From 1c49fea3ef2aa872f7292bc0d24282253dd87dd4 Mon Sep 17 00:00:00 2001 From: xisco Date: Thu, 14 May 2020 18:00:43 +0200 Subject: [PATCH 1/2] change goquery.NewDocument deprecated method and add language header --- profile.go | 19 ++++++++++++++++++- 1 file changed, 18 insertions(+), 1 deletion(-) diff --git a/profile.go b/profile.go index fce3711..4fc29dc 100644 --- a/profile.go +++ b/profile.go @@ -2,6 +2,7 @@ package twitterscraper import ( "fmt" + "net/http" "strconv" "strings" "time" @@ -29,7 +30,23 @@ type Profile struct { // GetProfile return parsed user profile func GetProfile(username string) (Profile, error) { url := "https://twitter.com/" + username - doc, err := goquery.NewDocument(url) + + req, err := http.NewRequest("GET", url, nil) + + if err != nil { + return Profile{}, err + } + + req.Header.Set("Accept-Language", "en-US") + + res, err := http.DefaultClient.Do(req) + + if res == nil { + return Profile{}, err + } + + doc, err := goquery.NewDocumentFromReader(res.Body) + if err != nil { return Profile{}, err } From b864ad2b4e8403297b69c997173682ca05b6a790 Mon Sep 17 00:00:00 2001 From: xisco Date: Thu, 14 May 2020 18:01:30 +0200 Subject: [PATCH 2/2] change TestGetProfile profile and data --- profile_test.go | 21 +++++++++++---------- 1 file changed, 11 insertions(+), 10 deletions(-) diff --git a/profile_test.go b/profile_test.go index 537cbff..8cb1d10 100644 --- a/profile_test.go +++ b/profile_test.go @@ -9,20 +9,21 @@ import ( ) func TestGetProfile(t *testing.T) { - joined := time.Unix(1245860880, 0) + loc := time.FixedZone("UTC", 0) + joined := time.Date(2007,02, 20,6,35,0,0, loc) sample := Profile{ - Avatar: "https://pbs.twimg.com/profile_images/1176439369596624896/Fkqe6qVj_400x400.jpg", - Biography: "Kenneth Reitz, n: a garden–variety self–referential loop. Not available for purchase. Made in USA.", - Birthday: "1988", + Avatar: "https://pbs.twimg.com/profile_images/1111729635610382336/_65QFl7B_400x400.png", + Biography: "What’s happening?!", + Birthday: "March 21", Joined: &joined, - Location: "Eden, Earth, Milky Way", - Name: "☿ Kenneth Reitz", - URL: "https://twitter.com/kennethreitz", - Username: "kennethreitz", - Website: "https://kennethreitz.org/values", + Location: "Everywhere", + Name: "Twitter", + URL: "https://twitter.com/Twitter", + Username: "Twitter", + Website: "https://about.twitter.com/", } - profile, err := GetProfile("kennethreitz") + profile, err := GetProfile("Twitter") if err != nil { t.Error(err) }