diff --git a/profile.go b/profile.go index a70e77c..6d1af96 100644 --- a/profile.go +++ b/profile.go @@ -31,14 +31,11 @@ type Profile struct { func GetProfile(username string) (Profile, error) { url := "https://twitter.com/" + username - req, err := http.NewRequest("GET", url, nil) + req, err := newRequest(url) if err != nil { return Profile{}, err } - req.Header.Set("Accept-Language", "en-US") - req.Header.Set("X-Requested-With", "XMLHttpRequest") - resp, err := http.DefaultClient.Do(req) if resp == nil { return Profile{}, err diff --git a/trends.go b/trends.go index 8017019..e6a2db9 100644 --- a/trends.go +++ b/trends.go @@ -1,8 +1,6 @@ package twitterscraper import ( - "net/http" - "github.com/PuerkitoBio/goquery" ) @@ -10,7 +8,7 @@ const trendsURL = "https://twitter.com/i/trends" // GetTrends return list of trends. func GetTrends() ([]string, error) { - req, err := http.NewRequest("GET", trendsURL, nil) + req, err := newRequest(trendsURL) if err != nil { return nil, err } diff --git a/tweets.go b/tweets.go index 2886e5b..1447279 100644 --- a/tweets.go +++ b/tweets.go @@ -3,7 +3,6 @@ package twitterscraper import ( "context" "fmt" - "net/http" "regexp" "strconv" "strings" @@ -84,17 +83,12 @@ func GetTweets(ctx context.Context, user string, pages int) <-chan *Result { // FetchTweets gets tweets for a given user, via the Twitter frontend API. func FetchTweets(user string, last string) ([]*Tweet, error) { - req, err := http.NewRequest("GET", fmt.Sprintf(ajaxURL, user), nil) + req, err := newRequest(fmt.Sprintf(ajaxURL, user)) if err != nil { return nil, err } req.Header.Set("Referer", "https://twitter.com/"+user) - req.Header.Set("Accept", "application/json, text/javascript, */*; q=0.01") - req.Header.Set("User-Agent", "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_12_6) AppleWebKit/603.3.8 (KHTML, like Gecko) Version/10.1.2 Safari/603.3.8") - req.Header.Set("Accept-Language", "en-US") - req.Header.Set("X-Twitter-Active-User", "yes") - req.Header.Set("X-Requested-With", "XMLHttpRequest") q := req.URL.Query() q.Add("include_available_features", "1") diff --git a/util.go b/util.go index 1d8bd26..0678cc2 100644 --- a/util.go +++ b/util.go @@ -7,6 +7,21 @@ import ( "strings" ) +func newRequest(url string) (*http.Request, error) { + req, err := http.NewRequest("GET", url, nil) + if err != nil { + return nil, err + } + + req.Header.Set("Accept", "application/json, text/javascript, */*; q=0.01") + req.Header.Set("Accept-Language", "en-US") + req.Header.Set("User-Agent", "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_12_6) AppleWebKit/603.3.8 (KHTML, like Gecko) Version/10.1.2 Safari/603.3.8") + req.Header.Set("X-Twitter-Active-User", "yes") + req.Header.Set("X-Requested-With", "XMLHttpRequest") + + return req, nil +} + func getHTMLFromJSON(req *http.Request, field string) (*strings.Reader, error) { resp, err := http.DefaultClient.Do(req) if err != nil {