Add newRequest function with common headers

profile.go

@@ -31,14 +31,11 @@ type Profile struct {
 func GetProfile(username string) (Profile, error) {
 	url := "https://twitter.com/" + username
 
-	req, err := http.NewRequest("GET", url, nil)
+	req, err := newRequest(url)
 	if err != nil {
 		return Profile{}, err
 	}
 
-	req.Header.Set("Accept-Language", "en-US")
-	req.Header.Set("X-Requested-With", "XMLHttpRequest")
-
 	resp, err := http.DefaultClient.Do(req)
 	if resp == nil {
 		return Profile{}, err

trends.go

@@ -1,8 +1,6 @@
 package twitterscraper
 
 import (
-	"net/http"
-
 	"github.com/PuerkitoBio/goquery"
 )
 
@@ -10,7 +8,7 @@ const trendsURL = "https://twitter.com/i/trends"
 
 // GetTrends return list of trends.
 func GetTrends() ([]string, error) {
-	req, err := http.NewRequest("GET", trendsURL, nil)
+	req, err := newRequest(trendsURL)
 	if err != nil {
 		return nil, err
 	}

tweets.go

@@ -3,7 +3,6 @@ package twitterscraper
 import (
 	"context"
 	"fmt"
-	"net/http"
 	"regexp"
 	"strconv"
 	"strings"
@@ -84,17 +83,12 @@ func GetTweets(ctx context.Context, user string, pages int) <-chan *Result {
 // FetchTweets gets tweets for a given user, via the Twitter frontend API.
 func FetchTweets(user string, last string) ([]*Tweet, error) {
 
-	req, err := http.NewRequest("GET", fmt.Sprintf(ajaxURL, user), nil)
+	req, err := newRequest(fmt.Sprintf(ajaxURL, user))
 	if err != nil {
 		return nil, err
 	}
 
 	req.Header.Set("Referer", "https://twitter.com/"+user)
-	req.Header.Set("Accept", "application/json, text/javascript, */*; q=0.01")
-	req.Header.Set("User-Agent", "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_12_6) AppleWebKit/603.3.8 (KHTML, like Gecko) Version/10.1.2 Safari/603.3.8")
-	req.Header.Set("Accept-Language", "en-US")
-	req.Header.Set("X-Twitter-Active-User", "yes")
-	req.Header.Set("X-Requested-With", "XMLHttpRequest")
 
 	q := req.URL.Query()
 	q.Add("include_available_features", "1")

util.go

@@ -7,6 +7,21 @@ import (
 	"strings"
 )
 
+func newRequest(url string) (*http.Request, error) {
+	req, err := http.NewRequest("GET", url, nil)
+	if err != nil {
+		return nil, err
+	}
+
+	req.Header.Set("Accept", "application/json, text/javascript, */*; q=0.01")
+	req.Header.Set("Accept-Language", "en-US")
+	req.Header.Set("User-Agent", "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_12_6) AppleWebKit/603.3.8 (KHTML, like Gecko) Version/10.1.2 Safari/603.3.8")
+	req.Header.Set("X-Twitter-Active-User", "yes")
+	req.Header.Set("X-Requested-With", "XMLHttpRequest")
+
+	return req, nil
+}
+
 func getHTMLFromJSON(req *http.Request, field string) (*strings.Reader, error) {
 	resp, err := http.DefaultClient.Do(req)
 	if err != nil {