lain/twitter-scrapper
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity

Add newRequest function with common headers

Browse source
This commit is contained in:
Alexander Sheiko 2020-06-15 11:59:30 +03:00
parent 2dc80b95cf
commit c12927172f
4 changed files with 18 additions and 14 deletions
Download patch file Download diff file Expand all files Collapse all files

15
util.go
View file

@ -7,6 +7,21 @@ import (
"strings"
)
func newRequest(url string) (*http.Request, error) {
req, err := http.NewRequest("GET", url, nil)
if err != nil {
return nil, err
}
req.Header.Set("Accept", "application/json, text/javascript, */*; q=0.01")
req.Header.Set("Accept-Language", "en-US")
req.Header.Set("User-Agent", "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_12_6) AppleWebKit/603.3.8 (KHTML, like Gecko) Version/10.1.2 Safari/603.3.8")
req.Header.Set("X-Twitter-Active-User", "yes")
req.Header.Set("X-Requested-With", "XMLHttpRequest")
return req, nil
}
func getHTMLFromJSON(req *http.Request, field string) (*strings.Reader, error) {
resp, err := http.DefaultClient.Do(req)
if err != nil {