diff --git a/api.go b/api.go index fb6dfcf..38360dc 100644 --- a/api.go +++ b/api.go @@ -14,36 +14,11 @@ const bearerToken string = "AAAAAAAAAAAAAAAAAAAAAPYXBAAAAAAACLXUNDekMxqa8h%2F40K func (s *Scraper) RequestAPI(req *http.Request, target interface{}) error { s.wg.Wait() if s.delay > 0 { - defer func() { - s.wg.Add(1) - go func() { - time.Sleep(time.Second * time.Duration(s.delay)) - s.wg.Done() - }() - }() + defer s.delayRequest() } - if !s.isLogged { - if !s.IsGuestToken() || s.guestCreatedAt.Before(time.Now().Add(-time.Hour*3)) { - err := s.GetGuestToken() - if err != nil { - return err - } - } - req.Header.Set("X-Guest-Token", s.guestToken) - } - - if s.oAuthToken != "" && s.oAuthSecret != "" { - req.Header.Set("Authorization", s.sign(req.Method, req.URL)) - } else { - req.Header.Set("Authorization", "Bearer "+s.bearerToken) - } - - for _, cookie := range s.client.Jar.Cookies(req.URL) { - if cookie.Name == "ct0" { - req.Header.Set("X-CSRF-Token", cookie.Value) - break - } + if err := s.prepareRequest(req); err != nil { + return err } resp, err := s.client.Do(req) @@ -52,13 +27,66 @@ func (s *Scraper) RequestAPI(req *http.Request, target interface{}) error { } defer resp.Body.Close() + return s.handleResponse(resp, target) +} + +func (s *Scraper) delayRequest() { + s.wg.Add(1) + go func() { + time.Sleep(time.Second * time.Duration(s.delay)) + s.wg.Done() + }() +} + +func (s *Scraper) prepareRequest(req *http.Request) error { + req.Header.Set("User-Agent", s.userAgent) + + if !s.isLogged { + if err := s.setGuestToken(req); err != nil { + return err + } + } + + s.setAuthorizationHeader(req) + s.setCSRFToken(req) + + return nil +} + +func (s *Scraper) setGuestToken(req *http.Request) error { + if !s.IsGuestToken() || s.guestCreatedAt.Before(time.Now().Add(-time.Hour*3)) { + if err := s.GetGuestToken(); err != nil { + return err + } + } + req.Header.Set("X-Guest-Token", s.guestToken) + return nil +} + +func (s *Scraper) setAuthorizationHeader(req *http.Request) { + if s.oAuthToken != "" && s.oAuthSecret != "" { + req.Header.Set("Authorization", s.sign(req.Method, req.URL)) + } else { + req.Header.Set("Authorization", "Bearer "+s.bearerToken) + } +} + +func (s *Scraper) setCSRFToken(req *http.Request) { + for _, cookie := range s.client.Jar.Cookies(req.URL) { + if cookie.Name == "ct0" { + req.Header.Set("X-CSRF-Token", cookie.Value) + break + } + } +} + +func (s *Scraper) handleResponse(resp *http.Response, target interface{}) error { content, err := io.ReadAll(resp.Body) if err != nil { return err } - statusOK := resp.StatusCode >= 200 && resp.StatusCode < 300 - if !statusOK { + if resp.StatusCode != http.StatusOK { return fmt.Errorf("response status %s: %s", resp.Status, content) } @@ -69,6 +97,7 @@ func (s *Scraper) RequestAPI(req *http.Request, target interface{}) error { if target == nil { return nil } + return json.Unmarshal(content, target) } diff --git a/auth.go b/auth.go index a7a7d8b..7478349 100644 --- a/auth.go +++ b/auth.go @@ -84,7 +84,7 @@ func (s *Scraper) getFlow(data map[string]interface{}) (*flow, error) { headers := http.Header{ "Authorization": []string{"Bearer " + s.bearerToken}, "Content-Type": []string{"application/json"}, - "User-Agent": []string{"TwitterAndroid/99"}, + "User-Agent": []string{s.userAgent}, "X-Guest-Token": []string{s.guestToken}, "X-Twitter-Auth-Type": []string{"OAuth2Client"}, "X-Twitter-Active-User": []string{"yes"}, diff --git a/scraper.go b/scraper.go index 98326c6..b60e921 100644 --- a/scraper.go +++ b/scraper.go @@ -27,6 +27,7 @@ type Scraper struct { oAuthToken string oAuthSecret string proxy string + userAgent string searchMode SearchMode wg sync.WaitGroup } @@ -49,12 +50,14 @@ const ( // default http client timeout const DefaultClientTimeout = 10 * time.Second +const DefaultUserAgent = "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/129.0.0.0 Safari/537.36" // New creates a Scraper object func New() *Scraper { jar, _ := cookiejar.New(nil) return &Scraper{ bearerToken: bearerToken, + userAgent: DefaultUserAgent, client: &http.Client{ Jar: jar, Timeout: DefaultClientTimeout, @@ -169,3 +172,11 @@ func (s *Scraper) SetProxy(proxyAddr string) error { } return errors.New("only support http(s) or socks5 protocol") } + +func (s *Scraper) SetUserAgent(userAgent string) { + s.userAgent = userAgent +} + +func (s *Scraper) GetUserAgent() string { + return s.userAgent +}