package twitterscraper import ( "encoding/json" "fmt" "io/ioutil" "net/http" "time" ) const bearerToken string = "AAAAAAAAAAAAAAAAAAAAAPYXBAAAAAAACLXUNDekMxqa8h%2F40K4moUkGsoc%3DTYfbDKbT3jJPCEVnMYqilB28NHfOPqkca3qaAxGfsyKCs0wRbw" // RequestAPI get JSON from frontend API and decodes it func (s *Scraper) RequestAPI(req *http.Request, target interface{}) error { s.wg.Wait() if s.delay > 0 { defer func() { s.wg.Add(1) go func() { time.Sleep(time.Second * time.Duration(s.delay)) s.wg.Done() }() }() } if !s.IsGuestToken() || s.guestCreatedAt.Before(time.Now().Add(-time.Hour*3)) { err := s.GetGuestToken() if err != nil { return err } } req.Header.Set("Authorization", "Bearer "+s.bearerToken) req.Header.Set("X-Guest-Token", s.guestToken) for _, cookie := range s.client.Jar.Cookies(req.URL) { if cookie.Name == "ct0" { req.Header.Set("X-CSRF-Token", cookie.Value) break } } resp, err := s.client.Do(req) if err != nil { return err } defer resp.Body.Close() // private profiles return forbidden, but also data if resp.StatusCode != http.StatusOK && resp.StatusCode != http.StatusForbidden { content, _ := ioutil.ReadAll(resp.Body) return fmt.Errorf("response status %s: %s", resp.Status, content) } if resp.Header.Get("X-Rate-Limit-Remaining") == "0" { s.guestToken = "" } b, err := ioutil.ReadAll(resp.Body) if err != nil { return err } // fmt.Println(string(b)) return json.Unmarshal(b, target) } // GetGuestToken from Twitter API func (s *Scraper) GetGuestToken() error { req, err := http.NewRequest("POST", "https://api.twitter.com/1.1/guest/activate.json", nil) if err != nil { return err } req.Header.Set("Authorization", "Bearer "+s.bearerToken) resp, err := s.client.Do(req) if err != nil { return err } defer resp.Body.Close() body, err := ioutil.ReadAll(resp.Body) if err != nil { return err } if resp.StatusCode != http.StatusOK { return fmt.Errorf("response status %s: %s", resp.Status, body) } var jsn map[string]interface{} if err := json.Unmarshal(body, &jsn); err != nil { return err } var ok bool if s.guestToken, ok = jsn["guest_token"].(string); !ok { return fmt.Errorf("guest_token not found") } s.guestCreatedAt = time.Now() return nil }