diff --git a/README.md b/README.md index 0fa990b..b4b0b9b 100644 --- a/README.md +++ b/README.md @@ -176,6 +176,14 @@ if err != nil { } ``` +### Delay requests + +Add delay between API requests (in seconds) + +```golang +scraper.WithDelay(5) +``` + ### Load timeline with tweet replies ```golang diff --git a/api.go b/api.go index 13ae736..590dd64 100644 --- a/api.go +++ b/api.go @@ -12,6 +12,17 @@ const bearerToken string = "AAAAAAAAAAAAAAAAAAAAANRILgAAAAAAnNwIzUejRCOuH5E6I8xn // RequestAPI get JSON from frontend API and decodes it func (s *Scraper) RequestAPI(req *http.Request, target interface{}) error { + s.wg.Wait() + if s.delay > 0 { + defer func() { + s.wg.Add(1) + go func() { + time.Sleep(time.Second * time.Duration(s.delay)) + s.wg.Done() + }() + }() + } + if s.guestToken == "" || s.guestCreatedAt.Before(time.Now().Add(-time.Hour*3)) { err := s.GetGuestToken() if err != nil { diff --git a/scraper.go b/scraper.go index 98e1743..5a7c8b3 100644 --- a/scraper.go +++ b/scraper.go @@ -7,16 +7,19 @@ import ( "net/http" "net/url" "strings" + "sync" "time" ) // Scraper object type Scraper struct { client *http.Client + delay int64 guestToken string guestCreatedAt time.Time includeReplies bool searchMode SearchMode + wg sync.WaitGroup } // SearchMode type @@ -55,6 +58,17 @@ func SetSearchMode(mode SearchMode) *Scraper { return defaultScraper.SetSearchMode(mode) } +// WithDelay add delay between API requests (in seconds) +func (s *Scraper) WithDelay(seconds int64) *Scraper { + s.delay = seconds + return s +} + +// WithDelay wrapper for default Scraper +func WithDelay(seconds int64) *Scraper { + return defaultScraper.WithDelay(seconds) +} + // WithReplies enable/disable load timeline with tweet replies func (s *Scraper) WithReplies(b bool) *Scraper { s.includeReplies = b