diff --git a/search.go b/search.go index 1fb625f..0df959a 100644 --- a/search.go +++ b/search.go @@ -19,8 +19,8 @@ func SearchTweets(ctx context.Context, query string, maxTweetsNbr int) <-chan *R // FetchSearchTweets gets tweets for a given search query, via the Twitter frontend API func (s *Scraper) FetchSearchTweets(query string, maxTweetsNbr int, cursor string) ([]*Tweet, string, error) { query = url.PathEscape(query) - if maxTweetsNbr > 200 { - maxTweetsNbr = 200 + if maxTweetsNbr > 100 { + maxTweetsNbr = 100 } req, err := s.newRequest("GET", "https://twitter.com/i/api/2/search/adaptive.json") diff --git a/search_test.go b/search_test.go index 4149b2d..d572f96 100644 --- a/search_test.go +++ b/search_test.go @@ -5,10 +5,28 @@ import ( "testing" ) +func TestFetchSearchCursor(t *testing.T) { + scraper := New() + maxTweetsNbr := 150 + tweetsNbr := 0 + nextCursor := "" + for tweetsNbr < maxTweetsNbr { + tweets, cursor, err := scraper.FetchSearchTweets("twitter", maxTweetsNbr, nextCursor) + if err != nil { + t.Fatal(err) + } + if cursor == "" { + t.Fatal("Expected search cursor is not empty") + } + tweetsNbr += len(tweets) + nextCursor = cursor + } +} + func TestGetSearchTweets(t *testing.T) { count := 0 maxTweetsNbr := 250 - for tweet := range SearchTweets(context.Background(), "twitter scraper data -filter:retweets", maxTweetsNbr) { + for tweet := range SearchTweets(context.Background(), "twitter -filter:retweets", maxTweetsNbr) { if tweet.Error != nil { t.Error(tweet.Error) } else { diff --git a/types.go b/types.go index 73795ef..1520f87 100644 --- a/types.go +++ b/types.go @@ -169,6 +169,18 @@ type ( } `json:"content"` } `json:"entry"` } `json:"pinEntry,omitempty"` + ReplaceEntry struct { + Entry struct { + Content struct { + Operation struct { + Cursor struct { + Value string `json:"value"` + CursorType string `json:"cursorType"` + } `json:"cursor"` + } `json:"operation"` + } `json:"content"` + } `json:"entry"` + } `json:"replaceEntry,omitempty"` } `json:"instructions"` } `json:"timeline"` } diff --git a/util.go b/util.go index 69486b3..5187dea 100644 --- a/util.go +++ b/util.go @@ -225,6 +225,9 @@ func parseTimeline(timeline *timeline) ([]*Tweet, string) { cursor = entry.Content.Operation.Cursor.Value } } + if instruction.ReplaceEntry.Entry.Content.Operation.Cursor.CursorType == "Bottom" { + cursor = instruction.ReplaceEntry.Entry.Content.Operation.Cursor.Value + } } if pinnedTweet != nil && len(orderedTweets) > 0 { orderedTweets = append([]*Tweet{pinnedTweet}, orderedTweets...)