Fix search cursor

search.go

@@ -19,8 +19,8 @@ func SearchTweets(ctx context.Context, query string, maxTweetsNbr int) <-chan *R
 // FetchSearchTweets gets tweets for a given search query, via the Twitter frontend API
 func (s *Scraper) FetchSearchTweets(query string, maxTweetsNbr int, cursor string) ([]*Tweet, string, error) {
 	query = url.PathEscape(query)
-	if maxTweetsNbr > 200 {
+	if maxTweetsNbr > 100 {
-		maxTweetsNbr = 200
+		maxTweetsNbr = 100
 	}
 
 	req, err := s.newRequest("GET", "https://twitter.com/i/api/2/search/adaptive.json")

search_test.go

@@ -5,10 +5,28 @@ import (
 	"testing"
 )
 
+func TestFetchSearchCursor(t *testing.T) {
+	scraper := New()
+	maxTweetsNbr := 150
+	tweetsNbr := 0
+	nextCursor := ""
+	for tweetsNbr < maxTweetsNbr {
+		tweets, cursor, err := scraper.FetchSearchTweets("twitter", maxTweetsNbr, nextCursor)
+		if err != nil {
+			t.Fatal(err)
+		}
+		if cursor == "" {
+			t.Fatal("Expected search cursor is not empty")
+		}
+		tweetsNbr += len(tweets)
+		nextCursor = cursor
+	}
+}
+
 func TestGetSearchTweets(t *testing.T) {
 	count := 0
 	maxTweetsNbr := 250
-	for tweet := range SearchTweets(context.Background(), "twitter scraper data -filter:retweets", maxTweetsNbr) {
+	for tweet := range SearchTweets(context.Background(), "twitter -filter:retweets", maxTweetsNbr) {
 		if tweet.Error != nil {
 			t.Error(tweet.Error)
 		} else {

types.go

@@ -169,6 +169,18 @@ type (
 						} `json:"content"`
 					} `json:"entry"`
 				} `json:"pinEntry,omitempty"`
+				ReplaceEntry struct {
+					Entry struct {
+						Content struct {
+							Operation struct {
+								Cursor struct {
+									Value      string `json:"value"`
+									CursorType string `json:"cursorType"`
+								} `json:"cursor"`
+							} `json:"operation"`
+						} `json:"content"`
+					} `json:"entry"`
+				} `json:"replaceEntry,omitempty"`
 			} `json:"instructions"`
 		} `json:"timeline"`
 	}

util.go

@@ -225,6 +225,9 @@ func parseTimeline(timeline *timeline) ([]*Tweet, string) {
 				cursor = entry.Content.Operation.Cursor.Value
 			}
 		}
+		if instruction.ReplaceEntry.Entry.Content.Operation.Cursor.CursorType == "Bottom" {
+			cursor = instruction.ReplaceEntry.Entry.Content.Operation.Cursor.Value
+		}
 	}
 	if pinnedTweet != nil && len(orderedTweets) > 0 {
 		orderedTweets = append([]*Tweet{pinnedTweet}, orderedTweets...)