diff --git a/search.go b/search.go index 0eeb828..549cab8 100644 --- a/search.go +++ b/search.go @@ -41,7 +41,9 @@ func (s *Scraper) getSearchTimeline(query string, maxNbr int, cursor string) (*t q.Add("count", strconv.Itoa(maxNbr)) q.Add("query_source", "typed_query") q.Add("pc", "1") + q.Add("requestContext", "launch") q.Add("spelling_corrections", "1") + q.Add("include_ext_edit_control", "true") if cursor != "" { q.Add("cursor", cursor) } diff --git a/util.go b/util.go index f1b062f..5de9c89 100644 --- a/util.go +++ b/util.go @@ -5,6 +5,7 @@ import ( "net/http" "regexp" "strconv" + "strings" "time" ) @@ -30,22 +31,28 @@ func (s *Scraper) newRequest(method string, url string) (*http.Request, error) { q.Add("include_can_dm", "1") q.Add("include_can_media_tag", "1") q.Add("include_ext_has_nft_avatar", "1") + q.Add("include_ext_is_blue_verified", "1") + q.Add("include_ext_verified_type", "1") q.Add("skip_status", "1") q.Add("cards_platform", "Web-12") q.Add("include_cards", "1") q.Add("include_ext_alt_text", "true") + q.Add("include_ext_limited_action_results", "false") q.Add("include_quote_count", "true") q.Add("include_reply_count", "1") q.Add("tweet_mode", "extended") + q.Add("include_ext_collab_control", "true") + q.Add("include_ext_views", "true") q.Add("include_entities", "true") q.Add("include_user_entities", "true") q.Add("include_ext_media_color", "true") q.Add("include_ext_media_availability", "true") q.Add("include_ext_sensitive_media_warning", "true") + q.Add("include_ext_trusted_friends_metadata", "true") q.Add("send_error_codes", "true") q.Add("simple_quoted_tweet", "true") q.Add("include_tweet_replies", strconv.FormatBool(s.includeReplies)) - q.Add("ext", "mediaStats,highlightedLabel,hasNftAvatar,voiceInfo,superFollowMetadata") + q.Add("ext", "mediaStats,highlightedLabel,hasNftAvatar,voiceInfo,birdwatchPivot,enrichments,superFollowMetadata,unmentionInfo,editControl,collab_control,vibe") req.URL.RawQuery = q.Encode() return req, nil @@ -75,6 +82,10 @@ func getUserTimeline(ctx context.Context, query string, maxProfilesNbr int, fetc break } + if strings.HasPrefix(next, "scroll:") { + continue + } + for _, profile := range profiles { select { case <-ctx.Done(): @@ -120,6 +131,10 @@ func getTweetTimeline(ctx context.Context, query string, maxTweetsNbr int, fetch break } + if strings.HasPrefix(next, "scroll:") { + continue + } + for _, tweet := range tweets { select { case <-ctx.Done():