lain/twitter-scrapper
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity

Fix search

Browse source
This commit is contained in:
Alexander Sheiko 2023-02-16 11:15:13 +02:00
parent 96325b0173
commit 7d5b51da84
2 changed files with 18 additions and 1 deletions
Download patch file Download diff file Expand all files Collapse all files

2
search.go
View file

@ -41,7 +41,9 @@ func (s *Scraper) getSearchTimeline(query string, maxNbr int, cursor string) (*t
q.Add("count", strconv.Itoa(maxNbr))
q.Add("query_source", "typed_query")
q.Add("pc", "1")
q.Add("requestContext", "launch")
q.Add("spelling_corrections", "1")
q.Add("include_ext_edit_control", "true")
if cursor != "" {
q.Add("cursor", cursor)
}

17
util.go
View file

@ -5,6 +5,7 @@ import (
"net/http"
"regexp"
"strconv"
"strings"
"time"
)
@ -30,22 +31,28 @@ func (s *Scraper) newRequest(method string, url string) (*http.Request, error) {
q.Add("include_can_dm", "1")
q.Add("include_can_media_tag", "1")
q.Add("include_ext_has_nft_avatar", "1")
q.Add("include_ext_is_blue_verified", "1")
q.Add("include_ext_verified_type", "1")
q.Add("skip_status", "1")
q.Add("cards_platform", "Web-12")
q.Add("include_cards", "1")
q.Add("include_ext_alt_text", "true")
q.Add("include_ext_limited_action_results", "false")
q.Add("include_quote_count", "true")
q.Add("include_reply_count", "1")
q.Add("tweet_mode", "extended")
q.Add("include_ext_collab_control", "true")
q.Add("include_ext_views", "true")
q.Add("include_entities", "true")
q.Add("include_user_entities", "true")
q.Add("include_ext_media_color", "true")
q.Add("include_ext_media_availability", "true")
q.Add("include_ext_sensitive_media_warning", "true")
q.Add("include_ext_trusted_friends_metadata", "true")
q.Add("send_error_codes", "true")
q.Add("simple_quoted_tweet", "true")
q.Add("include_tweet_replies", strconv.FormatBool(s.includeReplies))
q.Add("ext", "mediaStats,highlightedLabel,hasNftAvatar,voiceInfo,superFollowMetadata")
q.Add("ext", "mediaStats,highlightedLabel,hasNftAvatar,voiceInfo,birdwatchPivot,enrichments,superFollowMetadata,unmentionInfo,editControl,collab_control,vibe")
req.URL.RawQuery = q.Encode()
return req, nil
@ -75,6 +82,10 @@ func getUserTimeline(ctx context.Context, query string, maxProfilesNbr int, fetc
break
}
if strings.HasPrefix(next, "scroll:") {
continue
}
for _, profile := range profiles {
select {
case <-ctx.Done():
@ -120,6 +131,10 @@ func getTweetTimeline(ctx context.Context, query string, maxTweetsNbr int, fetch
break
}
if strings.HasPrefix(next, "scroll:") {
continue
}
for _, tweet := range tweets {
select {
case <-ctx.Done():