From c967fdefdd9a2ee0a9fc5bdb2dbfeaedb98264a4 Mon Sep 17 00:00:00 2001 From: Joaco Esteban Date: Sat, 3 Feb 2024 14:53:11 +0100 Subject: [PATCH] FEAT, list following --- follow.go | 64 ++++++++++++++++++++++++++++++++++++++++++++++++++ search.go | 2 +- timeline_v2.go | 48 +++++++++++++++++++++++++++++++++---- types.go | 46 ++++++++++++++++++++++++++++++++++++ util.go | 35 ++++++++++++++++++++++++++- 5 files changed, 189 insertions(+), 6 deletions(-) create mode 100644 follow.go diff --git a/follow.go b/follow.go new file mode 100644 index 0000000..4a355cf --- /dev/null +++ b/follow.go @@ -0,0 +1,64 @@ +package twitterscraper + +import ( + "net/url" +) + +// FetchFollowingByUserID gets following profiles list for a given userID, via the Twitter frontend GraphQL API. +func (s *Scraper) FetchFollowingByUserID(userID string, maxFollowingNbr int, cursor string) ([]*Profile, string, error) { + if maxFollowingNbr > 200 { + maxFollowingNbr = 200 + } + + req, err := s.newRequest("GET", "https://twitter.com/i/api/graphql/oUxds-Fprv5NKsH67zPt5w/Following") + if err != nil { + return nil, "", err + } + + variables := map[string]interface{}{ + "userId": userID, + "count": maxFollowingNbr, + "includePromotedContent": false, + } + features := map[string]interface{}{ + "responsive_web_graphql_exclude_directive_enabled": true, + "verified_phone_label_enabled": false, + "creator_subscriptions_tweet_preview_api_enabled": true, + "responsive_web_graphql_timeline_navigation_enabled": true, + "responsive_web_graphql_skip_user_profile_image_extensions_enabled": false, + "c9s_tweet_anatomy_moderator_badge_enabled": true, + "tweetypie_unmention_optimization_enabled": true, + "responsive_web_edit_tweet_api_enabled": true, + "graphql_is_translatable_rweb_tweet_is_translatable_enabled": true, + "view_counts_everywhere_api_enabled": true, + "longform_notetweets_consumption_enabled": true, + "responsive_web_twitter_article_tweet_consumption_enabled": true, + "tweet_awards_web_tipping_enabled": false, + "freedom_of_speech_not_reach_fetch_enabled": true, + "standardized_nudges_misinfo": true, + "tweet_with_visibility_results_prefer_gql_limited_actions_policy_enabled": true, + "rweb_video_timestamps_enabled": true, + "longform_notetweets_rich_text_read_enabled": true, + "longform_notetweets_inline_media_enabled": true, + "responsive_web_media_download_video_enabled": false, + "responsive_web_enhance_cards_enabled": false, + } + + if cursor != "" { + variables["cursor"] = cursor + } + + query := url.Values{} + query.Set("variables", mapToJSONString(variables)) + query.Set("features", mapToJSONString(features)) + req.URL.RawQuery = query.Encode() + + var timeline timelineV2 + err = s.RequestAPI(req, &timeline) + if err != nil { + return nil, "", err + } + + users, nextCursor := timeline.parseUsers() + return users, nextCursor, nil +} diff --git a/search.go b/search.go index b4254f9..9f8f04f 100644 --- a/search.go +++ b/search.go @@ -62,7 +62,7 @@ func (timeline *searchTimeline) parseUsers() ([]*Profile, string) { } for _, entry := range instruction.Entries { if entry.Content.ItemContent.UserDisplayType == "User" { - if profile := parseProfile(entry.Content.ItemContent.UserResults.Result.Legacy); profile.Name != "" { + if profile := parseProfileV2(entry.Content.ItemContent.UserResults.Result); profile.Name != "" { if profile.UserID == "" { profile.UserID = entry.Content.ItemContent.UserResults.Result.RestID } diff --git a/timeline_v2.go b/timeline_v2.go index e33bb9b..cd4c164 100644 --- a/timeline_v2.go +++ b/timeline_v2.go @@ -47,6 +47,21 @@ func (result *result) parse() *Tweet { return tw } +type userResult struct { + Typename string `json:"__typename"` + ID string `json:"id"` + RestID string `json:"rest_id"` + AffiliatesHighlightedLabel struct{} `json:"affiliates_highlighted_label"` + HasGraduatedAccess bool `json:"has_graduated_access"` + IsBlueVerified bool `json:"is_blue_verified"` + ProfileImageShape string `json:"profile_image_shape"` + Legacy legacyUserV2 `json:"legacy"` +} + +func (result *userResult) parse() Profile { + return parseProfileV2(*result) +} + type entry struct { Content struct { CursorType string `json:"cursorType"` @@ -68,10 +83,7 @@ type entry struct { } `json:"tweet_results"` UserDisplayType string `json:"userDisplayType"` UserResults struct { - Result struct { - RestID string `json:"rest_id"` - Legacy legacyUser `json:"legacy"` - } `json:"result"` + Result userResult `json:"result"` } `json:"user_results"` } `json:"itemContent"` } `json:"content"` @@ -91,6 +103,16 @@ type timelineV2 struct { } `json:"instructions"` } `json:"timeline"` } `json:"timeline_v2"` + + Timeline struct { + Timeline struct { + Instructions []struct { + Entries []entry `json:"entries"` + Entry entry `json:"entry"` + Type string `json:"type"` + } `json:"instructions"` + } `json:"timeline"` + } `json:"timeline"` } `json:"result"` } `json:"user"` } `json:"data"` @@ -115,6 +137,24 @@ func (timeline *timelineV2) parseTweets() ([]*Tweet, string) { return tweets, cursor } +func (timeline *timelineV2) parseUsers() ([]*Profile, string) { + var cursor string + var users []*Profile + for _, instruction := range timeline.Data.User.Result.Timeline.Timeline.Instructions { + for _, entry := range instruction.Entries { + if entry.Content.CursorType == "Bottom" { + cursor = entry.Content.Value + continue + } + if entry.Content.ItemContent.UserResults.Result.Typename == "User" { + user := entry.Content.ItemContent.UserResults.Result.parse() + users = append(users, &user) + } + } + } + return users, cursor +} + type threadedConversation struct { Data struct { ThreadedConversationWithInjectionsV2 struct { diff --git a/types.go b/types.go index 344d80c..3e96331 100644 --- a/types.go +++ b/types.go @@ -170,6 +170,52 @@ type ( Verified bool `json:"verified"` } + legacyUserV2 struct { + Following bool `json:"following"` + CanDm bool `json:"can_dm"` + CanMediaTag bool `json:"can_media_tag"` + CreatedAt string `json:"created_at"` + DefaultProfile bool `json:"default_profile"` + DefaultProfileImage bool `json:"default_profile_image"` + Description string `json:"description"` + Entities struct { + Description struct { + Urls []interface{} `json:"urls"` + } `json:"description"` + URL struct { + Urls []struct { + DisplayURL string `json:"display_url"` + ExpandedURL string `json:"expanded_url"` + URL string `json:"url"` + Indices []int `json:"indices"` + } `json:"urls"` + } `json:"url"` + } `json:"entities"` + FastFollowersCount int `json:"fast_followers_count"` + FavouritesCount int `json:"favourites_count"` + FollowersCount int `json:"followers_count"` + FriendsCount int `json:"friends_count"` + HasCustomTimelines bool `json:"has_custom_timelines"` + IsTranslator bool `json:"is_translator"` + ListedCount int `json:"listed_count"` + Location string `json:"location"` + MediaCount int `json:"media_count"` + Name string `json:"name"` + NormalFollowersCount int `json:"normal_followers_count"` + PinnedTweetIdsStr []string `json:"pinned_tweet_ids_str"` + PossiblySensitive bool `json:"possibly_sensitive"` + ProfileBannerURL string `json:"profile_banner_url"` + ProfileImageURLHTTPS string `json:"profile_image_url_https"` + ProfileInterstitialType string `json:"profile_interstitial_type"` + ScreenName string `json:"screen_name"` + StatusesCount int `json:"statuses_count"` + TranslatorType string `json:"translator_type"` + URL string `json:"url"` + Verified bool `json:"verified"` + WantRetweets bool `json:"want_retweets"` + WithheldInCountries []interface{} `json:"withheld_in_countries"` + } + Place struct { ID string `json:"id"` PlaceType string `json:"place_type"` diff --git a/util.go b/util.go index 9bf6d55..7851ad7 100644 --- a/util.go +++ b/util.go @@ -342,7 +342,6 @@ func parseProfile(user legacyUser) Profile { FollowersCount: user.FollowersCount, FollowingCount: user.FavouritesCount, FriendsCount: user.FriendsCount, - IsPrivate: user.Protected, IsVerified: user.Verified, LikesCount: user.FavouritesCount, ListedCount: user.ListedCount, @@ -368,6 +367,40 @@ func parseProfile(user legacyUser) Profile { return profile } +func parseProfileV2(user userResult) Profile { + u := user.Legacy + profile := Profile{ + Avatar: u.ProfileImageURLHTTPS, + Banner: u.ProfileBannerURL, + Biography: u.Description, + FollowersCount: u.FollowersCount, + FollowingCount: u.FavouritesCount, + FriendsCount: u.FriendsCount, + IsVerified: u.Verified, + LikesCount: u.FavouritesCount, + ListedCount: u.ListedCount, + Location: u.Location, + Name: u.Name, + PinnedTweetIDs: u.PinnedTweetIdsStr, + TweetsCount: u.StatusesCount, + URL: "https://twitter.com/" + u.ScreenName, + UserID: user.ID, + Username: u.ScreenName, + } + + tm, err := time.Parse(time.RubyDate, u.CreatedAt) + if err == nil { + tm = tm.UTC() + profile.Joined = &tm + } + + if len(u.Entities.URL.Urls) > 0 { + profile.Website = u.Entities.URL.Urls[0].ExpandedURL + } + + return profile +} + func mapToJSONString(data map[string]interface{}) string { jsonBytes, err := json.Marshal(data) if err != nil {