diff --git a/README.md b/README.md index 5d4f3b9..718bc81 100644 --- a/README.md +++ b/README.md @@ -29,6 +29,8 @@ You can use this library to get tweets, profiles, and trends trivially. - [Get profile](#get-profile) - [Search profile](#search-profile) - [Get trends](#get-trends) + - [Get following](#get-following) + - [Get followers](#get-followers) - [Connection](#connection) - [Proxy](#proxy) - [HTTP(s)](#https) @@ -332,6 +334,30 @@ profiles, cursor, err := scraper.FetchSearchProfiles("taylorswift13", 20, cursor trends, err := scraper.GetTrends() ``` +### Get following + +> [!IMPORTANT] +> Requires authentication! + +500 requests / 15 minutes + +```golang +var cursor string +users, cursor, err := testScraper.FetchFollowing("Support", 20, cursor) +``` + +### Get followers + +> [!IMPORTANT] +> Requires authentication! + +50 requests / 15 minutes + +```golang +var cursor string +users, cursor, err := testScraper.FetchFollowers("Support", 20, cursor) +``` + ## Connection ### Proxy diff --git a/follows.go b/follows.go new file mode 100644 index 0000000..b849afa --- /dev/null +++ b/follows.go @@ -0,0 +1,152 @@ +package twitterscraper + +import ( + "net/url" + "strings" +) + +// FetchFollowing gets following profiles list for a given user, via the Twitter frontend GraphQL API. +func (s *Scraper) FetchFollowing(user string, maxUsersNbr int, cursor string) ([]*Profile, string, error) { + userID, err := s.GetUserIDByScreenName(user) + if err != nil { + return nil, "", err + } + + return s.FetchFollowingByUserID(userID, maxUsersNbr, cursor) +} + +// FetchFollowingByUserID gets following profiles list for a given userID, via the Twitter frontend GraphQL API. +func (s *Scraper) FetchFollowingByUserID(userID string, maxUsersNbr int, cursor string) ([]*Profile, string, error) { + if maxUsersNbr > 200 { + maxUsersNbr = 200 + } + + req, err := s.newRequest("GET", "https://twitter.com/i/api/graphql/g5P4cbXR4ta4oCeE7y2vLQ/Following") + if err != nil { + return nil, "", err + } + + variables := map[string]interface{}{ + "userId": userID, + "includePromotedContent": false, + "count": maxUsersNbr, + } + features := map[string]interface{}{ + "responsive_web_graphql_exclude_directive_enabled": true, + "verified_phone_label_enabled": false, + "creator_subscriptions_tweet_preview_api_enabled": true, + "responsive_web_graphql_timeline_navigation_enabled": true, + "responsive_web_graphql_skip_user_profile_image_extensions_enabled": false, + "c9s_tweet_anatomy_moderator_badge_enabled": true, + "tweetypie_unmention_optimization_enabled": true, + "responsive_web_edit_tweet_api_enabled": true, + "graphql_is_translatable_rweb_tweet_is_translatable_enabled": true, + "view_counts_everywhere_api_enabled": true, + "longform_notetweets_consumption_enabled": true, + "responsive_web_twitter_article_tweet_consumption_enabled": true, + "tweet_awards_web_tipping_enabled": false, + "freedom_of_speech_not_reach_fetch_enabled": true, + "standardized_nudges_misinfo": true, + "tweet_with_visibility_results_prefer_gql_limited_actions_policy_enabled": true, + "rweb_video_timestamps_enabled": true, + "longform_notetweets_rich_text_read_enabled": true, + "longform_notetweets_inline_media_enabled": true, + "responsive_web_enhance_cards_enabled": false, + } + + if cursor != "" { + variables["cursor"] = cursor + } + + query := url.Values{} + query.Set("variables", mapToJSONString(variables)) + query.Set("features", mapToJSONString(features)) + req.URL.RawQuery = query.Encode() + + var timeline timelineV2 + err = s.RequestAPI(req, &timeline) + if err != nil { + return nil, "", err + } + + users, nextCursor := timeline.parseUsers() + + if strings.HasPrefix(nextCursor, "0|") { + nextCursor = "" + } + + return users, nextCursor, nil +} + +// FetchFollowers gets following profiles list for a given user, via the Twitter frontend GraphQL API. +func (s *Scraper) FetchFollowers(user string, maxUsersNbr int, cursor string) ([]*Profile, string, error) { + userID, err := s.GetUserIDByScreenName(user) + if err != nil { + return nil, "", err + } + + return s.FetchFollowersByUserID(userID, maxUsersNbr, cursor) +} + +// FetchFollowersByUserID gets followers profiles list for a given userID, via the Twitter frontend GraphQL API. +func (s *Scraper) FetchFollowersByUserID(userID string, maxUsersNbr int, cursor string) ([]*Profile, string, error) { + if maxUsersNbr > 200 { + maxUsersNbr = 200 + } + + req, err := s.newRequest("GET", "https://twitter.com/i/api/graphql/jwbfbSzn0FRL_AMZGsYDag/Followers") + if err != nil { + return nil, "", err + } + + variables := map[string]interface{}{ + "userId": userID, + "includePromotedContent": false, + "count": maxUsersNbr, + } + features := map[string]interface{}{ + "responsive_web_graphql_exclude_directive_enabled": true, + "verified_phone_label_enabled": false, + "creator_subscriptions_tweet_preview_api_enabled": true, + "responsive_web_graphql_timeline_navigation_enabled": true, + "responsive_web_graphql_skip_user_profile_image_extensions_enabled": false, + "c9s_tweet_anatomy_moderator_badge_enabled": true, + "tweetypie_unmention_optimization_enabled": true, + "responsive_web_edit_tweet_api_enabled": true, + "graphql_is_translatable_rweb_tweet_is_translatable_enabled": true, + "view_counts_everywhere_api_enabled": true, + "longform_notetweets_consumption_enabled": true, + "responsive_web_twitter_article_tweet_consumption_enabled": true, + "tweet_awards_web_tipping_enabled": false, + "freedom_of_speech_not_reach_fetch_enabled": true, + "standardized_nudges_misinfo": true, + "tweet_with_visibility_results_prefer_gql_limited_actions_policy_enabled": true, + "rweb_video_timestamps_enabled": true, + "longform_notetweets_rich_text_read_enabled": true, + "longform_notetweets_inline_media_enabled": true, + "responsive_web_enhance_cards_enabled": false, + } + + if cursor != "" { + variables["cursor"] = cursor + } + + query := url.Values{} + query.Set("variables", mapToJSONString(variables)) + query.Set("features", mapToJSONString(features)) + req.URL.RawQuery = query.Encode() + + var timeline timelineV2 + err = s.RequestAPI(req, &timeline) + if err != nil { + return nil, "", err + } + + users, nextCursor := timeline.parseUsers() + + if strings.HasPrefix(nextCursor, "0|") { + nextCursor = "" + } + + return users, nextCursor, nil +} diff --git a/follows_test.go b/follows_test.go new file mode 100644 index 0000000..3aa9227 --- /dev/null +++ b/follows_test.go @@ -0,0 +1,25 @@ +package twitterscraper_test + +import ( + "testing" +) + +func TestFetchFollowing(t *testing.T) { + users, _, err := testScraper.FetchFollowing("Support", 20, "") + if err != nil { + t.Error(err) + } + if len(users) < 1 || users[len(users)-1].Username == "" { + t.Error("error FetchFollowing() No users found") + } +} + +func TestFetchFollowers(t *testing.T) { + users, _, err := testScraper.FetchFollowers("Support", 20, "") + if err != nil { + t.Error(err) + } + if len(users) < 1 || users[len(users)-1].Username == "" { + t.Error("error FetchFollowing() No users found") + } +} diff --git a/profile.go b/profile.go index 3cfc5ad..98e3517 100644 --- a/profile.go +++ b/profile.go @@ -33,6 +33,9 @@ type Profile struct { UserID string Username string Website string + Sensitive bool + Following bool + FollowedBy bool } type user struct { diff --git a/search.go b/search.go index b4254f9..9f8f04f 100644 --- a/search.go +++ b/search.go @@ -62,7 +62,7 @@ func (timeline *searchTimeline) parseUsers() ([]*Profile, string) { } for _, entry := range instruction.Entries { if entry.Content.ItemContent.UserDisplayType == "User" { - if profile := parseProfile(entry.Content.ItemContent.UserResults.Result.Legacy); profile.Name != "" { + if profile := parseProfileV2(entry.Content.ItemContent.UserResults.Result); profile.Name != "" { if profile.UserID == "" { profile.UserID = entry.Content.ItemContent.UserResults.Result.RestID } diff --git a/timeline_v2.go b/timeline_v2.go index 7ed2cfd..d9a68a7 100644 --- a/timeline_v2.go +++ b/timeline_v2.go @@ -47,6 +47,21 @@ func (result *result) parse() *Tweet { return tw } +type userResult struct { + Typename string `json:"__typename"` + ID string `json:"id"` + RestID string `json:"rest_id"` + AffiliatesHighlightedLabel struct{} `json:"affiliates_highlighted_label"` + HasGraduatedAccess bool `json:"has_graduated_access"` + IsBlueVerified bool `json:"is_blue_verified"` + ProfileImageShape string `json:"profile_image_shape"` + Legacy legacyUserV2 `json:"legacy"` +} + +func (result *userResult) parse() Profile { + return parseProfileV2(*result) +} + type item struct { Item struct { ItemContent struct { @@ -70,10 +85,7 @@ type entry struct { } `json:"tweet_results"` UserDisplayType string `json:"userDisplayType"` UserResults struct { - Result struct { - RestID string `json:"rest_id"` - Legacy legacyUser `json:"legacy"` - } `json:"result"` + Result userResult `json:"result"` } `json:"user_results"` } `json:"itemContent"` } `json:"content"` @@ -94,6 +106,16 @@ type timelineV2 struct { } `json:"instructions"` } `json:"timeline"` } `json:"timeline_v2"` + + Timeline struct { + Timeline struct { + Instructions []struct { + Entries []entry `json:"entries"` + Entry entry `json:"entry"` + Type string `json:"type"` + } `json:"instructions"` + } `json:"timeline"` + } `json:"timeline"` } `json:"result"` } `json:"user"` } `json:"data"` @@ -166,6 +188,24 @@ func (timeline *bookmarksTimelineV2) parseTweets() ([]*Tweet, string) { return tweets, cursor } +func (timeline *timelineV2) parseUsers() ([]*Profile, string) { + var cursor string + var users []*Profile + for _, instruction := range timeline.Data.User.Result.Timeline.Timeline.Instructions { + for _, entry := range instruction.Entries { + if entry.Content.CursorType == "Bottom" { + cursor = entry.Content.Value + continue + } + if entry.Content.ItemContent.UserResults.Result.Typename == "User" { + user := entry.Content.ItemContent.UserResults.Result.parse() + users = append(users, &user) + } + } + } + return users, cursor +} + type threadedConversation struct { Data struct { ThreadedConversationWithInjectionsV2 struct { diff --git a/types.go b/types.go index 344d80c..7a13c95 100644 --- a/types.go +++ b/types.go @@ -168,6 +168,55 @@ type ( ScreenName string `json:"screen_name"` StatusesCount int `json:"statuses_count"` Verified bool `json:"verified"` + FollowedBy bool `json:"followed_by"` + Following bool `json:"following"` + } + + legacyUserV2 struct { + FollowedBy bool `json:"followed_by"` + Following bool `json:"following"` + CanDm bool `json:"can_dm"` + CanMediaTag bool `json:"can_media_tag"` + CreatedAt string `json:"created_at"` + DefaultProfile bool `json:"default_profile"` + DefaultProfileImage bool `json:"default_profile_image"` + Description string `json:"description"` + Entities struct { + Description struct { + Urls []interface{} `json:"urls"` + } `json:"description"` + URL struct { + Urls []struct { + DisplayURL string `json:"display_url"` + ExpandedURL string `json:"expanded_url"` + URL string `json:"url"` + Indices []int `json:"indices"` + } `json:"urls"` + } `json:"url"` + } `json:"entities"` + FastFollowersCount int `json:"fast_followers_count"` + FavouritesCount int `json:"favourites_count"` + FollowersCount int `json:"followers_count"` + FriendsCount int `json:"friends_count"` + HasCustomTimelines bool `json:"has_custom_timelines"` + IsTranslator bool `json:"is_translator"` + ListedCount int `json:"listed_count"` + Location string `json:"location"` + MediaCount int `json:"media_count"` + Name string `json:"name"` + NormalFollowersCount int `json:"normal_followers_count"` + PinnedTweetIdsStr []string `json:"pinned_tweet_ids_str"` + PossiblySensitive bool `json:"possibly_sensitive"` + ProfileBannerURL string `json:"profile_banner_url"` + ProfileImageURLHTTPS string `json:"profile_image_url_https"` + ProfileInterstitialType string `json:"profile_interstitial_type"` + ScreenName string `json:"screen_name"` + StatusesCount int `json:"statuses_count"` + TranslatorType string `json:"translator_type"` + URL string `json:"url"` + Verified bool `json:"verified"` + WantRetweets bool `json:"want_retweets"` + WithheldInCountries []interface{} `json:"withheld_in_countries"` } Place struct { diff --git a/util.go b/util.go index 9bf6d55..0eda4be 100644 --- a/util.go +++ b/util.go @@ -342,7 +342,6 @@ func parseProfile(user legacyUser) Profile { FollowersCount: user.FollowersCount, FollowingCount: user.FavouritesCount, FriendsCount: user.FriendsCount, - IsPrivate: user.Protected, IsVerified: user.Verified, LikesCount: user.FavouritesCount, ListedCount: user.ListedCount, @@ -353,6 +352,8 @@ func parseProfile(user legacyUser) Profile { URL: "https://twitter.com/" + user.ScreenName, UserID: user.IDStr, Username: user.ScreenName, + FollowedBy: user.FollowedBy, + Following: user.Following, } tm, err := time.Parse(time.RubyDate, user.CreatedAt) @@ -368,6 +369,43 @@ func parseProfile(user legacyUser) Profile { return profile } +func parseProfileV2(user userResult) Profile { + u := user.Legacy + profile := Profile{ + Avatar: u.ProfileImageURLHTTPS, + Banner: u.ProfileBannerURL, + Biography: u.Description, + FollowersCount: u.FollowersCount, + FollowingCount: u.FavouritesCount, + FriendsCount: u.FriendsCount, + IsVerified: u.Verified, + LikesCount: u.FavouritesCount, + ListedCount: u.ListedCount, + Location: u.Location, + Name: u.Name, + PinnedTweetIDs: u.PinnedTweetIdsStr, + TweetsCount: u.StatusesCount, + URL: "https://twitter.com/" + u.ScreenName, + UserID: user.ID, + Username: u.ScreenName, + Sensitive: u.PossiblySensitive, + Following: u.Following, + FollowedBy: u.FollowedBy, + } + + tm, err := time.Parse(time.RubyDate, u.CreatedAt) + if err == nil { + tm = tm.UTC() + profile.Joined = &tm + } + + if len(u.Entities.URL.Urls) > 0 { + profile.Website = u.Entities.URL.Urls[0].ExpandedURL + } + + return profile +} + func mapToJSONString(data map[string]interface{}) string { jsonBytes, err := json.Marshal(data) if err != nil {