From be3eeb90916341a091b0bc162c13a598bfc5deb9 Mon Sep 17 00:00:00 2001 From: Joaco Esteban Date: Sat, 3 Feb 2024 14:52:47 +0100 Subject: [PATCH 01/12] Add tmp to .gitignore --- .gitignore | 1 + 1 file changed, 1 insertion(+) diff --git a/.gitignore b/.gitignore index 0a035ec..d0d7dab 100644 --- a/.gitignore +++ b/.gitignore @@ -1,3 +1,4 @@ *.htm? *.json .idea/ +tmp \ No newline at end of file From c967fdefdd9a2ee0a9fc5bdb2dbfeaedb98264a4 Mon Sep 17 00:00:00 2001 From: Joaco Esteban Date: Sat, 3 Feb 2024 14:53:11 +0100 Subject: [PATCH 02/12] FEAT, list following --- follow.go | 64 ++++++++++++++++++++++++++++++++++++++++++++++++++ search.go | 2 +- timeline_v2.go | 48 +++++++++++++++++++++++++++++++++---- types.go | 46 ++++++++++++++++++++++++++++++++++++ util.go | 35 ++++++++++++++++++++++++++- 5 files changed, 189 insertions(+), 6 deletions(-) create mode 100644 follow.go diff --git a/follow.go b/follow.go new file mode 100644 index 0000000..4a355cf --- /dev/null +++ b/follow.go @@ -0,0 +1,64 @@ +package twitterscraper + +import ( + "net/url" +) + +// FetchFollowingByUserID gets following profiles list for a given userID, via the Twitter frontend GraphQL API. +func (s *Scraper) FetchFollowingByUserID(userID string, maxFollowingNbr int, cursor string) ([]*Profile, string, error) { + if maxFollowingNbr > 200 { + maxFollowingNbr = 200 + } + + req, err := s.newRequest("GET", "https://twitter.com/i/api/graphql/oUxds-Fprv5NKsH67zPt5w/Following") + if err != nil { + return nil, "", err + } + + variables := map[string]interface{}{ + "userId": userID, + "count": maxFollowingNbr, + "includePromotedContent": false, + } + features := map[string]interface{}{ + "responsive_web_graphql_exclude_directive_enabled": true, + "verified_phone_label_enabled": false, + "creator_subscriptions_tweet_preview_api_enabled": true, + "responsive_web_graphql_timeline_navigation_enabled": true, + "responsive_web_graphql_skip_user_profile_image_extensions_enabled": false, + "c9s_tweet_anatomy_moderator_badge_enabled": true, + "tweetypie_unmention_optimization_enabled": true, + "responsive_web_edit_tweet_api_enabled": true, + "graphql_is_translatable_rweb_tweet_is_translatable_enabled": true, + "view_counts_everywhere_api_enabled": true, + "longform_notetweets_consumption_enabled": true, + "responsive_web_twitter_article_tweet_consumption_enabled": true, + "tweet_awards_web_tipping_enabled": false, + "freedom_of_speech_not_reach_fetch_enabled": true, + "standardized_nudges_misinfo": true, + "tweet_with_visibility_results_prefer_gql_limited_actions_policy_enabled": true, + "rweb_video_timestamps_enabled": true, + "longform_notetweets_rich_text_read_enabled": true, + "longform_notetweets_inline_media_enabled": true, + "responsive_web_media_download_video_enabled": false, + "responsive_web_enhance_cards_enabled": false, + } + + if cursor != "" { + variables["cursor"] = cursor + } + + query := url.Values{} + query.Set("variables", mapToJSONString(variables)) + query.Set("features", mapToJSONString(features)) + req.URL.RawQuery = query.Encode() + + var timeline timelineV2 + err = s.RequestAPI(req, &timeline) + if err != nil { + return nil, "", err + } + + users, nextCursor := timeline.parseUsers() + return users, nextCursor, nil +} diff --git a/search.go b/search.go index b4254f9..9f8f04f 100644 --- a/search.go +++ b/search.go @@ -62,7 +62,7 @@ func (timeline *searchTimeline) parseUsers() ([]*Profile, string) { } for _, entry := range instruction.Entries { if entry.Content.ItemContent.UserDisplayType == "User" { - if profile := parseProfile(entry.Content.ItemContent.UserResults.Result.Legacy); profile.Name != "" { + if profile := parseProfileV2(entry.Content.ItemContent.UserResults.Result); profile.Name != "" { if profile.UserID == "" { profile.UserID = entry.Content.ItemContent.UserResults.Result.RestID } diff --git a/timeline_v2.go b/timeline_v2.go index e33bb9b..cd4c164 100644 --- a/timeline_v2.go +++ b/timeline_v2.go @@ -47,6 +47,21 @@ func (result *result) parse() *Tweet { return tw } +type userResult struct { + Typename string `json:"__typename"` + ID string `json:"id"` + RestID string `json:"rest_id"` + AffiliatesHighlightedLabel struct{} `json:"affiliates_highlighted_label"` + HasGraduatedAccess bool `json:"has_graduated_access"` + IsBlueVerified bool `json:"is_blue_verified"` + ProfileImageShape string `json:"profile_image_shape"` + Legacy legacyUserV2 `json:"legacy"` +} + +func (result *userResult) parse() Profile { + return parseProfileV2(*result) +} + type entry struct { Content struct { CursorType string `json:"cursorType"` @@ -68,10 +83,7 @@ type entry struct { } `json:"tweet_results"` UserDisplayType string `json:"userDisplayType"` UserResults struct { - Result struct { - RestID string `json:"rest_id"` - Legacy legacyUser `json:"legacy"` - } `json:"result"` + Result userResult `json:"result"` } `json:"user_results"` } `json:"itemContent"` } `json:"content"` @@ -91,6 +103,16 @@ type timelineV2 struct { } `json:"instructions"` } `json:"timeline"` } `json:"timeline_v2"` + + Timeline struct { + Timeline struct { + Instructions []struct { + Entries []entry `json:"entries"` + Entry entry `json:"entry"` + Type string `json:"type"` + } `json:"instructions"` + } `json:"timeline"` + } `json:"timeline"` } `json:"result"` } `json:"user"` } `json:"data"` @@ -115,6 +137,24 @@ func (timeline *timelineV2) parseTweets() ([]*Tweet, string) { return tweets, cursor } +func (timeline *timelineV2) parseUsers() ([]*Profile, string) { + var cursor string + var users []*Profile + for _, instruction := range timeline.Data.User.Result.Timeline.Timeline.Instructions { + for _, entry := range instruction.Entries { + if entry.Content.CursorType == "Bottom" { + cursor = entry.Content.Value + continue + } + if entry.Content.ItemContent.UserResults.Result.Typename == "User" { + user := entry.Content.ItemContent.UserResults.Result.parse() + users = append(users, &user) + } + } + } + return users, cursor +} + type threadedConversation struct { Data struct { ThreadedConversationWithInjectionsV2 struct { diff --git a/types.go b/types.go index 344d80c..3e96331 100644 --- a/types.go +++ b/types.go @@ -170,6 +170,52 @@ type ( Verified bool `json:"verified"` } + legacyUserV2 struct { + Following bool `json:"following"` + CanDm bool `json:"can_dm"` + CanMediaTag bool `json:"can_media_tag"` + CreatedAt string `json:"created_at"` + DefaultProfile bool `json:"default_profile"` + DefaultProfileImage bool `json:"default_profile_image"` + Description string `json:"description"` + Entities struct { + Description struct { + Urls []interface{} `json:"urls"` + } `json:"description"` + URL struct { + Urls []struct { + DisplayURL string `json:"display_url"` + ExpandedURL string `json:"expanded_url"` + URL string `json:"url"` + Indices []int `json:"indices"` + } `json:"urls"` + } `json:"url"` + } `json:"entities"` + FastFollowersCount int `json:"fast_followers_count"` + FavouritesCount int `json:"favourites_count"` + FollowersCount int `json:"followers_count"` + FriendsCount int `json:"friends_count"` + HasCustomTimelines bool `json:"has_custom_timelines"` + IsTranslator bool `json:"is_translator"` + ListedCount int `json:"listed_count"` + Location string `json:"location"` + MediaCount int `json:"media_count"` + Name string `json:"name"` + NormalFollowersCount int `json:"normal_followers_count"` + PinnedTweetIdsStr []string `json:"pinned_tweet_ids_str"` + PossiblySensitive bool `json:"possibly_sensitive"` + ProfileBannerURL string `json:"profile_banner_url"` + ProfileImageURLHTTPS string `json:"profile_image_url_https"` + ProfileInterstitialType string `json:"profile_interstitial_type"` + ScreenName string `json:"screen_name"` + StatusesCount int `json:"statuses_count"` + TranslatorType string `json:"translator_type"` + URL string `json:"url"` + Verified bool `json:"verified"` + WantRetweets bool `json:"want_retweets"` + WithheldInCountries []interface{} `json:"withheld_in_countries"` + } + Place struct { ID string `json:"id"` PlaceType string `json:"place_type"` diff --git a/util.go b/util.go index 9bf6d55..7851ad7 100644 --- a/util.go +++ b/util.go @@ -342,7 +342,6 @@ func parseProfile(user legacyUser) Profile { FollowersCount: user.FollowersCount, FollowingCount: user.FavouritesCount, FriendsCount: user.FriendsCount, - IsPrivate: user.Protected, IsVerified: user.Verified, LikesCount: user.FavouritesCount, ListedCount: user.ListedCount, @@ -368,6 +367,40 @@ func parseProfile(user legacyUser) Profile { return profile } +func parseProfileV2(user userResult) Profile { + u := user.Legacy + profile := Profile{ + Avatar: u.ProfileImageURLHTTPS, + Banner: u.ProfileBannerURL, + Biography: u.Description, + FollowersCount: u.FollowersCount, + FollowingCount: u.FavouritesCount, + FriendsCount: u.FriendsCount, + IsVerified: u.Verified, + LikesCount: u.FavouritesCount, + ListedCount: u.ListedCount, + Location: u.Location, + Name: u.Name, + PinnedTweetIDs: u.PinnedTweetIdsStr, + TweetsCount: u.StatusesCount, + URL: "https://twitter.com/" + u.ScreenName, + UserID: user.ID, + Username: u.ScreenName, + } + + tm, err := time.Parse(time.RubyDate, u.CreatedAt) + if err == nil { + tm = tm.UTC() + profile.Joined = &tm + } + + if len(u.Entities.URL.Urls) > 0 { + profile.Website = u.Entities.URL.Urls[0].ExpandedURL + } + + return profile +} + func mapToJSONString(data map[string]interface{}) string { jsonBytes, err := json.Marshal(data) if err != nil { From 985351070de8b6475617c43562d44ddedb136417 Mon Sep 17 00:00:00 2001 From: Joaco Esteban Date: Sat, 3 Feb 2024 14:56:01 +0100 Subject: [PATCH 03/12] rename --- README.md | 27 +++++++++++++-------------- api_test.go | 2 +- auth_test.go | 2 +- go.mod | 2 +- profile_test.go | 2 +- search_test.go | 2 +- tweets_test.go | 2 +- 7 files changed, 19 insertions(+), 20 deletions(-) diff --git a/README.md b/README.md index 0dacc97..bb3d37a 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,6 @@ # Twitter Scraper -[![Go Reference](https://pkg.go.dev/badge/github.com/n0madic/twitter-scraper.svg)](https://pkg.go.dev/github.com/n0madic/twitter-scraper) +[![Go Reference](https://pkg.go.dev/badge/github.com/joacoesteban/twitter-scraper.svg)](https://pkg.go.dev/github.com/joacoesteban/twitter-scraper) Twitter's API is annoying to work with, and has lots of limitations — luckily their frontend (JavaScript) has it's own API, which I reverse-engineered. @@ -11,7 +11,7 @@ You can use this library to get the text of any user's Tweets trivially. ## Installation ```shell -go get -u github.com/n0madic/twitter-scraper +go get -u github.com/joacoesteban/twitter-scraper ``` ## Usage @@ -93,7 +93,7 @@ package main import ( "context" "fmt" - twitterscraper "github.com/n0madic/twitter-scraper" + twitterscraper "github.com/joacoesteban/twitter-scraper" ) func main() { @@ -121,7 +121,7 @@ package main import ( "fmt" - twitterscraper "github.com/n0madic/twitter-scraper" + twitterscraper "github.com/joacoesteban/twitter-scraper" ) func main() { @@ -150,7 +150,7 @@ package main import ( "context" "fmt" - twitterscraper "github.com/n0madic/twitter-scraper" + twitterscraper "github.com/joacoesteban/twitter-scraper" ) func main() { @@ -173,7 +173,6 @@ The search ends if we have 50 tweets. See [Rules and filtering](https://developer.twitter.com/en/docs/tweets/rules-and-filtering/overview/standard-operators) for build standard queries. - #### Set search mode ```golang @@ -182,11 +181,11 @@ scraper.SetSearchMode(twitterscraper.SearchLatest) Options: -* `twitterscraper.SearchTop` - default mode -* `twitterscraper.SearchLatest` - live mode -* `twitterscraper.SearchPhotos` - image mode -* `twitterscraper.SearchVideos` - video mode -* `twitterscraper.SearchUsers` - user mode +- `twitterscraper.SearchTop` - default mode +- `twitterscraper.SearchLatest` - live mode +- `twitterscraper.SearchPhotos` - image mode +- `twitterscraper.SearchVideos` - video mode +- `twitterscraper.SearchUsers` - user mode ### Get profile @@ -195,7 +194,7 @@ package main import ( "fmt" - twitterscraper "github.com/n0madic/twitter-scraper" + twitterscraper "github.com/joacoesteban/twitter-scraper" ) func main() { @@ -217,7 +216,7 @@ package main import ( "context" "fmt" - twitterscraper "github.com/n0madic/twitter-scraper" + twitterscraper "github.com/joacoesteban/twitter-scraper" ) func main() { @@ -242,7 +241,7 @@ package main import ( "fmt" - twitterscraper "github.com/n0madic/twitter-scraper" + twitterscraper "github.com/joacoesteban/twitter-scraper" ) func main() { diff --git a/api_test.go b/api_test.go index 2168de9..8436127 100644 --- a/api_test.go +++ b/api_test.go @@ -3,7 +3,7 @@ package twitterscraper_test import ( "testing" - twitterscraper "github.com/n0madic/twitter-scraper" + twitterscraper "github.com/joacoesteban/twitter-scraper" ) func TestGetGuestToken(t *testing.T) { diff --git a/auth_test.go b/auth_test.go index 88bfd68..852759e 100644 --- a/auth_test.go +++ b/auth_test.go @@ -5,7 +5,7 @@ import ( "os" "testing" - twitterscraper "github.com/n0madic/twitter-scraper" + twitterscraper "github.com/joacoesteban/twitter-scraper" ) var ( diff --git a/go.mod b/go.mod index a41c030..3ec16df 100644 --- a/go.mod +++ b/go.mod @@ -1,4 +1,4 @@ -module github.com/n0madic/twitter-scraper +module github.com/joacoesteban/twitter-scraper go 1.16 diff --git a/profile_test.go b/profile_test.go index bdde60d..ae702d1 100644 --- a/profile_test.go +++ b/profile_test.go @@ -7,7 +7,7 @@ import ( "github.com/google/go-cmp/cmp" "github.com/google/go-cmp/cmp/cmpopts" - twitterscraper "github.com/n0madic/twitter-scraper" + twitterscraper "github.com/joacoesteban/twitter-scraper" ) func TestGetProfile(t *testing.T) { diff --git a/search_test.go b/search_test.go index 36c61bc..13ca2f9 100644 --- a/search_test.go +++ b/search_test.go @@ -4,7 +4,7 @@ import ( "context" "testing" - twitterscraper "github.com/n0madic/twitter-scraper" + twitterscraper "github.com/joacoesteban/twitter-scraper" ) func TestFetchSearchCursor(t *testing.T) { diff --git a/tweets_test.go b/tweets_test.go index 71960c1..4db5fd6 100644 --- a/tweets_test.go +++ b/tweets_test.go @@ -7,7 +7,7 @@ import ( "github.com/google/go-cmp/cmp" "github.com/google/go-cmp/cmp/cmpopts" - twitterscraper "github.com/n0madic/twitter-scraper" + twitterscraper "github.com/joacoesteban/twitter-scraper" ) var cmpOptions = cmp.Options{ From a1511c386907faddcb18e5b1618a71acdd8293e4 Mon Sep 17 00:00:00 2001 From: Joaco Esteban Date: Sat, 3 Feb 2024 15:34:03 +0100 Subject: [PATCH 04/12] Remove maxFollowingNbr parameter from FetchFollowingByUserID function --- follow.go | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/follow.go b/follow.go index 4a355cf..329b636 100644 --- a/follow.go +++ b/follow.go @@ -5,10 +5,7 @@ import ( ) // FetchFollowingByUserID gets following profiles list for a given userID, via the Twitter frontend GraphQL API. -func (s *Scraper) FetchFollowingByUserID(userID string, maxFollowingNbr int, cursor string) ([]*Profile, string, error) { - if maxFollowingNbr > 200 { - maxFollowingNbr = 200 - } +func (s *Scraper) FetchFollowingByUserID(userID string, cursor string) ([]*Profile, string, error) { req, err := s.newRequest("GET", "https://twitter.com/i/api/graphql/oUxds-Fprv5NKsH67zPt5w/Following") if err != nil { @@ -17,7 +14,6 @@ func (s *Scraper) FetchFollowingByUserID(userID string, maxFollowingNbr int, cur variables := map[string]interface{}{ "userId": userID, - "count": maxFollowingNbr, "includePromotedContent": false, } features := map[string]interface{}{ From bd65223b4146e94a49f4e48385c3c8c204329563 Mon Sep 17 00:00:00 2001 From: Joaco Esteban Date: Wed, 14 Feb 2024 12:27:29 +0100 Subject: [PATCH 05/12] Add Sensitive field to Profile struct --- profile.go | 1 + util.go | 1 + 2 files changed, 2 insertions(+) diff --git a/profile.go b/profile.go index 3cfc5ad..a96e2fe 100644 --- a/profile.go +++ b/profile.go @@ -33,6 +33,7 @@ type Profile struct { UserID string Username string Website string + Sensitive bool } type user struct { diff --git a/util.go b/util.go index 7851ad7..0ed2b1f 100644 --- a/util.go +++ b/util.go @@ -386,6 +386,7 @@ func parseProfileV2(user userResult) Profile { URL: "https://twitter.com/" + u.ScreenName, UserID: user.ID, Username: u.ScreenName, + Sensitive: u.PossiblySensitive, } tm, err := time.Parse(time.RubyDate, u.CreatedAt) From db699c17122e4ef3c68d6737356a265038661b0f Mon Sep 17 00:00:00 2001 From: Valentine Date: Wed, 21 Feb 2024 05:56:40 +0300 Subject: [PATCH 06/12] rename follow --- follow.go => follows.go | 22 +++++++++++++++++++++- 1 file changed, 21 insertions(+), 1 deletion(-) rename follow.go => follows.go (80%) diff --git a/follow.go b/follows.go similarity index 80% rename from follow.go rename to follows.go index 329b636..de2958d 100644 --- a/follow.go +++ b/follows.go @@ -2,10 +2,24 @@ package twitterscraper import ( "net/url" + "strings" ) +// FetchFollowing gets tweets with medias for a given user, via the Twitter frontend API. +func (s *Scraper) FetchFollowing(user string, maxUsersNbr int, cursor string) ([]*Profile, string, error) { + userID, err := s.GetUserIDByScreenName(user) + if err != nil { + return nil, "", err + } + + return s.FetchFollowingByUserID(userID, maxUsersNbr, cursor) +} + // FetchFollowingByUserID gets following profiles list for a given userID, via the Twitter frontend GraphQL API. -func (s *Scraper) FetchFollowingByUserID(userID string, cursor string) ([]*Profile, string, error) { +func (s *Scraper) FetchFollowingByUserID(userID string, maxUsersNbr int, cursor string) ([]*Profile, string, error) { + if maxUsersNbr > 200 { + maxUsersNbr = 200 + } req, err := s.newRequest("GET", "https://twitter.com/i/api/graphql/oUxds-Fprv5NKsH67zPt5w/Following") if err != nil { @@ -15,6 +29,7 @@ func (s *Scraper) FetchFollowingByUserID(userID string, cursor string) ([]*Profi variables := map[string]interface{}{ "userId": userID, "includePromotedContent": false, + "count": maxUsersNbr, } features := map[string]interface{}{ "responsive_web_graphql_exclude_directive_enabled": true, @@ -56,5 +71,10 @@ func (s *Scraper) FetchFollowingByUserID(userID string, cursor string) ([]*Profi } users, nextCursor := timeline.parseUsers() + + if strings.HasPrefix(nextCursor, "0|") { + nextCursor = "" + } + return users, nextCursor, nil } From 0046ed18c96bded24526d418b81f7c46bc385b7d Mon Sep 17 00:00:00 2001 From: Valentine Date: Wed, 21 Feb 2024 05:56:50 +0300 Subject: [PATCH 07/12] add following test --- follows_test.go | 15 +++++++++++++++ 1 file changed, 15 insertions(+) create mode 100644 follows_test.go diff --git a/follows_test.go b/follows_test.go new file mode 100644 index 0000000..26ed836 --- /dev/null +++ b/follows_test.go @@ -0,0 +1,15 @@ +package twitterscraper_test + +import ( + "testing" +) + +func TestFetchFollowing(t *testing.T) { + users, _, err := testScraper.FetchFollowing("Support", 20, "") + if err != nil { + t.Error(err) + } + if len(users) < 1 || users[len(users)-1].Username == "" { + t.Error("error FetchFollowing() No users found") + } +} From 5d3a5724f9d8fa9c497febeb7c4493b80ec8bc0f Mon Sep 17 00:00:00 2001 From: Valentine Date: Wed, 21 Feb 2024 06:15:04 +0300 Subject: [PATCH 08/12] add followers method --- follows.go | 78 +++++++++++++++++++++++++++++++++++++++++++++++-- follows_test.go | 12 ++++++++ 2 files changed, 87 insertions(+), 3 deletions(-) diff --git a/follows.go b/follows.go index de2958d..b849afa 100644 --- a/follows.go +++ b/follows.go @@ -5,7 +5,7 @@ import ( "strings" ) -// FetchFollowing gets tweets with medias for a given user, via the Twitter frontend API. +// FetchFollowing gets following profiles list for a given user, via the Twitter frontend GraphQL API. func (s *Scraper) FetchFollowing(user string, maxUsersNbr int, cursor string) ([]*Profile, string, error) { userID, err := s.GetUserIDByScreenName(user) if err != nil { @@ -21,7 +21,80 @@ func (s *Scraper) FetchFollowingByUserID(userID string, maxUsersNbr int, cursor maxUsersNbr = 200 } - req, err := s.newRequest("GET", "https://twitter.com/i/api/graphql/oUxds-Fprv5NKsH67zPt5w/Following") + req, err := s.newRequest("GET", "https://twitter.com/i/api/graphql/g5P4cbXR4ta4oCeE7y2vLQ/Following") + if err != nil { + return nil, "", err + } + + variables := map[string]interface{}{ + "userId": userID, + "includePromotedContent": false, + "count": maxUsersNbr, + } + features := map[string]interface{}{ + "responsive_web_graphql_exclude_directive_enabled": true, + "verified_phone_label_enabled": false, + "creator_subscriptions_tweet_preview_api_enabled": true, + "responsive_web_graphql_timeline_navigation_enabled": true, + "responsive_web_graphql_skip_user_profile_image_extensions_enabled": false, + "c9s_tweet_anatomy_moderator_badge_enabled": true, + "tweetypie_unmention_optimization_enabled": true, + "responsive_web_edit_tweet_api_enabled": true, + "graphql_is_translatable_rweb_tweet_is_translatable_enabled": true, + "view_counts_everywhere_api_enabled": true, + "longform_notetweets_consumption_enabled": true, + "responsive_web_twitter_article_tweet_consumption_enabled": true, + "tweet_awards_web_tipping_enabled": false, + "freedom_of_speech_not_reach_fetch_enabled": true, + "standardized_nudges_misinfo": true, + "tweet_with_visibility_results_prefer_gql_limited_actions_policy_enabled": true, + "rweb_video_timestamps_enabled": true, + "longform_notetweets_rich_text_read_enabled": true, + "longform_notetweets_inline_media_enabled": true, + "responsive_web_enhance_cards_enabled": false, + } + + if cursor != "" { + variables["cursor"] = cursor + } + + query := url.Values{} + query.Set("variables", mapToJSONString(variables)) + query.Set("features", mapToJSONString(features)) + req.URL.RawQuery = query.Encode() + + var timeline timelineV2 + err = s.RequestAPI(req, &timeline) + if err != nil { + return nil, "", err + } + + users, nextCursor := timeline.parseUsers() + + if strings.HasPrefix(nextCursor, "0|") { + nextCursor = "" + } + + return users, nextCursor, nil +} + +// FetchFollowers gets following profiles list for a given user, via the Twitter frontend GraphQL API. +func (s *Scraper) FetchFollowers(user string, maxUsersNbr int, cursor string) ([]*Profile, string, error) { + userID, err := s.GetUserIDByScreenName(user) + if err != nil { + return nil, "", err + } + + return s.FetchFollowersByUserID(userID, maxUsersNbr, cursor) +} + +// FetchFollowersByUserID gets followers profiles list for a given userID, via the Twitter frontend GraphQL API. +func (s *Scraper) FetchFollowersByUserID(userID string, maxUsersNbr int, cursor string) ([]*Profile, string, error) { + if maxUsersNbr > 200 { + maxUsersNbr = 200 + } + + req, err := s.newRequest("GET", "https://twitter.com/i/api/graphql/jwbfbSzn0FRL_AMZGsYDag/Followers") if err != nil { return nil, "", err } @@ -51,7 +124,6 @@ func (s *Scraper) FetchFollowingByUserID(userID string, maxUsersNbr int, cursor "rweb_video_timestamps_enabled": true, "longform_notetweets_rich_text_read_enabled": true, "longform_notetweets_inline_media_enabled": true, - "responsive_web_media_download_video_enabled": false, "responsive_web_enhance_cards_enabled": false, } diff --git a/follows_test.go b/follows_test.go index 26ed836..112a4fb 100644 --- a/follows_test.go +++ b/follows_test.go @@ -1,6 +1,7 @@ package twitterscraper_test import ( + "fmt" "testing" ) @@ -9,6 +10,17 @@ func TestFetchFollowing(t *testing.T) { if err != nil { t.Error(err) } + fmt.Println(users[0].Username) + if len(users) < 1 || users[len(users)-1].Username == "" { + t.Error("error FetchFollowing() No users found") + } +} + +func TestFetchFollowers(t *testing.T) { + users, _, err := testScraper.FetchFollowers("Support", 20, "") + if err != nil { + t.Error(err) + } if len(users) < 1 || users[len(users)-1].Username == "" { t.Error("error FetchFollowing() No users found") } From cf6de782e0cf341d32ea1e1b8848d36cafceb037 Mon Sep 17 00:00:00 2001 From: Valentine Date: Wed, 21 Feb 2024 06:15:17 +0300 Subject: [PATCH 09/12] add follows to docs --- README.md | 26 ++++++++++++++++++++++++++ 1 file changed, 26 insertions(+) diff --git a/README.md b/README.md index 5d4f3b9..718bc81 100644 --- a/README.md +++ b/README.md @@ -29,6 +29,8 @@ You can use this library to get tweets, profiles, and trends trivially. - [Get profile](#get-profile) - [Search profile](#search-profile) - [Get trends](#get-trends) + - [Get following](#get-following) + - [Get followers](#get-followers) - [Connection](#connection) - [Proxy](#proxy) - [HTTP(s)](#https) @@ -332,6 +334,30 @@ profiles, cursor, err := scraper.FetchSearchProfiles("taylorswift13", 20, cursor trends, err := scraper.GetTrends() ``` +### Get following + +> [!IMPORTANT] +> Requires authentication! + +500 requests / 15 minutes + +```golang +var cursor string +users, cursor, err := testScraper.FetchFollowing("Support", 20, cursor) +``` + +### Get followers + +> [!IMPORTANT] +> Requires authentication! + +50 requests / 15 minutes + +```golang +var cursor string +users, cursor, err := testScraper.FetchFollowers("Support", 20, cursor) +``` + ## Connection ### Proxy From 5dee67c2dafd4e5a5fbd707d8c95ab4addfbb161 Mon Sep 17 00:00:00 2001 From: Valentine Date: Wed, 21 Feb 2024 06:21:49 +0300 Subject: [PATCH 10/12] add following & followed by to profile --- profile.go | 2 ++ types.go | 1 + util.go | 2 ++ 3 files changed, 5 insertions(+) diff --git a/profile.go b/profile.go index a96e2fe..98e3517 100644 --- a/profile.go +++ b/profile.go @@ -34,6 +34,8 @@ type Profile struct { Username string Website string Sensitive bool + Following bool + FollowedBy bool } type user struct { diff --git a/types.go b/types.go index 3e96331..715f588 100644 --- a/types.go +++ b/types.go @@ -171,6 +171,7 @@ type ( } legacyUserV2 struct { + FollowedBy bool `json:"followed_by"` Following bool `json:"following"` CanDm bool `json:"can_dm"` CanMediaTag bool `json:"can_media_tag"` diff --git a/util.go b/util.go index 0ed2b1f..6912ee1 100644 --- a/util.go +++ b/util.go @@ -387,6 +387,8 @@ func parseProfileV2(user userResult) Profile { UserID: user.ID, Username: u.ScreenName, Sensitive: u.PossiblySensitive, + Following: u.Following, + FollowedBy: u.FollowedBy, } tm, err := time.Parse(time.RubyDate, u.CreatedAt) From 4f710ab596ed3da81f332ee1d5ace127c46dcd4b Mon Sep 17 00:00:00 2001 From: Valentine Date: Wed, 21 Feb 2024 06:21:57 +0300 Subject: [PATCH 11/12] rm fmt from test --- follows_test.go | 2 -- 1 file changed, 2 deletions(-) diff --git a/follows_test.go b/follows_test.go index 112a4fb..3aa9227 100644 --- a/follows_test.go +++ b/follows_test.go @@ -1,7 +1,6 @@ package twitterscraper_test import ( - "fmt" "testing" ) @@ -10,7 +9,6 @@ func TestFetchFollowing(t *testing.T) { if err != nil { t.Error(err) } - fmt.Println(users[0].Username) if len(users) < 1 || users[len(users)-1].Username == "" { t.Error("error FetchFollowing() No users found") } From 43897cad9f07966553d76eb49a11b5363a4cc5e4 Mon Sep 17 00:00:00 2001 From: Valentine Date: Wed, 21 Feb 2024 06:33:06 +0300 Subject: [PATCH 12/12] add follows & followed by to legacyUser --- types.go | 2 ++ util.go | 2 ++ 2 files changed, 4 insertions(+) diff --git a/types.go b/types.go index 715f588..7a13c95 100644 --- a/types.go +++ b/types.go @@ -168,6 +168,8 @@ type ( ScreenName string `json:"screen_name"` StatusesCount int `json:"statuses_count"` Verified bool `json:"verified"` + FollowedBy bool `json:"followed_by"` + Following bool `json:"following"` } legacyUserV2 struct { diff --git a/util.go b/util.go index 6912ee1..0eda4be 100644 --- a/util.go +++ b/util.go @@ -352,6 +352,8 @@ func parseProfile(user legacyUser) Profile { URL: "https://twitter.com/" + user.ScreenName, UserID: user.IDStr, Username: user.ScreenName, + FollowedBy: user.FollowedBy, + Following: user.Following, } tm, err := time.Parse(time.RubyDate, user.CreatedAt)