Merge pull request #4 from imperatrona/following

Follows

README.md

@@ -29,6 +29,8 @@ You can use this library to get tweets, profiles, and trends trivially.
   - [Get profile](#get-profile)
   - [Search profile](#search-profile)
   - [Get trends](#get-trends)
+  - [Get following](#get-following)
+  - [Get followers](#get-followers)
 - [Connection](#connection)
   - [Proxy](#proxy)
   - [HTTP(s)](#https)
@@ -332,6 +334,30 @@ profiles, cursor, err := scraper.FetchSearchProfiles("taylorswift13", 20, cursor
 trends, err := scraper.GetTrends()
 ```
 
+### Get following
+
+> [!IMPORTANT]  
+> Requires authentication!
+
+500 requests / 15 minutes
+
+```golang
+var cursor string
+users, cursor, err := testScraper.FetchFollowing("Support", 20, cursor)
+```
+
+### Get followers
+
+> [!IMPORTANT]  
+> Requires authentication!
+
+50 requests / 15 minutes
+
+```golang
+var cursor string
+users, cursor, err := testScraper.FetchFollowers("Support", 20, cursor)
+```
+
 ## Connection
 
 ### Proxy

follows.go

@@ -0,0 +1,152 @@
+package twitterscraper
+
+import (
+	"net/url"
+	"strings"
+)
+
+// FetchFollowing gets following profiles list for a given user, via the Twitter frontend GraphQL API.
+func (s *Scraper) FetchFollowing(user string, maxUsersNbr int, cursor string) ([]*Profile, string, error) {
+	userID, err := s.GetUserIDByScreenName(user)
+	if err != nil {
+		return nil, "", err
+	}
+
+	return s.FetchFollowingByUserID(userID, maxUsersNbr, cursor)
+}
+
+// FetchFollowingByUserID gets following profiles list for a given userID, via the Twitter frontend GraphQL API.
+func (s *Scraper) FetchFollowingByUserID(userID string, maxUsersNbr int, cursor string) ([]*Profile, string, error) {
+	if maxUsersNbr > 200 {
+		maxUsersNbr = 200
+	}
+
+	req, err := s.newRequest("GET", "https://twitter.com/i/api/graphql/g5P4cbXR4ta4oCeE7y2vLQ/Following")
+	if err != nil {
+		return nil, "", err
+	}
+
+	variables := map[string]interface{}{
+		"userId":                 userID,
+		"includePromotedContent": false,
+		"count":                  maxUsersNbr,
+	}
+	features := map[string]interface{}{
+		"responsive_web_graphql_exclude_directive_enabled":                        true,
+		"verified_phone_label_enabled":                                            false,
+		"creator_subscriptions_tweet_preview_api_enabled":                         true,
+		"responsive_web_graphql_timeline_navigation_enabled":                      true,
+		"responsive_web_graphql_skip_user_profile_image_extensions_enabled":       false,
+		"c9s_tweet_anatomy_moderator_badge_enabled":                               true,
+		"tweetypie_unmention_optimization_enabled":                                true,
+		"responsive_web_edit_tweet_api_enabled":                                   true,
+		"graphql_is_translatable_rweb_tweet_is_translatable_enabled":              true,
+		"view_counts_everywhere_api_enabled":                                      true,
+		"longform_notetweets_consumption_enabled":                                 true,
+		"responsive_web_twitter_article_tweet_consumption_enabled":                true,
+		"tweet_awards_web_tipping_enabled":                                        false,
+		"freedom_of_speech_not_reach_fetch_enabled":                               true,
+		"standardized_nudges_misinfo":                                             true,
+		"tweet_with_visibility_results_prefer_gql_limited_actions_policy_enabled": true,
+		"rweb_video_timestamps_enabled":                                           true,
+		"longform_notetweets_rich_text_read_enabled":                              true,
+		"longform_notetweets_inline_media_enabled":                                true,
+		"responsive_web_enhance_cards_enabled":                                    false,
+	}
+
+	if cursor != "" {
+		variables["cursor"] = cursor
+	}
+
+	query := url.Values{}
+	query.Set("variables", mapToJSONString(variables))
+	query.Set("features", mapToJSONString(features))
+	req.URL.RawQuery = query.Encode()
+
+	var timeline timelineV2
+	err = s.RequestAPI(req, &timeline)
+	if err != nil {
+		return nil, "", err
+	}
+
+	users, nextCursor := timeline.parseUsers()
+
+	if strings.HasPrefix(nextCursor, "0|") {
+		nextCursor = ""
+	}
+
+	return users, nextCursor, nil
+}
+
+// FetchFollowers gets following profiles list for a given user, via the Twitter frontend GraphQL API.
+func (s *Scraper) FetchFollowers(user string, maxUsersNbr int, cursor string) ([]*Profile, string, error) {
+	userID, err := s.GetUserIDByScreenName(user)
+	if err != nil {
+		return nil, "", err
+	}
+
+	return s.FetchFollowersByUserID(userID, maxUsersNbr, cursor)
+}
+
+// FetchFollowersByUserID gets followers profiles list for a given userID, via the Twitter frontend GraphQL API.
+func (s *Scraper) FetchFollowersByUserID(userID string, maxUsersNbr int, cursor string) ([]*Profile, string, error) {
+	if maxUsersNbr > 200 {
+		maxUsersNbr = 200
+	}
+
+	req, err := s.newRequest("GET", "https://twitter.com/i/api/graphql/jwbfbSzn0FRL_AMZGsYDag/Followers")
+	if err != nil {
+		return nil, "", err
+	}
+
+	variables := map[string]interface{}{
+		"userId":                 userID,
+		"includePromotedContent": false,
+		"count":                  maxUsersNbr,
+	}
+	features := map[string]interface{}{
+		"responsive_web_graphql_exclude_directive_enabled":                        true,
+		"verified_phone_label_enabled":                                            false,
+		"creator_subscriptions_tweet_preview_api_enabled":                         true,
+		"responsive_web_graphql_timeline_navigation_enabled":                      true,
+		"responsive_web_graphql_skip_user_profile_image_extensions_enabled":       false,
+		"c9s_tweet_anatomy_moderator_badge_enabled":                               true,
+		"tweetypie_unmention_optimization_enabled":                                true,
+		"responsive_web_edit_tweet_api_enabled":                                   true,
+		"graphql_is_translatable_rweb_tweet_is_translatable_enabled":              true,
+		"view_counts_everywhere_api_enabled":                                      true,
+		"longform_notetweets_consumption_enabled":                                 true,
+		"responsive_web_twitter_article_tweet_consumption_enabled":                true,
+		"tweet_awards_web_tipping_enabled":                                        false,
+		"freedom_of_speech_not_reach_fetch_enabled":                               true,
+		"standardized_nudges_misinfo":                                             true,
+		"tweet_with_visibility_results_prefer_gql_limited_actions_policy_enabled": true,
+		"rweb_video_timestamps_enabled":                                           true,
+		"longform_notetweets_rich_text_read_enabled":                              true,
+		"longform_notetweets_inline_media_enabled":                                true,
+		"responsive_web_enhance_cards_enabled":                                    false,
+	}
+
+	if cursor != "" {
+		variables["cursor"] = cursor
+	}
+
+	query := url.Values{}
+	query.Set("variables", mapToJSONString(variables))
+	query.Set("features", mapToJSONString(features))
+	req.URL.RawQuery = query.Encode()
+
+	var timeline timelineV2
+	err = s.RequestAPI(req, &timeline)
+	if err != nil {
+		return nil, "", err
+	}
+
+	users, nextCursor := timeline.parseUsers()
+
+	if strings.HasPrefix(nextCursor, "0|") {
+		nextCursor = ""
+	}
+
+	return users, nextCursor, nil
+}

follows_test.go

@@ -0,0 +1,25 @@
+package twitterscraper_test
+
+import (
+	"testing"
+)
+
+func TestFetchFollowing(t *testing.T) {
+	users, _, err := testScraper.FetchFollowing("Support", 20, "")
+	if err != nil {
+		t.Error(err)
+	}
+	if len(users) < 1 || users[len(users)-1].Username == "" {
+		t.Error("error FetchFollowing() No users found")
+	}
+}
+
+func TestFetchFollowers(t *testing.T) {
+	users, _, err := testScraper.FetchFollowers("Support", 20, "")
+	if err != nil {
+		t.Error(err)
+	}
+	if len(users) < 1 || users[len(users)-1].Username == "" {
+		t.Error("error FetchFollowing() No users found")
+	}
+}

profile.go

@@ -33,6 +33,9 @@ type Profile struct {
 	UserID         string
 	Username       string
 	Website        string
+	Sensitive      bool
+	Following      bool
+	FollowedBy     bool
 }
 
 type user struct {

search.go

@@ -62,7 +62,7 @@ func (timeline *searchTimeline) parseUsers() ([]*Profile, string) {
 			}
 			for _, entry := range instruction.Entries {
 				if entry.Content.ItemContent.UserDisplayType == "User" {
-					if profile := parseProfile(entry.Content.ItemContent.UserResults.Result.Legacy); profile.Name != "" {
+					if profile := parseProfileV2(entry.Content.ItemContent.UserResults.Result); profile.Name != "" {
 						if profile.UserID == "" {
 							profile.UserID = entry.Content.ItemContent.UserResults.Result.RestID
 						}

timeline_v2.go

@@ -47,6 +47,21 @@ func (result *result) parse() *Tweet {
 	return tw
 }
 
+type userResult struct {
+	Typename                   string       `json:"__typename"`
+	ID                         string       `json:"id"`
+	RestID                     string       `json:"rest_id"`
+	AffiliatesHighlightedLabel struct{}     `json:"affiliates_highlighted_label"`
+	HasGraduatedAccess         bool         `json:"has_graduated_access"`
+	IsBlueVerified             bool         `json:"is_blue_verified"`
+	ProfileImageShape          string       `json:"profile_image_shape"`
+	Legacy                     legacyUserV2 `json:"legacy"`
+}
+
+func (result *userResult) parse() Profile {
+	return parseProfileV2(*result)
+}
+
 type item struct {
 	Item struct {
 		ItemContent struct {
@@ -70,10 +85,7 @@ type entry struct {
 			} `json:"tweet_results"`
 			UserDisplayType string `json:"userDisplayType"`
 			UserResults     struct {
-				Result struct {
+				Result userResult `json:"result"`
-					RestID string     `json:"rest_id"`
-					Legacy legacyUser `json:"legacy"`
-				} `json:"result"`
 			} `json:"user_results"`
 		} `json:"itemContent"`
 	} `json:"content"`
@@ -94,6 +106,16 @@ type timelineV2 struct {
 						} `json:"instructions"`
 					} `json:"timeline"`
 				} `json:"timeline_v2"`
+
+				Timeline struct {
+					Timeline struct {
+						Instructions []struct {
+							Entries []entry `json:"entries"`
+							Entry   entry   `json:"entry"`
+							Type    string  `json:"type"`
+						} `json:"instructions"`
+					} `json:"timeline"`
+				} `json:"timeline"`
 			} `json:"result"`
 		} `json:"user"`
 	} `json:"data"`
@@ -166,6 +188,24 @@ func (timeline *bookmarksTimelineV2) parseTweets() ([]*Tweet, string) {
 	return tweets, cursor
 }
 
+func (timeline *timelineV2) parseUsers() ([]*Profile, string) {
+	var cursor string
+	var users []*Profile
+	for _, instruction := range timeline.Data.User.Result.Timeline.Timeline.Instructions {
+		for _, entry := range instruction.Entries {
+			if entry.Content.CursorType == "Bottom" {
+				cursor = entry.Content.Value
+				continue
+			}
+			if entry.Content.ItemContent.UserResults.Result.Typename == "User" {
+				user := entry.Content.ItemContent.UserResults.Result.parse()
+				users = append(users, &user)
+			}
+		}
+	}
+	return users, cursor
+}
+
 type threadedConversation struct {
 	Data struct {
 		ThreadedConversationWithInjectionsV2 struct {

types.go

@@ -168,6 +168,55 @@ type (
 		ScreenName           string   `json:"screen_name"`
 		StatusesCount        int      `json:"statuses_count"`
 		Verified             bool     `json:"verified"`
+		FollowedBy           bool     `json:"followed_by"`
+		Following            bool     `json:"following"`
+	}
+
+	legacyUserV2 struct {
+		FollowedBy          bool   `json:"followed_by"`
+		Following           bool   `json:"following"`
+		CanDm               bool   `json:"can_dm"`
+		CanMediaTag         bool   `json:"can_media_tag"`
+		CreatedAt           string `json:"created_at"`
+		DefaultProfile      bool   `json:"default_profile"`
+		DefaultProfileImage bool   `json:"default_profile_image"`
+		Description         string `json:"description"`
+		Entities            struct {
+			Description struct {
+				Urls []interface{} `json:"urls"`
+			} `json:"description"`
+			URL struct {
+				Urls []struct {
+					DisplayURL  string `json:"display_url"`
+					ExpandedURL string `json:"expanded_url"`
+					URL         string `json:"url"`
+					Indices     []int  `json:"indices"`
+				} `json:"urls"`
+			} `json:"url"`
+		} `json:"entities"`
+		FastFollowersCount      int           `json:"fast_followers_count"`
+		FavouritesCount         int           `json:"favourites_count"`
+		FollowersCount          int           `json:"followers_count"`
+		FriendsCount            int           `json:"friends_count"`
+		HasCustomTimelines      bool          `json:"has_custom_timelines"`
+		IsTranslator            bool          `json:"is_translator"`
+		ListedCount             int           `json:"listed_count"`
+		Location                string        `json:"location"`
+		MediaCount              int           `json:"media_count"`
+		Name                    string        `json:"name"`
+		NormalFollowersCount    int           `json:"normal_followers_count"`
+		PinnedTweetIdsStr       []string      `json:"pinned_tweet_ids_str"`
+		PossiblySensitive       bool          `json:"possibly_sensitive"`
+		ProfileBannerURL        string        `json:"profile_banner_url"`
+		ProfileImageURLHTTPS    string        `json:"profile_image_url_https"`
+		ProfileInterstitialType string        `json:"profile_interstitial_type"`
+		ScreenName              string        `json:"screen_name"`
+		StatusesCount           int           `json:"statuses_count"`
+		TranslatorType          string        `json:"translator_type"`
+		URL                     string        `json:"url"`
+		Verified                bool          `json:"verified"`
+		WantRetweets            bool          `json:"want_retweets"`
+		WithheldInCountries     []interface{} `json:"withheld_in_countries"`
 	}
 
 	Place struct {

util.go

@@ -342,7 +342,6 @@ func parseProfile(user legacyUser) Profile {
 		FollowersCount: user.FollowersCount,
 		FollowingCount: user.FavouritesCount,
 		FriendsCount:   user.FriendsCount,
-		IsPrivate:      user.Protected,
 		IsVerified:     user.Verified,
 		LikesCount:     user.FavouritesCount,
 		ListedCount:    user.ListedCount,
@@ -353,6 +352,8 @@ func parseProfile(user legacyUser) Profile {
 		URL:            "https://twitter.com/" + user.ScreenName,
 		UserID:         user.IDStr,
 		Username:       user.ScreenName,
+		FollowedBy:     user.FollowedBy,
+		Following:      user.Following,
 	}
 
 	tm, err := time.Parse(time.RubyDate, user.CreatedAt)
@@ -368,6 +369,43 @@ func parseProfile(user legacyUser) Profile {
 	return profile
 }
 
+func parseProfileV2(user userResult) Profile {
+	u := user.Legacy
+	profile := Profile{
+		Avatar:         u.ProfileImageURLHTTPS,
+		Banner:         u.ProfileBannerURL,
+		Biography:      u.Description,
+		FollowersCount: u.FollowersCount,
+		FollowingCount: u.FavouritesCount,
+		FriendsCount:   u.FriendsCount,
+		IsVerified:     u.Verified,
+		LikesCount:     u.FavouritesCount,
+		ListedCount:    u.ListedCount,
+		Location:       u.Location,
+		Name:           u.Name,
+		PinnedTweetIDs: u.PinnedTweetIdsStr,
+		TweetsCount:    u.StatusesCount,
+		URL:            "https://twitter.com/" + u.ScreenName,
+		UserID:         user.ID,
+		Username:       u.ScreenName,
+		Sensitive:      u.PossiblySensitive,
+		Following:      u.Following,
+		FollowedBy:     u.FollowedBy,
+	}
+
+	tm, err := time.Parse(time.RubyDate, u.CreatedAt)
+	if err == nil {
+		tm = tm.UTC()
+		profile.Joined = &tm
+	}
+
+	if len(u.Entities.URL.Urls) > 0 {
+		profile.Website = u.Entities.URL.Urls[0].ExpandedURL
+	}
+
+	return profile
+}
+
 func mapToJSONString(data map[string]interface{}) string {
 	jsonBytes, err := json.Marshal(data)
 	if err != nil {