lain/twitter-scrapper
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity

Merge pull request #4 from imperatrona/following

Browse source 
Follows
This commit is contained in:
Valentine 2024-02-21 06:34:49 +03:00 • committed by GitHub
commit 14ba33db9d
8 changed files with 339 additions and 6 deletions
Download patch file Download diff file Expand all files Collapse all files

26
README.md
View file

@ -29,6 +29,8 @@ You can use this library to get tweets, profiles, and trends trivially.
- [Get profile](#get-profile)
- [Search profile](#search-profile)
- [Get trends](#get-trends)
- [Get following](#get-following)
- [Get followers](#get-followers)
- [Connection](#connection)
- [Proxy](#proxy)
- [HTTP(s)](#https)
@ -332,6 +334,30 @@ profiles, cursor, err := scraper.FetchSearchProfiles("taylorswift13", 20, cursor
trends, err := scraper.GetTrends()
```
### Get following
> [!IMPORTANT]
> Requires authentication!
500 requests / 15 minutes
```golang
var cursor string
users, cursor, err := testScraper.FetchFollowing("Support", 20, cursor)
```
### Get followers
> [!IMPORTANT]
> Requires authentication!
50 requests / 15 minutes
```golang
var cursor string
users, cursor, err := testScraper.FetchFollowers("Support", 20, cursor)
```
## Connection
### Proxy

152
follows.go Normal file
View file

@ -0,0 +1,152 @@
package twitterscraper
import (
"net/url"
"strings"
)
// FetchFollowing gets following profiles list for a given user, via the Twitter frontend GraphQL API.
func (s *Scraper) FetchFollowing(user string, maxUsersNbr int, cursor string) ([]*Profile, string, error) {
userID, err := s.GetUserIDByScreenName(user)
if err != nil {
return nil, "", err
}
return s.FetchFollowingByUserID(userID, maxUsersNbr, cursor)
}
// FetchFollowingByUserID gets following profiles list for a given userID, via the Twitter frontend GraphQL API.
func (s *Scraper) FetchFollowingByUserID(userID string, maxUsersNbr int, cursor string) ([]*Profile, string, error) {
if maxUsersNbr > 200 {
maxUsersNbr = 200
}
req, err := s.newRequest("GET", "https://twitter.com/i/api/graphql/g5P4cbXR4ta4oCeE7y2vLQ/Following")
if err != nil {
return nil, "", err
}
variables := map[string]interface{}{
"userId": userID,
"includePromotedContent": false,
"count": maxUsersNbr,
}
features := map[string]interface{}{
"responsive_web_graphql_exclude_directive_enabled": true,
"verified_phone_label_enabled": false,
"creator_subscriptions_tweet_preview_api_enabled": true,
"responsive_web_graphql_timeline_navigation_enabled": true,
"responsive_web_graphql_skip_user_profile_image_extensions_enabled": false,
"c9s_tweet_anatomy_moderator_badge_enabled": true,
"tweetypie_unmention_optimization_enabled": true,
"responsive_web_edit_tweet_api_enabled": true,
"graphql_is_translatable_rweb_tweet_is_translatable_enabled": true,
"view_counts_everywhere_api_enabled": true,
"longform_notetweets_consumption_enabled": true,
"responsive_web_twitter_article_tweet_consumption_enabled": true,
"tweet_awards_web_tipping_enabled": false,
"freedom_of_speech_not_reach_fetch_enabled": true,
"standardized_nudges_misinfo": true,
"tweet_with_visibility_results_prefer_gql_limited_actions_policy_enabled": true,
"rweb_video_timestamps_enabled": true,
"longform_notetweets_rich_text_read_enabled": true,
"longform_notetweets_inline_media_enabled": true,
"responsive_web_enhance_cards_enabled": false,
}
if cursor != "" {
variables["cursor"] = cursor
}
query := url.Values{}
query.Set("variables", mapToJSONString(variables))
query.Set("features", mapToJSONString(features))
req.URL.RawQuery = query.Encode()
var timeline timelineV2
err = s.RequestAPI(req, &timeline)
if err != nil {
return nil, "", err
}
users, nextCursor := timeline.parseUsers()
if strings.HasPrefix(nextCursor, "0|") {
nextCursor = ""
}
return users, nextCursor, nil
}
// FetchFollowers gets following profiles list for a given user, via the Twitter frontend GraphQL API.
func (s *Scraper) FetchFollowers(user string, maxUsersNbr int, cursor string) ([]*Profile, string, error) {
userID, err := s.GetUserIDByScreenName(user)
if err != nil {
return nil, "", err
}
return s.FetchFollowersByUserID(userID, maxUsersNbr, cursor)
}
// FetchFollowersByUserID gets followers profiles list for a given userID, via the Twitter frontend GraphQL API.
func (s *Scraper) FetchFollowersByUserID(userID string, maxUsersNbr int, cursor string) ([]*Profile, string, error) {
if maxUsersNbr > 200 {
maxUsersNbr = 200
}
req, err := s.newRequest("GET", "https://twitter.com/i/api/graphql/jwbfbSzn0FRL_AMZGsYDag/Followers")
if err != nil {
return nil, "", err
}
variables := map[string]interface{}{
"userId": userID,
"includePromotedContent": false,
"count": maxUsersNbr,
}
features := map[string]interface{}{
"responsive_web_graphql_exclude_directive_enabled": true,
"verified_phone_label_enabled": false,
"creator_subscriptions_tweet_preview_api_enabled": true,
"responsive_web_graphql_timeline_navigation_enabled": true,
"responsive_web_graphql_skip_user_profile_image_extensions_enabled": false,
"c9s_tweet_anatomy_moderator_badge_enabled": true,
"tweetypie_unmention_optimization_enabled": true,
"responsive_web_edit_tweet_api_enabled": true,
"graphql_is_translatable_rweb_tweet_is_translatable_enabled": true,
"view_counts_everywhere_api_enabled": true,
"longform_notetweets_consumption_enabled": true,
"responsive_web_twitter_article_tweet_consumption_enabled": true,
"tweet_awards_web_tipping_enabled": false,
"freedom_of_speech_not_reach_fetch_enabled": true,
"standardized_nudges_misinfo": true,
"tweet_with_visibility_results_prefer_gql_limited_actions_policy_enabled": true,
"rweb_video_timestamps_enabled": true,
"longform_notetweets_rich_text_read_enabled": true,
"longform_notetweets_inline_media_enabled": true,
"responsive_web_enhance_cards_enabled": false,
}
if cursor != "" {
variables["cursor"] = cursor
}
query := url.Values{}
query.Set("variables", mapToJSONString(variables))
query.Set("features", mapToJSONString(features))
req.URL.RawQuery = query.Encode()
var timeline timelineV2
err = s.RequestAPI(req, &timeline)
if err != nil {
return nil, "", err
}
users, nextCursor := timeline.parseUsers()
if strings.HasPrefix(nextCursor, "0|") {
nextCursor = ""
}
return users, nextCursor, nil
}

25
follows_test.go Normal file
View file

@ -0,0 +1,25 @@
package twitterscraper_test
import (
"testing"
)
func TestFetchFollowing(t *testing.T) {
users, _, err := testScraper.FetchFollowing("Support", 20, "")
if err != nil {
t.Error(err)
}
if len(users) < 1 || users[len(users)-1].Username == "" {
t.Error("error FetchFollowing() No users found")
}
}
func TestFetchFollowers(t *testing.T) {
users, _, err := testScraper.FetchFollowers("Support", 20, "")
if err != nil {
t.Error(err)
}
if len(users) < 1 || users[len(users)-1].Username == "" {
t.Error("error FetchFollowing() No users found")
}
}

3
profile.go
View file

@ -33,6 +33,9 @@ type Profile struct {
UserID string
Username string
Website string
Sensitive bool
Following bool
FollowedBy bool
}
type user struct {

2
search.go
View file

@ -62,7 +62,7 @@ func (timeline *searchTimeline) parseUsers() ([]*Profile, string) {
}
for _, entry := range instruction.Entries {
if entry.Content.ItemContent.UserDisplayType == "User" {
if profile := parseProfile(entry.Content.ItemContent.UserResults.Result.Legacy); profile.Name != "" {
if profile := parseProfileV2(entry.Content.ItemContent.UserResults.Result); profile.Name != "" {
if profile.UserID == "" {
profile.UserID = entry.Content.ItemContent.UserResults.Result.RestID
}

48
timeline_v2.go
View file

@ -47,6 +47,21 @@ func (result *result) parse() *Tweet {
return tw
}
type userResult struct {
Typename string `json:"__typename"`
ID string `json:"id"`
RestID string `json:"rest_id"`
AffiliatesHighlightedLabel struct{} `json:"affiliates_highlighted_label"`
HasGraduatedAccess bool `json:"has_graduated_access"`
IsBlueVerified bool `json:"is_blue_verified"`
ProfileImageShape string `json:"profile_image_shape"`
Legacy legacyUserV2 `json:"legacy"`
}
func (result *userResult) parse() Profile {
return parseProfileV2(*result)
}
type item struct {
Item struct {
ItemContent struct {
@ -70,10 +85,7 @@ type entry struct {
} `json:"tweet_results"`
UserDisplayType string `json:"userDisplayType"`
UserResults struct {
Result struct {
RestID string `json:"rest_id"`
Legacy legacyUser `json:"legacy"`
} `json:"result"`
Result userResult `json:"result"`
} `json:"user_results"`
} `json:"itemContent"`
} `json:"content"`
@ -94,6 +106,16 @@ type timelineV2 struct {
} `json:"instructions"`
} `json:"timeline"`
} `json:"timeline_v2"`
Timeline struct {
Timeline struct {
Instructions []struct {
Entries []entry `json:"entries"`
Entry entry `json:"entry"`
Type string `json:"type"`
} `json:"instructions"`
} `json:"timeline"`
} `json:"timeline"`
} `json:"result"`
} `json:"user"`
} `json:"data"`
@ -166,6 +188,24 @@ func (timeline *bookmarksTimelineV2) parseTweets() ([]*Tweet, string) {
return tweets, cursor
}
func (timeline *timelineV2) parseUsers() ([]*Profile, string) {
var cursor string
var users []*Profile
for _, instruction := range timeline.Data.User.Result.Timeline.Timeline.Instructions {
for _, entry := range instruction.Entries {
if entry.Content.CursorType == "Bottom" {
cursor = entry.Content.Value
continue
}
if entry.Content.ItemContent.UserResults.Result.Typename == "User" {
user := entry.Content.ItemContent.UserResults.Result.parse()
users = append(users, &user)
}
}
}
return users, cursor
}
type threadedConversation struct {
Data struct {
ThreadedConversationWithInjectionsV2 struct {

49
types.go
View file

@ -168,6 +168,55 @@ type (
ScreenName string `json:"screen_name"`
StatusesCount int `json:"statuses_count"`
Verified bool `json:"verified"`
FollowedBy bool `json:"followed_by"`
Following bool `json:"following"`
}
legacyUserV2 struct {
FollowedBy bool `json:"followed_by"`
Following bool `json:"following"`
CanDm bool `json:"can_dm"`
CanMediaTag bool `json:"can_media_tag"`
CreatedAt string `json:"created_at"`
DefaultProfile bool `json:"default_profile"`
DefaultProfileImage bool `json:"default_profile_image"`
Description string `json:"description"`
Entities struct {
Description struct {
Urls []interface{} `json:"urls"`
} `json:"description"`
URL struct {
Urls []struct {
DisplayURL string `json:"display_url"`
ExpandedURL string `json:"expanded_url"`
URL string `json:"url"`
Indices []int `json:"indices"`
} `json:"urls"`
} `json:"url"`
} `json:"entities"`
FastFollowersCount int `json:"fast_followers_count"`
FavouritesCount int `json:"favourites_count"`
FollowersCount int `json:"followers_count"`
FriendsCount int `json:"friends_count"`
HasCustomTimelines bool `json:"has_custom_timelines"`
IsTranslator bool `json:"is_translator"`
ListedCount int `json:"listed_count"`
Location string `json:"location"`
MediaCount int `json:"media_count"`
Name string `json:"name"`
NormalFollowersCount int `json:"normal_followers_count"`
PinnedTweetIdsStr []string `json:"pinned_tweet_ids_str"`
PossiblySensitive bool `json:"possibly_sensitive"`
ProfileBannerURL string `json:"profile_banner_url"`
ProfileImageURLHTTPS string `json:"profile_image_url_https"`
ProfileInterstitialType string `json:"profile_interstitial_type"`
ScreenName string `json:"screen_name"`
StatusesCount int `json:"statuses_count"`
TranslatorType string `json:"translator_type"`
URL string `json:"url"`
Verified bool `json:"verified"`
WantRetweets bool `json:"want_retweets"`
WithheldInCountries []interface{} `json:"withheld_in_countries"`
}
Place struct {

40
util.go
View file

@ -342,7 +342,6 @@ func parseProfile(user legacyUser) Profile {
FollowersCount: user.FollowersCount,
FollowingCount: user.FavouritesCount,
FriendsCount: user.FriendsCount,
IsPrivate: user.Protected,
IsVerified: user.Verified,
LikesCount: user.FavouritesCount,
ListedCount: user.ListedCount,
@ -353,6 +352,8 @@ func parseProfile(user legacyUser) Profile {
URL: "https://twitter.com/" + user.ScreenName,
UserID: user.IDStr,
Username: user.ScreenName,
FollowedBy: user.FollowedBy,
Following: user.Following,
}
tm, err := time.Parse(time.RubyDate, user.CreatedAt)
@ -368,6 +369,43 @@ func parseProfile(user legacyUser) Profile {
return profile
}
func parseProfileV2(user userResult) Profile {
u := user.Legacy
profile := Profile{
Avatar: u.ProfileImageURLHTTPS,
Banner: u.ProfileBannerURL,
Biography: u.Description,
FollowersCount: u.FollowersCount,
FollowingCount: u.FavouritesCount,
FriendsCount: u.FriendsCount,
IsVerified: u.Verified,
LikesCount: u.FavouritesCount,
ListedCount: u.ListedCount,
Location: u.Location,
Name: u.Name,
PinnedTweetIDs: u.PinnedTweetIdsStr,
TweetsCount: u.StatusesCount,
URL: "https://twitter.com/" + u.ScreenName,
UserID: user.ID,
Username: u.ScreenName,
Sensitive: u.PossiblySensitive,
Following: u.Following,
FollowedBy: u.FollowedBy,
}
tm, err := time.Parse(time.RubyDate, u.CreatedAt)
if err == nil {
tm = tm.UTC()
profile.Joined = &tm
}
if len(u.Entities.URL.Urls) > 0 {
profile.Website = u.Entities.URL.Urls[0].ExpandedURL
}
return profile
}
func mapToJSONString(data map[string]interface{}) string {
jsonBytes, err := json.Marshal(data)
if err != nil {