lain/twitter-scrapper
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity

FEAT, list following

Browse source
This commit is contained in:
Joaco Esteban 2024-02-03 14:53:11 +01:00
parent be3eeb9091
commit c967fdefdd
5 changed files with 189 additions and 6 deletions
Download patch file Download diff file Expand all files Collapse all files

64
follow.go Normal file
View file

@ -0,0 +1,64 @@
package twitterscraper
import (
"net/url"
)
// FetchFollowingByUserID gets following profiles list for a given userID, via the Twitter frontend GraphQL API.
func (s *Scraper) FetchFollowingByUserID(userID string, maxFollowingNbr int, cursor string) ([]*Profile, string, error) {
if maxFollowingNbr > 200 {
maxFollowingNbr = 200
}
req, err := s.newRequest("GET", "https://twitter.com/i/api/graphql/oUxds-Fprv5NKsH67zPt5w/Following")
if err != nil {
return nil, "", err
}
variables := map[string]interface{}{
"userId": userID,
"count": maxFollowingNbr,
"includePromotedContent": false,
}
features := map[string]interface{}{
"responsive_web_graphql_exclude_directive_enabled": true,
"verified_phone_label_enabled": false,
"creator_subscriptions_tweet_preview_api_enabled": true,
"responsive_web_graphql_timeline_navigation_enabled": true,
"responsive_web_graphql_skip_user_profile_image_extensions_enabled": false,
"c9s_tweet_anatomy_moderator_badge_enabled": true,
"tweetypie_unmention_optimization_enabled": true,
"responsive_web_edit_tweet_api_enabled": true,
"graphql_is_translatable_rweb_tweet_is_translatable_enabled": true,
"view_counts_everywhere_api_enabled": true,
"longform_notetweets_consumption_enabled": true,
"responsive_web_twitter_article_tweet_consumption_enabled": true,
"tweet_awards_web_tipping_enabled": false,
"freedom_of_speech_not_reach_fetch_enabled": true,
"standardized_nudges_misinfo": true,
"tweet_with_visibility_results_prefer_gql_limited_actions_policy_enabled": true,
"rweb_video_timestamps_enabled": true,
"longform_notetweets_rich_text_read_enabled": true,
"longform_notetweets_inline_media_enabled": true,
"responsive_web_media_download_video_enabled": false,
"responsive_web_enhance_cards_enabled": false,
}
if cursor != "" {
variables["cursor"] = cursor
}
query := url.Values{}
query.Set("variables", mapToJSONString(variables))
query.Set("features", mapToJSONString(features))
req.URL.RawQuery = query.Encode()
var timeline timelineV2
err = s.RequestAPI(req, &timeline)
if err != nil {
return nil, "", err
}
users, nextCursor := timeline.parseUsers()
return users, nextCursor, nil
}

2
search.go
View file

@ -62,7 +62,7 @@ func (timeline *searchTimeline) parseUsers() ([]*Profile, string) {
}
for _, entry := range instruction.Entries {
if entry.Content.ItemContent.UserDisplayType == "User" {
if profile := parseProfile(entry.Content.ItemContent.UserResults.Result.Legacy); profile.Name != "" {
if profile := parseProfileV2(entry.Content.ItemContent.UserResults.Result); profile.Name != "" {
if profile.UserID == "" {
profile.UserID = entry.Content.ItemContent.UserResults.Result.RestID
}

48
timeline_v2.go
View file

@ -47,6 +47,21 @@ func (result *result) parse() *Tweet {
return tw
}
type userResult struct {
Typename string `json:"__typename"`
ID string `json:"id"`
RestID string `json:"rest_id"`
AffiliatesHighlightedLabel struct{} `json:"affiliates_highlighted_label"`
HasGraduatedAccess bool `json:"has_graduated_access"`
IsBlueVerified bool `json:"is_blue_verified"`
ProfileImageShape string `json:"profile_image_shape"`
Legacy legacyUserV2 `json:"legacy"`
}
func (result *userResult) parse() Profile {
return parseProfileV2(*result)
}
type entry struct {
Content struct {
CursorType string `json:"cursorType"`
@ -68,10 +83,7 @@ type entry struct {
} `json:"tweet_results"`
UserDisplayType string `json:"userDisplayType"`
UserResults struct {
Result struct {
RestID string `json:"rest_id"`
Legacy legacyUser `json:"legacy"`
} `json:"result"`
Result userResult `json:"result"`
} `json:"user_results"`
} `json:"itemContent"`
} `json:"content"`
@ -91,6 +103,16 @@ type timelineV2 struct {
} `json:"instructions"`
} `json:"timeline"`
} `json:"timeline_v2"`
Timeline struct {
Timeline struct {
Instructions []struct {
Entries []entry `json:"entries"`
Entry entry `json:"entry"`
Type string `json:"type"`
} `json:"instructions"`
} `json:"timeline"`
} `json:"timeline"`
} `json:"result"`
} `json:"user"`
} `json:"data"`
@ -115,6 +137,24 @@ func (timeline *timelineV2) parseTweets() ([]*Tweet, string) {
return tweets, cursor
}
func (timeline *timelineV2) parseUsers() ([]*Profile, string) {
var cursor string
var users []*Profile
for _, instruction := range timeline.Data.User.Result.Timeline.Timeline.Instructions {
for _, entry := range instruction.Entries {
if entry.Content.CursorType == "Bottom" {
cursor = entry.Content.Value
continue
}
if entry.Content.ItemContent.UserResults.Result.Typename == "User" {
user := entry.Content.ItemContent.UserResults.Result.parse()
users = append(users, &user)
}
}
}
return users, cursor
}
type threadedConversation struct {
Data struct {
ThreadedConversationWithInjectionsV2 struct {

46
types.go
View file

@ -170,6 +170,52 @@ type (
Verified bool `json:"verified"`
}
legacyUserV2 struct {
Following bool `json:"following"`
CanDm bool `json:"can_dm"`
CanMediaTag bool `json:"can_media_tag"`
CreatedAt string `json:"created_at"`
DefaultProfile bool `json:"default_profile"`
DefaultProfileImage bool `json:"default_profile_image"`
Description string `json:"description"`
Entities struct {
Description struct {
Urls []interface{} `json:"urls"`
} `json:"description"`
URL struct {
Urls []struct {
DisplayURL string `json:"display_url"`
ExpandedURL string `json:"expanded_url"`
URL string `json:"url"`
Indices []int `json:"indices"`
} `json:"urls"`
} `json:"url"`
} `json:"entities"`
FastFollowersCount int `json:"fast_followers_count"`
FavouritesCount int `json:"favourites_count"`
FollowersCount int `json:"followers_count"`
FriendsCount int `json:"friends_count"`
HasCustomTimelines bool `json:"has_custom_timelines"`
IsTranslator bool `json:"is_translator"`
ListedCount int `json:"listed_count"`
Location string `json:"location"`
MediaCount int `json:"media_count"`
Name string `json:"name"`
NormalFollowersCount int `json:"normal_followers_count"`
PinnedTweetIdsStr []string `json:"pinned_tweet_ids_str"`
PossiblySensitive bool `json:"possibly_sensitive"`
ProfileBannerURL string `json:"profile_banner_url"`
ProfileImageURLHTTPS string `json:"profile_image_url_https"`
ProfileInterstitialType string `json:"profile_interstitial_type"`
ScreenName string `json:"screen_name"`
StatusesCount int `json:"statuses_count"`
TranslatorType string `json:"translator_type"`
URL string `json:"url"`
Verified bool `json:"verified"`
WantRetweets bool `json:"want_retweets"`
WithheldInCountries []interface{} `json:"withheld_in_countries"`
}
Place struct {
ID string `json:"id"`
PlaceType string `json:"place_type"`

35
util.go
View file

@ -342,7 +342,6 @@ func parseProfile(user legacyUser) Profile {
FollowersCount: user.FollowersCount,
FollowingCount: user.FavouritesCount,
FriendsCount: user.FriendsCount,
IsPrivate: user.Protected,
IsVerified: user.Verified,
LikesCount: user.FavouritesCount,
ListedCount: user.ListedCount,
@ -368,6 +367,40 @@ func parseProfile(user legacyUser) Profile {
return profile
}
func parseProfileV2(user userResult) Profile {
u := user.Legacy
profile := Profile{
Avatar: u.ProfileImageURLHTTPS,
Banner: u.ProfileBannerURL,
Biography: u.Description,
FollowersCount: u.FollowersCount,
FollowingCount: u.FavouritesCount,
FriendsCount: u.FriendsCount,
IsVerified: u.Verified,
LikesCount: u.FavouritesCount,
ListedCount: u.ListedCount,
Location: u.Location,
Name: u.Name,
PinnedTweetIDs: u.PinnedTweetIdsStr,
TweetsCount: u.StatusesCount,
URL: "https://twitter.com/" + u.ScreenName,
UserID: user.ID,
Username: u.ScreenName,
}
tm, err := time.Parse(time.RubyDate, u.CreatedAt)
if err == nil {
tm = tm.UTC()
profile.Joined = &tm
}
if len(u.Entities.URL.Urls) > 0 {
profile.Website = u.Entities.URL.Urls[0].ExpandedURL
}
return profile
}
func mapToJSONString(data map[string]interface{}) string {
jsonBytes, err := json.Marshal(data)
if err != nil {