From 8bb70a9d2c442c87a42dd51cd82ce185cf4cf6c9 Mon Sep 17 00:00:00 2001 From: thewh1teagle <61390950+thewh1teagle@users.noreply.github.com> Date: Sat, 23 Nov 2024 16:38:28 +0200 Subject: [PATCH 1/6] fix: change bearer token for authentication that is less restrictive --- auth.go | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/auth.go b/auth.go index 2d143c3..9437d7b 100644 --- a/auth.go +++ b/auth.go @@ -19,9 +19,12 @@ import ( ) const ( - loginURL = "https://api.twitter.com/1.1/onboarding/task.json" - logoutURL = "https://api.twitter.com/1.1/account/logout.json" - oAuthURL = "https://api.twitter.com/oauth2/token" + loginURL = "https://api.twitter.com/1.1/onboarding/task.json" + logoutURL = "https://api.twitter.com/1.1/account/logout.json" + oAuthURL = "https://api.twitter.com/oauth2/token" + // Doesn't require x-client-transaction-id header + bearerToken1 = "AAAAAAAAAAAAAAAAAAAAAG5LOQEAAAAAbEKsIYYIhrfOQqm4H8u7xcahRkU%3Dz98HKmzbeXdKqBfUDmElcqYl0cmmKY9KdS2UoNIz3Phapgsowi" + // Requires x-client-transaction-id header bearerToken2 = "AAAAAAAAAAAAAAAAAAAAANRILgAAAAAAnNwIzUejRCOuH5E6I8xnZz4puTs%3D1Zv7ttfk8LF81IUq16cHjhLTvJu4FA33AGWWjCpTnA" appConsumerKey = "3nVuSoBZnx6U4vzUxf5w" appConsumerSecret = "Bcs59EFbbsdF6Sl9Ng71smgStWEGwXXKSjYvPVt7qys" @@ -147,7 +150,7 @@ func (s *Scraper) getFlowToken(data map[string]interface{}) (string, error) { // IsLoggedIn check if scraper logged in func (s *Scraper) IsLoggedIn() bool { s.isLogged = true - s.setBearerToken(bearerToken2) + s.setBearerToken(bearerToken1) req, err := http.NewRequest("GET", "https://api.twitter.com/1.1/account/verify_credentials.json", nil) if err != nil { return false From 3f34db1aa9c1a0fdce5f5e7689239d84da06e1ee Mon Sep 17 00:00:00 2001 From: thewh1teagle <61390950+thewh1teagle@users.noreply.github.com> Date: Sun, 24 Nov 2024 00:36:29 +0200 Subject: [PATCH 2/6] feat: use bearer token with higher rate limit --- auth.go | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/auth.go b/auth.go index 9437d7b..d0a1727 100644 --- a/auth.go +++ b/auth.go @@ -22,9 +22,9 @@ const ( loginURL = "https://api.twitter.com/1.1/onboarding/task.json" logoutURL = "https://api.twitter.com/1.1/account/logout.json" oAuthURL = "https://api.twitter.com/oauth2/token" - // Doesn't require x-client-transaction-id header - bearerToken1 = "AAAAAAAAAAAAAAAAAAAAAG5LOQEAAAAAbEKsIYYIhrfOQqm4H8u7xcahRkU%3Dz98HKmzbeXdKqBfUDmElcqYl0cmmKY9KdS2UoNIz3Phapgsowi" - // Requires x-client-transaction-id header + // Doesn't require x-client-transaction-id header in auth. x-rate-limit-limit: 2000 + bearerToken1 = "AAAAAAAAAAAAAAAAAAAAAFQODgEAAAAAVHTp76lzh3rFzcHbmHVvQxYYpTw%3DckAlMINMjmCwxUcaXbAN4XqJVdgMJaHqNOFgPMK0zN1qLqLQCF" + // Requires x-client-transaction-id header in auth. bearerToken2 = "AAAAAAAAAAAAAAAAAAAAANRILgAAAAAAnNwIzUejRCOuH5E6I8xnZz4puTs%3D1Zv7ttfk8LF81IUq16cHjhLTvJu4FA33AGWWjCpTnA" appConsumerKey = "3nVuSoBZnx6U4vzUxf5w" appConsumerSecret = "Bcs59EFbbsdF6Sl9Ng71smgStWEGwXXKSjYvPVt7qys" From 7226460e051d65f8a80db72c2e2f5e8abb05c7db Mon Sep 17 00:00:00 2001 From: Brendan Playford <34052452+teslashibe@users.noreply.github.com> Date: Wed, 22 Jan 2025 11:05:38 -0800 Subject: [PATCH 3/6] feat(types): extend Twitter profile data structures - Add new fields to Profile struct (IsBlueVerified, MediaCount, etc.) - Extend legacyUser struct with additional Twitter API fields - Add new types for extended profile, verification and highlights info - Update profile parsing to include new metrics --- profile.go | 53 ++++++++++++++++++--------------- timeline_v2.go | 23 ++++++++++----- types.go | 80 ++++++++++++++++++++++++++++++++++++++++---------- util.go | 41 ++++++++++++++------------ 4 files changed, 131 insertions(+), 66 deletions(-) diff --git a/profile.go b/profile.go index 3fc0661..bba0ee1 100644 --- a/profile.go +++ b/profile.go @@ -14,29 +14,36 @@ var cacheIDs sync.Map // Profile of twitter user. type Profile struct { - Avatar string - Banner string - Biography string - Birthday string - FollowersCount int - FollowingCount int - FriendsCount int - IsPrivate bool - IsVerified bool - Joined *time.Time - LikesCount int - ListedCount int - Location string - Name string - PinnedTweetIDs []string - TweetsCount int - URL string - UserID string - Username string - Website string - Sensitive bool - Following bool - FollowedBy bool + Avatar string + Banner string + Biography string + Birthday string + FollowersCount int + FollowingCount int + FriendsCount int + IsPrivate bool + IsVerified bool + IsBlueVerified bool + Joined *time.Time + LikesCount int + ListedCount int + Location string + Name string + PinnedTweetIDs []string + TweetsCount int + URL string + UserID string + Username string + Website string + Sensitive bool + Following bool + FollowedBy bool + MediaCount int + FastFollowersCount int + NormalFollowersCount int + ProfileImageShape string + HasGraduatedAccess bool + CanHighlightTweets bool } type user struct { diff --git a/timeline_v2.go b/timeline_v2.go index 2d7f10e..372941d 100644 --- a/timeline_v2.go +++ b/timeline_v2.go @@ -60,14 +60,21 @@ func (result *result) parse() *Tweet { } type userResult struct { - Typename string `json:"__typename"` - ID string `json:"id"` - RestID string `json:"rest_id"` - AffiliatesHighlightedLabel struct{} `json:"affiliates_highlighted_label"` - HasGraduatedAccess bool `json:"has_graduated_access"` - IsBlueVerified bool `json:"is_blue_verified"` - ProfileImageShape string `json:"profile_image_shape"` - Legacy legacyUserV2 `json:"legacy"` + Typename string `json:"__typename"` + ID string `json:"id"` + RestID string `json:"rest_id"` + AffiliatesHighlightedLabel struct{} `json:"affiliates_highlighted_label"` + HasGraduatedAccess bool `json:"has_graduated_access"` + IsBlueVerified bool `json:"is_blue_verified"` + ProfileImageShape string `json:"profile_image_shape"` + Legacy legacyUserV2 `json:"legacy"` + LegacyExtendedProfile legacyExtendedProfile `json:"legacy_extended_profile"` + IsProfileTranslatable bool `json:"is_profile_translatable"` + VerificationInfo verificationInfo `json:"verification_info"` + HighlightsInfo highlightsInfo `json:"highlights_info"` + UserSeedTweetCount int `json:"user_seed_tweet_count"` + PremiumGiftingEligible bool `json:"premium_gifting_eligible"` + CreatorSubscriptionsCount int `json:"creator_subscriptions_count"` } func (result *userResult) parse() Profile { diff --git a/types.go b/types.go index 35fccd5..a948b39 100644 --- a/types.go +++ b/types.go @@ -166,22 +166,37 @@ type ( } `json:"urls"` } `json:"url"` } `json:"entities"` - FavouritesCount int `json:"favourites_count"` - FollowersCount int `json:"followers_count"` - FriendsCount int `json:"friends_count"` - IDStr string `json:"id_str"` - ListedCount int `json:"listed_count"` - Name string `json:"name"` - Location string `json:"location"` - PinnedTweetIdsStr []string `json:"pinned_tweet_ids_str"` - ProfileBannerURL string `json:"profile_banner_url"` - ProfileImageURLHTTPS string `json:"profile_image_url_https"` - Protected bool `json:"protected"` - ScreenName string `json:"screen_name"` - StatusesCount int `json:"statuses_count"` - Verified bool `json:"verified"` - FollowedBy bool `json:"followed_by"` - Following bool `json:"following"` + FavouritesCount int `json:"favourites_count"` + FollowersCount int `json:"followers_count"` + FriendsCount int `json:"friends_count"` + IDStr string `json:"id_str"` + ListedCount int `json:"listed_count"` + Name string `json:"name"` + Location string `json:"location"` + PinnedTweetIdsStr []string `json:"pinned_tweet_ids_str"` + ProfileBannerURL string `json:"profile_banner_url"` + ProfileImageURLHTTPS string `json:"profile_image_url_https"` + Protected bool `json:"protected"` + ScreenName string `json:"screen_name"` + StatusesCount int `json:"statuses_count"` + Verified bool `json:"verified"` + FollowedBy bool `json:"followed_by"` + Following bool `json:"following"` + CanDm bool `json:"can_dm"` + CanMediaTag bool `json:"can_media_tag"` + DefaultProfile bool `json:"default_profile"` + DefaultProfileImage bool `json:"default_profile_image"` + FastFollowersCount int `json:"fast_followers_count"` + HasCustomTimelines bool `json:"has_custom_timelines"` + IsTranslator bool `json:"is_translator"` + MediaCount int `json:"media_count"` + NeedsPhoneVerification bool `json:"needs_phone_verification"` + NormalFollowersCount int `json:"normal_followers_count"` + PossiblySensitive bool `json:"possibly_sensitive"` + ProfileInterstitialType string `json:"profile_interstitial_type"` + TranslatorType string `json:"translator_type"` + WantRetweets bool `json:"want_retweets"` + WithheldInCountries []string `json:"withheld_in_countries"` } legacyUserV2 struct { @@ -246,4 +261,37 @@ type ( fetchProfileFunc func(query string, maxProfilesNbr int, cursor string) ([]*Profile, string, error) fetchTweetFunc func(query string, maxTweetsNbr int, cursor string) ([]*Tweet, string, error) + + legacyExtendedProfile struct { + Birthdate struct { + Day int `json:"day"` + Month int `json:"month"` + Year int `json:"year"` + Visibility string `json:"visibility"` + YearVisibility string `json:"year_visibility"` + } `json:"birthdate"` + } + + verificationInfo struct { + IsIdentityVerified bool `json:"is_identity_verified"` + Reason struct { + Description struct { + Text string `json:"text"` + Entities []struct { + FromIndex int `json:"from_index"` + ToIndex int `json:"to_index"` + Ref struct { + URL string `json:"url"` + URLType string `json:"url_type"` + } `json:"ref"` + } `json:"entities"` + } `json:"description"` + VerifiedSinceMsec string `json:"verified_since_msec"` + } `json:"reason"` + } + + highlightsInfo struct { + CanHighlightTweets bool `json:"can_highlight_tweets"` + HighlightedTweets string `json:"highlighted_tweets"` + } ) diff --git a/util.go b/util.go index 0a2f658..c6a4fb8 100644 --- a/util.go +++ b/util.go @@ -345,25 +345,28 @@ func parseLegacyTweet(user *legacyUser, tweet *legacyTweet) *Tweet { func parseProfile(user legacyUser) Profile { profile := Profile{ - Avatar: user.ProfileImageURLHTTPS, - Banner: user.ProfileBannerURL, - Biography: user.Description, - FollowersCount: user.FollowersCount, - FollowingCount: user.FavouritesCount, - FriendsCount: user.FriendsCount, - IsVerified: user.Verified, - IsPrivate: user.Protected, - LikesCount: user.FavouritesCount, - ListedCount: user.ListedCount, - Location: user.Location, - Name: user.Name, - PinnedTweetIDs: user.PinnedTweetIdsStr, - TweetsCount: user.StatusesCount, - URL: "https://twitter.com/" + user.ScreenName, - UserID: user.IDStr, - Username: user.ScreenName, - FollowedBy: user.FollowedBy, - Following: user.Following, + Avatar: user.ProfileImageURLHTTPS, + Banner: user.ProfileBannerURL, + Biography: user.Description, + FollowersCount: user.FollowersCount, + FollowingCount: user.FavouritesCount, + FriendsCount: user.FriendsCount, + IsVerified: user.Verified, + IsPrivate: user.Protected, + LikesCount: user.FavouritesCount, + ListedCount: user.ListedCount, + Location: user.Location, + Name: user.Name, + PinnedTweetIDs: user.PinnedTweetIdsStr, + TweetsCount: user.StatusesCount, + URL: "https://twitter.com/" + user.ScreenName, + UserID: user.IDStr, + Username: user.ScreenName, + FollowedBy: user.FollowedBy, + Following: user.Following, + MediaCount: user.MediaCount, + FastFollowersCount: user.FastFollowersCount, + NormalFollowersCount: user.NormalFollowersCount, } tm, err := time.Parse(time.RubyDate, user.CreatedAt) From 388445d4c2e2d5efa46922772a40dfd13a9995c5 Mon Sep 17 00:00:00 2001 From: Brendan Playford <34052452+teslashibe@users.noreply.github.com> Date: Wed, 22 Jan 2025 11:14:03 -0800 Subject: [PATCH 4/6] feat: add IsBlueVerified to profile responses Add support for Twitter's blue verification status in profile responses: - Add IsBlueVerified field to user struct in GetProfile response - Update parseProfile and parseProfileV2 to include blue verification status - Add ProfileImageShape and HasGraduatedAccess fields to profile responses This change ensures the API correctly returns blue verification status for both direct profile queries and timeline responses. --- profile.go | 15 ++++++++++----- util.go | 41 ++++++++++++++++++++++------------------- 2 files changed, 32 insertions(+), 24 deletions(-) diff --git a/profile.go b/profile.go index bba0ee1..e79a92c 100644 --- a/profile.go +++ b/profile.go @@ -50,9 +50,10 @@ type user struct { Data struct { User struct { Result struct { - RestID string `json:"rest_id"` - Legacy legacyUser `json:"legacy"` - Message string `json:"message"` + RestID string `json:"rest_id"` + Legacy legacyUser `json:"legacy"` + Message string `json:"message"` + IsBlueVerified bool `json:"is_blue_verified"` } `json:"result"` } `json:"user"` } `json:"data"` @@ -118,7 +119,9 @@ func (s *Scraper) GetProfile(username string) (Profile, error) { return Profile{}, fmt.Errorf("either @%s does not exist or is private", username) } - return parseProfile(jsn.Data.User.Result.Legacy), nil + profile := parseProfile(jsn.Data.User.Result.Legacy) + profile.IsBlueVerified = jsn.Data.User.Result.IsBlueVerified + return profile, nil } func (s *Scraper) GetProfileByID(userID string) (Profile, error) { @@ -175,7 +178,9 @@ func (s *Scraper) GetProfileByID(userID string) (Profile, error) { return Profile{}, fmt.Errorf("either @%s does not exist or is private", userID) } - return parseProfile(jsn.Data.User.Result.Legacy), nil + profile := parseProfile(jsn.Data.User.Result.Legacy) + profile.IsBlueVerified = jsn.Data.User.Result.IsBlueVerified + return profile, nil } // GetUserIDByScreenName from API diff --git a/util.go b/util.go index c6a4fb8..26264e0 100644 --- a/util.go +++ b/util.go @@ -385,25 +385,28 @@ func parseProfile(user legacyUser) Profile { func parseProfileV2(user userResult) Profile { u := user.Legacy profile := Profile{ - Avatar: u.ProfileImageURLHTTPS, - Banner: u.ProfileBannerURL, - Biography: u.Description, - FollowersCount: u.FollowersCount, - FollowingCount: u.FavouritesCount, - FriendsCount: u.FriendsCount, - IsVerified: u.Verified, - LikesCount: u.FavouritesCount, - ListedCount: u.ListedCount, - Location: u.Location, - Name: u.Name, - PinnedTweetIDs: u.PinnedTweetIdsStr, - TweetsCount: u.StatusesCount, - URL: "https://twitter.com/" + u.ScreenName, - UserID: user.ID, - Username: u.ScreenName, - Sensitive: u.PossiblySensitive, - Following: u.Following, - FollowedBy: u.FollowedBy, + Avatar: u.ProfileImageURLHTTPS, + Banner: u.ProfileBannerURL, + Biography: u.Description, + FollowersCount: u.FollowersCount, + FollowingCount: u.FavouritesCount, + FriendsCount: u.FriendsCount, + IsVerified: u.Verified, + IsBlueVerified: user.IsBlueVerified, + ProfileImageShape: user.ProfileImageShape, + HasGraduatedAccess: user.HasGraduatedAccess, + LikesCount: u.FavouritesCount, + ListedCount: u.ListedCount, + Location: u.Location, + Name: u.Name, + PinnedTweetIDs: u.PinnedTweetIdsStr, + TweetsCount: u.StatusesCount, + URL: "https://twitter.com/" + u.ScreenName, + UserID: user.ID, + Username: u.ScreenName, + Sensitive: u.PossiblySensitive, + Following: u.Following, + FollowedBy: u.FollowedBy, } tm, err := time.Parse(time.RubyDate, u.CreatedAt) From 4276e243ac1c4631354f26800aceccebb97ccf86 Mon Sep 17 00:00:00 2001 From: Valentine Date: Wed, 22 Jan 2025 23:57:00 +0300 Subject: [PATCH 5/6] fix test fail --- profile_test.go | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/profile_test.go b/profile_test.go index eb82891..6b7b940 100644 --- a/profile_test.go +++ b/profile_test.go @@ -42,6 +42,8 @@ func TestGetProfile(t *testing.T) { cmpopts.IgnoreFields(twitterscraper.Profile{}, "LikesCount"), cmpopts.IgnoreFields(twitterscraper.Profile{}, "ListedCount"), cmpopts.IgnoreFields(twitterscraper.Profile{}, "TweetsCount"), + cmpopts.IgnoreFields(twitterscraper.Profile{}, "MediaCount"), + cmpopts.IgnoreFields(twitterscraper.Profile{}, "NormalFollowersCount"), } if diff := cmp.Diff(sample, profile, cmpOptions...); diff != "" { t.Error("Resulting profile does not match the sample", diff) @@ -94,6 +96,8 @@ func TestGetProfilePrivate(t *testing.T) { cmpopts.IgnoreFields(twitterscraper.Profile{}, "LikesCount"), cmpopts.IgnoreFields(twitterscraper.Profile{}, "ListedCount"), cmpopts.IgnoreFields(twitterscraper.Profile{}, "TweetsCount"), + cmpopts.IgnoreFields(twitterscraper.Profile{}, "MediaCount"), + cmpopts.IgnoreFields(twitterscraper.Profile{}, "NormalFollowersCount"), } if diff := cmp.Diff(sample, profile, cmpOptions...); diff != "" { t.Error("Resulting profile does not match the sample", diff) From 44183226af2de76c0e90890ecaa89f6a104070cf Mon Sep 17 00:00:00 2001 From: Valentine Date: Thu, 23 Jan 2025 00:00:32 +0300 Subject: [PATCH 6/6] update twitter account in test --- profile_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/profile_test.go b/profile_test.go index 6b7b940..7e9a766 100644 --- a/profile_test.go +++ b/profile_test.go @@ -150,7 +150,7 @@ func TestGetProfileByID(t *testing.T) { } func TestGetUserIDByScreenName(t *testing.T) { - userID, err := testScraper.GetUserIDByScreenName("Twitter") + userID, err := testScraper.GetUserIDByScreenName("X") if err != nil { t.Errorf("getUserByScreenName() error = %v", err) }