diff --git a/auth.go b/auth.go index 2d143c3..d0a1727 100644 --- a/auth.go +++ b/auth.go @@ -19,9 +19,12 @@ import ( ) const ( - loginURL = "https://api.twitter.com/1.1/onboarding/task.json" - logoutURL = "https://api.twitter.com/1.1/account/logout.json" - oAuthURL = "https://api.twitter.com/oauth2/token" + loginURL = "https://api.twitter.com/1.1/onboarding/task.json" + logoutURL = "https://api.twitter.com/1.1/account/logout.json" + oAuthURL = "https://api.twitter.com/oauth2/token" + // Doesn't require x-client-transaction-id header in auth. x-rate-limit-limit: 2000 + bearerToken1 = "AAAAAAAAAAAAAAAAAAAAAFQODgEAAAAAVHTp76lzh3rFzcHbmHVvQxYYpTw%3DckAlMINMjmCwxUcaXbAN4XqJVdgMJaHqNOFgPMK0zN1qLqLQCF" + // Requires x-client-transaction-id header in auth. bearerToken2 = "AAAAAAAAAAAAAAAAAAAAANRILgAAAAAAnNwIzUejRCOuH5E6I8xnZz4puTs%3D1Zv7ttfk8LF81IUq16cHjhLTvJu4FA33AGWWjCpTnA" appConsumerKey = "3nVuSoBZnx6U4vzUxf5w" appConsumerSecret = "Bcs59EFbbsdF6Sl9Ng71smgStWEGwXXKSjYvPVt7qys" @@ -147,7 +150,7 @@ func (s *Scraper) getFlowToken(data map[string]interface{}) (string, error) { // IsLoggedIn check if scraper logged in func (s *Scraper) IsLoggedIn() bool { s.isLogged = true - s.setBearerToken(bearerToken2) + s.setBearerToken(bearerToken1) req, err := http.NewRequest("GET", "https://api.twitter.com/1.1/account/verify_credentials.json", nil) if err != nil { return false diff --git a/profile.go b/profile.go index 3fc0661..e79a92c 100644 --- a/profile.go +++ b/profile.go @@ -14,38 +14,46 @@ var cacheIDs sync.Map // Profile of twitter user. type Profile struct { - Avatar string - Banner string - Biography string - Birthday string - FollowersCount int - FollowingCount int - FriendsCount int - IsPrivate bool - IsVerified bool - Joined *time.Time - LikesCount int - ListedCount int - Location string - Name string - PinnedTweetIDs []string - TweetsCount int - URL string - UserID string - Username string - Website string - Sensitive bool - Following bool - FollowedBy bool + Avatar string + Banner string + Biography string + Birthday string + FollowersCount int + FollowingCount int + FriendsCount int + IsPrivate bool + IsVerified bool + IsBlueVerified bool + Joined *time.Time + LikesCount int + ListedCount int + Location string + Name string + PinnedTweetIDs []string + TweetsCount int + URL string + UserID string + Username string + Website string + Sensitive bool + Following bool + FollowedBy bool + MediaCount int + FastFollowersCount int + NormalFollowersCount int + ProfileImageShape string + HasGraduatedAccess bool + CanHighlightTweets bool } type user struct { Data struct { User struct { Result struct { - RestID string `json:"rest_id"` - Legacy legacyUser `json:"legacy"` - Message string `json:"message"` + RestID string `json:"rest_id"` + Legacy legacyUser `json:"legacy"` + Message string `json:"message"` + IsBlueVerified bool `json:"is_blue_verified"` } `json:"result"` } `json:"user"` } `json:"data"` @@ -111,7 +119,9 @@ func (s *Scraper) GetProfile(username string) (Profile, error) { return Profile{}, fmt.Errorf("either @%s does not exist or is private", username) } - return parseProfile(jsn.Data.User.Result.Legacy), nil + profile := parseProfile(jsn.Data.User.Result.Legacy) + profile.IsBlueVerified = jsn.Data.User.Result.IsBlueVerified + return profile, nil } func (s *Scraper) GetProfileByID(userID string) (Profile, error) { @@ -168,7 +178,9 @@ func (s *Scraper) GetProfileByID(userID string) (Profile, error) { return Profile{}, fmt.Errorf("either @%s does not exist or is private", userID) } - return parseProfile(jsn.Data.User.Result.Legacy), nil + profile := parseProfile(jsn.Data.User.Result.Legacy) + profile.IsBlueVerified = jsn.Data.User.Result.IsBlueVerified + return profile, nil } // GetUserIDByScreenName from API diff --git a/profile_test.go b/profile_test.go index eb82891..7e9a766 100644 --- a/profile_test.go +++ b/profile_test.go @@ -42,6 +42,8 @@ func TestGetProfile(t *testing.T) { cmpopts.IgnoreFields(twitterscraper.Profile{}, "LikesCount"), cmpopts.IgnoreFields(twitterscraper.Profile{}, "ListedCount"), cmpopts.IgnoreFields(twitterscraper.Profile{}, "TweetsCount"), + cmpopts.IgnoreFields(twitterscraper.Profile{}, "MediaCount"), + cmpopts.IgnoreFields(twitterscraper.Profile{}, "NormalFollowersCount"), } if diff := cmp.Diff(sample, profile, cmpOptions...); diff != "" { t.Error("Resulting profile does not match the sample", diff) @@ -94,6 +96,8 @@ func TestGetProfilePrivate(t *testing.T) { cmpopts.IgnoreFields(twitterscraper.Profile{}, "LikesCount"), cmpopts.IgnoreFields(twitterscraper.Profile{}, "ListedCount"), cmpopts.IgnoreFields(twitterscraper.Profile{}, "TweetsCount"), + cmpopts.IgnoreFields(twitterscraper.Profile{}, "MediaCount"), + cmpopts.IgnoreFields(twitterscraper.Profile{}, "NormalFollowersCount"), } if diff := cmp.Diff(sample, profile, cmpOptions...); diff != "" { t.Error("Resulting profile does not match the sample", diff) @@ -146,7 +150,7 @@ func TestGetProfileByID(t *testing.T) { } func TestGetUserIDByScreenName(t *testing.T) { - userID, err := testScraper.GetUserIDByScreenName("Twitter") + userID, err := testScraper.GetUserIDByScreenName("X") if err != nil { t.Errorf("getUserByScreenName() error = %v", err) } diff --git a/timeline_v2.go b/timeline_v2.go index d6fc82e..3246b03 100644 --- a/timeline_v2.go +++ b/timeline_v2.go @@ -62,14 +62,21 @@ func (result *result) parse() *Tweet { } type userResult struct { - Typename string `json:"__typename"` - ID string `json:"id"` - RestID string `json:"rest_id"` - AffiliatesHighlightedLabel struct{} `json:"affiliates_highlighted_label"` - HasGraduatedAccess bool `json:"has_graduated_access"` - IsBlueVerified bool `json:"is_blue_verified"` - ProfileImageShape string `json:"profile_image_shape"` - Legacy legacyUserV2 `json:"legacy"` + Typename string `json:"__typename"` + ID string `json:"id"` + RestID string `json:"rest_id"` + AffiliatesHighlightedLabel struct{} `json:"affiliates_highlighted_label"` + HasGraduatedAccess bool `json:"has_graduated_access"` + IsBlueVerified bool `json:"is_blue_verified"` + ProfileImageShape string `json:"profile_image_shape"` + Legacy legacyUserV2 `json:"legacy"` + LegacyExtendedProfile legacyExtendedProfile `json:"legacy_extended_profile"` + IsProfileTranslatable bool `json:"is_profile_translatable"` + VerificationInfo verificationInfo `json:"verification_info"` + HighlightsInfo highlightsInfo `json:"highlights_info"` + UserSeedTweetCount int `json:"user_seed_tweet_count"` + PremiumGiftingEligible bool `json:"premium_gifting_eligible"` + CreatorSubscriptionsCount int `json:"creator_subscriptions_count"` } func (result *userResult) parse() Profile { diff --git a/types.go b/types.go index f18f109..7bd9437 100644 --- a/types.go +++ b/types.go @@ -173,22 +173,37 @@ type ( } `json:"urls"` } `json:"url"` } `json:"entities"` - FavouritesCount int `json:"favourites_count"` - FollowersCount int `json:"followers_count"` - FriendsCount int `json:"friends_count"` - IDStr string `json:"id_str"` - ListedCount int `json:"listed_count"` - Name string `json:"name"` - Location string `json:"location"` - PinnedTweetIdsStr []string `json:"pinned_tweet_ids_str"` - ProfileBannerURL string `json:"profile_banner_url"` - ProfileImageURLHTTPS string `json:"profile_image_url_https"` - Protected bool `json:"protected"` - ScreenName string `json:"screen_name"` - StatusesCount int `json:"statuses_count"` - Verified bool `json:"verified"` - FollowedBy bool `json:"followed_by"` - Following bool `json:"following"` + FavouritesCount int `json:"favourites_count"` + FollowersCount int `json:"followers_count"` + FriendsCount int `json:"friends_count"` + IDStr string `json:"id_str"` + ListedCount int `json:"listed_count"` + Name string `json:"name"` + Location string `json:"location"` + PinnedTweetIdsStr []string `json:"pinned_tweet_ids_str"` + ProfileBannerURL string `json:"profile_banner_url"` + ProfileImageURLHTTPS string `json:"profile_image_url_https"` + Protected bool `json:"protected"` + ScreenName string `json:"screen_name"` + StatusesCount int `json:"statuses_count"` + Verified bool `json:"verified"` + FollowedBy bool `json:"followed_by"` + Following bool `json:"following"` + CanDm bool `json:"can_dm"` + CanMediaTag bool `json:"can_media_tag"` + DefaultProfile bool `json:"default_profile"` + DefaultProfileImage bool `json:"default_profile_image"` + FastFollowersCount int `json:"fast_followers_count"` + HasCustomTimelines bool `json:"has_custom_timelines"` + IsTranslator bool `json:"is_translator"` + MediaCount int `json:"media_count"` + NeedsPhoneVerification bool `json:"needs_phone_verification"` + NormalFollowersCount int `json:"normal_followers_count"` + PossiblySensitive bool `json:"possibly_sensitive"` + ProfileInterstitialType string `json:"profile_interstitial_type"` + TranslatorType string `json:"translator_type"` + WantRetweets bool `json:"want_retweets"` + WithheldInCountries []string `json:"withheld_in_countries"` } legacyUserV2 struct { @@ -248,4 +263,37 @@ type ( fetchProfileFunc func(query string, maxProfilesNbr int, cursor string) ([]*Profile, string, error) fetchTweetFunc func(query string, maxTweetsNbr int, cursor string) ([]*Tweet, string, error) + + legacyExtendedProfile struct { + Birthdate struct { + Day int `json:"day"` + Month int `json:"month"` + Year int `json:"year"` + Visibility string `json:"visibility"` + YearVisibility string `json:"year_visibility"` + } `json:"birthdate"` + } + + verificationInfo struct { + IsIdentityVerified bool `json:"is_identity_verified"` + Reason struct { + Description struct { + Text string `json:"text"` + Entities []struct { + FromIndex int `json:"from_index"` + ToIndex int `json:"to_index"` + Ref struct { + URL string `json:"url"` + URLType string `json:"url_type"` + } `json:"ref"` + } `json:"entities"` + } `json:"description"` + VerifiedSinceMsec string `json:"verified_since_msec"` + } `json:"reason"` + } + + highlightsInfo struct { + CanHighlightTweets bool `json:"can_highlight_tweets"` + HighlightedTweets string `json:"highlighted_tweets"` + } ) diff --git a/util.go b/util.go index abce237..e7e5311 100644 --- a/util.go +++ b/util.go @@ -346,25 +346,28 @@ func parseLegacyTweet(user *legacyUser, tweet *legacyTweet) *Tweet { func parseProfile(user legacyUser) Profile { profile := Profile{ - Avatar: user.ProfileImageURLHTTPS, - Banner: user.ProfileBannerURL, - Biography: user.Description, - FollowersCount: user.FollowersCount, - FollowingCount: user.FavouritesCount, - FriendsCount: user.FriendsCount, - IsVerified: user.Verified, - IsPrivate: user.Protected, - LikesCount: user.FavouritesCount, - ListedCount: user.ListedCount, - Location: user.Location, - Name: user.Name, - PinnedTweetIDs: user.PinnedTweetIdsStr, - TweetsCount: user.StatusesCount, - URL: "https://twitter.com/" + user.ScreenName, - UserID: user.IDStr, - Username: user.ScreenName, - FollowedBy: user.FollowedBy, - Following: user.Following, + Avatar: user.ProfileImageURLHTTPS, + Banner: user.ProfileBannerURL, + Biography: user.Description, + FollowersCount: user.FollowersCount, + FollowingCount: user.FavouritesCount, + FriendsCount: user.FriendsCount, + IsVerified: user.Verified, + IsPrivate: user.Protected, + LikesCount: user.FavouritesCount, + ListedCount: user.ListedCount, + Location: user.Location, + Name: user.Name, + PinnedTweetIDs: user.PinnedTweetIdsStr, + TweetsCount: user.StatusesCount, + URL: "https://twitter.com/" + user.ScreenName, + UserID: user.IDStr, + Username: user.ScreenName, + FollowedBy: user.FollowedBy, + Following: user.Following, + MediaCount: user.MediaCount, + FastFollowersCount: user.FastFollowersCount, + NormalFollowersCount: user.NormalFollowersCount, } tm, err := time.Parse(time.RubyDate, user.CreatedAt) @@ -396,25 +399,28 @@ func parseProfileV2(user userResult) Profile { u := user.Legacy description := expandURLs(u.Description, u.Entities.Description.Urls, []ExtendedMedia{}) profile := Profile{ - Avatar: u.ProfileImageURLHTTPS, - Banner: u.ProfileBannerURL, - Biography: description, - FollowersCount: u.FollowersCount, - FollowingCount: u.FavouritesCount, - FriendsCount: u.FriendsCount, - IsVerified: u.Verified, - LikesCount: u.FavouritesCount, - ListedCount: u.ListedCount, - Location: u.Location, - Name: u.Name, - PinnedTweetIDs: u.PinnedTweetIdsStr, - TweetsCount: u.StatusesCount, - URL: "https://twitter.com/" + u.ScreenName, - UserID: user.ID, - Username: u.ScreenName, - Sensitive: u.PossiblySensitive, - Following: u.Following, - FollowedBy: u.FollowedBy, + Avatar: u.ProfileImageURLHTTPS, + Banner: u.ProfileBannerURL, + Biography: description, + FollowersCount: u.FollowersCount, + FollowingCount: u.FavouritesCount, + FriendsCount: u.FriendsCount, + IsVerified: u.Verified, + IsBlueVerified: user.IsBlueVerified, + ProfileImageShape: user.ProfileImageShape, + HasGraduatedAccess: user.HasGraduatedAccess, + LikesCount: u.FavouritesCount, + ListedCount: u.ListedCount, + Location: u.Location, + Name: u.Name, + PinnedTweetIDs: u.PinnedTweetIdsStr, + TweetsCount: u.StatusesCount, + URL: "https://twitter.com/" + u.ScreenName, + UserID: user.ID, + Username: u.ScreenName, + Sensitive: u.PossiblySensitive, + Following: u.Following, + FollowedBy: u.FollowedBy, } tm, err := time.Parse(time.RubyDate, u.CreatedAt)