Merge branch 'master' into feat/expand-description-urls

auth.go

@@ -22,6 +22,9 @@ const (
 	loginURL  = "https://api.twitter.com/1.1/onboarding/task.json"
 	logoutURL = "https://api.twitter.com/1.1/account/logout.json"
 	oAuthURL  = "https://api.twitter.com/oauth2/token"
+	// Doesn't require x-client-transaction-id header in auth. x-rate-limit-limit: 2000
+	bearerToken1 = "AAAAAAAAAAAAAAAAAAAAAFQODgEAAAAAVHTp76lzh3rFzcHbmHVvQxYYpTw%3DckAlMINMjmCwxUcaXbAN4XqJVdgMJaHqNOFgPMK0zN1qLqLQCF"
+	// Requires x-client-transaction-id header in auth.
 	bearerToken2      = "AAAAAAAAAAAAAAAAAAAAANRILgAAAAAAnNwIzUejRCOuH5E6I8xnZz4puTs%3D1Zv7ttfk8LF81IUq16cHjhLTvJu4FA33AGWWjCpTnA"
 	appConsumerKey    = "3nVuSoBZnx6U4vzUxf5w"
 	appConsumerSecret = "Bcs59EFbbsdF6Sl9Ng71smgStWEGwXXKSjYvPVt7qys"
@@ -147,7 +150,7 @@ func (s *Scraper) getFlowToken(data map[string]interface{}) (string, error) {
 // IsLoggedIn check if scraper logged in
 func (s *Scraper) IsLoggedIn() bool {
 	s.isLogged = true
-	s.setBearerToken(bearerToken2)
+	s.setBearerToken(bearerToken1)
 	req, err := http.NewRequest("GET", "https://api.twitter.com/1.1/account/verify_credentials.json", nil)
 	if err != nil {
 		return false

profile.go

@@ -23,6 +23,7 @@ type Profile struct {
 	FriendsCount         int
 	IsPrivate            bool
 	IsVerified           bool
+	IsBlueVerified       bool
 	Joined               *time.Time
 	LikesCount           int
 	ListedCount          int
@@ -37,6 +38,12 @@ type Profile struct {
 	Sensitive            bool
 	Following            bool
 	FollowedBy           bool
+	MediaCount           int
+	FastFollowersCount   int
+	NormalFollowersCount int
+	ProfileImageShape    string
+	HasGraduatedAccess   bool
+	CanHighlightTweets   bool
 }
 
 type user struct {
@@ -46,6 +53,7 @@ type user struct {
 				RestID         string     `json:"rest_id"`
 				Legacy         legacyUser `json:"legacy"`
 				Message        string     `json:"message"`
+				IsBlueVerified bool       `json:"is_blue_verified"`
 			} `json:"result"`
 		} `json:"user"`
 	} `json:"data"`
@@ -111,7 +119,9 @@ func (s *Scraper) GetProfile(username string) (Profile, error) {
 		return Profile{}, fmt.Errorf("either @%s does not exist or is private", username)
 	}
 
-	return parseProfile(jsn.Data.User.Result.Legacy), nil
+	profile := parseProfile(jsn.Data.User.Result.Legacy)
+	profile.IsBlueVerified = jsn.Data.User.Result.IsBlueVerified
+	return profile, nil
 }
 
 func (s *Scraper) GetProfileByID(userID string) (Profile, error) {
@@ -168,7 +178,9 @@ func (s *Scraper) GetProfileByID(userID string) (Profile, error) {
 		return Profile{}, fmt.Errorf("either @%s does not exist or is private", userID)
 	}
 
-	return parseProfile(jsn.Data.User.Result.Legacy), nil
+	profile := parseProfile(jsn.Data.User.Result.Legacy)
+	profile.IsBlueVerified = jsn.Data.User.Result.IsBlueVerified
+	return profile, nil
 }
 
 // GetUserIDByScreenName from API

profile_test.go

@@ -42,6 +42,8 @@ func TestGetProfile(t *testing.T) {
 		cmpopts.IgnoreFields(twitterscraper.Profile{}, "LikesCount"),
 		cmpopts.IgnoreFields(twitterscraper.Profile{}, "ListedCount"),
 		cmpopts.IgnoreFields(twitterscraper.Profile{}, "TweetsCount"),
+		cmpopts.IgnoreFields(twitterscraper.Profile{}, "MediaCount"),
+		cmpopts.IgnoreFields(twitterscraper.Profile{}, "NormalFollowersCount"),
 	}
 	if diff := cmp.Diff(sample, profile, cmpOptions...); diff != "" {
 		t.Error("Resulting profile does not match the sample", diff)
@@ -94,6 +96,8 @@ func TestGetProfilePrivate(t *testing.T) {
 		cmpopts.IgnoreFields(twitterscraper.Profile{}, "LikesCount"),
 		cmpopts.IgnoreFields(twitterscraper.Profile{}, "ListedCount"),
 		cmpopts.IgnoreFields(twitterscraper.Profile{}, "TweetsCount"),
+		cmpopts.IgnoreFields(twitterscraper.Profile{}, "MediaCount"),
+		cmpopts.IgnoreFields(twitterscraper.Profile{}, "NormalFollowersCount"),
 	}
 	if diff := cmp.Diff(sample, profile, cmpOptions...); diff != "" {
 		t.Error("Resulting profile does not match the sample", diff)
@@ -146,7 +150,7 @@ func TestGetProfileByID(t *testing.T) {
 }
 
 func TestGetUserIDByScreenName(t *testing.T) {
-	userID, err := testScraper.GetUserIDByScreenName("Twitter")
+	userID, err := testScraper.GetUserIDByScreenName("X")
 	if err != nil {
 		t.Errorf("getUserByScreenName() error = %v", err)
 	}

timeline_v2.go

@@ -70,6 +70,13 @@ type userResult struct {
 	IsBlueVerified             bool                  `json:"is_blue_verified"`
 	ProfileImageShape          string                `json:"profile_image_shape"`
 	Legacy                     legacyUserV2          `json:"legacy"`
+	LegacyExtendedProfile      legacyExtendedProfile `json:"legacy_extended_profile"`
+	IsProfileTranslatable      bool                  `json:"is_profile_translatable"`
+	VerificationInfo           verificationInfo      `json:"verification_info"`
+	HighlightsInfo             highlightsInfo        `json:"highlights_info"`
+	UserSeedTweetCount         int                   `json:"user_seed_tweet_count"`
+	PremiumGiftingEligible     bool                  `json:"premium_gifting_eligible"`
+	CreatorSubscriptionsCount  int                   `json:"creator_subscriptions_count"`
 }
 
 func (result *userResult) parse() Profile {

types.go

@@ -189,6 +189,21 @@ type (
 		Verified                bool     `json:"verified"`
 		FollowedBy              bool     `json:"followed_by"`
 		Following               bool     `json:"following"`
+		CanDm                   bool     `json:"can_dm"`
+		CanMediaTag             bool     `json:"can_media_tag"`
+		DefaultProfile          bool     `json:"default_profile"`
+		DefaultProfileImage     bool     `json:"default_profile_image"`
+		FastFollowersCount      int      `json:"fast_followers_count"`
+		HasCustomTimelines      bool     `json:"has_custom_timelines"`
+		IsTranslator            bool     `json:"is_translator"`
+		MediaCount              int      `json:"media_count"`
+		NeedsPhoneVerification  bool     `json:"needs_phone_verification"`
+		NormalFollowersCount    int      `json:"normal_followers_count"`
+		PossiblySensitive       bool     `json:"possibly_sensitive"`
+		ProfileInterstitialType string   `json:"profile_interstitial_type"`
+		TranslatorType          string   `json:"translator_type"`
+		WantRetweets            bool     `json:"want_retweets"`
+		WithheldInCountries     []string `json:"withheld_in_countries"`
 	}
 
 	legacyUserV2 struct {
@@ -248,4 +263,37 @@ type (
 
 	fetchProfileFunc func(query string, maxProfilesNbr int, cursor string) ([]*Profile, string, error)
 	fetchTweetFunc   func(query string, maxTweetsNbr int, cursor string) ([]*Tweet, string, error)
+
+	legacyExtendedProfile struct {
+		Birthdate struct {
+			Day            int    `json:"day"`
+			Month          int    `json:"month"`
+			Year           int    `json:"year"`
+			Visibility     string `json:"visibility"`
+			YearVisibility string `json:"year_visibility"`
+		} `json:"birthdate"`
+	}
+
+	verificationInfo struct {
+		IsIdentityVerified bool `json:"is_identity_verified"`
+		Reason             struct {
+			Description struct {
+				Text     string `json:"text"`
+				Entities []struct {
+					FromIndex int `json:"from_index"`
+					ToIndex   int `json:"to_index"`
+					Ref       struct {
+						URL     string `json:"url"`
+						URLType string `json:"url_type"`
+					} `json:"ref"`
+				} `json:"entities"`
+			} `json:"description"`
+			VerifiedSinceMsec string `json:"verified_since_msec"`
+		} `json:"reason"`
+	}
+
+	highlightsInfo struct {
+		CanHighlightTweets bool   `json:"can_highlight_tweets"`
+		HighlightedTweets  string `json:"highlighted_tweets"`
+	}
 )

util.go

@@ -365,6 +365,9 @@ func parseProfile(user legacyUser) Profile {
 		Username:             user.ScreenName,
 		FollowedBy:           user.FollowedBy,
 		Following:            user.Following,
+		MediaCount:           user.MediaCount,
+		FastFollowersCount:   user.FastFollowersCount,
+		NormalFollowersCount: user.NormalFollowersCount,
 	}
 
 	tm, err := time.Parse(time.RubyDate, user.CreatedAt)
@@ -403,6 +406,9 @@ func parseProfileV2(user userResult) Profile {
 		FollowingCount:     u.FavouritesCount,
 		FriendsCount:       u.FriendsCount,
 		IsVerified:         u.Verified,
+		IsBlueVerified:     user.IsBlueVerified,
+		ProfileImageShape:  user.ProfileImageShape,
+		HasGraduatedAccess: user.HasGraduatedAccess,
 		LikesCount:         u.FavouritesCount,
 		ListedCount:        u.ListedCount,
 		Location:           u.Location,