Merge pull request #48 from imperatrona/hotfix

rm bearerToken2

auth.go

@@ -24,8 +24,9 @@ const (
 	oAuthURL  = "https://api.twitter.com/oauth2/token"
 	// Doesn't require x-client-transaction-id header in auth. x-rate-limit-limit: 2000
 	bearerToken1 = "AAAAAAAAAAAAAAAAAAAAAFQODgEAAAAAVHTp76lzh3rFzcHbmHVvQxYYpTw%3DckAlMINMjmCwxUcaXbAN4XqJVdgMJaHqNOFgPMK0zN1qLqLQCF"
-	// Requires x-client-transaction-id header in auth.
+	// HOTFIX: Returns 404 error; Requires x-client-transaction-id header in auth.
-	bearerToken2      = "AAAAAAAAAAAAAAAAAAAAANRILgAAAAAAnNwIzUejRCOuH5E6I8xnZz4puTs%3D1Zv7ttfk8LF81IUq16cHjhLTvJu4FA33AGWWjCpTnA"
+	// bearerToken2      = "AAAAAAAAAAAAAAAAAAAAANRILgAAAAAAnNwIzUejRCOuH5E6I8xnZz4puTs%3D1Zv7ttfk8LF81IUq16cHjhLTvJu4FA33AGWWjCpTnA"
+	bearerToken2      = "AAAAAAAAAAAAAAAAAAAAAFQODgEAAAAAVHTp76lzh3rFzcHbmHVvQxYYpTw%3DckAlMINMjmCwxUcaXbAN4XqJVdgMJaHqNOFgPMK0zN1qLqLQCF"
 	appConsumerKey    = "3nVuSoBZnx6U4vzUxf5w"
 	appConsumerSecret = "Bcs59EFbbsdF6Sl9Ng71smgStWEGwXXKSjYvPVt7qys"
 )

profile.go

@@ -14,38 +14,46 @@ var cacheIDs sync.Map
 
 // Profile of twitter user.
 type Profile struct {
-	Avatar         string
+	Avatar               string
-	Banner         string
+	Banner               string
-	Biography      string
+	Biography            string
-	Birthday       string
+	Birthday             string
-	FollowersCount int
+	FollowersCount       int
-	FollowingCount int
+	FollowingCount       int
-	FriendsCount   int
+	FriendsCount         int
-	IsPrivate      bool
+	IsPrivate            bool
-	IsVerified     bool
+	IsVerified           bool
-	Joined         *time.Time
+	IsBlueVerified       bool
-	LikesCount     int
+	Joined               *time.Time
-	ListedCount    int
+	LikesCount           int
-	Location       string
+	ListedCount          int
-	Name           string
+	Location             string
-	PinnedTweetIDs []string
+	Name                 string
-	TweetsCount    int
+	PinnedTweetIDs       []string
-	URL            string
+	TweetsCount          int
-	UserID         string
+	URL                  string
-	Username       string
+	UserID               string
-	Website        string
+	Username             string
-	Sensitive      bool
+	Website              string
-	Following      bool
+	Sensitive            bool
-	FollowedBy     bool
+	Following            bool
+	FollowedBy           bool
+	MediaCount           int
+	FastFollowersCount   int
+	NormalFollowersCount int
+	ProfileImageShape    string
+	HasGraduatedAccess   bool
+	CanHighlightTweets   bool
 }
 
 type user struct {
 	Data struct {
 		User struct {
 			Result struct {
-				RestID  string     `json:"rest_id"`
+				RestID         string     `json:"rest_id"`
-				Legacy  legacyUser `json:"legacy"`
+				Legacy         legacyUser `json:"legacy"`
-				Message string     `json:"message"`
+				Message        string     `json:"message"`
+				IsBlueVerified bool       `json:"is_blue_verified"`
 			} `json:"result"`
 		} `json:"user"`
 	} `json:"data"`
@@ -111,7 +119,9 @@ func (s *Scraper) GetProfile(username string) (Profile, error) {
 		return Profile{}, fmt.Errorf("either @%s does not exist or is private", username)
 	}
 
-	return parseProfile(jsn.Data.User.Result.Legacy), nil
+	profile := parseProfile(jsn.Data.User.Result.Legacy)
+	profile.IsBlueVerified = jsn.Data.User.Result.IsBlueVerified
+	return profile, nil
 }
 
 func (s *Scraper) GetProfileByID(userID string) (Profile, error) {
@@ -168,7 +178,9 @@ func (s *Scraper) GetProfileByID(userID string) (Profile, error) {
 		return Profile{}, fmt.Errorf("either @%s does not exist or is private", userID)
 	}
 
-	return parseProfile(jsn.Data.User.Result.Legacy), nil
+	profile := parseProfile(jsn.Data.User.Result.Legacy)
+	profile.IsBlueVerified = jsn.Data.User.Result.IsBlueVerified
+	return profile, nil
 }
 
 // GetUserIDByScreenName from API

profile_test.go

@@ -42,6 +42,8 @@ func TestGetProfile(t *testing.T) {
 		cmpopts.IgnoreFields(twitterscraper.Profile{}, "LikesCount"),
 		cmpopts.IgnoreFields(twitterscraper.Profile{}, "ListedCount"),
 		cmpopts.IgnoreFields(twitterscraper.Profile{}, "TweetsCount"),
+		cmpopts.IgnoreFields(twitterscraper.Profile{}, "MediaCount"),
+		cmpopts.IgnoreFields(twitterscraper.Profile{}, "NormalFollowersCount"),
 	}
 	if diff := cmp.Diff(sample, profile, cmpOptions...); diff != "" {
 		t.Error("Resulting profile does not match the sample", diff)
@@ -94,6 +96,8 @@ func TestGetProfilePrivate(t *testing.T) {
 		cmpopts.IgnoreFields(twitterscraper.Profile{}, "LikesCount"),
 		cmpopts.IgnoreFields(twitterscraper.Profile{}, "ListedCount"),
 		cmpopts.IgnoreFields(twitterscraper.Profile{}, "TweetsCount"),
+		cmpopts.IgnoreFields(twitterscraper.Profile{}, "MediaCount"),
+		cmpopts.IgnoreFields(twitterscraper.Profile{}, "NormalFollowersCount"),
 	}
 	if diff := cmp.Diff(sample, profile, cmpOptions...); diff != "" {
 		t.Error("Resulting profile does not match the sample", diff)
@@ -146,7 +150,7 @@ func TestGetProfileByID(t *testing.T) {
 }
 
 func TestGetUserIDByScreenName(t *testing.T) {
-	userID, err := testScraper.GetUserIDByScreenName("Twitter")
+	userID, err := testScraper.GetUserIDByScreenName("X")
 	if err != nil {
 		t.Errorf("getUserByScreenName() error = %v", err)
 	}

timeline_v2.go

@@ -1,6 +1,7 @@
 package twitterscraper
 
 import (
+	"encoding/json"
 	"strconv"
 	"strings"
 )
@@ -28,6 +29,17 @@ type tweet struct {
 		Result *result `json:"result"`
 	} `json:"quoted_status_result"`
 	Legacy legacyTweet `json:"legacy"`
+	Card   struct {
+		RestID string `json:"rest_id"`
+		Legacy struct {
+			BindingValues []struct {
+				Key   string `json:"key"`
+				Value struct {
+					StringValue string `json:"string_value"`
+				} `json:"value"`
+			} `json:"binding_values"`
+		} `json:"legacy"`
+	} `json:"card"`
 }
 
 type result struct {
@@ -36,6 +48,31 @@ type result struct {
 	Tweet tweet `json:"tweet"`
 }
 
+type UnifiedCard struct {
+	Type          string   `json:"type"`
+	Components    []string `json:"components"`
+	MediaEntities map[string]struct {
+		ID            int64  `json:"id"`
+		IDStr         string `json:"id_str"`
+		MediaURLHTTPS string `json:"media_url_https"`
+		Type          string `json:"type"`
+		OriginalInfo  struct {
+			Width  int `json:"width"`
+			Height int `json:"height"`
+		} `json:"original_info"`
+		SourceUserID int64 `json:"source_user_id"`
+		VideoInfo    struct {
+			AspectRatio    []int `json:"aspect_ratio"`
+			DurationMillis int   `json:"duration_millis"`
+			Variants       []struct {
+				Bitrate     int    `json:"bitrate,omitempty"`
+				ContentType string `json:"content_type"`
+				URL         string `json:"url"`
+			} `json:"variants"`
+		} `json:"video_info"`
+	} `json:"media_entities"`
+}
+
 func (result *result) parse() *Tweet {
 	if result.NoteTweet.NoteTweetResults.Result.Text != "" {
 		result.Legacy.FullText = result.NoteTweet.NoteTweetResults.Result.Text
@@ -56,18 +93,62 @@ func (result *result) parse() *Tweet {
 	if result.QuotedStatusResult.Result != nil {
 		tw.QuotedStatus = result.QuotedStatusResult.Result.parse()
 	}
+
+	// Get videos from cards
+	for _, v := range result.Tweet.Card.Legacy.BindingValues {
+		if v.Key == "unified_card" {
+			var card UnifiedCard
+			err := json.Unmarshal([]byte(v.Value.StringValue), &card)
+			if err != nil {
+				continue
+			}
+
+			for _, media := range card.MediaEntities {
+				if media.Type == "video" {
+					var vid Video
+
+					vid.ID = media.IDStr
+					vid.Preview = media.MediaURLHTTPS
+
+					var bitrate int
+					for _, variant := range media.VideoInfo.Variants {
+						if variant.ContentType == "video/mp4" {
+							if variant.Bitrate > bitrate {
+								bitrate = variant.Bitrate
+								vid.URL = variant.URL
+							}
+						} else if variant.ContentType == "application/x-mpegURL" {
+							vid.HLSURL = variant.URL
+						}
+					}
+
+					if vid.URL != "" {
+						tw.Videos = append(tw.Videos, vid)
+					}
+				}
+			}
+		}
+	}
+
 	return tw
 }
 
 type userResult struct {
-	Typename                   string       `json:"__typename"`
+	Typename                   string                `json:"__typename"`
-	ID                         string       `json:"id"`
+	ID                         string                `json:"id"`
-	RestID                     string       `json:"rest_id"`
+	RestID                     string                `json:"rest_id"`
-	AffiliatesHighlightedLabel struct{}     `json:"affiliates_highlighted_label"`
+	AffiliatesHighlightedLabel struct{}              `json:"affiliates_highlighted_label"`
-	HasGraduatedAccess         bool         `json:"has_graduated_access"`
+	HasGraduatedAccess         bool                  `json:"has_graduated_access"`
-	IsBlueVerified             bool         `json:"is_blue_verified"`
+	IsBlueVerified             bool                  `json:"is_blue_verified"`
-	ProfileImageShape          string       `json:"profile_image_shape"`
+	ProfileImageShape          string                `json:"profile_image_shape"`
-	Legacy                     legacyUserV2 `json:"legacy"`
+	Legacy                     legacyUserV2          `json:"legacy"`
+	LegacyExtendedProfile      legacyExtendedProfile `json:"legacy_extended_profile"`
+	IsProfileTranslatable      bool                  `json:"is_profile_translatable"`
+	VerificationInfo           verificationInfo      `json:"verification_info"`
+	HighlightsInfo             highlightsInfo        `json:"highlights_info"`
+	UserSeedTweetCount         int                   `json:"user_seed_tweet_count"`
+	PremiumGiftingEligible     bool                  `json:"premium_gifting_eligible"`
+	CreatorSubscriptionsCount  int                   `json:"creator_subscriptions_count"`
 }
 
 func (result *userResult) parse() Profile {

tweets_test.go

@@ -92,7 +92,7 @@ func TestGetTweetWithVideo(t *testing.T) {
 		Name:           "X",
 		PermanentURL:   "https://twitter.com/X/status/1697304622749086011",
 		Photos:         nil,
-		Text:           "on iOS & Android, you can now swipe to reply when you slide into their DMs https://t.co/evuWpMfBxQ",
+		Text:           "on iOS & Android, you can now swipe to reply when you slide into their DMs https://pbs.twimg.com/amplify_video_thumb/1697304568550330368/img/BUlESpef6FmWV_j2.jpg",
 		Timestamp:      1693503931,
 		UserID:         "783214",
 		Username:       "X",
@@ -125,7 +125,7 @@ func TestGetTweetWithMultiplePhotos(t *testing.T) {
 				URL: "https://pbs.twimg.com/media/FeUJKuxXEAAa6t7.jpg",
 			},
 		},
-		Text:      "More ways to discover videos on Twitter are here!\n\nNow on iOS, videos on your timeline will open in our full screen immersive video player, where you can swipe up to keep discovering more content. https://t.co/XI2vM8DKXA",
+		Text:      "More ways to discover videos on Twitter are here!\n\nNow on iOS, videos on your timeline will open in our full screen immersive video player, where you can swipe up to keep discovering more content. https://pbs.twimg.com/media/FeUJKdnXEAEFe2j.jpg",
 		Timestamp: 1664982561,
 		UserID:    "17874544",
 		Username:  "Support",
@@ -147,7 +147,7 @@ func TestGetTweetWithGIF(t *testing.T) {
 		ID:           "1517535384833605632",
 		Name:         "Support",
 		PermanentURL: "https://twitter.com/Support/status/1517535384833605632",
-		Text:         "Video captions or no captions, it’s now easier to choose for some of you on iOS, and soon on Android.\n\nOn videos that have captions available, we’re testing the option to turn captions off/on with a new “CC” button. https://t.co/Q2Q2Wmr78U",
+		Text:         "Video captions or no captions, it’s now easier to choose for some of you on iOS, and soon on Android.\n\nOn videos that have captions available, we’re testing the option to turn captions off/on with a new “CC” button. https://pbs.twimg.com/tweet_video_thumb/FQ9eXEhXEAA-haj.jpg",
 		Timestamp:    1650643604,
 		UserID:       "17874544",
 		Username:     "Support",
@@ -170,7 +170,7 @@ func TestGetTweetWithPhotoAndGIF(t *testing.T) {
 		Name:         "Spaces",
 		PermanentURL: "https://twitter.com/XSpaces/status/1583186305722507265",
 		Photos:       []twitterscraper.Photo{{ID: "1583186295626539020", URL: "https://pbs.twimg.com/media/FfibjDwWIAwvbtJ.jpg"}},
-		Text:         "“we need to talk” \n\nirl vs on Spaces https://t.co/hrflPpbpif",
+		Text:         "“we need to talk” \n\nirl vs on Spaces https://pbs.twimg.com/tweet_video_thumb/FfibjDnWIBIt5fn.jpg",
 		Timestamp:    1666296004,
 		UserID:       "1065249714214457345",
 		Username:     "XSpaces",
@@ -197,7 +197,7 @@ func TestTweetMentions(t *testing.T) {
 func TestQuotedAndReply(t *testing.T) {
 	sample := &twitterscraper.Tweet{
 		ConversationID: "1237110546383724547",
-		HTML:           "The Easiest Problem Everyone Gets Wrong <br><br>[new video] --&gt; <a href=\"https://youtu.be/ytfCdqWhmdg\">https://t.co/YdaeDYmPAU</a> <br><a href=\"https://t.co/iKu4Xs6o2V\"><img src=\"https://pbs.twimg.com/media/ESsZa9AXgAIAYnF.jpg\"/></a>",
+		HTML:           "The Easiest Problem Everyone Gets Wrong <br><br>[new video] --&gt; <a href=\"https://youtu.be/ytfCdqWhmdg\">https://youtu.be/ytfCdqWhmdg</a> <br><a href=\"https://t.co/iKu4Xs6o2V\"><img src=\"https://pbs.twimg.com/media/ESsZa9AXgAIAYnF.jpg\"/></a>",
 		ID:             "1237110546383724547",
 		Likes:          485,
 		Name:           "Vsauce2",
@@ -208,7 +208,7 @@ func TestQuotedAndReply(t *testing.T) {
 		}},
 		Replies:   12,
 		Retweets:  18,
-		Text:      "The Easiest Problem Everyone Gets Wrong \n\n[new video] --&gt; https://t.co/YdaeDYmPAU https://t.co/iKu4Xs6o2V",
+		Text:      "The Easiest Problem Everyone Gets Wrong \n\n[new video] --&gt; https://youtu.be/ytfCdqWhmdg https://pbs.twimg.com/media/ESsZa9AXgAIAYnF.jpg",
 		Timestamp: 1583785113,
 		URLs:      []string{"https://youtu.be/ytfCdqWhmdg"},
 		UserID:    "978944851",
@@ -241,13 +241,13 @@ func TestQuotedAndReply(t *testing.T) {
 func TestRetweet(t *testing.T) {
 	sample := &twitterscraper.Tweet{
 		ConversationID: "1758837061786779942",
-		HTML:           "no ads, just bangers<br><br>aka your For You feed with Premium+<br><br>subscribe here → <a href=\"https://x.com/i/premium_sign_up\">https://t.co/APTO1t7kMk</a>",
+		HTML:           "no ads, just bangers<br><br>aka your For You feed with Premium+<br><br>subscribe here → <a href=\"https://x.com/i/premium_sign_up\">https://x.com/i/premium_sign_up</a>",
 		ID:             "1758837061786779942",
 		URLs:           []string{"https://x.com/i/premium_sign_up"},
 		IsSelfThread:   false,
 		Name:           "Premium",
 		PermanentURL:   "https://twitter.com/premium/status/1758837061786779942",
-		Text:           "no ads, just bangers\n\naka your For You feed with Premium+\n\nsubscribe here → https://t.co/APTO1t7kMk",
+		Text:           "no ads, just bangers\n\naka your For You feed with Premium+\n\nsubscribe here → https://x.com/i/premium_sign_up",
 		Timestamp:      1708174407,
 		UserID:         "1399766153053061121",
 		Username:       "premium",

types.go

@@ -10,6 +10,14 @@ type (
 		Name     string
 	}
 
+	// Url represents a URL with display, expanded, and index data.
+	Url struct {
+		DisplayURL  string `json:"display_url"`
+		ExpandedURL string `json:"expanded_url"`
+		URL         string `json:"url"`
+		Indices     []int  `json:"indices"`
+	}
+
 	// Photo type.
 	Photo struct {
 		ID  string
@@ -91,6 +99,25 @@ type (
 		GIFs      []GIF
 	}
 
+	ExtendedMedia struct {
+		IDStr                    string `json:"id_str"`
+		MediaURLHttps            string `json:"media_url_https"`
+		ExtSensitiveMediaWarning struct {
+			AdultContent    bool `json:"adult_content"`
+			GraphicViolence bool `json:"graphic_violence"`
+			Other           bool `json:"other"`
+		} `json:"ext_sensitive_media_warning"`
+		Type      string `json:"type"`
+		URL       string `json:"url"`
+		VideoInfo struct {
+			Variants []struct {
+				Type    string `json:"content_type"`
+				Bitrate int    `json:"bitrate"`
+				URL     string `json:"url"`
+			} `json:"variants"`
+		} `json:"video_info"`
+	}
+
 	legacyTweet struct {
 		ConversationIDStr string `json:"conversation_id_str"`
 		CreatedAt         string `json:"created_at"`
@@ -105,10 +132,7 @@ type (
 				Type          string `json:"type"`
 				URL           string `json:"url"`
 			} `json:"media"`
-			URLs []struct {
+			URLs         []Url `json:"urls"`
-				ExpandedURL string `json:"expanded_url"`
-				URL         string `json:"url"`
-			} `json:"urls"`
 			UserMentions []struct {
 				IDStr      string `json:"id_str"`
 				Name       string `json:"name"`
@@ -116,24 +140,7 @@ type (
 			} `json:"user_mentions"`
 		} `json:"entities"`
 		ExtendedEntities struct {
-			Media []struct {
+			Media []ExtendedMedia `json:"media"`
-				IDStr                    string `json:"id_str"`
-				MediaURLHttps            string `json:"media_url_https"`
-				ExtSensitiveMediaWarning struct {
-					AdultContent    bool `json:"adult_content"`
-					GraphicViolence bool `json:"graphic_violence"`
-					Other           bool `json:"other"`
-				} `json:"ext_sensitive_media_warning"`
-				Type      string `json:"type"`
-				URL       string `json:"url"`
-				VideoInfo struct {
-					Variants []struct {
-						Type    string `json:"content_type"`
-						Bitrate int    `json:"bitrate"`
-						URL     string `json:"url"`
-					} `json:"variants"`
-				} `json:"video_info"`
-			} `json:"media"`
 		} `json:"extended_entities"`
 		IDStr                 string `json:"id_str"`
 		InReplyToStatusIDStr  string `json:"in_reply_to_status_id_str"`
@@ -166,22 +173,37 @@ type (
 				} `json:"urls"`
 			} `json:"url"`
 		} `json:"entities"`
-		FavouritesCount      int      `json:"favourites_count"`
+		FavouritesCount         int      `json:"favourites_count"`
-		FollowersCount       int      `json:"followers_count"`
+		FollowersCount          int      `json:"followers_count"`
-		FriendsCount         int      `json:"friends_count"`
+		FriendsCount            int      `json:"friends_count"`
-		IDStr                string   `json:"id_str"`
+		IDStr                   string   `json:"id_str"`
-		ListedCount          int      `json:"listed_count"`
+		ListedCount             int      `json:"listed_count"`
-		Name                 string   `json:"name"`
+		Name                    string   `json:"name"`
-		Location             string   `json:"location"`
+		Location                string   `json:"location"`
-		PinnedTweetIdsStr    []string `json:"pinned_tweet_ids_str"`
+		PinnedTweetIdsStr       []string `json:"pinned_tweet_ids_str"`
-		ProfileBannerURL     string   `json:"profile_banner_url"`
+		ProfileBannerURL        string   `json:"profile_banner_url"`
-		ProfileImageURLHTTPS string   `json:"profile_image_url_https"`
+		ProfileImageURLHTTPS    string   `json:"profile_image_url_https"`
-		Protected            bool     `json:"protected"`
+		Protected               bool     `json:"protected"`
-		ScreenName           string   `json:"screen_name"`
+		ScreenName              string   `json:"screen_name"`
-		StatusesCount        int      `json:"statuses_count"`
+		StatusesCount           int      `json:"statuses_count"`
-		Verified             bool     `json:"verified"`
+		Verified                bool     `json:"verified"`
-		FollowedBy           bool     `json:"followed_by"`
+		FollowedBy              bool     `json:"followed_by"`
-		Following            bool     `json:"following"`
+		Following               bool     `json:"following"`
+		CanDm                   bool     `json:"can_dm"`
+		CanMediaTag             bool     `json:"can_media_tag"`
+		DefaultProfile          bool     `json:"default_profile"`
+		DefaultProfileImage     bool     `json:"default_profile_image"`
+		FastFollowersCount      int      `json:"fast_followers_count"`
+		HasCustomTimelines      bool     `json:"has_custom_timelines"`
+		IsTranslator            bool     `json:"is_translator"`
+		MediaCount              int      `json:"media_count"`
+		NeedsPhoneVerification  bool     `json:"needs_phone_verification"`
+		NormalFollowersCount    int      `json:"normal_followers_count"`
+		PossiblySensitive       bool     `json:"possibly_sensitive"`
+		ProfileInterstitialType string   `json:"profile_interstitial_type"`
+		TranslatorType          string   `json:"translator_type"`
+		WantRetweets            bool     `json:"want_retweets"`
+		WithheldInCountries     []string `json:"withheld_in_countries"`
 	}
 
 	legacyUserV2 struct {
@@ -195,15 +217,10 @@ type (
 		Description         string `json:"description"`
 		Entities            struct {
 			Description struct {
-				Urls []interface{} `json:"urls"`
+				Urls []Url `json:"urls"`
 			} `json:"description"`
 			URL struct {
-				Urls []struct {
+				Urls []Url `json:"urls"`
-					DisplayURL  string `json:"display_url"`
-					ExpandedURL string `json:"expanded_url"`
-					URL         string `json:"url"`
-					Indices     []int  `json:"indices"`
-				} `json:"urls"`
 			} `json:"url"`
 		} `json:"entities"`
 		FastFollowersCount      int           `json:"fast_followers_count"`
@@ -246,4 +263,37 @@ type (
 
 	fetchProfileFunc func(query string, maxProfilesNbr int, cursor string) ([]*Profile, string, error)
 	fetchTweetFunc   func(query string, maxTweetsNbr int, cursor string) ([]*Tweet, string, error)
+
+	legacyExtendedProfile struct {
+		Birthdate struct {
+			Day            int    `json:"day"`
+			Month          int    `json:"month"`
+			Year           int    `json:"year"`
+			Visibility     string `json:"visibility"`
+			YearVisibility string `json:"year_visibility"`
+		} `json:"birthdate"`
+	}
+
+	verificationInfo struct {
+		IsIdentityVerified bool `json:"is_identity_verified"`
+		Reason             struct {
+			Description struct {
+				Text     string `json:"text"`
+				Entities []struct {
+					FromIndex int `json:"from_index"`
+					ToIndex   int `json:"to_index"`
+					Ref       struct {
+						URL     string `json:"url"`
+						URLType string `json:"url_type"`
+					} `json:"ref"`
+				} `json:"entities"`
+			} `json:"description"`
+			VerifiedSinceMsec string `json:"verified_since_msec"`
+		} `json:"reason"`
+	}
+
+	highlightsInfo struct {
+		CanHighlightTweets bool   `json:"can_highlight_tweets"`
+		HighlightedTweets  string `json:"highlighted_tweets"`
+	}
 )

util.go

@@ -157,6 +157,7 @@ func parseLegacyTweet(user *legacyUser, tweet *legacyTweet) *Tweet {
 	if tweetID == "" {
 		return nil
 	}
+	text := expandURLs(tweet.FullText, tweet.Entities.URLs, tweet.ExtendedEntities.Media)
 	username := user.ScreenName
 	name := user.Name
 	tw := &Tweet{
@@ -167,7 +168,7 @@ func parseLegacyTweet(user *legacyUser, tweet *legacyTweet) *Tweet {
 		PermanentURL:   fmt.Sprintf("https://twitter.com/%s/status/%s", username, tweetID),
 		Replies:        tweet.ReplyCount,
 		Retweets:       tweet.RetweetCount,
-		Text:           tweet.FullText,
+		Text:           text,
 		UserID:         tweet.UserIDStr,
 		Username:       username,
 	}
@@ -307,7 +308,7 @@ func parseLegacyTweet(user *legacyUser, tweet *legacyTweet) *Tweet {
 	tw.HTML = reTwitterURL.ReplaceAllStringFunc(tw.HTML, func(tco string) string {
 		for _, entity := range tweet.Entities.URLs {
 			if tco == entity.URL {
-				return fmt.Sprintf(`<a href="%s">%s</a>`, entity.ExpandedURL, tco)
+				return fmt.Sprintf(`<a href="%s">%s</a>`, entity.ExpandedURL, entity.ExpandedURL)
 			}
 		}
 		for _, entity := range tweet.ExtendedEntities.Media {
@@ -345,25 +346,28 @@ func parseLegacyTweet(user *legacyUser, tweet *legacyTweet) *Tweet {
 
 func parseProfile(user legacyUser) Profile {
 	profile := Profile{
-		Avatar:         user.ProfileImageURLHTTPS,
+		Avatar:               user.ProfileImageURLHTTPS,
-		Banner:         user.ProfileBannerURL,
+		Banner:               user.ProfileBannerURL,
-		Biography:      user.Description,
+		Biography:            user.Description,
-		FollowersCount: user.FollowersCount,
+		FollowersCount:       user.FollowersCount,
-		FollowingCount: user.FavouritesCount,
+		FollowingCount:       user.FavouritesCount,
-		FriendsCount:   user.FriendsCount,
+		FriendsCount:         user.FriendsCount,
-		IsVerified:     user.Verified,
+		IsVerified:           user.Verified,
-		IsPrivate:      user.Protected,
+		IsPrivate:            user.Protected,
-		LikesCount:     user.FavouritesCount,
+		LikesCount:           user.FavouritesCount,
-		ListedCount:    user.ListedCount,
+		ListedCount:          user.ListedCount,
-		Location:       user.Location,
+		Location:             user.Location,
-		Name:           user.Name,
+		Name:                 user.Name,
-		PinnedTweetIDs: user.PinnedTweetIdsStr,
+		PinnedTweetIDs:       user.PinnedTweetIdsStr,
-		TweetsCount:    user.StatusesCount,
+		TweetsCount:          user.StatusesCount,
-		URL:            "https://twitter.com/" + user.ScreenName,
+		URL:                  "https://twitter.com/" + user.ScreenName,
-		UserID:         user.IDStr,
+		UserID:               user.IDStr,
-		Username:       user.ScreenName,
+		Username:             user.ScreenName,
-		FollowedBy:     user.FollowedBy,
+		FollowedBy:           user.FollowedBy,
-		Following:      user.Following,
+		Following:            user.Following,
+		MediaCount:           user.MediaCount,
+		FastFollowersCount:   user.FastFollowersCount,
+		NormalFollowersCount: user.NormalFollowersCount,
 	}
 
 	tm, err := time.Parse(time.RubyDate, user.CreatedAt)
@@ -379,28 +383,44 @@ func parseProfile(user legacyUser) Profile {
 	return profile
 }
 
+func expandURLs(text string, urls []Url, extendedMediaEntities []ExtendedMedia) string {
+	expandedText := text
+	for _, url := range urls {
+		expandedText = strings.ReplaceAll(expandedText, url.URL, url.ExpandedURL)
+	}
+	for _, entity := range extendedMediaEntities {
+		expandedText = strings.ReplaceAll(expandedText, entity.URL, entity.MediaURLHttps)
+	}
+
+	return expandedText
+}
+
 func parseProfileV2(user userResult) Profile {
 	u := user.Legacy
+	description := expandURLs(u.Description, u.Entities.Description.Urls, []ExtendedMedia{})
 	profile := Profile{
-		Avatar:         u.ProfileImageURLHTTPS,
+		Avatar:             u.ProfileImageURLHTTPS,
-		Banner:         u.ProfileBannerURL,
+		Banner:             u.ProfileBannerURL,
-		Biography:      u.Description,
+		Biography:          description,
-		FollowersCount: u.FollowersCount,
+		FollowersCount:     u.FollowersCount,
-		FollowingCount: u.FavouritesCount,
+		FollowingCount:     u.FavouritesCount,
-		FriendsCount:   u.FriendsCount,
+		FriendsCount:       u.FriendsCount,
-		IsVerified:     u.Verified,
+		IsVerified:         u.Verified,
-		LikesCount:     u.FavouritesCount,
+		IsBlueVerified:     user.IsBlueVerified,
-		ListedCount:    u.ListedCount,
+		ProfileImageShape:  user.ProfileImageShape,
-		Location:       u.Location,
+		HasGraduatedAccess: user.HasGraduatedAccess,
-		Name:           u.Name,
+		LikesCount:         u.FavouritesCount,
-		PinnedTweetIDs: u.PinnedTweetIdsStr,
+		ListedCount:        u.ListedCount,
-		TweetsCount:    u.StatusesCount,
+		Location:           u.Location,
-		URL:            "https://twitter.com/" + u.ScreenName,
+		Name:               u.Name,
-		UserID:         user.ID,
+		PinnedTweetIDs:     u.PinnedTweetIdsStr,
-		Username:       u.ScreenName,
+		TweetsCount:        u.StatusesCount,
-		Sensitive:      u.PossiblySensitive,
+		URL:                "https://twitter.com/" + u.ScreenName,
-		Following:      u.Following,
+		UserID:             user.ID,
-		FollowedBy:     u.FollowedBy,
+		Username:           u.ScreenName,
+		Sensitive:          u.PossiblySensitive,
+		Following:          u.Following,
+		FollowedBy:         u.FollowedBy,
 	}
 
 	tm, err := time.Parse(time.RubyDate, u.CreatedAt)