Merge pull request #48 from imperatrona/hotfix

rm bearerToken2
2025-04-30 01:06:44 +03:00 · 2025-04-30 01:03:40 +03:00 · 2025-03-28 14:27:00 +03:00 · 2025-03-28 14:14:16 +03:00 · 2025-01-23 03:13:18 +03:00 · 2025-01-23 03:03:43 +03:00
7 changed files with 300 additions and 132 deletions
--- a/auth.go
+++ b/auth.go
@ -24,8 +24,9 @@ const (
 	oAuthURL  = "https://api.twitter.com/oauth2/token"
 	// Doesn't require x-client-transaction-id header in auth. x-rate-limit-limit: 2000
 	bearerToken1 = "AAAAAAAAAAAAAAAAAAAAAFQODgEAAAAAVHTp76lzh3rFzcHbmHVvQxYYpTw%3DckAlMINMjmCwxUcaXbAN4XqJVdgMJaHqNOFgPMK0zN1qLqLQCF"
-	// Requires x-client-transaction-id header in auth.
-	bearerToken2      = "AAAAAAAAAAAAAAAAAAAAANRILgAAAAAAnNwIzUejRCOuH5E6I8xnZz4puTs%3D1Zv7ttfk8LF81IUq16cHjhLTvJu4FA33AGWWjCpTnA"
+	// HOTFIX: Returns 404 error; Requires x-client-transaction-id header in auth.
+	// bearerToken2      = "AAAAAAAAAAAAAAAAAAAAANRILgAAAAAAnNwIzUejRCOuH5E6I8xnZz4puTs%3D1Zv7ttfk8LF81IUq16cHjhLTvJu4FA33AGWWjCpTnA"
+	bearerToken2      = "AAAAAAAAAAAAAAAAAAAAAFQODgEAAAAAVHTp76lzh3rFzcHbmHVvQxYYpTw%3DckAlMINMjmCwxUcaXbAN4XqJVdgMJaHqNOFgPMK0zN1qLqLQCF"
 	appConsumerKey    = "3nVuSoBZnx6U4vzUxf5w"
 	appConsumerSecret = "Bcs59EFbbsdF6Sl9Ng71smgStWEGwXXKSjYvPVt7qys"
 )
--- a/profile.go
+++ b/profile.go
@ -14,38 +14,46 @@ var cacheIDs sync.Map

 // Profile of twitter user.
 type Profile struct {
-	Avatar         string
-	Banner         string
-	Biography      string
-	Birthday       string
-	FollowersCount int
-	FollowingCount int
-	FriendsCount   int
-	IsPrivate      bool
-	IsVerified     bool
-	Joined         *time.Time
-	LikesCount     int
-	ListedCount    int
-	Location       string
-	Name           string
-	PinnedTweetIDs []string
-	TweetsCount    int
-	URL            string
-	UserID         string
-	Username       string
-	Website        string
-	Sensitive      bool
-	Following      bool
-	FollowedBy     bool
+	Avatar               string
+	Banner               string
+	Biography            string
+	Birthday             string
+	FollowersCount       int
+	FollowingCount       int
+	FriendsCount         int
+	IsPrivate            bool
+	IsVerified           bool
+	IsBlueVerified       bool
+	Joined               *time.Time
+	LikesCount           int
+	ListedCount          int
+	Location             string
+	Name                 string
+	PinnedTweetIDs       []string
+	TweetsCount          int
+	URL                  string
+	UserID               string
+	Username             string
+	Website              string
+	Sensitive            bool
+	Following            bool
+	FollowedBy           bool
+	MediaCount           int
+	FastFollowersCount   int
+	NormalFollowersCount int
+	ProfileImageShape    string
+	HasGraduatedAccess   bool
+	CanHighlightTweets   bool
 }

 type user struct {
 	Data struct {
 		User struct {
 			Result struct {
-				RestID  string     `json:"rest_id"`
-				Legacy  legacyUser `json:"legacy"`
-				Message string     `json:"message"`
+				RestID         string     `json:"rest_id"`
+				Legacy         legacyUser `json:"legacy"`
+				Message        string     `json:"message"`
+				IsBlueVerified bool       `json:"is_blue_verified"`
 			} `json:"result"`
 		} `json:"user"`
 	} `json:"data"`
@ -111,7 +119,9 @@ func (s *Scraper) GetProfile(username string) (Profile, error) {
 		return Profile{}, fmt.Errorf("either @%s does not exist or is private", username)
 	}

-	return parseProfile(jsn.Data.User.Result.Legacy), nil
+	profile := parseProfile(jsn.Data.User.Result.Legacy)
+	profile.IsBlueVerified = jsn.Data.User.Result.IsBlueVerified
+	return profile, nil
 }

 func (s *Scraper) GetProfileByID(userID string) (Profile, error) {
@ -168,7 +178,9 @@ func (s *Scraper) GetProfileByID(userID string) (Profile, error) {
 		return Profile{}, fmt.Errorf("either @%s does not exist or is private", userID)
 	}

-	return parseProfile(jsn.Data.User.Result.Legacy), nil
+	profile := parseProfile(jsn.Data.User.Result.Legacy)
+	profile.IsBlueVerified = jsn.Data.User.Result.IsBlueVerified
+	return profile, nil
 }

 // GetUserIDByScreenName from API
--- a/profile_test.go
+++ b/profile_test.go
@ -42,6 +42,8 @@ func TestGetProfile(t *testing.T) {
 		cmpopts.IgnoreFields(twitterscraper.Profile{}, "LikesCount"),
 		cmpopts.IgnoreFields(twitterscraper.Profile{}, "ListedCount"),
 		cmpopts.IgnoreFields(twitterscraper.Profile{}, "TweetsCount"),
+		cmpopts.IgnoreFields(twitterscraper.Profile{}, "MediaCount"),
+		cmpopts.IgnoreFields(twitterscraper.Profile{}, "NormalFollowersCount"),
 	}
 	if diff := cmp.Diff(sample, profile, cmpOptions...); diff != "" {
 		t.Error("Resulting profile does not match the sample", diff)
@ -94,6 +96,8 @@ func TestGetProfilePrivate(t *testing.T) {
 		cmpopts.IgnoreFields(twitterscraper.Profile{}, "LikesCount"),
 		cmpopts.IgnoreFields(twitterscraper.Profile{}, "ListedCount"),
 		cmpopts.IgnoreFields(twitterscraper.Profile{}, "TweetsCount"),
+		cmpopts.IgnoreFields(twitterscraper.Profile{}, "MediaCount"),
+		cmpopts.IgnoreFields(twitterscraper.Profile{}, "NormalFollowersCount"),
 	}
 	if diff := cmp.Diff(sample, profile, cmpOptions...); diff != "" {
 		t.Error("Resulting profile does not match the sample", diff)
@ -146,7 +150,7 @@ func TestGetProfileByID(t *testing.T) {
 }

 func TestGetUserIDByScreenName(t *testing.T) {
-	userID, err := testScraper.GetUserIDByScreenName("Twitter")
+	userID, err := testScraper.GetUserIDByScreenName("X")
 	if err != nil {
 		t.Errorf("getUserByScreenName() error = %v", err)
 	}
--- a/timeline_v2.go
+++ b/timeline_v2.go
@ -1,6 +1,7 @@
 package twitterscraper

 import (
+	"encoding/json"
 	"strconv"
 	"strings"
 )
@ -28,6 +29,17 @@ type tweet struct {
 		Result *result `json:"result"`
 	} `json:"quoted_status_result"`
 	Legacy legacyTweet `json:"legacy"`
+	Card   struct {
+		RestID string `json:"rest_id"`
+		Legacy struct {
+			BindingValues []struct {
+				Key   string `json:"key"`
+				Value struct {
+					StringValue string `json:"string_value"`
+				} `json:"value"`
+			} `json:"binding_values"`
+		} `json:"legacy"`
+	} `json:"card"`
 }

 type result struct {
@ -36,6 +48,31 @@ type result struct {
 	Tweet tweet `json:"tweet"`
 }

+type UnifiedCard struct {
+	Type          string   `json:"type"`
+	Components    []string `json:"components"`
+	MediaEntities map[string]struct {
+		ID            int64  `json:"id"`
+		IDStr         string `json:"id_str"`
+		MediaURLHTTPS string `json:"media_url_https"`
+		Type          string `json:"type"`
+		OriginalInfo  struct {
+			Width  int `json:"width"`
+			Height int `json:"height"`
+		} `json:"original_info"`
+		SourceUserID int64 `json:"source_user_id"`
+		VideoInfo    struct {
+			AspectRatio    []int `json:"aspect_ratio"`
+			DurationMillis int   `json:"duration_millis"`
+			Variants       []struct {
+				Bitrate     int    `json:"bitrate,omitempty"`
+				ContentType string `json:"content_type"`
+				URL         string `json:"url"`
+			} `json:"variants"`
+		} `json:"video_info"`
+	} `json:"media_entities"`
+}
+
 func (result *result) parse() *Tweet {
 	if result.NoteTweet.NoteTweetResults.Result.Text != "" {
 		result.Legacy.FullText = result.NoteTweet.NoteTweetResults.Result.Text
@ -56,18 +93,62 @@ func (result *result) parse() *Tweet {
 	if result.QuotedStatusResult.Result != nil {
 		tw.QuotedStatus = result.QuotedStatusResult.Result.parse()
 	}
+
+	// Get videos from cards
+	for _, v := range result.Tweet.Card.Legacy.BindingValues {
+		if v.Key == "unified_card" {
+			var card UnifiedCard
+			err := json.Unmarshal([]byte(v.Value.StringValue), &card)
+			if err != nil {
+				continue
+			}
+
+			for _, media := range card.MediaEntities {
+				if media.Type == "video" {
+					var vid Video
+
+					vid.ID = media.IDStr
+					vid.Preview = media.MediaURLHTTPS
+
+					var bitrate int
+					for _, variant := range media.VideoInfo.Variants {
+						if variant.ContentType == "video/mp4" {
+							if variant.Bitrate > bitrate {
+								bitrate = variant.Bitrate
+								vid.URL = variant.URL
+							}
+						} else if variant.ContentType == "application/x-mpegURL" {
+							vid.HLSURL = variant.URL
+						}
+					}
+
+					if vid.URL != "" {
+						tw.Videos = append(tw.Videos, vid)
+					}
+				}
+			}
+		}
+	}
+
 	return tw
 }

 type userResult struct {
-	Typename                   string       `json:"__typename"`
-	ID                         string       `json:"id"`
-	RestID                     string       `json:"rest_id"`
-	AffiliatesHighlightedLabel struct{}     `json:"affiliates_highlighted_label"`
-	HasGraduatedAccess         bool         `json:"has_graduated_access"`
-	IsBlueVerified             bool         `json:"is_blue_verified"`
-	ProfileImageShape          string       `json:"profile_image_shape"`
-	Legacy                     legacyUserV2 `json:"legacy"`
+	Typename                   string                `json:"__typename"`
+	ID                         string                `json:"id"`
+	RestID                     string                `json:"rest_id"`
+	AffiliatesHighlightedLabel struct{}              `json:"affiliates_highlighted_label"`
+	HasGraduatedAccess         bool                  `json:"has_graduated_access"`
+	IsBlueVerified             bool                  `json:"is_blue_verified"`
+	ProfileImageShape          string                `json:"profile_image_shape"`
+	Legacy                     legacyUserV2          `json:"legacy"`
+	LegacyExtendedProfile      legacyExtendedProfile `json:"legacy_extended_profile"`
+	IsProfileTranslatable      bool                  `json:"is_profile_translatable"`
+	VerificationInfo           verificationInfo      `json:"verification_info"`
+	HighlightsInfo             highlightsInfo        `json:"highlights_info"`
+	UserSeedTweetCount         int                   `json:"user_seed_tweet_count"`
+	PremiumGiftingEligible     bool                  `json:"premium_gifting_eligible"`
+	CreatorSubscriptionsCount  int                   `json:"creator_subscriptions_count"`
 }

 func (result *userResult) parse() Profile {
--- a/tweets_test.go
+++ b/tweets_test.go
@ -92,7 +92,7 @@ func TestGetTweetWithVideo(t *testing.T) {
 		Name:           "X",
 		PermanentURL:   "https://twitter.com/X/status/1697304622749086011",
 		Photos:         nil,
-		Text:           "on iOS &amp; Android, you can now swipe to reply when you slide into their DMs https://t.co/evuWpMfBxQ",
+		Text:           "on iOS &amp; Android, you can now swipe to reply when you slide into their DMs https://pbs.twimg.com/amplify_video_thumb/1697304568550330368/img/BUlESpef6FmWV_j2.jpg",
 		Timestamp:      1693503931,
 		UserID:         "783214",
 		Username:       "X",
@ -125,7 +125,7 @@ func TestGetTweetWithMultiplePhotos(t *testing.T) {
 				URL: "https://pbs.twimg.com/media/FeUJKuxXEAAa6t7.jpg",
 			},
 		},
-		Text:      "More ways to discover videos on Twitter are here!\n\nNow on iOS, videos on your timeline will open in our full screen immersive video player, where you can swipe up to keep discovering more content. https://t.co/XI2vM8DKXA",
+		Text:      "More ways to discover videos on Twitter are here!\n\nNow on iOS, videos on your timeline will open in our full screen immersive video player, where you can swipe up to keep discovering more content. https://pbs.twimg.com/media/FeUJKdnXEAEFe2j.jpg",
 		Timestamp: 1664982561,
 		UserID:    "17874544",
 		Username:  "Support",
@ -147,7 +147,7 @@ func TestGetTweetWithGIF(t *testing.T) {
 		ID:           "1517535384833605632",
 		Name:         "Support",
 		PermanentURL: "https://twitter.com/Support/status/1517535384833605632",
-		Text:         "Video captions or no captions, it’s now easier to choose for some of you on iOS, and soon on Android.\n\nOn videos that have captions available, we’re testing the option to turn captions off/on with a new “CC” button. https://t.co/Q2Q2Wmr78U",
+		Text:         "Video captions or no captions, it’s now easier to choose for some of you on iOS, and soon on Android.\n\nOn videos that have captions available, we’re testing the option to turn captions off/on with a new “CC” button. https://pbs.twimg.com/tweet_video_thumb/FQ9eXEhXEAA-haj.jpg",
 		Timestamp:    1650643604,
 		UserID:       "17874544",
 		Username:     "Support",
@ -170,7 +170,7 @@ func TestGetTweetWithPhotoAndGIF(t *testing.T) {
 		Name:         "Spaces",
 		PermanentURL: "https://twitter.com/XSpaces/status/1583186305722507265",
 		Photos:       []twitterscraper.Photo{{ID: "1583186295626539020", URL: "https://pbs.twimg.com/media/FfibjDwWIAwvbtJ.jpg"}},
-		Text:         "“we need to talk” \n\nirl vs on Spaces https://t.co/hrflPpbpif",
+		Text:         "“we need to talk” \n\nirl vs on Spaces https://pbs.twimg.com/tweet_video_thumb/FfibjDnWIBIt5fn.jpg",
 		Timestamp:    1666296004,
 		UserID:       "1065249714214457345",
 		Username:     "XSpaces",
@ -197,7 +197,7 @@ func TestTweetMentions(t *testing.T) {
 func TestQuotedAndReply(t *testing.T) {
 	sample := &twitterscraper.Tweet{
 		ConversationID: "1237110546383724547",
-		HTML:           "The Easiest Problem Everyone Gets Wrong <br><br>[new video] --&gt; <a href=\"https://youtu.be/ytfCdqWhmdg\">https://t.co/YdaeDYmPAU</a> <br><a href=\"https://t.co/iKu4Xs6o2V\"><img src=\"https://pbs.twimg.com/media/ESsZa9AXgAIAYnF.jpg\"/></a>",
+		HTML:           "The Easiest Problem Everyone Gets Wrong <br><br>[new video] --&gt; <a href=\"https://youtu.be/ytfCdqWhmdg\">https://youtu.be/ytfCdqWhmdg</a> <br><a href=\"https://t.co/iKu4Xs6o2V\"><img src=\"https://pbs.twimg.com/media/ESsZa9AXgAIAYnF.jpg\"/></a>",
 		ID:             "1237110546383724547",
 		Likes:          485,
 		Name:           "Vsauce2",
@ -208,7 +208,7 @@ func TestQuotedAndReply(t *testing.T) {
 		}},
 		Replies:   12,
 		Retweets:  18,
-		Text:      "The Easiest Problem Everyone Gets Wrong \n\n[new video] --&gt; https://t.co/YdaeDYmPAU https://t.co/iKu4Xs6o2V",
+		Text:      "The Easiest Problem Everyone Gets Wrong \n\n[new video] --&gt; https://youtu.be/ytfCdqWhmdg https://pbs.twimg.com/media/ESsZa9AXgAIAYnF.jpg",
 		Timestamp: 1583785113,
 		URLs:      []string{"https://youtu.be/ytfCdqWhmdg"},
 		UserID:    "978944851",
@ -241,13 +241,13 @@ func TestQuotedAndReply(t *testing.T) {
 func TestRetweet(t *testing.T) {
 	sample := &twitterscraper.Tweet{
 		ConversationID: "1758837061786779942",
-		HTML:           "no ads, just bangers<br><br>aka your For You feed with Premium+<br><br>subscribe here → <a href=\"https://x.com/i/premium_sign_up\">https://t.co/APTO1t7kMk</a>",
+		HTML:           "no ads, just bangers<br><br>aka your For You feed with Premium+<br><br>subscribe here → <a href=\"https://x.com/i/premium_sign_up\">https://x.com/i/premium_sign_up</a>",
 		ID:             "1758837061786779942",
 		URLs:           []string{"https://x.com/i/premium_sign_up"},
 		IsSelfThread:   false,
 		Name:           "Premium",
 		PermanentURL:   "https://twitter.com/premium/status/1758837061786779942",
-		Text:           "no ads, just bangers\n\naka your For You feed with Premium+\n\nsubscribe here → https://t.co/APTO1t7kMk",
+		Text:           "no ads, just bangers\n\naka your For You feed with Premium+\n\nsubscribe here → https://x.com/i/premium_sign_up",
 		Timestamp:      1708174407,
 		UserID:         "1399766153053061121",
 		Username:       "premium",
--- a/types.go
+++ b/types.go
@ -10,6 +10,14 @@ type (
 		Name     string
 	}

+	// Url represents a URL with display, expanded, and index data.
+	Url struct {
+		DisplayURL  string `json:"display_url"`
+		ExpandedURL string `json:"expanded_url"`
+		URL         string `json:"url"`
+		Indices     []int  `json:"indices"`
+	}
+
 	// Photo type.
 	Photo struct {
 		ID  string
@ -91,6 +99,25 @@ type (
 		GIFs      []GIF
 	}

+	ExtendedMedia struct {
+		IDStr                    string `json:"id_str"`
+		MediaURLHttps            string `json:"media_url_https"`
+		ExtSensitiveMediaWarning struct {
+			AdultContent    bool `json:"adult_content"`
+			GraphicViolence bool `json:"graphic_violence"`
+			Other           bool `json:"other"`
+		} `json:"ext_sensitive_media_warning"`
+		Type      string `json:"type"`
+		URL       string `json:"url"`
+		VideoInfo struct {
+			Variants []struct {
+				Type    string `json:"content_type"`
+				Bitrate int    `json:"bitrate"`
+				URL     string `json:"url"`
+			} `json:"variants"`
+		} `json:"video_info"`
+	}
+
 	legacyTweet struct {
 		ConversationIDStr string `json:"conversation_id_str"`
 		CreatedAt         string `json:"created_at"`
@ -105,10 +132,7 @@ type (
 				Type          string `json:"type"`
 				URL           string `json:"url"`
 			} `json:"media"`
-			URLs []struct {
-				ExpandedURL string `json:"expanded_url"`
-				URL         string `json:"url"`
-			} `json:"urls"`
+			URLs         []Url `json:"urls"`
 			UserMentions []struct {
 				IDStr      string `json:"id_str"`
 				Name       string `json:"name"`
@ -116,24 +140,7 @@ type (
 			} `json:"user_mentions"`
 		} `json:"entities"`
 		ExtendedEntities struct {
-			Media []struct {
-				IDStr                    string `json:"id_str"`
-				MediaURLHttps            string `json:"media_url_https"`
-				ExtSensitiveMediaWarning struct {
-					AdultContent    bool `json:"adult_content"`
-					GraphicViolence bool `json:"graphic_violence"`
-					Other           bool `json:"other"`
-				} `json:"ext_sensitive_media_warning"`
-				Type      string `json:"type"`
-				URL       string `json:"url"`
-				VideoInfo struct {
-					Variants []struct {
-						Type    string `json:"content_type"`
-						Bitrate int    `json:"bitrate"`
-						URL     string `json:"url"`
-					} `json:"variants"`
-				} `json:"video_info"`
-			} `json:"media"`
+			Media []ExtendedMedia `json:"media"`
 		} `json:"extended_entities"`
 		IDStr                 string `json:"id_str"`
 		InReplyToStatusIDStr  string `json:"in_reply_to_status_id_str"`
@ -166,22 +173,37 @@ type (
 				} `json:"urls"`
 			} `json:"url"`
 		} `json:"entities"`
-		FavouritesCount      int      `json:"favourites_count"`
-		FollowersCount       int      `json:"followers_count"`
-		FriendsCount         int      `json:"friends_count"`
-		IDStr                string   `json:"id_str"`
-		ListedCount          int      `json:"listed_count"`
-		Name                 string   `json:"name"`
-		Location             string   `json:"location"`
-		PinnedTweetIdsStr    []string `json:"pinned_tweet_ids_str"`
-		ProfileBannerURL     string   `json:"profile_banner_url"`
-		ProfileImageURLHTTPS string   `json:"profile_image_url_https"`
-		Protected            bool     `json:"protected"`
-		ScreenName           string   `json:"screen_name"`
-		StatusesCount        int      `json:"statuses_count"`
-		Verified             bool     `json:"verified"`
-		FollowedBy           bool     `json:"followed_by"`
-		Following            bool     `json:"following"`
+		FavouritesCount         int      `json:"favourites_count"`
+		FollowersCount          int      `json:"followers_count"`
+		FriendsCount            int      `json:"friends_count"`
+		IDStr                   string   `json:"id_str"`
+		ListedCount             int      `json:"listed_count"`
+		Name                    string   `json:"name"`
+		Location                string   `json:"location"`
+		PinnedTweetIdsStr       []string `json:"pinned_tweet_ids_str"`
+		ProfileBannerURL        string   `json:"profile_banner_url"`
+		ProfileImageURLHTTPS    string   `json:"profile_image_url_https"`
+		Protected               bool     `json:"protected"`
+		ScreenName              string   `json:"screen_name"`
+		StatusesCount           int      `json:"statuses_count"`
+		Verified                bool     `json:"verified"`
+		FollowedBy              bool     `json:"followed_by"`
+		Following               bool     `json:"following"`
+		CanDm                   bool     `json:"can_dm"`
+		CanMediaTag             bool     `json:"can_media_tag"`
+		DefaultProfile          bool     `json:"default_profile"`
+		DefaultProfileImage     bool     `json:"default_profile_image"`
+		FastFollowersCount      int      `json:"fast_followers_count"`
+		HasCustomTimelines      bool     `json:"has_custom_timelines"`
+		IsTranslator            bool     `json:"is_translator"`
+		MediaCount              int      `json:"media_count"`
+		NeedsPhoneVerification  bool     `json:"needs_phone_verification"`
+		NormalFollowersCount    int      `json:"normal_followers_count"`
+		PossiblySensitive       bool     `json:"possibly_sensitive"`
+		ProfileInterstitialType string   `json:"profile_interstitial_type"`
+		TranslatorType          string   `json:"translator_type"`
+		WantRetweets            bool     `json:"want_retweets"`
+		WithheldInCountries     []string `json:"withheld_in_countries"`
 	}

 	legacyUserV2 struct {
@ -195,15 +217,10 @@ type (
 		Description         string `json:"description"`
 		Entities            struct {
 			Description struct {
-				Urls []interface{} `json:"urls"`
+				Urls []Url `json:"urls"`
 			} `json:"description"`
 			URL struct {
-				Urls []struct {
-					DisplayURL  string `json:"display_url"`
-					ExpandedURL string `json:"expanded_url"`
-					URL         string `json:"url"`
-					Indices     []int  `json:"indices"`
-				} `json:"urls"`
+				Urls []Url `json:"urls"`
 			} `json:"url"`
 		} `json:"entities"`
 		FastFollowersCount      int           `json:"fast_followers_count"`
@ -246,4 +263,37 @@ type (

 	fetchProfileFunc func(query string, maxProfilesNbr int, cursor string) ([]*Profile, string, error)
 	fetchTweetFunc   func(query string, maxTweetsNbr int, cursor string) ([]*Tweet, string, error)
+
+	legacyExtendedProfile struct {
+		Birthdate struct {
+			Day            int    `json:"day"`
+			Month          int    `json:"month"`
+			Year           int    `json:"year"`
+			Visibility     string `json:"visibility"`
+			YearVisibility string `json:"year_visibility"`
+		} `json:"birthdate"`
+	}
+
+	verificationInfo struct {
+		IsIdentityVerified bool `json:"is_identity_verified"`
+		Reason             struct {
+			Description struct {
+				Text     string `json:"text"`
+				Entities []struct {
+					FromIndex int `json:"from_index"`
+					ToIndex   int `json:"to_index"`
+					Ref       struct {
+						URL     string `json:"url"`
+						URLType string `json:"url_type"`
+					} `json:"ref"`
+				} `json:"entities"`
+			} `json:"description"`
+			VerifiedSinceMsec string `json:"verified_since_msec"`
+		} `json:"reason"`
+	}
+
+	highlightsInfo struct {
+		CanHighlightTweets bool   `json:"can_highlight_tweets"`
+		HighlightedTweets  string `json:"highlighted_tweets"`
+	}
 )
--- a/util.go
+++ b/util.go
@ -157,6 +157,7 @@ func parseLegacyTweet(user *legacyUser, tweet *legacyTweet) *Tweet {
 	if tweetID == "" {
 		return nil
 	}
+	text := expandURLs(tweet.FullText, tweet.Entities.URLs, tweet.ExtendedEntities.Media)
 	username := user.ScreenName
 	name := user.Name
 	tw := &Tweet{
@ -167,7 +168,7 @@ func parseLegacyTweet(user *legacyUser, tweet *legacyTweet) *Tweet {
 		PermanentURL:   fmt.Sprintf("https://twitter.com/%s/status/%s", username, tweetID),
 		Replies:        tweet.ReplyCount,
 		Retweets:       tweet.RetweetCount,
-		Text:           tweet.FullText,
+		Text:           text,
 		UserID:         tweet.UserIDStr,
 		Username:       username,
 	}
@ -307,7 +308,7 @@ func parseLegacyTweet(user *legacyUser, tweet *legacyTweet) *Tweet {
 	tw.HTML = reTwitterURL.ReplaceAllStringFunc(tw.HTML, func(tco string) string {
 		for _, entity := range tweet.Entities.URLs {
 			if tco == entity.URL {
-				return fmt.Sprintf(`<a href="%s">%s</a>`, entity.ExpandedURL, tco)
+				return fmt.Sprintf(`<a href="%s">%s</a>`, entity.ExpandedURL, entity.ExpandedURL)
 			}
 		}
 		for _, entity := range tweet.ExtendedEntities.Media {
@ -345,25 +346,28 @@ func parseLegacyTweet(user *legacyUser, tweet *legacyTweet) *Tweet {

 func parseProfile(user legacyUser) Profile {
 	profile := Profile{
-		Avatar:         user.ProfileImageURLHTTPS,
-		Banner:         user.ProfileBannerURL,
-		Biography:      user.Description,
-		FollowersCount: user.FollowersCount,
-		FollowingCount: user.FavouritesCount,
-		FriendsCount:   user.FriendsCount,
-		IsVerified:     user.Verified,
-		IsPrivate:      user.Protected,
-		LikesCount:     user.FavouritesCount,
-		ListedCount:    user.ListedCount,
-		Location:       user.Location,
-		Name:           user.Name,
-		PinnedTweetIDs: user.PinnedTweetIdsStr,
-		TweetsCount:    user.StatusesCount,
-		URL:            "https://twitter.com/" + user.ScreenName,
-		UserID:         user.IDStr,
-		Username:       user.ScreenName,
-		FollowedBy:     user.FollowedBy,
-		Following:      user.Following,
+		Avatar:               user.ProfileImageURLHTTPS,
+		Banner:               user.ProfileBannerURL,
+		Biography:            user.Description,
+		FollowersCount:       user.FollowersCount,
+		FollowingCount:       user.FavouritesCount,
+		FriendsCount:         user.FriendsCount,
+		IsVerified:           user.Verified,
+		IsPrivate:            user.Protected,
+		LikesCount:           user.FavouritesCount,
+		ListedCount:          user.ListedCount,
+		Location:             user.Location,
+		Name:                 user.Name,
+		PinnedTweetIDs:       user.PinnedTweetIdsStr,
+		TweetsCount:          user.StatusesCount,
+		URL:                  "https://twitter.com/" + user.ScreenName,
+		UserID:               user.IDStr,
+		Username:             user.ScreenName,
+		FollowedBy:           user.FollowedBy,
+		Following:            user.Following,
+		MediaCount:           user.MediaCount,
+		FastFollowersCount:   user.FastFollowersCount,
+		NormalFollowersCount: user.NormalFollowersCount,
 	}

 	tm, err := time.Parse(time.RubyDate, user.CreatedAt)
@ -379,28 +383,44 @@ func parseProfile(user legacyUser) Profile {
 	return profile
 }

+func expandURLs(text string, urls []Url, extendedMediaEntities []ExtendedMedia) string {
+	expandedText := text
+	for _, url := range urls {
+		expandedText = strings.ReplaceAll(expandedText, url.URL, url.ExpandedURL)
+	}
+	for _, entity := range extendedMediaEntities {
+		expandedText = strings.ReplaceAll(expandedText, entity.URL, entity.MediaURLHttps)
+	}
+
+	return expandedText
+}
+
 func parseProfileV2(user userResult) Profile {
 	u := user.Legacy
+	description := expandURLs(u.Description, u.Entities.Description.Urls, []ExtendedMedia{})
 	profile := Profile{
-		Avatar:         u.ProfileImageURLHTTPS,
-		Banner:         u.ProfileBannerURL,
-		Biography:      u.Description,
-		FollowersCount: u.FollowersCount,
-		FollowingCount: u.FavouritesCount,
-		FriendsCount:   u.FriendsCount,
-		IsVerified:     u.Verified,
-		LikesCount:     u.FavouritesCount,
-		ListedCount:    u.ListedCount,
-		Location:       u.Location,
-		Name:           u.Name,
-		PinnedTweetIDs: u.PinnedTweetIdsStr,
-		TweetsCount:    u.StatusesCount,
-		URL:            "https://twitter.com/" + u.ScreenName,
-		UserID:         user.ID,
-		Username:       u.ScreenName,
-		Sensitive:      u.PossiblySensitive,
-		Following:      u.Following,
-		FollowedBy:     u.FollowedBy,
+		Avatar:             u.ProfileImageURLHTTPS,
+		Banner:             u.ProfileBannerURL,
+		Biography:          description,
+		FollowersCount:     u.FollowersCount,
+		FollowingCount:     u.FavouritesCount,
+		FriendsCount:       u.FriendsCount,
+		IsVerified:         u.Verified,
+		IsBlueVerified:     user.IsBlueVerified,
+		ProfileImageShape:  user.ProfileImageShape,
+		HasGraduatedAccess: user.HasGraduatedAccess,
+		LikesCount:         u.FavouritesCount,
+		ListedCount:        u.ListedCount,
+		Location:           u.Location,
+		Name:               u.Name,
+		PinnedTweetIDs:     u.PinnedTweetIdsStr,
+		TweetsCount:        u.StatusesCount,
+		URL:                "https://twitter.com/" + u.ScreenName,
+		UserID:             user.ID,
+		Username:           u.ScreenName,
+		Sensitive:          u.PossiblySensitive,
+		Following:          u.Following,
+		FollowedBy:         u.FollowedBy,
 	}

 	tm, err := time.Parse(time.RubyDate, u.CreatedAt)
Author	SHA1	Message	Date
Valentine	76cb95cd3e	Merge pull request #48 from imperatrona/hotfix Some checks are pending CodeQL / Analyze (push) Waiting to run Go / Test (push) Waiting to run rm bearerToken2	2025-04-30 01:06:44 +03:00
imperatrona	d31ee7d09f	rm bearerToken2	2025-04-30 01:03:40 +03:00
Valentine	f5a2629e42	Merge pull request #44 from imperatrona/video_website parse videos from cards	2025-03-28 14:27:00 +03:00
Valentine	a48dcf2587	parse videos from cards	2025-03-28 14:14:16 +03:00
Valentine	3684e678e3	update tests	2025-01-23 03:13:18 +03:00
Valentine	c269a9ba92	Merge pull request #24 from thewh1teagle/feat/expand-description-urls feat: expand description URLs	2025-01-23 03:03:43 +03:00
Valentine	e5c170986e	Merge branch 'master' into feat/expand-description-urls	2025-01-23 02:16:38 +03:00
Valentine	44183226af	update twitter account in test	2025-01-23 00:00:32 +03:00
Valentine	4276e243ac	fix test fail	2025-01-22 23:57:00 +03:00
Valentine	4519142ae8	Merge pull request #35 from masa-finance/master Add Twitter Blue Verification Status to Profile Responses	2025-01-22 23:52:05 +03:00
Brendan Playford	48d27acfb1	Merge pull request #1 from masa-finance/feat-update-profile-to-twitter-v2-api Add Twitter Blue Verification Status to Profile Responses	2025-01-22 11:22:20 -08:00
Brendan Playford	388445d4c2	feat: add IsBlueVerified to profile responses Add support for Twitter's blue verification status in profile responses: - Add IsBlueVerified field to user struct in GetProfile response - Update parseProfile and parseProfileV2 to include blue verification status - Add ProfileImageShape and HasGraduatedAccess fields to profile responses This change ensures the API correctly returns blue verification status for both direct profile queries and timeline responses.	2025-01-22 11:14:03 -08:00
Brendan Playford	7226460e05	feat(types): extend Twitter profile data structures - Add new fields to Profile struct (IsBlueVerified, MediaCount, etc.) - Extend legacyUser struct with additional Twitter API fields - Add new types for extended profile, verification and highlights info - Update profile parsing to include new metrics	2025-01-22 11:05:38 -08:00
thewh1teagle	40066f125a	feat: single function to expand URLs	2024-11-08 21:57:56 +02:00
thewh1teagle	cc1eb793d6	feat: expand media URLs in tweet	2024-11-08 21:41:52 +02:00
thewh1teagle	9f31f3890f	feat: expand URLs for profile and tweets	2024-10-14 21:31:13 +03:00