Merge pull request #24 from thewh1teagle/feat/expand-description-urls

feat: expand description URLs

timeline_v2.go

@@ -56,6 +56,8 @@ func (result *result) parse() *Tweet {
 	if result.QuotedStatusResult.Result != nil {
 		tw.QuotedStatus = result.QuotedStatusResult.Result.parse()
 	}
+	tw.HTML = expandURLs(tw.HTML, legacy.Entities.URLs, legacy.ExtendedEntities.Media)
+	tw.HTML = expandURLs(tw.Text, legacy.Entities.URLs, legacy.ExtendedEntities.Media)
 	return tw
 }
 

types.go

@@ -10,6 +10,14 @@ type (
 		Name     string
 	}
 
+	// Url represents a URL with display, expanded, and index data.
+	Url struct {
+		DisplayURL  string `json:"display_url"`
+		ExpandedURL string `json:"expanded_url"`
+		URL         string `json:"url"`
+		Indices     []int  `json:"indices"`
+	}
+
 	// Photo type.
 	Photo struct {
 		ID  string
@@ -91,6 +99,25 @@ type (
 		GIFs      []GIF
 	}
 
+	ExtendedMedia struct {
+		IDStr                    string `json:"id_str"`
+		MediaURLHttps            string `json:"media_url_https"`
+		ExtSensitiveMediaWarning struct {
+			AdultContent    bool `json:"adult_content"`
+			GraphicViolence bool `json:"graphic_violence"`
+			Other           bool `json:"other"`
+		} `json:"ext_sensitive_media_warning"`
+		Type      string `json:"type"`
+		URL       string `json:"url"`
+		VideoInfo struct {
+			Variants []struct {
+				Type    string `json:"content_type"`
+				Bitrate int    `json:"bitrate"`
+				URL     string `json:"url"`
+			} `json:"variants"`
+		} `json:"video_info"`
+	}
+
 	legacyTweet struct {
 		ConversationIDStr string `json:"conversation_id_str"`
 		CreatedAt         string `json:"created_at"`
@@ -105,10 +132,7 @@ type (
 				Type          string `json:"type"`
 				URL           string `json:"url"`
 			} `json:"media"`
-			URLs []struct {
+			URLs         []Url `json:"urls"`
-				ExpandedURL string `json:"expanded_url"`
-				URL         string `json:"url"`
-			} `json:"urls"`
 			UserMentions []struct {
 				IDStr      string `json:"id_str"`
 				Name       string `json:"name"`
@@ -116,24 +140,7 @@ type (
 			} `json:"user_mentions"`
 		} `json:"entities"`
 		ExtendedEntities struct {
-			Media []struct {
+			Media []ExtendedMedia `json:"media"`
-				IDStr                    string `json:"id_str"`
-				MediaURLHttps            string `json:"media_url_https"`
-				ExtSensitiveMediaWarning struct {
-					AdultContent    bool `json:"adult_content"`
-					GraphicViolence bool `json:"graphic_violence"`
-					Other           bool `json:"other"`
-				} `json:"ext_sensitive_media_warning"`
-				Type      string `json:"type"`
-				URL       string `json:"url"`
-				VideoInfo struct {
-					Variants []struct {
-						Type    string `json:"content_type"`
-						Bitrate int    `json:"bitrate"`
-						URL     string `json:"url"`
-					} `json:"variants"`
-				} `json:"video_info"`
-			} `json:"media"`
 		} `json:"extended_entities"`
 		IDStr                 string `json:"id_str"`
 		InReplyToStatusIDStr  string `json:"in_reply_to_status_id_str"`
@@ -210,15 +217,10 @@ type (
 		Description         string `json:"description"`
 		Entities            struct {
 			Description struct {
-				Urls []interface{} `json:"urls"`
+				Urls []Url `json:"urls"`
 			} `json:"description"`
 			URL struct {
-				Urls []struct {
+				Urls []Url `json:"urls"`
-					DisplayURL  string `json:"display_url"`
-					ExpandedURL string `json:"expanded_url"`
-					URL         string `json:"url"`
-					Indices     []int  `json:"indices"`
-				} `json:"urls"`
 			} `json:"url"`
 		} `json:"entities"`
 		FastFollowersCount      int           `json:"fast_followers_count"`

util.go

@@ -157,6 +157,7 @@ func parseLegacyTweet(user *legacyUser, tweet *legacyTweet) *Tweet {
 	if tweetID == "" {
 		return nil
 	}
+	text := expandURLs(tweet.FullText, tweet.Entities.URLs, tweet.ExtendedEntities.Media)
 	username := user.ScreenName
 	name := user.Name
 	tw := &Tweet{
@@ -167,7 +168,7 @@ func parseLegacyTweet(user *legacyUser, tweet *legacyTweet) *Tweet {
 		PermanentURL:   fmt.Sprintf("https://twitter.com/%s/status/%s", username, tweetID),
 		Replies:        tweet.ReplyCount,
 		Retweets:       tweet.RetweetCount,
-		Text:           tweet.FullText,
+		Text:           text,
 		UserID:         tweet.UserIDStr,
 		Username:       username,
 	}
@@ -382,12 +383,25 @@ func parseProfile(user legacyUser) Profile {
 	return profile
 }
 
+func expandURLs(text string, urls []Url, extendedMediaEntities []ExtendedMedia) string {
+	expandedText := text
+	for _, url := range urls {
+		expandedText = strings.ReplaceAll(expandedText, url.URL, url.ExpandedURL)
+	}
+	for _, entity := range extendedMediaEntities {
+		expandedText = strings.ReplaceAll(expandedText, entity.URL, entity.MediaURLHttps)
+	}
+
+	return expandedText
+}
+
 func parseProfileV2(user userResult) Profile {
 	u := user.Legacy
+	description := expandURLs(u.Description, u.Entities.Description.Urls, []ExtendedMedia{})
 	profile := Profile{
 		Avatar:             u.ProfileImageURLHTTPS,
 		Banner:             u.ProfileBannerURL,
-		Biography:          u.Description,
+		Biography:          description,
 		FollowersCount:     u.FollowersCount,
 		FollowingCount:     u.FavouritesCount,
 		FriendsCount:       u.FriendsCount,