diff --git a/timeline_v2.go b/timeline_v2.go index 372941d..3246b03 100644 --- a/timeline_v2.go +++ b/timeline_v2.go @@ -56,6 +56,8 @@ func (result *result) parse() *Tweet { if result.QuotedStatusResult.Result != nil { tw.QuotedStatus = result.QuotedStatusResult.Result.parse() } + tw.HTML = expandURLs(tw.HTML, legacy.Entities.URLs, legacy.ExtendedEntities.Media) + tw.HTML = expandURLs(tw.Text, legacy.Entities.URLs, legacy.ExtendedEntities.Media) return tw } diff --git a/types.go b/types.go index a948b39..7bd9437 100644 --- a/types.go +++ b/types.go @@ -10,6 +10,14 @@ type ( Name string } + // Url represents a URL with display, expanded, and index data. + Url struct { + DisplayURL string `json:"display_url"` + ExpandedURL string `json:"expanded_url"` + URL string `json:"url"` + Indices []int `json:"indices"` + } + // Photo type. Photo struct { ID string @@ -91,6 +99,25 @@ type ( GIFs []GIF } + ExtendedMedia struct { + IDStr string `json:"id_str"` + MediaURLHttps string `json:"media_url_https"` + ExtSensitiveMediaWarning struct { + AdultContent bool `json:"adult_content"` + GraphicViolence bool `json:"graphic_violence"` + Other bool `json:"other"` + } `json:"ext_sensitive_media_warning"` + Type string `json:"type"` + URL string `json:"url"` + VideoInfo struct { + Variants []struct { + Type string `json:"content_type"` + Bitrate int `json:"bitrate"` + URL string `json:"url"` + } `json:"variants"` + } `json:"video_info"` + } + legacyTweet struct { ConversationIDStr string `json:"conversation_id_str"` CreatedAt string `json:"created_at"` @@ -105,10 +132,7 @@ type ( Type string `json:"type"` URL string `json:"url"` } `json:"media"` - URLs []struct { - ExpandedURL string `json:"expanded_url"` - URL string `json:"url"` - } `json:"urls"` + URLs []Url `json:"urls"` UserMentions []struct { IDStr string `json:"id_str"` Name string `json:"name"` @@ -116,24 +140,7 @@ type ( } `json:"user_mentions"` } `json:"entities"` ExtendedEntities struct { - Media []struct { - IDStr string `json:"id_str"` - MediaURLHttps string `json:"media_url_https"` - ExtSensitiveMediaWarning struct { - AdultContent bool `json:"adult_content"` - GraphicViolence bool `json:"graphic_violence"` - Other bool `json:"other"` - } `json:"ext_sensitive_media_warning"` - Type string `json:"type"` - URL string `json:"url"` - VideoInfo struct { - Variants []struct { - Type string `json:"content_type"` - Bitrate int `json:"bitrate"` - URL string `json:"url"` - } `json:"variants"` - } `json:"video_info"` - } `json:"media"` + Media []ExtendedMedia `json:"media"` } `json:"extended_entities"` IDStr string `json:"id_str"` InReplyToStatusIDStr string `json:"in_reply_to_status_id_str"` @@ -210,15 +217,10 @@ type ( Description string `json:"description"` Entities struct { Description struct { - Urls []interface{} `json:"urls"` + Urls []Url `json:"urls"` } `json:"description"` URL struct { - Urls []struct { - DisplayURL string `json:"display_url"` - ExpandedURL string `json:"expanded_url"` - URL string `json:"url"` - Indices []int `json:"indices"` - } `json:"urls"` + Urls []Url `json:"urls"` } `json:"url"` } `json:"entities"` FastFollowersCount int `json:"fast_followers_count"` diff --git a/util.go b/util.go index 26264e0..e7e5311 100644 --- a/util.go +++ b/util.go @@ -157,6 +157,7 @@ func parseLegacyTweet(user *legacyUser, tweet *legacyTweet) *Tweet { if tweetID == "" { return nil } + text := expandURLs(tweet.FullText, tweet.Entities.URLs, tweet.ExtendedEntities.Media) username := user.ScreenName name := user.Name tw := &Tweet{ @@ -167,7 +168,7 @@ func parseLegacyTweet(user *legacyUser, tweet *legacyTweet) *Tweet { PermanentURL: fmt.Sprintf("https://twitter.com/%s/status/%s", username, tweetID), Replies: tweet.ReplyCount, Retweets: tweet.RetweetCount, - Text: tweet.FullText, + Text: text, UserID: tweet.UserIDStr, Username: username, } @@ -382,12 +383,25 @@ func parseProfile(user legacyUser) Profile { return profile } +func expandURLs(text string, urls []Url, extendedMediaEntities []ExtendedMedia) string { + expandedText := text + for _, url := range urls { + expandedText = strings.ReplaceAll(expandedText, url.URL, url.ExpandedURL) + } + for _, entity := range extendedMediaEntities { + expandedText = strings.ReplaceAll(expandedText, entity.URL, entity.MediaURLHttps) + } + + return expandedText +} + func parseProfileV2(user userResult) Profile { u := user.Legacy + description := expandURLs(u.Description, u.Entities.Description.Urls, []ExtendedMedia{}) profile := Profile{ Avatar: u.ProfileImageURLHTTPS, Banner: u.ProfileBannerURL, - Biography: u.Description, + Biography: description, FollowersCount: u.FollowersCount, FollowingCount: u.FavouritesCount, FriendsCount: u.FriendsCount,