lain/twitter-scrapper
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity

Merge pull request #24 from thewh1teagle/feat/expand-description-urls

Browse source 
feat: expand description URLs
This commit is contained in:
Valentine 2025-01-23 03:03:43 +03:00 • committed by GitHub
commit c269a9ba92
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
3 changed files with 49 additions and 31 deletions
Download patch file Download diff file Expand all files Collapse all files

2
timeline_v2.go
View file

@ -56,6 +56,8 @@ func (result *result) parse() *Tweet {
if result.QuotedStatusResult.Result != nil {
tw.QuotedStatus = result.QuotedStatusResult.Result.parse()
}
tw.HTML = expandURLs(tw.HTML, legacy.Entities.URLs, legacy.ExtendedEntities.Media)
tw.HTML = expandURLs(tw.Text, legacy.Entities.URLs, legacy.ExtendedEntities.Media)
return tw
}

68
types.go
View file

@ -10,6 +10,14 @@ type (
Name string
}
// Url represents a URL with display, expanded, and index data.
Url struct {
DisplayURL string `json:"display_url"`
ExpandedURL string `json:"expanded_url"`
URL string `json:"url"`
Indices []int `json:"indices"`
}
// Photo type.
Photo struct {
ID string
@ -91,32 +99,7 @@ type (
GIFs []GIF
}
legacyTweet struct {
ConversationIDStr string `json:"conversation_id_str"`
CreatedAt string `json:"created_at"`
FavoriteCount int `json:"favorite_count"`
FullText string `json:"full_text"`
Entities struct {
Hashtags []struct {
Text string `json:"text"`
} `json:"hashtags"`
Media []struct {
MediaURLHttps string `json:"media_url_https"`
Type string `json:"type"`
URL string `json:"url"`
} `json:"media"`
URLs []struct {
ExpandedURL string `json:"expanded_url"`
URL string `json:"url"`
} `json:"urls"`
UserMentions []struct {
IDStr string `json:"id_str"`
Name string `json:"name"`
ScreenName string `json:"screen_name"`
} `json:"user_mentions"`
} `json:"entities"`
ExtendedEntities struct {
Media []struct {
ExtendedMedia struct {
IDStr string `json:"id_str"`
MediaURLHttps string `json:"media_url_https"`
ExtSensitiveMediaWarning struct {
@ -133,7 +116,31 @@ type (
URL string `json:"url"`
} `json:"variants"`
} `json:"video_info"`
}
legacyTweet struct {
ConversationIDStr string `json:"conversation_id_str"`
CreatedAt string `json:"created_at"`
FavoriteCount int `json:"favorite_count"`
FullText string `json:"full_text"`
Entities struct {
Hashtags []struct {
Text string `json:"text"`
} `json:"hashtags"`
Media []struct {
MediaURLHttps string `json:"media_url_https"`
Type string `json:"type"`
URL string `json:"url"`
} `json:"media"`
URLs []Url `json:"urls"`
UserMentions []struct {
IDStr string `json:"id_str"`
Name string `json:"name"`
ScreenName string `json:"screen_name"`
} `json:"user_mentions"`
} `json:"entities"`
ExtendedEntities struct {
Media []ExtendedMedia `json:"media"`
} `json:"extended_entities"`
IDStr string `json:"id_str"`
InReplyToStatusIDStr string `json:"in_reply_to_status_id_str"`
@ -210,15 +217,10 @@ type (
Description string `json:"description"`
Entities struct {
Description struct {
Urls []interface{} `json:"urls"`
Urls []Url `json:"urls"`
} `json:"description"`
URL struct {
Urls []struct {
DisplayURL string `json:"display_url"`
ExpandedURL string `json:"expanded_url"`
URL string `json:"url"`
Indices []int `json:"indices"`
} `json:"urls"`
Urls []Url `json:"urls"`
} `json:"url"`
} `json:"entities"`
FastFollowersCount int `json:"fast_followers_count"`

18
util.go
View file

@ -157,6 +157,7 @@ func parseLegacyTweet(user *legacyUser, tweet *legacyTweet) *Tweet {
if tweetID == "" {
return nil
}
text := expandURLs(tweet.FullText, tweet.Entities.URLs, tweet.ExtendedEntities.Media)
username := user.ScreenName
name := user.Name
tw := &Tweet{
@ -167,7 +168,7 @@ func parseLegacyTweet(user *legacyUser, tweet *legacyTweet) *Tweet {
PermanentURL: fmt.Sprintf("https://twitter.com/%s/status/%s", username, tweetID),
Replies: tweet.ReplyCount,
Retweets: tweet.RetweetCount,
Text: tweet.FullText,
Text: text,
UserID: tweet.UserIDStr,
Username: username,
}
@ -382,12 +383,25 @@ func parseProfile(user legacyUser) Profile {
return profile
}
func expandURLs(text string, urls []Url, extendedMediaEntities []ExtendedMedia) string {
expandedText := text
for _, url := range urls {
expandedText = strings.ReplaceAll(expandedText, url.URL, url.ExpandedURL)
}
for _, entity := range extendedMediaEntities {
expandedText = strings.ReplaceAll(expandedText, entity.URL, entity.MediaURLHttps)
}
return expandedText
}
func parseProfileV2(user userResult) Profile {
u := user.Legacy
description := expandURLs(u.Description, u.Entities.Description.Urls, []ExtendedMedia{})
profile := Profile{
Avatar: u.ProfileImageURLHTTPS,
Banner: u.ProfileBannerURL,
Biography: u.Description,
Biography: description,
FollowersCount: u.FollowersCount,
FollowingCount: u.FavouritesCount,
FriendsCount: u.FriendsCount,