lain/twitter-scrapper
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity

feat: single function to expand URLs

Browse source
This commit is contained in:
thewh1teagle 2024-11-08 21:57:56 +02:00
parent cc1eb793d6
commit 40066f125a
No known key found for this signature in database
2 changed files with 6 additions and 9 deletions
Download patch file Download diff file Expand all files Collapse all files

3
timeline_v2.go
View file

@ -56,7 +56,8 @@ func (result *result) parse() *Tweet {
if result.QuotedStatusResult.Result != nil { if result.QuotedStatusResult.Result != nil {
tw.QuotedStatus = result.QuotedStatusResult.Result.parse() tw.QuotedStatus = result.QuotedStatusResult.Result.parse()
} }
tw.HTML = expandMediaURLs(tw.HTML, legacy.ExtendedEntities.Media) tw.HTML = expandURLs(tw.HTML, legacy.Entities.URLs, legacy.ExtendedEntities.Media)
tw.HTML = expandURLs(tw.Text, legacy.Entities.URLs, legacy.ExtendedEntities.Media)
return tw return tw
} }

12
util.go
View file

@ -157,7 +157,7 @@ func parseLegacyTweet(user *legacyUser, tweet *legacyTweet) *Tweet {
if tweetID == "" { if tweetID == "" {
return nil return nil
} }
text := expandURLs(tweet.FullText, tweet.Entities.URLs) text := expandURLs(tweet.FullText, tweet.Entities.URLs, tweet.ExtendedEntities.Media)
username := user.ScreenName username := user.ScreenName
name := user.Name name := user.Name
tw := &Tweet{ tw := &Tweet{
@ -380,25 +380,21 @@ func parseProfile(user legacyUser) Profile {
return profile return profile
} }
func expandURLs(text string, urls []Url) string { func expandURLs(text string, urls []Url, extendedMediaEntities []ExtendedMedia) string {
expandedText := text expandedText := text
for _, url := range urls { for _, url := range urls {
expandedText = strings.ReplaceAll(expandedText, url.URL, url.ExpandedURL) expandedText = strings.ReplaceAll(expandedText, url.URL, url.ExpandedURL)
} }
return expandedText
}
func expandMediaURLs(text string, extendedMediaEntities []ExtendedMedia) string {
expandedText := text
for _, entity := range extendedMediaEntities { for _, entity := range extendedMediaEntities {
expandedText = strings.ReplaceAll(expandedText, entity.URL, entity.MediaURLHttps) expandedText = strings.ReplaceAll(expandedText, entity.URL, entity.MediaURLHttps)
} }
return expandedText return expandedText
} }
func parseProfileV2(user userResult) Profile { func parseProfileV2(user userResult) Profile {
u := user.Legacy u := user.Legacy
description := expandURLs(u.Description, u.Entities.Description.Urls) description := expandURLs(u.Description, u.Entities.Description.Urls, []ExtendedMedia{})
profile := Profile{ profile := Profile{
Avatar: u.ProfileImageURLHTTPS, Avatar: u.ProfileImageURLHTTPS,
Banner: u.ProfileBannerURL, Banner: u.ProfileBannerURL,