lain/twitter-scrapper
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity

feat: single function to expand URLs

Browse source
This commit is contained in:
thewh1teagle 2024-11-08 21:57:56 +02:00
parent cc1eb793d6
commit 40066f125a
No known key found for this signature in database
2 changed files with 6 additions and 9 deletions
Download patch file Download diff file Expand all files Collapse all files

3
timeline_v2.go
View file

@ -56,7 +56,8 @@ func (result *result) parse() *Tweet {
if result.QuotedStatusResult.Result != nil {
tw.QuotedStatus = result.QuotedStatusResult.Result.parse()
}
tw.HTML = expandMediaURLs(tw.HTML, legacy.ExtendedEntities.Media)
tw.HTML = expandURLs(tw.HTML, legacy.Entities.URLs, legacy.ExtendedEntities.Media)
tw.HTML = expandURLs(tw.Text, legacy.Entities.URLs, legacy.ExtendedEntities.Media)
return tw
}

12
util.go
View file

@ -157,7 +157,7 @@ func parseLegacyTweet(user *legacyUser, tweet *legacyTweet) *Tweet {
if tweetID == "" {
return nil
}
text := expandURLs(tweet.FullText, tweet.Entities.URLs)
text := expandURLs(tweet.FullText, tweet.Entities.URLs, tweet.ExtendedEntities.Media)
username := user.ScreenName
name := user.Name
tw := &Tweet{
@ -380,25 +380,21 @@ func parseProfile(user legacyUser) Profile {
return profile
}
func expandURLs(text string, urls []Url) string {
func expandURLs(text string, urls []Url, extendedMediaEntities []ExtendedMedia) string {
expandedText := text
for _, url := range urls {
expandedText = strings.ReplaceAll(expandedText, url.URL, url.ExpandedURL)
}
return expandedText
}
func expandMediaURLs(text string, extendedMediaEntities []ExtendedMedia) string {
expandedText := text
for _, entity := range extendedMediaEntities {
expandedText = strings.ReplaceAll(expandedText, entity.URL, entity.MediaURLHttps)
}
return expandedText
}
func parseProfileV2(user userResult) Profile {
u := user.Legacy
description := expandURLs(u.Description, u.Entities.Description.Urls)
description := expandURLs(u.Description, u.Entities.Description.Urls, []ExtendedMedia{})
profile := Profile{
Avatar: u.ProfileImageURLHTTPS,
Banner: u.ProfileBannerURL,