lain/twitter-scrapper
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity

feat: expand URLs for profile and tweets

Browse source
This commit is contained in:
thewh1teagle 2024-10-13 14:47:05 +03:00
parent 033e8609fb
commit 9f31f3890f
No known key found for this signature in database
2 changed files with 23 additions and 13 deletions
Download patch file Download diff file Expand all files Collapse all files

22
types.go
View file

@ -10,6 +10,14 @@ type (
Name string Name string
} }
// Url represents a URL with display, expanded, and index data.
Url struct {
DisplayURL string `json:"display_url"`
ExpandedURL string `json:"expanded_url"`
URL string `json:"url"`
Indices []int `json:"indices"`
}
// Photo type. // Photo type.
Photo struct { Photo struct {
ID string ID string
@ -105,10 +113,7 @@ type (
Type string `json:"type"` Type string `json:"type"`
URL string `json:"url"` URL string `json:"url"`
} `json:"media"` } `json:"media"`
URLs []struct { URLs []Url `json:"urls"`
ExpandedURL string `json:"expanded_url"`
URL string `json:"url"`
} `json:"urls"`
UserMentions []struct { UserMentions []struct {
IDStr string `json:"id_str"` IDStr string `json:"id_str"`
Name string `json:"name"` Name string `json:"name"`
@ -195,15 +200,10 @@ type (
Description string `json:"description"` Description string `json:"description"`
Entities struct { Entities struct {
Description struct { Description struct {
Urls []interface{} `json:"urls"` Urls []Url `json:"urls"`
} `json:"description"` } `json:"description"`
URL struct { URL struct {
Urls []struct { Urls []Url `json:"urls"`
DisplayURL string `json:"display_url"`
ExpandedURL string `json:"expanded_url"`
URL string `json:"url"`
Indices []int `json:"indices"`
} `json:"urls"`
} `json:"url"` } `json:"url"`
} `json:"entities"` } `json:"entities"`
FastFollowersCount int `json:"fast_followers_count"` FastFollowersCount int `json:"fast_followers_count"`

14
util.go
View file

@ -157,6 +157,7 @@ func parseLegacyTweet(user *legacyUser, tweet *legacyTweet) *Tweet {
if tweetID == "" { if tweetID == "" {
return nil return nil
} }
text := expandURLs(tweet.FullText, tweet.Entities.URLs)
username := user.ScreenName username := user.ScreenName
name := user.Name name := user.Name
tw := &Tweet{ tw := &Tweet{
@ -167,7 +168,7 @@ func parseLegacyTweet(user *legacyUser, tweet *legacyTweet) *Tweet {
PermanentURL: fmt.Sprintf("https://twitter.com/%s/status/%s", username, tweetID), PermanentURL: fmt.Sprintf("https://twitter.com/%s/status/%s", username, tweetID),
Replies: tweet.ReplyCount, Replies: tweet.ReplyCount,
Retweets: tweet.RetweetCount, Retweets: tweet.RetweetCount,
Text: tweet.FullText, Text: text,
UserID: tweet.UserIDStr, UserID: tweet.UserIDStr,
Username: username, Username: username,
} }
@ -379,12 +380,21 @@ func parseProfile(user legacyUser) Profile {
return profile return profile
} }
func expandURLs(text string, urls []Url) string {
expandedText := text
for _, url := range urls {
expandedText = strings.ReplaceAll(expandedText, url.URL, url.ExpandedURL)
}
return expandedText
}
func parseProfileV2(user userResult) Profile { func parseProfileV2(user userResult) Profile {
u := user.Legacy u := user.Legacy
description := expandURLs(u.Description, u.Entities.Description.Urls)
profile := Profile{ profile := Profile{
Avatar: u.ProfileImageURLHTTPS, Avatar: u.ProfileImageURLHTTPS,
Banner: u.ProfileBannerURL, Banner: u.ProfileBannerURL,
Biography: u.Description, Biography: description,
FollowersCount: u.FollowersCount, FollowersCount: u.FollowersCount,
FollowingCount: u.FavouritesCount, FollowingCount: u.FavouritesCount,
FriendsCount: u.FriendsCount, FriendsCount: u.FriendsCount,