lain/twitter-scrapper
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity

Extend timeline object

Browse source
This commit is contained in:
Alexander Sheiko 2021-07-16 11:08:43 +03:00
parent 0183c20011
commit 34aae0d008
5 changed files with 311 additions and 306 deletions
Download patch file Download diff file Expand all files Collapse all files

163
util.go
View file

@ -2,11 +2,9 @@ package twitterscraper
import (
"context"
"fmt"
"net/http"
"regexp"
"strconv"
"strings"
"time"
)
@ -177,164 +175,3 @@ func parseProfile(user legacyUser) Profile {
return profile
}
func parseTimeline(timeline *timeline) ([]*Tweet, string) {
tweets := make(map[string]Tweet)
for id, tweet := range timeline.GlobalObjects.Tweets {
username := timeline.GlobalObjects.Users[tweet.UserIDStr].ScreenName
tw := Tweet{
ID: id,
Likes: tweet.FavoriteCount,
PermanentURL: fmt.Sprintf("https://twitter.com/%s/status/%s", username, id),
Replies: tweet.ReplyCount,
Retweets: tweet.RetweetCount,
Text: tweet.FullText,
UserID: tweet.UserIDStr,
Username: username,
}
tm, err := time.Parse(time.RubyDate, tweet.CreatedAt)
if err == nil {
tw.TimeParsed = tm
tw.Timestamp = tm.Unix()
}
if tweet.QuotedStatusIDStr != "" {
tw.IsQuoted = true
}
if tweet.InReplyToStatusIDStr != "" {
tw.IsReply = true
}
if tweet.RetweetedStatusIDStr != "" {
tw.IsRetweet = true
if retweet, ok := timeline.GlobalObjects.Tweets[tweet.RetweetedStatusIDStr]; ok {
tw.Retweet = Retweet{
ID: tweet.RetweetedStatusIDStr,
UserID: retweet.UserIDStr,
Username: timeline.GlobalObjects.Users[retweet.UserIDStr].ScreenName,
}
tm, err := time.Parse(time.RubyDate, retweet.CreatedAt)
if err == nil {
tw.Retweet.TimeParsed = tm
tw.Retweet.Timestamp = tm.Unix()
}
}
}
for _, pinned := range timeline.GlobalObjects.Users[tweet.UserIDStr].PinnedTweetIdsStr {
if tweet.ConversationIDStr == pinned {
tw.IsPin = true
break
}
}
for _, hash := range tweet.Entities.Hashtags {
tw.Hashtags = append(tw.Hashtags, hash.Text)
}
for _, media := range tweet.Entities.Media {
if media.Type == "photo" {
tw.Photos = append(tw.Photos, media.MediaURLHttps)
}
}
for _, media := range tweet.ExtendedEntities.Media {
if media.Type == "video" {
video := Video{
ID: media.IDStr,
Preview: media.MediaURLHttps,
}
maxBitrate := 0
for _, variant := range media.VideoInfo.Variants {
if variant.Bitrate > maxBitrate {
video.URL = strings.TrimSuffix(variant.URL, "?tag=10")
}
}
tw.Videos = append(tw.Videos, video)
}
}
for _, url := range tweet.Entities.URLs {
tw.URLs = append(tw.URLs, url.ExpandedURL)
}
tw.HTML = tweet.FullText
tw.HTML = reHashtag.ReplaceAllStringFunc(tw.HTML, func(hashtag string) string {
return fmt.Sprintf(`<a href="https://twitter.com/hashtag/%s">%s</a>`,
strings.TrimPrefix(hashtag, "#"),
hashtag,
)
})
tw.HTML = reUsername.ReplaceAllStringFunc(tw.HTML, func(username string) string {
return fmt.Sprintf(`<a href="https://twitter.com/%s">%s</a>`,
strings.TrimPrefix(username, "@"),
username,
)
})
tw.HTML = reTwitterURL.ReplaceAllStringFunc(tw.HTML, func(tco string) string {
for _, entity := range tweet.Entities.URLs {
if tco == entity.URL {
return fmt.Sprintf(`<a href="%s">%s</a>`, entity.ExpandedURL, tco)
}
}
for _, entity := range tweet.Entities.Media {
if tco == entity.URL {
return fmt.Sprintf(`<br><a href="%s"><img src="%s"/></a>`, tco, entity.MediaURLHttps)
}
}
return tco
})
tw.HTML = strings.Replace(tw.HTML, "\n", "<br>", -1)
tweets[tw.ID] = tw
}
var cursor string
var pinnedTweet *Tweet
var orderedTweets []*Tweet
for _, instruction := range timeline.Timeline.Instructions {
if instruction.PinEntry.Entry.Content.Item.Content.Tweet.ID != "" {
if tweet, ok := tweets[instruction.PinEntry.Entry.Content.Item.Content.Tweet.ID]; ok {
pinnedTweet = &tweet
}
}
for _, entry := range instruction.AddEntries.Entries {
if tweet, ok := tweets[entry.Content.Item.Content.Tweet.ID]; ok {
orderedTweets = append(orderedTweets, &tweet)
}
if entry.Content.Operation.Cursor.CursorType == "Bottom" {
cursor = entry.Content.Operation.Cursor.Value
}
}
if instruction.ReplaceEntry.Entry.Content.Operation.Cursor.CursorType == "Bottom" {
cursor = instruction.ReplaceEntry.Entry.Content.Operation.Cursor.Value
}
}
if pinnedTweet != nil && len(orderedTweets) > 0 {
orderedTweets = append([]*Tweet{pinnedTweet}, orderedTweets...)
}
return orderedTweets, cursor
}
func parseUsers(timeline *timeline) ([]*Profile, string) {
users := make(map[string]Profile)
for id, user := range timeline.GlobalObjects.Users {
users[id] = parseProfile(user)
}
var cursor string
var orderedProfiles []*Profile
for _, instruction := range timeline.Timeline.Instructions {
for _, entry := range instruction.AddEntries.Entries {
if profile, ok := users[entry.Content.Item.Content.User.ID]; ok {
orderedProfiles = append(orderedProfiles, &profile)
}
if entry.Content.Operation.Cursor.CursorType == "Bottom" {
cursor = entry.Content.Operation.Cursor.Value
}
}
if instruction.ReplaceEntry.Entry.Content.Operation.Cursor.CursorType == "Bottom" {
cursor = instruction.ReplaceEntry.Entry.Content.Operation.Cursor.Value
}
}
return orderedProfiles, cursor
}