From 34aae0d00836276f475f7afb6b13990fc78e576d Mon Sep 17 00:00:00 2001 From: Alexander Sheiko Date: Fri, 16 Jul 2021 11:08:43 +0300 Subject: [PATCH] Extend timeline object --- search.go | 4 +- timeline.go | 307 ++++++++++++++++++++++++++++++++++++++++++++++++++++ tweets.go | 4 +- types.go | 139 ------------------------ util.go | 163 ---------------------------- 5 files changed, 311 insertions(+), 306 deletions(-) create mode 100644 timeline.go diff --git a/search.go b/search.go index fdd3162..81886e4 100644 --- a/search.go +++ b/search.go @@ -74,7 +74,7 @@ func (s *Scraper) FetchSearchTweets(query string, maxTweetsNbr int, cursor strin if err != nil { return nil, "", err } - tweets, nextCursor := parseTimeline(timeline) + tweets, nextCursor := timeline.parseTweets() return tweets, nextCursor, nil } @@ -84,6 +84,6 @@ func (s *Scraper) FetchSearchProfiles(query string, maxProfilesNbr int, cursor s if err != nil { return nil, "", err } - users, nextCursor := parseUsers(timeline) + users, nextCursor := timeline.parseUsers() return users, nextCursor, nil } diff --git a/timeline.go b/timeline.go new file mode 100644 index 0000000..faf791b --- /dev/null +++ b/timeline.go @@ -0,0 +1,307 @@ +package twitterscraper + +import ( + "fmt" + "strings" + "time" +) + +// timeline JSON object +type timeline struct { + GlobalObjects struct { + Tweets map[string]struct { + ConversationIDStr string `json:"conversation_id_str"` + CreatedAt string `json:"created_at"` + FavoriteCount int `json:"favorite_count"` + FullText string `json:"full_text"` + Entities struct { + Hashtags []struct { + Text string `json:"text"` + } `json:"hashtags"` + Media []struct { + MediaURLHttps string `json:"media_url_https"` + Type string `json:"type"` + URL string `json:"url"` + } `json:"media"` + URLs []struct { + ExpandedURL string `json:"expanded_url"` + URL string `json:"url"` + } `json:"urls"` + } `json:"entities"` + ExtendedEntities struct { + Media []struct { + IDStr string `json:"id_str"` + MediaURLHttps string `json:"media_url_https"` + Type string `json:"type"` + VideoInfo struct { + Variants []struct { + Bitrate int `json:"bitrate,omitempty"` + URL string `json:"url"` + } `json:"variants"` + } `json:"video_info"` + } `json:"media"` + } `json:"extended_entities"` + InReplyToStatusIDStr string `json:"in_reply_to_status_id_str"` + ReplyCount int `json:"reply_count"` + RetweetCount int `json:"retweet_count"` + RetweetedStatusIDStr string `json:"retweeted_status_id_str"` + QuotedStatusIDStr string `json:"quoted_status_id_str"` + Time time.Time `json:"time"` + UserIDStr string `json:"user_id_str"` + } `json:"tweets"` + Users map[string]struct { + CreatedAt string `json:"created_at"` + Description string `json:"description"` + Entities struct { + URL struct { + Urls []struct { + ExpandedURL string `json:"expanded_url"` + } `json:"urls"` + } `json:"url"` + } `json:"entities"` + FavouritesCount int `json:"favourites_count"` + FollowersCount int `json:"followers_count"` + FriendsCount int `json:"friends_count"` + IDStr string `json:"id_str"` + ListedCount int `json:"listed_count"` + Name string `json:"name"` + Location string `json:"location"` + PinnedTweetIdsStr []string `json:"pinned_tweet_ids_str"` + ProfileBannerURL string `json:"profile_banner_url"` + ProfileImageURLHTTPS string `json:"profile_image_url_https"` + Protected bool `json:"protected"` + ScreenName string `json:"screen_name"` + StatusesCount int `json:"statuses_count"` + Verified bool `json:"verified"` + } `json:"users"` + } `json:"globalObjects"` + Timeline struct { + Instructions []struct { + AddEntries struct { + Entries []struct { + Content struct { + Item struct { + Content struct { + Tweet struct { + ID string `json:"id"` + } `json:"tweet"` + User struct { + ID string `json:"id"` + } `json:"user"` + } `json:"content"` + } `json:"item"` + Operation struct { + Cursor struct { + Value string `json:"value"` + CursorType string `json:"cursorType"` + } `json:"cursor"` + } `json:"operation"` + TimelineModule struct { + Items []struct { + Item struct { + ClientEventInfo struct { + Details struct { + GuideDetails struct { + TransparentGuideDetails struct { + TrendMetadata struct { + TrendName string `json:"trendName"` + } `json:"trendMetadata"` + } `json:"transparentGuideDetails"` + } `json:"guideDetails"` + } `json:"details"` + } `json:"clientEventInfo"` + } `json:"item"` + } `json:"items"` + } `json:"timelineModule"` + } `json:"content,omitempty"` + } `json:"entries"` + } `json:"addEntries"` + PinEntry struct { + Entry struct { + Content struct { + Item struct { + Content struct { + Tweet struct { + ID string `json:"id"` + } `json:"tweet"` + } `json:"content"` + } `json:"item"` + } `json:"content"` + } `json:"entry"` + } `json:"pinEntry,omitempty"` + ReplaceEntry struct { + Entry struct { + Content struct { + Operation struct { + Cursor struct { + Value string `json:"value"` + CursorType string `json:"cursorType"` + } `json:"cursor"` + } `json:"operation"` + } `json:"content"` + } `json:"entry"` + } `json:"replaceEntry,omitempty"` + } `json:"instructions"` + } `json:"timeline"` +} + +func (timeline *timeline) parseTweet(id string) *Tweet { + if tweet, ok := timeline.GlobalObjects.Tweets[id]; ok { + username := timeline.GlobalObjects.Users[tweet.UserIDStr].ScreenName + tw := &Tweet{ + ID: id, + Likes: tweet.FavoriteCount, + PermanentURL: fmt.Sprintf("https://twitter.com/%s/status/%s", username, id), + Replies: tweet.ReplyCount, + Retweets: tweet.RetweetCount, + Text: tweet.FullText, + UserID: tweet.UserIDStr, + Username: username, + } + + tm, err := time.Parse(time.RubyDate, tweet.CreatedAt) + if err == nil { + tw.TimeParsed = tm + tw.Timestamp = tm.Unix() + } + + if tweet.QuotedStatusIDStr != "" { + tw.IsQuoted = true + } + if tweet.InReplyToStatusIDStr != "" { + tw.IsReply = true + } + if tweet.RetweetedStatusIDStr != "" { + tw.IsRetweet = true + if retweet, ok := timeline.GlobalObjects.Tweets[tweet.RetweetedStatusIDStr]; ok { + tw.Retweet = Retweet{ + ID: tweet.RetweetedStatusIDStr, + UserID: retweet.UserIDStr, + Username: timeline.GlobalObjects.Users[retweet.UserIDStr].ScreenName, + } + tm, err := time.Parse(time.RubyDate, retweet.CreatedAt) + if err == nil { + tw.Retweet.TimeParsed = tm + tw.Retweet.Timestamp = tm.Unix() + } + } + } + + for _, pinned := range timeline.GlobalObjects.Users[tweet.UserIDStr].PinnedTweetIdsStr { + if tweet.ConversationIDStr == pinned { + tw.IsPin = true + break + } + } + + for _, hash := range tweet.Entities.Hashtags { + tw.Hashtags = append(tw.Hashtags, hash.Text) + } + for _, media := range tweet.Entities.Media { + if media.Type == "photo" { + tw.Photos = append(tw.Photos, media.MediaURLHttps) + } + } + for _, media := range tweet.ExtendedEntities.Media { + if media.Type == "video" { + video := Video{ + ID: media.IDStr, + Preview: media.MediaURLHttps, + } + maxBitrate := 0 + for _, variant := range media.VideoInfo.Variants { + if variant.Bitrate > maxBitrate { + video.URL = strings.TrimSuffix(variant.URL, "?tag=10") + } + } + tw.Videos = append(tw.Videos, video) + } + } + for _, url := range tweet.Entities.URLs { + tw.URLs = append(tw.URLs, url.ExpandedURL) + } + + tw.HTML = tweet.FullText + tw.HTML = reHashtag.ReplaceAllStringFunc(tw.HTML, func(hashtag string) string { + return fmt.Sprintf(`%s`, + strings.TrimPrefix(hashtag, "#"), + hashtag, + ) + }) + tw.HTML = reUsername.ReplaceAllStringFunc(tw.HTML, func(username string) string { + return fmt.Sprintf(`%s`, + strings.TrimPrefix(username, "@"), + username, + ) + }) + tw.HTML = reTwitterURL.ReplaceAllStringFunc(tw.HTML, func(tco string) string { + for _, entity := range tweet.Entities.URLs { + if tco == entity.URL { + return fmt.Sprintf(`%s`, entity.ExpandedURL, tco) + } + } + for _, entity := range tweet.Entities.Media { + if tco == entity.URL { + return fmt.Sprintf(`
`, tco, entity.MediaURLHttps) + } + } + return tco + }) + tw.HTML = strings.Replace(tw.HTML, "\n", "
", -1) + return tw + } + return nil +} + +func (timeline *timeline) parseTweets() ([]*Tweet, string) { + var cursor string + var pinnedTweet *Tweet + var orderedTweets []*Tweet + for _, instruction := range timeline.Timeline.Instructions { + if instruction.PinEntry.Entry.Content.Item.Content.Tweet.ID != "" { + if tweet := timeline.parseTweet(instruction.PinEntry.Entry.Content.Item.Content.Tweet.ID); tweet != nil { + pinnedTweet = tweet + } + } + for _, entry := range instruction.AddEntries.Entries { + if tweet := timeline.parseTweet(entry.Content.Item.Content.Tweet.ID); tweet != nil { + orderedTweets = append(orderedTweets, tweet) + } + if entry.Content.Operation.Cursor.CursorType == "Bottom" { + cursor = entry.Content.Operation.Cursor.Value + } + } + if instruction.ReplaceEntry.Entry.Content.Operation.Cursor.CursorType == "Bottom" { + cursor = instruction.ReplaceEntry.Entry.Content.Operation.Cursor.Value + } + } + if pinnedTweet != nil && len(orderedTweets) > 0 { + orderedTweets = append([]*Tweet{pinnedTweet}, orderedTweets...) + } + return orderedTweets, cursor +} + +func (timeline *timeline) parseUsers() ([]*Profile, string) { + users := make(map[string]Profile) + + for id, user := range timeline.GlobalObjects.Users { + users[id] = parseProfile(user) + } + + var cursor string + var orderedProfiles []*Profile + for _, instruction := range timeline.Timeline.Instructions { + for _, entry := range instruction.AddEntries.Entries { + if profile, ok := users[entry.Content.Item.Content.User.ID]; ok { + orderedProfiles = append(orderedProfiles, &profile) + } + if entry.Content.Operation.Cursor.CursorType == "Bottom" { + cursor = entry.Content.Operation.Cursor.Value + } + } + if instruction.ReplaceEntry.Entry.Content.Operation.Cursor.CursorType == "Bottom" { + cursor = instruction.ReplaceEntry.Entry.Content.Operation.Cursor.Value + } + } + return orderedProfiles, cursor +} diff --git a/tweets.go b/tweets.go index 9c6df0f..4bf6d93 100644 --- a/tweets.go +++ b/tweets.go @@ -46,7 +46,7 @@ func (s *Scraper) FetchTweets(user string, maxTweetsNbr int, cursor string) ([]* return nil, "", err } - tweets, nextCursor := parseTimeline(&timeline) + tweets, nextCursor := timeline.parseTweets() return tweets, nextCursor, nil } @@ -63,7 +63,7 @@ func (s *Scraper) GetTweet(id string) (*Tweet, error) { return nil, err } - tweets, _ := parseTimeline(&timeline) + tweets, _ := timeline.parseTweets() for _, tweet := range tweets { if tweet.ID == id { return tweet, nil diff --git a/types.go b/types.go index 3b2b2e3..cde04cf 100644 --- a/types.go +++ b/types.go @@ -81,145 +81,6 @@ type ( Verified bool `json:"verified"` } - // timeline JSON - timeline struct { - GlobalObjects struct { - Tweets map[string]struct { - ConversationIDStr string `json:"conversation_id_str"` - CreatedAt string `json:"created_at"` - FavoriteCount int `json:"favorite_count"` - FullText string `json:"full_text"` - Entities struct { - Hashtags []struct { - Text string `json:"text"` - } `json:"hashtags"` - Media []struct { - MediaURLHttps string `json:"media_url_https"` - Type string `json:"type"` - URL string `json:"url"` - } `json:"media"` - URLs []struct { - ExpandedURL string `json:"expanded_url"` - URL string `json:"url"` - } `json:"urls"` - } `json:"entities"` - ExtendedEntities struct { - Media []struct { - IDStr string `json:"id_str"` - MediaURLHttps string `json:"media_url_https"` - Type string `json:"type"` - VideoInfo struct { - Variants []struct { - Bitrate int `json:"bitrate,omitempty"` - URL string `json:"url"` - } `json:"variants"` - } `json:"video_info"` - } `json:"media"` - } `json:"extended_entities"` - InReplyToStatusIDStr string `json:"in_reply_to_status_id_str"` - ReplyCount int `json:"reply_count"` - RetweetCount int `json:"retweet_count"` - RetweetedStatusIDStr string `json:"retweeted_status_id_str"` - QuotedStatusIDStr string `json:"quoted_status_id_str"` - Time time.Time `json:"time"` - UserIDStr string `json:"user_id_str"` - } `json:"tweets"` - Users map[string]struct { - CreatedAt string `json:"created_at"` - Description string `json:"description"` - Entities struct { - URL struct { - Urls []struct { - ExpandedURL string `json:"expanded_url"` - } `json:"urls"` - } `json:"url"` - } `json:"entities"` - FavouritesCount int `json:"favourites_count"` - FollowersCount int `json:"followers_count"` - FriendsCount int `json:"friends_count"` - IDStr string `json:"id_str"` - ListedCount int `json:"listed_count"` - Name string `json:"name"` - Location string `json:"location"` - PinnedTweetIdsStr []string `json:"pinned_tweet_ids_str"` - ProfileBannerURL string `json:"profile_banner_url"` - ProfileImageURLHTTPS string `json:"profile_image_url_https"` - Protected bool `json:"protected"` - ScreenName string `json:"screen_name"` - StatusesCount int `json:"statuses_count"` - Verified bool `json:"verified"` - } `json:"users"` - } `json:"globalObjects"` - Timeline struct { - Instructions []struct { - AddEntries struct { - Entries []struct { - Content struct { - Item struct { - Content struct { - Tweet struct { - ID string `json:"id"` - } `json:"tweet"` - User struct { - ID string `json:"id"` - } `json:"user"` - } `json:"content"` - } `json:"item"` - Operation struct { - Cursor struct { - Value string `json:"value"` - CursorType string `json:"cursorType"` - } `json:"cursor"` - } `json:"operation"` - TimelineModule struct { - Items []struct { - Item struct { - ClientEventInfo struct { - Details struct { - GuideDetails struct { - TransparentGuideDetails struct { - TrendMetadata struct { - TrendName string `json:"trendName"` - } `json:"trendMetadata"` - } `json:"transparentGuideDetails"` - } `json:"guideDetails"` - } `json:"details"` - } `json:"clientEventInfo"` - } `json:"item"` - } `json:"items"` - } `json:"timelineModule"` - } `json:"content,omitempty"` - } `json:"entries"` - } `json:"addEntries"` - PinEntry struct { - Entry struct { - Content struct { - Item struct { - Content struct { - Tweet struct { - ID string `json:"id"` - } `json:"tweet"` - } `json:"content"` - } `json:"item"` - } `json:"content"` - } `json:"entry"` - } `json:"pinEntry,omitempty"` - ReplaceEntry struct { - Entry struct { - Content struct { - Operation struct { - Cursor struct { - Value string `json:"value"` - CursorType string `json:"cursorType"` - } `json:"cursor"` - } `json:"operation"` - } `json:"content"` - } `json:"entry"` - } `json:"replaceEntry,omitempty"` - } `json:"instructions"` - } `json:"timeline"` - } - fetchProfileFunc func(query string, maxProfilesNbr int, cursor string) ([]*Profile, string, error) fetchTweetFunc func(query string, maxTweetsNbr int, cursor string) ([]*Tweet, string, error) ) diff --git a/util.go b/util.go index 70a99d0..aec0b67 100644 --- a/util.go +++ b/util.go @@ -2,11 +2,9 @@ package twitterscraper import ( "context" - "fmt" "net/http" "regexp" "strconv" - "strings" "time" ) @@ -177,164 +175,3 @@ func parseProfile(user legacyUser) Profile { return profile } - -func parseTimeline(timeline *timeline) ([]*Tweet, string) { - tweets := make(map[string]Tweet) - - for id, tweet := range timeline.GlobalObjects.Tweets { - username := timeline.GlobalObjects.Users[tweet.UserIDStr].ScreenName - tw := Tweet{ - ID: id, - Likes: tweet.FavoriteCount, - PermanentURL: fmt.Sprintf("https://twitter.com/%s/status/%s", username, id), - Replies: tweet.ReplyCount, - Retweets: tweet.RetweetCount, - Text: tweet.FullText, - UserID: tweet.UserIDStr, - Username: username, - } - - tm, err := time.Parse(time.RubyDate, tweet.CreatedAt) - if err == nil { - tw.TimeParsed = tm - tw.Timestamp = tm.Unix() - } - - if tweet.QuotedStatusIDStr != "" { - tw.IsQuoted = true - } - if tweet.InReplyToStatusIDStr != "" { - tw.IsReply = true - } - if tweet.RetweetedStatusIDStr != "" { - tw.IsRetweet = true - if retweet, ok := timeline.GlobalObjects.Tweets[tweet.RetweetedStatusIDStr]; ok { - tw.Retweet = Retweet{ - ID: tweet.RetweetedStatusIDStr, - UserID: retweet.UserIDStr, - Username: timeline.GlobalObjects.Users[retweet.UserIDStr].ScreenName, - } - tm, err := time.Parse(time.RubyDate, retweet.CreatedAt) - if err == nil { - tw.Retweet.TimeParsed = tm - tw.Retweet.Timestamp = tm.Unix() - } - } - } - - for _, pinned := range timeline.GlobalObjects.Users[tweet.UserIDStr].PinnedTweetIdsStr { - if tweet.ConversationIDStr == pinned { - tw.IsPin = true - break - } - } - - for _, hash := range tweet.Entities.Hashtags { - tw.Hashtags = append(tw.Hashtags, hash.Text) - } - for _, media := range tweet.Entities.Media { - if media.Type == "photo" { - tw.Photos = append(tw.Photos, media.MediaURLHttps) - } - } - for _, media := range tweet.ExtendedEntities.Media { - if media.Type == "video" { - video := Video{ - ID: media.IDStr, - Preview: media.MediaURLHttps, - } - maxBitrate := 0 - for _, variant := range media.VideoInfo.Variants { - if variant.Bitrate > maxBitrate { - video.URL = strings.TrimSuffix(variant.URL, "?tag=10") - } - } - tw.Videos = append(tw.Videos, video) - } - } - for _, url := range tweet.Entities.URLs { - tw.URLs = append(tw.URLs, url.ExpandedURL) - } - - tw.HTML = tweet.FullText - tw.HTML = reHashtag.ReplaceAllStringFunc(tw.HTML, func(hashtag string) string { - return fmt.Sprintf(`%s`, - strings.TrimPrefix(hashtag, "#"), - hashtag, - ) - }) - tw.HTML = reUsername.ReplaceAllStringFunc(tw.HTML, func(username string) string { - return fmt.Sprintf(`%s`, - strings.TrimPrefix(username, "@"), - username, - ) - }) - tw.HTML = reTwitterURL.ReplaceAllStringFunc(tw.HTML, func(tco string) string { - for _, entity := range tweet.Entities.URLs { - if tco == entity.URL { - return fmt.Sprintf(`%s`, entity.ExpandedURL, tco) - } - } - for _, entity := range tweet.Entities.Media { - if tco == entity.URL { - return fmt.Sprintf(`
`, tco, entity.MediaURLHttps) - } - } - return tco - }) - tw.HTML = strings.Replace(tw.HTML, "\n", "
", -1) - - tweets[tw.ID] = tw - } - - var cursor string - var pinnedTweet *Tweet - var orderedTweets []*Tweet - for _, instruction := range timeline.Timeline.Instructions { - if instruction.PinEntry.Entry.Content.Item.Content.Tweet.ID != "" { - if tweet, ok := tweets[instruction.PinEntry.Entry.Content.Item.Content.Tweet.ID]; ok { - pinnedTweet = &tweet - } - } - for _, entry := range instruction.AddEntries.Entries { - if tweet, ok := tweets[entry.Content.Item.Content.Tweet.ID]; ok { - orderedTweets = append(orderedTweets, &tweet) - } - if entry.Content.Operation.Cursor.CursorType == "Bottom" { - cursor = entry.Content.Operation.Cursor.Value - } - } - if instruction.ReplaceEntry.Entry.Content.Operation.Cursor.CursorType == "Bottom" { - cursor = instruction.ReplaceEntry.Entry.Content.Operation.Cursor.Value - } - } - if pinnedTweet != nil && len(orderedTweets) > 0 { - orderedTweets = append([]*Tweet{pinnedTweet}, orderedTweets...) - } - return orderedTweets, cursor -} - -func parseUsers(timeline *timeline) ([]*Profile, string) { - users := make(map[string]Profile) - - for id, user := range timeline.GlobalObjects.Users { - users[id] = parseProfile(user) - } - - var cursor string - var orderedProfiles []*Profile - for _, instruction := range timeline.Timeline.Instructions { - for _, entry := range instruction.AddEntries.Entries { - if profile, ok := users[entry.Content.Item.Content.User.ID]; ok { - orderedProfiles = append(orderedProfiles, &profile) - } - if entry.Content.Operation.Cursor.CursorType == "Bottom" { - cursor = entry.Content.Operation.Cursor.Value - } - } - if instruction.ReplaceEntry.Entry.Content.Operation.Cursor.CursorType == "Bottom" { - cursor = instruction.ReplaceEntry.Entry.Content.Operation.Cursor.Value - } - } - return orderedProfiles, cursor -}