twitter-scrapper/timeline_v2.go

package twitterscraper

import (
	"strconv"
	"strings"
)

type tweet struct {
	Core struct {
		UserResults struct {
			Result struct {
				IsBlueVerified bool       `json:"is_blue_verified"`
				Legacy         legacyUser `json:"legacy"`
			} `json:"result"`
		} `json:"user_results"`
	} `json:"core"`
	Views struct {
		Count string `json:"count"`
	} `json:"views"`
	NoteTweet struct {
		NoteTweetResults struct {
			Result struct {
				Text string `json:"text"`
			} `json:"result"`
		} `json:"note_tweet_results"`
	} `json:"note_tweet"`
	QuotedStatusResult struct {
		Result *result `json:"result"`
	} `json:"quoted_status_result"`
	Legacy legacyTweet `json:"legacy"`
}

type result struct {
	Typename string `json:"__typename"`
	tweet
	Tweet tweet `json:"tweet"`
}

func (result *result) parse() *Tweet {
	if result.NoteTweet.NoteTweetResults.Result.Text != "" {
		result.Legacy.FullText = result.NoteTweet.NoteTweetResults.Result.Text
	}
	var legacy *legacyTweet = &result.Legacy
	var user *legacyUser = &result.Core.UserResults.Result.Legacy
	if result.Typename == "TweetWithVisibilityResults" {
		legacy = &result.Tweet.Legacy
		user = &result.Tweet.Core.UserResults.Result.Legacy
	}
	tw := parseLegacyTweet(user, legacy)
	if tw == nil {
		return nil
	}
	if tw.Views == 0 && result.Views.Count != "" {
		tw.Views, _ = strconv.Atoi(result.Views.Count)
	}
	if result.QuotedStatusResult.Result != nil {
		tw.QuotedStatus = result.QuotedStatusResult.Result.parse()
	}
	tw.HTML = expandURLs(tw.HTML, legacy.Entities.URLs, legacy.ExtendedEntities.Media)
	tw.HTML = expandURLs(tw.Text, legacy.Entities.URLs, legacy.ExtendedEntities.Media)
	return tw
}

type userResult struct {
	Typename                   string       `json:"__typename"`
	ID                         string       `json:"id"`
	RestID                     string       `json:"rest_id"`
	AffiliatesHighlightedLabel struct{}     `json:"affiliates_highlighted_label"`
	HasGraduatedAccess         bool         `json:"has_graduated_access"`
	IsBlueVerified             bool         `json:"is_blue_verified"`
	ProfileImageShape          string       `json:"profile_image_shape"`
	Legacy                     legacyUserV2 `json:"legacy"`
}

func (result *userResult) parse() Profile {
	return parseProfileV2(*result)
}

type item struct {
	EntryID string `json:"entryId"`
	Item    struct {
		ItemContent struct {
			ItemType         string `json:"itemType"`
			TweetDisplayType string `json:"tweetDisplayType"`
			TweetResults     struct {
				Result result `json:"result"`
			} `json:"tweet_results"`
			CursorType string `json:"cursorType"`
			Value      string `json:"value"`
		} `json:"itemContent"`
	} `json:"item"`
}

type entry struct {
	Content struct {
		CursorType  string `json:"cursorType"`
		Value       string `json:"value"`
		Items       []item `json:"items"`
		ItemContent struct {
			ItemType         string `json:"itemType"`
			TweetDisplayType string `json:"tweetDisplayType"`
			TweetResults     struct {
				Result result `json:"result"`
			} `json:"tweet_results"`
			UserDisplayType string `json:"userDisplayType"`
			UserResults     struct {
				Result userResult `json:"result"`
			} `json:"user_results"`
			CursorType string `json:"cursorType"`
			Value      string `json:"value"`
		} `json:"itemContent"`
	} `json:"content"`
}

// timeline v2 JSON object
type timelineV2 struct {
	Data struct {
		User struct {
			Result struct {
				TimelineV2 struct {
					Timeline struct {
						Instructions []struct {
							ModuleItems []item  `json:"moduleItems"`
							Entries     []entry `json:"entries"`
							Entry       entry   `json:"entry"`
							Type        string  `json:"type"`
						} `json:"instructions"`
					} `json:"timeline"`
				} `json:"timeline_v2"`

				Timeline struct {
					Timeline struct {
						Instructions []struct {
							Entries []entry `json:"entries"`
							Entry   entry   `json:"entry"`
							Type    string  `json:"type"`
						} `json:"instructions"`
					} `json:"timeline"`
				} `json:"timeline"`
			} `json:"result"`
		} `json:"user"`
	} `json:"data"`
}

func (timeline *timelineV2) parseTweets() ([]*Tweet, string) {
	var cursor string
	var tweets []*Tweet
	for _, instruction := range timeline.Data.User.Result.TimelineV2.Timeline.Instructions {
		for _, entry := range instruction.Entries {
			if entry.Content.CursorType == "Bottom" {
				cursor = entry.Content.Value
				continue
			}
			if entry.Content.ItemContent.TweetResults.Result.Typename == "Tweet" || entry.Content.ItemContent.TweetResults.Result.Typename == "TweetWithVisibilityResults" {
				if tweet := entry.Content.ItemContent.TweetResults.Result.parse(); tweet != nil {
					tweets = append(tweets, tweet)
				}
			}
			if len(entry.Content.Items) > 0 {
				for _, item := range entry.Content.Items {
					if tweet := item.Item.ItemContent.TweetResults.Result.parse(); tweet != nil {
						tweets = append(tweets, tweet)
					}
				}
			}
		}
		if len(instruction.ModuleItems) > 0 {
			for _, entry := range instruction.ModuleItems {
				if entry.Item.ItemContent.TweetResults.Result.Typename == "Tweet" || entry.Item.ItemContent.TweetResults.Result.Typename == "TweetWithVisibilityResults" {
					if tweet := entry.Item.ItemContent.TweetResults.Result.parse(); tweet != nil {
						tweets = append(tweets, tweet)
					}
				}
			}
		}
	}
	return tweets, cursor
}

type bookmarksTimelineV2 struct {
	Data struct {
		Bookmarks struct {
			Timeline struct {
				Instructions []struct {
					Entries []entry `json:"entries"`
					Type    string  `json:"type"`
				} `json:"instructions"`
			} `json:"timeline"`
		} `json:"bookmark_timeline_v2"`
	} `json:"data"`
}

func (timeline *bookmarksTimelineV2) parseTweets() ([]*Tweet, string) {
	var cursor string
	var tweets []*Tweet
	for _, instruction := range timeline.Data.Bookmarks.Timeline.Instructions {
		for _, entry := range instruction.Entries {
			if entry.Content.CursorType == "Bottom" {
				cursor = entry.Content.Value
				continue
			}
			if entry.Content.ItemContent.TweetResults.Result.Typename == "Tweet" {
				if tweet := entry.Content.ItemContent.TweetResults.Result.parse(); tweet != nil {
					tweets = append(tweets, tweet)
				}
			}
		}
	}
	return tweets, cursor
}

type retweetersTimelineV2 struct {
	Data struct {
		RetweetersTimeline struct {
			Timeline struct {
				Instructions []struct {
					Type    string  `json:"type"`
					Entries []entry `json:"entries"`
				} `json:"instructions"`
			} `json:"timeline"`
		} `json:"retweeters_timeline"`
	} `json:"data"`
}

func (timeline *retweetersTimelineV2) parseUsers() ([]*Profile, string) {
	var cursor string
	var users []*Profile
	for _, instruction := range timeline.Data.RetweetersTimeline.Timeline.Instructions {
		for _, entry := range instruction.Entries {
			if entry.Content.CursorType == "Bottom" {
				cursor = entry.Content.Value
				continue
			}
			if entry.Content.ItemContent.UserResults.Result.Typename == "User" {
				user := entry.Content.ItemContent.UserResults.Result.parse()
				users = append(users, &user)
			}
		}
	}
	return users, cursor
}

func (timeline *timelineV2) parseUsers() ([]*Profile, string) {
	var cursor string
	var users []*Profile
	for _, instruction := range timeline.Data.User.Result.Timeline.Timeline.Instructions {
		for _, entry := range instruction.Entries {
			if entry.Content.CursorType == "Bottom" {
				cursor = entry.Content.Value
				continue
			}
			if entry.Content.ItemContent.UserResults.Result.Typename == "User" {
				user := entry.Content.ItemContent.UserResults.Result.parse()
				users = append(users, &user)
			}
		}
	}
	return users, cursor
}

type threadedConversation struct {
	Data struct {
		ThreadedConversationWithInjectionsV2 struct {
			Instructions []struct {
				Type        string  `json:"type"`
				Entry       entry   `json:"entry"`
				Entries     []entry `json:"entries"`
				ModuleItems []item  `json:"moduleItems"`
			} `json:"instructions"`
		} `json:"threaded_conversation_with_injections_v2"`
	} `json:"data"`
}

func (conversation *threadedConversation) parse(focalTweetID string) ([]*Tweet, []*ThreadCursor) {
	var tweets []*Tweet
	var cursors []*ThreadCursor
	for _, instruction := range conversation.Data.ThreadedConversationWithInjectionsV2.Instructions {
		for _, entry := range instruction.Entries {
			if entry.Content.ItemContent.TweetResults.Result.Typename == "Tweet" || entry.Content.ItemContent.TweetResults.Result.Typename == "TweetWithVisibilityResults" {
				if tweet := entry.Content.ItemContent.TweetResults.Result.parse(); tweet != nil {
					if entry.Content.ItemContent.TweetDisplayType == "SelfThread" {
						tweet.IsSelfThread = true
					}
					tweets = append(tweets, tweet)
				}
			}

			if entry.Content.ItemContent.CursorType != "" && entry.Content.ItemContent.Value != "" {
				cursors = append(cursors, &ThreadCursor{
					FocalTweetID: focalTweetID,
					ThreadID:     focalTweetID,
					Cursor:       entry.Content.ItemContent.Value,
					CursorType:   entry.Content.ItemContent.CursorType,
				})
			}

			for _, item := range entry.Content.Items {
				if item.Item.ItemContent.TweetResults.Result.Typename == "Tweet" || item.Item.ItemContent.TweetResults.Result.Typename == "TweetWithVisibilityResults" {
					if tweet := item.Item.ItemContent.TweetResults.Result.parse(); tweet != nil {
						if item.Item.ItemContent.TweetDisplayType == "SelfThread" {
							tweet.IsSelfThread = true
						}
						tweets = append(tweets, tweet)
					}
				}

				if item.Item.ItemContent.CursorType != "" && item.Item.ItemContent.Value != "" {
					threadID := ""

					entryId := strings.Split(item.EntryID, "-")
					if len(entryId) > 1 && entryId[0] == "conversationthread" {
						if i, _ := strconv.Atoi(entryId[1]); i != 0 {
							threadID = entryId[1]
						}
					}

					cursors = append(cursors, &ThreadCursor{
						FocalTweetID: focalTweetID,
						ThreadID:     threadID,
						Cursor:       item.Item.ItemContent.Value,
						CursorType:   item.Item.ItemContent.CursorType,
					})
				}
			}
		}
		for _, item := range instruction.ModuleItems {
			if item.Item.ItemContent.TweetResults.Result.Typename == "Tweet" || item.Item.ItemContent.TweetResults.Result.Typename == "TweetWithVisibilityResults" {
				if tweet := item.Item.ItemContent.TweetResults.Result.parse(); tweet != nil {
					if item.Item.ItemContent.TweetDisplayType == "SelfThread" {
						tweet.IsSelfThread = true
					}
					tweets = append(tweets, tweet)
				}
			}

			if item.Item.ItemContent.CursorType != "" && item.Item.ItemContent.Value != "" {
				threadID := ""

				entryId := strings.Split(item.EntryID, "-")
				if len(entryId) > 1 && entryId[0] == "conversationthread" {
					if i, _ := strconv.Atoi(entryId[1]); i != 0 {
						threadID = entryId[1]
					}
				}

				cursors = append(cursors, &ThreadCursor{
					FocalTweetID: focalTweetID,
					ThreadID:     threadID,
					Cursor:       item.Item.ItemContent.Value,
					CursorType:   item.Item.ItemContent.CursorType,
				})
			}
		}
	}

	for _, tweet := range tweets {
		if tweet.InReplyToStatusID != "" {
			for _, parentTweet := range tweets {
				if parentTweet.ID == tweet.InReplyToStatusID {
					tweet.InReplyToStatus = parentTweet
					break
				}
			}
		}
		if tweet.IsSelfThread && tweet.ConversationID == tweet.ID {
			for _, childTweet := range tweets {
				if childTweet.IsSelfThread && childTweet.ID != tweet.ID {
					tweet.Thread = append(tweet.Thread, childTweet)
				}
			}
			if len(tweet.Thread) == 0 {
				tweet.IsSelfThread = false
			}
		}
	}
	return tweets, cursors
}

type tweetResult struct {
	Data struct {
		TweetResult struct {
			Result result `json:"result"`
		} `json:"tweetResult"`
	} `json:"data"`
}

func (tweetResult *tweetResult) parse() *Tweet {
	return tweetResult.Data.TweetResult.Result.parse()
}