Use GraphQL API with timeline v2

Close #85
Close #82
Close #77
Close #76

search.go

@@ -19,7 +19,7 @@ func (s *Scraper) SearchProfiles(ctx context.Context, query string, maxProfilesN
 }
 
 // getSearchTimeline gets results for a given search query, via the Twitter frontend API
-func (s *Scraper) getSearchTimeline(query string, maxNbr int, cursor string) (*timeline, error) {
+func (s *Scraper) getSearchTimeline(query string, maxNbr int, cursor string) (*timelineV1, error) {
 	if !s.isLogged {
 		return nil, errors.New("scraper is not logged in for search")
 	}
@@ -57,7 +57,7 @@ func (s *Scraper) getSearchTimeline(query string, maxNbr int, cursor string) (*t
 
 	req.URL.RawQuery = q.Encode()
 
-	var timeline timeline
+	var timeline timelineV1
 	err = s.RequestAPI(req, &timeline)
 	if err != nil {
 		return nil, err

timeline_v1.go

@@ -7,90 +7,11 @@ import (
 	"time"
 )
 
-// timeline JSON object
+// legacy timeline JSON object
-type timeline struct {
+type timelineV1 struct {
 	GlobalObjects struct {
-		Tweets map[string]struct {
+		Tweets map[string]legacyTweet `json:"tweets"`
-			ConversationIDStr string `json:"conversation_id_str"`
+		Users  map[string]legacyUser  `json:"users"`
-			CreatedAt         string `json:"created_at"`
-			FavoriteCount     int    `json:"favorite_count"`
-			FullText          string `json:"full_text"`
-			Entities          struct {
-				Hashtags []struct {
-					Text string `json:"text"`
-				} `json:"hashtags"`
-				Media []struct {
-					MediaURLHttps string `json:"media_url_https"`
-					Type          string `json:"type"`
-					URL           string `json:"url"`
-				} `json:"media"`
-				URLs []struct {
-					ExpandedURL string `json:"expanded_url"`
-					URL         string `json:"url"`
-				} `json:"urls"`
-				UserMentions []struct {
-					IDStr      string `json:"id_str"`
-					Name       string `json:"name"`
-					ScreenName string `json:"screen_name"`
-				} `json:"user_mentions"`
-			} `json:"entities"`
-			ExtendedEntities struct {
-				Media []struct {
-					IDStr                    string `json:"id_str"`
-					MediaURLHttps            string `json:"media_url_https"`
-					ExtSensitiveMediaWarning struct {
-						AdultContent    bool `json:"adult_content"`
-						GraphicViolence bool `json:"graphic_violence"`
-						Other           bool `json:"other"`
-					} `json:"ext_sensitive_media_warning"`
-					Type      string `json:"type"`
-					URL       string `json:"url"`
-					VideoInfo struct {
-						Variants []struct {
-							Bitrate int    `json:"bitrate,omitempty"`
-							URL     string `json:"url"`
-						} `json:"variants"`
-					} `json:"video_info"`
-				} `json:"media"`
-			} `json:"extended_entities"`
-			InReplyToStatusIDStr string    `json:"in_reply_to_status_id_str"`
-			Place                Place     `json:"place"`
-			ReplyCount           int       `json:"reply_count"`
-			RetweetCount         int       `json:"retweet_count"`
-			RetweetedStatusIDStr string    `json:"retweeted_status_id_str"`
-			QuotedStatusIDStr    string    `json:"quoted_status_id_str"`
-			Time                 time.Time `json:"time"`
-			UserIDStr            string    `json:"user_id_str"`
-			Views                struct {
-				State string `json:"state"`
-				Count string `json:"count"`
-			} `json:"ext_views"`
-		} `json:"tweets"`
-		Users map[string]struct {
-			CreatedAt   string `json:"created_at"`
-			Description string `json:"description"`
-			Entities    struct {
-				URL struct {
-					Urls []struct {
-						ExpandedURL string `json:"expanded_url"`
-					} `json:"urls"`
-				} `json:"url"`
-			} `json:"entities"`
-			FavouritesCount      int      `json:"favourites_count"`
-			FollowersCount       int      `json:"followers_count"`
-			FriendsCount         int      `json:"friends_count"`
-			IDStr                string   `json:"id_str"`
-			ListedCount          int      `json:"listed_count"`
-			Name                 string   `json:"name"`
-			Location             string   `json:"location"`
-			PinnedTweetIdsStr    []string `json:"pinned_tweet_ids_str"`
-			ProfileBannerURL     string   `json:"profile_banner_url"`
-			ProfileImageURLHTTPS string   `json:"profile_image_url_https"`
-			Protected            bool     `json:"protected"`
-			ScreenName           string   `json:"screen_name"`
-			StatusesCount        int      `json:"statuses_count"`
-			Verified             bool     `json:"verified"`
-		} `json:"users"`
 	} `json:"globalObjects"`
 	Timeline struct {
 		Instructions []struct {
@@ -162,7 +83,7 @@ type timeline struct {
 	} `json:"timeline"`
 }
 
-func (timeline *timeline) parseTweet(id string) *Tweet {
+func (timeline *timelineV1) parseTweet(id string) *Tweet {
 	if tweet, ok := timeline.GlobalObjects.Tweets[id]; ok {
 		username := timeline.GlobalObjects.Users[tweet.UserIDStr].ScreenName
 		name := timeline.GlobalObjects.Users[tweet.UserIDStr].Name
@@ -191,14 +112,17 @@ func (timeline *timeline) parseTweet(id string) *Tweet {
 		if tweet.QuotedStatusIDStr != "" {
 			tw.IsQuoted = true
 			tw.QuotedStatus = timeline.parseTweet(tweet.QuotedStatusIDStr)
+			tw.QuotedStatusID = tweet.QuotedStatusIDStr
 		}
 		if tweet.InReplyToStatusIDStr != "" {
 			tw.IsReply = true
 			tw.InReplyToStatus = timeline.parseTweet(tweet.InReplyToStatusIDStr)
+			tw.InReplyToStatusID = tweet.InReplyToStatusIDStr
 		}
 		if tweet.RetweetedStatusIDStr != "" {
 			tw.IsRetweet = true
 			tw.RetweetedStatus = timeline.parseTweet(tweet.RetweetedStatusIDStr)
+			tw.RetweetedStatusID = tweet.RetweetedStatusIDStr
 		}
 
 		if tweet.Views.Count != "" {
@@ -210,7 +134,7 @@ func (timeline *timeline) parseTweet(id string) *Tweet {
 		}
 
 		for _, pinned := range timeline.GlobalObjects.Users[tweet.UserIDStr].PinnedTweetIdsStr {
-			if tweet.ConversationIDStr == pinned {
+			if tweet.IDStr == pinned {
 				tw.IsPin = true
 				break
 			}
@@ -311,7 +235,7 @@ func (timeline *timeline) parseTweet(id string) *Tweet {
 	return nil
 }
 
-func (timeline *timeline) parseTweets() ([]*Tweet, string) {
+func (timeline *timelineV1) parseTweets() ([]*Tweet, string) {
 	var cursor string
 	var pinnedTweet *Tweet
 	var orderedTweets []*Tweet
@@ -339,7 +263,7 @@ func (timeline *timeline) parseTweets() ([]*Tweet, string) {
 	return orderedTweets, cursor
 }
 
-func (timeline *timeline) parseUsers() ([]*Profile, string) {
+func (timeline *timelineV1) parseUsers() ([]*Profile, string) {
 	users := make(map[string]Profile)
 
 	for id, user := range timeline.GlobalObjects.Users {

timeline_v2.go

@@ -0,0 +1,301 @@
+package twitterscraper
+
+import (
+	"fmt"
+	"strconv"
+	"strings"
+	"time"
+)
+
+type result struct {
+	Typename string `json:"__typename"`
+	Core     struct {
+		UserResults struct {
+			Result struct {
+				IsBlueVerified bool       `json:"is_blue_verified"`
+				Legacy         legacyUser `json:"legacy"`
+			} `json:"result"`
+		} `json:"user_results"`
+	} `json:"core"`
+	Views struct {
+		Count string `json:"count"`
+	} `json:"views"`
+	NoteTweet struct {
+		NoteTweetResults struct {
+			Result struct {
+				Text string `json:"text"`
+			} `json:"result"`
+		} `json:"note_tweet_results"`
+	} `json:"note_tweet"`
+	QuotedStatusResult struct {
+		Result *result `json:"result"`
+	} `json:"quoted_status_result"`
+	Legacy legacyTweet `json:"legacy"`
+}
+
+func (result *result) parse() *Tweet {
+	if result.NoteTweet.NoteTweetResults.Result.Text != "" {
+		result.Legacy.FullText = result.NoteTweet.NoteTweetResults.Result.Text
+	}
+	tw := parseLegacyTweet(&result.Core.UserResults.Result.Legacy, &result.Legacy)
+	if tw.Views == 0 && result.Views.Count != "" {
+		tw.Views, _ = strconv.Atoi(result.Views.Count)
+	}
+	if result.QuotedStatusResult.Result != nil {
+		tw.QuotedStatus = result.QuotedStatusResult.Result.parse()
+	}
+	return tw
+}
+
+type entry struct {
+	Content struct {
+		CursorType string `json:"cursorType"`
+		Value      string `json:"value"`
+		Items      []struct {
+			Item struct {
+				ItemContent struct {
+					TweetResults struct {
+						Result result `json:"result"`
+					} `json:"tweet_results"`
+				} `json:"itemContent"`
+			} `json:"item"`
+		} `json:"items"`
+		ItemContent struct {
+			TweetResults struct {
+				Result result `json:"result"`
+			} `json:"tweet_results"`
+		} `json:"itemContent"`
+	} `json:"content"`
+}
+
+// timeline v2 JSON object
+type timelineV2 struct {
+	Data struct {
+		User struct {
+			Result struct {
+				TimelineV2 struct {
+					Timeline struct {
+						Instructions []struct {
+							Entries []entry `json:"entries"`
+							Entry   entry   `json:"entry"`
+							Type    string  `json:"type"`
+						} `json:"instructions"`
+					} `json:"timeline"`
+				} `json:"timeline_v2"`
+			} `json:"result"`
+		} `json:"user"`
+	} `json:"data"`
+}
+
+func (timeline *timelineV2) parseTweets() ([]*Tweet, string) {
+	var cursor string
+	var tweets []*Tweet
+	for _, instruction := range timeline.Data.User.Result.TimelineV2.Timeline.Instructions {
+		for _, entry := range instruction.Entries {
+			if entry.Content.CursorType == "Bottom" {
+				cursor = entry.Content.Value
+				continue
+			}
+			if entry.Content.ItemContent.TweetResults.Result.Typename == "Tweet" {
+				if tweet := entry.Content.ItemContent.TweetResults.Result.parse(); tweet != nil {
+					tweets = append(tweets, tweet)
+				}
+			}
+		}
+	}
+	return tweets, cursor
+}
+
+type threadedConversation struct {
+	Data struct {
+		ThreadedConversationWithInjectionsV2 struct {
+			Instructions []struct {
+				Type    string  `json:"type"`
+				Entries []entry `json:"entries"`
+				Entry   entry   `json:"entry"`
+			} `json:"instructions"`
+		} `json:"threaded_conversation_with_injections_v2"`
+	} `json:"data"`
+}
+
+func (conversation *threadedConversation) parse() []*Tweet {
+	var tweets []*Tweet
+	for _, instruction := range conversation.Data.ThreadedConversationWithInjectionsV2.Instructions {
+		for _, entry := range instruction.Entries {
+			if entry.Content.ItemContent.TweetResults.Result.Typename == "Tweet" {
+				if tweet := entry.Content.ItemContent.TweetResults.Result.parse(); tweet != nil {
+					tweets = append(tweets, tweet)
+				}
+			}
+			for _, item := range entry.Content.Items {
+				if item.Item.ItemContent.TweetResults.Result.Typename == "Tweet" {
+					if tweet := item.Item.ItemContent.TweetResults.Result.parse(); tweet != nil {
+						tweets = append(tweets, tweet)
+					}
+				}
+			}
+		}
+	}
+	for _, tweet := range tweets {
+		if tweet.InReplyToStatusID != "" {
+			for _, parentTweet := range tweets {
+				if parentTweet.ID == tweet.InReplyToStatusID {
+					tweet.InReplyToStatus = parentTweet
+					break
+				}
+			}
+		}
+	}
+	return tweets
+}
+
+func parseLegacyTweet(user *legacyUser, tweet *legacyTweet) *Tweet {
+	username := user.ScreenName
+	name := user.Name
+	tweetID := tweet.IDStr
+	tw := &Tweet{
+		ID:           tweetID,
+		Likes:        tweet.FavoriteCount,
+		Name:         name,
+		PermanentURL: fmt.Sprintf("https://twitter.com/%s/status/%s", username, tweetID),
+		Replies:      tweet.ReplyCount,
+		Retweets:     tweet.RetweetCount,
+		Text:         tweet.FullText,
+		UserID:       tweet.UserIDStr,
+		Username:     username,
+	}
+
+	tm, err := time.Parse(time.RubyDate, tweet.CreatedAt)
+	if err == nil {
+		tw.TimeParsed = tm
+		tw.Timestamp = tm.Unix()
+	}
+
+	if tweet.Place.ID != "" {
+		tw.Place = &tweet.Place
+	}
+
+	if tweet.QuotedStatusIDStr != "" {
+		tw.IsQuoted = true
+		tw.QuotedStatusID = tweet.QuotedStatusIDStr
+	}
+	if tweet.InReplyToStatusIDStr != "" {
+		tw.IsReply = true
+		tw.InReplyToStatusID = tweet.InReplyToStatusIDStr
+	}
+	if tweet.RetweetedStatusIDStr != "" || tweet.RetweetedStatusResult.Result != nil {
+		tw.IsRetweet = true
+		tw.RetweetedStatusID = tweet.RetweetedStatusIDStr
+		if tweet.RetweetedStatusResult.Result != nil {
+			tw.RetweetedStatus = parseLegacyTweet(&tweet.RetweetedStatusResult.Result.Core.UserResults.Result.Legacy, &tweet.RetweetedStatusResult.Result.Legacy)
+			tw.RetweetedStatusID = tw.RetweetedStatus.ID
+		}
+	}
+
+	if tweet.Views.Count != "" {
+		views, viewsErr := strconv.Atoi(tweet.Views.Count)
+		if viewsErr != nil {
+			views = 0
+		}
+		tw.Views = views
+	}
+
+	for _, pinned := range user.PinnedTweetIdsStr {
+		if tweet.IDStr == pinned {
+			tw.IsPin = true
+			break
+		}
+	}
+
+	for _, hash := range tweet.Entities.Hashtags {
+		tw.Hashtags = append(tw.Hashtags, hash.Text)
+	}
+
+	for _, mention := range tweet.Entities.UserMentions {
+		tw.Mentions = append(tw.Mentions, Mention{
+			ID:       mention.IDStr,
+			Username: mention.ScreenName,
+			Name:     mention.Name,
+		})
+	}
+
+	for _, media := range tweet.ExtendedEntities.Media {
+		if media.Type == "photo" {
+			photo := Photo{
+				ID:  media.IDStr,
+				URL: media.MediaURLHttps,
+			}
+
+			tw.Photos = append(tw.Photos, photo)
+		} else if media.Type == "video" {
+			video := Video{
+				ID:      media.IDStr,
+				Preview: media.MediaURLHttps,
+			}
+
+			maxBitrate := 0
+			for _, variant := range media.VideoInfo.Variants {
+				if variant.Bitrate > maxBitrate {
+					video.URL = strings.TrimSuffix(variant.URL, "?tag=10")
+					maxBitrate = variant.Bitrate
+				}
+			}
+
+			tw.Videos = append(tw.Videos, video)
+		}
+
+		if !tw.SensitiveContent {
+			sensitive := media.ExtSensitiveMediaWarning
+			tw.SensitiveContent = sensitive.AdultContent || sensitive.GraphicViolence || sensitive.Other
+		}
+	}
+
+	for _, url := range tweet.Entities.URLs {
+		tw.URLs = append(tw.URLs, url.ExpandedURL)
+	}
+
+	tw.HTML = tweet.FullText
+	tw.HTML = reHashtag.ReplaceAllStringFunc(tw.HTML, func(hashtag string) string {
+		return fmt.Sprintf(`<a href="https://twitter.com/hashtag/%s">%s</a>`,
+			strings.TrimPrefix(hashtag, "#"),
+			hashtag,
+		)
+	})
+	tw.HTML = reUsername.ReplaceAllStringFunc(tw.HTML, func(username string) string {
+		return fmt.Sprintf(`<a href="https://twitter.com/%s">%s</a>`,
+			strings.TrimPrefix(username, "@"),
+			username,
+		)
+	})
+	var foundedMedia []string
+	tw.HTML = reTwitterURL.ReplaceAllStringFunc(tw.HTML, func(tco string) string {
+		for _, entity := range tweet.Entities.URLs {
+			if tco == entity.URL {
+				return fmt.Sprintf(`<a href="%s">%s</a>`, entity.ExpandedURL, tco)
+			}
+		}
+		for _, entity := range tweet.ExtendedEntities.Media {
+			if tco == entity.URL {
+				foundedMedia = append(foundedMedia, entity.MediaURLHttps)
+				return fmt.Sprintf(`<br><a href="%s"><img src="%s"/></a>`, tco, entity.MediaURLHttps)
+			}
+		}
+		return tco
+	})
+	for _, photo := range tw.Photos {
+		url := photo.URL
+		if stringInSlice(url, foundedMedia) {
+			continue
+		}
+		tw.HTML += fmt.Sprintf(`<br><img src="%s"/>`, url)
+	}
+	for _, video := range tw.Videos {
+		url := video.Preview
+		if stringInSlice(url, foundedMedia) {
+			continue
+		}
+		tw.HTML += fmt.Sprintf(`<br><img src="%s"/>`, url)
+	}
+	tw.HTML = strings.Replace(tw.HTML, "\n", "<br>", -1)
+	return tw
+}

trends.go

@@ -16,7 +16,7 @@ func (s *Scraper) GetTrends() ([]string, error) {
 	q.Add("entity_tokens", "false")
 	req.URL.RawQuery = q.Encode()
 
-	var jsn timeline
+	var jsn timelineV1
 	curBearerToken := s.bearerToken
 	if curBearerToken != bearerToken2 {
 		s.setBearerToken(bearerToken2)

tweets.go

@@ -3,7 +3,7 @@ package twitterscraper
 import (
 	"context"
 	"fmt"
-	"strconv"
+	"net/url"
 )
 
 // GetTweets returns channel with tweets for a given user.
@@ -21,26 +21,59 @@ func (s *Scraper) FetchTweets(user string, maxTweetsNbr int, cursor string) ([]*
 	return s.FetchTweetsByUserID(userID, maxTweetsNbr, cursor)
 }
 
-// FetchTweetsByUserID gets tweets for a given userID, via the Twitter frontend API.
+// FetchTweetsByUserID gets tweets for a given userID, via the Twitter frontend GraphQL API.
 func (s *Scraper) FetchTweetsByUserID(userID string, maxTweetsNbr int, cursor string) ([]*Tweet, string, error) {
 	if maxTweetsNbr > 200 {
 		maxTweetsNbr = 200
 	}
 
-	req, err := s.newRequest("GET", "https://api.twitter.com/2/timeline/profile/"+userID+".json")
+	req, err := s.newRequest("GET", "https://twitter.com/i/api/graphql/UGi7tjRPr-d_U3bCPIko5Q/UserTweets")
 	if err != nil {
 		return nil, "", err
 	}
 
-	q := req.URL.Query()
+	variables := map[string]interface{}{
-	q.Add("count", strconv.Itoa(maxTweetsNbr))
+		"userId":                                 userID,
-	q.Add("userId", userID)
+		"count":                                  maxTweetsNbr,
-	if cursor != "" {
+		"includePromotedContent":                 false,
-		q.Add("cursor", cursor)
+		"withQuickPromoteEligibilityTweetFields": false,
+		"withVoice":                              true,
+		"withV2Timeline":                         true,
+	}
+	features := map[string]interface{}{
+		"rweb_lists_timeline_redesign_enabled":                              true,
+		"responsive_web_graphql_exclude_directive_enabled":                  true,
+		"verified_phone_label_enabled":                                      false,
+		"creator_subscriptions_tweet_preview_api_enabled":                   true,
+		"responsive_web_graphql_timeline_navigation_enabled":                true,
+		"responsive_web_graphql_skip_user_profile_image_extensions_enabled": false,
+		"tweetypie_unmention_optimization_enabled":                          true,
+		"vibe_api_enabled":                                                        true,
+		"responsive_web_edit_tweet_api_enabled":                                   true,
+		"graphql_is_translatable_rweb_tweet_is_translatable_enabled":              true,
+		"view_counts_everywhere_api_enabled":                                      true,
+		"longform_notetweets_consumption_enabled":                                 true,
+		"tweet_awards_web_tipping_enabled":                                        false,
+		"freedom_of_speech_not_reach_fetch_enabled":                               true,
+		"standardized_nudges_misinfo":                                             true,
+		"tweet_with_visibility_results_prefer_gql_limited_actions_policy_enabled": false,
+		"interactive_text_enabled":                                                true,
+		"responsive_web_text_conversations_enabled":                               false,
+		"longform_notetweets_rich_text_read_enabled":                              true,
+		"longform_notetweets_inline_media_enabled":                                false,
+		"responsive_web_enhance_cards_enabled":                                    false,
 	}
-	req.URL.RawQuery = q.Encode()
 
-	var timeline timeline
+	if cursor != "" {
+		variables["cursor"] = cursor
+	}
+
+	query := url.Values{}
+	query.Set("variables", mapToJSONString(variables))
+	query.Set("features", mapToJSONString(features))
+	req.URL.RawQuery = query.Encode()
+
+	var timeline timelineV2
 	err = s.RequestAPI(req, &timeline)
 	if err != nil {
 		return nil, "", err
@@ -52,18 +85,59 @@ func (s *Scraper) FetchTweetsByUserID(userID string, maxTweetsNbr int, cursor st
 
 // GetTweet get a single tweet by ID.
 func (s *Scraper) GetTweet(id string) (*Tweet, error) {
-	req, err := s.newRequest("GET", "https://api.twitter.com/2/timeline/conversation/"+id+".json")
+	req, err := s.newRequest("GET", "https://twitter.com/i/api/graphql/wETHelmSuBQR5r-dgUlPxg/TweetDetail")
 	if err != nil {
 		return nil, err
 	}
 
-	var timeline timeline
+	variables := map[string]interface{}{
-	err = s.RequestAPI(req, &timeline)
+		"focalTweetId":                           id,
+		"referrer":                               "profile",
+		"with_rux_injections":                    false,
+		"includePromotedContent":                 true,
+		"withCommunity":                          true,
+		"withQuickPromoteEligibilityTweetFields": true,
+		"withBirdwatchNotes":                     true,
+		"withVoice":                              true,
+		"withV2Timeline":                         true,
+	}
+
+	features := map[string]interface{}{
+		"rweb_lists_timeline_redesign_enabled":                              true,
+		"responsive_web_graphql_exclude_directive_enabled":                  true,
+		"verified_phone_label_enabled":                                      false,
+		"creator_subscriptions_tweet_preview_api_enabled":                   true,
+		"responsive_web_graphql_timeline_navigation_enabled":                true,
+		"responsive_web_graphql_skip_user_profile_image_extensions_enabled": false,
+		"tweetypie_unmention_optimization_enabled":                          true,
+		"vibe_api_enabled":                                                        true,
+		"responsive_web_edit_tweet_api_enabled":                                   true,
+		"graphql_is_translatable_rweb_tweet_is_translatable_enabled":              true,
+		"view_counts_everywhere_api_enabled":                                      true,
+		"longform_notetweets_consumption_enabled":                                 true,
+		"tweet_awards_web_tipping_enabled":                                        false,
+		"freedom_of_speech_not_reach_fetch_enabled":                               true,
+		"standardized_nudges_misinfo":                                             true,
+		"tweet_with_visibility_results_prefer_gql_limited_actions_policy_enabled": false,
+		"interactive_text_enabled":                                                true,
+		"responsive_web_text_conversations_enabled":                               false,
+		"longform_notetweets_rich_text_read_enabled":                              true,
+		"longform_notetweets_inline_media_enabled":                                false,
+		"responsive_web_enhance_cards_enabled":                                    false,
+	}
+
+	query := url.Values{}
+	query.Set("variables", mapToJSONString(variables))
+	query.Set("features", mapToJSONString(features))
+	req.URL.RawQuery = query.Encode()
+
+	var conversation threadedConversation
+	err = s.RequestAPI(req, &conversation)
 	if err != nil {
 		return nil, err
 	}
 
-	tweets, _ := timeline.parseTweets()
+	tweets := conversation.parse()
 	for _, tweet := range tweets {
 		if tweet.ID == id {
 			return tweet, nil

types.go

@@ -29,6 +29,7 @@ type (
 		HTML              string
 		ID                string
 		InReplyToStatus   *Tweet
+		InReplyToStatusID string
 		IsQuoted          bool
 		IsPin             bool
 		IsReply           bool
@@ -40,9 +41,11 @@ type (
 		Photos            []Photo
 		Place             *Place
 		QuotedStatus      *Tweet
+		QuotedStatusID    string
 		Replies           int
 		Retweets          int
 		RetweetedStatus   *Tweet
+		RetweetedStatusID string
 		Text              string
 		TimeParsed        time.Time
 		Timestamp         int64
@@ -66,6 +69,66 @@ type (
 		Error error
 	}
 
+	legacyTweet struct {
+		CreatedAt     string `json:"created_at"`
+		FavoriteCount int    `json:"favorite_count"`
+		FullText      string `json:"full_text"`
+		Entities      struct {
+			Hashtags []struct {
+				Text string `json:"text"`
+			} `json:"hashtags"`
+			Media []struct {
+				MediaURLHttps string `json:"media_url_https"`
+				Type          string `json:"type"`
+				URL           string `json:"url"`
+			} `json:"media"`
+			URLs []struct {
+				ExpandedURL string `json:"expanded_url"`
+				URL         string `json:"url"`
+			} `json:"urls"`
+			UserMentions []struct {
+				IDStr      string `json:"id_str"`
+				Name       string `json:"name"`
+				ScreenName string `json:"screen_name"`
+			} `json:"user_mentions"`
+		} `json:"entities"`
+		ExtendedEntities struct {
+			Media []struct {
+				IDStr                    string `json:"id_str"`
+				MediaURLHttps            string `json:"media_url_https"`
+				ExtSensitiveMediaWarning struct {
+					AdultContent    bool `json:"adult_content"`
+					GraphicViolence bool `json:"graphic_violence"`
+					Other           bool `json:"other"`
+				} `json:"ext_sensitive_media_warning"`
+				Type      string `json:"type"`
+				URL       string `json:"url"`
+				VideoInfo struct {
+					Variants []struct {
+						Bitrate int    `json:"bitrate"`
+						URL     string `json:"url"`
+					} `json:"variants"`
+				} `json:"video_info"`
+			} `json:"media"`
+		} `json:"extended_entities"`
+		IDStr                 string `json:"id_str"`
+		InReplyToStatusIDStr  string `json:"in_reply_to_status_id_str"`
+		Place                 Place  `json:"place"`
+		ReplyCount            int    `json:"reply_count"`
+		RetweetCount          int    `json:"retweet_count"`
+		RetweetedStatusIDStr  string `json:"retweeted_status_id_str"`
+		RetweetedStatusResult struct {
+			Result *result `json:"result"`
+		} `json:"retweeted_status_result"`
+		QuotedStatusIDStr string    `json:"quoted_status_id_str"`
+		Time              time.Time `json:"time"`
+		UserIDStr         string    `json:"user_id_str"`
+		Views             struct {
+			State string `json:"state"`
+			Count string `json:"count"`
+		} `json:"ext_views"`
+	}
+
 	legacyUser struct {
 		CreatedAt   string `json:"created_at"`
 		Description string `json:"description"`

util.go

@@ -2,6 +2,7 @@ package twitterscraper
 
 import (
 	"context"
+	"encoding/json"
 	"net/http"
 	"net/url"
 	"regexp"
@@ -186,6 +187,14 @@ func parseProfile(user legacyUser) Profile {
 	return profile
 }
 
+func mapToJSONString(data map[string]interface{}) string {
+	jsonBytes, err := json.Marshal(data)
+	if err != nil {
+		return ""
+	}
+	return string(jsonBytes)
+}
+
 func stringInSlice(a string, list []string) bool {
 	for _, b := range list {
 		if b == a {