From d9d8ee67b664e3193bf6e0a5aa2962c4f5d19e01 Mon Sep 17 00:00:00 2001 From: Alexander Sheiko Date: Mon, 5 Jun 2023 13:49:56 +0300 Subject: [PATCH] Support for threaded tweets Close #84 --- timeline_v2.go | 41 ++++++++++++++++++++-------- tweets_test.go | 72 +++++++++++++++++++++++++++++++------------------- types.go | 12 ++++++--- 3 files changed, 83 insertions(+), 42 deletions(-) diff --git a/timeline_v2.go b/timeline_v2.go index f7ac971..c3d031f 100644 --- a/timeline_v2.go +++ b/timeline_v2.go @@ -54,14 +54,16 @@ type entry struct { Items []struct { Item struct { ItemContent struct { - TweetResults struct { + TweetDisplayType string `json:"tweetDisplayType"` + TweetResults struct { Result result `json:"result"` } `json:"tweet_results"` } `json:"itemContent"` } `json:"item"` } `json:"items"` ItemContent struct { - TweetResults struct { + TweetDisplayType string `json:"tweetDisplayType"` + TweetResults struct { Result result `json:"result"` } `json:"tweet_results"` } `json:"itemContent"` @@ -124,12 +126,18 @@ func (conversation *threadedConversation) parse() []*Tweet { for _, entry := range instruction.Entries { if entry.Content.ItemContent.TweetResults.Result.Typename == "Tweet" { if tweet := entry.Content.ItemContent.TweetResults.Result.parse(); tweet != nil { + if entry.Content.ItemContent.TweetDisplayType == "SelfThread" { + tweet.IsSelfThread = true + } tweets = append(tweets, tweet) } } for _, item := range entry.Content.Items { if item.Item.ItemContent.TweetResults.Result.Typename == "Tweet" { if tweet := item.Item.ItemContent.TweetResults.Result.parse(); tweet != nil { + if item.Item.ItemContent.TweetDisplayType == "SelfThread" { + tweet.IsSelfThread = true + } tweets = append(tweets, tweet) } } @@ -145,6 +153,16 @@ func (conversation *threadedConversation) parse() []*Tweet { } } } + if tweet.IsSelfThread && tweet.ConversationID == tweet.ID { + for _, childTweet := range tweets { + if childTweet.IsSelfThread && childTweet.ID != tweet.ID { + tweet.Thread = append(tweet.Thread, childTweet) + } + } + if len(tweet.Thread) == 0 { + tweet.IsSelfThread = false + } + } } return tweets } @@ -154,15 +172,16 @@ func parseLegacyTweet(user *legacyUser, tweet *legacyTweet) *Tweet { name := user.Name tweetID := tweet.IDStr tw := &Tweet{ - ID: tweetID, - Likes: tweet.FavoriteCount, - Name: name, - PermanentURL: fmt.Sprintf("https://twitter.com/%s/status/%s", username, tweetID), - Replies: tweet.ReplyCount, - Retweets: tweet.RetweetCount, - Text: tweet.FullText, - UserID: tweet.UserIDStr, - Username: username, + ConversationID: tweet.ConversationIDStr, + ID: tweetID, + Likes: tweet.FavoriteCount, + Name: name, + PermanentURL: fmt.Sprintf("https://twitter.com/%s/status/%s", username, tweetID), + Replies: tweet.ReplyCount, + Retweets: tweet.RetweetCount, + Text: tweet.FullText, + UserID: tweet.UserIDStr, + Username: username, } tm, err := time.Parse(time.RubyDate, tweet.CreatedAt) diff --git a/tweets_test.go b/tweets_test.go index b548fda..25c5b21 100644 --- a/tweets_test.go +++ b/tweets_test.go @@ -73,16 +73,17 @@ func TestGetTweets(t *testing.T) { func TestGetTweet(t *testing.T) { sample := twitterscraper.Tweet{ - HTML: "That thing you didn’t Tweet but wanted to but didn’t but got so close but then were like nah.

We have a place for that now—Fleets!

Rolling out to everyone starting today.
", - ID: "1328684389388185600", - Name: "Twitter", - PermanentURL: "https://twitter.com/Twitter/status/1328684389388185600", - Photos: nil, - Text: "That thing you didn’t Tweet but wanted to but didn’t but got so close but then were like nah. \n\nWe have a place for that now—Fleets! \n\nRolling out to everyone starting today. https://t.co/auQAHXZMfH", - TimeParsed: time.Date(2020, 11, 17, 13, 0, 18, 0, time.FixedZone("UTC", 0)), - Timestamp: 1605618018, - UserID: "783214", - Username: "Twitter", + ConversationID: "1328684389388185600", + HTML: "That thing you didn’t Tweet but wanted to but didn’t but got so close but then were like nah.

We have a place for that now—Fleets!

Rolling out to everyone starting today.
", + ID: "1328684389388185600", + Name: "Twitter", + PermanentURL: "https://twitter.com/Twitter/status/1328684389388185600", + Photos: nil, + Text: "That thing you didn’t Tweet but wanted to but didn’t but got so close but then were like nah. \n\nWe have a place for that now—Fleets! \n\nRolling out to everyone starting today. https://t.co/auQAHXZMfH", + TimeParsed: time.Date(2020, 11, 17, 13, 0, 18, 0, time.FixedZone("UTC", 0)), + Timestamp: 1605618018, + UserID: "783214", + Username: "Twitter", Videos: []twitterscraper.Video{{ ID: "1328684333599756289", Preview: "https://pbs.twimg.com/amplify_video_thumb/1328684333599756289/img/cP5KwbIXbGunNSBy.jpg", @@ -119,11 +120,12 @@ func TestTweetMentions(t *testing.T) { func TestQuotedAndReply(t *testing.T) { sample := &twitterscraper.Tweet{ - HTML: "The Easiest Problem Everyone Gets Wrong

[new video] --> https://t.co/YdaeDYmPAU
", - ID: "1237110546383724547", - Likes: 485, - Name: "Vsauce2", - PermanentURL: "https://twitter.com/VsauceTwo/status/1237110546383724547", + ConversationID: "1237110546383724547", + HTML: "The Easiest Problem Everyone Gets Wrong

[new video] --> https://t.co/YdaeDYmPAU
", + ID: "1237110546383724547", + Likes: 485, + Name: "Vsauce2", + PermanentURL: "https://twitter.com/VsauceTwo/status/1237110546383724547", Photos: []twitterscraper.Photo{{ ID: "1237110473486729218", URL: "https://pbs.twimg.com/media/ESsZa9AXgAIAYnF.jpg", @@ -164,18 +166,19 @@ func TestQuotedAndReply(t *testing.T) { } func TestRetweet(t *testing.T) { sample := &twitterscraper.Tweet{ - HTML: "We’ve seen an increase in attacks against Asian communities and individuals around the world. It’s important to know that this isn’t new; throughout history, Asians have experienced violence and exclusion. However, their diverse lived experiences have largely been overlooked.", - ID: "1359151057872580612", - Likes: 6683, - Name: "Twitter Together", - PermanentURL: "https://twitter.com/TwitterTogether/status/1359151057872580612", - Replies: 456, - Retweets: 1495, - Text: "We’ve seen an increase in attacks against Asian communities and individuals around the world. It’s important to know that this isn’t new; throughout history, Asians have experienced violence and exclusion. However, their diverse lived experiences have largely been overlooked.", - TimeParsed: time.Date(2021, 02, 9, 14, 43, 58, 0, time.FixedZone("UTC", 0)), - Timestamp: 1612881838, - UserID: "773578328498372608", - Username: "TwitterTogether", + ConversationID: "1359151057872580612", + HTML: "We’ve seen an increase in attacks against Asian communities and individuals around the world. It’s important to know that this isn’t new; throughout history, Asians have experienced violence and exclusion. However, their diverse lived experiences have largely been overlooked.", + ID: "1359151057872580612", + Likes: 6683, + Name: "Twitter Together", + PermanentURL: "https://twitter.com/TwitterTogether/status/1359151057872580612", + Replies: 456, + Retweets: 1495, + Text: "We’ve seen an increase in attacks against Asian communities and individuals around the world. It’s important to know that this isn’t new; throughout history, Asians have experienced violence and exclusion. However, their diverse lived experiences have largely been overlooked.", + TimeParsed: time.Date(2021, 02, 9, 14, 43, 58, 0, time.FixedZone("UTC", 0)), + Timestamp: 1612881838, + UserID: "773578328498372608", + Username: "TwitterTogether", } scraper := twitterscraper.New() tweet, err := scraper.GetTweet("1362849141248974853") @@ -217,3 +220,18 @@ func TestTweetViews(t *testing.T) { } } } + +func TestTweetThread(t *testing.T) { + scraper := twitterscraper.New() + tweet, err := scraper.GetTweet("1665602315745673217") + if err != nil { + t.Fatal(err) + } else { + if !tweet.IsSelfThread { + t.Error("IsSelfThread must be True") + } + if len(tweet.Thread) != 7 { + t.Error("Thread length must be 7") + } + } +} diff --git a/types.go b/types.go index 5ac7aa8..0528aa5 100644 --- a/types.go +++ b/types.go @@ -25,6 +25,7 @@ type ( // Tweet type. Tweet struct { + ConversationID string Hashtags []string HTML string ID string @@ -34,6 +35,7 @@ type ( IsPin bool IsReply bool IsRetweet bool + IsSelfThread bool Likes int Name string Mentions []Mention @@ -47,6 +49,7 @@ type ( RetweetedStatus *Tweet RetweetedStatusID string Text string + Thread []*Tweet TimeParsed time.Time Timestamp int64 URLs []string @@ -70,10 +73,11 @@ type ( } legacyTweet struct { - CreatedAt string `json:"created_at"` - FavoriteCount int `json:"favorite_count"` - FullText string `json:"full_text"` - Entities struct { + ConversationIDStr string `json:"conversation_id_str"` + CreatedAt string `json:"created_at"` + FavoriteCount int `json:"favorite_count"` + FullText string `json:"full_text"` + Entities struct { Hashtags []struct { Text string `json:"text"` } `json:"hashtags"`