lain/twitter-scrapper
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity

Support for threaded tweets

Browse source 
Close #84
This commit is contained in:
Alexander Sheiko 2023-06-05 13:49:56 +03:00
parent 0f96589c74
commit d9d8ee67b6
3 changed files with 83 additions and 42 deletions
Download patch file Download diff file Expand all files Collapse all files

41
timeline_v2.go
View file

@ -54,14 +54,16 @@ type entry struct {
Items []struct { Items []struct {
Item struct { Item struct {
ItemContent struct { ItemContent struct {
TweetResults struct { TweetDisplayType string `json:"tweetDisplayType"`
TweetResults struct {
Result result `json:"result"` Result result `json:"result"`
} `json:"tweet_results"` } `json:"tweet_results"`
} `json:"itemContent"` } `json:"itemContent"`
} `json:"item"` } `json:"item"`
} `json:"items"` } `json:"items"`
ItemContent struct { ItemContent struct {
TweetResults struct { TweetDisplayType string `json:"tweetDisplayType"`
TweetResults struct {
Result result `json:"result"` Result result `json:"result"`
} `json:"tweet_results"` } `json:"tweet_results"`
} `json:"itemContent"` } `json:"itemContent"`
@ -124,12 +126,18 @@ func (conversation *threadedConversation) parse() []*Tweet {
for _, entry := range instruction.Entries { for _, entry := range instruction.Entries {
if entry.Content.ItemContent.TweetResults.Result.Typename == "Tweet" { if entry.Content.ItemContent.TweetResults.Result.Typename == "Tweet" {
if tweet := entry.Content.ItemContent.TweetResults.Result.parse(); tweet != nil { if tweet := entry.Content.ItemContent.TweetResults.Result.parse(); tweet != nil {
if entry.Content.ItemContent.TweetDisplayType == "SelfThread" {
tweet.IsSelfThread = true
}
tweets = append(tweets, tweet) tweets = append(tweets, tweet)
} }
} }
for _, item := range entry.Content.Items { for _, item := range entry.Content.Items {
if item.Item.ItemContent.TweetResults.Result.Typename == "Tweet" { if item.Item.ItemContent.TweetResults.Result.Typename == "Tweet" {
if tweet := item.Item.ItemContent.TweetResults.Result.parse(); tweet != nil { if tweet := item.Item.ItemContent.TweetResults.Result.parse(); tweet != nil {
if item.Item.ItemContent.TweetDisplayType == "SelfThread" {
tweet.IsSelfThread = true
}
tweets = append(tweets, tweet) tweets = append(tweets, tweet)
} }
} }
@ -145,6 +153,16 @@ func (conversation *threadedConversation) parse() []*Tweet {
} }
} }
} }
if tweet.IsSelfThread && tweet.ConversationID == tweet.ID {
for _, childTweet := range tweets {
if childTweet.IsSelfThread && childTweet.ID != tweet.ID {
tweet.Thread = append(tweet.Thread, childTweet)
}
}
if len(tweet.Thread) == 0 {
tweet.IsSelfThread = false
}
}
} }
return tweets return tweets
} }
@ -154,15 +172,16 @@ func parseLegacyTweet(user *legacyUser, tweet *legacyTweet) *Tweet {
name := user.Name name := user.Name
tweetID := tweet.IDStr tweetID := tweet.IDStr
tw := &Tweet{ tw := &Tweet{
ID: tweetID, ConversationID: tweet.ConversationIDStr,
Likes: tweet.FavoriteCount, ID: tweetID,
Name: name, Likes: tweet.FavoriteCount,
PermanentURL: fmt.Sprintf("https://twitter.com/%s/status/%s", username, tweetID), Name: name,
Replies: tweet.ReplyCount, PermanentURL: fmt.Sprintf("https://twitter.com/%s/status/%s", username, tweetID),
Retweets: tweet.RetweetCount, Replies: tweet.ReplyCount,
Text: tweet.FullText, Retweets: tweet.RetweetCount,
UserID: tweet.UserIDStr, Text: tweet.FullText,
Username: username, UserID: tweet.UserIDStr,
Username: username,
} }
tm, err := time.Parse(time.RubyDate, tweet.CreatedAt) tm, err := time.Parse(time.RubyDate, tweet.CreatedAt)

72
tweets_test.go
View file

@ -73,16 +73,17 @@ func TestGetTweets(t *testing.T) {
func TestGetTweet(t *testing.T) { func TestGetTweet(t *testing.T) {
sample := twitterscraper.Tweet{ sample := twitterscraper.Tweet{
HTML: "That thing you didn’t Tweet but wanted to but didn’t but got so close but then were like nah. <br><br>We have a place for that now—Fleets! <br><br>Rolling out to everyone starting today. <br><a href=\"https://t.co/auQAHXZMfH\"><img src=\"https://pbs.twimg.com/amplify_video_thumb/1328684333599756289/img/cP5KwbIXbGunNSBy.jpg\"/></a>", ConversationID: "1328684389388185600",
ID: "1328684389388185600", HTML: "That thing you didn’t Tweet but wanted to but didn’t but got so close but then were like nah. <br><br>We have a place for that now—Fleets! <br><br>Rolling out to everyone starting today. <br><a href=\"https://t.co/auQAHXZMfH\"><img src=\"https://pbs.twimg.com/amplify_video_thumb/1328684333599756289/img/cP5KwbIXbGunNSBy.jpg\"/></a>",
Name: "Twitter", ID: "1328684389388185600",
PermanentURL: "https://twitter.com/Twitter/status/1328684389388185600", Name: "Twitter",
Photos: nil, PermanentURL: "https://twitter.com/Twitter/status/1328684389388185600",
Text: "That thing you didn’t Tweet but wanted to but didn’t but got so close but then were like nah. \n\nWe have a place for that now—Fleets! \n\nRolling out to everyone starting today. https://t.co/auQAHXZMfH", Photos: nil,
TimeParsed: time.Date(2020, 11, 17, 13, 0, 18, 0, time.FixedZone("UTC", 0)), Text: "That thing you didn’t Tweet but wanted to but didn’t but got so close but then were like nah. \n\nWe have a place for that now—Fleets! \n\nRolling out to everyone starting today. https://t.co/auQAHXZMfH",
Timestamp: 1605618018, TimeParsed: time.Date(2020, 11, 17, 13, 0, 18, 0, time.FixedZone("UTC", 0)),
UserID: "783214", Timestamp: 1605618018,
Username: "Twitter", UserID: "783214",
Username: "Twitter",
Videos: []twitterscraper.Video{{ Videos: []twitterscraper.Video{{
ID: "1328684333599756289", ID: "1328684333599756289",
Preview: "https://pbs.twimg.com/amplify_video_thumb/1328684333599756289/img/cP5KwbIXbGunNSBy.jpg", Preview: "https://pbs.twimg.com/amplify_video_thumb/1328684333599756289/img/cP5KwbIXbGunNSBy.jpg",
@ -119,11 +120,12 @@ func TestTweetMentions(t *testing.T) {
func TestQuotedAndReply(t *testing.T) { func TestQuotedAndReply(t *testing.T) {
sample := &twitterscraper.Tweet{ sample := &twitterscraper.Tweet{
HTML: "The Easiest Problem Everyone Gets Wrong <br><br>[new video] --&gt; <a href=\"https://youtu.be/ytfCdqWhmdg\">https://t.co/YdaeDYmPAU</a> <br><a href=\"https://t.co/iKu4Xs6o2V\"><img src=\"https://pbs.twimg.com/media/ESsZa9AXgAIAYnF.jpg\"/></a>", ConversationID: "1237110546383724547",
ID: "1237110546383724547", HTML: "The Easiest Problem Everyone Gets Wrong <br><br>[new video] --&gt; <a href=\"https://youtu.be/ytfCdqWhmdg\">https://t.co/YdaeDYmPAU</a> <br><a href=\"https://t.co/iKu4Xs6o2V\"><img src=\"https://pbs.twimg.com/media/ESsZa9AXgAIAYnF.jpg\"/></a>",
Likes: 485, ID: "1237110546383724547",
Name: "Vsauce2", Likes: 485,
PermanentURL: "https://twitter.com/VsauceTwo/status/1237110546383724547", Name: "Vsauce2",
PermanentURL: "https://twitter.com/VsauceTwo/status/1237110546383724547",
Photos: []twitterscraper.Photo{{ Photos: []twitterscraper.Photo{{
ID: "1237110473486729218", ID: "1237110473486729218",
URL: "https://pbs.twimg.com/media/ESsZa9AXgAIAYnF.jpg", URL: "https://pbs.twimg.com/media/ESsZa9AXgAIAYnF.jpg",
@ -164,18 +166,19 @@ func TestQuotedAndReply(t *testing.T) {
} }
func TestRetweet(t *testing.T) { func TestRetweet(t *testing.T) {
sample := &twitterscraper.Tweet{ sample := &twitterscraper.Tweet{
HTML: "We’ve seen an increase in attacks against Asian communities and individuals around the world. It’s important to know that this isn’t new; throughout history, Asians have experienced violence and exclusion. However, their diverse lived experiences have largely been overlooked.", ConversationID: "1359151057872580612",
ID: "1359151057872580612", HTML: "We’ve seen an increase in attacks against Asian communities and individuals around the world. It’s important to know that this isn’t new; throughout history, Asians have experienced violence and exclusion. However, their diverse lived experiences have largely been overlooked.",
Likes: 6683, ID: "1359151057872580612",
Name: "Twitter Together", Likes: 6683,
PermanentURL: "https://twitter.com/TwitterTogether/status/1359151057872580612", Name: "Twitter Together",
Replies: 456, PermanentURL: "https://twitter.com/TwitterTogether/status/1359151057872580612",
Retweets: 1495, Replies: 456,
Text: "We’ve seen an increase in attacks against Asian communities and individuals around the world. It’s important to know that this isn’t new; throughout history, Asians have experienced violence and exclusion. However, their diverse lived experiences have largely been overlooked.", Retweets: 1495,
TimeParsed: time.Date(2021, 02, 9, 14, 43, 58, 0, time.FixedZone("UTC", 0)), Text: "We’ve seen an increase in attacks against Asian communities and individuals around the world. It’s important to know that this isn’t new; throughout history, Asians have experienced violence and exclusion. However, their diverse lived experiences have largely been overlooked.",
Timestamp: 1612881838, TimeParsed: time.Date(2021, 02, 9, 14, 43, 58, 0, time.FixedZone("UTC", 0)),
UserID: "773578328498372608", Timestamp: 1612881838,
Username: "TwitterTogether", UserID: "773578328498372608",
Username: "TwitterTogether",
} }
scraper := twitterscraper.New() scraper := twitterscraper.New()
tweet, err := scraper.GetTweet("1362849141248974853") tweet, err := scraper.GetTweet("1362849141248974853")
@ -217,3 +220,18 @@ func TestTweetViews(t *testing.T) {
} }
} }
} }
func TestTweetThread(t *testing.T) {
scraper := twitterscraper.New()
tweet, err := scraper.GetTweet("1665602315745673217")
if err != nil {
t.Fatal(err)
} else {
if !tweet.IsSelfThread {
t.Error("IsSelfThread must be True")
}
if len(tweet.Thread) != 7 {
t.Error("Thread length must be 7")
}
}
}

12
types.go
View file

@ -25,6 +25,7 @@ type (
// Tweet type. // Tweet type.
Tweet struct { Tweet struct {
ConversationID string
Hashtags []string Hashtags []string
HTML string HTML string
ID string ID string
@ -34,6 +35,7 @@ type (
IsPin bool IsPin bool
IsReply bool IsReply bool
IsRetweet bool IsRetweet bool
IsSelfThread bool
Likes int Likes int
Name string Name string
Mentions []Mention Mentions []Mention
@ -47,6 +49,7 @@ type (
RetweetedStatus *Tweet RetweetedStatus *Tweet
RetweetedStatusID string RetweetedStatusID string
Text string Text string
Thread []*Tweet
TimeParsed time.Time TimeParsed time.Time
Timestamp int64 Timestamp int64
URLs []string URLs []string
@ -70,10 +73,11 @@ type (
} }
legacyTweet struct { legacyTweet struct {
CreatedAt string `json:"created_at"` ConversationIDStr string `json:"conversation_id_str"`
FavoriteCount int `json:"favorite_count"` CreatedAt string `json:"created_at"`
FullText string `json:"full_text"` FavoriteCount int `json:"favorite_count"`
Entities struct { FullText string `json:"full_text"`
Entities struct {
Hashtags []struct { Hashtags []struct {
Text string `json:"text"` Text string `json:"text"`
} `json:"hashtags"` } `json:"hashtags"`