package twitterscraper_test
import (
"context"
"testing"
"time"
"github.com/google/go-cmp/cmp"
"github.com/google/go-cmp/cmp/cmpopts"
twitterscraper "github.com/imperatrona/twitter-scraper"
)
var cmpOptions = cmp.Options{
cmpopts.IgnoreFields(twitterscraper.Tweet{}, "Likes"),
cmpopts.IgnoreFields(twitterscraper.Tweet{}, "Replies"),
cmpopts.IgnoreFields(twitterscraper.Tweet{}, "Retweets"),
}
func TestGetTweets(t *testing.T) {
count := 0
maxTweetsNbr := 300
dupcheck := make(map[string]bool)
scraper := twitterscraper.New()
err := scraper.LoginOpenAccount()
if err != nil {
t.Fatalf("LoginOpenAccount() error = %v", err)
}
for tweet := range scraper.GetTweets(context.Background(), "Twitter", maxTweetsNbr) {
if tweet.Error != nil {
t.Error(tweet.Error)
} else {
count++
if tweet.ID == "" {
t.Error("Expected tweet ID is empty")
} else {
if dupcheck[tweet.ID] {
t.Errorf("Detect duplicated tweet ID: %s", tweet.ID)
} else {
dupcheck[tweet.ID] = true
}
}
if tweet.UserID == "" {
t.Error("Expected tweet UserID is empty")
}
if tweet.Username == "" {
t.Error("Expected tweet Username is empty")
}
if tweet.PermanentURL == "" {
t.Error("Expected tweet PermanentURL is empty")
}
if tweet.Text == "" {
t.Error("Expected tweet Text is empty")
}
if tweet.TimeParsed.IsZero() {
t.Error("Expected tweet TimeParsed is zero")
}
if tweet.Timestamp == 0 {
t.Error("Expected tweet Timestamp is greater than zero")
}
for _, video := range tweet.Videos {
if video.ID == "" {
t.Error("Expected tweet video ID is empty")
}
if video.Preview == "" {
t.Error("Expected tweet video Preview is empty")
}
if video.URL == "" {
t.Error("Expected tweet video URL is empty")
}
}
}
}
if count != maxTweetsNbr {
t.Errorf("Expected tweets count=%v, got: %v", maxTweetsNbr, count)
}
}
func assertGetTweet(t *testing.T, expectedTweet *twitterscraper.Tweet) {
actualTweet, err := testScraper.GetTweet(expectedTweet.ID)
if err != nil {
t.Error(err)
} else if diff := cmp.Diff(expectedTweet, actualTweet, cmpOptions...); diff != "" {
t.Error("Resulting tweet does not match the sample", diff)
}
}
func TestGetTweetWithVideo(t *testing.T) {
expectedTweet := twitterscraper.Tweet{
ConversationID: "1328684389388185600",
HTML: "That thing you didn’t Tweet but wanted to but didn’t but got so close but then were like nah.
We have a place for that now—Fleets!
Rolling out to everyone starting today.
![](\"https://pbs.twimg.com/amplify_video_thumb/1328684333599756289/img/cP5KwbIXbGunNSBy.jpg\"/)
",
ID: "1328684389388185600",
Name: "Twitter",
PermanentURL: "https://twitter.com/Twitter/status/1328684389388185600",
Photos: nil,
Text: "That thing you didn’t Tweet but wanted to but didn’t but got so close but then were like nah. \n\nWe have a place for that now—Fleets! \n\nRolling out to everyone starting today. https://t.co/auQAHXZMfH",
TimeParsed: time.Date(2020, 11, 17, 13, 0, 18, 0, time.FixedZone("UTC", 0)),
Timestamp: 1605618018,
UserID: "783214",
Username: "Twitter",
Videos: []twitterscraper.Video{{
ID: "1328684333599756289",
Preview: "https://pbs.twimg.com/amplify_video_thumb/1328684333599756289/img/cP5KwbIXbGunNSBy.jpg",
URL: "https://video.twimg.com/amplify_video/1328684333599756289/vid/960x720/PcL8yv8KhgQ48Qpt.mp4?tag=13",
}},
}
assertGetTweet(t, &expectedTweet)
}
func TestGetTweetWithMultiplePhotos(t *testing.T) {
expectedTweet := twitterscraper.Tweet{
ConversationID: "1390026628957417473",
HTML: `no bird too tall, no crop too short
introducing bigger and better images on iOS and Android, now available to everyone
`,
ID: "1390026628957417473",
Name: "Twitter",
PermanentURL: "https://twitter.com/Twitter/status/1390026628957417473",
Photos: []twitterscraper.Photo{
{ID: "1390026620472332292", URL: "https://pbs.twimg.com/media/E0pd2L2XEAQ_gnn.jpg"},
{ID: "1390026626214371334", URL: "https://pbs.twimg.com/media/E0pd2hPXoAY9-TZ.jpg"},
},
Text: "no bird too tall, no crop too short\n\nintroducing bigger and better images on iOS and Android, now available to everyone https://t.co/2buHfhfRAx",
TimeParsed: time.Date(2021, 5, 5, 19, 32, 28, 0, time.FixedZone("UTC", 0)),
Timestamp: 1620243148,
UserID: "783214",
Username: "Twitter",
}
assertGetTweet(t, &expectedTweet)
}
func TestGetTweetWithGIF(t *testing.T) {
if skipAuthTest {
t.Skip("Skipping test due to environment variable")
}
expectedTweet := twitterscraper.Tweet{
ConversationID: "1288540609310056450",
GIFs: []twitterscraper.GIF{
{
ID: "1288540582768517123",
Preview: "https://pbs.twimg.com/tweet_video_thumb/EeHQ1UKXoAMVxWB.jpg",
URL: "https://video.twimg.com/tweet_video/EeHQ1UKXoAMVxWB.mp4",
},
},
Hashtags: []string{"CountdownToMars"},
HTML: `Like for liftoff! #CountdownToMars
`,
ID: "1288540609310056450",
Name: "Twitter",
PermanentURL: "https://twitter.com/Twitter/status/1288540609310056450",
Text: "Like for liftoff! #CountdownToMars https://t.co/yLe331pHfY",
TimeParsed: time.Date(2020, 7, 29, 18, 23, 15, 0, time.FixedZone("UTC", 0)),
Timestamp: 1596046995,
UserID: "783214",
Username: "Twitter",
}
assertGetTweet(t, &expectedTweet)
}
func TestGetTweetWithPhotoAndGIF(t *testing.T) {
if skipAuthTest {
t.Skip("Skipping test due to environment variable")
}
expectedTweet := twitterscraper.Tweet{
ConversationID: "1580661436132757506",
GIFs: []twitterscraper.GIF{
{
ID: "1580661428335382531",
Preview: "https://pbs.twimg.com/tweet_video_thumb/Fe-jMcIXkAMXK_W.jpg",
URL: "https://video.twimg.com/tweet_video/Fe-jMcIXkAMXK_W.mp4",
},
},
HTML: `a hit Tweet
`,
ID: "1580661436132757506",
Name: "Twitter",
PermanentURL: "https://twitter.com/Twitter/status/1580661436132757506",
Photos: []twitterscraper.Photo{{ID: "1580661428326907904", URL: "https://pbs.twimg.com/media/Fe-jMcGWQAAFWoG.jpg"}},
Text: "a hit Tweet https://t.co/2C7cah4KzW",
TimeParsed: time.Date(2022, 10, 13, 20, 47, 8, 0, time.FixedZone("UTC", 0)),
Timestamp: 1665694028,
UserID: "783214",
Username: "Twitter",
}
assertGetTweet(t, &expectedTweet)
}
func TestTweetMentions(t *testing.T) {
sample := []twitterscraper.Mention{{
ID: "7018222",
Username: "davidmcraney",
Name: "David McRaney",
}}
tweet, err := testScraper.GetTweet("1554522888904101890")
if err != nil {
t.Error(err)
} else {
if diff := cmp.Diff(sample, tweet.Mentions, cmpOptions...); diff != "" {
t.Error("Resulting tweet does not match the sample", diff)
}
}
}
func TestQuotedAndReply(t *testing.T) {
sample := &twitterscraper.Tweet{
ConversationID: "1237110546383724547",
HTML: "The Easiest Problem Everyone Gets Wrong
[new video] --> https://t.co/YdaeDYmPAU
![](\"https://pbs.twimg.com/media/ESsZa9AXgAIAYnF.jpg\"/)
",
ID: "1237110546383724547",
Likes: 485,
Name: "Vsauce2",
PermanentURL: "https://twitter.com/VsauceTwo/status/1237110546383724547",
Photos: []twitterscraper.Photo{{
ID: "1237110473486729218",
URL: "https://pbs.twimg.com/media/ESsZa9AXgAIAYnF.jpg",
}},
Replies: 12,
Retweets: 18,
Text: "The Easiest Problem Everyone Gets Wrong \n\n[new video] --> https://t.co/YdaeDYmPAU https://t.co/iKu4Xs6o2V",
TimeParsed: time.Date(2020, 0o3, 9, 20, 18, 33, 0, time.FixedZone("UTC", 0)),
Timestamp: 1583785113,
URLs: []string{"https://youtu.be/ytfCdqWhmdg"},
UserID: "978944851",
Username: "VsauceTwo",
}
tweet, err := testScraper.GetTweet("1237110897597976576")
if err != nil {
t.Error(err)
} else {
if !tweet.IsQuoted {
t.Error("IsQuoted must be True")
}
if diff := cmp.Diff(sample, tweet.QuotedStatus, cmpOptions...); diff != "" {
t.Error("Resulting quote does not match the sample", diff)
}
}
tweet, err = testScraper.GetTweet("1237111868445134850")
if err != nil {
t.Error(err)
} else {
if !tweet.IsReply {
t.Error("IsReply must be True")
}
if diff := cmp.Diff(sample, tweet.InReplyToStatus, cmpOptions...); diff != "" {
t.Error("Resulting reply does not match the sample", diff)
}
}
}
func TestRetweet(t *testing.T) {
sample := &twitterscraper.Tweet{
ConversationID: "1359151057872580612",
HTML: "We’ve seen an increase in attacks against Asian communities and individuals around the world. It’s important to know that this isn’t new; throughout history, Asians have experienced violence and exclusion. However, their diverse lived experiences have largely been overlooked.",
ID: "1359151057872580612",
IsSelfThread: false,
Likes: 6683,
Name: "Twitter Together",
PermanentURL: "https://twitter.com/TwitterTogether/status/1359151057872580612",
Replies: 456,
Retweets: 1495,
Text: "We’ve seen an increase in attacks against Asian communities and individuals around the world. It’s important to know that this isn’t new; throughout history, Asians have experienced violence and exclusion. However, their diverse lived experiences have largely been overlooked.",
TimeParsed: time.Date(2021, 02, 9, 14, 43, 58, 0, time.FixedZone("UTC", 0)),
Timestamp: 1612881838,
UserID: "773578328498372608",
Username: "TwitterTogether",
}
tweet, err := testScraper.GetTweet("1362849141248974853")
if err != nil {
t.Error(err)
} else {
if !tweet.IsRetweet {
t.Error("IsRetweet must be True")
}
if diff := cmp.Diff(sample, tweet.RetweetedStatus, cmpOptions...); diff != "" {
t.Error("Resulting retweet does not match the sample", diff)
}
}
}
func TestTweetViews(t *testing.T) {
sample := &twitterscraper.Tweet{
HTML: "Replies and likes don’t tell the whole story. We’re making it easier to tell *just* how many people have seen your Tweets with the addition of view counts, shown right next to likes. Now on iOS and Android, web coming soon.",
ID: "1606055187348688896",
Likes: 2839,
Name: "Twitter Support",
PermanentURL: "https://twitter.com/TwitterSupport/status/1606055187348688896",
Replies: 3427,
Retweets: 783,
Text: "Replies and likes don’t tell the whole story. We’re making it easier to tell *just* how many people have seen your Tweets with the addition of view counts, shown right next to likes. Now on iOS and Android, web coming soon.",
TimeParsed: time.Date(2022, 12, 22, 22, 32, 50, 0, time.FixedZone("UTC", 0)),
Timestamp: 1612881838,
UserID: "17874544",
Username: "TwitterSupport",
Views: 3189278,
}
tweet, err := testScraper.GetTweet("1606055187348688896")
if err != nil {
t.Error(err)
} else {
if tweet.Views < sample.Views {
t.Error("Views must be greater than or equal to the sample")
}
}
}
func TestTweetThread(t *testing.T) {
if skipAuthTest {
t.Skip("Skipping test due to environment variable")
}
tweet, err := testScraper.GetTweet("1665602315745673217")
if err != nil {
t.Fatal(err)
} else {
if !tweet.IsSelfThread {
t.Error("IsSelfThread must be True")
}
if len(tweet.Thread) != 7 {
t.Error("Thread length must be 7")
}
}
}