From cc633cb1ead91cff8afa3d7d67a0c94d6e16b92c Mon Sep 17 00:00:00 2001 From: Valentine Date: Tue, 20 Feb 2024 23:27:22 +0300 Subject: [PATCH] fix tests --- .gitignore | 2 + api.go | 7 ++ api_test.go | 15 ++++- auth_test.go | 114 ++++++++++++++++++++++++++----- profile_test.go | 40 +++-------- tweets_test.go | 174 ++++++++++++++++++++++++------------------------ 6 files changed, 214 insertions(+), 138 deletions(-) diff --git a/.gitignore b/.gitignore index 0a035ec..768ec23 100644 --- a/.gitignore +++ b/.gitignore @@ -1,3 +1,5 @@ *.htm? *.json .idea/ + +!.vscode/settings.json \ No newline at end of file diff --git a/api.go b/api.go index 64ad37a..bd1e3ef 100644 --- a/api.go +++ b/api.go @@ -105,3 +105,10 @@ func (s *Scraper) GetGuestToken() error { return nil } + +func (s *Scraper) ClearGuestToken() error { + s.guestToken = "" + s.guestCreatedAt = time.Time{} + + return nil +} diff --git a/api_test.go b/api_test.go index d35b8f6..16af4c1 100644 --- a/api_test.go +++ b/api_test.go @@ -2,12 +2,11 @@ package twitterscraper_test import ( "testing" - - twitterscraper "github.com/imperatrona/twitter-scraper" ) func TestGetGuestToken(t *testing.T) { - scraper := twitterscraper.New() + scraper := newTestScraper(true) + if err := scraper.GetGuestToken(); err != nil { t.Errorf("getGuestToken() error = %v", err) } @@ -15,3 +14,13 @@ func TestGetGuestToken(t *testing.T) { t.Error("Expected non-empty guestToken") } } + +func TestClearGuestToken(t *testing.T) { + scraper := newTestScraper(false) + + scraper.ClearGuestToken() + + if scraper.IsGuestToken() { + t.Error("Expected empty guestToken") + } +} diff --git a/auth_test.go b/auth_test.go index 10dd00b..9dfc4fa 100644 --- a/auth_test.go +++ b/auth_test.go @@ -1,47 +1,97 @@ package twitterscraper_test import ( + "encoding/json" "fmt" + "net/http" "os" + "strings" "testing" twitterscraper "github.com/imperatrona/twitter-scraper" ) var ( - username = os.Getenv("TWITTER_USERNAME") - password = os.Getenv("TWITTER_PASSWORD") - email = os.Getenv("TWITTER_EMAIL") - skipAuthTest = os.Getenv("SKIP_AUTH_TEST") != "" - testScraper = twitterscraper.New() + proxy = os.Getenv("PROXY") + proxyRequired = os.Getenv("PROXY_REQUIRED") != "" + authToken = os.Getenv("AUTH_TOKEN") + ct0 = os.Getenv("CT0") + cookies = os.Getenv("COOKIES") + username = os.Getenv("TWITTER_USERNAME") + password = os.Getenv("TWITTER_PASSWORD") + email = os.Getenv("TWITTER_EMAIL") + skipAuthTest = os.Getenv("SKIP_AUTH_TEST") != "" + testScraper = newTestScraper(false) ) func init() { - if username != "" && password != "" && !skipAuthTest { + if skipAuthTest { + return + } + + if authToken != "" && ct0 != "" { + testScraper.SetAuthToken(authToken, ct0) + if !testScraper.IsLoggedIn() { + panic("Invalid AuthToken") + } + return + } + + if cookies != "" { + var parsedCookies []*http.Cookie + json.NewDecoder(strings.NewReader(cookies)).Decode(&parsedCookies) + testScraper.SetCookies(parsedCookies) + if !testScraper.IsLoggedIn() { + panic("Invalid Cookies") + } + return + } + + if username != "" && password != "" { err := testScraper.Login(username, password, email) if err != nil { panic(fmt.Sprintf("Login() error = %v", err)) } + return } + + panic("None of any auth data provided, provide any variables or set SKIP_AUTH_TEST.") } -func TestAuth(t *testing.T) { - if skipAuthTest { +func newTestScraper(skip_auth bool) *twitterscraper.Scraper { + s := twitterscraper.New() + + if proxy != "" && proxyRequired { + err := s.SetProxy(proxy) + if err != nil { + panic(fmt.Sprintf("SetProxy() error = %v", err)) + } + } + + // Check connection by getting guest token + if err := s.GetGuestToken(); err != nil { + panic(fmt.Sprintf("cannot get guest token, can also be error with connection to twitter.\n %v", err)) + } + + if skip_auth == true || !skipAuthTest { + s.ClearGuestToken() + return s + } + + return s +} + +func TestLoginPassword(t *testing.T) { + if skipAuthTest || username == "" || password == "" { t.Skip("Skipping test due to environment variable") } - scraper := twitterscraper.New() + scraper := newTestScraper(true) if err := scraper.Login(username, password, email); err != nil { t.Fatalf("Login() error = %v", err) } if !scraper.IsLoggedIn() { t.Fatalf("Expected IsLoggedIn() = true") } - cookies := scraper.GetCookies() - scraper2 := twitterscraper.New() - scraper2.SetCookies(cookies) - if !scraper2.IsLoggedIn() { - t.Error("Expected restored IsLoggedIn() = true") - } if err := scraper.Logout(); err != nil { t.Errorf("Logout() error = %v", err) } @@ -50,7 +100,41 @@ func TestAuth(t *testing.T) { } } +func TestLoginToken(t *testing.T) { + if skipAuthTest || authToken == "" || ct0 == "" { + t.Skip("Skipping test due to environment variable") + } + + scraper := newTestScraper(true) + + scraper.SetAuthToken(authToken, ct0) + if !scraper.IsLoggedIn() { + t.Error("Expected IsLoggedIn() = true") + } +} + +func TestLoginCookie(t *testing.T) { + if skipAuthTest || cookies == "" { + t.Skip("Skipping test due to environment variable") + } + + scraper := newTestScraper(true) + + var c []*http.Cookie + + json.NewDecoder(strings.NewReader(cookies)).Decode(&c) + + scraper.SetCookies(c) + if !scraper.IsLoggedIn() { + t.Error("Expected IsLoggedIn() = true") + } +} + func TestLoginOpenAccount(t *testing.T) { + if os.Getenv("TEST_OPEN_ACCOUNT") == "" { + t.Skip("Skipping test due to environment variable") + } + scraper := twitterscraper.New() _, err := scraper.LoginOpenAccount() diff --git a/profile_test.go b/profile_test.go index 7c12e34..3031eac 100644 --- a/profile_test.go +++ b/profile_test.go @@ -2,6 +2,7 @@ package twitterscraper_test import ( "fmt" + "strings" "testing" "time" @@ -30,12 +31,7 @@ func TestGetProfile(t *testing.T) { Website: "https://nomadic.name", } - scraper := twitterscraper.New() - _, err := scraper.LoginOpenAccount() - if err != nil { - t.Fatalf("LoginOpenAccount() error = %v", err) - } - profile, err := scraper.GetProfile("nomadic_ua") + profile, err := testScraper.GetProfile("nomadic_ua") if err != nil { t.Error(err) } @@ -86,13 +82,8 @@ func TestGetProfilePrivate(t *testing.T) { Website: "", } - scraper := twitterscraper.New() - _, err := scraper.LoginOpenAccount() - if err != nil { - t.Fatalf("LoginOpenAccount() error = %v", err) - } // some random private profile (found via google) - profile, err := scraper.GetProfile("tomdumont") + profile, err := testScraper.GetProfile("tomdumont") if err != nil { t.Error(err) } @@ -121,17 +112,12 @@ func TestGetProfilePrivate(t *testing.T) { } func TestGetProfileErrorSuspended(t *testing.T) { - scraper := twitterscraper.New() - _, err := scraper.LoginOpenAccount() - if err != nil { - t.Fatalf("LoginOpenAccount() error = %v", err) - } - _, err = scraper.GetProfile("123") + _, err := testScraper.GetProfile("123") if err == nil { t.Error("Expected Error, got success") } else { - if err.Error() != "Authorization: User has been suspended. (63)" { - t.Errorf("Expected error 'Authorization: User has been suspended. (63)', got '%s'", err) + if !strings.Contains(err.Error(), "Missing LdapGroup(visibility-custom-suspension)") { + t.Error("Expected error to contain 'Missing LdapGroup(visibility-custom-suspension)', got", err) } } } @@ -139,12 +125,7 @@ func TestGetProfileErrorSuspended(t *testing.T) { func TestGetProfileErrorNotFound(t *testing.T) { neUser := "sample3123131" expectedError := fmt.Sprintf("User '%s' not found", neUser) - scraper := twitterscraper.New() - _, err := scraper.LoginOpenAccount() - if err != nil { - t.Fatalf("LoginOpenAccount() error = %v", err) - } - _, err = scraper.GetProfile(neUser) + _, err := testScraper.GetProfile(neUser) if err == nil { t.Error("Expected Error, got success") } else { @@ -155,12 +136,7 @@ func TestGetProfileErrorNotFound(t *testing.T) { } func TestGetUserIDByScreenName(t *testing.T) { - scraper := twitterscraper.New() - _, err := scraper.LoginOpenAccount() - if err != nil { - t.Fatalf("LoginOpenAccount() error = %v", err) - } - userID, err := scraper.GetUserIDByScreenName("Twitter") + userID, err := testScraper.GetUserIDByScreenName("Twitter") if err != nil { t.Errorf("getUserByScreenName() error = %v", err) } diff --git a/tweets_test.go b/tweets_test.go index 1d73064..1642357 100644 --- a/tweets_test.go +++ b/tweets_test.go @@ -3,7 +3,6 @@ package twitterscraper_test import ( "context" "testing" - "time" "github.com/google/go-cmp/cmp" "github.com/google/go-cmp/cmp/cmpopts" @@ -14,18 +13,18 @@ var cmpOptions = cmp.Options{ cmpopts.IgnoreFields(twitterscraper.Tweet{}, "Likes"), cmpopts.IgnoreFields(twitterscraper.Tweet{}, "Replies"), cmpopts.IgnoreFields(twitterscraper.Tweet{}, "Retweets"), + cmpopts.IgnoreFields(twitterscraper.Tweet{}, "Views"), + + cmpopts.IgnoreFields(twitterscraper.Tweet{}, "IsSelfThread"), + cmpopts.IgnoreFields(twitterscraper.Tweet{}, "Thread"), + cmpopts.IgnoreFields(twitterscraper.Tweet{}, "TimeParsed"), } func TestGetTweets(t *testing.T) { count := 0 - maxTweetsNbr := 300 + maxTweetsNbr := 100 dupcheck := make(map[string]bool) - scraper := twitterscraper.New() - _, err := scraper.LoginOpenAccount() - if err != nil { - t.Fatalf("LoginOpenAccount() error = %v", err) - } - for tweet := range scraper.GetTweets(context.Background(), "Twitter", maxTweetsNbr) { + for tweet := range testScraper.GetTweets(context.Background(), "x", maxTweetsNbr) { if tweet.Error != nil { t.Error(tweet.Error) } else { @@ -76,6 +75,7 @@ func TestGetTweets(t *testing.T) { } func assertGetTweet(t *testing.T, expectedTweet *twitterscraper.Tweet) { + // to get tweet as struct fmt.Printf("%#v", actualTweet) actualTweet, err := testScraper.GetTweet(expectedTweet.ID) if err != nil { t.Error(err) @@ -86,42 +86,48 @@ func assertGetTweet(t *testing.T, expectedTweet *twitterscraper.Tweet) { func TestGetTweetWithVideo(t *testing.T) { expectedTweet := twitterscraper.Tweet{ - ConversationID: "1328684389388185600", - HTML: "That thing you didn’t Tweet but wanted to but didn’t but got so close but then were like nah.

We have a place for that now—Fleets!

Rolling out to everyone starting today.
", - ID: "1328684389388185600", - Name: "Twitter", - PermanentURL: "https://twitter.com/Twitter/status/1328684389388185600", + ConversationID: "1697304622749086011", + HTML: "on iOS & Android, you can now swipe to reply when you slide into their DMs
", + ID: "1697304622749086011", + Name: "X", + PermanentURL: "https://twitter.com/X/status/1697304622749086011", Photos: nil, - Text: "That thing you didn’t Tweet but wanted to but didn’t but got so close but then were like nah. \n\nWe have a place for that now—Fleets! \n\nRolling out to everyone starting today. https://t.co/auQAHXZMfH", - TimeParsed: time.Date(2020, 11, 17, 13, 0, 18, 0, time.FixedZone("UTC", 0)), - Timestamp: 1605618018, + Text: "on iOS & Android, you can now swipe to reply when you slide into their DMs https://t.co/evuWpMfBxQ", + Timestamp: 1693503931, UserID: "783214", - Username: "Twitter", - Videos: []twitterscraper.Video{{ - ID: "1328684333599756289", - Preview: "https://pbs.twimg.com/amplify_video_thumb/1328684333599756289/img/cP5KwbIXbGunNSBy.jpg", - URL: "https://video.twimg.com/amplify_video/1328684333599756289/vid/960x720/PcL8yv8KhgQ48Qpt.mp4?tag=13", - }}, + Username: "X", + Videos: []twitterscraper.Video{ + { + ID: "1697304568550330368", + Preview: "https://pbs.twimg.com/amplify_video_thumb/1697304568550330368/img/BUlESpef6FmWV_j2.jpg", + URL: "https://video.twimg.com/amplify_video/1697304568550330368/vid/720x720/KyQlZA9zaf0kqY9Z.mp4?tag=14", + }, + }, } assertGetTweet(t, &expectedTweet) } func TestGetTweetWithMultiplePhotos(t *testing.T) { expectedTweet := twitterscraper.Tweet{ - ConversationID: "1390026628957417473", - HTML: `no bird too tall, no crop too short

introducing bigger and better images on iOS and Android, now available to everyone

`, - ID: "1390026628957417473", - Name: "Twitter", - PermanentURL: "https://twitter.com/Twitter/status/1390026628957417473", + ConversationID: "1577677328968204291", + HTML: "More ways to discover videos on Twitter are here!

Now on iOS, videos on your timeline will open in our full screen immersive video player, where you can swipe up to keep discovering more content.

", + ID: "1577677328968204291", + Name: "Support", + PermanentURL: "https://twitter.com/Support/status/1577677328968204291", Photos: []twitterscraper.Photo{ - {ID: "1390026620472332292", URL: "https://pbs.twimg.com/media/E0pd2L2XEAQ_gnn.jpg"}, - {ID: "1390026626214371334", URL: "https://pbs.twimg.com/media/E0pd2hPXoAY9-TZ.jpg"}, + { + ID: "1577677319816286209", + URL: "https://pbs.twimg.com/media/FeUJKdnXEAEFe2j.jpg", + }, + { + ID: "1577677324421632000", + URL: "https://pbs.twimg.com/media/FeUJKuxXEAAa6t7.jpg", + }, }, - Text: "no bird too tall, no crop too short\n\nintroducing bigger and better images on iOS and Android, now available to everyone https://t.co/2buHfhfRAx", - TimeParsed: time.Date(2021, 5, 5, 19, 32, 28, 0, time.FixedZone("UTC", 0)), - Timestamp: 1620243148, - UserID: "783214", - Username: "Twitter", + Text: "More ways to discover videos on Twitter are here!\n\nNow on iOS, videos on your timeline will open in our full screen immersive video player, where you can swipe up to keep discovering more content. https://t.co/XI2vM8DKXA", + Timestamp: 1664982561, + UserID: "17874544", + Username: "Support", } assertGetTweet(t, &expectedTweet) } @@ -131,24 +137,22 @@ func TestGetTweetWithGIF(t *testing.T) { t.Skip("Skipping test due to environment variable") } expectedTweet := twitterscraper.Tweet{ - ConversationID: "1288540609310056450", + ConversationID: "1517535384833605632", GIFs: []twitterscraper.GIF{ { - ID: "1288540582768517123", - Preview: "https://pbs.twimg.com/tweet_video_thumb/EeHQ1UKXoAMVxWB.jpg", - URL: "https://video.twimg.com/tweet_video/EeHQ1UKXoAMVxWB.mp4", + ID: "1517535349890813952", + Preview: "https://pbs.twimg.com/tweet_video_thumb/FQ9eXEhXEAA-haj.jpg", + URL: "https://video.twimg.com/tweet_video/FQ9eXEhXEAA-haj.mp4", }, }, - Hashtags: []string{"CountdownToMars"}, - HTML: `Like for liftoff! #CountdownToMars
`, - ID: "1288540609310056450", - Name: "Twitter", - PermanentURL: "https://twitter.com/Twitter/status/1288540609310056450", - Text: "Like for liftoff! #CountdownToMars https://t.co/yLe331pHfY", - TimeParsed: time.Date(2020, 7, 29, 18, 23, 15, 0, time.FixedZone("UTC", 0)), - Timestamp: 1596046995, - UserID: "783214", - Username: "Twitter", + HTML: "Video captions or no captions, it’s now easier to choose for some of you on iOS, and soon on Android.

On videos that have captions available, we’re testing the option to turn captions off/on with a new “CC” button.
", + ID: "1517535384833605632", + Name: "Support", + PermanentURL: "https://twitter.com/Support/status/1517535384833605632", + Text: "Video captions or no captions, it’s now easier to choose for some of you on iOS, and soon on Android.\n\nOn videos that have captions available, we’re testing the option to turn captions off/on with a new “CC” button. https://t.co/Q2Q2Wmr78U", + Timestamp: 1650643604, + UserID: "17874544", + Username: "Support", } assertGetTweet(t, &expectedTweet) } @@ -158,24 +162,23 @@ func TestGetTweetWithPhotoAndGIF(t *testing.T) { t.Skip("Skipping test due to environment variable") } expectedTweet := twitterscraper.Tweet{ - ConversationID: "1580661436132757506", + ConversationID: "1583186305722507265", GIFs: []twitterscraper.GIF{ { - ID: "1580661428335382531", - Preview: "https://pbs.twimg.com/tweet_video_thumb/Fe-jMcIXkAMXK_W.jpg", - URL: "https://video.twimg.com/tweet_video/Fe-jMcIXkAMXK_W.mp4", + ID: "1583186295588790290", + Preview: "https://pbs.twimg.com/tweet_video_thumb/FfibjDnWIBIt5fn.jpg", + URL: "https://video.twimg.com/tweet_video/FfibjDnWIBIt5fn.mp4", }, }, - HTML: `a hit Tweet

`, - ID: "1580661436132757506", - Name: "Twitter", - PermanentURL: "https://twitter.com/Twitter/status/1580661436132757506", - Photos: []twitterscraper.Photo{{ID: "1580661428326907904", URL: "https://pbs.twimg.com/media/Fe-jMcGWQAAFWoG.jpg"}}, - Text: "a hit Tweet https://t.co/2C7cah4KzW", - TimeParsed: time.Date(2022, 10, 13, 20, 47, 8, 0, time.FixedZone("UTC", 0)), - Timestamp: 1665694028, - UserID: "783214", - Username: "Twitter", + HTML: "“we need to talk”

irl vs on Spaces

", + ID: "1583186305722507265", + Name: "Spaces", + PermanentURL: "https://twitter.com/XSpaces/status/1583186305722507265", + Photos: []twitterscraper.Photo{{ID: "1583186295626539020", URL: "https://pbs.twimg.com/media/FfibjDwWIAwvbtJ.jpg"}}, + Text: "“we need to talk” \n\nirl vs on Spaces https://t.co/hrflPpbpif", + Timestamp: 1666296004, + UserID: "1065249714214457345", + Username: "XSpaces", } assertGetTweet(t, &expectedTweet) } @@ -208,14 +211,13 @@ func TestQuotedAndReply(t *testing.T) { ID: "1237110473486729218", URL: "https://pbs.twimg.com/media/ESsZa9AXgAIAYnF.jpg", }}, - Replies: 12, - Retweets: 18, - Text: "The Easiest Problem Everyone Gets Wrong \n\n[new video] --> https://t.co/YdaeDYmPAU https://t.co/iKu4Xs6o2V", - TimeParsed: time.Date(2020, 0o3, 9, 20, 18, 33, 0, time.FixedZone("UTC", 0)), - Timestamp: 1583785113, - URLs: []string{"https://youtu.be/ytfCdqWhmdg"}, - UserID: "978944851", - Username: "VsauceTwo", + Replies: 12, + Retweets: 18, + Text: "The Easiest Problem Everyone Gets Wrong \n\n[new video] --> https://t.co/YdaeDYmPAU https://t.co/iKu4Xs6o2V", + Timestamp: 1583785113, + URLs: []string{"https://youtu.be/ytfCdqWhmdg"}, + UserID: "978944851", + Username: "VsauceTwo", } tweet, err := testScraper.GetTweet("1237110897597976576") if err != nil { @@ -243,22 +245,19 @@ func TestQuotedAndReply(t *testing.T) { } func TestRetweet(t *testing.T) { sample := &twitterscraper.Tweet{ - ConversationID: "1359151057872580612", - HTML: "We’ve seen an increase in attacks against Asian communities and individuals around the world. It’s important to know that this isn’t new; throughout history, Asians have experienced violence and exclusion. However, their diverse lived experiences have largely been overlooked.", - ID: "1359151057872580612", + ConversationID: "1758837061786779942", + HTML: "no ads, just bangers

aka your For You feed with Premium+

subscribe here → https://t.co/APTO1t7kMk", + ID: "1758837061786779942", + URLs: []string{"https://x.com/i/premium_sign_up"}, IsSelfThread: false, - Likes: 6683, - Name: "Twitter Together", - PermanentURL: "https://twitter.com/TwitterTogether/status/1359151057872580612", - Replies: 456, - Retweets: 1495, - Text: "We’ve seen an increase in attacks against Asian communities and individuals around the world. It’s important to know that this isn’t new; throughout history, Asians have experienced violence and exclusion. However, their diverse lived experiences have largely been overlooked.", - TimeParsed: time.Date(2021, 02, 9, 14, 43, 58, 0, time.FixedZone("UTC", 0)), - Timestamp: 1612881838, - UserID: "773578328498372608", - Username: "TwitterTogether", + Name: "Premium", + PermanentURL: "https://twitter.com/premium/status/1758837061786779942", + Text: "no ads, just bangers\n\naka your For You feed with Premium+\n\nsubscribe here → https://t.co/APTO1t7kMk", + Timestamp: 1708174407, + UserID: "1399766153053061121", + Username: "premium", } - tweet, err := testScraper.GetTweet("1362849141248974853") + tweet, err := testScraper.GetTweet("1758837226379596068") if err != nil { t.Error(err) } else { @@ -276,15 +275,14 @@ func TestTweetViews(t *testing.T) { HTML: "Replies and likes don’t tell the whole story. We’re making it easier to tell *just* how many people have seen your Tweets with the addition of view counts, shown right next to likes. Now on iOS and Android, web coming soon.", ID: "1606055187348688896", Likes: 2839, - Name: "Twitter Support", - PermanentURL: "https://twitter.com/TwitterSupport/status/1606055187348688896", + Name: "Support", + PermanentURL: "https://twitter.com/Support/status/1606055187348688896", Replies: 3427, Retweets: 783, Text: "Replies and likes don’t tell the whole story. We’re making it easier to tell *just* how many people have seen your Tweets with the addition of view counts, shown right next to likes. Now on iOS and Android, web coming soon.", - TimeParsed: time.Date(2022, 12, 22, 22, 32, 50, 0, time.FixedZone("UTC", 0)), Timestamp: 1612881838, UserID: "17874544", - Username: "TwitterSupport", + Username: "Support", Views: 3189278, } tweet, err := testScraper.GetTweet("1606055187348688896")