diff --git a/api.go b/api.go index 2f4d657..bfee5f2 100644 --- a/api.go +++ b/api.go @@ -23,7 +23,7 @@ func (s *Scraper) RequestAPI(req *http.Request, target interface{}) error { }() } - if s.guestToken == "" || s.guestCreatedAt.Before(time.Now().Add(-time.Hour*3)) { + if !s.IsGuestToken() || s.guestCreatedAt.Before(time.Now().Add(-time.Hour*3)) { err := s.GetGuestToken() if err != nil { return err diff --git a/api_test.go b/api_test.go index 2267a4c..2168de9 100644 --- a/api_test.go +++ b/api_test.go @@ -1,15 +1,17 @@ -package twitterscraper +package twitterscraper_test import ( "testing" + + twitterscraper "github.com/n0madic/twitter-scraper" ) func TestGetGuestToken(t *testing.T) { - scraper := New() + scraper := twitterscraper.New() if err := scraper.GetGuestToken(); err != nil { t.Errorf("getGuestToken() error = %v", err) } - if scraper.guestToken == "" { + if !scraper.IsGuestToken() { t.Error("Expected non-empty guestToken") } } diff --git a/profile_test.go b/profile_test.go index 767a62a..9967173 100644 --- a/profile_test.go +++ b/profile_test.go @@ -1,4 +1,4 @@ -package twitterscraper +package twitterscraper_test import ( "fmt" @@ -7,12 +7,13 @@ import ( "github.com/google/go-cmp/cmp" "github.com/google/go-cmp/cmp/cmpopts" + twitterscraper "github.com/n0madic/twitter-scraper" ) func TestGetProfile(t *testing.T) { loc := time.FixedZone("UTC", 0) joined := time.Date(2010, 01, 18, 8, 49, 30, 0, loc) - sample := Profile{ + sample := twitterscraper.Profile{ Avatar: "https://pbs.twimg.com/profile_images/436075027193004032/XlDa2oaz_normal.jpeg", Banner: "https://pbs.twimg.com/profile_banners/106037940/1541084318", Biography: "nothing", @@ -29,18 +30,18 @@ func TestGetProfile(t *testing.T) { Website: "https://nomadic.name", } - profile, err := GetProfile("nomadic_ua") + profile, err := twitterscraper.GetProfile("nomadic_ua") if err != nil { t.Error(err) } cmpOptions := cmp.Options{ - cmpopts.IgnoreFields(Profile{}, "FollowersCount"), - cmpopts.IgnoreFields(Profile{}, "FollowingCount"), - cmpopts.IgnoreFields(Profile{}, "FriendsCount"), - cmpopts.IgnoreFields(Profile{}, "LikesCount"), - cmpopts.IgnoreFields(Profile{}, "ListedCount"), - cmpopts.IgnoreFields(Profile{}, "TweetsCount"), + cmpopts.IgnoreFields(twitterscraper.Profile{}, "FollowersCount"), + cmpopts.IgnoreFields(twitterscraper.Profile{}, "FollowingCount"), + cmpopts.IgnoreFields(twitterscraper.Profile{}, "FriendsCount"), + cmpopts.IgnoreFields(twitterscraper.Profile{}, "LikesCount"), + cmpopts.IgnoreFields(twitterscraper.Profile{}, "ListedCount"), + cmpopts.IgnoreFields(twitterscraper.Profile{}, "TweetsCount"), } if diff := cmp.Diff(sample, profile, cmpOptions...); diff != "" { t.Error("Resulting profile does not match the sample", diff) @@ -63,7 +64,7 @@ func TestGetProfile(t *testing.T) { func TestGetProfilePrivate(t *testing.T) { loc := time.FixedZone("UTC", 0) joined := time.Date(2020, 1, 26, 0, 3, 5, 0, loc) - sample := Profile{ + sample := twitterscraper.Profile{ Avatar: "https://pbs.twimg.com/profile_images/1222218816484020224/ik9P1QZt_normal.jpg", Banner: "", Biography: "private account", @@ -81,18 +82,18 @@ func TestGetProfilePrivate(t *testing.T) { } // some random private profile (found via google) - profile, err := GetProfile("tomdumont") + profile, err := twitterscraper.GetProfile("tomdumont") if err != nil { t.Error(err) } cmpOptions := cmp.Options{ - cmpopts.IgnoreFields(Profile{}, "FollowersCount"), - cmpopts.IgnoreFields(Profile{}, "FollowingCount"), - cmpopts.IgnoreFields(Profile{}, "FriendsCount"), - cmpopts.IgnoreFields(Profile{}, "LikesCount"), - cmpopts.IgnoreFields(Profile{}, "ListedCount"), - cmpopts.IgnoreFields(Profile{}, "TweetsCount"), + cmpopts.IgnoreFields(twitterscraper.Profile{}, "FollowersCount"), + cmpopts.IgnoreFields(twitterscraper.Profile{}, "FollowingCount"), + cmpopts.IgnoreFields(twitterscraper.Profile{}, "FriendsCount"), + cmpopts.IgnoreFields(twitterscraper.Profile{}, "LikesCount"), + cmpopts.IgnoreFields(twitterscraper.Profile{}, "ListedCount"), + cmpopts.IgnoreFields(twitterscraper.Profile{}, "TweetsCount"), } if diff := cmp.Diff(sample, profile, cmpOptions...); diff != "" { t.Error("Resulting profile does not match the sample", diff) @@ -110,7 +111,7 @@ func TestGetProfilePrivate(t *testing.T) { } func TestGetProfileErrorSuspended(t *testing.T) { - _, err := GetProfile("123") + _, err := twitterscraper.GetProfile("123") if err == nil { t.Error("Expected Error, got success") } else { @@ -123,7 +124,7 @@ func TestGetProfileErrorSuspended(t *testing.T) { func TestGetProfileErrorNotFound(t *testing.T) { neUser := "sample3123131" expectedError := fmt.Sprintf("User '%s' not found", neUser) - _, err := GetProfile(neUser) + _, err := twitterscraper.GetProfile(neUser) if err == nil { t.Error("Expected Error, got success") } else { @@ -134,7 +135,7 @@ func TestGetProfileErrorNotFound(t *testing.T) { } func TestGetUserIDByScreenName(t *testing.T) { - scraper := New() + scraper := twitterscraper.New() userID, err := scraper.GetUserIDByScreenName("Twitter") if err != nil { t.Errorf("getUserByScreenName() error = %v", err) diff --git a/scraper.go b/scraper.go index a5a5d4a..769e3b5 100644 --- a/scraper.go +++ b/scraper.go @@ -3,13 +3,14 @@ package twitterscraper import ( "crypto/tls" "errors" - "golang.org/x/net/proxy" "net" "net/http" "net/url" "strings" "sync" "time" + + "golang.org/x/net/proxy" ) // Scraper object @@ -56,6 +57,11 @@ func New() *Scraper { } } +// IsGuestToken check if guest token not empty +func (s *Scraper) IsGuestToken() bool { + return s.guestToken != "" +} + // SetSearchMode switcher func (s *Scraper) SetSearchMode(mode SearchMode) *Scraper { s.searchMode = mode diff --git a/search_test.go b/search_test.go index 957564e..ad09dad 100644 --- a/search_test.go +++ b/search_test.go @@ -1,12 +1,14 @@ -package twitterscraper +package twitterscraper_test import ( "context" "testing" + + twitterscraper "github.com/n0madic/twitter-scraper" ) func TestFetchSearchCursor(t *testing.T) { - scraper := New() + scraper := twitterscraper.New() maxTweetsNbr := 150 tweetsNbr := 0 nextCursor := "" @@ -27,7 +29,7 @@ func TestGetSearchProfiles(t *testing.T) { count := 0 maxProfilesNbr := 150 dupcheck := make(map[string]bool) - scraper := New().SetSearchMode(SearchUsers) + scraper := twitterscraper.New().SetSearchMode(twitterscraper.SearchUsers) for profile := range scraper.SearchProfiles(context.Background(), "Twitter", maxProfilesNbr) { if profile.Error != nil { t.Error(profile.Error) @@ -53,7 +55,7 @@ func TestGetSearchTweets(t *testing.T) { count := 0 maxTweetsNbr := 150 dupcheck := make(map[string]bool) - scraper := New().WithDelay(4) + scraper := twitterscraper.New().WithDelay(4) for tweet := range scraper.SearchTweets(context.Background(), "twitter -filter:retweets", maxTweetsNbr) { if tweet.Error != nil { t.Error(tweet.Error) diff --git a/trends_test.go b/trends_test.go index 1c3564b..71d3eeb 100644 --- a/trends_test.go +++ b/trends_test.go @@ -1,11 +1,13 @@ -package twitterscraper +package twitterscraper_test import ( "testing" + + twitterscraper "github.com/n0madic/twitter-scraper" ) func TestGetTrends(t *testing.T) { - trends, err := GetTrends() + trends, err := twitterscraper.GetTrends() if err != nil { t.Error(err) } diff --git a/tweets_test.go b/tweets_test.go index ac2c4b1..cd31496 100644 --- a/tweets_test.go +++ b/tweets_test.go @@ -1,4 +1,4 @@ -package twitterscraper +package twitterscraper_test import ( "context" @@ -7,19 +7,20 @@ import ( "github.com/google/go-cmp/cmp" "github.com/google/go-cmp/cmp/cmpopts" + twitterscraper "github.com/n0madic/twitter-scraper" ) var cmpOptions = cmp.Options{ - cmpopts.IgnoreFields(Tweet{}, "Likes"), - cmpopts.IgnoreFields(Tweet{}, "Replies"), - cmpopts.IgnoreFields(Tweet{}, "Retweets"), + cmpopts.IgnoreFields(twitterscraper.Tweet{}, "Likes"), + cmpopts.IgnoreFields(twitterscraper.Tweet{}, "Replies"), + cmpopts.IgnoreFields(twitterscraper.Tweet{}, "Retweets"), } func TestGetTweets(t *testing.T) { count := 0 maxTweetsNbr := 300 dupcheck := make(map[string]bool) - for tweet := range GetTweets(context.Background(), "Twitter", maxTweetsNbr) { + for tweet := range twitterscraper.GetTweets(context.Background(), "Twitter", maxTweetsNbr) { if tweet.Error != nil { t.Error(tweet.Error) } else { @@ -70,7 +71,7 @@ func TestGetTweets(t *testing.T) { } func TestGetTweet(t *testing.T) { - sample := Tweet{ + sample := twitterscraper.Tweet{ HTML: "That thing you didn’t Tweet but wanted to but didn’t but got so close but then were like nah.

We have a place for that now—Fleets!

Rolling out to everyone starting today.
", ID: "1328684389388185600", PermanentURL: "https://twitter.com/Twitter/status/1328684389388185600", @@ -80,13 +81,13 @@ func TestGetTweet(t *testing.T) { Timestamp: 1605618018, UserID: "783214", Username: "Twitter", - Videos: []Video{{ + Videos: []twitterscraper.Video{{ ID: "1328684333599756289", Preview: "https://pbs.twimg.com/amplify_video_thumb/1328684333599756289/img/cP5KwbIXbGunNSBy.jpg", URL: "https://video.twimg.com/amplify_video/1328684333599756289/vid/960x720/PcL8yv8KhgQ48Qpt.mp4?tag=13", }}, } - tweet, err := defaultScraper.GetTweet("1328684389388185600") + tweet, err := twitterscraper.GetTweet("1328684389388185600") if err != nil { t.Error(err) } else { @@ -97,7 +98,7 @@ func TestGetTweet(t *testing.T) { } func TestQuotedAndReply(t *testing.T) { - sample := &Tweet{ + sample := &twitterscraper.Tweet{ HTML: "The Easiest Problem Everyone Gets Wrong

[new video] --> https://t.co/YdaeDYmPAU
", ID: "1237110546383724547", Likes: 485, @@ -112,7 +113,7 @@ func TestQuotedAndReply(t *testing.T) { UserID: "978944851", Username: "VsauceTwo", } - tweet, err := defaultScraper.GetTweet("1237110897597976576") + tweet, err := twitterscraper.GetTweet("1237110897597976576") if err != nil { t.Error(err) } else { @@ -123,7 +124,7 @@ func TestQuotedAndReply(t *testing.T) { t.Error("Resulting quote does not match the sample", diff) } } - tweet, err = defaultScraper.GetTweet("1237111868445134850") + tweet, err = twitterscraper.GetTweet("1237111868445134850") if err != nil { t.Error(err) } else { @@ -137,7 +138,7 @@ func TestQuotedAndReply(t *testing.T) { } func TestRetweet(t *testing.T) { - sample := &Tweet{ + sample := &twitterscraper.Tweet{ HTML: "We’ve seen an increase in attacks against Asian communities and individuals around the world. It’s important to know that this isn’t new; throughout history, Asians have experienced violence and exclusion. However, their diverse lived experiences have largely been overlooked.", ID: "1359151057872580612", Likes: 6683, @@ -150,7 +151,7 @@ func TestRetweet(t *testing.T) { UserID: "773578328498372608", Username: "TwitterTogether", } - tweet, err := defaultScraper.GetTweet("1362849141248974853") + tweet, err := twitterscraper.GetTweet("1362849141248974853") if err != nil { t.Error(err) } else {