From bfb40205971f09a3effbadf75fb6d21847f0af1d Mon Sep 17 00:00:00 2001 From: Alexander Sheiko Date: Wed, 4 May 2022 11:55:12 +0300 Subject: [PATCH] Deprecate default scraper --- README.md | 22 ---------------------- profile.go | 2 +- profile_test.go | 12 ++++++++---- scraper.go | 8 ++++---- search.go | 4 ++-- trends.go | 2 +- trends_test.go | 3 ++- tweets.go | 4 ++-- tweets_test.go | 14 +++++++++----- 9 files changed, 29 insertions(+), 42 deletions(-) diff --git a/README.md b/README.md index 14375d3..c2cfd0d 100644 --- a/README.md +++ b/README.md @@ -213,25 +213,3 @@ scraper.WithDelay(5) ```golang scraper.WithReplies(true) ``` - -### Default Scraper (Ad hoc) - -In simple cases, you can use the default scraper without creating an object instance - -```golang -import twitterscraper "github.com/n0madic/twitter-scraper" - -// for tweets -twitterscraper.GetTweets(context.Background(), "Twitter", 50) -// for tweets with replies -twitterscraper.WithReplies(true).GetTweets(context.Background(), "Twitter", 50) - -// for search -twitterscraper.SearchTweets(context.Background(), "twitter", 50) - -// for profile -twitterscraper.GetProfile("Twitter") - -// for trends -twitterscraper.GetTrends() -``` diff --git a/profile.go b/profile.go index 1899bd7..f295910 100644 --- a/profile.go +++ b/profile.go @@ -75,7 +75,7 @@ func (s *Scraper) GetProfile(username string) (Profile, error) { return parseProfile(jsn.Data.User.Legacy), nil } -// GetProfile wrapper for default scraper +// Deprecated: GetProfile wrapper for default scraper func GetProfile(username string) (Profile, error) { return defaultScraper.GetProfile(username) } diff --git a/profile_test.go b/profile_test.go index f940b61..1ea6180 100644 --- a/profile_test.go +++ b/profile_test.go @@ -30,7 +30,8 @@ func TestGetProfile(t *testing.T) { Website: "https://nomadic.name", } - profile, err := twitterscraper.GetProfile("nomadic_ua") + scraper := twitterscraper.New() + profile, err := scraper.GetProfile("nomadic_ua") if err != nil { t.Error(err) } @@ -81,8 +82,9 @@ func TestGetProfilePrivate(t *testing.T) { Website: "", } + scraper := twitterscraper.New() // some random private profile (found via google) - profile, err := twitterscraper.GetProfile("tomdumont") + profile, err := scraper.GetProfile("tomdumont") if err != nil { t.Error(err) } @@ -111,7 +113,8 @@ func TestGetProfilePrivate(t *testing.T) { } func TestGetProfileErrorSuspended(t *testing.T) { - _, err := twitterscraper.GetProfile("123") + scraper := twitterscraper.New() + _, err := scraper.GetProfile("123") if err == nil { t.Error("Expected Error, got success") } else { @@ -124,7 +127,8 @@ func TestGetProfileErrorSuspended(t *testing.T) { func TestGetProfileErrorNotFound(t *testing.T) { neUser := "sample3123131" expectedError := fmt.Sprintf("User '%s' not found", neUser) - _, err := twitterscraper.GetProfile(neUser) + scraper := twitterscraper.New() + _, err := scraper.GetProfile(neUser) if err == nil { t.Error("Expected Error, got success") } else { diff --git a/scraper.go b/scraper.go index 3a216dc..16b09c0 100644 --- a/scraper.go +++ b/scraper.go @@ -73,7 +73,7 @@ func (s *Scraper) SetSearchMode(mode SearchMode) *Scraper { return s } -// SetSearchMode wrapper for default Scraper +// Deprecated: SetSearchMode wrapper for default Scraper func SetSearchMode(mode SearchMode) *Scraper { return defaultScraper.SetSearchMode(mode) } @@ -84,7 +84,7 @@ func (s *Scraper) WithDelay(seconds int64) *Scraper { return s } -// WithDelay wrapper for default Scraper +// Deprecated: WithDelay wrapper for default Scraper func WithDelay(seconds int64) *Scraper { return defaultScraper.WithDelay(seconds) } @@ -95,7 +95,7 @@ func (s *Scraper) WithReplies(b bool) *Scraper { return s } -// WithReplies wrapper for default Scraper +// Deprecated: WithReplies wrapper for default Scraper func WithReplies(b bool) *Scraper { return defaultScraper.WithReplies(b) } @@ -163,7 +163,7 @@ func (s *Scraper) SetProxy(proxyAddr string) error { return errors.New("only support http(s) or socks5 protocol") } -// SetProxy wrapper for default Scraper +// Deprecated: SetProxy wrapper for default Scraper func SetProxy(proxy string) error { return defaultScraper.SetProxy(proxy) } diff --git a/search.go b/search.go index ec74942..0eeb828 100644 --- a/search.go +++ b/search.go @@ -10,7 +10,7 @@ func (s *Scraper) SearchTweets(ctx context.Context, query string, maxTweetsNbr i return getTweetTimeline(ctx, query, maxTweetsNbr, s.FetchSearchTweets) } -// SearchTweets wrapper for default Scraper +// Deprecated: SearchTweets wrapper for default Scraper func SearchTweets(ctx context.Context, query string, maxTweetsNbr int) <-chan *TweetResult { return defaultScraper.SearchTweets(ctx, query, maxTweetsNbr) } @@ -20,7 +20,7 @@ func (s *Scraper) SearchProfiles(ctx context.Context, query string, maxProfilesN return getUserTimeline(ctx, query, maxProfilesNbr, s.FetchSearchProfiles) } -// SearchProfiles wrapper for default Scraper +// Deprecated: SearchProfiles wrapper for default Scraper func SearchProfiles(ctx context.Context, query string, maxProfilesNbr int) <-chan *ProfileResult { return defaultScraper.SearchProfiles(ctx, query, maxProfilesNbr) } diff --git a/trends.go b/trends.go index a103195..cdcc3bd 100644 --- a/trends.go +++ b/trends.go @@ -38,7 +38,7 @@ func (s *Scraper) GetTrends() ([]string, error) { return trends, nil } -// GetTrends wrapper for default Scraper +// Deprecated: GetTrends wrapper for default Scraper func GetTrends() ([]string, error) { return defaultScraper.GetTrends() } diff --git a/trends_test.go b/trends_test.go index 71d3eeb..98ef5a0 100644 --- a/trends_test.go +++ b/trends_test.go @@ -7,7 +7,8 @@ import ( ) func TestGetTrends(t *testing.T) { - trends, err := twitterscraper.GetTrends() + scraper := twitterscraper.New() + trends, err := scraper.GetTrends() if err != nil { t.Error(err) } diff --git a/tweets.go b/tweets.go index 4bf6d93..4e6ccfb 100644 --- a/tweets.go +++ b/tweets.go @@ -11,7 +11,7 @@ func (s *Scraper) GetTweets(ctx context.Context, user string, maxTweetsNbr int) return getTweetTimeline(ctx, user, maxTweetsNbr, s.FetchTweets) } -// GetTweets wrapper for default Scraper +// Deprecated: GetTweets wrapper for default Scraper func GetTweets(ctx context.Context, user string, maxTweetsNbr int) <-chan *TweetResult { return defaultScraper.GetTweets(ctx, user, maxTweetsNbr) } @@ -72,7 +72,7 @@ func (s *Scraper) GetTweet(id string) (*Tweet, error) { return nil, fmt.Errorf("tweet with ID %s not found", id) } -// GetTweet wrapper for default Scraper +// Deprecated: GetTweet wrapper for default Scraper func GetTweet(id string) (*Tweet, error) { return defaultScraper.GetTweet(id) } diff --git a/tweets_test.go b/tweets_test.go index cd31496..7856ade 100644 --- a/tweets_test.go +++ b/tweets_test.go @@ -20,7 +20,8 @@ func TestGetTweets(t *testing.T) { count := 0 maxTweetsNbr := 300 dupcheck := make(map[string]bool) - for tweet := range twitterscraper.GetTweets(context.Background(), "Twitter", maxTweetsNbr) { + scraper := twitterscraper.New() + for tweet := range scraper.GetTweets(context.Background(), "Twitter", maxTweetsNbr) { if tweet.Error != nil { t.Error(tweet.Error) } else { @@ -87,7 +88,8 @@ func TestGetTweet(t *testing.T) { URL: "https://video.twimg.com/amplify_video/1328684333599756289/vid/960x720/PcL8yv8KhgQ48Qpt.mp4?tag=13", }}, } - tweet, err := twitterscraper.GetTweet("1328684389388185600") + scraper := twitterscraper.New() + tweet, err := scraper.GetTweet("1328684389388185600") if err != nil { t.Error(err) } else { @@ -113,7 +115,8 @@ func TestQuotedAndReply(t *testing.T) { UserID: "978944851", Username: "VsauceTwo", } - tweet, err := twitterscraper.GetTweet("1237110897597976576") + scraper := twitterscraper.New() + tweet, err := scraper.GetTweet("1237110897597976576") if err != nil { t.Error(err) } else { @@ -124,7 +127,7 @@ func TestQuotedAndReply(t *testing.T) { t.Error("Resulting quote does not match the sample", diff) } } - tweet, err = twitterscraper.GetTweet("1237111868445134850") + tweet, err = scraper.GetTweet("1237111868445134850") if err != nil { t.Error(err) } else { @@ -151,7 +154,8 @@ func TestRetweet(t *testing.T) { UserID: "773578328498372608", Username: "TwitterTogether", } - tweet, err := twitterscraper.GetTweet("1362849141248974853") + scraper := twitterscraper.New() + tweet, err := scraper.GetTweet("1362849141248974853") if err != nil { t.Error(err) } else {