Deprecate default scraper
This commit is contained in:
Alexander Sheiko
parent
6bf6dc4b69
commit
bfb4020597
9 changed files with 29 additions and 42 deletions
22
README.md
22
README.md
|
|
@ -213,25 +213,3 @@ scraper.WithDelay(5)
|
|||
```golang
|
||||
scraper.WithReplies(true)
|
||||
```
|
||||
|
||||
### Default Scraper (Ad hoc)
|
||||
|
||||
In simple cases, you can use the default scraper without creating an object instance
|
||||
|
||||
```golang
|
||||
import twitterscraper "github.com/n0madic/twitter-scraper"
|
||||
|
||||
// for tweets
|
||||
twitterscraper.GetTweets(context.Background(), "Twitter", 50)
|
||||
// for tweets with replies
|
||||
twitterscraper.WithReplies(true).GetTweets(context.Background(), "Twitter", 50)
|
||||
|
||||
// for search
|
||||
twitterscraper.SearchTweets(context.Background(), "twitter", 50)
|
||||
|
||||
// for profile
|
||||
twitterscraper.GetProfile("Twitter")
|
||||
|
||||
// for trends
|
||||
twitterscraper.GetTrends()
|
||||
```
|
||||
|
|
|
|||
|
|
@ -75,7 +75,7 @@ func (s *Scraper) GetProfile(username string) (Profile, error) {
|
|||
return parseProfile(jsn.Data.User.Legacy), nil
|
||||
}
|
||||
|
||||
// GetProfile wrapper for default scraper
|
||||
// Deprecated: GetProfile wrapper for default scraper
|
||||
func GetProfile(username string) (Profile, error) {
|
||||
return defaultScraper.GetProfile(username)
|
||||
}
|
||||
|
|
|
|||
|
|
@ -30,7 +30,8 @@ func TestGetProfile(t *testing.T) {
|
|||
Website: "https://nomadic.name",
|
||||
}
|
||||
|
||||
profile, err := twitterscraper.GetProfile("nomadic_ua")
|
||||
scraper := twitterscraper.New()
|
||||
profile, err := scraper.GetProfile("nomadic_ua")
|
||||
if err != nil {
|
||||
t.Error(err)
|
||||
}
|
||||
|
|
@ -81,8 +82,9 @@ func TestGetProfilePrivate(t *testing.T) {
|
|||
Website: "",
|
||||
}
|
||||
|
||||
scraper := twitterscraper.New()
|
||||
// some random private profile (found via google)
|
||||
profile, err := twitterscraper.GetProfile("tomdumont")
|
||||
profile, err := scraper.GetProfile("tomdumont")
|
||||
if err != nil {
|
||||
t.Error(err)
|
||||
}
|
||||
|
|
@ -111,7 +113,8 @@ func TestGetProfilePrivate(t *testing.T) {
|
|||
}
|
||||
|
||||
func TestGetProfileErrorSuspended(t *testing.T) {
|
||||
_, err := twitterscraper.GetProfile("123")
|
||||
scraper := twitterscraper.New()
|
||||
_, err := scraper.GetProfile("123")
|
||||
if err == nil {
|
||||
t.Error("Expected Error, got success")
|
||||
} else {
|
||||
|
|
@ -124,7 +127,8 @@ func TestGetProfileErrorSuspended(t *testing.T) {
|
|||
func TestGetProfileErrorNotFound(t *testing.T) {
|
||||
neUser := "sample3123131"
|
||||
expectedError := fmt.Sprintf("User '%s' not found", neUser)
|
||||
_, err := twitterscraper.GetProfile(neUser)
|
||||
scraper := twitterscraper.New()
|
||||
_, err := scraper.GetProfile(neUser)
|
||||
if err == nil {
|
||||
t.Error("Expected Error, got success")
|
||||
} else {
|
||||
|
|
|
|||
|
|
@ -73,7 +73,7 @@ func (s *Scraper) SetSearchMode(mode SearchMode) *Scraper {
|
|||
return s
|
||||
}
|
||||
|
||||
// SetSearchMode wrapper for default Scraper
|
||||
// Deprecated: SetSearchMode wrapper for default Scraper
|
||||
func SetSearchMode(mode SearchMode) *Scraper {
|
||||
return defaultScraper.SetSearchMode(mode)
|
||||
}
|
||||
|
|
@ -84,7 +84,7 @@ func (s *Scraper) WithDelay(seconds int64) *Scraper {
|
|||
return s
|
||||
}
|
||||
|
||||
// WithDelay wrapper for default Scraper
|
||||
// Deprecated: WithDelay wrapper for default Scraper
|
||||
func WithDelay(seconds int64) *Scraper {
|
||||
return defaultScraper.WithDelay(seconds)
|
||||
}
|
||||
|
|
@ -95,7 +95,7 @@ func (s *Scraper) WithReplies(b bool) *Scraper {
|
|||
return s
|
||||
}
|
||||
|
||||
// WithReplies wrapper for default Scraper
|
||||
// Deprecated: WithReplies wrapper for default Scraper
|
||||
func WithReplies(b bool) *Scraper {
|
||||
return defaultScraper.WithReplies(b)
|
||||
}
|
||||
|
|
@ -163,7 +163,7 @@ func (s *Scraper) SetProxy(proxyAddr string) error {
|
|||
return errors.New("only support http(s) or socks5 protocol")
|
||||
}
|
||||
|
||||
// SetProxy wrapper for default Scraper
|
||||
// Deprecated: SetProxy wrapper for default Scraper
|
||||
func SetProxy(proxy string) error {
|
||||
return defaultScraper.SetProxy(proxy)
|
||||
}
|
||||
|
|
|
|||
|
|
@ -10,7 +10,7 @@ func (s *Scraper) SearchTweets(ctx context.Context, query string, maxTweetsNbr i
|
|||
return getTweetTimeline(ctx, query, maxTweetsNbr, s.FetchSearchTweets)
|
||||
}
|
||||
|
||||
// SearchTweets wrapper for default Scraper
|
||||
// Deprecated: SearchTweets wrapper for default Scraper
|
||||
func SearchTweets(ctx context.Context, query string, maxTweetsNbr int) <-chan *TweetResult {
|
||||
return defaultScraper.SearchTweets(ctx, query, maxTweetsNbr)
|
||||
}
|
||||
|
|
@ -20,7 +20,7 @@ func (s *Scraper) SearchProfiles(ctx context.Context, query string, maxProfilesN
|
|||
return getUserTimeline(ctx, query, maxProfilesNbr, s.FetchSearchProfiles)
|
||||
}
|
||||
|
||||
// SearchProfiles wrapper for default Scraper
|
||||
// Deprecated: SearchProfiles wrapper for default Scraper
|
||||
func SearchProfiles(ctx context.Context, query string, maxProfilesNbr int) <-chan *ProfileResult {
|
||||
return defaultScraper.SearchProfiles(ctx, query, maxProfilesNbr)
|
||||
}
|
||||
|
|
|
|||
|
|
@ -38,7 +38,7 @@ func (s *Scraper) GetTrends() ([]string, error) {
|
|||
return trends, nil
|
||||
}
|
||||
|
||||
// GetTrends wrapper for default Scraper
|
||||
// Deprecated: GetTrends wrapper for default Scraper
|
||||
func GetTrends() ([]string, error) {
|
||||
return defaultScraper.GetTrends()
|
||||
}
|
||||
|
|
|
|||
|
|
@ -7,7 +7,8 @@ import (
|
|||
)
|
||||
|
||||
func TestGetTrends(t *testing.T) {
|
||||
trends, err := twitterscraper.GetTrends()
|
||||
scraper := twitterscraper.New()
|
||||
trends, err := scraper.GetTrends()
|
||||
if err != nil {
|
||||
t.Error(err)
|
||||
}
|
||||
|
|
|
|||
|
|
@ -11,7 +11,7 @@ func (s *Scraper) GetTweets(ctx context.Context, user string, maxTweetsNbr int)
|
|||
return getTweetTimeline(ctx, user, maxTweetsNbr, s.FetchTweets)
|
||||
}
|
||||
|
||||
// GetTweets wrapper for default Scraper
|
||||
// Deprecated: GetTweets wrapper for default Scraper
|
||||
func GetTweets(ctx context.Context, user string, maxTweetsNbr int) <-chan *TweetResult {
|
||||
return defaultScraper.GetTweets(ctx, user, maxTweetsNbr)
|
||||
}
|
||||
|
|
@ -72,7 +72,7 @@ func (s *Scraper) GetTweet(id string) (*Tweet, error) {
|
|||
return nil, fmt.Errorf("tweet with ID %s not found", id)
|
||||
}
|
||||
|
||||
// GetTweet wrapper for default Scraper
|
||||
// Deprecated: GetTweet wrapper for default Scraper
|
||||
func GetTweet(id string) (*Tweet, error) {
|
||||
return defaultScraper.GetTweet(id)
|
||||
}
|
||||
|
|
|
|||
|
|
@ -20,7 +20,8 @@ func TestGetTweets(t *testing.T) {
|
|||
count := 0
|
||||
maxTweetsNbr := 300
|
||||
dupcheck := make(map[string]bool)
|
||||
for tweet := range twitterscraper.GetTweets(context.Background(), "Twitter", maxTweetsNbr) {
|
||||
scraper := twitterscraper.New()
|
||||
for tweet := range scraper.GetTweets(context.Background(), "Twitter", maxTweetsNbr) {
|
||||
if tweet.Error != nil {
|
||||
t.Error(tweet.Error)
|
||||
} else {
|
||||
|
|
@ -87,7 +88,8 @@ func TestGetTweet(t *testing.T) {
|
|||
URL: "https://video.twimg.com/amplify_video/1328684333599756289/vid/960x720/PcL8yv8KhgQ48Qpt.mp4?tag=13",
|
||||
}},
|
||||
}
|
||||
tweet, err := twitterscraper.GetTweet("1328684389388185600")
|
||||
scraper := twitterscraper.New()
|
||||
tweet, err := scraper.GetTweet("1328684389388185600")
|
||||
if err != nil {
|
||||
t.Error(err)
|
||||
} else {
|
||||
|
|
@ -113,7 +115,8 @@ func TestQuotedAndReply(t *testing.T) {
|
|||
UserID: "978944851",
|
||||
Username: "VsauceTwo",
|
||||
}
|
||||
tweet, err := twitterscraper.GetTweet("1237110897597976576")
|
||||
scraper := twitterscraper.New()
|
||||
tweet, err := scraper.GetTweet("1237110897597976576")
|
||||
if err != nil {
|
||||
t.Error(err)
|
||||
} else {
|
||||
|
|
@ -124,7 +127,7 @@ func TestQuotedAndReply(t *testing.T) {
|
|||
t.Error("Resulting quote does not match the sample", diff)
|
||||
}
|
||||
}
|
||||
tweet, err = twitterscraper.GetTweet("1237111868445134850")
|
||||
tweet, err = scraper.GetTweet("1237111868445134850")
|
||||
if err != nil {
|
||||
t.Error(err)
|
||||
} else {
|
||||
|
|
@ -151,7 +154,8 @@ func TestRetweet(t *testing.T) {
|
|||
UserID: "773578328498372608",
|
||||
Username: "TwitterTogether",
|
||||
}
|
||||
tweet, err := twitterscraper.GetTweet("1362849141248974853")
|
||||
scraper := twitterscraper.New()
|
||||
tweet, err := scraper.GetTweet("1362849141248974853")
|
||||
if err != nil {
|
||||
t.Error(err)
|
||||
} else {
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue