lain/twitter-scrapper
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity

Getting tweets by the maximum number, not by pages as before

Browse source 
BREAKING CHANGE!
This commit is contained in:
Alexander Sheiko 2020-06-15 16:16:08 +03:00
parent a5117bb5c4
commit 49baceb5b4
3 changed files with 19 additions and 10 deletions
Download patch file Download diff file Expand all files Collapse all files

4
README.md
View file

@ -28,7 +28,7 @@ import (
) )
func main() { func main() {
for tweet := range twitterscraper.GetTweets(context.Background(), "Twitter", 5) { for tweet := range twitterscraper.GetTweets(context.Background(), "Twitter", 50) {
if tweet.Error != nil { if tweet.Error != nil {
panic(tweet.Error) panic(tweet.Error)
} }
@ -37,7 +37,7 @@ func main() {
} }
``` ```
It appears you can ask for up to 5 pages of tweets reliably. It appears you can ask for up to 50 tweets.
### Search tweets by query standard operators ### Search tweets by query standard operators

18
tweets.go
View file

@ -45,12 +45,13 @@ type Result struct {
} }
// GetTweets returns channel with tweets for a given user. // GetTweets returns channel with tweets for a given user.
func GetTweets(ctx context.Context, user string, pages int) <-chan *Result { func GetTweets(ctx context.Context, user string, maxTweetsNbr int) <-chan *Result {
channel := make(chan *Result) channel := make(chan *Result)
go func(user string) { go func(user string) {
defer close(channel) defer close(channel)
var lastTweetID string var lastTweetID string
for pages > 0 { tweetsNbr := 0
for tweetsNbr < maxTweetsNbr {
select { select {
case <-ctx.Done(): case <-ctx.Done():
channel <- &Result{Error: ctx.Err()} channel <- &Result{Error: ctx.Err()}
@ -63,6 +64,11 @@ func GetTweets(ctx context.Context, user string, pages int) <-chan *Result {
channel <- &Result{Error: err} channel <- &Result{Error: err}
return return
} }
if len(tweets) == 0 {
break
}
for _, tweet := range tweets { for _, tweet := range tweets {
select { select {
case <-ctx.Done(): case <-ctx.Done():
@ -71,10 +77,12 @@ func GetTweets(ctx context.Context, user string, pages int) <-chan *Result {
default: default:
} }
lastTweetID = tweet.ID if tweetsNbr < maxTweetsNbr {
channel <- &Result{Tweet: *tweet} lastTweetID = tweet.ID
channel <- &Result{Tweet: *tweet}
}
tweetsNbr++
} }
pages--
} }
}(user) }(user)
return channel return channel

7
tweets_test.go
View file

@ -7,7 +7,8 @@ import (
func TestGetTweets(t *testing.T) { func TestGetTweets(t *testing.T) {
count := 0 count := 0
for tweet := range GetTweets(context.Background(), "nomadic_ua", 2) { maxTweetsNbr := 50
for tweet := range GetTweets(context.Background(), "Twitter", maxTweetsNbr) {
if tweet.Error != nil { if tweet.Error != nil {
t.Error(tweet.Error) t.Error(tweet.Error)
} else { } else {
@ -32,7 +33,7 @@ func TestGetTweets(t *testing.T) {
} }
} }
} }
if count == 0 { if count != maxTweetsNbr {
t.Error("Expected tweets count is greater than zero") t.Errorf("Expected tweets count=%v, got: %v", maxTweetsNbr, count)
} }
} }