lain/twitter-scrapper
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity

Separate test package

Browse source
This commit is contained in:
Alexander Sheiko 2021-12-07 10:18:01 +02:00
parent 55a0c83804
commit 4c9f06806a
7 changed files with 58 additions and 44 deletions
Download patch file Download diff file Expand all files Collapse all files

2
api.go
View file

@ -23,7 +23,7 @@ func (s *Scraper) RequestAPI(req *http.Request, target interface{}) error {
}() }()
} }
if s.guestToken == "" || s.guestCreatedAt.Before(time.Now().Add(-time.Hour*3)) { if !s.IsGuestToken() || s.guestCreatedAt.Before(time.Now().Add(-time.Hour*3)) {
err := s.GetGuestToken() err := s.GetGuestToken()
if err != nil { if err != nil {
return err return err

8
api_test.go
View file

@ -1,15 +1,17 @@
package twitterscraper package twitterscraper_test
import ( import (
"testing" "testing"
twitterscraper "github.com/n0madic/twitter-scraper"
) )
func TestGetGuestToken(t *testing.T) { func TestGetGuestToken(t *testing.T) {
scraper := New() scraper := twitterscraper.New()
if err := scraper.GetGuestToken(); err != nil { if err := scraper.GetGuestToken(); err != nil {
t.Errorf("getGuestToken() error = %v", err) t.Errorf("getGuestToken() error = %v", err)
} }
if scraper.guestToken == "" { if !scraper.IsGuestToken() {
t.Error("Expected non-empty guestToken") t.Error("Expected non-empty guestToken")
} }
} }

41
profile_test.go
View file

@ -1,4 +1,4 @@
package twitterscraper package twitterscraper_test
import ( import (
"fmt" "fmt"
@ -7,12 +7,13 @@ import (
"github.com/google/go-cmp/cmp" "github.com/google/go-cmp/cmp"
"github.com/google/go-cmp/cmp/cmpopts" "github.com/google/go-cmp/cmp/cmpopts"
twitterscraper "github.com/n0madic/twitter-scraper"
) )
func TestGetProfile(t *testing.T) { func TestGetProfile(t *testing.T) {
loc := time.FixedZone("UTC", 0) loc := time.FixedZone("UTC", 0)
joined := time.Date(2010, 01, 18, 8, 49, 30, 0, loc) joined := time.Date(2010, 01, 18, 8, 49, 30, 0, loc)
sample := Profile{ sample := twitterscraper.Profile{
Avatar: "https://pbs.twimg.com/profile_images/436075027193004032/XlDa2oaz_normal.jpeg", Avatar: "https://pbs.twimg.com/profile_images/436075027193004032/XlDa2oaz_normal.jpeg",
Banner: "https://pbs.twimg.com/profile_banners/106037940/1541084318", Banner: "https://pbs.twimg.com/profile_banners/106037940/1541084318",
Biography: "nothing", Biography: "nothing",
@ -29,18 +30,18 @@ func TestGetProfile(t *testing.T) {
Website: "https://nomadic.name", Website: "https://nomadic.name",
} }
profile, err := GetProfile("nomadic_ua") profile, err := twitterscraper.GetProfile("nomadic_ua")
if err != nil { if err != nil {
t.Error(err) t.Error(err)
} }
cmpOptions := cmp.Options{ cmpOptions := cmp.Options{
cmpopts.IgnoreFields(Profile{}, "FollowersCount"), cmpopts.IgnoreFields(twitterscraper.Profile{}, "FollowersCount"),
cmpopts.IgnoreFields(Profile{}, "FollowingCount"), cmpopts.IgnoreFields(twitterscraper.Profile{}, "FollowingCount"),
cmpopts.IgnoreFields(Profile{}, "FriendsCount"), cmpopts.IgnoreFields(twitterscraper.Profile{}, "FriendsCount"),
cmpopts.IgnoreFields(Profile{}, "LikesCount"), cmpopts.IgnoreFields(twitterscraper.Profile{}, "LikesCount"),
cmpopts.IgnoreFields(Profile{}, "ListedCount"), cmpopts.IgnoreFields(twitterscraper.Profile{}, "ListedCount"),
cmpopts.IgnoreFields(Profile{}, "TweetsCount"), cmpopts.IgnoreFields(twitterscraper.Profile{}, "TweetsCount"),
} }
if diff := cmp.Diff(sample, profile, cmpOptions...); diff != "" { if diff := cmp.Diff(sample, profile, cmpOptions...); diff != "" {
t.Error("Resulting profile does not match the sample", diff) t.Error("Resulting profile does not match the sample", diff)
@ -63,7 +64,7 @@ func TestGetProfile(t *testing.T) {
func TestGetProfilePrivate(t *testing.T) { func TestGetProfilePrivate(t *testing.T) {
loc := time.FixedZone("UTC", 0) loc := time.FixedZone("UTC", 0)
joined := time.Date(2020, 1, 26, 0, 3, 5, 0, loc) joined := time.Date(2020, 1, 26, 0, 3, 5, 0, loc)
sample := Profile{ sample := twitterscraper.Profile{
Avatar: "https://pbs.twimg.com/profile_images/1222218816484020224/ik9P1QZt_normal.jpg", Avatar: "https://pbs.twimg.com/profile_images/1222218816484020224/ik9P1QZt_normal.jpg",
Banner: "", Banner: "",
Biography: "private account", Biography: "private account",
@ -81,18 +82,18 @@ func TestGetProfilePrivate(t *testing.T) {
} }
// some random private profile (found via google) // some random private profile (found via google)
profile, err := GetProfile("tomdumont") profile, err := twitterscraper.GetProfile("tomdumont")
if err != nil { if err != nil {
t.Error(err) t.Error(err)
} }
cmpOptions := cmp.Options{ cmpOptions := cmp.Options{
cmpopts.IgnoreFields(Profile{}, "FollowersCount"), cmpopts.IgnoreFields(twitterscraper.Profile{}, "FollowersCount"),
cmpopts.IgnoreFields(Profile{}, "FollowingCount"), cmpopts.IgnoreFields(twitterscraper.Profile{}, "FollowingCount"),
cmpopts.IgnoreFields(Profile{}, "FriendsCount"), cmpopts.IgnoreFields(twitterscraper.Profile{}, "FriendsCount"),
cmpopts.IgnoreFields(Profile{}, "LikesCount"), cmpopts.IgnoreFields(twitterscraper.Profile{}, "LikesCount"),
cmpopts.IgnoreFields(Profile{}, "ListedCount"), cmpopts.IgnoreFields(twitterscraper.Profile{}, "ListedCount"),
cmpopts.IgnoreFields(Profile{}, "TweetsCount"), cmpopts.IgnoreFields(twitterscraper.Profile{}, "TweetsCount"),
} }
if diff := cmp.Diff(sample, profile, cmpOptions...); diff != "" { if diff := cmp.Diff(sample, profile, cmpOptions...); diff != "" {
t.Error("Resulting profile does not match the sample", diff) t.Error("Resulting profile does not match the sample", diff)
@ -110,7 +111,7 @@ func TestGetProfilePrivate(t *testing.T) {
} }
func TestGetProfileErrorSuspended(t *testing.T) { func TestGetProfileErrorSuspended(t *testing.T) {
_, err := GetProfile("123") _, err := twitterscraper.GetProfile("123")
if err == nil { if err == nil {
t.Error("Expected Error, got success") t.Error("Expected Error, got success")
} else { } else {
@ -123,7 +124,7 @@ func TestGetProfileErrorSuspended(t *testing.T) {
func TestGetProfileErrorNotFound(t *testing.T) { func TestGetProfileErrorNotFound(t *testing.T) {
neUser := "sample3123131" neUser := "sample3123131"
expectedError := fmt.Sprintf("User '%s' not found", neUser) expectedError := fmt.Sprintf("User '%s' not found", neUser)
_, err := GetProfile(neUser) _, err := twitterscraper.GetProfile(neUser)
if err == nil { if err == nil {
t.Error("Expected Error, got success") t.Error("Expected Error, got success")
} else { } else {
@ -134,7 +135,7 @@ func TestGetProfileErrorNotFound(t *testing.T) {
} }
func TestGetUserIDByScreenName(t *testing.T) { func TestGetUserIDByScreenName(t *testing.T) {
scraper := New() scraper := twitterscraper.New()
userID, err := scraper.GetUserIDByScreenName("Twitter") userID, err := scraper.GetUserIDByScreenName("Twitter")
if err != nil { if err != nil {
t.Errorf("getUserByScreenName() error = %v", err) t.Errorf("getUserByScreenName() error = %v", err)

8
scraper.go
View file

@ -3,13 +3,14 @@ package twitterscraper
import ( import (
"crypto/tls" "crypto/tls"
"errors" "errors"
"golang.org/x/net/proxy"
"net" "net"
"net/http" "net/http"
"net/url" "net/url"
"strings" "strings"
"sync" "sync"
"time" "time"
"golang.org/x/net/proxy"
) )
// Scraper object // Scraper object
@ -56,6 +57,11 @@ func New() *Scraper {
} }
} }
// IsGuestToken check if guest token not empty
func (s *Scraper) IsGuestToken() bool {
return s.guestToken != ""
}
// SetSearchMode switcher // SetSearchMode switcher
func (s *Scraper) SetSearchMode(mode SearchMode) *Scraper { func (s *Scraper) SetSearchMode(mode SearchMode) *Scraper {
s.searchMode = mode s.searchMode = mode

10
search_test.go
View file

@ -1,12 +1,14 @@
package twitterscraper package twitterscraper_test
import ( import (
"context" "context"
"testing" "testing"
twitterscraper "github.com/n0madic/twitter-scraper"
) )
func TestFetchSearchCursor(t *testing.T) { func TestFetchSearchCursor(t *testing.T) {
scraper := New() scraper := twitterscraper.New()
maxTweetsNbr := 150 maxTweetsNbr := 150
tweetsNbr := 0 tweetsNbr := 0
nextCursor := "" nextCursor := ""
@ -27,7 +29,7 @@ func TestGetSearchProfiles(t *testing.T) {
count := 0 count := 0
maxProfilesNbr := 150 maxProfilesNbr := 150
dupcheck := make(map[string]bool) dupcheck := make(map[string]bool)
scraper := New().SetSearchMode(SearchUsers) scraper := twitterscraper.New().SetSearchMode(twitterscraper.SearchUsers)
for profile := range scraper.SearchProfiles(context.Background(), "Twitter", maxProfilesNbr) { for profile := range scraper.SearchProfiles(context.Background(), "Twitter", maxProfilesNbr) {
if profile.Error != nil { if profile.Error != nil {
t.Error(profile.Error) t.Error(profile.Error)
@ -53,7 +55,7 @@ func TestGetSearchTweets(t *testing.T) {
count := 0 count := 0
maxTweetsNbr := 150 maxTweetsNbr := 150
dupcheck := make(map[string]bool) dupcheck := make(map[string]bool)
scraper := New().WithDelay(4) scraper := twitterscraper.New().WithDelay(4)
for tweet := range scraper.SearchTweets(context.Background(), "twitter -filter:retweets", maxTweetsNbr) { for tweet := range scraper.SearchTweets(context.Background(), "twitter -filter:retweets", maxTweetsNbr) {
if tweet.Error != nil { if tweet.Error != nil {
t.Error(tweet.Error) t.Error(tweet.Error)

6
trends_test.go
View file

@ -1,11 +1,13 @@
package twitterscraper package twitterscraper_test
import ( import (
"testing" "testing"
twitterscraper "github.com/n0madic/twitter-scraper"
) )
func TestGetTrends(t *testing.T) { func TestGetTrends(t *testing.T) {
trends, err := GetTrends() trends, err := twitterscraper.GetTrends()
if err != nil { if err != nil {
t.Error(err) t.Error(err)
} }

27
tweets_test.go
View file

@ -1,4 +1,4 @@
package twitterscraper package twitterscraper_test
import ( import (
"context" "context"
@ -7,19 +7,20 @@ import (
"github.com/google/go-cmp/cmp" "github.com/google/go-cmp/cmp"
"github.com/google/go-cmp/cmp/cmpopts" "github.com/google/go-cmp/cmp/cmpopts"
twitterscraper "github.com/n0madic/twitter-scraper"
) )
var cmpOptions = cmp.Options{ var cmpOptions = cmp.Options{
cmpopts.IgnoreFields(Tweet{}, "Likes"), cmpopts.IgnoreFields(twitterscraper.Tweet{}, "Likes"),
cmpopts.IgnoreFields(Tweet{}, "Replies"), cmpopts.IgnoreFields(twitterscraper.Tweet{}, "Replies"),
cmpopts.IgnoreFields(Tweet{}, "Retweets"), cmpopts.IgnoreFields(twitterscraper.Tweet{}, "Retweets"),
} }
func TestGetTweets(t *testing.T) { func TestGetTweets(t *testing.T) {
count := 0 count := 0
maxTweetsNbr := 300 maxTweetsNbr := 300
dupcheck := make(map[string]bool) dupcheck := make(map[string]bool)
for tweet := range GetTweets(context.Background(), "Twitter", maxTweetsNbr) { for tweet := range twitterscraper.GetTweets(context.Background(), "Twitter", maxTweetsNbr) {
if tweet.Error != nil { if tweet.Error != nil {
t.Error(tweet.Error) t.Error(tweet.Error)
} else { } else {
@ -70,7 +71,7 @@ func TestGetTweets(t *testing.T) {
} }
func TestGetTweet(t *testing.T) { func TestGetTweet(t *testing.T) {
sample := Tweet{ sample := twitterscraper.Tweet{
HTML: "That thing you didn’t Tweet but wanted to but didn’t but got so close but then were like nah. <br><br>We have a place for that now—Fleets! <br><br>Rolling out to everyone starting today. <br><a href=\"https://t.co/auQAHXZMfH\"><img src=\"https://pbs.twimg.com/amplify_video_thumb/1328684333599756289/img/cP5KwbIXbGunNSBy.jpg\"/></a>", HTML: "That thing you didn’t Tweet but wanted to but didn’t but got so close but then were like nah. <br><br>We have a place for that now—Fleets! <br><br>Rolling out to everyone starting today. <br><a href=\"https://t.co/auQAHXZMfH\"><img src=\"https://pbs.twimg.com/amplify_video_thumb/1328684333599756289/img/cP5KwbIXbGunNSBy.jpg\"/></a>",
ID: "1328684389388185600", ID: "1328684389388185600",
PermanentURL: "https://twitter.com/Twitter/status/1328684389388185600", PermanentURL: "https://twitter.com/Twitter/status/1328684389388185600",
@ -80,13 +81,13 @@ func TestGetTweet(t *testing.T) {
Timestamp: 1605618018, Timestamp: 1605618018,
UserID: "783214", UserID: "783214",
Username: "Twitter", Username: "Twitter",
Videos: []Video{{ Videos: []twitterscraper.Video{{
ID: "1328684333599756289", ID: "1328684333599756289",
Preview: "https://pbs.twimg.com/amplify_video_thumb/1328684333599756289/img/cP5KwbIXbGunNSBy.jpg", Preview: "https://pbs.twimg.com/amplify_video_thumb/1328684333599756289/img/cP5KwbIXbGunNSBy.jpg",
URL: "https://video.twimg.com/amplify_video/1328684333599756289/vid/960x720/PcL8yv8KhgQ48Qpt.mp4?tag=13", URL: "https://video.twimg.com/amplify_video/1328684333599756289/vid/960x720/PcL8yv8KhgQ48Qpt.mp4?tag=13",
}}, }},
} }
tweet, err := defaultScraper.GetTweet("1328684389388185600") tweet, err := twitterscraper.GetTweet("1328684389388185600")
if err != nil { if err != nil {
t.Error(err) t.Error(err)
} else { } else {
@ -97,7 +98,7 @@ func TestGetTweet(t *testing.T) {
} }
func TestQuotedAndReply(t *testing.T) { func TestQuotedAndReply(t *testing.T) {
sample := &Tweet{ sample := &twitterscraper.Tweet{
HTML: "The Easiest Problem Everyone Gets Wrong <br><br>[new video] --&gt; <a href=\"https://youtu.be/ytfCdqWhmdg\">https://t.co/YdaeDYmPAU</a> <br><a href=\"https://t.co/iKu4Xs6o2V\"><img src=\"https://pbs.twimg.com/media/ESsZa9AXgAIAYnF.jpg\"/></a>", HTML: "The Easiest Problem Everyone Gets Wrong <br><br>[new video] --&gt; <a href=\"https://youtu.be/ytfCdqWhmdg\">https://t.co/YdaeDYmPAU</a> <br><a href=\"https://t.co/iKu4Xs6o2V\"><img src=\"https://pbs.twimg.com/media/ESsZa9AXgAIAYnF.jpg\"/></a>",
ID: "1237110546383724547", ID: "1237110546383724547",
Likes: 485, Likes: 485,
@ -112,7 +113,7 @@ func TestQuotedAndReply(t *testing.T) {
UserID: "978944851", UserID: "978944851",
Username: "VsauceTwo", Username: "VsauceTwo",
} }
tweet, err := defaultScraper.GetTweet("1237110897597976576") tweet, err := twitterscraper.GetTweet("1237110897597976576")
if err != nil { if err != nil {
t.Error(err) t.Error(err)
} else { } else {
@ -123,7 +124,7 @@ func TestQuotedAndReply(t *testing.T) {
t.Error("Resulting quote does not match the sample", diff) t.Error("Resulting quote does not match the sample", diff)
} }
} }
tweet, err = defaultScraper.GetTweet("1237111868445134850") tweet, err = twitterscraper.GetTweet("1237111868445134850")
if err != nil { if err != nil {
t.Error(err) t.Error(err)
} else { } else {
@ -137,7 +138,7 @@ func TestQuotedAndReply(t *testing.T) {
} }
func TestRetweet(t *testing.T) { func TestRetweet(t *testing.T) {
sample := &Tweet{ sample := &twitterscraper.Tweet{
HTML: "We’ve seen an increase in attacks against Asian communities and individuals around the world. It’s important to know that this isn’t new; throughout history, Asians have experienced violence and exclusion. However, their diverse lived experiences have largely been overlooked.", HTML: "We’ve seen an increase in attacks against Asian communities and individuals around the world. It’s important to know that this isn’t new; throughout history, Asians have experienced violence and exclusion. However, their diverse lived experiences have largely been overlooked.",
ID: "1359151057872580612", ID: "1359151057872580612",
Likes: 6683, Likes: 6683,
@ -150,7 +151,7 @@ func TestRetweet(t *testing.T) {
UserID: "773578328498372608", UserID: "773578328498372608",
Username: "TwitterTogether", Username: "TwitterTogether",
} }
tweet, err := defaultScraper.GetTweet("1362849141248974853") tweet, err := twitterscraper.GetTweet("1362849141248974853")
if err != nil { if err != nil {
t.Error(err) t.Error(err)
} else { } else {