lain/twitter-scrapper
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity

Separate test package

Browse source
This commit is contained in:
Alexander Sheiko 2021-12-07 10:18:01 +02:00
parent 55a0c83804
commit 4c9f06806a
7 changed files with 58 additions and 44 deletions
Download patch file Download diff file Expand all files Collapse all files

2
api.go
View file

@ -23,7 +23,7 @@ func (s *Scraper) RequestAPI(req *http.Request, target interface{}) error {
}()
}
if s.guestToken == "" || s.guestCreatedAt.Before(time.Now().Add(-time.Hour*3)) {
if !s.IsGuestToken() || s.guestCreatedAt.Before(time.Now().Add(-time.Hour*3)) {
err := s.GetGuestToken()
if err != nil {
return err

8
api_test.go
View file

@ -1,15 +1,17 @@
package twitterscraper
package twitterscraper_test
import (
"testing"
twitterscraper "github.com/n0madic/twitter-scraper"
)
func TestGetGuestToken(t *testing.T) {
scraper := New()
scraper := twitterscraper.New()
if err := scraper.GetGuestToken(); err != nil {
t.Errorf("getGuestToken() error = %v", err)
}
if scraper.guestToken == "" {
if !scraper.IsGuestToken() {
t.Error("Expected non-empty guestToken")
}
}

41
profile_test.go
View file

@ -1,4 +1,4 @@
package twitterscraper
package twitterscraper_test
import (
"fmt"
@ -7,12 +7,13 @@ import (
"github.com/google/go-cmp/cmp"
"github.com/google/go-cmp/cmp/cmpopts"
twitterscraper "github.com/n0madic/twitter-scraper"
)
func TestGetProfile(t *testing.T) {
loc := time.FixedZone("UTC", 0)
joined := time.Date(2010, 01, 18, 8, 49, 30, 0, loc)
sample := Profile{
sample := twitterscraper.Profile{
Avatar: "https://pbs.twimg.com/profile_images/436075027193004032/XlDa2oaz_normal.jpeg",
Banner: "https://pbs.twimg.com/profile_banners/106037940/1541084318",
Biography: "nothing",
@ -29,18 +30,18 @@ func TestGetProfile(t *testing.T) {
Website: "https://nomadic.name",
}
profile, err := GetProfile("nomadic_ua")
profile, err := twitterscraper.GetProfile("nomadic_ua")
if err != nil {
t.Error(err)
}
cmpOptions := cmp.Options{
cmpopts.IgnoreFields(Profile{}, "FollowersCount"),
cmpopts.IgnoreFields(Profile{}, "FollowingCount"),
cmpopts.IgnoreFields(Profile{}, "FriendsCount"),
cmpopts.IgnoreFields(Profile{}, "LikesCount"),
cmpopts.IgnoreFields(Profile{}, "ListedCount"),
cmpopts.IgnoreFields(Profile{}, "TweetsCount"),
cmpopts.IgnoreFields(twitterscraper.Profile{}, "FollowersCount"),
cmpopts.IgnoreFields(twitterscraper.Profile{}, "FollowingCount"),
cmpopts.IgnoreFields(twitterscraper.Profile{}, "FriendsCount"),
cmpopts.IgnoreFields(twitterscraper.Profile{}, "LikesCount"),
cmpopts.IgnoreFields(twitterscraper.Profile{}, "ListedCount"),
cmpopts.IgnoreFields(twitterscraper.Profile{}, "TweetsCount"),
}
if diff := cmp.Diff(sample, profile, cmpOptions...); diff != "" {
t.Error("Resulting profile does not match the sample", diff)
@ -63,7 +64,7 @@ func TestGetProfile(t *testing.T) {
func TestGetProfilePrivate(t *testing.T) {
loc := time.FixedZone("UTC", 0)
joined := time.Date(2020, 1, 26, 0, 3, 5, 0, loc)
sample := Profile{
sample := twitterscraper.Profile{
Avatar: "https://pbs.twimg.com/profile_images/1222218816484020224/ik9P1QZt_normal.jpg",
Banner: "",
Biography: "private account",
@ -81,18 +82,18 @@ func TestGetProfilePrivate(t *testing.T) {
}
// some random private profile (found via google)
profile, err := GetProfile("tomdumont")
profile, err := twitterscraper.GetProfile("tomdumont")
if err != nil {
t.Error(err)
}
cmpOptions := cmp.Options{
cmpopts.IgnoreFields(Profile{}, "FollowersCount"),
cmpopts.IgnoreFields(Profile{}, "FollowingCount"),
cmpopts.IgnoreFields(Profile{}, "FriendsCount"),
cmpopts.IgnoreFields(Profile{}, "LikesCount"),
cmpopts.IgnoreFields(Profile{}, "ListedCount"),
cmpopts.IgnoreFields(Profile{}, "TweetsCount"),
cmpopts.IgnoreFields(twitterscraper.Profile{}, "FollowersCount"),
cmpopts.IgnoreFields(twitterscraper.Profile{}, "FollowingCount"),
cmpopts.IgnoreFields(twitterscraper.Profile{}, "FriendsCount"),
cmpopts.IgnoreFields(twitterscraper.Profile{}, "LikesCount"),
cmpopts.IgnoreFields(twitterscraper.Profile{}, "ListedCount"),
cmpopts.IgnoreFields(twitterscraper.Profile{}, "TweetsCount"),
}
if diff := cmp.Diff(sample, profile, cmpOptions...); diff != "" {
t.Error("Resulting profile does not match the sample", diff)
@ -110,7 +111,7 @@ func TestGetProfilePrivate(t *testing.T) {
}
func TestGetProfileErrorSuspended(t *testing.T) {
_, err := GetProfile("123")
_, err := twitterscraper.GetProfile("123")
if err == nil {
t.Error("Expected Error, got success")
} else {
@ -123,7 +124,7 @@ func TestGetProfileErrorSuspended(t *testing.T) {
func TestGetProfileErrorNotFound(t *testing.T) {
neUser := "sample3123131"
expectedError := fmt.Sprintf("User '%s' not found", neUser)
_, err := GetProfile(neUser)
_, err := twitterscraper.GetProfile(neUser)
if err == nil {
t.Error("Expected Error, got success")
} else {
@ -134,7 +135,7 @@ func TestGetProfileErrorNotFound(t *testing.T) {
}
func TestGetUserIDByScreenName(t *testing.T) {
scraper := New()
scraper := twitterscraper.New()
userID, err := scraper.GetUserIDByScreenName("Twitter")
if err != nil {
t.Errorf("getUserByScreenName() error = %v", err)

8
scraper.go
View file

@ -3,13 +3,14 @@ package twitterscraper
import (
"crypto/tls"
"errors"
"golang.org/x/net/proxy"
"net"
"net/http"
"net/url"
"strings"
"sync"
"time"
"golang.org/x/net/proxy"
)
// Scraper object
@ -56,6 +57,11 @@ func New() *Scraper {
}
}
// IsGuestToken check if guest token not empty
func (s *Scraper) IsGuestToken() bool {
return s.guestToken != ""
}
// SetSearchMode switcher
func (s *Scraper) SetSearchMode(mode SearchMode) *Scraper {
s.searchMode = mode

10
search_test.go
View file

@ -1,12 +1,14 @@
package twitterscraper
package twitterscraper_test
import (
"context"
"testing"
twitterscraper "github.com/n0madic/twitter-scraper"
)
func TestFetchSearchCursor(t *testing.T) {
scraper := New()
scraper := twitterscraper.New()
maxTweetsNbr := 150
tweetsNbr := 0
nextCursor := ""
@ -27,7 +29,7 @@ func TestGetSearchProfiles(t *testing.T) {
count := 0
maxProfilesNbr := 150
dupcheck := make(map[string]bool)
scraper := New().SetSearchMode(SearchUsers)
scraper := twitterscraper.New().SetSearchMode(twitterscraper.SearchUsers)
for profile := range scraper.SearchProfiles(context.Background(), "Twitter", maxProfilesNbr) {
if profile.Error != nil {
t.Error(profile.Error)
@ -53,7 +55,7 @@ func TestGetSearchTweets(t *testing.T) {
count := 0
maxTweetsNbr := 150
dupcheck := make(map[string]bool)
scraper := New().WithDelay(4)
scraper := twitterscraper.New().WithDelay(4)
for tweet := range scraper.SearchTweets(context.Background(), "twitter -filter:retweets", maxTweetsNbr) {
if tweet.Error != nil {
t.Error(tweet.Error)

6
trends_test.go
View file

@ -1,11 +1,13 @@
package twitterscraper
package twitterscraper_test
import (
"testing"
twitterscraper "github.com/n0madic/twitter-scraper"
)
func TestGetTrends(t *testing.T) {
trends, err := GetTrends()
trends, err := twitterscraper.GetTrends()
if err != nil {
t.Error(err)
}

27
tweets_test.go
View file

@ -1,4 +1,4 @@
package twitterscraper
package twitterscraper_test
import (
"context"
@ -7,19 +7,20 @@ import (
"github.com/google/go-cmp/cmp"
"github.com/google/go-cmp/cmp/cmpopts"
twitterscraper "github.com/n0madic/twitter-scraper"
)
var cmpOptions = cmp.Options{
cmpopts.IgnoreFields(Tweet{}, "Likes"),
cmpopts.IgnoreFields(Tweet{}, "Replies"),
cmpopts.IgnoreFields(Tweet{}, "Retweets"),
cmpopts.IgnoreFields(twitterscraper.Tweet{}, "Likes"),
cmpopts.IgnoreFields(twitterscraper.Tweet{}, "Replies"),
cmpopts.IgnoreFields(twitterscraper.Tweet{}, "Retweets"),
}
func TestGetTweets(t *testing.T) {
count := 0
maxTweetsNbr := 300
dupcheck := make(map[string]bool)
for tweet := range GetTweets(context.Background(), "Twitter", maxTweetsNbr) {
for tweet := range twitterscraper.GetTweets(context.Background(), "Twitter", maxTweetsNbr) {
if tweet.Error != nil {
t.Error(tweet.Error)
} else {
@ -70,7 +71,7 @@ func TestGetTweets(t *testing.T) {
}
func TestGetTweet(t *testing.T) {
sample := Tweet{
sample := twitterscraper.Tweet{
HTML: "That thing you didn’t Tweet but wanted to but didn’t but got so close but then were like nah. <br><br>We have a place for that now—Fleets! <br><br>Rolling out to everyone starting today. <br><a href=\"https://t.co/auQAHXZMfH\"><img src=\"https://pbs.twimg.com/amplify_video_thumb/1328684333599756289/img/cP5KwbIXbGunNSBy.jpg\"/></a>",
ID: "1328684389388185600",
PermanentURL: "https://twitter.com/Twitter/status/1328684389388185600",
@ -80,13 +81,13 @@ func TestGetTweet(t *testing.T) {
Timestamp: 1605618018,
UserID: "783214",
Username: "Twitter",
Videos: []Video{{
Videos: []twitterscraper.Video{{
ID: "1328684333599756289",
Preview: "https://pbs.twimg.com/amplify_video_thumb/1328684333599756289/img/cP5KwbIXbGunNSBy.jpg",
URL: "https://video.twimg.com/amplify_video/1328684333599756289/vid/960x720/PcL8yv8KhgQ48Qpt.mp4?tag=13",
}},
}
tweet, err := defaultScraper.GetTweet("1328684389388185600")
tweet, err := twitterscraper.GetTweet("1328684389388185600")
if err != nil {
t.Error(err)
} else {
@ -97,7 +98,7 @@ func TestGetTweet(t *testing.T) {
}
func TestQuotedAndReply(t *testing.T) {
sample := &Tweet{
sample := &twitterscraper.Tweet{
HTML: "The Easiest Problem Everyone Gets Wrong <br><br>[new video] --&gt; <a href=\"https://youtu.be/ytfCdqWhmdg\">https://t.co/YdaeDYmPAU</a> <br><a href=\"https://t.co/iKu4Xs6o2V\"><img src=\"https://pbs.twimg.com/media/ESsZa9AXgAIAYnF.jpg\"/></a>",
ID: "1237110546383724547",
Likes: 485,
@ -112,7 +113,7 @@ func TestQuotedAndReply(t *testing.T) {
UserID: "978944851",
Username: "VsauceTwo",
}
tweet, err := defaultScraper.GetTweet("1237110897597976576")
tweet, err := twitterscraper.GetTweet("1237110897597976576")
if err != nil {
t.Error(err)
} else {
@ -123,7 +124,7 @@ func TestQuotedAndReply(t *testing.T) {
t.Error("Resulting quote does not match the sample", diff)
}
}
tweet, err = defaultScraper.GetTweet("1237111868445134850")
tweet, err = twitterscraper.GetTweet("1237111868445134850")
if err != nil {
t.Error(err)
} else {
@ -137,7 +138,7 @@ func TestQuotedAndReply(t *testing.T) {
}
func TestRetweet(t *testing.T) {
sample := &Tweet{
sample := &twitterscraper.Tweet{
HTML: "We’ve seen an increase in attacks against Asian communities and individuals around the world. It’s important to know that this isn’t new; throughout history, Asians have experienced violence and exclusion. However, their diverse lived experiences have largely been overlooked.",
ID: "1359151057872580612",
Likes: 6683,
@ -150,7 +151,7 @@ func TestRetweet(t *testing.T) {
UserID: "773578328498372608",
Username: "TwitterTogether",
}
tweet, err := defaultScraper.GetTweet("1362849141248974853")
tweet, err := twitterscraper.GetTweet("1362849141248974853")
if err != nil {
t.Error(err)
} else {