Merge pull request #3 from imperatrona/user-medias

User medias

README.md

@@ -122,6 +122,32 @@ func main() {
 
 It appears you can ask for up to 50 tweets.
 
+### Get user medias
+
+```golang
+package main
+
+import (
+    "context"
+    "fmt"
+    twitterscraper "github.com/imperatrona/twitter-scraper"
+)
+
+func main() {
+    scraper := twitterscraper.New()
+    account, err := scraper.LoginOpenAccount()
+    if err != nil {
+        panic(err)
+    }
+    for tweet := range scraper.GetMediaTweets(context.Background(), "Twitter", 50) {
+        if tweet.Error != nil {
+            panic(tweet.Error)
+        }
+        fmt.Println(tweet.Text)
+    }
+}
+```
+
 ### Get single tweet
 
 ```golang

medias.go

@@ -0,0 +1,84 @@
+package twitterscraper
+
+import (
+	"context"
+	"net/url"
+)
+
+// GetTweets returns channel with tweets for a given user.
+func (s *Scraper) GetMediaTweets(ctx context.Context, user string, maxTweetsNbr int) <-chan *TweetResult {
+	return getTweetTimeline(ctx, user, maxTweetsNbr, s.FetchMediaTweets)
+}
+
+// FetchMediaTweets gets tweets with medias for a given user, via the Twitter frontend API.
+func (s *Scraper) FetchMediaTweets(user string, maxTweetsNbr int, cursor string) ([]*Tweet, string, error) {
+	userID, err := s.GetUserIDByScreenName(user)
+	if err != nil {
+		return nil, "", err
+	}
+
+	return s.FetchMediaTweetsByUserID(userID, maxTweetsNbr, cursor)
+}
+
+// FetchMediaTweetsByUserID gets tweets with medias for a given userID, via the Twitter frontend GraphQL API.
+func (s *Scraper) FetchMediaTweetsByUserID(userID string, maxTweetsNbr int, cursor string) ([]*Tweet, string, error) {
+	if maxTweetsNbr > 200 {
+		maxTweetsNbr = 200
+	}
+
+	req, err := s.newRequest("GET", "https://twitter.com/i/api/graphql/2tLOJWwGuCTytDrGBg8VwQ/UserMedia")
+	if err != nil {
+		return nil, "", err
+	}
+
+	variables := map[string]interface{}{
+		"userId":                 userID,
+		"count":                  maxTweetsNbr,
+		"includePromotedContent": false,
+		"withClientEventToken":   false,
+		"withBirdwatchNotes":     false,
+		"withVoice":              true,
+		"withV2Timeline":         true,
+	}
+	features := map[string]interface{}{
+		"responsive_web_graphql_exclude_directive_enabled":                        true,
+		"verified_phone_label_enabled":                                            false,
+		"creator_subscriptions_tweet_preview_api_enabled":                         true,
+		"responsive_web_graphql_timeline_navigation_enabled":                      true,
+		"responsive_web_graphql_skip_user_profile_image_extensions_enabled":       false,
+		"c9s_tweet_anatomy_moderator_badge_enabled":                               true,
+		"tweetypie_unmention_optimization_enabled":                                true,
+		"responsive_web_edit_tweet_api_enabled":                                   true,
+		"graphql_is_translatable_rweb_tweet_is_translatable_enabled":              true,
+		"view_counts_everywhere_api_enabled":                                      true,
+		"longform_notetweets_consumption_enabled":                                 true,
+		"responsive_web_twitter_article_tweet_consumption_enabled":                true,
+		"tweet_awards_web_tipping_enabled":                                        false,
+		"freedom_of_speech_not_reach_fetch_enabled":                               true,
+		"standardized_nudges_misinfo":                                             true,
+		"tweet_with_visibility_results_prefer_gql_limited_actions_policy_enabled": true,
+		"rweb_video_timestamps_enabled":                                           true,
+		"longform_notetweets_rich_text_read_enabled":                              true,
+		"longform_notetweets_inline_media_enabled":                                true,
+		"responsive_web_media_download_video_enabled":                             false,
+		"responsive_web_enhance_cards_enabled":                                    false,
+	}
+
+	if cursor != "" {
+		variables["cursor"] = cursor
+	}
+
+	query := url.Values{}
+	query.Set("variables", mapToJSONString(variables))
+	query.Set("features", mapToJSONString(features))
+	req.URL.RawQuery = query.Encode()
+
+	var timeline timelineV2
+	err = s.RequestAPI(req, &timeline)
+	if err != nil {
+		return nil, "", err
+	}
+
+	tweets, nextCursor := timeline.parseTweets()
+	return tweets, nextCursor, nil
+}

medias_test.go

@@ -0,0 +1,60 @@
+package twitterscraper_test
+
+import (
+	"context"
+	"testing"
+)
+
+func TestGetMedias(t *testing.T) {
+	count := 0
+	maxTweetsNbr := 20
+	dupcheck := make(map[string]bool)
+	for tweet := range testScraper.GetMediaTweets(context.Background(), "XSpaces", maxTweetsNbr) {
+		if tweet.Error != nil {
+			t.Error(tweet.Error)
+		} else {
+			count++
+			if tweet.ID == "" {
+				t.Error("Expected tweet ID is empty")
+			} else {
+				if dupcheck[tweet.ID] {
+					t.Errorf("Detect duplicated tweet ID: %s", tweet.ID)
+				} else {
+					dupcheck[tweet.ID] = true
+				}
+			}
+			if tweet.UserID == "" {
+				t.Error("Expected tweet UserID is empty")
+			}
+			if tweet.Username == "" {
+				t.Error("Expected tweet Username is empty")
+			}
+			if tweet.PermanentURL == "" {
+				t.Error("Expected tweet PermanentURL is empty")
+			}
+			if tweet.Text == "" {
+				t.Error("Expected tweet Text is empty")
+			}
+			if tweet.TimeParsed.IsZero() {
+				t.Error("Expected tweet TimeParsed is zero")
+			}
+			if tweet.Timestamp == 0 {
+				t.Error("Expected tweet Timestamp is greater than zero")
+			}
+			for _, video := range tweet.Videos {
+				if video.ID == "" {
+					t.Error("Expected tweet video ID is empty")
+				}
+				if video.Preview == "" {
+					t.Error("Expected tweet video Preview is empty")
+				}
+				if video.URL == "" {
+					t.Error("Expected tweet video URL is empty")
+				}
+			}
+		}
+	}
+	if count != maxTweetsNbr {
+		t.Errorf("Expected tweets count=%v, got: %v", maxTweetsNbr, count)
+	}
+}

scraper.go

@@ -143,7 +143,16 @@ func (s *Scraper) SetProxy(proxyAddr string) error {
 		host := proxyURL.Hostname()
 		port := proxyURL.Port()
 
-		dialSocksProxy, err := proxy.SOCKS5("tcp", host+":"+port, &proxy.Auth{User: username, Password: password}, baseDialer)
+		var auth *proxy.Auth
+
+		if username != "" || password != "" {
+			auth = &proxy.Auth{
+				User:     username,
+				Password: password,
+			}
+		}
+
+		dialSocksProxy, err := proxy.SOCKS5("tcp", host+":"+port, auth, baseDialer)
 		if err != nil {
 			return errors.New("error creating socks5 proxy :" + err.Error())
 		}

timeline_v2.go

@@ -47,11 +47,7 @@ func (result *result) parse() *Tweet {
 	return tw
 }
 
-type entry struct {
+type item struct {
-	Content struct {
-		CursorType string `json:"cursorType"`
-		Value      string `json:"value"`
-		Items      []struct {
 	Item struct {
 		ItemContent struct {
 			TweetDisplayType string `json:"tweetDisplayType"`
@@ -60,7 +56,13 @@ type entry struct {
 			} `json:"tweet_results"`
 		} `json:"itemContent"`
 	} `json:"item"`
-		} `json:"items"`
+}
+
+type entry struct {
+	Content struct {
+		CursorType  string `json:"cursorType"`
+		Value       string `json:"value"`
+		Items       []item `json:"items"`
 		ItemContent struct {
 			TweetDisplayType string `json:"tweetDisplayType"`
 			TweetResults     struct {
@@ -85,6 +87,7 @@ type timelineV2 struct {
 				TimelineV2 struct {
 					Timeline struct {
 						Instructions []struct {
+							ModuleItems []item  `json:"moduleItems"`
 							Entries     []entry `json:"entries"`
 							Entry       entry   `json:"entry"`
 							Type        string  `json:"type"`
@@ -110,6 +113,22 @@ func (timeline *timelineV2) parseTweets() ([]*Tweet, string) {
 					tweets = append(tweets, tweet)
 				}
 			}
+			if len(entry.Content.Items) > 0 {
+				for _, item := range entry.Content.Items {
+					if tweet := item.Item.ItemContent.TweetResults.Result.parse(); tweet != nil {
+						tweets = append(tweets, tweet)
+					}
+				}
+			}
+		}
+		if len(instruction.ModuleItems) > 0 {
+			for _, entry := range instruction.ModuleItems {
+				if entry.Item.ItemContent.TweetResults.Result.Typename == "Tweet" {
+					if tweet := entry.Item.ItemContent.TweetResults.Result.parse(); tweet != nil {
+						tweets = append(tweets, tweet)
+					}
+				}
+			}
 		}
 	}
 	return tweets, cursor