From 56af56d23031fc6572b4fef9c96a471f676ec92d Mon Sep 17 00:00:00 2001 From: Valentine Date: Wed, 21 Feb 2024 04:41:51 +0300 Subject: [PATCH] add bookmark method --- README.md | 26 ++++++++++++++++- bookmarks.go | 71 +++++++++++++++++++++++++++++++++++++++++++++++ bookmarks_test.go | 60 +++++++++++++++++++++++++++++++++++++++ timeline_v2.go | 32 +++++++++++++++++++++ 4 files changed, 188 insertions(+), 1 deletion(-) create mode 100644 bookmarks.go create mode 100644 bookmarks_test.go diff --git a/README.md b/README.md index 5e2749c..5d4f3b9 100644 --- a/README.md +++ b/README.md @@ -23,6 +23,7 @@ You can use this library to get tweets, profiles, and trends trivially. - [Get tweet](#get-tweet) - [Get user tweets](#get-user-tweets) - [Get user medias](#get-user-medias) + - [Get bookmarks](#get-bookmarks) - [Search tweets](#search-tweets) - [Search params](#search-params) - [Get profile](#get-profile) @@ -233,7 +234,30 @@ var cursor string tweets, cursor, err := scraper.FetchMediaTweets("taylorswift13", 20, cursor) ``` - +### Get bookmarks + +> [!IMPORTANT] +> Requires authentication! + +500 requests / 15 minutes + +`GetBookmarks` returns a channel with the specified number of bookmarked tweets. It’s using the `FetchBookmarks` method under the hood. + +```golang +for tweet := range scraper.GetBookmarks(context.Background(), 50) { + if tweet.Error != nil { + panic(tweet.Error) + } + fmt.Println(tweet.Text) +} +``` + +`FetchBookmarks` returns bookmarked tweets and cursor for fetching the next page. Each request returns up to 20 tweets. + +```golang +var cursor string +tweets, cursor, err := scraper.FetchBookmarks(20, cursor) +``` ### Search tweets diff --git a/bookmarks.go b/bookmarks.go new file mode 100644 index 0000000..89d7fe8 --- /dev/null +++ b/bookmarks.go @@ -0,0 +1,71 @@ +package twitterscraper + +import ( + "context" + "net/url" +) + +// GetBookmarks returns channel with tweets from user bookmarks. +func (s *Scraper) GetBookmarks(ctx context.Context, maxTweetsNbr int) <-chan *TweetResult { + return getTweetTimeline(ctx, "", maxTweetsNbr, func(unused string, maxTweetsNbr int, cursor string) ([]*Tweet, string, error) { + return s.FetchBookmarks(maxTweetsNbr, cursor) + }) +} + +// FetchBookmarks gets bookmarked tweets via the Twitter frontend GraphQL API. +func (s *Scraper) FetchBookmarks(maxTweetsNbr int, cursor string) ([]*Tweet, string, error) { + if maxTweetsNbr > 200 { + maxTweetsNbr = 200 + } + + req, err := s.newRequest("GET", "https://twitter.com/i/api/graphql/-IyJFt9_jS_9d_vS3NN-fA/Bookmarks") + if err != nil { + return nil, "", err + } + + variables := map[string]interface{}{ + "count": maxTweetsNbr, + "includePromotedContent": false, + } + features := map[string]interface{}{ + "graphql_timeline_v2_bookmark_timeline": true, + "responsive_web_graphql_exclude_directive_enabled": true, + "verified_phone_label_enabled": false, + "creator_subscriptions_tweet_preview_api_enabled": true, + "responsive_web_graphql_timeline_navigation_enabled": true, + "responsive_web_graphql_skip_user_profile_image_extensions_enabled": false, + "c9s_tweet_anatomy_moderator_badge_enabled": true, + "tweetypie_unmention_optimization_enabled": true, + "responsive_web_edit_tweet_api_enabled": true, + "graphql_is_translatable_rweb_tweet_is_translatable_enabled": true, + "view_counts_everywhere_api_enabled": true, + "longform_notetweets_consumption_enabled": true, + "responsive_web_twitter_article_tweet_consumption_enabled": true, + "tweet_awards_web_tipping_enabled": false, + "freedom_of_speech_not_reach_fetch_enabled": true, + "standardized_nudges_misinfo": true, + "tweet_with_visibility_results_prefer_gql_limited_actions_policy_enabled": true, + "rweb_video_timestamps_enabled": true, + "longform_notetweets_rich_text_read_enabled": true, + "longform_notetweets_inline_media_enabled": true, + "responsive_web_enhance_cards_enabled": false, + } + + if cursor != "" { + variables["cursor"] = cursor + } + + query := url.Values{} + query.Set("variables", mapToJSONString(variables)) + query.Set("features", mapToJSONString(features)) + req.URL.RawQuery = query.Encode() + + var timeline bookmarksTimelineV2 + err = s.RequestAPI(req, &timeline) + if err != nil { + return nil, "", err + } + + tweets, nextCursor := timeline.parseTweets() + return tweets, nextCursor, nil +} diff --git a/bookmarks_test.go b/bookmarks_test.go new file mode 100644 index 0000000..2aa53d0 --- /dev/null +++ b/bookmarks_test.go @@ -0,0 +1,60 @@ +package twitterscraper_test + +import ( + "context" + "testing" +) + +func TestGetBookmarks(t *testing.T) { + count := 0 + maxTweetsNbr := 40 + dupcheck := make(map[string]bool) + for tweet := range testScraper.GetBookmarks(context.Background(), maxTweetsNbr) { + if tweet.Error != nil { + t.Error(tweet.Error) + } else { + count++ + if tweet.ID == "" { + t.Error("Expected tweet ID is empty") + } else { + if dupcheck[tweet.ID] { + t.Errorf("Detect duplicated tweet ID: %s", tweet.ID) + } else { + dupcheck[tweet.ID] = true + } + } + if tweet.UserID == "" { + t.Error("Expected tweet UserID is empty") + } + if tweet.Username == "" { + t.Error("Expected tweet Username is empty") + } + if tweet.PermanentURL == "" { + t.Error("Expected tweet PermanentURL is empty") + } + if tweet.Text == "" { + t.Error("Expected tweet Text is empty") + } + if tweet.TimeParsed.IsZero() { + t.Error("Expected tweet TimeParsed is zero") + } + if tweet.Timestamp == 0 { + t.Error("Expected tweet Timestamp is greater than zero") + } + for _, video := range tweet.Videos { + if video.ID == "" { + t.Error("Expected tweet video ID is empty") + } + if video.Preview == "" { + t.Error("Expected tweet video Preview is empty") + } + if video.URL == "" { + t.Error("Expected tweet video URL is empty") + } + } + } + } + if count != maxTweetsNbr { + t.Errorf("Expected tweets count=%v, got: %v", maxTweetsNbr, count) + } +} diff --git a/timeline_v2.go b/timeline_v2.go index 2ab8124..7ed2cfd 100644 --- a/timeline_v2.go +++ b/timeline_v2.go @@ -134,6 +134,38 @@ func (timeline *timelineV2) parseTweets() ([]*Tweet, string) { return tweets, cursor } +type bookmarksTimelineV2 struct { + Data struct { + Bookmarks struct { + Timeline struct { + Instructions []struct { + Entries []entry `json:"entries"` + Type string `json:"type"` + } `json:"instructions"` + } `json:"timeline"` + } `json:"bookmark_timeline_v2"` + } `json:"data"` +} + +func (timeline *bookmarksTimelineV2) parseTweets() ([]*Tweet, string) { + var cursor string + var tweets []*Tweet + for _, instruction := range timeline.Data.Bookmarks.Timeline.Instructions { + for _, entry := range instruction.Entries { + if entry.Content.CursorType == "Bottom" { + cursor = entry.Content.Value + continue + } + if entry.Content.ItemContent.TweetResults.Result.Typename == "Tweet" { + if tweet := entry.Content.ItemContent.TweetResults.Result.parse(); tweet != nil { + tweets = append(tweets, tweet) + } + } + } + } + return tweets, cursor +} + type threadedConversation struct { Data struct { ThreadedConversationWithInjectionsV2 struct {