lain/twitter-scrapper
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity

Use GraphQL API with timeline v2

Browse source 
Close #85
Close #82
Close #77
Close #76
This commit is contained in:
Alexander Sheiko 2023-06-01 23:20:11 +03:00
parent c07bd1d1d4
commit 50440667ed
7 changed files with 502 additions and 131 deletions
Download patch file Download diff file Expand all files Collapse all files

102
tweets.go
View file

@ -3,7 +3,7 @@ package twitterscraper
import (
"context"
"fmt"
"strconv"
"net/url"
)
// GetTweets returns channel with tweets for a given user.
@ -21,26 +21,59 @@ func (s *Scraper) FetchTweets(user string, maxTweetsNbr int, cursor string) ([]*
return s.FetchTweetsByUserID(userID, maxTweetsNbr, cursor)
}
// FetchTweetsByUserID gets tweets for a given userID, via the Twitter frontend API.
// FetchTweetsByUserID gets tweets for a given userID, via the Twitter frontend GraphQL API.
func (s *Scraper) FetchTweetsByUserID(userID string, maxTweetsNbr int, cursor string) ([]*Tweet, string, error) {
if maxTweetsNbr > 200 {
maxTweetsNbr = 200
}
req, err := s.newRequest("GET", "https://api.twitter.com/2/timeline/profile/"+userID+".json")
req, err := s.newRequest("GET", "https://twitter.com/i/api/graphql/UGi7tjRPr-d_U3bCPIko5Q/UserTweets")
if err != nil {
return nil, "", err
}
q := req.URL.Query()
q.Add("count", strconv.Itoa(maxTweetsNbr))
q.Add("userId", userID)
if cursor != "" {
q.Add("cursor", cursor)
variables := map[string]interface{}{
"userId": userID,
"count": maxTweetsNbr,
"includePromotedContent": false,
"withQuickPromoteEligibilityTweetFields": false,
"withVoice": true,
"withV2Timeline": true,
}
features := map[string]interface{}{
"rweb_lists_timeline_redesign_enabled": true,
"responsive_web_graphql_exclude_directive_enabled": true,
"verified_phone_label_enabled": false,
"creator_subscriptions_tweet_preview_api_enabled": true,
"responsive_web_graphql_timeline_navigation_enabled": true,
"responsive_web_graphql_skip_user_profile_image_extensions_enabled": false,
"tweetypie_unmention_optimization_enabled": true,
"vibe_api_enabled": true,
"responsive_web_edit_tweet_api_enabled": true,
"graphql_is_translatable_rweb_tweet_is_translatable_enabled": true,
"view_counts_everywhere_api_enabled": true,
"longform_notetweets_consumption_enabled": true,
"tweet_awards_web_tipping_enabled": false,
"freedom_of_speech_not_reach_fetch_enabled": true,
"standardized_nudges_misinfo": true,
"tweet_with_visibility_results_prefer_gql_limited_actions_policy_enabled": false,
"interactive_text_enabled": true,
"responsive_web_text_conversations_enabled": false,
"longform_notetweets_rich_text_read_enabled": true,
"longform_notetweets_inline_media_enabled": false,
"responsive_web_enhance_cards_enabled": false,
}
req.URL.RawQuery = q.Encode()
var timeline timeline
if cursor != "" {
variables["cursor"] = cursor
}
query := url.Values{}
query.Set("variables", mapToJSONString(variables))
query.Set("features", mapToJSONString(features))
req.URL.RawQuery = query.Encode()
var timeline timelineV2
err = s.RequestAPI(req, &timeline)
if err != nil {
return nil, "", err
@ -52,18 +85,59 @@ func (s *Scraper) FetchTweetsByUserID(userID string, maxTweetsNbr int, cursor st
// GetTweet get a single tweet by ID.
func (s *Scraper) GetTweet(id string) (*Tweet, error) {
req, err := s.newRequest("GET", "https://api.twitter.com/2/timeline/conversation/"+id+".json")
req, err := s.newRequest("GET", "https://twitter.com/i/api/graphql/wETHelmSuBQR5r-dgUlPxg/TweetDetail")
if err != nil {
return nil, err
}
var timeline timeline
err = s.RequestAPI(req, &timeline)
variables := map[string]interface{}{
"focalTweetId": id,
"referrer": "profile",
"with_rux_injections": false,
"includePromotedContent": true,
"withCommunity": true,
"withQuickPromoteEligibilityTweetFields": true,
"withBirdwatchNotes": true,
"withVoice": true,
"withV2Timeline": true,
}
features := map[string]interface{}{
"rweb_lists_timeline_redesign_enabled": true,
"responsive_web_graphql_exclude_directive_enabled": true,
"verified_phone_label_enabled": false,
"creator_subscriptions_tweet_preview_api_enabled": true,
"responsive_web_graphql_timeline_navigation_enabled": true,
"responsive_web_graphql_skip_user_profile_image_extensions_enabled": false,
"tweetypie_unmention_optimization_enabled": true,
"vibe_api_enabled": true,
"responsive_web_edit_tweet_api_enabled": true,
"graphql_is_translatable_rweb_tweet_is_translatable_enabled": true,
"view_counts_everywhere_api_enabled": true,
"longform_notetweets_consumption_enabled": true,
"tweet_awards_web_tipping_enabled": false,
"freedom_of_speech_not_reach_fetch_enabled": true,
"standardized_nudges_misinfo": true,
"tweet_with_visibility_results_prefer_gql_limited_actions_policy_enabled": false,
"interactive_text_enabled": true,
"responsive_web_text_conversations_enabled": false,
"longform_notetweets_rich_text_read_enabled": true,
"longform_notetweets_inline_media_enabled": false,
"responsive_web_enhance_cards_enabled": false,
}
query := url.Values{}
query.Set("variables", mapToJSONString(variables))
query.Set("features", mapToJSONString(features))
req.URL.RawQuery = query.Encode()
var conversation threadedConversation
err = s.RequestAPI(req, &conversation)
if err != nil {
return nil, err
}
tweets, _ := timeline.parseTweets()
tweets := conversation.parse()
for _, tweet := range tweets {
if tweet.ID == id {
return tweet, nil