lain/twitter-scrapper
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity

feat: x.com migration + public API for cookie-based auth

Browse source 
- migrate all API URLs from twitter.com to x.com
- add SetBearerToken, SetHTTPClient, SetLoggedIn public methods
- accept 202/204 status codes in handleResponse
- module path -> src.cultist.club/lain/twitter-scrapper
This commit is contained in:
Lain Iwakura 2026-05-21 18:54:10 +03:00
parent 76cb95cd3e
commit 34db837a9e
Signed by: lain
GPG key ID: 8160466B2E8D1441
19 changed files with 75 additions and 63 deletions
Download patch file Download diff file Expand all files Collapse all files

4
account.go
View file

@ -46,7 +46,7 @@ type AccountList struct {
func (s *Scraper) GetAccountSettings() (AccountSettings, error) { func (s *Scraper) GetAccountSettings() (AccountSettings, error) {
var settings AccountSettings var settings AccountSettings
req, err := s.newRequest("GET", "https://api.twitter.com/1.1/account/settings.json") req, err := s.newRequest("GET", "https://api.x.com/1.1/account/settings.json")
if err != nil { if err != nil {
return settings, err return settings, err
} }
@ -57,7 +57,7 @@ func (s *Scraper) GetAccountSettings() (AccountSettings, error) {
func (s *Scraper) GetAccountList() ([]Account, error) { func (s *Scraper) GetAccountList() ([]Account, error) {
var list AccountList var list AccountList
req, err := s.newRequest("GET", "https://api.twitter.com/1.1/account/multi/list.json") req, err := s.newRequest("GET", "https://api.x.com/1.1/account/multi/list.json")
if err != nil { if err != nil {
return list.Users, err return list.Users, err
} }

6
api.go
View file

@ -86,7 +86,7 @@ func (s *Scraper) handleResponse(resp *http.Response, target interface{}) error
return err return err
} }
if resp.StatusCode != http.StatusOK { if resp.StatusCode != http.StatusOK && resp.StatusCode != http.StatusAccepted && resp.StatusCode != http.StatusNoContent {
return fmt.Errorf("response status %s: %s", resp.Status, content) return fmt.Errorf("response status %s: %s", resp.Status, content)
} }
@ -103,7 +103,7 @@ func (s *Scraper) handleResponse(resp *http.Response, target interface{}) error
// GetGuestToken from Twitter API // GetGuestToken from Twitter API
func (s *Scraper) GetGuestToken() error { func (s *Scraper) GetGuestToken() error {
req, err := http.NewRequest("POST", "https://api.twitter.com/1.1/guest/activate.json", nil) req, err := http.NewRequest("POST", "https://api.x.com/1.1/guest/activate.json", nil)
if err != nil { if err != nil {
return err return err
} }
@ -119,7 +119,7 @@ func (s *Scraper) GetGuestToken() error {
if err != nil { if err != nil {
return err return err
} }
if resp.StatusCode != http.StatusOK { if resp.StatusCode != http.StatusOK && resp.StatusCode != http.StatusAccepted && resp.StatusCode != http.StatusNoContent {
return fmt.Errorf("response status %s: %s", resp.Status, body) return fmt.Errorf("response status %s: %s", resp.Status, body)
} }

12
auth.go
View file

@ -19,9 +19,9 @@ import (
) )
const ( const (
loginURL = "https://api.twitter.com/1.1/onboarding/task.json" loginURL = "https://api.x.com/1.1/onboarding/task.json"
logoutURL = "https://api.twitter.com/1.1/account/logout.json" logoutURL = "https://api.x.com/1.1/account/logout.json"
oAuthURL = "https://api.twitter.com/oauth2/token" oAuthURL = "https://api.x.com/oauth2/token"
// Doesn't require x-client-transaction-id header in auth. x-rate-limit-limit: 2000 // Doesn't require x-client-transaction-id header in auth. x-rate-limit-limit: 2000
bearerToken1 = "AAAAAAAAAAAAAAAAAAAAAFQODgEAAAAAVHTp76lzh3rFzcHbmHVvQxYYpTw%3DckAlMINMjmCwxUcaXbAN4XqJVdgMJaHqNOFgPMK0zN1qLqLQCF" bearerToken1 = "AAAAAAAAAAAAAAAAAAAAAFQODgEAAAAAVHTp76lzh3rFzcHbmHVvQxYYpTw%3DckAlMINMjmCwxUcaXbAN4XqJVdgMJaHqNOFgPMK0zN1qLqLQCF"
// HOTFIX: Returns 404 error; Requires x-client-transaction-id header in auth. // HOTFIX: Returns 404 error; Requires x-client-transaction-id header in auth.
@ -152,7 +152,7 @@ func (s *Scraper) getFlowToken(data map[string]interface{}) (string, error) {
func (s *Scraper) IsLoggedIn() bool { func (s *Scraper) IsLoggedIn() bool {
s.isLogged = true s.isLogged = true
s.setBearerToken(bearerToken1) s.setBearerToken(bearerToken1)
req, err := http.NewRequest("GET", "https://api.twitter.com/1.1/account/verify_credentials.json", nil) req, err := http.NewRequest("GET", "https://api.x.com/1.1/account/verify_credentials.json", nil)
if err != nil { if err != nil {
return false return false
} }
@ -444,7 +444,7 @@ func (s *Scraper) SetAuthToken(token AuthToken) {
Name: "auth_token", Name: "auth_token",
Value: token.Token, Value: token.Token,
Path: "", Path: "",
Domain: "twitter.com", Domain: "x.com",
Expires: expires, Expires: expires,
RawExpires: "", RawExpires: "",
MaxAge: 0, MaxAge: 0,
@ -457,7 +457,7 @@ func (s *Scraper) SetAuthToken(token AuthToken) {
Name: "ct0", Name: "ct0",
Value: token.CSRFToken, Value: token.CSRFToken,
Path: "", Path: "",
Domain: "twitter.com", Domain: "x.com",
Expires: expires, Expires: expires,
RawExpires: "", RawExpires: "",
MaxAge: 0, MaxAge: 0,

2
bookmarks.go
View file

@ -18,7 +18,7 @@ func (s *Scraper) FetchBookmarks(maxTweetsNbr int, cursor string) ([]*Tweet, str
maxTweetsNbr = 200 maxTweetsNbr = 200
} }
req, err := s.newRequest("GET", "https://twitter.com/i/api/graphql/-IyJFt9_jS_9d_vS3NN-fA/Bookmarks") req, err := s.newRequest("GET", "https://x.com/i/api/graphql/-IyJFt9_jS_9d_vS3NN-fA/Bookmarks")
if err != nil { if err != nil {
return nil, "", err return nil, "", err
} }

4
follows.go
View file

@ -21,7 +21,7 @@ func (s *Scraper) FetchFollowingByUserID(userID string, maxUsersNbr int, cursor
maxUsersNbr = 200 maxUsersNbr = 200
} }
req, err := s.newRequest("GET", "https://twitter.com/i/api/graphql/g5P4cbXR4ta4oCeE7y2vLQ/Following") req, err := s.newRequest("GET", "https://x.com/i/api/graphql/g5P4cbXR4ta4oCeE7y2vLQ/Following")
if err != nil { if err != nil {
return nil, "", err return nil, "", err
} }
@ -94,7 +94,7 @@ func (s *Scraper) FetchFollowersByUserID(userID string, maxUsersNbr int, cursor
maxUsersNbr = 200 maxUsersNbr = 200
} }
req, err := s.newRequest("GET", "https://twitter.com/i/api/graphql/jwbfbSzn0FRL_AMZGsYDag/Followers") req, err := s.newRequest("GET", "https://x.com/i/api/graphql/jwbfbSzn0FRL_AMZGsYDag/Followers")
if err != nil { if err != nil {
return nil, "", err return nil, "", err
} }

2
go.mod
View file

@ -1,4 +1,4 @@
module github.com/imperatrona/twitter-scraper module src.cultist.club/lain/twitter-scrapper
go 1.16 go 1.16

2
medias.go
View file

@ -26,7 +26,7 @@ func (s *Scraper) FetchMediaTweetsByUserID(userID string, maxTweetsNbr int, curs
maxTweetsNbr = 200 maxTweetsNbr = 200
} }
req, err := s.newRequest("GET", "https://twitter.com/i/api/graphql/2tLOJWwGuCTytDrGBg8VwQ/UserMedia") req, err := s.newRequest("GET", "https://x.com/i/api/graphql/2tLOJWwGuCTytDrGBg8VwQ/UserMedia")
if err != nil { if err != nil {
return nil, "", err return nil, "", err
} }

4
profile.go
View file

@ -65,7 +65,7 @@ type user struct {
// GetProfile return parsed user profile. // GetProfile return parsed user profile.
func (s *Scraper) GetProfile(username string) (Profile, error) { func (s *Scraper) GetProfile(username string) (Profile, error) {
var jsn user var jsn user
req, err := http.NewRequest("GET", "https://api.twitter.com/graphql/Yka-W8dz7RaEuQNkroPkYw/UserByScreenName", nil) req, err := http.NewRequest("GET", "https://api.x.com/graphql/Yka-W8dz7RaEuQNkroPkYw/UserByScreenName", nil)
if err != nil { if err != nil {
return Profile{}, err return Profile{}, err
} }
@ -126,7 +126,7 @@ func (s *Scraper) GetProfile(username string) (Profile, error) {
func (s *Scraper) GetProfileByID(userID string) (Profile, error) { func (s *Scraper) GetProfileByID(userID string) (Profile, error) {
var jsn user var jsn user
req, err := http.NewRequest("GET", "https://twitter.com/i/api/graphql/Qw77dDjp9xCpUY-AXwt-yQ/UserByRestId", nil) req, err := http.NewRequest("GET", "https://x.com/i/api/graphql/Qw77dDjp9xCpUY-AXwt-yQ/UserByRestId", nil)
if err != nil { if err != nil {
return Profile{}, err return Profile{}, err
} }

2
replies.go
View file

@ -10,7 +10,7 @@ type ThreadCursor struct {
} }
func (s *Scraper) GetTweetReplies(id string, cursor string) ([]*Tweet, []*ThreadCursor, error) { func (s *Scraper) GetTweetReplies(id string, cursor string) ([]*Tweet, []*ThreadCursor, error) {
req, err := s.newRequest("GET", "https://twitter.com/i/api/graphql/ldqoq5MmFHN1FhMGvzC9Jg/TweetDetail") req, err := s.newRequest("GET", "https://x.com/i/api/graphql/ldqoq5MmFHN1FhMGvzC9Jg/TweetDetail")
if err != nil { if err != nil {
return nil, nil, err return nil, nil, err
} }

6
schedule.go
View file

@ -129,7 +129,7 @@ func (timeline *scheduleTweets) parseTweets() []*ScheduledTweet {
// FetchScheduledTweets gets scheduled tweets via the Twitter frontend GraphQL API. // FetchScheduledTweets gets scheduled tweets via the Twitter frontend GraphQL API.
func (s *Scraper) FetchScheduledTweets() ([]*ScheduledTweet, error) { func (s *Scraper) FetchScheduledTweets() ([]*ScheduledTweet, error) {
req, err := s.newRequest("GET", "https://twitter.com/i/api/graphql/ITtjAzvlZni2wWXwf295Qg/FetchScheduledTweets") req, err := s.newRequest("GET", "https://x.com/i/api/graphql/ITtjAzvlZni2wWXwf295Qg/FetchScheduledTweets")
if err != nil { if err != nil {
return nil, err return nil, err
} }
@ -154,7 +154,7 @@ func (s *Scraper) FetchScheduledTweets() ([]*ScheduledTweet, error) {
// DeleteScheduledTweet removes tweet from scheduled. // DeleteScheduledTweet removes tweet from scheduled.
func (s *Scraper) DeleteScheduledTweet(id string) error { func (s *Scraper) DeleteScheduledTweet(id string) error {
req, err := s.newRequest("POST", "https://twitter.com/i/api/graphql/CTOVqej0JBXAZSwkp1US0g/DeleteScheduledTweet") req, err := s.newRequest("POST", "https://x.com/i/api/graphql/CTOVqej0JBXAZSwkp1US0g/DeleteScheduledTweet")
if err != nil { if err != nil {
return err return err
} }
@ -197,7 +197,7 @@ func (s *Scraper) CreateScheduledTweet(schedule TweetSchedule) (string, error) {
return "", errors.New("date can't be in past") return "", errors.New("date can't be in past")
} }
req, err := s.newRequest("POST", "https://twitter.com/i/api/graphql/LCVzRQGxOaGnOnYH01NQXg/CreateScheduledTweet") req, err := s.newRequest("POST", "https://x.com/i/api/graphql/LCVzRQGxOaGnOnYH01NQXg/CreateScheduledTweet")
if err != nil { if err != nil {
return "", err return "", err
} }

12
scraper.go
View file

@ -180,3 +180,15 @@ func (s *Scraper) SetUserAgent(userAgent string) {
func (s *Scraper) GetUserAgent() string { func (s *Scraper) GetUserAgent() string {
return s.userAgent return s.userAgent
} }
func (s *Scraper) SetBearerToken(token string) {
s.setBearerToken(token)
}
func (s *Scraper) SetHTTPClient(client *http.Client) {
s.client = client
}
func (s *Scraper) SetLoggedIn(v bool) {
s.isLogged = v
}

2
search.go
View file

@ -7,7 +7,7 @@ import (
"strconv" "strconv"
) )
const searchURL = "https://twitter.com/i/api/graphql/nK1dw4oV3k4w5TdtcAdSww/SearchTimeline" const searchURL = "https://x.com/i/api/graphql/nK1dw4oV3k4w5TdtcAdSww/SearchTimeline"
type searchTimeline struct { type searchTimeline struct {
Data struct { Data struct {

2
spaces.go
View file

@ -11,7 +11,7 @@ func (s *Scraper) GetSpace(id string) (*Space, error) {
return nil, errors.New("scraper is not logged in") return nil, errors.New("scraper is not logged in")
} }
req, err := s.newRequest("GET", "https://twitter.com/i/api/graphql/d03OdorPdZ_sH9V3D1_yWQ/AudioSpaceById") req, err := s.newRequest("GET", "https://x.com/i/api/graphql/d03OdorPdZ_sH9V3D1_yWQ/AudioSpaceById")
if err != nil { if err != nil {
return nil, err return nil, err
} }

6
timeline_v1.go
View file

@ -92,7 +92,7 @@ func (timeline *timelineV1) parseTweet(id string) *Tweet {
ConversationID: tweet.ConversationIDStr, ConversationID: tweet.ConversationIDStr,
Likes: tweet.FavoriteCount, Likes: tweet.FavoriteCount,
Name: name, Name: name,
PermanentURL: fmt.Sprintf("https://twitter.com/%s/status/%s", username, id), PermanentURL: fmt.Sprintf("https://x.com/%s/status/%s", username, id),
Replies: tweet.ReplyCount, Replies: tweet.ReplyCount,
Retweets: tweet.RetweetCount, Retweets: tweet.RetweetCount,
Text: tweet.FullText, Text: tweet.FullText,
@ -194,13 +194,13 @@ func (timeline *timelineV1) parseTweet(id string) *Tweet {
tw.HTML = tweet.FullText tw.HTML = tweet.FullText
tw.HTML = reHashtag.ReplaceAllStringFunc(tw.HTML, func(hashtag string) string { tw.HTML = reHashtag.ReplaceAllStringFunc(tw.HTML, func(hashtag string) string {
return fmt.Sprintf(`<a href="https://twitter.com/hashtag/%s">%s</a>`, return fmt.Sprintf(`<a href="https://x.com/hashtag/%s">%s</a>`,
strings.TrimPrefix(hashtag, "#"), strings.TrimPrefix(hashtag, "#"),
hashtag, hashtag,
) )
}) })
tw.HTML = reUsername.ReplaceAllStringFunc(tw.HTML, func(username string) string { tw.HTML = reUsername.ReplaceAllStringFunc(tw.HTML, func(username string) string {
return fmt.Sprintf(`<a href="https://twitter.com/%s">%s</a>`, return fmt.Sprintf(`<a href="https://x.com/%s">%s</a>`,
strings.TrimPrefix(username, "@"), strings.TrimPrefix(username, "@"),
username, username,
) )

2
trends.go
View file

@ -4,7 +4,7 @@ import "fmt"
// GetTrends return list of trends. // GetTrends return list of trends.
func (s *Scraper) GetTrends() ([]string, error) { func (s *Scraper) GetTrends() ([]string, error) {
req, err := s.newRequest("GET", "https://api.twitter.com/2/guide.json") req, err := s.newRequest("GET", "https://api.x.com/2/guide.json")
if err != nil { if err != nil {
return nil, err return nil, err
} }

14
tweet.go
View file

@ -47,7 +47,7 @@ func (newTweet *newTweet) parse() *Tweet {
} }
func (s *Scraper) CreateTweet(tweet NewTweet) (*Tweet, error) { func (s *Scraper) CreateTweet(tweet NewTweet) (*Tweet, error) {
req, err := s.newRequest("POST", "https://twitter.com/i/api/graphql/oB-5XsHNAbjvARJEc8CZFw/CreateTweet") req, err := s.newRequest("POST", "https://x.com/i/api/graphql/oB-5XsHNAbjvARJEc8CZFw/CreateTweet")
if err != nil { if err != nil {
return nil, err return nil, err
} }
@ -126,7 +126,7 @@ func (s *Scraper) CreateTweet(tweet NewTweet) (*Tweet, error) {
} }
func (s *Scraper) DeleteTweet(tweetId string) error { func (s *Scraper) DeleteTweet(tweetId string) error {
req, err := s.newRequest("POST", "https://twitter.com/i/api/graphql/VaenaVgh5q5ih7kvyVjgtg/DeleteTweet") req, err := s.newRequest("POST", "https://x.com/i/api/graphql/VaenaVgh5q5ih7kvyVjgtg/DeleteTweet")
if err != nil { if err != nil {
return err return err
} }
@ -163,7 +163,7 @@ func (s *Scraper) DeleteTweet(tweetId string) error {
} }
func (s *Scraper) CreateRetweet(tweetId string) (string, error) { func (s *Scraper) CreateRetweet(tweetId string) (string, error) {
req, err := s.newRequest("POST", "https://twitter.com/i/api/graphql/ojPdsZsimiJrUGLR1sjUtA/CreateRetweet") req, err := s.newRequest("POST", "https://x.com/i/api/graphql/ojPdsZsimiJrUGLR1sjUtA/CreateRetweet")
if err != nil { if err != nil {
return "", err return "", err
} }
@ -211,7 +211,7 @@ func (s *Scraper) CreateRetweet(tweetId string) (string, error) {
// Retweeted tweets has their own id, but to delete retweet twitter using id of source tweet // Retweeted tweets has their own id, but to delete retweet twitter using id of source tweet
func (s *Scraper) DeleteRetweet(tweetId string) error { func (s *Scraper) DeleteRetweet(tweetId string) error {
req, err := s.newRequest("POST", "https://twitter.com/i/api/graphql/iQtK4dl5hBmXewYZuEOKVw/DeleteRetweet") req, err := s.newRequest("POST", "https://x.com/i/api/graphql/iQtK4dl5hBmXewYZuEOKVw/DeleteRetweet")
if err != nil { if err != nil {
return err return err
} }
@ -253,7 +253,7 @@ func (s *Scraper) DeleteRetweet(tweetId string) error {
} }
func (s *Scraper) LikeTweet(tweetId string) error { func (s *Scraper) LikeTweet(tweetId string) error {
req, err := s.newRequest("POST", "https://twitter.com/i/api/graphql/lI07N6Otwv1PhnEgXILM7A/FavoriteTweet") req, err := s.newRequest("POST", "https://x.com/i/api/graphql/lI07N6Otwv1PhnEgXILM7A/FavoriteTweet")
if err != nil { if err != nil {
return err return err
} }
@ -297,7 +297,7 @@ func (s *Scraper) LikeTweet(tweetId string) error {
} }
func (s *Scraper) UnlikeTweet(tweetId string) error { func (s *Scraper) UnlikeTweet(tweetId string) error {
req, err := s.newRequest("POST", "https://twitter.com/i/api/graphql/ZYKSe-w7KEslx3JhSIk5LA/UnfavoriteTweet") req, err := s.newRequest("POST", "https://x.com/i/api/graphql/ZYKSe-w7KEslx3JhSIk5LA/UnfavoriteTweet")
if err != nil { if err != nil {
return err return err
} }
@ -344,7 +344,7 @@ func (s *Scraper) GetTweetRetweeters(tweetId string, maxUsersNbr int, cursor str
maxUsersNbr = 200 maxUsersNbr = 200
} }
req, err := s.newRequest("GET", "https://twitter.com/i/api/graphql/8019obfgnveiPiJuS2Rtow/Retweeters") req, err := s.newRequest("GET", "https://x.com/i/api/graphql/8019obfgnveiPiJuS2Rtow/Retweeters")
if err != nil { if err != nil {
return nil, "", err return nil, "", err
} }

20
tweets.go
View file

@ -46,7 +46,7 @@ func (s *Scraper) FetchTweetsAndRepliesByUserID(userID string, maxReplysNbr int,
maxReplysNbr = 200 maxReplysNbr = 200
} }
req, err := s.newRequest("GET", "https://twitter.com/i/api/graphql/bt4TKuFz4T7Ckk-VvQVSow/UserTweetsAndReplies") req, err := s.newRequest("GET", "https://x.com/i/api/graphql/bt4TKuFz4T7Ckk-VvQVSow/UserTweetsAndReplies")
if err != nil { if err != nil {
return nil, "", err return nil, "", err
} }
@ -111,7 +111,7 @@ func (s *Scraper) FetchTweetsByUserID(userID string, maxTweetsNbr int, cursor st
maxTweetsNbr = 200 maxTweetsNbr = 200
} }
req, err := s.newRequest("GET", "https://twitter.com/i/api/graphql/UGi7tjRPr-d_U3bCPIko5Q/UserTweets") req, err := s.newRequest("GET", "https://x.com/i/api/graphql/UGi7tjRPr-d_U3bCPIko5Q/UserTweets")
if err != nil { if err != nil {
return nil, "", err return nil, "", err
} }
@ -173,7 +173,7 @@ func (s *Scraper) FetchTweetsByUserIDLegacy(userID string, maxTweetsNbr int, cur
maxTweetsNbr = 200 maxTweetsNbr = 200
} }
req, err := s.newRequest("GET", "https://api.twitter.com/2/timeline/profile/"+userID+".json") req, err := s.newRequest("GET", "https://api.x.com/2/timeline/profile/"+userID+".json")
if err != nil { if err != nil {
return nil, "", err return nil, "", err
} }
@ -199,7 +199,7 @@ func (s *Scraper) FetchTweetsByUserIDLegacy(userID string, maxTweetsNbr int, cur
// GetTweet get a single tweet by ID. // GetTweet get a single tweet by ID.
func (s *Scraper) GetTweet(id string) (*Tweet, error) { func (s *Scraper) GetTweet(id string) (*Tweet, error) {
if s.isOpenAccount { if s.isOpenAccount {
req, err := s.newRequest("GET", "https://api.twitter.com/2/timeline/conversation/"+id+".json") req, err := s.newRequest("GET", "https://api.x.com/2/timeline/conversation/"+id+".json")
if err != nil { if err != nil {
return nil, err return nil, err
} }
@ -217,7 +217,7 @@ func (s *Scraper) GetTweet(id string) (*Tweet, error) {
} }
} }
} else if s.isLogged { } else if s.isLogged {
req, err := s.newRequest("GET", "https://twitter.com/i/api/graphql/VWFGPVAGkZMGRKGe3GFFnA/TweetDetail") req, err := s.newRequest("GET", "https://x.com/i/api/graphql/VWFGPVAGkZMGRKGe3GFFnA/TweetDetail")
if err != nil { if err != nil {
return nil, err return nil, err
} }
@ -263,7 +263,7 @@ func (s *Scraper) GetTweet(id string) (*Tweet, error) {
// Surprisingly, if bearerToken2 is not set, then animated GIFs are not // Surprisingly, if bearerToken2 is not set, then animated GIFs are not
// present in the response for tweets with a GIF + a photo like this one: // present in the response for tweets with a GIF + a photo like this one:
// https://twitter.com/Twitter/status/1580661436132757506 // https://x.com/Twitter/status/1580661436132757506
curBearerToken := s.bearerToken curBearerToken := s.bearerToken
if curBearerToken != bearerToken2 { if curBearerToken != bearerToken2 {
s.setBearerToken(bearerToken2) s.setBearerToken(bearerToken2)
@ -286,7 +286,7 @@ func (s *Scraper) GetTweet(id string) (*Tweet, error) {
} }
} }
} else { } else {
req, err := s.newRequest("GET", "https://twitter.com/i/api/graphql/xBtHv5-Xsk268T5ng_OGNg/TweetResultByRestId") req, err := s.newRequest("GET", "https://x.com/i/api/graphql/xBtHv5-Xsk268T5ng_OGNg/TweetResultByRestId")
if err != nil { if err != nil {
return nil, err return nil, err
} }
@ -332,7 +332,7 @@ func (s *Scraper) GetTweet(id string) (*Tweet, error) {
// Surprisingly, if bearerToken2 is not set, then animated GIFs are not // Surprisingly, if bearerToken2 is not set, then animated GIFs are not
// present in the response for tweets with a GIF + a photo like this one: // present in the response for tweets with a GIF + a photo like this one:
// https://twitter.com/Twitter/status/1580661436132757506 // https://x.com/Twitter/status/1580661436132757506
curBearerToken := s.bearerToken curBearerToken := s.bearerToken
if curBearerToken != bearerToken2 { if curBearerToken != bearerToken2 {
s.setBearerToken(bearerToken2) s.setBearerToken(bearerToken2)
@ -421,7 +421,7 @@ func (s *Scraper) fetchHomeTweets(_ string, maxTweetsNbr int, cursor string) ([]
maxTweetsNbr = 200 maxTweetsNbr = 200
} }
req, err := s.newRequest("GET", "https://twitter.com/i/api/graphql/9EwYy8pLBOSFlEoSP2STiQ/HomeLatestTimeline") req, err := s.newRequest("GET", "https://x.com/i/api/graphql/9EwYy8pLBOSFlEoSP2STiQ/HomeLatestTimeline")
if err != nil { if err != nil {
return nil, "", err return nil, "", err
} }
@ -496,7 +496,7 @@ func (s *Scraper) fetchForYouTweets(_ string, maxTweetsNbr int, cursor string) (
maxTweetsNbr = 200 maxTweetsNbr = 200
} }
req, err := s.newRequest("GET", "https://twitter.com/i/api/graphql/1u0Wlkw6Ru1NwBUD-pDiww/HomeTimeline") req, err := s.newRequest("GET", "https://x.com/i/api/graphql/1u0Wlkw6Ru1NwBUD-pDiww/HomeTimeline")
if err != nil { if err != nil {
return nil, "", err return nil, "", err
} }

24
upload.go
View file

@ -104,7 +104,7 @@ func (s *Scraper) uploadInit(filePath string, fileContent []byte) (*Media, error
return nil, fmt.Errorf("file type %s unsupported by twitter, make sure you uploading photo, video or gif", fileType) return nil, fmt.Errorf("file type %s unsupported by twitter, make sure you uploading photo, video or gif", fileType)
} }
req, err := s.newRequest("POST", "https://upload.twitter.com/i/media/upload.json") req, err := s.newRequest("POST", "https://upload.x.com/i/media/upload.json")
if err != nil { if err != nil {
return nil, err return nil, err
} }
@ -118,8 +118,8 @@ func (s *Scraper) uploadInit(filePath string, fileContent []byte) (*Media, error
query.Set("video_duration_ms", strconv.FormatFloat(videoDuration*1000, 'f', -1, 64)) query.Set("video_duration_ms", strconv.FormatFloat(videoDuration*1000, 'f', -1, 64))
} }
req.URL.RawQuery = query.Encode() req.URL.RawQuery = query.Encode()
req.Header.Set("Origin", "https://twitter.com") req.Header.Set("Origin", "https://x.com")
req.Header.Set("Referer", "https://twitter.com/") req.Header.Set("Referer", "https://x.com/")
var uploadInit uploadInitResponse var uploadInit uploadInitResponse
@ -158,7 +158,7 @@ func (s *Scraper) uploadAppend(media *Media, fileContent []byte) error {
} }
w.Close() w.Close()
req, err := s.newRequest("POST", "https://upload.twitter.com/i/media/upload.json") req, err := s.newRequest("POST", "https://upload.x.com/i/media/upload.json")
if err != nil { if err != nil {
return err return err
} }
@ -169,8 +169,8 @@ func (s *Scraper) uploadAppend(media *Media, fileContent []byte) error {
query.Set("segment_index", strconv.Itoa(i)) query.Set("segment_index", strconv.Itoa(i))
req.URL.RawQuery = query.Encode() req.URL.RawQuery = query.Encode()
req.Header.Set("Content-Type", w.FormDataContentType()) req.Header.Set("Content-Type", w.FormDataContentType())
req.Header.Set("Origin", "https://twitter.com") req.Header.Set("Origin", "https://x.com")
req.Header.Set("Referer", "https://twitter.com/") req.Header.Set("Referer", "https://x.com/")
req.Body = io.NopCloser(&buf) req.Body = io.NopCloser(&buf)
err = s.RequestAPI(req, nil) err = s.RequestAPI(req, nil)
@ -183,7 +183,7 @@ func (s *Scraper) uploadAppend(media *Media, fileContent []byte) error {
} }
func (s *Scraper) uploadFinalize(media *Media) (*ProcessingInfo, error) { func (s *Scraper) uploadFinalize(media *Media) (*ProcessingInfo, error) {
req, err := s.newRequest("POST", "https://upload.twitter.com/i/media/upload.json") req, err := s.newRequest("POST", "https://upload.x.com/i/media/upload.json")
if err != nil { if err != nil {
return nil, err return nil, err
} }
@ -193,8 +193,8 @@ func (s *Scraper) uploadFinalize(media *Media) (*ProcessingInfo, error) {
query.Set("media_id", strconv.Itoa(media.ID)) query.Set("media_id", strconv.Itoa(media.ID))
query.Set("allow_async", "true") query.Set("allow_async", "true")
req.URL.RawQuery = query.Encode() req.URL.RawQuery = query.Encode()
req.Header.Set("Origin", "https://twitter.com") req.Header.Set("Origin", "https://x.com")
req.Header.Set("Referer", "https://twitter.com/") req.Header.Set("Referer", "https://x.com/")
var response uploadStatusResponse var response uploadStatusResponse
@ -207,7 +207,7 @@ func (s *Scraper) uploadFinalize(media *Media) (*ProcessingInfo, error) {
} }
func (s *Scraper) uploadStatus(media *Media) (*ProcessingInfo, error) { func (s *Scraper) uploadStatus(media *Media) (*ProcessingInfo, error) {
req, err := s.newRequest("GET", "https://upload.twitter.com/i/media/upload.json") req, err := s.newRequest("GET", "https://upload.x.com/i/media/upload.json")
if err != nil { if err != nil {
return nil, err return nil, err
} }
@ -216,8 +216,8 @@ func (s *Scraper) uploadStatus(media *Media) (*ProcessingInfo, error) {
query.Set("command", "STATUS") query.Set("command", "STATUS")
query.Set("media_id", strconv.Itoa(media.ID)) query.Set("media_id", strconv.Itoa(media.ID))
req.URL.RawQuery = query.Encode() req.URL.RawQuery = query.Encode()
req.Header.Set("Origin", "https://twitter.com") req.Header.Set("Origin", "https://x.com")
req.Header.Set("Referer", "https://twitter.com/") req.Header.Set("Referer", "https://x.com/")
var response uploadStatusResponse var response uploadStatusResponse

12
util.go
View file

@ -16,7 +16,7 @@ var (
reHashtag = regexp.MustCompile(`\B(\#\S+\b)`) reHashtag = regexp.MustCompile(`\B(\#\S+\b)`)
reTwitterURL = regexp.MustCompile(`https:(\/\/t\.co\/([A-Za-z0-9]|[A-Za-z]){10})`) reTwitterURL = regexp.MustCompile(`https:(\/\/t\.co\/([A-Za-z0-9]|[A-Za-z]){10})`)
reUsername = regexp.MustCompile(`\B(\@\S{1,15}\b)`) reUsername = regexp.MustCompile(`\B(\@\S{1,15}\b)`)
twURL = urlParse("https://twitter.com") twURL = urlParse("https://x.com")
) )
func (s *Scraper) newRequest(method string, url string) (*http.Request, error) { func (s *Scraper) newRequest(method string, url string) (*http.Request, error) {
@ -165,7 +165,7 @@ func parseLegacyTweet(user *legacyUser, tweet *legacyTweet) *Tweet {
ID: tweetID, ID: tweetID,
Likes: tweet.FavoriteCount, Likes: tweet.FavoriteCount,
Name: name, Name: name,
PermanentURL: fmt.Sprintf("https://twitter.com/%s/status/%s", username, tweetID), PermanentURL: fmt.Sprintf("https://x.com/%s/status/%s", username, tweetID),
Replies: tweet.ReplyCount, Replies: tweet.ReplyCount,
Retweets: tweet.RetweetCount, Retweets: tweet.RetweetCount,
Text: text, Text: text,
@ -293,13 +293,13 @@ func parseLegacyTweet(user *legacyUser, tweet *legacyTweet) *Tweet {
tw.HTML = tweet.FullText tw.HTML = tweet.FullText
tw.HTML = reHashtag.ReplaceAllStringFunc(tw.HTML, func(hashtag string) string { tw.HTML = reHashtag.ReplaceAllStringFunc(tw.HTML, func(hashtag string) string {
return fmt.Sprintf(`<a href="https://twitter.com/hashtag/%s">%s</a>`, return fmt.Sprintf(`<a href="https://x.com/hashtag/%s">%s</a>`,
strings.TrimPrefix(hashtag, "#"), strings.TrimPrefix(hashtag, "#"),
hashtag, hashtag,
) )
}) })
tw.HTML = reUsername.ReplaceAllStringFunc(tw.HTML, func(username string) string { tw.HTML = reUsername.ReplaceAllStringFunc(tw.HTML, func(username string) string {
return fmt.Sprintf(`<a href="https://twitter.com/%s">%s</a>`, return fmt.Sprintf(`<a href="https://x.com/%s">%s</a>`,
strings.TrimPrefix(username, "@"), strings.TrimPrefix(username, "@"),
username, username,
) )
@ -360,7 +360,7 @@ func parseProfile(user legacyUser) Profile {
Name: user.Name, Name: user.Name,
PinnedTweetIDs: user.PinnedTweetIdsStr, PinnedTweetIDs: user.PinnedTweetIdsStr,
TweetsCount: user.StatusesCount, TweetsCount: user.StatusesCount,
URL: "https://twitter.com/" + user.ScreenName, URL: "https://x.com/" + user.ScreenName,
UserID: user.IDStr, UserID: user.IDStr,
Username: user.ScreenName, Username: user.ScreenName,
FollowedBy: user.FollowedBy, FollowedBy: user.FollowedBy,
@ -415,7 +415,7 @@ func parseProfileV2(user userResult) Profile {
Name: u.Name, Name: u.Name,
PinnedTweetIDs: u.PinnedTweetIdsStr, PinnedTweetIDs: u.PinnedTweetIdsStr,
TweetsCount: u.StatusesCount, TweetsCount: u.StatusesCount,
URL: "https://twitter.com/" + u.ScreenName, URL: "https://x.com/" + u.ScreenName,
UserID: user.ID, UserID: user.ID,
Username: u.ScreenName, Username: u.ScreenName,
Sensitive: u.PossiblySensitive, Sensitive: u.PossiblySensitive,