feat: x.com migration + public API for cookie-based auth
- migrate all API URLs from twitter.com to x.com - add SetBearerToken, SetHTTPClient, SetLoggedIn public methods - accept 202/204 status codes in handleResponse - module path -> src.cultist.club/lain/twitter-scrapper
This commit is contained in:
parent
76cb95cd3e
commit
34db837a9e
GPG key ID:
8160466B2E8D1441
19 changed files with 75 additions and 63 deletions
|
|
@ -46,7 +46,7 @@ type AccountList struct {
|
|||
|
||||
func (s *Scraper) GetAccountSettings() (AccountSettings, error) {
|
||||
var settings AccountSettings
|
||||
req, err := s.newRequest("GET", "https://api.twitter.com/1.1/account/settings.json")
|
||||
req, err := s.newRequest("GET", "https://api.x.com/1.1/account/settings.json")
|
||||
if err != nil {
|
||||
return settings, err
|
||||
}
|
||||
|
|
@ -57,7 +57,7 @@ func (s *Scraper) GetAccountSettings() (AccountSettings, error) {
|
|||
|
||||
func (s *Scraper) GetAccountList() ([]Account, error) {
|
||||
var list AccountList
|
||||
req, err := s.newRequest("GET", "https://api.twitter.com/1.1/account/multi/list.json")
|
||||
req, err := s.newRequest("GET", "https://api.x.com/1.1/account/multi/list.json")
|
||||
if err != nil {
|
||||
return list.Users, err
|
||||
}
|
||||
|
|
|
|||
6
api.go
6
api.go
|
|
@ -86,7 +86,7 @@ func (s *Scraper) handleResponse(resp *http.Response, target interface{}) error
|
|||
return err
|
||||
}
|
||||
|
||||
if resp.StatusCode != http.StatusOK {
|
||||
if resp.StatusCode != http.StatusOK && resp.StatusCode != http.StatusAccepted && resp.StatusCode != http.StatusNoContent {
|
||||
return fmt.Errorf("response status %s: %s", resp.Status, content)
|
||||
}
|
||||
|
||||
|
|
@ -103,7 +103,7 @@ func (s *Scraper) handleResponse(resp *http.Response, target interface{}) error
|
|||
|
||||
// GetGuestToken from Twitter API
|
||||
func (s *Scraper) GetGuestToken() error {
|
||||
req, err := http.NewRequest("POST", "https://api.twitter.com/1.1/guest/activate.json", nil)
|
||||
req, err := http.NewRequest("POST", "https://api.x.com/1.1/guest/activate.json", nil)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
|
@ -119,7 +119,7 @@ func (s *Scraper) GetGuestToken() error {
|
|||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if resp.StatusCode != http.StatusOK {
|
||||
if resp.StatusCode != http.StatusOK && resp.StatusCode != http.StatusAccepted && resp.StatusCode != http.StatusNoContent {
|
||||
return fmt.Errorf("response status %s: %s", resp.Status, body)
|
||||
}
|
||||
|
||||
|
|
|
|||
12
auth.go
12
auth.go
|
|
@ -19,9 +19,9 @@ import (
|
|||
)
|
||||
|
||||
const (
|
||||
loginURL = "https://api.twitter.com/1.1/onboarding/task.json"
|
||||
logoutURL = "https://api.twitter.com/1.1/account/logout.json"
|
||||
oAuthURL = "https://api.twitter.com/oauth2/token"
|
||||
loginURL = "https://api.x.com/1.1/onboarding/task.json"
|
||||
logoutURL = "https://api.x.com/1.1/account/logout.json"
|
||||
oAuthURL = "https://api.x.com/oauth2/token"
|
||||
// Doesn't require x-client-transaction-id header in auth. x-rate-limit-limit: 2000
|
||||
bearerToken1 = "AAAAAAAAAAAAAAAAAAAAAFQODgEAAAAAVHTp76lzh3rFzcHbmHVvQxYYpTw%3DckAlMINMjmCwxUcaXbAN4XqJVdgMJaHqNOFgPMK0zN1qLqLQCF"
|
||||
// HOTFIX: Returns 404 error; Requires x-client-transaction-id header in auth.
|
||||
|
|
@ -152,7 +152,7 @@ func (s *Scraper) getFlowToken(data map[string]interface{}) (string, error) {
|
|||
func (s *Scraper) IsLoggedIn() bool {
|
||||
s.isLogged = true
|
||||
s.setBearerToken(bearerToken1)
|
||||
req, err := http.NewRequest("GET", "https://api.twitter.com/1.1/account/verify_credentials.json", nil)
|
||||
req, err := http.NewRequest("GET", "https://api.x.com/1.1/account/verify_credentials.json", nil)
|
||||
if err != nil {
|
||||
return false
|
||||
}
|
||||
|
|
@ -444,7 +444,7 @@ func (s *Scraper) SetAuthToken(token AuthToken) {
|
|||
Name: "auth_token",
|
||||
Value: token.Token,
|
||||
Path: "",
|
||||
Domain: "twitter.com",
|
||||
Domain: "x.com",
|
||||
Expires: expires,
|
||||
RawExpires: "",
|
||||
MaxAge: 0,
|
||||
|
|
@ -457,7 +457,7 @@ func (s *Scraper) SetAuthToken(token AuthToken) {
|
|||
Name: "ct0",
|
||||
Value: token.CSRFToken,
|
||||
Path: "",
|
||||
Domain: "twitter.com",
|
||||
Domain: "x.com",
|
||||
Expires: expires,
|
||||
RawExpires: "",
|
||||
MaxAge: 0,
|
||||
|
|
|
|||
|
|
@ -18,7 +18,7 @@ func (s *Scraper) FetchBookmarks(maxTweetsNbr int, cursor string) ([]*Tweet, str
|
|||
maxTweetsNbr = 200
|
||||
}
|
||||
|
||||
req, err := s.newRequest("GET", "https://twitter.com/i/api/graphql/-IyJFt9_jS_9d_vS3NN-fA/Bookmarks")
|
||||
req, err := s.newRequest("GET", "https://x.com/i/api/graphql/-IyJFt9_jS_9d_vS3NN-fA/Bookmarks")
|
||||
if err != nil {
|
||||
return nil, "", err
|
||||
}
|
||||
|
|
|
|||
|
|
@ -21,7 +21,7 @@ func (s *Scraper) FetchFollowingByUserID(userID string, maxUsersNbr int, cursor
|
|||
maxUsersNbr = 200
|
||||
}
|
||||
|
||||
req, err := s.newRequest("GET", "https://twitter.com/i/api/graphql/g5P4cbXR4ta4oCeE7y2vLQ/Following")
|
||||
req, err := s.newRequest("GET", "https://x.com/i/api/graphql/g5P4cbXR4ta4oCeE7y2vLQ/Following")
|
||||
if err != nil {
|
||||
return nil, "", err
|
||||
}
|
||||
|
|
@ -94,7 +94,7 @@ func (s *Scraper) FetchFollowersByUserID(userID string, maxUsersNbr int, cursor
|
|||
maxUsersNbr = 200
|
||||
}
|
||||
|
||||
req, err := s.newRequest("GET", "https://twitter.com/i/api/graphql/jwbfbSzn0FRL_AMZGsYDag/Followers")
|
||||
req, err := s.newRequest("GET", "https://x.com/i/api/graphql/jwbfbSzn0FRL_AMZGsYDag/Followers")
|
||||
if err != nil {
|
||||
return nil, "", err
|
||||
}
|
||||
|
|
|
|||
2
go.mod
2
go.mod
|
|
@ -1,4 +1,4 @@
|
|||
module github.com/imperatrona/twitter-scraper
|
||||
module src.cultist.club/lain/twitter-scrapper
|
||||
|
||||
go 1.16
|
||||
|
||||
|
|
|
|||
|
|
@ -26,7 +26,7 @@ func (s *Scraper) FetchMediaTweetsByUserID(userID string, maxTweetsNbr int, curs
|
|||
maxTweetsNbr = 200
|
||||
}
|
||||
|
||||
req, err := s.newRequest("GET", "https://twitter.com/i/api/graphql/2tLOJWwGuCTytDrGBg8VwQ/UserMedia")
|
||||
req, err := s.newRequest("GET", "https://x.com/i/api/graphql/2tLOJWwGuCTytDrGBg8VwQ/UserMedia")
|
||||
if err != nil {
|
||||
return nil, "", err
|
||||
}
|
||||
|
|
|
|||
|
|
@ -65,7 +65,7 @@ type user struct {
|
|||
// GetProfile return parsed user profile.
|
||||
func (s *Scraper) GetProfile(username string) (Profile, error) {
|
||||
var jsn user
|
||||
req, err := http.NewRequest("GET", "https://api.twitter.com/graphql/Yka-W8dz7RaEuQNkroPkYw/UserByScreenName", nil)
|
||||
req, err := http.NewRequest("GET", "https://api.x.com/graphql/Yka-W8dz7RaEuQNkroPkYw/UserByScreenName", nil)
|
||||
if err != nil {
|
||||
return Profile{}, err
|
||||
}
|
||||
|
|
@ -126,7 +126,7 @@ func (s *Scraper) GetProfile(username string) (Profile, error) {
|
|||
|
||||
func (s *Scraper) GetProfileByID(userID string) (Profile, error) {
|
||||
var jsn user
|
||||
req, err := http.NewRequest("GET", "https://twitter.com/i/api/graphql/Qw77dDjp9xCpUY-AXwt-yQ/UserByRestId", nil)
|
||||
req, err := http.NewRequest("GET", "https://x.com/i/api/graphql/Qw77dDjp9xCpUY-AXwt-yQ/UserByRestId", nil)
|
||||
if err != nil {
|
||||
return Profile{}, err
|
||||
}
|
||||
|
|
|
|||
|
|
@ -10,7 +10,7 @@ type ThreadCursor struct {
|
|||
}
|
||||
|
||||
func (s *Scraper) GetTweetReplies(id string, cursor string) ([]*Tweet, []*ThreadCursor, error) {
|
||||
req, err := s.newRequest("GET", "https://twitter.com/i/api/graphql/ldqoq5MmFHN1FhMGvzC9Jg/TweetDetail")
|
||||
req, err := s.newRequest("GET", "https://x.com/i/api/graphql/ldqoq5MmFHN1FhMGvzC9Jg/TweetDetail")
|
||||
if err != nil {
|
||||
return nil, nil, err
|
||||
}
|
||||
|
|
|
|||
|
|
@ -129,7 +129,7 @@ func (timeline *scheduleTweets) parseTweets() []*ScheduledTweet {
|
|||
|
||||
// FetchScheduledTweets gets scheduled tweets via the Twitter frontend GraphQL API.
|
||||
func (s *Scraper) FetchScheduledTweets() ([]*ScheduledTweet, error) {
|
||||
req, err := s.newRequest("GET", "https://twitter.com/i/api/graphql/ITtjAzvlZni2wWXwf295Qg/FetchScheduledTweets")
|
||||
req, err := s.newRequest("GET", "https://x.com/i/api/graphql/ITtjAzvlZni2wWXwf295Qg/FetchScheduledTweets")
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
|
@ -154,7 +154,7 @@ func (s *Scraper) FetchScheduledTweets() ([]*ScheduledTweet, error) {
|
|||
|
||||
// DeleteScheduledTweet removes tweet from scheduled.
|
||||
func (s *Scraper) DeleteScheduledTweet(id string) error {
|
||||
req, err := s.newRequest("POST", "https://twitter.com/i/api/graphql/CTOVqej0JBXAZSwkp1US0g/DeleteScheduledTweet")
|
||||
req, err := s.newRequest("POST", "https://x.com/i/api/graphql/CTOVqej0JBXAZSwkp1US0g/DeleteScheduledTweet")
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
|
@ -197,7 +197,7 @@ func (s *Scraper) CreateScheduledTweet(schedule TweetSchedule) (string, error) {
|
|||
return "", errors.New("date can't be in past")
|
||||
}
|
||||
|
||||
req, err := s.newRequest("POST", "https://twitter.com/i/api/graphql/LCVzRQGxOaGnOnYH01NQXg/CreateScheduledTweet")
|
||||
req, err := s.newRequest("POST", "https://x.com/i/api/graphql/LCVzRQGxOaGnOnYH01NQXg/CreateScheduledTweet")
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
|
|
|
|||
12
scraper.go
12
scraper.go
|
|
@ -180,3 +180,15 @@ func (s *Scraper) SetUserAgent(userAgent string) {
|
|||
func (s *Scraper) GetUserAgent() string {
|
||||
return s.userAgent
|
||||
}
|
||||
|
||||
func (s *Scraper) SetBearerToken(token string) {
|
||||
s.setBearerToken(token)
|
||||
}
|
||||
|
||||
func (s *Scraper) SetHTTPClient(client *http.Client) {
|
||||
s.client = client
|
||||
}
|
||||
|
||||
func (s *Scraper) SetLoggedIn(v bool) {
|
||||
s.isLogged = v
|
||||
}
|
||||
|
|
|
|||
|
|
@ -7,7 +7,7 @@ import (
|
|||
"strconv"
|
||||
)
|
||||
|
||||
const searchURL = "https://twitter.com/i/api/graphql/nK1dw4oV3k4w5TdtcAdSww/SearchTimeline"
|
||||
const searchURL = "https://x.com/i/api/graphql/nK1dw4oV3k4w5TdtcAdSww/SearchTimeline"
|
||||
|
||||
type searchTimeline struct {
|
||||
Data struct {
|
||||
|
|
|
|||
|
|
@ -11,7 +11,7 @@ func (s *Scraper) GetSpace(id string) (*Space, error) {
|
|||
return nil, errors.New("scraper is not logged in")
|
||||
}
|
||||
|
||||
req, err := s.newRequest("GET", "https://twitter.com/i/api/graphql/d03OdorPdZ_sH9V3D1_yWQ/AudioSpaceById")
|
||||
req, err := s.newRequest("GET", "https://x.com/i/api/graphql/d03OdorPdZ_sH9V3D1_yWQ/AudioSpaceById")
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
|
|
|||
|
|
@ -92,7 +92,7 @@ func (timeline *timelineV1) parseTweet(id string) *Tweet {
|
|||
ConversationID: tweet.ConversationIDStr,
|
||||
Likes: tweet.FavoriteCount,
|
||||
Name: name,
|
||||
PermanentURL: fmt.Sprintf("https://twitter.com/%s/status/%s", username, id),
|
||||
PermanentURL: fmt.Sprintf("https://x.com/%s/status/%s", username, id),
|
||||
Replies: tweet.ReplyCount,
|
||||
Retweets: tweet.RetweetCount,
|
||||
Text: tweet.FullText,
|
||||
|
|
@ -194,13 +194,13 @@ func (timeline *timelineV1) parseTweet(id string) *Tweet {
|
|||
|
||||
tw.HTML = tweet.FullText
|
||||
tw.HTML = reHashtag.ReplaceAllStringFunc(tw.HTML, func(hashtag string) string {
|
||||
return fmt.Sprintf(`<a href="https://twitter.com/hashtag/%s">%s</a>`,
|
||||
return fmt.Sprintf(`<a href="https://x.com/hashtag/%s">%s</a>`,
|
||||
strings.TrimPrefix(hashtag, "#"),
|
||||
hashtag,
|
||||
)
|
||||
})
|
||||
tw.HTML = reUsername.ReplaceAllStringFunc(tw.HTML, func(username string) string {
|
||||
return fmt.Sprintf(`<a href="https://twitter.com/%s">%s</a>`,
|
||||
return fmt.Sprintf(`<a href="https://x.com/%s">%s</a>`,
|
||||
strings.TrimPrefix(username, "@"),
|
||||
username,
|
||||
)
|
||||
|
|
|
|||
|
|
@ -4,7 +4,7 @@ import "fmt"
|
|||
|
||||
// GetTrends return list of trends.
|
||||
func (s *Scraper) GetTrends() ([]string, error) {
|
||||
req, err := s.newRequest("GET", "https://api.twitter.com/2/guide.json")
|
||||
req, err := s.newRequest("GET", "https://api.x.com/2/guide.json")
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
|
|
|||
14
tweet.go
14
tweet.go
|
|
@ -47,7 +47,7 @@ func (newTweet *newTweet) parse() *Tweet {
|
|||
}
|
||||
|
||||
func (s *Scraper) CreateTweet(tweet NewTweet) (*Tweet, error) {
|
||||
req, err := s.newRequest("POST", "https://twitter.com/i/api/graphql/oB-5XsHNAbjvARJEc8CZFw/CreateTweet")
|
||||
req, err := s.newRequest("POST", "https://x.com/i/api/graphql/oB-5XsHNAbjvARJEc8CZFw/CreateTweet")
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
|
@ -126,7 +126,7 @@ func (s *Scraper) CreateTweet(tweet NewTweet) (*Tweet, error) {
|
|||
}
|
||||
|
||||
func (s *Scraper) DeleteTweet(tweetId string) error {
|
||||
req, err := s.newRequest("POST", "https://twitter.com/i/api/graphql/VaenaVgh5q5ih7kvyVjgtg/DeleteTweet")
|
||||
req, err := s.newRequest("POST", "https://x.com/i/api/graphql/VaenaVgh5q5ih7kvyVjgtg/DeleteTweet")
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
|
@ -163,7 +163,7 @@ func (s *Scraper) DeleteTweet(tweetId string) error {
|
|||
}
|
||||
|
||||
func (s *Scraper) CreateRetweet(tweetId string) (string, error) {
|
||||
req, err := s.newRequest("POST", "https://twitter.com/i/api/graphql/ojPdsZsimiJrUGLR1sjUtA/CreateRetweet")
|
||||
req, err := s.newRequest("POST", "https://x.com/i/api/graphql/ojPdsZsimiJrUGLR1sjUtA/CreateRetweet")
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
|
|
@ -211,7 +211,7 @@ func (s *Scraper) CreateRetweet(tweetId string) (string, error) {
|
|||
|
||||
// Retweeted tweets has their own id, but to delete retweet twitter using id of source tweet
|
||||
func (s *Scraper) DeleteRetweet(tweetId string) error {
|
||||
req, err := s.newRequest("POST", "https://twitter.com/i/api/graphql/iQtK4dl5hBmXewYZuEOKVw/DeleteRetweet")
|
||||
req, err := s.newRequest("POST", "https://x.com/i/api/graphql/iQtK4dl5hBmXewYZuEOKVw/DeleteRetweet")
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
|
@ -253,7 +253,7 @@ func (s *Scraper) DeleteRetweet(tweetId string) error {
|
|||
}
|
||||
|
||||
func (s *Scraper) LikeTweet(tweetId string) error {
|
||||
req, err := s.newRequest("POST", "https://twitter.com/i/api/graphql/lI07N6Otwv1PhnEgXILM7A/FavoriteTweet")
|
||||
req, err := s.newRequest("POST", "https://x.com/i/api/graphql/lI07N6Otwv1PhnEgXILM7A/FavoriteTweet")
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
|
@ -297,7 +297,7 @@ func (s *Scraper) LikeTweet(tweetId string) error {
|
|||
}
|
||||
|
||||
func (s *Scraper) UnlikeTweet(tweetId string) error {
|
||||
req, err := s.newRequest("POST", "https://twitter.com/i/api/graphql/ZYKSe-w7KEslx3JhSIk5LA/UnfavoriteTweet")
|
||||
req, err := s.newRequest("POST", "https://x.com/i/api/graphql/ZYKSe-w7KEslx3JhSIk5LA/UnfavoriteTweet")
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
|
@ -344,7 +344,7 @@ func (s *Scraper) GetTweetRetweeters(tweetId string, maxUsersNbr int, cursor str
|
|||
maxUsersNbr = 200
|
||||
}
|
||||
|
||||
req, err := s.newRequest("GET", "https://twitter.com/i/api/graphql/8019obfgnveiPiJuS2Rtow/Retweeters")
|
||||
req, err := s.newRequest("GET", "https://x.com/i/api/graphql/8019obfgnveiPiJuS2Rtow/Retweeters")
|
||||
if err != nil {
|
||||
return nil, "", err
|
||||
}
|
||||
|
|
|
|||
20
tweets.go
20
tweets.go
|
|
@ -46,7 +46,7 @@ func (s *Scraper) FetchTweetsAndRepliesByUserID(userID string, maxReplysNbr int,
|
|||
maxReplysNbr = 200
|
||||
}
|
||||
|
||||
req, err := s.newRequest("GET", "https://twitter.com/i/api/graphql/bt4TKuFz4T7Ckk-VvQVSow/UserTweetsAndReplies")
|
||||
req, err := s.newRequest("GET", "https://x.com/i/api/graphql/bt4TKuFz4T7Ckk-VvQVSow/UserTweetsAndReplies")
|
||||
if err != nil {
|
||||
return nil, "", err
|
||||
}
|
||||
|
|
@ -111,7 +111,7 @@ func (s *Scraper) FetchTweetsByUserID(userID string, maxTweetsNbr int, cursor st
|
|||
maxTweetsNbr = 200
|
||||
}
|
||||
|
||||
req, err := s.newRequest("GET", "https://twitter.com/i/api/graphql/UGi7tjRPr-d_U3bCPIko5Q/UserTweets")
|
||||
req, err := s.newRequest("GET", "https://x.com/i/api/graphql/UGi7tjRPr-d_U3bCPIko5Q/UserTweets")
|
||||
if err != nil {
|
||||
return nil, "", err
|
||||
}
|
||||
|
|
@ -173,7 +173,7 @@ func (s *Scraper) FetchTweetsByUserIDLegacy(userID string, maxTweetsNbr int, cur
|
|||
maxTweetsNbr = 200
|
||||
}
|
||||
|
||||
req, err := s.newRequest("GET", "https://api.twitter.com/2/timeline/profile/"+userID+".json")
|
||||
req, err := s.newRequest("GET", "https://api.x.com/2/timeline/profile/"+userID+".json")
|
||||
if err != nil {
|
||||
return nil, "", err
|
||||
}
|
||||
|
|
@ -199,7 +199,7 @@ func (s *Scraper) FetchTweetsByUserIDLegacy(userID string, maxTweetsNbr int, cur
|
|||
// GetTweet get a single tweet by ID.
|
||||
func (s *Scraper) GetTweet(id string) (*Tweet, error) {
|
||||
if s.isOpenAccount {
|
||||
req, err := s.newRequest("GET", "https://api.twitter.com/2/timeline/conversation/"+id+".json")
|
||||
req, err := s.newRequest("GET", "https://api.x.com/2/timeline/conversation/"+id+".json")
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
|
@ -217,7 +217,7 @@ func (s *Scraper) GetTweet(id string) (*Tweet, error) {
|
|||
}
|
||||
}
|
||||
} else if s.isLogged {
|
||||
req, err := s.newRequest("GET", "https://twitter.com/i/api/graphql/VWFGPVAGkZMGRKGe3GFFnA/TweetDetail")
|
||||
req, err := s.newRequest("GET", "https://x.com/i/api/graphql/VWFGPVAGkZMGRKGe3GFFnA/TweetDetail")
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
|
@ -263,7 +263,7 @@ func (s *Scraper) GetTweet(id string) (*Tweet, error) {
|
|||
|
||||
// Surprisingly, if bearerToken2 is not set, then animated GIFs are not
|
||||
// present in the response for tweets with a GIF + a photo like this one:
|
||||
// https://twitter.com/Twitter/status/1580661436132757506
|
||||
// https://x.com/Twitter/status/1580661436132757506
|
||||
curBearerToken := s.bearerToken
|
||||
if curBearerToken != bearerToken2 {
|
||||
s.setBearerToken(bearerToken2)
|
||||
|
|
@ -286,7 +286,7 @@ func (s *Scraper) GetTweet(id string) (*Tweet, error) {
|
|||
}
|
||||
}
|
||||
} else {
|
||||
req, err := s.newRequest("GET", "https://twitter.com/i/api/graphql/xBtHv5-Xsk268T5ng_OGNg/TweetResultByRestId")
|
||||
req, err := s.newRequest("GET", "https://x.com/i/api/graphql/xBtHv5-Xsk268T5ng_OGNg/TweetResultByRestId")
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
|
@ -332,7 +332,7 @@ func (s *Scraper) GetTweet(id string) (*Tweet, error) {
|
|||
|
||||
// Surprisingly, if bearerToken2 is not set, then animated GIFs are not
|
||||
// present in the response for tweets with a GIF + a photo like this one:
|
||||
// https://twitter.com/Twitter/status/1580661436132757506
|
||||
// https://x.com/Twitter/status/1580661436132757506
|
||||
curBearerToken := s.bearerToken
|
||||
if curBearerToken != bearerToken2 {
|
||||
s.setBearerToken(bearerToken2)
|
||||
|
|
@ -421,7 +421,7 @@ func (s *Scraper) fetchHomeTweets(_ string, maxTweetsNbr int, cursor string) ([]
|
|||
maxTweetsNbr = 200
|
||||
}
|
||||
|
||||
req, err := s.newRequest("GET", "https://twitter.com/i/api/graphql/9EwYy8pLBOSFlEoSP2STiQ/HomeLatestTimeline")
|
||||
req, err := s.newRequest("GET", "https://x.com/i/api/graphql/9EwYy8pLBOSFlEoSP2STiQ/HomeLatestTimeline")
|
||||
if err != nil {
|
||||
return nil, "", err
|
||||
}
|
||||
|
|
@ -496,7 +496,7 @@ func (s *Scraper) fetchForYouTweets(_ string, maxTweetsNbr int, cursor string) (
|
|||
maxTweetsNbr = 200
|
||||
}
|
||||
|
||||
req, err := s.newRequest("GET", "https://twitter.com/i/api/graphql/1u0Wlkw6Ru1NwBUD-pDiww/HomeTimeline")
|
||||
req, err := s.newRequest("GET", "https://x.com/i/api/graphql/1u0Wlkw6Ru1NwBUD-pDiww/HomeTimeline")
|
||||
if err != nil {
|
||||
return nil, "", err
|
||||
}
|
||||
|
|
|
|||
24
upload.go
24
upload.go
|
|
@ -104,7 +104,7 @@ func (s *Scraper) uploadInit(filePath string, fileContent []byte) (*Media, error
|
|||
return nil, fmt.Errorf("file type %s unsupported by twitter, make sure you uploading photo, video or gif", fileType)
|
||||
}
|
||||
|
||||
req, err := s.newRequest("POST", "https://upload.twitter.com/i/media/upload.json")
|
||||
req, err := s.newRequest("POST", "https://upload.x.com/i/media/upload.json")
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
|
@ -118,8 +118,8 @@ func (s *Scraper) uploadInit(filePath string, fileContent []byte) (*Media, error
|
|||
query.Set("video_duration_ms", strconv.FormatFloat(videoDuration*1000, 'f', -1, 64))
|
||||
}
|
||||
req.URL.RawQuery = query.Encode()
|
||||
req.Header.Set("Origin", "https://twitter.com")
|
||||
req.Header.Set("Referer", "https://twitter.com/")
|
||||
req.Header.Set("Origin", "https://x.com")
|
||||
req.Header.Set("Referer", "https://x.com/")
|
||||
|
||||
var uploadInit uploadInitResponse
|
||||
|
||||
|
|
@ -158,7 +158,7 @@ func (s *Scraper) uploadAppend(media *Media, fileContent []byte) error {
|
|||
}
|
||||
w.Close()
|
||||
|
||||
req, err := s.newRequest("POST", "https://upload.twitter.com/i/media/upload.json")
|
||||
req, err := s.newRequest("POST", "https://upload.x.com/i/media/upload.json")
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
|
@ -169,8 +169,8 @@ func (s *Scraper) uploadAppend(media *Media, fileContent []byte) error {
|
|||
query.Set("segment_index", strconv.Itoa(i))
|
||||
req.URL.RawQuery = query.Encode()
|
||||
req.Header.Set("Content-Type", w.FormDataContentType())
|
||||
req.Header.Set("Origin", "https://twitter.com")
|
||||
req.Header.Set("Referer", "https://twitter.com/")
|
||||
req.Header.Set("Origin", "https://x.com")
|
||||
req.Header.Set("Referer", "https://x.com/")
|
||||
req.Body = io.NopCloser(&buf)
|
||||
|
||||
err = s.RequestAPI(req, nil)
|
||||
|
|
@ -183,7 +183,7 @@ func (s *Scraper) uploadAppend(media *Media, fileContent []byte) error {
|
|||
}
|
||||
|
||||
func (s *Scraper) uploadFinalize(media *Media) (*ProcessingInfo, error) {
|
||||
req, err := s.newRequest("POST", "https://upload.twitter.com/i/media/upload.json")
|
||||
req, err := s.newRequest("POST", "https://upload.x.com/i/media/upload.json")
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
|
@ -193,8 +193,8 @@ func (s *Scraper) uploadFinalize(media *Media) (*ProcessingInfo, error) {
|
|||
query.Set("media_id", strconv.Itoa(media.ID))
|
||||
query.Set("allow_async", "true")
|
||||
req.URL.RawQuery = query.Encode()
|
||||
req.Header.Set("Origin", "https://twitter.com")
|
||||
req.Header.Set("Referer", "https://twitter.com/")
|
||||
req.Header.Set("Origin", "https://x.com")
|
||||
req.Header.Set("Referer", "https://x.com/")
|
||||
|
||||
var response uploadStatusResponse
|
||||
|
||||
|
|
@ -207,7 +207,7 @@ func (s *Scraper) uploadFinalize(media *Media) (*ProcessingInfo, error) {
|
|||
}
|
||||
|
||||
func (s *Scraper) uploadStatus(media *Media) (*ProcessingInfo, error) {
|
||||
req, err := s.newRequest("GET", "https://upload.twitter.com/i/media/upload.json")
|
||||
req, err := s.newRequest("GET", "https://upload.x.com/i/media/upload.json")
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
|
@ -216,8 +216,8 @@ func (s *Scraper) uploadStatus(media *Media) (*ProcessingInfo, error) {
|
|||
query.Set("command", "STATUS")
|
||||
query.Set("media_id", strconv.Itoa(media.ID))
|
||||
req.URL.RawQuery = query.Encode()
|
||||
req.Header.Set("Origin", "https://twitter.com")
|
||||
req.Header.Set("Referer", "https://twitter.com/")
|
||||
req.Header.Set("Origin", "https://x.com")
|
||||
req.Header.Set("Referer", "https://x.com/")
|
||||
|
||||
var response uploadStatusResponse
|
||||
|
||||
|
|
|
|||
12
util.go
12
util.go
|
|
@ -16,7 +16,7 @@ var (
|
|||
reHashtag = regexp.MustCompile(`\B(\#\S+\b)`)
|
||||
reTwitterURL = regexp.MustCompile(`https:(\/\/t\.co\/([A-Za-z0-9]|[A-Za-z]){10})`)
|
||||
reUsername = regexp.MustCompile(`\B(\@\S{1,15}\b)`)
|
||||
twURL = urlParse("https://twitter.com")
|
||||
twURL = urlParse("https://x.com")
|
||||
)
|
||||
|
||||
func (s *Scraper) newRequest(method string, url string) (*http.Request, error) {
|
||||
|
|
@ -165,7 +165,7 @@ func parseLegacyTweet(user *legacyUser, tweet *legacyTweet) *Tweet {
|
|||
ID: tweetID,
|
||||
Likes: tweet.FavoriteCount,
|
||||
Name: name,
|
||||
PermanentURL: fmt.Sprintf("https://twitter.com/%s/status/%s", username, tweetID),
|
||||
PermanentURL: fmt.Sprintf("https://x.com/%s/status/%s", username, tweetID),
|
||||
Replies: tweet.ReplyCount,
|
||||
Retweets: tweet.RetweetCount,
|
||||
Text: text,
|
||||
|
|
@ -293,13 +293,13 @@ func parseLegacyTweet(user *legacyUser, tweet *legacyTweet) *Tweet {
|
|||
|
||||
tw.HTML = tweet.FullText
|
||||
tw.HTML = reHashtag.ReplaceAllStringFunc(tw.HTML, func(hashtag string) string {
|
||||
return fmt.Sprintf(`<a href="https://twitter.com/hashtag/%s">%s</a>`,
|
||||
return fmt.Sprintf(`<a href="https://x.com/hashtag/%s">%s</a>`,
|
||||
strings.TrimPrefix(hashtag, "#"),
|
||||
hashtag,
|
||||
)
|
||||
})
|
||||
tw.HTML = reUsername.ReplaceAllStringFunc(tw.HTML, func(username string) string {
|
||||
return fmt.Sprintf(`<a href="https://twitter.com/%s">%s</a>`,
|
||||
return fmt.Sprintf(`<a href="https://x.com/%s">%s</a>`,
|
||||
strings.TrimPrefix(username, "@"),
|
||||
username,
|
||||
)
|
||||
|
|
@ -360,7 +360,7 @@ func parseProfile(user legacyUser) Profile {
|
|||
Name: user.Name,
|
||||
PinnedTweetIDs: user.PinnedTweetIdsStr,
|
||||
TweetsCount: user.StatusesCount,
|
||||
URL: "https://twitter.com/" + user.ScreenName,
|
||||
URL: "https://x.com/" + user.ScreenName,
|
||||
UserID: user.IDStr,
|
||||
Username: user.ScreenName,
|
||||
FollowedBy: user.FollowedBy,
|
||||
|
|
@ -415,7 +415,7 @@ func parseProfileV2(user userResult) Profile {
|
|||
Name: u.Name,
|
||||
PinnedTweetIDs: u.PinnedTweetIdsStr,
|
||||
TweetsCount: u.StatusesCount,
|
||||
URL: "https://twitter.com/" + u.ScreenName,
|
||||
URL: "https://x.com/" + u.ScreenName,
|
||||
UserID: user.ID,
|
||||
Username: u.ScreenName,
|
||||
Sensitive: u.PossiblySensitive,
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue