Compare commits
16 commits
152a0a1c2c
...
76cb95cd3e
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
76cb95cd3e | ||
|
|
d31ee7d09f | ||
|
|
f5a2629e42 | ||
|
|
a48dcf2587 | ||
|
|
3684e678e3 | ||
|
|
c269a9ba92 | ||
|
|
e5c170986e | ||
|
|
44183226af | ||
|
|
4276e243ac | ||
|
|
4519142ae8 | ||
|
|
48d27acfb1 | ||
|
|
388445d4c2 | ||
|
|
7226460e05 | ||
|
|
40066f125a | ||
|
|
cc1eb793d6 | ||
|
|
9f31f3890f |
7 changed files with 300 additions and 132 deletions
5
auth.go
5
auth.go
|
|
@ -24,8 +24,9 @@ const (
|
||||||
oAuthURL = "https://api.twitter.com/oauth2/token"
|
oAuthURL = "https://api.twitter.com/oauth2/token"
|
||||||
// Doesn't require x-client-transaction-id header in auth. x-rate-limit-limit: 2000
|
// Doesn't require x-client-transaction-id header in auth. x-rate-limit-limit: 2000
|
||||||
bearerToken1 = "AAAAAAAAAAAAAAAAAAAAAFQODgEAAAAAVHTp76lzh3rFzcHbmHVvQxYYpTw%3DckAlMINMjmCwxUcaXbAN4XqJVdgMJaHqNOFgPMK0zN1qLqLQCF"
|
bearerToken1 = "AAAAAAAAAAAAAAAAAAAAAFQODgEAAAAAVHTp76lzh3rFzcHbmHVvQxYYpTw%3DckAlMINMjmCwxUcaXbAN4XqJVdgMJaHqNOFgPMK0zN1qLqLQCF"
|
||||||
// Requires x-client-transaction-id header in auth.
|
// HOTFIX: Returns 404 error; Requires x-client-transaction-id header in auth.
|
||||||
bearerToken2 = "AAAAAAAAAAAAAAAAAAAAANRILgAAAAAAnNwIzUejRCOuH5E6I8xnZz4puTs%3D1Zv7ttfk8LF81IUq16cHjhLTvJu4FA33AGWWjCpTnA"
|
// bearerToken2 = "AAAAAAAAAAAAAAAAAAAAANRILgAAAAAAnNwIzUejRCOuH5E6I8xnZz4puTs%3D1Zv7ttfk8LF81IUq16cHjhLTvJu4FA33AGWWjCpTnA"
|
||||||
|
bearerToken2 = "AAAAAAAAAAAAAAAAAAAAAFQODgEAAAAAVHTp76lzh3rFzcHbmHVvQxYYpTw%3DckAlMINMjmCwxUcaXbAN4XqJVdgMJaHqNOFgPMK0zN1qLqLQCF"
|
||||||
appConsumerKey = "3nVuSoBZnx6U4vzUxf5w"
|
appConsumerKey = "3nVuSoBZnx6U4vzUxf5w"
|
||||||
appConsumerSecret = "Bcs59EFbbsdF6Sl9Ng71smgStWEGwXXKSjYvPVt7qys"
|
appConsumerSecret = "Bcs59EFbbsdF6Sl9Ng71smgStWEGwXXKSjYvPVt7qys"
|
||||||
)
|
)
|
||||||
|
|
|
||||||
16
profile.go
16
profile.go
|
|
@ -23,6 +23,7 @@ type Profile struct {
|
||||||
FriendsCount int
|
FriendsCount int
|
||||||
IsPrivate bool
|
IsPrivate bool
|
||||||
IsVerified bool
|
IsVerified bool
|
||||||
|
IsBlueVerified bool
|
||||||
Joined *time.Time
|
Joined *time.Time
|
||||||
LikesCount int
|
LikesCount int
|
||||||
ListedCount int
|
ListedCount int
|
||||||
|
|
@ -37,6 +38,12 @@ type Profile struct {
|
||||||
Sensitive bool
|
Sensitive bool
|
||||||
Following bool
|
Following bool
|
||||||
FollowedBy bool
|
FollowedBy bool
|
||||||
|
MediaCount int
|
||||||
|
FastFollowersCount int
|
||||||
|
NormalFollowersCount int
|
||||||
|
ProfileImageShape string
|
||||||
|
HasGraduatedAccess bool
|
||||||
|
CanHighlightTweets bool
|
||||||
}
|
}
|
||||||
|
|
||||||
type user struct {
|
type user struct {
|
||||||
|
|
@ -46,6 +53,7 @@ type user struct {
|
||||||
RestID string `json:"rest_id"`
|
RestID string `json:"rest_id"`
|
||||||
Legacy legacyUser `json:"legacy"`
|
Legacy legacyUser `json:"legacy"`
|
||||||
Message string `json:"message"`
|
Message string `json:"message"`
|
||||||
|
IsBlueVerified bool `json:"is_blue_verified"`
|
||||||
} `json:"result"`
|
} `json:"result"`
|
||||||
} `json:"user"`
|
} `json:"user"`
|
||||||
} `json:"data"`
|
} `json:"data"`
|
||||||
|
|
@ -111,7 +119,9 @@ func (s *Scraper) GetProfile(username string) (Profile, error) {
|
||||||
return Profile{}, fmt.Errorf("either @%s does not exist or is private", username)
|
return Profile{}, fmt.Errorf("either @%s does not exist or is private", username)
|
||||||
}
|
}
|
||||||
|
|
||||||
return parseProfile(jsn.Data.User.Result.Legacy), nil
|
profile := parseProfile(jsn.Data.User.Result.Legacy)
|
||||||
|
profile.IsBlueVerified = jsn.Data.User.Result.IsBlueVerified
|
||||||
|
return profile, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
func (s *Scraper) GetProfileByID(userID string) (Profile, error) {
|
func (s *Scraper) GetProfileByID(userID string) (Profile, error) {
|
||||||
|
|
@ -168,7 +178,9 @@ func (s *Scraper) GetProfileByID(userID string) (Profile, error) {
|
||||||
return Profile{}, fmt.Errorf("either @%s does not exist or is private", userID)
|
return Profile{}, fmt.Errorf("either @%s does not exist or is private", userID)
|
||||||
}
|
}
|
||||||
|
|
||||||
return parseProfile(jsn.Data.User.Result.Legacy), nil
|
profile := parseProfile(jsn.Data.User.Result.Legacy)
|
||||||
|
profile.IsBlueVerified = jsn.Data.User.Result.IsBlueVerified
|
||||||
|
return profile, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
// GetUserIDByScreenName from API
|
// GetUserIDByScreenName from API
|
||||||
|
|
|
||||||
|
|
@ -42,6 +42,8 @@ func TestGetProfile(t *testing.T) {
|
||||||
cmpopts.IgnoreFields(twitterscraper.Profile{}, "LikesCount"),
|
cmpopts.IgnoreFields(twitterscraper.Profile{}, "LikesCount"),
|
||||||
cmpopts.IgnoreFields(twitterscraper.Profile{}, "ListedCount"),
|
cmpopts.IgnoreFields(twitterscraper.Profile{}, "ListedCount"),
|
||||||
cmpopts.IgnoreFields(twitterscraper.Profile{}, "TweetsCount"),
|
cmpopts.IgnoreFields(twitterscraper.Profile{}, "TweetsCount"),
|
||||||
|
cmpopts.IgnoreFields(twitterscraper.Profile{}, "MediaCount"),
|
||||||
|
cmpopts.IgnoreFields(twitterscraper.Profile{}, "NormalFollowersCount"),
|
||||||
}
|
}
|
||||||
if diff := cmp.Diff(sample, profile, cmpOptions...); diff != "" {
|
if diff := cmp.Diff(sample, profile, cmpOptions...); diff != "" {
|
||||||
t.Error("Resulting profile does not match the sample", diff)
|
t.Error("Resulting profile does not match the sample", diff)
|
||||||
|
|
@ -94,6 +96,8 @@ func TestGetProfilePrivate(t *testing.T) {
|
||||||
cmpopts.IgnoreFields(twitterscraper.Profile{}, "LikesCount"),
|
cmpopts.IgnoreFields(twitterscraper.Profile{}, "LikesCount"),
|
||||||
cmpopts.IgnoreFields(twitterscraper.Profile{}, "ListedCount"),
|
cmpopts.IgnoreFields(twitterscraper.Profile{}, "ListedCount"),
|
||||||
cmpopts.IgnoreFields(twitterscraper.Profile{}, "TweetsCount"),
|
cmpopts.IgnoreFields(twitterscraper.Profile{}, "TweetsCount"),
|
||||||
|
cmpopts.IgnoreFields(twitterscraper.Profile{}, "MediaCount"),
|
||||||
|
cmpopts.IgnoreFields(twitterscraper.Profile{}, "NormalFollowersCount"),
|
||||||
}
|
}
|
||||||
if diff := cmp.Diff(sample, profile, cmpOptions...); diff != "" {
|
if diff := cmp.Diff(sample, profile, cmpOptions...); diff != "" {
|
||||||
t.Error("Resulting profile does not match the sample", diff)
|
t.Error("Resulting profile does not match the sample", diff)
|
||||||
|
|
@ -146,7 +150,7 @@ func TestGetProfileByID(t *testing.T) {
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestGetUserIDByScreenName(t *testing.T) {
|
func TestGetUserIDByScreenName(t *testing.T) {
|
||||||
userID, err := testScraper.GetUserIDByScreenName("Twitter")
|
userID, err := testScraper.GetUserIDByScreenName("X")
|
||||||
if err != nil {
|
if err != nil {
|
||||||
t.Errorf("getUserByScreenName() error = %v", err)
|
t.Errorf("getUserByScreenName() error = %v", err)
|
||||||
}
|
}
|
||||||
|
|
|
||||||
|
|
@ -1,6 +1,7 @@
|
||||||
package twitterscraper
|
package twitterscraper
|
||||||
|
|
||||||
import (
|
import (
|
||||||
|
"encoding/json"
|
||||||
"strconv"
|
"strconv"
|
||||||
"strings"
|
"strings"
|
||||||
)
|
)
|
||||||
|
|
@ -28,6 +29,17 @@ type tweet struct {
|
||||||
Result *result `json:"result"`
|
Result *result `json:"result"`
|
||||||
} `json:"quoted_status_result"`
|
} `json:"quoted_status_result"`
|
||||||
Legacy legacyTweet `json:"legacy"`
|
Legacy legacyTweet `json:"legacy"`
|
||||||
|
Card struct {
|
||||||
|
RestID string `json:"rest_id"`
|
||||||
|
Legacy struct {
|
||||||
|
BindingValues []struct {
|
||||||
|
Key string `json:"key"`
|
||||||
|
Value struct {
|
||||||
|
StringValue string `json:"string_value"`
|
||||||
|
} `json:"value"`
|
||||||
|
} `json:"binding_values"`
|
||||||
|
} `json:"legacy"`
|
||||||
|
} `json:"card"`
|
||||||
}
|
}
|
||||||
|
|
||||||
type result struct {
|
type result struct {
|
||||||
|
|
@ -36,6 +48,31 @@ type result struct {
|
||||||
Tweet tweet `json:"tweet"`
|
Tweet tweet `json:"tweet"`
|
||||||
}
|
}
|
||||||
|
|
||||||
|
type UnifiedCard struct {
|
||||||
|
Type string `json:"type"`
|
||||||
|
Components []string `json:"components"`
|
||||||
|
MediaEntities map[string]struct {
|
||||||
|
ID int64 `json:"id"`
|
||||||
|
IDStr string `json:"id_str"`
|
||||||
|
MediaURLHTTPS string `json:"media_url_https"`
|
||||||
|
Type string `json:"type"`
|
||||||
|
OriginalInfo struct {
|
||||||
|
Width int `json:"width"`
|
||||||
|
Height int `json:"height"`
|
||||||
|
} `json:"original_info"`
|
||||||
|
SourceUserID int64 `json:"source_user_id"`
|
||||||
|
VideoInfo struct {
|
||||||
|
AspectRatio []int `json:"aspect_ratio"`
|
||||||
|
DurationMillis int `json:"duration_millis"`
|
||||||
|
Variants []struct {
|
||||||
|
Bitrate int `json:"bitrate,omitempty"`
|
||||||
|
ContentType string `json:"content_type"`
|
||||||
|
URL string `json:"url"`
|
||||||
|
} `json:"variants"`
|
||||||
|
} `json:"video_info"`
|
||||||
|
} `json:"media_entities"`
|
||||||
|
}
|
||||||
|
|
||||||
func (result *result) parse() *Tweet {
|
func (result *result) parse() *Tweet {
|
||||||
if result.NoteTweet.NoteTweetResults.Result.Text != "" {
|
if result.NoteTweet.NoteTweetResults.Result.Text != "" {
|
||||||
result.Legacy.FullText = result.NoteTweet.NoteTweetResults.Result.Text
|
result.Legacy.FullText = result.NoteTweet.NoteTweetResults.Result.Text
|
||||||
|
|
@ -56,6 +93,43 @@ func (result *result) parse() *Tweet {
|
||||||
if result.QuotedStatusResult.Result != nil {
|
if result.QuotedStatusResult.Result != nil {
|
||||||
tw.QuotedStatus = result.QuotedStatusResult.Result.parse()
|
tw.QuotedStatus = result.QuotedStatusResult.Result.parse()
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// Get videos from cards
|
||||||
|
for _, v := range result.Tweet.Card.Legacy.BindingValues {
|
||||||
|
if v.Key == "unified_card" {
|
||||||
|
var card UnifiedCard
|
||||||
|
err := json.Unmarshal([]byte(v.Value.StringValue), &card)
|
||||||
|
if err != nil {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
|
||||||
|
for _, media := range card.MediaEntities {
|
||||||
|
if media.Type == "video" {
|
||||||
|
var vid Video
|
||||||
|
|
||||||
|
vid.ID = media.IDStr
|
||||||
|
vid.Preview = media.MediaURLHTTPS
|
||||||
|
|
||||||
|
var bitrate int
|
||||||
|
for _, variant := range media.VideoInfo.Variants {
|
||||||
|
if variant.ContentType == "video/mp4" {
|
||||||
|
if variant.Bitrate > bitrate {
|
||||||
|
bitrate = variant.Bitrate
|
||||||
|
vid.URL = variant.URL
|
||||||
|
}
|
||||||
|
} else if variant.ContentType == "application/x-mpegURL" {
|
||||||
|
vid.HLSURL = variant.URL
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
if vid.URL != "" {
|
||||||
|
tw.Videos = append(tw.Videos, vid)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
return tw
|
return tw
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
@ -68,6 +142,13 @@ type userResult struct {
|
||||||
IsBlueVerified bool `json:"is_blue_verified"`
|
IsBlueVerified bool `json:"is_blue_verified"`
|
||||||
ProfileImageShape string `json:"profile_image_shape"`
|
ProfileImageShape string `json:"profile_image_shape"`
|
||||||
Legacy legacyUserV2 `json:"legacy"`
|
Legacy legacyUserV2 `json:"legacy"`
|
||||||
|
LegacyExtendedProfile legacyExtendedProfile `json:"legacy_extended_profile"`
|
||||||
|
IsProfileTranslatable bool `json:"is_profile_translatable"`
|
||||||
|
VerificationInfo verificationInfo `json:"verification_info"`
|
||||||
|
HighlightsInfo highlightsInfo `json:"highlights_info"`
|
||||||
|
UserSeedTweetCount int `json:"user_seed_tweet_count"`
|
||||||
|
PremiumGiftingEligible bool `json:"premium_gifting_eligible"`
|
||||||
|
CreatorSubscriptionsCount int `json:"creator_subscriptions_count"`
|
||||||
}
|
}
|
||||||
|
|
||||||
func (result *userResult) parse() Profile {
|
func (result *userResult) parse() Profile {
|
||||||
|
|
|
||||||
|
|
@ -92,7 +92,7 @@ func TestGetTweetWithVideo(t *testing.T) {
|
||||||
Name: "X",
|
Name: "X",
|
||||||
PermanentURL: "https://twitter.com/X/status/1697304622749086011",
|
PermanentURL: "https://twitter.com/X/status/1697304622749086011",
|
||||||
Photos: nil,
|
Photos: nil,
|
||||||
Text: "on iOS & Android, you can now swipe to reply when you slide into their DMs https://t.co/evuWpMfBxQ",
|
Text: "on iOS & Android, you can now swipe to reply when you slide into their DMs https://pbs.twimg.com/amplify_video_thumb/1697304568550330368/img/BUlESpef6FmWV_j2.jpg",
|
||||||
Timestamp: 1693503931,
|
Timestamp: 1693503931,
|
||||||
UserID: "783214",
|
UserID: "783214",
|
||||||
Username: "X",
|
Username: "X",
|
||||||
|
|
@ -125,7 +125,7 @@ func TestGetTweetWithMultiplePhotos(t *testing.T) {
|
||||||
URL: "https://pbs.twimg.com/media/FeUJKuxXEAAa6t7.jpg",
|
URL: "https://pbs.twimg.com/media/FeUJKuxXEAAa6t7.jpg",
|
||||||
},
|
},
|
||||||
},
|
},
|
||||||
Text: "More ways to discover videos on Twitter are here!\n\nNow on iOS, videos on your timeline will open in our full screen immersive video player, where you can swipe up to keep discovering more content. https://t.co/XI2vM8DKXA",
|
Text: "More ways to discover videos on Twitter are here!\n\nNow on iOS, videos on your timeline will open in our full screen immersive video player, where you can swipe up to keep discovering more content. https://pbs.twimg.com/media/FeUJKdnXEAEFe2j.jpg",
|
||||||
Timestamp: 1664982561,
|
Timestamp: 1664982561,
|
||||||
UserID: "17874544",
|
UserID: "17874544",
|
||||||
Username: "Support",
|
Username: "Support",
|
||||||
|
|
@ -147,7 +147,7 @@ func TestGetTweetWithGIF(t *testing.T) {
|
||||||
ID: "1517535384833605632",
|
ID: "1517535384833605632",
|
||||||
Name: "Support",
|
Name: "Support",
|
||||||
PermanentURL: "https://twitter.com/Support/status/1517535384833605632",
|
PermanentURL: "https://twitter.com/Support/status/1517535384833605632",
|
||||||
Text: "Video captions or no captions, it’s now easier to choose for some of you on iOS, and soon on Android.\n\nOn videos that have captions available, we’re testing the option to turn captions off/on with a new “CC” button. https://t.co/Q2Q2Wmr78U",
|
Text: "Video captions or no captions, it’s now easier to choose for some of you on iOS, and soon on Android.\n\nOn videos that have captions available, we’re testing the option to turn captions off/on with a new “CC” button. https://pbs.twimg.com/tweet_video_thumb/FQ9eXEhXEAA-haj.jpg",
|
||||||
Timestamp: 1650643604,
|
Timestamp: 1650643604,
|
||||||
UserID: "17874544",
|
UserID: "17874544",
|
||||||
Username: "Support",
|
Username: "Support",
|
||||||
|
|
@ -170,7 +170,7 @@ func TestGetTweetWithPhotoAndGIF(t *testing.T) {
|
||||||
Name: "Spaces",
|
Name: "Spaces",
|
||||||
PermanentURL: "https://twitter.com/XSpaces/status/1583186305722507265",
|
PermanentURL: "https://twitter.com/XSpaces/status/1583186305722507265",
|
||||||
Photos: []twitterscraper.Photo{{ID: "1583186295626539020", URL: "https://pbs.twimg.com/media/FfibjDwWIAwvbtJ.jpg"}},
|
Photos: []twitterscraper.Photo{{ID: "1583186295626539020", URL: "https://pbs.twimg.com/media/FfibjDwWIAwvbtJ.jpg"}},
|
||||||
Text: "“we need to talk” \n\nirl vs on Spaces https://t.co/hrflPpbpif",
|
Text: "“we need to talk” \n\nirl vs on Spaces https://pbs.twimg.com/tweet_video_thumb/FfibjDnWIBIt5fn.jpg",
|
||||||
Timestamp: 1666296004,
|
Timestamp: 1666296004,
|
||||||
UserID: "1065249714214457345",
|
UserID: "1065249714214457345",
|
||||||
Username: "XSpaces",
|
Username: "XSpaces",
|
||||||
|
|
@ -197,7 +197,7 @@ func TestTweetMentions(t *testing.T) {
|
||||||
func TestQuotedAndReply(t *testing.T) {
|
func TestQuotedAndReply(t *testing.T) {
|
||||||
sample := &twitterscraper.Tweet{
|
sample := &twitterscraper.Tweet{
|
||||||
ConversationID: "1237110546383724547",
|
ConversationID: "1237110546383724547",
|
||||||
HTML: "The Easiest Problem Everyone Gets Wrong <br><br>[new video] --> <a href=\"https://youtu.be/ytfCdqWhmdg\">https://t.co/YdaeDYmPAU</a> <br><a href=\"https://t.co/iKu4Xs6o2V\"><img src=\"https://pbs.twimg.com/media/ESsZa9AXgAIAYnF.jpg\"/></a>",
|
HTML: "The Easiest Problem Everyone Gets Wrong <br><br>[new video] --> <a href=\"https://youtu.be/ytfCdqWhmdg\">https://youtu.be/ytfCdqWhmdg</a> <br><a href=\"https://t.co/iKu4Xs6o2V\"><img src=\"https://pbs.twimg.com/media/ESsZa9AXgAIAYnF.jpg\"/></a>",
|
||||||
ID: "1237110546383724547",
|
ID: "1237110546383724547",
|
||||||
Likes: 485,
|
Likes: 485,
|
||||||
Name: "Vsauce2",
|
Name: "Vsauce2",
|
||||||
|
|
@ -208,7 +208,7 @@ func TestQuotedAndReply(t *testing.T) {
|
||||||
}},
|
}},
|
||||||
Replies: 12,
|
Replies: 12,
|
||||||
Retweets: 18,
|
Retweets: 18,
|
||||||
Text: "The Easiest Problem Everyone Gets Wrong \n\n[new video] --> https://t.co/YdaeDYmPAU https://t.co/iKu4Xs6o2V",
|
Text: "The Easiest Problem Everyone Gets Wrong \n\n[new video] --> https://youtu.be/ytfCdqWhmdg https://pbs.twimg.com/media/ESsZa9AXgAIAYnF.jpg",
|
||||||
Timestamp: 1583785113,
|
Timestamp: 1583785113,
|
||||||
URLs: []string{"https://youtu.be/ytfCdqWhmdg"},
|
URLs: []string{"https://youtu.be/ytfCdqWhmdg"},
|
||||||
UserID: "978944851",
|
UserID: "978944851",
|
||||||
|
|
@ -241,13 +241,13 @@ func TestQuotedAndReply(t *testing.T) {
|
||||||
func TestRetweet(t *testing.T) {
|
func TestRetweet(t *testing.T) {
|
||||||
sample := &twitterscraper.Tweet{
|
sample := &twitterscraper.Tweet{
|
||||||
ConversationID: "1758837061786779942",
|
ConversationID: "1758837061786779942",
|
||||||
HTML: "no ads, just bangers<br><br>aka your For You feed with Premium+<br><br>subscribe here → <a href=\"https://x.com/i/premium_sign_up\">https://t.co/APTO1t7kMk</a>",
|
HTML: "no ads, just bangers<br><br>aka your For You feed with Premium+<br><br>subscribe here → <a href=\"https://x.com/i/premium_sign_up\">https://x.com/i/premium_sign_up</a>",
|
||||||
ID: "1758837061786779942",
|
ID: "1758837061786779942",
|
||||||
URLs: []string{"https://x.com/i/premium_sign_up"},
|
URLs: []string{"https://x.com/i/premium_sign_up"},
|
||||||
IsSelfThread: false,
|
IsSelfThread: false,
|
||||||
Name: "Premium",
|
Name: "Premium",
|
||||||
PermanentURL: "https://twitter.com/premium/status/1758837061786779942",
|
PermanentURL: "https://twitter.com/premium/status/1758837061786779942",
|
||||||
Text: "no ads, just bangers\n\naka your For You feed with Premium+\n\nsubscribe here → https://t.co/APTO1t7kMk",
|
Text: "no ads, just bangers\n\naka your For You feed with Premium+\n\nsubscribe here → https://x.com/i/premium_sign_up",
|
||||||
Timestamp: 1708174407,
|
Timestamp: 1708174407,
|
||||||
UserID: "1399766153053061121",
|
UserID: "1399766153053061121",
|
||||||
Username: "premium",
|
Username: "premium",
|
||||||
|
|
|
||||||
116
types.go
116
types.go
|
|
@ -10,6 +10,14 @@ type (
|
||||||
Name string
|
Name string
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// Url represents a URL with display, expanded, and index data.
|
||||||
|
Url struct {
|
||||||
|
DisplayURL string `json:"display_url"`
|
||||||
|
ExpandedURL string `json:"expanded_url"`
|
||||||
|
URL string `json:"url"`
|
||||||
|
Indices []int `json:"indices"`
|
||||||
|
}
|
||||||
|
|
||||||
// Photo type.
|
// Photo type.
|
||||||
Photo struct {
|
Photo struct {
|
||||||
ID string
|
ID string
|
||||||
|
|
@ -91,32 +99,7 @@ type (
|
||||||
GIFs []GIF
|
GIFs []GIF
|
||||||
}
|
}
|
||||||
|
|
||||||
legacyTweet struct {
|
ExtendedMedia struct {
|
||||||
ConversationIDStr string `json:"conversation_id_str"`
|
|
||||||
CreatedAt string `json:"created_at"`
|
|
||||||
FavoriteCount int `json:"favorite_count"`
|
|
||||||
FullText string `json:"full_text"`
|
|
||||||
Entities struct {
|
|
||||||
Hashtags []struct {
|
|
||||||
Text string `json:"text"`
|
|
||||||
} `json:"hashtags"`
|
|
||||||
Media []struct {
|
|
||||||
MediaURLHttps string `json:"media_url_https"`
|
|
||||||
Type string `json:"type"`
|
|
||||||
URL string `json:"url"`
|
|
||||||
} `json:"media"`
|
|
||||||
URLs []struct {
|
|
||||||
ExpandedURL string `json:"expanded_url"`
|
|
||||||
URL string `json:"url"`
|
|
||||||
} `json:"urls"`
|
|
||||||
UserMentions []struct {
|
|
||||||
IDStr string `json:"id_str"`
|
|
||||||
Name string `json:"name"`
|
|
||||||
ScreenName string `json:"screen_name"`
|
|
||||||
} `json:"user_mentions"`
|
|
||||||
} `json:"entities"`
|
|
||||||
ExtendedEntities struct {
|
|
||||||
Media []struct {
|
|
||||||
IDStr string `json:"id_str"`
|
IDStr string `json:"id_str"`
|
||||||
MediaURLHttps string `json:"media_url_https"`
|
MediaURLHttps string `json:"media_url_https"`
|
||||||
ExtSensitiveMediaWarning struct {
|
ExtSensitiveMediaWarning struct {
|
||||||
|
|
@ -133,7 +116,31 @@ type (
|
||||||
URL string `json:"url"`
|
URL string `json:"url"`
|
||||||
} `json:"variants"`
|
} `json:"variants"`
|
||||||
} `json:"video_info"`
|
} `json:"video_info"`
|
||||||
|
}
|
||||||
|
|
||||||
|
legacyTweet struct {
|
||||||
|
ConversationIDStr string `json:"conversation_id_str"`
|
||||||
|
CreatedAt string `json:"created_at"`
|
||||||
|
FavoriteCount int `json:"favorite_count"`
|
||||||
|
FullText string `json:"full_text"`
|
||||||
|
Entities struct {
|
||||||
|
Hashtags []struct {
|
||||||
|
Text string `json:"text"`
|
||||||
|
} `json:"hashtags"`
|
||||||
|
Media []struct {
|
||||||
|
MediaURLHttps string `json:"media_url_https"`
|
||||||
|
Type string `json:"type"`
|
||||||
|
URL string `json:"url"`
|
||||||
} `json:"media"`
|
} `json:"media"`
|
||||||
|
URLs []Url `json:"urls"`
|
||||||
|
UserMentions []struct {
|
||||||
|
IDStr string `json:"id_str"`
|
||||||
|
Name string `json:"name"`
|
||||||
|
ScreenName string `json:"screen_name"`
|
||||||
|
} `json:"user_mentions"`
|
||||||
|
} `json:"entities"`
|
||||||
|
ExtendedEntities struct {
|
||||||
|
Media []ExtendedMedia `json:"media"`
|
||||||
} `json:"extended_entities"`
|
} `json:"extended_entities"`
|
||||||
IDStr string `json:"id_str"`
|
IDStr string `json:"id_str"`
|
||||||
InReplyToStatusIDStr string `json:"in_reply_to_status_id_str"`
|
InReplyToStatusIDStr string `json:"in_reply_to_status_id_str"`
|
||||||
|
|
@ -182,6 +189,21 @@ type (
|
||||||
Verified bool `json:"verified"`
|
Verified bool `json:"verified"`
|
||||||
FollowedBy bool `json:"followed_by"`
|
FollowedBy bool `json:"followed_by"`
|
||||||
Following bool `json:"following"`
|
Following bool `json:"following"`
|
||||||
|
CanDm bool `json:"can_dm"`
|
||||||
|
CanMediaTag bool `json:"can_media_tag"`
|
||||||
|
DefaultProfile bool `json:"default_profile"`
|
||||||
|
DefaultProfileImage bool `json:"default_profile_image"`
|
||||||
|
FastFollowersCount int `json:"fast_followers_count"`
|
||||||
|
HasCustomTimelines bool `json:"has_custom_timelines"`
|
||||||
|
IsTranslator bool `json:"is_translator"`
|
||||||
|
MediaCount int `json:"media_count"`
|
||||||
|
NeedsPhoneVerification bool `json:"needs_phone_verification"`
|
||||||
|
NormalFollowersCount int `json:"normal_followers_count"`
|
||||||
|
PossiblySensitive bool `json:"possibly_sensitive"`
|
||||||
|
ProfileInterstitialType string `json:"profile_interstitial_type"`
|
||||||
|
TranslatorType string `json:"translator_type"`
|
||||||
|
WantRetweets bool `json:"want_retweets"`
|
||||||
|
WithheldInCountries []string `json:"withheld_in_countries"`
|
||||||
}
|
}
|
||||||
|
|
||||||
legacyUserV2 struct {
|
legacyUserV2 struct {
|
||||||
|
|
@ -195,15 +217,10 @@ type (
|
||||||
Description string `json:"description"`
|
Description string `json:"description"`
|
||||||
Entities struct {
|
Entities struct {
|
||||||
Description struct {
|
Description struct {
|
||||||
Urls []interface{} `json:"urls"`
|
Urls []Url `json:"urls"`
|
||||||
} `json:"description"`
|
} `json:"description"`
|
||||||
URL struct {
|
URL struct {
|
||||||
Urls []struct {
|
Urls []Url `json:"urls"`
|
||||||
DisplayURL string `json:"display_url"`
|
|
||||||
ExpandedURL string `json:"expanded_url"`
|
|
||||||
URL string `json:"url"`
|
|
||||||
Indices []int `json:"indices"`
|
|
||||||
} `json:"urls"`
|
|
||||||
} `json:"url"`
|
} `json:"url"`
|
||||||
} `json:"entities"`
|
} `json:"entities"`
|
||||||
FastFollowersCount int `json:"fast_followers_count"`
|
FastFollowersCount int `json:"fast_followers_count"`
|
||||||
|
|
@ -246,4 +263,37 @@ type (
|
||||||
|
|
||||||
fetchProfileFunc func(query string, maxProfilesNbr int, cursor string) ([]*Profile, string, error)
|
fetchProfileFunc func(query string, maxProfilesNbr int, cursor string) ([]*Profile, string, error)
|
||||||
fetchTweetFunc func(query string, maxTweetsNbr int, cursor string) ([]*Tweet, string, error)
|
fetchTweetFunc func(query string, maxTweetsNbr int, cursor string) ([]*Tweet, string, error)
|
||||||
|
|
||||||
|
legacyExtendedProfile struct {
|
||||||
|
Birthdate struct {
|
||||||
|
Day int `json:"day"`
|
||||||
|
Month int `json:"month"`
|
||||||
|
Year int `json:"year"`
|
||||||
|
Visibility string `json:"visibility"`
|
||||||
|
YearVisibility string `json:"year_visibility"`
|
||||||
|
} `json:"birthdate"`
|
||||||
|
}
|
||||||
|
|
||||||
|
verificationInfo struct {
|
||||||
|
IsIdentityVerified bool `json:"is_identity_verified"`
|
||||||
|
Reason struct {
|
||||||
|
Description struct {
|
||||||
|
Text string `json:"text"`
|
||||||
|
Entities []struct {
|
||||||
|
FromIndex int `json:"from_index"`
|
||||||
|
ToIndex int `json:"to_index"`
|
||||||
|
Ref struct {
|
||||||
|
URL string `json:"url"`
|
||||||
|
URLType string `json:"url_type"`
|
||||||
|
} `json:"ref"`
|
||||||
|
} `json:"entities"`
|
||||||
|
} `json:"description"`
|
||||||
|
VerifiedSinceMsec string `json:"verified_since_msec"`
|
||||||
|
} `json:"reason"`
|
||||||
|
}
|
||||||
|
|
||||||
|
highlightsInfo struct {
|
||||||
|
CanHighlightTweets bool `json:"can_highlight_tweets"`
|
||||||
|
HighlightedTweets string `json:"highlighted_tweets"`
|
||||||
|
}
|
||||||
)
|
)
|
||||||
|
|
|
||||||
26
util.go
26
util.go
|
|
@ -157,6 +157,7 @@ func parseLegacyTweet(user *legacyUser, tweet *legacyTweet) *Tweet {
|
||||||
if tweetID == "" {
|
if tweetID == "" {
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
text := expandURLs(tweet.FullText, tweet.Entities.URLs, tweet.ExtendedEntities.Media)
|
||||||
username := user.ScreenName
|
username := user.ScreenName
|
||||||
name := user.Name
|
name := user.Name
|
||||||
tw := &Tweet{
|
tw := &Tweet{
|
||||||
|
|
@ -167,7 +168,7 @@ func parseLegacyTweet(user *legacyUser, tweet *legacyTweet) *Tweet {
|
||||||
PermanentURL: fmt.Sprintf("https://twitter.com/%s/status/%s", username, tweetID),
|
PermanentURL: fmt.Sprintf("https://twitter.com/%s/status/%s", username, tweetID),
|
||||||
Replies: tweet.ReplyCount,
|
Replies: tweet.ReplyCount,
|
||||||
Retweets: tweet.RetweetCount,
|
Retweets: tweet.RetweetCount,
|
||||||
Text: tweet.FullText,
|
Text: text,
|
||||||
UserID: tweet.UserIDStr,
|
UserID: tweet.UserIDStr,
|
||||||
Username: username,
|
Username: username,
|
||||||
}
|
}
|
||||||
|
|
@ -307,7 +308,7 @@ func parseLegacyTweet(user *legacyUser, tweet *legacyTweet) *Tweet {
|
||||||
tw.HTML = reTwitterURL.ReplaceAllStringFunc(tw.HTML, func(tco string) string {
|
tw.HTML = reTwitterURL.ReplaceAllStringFunc(tw.HTML, func(tco string) string {
|
||||||
for _, entity := range tweet.Entities.URLs {
|
for _, entity := range tweet.Entities.URLs {
|
||||||
if tco == entity.URL {
|
if tco == entity.URL {
|
||||||
return fmt.Sprintf(`<a href="%s">%s</a>`, entity.ExpandedURL, tco)
|
return fmt.Sprintf(`<a href="%s">%s</a>`, entity.ExpandedURL, entity.ExpandedURL)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
for _, entity := range tweet.ExtendedEntities.Media {
|
for _, entity := range tweet.ExtendedEntities.Media {
|
||||||
|
|
@ -364,6 +365,9 @@ func parseProfile(user legacyUser) Profile {
|
||||||
Username: user.ScreenName,
|
Username: user.ScreenName,
|
||||||
FollowedBy: user.FollowedBy,
|
FollowedBy: user.FollowedBy,
|
||||||
Following: user.Following,
|
Following: user.Following,
|
||||||
|
MediaCount: user.MediaCount,
|
||||||
|
FastFollowersCount: user.FastFollowersCount,
|
||||||
|
NormalFollowersCount: user.NormalFollowersCount,
|
||||||
}
|
}
|
||||||
|
|
||||||
tm, err := time.Parse(time.RubyDate, user.CreatedAt)
|
tm, err := time.Parse(time.RubyDate, user.CreatedAt)
|
||||||
|
|
@ -379,16 +383,32 @@ func parseProfile(user legacyUser) Profile {
|
||||||
return profile
|
return profile
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func expandURLs(text string, urls []Url, extendedMediaEntities []ExtendedMedia) string {
|
||||||
|
expandedText := text
|
||||||
|
for _, url := range urls {
|
||||||
|
expandedText = strings.ReplaceAll(expandedText, url.URL, url.ExpandedURL)
|
||||||
|
}
|
||||||
|
for _, entity := range extendedMediaEntities {
|
||||||
|
expandedText = strings.ReplaceAll(expandedText, entity.URL, entity.MediaURLHttps)
|
||||||
|
}
|
||||||
|
|
||||||
|
return expandedText
|
||||||
|
}
|
||||||
|
|
||||||
func parseProfileV2(user userResult) Profile {
|
func parseProfileV2(user userResult) Profile {
|
||||||
u := user.Legacy
|
u := user.Legacy
|
||||||
|
description := expandURLs(u.Description, u.Entities.Description.Urls, []ExtendedMedia{})
|
||||||
profile := Profile{
|
profile := Profile{
|
||||||
Avatar: u.ProfileImageURLHTTPS,
|
Avatar: u.ProfileImageURLHTTPS,
|
||||||
Banner: u.ProfileBannerURL,
|
Banner: u.ProfileBannerURL,
|
||||||
Biography: u.Description,
|
Biography: description,
|
||||||
FollowersCount: u.FollowersCount,
|
FollowersCount: u.FollowersCount,
|
||||||
FollowingCount: u.FavouritesCount,
|
FollowingCount: u.FavouritesCount,
|
||||||
FriendsCount: u.FriendsCount,
|
FriendsCount: u.FriendsCount,
|
||||||
IsVerified: u.Verified,
|
IsVerified: u.Verified,
|
||||||
|
IsBlueVerified: user.IsBlueVerified,
|
||||||
|
ProfileImageShape: user.ProfileImageShape,
|
||||||
|
HasGraduatedAccess: user.HasGraduatedAccess,
|
||||||
LikesCount: u.FavouritesCount,
|
LikesCount: u.FavouritesCount,
|
||||||
ListedCount: u.ListedCount,
|
ListedCount: u.ListedCount,
|
||||||
Location: u.Location,
|
Location: u.Location,
|
||||||
|
|
|
||||||
Loading…
Add table
Add a link
Reference in a new issue