Compare commits
No commits in common. "76cb95cd3ebe677a3d5afccdab557364c9f5e2bf" and "152a0a1c2c60df3812b3fd9cfcaf61905f68a80f" have entirely different histories.
76cb95cd3e
...
152a0a1c2c
7 changed files with 132 additions and 300 deletions
5
auth.go
5
auth.go
|
|
@ -24,9 +24,8 @@ const (
|
|||
oAuthURL = "https://api.twitter.com/oauth2/token"
|
||||
// Doesn't require x-client-transaction-id header in auth. x-rate-limit-limit: 2000
|
||||
bearerToken1 = "AAAAAAAAAAAAAAAAAAAAAFQODgEAAAAAVHTp76lzh3rFzcHbmHVvQxYYpTw%3DckAlMINMjmCwxUcaXbAN4XqJVdgMJaHqNOFgPMK0zN1qLqLQCF"
|
||||
// HOTFIX: Returns 404 error; Requires x-client-transaction-id header in auth.
|
||||
// bearerToken2 = "AAAAAAAAAAAAAAAAAAAAANRILgAAAAAAnNwIzUejRCOuH5E6I8xnZz4puTs%3D1Zv7ttfk8LF81IUq16cHjhLTvJu4FA33AGWWjCpTnA"
|
||||
bearerToken2 = "AAAAAAAAAAAAAAAAAAAAAFQODgEAAAAAVHTp76lzh3rFzcHbmHVvQxYYpTw%3DckAlMINMjmCwxUcaXbAN4XqJVdgMJaHqNOFgPMK0zN1qLqLQCF"
|
||||
// Requires x-client-transaction-id header in auth.
|
||||
bearerToken2 = "AAAAAAAAAAAAAAAAAAAAANRILgAAAAAAnNwIzUejRCOuH5E6I8xnZz4puTs%3D1Zv7ttfk8LF81IUq16cHjhLTvJu4FA33AGWWjCpTnA"
|
||||
appConsumerKey = "3nVuSoBZnx6U4vzUxf5w"
|
||||
appConsumerSecret = "Bcs59EFbbsdF6Sl9Ng71smgStWEGwXXKSjYvPVt7qys"
|
||||
)
|
||||
|
|
|
|||
68
profile.go
68
profile.go
|
|
@ -14,46 +14,38 @@ var cacheIDs sync.Map
|
|||
|
||||
// Profile of twitter user.
|
||||
type Profile struct {
|
||||
Avatar string
|
||||
Banner string
|
||||
Biography string
|
||||
Birthday string
|
||||
FollowersCount int
|
||||
FollowingCount int
|
||||
FriendsCount int
|
||||
IsPrivate bool
|
||||
IsVerified bool
|
||||
IsBlueVerified bool
|
||||
Joined *time.Time
|
||||
LikesCount int
|
||||
ListedCount int
|
||||
Location string
|
||||
Name string
|
||||
PinnedTweetIDs []string
|
||||
TweetsCount int
|
||||
URL string
|
||||
UserID string
|
||||
Username string
|
||||
Website string
|
||||
Sensitive bool
|
||||
Following bool
|
||||
FollowedBy bool
|
||||
MediaCount int
|
||||
FastFollowersCount int
|
||||
NormalFollowersCount int
|
||||
ProfileImageShape string
|
||||
HasGraduatedAccess bool
|
||||
CanHighlightTweets bool
|
||||
Avatar string
|
||||
Banner string
|
||||
Biography string
|
||||
Birthday string
|
||||
FollowersCount int
|
||||
FollowingCount int
|
||||
FriendsCount int
|
||||
IsPrivate bool
|
||||
IsVerified bool
|
||||
Joined *time.Time
|
||||
LikesCount int
|
||||
ListedCount int
|
||||
Location string
|
||||
Name string
|
||||
PinnedTweetIDs []string
|
||||
TweetsCount int
|
||||
URL string
|
||||
UserID string
|
||||
Username string
|
||||
Website string
|
||||
Sensitive bool
|
||||
Following bool
|
||||
FollowedBy bool
|
||||
}
|
||||
|
||||
type user struct {
|
||||
Data struct {
|
||||
User struct {
|
||||
Result struct {
|
||||
RestID string `json:"rest_id"`
|
||||
Legacy legacyUser `json:"legacy"`
|
||||
Message string `json:"message"`
|
||||
IsBlueVerified bool `json:"is_blue_verified"`
|
||||
RestID string `json:"rest_id"`
|
||||
Legacy legacyUser `json:"legacy"`
|
||||
Message string `json:"message"`
|
||||
} `json:"result"`
|
||||
} `json:"user"`
|
||||
} `json:"data"`
|
||||
|
|
@ -119,9 +111,7 @@ func (s *Scraper) GetProfile(username string) (Profile, error) {
|
|||
return Profile{}, fmt.Errorf("either @%s does not exist or is private", username)
|
||||
}
|
||||
|
||||
profile := parseProfile(jsn.Data.User.Result.Legacy)
|
||||
profile.IsBlueVerified = jsn.Data.User.Result.IsBlueVerified
|
||||
return profile, nil
|
||||
return parseProfile(jsn.Data.User.Result.Legacy), nil
|
||||
}
|
||||
|
||||
func (s *Scraper) GetProfileByID(userID string) (Profile, error) {
|
||||
|
|
@ -178,9 +168,7 @@ func (s *Scraper) GetProfileByID(userID string) (Profile, error) {
|
|||
return Profile{}, fmt.Errorf("either @%s does not exist or is private", userID)
|
||||
}
|
||||
|
||||
profile := parseProfile(jsn.Data.User.Result.Legacy)
|
||||
profile.IsBlueVerified = jsn.Data.User.Result.IsBlueVerified
|
||||
return profile, nil
|
||||
return parseProfile(jsn.Data.User.Result.Legacy), nil
|
||||
}
|
||||
|
||||
// GetUserIDByScreenName from API
|
||||
|
|
|
|||
|
|
@ -42,8 +42,6 @@ func TestGetProfile(t *testing.T) {
|
|||
cmpopts.IgnoreFields(twitterscraper.Profile{}, "LikesCount"),
|
||||
cmpopts.IgnoreFields(twitterscraper.Profile{}, "ListedCount"),
|
||||
cmpopts.IgnoreFields(twitterscraper.Profile{}, "TweetsCount"),
|
||||
cmpopts.IgnoreFields(twitterscraper.Profile{}, "MediaCount"),
|
||||
cmpopts.IgnoreFields(twitterscraper.Profile{}, "NormalFollowersCount"),
|
||||
}
|
||||
if diff := cmp.Diff(sample, profile, cmpOptions...); diff != "" {
|
||||
t.Error("Resulting profile does not match the sample", diff)
|
||||
|
|
@ -96,8 +94,6 @@ func TestGetProfilePrivate(t *testing.T) {
|
|||
cmpopts.IgnoreFields(twitterscraper.Profile{}, "LikesCount"),
|
||||
cmpopts.IgnoreFields(twitterscraper.Profile{}, "ListedCount"),
|
||||
cmpopts.IgnoreFields(twitterscraper.Profile{}, "TweetsCount"),
|
||||
cmpopts.IgnoreFields(twitterscraper.Profile{}, "MediaCount"),
|
||||
cmpopts.IgnoreFields(twitterscraper.Profile{}, "NormalFollowersCount"),
|
||||
}
|
||||
if diff := cmp.Diff(sample, profile, cmpOptions...); diff != "" {
|
||||
t.Error("Resulting profile does not match the sample", diff)
|
||||
|
|
@ -150,7 +146,7 @@ func TestGetProfileByID(t *testing.T) {
|
|||
}
|
||||
|
||||
func TestGetUserIDByScreenName(t *testing.T) {
|
||||
userID, err := testScraper.GetUserIDByScreenName("X")
|
||||
userID, err := testScraper.GetUserIDByScreenName("Twitter")
|
||||
if err != nil {
|
||||
t.Errorf("getUserByScreenName() error = %v", err)
|
||||
}
|
||||
|
|
|
|||
|
|
@ -1,7 +1,6 @@
|
|||
package twitterscraper
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"strconv"
|
||||
"strings"
|
||||
)
|
||||
|
|
@ -29,17 +28,6 @@ type tweet struct {
|
|||
Result *result `json:"result"`
|
||||
} `json:"quoted_status_result"`
|
||||
Legacy legacyTweet `json:"legacy"`
|
||||
Card struct {
|
||||
RestID string `json:"rest_id"`
|
||||
Legacy struct {
|
||||
BindingValues []struct {
|
||||
Key string `json:"key"`
|
||||
Value struct {
|
||||
StringValue string `json:"string_value"`
|
||||
} `json:"value"`
|
||||
} `json:"binding_values"`
|
||||
} `json:"legacy"`
|
||||
} `json:"card"`
|
||||
}
|
||||
|
||||
type result struct {
|
||||
|
|
@ -48,31 +36,6 @@ type result struct {
|
|||
Tweet tweet `json:"tweet"`
|
||||
}
|
||||
|
||||
type UnifiedCard struct {
|
||||
Type string `json:"type"`
|
||||
Components []string `json:"components"`
|
||||
MediaEntities map[string]struct {
|
||||
ID int64 `json:"id"`
|
||||
IDStr string `json:"id_str"`
|
||||
MediaURLHTTPS string `json:"media_url_https"`
|
||||
Type string `json:"type"`
|
||||
OriginalInfo struct {
|
||||
Width int `json:"width"`
|
||||
Height int `json:"height"`
|
||||
} `json:"original_info"`
|
||||
SourceUserID int64 `json:"source_user_id"`
|
||||
VideoInfo struct {
|
||||
AspectRatio []int `json:"aspect_ratio"`
|
||||
DurationMillis int `json:"duration_millis"`
|
||||
Variants []struct {
|
||||
Bitrate int `json:"bitrate,omitempty"`
|
||||
ContentType string `json:"content_type"`
|
||||
URL string `json:"url"`
|
||||
} `json:"variants"`
|
||||
} `json:"video_info"`
|
||||
} `json:"media_entities"`
|
||||
}
|
||||
|
||||
func (result *result) parse() *Tweet {
|
||||
if result.NoteTweet.NoteTweetResults.Result.Text != "" {
|
||||
result.Legacy.FullText = result.NoteTweet.NoteTweetResults.Result.Text
|
||||
|
|
@ -93,62 +56,18 @@ func (result *result) parse() *Tweet {
|
|||
if result.QuotedStatusResult.Result != nil {
|
||||
tw.QuotedStatus = result.QuotedStatusResult.Result.parse()
|
||||
}
|
||||
|
||||
// Get videos from cards
|
||||
for _, v := range result.Tweet.Card.Legacy.BindingValues {
|
||||
if v.Key == "unified_card" {
|
||||
var card UnifiedCard
|
||||
err := json.Unmarshal([]byte(v.Value.StringValue), &card)
|
||||
if err != nil {
|
||||
continue
|
||||
}
|
||||
|
||||
for _, media := range card.MediaEntities {
|
||||
if media.Type == "video" {
|
||||
var vid Video
|
||||
|
||||
vid.ID = media.IDStr
|
||||
vid.Preview = media.MediaURLHTTPS
|
||||
|
||||
var bitrate int
|
||||
for _, variant := range media.VideoInfo.Variants {
|
||||
if variant.ContentType == "video/mp4" {
|
||||
if variant.Bitrate > bitrate {
|
||||
bitrate = variant.Bitrate
|
||||
vid.URL = variant.URL
|
||||
}
|
||||
} else if variant.ContentType == "application/x-mpegURL" {
|
||||
vid.HLSURL = variant.URL
|
||||
}
|
||||
}
|
||||
|
||||
if vid.URL != "" {
|
||||
tw.Videos = append(tw.Videos, vid)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return tw
|
||||
}
|
||||
|
||||
type userResult struct {
|
||||
Typename string `json:"__typename"`
|
||||
ID string `json:"id"`
|
||||
RestID string `json:"rest_id"`
|
||||
AffiliatesHighlightedLabel struct{} `json:"affiliates_highlighted_label"`
|
||||
HasGraduatedAccess bool `json:"has_graduated_access"`
|
||||
IsBlueVerified bool `json:"is_blue_verified"`
|
||||
ProfileImageShape string `json:"profile_image_shape"`
|
||||
Legacy legacyUserV2 `json:"legacy"`
|
||||
LegacyExtendedProfile legacyExtendedProfile `json:"legacy_extended_profile"`
|
||||
IsProfileTranslatable bool `json:"is_profile_translatable"`
|
||||
VerificationInfo verificationInfo `json:"verification_info"`
|
||||
HighlightsInfo highlightsInfo `json:"highlights_info"`
|
||||
UserSeedTweetCount int `json:"user_seed_tweet_count"`
|
||||
PremiumGiftingEligible bool `json:"premium_gifting_eligible"`
|
||||
CreatorSubscriptionsCount int `json:"creator_subscriptions_count"`
|
||||
Typename string `json:"__typename"`
|
||||
ID string `json:"id"`
|
||||
RestID string `json:"rest_id"`
|
||||
AffiliatesHighlightedLabel struct{} `json:"affiliates_highlighted_label"`
|
||||
HasGraduatedAccess bool `json:"has_graduated_access"`
|
||||
IsBlueVerified bool `json:"is_blue_verified"`
|
||||
ProfileImageShape string `json:"profile_image_shape"`
|
||||
Legacy legacyUserV2 `json:"legacy"`
|
||||
}
|
||||
|
||||
func (result *userResult) parse() Profile {
|
||||
|
|
|
|||
|
|
@ -92,7 +92,7 @@ func TestGetTweetWithVideo(t *testing.T) {
|
|||
Name: "X",
|
||||
PermanentURL: "https://twitter.com/X/status/1697304622749086011",
|
||||
Photos: nil,
|
||||
Text: "on iOS & Android, you can now swipe to reply when you slide into their DMs https://pbs.twimg.com/amplify_video_thumb/1697304568550330368/img/BUlESpef6FmWV_j2.jpg",
|
||||
Text: "on iOS & Android, you can now swipe to reply when you slide into their DMs https://t.co/evuWpMfBxQ",
|
||||
Timestamp: 1693503931,
|
||||
UserID: "783214",
|
||||
Username: "X",
|
||||
|
|
@ -125,7 +125,7 @@ func TestGetTweetWithMultiplePhotos(t *testing.T) {
|
|||
URL: "https://pbs.twimg.com/media/FeUJKuxXEAAa6t7.jpg",
|
||||
},
|
||||
},
|
||||
Text: "More ways to discover videos on Twitter are here!\n\nNow on iOS, videos on your timeline will open in our full screen immersive video player, where you can swipe up to keep discovering more content. https://pbs.twimg.com/media/FeUJKdnXEAEFe2j.jpg",
|
||||
Text: "More ways to discover videos on Twitter are here!\n\nNow on iOS, videos on your timeline will open in our full screen immersive video player, where you can swipe up to keep discovering more content. https://t.co/XI2vM8DKXA",
|
||||
Timestamp: 1664982561,
|
||||
UserID: "17874544",
|
||||
Username: "Support",
|
||||
|
|
@ -147,7 +147,7 @@ func TestGetTweetWithGIF(t *testing.T) {
|
|||
ID: "1517535384833605632",
|
||||
Name: "Support",
|
||||
PermanentURL: "https://twitter.com/Support/status/1517535384833605632",
|
||||
Text: "Video captions or no captions, it’s now easier to choose for some of you on iOS, and soon on Android.\n\nOn videos that have captions available, we’re testing the option to turn captions off/on with a new “CC” button. https://pbs.twimg.com/tweet_video_thumb/FQ9eXEhXEAA-haj.jpg",
|
||||
Text: "Video captions or no captions, it’s now easier to choose for some of you on iOS, and soon on Android.\n\nOn videos that have captions available, we’re testing the option to turn captions off/on with a new “CC” button. https://t.co/Q2Q2Wmr78U",
|
||||
Timestamp: 1650643604,
|
||||
UserID: "17874544",
|
||||
Username: "Support",
|
||||
|
|
@ -170,7 +170,7 @@ func TestGetTweetWithPhotoAndGIF(t *testing.T) {
|
|||
Name: "Spaces",
|
||||
PermanentURL: "https://twitter.com/XSpaces/status/1583186305722507265",
|
||||
Photos: []twitterscraper.Photo{{ID: "1583186295626539020", URL: "https://pbs.twimg.com/media/FfibjDwWIAwvbtJ.jpg"}},
|
||||
Text: "“we need to talk” \n\nirl vs on Spaces https://pbs.twimg.com/tweet_video_thumb/FfibjDnWIBIt5fn.jpg",
|
||||
Text: "“we need to talk” \n\nirl vs on Spaces https://t.co/hrflPpbpif",
|
||||
Timestamp: 1666296004,
|
||||
UserID: "1065249714214457345",
|
||||
Username: "XSpaces",
|
||||
|
|
@ -197,7 +197,7 @@ func TestTweetMentions(t *testing.T) {
|
|||
func TestQuotedAndReply(t *testing.T) {
|
||||
sample := &twitterscraper.Tweet{
|
||||
ConversationID: "1237110546383724547",
|
||||
HTML: "The Easiest Problem Everyone Gets Wrong <br><br>[new video] --> <a href=\"https://youtu.be/ytfCdqWhmdg\">https://youtu.be/ytfCdqWhmdg</a> <br><a href=\"https://t.co/iKu4Xs6o2V\"><img src=\"https://pbs.twimg.com/media/ESsZa9AXgAIAYnF.jpg\"/></a>",
|
||||
HTML: "The Easiest Problem Everyone Gets Wrong <br><br>[new video] --> <a href=\"https://youtu.be/ytfCdqWhmdg\">https://t.co/YdaeDYmPAU</a> <br><a href=\"https://t.co/iKu4Xs6o2V\"><img src=\"https://pbs.twimg.com/media/ESsZa9AXgAIAYnF.jpg\"/></a>",
|
||||
ID: "1237110546383724547",
|
||||
Likes: 485,
|
||||
Name: "Vsauce2",
|
||||
|
|
@ -208,7 +208,7 @@ func TestQuotedAndReply(t *testing.T) {
|
|||
}},
|
||||
Replies: 12,
|
||||
Retweets: 18,
|
||||
Text: "The Easiest Problem Everyone Gets Wrong \n\n[new video] --> https://youtu.be/ytfCdqWhmdg https://pbs.twimg.com/media/ESsZa9AXgAIAYnF.jpg",
|
||||
Text: "The Easiest Problem Everyone Gets Wrong \n\n[new video] --> https://t.co/YdaeDYmPAU https://t.co/iKu4Xs6o2V",
|
||||
Timestamp: 1583785113,
|
||||
URLs: []string{"https://youtu.be/ytfCdqWhmdg"},
|
||||
UserID: "978944851",
|
||||
|
|
@ -241,13 +241,13 @@ func TestQuotedAndReply(t *testing.T) {
|
|||
func TestRetweet(t *testing.T) {
|
||||
sample := &twitterscraper.Tweet{
|
||||
ConversationID: "1758837061786779942",
|
||||
HTML: "no ads, just bangers<br><br>aka your For You feed with Premium+<br><br>subscribe here → <a href=\"https://x.com/i/premium_sign_up\">https://x.com/i/premium_sign_up</a>",
|
||||
HTML: "no ads, just bangers<br><br>aka your For You feed with Premium+<br><br>subscribe here → <a href=\"https://x.com/i/premium_sign_up\">https://t.co/APTO1t7kMk</a>",
|
||||
ID: "1758837061786779942",
|
||||
URLs: []string{"https://x.com/i/premium_sign_up"},
|
||||
IsSelfThread: false,
|
||||
Name: "Premium",
|
||||
PermanentURL: "https://twitter.com/premium/status/1758837061786779942",
|
||||
Text: "no ads, just bangers\n\naka your For You feed with Premium+\n\nsubscribe here → https://x.com/i/premium_sign_up",
|
||||
Text: "no ads, just bangers\n\naka your For You feed with Premium+\n\nsubscribe here → https://t.co/APTO1t7kMk",
|
||||
Timestamp: 1708174407,
|
||||
UserID: "1399766153053061121",
|
||||
Username: "premium",
|
||||
|
|
|
|||
140
types.go
140
types.go
|
|
@ -10,14 +10,6 @@ type (
|
|||
Name string
|
||||
}
|
||||
|
||||
// Url represents a URL with display, expanded, and index data.
|
||||
Url struct {
|
||||
DisplayURL string `json:"display_url"`
|
||||
ExpandedURL string `json:"expanded_url"`
|
||||
URL string `json:"url"`
|
||||
Indices []int `json:"indices"`
|
||||
}
|
||||
|
||||
// Photo type.
|
||||
Photo struct {
|
||||
ID string
|
||||
|
|
@ -99,25 +91,6 @@ type (
|
|||
GIFs []GIF
|
||||
}
|
||||
|
||||
ExtendedMedia struct {
|
||||
IDStr string `json:"id_str"`
|
||||
MediaURLHttps string `json:"media_url_https"`
|
||||
ExtSensitiveMediaWarning struct {
|
||||
AdultContent bool `json:"adult_content"`
|
||||
GraphicViolence bool `json:"graphic_violence"`
|
||||
Other bool `json:"other"`
|
||||
} `json:"ext_sensitive_media_warning"`
|
||||
Type string `json:"type"`
|
||||
URL string `json:"url"`
|
||||
VideoInfo struct {
|
||||
Variants []struct {
|
||||
Type string `json:"content_type"`
|
||||
Bitrate int `json:"bitrate"`
|
||||
URL string `json:"url"`
|
||||
} `json:"variants"`
|
||||
} `json:"video_info"`
|
||||
}
|
||||
|
||||
legacyTweet struct {
|
||||
ConversationIDStr string `json:"conversation_id_str"`
|
||||
CreatedAt string `json:"created_at"`
|
||||
|
|
@ -132,7 +105,10 @@ type (
|
|||
Type string `json:"type"`
|
||||
URL string `json:"url"`
|
||||
} `json:"media"`
|
||||
URLs []Url `json:"urls"`
|
||||
URLs []struct {
|
||||
ExpandedURL string `json:"expanded_url"`
|
||||
URL string `json:"url"`
|
||||
} `json:"urls"`
|
||||
UserMentions []struct {
|
||||
IDStr string `json:"id_str"`
|
||||
Name string `json:"name"`
|
||||
|
|
@ -140,7 +116,24 @@ type (
|
|||
} `json:"user_mentions"`
|
||||
} `json:"entities"`
|
||||
ExtendedEntities struct {
|
||||
Media []ExtendedMedia `json:"media"`
|
||||
Media []struct {
|
||||
IDStr string `json:"id_str"`
|
||||
MediaURLHttps string `json:"media_url_https"`
|
||||
ExtSensitiveMediaWarning struct {
|
||||
AdultContent bool `json:"adult_content"`
|
||||
GraphicViolence bool `json:"graphic_violence"`
|
||||
Other bool `json:"other"`
|
||||
} `json:"ext_sensitive_media_warning"`
|
||||
Type string `json:"type"`
|
||||
URL string `json:"url"`
|
||||
VideoInfo struct {
|
||||
Variants []struct {
|
||||
Type string `json:"content_type"`
|
||||
Bitrate int `json:"bitrate"`
|
||||
URL string `json:"url"`
|
||||
} `json:"variants"`
|
||||
} `json:"video_info"`
|
||||
} `json:"media"`
|
||||
} `json:"extended_entities"`
|
||||
IDStr string `json:"id_str"`
|
||||
InReplyToStatusIDStr string `json:"in_reply_to_status_id_str"`
|
||||
|
|
@ -173,37 +166,22 @@ type (
|
|||
} `json:"urls"`
|
||||
} `json:"url"`
|
||||
} `json:"entities"`
|
||||
FavouritesCount int `json:"favourites_count"`
|
||||
FollowersCount int `json:"followers_count"`
|
||||
FriendsCount int `json:"friends_count"`
|
||||
IDStr string `json:"id_str"`
|
||||
ListedCount int `json:"listed_count"`
|
||||
Name string `json:"name"`
|
||||
Location string `json:"location"`
|
||||
PinnedTweetIdsStr []string `json:"pinned_tweet_ids_str"`
|
||||
ProfileBannerURL string `json:"profile_banner_url"`
|
||||
ProfileImageURLHTTPS string `json:"profile_image_url_https"`
|
||||
Protected bool `json:"protected"`
|
||||
ScreenName string `json:"screen_name"`
|
||||
StatusesCount int `json:"statuses_count"`
|
||||
Verified bool `json:"verified"`
|
||||
FollowedBy bool `json:"followed_by"`
|
||||
Following bool `json:"following"`
|
||||
CanDm bool `json:"can_dm"`
|
||||
CanMediaTag bool `json:"can_media_tag"`
|
||||
DefaultProfile bool `json:"default_profile"`
|
||||
DefaultProfileImage bool `json:"default_profile_image"`
|
||||
FastFollowersCount int `json:"fast_followers_count"`
|
||||
HasCustomTimelines bool `json:"has_custom_timelines"`
|
||||
IsTranslator bool `json:"is_translator"`
|
||||
MediaCount int `json:"media_count"`
|
||||
NeedsPhoneVerification bool `json:"needs_phone_verification"`
|
||||
NormalFollowersCount int `json:"normal_followers_count"`
|
||||
PossiblySensitive bool `json:"possibly_sensitive"`
|
||||
ProfileInterstitialType string `json:"profile_interstitial_type"`
|
||||
TranslatorType string `json:"translator_type"`
|
||||
WantRetweets bool `json:"want_retweets"`
|
||||
WithheldInCountries []string `json:"withheld_in_countries"`
|
||||
FavouritesCount int `json:"favourites_count"`
|
||||
FollowersCount int `json:"followers_count"`
|
||||
FriendsCount int `json:"friends_count"`
|
||||
IDStr string `json:"id_str"`
|
||||
ListedCount int `json:"listed_count"`
|
||||
Name string `json:"name"`
|
||||
Location string `json:"location"`
|
||||
PinnedTweetIdsStr []string `json:"pinned_tweet_ids_str"`
|
||||
ProfileBannerURL string `json:"profile_banner_url"`
|
||||
ProfileImageURLHTTPS string `json:"profile_image_url_https"`
|
||||
Protected bool `json:"protected"`
|
||||
ScreenName string `json:"screen_name"`
|
||||
StatusesCount int `json:"statuses_count"`
|
||||
Verified bool `json:"verified"`
|
||||
FollowedBy bool `json:"followed_by"`
|
||||
Following bool `json:"following"`
|
||||
}
|
||||
|
||||
legacyUserV2 struct {
|
||||
|
|
@ -217,10 +195,15 @@ type (
|
|||
Description string `json:"description"`
|
||||
Entities struct {
|
||||
Description struct {
|
||||
Urls []Url `json:"urls"`
|
||||
Urls []interface{} `json:"urls"`
|
||||
} `json:"description"`
|
||||
URL struct {
|
||||
Urls []Url `json:"urls"`
|
||||
Urls []struct {
|
||||
DisplayURL string `json:"display_url"`
|
||||
ExpandedURL string `json:"expanded_url"`
|
||||
URL string `json:"url"`
|
||||
Indices []int `json:"indices"`
|
||||
} `json:"urls"`
|
||||
} `json:"url"`
|
||||
} `json:"entities"`
|
||||
FastFollowersCount int `json:"fast_followers_count"`
|
||||
|
|
@ -263,37 +246,4 @@ type (
|
|||
|
||||
fetchProfileFunc func(query string, maxProfilesNbr int, cursor string) ([]*Profile, string, error)
|
||||
fetchTweetFunc func(query string, maxTweetsNbr int, cursor string) ([]*Tweet, string, error)
|
||||
|
||||
legacyExtendedProfile struct {
|
||||
Birthdate struct {
|
||||
Day int `json:"day"`
|
||||
Month int `json:"month"`
|
||||
Year int `json:"year"`
|
||||
Visibility string `json:"visibility"`
|
||||
YearVisibility string `json:"year_visibility"`
|
||||
} `json:"birthdate"`
|
||||
}
|
||||
|
||||
verificationInfo struct {
|
||||
IsIdentityVerified bool `json:"is_identity_verified"`
|
||||
Reason struct {
|
||||
Description struct {
|
||||
Text string `json:"text"`
|
||||
Entities []struct {
|
||||
FromIndex int `json:"from_index"`
|
||||
ToIndex int `json:"to_index"`
|
||||
Ref struct {
|
||||
URL string `json:"url"`
|
||||
URLType string `json:"url_type"`
|
||||
} `json:"ref"`
|
||||
} `json:"entities"`
|
||||
} `json:"description"`
|
||||
VerifiedSinceMsec string `json:"verified_since_msec"`
|
||||
} `json:"reason"`
|
||||
}
|
||||
|
||||
highlightsInfo struct {
|
||||
CanHighlightTweets bool `json:"can_highlight_tweets"`
|
||||
HighlightedTweets string `json:"highlighted_tweets"`
|
||||
}
|
||||
)
|
||||
|
|
|
|||
100
util.go
100
util.go
|
|
@ -157,7 +157,6 @@ func parseLegacyTweet(user *legacyUser, tweet *legacyTweet) *Tweet {
|
|||
if tweetID == "" {
|
||||
return nil
|
||||
}
|
||||
text := expandURLs(tweet.FullText, tweet.Entities.URLs, tweet.ExtendedEntities.Media)
|
||||
username := user.ScreenName
|
||||
name := user.Name
|
||||
tw := &Tweet{
|
||||
|
|
@ -168,7 +167,7 @@ func parseLegacyTweet(user *legacyUser, tweet *legacyTweet) *Tweet {
|
|||
PermanentURL: fmt.Sprintf("https://twitter.com/%s/status/%s", username, tweetID),
|
||||
Replies: tweet.ReplyCount,
|
||||
Retweets: tweet.RetweetCount,
|
||||
Text: text,
|
||||
Text: tweet.FullText,
|
||||
UserID: tweet.UserIDStr,
|
||||
Username: username,
|
||||
}
|
||||
|
|
@ -308,7 +307,7 @@ func parseLegacyTweet(user *legacyUser, tweet *legacyTweet) *Tweet {
|
|||
tw.HTML = reTwitterURL.ReplaceAllStringFunc(tw.HTML, func(tco string) string {
|
||||
for _, entity := range tweet.Entities.URLs {
|
||||
if tco == entity.URL {
|
||||
return fmt.Sprintf(`<a href="%s">%s</a>`, entity.ExpandedURL, entity.ExpandedURL)
|
||||
return fmt.Sprintf(`<a href="%s">%s</a>`, entity.ExpandedURL, tco)
|
||||
}
|
||||
}
|
||||
for _, entity := range tweet.ExtendedEntities.Media {
|
||||
|
|
@ -346,28 +345,25 @@ func parseLegacyTweet(user *legacyUser, tweet *legacyTweet) *Tweet {
|
|||
|
||||
func parseProfile(user legacyUser) Profile {
|
||||
profile := Profile{
|
||||
Avatar: user.ProfileImageURLHTTPS,
|
||||
Banner: user.ProfileBannerURL,
|
||||
Biography: user.Description,
|
||||
FollowersCount: user.FollowersCount,
|
||||
FollowingCount: user.FavouritesCount,
|
||||
FriendsCount: user.FriendsCount,
|
||||
IsVerified: user.Verified,
|
||||
IsPrivate: user.Protected,
|
||||
LikesCount: user.FavouritesCount,
|
||||
ListedCount: user.ListedCount,
|
||||
Location: user.Location,
|
||||
Name: user.Name,
|
||||
PinnedTweetIDs: user.PinnedTweetIdsStr,
|
||||
TweetsCount: user.StatusesCount,
|
||||
URL: "https://twitter.com/" + user.ScreenName,
|
||||
UserID: user.IDStr,
|
||||
Username: user.ScreenName,
|
||||
FollowedBy: user.FollowedBy,
|
||||
Following: user.Following,
|
||||
MediaCount: user.MediaCount,
|
||||
FastFollowersCount: user.FastFollowersCount,
|
||||
NormalFollowersCount: user.NormalFollowersCount,
|
||||
Avatar: user.ProfileImageURLHTTPS,
|
||||
Banner: user.ProfileBannerURL,
|
||||
Biography: user.Description,
|
||||
FollowersCount: user.FollowersCount,
|
||||
FollowingCount: user.FavouritesCount,
|
||||
FriendsCount: user.FriendsCount,
|
||||
IsVerified: user.Verified,
|
||||
IsPrivate: user.Protected,
|
||||
LikesCount: user.FavouritesCount,
|
||||
ListedCount: user.ListedCount,
|
||||
Location: user.Location,
|
||||
Name: user.Name,
|
||||
PinnedTweetIDs: user.PinnedTweetIdsStr,
|
||||
TweetsCount: user.StatusesCount,
|
||||
URL: "https://twitter.com/" + user.ScreenName,
|
||||
UserID: user.IDStr,
|
||||
Username: user.ScreenName,
|
||||
FollowedBy: user.FollowedBy,
|
||||
Following: user.Following,
|
||||
}
|
||||
|
||||
tm, err := time.Parse(time.RubyDate, user.CreatedAt)
|
||||
|
|
@ -383,44 +379,28 @@ func parseProfile(user legacyUser) Profile {
|
|||
return profile
|
||||
}
|
||||
|
||||
func expandURLs(text string, urls []Url, extendedMediaEntities []ExtendedMedia) string {
|
||||
expandedText := text
|
||||
for _, url := range urls {
|
||||
expandedText = strings.ReplaceAll(expandedText, url.URL, url.ExpandedURL)
|
||||
}
|
||||
for _, entity := range extendedMediaEntities {
|
||||
expandedText = strings.ReplaceAll(expandedText, entity.URL, entity.MediaURLHttps)
|
||||
}
|
||||
|
||||
return expandedText
|
||||
}
|
||||
|
||||
func parseProfileV2(user userResult) Profile {
|
||||
u := user.Legacy
|
||||
description := expandURLs(u.Description, u.Entities.Description.Urls, []ExtendedMedia{})
|
||||
profile := Profile{
|
||||
Avatar: u.ProfileImageURLHTTPS,
|
||||
Banner: u.ProfileBannerURL,
|
||||
Biography: description,
|
||||
FollowersCount: u.FollowersCount,
|
||||
FollowingCount: u.FavouritesCount,
|
||||
FriendsCount: u.FriendsCount,
|
||||
IsVerified: u.Verified,
|
||||
IsBlueVerified: user.IsBlueVerified,
|
||||
ProfileImageShape: user.ProfileImageShape,
|
||||
HasGraduatedAccess: user.HasGraduatedAccess,
|
||||
LikesCount: u.FavouritesCount,
|
||||
ListedCount: u.ListedCount,
|
||||
Location: u.Location,
|
||||
Name: u.Name,
|
||||
PinnedTweetIDs: u.PinnedTweetIdsStr,
|
||||
TweetsCount: u.StatusesCount,
|
||||
URL: "https://twitter.com/" + u.ScreenName,
|
||||
UserID: user.ID,
|
||||
Username: u.ScreenName,
|
||||
Sensitive: u.PossiblySensitive,
|
||||
Following: u.Following,
|
||||
FollowedBy: u.FollowedBy,
|
||||
Avatar: u.ProfileImageURLHTTPS,
|
||||
Banner: u.ProfileBannerURL,
|
||||
Biography: u.Description,
|
||||
FollowersCount: u.FollowersCount,
|
||||
FollowingCount: u.FavouritesCount,
|
||||
FriendsCount: u.FriendsCount,
|
||||
IsVerified: u.Verified,
|
||||
LikesCount: u.FavouritesCount,
|
||||
ListedCount: u.ListedCount,
|
||||
Location: u.Location,
|
||||
Name: u.Name,
|
||||
PinnedTweetIDs: u.PinnedTweetIdsStr,
|
||||
TweetsCount: u.StatusesCount,
|
||||
URL: "https://twitter.com/" + u.ScreenName,
|
||||
UserID: user.ID,
|
||||
Username: u.ScreenName,
|
||||
Sensitive: u.PossiblySensitive,
|
||||
Following: u.Following,
|
||||
FollowedBy: u.FollowedBy,
|
||||
}
|
||||
|
||||
tm, err := time.Parse(time.RubyDate, u.CreatedAt)
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue