lain/twitter-scrapper
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity

Compare commits

base: lain:76cb95cd3ebe677a3d5afccdab557364c9f5e2bf
Branches Tags
lain:master
..
compare: lain:152a0a1c2c60df3812b3fd9cfcaf61905f68a80f
Branches Tags
lain:master

No commits in common. "76cb95cd3ebe677a3d5afccdab557364c9f5e2bf" and "152a0a1c2c60df3812b3fd9cfcaf61905f68a80f" have entirely different histories.
76cb95cd3e ... 152a0a1c2c

7 changed files with 132 additions and 300 deletions
Download patch file Download diff file Expand all files Collapse all files

5
auth.go
View file

@ -24,9 +24,8 @@ const (
oAuthURL = "https://api.twitter.com/oauth2/token" oAuthURL = "https://api.twitter.com/oauth2/token"
// Doesn't require x-client-transaction-id header in auth. x-rate-limit-limit: 2000 // Doesn't require x-client-transaction-id header in auth. x-rate-limit-limit: 2000
bearerToken1 = "AAAAAAAAAAAAAAAAAAAAAFQODgEAAAAAVHTp76lzh3rFzcHbmHVvQxYYpTw%3DckAlMINMjmCwxUcaXbAN4XqJVdgMJaHqNOFgPMK0zN1qLqLQCF" bearerToken1 = "AAAAAAAAAAAAAAAAAAAAAFQODgEAAAAAVHTp76lzh3rFzcHbmHVvQxYYpTw%3DckAlMINMjmCwxUcaXbAN4XqJVdgMJaHqNOFgPMK0zN1qLqLQCF"
// HOTFIX: Returns 404 error; Requires x-client-transaction-id header in auth. // Requires x-client-transaction-id header in auth.
// bearerToken2 = "AAAAAAAAAAAAAAAAAAAAANRILgAAAAAAnNwIzUejRCOuH5E6I8xnZz4puTs%3D1Zv7ttfk8LF81IUq16cHjhLTvJu4FA33AGWWjCpTnA" bearerToken2 = "AAAAAAAAAAAAAAAAAAAAANRILgAAAAAAnNwIzUejRCOuH5E6I8xnZz4puTs%3D1Zv7ttfk8LF81IUq16cHjhLTvJu4FA33AGWWjCpTnA"
bearerToken2 = "AAAAAAAAAAAAAAAAAAAAAFQODgEAAAAAVHTp76lzh3rFzcHbmHVvQxYYpTw%3DckAlMINMjmCwxUcaXbAN4XqJVdgMJaHqNOFgPMK0zN1qLqLQCF"
appConsumerKey = "3nVuSoBZnx6U4vzUxf5w" appConsumerKey = "3nVuSoBZnx6U4vzUxf5w"
appConsumerSecret = "Bcs59EFbbsdF6Sl9Ng71smgStWEGwXXKSjYvPVt7qys" appConsumerSecret = "Bcs59EFbbsdF6Sl9Ng71smgStWEGwXXKSjYvPVt7qys"
) )

68
profile.go
View file

@ -14,46 +14,38 @@ var cacheIDs sync.Map
// Profile of twitter user. // Profile of twitter user.
type Profile struct { type Profile struct {
Avatar string Avatar string
Banner string Banner string
Biography string Biography string
Birthday string Birthday string
FollowersCount int FollowersCount int
FollowingCount int FollowingCount int
FriendsCount int FriendsCount int
IsPrivate bool IsPrivate bool
IsVerified bool IsVerified bool
IsBlueVerified bool Joined *time.Time
Joined *time.Time LikesCount int
LikesCount int ListedCount int
ListedCount int Location string
Location string Name string
Name string PinnedTweetIDs []string
PinnedTweetIDs []string TweetsCount int
TweetsCount int URL string
URL string UserID string
UserID string Username string
Username string Website string
Website string Sensitive bool
Sensitive bool Following bool
Following bool FollowedBy bool
FollowedBy bool
MediaCount int
FastFollowersCount int
NormalFollowersCount int
ProfileImageShape string
HasGraduatedAccess bool
CanHighlightTweets bool
} }
type user struct { type user struct {
Data struct { Data struct {
User struct { User struct {
Result struct { Result struct {
RestID string `json:"rest_id"` RestID string `json:"rest_id"`
Legacy legacyUser `json:"legacy"` Legacy legacyUser `json:"legacy"`
Message string `json:"message"` Message string `json:"message"`
IsBlueVerified bool `json:"is_blue_verified"`
} `json:"result"` } `json:"result"`
} `json:"user"` } `json:"user"`
} `json:"data"` } `json:"data"`
@ -119,9 +111,7 @@ func (s *Scraper) GetProfile(username string) (Profile, error) {
return Profile{}, fmt.Errorf("either @%s does not exist or is private", username) return Profile{}, fmt.Errorf("either @%s does not exist or is private", username)
} }
profile := parseProfile(jsn.Data.User.Result.Legacy) return parseProfile(jsn.Data.User.Result.Legacy), nil
profile.IsBlueVerified = jsn.Data.User.Result.IsBlueVerified
return profile, nil
} }
func (s *Scraper) GetProfileByID(userID string) (Profile, error) { func (s *Scraper) GetProfileByID(userID string) (Profile, error) {
@ -178,9 +168,7 @@ func (s *Scraper) GetProfileByID(userID string) (Profile, error) {
return Profile{}, fmt.Errorf("either @%s does not exist or is private", userID) return Profile{}, fmt.Errorf("either @%s does not exist or is private", userID)
} }
profile := parseProfile(jsn.Data.User.Result.Legacy) return parseProfile(jsn.Data.User.Result.Legacy), nil
profile.IsBlueVerified = jsn.Data.User.Result.IsBlueVerified
return profile, nil
} }
// GetUserIDByScreenName from API // GetUserIDByScreenName from API

6
profile_test.go
View file

@ -42,8 +42,6 @@ func TestGetProfile(t *testing.T) {
cmpopts.IgnoreFields(twitterscraper.Profile{}, "LikesCount"), cmpopts.IgnoreFields(twitterscraper.Profile{}, "LikesCount"),
cmpopts.IgnoreFields(twitterscraper.Profile{}, "ListedCount"), cmpopts.IgnoreFields(twitterscraper.Profile{}, "ListedCount"),
cmpopts.IgnoreFields(twitterscraper.Profile{}, "TweetsCount"), cmpopts.IgnoreFields(twitterscraper.Profile{}, "TweetsCount"),
cmpopts.IgnoreFields(twitterscraper.Profile{}, "MediaCount"),
cmpopts.IgnoreFields(twitterscraper.Profile{}, "NormalFollowersCount"),
} }
if diff := cmp.Diff(sample, profile, cmpOptions...); diff != "" { if diff := cmp.Diff(sample, profile, cmpOptions...); diff != "" {
t.Error("Resulting profile does not match the sample", diff) t.Error("Resulting profile does not match the sample", diff)
@ -96,8 +94,6 @@ func TestGetProfilePrivate(t *testing.T) {
cmpopts.IgnoreFields(twitterscraper.Profile{}, "LikesCount"), cmpopts.IgnoreFields(twitterscraper.Profile{}, "LikesCount"),
cmpopts.IgnoreFields(twitterscraper.Profile{}, "ListedCount"), cmpopts.IgnoreFields(twitterscraper.Profile{}, "ListedCount"),
cmpopts.IgnoreFields(twitterscraper.Profile{}, "TweetsCount"), cmpopts.IgnoreFields(twitterscraper.Profile{}, "TweetsCount"),
cmpopts.IgnoreFields(twitterscraper.Profile{}, "MediaCount"),
cmpopts.IgnoreFields(twitterscraper.Profile{}, "NormalFollowersCount"),
} }
if diff := cmp.Diff(sample, profile, cmpOptions...); diff != "" { if diff := cmp.Diff(sample, profile, cmpOptions...); diff != "" {
t.Error("Resulting profile does not match the sample", diff) t.Error("Resulting profile does not match the sample", diff)
@ -150,7 +146,7 @@ func TestGetProfileByID(t *testing.T) {
} }
func TestGetUserIDByScreenName(t *testing.T) { func TestGetUserIDByScreenName(t *testing.T) {
userID, err := testScraper.GetUserIDByScreenName("X") userID, err := testScraper.GetUserIDByScreenName("Twitter")
if err != nil { if err != nil {
t.Errorf("getUserByScreenName() error = %v", err) t.Errorf("getUserByScreenName() error = %v", err)
} }

97
timeline_v2.go
View file

@ -1,7 +1,6 @@
package twitterscraper package twitterscraper
import ( import (
"encoding/json"
"strconv" "strconv"
"strings" "strings"
) )
@ -29,17 +28,6 @@ type tweet struct {
Result *result `json:"result"` Result *result `json:"result"`
} `json:"quoted_status_result"` } `json:"quoted_status_result"`
Legacy legacyTweet `json:"legacy"` Legacy legacyTweet `json:"legacy"`
Card struct {
RestID string `json:"rest_id"`
Legacy struct {
BindingValues []struct {
Key string `json:"key"`
Value struct {
StringValue string `json:"string_value"`
} `json:"value"`
} `json:"binding_values"`
} `json:"legacy"`
} `json:"card"`
} }
type result struct { type result struct {
@ -48,31 +36,6 @@ type result struct {
Tweet tweet `json:"tweet"` Tweet tweet `json:"tweet"`
} }
type UnifiedCard struct {
Type string `json:"type"`
Components []string `json:"components"`
MediaEntities map[string]struct {
ID int64 `json:"id"`
IDStr string `json:"id_str"`
MediaURLHTTPS string `json:"media_url_https"`
Type string `json:"type"`
OriginalInfo struct {
Width int `json:"width"`
Height int `json:"height"`
} `json:"original_info"`
SourceUserID int64 `json:"source_user_id"`
VideoInfo struct {
AspectRatio []int `json:"aspect_ratio"`
DurationMillis int `json:"duration_millis"`
Variants []struct {
Bitrate int `json:"bitrate,omitempty"`
ContentType string `json:"content_type"`
URL string `json:"url"`
} `json:"variants"`
} `json:"video_info"`
} `json:"media_entities"`
}
func (result *result) parse() *Tweet { func (result *result) parse() *Tweet {
if result.NoteTweet.NoteTweetResults.Result.Text != "" { if result.NoteTweet.NoteTweetResults.Result.Text != "" {
result.Legacy.FullText = result.NoteTweet.NoteTweetResults.Result.Text result.Legacy.FullText = result.NoteTweet.NoteTweetResults.Result.Text
@ -93,62 +56,18 @@ func (result *result) parse() *Tweet {
if result.QuotedStatusResult.Result != nil { if result.QuotedStatusResult.Result != nil {
tw.QuotedStatus = result.QuotedStatusResult.Result.parse() tw.QuotedStatus = result.QuotedStatusResult.Result.parse()
} }
// Get videos from cards
for _, v := range result.Tweet.Card.Legacy.BindingValues {
if v.Key == "unified_card" {
var card UnifiedCard
err := json.Unmarshal([]byte(v.Value.StringValue), &card)
if err != nil {
continue
}
for _, media := range card.MediaEntities {
if media.Type == "video" {
var vid Video
vid.ID = media.IDStr
vid.Preview = media.MediaURLHTTPS
var bitrate int
for _, variant := range media.VideoInfo.Variants {
if variant.ContentType == "video/mp4" {
if variant.Bitrate > bitrate {
bitrate = variant.Bitrate
vid.URL = variant.URL
}
} else if variant.ContentType == "application/x-mpegURL" {
vid.HLSURL = variant.URL
}
}
if vid.URL != "" {
tw.Videos = append(tw.Videos, vid)
}
}
}
}
}
return tw return tw
} }
type userResult struct { type userResult struct {
Typename string `json:"__typename"` Typename string `json:"__typename"`
ID string `json:"id"` ID string `json:"id"`
RestID string `json:"rest_id"` RestID string `json:"rest_id"`
AffiliatesHighlightedLabel struct{} `json:"affiliates_highlighted_label"` AffiliatesHighlightedLabel struct{} `json:"affiliates_highlighted_label"`
HasGraduatedAccess bool `json:"has_graduated_access"` HasGraduatedAccess bool `json:"has_graduated_access"`
IsBlueVerified bool `json:"is_blue_verified"` IsBlueVerified bool `json:"is_blue_verified"`
ProfileImageShape string `json:"profile_image_shape"` ProfileImageShape string `json:"profile_image_shape"`
Legacy legacyUserV2 `json:"legacy"` Legacy legacyUserV2 `json:"legacy"`
LegacyExtendedProfile legacyExtendedProfile `json:"legacy_extended_profile"`
IsProfileTranslatable bool `json:"is_profile_translatable"`
VerificationInfo verificationInfo `json:"verification_info"`
HighlightsInfo highlightsInfo `json:"highlights_info"`
UserSeedTweetCount int `json:"user_seed_tweet_count"`
PremiumGiftingEligible bool `json:"premium_gifting_eligible"`
CreatorSubscriptionsCount int `json:"creator_subscriptions_count"`
} }
func (result *userResult) parse() Profile { func (result *userResult) parse() Profile {

16
tweets_test.go
View file

@ -92,7 +92,7 @@ func TestGetTweetWithVideo(t *testing.T) {
Name: "X", Name: "X",
PermanentURL: "https://twitter.com/X/status/1697304622749086011", PermanentURL: "https://twitter.com/X/status/1697304622749086011",
Photos: nil, Photos: nil,
Text: "on iOS & Android, you can now swipe to reply when you slide into their DMs https://pbs.twimg.com/amplify_video_thumb/1697304568550330368/img/BUlESpef6FmWV_j2.jpg", Text: "on iOS & Android, you can now swipe to reply when you slide into their DMs https://t.co/evuWpMfBxQ",
Timestamp: 1693503931, Timestamp: 1693503931,
UserID: "783214", UserID: "783214",
Username: "X", Username: "X",
@ -125,7 +125,7 @@ func TestGetTweetWithMultiplePhotos(t *testing.T) {
URL: "https://pbs.twimg.com/media/FeUJKuxXEAAa6t7.jpg", URL: "https://pbs.twimg.com/media/FeUJKuxXEAAa6t7.jpg",
}, },
}, },
Text: "More ways to discover videos on Twitter are here!\n\nNow on iOS, videos on your timeline will open in our full screen immersive video player, where you can swipe up to keep discovering more content. https://pbs.twimg.com/media/FeUJKdnXEAEFe2j.jpg", Text: "More ways to discover videos on Twitter are here!\n\nNow on iOS, videos on your timeline will open in our full screen immersive video player, where you can swipe up to keep discovering more content. https://t.co/XI2vM8DKXA",
Timestamp: 1664982561, Timestamp: 1664982561,
UserID: "17874544", UserID: "17874544",
Username: "Support", Username: "Support",
@ -147,7 +147,7 @@ func TestGetTweetWithGIF(t *testing.T) {
ID: "1517535384833605632", ID: "1517535384833605632",
Name: "Support", Name: "Support",
PermanentURL: "https://twitter.com/Support/status/1517535384833605632", PermanentURL: "https://twitter.com/Support/status/1517535384833605632",
Text: "Video captions or no captions, it’s now easier to choose for some of you on iOS, and soon on Android.\n\nOn videos that have captions available, we’re testing the option to turn captions off/on with a new “CC” button. https://pbs.twimg.com/tweet_video_thumb/FQ9eXEhXEAA-haj.jpg", Text: "Video captions or no captions, it’s now easier to choose for some of you on iOS, and soon on Android.\n\nOn videos that have captions available, we’re testing the option to turn captions off/on with a new “CC” button. https://t.co/Q2Q2Wmr78U",
Timestamp: 1650643604, Timestamp: 1650643604,
UserID: "17874544", UserID: "17874544",
Username: "Support", Username: "Support",
@ -170,7 +170,7 @@ func TestGetTweetWithPhotoAndGIF(t *testing.T) {
Name: "Spaces", Name: "Spaces",
PermanentURL: "https://twitter.com/XSpaces/status/1583186305722507265", PermanentURL: "https://twitter.com/XSpaces/status/1583186305722507265",
Photos: []twitterscraper.Photo{{ID: "1583186295626539020", URL: "https://pbs.twimg.com/media/FfibjDwWIAwvbtJ.jpg"}}, Photos: []twitterscraper.Photo{{ID: "1583186295626539020", URL: "https://pbs.twimg.com/media/FfibjDwWIAwvbtJ.jpg"}},
Text: "“we need to talk” \n\nirl vs on Spaces https://pbs.twimg.com/tweet_video_thumb/FfibjDnWIBIt5fn.jpg", Text: "“we need to talk” \n\nirl vs on Spaces https://t.co/hrflPpbpif",
Timestamp: 1666296004, Timestamp: 1666296004,
UserID: "1065249714214457345", UserID: "1065249714214457345",
Username: "XSpaces", Username: "XSpaces",
@ -197,7 +197,7 @@ func TestTweetMentions(t *testing.T) {
func TestQuotedAndReply(t *testing.T) { func TestQuotedAndReply(t *testing.T) {
sample := &twitterscraper.Tweet{ sample := &twitterscraper.Tweet{
ConversationID: "1237110546383724547", ConversationID: "1237110546383724547",
HTML: "The Easiest Problem Everyone Gets Wrong <br><br>[new video] --&gt; <a href=\"https://youtu.be/ytfCdqWhmdg\">https://youtu.be/ytfCdqWhmdg</a> <br><a href=\"https://t.co/iKu4Xs6o2V\"><img src=\"https://pbs.twimg.com/media/ESsZa9AXgAIAYnF.jpg\"/></a>", HTML: "The Easiest Problem Everyone Gets Wrong <br><br>[new video] --&gt; <a href=\"https://youtu.be/ytfCdqWhmdg\">https://t.co/YdaeDYmPAU</a> <br><a href=\"https://t.co/iKu4Xs6o2V\"><img src=\"https://pbs.twimg.com/media/ESsZa9AXgAIAYnF.jpg\"/></a>",
ID: "1237110546383724547", ID: "1237110546383724547",
Likes: 485, Likes: 485,
Name: "Vsauce2", Name: "Vsauce2",
@ -208,7 +208,7 @@ func TestQuotedAndReply(t *testing.T) {
}}, }},
Replies: 12, Replies: 12,
Retweets: 18, Retweets: 18,
Text: "The Easiest Problem Everyone Gets Wrong \n\n[new video] --&gt; https://youtu.be/ytfCdqWhmdg https://pbs.twimg.com/media/ESsZa9AXgAIAYnF.jpg", Text: "The Easiest Problem Everyone Gets Wrong \n\n[new video] --&gt; https://t.co/YdaeDYmPAU https://t.co/iKu4Xs6o2V",
Timestamp: 1583785113, Timestamp: 1583785113,
URLs: []string{"https://youtu.be/ytfCdqWhmdg"}, URLs: []string{"https://youtu.be/ytfCdqWhmdg"},
UserID: "978944851", UserID: "978944851",
@ -241,13 +241,13 @@ func TestQuotedAndReply(t *testing.T) {
func TestRetweet(t *testing.T) { func TestRetweet(t *testing.T) {
sample := &twitterscraper.Tweet{ sample := &twitterscraper.Tweet{
ConversationID: "1758837061786779942", ConversationID: "1758837061786779942",
HTML: "no ads, just bangers<br><br>aka your For You feed with Premium+<br><br>subscribe here → <a href=\"https://x.com/i/premium_sign_up\">https://x.com/i/premium_sign_up</a>", HTML: "no ads, just bangers<br><br>aka your For You feed with Premium+<br><br>subscribe here → <a href=\"https://x.com/i/premium_sign_up\">https://t.co/APTO1t7kMk</a>",
ID: "1758837061786779942", ID: "1758837061786779942",
URLs: []string{"https://x.com/i/premium_sign_up"}, URLs: []string{"https://x.com/i/premium_sign_up"},
IsSelfThread: false, IsSelfThread: false,
Name: "Premium", Name: "Premium",
PermanentURL: "https://twitter.com/premium/status/1758837061786779942", PermanentURL: "https://twitter.com/premium/status/1758837061786779942",
Text: "no ads, just bangers\n\naka your For You feed with Premium+\n\nsubscribe here → https://x.com/i/premium_sign_up", Text: "no ads, just bangers\n\naka your For You feed with Premium+\n\nsubscribe here → https://t.co/APTO1t7kMk",
Timestamp: 1708174407, Timestamp: 1708174407,
UserID: "1399766153053061121", UserID: "1399766153053061121",
Username: "premium", Username: "premium",

140
types.go
View file

@ -10,14 +10,6 @@ type (
Name string Name string
} }
// Url represents a URL with display, expanded, and index data.
Url struct {
DisplayURL string `json:"display_url"`
ExpandedURL string `json:"expanded_url"`
URL string `json:"url"`
Indices []int `json:"indices"`
}
// Photo type. // Photo type.
Photo struct { Photo struct {
ID string ID string
@ -99,25 +91,6 @@ type (
GIFs []GIF GIFs []GIF
} }
ExtendedMedia struct {
IDStr string `json:"id_str"`
MediaURLHttps string `json:"media_url_https"`
ExtSensitiveMediaWarning struct {
AdultContent bool `json:"adult_content"`
GraphicViolence bool `json:"graphic_violence"`
Other bool `json:"other"`
} `json:"ext_sensitive_media_warning"`
Type string `json:"type"`
URL string `json:"url"`
VideoInfo struct {
Variants []struct {
Type string `json:"content_type"`
Bitrate int `json:"bitrate"`
URL string `json:"url"`
} `json:"variants"`
} `json:"video_info"`
}
legacyTweet struct { legacyTweet struct {
ConversationIDStr string `json:"conversation_id_str"` ConversationIDStr string `json:"conversation_id_str"`
CreatedAt string `json:"created_at"` CreatedAt string `json:"created_at"`
@ -132,7 +105,10 @@ type (
Type string `json:"type"` Type string `json:"type"`
URL string `json:"url"` URL string `json:"url"`
} `json:"media"` } `json:"media"`
URLs []Url `json:"urls"` URLs []struct {
ExpandedURL string `json:"expanded_url"`
URL string `json:"url"`
} `json:"urls"`
UserMentions []struct { UserMentions []struct {
IDStr string `json:"id_str"` IDStr string `json:"id_str"`
Name string `json:"name"` Name string `json:"name"`
@ -140,7 +116,24 @@ type (
} `json:"user_mentions"` } `json:"user_mentions"`
} `json:"entities"` } `json:"entities"`
ExtendedEntities struct { ExtendedEntities struct {
Media []ExtendedMedia `json:"media"` Media []struct {
IDStr string `json:"id_str"`
MediaURLHttps string `json:"media_url_https"`
ExtSensitiveMediaWarning struct {
AdultContent bool `json:"adult_content"`
GraphicViolence bool `json:"graphic_violence"`
Other bool `json:"other"`
} `json:"ext_sensitive_media_warning"`
Type string `json:"type"`
URL string `json:"url"`
VideoInfo struct {
Variants []struct {
Type string `json:"content_type"`
Bitrate int `json:"bitrate"`
URL string `json:"url"`
} `json:"variants"`
} `json:"video_info"`
} `json:"media"`
} `json:"extended_entities"` } `json:"extended_entities"`
IDStr string `json:"id_str"` IDStr string `json:"id_str"`
InReplyToStatusIDStr string `json:"in_reply_to_status_id_str"` InReplyToStatusIDStr string `json:"in_reply_to_status_id_str"`
@ -173,37 +166,22 @@ type (
} `json:"urls"` } `json:"urls"`
} `json:"url"` } `json:"url"`
} `json:"entities"` } `json:"entities"`
FavouritesCount int `json:"favourites_count"` FavouritesCount int `json:"favourites_count"`
FollowersCount int `json:"followers_count"` FollowersCount int `json:"followers_count"`
FriendsCount int `json:"friends_count"` FriendsCount int `json:"friends_count"`
IDStr string `json:"id_str"` IDStr string `json:"id_str"`
ListedCount int `json:"listed_count"` ListedCount int `json:"listed_count"`
Name string `json:"name"` Name string `json:"name"`
Location string `json:"location"` Location string `json:"location"`
PinnedTweetIdsStr []string `json:"pinned_tweet_ids_str"` PinnedTweetIdsStr []string `json:"pinned_tweet_ids_str"`
ProfileBannerURL string `json:"profile_banner_url"` ProfileBannerURL string `json:"profile_banner_url"`
ProfileImageURLHTTPS string `json:"profile_image_url_https"` ProfileImageURLHTTPS string `json:"profile_image_url_https"`
Protected bool `json:"protected"` Protected bool `json:"protected"`
ScreenName string `json:"screen_name"` ScreenName string `json:"screen_name"`
StatusesCount int `json:"statuses_count"` StatusesCount int `json:"statuses_count"`
Verified bool `json:"verified"` Verified bool `json:"verified"`
FollowedBy bool `json:"followed_by"` FollowedBy bool `json:"followed_by"`
Following bool `json:"following"` Following bool `json:"following"`
CanDm bool `json:"can_dm"`
CanMediaTag bool `json:"can_media_tag"`
DefaultProfile bool `json:"default_profile"`
DefaultProfileImage bool `json:"default_profile_image"`
FastFollowersCount int `json:"fast_followers_count"`
HasCustomTimelines bool `json:"has_custom_timelines"`
IsTranslator bool `json:"is_translator"`
MediaCount int `json:"media_count"`
NeedsPhoneVerification bool `json:"needs_phone_verification"`
NormalFollowersCount int `json:"normal_followers_count"`
PossiblySensitive bool `json:"possibly_sensitive"`
ProfileInterstitialType string `json:"profile_interstitial_type"`
TranslatorType string `json:"translator_type"`
WantRetweets bool `json:"want_retweets"`
WithheldInCountries []string `json:"withheld_in_countries"`
} }
legacyUserV2 struct { legacyUserV2 struct {
@ -217,10 +195,15 @@ type (
Description string `json:"description"` Description string `json:"description"`
Entities struct { Entities struct {
Description struct { Description struct {
Urls []Url `json:"urls"` Urls []interface{} `json:"urls"`
} `json:"description"` } `json:"description"`
URL struct { URL struct {
Urls []Url `json:"urls"` Urls []struct {
DisplayURL string `json:"display_url"`
ExpandedURL string `json:"expanded_url"`
URL string `json:"url"`
Indices []int `json:"indices"`
} `json:"urls"`
} `json:"url"` } `json:"url"`
} `json:"entities"` } `json:"entities"`
FastFollowersCount int `json:"fast_followers_count"` FastFollowersCount int `json:"fast_followers_count"`
@ -263,37 +246,4 @@ type (
fetchProfileFunc func(query string, maxProfilesNbr int, cursor string) ([]*Profile, string, error) fetchProfileFunc func(query string, maxProfilesNbr int, cursor string) ([]*Profile, string, error)
fetchTweetFunc func(query string, maxTweetsNbr int, cursor string) ([]*Tweet, string, error) fetchTweetFunc func(query string, maxTweetsNbr int, cursor string) ([]*Tweet, string, error)
legacyExtendedProfile struct {
Birthdate struct {
Day int `json:"day"`
Month int `json:"month"`
Year int `json:"year"`
Visibility string `json:"visibility"`
YearVisibility string `json:"year_visibility"`
} `json:"birthdate"`
}
verificationInfo struct {
IsIdentityVerified bool `json:"is_identity_verified"`
Reason struct {
Description struct {
Text string `json:"text"`
Entities []struct {
FromIndex int `json:"from_index"`
ToIndex int `json:"to_index"`
Ref struct {
URL string `json:"url"`
URLType string `json:"url_type"`
} `json:"ref"`
} `json:"entities"`
} `json:"description"`
VerifiedSinceMsec string `json:"verified_since_msec"`
} `json:"reason"`
}
highlightsInfo struct {
CanHighlightTweets bool `json:"can_highlight_tweets"`
HighlightedTweets string `json:"highlighted_tweets"`
}
) )

100
util.go
View file

@ -157,7 +157,6 @@ func parseLegacyTweet(user *legacyUser, tweet *legacyTweet) *Tweet {
if tweetID == "" { if tweetID == "" {
return nil return nil
} }
text := expandURLs(tweet.FullText, tweet.Entities.URLs, tweet.ExtendedEntities.Media)
username := user.ScreenName username := user.ScreenName
name := user.Name name := user.Name
tw := &Tweet{ tw := &Tweet{
@ -168,7 +167,7 @@ func parseLegacyTweet(user *legacyUser, tweet *legacyTweet) *Tweet {
PermanentURL: fmt.Sprintf("https://twitter.com/%s/status/%s", username, tweetID), PermanentURL: fmt.Sprintf("https://twitter.com/%s/status/%s", username, tweetID),
Replies: tweet.ReplyCount, Replies: tweet.ReplyCount,
Retweets: tweet.RetweetCount, Retweets: tweet.RetweetCount,
Text: text, Text: tweet.FullText,
UserID: tweet.UserIDStr, UserID: tweet.UserIDStr,
Username: username, Username: username,
} }
@ -308,7 +307,7 @@ func parseLegacyTweet(user *legacyUser, tweet *legacyTweet) *Tweet {
tw.HTML = reTwitterURL.ReplaceAllStringFunc(tw.HTML, func(tco string) string { tw.HTML = reTwitterURL.ReplaceAllStringFunc(tw.HTML, func(tco string) string {
for _, entity := range tweet.Entities.URLs { for _, entity := range tweet.Entities.URLs {
if tco == entity.URL { if tco == entity.URL {
return fmt.Sprintf(`<a href="%s">%s</a>`, entity.ExpandedURL, entity.ExpandedURL) return fmt.Sprintf(`<a href="%s">%s</a>`, entity.ExpandedURL, tco)
} }
} }
for _, entity := range tweet.ExtendedEntities.Media { for _, entity := range tweet.ExtendedEntities.Media {
@ -346,28 +345,25 @@ func parseLegacyTweet(user *legacyUser, tweet *legacyTweet) *Tweet {
func parseProfile(user legacyUser) Profile { func parseProfile(user legacyUser) Profile {
profile := Profile{ profile := Profile{
Avatar: user.ProfileImageURLHTTPS, Avatar: user.ProfileImageURLHTTPS,
Banner: user.ProfileBannerURL, Banner: user.ProfileBannerURL,
Biography: user.Description, Biography: user.Description,
FollowersCount: user.FollowersCount, FollowersCount: user.FollowersCount,
FollowingCount: user.FavouritesCount, FollowingCount: user.FavouritesCount,
FriendsCount: user.FriendsCount, FriendsCount: user.FriendsCount,
IsVerified: user.Verified, IsVerified: user.Verified,
IsPrivate: user.Protected, IsPrivate: user.Protected,
LikesCount: user.FavouritesCount, LikesCount: user.FavouritesCount,
ListedCount: user.ListedCount, ListedCount: user.ListedCount,
Location: user.Location, Location: user.Location,
Name: user.Name, Name: user.Name,
PinnedTweetIDs: user.PinnedTweetIdsStr, PinnedTweetIDs: user.PinnedTweetIdsStr,
TweetsCount: user.StatusesCount, TweetsCount: user.StatusesCount,
URL: "https://twitter.com/" + user.ScreenName, URL: "https://twitter.com/" + user.ScreenName,
UserID: user.IDStr, UserID: user.IDStr,
Username: user.ScreenName, Username: user.ScreenName,
FollowedBy: user.FollowedBy, FollowedBy: user.FollowedBy,
Following: user.Following, Following: user.Following,
MediaCount: user.MediaCount,
FastFollowersCount: user.FastFollowersCount,
NormalFollowersCount: user.NormalFollowersCount,
} }
tm, err := time.Parse(time.RubyDate, user.CreatedAt) tm, err := time.Parse(time.RubyDate, user.CreatedAt)
@ -383,44 +379,28 @@ func parseProfile(user legacyUser) Profile {
return profile return profile
} }
func expandURLs(text string, urls []Url, extendedMediaEntities []ExtendedMedia) string {
expandedText := text
for _, url := range urls {
expandedText = strings.ReplaceAll(expandedText, url.URL, url.ExpandedURL)
}
for _, entity := range extendedMediaEntities {
expandedText = strings.ReplaceAll(expandedText, entity.URL, entity.MediaURLHttps)
}
return expandedText
}
func parseProfileV2(user userResult) Profile { func parseProfileV2(user userResult) Profile {
u := user.Legacy u := user.Legacy
description := expandURLs(u.Description, u.Entities.Description.Urls, []ExtendedMedia{})
profile := Profile{ profile := Profile{
Avatar: u.ProfileImageURLHTTPS, Avatar: u.ProfileImageURLHTTPS,
Banner: u.ProfileBannerURL, Banner: u.ProfileBannerURL,
Biography: description, Biography: u.Description,
FollowersCount: u.FollowersCount, FollowersCount: u.FollowersCount,
FollowingCount: u.FavouritesCount, FollowingCount: u.FavouritesCount,
FriendsCount: u.FriendsCount, FriendsCount: u.FriendsCount,
IsVerified: u.Verified, IsVerified: u.Verified,
IsBlueVerified: user.IsBlueVerified, LikesCount: u.FavouritesCount,
ProfileImageShape: user.ProfileImageShape, ListedCount: u.ListedCount,
HasGraduatedAccess: user.HasGraduatedAccess, Location: u.Location,
LikesCount: u.FavouritesCount, Name: u.Name,
ListedCount: u.ListedCount, PinnedTweetIDs: u.PinnedTweetIdsStr,
Location: u.Location, TweetsCount: u.StatusesCount,
Name: u.Name, URL: "https://twitter.com/" + u.ScreenName,
PinnedTweetIDs: u.PinnedTweetIdsStr, UserID: user.ID,
TweetsCount: u.StatusesCount, Username: u.ScreenName,
URL: "https://twitter.com/" + u.ScreenName, Sensitive: u.PossiblySensitive,
UserID: user.ID, Following: u.Following,
Username: u.ScreenName, FollowedBy: u.FollowedBy,
Sensitive: u.PossiblySensitive,
Following: u.Following,
FollowedBy: u.FollowedBy,
} }
tm, err := time.Parse(time.RubyDate, u.CreatedAt) tm, err := time.Parse(time.RubyDate, u.CreatedAt)