lain/twitter-scrapper
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity

Merge pull request #35 from masa-finance/master

Browse source 
Add Twitter Blue Verification Status to Profile Responses
This commit is contained in:
Valentine 2025-01-22 23:52:05 +03:00 • committed by GitHub
commit 4519142ae8
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
4 changed files with 163 additions and 90 deletions
Download patch file Download diff file Expand all files Collapse all files

68
profile.go
View file

@ -14,38 +14,46 @@ var cacheIDs sync.Map
// Profile of twitter user. // Profile of twitter user.
type Profile struct { type Profile struct {
Avatar string Avatar string
Banner string Banner string
Biography string Biography string
Birthday string Birthday string
FollowersCount int FollowersCount int
FollowingCount int FollowingCount int
FriendsCount int FriendsCount int
IsPrivate bool IsPrivate bool
IsVerified bool IsVerified bool
Joined *time.Time IsBlueVerified bool
LikesCount int Joined *time.Time
ListedCount int LikesCount int
Location string ListedCount int
Name string Location string
PinnedTweetIDs []string Name string
TweetsCount int PinnedTweetIDs []string
URL string TweetsCount int
UserID string URL string
Username string UserID string
Website string Username string
Sensitive bool Website string
Following bool Sensitive bool
FollowedBy bool Following bool
FollowedBy bool
MediaCount int
FastFollowersCount int
NormalFollowersCount int
ProfileImageShape string
HasGraduatedAccess bool
CanHighlightTweets bool
} }
type user struct { type user struct {
Data struct { Data struct {
User struct { User struct {
Result struct { Result struct {
RestID string `json:"rest_id"` RestID string `json:"rest_id"`
Legacy legacyUser `json:"legacy"` Legacy legacyUser `json:"legacy"`
Message string `json:"message"` Message string `json:"message"`
IsBlueVerified bool `json:"is_blue_verified"`
} `json:"result"` } `json:"result"`
} `json:"user"` } `json:"user"`
} `json:"data"` } `json:"data"`
@ -111,7 +119,9 @@ func (s *Scraper) GetProfile(username string) (Profile, error) {
return Profile{}, fmt.Errorf("either @%s does not exist or is private", username) return Profile{}, fmt.Errorf("either @%s does not exist or is private", username)
} }
return parseProfile(jsn.Data.User.Result.Legacy), nil profile := parseProfile(jsn.Data.User.Result.Legacy)
profile.IsBlueVerified = jsn.Data.User.Result.IsBlueVerified
return profile, nil
} }
func (s *Scraper) GetProfileByID(userID string) (Profile, error) { func (s *Scraper) GetProfileByID(userID string) (Profile, error) {
@ -168,7 +178,9 @@ func (s *Scraper) GetProfileByID(userID string) (Profile, error) {
return Profile{}, fmt.Errorf("either @%s does not exist or is private", userID) return Profile{}, fmt.Errorf("either @%s does not exist or is private", userID)
} }
return parseProfile(jsn.Data.User.Result.Legacy), nil profile := parseProfile(jsn.Data.User.Result.Legacy)
profile.IsBlueVerified = jsn.Data.User.Result.IsBlueVerified
return profile, nil
} }
// GetUserIDByScreenName from API // GetUserIDByScreenName from API

23
timeline_v2.go
View file

@ -60,14 +60,21 @@ func (result *result) parse() *Tweet {
} }
type userResult struct { type userResult struct {
Typename string `json:"__typename"` Typename string `json:"__typename"`
ID string `json:"id"` ID string `json:"id"`
RestID string `json:"rest_id"` RestID string `json:"rest_id"`
AffiliatesHighlightedLabel struct{} `json:"affiliates_highlighted_label"` AffiliatesHighlightedLabel struct{} `json:"affiliates_highlighted_label"`
HasGraduatedAccess bool `json:"has_graduated_access"` HasGraduatedAccess bool `json:"has_graduated_access"`
IsBlueVerified bool `json:"is_blue_verified"` IsBlueVerified bool `json:"is_blue_verified"`
ProfileImageShape string `json:"profile_image_shape"` ProfileImageShape string `json:"profile_image_shape"`
Legacy legacyUserV2 `json:"legacy"` Legacy legacyUserV2 `json:"legacy"`
LegacyExtendedProfile legacyExtendedProfile `json:"legacy_extended_profile"`
IsProfileTranslatable bool `json:"is_profile_translatable"`
VerificationInfo verificationInfo `json:"verification_info"`
HighlightsInfo highlightsInfo `json:"highlights_info"`
UserSeedTweetCount int `json:"user_seed_tweet_count"`
PremiumGiftingEligible bool `json:"premium_gifting_eligible"`
CreatorSubscriptionsCount int `json:"creator_subscriptions_count"`
} }
func (result *userResult) parse() Profile { func (result *userResult) parse() Profile {

80
types.go
View file

@ -166,22 +166,37 @@ type (
} `json:"urls"` } `json:"urls"`
} `json:"url"` } `json:"url"`
} `json:"entities"` } `json:"entities"`
FavouritesCount int `json:"favourites_count"` FavouritesCount int `json:"favourites_count"`
FollowersCount int `json:"followers_count"` FollowersCount int `json:"followers_count"`
FriendsCount int `json:"friends_count"` FriendsCount int `json:"friends_count"`
IDStr string `json:"id_str"` IDStr string `json:"id_str"`
ListedCount int `json:"listed_count"` ListedCount int `json:"listed_count"`
Name string `json:"name"` Name string `json:"name"`
Location string `json:"location"` Location string `json:"location"`
PinnedTweetIdsStr []string `json:"pinned_tweet_ids_str"` PinnedTweetIdsStr []string `json:"pinned_tweet_ids_str"`
ProfileBannerURL string `json:"profile_banner_url"` ProfileBannerURL string `json:"profile_banner_url"`
ProfileImageURLHTTPS string `json:"profile_image_url_https"` ProfileImageURLHTTPS string `json:"profile_image_url_https"`
Protected bool `json:"protected"` Protected bool `json:"protected"`
ScreenName string `json:"screen_name"` ScreenName string `json:"screen_name"`
StatusesCount int `json:"statuses_count"` StatusesCount int `json:"statuses_count"`
Verified bool `json:"verified"` Verified bool `json:"verified"`
FollowedBy bool `json:"followed_by"` FollowedBy bool `json:"followed_by"`
Following bool `json:"following"` Following bool `json:"following"`
CanDm bool `json:"can_dm"`
CanMediaTag bool `json:"can_media_tag"`
DefaultProfile bool `json:"default_profile"`
DefaultProfileImage bool `json:"default_profile_image"`
FastFollowersCount int `json:"fast_followers_count"`
HasCustomTimelines bool `json:"has_custom_timelines"`
IsTranslator bool `json:"is_translator"`
MediaCount int `json:"media_count"`
NeedsPhoneVerification bool `json:"needs_phone_verification"`
NormalFollowersCount int `json:"normal_followers_count"`
PossiblySensitive bool `json:"possibly_sensitive"`
ProfileInterstitialType string `json:"profile_interstitial_type"`
TranslatorType string `json:"translator_type"`
WantRetweets bool `json:"want_retweets"`
WithheldInCountries []string `json:"withheld_in_countries"`
} }
legacyUserV2 struct { legacyUserV2 struct {
@ -246,4 +261,37 @@ type (
fetchProfileFunc func(query string, maxProfilesNbr int, cursor string) ([]*Profile, string, error) fetchProfileFunc func(query string, maxProfilesNbr int, cursor string) ([]*Profile, string, error)
fetchTweetFunc func(query string, maxTweetsNbr int, cursor string) ([]*Tweet, string, error) fetchTweetFunc func(query string, maxTweetsNbr int, cursor string) ([]*Tweet, string, error)
legacyExtendedProfile struct {
Birthdate struct {
Day int `json:"day"`
Month int `json:"month"`
Year int `json:"year"`
Visibility string `json:"visibility"`
YearVisibility string `json:"year_visibility"`
} `json:"birthdate"`
}
verificationInfo struct {
IsIdentityVerified bool `json:"is_identity_verified"`
Reason struct {
Description struct {
Text string `json:"text"`
Entities []struct {
FromIndex int `json:"from_index"`
ToIndex int `json:"to_index"`
Ref struct {
URL string `json:"url"`
URLType string `json:"url_type"`
} `json:"ref"`
} `json:"entities"`
} `json:"description"`
VerifiedSinceMsec string `json:"verified_since_msec"`
} `json:"reason"`
}
highlightsInfo struct {
CanHighlightTweets bool `json:"can_highlight_tweets"`
HighlightedTweets string `json:"highlighted_tweets"`
}
) )

82
util.go
View file

@ -345,25 +345,28 @@ func parseLegacyTweet(user *legacyUser, tweet *legacyTweet) *Tweet {
func parseProfile(user legacyUser) Profile { func parseProfile(user legacyUser) Profile {
profile := Profile{ profile := Profile{
Avatar: user.ProfileImageURLHTTPS, Avatar: user.ProfileImageURLHTTPS,
Banner: user.ProfileBannerURL, Banner: user.ProfileBannerURL,
Biography: user.Description, Biography: user.Description,
FollowersCount: user.FollowersCount, FollowersCount: user.FollowersCount,
FollowingCount: user.FavouritesCount, FollowingCount: user.FavouritesCount,
FriendsCount: user.FriendsCount, FriendsCount: user.FriendsCount,
IsVerified: user.Verified, IsVerified: user.Verified,
IsPrivate: user.Protected, IsPrivate: user.Protected,
LikesCount: user.FavouritesCount, LikesCount: user.FavouritesCount,
ListedCount: user.ListedCount, ListedCount: user.ListedCount,
Location: user.Location, Location: user.Location,
Name: user.Name, Name: user.Name,
PinnedTweetIDs: user.PinnedTweetIdsStr, PinnedTweetIDs: user.PinnedTweetIdsStr,
TweetsCount: user.StatusesCount, TweetsCount: user.StatusesCount,
URL: "https://twitter.com/" + user.ScreenName, URL: "https://twitter.com/" + user.ScreenName,
UserID: user.IDStr, UserID: user.IDStr,
Username: user.ScreenName, Username: user.ScreenName,
FollowedBy: user.FollowedBy, FollowedBy: user.FollowedBy,
Following: user.Following, Following: user.Following,
MediaCount: user.MediaCount,
FastFollowersCount: user.FastFollowersCount,
NormalFollowersCount: user.NormalFollowersCount,
} }
tm, err := time.Parse(time.RubyDate, user.CreatedAt) tm, err := time.Parse(time.RubyDate, user.CreatedAt)
@ -382,25 +385,28 @@ func parseProfile(user legacyUser) Profile {
func parseProfileV2(user userResult) Profile { func parseProfileV2(user userResult) Profile {
u := user.Legacy u := user.Legacy
profile := Profile{ profile := Profile{
Avatar: u.ProfileImageURLHTTPS, Avatar: u.ProfileImageURLHTTPS,
Banner: u.ProfileBannerURL, Banner: u.ProfileBannerURL,
Biography: u.Description, Biography: u.Description,
FollowersCount: u.FollowersCount, FollowersCount: u.FollowersCount,
FollowingCount: u.FavouritesCount, FollowingCount: u.FavouritesCount,
FriendsCount: u.FriendsCount, FriendsCount: u.FriendsCount,
IsVerified: u.Verified, IsVerified: u.Verified,
LikesCount: u.FavouritesCount, IsBlueVerified: user.IsBlueVerified,
ListedCount: u.ListedCount, ProfileImageShape: user.ProfileImageShape,
Location: u.Location, HasGraduatedAccess: user.HasGraduatedAccess,
Name: u.Name, LikesCount: u.FavouritesCount,
PinnedTweetIDs: u.PinnedTweetIdsStr, ListedCount: u.ListedCount,
TweetsCount: u.StatusesCount, Location: u.Location,
URL: "https://twitter.com/" + u.ScreenName, Name: u.Name,
UserID: user.ID, PinnedTweetIDs: u.PinnedTweetIdsStr,
Username: u.ScreenName, TweetsCount: u.StatusesCount,
Sensitive: u.PossiblySensitive, URL: "https://twitter.com/" + u.ScreenName,
Following: u.Following, UserID: user.ID,
FollowedBy: u.FollowedBy, Username: u.ScreenName,
Sensitive: u.PossiblySensitive,
Following: u.Following,
FollowedBy: u.FollowedBy,
} }
tm, err := time.Parse(time.RubyDate, u.CreatedAt) tm, err := time.Parse(time.RubyDate, u.CreatedAt)