lain/twitter-scrapper
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity

Merge branch 'master' into feat/expand-description-urls

Browse source
This commit is contained in:
Valentine 2025-01-23 02:16:38 +03:00 • committed by GitHub
commit e5c170986e
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
6 changed files with 175 additions and 95 deletions
Download patch file Download diff file Expand all files Collapse all files

11
auth.go
View file

@ -19,9 +19,12 @@ import (
)
const (
loginURL = "https://api.twitter.com/1.1/onboarding/task.json"
logoutURL = "https://api.twitter.com/1.1/account/logout.json"
oAuthURL = "https://api.twitter.com/oauth2/token"
loginURL = "https://api.twitter.com/1.1/onboarding/task.json"
logoutURL = "https://api.twitter.com/1.1/account/logout.json"
oAuthURL = "https://api.twitter.com/oauth2/token"
// Doesn't require x-client-transaction-id header in auth. x-rate-limit-limit: 2000
bearerToken1 = "AAAAAAAAAAAAAAAAAAAAAFQODgEAAAAAVHTp76lzh3rFzcHbmHVvQxYYpTw%3DckAlMINMjmCwxUcaXbAN4XqJVdgMJaHqNOFgPMK0zN1qLqLQCF"
// Requires x-client-transaction-id header in auth.
bearerToken2 = "AAAAAAAAAAAAAAAAAAAAANRILgAAAAAAnNwIzUejRCOuH5E6I8xnZz4puTs%3D1Zv7ttfk8LF81IUq16cHjhLTvJu4FA33AGWWjCpTnA"
appConsumerKey = "3nVuSoBZnx6U4vzUxf5w"
appConsumerSecret = "Bcs59EFbbsdF6Sl9Ng71smgStWEGwXXKSjYvPVt7qys"
@ -147,7 +150,7 @@ func (s *Scraper) getFlowToken(data map[string]interface{}) (string, error) {
// IsLoggedIn check if scraper logged in
func (s *Scraper) IsLoggedIn() bool {
s.isLogged = true
s.setBearerToken(bearerToken2)
s.setBearerToken(bearerToken1)
req, err := http.NewRequest("GET", "https://api.twitter.com/1.1/account/verify_credentials.json", nil)
if err != nil {
return false

68
profile.go
View file

@ -14,38 +14,46 @@ var cacheIDs sync.Map
// Profile of twitter user.
type Profile struct {
Avatar string
Banner string
Biography string
Birthday string
FollowersCount int
FollowingCount int
FriendsCount int
IsPrivate bool
IsVerified bool
Joined *time.Time
LikesCount int
ListedCount int
Location string
Name string
PinnedTweetIDs []string
TweetsCount int
URL string
UserID string
Username string
Website string
Sensitive bool
Following bool
FollowedBy bool
Avatar string
Banner string
Biography string
Birthday string
FollowersCount int
FollowingCount int
FriendsCount int
IsPrivate bool
IsVerified bool
IsBlueVerified bool
Joined *time.Time
LikesCount int
ListedCount int
Location string
Name string
PinnedTweetIDs []string
TweetsCount int
URL string
UserID string
Username string
Website string
Sensitive bool
Following bool
FollowedBy bool
MediaCount int
FastFollowersCount int
NormalFollowersCount int
ProfileImageShape string
HasGraduatedAccess bool
CanHighlightTweets bool
}
type user struct {
Data struct {
User struct {
Result struct {
RestID string `json:"rest_id"`
Legacy legacyUser `json:"legacy"`
Message string `json:"message"`
RestID string `json:"rest_id"`
Legacy legacyUser `json:"legacy"`
Message string `json:"message"`
IsBlueVerified bool `json:"is_blue_verified"`
} `json:"result"`
} `json:"user"`
} `json:"data"`
@ -111,7 +119,9 @@ func (s *Scraper) GetProfile(username string) (Profile, error) {
return Profile{}, fmt.Errorf("either @%s does not exist or is private", username)
}
return parseProfile(jsn.Data.User.Result.Legacy), nil
profile := parseProfile(jsn.Data.User.Result.Legacy)
profile.IsBlueVerified = jsn.Data.User.Result.IsBlueVerified
return profile, nil
}
func (s *Scraper) GetProfileByID(userID string) (Profile, error) {
@ -168,7 +178,9 @@ func (s *Scraper) GetProfileByID(userID string) (Profile, error) {
return Profile{}, fmt.Errorf("either @%s does not exist or is private", userID)
}
return parseProfile(jsn.Data.User.Result.Legacy), nil
profile := parseProfile(jsn.Data.User.Result.Legacy)
profile.IsBlueVerified = jsn.Data.User.Result.IsBlueVerified
return profile, nil
}
// GetUserIDByScreenName from API

6
profile_test.go
View file

@ -42,6 +42,8 @@ func TestGetProfile(t *testing.T) {
cmpopts.IgnoreFields(twitterscraper.Profile{}, "LikesCount"),
cmpopts.IgnoreFields(twitterscraper.Profile{}, "ListedCount"),
cmpopts.IgnoreFields(twitterscraper.Profile{}, "TweetsCount"),
cmpopts.IgnoreFields(twitterscraper.Profile{}, "MediaCount"),
cmpopts.IgnoreFields(twitterscraper.Profile{}, "NormalFollowersCount"),
}
if diff := cmp.Diff(sample, profile, cmpOptions...); diff != "" {
t.Error("Resulting profile does not match the sample", diff)
@ -94,6 +96,8 @@ func TestGetProfilePrivate(t *testing.T) {
cmpopts.IgnoreFields(twitterscraper.Profile{}, "LikesCount"),
cmpopts.IgnoreFields(twitterscraper.Profile{}, "ListedCount"),
cmpopts.IgnoreFields(twitterscraper.Profile{}, "TweetsCount"),
cmpopts.IgnoreFields(twitterscraper.Profile{}, "MediaCount"),
cmpopts.IgnoreFields(twitterscraper.Profile{}, "NormalFollowersCount"),
}
if diff := cmp.Diff(sample, profile, cmpOptions...); diff != "" {
t.Error("Resulting profile does not match the sample", diff)
@ -146,7 +150,7 @@ func TestGetProfileByID(t *testing.T) {
}
func TestGetUserIDByScreenName(t *testing.T) {
userID, err := testScraper.GetUserIDByScreenName("Twitter")
userID, err := testScraper.GetUserIDByScreenName("X")
if err != nil {
t.Errorf("getUserByScreenName() error = %v", err)
}

23
timeline_v2.go
View file

@ -62,14 +62,21 @@ func (result *result) parse() *Tweet {
}
type userResult struct {
Typename string `json:"__typename"`
ID string `json:"id"`
RestID string `json:"rest_id"`
AffiliatesHighlightedLabel struct{} `json:"affiliates_highlighted_label"`
HasGraduatedAccess bool `json:"has_graduated_access"`
IsBlueVerified bool `json:"is_blue_verified"`
ProfileImageShape string `json:"profile_image_shape"`
Legacy legacyUserV2 `json:"legacy"`
Typename string `json:"__typename"`
ID string `json:"id"`
RestID string `json:"rest_id"`
AffiliatesHighlightedLabel struct{} `json:"affiliates_highlighted_label"`
HasGraduatedAccess bool `json:"has_graduated_access"`
IsBlueVerified bool `json:"is_blue_verified"`
ProfileImageShape string `json:"profile_image_shape"`
Legacy legacyUserV2 `json:"legacy"`
LegacyExtendedProfile legacyExtendedProfile `json:"legacy_extended_profile"`
IsProfileTranslatable bool `json:"is_profile_translatable"`
VerificationInfo verificationInfo `json:"verification_info"`
HighlightsInfo highlightsInfo `json:"highlights_info"`
UserSeedTweetCount int `json:"user_seed_tweet_count"`
PremiumGiftingEligible bool `json:"premium_gifting_eligible"`
CreatorSubscriptionsCount int `json:"creator_subscriptions_count"`
}
func (result *userResult) parse() Profile {

80
types.go
View file

@ -173,22 +173,37 @@ type (
} `json:"urls"`
} `json:"url"`
} `json:"entities"`
FavouritesCount int `json:"favourites_count"`
FollowersCount int `json:"followers_count"`
FriendsCount int `json:"friends_count"`
IDStr string `json:"id_str"`
ListedCount int `json:"listed_count"`
Name string `json:"name"`
Location string `json:"location"`
PinnedTweetIdsStr []string `json:"pinned_tweet_ids_str"`
ProfileBannerURL string `json:"profile_banner_url"`
ProfileImageURLHTTPS string `json:"profile_image_url_https"`
Protected bool `json:"protected"`
ScreenName string `json:"screen_name"`
StatusesCount int `json:"statuses_count"`
Verified bool `json:"verified"`
FollowedBy bool `json:"followed_by"`
Following bool `json:"following"`
FavouritesCount int `json:"favourites_count"`
FollowersCount int `json:"followers_count"`
FriendsCount int `json:"friends_count"`
IDStr string `json:"id_str"`
ListedCount int `json:"listed_count"`
Name string `json:"name"`
Location string `json:"location"`
PinnedTweetIdsStr []string `json:"pinned_tweet_ids_str"`
ProfileBannerURL string `json:"profile_banner_url"`
ProfileImageURLHTTPS string `json:"profile_image_url_https"`
Protected bool `json:"protected"`
ScreenName string `json:"screen_name"`
StatusesCount int `json:"statuses_count"`
Verified bool `json:"verified"`
FollowedBy bool `json:"followed_by"`
Following bool `json:"following"`
CanDm bool `json:"can_dm"`
CanMediaTag bool `json:"can_media_tag"`
DefaultProfile bool `json:"default_profile"`
DefaultProfileImage bool `json:"default_profile_image"`
FastFollowersCount int `json:"fast_followers_count"`
HasCustomTimelines bool `json:"has_custom_timelines"`
IsTranslator bool `json:"is_translator"`
MediaCount int `json:"media_count"`
NeedsPhoneVerification bool `json:"needs_phone_verification"`
NormalFollowersCount int `json:"normal_followers_count"`
PossiblySensitive bool `json:"possibly_sensitive"`
ProfileInterstitialType string `json:"profile_interstitial_type"`
TranslatorType string `json:"translator_type"`
WantRetweets bool `json:"want_retweets"`
WithheldInCountries []string `json:"withheld_in_countries"`
}
legacyUserV2 struct {
@ -248,4 +263,37 @@ type (
fetchProfileFunc func(query string, maxProfilesNbr int, cursor string) ([]*Profile, string, error)
fetchTweetFunc func(query string, maxTweetsNbr int, cursor string) ([]*Tweet, string, error)
legacyExtendedProfile struct {
Birthdate struct {
Day int `json:"day"`
Month int `json:"month"`
Year int `json:"year"`
Visibility string `json:"visibility"`
YearVisibility string `json:"year_visibility"`
} `json:"birthdate"`
}
verificationInfo struct {
IsIdentityVerified bool `json:"is_identity_verified"`
Reason struct {
Description struct {
Text string `json:"text"`
Entities []struct {
FromIndex int `json:"from_index"`
ToIndex int `json:"to_index"`
Ref struct {
URL string `json:"url"`
URLType string `json:"url_type"`
} `json:"ref"`
} `json:"entities"`
} `json:"description"`
VerifiedSinceMsec string `json:"verified_since_msec"`
} `json:"reason"`
}
highlightsInfo struct {
CanHighlightTweets bool `json:"can_highlight_tweets"`
HighlightedTweets string `json:"highlighted_tweets"`
}
)

82
util.go
View file

@ -346,25 +346,28 @@ func parseLegacyTweet(user *legacyUser, tweet *legacyTweet) *Tweet {
func parseProfile(user legacyUser) Profile {
profile := Profile{
Avatar: user.ProfileImageURLHTTPS,
Banner: user.ProfileBannerURL,
Biography: user.Description,
FollowersCount: user.FollowersCount,
FollowingCount: user.FavouritesCount,
FriendsCount: user.FriendsCount,
IsVerified: user.Verified,
IsPrivate: user.Protected,
LikesCount: user.FavouritesCount,
ListedCount: user.ListedCount,
Location: user.Location,
Name: user.Name,
PinnedTweetIDs: user.PinnedTweetIdsStr,
TweetsCount: user.StatusesCount,
URL: "https://twitter.com/" + user.ScreenName,
UserID: user.IDStr,
Username: user.ScreenName,
FollowedBy: user.FollowedBy,
Following: user.Following,
Avatar: user.ProfileImageURLHTTPS,
Banner: user.ProfileBannerURL,
Biography: user.Description,
FollowersCount: user.FollowersCount,
FollowingCount: user.FavouritesCount,
FriendsCount: user.FriendsCount,
IsVerified: user.Verified,
IsPrivate: user.Protected,
LikesCount: user.FavouritesCount,
ListedCount: user.ListedCount,
Location: user.Location,
Name: user.Name,
PinnedTweetIDs: user.PinnedTweetIdsStr,
TweetsCount: user.StatusesCount,
URL: "https://twitter.com/" + user.ScreenName,
UserID: user.IDStr,
Username: user.ScreenName,
FollowedBy: user.FollowedBy,
Following: user.Following,
MediaCount: user.MediaCount,
FastFollowersCount: user.FastFollowersCount,
NormalFollowersCount: user.NormalFollowersCount,
}
tm, err := time.Parse(time.RubyDate, user.CreatedAt)
@ -396,25 +399,28 @@ func parseProfileV2(user userResult) Profile {
u := user.Legacy
description := expandURLs(u.Description, u.Entities.Description.Urls, []ExtendedMedia{})
profile := Profile{
Avatar: u.ProfileImageURLHTTPS,
Banner: u.ProfileBannerURL,
Biography: description,
FollowersCount: u.FollowersCount,
FollowingCount: u.FavouritesCount,
FriendsCount: u.FriendsCount,
IsVerified: u.Verified,
LikesCount: u.FavouritesCount,
ListedCount: u.ListedCount,
Location: u.Location,
Name: u.Name,
PinnedTweetIDs: u.PinnedTweetIdsStr,
TweetsCount: u.StatusesCount,
URL: "https://twitter.com/" + u.ScreenName,
UserID: user.ID,
Username: u.ScreenName,
Sensitive: u.PossiblySensitive,
Following: u.Following,
FollowedBy: u.FollowedBy,
Avatar: u.ProfileImageURLHTTPS,
Banner: u.ProfileBannerURL,
Biography: description,
FollowersCount: u.FollowersCount,
FollowingCount: u.FavouritesCount,
FriendsCount: u.FriendsCount,
IsVerified: u.Verified,
IsBlueVerified: user.IsBlueVerified,
ProfileImageShape: user.ProfileImageShape,
HasGraduatedAccess: user.HasGraduatedAccess,
LikesCount: u.FavouritesCount,
ListedCount: u.ListedCount,
Location: u.Location,
Name: u.Name,
PinnedTweetIDs: u.PinnedTweetIdsStr,
TweetsCount: u.StatusesCount,
URL: "https://twitter.com/" + u.ScreenName,
UserID: user.ID,
Username: u.ScreenName,
Sensitive: u.PossiblySensitive,
Following: u.Following,
FollowedBy: u.FollowedBy,
}
tm, err := time.Parse(time.RubyDate, u.CreatedAt)