diff --git a/CHANGELOG.md b/CHANGELOG.md index 838e2e3..88cb146 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,11 @@ # Changelog +## v0.0.9 + +24.07.2024 + +- Added method `GetSpace` + ## v0.0.8 09.07.2024 diff --git a/README.md b/README.md index 89cce4e..ec6aedd 100644 --- a/README.md +++ b/README.md @@ -34,6 +34,7 @@ You can use this library to get tweets, profiles, and trends trivially. - [Get trends](#get-trends) - [Get following](#get-following) - [Get followers](#get-followers) + - [Get space](#get-space) - [Create tweet](#create-tweet) - [Delete tweet](#delete-tweet) - [Create retweet](#create-retweet) @@ -237,7 +238,6 @@ tweets, cursor, err := scraper.FetchTweets("taylorswift13", 20, cursor) `GetMediaTweets` returns a channel with the specified number of user tweets that contain media. It’s using the `FetchMediaTweets` method under the hood. Read how this method works in [Methods that returns channels](#methods-that-returns-channels). - ```golang for tweet := range scraper.GetMediaTweets(context.Background(), "taylorswift13", 50) { if tweet.Error != nil { @@ -263,7 +263,6 @@ tweets, cursor, err := scraper.FetchMediaTweets("taylorswift13", 20, cursor) `GetBookmarks` returns a channel with the specified number of bookmarked tweets. It’s using the `FetchBookmarks` method under the hood. Read how this method works in [Methods that returns channels](#methods-that-returns-channels). - ```golang for tweet := range scraper.GetBookmarks(context.Background(), 50) { if tweet.Error != nil { @@ -289,7 +288,6 @@ tweets, cursor, err := scraper.FetchBookmarks(20, cursor) `GetHomeTweets` returns a channel with the specified number of latest home tweets. It’s using the `FetchHomeTweets` method under the hood. Read how this method works in [Methods that returns channels](#methods-that-returns-channels). - ```golang for tweet := range scraper.GetHomeTweets(context.Background(), 50) { if tweet.Error != nil { @@ -315,7 +313,6 @@ tweets, cursor, err := scraper.FetchHomeTweets(20, cursor) `GetForYouTweets` returns a channel with the specified number of for you home tweets. It’s using the `FetchForYouTweets` method under the hood. Read how this method works in [Methods that returns channels](#methods-that-returns-channels). - ```golang for tweet := range scraper.GetForYouTweets(context.Background(), 50) { if tweet.Error != nil { @@ -341,7 +338,6 @@ tweets, cursor, err := scraper.FetchForYouTweets(20, cursor) `SearchTweets` returns a channel with the specified number of tweets that contain media. It’s using the `FetchSearchTweets` method under the hood. Read how this method works in [Methods that returns channels](#methods-that-returns-channels). - ```golang for tweet := range scraper.SearchTweets(context.Background(), "twitter scraper data -filter:retweets", 50) { @@ -385,7 +381,6 @@ profile, err := scraper.GetProfile("taylorswift13") `SearchProfiles` returns a channel with the specified number of tweets that contain media. It’s using the `FetchSearchProfiles` method under the hood. Read how this method works in [Methods that returns channels](#methods-that-returns-channels). - ```golang for profile := range scraper.SearchProfiles(context.Background(), "Twitter", 50) { if profile.Error != nil { @@ -431,6 +426,37 @@ var cursor string users, cursor, err := scraper.FetchFollowers("Support", 20, cursor) ``` +### Get space + +> [!IMPORTANT] +> Requires authentication! + +500 requests / 15 minutes + +Use to retrvie data about space and it's participants. You can get up to 1000 participants of space. If method returns less, it's probably because listeners is anonymous. + +```golang +space, err := scraper.GetSpace("space_id") +``` + +You can get `space_id` from space url which can be retrived from tweet. For example: + +```golang +tweet, err := testScraper.GetTweet("1815884577040445599") +if err != nil { + t.Fatal(err) +} + +var spaceId string +spaceUrl := tweet.URLs[0] // https://twitter.com/i/spaces/1mnxeAMPEqqxX + +if strings.HasPrefix(spaceUrl, "https://twitter.com/i/spaces/") { + spaceId = strings.Replace(spaceUrl, "https://twitter.com/i/spaces/", "", 1) // 1mnxeAMPEqqxX +} + +space, err := scraper.GetSpace(spaceId) +``` + ### Create tweet > [!IMPORTANT] diff --git a/spaces.go b/spaces.go new file mode 100644 index 0000000..367fd8e --- /dev/null +++ b/spaces.go @@ -0,0 +1,279 @@ +package twitterscraper + +import ( + "errors" + "net/url" + "time" +) + +func (s *Scraper) GetSpace(id string) (*Space, error) { + if !s.isLogged { + return nil, errors.New("scraper is not logged in") + } + + req, err := s.newRequest("GET", "https://twitter.com/i/api/graphql/d03OdorPdZ_sH9V3D1_yWQ/AudioSpaceById") + if err != nil { + return nil, err + } + + variables := map[string]interface{}{ + "id": id, + "isMetatagsQuery": false, + "withReplays": true, + "withListeners": true, + } + + features := map[string]interface{}{ + "spaces_2022_h2_spaces_communities": true, + "spaces_2022_h2_clipping": true, + "creator_subscriptions_tweet_preview_api_enabled": true, + "rweb_tipjar_consumption_enabled": true, + "responsive_web_graphql_exclude_directive_enabled": true, + "verified_phone_label_enabled": false, + "communities_web_enable_tweet_community_results_fetch": true, + "c9s_tweet_anatomy_moderator_badge_enabled": true, + "articles_preview_enabled": true, + "responsive_web_graphql_skip_user_profile_image_extensions_enabled": false, + "tweetypie_unmention_optimization_enabled": true, + "responsive_web_edit_tweet_api_enabled": true, + "graphql_is_translatable_rweb_tweet_is_translatable_enabled": true, + "view_counts_everywhere_api_enabled": true, + "longform_notetweets_consumption_enabled": true, + "responsive_web_twitter_article_tweet_consumption_enabled": true, + "tweet_awards_web_tipping_enabled": false, + "creator_subscriptions_quote_tweet_preview_enabled": false, + "freedom_of_speech_not_reach_fetch_enabled": true, + "standardized_nudges_misinfo": true, + "tweet_with_visibility_results_prefer_gql_limited_actions_policy_enabled": true, + "rweb_video_timestamps_enabled": true, + "longform_notetweets_rich_text_read_enabled": true, + "longform_notetweets_inline_media_enabled": true, + "responsive_web_graphql_timeline_navigation_enabled": true, + "responsive_web_enhance_cards_enabled": false, + } + + query := url.Values{} + query.Set("variables", mapToJSONString(variables)) + query.Set("features", mapToJSONString(features)) + req.URL.RawQuery = query.Encode() + + var spaceData space + err = s.RequestAPI(req, &spaceData) + if err != nil { + return nil, err + } + + space := spaceData.parse() + + if space.ID == "" { + return nil, errors.New("some erorr happend") + } + + return space, nil +} + +type Topic struct { + ID string + Title string +} + +type SpaceUser struct { + UserID string + Username string + Name string + Avatar string + IsVerified bool + ConnectedAt time.Time +} + +type SpaceParticipants struct { + TotalCount int + CurrentCount int + Admins []*SpaceUser + Speakers []*SpaceUser + Listeners []*SpaceUser +} + +type Space struct { + ID string + State string + Title string + ContentType string + Topics []Topic + Participants SpaceParticipants + CreatedAt time.Time + ScheduledStart time.Time + StartedAt time.Time + UpdatedAt time.Time +} + +type spaceUser struct { + PeriscopeUserID string `json:"periscope_user_id"` + Start int64 `json:"start"` + TwitterScreenName string `json:"twitter_screen_name"` + DisplayName string `json:"display_name"` + AvatarURL string `json:"avatar_url"` + IsVerified bool `json:"is_verified"` + IsMutedByAdmin bool `json:"is_muted_by_admin"` + IsMutedByGuest bool `json:"is_muted_by_guest"` + UserResults struct { + RestID string `json:"rest_id"` + Result struct { + Typename string `json:"__typename"` + IdentityProfileLabelsHighlightedLabel struct { + } `json:"identity_profile_labels_highlighted_label"` + IsBlueVerified bool `json:"is_blue_verified"` + Legacy struct { + } `json:"legacy"` + } `json:"result"` + } `json:"user_results"` +} + +type space struct { + Data struct { + AudioSpace struct { + Metadata struct { + RestID string `json:"rest_id"` + State string `json:"state"` + Title string `json:"title"` + MediaKey string `json:"media_key"` + CreatedAt int64 `json:"created_at"` + ScheduledStart int64 `json:"scheduled_start"` + StartedAt int64 `json:"started_at"` + UpdatedAt int64 `json:"updated_at"` + ContentType string `json:"content_type"` + CreatorResults struct { + Result struct { + Typename string `json:"__typename"` + ID string `json:"id"` + RestID string `json:"rest_id"` + HasGraduatedAccess bool `json:"has_graduated_access"` + IsBlueVerified bool `json:"is_blue_verified"` + ProfileImageShape string `json:"profile_image_shape"` + Legacy legacyUser `json:"legacy"` + } `json:"result"` + } `json:"creator_results"` + ConversationControls int `json:"conversation_controls"` + DisallowJoin bool `json:"disallow_join"` + IsEmployeeOnly bool `json:"is_employee_only"` + IsLocked bool `json:"is_locked"` + IsMuted bool `json:"is_muted"` + IsSpaceAvailableForClipping bool `json:"is_space_available_for_clipping"` + IsSpaceAvailableForReplay bool `json:"is_space_available_for_replay"` + MentionedUsers []struct { + RestID string `json:"rest_id"` + } `json:"mentioned_users"` + NarrowCastSpaceType int `json:"narrow_cast_space_type"` + NoIncognito bool `json:"no_incognito"` + TotalReplayWatched int `json:"total_replay_watched"` + TotalLiveListeners int `json:"total_live_listeners"` + Topics []struct { + Topic struct { + TopicID string `json:"topic_id"` + Name string `json:"name"` + } `json:"topic"` + } `json:"topics"` + TweetResults struct { + Result tweet `json:"result"` + } `json:"tweet_results"` + MaxGuestSessions int `json:"max_guest_sessions"` + MaxAdminCapacity int `json:"max_admin_capacity"` + } `json:"metadata"` + + IsSubscribed bool `json:"is_subscribed"` + Participants struct { + Total int `json:"total"` + Admins []spaceUser `json:"admins"` + Speakers []spaceUser `json:"speakers"` + Listeners []spaceUser `json:"listeners"` + } `json:"participants"` + Sharings struct { + Items []struct { + SharingID string `json:"sharing_id"` + CreatedAtMs int64 `json:"created_at_ms"` + UpdatedAtMs int64 `json:"updated_at_ms"` + SharedItem struct { + Typename string `json:"__typename"` + TweetResults struct { + Result struct { + Typename string `json:"__typename"` + RestID string `json:"rest_id"` + Core tweet `json:"result"` + } `json:"tweet_results"` + } `json:"shared_item"` + UserResults struct { + Result struct { + Typename string `json:"__typename"` + ID string `json:"id"` + RestID string `json:"rest_id"` + AffiliatesHighlightedLabel struct { + } `json:"affiliates_highlighted_label"` + HasGraduatedAccess bool `json:"has_graduated_access"` + IsBlueVerified bool `json:"is_blue_verified"` + ProfileImageShape string `json:"profile_image_shape"` + Legacy legacyUser `json:"legacy"` + TipjarSettings struct { + IsEnabled bool `json:"is_enabled"` + CashAppHandle string `json:"cash_app_handle"` + } `json:"tipjar_settings"` + } `json:"result"` + } `json:"user_results"` + } `json:"shared_item"` + } `json:"items"` + } `json:"sharings"` + } `json:"audioSpace"` + } `json:"data"` +} + +func (user *spaceUser) parse() *SpaceUser { + result := &SpaceUser{ + UserID: user.UserResults.RestID, + Username: user.TwitterScreenName, + Name: user.DisplayName, + Avatar: user.AvatarURL, + IsVerified: user.IsVerified, + ConnectedAt: time.UnixMilli(user.Start), + } + + return result +} + +func (space *space) parse() *Space { + result := &Space{ + ID: space.Data.AudioSpace.Metadata.RestID, + State: space.Data.AudioSpace.Metadata.State, + Title: space.Data.AudioSpace.Metadata.Title, + ContentType: space.Data.AudioSpace.Metadata.ContentType, + Topics: []Topic{}, + Participants: SpaceParticipants{ + TotalCount: space.Data.AudioSpace.Metadata.TotalLiveListeners, + CurrentCount: space.Data.AudioSpace.Participants.Total, + }, + CreatedAt: time.UnixMilli(space.Data.AudioSpace.Metadata.CreatedAt), + ScheduledStart: time.UnixMilli(space.Data.AudioSpace.Metadata.ScheduledStart), + StartedAt: time.UnixMilli(space.Data.AudioSpace.Metadata.StartedAt), + UpdatedAt: time.UnixMilli(space.Data.AudioSpace.Metadata.UpdatedAt), + } + + for _, topic := range space.Data.AudioSpace.Metadata.Topics { + result.Topics = append(result.Topics, Topic{ + ID: topic.Topic.TopicID, + Title: topic.Topic.Name, + }) + } + + for _, admin := range space.Data.AudioSpace.Participants.Admins { + result.Participants.Admins = append(result.Participants.Admins, admin.parse()) + } + + for _, speaker := range space.Data.AudioSpace.Participants.Speakers { + result.Participants.Speakers = append(result.Participants.Speakers, speaker.parse()) + } + + for _, listener := range space.Data.AudioSpace.Participants.Listeners { + result.Participants.Listeners = append(result.Participants.Listeners, listener.parse()) + + } + + return result +} diff --git a/spaces_test.go b/spaces_test.go new file mode 100644 index 0000000..ab1fc96 --- /dev/null +++ b/spaces_test.go @@ -0,0 +1,23 @@ +package twitterscraper_test + +import ( + "errors" + "testing" +) + +func TestGetSpace(t *testing.T) { + if skipAuthTest { + t.Skip("Skipping test due to environment variable") + } + + spaceId := "1OdJrXPVLEnKX" + + space, err := testScraper.GetSpace(spaceId) + if err != nil { + t.Fatal(err) + } + + if space.ID != spaceId { + t.Fatal(errors.New("returned space id is not requested")) + } +}