2018-11-29 17:33:44 +02:00
|
|
|
# Twitter Scraper
|
|
|
|
|
|
2020-06-15 14:56:52 +03:00
|
|
|
Golang implementation of python library <https://github.com/bisguzar/twitter-scraper>
|
2018-11-29 17:33:44 +02:00
|
|
|
|
|
|
|
|
Twitter's API is annoying to work with, and has lots of limitations —
|
|
|
|
|
luckily their frontend (JavaScript) has it's own API, which I reverse-engineered.
|
|
|
|
|
No API rate limits. No tokens needed. No restrictions. Extremely fast.
|
|
|
|
|
|
|
|
|
|
You can use this library to get the text of any user's Tweets trivially.
|
|
|
|
|
|
2020-06-15 15:05:41 +03:00
|
|
|
## Installation
|
|
|
|
|
|
|
|
|
|
```shell
|
|
|
|
|
go get -u github.com/n0madic/twitter-scraper
|
|
|
|
|
```
|
|
|
|
|
|
2018-11-29 17:33:44 +02:00
|
|
|
## Usage
|
|
|
|
|
|
2020-05-14 14:59:33 +02:00
|
|
|
### Get user tweets
|
2019-09-21 10:59:45 +03:00
|
|
|
|
2018-11-29 17:33:44 +02:00
|
|
|
```golang
|
|
|
|
|
package main
|
|
|
|
|
|
|
|
|
|
import (
|
2020-06-15 15:19:14 +03:00
|
|
|
"context"
|
2018-11-29 17:33:44 +02:00
|
|
|
"fmt"
|
|
|
|
|
twitterscraper "github.com/n0madic/twitter-scraper"
|
|
|
|
|
)
|
|
|
|
|
|
|
|
|
|
func main() {
|
2020-06-15 16:16:08 +03:00
|
|
|
for tweet := range twitterscraper.GetTweets(context.Background(), "Twitter", 50) {
|
2019-09-21 11:02:22 +03:00
|
|
|
if tweet.Error != nil {
|
|
|
|
|
panic(tweet.Error)
|
|
|
|
|
}
|
2018-11-29 17:33:44 +02:00
|
|
|
fmt.Println(tweet.HTML)
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
```
|
|
|
|
|
|
2020-06-15 16:16:08 +03:00
|
|
|
It appears you can ask for up to 50 tweets.
|
2018-11-29 17:33:44 +02:00
|
|
|
|
2020-05-15 17:56:57 +02:00
|
|
|
### Search tweets by query standard operators
|
2020-05-14 14:59:33 +02:00
|
|
|
|
|
|
|
|
Tweets containing “twitter” and “scraper” and “data“, filtering out retweets:
|
|
|
|
|
|
|
|
|
|
```golang
|
|
|
|
|
package main
|
|
|
|
|
|
|
|
|
|
import (
|
2020-06-15 15:19:14 +03:00
|
|
|
"context"
|
2020-05-14 14:59:33 +02:00
|
|
|
"fmt"
|
|
|
|
|
twitterscraper "github.com/n0madic/twitter-scraper"
|
|
|
|
|
)
|
|
|
|
|
|
|
|
|
|
func main() {
|
2020-06-15 14:56:52 +03:00
|
|
|
for tweet := range twitterscraper.SearchTweets(context.Background(),
|
|
|
|
|
"twitter scraper data -filter:retweets", 50) {
|
2020-05-14 14:59:33 +02:00
|
|
|
if tweet.Error != nil {
|
|
|
|
|
panic(tweet.Error)
|
|
|
|
|
}
|
|
|
|
|
fmt.Println(tweet.HTML)
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
```
|
2020-12-04 15:08:33 +07:00
|
|
|
#### With http proxy
|
2020-05-14 14:59:33 +02:00
|
|
|
|
2020-12-04 15:08:33 +07:00
|
|
|
```golang
|
|
|
|
|
package main
|
|
|
|
|
|
|
|
|
|
import (
|
|
|
|
|
"context"
|
|
|
|
|
"fmt"
|
|
|
|
|
twitterscraper "github.com/n0madic/twitter-scraper"
|
|
|
|
|
)
|
|
|
|
|
|
|
|
|
|
func main() {
|
|
|
|
|
twitterscraper.SetProxy("http://localhost:16379")
|
|
|
|
|
for tweet := range twitterscraper.SearchTweets(context.Background(),
|
|
|
|
|
"twitter scraper data -filter:retweets", 50) {
|
|
|
|
|
if tweet.Error != nil {
|
|
|
|
|
panic(tweet.Error)
|
|
|
|
|
}
|
|
|
|
|
fmt.Println(tweet.HTML)
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
```
|
2020-05-14 14:59:33 +02:00
|
|
|
The search ends if we have 50 tweets.
|
|
|
|
|
|
2020-06-15 15:05:41 +03:00
|
|
|
See [Rules and filtering](https://developer.twitter.com/en/docs/tweets/rules-and-filtering/overview/standard-operators) for build standard queries.
|
2020-05-14 14:59:33 +02:00
|
|
|
|
2019-09-21 10:59:45 +03:00
|
|
|
### Get profile
|
|
|
|
|
|
|
|
|
|
```golang
|
|
|
|
|
package main
|
|
|
|
|
|
|
|
|
|
import (
|
2019-09-21 11:02:22 +03:00
|
|
|
"fmt"
|
|
|
|
|
twitterscraper "github.com/n0madic/twitter-scraper"
|
2019-09-21 10:59:45 +03:00
|
|
|
)
|
|
|
|
|
|
|
|
|
|
func main() {
|
2020-06-15 14:56:52 +03:00
|
|
|
profile, err := twitterscraper.GetProfile("Twitter")
|
2019-09-21 11:02:22 +03:00
|
|
|
if err != nil {
|
|
|
|
|
panic(err)
|
|
|
|
|
}
|
|
|
|
|
fmt.Printf("%+v\n", profile)
|
2019-09-21 10:59:45 +03:00
|
|
|
}
|
|
|
|
|
```
|
|
|
|
|
|
2020-02-12 10:45:19 +02:00
|
|
|
### Get trends
|
|
|
|
|
|
|
|
|
|
```golang
|
|
|
|
|
package main
|
|
|
|
|
|
|
|
|
|
import (
|
|
|
|
|
"fmt"
|
|
|
|
|
twitterscraper "github.com/n0madic/twitter-scraper"
|
|
|
|
|
)
|
|
|
|
|
|
|
|
|
|
func main() {
|
|
|
|
|
trends, err := twitterscraper.GetTrends()
|
|
|
|
|
if err != nil {
|
|
|
|
|
panic(err)
|
|
|
|
|
}
|
|
|
|
|
fmt.Println(trends)
|
|
|
|
|
}
|
|
|
|
|
```
|