make a couple of changes pr:43
This commit is contained in:
parent
d3057f34fb
commit
7e61608f79
3 changed files with 68 additions and 45 deletions
21
README.md
21
README.md
|
|
@ -29,6 +29,12 @@ import (
|
||||||
|
|
||||||
func main() {
|
func main() {
|
||||||
scraper := twitterscraper.New()
|
scraper := twitterscraper.New()
|
||||||
|
|
||||||
|
// Cookie and xCsrfToken is optional
|
||||||
|
// Some specified user tweets are protected that you must login and follow
|
||||||
|
scraper.WithCookie("twitter cookie after login")
|
||||||
|
scraper.WithXCsrfToken("twitter X-Csrf-Token after login")
|
||||||
|
|
||||||
for tweet := range scraper.GetTweets(context.Background(), "Twitter", 50) {
|
for tweet := range scraper.GetTweets(context.Background(), "Twitter", 50) {
|
||||||
if tweet.Error != nil {
|
if tweet.Error != nil {
|
||||||
panic(tweet.Error)
|
panic(tweet.Error)
|
||||||
|
|
@ -167,7 +173,11 @@ func main() {
|
||||||
}
|
}
|
||||||
```
|
```
|
||||||
|
|
||||||
### Use http proxy
|
### Use Proxy
|
||||||
|
|
||||||
|
Support http and socks5 proxy
|
||||||
|
|
||||||
|
#### with http
|
||||||
|
|
||||||
```golang
|
```golang
|
||||||
err := scraper.SetProxy("http://localhost:3128")
|
err := scraper.SetProxy("http://localhost:3128")
|
||||||
|
|
@ -176,6 +186,15 @@ if err != nil {
|
||||||
}
|
}
|
||||||
```
|
```
|
||||||
|
|
||||||
|
#### with socks5
|
||||||
|
|
||||||
|
```golang
|
||||||
|
err := scraper.SetProxy("socks5://localhost:3128")
|
||||||
|
if err != nil {
|
||||||
|
panic(err)
|
||||||
|
}
|
||||||
|
```
|
||||||
|
|
||||||
### Delay requests
|
### Delay requests
|
||||||
|
|
||||||
Add delay between API requests (in seconds)
|
Add delay between API requests (in seconds)
|
||||||
|
|
|
||||||
6
api.go
6
api.go
|
|
@ -34,9 +34,9 @@ func (s *Scraper) RequestAPI(req *http.Request, target interface{}) error {
|
||||||
req.Header.Set("X-Guest-Token", s.guestToken)
|
req.Header.Set("X-Guest-Token", s.guestToken)
|
||||||
|
|
||||||
// use cookie
|
// use cookie
|
||||||
if len(s.Cookie) > 0 && len(s.XCsrfToken) > 0 {
|
if len(s.cookie) > 0 && len(s.xCsrfToken) > 0 {
|
||||||
req.Header.Set("Cookie", s.Cookie)
|
req.Header.Set("Cookie", s.cookie)
|
||||||
req.Header.Set("x-csrf-token", s.XCsrfToken)
|
req.Header.Set("x-csrf-token", s.xCsrfToken)
|
||||||
}
|
}
|
||||||
|
|
||||||
resp, err := s.client.Do(req)
|
resp, err := s.client.Do(req)
|
||||||
|
|
|
||||||
64
scraper.go
64
scraper.go
|
|
@ -15,6 +15,7 @@ import (
|
||||||
// Scraper object
|
// Scraper object
|
||||||
type Scraper struct {
|
type Scraper struct {
|
||||||
client *http.Client
|
client *http.Client
|
||||||
|
clientTimeout time.Duration
|
||||||
delay int64
|
delay int64
|
||||||
guestToken string
|
guestToken string
|
||||||
guestCreatedAt time.Time
|
guestCreatedAt time.Time
|
||||||
|
|
@ -22,8 +23,8 @@ type Scraper struct {
|
||||||
searchMode SearchMode
|
searchMode SearchMode
|
||||||
wg sync.WaitGroup
|
wg sync.WaitGroup
|
||||||
|
|
||||||
Cookie string
|
cookie string
|
||||||
XCsrfToken string
|
xCsrfToken string
|
||||||
}
|
}
|
||||||
|
|
||||||
// SearchMode type
|
// SearchMode type
|
||||||
|
|
@ -42,12 +43,16 @@ const (
|
||||||
SearchUsers
|
SearchUsers
|
||||||
)
|
)
|
||||||
|
|
||||||
|
// default http client timeout
|
||||||
|
const DefaultClientTimeout = 10 * time.Second
|
||||||
|
|
||||||
var defaultScraper *Scraper
|
var defaultScraper *Scraper
|
||||||
|
|
||||||
// New creates a Scraper object
|
// New creates a Scraper object
|
||||||
func New() *Scraper {
|
func New() *Scraper {
|
||||||
return &Scraper{
|
return &Scraper{
|
||||||
client: &http.Client{Timeout: 10 * time.Second},
|
client: &http.Client{Timeout: DefaultClientTimeout},
|
||||||
|
clientTimeout: DefaultClientTimeout,
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
@ -86,22 +91,28 @@ func WithReplies(b bool) *Scraper {
|
||||||
|
|
||||||
// cookie
|
// cookie
|
||||||
func (s *Scraper) WithCookie(cookie string) *Scraper {
|
func (s *Scraper) WithCookie(cookie string) *Scraper {
|
||||||
s.Cookie = cookie
|
s.cookie = cookie
|
||||||
return s
|
return s
|
||||||
}
|
}
|
||||||
|
|
||||||
// x csrf token
|
// x csrf token
|
||||||
func (s *Scraper) WithXCsrfToken(xcsrfToken string) *Scraper {
|
func (s *Scraper) WithXCsrfToken(xcsrfToken string) *Scraper {
|
||||||
s.XCsrfToken = xcsrfToken
|
s.xCsrfToken = xcsrfToken
|
||||||
return s
|
return s
|
||||||
}
|
}
|
||||||
|
|
||||||
// SetProxy set http proxy in the format `http://HOST:PORT`
|
// client timeout
|
||||||
func (s *Scraper) SetProxy(proxy string) error {
|
func (s *Scraper) WithClientTimeout(timeout time.Duration) *Scraper {
|
||||||
if !strings.HasPrefix(proxy, "http") {
|
s.clientTimeout = timeout
|
||||||
return errors.New("only support http(s) protocol")
|
return s
|
||||||
}
|
}
|
||||||
urlproxy, err := url.Parse(proxy)
|
|
||||||
|
// SetProxy
|
||||||
|
// set http proxy in the format `http://HOST:PORT`
|
||||||
|
// set socket proxy in the format `socks5://HOST:PORT`
|
||||||
|
func (s *Scraper) SetProxy(proxyAddr string) error {
|
||||||
|
if strings.HasPrefix(proxyAddr, "http") {
|
||||||
|
urlproxy, err := url.Parse(proxyAddr)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
|
|
@ -110,23 +121,21 @@ func (s *Scraper) SetProxy(proxy string) error {
|
||||||
Proxy: http.ProxyURL(urlproxy),
|
Proxy: http.ProxyURL(urlproxy),
|
||||||
TLSNextProto: make(map[string]func(authority string, c *tls.Conn) http.RoundTripper),
|
TLSNextProto: make(map[string]func(authority string, c *tls.Conn) http.RoundTripper),
|
||||||
DialContext: (&net.Dialer{
|
DialContext: (&net.Dialer{
|
||||||
Timeout: 10 * time.Second,
|
Timeout: s.clientTimeout,
|
||||||
}).DialContext,
|
}).DialContext,
|
||||||
},
|
},
|
||||||
}
|
}
|
||||||
return nil
|
return nil
|
||||||
}
|
|
||||||
|
|
||||||
// SetProxy set socks5 proxy in the format `HOST:PORT`
|
|
||||||
func (s *Scraper) SetSocks5Proxy(socks5 string) error {
|
|
||||||
baseDialer := &net.Dialer{
|
|
||||||
Timeout: 30 * time.Second,
|
|
||||||
KeepAlive: 30 * time.Second,
|
|
||||||
}
|
}
|
||||||
if socks5 != "" {
|
if strings.HasPrefix(proxyAddr, "socks5") {
|
||||||
dialSocksProxy, err := proxy.SOCKS5("tcp", socks5, nil, baseDialer)
|
baseDialer := &net.Dialer{
|
||||||
|
Timeout: s.clientTimeout,
|
||||||
|
KeepAlive: s.clientTimeout,
|
||||||
|
}
|
||||||
|
socksHostPort := strings.ReplaceAll(proxyAddr, "socks5://", "")
|
||||||
|
dialSocksProxy, err := proxy.SOCKS5("tcp", socksHostPort, nil, baseDialer)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return errors.New("Error creating SOCKS5 proxy")
|
return errors.New("error creating socks5 proxy :" + err.Error())
|
||||||
}
|
}
|
||||||
if contextDialer, ok := dialSocksProxy.(proxy.ContextDialer); ok {
|
if contextDialer, ok := dialSocksProxy.(proxy.ContextDialer); ok {
|
||||||
dialContext := contextDialer.DialContext
|
dialContext := contextDialer.DialContext
|
||||||
|
|
@ -136,16 +145,11 @@ func (s *Scraper) SetSocks5Proxy(socks5 string) error {
|
||||||
},
|
},
|
||||||
}
|
}
|
||||||
} else {
|
} else {
|
||||||
return errors.New("Failed type assertion to DialContext")
|
return errors.New("failed type assertion to DialContext")
|
||||||
}
|
|
||||||
} else {
|
|
||||||
s.client = &http.Client{
|
|
||||||
Transport: &http.Transport{
|
|
||||||
DialContext: (baseDialer).DialContext,
|
|
||||||
},
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
return nil
|
return nil
|
||||||
|
}
|
||||||
|
return errors.New("only support http(s) or socks5 protocol")
|
||||||
}
|
}
|
||||||
|
|
||||||
// SetProxy wrapper for default Scraper
|
// SetProxy wrapper for default Scraper
|
||||||
|
|
|
||||||
Loading…
Add table
Add a link
Reference in a new issue