make a couple of changes pr:43

This commit is contained in:
vanilla 2021-09-09 11:15:53 +08:00
parent d3057f34fb
commit 7e61608f79
3 changed files with 68 additions and 45 deletions

View file

@ -29,6 +29,12 @@ import (
func main() { func main() {
scraper := twitterscraper.New() scraper := twitterscraper.New()
// Cookie and xCsrfToken is optional
// Some specified user tweets are protected that you must login and follow
scraper.WithCookie("twitter cookie after login")
scraper.WithXCsrfToken("twitter X-Csrf-Token after login")
for tweet := range scraper.GetTweets(context.Background(), "Twitter", 50) { for tweet := range scraper.GetTweets(context.Background(), "Twitter", 50) {
if tweet.Error != nil { if tweet.Error != nil {
panic(tweet.Error) panic(tweet.Error)
@ -167,7 +173,11 @@ func main() {
} }
``` ```
### Use http proxy ### Use Proxy
Support http and socks5 proxy
#### with http
```golang ```golang
err := scraper.SetProxy("http://localhost:3128") err := scraper.SetProxy("http://localhost:3128")
@ -176,6 +186,15 @@ if err != nil {
} }
``` ```
#### with socks5
```golang
err := scraper.SetProxy("socks5://localhost:3128")
if err != nil {
panic(err)
}
```
### Delay requests ### Delay requests
Add delay between API requests (in seconds) Add delay between API requests (in seconds)

6
api.go
View file

@ -34,9 +34,9 @@ func (s *Scraper) RequestAPI(req *http.Request, target interface{}) error {
req.Header.Set("X-Guest-Token", s.guestToken) req.Header.Set("X-Guest-Token", s.guestToken)
// use cookie // use cookie
if len(s.Cookie) > 0 && len(s.XCsrfToken) > 0 { if len(s.cookie) > 0 && len(s.xCsrfToken) > 0 {
req.Header.Set("Cookie", s.Cookie) req.Header.Set("Cookie", s.cookie)
req.Header.Set("x-csrf-token", s.XCsrfToken) req.Header.Set("x-csrf-token", s.xCsrfToken)
} }
resp, err := s.client.Do(req) resp, err := s.client.Do(req)

View file

@ -15,6 +15,7 @@ import (
// Scraper object // Scraper object
type Scraper struct { type Scraper struct {
client *http.Client client *http.Client
clientTimeout time.Duration
delay int64 delay int64
guestToken string guestToken string
guestCreatedAt time.Time guestCreatedAt time.Time
@ -22,8 +23,8 @@ type Scraper struct {
searchMode SearchMode searchMode SearchMode
wg sync.WaitGroup wg sync.WaitGroup
Cookie string cookie string
XCsrfToken string xCsrfToken string
} }
// SearchMode type // SearchMode type
@ -42,12 +43,16 @@ const (
SearchUsers SearchUsers
) )
// default http client timeout
const DefaultClientTimeout = 10 * time.Second
var defaultScraper *Scraper var defaultScraper *Scraper
// New creates a Scraper object // New creates a Scraper object
func New() *Scraper { func New() *Scraper {
return &Scraper{ return &Scraper{
client: &http.Client{Timeout: 10 * time.Second}, client: &http.Client{Timeout: DefaultClientTimeout},
clientTimeout: DefaultClientTimeout,
} }
} }
@ -86,22 +91,28 @@ func WithReplies(b bool) *Scraper {
// cookie // cookie
func (s *Scraper) WithCookie(cookie string) *Scraper { func (s *Scraper) WithCookie(cookie string) *Scraper {
s.Cookie = cookie s.cookie = cookie
return s return s
} }
// x csrf token // x csrf token
func (s *Scraper) WithXCsrfToken(xcsrfToken string) *Scraper { func (s *Scraper) WithXCsrfToken(xcsrfToken string) *Scraper {
s.XCsrfToken = xcsrfToken s.xCsrfToken = xcsrfToken
return s return s
} }
// SetProxy set http proxy in the format `http://HOST:PORT` // client timeout
func (s *Scraper) SetProxy(proxy string) error { func (s *Scraper) WithClientTimeout(timeout time.Duration) *Scraper {
if !strings.HasPrefix(proxy, "http") { s.clientTimeout = timeout
return errors.New("only support http(s) protocol") return s
} }
urlproxy, err := url.Parse(proxy)
// SetProxy
// set http proxy in the format `http://HOST:PORT`
// set socket proxy in the format `socks5://HOST:PORT`
func (s *Scraper) SetProxy(proxyAddr string) error {
if strings.HasPrefix(proxyAddr, "http") {
urlproxy, err := url.Parse(proxyAddr)
if err != nil { if err != nil {
return err return err
} }
@ -110,23 +121,21 @@ func (s *Scraper) SetProxy(proxy string) error {
Proxy: http.ProxyURL(urlproxy), Proxy: http.ProxyURL(urlproxy),
TLSNextProto: make(map[string]func(authority string, c *tls.Conn) http.RoundTripper), TLSNextProto: make(map[string]func(authority string, c *tls.Conn) http.RoundTripper),
DialContext: (&net.Dialer{ DialContext: (&net.Dialer{
Timeout: 10 * time.Second, Timeout: s.clientTimeout,
}).DialContext, }).DialContext,
}, },
} }
return nil return nil
} }
if strings.HasPrefix(proxyAddr, "socks5") {
// SetProxy set socks5 proxy in the format `HOST:PORT`
func (s *Scraper) SetSocks5Proxy(socks5 string) error {
baseDialer := &net.Dialer{ baseDialer := &net.Dialer{
Timeout: 30 * time.Second, Timeout: s.clientTimeout,
KeepAlive: 30 * time.Second, KeepAlive: s.clientTimeout,
} }
if socks5 != "" { socksHostPort := strings.ReplaceAll(proxyAddr, "socks5://", "")
dialSocksProxy, err := proxy.SOCKS5("tcp", socks5, nil, baseDialer) dialSocksProxy, err := proxy.SOCKS5("tcp", socksHostPort, nil, baseDialer)
if err != nil { if err != nil {
return errors.New("Error creating SOCKS5 proxy") return errors.New("error creating socks5 proxy :" + err.Error())
} }
if contextDialer, ok := dialSocksProxy.(proxy.ContextDialer); ok { if contextDialer, ok := dialSocksProxy.(proxy.ContextDialer); ok {
dialContext := contextDialer.DialContext dialContext := contextDialer.DialContext
@ -136,17 +145,12 @@ func (s *Scraper) SetSocks5Proxy(socks5 string) error {
}, },
} }
} else { } else {
return errors.New("Failed type assertion to DialContext") return errors.New("failed type assertion to DialContext")
}
} else {
s.client = &http.Client{
Transport: &http.Transport{
DialContext: (baseDialer).DialContext,
},
}
} }
return nil return nil
} }
return errors.New("only support http(s) or socks5 protocol")
}
// SetProxy wrapper for default Scraper // SetProxy wrapper for default Scraper
func SetProxy(proxy string) error { func SetProxy(proxy string) error {