make a couple of changes pr:43
This commit is contained in:
parent
d3057f34fb
commit
7e61608f79
3 changed files with 68 additions and 45 deletions
21
README.md
21
README.md
|
|
@ -29,6 +29,12 @@ import (
|
|||
|
||||
func main() {
|
||||
scraper := twitterscraper.New()
|
||||
|
||||
// Cookie and xCsrfToken is optional
|
||||
// Some specified user tweets are protected that you must login and follow
|
||||
scraper.WithCookie("twitter cookie after login")
|
||||
scraper.WithXCsrfToken("twitter X-Csrf-Token after login")
|
||||
|
||||
for tweet := range scraper.GetTweets(context.Background(), "Twitter", 50) {
|
||||
if tweet.Error != nil {
|
||||
panic(tweet.Error)
|
||||
|
|
@ -167,7 +173,11 @@ func main() {
|
|||
}
|
||||
```
|
||||
|
||||
### Use http proxy
|
||||
### Use Proxy
|
||||
|
||||
Support http and socks5 proxy
|
||||
|
||||
#### with http
|
||||
|
||||
```golang
|
||||
err := scraper.SetProxy("http://localhost:3128")
|
||||
|
|
@ -176,6 +186,15 @@ if err != nil {
|
|||
}
|
||||
```
|
||||
|
||||
#### with socks5
|
||||
|
||||
```golang
|
||||
err := scraper.SetProxy("socks5://localhost:3128")
|
||||
if err != nil {
|
||||
panic(err)
|
||||
}
|
||||
```
|
||||
|
||||
### Delay requests
|
||||
|
||||
Add delay between API requests (in seconds)
|
||||
|
|
|
|||
6
api.go
6
api.go
|
|
@ -34,9 +34,9 @@ func (s *Scraper) RequestAPI(req *http.Request, target interface{}) error {
|
|||
req.Header.Set("X-Guest-Token", s.guestToken)
|
||||
|
||||
// use cookie
|
||||
if len(s.Cookie) > 0 && len(s.XCsrfToken) > 0 {
|
||||
req.Header.Set("Cookie", s.Cookie)
|
||||
req.Header.Set("x-csrf-token", s.XCsrfToken)
|
||||
if len(s.cookie) > 0 && len(s.xCsrfToken) > 0 {
|
||||
req.Header.Set("Cookie", s.cookie)
|
||||
req.Header.Set("x-csrf-token", s.xCsrfToken)
|
||||
}
|
||||
|
||||
resp, err := s.client.Do(req)
|
||||
|
|
|
|||
58
scraper.go
58
scraper.go
|
|
@ -15,6 +15,7 @@ import (
|
|||
// Scraper object
|
||||
type Scraper struct {
|
||||
client *http.Client
|
||||
clientTimeout time.Duration
|
||||
delay int64
|
||||
guestToken string
|
||||
guestCreatedAt time.Time
|
||||
|
|
@ -22,8 +23,8 @@ type Scraper struct {
|
|||
searchMode SearchMode
|
||||
wg sync.WaitGroup
|
||||
|
||||
Cookie string
|
||||
XCsrfToken string
|
||||
cookie string
|
||||
xCsrfToken string
|
||||
}
|
||||
|
||||
// SearchMode type
|
||||
|
|
@ -42,12 +43,16 @@ const (
|
|||
SearchUsers
|
||||
)
|
||||
|
||||
// default http client timeout
|
||||
const DefaultClientTimeout = 10 * time.Second
|
||||
|
||||
var defaultScraper *Scraper
|
||||
|
||||
// New creates a Scraper object
|
||||
func New() *Scraper {
|
||||
return &Scraper{
|
||||
client: &http.Client{Timeout: 10 * time.Second},
|
||||
client: &http.Client{Timeout: DefaultClientTimeout},
|
||||
clientTimeout: DefaultClientTimeout,
|
||||
}
|
||||
}
|
||||
|
||||
|
|
@ -86,22 +91,28 @@ func WithReplies(b bool) *Scraper {
|
|||
|
||||
// cookie
|
||||
func (s *Scraper) WithCookie(cookie string) *Scraper {
|
||||
s.Cookie = cookie
|
||||
s.cookie = cookie
|
||||
return s
|
||||
}
|
||||
|
||||
// x csrf token
|
||||
func (s *Scraper) WithXCsrfToken(xcsrfToken string) *Scraper {
|
||||
s.XCsrfToken = xcsrfToken
|
||||
s.xCsrfToken = xcsrfToken
|
||||
return s
|
||||
}
|
||||
|
||||
// SetProxy set http proxy in the format `http://HOST:PORT`
|
||||
func (s *Scraper) SetProxy(proxy string) error {
|
||||
if !strings.HasPrefix(proxy, "http") {
|
||||
return errors.New("only support http(s) protocol")
|
||||
// client timeout
|
||||
func (s *Scraper) WithClientTimeout(timeout time.Duration) *Scraper {
|
||||
s.clientTimeout = timeout
|
||||
return s
|
||||
}
|
||||
urlproxy, err := url.Parse(proxy)
|
||||
|
||||
// SetProxy
|
||||
// set http proxy in the format `http://HOST:PORT`
|
||||
// set socket proxy in the format `socks5://HOST:PORT`
|
||||
func (s *Scraper) SetProxy(proxyAddr string) error {
|
||||
if strings.HasPrefix(proxyAddr, "http") {
|
||||
urlproxy, err := url.Parse(proxyAddr)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
|
@ -110,23 +121,21 @@ func (s *Scraper) SetProxy(proxy string) error {
|
|||
Proxy: http.ProxyURL(urlproxy),
|
||||
TLSNextProto: make(map[string]func(authority string, c *tls.Conn) http.RoundTripper),
|
||||
DialContext: (&net.Dialer{
|
||||
Timeout: 10 * time.Second,
|
||||
Timeout: s.clientTimeout,
|
||||
}).DialContext,
|
||||
},
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// SetProxy set socks5 proxy in the format `HOST:PORT`
|
||||
func (s *Scraper) SetSocks5Proxy(socks5 string) error {
|
||||
if strings.HasPrefix(proxyAddr, "socks5") {
|
||||
baseDialer := &net.Dialer{
|
||||
Timeout: 30 * time.Second,
|
||||
KeepAlive: 30 * time.Second,
|
||||
Timeout: s.clientTimeout,
|
||||
KeepAlive: s.clientTimeout,
|
||||
}
|
||||
if socks5 != "" {
|
||||
dialSocksProxy, err := proxy.SOCKS5("tcp", socks5, nil, baseDialer)
|
||||
socksHostPort := strings.ReplaceAll(proxyAddr, "socks5://", "")
|
||||
dialSocksProxy, err := proxy.SOCKS5("tcp", socksHostPort, nil, baseDialer)
|
||||
if err != nil {
|
||||
return errors.New("Error creating SOCKS5 proxy")
|
||||
return errors.New("error creating socks5 proxy :" + err.Error())
|
||||
}
|
||||
if contextDialer, ok := dialSocksProxy.(proxy.ContextDialer); ok {
|
||||
dialContext := contextDialer.DialContext
|
||||
|
|
@ -136,17 +145,12 @@ func (s *Scraper) SetSocks5Proxy(socks5 string) error {
|
|||
},
|
||||
}
|
||||
} else {
|
||||
return errors.New("Failed type assertion to DialContext")
|
||||
}
|
||||
} else {
|
||||
s.client = &http.Client{
|
||||
Transport: &http.Transport{
|
||||
DialContext: (baseDialer).DialContext,
|
||||
},
|
||||
}
|
||||
return errors.New("failed type assertion to DialContext")
|
||||
}
|
||||
return nil
|
||||
}
|
||||
return errors.New("only support http(s) or socks5 protocol")
|
||||
}
|
||||
|
||||
// SetProxy wrapper for default Scraper
|
||||
func SetProxy(proxy string) error {
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue