lain/twitter-scrapper
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity

Remove clientTimeout property

Browse source
This commit is contained in:
Alexander Sheiko 2022-01-11 14:47:17 +02:00
parent ed0186bad8
commit e6ddf32a93
1 changed files with 5 additions and 7 deletions
Download patch file Download diff file Expand all files Collapse all files

12
scraper.go
View file

@ -16,7 +16,6 @@ import (
// Scraper object // Scraper object
type Scraper struct { type Scraper struct {
client *http.Client client *http.Client
clientTimeout time.Duration
delay int64 delay int64
guestToken string guestToken string
guestCreatedAt time.Time guestCreatedAt time.Time
@ -52,8 +51,7 @@ var defaultScraper *Scraper
// New creates a Scraper object // New creates a Scraper object
func New() *Scraper { func New() *Scraper {
return &Scraper{ return &Scraper{
client: &http.Client{Timeout: DefaultClientTimeout}, client: &http.Client{Timeout: DefaultClientTimeout},
clientTimeout: DefaultClientTimeout,
} }
} }
@ -109,7 +107,7 @@ func (s *Scraper) WithXCsrfToken(xcsrfToken string) *Scraper {
// client timeout // client timeout
func (s *Scraper) WithClientTimeout(timeout time.Duration) *Scraper { func (s *Scraper) WithClientTimeout(timeout time.Duration) *Scraper {
s.clientTimeout = timeout s.client.Timeout = timeout
return s return s
} }
@ -127,7 +125,7 @@ func (s *Scraper) SetProxy(proxyAddr string) error {
Proxy: http.ProxyURL(urlproxy), Proxy: http.ProxyURL(urlproxy),
TLSNextProto: make(map[string]func(authority string, c *tls.Conn) http.RoundTripper), TLSNextProto: make(map[string]func(authority string, c *tls.Conn) http.RoundTripper),
DialContext: (&net.Dialer{ DialContext: (&net.Dialer{
Timeout: s.clientTimeout, Timeout: s.client.Timeout,
}).DialContext, }).DialContext,
}, },
} }
@ -135,8 +133,8 @@ func (s *Scraper) SetProxy(proxyAddr string) error {
} }
if strings.HasPrefix(proxyAddr, "socks5") { if strings.HasPrefix(proxyAddr, "socks5") {
baseDialer := &net.Dialer{ baseDialer := &net.Dialer{
Timeout: s.clientTimeout, Timeout: s.client.Timeout,
KeepAlive: s.clientTimeout, KeepAlive: s.client.Timeout,
} }
socksHostPort := strings.ReplaceAll(proxyAddr, "socks5://", "") socksHostPort := strings.ReplaceAll(proxyAddr, "socks5://", "")
dialSocksProxy, err := proxy.SOCKS5("tcp", socksHostPort, nil, baseDialer) dialSocksProxy, err := proxy.SOCKS5("tcp", socksHostPort, nil, baseDialer)