parent
16e8b34638
commit
e52022e31a
10 changed files with 224 additions and 34 deletions
15
scraper.go
15
scraper.go
|
|
@ -23,6 +23,9 @@ type Scraper struct {
|
|||
guestCreatedAt time.Time
|
||||
includeReplies bool
|
||||
isLogged bool
|
||||
oAuthToken string
|
||||
oAuthSecret string
|
||||
proxy string
|
||||
searchMode SearchMode
|
||||
wg sync.WaitGroup
|
||||
}
|
||||
|
|
@ -96,6 +99,16 @@ func (s *Scraper) WithClientTimeout(timeout time.Duration) *Scraper {
|
|||
// set http proxy in the format `http://HOST:PORT`
|
||||
// set socket proxy in the format `socks5://HOST:PORT`
|
||||
func (s *Scraper) SetProxy(proxyAddr string) error {
|
||||
if proxyAddr == "" {
|
||||
s.client.Transport = &http.Transport{
|
||||
TLSNextProto: make(map[string]func(authority string, c *tls.Conn) http.RoundTripper),
|
||||
DialContext: (&net.Dialer{
|
||||
Timeout: s.client.Timeout,
|
||||
}).DialContext,
|
||||
}
|
||||
s.proxy = ""
|
||||
return nil
|
||||
}
|
||||
if strings.HasPrefix(proxyAddr, "http") {
|
||||
urlproxy, err := url.Parse(proxyAddr)
|
||||
if err != nil {
|
||||
|
|
@ -108,6 +121,7 @@ func (s *Scraper) SetProxy(proxyAddr string) error {
|
|||
Timeout: s.client.Timeout,
|
||||
}).DialContext,
|
||||
}
|
||||
s.proxy = proxyAddr
|
||||
return nil
|
||||
}
|
||||
if strings.HasPrefix(proxyAddr, "socks5") {
|
||||
|
|
@ -128,6 +142,7 @@ func (s *Scraper) SetProxy(proxyAddr string) error {
|
|||
} else {
|
||||
return errors.New("failed type assertion to DialContext")
|
||||
}
|
||||
s.proxy = proxyAddr
|
||||
return nil
|
||||
}
|
||||
return errors.New("only support http(s) or socks5 protocol")
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue