lain/twitter-scrapper
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity

Support https proxy

Browse source
This commit is contained in:
Alexander Sheiko 2020-12-13 00:04:34 +02:00
parent c299d7f479
commit 6414a834ea
1 changed files with 5 additions and 3 deletions
Download patch file Download diff file Expand all files Collapse all files

6
scraper.go
View file

@ -1,6 +1,7 @@
package twitterscraper package twitterscraper
import ( import (
"crypto/tls"
"errors" "errors"
"net" "net"
"net/http" "net/http"
@ -38,8 +39,8 @@ func WithReplies(b bool) *Scraper {
// SetProxy set http proxy in the format `http://HOST:PORT` // SetProxy set http proxy in the format `http://HOST:PORT`
func (s *Scraper) SetProxy(proxy string) error { func (s *Scraper) SetProxy(proxy string) error {
if !strings.HasPrefix(proxy, "http://") { if !strings.HasPrefix(proxy, "http") {
return errors.New("only support http protocol") return errors.New("only support http(s) protocol")
} }
urlproxy, err := url.Parse(proxy) urlproxy, err := url.Parse(proxy)
if err != nil { if err != nil {
@ -48,6 +49,7 @@ func (s *Scraper) SetProxy(proxy string) error {
s.client = &http.Client{ s.client = &http.Client{
Transport: &http.Transport{ Transport: &http.Transport{
Proxy: http.ProxyURL(urlproxy), Proxy: http.ProxyURL(urlproxy),
TLSNextProto: make(map[string]func(authority string, c *tls.Conn) http.RoundTripper),
DialContext: (&net.Dialer{ DialContext: (&net.Dialer{
Timeout: 10 * time.Second, Timeout: 10 * time.Second,
}).DialContext, }).DialContext,