111 lines
2.3 KiB
Go
111 lines
2.3 KiB
Go
package twitterscraper
|
|
|
|
import (
|
|
"crypto/tls"
|
|
"errors"
|
|
"net"
|
|
"net/http"
|
|
"net/url"
|
|
"strings"
|
|
"time"
|
|
)
|
|
|
|
// Scraper object
|
|
type Scraper struct {
|
|
client *http.Client
|
|
guestToken string
|
|
includeReplies bool
|
|
liveSearch bool
|
|
resultFilter string
|
|
}
|
|
|
|
var defaultScraper *Scraper
|
|
|
|
// New creates a Scraper object
|
|
func New() *Scraper {
|
|
return &Scraper{
|
|
client: &http.Client{Timeout: 10 * time.Second},
|
|
}
|
|
}
|
|
|
|
// SetSearchLive enable/disable realtime search
|
|
func (s *Scraper) SetSearchLive(b bool) *Scraper {
|
|
s.liveSearch = b
|
|
return s
|
|
}
|
|
|
|
// SetSearchLive wrapper for default SetSearchLive
|
|
func SetSearchLive(b bool) *Scraper {
|
|
return defaultScraper.SetSearchLive(b)
|
|
}
|
|
|
|
// SetSearchPhotos filter search for photos only
|
|
func (s *Scraper) SetSearchPhotos(srctype bool) *Scraper {
|
|
if srctype {
|
|
s.resultFilter = "image"
|
|
} else {
|
|
s.resultFilter = ""
|
|
}
|
|
return s
|
|
}
|
|
|
|
// SetSearchPhotos wrapper for default SetSearchPhotos
|
|
func SetSearchPhotos(srctype bool) *Scraper {
|
|
return defaultScraper.SetSearchPhotos(srctype)
|
|
}
|
|
|
|
// SetSearchVideos filter search for videos only
|
|
func (s *Scraper) SetSearchVideos(srctype bool) *Scraper {
|
|
if srctype {
|
|
s.resultFilter = "video"
|
|
} else {
|
|
s.resultFilter = ""
|
|
}
|
|
return s
|
|
}
|
|
|
|
// SetSearchVideos wrapper for default SetSearchVideos
|
|
func SetSearchVideos(srctype bool) *Scraper {
|
|
return defaultScraper.SetSearchVideos(srctype)
|
|
}
|
|
|
|
// WithReplies enable/disable load timeline with tweet replies
|
|
func (s *Scraper) WithReplies(b bool) *Scraper {
|
|
s.includeReplies = b
|
|
return s
|
|
}
|
|
|
|
// WithReplies wrapper for default Scraper
|
|
func WithReplies(b bool) *Scraper {
|
|
return defaultScraper.WithReplies(b)
|
|
}
|
|
|
|
// SetProxy set http proxy in the format `http://HOST:PORT`
|
|
func (s *Scraper) SetProxy(proxy string) error {
|
|
if !strings.HasPrefix(proxy, "http") {
|
|
return errors.New("only support http(s) protocol")
|
|
}
|
|
urlproxy, err := url.Parse(proxy)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
s.client = &http.Client{
|
|
Transport: &http.Transport{
|
|
Proxy: http.ProxyURL(urlproxy),
|
|
TLSNextProto: make(map[string]func(authority string, c *tls.Conn) http.RoundTripper),
|
|
DialContext: (&net.Dialer{
|
|
Timeout: 10 * time.Second,
|
|
}).DialContext,
|
|
},
|
|
}
|
|
return nil
|
|
}
|
|
|
|
// SetProxy wrapper for default Scraper
|
|
func SetProxy(proxy string) error {
|
|
return defaultScraper.SetProxy(proxy)
|
|
}
|
|
|
|
func init() {
|
|
defaultScraper = New()
|
|
}
|