55 lines
1.1 KiB
Go
55 lines
1.1 KiB
Go
package twitterscraper
|
|
|
|
import (
|
|
"fmt"
|
|
"net"
|
|
"net/http"
|
|
"strings"
|
|
"time"
|
|
|
|
"github.com/PuerkitoBio/goquery"
|
|
)
|
|
|
|
const trendsURL = "https://mobile.twitter.com/trends"
|
|
|
|
// GetTrends return list of trends.
|
|
func GetTrends() ([]string, error) {
|
|
client := http.DefaultClient
|
|
if HTTPProxy != nil {
|
|
client = &http.Client{
|
|
Transport: &http.Transport{
|
|
Proxy: http.ProxyURL(HTTPProxy),
|
|
DialContext: (&net.Dialer{
|
|
Timeout: 10 * time.Second,
|
|
}).DialContext,
|
|
},
|
|
}
|
|
}
|
|
|
|
req, err := http.NewRequest("GET", trendsURL, nil)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
req.Header.Set("Accept-Language", "en-US")
|
|
|
|
resp, err := client.Do(req)
|
|
if resp == nil {
|
|
return nil, err
|
|
}
|
|
defer resp.Body.Close()
|
|
|
|
if resp.StatusCode != http.StatusOK {
|
|
return nil, fmt.Errorf("response status: %s", resp.Status)
|
|
}
|
|
|
|
doc, err := goquery.NewDocumentFromReader(resp.Body)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
|
|
var trends []string
|
|
doc.Find("li.topic").Each(func(i int, s *goquery.Selection) {
|
|
trends = append(trends, strings.TrimSpace(s.Text()))
|
|
})
|
|
return trends, nil
|
|
}
|