twitter-scrapper/trends.go

package twitterscraper

import (
	"fmt"
	"net"
	"net/http"
	"strings"
	"time"

	"github.com/PuerkitoBio/goquery"
)

const trendsURL = "https://mobile.twitter.com/trends"

// GetTrends return list of trends.
func GetTrends() ([]string, error) {
	client := http.DefaultClient
	if HTTPProxy != nil {
		client = &http.Client{
			Transport: &http.Transport{
				Proxy: http.ProxyURL(HTTPProxy),
				DialContext: (&net.Dialer{
					Timeout: 10 * time.Second,
				}).DialContext,
			},
		}
	}

	req, err := http.NewRequest("GET", trendsURL, nil)
	if err != nil {
		return nil, err
	}
	req.Header.Set("Accept-Language", "en-US")

	resp, err := client.Do(req)
	if resp == nil {
		return nil, err
	}
	defer resp.Body.Close()

	if resp.StatusCode != http.StatusOK {
		return nil, fmt.Errorf("response status: %s", resp.Status)
	}

	doc, err := goquery.NewDocumentFromReader(resp.Body)
	if err != nil {
		return nil, err
	}

	var trends []string
	doc.Find("li.topic").Each(func(i int, s *goquery.Selection) {
		trends = append(trends, strings.TrimSpace(s.Text()))
	})
	return trends, nil
}