lain/twitter-scrapper
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity

Fix trends

Browse source
This commit is contained in:
Alexander Sheiko 2020-08-13 17:39:32 +03:00
parent 2f2b073de7
commit 7f033acb89
1 changed files with 17 additions and 9 deletions
Download patch file Download diff file Expand all files Collapse all files

26
trends.go
View file

@ -1,33 +1,41 @@
package twitterscraper package twitterscraper
import ( import (
"fmt"
"net/http"
"strings"
"github.com/PuerkitoBio/goquery" "github.com/PuerkitoBio/goquery"
) )
const trendsURL = "https://twitter.com/i/trends" const trendsURL = "https://mobile.twitter.com/trends"
// GetTrends return list of trends. // GetTrends return list of trends.
func GetTrends() ([]string, error) { func GetTrends() ([]string, error) {
req, err := newRequest(trendsURL) req, err := http.NewRequest("GET", trendsURL, nil)
if err != nil { if err != nil {
return nil, err return nil, err
} }
req.Header.Set("Accept-Language", "en-US")
htm, err := getHTMLFromJSON(req, "module_html") resp, err := http.DefaultClient.Do(req)
if err != nil { if resp == nil {
return nil, err return nil, err
} }
defer resp.Body.Close()
doc, err := goquery.NewDocumentFromReader(htm) if resp.StatusCode != http.StatusOK {
return nil, fmt.Errorf("response status: %s", resp.Status)
}
doc, err := goquery.NewDocumentFromReader(resp.Body)
if err != nil { if err != nil {
return nil, err return nil, err
} }
var trends []string var trends []string
doc.Find("li").Each(func(i int, s *goquery.Selection) { doc.Find("li.topic").Each(func(i int, s *goquery.Selection) {
if trend, ok := s.Attr("data-trend-name"); ok { trends = append(trends, strings.TrimSpace(s.Text()))
trends = append(trends, trend)
}
}) })
return trends, nil return trends, nil
} }