Add Trends

README.md

@@ -51,6 +51,25 @@ func main() {
 }
 ```
 
+### Get trends
+
+```golang
+package main
+
+import (
+    "fmt"
+    twitterscraper "github.com/n0madic/twitter-scraper"
+)
+
+func main() {
+    trends, err := twitterscraper.GetTrends()
+    if err != nil {
+        panic(err)
+    }
+    fmt.Println(trends)
+}
+```
+
 ## Installation
 
 ```shell

trends.go

@@ -0,0 +1,35 @@
+package twitterscraper
+
+import (
+	"net/http"
+
+	"github.com/PuerkitoBio/goquery"
+)
+
+const trendsURL = "https://twitter.com/i/trends"
+
+// GetTrends return list of trends
+func GetTrends() ([]string, error) {
+	req, err := http.NewRequest("GET", trendsURL, nil)
+	if err != nil {
+		return nil, err
+	}
+
+	htm, err := getHTMLFromJSON(req, "module_html")
+	if err != nil {
+		return nil, err
+	}
+
+	doc, err := goquery.NewDocumentFromReader(htm)
+	if err != nil {
+		return nil, err
+	}
+
+	var trends []string
+	doc.Find("li").Each(func(i int, s *goquery.Selection) {
+		if trend, ok := s.Attr("data-trend-name"); ok {
+			trends = append(trends, trend)
+		}
+	})
+	return trends, nil
+}

trends_test.go

@@ -0,0 +1,16 @@
+package twitterscraper
+
+import (
+	"testing"
+)
+
+func TestGetTrends(t *testing.T) {
+	trends, err := GetTrends()
+	if err != nil {
+		t.Error(err)
+	}
+
+	if len(trends) != 10 {
+		t.Error("Expected 10 trends")
+	}
+}

tweets.go

@@ -1,7 +1,6 @@
 package twitterscraper
 
 import (
-	"encoding/json"
 	"fmt"
 	"net/http"
 	"regexp"
@@ -90,20 +89,12 @@ func FetchTweets(user string, last string) ([]*Tweet, error) {
 	}
 	req.URL.RawQuery = q.Encode()
 
-	resp, err := http.DefaultClient.Do(req)
+	htm, err := getHTMLFromJSON(req, "items_html")
-	if err != nil {
-		return nil, err
-	}
-	defer resp.Body.Close()
-
-	if resp.StatusCode == http.StatusOK {
-		ajaxJSON := make(map[string]interface{})
-		err = json.NewDecoder(resp.Body).Decode(&ajaxJSON)
 	if err != nil {
 		return nil, err
 	}
 
-		doc, err := goquery.NewDocumentFromReader(strings.NewReader(ajaxJSON["items_html"].(string)))
+	doc, err := goquery.NewDocumentFromReader(htm)
 	if err != nil {
 		return nil, err
 	}
@@ -160,11 +151,6 @@ func FetchTweets(user string, last string) ([]*Tweet, error) {
 			tweets = append(tweets, &tweet)
 		}
 	})
-	} else if resp.StatusCode == http.StatusNotFound {
-		return nil, fmt.Errorf("user %s not found", user)
-	} else {
-		return nil, fmt.Errorf("response status: %s", resp.Status)
-	}
 
 	return tweets, nil
 }

util.go

@@ -0,0 +1,33 @@
+package twitterscraper
+
+import (
+	"encoding/json"
+	"fmt"
+	"net/http"
+	"strings"
+)
+
+func getHTMLFromJSON(req *http.Request, field string) (*strings.Reader, error) {
+	resp, err := http.DefaultClient.Do(req)
+	if err != nil {
+		return nil, err
+	}
+	defer resp.Body.Close()
+
+	if resp.StatusCode != http.StatusOK {
+		return nil, fmt.Errorf("response status: %s", resp.Status)
+	}
+
+	ajaxJSON := make(map[string]interface{})
+	err = json.NewDecoder(resp.Body).Decode(&ajaxJSON)
+	if err != nil {
+		return nil, err
+	}
+
+	htm, ok := ajaxJSON[field].(string)
+	if !ok {
+		return nil, fmt.Errorf("filed not found in JSON")
+	}
+
+	return strings.NewReader(htm), nil
+}