lain/twitter-scrapper
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity

Fix some golangci-lint warnings

Browse source 
Minor changes
This commit is contained in:
Alexander Sheiko 2020-05-14 21:52:55 +03:00
parent d0f72d53e4
commit a34a14d25a
4 changed files with 20 additions and 17 deletions
Download patch file Download diff file Expand all files Collapse all files

16
profile.go
View file

@ -10,7 +10,7 @@ import (
"github.com/PuerkitoBio/goquery" "github.com/PuerkitoBio/goquery"
) )
// Profile of twitter user // Profile of twitter user.
type Profile struct { type Profile struct {
Avatar string Avatar string
Biography string Biography string
@ -27,26 +27,28 @@ type Profile struct {
Website string Website string
} }
// GetProfile return parsed user profile // GetProfile return parsed user profile.
func GetProfile(username string) (Profile, error) { func GetProfile(username string) (Profile, error) {
url := "https://twitter.com/" + username url := "https://twitter.com/" + username
req, err := http.NewRequest("GET", url, nil) req, err := http.NewRequest("GET", url, nil)
if err != nil { if err != nil {
return Profile{}, err return Profile{}, err
} }
req.Header.Set("Accept-Language", "en-US") req.Header.Set("Accept-Language", "en-US")
res, err := http.DefaultClient.Do(req) resp, err := http.DefaultClient.Do(req)
if resp == nil {
if res == nil {
return Profile{}, err return Profile{}, err
} }
defer resp.Body.Close()
doc, err := goquery.NewDocumentFromReader(res.Body) if resp.StatusCode != http.StatusOK {
return Profile{}, fmt.Errorf("response status: %s", resp.Status)
}
doc, err := goquery.NewDocumentFromReader(resp.Body)
if err != nil { if err != nil {
return Profile{}, err return Profile{}, err
} }

2
trends.go
View file

@ -8,7 +8,7 @@ import (
const trendsURL = "https://twitter.com/i/trends" const trendsURL = "https://twitter.com/i/trends"
// GetTrends return list of trends // GetTrends return list of trends.
func GetTrends() ([]string, error) { func GetTrends() ([]string, error) {
req, err := http.NewRequest("GET", trendsURL, nil) req, err := http.NewRequest("GET", trendsURL, nil)
if err != nil { if err != nil {

17
tweets.go
View file

@ -13,13 +13,13 @@ import (
const ajaxURL = "https://twitter.com/i/profiles/show/%s/timeline/tweets" const ajaxURL = "https://twitter.com/i/profiles/show/%s/timeline/tweets"
// Video type // Video type.
type Video struct { type Video struct {
ID string ID string
Preview string Preview string
} }
// Tweet type // Tweet type.
type Tweet struct { type Tweet struct {
Hashtags []string Hashtags []string
HTML string HTML string
@ -38,13 +38,13 @@ type Tweet struct {
Videos []Video Videos []Video
} }
// Result of scrapping // Result of scrapping.
type Result struct { type Result struct {
Tweet Tweet
Error error Error error
} }
// GetTweets returns channel with tweets for a given user // GetTweets returns channel with tweets for a given user.
func GetTweets(user string, pages int) <-chan *Result { func GetTweets(user string, pages int) <-chan *Result {
channel := make(chan *Result) channel := make(chan *Result)
go func(user string) { go func(user string) {
@ -66,7 +66,7 @@ func GetTweets(user string, pages int) <-chan *Result {
return channel return channel
} }
// FetchTweets gets tweets for a given user, via the Twitter frontend API // FetchTweets gets tweets for a given user, via the Twitter frontend API.
func FetchTweets(user string, last string) ([]*Tweet, error) { func FetchTweets(user string, last string) ([]*Tweet, error) {
var tweets []*Tweet var tweets []*Tweet
@ -119,13 +119,14 @@ func FetchTweets(user string, last string) ([]*Tweet, error) {
}) })
s.Find(".ProfileTweet-actionCount").Each(func(i int, c *goquery.Selection) { s.Find(".ProfileTweet-actionCount").Each(func(i int, c *goquery.Selection) {
txt := strings.TrimSpace(c.Text()) txt := strings.TrimSpace(c.Text())
if strings.HasSuffix(txt, "likes") { switch {
case strings.HasSuffix(txt, "likes"):
l := strings.Split(txt, " ") l := strings.Split(txt, " ")
tweet.Likes, _ = strconv.Atoi(l[0]) tweet.Likes, _ = strconv.Atoi(l[0])
} else if strings.HasSuffix(txt, "replies") { case strings.HasSuffix(txt, "replies"):
l := strings.Split(txt, " ") l := strings.Split(txt, " ")
tweet.Replies, _ = strconv.Atoi(l[0]) tweet.Replies, _ = strconv.Atoi(l[0])
} else if strings.HasSuffix(txt, "retweets") { case strings.HasSuffix(txt, "retweets"):
l := strings.Split(txt, " ") l := strings.Split(txt, " ")
tweet.Retweets, _ = strconv.Atoi(l[0]) tweet.Retweets, _ = strconv.Atoi(l[0])
} }