lain/twitter-scrapper
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity

create readTweetsFromHtml func for recycle code

Browse source
This commit is contained in:
xisco 2020-05-13 17:35:44 +02:00
parent 7c9eb133e8
commit 37bb62a82a
2 changed files with 13 additions and 1 deletions
Download patch file Download diff file Expand all files Collapse all files

1
search.go Normal file
View file

@ -0,0 +1 @@
package twitterscraper

13
tweets.go
View file

@ -68,7 +68,6 @@ func GetTweets(user string, pages int) <-chan *Result {
// FetchTweets gets tweets for a given user, via the Twitter frontend API // FetchTweets gets tweets for a given user, via the Twitter frontend API
func FetchTweets(user string, last string) ([]*Tweet, error) { func FetchTweets(user string, last string) ([]*Tweet, error) {
var tweets []*Tweet
req, err := http.NewRequest("GET", fmt.Sprintf(ajaxURL, user), nil) req, err := http.NewRequest("GET", fmt.Sprintf(ajaxURL, user), nil)
if err != nil { if err != nil {
@ -96,6 +95,17 @@ func FetchTweets(user string, last string) ([]*Tweet, error) {
return nil, err return nil, err
} }
tweets, err := readTweetsFromHTML(htm)
if err != nil {
return nil, err
}
return tweets, nil
}
func readTweetsFromHTML (htm *strings.Reader) ([]*Tweet, error) {
var tweets []*Tweet
doc, err := goquery.NewDocumentFromReader(htm) doc, err := goquery.NewDocumentFromReader(htm)
if err != nil { if err != nil {
return nil, err return nil, err
@ -108,6 +118,7 @@ func FetchTweets(user string, last string) ([]*Tweet, error) {
tweet.Timestamp, _ = strconv.ParseInt(timeStr, 10, 64) tweet.Timestamp, _ = strconv.ParseInt(timeStr, 10, 64)
tweet.TimeParsed = time.Unix(tweet.Timestamp, 0) tweet.TimeParsed = time.Unix(tweet.Timestamp, 0)
tweet.ID = s.AttrOr("data-item-id", "") tweet.ID = s.AttrOr("data-item-id", "")
user, _ := s.Find(".tweet").Attr("data-screen-name")
tweet.PermanentURL = fmt.Sprintf("https://twitter.com/%s/status/%s", user, tweet.ID) tweet.PermanentURL = fmt.Sprintf("https://twitter.com/%s/status/%s", user, tweet.ID)
tweet.Text = s.Find(".tweet-text").Text() tweet.Text = s.Find(".tweet-text").Text()
tweet.HTML, _ = s.Find(".tweet-text").Html() tweet.HTML, _ = s.Find(".tweet-text").Html()