Total refactoring

Used guest frontend API
BREAKING CHANGE: remove tweet.HTML property
Loading more information
Minor fixes and changes

profile.go

@@ -2,13 +2,7 @@ package twitterscraper
 
 import (
 	"fmt"
-	"net"
-	"net/http"
-	"strconv"
-	"strings"
 	"time"
-
-	"github.com/PuerkitoBio/goquery"
 )
 
 // Profile of twitter user.
@@ -19,12 +13,15 @@ type Profile struct {
 	Birthday       string
 	FollowersCount int
 	FollowingCount int
+	FriendsCount   int
 	IsPrivate      bool
 	IsVerified     bool
 	Joined         *time.Time
 	LikesCount     int
+	ListedCount    int
 	Location       string
 	Name           string
+	PinnedTweetIDs []string
 	TweetsCount    int
 	URL            string
 	UserID         string
@@ -34,66 +31,61 @@ type Profile struct {
 
 // GetProfile return parsed user profile.
 func GetProfile(username string) (Profile, error) {
-	url := "https://mobile.twitter.com/" + username
-
-	client := http.DefaultClient
-	if HTTPProxy != nil {
-		client = &http.Client{
-			Transport: &http.Transport{
-				Proxy: http.ProxyURL(HTTPProxy),
-				DialContext: (&net.Dialer{
-					Timeout: 10 * time.Second,
-				}).DialContext,
-			},
-		}
-	}
-
-	req, err := http.NewRequest("GET", url, nil)
-	if err != nil {
-		return Profile{}, err
-	}
-	req.Header.Set("Accept-Language", "en-US")
-
-	resp, err := client.Do(req)
-	if resp == nil {
-		return Profile{}, err
-	}
-	defer resp.Body.Close()
-
-	if resp.StatusCode != http.StatusOK {
-		return Profile{}, fmt.Errorf("response status: %s", resp.Status)
-	}
-
-	doc, err := goquery.NewDocumentFromReader(resp.Body)
+	userID, err := GetUserIDByScreenName(username)
 	if err != nil {
 		return Profile{}, err
 	}
 
-	// parse join date text
-	screenName := doc.Find(".screen-name").First().Text()
+	req, err := newRequest("GET", "https://twitter.com/i/api/2/timeline/profile/"+userID+".json")
+	if err != nil {
+		return Profile{}, err
+	}
 
-	// check is username valid
-	if screenName == "" {
+	q := req.URL.Query()
+	q.Add("count", "20")
+	q.Add("userId", userID)
+	req.URL.RawQuery = q.Encode()
+
+	var timeline timeline
+	err = requestAPI(req, &timeline)
+	if err != nil {
+		return Profile{}, err
+	}
+
+	user, found := timeline.GlobalObjects.Users[userID]
+	if !found {
 		return Profile{}, fmt.Errorf("either @%s does not exist or is private", username)
 	}
 
-	return Profile{
-		Avatar:         doc.Find("td.avatar > img").First().AttrOr("src", ""),
-		Biography:      strings.TrimSpace(doc.Find(".bio").First().Text()),
-		FollowersCount: parseCount(doc.Find("table.profile-stats > tbody > tr > td:nth-child(3) > a > div.statnum").First().Text()),
-		FollowingCount: parseCount(doc.Find("table.profile-stats > tbody > tr > td:nth-child(2) > a > div.statnum").First().Text()),
-		IsPrivate:      strings.Contains(doc.Find("div.fullname > a.badge > img").First().AttrOr("src", ""), "protected"),
-		IsVerified:     strings.Contains(doc.Find("div.fullname > a.badge > img").First().AttrOr("src", ""), "verified"),
-		Location:       strings.TrimSpace(doc.Find(".location").First().Text()),
-		Name:           strings.TrimSpace(doc.Find(".fullname").First().Text()),
-		TweetsCount:    parseCount(doc.Find("table.profile-stats > tbody > tr > td:nth-child(1) > div.statnum").First().Text()),
-		URL:            "https://twitter.com/" + screenName,
-		Username:       screenName,
-		Website:        strings.TrimSpace(doc.Find("div.url > div > a").First().AttrOr("data-url", "")),
-	}, nil
-}
+	profile := Profile{
+		Avatar:         user.ProfileImageURLHTTPS,
+		Banner:         user.ProfileBannerURL,
+		Biography:      user.Description,
+		FollowersCount: user.FollowersCount,
+		FollowingCount: user.FavouritesCount,
+		FriendsCount:   user.FriendsCount,
+		IsPrivate:      user.Protected,
+		IsVerified:     user.Verified,
+		LikesCount:     user.FavouritesCount,
+		ListedCount:    user.ListedCount,
+		Location:       user.Location,
+		Name:           user.Name,
+		PinnedTweetIDs: user.PinnedTweetIdsStr,
+		TweetsCount:    user.StatusesCount,
+		URL:            "https://twitter.com/" + user.ScreenName,
+		UserID:         user.IDStr,
+		Username:       user.ScreenName,
+	}
 
-func parseCount(str string) (i int) {
-	i, _ = strconv.Atoi(strings.Replace(str, ",", "", -1))
-	return
+	tm, err := time.Parse(time.RubyDate, user.CreatedAt)
+	if err == nil {
+		tm = tm.UTC()
+		profile.Joined = &tm
+	}
+
+	if len(user.Entities.URL.Urls) > 0 {
+		profile.Website = user.Entities.URL.Urls[0].ExpandedURL
+	}
+
+	return profile, nil
 }