From f257b131ff34f9b7514c39631b02483d396ec25d Mon Sep 17 00:00:00 2001 From: Valentine Date: Tue, 1 Oct 2024 01:30:08 +0300 Subject: [PATCH 1/3] refactor auth api --- api.go | 89 ++++++++++++++++++++++++++++++++++++------------------ auth.go | 2 +- scraper.go | 11 +++++++ 3 files changed, 71 insertions(+), 31 deletions(-) diff --git a/api.go b/api.go index fb6dfcf..38360dc 100644 --- a/api.go +++ b/api.go @@ -14,36 +14,11 @@ const bearerToken string = "AAAAAAAAAAAAAAAAAAAAAPYXBAAAAAAACLXUNDekMxqa8h%2F40K func (s *Scraper) RequestAPI(req *http.Request, target interface{}) error { s.wg.Wait() if s.delay > 0 { - defer func() { - s.wg.Add(1) - go func() { - time.Sleep(time.Second * time.Duration(s.delay)) - s.wg.Done() - }() - }() + defer s.delayRequest() } - if !s.isLogged { - if !s.IsGuestToken() || s.guestCreatedAt.Before(time.Now().Add(-time.Hour*3)) { - err := s.GetGuestToken() - if err != nil { - return err - } - } - req.Header.Set("X-Guest-Token", s.guestToken) - } - - if s.oAuthToken != "" && s.oAuthSecret != "" { - req.Header.Set("Authorization", s.sign(req.Method, req.URL)) - } else { - req.Header.Set("Authorization", "Bearer "+s.bearerToken) - } - - for _, cookie := range s.client.Jar.Cookies(req.URL) { - if cookie.Name == "ct0" { - req.Header.Set("X-CSRF-Token", cookie.Value) - break - } + if err := s.prepareRequest(req); err != nil { + return err } resp, err := s.client.Do(req) @@ -52,13 +27,66 @@ func (s *Scraper) RequestAPI(req *http.Request, target interface{}) error { } defer resp.Body.Close() + return s.handleResponse(resp, target) +} + +func (s *Scraper) delayRequest() { + s.wg.Add(1) + go func() { + time.Sleep(time.Second * time.Duration(s.delay)) + s.wg.Done() + }() +} + +func (s *Scraper) prepareRequest(req *http.Request) error { + req.Header.Set("User-Agent", s.userAgent) + + if !s.isLogged { + if err := s.setGuestToken(req); err != nil { + return err + } + } + + s.setAuthorizationHeader(req) + s.setCSRFToken(req) + + return nil +} + +func (s *Scraper) setGuestToken(req *http.Request) error { + if !s.IsGuestToken() || s.guestCreatedAt.Before(time.Now().Add(-time.Hour*3)) { + if err := s.GetGuestToken(); err != nil { + return err + } + } + req.Header.Set("X-Guest-Token", s.guestToken) + return nil +} + +func (s *Scraper) setAuthorizationHeader(req *http.Request) { + if s.oAuthToken != "" && s.oAuthSecret != "" { + req.Header.Set("Authorization", s.sign(req.Method, req.URL)) + } else { + req.Header.Set("Authorization", "Bearer "+s.bearerToken) + } +} + +func (s *Scraper) setCSRFToken(req *http.Request) { + for _, cookie := range s.client.Jar.Cookies(req.URL) { + if cookie.Name == "ct0" { + req.Header.Set("X-CSRF-Token", cookie.Value) + break + } + } +} + +func (s *Scraper) handleResponse(resp *http.Response, target interface{}) error { content, err := io.ReadAll(resp.Body) if err != nil { return err } - statusOK := resp.StatusCode >= 200 && resp.StatusCode < 300 - if !statusOK { + if resp.StatusCode != http.StatusOK { return fmt.Errorf("response status %s: %s", resp.Status, content) } @@ -69,6 +97,7 @@ func (s *Scraper) RequestAPI(req *http.Request, target interface{}) error { if target == nil { return nil } + return json.Unmarshal(content, target) } diff --git a/auth.go b/auth.go index a7a7d8b..7478349 100644 --- a/auth.go +++ b/auth.go @@ -84,7 +84,7 @@ func (s *Scraper) getFlow(data map[string]interface{}) (*flow, error) { headers := http.Header{ "Authorization": []string{"Bearer " + s.bearerToken}, "Content-Type": []string{"application/json"}, - "User-Agent": []string{"TwitterAndroid/99"}, + "User-Agent": []string{s.userAgent}, "X-Guest-Token": []string{s.guestToken}, "X-Twitter-Auth-Type": []string{"OAuth2Client"}, "X-Twitter-Active-User": []string{"yes"}, diff --git a/scraper.go b/scraper.go index 98326c6..b60e921 100644 --- a/scraper.go +++ b/scraper.go @@ -27,6 +27,7 @@ type Scraper struct { oAuthToken string oAuthSecret string proxy string + userAgent string searchMode SearchMode wg sync.WaitGroup } @@ -49,12 +50,14 @@ const ( // default http client timeout const DefaultClientTimeout = 10 * time.Second +const DefaultUserAgent = "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/129.0.0.0 Safari/537.36" // New creates a Scraper object func New() *Scraper { jar, _ := cookiejar.New(nil) return &Scraper{ bearerToken: bearerToken, + userAgent: DefaultUserAgent, client: &http.Client{ Jar: jar, Timeout: DefaultClientTimeout, @@ -169,3 +172,11 @@ func (s *Scraper) SetProxy(proxyAddr string) error { } return errors.New("only support http(s) or socks5 protocol") } + +func (s *Scraper) SetUserAgent(userAgent string) { + s.userAgent = userAgent +} + +func (s *Scraper) GetUserAgent() string { + return s.userAgent +} From 4152dfe0867fb1f64de466645a4fac192fe60c35 Mon Sep 17 00:00:00 2001 From: Valentine Date: Tue, 1 Oct 2024 01:33:38 +0300 Subject: [PATCH 2/3] add random delay to auth flow --- auth.go | 21 +++++++++++++++++++++ 1 file changed, 21 insertions(+) diff --git a/auth.go b/auth.go index 7478349..edea85a 100644 --- a/auth.go +++ b/auth.go @@ -14,6 +14,8 @@ import ( "strconv" "strings" "time" + + "math/rand" ) const ( @@ -160,6 +162,12 @@ func (s *Scraper) IsLoggedIn() bool { return s.isLogged } +// randomDelay introduces a random delay between 1 and 3 seconds +func randomDelay() { + delay := time.Duration(3000+rand.Intn(5000)) * time.Millisecond + time.Sleep(delay) +} + // Login to Twitter // Use Login(username, password) for ordinary login // or Login(username, password, email) for login if you have email confirmation @@ -182,6 +190,8 @@ func (s *Scraper) Login(credentials ...string) error { return err } + randomDelay() + // flow start data := map[string]interface{}{ "flow_name": "login", @@ -197,6 +207,8 @@ func (s *Scraper) Login(credentials ...string) error { return err } + randomDelay() + // flow instrumentation step data = map[string]interface{}{ "flow_token": flowToken, @@ -212,6 +224,8 @@ func (s *Scraper) Login(credentials ...string) error { return err } + randomDelay() + // flow username step data = map[string]interface{}{ "flow_token": flowToken, @@ -235,6 +249,8 @@ func (s *Scraper) Login(credentials ...string) error { return err } + randomDelay() + // flow password step data = map[string]interface{}{ "flow_token": flowToken, @@ -250,6 +266,8 @@ func (s *Scraper) Login(credentials ...string) error { return err } + randomDelay() + // flow duplication check data = map[string]interface{}{ "flow_token": flowToken, @@ -273,6 +291,9 @@ func (s *Scraper) Login(credentials ...string) error { if confirmation == "" { return fmt.Errorf("confirmation data required for %v", confirmationSubtask) } + + randomDelay() + // flow confirmation data = map[string]interface{}{ "flow_token": flowToken, From 97db62752d23f1fb6a18402f526071da34fca7df Mon Sep 17 00:00:00 2001 From: Valentine Date: Tue, 1 Oct 2024 01:36:46 +0300 Subject: [PATCH 3/3] bump net to v0.29.0 --- go.mod | 2 +- go.sum | 31 ++++++++++++++++++++++++++----- 2 files changed, 27 insertions(+), 6 deletions(-) diff --git a/go.mod b/go.mod index a762b76..8e36457 100644 --- a/go.mod +++ b/go.mod @@ -5,5 +5,5 @@ go 1.16 require ( github.com/AlexEidt/Vidio v1.5.1 github.com/google/go-cmp v0.6.0 - golang.org/x/net v0.22.0 + golang.org/x/net v0.29.0 ) diff --git a/go.sum b/go.sum index 8b2238c..1649b9e 100644 --- a/go.sum +++ b/go.sum @@ -5,21 +5,32 @@ github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeN github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= +golang.org/x/crypto v0.13.0/go.mod h1:y6Z2r+Rw4iayiXXAIxJIDAJ1zMW4yaTpebo8fPOliYc= golang.org/x/crypto v0.19.0/go.mod h1:Iy9bg/ha4yyC70EfRS8jz+B6ybOBKMaSxLj6P6oBDfU= -golang.org/x/crypto v0.21.0/go.mod h1:0BP7YvVV9gBbVKyeTG0Gyn+gZm94bibOW5BjDEYAOMs= +golang.org/x/crypto v0.23.0/go.mod h1:CKFgDieR+mRhux2Lsu27y0fO304Db0wZe70UKqHu0v8= +golang.org/x/crypto v0.27.0/go.mod h1:1Xngt8kV6Dvbssa53Ziq6Eqn0HqbZi5Z6R0ZpwQzt70= golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4= golang.org/x/mod v0.8.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= +golang.org/x/mod v0.12.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= +golang.org/x/mod v0.15.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c= +golang.org/x/mod v0.17.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c= golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= golang.org/x/net v0.6.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs= golang.org/x/net v0.10.0/go.mod h1:0qNGK6F8kojg2nk9dLZ2mShWaEBan6FAoqfSigmmuDg= +golang.org/x/net v0.15.0/go.mod h1:idbUs1IY1+zTqbi8yxTbhexhEEk5ur9LInksu6HrEpk= golang.org/x/net v0.21.0/go.mod h1:bIjVDfnllIU7BJ2DNgfnXvpSvtn8VRwhlsaeUTyUS44= -golang.org/x/net v0.22.0 h1:9sGLhx7iRIHEiX0oAJ3MRZMUCElJgy7Br1nO+AMN3Tc= -golang.org/x/net v0.22.0/go.mod h1:JKghWKKOSdJwpW2GEx0Ja7fmaKnMsbu+MWVZTokSYmg= +golang.org/x/net v0.25.0/go.mod h1:JkAGAh7GEvH74S6FOH42FLoXpXbE/aqXSrIQjXgsiwM= +golang.org/x/net v0.29.0 h1:5ORfpBpCs4HzDYoodCDBbwHzdR5UrLBZ3sOnUJmFoHo= +golang.org/x/net v0.29.0/go.mod h1:gLkgy8jTGERgjzMic6DS9+SP0ajcu6Xu3Orq/SpETg0= golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.1.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.3.0/go.mod h1:FU7BRWz2tNW+3quACPkgCx/L+uEAv1htQ0V83Z9Rj+Y= +golang.org/x/sync v0.6.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= +golang.org/x/sync v0.7.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= +golang.org/x/sync v0.8.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= @@ -27,22 +38,32 @@ golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBc golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.8.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.12.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.17.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= -golang.org/x/sys v0.18.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/sys v0.20.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/sys v0.25.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/telemetry v0.0.0-20240228155512-f48c80bd79b2/go.mod h1:TeRTkGYfJXctD9OcfyVLyj2J3IxLnKwHJR8f4D8a3YE= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= golang.org/x/term v0.5.0/go.mod h1:jMB1sMXY+tzblOD4FWmEbocvup2/aLOaQEp7JmGp78k= golang.org/x/term v0.8.0/go.mod h1:xPskH00ivmX89bAKVGSKKtLOWNx2+17Eiy94tnKShWo= +golang.org/x/term v0.12.0/go.mod h1:owVbMEjm3cBLCHdkQu9b1opXd4ETQWc3BhuQGKgXgvU= golang.org/x/term v0.17.0/go.mod h1:lLRBjIVuehSbZlaOtGMbcMncT+aqLLLmKrsjNrUguwk= -golang.org/x/term v0.18.0/go.mod h1:ILwASektA3OnRv7amZ1xhE/KTR+u50pbXfZ03+6Nx58= +golang.org/x/term v0.20.0/go.mod h1:8UkIAJTvZgivsXaD6/pH6U9ecQzZ45awqEOzuCvwpFY= +golang.org/x/term v0.24.0/go.mod h1:lOBK/LVxemqiMij05LGJ0tzNr8xlmwBRJ81PX6wVLH8= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= golang.org/x/text v0.9.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8= +golang.org/x/text v0.13.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE= golang.org/x/text v0.14.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU= +golang.org/x/text v0.15.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU= +golang.org/x/text v0.18.0/go.mod h1:BuEKDfySbSR4drPmRPG/7iBdf8hvFMuRexcpahXilzY= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc= golang.org/x/tools v0.6.0/go.mod h1:Xwgl3UAJ/d3gWutnCtw505GrjyAbvKui8lOU390QaIU= +golang.org/x/tools v0.13.0/go.mod h1:HvlwmtVNQAhOuCjW7xxvovg8wbNq7LwfXh/k7wXUl58= +golang.org/x/tools v0.21.1-0.20240508182429-e35e4ccd0d2d/go.mod h1:aiJjzUbINMkxbQROHiO6hDPo2LHcIPhhQsa9DLh0yGk= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=