From 341e1de6d0de12227daeb276fc5b6593a2b775cd Mon Sep 17 00:00:00 2001
From: JustHumanz <Aldinsetiawan99@gmail.com>
Date: Thu, 3 Dec 2020 21:42:16 +0700
Subject: [PATCH 1/2] add http proxy support

---
 profile.go | 15 ++++++++++++++-
 search.go  | 16 +++++++++++++++-
 trends.go  | 16 +++++++++++++++-
 util.go    | 20 ++++++++++++++++++++
 4 files changed, 64 insertions(+), 3 deletions(-)

diff --git a/profile.go b/profile.go
index 2efa234..54bdae2 100644
--- a/profile.go
+++ b/profile.go
@@ -2,6 +2,7 @@ package twitterscraper
 
 import (
 	"fmt"
+	"net"
 	"net/http"
 	"strconv"
 	"strings"
@@ -35,13 +36,25 @@ type Profile struct {
 func GetProfile(username string) (Profile, error) {
 	url := "https://mobile.twitter.com/" + username
 
+	client := http.DefaultClient
+	if HTTPProxy != nil {
+		client = &http.Client{
+			Transport: &http.Transport{
+				Proxy: http.ProxyURL(HTTPProxy),
+				DialContext: (&net.Dialer{
+					Timeout: 10 * time.Second,
+				}).DialContext,
+			},
+		}
+	}
+
 	req, err := http.NewRequest("GET", url, nil)
 	if err != nil {
 		return Profile{}, err
 	}
 	req.Header.Set("Accept-Language", "en-US")
 
-	resp, err := http.DefaultClient.Do(req)
+	resp, err := client.Do(req)
 	if resp == nil {
 		return Profile{}, err
 	}
diff --git a/search.go b/search.go
index 0734161..52eb540 100644
--- a/search.go
+++ b/search.go
@@ -4,9 +4,11 @@ import (
 	"context"
 	"fmt"
 	"io"
+	"net"
 	"net/http"
 	"net/url"
 	"strings"
+	"time"
 
 	"github.com/PuerkitoBio/goquery"
 )
@@ -64,6 +66,18 @@ func FetchSearchTweets(query, nextCursor string) ([]*Tweet, string, error) {
 		url = "https://mobile.twitter.com" + nextCursor
 	}
 
+	client := http.DefaultClient
+	if HTTPProxy != nil {
+		client = &http.Client{
+			Transport: &http.Transport{
+				Proxy: http.ProxyURL(HTTPProxy),
+				DialContext: (&net.Dialer{
+					Timeout: 10 * time.Second,
+				}).DialContext,
+			},
+		}
+	}
+
 	req, err := http.NewRequest("GET", url, nil)
 	if err != nil {
 		return nil, "", err
@@ -72,7 +86,7 @@ func FetchSearchTweets(query, nextCursor string) ([]*Tweet, string, error) {
 	req.Header.Set("Referer", "https://mobile.twitter.com/")
 	req.Header.Set("User-Agent", "Opera/9.80 (J2ME/MIDP; Opera Mini/5.1.21214/28.2725; U; ru) Presto/2.8.119 Version/11.10")
 
-	resp, err := http.DefaultClient.Do(req)
+	resp, err := client.Do(req)
 	if resp == nil {
 		return nil, "", err
 	}
diff --git a/trends.go b/trends.go
index 194438f..69a316f 100644
--- a/trends.go
+++ b/trends.go
@@ -2,8 +2,10 @@ package twitterscraper
 
 import (
 	"fmt"
+	"net"
 	"net/http"
 	"strings"
+	"time"
 
 	"github.com/PuerkitoBio/goquery"
 )
@@ -12,13 +14,25 @@ const trendsURL = "https://mobile.twitter.com/trends"
 
 // GetTrends return list of trends.
 func GetTrends() ([]string, error) {
+	client := http.DefaultClient
+	if HTTPProxy != nil {
+		client = &http.Client{
+			Transport: &http.Transport{
+				Proxy: http.ProxyURL(HTTPProxy),
+				DialContext: (&net.Dialer{
+					Timeout: 10 * time.Second,
+				}).DialContext,
+			},
+		}
+	}
+
 	req, err := http.NewRequest("GET", trendsURL, nil)
 	if err != nil {
 		return nil, err
 	}
 	req.Header.Set("Accept-Language", "en-US")
 
-	resp, err := http.DefaultClient.Do(req)
+	resp, err := client.Do(req)
 	if resp == nil {
 		return nil, err
 	}
diff --git a/util.go b/util.go
index d3c8630..058f746 100644
--- a/util.go
+++ b/util.go
@@ -2,11 +2,31 @@ package twitterscraper
 
 import (
 	"encoding/json"
+	"errors"
 	"fmt"
 	"net/http"
+	"net/url"
+	"regexp"
 	"strings"
 )
 
+//HttpProxy Public variable for Http proxy
+var HTTPProxy *url.URL
+
+//SetProxy set http proxy
+func SetProxy(Proxy string) error {
+	match, _ := regexp.MatchString("http.+", Proxy)
+	if !match {
+		return errors.New("only support http protocol")
+	}
+	urlproxy, err := url.Parse(Proxy)
+	if err != nil {
+		return err
+	}
+	HTTPProxy = urlproxy
+	return nil
+}
+
 func newRequest(url string) (*http.Request, error) {
 	req, err := http.NewRequest("GET", url, nil)
 	if err != nil {

From 8c930602c1494f8f12b8b140bb0407f0db5897f7 Mon Sep 17 00:00:00 2001
From: JustHumanz <Aldinsetiawan99@gmail.com>
Date: Fri, 4 Dec 2020 15:08:33 +0700
Subject: [PATCH 2/2] update readme

---
 README.md | 21 +++++++++++++++++++++
 util.go   |  2 +-
 2 files changed, 22 insertions(+), 1 deletion(-)

diff --git a/README.md b/README.md
index b03d9fc..5b3a2d5 100644
--- a/README.md
+++ b/README.md
@@ -62,7 +62,28 @@ func main() {
     }
 }
 ```
+#### With http proxy
 
+```golang
+package main
+
+import (
+    "context"
+    "fmt"
+    twitterscraper "github.com/n0madic/twitter-scraper"
+)
+
+func main() {
+    twitterscraper.SetProxy("http://localhost:16379")
+    for tweet := range twitterscraper.SearchTweets(context.Background(),
+        "twitter scraper data -filter:retweets", 50) {
+        if tweet.Error != nil {
+            panic(tweet.Error)
+        }
+        fmt.Println(tweet.HTML)
+    }
+}
+```
 The search ends if we have 50 tweets.
 
 See [Rules and filtering](https://developer.twitter.com/en/docs/tweets/rules-and-filtering/overview/standard-operators) for build standard queries.
diff --git a/util.go b/util.go
index 058f746..155d0e0 100644
--- a/util.go
+++ b/util.go
@@ -13,7 +13,7 @@ import (
 //HttpProxy Public variable for Http proxy
 var HTTPProxy *url.URL
 
-//SetProxy set http proxy
+//SetProxy set http proxy format `http://HOST:PORT`
 func SetProxy(Proxy string) error {
 	match, _ := regexp.MatchString("http.+", Proxy)
 	if !match {