From 341e1de6d0de12227daeb276fc5b6593a2b775cd Mon Sep 17 00:00:00 2001
From: JustHumanz <Aldinsetiawan99@gmail.com>
Date: Thu, 3 Dec 2020 21:42:16 +0700
Subject: [PATCH] add http proxy support

---
 profile.go | 15 ++++++++++++++-
 search.go  | 16 +++++++++++++++-
 trends.go  | 16 +++++++++++++++-
 util.go    | 20 ++++++++++++++++++++
 4 files changed, 64 insertions(+), 3 deletions(-)

diff --git a/profile.go b/profile.go
index 2efa234..54bdae2 100644
--- a/profile.go
+++ b/profile.go
@@ -2,6 +2,7 @@ package twitterscraper
 
 import (
 	"fmt"
+	"net"
 	"net/http"
 	"strconv"
 	"strings"
@@ -35,13 +36,25 @@ type Profile struct {
 func GetProfile(username string) (Profile, error) {
 	url := "https://mobile.twitter.com/" + username
 
+	client := http.DefaultClient
+	if HTTPProxy != nil {
+		client = &http.Client{
+			Transport: &http.Transport{
+				Proxy: http.ProxyURL(HTTPProxy),
+				DialContext: (&net.Dialer{
+					Timeout: 10 * time.Second,
+				}).DialContext,
+			},
+		}
+	}
+
 	req, err := http.NewRequest("GET", url, nil)
 	if err != nil {
 		return Profile{}, err
 	}
 	req.Header.Set("Accept-Language", "en-US")
 
-	resp, err := http.DefaultClient.Do(req)
+	resp, err := client.Do(req)
 	if resp == nil {
 		return Profile{}, err
 	}
diff --git a/search.go b/search.go
index 0734161..52eb540 100644
--- a/search.go
+++ b/search.go
@@ -4,9 +4,11 @@ import (
 	"context"
 	"fmt"
 	"io"
+	"net"
 	"net/http"
 	"net/url"
 	"strings"
+	"time"
 
 	"github.com/PuerkitoBio/goquery"
 )
@@ -64,6 +66,18 @@ func FetchSearchTweets(query, nextCursor string) ([]*Tweet, string, error) {
 		url = "https://mobile.twitter.com" + nextCursor
 	}
 
+	client := http.DefaultClient
+	if HTTPProxy != nil {
+		client = &http.Client{
+			Transport: &http.Transport{
+				Proxy: http.ProxyURL(HTTPProxy),
+				DialContext: (&net.Dialer{
+					Timeout: 10 * time.Second,
+				}).DialContext,
+			},
+		}
+	}
+
 	req, err := http.NewRequest("GET", url, nil)
 	if err != nil {
 		return nil, "", err
@@ -72,7 +86,7 @@ func FetchSearchTweets(query, nextCursor string) ([]*Tweet, string, error) {
 	req.Header.Set("Referer", "https://mobile.twitter.com/")
 	req.Header.Set("User-Agent", "Opera/9.80 (J2ME/MIDP; Opera Mini/5.1.21214/28.2725; U; ru) Presto/2.8.119 Version/11.10")
 
-	resp, err := http.DefaultClient.Do(req)
+	resp, err := client.Do(req)
 	if resp == nil {
 		return nil, "", err
 	}
diff --git a/trends.go b/trends.go
index 194438f..69a316f 100644
--- a/trends.go
+++ b/trends.go
@@ -2,8 +2,10 @@ package twitterscraper
 
 import (
 	"fmt"
+	"net"
 	"net/http"
 	"strings"
+	"time"
 
 	"github.com/PuerkitoBio/goquery"
 )
@@ -12,13 +14,25 @@ const trendsURL = "https://mobile.twitter.com/trends"
 
 // GetTrends return list of trends.
 func GetTrends() ([]string, error) {
+	client := http.DefaultClient
+	if HTTPProxy != nil {
+		client = &http.Client{
+			Transport: &http.Transport{
+				Proxy: http.ProxyURL(HTTPProxy),
+				DialContext: (&net.Dialer{
+					Timeout: 10 * time.Second,
+				}).DialContext,
+			},
+		}
+	}
+
 	req, err := http.NewRequest("GET", trendsURL, nil)
 	if err != nil {
 		return nil, err
 	}
 	req.Header.Set("Accept-Language", "en-US")
 
-	resp, err := http.DefaultClient.Do(req)
+	resp, err := client.Do(req)
 	if resp == nil {
 		return nil, err
 	}
diff --git a/util.go b/util.go
index d3c8630..058f746 100644
--- a/util.go
+++ b/util.go
@@ -2,11 +2,31 @@ package twitterscraper
 
 import (
 	"encoding/json"
+	"errors"
 	"fmt"
 	"net/http"
+	"net/url"
+	"regexp"
 	"strings"
 )
 
+//HttpProxy Public variable for Http proxy
+var HTTPProxy *url.URL
+
+//SetProxy set http proxy
+func SetProxy(Proxy string) error {
+	match, _ := regexp.MatchString("http.+", Proxy)
+	if !match {
+		return errors.New("only support http protocol")
+	}
+	urlproxy, err := url.Parse(Proxy)
+	if err != nil {
+		return err
+	}
+	HTTPProxy = urlproxy
+	return nil
+}
+
 func newRequest(url string) (*http.Request, error) {
 	req, err := http.NewRequest("GET", url, nil)
 	if err != nil {