From cc5c44a08a43865de686e3b5a0d0bfb19a18d58b Mon Sep 17 00:00:00 2001 From: Dwi Siswanto Date: Tue, 3 Sep 2024 16:34:05 +0700 Subject: [PATCH 1/8] feat(server): use request var from goroutine Signed-off-by: Dwi Siswanto --- internal/server/handler.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/internal/server/handler.go b/internal/server/handler.go index bccd04f..7bec20f 100644 --- a/internal/server/handler.go +++ b/internal/server/handler.go @@ -60,7 +60,7 @@ func (p *Proxy) onRequest(req *http.Request, ctx *goproxy.ProxyCtx) (*http.Reque Transport: tr, } - client, err := proxy.New(req) + client, err := proxy.New(r) if err != nil { resChan <- err return @@ -71,7 +71,7 @@ func (p *Proxy) onRequest(req *http.Request, ctx *goproxy.ProxyCtx) (*http.Reque client.Transport = dump.RoundTripper(tr) } - resp, err := client.Do(req) + resp, err := client.Do(r) if err != nil { resChan <- err return From a55d2b402edc8930922475226a4e2214dfee38c4 Mon Sep 17 00:00:00 2001 From: Dwi Siswanto Date: Tue, 3 Sep 2024 16:37:51 +0700 Subject: [PATCH 2/8] feat(mubeng): add `ToRetryableHTTPClient` func Signed-off-by: Dwi Siswanto --- go.mod | 8 +++++--- go.sum | 11 +++++++++++ pkg/mubeng/mubeng.go | 12 +++++++++++- 3 files changed, 27 insertions(+), 4 deletions(-) diff --git a/go.mod b/go.mod index c5b25b0..5371280 100644 --- a/go.mod +++ b/go.mod @@ -22,15 +22,17 @@ require ( ) require ( - github.com/fatih/color v1.13.0 // indirect + github.com/fatih/color v1.16.0 // indirect github.com/google/go-cmp v0.5.8 // indirect github.com/google/go-github v17.0.0+incompatible // indirect github.com/google/go-querystring v1.1.0 // indirect + github.com/hashicorp/go-cleanhttp v0.5.2 // indirect + github.com/hashicorp/go-retryablehttp v0.7.7 // indirect github.com/hashicorp/go-version v1.3.0 // indirect github.com/json-iterator/go v1.1.12 // indirect github.com/kballard/go-shellquote v0.0.0-20180428030007-95032a82bc51 // indirect github.com/mattn/go-colorable v0.1.13 // indirect - github.com/mattn/go-isatty v0.0.16 // indirect + github.com/mattn/go-isatty v0.0.20 // indirect github.com/mgutz/ansi v0.0.0-20170206155736-9520e82c474b // indirect github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect github.com/modern-go/reflect2 v1.0.2 // indirect @@ -38,7 +40,7 @@ require ( go.uber.org/atomic v1.10.0 // indirect go.uber.org/multierr v1.9.0 // indirect golang.org/x/net v0.0.0-20220722155237-a158d28d115b // indirect - golang.org/x/sys v0.0.0-20220908164124-27713097b956 // indirect + golang.org/x/sys v0.20.0 // indirect golang.org/x/term v0.1.0 // indirect golang.org/x/text v0.4.0 // indirect ) diff --git a/go.sum b/go.sum index a6a7078..e43e02a 100644 --- a/go.sum +++ b/go.sum @@ -16,6 +16,8 @@ github.com/elazarl/goproxy/ext v0.0.0-20190711103511-473e67f1d7d2 h1:dWB6v3RcOy0 github.com/elazarl/goproxy/ext v0.0.0-20190711103511-473e67f1d7d2/go.mod h1:gNh8nYJoAm43RfaxurUnxr+N1PwuFV3ZMl/efxlIlY8= github.com/fatih/color v1.13.0 h1:8LOYc1KYPPmyKMuN8QV2DNRWNbLo6LZ0iLs8+mlH53w= github.com/fatih/color v1.13.0/go.mod h1:kLAiJbzzSOZDVNGyDpeOxJ47H46qBXwg5ILebYFFOfk= +github.com/fatih/color v1.16.0 h1:zmkK9Ngbjj+K0yRhTVONQh1p/HknKYSlNT+vZCzyokM= +github.com/fatih/color v1.16.0/go.mod h1:fL2Sau1YI5c0pdGEVCbKQbLXB6edEj1ZgiY4NijnWvE= github.com/fsnotify/fsnotify v1.6.0 h1:n+5WquG0fcWoWp6xPWfHdbskMCQaFnG6PfBrh1Ky4HY= github.com/fsnotify/fsnotify v1.6.0/go.mod h1:sl3t1tCWJFWoRz9R8WJCbQihKKwmorjAbSClcnxKAGw= github.com/go-test/deep v1.0.7 h1:/VSMRlnY/JSyqxQUzQLKVMAskpY/NZKFA5j2P+0pP2M= @@ -30,6 +32,10 @@ github.com/google/go-querystring v1.1.0/go.mod h1:Kcdr2DB4koayq7X8pmAG4sNG59So17 github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= github.com/h12w/go-socks5 v0.0.0-20200522160539-76189e178364 h1:5XxdakFhqd9dnXoAZy1Mb2R/DZ6D1e+0bGC/JhucGYI= github.com/h12w/go-socks5 v0.0.0-20200522160539-76189e178364/go.mod h1:eDJQioIyy4Yn3MVivT7rv/39gAJTrA7lgmYr8EW950c= +github.com/hashicorp/go-cleanhttp v0.5.2 h1:035FKYIWjmULyFRBKPs8TBQoi0x6d9G4xc9neXJWAZQ= +github.com/hashicorp/go-cleanhttp v0.5.2/go.mod h1:kO/YDlP8L1346E6Sodw+PrpBSV4/SoxCXGY6BqNFT48= +github.com/hashicorp/go-retryablehttp v0.7.7 h1:C8hUCYzor8PIfXHa4UrZkU4VvK8o9ISHxT2Q8+VepXU= +github.com/hashicorp/go-retryablehttp v0.7.7/go.mod h1:pkQpWZeYWskR+D1tR2O5OcBFOxfA7DoAO6xtkuQnHTk= github.com/hashicorp/go-version v1.3.0 h1:McDWVJIU/y+u1BRV06dPaLfLCaT7fUTJLp5r04x7iNw= github.com/hashicorp/go-version v1.3.0/go.mod h1:fltr4n8CU8Ke44wwGCBoEymUuxUHl09ZGVZPK5anwXA= github.com/henvic/httpretty v0.1.2 h1:EQo556sO0xeXAjP10eB+BZARMuvkdGqtfeS4Ntjvkiw= @@ -60,6 +66,8 @@ github.com/mattn/go-isatty v0.0.12/go.mod h1:cbi8OIDigv2wuxKPP5vlRcQ1OAZbq2CE4Ky github.com/mattn/go-isatty v0.0.14/go.mod h1:7GGIvUiUoEMVVmxf/4nioHXj79iQHKdU27kJ6hsGG94= github.com/mattn/go-isatty v0.0.16 h1:bq3VjFmv/sOjHtdEhmkEV4x1AJtvUvOJ2PFAZ5+peKQ= github.com/mattn/go-isatty v0.0.16/go.mod h1:kYGgaQfpe5nmfYZH+SKPsOc2e4SrIfOl2e/yFXSvRLM= +github.com/mattn/go-isatty v0.0.20 h1:xfD0iDuEKnDkl03q4limB+vH+GxLEtL/jb4xVJSWWEY= +github.com/mattn/go-isatty v0.0.20/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y= github.com/mbndr/logo v0.0.1 h1:qWCRjgCP9tao2jGsYiGXceEUQEoILRFfbZaf+o4r868= github.com/mbndr/logo v0.0.1/go.mod h1:eD5FeNoALymRumX35fnzCm0RtoS9HQ+jDC+GGmE6/KM= github.com/mgutz/ansi v0.0.0-20170206155736-9520e82c474b h1:j7+1HpAFS1zy5+Q4qx1fWh90gTKwiN4QCGoY9TWyyO4= @@ -126,6 +134,9 @@ golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBc golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220908164124-27713097b956 h1:XeJjHH1KiLpKGb6lvMiksZ9l0fVUh+AmGcm0nOMEBOY= golang.org/x/sys v0.0.0-20220908164124-27713097b956/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.20.0 h1:Od9JTbYCk261bKm4M/mw7AklTlFYIa0bIp9BgSm1S8Y= +golang.org/x/sys v0.20.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= golang.org/x/term v0.1.0 h1:g6Z6vPFA9dYBAF7DWcH6sCcOntplXsDKcliusYijMlw= diff --git a/pkg/mubeng/mubeng.go b/pkg/mubeng/mubeng.go index 3d653ee..3f92cc2 100644 --- a/pkg/mubeng/mubeng.go +++ b/pkg/mubeng/mubeng.go @@ -5,9 +5,11 @@ import ( "net/http" "net/url" "strings" + + "github.com/hashicorp/go-retryablehttp" ) -// New define HTTP client & request of http.Request itself. +// New define HTTP client request of the [http.Request] itself. // // also removes Hop-by-hop headers when it is sent to backend (see http://www.w3.org/Protocols/rfc2616/rfc2616-sec13.html), // then add X-Forwarded-For header value with the IP address value of rotator proxy IP. @@ -38,3 +40,11 @@ func (proxy *Proxy) New(req *http.Request) (*http.Client, error) { return client, nil } + +// ToRetryableHTTPClient converts standard [http.Client] to [retryablehttp.Client] +func ToRetryableHTTPClient(client *http.Client) *retryablehttp.Client { + retryablehttpClient := retryablehttp.NewClient() + retryablehttpClient.HTTPClient = client + + return retryablehttpClient +} From 02867d13bf1bd7e4b99521a225a21db0345645c9 Mon Sep 17 00:00:00 2001 From: Dwi Siswanto Date: Tue, 3 Sep 2024 18:16:24 +0700 Subject: [PATCH 3/8] feat(server): add `ReleveledLogo` logger interface Signed-off-by: Dwi Siswanto --- internal/server/releveledlogo.go | 71 ++++++++++++++++++++++++++++++++ 1 file changed, 71 insertions(+) create mode 100644 internal/server/releveledlogo.go diff --git a/internal/server/releveledlogo.go b/internal/server/releveledlogo.go new file mode 100644 index 0000000..a6e13b0 --- /dev/null +++ b/internal/server/releveledlogo.go @@ -0,0 +1,71 @@ +package server + +import ( + "fmt" + "net/http" + + "github.com/mbndr/logo" +) + +// repo:hashicorp/go-retryablehttp /v\.(Error|Info|Debug|Warn)\("/ +type ReleveledLogo struct { + *logo.Logger + *http.Request + + Verbose bool +} + +func (l ReleveledLogo) toArgs(msg string, keysAndValues ...interface{}) []interface{} { + args := []interface{}{msg} + + for i := 0; i < len(keysAndValues); i += 2 { + if i+1 < len(keysAndValues) { + key := keysAndValues[i] + value := keysAndValues[i+1] + + switch key { + case "request", "timeout": + continue + case "error": + return []interface{}{value} + case "remaining": + value = fmt.Sprintf("%d", value) + } + + args = append(args, fmt.Sprintf(" [%v=%q]", key, value)) + } + } + + return args +} + +func (l ReleveledLogo) Error(msg string, keysAndValues ...interface{}) { + if l.Verbose { + return + } + + args := l.toArgs(msg, keysAndValues...) + l.Logger.Error(args...) +} + +func (l ReleveledLogo) Info(msg string, keysAndValues ...interface{}) { + args := l.toArgs(msg, keysAndValues...) + l.Logger.Info(args...) +} + +func (l ReleveledLogo) Debug(msg string, keysAndValues ...interface{}) { + switch msg { + case "performing request": + return + case "retrying request": + msg = fmt.Sprintf("%s Retrying %s %s", l.Request.RemoteAddr, l.Request.Method, l.Request.URL) + } + + args := l.toArgs(msg, keysAndValues...) + l.Logger.Debug(args...) +} + +func (l ReleveledLogo) Warn(msg string, keysAndValues ...interface{}) { + args := l.toArgs(msg, keysAndValues...) + l.Logger.Warn(args...) +} From 1730c4741cc96b9171402ec424b833085442d93f Mon Sep 17 00:00:00 2001 From: Dwi Siswanto Date: Tue, 3 Sep 2024 18:16:42 +0700 Subject: [PATCH 4/8] feat(server): use retryablehttp client Signed-off-by: Dwi Siswanto --- internal/server/handler.go | 19 ++++++++++++++++++- 1 file changed, 18 insertions(+), 1 deletion(-) diff --git a/internal/server/handler.go b/internal/server/handler.go index 7bec20f..3045cb5 100644 --- a/internal/server/handler.go +++ b/internal/server/handler.go @@ -9,6 +9,7 @@ import ( "strings" "github.com/elazarl/goproxy" + "github.com/hashicorp/go-retryablehttp" "github.com/kitabisa/mubeng/common" "github.com/kitabisa/mubeng/pkg/helper" "github.com/kitabisa/mubeng/pkg/mubeng" @@ -71,7 +72,23 @@ func (p *Proxy) onRequest(req *http.Request, ctx *goproxy.ProxyCtx) (*http.Reque client.Transport = dump.RoundTripper(tr) } - resp, err := client.Do(r) + retryablehttpClient := mubeng.ToRetryableHTTPClient(client) + retryablehttpClient.RetryMax = 3 + retryablehttpClient.RetryWaitMin = client.Timeout + retryablehttpClient.RetryWaitMax = client.Timeout + retryablehttpClient.Logger = ReleveledLogo{ + Logger: log, + Request: r, + Verbose: p.Options.Verbose, + } + + retryablehttpRequest, err := retryablehttp.FromRequest(r) + if err != nil { + resChan <- err + return + } + + resp, err := retryablehttpClient.Do(retryablehttpRequest) if err != nil { resChan <- err return From 3129e47635ce6ee5bc6219eb252dd62a27f27303 Mon Sep 17 00:00:00 2001 From: Dwi Siswanto Date: Tue, 3 Sep 2024 20:02:26 +0700 Subject: [PATCH 5/8] feat(common): add `MaxRetries` field Signed-off-by: Dwi Siswanto --- common/options.go | 29 +++++++++++++++-------------- common/vars.go | 1 + 2 files changed, 16 insertions(+), 14 deletions(-) diff --git a/common/options.go b/common/options.go index 1bb11a3..bad3a2f 100644 --- a/common/options.go +++ b/common/options.go @@ -13,18 +13,19 @@ type Options struct { Result *os.File Timeout time.Duration - Address string - Auth string - CC string - Check bool - Countries []string - Daemon bool - File string - Goroutine int - Method string - Output string - Rotate int - Sync bool - Verbose bool - Watch bool + Address string + Auth string + CC string + Check bool + Countries []string + Daemon bool + File string + Goroutine int + Method string + Output string + Rotate int + Sync bool + Verbose bool + Watch bool + MaxRetries int } diff --git a/common/vars.go b/common/vars.go index 186a2c5..61b9ee9 100644 --- a/common/vars.go +++ b/common/vars.go @@ -39,6 +39,7 @@ Options: -d, --daemon Daemonize proxy server -m, --method Rotation method (sequent/random) (default: sequent) -r, --rotate Rotate proxy IP after N request (default: 1) + --max-retries Max. retries for failed HTTP requests (default: 3) -s, --sync Syncrounus mode -w, --watch Watch proxy file, live-reload from changes From 5c5f8ac5866413901b3682cede7d613954966819 Mon Sep 17 00:00:00 2001 From: Dwi Siswanto Date: Tue, 3 Sep 2024 20:02:36 +0700 Subject: [PATCH 6/8] feat(runner): add `MaxRetries` field Signed-off-by: Dwi Siswanto --- internal/runner/options.go | 2 ++ 1 file changed, 2 insertions(+) diff --git a/internal/runner/options.go b/internal/runner/options.go index 9f7fec6..2acda77 100644 --- a/internal/runner/options.go +++ b/internal/runner/options.go @@ -60,6 +60,8 @@ func Options() *common.Options { flag.IntVar(&opt.Goroutine, "g", 50, "") flag.IntVar(&opt.Goroutine, "goroutine", 50, "") + flag.IntVar(&opt.MaxRetries, "max-retries", 3, "") + flag.Usage = func() { showBanner() showUsage() From c8dbbaad3c90c751ccca7385a2683f284d2d9531 Mon Sep 17 00:00:00 2001 From: Dwi Siswanto Date: Tue, 3 Sep 2024 20:02:54 +0700 Subject: [PATCH 7/8] feat(server): impl `MaxRetries` opt Signed-off-by: Dwi Siswanto --- internal/server/handler.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/internal/server/handler.go b/internal/server/handler.go index 3045cb5..7ba3be1 100644 --- a/internal/server/handler.go +++ b/internal/server/handler.go @@ -73,7 +73,7 @@ func (p *Proxy) onRequest(req *http.Request, ctx *goproxy.ProxyCtx) (*http.Reque } retryablehttpClient := mubeng.ToRetryableHTTPClient(client) - retryablehttpClient.RetryMax = 3 + retryablehttpClient.RetryMax = p.Options.MaxRetries retryablehttpClient.RetryWaitMin = client.Timeout retryablehttpClient.RetryWaitMax = client.Timeout retryablehttpClient.Logger = ReleveledLogo{ From fce4c22eb2f4f0d6211eda930b429cc497bcdb9e Mon Sep 17 00:00:00 2001 From: Dwi Siswanto Date: Tue, 3 Sep 2024 20:03:12 +0700 Subject: [PATCH 8/8] docs(README): add `--max-retry` flag Signed-off-by: Dwi Siswanto --- README.md | 1 + 1 file changed, 1 insertion(+) diff --git a/README.md b/README.md index bad44d3..750c6a6 100644 --- a/README.md +++ b/README.md @@ -141,6 +141,7 @@ Here are all the options it supports. | --only-cc `,` | Only show specific country code (comma separated). | | -t, --timeout | Max. time allowed for proxy server/check (default: 30s). | | -r, --rotate `` | Rotate proxy IP for every `AFTER` request (default: 1). | +| --max-retries `` | Max. retries for failed HTTP requests (default: 3). | | -m, --method `` | Rotation method (sequent/random) (default: sequent). | | -s, --sync | Sync will wait for the previous request to complete. | | -v, --verbose | Dump HTTP request/responses or show died proxy on check. |