diff --git a/README.md b/README.md index ac22036..9056e24 100644 --- a/README.md +++ b/README.md @@ -142,7 +142,8 @@ Here are all the options it supports. | -t, --timeout | Max. time allowed for proxy server/check (default: 30s). | | -r, --rotate `` | Rotate proxy IP for every `AFTER` request (default: 1). | | --rotate-on-error | Rotate proxy IP and retry failed HTTP requests. | -| --max-errors `` | Max. errors allowed during rotation (default:3). | +| --remove-on-error | Remove proxy IP from proxy pool on failed HTTP requests. | +| --max-errors `` | Max. errors allowed during rotation (default: 3). | | | Use this with `--rotate-on-error`. | | --max-redirs `` | Max. redirects allowed (default: 10). | | --max-retries `` | Max. retries for failed HTTP requests (default: 0). | diff --git a/common/errors/errors.go b/common/errors/errors.go new file mode 100644 index 0000000..015d4c1 --- /dev/null +++ b/common/errors/errors.go @@ -0,0 +1,5 @@ +package errors + +import "errors" + +var ErrNoProxyLeft = errors.New("no proxy left in the pool") diff --git a/common/options.go b/common/options.go index 3612773..9189607 100644 --- a/common/options.go +++ b/common/options.go @@ -25,6 +25,7 @@ type Options struct { Output string Rotate int RotateOnErr bool + RemoveOnErr bool Sync bool Verbose bool Watch bool diff --git a/common/vars.go b/common/vars.go index 9564f54..3efdb8d 100644 --- a/common/vars.go +++ b/common/vars.go @@ -40,6 +40,7 @@ Options: -m, --method Rotation method (sequent/random) (default: sequent) -r, --rotate Rotate proxy IP after N request (default: 1) --rotate-on-error Rotate proxy IP and retry failed HTTP requests + --remove-on-error Remove proxy IP from proxy pool on failed HTTP requests --max-errors Max. errors allowed during rotation (default: 3) Use this with --rotate-on-error --max-redirs Max. redirects allowed (default: 10) diff --git a/internal/proxymanager/proxymanager.go b/internal/proxymanager/proxymanager.go index 851b439..b9c8de0 100644 --- a/internal/proxymanager/proxymanager.go +++ b/internal/proxymanager/proxymanager.go @@ -50,7 +50,8 @@ func New(filename string) (*ProxyManager, error) { } } - manager.Length = len(manager.Proxies) + manager.Count() + if manager.Length < 1 { return manager, fmt.Errorf("open %s: has no valid proxy URLs", filename) } diff --git a/internal/proxymanager/utils.go b/internal/proxymanager/utils.go index 2b33207..0232e7d 100644 --- a/internal/proxymanager/utils.go +++ b/internal/proxymanager/utils.go @@ -1,47 +1,86 @@ package proxymanager import ( + "fmt" "math/rand" "github.com/fsnotify/fsnotify" + "github.com/kitabisa/mubeng/common/errors" "github.com/kitabisa/mubeng/pkg/helper" ) +// Count counts total proxies +func (p *ProxyManager) Count() int { + p.Length = len(p.Proxies) + + return p.Length +} + // NextProxy will navigate the next proxy to use -func (p *ProxyManager) NextProxy() string { +func (p *ProxyManager) NextProxy() (string, error) { + var proxy string + + count := p.Count() + if count <= 0 { + return proxy, errors.ErrNoProxyLeft + } + p.CurrentIndex++ - if p.CurrentIndex > len(p.Proxies)-1 { + if p.CurrentIndex > count-1 { p.CurrentIndex = 0 } - proxy := p.Proxies[p.CurrentIndex] + proxy = p.Proxies[p.CurrentIndex] - return proxy + return proxy, nil } // RandomProxy will choose a proxy randomly from the list -func (p *ProxyManager) RandomProxy() string { - return p.Proxies[rand.Intn(len(p.Proxies))] +func (p *ProxyManager) RandomProxy() (string, error) { + var proxy string + + count := p.Count() + if count <= 0 { + return proxy, errors.ErrNoProxyLeft + } + + proxy = p.Proxies[rand.Intn(count)] + + return proxy, nil +} + +// RemoveProxy removes target proxy from proxy pool +func (p *ProxyManager) RemoveProxy(target string) error { + for i, v := range p.Proxies { + if v == target { + p.Proxies = append(p.Proxies[:i], p.Proxies[i+1:]...) + + return nil + } + } + + return fmt.Errorf("Unable to find %q in the proxy pool", target) } // Rotate proxy based on method // // Valid methods are "sequent" and "random", default return empty string. -func (p *ProxyManager) Rotate(method string) string { +func (p *ProxyManager) Rotate(method string) (string, error) { var proxy string + var err error switch method { case "sequent": - proxy = p.NextProxy() + proxy, err = p.NextProxy() case "random": - proxy = p.RandomProxy() + proxy, err = p.RandomProxy() } if proxy != "" { proxy = helper.EvalFunc(proxy) } - return proxy + return proxy, err } // Watch proxy file from events diff --git a/internal/runner/options.go b/internal/runner/options.go index 54e21c4..6f90c1d 100644 --- a/internal/runner/options.go +++ b/internal/runner/options.go @@ -34,6 +34,7 @@ func Options() *common.Options { flag.IntVar(&opt.Rotate, "rotate", 1, "") flag.BoolVar(&opt.RotateOnErr, "rotate-on-error", false, "") + flag.BoolVar(&opt.RemoveOnErr, "remove-on-error", false, "") flag.StringVar(&opt.Method, "m", "sequent", "") flag.StringVar(&opt.Method, "method", "sequent", "") diff --git a/internal/server/handler.go b/internal/server/handler.go index 92a4765..b244023 100644 --- a/internal/server/handler.go +++ b/internal/server/handler.go @@ -31,7 +31,9 @@ func (p *Proxy) onRequest(req *http.Request, ctx *goproxy.ProxyCtx) (*http.Reque log.Debugf("%s %s %s", r.RemoteAddr, r.Method, r.URL) for i := 0; i <= p.Options.MaxErrors; i++ { - retryablehttpClient, err := p.getClient(r, p.rotateProxy()) + proxy := p.rotateProxy() + + retryablehttpClient, err := p.getClient(r, proxy) if err != nil { resChan <- err @@ -47,6 +49,15 @@ func (p *Proxy) onRequest(req *http.Request, ctx *goproxy.ProxyCtx) (*http.Reque resp, err := retryablehttpClient.Do(retryablehttpRequest) if err != nil { + if p.Options.RemoveOnErr { + p.removeProxy(proxy) + + log.Debugf( + "%s Removing proxy IP from proxy pool [proxies=%q]", + r.RemoteAddr, fmt.Sprint(p.Options.ProxyManager.Count()), + ) + } + if p.Options.RotateOnErr && i < p.Options.MaxErrors { log.Debugf( "%s Retrying (rotated) %s %s [remaining=%q]", @@ -131,10 +142,14 @@ func (p *Proxy) onResponse(resp *http.Response, ctx *goproxy.ProxyCtx) *http.Res } func (p *Proxy) rotateProxy() string { - var proxyAddr string + var proxy string + var err error if ok >= p.Options.Rotate { - proxyAddr = p.Options.ProxyManager.Rotate(p.Options.Method) + proxy, err = p.Options.ProxyManager.Rotate(p.Options.Method) + if err != nil { + log.Fatalf("Could not rotate proxy IP: %s", err) + } if ok >= p.Options.Rotate { ok = 1 @@ -143,7 +158,14 @@ func (p *Proxy) rotateProxy() string { ok++ } - return proxyAddr + return proxy +} + +func (p *Proxy) removeProxy(target string) { + err := p.Options.ProxyManager.RemoveProxy(target) + if err != nil { + log.Error(err) + } } func (p *Proxy) getClient(req *http.Request, proxyAddr string) (*retryablehttp.Client, error) { diff --git a/internal/server/server.go b/internal/server/server.go index 172cb94..fae1419 100644 --- a/internal/server/server.go +++ b/internal/server/server.go @@ -7,8 +7,8 @@ import ( "github.com/elazarl/goproxy" "github.com/henvic/httpretty" - "github.com/mbndr/logo" "github.com/kitabisa/mubeng/common" + "github.com/mbndr/logo" ) // Run proxy server with a user defined listener. @@ -65,6 +65,8 @@ func Run(opt *common.Options) { signal.Notify(stop, os.Interrupt) go interrupt(stop) + log.Infof("%d proxies loaded", opt.ProxyManager.Count()) + log.Infof("[PID: %d] Starting proxy server on %s", os.Getpid(), opt.Address) if err := server.ListenAndServe(); err != nil && err != http.ErrServerClosed { log.Fatal(err)