feat: add remove-on-error option

README.md

@@ -142,7 +142,8 @@ Here are all the options it supports.
 | -t, --timeout                   | Max. time allowed for proxy server/check (default: 30s).      |
 | -r, --rotate `<AFTER>`          | Rotate proxy IP for every `AFTER` request (default: 1).       |
 |     --rotate-on-error           | Rotate proxy IP and retry failed HTTP requests.               |
-|     --max-errors `<N>`          | Max. errors allowed during rotation (default:3).              |
+|     --remove-on-error           | Remove proxy IP from proxy pool on failed HTTP requests.      |
+|     --max-errors `<N>`          | Max. errors allowed during rotation (default: 3).             |
 |                                 | Use this with `--rotate-on-error`.                            |
 |     --max-redirs `<N>`          | Max. redirects allowed (default: 10).                         |
 |     --max-retries `<N>`         | Max. retries for failed HTTP requests (default: 0).           |

common/errors/errors.go

@@ -0,0 +1,5 @@
+package errors
+
+import "errors"
+
+var ErrNoProxyLeft = errors.New("no proxy left in the pool")

common/options.go

@@ -25,6 +25,7 @@ type Options struct {
 	Output       string
 	Rotate       int
 	RotateOnErr  bool
+	RemoveOnErr  bool
 	Sync         bool
 	Verbose      bool
 	Watch        bool

common/vars.go

@@ -40,6 +40,7 @@ Options:
     -m, --method <METHOD>            Rotation method (sequent/random) (default: sequent)
     -r, --rotate <N>                 Rotate proxy IP after N request (default: 1)
         --rotate-on-error            Rotate proxy IP and retry failed HTTP requests
+        --remove-on-error            Remove proxy IP from proxy pool on failed HTTP requests
         --max-errors <N>             Max. errors allowed during rotation (default: 3)
                                      Use this with --rotate-on-error
         --max-redirs <N>             Max. redirects allowed (default: 10)

internal/proxymanager/proxymanager.go

@@ -50,7 +50,8 @@ func New(filename string) (*ProxyManager, error) {
 		}
 	}
 
-	manager.Length = len(manager.Proxies)
+	manager.Count()
+
 	if manager.Length < 1 {
 		return manager, fmt.Errorf("open %s: has no valid proxy URLs", filename)
 	}

internal/proxymanager/utils.go

@@ -1,47 +1,86 @@
 package proxymanager
 
 import (
+	"fmt"
 	"math/rand"
 
 	"github.com/fsnotify/fsnotify"
+	"github.com/kitabisa/mubeng/common/errors"
 	"github.com/kitabisa/mubeng/pkg/helper"
 )
 
+// Count counts total proxies
+func (p *ProxyManager) Count() int {
+	p.Length = len(p.Proxies)
+
+	return p.Length
+}
+
 // NextProxy will navigate the next proxy to use
-func (p *ProxyManager) NextProxy() string {
+func (p *ProxyManager) NextProxy() (string, error) {
+	var proxy string
+
+	count := p.Count()
+	if count <= 0 {
+		return proxy, errors.ErrNoProxyLeft
+	}
+
 	p.CurrentIndex++
-	if p.CurrentIndex > len(p.Proxies)-1 {
+	if p.CurrentIndex > count-1 {
 		p.CurrentIndex = 0
 	}
 
-	proxy := p.Proxies[p.CurrentIndex]
+	proxy = p.Proxies[p.CurrentIndex]
 
-	return proxy
+	return proxy, nil
 }
 
 // RandomProxy will choose a proxy randomly from the list
-func (p *ProxyManager) RandomProxy() string {
-	return p.Proxies[rand.Intn(len(p.Proxies))]
+func (p *ProxyManager) RandomProxy() (string, error) {
+	var proxy string
+
+	count := p.Count()
+	if count <= 0 {
+		return proxy, errors.ErrNoProxyLeft
+	}
+
+	proxy = p.Proxies[rand.Intn(count)]
+
+	return proxy, nil
+}
+
+// RemoveProxy removes target proxy from proxy pool
+func (p *ProxyManager) RemoveProxy(target string) error {
+	for i, v := range p.Proxies {
+		if v == target {
+			p.Proxies = append(p.Proxies[:i], p.Proxies[i+1:]...)
+
+			return nil
+		}
+	}
+
+	return fmt.Errorf("Unable to find %q in the proxy pool", target)
 }
 
 // Rotate proxy based on method
 //
 // Valid methods are "sequent" and "random", default return empty string.
-func (p *ProxyManager) Rotate(method string) string {
+func (p *ProxyManager) Rotate(method string) (string, error) {
 	var proxy string
+	var err error
 
 	switch method {
 	case "sequent":
-		proxy = p.NextProxy()
+		proxy, err = p.NextProxy()
 	case "random":
-		proxy = p.RandomProxy()
+		proxy, err = p.RandomProxy()
 	}
 
 	if proxy != "" {
 		proxy = helper.EvalFunc(proxy)
 	}
 
-	return proxy
+	return proxy, err
 }
 
 // Watch proxy file from events

internal/runner/options.go

@@ -34,6 +34,7 @@ func Options() *common.Options {
 	flag.IntVar(&opt.Rotate, "rotate", 1, "")
 
 	flag.BoolVar(&opt.RotateOnErr, "rotate-on-error", false, "")
+	flag.BoolVar(&opt.RemoveOnErr, "remove-on-error", false, "")
 
 	flag.StringVar(&opt.Method, "m", "sequent", "")
 	flag.StringVar(&opt.Method, "method", "sequent", "")

internal/server/handler.go

@@ -31,7 +31,9 @@ func (p *Proxy) onRequest(req *http.Request, ctx *goproxy.ProxyCtx) (*http.Reque
 		log.Debugf("%s %s %s", r.RemoteAddr, r.Method, r.URL)
 
 		for i := 0; i <= p.Options.MaxErrors; i++ {
-			retryablehttpClient, err := p.getClient(r, p.rotateProxy())
+			proxy := p.rotateProxy()
+
+			retryablehttpClient, err := p.getClient(r, proxy)
 			if err != nil {
 				resChan <- err
 
@@ -47,6 +49,15 @@ func (p *Proxy) onRequest(req *http.Request, ctx *goproxy.ProxyCtx) (*http.Reque
 
 			resp, err := retryablehttpClient.Do(retryablehttpRequest)
 			if err != nil {
+				if p.Options.RemoveOnErr {
+					p.removeProxy(proxy)
+
+					log.Debugf(
+						"%s Removing proxy IP from proxy pool [proxies=%q]",
+						r.RemoteAddr, fmt.Sprint(p.Options.ProxyManager.Count()),
+					)
+				}
+
 				if p.Options.RotateOnErr && i < p.Options.MaxErrors {
 					log.Debugf(
 						"%s Retrying (rotated) %s %s [remaining=%q]",
@@ -131,10 +142,14 @@ func (p *Proxy) onResponse(resp *http.Response, ctx *goproxy.ProxyCtx) *http.Res
 }
 
 func (p *Proxy) rotateProxy() string {
-	var proxyAddr string
+	var proxy string
+	var err error
 
 	if ok >= p.Options.Rotate {
-		proxyAddr = p.Options.ProxyManager.Rotate(p.Options.Method)
+		proxy, err = p.Options.ProxyManager.Rotate(p.Options.Method)
+		if err != nil {
+			log.Fatalf("Could not rotate proxy IP: %s", err)
+		}
 
 		if ok >= p.Options.Rotate {
 			ok = 1
@@ -143,7 +158,14 @@ func (p *Proxy) rotateProxy() string {
 		ok++
 	}
 
-	return proxyAddr
+	return proxy
+}
+
+func (p *Proxy) removeProxy(target string) {
+	err := p.Options.ProxyManager.RemoveProxy(target)
+	if err != nil {
+		log.Error(err)
+	}
 }
 
 func (p *Proxy) getClient(req *http.Request, proxyAddr string) (*retryablehttp.Client, error) {

internal/server/server.go

@@ -7,8 +7,8 @@ import (
 
 	"github.com/elazarl/goproxy"
 	"github.com/henvic/httpretty"
-	"github.com/mbndr/logo"
 	"github.com/kitabisa/mubeng/common"
+	"github.com/mbndr/logo"
 )
 
 // Run proxy server with a user defined listener.
@@ -65,6 +65,8 @@ func Run(opt *common.Options) {
 	signal.Notify(stop, os.Interrupt)
 	go interrupt(stop)
 
+	log.Infof("%d proxies loaded", opt.ProxyManager.Count())
+
 	log.Infof("[PID: %d] Starting proxy server on %s", os.Getpid(), opt.Address)
 	if err := server.ListenAndServe(); err != nil && err != http.ErrServerClosed {
 		log.Fatal(err)