Add --stop-on-429 flag to stop scanning when the server rate limits

README.md

@@ -286,6 +286,7 @@ gobuster gcs -w company-names.txt -v
 - Try reducing thread count with `-t` flag
 - Add delays between requests with `--delay`
 - Use different user agent with `-a` flag
+- Use `--stop-on-429` to stop the scan as soon as the server starts rate limiting (HTTP 429) instead of hammering it and risking a block
 
 #### "Connection Timeout"
 

cli/options.go

@@ -234,6 +234,7 @@ func GlobalOptions() []cli.Flag {
 		&cli.StringFlag{Name: "pattern", Aliases: []string{"p"}, Usage: "File containing replacement patterns"},
 		&cli.StringFlag{Name: "discover-pattern", Aliases: []string{"pd"}, Usage: "File containing replacement patterns applied to successful guesses"},
 		&cli.BoolFlag{Name: "no-color", Aliases: []string{"nc"}, Value: false, Usage: "Disable color output"},
+		&cli.BoolFlag{Name: "stop-on-429", Value: false, Usage: "Stop the scan when the server responds with HTTP 429 Too Many Requests (HTTP based modes only)"},
 		&cli.BoolFlag{Name: "debug", Value: false, Usage: "enable debug output"},
 	}
 }
@@ -305,6 +306,7 @@ func ParseGlobalOptions(c *cli.Context) (libgobuster.Options, error) {
 		color.NoColor = true
 	}
 
+	opts.StopOnRateLimit = c.Bool("stop-on-429")
 	opts.Debug = c.Bool("debug")
 	return opts, nil
 }

gobusterdir/gobusterdir.go

@@ -307,6 +307,10 @@ func (d *GobusterDir) ProcessWord(ctx context.Context, word string, progress *li
 		break
 	}
 
+	if d.globalopts.StopOnRateLimit && statusCode == http.StatusTooManyRequests {
+		return nil, libgobuster.ErrRateLimited
+	}
+
 	if d.options.BodyOutputDir != "" && body != nil {
 		fname := libgobuster.SanitizeFilename(fmt.Sprintf("%s_%d.html", strings.Trim(entity, "/"), statusCode))
 		fpath := filepath.Join(d.options.BodyOutputDir, fname)
@@ -385,6 +389,12 @@ func (d *GobusterDir) GetConfigString() (string, error) {
 		}
 	}
 
+	if d.globalopts.StopOnRateLimit {
+		if _, err := fmt.Fprintf(tw, "[+] Stop on 429:\ttrue\n"); err != nil {
+			return "", err
+		}
+	}
+
 	wordlist := "stdin (pipe)"
 	if d.globalopts.Wordlist != "-" {
 		wordlist = d.globalopts.Wordlist

gobusterdir/ratelimit_test.go

@@ -0,0 +1,70 @@
+package gobusterdir
+
+import (
+	"context"
+	"errors"
+	"net/http"
+	"net/http/httptest"
+	"net/url"
+	"testing"
+	"time"
+
+	"github.com/OJ/gobuster/v3/libgobuster"
+)
+
+func newDirForRateLimit(t *testing.T, target string, stopOn429 bool) *GobusterDir {
+	t.Helper()
+
+	u, err := url.Parse(target)
+	if err != nil {
+		t.Fatalf("could not parse url: %v", err)
+	}
+
+	globalOpts := &libgobuster.Options{StopOnRateLimit: stopOn429}
+	opts := NewOptions()
+	opts.URL = u
+	opts.Timeout = 10 * time.Second
+	// treat 429 as a matching status so, without the flag, it is reported normally
+	opts.StatusCodesParsed.Add(http.StatusTooManyRequests)
+
+	d, err := New(globalOpts, opts, libgobuster.NewLogger(false))
+	if err != nil {
+		t.Fatalf("could not create gobusterdir: %v", err)
+	}
+	return d
+}
+
+func TestProcessWordStopsOn429(t *testing.T) {
+	t.Parallel()
+
+	ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {
+		w.WriteHeader(http.StatusTooManyRequests)
+	}))
+	defer ts.Close()
+
+	d := newDirForRateLimit(t, ts.URL, true)
+
+	_, err := d.ProcessWord(context.Background(), "admin", libgobuster.NewProgress())
+	if !errors.Is(err, libgobuster.ErrRateLimited) {
+		t.Fatalf("expected ErrRateLimited, got %v", err)
+	}
+}
+
+func TestProcessWord429WithoutFlag(t *testing.T) {
+	t.Parallel()
+
+	ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {
+		w.WriteHeader(http.StatusTooManyRequests)
+	}))
+	defer ts.Close()
+
+	d := newDirForRateLimit(t, ts.URL, false)
+
+	res, err := d.ProcessWord(context.Background(), "admin", libgobuster.NewProgress())
+	if err != nil {
+		t.Fatalf("did not expect an error, got %v", err)
+	}
+	if res == nil {
+		t.Fatal("expected a result for the 429 response when the flag is disabled")
+	}
+}

gobusterfuzz/gobusterfuzz.go

@@ -194,6 +194,10 @@ func (d *GobusterFuzz) ProcessWord(ctx context.Context, word string, progress *l
 		break
 	}
 
+	if d.globalopts.StopOnRateLimit && statusCode == http.StatusTooManyRequests {
+		return nil, libgobuster.ErrRateLimited
+	}
+
 	if d.options.BodyOutputDir != "" && body != nil {
 		fname := libgobuster.SanitizeFilename(fmt.Sprintf("%s_%d.html", strings.Trim(word, "/"), statusCode))
 		fpath := filepath.Join(d.options.BodyOutputDir, fname)
@@ -269,6 +273,12 @@ func (d *GobusterFuzz) GetConfigString() (string, error) {
 		}
 	}
 
+	if d.globalopts.StopOnRateLimit {
+		if _, err := fmt.Fprintf(tw, "[+] Stop on 429:\ttrue\n"); err != nil {
+			return "", err
+		}
+	}
+
 	wordlist := "stdin (pipe)"
 	if d.globalopts.Wordlist != "-" {
 		wordlist = d.globalopts.Wordlist

gobustervhost/gobustervhost.go

@@ -201,6 +201,10 @@ func (v *GobusterVhost) ProcessWord(ctx context.Context, word string, progress *
 		break
 	}
 
+	if v.globalopts.StopOnRateLimit && statusCode == http.StatusTooManyRequests {
+		return nil, libgobuster.ErrRateLimited
+	}
+
 	if v.options.BodyOutputDir != "" && body != nil {
 		fname := libgobuster.SanitizeFilename(fmt.Sprintf("%s_%d.html", strings.Trim(word, "/"), statusCode))
 		fpath := filepath.Join(v.options.BodyOutputDir, fname)
@@ -264,6 +268,12 @@ func (v *GobusterVhost) GetConfigString() (string, error) {
 		}
 	}
 
+	if v.globalopts.StopOnRateLimit {
+		if _, err := fmt.Fprintf(tw, "[+] Stop on 429:\ttrue\n"); err != nil {
+			return "", err
+		}
+	}
+
 	wordlist := "stdin (pipe)"
 	if v.globalopts.Wordlist != "-" {
 		wordlist = v.globalopts.Wordlist

libgobuster/errors.go

@@ -6,4 +6,5 @@ var (
 	ErrTimeout           = errors.New("timeout occurred during the request")
 	ErrEOF               = errors.New("server closed connection without sending any data back. Maybe you are connecting via https to on http port or vice versa?")
 	ErrConnectionRefused = errors.New("connection refused")
+	ErrRateLimited       = errors.New("server responded with HTTP 429 Too Many Requests")
 )

libgobuster/libgobuster.go

@@ -9,6 +9,7 @@ import (
 	"os"
 	"strings"
 	"sync"
+	"sync/atomic"
 	"time"
 )
 
@@ -26,10 +27,12 @@ type ResultToStringFunc func(*Gobuster, *Result) (*string, error)
 
 // Gobuster is the main object when creating a new run
 type Gobuster struct {
-	Opts     *Options
-	Logger   *Logger
-	plugin   GobusterPlugin
-	Progress *Progress
+	Opts        *Options
+	Logger      *Logger
+	plugin      GobusterPlugin
+	Progress    *Progress
+	cancel      context.CancelFunc
+	rateLimited atomic.Bool
 }
 
 type Guess struct {
@@ -71,6 +74,19 @@ func (g *Gobuster) worker(ctx context.Context, guessChan <-chan *Guess, successC
 			// Mode-specific processing
 			res, err := g.plugin.ProcessWord(ctx, guess.word, g.Progress)
 			if err != nil {
+				if g.Opts.StopOnRateLimit && errors.Is(err, ErrRateLimited) {
+					// only the first worker to hit the limit prints the message
+					if g.rateLimited.CompareAndSwap(false, true) {
+						g.Progress.MessageChan <- Message{
+							Level:   LevelError,
+							Message: "hit rate limit (HTTP 429), stopping. Use a delay or lower the thread count to avoid this",
+						}
+					}
+					if g.cancel != nil {
+						g.cancel()
+					}
+					return
+				}
 				// do not exit and continue
 				g.Progress.ErrorChan <- fmt.Errorf("error on word %s: %w", guess.word, err)
 			}
@@ -232,9 +248,15 @@ func (g *Gobuster) Run(ctx context.Context) error {
 		return err
 	}
 
-	workerCtx, workerCancel := context.WithCancel(ctx)
+	// runCtx lets a worker stop the whole run early, for example when the
+	// server starts rate limiting and StopOnRateLimit is set
+	runCtx, runCancel := context.WithCancel(ctx)
+	defer runCancel()
+	g.cancel = runCancel
+
+	workerCtx, workerCancel := context.WithCancel(runCtx)
 	defer workerCancel()
-	feederCtx, feederCancel := context.WithCancel(ctx)
+	feederCtx, feederCancel := context.WithCancel(runCtx)
 	defer feederCancel()
 
 	var workerGroup, feederGroup sync.WaitGroup
@@ -271,13 +293,13 @@ ListenForMore:
 	for {
 		// Prioritize stopping when the context is done
 		select {
-		case <-ctx.Done():
+		case <-runCtx.Done():
 			break ListenForMore
 		default:
 		}
 
 		select {
-		case <-ctx.Done():
+		case <-runCtx.Done():
 			break ListenForMore
 		case successGuess := <-successChan:
 			// Add more guesses based on the results of previous attempts

libgobuster/options.go

@@ -19,4 +19,5 @@ type Options struct {
 	NoError             bool
 	Quiet               bool
 	Delay               time.Duration
+	StopOnRateLimit     bool
 }