diff --git a/README.md b/README.md index fb22a14c..56786fa9 100644 --- a/README.md +++ b/README.md @@ -286,6 +286,7 @@ gobuster gcs -w company-names.txt -v - Try reducing thread count with `-t` flag - Add delays between requests with `--delay` - Use different user agent with `-a` flag +- Use `--stop-on-429` to stop the scan as soon as the server starts rate limiting (HTTP 429) instead of hammering it and risking a block #### "Connection Timeout" diff --git a/cli/options.go b/cli/options.go index 95302578..4b714a24 100644 --- a/cli/options.go +++ b/cli/options.go @@ -234,6 +234,7 @@ func GlobalOptions() []cli.Flag { &cli.StringFlag{Name: "pattern", Aliases: []string{"p"}, Usage: "File containing replacement patterns"}, &cli.StringFlag{Name: "discover-pattern", Aliases: []string{"pd"}, Usage: "File containing replacement patterns applied to successful guesses"}, &cli.BoolFlag{Name: "no-color", Aliases: []string{"nc"}, Value: false, Usage: "Disable color output"}, + &cli.BoolFlag{Name: "stop-on-429", Value: false, Usage: "Stop the scan when the server responds with HTTP 429 Too Many Requests (HTTP based modes only)"}, &cli.BoolFlag{Name: "debug", Value: false, Usage: "enable debug output"}, } } @@ -305,6 +306,7 @@ func ParseGlobalOptions(c *cli.Context) (libgobuster.Options, error) { color.NoColor = true } + opts.StopOnRateLimit = c.Bool("stop-on-429") opts.Debug = c.Bool("debug") return opts, nil } diff --git a/gobusterdir/gobusterdir.go b/gobusterdir/gobusterdir.go index bd683711..1ec785ab 100644 --- a/gobusterdir/gobusterdir.go +++ b/gobusterdir/gobusterdir.go @@ -307,6 +307,10 @@ func (d *GobusterDir) ProcessWord(ctx context.Context, word string, progress *li break } + if d.globalopts.StopOnRateLimit && statusCode == http.StatusTooManyRequests { + return nil, libgobuster.ErrRateLimited + } + if d.options.BodyOutputDir != "" && body != nil { fname := libgobuster.SanitizeFilename(fmt.Sprintf("%s_%d.html", strings.Trim(entity, "/"), statusCode)) fpath := filepath.Join(d.options.BodyOutputDir, fname) @@ -385,6 +389,12 @@ func (d *GobusterDir) GetConfigString() (string, error) { } } + if d.globalopts.StopOnRateLimit { + if _, err := fmt.Fprintf(tw, "[+] Stop on 429:\ttrue\n"); err != nil { + return "", err + } + } + wordlist := "stdin (pipe)" if d.globalopts.Wordlist != "-" { wordlist = d.globalopts.Wordlist diff --git a/gobusterdir/ratelimit_test.go b/gobusterdir/ratelimit_test.go new file mode 100644 index 00000000..1e53890e --- /dev/null +++ b/gobusterdir/ratelimit_test.go @@ -0,0 +1,70 @@ +package gobusterdir + +import ( + "context" + "errors" + "net/http" + "net/http/httptest" + "net/url" + "testing" + "time" + + "github.com/OJ/gobuster/v3/libgobuster" +) + +func newDirForRateLimit(t *testing.T, target string, stopOn429 bool) *GobusterDir { + t.Helper() + + u, err := url.Parse(target) + if err != nil { + t.Fatalf("could not parse url: %v", err) + } + + globalOpts := &libgobuster.Options{StopOnRateLimit: stopOn429} + opts := NewOptions() + opts.URL = u + opts.Timeout = 10 * time.Second + // treat 429 as a matching status so, without the flag, it is reported normally + opts.StatusCodesParsed.Add(http.StatusTooManyRequests) + + d, err := New(globalOpts, opts, libgobuster.NewLogger(false)) + if err != nil { + t.Fatalf("could not create gobusterdir: %v", err) + } + return d +} + +func TestProcessWordStopsOn429(t *testing.T) { + t.Parallel() + + ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + w.WriteHeader(http.StatusTooManyRequests) + })) + defer ts.Close() + + d := newDirForRateLimit(t, ts.URL, true) + + _, err := d.ProcessWord(context.Background(), "admin", libgobuster.NewProgress()) + if !errors.Is(err, libgobuster.ErrRateLimited) { + t.Fatalf("expected ErrRateLimited, got %v", err) + } +} + +func TestProcessWord429WithoutFlag(t *testing.T) { + t.Parallel() + + ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + w.WriteHeader(http.StatusTooManyRequests) + })) + defer ts.Close() + + d := newDirForRateLimit(t, ts.URL, false) + + res, err := d.ProcessWord(context.Background(), "admin", libgobuster.NewProgress()) + if err != nil { + t.Fatalf("did not expect an error, got %v", err) + } + if res == nil { + t.Fatal("expected a result for the 429 response when the flag is disabled") + } +} diff --git a/gobusterfuzz/gobusterfuzz.go b/gobusterfuzz/gobusterfuzz.go index f30c55d0..b5fc92a1 100644 --- a/gobusterfuzz/gobusterfuzz.go +++ b/gobusterfuzz/gobusterfuzz.go @@ -194,6 +194,10 @@ func (d *GobusterFuzz) ProcessWord(ctx context.Context, word string, progress *l break } + if d.globalopts.StopOnRateLimit && statusCode == http.StatusTooManyRequests { + return nil, libgobuster.ErrRateLimited + } + if d.options.BodyOutputDir != "" && body != nil { fname := libgobuster.SanitizeFilename(fmt.Sprintf("%s_%d.html", strings.Trim(word, "/"), statusCode)) fpath := filepath.Join(d.options.BodyOutputDir, fname) @@ -269,6 +273,12 @@ func (d *GobusterFuzz) GetConfigString() (string, error) { } } + if d.globalopts.StopOnRateLimit { + if _, err := fmt.Fprintf(tw, "[+] Stop on 429:\ttrue\n"); err != nil { + return "", err + } + } + wordlist := "stdin (pipe)" if d.globalopts.Wordlist != "-" { wordlist = d.globalopts.Wordlist diff --git a/gobustervhost/gobustervhost.go b/gobustervhost/gobustervhost.go index 113d7744..9b2cc795 100644 --- a/gobustervhost/gobustervhost.go +++ b/gobustervhost/gobustervhost.go @@ -201,6 +201,10 @@ func (v *GobusterVhost) ProcessWord(ctx context.Context, word string, progress * break } + if v.globalopts.StopOnRateLimit && statusCode == http.StatusTooManyRequests { + return nil, libgobuster.ErrRateLimited + } + if v.options.BodyOutputDir != "" && body != nil { fname := libgobuster.SanitizeFilename(fmt.Sprintf("%s_%d.html", strings.Trim(word, "/"), statusCode)) fpath := filepath.Join(v.options.BodyOutputDir, fname) @@ -264,6 +268,12 @@ func (v *GobusterVhost) GetConfigString() (string, error) { } } + if v.globalopts.StopOnRateLimit { + if _, err := fmt.Fprintf(tw, "[+] Stop on 429:\ttrue\n"); err != nil { + return "", err + } + } + wordlist := "stdin (pipe)" if v.globalopts.Wordlist != "-" { wordlist = v.globalopts.Wordlist diff --git a/libgobuster/errors.go b/libgobuster/errors.go index 228d1662..c6765b8d 100644 --- a/libgobuster/errors.go +++ b/libgobuster/errors.go @@ -6,4 +6,5 @@ var ( ErrTimeout = errors.New("timeout occurred during the request") ErrEOF = errors.New("server closed connection without sending any data back. Maybe you are connecting via https to on http port or vice versa?") ErrConnectionRefused = errors.New("connection refused") + ErrRateLimited = errors.New("server responded with HTTP 429 Too Many Requests") ) diff --git a/libgobuster/libgobuster.go b/libgobuster/libgobuster.go index 1e803d62..24bb4abf 100644 --- a/libgobuster/libgobuster.go +++ b/libgobuster/libgobuster.go @@ -9,6 +9,7 @@ import ( "os" "strings" "sync" + "sync/atomic" "time" ) @@ -26,10 +27,12 @@ type ResultToStringFunc func(*Gobuster, *Result) (*string, error) // Gobuster is the main object when creating a new run type Gobuster struct { - Opts *Options - Logger *Logger - plugin GobusterPlugin - Progress *Progress + Opts *Options + Logger *Logger + plugin GobusterPlugin + Progress *Progress + cancel context.CancelFunc + rateLimited atomic.Bool } type Guess struct { @@ -71,6 +74,19 @@ func (g *Gobuster) worker(ctx context.Context, guessChan <-chan *Guess, successC // Mode-specific processing res, err := g.plugin.ProcessWord(ctx, guess.word, g.Progress) if err != nil { + if g.Opts.StopOnRateLimit && errors.Is(err, ErrRateLimited) { + // only the first worker to hit the limit prints the message + if g.rateLimited.CompareAndSwap(false, true) { + g.Progress.MessageChan <- Message{ + Level: LevelError, + Message: "hit rate limit (HTTP 429), stopping. Use a delay or lower the thread count to avoid this", + } + } + if g.cancel != nil { + g.cancel() + } + return + } // do not exit and continue g.Progress.ErrorChan <- fmt.Errorf("error on word %s: %w", guess.word, err) } @@ -232,9 +248,15 @@ func (g *Gobuster) Run(ctx context.Context) error { return err } - workerCtx, workerCancel := context.WithCancel(ctx) + // runCtx lets a worker stop the whole run early, for example when the + // server starts rate limiting and StopOnRateLimit is set + runCtx, runCancel := context.WithCancel(ctx) + defer runCancel() + g.cancel = runCancel + + workerCtx, workerCancel := context.WithCancel(runCtx) defer workerCancel() - feederCtx, feederCancel := context.WithCancel(ctx) + feederCtx, feederCancel := context.WithCancel(runCtx) defer feederCancel() var workerGroup, feederGroup sync.WaitGroup @@ -271,13 +293,13 @@ ListenForMore: for { // Prioritize stopping when the context is done select { - case <-ctx.Done(): + case <-runCtx.Done(): break ListenForMore default: } select { - case <-ctx.Done(): + case <-runCtx.Done(): break ListenForMore case successGuess := <-successChan: // Add more guesses based on the results of previous attempts diff --git a/libgobuster/options.go b/libgobuster/options.go index 3054aebf..6df1b668 100644 --- a/libgobuster/options.go +++ b/libgobuster/options.go @@ -19,4 +19,5 @@ type Options struct { NoError bool Quiet bool Delay time.Duration + StopOnRateLimit bool } diff --git a/libgobuster/ratelimit_test.go b/libgobuster/ratelimit_test.go new file mode 100644 index 00000000..ca5af099 --- /dev/null +++ b/libgobuster/ratelimit_test.go @@ -0,0 +1,142 @@ +package libgobuster + +import ( + "context" + "fmt" + "os" + "path/filepath" + "sync" + "sync/atomic" + "testing" +) + +// ratePlugin is a minimal GobusterPlugin used to exercise the stop on rate +// limit behaviour without doing any real network work. It returns +// ErrRateLimited once it has processed failAfter words. +type ratePlugin struct { + processed atomic.Int64 + failAfter int64 +} + +func (p *ratePlugin) Name() string { return "ratemock" } + +func (p *ratePlugin) PreRun(_ context.Context, _ *Progress) error { return nil } + +func (p *ratePlugin) ProcessWord(_ context.Context, _ string, _ *Progress) (Result, error) { + n := p.processed.Add(1) + if p.failAfter > 0 && n >= p.failAfter { + return nil, ErrRateLimited + } + return nil, nil // nolint:nilnil +} + +func (p *ratePlugin) AdditionalWords(_ string) []string { return nil } + +func (p *ratePlugin) AdditionalWordsLen() int { return 0 } + +func (p *ratePlugin) AdditionalSuccessWords(_ string) []string { return nil } + +func (p *ratePlugin) GetConfigString() (string, error) { return "", nil } + +func writeWordlist(t *testing.T, words int) string { + t.Helper() + f := filepath.Join(t.TempDir(), "wordlist.txt") + var b []byte + for i := range words { + b = append(b, []byte(fmt.Sprintf("word%d\n", i))...) + } + if err := os.WriteFile(f, b, 0o600); err != nil { + t.Fatalf("could not write wordlist: %v", err) + } + return f +} + +// drain reads from the progress channels until they are closed by Run so the +// worker never blocks on a send. +func drain(g *Gobuster) *sync.WaitGroup { + var wg sync.WaitGroup + wg.Add(3) + go func() { + defer wg.Done() + for r := range g.Progress.ResultChan { + _ = r + } + }() + go func() { + defer wg.Done() + for e := range g.Progress.ErrorChan { + _ = e + } + }() + go func() { + defer wg.Done() + for m := range g.Progress.MessageChan { + _ = m + } + }() + return &wg +} + +func TestStopOnRateLimit(t *testing.T) { + t.Parallel() + + const words = 100 + wordlist := writeWordlist(t, words) + + opts := &Options{ + Threads: 1, + Wordlist: wordlist, + StopOnRateLimit: true, + } + plugin := &ratePlugin{failAfter: 2} + g, err := NewGobuster(opts, plugin, NewLogger(false)) + if err != nil { + t.Fatalf("could not create gobuster: %v", err) + } + + wg := drain(g) + if err := g.Run(context.Background()); err != nil { + t.Fatalf("run returned an error: %v", err) + } + wg.Wait() + + if !g.rateLimited.Load() { + t.Fatal("expected the run to be flagged as rate limited") + } + if got := plugin.processed.Load(); got >= words { + t.Fatalf("expected the run to stop early, but processed %d of %d words", got, words) + } +} + +func TestStopOnRateLimitDisabled(t *testing.T) { + t.Parallel() + + const words = 100 + wordlist := writeWordlist(t, words) + + opts := &Options{ + Threads: 1, + Wordlist: wordlist, + StopOnRateLimit: false, + } + // the plugin still returns ErrRateLimited, but with the flag disabled it + // is treated as a normal error and the run keeps going + plugin := &ratePlugin{failAfter: 2} + g, err := NewGobuster(opts, plugin, NewLogger(false)) + if err != nil { + t.Fatalf("could not create gobuster: %v", err) + } + + wg := drain(g) + if err := g.Run(context.Background()); err != nil { + t.Fatalf("run returned an error: %v", err) + } + wg.Wait() + + if g.rateLimited.Load() { + t.Fatal("run should not be flagged as rate limited when the flag is disabled") + } + if got := plugin.processed.Load(); got != words { + t.Fatalf("expected all %d words to be processed, got %d", words, got) + } +} diff --git a/vhs/gobuster_dir.gif b/vhs/gobuster_dir.gif index e320fa80..83a1b4b2 100644 Binary files a/vhs/gobuster_dir.gif and b/vhs/gobuster_dir.gif differ