fix(honeypot/naive): apply robot9001 style delays

Currently the honeypotting feature has no limits or delays anywhere and
uses that to feed an internal greylist of IP networks. This can cause
issues such as in #1613 where Claude's crawler seemed to pick up on it
and egress data at over one megabit per second until the administrator
noticed and blocked the address range.

This takes a different approach by inspiration of how the classic #xkcd
IRC bot Robot9000 works. The first time a given IPv4 /24 or IPv6 /48
visits a honepot page, Anubis sleeps for 1 millisecond. The second it
sleeps for two milliseconds. The third is four milliseconds and so on.
The goal of this is to make the scraping inherently self-limiting such
that the scrapers go off in their own corner where they won't really
hurt anyone.

Let's see if this works out according to keikaku.

Ref: https://github.com/TecharoHQ/anubis/issues/1613
Signed-off-by: Xe Iaso <me@xeiaso.net>

.github/actions/spelling/expect.txt

@@ -120,6 +120,7 @@ fahedouch
 fastcgi
 FCr
 fcrdns
+fcvg
 fediverse
 ffprobe
 fhdr
@@ -238,6 +239,7 @@ mnt
 Mojeek
 mojeekbot
 mozilla
+mqvh
 myclient
 mymaster
 mypass
@@ -387,6 +389,7 @@ vnd
 VPS
 Vultr
 WAIFU
+wcg
 weblate
 webmaster
 webpage

cmd/anubis/main.go

@@ -259,7 +259,7 @@ func main() {
 	}
 
 	lg.Info("loading policy file", "fname", *policyFname)
-	policy, err := libanubis.LoadPoliciesOrDefault(ctx, *policyFname, *challengeDifficulty, *slogLevel)
+	policy, err := libanubis.LoadPoliciesOrDefault(ctx, *policyFname, *challengeDifficulty, *slogLevel, strings.TrimSpace(*target) == "")
 	if err != nil {
 		log.Fatalf("can't parse policy file: %v", err)
 	}

docs/docs/CHANGELOG.md

@@ -13,6 +13,8 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 
 <!-- This changes the project to: -->
 
+- Patch [GHSA-6wcg-mqvh-fcvg](https://github.com/TecharoHQ/anubis/security/advisories/GHSA-6wcg-mqvh-fcvg) by containing subrequest logic to Anubis instances in subrequest mode.
+- Implement robot9001 style delays on the honeypot feature so that the first hit takes 1 millisecond, the second takes 2, etc.
 - Move metrics server configuration to [the policy file](./admin/policies.mdx#metrics-server).
 - Expose [pprof endpoints](https://pkg.go.dev/net/http/pprof) on the metrics listener to enable profiling Anubis in production.
 - fix: prevent nil pointer panic in challenge validation when threshold rules match during PassChallenge (#1463)
@@ -26,6 +28,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 - Enable [HTTP basic auth](./admin/policies.mdx#http-basic-authentication) for the metrics server.
 - Fix a bug in the dataset poisoning maze that could allow denial of service [#1580](https://github.com/TecharoHQ/anubis/issues/1580).
 - Add config option to add ASN to logs/metrics.
+- Log weight when issuing challenge
 
 ## v1.25.0: Necron
 

go.mod

@@ -106,7 +106,7 @@ require (
 	github.com/go-git/gcfg v1.5.1-0.20230307220236-3a3c6141e376 // indirect
 	github.com/go-git/go-billy/v5 v5.6.2 // indirect
 	github.com/go-git/go-git/v5 v5.16.2 // indirect
-	github.com/go-jose/go-jose/v3 v3.0.4 // indirect
+	github.com/go-jose/go-jose/v3 v3.0.5 // indirect
 	github.com/go-logr/logr v1.4.3 // indirect
 	github.com/go-logr/stdr v1.2.2 // indirect
 	github.com/go-ole/go-ole v1.3.0 // indirect

go.sum

@@ -189,8 +189,8 @@ github.com/go-git/go-git-fixtures/v4 v4.3.2-0.20231010084843-55a94097c399 h1:eMj
 github.com/go-git/go-git-fixtures/v4 v4.3.2-0.20231010084843-55a94097c399/go.mod h1:1OCfN199q1Jm3HZlxleg+Dw/mwps2Wbk9frAWm+4FII=
 github.com/go-git/go-git/v5 v5.16.2 h1:fT6ZIOjE5iEnkzKyxTHK1W4HGAsPhqEqiSAssSO77hM=
 github.com/go-git/go-git/v5 v5.16.2/go.mod h1:4Ge4alE/5gPs30F2H1esi2gPd69R0C39lolkucHBOp8=
-github.com/go-jose/go-jose/v3 v3.0.4 h1:Wp5HA7bLQcKnf6YYao/4kpRpVMp/yf6+pJKV8WFSaNY=
+github.com/go-jose/go-jose/v3 v3.0.5 h1:BLLJWbC4nMZOfuPVxoZIxeYsn6Nl2r1fITaJ78UQlVQ=
-github.com/go-jose/go-jose/v3 v3.0.4/go.mod h1:5b+7YgP7ZICgJDBdfjZaIt+H/9L9T/YQrVfLAMboGkQ=
+github.com/go-jose/go-jose/v3 v3.0.5/go.mod h1:5b+7YgP7ZICgJDBdfjZaIt+H/9L9T/YQrVfLAMboGkQ=
 github.com/go-logr/logr v1.2.2/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A=
 github.com/go-logr/logr v1.4.3 h1:CjnDlHq8ikf6E492q6eKboGOC0T8CDaOvkHCIg8idEI=
 github.com/go-logr/logr v1.4.3/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY=

internal/honeypot/naive/naive.go

@@ -5,6 +5,7 @@ import (
 	_ "embed"
 	"fmt"
 	"log/slog"
+	"math"
 	"math/rand/v2"
 	"net/http"
 	"net/netip"
@@ -168,6 +169,9 @@ func (i *Impl) ServeHTTP(w http.ResponseWriter, r *http.Request) {
 		}
 	}
 
+	millisecondAmount := math.Pow(float64(networkCount), 2)
+	time.Sleep(time.Duration(millisecondAmount) * time.Millisecond)
+
 	spins := i.makeSpins()
 	affirmations := i.makeAffirmations()
 	title := i.makeTitle()

internal/test/playwright_test.go

@@ -595,7 +595,7 @@ func spawnAnubisWithOptions(t *testing.T, basePrefix string) string {
 		fmt.Fprintf(w, "<html><body><span id=anubis-test>%d</span></body></html>", time.Now().Unix())
 	})
 
-	policy, err := libanubis.LoadPoliciesOrDefault(t.Context(), "", anubis.DefaultDifficulty, "info")
+	policy, err := libanubis.LoadPoliciesOrDefault(t.Context(), "", anubis.DefaultDifficulty, "info", false)
 	if err != nil {
 		t.Fatal(err)
 	}

lib/anubis.go

@@ -186,7 +186,7 @@ func (s *Server) issueChallenge(ctx context.Context, r *http.Request, lg *slog.L
 		return nil, err
 	}
 
-	lg.Info("new challenge issued", "challenge", id.String())
+	lg.Info("new challenge issued", "challenge", id.String(), "weight", cr.Weight)
 
 	return &chall, err
 }

lib/anubis_test.go

@@ -58,7 +58,7 @@ func loadPolicies(t *testing.T, fname string, difficulty int) *policy.ParsedConf
 
 	t.Logf("loading policy file: %s", fname)
 
-	anubisPolicy, err := LoadPoliciesOrDefault(ctx, fname, difficulty, "info")
+	anubisPolicy, err := LoadPoliciesOrDefault(ctx, fname, difficulty, "info", false)
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -250,7 +250,7 @@ func TestLoadPolicies(t *testing.T) {
 			}
 			defer fin.Close()
 
-			if _, err := policy.ParseConfig(t.Context(), fin, fname, 4, "info"); err != nil {
+			if _, err := policy.ParseConfig(t.Context(), fin, fname, 4, "info", false); err != nil {
 				t.Fatal(err)
 			}
 		})

lib/config.go

@@ -55,7 +55,7 @@ type Options struct {
 	DifficultyInJWT          bool
 }
 
-func LoadPoliciesOrDefault(ctx context.Context, fname string, defaultDifficulty int, logLevel string) (*policy.ParsedConfig, error) {
+func LoadPoliciesOrDefault(ctx context.Context, fname string, defaultDifficulty int, logLevel string, subrequestMode bool) (*policy.ParsedConfig, error) {
 	var fin io.ReadCloser
 	var err error
 
@@ -79,7 +79,7 @@ func LoadPoliciesOrDefault(ctx context.Context, fname string, defaultDifficulty
 		}
 	}(fin)
 
-	anubisPolicy, err := policy.ParseConfig(ctx, fin, fname, defaultDifficulty, logLevel)
+	anubisPolicy, err := policy.ParseConfig(ctx, fin, fname, defaultDifficulty, logLevel, subrequestMode)
 	if err != nil {
 		return nil, fmt.Errorf("can't parse policy file %s: %w", fname, err)
 	}

lib/config_test.go

@@ -12,7 +12,7 @@ import (
 )
 
 func TestInvalidChallengeMethod(t *testing.T) {
-	if _, err := LoadPoliciesOrDefault(t.Context(), "testdata/invalid-challenge-method.yaml", 4, "info"); !errors.Is(err, policy.ErrChallengeRuleHasWrongAlgorithm) {
+	if _, err := LoadPoliciesOrDefault(t.Context(), "testdata/invalid-challenge-method.yaml", 4, "info", false); !errors.Is(err, policy.ErrChallengeRuleHasWrongAlgorithm) {
 		t.Fatalf("wanted error %v but got %v", policy.ErrChallengeRuleHasWrongAlgorithm, err)
 	}
 }
@@ -25,7 +25,7 @@ func TestBadConfigs(t *testing.T) {
 
 	for _, st := range finfos {
 		t.Run(st.Name(), func(t *testing.T) {
-			if _, err := LoadPoliciesOrDefault(t.Context(), filepath.Join("config", "testdata", "bad", st.Name()), anubis.DefaultDifficulty, "info"); err == nil {
+			if _, err := LoadPoliciesOrDefault(t.Context(), filepath.Join("config", "testdata", "bad", st.Name()), anubis.DefaultDifficulty, "info", false); err == nil {
 				t.Fatal(err)
 			} else {
 				t.Log(err)
@@ -44,13 +44,13 @@ func TestGoodConfigs(t *testing.T) {
 		t.Run(st.Name(), func(t *testing.T) {
 			t.Run("with-thoth", func(t *testing.T) {
 				ctx := thothmock.WithMockThoth(t)
-				if _, err := LoadPoliciesOrDefault(ctx, filepath.Join("config", "testdata", "good", st.Name()), anubis.DefaultDifficulty, "info"); err != nil {
+				if _, err := LoadPoliciesOrDefault(ctx, filepath.Join("config", "testdata", "good", st.Name()), anubis.DefaultDifficulty, "info", false); err != nil {
 					t.Fatal(err)
 				}
 			})
 
 			t.Run("without-thoth", func(t *testing.T) {
-				if _, err := LoadPoliciesOrDefault(t.Context(), filepath.Join("config", "testdata", "good", st.Name()), anubis.DefaultDifficulty, "info"); err != nil {
+				if _, err := LoadPoliciesOrDefault(t.Context(), filepath.Join("config", "testdata", "good", st.Name()), anubis.DefaultDifficulty, "info", false); err != nil {
 					t.Fatal(err)
 				}
 			})

lib/policy/checker.go

@@ -94,23 +94,26 @@ func (hmc *HeaderMatchesChecker) Hash() string {
 }
 
 type PathChecker struct {
-	regexp *regexp.Regexp
+	regexp         *regexp.Regexp
-	hash   string
+	hash           string
+	subRequestMode bool
 }
 
-func NewPathChecker(rexStr string) (checker.Impl, error) {
+func NewPathChecker(rexStr string, subrequestMode bool) (checker.Impl, error) {
 	rex, err := regexp.Compile(strings.TrimSpace(rexStr))
 	if err != nil {
 		return nil, fmt.Errorf("%w: regex %s failed parse: %w", ErrMisconfiguration, rexStr, err)
 	}
-	return &PathChecker{rex, internal.FastHash(rexStr)}, nil
+	return &PathChecker{rex, internal.FastHash(rexStr), subrequestMode}, nil
 }
 
 func (pc *PathChecker) Check(r *http.Request) (bool, error) {
-	originalUrl := r.Header.Get("X-Original-URI")
+	if pc.subRequestMode {
-	if originalUrl != "" {
+		originalUrl := r.Header.Get("X-Original-URI")
-		if pc.regexp.MatchString(originalUrl) {
+		if originalUrl != "" {
-			return true, nil
+			if pc.regexp.MatchString(originalUrl) {
+				return true, nil
+			}
 		}
 	}
 

lib/policy/checker_test.go

@@ -272,8 +272,8 @@ func TestPathChecker_XOriginalURI(t *testing.T) {
 
 	for _, tt := range tests {
 		t.Run(tt.name, func(t *testing.T) {
-			// Create the PathChecker
+			// Create the PathChecker in subrequest mode so X-Original-URI is honored.
-			pc, err := NewPathChecker(tt.regex)
+			pc, err := NewPathChecker(tt.regex, true)
 			if err != nil {
 				if !tt.expectError {
 					t.Fatalf("NewPathChecker() unexpected error: %v", err)
@@ -305,3 +305,108 @@ func TestPathChecker_XOriginalURI(t *testing.T) {
 		})
 	}
 }
+
+// TestPathChecker_GHSA_6wcg_mqvh_fcvg is a regression test for
+// https://github.com/TecharoHQ/anubis/security/advisories/GHSA-6wcg-mqvh-fcvg.
+//
+// PR https://github.com/TecharoHQ/anubis/pull/1015 added the ability for
+// reverse proxies using Anubis in subrequest auth mode to look at the path
+// of a request as there are many rules in the wild that rely on checking
+// the path. This is how access to things like robots.txt or anything in the
+// .well-known directory is unaffected by Anubis.
+//
+// However this logic was also enabled for non-subrequest deployments of Anubis,
+// meaning that a specially crafted request could include a /.well-known/
+// path in it and then get around Anubis with little effort.
+//
+// This fix gates the logic behind a new plumbed variable named subrequestMode
+// that only fires when Anubis is running in subrequest auth mode. This
+// properly contains that workaround so that the logic does not fire in
+// most deployments.
+func TestPathChecker_GHSA_6wcg_mqvh_fcvg(t *testing.T) {
+	tests := []struct {
+		name           string
+		regex          string
+		urlPath        string
+		xOriginalURI   string
+		subRequestMode bool
+		want           bool
+	}{
+		{
+			name:           "default mode ignores spoofed X-Original-URI when real path matches",
+			regex:          "^/admin/.*",
+			urlPath:        "/admin/secret",
+			xOriginalURI:   "/public/index",
+			subRequestMode: false,
+			want:           true,
+		},
+		{
+			name:           "default mode ignores spoofed X-Original-URI when real path does not match",
+			regex:          "^/admin/.*",
+			urlPath:        "/public/index",
+			xOriginalURI:   "/admin/secret",
+			subRequestMode: false,
+			want:           false,
+		},
+		{
+			name:           "default mode without X-Original-URI matches real path",
+			regex:          "^/admin/.*",
+			urlPath:        "/admin/dashboard",
+			xOriginalURI:   "",
+			subRequestMode: false,
+			want:           true,
+		},
+		{
+			name:           "subrequest mode honors X-Original-URI",
+			regex:          "^/admin/.*",
+			urlPath:        "/auth",
+			xOriginalURI:   "/admin/secret",
+			subRequestMode: true,
+			want:           true,
+		},
+		{
+			name:           "subrequest mode falls back to URL.Path when X-Original-URI does not match",
+			regex:          "^/admin/.*",
+			urlPath:        "/admin/dashboard",
+			xOriginalURI:   "/public/index",
+			subRequestMode: true,
+			want:           true,
+		},
+		{
+			name:           "subrequest mode with empty X-Original-URI uses URL.Path",
+			regex:          "^/admin/.*",
+			urlPath:        "/admin/dashboard",
+			xOriginalURI:   "",
+			subRequestMode: true,
+			want:           true,
+		},
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			pc, err := NewPathChecker(tt.regex, tt.subRequestMode)
+			if err != nil {
+				t.Fatalf("NewPathChecker(%q, %v) returned error: %v", tt.regex, tt.subRequestMode, err)
+			}
+
+			req, err := http.NewRequest(http.MethodGet, "http://example.com"+tt.urlPath, nil)
+			if err != nil {
+				t.Fatalf("http.NewRequest: %v", err)
+			}
+
+			if tt.xOriginalURI != "" {
+				req.Header.Set("X-Original-URI", tt.xOriginalURI)
+			}
+
+			got, err := pc.Check(req)
+			if err != nil {
+				t.Fatalf("Check() unexpected error: %v", err)
+			}
+
+			if got != tt.want {
+				t.Errorf("Check() = %v, want %v (subRequestMode=%v, urlPath=%q, X-Original-URI=%q)",
+					got, tt.want, tt.subRequestMode, tt.urlPath, tt.xOriginalURI)
+			}
+		})
+	}
+}

lib/policy/policy.go

@@ -60,7 +60,7 @@ func newParsedConfig(orig *config.Config) *ParsedConfig {
 	}
 }
 
-func ParseConfig(ctx context.Context, fin io.Reader, fname string, defaultDifficulty int, logLevel string) (*ParsedConfig, error) {
+func ParseConfig(ctx context.Context, fin io.Reader, fname string, defaultDifficulty int, logLevel string, subrequestMode bool) (*ParsedConfig, error) {
 	c, err := config.Load(fin, fname)
 	if err != nil {
 		return nil, err
@@ -152,7 +152,7 @@ func ParseConfig(ctx context.Context, fin io.Reader, fname string, defaultDiffic
 		}
 
 		if b.PathRegex != nil {
-			c, err := NewPathChecker(*b.PathRegex)
+			c, err := NewPathChecker(*b.PathRegex, subrequestMode)
 			if err != nil {
 				validationErrs = append(validationErrs, fmt.Errorf("while processing rule %s path regex: %w", b.Name, err))
 			} else {

lib/policy/policy_test.go

@@ -19,7 +19,7 @@ func TestDefaultPolicyMustParse(t *testing.T) {
 	}
 	defer fin.Close()
 
-	if _, err := ParseConfig(ctx, fin, "botPolicies.yaml", anubis.DefaultDifficulty, "info"); err != nil {
+	if _, err := ParseConfig(ctx, fin, "botPolicies.yaml", anubis.DefaultDifficulty, "info", false); err != nil {
 		t.Fatalf("can't parse config: %v", err)
 	}
 }
@@ -41,7 +41,7 @@ func TestGoodConfigs(t *testing.T) {
 				defer fin.Close()
 
 				ctx := thothmock.WithMockThoth(t)
-				if _, err := ParseConfig(ctx, fin, fin.Name(), anubis.DefaultDifficulty, "info"); err != nil {
+				if _, err := ParseConfig(ctx, fin, fin.Name(), anubis.DefaultDifficulty, "info", false); err != nil {
 					t.Fatal(err)
 				}
 			})
@@ -53,7 +53,7 @@ func TestGoodConfigs(t *testing.T) {
 				}
 				defer fin.Close()
 
-				if _, err := ParseConfig(t.Context(), fin, fin.Name(), anubis.DefaultDifficulty, "info"); err != nil {
+				if _, err := ParseConfig(t.Context(), fin, fin.Name(), anubis.DefaultDifficulty, "info", false); err != nil {
 					t.Fatal(err)
 				}
 			})
@@ -77,7 +77,7 @@ func TestBadConfigs(t *testing.T) {
 			}
 			defer fin.Close()
 
-			if _, err := ParseConfig(ctx, fin, fin.Name(), anubis.DefaultDifficulty, "info"); err == nil {
+			if _, err := ParseConfig(ctx, fin, fin.Name(), anubis.DefaultDifficulty, "info", false); err == nil {
 				t.Fatal(err)
 			} else {
 				t.Log(err)

package.json

@@ -20,11 +20,11 @@
   "author": "",
   "license": "ISC",
   "devDependencies": {
-    "@commitlint/cli": "^20.5.0",
+    "@commitlint/cli": "^20.5.3",
-    "@commitlint/config-conventional": "^20.5.0",
+    "@commitlint/config-conventional": "^20.5.3",
-    "baseline-browser-mapping": "^2.10.15",
+    "baseline-browser-mapping": "^2.10.27",
-    "cssnano": "^7.1.4",
+    "cssnano": "^7.1.8",
-    "cssnano-preset-advanced": "^7.0.12",
+    "cssnano-preset-advanced": "^7.0.16",
     "esbuild": "^0.28.0",
     "husky": "^9.1.7",
     "playwright": "^1.52.0",
@@ -32,7 +32,7 @@
     "postcss-import": "^16.1.1",
     "postcss-import-url": "^7.2.0",
     "postcss-url": "^10.1.3",
-    "prettier": "^3.8.1"
+    "prettier": "^3.8.3"
   },
   "dependencies": {
     "@aws-crypto/sha256-js": "^5.2.0",