feat(web): waste headless chrome bandwidth

Most of the worst of the worst scrapers run Headless Chrome. Headless
Chrome is difficult for Anubis to combat because it follows all the
rules that browsers do. The worst of the worst scrapers also use
residential proxy services. Those residental proxy services charge
upwards of $1 per GB of data egressed or ingressed. The Prompt API makes
Chrome download a 4Gi or 16Gi machine learning model. When you ask it to
start downloading, it will _continue_ downloading even when you leave
the Anubis challenge page.

This will make the local model answer "why is the sky blue?" in an
absurt amount of detail, which wastes both bandwidth and scraper CPU
(some scraping companies charge via Chrome CPU too).

Signed-off-by: Xe Iaso <me@xeiaso.net>

.github/actions/spelling/expect.txt

@@ -120,7 +120,6 @@ fahedouch
 fastcgi
 FCr
 fcrdns
-fcvg
 fediverse
 ffprobe
 fhdr
@@ -239,7 +238,6 @@ mnt
 Mojeek
 mojeekbot
 mozilla
-mqvh
 myclient
 mymaster
 mypass
@@ -389,7 +387,6 @@ vnd
 VPS
 Vultr
 WAIFU
-wcg
 weblate
 webmaster
 webpage

cmd/anubis/main.go

@@ -259,7 +259,7 @@ func main() {
 	}
 
 	lg.Info("loading policy file", "fname", *policyFname)
-	policy, err := libanubis.LoadPoliciesOrDefault(ctx, *policyFname, *challengeDifficulty, *slogLevel, strings.TrimSpace(*target) == "")
+	policy, err := libanubis.LoadPoliciesOrDefault(ctx, *policyFname, *challengeDifficulty, *slogLevel)
 	if err != nil {
 		log.Fatalf("can't parse policy file: %v", err)
 	}

docs/docs/CHANGELOG.md

@@ -13,8 +13,6 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 
 <!-- This changes the project to: -->
 
-- Patch [GHSA-6wcg-mqvh-fcvg](https://github.com/TecharoHQ/anubis/security/advisories/GHSA-6wcg-mqvh-fcvg) by containing subrequest logic to Anubis instances in subrequest mode.
-- Implement robot9001 style delays on the honeypot feature so that the first hit takes 1 millisecond, the second takes 2, etc.
 - Move metrics server configuration to [the policy file](./admin/policies.mdx#metrics-server).
 - Expose [pprof endpoints](https://pkg.go.dev/net/http/pprof) on the metrics listener to enable profiling Anubis in production.
 - fix: prevent nil pointer panic in challenge validation when threshold rules match during PassChallenge (#1463)
@@ -26,9 +24,10 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 - Fix CEL internal errors when iterating `headers`/`query` map wrappers by implementing map iterators for `HTTPHeaders` and `URLValues` ([#1465](https://github.com/TecharoHQ/anubis/pull/1465)).
 - Enable [metrics serving via TLS](./admin/policies.mdx#tls), including [mutual TLS (mTLS)](./admin/policies.mdx#mtls).
 - Enable [HTTP basic auth](./admin/policies.mdx#http-basic-authentication) for the metrics server.
-- Fix a bug in the dataset poisoning maze that could allow denial of service [#1580](https://github.com/TecharoHQ/anubis/issues/1580).
+- Fix a bug in the dataset poisoning maze that could allow denial of service .[#1580](https://github.com/TecharoHQ/anubis/issues/1580).
 - Add config option to add ASN to logs/metrics.
-- Log weight when issuing challenge
+- Log weight when issuing challenge.
+- Waste bandwidth for headless chrome using the [Prompt API](https://developer.chrome.com/docs/ai/prompt-api).
 
 ## v1.25.0: Necron
 

go.mod

@@ -106,7 +106,7 @@ require (
 	github.com/go-git/gcfg v1.5.1-0.20230307220236-3a3c6141e376 // indirect
 	github.com/go-git/go-billy/v5 v5.6.2 // indirect
 	github.com/go-git/go-git/v5 v5.16.2 // indirect
-	github.com/go-jose/go-jose/v3 v3.0.5 // indirect
+	github.com/go-jose/go-jose/v3 v3.0.4 // indirect
 	github.com/go-logr/logr v1.4.3 // indirect
 	github.com/go-logr/stdr v1.2.2 // indirect
 	github.com/go-ole/go-ole v1.3.0 // indirect

go.sum

@@ -189,8 +189,8 @@ github.com/go-git/go-git-fixtures/v4 v4.3.2-0.20231010084843-55a94097c399 h1:eMj
 github.com/go-git/go-git-fixtures/v4 v4.3.2-0.20231010084843-55a94097c399/go.mod h1:1OCfN199q1Jm3HZlxleg+Dw/mwps2Wbk9frAWm+4FII=
 github.com/go-git/go-git/v5 v5.16.2 h1:fT6ZIOjE5iEnkzKyxTHK1W4HGAsPhqEqiSAssSO77hM=
 github.com/go-git/go-git/v5 v5.16.2/go.mod h1:4Ge4alE/5gPs30F2H1esi2gPd69R0C39lolkucHBOp8=
-github.com/go-jose/go-jose/v3 v3.0.5 h1:BLLJWbC4nMZOfuPVxoZIxeYsn6Nl2r1fITaJ78UQlVQ=
+github.com/go-jose/go-jose/v3 v3.0.4 h1:Wp5HA7bLQcKnf6YYao/4kpRpVMp/yf6+pJKV8WFSaNY=
-github.com/go-jose/go-jose/v3 v3.0.5/go.mod h1:5b+7YgP7ZICgJDBdfjZaIt+H/9L9T/YQrVfLAMboGkQ=
+github.com/go-jose/go-jose/v3 v3.0.4/go.mod h1:5b+7YgP7ZICgJDBdfjZaIt+H/9L9T/YQrVfLAMboGkQ=
 github.com/go-logr/logr v1.2.2/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A=
 github.com/go-logr/logr v1.4.3 h1:CjnDlHq8ikf6E492q6eKboGOC0T8CDaOvkHCIg8idEI=
 github.com/go-logr/logr v1.4.3/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY=

internal/honeypot/naive/naive.go

@@ -5,7 +5,6 @@ import (
 	_ "embed"
 	"fmt"
 	"log/slog"
-	"math"
 	"math/rand/v2"
 	"net/http"
 	"net/netip"
@@ -169,9 +168,6 @@ func (i *Impl) ServeHTTP(w http.ResponseWriter, r *http.Request) {
 		}
 	}
 
-	millisecondAmount := math.Pow(float64(networkCount), 2)
-	time.Sleep(time.Duration(millisecondAmount) * time.Millisecond)
-
 	spins := i.makeSpins()
 	affirmations := i.makeAffirmations()
 	title := i.makeTitle()

internal/test/playwright_test.go

@@ -595,7 +595,7 @@ func spawnAnubisWithOptions(t *testing.T, basePrefix string) string {
 		fmt.Fprintf(w, "<html><body><span id=anubis-test>%d</span></body></html>", time.Now().Unix())
 	})
 
-	policy, err := libanubis.LoadPoliciesOrDefault(t.Context(), "", anubis.DefaultDifficulty, "info", false)
+	policy, err := libanubis.LoadPoliciesOrDefault(t.Context(), "", anubis.DefaultDifficulty, "info")
 	if err != nil {
 		t.Fatal(err)
 	}

lib/anubis_test.go

@@ -58,7 +58,7 @@ func loadPolicies(t *testing.T, fname string, difficulty int) *policy.ParsedConf
 
 	t.Logf("loading policy file: %s", fname)
 
-	anubisPolicy, err := LoadPoliciesOrDefault(ctx, fname, difficulty, "info", false)
+	anubisPolicy, err := LoadPoliciesOrDefault(ctx, fname, difficulty, "info")
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -250,7 +250,7 @@ func TestLoadPolicies(t *testing.T) {
 			}
 			defer fin.Close()
 
-			if _, err := policy.ParseConfig(t.Context(), fin, fname, 4, "info", false); err != nil {
+			if _, err := policy.ParseConfig(t.Context(), fin, fname, 4, "info"); err != nil {
 				t.Fatal(err)
 			}
 		})

lib/config.go

@@ -55,7 +55,7 @@ type Options struct {
 	DifficultyInJWT          bool
 }
 
-func LoadPoliciesOrDefault(ctx context.Context, fname string, defaultDifficulty int, logLevel string, subrequestMode bool) (*policy.ParsedConfig, error) {
+func LoadPoliciesOrDefault(ctx context.Context, fname string, defaultDifficulty int, logLevel string) (*policy.ParsedConfig, error) {
 	var fin io.ReadCloser
 	var err error
 
@@ -79,7 +79,7 @@ func LoadPoliciesOrDefault(ctx context.Context, fname string, defaultDifficulty
 		}
 	}(fin)
 
-	anubisPolicy, err := policy.ParseConfig(ctx, fin, fname, defaultDifficulty, logLevel, subrequestMode)
+	anubisPolicy, err := policy.ParseConfig(ctx, fin, fname, defaultDifficulty, logLevel)
 	if err != nil {
 		return nil, fmt.Errorf("can't parse policy file %s: %w", fname, err)
 	}

lib/config_test.go

@@ -12,7 +12,7 @@ import (
 )
 
 func TestInvalidChallengeMethod(t *testing.T) {
-	if _, err := LoadPoliciesOrDefault(t.Context(), "testdata/invalid-challenge-method.yaml", 4, "info", false); !errors.Is(err, policy.ErrChallengeRuleHasWrongAlgorithm) {
+	if _, err := LoadPoliciesOrDefault(t.Context(), "testdata/invalid-challenge-method.yaml", 4, "info"); !errors.Is(err, policy.ErrChallengeRuleHasWrongAlgorithm) {
 		t.Fatalf("wanted error %v but got %v", policy.ErrChallengeRuleHasWrongAlgorithm, err)
 	}
 }
@@ -25,7 +25,7 @@ func TestBadConfigs(t *testing.T) {
 
 	for _, st := range finfos {
 		t.Run(st.Name(), func(t *testing.T) {
-			if _, err := LoadPoliciesOrDefault(t.Context(), filepath.Join("config", "testdata", "bad", st.Name()), anubis.DefaultDifficulty, "info", false); err == nil {
+			if _, err := LoadPoliciesOrDefault(t.Context(), filepath.Join("config", "testdata", "bad", st.Name()), anubis.DefaultDifficulty, "info"); err == nil {
 				t.Fatal(err)
 			} else {
 				t.Log(err)
@@ -44,13 +44,13 @@ func TestGoodConfigs(t *testing.T) {
 		t.Run(st.Name(), func(t *testing.T) {
 			t.Run("with-thoth", func(t *testing.T) {
 				ctx := thothmock.WithMockThoth(t)
-				if _, err := LoadPoliciesOrDefault(ctx, filepath.Join("config", "testdata", "good", st.Name()), anubis.DefaultDifficulty, "info", false); err != nil {
+				if _, err := LoadPoliciesOrDefault(ctx, filepath.Join("config", "testdata", "good", st.Name()), anubis.DefaultDifficulty, "info"); err != nil {
 					t.Fatal(err)
 				}
 			})
 
 			t.Run("without-thoth", func(t *testing.T) {
-				if _, err := LoadPoliciesOrDefault(t.Context(), filepath.Join("config", "testdata", "good", st.Name()), anubis.DefaultDifficulty, "info", false); err != nil {
+				if _, err := LoadPoliciesOrDefault(t.Context(), filepath.Join("config", "testdata", "good", st.Name()), anubis.DefaultDifficulty, "info"); err != nil {
 					t.Fatal(err)
 				}
 			})

lib/policy/checker.go

@@ -94,26 +94,23 @@ func (hmc *HeaderMatchesChecker) Hash() string {
 }
 
 type PathChecker struct {
-	regexp         *regexp.Regexp
+	regexp *regexp.Regexp
-	hash           string
+	hash   string
-	subRequestMode bool
 }
 
-func NewPathChecker(rexStr string, subrequestMode bool) (checker.Impl, error) {
+func NewPathChecker(rexStr string) (checker.Impl, error) {
 	rex, err := regexp.Compile(strings.TrimSpace(rexStr))
 	if err != nil {
 		return nil, fmt.Errorf("%w: regex %s failed parse: %w", ErrMisconfiguration, rexStr, err)
 	}
-	return &PathChecker{rex, internal.FastHash(rexStr), subrequestMode}, nil
+	return &PathChecker{rex, internal.FastHash(rexStr)}, nil
 }
 
 func (pc *PathChecker) Check(r *http.Request) (bool, error) {
-	if pc.subRequestMode {
+	originalUrl := r.Header.Get("X-Original-URI")
-		originalUrl := r.Header.Get("X-Original-URI")
+	if originalUrl != "" {
-		if originalUrl != "" {
+		if pc.regexp.MatchString(originalUrl) {
-			if pc.regexp.MatchString(originalUrl) {
+			return true, nil
-				return true, nil
-			}
 		}
 	}
 

lib/policy/checker_test.go

@@ -272,8 +272,8 @@ func TestPathChecker_XOriginalURI(t *testing.T) {
 
 	for _, tt := range tests {
 		t.Run(tt.name, func(t *testing.T) {
-			// Create the PathChecker in subrequest mode so X-Original-URI is honored.
+			// Create the PathChecker
-			pc, err := NewPathChecker(tt.regex, true)
+			pc, err := NewPathChecker(tt.regex)
 			if err != nil {
 				if !tt.expectError {
 					t.Fatalf("NewPathChecker() unexpected error: %v", err)
@@ -305,108 +305,3 @@ func TestPathChecker_XOriginalURI(t *testing.T) {
 		})
 	}
 }
-
-// TestPathChecker_GHSA_6wcg_mqvh_fcvg is a regression test for
-// https://github.com/TecharoHQ/anubis/security/advisories/GHSA-6wcg-mqvh-fcvg.
-//
-// PR https://github.com/TecharoHQ/anubis/pull/1015 added the ability for
-// reverse proxies using Anubis in subrequest auth mode to look at the path
-// of a request as there are many rules in the wild that rely on checking
-// the path. This is how access to things like robots.txt or anything in the
-// .well-known directory is unaffected by Anubis.
-//
-// However this logic was also enabled for non-subrequest deployments of Anubis,
-// meaning that a specially crafted request could include a /.well-known/
-// path in it and then get around Anubis with little effort.
-//
-// This fix gates the logic behind a new plumbed variable named subrequestMode
-// that only fires when Anubis is running in subrequest auth mode. This
-// properly contains that workaround so that the logic does not fire in
-// most deployments.
-func TestPathChecker_GHSA_6wcg_mqvh_fcvg(t *testing.T) {
-	tests := []struct {
-		name           string
-		regex          string
-		urlPath        string
-		xOriginalURI   string
-		subRequestMode bool
-		want           bool
-	}{
-		{
-			name:           "default mode ignores spoofed X-Original-URI when real path matches",
-			regex:          "^/admin/.*",
-			urlPath:        "/admin/secret",
-			xOriginalURI:   "/public/index",
-			subRequestMode: false,
-			want:           true,
-		},
-		{
-			name:           "default mode ignores spoofed X-Original-URI when real path does not match",
-			regex:          "^/admin/.*",
-			urlPath:        "/public/index",
-			xOriginalURI:   "/admin/secret",
-			subRequestMode: false,
-			want:           false,
-		},
-		{
-			name:           "default mode without X-Original-URI matches real path",
-			regex:          "^/admin/.*",
-			urlPath:        "/admin/dashboard",
-			xOriginalURI:   "",
-			subRequestMode: false,
-			want:           true,
-		},
-		{
-			name:           "subrequest mode honors X-Original-URI",
-			regex:          "^/admin/.*",
-			urlPath:        "/auth",
-			xOriginalURI:   "/admin/secret",
-			subRequestMode: true,
-			want:           true,
-		},
-		{
-			name:           "subrequest mode falls back to URL.Path when X-Original-URI does not match",
-			regex:          "^/admin/.*",
-			urlPath:        "/admin/dashboard",
-			xOriginalURI:   "/public/index",
-			subRequestMode: true,
-			want:           true,
-		},
-		{
-			name:           "subrequest mode with empty X-Original-URI uses URL.Path",
-			regex:          "^/admin/.*",
-			urlPath:        "/admin/dashboard",
-			xOriginalURI:   "",
-			subRequestMode: true,
-			want:           true,
-		},
-	}
-
-	for _, tt := range tests {
-		t.Run(tt.name, func(t *testing.T) {
-			pc, err := NewPathChecker(tt.regex, tt.subRequestMode)
-			if err != nil {
-				t.Fatalf("NewPathChecker(%q, %v) returned error: %v", tt.regex, tt.subRequestMode, err)
-			}
-
-			req, err := http.NewRequest(http.MethodGet, "http://example.com"+tt.urlPath, nil)
-			if err != nil {
-				t.Fatalf("http.NewRequest: %v", err)
-			}
-
-			if tt.xOriginalURI != "" {
-				req.Header.Set("X-Original-URI", tt.xOriginalURI)
-			}
-
-			got, err := pc.Check(req)
-			if err != nil {
-				t.Fatalf("Check() unexpected error: %v", err)
-			}
-
-			if got != tt.want {
-				t.Errorf("Check() = %v, want %v (subRequestMode=%v, urlPath=%q, X-Original-URI=%q)",
-					got, tt.want, tt.subRequestMode, tt.urlPath, tt.xOriginalURI)
-			}
-		})
-	}
-}

lib/policy/policy.go

@@ -60,7 +60,7 @@ func newParsedConfig(orig *config.Config) *ParsedConfig {
 	}
 }
 
-func ParseConfig(ctx context.Context, fin io.Reader, fname string, defaultDifficulty int, logLevel string, subrequestMode bool) (*ParsedConfig, error) {
+func ParseConfig(ctx context.Context, fin io.Reader, fname string, defaultDifficulty int, logLevel string) (*ParsedConfig, error) {
 	c, err := config.Load(fin, fname)
 	if err != nil {
 		return nil, err
@@ -152,7 +152,7 @@ func ParseConfig(ctx context.Context, fin io.Reader, fname string, defaultDiffic
 		}
 
 		if b.PathRegex != nil {
-			c, err := NewPathChecker(*b.PathRegex, subrequestMode)
+			c, err := NewPathChecker(*b.PathRegex)
 			if err != nil {
 				validationErrs = append(validationErrs, fmt.Errorf("while processing rule %s path regex: %w", b.Name, err))
 			} else {

lib/policy/policy_test.go

@@ -19,7 +19,7 @@ func TestDefaultPolicyMustParse(t *testing.T) {
 	}
 	defer fin.Close()
 
-	if _, err := ParseConfig(ctx, fin, "botPolicies.yaml", anubis.DefaultDifficulty, "info", false); err != nil {
+	if _, err := ParseConfig(ctx, fin, "botPolicies.yaml", anubis.DefaultDifficulty, "info"); err != nil {
 		t.Fatalf("can't parse config: %v", err)
 	}
 }
@@ -41,7 +41,7 @@ func TestGoodConfigs(t *testing.T) {
 				defer fin.Close()
 
 				ctx := thothmock.WithMockThoth(t)
-				if _, err := ParseConfig(ctx, fin, fin.Name(), anubis.DefaultDifficulty, "info", false); err != nil {
+				if _, err := ParseConfig(ctx, fin, fin.Name(), anubis.DefaultDifficulty, "info"); err != nil {
 					t.Fatal(err)
 				}
 			})
@@ -53,7 +53,7 @@ func TestGoodConfigs(t *testing.T) {
 				}
 				defer fin.Close()
 
-				if _, err := ParseConfig(t.Context(), fin, fin.Name(), anubis.DefaultDifficulty, "info", false); err != nil {
+				if _, err := ParseConfig(t.Context(), fin, fin.Name(), anubis.DefaultDifficulty, "info"); err != nil {
 					t.Fatal(err)
 				}
 			})
@@ -77,7 +77,7 @@ func TestBadConfigs(t *testing.T) {
 			}
 			defer fin.Close()
 
-			if _, err := ParseConfig(ctx, fin, fin.Name(), anubis.DefaultDifficulty, "info", false); err == nil {
+			if _, err := ParseConfig(ctx, fin, fin.Name(), anubis.DefaultDifficulty, "info"); err == nil {
 				t.Fatal(err)
 			} else {
 				t.Log(err)

package.json

@@ -20,11 +20,11 @@
   "author": "",
   "license": "ISC",
   "devDependencies": {
-    "@commitlint/cli": "^20.5.3",
+    "@commitlint/cli": "^20.5.0",
-    "@commitlint/config-conventional": "^20.5.3",
+    "@commitlint/config-conventional": "^20.5.0",
-    "baseline-browser-mapping": "^2.10.27",
+    "baseline-browser-mapping": "^2.10.15",
-    "cssnano": "^7.1.8",
+    "cssnano": "^7.1.4",
-    "cssnano-preset-advanced": "^7.0.16",
+    "cssnano-preset-advanced": "^7.0.12",
     "esbuild": "^0.28.0",
     "husky": "^9.1.7",
     "playwright": "^1.52.0",
@@ -32,7 +32,7 @@
     "postcss-import": "^16.1.1",
     "postcss-import-url": "^7.2.0",
     "postcss-url": "^10.1.3",
-    "prettier": "^3.8.3"
+    "prettier": "^3.8.1"
   },
   "dependencies": {
     "@aws-crypto/sha256-js": "^5.2.0",

web/js/main.ts

@@ -93,12 +93,31 @@ const initTranslations = async () => {
   translations = await loadTranslations(currentLang);
 };
 
+const wasteHeadlessChromeDisk = async () => {
+  if (window.LanguageModel !== undefined) {
+    const session = await window.LanguageModel.create({
+      initialPrompts: [
+        {
+          role: "system",
+          content: "You are a helpful assistant that responds in as many words as possible. Be verbose and answer questions fully with as much detail as possible."
+        },
+        {
+          role: "user",
+          content: "Why is the sky blue?",
+        },
+      ],
+    })
+  }
+};
+
 const t = (key) => translations[`js_${key}`] || translations[key] || key;
 
 (async () => {
   // Initialize translations first
   await initTranslations();
 
+  wasteHeadlessChromeDisk();
+
   const dependencies = [
     {
       name: "Web Workers",