chore: spelling

Signed-off-by: Xe Iaso <me@xeiaso.net>

.dockerignore

@@ -1,25 +0,0 @@
-.env
-*.deb
-*.rpm
-
-# Additional package locks
-pnpm-lock.yaml
-yarn.lock
-
-# Go binaries and test artifacts
-main
-*.test
-
-node_modules
-
-# MacOS
-.DS_store
-
-# Intellij
-.idea
-
-# how does this get here
-doc/VERSION
-
-web/static/js/*
-!web/static/js/.gitignore

.github/actions/spelling/expect.txt

@@ -6,46 +6,42 @@ amazonbot
 anthro
 anubis
 anubistest
-Applebot
 archlinux
+asnc
+asnchecker
+asns
 badregexes
-bdba
 berr
 bingbot
-bitcoin
+Bitcoin
 blogging
 Bluesky
 blueskybot
 boi
 botnet
 BPort
-Brightbot
 broked
-Bytespider
 cachebuster
+cachediptoasn
 Caddyfile
 caninetools
 Cardyb
 celchecker
 CELPHASE
-cerr
 certresolver
 CGNAT
 cgr
 chainguard
 chall
 challengemozilla
-checkpath
 checkresult
 chen
 chibi
 cidranger
 ckie
 cloudflare
-confd
 containerbuild
 coreutils
-Cotoyogi
 CRDs
 crt
 daemonizing
@@ -54,7 +50,6 @@ Debian
 debrpm
 decaymap
 decompiling
-Diffbot
 discordapp
 discordbot
 distros
@@ -65,61 +60,57 @@ dracula
 dronebl
 droneblresponse
 duckduckbot
-eerror
 ellenjoe
 enbyware
 everyones
 evilbot
 evilsite
 expressionorlist
-externalagent
-externalfetcher
 extldflags
 facebookgo
-Factset
 fastcgi
 fediverse
 finfos
-Firecrawl
 flagenv
 Fordola
 forgejo
 fsys
 fullchain
+gaissmai
 Galvus
+geoip
+geoipchecker
 gha
+gipc
 gitea
+godotenv
 goland
 gomod
 goodbot
 googlebot
 govulncheck
 GPG
-GPT
-gptbot
+grpcprom
 grw
 Hashcash
 hashrate
 headermap
 healthcheck
-hebis
 hec
 hmc
 hostable
 htmx
 httpdebug
 hypertext
-iaskspider
 iat
 ifm
-Imagesift
-imgproxy
 inp
+IPTo
+iptoasn
 iss
-isset
 ivh
-Jenomis
 JGit
+joho
 journalctl
 jshelter
 JWTs
@@ -130,10 +121,8 @@ keypair
 KHTML
 kinda
 KUBECONFIG
-lcj
 ldflags
 letsencrypt
-Lexentale
 lgbt
 licend
 licstart
@@ -149,30 +138,19 @@ maintainership
 malware
 mcr
 memes
-metarefresh
-metrix
 mimi
 minica
-mistralai
 Mojeek
 mojeekbot
 mozilla
 nbf
-netsurf
 nginx
 nobots
 NONINFRINGEMENT
 nosleep
-OCOB
 ogtags
-omgili
-omgilibot
 onionservice
-openai
-openrc
 pag
-palemoon
-Pangu
 parseable
 passthrough
 Patreon
@@ -188,7 +166,6 @@ prebaked
 privkey
 promauto
 promhttp
-proofofwork
 pwcmd
 pwuser
 qualys
@@ -203,27 +180,20 @@ reputational
 reqmeta
 risc
 ruleset
-runlevels
 RUnlock
 sas
 sasl
 Scumm
-searchbot
 searx
 sebest
 secretplans
 selfsigned
-Semrush
 setsebool
-shellcheck
-Sidetrade
 sitemap
 sls
-sni
 Sourceware
 Spambot
 sparkline
-spyderbot
 srv
 stackoverflow
 startprecmd
@@ -231,25 +201,22 @@ stoppostcmd
 subgrid
 subr
 subrequest
-SVCNAME
 tagline
 tarballs
+tarrif
 techaro
 techarohq
 templ
 templruntime
 testarea
-Tik
-Timpibot
+thoth
+thothmock
 torproject
 traefik
-uberspace
 unixhttpd
 unmarshal
 uvx
-UXP
 Varis
-Velen
 vendored
 vhosts
 videotest
@@ -259,12 +226,8 @@ webmaster
 webpage
 websecure
 websites
-Webzio
-wildbase
-wordpress
 Workaround
 workdir
-wpbot
 xcaddy
 Xeact
 xeiaso

.github/actions/spelling/line_forbidden.patterns

@@ -20,6 +20,9 @@
 # https://twitter.com/nyttypos/status/1898844061873639490
 #\([A-Z][a-z]{2,}(?: [a-z]+){3,}\)\.\s
 
+# Complete sentences shouldn't be in the middle of another sentence as a parenthetical.
+(?<!\.)\.\),
+
 # Complete sentences in parentheticals should not have a space before the period.
 \s\.\)(?!.*\}\})
 

.github/actions/spelling/patterns.txt

@@ -128,7 +128,3 @@ go install(?:\s+[a-z]+\.[-@\w/.]+)+
 
 # ignore long runs of a single character:
 \b([A-Za-z])\g{-1}{3,}\b
-
-# hit-count: 1 file-count: 1
-# microsoft
-\b(?:https?://|)(?:(?:(?:blogs|download\.visualstudio|docs|msdn2?|research)\.|)microsoft|blogs\.msdn)\.co(?:m|\.\w\w)/[-_a-zA-Z0-9()=./%]*

.github/workflows/docs-deploy.yml

@@ -39,7 +39,7 @@ jobs:
 
       - name: Build and push
         id: build
-        uses: docker/build-push-action@263435318d21b8e681c14492fe198d362a7d2c83 # v6.18.0
+        uses: docker/build-push-action@1dc73863535b631f98b2378be8619f83b136f4a0 # v6.17.0
         with:
           context: ./docs
           cache-to: type=gha

.github/workflows/docs-test.yml

@@ -28,7 +28,7 @@ jobs:
 
       - name: Build and push
         id: build
-        uses: docker/build-push-action@263435318d21b8e681c14492fe198d362a7d2c83 # v6.18.0
+        uses: docker/build-push-action@1dc73863535b631f98b2378be8619f83b136f4a0 # v6.17.0
         with:
           context: ./docs
           cache-to: type=gha

.github/workflows/zizmor.yml

@@ -21,7 +21,7 @@ jobs:
           persist-credentials: false
 
       - name: Install the latest version of uv
-        uses: astral-sh/setup-uv@f0ec1fc3b38f5e7cd731bb6ce540c5af426746bb # v6.1.0
+        uses: astral-sh/setup-uv@6b9c6063abd6010835644d4c2e1bef4cf5cd0fca # v6.0.1
 
       - name: Run zizmor 🌈
         run: uvx zizmor --format sarif . > results.sarif 
@@ -29,7 +29,7 @@ jobs:
           GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} 
 
       - name: Upload SARIF file
-        uses: github/codeql-action/upload-sarif@fca7ace96b7d713c7035871441bd52efbe39e27e # v3.28.19
+        uses: github/codeql-action/upload-sarif@ff0a06e83cb2de871e5a09832bc6a81e7276941f # v3.28.18
         with:
           sarif_file: results.sarif
           category: zizmor

Dockerfile

@@ -1,30 +0,0 @@
-ARG ALPINE_VERSION=edge
-FROM --platform=${BUILDPLATFORM} alpine:${ALPINE_VERSION} AS build
-
-ARG TARGETOS
-ARG TARGETARCH
-ARG COMPONENT=anubis
-ARG VERSION=devel-docker
-
-RUN apk -U add go nodejs git build-base git npm bash zstd brotli gzip
-
-WORKDIR /app
-
-COPY go.mod go.sum ./
-RUN go mod download
-
-COPY . .
-RUN --mount=type=cache,target=/root/.cache npm ci && npm run assets
-RUN --mount=type=cache,target=/root/.cache GOOS=${TARGETOS} GOARCH=${TARGETARCH} CGO_ENABLED=0 GOARM=7 go build -gcflags "all=-N -l" -o /app/bin/${COMPONENT} -ldflags "-s -w -extldflags -static -X github.com/TecharoHQ/anubis.Version=${VERSION}" ./cmd/${COMPONENT}
-
-FROM alpine:${ALPINE_VERSION} AS run
-WORKDIR /app
-
-RUN apk -U add ca-certificates mailcap
-
-COPY --from=build /app/bin/anubis /app/bin/anubis
-
-CMD ["/app/bin/anubis"]
-HEALTHCHECK --interval=30s --timeout=30s --start-period=5s --retries=3 CMD [ "/app/bin/anubis", "--healthcheck" ]
-
-LABEL org.opencontainers.image.source="https://github.com/TecharoHQ/anubis"

README.md

@@ -14,36 +14,14 @@
 
 Anubis is brought to you by sponsors and donors like:
 
-### Diamond Tier
-
-<a href="https://www.raptorcs.com/content/base/products.html">
-  <img src="./docs/static/img/sponsors/raptor-computing-logo.webp" alt="Raptor Computing Systems" height=64 />
-</a>
-
-### Gold Tier
-
-<a href="https://distrust.co?utm_campaign=github&utm_medium=referral&utm_content=anubis">
-  <img src="./docs/static/img/sponsors/distrust-logo.webp" alt="Distrust" height="64">
-</a>
-<a href="https://terminaltrove.com/?utm_campaign=github&utm_medium=referral&utm_content=anubis&utm_source=abgh">
-  <img src="./docs/static/img/sponsors/terminal-trove.webp" alt="Terminal Trove" height="64">
-</a>
-<a href="https://canine.tools?utm_campaign=github&utm_medium=referral&utm_content=anubis">
-  <img src="./docs/static/img/sponsors/caninetools-logo.webp" alt="canine.tools" height="64">
-</a>
-<a href="https://weblate.org/">
-  <img src="./docs/static/img/sponsors/weblate-logo.webp" alt="Weblate" height="64">
-</a>
-<a href="https://uberspace.de/">
-  <img src="./docs/static/img/sponsors/uberspace-logo.webp" alt="Uberspace" height="64">
-</a>
-<a href="https://wildbase.xyz/">
-  <img src="./docs/static/img/sponsors/wildbase-logo.webp" alt="Wildbase" height="64">
-</a>
+[![Distrust](./docs/static/img/sponsors/distrust-logo.webp)](https://distrust.co?utm_campaign=github&utm_medium=referral&utm_content=anubis)
+[![Terminal Trove](./docs/static/img/sponsors/terminal-trove.webp)](https://terminaltrove.com/?utm_campaign=github&utm_medium=referral&utm_content=anubis&utm_source=abgh)
+[![canine.tools](./docs/static/img/sponsors/caninetools-logo.webp)](https://canine.tools?utm_campaign=github&utm_medium=referral&utm_content=anubis)
+[![Weblate](./docs/static/img/sponsors/weblate-logo.webp)](https://weblate.org/?utm_campaign=github&utm_medium=referral&utm_content=anubis)
 
 ## Overview
 
-Anubis is a Web AI Firewall Utility that [weighs the soul of your connection](https://en.wikipedia.org/wiki/Weighing_of_souls) using one or more challenges in order to protect upstream resources from scraper bots.
+Anubis [weighs the soul of your connection](https://en.wikipedia.org/wiki/Weighing_of_souls) using a proof-of-work challenge in order to protect upstream resources from scraper bots.
 
 This program is designed to help protect the small internet from the endless storm of requests that flood in from AI companies. Anubis is as lightweight as possible to ensure that everyone can afford to protect the communities closest to them.
 

VERSION

@@ -1 +1 @@
-1.19.1
+1.18.0

cmd/anubis/main.go

@@ -30,11 +30,13 @@ import (
 	"github.com/TecharoHQ/anubis"
 	"github.com/TecharoHQ/anubis/data"
 	"github.com/TecharoHQ/anubis/internal"
+	"github.com/TecharoHQ/anubis/internal/thoth"
 	libanubis "github.com/TecharoHQ/anubis/lib"
 	botPolicy "github.com/TecharoHQ/anubis/lib/policy"
 	"github.com/TecharoHQ/anubis/lib/policy/config"
 	"github.com/TecharoHQ/anubis/web"
 	"github.com/facebookgo/flagenv"
+	_ "github.com/joho/godotenv/autoload"
 	"github.com/prometheus/client_golang/prometheus/promhttp"
 )
 
@@ -56,7 +58,6 @@ var (
 	redirectDomains          = flag.String("redirect-domains", "", "list of domains separated by commas which anubis is allowed to redirect to. Leaving this unset allows any domain.")
 	slogLevel                = flag.String("slog-level", "INFO", "logging level (see https://pkg.go.dev/log/slog#hdr-Levels)")
 	target                   = flag.String("target", "http://localhost:3923", "target to reverse proxy to, set to an empty string to disable proxying when only using auth request")
-	targetSNI                = flag.String("target-sni", "", "if set, the value of the TLS handshake hostname when forwarding requests to the target")
 	targetHost               = flag.String("target-host", "", "if set, the value of the Host header when forwarding requests to the target")
 	targetInsecureSkipVerify = flag.Bool("target-insecure-skip-verify", false, "if true, skips TLS validation for the backend")
 	healthcheck              = flag.Bool("healthcheck", false, "run a health check against Anubis")
@@ -67,8 +68,9 @@ var (
 	ogCacheConsiderHost      = flag.Bool("og-cache-consider-host", false, "enable or disable the use of the host in the Open Graph tag cache")
 	extractResources         = flag.String("extract-resources", "", "if set, extract the static resources to the specified folder")
 	webmasterEmail           = flag.String("webmaster-email", "", "if set, displays webmaster's email on the reject page for appeals")
-	versionFlag              = flag.Bool("version", false, "print Anubis version")
-	xffStripPrivate          = flag.Bool("xff-strip-private", true, "if set, strip private addresses from X-Forwarded-For")
+
+	thothURL   = flag.String("thoth-url", "", "if set, URL for Thoth, the IP reputation database for Anubis")
+	thothToken = flag.String("thoth-token", "", "if set, API token for Thoth, the IP reputation database for Anubis")
 )
 
 func keyFromHex(value string) (ed25519.PrivateKey, error) {
@@ -139,7 +141,7 @@ func setupListener(network string, address string) (net.Listener, string) {
 	return listener, formattedAddress
 }
 
-func makeReverseProxy(target string, targetSNI string, targetHost string, insecureSkipVerify bool) (http.Handler, error) {
+func makeReverseProxy(target string, targetHost string, insecureSkipVerify bool) (http.Handler, error) {
 	targetUri, err := url.Parse(target)
 	if err != nil {
 		return nil, fmt.Errorf("failed to parse target URL: %w", err)
@@ -161,14 +163,10 @@ func makeReverseProxy(target string, targetSNI string, targetHost string, insecu
 		transport.RegisterProtocol("unix", libanubis.UnixRoundTripper{Transport: transport})
 	}
 
-	if insecureSkipVerify || targetSNI != "" {
-		transport.TLSClientConfig = &tls.Config{}
-		if insecureSkipVerify {
-			slog.Warn("TARGET_INSECURE_SKIP_VERIFY is set to true, TLS certificate validation will not be performed", "target", target)
-			transport.TLSClientConfig.InsecureSkipVerify = true
-		}
-		if targetSNI != "" {
-			transport.TLSClientConfig.ServerName = targetSNI
+	if insecureSkipVerify {
+		slog.Warn("TARGET_INSECURE_SKIP_VERIFY is set to true, TLS certificate validation will not be performed", "target", target)
+		transport.TLSClientConfig = &tls.Config{
+			InsecureSkipVerify: true,
 		}
 	}
 
@@ -204,11 +202,6 @@ func main() {
 	flagenv.Parse()
 	flag.Parse()
 
-	if *versionFlag {
-		fmt.Println("Anubis", anubis.Version)
-		return
-	}
-
 	internal.InitSlog(*slogLevel)
 
 	if *extractResources != "" {
@@ -226,13 +219,25 @@ func main() {
 	// when using anubis via Systemd and environment variables, then it is not possible to set targe to an empty string but only to space
 	if strings.TrimSpace(*target) != "" {
 		var err error
-		rp, err = makeReverseProxy(*target, *targetSNI, *targetHost, *targetInsecureSkipVerify)
+		rp, err = makeReverseProxy(*target, *targetHost, *targetInsecureSkipVerify)
 		if err != nil {
 			log.Fatalf("can't make reverse proxy: %v", err)
 		}
 	}
 
-	policy, err := libanubis.LoadPoliciesOrDefault(*policyFname, *challengeDifficulty)
+	ctx := context.Background()
+
+	if *thothURL != "" && *thothToken != "" {
+		slog.Debug("connecting to Thoth")
+		thothClient, err := thoth.New(ctx, *thothURL, *thothToken)
+		if err != nil {
+			log.Fatalf("can't dial thoth at %s: %v", *thothURL, err)
+		}
+
+		ctx = thoth.With(ctx, thothClient)
+	}
+
+	policy, err := libanubis.LoadPoliciesOrDefault(ctx, *policyFname, *challengeDifficulty)
 	if err != nil {
 		log.Fatalf("can't parse policy file: %v", err)
 	}
@@ -337,7 +342,7 @@ func main() {
 	h = s
 	h = internal.RemoteXRealIP(*useRemoteAddress, *bindNetwork, h)
 	h = internal.XForwardedForToXRealIP(h)
-	h = internal.XForwardedForUpdate(*xffStripPrivate, h)
+	h = internal.XForwardedForUpdate(h)
 
 	srv := http.Server{Handler: h, ErrorLog: internal.GetFilteredHTTPLogger()}
 	listener, listenerUrl := setupListener(*bindNetwork, *bind)
@@ -421,11 +426,11 @@ func extractEmbedFS(fsys embed.FS, root string, destDir string) error {
 			return os.MkdirAll(destPath, 0o700)
 		}
 
-		embeddedData, err := fs.ReadFile(fsys, path)
+		data, err := fs.ReadFile(fsys, path)
 		if err != nil {
 			return err
 		}
 
-		return os.WriteFile(destPath, embeddedData, 0o644)
+		return os.WriteFile(destPath, data, 0o644)
 	})
 }

data/botPolicies.json

@@ -4,7 +4,7 @@
       "import": "(data)/bots/_deny-pathological.yaml"
     },
     {
-      "import": "(data)/meta/ai-block-aggressive.yaml"
+      "import": "(data)/bots/ai-robots-txt.yaml"
     },
     {
       "import": "(data)/crawlers/_allow-good.yaml"

data/botPolicies.yaml

@@ -11,53 +11,44 @@
 ## /usr/share/docs/anubis/data or in the tarball you extracted Anubis from.
 
 bots:
-  # Pathological bots to deny
-  - # This correlates to data/bots/deny-pathological.yaml in the source tree
-    # https://github.com/TecharoHQ/anubis/blob/main/data/bots/deny-pathological.yaml
-    import: (data)/bots/_deny-pathological.yaml
-  - import: (data)/bots/aggressive-brazilian-scrapers.yaml
+# Pathological bots to deny
+- # This correlates to data/bots/deny-pathological.yaml in the source tree
+  # https://github.com/TecharoHQ/anubis/blob/main/data/bots/deny-pathological.yaml
+  import: (data)/bots/_deny-pathological.yaml
+- import: (data)/bots/aggressive-brazilian-scrapers.yaml
 
-  # Aggressively block AI/LLM related bots/agents by default
-  - import: (data)/meta/ai-block-aggressive.yaml
+# Enforce https://github.com/ai-robots-txt/ai.robots.txt
+- import: (data)/bots/ai-robots-txt.yaml
 
-  # Consider replacing the aggressive AI policy with more selective policies:
-  # - import: (data)/meta/ai-block-moderate.yaml
-  # - import: (data)/meta/ai-block-permissive.yaml
+# Search engine crawlers to allow, defaults to:
+#   - Google (so they don't try to bypass Anubis)
+#   - Bing
+#   - DuckDuckGo
+#   - Qwant
+#   - The Internet Archive
+#   - Kagi
+#   - Marginalia
+#   - Mojeek
+- import: (data)/crawlers/_allow-good.yaml
 
-  # Search engine crawlers to allow, defaults to:
-  #   - Google (so they don't try to bypass Anubis)
-  #   - Apple
-  #   - Bing
-  #   - DuckDuckGo
-  #   - Qwant
-  #   - The Internet Archive
-  #   - Kagi
-  #   - Marginalia
-  #   - Mojeek
-  - import: (data)/crawlers/_allow-good.yaml
-  # Challenge Firefox AI previews
-  - import: (data)/clients/x-firefox-ai.yaml
+# Allow common "keeping the internet working" routes (well-known, favicon, robots.txt)
+- import: (data)/common/keep-internet-working.yaml
 
-  # Allow common "keeping the internet working" routes (well-known, favicon, robots.txt)
-  - import: (data)/common/keep-internet-working.yaml
+# # Punish any bot with "bot" in the user-agent string
+# # This is known to have a high false-positive rate, use at your own risk
+# - name: generic-bot-catchall
+#   user_agent_regex: (?i:bot|crawler)
+#   action: CHALLENGE
+#   challenge:
+#     difficulty: 16  # impossible
+#     report_as: 4    # lie to the operator
+#     algorithm: slow # intentionally waste CPU cycles and time
 
-  # # Punish any bot with "bot" in the user-agent string
-  # # This is known to have a high false-positive rate, use at your own risk
-  # - name: generic-bot-catchall
-  #   user_agent_regex: (?i:bot|crawler)
-  #   action: CHALLENGE
-  #   challenge:
-  #     difficulty: 16  # impossible
-  #     report_as: 4    # lie to the operator
-  #     algorithm: slow # intentionally waste CPU cycles and time
-
-  # Generic catchall rule
-  - name: generic-browser
-    user_agent_regex: >-
-      Mozilla|Opera
-    action: WEIGH
-    weight:
-      adjust: 10
+# Generic catchall rule
+- name: generic-browser
+  user_agent_regex: >-
+    Mozilla|Opera
+  action: CHALLENGE
 
 dnsbl: false
 
@@ -67,4 +58,4 @@ dnsbl: false
 # will stop sending requests once they get it.
 status_codes:
   CHALLENGE: 200
-  DENY: 200
+  DENY: 200 

data/bots/aggressive-brazilian-scrapers.yaml

@@ -1,26 +1,28 @@
 - name: deny-aggressive-brazilian-scrapers
-  action: WEIGH
-  weight:
-    adjust: 20
+  action: DENY
   expression:
     any:
-      # Internet Explorer should be out of support
-      - userAgent.contains("MSIE")
-      # Trident is the Internet Explorer browser engine
-      - userAgent.contains("Trident")
-      # Opera is a fork of chrome now
-      - userAgent.contains("Presto")
-      # Windows CE is discontinued
-      - userAgent.contains("Windows CE")
-      # Windows 95 is discontinued
-      - userAgent.contains("Windows 95")
-      # Windows 98 is discontinued
-      - userAgent.contains("Windows 98")
-      # Windows 9.x is discontinued
-      - userAgent.contains("Win 9x")
-      # Amazon does not have an Alexa Toolbar.
-      - userAgent.contains("Alexa Toolbar")
-      # This is not released, even Windows 11 calls itself Windows 10
-      - userAgent.contains("Windows NT 11.0")
-      # iPods are not in common use
-      - userAgent.contains("iPod")
+    # Internet Explorer should be out of support
+    - userAgent.contains("MSIE")
+    # Trident is the Internet Explorer browser engine
+    - userAgent.contains("Trident")
+    # Opera is a fork of chrome now
+    - userAgent.contains("Presto")
+    # Windows CE is discontinued
+    - userAgent.contains("Windows CE")
+    # Windows 95 is discontinued
+    - userAgent.contains("Windows 95")
+    # Windows 98 is discontinued
+    - userAgent.contains("Windows 98")
+    # Windows 9.x is discontinued
+    - userAgent.contains("Win 9x")
+    # Amazon does not have an Alexa Toolbar.
+    - userAgent.contains("Alexa Toolbar")
+- name: challenge-aggressive-brazilian-scrapers
+  action: CHALLENGE
+  expression:
+    any:
+    # This is not released, even Windows 11 calls itself Windows 10
+    - userAgent.contains("Windows NT 11.0")
+    # iPods are not in common use
+    - userAgent.contains("iPod")

data/bots/ai-catchall.yaml

@@ -1,11 +0,0 @@
-# Extensive list of AI-affiliated agents based on https://github.com/ai-robots-txt/ai.robots.txt
-# Add new/undocumented agents here. Where documentation exists, consider moving to dedicated policy files.
-# Notes on various agents:
-#  - Amazonbot: Well documented, but they refuse to state which agent collects training data.
-#  - anthropic-ai/Claude-Web: Undocumented by Anthropic. Possibly deprecated or hallucinations?
-#  - Perplexity*: Well documented, but they refuse to state which agent collects training data.
-# Warning: May contain user agents that _must_ be blocked in robots.txt, or the opt-out will have no effect.
-- name: "ai-catchall"
-  user_agent_regex: >-
-    AI2Bot|Ai2Bot-Dolma|aiHitBot|Amazonbot|anthropic-ai|Brightbot 1.0|Bytespider|CCBot|Claude-Web|cohere-ai|cohere-training-data-crawler|Cotoyogi|Crawlspace|Diffbot|DuckAssistBot|FacebookBot|Factset_spyderbot|FirecrawlAgent|FriendlyCrawler|Google-CloudVertexBot|GoogleOther|GoogleOther-Image|GoogleOther-Video|iaskspider/2.0|ICC-Crawler|ImagesiftBot|img2dataset|imgproxy|ISSCyberRiskCrawler|Kangaroo Bot|meta-externalagent|Meta-ExternalAgent|meta-externalfetcher|Meta-ExternalFetcher|NovaAct|omgili|omgilibot|Operator|PanguBot|Perplexity-User|PerplexityBot|PetalBot|QualifiedBot|Scrapy|SemrushBot-OCOB|SemrushBot-SWA|Sidetrade indexer bot|TikTokSpider|Timpibot|VelenPublicWebCrawler|Webzio-Extended|wpbot|YouBot
-  action: DENY

data/bots/ai-robots-txt.yaml

@@ -1,6 +1,4 @@
-# Warning: Contains user agents that _must_ be blocked in robots.txt, or the opt-out will have no effect.
-# Note: Blocks human-directed/non-training user agents
 - name: "ai-robots-txt"
   user_agent_regex: >-
-    AI2Bot|Ai2Bot-Dolma|aiHitBot|Amazonbot|Andibot|anthropic-ai|Applebot|Applebot-Extended|bedrockbot|Brightbot 1.0|Bytespider|CCBot|ChatGPT-User|Claude-SearchBot|Claude-User|Claude-Web|ClaudeBot|cohere-ai|cohere-training-data-crawler|Cotoyogi|Crawlspace|Diffbot|DuckAssistBot|FacebookBot|Factset_spyderbot|FirecrawlAgent|FriendlyCrawler|Google-CloudVertexBot|Google-Extended|GoogleOther|GoogleOther-Image|GoogleOther-Video|GPTBot|iaskspider/2.0|ICC-Crawler|ImagesiftBot|img2dataset|ISSCyberRiskCrawler|Kangaroo Bot|meta-externalagent|Meta-ExternalAgent|meta-externalfetcher|Meta-ExternalFetcher|MistralAI-User/1.0|NovaAct|OAI-SearchBot|omgili|omgilibot|Operator|PanguBot|Panscient|panscient.com|Perplexity-User|PerplexityBot|PetalBot|PhindBot|QualifiedBot|QuillBot|quillbot.com|SBIntuitionsBot|Scrapy|SemrushBot-OCOB|SemrushBot-SWA|Sidetrade indexer bot|TikTokSpider|Timpibot|VelenPublicWebCrawler|Webzio-Extended|wpbot|YandexAdditional|YandexAdditionalBot|YouBot
+    AI2Bot|Ai2Bot-Dolma|aiHitBot|Amazonbot|anthropic-ai|Applebot|Applebot-Extended|Brightbot 1.0|Bytespider|CCBot|ChatGPT-User|Claude-Web|ClaudeBot|cohere-ai|cohere-training-data-crawler|Cotoyogi|Crawlspace|Diffbot|DuckAssistBot|FacebookBot|Factset_spyderbot|FirecrawlAgent|FriendlyCrawler|Google-Extended|GoogleOther|GoogleOther-Image|GoogleOther-Video|GPTBot|iaskspider/2.0|ICC-Crawler|ImagesiftBot|img2dataset|imgproxy|ISSCyberRiskCrawler|Kangaroo Bot|meta-externalagent|Meta-ExternalAgent|meta-externalfetcher|Meta-ExternalFetcher|NovaAct|OAI-SearchBot|omgili|omgilibot|Operator|PanguBot|Perplexity-User|PerplexityBot|PetalBot|QualifiedBot|Scrapy|SemrushBot-OCOB|SemrushBot-SWA|Sidetrade indexer bot|TikTokSpider|Timpibot|VelenPublicWebCrawler|Webzio-Extended|YouBot
   action: DENY

data/bots/cloudflare-workers.yaml

@@ -1,6 +1,4 @@
 - name: cloudflare-workers
   headers_regex:
     CF-Worker: .*
-  action: WEIGH
-  weight:
-    adjust: 15
+  action: DENY

data/clients/ai.yaml

@@ -1,8 +0,0 @@
-# User agents that act on behalf of humans in AI tools, e.g. searching the web.
-# Each entry should have a positive/ALLOW entry created as well, with further documentation.
-# Exceptions:
-#  - Claude-User: No published IP allowlist
-- name: "ai-clients"
-  user_agent_regex: >-
-    ChatGPT-User|Claude-User|MistralAI-User
-  action: DENY

data/clients/mistral-mistralai-user.yaml

@@ -1,10 +0,0 @@
-# Acts on behalf of user requests
-# https://docs.mistral.ai/robots/
-- name: mistral-mistralai-user
-  user_agent_regex: MistralAI-User/.+; \+https\://docs\.mistral\.ai/robots
-  action: ALLOW
-  # https://mistral.ai/mistralai-user-ips.json
-  remote_addresses: [
-    "20.240.160.161/32",
-    "20.240.160.1/32",
-  ]

data/clients/openai-chatgpt-user.yaml

@@ -1,93 +0,0 @@
-# Acts on behalf of user requests
-# https://platform.openai.com/docs/bots/overview-of-openai-crawlers
-- name: openai-chatgpt-user
-  user_agent_regex: ChatGPT-User/.+; \+https\://openai\.com/bot
-  action: ALLOW
-  # https://openai.com/chatgpt-user.json
-  # curl 'https://openai.com/chatgpt-user.json' | jq '.prefixes.[].ipv4Prefix' | sed 's/$/,/'
-  remote_addresses: [
-    "13.65.138.112/28",
-    "23.98.179.16/28",
-    "13.65.138.96/28",
-    "172.183.222.128/28",
-    "20.102.212.144/28",
-    "40.116.73.208/28",
-    "172.183.143.224/28",
-    "52.190.190.16/28",
-    "13.83.237.176/28",
-    "51.8.155.64/28",
-    "74.249.86.176/28",
-    "51.8.155.48/28",
-    "20.55.229.144/28",
-    "135.237.131.208/28",
-    "135.237.133.48/28",
-    "51.8.155.112/28",
-    "135.237.133.112/28",
-    "52.159.249.96/28",
-    "52.190.137.16/28",
-    "52.255.111.112/28",
-    "40.84.181.32/28",
-    "172.178.141.112/28",
-    "52.190.142.64/28",
-    "172.178.140.144/28",
-    "52.190.137.144/28",
-    "172.178.141.128/28",
-    "57.154.187.32/28",
-    "4.196.118.112/28",
-    "20.193.50.32/28",
-    "20.215.188.192/28",
-    "20.215.214.16/28",
-    "4.197.22.112/28",
-    "4.197.115.112/28",
-    "172.213.21.16/28",
-    "172.213.11.144/28",
-    "172.213.12.112/28",
-    "172.213.21.144/28",
-    "20.90.7.144/28",
-    "57.154.175.0/28",
-    "57.154.174.112/28",
-    "52.236.94.144/28",
-    "137.135.191.176/28",
-    "23.98.186.192/28",
-    "23.98.186.96/28",
-    "23.98.186.176/28",
-    "23.98.186.64/28",
-    "68.221.67.192/28",
-    "68.221.67.160/28",
-    "13.83.167.128/28",
-    "20.228.106.176/28",
-    "52.159.227.32/28",
-    "68.220.57.64/28",
-    "172.213.21.112/28",
-    "68.221.67.224/28",
-    "68.221.75.16/28",
-    "20.97.189.96/28",
-    "52.252.113.240/28",
-    "52.230.163.32/28",
-    "172.212.159.64/28",
-    "52.255.111.80/28",
-    "52.255.111.0/28",
-    "4.151.241.240/28",
-    "52.255.111.32/28",
-    "52.255.111.48/28",
-    "52.255.111.16/28",
-    "52.230.164.176/28",
-    "52.176.139.176/28",
-    "52.173.234.16/28",
-    "4.151.71.176/28",
-    "4.151.119.48/28",
-    "52.255.109.112/28",
-    "52.255.109.80/28",
-    "20.161.75.208/28",
-    "68.154.28.96/28",
-    "52.255.109.128/28",
-    "52.225.75.208/28",
-    "52.190.139.48/28",
-    "68.221.67.240/28",
-    "52.156.77.144/28",
-    "52.148.129.32/28",
-    "40.84.221.208/28",
-    "104.210.139.224/28",
-    "40.84.221.224/28",
-    "104.210.139.192/28",
-  ]

data/clients/small-internet-browsers/_permissive.yaml

@@ -1,2 +0,0 @@
-- import: (data)/clients/small-internet-browsers/netsurf.yaml
-- import: (data)/clients/small-internet-browsers/palemoon.yaml

data/clients/small-internet-browsers/netsurf.yaml

@@ -1,5 +0,0 @@
-- name: "reduce-weight-netsurf"
-  user_agent_regex: "NetSurf"
-  action: WEIGH
-  weight:
-    adjust: -5

data/clients/small-internet-browsers/palemoon.yaml

@@ -1,5 +0,0 @@
-- name: "reduce-weight-palemoon"
-  user_agent_regex: "PaleMoon"
-  action: WEIGH
-  weight:
-    adjust: -5

data/clients/x-firefox-ai.yaml

@@ -1,6 +0,0 @@
-# https://connect.mozilla.org/t5/firefox-labs/try-out-link-previews-in-firefox-labs-138-and-share-your/td-p/92012
-- name: x-firefox-ai
-  action: WEIGH
-  expression: '"X-Firefox-Ai" in headers'
-  weight:
-    adjust: 5

data/common/allow-private-addresses.yaml

@@ -1,15 +1,15 @@
 - name: ipv4-rfc-1918
   action: ALLOW
   remote_addresses:
-    - 10.0.0.0/8
-    - 172.16.0.0/12
-    - 192.168.0.0/16
-    - 100.64.0.0/10
+  - 10.0.0.0/8
+  - 172.16.0.0/12
+  - 192.168.0.0/16
+  - 100.64.0.0/10
 - name: ipv6-ula
   action: ALLOW
   remote_addresses:
-    - fc00::/7
+  - fc00::/7
 - name: ipv6-link-local
   action: ALLOW
   remote_addresses:
-    - fe80::/10
+  - fe80::/10

data/crawlers/_allow-good.yaml

@@ -1,5 +1,4 @@
 - import: (data)/crawlers/googlebot.yaml
-- import: (data)/crawlers/applebot.yaml
 - import: (data)/crawlers/bingbot.yaml
 - import: (data)/crawlers/duckduckbot.yaml
 - import: (data)/crawlers/qwantbot.yaml

data/crawlers/ai-search.yaml

@@ -1,8 +0,0 @@
-# User agents that index exclusively for search in for AI systems.
-# Each entry should have a positive/ALLOW entry created as well, with further documentation.
-# Exceptions:
-#  - Claude-SearchBot: No published IP allowlist
-- name: "ai-crawlers-search"
-  user_agent_regex: >-
-    OAI-SearchBot|Claude-SearchBot
-  action: DENY

data/crawlers/ai-training.yaml

@@ -1,8 +0,0 @@
-# User agents that crawl for training AI/LLM systems
-# Each entry should have a positive/ALLOW entry created as well, with further documentation.
-# Exceptions:
-#  - ClaudeBot: No published IP allowlist
-- name: "ai-crawlers-training"
-  user_agent_regex: >-
-    GPTBot|ClaudeBot
-  action: DENY

data/crawlers/applebot.yaml

@@ -1,20 +0,0 @@
-# Indexing for search and Siri
-# https://support.apple.com/en-us/119829
-- name: applebot
-  user_agent_regex: Applebot
-  action: ALLOW
-  # https://search.developer.apple.com/applebot.json
-  remote_addresses: [
-    "17.241.208.160/27",
-    "17.241.193.160/27",
-    "17.241.200.160/27",
-    "17.22.237.0/24",
-    "17.22.245.0/24",
-    "17.22.253.0/24",
-    "17.241.75.0/24",
-    "17.241.219.0/24",
-    "17.241.227.0/24",
-    "17.246.15.0/24",
-    "17.246.19.0/24",
-    "17.246.23.0/24",
-  ]

data/crawlers/openai-gptbot.yaml

@@ -1,16 +0,0 @@
-# Collects AI training data
-# https://platform.openai.com/docs/bots/overview-of-openai-crawlers
-- name: openai-gptbot
-  user_agent_regex: GPTBot/1\.1; \+https\://openai\.com/gptbot
-  action: ALLOW
-  # https://openai.com/gptbot.json
-  remote_addresses: [
-    "52.230.152.0/24",
-    "20.171.206.0/24",
-    "20.171.207.0/24",
-    "4.227.36.0/25",
-    "20.125.66.80/28",
-    "172.182.204.0/24",
-    "172.182.214.0/24",
-    "172.182.215.0/24",
-  ]

data/crawlers/openai-searchbot.yaml

@@ -1,13 +0,0 @@
-# Indexing for search, does not collect training data
-# https://platform.openai.com/docs/bots/overview-of-openai-crawlers
-- name: openai-searchbot
-  user_agent_regex: OAI-SearchBot/1\.0; \+https\://openai\.com/searchbot
-  action: ALLOW
-  # https://openai.com/searchbot.json
-  remote_addresses: [
-    "20.42.10.176/28",
-    "172.203.190.128/28",
-    "104.210.140.128/28",
-    "51.8.102.0/24",
-    "135.234.64.0/24"
-  ]

data/embed.go

@@ -3,6 +3,6 @@ package data
 import "embed"
 
 var (
-	//go:embed botPolicies.yaml botPolicies.json all:apps all:bots all:clients all:common all:crawlers all:meta
+	//go:embed botPolicies.yaml botPolicies.json all:apps all:bots all:clients all:common all:crawlers
 	BotPolicies embed.FS
 )

data/meta/README.md

@@ -1,5 +0,0 @@
-# meta policies
-
-Contains policies that exclusively reference policies in _multiple_ other data folders.
-
-Akin to "stances" that the administrator can take, with reference to various topics, such as AI/LLM systems.

data/meta/ai-block-aggressive.yaml

@@ -1,6 +0,0 @@
-# Blocks all AI/LLM associated user agents, regardless of purpose or human agency
-# Warning: To completely block some AI/LLM training, such as with Google, you _must_ place flags in robots.txt.
-- import: (data)/bots/ai-catchall.yaml
-- import: (data)/clients/ai.yaml
-- import: (data)/crawlers/ai-search.yaml
-- import: (data)/crawlers/ai-training.yaml

data/meta/ai-block-moderate.yaml

@@ -1,7 +0,0 @@
-# Blocks all AI/LLM bots used for training or unknown/undocumented purposes.
-# Permits user agents with explicitly documented non-training use, and published IP allowlists.
-- import: (data)/bots/ai-catchall.yaml
-- import: (data)/crawlers/ai-training.yaml
-- import: (data)/crawlers/openai-searchbot.yaml
-- import: (data)/clients/openai-chatgpt-user.yaml
-- import: (data)/clients/mistral-mistralai-user.yaml

data/meta/ai-block-permissive.yaml

@@ -1,6 +0,0 @@
-# Permits all well documented AI/LLM user agents with published IP allowlists.
-- import: (data)/bots/ai-catchall.yaml
-- import: (data)/crawlers/openai-searchbot.yaml
-- import: (data)/crawlers/openai-gptbot.yaml
-- import: (data)/clients/openai-chatgpt-user.yaml
-- import: (data)/clients/mistral-mistralai-user.yaml

docker-bake.hcl

@@ -1,27 +0,0 @@
-variable "ALPINE_VERSION" { default = "3.22" }
-variable "GITHUB_SHA" { default = "devel" }
-
-group "default" {
-  targets = [
-    "anubis",
-  ]
-}
-
-target "anubis" {
-  args = {
-    ALPINE_VERSION = "3.22"
-  }
-  context = "."
-  dockerfile = "./Dockerfile"
-  platforms = [
-    "linux/amd64",
-    "linux/arm64",
-    "linux/arm/v7",
-    "linux/ppc64le",
-    "linux/riscv64",
-  ]
-  pull = true
-  tags = [
-    "ghcr.io/techarohq/anubis:${GITHUB_SHA}"
-  ]
-}

docs/docs/CHANGELOG.md

@@ -11,54 +11,22 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 
 ## [Unreleased]
 
-- Remove the unused `/test-error` endpoint and update the testing endpoint `/make-challenge` to only be enabled in
-  development
-- Add `--xff-strip-private` flag/envvar to toggle skipping X-Forwarded-For private addresses or not
-- Requests can have their weight be adjusted, if a request weighs zero or less than it is allowed through
-- Refactor challenge presentation logic to use a challenge registry
-- Allow challenge implementations to register HTTP routes
-- Implement a no-JS challenge method: [`metarefresh`](./admin/configuration/challenges/metarefresh.mdx) ([#95](https://github.com/TecharoHQ/anubis/issues/95))
-- Bump AI-robots.txt to version 1.34
-- Make progress bar styling more compatible (UXP, etc)
-
-## v1.19.1: Jenomis cen Lexentale - Echo 1
-
-- Return `data/bots/ai-robots-txt.yaml` to avoid breaking configs [#599](https://github.com/TecharoHQ/anubis/issues/599)
-
-## v1.19.0: Jenomis cen Lexentale
-
-Mostly a bunch of small features, no big ticket things this time.
-
-- Record if challenges were issued via the API or via embedded JSON in the challenge page HTML ([#531](https://github.com/TecharoHQ/anubis/issues/531))
 - Ensure that clients that are shown a challenge support storing cookies
-- Imprint the version number into challenge pages
 - Encode challenge pages with gzip level 1
-- Add PowerPC 64 bit little-endian builds (`GOARCH=ppc64le`)
 - Add `check-spelling` for spell checking
 - Add `--target-insecure-skip-verify` flag/envvar to allow Anubis to hit a self-signed HTTPS backend
 - Minor adjustments to FreeBSD rc.d script to allow for more flexible configuration.
 - Added Podman and Docker support for running Playwright tests
-- Add a default rule to throw challenges when a request with the `X-Firefox-Ai` header is set
 - Updated the nonce value in the challenge JWT cookie to be a string instead of a number
 - Rename cookies in response to user feedback
 - Ensure cookie renaming is consistent across configuration options
 - Add Bookstack app in data
-- Truncate everything but the first five characters of Accept-Language headers when making challenges
-- Ensure client JavaScript is served with Content-Type text/javascript.
-- Add `--target-host` flag/envvar to allow changing the value of the Host header in requests forwarded to the target service
-- Bump AI-robots.txt to version 1.31
+- Add `--target-host` flag/envvar to allow changing the value of the Host header in requests forwarded to the target service.
+- Bump AI-robots.txt to version 1.30 (add QualifiedBot)
 - Add `RuntimeDirectory` to systemd unit settings so native packages can listen over unix sockets
 - Added SearXNG instance tracker whitelist policy
 - Added Qualys SSL Labs whitelist policy
 - Fixed cookie deletion logic ([#520](https://github.com/TecharoHQ/anubis/issues/520), [#522](https://github.com/TecharoHQ/anubis/pull/522))
-- Add `--target-sni` flag/envvar to allow changing the value of the TLS handshake hostname in requests forwarded to the target service
-- Fixed CEL expression matching validator to now properly error out when it receives empty expressions
-- Added OpenRC init.d script
-- Added `--version` flag
-- Added `anubis_proxied_requests_total` metric to count proxied requests
-- Add `Applebot` as "good" web crawler
-- Reorganize AI/LLM crawler blocking into three separate stances, maintaining existing status quo as default
-- Split out AI/LLM user agent blocking policies, adding documentation for each
 
 ## v1.18.0: Varis zos Galvus
 

docs/docs/admin/configuration/challenges/_category_.json

@@ -1,8 +0,0 @@
-{
-  "label": "Challenges",
-  "position": 10,
-  "link": {
-    "type": "generated-index",
-    "description": "The different challenge methods that Anubis supports."
-  }
-}

docs/docs/admin/configuration/challenges/metarefresh.mdx

@@ -1,19 +0,0 @@
-# Meta Refresh (No JavaScript)
-
-The `metarefresh` challenge sends a browser a much simpler challenge that makes it refresh the page after a set period of time. This enables clients to pass challenges without executing JavaScript.
-
-To use it in your Anubis configuration:
-
-```yaml
-# Generic catchall rule
-- name: generic-browser
-  user_agent_regex: >-
-    Mozilla|Opera
-  action: CHALLENGE
-  challenge:
-    difficulty: 1 # Number of seconds to wait before refreshing the page
-    report_as: 4 # Unused by this challenge method
-    algorithm: metarefresh # Specify a non-JS challenge method
-```
-
-This is not enabled by default while this method is tested and its false positive rate is ascertained. Many modern scrapers use headless Google Chrome, so this will have a much higher false positive rate.

docs/docs/admin/configuration/challenges/proof-of-work.mdx

@@ -1,5 +0,0 @@
-# Proof of Work (JavaScript)
-
-When Anubis is configured to use the `fast` or `slow` challenge methods, clients will be sent a small [proof of work](https://en.wikipedia.org/wiki/Proof_of_work) challenge. In order to get a token used to access the upstream resource, clients must calculate a complicated math puzzle with JavaScript.
-
-A `fast` challenge uses a heavily optimized multithreaded implementation and a `slow` challenge uses a simplistic single-threaded implementation. The `slow` method is kept around for legacy compatibility.

docs/docs/admin/configuration/expressions.mdx

@@ -143,29 +143,7 @@ Anubis would return a challenge because all of those conditions are true.
 
 ## Functions exposed to Anubis expressions
 
-Anubis expressions can be augmented with the following functions:
-
-### `randInt`
-
-```ts
-function randInt(n: int): int;
-```
-
-randInt returns a randomly selected integer value in the range of `[0,n)`. This is a thin wrapper around [Go's math/rand#Intn](https://pkg.go.dev/math/rand#Intn). Be careful with this as it may cause inconsistent behavior for genuine users.
-
-This is best applied when doing explicit block rules, eg:
-
-```yaml
-# Denies LightPanda about 75% of the time on average
-- name: deny-lightpanda-sometimes
-  action: DENY
-  expression:
-    all:
-      - userAgent.matches("LightPanda")
-      - randInt(16) >= 4
-```
-
-It seems counter-intuitive to allow known bad clients through sometimes, but this allows you to confuse attackers by making Anubis' behavior random. Adjust the thresholds and numbers as facts and circumstances demand.
+There are currently no functions from the Anubis runtime exposed to expressions. This will change in the future.
 
 ## Life advice
 

docs/docs/admin/configuration/import.mdx

@@ -14,7 +14,7 @@ EG:
 {
   "bots": [
     {
-      "import": "(data)/bots/ai-catchall.yaml"
+      "import": "(data)/bots/ai-robots-txt.yaml"
     },
     {
       "import": "(data)/bots/cloudflare-workers.yaml"
@@ -29,8 +29,8 @@ EG:
 ```yaml
 bots:
   # Pathological bots to deny
-  - # This correlates to data/bots/ai-catchall.yaml in the source tree
-    import: (data)/bots/ai-catchall.yaml
+  - # This correlates to data/bots/ai-robots-txt.yaml in the source tree
+    import: (data)/bots/ai-robots-txt.yaml
   - import: (data)/bots/cloudflare-workers.yaml
 ```
 
@@ -46,7 +46,7 @@ Of note, a bot rule can either have inline bot configuration or import a bot con
 {
   "bots": [
     {
-      "import": "(data)/bots/ai-catchall.yaml",
+      "import": "(data)/bots/ai-robots-txt.yaml",
       "name": "generic-browser",
       "user_agent_regex": "Mozilla|Opera\n",
       "action": "CHALLENGE"
@@ -60,7 +60,7 @@ Of note, a bot rule can either have inline bot configuration or import a bot con
 
 ```yaml
 bots:
-  - import: (data)/bots/ai-catchall.yaml
+  - import: (data)/bots/ai-robots-txt.yaml
     name: generic-browser
     user_agent_regex: >
       Mozilla|Opera
@@ -167,7 +167,7 @@ static
 ├── botPolicies.json
 ├── botPolicies.yaml
 ├── bots
-│   ├── ai-catchall.yaml
+│   ├── ai-robots-txt.yaml
 │   ├── cloudflare-workers.yaml
 │   ├── headless-browsers.yaml
 │   └── us-ai-scraper.yaml

docs/docs/admin/configuration/subrequest-auth.mdx

@@ -10,20 +10,6 @@ Anubis can act in one of two modes:
 1. Reverse proxy (the default): Anubis sits in the middle of all traffic and then will reverse proxy it to its destination. This is the moral equivalent of a middleware in your favorite web framework.
 2. Subrequest authentication mode: Anubis listens for requests and if they don't pass muster then they are forwarded to Anubis for challenge processing. This is the equivalent of Anubis being a sidecar service.
 
-:::note
-
-Subrequest authentication requires changing the default policy because nginx interprets the default `DENY` status code `200` as successful authentication and allows the request.
-
-```yaml
-status_codes:
-  CHALLENGE: 200
-  DENY: 403
-```
-
-[See policy definitions](../policies.mdx).
-
-:::
-
 ## Nginx
 
 Anubis can perform [subrequest authentication](https://docs.nginx.com/nginx/admin-guide/security-controls/configuring-subrequest-authentication/) with the `auth_request` module in Nginx. In order to set this up, keep the following things in mind:

docs/docs/admin/environments/apache.mdx

@@ -92,7 +92,6 @@ Assuming you are protecting `anubistest.techaro.lol`, you need the following ser
        # throw an "admin misconfiguration" error.
        RequestHeader set "X-Real-Ip" expr=%{REMOTE_ADDR}
        RequestHeader set X-Forwarded-Proto "https"
-       RequestHeader set "X-Http-Version" "%{SERVER_PROTOCOL}s"
 
        ProxyPreserveHost On
 
@@ -120,14 +119,6 @@ Make sure to add a separate configuration file for the listener on port 3001:
 ```text
 # /etc/httpd/conf.d/listener-3001.conf
 
-Listen [::1]:3001
-```
-
-In case you are running an IPv4-only system, use the following configuration instead:
-
-```text
-# /etc/httpd/conf.d/listener-3001.conf
-
 Listen 127.0.0.1:3001
 ```
 

docs/docs/admin/environments/nginx.mdx

@@ -61,7 +61,6 @@ server {
   location / {
     proxy_set_header Host $host;
     proxy_set_header X-Real-IP $remote_addr;
-    proxy_set_header X-Http-Version $server_protocol;
     proxy_pass http://anubis;
   }
 

docs/docs/admin/environments/traefik.mdx

@@ -3,10 +3,11 @@ id: traefik
 title: Traefik
 ---
 
+
 :::note
 
-This only talks about integration through Compose,
-but it also applies to docker cli options.
+ This only talks about integration through Compose,
+ but it also applies to docker cli options.
 
 :::
 

docs/docs/admin/frameworks/wordpress.mdx

@@ -1,39 +0,0 @@
-# Wordpress
-
-Wordpress is the most popular blog engine on the planet.
-
-## Using a multi-site setup with Anubis
-
-If you have a multi-site setup where traffic goes through Anubis like this:
-
-```mermaid
----
-title: Apache as tls terminator and HTTP router
----
-
-flowchart LR
-    T(User Traffic)
-    subgraph Apache 2
-        TCP(TCP 80/443)
-        US(TCP 3001)
-    end
-
-    An(Anubis)
-    B(Backend)
-
-    T --> |TLS termination| TCP
-    TCP --> |Traffic filtering| An
-    An --> |Happy traffic| US
-    US --> |whatever you're doing| B
-```
-
-Wordpress may not realize that the underlying connection is being done over HTTPS. This could lead to a redirect loop in the `/wp-admin/` routes. In order to fix this, add the following to your `wp-config.php` file:
-
-```php
-if (isset($_SERVER['HTTP_X_FORWARDED_PROTO']) && $_SERVER['HTTP_X_FORWARDED_PROTO'] === 'https') {
-    $_SERVER['HTTPS'] = 'on';
-    $_SERVER['SERVER_PORT'] = 443;
-}
-```
-
-This will make Wordpress think that your connection is over HTTPS instead of plain HTTP.

docs/docs/admin/installation.mdx

@@ -49,30 +49,29 @@ For more detailed information on installing Anubis with native packages, please
 
 Anubis uses these environment variables for configuration:
 
-| Environment Variable           | Default value           | Explanation                                                                                                                                                                                                                                                                                                                                                                                                                                                                                             |
-| :----------------------------- | :---------------------- | :------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ |
-| `BASE_PREFIX`                  | unset                   | If set, adds a global prefix to all Anubis endpoints. For example, setting this to `/myapp` would make Anubis accessible at `/myapp/` instead of `/`. This is useful when running Anubis behind a reverse proxy that routes based on path prefixes.                                                                                                                                                                                                                                                     |
-| `BIND`                         | `:8923`                 | The network address that Anubis listens on. For `unix`, set this to a path: `/run/anubis/instance.sock`                                                                                                                                                                                                                                                                                                                                                                                                 |
-| `BIND_NETWORK`                 | `tcp`                   | The address family that Anubis listens on. Accepts `tcp`, `unix` and anything Go's [`net.Listen`](https://pkg.go.dev/net#Listen) supports.                                                                                                                                                                                                                                                                                                                                                              |
-| `COOKIE_DOMAIN`                | unset                   | The domain the Anubis challenge pass cookie should be set to. This should be set to the domain you bought from your registrar (EG: `techaro.lol` if your webapp is running on `anubis.techaro.lol`). See this [stackoverflow explanation of cookies](https://stackoverflow.com/a/1063760) for more information.<br/><br/>Note that unlike `REDIRECT_DOMAINS`, you should never include a port number in this variable.                                                                                  |
-| `COOKIE_EXPIRATION_TIME`       | `168h`                  | The amount of time the authorization cookie is valid for.                                                                                                                                                                                                                                                                                                                                                                                                                                               |
-| `COOKIE_PARTITIONED`           | `false`                 | If set to `true`, enables the [partitioned (CHIPS) flag](https://developers.google.com/privacy-sandbox/cookies/chips), meaning that Anubis inside an iframe has a different set of cookies than the domain hosting the iframe.                                                                                                                                                                                                                                                                          |
-| `DIFFICULTY`                   | `4`                     | The difficulty of the challenge, or the number of leading zeroes that must be in successful responses.                                                                                                                                                                                                                                                                                                                                                                                                  |
-| `ED25519_PRIVATE_KEY_HEX`      | unset                   | The hex-encoded ed25519 private key used to sign Anubis responses. If this is not set, Anubis will generate one for you. This should be exactly 64 characters long. See below for details.                                                                                                                                                                                                                                                                                                              |
-| `ED25519_PRIVATE_KEY_HEX_FILE` | unset                   | Path to a file containing the hex-encoded ed25519 private key. Only one of this or its sister option may be set.                                                                                                                                                                                                                                                                                                                                                                                        |
-| `METRICS_BIND`                 | `:9090`                 | The network address that Anubis serves Prometheus metrics on. See `BIND` for more information.                                                                                                                                                                                                                                                                                                                                                                                                          |
-| `METRICS_BIND_NETWORK`         | `tcp`                   | The address family that the Anubis metrics server listens on. See `BIND_NETWORK` for more information.                                                                                                                                                                                                                                                                                                                                                                                                  |
-| `OG_EXPIRY_TIME`               | `24h`                   | The expiration time for the Open Graph tag cache.                                                                                                                                                                                                                                                                                                                                                                                                                                                       |
-| `OG_PASSTHROUGH`               | `false`                 | If set to `true`, Anubis will enable Open Graph tag passthrough.                                                                                                                                                                                                                                                                                                                                                                                                                                        |
-| `OG_CACHE_CONSIDER_HOST`       | `false`                 | If set to `true`, Anubis will consider the host in the Open Graph tag cache key.                                                                                                                                                                                                                                                                                                                                                                                                                        |
-| `POLICY_FNAME`                 | unset                   | The file containing [bot policy configuration](./policies.mdx). See the bot policy documentation for more details. If unset, the default bot policy configuration is used.                                                                                                                                                                                                                                                                                                                              |
+| Environment Variable           | Default value           | Explanation                                                                                                                                                                                                                                                                                                          |
+| :----------------------------- | :---------------------- | :------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- |
+| `BASE_PREFIX`                  | unset                   | If set, adds a global prefix to all Anubis endpoints. For example, setting this to `/myapp` would make Anubis accessible at `/myapp/` instead of `/`. This is useful when running Anubis behind a reverse proxy that routes based on path prefixes.                                                                  |
+| `BIND`                         | `:8923`                 | The network address that Anubis listens on. For `unix`, set this to a path: `/run/anubis/instance.sock`                                                                                                                                                                                                              |
+| `BIND_NETWORK`                 | `tcp`                   | The address family that Anubis listens on. Accepts `tcp`, `unix` and anything Go's [`net.Listen`](https://pkg.go.dev/net#Listen) supports.                                                                                                                                                                           |
+| `COOKIE_DOMAIN`                | unset                   | The domain the Anubis challenge pass cookie should be set to. This should be set to the domain you bought from your registrar (EG: `techaro.lol` if your webapp is running on `anubis.techaro.lol`). See this [stackoverflow explanation of cookies](https://stackoverflow.com/a/1063760) for more information.<br/><br/>Note that unlike `REDIRECT_DOMAINS`, you should never include a port number in this variable. |
+| `COOKIE_EXPIRATION_TIME`       | `168h`                  | The amount of time the authorization cookie is valid for.                                                                                                                                                                                                                                                            |
+| `COOKIE_PARTITIONED`           | `false`                 | If set to `true`, enables the [partitioned (CHIPS) flag](https://developers.google.com/privacy-sandbox/cookies/chips), meaning that Anubis inside an iframe has a different set of cookies than the domain hosting the iframe.                                                                                       |
+| `DIFFICULTY`                   | `4`                     | The difficulty of the challenge, or the number of leading zeroes that must be in successful responses.                                                                                                                                                                                                               |
+| `ED25519_PRIVATE_KEY_HEX`      | unset                   | The hex-encoded ed25519 private key used to sign Anubis responses. If this is not set, Anubis will generate one for you. This should be exactly 64 characters long. See below for details.                                                                                                                           |
+| `ED25519_PRIVATE_KEY_HEX_FILE` | unset                   | Path to a file containing the hex-encoded ed25519 private key. Only one of this or its sister option may be set.                                                                                                                                                                                                     |
+| `METRICS_BIND`                 | `:9090`                 | The network address that Anubis serves Prometheus metrics on. See `BIND` for more information.                                                                                                                                                                                                                       |
+| `METRICS_BIND_NETWORK`         | `tcp`                   | The address family that the Anubis metrics server listens on. See `BIND_NETWORK` for more information.                                                                                                                                                                                                               |
+| `OG_EXPIRY_TIME`               | `24h`                   | The expiration time for the Open Graph tag cache.                                                                                                                                                                                                                                                                    |
+| `OG_PASSTHROUGH`               | `false`                 | If set to `true`, Anubis will enable Open Graph tag passthrough.                                                                                                                                                                                                                                                     |
+| `OG_CACHE_CONSIDER_HOST`       | `false`                 | If set to `true`, Anubis will consider the host in the Open Graph tag cache key.                                                                                                                                                                                                                                     |
+| `POLICY_FNAME`                 | unset                   | The file containing [bot policy configuration](./policies.mdx). See the bot policy documentation for more details. If unset, the default bot policy configuration is used.                                                                                                                                           |
 | `REDIRECT_DOMAINS`             | unset                   | If set, restrict the domains that Anubis can redirect to when passing a challenge.<br/><br/>If this is unset, Anubis may redirect to any domain which could cause security issues in the unlikely case that an attacker passes a challenge for your browser and then tricks you into clicking a link to your domain.<br/><br/>Note that if you are hosting Anubis on a non-standard port (`https://example:com:8443`, `http://www.example.net:8080`, etc.), you must also include the port number here. |
-| `SERVE_ROBOTS_TXT`             | `false`                 | If set `true`, Anubis will serve a default `robots.txt` file that disallows all known AI scrapers by name and then additionally disallows every scraper. This is useful if facts and circumstances make it difficult to change the underlying service to serve such a `robots.txt` file.                                                                                                                                                                                                                |
-| `SOCKET_MODE`                  | `0770`                  | _Only used when at least one of the `*_BIND_NETWORK` variables are set to `unix`._ The socket mode (permissions) for Unix domain sockets.                                                                                                                                                                                                                                                                                                                                                               |
-| `TARGET`                       | `http://localhost:3923` | The URL of the service that Anubis should forward valid requests to. Supports Unix domain sockets, set this to a URI like so: `unix:///path/to/socket.sock`.                                                                                                                                                                                                                                                                                                                                            |
-| `USE_REMOTE_ADDRESS`           | unset                   | If set to `true`, Anubis will take the client's IP from the network socket. For production deployments, it is expected that a reverse proxy is used in front of Anubis, which pass the IP using headers, instead.                                                                                                                                                                                                                                                                                       |
-| `WEBMASTER_EMAIL`              | unset                   | If set, shows a contact email address when rendering error pages. This email address will be how users can get in contact with administrators.                                                                                                                                                                                                                                                                                                                                                          |
-| `XFF_STRIP_PRIVATE`            | `true`                  | If set, strip private addresses from `X-Forwarded-For` headers. To unset this, you must set `XFF_STRIP_PRIVATE=false` or `--xff-strip-private=false`.                                                                                                                                                                                                                                                                                                                                                   |
+| `SERVE_ROBOTS_TXT`             | `false`                 | If set `true`, Anubis will serve a default `robots.txt` file that disallows all known AI scrapers by name and then additionally disallows every scraper. This is useful if facts and circumstances make it difficult to change the underlying service to serve such a `robots.txt` file.                             |
+| `SOCKET_MODE`                  | `0770`                  | _Only used when at least one of the `*_BIND_NETWORK` variables are set to `unix`._ The socket mode (permissions) for Unix domain sockets.                                                                                                                                                                            |
+| `TARGET`                       | `http://localhost:3923` | The URL of the service that Anubis should forward valid requests to. Supports Unix domain sockets, set this to a URI like so: `unix:///path/to/socket.sock`.                                                                                                                                                         |
+| `USE_REMOTE_ADDRESS`           | unset                   | If set to `true`, Anubis will take the client's IP from the network socket. For production deployments, it is expected that a reverse proxy is used in front of Anubis, which pass the IP using headers, instead.                                                                                                    |
+| `WEBMASTER_EMAIL`              | unset                   | If set, shows a contact email address when rendering error pages. This email address will be how users can get in contact with administrators.                                                                                                                                                                       |
 
 <details>
 <summary>Advanced configuration settings</summary>
@@ -85,7 +84,6 @@ If you don't know or understand what these settings mean, ignore them. These are
 
 | Environment Variable          | Default value | Explanation                                                                                                                                         |
 | :---------------------------- | :------------ | :-------------------------------------------------------------------------------------------------------------------------------------------------- |
-| `TARGET_SNI`                  | unset         | If set, overrides the TLS handshake hostname in requests forwarded to `TARGET`.                                                                     |
 | `TARGET_HOST`                 | unset         | If set, overrides the Host header in requests forwarded to `TARGET`.                                                                                |
 | `TARGET_INSECURE_SKIP_VERIFY` | `false`       | If `true`, skip TLS certificate validation for targets that listen over `https`. If your backend does not listen over `https`, ignore this setting. |
 

docs/docs/admin/native-install.mdx

@@ -49,7 +49,7 @@ sudo install -D ./run/anubis@.service /etc/systemd/system
 Install the default configuration file to your system:
 
 ```text
-sudo install -D ./run/default.env /etc/anubis/default.env
+sudo install -D ./run/default.env /etc/anubis
 ```
 
   </TabItem>
@@ -77,13 +77,6 @@ Install Anubis with `rpm`:
 sudo rpm -ivh ./anubis-$VERSION.$ARCH.rpm
 ```
 
-  </TabItem>
-  <TabItem value="distro" label="Package managers">
-
-Some Linux distributions offer Anubis [as a native package](https://repology.org/project/anubis-anti-crawler/versions). If you want to install Anubis from your distribution's package manager, consult any upstream documentation for how to install the package. It will either be named `anubis`, `www-apps/anubis` or `www/anubis`.
-
-If you use a systemd-flavoured distribution, then follow the setup instructions for Debian or Red Hat Linux.
-
   </TabItem>
 </Tabs>
 

docs/docs/admin/policies.mdx

@@ -244,39 +244,3 @@ In case your service needs it for risk calculation reasons, Anubis exposes infor
 | `X-Anubis-Status` | The status and how strict Anubis was in its checks   | `PASS`           |
 
 Policy rules are matched using [Go's standard library regular expressions package](https://pkg.go.dev/regexp). You can mess around with the syntax at [regex101.com](https://regex101.com), make sure to select the Golang option.
-
-## Request Weight
-
-Anubis rules can also add or remove "weight" from requests, allowing administrators to configure custom levels of suspicion. For example, if your application uses session tokens named `i_love_gitea`:
-
-```yaml
-- name: gitea-session-token
-  action: WEIGH
-  expression:
-    all:
-      - '"Cookie" in headers'
-      - headers["Cookie"].contains("i_love_gitea=")
-  # Remove 5 weight points
-  weight:
-    adjust: -5
-```
-
-This would remove five weight points from the request, making Anubis present the [Meta Refresh challenge](./configuration/challenges/metarefresh.mdx).
-
-### Weight Thresholds
-
-Weight thresholds and challenge associations will be configurable with CEL expressions in the configuration file in an upcoming patch, for now here's how Anubis configures the weight thresholds:
-
-|                                      Weight Expression | Action                                                                                                                                 |
-| -----------------------------------------------------: | :------------------------------------------------------------------------------------------------------------------------------------- |
-|                   `weight < 0` (weight is less than 0) | Allow the request through.                                                                                                             |
-|                 `weight < 10` (weight is less than 10) | Challenge the client with the [Meta Refresh challenge](./configuration/challenges/metarefresh.mdx) at the default difficulty level.    |
-| `weight >= 10` (weight is greater than or equal to 10) | Challenge the client with the [Proof of Work challenge](./configuration/challenges/proof-of-work.mdx) at the default difficulty level. |
-
-### Advice
-
-Weight is still very new and needs work. This is an experimental feature and should be treated as such. Here's some advice to help you better tune requests:
-
-- The default weight for browser-like clients is 10. This triggers an aggressive challenge.
-- Remove and add weight in multiples of five.
-- Be careful with how you configure weight.

docs/docs/design/how-anubis-works.mdx

@@ -91,7 +91,7 @@ work valid?"}
 
 ## Proof of passing challenges
 
-When a client passes a challenge, Anubis sets an HTTP cookie named `"techaro.lol-anubis-auth"` containing a signed [JWT](https://jwt.io/) (JSON Web Token). This JWT contains the following claims:
+When a client passes a challenge, Anubis sets an HTTP cookie named `"within.website-x-cmd-anubis-auth"` containing a signed [JWT](https://jwt.io/) (JSON Web Token). This JWT contains the following claims:
 
 - `challenge`: The challenge string derived from user request metadata
 - `nonce`: The nonce / iteration number used to generate the passing response

docs/docs/index.mdx

@@ -19,48 +19,14 @@ title: Anubis
 
 Anubis is brought to you by sponsors and donors like:
 
-### Diamond Tier
-
-<a href="https://www.raptorcs.com/content/base/products.html">
-  <img
-    src="/img/sponsors/raptor-computing-logo.webp"
-    alt="Raptor Computing Systems"
-    height="64"
-  />
-</a>
-
-### Gold Tier
-
-<a href="https://distrust.co?utm_campaign=github&utm_medium=referral&utm_content=anubis">
-  <img src="/img/sponsors/distrust-logo.webp" alt="Distrust" height="64" />
-</a>
-<a href="https://terminaltrove.com/?utm_campaign=github&utm_medium=referral&utm_content=anubis&utm_source=abgh">
-  <img
-    src="/img/sponsors/terminal-trove.webp"
-    alt="Terminal Trove"
-    height="64"
-  />
-</a>
-<a href="https://canine.tools?utm_campaign=github&utm_medium=referral&utm_content=anubis">
-  <img
-    src="/img/sponsors/caninetools-logo.webp"
-    alt="canine.tools"
-    height="64"
-  />
-</a>
-<a href="https://weblate.org/">
-  <img src="/img/sponsors/weblate-logo.webp" alt="Weblate" height="64" />
-</a>
-<a href="https://uberspace.de/">
-  <img src="/img/sponsors/uberspace-logo.webp" alt="Uberspace" height="64" />
-</a>
-<a href="https://wildbase.xyz/">
-  <img src="/img/sponsors/wildbase-logo.webp" alt="Wildbase" height="64" />
-</a>
+[![Distrust](/img/sponsors/distrust-logo.webp)](https://distrust.co?utm_campaign=github&utm_medium=referral&utm_content=anubis)
+[![Terminal Trove](/img/sponsors/terminal-trove.webp)](https://terminaltrove.com/?utm_campaign=github&utm_medium=referral&utm_content=anubis&utm_source=abgh)
+[![canine.tools](/img/sponsors/caninetools-logo.webp)](https://canine.tools?utm_campaign=github&utm_medium=referral&utm_content=anubis)
+[![Weblate](/img/sponsors/weblate-logo.webp)](https://weblate.org/?utm_campaign=github&utm_medium=referral&utm_content=anubis)
 
 ## Overview
 
-Anubis is a Web AI Firewall Utility that [weighs the soul of your connection](https://en.wikipedia.org/wiki/Weighing_of_souls) using one or more challenges in order to protect upstream resources from scraper bots.
+Anubis [weighs the soul of your connection](https://en.wikipedia.org/wiki/Weighing_of_souls) using a proof-of-work challenge in order to protect upstream resources from scraper bots.
 
 This program is designed to help protect the small internet from the endless storm of requests that flood in from AI companies. Anubis is as lightweight as possible to ensure that everyone can afford to protect the communities closest to them.
 

docs/docs/user/frequently-asked-questions.mdx

@@ -18,11 +18,3 @@ Anubis uses [Web Workers](https://developer.mozilla.org/en-US/docs/Web/API/Web_W
 2. Web Workers allow you to do multithreaded execution of JavaScript code. This lets Anubis run its checks in parallel across all your system cores so that the challenge can complete as fast as possible. In the last decade, most CPU advancements have come from making cores and code extremely parallel. Using Web Workers lets Anubis take advantage of your hardware as much as possible so that the challenge finishes as fast as possible.
 
 If you use a browser extension such as [JShelter](https://jshelter.org/), you will need to [modify your JShelter configuration](./known-broken-extensions.md#jshelter) to allow Anubis' proof of work computation to complete.
-
-## Does Anubis mine Bitcoin?
-
-No. Anubis does not mine Bitcoin.
-
-In order to mine bitcoin, you need to download a copy of the blockchain (so you have the state required to do mining) and also broadcast your mined blocks to the network should you reach a hash with the right number of leading zeroes. You also need to continuously read for newly broadcasted transactions so you can batch them into a block. This requires gigabytes of data to be transferred from the server to the client.
-
-Anubis transfers two digit numbers of kilobytes from the server to the client (which you can independently verify with your browser's Developer Tools feature). This is orders of magnitude below what is required to mine Bitcoin.

docs/docs/user/known-instances.md

@@ -35,12 +35,6 @@ This page contains a non-exhaustive list with all websites using Anubis.
 - https://hofstede.io/
 - https://www.indiemag.fr/
 - https://reddit.nerdvpn.de/
-- https://hosted.weblate.org/
-- https://gitea.com/
-- https://openwrt.org/
-- https://minihoot.site
-- https://catgirl.click/
-- https://wiki.dolphin-emu.org/
 - <details>
   <summary>FreeCAD</summary>
   - https://forum.freecad.org/
@@ -64,10 +58,3 @@ This page contains a non-exhaustive list with all websites using Anubis.
   <summary>The United Nations</summary>
   - https://policytoolbox.iiep.unesco.org/
   </details>
-- <details>
-  <summary>hebis (Alliance of Hessian Libraries)</summary>
-  - https://ubmr.hds.hebis.de/
-  - https://tufind.hds.hebis.de/
-  - https://karla.hds.hebis.de/
-  - and many more (see https://www.hebis.de/dienste/hebis-discovery-system/)
-  </details>

docs/manifest/cfg/anubis/botPolicies.yaml

@@ -1,72 +0,0 @@
-## Anubis has the ability to let you import snippets of configuration into the main
-## configuration file. This allows you to break up your config into smaller parts
-## that get logically assembled into one big file.
-##
-## Of note, a bot rule can either have inline bot configuration or import a
-## bot config snippet. You cannot do both in a single bot rule.
-##
-## Import paths can either be prefixed with (data) to import from the common/shared
-## rules in the data folder in the Anubis source tree or will point to absolute/relative
-## paths in your filesystem. If you don't have access to the Anubis source tree, check
-## /usr/share/docs/anubis/data or in the tarball you extracted Anubis from.
-
-bots:
-  # Pathological bots to deny
-  - # This correlates to data/bots/deny-pathological.yaml in the source tree
-    # https://github.com/TecharoHQ/anubis/blob/main/data/bots/deny-pathological.yaml
-    import: (data)/bots/_deny-pathological.yaml
-  - import: (data)/bots/aggressive-brazilian-scrapers.yaml
-
-  # Aggressively block AI/LLM related bots/agents by default
-  - import: (data)/meta/ai-block-aggressive.yaml
-
-  # Consider replacing the aggressive AI policy with more selective policies:
-  # - import: (data)/meta/ai-block-moderate.yaml
-  # - import: (data)/meta/ai-block-permissive.yaml
-
-  # Search engine crawlers to allow, defaults to:
-  #   - Google (so they don't try to bypass Anubis)
-  #   - Apple
-  #   - Bing
-  #   - DuckDuckGo
-  #   - Qwant
-  #   - The Internet Archive
-  #   - Kagi
-  #   - Marginalia
-  #   - Mojeek
-  - import: (data)/crawlers/_allow-good.yaml
-  # Challenge Firefox AI previews
-  - import: (data)/clients/x-firefox-ai.yaml
-
-  # Allow common "keeping the internet working" routes (well-known, favicon, robots.txt)
-  - import: (data)/common/keep-internet-working.yaml
-
-  # # Punish any bot with "bot" in the user-agent string
-  # # This is known to have a high false-positive rate, use at your own risk
-  # - name: generic-bot-catchall
-  #   user_agent_regex: (?i:bot|crawler)
-  #   action: CHALLENGE
-  #   challenge:
-  #     difficulty: 16  # impossible
-  #     report_as: 4    # lie to the operator
-  #     algorithm: slow # intentionally waste CPU cycles and time
-
-  # Generic catchall rule
-  - name: generic-browser
-    user_agent_regex: >-
-      Mozilla|Opera
-    action: CHALLENGE
-    challenge:
-      difficulty: 1 # Number of seconds to wait before refreshing the page
-      report_as: 4 # Unused by this challenge method
-      algorithm: metarefresh # Specify a non-JS challenge method
-
-dnsbl: false
-
-# By default, send HTTP 200 back to clients that either get issued a challenge
-# or a denial. This seems weird, but this is load-bearing due to the fact that
-# the most aggressive scraper bots seem to really, really, want an HTTP 200 and
-# will stop sending requests once they get it.
-status_codes:
-  CHALLENGE: 200
-  DENY: 200

docs/manifest/deployment.yaml

@@ -11,58 +11,48 @@ spec:
       labels:
         app: anubis-docs
     spec:
-      volumes:
-        - name: anubis
-          configMap:
-            name: anubis-cfg
       containers:
-        - name: anubis-docs
-          image: ghcr.io/techarohq/anubis/docs:main
-          imagePullPolicy: Always
-          resources:
-            limits:
-              memory: "128Mi"
-              cpu: "500m"
-            requests:
-              cpu: 250m
-              memory: 128Mi
-          ports:
-            - containerPort: 80
-        - name: anubis
-          image: ghcr.io/techarohq/anubis:main
-          imagePullPolicy: Always
-          env:
-            - name: "BIND"
-              value: ":8081"
-            - name: "DIFFICULTY"
-              value: "4"
-            - name: "METRICS_BIND"
-              value: ":9090"
-            - name: "POLICY_FNAME"
-              value: "/xe/cfg/anubis/botPolicies.yaml"
-            - name: "SERVE_ROBOTS_TXT"
-              value: "false"
-            - name: "TARGET"
-              value: "http://localhost:80"
-            # - name: "SLOG_LEVEL"
-            #   value: "debug"
-          volumeMounts:
-            - name: anubis
-              mountPath: /xe/cfg/anubis
-          resources:
-            limits:
-              cpu: 500m
-              memory: 128Mi
-            requests:
-              cpu: 250m
-              memory: 128Mi
-          securityContext:
-            runAsUser: 1000
-            runAsGroup: 1000
-            runAsNonRoot: true
-            allowPrivilegeEscalation: false
-            capabilities:
-              drop:
-                - ALL
-            seccompProfile:
-              type: RuntimeDefault
+      - name: anubis-docs
+        image: ghcr.io/techarohq/anubis/docs:main
+        imagePullPolicy: Always
+        resources:
+          limits:
+            memory: "128Mi"
+            cpu: "500m"
+        ports:
+        - containerPort: 80
+      - name: anubis
+        image: ghcr.io/techarohq/anubis:main
+        imagePullPolicy: Always
+        env:
+          - name: "BIND"
+            value: ":8081"
+          - name: "DIFFICULTY"
+            value: "4"
+          - name: "METRICS_BIND"
+            value: ":9090"
+          - name: "POLICY_FNAME"
+            value: ""
+          - name: "SERVE_ROBOTS_TXT"
+            value: "false"
+          - name: "TARGET"
+            value: "http://localhost:80"
+          # - name: "SLOG_LEVEL"
+          #   value: "debug"
+        resources:
+          limits:
+            cpu: 500m
+            memory: 128Mi
+          requests:
+            cpu: 250m
+            memory: 128Mi
+        securityContext:
+          runAsUser: 1000
+          runAsGroup: 1000
+          runAsNonRoot: true
+          allowPrivilegeEscalation: false
+          capabilities:
+            drop:
+              - ALL
+          seccompProfile:
+            type: RuntimeDefault

docs/manifest/kustomization.yaml

@@ -2,10 +2,4 @@ resources:
   - deployment.yaml
   - ingress.yaml
   - onionservice.yaml
-  - service.yaml
-
-configMapGenerator:
-  - name: anubis-cfg
-    behavior: create
-    files:
-      - ./cfg/anubis/botPolicies.yaml
+  - service.yaml

go.mod

@@ -3,31 +3,37 @@ module github.com/TecharoHQ/anubis
 go 1.24.2
 
 require (
-	github.com/a-h/templ v0.3.898
+	github.com/TecharoHQ/thoth-proto v0.2.0
+	github.com/a-h/templ v0.3.865
 	github.com/facebookgo/flagenv v0.0.0-20160425205200-fcd59fca7456
+	github.com/gaissmai/bart v0.20.4
 	github.com/golang-jwt/jwt/v5 v5.2.2
 	github.com/google/cel-go v0.25.0
+	github.com/grpc-ecosystem/go-grpc-middleware/providers/prometheus v1.0.1
+	github.com/joho/godotenv v1.5.1
 	github.com/playwright-community/playwright-go v0.5200.0
 	github.com/prometheus/client_golang v1.22.0
 	github.com/sebest/xff v0.0.0-20210106013422-671bd2870b3a
 	github.com/yl2chen/cidranger v1.0.2
-	golang.org/x/net v0.41.0
-	k8s.io/apimachinery v0.33.1
+	golang.org/x/net v0.40.0
+	google.golang.org/grpc v1.72.1
+	k8s.io/apimachinery v0.33.0
 )
 
 require (
 	al.essio.dev/pkg/shellescape v1.6.0 // indirect
+	buf.build/gen/go/bufbuild/protovalidate/protocolbuffers/go v1.36.6-20250425153114-8976f5be98c1.1 // indirect
 	cel.dev/expr v0.23.1 // indirect
-	dario.cat/mergo v1.0.2 // indirect
+	dario.cat/mergo v1.0.1 // indirect
 	github.com/AlekSi/pointer v1.2.0 // indirect
 	github.com/BurntSushi/toml v1.4.1-0.20240526193622-a339e1f7089c // indirect
 	github.com/Masterminds/goutils v1.1.1 // indirect
 	github.com/Masterminds/semver/v3 v3.3.1 // indirect
 	github.com/Masterminds/sprig/v3 v3.3.0 // indirect
 	github.com/Microsoft/go-winio v0.6.2 // indirect
-	github.com/ProtonMail/go-crypto v1.2.0 // indirect
+	github.com/ProtonMail/go-crypto v1.1.6 // indirect
 	github.com/Songmu/gitconfig v0.2.0 // indirect
-	github.com/TecharoHQ/yeet v0.6.0 // indirect
+	github.com/TecharoHQ/yeet v0.2.3 // indirect
 	github.com/a-h/parse v0.0.0-20250122154542-74294addb73e // indirect
 	github.com/andybalholm/brotli v1.1.0 // indirect
 	github.com/antlr4-go/antlr/v4 v4.13.0 // indirect
@@ -40,7 +46,7 @@ require (
 	github.com/cli/go-gh v0.1.0 // indirect
 	github.com/cloudflare/circl v1.6.0 // indirect
 	github.com/cyphar/filepath-securejoin v0.4.1 // indirect
-	github.com/deckarep/golang-set/v2 v2.8.0 // indirect
+	github.com/deckarep/golang-set/v2 v2.7.0 // indirect
 	github.com/dlclark/regexp2 v1.11.4 // indirect
 	github.com/dop251/goja v0.0.0-20250309171923-bcd7cc6bf64c // indirect
 	github.com/emirpasic/gods v1.18.1 // indirect
@@ -59,11 +65,12 @@ require (
 	github.com/goccy/go-yaml v1.12.0 // indirect
 	github.com/golang/groupcache v0.0.0-20241129210726-2c02b8208cf8 // indirect
 	github.com/google/pprof v0.0.0-20230207041349-798e818bf904 // indirect
-	github.com/google/rpmpack v0.6.1-0.20250405124433-758cc6896cbc // indirect
+	github.com/google/rpmpack v0.6.1-0.20240329070804-c2247cbb881a // indirect
 	github.com/google/uuid v1.6.0 // indirect
 	github.com/goreleaser/chglog v0.7.0 // indirect
 	github.com/goreleaser/fileglob v1.3.0 // indirect
-	github.com/goreleaser/nfpm/v2 v2.42.1 // indirect
+	github.com/goreleaser/nfpm/v2 v2.42.0 // indirect
+	github.com/grpc-ecosystem/go-grpc-middleware/v2 v2.1.0 // indirect
 	github.com/huandu/xstrings v1.5.0 // indirect
 	github.com/jbenet/go-context v0.0.0-20150711004518-d14ea06fba99 // indirect
 	github.com/kevinburke/ssh_config v1.2.0 // indirect
@@ -84,29 +91,27 @@ require (
 	github.com/shopspring/decimal v1.4.0 // indirect
 	github.com/skeema/knownhosts v1.3.1 // indirect
 	github.com/spf13/cast v1.7.1 // indirect
-	github.com/stoewer/go-strcase v1.2.0 // indirect
+	github.com/stoewer/go-strcase v1.3.0 // indirect
 	github.com/ulikunitz/xz v0.5.12 // indirect
 	github.com/xanzy/ssh-agent v0.3.3 // indirect
 	gitlab.com/digitalxero/go-conventional-commit v1.0.7 // indirect
-	golang.org/x/crypto v0.39.0 // indirect
+	golang.org/x/crypto v0.38.0 // indirect
 	golang.org/x/exp v0.0.0-20240719175910-8a7402abbf56 // indirect
 	golang.org/x/exp/typeparams v0.0.0-20231108232855-2478ac86f678 // indirect
-	golang.org/x/mod v0.25.0 // indirect
-	golang.org/x/sync v0.15.0 // indirect
+	golang.org/x/mod v0.24.0 // indirect
+	golang.org/x/sync v0.14.0 // indirect
 	golang.org/x/sys v0.33.0 // indirect
 	golang.org/x/telemetry v0.0.0-20240522233618-39ace7a40ae7 // indirect
-	golang.org/x/term v0.32.0 // indirect
-	golang.org/x/text v0.26.0 // indirect
-	golang.org/x/tools v0.33.0 // indirect
+	golang.org/x/text v0.25.0 // indirect
+	golang.org/x/tools v0.32.0 // indirect
 	golang.org/x/vuln v1.1.4 // indirect
 	golang.org/x/xerrors v0.0.0-20240716161551-93cc26a95ae9 // indirect
-	google.golang.org/genproto/googleapis/api v0.0.0-20240826202546-f6391c0de4c7 // indirect
-	google.golang.org/genproto/googleapis/rpc v0.0.0-20240826202546-f6391c0de4c7 // indirect
-	google.golang.org/protobuf v1.36.5 // indirect
+	google.golang.org/genproto/googleapis/api v0.0.0-20250218202821-56aae31c358a // indirect
+	google.golang.org/genproto/googleapis/rpc v0.0.0-20250218202821-56aae31c358a // indirect
+	google.golang.org/protobuf v1.36.6 // indirect
 	gopkg.in/warnings.v0 v0.1.2 // indirect
 	gopkg.in/yaml.v3 v3.0.1 // indirect
 	honnef.co/go/tools v0.6.1 // indirect
-	mvdan.cc/sh/v3 v3.11.0 // indirect
 	sigs.k8s.io/json v0.0.0-20241010143419-9aa6b5e7a4b3 // indirect
 	sigs.k8s.io/yaml v1.4.0 // indirect
 )

go.sum

@@ -1,9 +1,11 @@
 al.essio.dev/pkg/shellescape v1.6.0 h1:NxFcEqzFSEVCGN2yq7Huv/9hyCEGVa/TncnOOBBeXHA=
 al.essio.dev/pkg/shellescape v1.6.0/go.mod h1:6sIqp7X2P6mThCQ7twERpZTuigpr6KbZWtls1U8I890=
+buf.build/gen/go/bufbuild/protovalidate/protocolbuffers/go v1.36.6-20250425153114-8976f5be98c1.1 h1:YhMSc48s25kr7kv31Z8vf7sPUIq5YJva9z1mn/hAt0M=
+buf.build/gen/go/bufbuild/protovalidate/protocolbuffers/go v1.36.6-20250425153114-8976f5be98c1.1/go.mod h1:avRlCjnFzl98VPaeCtJ24RrV/wwHFzB8sWXhj26+n/U=
 cel.dev/expr v0.23.1 h1:K4KOtPCJQjVggkARsjG9RWXP6O4R73aHeJMa/dmCQQg=
 cel.dev/expr v0.23.1/go.mod h1:hLPLo1W4QUmuYdA72RBX06QTs6MXw941piREPl3Yfiw=
-dario.cat/mergo v1.0.2 h1:85+piFYR1tMbRrLcDwR18y4UKJ3aH1Tbzi24VRW1TK8=
-dario.cat/mergo v1.0.2/go.mod h1:E/hbnu0NxMFBjpMIE34DRGLWqDy0g5FuKDhCb31ngxA=
+dario.cat/mergo v1.0.1 h1:Ra4+bf83h2ztPIQYNP99R6m+Y7KfnARDfID+a+vLl4s=
+dario.cat/mergo v1.0.1/go.mod h1:uNxQE+84aUszobStD9th8a29P2fMDhsBdgRYvZOxGmk=
 github.com/AlekSi/pointer v1.2.0 h1:glcy/gc4h8HnG2Z3ZECSzZ1IX1x2JxRVuDzaJwQE0+w=
 github.com/AlekSi/pointer v1.2.0/go.mod h1:gZGfd3dpW4vEc/UlyfKKi1roIqcCgwOIvb0tSNSBle0=
 github.com/BurntSushi/toml v1.4.1-0.20240526193622-a339e1f7089c h1:pxW6RcqyfI9/kWtOwnv/G+AzdKuy2ZrqINhenH4HyNs=
@@ -20,20 +22,22 @@ github.com/Masterminds/sprig/v3 v3.3.0/go.mod h1:Zy1iXRYNqNLUolqCpL4uhk6SHUMAOSC
 github.com/Microsoft/go-winio v0.5.2/go.mod h1:WpS1mjBmmwHBEWmogvA2mj8546UReBk4v8QkMxJ6pZY=
 github.com/Microsoft/go-winio v0.6.2 h1:F2VQgta7ecxGYO8k3ZZz3RS8fVIXVxONVUPlNERoyfY=
 github.com/Microsoft/go-winio v0.6.2/go.mod h1:yd8OoFMLzJbo9gZq8j5qaps8bJ9aShtEA8Ipt1oGCvU=
-github.com/ProtonMail/go-crypto v1.2.0 h1:+PhXXn4SPGd+qk76TlEePBfOfivE0zkWFenhGhFLzWs=
-github.com/ProtonMail/go-crypto v1.2.0/go.mod h1:9whxjD8Rbs29b4XWbB8irEcE8KHMqaR2e7GWU1R+/PE=
+github.com/ProtonMail/go-crypto v1.1.6 h1:ZcV+Ropw6Qn0AX9brlQLAUXfqLBc7Bl+f/DmNxpLfdw=
+github.com/ProtonMail/go-crypto v1.1.6/go.mod h1:rA3QumHc/FZ8pAHreoekgiAbzpNsfQAosU5td4SnOrE=
 github.com/ProtonMail/go-mime v0.0.0-20230322103455-7d82a3887f2f h1:tCbYj7/299ekTTXpdwKYF8eBlsYsDVoggDAuAjoK66k=
 github.com/ProtonMail/go-mime v0.0.0-20230322103455-7d82a3887f2f/go.mod h1:gcr0kNtGBqin9zDW9GOHcVntrwnjrK+qdJ06mWYBybw=
 github.com/ProtonMail/gopenpgp/v2 v2.7.1 h1:Awsg7MPc2gD3I7IFac2qE3Gdls0lZW8SzrFZ3k1oz0s=
 github.com/ProtonMail/gopenpgp/v2 v2.7.1/go.mod h1:/BU5gfAVwqyd8EfC3Eu7zmuhwYQpKs+cGD8M//iiaxs=
 github.com/Songmu/gitconfig v0.2.0 h1:pX2++u4KUq+K2k/ZCzGXLtkD3ceCqIdi0tDyb+IbSyo=
 github.com/Songmu/gitconfig v0.2.0/go.mod h1:cB5bYJer+pl7W8g6RHFwL/0X6aJROVrYuHlvc7PT+hE=
-github.com/TecharoHQ/yeet v0.6.0 h1:RCBAjr7wIlllsgy0tpvWpLX7jsZgu2tiuBY3RrprcR0=
-github.com/TecharoHQ/yeet v0.6.0/go.mod h1:bj2V4Fg8qKQXoiuPZa3HuawrE8g+LsOQv/9q2WyGSsA=
+github.com/TecharoHQ/thoth-proto v0.2.0 h1:IR/LMbr4phOPgfgmQ+VNBYfckGoo/xr5xlWqsORF8/8=
+github.com/TecharoHQ/thoth-proto v0.2.0/go.mod h1:wIkQ7hMmNk2XZXRVeL1WcioD4sc1pCCEAHbJ8hKG51A=
+github.com/TecharoHQ/yeet v0.2.3 h1:Pcsnq5HTnk4Xntlu/FNEidH7x55bIx+f5Mk1hpVIngs=
+github.com/TecharoHQ/yeet v0.2.3/go.mod h1:avLiwxZpNY37A/o35XledvdmGnTkm3G7+Oskxca6Z7Y=
 github.com/a-h/parse v0.0.0-20250122154542-74294addb73e h1:HjVbSQHy+dnlS6C3XajZ69NYAb5jbGNfHanvm1+iYlo=
 github.com/a-h/parse v0.0.0-20250122154542-74294addb73e/go.mod h1:3mnrkvGpurZ4ZrTDbYU84xhwXW2TjTKShSwjRi2ihfQ=
-github.com/a-h/templ v0.3.898 h1:g9oxL/dmM6tvwRe2egJS8hBDQTncokbMoOFk1oJMX7s=
-github.com/a-h/templ v0.3.898/go.mod h1:oLBbZVQ6//Q6zpvSMPTuBK0F3qOtBdFBcGRspcT+VNQ=
+github.com/a-h/templ v0.3.865 h1:nYn5EWm9EiXaDgWcMQaKiKvrydqgxDUtT1+4zU2C43A=
+github.com/a-h/templ v0.3.865/go.mod h1:oLBbZVQ6//Q6zpvSMPTuBK0F3qOtBdFBcGRspcT+VNQ=
 github.com/andybalholm/brotli v1.1.0 h1:eLKJA0d02Lf0mVpIDgYnqXcUn0GqVmEFny3VuID1U3M=
 github.com/andybalholm/brotli v1.1.0/go.mod h1:sms7XGricyQI9K10gOSf56VKKWS4oLer58Q+mhRPtnY=
 github.com/anmitsu/go-shlex v0.0.0-20200514113438-38f4b401e2be h1:9AeTilPcZAjCFIImctFaOjnTIavg87rW78vTPkQqLI8=
@@ -65,16 +69,14 @@ github.com/cli/safeexec v1.0.0/go.mod h1:Z/D4tTN8Vs5gXYHDCbaM1S/anmEDnJb1iW0+EJ5
 github.com/cli/shurcooL-graphql v0.0.1/go.mod h1:U7gCSuMZP/Qy7kbqkk5PrqXEeDgtfG5K+W+u8weorps=
 github.com/cloudflare/circl v1.6.0 h1:cr5JKic4HI+LkINy2lg3W2jF8sHCVTBncJr5gIIq7qk=
 github.com/cloudflare/circl v1.6.0/go.mod h1:uddAzsPgqdMAYatqJ0lsjX1oECcQLIlRpzZh3pJrofs=
-github.com/creack/pty v1.1.24 h1:bJrF4RRfyJnbTJqzRLHzcGaZK1NeM5kTC9jGgovnR1s=
-github.com/creack/pty v1.1.24/go.mod h1:08sCNb52WyoAwi2QDyzUCTgcvVFhUzewun7wtTfvcwE=
 github.com/cyphar/filepath-securejoin v0.4.1 h1:JyxxyPEaktOD+GAnqIqTf9A8tHyAG22rowi7HkoSU1s=
 github.com/cyphar/filepath-securejoin v0.4.1/go.mod h1:Sdj7gXlvMcPZsbhwhQ33GguGLDGQL7h7bg04C/+u9jI=
 github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
 github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
 github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc h1:U9qPSI2PIWSS1VwoXQT9A3Wy9MM3WgvqSxFWenqJduM=
 github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
-github.com/deckarep/golang-set/v2 v2.8.0 h1:swm0rlPCmdWn9mESxKOjWk8hXSqoxOp+ZlfuyaAdFlQ=
-github.com/deckarep/golang-set/v2 v2.8.0/go.mod h1:VAky9rY/yGXJOLEDv3OMci+7wtDpOF4IN+y82NBOac4=
+github.com/deckarep/golang-set/v2 v2.7.0 h1:gIloKvD7yH2oip4VLhsv3JyLLFnC0Y2mlusgcvJYW5k=
+github.com/deckarep/golang-set/v2 v2.7.0/go.mod h1:VAky9rY/yGXJOLEDv3OMci+7wtDpOF4IN+y82NBOac4=
 github.com/dlclark/regexp2 v1.11.4 h1:rPYF9/LECdNymJufQKmri9gV604RvvABwgOA8un7yAo=
 github.com/dlclark/regexp2 v1.11.4/go.mod h1:DHkYz0B9wPfa6wondMfaivmHpzrQ3v9q8cnmRbL6yW8=
 github.com/dop251/goja v0.0.0-20250309171923-bcd7cc6bf64c h1:mxWGS0YyquJ/ikZOjSrRjjFIbUqIP9ojyYQ+QZTU3Rg=
@@ -99,6 +101,8 @@ github.com/frankban/quicktest v1.14.6 h1:7Xjx+VpznH+oBnejlPUj8oUpdxnVs4f8XU8WnHk
 github.com/frankban/quicktest v1.14.6/go.mod h1:4ptaffx2x8+WTWXmUCuVU6aPUX1/Mz7zb5vbUoiM6w0=
 github.com/fsnotify/fsnotify v1.8.0 h1:dAwr6QBTBZIkG8roQaJjGof0pp0EeF+tNV7YBP3F/8M=
 github.com/fsnotify/fsnotify v1.8.0/go.mod h1:8jBTzvmWwFyi3Pb8djgCCO5IBqzKJ/Jwo8TRcHyHii0=
+github.com/gaissmai/bart v0.20.4 h1:Ik47r1fy3jRVU+1eYzKSW3ho2UgBVTVnUS8O993584U=
+github.com/gaissmai/bart v0.20.4/go.mod h1:cEed+ge8dalcbpi8wtS9x9m2hn/fNJH5suhdGQOHnYk=
 github.com/gliderlabs/ssh v0.3.8 h1:a4YXD1V7xMF9g5nTkdfnja3Sxy1PVDCj1Zg4Wb8vY6c=
 github.com/gliderlabs/ssh v0.3.8/go.mod h1:xYoytBv1sV0aL3CavoDuJIQNURXkkfPA/wxQ1pL1fAU=
 github.com/go-git/gcfg v1.5.1-0.20230307220236-3a3c6141e376 h1:+zs/tPmkDkHx3U66DAb0lQFJrpS6731Oaa12ikc+DiI=
@@ -111,6 +115,10 @@ github.com/go-git/go-git/v5 v5.14.0 h1:/MD3lCrGjCen5WfEAzKg00MJJffKhC8gzS80ycmCi
 github.com/go-git/go-git/v5 v5.14.0/go.mod h1:Z5Xhoia5PcWA3NF8vRLURn9E5FRhSl7dGj9ItW3Wk5k=
 github.com/go-jose/go-jose/v3 v3.0.4 h1:Wp5HA7bLQcKnf6YYao/4kpRpVMp/yf6+pJKV8WFSaNY=
 github.com/go-jose/go-jose/v3 v3.0.4/go.mod h1:5b+7YgP7ZICgJDBdfjZaIt+H/9L9T/YQrVfLAMboGkQ=
+github.com/go-logr/logr v1.4.2 h1:6pFjapn8bFcIbiKo3XT4j/BhANplGihG6tvd+8rYgrY=
+github.com/go-logr/logr v1.4.2/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY=
+github.com/go-logr/stdr v1.2.2 h1:hSWxHoqTgW2S2qGc0LTAI563KZ5YKYRhT3MFKZMbjag=
+github.com/go-logr/stdr v1.2.2/go.mod h1:mMo/vtBO5dYbehREoey6XUKy/eSumjCCveDpRre4VKE=
 github.com/go-playground/assert/v2 v2.0.1/go.mod h1:VDjEfimB/XKnb+ZQfWdccd7VUvScMdVu0Titje2rxJ4=
 github.com/go-playground/locales v0.13.0 h1:HyWk6mgj5qFqCT5fjGBuRArbVDfE4hi8+e8ceBS/t7Q=
 github.com/go-playground/locales v0.13.0/go.mod h1:taPMhCMXrRLJO55olJkUXHZBHCxTMfnGwq/HNwmWNS8=
@@ -119,8 +127,6 @@ github.com/go-playground/universal-translator v0.17.0/go.mod h1:UkSxE5sNxxRwHyU+
 github.com/go-playground/validator/v10 v10.4.1/go.mod h1:nlOn6nFhuKACm19sB/8EGNn9GlaMV7XkbRSipzJ0Ii4=
 github.com/go-playground/validator/v10 v10.10.0 h1:I7mrTYv78z8k8VXa/qJlOlEXn/nBh+BF8dHX5nt/dr0=
 github.com/go-playground/validator/v10 v10.10.0/go.mod h1:74x4gJWsvQexRdW8Pn3dXSGrTK4nAUsbPlLADvpJkos=
-github.com/go-quicktest/qt v1.101.0 h1:O1K29Txy5P2OK0dGo59b7b0LR6wKfIhttaAhHUyn7eI=
-github.com/go-quicktest/qt v1.101.0/go.mod h1:14Bz/f7NwaXPtdYEgzsx46kqSxVwTbzVZsDC26tQJow=
 github.com/go-sourcemap/sourcemap v2.1.3+incompatible h1:W1iEw64niKVGogNgBN3ePyLFfuisuzeidWPMPWmECqU=
 github.com/go-sourcemap/sourcemap v2.1.3+incompatible/go.mod h1:F8jJfvm2KbVjc5NqelyYJmf/v5J0dwNLS2mL4sNA1Jg=
 github.com/go-stack/stack v1.8.1 h1:ntEHSVwIt7PNXNpgPmVfMrNhLtgjlmnZha2kOpuRiDw=
@@ -134,6 +140,8 @@ github.com/golang-jwt/jwt/v5 v5.2.2 h1:Rl4B7itRWVtYIHFrSNd7vhTiz9UpLdi6gZhZ3wEeD
 github.com/golang-jwt/jwt/v5 v5.2.2/go.mod h1:pqrtFR0X4osieyHYxtmOUWsAWrfe1Q5UVIyoH402zdk=
 github.com/golang/groupcache v0.0.0-20241129210726-2c02b8208cf8 h1:f+oWsMOmNPc8JmEHVZIycC7hBoQxHH9pNKQORJNozsQ=
 github.com/golang/groupcache v0.0.0-20241129210726-2c02b8208cf8/go.mod h1:wcDNUvekVysuuOpQKo3191zZyTpiI6se1N1ULghS0sw=
+github.com/golang/protobuf v1.5.4 h1:i7eJL8qZTpSEXOPTxNKhASYpMn+8e5Q6AdndVa1dWek=
+github.com/golang/protobuf v1.5.4/go.mod h1:lnTiLA8Wa4RWRcIUkrtSVa5nRhsEGBg48fD6rSs7xps=
 github.com/google/cel-go v0.25.0 h1:jsFw9Fhn+3y2kBbltZR4VEz5xKkcIFRPDnuEzAGv5GY=
 github.com/google/cel-go v0.25.0/go.mod h1:hjEb6r5SuOSlhCHmFoLzu8HGCERvIsDAbxDAyNU/MmI=
 github.com/google/go-cmdtest v0.4.1-0.20220921163831-55ab3332a786 h1:rcv+Ippz6RAtvaGgKxc+8FQIpxHgsF+HBzPyYL2cyVU=
@@ -145,8 +153,8 @@ github.com/google/pprof v0.0.0-20230207041349-798e818bf904 h1:4/hN5RUoecvl+RmJRE
 github.com/google/pprof v0.0.0-20230207041349-798e818bf904/go.mod h1:uglQLonpP8qtYCYyzA+8c/9qtqgA3qsXGYqCPKARAFg=
 github.com/google/renameio v0.1.0 h1:GOZbcHa3HfsPKPlmyPyN2KEohoMXOhdMbHrvbpl2QaA=
 github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI=
-github.com/google/rpmpack v0.6.1-0.20250405124433-758cc6896cbc h1:qES+d3PvR9CN+zARQQH/bNXH0ybzmdjNMHICrBwXD28=
-github.com/google/rpmpack v0.6.1-0.20250405124433-758cc6896cbc/go.mod h1:uqVAUVQLq8UY2hCDfmJ/+rtO3aw7qyhc90rCVEabEfI=
+github.com/google/rpmpack v0.6.1-0.20240329070804-c2247cbb881a h1:JJBdjSfqSy3mnDT0940ASQFghwcZ4y4cb6ttjAoXqwE=
+github.com/google/rpmpack v0.6.1-0.20240329070804-c2247cbb881a/go.mod h1:uqVAUVQLq8UY2hCDfmJ/+rtO3aw7qyhc90rCVEabEfI=
 github.com/google/shlex v0.0.0-20191202100458-e7afc7fbc510 h1:El6M4kTTCOh6aBiKaUGG7oYTSPP8MxqL4YI3kZKwcP4=
 github.com/google/shlex v0.0.0-20191202100458-e7afc7fbc510/go.mod h1:pupxD2MaaD3pAXIBCelhxNneeOaAeabZDe5s4K6zSpQ=
 github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0=
@@ -157,14 +165,20 @@ github.com/goreleaser/chglog v0.7.0 h1:/KzXWAeg4DrEz4r3OI6K2Yb8RAsVGeInCUfLWFXL9
 github.com/goreleaser/chglog v0.7.0/go.mod h1:2h/yyq9xvTUeM9tOoucBP+jri8Dj28splx+SjlYkklc=
 github.com/goreleaser/fileglob v1.3.0 h1:/X6J7U8lbDpQtBvGcwwPS6OpzkNVlVEsFUVRx9+k+7I=
 github.com/goreleaser/fileglob v1.3.0/go.mod h1:Jx6BoXv3mbYkEzwm9THo7xbr5egkAraxkGorbJb4RxU=
-github.com/goreleaser/nfpm/v2 v2.42.1 h1:xu2pLRgQuz2ab+YZFoeIzwU/M5jjjCKDGwv1lRbVGvk=
-github.com/goreleaser/nfpm/v2 v2.42.1/go.mod h1:dY53KWYKebkOocxgkmpM7SRX0Nv5hU+jEu2kIaM4/LI=
+github.com/goreleaser/nfpm/v2 v2.42.0 h1:7BW4WQWyvZDrT0C7SyWop+J8rtqFyTB17Sb2/j/NxMI=
+github.com/goreleaser/nfpm/v2 v2.42.0/go.mod h1:DtNL+nKpfB8sMFZp+X7Xu3W64atyZYtTnYe8O925/mg=
+github.com/grpc-ecosystem/go-grpc-middleware/providers/prometheus v1.0.1 h1:qnpSQwGEnkcRpTqNOIR6bJbR0gAorgP9CSALpRcKoAA=
+github.com/grpc-ecosystem/go-grpc-middleware/providers/prometheus v1.0.1/go.mod h1:lXGCsh6c22WGtjr+qGHj1otzZpV/1kwTMAqkwZsnWRU=
+github.com/grpc-ecosystem/go-grpc-middleware/v2 v2.1.0 h1:pRhl55Yx1eC7BZ1N+BBWwnKaMyD8uC+34TLdndZMAKk=
+github.com/grpc-ecosystem/go-grpc-middleware/v2 v2.1.0/go.mod h1:XKMd7iuf/RGPSMJ/U4HP0zS2Z9Fh8Ps9a+6X26m/tmI=
 github.com/h2non/parth v0.0.0-20190131123155-b4df798d6542/go.mod h1:Ow0tF8D4Kplbc8s8sSb3V2oUCygFHVp8gC3Dn6U4MNI=
 github.com/henvic/httpretty v0.0.6/go.mod h1:X38wLjWXHkXT7r2+uK8LjCMne9rsuNaBLJ+5cU2/Pmo=
 github.com/huandu/xstrings v1.5.0 h1:2ag3IFq9ZDANvthTwTiqSSZLjDc+BedvHPAp5tJy2TI=
 github.com/huandu/xstrings v1.5.0/go.mod h1:y5/lhBue+AyNmUVz9RLU9xbLR0o4KIIExikq4ovT0aE=
 github.com/jbenet/go-context v0.0.0-20150711004518-d14ea06fba99 h1:BQSFePA1RWJOlocH6Fxy8MmwDt+yVQYULKfN0RoTN8A=
 github.com/jbenet/go-context v0.0.0-20150711004518-d14ea06fba99/go.mod h1:1lJo3i6rXxKeerYnT8Nvf0QmHCRC1n8sfWVwXF2Frvo=
+github.com/joho/godotenv v1.5.1 h1:7eLL/+HRGLY0ldzfGMeQkb7vMd0as4CfYvUVzLqw0N0=
+github.com/joho/godotenv v1.5.1/go.mod h1:f4LDr5Voq0i2e/R5DDNOoa2zzDfwtkZa6DnEwAbqwq4=
 github.com/jtolds/gls v4.20.0+incompatible h1:xdiiI2gbIgH/gLH7ADydsJ1uDOEzR8yvV7C0MuV77Wo=
 github.com/jtolds/gls v4.20.0+incompatible/go.mod h1:QJZ7F/aHp+rZTRtaJ1ow/lLfFfVYBRgL+9YlvaHOwJU=
 github.com/kevinburke/ssh_config v1.2.0 h1:x584FjTGwHzMwvHx18PXxbBVzfnxogHaAReU4gf13a4=
@@ -248,13 +262,17 @@ github.com/smartystreets/goconvey v1.8.1 h1:qGjIddxOk4grTu9JPOU31tVfq3cNdBlNa5sS
 github.com/smartystreets/goconvey v1.8.1/go.mod h1:+/u4qLyY6x1jReYOp7GOM2FSt8aP9CzCZL03bI28W60=
 github.com/spf13/cast v1.7.1 h1:cuNEagBQEHWN1FnbGEjCXL2szYEXqfJPbP2HNUaca9Y=
 github.com/spf13/cast v1.7.1/go.mod h1:ancEpBxwJDODSW/UG4rDrAqiKolqNNh2DX3mk86cAdo=
-github.com/stoewer/go-strcase v1.2.0 h1:Z2iHWqGXH00XYgqDmNgQbIBxf3wrNq0F3feEy0ainaU=
-github.com/stoewer/go-strcase v1.2.0/go.mod h1:IBiWB2sKIp3wVVQ3Y035++gc+knqhUQag1KpM8ahLw8=
+github.com/stoewer/go-strcase v1.3.0 h1:g0eASXYtp+yvN9fK8sH94oCIk0fau9uV1/ZdJ0AVEzs=
+github.com/stoewer/go-strcase v1.3.0/go.mod h1:fAH5hQ5pehh+j3nZfvwdk2RgEgQjAoM8wodgtPmh1xo=
 github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
+github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw=
+github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo=
 github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs=
 github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4=
-github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA=
 github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
+github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
+github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU=
+github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4=
 github.com/stretchr/testify v1.10.0 h1:Xv5erBjTwe/5IxqUQTdXv5kgmIvbHo3QQyRwhJsOfJA=
 github.com/stretchr/testify v1.10.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY=
 github.com/thlib/go-timezone-local v0.0.0-20210907160436-ef149e42d28e/go.mod h1:/Tnicc6m/lsJE0irFMA0LfIwTBo4QP7A8IfyIv4zZKI=
@@ -269,21 +287,33 @@ github.com/yl2chen/cidranger v1.0.2/go.mod h1:9U1yz7WPYDwf0vpNWFaeRh0bjwz5RVgRy/
 github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY=
 gitlab.com/digitalxero/go-conventional-commit v1.0.7 h1:8/dO6WWG+98PMhlZowt/YjuiKhqhGlOCwlIV8SqqGh8=
 gitlab.com/digitalxero/go-conventional-commit v1.0.7/go.mod h1:05Xc2BFsSyC5tKhK0y+P3bs0AwUtNuTp+mTpbCU/DZ0=
+go.opentelemetry.io/auto/sdk v1.1.0 h1:cH53jehLUN6UFLY71z+NDOiNJqDdPRaXzTel0sJySYA=
+go.opentelemetry.io/auto/sdk v1.1.0/go.mod h1:3wSPjt5PWp2RhlCcmmOial7AvC4DQqZb7a7wCow3W8A=
+go.opentelemetry.io/otel v1.34.0 h1:zRLXxLCgL1WyKsPVrgbSdMN4c0FMkDAskSTQP+0hdUY=
+go.opentelemetry.io/otel v1.34.0/go.mod h1:OWFPOQ+h4G8xpyjgqo4SxJYdDQ/qmRH+wivy7zzx9oI=
+go.opentelemetry.io/otel/metric v1.34.0 h1:+eTR3U0MyfWjRDhmFMxe2SsW64QrZ84AOhvqS7Y+PoQ=
+go.opentelemetry.io/otel/metric v1.34.0/go.mod h1:CEDrp0fy2D0MvkXE+dPV7cMi8tWZwX3dmaIhwPOaqHE=
+go.opentelemetry.io/otel/sdk v1.34.0 h1:95zS4k/2GOy069d321O8jWgYsW3MzVV+KuSPKp7Wr1A=
+go.opentelemetry.io/otel/sdk v1.34.0/go.mod h1:0e/pNiaMAqaykJGKbi+tSjWfNNHMTxoC9qANsCzbyxU=
+go.opentelemetry.io/otel/sdk/metric v1.34.0 h1:5CeK9ujjbFVL5c1PhLuStg1wxA7vQv7ce1EK0Gyvahk=
+go.opentelemetry.io/otel/sdk/metric v1.34.0/go.mod h1:jQ/r8Ze28zRKoNRdkjCZxfs6YvBTG1+YIqyFVFYec5w=
+go.opentelemetry.io/otel/trace v1.34.0 h1:+ouXS2V8Rd4hp4580a8q23bg0azF2nI8cqLYnC8mh/k=
+go.opentelemetry.io/otel/trace v1.34.0/go.mod h1:Svm7lSjQD7kG7KJ/MUHPVXSDGz2OX4h0M2jHBhmSfRE=
 golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
 golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
 golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc=
 golang.org/x/crypto v0.0.0-20220622213112-05595931fe9d/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4=
 golang.org/x/crypto v0.19.0/go.mod h1:Iy9bg/ha4yyC70EfRS8jz+B6ybOBKMaSxLj6P6oBDfU=
-golang.org/x/crypto v0.39.0 h1:SHs+kF4LP+f+p14esP5jAoDpHU8Gu/v9lFRK6IT5imM=
-golang.org/x/crypto v0.39.0/go.mod h1:L+Xg3Wf6HoL4Bn4238Z6ft6KfEpN0tJGo53AAPC632U=
+golang.org/x/crypto v0.38.0 h1:jt+WWG8IZlBnVbomuhg2Mdq0+BBQaHbtqHEFEigjUV8=
+golang.org/x/crypto v0.38.0/go.mod h1:MvrbAqul58NNYPKnOra203SB9vpuZW0e+RRZV+Ggqjw=
 golang.org/x/exp v0.0.0-20240719175910-8a7402abbf56 h1:2dVuKD2vS7b0QIHQbpyTISPd0LeHDbnYEryqj5Q1ug8=
 golang.org/x/exp v0.0.0-20240719175910-8a7402abbf56/go.mod h1:M4RDyNAINzryxdtnbRXRL/OHtkFuWGRjvuhBJpk2IlY=
 golang.org/x/exp/typeparams v0.0.0-20231108232855-2478ac86f678 h1:1P7xPZEwZMoBoz0Yze5Nx2/4pxj6nw9ZqHWXqP0iRgQ=
 golang.org/x/exp/typeparams v0.0.0-20231108232855-2478ac86f678/go.mod h1:AbB0pIl9nAr9wVwH+Z2ZpaocVmF5I4GyWCDIsVjR0bk=
 golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4=
 golang.org/x/mod v0.8.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs=
-golang.org/x/mod v0.25.0 h1:n7a+ZbQKQA/Ysbyb0/6IbB1H/X41mKgbhfv7AfG/44w=
-golang.org/x/mod v0.25.0/go.mod h1:IXM97Txy2VM4PJ3gI61r1YEk/gAj6zAHN3AdZt6S9Ww=
+golang.org/x/mod v0.24.0 h1:ZfthKaKaT4NrhGVZHO1/WDTwGES4De8KtWO0SIbNJMU=
+golang.org/x/mod v0.24.0/go.mod h1:IXM97Txy2VM4PJ3gI61r1YEk/gAj6zAHN3AdZt6S9Ww=
 golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
 golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
 golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg=
@@ -291,13 +321,13 @@ golang.org/x/net v0.0.0-20211112202133-69e39bad7dc2/go.mod h1:9nx3DQGgdP8bBQD5qx
 golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c=
 golang.org/x/net v0.6.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs=
 golang.org/x/net v0.10.0/go.mod h1:0qNGK6F8kojg2nk9dLZ2mShWaEBan6FAoqfSigmmuDg=
-golang.org/x/net v0.41.0 h1:vBTly1HeNPEn3wtREYfy4GZ/NECgw2Cnl+nK6Nz3uvw=
-golang.org/x/net v0.41.0/go.mod h1:B/K4NNqkfmg07DQYrbwvSluqCJOOXwUjeb/5lOisjbA=
+golang.org/x/net v0.40.0 h1:79Xs7wF06Gbdcg4kdCCIQArK11Z1hr5POQ6+fIYHNuY=
+golang.org/x/net v0.40.0/go.mod h1:y0hY0exeL2Pku80/zKK7tpntoX23cqL3Oa6njdgRtds=
 golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.1.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
-golang.org/x/sync v0.15.0 h1:KWH3jNZsfyT6xfAfKiz6MRNmd46ByHDYaZ7KSkCtdW8=
-golang.org/x/sync v0.15.0/go.mod h1:1dzgHSNfp02xaA81J2MS99Qcpr2w7fw1gpm99rleRqA=
+golang.org/x/sync v0.14.0 h1:woo0S4Yywslg6hp4eUFjTVOyKt0RookbpAHG4c1HmhQ=
+golang.org/x/sync v0.14.0/go.mod h1:1dzgHSNfp02xaA81J2MS99Qcpr2w7fw1gpm99rleRqA=
 golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
 golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20191026070338-33540a1f6037/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
@@ -338,14 +368,14 @@ golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ=
 golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8=
 golang.org/x/text v0.9.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8=
 golang.org/x/text v0.14.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU=
-golang.org/x/text v0.26.0 h1:P42AVeLghgTYr4+xUnTRKDMqpar+PtX7KWuNQL21L8M=
-golang.org/x/text v0.26.0/go.mod h1:QK15LZJUUQVJxhz7wXgxSy/CJaTFjd0G+YLonydOVQA=
+golang.org/x/text v0.25.0 h1:qVyWApTSYLk/drJRO5mDlNYskwQznZmkpV2c8q9zls4=
+golang.org/x/text v0.25.0/go.mod h1:WEdwpYrmk1qmdHvhkSTNPm3app7v4rsT8F2UD6+VHIA=
 golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
 golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
 golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc=
 golang.org/x/tools v0.6.0/go.mod h1:Xwgl3UAJ/d3gWutnCtw505GrjyAbvKui8lOU390QaIU=
-golang.org/x/tools v0.33.0 h1:4qz2S3zmRxbGIhDIAgjxvFutSvH5EfnsYrRBj0UI0bc=
-golang.org/x/tools v0.33.0/go.mod h1:CIJMaWEY88juyUfo7UbgPqbC8rU2OqfAV1h2Qp0oMYI=
+golang.org/x/tools v0.32.0 h1:Q7N1vhpkQv7ybVzLFtTjvQya2ewbwNDZzUgfXGqtMWU=
+golang.org/x/tools v0.32.0/go.mod h1:ZxrU41P/wAbZD8EDa6dDCa6XfpkhJ7HFMjHJXfBDu8s=
 golang.org/x/vuln v1.1.4 h1:Ju8QsuyhX3Hk8ma3CesTbO8vfJD9EvUBgHvkxHBzj0I=
 golang.org/x/vuln v1.1.4/go.mod h1:F+45wmU18ym/ca5PLTPLsSzr2KppzswxPP603ldA67s=
 golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
@@ -353,12 +383,14 @@ golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8T
 golang.org/x/xerrors v0.0.0-20220609144429-65e65417b02f/go.mod h1:K8+ghG5WaK9qNqU5K3HdILfMLy1f3aNYFI/wnl100a8=
 golang.org/x/xerrors v0.0.0-20240716161551-93cc26a95ae9 h1:LLhsEBxRTBLuKlQxFBYUOU8xyFgXv6cOTp2HASDlsDk=
 golang.org/x/xerrors v0.0.0-20240716161551-93cc26a95ae9/go.mod h1:NDW/Ps6MPRej6fsCIbMTohpP40sJ/P/vI1MoTEGwX90=
-google.golang.org/genproto/googleapis/api v0.0.0-20240826202546-f6391c0de4c7 h1:YcyjlL1PRr2Q17/I0dPk2JmYS5CDXfcdb2Z3YRioEbw=
-google.golang.org/genproto/googleapis/api v0.0.0-20240826202546-f6391c0de4c7/go.mod h1:OCdP9MfskevB/rbYvHTsXTtKC+3bHWajPdoKgjcYkfo=
-google.golang.org/genproto/googleapis/rpc v0.0.0-20240826202546-f6391c0de4c7 h1:2035KHhUv+EpyB+hWgJnaWKJOdX1E95w2S8Rr4uWKTs=
-google.golang.org/genproto/googleapis/rpc v0.0.0-20240826202546-f6391c0de4c7/go.mod h1:UqMtugtsSgubUsoxbuAoiCXvqvErP7Gf0so0mK9tHxU=
-google.golang.org/protobuf v1.36.5 h1:tPhr+woSbjfYvY6/GPufUoYizxw1cF/yFoxJ2fmpwlM=
-google.golang.org/protobuf v1.36.5/go.mod h1:9fA7Ob0pmnwhb644+1+CVWFRbNajQ6iRojtC/QF5bRE=
+google.golang.org/genproto/googleapis/api v0.0.0-20250218202821-56aae31c358a h1:nwKuGPlUAt+aR+pcrkfFRrTU1BVrSmYyYMxYbUIVHr0=
+google.golang.org/genproto/googleapis/api v0.0.0-20250218202821-56aae31c358a/go.mod h1:3kWAYMk1I75K4vykHtKt2ycnOgpA6974V7bREqbsenU=
+google.golang.org/genproto/googleapis/rpc v0.0.0-20250218202821-56aae31c358a h1:51aaUVRocpvUOSQKM6Q7VuoaktNIaMCLuhZB6DKksq4=
+google.golang.org/genproto/googleapis/rpc v0.0.0-20250218202821-56aae31c358a/go.mod h1:uRxBH1mhmO8PGhU89cMcHaXKZqO+OfakD8QQO0oYwlQ=
+google.golang.org/grpc v1.72.1 h1:HR03wO6eyZ7lknl75XlxABNVLLFc2PAb6mHlYh756mA=
+google.golang.org/grpc v1.72.1/go.mod h1:wH5Aktxcg25y1I3w7H69nHfXdOG3UiadoBtjh3izSDM=
+google.golang.org/protobuf v1.36.6 h1:z1NpPI8ku2WgiWnf+t9wTPsn6eP1L7ksHUlkfLvd9xY=
+google.golang.org/protobuf v1.36.6/go.mod h1:jduwjTPXsFjZGTmRluh+L6NjiWu7pchiJ2/5YcXBHnY=
 gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
 gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
 gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
@@ -375,10 +407,8 @@ gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
 gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
 honnef.co/go/tools v0.6.1 h1:R094WgE8K4JirYjBaOpz/AvTyUu/3wbmAoskKN/pxTI=
 honnef.co/go/tools v0.6.1/go.mod h1:3puzxxljPCe8RGJX7BIy1plGbxEOZni5mR2aXe3/uk4=
-k8s.io/apimachinery v0.33.1 h1:mzqXWV8tW9Rw4VeW9rEkqvnxj59k1ezDUl20tFK/oM4=
-k8s.io/apimachinery v0.33.1/go.mod h1:BHW0YOu7n22fFv/JkYOEfkUYNRN0fj0BlvMFWA7b+SM=
-mvdan.cc/sh/v3 v3.11.0 h1:q5h+XMDRfUGUedCqFFsjoFjrhwf2Mvtt1rkMvVz0blw=
-mvdan.cc/sh/v3 v3.11.0/go.mod h1:LRM+1NjoYCzuq/WZ6y44x14YNAI0NK7FLPeQSaFagGg=
+k8s.io/apimachinery v0.33.0 h1:1a6kHrJxb2hs4t8EE5wuR/WxKDwGN1FKH3JvDtA0CIQ=
+k8s.io/apimachinery v0.33.0/go.mod h1:BHW0YOu7n22fFv/JkYOEfkUYNRN0fj0BlvMFWA7b+SM=
 pault.ag/go/debian v0.18.0 h1:nr0iiyOU5QlG1VPnhZLNhnCcHx58kukvBJp+dvaM6CQ=
 pault.ag/go/debian v0.18.0/go.mod h1:JFl0XWRCv9hWBrB5MDDZjA5GSEs1X3zcFK/9kCNIUmE=
 pault.ag/go/topsort v0.1.1 h1:L0QnhUly6LmTv0e3DEzbN2q6/FGgAcQvaEw65S53Bg4=

internal/headers.go

@@ -81,12 +81,12 @@ func XForwardedForToXRealIP(next http.Handler) http.Handler {
 
 // XForwardedForUpdate sets or updates the X-Forwarded-For header, adding
 // the known remote address to an existing chain if present
-func XForwardedForUpdate(stripPrivate bool, next http.Handler) http.Handler {
+func XForwardedForUpdate(next http.Handler) http.Handler {
 	return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
 		defer next.ServeHTTP(w, r)
 
 		pref := XFFComputePreferences{
-			StripPrivate:  stripPrivate,
+			StripPrivate:  true,
 			StripLoopback: true,
 			StripCGNAT:    true,
 			Flatten:       true,

internal/mimetype.go

@@ -1,7 +0,0 @@
-package internal
-
-import "mime"
-
-func init() {
-	mime.AddExtensionType(".mjs", "text/javascript")
-}

internal/test/playwright_test.go

@@ -214,11 +214,6 @@ func TestPlaywrightBrowser(t *testing.T) {
 		return
 	}
 
-	if os.Getenv("SKIP_INTEGRATION") != "" {
-		t.Skip("SKIP_INTEGRATION was set")
-		return
-	}
-
 	startPlaywright(t)
 
 	pw := setupPlaywright(t)
@@ -294,11 +289,6 @@ func TestPlaywrightWithBasePrefix(t *testing.T) {
 		return
 	}
 
-	if os.Getenv("SKIP_INTEGRATION") != "" {
-		t.Skip("SKIP_INTEGRATION was set")
-		return
-	}
-
 	t.Skip("NOTE(Xe)\\ these tests require HTTPS support in #364")
 
 	startPlaywright(t)

internal/thoth/asnchecker.go

@@ -0,0 +1,39 @@
+package thoth
+
+import (
+	"context"
+	"net/http"
+	"time"
+
+	iptoasnv1 "github.com/TecharoHQ/thoth-proto/gen/techaro/thoth/iptoasn/v1"
+)
+
+type ASNChecker struct {
+	iptoasn iptoasnv1.IpToASNServiceClient
+	asns    map[uint32]struct{}
+	hash    string
+}
+
+func (asnc *ASNChecker) Check(r *http.Request) (bool, error) {
+	ctx, cancel := context.WithTimeout(r.Context(), 500*time.Millisecond)
+	defer cancel()
+
+	ipInfo, err := asnc.iptoasn.Lookup(ctx, &iptoasnv1.LookupRequest{
+		IpAddress: r.Header.Get("X-Real-Ip"),
+	})
+	if err != nil {
+		return false, err
+	}
+
+	if !ipInfo.GetAnnounced() {
+		return false, nil
+	}
+
+	_, ok := asnc.asns[uint32(ipInfo.GetAsNumber())]
+
+	return ok, nil
+}
+
+func (asnc *ASNChecker) Hash() string {
+	return asnc.hash
+}

internal/thoth/asnchecker_test.go

@@ -0,0 +1,81 @@
+package thoth
+
+import (
+	"fmt"
+	"net/http/httptest"
+	"testing"
+
+	"github.com/TecharoHQ/anubis/lib/policy/checker"
+	iptoasnv1 "github.com/TecharoHQ/thoth-proto/gen/techaro/thoth/iptoasn/v1"
+)
+
+var _ checker.Impl = &ASNChecker{}
+
+func TestASNChecker(t *testing.T) {
+	cli := loadSecrets(t)
+
+	asnc := &ASNChecker{
+		iptoasn: cli.iptoasn,
+		asns: map[uint32]struct{}{
+			13335: {},
+		},
+		hash: "foobar",
+	}
+
+	for _, cs := range []struct {
+		ipAddress string
+		wantMatch bool
+		wantError bool
+	}{
+		{
+			ipAddress: "1.1.1.1",
+			wantMatch: true,
+			wantError: false,
+		},
+		{
+			ipAddress: "8.8.8.8",
+			wantMatch: false,
+			wantError: false,
+		},
+		{
+			ipAddress: "taco",
+			wantMatch: false,
+			wantError: true,
+		},
+	} {
+		t.Run(fmt.Sprintf("%v", cs), func(t *testing.T) {
+			req := httptest.NewRequest("GET", "/", nil)
+			req.Header.Set("X-Real-Ip", cs.ipAddress)
+
+			match, err := asnc.Check(req)
+
+			if match != cs.wantMatch {
+				t.Errorf("Wanted match: %v, got: %v", cs.wantMatch, match)
+			}
+
+			switch {
+			case err != nil && !cs.wantError:
+				t.Errorf("Did not want error but got: %v", err)
+			case err == nil && cs.wantError:
+				t.Error("Wanted error but got none")
+			}
+		})
+	}
+}
+
+func BenchmarkWithCache(b *testing.B) {
+	cli := loadSecrets(b)
+	req := &iptoasnv1.LookupRequest{IpAddress: "1.1.1.1"}
+
+	_, err := cli.iptoasn.Lookup(b.Context(), req)
+	if err != nil {
+		b.Error(err)
+	}
+
+	for b.Loop() {
+		_, err := cli.iptoasn.Lookup(b.Context(), req)
+		if err != nil {
+			b.Error(err)
+		}
+	}
+}

internal/thoth/auth.go

@@ -0,0 +1,39 @@
+package thoth
+
+import (
+	"context"
+
+	"google.golang.org/grpc"
+	"google.golang.org/grpc/metadata"
+)
+
+func authUnaryClientInterceptor(token string) grpc.UnaryClientInterceptor {
+	return func(
+		ctx context.Context,
+		method string,
+		req interface{},
+		reply interface{},
+		cc *grpc.ClientConn,
+		invoker grpc.UnaryInvoker,
+		opts ...grpc.CallOption,
+	) error {
+		md := metadata.Pairs("authorization", "Bearer "+token)
+		ctx = metadata.NewOutgoingContext(ctx, md)
+		return invoker(ctx, method, req, reply, cc, opts...)
+	}
+}
+
+func authStreamClientInterceptor(token string) grpc.StreamClientInterceptor {
+	return func(
+		ctx context.Context,
+		desc *grpc.StreamDesc,
+		cc *grpc.ClientConn,
+		method string,
+		streamer grpc.Streamer,
+		opts ...grpc.CallOption,
+	) (grpc.ClientStream, error) {
+		md := metadata.Pairs("authorization", "Bearer "+token)
+		ctx = metadata.NewOutgoingContext(ctx, md)
+		return streamer(ctx, desc, cc, method, opts...)
+	}
+}

internal/thoth/cachediptoasn.go

@@ -0,0 +1,84 @@
+package thoth
+
+import (
+	"context"
+	"errors"
+	"fmt"
+	"log/slog"
+	"net/netip"
+
+	iptoasnv1 "github.com/TecharoHQ/thoth-proto/gen/techaro/thoth/iptoasn/v1"
+	"github.com/gaissmai/bart"
+	"google.golang.org/grpc"
+)
+
+type IPToASNWithCache struct {
+	next  iptoasnv1.IpToASNServiceClient
+	table *bart.Table[*iptoasnv1.LookupResponse]
+}
+
+func NewIpToASNWithCache(next iptoasnv1.IpToASNServiceClient) *IPToASNWithCache {
+	result := &IPToASNWithCache{
+		next:  next,
+		table: &bart.Table[*iptoasnv1.LookupResponse]{},
+	}
+
+	for _, pfx := range []netip.Prefix{
+		netip.MustParsePrefix("10.0.0.0/8"),         // RFC 1918
+		netip.MustParsePrefix("172.16.0.0/12"),      // RFC 1918
+		netip.MustParsePrefix("192.168.0.0/16"),     // RFC 1918
+		netip.MustParsePrefix("127.0.0.0/8"),        // Loopback
+		netip.MustParsePrefix("169.254.0.0/16"),     // Link-local
+		netip.MustParsePrefix("100.64.0.0/10"),      // CGNAT
+		netip.MustParsePrefix("192.0.0.0/24"),       // Protocol assignments
+		netip.MustParsePrefix("192.0.2.0/24"),       // TEST-NET-1
+		netip.MustParsePrefix("198.18.0.0/15"),      // Benchmarking
+		netip.MustParsePrefix("198.51.100.0/24"),    // TEST-NET-2
+		netip.MustParsePrefix("203.0.113.0/24"),     // TEST-NET-3
+		netip.MustParsePrefix("240.0.0.0/4"),        // Reserved
+		netip.MustParsePrefix("255.255.255.255/32"), // Broadcast
+		netip.MustParsePrefix("fc00::/7"),           // Unique local address
+		netip.MustParsePrefix("fe80::/10"),          // Link-local
+		netip.MustParsePrefix("::1/128"),            // Loopback
+		netip.MustParsePrefix("::/128"),             // Unspecified
+		netip.MustParsePrefix("100::/64"),           // Discard-only
+		netip.MustParsePrefix("2001:db8::/32"),      // Documentation
+	} {
+		result.table.Insert(pfx, &iptoasnv1.LookupResponse{Announced: false})
+	}
+
+	return result
+}
+
+func (ip2asn *IPToASNWithCache) Lookup(ctx context.Context, lr *iptoasnv1.LookupRequest, opts ...grpc.CallOption) (*iptoasnv1.LookupResponse, error) {
+	addr, err := netip.ParseAddr(lr.GetIpAddress())
+	if err != nil {
+		return nil, fmt.Errorf("input is not an IP address: %w", err)
+	}
+
+	cachedResponse, ok := ip2asn.table.Lookup(addr)
+	if ok {
+		return cachedResponse, nil
+	}
+
+	resp, err := ip2asn.next.Lookup(ctx, lr, opts...)
+	if err != nil {
+		return nil, err
+	}
+
+	var errs []error
+	for _, cidr := range resp.GetCidr() {
+		pfx, err := netip.ParsePrefix(cidr)
+		if err != nil {
+			errs = append(errs, err)
+			continue
+		}
+		ip2asn.table.Insert(pfx, resp)
+	}
+
+	if len(errs) != 0 {
+		slog.Error("errors parsing IP prefixes", "err", errors.Join(errs...))
+	}
+
+	return resp, nil
+}

internal/thoth/context.go

@@ -0,0 +1,14 @@
+package thoth
+
+import "context"
+
+type ctxKey struct{}
+
+func With(ctx context.Context, cli *Client) context.Context {
+	return context.WithValue(ctx, ctxKey{}, cli)
+}
+
+func FromContext(ctx context.Context) (*Client, bool) {
+	cli, ok := ctx.Value(ctxKey{}).(*Client)
+	return cli, ok
+}

internal/thoth/geoipchecker.go

@@ -0,0 +1,40 @@
+package thoth
+
+import (
+	"context"
+	"net/http"
+	"strings"
+	"time"
+
+	iptoasnv1 "github.com/TecharoHQ/thoth-proto/gen/techaro/thoth/iptoasn/v1"
+)
+
+type GeoIPChecker struct {
+	iptoasn   iptoasnv1.IpToASNServiceClient
+	countries map[string]struct{}
+	hash      string
+}
+
+func (gipc *GeoIPChecker) Check(r *http.Request) (bool, error) {
+	ctx, cancel := context.WithTimeout(r.Context(), 500*time.Millisecond)
+	defer cancel()
+
+	ipInfo, err := gipc.iptoasn.Lookup(ctx, &iptoasnv1.LookupRequest{
+		IpAddress: r.Header.Get("X-Real-Ip"),
+	})
+	if err != nil {
+		return false, err
+	}
+
+	if !ipInfo.GetAnnounced() {
+		return false, nil
+	}
+
+	_, ok := gipc.countries[strings.ToLower(ipInfo.GetCountryCode())]
+
+	return ok, nil
+}
+
+func (gipc *GeoIPChecker) Hash() string {
+	return gipc.hash
+}

internal/thoth/geoipchecker_test.go

@@ -0,0 +1,63 @@
+package thoth
+
+import (
+	"fmt"
+	"net/http/httptest"
+	"testing"
+
+	"github.com/TecharoHQ/anubis/lib/policy/checker"
+)
+
+var _ checker.Impl = &ASNChecker{}
+
+func TestGeoIPChecker(t *testing.T) {
+	cli := loadSecrets(t)
+
+	asnc := &GeoIPChecker{
+		iptoasn: cli.iptoasn,
+		countries: map[string]struct{}{
+			"us": {},
+		},
+		hash: "foobar",
+	}
+
+	for _, cs := range []struct {
+		ipAddress string
+		wantMatch bool
+		wantError bool
+	}{
+		{
+			ipAddress: "1.1.1.1",
+			wantMatch: true,
+			wantError: false,
+		},
+		{
+			ipAddress: "70.31.0.1",
+			wantMatch: false,
+			wantError: false,
+		},
+		{
+			ipAddress: "taco",
+			wantMatch: false,
+			wantError: true,
+		},
+	} {
+		t.Run(fmt.Sprintf("%v", cs), func(t *testing.T) {
+			req := httptest.NewRequest("GET", "/", nil)
+			req.Header.Set("X-Real-Ip", cs.ipAddress)
+
+			match, err := asnc.Check(req)
+
+			if match != cs.wantMatch {
+				t.Errorf("Wanted match: %v, got: %v", cs.wantMatch, match)
+			}
+
+			switch {
+			case err != nil && !cs.wantError:
+				t.Errorf("Did not want error but got: %v", err)
+			case err == nil && cs.wantError:
+				t.Error("Wanted error but got none")
+			}
+		})
+	}
+}

internal/thoth/thoth.go

@@ -0,0 +1,114 @@
+package thoth
+
+import (
+	"context"
+	"crypto/tls"
+	"fmt"
+	"strings"
+	"time"
+
+	"github.com/TecharoHQ/anubis/internal"
+	"github.com/TecharoHQ/anubis/lib/policy/checker"
+	iptoasnv1 "github.com/TecharoHQ/thoth-proto/gen/techaro/thoth/iptoasn/v1"
+	grpcprom "github.com/grpc-ecosystem/go-grpc-middleware/providers/prometheus"
+	"github.com/grpc-ecosystem/go-grpc-middleware/v2/interceptors/timeout"
+	"github.com/prometheus/client_golang/prometheus"
+	"google.golang.org/grpc"
+	"google.golang.org/grpc/credentials"
+	healthv1 "google.golang.org/grpc/health/grpc_health_v1"
+)
+
+type Client struct {
+	thothURL string
+
+	conn    *grpc.ClientConn
+	health  healthv1.HealthClient
+	iptoasn iptoasnv1.IpToASNServiceClient
+}
+
+func New(ctx context.Context, thothURL, apiToken string) (*Client, error) {
+	clMetrics := grpcprom.NewClientMetrics(
+		grpcprom.WithClientHandlingTimeHistogram(
+			grpcprom.WithHistogramBuckets([]float64{0.001, 0.01, 0.1, 0.3, 0.6, 1, 3, 6, 9, 20, 30, 60, 90, 120}),
+		),
+	)
+	prometheus.DefaultRegisterer.Register(clMetrics)
+
+	conn, err := grpc.DialContext(
+		ctx,
+		thothURL,
+		grpc.WithTransportCredentials(credentials.NewTLS(&tls.Config{})),
+		grpc.WithChainUnaryInterceptor(
+			timeout.UnaryClientInterceptor(500*time.Millisecond),
+			clMetrics.UnaryClientInterceptor(),
+			authUnaryClientInterceptor(apiToken),
+		),
+		grpc.WithChainStreamInterceptor(
+			clMetrics.StreamClientInterceptor(),
+			authStreamClientInterceptor(apiToken),
+		),
+	)
+	if err != nil {
+		return nil, fmt.Errorf("can't dial thoth at %s: %w", thothURL, err)
+	}
+
+	hc := healthv1.NewHealthClient(conn)
+
+	resp, err := hc.Check(ctx, &healthv1.HealthCheckRequest{})
+	if err != nil {
+		return nil, fmt.Errorf("can't verify thoth health at %s: %w", thothURL, err)
+	}
+
+	if resp.Status != healthv1.HealthCheckResponse_SERVING {
+		return nil, fmt.Errorf("thoth is not healthy, wanted %s but got %s", healthv1.HealthCheckResponse_SERVING, resp.Status)
+	}
+
+	return &Client{
+		conn:    conn,
+		health:  hc,
+		iptoasn: NewIpToASNWithCache(iptoasnv1.NewIpToASNServiceClient(conn)),
+	}, nil
+}
+
+func (c *Client) Close() error {
+	if c.conn != nil {
+		return c.conn.Close()
+	}
+	return nil
+}
+
+func (c *Client) WithIPToASNService(impl iptoasnv1.IpToASNServiceClient) {
+	c.iptoasn = impl
+}
+
+func (c *Client) ASNCheckerFor(asns []uint32) checker.Impl {
+	asnMap := map[uint32]struct{}{}
+	var sb strings.Builder
+	fmt.Fprintln(&sb, "ASNChecker")
+	for _, asn := range asns {
+		asnMap[asn] = struct{}{}
+		fmt.Fprintln(&sb, "AS", asn)
+	}
+
+	return &ASNChecker{
+		iptoasn: c.iptoasn,
+		asns:    asnMap,
+		hash:    internal.SHA256sum(sb.String()),
+	}
+}
+
+func (c *Client) GeoIPCheckerFor(countries []string) checker.Impl {
+	countryMap := map[string]struct{}{}
+	var sb strings.Builder
+	fmt.Fprintln(&sb, "GeoIPChecker")
+	for _, cc := range countries {
+		countryMap[cc] = struct{}{}
+		fmt.Fprintln(&sb, cc)
+	}
+
+	return &GeoIPChecker{
+		iptoasn:   c.iptoasn,
+		countries: countryMap,
+		hash:      sb.String(),
+	}
+}

internal/thoth/thoth_test.go

@@ -0,0 +1,29 @@
+package thoth
+
+import (
+	"os"
+	"testing"
+
+	"github.com/joho/godotenv"
+)
+
+func loadSecrets(t testing.TB) *Client {
+	if err := godotenv.Load(); err != nil {
+		t.Skip(".env not defined, can't load thoth secrets")
+	}
+
+	cli, err := New(t.Context(), os.Getenv("THOTH_URL"), os.Getenv("THOTH_API_KEY"))
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	return cli
+}
+
+func TestNew(t *testing.T) {
+	cli := loadSecrets(t)
+
+	if err := cli.Close(); err != nil {
+		t.Fatal(err)
+	}
+}

internal/thoth/thothmock/iptoasn.go

@@ -0,0 +1,44 @@
+package thothmock
+
+import (
+	"context"
+
+	iptoasnv1 "github.com/TecharoHQ/thoth-proto/gen/techaro/thoth/iptoasn/v1"
+	"google.golang.org/grpc"
+	"google.golang.org/grpc/codes"
+	"google.golang.org/grpc/status"
+)
+
+func MockIpToASNService() *IpToASNService {
+	responses := map[string]*iptoasnv1.LookupResponse{
+		"1.1.1.1": {
+			Announced:   true,
+			AsNumber:    13335,
+			Cidr:        []string{"1.1.1.0/24"},
+			CountryCode: "US",
+			Description: "Cloudflare",
+		},
+		"2.2.2.2": {
+			Announced:   true,
+			AsNumber:    420,
+			Cidr:        []string{"2.2.2.0/24"},
+			CountryCode: "CA",
+			Description: "test canada",
+		},
+	}
+
+	return &IpToASNService{Responses: responses}
+}
+
+type IpToASNService struct {
+	Responses map[string]*iptoasnv1.LookupResponse
+}
+
+func (ip2asn *IpToASNService) Lookup(ctx context.Context, lr *iptoasnv1.LookupRequest, opts ...grpc.CallOption) (*iptoasnv1.LookupResponse, error) {
+	resp, ok := ip2asn.Responses[lr.GetIpAddress()]
+	if !ok {
+		return nil, status.Error(codes.NotFound, "IP address not found in mock")
+	}
+
+	return resp, nil
+}

internal/xff_test.go

@@ -23,7 +23,7 @@ func TestXForwardedForUpdateIgnoreUnix(t *testing.T) {
 
 	w := httptest.NewRecorder()
 
-	XForwardedForUpdate(true, h).ServeHTTP(w, r)
+	XForwardedForUpdate(h).ServeHTTP(w, r)
 
 	if r.RemoteAddr != remoteAddr {
 		t.Errorf("wanted remoteAddr to be %s, got: %s", r.RemoteAddr, remoteAddr)
@@ -43,7 +43,7 @@ func TestXForwardedForUpdateAddToChain(t *testing.T) {
 		w.WriteHeader(http.StatusOK)
 	})
 
-	srv := httptest.NewServer(XForwardedForUpdate(true, h))
+	srv := httptest.NewServer(XForwardedForUpdate(h))
 
 	r, err := http.NewRequest(http.MethodGet, srv.URL, nil)
 	if err != nil {
@@ -81,15 +81,6 @@ func TestComputeXFFHeader(t *testing.T) {
 			},
 			result: "1.1.1.1,127.0.0.1",
 		},
-		{
-			name:          "StripPrivate",
-			remoteAddr:    "127.0.0.1:80",
-			origXFFHeader: "1.1.1.1,10.0.0.1",
-			pref: XFFComputePreferences{
-				StripPrivate: false,
-			},
-			result: "1.1.1.1,10.0.0.1,127.0.0.1",
-		},
 		{
 			name:          "StripLoopback",
 			remoteAddr:    "127.0.0.1:80",

lib/anubis.go

@@ -3,14 +3,16 @@ package lib
 import (
 	"crypto/ed25519"
 	"crypto/sha256"
+	"crypto/subtle"
 	"encoding/json"
-	"errors"
 	"fmt"
 	"log/slog"
+	"math"
 	"net"
 	"net/http"
 	"net/url"
 	"slices"
+	"strconv"
 	"strings"
 	"time"
 
@@ -24,40 +26,36 @@ import (
 	"github.com/TecharoHQ/anubis/internal"
 	"github.com/TecharoHQ/anubis/internal/dnsbl"
 	"github.com/TecharoHQ/anubis/internal/ogtags"
-	"github.com/TecharoHQ/anubis/lib/challenge"
 	"github.com/TecharoHQ/anubis/lib/policy"
 	"github.com/TecharoHQ/anubis/lib/policy/config"
-
-	// challenge implementations
-	_ "github.com/TecharoHQ/anubis/lib/challenge/metarefresh"
-	_ "github.com/TecharoHQ/anubis/lib/challenge/proofofwork"
 )
 
 var (
-	challengesIssued = promauto.NewCounterVec(prometheus.CounterOpts{
+	challengesIssued = promauto.NewCounter(prometheus.CounterOpts{
 		Name: "anubis_challenges_issued",
 		Help: "The total number of challenges issued",
-	}, []string{"method"})
+	})
 
-	challengesValidated = promauto.NewCounterVec(prometheus.CounterOpts{
+	challengesValidated = promauto.NewCounter(prometheus.CounterOpts{
 		Name: "anubis_challenges_validated",
 		Help: "The total number of challenges validated",
-	}, []string{"method"})
+	})
 
 	droneBLHits = promauto.NewCounterVec(prometheus.CounterOpts{
 		Name: "anubis_dronebl_hits",
 		Help: "The total number of hits from DroneBL",
 	}, []string{"status"})
 
-	failedValidations = promauto.NewCounterVec(prometheus.CounterOpts{
+	failedValidations = promauto.NewCounter(prometheus.CounterOpts{
 		Name: "anubis_failed_validations",
 		Help: "The total number of failed validations",
-	}, []string{"method"})
+	})
 
-	requestsProxied = promauto.NewCounterVec(prometheus.CounterOpts{
-		Name: "anubis_proxied_requests_total",
-		Help: "Number of requests proxied through Anubis to upstream targets",
-	}, []string{"host"})
+	timeTaken = promauto.NewHistogram(prometheus.HistogramOpts{
+		Name:    "anubis_time_taken",
+		Help:    "The time taken for a browser to generate a response (milliseconds)",
+		Buckets: prometheus.ExponentialBucketsRange(1, math.Pow(2, 18), 19),
+	})
 )
 
 type Server struct {
@@ -66,23 +64,18 @@ type Server struct {
 	policy     *policy.ParsedConfig
 	DNSBLCache *decaymap.Impl[string, dnsbl.DroneBLResponse]
 	OGTags     *ogtags.OGTagCache
-	cookieName string
 	priv       ed25519.PrivateKey
 	pub        ed25519.PublicKey
 	opts       Options
+	cookieName string
 }
 
 func (s *Server) challengeFor(r *http.Request, difficulty int) string {
-	fp := sha256.Sum256(s.pub[:])
-
-	acceptLanguage := r.Header.Get("Accept-Language")
-	if len(acceptLanguage) > 5 {
-		acceptLanguage = acceptLanguage[:5]
-	}
+	fp := sha256.Sum256(s.priv.Seed())
 
 	challengeData := fmt.Sprintf(
 		"Accept-Language=%s,X-Real-IP=%s,User-Agent=%s,WeekTime=%s,Fingerprint=%x,Difficulty=%d",
-		acceptLanguage,
+		r.Header.Get("Accept-Language"),
 		r.Header.Get("X-Real-Ip"),
 		r.UserAgent(),
 		time.Now().UTC().Round(24*7*time.Hour).Format(time.RFC3339),
@@ -164,29 +157,6 @@ func (s *Server) maybeReverseProxy(w http.ResponseWriter, r *http.Request, httpS
 		return
 	}
 
-	claims, ok := token.Claims.(jwt.MapClaims)
-	if !ok {
-		lg.Debug("invalid token claims type", "path", r.URL.Path)
-		s.ClearCookie(w, s.cookieName, cookiePath)
-		s.RenderIndex(w, r, rule, httpStatusOnly)
-		return
-	}
-
-	policyRule, ok := claims["policyRule"].(string)
-	if !ok {
-		lg.Debug("policyRule claim is not a string")
-		s.ClearCookie(w, s.cookieName, cookiePath)
-		s.RenderIndex(w, r, rule, httpStatusOnly)
-		return
-	}
-
-	if policyRule != rule.Hash() {
-		lg.Debug("user originally passed with a different rule, issuing new challenge", "old", policyRule, "new", rule.Name)
-		s.ClearCookie(w, s.cookieName, cookiePath)
-		s.RenderIndex(w, r, rule, httpStatusOnly)
-		return
-	}
-
 	r.Header.Add("X-Anubis-Status", "PASS")
 	s.ServeHTTPNext(w, r)
 }
@@ -256,21 +226,6 @@ func (s *Server) handleDNSBL(w http.ResponseWriter, r *http.Request, ip string,
 func (s *Server) MakeChallenge(w http.ResponseWriter, r *http.Request) {
 	lg := internal.GetRequestLogger(r)
 
-	redir := r.FormValue("redir")
-	if redir == "" {
-		w.WriteHeader(http.StatusBadRequest)
-		encoder := json.NewEncoder(w)
-		lg.Error("invalid invocation of MakeChallenge", "redir", redir)
-		encoder.Encode(struct {
-			Error string `json:"error"`
-		}{
-			Error: "Invalid invocation of MakeChallenge",
-		})
-		return
-	}
-
-	r.URL.Path = redir
-
 	encoder := json.NewEncoder(w)
 	cr, rule, err := s.check(r)
 	if err != nil {
@@ -305,7 +260,7 @@ func (s *Server) MakeChallenge(w http.ResponseWriter, r *http.Request) {
 		return
 	}
 	lg.Debug("made challenge", "challenge", challenge, "rules", rule.Challenge, "cr", cr)
-	challengesIssued.WithLabelValues("api").Inc()
+	challengesIssued.Inc()
 }
 
 func (s *Server) PassChallenge(w http.ResponseWriter, r *http.Request) {
@@ -317,14 +272,6 @@ func (s *Server) PassChallenge(w http.ResponseWriter, r *http.Request) {
 		cookiePath = strings.TrimSuffix(anubis.BasePrefix, "/") + "/"
 	}
 
-	if _, err := r.Cookie(anubis.TestCookieName); err == http.ErrNoCookie {
-		s.ClearCookie(w, s.cookieName, cookiePath)
-		s.ClearCookie(w, anubis.TestCookieName, "/")
-		lg.Warn("user has cookies disabled, this is not an anubis bug")
-		s.respondWithError(w, r, "Your browser is configured to disable cookies. Anubis requires cookies for the legitimate interest of making sure you are a valid client. Please enable cookies for this domain")
-		return
-	}
-
 	s.ClearCookie(w, anubis.TestCookieName, "/")
 
 	redir := r.FormValue("redir")
@@ -337,6 +284,42 @@ func (s *Server) PassChallenge(w http.ResponseWriter, r *http.Request) {
 	// used by the path checker rule
 	r.URL = redirURL
 
+	cr, rule, err := s.check(r)
+	if err != nil {
+		lg.Error("check failed", "err", err)
+		s.respondWithError(w, r, "Internal Server Error: administrator has misconfigured Anubis. Please contact the administrator and ask them to look for the logs around \"passChallenge\".")
+		return
+	}
+	lg = lg.With("check_result", cr)
+
+	nonceStr := r.FormValue("nonce")
+	if nonceStr == "" {
+		s.ClearCookie(w, s.cookieName, cookiePath)
+		lg.Debug("no nonce")
+		s.respondWithError(w, r, "missing nonce")
+		return
+	}
+
+	elapsedTimeStr := r.FormValue("elapsedTime")
+	if elapsedTimeStr == "" {
+		s.ClearCookie(w, s.cookieName, cookiePath)
+		lg.Debug("no elapsedTime")
+		s.respondWithError(w, r, "missing elapsedTime")
+		return
+	}
+
+	elapsedTime, err := strconv.ParseFloat(elapsedTimeStr, 64)
+	if err != nil {
+		s.ClearCookie(w, s.cookieName, cookiePath)
+		lg.Debug("elapsedTime doesn't parse", "err", err)
+		s.respondWithError(w, r, "invalid elapsedTime")
+		return
+	}
+
+	lg.Info("challenge took", "elapsedTime", elapsedTime)
+	timeTaken.Observe(elapsedTime)
+
+	response := r.FormValue("response")
 	urlParsed, err := r.URL.Parse(redir)
 	if err != nil {
 		s.respondWithError(w, r, "Redirect URL not parseable")
@@ -347,47 +330,54 @@ func (s *Server) PassChallenge(w http.ResponseWriter, r *http.Request) {
 		return
 	}
 
-	cr, rule, err := s.check(r)
-	if err != nil {
-		lg.Error("check failed", "err", err)
-		s.respondWithError(w, r, "Internal Server Error: administrator has misconfigured Anubis. Please contact the administrator and ask them to look for the logs around \"passChallenge\"")
-		return
-	}
-	lg = lg.With("check_result", cr)
+	challenge := s.challengeFor(r, rule.Challenge.Difficulty)
 
-	impl, ok := challenge.Get(rule.Challenge.Algorithm)
-	if !ok {
-		lg.Error("check failed", "err", err)
-		s.respondWithError(w, r, fmt.Sprintf("Internal Server Error: administrator has misconfigured Anubis. Please contact the administrator and ask them to file a bug as Anubis is trying to use challenge method %s but it does not exist in the challenge registry", rule.Challenge.Algorithm))
-		return
-	}
-
-	challengeStr := s.challengeFor(r, rule.Challenge.Difficulty)
-
-	if err := impl.Validate(r, lg, rule, challengeStr); err != nil {
-		failedValidations.WithLabelValues(string(rule.Challenge.Algorithm)).Inc()
-		var cerr *challenge.Error
+	if _, err := r.Cookie(anubis.TestCookieName); err == http.ErrNoCookie {
 		s.ClearCookie(w, s.cookieName, cookiePath)
-		lg.Debug("challenge validate call failed", "err", err)
+		s.ClearCookie(w, anubis.TestCookieName, cookiePath)
+		lg.Warn("user has cookies disabled, this is not an anubis bug")
+		s.respondWithError(w, r, "Your browser is configured to disable cookies. Anubis requires cookies for the legitimate interest of making sure you are a valid client. Please enable cookies for this domain")
+		return
+	}
 
-		switch {
-		case errors.As(err, &cerr):
-			switch {
-			case errors.Is(err, challenge.ErrFailed):
-				s.respondWithStatus(w, r, cerr.PublicReason, cerr.StatusCode)
-			case errors.Is(err, challenge.ErrInvalidFormat), errors.Is(err, challenge.ErrMissingField):
-				s.respondWithError(w, r, cerr.PublicReason)
-			}
-		}
+	nonce, err := strconv.Atoi(nonceStr)
+	if err != nil {
+		s.ClearCookie(w, s.cookieName, cookiePath)
+		lg.Debug("nonce doesn't parse", "err", err)
+		s.respondWithError(w, r, "invalid response")
+		return
+	}
+
+	calcString := fmt.Sprintf("%s%d", challenge, nonce)
+	calculated := internal.SHA256sum(calcString)
+
+	if subtle.ConstantTimeCompare([]byte(response), []byte(calculated)) != 1 {
+		s.ClearCookie(w, s.cookieName, cookiePath)
+		lg.Debug("hash does not match", "got", response, "want", calculated)
+		s.respondWithStatus(w, r, "invalid response", http.StatusForbidden)
+		failedValidations.Inc()
+		return
+	}
+
+	// compare the leading zeroes
+	if !strings.HasPrefix(response, strings.Repeat("0", rule.Challenge.Difficulty)) {
+		s.ClearCookie(w, s.cookieName, cookiePath)
+		lg.Debug("difficulty check failed", "response", response, "difficulty", rule.Challenge.Difficulty)
+		s.respondWithStatus(w, r, "invalid response", http.StatusForbidden)
+		failedValidations.Inc()
+		return
 	}
 
 	// generate JWT cookie
-	tokenString, err := s.signJWT(jwt.MapClaims{
-		"challenge":  challengeStr,
-		"method":     rule.Challenge.Algorithm,
-		"policyRule": rule.Hash(),
-		"action":     string(cr.Rule),
+	token := jwt.NewWithClaims(jwt.SigningMethodEdDSA, jwt.MapClaims{
+		"challenge": challenge,
+		"nonce":     nonceStr,
+		"response":  response,
+		"iat":       time.Now().Unix(),
+		"nbf":       time.Now().Add(-1 * time.Minute).Unix(),
+		"exp":       time.Now().Add(s.opts.CookieExpiration).Unix(),
 	})
+	tokenString, err := token.SignedString(s.priv)
 	if err != nil {
 		lg.Error("failed to sign JWT", "err", err)
 		s.ClearCookie(w, s.cookieName, cookiePath)
@@ -397,24 +387,22 @@ func (s *Server) PassChallenge(w http.ResponseWriter, r *http.Request) {
 
 	s.SetCookie(w, s.cookieName, tokenString, cookiePath)
 
-	challengesValidated.WithLabelValues(rule.Challenge.Algorithm).Inc()
+	challengesValidated.Inc()
 	lg.Debug("challenge passed, redirecting to app")
 	http.Redirect(w, r, redir, http.StatusFound)
 }
 
-func cr(name string, rule config.Rule, weight int) policy.CheckResult {
-	return policy.CheckResult{
-		Name:   name,
-		Rule:   rule,
-		Weight: weight,
-	}
+func (s *Server) TestError(w http.ResponseWriter, r *http.Request) {
+	err := r.FormValue("err")
+	s.respondWithError(w, r, err)
 }
 
-var (
-	weightOkayStatic    = policy.NewStaticHashChecker("weight/okay")
-	weightMildSusStatic = policy.NewStaticHashChecker("weight/mild-suspicion")
-	weightVerySusStatic = policy.NewStaticHashChecker("weight/extreme-suspicion")
-)
+func cr(name string, rule config.Rule) policy.CheckResult {
+	return policy.CheckResult{
+		Name: name,
+		Rule: rule,
+	}
+}
 
 // Check evaluates the list of rules, and returns the result
 func (s *Server) check(r *http.Request) (policy.CheckResult, *policy.Bot, error) {
@@ -428,8 +416,6 @@ func (s *Server) check(r *http.Request) (policy.CheckResult, *policy.Bot, error)
 		return decaymap.Zilch[policy.CheckResult](), nil, fmt.Errorf("[misconfiguration] %q is not an IP address", host)
 	}
 
-	weight := 0
-
 	for _, b := range s.policy.Bots {
 		match, err := b.Rules.Check(r)
 		if err != nil {
@@ -437,53 +423,16 @@ func (s *Server) check(r *http.Request) (policy.CheckResult, *policy.Bot, error)
 		}
 
 		if match {
-			switch b.Action {
-			case config.RuleDeny, config.RuleAllow, config.RuleBenchmark, config.RuleChallenge:
-				return cr("bot/"+b.Name, b.Action, weight), &b, nil
-			case config.RuleWeigh:
-				slog.Debug("adjusting weight", "name", b.Name, "delta", b.Weight.Adjust)
-				weight += b.Weight.Adjust
-			}
+			return cr("bot/"+b.Name, b.Action), &b, nil
 		}
 	}
 
-	switch {
-	case weight <= 0:
-		return cr("weight/okay", config.RuleAllow, weight), &policy.Bot{
-			Challenge: &config.ChallengeRules{
-				Difficulty: s.policy.DefaultDifficulty,
-				ReportAs:   s.policy.DefaultDifficulty,
-				Algorithm:  config.DefaultAlgorithm,
-			},
-			Rules: weightOkayStatic,
-		}, nil
-	case weight > 0 && weight < 10:
-		return cr("weight/mild-suspicion", config.RuleChallenge, weight), &policy.Bot{
-			Challenge: &config.ChallengeRules{
-				Difficulty: s.policy.DefaultDifficulty,
-				ReportAs:   s.policy.DefaultDifficulty,
-				Algorithm:  "metarefresh",
-			},
-			Rules: weightMildSusStatic,
-		}, nil
-	case weight >= 10:
-		return cr("weight/extreme-suspicion", config.RuleChallenge, weight), &policy.Bot{
-			Challenge: &config.ChallengeRules{
-				Difficulty: s.policy.DefaultDifficulty,
-				ReportAs:   s.policy.DefaultDifficulty,
-				Algorithm:  "fast",
-			},
-			Rules: weightVerySusStatic,
-		}, nil
-	}
-
-	return cr("default/allow", config.RuleAllow, weight), &policy.Bot{
+	return cr("default/allow", config.RuleAllow), &policy.Bot{
 		Challenge: &config.ChallengeRules{
 			Difficulty: s.policy.DefaultDifficulty,
 			ReportAs:   s.policy.DefaultDifficulty,
-			Algorithm:  config.DefaultAlgorithm,
+			Algorithm:  config.AlgorithmFast,
 		},
-		Rules: &policy.CheckerList{},
 	}, nil
 }
 

lib/anubis_test.go

@@ -4,29 +4,30 @@ import (
 	"encoding/json"
 	"fmt"
 	"net/http"
+	"net/http/cookiejar"
 	"net/http/httptest"
-	"net/url"
 	"os"
 	"strings"
-	"sync"
 	"testing"
 	"time"
 
 	"github.com/TecharoHQ/anubis"
 	"github.com/TecharoHQ/anubis/data"
 	"github.com/TecharoHQ/anubis/internal"
+	"github.com/TecharoHQ/anubis/internal/thoth"
+	"github.com/TecharoHQ/anubis/internal/thoth/thothmock"
 	"github.com/TecharoHQ/anubis/lib/policy"
 	"github.com/TecharoHQ/anubis/lib/policy/config"
 )
 
-func init() {
-	internal.InitSlog("debug")
-}
-
 func loadPolicies(t *testing.T, fname string) *policy.ParsedConfig {
 	t.Helper()
 
-	anubisPolicy, err := LoadPoliciesOrDefault(fname, anubis.DefaultDifficulty)
+	thothCli := &thoth.Client{}
+	thothCli.WithIPToASNService(thothmock.MockIpToASNService())
+	ctx := thoth.With(t.Context(), thothCli)
+
+	anubisPolicy, err := LoadPoliciesOrDefault(ctx, fname, anubis.DefaultDifficulty)
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -45,29 +46,20 @@ func spawnAnubis(t *testing.T, opts Options) *Server {
 	return s
 }
 
-type challengeResp struct {
+type challenge struct {
 	Challenge string `json:"challenge"`
 }
 
-func makeChallenge(t *testing.T, ts *httptest.Server, cli *http.Client) challengeResp {
+func makeChallenge(t *testing.T, ts *httptest.Server, cli *http.Client) challenge {
 	t.Helper()
 
-	req, err := http.NewRequest(http.MethodPost, ts.URL+"/.within.website/x/cmd/anubis/api/make-challenge", nil)
-	if err != nil {
-		t.Fatalf("can't make request: %v", err)
-	}
-
-	q := req.URL.Query()
-	q.Set("redir", "/")
-	req.URL.RawQuery = q.Encode()
-
-	resp, err := cli.Do(req)
+	resp, err := cli.Post(ts.URL+"/.within.website/x/cmd/anubis/api/make-challenge", "", nil)
 	if err != nil {
 		t.Fatalf("can't request challenge: %v", err)
 	}
 	defer resp.Body.Close()
 
-	var chall challengeResp
+	var chall challenge
 	if err := json.NewDecoder(resp.Body).Decode(&chall); err != nil {
 		t.Fatalf("can't read challenge response body: %v", err)
 	}
@@ -75,7 +67,7 @@ func makeChallenge(t *testing.T, ts *httptest.Server, cli *http.Client) challeng
 	return chall
 }
 
-func handleChallengeZeroDifficulty(t *testing.T, ts *httptest.Server, cli *http.Client, chall challengeResp) *http.Response {
+func handleChallengeZeroDifficulty(t *testing.T, ts *httptest.Server, cli *http.Client, chall challenge) *http.Response {
 	t.Helper()
 
 	nonce := 0
@@ -105,48 +97,16 @@ func handleChallengeZeroDifficulty(t *testing.T, ts *httptest.Server, cli *http.
 	return resp
 }
 
-type loggingCookieJar struct {
-	t       *testing.T
-	lock    sync.Mutex
-	cookies map[string][]*http.Cookie
-}
-
-func (lcj *loggingCookieJar) Cookies(u *url.URL) []*http.Cookie {
-	lcj.lock.Lock()
-	defer lcj.lock.Unlock()
-
-	// XXX(Xe): This is not RFC compliant in the slightest.
-	result, ok := lcj.cookies[u.Host]
-	if !ok {
-		return nil
-	}
-
-	lcj.t.Logf("requested cookies for %s", u)
-
-	for _, ckie := range result {
-		lcj.t.Logf("get cookie: <- %s", ckie)
-	}
-
-	return result
-}
-
-func (lcj *loggingCookieJar) SetCookies(u *url.URL, cookies []*http.Cookie) {
-	lcj.lock.Lock()
-	defer lcj.lock.Unlock()
-
-	for _, ckie := range cookies {
-		lcj.t.Logf("set cookie: %s -> %s", u, ckie)
-	}
-
-	// XXX(Xe): This is not RFC compliant in the slightest.
-	lcj.cookies[u.Host] = append(lcj.cookies[u.Host], cookies...)
-}
-
 func httpClient(t *testing.T) *http.Client {
 	t.Helper()
 
+	jar, err := cookiejar.New(nil)
+	if err != nil {
+		t.Fatal(err)
+	}
+
 	cli := &http.Client{
-		Jar: &loggingCookieJar{t: t, cookies: map[string][]*http.Cookie{}},
+		Jar: jar,
 		CheckRedirect: func(req *http.Request, via []*http.Request) error {
 			return http.ErrUseLastResponse
 		},
@@ -164,7 +124,7 @@ func TestLoadPolicies(t *testing.T) {
 			}
 			defer fin.Close()
 
-			if _, err := policy.ParseConfig(fin, fname, 4); err != nil {
+			if _, err := policy.ParseConfig(t.Context(), fin, fname, 4); err != nil {
 				t.Fatal(err)
 			}
 		})
@@ -180,7 +140,8 @@ func TestCVE2025_24369(t *testing.T) {
 		Next:   http.NewServeMux(),
 		Policy: pol,
 
-		CookieName: t.Name(),
+		CookiePartitioned: true,
+		CookieName:        t.Name(),
 	})
 
 	ts := httptest.NewServer(internal.RemoteXRealIP(true, "tcp", srv))
@@ -313,7 +274,7 @@ func TestCheckDefaultDifficultyMatchesPolicy(t *testing.T) {
 
 	for i := 1; i < 10; i++ {
 		t.Run(fmt.Sprint(i), func(t *testing.T) {
-			anubisPolicy, err := LoadPoliciesOrDefault("", i)
+			anubisPolicy, err := LoadPoliciesOrDefault(t.Context(), "", i)
 			if err != nil {
 				t.Fatal(err)
 			}
@@ -363,7 +324,7 @@ func TestBasePrefix(t *testing.T) {
 	}{
 		{
 			name:       "no prefix",
-			basePrefix: "/",
+			basePrefix: "",
 			path:       "/.within.website/x/cmd/anubis/api/make-challenge",
 			expected:   "/.within.website/x/cmd/anubis/api/make-challenge",
 		},
@@ -398,19 +359,8 @@ func TestBasePrefix(t *testing.T) {
 			ts := httptest.NewServer(internal.RemoteXRealIP(true, "tcp", srv))
 			defer ts.Close()
 
-			cli := httpClient(t)
-
-			req, err := http.NewRequest(http.MethodPost, ts.URL+tc.path, nil)
-			if err != nil {
-				t.Fatal(err)
-			}
-
-			q := req.URL.Query()
-			q.Set("redir", tc.basePrefix)
-			req.URL.RawQuery = q.Encode()
-
 			// Test API endpoint with prefix
-			resp, err := cli.Do(req)
+			resp, err := ts.Client().Post(ts.URL+tc.path, "", nil)
 			if err != nil {
 				t.Fatalf("can't request challenge: %v", err)
 			}
@@ -420,7 +370,7 @@ func TestBasePrefix(t *testing.T) {
 				t.Errorf("expected status code %d, got: %d", http.StatusOK, resp.StatusCode)
 			}
 
-			var chall challengeResp
+			var chall challenge
 			if err := json.NewDecoder(resp.Body).Decode(&chall); err != nil {
 				t.Fatalf("can't read challenge response body: %v", err)
 			}
@@ -444,6 +394,7 @@ func TestBasePrefix(t *testing.T) {
 			elapsedTime := 420
 			redir := "/"
 
+			cli := ts.Client()
 			cli.CheckRedirect = func(req *http.Request, via []*http.Request) error {
 				return http.ErrUseLastResponse
 			}
@@ -452,7 +403,7 @@ func TestBasePrefix(t *testing.T) {
 			passChallengePath := tc.path
 			passChallengePath = passChallengePath[:strings.LastIndex(passChallengePath, "/")+1] + "pass-challenge"
 
-			req, err = http.NewRequest(http.MethodGet, ts.URL+passChallengePath, nil)
+			req, err := http.NewRequest(http.MethodGet, ts.URL+passChallengePath, nil)
 			if err != nil {
 				t.Fatalf("can't make request: %v", err)
 			}
@@ -461,7 +412,7 @@ func TestBasePrefix(t *testing.T) {
 				req.AddCookie(ckie)
 			}
 
-			q = req.URL.Query()
+			q := req.URL.Query()
 			q.Set("response", calculated)
 			q.Set("nonce", fmt.Sprint(nonce))
 			q.Set("redir", redir)
@@ -604,31 +555,3 @@ func TestCloudflareWorkersRule(t *testing.T) {
 		})
 	}
 }
-
-func TestRuleChange(t *testing.T) {
-	pol := loadPolicies(t, "testdata/rule_change.yaml")
-	pol.DefaultDifficulty = 0
-	ckieExpiration := 10 * time.Minute
-
-	srv := spawnAnubis(t, Options{
-		Next:   http.NewServeMux(),
-		Policy: pol,
-
-		CookieDomain:     "127.0.0.1",
-		CookieName:       t.Name(),
-		CookieExpiration: ckieExpiration,
-	})
-
-	ts := httptest.NewServer(internal.RemoteXRealIP(true, "tcp", srv))
-	defer ts.Close()
-
-	cli := httpClient(t)
-
-	chall := makeChallenge(t, ts, cli)
-	resp := handleChallengeZeroDifficulty(t, ts, cli, chall)
-
-	if resp.StatusCode != http.StatusFound {
-		resp.Write(os.Stderr)
-		t.Errorf("wanted %d, got: %d", http.StatusFound, resp.StatusCode)
-	}
-}

lib/challenge/challenge.go

@@ -1,52 +0,0 @@
-package challenge
-
-import (
-	"log/slog"
-	"net/http"
-	"sort"
-	"sync"
-
-	"github.com/TecharoHQ/anubis/lib/policy"
-	"github.com/a-h/templ"
-)
-
-var (
-	registry map[string]Impl = map[string]Impl{}
-	regLock  sync.RWMutex
-)
-
-func Register(name string, impl Impl) {
-	regLock.Lock()
-	defer regLock.Unlock()
-
-	registry[name] = impl
-}
-
-func Get(name string) (Impl, bool) {
-	regLock.RLock()
-	defer regLock.RUnlock()
-	result, ok := registry[name]
-	return result, ok
-}
-
-func Methods() []string {
-	regLock.RLock()
-	defer regLock.RUnlock()
-	var result []string
-	for method := range registry {
-		result = append(result, method)
-	}
-	sort.Strings(result)
-	return result
-}
-
-type Impl interface {
-	// Setup registers any additional routes with the Impl for assets or API routes.
-	Setup(mux *http.ServeMux)
-
-	// Issue a new challenge to the user, called by the Anubis.
-	Issue(r *http.Request, lg *slog.Logger, rule *policy.Bot, challenge string, ogTags map[string]string) (templ.Component, error)
-
-	// Validate a challenge, making sure that it passes muster.
-	Validate(r *http.Request, lg *slog.Logger, rule *policy.Bot, challenge string) error
-}

lib/challenge/error.go

@@ -1,37 +0,0 @@
-package challenge
-
-import (
-	"errors"
-	"fmt"
-	"net/http"
-)
-
-var (
-	ErrFailed        = errors.New("challenge: user failed challenge")
-	ErrMissingField  = errors.New("challenge: missing field")
-	ErrInvalidFormat = errors.New("challenge: field has invalid format")
-)
-
-func NewError(verb, publicReason string, privateReason error) *Error {
-	return &Error{
-		Verb:          verb,
-		PublicReason:  publicReason,
-		PrivateReason: privateReason,
-		StatusCode:    http.StatusForbidden,
-	}
-}
-
-type Error struct {
-	Verb          string
-	PublicReason  string
-	PrivateReason error
-	StatusCode    int
-}
-
-func (e *Error) Error() string {
-	return fmt.Sprintf("challenge: error when processing challenge: %s: %v", e.Verb, e.PrivateReason)
-}
-
-func (e *Error) Unwrap() error {
-	return e.PrivateReason
-}

lib/challenge/metarefresh/metarefresh.go

@@ -1,53 +0,0 @@
-package metarefresh
-
-import (
-	"crypto/subtle"
-	"fmt"
-	"log/slog"
-	"net/http"
-
-	"github.com/TecharoHQ/anubis"
-	"github.com/TecharoHQ/anubis/lib/challenge"
-	"github.com/TecharoHQ/anubis/lib/policy"
-	"github.com/TecharoHQ/anubis/web"
-	"github.com/a-h/templ"
-)
-
-//go:generate go tool github.com/a-h/templ/cmd/templ generate
-
-func init() {
-	challenge.Register("metarefresh", &Impl{})
-}
-
-type Impl struct{}
-
-func (i *Impl) Setup(mux *http.ServeMux) {}
-
-func (i *Impl) Issue(r *http.Request, lg *slog.Logger, rule *policy.Bot, challenge string, ogTags map[string]string) (templ.Component, error) {
-	u, err := r.URL.Parse(anubis.BasePrefix + "/.within.website/x/cmd/anubis/api/pass-challenge")
-	if err != nil {
-		return nil, fmt.Errorf("can't render page: %w", err)
-	}
-
-	q := u.Query()
-	q.Set("redir", r.URL.String())
-	q.Set("challenge", challenge)
-	u.RawQuery = q.Encode()
-
-	component, err := web.BaseWithChallengeAndOGTags("Making sure you're not a bot!", page(challenge, u.String(), rule.Challenge.Difficulty), challenge, rule.Challenge, ogTags)
-	if err != nil {
-		return nil, fmt.Errorf("can't render page: %w", err)
-	}
-
-	return component, nil
-}
-
-func (i *Impl) Validate(r *http.Request, lg *slog.Logger, rule *policy.Bot, wantChallenge string) error {
-	gotChallenge := r.FormValue("challenge")
-
-	if subtle.ConstantTimeCompare([]byte(wantChallenge), []byte(gotChallenge)) != 1 {
-		return challenge.NewError("validate", "invalid response", fmt.Errorf("%w: wanted response %s but got %s", challenge.ErrFailed, wantChallenge, gotChallenge))
-	}
-
-	return nil
-}

lib/challenge/metarefresh/metarefresh.templ

@@ -1,17 +0,0 @@
-package metarefresh
-
-import (
-	"fmt"
-
-	"github.com/TecharoHQ/anubis"
-)
-
-templ page(challenge, redir string, difficulty int) {
-	<div class="centered-div">
-		<img id="image" style="width:100%;max-width:256px;" src={ anubis.BasePrefix + "/.within.website/x/cmd/anubis/static/img/pensive.webp?cacheBuster=" + anubis.Version }/>
-		<img style="display:none;" style="width:100%;max-width:256px;" src={ anubis.BasePrefix + "/.within.website/x/cmd/anubis/static/img/happy.webp?cacheBuster=" + anubis.Version }/>
-		<p id="status">Loading...</p>
-		<p>Please wait a moment while we ensure the security of your connection.</p>
-		<meta http-equiv="refresh" content={ fmt.Sprintf("%d; url=%s", difficulty, redir) }/>
-	</div>
-}

lib/challenge/metarefresh/metarefresh_templ.go

@@ -1,85 +0,0 @@
-// Code generated by templ - DO NOT EDIT.
-
-// templ: version: v0.3.898
-package metarefresh
-
-//lint:file-ignore SA4006 This context is only used if a nested component is present.
-
-import "github.com/a-h/templ"
-import templruntime "github.com/a-h/templ/runtime"
-
-import (
-	"fmt"
-
-	"github.com/TecharoHQ/anubis"
-)
-
-func page(challenge, redir string, difficulty int) templ.Component {
-	return templruntime.GeneratedTemplate(func(templ_7745c5c3_Input templruntime.GeneratedComponentInput) (templ_7745c5c3_Err error) {
-		templ_7745c5c3_W, ctx := templ_7745c5c3_Input.Writer, templ_7745c5c3_Input.Context
-		if templ_7745c5c3_CtxErr := ctx.Err(); templ_7745c5c3_CtxErr != nil {
-			return templ_7745c5c3_CtxErr
-		}
-		templ_7745c5c3_Buffer, templ_7745c5c3_IsBuffer := templruntime.GetBuffer(templ_7745c5c3_W)
-		if !templ_7745c5c3_IsBuffer {
-			defer func() {
-				templ_7745c5c3_BufErr := templruntime.ReleaseBuffer(templ_7745c5c3_Buffer)
-				if templ_7745c5c3_Err == nil {
-					templ_7745c5c3_Err = templ_7745c5c3_BufErr
-				}
-			}()
-		}
-		ctx = templ.InitializeContext(ctx)
-		templ_7745c5c3_Var1 := templ.GetChildren(ctx)
-		if templ_7745c5c3_Var1 == nil {
-			templ_7745c5c3_Var1 = templ.NopComponent
-		}
-		ctx = templ.ClearChildren(ctx)
-		templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 1, "<div class=\"centered-div\"><img id=\"image\" style=\"width:100%;max-width:256px;\" src=\"")
-		if templ_7745c5c3_Err != nil {
-			return templ_7745c5c3_Err
-		}
-		var templ_7745c5c3_Var2 string
-		templ_7745c5c3_Var2, templ_7745c5c3_Err = templ.JoinStringErrs(anubis.BasePrefix + "/.within.website/x/cmd/anubis/static/img/pensive.webp?cacheBuster=" + anubis.Version)
-		if templ_7745c5c3_Err != nil {
-			return templ.Error{Err: templ_7745c5c3_Err, FileName: `metarefresh.templ`, Line: 11, Col: 165}
-		}
-		_, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ.EscapeString(templ_7745c5c3_Var2))
-		if templ_7745c5c3_Err != nil {
-			return templ_7745c5c3_Err
-		}
-		templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 2, "\"> <img style=\"display:none;\" style=\"width:100%;max-width:256px;\" src=\"")
-		if templ_7745c5c3_Err != nil {
-			return templ_7745c5c3_Err
-		}
-		var templ_7745c5c3_Var3 string
-		templ_7745c5c3_Var3, templ_7745c5c3_Err = templ.JoinStringErrs(anubis.BasePrefix + "/.within.website/x/cmd/anubis/static/img/happy.webp?cacheBuster=" + anubis.Version)
-		if templ_7745c5c3_Err != nil {
-			return templ.Error{Err: templ_7745c5c3_Err, FileName: `metarefresh.templ`, Line: 12, Col: 174}
-		}
-		_, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ.EscapeString(templ_7745c5c3_Var3))
-		if templ_7745c5c3_Err != nil {
-			return templ_7745c5c3_Err
-		}
-		templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 3, "\"><p id=\"status\">Loading...</p><p>Please wait a moment while we ensure the security of your connection.</p><meta http-equiv=\"refresh\" content=\"")
-		if templ_7745c5c3_Err != nil {
-			return templ_7745c5c3_Err
-		}
-		var templ_7745c5c3_Var4 string
-		templ_7745c5c3_Var4, templ_7745c5c3_Err = templ.JoinStringErrs(fmt.Sprintf("%d; url=%s", difficulty, redir))
-		if templ_7745c5c3_Err != nil {
-			return templ.Error{Err: templ_7745c5c3_Err, FileName: `metarefresh.templ`, Line: 15, Col: 83}
-		}
-		_, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ.EscapeString(templ_7745c5c3_Var4))
-		if templ_7745c5c3_Err != nil {
-			return templ_7745c5c3_Err
-		}
-		templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 4, "\"></div>")
-		if templ_7745c5c3_Err != nil {
-			return templ_7745c5c3_Err
-		}
-		return nil
-	})
-}
-
-var _ = templruntime.GeneratedTemplate

lib/challenge/metrics.go

@@ -1,14 +0,0 @@
-package challenge
-
-import (
-	"math"
-
-	"github.com/prometheus/client_golang/prometheus"
-	"github.com/prometheus/client_golang/prometheus/promauto"
-)
-
-var TimeTaken = promauto.NewHistogramVec(prometheus.HistogramOpts{
-	Name:    "anubis_time_taken",
-	Help:    "The time taken for a browser to generate a response (milliseconds)",
-	Buckets: prometheus.ExponentialBucketsRange(1, math.Pow(2, 20), 20),
-}, []string{"method"})

lib/challenge/proofofwork/proofofwork.go

@@ -1,83 +0,0 @@
-package proofofwork
-
-import (
-	"crypto/subtle"
-	"fmt"
-	"log/slog"
-	"net/http"
-	"strconv"
-	"strings"
-
-	"github.com/TecharoHQ/anubis/internal"
-	chall "github.com/TecharoHQ/anubis/lib/challenge"
-	"github.com/TecharoHQ/anubis/lib/policy"
-	"github.com/TecharoHQ/anubis/web"
-	"github.com/a-h/templ"
-)
-
-func init() {
-	chall.Register("fast", &Impl{Algorithm: "fast"})
-	chall.Register("slow", &Impl{Algorithm: "slow"})
-}
-
-type Impl struct {
-	Algorithm string
-}
-
-func (i *Impl) Setup(mux *http.ServeMux) {
-	/* no implementation required */
-}
-
-func (i *Impl) Issue(r *http.Request, lg *slog.Logger, rule *policy.Bot, challenge string, ogTags map[string]string) (templ.Component, error) {
-	component, err := web.BaseWithChallengeAndOGTags("Making sure you're not a bot!", web.Index(), challenge, rule.Challenge, ogTags)
-	if err != nil {
-		return nil, fmt.Errorf("can't render page: %w", err)
-	}
-
-	return component, nil
-}
-
-func (i *Impl) Validate(r *http.Request, lg *slog.Logger, rule *policy.Bot, challenge string) error {
-	nonceStr := r.FormValue("nonce")
-	if nonceStr == "" {
-		return chall.NewError("validate", "invalid response", fmt.Errorf("%w nonce", chall.ErrMissingField))
-	}
-
-	nonce, err := strconv.Atoi(nonceStr)
-	if err != nil {
-		return chall.NewError("validate", "invalid response", fmt.Errorf("%w: nonce: %w", chall.ErrInvalidFormat, err))
-
-	}
-
-	elapsedTimeStr := r.FormValue("elapsedTime")
-	if elapsedTimeStr == "" {
-		return chall.NewError("validate", "invalid response", fmt.Errorf("%w elapsedTime", chall.ErrMissingField))
-	}
-
-	elapsedTime, err := strconv.ParseFloat(elapsedTimeStr, 64)
-	if err != nil {
-		return chall.NewError("validate", "invalid response", fmt.Errorf("%w: elapsedTime: %w", chall.ErrInvalidFormat, err))
-	}
-
-	response := r.FormValue("response")
-	if response == "" {
-		return chall.NewError("validate", "invalid response", fmt.Errorf("%w response", chall.ErrMissingField))
-	}
-
-	calcString := fmt.Sprintf("%s%d", challenge, nonce)
-	calculated := internal.SHA256sum(calcString)
-
-	if subtle.ConstantTimeCompare([]byte(response), []byte(calculated)) != 1 {
-		return chall.NewError("validate", "invalid response", fmt.Errorf("%w: wanted response %s but got %s", chall.ErrFailed, calculated, response))
-	}
-
-	// compare the leading zeroes
-	if !strings.HasPrefix(response, strings.Repeat("0", rule.Challenge.Difficulty)) {
-		return chall.NewError("validate", "invalid response", fmt.Errorf("%w: wanted %d leading zeros but got %s", chall.ErrFailed, rule.Challenge.Difficulty, response))
-	}
-
-	lg.Debug("challenge took", "elapsedTime", elapsedTime)
-	chall.TimeTaken.WithLabelValues(i.Algorithm).Observe(elapsedTime)
-
-	return nil
-}

lib/challenge/proofofwork/proofofwork_test.go

@@ -1,136 +0,0 @@
-package proofofwork
-
-import (
-	"errors"
-	"log/slog"
-	"net/http"
-	"testing"
-
-	"github.com/TecharoHQ/anubis/lib/challenge"
-	"github.com/TecharoHQ/anubis/lib/policy"
-	"github.com/TecharoHQ/anubis/lib/policy/config"
-)
-
-func mkRequest(t *testing.T, values map[string]string) *http.Request {
-	t.Helper()
-	req, err := http.NewRequestWithContext(t.Context(), http.MethodGet, "/", nil)
-	if err != nil {
-		t.Fatal(err)
-	}
-
-	q := req.URL.Query()
-
-	for k, v := range values {
-		q.Set(k, v)
-	}
-
-	req.URL.RawQuery = q.Encode()
-
-	return req
-}
-
-func TestBasic(t *testing.T) {
-	i := &Impl{Algorithm: "fast"}
-	bot := &policy.Bot{
-		Challenge: &config.ChallengeRules{
-			Algorithm:  "fast",
-			Difficulty: 0,
-			ReportAs:   0,
-		},
-	}
-	const challengeStr = "hunter"
-	const response = "2652bdba8fb4d2ab39ef28d8534d7694c557a4ae146c1e9237bd8d950280500e"
-
-	for _, cs := range []struct {
-		name         string
-		req          *http.Request
-		err          error
-		challengeStr string
-	}{
-		{
-			name: "allgood",
-			req: mkRequest(t, map[string]string{
-				"nonce":       "0",
-				"elapsedTime": "69",
-				"response":    response,
-			}),
-			err:          nil,
-			challengeStr: challengeStr,
-		},
-		{
-			name:         "no-params",
-			req:          mkRequest(t, map[string]string{}),
-			err:          challenge.ErrMissingField,
-			challengeStr: challengeStr,
-		},
-		{
-			name: "missing-nonce",
-			req: mkRequest(t, map[string]string{
-				"elapsedTime": "69",
-				"response":    response,
-			}),
-			err:          challenge.ErrMissingField,
-			challengeStr: challengeStr,
-		},
-		{
-			name: "missing-elapsedTime",
-			req: mkRequest(t, map[string]string{
-				"nonce":    "0",
-				"response": response,
-			}),
-			err:          challenge.ErrMissingField,
-			challengeStr: challengeStr,
-		},
-		{
-			name: "missing-response",
-			req: mkRequest(t, map[string]string{
-				"nonce":       "0",
-				"elapsedTime": "69",
-			}),
-			err:          challenge.ErrMissingField,
-			challengeStr: challengeStr,
-		},
-		{
-			name: "wrong-nonce-format",
-			req: mkRequest(t, map[string]string{
-				"nonce":       "taco",
-				"elapsedTime": "69",
-				"response":    response,
-			}),
-			err:          challenge.ErrInvalidFormat,
-			challengeStr: challengeStr,
-		},
-		{
-			name: "wrong-elapsedTime-format",
-			req: mkRequest(t, map[string]string{
-				"nonce":       "0",
-				"elapsedTime": "taco",
-				"response":    response,
-			}),
-			err:          challenge.ErrInvalidFormat,
-			challengeStr: challengeStr,
-		},
-		{
-			name: "invalid-response",
-			req: mkRequest(t, map[string]string{
-				"nonce":       "0",
-				"elapsedTime": "69",
-				"response":    response,
-			}),
-			err:          challenge.ErrFailed,
-			challengeStr: "Tacos are tasty",
-		},
-	} {
-		t.Run(cs.name, func(t *testing.T) {
-			lg := slog.With()
-
-			if _, err := i.Issue(cs.req, lg, bot, cs.challengeStr, nil); err != nil {
-				t.Errorf("can't issue challenge: %v", err)
-			}
-
-			if err := i.Validate(cs.req, lg, bot, cs.challengeStr); !errors.Is(err, cs.err) {
-				t.Errorf("got wrong error from Validate, got %v but wanted %v", err, cs.err)
-			}
-		})
-	}
-}

lib/config.go

@@ -1,9 +1,9 @@
 package lib
 
 import (
+	"context"
 	"crypto/ed25519"
 	"crypto/rand"
-	"errors"
 	"fmt"
 	"io"
 	"log/slog"
@@ -18,7 +18,6 @@ import (
 	"github.com/TecharoHQ/anubis/internal"
 	"github.com/TecharoHQ/anubis/internal/dnsbl"
 	"github.com/TecharoHQ/anubis/internal/ogtags"
-	"github.com/TecharoHQ/anubis/lib/challenge"
 	"github.com/TecharoHQ/anubis/lib/policy"
 	"github.com/TecharoHQ/anubis/web"
 	"github.com/TecharoHQ/anubis/xess"
@@ -42,7 +41,7 @@ type Options struct {
 	ServeRobotsTXT       bool
 }
 
-func LoadPoliciesOrDefault(fname string, defaultDifficulty int) (*policy.ParsedConfig, error) {
+func LoadPoliciesOrDefault(ctx context.Context, fname string, defaultDifficulty int) (*policy.ParsedConfig, error) {
 	var fin io.ReadCloser
 	var err error
 
@@ -66,18 +65,7 @@ func LoadPoliciesOrDefault(fname string, defaultDifficulty int) (*policy.ParsedC
 		}
 	}(fin)
 
-	anubisPolicy, err := policy.ParseConfig(fin, fname, defaultDifficulty)
-	var validationErrs []error
-
-	for _, b := range anubisPolicy.Bots {
-		if _, ok := challenge.Get(b.Challenge.Algorithm); !ok {
-			validationErrs = append(validationErrs, fmt.Errorf("%w %s", policy.ErrChallengeRuleHasWrongAlgorithm, b.Challenge.Algorithm))
-		}
-	}
-
-	if len(validationErrs) != 0 {
-		return nil, fmt.Errorf("can't do final validation of Anubis config: %w", errors.Join(validationErrs...))
-	}
+	anubisPolicy, err := policy.ParseConfig(ctx, fin, fname, defaultDifficulty)
 
 	return anubisPolicy, err
 }
@@ -145,21 +133,12 @@ func New(opts Options) (*Server, error) {
 		}), "GET")
 	}
 
+	registerWithPrefix(anubis.APIPrefix+"make-challenge", http.HandlerFunc(result.MakeChallenge), "POST")
 	registerWithPrefix(anubis.APIPrefix+"pass-challenge", http.HandlerFunc(result.PassChallenge), "GET")
 	registerWithPrefix(anubis.APIPrefix+"check", http.HandlerFunc(result.maybeReverseProxyHttpStatusOnly), "")
+	registerWithPrefix(anubis.APIPrefix+"test-error", http.HandlerFunc(result.TestError), "GET")
 	registerWithPrefix("/", http.HandlerFunc(result.maybeReverseProxyOrPage), "")
 
-	//goland:noinspection GoBoolExpressions
-	if anubis.Version == "devel" {
-		// make-challenge is only used in tests. Only enable while version is devel
-		registerWithPrefix(anubis.APIPrefix+"make-challenge", http.HandlerFunc(result.MakeChallenge), "POST")
-	}
-  
-	for _, implKind := range challenge.Methods() {
-		impl, _ := challenge.Get(implKind)
-		impl.Setup(mux)
-	}
-
 	result.mux = mux
 
 	return result, nil

lib/config_test.go

@@ -1,51 +0,0 @@
-package lib
-
-import (
-	"errors"
-	"os"
-	"path/filepath"
-	"testing"
-
-	"github.com/TecharoHQ/anubis"
-	"github.com/TecharoHQ/anubis/lib/policy"
-)
-
-func TestInvalidChallengeMethod(t *testing.T) {
-	if _, err := LoadPoliciesOrDefault("testdata/invalid-challenge-method.yaml", 4); !errors.Is(err, policy.ErrChallengeRuleHasWrongAlgorithm) {
-		t.Fatalf("wanted error %v but got %v", policy.ErrChallengeRuleHasWrongAlgorithm, err)
-	}
-}
-
-func TestBadConfigs(t *testing.T) {
-	finfos, err := os.ReadDir("policy/config/testdata/bad")
-	if err != nil {
-		t.Fatal(err)
-	}
-
-	for _, st := range finfos {
-		st := st
-		t.Run(st.Name(), func(t *testing.T) {
-			if _, err := LoadPoliciesOrDefault(filepath.Join("policy", "config", "testdata", "good", st.Name()), anubis.DefaultDifficulty); err == nil {
-				t.Fatal(err)
-			} else {
-				t.Log(err)
-			}
-		})
-	}
-}
-
-func TestGoodConfigs(t *testing.T) {
-	finfos, err := os.ReadDir("policy/config/testdata/good")
-	if err != nil {
-		t.Fatal(err)
-	}
-
-	for _, st := range finfos {
-		st := st
-		t.Run(st.Name(), func(t *testing.T) {
-			if _, err := LoadPoliciesOrDefault(filepath.Join("policy", "config", "testdata", "good", st.Name()), anubis.DefaultDifficulty); err != nil {
-				t.Fatal(err)
-			}
-		})
-	}
-}

lib/http.go

@@ -1,7 +1,6 @@
 package lib
 
 import (
-	"fmt"
 	"math/rand"
 	"net/http"
 	"slices"
@@ -10,11 +9,9 @@ import (
 
 	"github.com/TecharoHQ/anubis"
 	"github.com/TecharoHQ/anubis/internal"
-	"github.com/TecharoHQ/anubis/lib/challenge"
 	"github.com/TecharoHQ/anubis/lib/policy"
 	"github.com/TecharoHQ/anubis/web"
 	"github.com/a-h/templ"
-	"github.com/golang-jwt/jwt/v5"
 )
 
 func (s *Server) SetCookie(w http.ResponseWriter, name, value, path string) {
@@ -76,8 +73,7 @@ func (s *Server) RenderIndex(w http.ResponseWriter, r *http.Request, rule *polic
 		s.respondWithError(w, r, "Client Error: Please ensure your browser is up to date and try again later.")
 	}
 
-	challengesIssued.WithLabelValues("embedded").Add(1)
-	challengeStr := s.challengeFor(r, rule.Challenge.Difficulty)
+	challenge := s.challengeFor(r, rule.Challenge.Difficulty)
 
 	var ogTags map[string]string = nil
 	if s.opts.OGPassthrough {
@@ -90,21 +86,14 @@ func (s *Server) RenderIndex(w http.ResponseWriter, r *http.Request, rule *polic
 
 	http.SetCookie(w, &http.Cookie{
 		Name:    anubis.TestCookieName,
-		Value:   challengeStr,
+		Value:   challenge,
 		Expires: time.Now().Add(30 * time.Minute),
 		Path:    "/",
 	})
 
-	impl, ok := challenge.Get(rule.Challenge.Algorithm)
-	if !ok {
-		lg.Error("check failed", "err", "can't get algorithm", "algorithm", rule.Challenge.Algorithm)
-		s.respondWithError(w, r, fmt.Sprintf("Internal Server Error: administrator has misconfigured Anubis. Please contact the administrator and ask them to file a bug as Anubis is trying to use challenge method %s but it does not exist in the challenge registry", rule.Challenge.Algorithm))
-		return
-	}
-
-	component, err := impl.Issue(r, lg, rule, challengeStr, ogTags)
+	component, err := web.BaseWithChallengeAndOGTags("Making sure you're not a bot!", web.Index(), challenge, rule.Challenge, ogTags)
 	if err != nil {
-		lg.Error("[unexpected] render failed, please open an issue", "err", err) // This is likely a bug in the template. Should never be triggered as CI tests for this.
+		lg.Error("render failed, please open an issue", "err", err) // This is likely a bug in the template. Should never be triggered as CI tests for this.
 		s.respondWithError(w, r, "Internal Server Error: please contact the administrator and ask them to look for the logs around \"RenderIndex\"")
 		return
 	}
@@ -157,15 +146,6 @@ func (s *Server) ServeHTTPNext(w http.ResponseWriter, r *http.Request) {
 			web.Base("You are not a bot!", web.StaticHappy()),
 		).ServeHTTP(w, r)
 	} else {
-		requestsProxied.WithLabelValues(r.Host).Inc()
 		s.next.ServeHTTP(w, r)
 	}
 }
-
-func (s *Server) signJWT(claims jwt.MapClaims) (string, error) {
-	claims["iat"] = time.Now().Unix()
-	claims["nbf"] = time.Now().Add(-1 * time.Minute).Unix()
-	claims["exp"] = time.Now().Add(s.opts.CookieExpiration).Unix()
-
-	return jwt.NewWithClaims(jwt.SigningMethodEdDSA, claims).SignedString(s.priv)
-}

lib/policy/bot.go

@@ -4,15 +4,15 @@ import (
 	"fmt"
 
 	"github.com/TecharoHQ/anubis/internal"
+	"github.com/TecharoHQ/anubis/lib/policy/checker"
 	"github.com/TecharoHQ/anubis/lib/policy/config"
 )
 
 type Bot struct {
-	Rules     Checker
+	Rules     checker.Impl
 	Challenge *config.ChallengeRules
 	Name      string
 	Action    config.Rule
-	Weight    *config.Weight
 }
 
 func (b Bot) Hash() string {

lib/policy/checker.go

@@ -9,6 +9,7 @@ import (
 	"strings"
 
 	"github.com/TecharoHQ/anubis/internal"
+	"github.com/TecharoHQ/anubis/lib/policy/checker"
 	"github.com/yl2chen/cidranger"
 )
 
@@ -16,57 +17,12 @@ var (
 	ErrMisconfiguration = errors.New("[unexpected] policy: administrator misconfiguration")
 )
 
-type Checker interface {
-	Check(*http.Request) (bool, error)
-	Hash() string
-}
-
-type CheckerList []Checker
-
-func (cl CheckerList) Check(r *http.Request) (bool, error) {
-	for _, c := range cl {
-		ok, err := c.Check(r)
-		if err != nil {
-			return ok, err
-		}
-		if ok {
-			return ok, nil
-		}
-	}
-
-	return false, nil
-}
-
-func (cl CheckerList) Hash() string {
-	var sb strings.Builder
-
-	for _, c := range cl {
-		fmt.Fprintln(&sb, c.Hash())
-	}
-
-	return internal.SHA256sum(sb.String())
-}
-
-type staticHashChecker struct {
-	hash string
-}
-
-func (staticHashChecker) Check(r *http.Request) (bool, error) {
-	return true, nil
-}
-
-func (s staticHashChecker) Hash() string { return s.hash }
-
-func NewStaticHashChecker(hashable string) Checker {
-	return staticHashChecker{hash: internal.SHA256sum(hashable)}
-}
-
 type RemoteAddrChecker struct {
 	ranger cidranger.Ranger
 	hash   string
 }
 
-func NewRemoteAddrChecker(cidrs []string) (Checker, error) {
+func NewRemoteAddrChecker(cidrs []string) (checker.Impl, error) {
 	ranger := cidranger.NewPCTrieRanger()
 	var sb strings.Builder
 
@@ -76,10 +32,7 @@ func NewRemoteAddrChecker(cidrs []string) (Checker, error) {
 			return nil, fmt.Errorf("%w: range %s not parsing: %w", ErrMisconfiguration, cidr, err)
 		}
 
-		err = ranger.Insert(cidranger.NewBasicRangerEntry(*rng))
-		if err != nil {
-			return nil, fmt.Errorf("%w: error inserting ip range: %w", ErrMisconfiguration, err)
-		}
+		ranger.Insert(cidranger.NewBasicRangerEntry(*rng))
 		fmt.Fprintln(&sb, cidr)
 	}
 
@@ -122,11 +75,11 @@ type HeaderMatchesChecker struct {
 	hash   string
 }
 
-func NewUserAgentChecker(rexStr string) (Checker, error) {
+func NewUserAgentChecker(rexStr string) (checker.Impl, error) {
 	return NewHeaderMatchesChecker("User-Agent", rexStr)
 }
 
-func NewHeaderMatchesChecker(header, rexStr string) (Checker, error) {
+func NewHeaderMatchesChecker(header, rexStr string) (checker.Impl, error) {
 	rex, err := regexp.Compile(strings.TrimSpace(rexStr))
 	if err != nil {
 		return nil, fmt.Errorf("%w: regex %s failed parse: %w", ErrMisconfiguration, rexStr, err)
@@ -151,7 +104,7 @@ type PathChecker struct {
 	hash   string
 }
 
-func NewPathChecker(rexStr string) (Checker, error) {
+func NewPathChecker(rexStr string) (checker.Impl, error) {
 	rex, err := regexp.Compile(strings.TrimSpace(rexStr))
 	if err != nil {
 		return nil, fmt.Errorf("%w: regex %s failed parse: %w", ErrMisconfiguration, rexStr, err)
@@ -171,7 +124,7 @@ func (pc *PathChecker) Hash() string {
 	return pc.hash
 }
 
-func NewHeaderExistsChecker(key string) Checker {
+func NewHeaderExistsChecker(key string) checker.Impl {
 	return headerExistsChecker{strings.TrimSpace(key)}
 }
 
@@ -191,8 +144,8 @@ func (hec headerExistsChecker) Hash() string {
 	return internal.SHA256sum(hec.header)
 }
 
-func NewHeadersChecker(headermap map[string]string) (Checker, error) {
-	var result CheckerList
+func NewHeadersChecker(headermap map[string]string) (checker.Impl, error) {
+	var result checker.List
 	var errs []error
 
 	for key, rexStr := range headermap {

lib/policy/checker/checker.go

@@ -0,0 +1,41 @@
+// Package checker defines the Checker interface and a helper utility to avoid import cycles.
+package checker
+
+import (
+	"fmt"
+	"net/http"
+	"strings"
+
+	"github.com/TecharoHQ/anubis/internal"
+)
+
+type Impl interface {
+	Check(*http.Request) (bool, error)
+	Hash() string
+}
+
+type List []Impl
+
+func (l List) Check(r *http.Request) (bool, error) {
+	for _, c := range l {
+		ok, err := c.Check(r)
+		if err != nil {
+			return ok, err
+		}
+		if ok {
+			return ok, nil
+		}
+	}
+
+	return false, nil
+}
+
+func (l List) Hash() string {
+	var sb strings.Builder
+
+	for _, c := range l {
+		fmt.Fprintln(&sb, c.Hash())
+	}
+
+	return internal.SHA256sum(sb.String())
+}

lib/policy/checkresult.go

@@ -7,15 +7,12 @@ import (
 )
 
 type CheckResult struct {
-	Name   string
-	Rule   config.Rule
-	Weight int
+	Name string
+	Rule config.Rule
 }
 
 func (cr CheckResult) LogValue() slog.Value {
 	return slog.GroupValue(
 		slog.String("name", cr.Name),
-		slog.String("rule", string(cr.Rule)),
-		slog.Int("weight", cr.Weight),
-	)
+		slog.String("rule", string(cr.Rule)))
 }

lib/policy/config/asn.go

@@ -0,0 +1,44 @@
+package config
+
+import (
+	"errors"
+	"fmt"
+)
+
+var (
+	ErrPrivateASN = errors.New("bot.ASNs: you have specified a private use ASN")
+)
+
+type ASNs struct {
+	Match []uint32 `json:"match"`
+}
+
+func (a *ASNs) Valid() error {
+	var errs []error
+
+	for _, asn := range a.Match {
+		if isPrivateASN(asn) {
+			errs = append(errs, fmt.Errorf("%w: %d is private (see RFC 6996)", ErrPrivateASN, asn))
+		}
+	}
+
+	if len(errs) != 0 {
+		return fmt.Errorf("bot.ASNs: invalid ASN settings: %w", errors.Join(errs...))
+	}
+
+	return nil
+}
+
+// isPrivateASN checks if an ASN is in the private use area.
+//
+// Based on RFC 6996 and IANA allocations.
+func isPrivateASN(asn uint32) bool {
+	switch {
+	case asn >= 64512 && asn <= 65534:
+		return true
+	case asn >= 4200000000 && asn <= 4294967294:
+		return true
+	default:
+		return false
+	}
+}

lib/policy/config/config.go

@@ -39,11 +39,16 @@ const (
 	RuleAllow     Rule = "ALLOW"
 	RuleDeny      Rule = "DENY"
 	RuleChallenge Rule = "CHALLENGE"
-	RuleWeigh     Rule = "WEIGH"
 	RuleBenchmark Rule = "DEBUG_BENCHMARK"
 )
 
-const DefaultAlgorithm = "fast"
+type Algorithm string
+
+const (
+	AlgorithmUnknown Algorithm = ""
+	AlgorithmFast    Algorithm = "fast"
+	AlgorithmSlow    Algorithm = "slow"
+)
 
 type BotConfig struct {
 	UserAgentRegex *string           `json:"user_agent_regex,omitempty"`
@@ -51,7 +56,8 @@ type BotConfig struct {
 	HeadersRegex   map[string]string `json:"headers_regex,omitempty"`
 	Expression     *ExpressionOrList `json:"expression,omitempty"`
 	Challenge      *ChallengeRules   `json:"challenge,omitempty"`
-	Weight         *Weight           `json:"weight,omitempty"`
+	GeoIP          *GeoIP            `json:"geoip,omitempty"`
+	ASNs           *ASNs             `json:"asns,omitempty"`
 	Name           string            `json:"name"`
 	Action         Rule              `json:"action"`
 	RemoteAddr     []string          `json:"remote_addresses,omitempty"`
@@ -75,7 +81,7 @@ func (b BotConfig) Zero() bool {
 	return true
 }
 
-func (b *BotConfig) Valid() error {
+func (b BotConfig) Valid() error {
 	var errs []error
 
 	if b.Name == "" {
@@ -85,7 +91,9 @@ func (b *BotConfig) Valid() error {
 	allFieldsEmpty := b.UserAgentRegex == nil &&
 		b.PathRegex == nil &&
 		len(b.RemoteAddr) == 0 &&
-		len(b.HeadersRegex) == 0
+		len(b.HeadersRegex) == 0 &&
+		b.ASNs == nil &&
+		b.GeoIP == nil
 
 	if allFieldsEmpty && b.Expression == nil {
 		errs = append(errs, ErrBotMustHaveUserAgentOrPath)
@@ -146,7 +154,7 @@ func (b *BotConfig) Valid() error {
 	}
 
 	switch b.Action {
-	case RuleAllow, RuleBenchmark, RuleChallenge, RuleDeny, RuleWeigh:
+	case RuleAllow, RuleBenchmark, RuleChallenge, RuleDeny:
 		// okay
 	default:
 		errs = append(errs, fmt.Errorf("%w: %q", ErrUnknownAction, b.Action))
@@ -158,10 +166,6 @@ func (b *BotConfig) Valid() error {
 		}
 	}
 
-	if b.Action == RuleWeigh && b.Weight == nil {
-		b.Weight = &Weight{Adjust: 5}
-	}
-
 	if len(errs) != 0 {
 		return fmt.Errorf("config: bot entry for %q is not valid:\n%w", b.Name, errors.Join(errs...))
 	}
@@ -170,14 +174,15 @@ func (b *BotConfig) Valid() error {
 }
 
 type ChallengeRules struct {
-	Algorithm  string `json:"algorithm"`
-	Difficulty int    `json:"difficulty"`
-	ReportAs   int    `json:"report_as"`
+	Algorithm  Algorithm `json:"algorithm"`
+	Difficulty int       `json:"difficulty"`
+	ReportAs   int       `json:"report_as"`
 }
 
 var (
-	ErrChallengeDifficultyTooLow  = errors.New("config.Bot.ChallengeRules: difficulty is too low (must be >= 1)")
-	ErrChallengeDifficultyTooHigh = errors.New("config.Bot.ChallengeRules: difficulty is too high (must be <= 64)")
+	ErrChallengeRuleHasWrongAlgorithm = errors.New("config.Bot.ChallengeRules: algorithm is invalid")
+	ErrChallengeDifficultyTooLow      = errors.New("config.Bot.ChallengeRules: difficulty is too low (must be >= 1)")
+	ErrChallengeDifficultyTooHigh     = errors.New("config.Bot.ChallengeRules: difficulty is too high (must be <= 64)")
 )
 
 func (cr ChallengeRules) Valid() error {
@@ -191,6 +196,13 @@ func (cr ChallengeRules) Valid() error {
 		errs = append(errs, fmt.Errorf("%w, got: %d", ErrChallengeDifficultyTooHigh, cr.Difficulty))
 	}
 
+	switch cr.Algorithm {
+	case AlgorithmFast, AlgorithmSlow, AlgorithmUnknown:
+		// do nothing, it's all good
+	default:
+		errs = append(errs, fmt.Errorf("%w: %q", ErrChallengeRuleHasWrongAlgorithm, cr.Algorithm))
+	}
+
 	if len(errs) != 0 {
 		return fmt.Errorf("config: challenge rules entry is not valid:\n%w", errors.Join(errs...))
 	}
@@ -216,7 +228,7 @@ func (is *ImportStatement) open() (fs.File, error) {
 func (is *ImportStatement) load() error {
 	fin, err := is.open()
 	if err != nil {
-		return fmt.Errorf("%w: %s: %w", ErrInvalidImportStatement, is.Import, err)
+		return fmt.Errorf("can't open %s: %w", is.Import, err)
 	}
 	defer fin.Close()
 

lib/policy/config/config_test.go

@@ -130,6 +130,20 @@ func TestBotValid(t *testing.T) {
 			},
 			err: ErrChallengeDifficultyTooHigh,
 		},
+		{
+			name: "challenge wrong algorithm",
+			bot: BotConfig{
+				Name:      "mozilla-ua",
+				Action:    RuleChallenge,
+				PathRegex: p("Mozilla"),
+				Challenge: &ChallengeRules{
+					Difficulty: 420,
+					ReportAs:   4,
+					Algorithm:  "high quality rips",
+				},
+			},
+			err: ErrChallengeRuleHasWrongAlgorithm,
+		},
 		{
 			name: "invalid cidr range",
 			bot: BotConfig{
@@ -168,25 +182,6 @@ func TestBotValid(t *testing.T) {
 			},
 			err: nil,
 		},
-		{
-			name: "weight rule without weight",
-			bot: BotConfig{
-				Name:           "weight-adjust-if-mozilla",
-				Action:         RuleWeigh,
-				UserAgentRegex: p("Mozilla"),
-			},
-		},
-		{
-			name: "weight rule with weight adjust",
-			bot: BotConfig{
-				Name:           "weight-adjust-if-mozilla",
-				Action:         RuleWeigh,
-				UserAgentRegex: p("Mozilla"),
-				Weight: &Weight{
-					Adjust: 5,
-				},
-			},
-		},
 	}
 
 	for _, cs := range tests {
@@ -256,7 +251,6 @@ func TestImportStatement(t *testing.T) {
 		"bots",
 		"common",
 		"crawlers",
-		"meta",
 	} {
 		if err := fs.WalkDir(data.BotPolicies, folderName, func(path string, d fs.DirEntry, err error) error {
 			if err != nil {
@@ -265,9 +259,6 @@ func TestImportStatement(t *testing.T) {
 			if d.IsDir() {
 				return nil
 			}
-			if d.Name() == "README.md" {
-				return nil
-			}
 
 			tests = append(tests, testCase{
 				name:       "(data)/" + path,
@@ -366,7 +357,7 @@ func TestBotConfigZero(t *testing.T) {
 	b.Challenge = &ChallengeRules{
 		Difficulty: 4,
 		ReportAs:   4,
-		Algorithm:  DefaultAlgorithm,
+		Algorithm:  AlgorithmFast,
 	}
 	if b.Zero() {
 		t.Error("BotConfig with challenge rules is zero value")

lib/policy/config/expressionorlist.go

@@ -14,8 +14,8 @@ var (
 
 type ExpressionOrList struct {
 	Expression string   `json:"-"`
-	All        []string `json:"all,omitempty"`
-	Any        []string `json:"any,omitempty"`
+	All        []string `json:"all"`
+	Any        []string `json:"any"`
 }
 
 func (eol ExpressionOrList) Equal(rhs *ExpressionOrList) bool {
@@ -54,9 +54,6 @@ func (eol *ExpressionOrList) UnmarshalJSON(data []byte) error {
 }
 
 func (eol *ExpressionOrList) Valid() error {
-	if eol.Expression == "" && len(eol.All) == 0 && len(eol.Any) == 0 {
-		return ErrExpressionEmpty
-	}
 	if len(eol.All) != 0 && len(eol.Any) != 0 {
 		return ErrExpressionCantHaveBoth
 	}

lib/policy/config/expressionorlist_test.go

@@ -51,13 +51,6 @@ func TestExpressionOrListUnmarshal(t *testing.T) {
 			}`,
 			validErr: ErrExpressionCantHaveBoth,
 		},
-		{
-			name: "expression-empty",
-			inp: `{
-			"any": []
-			}`,
-			validErr: ErrExpressionEmpty,
-		},
 	} {
 		t.Run(tt.name, func(t *testing.T) {
 			var eol ExpressionOrList

lib/policy/config/geoip.go

@@ -0,0 +1,36 @@
+package config
+
+import (
+	"errors"
+	"fmt"
+	"regexp"
+	"strings"
+)
+
+var (
+	countryCodeRegexp = regexp.MustCompile(`^\w{2}$`)
+
+	ErrNotCountryCode = errors.New("config.Bot: invalid country code")
+)
+
+type GeoIP struct {
+	Countries []string `json:"countries"`
+}
+
+func (g *GeoIP) Valid() error {
+	var errs []error
+
+	for i, cc := range g.Countries {
+		if !countryCodeRegexp.MatchString(cc) {
+			errs = append(errs, fmt.Errorf("%w: %s", ErrNotCountryCode, cc))
+		}
+
+		g.Countries[i] = strings.ToLower(cc)
+	}
+
+	if len(errs) != 0 {
+		return fmt.Errorf("bot.GeoIP: invalid GeoIP settings: %w", errors.Join(errs...))
+	}
+
+	return nil
+}