feat: add valkey backed store

Signed-off-by: Xe Iaso <me@xeiaso.net>

.github/actions/spelling/expect.txt

@@ -1,4 +1,3 @@
-acs
 aeacus
 Aibrew
 alrest
@@ -81,7 +80,6 @@ goodbot
 googlebot
 govulncheck
 GPG
-grw
 Hashcash
 hashrate
 headermap
@@ -89,9 +87,7 @@ healthcheck
 hec
 hmc
 hostable
-htmx
 httpdebug
-hypertext
 iat
 ifm
 inp
@@ -114,13 +110,11 @@ lgbt
 licend
 licstart
 lightpanda
-LIMSA
 Linting
 linuxbrew
 LLU
 loadbalancer
 lol
-LOMINSA
 maintainership
 malware
 mcr
@@ -155,7 +149,6 @@ promauto
 promhttp
 pwcmd
 pwuser
-qualys
 qwant
 qwantbot
 rac
@@ -169,16 +162,12 @@ risc
 ruleset
 RUnlock
 sas
-sasl
 Scumm
-searx
 sebest
 secretplans
 selfsigned
 setsebool
 sitemap
-sls
-sni
 Sourceware
 Spambot
 sparkline
@@ -211,7 +200,7 @@ webmaster
 webpage
 websecure
 websites
-Workaround
+workaround
 workdir
 xcaddy
 Xeact
@@ -221,7 +210,6 @@ xesite
 xess
 xff
 XForwarded
-XNG
 XReal
 yae
 YAMLTo

.github/actions/spelling/line_forbidden.patterns

@@ -20,6 +20,9 @@
 # https://twitter.com/nyttypos/status/1898844061873639490
 #\([A-Z][a-z]{2,}(?: [a-z]+){3,}\)\.\s
 
+# Complete sentences shouldn't be in the middle of another sentence as a parenthetical.
+(?<!\.)\.\),
+
 # Complete sentences in parentheticals should not have a space before the period.
 \s\.\)(?!.*\}\})
 

.github/actions/spelling/patterns.txt

@@ -128,7 +128,3 @@ go install(?:\s+[a-z]+\.[-@\w/.]+)+
 
 # ignore long runs of a single character:
 \b([A-Za-z])\g{-1}{3,}\b
-
-# hit-count: 1 file-count: 1
-# microsoft
-\b(?:https?://|)(?:(?:(?:blogs|download\.visualstudio|docs|msdn2?|research)\.|)microsoft|blogs\.msdn)\.co(?:m|\.\w\w)/[-_a-zA-Z0-9()=./%]*

.github/workflows/docs-deploy.yml

@@ -3,7 +3,7 @@ name: Docs deploy
 on:
   workflow_dispatch:
   push:
-    branches: ["main"]
+    branches: [ "main" ]
 
 permissions:
   contents: read
@@ -24,7 +24,7 @@ jobs:
       - name: Set up Docker Buildx
         uses: docker/setup-buildx-action@b5ca514318bd6ebac0fb2aedd5d36ec1b5c232a2 # v3.10.0
 
-      - name: Log into registry
+      - name: Log into registry 
         uses: docker/login-action@74a5d142397b4f367a81961eba4e8cd7edddf772 # v3.4.0
         with:
           registry: ghcr.io
@@ -39,7 +39,7 @@ jobs:
 
       - name: Build and push
         id: build
-        uses: docker/build-push-action@1dc73863535b631f98b2378be8619f83b136f4a0 # v6.17.0
+        uses: docker/build-push-action@14487ce63c7a62a4a324b0bfb37086795e31c6c1 # v6.16.0
         with:
           context: ./docs
           cache-to: type=gha
@@ -50,15 +50,15 @@ jobs:
           push: true
 
       - name: Apply k8s manifests to aeacus
-        uses: actions-hub/kubectl@f632a31512a74cb35940627c49c20f67723cbaaf # v1.33.1
+        uses: actions-hub/kubectl@e81783053d902f50d752d21a6d99cf9689a652e1 # v1.33.0
         env:
-          KUBE_CONFIG: ${{ secrets.LIMSA_LOMINSA_KUBECONFIG }}
+          KUBE_CONFIG: ${{ secrets.AEACUS_KUBECONFIG }}
         with:
           args: apply -k docs/manifest
 
       - name: Apply k8s manifests to aeacus
-        uses: actions-hub/kubectl@f632a31512a74cb35940627c49c20f67723cbaaf # v1.33.1
+        uses: actions-hub/kubectl@e81783053d902f50d752d21a6d99cf9689a652e1 # v1.33.0
         env:
-          KUBE_CONFIG: ${{ secrets.LIMSA_LOMINSA_KUBECONFIG }}
+          KUBE_CONFIG: ${{ secrets.AEACUS_KUBECONFIG }}
         with:
           args: rollout restart -n default deploy/anubis-docs

.github/workflows/docs-test.yml

@@ -28,7 +28,7 @@ jobs:
 
       - name: Build and push
         id: build
-        uses: docker/build-push-action@1dc73863535b631f98b2378be8619f83b136f4a0 # v6.17.0
+        uses: docker/build-push-action@14487ce63c7a62a4a324b0bfb37086795e31c6c1 # v6.16.0
         with:
           context: ./docs
           cache-to: type=gha

.github/workflows/spelling.yml

@@ -89,7 +89,7 @@ jobs:
     steps:
       - name: check-spelling
         id: spelling
-        uses: check-spelling/check-spelling@c635c2f3f714eec2fcf27b643a1919b9a811ef2e # v0.0.25
+        uses: check-spelling/check-spelling@67debf50669c7fc76fc8f5d7f996384535a72b77 # v0.0.24
         with:
           suppress_push_for_open_pull_request: ${{ github.actor != 'dependabot[bot]' && 1 }}
           checkout: true

.github/workflows/zizmor.yml

@@ -29,7 +29,7 @@ jobs:
           GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} 
 
       - name: Upload SARIF file
-        uses: github/codeql-action/upload-sarif@ff0a06e83cb2de871e5a09832bc6a81e7276941f # v3.28.18
+        uses: github/codeql-action/upload-sarif@60168efe1c415ce0f5521ea06d5c2062adbeed1b # v3.28.17
         with:
           sarif_file: results.sarif
           category: zizmor

VERSION

@@ -1 +1 @@
-1.19.0-pre1
+1.18.0

anubis.go

@@ -16,8 +16,6 @@ const CookieName = "techaro.lol-anubis-auth"
 // WithDomainCookieName is the name that is prepended to the per-domain cookie used when COOKIE_DOMAIN is set.
 const WithDomainCookieName = "techaro.lol-anubis-auth-for-"
 
-const TestCookieName = "techaro.lol-anubis-cookie-test-if-you-block-this-anubis-wont-work"
-
 // CookieDefaultExpirationTime is the amount of time before the cookie/JWT expires.
 const CookieDefaultExpirationTime = 7 * 24 * time.Hour
 

cmd/anubis/main.go

@@ -56,8 +56,6 @@ var (
 	redirectDomains          = flag.String("redirect-domains", "", "list of domains separated by commas which anubis is allowed to redirect to. Leaving this unset allows any domain.")
 	slogLevel                = flag.String("slog-level", "INFO", "logging level (see https://pkg.go.dev/log/slog#hdr-Levels)")
 	target                   = flag.String("target", "http://localhost:3923", "target to reverse proxy to, set to an empty string to disable proxying when only using auth request")
-	targetSNI                = flag.String("target-sni", "", "if set, the value of the TLS handshake hostname when forwarding requests to the target")
-	targetHost               = flag.String("target-host", "", "if set, the value of the Host header when forwarding requests to the target")
 	targetInsecureSkipVerify = flag.Bool("target-insecure-skip-verify", false, "if true, skips TLS validation for the backend")
 	healthcheck              = flag.Bool("healthcheck", false, "run a health check against Anubis")
 	useRemoteAddress         = flag.Bool("use-remote-address", false, "read the client's IP address from the network request, useful for debugging and running Anubis on bare metal")
@@ -67,6 +65,7 @@ var (
 	ogCacheConsiderHost      = flag.Bool("og-cache-consider-host", false, "enable or disable the use of the host in the Open Graph tag cache")
 	extractResources         = flag.String("extract-resources", "", "if set, extract the static resources to the specified folder")
 	webmasterEmail           = flag.String("webmaster-email", "", "if set, displays webmaster's email on the reject page for appeals")
+	valkeyURL                = flag.String("valkey-url", "", "Valkey URL for Anubis' state layer")
 )
 
 func keyFromHex(value string) (ed25519.PrivateKey, error) {
@@ -137,7 +136,7 @@ func setupListener(network string, address string) (net.Listener, string) {
 	return listener, formattedAddress
 }
 
-func makeReverseProxy(target string, targetSNI string, targetHost string, insecureSkipVerify bool) (http.Handler, error) {
+func makeReverseProxy(target string, insecureSkipVerify bool) (http.Handler, error) {
 	targetUri, err := url.Parse(target)
 	if err != nil {
 		return nil, fmt.Errorf("failed to parse target URL: %w", err)
@@ -159,28 +158,16 @@ func makeReverseProxy(target string, targetSNI string, targetHost string, insecu
 		transport.RegisterProtocol("unix", libanubis.UnixRoundTripper{Transport: transport})
 	}
 
-	if insecureSkipVerify || targetSNI != "" {
-		transport.TLSClientConfig = &tls.Config{}
-		if insecureSkipVerify {
-			slog.Warn("TARGET_INSECURE_SKIP_VERIFY is set to true, TLS certificate validation will not be performed", "target", target)
-			transport.TLSClientConfig.InsecureSkipVerify = true
-		}
-		if targetSNI != "" {
-			transport.TLSClientConfig.ServerName = targetSNI
+	if insecureSkipVerify {
+		slog.Warn("TARGET_INSECURE_SKIP_VERIFY is set to true, TLS certificate validation will not be performed", "target", target)
+		transport.TLSClientConfig = &tls.Config{
+			InsecureSkipVerify: true,
 		}
 	}
 
 	rp := httputil.NewSingleHostReverseProxy(targetUri)
 	rp.Transport = transport
 
-	if targetHost != "" {
-		originalDirector := rp.Director
-		rp.Director = func(req *http.Request) {
-			originalDirector(req)
-			req.Host = targetHost
-		}
-	}
-
 	return rp, nil
 }
 
@@ -219,7 +206,7 @@ func main() {
 	// when using anubis via Systemd and environment variables, then it is not possible to set targe to an empty string but only to space
 	if strings.TrimSpace(*target) != "" {
 		var err error
-		rp, err = makeReverseProxy(*target, *targetSNI, *targetHost, *targetInsecureSkipVerify)
+		rp, err = makeReverseProxy(*target, *targetInsecureSkipVerify)
 		if err != nil {
 			log.Fatalf("can't make reverse proxy: %v", err)
 		}

data/apps/bookstack-saml.yaml

@@ -1,20 +0,0 @@
-# Make SASL login work on bookstack with Anubis
-# https://www.bookstackapp.com/docs/admin/saml2-auth/
-- name: allow-bookstack-sasl-login-routes
-  action: ALLOW
-  expression:
-    all:
-      - 'method == "POST"'
-      - path.startsWith("/saml2/acs")
-- name: allow-bookstack-sasl-metadata-routes
-  action: ALLOW
-  expression:
-    all:
-      - 'method == "GET"'
-      - path.startsWith("/saml2/metadata")
-- name: allow-bookstack-sasl-logout-routes
-  action: ALLOW
-  expression:
-    all:
-      - 'method == "GET"'
-      - path.startsWith("/saml2/sls")

data/apps/qualys-ssl-labs.yml

@@ -1,7 +0,0 @@
-# This policy allows Qualys SSL Labs to fully work. (https://www.ssllabs.com/ssltest)
-# IP ranges are taken from: https://qualys.my.site.com/discussions/s/article/000005823
-- name: qualys-ssl-labs
-  action: ALLOW
-  remote_addresses:
-  - 64.41.200.0/24
-  - 2600:C02:1020:4202::/64

data/apps/searx-checker.yml

@@ -1,9 +0,0 @@
-# This policy allows SearXNG's instance tracker to work. (https://searx.space)
-# IPs are taken from `check.searx.space` DNS records.
-# https://toolbox.googleapps.com/apps/dig/#A/check.searx.space
-# https://toolbox.googleapps.com/apps/dig/#AAAA/check.searx.space
-- name: searx-checker
-  action: ALLOW
-  remote_addresses:
-  - 167.235.158.251/32
-  - 2a01:4f8:1c1c:8fc2::1/128

data/botPolicies.yaml

@@ -48,7 +48,16 @@ bots:
 - name: generic-browser
   user_agent_regex: >-
     Mozilla|Opera
-  action: CHALLENGE
+  action: WEIGH
+  weight:
+    adjust: 5
+
+- name: high-pass-rate
+  pass_rate:
+    rate: 0.8
+  action: WEIGH
+  weight:
+    adjust: -15
 
 dnsbl: false
 

data/bots/aggressive-brazilian-scrapers.yaml

@@ -1,5 +1,5 @@
-- name: deny-aggressive-brazilian-scrapers
-  action: DENY
+- name: aggressive-brazilian-scrapers
+  action: WEIGH
   expression:
     any:
     # Internet Explorer should be out of support
@@ -18,11 +18,9 @@
     - userAgent.contains("Win 9x")
     # Amazon does not have an Alexa Toolbar.
     - userAgent.contains("Alexa Toolbar")
-- name: challenge-aggressive-brazilian-scrapers
-  action: CHALLENGE
-  expression:
-    any:
     # This is not released, even Windows 11 calls itself Windows 10
     - userAgent.contains("Windows NT 11.0")
     # iPods are not in common use
-    - userAgent.contains("iPod")
+    - userAgent.contains("iPod")
+  weight:
+    adjust: 10

data/bots/ai-robots-txt.yaml

@@ -1,4 +1,4 @@
 - name: "ai-robots-txt"
   user_agent_regex: >-
-    AI2Bot|Ai2Bot-Dolma|aiHitBot|Amazonbot|anthropic-ai|Applebot|Applebot-Extended|Brightbot 1.0|Bytespider|CCBot|ChatGPT-User|Claude-SearchBot|Claude-User|Claude-Web|ClaudeBot|cohere-ai|cohere-training-data-crawler|Cotoyogi|Crawlspace|Diffbot|DuckAssistBot|FacebookBot|Factset_spyderbot|FirecrawlAgent|FriendlyCrawler|Google-CloudVertexBot|Google-Extended|GoogleOther|GoogleOther-Image|GoogleOther-Video|GPTBot|iaskspider/2.0|ICC-Crawler|ImagesiftBot|img2dataset|imgproxy|ISSCyberRiskCrawler|Kangaroo Bot|meta-externalagent|Meta-ExternalAgent|meta-externalfetcher|Meta-ExternalFetcher|MistralAI-User/1.0|NovaAct|OAI-SearchBot|omgili|omgilibot|Operator|PanguBot|Perplexity-User|PerplexityBot|PetalBot|QualifiedBot|Scrapy|SemrushBot-OCOB|SemrushBot-SWA|Sidetrade indexer bot|TikTokSpider|Timpibot|VelenPublicWebCrawler|Webzio-Extended|wpbot|YouBot
+    AI2Bot|Ai2Bot-Dolma|aiHitBot|Amazonbot|anthropic-ai|Applebot|Applebot-Extended|Brightbot 1.0|Bytespider|CCBot|ChatGPT-User|Claude-Web|ClaudeBot|cohere-ai|cohere-training-data-crawler|Cotoyogi|Crawlspace|Diffbot|DuckAssistBot|FacebookBot|Factset_spyderbot|FirecrawlAgent|FriendlyCrawler|Google-Extended|GoogleOther|GoogleOther-Image|GoogleOther-Video|GPTBot|iaskspider/2.0|ICC-Crawler|ImagesiftBot|img2dataset|imgproxy|ISSCyberRiskCrawler|Kangaroo Bot|meta-externalagent|Meta-ExternalAgent|meta-externalfetcher|Meta-ExternalFetcher|NovaAct|OAI-SearchBot|omgili|omgilibot|Operator|PanguBot|Perplexity-User|PerplexityBot|PetalBot|Scrapy|SemrushBot-OCOB|SemrushBot-SWA|Sidetrade indexer bot|TikTokSpider|Timpibot|VelenPublicWebCrawler|Webzio-Extended|YouBot
   action: DENY

data/bots/cloudflare-workers.yaml

@@ -1,4 +1,6 @@
 - name: cloudflare-workers
   headers_regex:
     CF-Worker: .*
-  action: DENY
+  action: WEIGH
+  weight:
+    adjust: 5

data/bots/headless-browsers.yaml

@@ -1,9 +1,14 @@
 - name: lightpanda
   user_agent_regex: ^LightPanda/.*$
-  action: DENY
+  action: WEIGH
+  weight:
+    adjust: 5
 - name: headless-chrome
   user_agent_regex: HeadlessChrome
-  action: DENY
+  action: WEIGH
+  weight:
+    adjust: 5
 - name: headless-chromium
   user_agent_regex: HeadlessChromium
-  action: DENY
+  weight:
+    adjust: 5

data/common/rfc-violations.yaml

@@ -1,3 +1,5 @@
 - name: no-user-agent-string
-  action: DENY
-  expression: userAgent == ""
+  action: WEIGH
+  expression: userAgent == ""
+  weight:
+    adjust: 10

docs/docs/CHANGELOG.md

@@ -11,11 +11,6 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 
 ## [Unreleased]
 
-## v1.19.0: Jenomis cen Lexentale
-
-- Record if challenges were issued via the API or via embedded JSON in the challenge page HTML ([#531](https://github.com/TecharoHQ/anubis/issues/531))
-- Ensure that clients that are shown a challenge support storing cookies
-- Encode challenge pages with gzip level 1
 - Add `check-spelling` for spell checking
 - Add `--target-insecure-skip-verify` flag/envvar to allow Anubis to hit a self-signed HTTPS backend
 - Minor adjustments to FreeBSD rc.d script to allow for more flexible configuration.
@@ -23,15 +18,6 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 - Updated the nonce value in the challenge JWT cookie to be a string instead of a number
 - Rename cookies in response to user feedback
 - Ensure cookie renaming is consistent across configuration options
-- Add Bookstack app in data
-- Add `--target-host` flag/envvar to allow changing the value of the Host header in requests forwarded to the target service.
-- Bump AI-robots.txt to version 1.31
-- Add `RuntimeDirectory` to systemd unit settings so native packages can listen over unix sockets
-- Added SearXNG instance tracker whitelist policy
-- Added Qualys SSL Labs whitelist policy
-- Fixed cookie deletion logic ([#520](https://github.com/TecharoHQ/anubis/issues/520), [#522](https://github.com/TecharoHQ/anubis/pull/522))
-- Add `--target-sni` flag/envvar to allow changing the value of the TLS handshake hostname in requests forwarded to the target service.
-- Fixed CEL expression matching validator to now properly error out when it receives empty expressions
 
 ## v1.18.0: Varis zos Galvus
 
@@ -57,7 +43,7 @@ Or as complicated as:
   expression:
     all:
       - >-
-        (
+        (  
           userAgent.startsWith("git/") ||
           userAgent.contains("libgit") ||
           userAgent.startsWith("go-git") ||

docs/docs/admin/frameworks/_category_.json

@@ -1,8 +0,0 @@
-{
-  "label": "Frameworks",
-  "position": 30,
-  "link": {
-    "type": "generated-index",
-    "description": "Information about getting specific frameworks or tools working with Anubis."
-  }
-}

docs/docs/admin/frameworks/htmx.mdx

@@ -1,45 +0,0 @@
-# HTMX
-
-import Tabs from "@theme/Tabs";
-import TabItem from "@theme/TabItem";
-
-[HTMX](https://htmx.org) is a framework that enables you to write applications using hypertext as the engine of application state. This enables you to simplify you server side code by having it return HTML instead of JSON. This can interfere with Anubis because Anubis challenge pages also return HTML.
-
-To work around this, you can make a custom [expression](../configuration/expressions.mdx) rule that allows HTMX requests if the user has passed a challenge in the past:
-
-<Tabs>
-<TabItem value="json" label="JSON">
-
-```json
-{
-  "name": "allow-htmx-iff-already-passed-challenge",
-  "action": "ALLOW",
-  "expression": {
-    "all": [
-      "\"Cookie\" in headers",
-      "headers[\"Cookie\"].contains(\"anubis-auth\")",
-      "\"Hx-Request\" in headers",
-      "headers[\"Hx-Request\"] == \"true\""
-    ]
-  }
-}
-```
-
-</TabItem>
-<TabItem value="yaml" label="YAML" default>
-
-```yaml
-- name: allow-htmx-iff-already-passed-challenge
-  action: ALLOW
-  expression:
-    all:
-      - '"Cookie" in headers'
-      - 'headers["Cookie"].contains("anubis-auth")'
-      - '"Hx-Request" in headers'
-      - 'headers["Hx-Request"] == "true"'
-```
-
-</TabItem>
-</Tabs>
-
-This will reduce some security because it does not assert the validity of the Anubis auth cookie, however in trade it improves the experience for existing users.

docs/docs/admin/frameworks/wordpress.mdx

@@ -1,39 +0,0 @@
-# Wordpress
-
-Wordpress is the most popular blog engine on the planet.
-
-## Using a multi-site setup with Anubis
-
-If you have a multi-site setup where traffic goes through Anubis like this:
-
-```mermaid
----
-title: Apache as tls terminator and HTTP router
----
-
-flowchart LR
-    T(User Traffic)
-    subgraph Apache 2
-        TCP(TCP 80/443)
-        US(TCP 3001)
-    end
-
-    An(Anubis)
-    B(Backend)
-
-    T --> |TLS termination| TCP
-    TCP --> |Traffic filtering| An
-    An --> |Happy traffic| US
-    US --> |whatever you're doing| B
-```
-
-Wordpress may not realize that the underlying connection is being done over HTTPS. This could lead to a redirect loop in the `/wp-admin/` routes. In order to fix this, add the following to your `wp-config.php` file:
-
-```php
-if (isset($_SERVER['HTTP_X_FORWARDED_PROTO']) && $_SERVER['HTTP_X_FORWARDED_PROTO'] === 'https') {
-    $_SERVER['HTTPS'] = 'on';
-    $_SERVER['SERVER_PORT'] = 443;
-}
-```
-
-This will make Wordpress think that your connection is over HTTPS instead of plain HTTP.

docs/docs/admin/installation.mdx

@@ -54,7 +54,7 @@ Anubis uses these environment variables for configuration:
 | `BASE_PREFIX`                  | unset                   | If set, adds a global prefix to all Anubis endpoints. For example, setting this to `/myapp` would make Anubis accessible at `/myapp/` instead of `/`. This is useful when running Anubis behind a reverse proxy that routes based on path prefixes.                                                                  |
 | `BIND`                         | `:8923`                 | The network address that Anubis listens on. For `unix`, set this to a path: `/run/anubis/instance.sock`                                                                                                                                                                                                              |
 | `BIND_NETWORK`                 | `tcp`                   | The address family that Anubis listens on. Accepts `tcp`, `unix` and anything Go's [`net.Listen`](https://pkg.go.dev/net#Listen) supports.                                                                                                                                                                           |
-| `COOKIE_DOMAIN`                | unset                   | The domain the Anubis challenge pass cookie should be set to. This should be set to the domain you bought from your registrar (EG: `techaro.lol` if your webapp is running on `anubis.techaro.lol`). See this [stackoverflow explanation of cookies](https://stackoverflow.com/a/1063760) for more information.<br/><br/>Note that unlike `REDIRECT_DOMAINS`, you should never include a port number in this variable. |
+| `COOKIE_DOMAIN`                | unset                   | The domain the Anubis challenge pass cookie should be set to. This should be set to the domain you bought from your registrar (EG: `techaro.lol` if your webapp is running on `anubis.techaro.lol`). See this [stackoverflow explanation of cookies](https://stackoverflow.com/a/1063760) for more information.      |
 | `COOKIE_EXPIRATION_TIME`       | `168h`                  | The amount of time the authorization cookie is valid for.                                                                                                                                                                                                                                                            |
 | `COOKIE_PARTITIONED`           | `false`                 | If set to `true`, enables the [partitioned (CHIPS) flag](https://developers.google.com/privacy-sandbox/cookies/chips), meaning that Anubis inside an iframe has a different set of cookies than the domain hosting the iframe.                                                                                       |
 | `DIFFICULTY`                   | `4`                     | The difficulty of the challenge, or the number of leading zeroes that must be in successful responses.                                                                                                                                                                                                               |
@@ -66,7 +66,7 @@ Anubis uses these environment variables for configuration:
 | `OG_PASSTHROUGH`               | `false`                 | If set to `true`, Anubis will enable Open Graph tag passthrough.                                                                                                                                                                                                                                                     |
 | `OG_CACHE_CONSIDER_HOST`       | `false`                 | If set to `true`, Anubis will consider the host in the Open Graph tag cache key.                                                                                                                                                                                                                                     |
 | `POLICY_FNAME`                 | unset                   | The file containing [bot policy configuration](./policies.mdx). See the bot policy documentation for more details. If unset, the default bot policy configuration is used.                                                                                                                                           |
-| `REDIRECT_DOMAINS`             | unset                   | If set, restrict the domains that Anubis can redirect to when passing a challenge.<br/><br/>If this is unset, Anubis may redirect to any domain which could cause security issues in the unlikely case that an attacker passes a challenge for your browser and then tricks you into clicking a link to your domain.<br/><br/>Note that if you are hosting Anubis on a non-standard port (`https://example:com:8443`, `http://www.example.net:8080`, etc.), you must also include the port number here. |
+| `REDIRECT_DOMAINS`             | unset                   | If set, restrict the domains that Anubis can redirect to when passing a challenge.<br/><br/>If this is unset, Anubis may redirect to any domain which could cause security issues in the unlikely case that an attacker passes a challenge for your browser and then tricks you into clicking a link to your domain. |
 | `SERVE_ROBOTS_TXT`             | `false`                 | If set `true`, Anubis will serve a default `robots.txt` file that disallows all known AI scrapers by name and then additionally disallows every scraper. This is useful if facts and circumstances make it difficult to change the underlying service to serve such a `robots.txt` file.                             |
 | `SOCKET_MODE`                  | `0770`                  | _Only used when at least one of the `*_BIND_NETWORK` variables are set to `unix`._ The socket mode (permissions) for Unix domain sockets.                                                                                                                                                                            |
 | `TARGET`                       | `http://localhost:3923` | The URL of the service that Anubis should forward valid requests to. Supports Unix domain sockets, set this to a URI like so: `unix:///path/to/socket.sock`.                                                                                                                                                         |
@@ -84,8 +84,6 @@ If you don't know or understand what these settings mean, ignore them. These are
 
 | Environment Variable          | Default value | Explanation                                                                                                                                         |
 | :---------------------------- | :------------ | :-------------------------------------------------------------------------------------------------------------------------------------------------- |
-| `TARGET_SNI`                  | unset         | If set, overrides the TLS handshake hostname in requests forwarded to `TARGET`.                                                                     |
-| `TARGET_HOST`                 | unset         | If set, overrides the Host header in requests forwarded to `TARGET`.                                                                                |
 | `TARGET_INSECURE_SKIP_VERIFY` | `false`       | If `true`, skip TLS certificate validation for targets that listen over `https`. If your backend does not listen over `https`, ignore this setting. |
 
 </details>

docs/docs/admin/native-install.mdx

@@ -49,7 +49,7 @@ sudo install -D ./run/anubis@.service /etc/systemd/system
 Install the default configuration file to your system:
 
 ```text
-sudo install -D ./run/default.env /etc/anubis/default.env
+sudo install -D ./run/default.env /etc/anubis
 ```
 
   </TabItem>
@@ -77,13 +77,6 @@ Install Anubis with `rpm`:
 sudo rpm -ivh ./anubis-$VERSION.$ARCH.rpm
 ```
 
-  </TabItem>
-  <TabItem value="distro" label="Package managers">
-
-Some Linux distributions offer Anubis [as a native package](https://repology.org/project/anubis-anti-crawler/versions). If you want to install Anubis from your distribution's package manager, consult any upstream documentation for how to install the package. It will either be named `anubis`, `www-apps/anubis` or `www/anubis`.
-
-If you use a systemd-flavoured distribution, then follow the setup instructions for Debian or Red Hat Linux.
-
   </TabItem>
 </Tabs>
 

docs/docs/user/known-instances.md

@@ -34,7 +34,6 @@ This page contains a non-exhaustive list with all websites using Anubis.
 - https://bugzilla.proxmox.com
 - https://hofstede.io/
 - https://www.indiemag.fr/
-- https://reddit.nerdvpn.de/
 - <details>
   <summary>FreeCAD</summary>
   - https://forum.freecad.org/

go.mod

@@ -9,6 +9,7 @@ require (
 	github.com/google/cel-go v0.25.0
 	github.com/playwright-community/playwright-go v0.5200.0
 	github.com/prometheus/client_golang v1.22.0
+	github.com/redis/go-redis/v9 v9.8.0
 	github.com/sebest/xff v0.0.0-20210106013422-671bd2870b3a
 	github.com/yl2chen/cidranger v1.0.2
 	golang.org/x/net v0.40.0
@@ -41,6 +42,7 @@ require (
 	github.com/cloudflare/circl v1.6.0 // indirect
 	github.com/cyphar/filepath-securejoin v0.4.1 // indirect
 	github.com/deckarep/golang-set/v2 v2.7.0 // indirect
+	github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f // indirect
 	github.com/dlclark/regexp2 v1.11.4 // indirect
 	github.com/dop251/goja v0.0.0-20250309171923-bcd7cc6bf64c // indirect
 	github.com/emirpasic/gods v1.18.1 // indirect

go.sum

@@ -46,6 +46,10 @@ github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM=
 github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw=
 github.com/blakesmith/ar v0.0.0-20190502131153-809d4375e1fb h1:m935MPodAbYS46DG4pJSv7WO+VECIWUQ7OJYSoTrMh4=
 github.com/blakesmith/ar v0.0.0-20190502131153-809d4375e1fb/go.mod h1:PkYb9DJNAwrSvRx5DYA+gUcOIgTGVMNkfSCbZM8cWpI=
+github.com/bsm/ginkgo/v2 v2.12.0 h1:Ny8MWAHyOepLGlLKYmXG4IEkioBysk6GpaRTLC8zwWs=
+github.com/bsm/ginkgo/v2 v2.12.0/go.mod h1:SwYbGRRDovPVboqFv0tPTcG1sN61LM1Z4ARdbAV9g4c=
+github.com/bsm/gomega v1.27.10 h1:yeMWxP2pV2fG3FgAODIY8EiRE3dy0aeFYt4l7wh6yKA=
+github.com/bsm/gomega v1.27.10/go.mod h1:JyEr/xRbxbtgWNi8tIEVPUYZ5Dzef52k01W3YH0H+O0=
 github.com/caarlos0/testfs v0.4.4 h1:3PHvzHi5Lt+g332CiShwS8ogTgS3HjrmzZxCm6JCDr8=
 github.com/caarlos0/testfs v0.4.4/go.mod h1:bRN55zgG4XCUVVHZCeU+/Tz1Q6AxEJOEJTliBy+1DMk=
 github.com/cavaliergopher/cpio v1.0.1 h1:KQFSeKmZhv0cr+kawA3a0xTQCU4QxXF1vhU7P7av2KM=
@@ -73,6 +77,8 @@ github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc h1:U9qPSI2PIWSS1
 github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
 github.com/deckarep/golang-set/v2 v2.7.0 h1:gIloKvD7yH2oip4VLhsv3JyLLFnC0Y2mlusgcvJYW5k=
 github.com/deckarep/golang-set/v2 v2.7.0/go.mod h1:VAky9rY/yGXJOLEDv3OMci+7wtDpOF4IN+y82NBOac4=
+github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f h1:lO4WD4F/rVNCu3HqELle0jiPLLBs70cWOduZpkS1E78=
+github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f/go.mod h1:cuUVRXasLTGF7a8hSLbxyZXjz+1KgoB3wDUb6vlszIc=
 github.com/dlclark/regexp2 v1.11.4 h1:rPYF9/LECdNymJufQKmri9gV604RvvABwgOA8un7yAo=
 github.com/dlclark/regexp2 v1.11.4/go.mod h1:DHkYz0B9wPfa6wondMfaivmHpzrQ3v9q8cnmRbL6yW8=
 github.com/dop251/goja v0.0.0-20250309171923-bcd7cc6bf64c h1:mxWGS0YyquJ/ikZOjSrRjjFIbUqIP9ojyYQ+QZTU3Rg=
@@ -224,6 +230,8 @@ github.com/prometheus/common v0.62.0 h1:xasJaQlnWAeyHdUBeGjXmutelfJHWMRr+Fg4QszZ
 github.com/prometheus/common v0.62.0/go.mod h1:vyBcEuLSvWos9B1+CyL7JZ2up+uFzXhkqml0W5zIY1I=
 github.com/prometheus/procfs v0.15.1 h1:YagwOFzUgYfKKHX6Dr+sHT7km/hxC76UB0learggepc=
 github.com/prometheus/procfs v0.15.1/go.mod h1:fB45yRUv8NstnjriLhBQLuOUt+WW4BsoGhij/e3PBqk=
+github.com/redis/go-redis/v9 v9.8.0 h1:q3nRvjrlge/6UD7eTu/DSg2uYiU2mCL0G/uzBWqhicI=
+github.com/redis/go-redis/v9 v9.8.0/go.mod h1:huWgSWd8mW6+m0VPhJjSSQ+d6Nh1VICQ6Q5lHuCH/Iw=
 github.com/rivo/uniseg v0.2.0/go.mod h1:J6wj4VEh+S6ZtnVlnTBMWIodfgj8LQOQFoIToxlJtxc=
 github.com/rogpeppe/go-internal v1.14.1 h1:UQB4HGPB6osV0SQTLymcB4TgvyWu6ZyliaW0tI/otEQ=
 github.com/rogpeppe/go-internal v1.14.1/go.mod h1:MaRKkUm5W0goXpeCfT7UZI6fk/L7L7so1lCWt35ZSgc=

internal/gzip.go

@@ -1,35 +0,0 @@
-package internal
-
-import (
-	"compress/gzip"
-	"net/http"
-	"strings"
-)
-
-func GzipMiddleware(level int, next http.Handler) http.Handler {
-	return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
-		if !strings.Contains(r.Header.Get("Accept-Encoding"), "gzip") {
-			next.ServeHTTP(w, r)
-			return
-		}
-
-		w.Header().Set("Content-Encoding", "gzip")
-		gz, err := gzip.NewWriterLevel(w, level)
-		if err != nil {
-			panic(err)
-		}
-		defer gz.Close()
-
-		grw := gzipResponseWriter{ResponseWriter: w, sink: gz}
-		next.ServeHTTP(grw, r)
-	})
-}
-
-type gzipResponseWriter struct {
-	http.ResponseWriter
-	sink *gzip.Writer
-}
-
-func (w gzipResponseWriter) Write(b []byte) (int, error) {
-	return w.sink.Write(b)
-}

internal/store/valkey/valkey.go

@@ -0,0 +1,86 @@
+package valkey
+
+import (
+	"context"
+	"errors"
+	"fmt"
+	"strconv"
+	"strings"
+
+	valkey "github.com/redis/go-redis/v9"
+)
+
+type Store struct {
+	rdb *valkey.Client
+}
+
+func New(rdb *valkey.Client) *Store {
+	return &Store{rdb: rdb}
+}
+
+func (s *Store) Increment(ctx context.Context, segments []string) error {
+	key := fmt.Sprintf("anubis:%s", strings.Join(segments, ":"))
+	if err := s.rdb.Incr(ctx, key).Err(); err != nil {
+		return err
+	}
+
+	return nil
+}
+
+func (s *Store) GetInt(ctx context.Context, segments []string) (int, error) {
+	key := fmt.Sprintf("anubis:%s", strings.Join(segments, ":"))
+	numStr, err := s.rdb.Get(ctx, key).Result()
+	if err != nil {
+		return 0, err
+	}
+
+	num, err := strconv.Atoi(numStr)
+	if err != nil {
+		return 0, err
+	}
+
+	return num, nil
+}
+
+func (s *Store) MultiGetInt(ctx context.Context, segments [][]string) ([]int, error) {
+	var keys []string
+	for _, segment := range segments {
+		key := fmt.Sprintf("anubis:%s", strings.Join(segment, ":"))
+		keys = append(keys, key)
+	}
+
+	values, err := s.rdb.MGet(ctx, keys...).Result()
+	if err != nil {
+		return nil, err
+	}
+
+	var errs []error
+
+	result := make([]int, len(values))
+	for i, val := range values {
+		if val == nil {
+			result[i] = 0
+			errs = append(errs, fmt.Errorf("can't get key %s: value is null", keys[i]))
+			continue
+		}
+
+		switch v := val.(type) {
+		case string:
+			num, err := strconv.Atoi(v)
+			if err != nil {
+				errs = append(errs, fmt.Errorf("can't parse key %s: %w", keys[i], err))
+				continue
+			}
+
+			result[i] = num
+		default:
+			errs = append(errs, fmt.Errorf("can't parse key %s: wanted type string but got type %T", keys[i], val))
+		}
+	}
+
+	if len(errs) != 0 {
+		return nil, fmt.Errorf("can't read from valkey: %w", errors.Join(errs...))
+	}
+
+	return result, nil
+}

lib/anubis.go

@@ -26,15 +26,16 @@ import (
 	"github.com/TecharoHQ/anubis/internal"
 	"github.com/TecharoHQ/anubis/internal/dnsbl"
 	"github.com/TecharoHQ/anubis/internal/ogtags"
+	"github.com/TecharoHQ/anubis/internal/store/valkey"
 	"github.com/TecharoHQ/anubis/lib/policy"
 	"github.com/TecharoHQ/anubis/lib/policy/config"
 )
 
 var (
-	challengesIssued = promauto.NewCounterVec(prometheus.CounterOpts{
+	challengesIssued = promauto.NewCounter(prometheus.CounterOpts{
 		Name: "anubis_challenges_issued",
 		Help: "The total number of challenges issued",
-	}, []string{"method"})
+	})
 
 	challengesValidated = promauto.NewCounter(prometheus.CounterOpts{
 		Name: "anubis_challenges_validated",
@@ -68,6 +69,7 @@ type Server struct {
 	pub        ed25519.PublicKey
 	opts       Options
 	cookieName string
+	store      *valkey.Store
 }
 
 func (s *Server) challengeFor(r *http.Request, difficulty int) string {
@@ -96,12 +98,6 @@ func (s *Server) maybeReverseProxyOrPage(w http.ResponseWriter, r *http.Request)
 func (s *Server) maybeReverseProxy(w http.ResponseWriter, r *http.Request, httpStatusOnly bool) {
 	lg := internal.GetRequestLogger(r)
 
-	// Adjust cookie path if base prefix is not empty
-	cookiePath := "/"
-	if anubis.BasePrefix != "" {
-		cookiePath = strings.TrimSuffix(anubis.BasePrefix, "/") + "/"
-	}
-
 	cr, rule, err := s.check(r)
 	if err != nil {
 		lg.Error("check failed", "err", err)
@@ -127,21 +123,21 @@ func (s *Server) maybeReverseProxy(w http.ResponseWriter, r *http.Request, httpS
 	ckie, err := r.Cookie(s.cookieName)
 	if err != nil {
 		lg.Debug("cookie not found", "path", r.URL.Path)
-		s.ClearCookie(w, s.cookieName, cookiePath)
+		s.ClearCookie(w)
 		s.RenderIndex(w, r, rule, httpStatusOnly)
 		return
 	}
 
 	if err := ckie.Valid(); err != nil {
 		lg.Debug("cookie is invalid", "err", err)
-		s.ClearCookie(w, s.cookieName, cookiePath)
+		s.ClearCookie(w)
 		s.RenderIndex(w, r, rule, httpStatusOnly)
 		return
 	}
 
 	if time.Now().After(ckie.Expires) && !ckie.Expires.IsZero() {
 		lg.Debug("cookie expired", "path", r.URL.Path)
-		s.ClearCookie(w, s.cookieName, cookiePath)
+		s.ClearCookie(w)
 		s.RenderIndex(w, r, rule, httpStatusOnly)
 		return
 	}
@@ -152,7 +148,7 @@ func (s *Server) maybeReverseProxy(w http.ResponseWriter, r *http.Request, httpS
 
 	if err != nil || !token.Valid {
 		lg.Debug("invalid token", "path", r.URL.Path, "err", err)
-		s.ClearCookie(w, s.cookieName, cookiePath)
+		s.ClearCookie(w)
 		s.RenderIndex(w, r, rule, httpStatusOnly)
 		return
 	}
@@ -162,19 +158,13 @@ func (s *Server) maybeReverseProxy(w http.ResponseWriter, r *http.Request, httpS
 }
 
 func (s *Server) checkRules(w http.ResponseWriter, r *http.Request, cr policy.CheckResult, lg *slog.Logger, rule *policy.Bot) bool {
-	// Adjust cookie path if base prefix is not empty
-	cookiePath := "/"
-	if anubis.BasePrefix != "" {
-		cookiePath = strings.TrimSuffix(anubis.BasePrefix, "/") + "/"
-	}
-
 	switch cr.Rule {
 	case config.RuleAllow:
 		lg.Debug("allowing traffic to origin (explicit)")
 		s.ServeHTTPNext(w, r)
 		return true
 	case config.RuleDeny:
-		s.ClearCookie(w, s.cookieName, cookiePath)
+		s.ClearCookie(w)
 		lg.Info("explicit deny")
 		if rule == nil {
 			lg.Error("rule is nil, cannot calculate checksum")
@@ -193,7 +183,7 @@ func (s *Server) checkRules(w http.ResponseWriter, r *http.Request, cr policy.Ch
 		s.RenderBench(w, r)
 		return true
 	default:
-		s.ClearCookie(w, s.cookieName, cookiePath)
+		s.ClearCookie(w)
 		slog.Error("CONFIG ERROR: unknown rule", "rule", cr.Rule)
 		s.respondWithError(w, r, "Internal Server Error: administrator has misconfigured Anubis. Please contact the administrator and ask them to look for the logs around \"maybeReverseProxy.Rules\"")
 		return true
@@ -245,7 +235,9 @@ func (s *Server) MakeChallenge(w http.ResponseWriter, r *http.Request) {
 	lg = lg.With("check_result", cr)
 	challenge := s.challengeFor(r, rule.Challenge.Difficulty)
 
-	s.SetCookie(w, anubis.TestCookieName, challenge, "/")
+	if s.store != nil {
+		s.store.Increment(r.Context(), []string{"pass_rate", "User-Agent", r.UserAgent(), "challenges_issued"})
+	}
 
 	err = encoder.Encode(struct {
 		Rules     *config.ChallengeRules `json:"rules"`
@@ -260,20 +252,12 @@ func (s *Server) MakeChallenge(w http.ResponseWriter, r *http.Request) {
 		return
 	}
 	lg.Debug("made challenge", "challenge", challenge, "rules", rule.Challenge, "cr", cr)
-	challengesIssued.WithLabelValues("api").Inc()
+	challengesIssued.Inc()
 }
 
 func (s *Server) PassChallenge(w http.ResponseWriter, r *http.Request) {
 	lg := internal.GetRequestLogger(r)
 
-	// Adjust cookie path if base prefix is not empty
-	cookiePath := "/"
-	if anubis.BasePrefix != "" {
-		cookiePath = strings.TrimSuffix(anubis.BasePrefix, "/") + "/"
-	}
-
-	s.ClearCookie(w, anubis.TestCookieName, "/")
-
 	redir := r.FormValue("redir")
 	redirURL, err := url.ParseRequestURI(redir)
 	if err != nil {
@@ -287,14 +271,14 @@ func (s *Server) PassChallenge(w http.ResponseWriter, r *http.Request) {
 	cr, rule, err := s.check(r)
 	if err != nil {
 		lg.Error("check failed", "err", err)
-		s.respondWithError(w, r, "Internal Server Error: administrator has misconfigured Anubis. Please contact the administrator and ask them to look for the logs around \"passChallenge\".")
+		s.respondWithError(w, r, "Internal Server Error: administrator has misconfigured Anubis. Please contact the administrator and ask them to look for the logs around \"passChallenge\".\"")
 		return
 	}
 	lg = lg.With("check_result", cr)
 
 	nonceStr := r.FormValue("nonce")
 	if nonceStr == "" {
-		s.ClearCookie(w, s.cookieName, cookiePath)
+		s.ClearCookie(w)
 		lg.Debug("no nonce")
 		s.respondWithError(w, r, "missing nonce")
 		return
@@ -302,7 +286,7 @@ func (s *Server) PassChallenge(w http.ResponseWriter, r *http.Request) {
 
 	elapsedTimeStr := r.FormValue("elapsedTime")
 	if elapsedTimeStr == "" {
-		s.ClearCookie(w, s.cookieName, cookiePath)
+		s.ClearCookie(w)
 		lg.Debug("no elapsedTime")
 		s.respondWithError(w, r, "missing elapsedTime")
 		return
@@ -310,7 +294,7 @@ func (s *Server) PassChallenge(w http.ResponseWriter, r *http.Request) {
 
 	elapsedTime, err := strconv.ParseFloat(elapsedTimeStr, 64)
 	if err != nil {
-		s.ClearCookie(w, s.cookieName, cookiePath)
+		s.ClearCookie(w)
 		lg.Debug("elapsedTime doesn't parse", "err", err)
 		s.respondWithError(w, r, "invalid elapsedTime")
 		return
@@ -332,19 +316,11 @@ func (s *Server) PassChallenge(w http.ResponseWriter, r *http.Request) {
 
 	challenge := s.challengeFor(r, rule.Challenge.Difficulty)
 
-	if _, err := r.Cookie(anubis.TestCookieName); err == http.ErrNoCookie {
-		s.ClearCookie(w, s.cookieName, cookiePath)
-		s.ClearCookie(w, anubis.TestCookieName, cookiePath)
-		lg.Warn("user has cookies disabled, this is not an anubis bug")
-		s.respondWithError(w, r, "Your browser is configured to disable cookies. Anubis requires cookies for the legitimate interest of making sure you are a valid client. Please enable cookies for this domain")
-		return
-	}
-
 	nonce, err := strconv.Atoi(nonceStr)
 	if err != nil {
-		s.ClearCookie(w, s.cookieName, cookiePath)
+		s.ClearCookie(w)
 		lg.Debug("nonce doesn't parse", "err", err)
-		s.respondWithError(w, r, "invalid response")
+		s.respondWithError(w, r, "invalid nonce")
 		return
 	}
 
@@ -352,22 +328,33 @@ func (s *Server) PassChallenge(w http.ResponseWriter, r *http.Request) {
 	calculated := internal.SHA256sum(calcString)
 
 	if subtle.ConstantTimeCompare([]byte(response), []byte(calculated)) != 1 {
-		s.ClearCookie(w, s.cookieName, cookiePath)
+		s.ClearCookie(w)
 		lg.Debug("hash does not match", "got", response, "want", calculated)
 		s.respondWithStatus(w, r, "invalid response", http.StatusForbidden)
+		if s.store != nil {
+			s.store.Increment(r.Context(), []string{"pass_rate", "User-Agent", r.UserAgent(), "fail"})
+		}
 		failedValidations.Inc()
 		return
 	}
 
 	// compare the leading zeroes
 	if !strings.HasPrefix(response, strings.Repeat("0", rule.Challenge.Difficulty)) {
-		s.ClearCookie(w, s.cookieName, cookiePath)
+		s.ClearCookie(w)
 		lg.Debug("difficulty check failed", "response", response, "difficulty", rule.Challenge.Difficulty)
 		s.respondWithStatus(w, r, "invalid response", http.StatusForbidden)
+		if s.store != nil {
+			s.store.Increment(r.Context(), []string{"pass_rate", "User-Agent", r.UserAgent(), "fail"})
+		}
 		failedValidations.Inc()
 		return
 	}
 
+	// Adjust cookie path if base prefix is not empty
+	cookiePath := "/"
+	if anubis.BasePrefix != "" {
+		cookiePath = strings.TrimSuffix(anubis.BasePrefix, "/") + "/"
+	}
 	// generate JWT cookie
 	token := jwt.NewWithClaims(jwt.SigningMethodEdDSA, jwt.MapClaims{
 		"challenge": challenge,
@@ -380,12 +367,24 @@ func (s *Server) PassChallenge(w http.ResponseWriter, r *http.Request) {
 	tokenString, err := token.SignedString(s.priv)
 	if err != nil {
 		lg.Error("failed to sign JWT", "err", err)
-		s.ClearCookie(w, s.cookieName, cookiePath)
+		s.ClearCookie(w)
 		s.respondWithError(w, r, "failed to sign JWT")
 		return
 	}
 
-	s.SetCookie(w, s.cookieName, tokenString, cookiePath)
+	http.SetCookie(w, &http.Cookie{
+		Name:        s.cookieName,
+		Value:       tokenString,
+		Expires:     time.Now().Add(s.opts.CookieExpiration),
+		SameSite:    http.SameSiteLaxMode,
+		Domain:      s.opts.CookieDomain,
+		Partitioned: s.opts.CookiePartitioned,
+		Path:        cookiePath,
+	})
+
+	if s.store != nil {
+		s.store.Increment(r.Context(), []string{"pass_rate", "User-Agent", r.UserAgent(), "pass"})
+	}
 
 	challengesValidated.Inc()
 	lg.Debug("challenge passed, redirecting to app")
@@ -416,6 +415,8 @@ func (s *Server) check(r *http.Request) (policy.CheckResult, *policy.Bot, error)
 		return decaymap.Zilch[policy.CheckResult](), nil, fmt.Errorf("[misconfiguration] %q is not an IP address", host)
 	}
 
+	weight := 0
+
 	for _, b := range s.policy.Bots {
 		match, err := b.Rules.Check(r)
 		if err != nil {
@@ -423,10 +424,27 @@ func (s *Server) check(r *http.Request) (policy.CheckResult, *policy.Bot, error)
 		}
 
 		if match {
-			return cr("bot/"+b.Name, b.Action), &b, nil
+			switch b.Action {
+			case config.RuleDeny, config.RuleAllow, config.RuleBenchmark:
+				return cr("bot/"+b.Name, b.Action), &b, nil
+			case config.RuleChallenge:
+				weight += 5
+			case config.RuleWeigh:
+				weight += b.Weight.Adjust
+			}
 		}
 	}
 
+	if weight < 0 {
+		return cr("weight/okay", config.RuleAllow), &policy.Bot{
+			Challenge: &config.ChallengeRules{
+				Difficulty: s.policy.DefaultDifficulty,
+				ReportAs:   s.policy.DefaultDifficulty,
+				Algorithm:  config.AlgorithmFast,
+			},
+		}, nil
+	}
+
 	return cr("default/allow", config.RuleAllow), &policy.Bot{
 		Challenge: &config.ChallengeRules{
 			Difficulty: s.policy.DefaultDifficulty,

lib/anubis_test.go

@@ -4,7 +4,6 @@ import (
 	"encoding/json"
 	"fmt"
 	"net/http"
-	"net/http/cookiejar"
 	"net/http/httptest"
 	"os"
 	"strings"
@@ -44,10 +43,10 @@ type challenge struct {
 	Challenge string `json:"challenge"`
 }
 
-func makeChallenge(t *testing.T, ts *httptest.Server, cli *http.Client) challenge {
+func makeChallenge(t *testing.T, ts *httptest.Server) challenge {
 	t.Helper()
 
-	resp, err := cli.Post(ts.URL+"/.within.website/x/cmd/anubis/api/make-challenge", "", nil)
+	resp, err := ts.Client().Post(ts.URL+"/.within.website/x/cmd/anubis/api/make-challenge", "", nil)
 	if err != nil {
 		t.Fatalf("can't request challenge: %v", err)
 	}
@@ -61,54 +60,6 @@ func makeChallenge(t *testing.T, ts *httptest.Server, cli *http.Client) challeng
 	return chall
 }
 
-func handleChallengeZeroDifficulty(t *testing.T, ts *httptest.Server, cli *http.Client, chall challenge) *http.Response {
-	t.Helper()
-
-	nonce := 0
-	elapsedTime := 420
-	redir := "/"
-	calculated := ""
-	calcString := fmt.Sprintf("%s%d", chall.Challenge, nonce)
-	calculated = internal.SHA256sum(calcString)
-
-	req, err := http.NewRequest(http.MethodGet, ts.URL+"/.within.website/x/cmd/anubis/api/pass-challenge", nil)
-	if err != nil {
-		t.Fatalf("can't make request: %v", err)
-	}
-
-	q := req.URL.Query()
-	q.Set("response", calculated)
-	q.Set("nonce", fmt.Sprint(nonce))
-	q.Set("redir", redir)
-	q.Set("elapsedTime", fmt.Sprint(elapsedTime))
-	req.URL.RawQuery = q.Encode()
-
-	resp, err := cli.Do(req)
-	if err != nil {
-		t.Fatalf("can't do request: %v", err)
-	}
-
-	return resp
-}
-
-func httpClient(t *testing.T) *http.Client {
-	t.Helper()
-
-	jar, err := cookiejar.New(nil)
-	if err != nil {
-		t.Fatal(err)
-	}
-
-	cli := &http.Client{
-		Jar: jar,
-		CheckRedirect: func(req *http.Request, via []*http.Request) error {
-			return http.ErrUseLastResponse
-		},
-	}
-
-	return cli
-}
-
 func TestLoadPolicies(t *testing.T) {
 	for _, fname := range []string{"botPolicies.json", "botPolicies.yaml"} {
 		t.Run(fname, func(t *testing.T) {
@@ -134,6 +85,7 @@ func TestCVE2025_24369(t *testing.T) {
 		Next:   http.NewServeMux(),
 		Policy: pol,
 
+		CookieDomain:      ".local.cetacean.club",
 		CookiePartitioned: true,
 		CookieName:        t.Name(),
 	})
@@ -141,9 +93,34 @@ func TestCVE2025_24369(t *testing.T) {
 	ts := httptest.NewServer(internal.RemoteXRealIP(true, "tcp", srv))
 	defer ts.Close()
 
-	cli := httpClient(t)
-	chall := makeChallenge(t, ts, cli)
-	resp := handleChallengeZeroDifficulty(t, ts, cli, chall)
+	chall := makeChallenge(t, ts)
+	calcString := fmt.Sprintf("%s%d", chall.Challenge, 0)
+	calculated := internal.SHA256sum(calcString)
+	nonce := 0
+	elapsedTime := 420
+	redir := "/"
+
+	cli := ts.Client()
+	cli.CheckRedirect = func(req *http.Request, via []*http.Request) error {
+		return http.ErrUseLastResponse
+	}
+
+	req, err := http.NewRequest(http.MethodGet, ts.URL+"/.within.website/x/cmd/anubis/api/pass-challenge", nil)
+	if err != nil {
+		t.Fatalf("can't make request: %v", err)
+	}
+
+	q := req.URL.Query()
+	q.Set("response", calculated)
+	q.Set("nonce", fmt.Sprint(nonce))
+	q.Set("redir", redir)
+	q.Set("elapsedTime", fmt.Sprint(elapsedTime))
+	req.URL.RawQuery = q.Encode()
+
+	resp, err := cli.Do(req)
+	if err != nil {
+		t.Fatalf("can't do challenge passing")
+	}
 
 	if resp.StatusCode == http.StatusFound {
 		t.Log("Regression on CVE-2025-24369")
@@ -160,18 +137,58 @@ func TestCookieCustomExpiration(t *testing.T) {
 		Next:   http.NewServeMux(),
 		Policy: pol,
 
+		CookieDomain:     "local.cetacean.club",
+		CookieName:       t.Name(),
 		CookieExpiration: ckieExpiration,
 	})
 
 	ts := httptest.NewServer(internal.RemoteXRealIP(true, "tcp", srv))
 	defer ts.Close()
 
-	cli := httpClient(t)
-	chall := makeChallenge(t, ts, cli)
+	cli := &http.Client{
+		CheckRedirect: func(req *http.Request, via []*http.Request) error {
+			return http.ErrUseLastResponse
+		},
+	}
 
-	requestReceiveLowerBound := time.Now().Add(-1 * time.Minute)
-	resp := handleChallengeZeroDifficulty(t, ts, cli, chall)
+	resp, err := cli.Post(ts.URL+"/.within.website/x/cmd/anubis/api/make-challenge", "", nil)
+	if err != nil {
+		t.Fatalf("can't request challenge: %v", err)
+	}
+	defer resp.Body.Close()
+
+	var chall = struct {
+		Challenge string `json:"challenge"`
+	}{}
+	if err := json.NewDecoder(resp.Body).Decode(&chall); err != nil {
+		t.Fatalf("can't read challenge response body: %v", err)
+	}
+
+	nonce := 0
+	elapsedTime := 420
+	redir := "/"
+	calculated := ""
+	calcString := fmt.Sprintf("%s%d", chall.Challenge, nonce)
+	calculated = internal.SHA256sum(calcString)
+
+	req, err := http.NewRequest(http.MethodGet, ts.URL+"/.within.website/x/cmd/anubis/api/pass-challenge", nil)
+	if err != nil {
+		t.Fatalf("can't make request: %v", err)
+	}
+
+	q := req.URL.Query()
+	q.Set("response", calculated)
+	q.Set("nonce", fmt.Sprint(nonce))
+	q.Set("redir", redir)
+	q.Set("elapsedTime", fmt.Sprint(elapsedTime))
+	req.URL.RawQuery = q.Encode()
+
+	requestReceiveLowerBound := time.Now()
+	resp, err = cli.Do(req)
 	requestReceiveUpperBound := time.Now()
+	if err != nil {
+		t.Fatalf("can't do challenge passing")
+	}
 
 	if resp.StatusCode != http.StatusFound {
 		resp.Write(os.Stderr)
@@ -209,21 +226,59 @@ func TestCookieSettings(t *testing.T) {
 		Next:   http.NewServeMux(),
 		Policy: pol,
 
-		CookieDomain:      "127.0.0.1",
+		CookieDomain:      "local.cetacean.club",
 		CookiePartitioned: true,
 		CookieName:        t.Name(),
 		CookieExpiration:  anubis.CookieDefaultExpirationTime,
 	})
 
-	requestReceiveLowerBound := time.Now()
 	ts := httptest.NewServer(internal.RemoteXRealIP(true, "tcp", srv))
 	defer ts.Close()
 
-	cli := httpClient(t)
-	chall := makeChallenge(t, ts, cli)
+	cli := &http.Client{
+		CheckRedirect: func(req *http.Request, via []*http.Request) error {
+			return http.ErrUseLastResponse
+		},
+	}
 
-	resp := handleChallengeZeroDifficulty(t, ts, cli, chall)
+	resp, err := cli.Post(ts.URL+"/.within.website/x/cmd/anubis/api/make-challenge", "", nil)
+	if err != nil {
+		t.Fatalf("can't request challenge: %v", err)
+	}
+	defer resp.Body.Close()
+
+	var chall = struct {
+		Challenge string `json:"challenge"`
+	}{}
+	if err := json.NewDecoder(resp.Body).Decode(&chall); err != nil {
+		t.Fatalf("can't read challenge response body: %v", err)
+	}
+
+	nonce := 0
+	elapsedTime := 420
+	redir := "/"
+	calculated := ""
+	calcString := fmt.Sprintf("%s%d", chall.Challenge, nonce)
+	calculated = internal.SHA256sum(calcString)
+
+	req, err := http.NewRequest(http.MethodGet, ts.URL+"/.within.website/x/cmd/anubis/api/pass-challenge", nil)
+	if err != nil {
+		t.Fatalf("can't make request: %v", err)
+	}
+
+	q := req.URL.Query()
+	q.Set("response", calculated)
+	q.Set("nonce", fmt.Sprint(nonce))
+	q.Set("redir", redir)
+	q.Set("elapsedTime", fmt.Sprint(elapsedTime))
+	req.URL.RawQuery = q.Encode()
+
+	requestReceiveLowerBound := time.Now()
+	resp, err = cli.Do(req)
 	requestReceiveUpperBound := time.Now()
+	if err != nil {
+		t.Fatalf("can't do challenge passing")
+	}
 
 	if resp.StatusCode != http.StatusFound {
 		resp.Write(os.Stderr)
@@ -243,8 +298,8 @@ func TestCookieSettings(t *testing.T) {
 		return
 	}
 
-	if ckie.Domain != "127.0.0.1" {
-		t.Errorf("cookie domain is wrong, wanted 127.0.0.1, got: %s", ckie.Domain)
+	if ckie.Domain != "local.cetacean.club" {
+		t.Errorf("cookie domain is wrong, wanted local.cetacean.club, got: %s", ckie.Domain)
 	}
 
 	expirationLowerBound := requestReceiveLowerBound.Add(anubis.CookieDefaultExpirationTime)
@@ -402,10 +457,6 @@ func TestBasePrefix(t *testing.T) {
 				t.Fatalf("can't make request: %v", err)
 			}
 
-			for _, ckie := range resp.Cookies() {
-				req.AddCookie(ckie)
-			}
-
 			q := req.URL.Query()
 			q.Set("response", calculated)
 			q.Set("nonce", fmt.Sprint(nonce))
@@ -510,25 +561,6 @@ func TestCloudflareWorkersRule(t *testing.T) {
 				t.Fatalf("can't construct libanubis.Server: %v", err)
 			}
 
-			t.Run("with-cf-worker-header", func(t *testing.T) {
-				req, err := http.NewRequest(http.MethodGet, "/", nil)
-				if err != nil {
-					t.Fatal(err)
-				}
-
-				req.Header.Add("X-Real-Ip", "127.0.0.1")
-				req.Header.Add("Cf-Worker", "true")
-
-				cr, _, err := s.check(req)
-				if err != nil {
-					t.Fatal(err)
-				}
-
-				if cr.Rule != config.RuleDeny {
-					t.Errorf("rule is wrong, wanted %s, got: %s", config.RuleDeny, cr.Rule)
-				}
-			})
-
 			t.Run("no-cf-worker-header", func(t *testing.T) {
 				req, err := http.NewRequest(http.MethodGet, "/", nil)
 				if err != nil {

lib/http.go

@@ -1,41 +1,24 @@
 package lib
 
 import (
-	"math/rand"
 	"net/http"
 	"slices"
-	"strings"
 	"time"
 
-	"github.com/TecharoHQ/anubis"
 	"github.com/TecharoHQ/anubis/internal"
 	"github.com/TecharoHQ/anubis/lib/policy"
 	"github.com/TecharoHQ/anubis/web"
 	"github.com/a-h/templ"
 )
 
-func (s *Server) SetCookie(w http.ResponseWriter, name, value, path string) {
+func (s *Server) ClearCookie(w http.ResponseWriter) {
 	http.SetCookie(w, &http.Cookie{
-		Name:        name,
-		Value:       value,
-		Expires:     time.Now().Add(s.opts.CookieExpiration),
-		SameSite:    http.SameSiteLaxMode,
-		Domain:      s.opts.CookieDomain,
-		Partitioned: s.opts.CookiePartitioned,
-		Path:        path,
-	})
-}
-
-func (s *Server) ClearCookie(w http.ResponseWriter, name, path string) {
-	http.SetCookie(w, &http.Cookie{
-		Name:        name,
-		Value:       "",
-		MaxAge:      -1,
-		Expires:     time.Now().Add(-1 * time.Minute),
-		SameSite:    http.SameSiteLaxMode,
-		Partitioned: s.opts.CookiePartitioned,
-		Domain:      s.opts.CookieDomain,
-		Path:        path,
+		Name:     s.cookieName,
+		Value:    "",
+		Expires:  time.Now().Add(-1 * time.Hour),
+		MaxAge:   -1,
+		SameSite: http.SameSiteLaxMode,
+		Domain:   s.opts.CookieDomain,
 	})
 }
 
@@ -55,10 +38,6 @@ func (t UnixRoundTripper) RoundTrip(req *http.Request) (*http.Response, error) {
 	return t.Transport.RoundTrip(req)
 }
 
-func randomChance(n int) bool {
-	return rand.Intn(n) == 0
-}
-
 func (s *Server) RenderIndex(w http.ResponseWriter, r *http.Request, rule *policy.Bot, returnHTTPStatusOnly bool) {
 	if returnHTTPStatusOnly {
 		w.WriteHeader(http.StatusUnauthorized)
@@ -68,12 +47,6 @@ func (s *Server) RenderIndex(w http.ResponseWriter, r *http.Request, rule *polic
 
 	lg := internal.GetRequestLogger(r)
 
-	if !strings.Contains(r.Header.Get("Accept-Encoding"), "gzip") && randomChance(64) {
-		lg.Error("client was given a challenge but does not in fact support gzip compression")
-		s.respondWithError(w, r, "Client Error: Please ensure your browser is up to date and try again later.")
-	}
-
-	challengesIssued.WithLabelValues("embedded").Add(1)
 	challenge := s.challengeFor(r, rule.Challenge.Difficulty)
 
 	var ogTags map[string]string = nil
@@ -85,13 +58,6 @@ func (s *Server) RenderIndex(w http.ResponseWriter, r *http.Request, rule *polic
 		}
 	}
 
-	http.SetCookie(w, &http.Cookie{
-		Name:    anubis.TestCookieName,
-		Value:   challenge,
-		Expires: time.Now().Add(30 * time.Minute),
-		Path:    "/",
-	})
-
 	component, err := web.BaseWithChallengeAndOGTags("Making sure you're not a bot!", web.Index(), challenge, rule.Challenge, ogTags)
 	if err != nil {
 		lg.Error("render failed, please open an issue", "err", err) // This is likely a bug in the template. Should never be triggered as CI tests for this.
@@ -99,10 +65,14 @@ func (s *Server) RenderIndex(w http.ResponseWriter, r *http.Request, rule *polic
 		return
 	}
 
-	handler := internal.GzipMiddleware(1, internal.NoStoreCache(templ.Handler(
+	if s.store != nil {
+		s.store.Increment(r.Context(), []string{"pass_rate", "User-Agent", r.UserAgent(), "challenges_issued"})
+	}
+
+	handler := internal.NoStoreCache(templ.Handler(
 		component,
 		templ.WithStatus(s.opts.Policy.StatusCodes.Challenge),
-	)))
+	))
 	handler.ServeHTTP(w, r)
 }
 

lib/http_test.go

@@ -11,7 +11,7 @@ func TestClearCookie(t *testing.T) {
 	srv := spawnAnubis(t, Options{})
 	rw := httptest.NewRecorder()
 
-	srv.ClearCookie(rw, srv.cookieName, "/")
+	srv.ClearCookie(rw)
 
 	resp := rw.Result()
 
@@ -36,7 +36,7 @@ func TestClearCookieWithDomain(t *testing.T) {
 	srv := spawnAnubis(t, Options{CookieDomain: "techaro.lol"})
 	rw := httptest.NewRecorder()
 
-	srv.ClearCookie(rw, srv.cookieName, "/")
+	srv.ClearCookie(rw)
 
 	resp := rw.Result()
 

lib/policy/bot.go

@@ -12,6 +12,7 @@ type Bot struct {
 	Challenge *config.ChallengeRules
 	Name      string
 	Action    config.Rule
+	Weight    *config.Weight
 }
 
 func (b Bot) Hash() string {

lib/policy/checkresult.go

@@ -7,12 +7,15 @@ import (
 )
 
 type CheckResult struct {
-	Name string
-	Rule config.Rule
+	Name   string
+	Rule   config.Rule
+	Weight int
 }
 
 func (cr CheckResult) LogValue() slog.Value {
 	return slog.GroupValue(
 		slog.String("name", cr.Name),
-		slog.String("rule", string(cr.Rule)))
+		slog.String("rule", string(cr.Rule)),
+		slog.Int("weight", cr.Weight),
+	)
 }

lib/policy/config/config.go

@@ -39,6 +39,7 @@ const (
 	RuleAllow     Rule = "ALLOW"
 	RuleDeny      Rule = "DENY"
 	RuleChallenge Rule = "CHALLENGE"
+	RuleWeigh     Rule = "WEIGH"
 	RuleBenchmark Rule = "DEBUG_BENCHMARK"
 )
 
@@ -51,14 +52,15 @@ const (
 )
 
 type BotConfig struct {
-	UserAgentRegex *string           `json:"user_agent_regex"`
-	PathRegex      *string           `json:"path_regex"`
-	HeadersRegex   map[string]string `json:"headers_regex"`
-	Expression     *ExpressionOrList `json:"expression"`
+	UserAgentRegex *string           `json:"user_agent_regex,omitempty"`
+	PathRegex      *string           `json:"path_regex,omitempty"`
+	HeadersRegex   map[string]string `json:"headers_regex,omitempty"`
+	Expression     *ExpressionOrList `json:"expression,omitempty"`
 	Challenge      *ChallengeRules   `json:"challenge,omitempty"`
+	Weight         *Weight           `json:"weight,omitempty"`
 	Name           string            `json:"name"`
 	Action         Rule              `json:"action"`
-	RemoteAddr     []string          `json:"remote_addresses"`
+	RemoteAddr     []string          `json:"remote_addresses,omitempty"`
 }
 
 func (b BotConfig) Zero() bool {
@@ -150,7 +152,7 @@ func (b BotConfig) Valid() error {
 	}
 
 	switch b.Action {
-	case RuleAllow, RuleBenchmark, RuleChallenge, RuleDeny:
+	case RuleAllow, RuleBenchmark, RuleChallenge, RuleDeny, RuleWeigh:
 		// okay
 	default:
 		errs = append(errs, fmt.Errorf("%w: %q", ErrUnknownAction, b.Action))
@@ -162,6 +164,10 @@ func (b BotConfig) Valid() error {
 		}
 	}
 
+	if b.Action == RuleWeigh && b.Weight == nil {
+		b.Weight = &Weight{Adjust: 5}
+	}
+
 	if len(errs) != 0 {
 		return fmt.Errorf("config: bot entry for %q is not valid:\n%w", b.Name, errors.Join(errs...))
 	}
@@ -224,7 +230,7 @@ func (is *ImportStatement) open() (fs.File, error) {
 func (is *ImportStatement) load() error {
 	fin, err := is.open()
 	if err != nil {
-		return fmt.Errorf("%w: %s: %w", ErrInvalidImportStatement, is.Import, err)
+		return fmt.Errorf("can't open %s: %w", is.Import, err)
 	}
 	defer fin.Close()
 

lib/policy/config/config_test.go

@@ -182,6 +182,25 @@ func TestBotValid(t *testing.T) {
 			},
 			err: nil,
 		},
+		{
+			name: "weight rule without weight",
+			bot: BotConfig{
+				Name:           "weight-adjust-if-mozilla",
+				Action:         RuleWeigh,
+				UserAgentRegex: p("Mozilla"),
+			},
+		},
+		{
+			name: "weight rule with weight adjust",
+			bot: BotConfig{
+				Name:           "weight-adjust-if-mozilla",
+				Action:         RuleWeigh,
+				UserAgentRegex: p("Mozilla"),
+				Weight: &Weight{
+					Adjust: 5,
+				},
+			},
+		},
 	}
 
 	for _, cs := range tests {

lib/policy/config/expressionorlist.go

@@ -14,8 +14,8 @@ var (
 
 type ExpressionOrList struct {
 	Expression string   `json:"-"`
-	All        []string `json:"all"`
-	Any        []string `json:"any"`
+	All        []string `json:"all,omitempty"`
+	Any        []string `json:"any,omitempty"`
 }
 
 func (eol ExpressionOrList) Equal(rhs *ExpressionOrList) bool {
@@ -54,9 +54,6 @@ func (eol *ExpressionOrList) UnmarshalJSON(data []byte) error {
 }
 
 func (eol *ExpressionOrList) Valid() error {
-	if eol.Expression == "" && len(eol.All) == 0 && len(eol.Any) == 0 {
-		return ErrExpressionEmpty
-	}
 	if len(eol.All) != 0 && len(eol.Any) != 0 {
 		return ErrExpressionCantHaveBoth
 	}

lib/policy/config/expressionorlist_test.go

@@ -51,13 +51,6 @@ func TestExpressionOrListUnmarshal(t *testing.T) {
 			}`,
 			validErr: ErrExpressionCantHaveBoth,
 		},
-		{
-			name: "expression-empty",
-			inp: `{
-			"any": []
-			}`,
-			validErr: ErrExpressionEmpty,
-		},
 	} {
 		t.Run(tt.name, func(t *testing.T) {
 			var eol ExpressionOrList

lib/policy/config/testdata/good/simple-weight.yaml

@@ -0,0 +1,6 @@
+bots:
+- name: simple-weight-adjust
+  action: WEIGH
+  user_agent_regex: Mozilla
+  weight:
+    adjust: 5

lib/policy/config/testdata/good/weight-no-weight.yaml

@@ -0,0 +1,4 @@
+bots:
+- name: weight
+  action: WEIGH
+  user_agent_regex: Mozilla

lib/policy/config/weight.go

@@ -0,0 +1,5 @@
+package config
+
+type Weight struct {
+	Adjust int `json:"adjust"`
+}

lib/policy/passratechecker.go

@@ -0,0 +1,47 @@
+package policy
+
+import (
+	"fmt"
+	"net/http"
+
+	"github.com/TecharoHQ/anubis/internal"
+	"github.com/TecharoHQ/anubis/internal/store/valkey"
+)
+
+type PassRateChecker struct {
+	store  *valkey.Store
+	header string
+	rate   float64
+}
+
+func NewPassRateChecker(store *valkey.Store, rate float64) Checker {
+	return &PassRateChecker{
+		store:  store,
+		rate:   rate,
+		header: "User-Agent",
+	}
+}
+
+func (prc *PassRateChecker) Hash() string {
+	return internal.SHA256sum(fmt.Sprintf("pass rate checker::%s", prc.header))
+}
+
+func (prc *PassRateChecker) Check(r *http.Request) (bool, error) {
+	data, err := prc.store.MultiGetInt(r.Context(), [][]string{
+		{"pass_rate", prc.header, r.Header.Get(prc.header), "pass"},
+		{"pass_rate", prc.header, r.Header.Get(prc.header), "challenges_issued"},
+		{"pass_rate", prc.header, r.Header.Get(prc.header), "fail"},
+	})
+	if err != nil {
+		return false, err
+	}
+
+	passCount, challengeCount, failCount := data[0], data[1], data[2]
+	passRate := float64(passCount-failCount) / float64(challengeCount)
+
+	if passRate >= prc.rate {
+		return true, nil
+	}
+
+	return false, nil
+}

lib/policy/policy.go

@@ -116,6 +116,10 @@ func ParseConfig(fin io.Reader, fname string, defaultDifficulty int) (*ParsedCon
 			}
 		}
 
+		if b.Weight != nil {
+			parsedBot.Weight = b.Weight
+		}
+
 		parsedBot.Rules = cl
 
 		result.Bots = append(result.Bots, parsedBot)

lib/testdata/cloudflare-workers-cel.yaml

@@ -1,8 +1,4 @@
 bots:
 - name: cloudflare-workers
   expression: '"Cf-Worker" in headers'
-  action: DENY
-
-status_codes:
-  CHALLENGE: 401
-  DENY: 403
+  action: DENY

lib/testdata/cloudflare-workers-header.yaml

@@ -2,8 +2,4 @@ bots:
 - name: cloudflare-workers
   headers_regex:
     CF-Worker: .*
-  action: DENY
-
-status_codes:
-  CHALLENGE: 401
-  DENY: 403
+  action: DENY

package-lock.json

@@ -1,12 +1,12 @@
 {
   "name": "@techaro/anubis",
-  "version": "1.19.0-pre1",
+  "version": "1.18.0",
   "lockfileVersion": 3,
   "requires": true,
   "packages": {
     "": {
       "name": "@techaro/anubis",
-      "version": "1.19.0-pre1",
+      "version": "1.18.0",
       "license": "ISC",
       "devDependencies": {
         "cssnano": "^7.0.7",

package.json

@@ -1,6 +1,6 @@
 {
   "name": "@techaro/anubis",
-  "version": "1.19.0-pre1",
+  "version": "1.18.0",
   "description": "",
   "main": "index.js",
   "scripts": {
@@ -10,7 +10,7 @@
     "test:integration:docker": "npm run assets && go test -v ./internal/test --playwright-runner=docker",
     "assets": "go generate ./... && ./web/build.sh && ./xess/build.sh",
     "build": "npm run assets && go build -o ./var/anubis ./cmd/anubis",
-    "dev": "npm run assets && go run ./cmd/anubis --use-remote-address --target http://localhost:3000",
+    "dev": "npm run assets && go run ./cmd/anubis --use-remote-address",
     "container": "npm run assets && go run ./cmd/containerbuild",
     "package": "yeet",
     "lint": "make lint"
@@ -27,4 +27,4 @@
     "postcss-import-url": "^7.2.0",
     "postcss-url": "^10.1.3"
   }
-}
+}

run/anubis@.service

@@ -12,8 +12,6 @@ CacheDirectory=anubis/%i
 CacheDirectoryMode=0755
 StateDirectory=anubis/%i
 StateDirectoryMode=0755
-RuntimeDirectory=anubis
-RuntimeDirectoryMode=0755
 ReadWritePaths=/run
 
 [Install]

web/static/robots.txt

@@ -9,8 +9,6 @@ User-agent: Brightbot 1.0
 User-agent: Bytespider
 User-agent: CCBot
 User-agent: ChatGPT-User
-User-agent: Claude-SearchBot
-User-agent: Claude-User
 User-agent: Claude-Web
 User-agent: ClaudeBot
 User-agent: cohere-ai
@@ -23,7 +21,6 @@ User-agent: FacebookBot
 User-agent: Factset_spyderbot
 User-agent: FirecrawlAgent
 User-agent: FriendlyCrawler
-User-agent: Google-CloudVertexBot
 User-agent: Google-Extended
 User-agent: GoogleOther
 User-agent: GoogleOther-Image
@@ -40,7 +37,6 @@ User-agent: meta-externalagent
 User-agent: Meta-ExternalAgent
 User-agent: meta-externalfetcher
 User-agent: Meta-ExternalFetcher
-User-agent: MistralAI-User/1.0
 User-agent: NovaAct
 User-agent: OAI-SearchBot
 User-agent: omgili
@@ -50,7 +46,6 @@ User-agent: PanguBot
 User-agent: Perplexity-User
 User-agent: PerplexityBot
 User-agent: PetalBot
-User-agent: QualifiedBot
 User-agent: Scrapy
 User-agent: SemrushBot-OCOB
 User-agent: SemrushBot-SWA
@@ -59,7 +54,6 @@ User-agent: TikTokSpider
 User-agent: Timpibot
 User-agent: VelenPublicWebCrawler
 User-agent: Webzio-Extended
-User-agent: wpbot
 User-agent: YouBot
 Disallow: /