Version 1.17.1: Asahi sas Brutus: Echo 1

- Added customization of authorization cookie expiration time with `--cookie-expiration-time` flag or envvar
- Updated the `OG_PASSTHROUGH` to be true by default, thereby allowing OpenGraph tags to be passed through by default
- Added the ability to [customize Anubis' HTTP status codes](./admin/configuration/custom-status-codes.mdx) ([#355](https://github.com/TecharoHQ/anubis/issues/355))

Signed-off-by: Xe Iaso <me@xeiaso.net>

.github/workflows/docs-deploy.yml

@@ -38,7 +38,7 @@ jobs:
 
       - name: Build and push
         id: build
-        uses: docker/build-push-action@471d1dc4e07e5cdedd4c2171150001c434f0b7a4 # v6.15.0
+        uses: docker/build-push-action@14487ce63c7a62a4a324b0bfb37086795e31c6c1 # v6.16.0
         with:
           context: ./docs
           cache-to: type=gha
@@ -49,14 +49,14 @@ jobs:
           push: true
 
       - name: Apply k8s manifests to aeacus
-        uses: actions-hub/kubectl@9270913c29699788b51bc04becd0ebdf048ffb49 # v1.32.3
+        uses: actions-hub/kubectl@e81783053d902f50d752d21a6d99cf9689a652e1 # v1.33.0
         env:
           KUBE_CONFIG: ${{ secrets.AEACUS_KUBECONFIG }}
         with:
           args: apply -k docs/manifest
 
       - name: Apply k8s manifests to aeacus
-        uses: actions-hub/kubectl@9270913c29699788b51bc04becd0ebdf048ffb49 # v1.32.3
+        uses: actions-hub/kubectl@e81783053d902f50d752d21a6d99cf9689a652e1 # v1.33.0
         env:
           KUBE_CONFIG: ${{ secrets.AEACUS_KUBECONFIG }}
         with:

.github/workflows/docs-test.yml

@@ -28,7 +28,7 @@ jobs:
 
       - name: Build and push
         id: build
-        uses: docker/build-push-action@471d1dc4e07e5cdedd4c2171150001c434f0b7a4 # v6.15.0
+        uses: docker/build-push-action@14487ce63c7a62a4a324b0bfb37086795e31c6c1 # v6.16.0
         with:
           context: ./docs
           cache-to: type=gha

.github/workflows/package-builds-stable.yml

@@ -64,7 +64,7 @@ jobs:
 
     - name: Build Packages
       run: |
-        wget https://github.com/TecharoHQ/yeet/releases/download/v0.1.1/yeet_0.1.1_amd64.deb -O var/yeet.deb
+        wget https://github.com/TecharoHQ/yeet/releases/download/v0.2.1/yeet_0.2.1_amd64.deb -O var/yeet.deb
         sudo apt -y install -f ./var/yeet.deb
         rm ./var/yeet.deb
         yeet

.github/workflows/package-builds-unstable.yml

@@ -66,7 +66,7 @@ jobs:
 
     - name: Build Packages
       run: |
-        wget https://github.com/TecharoHQ/yeet/releases/download/v0.1.1/yeet_0.1.1_amd64.deb -O var/yeet.deb
+        wget https://github.com/TecharoHQ/yeet/releases/download/v0.2.1/yeet_0.2.1_amd64.deb -O var/yeet.deb
         sudo apt -y install -f ./var/yeet.deb
         rm ./var/yeet.deb
         yeet

.github/workflows/zizmor.yml

@@ -21,7 +21,7 @@ jobs:
           persist-credentials: false
 
       - name: Install the latest version of uv
-        uses: astral-sh/setup-uv@d4b2f3b6ecc6e67c4457f6d3e41ec42d3d0fcb86 # v5.4.2
+        uses: astral-sh/setup-uv@c7f87aa956e4c323abf06d5dec078e358f6b4d04 # v6.0.0
 
       - name: Run zizmor 🌈
         run: uvx zizmor --format sarif . > results.sarif 
@@ -29,7 +29,7 @@ jobs:
           GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} 
 
       - name: Upload SARIF file
-        uses: github/codeql-action/upload-sarif@45775bd8235c68ba998cffa5171334d58593da47 # v3.28.15
+        uses: github/codeql-action/upload-sarif@28deaeda66b76a05916b6923827895f2b14ab387 # v3.28.16
         with:
           sarif_file: results.sarif
           category: zizmor

README.md

@@ -22,7 +22,7 @@ Anubis [weighs the soul of your connection](https://en.wikipedia.org/wiki/Weighi
 
 This program is designed to help protect the small internet from the endless storm of requests that flood in from AI companies. Anubis is as lightweight as possible to ensure that everyone can afford to protect the communities closest to them.
 
-Anubis is a bit of a nuclear response. This will result in your website being blocked from smaller scrapers and may inhibit "good bots" like the Internet Archive. You can configure [bot policy definitions](./admin/policies.mdx) to explicitly allowlist them and we are working on a curated set of "known good" bots to allow for a compromise between discoverability and uptime.
+Anubis is a bit of a nuclear response. This will result in your website being blocked from smaller scrapers and may inhibit "good bots" like the Internet Archive. You can configure [bot policy definitions](./docs/docs/admin/policies.mdx) to explicitly allowlist them and we are working on a curated set of "known good" bots to allow for a compromise between discoverability and uptime.
 
 In most cases, you should not need this and can probably get by using Cloudflare to protect a given origin. However, for circumstances where you can't or won't use Cloudflare, Anubis is there for you.
 

VERSION

@@ -1 +1 @@
-1.16.0
+1.17.1

anubis.go

@@ -1,6 +1,8 @@
 // Package anubis contains the version number of Anubis.
 package anubis
 
+import "time"
+
 // Version is the current version of Anubis.
 //
 // This variable is set at build time using the -X linker flag. If not set,
@@ -11,6 +13,9 @@ var Version = "devel"
 // access.
 const CookieName = "within.website-x-cmd-anubis-auth"
 
+// CookieDefaultExpirationTime is the amount of time before the cookie/JWT expires.
+const CookieDefaultExpirationTime = 7 * 24 * time.Hour
+
 // BasePrefix is a global prefix for all Anubis endpoints. Can be emptied to remove the prefix entirely.
 var BasePrefix = ""
 

cmd/anubis/main.go

@@ -43,6 +43,7 @@ var (
 	bindNetwork              = flag.String("bind-network", "tcp", "network family to bind HTTP to, e.g. unix, tcp")
 	challengeDifficulty      = flag.Int("difficulty", anubis.DefaultDifficulty, "difficulty of the challenge")
 	cookieDomain             = flag.String("cookie-domain", "", "if set, the top-level domain that the Anubis cookie will be valid for")
+	cookieExpiration         = flag.Duration("cookie-expiration-time", anubis.CookieDefaultExpirationTime, "The amount of time the authorization cookie is valid for")
 	cookiePartitioned        = flag.Bool("cookie-partitioned", false, "if true, sets the partitioned flag on Anubis cookies, enabling CHIPS support")
 	ed25519PrivateKeyHex     = flag.String("ed25519-private-key-hex", "", "private key used to sign JWTs, if not set a random one will be assigned")
 	ed25519PrivateKeyHexFile = flag.String("ed25519-private-key-hex-file", "", "file name containing value for ed25519-private-key-hex")
@@ -57,8 +58,9 @@ var (
 	healthcheck              = flag.Bool("healthcheck", false, "run a health check against Anubis")
 	useRemoteAddress         = flag.Bool("use-remote-address", false, "read the client's IP address from the network request, useful for debugging and running Anubis on bare metal")
 	debugBenchmarkJS         = flag.Bool("debug-benchmark-js", false, "respond to every request with a challenge for benchmarking hashrate")
-	ogPassthrough            = flag.Bool("og-passthrough", false, "enable Open Graph tag passthrough")
+	ogPassthrough            = flag.Bool("og-passthrough", true, "enable Open Graph tag passthrough")
 	ogTimeToLive             = flag.Duration("og-expiry-time", 24*time.Hour, "Open Graph tag cache expiration time")
+	ogCacheConsiderHost      = flag.Bool("og-cache-consider-host", false, "enable or disable the use of the host in the Open Graph tag cache")
 	extractResources         = flag.String("extract-resources", "", "if set, extract the static resources to the specified folder")
 	webmasterEmail           = flag.String("webmaster-email", "", "if set, displays webmaster's email on the reject page for appeals")
 )
@@ -272,18 +274,20 @@ func main() {
 	}
 
 	s, err := libanubis.New(libanubis.Options{
-		BasePrefix:        *basePrefix,
-		Next:              rp,
-		Policy:            policy,
-		ServeRobotsTXT:    *robotsTxt,
-		PrivateKey:        priv,
-		CookieDomain:      *cookieDomain,
-		CookiePartitioned: *cookiePartitioned,
-		OGPassthrough:     *ogPassthrough,
-		OGTimeToLive:      *ogTimeToLive,
-		RedirectDomains:   redirectDomainsList,
-		Target:            *target,
-		WebmasterEmail:    *webmasterEmail,
+		BasePrefix:           *basePrefix,
+		Next:                 rp,
+		Policy:               policy,
+		ServeRobotsTXT:       *robotsTxt,
+		PrivateKey:           priv,
+		CookieDomain:         *cookieDomain,
+		CookieExpiration:     *cookieExpiration,
+		CookiePartitioned:    *cookiePartitioned,
+		OGPassthrough:        *ogPassthrough,
+		OGTimeToLive:         *ogTimeToLive,
+		RedirectDomains:      redirectDomainsList,
+		Target:               *target,
+		WebmasterEmail:       *webmasterEmail,
+		OGCacheConsidersHost: *ogCacheConsiderHost,
 	})
 	if err != nil {
 		log.Fatalf("can't construct libanubis.Server: %v", err)
@@ -320,6 +324,7 @@ func main() {
 		"og-passthrough", *ogPassthrough,
 		"og-expiry-time", *ogTimeToLive,
 		"base-prefix", *basePrefix,
+		"cookie-expiration-time", *cookieExpiration,
 	)
 
 	go func() {

data/botPolicies.yaml

@@ -48,3 +48,11 @@ bots:
   action: CHALLENGE
 
 dnsbl: false
+
+# By default, send HTTP 200 back to clients that either get issued a challenge
+# or a denial. This seems weird, but this is load-bearing due to the fact that
+# the most aggressive scraper bots seem to really really want an HTTP 200 and
+# will stop sending requests once they get it.
+status_codes:
+  CHALLENGE: 200
+  DENY: 200 

data/bots/ai-robots-txt.yaml

@@ -1,4 +1,4 @@
 - name: "ai-robots-txt"
   user_agent_regex: >-
-    AI2Bot|Ai2Bot-Dolma|Amazonbot|anthropic-ai|Applebot|Applebot-Extended|Brightbot 1.0|Bytespider|CCBot|ChatGPT-User|Claude-Web|ClaudeBot|cohere-ai|cohere-training-data-crawler|Crawlspace|Diffbot|DuckAssistBot|FacebookBot|FriendlyCrawler|Google-Extended|GoogleOther|GoogleOther-Image|GoogleOther-Video|GPTBot|iaskspider/2.0|ICC-Crawler|ImagesiftBot|img2dataset|ISSCyberRiskCrawler|Kangaroo Bot|Meta-ExternalAgent|Meta-ExternalFetcher|OAI-SearchBot|omgili|omgilibot|PanguBot|Perplexity-User|PerplexityBot|PetalBot|Scrapy|SemrushBot-OCOB|SemrushBot-SWA|Sidetrade indexer bot|Timpibot|VelenPublicWebCrawler|Webzio-Extended|YouBot
-  action: DENY
+    AI2Bot|Ai2Bot-Dolma|aiHitBot|Amazonbot|anthropic-ai|Applebot|Applebot-Extended|Brightbot 1.0|Bytespider|CCBot|ChatGPT-User|Claude-Web|ClaudeBot|cohere-ai|cohere-training-data-crawler|Cotoyogi|Crawlspace|Diffbot|DuckAssistBot|FacebookBot|Factset_spyderbot|FirecrawlAgent|FriendlyCrawler|Google-Extended|GoogleOther|GoogleOther-Image|GoogleOther-Video|GPTBot|iaskspider/2.0|ICC-Crawler|ImagesiftBot|img2dataset|imgproxy|ISSCyberRiskCrawler|Kangaroo Bot|meta-externalagent|Meta-ExternalAgent|meta-externalfetcher|Meta-ExternalFetcher|NovaAct|OAI-SearchBot|omgili|omgilibot|Operator|PanguBot|Perplexity-User|PerplexityBot|PetalBot|Scrapy|SemrushBot-OCOB|SemrushBot-SWA|Sidetrade indexer bot|TikTokSpider|Timpibot|VelenPublicWebCrawler|Webzio-Extended|YouBot
+  action: DENY

docs/docs/CHANGELOG.md

@@ -11,6 +11,14 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 
 ## [Unreleased]
 
+## v1.17.1: Asahi sas Brutus: Echo 1
+
+- Added customization of authorization cookie expiration time with `--cookie-expiration-time` flag or envvar
+- Updated the `OG_PASSTHROUGH` to be true by default, thereby allowing OpenGraph tags to be passed through by default
+- Added the ability to [customize Anubis' HTTP status codes](./admin/configuration/custom-status-codes.mdx) ([#355](https://github.com/TecharoHQ/anubis/issues/355))
+
+## v1.17.0: Asahi sas Brutus
+
 - Ensure regexes can't end in newlines ([#372](https://github.com/TecharoHQ/anubis/issues/372))
 - Add documentation for default allow behavior (implicit rule)
 - Enable [importing configuration snippets](./admin/configuration/import.mdx) ([#321](https://github.com/TecharoHQ/anubis/pull/321))
@@ -39,6 +47,9 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 - Fixed mojeekbot user agent regex
 - Added support for running anubis behind a base path (e.g. `/myapp`)
 - Reduce Anubis' paranoia with user cookies ([#365](https://github.com/TecharoHQ/anubis/pull/365))
+- Added support for Opengraph passthrough while using unix sockets
+- The opengraph subsystem now passes the HTTP `HOST` header through to the origin
+- Updated the `OG_PASSTHROUGH` to be true by default, thereby allowing OpenGraph tags to be passed through by default
 
 ## v1.16.0
 

docs/docs/admin/configuration/custom-status-codes.mdx

@@ -0,0 +1,19 @@
+# Custom status codes for Anubis errors
+
+Out of the box, Anubis will reply with `HTTP 200` for challenge and denial pages. This is intended to make AI scrapers have a hard time with your website because when they are faced with a non-200 response, they will hammer the page over and over until they get a 200 response. This behavior may not be desirable, as such Anubis lets you customize what HTTP status codes are returned when Anubis throws challenge and denial pages.
+
+This is configured in the `status_codes` block of your [bot policy file](../policies.mdx):
+
+```yaml
+status_codes:
+  CHALLENGE: 200
+  DENY: 200
+```
+
+To match CloudFlare's behavior, use a configuration like this:
+
+```yaml
+status_codes:
+  CHALLENGE: 403
+  DENY: 403
+```

docs/docs/admin/configuration/open-graph.mdx

@@ -9,10 +9,11 @@ This page provides detailed information on how to configure [OpenGraph tag](http
 
 ## Configuration Options
 
-| Name             | Description                                               | Type     | Default | Example                 |
-|------------------|-----------------------------------------------------------|----------|---------|-------------------------|
-| `OG_PASSTHROUGH` | Enables or disables the Open Graph tag passthrough system | Boolean  | `false` | `OG_PASSTHROUGH=true`   |
-| `OG_EXPIRY_TIME` | Configurable cache expiration time for Open Graph tags    | Duration | `24h`   | `OG_EXPIRY_TIME=1h`     |
+| Name                     | Description                                               | Type     | Default | Example                       |
+| ------------------------ | --------------------------------------------------------- | -------- | ------- | ----------------------------- |
+| `OG_PASSTHROUGH`         | Enables or disables the Open Graph tag passthrough system | Boolean  | `true`  | `OG_PASSTHROUGH=true`         |
+| `OG_EXPIRY_TIME`         | Configurable cache expiration time for Open Graph tags    | Duration | `24h`   | `OG_EXPIRY_TIME=1h`           |
+| `OG_CACHE_CONSIDER_HOST` | Enables or disables the use of the host in the cache key  | Boolean  | `false` | `OG_CACHE_CONSIDER_HOST=true` |
 
 ## Usage
 
@@ -21,6 +22,7 @@ To configure Open Graph tags, you can set the following environment variables, e
 ```sh
 export OG_PASSTHROUGH=true
 export OG_EXPIRY_TIME=1h
+export OG_CACHE_CONSIDER_HOST=false
 ```
 
 ## Implementation Details
@@ -33,6 +35,8 @@ When `OG_PASSTHROUGH` is enabled, Anubis will:
 
 The cache expiration time is controlled by `OG_EXPIRY_TIME`.
 
+When `OG_CACHE_CONSIDER_HOST` is enabled, Anubis will include the host in the cache key for Open Graph tags. This ensures that tags are cached separately for different hosts.
+
 ## Example
 
 Here is an example of how to configure Open Graph tags in your Anubis setup:
@@ -40,8 +44,19 @@ Here is an example of how to configure Open Graph tags in your Anubis setup:
 ```sh
 export OG_PASSTHROUGH=true
 export OG_EXPIRY_TIME=1h
+export OG_CACHE_CONSIDER_HOST=false
 ```
 
-With these settings, Anubis will cache Open Graph tags for 1 hour and pass them through to the challenge page.
+With these settings, Anubis will cache Open Graph tags for 1 hour and pass them through to the challenge page, not considering the host in the cache key.
+
+## When to Enable `OG_CACHE_CONSIDER_HOST`
+
+In most cases, you would want to keep `OG_CACHE_CONSIDER_HOST` set to `false` to avoid unnecessary cache fragmentation. However, there are some scenarios where enabling this option can be beneficial:
+
+1. **Multi-Tenant Applications**: If you are running a multi-tenant application where different tenants are hosted on different subdomains, enabling `OG_CACHE_CONSIDER_HOST` ensures that the Open Graph tags are cached separately for each tenant. This prevents one tenant's Open Graph tags from being served to another tenant's users.
+
+2. **Different Content for Different Hosts**: If your application serves different content based on the host, enabling `OG_CACHE_CONSIDER_HOST` ensures that the correct Open Graph tags are cached and served for each host. This is useful for applications that have different branding or content for different domains or subdomains.
+
+3. **Security and Privacy Concerns**: In some cases, you may want to ensure that Open Graph tags are not shared between different hosts for security or privacy reasons. Enabling `OG_CACHE_CONSIDER_HOST` ensures that the tags are cached separately for each host, preventing any potential leakage of information between hosts.
 
 For more information, refer to the [installation guide](../installation).

docs/docs/admin/environments/traefik.mdx

@@ -1,6 +1,6 @@
 ---
 id: traefik
-title: Integrate Anubis with Traefik in a Docker Compose Environment
+title: Traefik
 ---
 
 

docs/docs/admin/installation.mdx

@@ -55,6 +55,7 @@ Anubis uses these environment variables for configuration:
 | `BIND`                         | `:8923`                 | The network address that Anubis listens on. For `unix`, set this to a path: `/run/anubis/instance.sock`                                                                                                                                                                                                              |
 | `BIND_NETWORK`                 | `tcp`                   | The address family that Anubis listens on. Accepts `tcp`, `unix` and anything Go's [`net.Listen`](https://pkg.go.dev/net#Listen) supports.                                                                                                                                                                           |
 | `COOKIE_DOMAIN`                | unset                   | The domain the Anubis challenge pass cookie should be set to. This should be set to the domain you bought from your registrar (EG: `techaro.lol` if your webapp is running on `anubis.techaro.lol`). See [here](https://stackoverflow.com/a/1063760) for more information.                                           |
+| `COOKIE_EXPIRATION_TIME`       | `168h`                  | The amount of time the authorization cookie is valid for.                                                                                                                                                                                                                                                            |
 | `COOKIE_PARTITIONED`           | `false`                 | If set to `true`, enables the [partitioned (CHIPS) flag](https://developers.google.com/privacy-sandbox/cookies/chips), meaning that Anubis inside an iframe has a different set of cookies than the domain hosting the iframe.                                                                                       |
 | `DIFFICULTY`                   | `4`                     | The difficulty of the challenge, or the number of leading zeroes that must be in successful responses.                                                                                                                                                                                                               |
 | `ED25519_PRIVATE_KEY_HEX`      | unset                   | The hex-encoded ed25519 private key used to sign Anubis responses. If this is not set, Anubis will generate one for you. This should be exactly 64 characters long. See below for details.                                                                                                                           |
@@ -63,6 +64,7 @@ Anubis uses these environment variables for configuration:
 | `METRICS_BIND_NETWORK`         | `tcp`                   | The address family that the Anubis metrics server listens on. See `BIND_NETWORK` for more information.                                                                                                                                                                                                               |
 | `OG_EXPIRY_TIME`               | `24h`                   | The expiration time for the Open Graph tag cache.                                                                                                                                                                                                                                                                    |
 | `OG_PASSTHROUGH`               | `false`                 | If set to `true`, Anubis will enable Open Graph tag passthrough.                                                                                                                                                                                                                                                     |
+| `OG_CACHE_CONSIDER_HOST`       | `false`                 | If set to `true`, Anubis will consider the host in the Open Graph tag cache key.                                                                                                                                                                                                                                     |
 | `POLICY_FNAME`                 | unset                   | The file containing [bot policy configuration](./policies.mdx). See the bot policy documentation for more details. If unset, the default bot policy configuration is used.                                                                                                                                           |
 | `REDIRECT_DOMAINS`             | unset                   | If set, restrict the domains that Anubis can redirect to when passing a challenge.<br/><br/>If this is unset, Anubis may redirect to any domain which could cause security issues in the unlikely case that an attacker passes a challenge for your browser and then tricks you into clicking a link to your domain. |
 | `SERVE_ROBOTS_TXT`             | `false`                 | If set `true`, Anubis will serve a default `robots.txt` file that disallows all known AI scrapers by name and then additionally disallows every scraper. This is useful if facts and circumstances make it difficult to change the underlying service to serve such a `robots.txt` file.                             |

docs/docs/user/known-broken-extensions.md

@@ -3,17 +3,47 @@ title: List of known browser extensions that can break Anubis
 ---
 
 This page contains a list of all of the browser extensions that are known to break Anubis' functionality and their associated GitHub issues, along with instructions on how to work around the issue.
+
 ## [JShelter](https://jshelter.org/)
 
-| Extension    | JShelter                                      |
-| :----------- | :-------------------------------------------- |
-| Website      | [jshelter.org](https://jshelter.org/)         |
-| GitHub issue | https://github.com/TecharoHQ/anubis/issues/25 |
+| Extension    | JShelter                                                                                                                                           |
+| :----------- | :------------------------------------------------------------------------------------------------------------------------------------------------- |
+| Website      | [jshelter.org](https://jshelter.org/)                                                                                                              |
+| GitHub issue | https://github.com/TecharoHQ/anubis/issues/25                                                                                                      |
+| Be aware of  | [What are Web Workers, and what are the threats that I face?](https://jshelter.org/faq/#what-are-web-workers-and-what-are-the-threats-that-i-face) |
 
-Workaround steps:
+### Workaround steps (recommended):
+
+1. Click on the JShelter badge icon (typically in the toolbar next to your navigation bar; if you cannot locate the icon, see [this question](https://jshelter.org/faq/#can-i-see-a-jshelter-badge-icon-next-to-my-navigation-bar-i-want-to-interact-with-the-extension-easily-and-avoid-going-through-settings)).
+2. Expand JavaScript Shield settings by clicking on the `Modify` button.
+3. Click on the `Detail tweaks of JS shield for this site` button.
+4. Click and drag the `WebWorker` slider to the left until `Remove` is replaced by the `Unprotected`.
+5. Refresh the page, for example, by clicking on the `Refresh page` button at the top of the JShelter pop up window.
+6. You might want to restore the Worker settings once you go through the challenge.
+
+### Workaround steps (alternative if you do not want to dig in JShelter's pop up):
+
+1. Click on the JShelter badge icon (typically in the toolbar next to your navigation bar; if you cannot locate the icon, see [this question](https://jshelter.org/faq/#can-i-see-a-jshelter-badge-icon-next-to-my-navigation-bar-i-want-to-interact-with-the-extension-easily-and-avoid-going-through-settings)).
+2. Expand JavaScript Shield settings by clicking on the `Modify` button.
+3. Choose "Turn JavaScript Shield off"
+4. Refresh the page, for example, by clicking on the `Refresh page` button at the top of the JShelter pop up window.
+
+:::note
+
+Taking these actions will remove all protections of JavaScript Shield for all pages at the visited web site. You might want review and amend your JavaScript shield settings once you go through the challenge based on your operational security model.
+
+:::
+
+### Workaround steps (alternative if you do not like JShelter's pop up):
 
 1. Open JShelter extension settings
 2. Click on JS Shield details
 3. Enter in the domain for a website protected by Anubis
 4. Choose "Turn JavaScript Shield off"
 5. Hit "Add to list"
+
+:::note
+
+Taking these actions will remove all protections of JavaScript Shield for all pages at the visited web site. You might want review and amend your JavaScript shield settings once you go through the challenge based on your operational security model.
+
+:::

docs/docs/user/known-instances.md

@@ -29,9 +29,19 @@ This page contains a non-exhaustive list with all websites using Anubis.
 - https://wiki.archlinux.org/
 - https://git.devuan.org/
 - https://hydra.nixos.org/
-
+- https://codeberg.org/
+- https://www.cfaarchive.org/
+- https://forum.freecad.org/
+- <details>
+  <summary>Sourceware</summary>
+  - https://sourceware.org/cgit
+  - https://sourceware.org/glibc/wiki
+  - https://builder.sourceware.org/testruns/
+  - https://patchwork.sourceware.org/
+  - https://gcc.gnu.org/bugzilla/
+  - https://gcc.gnu.org/cgit
+  </details>
 - <details>
   <summary>The United Nations</summary>
-
   - https://policytoolbox.iiep.unesco.org/
   </details>

go.mod

@@ -40,9 +40,9 @@ require (
 	github.com/prometheus/procfs v0.15.1 // indirect
 	golang.org/x/exp/typeparams v0.0.0-20231108232855-2478ac86f678 // indirect
 	golang.org/x/mod v0.24.0 // indirect
-	golang.org/x/sync v0.12.0 // indirect
+	golang.org/x/sync v0.13.0 // indirect
 	golang.org/x/sys v0.32.0 // indirect
-	golang.org/x/tools v0.31.0 // indirect
+	golang.org/x/tools v0.32.0 // indirect
 	google.golang.org/protobuf v1.36.5 // indirect
 	honnef.co/go/tools v0.6.1 // indirect
 	k8s.io/apimachinery v0.32.3 // indirect
@@ -52,6 +52,7 @@ require (
 
 tool (
 	github.com/a-h/templ/cmd/templ
+	golang.org/x/tools/cmd/goimports
 	golang.org/x/tools/cmd/stringer
 	honnef.co/go/tools/cmd/staticcheck
 )

go.sum

@@ -99,6 +99,8 @@ golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJ
 golang.org/x/sync v0.1.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.12.0 h1:MHc5BpPuC30uJk597Ri8TV3CNZcTLu6B6z4lJy+g6Jw=
 golang.org/x/sync v0.12.0/go.mod h1:1dzgHSNfp02xaA81J2MS99Qcpr2w7fw1gpm99rleRqA=
+golang.org/x/sync v0.13.0 h1:AauUjRAJ9OSnvULf/ARrrVywoJDy0YS2AwQ98I37610=
+golang.org/x/sync v0.13.0/go.mod h1:1dzgHSNfp02xaA81J2MS99Qcpr2w7fw1gpm99rleRqA=
 golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
 golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
@@ -128,6 +130,8 @@ golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc
 golang.org/x/tools v0.6.0/go.mod h1:Xwgl3UAJ/d3gWutnCtw505GrjyAbvKui8lOU390QaIU=
 golang.org/x/tools v0.31.0 h1:0EedkvKDbh+qistFTd0Bcwe/YLh4vHwWEkiI0toFIBU=
 golang.org/x/tools v0.31.0/go.mod h1:naFTU+Cev749tSJRXJlna0T3WxKvb1kWEx15xA4SdmQ=
+golang.org/x/tools v0.32.0 h1:Q7N1vhpkQv7ybVzLFtTjvQya2ewbwNDZzUgfXGqtMWU=
+golang.org/x/tools v0.32.0/go.mod h1:ZxrU41P/wAbZD8EDa6dDCa6XfpkhJ7HFMjHJXfBDu8s=
 golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
 google.golang.org/protobuf v1.36.5 h1:tPhr+woSbjfYvY6/GPufUoYizxw1cF/yFoxJ2fmpwlM=
 google.golang.org/protobuf v1.36.5/go.mod h1:9fA7Ob0pmnwhb644+1+CVWFRbNajQ6iRojtC/QF5bRE=

internal/ogtags/cache.go

@@ -8,18 +8,21 @@ import (
 )
 
 // GetOGTags is the main function that retrieves Open Graph tags for a URL
-func (c *OGTagCache) GetOGTags(url *url.URL) (map[string]string, error) {
+func (c *OGTagCache) GetOGTags(url *url.URL, originalHost string) (map[string]string, error) {
 	if url == nil {
 		return nil, errors.New("nil URL provided, cannot fetch OG tags")
 	}
-	urlStr := c.getTarget(url)
+
+	target := c.getTarget(url)
+	cacheKey := c.generateCacheKey(target, originalHost)
+
 	// Check cache first
-	if cachedTags := c.checkCache(urlStr); cachedTags != nil {
+	if cachedTags := c.checkCache(cacheKey); cachedTags != nil {
 		return cachedTags, nil
 	}
 
-	// Fetch HTML content
-	doc, err := c.fetchHTMLDocument(urlStr)
+	// Fetch HTML content, passing the original host
+	doc, err := c.fetchHTMLDocumentWithCache(target, originalHost, cacheKey)
 	if errors.Is(err, syscall.ECONNREFUSED) {
 		slog.Debug("Connection refused, returning empty tags")
 		return nil, nil
@@ -35,17 +38,28 @@ func (c *OGTagCache) GetOGTags(url *url.URL) (map[string]string, error) {
 	ogTags := c.extractOGTags(doc)
 
 	// Store in cache
-	c.cache.Set(urlStr, ogTags, c.ogTimeToLive)
+	c.cache.Set(cacheKey, ogTags, c.ogTimeToLive)
 
 	return ogTags, nil
 }
 
+func (c *OGTagCache) generateCacheKey(target string, originalHost string) string {
+	var cacheKey string
+
+	if c.ogCacheConsiderHost {
+		cacheKey = target + "|" + originalHost
+	} else {
+		cacheKey = target
+	}
+	return cacheKey
+}
+
 // checkCache checks if we have the tags cached and returns them if so
-func (c *OGTagCache) checkCache(urlStr string) map[string]string {
-	if cachedTags, ok := c.cache.Get(urlStr); ok {
+func (c *OGTagCache) checkCache(cacheKey string) map[string]string {
+	if cachedTags, ok := c.cache.Get(cacheKey); ok {
 		slog.Debug("cache hit", "tags", cachedTags)
 		return cachedTags
 	}
-	slog.Debug("cache miss", "url", urlStr)
+	slog.Debug("cache miss", "url", cacheKey)
 	return nil
 }

internal/ogtags/cache_test.go

@@ -4,12 +4,13 @@ import (
 	"net/http"
 	"net/http/httptest"
 	"net/url"
+	"reflect"
 	"testing"
 	"time"
 )
 
 func TestCheckCache(t *testing.T) {
-	cache := NewOGTagCache("http://example.com", true, time.Minute)
+	cache := NewOGTagCache("http://example.com", true, time.Minute, false)
 
 	// Set up test data
 	urlStr := "http://example.com/page"
@@ -17,18 +18,19 @@ func TestCheckCache(t *testing.T) {
 		"og:title":       "Test Title",
 		"og:description": "Test Description",
 	}
+	cacheKey := cache.generateCacheKey(urlStr, "example.com")
 
 	// Test cache miss
-	tags := cache.checkCache(urlStr)
+	tags := cache.checkCache(cacheKey)
 	if tags != nil {
 		t.Errorf("expected nil tags on cache miss, got %v", tags)
 	}
 
 	// Manually add to cache
-	cache.cache.Set(urlStr, expectedTags, time.Minute)
+	cache.cache.Set(cacheKey, expectedTags, time.Minute)
 
 	// Test cache hit
-	tags = cache.checkCache(urlStr)
+	tags = cache.checkCache(cacheKey)
 	if tags == nil {
 		t.Fatal("expected non-nil tags on cache hit, got nil")
 	}
@@ -67,7 +69,7 @@ func TestGetOGTags(t *testing.T) {
 	defer ts.Close()
 
 	// Create an instance of OGTagCache with a short TTL for testing
-	cache := NewOGTagCache(ts.URL, true, 1*time.Minute)
+	cache := NewOGTagCache(ts.URL, true, 1*time.Minute, false)
 
 	// Parse the test server URL
 	parsedURL, err := url.Parse(ts.URL)
@@ -76,7 +78,8 @@ func TestGetOGTags(t *testing.T) {
 	}
 
 	// Test fetching OG tags from the test server
-	ogTags, err := cache.GetOGTags(parsedURL)
+	// Pass the host from the parsed test server URL
+	ogTags, err := cache.GetOGTags(parsedURL, parsedURL.Host)
 	if err != nil {
 		t.Fatalf("failed to get OG tags: %v", err)
 	}
@@ -95,13 +98,15 @@ func TestGetOGTags(t *testing.T) {
 	}
 
 	// Test fetching OG tags from the cache
-	ogTags, err = cache.GetOGTags(parsedURL)
+	// Pass the host from the parsed test server URL
+	ogTags, err = cache.GetOGTags(parsedURL, parsedURL.Host)
 	if err != nil {
 		t.Fatalf("failed to get OG tags from cache: %v", err)
 	}
 
 	// Test fetching OG tags from the cache (3rd time)
-	newOgTags, err := cache.GetOGTags(parsedURL)
+	// Pass the host from the parsed test server URL
+	newOgTags, err := cache.GetOGTags(parsedURL, parsedURL.Host)
 	if err != nil {
 		t.Fatalf("failed to get OG tags from cache: %v", err)
 	}
@@ -120,3 +125,116 @@ func TestGetOGTags(t *testing.T) {
 
 	}
 }
+
+// TestGetOGTagsWithHostConsideration tests the behavior of the cache with and without host consideration and for multiple hosts in a theoretical setup.
+func TestGetOGTagsWithHostConsideration(t *testing.T) {
+	var loadCount int // Counter to track how many times the test route is loaded
+
+	// Create a test server
+	ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		loadCount++ // Increment counter on each request to the server
+		w.Header().Set("Content-Type", "text/html")
+		w.Write([]byte(`
+			<!DOCTYPE html>
+			<html>
+			<head>
+				<meta property="og:title" content="Test Title" />
+				<meta property="og:description" content="Test Description" />
+			</head>
+			<body><p>Content</p></body>
+			</html>
+		`))
+	}))
+	defer ts.Close()
+
+	parsedURL, err := url.Parse(ts.URL)
+	if err != nil {
+		t.Fatalf("failed to parse test server URL: %v", err)
+	}
+
+	expectedTags := map[string]string{
+		"og:title":       "Test Title",
+		"og:description": "Test Description",
+	}
+
+	testCases := []struct {
+		name                string
+		ogCacheConsiderHost bool
+		requests            []struct {
+			host              string
+			expectedLoadCount int // Expected load count *after* this request
+		}
+	}{
+		{
+			name:                "Host Not Considered - Same Host",
+			ogCacheConsiderHost: false,
+			requests: []struct {
+				host              string
+				expectedLoadCount int
+			}{
+				{"host1", 1}, // First request, miss
+				{"host1", 1}, // Second request, same host, hit (host ignored)
+			},
+		},
+		{
+			name:                "Host Not Considered - Different Host",
+			ogCacheConsiderHost: false,
+			requests: []struct {
+				host              string
+				expectedLoadCount int
+			}{
+				{"host1", 1}, // First request, miss
+				{"host2", 1}, // Second request, different host, hit (host ignored)
+			},
+		},
+		{
+			name:                "Host Considered - Same Host",
+			ogCacheConsiderHost: true,
+			requests: []struct {
+				host              string
+				expectedLoadCount int
+			}{
+				{"host1", 1}, // First request, miss
+				{"host1", 1}, // Second request, same host, hit
+			},
+		},
+		{
+			name:                "Host Considered - Different Host",
+			ogCacheConsiderHost: true,
+			requests: []struct {
+				host              string
+				expectedLoadCount int
+			}{
+				{"host1", 1}, // First request, miss
+				{"host2", 2}, // Second request, different host, miss
+				{"host2", 2}, // Third request, same as second, hit
+				{"host1", 2}, // Fourth request, same as first, hit
+			},
+		},
+	}
+
+	for _, tc := range testCases {
+		t.Run(tc.name, func(t *testing.T) {
+			loadCount = 0 // Reset load count for each test case
+			cache := NewOGTagCache(ts.URL, true, 1*time.Minute, tc.ogCacheConsiderHost)
+
+			for i, req := range tc.requests {
+				ogTags, err := cache.GetOGTags(parsedURL, req.host)
+				if err != nil {
+					t.Errorf("Request %d (host: %s): unexpected error: %v", i+1, req.host, err)
+					continue // Skip further checks for this request if error occurred
+				}
+
+				// Verify tags are correct (should always be the same in this setup)
+				if !reflect.DeepEqual(ogTags, expectedTags) {
+					t.Errorf("Request %d (host: %s): expected tags %v, got %v", i+1, req.host, expectedTags, ogTags)
+				}
+
+				// Verify the load count to check cache hit/miss behavior
+				if loadCount != req.expectedLoadCount {
+					t.Errorf("Request %d (host: %s): expected load count %d, got %d (cache hit/miss mismatch)", i+1, req.host, req.expectedLoadCount, loadCount)
+				}
+			}
+		})
+	}
+}

internal/ogtags/fetch.go

@@ -1,6 +1,7 @@
 package ogtags
 
 import (
+	"context"
 	"errors"
 	"fmt"
 	"golang.org/x/net/html"
@@ -16,17 +17,35 @@ var (
 	emptyMap     = map[string]string{}             // used to indicate an empty result in the cache. Can't use nil as it would be a cache miss.
 )
 
-func (c *OGTagCache) fetchHTMLDocument(urlStr string) (*html.Node, error) {
-	resp, err := c.client.Get(urlStr)
+// fetchHTMLDocumentWithCache fetches the HTML document from the given URL string,
+// preserving the original host header.
+func (c *OGTagCache) fetchHTMLDocumentWithCache(urlStr string, originalHost string, cacheKey string) (*html.Node, error) {
+	req, err := http.NewRequestWithContext(context.Background(), "GET", urlStr, nil)
+	if err != nil {
+		return nil, fmt.Errorf("failed to create http request: %w", err)
+	}
+
+	// Set the Host header to the original host
+	if originalHost != "" {
+		req.Host = originalHost
+	}
+
+	// Add proxy headers
+	req.Header.Set("X-Forwarded-Proto", "https")
+	req.Header.Set("User-Agent", "Anubis-OGTag-Fetcher/1.0") // For tracking purposes
+
+	// Send the request
+	resp, err := c.client.Do(req)
 	if err != nil {
 		var netErr net.Error
 		if errors.As(err, &netErr) && netErr.Timeout() {
 			slog.Debug("og: request timed out", "url", urlStr)
-			c.cache.Set(urlStr, emptyMap, c.ogTimeToLive/2) // Cache empty result for half the TTL to not spam the server
+			c.cache.Set(cacheKey, emptyMap, c.ogTimeToLive/2) // Cache empty result for half the TTL to not spam the server
 		}
 		return nil, fmt.Errorf("http get failed: %w", err)
 	}
-	// this defer will call MaxBytesReader's Close, which closes the original body.
+
+	// Ensure the response body is closed
 	defer func(Body io.ReadCloser) {
 		err := Body.Close()
 		if err != nil {
@@ -36,19 +55,17 @@ func (c *OGTagCache) fetchHTMLDocument(urlStr string) (*html.Node, error) {
 
 	if resp.StatusCode != http.StatusOK {
 		slog.Debug("og: received non-OK status code", "url", urlStr, "status", resp.StatusCode)
-		c.cache.Set(urlStr, emptyMap, c.ogTimeToLive) // Cache empty result for non-successful status codes
+		c.cache.Set(cacheKey, emptyMap, c.ogTimeToLive) // Cache empty result for non-successful status codes
 		return nil, fmt.Errorf("%w: page not found", ErrOgHandled)
 	}
 
 	// Check content type
 	ct := resp.Header.Get("Content-Type")
 	if ct == "" {
-		// assume non html body
 		return nil, fmt.Errorf("missing Content-Type header")
 	} else {
 		mediaType, _, err := mime.ParseMediaType(ct)
 		if err != nil {
-			// Malformed Content-Type header
 			slog.Debug("og: malformed Content-Type header", "url", urlStr, "contentType", ct)
 			return nil, fmt.Errorf("%w malformed Content-Type header: %w", ErrOgHandled, err)
 		}
@@ -59,17 +76,16 @@ func (c *OGTagCache) fetchHTMLDocument(urlStr string) (*html.Node, error) {
 		}
 	}
 
-	resp.Body = http.MaxBytesReader(nil, resp.Body, c.maxContentLength)
+	resp.Body = http.MaxBytesReader(nil, resp.Body, maxContentLength)
 
 	doc, err := html.Parse(resp.Body)
 	if err != nil {
 		// Check if the error is specifically because the limit was exceeded
 		var maxBytesErr *http.MaxBytesError
 		if errors.As(err, &maxBytesErr) {
-			slog.Debug("og: content exceeded max length", "url", urlStr, "limit", c.maxContentLength)
-			return nil, fmt.Errorf("content too large: exceeded %d bytes", c.maxContentLength)
+			slog.Debug("og: content exceeded max length", "url", urlStr, "limit", maxContentLength)
+			return nil, fmt.Errorf("content too large: exceeded %d bytes", maxContentLength)
 		}
-		// parsing error (e.g., malformed HTML)
 		return nil, fmt.Errorf("failed to parse HTML: %w", err)
 	}
 

internal/ogtags/fetch_test.go

@@ -2,6 +2,7 @@ package ogtags
 
 import (
 	"fmt"
+	"golang.org/x/net/html"
 	"io"
 	"net/http"
 	"net/http/httptest"
@@ -78,8 +79,8 @@ func TestFetchHTMLDocument(t *testing.T) {
 			}))
 			defer ts.Close()
 
-			cache := NewOGTagCache("", true, time.Minute)
-			doc, err := cache.fetchHTMLDocument(ts.URL)
+			cache := NewOGTagCache("", true, time.Minute, false)
+			doc, err := cache.fetchHTMLDocument(ts.URL, "anything")
 
 			if tt.expectError {
 				if err == nil {
@@ -105,9 +106,9 @@ func TestFetchHTMLDocumentInvalidURL(t *testing.T) {
 		t.Skip("test requires theoretical network egress")
 	}
 
-	cache := NewOGTagCache("", true, time.Minute)
+	cache := NewOGTagCache("", true, time.Minute, false)
 
-	doc, err := cache.fetchHTMLDocument("http://invalid.url.that.doesnt.exist.example")
+	doc, err := cache.fetchHTMLDocument("http://invalid.url.that.doesnt.exist.example", "anything")
 
 	if err == nil {
 		t.Error("expected error for invalid URL, got nil")
@@ -117,3 +118,9 @@ func TestFetchHTMLDocumentInvalidURL(t *testing.T) {
 		t.Error("expected nil document for invalid URL, got non-nil")
 	}
 }
+
+// fetchHTMLDocument allows you to call fetchHTMLDocumentWithCache without a duplicate generateCacheKey call
+func (c *OGTagCache) fetchHTMLDocument(urlStr string, originalHost string) (*html.Node, error) {
+	cacheKey := c.generateCacheKey(urlStr, originalHost)
+	return c.fetchHTMLDocumentWithCache(urlStr, originalHost, cacheKey)
+}

internal/ogtags/integration_test.go

@@ -104,7 +104,7 @@ func TestIntegrationGetOGTags(t *testing.T) {
 	for _, tc := range testCases {
 		t.Run(tc.name, func(t *testing.T) {
 			// Create cache instance
-			cache := NewOGTagCache(ts.URL, true, 1*time.Minute)
+			cache := NewOGTagCache(ts.URL, true, 1*time.Minute, false)
 
 			// Create URL for test
 			testURL, _ := url.Parse(ts.URL)
@@ -112,7 +112,8 @@ func TestIntegrationGetOGTags(t *testing.T) {
 			testURL.RawQuery = tc.query
 
 			// Get OG tags
-			ogTags, err := cache.GetOGTags(testURL)
+			// Pass the host from the test URL
+			ogTags, err := cache.GetOGTags(testURL, testURL.Host)
 
 			// Check error expectation
 			if tc.expectError {
@@ -139,7 +140,8 @@ func TestIntegrationGetOGTags(t *testing.T) {
 			}
 
 			// Test cache retrieval
-			cachedOGTags, err := cache.GetOGTags(testURL)
+			// Pass the host from the test URL
+			cachedOGTags, err := cache.GetOGTags(testURL, testURL.Host)
 			if err != nil {
 				t.Fatalf("failed to get OG tags from cache: %v", err)
 			}

internal/ogtags/ogtags.go

@@ -1,51 +1,111 @@
 package ogtags
 
 import (
+	"context"
+	"log/slog"
+	"net"
 	"net/http"
 	"net/url"
+	"strings"
 	"time"
 
 	"github.com/TecharoHQ/anubis/decaymap"
 )
 
+const (
+	maxContentLength = 16 << 20        // 16 MiB in bytes, if there is a reasonable reason that you need more than this...Why?
+	httpTimeout      = 5 * time.Second /*todo: make this configurable?*/
+)
+
 type OGTagCache struct {
-	cache            *decaymap.Impl[string, map[string]string]
-	target           string
-	ogPassthrough    bool
-	ogTimeToLive     time.Duration
-	approvedTags     []string
-	approvedPrefixes []string
-	client           *http.Client
-	maxContentLength int64
+	cache               *decaymap.Impl[string, map[string]string]
+	targetURL           *url.URL
+	ogCacheConsiderHost bool
+	ogPassthrough       bool
+	ogTimeToLive        time.Duration
+	approvedTags        []string
+	approvedPrefixes    []string
+	client              *http.Client
 }
 
-func NewOGTagCache(target string, ogPassthrough bool, ogTimeToLive time.Duration) *OGTagCache {
+func NewOGTagCache(target string, ogPassthrough bool, ogTimeToLive time.Duration, ogTagsConsiderHost bool) *OGTagCache {
 	// Predefined approved tags and prefixes
 	// In the future, these could come from configuration
 	defaultApprovedTags := []string{"description", "keywords", "author"}
 	defaultApprovedPrefixes := []string{"og:", "twitter:", "fediverse:"}
-	client := &http.Client{
-		Timeout: 5 * time.Second, /*make this configurable?*/
+
+	var parsedTargetURL *url.URL
+	var err error
+
+	if target == "" {
+		// Default to localhost if target is empty
+		parsedTargetURL, _ = url.Parse("http://localhost")
+	} else {
+		parsedTargetURL, err = url.Parse(target)
+		if err != nil {
+			slog.Debug("og: failed to parse target URL, treating as non-unix", "target", target, "error", err)
+			// If parsing fails, treat it as a non-unix target for backward compatibility or default behavior
+			// For now, assume it's not a scheme issue but maybe an invalid char, etc.
+			// A simple string target might be intended if it's not a full URL.
+			parsedTargetURL = &url.URL{Scheme: "http", Host: target} // Assume http if scheme missing and host-like
+			if !strings.Contains(target, "://") && !strings.HasPrefix(target, "unix:") {
+				// If it looks like just a host/host:port (and not unix), prepend http:// (todo: is this bad...? Trace path to see if i can yell at user to do it right)
+				parsedTargetURL, _ = url.Parse("http://" + target) // fetch cares about scheme but anubis doesn't
+			}
+		}
 	}
 
-	const maxContentLength = 16 << 20 // 16 MiB in bytes
+	client := &http.Client{
+		Timeout: httpTimeout,
+	}
+
+	// Configure custom transport for Unix sockets
+	if parsedTargetURL.Scheme == "unix" {
+		socketPath := parsedTargetURL.Path // For unix scheme, path is the socket path
+		client.Transport = &http.Transport{
+			DialContext: func(_ context.Context, _, _ string) (net.Conn, error) {
+				return net.Dial("unix", socketPath)
+			},
+		}
+	}
 
 	return &OGTagCache{
-		cache:            decaymap.New[string, map[string]string](),
-		target:           target,
-		ogPassthrough:    ogPassthrough,
-		ogTimeToLive:     ogTimeToLive,
-		approvedTags:     defaultApprovedTags,
-		approvedPrefixes: defaultApprovedPrefixes,
-		client:           client,
-		maxContentLength: maxContentLength,
+		cache:               decaymap.New[string, map[string]string](),
+		targetURL:           parsedTargetURL, // Store the parsed URL
+		ogPassthrough:       ogPassthrough,
+		ogTimeToLive:        ogTimeToLive,
+		ogCacheConsiderHost: ogTagsConsiderHost, // todo: refactor to be a separate struct
+		approvedTags:        defaultApprovedTags,
+		approvedPrefixes:    defaultApprovedPrefixes,
+		client:              client,
 	}
 }
 
+// getTarget constructs the target URL string for fetching OG tags.
+// For Unix sockets, it creates a "fake" HTTP URL that the custom dialer understands.
 func (c *OGTagCache) getTarget(u *url.URL) string {
-	return c.target + u.Path
+	if c.targetURL.Scheme == "unix" {
+		// The custom dialer ignores the host, but we need a valid http URL structure.
+		// Use "unix" as a placeholder host. Path and Query from original request are appended.
+		fakeURL := &url.URL{
+			Scheme:   "http", // Scheme must be http/https for client.Get
+			Host:     "unix", // Arbitrary host, ignored by custom dialer
+			Path:     u.Path,
+			RawQuery: u.RawQuery,
+		}
+		return fakeURL.String()
+	}
+
+	// For regular http/https targets
+	target := *c.targetURL // Make a copy
+	target.Path = u.Path
+	target.RawQuery = u.RawQuery
+	return target.String()
+
 }
 
 func (c *OGTagCache) Cleanup() {
-	c.cache.Cleanup()
+	if c.cache != nil {
+		c.cache.Cleanup()
+	}
 }

internal/ogtags/ogtags_test.go

@@ -1,7 +1,16 @@
 package ogtags
 
 import (
+	"context"
+	"errors"
+	"fmt"
+	"net"
+	"net/http"
 	"net/url"
+	"os"
+	"path/filepath"
+	"reflect"
+	"strings"
 	"testing"
 	"time"
 )
@@ -29,14 +38,23 @@ func TestNewOGTagCache(t *testing.T) {
 
 	for _, tt := range tests {
 		t.Run(tt.name, func(t *testing.T) {
-			cache := NewOGTagCache(tt.target, tt.ogPassthrough, tt.ogTimeToLive)
+			cache := NewOGTagCache(tt.target, tt.ogPassthrough, tt.ogTimeToLive, false)
 
 			if cache == nil {
 				t.Fatal("expected non-nil cache, got nil")
 			}
 
-			if cache.target != tt.target {
-				t.Errorf("expected target %s, got %s", tt.target, cache.target)
+			// Check the parsed targetURL, handling the default case for empty target
+			expectedURLStr := tt.target
+			if tt.target == "" {
+				// Default behavior when target is empty is now http://localhost
+				expectedURLStr = "http://localhost"
+			} else if !strings.Contains(tt.target, "://") && !strings.HasPrefix(tt.target, "unix:") {
+				// Handle case where target is just host or host:port (and not unix)
+				expectedURLStr = "http://" + tt.target
+			}
+			if cache.targetURL.String() != expectedURLStr {
+				t.Errorf("expected targetURL %s, got %s", expectedURLStr, cache.targetURL.String())
 			}
 
 			if cache.ogPassthrough != tt.ogPassthrough {
@@ -50,6 +68,45 @@ func TestNewOGTagCache(t *testing.T) {
 	}
 }
 
+// TestNewOGTagCache_UnixSocket specifically tests unix socket initialization
+func TestNewOGTagCache_UnixSocket(t *testing.T) {
+	tempDir := t.TempDir()
+	socketPath := filepath.Join(tempDir, "test.sock")
+	target := "unix://" + socketPath
+
+	cache := NewOGTagCache(target, true, 5*time.Minute, false)
+
+	if cache == nil {
+		t.Fatal("expected non-nil cache, got nil")
+	}
+
+	if cache.targetURL.Scheme != "unix" {
+		t.Errorf("expected targetURL scheme 'unix', got '%s'", cache.targetURL.Scheme)
+	}
+	if cache.targetURL.Path != socketPath {
+		t.Errorf("expected targetURL path '%s', got '%s'", socketPath, cache.targetURL.Path)
+	}
+
+	// Check if the client transport is configured for Unix sockets
+	transport, ok := cache.client.Transport.(*http.Transport)
+	if !ok {
+		t.Fatalf("expected client transport to be *http.Transport, got %T", cache.client.Transport)
+	}
+	if transport.DialContext == nil {
+		t.Fatal("expected client transport DialContext to be non-nil for unix socket")
+	}
+
+	// Attempt a dummy dial to see if it uses the correct path (optional, more involved check)
+	dummyConn, err := transport.DialContext(context.Background(), "", "")
+	if err == nil {
+		dummyConn.Close()
+		t.Log("DialContext seems functional, but couldn't verify path without a listener")
+	} else if !strings.Contains(err.Error(), "connect: connection refused") && !strings.Contains(err.Error(), "connect: no such file or directory") {
+		// We expect connection refused or not found if nothing is listening
+		t.Errorf("DialContext failed with unexpected error: %v", err)
+	}
+}
+
 func TestGetTarget(t *testing.T) {
 	tests := []struct {
 		name     string
@@ -66,24 +123,39 @@ func TestGetTarget(t *testing.T) {
 			expected: "http://example.com",
 		},
 		{
-			name:     "With complex path",
-			target:   "http://example.com",
-			path:     "/pag(#*((#@)ΓΓΓΓe/Γ",
-			query:    "id=123",
-			expected: "http://example.com/pag(#*((#@)ΓΓΓΓe/Γ",
+			name:   "With complex path",
+			target: "http://example.com",
+			path:   "/pag(#*((#@)ΓΓΓΓe/Γ",
+			query:  "id=123",
+			// Expect URL encoding and query parameter
+			expected: "http://example.com/pag%28%23%2A%28%28%23@%29%CE%93%CE%93%CE%93%CE%93e/%CE%93?id=123",
 		},
 		{
 			name:     "With query and path",
 			target:   "http://example.com",
 			path:     "/page",
 			query:    "id=123",
-			expected: "http://example.com/page",
+			expected: "http://example.com/page?id=123",
+		},
+		{
+			name:     "Unix socket target",
+			target:   "unix:/tmp/anubis.sock",
+			path:     "/some/path",
+			query:    "key=value&flag=true",
+			expected: "http://unix/some/path?key=value&flag=true", // Scheme becomes http, host is 'unix'
+		},
+		{
+			name:     "Unix socket target with ///",
+			target:   "unix:///var/run/anubis.sock",
+			path:     "/",
+			query:    "",
+			expected: "http://unix/",
 		},
 	}
 
 	for _, tt := range tests {
 		t.Run(tt.name, func(t *testing.T) {
-			cache := NewOGTagCache(tt.target, false, time.Minute)
+			cache := NewOGTagCache(tt.target, false, time.Minute, false)
 
 			u := &url.URL{
 				Path:     tt.path,
@@ -98,3 +170,86 @@ func TestGetTarget(t *testing.T) {
 		})
 	}
 }
+
+// TestIntegrationGetOGTags_UnixSocket tests fetching OG tags via a Unix socket.
+func TestIntegrationGetOGTags_UnixSocket(t *testing.T) {
+	tempDir := t.TempDir()
+
+	socketPath := filepath.Join(tempDir, "anubis-test.sock")
+
+	// Ensure the socket does not exist initially
+	_ = os.Remove(socketPath)
+
+	// Create a simple HTTP server listening on the Unix socket
+	listener, err := net.Listen("unix", socketPath)
+	if err != nil {
+		t.Fatalf("Failed to listen on unix socket %s: %v", socketPath, err)
+	}
+	defer func(listener net.Listener, socketPath string) {
+		if listener != nil {
+			if err := listener.Close(); err != nil && !errors.Is(err, net.ErrClosed) {
+				t.Logf("Error closing listener: %v", err)
+			}
+		}
+
+		if _, err := os.Stat(socketPath); err == nil {
+			if err := os.Remove(socketPath); err != nil {
+				t.Logf("Error removing socket file %s: %v", socketPath, err)
+			}
+		}
+	}(listener, socketPath)
+
+	server := &http.Server{
+		Handler: http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+			w.Header().Set("Content-Type", "text/html")
+			fmt.Fprintln(w, `<!DOCTYPE html><html><head><meta property="og:title" content="Unix Socket Test" /></head><body>Test</body></html>`)
+		}),
+	}
+	go func() {
+		if err := server.Serve(listener); err != nil && !errors.Is(err, http.ErrServerClosed) {
+			t.Logf("Unix socket server error: %v", err)
+		}
+	}()
+	defer func(server *http.Server, ctx context.Context) {
+		err := server.Shutdown(ctx)
+		if err != nil {
+			t.Logf("Error shutting down server: %v", err)
+		}
+	}(server, context.Background()) // Ensure server is shut down
+
+	// Wait a moment for the server to start
+	time.Sleep(100 * time.Millisecond)
+
+	// Create cache instance pointing to the Unix socket
+	targetURL := "unix://" + socketPath
+	cache := NewOGTagCache(targetURL, true, 1*time.Minute, false)
+
+	// Create a dummy URL for the request (path and query matter)
+	testReqURL, _ := url.Parse("/some/page?query=1")
+
+	// Get OG tags
+	// Pass an empty string for host, as it's irrelevant for unix sockets
+	ogTags, err := cache.GetOGTags(testReqURL, "")
+
+	if err != nil {
+		t.Fatalf("GetOGTags failed for unix socket: %v", err)
+	}
+
+	expectedTags := map[string]string{
+		"og:title": "Unix Socket Test",
+	}
+
+	if !reflect.DeepEqual(ogTags, expectedTags) {
+		t.Errorf("Expected OG tags %v, got %v", expectedTags, ogTags)
+	}
+
+	// Test cache retrieval (should hit cache)
+	// Pass an empty string for host
+	cachedTags, err := cache.GetOGTags(testReqURL, "")
+	if err != nil {
+		t.Fatalf("GetOGTags (cache hit) failed for unix socket: %v", err)
+	}
+	if !reflect.DeepEqual(cachedTags, expectedTags) {
+		t.Errorf("Expected cached OG tags %v, got %v", expectedTags, cachedTags)
+	}
+}

internal/ogtags/parse_test.go

@@ -12,7 +12,7 @@ import (
 // TestExtractOGTags updated with correct expectations based on filtering logic
 func TestExtractOGTags(t *testing.T) {
 	// Use a cache instance that reflects the default approved lists
-	testCache := NewOGTagCache("", false, time.Minute)
+	testCache := NewOGTagCache("", false, time.Minute, false)
 	// Manually set approved tags/prefixes based on the user request for clarity
 	testCache.approvedTags = []string{"description"}
 	testCache.approvedPrefixes = []string{"og:"}
@@ -189,7 +189,7 @@ func TestIsOGMetaTag(t *testing.T) {
 
 func TestExtractMetaTagInfo(t *testing.T) {
 	// Use a cache instance that reflects the default approved lists
-	testCache := NewOGTagCache("", false, time.Minute)
+	testCache := NewOGTagCache("", false, time.Minute, false)
 	testCache.approvedTags = []string{"description"}
 	testCache.approvedPrefixes = []string{"og:"}
 

internal/slog.go

@@ -3,6 +3,7 @@ package internal
 import (
 	"fmt"
 	"log/slog"
+	"net/http"
 	"os"
 )
 
@@ -22,3 +23,14 @@ func InitSlog(level string) {
 	})
 	slog.SetDefault(slog.New(h))
 }
+
+func GetRequestLogger(r *http.Request) *slog.Logger {
+	return slog.With(
+		"user_agent", r.UserAgent(),
+		"accept_language", r.Header.Get("Accept-Language"),
+		"priority", r.Header.Get("Priority"),
+		"x-forwarded-for",
+		r.Header.Get("X-Forwarded-For"),
+		"x-real-ip", r.Header.Get("X-Real-Ip"),
+	)
+}

lib/anubis.go

@@ -2,38 +2,31 @@ package lib
 
 import (
 	"crypto/ed25519"
-	"crypto/rand"
 	"crypto/sha256"
 	"crypto/subtle"
 	"encoding/json"
 	"fmt"
-	"io"
 	"log/slog"
 	"math"
 	"net"
 	"net/http"
 	"net/url"
-	"os"
 	"slices"
 	"strconv"
 	"strings"
 	"time"
 
-	"github.com/a-h/templ"
 	"github.com/golang-jwt/jwt/v5"
 	"github.com/prometheus/client_golang/prometheus"
 	"github.com/prometheus/client_golang/prometheus/promauto"
 
 	"github.com/TecharoHQ/anubis"
-	"github.com/TecharoHQ/anubis/data"
 	"github.com/TecharoHQ/anubis/decaymap"
 	"github.com/TecharoHQ/anubis/internal"
 	"github.com/TecharoHQ/anubis/internal/dnsbl"
 	"github.com/TecharoHQ/anubis/internal/ogtags"
 	"github.com/TecharoHQ/anubis/lib/policy"
 	"github.com/TecharoHQ/anubis/lib/policy/config"
-	"github.com/TecharoHQ/anubis/web"
-	"github.com/TecharoHQ/anubis/xess"
 )
 
 var (
@@ -64,121 +57,6 @@ var (
 	})
 )
 
-type Options struct {
-	Next            http.Handler
-	Policy          *policy.ParsedConfig
-	RedirectDomains []string
-	ServeRobotsTXT  bool
-	PrivateKey      ed25519.PrivateKey
-
-	CookieDomain      string
-	CookieName        string
-	CookiePartitioned bool
-
-	OGPassthrough bool
-	OGTimeToLive  time.Duration
-	Target        string
-
-	WebmasterEmail string
-	BasePrefix     string
-}
-
-func LoadPoliciesOrDefault(fname string, defaultDifficulty int) (*policy.ParsedConfig, error) {
-	var fin io.ReadCloser
-	var err error
-
-	if fname != "" {
-		fin, err = os.Open(fname)
-		if err != nil {
-			return nil, fmt.Errorf("can't parse policy file %s: %w", fname, err)
-		}
-	} else {
-		fname = "(data)/botPolicies.yaml"
-		fin, err = data.BotPolicies.Open("botPolicies.yaml")
-		if err != nil {
-			return nil, fmt.Errorf("[unexpected] can't parse builtin policy file %s: %w", fname, err)
-		}
-	}
-
-	defer func(fin io.ReadCloser) {
-		err := fin.Close()
-		if err != nil {
-			slog.Error("failed to close policy file", "file", fname, "err", err)
-		}
-	}(fin)
-
-	anubisPolicy, err := policy.ParseConfig(fin, fname, defaultDifficulty)
-
-	return anubisPolicy, err
-}
-
-func New(opts Options) (*Server, error) {
-	if opts.PrivateKey == nil {
-		slog.Debug("opts.PrivateKey not set, generating a new one")
-		_, priv, err := ed25519.GenerateKey(rand.Reader)
-		if err != nil {
-			return nil, fmt.Errorf("lib: can't generate private key: %v", err)
-		}
-		opts.PrivateKey = priv
-	}
-
-	anubis.BasePrefix = opts.BasePrefix
-
-	result := &Server{
-		next:       opts.Next,
-		priv:       opts.PrivateKey,
-		pub:        opts.PrivateKey.Public().(ed25519.PublicKey),
-		policy:     opts.Policy,
-		opts:       opts,
-		DNSBLCache: decaymap.New[string, dnsbl.DroneBLResponse](),
-		OGTags:     ogtags.NewOGTagCache(opts.Target, opts.OGPassthrough, opts.OGTimeToLive),
-	}
-
-	mux := http.NewServeMux()
-	xess.Mount(mux)
-
-	// Helper to add global prefix
-	registerWithPrefix := func(pattern string, handler http.Handler, method string) {
-		if method != "" {
-			method = method + " " // methods must end with a space to register with them
-		}
-
-		// Ensure there's no double slash when concatenating BasePrefix and pattern
-		basePrefix := strings.TrimSuffix(anubis.BasePrefix, "/")
-		prefix := method + basePrefix
-
-		// If pattern doesn't start with a slash, add one
-		if !strings.HasPrefix(pattern, "/") {
-			pattern = "/" + pattern
-		}
-
-		mux.Handle(prefix+pattern, handler)
-	}
-
-	// Ensure there's no double slash when concatenating BasePrefix and StaticPath
-	stripPrefix := strings.TrimSuffix(anubis.BasePrefix, "/") + anubis.StaticPath
-	registerWithPrefix(anubis.StaticPath, internal.UnchangingCache(internal.NoBrowsing(http.StripPrefix(stripPrefix, http.FileServerFS(web.Static)))), "")
-
-	if opts.ServeRobotsTXT {
-		registerWithPrefix("/robots.txt", http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
-			http.ServeFileFS(w, r, web.Static, "static/robots.txt")
-		}), "GET")
-		registerWithPrefix("/.well-known/robots.txt", http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
-			http.ServeFileFS(w, r, web.Static, "static/robots.txt")
-		}), "GET")
-	}
-
-	registerWithPrefix(anubis.APIPrefix+"make-challenge", http.HandlerFunc(result.MakeChallenge), "POST")
-	registerWithPrefix(anubis.APIPrefix+"pass-challenge", http.HandlerFunc(result.PassChallenge), "GET")
-	registerWithPrefix(anubis.APIPrefix+"check", http.HandlerFunc(result.maybeReverseProxyHttpStatusOnly), "")
-	registerWithPrefix(anubis.APIPrefix+"test-error", http.HandlerFunc(result.TestError), "GET")
-	registerWithPrefix("/", http.HandlerFunc(result.maybeReverseProxyOrPage), "")
-
-	result.mux = mux
-
-	return result, nil
-}
-
 type Server struct {
 	mux        *http.ServeMux
 	next       http.Handler
@@ -190,40 +68,6 @@ type Server struct {
 	OGTags     *ogtags.OGTagCache
 }
 
-func (s *Server) ServeHTTP(w http.ResponseWriter, r *http.Request) {
-	s.mux.ServeHTTP(w, r)
-}
-
-func (s *Server) ServeHTTPNext(w http.ResponseWriter, r *http.Request) {
-	if s.next == nil {
-		redir := r.FormValue("redir")
-		urlParsed, err := r.URL.Parse(redir)
-		if err != nil {
-			templ.Handler(web.Base("Oh noes!", web.ErrorPage("Redirect URL not parseable", s.opts.WebmasterEmail)), templ.WithStatus(http.StatusInternalServerError)).ServeHTTP(w, r)
-			return
-		}
-
-		if len(urlParsed.Host) > 0 && len(s.opts.RedirectDomains) != 0 && !slices.Contains(s.opts.RedirectDomains, urlParsed.Host) {
-			templ.Handler(web.Base("Oh noes!", web.ErrorPage("Redirect domain not allowed", s.opts.WebmasterEmail)), templ.WithStatus(http.StatusInternalServerError)).ServeHTTP(w, r)
-			return
-		} else if urlParsed.Host != r.URL.Host {
-			templ.Handler(web.Base("Oh noes!", web.ErrorPage("Redirect domain not allowed", s.opts.WebmasterEmail)), templ.WithStatus(http.StatusInternalServerError)).ServeHTTP(w, r)
-			return
-		}
-
-		if redir != "" {
-			http.Redirect(w, r, redir, http.StatusFound)
-			return
-		}
-
-		templ.Handler(
-			web.Base("You are not a bot!", web.StaticHappy()),
-		).ServeHTTP(w, r)
-	} else {
-		s.next.ServeHTTP(w, r)
-	}
-}
-
 func (s *Server) challengeFor(r *http.Request, difficulty int) string {
 	fp := sha256.Sum256(s.priv.Seed())
 
@@ -248,19 +92,12 @@ func (s *Server) maybeReverseProxyOrPage(w http.ResponseWriter, r *http.Request)
 }
 
 func (s *Server) maybeReverseProxy(w http.ResponseWriter, r *http.Request, httpStatusOnly bool) {
-	lg := slog.With(
-		"user_agent", r.UserAgent(),
-		"accept_language", r.Header.Get("Accept-Language"),
-		"priority", r.Header.Get("Priority"),
-		"x-forwarded-for",
-		r.Header.Get("X-Forwarded-For"),
-		"x-real-ip", r.Header.Get("X-Real-Ip"),
-	)
+	lg := internal.GetRequestLogger(r)
 
 	cr, rule, err := s.check(r)
 	if err != nil {
 		lg.Error("check failed", "err", err)
-		templ.Handler(web.Base("Oh noes!", web.ErrorPage("Internal Server Error: administrator has misconfigured Anubis. Please contact the administrator and ask them to look for the logs around \"maybeReverseProxy\"", s.opts.WebmasterEmail)), templ.WithStatus(http.StatusInternalServerError)).ServeHTTP(w, r)
+		s.respondWithError(w, r, "Internal Server Error: administrator has misconfigured Anubis. Please contact the administrator and ask them to look for the logs around \"maybeReverseProxy\"")
 		return
 	}
 
@@ -271,52 +108,11 @@ func (s *Server) maybeReverseProxy(w http.ResponseWriter, r *http.Request, httpS
 
 	ip := r.Header.Get("X-Real-Ip")
 
-	if s.policy.DNSBL && ip != "" {
-		resp, ok := s.DNSBLCache.Get(ip)
-		if !ok {
-			lg.Debug("looking up ip in dnsbl")
-			resp, err := dnsbl.Lookup(ip)
-			if err != nil {
-				lg.Error("can't look up ip in dnsbl", "err", err)
-			}
-			s.DNSBLCache.Set(ip, resp, 24*time.Hour)
-			droneBLHits.WithLabelValues(resp.String()).Inc()
-		}
-
-		if resp != dnsbl.AllGood {
-			lg.Info("DNSBL hit", "status", resp.String())
-			templ.Handler(web.Base("Oh noes!", web.ErrorPage(fmt.Sprintf("DroneBL reported an entry: %s, see https://dronebl.org/lookup?ip=%s", resp.String(), ip), s.opts.WebmasterEmail)), templ.WithStatus(http.StatusOK)).ServeHTTP(w, r)
-			return
-		}
+	if s.handleDNSBL(w, r, ip, lg) {
+		return
 	}
 
-	switch cr.Rule {
-	case config.RuleAllow:
-		lg.Debug("allowing traffic to origin (explicit)")
-		s.ServeHTTPNext(w, r)
-		return
-	case config.RuleDeny:
-		s.ClearCookie(w)
-		lg.Info("explicit deny")
-		if rule == nil {
-			lg.Error("rule is nil, cannot calculate checksum")
-			templ.Handler(web.Base("Oh noes!", web.ErrorPage("Other internal server error (contact the admin)", s.opts.WebmasterEmail)), templ.WithStatus(http.StatusInternalServerError)).ServeHTTP(w, r)
-			return
-		}
-		hash := rule.Hash()
-
-		lg.Debug("rule hash", "hash", hash)
-		templ.Handler(web.Base("Oh noes!", web.ErrorPage(fmt.Sprintf("Access Denied: error code %s", hash), s.opts.WebmasterEmail)), templ.WithStatus(http.StatusOK)).ServeHTTP(w, r)
-		return
-	case config.RuleChallenge:
-		lg.Debug("challenge requested")
-	case config.RuleBenchmark:
-		lg.Debug("serving benchmark page")
-		s.RenderBench(w, r)
-		return
-	default:
-		s.ClearCookie(w)
-		templ.Handler(web.Base("Oh noes!", web.ErrorPage("Other internal server error (contact the admin)", s.opts.WebmasterEmail)), templ.WithStatus(http.StatusInternalServerError)).ServeHTTP(w, r)
+	if s.checkRules(w, r, cr, lg, rule) {
 		return
 	}
 
@@ -357,53 +153,64 @@ func (s *Server) maybeReverseProxy(w http.ResponseWriter, r *http.Request, httpS
 	s.ServeHTTPNext(w, r)
 }
 
-func (s *Server) RenderIndex(w http.ResponseWriter, r *http.Request, rule *policy.Bot, returnHTTPStatusOnly bool) {
-	if returnHTTPStatusOnly {
-		w.WriteHeader(http.StatusUnauthorized)
-		w.Write([]byte("Authorization required"))
-		return
-	}
-
-	lg := slog.With(
-		"user_agent", r.UserAgent(),
-		"accept_language", r.Header.Get("Accept-Language"),
-		"priority", r.Header.Get("Priority"),
-		"x-forwarded-for",
-		r.Header.Get("X-Forwarded-For"),
-		"x-real-ip", r.Header.Get("X-Real-Ip"),
-	)
-
-	challenge := s.challengeFor(r, rule.Challenge.Difficulty)
-
-	var ogTags map[string]string = nil
-	if s.opts.OGPassthrough {
-		var err error
-		ogTags, err = s.OGTags.GetOGTags(r.URL)
-		if err != nil {
-			lg.Error("failed to get OG tags", "err", err)
-			ogTags = nil
+func (s *Server) checkRules(w http.ResponseWriter, r *http.Request, cr policy.CheckResult, lg *slog.Logger, rule *policy.Bot) bool {
+	switch cr.Rule {
+	case config.RuleAllow:
+		lg.Debug("allowing traffic to origin (explicit)")
+		s.ServeHTTPNext(w, r)
+		return true
+	case config.RuleDeny:
+		s.ClearCookie(w)
+		lg.Info("explicit deny")
+		if rule == nil {
+			lg.Error("rule is nil, cannot calculate checksum")
+			s.respondWithError(w, r, "Internal Server Error: Please contact the administrator and ask them to look for the logs around \"maybeReverseProxy.RuleDeny\"")
+			return true
 		}
-	}
+		hash := rule.Hash()
 
-	component, err := web.BaseWithChallengeAndOGTags("Making sure you're not a bot!", web.Index(), challenge, rule.Challenge, ogTags)
-	if err != nil {
-		lg.Error("render failed", "err", err)
-		templ.Handler(web.Base("Oh noes!", web.ErrorPage("Other internal server error (contact the admin)", s.opts.WebmasterEmail)), templ.WithStatus(http.StatusInternalServerError)).ServeHTTP(w, r)
-		return
+		lg.Debug("rule hash", "hash", hash)
+		s.respondWithStatus(w, r, fmt.Sprintf("Access Denied: error code %s", hash), s.policy.StatusCodes.Deny)
+		return true
+	case config.RuleChallenge:
+		lg.Debug("challenge requested")
+	case config.RuleBenchmark:
+		lg.Debug("serving benchmark page")
+		s.RenderBench(w, r)
+		return true
+	default:
+		s.ClearCookie(w)
+		slog.Error("CONFIG ERROR: unknown rule", "rule", cr.Rule)
+		s.respondWithError(w, r, "Internal Server Error: administrator has misconfigured Anubis. Please contact the administrator and ask them to look for the logs around \"maybeReverseProxy.Rules\"")
+		return true
 	}
-
-	handler := internal.NoStoreCache(templ.Handler(component))
-	handler.ServeHTTP(w, r)
+	return false
 }
 
-func (s *Server) RenderBench(w http.ResponseWriter, r *http.Request) {
-	templ.Handler(
-		web.Base("Benchmarking Anubis!", web.Bench()),
-	).ServeHTTP(w, r)
+func (s *Server) handleDNSBL(w http.ResponseWriter, r *http.Request, ip string, lg *slog.Logger) bool {
+	if s.policy.DNSBL && ip != "" {
+		resp, ok := s.DNSBLCache.Get(ip)
+		if !ok {
+			lg.Debug("looking up ip in dnsbl")
+			resp, err := dnsbl.Lookup(ip)
+			if err != nil {
+				lg.Error("can't look up ip in dnsbl", "err", err)
+			}
+			s.DNSBLCache.Set(ip, resp, 24*time.Hour)
+			droneBLHits.WithLabelValues(resp.String()).Inc()
+		}
+
+		if resp != dnsbl.AllGood {
+			lg.Info("DNSBL hit", "status", resp.String())
+			s.respondWithStatus(w, r, fmt.Sprintf("DroneBL reported an entry: %s, see https://dronebl.org/lookup?ip=%s", resp.String(), ip), s.policy.StatusCodes.Deny)
+			return true
+		}
+	}
+	return false
 }
 
 func (s *Server) MakeChallenge(w http.ResponseWriter, r *http.Request) {
-	lg := slog.With("user_agent", r.UserAgent(), "accept_language", r.Header.Get("Accept-Language"), "priority", r.Header.Get("Priority"), "x-forwarded-for", r.Header.Get("X-Forwarded-For"), "x-real-ip", r.Header.Get("X-Real-Ip"))
+	lg := internal.GetRequestLogger(r)
 
 	encoder := json.NewEncoder(w)
 	cr, rule, err := s.check(r)
@@ -441,19 +248,13 @@ func (s *Server) MakeChallenge(w http.ResponseWriter, r *http.Request) {
 }
 
 func (s *Server) PassChallenge(w http.ResponseWriter, r *http.Request) {
-	lg := slog.With(
-		"user_agent", r.UserAgent(),
-		"accept_language", r.Header.Get("Accept-Language"),
-		"priority", r.Header.Get("Priority"),
-		"x-forwarded-for", r.Header.Get("X-Forwarded-For"),
-		"x-real-ip", r.Header.Get("X-Real-Ip"),
-	)
+	lg := internal.GetRequestLogger(r)
 
 	redir := r.FormValue("redir")
 	redirURL, err := url.ParseRequestURI(redir)
 	if err != nil {
 		lg.Error("invalid redirect", "err", err)
-		templ.Handler(web.Base("Oh noes!", web.ErrorPage("invalid redirect", s.opts.WebmasterEmail)), templ.WithStatus(http.StatusInternalServerError)).ServeHTTP(w, r)
+		s.respondWithError(w, r, "Invalid redirect")
 		return
 	}
 	// used by the path checker rule
@@ -462,7 +263,7 @@ func (s *Server) PassChallenge(w http.ResponseWriter, r *http.Request) {
 	cr, rule, err := s.check(r)
 	if err != nil {
 		lg.Error("check failed", "err", err)
-		templ.Handler(web.Base("Oh noes!", web.ErrorPage("Internal Server Error: administrator has misconfigured Anubis. Please contact the administrator and ask them to look for the logs around \"passChallenge\".", s.opts.WebmasterEmail)), templ.WithStatus(http.StatusInternalServerError)).ServeHTTP(w, r)
+		s.respondWithError(w, r, "Internal Server Error: administrator has misconfigured Anubis. Please contact the administrator and ask them to look for the logs around \"passChallenge\".\"")
 		return
 	}
 	lg = lg.With("check_result", cr)
@@ -471,7 +272,7 @@ func (s *Server) PassChallenge(w http.ResponseWriter, r *http.Request) {
 	if nonceStr == "" {
 		s.ClearCookie(w)
 		lg.Debug("no nonce")
-		templ.Handler(web.Base("Oh noes!", web.ErrorPage("missing nonce", s.opts.WebmasterEmail)), templ.WithStatus(http.StatusInternalServerError)).ServeHTTP(w, r)
+		s.respondWithError(w, r, "missing nonce")
 		return
 	}
 
@@ -479,7 +280,7 @@ func (s *Server) PassChallenge(w http.ResponseWriter, r *http.Request) {
 	if elapsedTimeStr == "" {
 		s.ClearCookie(w)
 		lg.Debug("no elapsedTime")
-		templ.Handler(web.Base("Oh noes!", web.ErrorPage("missing elapsedTime", s.opts.WebmasterEmail)), templ.WithStatus(http.StatusInternalServerError)).ServeHTTP(w, r)
+		s.respondWithError(w, r, "missing elapsedTime")
 		return
 	}
 
@@ -487,7 +288,7 @@ func (s *Server) PassChallenge(w http.ResponseWriter, r *http.Request) {
 	if err != nil {
 		s.ClearCookie(w)
 		lg.Debug("elapsedTime doesn't parse", "err", err)
-		templ.Handler(web.Base("Oh noes!", web.ErrorPage("invalid elapsedTime", s.opts.WebmasterEmail)), templ.WithStatus(http.StatusInternalServerError)).ServeHTTP(w, r)
+		s.respondWithError(w, r, "invalid elapsedTime")
 		return
 	}
 
@@ -497,15 +298,11 @@ func (s *Server) PassChallenge(w http.ResponseWriter, r *http.Request) {
 	response := r.FormValue("response")
 	urlParsed, err := r.URL.Parse(redir)
 	if err != nil {
-		templ.Handler(web.Base("Oh noes!", web.ErrorPage("Redirect URL not parseable", s.opts.WebmasterEmail)), templ.WithStatus(http.StatusInternalServerError)).ServeHTTP(w, r)
+		s.respondWithError(w, r, "Redirect URL not parseable")
 		return
 	}
-
-	if len(urlParsed.Host) > 0 && len(s.opts.RedirectDomains) != 0 && !slices.Contains(s.opts.RedirectDomains, urlParsed.Host) {
-		templ.Handler(web.Base("Oh noes!", web.ErrorPage("Redirect domain not allowed", s.opts.WebmasterEmail)), templ.WithStatus(http.StatusInternalServerError)).ServeHTTP(w, r)
-		return
-	} else if urlParsed.Host != r.URL.Host {
-		templ.Handler(web.Base("Oh noes!", web.ErrorPage("Redirect domain not allowed", s.opts.WebmasterEmail)), templ.WithStatus(http.StatusInternalServerError)).ServeHTTP(w, r)
+	if (len(urlParsed.Host) > 0 && len(s.opts.RedirectDomains) != 0 && !slices.Contains(s.opts.RedirectDomains, urlParsed.Host)) || urlParsed.Host != r.URL.Host {
+		s.respondWithError(w, r, "Redirect domain not allowed")
 		return
 	}
 
@@ -515,7 +312,7 @@ func (s *Server) PassChallenge(w http.ResponseWriter, r *http.Request) {
 	if err != nil {
 		s.ClearCookie(w)
 		lg.Debug("nonce doesn't parse", "err", err)
-		templ.Handler(web.Base("Oh noes!", web.ErrorPage("invalid nonce", s.opts.WebmasterEmail)), templ.WithStatus(http.StatusInternalServerError)).ServeHTTP(w, r)
+		s.respondWithError(w, r, "invalid nonce")
 		return
 	}
 
@@ -525,7 +322,7 @@ func (s *Server) PassChallenge(w http.ResponseWriter, r *http.Request) {
 	if subtle.ConstantTimeCompare([]byte(response), []byte(calculated)) != 1 {
 		s.ClearCookie(w)
 		lg.Debug("hash does not match", "got", response, "want", calculated)
-		templ.Handler(web.Base("Oh noes!", web.ErrorPage("invalid response", s.opts.WebmasterEmail)), templ.WithStatus(http.StatusForbidden)).ServeHTTP(w, r)
+		s.respondWithStatus(w, r, "invalid response", http.StatusForbidden)
 		failedValidations.Inc()
 		return
 	}
@@ -534,7 +331,7 @@ func (s *Server) PassChallenge(w http.ResponseWriter, r *http.Request) {
 	if !strings.HasPrefix(response, strings.Repeat("0", rule.Challenge.Difficulty)) {
 		s.ClearCookie(w)
 		lg.Debug("difficulty check failed", "response", response, "difficulty", rule.Challenge.Difficulty)
-		templ.Handler(web.Base("Oh noes!", web.ErrorPage("invalid response", s.opts.WebmasterEmail)), templ.WithStatus(http.StatusForbidden)).ServeHTTP(w, r)
+		s.respondWithStatus(w, r, "invalid response", http.StatusForbidden)
 		failedValidations.Inc()
 		return
 	}
@@ -551,20 +348,20 @@ func (s *Server) PassChallenge(w http.ResponseWriter, r *http.Request) {
 		"response":  response,
 		"iat":       time.Now().Unix(),
 		"nbf":       time.Now().Add(-1 * time.Minute).Unix(),
-		"exp":       time.Now().Add(24 * 7 * time.Hour).Unix(),
+		"exp":       time.Now().Add(s.opts.CookieExpiration).Unix(),
 	})
 	tokenString, err := token.SignedString(s.priv)
 	if err != nil {
 		lg.Error("failed to sign JWT", "err", err)
 		s.ClearCookie(w)
-		templ.Handler(web.Base("Oh noes!", web.ErrorPage("failed to sign JWT", s.opts.WebmasterEmail)), templ.WithStatus(http.StatusInternalServerError)).ServeHTTP(w, r)
+		s.respondWithError(w, r, "failed to sign JWT")
 		return
 	}
 
 	http.SetCookie(w, &http.Cookie{
 		Name:        anubis.CookieName,
 		Value:       tokenString,
-		Expires:     time.Now().Add(24 * 7 * time.Hour),
+		Expires:     time.Now().Add(s.opts.CookieExpiration),
 		SameSite:    http.SameSiteLaxMode,
 		Domain:      s.opts.CookieDomain,
 		Partitioned: s.opts.CookiePartitioned,
@@ -578,7 +375,7 @@ func (s *Server) PassChallenge(w http.ResponseWriter, r *http.Request) {
 
 func (s *Server) TestError(w http.ResponseWriter, r *http.Request) {
 	err := r.FormValue("err")
-	templ.Handler(web.Base("Oh noes!", web.ErrorPage(err, s.opts.WebmasterEmail)), templ.WithStatus(http.StatusInternalServerError)).ServeHTTP(w, r)
+	s.respondWithError(w, r, err)
 }
 
 func cr(name string, rule config.Rule) policy.CheckResult {

lib/anubis_test.go

@@ -8,6 +8,7 @@ import (
 	"os"
 	"strings"
 	"testing"
+	"time"
 
 	"github.com/TecharoHQ/anubis"
 	"github.com/TecharoHQ/anubis/data"
@@ -126,17 +127,18 @@ func TestCVE2025_24369(t *testing.T) {
 	}
 }
 
-func TestCookieSettings(t *testing.T) {
+func TestCookieCustomExpiration(t *testing.T) {
 	pol := loadPolicies(t, "")
 	pol.DefaultDifficulty = 0
+	ckieExpiration := 10 * time.Minute
 
 	srv := spawnAnubis(t, Options{
 		Next:   http.NewServeMux(),
 		Policy: pol,
 
-		CookieDomain:      "local.cetacean.club",
-		CookiePartitioned: true,
-		CookieName:        t.Name(),
+		CookieDomain:     "local.cetacean.club",
+		CookieName:       t.Name(),
+		CookieExpiration: ckieExpiration,
 	})
 
 	ts := httptest.NewServer(internal.RemoteXRealIP(true, "tcp", srv))
@@ -180,7 +182,99 @@ func TestCookieSettings(t *testing.T) {
 	q.Set("elapsedTime", fmt.Sprint(elapsedTime))
 	req.URL.RawQuery = q.Encode()
 
+	requestRecieveLowerBound := time.Now()
 	resp, err = cli.Do(req)
+	requestRecieveUpperBound := time.Now()
+	if err != nil {
+		t.Fatalf("can't do challenge passing")
+	}
+
+	if resp.StatusCode != http.StatusFound {
+		resp.Write(os.Stderr)
+		t.Errorf("wanted %d, got: %d", http.StatusFound, resp.StatusCode)
+	}
+
+	var ckie *http.Cookie
+	for _, cookie := range resp.Cookies() {
+		t.Logf("%#v", cookie)
+		if cookie.Name == anubis.CookieName {
+			ckie = cookie
+			break
+		}
+	}
+	if ckie == nil {
+		t.Errorf("Cookie %q not found", anubis.CookieName)
+		return
+	}
+
+	expirationLowerBound := requestRecieveLowerBound.Add(ckieExpiration)
+	expirationUpperBound := requestRecieveUpperBound.Add(ckieExpiration)
+	// Since the cookie expiration precision is only to the second due to the Unix() call, we can
+	// lower the level of expected precision.
+	if ckie.Expires.Unix() < expirationLowerBound.Unix() || ckie.Expires.Unix() > expirationUpperBound.Unix() {
+		t.Errorf("cookie expiration is not within the expected range. expected between: %v and %v. got: %v", expirationLowerBound, expirationUpperBound, ckie.Expires)
+		return
+	}
+}
+
+func TestCookieSettings(t *testing.T) {
+	pol := loadPolicies(t, "")
+	pol.DefaultDifficulty = 0
+
+	srv := spawnAnubis(t, Options{
+		Next:   http.NewServeMux(),
+		Policy: pol,
+
+		CookieDomain:      "local.cetacean.club",
+		CookiePartitioned: true,
+		CookieName:        t.Name(),
+		CookieExpiration:  anubis.CookieDefaultExpirationTime,
+	})
+
+	ts := httptest.NewServer(internal.RemoteXRealIP(true, "tcp", srv))
+	defer ts.Close()
+
+	cli := &http.Client{
+		CheckRedirect: func(req *http.Request, via []*http.Request) error {
+			return http.ErrUseLastResponse
+		},
+	}
+
+	resp, err := cli.Post(ts.URL+"/.within.website/x/cmd/anubis/api/make-challenge", "", nil)
+	if err != nil {
+		t.Fatalf("can't request challenge: %v", err)
+	}
+	defer resp.Body.Close()
+
+	var chall = struct {
+		Challenge string `json:"challenge"`
+	}{}
+	if err := json.NewDecoder(resp.Body).Decode(&chall); err != nil {
+		t.Fatalf("can't read challenge response body: %v", err)
+	}
+
+	nonce := 0
+	elapsedTime := 420
+	redir := "/"
+	calculated := ""
+	calcString := fmt.Sprintf("%s%d", chall.Challenge, nonce)
+	calculated = internal.SHA256sum(calcString)
+
+	req, err := http.NewRequest(http.MethodGet, ts.URL+"/.within.website/x/cmd/anubis/api/pass-challenge", nil)
+	if err != nil {
+		t.Fatalf("can't make request: %v", err)
+	}
+
+	q := req.URL.Query()
+	q.Set("response", calculated)
+	q.Set("nonce", fmt.Sprint(nonce))
+	q.Set("redir", redir)
+	q.Set("elapsedTime", fmt.Sprint(elapsedTime))
+	req.URL.RawQuery = q.Encode()
+
+	requestRecieveLowerBound := time.Now()
+	resp, err = cli.Do(req)
+	requestRecieveUpperBound := time.Now()
 	if err != nil {
 		t.Fatalf("can't do challenge passing")
 	}
@@ -207,6 +301,15 @@ func TestCookieSettings(t *testing.T) {
 		t.Errorf("cookie domain is wrong, wanted local.cetacean.club, got: %s", ckie.Domain)
 	}
 
+	expirationLowerBound := requestRecieveLowerBound.Add(anubis.CookieDefaultExpirationTime)
+	expirationUpperBound := requestRecieveUpperBound.Add(anubis.CookieDefaultExpirationTime)
+	// Since the cookie expiration precision is only to the second due to the Unix() call, we can
+	// lower the level of expected precision.
+	if ckie.Expires.Unix() < expirationLowerBound.Unix() || ckie.Expires.Unix() > expirationUpperBound.Unix() {
+		t.Errorf("cookie expiration is not within the expected range. expected between: %v and %v. got: %v", expirationLowerBound, expirationUpperBound, ckie.Expires)
+		return
+	}
+
 	if ckie.Partitioned != srv.opts.CookiePartitioned {
 		t.Errorf("wanted partitioned flag %v, got: %v", srv.opts.CookiePartitioned, ckie.Partitioned)
 	}
@@ -393,3 +496,48 @@ func TestBasePrefix(t *testing.T) {
 		})
 	}
 }
+
+func TestCustomStatusCodes(t *testing.T) {
+	h := http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		t.Log(r.UserAgent())
+		w.WriteHeader(http.StatusOK)
+		fmt.Fprintln(w, "OK")
+	})
+
+	statusMap := map[string]int{
+		"ALLOW":     200,
+		"CHALLENGE": 401,
+		"DENY":      403,
+	}
+
+	pol := loadPolicies(t, "./testdata/aggressive_403.yaml")
+	pol.DefaultDifficulty = 4
+
+	srv := spawnAnubis(t, Options{
+		Next:   h,
+		Policy: pol,
+	})
+
+	ts := httptest.NewServer(internal.RemoteXRealIP(true, "tcp", srv))
+	defer ts.Close()
+
+	for userAgent, statusCode := range statusMap {
+		t.Run(userAgent, func(t *testing.T) {
+			req, err := http.NewRequestWithContext(t.Context(), http.MethodGet, ts.URL, nil)
+			if err != nil {
+				t.Fatal(err)
+			}
+
+			req.Header.Set("User-Agent", userAgent)
+
+			resp, err := ts.Client().Do(req)
+			if err != nil {
+				t.Fatal(err)
+			}
+
+			if resp.StatusCode != statusCode {
+				t.Errorf("wanted status code %d but got: %d", statusCode, resp.StatusCode)
+			}
+		})
+	}
+}

lib/config.go

@@ -0,0 +1,140 @@
+package lib
+
+import (
+	"crypto/ed25519"
+	"crypto/rand"
+	"fmt"
+	"io"
+	"log/slog"
+	"net/http"
+	"os"
+	"strings"
+	"time"
+
+	"github.com/TecharoHQ/anubis"
+	"github.com/TecharoHQ/anubis/data"
+	"github.com/TecharoHQ/anubis/decaymap"
+	"github.com/TecharoHQ/anubis/internal"
+	"github.com/TecharoHQ/anubis/internal/dnsbl"
+	"github.com/TecharoHQ/anubis/internal/ogtags"
+	"github.com/TecharoHQ/anubis/lib/policy"
+	"github.com/TecharoHQ/anubis/web"
+	"github.com/TecharoHQ/anubis/xess"
+)
+
+type Options struct {
+	Next            http.Handler
+	Policy          *policy.ParsedConfig
+	RedirectDomains []string
+	ServeRobotsTXT  bool
+	PrivateKey      ed25519.PrivateKey
+
+	CookieExpiration  time.Duration
+	CookieDomain      string
+	CookieName        string
+	CookiePartitioned bool
+
+	OGPassthrough        bool
+	OGTimeToLive         time.Duration
+	OGCacheConsidersHost bool
+	Target               string
+
+	WebmasterEmail string
+	BasePrefix     string
+}
+
+func LoadPoliciesOrDefault(fname string, defaultDifficulty int) (*policy.ParsedConfig, error) {
+	var fin io.ReadCloser
+	var err error
+
+	if fname != "" {
+		fin, err = os.Open(fname)
+		if err != nil {
+			return nil, fmt.Errorf("can't parse policy file %s: %w", fname, err)
+		}
+	} else {
+		fname = "(data)/botPolicies.yaml"
+		fin, err = data.BotPolicies.Open("botPolicies.yaml")
+		if err != nil {
+			return nil, fmt.Errorf("[unexpected] can't parse builtin policy file %s: %w", fname, err)
+		}
+	}
+
+	defer func(fin io.ReadCloser) {
+		err := fin.Close()
+		if err != nil {
+			slog.Error("failed to close policy file", "file", fname, "err", err)
+		}
+	}(fin)
+
+	anubisPolicy, err := policy.ParseConfig(fin, fname, defaultDifficulty)
+
+	return anubisPolicy, err
+}
+
+func New(opts Options) (*Server, error) {
+	if opts.PrivateKey == nil {
+		slog.Debug("opts.PrivateKey not set, generating a new one")
+		_, priv, err := ed25519.GenerateKey(rand.Reader)
+		if err != nil {
+			return nil, fmt.Errorf("lib: can't generate private key: %v", err)
+		}
+		opts.PrivateKey = priv
+	}
+
+	anubis.BasePrefix = opts.BasePrefix
+
+	result := &Server{
+		next:       opts.Next,
+		priv:       opts.PrivateKey,
+		pub:        opts.PrivateKey.Public().(ed25519.PublicKey),
+		policy:     opts.Policy,
+		opts:       opts,
+		DNSBLCache: decaymap.New[string, dnsbl.DroneBLResponse](),
+		OGTags:     ogtags.NewOGTagCache(opts.Target, opts.OGPassthrough, opts.OGTimeToLive, opts.OGCacheConsidersHost),
+	}
+
+	mux := http.NewServeMux()
+	xess.Mount(mux)
+
+	// Helper to add global prefix
+	registerWithPrefix := func(pattern string, handler http.Handler, method string) {
+		if method != "" {
+			method = method + " " // methods must end with a space to register with them
+		}
+
+		// Ensure there's no double slash when concatenating BasePrefix and pattern
+		basePrefix := strings.TrimSuffix(anubis.BasePrefix, "/")
+		prefix := method + basePrefix
+
+		// If pattern doesn't start with a slash, add one
+		if !strings.HasPrefix(pattern, "/") {
+			pattern = "/" + pattern
+		}
+
+		mux.Handle(prefix+pattern, handler)
+	}
+
+	// Ensure there's no double slash when concatenating BasePrefix and StaticPath
+	stripPrefix := strings.TrimSuffix(anubis.BasePrefix, "/") + anubis.StaticPath
+	registerWithPrefix(anubis.StaticPath, internal.UnchangingCache(internal.NoBrowsing(http.StripPrefix(stripPrefix, http.FileServerFS(web.Static)))), "")
+
+	if opts.ServeRobotsTXT {
+		registerWithPrefix("/robots.txt", http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+			http.ServeFileFS(w, r, web.Static, "static/robots.txt")
+		}), "GET")
+		registerWithPrefix("/.well-known/robots.txt", http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+			http.ServeFileFS(w, r, web.Static, "static/robots.txt")
+		}), "GET")
+	}
+
+	registerWithPrefix(anubis.APIPrefix+"make-challenge", http.HandlerFunc(result.MakeChallenge), "POST")
+	registerWithPrefix(anubis.APIPrefix+"pass-challenge", http.HandlerFunc(result.PassChallenge), "GET")
+	registerWithPrefix(anubis.APIPrefix+"check", http.HandlerFunc(result.maybeReverseProxyHttpStatusOnly), "")
+	registerWithPrefix(anubis.APIPrefix+"test-error", http.HandlerFunc(result.TestError), "GET")
+	registerWithPrefix("/", http.HandlerFunc(result.maybeReverseProxyOrPage), "")
+
+	result.mux = mux
+
+	return result, nil
+}

lib/http.go

@@ -2,8 +2,14 @@ package lib
 
 import (
 	"net/http"
+	"slices"
 	"time"
 
+	"github.com/TecharoHQ/anubis/internal"
+	"github.com/TecharoHQ/anubis/lib/policy"
+	"github.com/TecharoHQ/anubis/web"
+	"github.com/a-h/templ"
+
 	"github.com/TecharoHQ/anubis"
 )
 
@@ -33,3 +39,82 @@ func (t UnixRoundTripper) RoundTrip(req *http.Request) (*http.Response, error) {
 	req.URL.Scheme = "http" // make http.Transport happy and avoid an infinite recursion
 	return t.Transport.RoundTrip(req)
 }
+
+func (s *Server) RenderIndex(w http.ResponseWriter, r *http.Request, rule *policy.Bot, returnHTTPStatusOnly bool) {
+	if returnHTTPStatusOnly {
+		w.WriteHeader(http.StatusUnauthorized)
+		w.Write([]byte("Authorization required"))
+		return
+	}
+
+	lg := internal.GetRequestLogger(r)
+
+	challenge := s.challengeFor(r, rule.Challenge.Difficulty)
+
+	var ogTags map[string]string = nil
+	if s.opts.OGPassthrough {
+		var err error
+		ogTags, err = s.OGTags.GetOGTags(r.URL, r.Host)
+		if err != nil {
+			lg.Error("failed to get OG tags", "err", err)
+		}
+	}
+
+	component, err := web.BaseWithChallengeAndOGTags("Making sure you're not a bot!", web.Index(), challenge, rule.Challenge, ogTags)
+	if err != nil {
+		lg.Error("render failed, please open an issue", "err", err) // This is likely a bug in the template. Should never be triggered as CI tests for this.
+		s.respondWithError(w, r, "Internal Server Error: please contact the administrator and ask them to look for the logs around \"RenderIndex\"")
+		return
+	}
+
+	handler := internal.NoStoreCache(templ.Handler(
+		component,
+		templ.WithStatus(s.opts.Policy.StatusCodes.Challenge),
+	))
+	handler.ServeHTTP(w, r)
+}
+
+func (s *Server) RenderBench(w http.ResponseWriter, r *http.Request) {
+	templ.Handler(
+		web.Base("Benchmarking Anubis!", web.Bench()),
+	).ServeHTTP(w, r)
+}
+
+func (s *Server) respondWithError(w http.ResponseWriter, r *http.Request, message string) {
+	s.respondWithStatus(w, r, message, http.StatusInternalServerError)
+}
+
+func (s *Server) respondWithStatus(w http.ResponseWriter, r *http.Request, msg string, status int) {
+	templ.Handler(web.Base("Oh noes!", web.ErrorPage(msg, s.opts.WebmasterEmail)), templ.WithStatus(status)).ServeHTTP(w, r)
+}
+
+func (s *Server) ServeHTTP(w http.ResponseWriter, r *http.Request) {
+	s.mux.ServeHTTP(w, r)
+}
+
+func (s *Server) ServeHTTPNext(w http.ResponseWriter, r *http.Request) {
+	if s.next == nil {
+		redir := r.FormValue("redir")
+		urlParsed, err := r.URL.Parse(redir)
+		if err != nil {
+			s.respondWithStatus(w, r, "Redirect URL not parseable", http.StatusBadRequest)
+			return
+		}
+
+		if (len(urlParsed.Host) > 0 && len(s.opts.RedirectDomains) != 0 && !slices.Contains(s.opts.RedirectDomains, urlParsed.Host)) || urlParsed.Host != r.URL.Host {
+			s.respondWithStatus(w, r, "Redirect domain not allowed", http.StatusBadRequest)
+			return
+		}
+
+		if redir != "" {
+			http.Redirect(w, r, redir, http.StatusFound)
+			return
+		}
+
+		templ.Handler(
+			web.Base("You are not a bot!", web.StaticHappy()),
+		).ServeHTTP(w, r)
+	} else {
+		s.next.ServeHTTP(w, r)
+	}
+}

lib/policy/config/config.go

@@ -6,6 +6,7 @@ import (
 	"io"
 	"io/fs"
 	"net"
+	"net/http"
 	"os"
 	"regexp"
 	"strings"
@@ -28,6 +29,7 @@ var (
 	ErrInvalidImportStatement            = errors.New("config.ImportStatement: invalid source file")
 	ErrCantSetBotAndImportValuesAtOnce   = errors.New("config.BotOrImport: can't set bot rules and import values at the same time")
 	ErrMustSetBotOrImportRules           = errors.New("config.BotOrImport: rule definition is invalid, you must set either bot rules or an import statement, not both")
+	ErrStatusCodeNotValid                = errors.New("config.StatusCode: status code not valid, must be between 100 and 599")
 )
 
 type Rule string
@@ -262,9 +264,33 @@ func (boi *BotOrImport) Valid() error {
 	return ErrMustSetBotOrImportRules
 }
 
+type StatusCodes struct {
+	Challenge int `json:"CHALLENGE"`
+	Deny      int `json:"DENY"`
+}
+
+func (sc StatusCodes) Valid() error {
+	var errs []error
+
+	if sc.Challenge == 0 || (sc.Challenge < 100 && sc.Challenge >= 599) {
+		errs = append(errs, fmt.Errorf("%w: challenge is %d", ErrStatusCodeNotValid, sc.Challenge))
+	}
+
+	if sc.Deny == 0 || (sc.Deny < 100 && sc.Deny >= 599) {
+		errs = append(errs, fmt.Errorf("%w: deny is %d", ErrStatusCodeNotValid, sc.Deny))
+	}
+
+	if len(errs) != 0 {
+		return fmt.Errorf("status codes not valid:\n%w", errors.Join(errs...))
+	}
+
+	return nil
+}
+
 type fileConfig struct {
-	Bots  []BotOrImport `json:"bots"`
-	DNSBL bool          `json:"dnsbl"`
+	Bots        []BotOrImport `json:"bots"`
+	DNSBL       bool          `json:"dnsbl"`
+	StatusCodes StatusCodes   `json:"status_codes"`
 }
 
 func (c fileConfig) Valid() error {
@@ -280,6 +306,10 @@ func (c fileConfig) Valid() error {
 		}
 	}
 
+	if err := c.StatusCodes.Valid(); err != nil {
+		errs = append(errs, err)
+	}
+
 	if len(errs) != 0 {
 		return fmt.Errorf("config is not valid:\n%w", errors.Join(errs...))
 	}
@@ -289,6 +319,10 @@ func (c fileConfig) Valid() error {
 
 func Load(fin io.Reader, fname string) (*Config, error) {
 	var c fileConfig
+	c.StatusCodes = StatusCodes{
+		Challenge: http.StatusOK,
+		Deny:      http.StatusOK,
+	}
 	if err := yaml.NewYAMLToJSONDecoder(fin).Decode(&c); err != nil {
 		return nil, fmt.Errorf("can't parse policy config YAML %s: %w", fname, err)
 	}
@@ -298,7 +332,8 @@ func Load(fin io.Reader, fname string) (*Config, error) {
 	}
 
 	result := &Config{
-		DNSBL: c.DNSBL,
+		DNSBL:       c.DNSBL,
+		StatusCodes: c.StatusCodes,
 	}
 
 	var validationErrs []error
@@ -331,8 +366,9 @@ func Load(fin io.Reader, fname string) (*Config, error) {
 }
 
 type Config struct {
-	Bots  []BotConfig
-	DNSBL bool
+	Bots        []BotConfig
+	DNSBL       bool
+	StatusCodes StatusCodes
 }
 
 func (c Config) Valid() error {

lib/policy/config/testdata/bad/status-codes-0.json

@@ -0,0 +1,13 @@
+{
+  "bots": [
+    {
+      "name": "everything",
+      "user_agent_regex": ".*",
+      "action": "DENY"
+    }
+  ],
+  "status_codes": {
+    "CHALLENGE": 0,
+    "DENY": 0
+  }
+}

lib/policy/config/testdata/bad/status-codes-0.yaml

@@ -0,0 +1,8 @@
+bots:
+- name: everything
+  user_agent_regex: .*
+  action: DENY
+
+status_codes:
+  CHALLENGE: 0
+  DENY: 0

lib/policy/config/testdata/good/old_xesite.json

@@ -0,0 +1,79 @@
+{
+  "bots": [
+    {
+      "name": "amazonbot",
+      "user_agent_regex": "Amazonbot",
+      "action": "DENY"
+    },
+    {
+      "name": "googlebot",
+      "user_agent_regex": "\\+http\\:\\/\\/www\\.google\\.com/bot\\.html",
+      "action": "ALLOW"
+    },
+    {
+      "name": "bingbot",
+      "user_agent_regex": "\\+http\\:\\/\\/www\\.bing\\.com/bingbot\\.htm",
+      "action": "ALLOW"
+    },
+    {
+      "name": "qwantbot",
+      "user_agent_regex": "\\+https\\:\\/\\/help\\.qwant\\.com/bot/",
+      "action": "ALLOW"
+    },
+    {
+      "name": "discordbot",
+      "user_agent_regex": "Discordbot/2\\.0; \\+https\\:\\/\\/discordapp\\.com",
+      "action": "ALLOW"
+    },
+    {
+      "name": "blueskybot",
+      "user_agent_regex": "Bluesky Cardyb",
+      "action": "ALLOW"
+    },
+    {
+      "name": "us-artificial-intelligence-scraper",
+      "user_agent_regex": "\\+https\\:\\/\\/github\\.com\\/US-Artificial-Intelligence\\/scraper",
+      "action": "DENY"
+    },
+    {
+      "name": "well-known",
+      "path_regex": "^/.well-known/.*$",
+      "action": "ALLOW"
+    },
+    {
+      "name": "favicon",
+      "path_regex": "^/favicon.ico$",
+      "action": "ALLOW"
+    },
+    {
+      "name": "robots-txt",
+      "path_regex": "^/robots.txt$",
+      "action": "ALLOW"
+    },
+    {
+      "name": "rss-readers",
+      "path_regex": ".*\\.(rss|xml|atom|json)$",
+      "action": "ALLOW"
+    },
+    {
+      "name": "lightpanda",
+      "user_agent_regex": "^Lightpanda/.*$",
+      "action": "DENY"
+    },
+    {
+      "name": "headless-chrome",
+      "user_agent_regex": "HeadlessChrome",
+      "action": "DENY"
+    },
+    {
+      "name": "headless-chromium",
+      "user_agent_regex": "HeadlessChromium",
+      "action": "DENY"
+    },
+    {
+      "name": "generic-browser",
+      "user_agent_regex": "Mozilla",
+      "action": "CHALLENGE"
+    }
+  ]
+}

lib/policy/config/testdata/good/status-codes-paranoid.json

@@ -0,0 +1,13 @@
+{
+  "bots": [
+    {
+      "name": "everything",
+      "user_agent_regex": ".*",
+      "action": "DENY"
+    }
+  ],
+  "status_codes": {
+    "CHALLENGE": 200,
+    "DENY": 200
+  }
+}

lib/policy/config/testdata/good/status-codes-paranoid.yaml

@@ -0,0 +1,8 @@
+bots:
+- name: everything
+  user_agent_regex: .*
+  action: DENY
+
+status_codes:
+  CHALLENGE: 200
+  DENY: 200

lib/policy/config/testdata/good/status-codes-rfc.json

@@ -0,0 +1,13 @@
+{
+  "bots": [
+    {
+      "name": "everything",
+      "user_agent_regex": ".*",
+      "action": "DENY"
+    }
+  ],
+  "status_codes": {
+    "CHALLENGE": 403,
+    "DENY": 403
+  }
+}

lib/policy/config/testdata/good/status-codes-rfc.yaml

@@ -0,0 +1,8 @@
+bots:
+- name: everything
+  user_agent_regex: .*
+  action: DENY
+
+status_codes:
+  CHALLENGE: 403
+  DENY: 403

lib/policy/policy.go

@@ -24,11 +24,13 @@ type ParsedConfig struct {
 	Bots              []Bot
 	DNSBL             bool
 	DefaultDifficulty int
+	StatusCodes       config.StatusCodes
 }
 
 func NewParsedConfig(orig *config.Config) *ParsedConfig {
 	return &ParsedConfig{
-		orig: orig,
+		orig:        orig,
+		StatusCodes: orig.StatusCodes,
 	}
 }
 

lib/testdata/aggressive_403.yaml

@@ -0,0 +1,12 @@
+bots:
+- name: deny
+  user_agent_regex: DENY
+  action: DENY
+
+- name: challenge
+  user_agent_regex: CHALLENGE
+  action: CHALLENGE
+
+status_codes:
+  CHALLENGE: 401
+  DENY: 403

package-lock.json

@@ -1,17 +1,17 @@
 {
   "name": "@techaro/anubis",
-  "version": "1.0.0-see-VERSION-file",
+  "version": "1.17.0",
   "lockfileVersion": 3,
   "requires": true,
   "packages": {
     "": {
       "name": "@techaro/anubis",
-      "version": "1.0.0-see-VERSION-file",
+      "version": "1.17.0",
       "license": "ISC",
       "devDependencies": {
         "cssnano": "^7.0.6",
         "cssnano-preset-advanced": "^7.0.6",
-        "esbuild": "^0.25.2",
+        "esbuild": "^0.25.3",
         "postcss-cli": "^11.0.1",
         "postcss-import": "^16.1.0",
         "postcss-import-url": "^7.2.0",
@@ -19,9 +19,9 @@
       }
     },
     "node_modules/@esbuild/aix-ppc64": {
-      "version": "0.25.2",
-      "resolved": "https://registry.npmjs.org/@esbuild/aix-ppc64/-/aix-ppc64-0.25.2.tgz",
-      "integrity": "sha512-wCIboOL2yXZym2cgm6mlA742s9QeJ8DjGVaL39dLN4rRwrOgOyYSnOaFPhKZGLb2ngj4EyfAFjsNJwPXZvseag==",
+      "version": "0.25.3",
+      "resolved": "https://registry.npmjs.org/@esbuild/aix-ppc64/-/aix-ppc64-0.25.3.tgz",
+      "integrity": "sha512-W8bFfPA8DowP8l//sxjJLSLkD8iEjMc7cBVyP+u4cEv9sM7mdUCkgsj+t0n/BWPFtv7WWCN5Yzj0N6FJNUUqBQ==",
       "cpu": [
         "ppc64"
       ],
@@ -36,9 +36,9 @@
       }
     },
     "node_modules/@esbuild/android-arm": {
-      "version": "0.25.2",
-      "resolved": "https://registry.npmjs.org/@esbuild/android-arm/-/android-arm-0.25.2.tgz",
-      "integrity": "sha512-NQhH7jFstVY5x8CKbcfa166GoV0EFkaPkCKBQkdPJFvo5u+nGXLEH/ooniLb3QI8Fk58YAx7nsPLozUWfCBOJA==",
+      "version": "0.25.3",
+      "resolved": "https://registry.npmjs.org/@esbuild/android-arm/-/android-arm-0.25.3.tgz",
+      "integrity": "sha512-PuwVXbnP87Tcff5I9ngV0lmiSu40xw1At6i3GsU77U7cjDDB4s0X2cyFuBiDa1SBk9DnvWwnGvVaGBqoFWPb7A==",
       "cpu": [
         "arm"
       ],
@@ -53,9 +53,9 @@
       }
     },
     "node_modules/@esbuild/android-arm64": {
-      "version": "0.25.2",
-      "resolved": "https://registry.npmjs.org/@esbuild/android-arm64/-/android-arm64-0.25.2.tgz",
-      "integrity": "sha512-5ZAX5xOmTligeBaeNEPnPaeEuah53Id2tX4c2CVP3JaROTH+j4fnfHCkr1PjXMd78hMst+TlkfKcW/DlTq0i4w==",
+      "version": "0.25.3",
+      "resolved": "https://registry.npmjs.org/@esbuild/android-arm64/-/android-arm64-0.25.3.tgz",
+      "integrity": "sha512-XelR6MzjlZuBM4f5z2IQHK6LkK34Cvv6Rj2EntER3lwCBFdg6h2lKbtRjpTTsdEjD/WSe1q8UyPBXP1x3i/wYQ==",
       "cpu": [
         "arm64"
       ],
@@ -70,9 +70,9 @@
       }
     },
     "node_modules/@esbuild/android-x64": {
-      "version": "0.25.2",
-      "resolved": "https://registry.npmjs.org/@esbuild/android-x64/-/android-x64-0.25.2.tgz",
-      "integrity": "sha512-Ffcx+nnma8Sge4jzddPHCZVRvIfQ0kMsUsCMcJRHkGJ1cDmhe4SsrYIjLUKn1xpHZybmOqCWwB0zQvsjdEHtkg==",
+      "version": "0.25.3",
+      "resolved": "https://registry.npmjs.org/@esbuild/android-x64/-/android-x64-0.25.3.tgz",
+      "integrity": "sha512-ogtTpYHT/g1GWS/zKM0cc/tIebFjm1F9Aw1boQ2Y0eUQ+J89d0jFY//s9ei9jVIlkYi8AfOjiixcLJSGNSOAdQ==",
       "cpu": [
         "x64"
       ],
@@ -87,9 +87,9 @@
       }
     },
     "node_modules/@esbuild/darwin-arm64": {
-      "version": "0.25.2",
-      "resolved": "https://registry.npmjs.org/@esbuild/darwin-arm64/-/darwin-arm64-0.25.2.tgz",
-      "integrity": "sha512-MpM6LUVTXAzOvN4KbjzU/q5smzryuoNjlriAIx+06RpecwCkL9JpenNzpKd2YMzLJFOdPqBpuub6eVRP5IgiSA==",
+      "version": "0.25.3",
+      "resolved": "https://registry.npmjs.org/@esbuild/darwin-arm64/-/darwin-arm64-0.25.3.tgz",
+      "integrity": "sha512-eESK5yfPNTqpAmDfFWNsOhmIOaQA59tAcF/EfYvo5/QWQCzXn5iUSOnqt3ra3UdzBv073ykTtmeLJZGt3HhA+w==",
       "cpu": [
         "arm64"
       ],
@@ -104,9 +104,9 @@
       }
     },
     "node_modules/@esbuild/darwin-x64": {
-      "version": "0.25.2",
-      "resolved": "https://registry.npmjs.org/@esbuild/darwin-x64/-/darwin-x64-0.25.2.tgz",
-      "integrity": "sha512-5eRPrTX7wFyuWe8FqEFPG2cU0+butQQVNcT4sVipqjLYQjjh8a8+vUTfgBKM88ObB85ahsnTwF7PSIt6PG+QkA==",
+      "version": "0.25.3",
+      "resolved": "https://registry.npmjs.org/@esbuild/darwin-x64/-/darwin-x64-0.25.3.tgz",
+      "integrity": "sha512-Kd8glo7sIZtwOLcPbW0yLpKmBNWMANZhrC1r6K++uDR2zyzb6AeOYtI6udbtabmQpFaxJ8uduXMAo1gs5ozz8A==",
       "cpu": [
         "x64"
       ],
@@ -121,9 +121,9 @@
       }
     },
     "node_modules/@esbuild/freebsd-arm64": {
-      "version": "0.25.2",
-      "resolved": "https://registry.npmjs.org/@esbuild/freebsd-arm64/-/freebsd-arm64-0.25.2.tgz",
-      "integrity": "sha512-mLwm4vXKiQ2UTSX4+ImyiPdiHjiZhIaE9QvC7sw0tZ6HoNMjYAqQpGyui5VRIi5sGd+uWq940gdCbY3VLvsO1w==",
+      "version": "0.25.3",
+      "resolved": "https://registry.npmjs.org/@esbuild/freebsd-arm64/-/freebsd-arm64-0.25.3.tgz",
+      "integrity": "sha512-EJiyS70BYybOBpJth3M0KLOus0n+RRMKTYzhYhFeMwp7e/RaajXvP+BWlmEXNk6uk+KAu46j/kaQzr6au+JcIw==",
       "cpu": [
         "arm64"
       ],
@@ -138,9 +138,9 @@
       }
     },
     "node_modules/@esbuild/freebsd-x64": {
-      "version": "0.25.2",
-      "resolved": "https://registry.npmjs.org/@esbuild/freebsd-x64/-/freebsd-x64-0.25.2.tgz",
-      "integrity": "sha512-6qyyn6TjayJSwGpm8J9QYYGQcRgc90nmfdUb0O7pp1s4lTY+9D0H9O02v5JqGApUyiHOtkz6+1hZNvNtEhbwRQ==",
+      "version": "0.25.3",
+      "resolved": "https://registry.npmjs.org/@esbuild/freebsd-x64/-/freebsd-x64-0.25.3.tgz",
+      "integrity": "sha512-Q+wSjaLpGxYf7zC0kL0nDlhsfuFkoN+EXrx2KSB33RhinWzejOd6AvgmP5JbkgXKmjhmpfgKZq24pneodYqE8Q==",
       "cpu": [
         "x64"
       ],
@@ -155,9 +155,9 @@
       }
     },
     "node_modules/@esbuild/linux-arm": {
-      "version": "0.25.2",
-      "resolved": "https://registry.npmjs.org/@esbuild/linux-arm/-/linux-arm-0.25.2.tgz",
-      "integrity": "sha512-UHBRgJcmjJv5oeQF8EpTRZs/1knq6loLxTsjc3nxO9eXAPDLcWW55flrMVc97qFPbmZP31ta1AZVUKQzKTzb0g==",
+      "version": "0.25.3",
+      "resolved": "https://registry.npmjs.org/@esbuild/linux-arm/-/linux-arm-0.25.3.tgz",
+      "integrity": "sha512-dUOVmAUzuHy2ZOKIHIKHCm58HKzFqd+puLaS424h6I85GlSDRZIA5ycBixb3mFgM0Jdh+ZOSB6KptX30DD8YOQ==",
       "cpu": [
         "arm"
       ],
@@ -172,9 +172,9 @@
       }
     },
     "node_modules/@esbuild/linux-arm64": {
-      "version": "0.25.2",
-      "resolved": "https://registry.npmjs.org/@esbuild/linux-arm64/-/linux-arm64-0.25.2.tgz",
-      "integrity": "sha512-gq/sjLsOyMT19I8obBISvhoYiZIAaGF8JpeXu1u8yPv8BE5HlWYobmlsfijFIZ9hIVGYkbdFhEqC0NvM4kNO0g==",
+      "version": "0.25.3",
+      "resolved": "https://registry.npmjs.org/@esbuild/linux-arm64/-/linux-arm64-0.25.3.tgz",
+      "integrity": "sha512-xCUgnNYhRD5bb1C1nqrDV1PfkwgbswTTBRbAd8aH5PhYzikdf/ddtsYyMXFfGSsb/6t6QaPSzxtbfAZr9uox4A==",
       "cpu": [
         "arm64"
       ],
@@ -189,9 +189,9 @@
       }
     },
     "node_modules/@esbuild/linux-ia32": {
-      "version": "0.25.2",
-      "resolved": "https://registry.npmjs.org/@esbuild/linux-ia32/-/linux-ia32-0.25.2.tgz",
-      "integrity": "sha512-bBYCv9obgW2cBP+2ZWfjYTU+f5cxRoGGQ5SeDbYdFCAZpYWrfjjfYwvUpP8MlKbP0nwZ5gyOU/0aUzZ5HWPuvQ==",
+      "version": "0.25.3",
+      "resolved": "https://registry.npmjs.org/@esbuild/linux-ia32/-/linux-ia32-0.25.3.tgz",
+      "integrity": "sha512-yplPOpczHOO4jTYKmuYuANI3WhvIPSVANGcNUeMlxH4twz/TeXuzEP41tGKNGWJjuMhotpGabeFYGAOU2ummBw==",
       "cpu": [
         "ia32"
       ],
@@ -206,9 +206,9 @@
       }
     },
     "node_modules/@esbuild/linux-loong64": {
-      "version": "0.25.2",
-      "resolved": "https://registry.npmjs.org/@esbuild/linux-loong64/-/linux-loong64-0.25.2.tgz",
-      "integrity": "sha512-SHNGiKtvnU2dBlM5D8CXRFdd+6etgZ9dXfaPCeJtz+37PIUlixvlIhI23L5khKXs3DIzAn9V8v+qb1TRKrgT5w==",
+      "version": "0.25.3",
+      "resolved": "https://registry.npmjs.org/@esbuild/linux-loong64/-/linux-loong64-0.25.3.tgz",
+      "integrity": "sha512-P4BLP5/fjyihmXCELRGrLd793q/lBtKMQl8ARGpDxgzgIKJDRJ/u4r1A/HgpBpKpKZelGct2PGI4T+axcedf6g==",
       "cpu": [
         "loong64"
       ],
@@ -223,9 +223,9 @@
       }
     },
     "node_modules/@esbuild/linux-mips64el": {
-      "version": "0.25.2",
-      "resolved": "https://registry.npmjs.org/@esbuild/linux-mips64el/-/linux-mips64el-0.25.2.tgz",
-      "integrity": "sha512-hDDRlzE6rPeoj+5fsADqdUZl1OzqDYow4TB4Y/3PlKBD0ph1e6uPHzIQcv2Z65u2K0kpeByIyAjCmjn1hJgG0Q==",
+      "version": "0.25.3",
+      "resolved": "https://registry.npmjs.org/@esbuild/linux-mips64el/-/linux-mips64el-0.25.3.tgz",
+      "integrity": "sha512-eRAOV2ODpu6P5divMEMa26RRqb2yUoYsuQQOuFUexUoQndm4MdpXXDBbUoKIc0iPa4aCO7gIhtnYomkn2x+bag==",
       "cpu": [
         "mips64el"
       ],
@@ -240,9 +240,9 @@
       }
     },
     "node_modules/@esbuild/linux-ppc64": {
-      "version": "0.25.2",
-      "resolved": "https://registry.npmjs.org/@esbuild/linux-ppc64/-/linux-ppc64-0.25.2.tgz",
-      "integrity": "sha512-tsHu2RRSWzipmUi9UBDEzc0nLc4HtpZEI5Ba+Omms5456x5WaNuiG3u7xh5AO6sipnJ9r4cRWQB2tUjPyIkc6g==",
+      "version": "0.25.3",
+      "resolved": "https://registry.npmjs.org/@esbuild/linux-ppc64/-/linux-ppc64-0.25.3.tgz",
+      "integrity": "sha512-ZC4jV2p7VbzTlnl8nZKLcBkfzIf4Yad1SJM4ZMKYnJqZFD4rTI+pBG65u8ev4jk3/MPwY9DvGn50wi3uhdaghg==",
       "cpu": [
         "ppc64"
       ],
@@ -257,9 +257,9 @@
       }
     },
     "node_modules/@esbuild/linux-riscv64": {
-      "version": "0.25.2",
-      "resolved": "https://registry.npmjs.org/@esbuild/linux-riscv64/-/linux-riscv64-0.25.2.tgz",
-      "integrity": "sha512-k4LtpgV7NJQOml/10uPU0s4SAXGnowi5qBSjaLWMojNCUICNu7TshqHLAEbkBdAszL5TabfvQ48kK84hyFzjnw==",
+      "version": "0.25.3",
+      "resolved": "https://registry.npmjs.org/@esbuild/linux-riscv64/-/linux-riscv64-0.25.3.tgz",
+      "integrity": "sha512-LDDODcFzNtECTrUUbVCs6j9/bDVqy7DDRsuIXJg6so+mFksgwG7ZVnTruYi5V+z3eE5y+BJZw7VvUadkbfg7QA==",
       "cpu": [
         "riscv64"
       ],
@@ -274,9 +274,9 @@
       }
     },
     "node_modules/@esbuild/linux-s390x": {
-      "version": "0.25.2",
-      "resolved": "https://registry.npmjs.org/@esbuild/linux-s390x/-/linux-s390x-0.25.2.tgz",
-      "integrity": "sha512-GRa4IshOdvKY7M/rDpRR3gkiTNp34M0eLTaC1a08gNrh4u488aPhuZOCpkF6+2wl3zAN7L7XIpOFBhnaE3/Q8Q==",
+      "version": "0.25.3",
+      "resolved": "https://registry.npmjs.org/@esbuild/linux-s390x/-/linux-s390x-0.25.3.tgz",
+      "integrity": "sha512-s+w/NOY2k0yC2p9SLen+ymflgcpRkvwwa02fqmAwhBRI3SC12uiS10edHHXlVWwfAagYSY5UpmT/zISXPMW3tQ==",
       "cpu": [
         "s390x"
       ],
@@ -291,9 +291,9 @@
       }
     },
     "node_modules/@esbuild/linux-x64": {
-      "version": "0.25.2",
-      "resolved": "https://registry.npmjs.org/@esbuild/linux-x64/-/linux-x64-0.25.2.tgz",
-      "integrity": "sha512-QInHERlqpTTZ4FRB0fROQWXcYRD64lAoiegezDunLpalZMjcUcld3YzZmVJ2H/Cp0wJRZ8Xtjtj0cEHhYc/uUg==",
+      "version": "0.25.3",
+      "resolved": "https://registry.npmjs.org/@esbuild/linux-x64/-/linux-x64-0.25.3.tgz",
+      "integrity": "sha512-nQHDz4pXjSDC6UfOE1Fw9Q8d6GCAd9KdvMZpfVGWSJztYCarRgSDfOVBY5xwhQXseiyxapkiSJi/5/ja8mRFFA==",
       "cpu": [
         "x64"
       ],
@@ -308,9 +308,9 @@
       }
     },
     "node_modules/@esbuild/netbsd-arm64": {
-      "version": "0.25.2",
-      "resolved": "https://registry.npmjs.org/@esbuild/netbsd-arm64/-/netbsd-arm64-0.25.2.tgz",
-      "integrity": "sha512-talAIBoY5M8vHc6EeI2WW9d/CkiO9MQJ0IOWX8hrLhxGbro/vBXJvaQXefW2cP0z0nQVTdQ/eNyGFV1GSKrxfw==",
+      "version": "0.25.3",
+      "resolved": "https://registry.npmjs.org/@esbuild/netbsd-arm64/-/netbsd-arm64-0.25.3.tgz",
+      "integrity": "sha512-1QaLtOWq0mzK6tzzp0jRN3eccmN3hezey7mhLnzC6oNlJoUJz4nym5ZD7mDnS/LZQgkrhEbEiTn515lPeLpgWA==",
       "cpu": [
         "arm64"
       ],
@@ -325,9 +325,9 @@
       }
     },
     "node_modules/@esbuild/netbsd-x64": {
-      "version": "0.25.2",
-      "resolved": "https://registry.npmjs.org/@esbuild/netbsd-x64/-/netbsd-x64-0.25.2.tgz",
-      "integrity": "sha512-voZT9Z+tpOxrvfKFyfDYPc4DO4rk06qamv1a/fkuzHpiVBMOhpjK+vBmWM8J1eiB3OLSMFYNaOaBNLXGChf5tg==",
+      "version": "0.25.3",
+      "resolved": "https://registry.npmjs.org/@esbuild/netbsd-x64/-/netbsd-x64-0.25.3.tgz",
+      "integrity": "sha512-i5Hm68HXHdgv8wkrt+10Bc50zM0/eonPb/a/OFVfB6Qvpiirco5gBA5bz7S2SHuU+Y4LWn/zehzNX14Sp4r27g==",
       "cpu": [
         "x64"
       ],
@@ -342,9 +342,9 @@
       }
     },
     "node_modules/@esbuild/openbsd-arm64": {
-      "version": "0.25.2",
-      "resolved": "https://registry.npmjs.org/@esbuild/openbsd-arm64/-/openbsd-arm64-0.25.2.tgz",
-      "integrity": "sha512-dcXYOC6NXOqcykeDlwId9kB6OkPUxOEqU+rkrYVqJbK2hagWOMrsTGsMr8+rW02M+d5Op5NNlgMmjzecaRf7Tg==",
+      "version": "0.25.3",
+      "resolved": "https://registry.npmjs.org/@esbuild/openbsd-arm64/-/openbsd-arm64-0.25.3.tgz",
+      "integrity": "sha512-zGAVApJEYTbOC6H/3QBr2mq3upG/LBEXr85/pTtKiv2IXcgKV0RT0QA/hSXZqSvLEpXeIxah7LczB4lkiYhTAQ==",
       "cpu": [
         "arm64"
       ],
@@ -359,9 +359,9 @@
       }
     },
     "node_modules/@esbuild/openbsd-x64": {
-      "version": "0.25.2",
-      "resolved": "https://registry.npmjs.org/@esbuild/openbsd-x64/-/openbsd-x64-0.25.2.tgz",
-      "integrity": "sha512-t/TkWwahkH0Tsgoq1Ju7QfgGhArkGLkF1uYz8nQS/PPFlXbP5YgRpqQR3ARRiC2iXoLTWFxc6DJMSK10dVXluw==",
+      "version": "0.25.3",
+      "resolved": "https://registry.npmjs.org/@esbuild/openbsd-x64/-/openbsd-x64-0.25.3.tgz",
+      "integrity": "sha512-fpqctI45NnCIDKBH5AXQBsD0NDPbEFczK98hk/aa6HJxbl+UtLkJV2+Bvy5hLSLk3LHmqt0NTkKNso1A9y1a4w==",
       "cpu": [
         "x64"
       ],
@@ -376,9 +376,9 @@
       }
     },
     "node_modules/@esbuild/sunos-x64": {
-      "version": "0.25.2",
-      "resolved": "https://registry.npmjs.org/@esbuild/sunos-x64/-/sunos-x64-0.25.2.tgz",
-      "integrity": "sha512-cfZH1co2+imVdWCjd+D1gf9NjkchVhhdpgb1q5y6Hcv9TP6Zi9ZG/beI3ig8TvwT9lH9dlxLq5MQBBgwuj4xvA==",
+      "version": "0.25.3",
+      "resolved": "https://registry.npmjs.org/@esbuild/sunos-x64/-/sunos-x64-0.25.3.tgz",
+      "integrity": "sha512-ROJhm7d8bk9dMCUZjkS8fgzsPAZEjtRJqCAmVgB0gMrvG7hfmPmz9k1rwO4jSiblFjYmNvbECL9uhaPzONMfgA==",
       "cpu": [
         "x64"
       ],
@@ -393,9 +393,9 @@
       }
     },
     "node_modules/@esbuild/win32-arm64": {
-      "version": "0.25.2",
-      "resolved": "https://registry.npmjs.org/@esbuild/win32-arm64/-/win32-arm64-0.25.2.tgz",
-      "integrity": "sha512-7Loyjh+D/Nx/sOTzV8vfbB3GJuHdOQyrOryFdZvPHLf42Tk9ivBU5Aedi7iyX+x6rbn2Mh68T4qq1SDqJBQO5Q==",
+      "version": "0.25.3",
+      "resolved": "https://registry.npmjs.org/@esbuild/win32-arm64/-/win32-arm64-0.25.3.tgz",
+      "integrity": "sha512-YWcow8peiHpNBiIXHwaswPnAXLsLVygFwCB3A7Bh5jRkIBFWHGmNQ48AlX4xDvQNoMZlPYzjVOQDYEzWCqufMQ==",
       "cpu": [
         "arm64"
       ],
@@ -410,9 +410,9 @@
       }
     },
     "node_modules/@esbuild/win32-ia32": {
-      "version": "0.25.2",
-      "resolved": "https://registry.npmjs.org/@esbuild/win32-ia32/-/win32-ia32-0.25.2.tgz",
-      "integrity": "sha512-WRJgsz9un0nqZJ4MfhabxaD9Ft8KioqU3JMinOTvobbX6MOSUigSBlogP8QB3uxpJDsFS6yN+3FDBdqE5lg9kg==",
+      "version": "0.25.3",
+      "resolved": "https://registry.npmjs.org/@esbuild/win32-ia32/-/win32-ia32-0.25.3.tgz",
+      "integrity": "sha512-qspTZOIGoXVS4DpNqUYUs9UxVb04khS1Degaw/MnfMe7goQ3lTfQ13Vw4qY/Nj0979BGvMRpAYbs/BAxEvU8ew==",
       "cpu": [
         "ia32"
       ],
@@ -427,9 +427,9 @@
       }
     },
     "node_modules/@esbuild/win32-x64": {
-      "version": "0.25.2",
-      "resolved": "https://registry.npmjs.org/@esbuild/win32-x64/-/win32-x64-0.25.2.tgz",
-      "integrity": "sha512-kM3HKb16VIXZyIeVrM1ygYmZBKybX8N4p754bw390wGO3Tf2j4L2/WYL+4suWujpgf6GBYs3jv7TyUivdd05JA==",
+      "version": "0.25.3",
+      "resolved": "https://registry.npmjs.org/@esbuild/win32-x64/-/win32-x64-0.25.3.tgz",
+      "integrity": "sha512-ICgUR+kPimx0vvRzf+N/7L7tVSQeE3BYY+NhHRHXS1kBuPO7z2+7ea2HbhDyZdTephgvNvKrlDDKUexuCVBVvg==",
       "cpu": [
         "x64"
       ],
@@ -1044,9 +1044,9 @@
       }
     },
     "node_modules/esbuild": {
-      "version": "0.25.2",
-      "resolved": "https://registry.npmjs.org/esbuild/-/esbuild-0.25.2.tgz",
-      "integrity": "sha512-16854zccKPnC+toMywC+uKNeYSv+/eXkevRAfwRD/G9Cleq66m8XFIrigkbvauLLlCfDL45Q2cWegSg53gGBnQ==",
+      "version": "0.25.3",
+      "resolved": "https://registry.npmjs.org/esbuild/-/esbuild-0.25.3.tgz",
+      "integrity": "sha512-qKA6Pvai73+M2FtftpNKRxJ78GIjmFXFxd/1DVBqGo/qNhLSfv+G12n9pNoWdytJC8U00TrViOwpjT0zgqQS8Q==",
       "dev": true,
       "hasInstallScript": true,
       "license": "MIT",
@@ -1057,31 +1057,31 @@
         "node": ">=18"
       },
       "optionalDependencies": {
-        "@esbuild/aix-ppc64": "0.25.2",
-        "@esbuild/android-arm": "0.25.2",
-        "@esbuild/android-arm64": "0.25.2",
-        "@esbuild/android-x64": "0.25.2",
-        "@esbuild/darwin-arm64": "0.25.2",
-        "@esbuild/darwin-x64": "0.25.2",
-        "@esbuild/freebsd-arm64": "0.25.2",
-        "@esbuild/freebsd-x64": "0.25.2",
-        "@esbuild/linux-arm": "0.25.2",
-        "@esbuild/linux-arm64": "0.25.2",
-        "@esbuild/linux-ia32": "0.25.2",
-        "@esbuild/linux-loong64": "0.25.2",
-        "@esbuild/linux-mips64el": "0.25.2",
-        "@esbuild/linux-ppc64": "0.25.2",
-        "@esbuild/linux-riscv64": "0.25.2",
-        "@esbuild/linux-s390x": "0.25.2",
-        "@esbuild/linux-x64": "0.25.2",
-        "@esbuild/netbsd-arm64": "0.25.2",
-        "@esbuild/netbsd-x64": "0.25.2",
-        "@esbuild/openbsd-arm64": "0.25.2",
-        "@esbuild/openbsd-x64": "0.25.2",
-        "@esbuild/sunos-x64": "0.25.2",
-        "@esbuild/win32-arm64": "0.25.2",
-        "@esbuild/win32-ia32": "0.25.2",
-        "@esbuild/win32-x64": "0.25.2"
+        "@esbuild/aix-ppc64": "0.25.3",
+        "@esbuild/android-arm": "0.25.3",
+        "@esbuild/android-arm64": "0.25.3",
+        "@esbuild/android-x64": "0.25.3",
+        "@esbuild/darwin-arm64": "0.25.3",
+        "@esbuild/darwin-x64": "0.25.3",
+        "@esbuild/freebsd-arm64": "0.25.3",
+        "@esbuild/freebsd-x64": "0.25.3",
+        "@esbuild/linux-arm": "0.25.3",
+        "@esbuild/linux-arm64": "0.25.3",
+        "@esbuild/linux-ia32": "0.25.3",
+        "@esbuild/linux-loong64": "0.25.3",
+        "@esbuild/linux-mips64el": "0.25.3",
+        "@esbuild/linux-ppc64": "0.25.3",
+        "@esbuild/linux-riscv64": "0.25.3",
+        "@esbuild/linux-s390x": "0.25.3",
+        "@esbuild/linux-x64": "0.25.3",
+        "@esbuild/netbsd-arm64": "0.25.3",
+        "@esbuild/netbsd-x64": "0.25.3",
+        "@esbuild/openbsd-arm64": "0.25.3",
+        "@esbuild/openbsd-x64": "0.25.3",
+        "@esbuild/sunos-x64": "0.25.3",
+        "@esbuild/win32-arm64": "0.25.3",
+        "@esbuild/win32-ia32": "0.25.3",
+        "@esbuild/win32-x64": "0.25.3"
       }
     },
     "node_modules/escalade": {

package.json

@@ -1,6 +1,6 @@
 {
   "name": "@techaro/anubis",
-  "version": "1.0.0-see-VERSION-file",
+  "version": "1.17.1",
   "description": "",
   "main": "index.js",
   "scripts": {
@@ -17,7 +17,7 @@
   "devDependencies": {
     "cssnano": "^7.0.6",
     "cssnano-preset-advanced": "^7.0.6",
-    "esbuild": "^0.25.2",
+    "esbuild": "^0.25.3",
     "postcss-cli": "^11.0.1",
     "postcss-import": "^16.1.0",
     "postcss-import-url": "^7.2.0",

test/anubis_configs/aggressive_403.yaml

@@ -0,0 +1,12 @@
+bots:
+- name: deny
+  user_agent_regex: DENY
+  action: DENY
+
+- name: challenge
+  user_agent_regex: CHALLENGE
+  action: CHALLENGE
+
+status_codes:
+  CHALLENGE: 401
+  DENY: 403

test/unix-socket-xff/start.sh

@@ -37,6 +37,7 @@ go run ../cmd/unixhttpd &
 go tool anubis \
   --bind=./anubis.sock \
   --bind-network=unix \
+  --policy-fname=../anubis_configs/aggressive_403.yaml \
   --target=unix://$(pwd)/unixhttpd.sock &
 
 # A simple TLS terminator that forwards to Anubis, which will forward to

test/unix-socket-xff/test.mjs

@@ -0,0 +1,30 @@
+async function testWithUserAgent(userAgent) {
+  const statusCode =
+    await fetch("https://relayd.local.cetacean.club:3004/reqmeta", {
+      headers: {
+        "User-Agent": userAgent,
+      }
+    })
+      .then(resp => resp.status);
+  return statusCode;
+}
+
+const codes = {
+  allow: await testWithUserAgent("ALLOW"),
+  challenge: await testWithUserAgent("CHALLENGE"),
+  deny: await testWithUserAgent("DENY")
+}
+
+const expected = {
+  allow: 200,
+  challenge: 401,
+  deny: 403,
+};
+
+console.log("ALLOW:    ", codes.allow);
+console.log("CHALLENGE:", codes.challenge);
+console.log("DENY:     ", codes.deny);
+
+if (JSON.stringify(codes) !== JSON.stringify(expected)) {
+  throw new Error(`wanted ${JSON.stringify(expected)}, got: ${JSON.stringify(codes)}`);
+}

web/static/robots.txt

@@ -1,9 +1,11 @@
 User-agent: AI2Bot
 User-agent: Ai2Bot-Dolma
+User-agent: aiHitBot
 User-agent: Amazonbot
 User-agent: anthropic-ai
 User-agent: Applebot
 User-agent: Applebot-Extended
+User-agent: Brightbot 1.0
 User-agent: Bytespider
 User-agent: CCBot
 User-agent: ChatGPT-User
@@ -11,9 +13,13 @@ User-agent: Claude-Web
 User-agent: ClaudeBot
 User-agent: cohere-ai
 User-agent: cohere-training-data-crawler
+User-agent: Cotoyogi
+User-agent: Crawlspace
 User-agent: Diffbot
 User-agent: DuckAssistBot
 User-agent: FacebookBot
+User-agent: Factset_spyderbot
+User-agent: FirecrawlAgent
 User-agent: FriendlyCrawler
 User-agent: Google-Extended
 User-agent: GoogleOther
@@ -24,19 +30,27 @@ User-agent: iaskspider/2.0
 User-agent: ICC-Crawler
 User-agent: ImagesiftBot
 User-agent: img2dataset
+User-agent: imgproxy
 User-agent: ISSCyberRiskCrawler
 User-agent: Kangaroo Bot
+User-agent: meta-externalagent
 User-agent: Meta-ExternalAgent
+User-agent: meta-externalfetcher
 User-agent: Meta-ExternalFetcher
+User-agent: NovaAct
 User-agent: OAI-SearchBot
 User-agent: omgili
 User-agent: omgilibot
+User-agent: Operator
 User-agent: PanguBot
+User-agent: Perplexity-User
 User-agent: PerplexityBot
 User-agent: PetalBot
 User-agent: Scrapy
-User-agent: SemrushBot
+User-agent: SemrushBot-OCOB
+User-agent: SemrushBot-SWA
 User-agent: Sidetrade indexer bot
+User-agent: TikTokSpider
 User-agent: Timpibot
 User-agent: VelenPublicWebCrawler
 User-agent: Webzio-Extended
@@ -44,4 +58,4 @@ User-agent: YouBot
 Disallow: /
 
 User-agent: *
-Disallow: /
+Disallow: /