fix: SERVE_ROBOTS_TXT works again

Signed-off-by: Xe Iaso <me@xeiaso.net>
chore: tag v1.23.0
2026-05-24 22:56:13 +00:00 · 2025-10-31 08:52:43 -04:00 · 2025-10-29 20:38:34 -04:00 · 2025-10-29 16:07:31 -04:00
10 changed files with 77 additions and 4 deletions
@@ -23,6 +23,7 @@ jobs:
          - i18n
          - palemoon/amd64
          #- palemoon/i386
+          - robots_txt
    runs-on: ubuntu-latest
    steps:
      - name: Checkout code
@@ -1 +1 @@
-1.23.0-pre2
+1.23.0
@@ -13,6 +13,8 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0

 <!-- This changes the project to: -->

+- Fix `SERVE_ROBOTS_TXT` setting file after the double slash fix broke it.
+
 ## v1.23.0: Lyse Hext

 - Add default tencent cloud DENY rule.
@@ -345,6 +345,15 @@ func (s *Server) ServeHTTP(w http.ResponseWriter, r *http.Request) {
 		return
 	}

+	// Forward robots.txt requests to mux when ServeRobotsTXT is enabled
+	if s.opts.ServeRobotsTXT {
+		path := strings.TrimPrefix(r.URL.Path, anubis.BasePrefix)
+		if path == "/robots.txt" || path == "/.well-known/robots.txt" {
+			s.mux.ServeHTTP(w, r)
+			return
+		}
+	}
+
 	s.maybeReverseProxyOrPage(w, r)
 }

@@ -1,12 +1,12 @@
 {
  "name": "@techaro/anubis",
-  "version": "1.23.0-pre2",
+  "version": "1.23.0",
  "lockfileVersion": 3,
  "requires": true,
  "packages": {
    "": {
      "name": "@techaro/anubis",
-      "version": "1.23.0-pre2",
+      "version": "1.23.0",
      "license": "ISC",
      "dependencies": {
        "@aws-crypto/sha256-js": "^5.2.0",
@@ -1,6 +1,6 @@
 {
  "name": "@techaro/anubis",
-  "version": "1.23.0-pre2",
+  "version": "1.23.0",
  "description": "",
  "main": "index.js",
  "scripts": {
@@ -0,0 +1,8 @@
+bots:
+  - name: challenge
+    user_agent_regex: CHALLENGE
+    action: CHALLENGE
+
+status_codes:
+  CHALLENGE: 200
+  DENY: 403
@@ -0,0 +1,27 @@
+async function getRobotsTxt() {
+  return fetch("http://localhost:8923/robots.txt", {
+    headers: {
+      "Accept-Language": "en",
+      "User-Agent": "Mozilla/5.0",
+    }
+  })
+    .then(resp => {
+      if (resp.status !== 200) {
+        throw new Error(`wanted status 200, got status: ${resp.status}`);
+      }
+      return resp;
+    })
+    .then(resp => resp.text());
+}
+
+(async () => {
+  const page = await getRobotsTxt();
+
+  if (page.includes(`<html>`)) {
+    console.log(page)
+    throw new Error("serve robots.txt smoke test failed");
+  }
+
+  console.log("serve-robots-txt serves robots.txt");
+  process.exit(0);
+})();
@@ -0,0 +1,24 @@
+#!/usr/bin/env bash
+
+set -euo pipefail
+
+function cleanup() {
+	pkill -P $$
+}
+
+trap cleanup EXIT SIGINT
+
+# Build static assets
+(cd ../.. && npm ci && npm run assets)
+
+go tool anubis --help 2>/dev/null || :
+
+go run ../cmd/unixhttpd &
+
+go tool anubis \
+	--policy-fname ./anubis.yaml \
+	--use-remote-address \
+	--serve-robots-txt \
+	--target=unix://$(pwd)/unixhttpd.sock &
+
+backoff-retry node ./test.mjs
@@ -0,0 +1,2 @@
+*
+!.gitignore