v1.17.0: Asahi sas Brutus

Signed-off-by: Xe Iaso <me@xeiaso.net>

.air.toml

@@ -0,0 +1,12 @@
+root = "."
+tmp_dir = "var"
+
+[build]
+cmd = "go build -o ./var/main ./cmd/anubis"
+bin = "./var/main"
+args = ["--use-remote-address"]
+exclude_dir = ["var", "vendor", "docs", "node_modules"]
+
+[logger]
+time = true
+# to change flags at runtime, prepend with -- e.g. $ air -- --target http://localhost:3000 --difficulty 20 --use-remote-address

.gitattributes

@@ -0,0 +1 @@
+web/index_templ.go linguist-generated

.github/FUNDING.yml

@@ -1 +1,2 @@
-patreon: cadey
+patreon: cadey
+github: xe

.github/PULL_REQUEST_TEMPLATE.md

@@ -0,0 +1,11 @@
+<!--
+delete me and describe your change here, give enough context for a maintainer to understand what and why
+
+See https://anubis.techaro.lol/docs/developer/code-quality for more information
+-->
+
+Checklist:
+
+- [ ] Added a description of the changes to the `[Unreleased]` section of docs/docs/CHANGELOG.md
+- [ ] Added test cases to [the relevant parts of the codebase](https://anubis.techaro.lol/docs/developer/code-quality)
+- [ ] Ran integration tests `npm run test:integration` (unsupported on Windows, please use WSL)

.github/dependabot.yml

@@ -0,0 +1,28 @@
+version: 2
+updates:
+  - package-ecosystem: github-actions
+    directory: /
+    schedule:
+      interval: weekly
+    groups:
+      github-actions:
+        patterns:
+          - "*"
+
+  - package-ecosystem: gomod
+    directory: /
+    schedule:
+      interval: weekly
+    groups:
+      gomod:
+        patterns:
+          - "*"
+
+  - package-ecosystem: npm
+    directory: /
+    schedule:
+      interval: weekly
+    groups:
+      npm:
+        patterns:
+          - "*"

.github/workflows/docker-pr.yml

@@ -0,0 +1,68 @@
+name: Docker image builds (pull requests)
+
+on:
+  pull_request:
+    branches: [ "main" ]
+
+env:
+  DOCKER_METADATA_SET_OUTPUT_ENV: "true"
+
+permissions:
+  contents: read
+
+jobs:
+  build:
+    runs-on: ubuntu-24.04
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
+        with:
+          fetch-tags: true
+          fetch-depth: 0
+          persist-credentials: false
+
+      - name: Set up Homebrew
+        uses: Homebrew/actions/setup-homebrew@master
+
+      - name: Setup Homebrew cellar cache
+        uses: actions/cache@5a3ec84eff668545956fd18022155c47e93e2684 # v4.2.3
+        with:
+          path: |
+            /home/linuxbrew/.linuxbrew/Cellar
+            /home/linuxbrew/.linuxbrew/bin
+            /home/linuxbrew/.linuxbrew/etc
+            /home/linuxbrew/.linuxbrew/include
+            /home/linuxbrew/.linuxbrew/lib
+            /home/linuxbrew/.linuxbrew/opt
+            /home/linuxbrew/.linuxbrew/sbin
+            /home/linuxbrew/.linuxbrew/share
+            /home/linuxbrew/.linuxbrew/var
+          key: ${{ runner.os }}-go-homebrew-cellar-${{ hashFiles('go.sum') }}
+          restore-keys: |
+            ${{ runner.os }}-go-homebrew-cellar-
+
+      - name: Install Brew dependencies
+        run: |
+          brew bundle
+
+      - name: Docker meta
+        id: meta
+        uses: docker/metadata-action@902fa8ec7d6ecbf8d84d538b9b233a880e428804 # v5.7.0
+        with:
+          images: ghcr.io/techarohq/anubis
+
+      - name: Build and push
+        id: build
+        run: |
+          npm ci
+          npm run container
+        env:
+          PULL_REQUEST_ID: ${{ github.event.number }}
+          DOCKER_REPO: ghcr.io/techarohq/anubis
+          SLOG_LEVEL: debug
+
+      - run: |
+          echo "Test this with:"
+          echo "docker pull ${DOCKER_IMAGE}"
+        env:
+          DOCKER_IMAGE: ${{ steps.build.outputs.docker_image }}

.github/workflows/docker.yml

@@ -5,8 +5,6 @@ on:
   push:
     branches: [ "main" ]
     tags: [ "v*" ]
-  pull_request:
-    branches: [ "main" ]
 
 env:
   DOCKER_METADATA_SET_OUTPUT_ENV: "true"
@@ -20,28 +18,41 @@ permissions:
 
 jobs:
   build:
-    runs-on: ubuntu-latest
+    runs-on: ubuntu-24.04
     steps:
       - name: Checkout code
-        uses: actions/checkout@v4
+        uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
         with:
           fetch-tags: true
           fetch-depth: 0
+          persist-credentials: false
 
-      - name: Set up QEMU
-        uses: docker/setup-qemu-action@v3
+      - name: Set up Homebrew
+        uses: Homebrew/actions/setup-homebrew@master
 
-      - name: Set up Docker Buildx
-        uses: docker/setup-buildx-action@v3
-
-      - uses: actions/setup-go@v5
+      - name: Setup Homebrew cellar cache
+        uses: actions/cache@5a3ec84eff668545956fd18022155c47e93e2684 # v4.2.3
         with:
-          go-version: '1.24.x'
+          path: |
+            /home/linuxbrew/.linuxbrew/Cellar
+            /home/linuxbrew/.linuxbrew/bin
+            /home/linuxbrew/.linuxbrew/etc
+            /home/linuxbrew/.linuxbrew/include
+            /home/linuxbrew/.linuxbrew/lib
+            /home/linuxbrew/.linuxbrew/opt
+            /home/linuxbrew/.linuxbrew/sbin
+            /home/linuxbrew/.linuxbrew/share
+            /home/linuxbrew/.linuxbrew/var
+          key: ${{ runner.os }}-go-homebrew-cellar-${{ hashFiles('go.sum') }}
+          restore-keys: |
+            ${{ runner.os }}-go-homebrew-cellar-
 
-      - uses: ko-build/setup-ko@v0.8
+      - name: Install Brew dependencies
+        run: |
+          brew bundle
 
       - name: Log into registry 
-        uses: docker/login-action@v3
+        uses: docker/login-action@74a5d142397b4f367a81961eba4e8cd7edddf772 # v3.4.0
         with:
           registry: ghcr.io
           username: techarohq
@@ -49,33 +60,22 @@ jobs:
 
       - name: Docker meta
         id: meta
-        uses: docker/metadata-action@v5
+        uses: docker/metadata-action@902fa8ec7d6ecbf8d84d538b9b233a880e428804 # v5.7.0
         with:
           images: ghcr.io/techarohq/anubis
 
       - name: Build and push
         id: build
         run: |
-          go run ./cmd/containerbuild --docker-repo ghcr.io/techarohq/anubis --slog-level debug
+          npm ci
+          npm run container
         env:
-          PULL_REQUEST_ID: ${{ github.event.number }}
-
-      # - name: "Comment about where to test this"
-      #   uses: thollander/actions-comment-pull-request@v3
-      #   if: ${{github.event_name == 'pull_request'}}
-      #   with:
-      #     message: |
-      #       You can try this PR out by using the following docker image:
-
-      #       ```
-      #       ${{ steps.build.outputs.docker_image }}
-      #       ```
-      #     comment-tag: ${{ steps.build.outputs.docker_image }}
+          DOCKER_REPO: ghcr.io/techarohq/anubis
+          SLOG_LEVEL: debug
       
       - name: Generate artifact attestation
-        uses: actions/attest-build-provenance@v2
-        if: ${{github.event_name == 'pull_request'}}
+        uses: actions/attest-build-provenance@c074443f1aee8d4aeeae555aebba3282517141b2 # v2.2.3
         with:
           subject-name: ghcr.io/techarohq/anubis
           subject-digest: ${{ steps.build.outputs.digest }}
-          push-to-registry: true
+          push-to-registry: true

.github/workflows/docs-deploy.yml

@@ -13,16 +13,18 @@ permissions:
 
 jobs:
   build:
-    runs-on: ubuntu-latest
+    runs-on: ubuntu-24.04
 
     steps:
-      - uses: actions/checkout@v4
+      - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
+        with:
+          persist-credentials: false
 
       - name: Set up Docker Buildx
-        uses: docker/setup-buildx-action@v3
+        uses: docker/setup-buildx-action@b5ca514318bd6ebac0fb2aedd5d36ec1b5c232a2 # v3.10.0
 
       - name: Log into registry 
-        uses: docker/login-action@v3
+        uses: docker/login-action@74a5d142397b4f367a81961eba4e8cd7edddf772 # v3.4.0
         with:
           registry: ghcr.io
           username: techarohq
@@ -30,13 +32,13 @@ jobs:
 
       - name: Docker meta
         id: meta
-        uses: docker/metadata-action@v5
+        uses: docker/metadata-action@902fa8ec7d6ecbf8d84d538b9b233a880e428804 # v5.7.0
         with:
           images: ghcr.io/techarohq/anubis/docs
 
       - name: Build and push
         id: build
-        uses: docker/build-push-action@v6
+        uses: docker/build-push-action@471d1dc4e07e5cdedd4c2171150001c434f0b7a4 # v6.15.0
         with:
           context: ./docs
           cache-to: type=gha
@@ -47,14 +49,14 @@ jobs:
           push: true
 
       - name: Apply k8s manifests to aeacus
-        uses: actions-hub/kubectl@master
+        uses: actions-hub/kubectl@9270913c29699788b51bc04becd0ebdf048ffb49 # v1.32.3
         env:
           KUBE_CONFIG: ${{ secrets.AEACUS_KUBECONFIG }}
         with:
           args: apply -k docs/manifest
 
       - name: Apply k8s manifests to aeacus
-        uses: actions-hub/kubectl@master
+        uses: actions-hub/kubectl@9270913c29699788b51bc04becd0ebdf048ffb49 # v1.32.3
         env:
           KUBE_CONFIG: ${{ secrets.AEACUS_KUBECONFIG }}
         with:

.github/workflows/docs-test.yml

@@ -0,0 +1,39 @@
+name: Docs test build
+
+on:
+  pull_request:
+    branches: [ "main" ]
+
+permissions:
+  contents: read
+  actions: write
+
+jobs:
+  build:
+    runs-on: ubuntu-24.04
+
+    steps:
+      - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
+        with:
+          persist-credentials: false
+
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@b5ca514318bd6ebac0fb2aedd5d36ec1b5c232a2 # v3.10.0
+
+      - name: Docker meta
+        id: meta
+        uses: docker/metadata-action@902fa8ec7d6ecbf8d84d538b9b233a880e428804 # v5.7.0
+        with:
+          images: ghcr.io/techarohq/anubis/docs
+
+      - name: Build and push
+        id: build
+        uses: docker/build-push-action@471d1dc4e07e5cdedd4c2171150001c434f0b7a4 # v6.15.0
+        with:
+          context: ./docs
+          cache-to: type=gha
+          cache-from: type=gha
+          tags: ${{ steps.meta.outputs.tags }}
+          labels: ${{ steps.meta.outputs.labels }}
+          platforms: linux/amd64
+          push: false

.github/workflows/go.yml

@@ -11,11 +11,13 @@ permissions:
   actions: write
 
 jobs:
-  build:
+  go_tests:
     #runs-on: alrest-techarohq
-    runs-on: ubuntu-latest
+    runs-on: ubuntu-24.04
     steps:
-    - uses: actions/checkout@v4
+    - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
+      with:
+        persist-credentials: false
 
     - name: build essential
       run: |
@@ -26,7 +28,7 @@ jobs:
       uses: Homebrew/actions/setup-homebrew@master
 
     - name: Setup Homebrew cellar cache
-      uses: actions/cache@v4
+      uses: actions/cache@5a3ec84eff668545956fd18022155c47e93e2684 # v4.2.3
       with:
         path: |
           /home/linuxbrew/.linuxbrew/Cellar
@@ -47,7 +49,7 @@ jobs:
         brew bundle
 
     - name: Setup Golang caches
-      uses: actions/cache@v4
+      uses: actions/cache@5a3ec84eff668545956fd18022155c47e93e2684 # v4.2.3
       with:
         path: |
           ~/.cache/go-build
@@ -56,8 +58,30 @@ jobs:
         restore-keys: |
           ${{ runner.os }}-golang-
 
+    - name: Cache playwright binaries
+      uses: actions/cache@5a3ec84eff668545956fd18022155c47e93e2684 # v4.2.3
+      id: playwright-cache
+      with:
+        path: |
+          ~/.cache/ms-playwright
+        key: ${{ runner.os }}-playwright-${{ hashFiles('**/go.sum') }}
+
+    - name: install playwright browsers
+      run: |
+        npx --yes playwright@1.51.1 install --with-deps
+        npx --yes playwright@1.51.1 run-server --port 9001 &
+
+    - name: install node deps
+      run: |
+        npm ci
+        npm run assets
+
     - name: Build
-      run: go build ./...
+      run: npm run build
 
     - name: Test
-      run: go test ./...
+      run: npm run test
+
+    - uses: dominikh/staticcheck-action@fe1dd0c3658873b46f8c9bb3291096a617310ca6 # v1.3.1
+      with:
+        version: "latest"

.github/workflows/package-builds-stable.yml

@@ -0,0 +1,82 @@
+name: Package builds (stable)
+
+on:
+  release:
+    types: [published]
+
+permissions:
+  contents: write
+  actions: write
+
+jobs:
+  package_builds:
+    #runs-on: alrest-techarohq
+    runs-on: ubuntu-24.04
+    steps:
+    - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
+      with:
+        persist-credentials: false
+        fetch-tags: true
+        fetch-depth: 0
+
+    - name: build essential
+      run: |
+        sudo apt-get update
+        sudo apt-get install -y build-essential
+
+    - name: Set up Homebrew
+      uses: Homebrew/actions/setup-homebrew@master
+
+    - name: Setup Homebrew cellar cache
+      uses: actions/cache@5a3ec84eff668545956fd18022155c47e93e2684 # v4.2.3
+      with:
+        path: |
+          /home/linuxbrew/.linuxbrew/Cellar
+          /home/linuxbrew/.linuxbrew/bin
+          /home/linuxbrew/.linuxbrew/etc
+          /home/linuxbrew/.linuxbrew/include
+          /home/linuxbrew/.linuxbrew/lib
+          /home/linuxbrew/.linuxbrew/opt
+          /home/linuxbrew/.linuxbrew/sbin
+          /home/linuxbrew/.linuxbrew/share
+          /home/linuxbrew/.linuxbrew/var
+        key: ${{ runner.os }}-go-homebrew-cellar-${{ hashFiles('go.sum') }}
+        restore-keys: |
+          ${{ runner.os }}-go-homebrew-cellar-
+
+    - name: Install Brew dependencies
+      run: |
+        brew bundle
+
+    - name: Setup Golang caches
+      uses: actions/cache@5a3ec84eff668545956fd18022155c47e93e2684 # v4.2.3
+      with:
+        path: |
+          ~/.cache/go-build
+          ~/go/pkg/mod
+        key: ${{ runner.os }}-golang-${{ hashFiles('**/go.sum') }}
+        restore-keys: |
+          ${{ runner.os }}-golang-
+
+    - name: install node deps
+      run: |
+        npm ci
+
+    - name: Build Packages
+      run: |
+        wget https://github.com/TecharoHQ/yeet/releases/download/v0.2.1/yeet_0.2.1_amd64.deb -O var/yeet.deb
+        sudo apt -y install -f ./var/yeet.deb
+        rm ./var/yeet.deb
+        yeet
+
+    - name: Upload released artifacts
+      env:
+        GITHUB_TOKEN: ${{ github.TOKEN }}
+        RELEASE_VERSION: ${{github.event.release.tag_name}}
+      shell: bash
+      run: |
+        RELEASE="${RELEASE_VERSION}"
+        cd var
+        for file in *; do
+          gh release upload $RELEASE $file
+        done

.github/workflows/package-builds-unstable.yml

@@ -0,0 +1,77 @@
+name: Package builds (unstable)
+
+on:
+  push:
+    branches: [ "main" ]
+  pull_request:
+    branches: [ "main" ]
+
+permissions:
+  contents: read
+  actions: write
+
+jobs:
+  package_builds:
+    #runs-on: alrest-techarohq
+    runs-on: ubuntu-24.04
+    steps:
+    - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
+      with:
+        persist-credentials: false
+        fetch-tags: true
+        fetch-depth: 0
+
+    - name: build essential
+      run: |
+        sudo apt-get update
+        sudo apt-get install -y build-essential
+
+    - name: Set up Homebrew
+      uses: Homebrew/actions/setup-homebrew@master
+
+    - name: Setup Homebrew cellar cache
+      uses: actions/cache@5a3ec84eff668545956fd18022155c47e93e2684 # v4.2.3
+      with:
+        path: |
+          /home/linuxbrew/.linuxbrew/Cellar
+          /home/linuxbrew/.linuxbrew/bin
+          /home/linuxbrew/.linuxbrew/etc
+          /home/linuxbrew/.linuxbrew/include
+          /home/linuxbrew/.linuxbrew/lib
+          /home/linuxbrew/.linuxbrew/opt
+          /home/linuxbrew/.linuxbrew/sbin
+          /home/linuxbrew/.linuxbrew/share
+          /home/linuxbrew/.linuxbrew/var
+        key: ${{ runner.os }}-go-homebrew-cellar-${{ hashFiles('go.sum') }}
+        restore-keys: |
+          ${{ runner.os }}-go-homebrew-cellar-
+
+    - name: Install Brew dependencies
+      run: |
+        brew bundle
+
+    - name: Setup Golang caches
+      uses: actions/cache@5a3ec84eff668545956fd18022155c47e93e2684 # v4.2.3
+      with:
+        path: |
+          ~/.cache/go-build
+          ~/go/pkg/mod
+        key: ${{ runner.os }}-golang-${{ hashFiles('**/go.sum') }}
+        restore-keys: |
+          ${{ runner.os }}-golang-
+
+    - name: install node deps
+      run: |
+        npm ci
+
+    - name: Build Packages
+      run: |
+        wget https://github.com/TecharoHQ/yeet/releases/download/v0.2.1/yeet_0.2.1_amd64.deb -O var/yeet.deb
+        sudo apt -y install -f ./var/yeet.deb
+        rm ./var/yeet.deb
+        yeet
+
+    - uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2
+      with:
+        name: packages
+        path: var/*

.github/workflows/zizmor.yml

@@ -0,0 +1,35 @@
+name: zizmor
+
+on:
+    push:
+      paths:
+        - '.github/workflows/*.ya?ml'
+    pull_request:
+      paths:
+        - '.github/workflows/*.ya?ml'
+
+jobs:
+  zizmor:
+    name: zizmor latest via PyPI
+    runs-on: ubuntu-24.04
+    permissions:
+      security-events: write
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
+        with:
+          persist-credentials: false
+
+      - name: Install the latest version of uv
+        uses: astral-sh/setup-uv@d4b2f3b6ecc6e67c4457f6d3e41ec42d3d0fcb86 # v5.4.2
+
+      - name: Run zizmor 🌈
+        run: uvx zizmor --format sarif . > results.sarif 
+        env:
+          GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} 
+
+      - name: Upload SARIF file
+        uses: github/codeql-action/upload-sarif@45775bd8235c68ba998cffa5171334d58593da47 # v3.28.15
+        with:
+          sarif_file: results.sarif
+          category: zizmor

.gitignore

@@ -1,2 +1,22 @@
 .env
-*.rpm
+*.deb
+*.rpm
+
+# Additional package locks
+pnpm-lock.yaml
+yarn.lock
+
+# Go binaries and test artifacts
+main
+*.test
+
+node_modules
+
+# MacOS
+.DS_store
+
+# Intellij
+.idea
+
+# how does this get here
+doc/VERSION

Brewfile

@@ -1,4 +1,7 @@
 # programming languages
 brew "go@1.24"
 brew "node"
-brew "ko"
+brew "ko"
+brew "esbuild"
+brew "zstd"
+brew "brotli"

Dockerfile

@@ -1,23 +0,0 @@
-FROM docker.io/library/golang:1.24 AS build
-ARG BUILDKIT_SBOM_SCAN_CONTEXT=true BUILDKIT_SBOM_SCAN_STAGE=true
-
-WORKDIR /app
-COPY go.mod go.sum /app/
-RUN go mod download
-
-COPY . .
-RUN --mount=type=cache,target=/root/.cache \
-  VERSION=$(git describe --tags --always --dirty) \
-  && go build -o /app/bin/anubis -ldflags="-X github.com/TecharoHQ/anubis.Version=${VERSION}" ./cmd/anubis
-
-FROM docker.io/library/debian:bookworm AS runtime
-ARG BUILDKIT_SBOM_SCAN_STAGE=true
-RUN apt-get update \
-  && apt-get -y install ca-certificates
-
-COPY --from=build /app/bin/anubis /app/bin/anubis
-
-HEALTHCHECK --interval=30s --timeout=5s --start-period=5s --retries=3 CMD ["/app/bin/anubis", "--healthcheck"]
-CMD ["/app/bin/anubis"]
-
-LABEL org.opencontainers.image.source="https://github.com/TecharoHQ/anubis"

Makefile

@@ -0,0 +1,31 @@
+VERSION= $(shell cat ./VERSION)
+GO?= go
+NPM?= npm
+
+.PHONY: build assets deps lint prebaked-build test
+
+all: build
+
+deps:
+	$(NPM) ci
+	$(GO) mod download
+
+assets: PATH:=$(PWD)/node_modules/.bin:$(PATH)
+assets: deps
+	$(GO) generate ./...
+	./web/build.sh
+	./xess/build.sh
+
+build: assets
+	$(GO) build -o ./var/anubis ./cmd/anubis
+	@echo "Anubis is now built to ./var/anubis"
+
+lint: assets
+	$(GO) vet ./...
+	$(GO) tool staticcheck ./...
+
+prebaked-build:
+	$(GO) build -o ./var/anubis -ldflags "-X 'github.com/TecharoHQ/anubis.Version=$(VERSION)'" ./cmd/anubis
+
+test: assets
+	$(GO) test ./...

PULL_REQUEST_TEMPLATE.md

@@ -1,6 +0,0 @@
-<!-- delete me and describe your change here -->
-
-Checklist:
-
-- [ ] Added a description of the changes to the `[Unreleased]` section of docs/docs/CHANGELOG.md
-- [ ] Tested this at least manually

README.md

@@ -1,7 +1,7 @@
 # Anubis
 
 <center>
-<img width=256 src="./cmd/anubis/static/img/happy.webp" alt="A smiling chibi dark-skinned anthro jackal with brown hair and tall ears looking victorious with a thumbs-up" />
+<img width=256 src="./web/static/img/happy.webp" alt="A smiling chibi dark-skinned anthro jackal with brown hair and tall ears looking victorious with a thumbs-up" />
 </center>
 
 ![enbyware](https://pride-badges.pony.workers.dev/static/v1?label=enbyware&labelColor=%23555&stripeWidth=8&stripeColors=FCF434%2CFFFFFF%2C9C59D1%2C2C2C2C)
@@ -10,11 +10,19 @@
 ![language count](https://img.shields.io/github/languages/count/TecharoHQ/anubis)
 ![repo size](https://img.shields.io/github/repo-size/TecharoHQ/anubis)
 
-Anubis [weighs the soul of your connection](https://en.wikipedia.org/wiki/Weighing_of_souls) using a sha256 proof-of-work challenge in order to protect upstream resources from scraper bots.
+## Sponsors
 
-Installing and using this will likely result in your website not being indexed by some search engines. This is considered a feature of Anubis, not a bug.
+Anubis is brought to you by sponsors and donors like:
 
-This is a bit of a nuclear response, but AI scraper bots scraping so aggressively have forced my hand. I hate that I have to do this, but this is what we get for the modern Internet because bots don't conform to standards like robots.txt, even when they claim to.
+[![Distrust](./docs/static/img/sponsors/distrust-logo.webp)](https://distrust.co)
+
+## Overview
+
+Anubis [weighs the soul of your connection](https://en.wikipedia.org/wiki/Weighing_of_souls) using a proof-of-work challenge in order to protect upstream resources from scraper bots.
+
+This program is designed to help protect the small internet from the endless storm of requests that flood in from AI companies. Anubis is as lightweight as possible to ensure that everyone can afford to protect the communities closest to them.
+
+Anubis is a bit of a nuclear response. This will result in your website being blocked from smaller scrapers and may inhibit "good bots" like the Internet Archive. You can configure [bot policy definitions](./docs/docs/admin/policies.mdx) to explicitly allowlist them and we are working on a curated set of "known good" bots to allow for a compromise between discoverability and uptime.
 
 In most cases, you should not need this and can probably get by using Cloudflare to protect a given origin. However, for circumstances where you can't or won't use Cloudflare, Anubis is there for you.
 
@@ -22,10 +30,28 @@ If you want to try this out, connect to [anubis.techaro.lol](https://anubis.tech
 
 ## Support
 
-If you run into any issues running Anubis, please [open an issue](https://github.com/TecharoHQ/anubis/issues/new?template=Blank+issue) and tag it with the Anubis tag. Please include all the information I would need to diagnose your issue.
+If you run into any issues running Anubis, please [open an issue](https://github.com/TecharoHQ/anubis/issues/new?template=Blank+issue). Please include all the information I would need to diagnose your issue.
 
 For live chat, please join the [Patreon](https://patreon.com/cadey) and ask in the Patron discord in the channel `#anubis`.
 
 ## Star History
 
-[![Star History Chart](https://api.star-history.com/svg?repos=TecharoHQ/anubis&type=Date)](https://www.star-history.com/#TecharoHQ/anubis&Date)
+<a href="https://www.star-history.com/#TecharoHQ/anubis&Date">
+ <picture>
+   <source media="(prefers-color-scheme: dark)" srcset="https://api.star-history.com/svg?repos=TecharoHQ/anubis&type=Date&theme=dark" />
+   <source media="(prefers-color-scheme: light)" srcset="https://api.star-history.com/svg?repos=TecharoHQ/anubis&type=Date" />
+   <img alt="Star History Chart" src="https://api.star-history.com/svg?repos=TecharoHQ/anubis&type=Date" />
+ </picture>
+</a>
+
+## Packaging Status
+
+[![Packaging status](https://repology.org/badge/vertical-allrepos/anubis-anti-crawler.svg?columns=3)](https://repology.org/project/anubis-anti-crawler/versions)
+
+## Contributors
+
+<a href="https://github.com/TecharoHQ/anubis/graphs/contributors">
+  <img src="https://contrib.rocks/image?repo=TecharoHQ/anubis" />
+</a>
+
+Made with [contrib.rocks](https://contrib.rocks).

VERSION

@@ -1 +1 @@
-1.14.2
+1.17.0

anubis.go

@@ -0,0 +1,25 @@
+// Package anubis contains the version number of Anubis.
+package anubis
+
+// Version is the current version of Anubis.
+//
+// This variable is set at build time using the -X linker flag. If not set,
+// it defaults to "devel".
+var Version = "devel"
+
+// CookieName is the name of the cookie that Anubis uses in order to validate
+// access.
+const CookieName = "within.website-x-cmd-anubis-auth"
+
+// BasePrefix is a global prefix for all Anubis endpoints. Can be emptied to remove the prefix entirely.
+var BasePrefix = ""
+
+// StaticPath is the location where all static Anubis assets are located.
+const StaticPath = "/.within.website/x/cmd/anubis/"
+
+// APIPrefix is the location where all Anubis API endpoints are located.
+const APIPrefix = "/.within.website/x/cmd/anubis/api/"
+
+// DefaultDifficulty is the default "difficulty" (number of leading zeroes)
+// that must be met by the client in order to pass the challenge.
+const DefaultDifficulty = 4

cmd/anubis/CHANGELOG.md

@@ -1,5 +0,0 @@
-# CHANGELOG
-
-## 2025-01-24
-
-- Added support for custom bot policy documentation, allowing administrators to change how Anubis works to meet their needs.

cmd/anubis/botPolicies.json

@@ -1,398 +0,0 @@
-{
-  "bots": [
-    {
-      "name": "amazonbot",
-      "user_agent_regex": "Amazonbot",
-      "action": "DENY"
-    },
-    {
-      "name": "googlebot",
-      "user_agent_regex": "\\+http\\:\\/\\/www\\.google\\.com/bot\\.html",
-      "action": "ALLOW",
-      "remote_addresses": [
-        "2001:4860:4801:10::/64",
-        "2001:4860:4801:11::/64",
-        "2001:4860:4801:12::/64",
-        "2001:4860:4801:13::/64",
-        "2001:4860:4801:14::/64",
-        "2001:4860:4801:15::/64",
-        "2001:4860:4801:16::/64",
-        "2001:4860:4801:17::/64",
-        "2001:4860:4801:18::/64",
-        "2001:4860:4801:19::/64",
-        "2001:4860:4801:1a::/64",
-        "2001:4860:4801:1b::/64",
-        "2001:4860:4801:1c::/64",
-        "2001:4860:4801:1d::/64",
-        "2001:4860:4801:1e::/64",
-        "2001:4860:4801:1f::/64",
-        "2001:4860:4801:20::/64",
-        "2001:4860:4801:21::/64",
-        "2001:4860:4801:22::/64",
-        "2001:4860:4801:23::/64",
-        "2001:4860:4801:24::/64",
-        "2001:4860:4801:25::/64",
-        "2001:4860:4801:26::/64",
-        "2001:4860:4801:27::/64",
-        "2001:4860:4801:28::/64",
-        "2001:4860:4801:29::/64",
-        "2001:4860:4801:2::/64",
-        "2001:4860:4801:2a::/64",
-        "2001:4860:4801:2b::/64",
-        "2001:4860:4801:2c::/64",
-        "2001:4860:4801:2d::/64",
-        "2001:4860:4801:2e::/64",
-        "2001:4860:4801:2f::/64",
-        "2001:4860:4801:31::/64",
-        "2001:4860:4801:32::/64",
-        "2001:4860:4801:33::/64",
-        "2001:4860:4801:34::/64",
-        "2001:4860:4801:35::/64",
-        "2001:4860:4801:36::/64",
-        "2001:4860:4801:37::/64",
-        "2001:4860:4801:38::/64",
-        "2001:4860:4801:39::/64",
-        "2001:4860:4801:3a::/64",
-        "2001:4860:4801:3b::/64",
-        "2001:4860:4801:3c::/64",
-        "2001:4860:4801:3d::/64",
-        "2001:4860:4801:3e::/64",
-        "2001:4860:4801:40::/64",
-        "2001:4860:4801:41::/64",
-        "2001:4860:4801:42::/64",
-        "2001:4860:4801:43::/64",
-        "2001:4860:4801:44::/64",
-        "2001:4860:4801:45::/64",
-        "2001:4860:4801:46::/64",
-        "2001:4860:4801:47::/64",
-        "2001:4860:4801:48::/64",
-        "2001:4860:4801:49::/64",
-        "2001:4860:4801:4a::/64",
-        "2001:4860:4801:4b::/64",
-        "2001:4860:4801:4c::/64",
-        "2001:4860:4801:50::/64",
-        "2001:4860:4801:51::/64",
-        "2001:4860:4801:52::/64",
-        "2001:4860:4801:53::/64",
-        "2001:4860:4801:54::/64",
-        "2001:4860:4801:55::/64",
-        "2001:4860:4801:56::/64",
-        "2001:4860:4801:60::/64",
-        "2001:4860:4801:61::/64",
-        "2001:4860:4801:62::/64",
-        "2001:4860:4801:63::/64",
-        "2001:4860:4801:64::/64",
-        "2001:4860:4801:65::/64",
-        "2001:4860:4801:66::/64",
-        "2001:4860:4801:67::/64",
-        "2001:4860:4801:68::/64",
-        "2001:4860:4801:69::/64",
-        "2001:4860:4801:6a::/64",
-        "2001:4860:4801:6b::/64",
-        "2001:4860:4801:6c::/64",
-        "2001:4860:4801:6d::/64",
-        "2001:4860:4801:6e::/64",
-        "2001:4860:4801:6f::/64",
-        "2001:4860:4801:70::/64",
-        "2001:4860:4801:71::/64",
-        "2001:4860:4801:72::/64",
-        "2001:4860:4801:73::/64",
-        "2001:4860:4801:74::/64",
-        "2001:4860:4801:75::/64",
-        "2001:4860:4801:76::/64",
-        "2001:4860:4801:77::/64",
-        "2001:4860:4801:78::/64",
-        "2001:4860:4801:79::/64",
-        "2001:4860:4801:80::/64",
-        "2001:4860:4801:81::/64",
-        "2001:4860:4801:82::/64",
-        "2001:4860:4801:83::/64",
-        "2001:4860:4801:84::/64",
-        "2001:4860:4801:85::/64",
-        "2001:4860:4801:86::/64",
-        "2001:4860:4801:87::/64",
-        "2001:4860:4801:88::/64",
-        "2001:4860:4801:90::/64",
-        "2001:4860:4801:91::/64",
-        "2001:4860:4801:92::/64",
-        "2001:4860:4801:93::/64",
-        "2001:4860:4801:94::/64",
-        "2001:4860:4801:95::/64",
-        "2001:4860:4801:96::/64",
-        "2001:4860:4801:a0::/64",
-        "2001:4860:4801:a1::/64",
-        "2001:4860:4801:a2::/64",
-        "2001:4860:4801:a3::/64",
-        "2001:4860:4801:a4::/64",
-        "2001:4860:4801:a5::/64",
-        "2001:4860:4801:c::/64",
-        "2001:4860:4801:f::/64",
-        "192.178.5.0/27",
-        "192.178.6.0/27",
-        "192.178.6.128/27",
-        "192.178.6.160/27",
-        "192.178.6.192/27",
-        "192.178.6.32/27",
-        "192.178.6.64/27",
-        "192.178.6.96/27",
-        "34.100.182.96/28",
-        "34.101.50.144/28",
-        "34.118.254.0/28",
-        "34.118.66.0/28",
-        "34.126.178.96/28",
-        "34.146.150.144/28",
-        "34.147.110.144/28",
-        "34.151.74.144/28",
-        "34.152.50.64/28",
-        "34.154.114.144/28",
-        "34.155.98.32/28",
-        "34.165.18.176/28",
-        "34.175.160.64/28",
-        "34.176.130.16/28",
-        "34.22.85.0/27",
-        "34.64.82.64/28",
-        "34.65.242.112/28",
-        "34.80.50.80/28",
-        "34.88.194.0/28",
-        "34.89.10.80/28",
-        "34.89.198.80/28",
-        "34.96.162.48/28",
-        "35.247.243.240/28",
-        "66.249.64.0/27",
-        "66.249.64.128/27",
-        "66.249.64.160/27",
-        "66.249.64.224/27",
-        "66.249.64.32/27",
-        "66.249.64.64/27",
-        "66.249.64.96/27",
-        "66.249.65.0/27",
-        "66.249.65.128/27",
-        "66.249.65.160/27",
-        "66.249.65.192/27",
-        "66.249.65.224/27",
-        "66.249.65.32/27",
-        "66.249.65.64/27",
-        "66.249.65.96/27",
-        "66.249.66.0/27",
-        "66.249.66.128/27",
-        "66.249.66.160/27",
-        "66.249.66.192/27",
-        "66.249.66.224/27",
-        "66.249.66.32/27",
-        "66.249.66.64/27",
-        "66.249.66.96/27",
-        "66.249.68.0/27",
-        "66.249.68.128/27",
-        "66.249.68.32/27",
-        "66.249.68.64/27",
-        "66.249.68.96/27",
-        "66.249.69.0/27",
-        "66.249.69.128/27",
-        "66.249.69.160/27",
-        "66.249.69.192/27",
-        "66.249.69.224/27",
-        "66.249.69.32/27",
-        "66.249.69.64/27",
-        "66.249.69.96/27",
-        "66.249.70.0/27",
-        "66.249.70.128/27",
-        "66.249.70.160/27",
-        "66.249.70.192/27",
-        "66.249.70.224/27",
-        "66.249.70.32/27",
-        "66.249.70.64/27",
-        "66.249.70.96/27",
-        "66.249.71.0/27",
-        "66.249.71.128/27",
-        "66.249.71.160/27",
-        "66.249.71.192/27",
-        "66.249.71.224/27",
-        "66.249.71.32/27",
-        "66.249.71.64/27",
-        "66.249.71.96/27",
-        "66.249.72.0/27",
-        "66.249.72.128/27",
-        "66.249.72.160/27",
-        "66.249.72.192/27",
-        "66.249.72.224/27",
-        "66.249.72.32/27",
-        "66.249.72.64/27",
-        "66.249.72.96/27",
-        "66.249.73.0/27",
-        "66.249.73.128/27",
-        "66.249.73.160/27",
-        "66.249.73.192/27",
-        "66.249.73.224/27",
-        "66.249.73.32/27",
-        "66.249.73.64/27",
-        "66.249.73.96/27",
-        "66.249.74.0/27",
-        "66.249.74.128/27",
-        "66.249.74.160/27",
-        "66.249.74.192/27",
-        "66.249.74.32/27",
-        "66.249.74.64/27",
-        "66.249.74.96/27",
-        "66.249.75.0/27",
-        "66.249.75.128/27",
-        "66.249.75.160/27",
-        "66.249.75.192/27",
-        "66.249.75.224/27",
-        "66.249.75.32/27",
-        "66.249.75.64/27",
-        "66.249.75.96/27",
-        "66.249.76.0/27",
-        "66.249.76.128/27",
-        "66.249.76.160/27",
-        "66.249.76.192/27",
-        "66.249.76.224/27",
-        "66.249.76.32/27",
-        "66.249.76.64/27",
-        "66.249.76.96/27",
-        "66.249.77.0/27",
-        "66.249.77.128/27",
-        "66.249.77.160/27",
-        "66.249.77.192/27",
-        "66.249.77.224/27",
-        "66.249.77.32/27",
-        "66.249.77.64/27",
-        "66.249.77.96/27",
-        "66.249.78.0/27",
-        "66.249.78.32/27",
-        "66.249.79.0/27",
-        "66.249.79.128/27",
-        "66.249.79.160/27",
-        "66.249.79.192/27",
-        "66.249.79.224/27",
-        "66.249.79.32/27",
-        "66.249.79.64/27",
-        "66.249.79.96/27"
-      ]
-    },
-    {
-      "name": "bingbot",
-      "user_agent_regex": "\\+http\\:\\/\\/www\\.bing\\.com/bingbot\\.htm",
-      "action": "ALLOW",
-      "remote_addresses": [
-        "157.55.39.0/24",
-        "207.46.13.0/24",
-        "40.77.167.0/24",
-        "13.66.139.0/24",
-        "13.66.144.0/24",
-        "52.167.144.0/24",
-        "13.67.10.16/28",
-        "13.69.66.240/28",
-        "13.71.172.224/28",
-        "139.217.52.0/28",
-        "191.233.204.224/28",
-        "20.36.108.32/28",
-        "20.43.120.16/28",
-        "40.79.131.208/28",
-        "40.79.186.176/28",
-        "52.231.148.0/28",
-        "20.79.107.240/28",
-        "51.105.67.0/28",
-        "20.125.163.80/28",
-        "40.77.188.0/22",
-        "65.55.210.0/24",
-        "199.30.24.0/23",
-        "40.77.202.0/24",
-        "40.77.139.0/25",
-        "20.74.197.0/28",
-        "20.15.133.160/27",
-        "40.77.177.0/24",
-        "40.77.178.0/23"
-      ]
-    },
-    {
-      "name": "qwantbot",
-      "user_agent_regex": "\\+https\\:\\/\\/help\\.qwant\\.com/bot/",
-      "action": "ALLOW",
-      "remote_addresses": [
-        "91.242.162.0/24"
-      ]
-    },
-    {
-      "name": "kagibot",
-      "user_agent_regex": "\\+https\\:\\/\\/kagi\\.com/bot",
-      "action": "ALLOW",
-      "remote_addresses": [
-        "216.18.205.234/32",
-        "35.212.27.76/32",
-        "104.254.65.50/32",
-        "209.151.156.194/32"
-      ]
-    },
-    {
-      "name": "marginalia",
-      "user_agent_regex": "search\\.marginalia\\.nu",
-      "action": "ALLOW",
-      "remote_addresses": [
-        "193.183.0.162/31",
-        "193.183.0.164/30",
-        "193.183.0.168/30",
-        "193.183.0.172/31",
-        "193.183.0.174/32"
-      ]
-    },
-    {
-      "name": "mojeekbot",
-      "user_agent_regex": "http\\://www\\.mojeek\\.com/bot\\.html",
-      "action": "ALLOW",
-      "remote_addresses": [
-        "5.102.173.71/32"
-      ]
-    },
-    {
-      "name": "us-artificial-intelligence-scraper",
-      "user_agent_regex": "\\+https\\:\\/\\/github\\.com\\/US-Artificial-Intelligence\\/scraper",
-      "action": "DENY"
-    },
-    {
-      "name": "well-known",
-      "path_regex": "^/.well-known/.*$",
-      "action": "ALLOW"
-    },
-    {
-      "name": "favicon",
-      "path_regex": "^/favicon.ico$",
-      "action": "ALLOW"
-    },
-    {
-      "name": "robots-txt",
-      "path_regex": "^/robots.txt$",
-      "action": "ALLOW"
-    },
-    {
-      "name": "lightpanda",
-      "user_agent_regex": "^Lightpanda/.*$",
-      "action": "DENY"
-    },
-    {
-      "name": "headless-chrome",
-      "user_agent_regex": "HeadlessChrome",
-      "action": "DENY"
-    },
-    {
-      "name": "headless-chromium",
-      "user_agent_regex": "HeadlessChromium",
-      "action": "DENY"
-    },
-    {
-      "name": "generic-bot-catchall",
-      "user_agent_regex": "(?i:bot|crawler)",
-      "action": "CHALLENGE",
-      "challenge": {
-        "difficulty": 16,
-        "report_as": 4,
-        "algorithm": "slow"
-      }
-    },
-    {
-      "name": "generic-browser",
-      "user_agent_regex": "(?i:gecko)",
-      "action": "CHALLENGE"
-    }
-  ],
-  "dnsbl": true
-}

cmd/anubis/decaymap_test.go

@@ -1,31 +0,0 @@
-package main
-
-import (
-	"testing"
-	"time"
-)
-
-func TestDecayMap(t *testing.T) {
-	dm := NewDecayMap[string, string]()
-
-	dm.Set("test", "hi", 5*time.Minute)
-
-	val, ok := dm.Get("test")
-	if !ok {
-		t.Error("somehow the test key was not set")
-	}
-
-	if val != "hi" {
-		t.Errorf("wanted value %q, got: %q", "hi", val)
-	}
-
-	ok = dm.expire("test")
-	if !ok {
-		t.Error("somehow could not force-expire the test key")
-	}
-
-	_, ok = dm.Get("test")
-	if ok {
-		t.Error("got value even though it was supposed to be expired")
-	}
-}

cmd/anubis/index.templ

@@ -1,217 +0,0 @@
-package main
-
-import (
-	"github.com/TecharoHQ/anubis"
-	"github.com/TecharoHQ/anubis/xess"
-)
-
-templ base(title string, body templ.Component) {
-	<!DOCTYPE html>
-	<html>
-		<head>
-			<title>{ title }</title>
-			<link rel="stylesheet" href={ xess.URL }/>
-			<meta name="viewport" content="width=device-width, initial-scale=1.0"/>
-			<style>
-    body,
-    html {
-      height: 100%;
-      display: flex;
-      justify-content: center;
-      align-items: center;
-      margin-left: auto;
-      margin-right: auto;
-    }
-
-    .centered-div {
-      text-align: center;
-    }
-
-    .lds-roller,
-    .lds-roller div,
-    .lds-roller div:after {
-      box-sizing: border-box;
-    }
-
-    .lds-roller {
-      display: inline-block;
-      position: relative;
-      width: 80px;
-      height: 80px;
-    }
-
-    .lds-roller div {
-      animation: lds-roller 1.2s cubic-bezier(0.5, 0, 0.5, 1) infinite;
-      transform-origin: 40px 40px;
-    }
-
-    .lds-roller div:after {
-      content: " ";
-      display: block;
-      position: absolute;
-      width: 7.2px;
-      height: 7.2px;
-      border-radius: 50%;
-      background: currentColor;
-      margin: -3.6px 0 0 -3.6px;
-    }
-
-    .lds-roller div:nth-child(1) {
-      animation-delay: -0.036s;
-    }
-
-    .lds-roller div:nth-child(1):after {
-      top: 62.62742px;
-      left: 62.62742px;
-    }
-
-    .lds-roller div:nth-child(2) {
-      animation-delay: -0.072s;
-    }
-
-    .lds-roller div:nth-child(2):after {
-      top: 67.71281px;
-      left: 56px;
-    }
-
-    .lds-roller div:nth-child(3) {
-      animation-delay: -0.108s;
-    }
-
-    .lds-roller div:nth-child(3):after {
-      top: 70.90963px;
-      left: 48.28221px;
-    }
-
-    .lds-roller div:nth-child(4) {
-      animation-delay: -0.144s;
-    }
-
-    .lds-roller div:nth-child(4):after {
-      top: 72px;
-      left: 40px;
-    }
-
-    .lds-roller div:nth-child(5) {
-      animation-delay: -0.18s;
-    }
-
-    .lds-roller div:nth-child(5):after {
-      top: 70.90963px;
-      left: 31.71779px;
-    }
-
-    .lds-roller div:nth-child(6) {
-      animation-delay: -0.216s;
-    }
-
-    .lds-roller div:nth-child(6):after {
-      top: 67.71281px;
-      left: 24px;
-    }
-
-    .lds-roller div:nth-child(7) {
-      animation-delay: -0.252s;
-    }
-
-    .lds-roller div:nth-child(7):after {
-      top: 62.62742px;
-      left: 17.37258px;
-    }
-
-    .lds-roller div:nth-child(8) {
-      animation-delay: -0.288s;
-    }
-
-    .lds-roller div:nth-child(8):after {
-      top: 56px;
-      left: 12.28719px;
-    }
-
-    @keyframes lds-roller {
-      0% {
-        transform: rotate(0deg);
-      }
-
-      100% {
-        transform: rotate(360deg);
-      }
-    }
-      </style>
-			@templ.JSONScript("anubis_version", anubis.Version)
-		</head>
-		<body id="top">
-			<main>
-				<center>
-					<h1 id="title" class=".centered-div">{ title }</h1>
-				</center>
-				@body
-				<footer>
-					<center>
-						<p>
-							Protected by <a href="https://github.com/TecharoHQ/anubis">Anubis</a> from <a
-	href="https://techaro.lol"
->Techaro</a>. Made with ❤️ in 🇨🇦.
-						</p>
-					</center>
-				</footer>
-			</main>
-		</body>
-	</html>
-}
-
-templ index() {
-	<div class="centered-div">
-		<img
-			id="image"
-			style="width:100%;max-width:256px;"
-			src={ "/.within.website/x/cmd/anubis/static/img/pensive.webp?cacheBuster=" +
-    anubis.Version }
-		/>
-		<img
-			style="display:none;"
-			style="width:100%;max-width:256px;"
-			src={ "/.within.website/x/cmd/anubis/static/img/happy.webp?cacheBuster=" +
-    anubis.Version }
-		/>
-		<p id="status">Loading...</p>
-		<script async type="module" src={ "/.within.website/x/cmd/anubis/static/js/main.mjs?cacheBuster=" + anubis.Version }></script>
-		<div id="spinner" class="lds-roller">
-			<div></div>
-			<div></div>
-			<div></div>
-			<div></div>
-			<div></div>
-			<div></div>
-			<div></div>
-			<div></div>
-		</div>
-		<details>
-			<summary>Why am I seeing this?</summary>
-			<p>You are seeing this because the administrator of this website has set up <a href="https://github.com/TecharoHQ/anubis">Anubis</a> to protect the server against the scourge of <a href="https://thelibre.news/foss-infrastructure-is-under-attack-by-ai-companies/">AI companies aggressively scraping websites</a>. This can and does cause downtime for the websites, which makes their resources inaccessible for everyone.</p>
-			<p>Anubis is a compromise. Anubis uses a <a href="https://anubis.techaro.lol/docs/design/why-proof-of-work">Proof-of-Work</a> scheme in the vein of <a href="https://en.wikipedia.org/wiki/Hashcash">Hashcash</a>, a proposed proof-of-work scheme for reducing email spam. The idea is that at individual scales the additional load is ignorable, but at mass scraper levels it adds up and makes scraping much more expensive.</p>
-			<p>Ultimately, this is a hack whose real purpose is to give a "good enough" placeholder solution so that more time can be spent on fingerprinting and identifying headless browsers (EG: via how they do font rendering) so that the challenge proof of work page doesn't need to be presented to users that are much more likely to be legitimate.</p>
-			<p>Please note that Anubis requires the use of modern JavaScript features that plugins like <a href="https://jshelter.org/">JShelter</a> will disable. Please disable JShelter or other such plugins for this domain.</p>
-		</details>
-		<noscript>
-			<p>
-				Sadly, you must enable JavaScript to get past this challenge. This is required because AI companies have changed
-				the social contract around how website hosting works. A no-JS solution is a work-in-progress.
-			</p>
-		</noscript>
-		<div id="testarea"></div>
-	</div>
-}
-
-templ errorPage(message string) {
-	<div class="centered-div">
-		<img
-			id="image"
-			style="width:100%;max-width:256px;"
-			src={ "/.within.website/x/cmd/anubis/static/img/sad.webp?cacheBuster=" + anubis.Version }
-		/>
-		<p>{ message }.</p>
-		<button onClick="window.location.reload();">Try again</button>
-		<p><a href="/">Go home</a></p>
-	</div>
-}

cmd/anubis/index_templ.go

@@ -1,225 +0,0 @@
-// Code generated by templ - DO NOT EDIT.
-
-// templ: version: v0.3.833
-package main
-
-//lint:file-ignore SA4006 This context is only used if a nested component is present.
-
-import "github.com/a-h/templ"
-import templruntime "github.com/a-h/templ/runtime"
-
-import (
-	"github.com/TecharoHQ/anubis"
-	"github.com/TecharoHQ/anubis/xess"
-)
-
-func base(title string, body templ.Component) templ.Component {
-	return templruntime.GeneratedTemplate(func(templ_7745c5c3_Input templruntime.GeneratedComponentInput) (templ_7745c5c3_Err error) {
-		templ_7745c5c3_W, ctx := templ_7745c5c3_Input.Writer, templ_7745c5c3_Input.Context
-		if templ_7745c5c3_CtxErr := ctx.Err(); templ_7745c5c3_CtxErr != nil {
-			return templ_7745c5c3_CtxErr
-		}
-		templ_7745c5c3_Buffer, templ_7745c5c3_IsBuffer := templruntime.GetBuffer(templ_7745c5c3_W)
-		if !templ_7745c5c3_IsBuffer {
-			defer func() {
-				templ_7745c5c3_BufErr := templruntime.ReleaseBuffer(templ_7745c5c3_Buffer)
-				if templ_7745c5c3_Err == nil {
-					templ_7745c5c3_Err = templ_7745c5c3_BufErr
-				}
-			}()
-		}
-		ctx = templ.InitializeContext(ctx)
-		templ_7745c5c3_Var1 := templ.GetChildren(ctx)
-		if templ_7745c5c3_Var1 == nil {
-			templ_7745c5c3_Var1 = templ.NopComponent
-		}
-		ctx = templ.ClearChildren(ctx)
-		templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 1, "<!doctype html><html><head><title>")
-		if templ_7745c5c3_Err != nil {
-			return templ_7745c5c3_Err
-		}
-		var templ_7745c5c3_Var2 string
-		templ_7745c5c3_Var2, templ_7745c5c3_Err = templ.JoinStringErrs(title)
-		if templ_7745c5c3_Err != nil {
-			return templ.Error{Err: templ_7745c5c3_Err, FileName: `index.templ`, Line: 12, Col: 17}
-		}
-		_, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ.EscapeString(templ_7745c5c3_Var2))
-		if templ_7745c5c3_Err != nil {
-			return templ_7745c5c3_Err
-		}
-		templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 2, "</title><link rel=\"stylesheet\" href=\"")
-		if templ_7745c5c3_Err != nil {
-			return templ_7745c5c3_Err
-		}
-		var templ_7745c5c3_Var3 string
-		templ_7745c5c3_Var3, templ_7745c5c3_Err = templ.JoinStringErrs(xess.URL)
-		if templ_7745c5c3_Err != nil {
-			return templ.Error{Err: templ_7745c5c3_Err, FileName: `index.templ`, Line: 13, Col: 41}
-		}
-		_, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ.EscapeString(templ_7745c5c3_Var3))
-		if templ_7745c5c3_Err != nil {
-			return templ_7745c5c3_Err
-		}
-		templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 3, "\"><meta name=\"viewport\" content=\"width=device-width, initial-scale=1.0\"><style>\n    body,\n    html {\n      height: 100%;\n      display: flex;\n      justify-content: center;\n      align-items: center;\n      margin-left: auto;\n      margin-right: auto;\n    }\n\n    .centered-div {\n      text-align: center;\n    }\n\n    .lds-roller,\n    .lds-roller div,\n    .lds-roller div:after {\n      box-sizing: border-box;\n    }\n\n    .lds-roller {\n      display: inline-block;\n      position: relative;\n      width: 80px;\n      height: 80px;\n    }\n\n    .lds-roller div {\n      animation: lds-roller 1.2s cubic-bezier(0.5, 0, 0.5, 1) infinite;\n      transform-origin: 40px 40px;\n    }\n\n    .lds-roller div:after {\n      content: \" \";\n      display: block;\n      position: absolute;\n      width: 7.2px;\n      height: 7.2px;\n      border-radius: 50%;\n      background: currentColor;\n      margin: -3.6px 0 0 -3.6px;\n    }\n\n    .lds-roller div:nth-child(1) {\n      animation-delay: -0.036s;\n    }\n\n    .lds-roller div:nth-child(1):after {\n      top: 62.62742px;\n      left: 62.62742px;\n    }\n\n    .lds-roller div:nth-child(2) {\n      animation-delay: -0.072s;\n    }\n\n    .lds-roller div:nth-child(2):after {\n      top: 67.71281px;\n      left: 56px;\n    }\n\n    .lds-roller div:nth-child(3) {\n      animation-delay: -0.108s;\n    }\n\n    .lds-roller div:nth-child(3):after {\n      top: 70.90963px;\n      left: 48.28221px;\n    }\n\n    .lds-roller div:nth-child(4) {\n      animation-delay: -0.144s;\n    }\n\n    .lds-roller div:nth-child(4):after {\n      top: 72px;\n      left: 40px;\n    }\n\n    .lds-roller div:nth-child(5) {\n      animation-delay: -0.18s;\n    }\n\n    .lds-roller div:nth-child(5):after {\n      top: 70.90963px;\n      left: 31.71779px;\n    }\n\n    .lds-roller div:nth-child(6) {\n      animation-delay: -0.216s;\n    }\n\n    .lds-roller div:nth-child(6):after {\n      top: 67.71281px;\n      left: 24px;\n    }\n\n    .lds-roller div:nth-child(7) {\n      animation-delay: -0.252s;\n    }\n\n    .lds-roller div:nth-child(7):after {\n      top: 62.62742px;\n      left: 17.37258px;\n    }\n\n    .lds-roller div:nth-child(8) {\n      animation-delay: -0.288s;\n    }\n\n    .lds-roller div:nth-child(8):after {\n      top: 56px;\n      left: 12.28719px;\n    }\n\n    @keyframes lds-roller {\n      0% {\n        transform: rotate(0deg);\n      }\n\n      100% {\n        transform: rotate(360deg);\n      }\n    }\n      </style>")
-		if templ_7745c5c3_Err != nil {
-			return templ_7745c5c3_Err
-		}
-		templ_7745c5c3_Err = templ.JSONScript("anubis_version", anubis.Version).Render(ctx, templ_7745c5c3_Buffer)
-		if templ_7745c5c3_Err != nil {
-			return templ_7745c5c3_Err
-		}
-		templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 4, "</head><body id=\"top\"><main><center><h1 id=\"title\" class=\".centered-div\">")
-		if templ_7745c5c3_Err != nil {
-			return templ_7745c5c3_Err
-		}
-		var templ_7745c5c3_Var4 string
-		templ_7745c5c3_Var4, templ_7745c5c3_Err = templ.JoinStringErrs(title)
-		if templ_7745c5c3_Err != nil {
-			return templ.Error{Err: templ_7745c5c3_Err, FileName: `index.templ`, Line: 146, Col: 49}
-		}
-		_, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ.EscapeString(templ_7745c5c3_Var4))
-		if templ_7745c5c3_Err != nil {
-			return templ_7745c5c3_Err
-		}
-		templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 5, "</h1></center>")
-		if templ_7745c5c3_Err != nil {
-			return templ_7745c5c3_Err
-		}
-		templ_7745c5c3_Err = body.Render(ctx, templ_7745c5c3_Buffer)
-		if templ_7745c5c3_Err != nil {
-			return templ_7745c5c3_Err
-		}
-		templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 6, "<footer><center><p>Protected by <a href=\"https://github.com/TecharoHQ/anubis\">Anubis</a> from <a href=\"https://techaro.lol\">Techaro</a>. Made with ❤️ in 🇨🇦.</p></center></footer></main></body></html>")
-		if templ_7745c5c3_Err != nil {
-			return templ_7745c5c3_Err
-		}
-		return nil
-	})
-}
-
-func index() templ.Component {
-	return templruntime.GeneratedTemplate(func(templ_7745c5c3_Input templruntime.GeneratedComponentInput) (templ_7745c5c3_Err error) {
-		templ_7745c5c3_W, ctx := templ_7745c5c3_Input.Writer, templ_7745c5c3_Input.Context
-		if templ_7745c5c3_CtxErr := ctx.Err(); templ_7745c5c3_CtxErr != nil {
-			return templ_7745c5c3_CtxErr
-		}
-		templ_7745c5c3_Buffer, templ_7745c5c3_IsBuffer := templruntime.GetBuffer(templ_7745c5c3_W)
-		if !templ_7745c5c3_IsBuffer {
-			defer func() {
-				templ_7745c5c3_BufErr := templruntime.ReleaseBuffer(templ_7745c5c3_Buffer)
-				if templ_7745c5c3_Err == nil {
-					templ_7745c5c3_Err = templ_7745c5c3_BufErr
-				}
-			}()
-		}
-		ctx = templ.InitializeContext(ctx)
-		templ_7745c5c3_Var5 := templ.GetChildren(ctx)
-		if templ_7745c5c3_Var5 == nil {
-			templ_7745c5c3_Var5 = templ.NopComponent
-		}
-		ctx = templ.ClearChildren(ctx)
-		templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 7, "<div class=\"centered-div\"><img id=\"image\" style=\"width:100%;max-width:256px;\" src=\"")
-		if templ_7745c5c3_Err != nil {
-			return templ_7745c5c3_Err
-		}
-		var templ_7745c5c3_Var6 string
-		templ_7745c5c3_Var6, templ_7745c5c3_Err = templ.JoinStringErrs("/.within.website/x/cmd/anubis/static/img/pensive.webp?cacheBuster=" +
-			anubis.Version)
-		if templ_7745c5c3_Err != nil {
-			return templ.Error{Err: templ_7745c5c3_Err, FileName: `index.templ`, Line: 169, Col: 18}
-		}
-		_, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ.EscapeString(templ_7745c5c3_Var6))
-		if templ_7745c5c3_Err != nil {
-			return templ_7745c5c3_Err
-		}
-		templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 8, "\"> <img style=\"display:none;\" style=\"width:100%;max-width:256px;\" src=\"")
-		if templ_7745c5c3_Err != nil {
-			return templ_7745c5c3_Err
-		}
-		var templ_7745c5c3_Var7 string
-		templ_7745c5c3_Var7, templ_7745c5c3_Err = templ.JoinStringErrs("/.within.website/x/cmd/anubis/static/img/happy.webp?cacheBuster=" +
-			anubis.Version)
-		if templ_7745c5c3_Err != nil {
-			return templ.Error{Err: templ_7745c5c3_Err, FileName: `index.templ`, Line: 175, Col: 18}
-		}
-		_, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ.EscapeString(templ_7745c5c3_Var7))
-		if templ_7745c5c3_Err != nil {
-			return templ_7745c5c3_Err
-		}
-		templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 9, "\"><p id=\"status\">Loading...</p><script async type=\"module\" src=\"")
-		if templ_7745c5c3_Err != nil {
-			return templ_7745c5c3_Err
-		}
-		var templ_7745c5c3_Var8 string
-		templ_7745c5c3_Var8, templ_7745c5c3_Err = templ.JoinStringErrs("/.within.website/x/cmd/anubis/static/js/main.mjs?cacheBuster=" + anubis.Version)
-		if templ_7745c5c3_Err != nil {
-			return templ.Error{Err: templ_7745c5c3_Err, FileName: `index.templ`, Line: 178, Col: 116}
-		}
-		_, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ.EscapeString(templ_7745c5c3_Var8))
-		if templ_7745c5c3_Err != nil {
-			return templ_7745c5c3_Err
-		}
-		templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 10, "\"></script><div id=\"spinner\" class=\"lds-roller\"><div></div><div></div><div></div><div></div><div></div><div></div><div></div><div></div></div><details><summary>Why am I seeing this?</summary><p>You are seeing this because the administrator of this website has set up <a href=\"https://github.com/TecharoHQ/anubis\">Anubis</a> to protect the server against the scourge of <a href=\"https://thelibre.news/foss-infrastructure-is-under-attack-by-ai-companies/\">AI companies aggressively scraping websites</a>. This can and does cause downtime for the websites, which makes their resources inaccessible for everyone.</p><p>Anubis is a compromise. Anubis uses a <a href=\"https://anubis.techaro.lol/docs/design/why-proof-of-work\">Proof-of-Work</a> scheme in the vein of <a href=\"https://en.wikipedia.org/wiki/Hashcash\">Hashcash</a>, a proposed proof-of-work scheme for reducing email spam. The idea is that at individual scales the additional load is ignorable, but at mass scraper levels it adds up and makes scraping much more expensive.</p><p>Ultimately, this is a hack whose real purpose is to give a \"good enough\" placeholder solution so that more time can be spent on fingerprinting and identifying headless browsers (EG: via how they do font rendering) so that the challenge proof of work page doesn't need to be presented to users that are much more likely to be legitimate.</p><p>Please note that Anubis requires the use of modern JavaScript features that plugins like <a href=\"https://jshelter.org/\">JShelter</a> will disable. Please disable JShelter or other such plugins for this domain.</p></details><noscript><p>Sadly, you must enable JavaScript to get past this challenge. This is required because AI companies have changed the social contract around how website hosting works. A no-JS solution is a work-in-progress.</p></noscript><div id=\"testarea\"></div></div>")
-		if templ_7745c5c3_Err != nil {
-			return templ_7745c5c3_Err
-		}
-		return nil
-	})
-}
-
-func errorPage(message string) templ.Component {
-	return templruntime.GeneratedTemplate(func(templ_7745c5c3_Input templruntime.GeneratedComponentInput) (templ_7745c5c3_Err error) {
-		templ_7745c5c3_W, ctx := templ_7745c5c3_Input.Writer, templ_7745c5c3_Input.Context
-		if templ_7745c5c3_CtxErr := ctx.Err(); templ_7745c5c3_CtxErr != nil {
-			return templ_7745c5c3_CtxErr
-		}
-		templ_7745c5c3_Buffer, templ_7745c5c3_IsBuffer := templruntime.GetBuffer(templ_7745c5c3_W)
-		if !templ_7745c5c3_IsBuffer {
-			defer func() {
-				templ_7745c5c3_BufErr := templruntime.ReleaseBuffer(templ_7745c5c3_Buffer)
-				if templ_7745c5c3_Err == nil {
-					templ_7745c5c3_Err = templ_7745c5c3_BufErr
-				}
-			}()
-		}
-		ctx = templ.InitializeContext(ctx)
-		templ_7745c5c3_Var9 := templ.GetChildren(ctx)
-		if templ_7745c5c3_Var9 == nil {
-			templ_7745c5c3_Var9 = templ.NopComponent
-		}
-		ctx = templ.ClearChildren(ctx)
-		templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 11, "<div class=\"centered-div\"><img id=\"image\" style=\"width:100%;max-width:256px;\" src=\"")
-		if templ_7745c5c3_Err != nil {
-			return templ_7745c5c3_Err
-		}
-		var templ_7745c5c3_Var10 string
-		templ_7745c5c3_Var10, templ_7745c5c3_Err = templ.JoinStringErrs("/.within.website/x/cmd/anubis/static/img/sad.webp?cacheBuster=" + anubis.Version)
-		if templ_7745c5c3_Err != nil {
-			return templ.Error{Err: templ_7745c5c3_Err, FileName: `index.templ`, Line: 211, Col: 90}
-		}
-		_, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ.EscapeString(templ_7745c5c3_Var10))
-		if templ_7745c5c3_Err != nil {
-			return templ_7745c5c3_Err
-		}
-		templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 12, "\"><p>")
-		if templ_7745c5c3_Err != nil {
-			return templ_7745c5c3_Err
-		}
-		var templ_7745c5c3_Var11 string
-		templ_7745c5c3_Var11, templ_7745c5c3_Err = templ.JoinStringErrs(message)
-		if templ_7745c5c3_Err != nil {
-			return templ.Error{Err: templ_7745c5c3_Err, FileName: `index.templ`, Line: 213, Col: 14}
-		}
-		_, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ.EscapeString(templ_7745c5c3_Var11))
-		if templ_7745c5c3_Err != nil {
-			return templ_7745c5c3_Err
-		}
-		templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 13, ".</p><button onClick=\"window.location.reload();\">Try again</button><p><a href=\"/\">Go home</a></p></div>")
-		if templ_7745c5c3_Err != nil {
-			return templ_7745c5c3_Err
-		}
-		return nil
-	})
-}
-
-var _ = templruntime.GeneratedTemplate

cmd/anubis/internal/config/config.go

@@ -1,162 +0,0 @@
-package config
-
-import (
-	"errors"
-	"fmt"
-	"net"
-	"regexp"
-)
-
-type Rule string
-
-const (
-	RuleUnknown   Rule = ""
-	RuleAllow     Rule = "ALLOW"
-	RuleDeny      Rule = "DENY"
-	RuleChallenge Rule = "CHALLENGE"
-)
-
-type Algorithm string
-
-const (
-	AlgorithmUnknown Algorithm = ""
-	AlgorithmFast    Algorithm = "fast"
-	AlgorithmSlow    Algorithm = "slow"
-)
-
-type Bot struct {
-	Name           string          `json:"name"`
-	UserAgentRegex *string         `json:"user_agent_regex"`
-	PathRegex      *string         `json:"path_regex"`
-	Action         Rule            `json:"action"`
-	RemoteAddr     []string        `json:"remote_addresses"`
-	Challenge      *ChallengeRules `json:"challenge,omitempty"`
-}
-
-var (
-	ErrNoBotRulesDefined                 = errors.New("config: must define at least one (1) bot rule")
-	ErrBotMustHaveName                   = errors.New("config.Bot: must set name")
-	ErrBotMustHaveUserAgentOrPath        = errors.New("config.Bot: must set either user_agent_regex, path_regex, or remote_addresses")
-	ErrBotMustHaveUserAgentOrPathNotBoth = errors.New("config.Bot: must set either user_agent_regex, path_regex, and not both")
-	ErrUnknownAction                     = errors.New("config.Bot: unknown action")
-	ErrInvalidUserAgentRegex             = errors.New("config.Bot: invalid user agent regex")
-	ErrInvalidPathRegex                  = errors.New("config.Bot: invalid path regex")
-	ErrInvalidCIDR                       = errors.New("config.Bot: invalid CIDR")
-)
-
-func (b Bot) Valid() error {
-	var errs []error
-
-	if b.Name == "" {
-		errs = append(errs, ErrBotMustHaveName)
-	}
-
-	if b.UserAgentRegex == nil && b.PathRegex == nil && (b.RemoteAddr == nil || len(b.RemoteAddr) == 0) {
-		errs = append(errs, ErrBotMustHaveUserAgentOrPath)
-	}
-
-	if b.UserAgentRegex != nil && b.PathRegex != nil {
-		errs = append(errs, ErrBotMustHaveUserAgentOrPathNotBoth)
-	}
-
-	if b.UserAgentRegex != nil {
-		if _, err := regexp.Compile(*b.UserAgentRegex); err != nil {
-			errs = append(errs, ErrInvalidUserAgentRegex, err)
-		}
-	}
-
-	if b.PathRegex != nil {
-		if _, err := regexp.Compile(*b.PathRegex); err != nil {
-			errs = append(errs, ErrInvalidPathRegex, err)
-		}
-	}
-
-	if b.RemoteAddr != nil && len(b.RemoteAddr) > 0 {
-		for _, cidr := range b.RemoteAddr {
-			if _, _, err := net.ParseCIDR(cidr); err != nil {
-				errs = append(errs, ErrInvalidCIDR, err)
-			}
-		}
-	}
-
-	switch b.Action {
-	case RuleAllow, RuleChallenge, RuleDeny:
-		// okay
-	default:
-		errs = append(errs, fmt.Errorf("%w: %q", ErrUnknownAction, b.Action))
-	}
-
-	if b.Action == RuleChallenge && b.Challenge != nil {
-		if err := b.Challenge.Valid(); err != nil {
-			errs = append(errs, err)
-		}
-	}
-
-	if len(errs) != 0 {
-		return fmt.Errorf("config: bot entry for %q is not valid:\n%w", b.Name, errors.Join(errs...))
-	}
-
-	return nil
-}
-
-type ChallengeRules struct {
-	Difficulty int       `json:"difficulty"`
-	ReportAs   int       `json:"report_as"`
-	Algorithm  Algorithm `json:"algorithm"`
-}
-
-var (
-	ErrChallengeRuleHasWrongAlgorithm = errors.New("config.Bot.ChallengeRules: algorithm is invalid")
-	ErrChallengeDifficultyTooLow      = errors.New("config.Bot.ChallengeRules: difficulty is too low (must be >= 1)")
-	ErrChallengeDifficultyTooHigh     = errors.New("config.Bot.ChallengeRules: difficulty is too high (must be <= 64)")
-)
-
-func (cr ChallengeRules) Valid() error {
-	var errs []error
-
-	if cr.Difficulty < 1 {
-		errs = append(errs, fmt.Errorf("%w, got: %d", ErrChallengeDifficultyTooLow, cr.Difficulty))
-	}
-
-	if cr.Difficulty > 64 {
-		errs = append(errs, fmt.Errorf("%w, got: %d", ErrChallengeDifficultyTooHigh, cr.Difficulty))
-	}
-
-	switch cr.Algorithm {
-	case AlgorithmFast, AlgorithmSlow, AlgorithmUnknown:
-		// do nothing, it's all good
-	default:
-		errs = append(errs, fmt.Errorf("%w: %q", ErrChallengeRuleHasWrongAlgorithm, cr.Algorithm))
-	}
-
-	if len(errs) != 0 {
-		return fmt.Errorf("config: challenge rules entry is not valid:\n%w", errors.Join(errs...))
-	}
-
-	return nil
-}
-
-type Config struct {
-	Bots  []Bot `json:"bots"`
-	DNSBL bool  `json:"dnsbl"`
-}
-
-func (c Config) Valid() error {
-	var errs []error
-
-	if len(c.Bots) == 0 {
-		errs = append(errs, ErrNoBotRulesDefined)
-	}
-
-	for _, b := range c.Bots {
-		if err := b.Valid(); err != nil {
-			errs = append(errs, err)
-		}
-	}
-
-	if len(errs) != 0 {
-		return fmt.Errorf("config is not valid:\n%w", errors.Join(errs...))
-	}
-
-	return nil
-}

cmd/anubis/js/main.mjs

@@ -1,89 +0,0 @@
-import processFast from "./proof-of-work.mjs";
-import processSlow from "./proof-of-work-slow.mjs";
-import { testVideo } from "./video.mjs";
-
-const algorithms = {
-  "fast": processFast,
-  "slow": processSlow,
-}
-
-// from Xeact
-const u = (url = "", params = {}) => {
-  let result = new URL(url, window.location.href);
-  Object.entries(params).forEach((kv) => {
-    let [k, v] = kv;
-    result.searchParams.set(k, v);
-  });
-  return result.toString();
-};
-
-const imageURL = (mood, cacheBuster) =>
-  u(`/.within.website/x/cmd/anubis/static/img/${mood}.webp`, { cacheBuster });
-
-(async () => {
-  const status = document.getElementById('status');
-  const image = document.getElementById('image');
-  const title = document.getElementById('title');
-  const spinner = document.getElementById('spinner');
-  const anubisVersion = JSON.parse(document.getElementById('anubis_version').textContent);
-
-  // const testarea = document.getElementById('testarea');
-
-  // const videoWorks = await testVideo(testarea);
-  // console.log(`videoWorks: ${videoWorks}`);
-
-  // if (!videoWorks) {
-  //   title.innerHTML = "Oh no!";
-  //   status.innerHTML = "Checks failed. Please check your browser's settings and try again.";
-  //   image.src = imageURL("sad");
-  //   spinner.innerHTML = "";
-  //   spinner.style.display = "none";
-  //   return;
-  // }
-
-  status.innerHTML = 'Calculating...';
-
-  const { challenge, rules } = await fetch("/.within.website/x/cmd/anubis/api/make-challenge", { method: "POST" })
-    .then(r => {
-      if (!r.ok) {
-        throw new Error("Failed to fetch config");
-      }
-      return r.json();
-    })
-    .catch(err => {
-      title.innerHTML = "Oh no!";
-      status.innerHTML = `Failed to fetch config: ${err.message}`;
-      image.src = imageURL("sad", anubisVersion);
-      spinner.innerHTML = "";
-      spinner.style.display = "none";
-      throw err;
-    });
-
-  const process = algorithms[rules.algorithm];
-  if (!process) {
-    title.innerHTML = "Oh no!";
-    status.innerHTML = `Failed to resolve check algorithm. You may want to reload the page.`;
-    image.src = imageURL("sad", anubisVersion);
-    spinner.innerHTML = "";
-    spinner.style.display = "none";
-    return;
-  }
-
-  status.innerHTML = `Calculating...<br/>Difficulty: ${rules.report_as}`;
-
-  const t0 = Date.now();
-  const { hash, nonce } = await process(challenge, rules.difficulty);
-  const t1 = Date.now();
-  console.log({ hash, nonce });
-
-  title.innerHTML = "Success!";
-  status.innerHTML = `Done! Took ${t1 - t0}ms, ${nonce} iterations`;
-  image.src = imageURL("happy", anubisVersion);
-  spinner.innerHTML = "";
-  spinner.style.display = "none";
-
-  setTimeout(() => {
-    const redir = window.location.href;
-    window.location.href = u("/.within.website/x/cmd/anubis/api/pass-challenge", { response: hash, nonce, redir, elapsedTime: t1 - t0 });
-  }, 250);
-})();

cmd/anubis/main.go

@@ -1,27 +1,25 @@
 package main
 
 import (
+	"bytes"
 	"context"
 	"crypto/ed25519"
 	"crypto/rand"
-	"crypto/sha256"
-	"crypto/subtle"
 	"embed"
 	"encoding/hex"
-	"encoding/json"
+	"errors"
 	"flag"
 	"fmt"
-	"io"
+	"io/fs"
 	"log"
 	"log/slog"
-	"math"
-	mrand "math/rand"
 	"net"
 	"net/http"
 	"net/http/httputil"
 	"net/url"
 	"os"
 	"os/signal"
+	"path/filepath"
 	"strconv"
 	"strings"
 	"sync"
@@ -29,76 +27,57 @@ import (
 	"time"
 
 	"github.com/TecharoHQ/anubis"
-	"github.com/TecharoHQ/anubis/cmd/anubis/internal/config"
-	"github.com/TecharoHQ/anubis/cmd/anubis/internal/dnsbl"
+	"github.com/TecharoHQ/anubis/data"
 	"github.com/TecharoHQ/anubis/internal"
-	"github.com/TecharoHQ/anubis/xess"
-	"github.com/a-h/templ"
+	libanubis "github.com/TecharoHQ/anubis/lib"
+	botPolicy "github.com/TecharoHQ/anubis/lib/policy"
+	"github.com/TecharoHQ/anubis/lib/policy/config"
+	"github.com/TecharoHQ/anubis/web"
 	"github.com/facebookgo/flagenv"
-	"github.com/golang-jwt/jwt/v5"
-	"github.com/prometheus/client_golang/prometheus"
-	"github.com/prometheus/client_golang/prometheus/promauto"
 	"github.com/prometheus/client_golang/prometheus/promhttp"
 )
 
 var (
-	bind                = flag.String("bind", ":8923", "network address to bind HTTP to")
-	bindNetwork         = flag.String("bind-network", "tcp", "network family to bind HTTP to, e.g. unix, tcp")
-	challengeDifficulty = flag.Int("difficulty", defaultDifficulty, "difficulty of the challenge")
-	metricsBind         = flag.String("metrics-bind", ":9090", "network address to bind metrics to")
-	metricsBindNetwork  = flag.String("metrics-bind-network", "tcp", "network family for the metrics server to bind to")
-	socketMode          = flag.String("socket-mode", "0770", "socket mode (permissions) for unix domain sockets.")
-	robotsTxt           = flag.Bool("serve-robots-txt", false, "serve a robots.txt file that disallows all robots")
-	policyFname         = flag.String("policy-fname", "", "full path to anubis policy document (defaults to a sensible built-in policy)")
-	slogLevel           = flag.String("slog-level", "INFO", "logging level (see https://pkg.go.dev/log/slog#hdr-Levels)")
-	target              = flag.String("target", "http://localhost:3923", "target to reverse proxy to")
-	healthcheck         = flag.Bool("healthcheck", false, "run a health check against Anubis")
-	debugXRealIPDefault = flag.String("debug-x-real-ip-default", "", "If set, replace empty X-Real-Ip headers with this value, useful only for debugging Anubis and running it locally")
-
-	//go:embed static botPolicies.json
-	static embed.FS
-
-	challengesIssued = promauto.NewCounter(prometheus.CounterOpts{
-		Name: "anubis_challenges_issued",
-		Help: "The total number of challenges issued",
-	})
-
-	challengesValidated = promauto.NewCounter(prometheus.CounterOpts{
-		Name: "anubis_challenges_validated",
-		Help: "The total number of challenges validated",
-	})
-
-	droneBLHits = promauto.NewCounterVec(prometheus.CounterOpts{
-		Name: "anubis_dronebl_hits",
-		Help: "The total number of hits from DroneBL",
-	}, []string{"status"})
-
-	failedValidations = promauto.NewCounter(prometheus.CounterOpts{
-		Name: "anubis_failed_validations",
-		Help: "The total number of failed validations",
-	})
-
-	timeTaken = promauto.NewHistogram(prometheus.HistogramOpts{
-		Name:    "anubis_time_taken",
-		Help:    "The time taken for a browser to generate a response (milliseconds)",
-		Buckets: prometheus.ExponentialBucketsRange(1, math.Pow(2, 18), 19),
-	})
+	basePrefix               = flag.String("base-prefix", "", "base prefix (root URL) the application is served under e.g. /myapp")
+	bind                     = flag.String("bind", ":8923", "network address to bind HTTP to")
+	bindNetwork              = flag.String("bind-network", "tcp", "network family to bind HTTP to, e.g. unix, tcp")
+	challengeDifficulty      = flag.Int("difficulty", anubis.DefaultDifficulty, "difficulty of the challenge")
+	cookieDomain             = flag.String("cookie-domain", "", "if set, the top-level domain that the Anubis cookie will be valid for")
+	cookiePartitioned        = flag.Bool("cookie-partitioned", false, "if true, sets the partitioned flag on Anubis cookies, enabling CHIPS support")
+	ed25519PrivateKeyHex     = flag.String("ed25519-private-key-hex", "", "private key used to sign JWTs, if not set a random one will be assigned")
+	ed25519PrivateKeyHexFile = flag.String("ed25519-private-key-hex-file", "", "file name containing value for ed25519-private-key-hex")
+	metricsBind              = flag.String("metrics-bind", ":9090", "network address to bind metrics to")
+	metricsBindNetwork       = flag.String("metrics-bind-network", "tcp", "network family for the metrics server to bind to")
+	socketMode               = flag.String("socket-mode", "0770", "socket mode (permissions) for unix domain sockets.")
+	robotsTxt                = flag.Bool("serve-robots-txt", false, "serve a robots.txt file that disallows all robots")
+	policyFname              = flag.String("policy-fname", "", "full path to anubis policy document (defaults to a sensible built-in policy)")
+	redirectDomains          = flag.String("redirect-domains", "", "list of domains separated by commas which anubis is allowed to redirect to. Leaving this unset allows any domain.")
+	slogLevel                = flag.String("slog-level", "INFO", "logging level (see https://pkg.go.dev/log/slog#hdr-Levels)")
+	target                   = flag.String("target", "http://localhost:3923", "target to reverse proxy to, set to an empty string to disable proxying when only using auth request")
+	healthcheck              = flag.Bool("healthcheck", false, "run a health check against Anubis")
+	useRemoteAddress         = flag.Bool("use-remote-address", false, "read the client's IP address from the network request, useful for debugging and running Anubis on bare metal")
+	debugBenchmarkJS         = flag.Bool("debug-benchmark-js", false, "respond to every request with a challenge for benchmarking hashrate")
+	ogPassthrough            = flag.Bool("og-passthrough", false, "enable Open Graph tag passthrough")
+	ogTimeToLive             = flag.Duration("og-expiry-time", 24*time.Hour, "Open Graph tag cache expiration time")
+	extractResources         = flag.String("extract-resources", "", "if set, extract the static resources to the specified folder")
+	webmasterEmail           = flag.String("webmaster-email", "", "if set, displays webmaster's email on the reject page for appeals")
 )
 
-const (
-	cookieName        = "within.website-x-cmd-anubis-auth"
-	staticPath        = "/.within.website/x/cmd/anubis/"
-	defaultDifficulty = 4
-)
+func keyFromHex(value string) (ed25519.PrivateKey, error) {
+	keyBytes, err := hex.DecodeString(value)
+	if err != nil {
+		return nil, fmt.Errorf("supplied key is not hex-encoded: %w", err)
+	}
 
-//go:generate go tool github.com/a-h/templ/cmd/templ generate
-//go:generate esbuild js/main.mjs --sourcemap --bundle --minify --outfile=static/js/main.mjs
-//go:generate gzip -f -k static/js/main.mjs
-//go:generate zstd -f -k --ultra -22 static/js/main.mjs
-//go:generate brotli -fZk static/js/main.mjs
+	if len(keyBytes) != ed25519.SeedSize {
+		return nil, fmt.Errorf("supplied key is not %d bytes long, got %d bytes", ed25519.SeedSize, len(keyBytes))
+	}
+
+	return ed25519.NewKeyFromSeed(keyBytes), nil
+}
 
 func doHealthCheck() error {
-	resp, err := http.Get("http://localhost" + *metricsBind + "/metrics")
+	resp, err := http.Get("http://localhost" + *metricsBind + anubis.BasePrefix + "/metrics")
 	if err != nil {
 		return fmt.Errorf("failed to fetch metrics: %w", err)
 	}
@@ -117,7 +96,11 @@ func setupListener(network string, address string) (net.Listener, string) {
 	case "unix":
 		formattedAddress = "unix:" + address
 	case "tcp":
-		formattedAddress = "http://localhost" + address
+		if strings.HasPrefix(address, ":") { // assume it's just a port e.g. :4259
+			formattedAddress = "http://localhost" + address
+		} else {
+			formattedAddress = "http://" + address
+		}
 	default:
 		formattedAddress = fmt.Sprintf(`(%s) %s`, network, address)
 	}
@@ -137,7 +120,10 @@ func setupListener(network string, address string) (net.Listener, string) {
 
 		err = os.Chmod(address, os.FileMode(mode))
 		if err != nil {
-			listener.Close()
+			err := listener.Close()
+			if err != nil {
+				log.Printf("failed to close listener: %v", err)
+			}
 			log.Fatal(fmt.Errorf("could not change socket mode: %w", err))
 		}
 	}
@@ -145,58 +131,162 @@ func setupListener(network string, address string) (net.Listener, string) {
 	return listener, formattedAddress
 }
 
+func makeReverseProxy(target string) (http.Handler, error) {
+	targetUri, err := url.Parse(target)
+	if err != nil {
+		return nil, fmt.Errorf("failed to parse target URL: %w", err)
+	}
+
+	transport := http.DefaultTransport.(*http.Transport).Clone()
+
+	// https://github.com/oauth2-proxy/oauth2-proxy/blob/4e2100a2879ef06aea1411790327019c1a09217c/pkg/upstream/http.go#L124
+	if targetUri.Scheme == "unix" {
+		// clean path up so we don't use the socket path in proxied requests
+		addr := targetUri.Path
+		targetUri.Path = ""
+		// tell transport how to dial unix sockets
+		transport.DialContext = func(ctx context.Context, _, _ string) (net.Conn, error) {
+			dialer := net.Dialer{}
+			return dialer.DialContext(ctx, "unix", addr)
+		}
+		// tell transport how to handle the unix url scheme
+		transport.RegisterProtocol("unix", libanubis.UnixRoundTripper{Transport: transport})
+	}
+
+	rp := httputil.NewSingleHostReverseProxy(targetUri)
+	rp.Transport = transport
+
+	return rp, nil
+}
+
+func startDecayMapCleanup(ctx context.Context, s *libanubis.Server) {
+	ticker := time.NewTicker(1 * time.Hour)
+	defer ticker.Stop()
+
+	for {
+		select {
+		case <-ticker.C:
+			s.CleanupDecayMap()
+		case <-ctx.Done():
+			return
+		}
+	}
+}
+
 func main() {
 	flagenv.Parse()
 	flag.Parse()
 
 	internal.InitSlog(*slogLevel)
 
-	if *healthcheck {
-		if err := doHealthCheck(); err != nil {
+	if *extractResources != "" {
+		if err := extractEmbedFS(data.BotPolicies, ".", *extractResources); err != nil {
 			log.Fatal(err)
 		}
+		if err := extractEmbedFS(web.Static, "static", *extractResources); err != nil {
+			log.Fatal(err)
+		}
+		fmt.Printf("Extracted embedded static files to %s\n", *extractResources)
 		return
 	}
 
-	s, err := New(*target, *policyFname)
+	var rp http.Handler
+	// when using anubis via Systemd and environment variables, then it is not possible to set targe to an empty string but only to space
+	if strings.TrimSpace(*target) != "" {
+		var err error
+		rp, err = makeReverseProxy(*target)
+		if err != nil {
+			log.Fatalf("can't make reverse proxy: %v", err)
+		}
+	}
+
+	policy, err := libanubis.LoadPoliciesOrDefault(*policyFname, *challengeDifficulty)
 	if err != nil {
-		log.Fatal(err)
+		log.Fatalf("can't parse policy file: %v", err)
 	}
 
 	fmt.Println("Rule error IDs:")
-	for _, rule := range s.policy.Bots {
+	for _, rule := range policy.Bots {
 		if rule.Action != config.RuleDeny {
 			continue
 		}
 
-		hash, err := rule.Hash()
-		if err != nil {
-			log.Fatalf("can't calculate checksum of rule %s: %v", rule.Name, err)
-		}
-
+		hash := rule.Hash()
 		fmt.Printf("* %s: %s\n", rule.Name, hash)
 	}
 	fmt.Println()
 
-	mux := http.NewServeMux()
-	xess.Mount(mux)
+	// replace the bot policy rules with a single rule that always benchmarks
+	if *debugBenchmarkJS {
+		policy.Bots = []botPolicy.Bot{{
+			Name:   "",
+			Rules:  botPolicy.NewHeaderExistsChecker("User-Agent"),
+			Action: config.RuleBenchmark,
+		}}
+	}
+	if *basePrefix != "" && !strings.HasPrefix(*basePrefix, "/") {
+		log.Fatalf("[misconfiguration] base-prefix must start with a slash, eg: /%s", *basePrefix)
+	} else if strings.HasSuffix(*basePrefix, "/") {
+		log.Fatalf("[misconfiguration] base-prefix must not end with a slash")
+	}
 
-	mux.Handle(staticPath, internal.UnchangingCache(http.StripPrefix(staticPath, http.FileServerFS(static))))
+	var priv ed25519.PrivateKey
+	if *ed25519PrivateKeyHex != "" && *ed25519PrivateKeyHexFile != "" {
+		log.Fatal("do not specify both ED25519_PRIVATE_KEY_HEX and ED25519_PRIVATE_KEY_HEX_FILE")
+	} else if *ed25519PrivateKeyHex != "" {
+		priv, err = keyFromHex(*ed25519PrivateKeyHex)
+		if err != nil {
+			log.Fatalf("failed to parse and validate ED25519_PRIVATE_KEY_HEX: %v", err)
+		}
+	} else if *ed25519PrivateKeyHexFile != "" {
+		hexFile, err := os.ReadFile(*ed25519PrivateKeyHexFile)
+		if err != nil {
+			log.Fatalf("failed to read ED25519_PRIVATE_KEY_HEX_FILE %s: %v", *ed25519PrivateKeyHexFile, err)
+		}
 
-	// mux.HandleFunc("GET /.within.website/x/cmd/anubis/static/js/main.mjs", serveMainJSWithBestEncoding)
+		priv, err = keyFromHex(string(bytes.TrimSpace(hexFile)))
+		if err != nil {
+			log.Fatalf("failed to parse and validate content of ED25519_PRIVATE_KEY_HEX_FILE: %v", err)
+		}
+	} else {
+		_, priv, err = ed25519.GenerateKey(rand.Reader)
+		if err != nil {
+			log.Fatalf("failed to generate ed25519 key: %v", err)
+		}
 
-	mux.HandleFunc("POST /.within.website/x/cmd/anubis/api/make-challenge", s.makeChallenge)
-	mux.HandleFunc("GET /.within.website/x/cmd/anubis/api/pass-challenge", s.passChallenge)
-	mux.HandleFunc("GET /.within.website/x/cmd/anubis/api/test-error", s.testError)
+		slog.Warn("generating random key, Anubis will have strange behavior when multiple instances are behind the same load balancer target, for more information: see https://anubis.techaro.lol/docs/admin/installation#key-generation")
+	}
 
-	if *robotsTxt {
-		mux.HandleFunc("/robots.txt", func(w http.ResponseWriter, r *http.Request) {
-			http.ServeFileFS(w, r, static, "static/robots.txt")
-		})
+	var redirectDomainsList []string
+	if *redirectDomains != "" {
+		domains := strings.Split(*redirectDomains, ",")
+		for _, domain := range domains {
+			_, err = url.Parse(domain)
+			if err != nil {
+				log.Fatalf("cannot parse redirect-domain %q: %s", domain, err.Error())
+			}
+			redirectDomainsList = append(redirectDomainsList, strings.TrimSpace(domain))
+		}
+	} else {
+		slog.Warn("REDIRECT_DOMAINS is not set, Anubis will only redirect to the same domain a request is coming from, see https://anubis.techaro.lol/docs/admin/configuration/redirect-domains")
+	}
 
-		mux.HandleFunc("/.well-known/robots.txt", func(w http.ResponseWriter, r *http.Request) {
-			http.ServeFileFS(w, r, static, "static/robots.txt")
-		})
+	s, err := libanubis.New(libanubis.Options{
+		BasePrefix:        *basePrefix,
+		Next:              rp,
+		Policy:            policy,
+		ServeRobotsTXT:    *robotsTxt,
+		PrivateKey:        priv,
+		CookieDomain:      *cookieDomain,
+		CookiePartitioned: *cookiePartitioned,
+		OGPassthrough:     *ogPassthrough,
+		OGTimeToLive:      *ogTimeToLive,
+		RedirectDomains:   redirectDomainsList,
+		Target:            *target,
+		WebmasterEmail:    *webmasterEmail,
+	})
+	if err != nil {
+		log.Fatalf("can't construct libanubis.Server: %v", err)
 	}
 
 	wg := new(sync.WaitGroup)
@@ -208,24 +298,28 @@ func main() {
 		wg.Add(1)
 		go metricsServer(ctx, wg.Done)
 	}
-
-	mux.HandleFunc("/", s.maybeReverseProxy)
+	go startDecayMapCleanup(ctx, s)
 
 	var h http.Handler
-	h = mux
-	h = internal.DefaultXRealIP(*debugXRealIPDefault, h)
+	h = s
+	h = internal.RemoteXRealIP(*useRemoteAddress, *bindNetwork, h)
 	h = internal.XForwardedForToXRealIP(h)
+	h = internal.XForwardedForUpdate(h)
 
 	srv := http.Server{Handler: h}
-	listener, url := setupListener(*bindNetwork, *bind)
+	listener, listenerUrl := setupListener(*bindNetwork, *bind)
 	slog.Info(
 		"listening",
-		"url", url,
+		"url", listenerUrl,
 		"difficulty", *challengeDifficulty,
 		"serveRobotsTXT", *robotsTxt,
 		"target", *target,
 		"version", anubis.Version,
-		"debug-x-real-ip-default", *debugXRealIPDefault,
+		"use-remote-address", *useRemoteAddress,
+		"debug-benchmark-js", *debugBenchmarkJS,
+		"og-passthrough", *ogPassthrough,
+		"og-expiry-time", *ogTimeToLive,
+		"base-prefix", *basePrefix,
 	)
 
 	go func() {
@@ -237,7 +331,7 @@ func main() {
 		}
 	}()
 
-	if err := srv.Serve(listener); err != http.ErrServerClosed {
+	if err := srv.Serve(listener); !errors.Is(err, http.ErrServerClosed) {
 		log.Fatal(err)
 	}
 	wg.Wait()
@@ -247,11 +341,19 @@ func metricsServer(ctx context.Context, done func()) {
 	defer done()
 
 	mux := http.NewServeMux()
-	mux.Handle("/metrics", promhttp.Handler())
+	mux.Handle(anubis.BasePrefix+"/metrics", promhttp.Handler())
 
 	srv := http.Server{Handler: mux}
-	listener, url := setupListener(*metricsBindNetwork, *metricsBind)
-	slog.Debug("listening for metrics", "url", url)
+	listener, metricsUrl := setupListener(*metricsBindNetwork, *metricsBind)
+	slog.Debug("listening for metrics", "url", metricsUrl)
+
+	if *healthcheck {
+		log.Println("running healthcheck")
+		if err := doHealthCheck(); err != nil {
+			log.Fatal(err)
+		}
+		return
+	}
 
 	go func() {
 		<-ctx.Done()
@@ -262,450 +364,33 @@ func metricsServer(ctx context.Context, done func()) {
 		}
 	}()
 
-	if err := srv.Serve(listener); err != http.ErrServerClosed {
+	if err := srv.Serve(listener); !errors.Is(err, http.ErrServerClosed) {
 		log.Fatal(err)
 	}
 }
 
-func sha256sum(text string) string {
-	hash := sha256.New()
-	hash.Write([]byte(text))
-	return hex.EncodeToString(hash.Sum(nil))
-}
-
-func (s *Server) challengeFor(r *http.Request, difficulty int) string {
-	fp := sha256.Sum256(s.priv.Seed())
-
-	data := fmt.Sprintf(
-		"Accept-Language=%s,X-Real-IP=%s,User-Agent=%s,WeekTime=%s,Fingerprint=%x,Difficulty=%d",
-		r.Header.Get("Accept-Language"),
-		r.Header.Get("X-Real-Ip"),
-		r.UserAgent(),
-		time.Now().UTC().Round(24*7*time.Hour).Format(time.RFC3339),
-		fp,
-		difficulty,
-	)
-	return sha256sum(data)
-}
-
-func New(target, policyFname string) (*Server, error) {
-	u, err := url.Parse(target)
-	if err != nil {
-		return nil, fmt.Errorf("failed to parse target URL: %w", err)
-	}
-
-	pub, priv, err := ed25519.GenerateKey(rand.Reader)
-	if err != nil {
-		return nil, fmt.Errorf("failed to generate ed25519 key: %w", err)
-	}
-
-	transport := http.DefaultTransport.(*http.Transport).Clone()
-
-	// https://github.com/oauth2-proxy/oauth2-proxy/blob/4e2100a2879ef06aea1411790327019c1a09217c/pkg/upstream/http.go#L124
-	if u.Scheme == "unix" {
-		// clean path up so we don't use the socket path in proxied requests
-		addr := u.Path
-		u.Path = ""
-		// tell transport how to dial unix sockets
-		transport.DialContext = func(ctx context.Context, _, _ string) (net.Conn, error) {
-			dialer := net.Dialer{}
-			return dialer.DialContext(ctx, "unix", addr)
-		}
-		// tell transport how to handle the unix url scheme
-		transport.RegisterProtocol("unix", unixRoundTripper{Transport: transport})
-	}
-
-	rp := httputil.NewSingleHostReverseProxy(u)
-	rp.Transport = transport
-
-	var fin io.ReadCloser
-
-	if policyFname != "" {
-		fin, err = os.Open(policyFname)
+func extractEmbedFS(fsys embed.FS, root string, destDir string) error {
+	return fs.WalkDir(fsys, root, func(path string, d fs.DirEntry, err error) error {
 		if err != nil {
-			return nil, fmt.Errorf("can't parse policy file %s: %w", policyFname, err)
+			return err
 		}
-	} else {
-		policyFname = "(static)/botPolicies.json"
-		fin, err = static.Open("botPolicies.json")
+
+		relPath, err := filepath.Rel(root, path)
 		if err != nil {
-			return nil, fmt.Errorf("[unexpected] can't parse builtin policy file %s: %w", policyFname, err)
-		}
-	}
-
-	defer fin.Close()
-
-	policy, err := parseConfig(fin, policyFname, *challengeDifficulty)
-	if err != nil {
-		return nil, err // parseConfig sets a fancy error for us
-	}
-
-	return &Server{
-		rp:         rp,
-		priv:       priv,
-		pub:        pub,
-		policy:     policy,
-		dnsblCache: NewDecayMap[string, dnsbl.DroneBLResponse](),
-	}, nil
-}
-
-// https://github.com/oauth2-proxy/oauth2-proxy/blob/master/pkg/upstream/http.go#L124
-type unixRoundTripper struct {
-	Transport *http.Transport
-}
-
-// set bare minimum stuff
-func (t unixRoundTripper) RoundTrip(req *http.Request) (*http.Response, error) {
-	req = req.Clone(req.Context())
-	if req.Host == "" {
-		req.Host = "localhost"
-	}
-	req.URL.Host = req.Host // proxy error: no Host in request URL
-	req.URL.Scheme = "http" // make http.Transport happy and avoid an infinite recursion
-	return t.Transport.RoundTrip(req)
-}
-
-type Server struct {
-	rp         *httputil.ReverseProxy
-	priv       ed25519.PrivateKey
-	pub        ed25519.PublicKey
-	policy     *ParsedConfig
-	dnsblCache *DecayMap[string, dnsbl.DroneBLResponse]
-}
-
-func (s *Server) maybeReverseProxy(w http.ResponseWriter, r *http.Request) {
-	lg := slog.With(
-		"user_agent", r.UserAgent(),
-		"accept_language", r.Header.Get("Accept-Language"),
-		"priority", r.Header.Get("Priority"),
-		"x-forwarded-for",
-		r.Header.Get("X-Forwarded-For"),
-		"x-real-ip", r.Header.Get("X-Real-Ip"),
-	)
-
-	cr, rule, err := s.check(r)
-	if err != nil {
-		lg.Error("check failed", "err", err)
-		templ.Handler(base("Oh noes!", errorPage("Internal Server Error: administrator has misconfigured Anubis. Please contact the administrator and ask them to look for the logs around \"maybeReverseProxy\"")), templ.WithStatus(http.StatusInternalServerError)).ServeHTTP(w, r)
-		return
-	}
-
-	r.Header.Add("X-Anubis-Rule", cr.Name)
-	r.Header.Add("X-Anubis-Action", string(cr.Rule))
-	lg = lg.With("check_result", cr)
-	policyApplications.WithLabelValues(cr.Name, string(cr.Rule)).Add(1)
-
-	ip := r.Header.Get("X-Real-Ip")
-
-	if s.policy.DNSBL && ip != "" {
-		resp, ok := s.dnsblCache.Get(ip)
-		if !ok {
-			lg.Debug("looking up ip in dnsbl")
-			resp, err := dnsbl.Lookup(ip)
-			if err != nil {
-				lg.Error("can't look up ip in dnsbl", "err", err)
-			}
-			s.dnsblCache.Set(ip, resp, 24*time.Hour)
-			droneBLHits.WithLabelValues(resp.String()).Inc()
+			return err
 		}
 
-		if resp != dnsbl.AllGood {
-			lg.Info("DNSBL hit", "status", resp.String())
-			templ.Handler(base("Oh noes!", errorPage(fmt.Sprintf("DroneBL reported an entry: %s, see https://dronebl.org/lookup?ip=%s", resp.String(), ip))), templ.WithStatus(http.StatusOK)).ServeHTTP(w, r)
-			return
-		}
-	}
+		destPath := filepath.Join(destDir, root, relPath)
 
-	switch cr.Rule {
-	case config.RuleAllow:
-		lg.Debug("allowing traffic to origin (explicit)")
-		s.rp.ServeHTTP(w, r)
-		return
-	case config.RuleDeny:
-		clearCookie(w)
-		lg.Info("explicit deny")
-		if rule == nil {
-			lg.Error("rule is nil, cannot calculate checksum")
-			templ.Handler(base("Oh noes!", errorPage("Other internal server error (contact the admin)")), templ.WithStatus(http.StatusInternalServerError)).ServeHTTP(w, r)
-			return
+		if d.IsDir() {
+			return os.MkdirAll(destPath, 0o700)
 		}
-		hash, err := rule.Hash()
+
+		data, err := fs.ReadFile(fsys, path)
 		if err != nil {
-			lg.Error("can't calculate checksum of rule", "err", err)
-			templ.Handler(base("Oh noes!", errorPage("Other internal server error (contact the admin)")), templ.WithStatus(http.StatusInternalServerError)).ServeHTTP(w, r)
-			return
+			return err
 		}
-		lg.Debug("rule hash", "hash", hash)
-		templ.Handler(base("Oh noes!", errorPage(fmt.Sprintf("Access Denied: error code %s", hash))), templ.WithStatus(http.StatusOK)).ServeHTTP(w, r)
-		return
-	case config.RuleChallenge:
-		lg.Debug("challenge requested")
-	default:
-		clearCookie(w)
-		templ.Handler(base("Oh noes!", errorPage("Other internal server error (contact the admin)")), templ.WithStatus(http.StatusInternalServerError)).ServeHTTP(w, r)
-		return
-	}
 
-	ckie, err := r.Cookie(cookieName)
-	if err != nil {
-		lg.Debug("cookie not found", "path", r.URL.Path)
-		clearCookie(w)
-		s.renderIndex(w, r)
-		return
-	}
-
-	if err := ckie.Valid(); err != nil {
-		lg.Debug("cookie is invalid", "err", err)
-		clearCookie(w)
-		s.renderIndex(w, r)
-		return
-	}
-
-	if time.Now().After(ckie.Expires) && !ckie.Expires.IsZero() {
-		lg.Debug("cookie expired", "path", r.URL.Path)
-		clearCookie(w)
-		s.renderIndex(w, r)
-		return
-	}
-
-	token, err := jwt.ParseWithClaims(ckie.Value, jwt.MapClaims{}, func(token *jwt.Token) (interface{}, error) {
-		return s.pub, nil
-	}, jwt.WithExpirationRequired(), jwt.WithStrictDecoding())
-
-	if err != nil || !token.Valid {
-		lg.Debug("invalid token", "path", r.URL.Path, "err", err)
-		clearCookie(w)
-		s.renderIndex(w, r)
-		return
-	}
-
-	if randomJitter() {
-		r.Header.Add("X-Anubis-Status", "PASS-BRIEF")
-		lg.Debug("cookie is not enrolled into secondary screening")
-		s.rp.ServeHTTP(w, r)
-		return
-	}
-
-	claims, ok := token.Claims.(jwt.MapClaims)
-	if !ok {
-		lg.Debug("invalid token claims type", "path", r.URL.Path)
-		clearCookie(w)
-		s.renderIndex(w, r)
-		return
-	}
-	challenge := s.challengeFor(r, rule.Challenge.Difficulty)
-
-	if claims["challenge"] != challenge {
-		lg.Debug("invalid challenge", "path", r.URL.Path)
-		clearCookie(w)
-		s.renderIndex(w, r)
-		return
-	}
-
-	var nonce int
-
-	if v, ok := claims["nonce"].(float64); ok {
-		nonce = int(v)
-	}
-
-	calcString := fmt.Sprintf("%s%d", challenge, nonce)
-	calculated := sha256sum(calcString)
-
-	if subtle.ConstantTimeCompare([]byte(claims["response"].(string)), []byte(calculated)) != 1 {
-		lg.Debug("invalid response", "path", r.URL.Path)
-		failedValidations.Inc()
-		clearCookie(w)
-		s.renderIndex(w, r)
-		return
-	}
-
-	slog.Debug("all checks passed")
-	r.Header.Add("X-Anubis-Status", "PASS-FULL")
-	s.rp.ServeHTTP(w, r)
-}
-
-func (s *Server) renderIndex(w http.ResponseWriter, r *http.Request) {
-	templ.Handler(
-		base("Making sure you're not a bot!", index()),
-	).ServeHTTP(w, r)
-}
-
-func (s *Server) makeChallenge(w http.ResponseWriter, r *http.Request) {
-	lg := slog.With("user_agent", r.UserAgent(), "accept_language", r.Header.Get("Accept-Language"), "priority", r.Header.Get("Priority"), "x-forwarded-for", r.Header.Get("X-Forwarded-For"), "x-real-ip", r.Header.Get("X-Real-Ip"))
-
-	cr, rule, err := s.check(r)
-	if err != nil {
-		lg.Error("check failed", "err", err)
-		w.WriteHeader(http.StatusInternalServerError)
-		json.NewEncoder(w).Encode(struct {
-			Error string `json:"error"`
-		}{
-			Error: "Internal Server Error: administrator has misconfigured Anubis. Please contact the administrator and ask them to look for the logs around \"makeChallenge\"",
-		})
-		return
-	}
-	lg = lg.With("check_result", cr)
-	challenge := s.challengeFor(r, rule.Challenge.Difficulty)
-
-	json.NewEncoder(w).Encode(struct {
-		Challenge string                 `json:"challenge"`
-		Rules     *config.ChallengeRules `json:"rules"`
-	}{
-		Challenge: challenge,
-		Rules:     rule.Challenge,
-	})
-	lg.Debug("made challenge", "challenge", challenge, "rules", rule.Challenge, "cr", cr)
-	challengesIssued.Inc()
-}
-
-func (s *Server) passChallenge(w http.ResponseWriter, r *http.Request) {
-	lg := slog.With(
-		"user_agent", r.UserAgent(),
-		"accept_language", r.Header.Get("Accept-Language"),
-		"priority", r.Header.Get("Priority"),
-		"x-forwarded-for", r.Header.Get("X-Forwarded-For"),
-		"x-real-ip", r.Header.Get("X-Real-Ip"),
-	)
-
-	cr, rule, err := s.check(r)
-	if err != nil {
-		lg.Error("check failed", "err", err)
-		templ.Handler(base("Oh noes!", errorPage("Internal Server Error: administrator has misconfigured Anubis. Please contact the administrator and ask them to look for the logs around \"passChallenge\".")), templ.WithStatus(http.StatusInternalServerError)).ServeHTTP(w, r)
-		return
-	}
-	lg = lg.With("check_result", cr)
-
-	nonceStr := r.FormValue("nonce")
-	if nonceStr == "" {
-		clearCookie(w)
-		lg.Debug("no nonce")
-		templ.Handler(base("Oh noes!", errorPage("missing nonce")), templ.WithStatus(http.StatusInternalServerError)).ServeHTTP(w, r)
-		return
-	}
-
-	elapsedTimeStr := r.FormValue("elapsedTime")
-	if elapsedTimeStr == "" {
-		clearCookie(w)
-		lg.Debug("no elapsedTime")
-		templ.Handler(base("Oh noes!", errorPage("missing elapsedTime")), templ.WithStatus(http.StatusInternalServerError)).ServeHTTP(w, r)
-		return
-	}
-
-	elapsedTime, err := strconv.ParseFloat(elapsedTimeStr, 64)
-	if err != nil {
-		clearCookie(w)
-		lg.Debug("elapsedTime doesn't parse", "err", err)
-		templ.Handler(base("Oh noes!", errorPage("invalid elapsedTime")), templ.WithStatus(http.StatusInternalServerError)).ServeHTTP(w, r)
-		return
-	}
-
-	lg.Info("challenge took", "elapsedTime", elapsedTime)
-	timeTaken.Observe(elapsedTime)
-
-	response := r.FormValue("response")
-	redir := r.FormValue("redir")
-
-	challenge := s.challengeFor(r, rule.Challenge.Difficulty)
-
-	nonce, err := strconv.Atoi(nonceStr)
-	if err != nil {
-		clearCookie(w)
-		lg.Debug("nonce doesn't parse", "err", err)
-		templ.Handler(base("Oh noes!", errorPage("invalid nonce")), templ.WithStatus(http.StatusInternalServerError)).ServeHTTP(w, r)
-		return
-	}
-
-	calcString := fmt.Sprintf("%s%d", challenge, nonce)
-	calculated := sha256sum(calcString)
-
-	if subtle.ConstantTimeCompare([]byte(response), []byte(calculated)) != 1 {
-		clearCookie(w)
-		lg.Debug("hash does not match", "got", response, "want", calculated)
-		templ.Handler(base("Oh noes!", errorPage("invalid response")), templ.WithStatus(http.StatusForbidden)).ServeHTTP(w, r)
-		failedValidations.Inc()
-		return
-	}
-
-	// compare the leading zeroes
-	if !strings.HasPrefix(response, strings.Repeat("0", *challengeDifficulty)) {
-		clearCookie(w)
-		lg.Debug("difficulty check failed", "response", response, "difficulty", *challengeDifficulty)
-		templ.Handler(base("Oh noes!", errorPage("invalid response")), templ.WithStatus(http.StatusForbidden)).ServeHTTP(w, r)
-		failedValidations.Inc()
-		return
-	}
-
-	// generate JWT cookie
-	token := jwt.NewWithClaims(jwt.SigningMethodEdDSA, jwt.MapClaims{
-		"challenge": challenge,
-		"nonce":     nonce,
-		"response":  response,
-		"iat":       time.Now().Unix(),
-		"nbf":       time.Now().Add(-1 * time.Minute).Unix(),
-		"exp":       time.Now().Add(24 * 7 * time.Hour).Unix(),
-	})
-	tokenString, err := token.SignedString(s.priv)
-	if err != nil {
-		lg.Error("failed to sign JWT", "err", err)
-		clearCookie(w)
-		templ.Handler(base("Oh noes!", errorPage("failed to sign JWT")), templ.WithStatus(http.StatusInternalServerError)).ServeHTTP(w, r)
-		return
-	}
-
-	http.SetCookie(w, &http.Cookie{
-		Name:     cookieName,
-		Value:    tokenString,
-		Expires:  time.Now().Add(24 * 7 * time.Hour),
-		SameSite: http.SameSiteLaxMode,
-		Path:     "/",
-	})
-
-	challengesValidated.Inc()
-	lg.Debug("challenge passed, redirecting to app")
-	http.Redirect(w, r, redir, http.StatusFound)
-}
-
-func (s *Server) testError(w http.ResponseWriter, r *http.Request) {
-	err := r.FormValue("err")
-	templ.Handler(base("Oh noes!", errorPage(err)), templ.WithStatus(http.StatusInternalServerError)).ServeHTTP(w, r)
-}
-
-func ohNoes(w http.ResponseWriter, r *http.Request, err error) {
-	slog.Error("super fatal error", "err", err)
-	templ.Handler(base("Oh noes!", errorPage("An internal server error happened")), templ.WithStatus(http.StatusInternalServerError)).ServeHTTP(w, r)
-}
-
-func clearCookie(w http.ResponseWriter) {
-	http.SetCookie(w, &http.Cookie{
-		Name:     cookieName,
-		Value:    "",
-		Expires:  time.Now().Add(-1 * time.Hour),
-		MaxAge:   -1,
-		SameSite: http.SameSiteLaxMode,
+		return os.WriteFile(destPath, data, 0o644)
 	})
 }
-
-func randomJitter() bool {
-	return mrand.Intn(100) > 10
-}
-
-func serveMainJSWithBestEncoding(w http.ResponseWriter, r *http.Request) {
-	priorityList := []string{"zstd", "br", "gzip"}
-	enc2ext := map[string]string{
-		"zstd": "zst",
-		"br":   "br",
-		"gzip": "gz",
-	}
-
-	for _, enc := range priorityList {
-		if strings.Contains(r.Header.Get("Accept-Encoding"), enc) {
-			w.Header().Set("Content-Type", "text/javascript")
-			w.Header().Set("Content-Encoding", enc)
-			http.ServeFileFS(w, r, static, "static/js/main.mjs."+enc2ext[enc])
-			return
-		}
-	}
-
-	w.Header().Set("Content-Type", "text/javascript")
-	http.ServeFileFS(w, r, static, "static/js/main.mjs")
-}

cmd/anubis/policy.go

@@ -1,212 +0,0 @@
-package main
-
-import (
-	"encoding/json"
-	"errors"
-	"fmt"
-	"io"
-	"log"
-	"log/slog"
-	"net"
-	"net/http"
-	"regexp"
-
-	"github.com/TecharoHQ/anubis/cmd/anubis/internal/config"
-	"github.com/prometheus/client_golang/prometheus"
-	"github.com/prometheus/client_golang/prometheus/promauto"
-	"github.com/yl2chen/cidranger"
-)
-
-var (
-	policyApplications = promauto.NewCounterVec(prometheus.CounterOpts{
-		Name: "anubis_policy_results",
-		Help: "The results of each policy rule",
-	}, []string{"rule", "action"})
-)
-
-type ParsedConfig struct {
-	orig config.Config
-
-	Bots  []Bot
-	DNSBL bool
-}
-
-type Bot struct {
-	Name      string
-	UserAgent *regexp.Regexp
-	Path      *regexp.Regexp
-	Action    config.Rule `json:"action"`
-	Challenge *config.ChallengeRules
-	Ranger    cidranger.Ranger
-}
-
-func (b Bot) Hash() (string, error) {
-	var pathRex string
-	if b.Path != nil {
-		pathRex = b.Path.String()
-	}
-	var userAgentRex string
-	if b.UserAgent != nil {
-		userAgentRex = b.UserAgent.String()
-	}
-
-	return sha256sum(fmt.Sprintf("%s::%s::%s", b.Name, pathRex, userAgentRex)), nil
-}
-
-func parseConfig(fin io.Reader, fname string, defaultDifficulty int) (*ParsedConfig, error) {
-	var c config.Config
-	if err := json.NewDecoder(fin).Decode(&c); err != nil {
-		return nil, fmt.Errorf("can't parse policy config JSON %s: %w", fname, err)
-	}
-
-	if err := c.Valid(); err != nil {
-		return nil, err
-	}
-
-	var err error
-
-	result := &ParsedConfig{
-		orig: c,
-	}
-
-	for _, b := range c.Bots {
-		if berr := b.Valid(); berr != nil {
-			err = errors.Join(err, berr)
-			continue
-		}
-
-		var botParseErr error
-		parsedBot := Bot{
-			Name:   b.Name,
-			Action: b.Action,
-		}
-
-		if b.RemoteAddr != nil && len(b.RemoteAddr) > 0 {
-			parsedBot.Ranger = cidranger.NewPCTrieRanger()
-
-			for _, cidr := range b.RemoteAddr {
-				_, rng, err := net.ParseCIDR(cidr)
-				if err != nil {
-					return nil, fmt.Errorf("[unexpected] range %s not parsing: %w", cidr, err)
-				}
-
-				parsedBot.Ranger.Insert(cidranger.NewBasicRangerEntry(*rng))
-			}
-		}
-
-		if b.UserAgentRegex != nil {
-			userAgent, err := regexp.Compile(*b.UserAgentRegex)
-			if err != nil {
-				botParseErr = errors.Join(botParseErr, fmt.Errorf("while compiling user agent regexp: %w", err))
-				continue
-			} else {
-				parsedBot.UserAgent = userAgent
-			}
-		}
-
-		if b.PathRegex != nil {
-			path, err := regexp.Compile(*b.PathRegex)
-			if err != nil {
-				botParseErr = errors.Join(botParseErr, fmt.Errorf("while compiling path regexp: %w", err))
-				continue
-			} else {
-				parsedBot.Path = path
-			}
-		}
-
-		if b.Challenge == nil {
-			parsedBot.Challenge = &config.ChallengeRules{
-				Difficulty: defaultDifficulty,
-				ReportAs:   defaultDifficulty,
-				Algorithm:  config.AlgorithmFast,
-			}
-		} else {
-			parsedBot.Challenge = b.Challenge
-			if parsedBot.Challenge.Algorithm == config.AlgorithmUnknown {
-				parsedBot.Challenge.Algorithm = config.AlgorithmFast
-			}
-		}
-
-		result.Bots = append(result.Bots, parsedBot)
-	}
-
-	if err != nil {
-		return nil, fmt.Errorf("errors validating policy config JSON %s: %w", fname, err)
-	}
-
-	result.DNSBL = c.DNSBL
-
-	return result, nil
-}
-
-type CheckResult struct {
-	Name string
-	Rule config.Rule
-}
-
-func (cr CheckResult) LogValue() slog.Value {
-	return slog.GroupValue(
-		slog.String("name", cr.Name),
-		slog.String("rule", string(cr.Rule)))
-}
-
-func cr(name string, rule config.Rule) CheckResult {
-	return CheckResult{
-		Name: name,
-		Rule: rule,
-	}
-}
-
-func (s *Server) checkRemoteAddress(b Bot, addr net.IP) bool {
-	if b.Ranger == nil {
-		return false
-	}
-
-	ok, err := b.Ranger.Contains(addr)
-	if err != nil {
-		log.Panicf("[unexpected] something very funky is going on, %q does not have a calculable network number: %v", addr.String(), err)
-	}
-
-	return ok
-}
-
-// Check evaluates the list of rules, and returns the result
-func (s *Server) check(r *http.Request) (CheckResult, *Bot, error) {
-	host := r.Header.Get("X-Real-Ip")
-	if host == "" {
-		return zilch[CheckResult](), nil, fmt.Errorf("[misconfiguration] X-Real-Ip header is not set")
-	}
-
-	addr := net.ParseIP(host)
-	if addr == nil {
-		return zilch[CheckResult](), nil, fmt.Errorf("[misconfiguration] %q is not an IP address", host)
-	}
-
-	for _, b := range s.policy.Bots {
-		if b.UserAgent != nil {
-			if uaMatch := b.UserAgent.MatchString(r.UserAgent()); uaMatch || (uaMatch && s.checkRemoteAddress(b, addr)) {
-				return cr("bot/"+b.Name, b.Action), &b, nil
-			}
-		}
-
-		if b.Path != nil {
-			if pathMatch := b.Path.MatchString(r.URL.Path); pathMatch || (pathMatch && s.checkRemoteAddress(b, addr)) {
-				return cr("bot/"+b.Name, b.Action), &b, nil
-			}
-		}
-
-		if b.Ranger != nil {
-			if s.checkRemoteAddress(b, addr) {
-				return cr("bot/"+b.Name, b.Action), &b, nil
-			}
-		}
-	}
-
-	return cr("default/allow", config.RuleAllow), &Bot{
-		Challenge: &config.ChallengeRules{
-			Difficulty: defaultDifficulty,
-			ReportAs:   defaultDifficulty,
-			Algorithm:  config.AlgorithmFast,
-		},
-	}, nil
-}

cmd/anubis/static/js/main.mjs

@@ -1,2 +0,0 @@
-(()=>{function p(r,n=5,t=navigator.hardwareConcurrency||1){return console.debug("fast algo"),new Promise((e,o)=>{let s=URL.createObjectURL(new Blob(["(",y(),")()"],{type:"application/javascript"})),a=[];for(let i=0;i<t;i++){let c=new Worker(s);c.onmessage=d=>{a.forEach(u=>u.terminate()),c.terminate(),e(d.data)},c.onerror=d=>{c.terminate(),o()},c.postMessage({data:r,difficulty:n,nonce:i,threads:t}),a.push(c)}URL.revokeObjectURL(s)})}function y(){return function(){let r=t=>{let e=new TextEncoder().encode(t);return crypto.subtle.digest("SHA-256",e.buffer)};function n(t){return Array.from(t).map(e=>e.toString(16).padStart(2,"0")).join("")}addEventListener("message",async t=>{let e=t.data.data,o=t.data.difficulty,s,a=t.data.nonce,i=t.data.threads;for(;;){let c=await r(e+a),d=new Uint8Array(c),u=!0;for(let m=0;m<o;m++){let l=Math.floor(m/2),g=m%2;if((d[l]>>(g===0?4:0)&15)!==0){u=!1;break}}if(u){s=n(d),console.log(s);break}a+=i}postMessage({hash:s,data:e,difficulty:o,nonce:a})})}.toString()}function f(r,n=5,t=1){return console.debug("slow algo"),new Promise((e,o)=>{let s=URL.createObjectURL(new Blob(["(",b(),")()"],{type:"application/javascript"})),a=new Worker(s);a.onmessage=i=>{a.terminate(),e(i.data)},a.onerror=i=>{a.terminate(),o()},a.postMessage({data:r,difficulty:n}),URL.revokeObjectURL(s)})}function b(){return function(){let r=n=>{let t=new TextEncoder().encode(n);return crypto.subtle.digest("SHA-256",t.buffer).then(e=>Array.from(new Uint8Array(e)).map(o=>o.toString(16).padStart(2,"0")).join(""))};addEventListener("message",async n=>{let t=n.data.data,e=n.data.difficulty,o,s=0;do o=await r(t+s++);while(o.substring(0,e)!==Array(e+1).join("0"));s-=1,postMessage({hash:o,data:t,difficulty:e,nonce:s})})}.toString()}var L={fast:p,slow:f},w=(r="",n={})=>{let t=new URL(r,window.location.href);return Object.entries(n).forEach(e=>{let[o,s]=e;t.searchParams.set(o,s)}),t.toString()},h=(r,n)=>w(`/.within.website/x/cmd/anubis/static/img/${r}.webp`,{cacheBuster:n});(async()=>{let r=document.getElementById("status"),n=document.getElementById("image"),t=document.getElementById("title"),e=document.getElementById("spinner"),o=JSON.parse(document.getElementById("anubis_version").textContent);r.innerHTML="Calculating...";let{challenge:s,rules:a}=await fetch("/.within.website/x/cmd/anubis/api/make-challenge",{method:"POST"}).then(l=>{if(!l.ok)throw new Error("Failed to fetch config");return l.json()}).catch(l=>{throw t.innerHTML="Oh no!",r.innerHTML=`Failed to fetch config: ${l.message}`,n.src=h("sad",o),e.innerHTML="",e.style.display="none",l}),i=L[a.algorithm];if(!i){t.innerHTML="Oh no!",r.innerHTML="Failed to resolve check algorithm. You may want to reload the page.",n.src=h("sad",o),e.innerHTML="",e.style.display="none";return}r.innerHTML=`Calculating...<br/>Difficulty: ${a.report_as}`;let c=Date.now(),{hash:d,nonce:u}=await i(s,a.difficulty),m=Date.now();console.log({hash:d,nonce:u}),t.innerHTML="Success!",r.innerHTML=`Done! Took ${m-c}ms, ${u} iterations`,n.src=h("happy",o),e.innerHTML="",e.style.display="none",setTimeout(()=>{let l=window.location.href;window.location.href=w("/.within.website/x/cmd/anubis/api/pass-challenge",{response:d,nonce:u,redir:l,elapsedTime:m-c})},250)})();})();
-//# sourceMappingURL=main.mjs.map

cmd/containerbuild/main.go

@@ -19,38 +19,20 @@ var (
 	dockerLabels      = flag.String("docker-labels", os.Getenv("DOCKER_METADATA_OUTPUT_LABELS"), "Docker image labels")
 	dockerRepo        = flag.String("docker-repo", "registry.int.xeserv.us/techaro/anubis", "Docker image repository for Anubis")
 	dockerTags        = flag.String("docker-tags", os.Getenv("DOCKER_METADATA_OUTPUT_TAGS"), "newline separated docker tags including the registry name")
-	githubActor       = flag.String("github-actor", "", "GitHub actor")
 	githubEventName   = flag.String("github-event-name", "", "GitHub event name")
 	pullRequestID     = flag.Int("pull-request-id", -1, "GitHub pull request ID")
 	slogLevel         = flag.String("slog-level", "INFO", "logging level (see https://pkg.go.dev/log/slog#hdr-Levels)")
-
-	knownContributors = []string{
-		"Xe",
-	}
 )
 
-func inList(needle string, haystack []string) bool {
-	for _, h := range haystack {
-		if h == needle {
-			return true
-		}
-	}
-	return false
-}
-
 func main() {
 	flagenv.Parse()
 	flag.Parse()
 
 	internal.InitSlog(*slogLevel)
 
-	koDockerRepo := strings.TrimRight(*dockerRepo, "/"+filepath.Base(*dockerRepo))
-
-	if *githubEventName == "pull_request" && !inList(*githubActor, knownContributors) {
-		if *pullRequestID == -1 {
-			log.Fatal("Must set --pull-request-id when --github-event-name=pull_request")
-		}
+	koDockerRepo := strings.TrimSuffix(*dockerRepo, "/"+filepath.Base(*dockerRepo))
 
+	if *githubEventName == "pull_request" && *pullRequestID != -1 {
 		*dockerRepo = fmt.Sprintf("ttl.sh/techaro/pr-%d/anubis", *pullRequestID)
 		*dockerTags = fmt.Sprintf("ttl.sh/techaro/pr-%d/anubis:24h", *pullRequestID)
 		koDockerRepo = fmt.Sprintf("ttl.sh/techaro/pr-%d", *pullRequestID)
@@ -130,11 +112,6 @@ type image struct {
 	tag        string
 }
 
-func newlineSep2Comma(inp string) string {
-	lines := strings.Split(inp, "\n")
-	return strings.Join(lines, ",")
-}
-
 func parseImageList(imageList string) ([]image, error) {
 	images := strings.Split(imageList, "\n")
 	var result []image
@@ -146,15 +123,15 @@ func parseImageList(imageList string) ([]image, error) {
 		// reg.xeiaso.net/techaro/anubis:latest
 		// repository: reg.xeiaso.net/techaro/anubis
 		// tag:        latest
-		parts := strings.SplitN(img, ":", 2)
+		index := strings.LastIndex(img, ":")
 		result = append(result, image{
-			repository: parts[0],
-			tag:        parts[1],
+			repository: img[:index],
+			tag:        img[index+1:],
 		})
 	}
 
 	if len(result) == 0 {
-		return nil, fmt.Errorf("no images provided, bad flags??")
+		return nil, fmt.Errorf("no images provided, bad flags")
 	}
 
 	return result, nil

data/apps/gitea-rss-feeds.yaml

@@ -0,0 +1,7 @@
+# By Aibrew: https://github.com/TecharoHQ/anubis/discussions/261#discussioncomment-12821065
+- name: gitea-feed-atom
+  action: ALLOW
+  path_regex: ^/[.A-Za-z0-9_-]{1,256}?[./A-Za-z0-9_-]*\.atom$
+- name: gitea-feed-rss
+  action: ALLOW
+  path_regex: ^/[.A-Za-z0-9_-]{1,256}?[./A-Za-z0-9_-]*\.rss$

data/botPolicies.json

@@ -0,0 +1,49 @@
+{
+  "bots": [
+    {
+      "import": "(data)/bots/ai-robots-txt.yaml"
+    },
+    {
+      "import": "(data)/bots/cloudflare-workers.yaml"
+    },
+    {
+      "import": "(data)/bots/headless-browsers.yaml"
+    },
+    {
+      "import": "(data)/bots/us-ai-scraper.yaml"
+    },
+    {
+      "import": "(data)/crawlers/googlebot.yaml"
+    },
+    {
+      "import": "(data)/crawlers/bingbot.yaml"
+    },
+    {
+      "import": "(data)/crawlers/duckduckbot.yaml"
+    },
+    {
+      "import": "(data)/crawlers/qwantbot.yaml"
+    },
+    {
+      "import": "(data)/crawlers/internet-archive.yaml"
+    },
+    {
+      "import": "(data)/crawlers/kagibot.yaml"
+    },
+    {
+      "import": "(data)/crawlers/marginalia.yaml"
+    },
+    {
+      "import": "(data)/crawlers/mojeekbot.yaml"
+    },
+    {
+      "import": "(data)/common/keep-internet-working.yaml"
+    },
+    {
+      "name": "generic-browser",
+      "user_agent_regex": "Mozilla|Opera",
+      "action": "CHALLENGE"
+    }
+  ],
+  "dnsbl": false
+}

data/botPolicies.yaml

@@ -0,0 +1,50 @@
+## Anubis has the ability to let you import snippets of configuration into the main
+## configuration file. This allows you to break up your config into smaller parts
+## that get logically assembled into one big file.
+##
+## Of note, a bot rule can either have inline bot configuration or import a
+## bot config snippet. You cannot do both in a single bot rule.
+##
+## Import paths can either be prefixed with (data) to import from the common/shared
+## rules in the data folder in the Anubis source tree or will point to absolute/relative
+## paths in your filesystem. If you don't have access to the Anubis source tree, check
+## /usr/share/docs/anubis/data or in the tarball you extracted Anubis from.
+
+bots:
+# Pathological bots to deny
+- # This correlates to data/bots/ai-robots-txt.yaml in the source tree
+  import: (data)/bots/ai-robots-txt.yaml
+- import: (data)/bots/cloudflare-workers.yaml 
+- import: (data)/bots/headless-browsers.yaml
+- import: (data)/bots/us-ai-scraper.yaml
+
+# Search engines to allow
+- import: (data)/crawlers/googlebot.yaml
+- import: (data)/crawlers/bingbot.yaml
+- import: (data)/crawlers/duckduckbot.yaml
+- import: (data)/crawlers/qwantbot.yaml
+- import: (data)/crawlers/internet-archive.yaml
+- import: (data)/crawlers/kagibot.yaml
+- import: (data)/crawlers/marginalia.yaml
+- import: (data)/crawlers/mojeekbot.yaml
+
+# Allow common "keeping the internet working" routes (well-known, favicon, robots.txt)
+- import: (data)/common/keep-internet-working.yaml
+
+# # Punish any bot with "bot" in the user-agent string
+# # This is known to have a high false-positive rate, use at your own risk
+# - name: generic-bot-catchall
+#   user_agent_regex: (?i:bot|crawler)
+#   action: CHALLENGE
+#   challenge:
+#     difficulty: 16  # impossible
+#     report_as: 4    # lie to the operator
+#     algorithm: slow # intentionally waste CPU cycles and time
+
+# Generic catchall rule
+- name: generic-browser
+  user_agent_regex: >-
+    Mozilla|Opera
+  action: CHALLENGE
+
+dnsbl: false

data/bots/ai-robots-txt.yaml

@@ -0,0 +1,4 @@
+- name: "ai-robots-txt"
+  user_agent_regex: >-
+    AI2Bot|Ai2Bot-Dolma|Amazonbot|anthropic-ai|Applebot|Applebot-Extended|Brightbot 1.0|Bytespider|CCBot|ChatGPT-User|Claude-Web|ClaudeBot|cohere-ai|cohere-training-data-crawler|Crawlspace|Diffbot|DuckAssistBot|FacebookBot|FriendlyCrawler|Google-Extended|GoogleOther|GoogleOther-Image|GoogleOther-Video|GPTBot|iaskspider/2.0|ICC-Crawler|ImagesiftBot|img2dataset|ISSCyberRiskCrawler|Kangaroo Bot|Meta-ExternalAgent|Meta-ExternalFetcher|OAI-SearchBot|omgili|omgilibot|PanguBot|Perplexity-User|PerplexityBot|PetalBot|Scrapy|SemrushBot-OCOB|SemrushBot-SWA|Sidetrade indexer bot|Timpibot|VelenPublicWebCrawler|Webzio-Extended|YouBot
+  action: DENY

data/bots/cloudflare-workers.yaml

@@ -0,0 +1,4 @@
+- name: cloudflare-workers
+  headers_regex:
+    CF-Worker: .*
+  action: DENY

data/bots/headless-browsers.yaml

@@ -0,0 +1,9 @@
+- name: lightpanda
+  user_agent_regex: ^LightPanda/.*$
+  action: DENY
+- name: headless-chrome
+  user_agent_regex: HeadlessChrome
+  action: DENY
+- name: headless-chromium
+  user_agent_regex: HeadlessChromium
+  action: DENY

data/bots/us-ai-scraper.yaml

@@ -0,0 +1,3 @@
+- name: us-artificial-intelligence-scraper
+  user_agent_regex: \+https\://github\.com/US-Artificial-Intelligence/scraper
+  action: DENY

data/common/allow-private-addresses.yaml

@@ -0,0 +1,15 @@
+- name: ipv4-rfc-1918
+  action: ALLOW
+  remote_addresses:
+  - 10.0.0.0/8
+  - 172.16.0.0/12
+  - 192.168.0.0/16
+  - 100.64.0.0/10
+- name: ipv6-ula
+  action: ALLOW
+  remote_addresses:
+  - fc00::/7
+- name: ipv6-link-local
+  action: ALLOW
+  remote_addresses:
+  - fe80::/10

data/common/keep-internet-working.yaml

@@ -0,0 +1,10 @@
+# Common "keeping the internet working" routes
+- name: well-known
+  path_regex: ^/.well-known/.*$
+  action: ALLOW
+- name: favicon
+  path_regex: ^/favicon.ico$
+  action: ALLOW
+- name: robots-txt
+  path_regex: ^/robots.txt$
+  action: ALLOW

data/crawlers/bingbot.yaml

@@ -0,0 +1,34 @@
+- name: bingbot
+  user_agent_regex: \+http\://www\.bing\.com/bingbot\.htm
+  action: ALLOW
+  # https://www.bing.com/toolbox/bingbot.json
+  remote_addresses: [
+    "157.55.39.0/24",
+    "207.46.13.0/24",
+    "40.77.167.0/24",
+    "13.66.139.0/24",
+    "13.66.144.0/24",
+    "52.167.144.0/24",
+    "13.67.10.16/28",
+    "13.69.66.240/28",
+    "13.71.172.224/28",
+    "139.217.52.0/28",
+    "191.233.204.224/28",
+    "20.36.108.32/28",
+    "20.43.120.16/28",
+    "40.79.131.208/28",
+    "40.79.186.176/28",
+    "52.231.148.0/28",
+    "20.79.107.240/28",
+    "51.105.67.0/28",
+    "20.125.163.80/28",
+    "40.77.188.0/22",
+    "65.55.210.0/24",
+    "199.30.24.0/23",
+    "40.77.202.0/24",
+    "40.77.139.0/25",
+    "20.74.197.0/28",
+    "20.15.133.160/27",
+    "40.77.177.0/24",
+    "40.77.178.0/23"
+  ]

data/crawlers/duckduckbot.yaml

@@ -0,0 +1,275 @@
+- name: duckduckbot
+  user_agent_regex: DuckDuckBot/1\.1; \(\+http\://duckduckgo\.com/duckduckbot\.html\)
+  action: ALLOW
+  # https://duckduckgo.com/duckduckgo-help-pages/results/duckduckbot
+  remote_addresses: [
+    "57.152.72.128/32",
+    "51.8.253.152/32",
+    "40.80.242.63/32",
+    "20.12.141.99/32",
+    "20.49.136.28/32",
+    "51.116.131.221/32",
+    "51.107.40.209/32",
+    "20.40.133.240/32",
+    "20.50.168.91/32",
+    "51.120.48.122/32",
+    "20.193.45.113/32",
+    "40.76.173.151/32",
+    "40.76.163.7/32",
+    "20.185.79.47/32",
+    "52.142.26.175/32",
+    "20.185.79.15/32",
+    "52.142.24.149/32",
+    "40.76.162.208/32",
+    "40.76.163.23/32",
+    "40.76.162.191/32",
+    "40.76.162.247/32",
+    "40.88.21.235/32",
+    "20.191.45.212/32",
+    "52.146.59.12/32",
+    "52.146.59.156/32",
+    "52.146.59.154/32",
+    "52.146.58.236/32",
+    "20.62.224.44/32",
+    "51.104.180.53/32",
+    "51.104.180.47/32",
+    "51.104.180.26/32",
+    "51.104.146.225/32",
+    "51.104.146.235/32",
+    "20.73.202.147/32",
+    "20.73.132.240/32",
+    "20.71.12.143/32",
+    "20.56.197.58/32",
+    "20.56.197.63/32",
+    "20.43.150.93/32",
+    "20.43.150.85/32",
+    "20.44.222.1/32",
+    "40.89.243.175/32",
+    "13.89.106.77/32",
+    "52.143.242.6/32",
+    "52.143.241.111/32",
+    "52.154.60.82/32",
+    "20.197.209.11/32",
+    "20.197.209.27/32",
+    "20.226.133.105/32",
+    "191.234.216.4/32",
+    "191.234.216.178/32",
+    "20.53.92.211/32",
+    "20.53.91.2/32",
+    "20.207.99.197/32",
+    "20.207.97.190/32",
+    "40.81.250.205/32",
+    "40.64.106.11/32",
+    "40.64.105.247/32",
+    "20.72.242.93/32",
+    "20.99.255.235/32",
+    "20.113.3.121/32",
+    "52.224.16.221/32",
+    "52.224.21.53/32",
+    "52.224.20.204/32",
+    "52.224.21.19/32",
+    "52.224.20.249/32",
+    "52.224.20.203/32",
+    "52.224.20.190/32",
+    "52.224.16.229/32",
+    "52.224.21.20/32",
+    "52.146.63.80/32",
+    "52.224.20.227/32",
+    "52.224.20.193/32",
+    "52.190.37.160/32",
+    "52.224.21.23/32",
+    "52.224.20.223/32",
+    "52.224.20.181/32",
+    "52.224.21.49/32",
+    "52.224.21.55/32",
+    "52.224.21.61/32",
+    "52.224.19.152/32",
+    "52.224.20.186/32",
+    "52.224.21.27/32",
+    "52.224.21.51/32",
+    "52.224.20.174/32",
+    "52.224.21.4/32",
+    "51.104.164.109/32",
+    "51.104.167.71/32",
+    "51.104.160.177/32",
+    "51.104.162.149/32",
+    "51.104.167.95/32",
+    "51.104.167.54/32",
+    "51.104.166.111/32",
+    "51.104.167.88/32",
+    "51.104.161.32/32",
+    "51.104.163.250/32",
+    "51.104.164.189/32",
+    "51.104.167.19/32",
+    "51.104.160.167/32",
+    "51.104.167.110/32",
+    "20.191.44.119/32",
+    "51.104.167.104/32",
+    "20.191.44.234/32",
+    "51.104.164.215/32",
+    "51.104.167.52/32",
+    "20.191.44.22/32",
+    "51.104.167.87/32",
+    "51.104.167.96/32",
+    "20.191.44.16/32",
+    "51.104.167.61/32",
+    "51.104.164.147/32",
+    "20.50.48.159/32",
+    "40.114.182.172/32",
+    "20.50.50.130/32",
+    "20.50.50.163/32",
+    "20.50.50.46/32",
+    "40.114.182.153/32",
+    "20.50.50.118/32",
+    "20.50.49.55/32",
+    "20.50.49.25/32",
+    "40.114.183.251/32",
+    "20.50.50.123/32",
+    "20.50.49.237/32",
+    "20.50.48.192/32",
+    "20.50.50.134/32",
+    "51.138.90.233/32",
+    "40.114.183.196/32",
+    "20.50.50.146/32",
+    "40.114.183.88/32",
+    "20.50.50.145/32",
+    "20.50.50.121/32",
+    "20.50.49.40/32",
+    "51.138.90.206/32",
+    "40.114.182.45/32",
+    "51.138.90.161/32",
+    "20.50.49.0/32",
+    "40.119.232.215/32",
+    "104.43.55.167/32",
+    "40.119.232.251/32",
+    "40.119.232.50/32",
+    "40.119.232.146/32",
+    "40.119.232.218/32",
+    "104.43.54.127/32",
+    "104.43.55.117/32",
+    "104.43.55.116/32",
+    "104.43.55.166/32",
+    "52.154.169.50/32",
+    "52.154.171.70/32",
+    "52.154.170.229/32",
+    "52.154.170.113/32",
+    "52.154.171.44/32",
+    "52.154.172.2/32",
+    "52.143.244.81/32",
+    "52.154.171.87/32",
+    "52.154.171.250/32",
+    "52.154.170.28/32",
+    "52.154.170.122/32",
+    "52.143.243.117/32",
+    "52.143.247.235/32",
+    "52.154.171.235/32",
+    "52.154.171.196/32",
+    "52.154.171.0/32",
+    "52.154.170.243/32",
+    "52.154.170.26/32",
+    "52.154.169.200/32",
+    "52.154.170.96/32",
+    "52.154.170.88/32",
+    "52.154.171.150/32",
+    "52.154.171.205/32",
+    "52.154.170.117/32",
+    "52.154.170.209/32",
+    "191.235.202.48/32",
+    "191.233.3.202/32",
+    "191.235.201.214/32",
+    "191.233.3.197/32",
+    "191.235.202.38/32",
+    "20.53.78.144/32",
+    "20.193.24.10/32",
+    "20.53.78.236/32",
+    "20.53.78.138/32",
+    "20.53.78.123/32",
+    "20.53.78.106/32",
+    "20.193.27.215/32",
+    "20.193.25.197/32",
+    "20.193.12.126/32",
+    "20.193.24.251/32",
+    "20.204.242.101/32",
+    "20.207.72.113/32",
+    "20.204.242.19/32",
+    "20.219.45.67/32",
+    "20.207.72.11/32",
+    "20.219.45.190/32",
+    "20.204.243.55/32",
+    "20.204.241.148/32",
+    "20.207.72.110/32",
+    "20.204.240.172/32",
+    "20.207.72.21/32",
+    "20.204.246.81/32",
+    "20.207.107.181/32",
+    "20.204.246.254/32",
+    "20.219.43.246/32",
+    "52.149.25.43/32",
+    "52.149.61.51/32",
+    "52.149.58.139/32",
+    "52.149.60.38/32",
+    "52.148.165.38/32",
+    "52.143.95.162/32",
+    "52.149.56.151/32",
+    "52.149.30.45/32",
+    "52.149.58.173/32",
+    "52.143.95.204/32",
+    "52.149.28.83/32",
+    "52.149.58.69/32",
+    "52.148.161.87/32",
+    "52.149.58.27/32",
+    "52.149.28.18/32",
+    "20.79.226.26/32",
+    "20.79.239.66/32",
+    "20.79.238.198/32",
+    "20.113.14.159/32",
+    "20.75.144.152/32",
+    "20.43.172.120/32",
+    "20.53.134.160/32",
+    "20.201.15.208/32",
+    "20.93.28.24/32",
+    "20.61.34.40/32",
+    "52.242.224.168/32",
+    "20.80.129.80/32",
+    "20.195.108.47/32",
+    "4.195.133.120/32",
+    "4.228.76.163/32",
+    "4.182.131.108/32",
+    "4.209.224.56/32",
+    "108.141.83.74/32",
+    "4.213.46.14/32",
+    "172.169.17.165/32",
+    "51.8.71.117/32",
+    "20.3.1.178/32",
+    "52.149.56.151/32",
+    "52.149.30.45/32",
+    "52.149.58.173/32",
+    "52.143.95.204/32",
+    "52.149.28.83/32",
+    "52.149.58.69/32",
+    "52.148.161.87/32",
+    "52.149.58.27/32",
+    "52.149.28.18/32",
+    "20.79.226.26/32",
+    "20.79.239.66/32",
+    "20.79.238.198/32",
+    "20.113.14.159/32",
+    "20.75.144.152/32",
+    "20.43.172.120/32",
+    "20.53.134.160/32",
+    "20.201.15.208/32",
+    "20.93.28.24/32",
+    "20.61.34.40/32",
+    "52.242.224.168/32",
+    "20.80.129.80/32",
+    "20.195.108.47/32",
+    "4.195.133.120/32",
+    "4.228.76.163/32",
+    "4.182.131.108/32",
+    "4.209.224.56/32",
+    "108.141.83.74/32",
+    "4.213.46.14/32",
+    "172.169.17.165/32",
+    "51.8.71.117/32",
+    "20.3.1.178/32"
+  ]

data/crawlers/googlebot.yaml

@@ -0,0 +1,263 @@
+- name: googlebot
+  user_agent_regex: \+http\://www\.google\.com/bot\.html
+  action: ALLOW
+  # https://developers.google.com/static/search/apis/ipranges/googlebot.json
+  remote_addresses: [
+    "2001:4860:4801:10::/64",
+    "2001:4860:4801:11::/64",
+    "2001:4860:4801:12::/64",
+    "2001:4860:4801:13::/64",
+    "2001:4860:4801:14::/64",
+    "2001:4860:4801:15::/64",
+    "2001:4860:4801:16::/64",
+    "2001:4860:4801:17::/64",
+    "2001:4860:4801:18::/64",
+    "2001:4860:4801:19::/64",
+    "2001:4860:4801:1a::/64",
+    "2001:4860:4801:1b::/64",
+    "2001:4860:4801:1c::/64",
+    "2001:4860:4801:1d::/64",
+    "2001:4860:4801:1e::/64",
+    "2001:4860:4801:1f::/64",
+    "2001:4860:4801:20::/64",
+    "2001:4860:4801:21::/64",
+    "2001:4860:4801:22::/64",
+    "2001:4860:4801:23::/64",
+    "2001:4860:4801:24::/64",
+    "2001:4860:4801:25::/64",
+    "2001:4860:4801:26::/64",
+    "2001:4860:4801:27::/64",
+    "2001:4860:4801:28::/64",
+    "2001:4860:4801:29::/64",
+    "2001:4860:4801:2::/64",
+    "2001:4860:4801:2a::/64",
+    "2001:4860:4801:2b::/64",
+    "2001:4860:4801:2c::/64",
+    "2001:4860:4801:2d::/64",
+    "2001:4860:4801:2e::/64",
+    "2001:4860:4801:2f::/64",
+    "2001:4860:4801:31::/64",
+    "2001:4860:4801:32::/64",
+    "2001:4860:4801:33::/64",
+    "2001:4860:4801:34::/64",
+    "2001:4860:4801:35::/64",
+    "2001:4860:4801:36::/64",
+    "2001:4860:4801:37::/64",
+    "2001:4860:4801:38::/64",
+    "2001:4860:4801:39::/64",
+    "2001:4860:4801:3a::/64",
+    "2001:4860:4801:3b::/64",
+    "2001:4860:4801:3c::/64",
+    "2001:4860:4801:3d::/64",
+    "2001:4860:4801:3e::/64",
+    "2001:4860:4801:40::/64",
+    "2001:4860:4801:41::/64",
+    "2001:4860:4801:42::/64",
+    "2001:4860:4801:43::/64",
+    "2001:4860:4801:44::/64",
+    "2001:4860:4801:45::/64",
+    "2001:4860:4801:46::/64",
+    "2001:4860:4801:47::/64",
+    "2001:4860:4801:48::/64",
+    "2001:4860:4801:49::/64",
+    "2001:4860:4801:4a::/64",
+    "2001:4860:4801:4b::/64",
+    "2001:4860:4801:4c::/64",
+    "2001:4860:4801:50::/64",
+    "2001:4860:4801:51::/64",
+    "2001:4860:4801:52::/64",
+    "2001:4860:4801:53::/64",
+    "2001:4860:4801:54::/64",
+    "2001:4860:4801:55::/64",
+    "2001:4860:4801:56::/64",
+    "2001:4860:4801:60::/64",
+    "2001:4860:4801:61::/64",
+    "2001:4860:4801:62::/64",
+    "2001:4860:4801:63::/64",
+    "2001:4860:4801:64::/64",
+    "2001:4860:4801:65::/64",
+    "2001:4860:4801:66::/64",
+    "2001:4860:4801:67::/64",
+    "2001:4860:4801:68::/64",
+    "2001:4860:4801:69::/64",
+    "2001:4860:4801:6a::/64",
+    "2001:4860:4801:6b::/64",
+    "2001:4860:4801:6c::/64",
+    "2001:4860:4801:6d::/64",
+    "2001:4860:4801:6e::/64",
+    "2001:4860:4801:6f::/64",
+    "2001:4860:4801:70::/64",
+    "2001:4860:4801:71::/64",
+    "2001:4860:4801:72::/64",
+    "2001:4860:4801:73::/64",
+    "2001:4860:4801:74::/64",
+    "2001:4860:4801:75::/64",
+    "2001:4860:4801:76::/64",
+    "2001:4860:4801:77::/64",
+    "2001:4860:4801:78::/64",
+    "2001:4860:4801:79::/64",
+    "2001:4860:4801:80::/64",
+    "2001:4860:4801:81::/64",
+    "2001:4860:4801:82::/64",
+    "2001:4860:4801:83::/64",
+    "2001:4860:4801:84::/64",
+    "2001:4860:4801:85::/64",
+    "2001:4860:4801:86::/64",
+    "2001:4860:4801:87::/64",
+    "2001:4860:4801:88::/64",
+    "2001:4860:4801:90::/64",
+    "2001:4860:4801:91::/64",
+    "2001:4860:4801:92::/64",
+    "2001:4860:4801:93::/64",
+    "2001:4860:4801:94::/64",
+    "2001:4860:4801:95::/64",
+    "2001:4860:4801:96::/64",
+    "2001:4860:4801:a0::/64",
+    "2001:4860:4801:a1::/64",
+    "2001:4860:4801:a2::/64",
+    "2001:4860:4801:a3::/64",
+    "2001:4860:4801:a4::/64",
+    "2001:4860:4801:a5::/64",
+    "2001:4860:4801:c::/64",
+    "2001:4860:4801:f::/64",
+    "192.178.5.0/27",
+    "192.178.6.0/27",
+    "192.178.6.128/27",
+    "192.178.6.160/27",
+    "192.178.6.192/27",
+    "192.178.6.32/27",
+    "192.178.6.64/27",
+    "192.178.6.96/27",
+    "34.100.182.96/28",
+    "34.101.50.144/28",
+    "34.118.254.0/28",
+    "34.118.66.0/28",
+    "34.126.178.96/28",
+    "34.146.150.144/28",
+    "34.147.110.144/28",
+    "34.151.74.144/28",
+    "34.152.50.64/28",
+    "34.154.114.144/28",
+    "34.155.98.32/28",
+    "34.165.18.176/28",
+    "34.175.160.64/28",
+    "34.176.130.16/28",
+    "34.22.85.0/27",
+    "34.64.82.64/28",
+    "34.65.242.112/28",
+    "34.80.50.80/28",
+    "34.88.194.0/28",
+    "34.89.10.80/28",
+    "34.89.198.80/28",
+    "34.96.162.48/28",
+    "35.247.243.240/28",
+    "66.249.64.0/27",
+    "66.249.64.128/27",
+    "66.249.64.160/27",
+    "66.249.64.224/27",
+    "66.249.64.32/27",
+    "66.249.64.64/27",
+    "66.249.64.96/27",
+    "66.249.65.0/27",
+    "66.249.65.128/27",
+    "66.249.65.160/27",
+    "66.249.65.192/27",
+    "66.249.65.224/27",
+    "66.249.65.32/27",
+    "66.249.65.64/27",
+    "66.249.65.96/27",
+    "66.249.66.0/27",
+    "66.249.66.128/27",
+    "66.249.66.160/27",
+    "66.249.66.192/27",
+    "66.249.66.224/27",
+    "66.249.66.32/27",
+    "66.249.66.64/27",
+    "66.249.66.96/27",
+    "66.249.68.0/27",
+    "66.249.68.128/27",
+    "66.249.68.32/27",
+    "66.249.68.64/27",
+    "66.249.68.96/27",
+    "66.249.69.0/27",
+    "66.249.69.128/27",
+    "66.249.69.160/27",
+    "66.249.69.192/27",
+    "66.249.69.224/27",
+    "66.249.69.32/27",
+    "66.249.69.64/27",
+    "66.249.69.96/27",
+    "66.249.70.0/27",
+    "66.249.70.128/27",
+    "66.249.70.160/27",
+    "66.249.70.192/27",
+    "66.249.70.224/27",
+    "66.249.70.32/27",
+    "66.249.70.64/27",
+    "66.249.70.96/27",
+    "66.249.71.0/27",
+    "66.249.71.128/27",
+    "66.249.71.160/27",
+    "66.249.71.192/27",
+    "66.249.71.224/27",
+    "66.249.71.32/27",
+    "66.249.71.64/27",
+    "66.249.71.96/27",
+    "66.249.72.0/27",
+    "66.249.72.128/27",
+    "66.249.72.160/27",
+    "66.249.72.192/27",
+    "66.249.72.224/27",
+    "66.249.72.32/27",
+    "66.249.72.64/27",
+    "66.249.72.96/27",
+    "66.249.73.0/27",
+    "66.249.73.128/27",
+    "66.249.73.160/27",
+    "66.249.73.192/27",
+    "66.249.73.224/27",
+    "66.249.73.32/27",
+    "66.249.73.64/27",
+    "66.249.73.96/27",
+    "66.249.74.0/27",
+    "66.249.74.128/27",
+    "66.249.74.160/27",
+    "66.249.74.192/27",
+    "66.249.74.32/27",
+    "66.249.74.64/27",
+    "66.249.74.96/27",
+    "66.249.75.0/27",
+    "66.249.75.128/27",
+    "66.249.75.160/27",
+    "66.249.75.192/27",
+    "66.249.75.224/27",
+    "66.249.75.32/27",
+    "66.249.75.64/27",
+    "66.249.75.96/27",
+    "66.249.76.0/27",
+    "66.249.76.128/27",
+    "66.249.76.160/27",
+    "66.249.76.192/27",
+    "66.249.76.224/27",
+    "66.249.76.32/27",
+    "66.249.76.64/27",
+    "66.249.76.96/27",
+    "66.249.77.0/27",
+    "66.249.77.128/27",
+    "66.249.77.160/27",
+    "66.249.77.192/27",
+    "66.249.77.224/27",
+    "66.249.77.32/27",
+    "66.249.77.64/27",
+    "66.249.77.96/27",
+    "66.249.78.0/27",
+    "66.249.78.32/27",
+    "66.249.79.0/27",
+    "66.249.79.128/27",
+    "66.249.79.160/27",
+    "66.249.79.192/27",
+    "66.249.79.224/27",
+    "66.249.79.32/27",
+    "66.249.79.64/27",
+    "66.249.79.96/27"
+  ]

data/crawlers/internet-archive.yaml

@@ -0,0 +1,8 @@
+- name: internet-archive
+  action: ALLOW
+  # https://ipinfo.io/AS7941
+  remote_addresses: [
+    "207.241.224.0/20",
+    "208.70.24.0/21",
+    "2620:0:9c0::/48"
+  ]

data/crawlers/kagibot.yaml

@@ -0,0 +1,10 @@
+- name: kagibot
+  user_agent_regex: \+https\://kagi\.com/bot
+  action: ALLOW
+  # https://kagi.com/bot
+  remote_addresses: [
+    "216.18.205.234/32",
+    "35.212.27.76/32",
+    "104.254.65.50/32",
+    "209.151.156.194/32"
+  ]

data/crawlers/marginalia.yaml

@@ -0,0 +1,11 @@
+- name: marginalia
+  user_agent_regex: search\.marginalia\.nu
+  action: ALLOW
+  # Received directly over email
+  remote_addresses: [
+    "193.183.0.162/31",
+    "193.183.0.164/30",
+    "193.183.0.168/30",
+    "193.183.0.172/31",
+    "193.183.0.174/32"
+  ]

data/crawlers/mojeekbot.yaml

@@ -0,0 +1,5 @@
+- name: mojeekbot
+  user_agent_regex: \+https\://www\.mojeek\.com/bot\.html
+  action: ALLOW
+  # https://www.mojeek.com/bot.html
+  remote_addresses: [ "5.102.173.71/32" ]

data/crawlers/qwantbot.yaml

@@ -0,0 +1,5 @@
+- name: qwantbot
+  user_agent_regex: \+https\://help\.qwant\.com/bot/
+  action: ALLOW
+  # https://help.qwant.com/wp-content/uploads/sites/2/2025/01/qwantbot.json
+  remote_addresses: [ "91.242.162.0/24" ]

data/embed.go

@@ -0,0 +1,8 @@
+package data
+
+import "embed"
+
+var (
+	//go:embed botPolicies.yaml botPolicies.json apps bots common crawlers
+	BotPolicies embed.FS
+)

decaymap/decaymap.go

@@ -1,17 +1,17 @@
-package main
+package decaymap
 
 import (
 	"sync"
 	"time"
 )
 
-func zilch[T any]() T {
+func Zilch[T any]() T {
 	var zero T
 	return zero
 }
 
-// DecayMap is a lazy key->value map. It's a wrapper around a map and a mutex. If values exceed their time-to-live, they are pruned at Get time.
-type DecayMap[K comparable, V any] struct {
+// Impl is a lazy key->value map. It's a wrapper around a map and a mutex. If values exceed their time-to-live, they are pruned at Get time.
+type Impl[K comparable, V any] struct {
 	data map[K]decayMapEntry[V]
 	lock sync.RWMutex
 }
@@ -21,17 +21,17 @@ type decayMapEntry[V any] struct {
 	expiry time.Time
 }
 
-// NewDecayMap creates a new DecayMap of key type K and value type V.
+// New creates a new DecayMap of key type K and value type V.
 //
 // Key types must be comparable to work with maps.
-func NewDecayMap[K comparable, V any]() *DecayMap[K, V] {
-	return &DecayMap[K, V]{
+func New[K comparable, V any]() *Impl[K, V] {
+	return &Impl[K, V]{
 		data: make(map[K]decayMapEntry[V]),
 	}
 }
 
 // expire forcibly expires a key by setting its time-to-live one second in the past.
-func (m *DecayMap[K, V]) expire(key K) bool {
+func (m *Impl[K, V]) expire(key K) bool {
 	m.lock.RLock()
 	val, ok := m.data[key]
 	m.lock.RUnlock()
@@ -51,32 +51,32 @@ func (m *DecayMap[K, V]) expire(key K) bool {
 // Get gets a value from the DecayMap by key.
 //
 // If a value has expired, forcibly delete it if it was not updated.
-func (m *DecayMap[K, V]) Get(key K) (V, bool) {
+func (m *Impl[K, V]) Get(key K) (V, bool) {
 	m.lock.RLock()
 	value, ok := m.data[key]
 	m.lock.RUnlock()
 
 	if !ok {
-		return zilch[V](), false
+		return Zilch[V](), false
 	}
 
 	if time.Now().After(value.expiry) {
 		m.lock.Lock()
 		// Since previously reading m.data[key], the value may have been updated.
 		// Delete the entry only if the expiry time is still the same.
-		if m.data[key].expiry == value.expiry {
+		if m.data[key].expiry.Equal(value.expiry) {
 			delete(m.data, key)
 		}
 		m.lock.Unlock()
 
-		return zilch[V](), false
+		return Zilch[V](), false
 	}
 
 	return value.Value, true
 }
 
 // Set sets a key value pair in the map.
-func (m *DecayMap[K, V]) Set(key K, value V, ttl time.Duration) {
+func (m *Impl[K, V]) Set(key K, value V, ttl time.Duration) {
 	m.lock.Lock()
 	defer m.lock.Unlock()
 
@@ -85,3 +85,23 @@ func (m *DecayMap[K, V]) Set(key K, value V, ttl time.Duration) {
 		expiry: time.Now().Add(ttl),
 	}
 }
+
+// Cleanup removes all expired entries from the DecayMap.
+func (m *Impl[K, V]) Cleanup() {
+	m.lock.Lock()
+	defer m.lock.Unlock()
+
+	now := time.Now()
+	for key, entry := range m.data {
+		if now.After(entry.expiry) {
+			delete(m.data, key)
+		}
+	}
+}
+
+// Len returns the number of entries in the DecayMap.
+func (m *Impl[K, V]) Len() int {
+	m.lock.RLock()
+	defer m.lock.RUnlock()
+	return len(m.data)
+}

decaymap/decaymap_test.go

@@ -0,0 +1,60 @@
+package decaymap
+
+import (
+	"testing"
+	"time"
+)
+
+func TestImpl(t *testing.T) {
+	dm := New[string, string]()
+
+	dm.Set("test", "hi", 5*time.Minute)
+
+	val, ok := dm.Get("test")
+	if !ok {
+		t.Error("somehow the test key was not set")
+	}
+
+	if val != "hi" {
+		t.Errorf("wanted value %q, got: %q", "hi", val)
+	}
+
+	ok = dm.expire("test")
+	if !ok {
+		t.Error("somehow could not force-expire the test key")
+	}
+
+	_, ok = dm.Get("test")
+	if ok {
+		t.Error("got value even though it was supposed to be expired")
+	}
+}
+
+func TestCleanup(t *testing.T) {
+	dm := New[string, string]()
+
+	dm.Set("test1", "hi1", 1*time.Second)
+	dm.Set("test2", "hi2", 2*time.Second)
+	dm.Set("test3", "hi3", 3*time.Second)
+
+	dm.expire("test1") // Force expire test1
+	dm.expire("test2") // Force expire test2
+
+	dm.Cleanup()
+
+	finalLen := dm.Len() // Get the length after cleanup
+
+	if finalLen != 1 { // "test3" should be the only one left
+		t.Errorf("Cleanup failed to remove expired entries. Expected length 1, got %d", finalLen)
+	}
+
+	if _, ok := dm.Get("test1"); ok { // Verify Get still behaves correctly after Cleanup
+		t.Error("test1 should not be found after cleanup")
+	}
+	if _, ok := dm.Get("test2"); ok {
+		t.Error("test2 should not be found after cleanup")
+	}
+	if val, ok := dm.Get("test3"); !ok || val != "hi3" {
+		t.Error("test3 should still be found after cleanup")
+	}
+}

doc.go

@@ -1,8 +0,0 @@
-// Package Anubis contains the version number of Anubis.
-package anubis
-
-// Version is the current version of Anubis.
-//
-// This variable is set at build time using the -X linker flag. If not set,
-// it defaults to "devel".
-var Version = "devel"

docs/blog/2019-05-28-first-blog-post.md

@@ -1,12 +0,0 @@
----
-slug: first-blog-post
-title: First Blog Post
-authors: [slorber, yangshun]
-tags: [hola, docusaurus]
----
-
-Lorem ipsum dolor sit amet...
-
-<!-- truncate -->
-
-...consectetur adipiscing elit. Pellentesque elementum dignissim ultricies. Fusce rhoncus ipsum tempor eros aliquam consequat. Lorem ipsum dolor sit amet

docs/blog/2019-05-29-long-blog-post.md

@@ -1,44 +0,0 @@
----
-slug: long-blog-post
-title: Long Blog Post
-authors: yangshun
-tags: [hello, docusaurus]
----
-
-This is the summary of a very long blog post,
-
-Use a `<!--` `truncate` `-->` comment to limit blog post size in the list view.
-
-<!-- truncate -->
-
-Lorem ipsum dolor sit amet, consectetur adipiscing elit. Pellentesque elementum dignissim ultricies. Fusce rhoncus ipsum tempor eros aliquam consequat. Lorem ipsum dolor sit amet
-
-Lorem ipsum dolor sit amet, consectetur adipiscing elit. Pellentesque elementum dignissim ultricies. Fusce rhoncus ipsum tempor eros aliquam consequat. Lorem ipsum dolor sit amet
-
-Lorem ipsum dolor sit amet, consectetur adipiscing elit. Pellentesque elementum dignissim ultricies. Fusce rhoncus ipsum tempor eros aliquam consequat. Lorem ipsum dolor sit amet
-
-Lorem ipsum dolor sit amet, consectetur adipiscing elit. Pellentesque elementum dignissim ultricies. Fusce rhoncus ipsum tempor eros aliquam consequat. Lorem ipsum dolor sit amet
-
-Lorem ipsum dolor sit amet, consectetur adipiscing elit. Pellentesque elementum dignissim ultricies. Fusce rhoncus ipsum tempor eros aliquam consequat. Lorem ipsum dolor sit amet
-
-Lorem ipsum dolor sit amet, consectetur adipiscing elit. Pellentesque elementum dignissim ultricies. Fusce rhoncus ipsum tempor eros aliquam consequat. Lorem ipsum dolor sit amet
-
-Lorem ipsum dolor sit amet, consectetur adipiscing elit. Pellentesque elementum dignissim ultricies. Fusce rhoncus ipsum tempor eros aliquam consequat. Lorem ipsum dolor sit amet
-
-Lorem ipsum dolor sit amet, consectetur adipiscing elit. Pellentesque elementum dignissim ultricies. Fusce rhoncus ipsum tempor eros aliquam consequat. Lorem ipsum dolor sit amet
-
-Lorem ipsum dolor sit amet, consectetur adipiscing elit. Pellentesque elementum dignissim ultricies. Fusce rhoncus ipsum tempor eros aliquam consequat. Lorem ipsum dolor sit amet
-
-Lorem ipsum dolor sit amet, consectetur adipiscing elit. Pellentesque elementum dignissim ultricies. Fusce rhoncus ipsum tempor eros aliquam consequat. Lorem ipsum dolor sit amet
-
-Lorem ipsum dolor sit amet, consectetur adipiscing elit. Pellentesque elementum dignissim ultricies. Fusce rhoncus ipsum tempor eros aliquam consequat. Lorem ipsum dolor sit amet
-
-Lorem ipsum dolor sit amet, consectetur adipiscing elit. Pellentesque elementum dignissim ultricies. Fusce rhoncus ipsum tempor eros aliquam consequat. Lorem ipsum dolor sit amet
-
-Lorem ipsum dolor sit amet, consectetur adipiscing elit. Pellentesque elementum dignissim ultricies. Fusce rhoncus ipsum tempor eros aliquam consequat. Lorem ipsum dolor sit amet
-
-Lorem ipsum dolor sit amet, consectetur adipiscing elit. Pellentesque elementum dignissim ultricies. Fusce rhoncus ipsum tempor eros aliquam consequat. Lorem ipsum dolor sit amet
-
-Lorem ipsum dolor sit amet, consectetur adipiscing elit. Pellentesque elementum dignissim ultricies. Fusce rhoncus ipsum tempor eros aliquam consequat. Lorem ipsum dolor sit amet
-
-Lorem ipsum dolor sit amet, consectetur adipiscing elit. Pellentesque elementum dignissim ultricies. Fusce rhoncus ipsum tempor eros aliquam consequat. Lorem ipsum dolor sit amet

docs/blog/2021-08-01-mdx-blog-post.mdx

@@ -1,24 +0,0 @@
----
-slug: mdx-blog-post
-title: MDX Blog Post
-authors: [slorber]
-tags: [docusaurus]
----
-
-Blog posts support [Docusaurus Markdown features](https://docusaurus.io/docs/markdown-features), such as [MDX](https://mdxjs.com/).
-
-:::tip
-
-Use the power of React to create interactive blog posts.
-
-:::
-
-{/* truncate */}
-
-For example, use JSX to create an interactive button:
-
-```js
-<button onClick={() => alert('button clicked!')}>Click me!</button>
-```
-
-<button onClick={() => alert('button clicked!')}>Click me!</button>

docs/blog/2021-08-26-welcome/index.md

@@ -1,29 +0,0 @@
----
-slug: welcome
-title: Welcome
-authors: [slorber, yangshun]
-tags: [facebook, hello, docusaurus]
----
-
-[Docusaurus blogging features](https://docusaurus.io/docs/blog) are powered by the [blog plugin](https://docusaurus.io/docs/api/plugins/@docusaurus/plugin-content-blog).
-
-Here are a few tips you might find useful.
-
-<!-- truncate -->
-
-Simply add Markdown files (or folders) to the `blog` directory.
-
-Regular blog authors can be added to `authors.yml`.
-
-The blog post date can be extracted from filenames, such as:
-
-- `2019-05-30-welcome.md`
-- `2019-05-30-welcome/index.md`
-
-A blog post folder can be convenient to co-locate blog post images:
-
-![Docusaurus Plushie](./docusaurus-plushie-banner.jpeg)
-
-The blog supports tags as well!
-
-**And if you don't want a blog**: just delete this directory, and use `blog: false` in your Docusaurus config.

docs/blog/authors.yml

@@ -1,23 +0,0 @@
-yangshun:
-  name: Yangshun Tay
-  title: Front End Engineer @ Facebook
-  url: https://github.com/yangshun
-  image_url: https://github.com/yangshun.png
-  page: true
-  socials:
-    x: yangshunz
-    github: yangshun
-
-slorber:
-  name: Sébastien Lorber
-  title: Docusaurus maintainer
-  url: https://sebastienlorber.com
-  image_url: https://github.com/slorber.png
-  page:
-    # customize the url of the author page at /blog/authors/<permalink>
-    permalink: '/all-sebastien-lorber-articles'
-  socials:
-    x: sebastienlorber
-    linkedin: sebastienlorber
-    github: slorber
-    newsletter: https://thisweekinreact.com

docs/blog/tags.yml

@@ -1,19 +0,0 @@
-facebook:
-  label: Facebook
-  permalink: /facebook
-  description: Facebook tag description
-
-hello:
-  label: Hello
-  permalink: /hello
-  description: Hello tag description
-
-docusaurus:
-  label: Docusaurus
-  permalink: /docusaurus
-  description: Docusaurus tag description
-
-hola:
-  label: Hola
-  permalink: /hola
-  description: Hola tag description

docs/docs/CHANGELOG.md

@@ -11,9 +11,117 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 
 ## [Unreleased]
 
+## v1.17.0: Asahi sas Brutus
+
+- Ensure regexes can't end in newlines ([#372](https://github.com/TecharoHQ/anubis/issues/372))
+- Add documentation for default allow behavior (implicit rule)
+- Enable [importing configuration snippets](./admin/configuration/import.mdx) ([#321](https://github.com/TecharoHQ/anubis/pull/321))
+- Refactor check logic to be more generic and work on a Checker type
+- Add more AI user agents based on the [ai.robots.txt](https://github.com/ai-robots-txt/ai.robots.txt) project
+- Embedded challenge data in initial HTML response to improve performance
+- Added support to use Nginx' `auth_request` directive with Anubis
+- Added support to allow to restrict the allowed redirect domains
+- Whitelisted [DuckDuckBot](https://duckduckgo.com/duckduckgo-help-pages/results/duckduckbot/) in botPolicies
+- Improvements to build scripts to make them less independent of the build host
+- Improved the OpenGraph error logging
+- Added `Opera` to the `generic-browser` bot policy rule
+- Added FreeBSD rc.d script so can be run as a FreeBSD daemon
+- Allow requests from the Internet Archive
+- Added example nginx configuration to documentation
+- Added example Apache configuration to the documentation [#277](https://github.com/TecharoHQ/anubis/issues/277)
+- Move per-environment configuration details into their own pages
+- Added support for running anubis behind a prefix (e.g. `/myapp`)
+- Added headers support to bot policy rules
+- Moved configuration file from JSON to YAML by default
+- Added documentation on how to use Anubis with Traefik in Docker
+- Improved error handling in some edge cases
+- Disable `generic-bot-catchall` rule because of its high false positive rate in real-world scenarios
+- Moved all CSS inline to the Xess package, changed colors to be CSS variables
+- Set or append to `X-Forwarded-For` header unless the remote connects over a loopback address [#328](https://github.com/TecharoHQ/anubis/issues/328)
+- Fixed mojeekbot user agent regex
+- Added support for running anubis behind a base path (e.g. `/myapp`)
+- Reduce Anubis' paranoia with user cookies ([#365](https://github.com/TecharoHQ/anubis/pull/365))
+
+## v1.16.0
+
+Fordola rem Lupis
+
+> I want to make them pay! All of them! Everyone who ever mocked or looked down on me -- I want the power to make them pay!
+
+The following features are the "big ticket" items:
+
+- Added support for native Debian, Red Hat, and tarball packaging strategies including installation and use directions
+- A prebaked tarball has been added, allowing distros to build Anubis like they could in v1.15.x
+- The placeholder Anubis mascot has been replaced with a design by [CELPHASE](https://bsky.app/profile/celphase.bsky.social)
+- Verification page now shows hash rate and a progress bar for completion probability
+- Added support for [OpenGraph tags](https://ogp.me/) when rendering the challenge page. This allows for social previews to be generated when sharing the challenge page on social media platforms ([#195](https://github.com/TecharoHQ/anubis/pull/195))
+- Added support for passing the ed25519 signing key in a file with `-ed25519-private-key-hex-file` or `ED25519_PRIVATE_KEY_HEX_FILE`
+
+The other small fixes have been made:
+
+- Added a periodic cleanup routine for the decaymap that removes expired entries, ensuring stale data is properly pruned
+- Added a no-store Cache-Control header to the challenge page
+- Hide the directory listings for Anubis' internal static content
+- Changed `--debug-x-real-ip-default` to `--use-remote-address`, getting the IP address from the request's socket address instead
+- DroneBL lookups have been disabled by default
+- Static asset builds are now done on demand instead of the results being committed to source control
+- The Dockerfile has been removed as it is no longer in use
+- Developer documentation has been added to the docs site
+- Show more errors when some predictable challenge page errors happen ([#150](https://github.com/TecharoHQ/anubis/issues/150))
+- Added the `--debug-benchmark-js` flag for testing proof-of-work performance during development
+- Use `TrimSuffix` instead of `TrimRight` on containerbuild
+- Fix the startup logs to correctly show the address and port the server is listening on
+- Add [LibreJS](https://www.gnu.org/software/librejs/) banner to Anubis JavaScript to allow LibreJS users to run the challenge
+- Added a wait with button continue + 30 second auto continue after 30s if you click "Why am I seeing this?"
+- Fixed a typo in the challenge page title
+- Disabled running integration tests on Windows hosts due to it's reliance on posix features (see [#133](https://github.com/TecharoHQ/anubis/pull/133#issuecomment-2764732309))
+- Fixed minor typos
+- Added a Makefile to enable comfortable workflows for downstream packagers
+- Added `zizmor` for GitHub Actions static analysis
+- Fixed most `zizmor` findings
+- Enabled Dependabot
+- Added an air config for autoreload support in development ([#195](https://github.com/TecharoHQ/anubis/pull/195))
+- Added an `--extract-resources` flag to extract static resources to a local folder
+- Add noindex flag to all Anubis pages ([#227](https://github.com/TecharoHQ/anubis/issues/227))
+- Added `WEBMASTER_EMAIL` variable, if it is present then display that email address on error pages ([#235](https://github.com/TecharoHQ/anubis/pull/235), [#115](https://github.com/TecharoHQ/anubis/issues/115))
+- Hash pinned all GitHub Actions
+
+## v1.15.1
+
+Zenos yae Galvus: Echo 1
+
+Fixes a recurrence of [CVE-2025-24369](https://github.com/Xe/x/security/advisories/GHSA-56w8-8ppj-2p4f)
+due to an incorrect logic change in a refactor. This allows an attacker to mint a valid
+access token by passing any SHA-256 hash instead of one that matches the proof-of-work
+test.
+
+This case has been added as a regression test. It was not when CVE-2025-24369 was released
+due to the project not having the maturity required to enable this kind of regression testing.
+
+## v1.15.0
+
+Zenos yae Galvus
+
+> Yes...the coming days promise to be most interesting. Most interesting.
+
+Headline changes:
+
+- ed25519 signing keys for Anubis can be stored in the flag `--ed25519-private-key-hex` or envvar `ED25519_PRIVATE_KEY_HEX`; if one is not provided when Anubis starts, a new one is generated and logged
+- Add the ability to set the cookie domain with the envvar `COOKIE_DOMAIN=techaro.lol` for all domains under `techaro.lol`
+- Add the ability to set the cookie partitioned flag with the envvar `COOKIE_PARTITIONED=true`
+
+Many other small changes were made, including but not limited to:
+
 - Fixed and clarified installation instructions
-- Changed default challenge logic for "Gecko" in the User-Agent string
-  instead of "Mozilla" [#78](https://github.com/TecharoHQ/anubis/pull/78)
+- Introduced integration tests using Playwright
+- Refactor & Split up Anubis into cmd and lib.go
+- Fixed bot check to only apply if address range matches
+- Fix default difficulty setting that was broken in a refactor
+- Linting fixes
+- Make dark mode diff lines readable in the documentation
+- Fix CI based browser smoke test
+
+Users running Anubis' test suite may run into issues with the integration tests on Windows hosts. This is a known issue and will be fixed at some point in the future. In the meantime, use the Windows Subsystem for Linux (WSL).
 
 ## v1.14.2
 
@@ -22,6 +130,7 @@ Livia sas Junius: Echo 2
 - Remove default RSS reader rule as it may allow for a targeted attack against rails apps
   [#67](https://github.com/TecharoHQ/anubis/pull/67)
 - Whitelist MojeekBot in botPolicies [#47](https://github.com/TecharoHQ/anubis/issues/47)
+- botPolicies regex has been cleaned up [#66](https://github.com/TecharoHQ/anubis/pull/66)
 
 ## v1.14.1
 
@@ -58,7 +167,7 @@ Livia sas Junius
   [#21](https://github.com/TecharoHQ/anubis/pull/21)
 - Don't overflow the image when browser windows are small (eg. on phones)
   [#27](https://github.com/TecharoHQ/anubis/pull/27)
-- Lower the default difficulty to 4 from 5
+- Lower the default difficulty to 5 from 4
 - Don't duplicate work across multiple threads [#36](https://github.com/TecharoHQ/anubis/pull/36)
 - Documentation has been moved to https://anubis.techaro.lol/ with sources in docs/
 - Removed several visible AI artifacts (e.g., 6 fingers) [#37](https://github.com/TecharoHQ/anubis/pull/37)
@@ -101,4 +210,4 @@ Livia sas Junius
   ([fd6903a](https://github.com/TecharoHQ/anubis/commit/fd6903aeed315b8fddee32890d7458a9271e4798)).
 - Footer links on the check page now point to Techaro's brand
   ([4ebccb1](https://github.com/TecharoHQ/anubis/commit/4ebccb197ec20d024328d7f92cad39bbbe4d6359))
-- Anubis was imported from [Xe/x](https://github.com/Xe/x).
+- Anubis was imported from [Xe/x](https://github.com/Xe/x)

docs/docs/admin/configuration/_category_.json

@@ -0,0 +1,8 @@
+{
+  "label": "Configuration",
+  "position": 10,
+  "link": {
+    "type": "generated-index",
+    "description": "Detailed information about configuring parts of Anubis."
+  }
+}

docs/docs/admin/configuration/import.mdx

@@ -0,0 +1,147 @@
+# Importing configuration rules
+
+import Tabs from "@theme/Tabs";
+import TabItem from "@theme/TabItem";
+
+Anubis has the ability to let you import snippets of configuration into the main configuration file. This allows you to break up your config into smaller parts that get logically assembled into one big file.
+
+EG:
+
+<Tabs>
+<TabItem value="json" label="JSON">
+
+```json
+{
+  "bots": [
+    {
+      "import": "(data)/bots/ai-robots-txt.yaml"
+    },
+    {
+      "import": "(data)/bots/cloudflare-workers.yaml"
+    }
+  ]
+}
+```
+
+</TabItem>
+<TabItem value="yaml" label="YAML" default>
+
+```yaml
+bots:
+  # Pathological bots to deny
+  - # This correlates to data/bots/ai-robots-txt.yaml in the source tree
+    import: (data)/bots/ai-robots-txt.yaml
+  - import: (data)/bots/cloudflare-workers.yaml
+```
+
+</TabItem>
+</Tabs>
+
+Of note, a bot rule can either have inline bot configuration or import a bot config snippet. You cannot do both in a single bot rule.
+
+<Tabs>
+<TabItem value="json" label="JSON">
+
+```json
+{
+  "bots": [
+    {
+      "import": "(data)/bots/ai-robots-txt.yaml",
+      "name": "generic-browser",
+      "user_agent_regex": "Mozilla|Opera\n",
+      "action": "CHALLENGE"
+    }
+  ]
+}
+```
+
+</TabItem>
+<TabItem value="yaml" label="YAML" default>
+
+```yaml
+bots:
+  - import: (data)/bots/ai-robots-txt.yaml
+    name: generic-browser
+    user_agent_regex: >
+      Mozilla|Opera
+    action: CHALLENGE
+```
+
+</TabItem>
+</Tabs>
+
+This will return an error like this:
+
+```text
+config is not valid:
+config.BotOrImport: rule definition is invalid, you must set either bot rules or an import statement, not both
+```
+
+Paths can either be prefixed with `(data)` to import from the [the data folder in the Anubis source tree](https://github.com/TecharoHQ/anubis/tree/main/data) or anywhere on the filesystem. If you don't have access to the Anubis source tree, check /usr/share/docs/anubis/data or in the tarball you extracted Anubis from.
+
+## Writing snippets
+
+Snippets can be written in either JSON or YAML, with a preference for YAML. When writing a snippet, write the bot rules you want directly at the top level of the file in a list.
+
+Here is an example snippet that allows [IPv6 Unique Local Addresses](https://en.wikipedia.org/wiki/Unique_local_address) through Anubis:
+
+<Tabs>
+<TabItem value="json" label="JSON">
+
+```json
+[
+  {
+    "name": "ipv6-ula",
+    "action": "ALLOW",
+    "remote_addresses": ["fc00::/7"]
+  }
+]
+```
+
+</TabItem>
+<TabItem value="yaml" label="YAML" default>
+
+```yaml
+- name: ipv6-ula
+  action: ALLOW
+  remote_addresses:
+    - fc00::/7
+```
+
+</TabItem>
+</Tabs>
+
+## Extracting Anubis' embedded filesystem
+
+You can always extract the list of rules embedded into the Anubis binary with this command:
+
+```text
+anubis --extract-resources=static
+```
+
+This will dump the contents of Anubis' embedded data to a new folder named `static`:
+
+```text
+static
+├── apps
+│   └── gitea-rss-feeds.yaml
+├── botPolicies.json
+├── botPolicies.yaml
+├── bots
+│   ├── ai-robots-txt.yaml
+│   ├── cloudflare-workers.yaml
+│   ├── headless-browsers.yaml
+│   └── us-ai-scraper.yaml
+├── common
+│   ├── allow-private-addresses.yaml
+│   └── keep-internet-working.yaml
+└── crawlers
+    ├── bingbot.yaml
+    ├── duckduckbot.yaml
+    ├── googlebot.yaml
+    ├── internet-archive.yaml
+    ├── kagibot.yaml
+    ├── marginalia.yaml
+    ├── mojeekbot.yaml
+    └── qwantbot.yaml
+```

docs/docs/admin/configuration/open-graph.mdx

@@ -0,0 +1,47 @@
+---
+id: open-graph
+title: Open Graph Configuration
+---
+
+# Open Graph Configuration
+
+This page provides detailed information on how to configure [OpenGraph tag](https://ogp.me/) passthrough in Anubis. This enables social previews of resources protected by Anubis without having to exempt each scraper individually.
+
+## Configuration Options
+
+| Name             | Description                                               | Type     | Default | Example                 |
+|------------------|-----------------------------------------------------------|----------|---------|-------------------------|
+| `OG_PASSTHROUGH` | Enables or disables the Open Graph tag passthrough system | Boolean  | `false` | `OG_PASSTHROUGH=true`   |
+| `OG_EXPIRY_TIME` | Configurable cache expiration time for Open Graph tags    | Duration | `24h`   | `OG_EXPIRY_TIME=1h`     |
+
+## Usage
+
+To configure Open Graph tags, you can set the following environment variables, environment file or as flags in your Anubis configuration:
+
+```sh
+export OG_PASSTHROUGH=true
+export OG_EXPIRY_TIME=1h
+```
+
+## Implementation Details
+
+When `OG_PASSTHROUGH` is enabled, Anubis will:
+
+1. Check a local cache for the requested URL's Open Graph tags.
+2. If a cached entry exists and is still valid, return the cached tags.
+3. If the cached entry is stale or not found, fetch the URL, parse the Open Graph tags, update the cache, and return the new tags.
+
+The cache expiration time is controlled by `OG_EXPIRY_TIME`.
+
+## Example
+
+Here is an example of how to configure Open Graph tags in your Anubis setup:
+
+```sh
+export OG_PASSTHROUGH=true
+export OG_EXPIRY_TIME=1h
+```
+
+With these settings, Anubis will cache Open Graph tags for 1 hour and pass them through to the challenge page.
+
+For more information, refer to the [installation guide](../installation).

docs/docs/admin/configuration/redirect-domains.mdx

@@ -0,0 +1,94 @@
+---
+title: Redirect Domain Configuration
+---
+
+import Tabs from "@theme/Tabs";
+import TabItem from "@theme/TabItem";
+
+Anubis has an HTTP redirect in the middle of its check validation logic. This redirect allows Anubis to set a cookie on validated requests so that users don't need to pass challenges on every page load.
+
+This flow looks something like this:
+
+```mermaid
+sequenceDiagram
+  participant User
+  participant Challenge
+  participant Validation
+  participant Backend
+
+  User->>+Challenge: GET /
+  Challenge->>+User: Solve this challenge
+  User->>+Validation: Here's the solution, send me to /
+  Validation->>+User: Here's a cookie, go to /
+  User->>+Backend: GET /
+```
+
+However, in some cases a sufficiently dedicated attacker could trick a user into clicking on a validation link with a solution pre-filled out. For example:
+
+```mermaid
+sequenceDiagram
+  participant Hacker
+  participant User
+  participant Validation
+  participant Evil Site
+
+  Hacker->>+User: Click on yoursite.com with this solution
+  User->>+Validation: Here's a solution, send me to evilsite.com
+  Validation->>+User: Here's a cookie, go to evilsite.com
+  User->>+Evil Site: GET evilsite.com
+```
+
+If this happens, Anubis will throw an error like this:
+
+```text
+Redirect domain not allowed
+```
+
+## Configuring allowed redirect domains
+
+By default, Anubis will limit redirects to be on the same HTTP Host that Anubis is running on (EG: requests to yoursite.com cannot redirect outside of yoursite.com). If you need to set more than one domain, fill the `REDIRECT_DOMAINS` environment variable with a comma-separated list of domain names that Anubis should allow redirects to.
+
+:::note
+
+These domains are _an exact string match_, they do not support wildcard matches.
+
+:::
+
+<Tabs>
+  <TabItem value="env-file" label="Environment file" default>
+
+```shell
+# anubis.env
+
+REDIRECT_DOMAINS="yoursite.com,secretplans.yoursite.com"
+# ...
+```
+
+  </TabItem>
+  <TabItem value="docker-compose" label="Docker Compose">
+
+```yaml
+services:
+  anubis-nginx:
+    image: ghcr.io/techarohq/anubis:latest
+    environment:
+      REDIRECT_DOMAINS: "yoursite.com,secretplans.yoursite.com"
+      # ...
+```
+
+  </TabItem>
+  <TabItem value="k8s" label="Kubernetes">
+
+Inside your Deployment, StatefulSet, or Pod:
+
+```yaml
+- name: anubis
+  image: ghcr.io/techarohq/anubis:latest
+  env:
+    - name: REDIRECT_DOMAINS
+      value: "yoursite.com,secretplans.yoursite.com"
+    # ...
+```
+
+  </TabItem>
+</Tabs>

docs/docs/admin/configuration/subrequest-auth.mdx

@@ -0,0 +1,139 @@
+---
+title: Subrequest Authentication
+---
+
+import Tabs from "@theme/Tabs";
+import TabItem from "@theme/TabItem";
+
+Anubis can act in one of two modes:
+
+1. Reverse proxy (the default): Anubis sits in the middle of all traffic and then will reverse proxy it to its destination. This is the moral equivalent of a middleware in your favorite web framework.
+2. Subrequest authentication mode: Anubis listens for requests and if they don't pass muster then they are forwarded to Anubis for challenge processing. This is the equivalent of Anubis being a sidecar service.
+
+## Nginx
+
+Anubis can perform [subrequest authentication](https://docs.nginx.com/nginx/admin-guide/security-controls/configuring-subrequest-authentication/) with the `auth_request` module in Nginx. In order to set this up, keep the following things in mind:
+
+The `TARGET` environment variable in Anubis must be set to a space, eg:
+
+<Tabs>
+  <TabItem value="env-file" label="Environment file" default>
+
+```shell
+# anubis.env
+
+TARGET=" "
+# ...
+```
+
+  </TabItem>
+  <TabItem value="docker-compose" label="Docker Compose">
+
+```yaml
+services:
+  anubis-nginx:
+    image: ghcr.io/techarohq/anubis:latest
+    environment:
+      TARGET: " "
+      # ...
+```
+
+  </TabItem>
+  <TabItem value="k8s" label="Kubernetes">
+
+Inside your Deployment, StatefulSet, or Pod:
+
+```yaml
+- name: anubis
+  image: ghcr.io/techarohq/anubis:latest
+  env:
+    - name: TARGET
+      value: " "
+    # ...
+```
+
+  </TabItem>
+</Tabs>
+
+In order to configure this, you need to add the following location blocks to each server pointing to the service you want to protect:
+
+```nginx
+location /.within.website/ {
+    # Assumption: Anubis is running in the same network namespace as
+    # nginx on localhost TCP port 8923
+    proxy_pass http://127.0.0.1:8923;
+    auth_request off;
+}
+
+location @redirectToAnubis {
+    return 307 /.within.website/?redir=$scheme://$host$request_uri;
+    auth_request off;
+}
+```
+
+This sets up `/.within.website` to point to Anubis. Any requests that Anubis rejects or throws a challenge to will be sent here. This also sets up a named location `@redirectToAnubis` that will redirect any requests to Anubis for advanced processing.
+
+Finally, add this to your root location block:
+
+```nginx
+location / {
+    # diff-add
+    auth_request /.within.website/x/cmd/anubis/api/check;
+    # diff-add
+    error_page 401 = @redirectToAnubis;
+}
+```
+
+This will check all requests that don't match other locations with Anubis to ensure the client is genuine.
+
+This will make every request get checked by Anubis before it hits your backend. If you have other locations that don't need Anubis to do validation, add the `auth_request off` directive to their blocks:
+
+```nginx
+location /secret {
+    # diff-add
+    auth_request off;
+
+    # ...
+}
+```
+
+Here is a complete example of an Nginx server listening over TLS and pointing to Anubis:
+
+<details>
+  <summary>Complete example</summary>
+
+```nginx
+# /etc/nginx/conf.d/nginx.local.cetacean.club.conf
+
+server {
+  listen 443 ssl;
+  listen [::]:443 ssl;
+  server_name         nginx.local.cetacean.club;
+  ssl_certificate     /etc/techaro/pki/nginx.local.cetacean.club/tls.crt;
+  ssl_certificate_key /etc/techaro/pki/nginx.local.cetacean.club/tls.key;
+  ssl_protocols       TLSv1.2 TLSv1.3;
+  ssl_ciphers         HIGH:!aNULL:!MD5;
+
+  proxy_set_header X-Real-IP $remote_addr;
+  proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
+
+  location /.within.website/ {
+    proxy_pass http://localhost:8923;
+    auth_request off;
+  }
+
+  location @redirectToAnubis {
+    return 307 /.within.website/?redir=$scheme://$host$request_uri;
+    auth_request off;
+  }
+
+  location / {
+    auth_request /.within.website/x/cmd/anubis/api/check;
+    error_page 401 = @redirectToAnubis;
+    root /usr/share/nginx/html;
+    index index.html index.htm;
+  }
+}
+```
+
+</details>

docs/docs/admin/default-allow-behavior.mdx

@@ -0,0 +1,92 @@
+---
+title: Default allow behavior
+---
+
+import Tabs from "@theme/Tabs";
+import TabItem from "@theme/TabItem";
+
+# Default allow behavior
+
+Anubis is designed to be as unintrusive as possible to your existing infrastructure.
+
+By default, it allows all traffic unless a request matches a rule that explicitly denies or challenges it.
+
+Only requests matching a DENY or CHALLENGE rule are blocked or challenged. All other requests are allowed. This is called "the implicit rule".
+
+## Example: Minimal policy
+
+If your policy only blocks a specific bot, all other requests will be allowed:
+
+<Tabs>
+<TabItem value="json" label="JSON" default>
+
+```json
+{
+  "bots": [
+    {
+      "name": "block-amazonbot",
+      "user_agent_regex": "Amazonbot",
+      "action": "DENY"
+    }
+  ]
+}
+```
+
+</TabItem>
+<TabItem value="yaml" label="YAML">
+
+```yaml
+- name: block-amazonbot
+  user_agent_regex: Amazonbot
+  action: DENY
+```
+
+</TabItem>
+</Tabs>
+
+## How to deny by default
+
+If you want to deny all traffic except what you explicitly allow, add a catch-all deny rule at the end of your policy list. Make sure to add ALLOW rules for any traffic you want to permit before this rule.
+
+<Tabs>
+<TabItem value="json" label="JSON" default>
+
+```json
+{
+  "bots": [
+    {
+      "name": "allow-goodbot",
+      "user_agent_regex": "GoodBot",
+      "action": "ALLOW"
+    },
+    {
+      "name": "catch-all-deny",
+      "path_regex": ".*",
+      "action": "DENY"
+    }
+  ]
+}
+```
+
+</TabItem>
+<TabItem value="yaml" label="YAML">
+
+```yaml
+- name: allow-goodbot
+  user_agent_regex: GoodBot
+  action: ALLOW
+- name: catch-all-deny
+  path_regex: .*
+  action: DENY
+```
+
+</TabItem>
+</Tabs>
+
+## Final remarks
+
+- Rules are evaluated in order; the first match wins.
+- The implicit allow rule is always last and cannot be removed.
+- Use your logs to monitor what traffic is being allowed by default.
+
+See [Policy Definitions](./policies) for more details on writing rules.

docs/docs/admin/environments/_category_.json

@@ -0,0 +1,8 @@
+{
+  "label": "Environments",
+  "position": 20,
+  "link": {
+    "type": "generated-index",
+    "description": "Detailed information about individual environments (such as HTTP servers, platforms, etc.) Anubis is known to work with."
+  }
+}

docs/docs/admin/environments/apache.mdx

@@ -0,0 +1,151 @@
+# Apache
+
+import Tabs from "@theme/Tabs";
+import TabItem from "@theme/TabItem";
+
+Anubis is intended to be a filter proxy. The way to integrate this is to break your configuration up into two parts: TLS termination and then HTTP routing. Consider this diagram:
+
+```mermaid
+---
+title: Apache as tls terminator and HTTP router
+---
+
+flowchart LR
+    T(User Traffic)
+    subgraph Apache 2
+        TCP(TCP 80/443)
+        US(TCP 3001)
+    end
+
+    An(Anubis)
+    B(Backend)
+
+    T --> |TLS termination| TCP
+    TCP --> |Traffic filtering| An
+    An --> |Happy traffic| US
+    US --> |whatever you're doing| B
+```
+
+Effectively you have one trip through Apache to do TLS termination, a detour through Anubis for traffic scrubbing, and then going to the backend directly. This final socket is what will do HTTP routing.
+
+:::note
+
+These examples assume that you are using a setup where your nginx configuration is made up of a bunch of files in `/etc/httpd/conf.d/*.conf`. This is not true for all deployments of Apache. If you are not in such an environment, append these snippets to your `/etc/httpd/conf/httpd.conf` file.
+
+:::
+
+## Dependencies
+
+Install the following dependencies for proxying HTTP:
+
+<Tabs>
+  <TabItem value="rpm" label="Red Hat / RPM" default>
+
+```text
+dnf -y install mod_proxy_html
+```
+
+  </TabItem>
+  <TabItem value="deb" label="Debian / Ubuntu / apt">
+
+```text
+apt-get install -y libapache2-mod-proxy-html libxml2-dev
+```
+
+  </TabItem>
+</Tabs>
+
+## Configuration
+
+Assuming you are protecting `anubistest.techaro.lol`, you need the following server configuration blocks:
+
+1. A block on port 80 that forwards HTTP to HTTPS
+2. A block on port 443 that terminates TLS and forwards to Anubis
+3. A block on port 3001 that actually serves your websites
+
+```text
+# Plain HTTP redirect to HTTPS
+<VirtualHost *:80>
+       ServerAdmin your@email.here
+       ServerName anubistest.techaro.lol
+       DocumentRoot /var/www/anubistest.techaro.lol
+       ErrorLog /var/log/httpd/anubistest.techaro.lol_error.log
+       CustomLog /var/log/httpd/anubistest.techaro.lol_access.log combined
+       RewriteEngine on
+       RewriteCond %{SERVER_NAME} =anubistest.techaro.lol
+       RewriteRule ^ https://%{SERVER_NAME}%{REQUEST_URI} [END,NE,R=permanent]
+</VirtualHost>
+
+# HTTPS listener that forwards to Anubis
+<VirtualHost *:443>
+       ServerAdmin your@email.here
+       ServerName anubistest.techaro.lol
+       DocumentRoot /var/www/anubistest.techaro.lol
+       ErrorLog /var/log/httpd/anubistest.techaro.lol_error.log
+       CustomLog /var/log/httpd/anubistest.techaro.lol_access.log combined
+
+       SSLCertificateFile /etc/letsencrypt/live/anubistest.techaro.lol/fullchain.pem
+       SSLCertificateKeyFile /etc/letsencrypt/live/anubistest.techaro.lol/privkey.pem
+       Include /etc/letsencrypt/options-ssl-apache.conf
+
+       # These headers need to be set or else Anubis will
+       # throw an "admin misconfiguration" error.
+       RequestHeader set "X-Real-Ip" expr=%{REMOTE_ADDR}
+       RequestHeader set X-Forwarded-Proto "https"
+
+       ProxyPreserveHost On
+
+       ProxyRequests Off
+       ProxyVia Off
+
+       # Replace 9000 with the port Anubis listens on
+       ProxyPass / http://[::1]:9000/
+       ProxyPassReverse / http://[::1]:9000/
+</VirtualHost>
+</IfModule>
+
+# Actual website config
+<VirtualHost *:3001>
+       ServerAdmin your@email.here
+       ServerName anubistest.techaro.lol
+       DocumentRoot /var/www/anubistest.techaro.lol
+       ErrorLog /var/log/httpd/anubistest.techaro.lol_error.log
+       CustomLog /var/log/httpd/anubistest.techaro.lol_access.log combined
+</VirtualHost>
+```
+
+Make sure to add a separate configuration file for the listener on port 3001:
+
+```text
+# /etc/httpd/conf.d/listener-3001.conf
+
+Listen 3001
+```
+
+This can be repeated for multiple sites. Anubis does not care about the HTTP `Host` header and will happily cope with multiple websites via the same instance.
+
+Then reload your Apache config and load your website. You should see Anubis protecting your apps!
+
+```text
+sudo systemctl reload httpd.service
+```
+
+## Troubleshooting
+
+Here are some answers to questions that came in in testing:
+
+### I'm running on a Red Hat distribution and Apache is saying "service unavailable" for every page load
+
+If you see a "Service unavailable" error on every page load and run a Red Hat derived distribution, you are missing a `selinux` setting. The exact command will be in a journalctl log message like this:
+
+```text
+*****  Plugin catchall_boolean (89.3 confidence) suggests   ******************
+
+If you want to allow HTTPD scripts and modules to connect to the network using TCP.
+Then you must tell SELinux about this by enabling the 'httpd_can_network_connect' boolean.
+
+Do
+setsebool -P httpd_can_network_connect 1
+```
+
+This will fix the error immediately.

docs/docs/admin/environments/docker-compose.mdx

@@ -0,0 +1,26 @@
+# Docker compose
+
+Docker compose is typically used in concert with other load balancers such as [Apache](./apache.mdx) or [Nginx](./nginx.mdx). Below is a minimal example showing you how to set up an instance of Anubis listening on host port 8080 that points to a static website containing data in `./www`:
+
+```yaml
+services:
+  anubis-nginx:
+    image: ghcr.io/techarohq/anubis:latest
+    environment:
+      BIND: ":8080"
+      DIFFICULTY: "4"
+      METRICS_BIND: ":9090"
+      SERVE_ROBOTS_TXT: "true"
+      TARGET: "http://nginx"
+      POLICY_FNAME: "/data/cfg/botPolicy.yaml"
+      OG_PASSTHROUGH: "true"
+      OG_EXPIRY_TIME: "24h"
+    ports:
+      - 8080:8080
+    volumes:
+      - "./botPolicy.yaml:/data/cfg/botPolicy.yaml:ro"
+  nginx:
+    image: nginx
+    volumes:
+      - "./www:/usr/share/nginx/html"
+```

docs/docs/admin/environments/kubernetes.mdx

@@ -0,0 +1,128 @@
+# Kubernetes
+
+When setting up Anubis in Kubernetes, you want to make sure that you thread requests through Anubis kinda like this:
+
+```mermaid
+---
+title: Anubis embedded into workload pods
+---
+
+flowchart LR
+    T(User Traffic)
+
+    IngressController(IngressController)
+
+    subgraph Service
+        AnPort(Anubis Port)
+        BPort(Backend Port)
+    end
+
+    subgraph Pod
+        An(Anubis)
+        B(Backend)
+    end
+
+    T -->  IngressController
+    IngressController --> AnPort
+    AnPort --> An
+    An --> B
+```
+
+Anubis is lightweight enough that you should be able to have many instances of it running without many problems. If this is a concern for you, please check out [ingress-anubis](https://github.com/jaredallard/ingress-anubis?ref=anubis.techaro.lol).
+
+This example makes the following assumptions:
+
+- Your target service is listening on TCP port `5000`.
+- Anubis will be listening on port `8080`.
+
+Adjust these values as facts and circumstances demand.
+
+Create a secret with the signing key Anubis should use for its responses:
+
+```
+kubectl create secret generic anubis-key \
+  --namespace default \
+  --from-literal=ED25519_PRIVATE_KEY_HEX=$(openssl rand -hex 32)
+```
+
+Attach Anubis to your Deployment:
+
+```yaml
+containers:
+  # ...
+  - name: anubis
+    image: ghcr.io/techarohq/anubis:latest
+    imagePullPolicy: Always
+    env:
+      - name: "BIND"
+        value: ":8080"
+      - name: "DIFFICULTY"
+        value: "4"
+      - name: ED25519_PRIVATE_KEY_HEX
+        valueFrom:
+          secretKeyRef:
+            name: anubis-key
+            key: ED25519_PRIVATE_KEY_HEX
+      - name: "METRICS_BIND"
+        value: ":9090"
+      - name: "SERVE_ROBOTS_TXT"
+        value: "true"
+      - name: "TARGET"
+        value: "http://localhost:5000"
+      - name: "OG_PASSTHROUGH"
+        value: "true"
+      - name: "OG_EXPIRY_TIME"
+        value: "24h"
+    resources:
+      limits:
+        cpu: 750m
+        memory: 256Mi
+      requests:
+        cpu: 250m
+        memory: 256Mi
+    securityContext:
+      runAsUser: 1000
+      runAsGroup: 1000
+      runAsNonRoot: true
+      allowPrivilegeEscalation: false
+      capabilities:
+        drop:
+          - ALL
+      seccompProfile:
+        type: RuntimeDefault
+```
+
+Then add a Service entry for Anubis:
+
+```yaml
+# ...
+spec:
+  ports:
+    # diff-add
+    - protocol: TCP
+      # diff-add
+      port: 8080
+      # diff-add
+      targetPort: 8080
+      # diff-add
+      name: anubis
+```
+
+Then point your Ingress to the Anubis port:
+
+```yaml
+  rules:
+  - host: git.xeserv.us
+    http:
+      paths:
+      - pathType: Prefix
+        path: "/"
+        backend:
+          service:
+            name: git
+            port:
+              # diff-remove
+              name: http
+              # diff-add
+              name: anubis
+```

docs/docs/admin/environments/nginx.mdx

@@ -0,0 +1,166 @@
+# Nginx
+
+Anubis is intended to be a filter proxy. The way to integrate this with nginx is to break your configuration up into two parts: TLS termination and then HTTP routing. Consider this diagram:
+
+```mermaid
+---
+title: Nginx as tls terminator and HTTP router
+---
+
+flowchart LR
+    T(User Traffic)
+    subgraph Nginx
+        TCP(TCP 80/443)
+        US(Unix Socket or
+another TCP port)
+    end
+
+    An(Anubis)
+    B(Backend)
+
+    T --> |TLS termination| TCP
+    TCP --> |Traffic filtering| An
+    An --> |Happy traffic| US
+    US --> |whatever you're doing| B
+```
+
+Instead of your traffic going right from TLS termination into the backend, it takes a detour through Anubis. Anubis filters out the "bad" traffic and then passes the "good" traffic to another socket that Nginx has open. This final socket is what you will use to do HTTP routing.
+
+Effectively, you have two roles for nginx: TLS termination (converting HTTPS to HTTP) and HTTP routing (distributing requests to the individual vhosts). This can stack with something like Apache in case you have a legacy deployment. Make sure you have the right [TLS certificates configured](https://code.kuederle.com/letsencrypt/) at the TLS termination level.
+
+:::note
+
+These examples assume that you are using a setup where your nginx configuration is made up of a bunch of files in `/etc/nginx/conf.d/*.conf`. This is not true for all deployments of nginx. If you are not in such an environment, append these snippets to your `/etc/nginx/nginx.conf` file.
+
+:::
+
+Assuming that we are protecting `anubistest.techaro.lol`, here's what the server configuration file would look like:
+
+```nginx
+# /etc/nginx/conf.d/server-anubistest-techaro-lol.conf
+
+# HTTP - Redirect all HTTP traffic to HTTPS
+server {
+  listen 80;
+  listen [::]:80;
+
+  server_name anubistest.techaro.lol;
+
+  location / {
+    return 301 https://$host$request_uri;
+  }
+}
+
+# TLS termination server, this will listen over TLS (https) and then
+# proxy all traffic to the target via Anubis.
+server {
+  # Listen on TCP port 443 with TLS (https) and HTTP/2
+  listen 443 ssl http2;
+  listen [::]:443 ssl http2;
+
+  location / {
+    proxy_set_header Host $host;
+    proxy_set_header X-Real-IP $remote_addr;
+    proxy_pass http://anubis;
+  }
+
+  server_name anubistest.techaro.lol;
+
+  ssl_certificate     /path/to/your/certs/anubistest.techaro.lol.crt;
+  ssl_certificate_key /path/to/your/certs/anubistest.techaro.lol.key;
+}
+
+# Backend server, this is where your webapp should actually live.
+server {
+  listen unix:/run/nginx/nginx.sock;
+
+  server_name anubistest.techaro.lol;
+  root "/srv/http/anubistest.techaro.lol";
+  index index.html;
+
+  # Your normal configuration can go here
+  # location .php { fastcgi...} etc.
+}
+```
+
+:::tip
+
+You can copy the `location /` block into a separate file named something like `conf-anubis.inc` and then include it inline to other `server` blocks:
+
+```nginx
+# /etc/nginx/conf.d/conf-anubis.inc
+
+# Forward to anubis
+location / {
+  proxy_set_header Host $host;
+  proxy_set_header X-Real-IP $remote_addr;
+  proxy_pass http://anubis;
+}
+```
+
+Then in a server block:
+
+<details>
+<summary>Full nginx config</summary>
+
+```nginx
+# /etc/nginx/conf.d/server-mimi-techaro-lol.conf
+
+server {
+  # Listen on 443 with SSL
+  listen 443 ssl http2;
+  listen [::]:443 ssl http2;
+
+  # Slipstream via Anubis
+  include "conf-anubis.inc";
+
+  server_name mimi.techaro.lol;
+
+  ssl_certificate     /path/to/your/certs/mimi.techaro.lol.crt;
+  ssl_certificate_key /path/to/your/certs/mimi.techaro.lol.key;
+}
+
+server {
+  listen unix:/run/nginx/nginx.sock;
+
+  server_name mimi.techaro.lol;
+  root "/srv/http/mimi.techaro.lol";
+  index index.html;
+
+  # Your normal configuration can go here
+  # location .php { fastcgi...} etc.
+}
+```
+
+</details>
+
+:::
+
+Create an upstream for Anubis.
+
+```nginx
+# /etc/nginx/conf.d/upstream-anubis.conf
+
+upstream anubis {
+  # Make sure this matches the values you set for `BIND` and `BIND_NETWORK`.
+  # If this does not match, your services will not be protected by Anubis.
+
+  # Try anubis first over a UNIX socket
+  server unix:/run/anubis/nginx.sock;
+  #server http://127.0.0.1:8923;
+
+  # Optional: fall back to serving the websites directly. This allows your
+  # websites to be resilient against Anubis failing, at the risk of exposing
+  # them to the raw internet without protection. This is a tradeoff and can
+  # be worth it in some edge cases.
+  #server unix:/run/nginx.sock backup;
+}
+```
+
+This can be repeated for multiple sites. Anubis does not care about the HTTP `Host` header and will happily cope with multiple websites via the same instance.
+
+Then reload your nginx config and load your website. You should see Anubis protecting your apps!
+
+```text
+sudo systemctl reload nginx.service
+```

docs/docs/admin/environments/traefik.mdx

@@ -0,0 +1,215 @@
+---
+id: traefik
+title: Integrate Anubis with Traefik in a Docker Compose Environment
+---
+
+
+:::note
+
+ This only talks about integration through Compose,
+ but it also applies to docker cli options.
+
+:::
+
+Currently, Anubis doesn't have any Traefik middleware,
+so you need to manually route it between Traefik and your target service.
+This routing is done per labels in Traefik.
+
+In this example, we will use 4 Containers:
+
+- `traefik` - the Traefik instance
+- `anubis` - the Anubis instance
+- `target` - our service to protect (`traefik/whoami` in this case)
+- `target2` - a second service that isn't supposed to be protected (`traefik/whoami` in this case)
+
+There are 3 steps we need to follow:
+
+1. Create a new exclusive Traefik endpoint for Anubis
+2. Pass all unspecified requests to Anubis
+3. Let Anubis pass all verified requests back to Traefik on its exclusive endpoint
+
+## Diagram of Flow
+
+This is a small diagram depicting the flow.
+Keep in mind that `8080` or `80` can be anything depending on your containers.
+
+```mermaid
+flowchart LR
+user[User]
+traefik[Traefik]
+anubis[Anubis]
+target[Target]
+
+user-->|:443 - Requesting Service|traefik
+traefik-->|:8080 - Passing to Anubis|anubis
+anubis-->|:3923 - Passing back to Traefik|traefik
+traefik-->|:80 - Passing to the target|target
+```
+
+## Create an Exclusive Anubis Endpoint in Traefik
+
+There are 2 ways of registering a new endpoint in Traefik.
+Which one to use depends on how you configured your Traefik so far.
+
+**CLI Options:**
+
+```yml
+--entrypoints.anubis.address=:3923
+```
+
+**traefik.yml:**
+
+```yml
+entryPoints:
+  anubis:
+    address: ":3923"
+```
+
+It is important that the specified port isn't actually reachable from the outside,
+but only exposed in the Docker network.
+Exposing the Anubis port on Traefik directly will allow direct unprotected access to all containers behind it.
+
+## Passing all unspecified Web Requests to Anubis
+
+There are cases where you want Traefik to still route some requests without protection, just like before.
+To achieve this, we can register Anubis as the default handler for non-protected requests.
+
+We also don't want users to get SSL Errors during the checking phase,
+thus we also need to let Traefik provide SSL Certs for our endpoint.
+This example expects an TLS cert resolver called `le`.
+
+We also expect there to be an endpoint called `websecure` for HTTPS in this example.
+
+This is an example of the required labels to configure Traefik on the Anubis container:
+
+```yml
+labels:
+  - traefik.enable=true # Enabling Traefik
+  - traefik.docker.network=traefik # Telling Traefik which network to use
+  - traefik.http.routers.anubis.priority=1 # Setting Anubis to the lowest priority, so it only takes the slack
+  - traefik.http.routers.anubis.rule=PathRegexp(`.*`) # Wildcard match every path
+  - traefik.http.routers.anubis.entrypoints=websecure # Listen on HTTPS
+  - traefik.http.services.anubis.loadbalancer.server.port=8080 # Telling Traefik to which port it should route requests
+  - traefik.http.routers.anubis.service=anubis # Telling Traefik to use the above specified port
+  - traefik.http.routers.anubis.tls.certresolver=le # Telling Traefik to resolve a Cert for Anubis
+```
+
+## Passing all Verified Requests Back Correctly to Traefik
+
+To pass verified requests back to Traefik,
+we only need to configure Anubis using its environment variables:
+
+```yml
+environment:
+  - BIND=:8080
+  - TARGET=http://traefik:3923
+```
+
+## Full Example Config
+
+Now that we know how to pass all requests back and forth, here is the example.
+This example contains 2 services: one that is protected and the other one that is not.
+
+**compose.yml**
+
+```yml
+services:
+  traefik:
+    image: traefik:v3.3
+    ports:
+      - 80:80
+      - 443:443
+    volumes:
+      - /var/run/docker.sock:/var/run/docker.sock
+      - ./letsencrypt:/letsencrypt
+      - ./traefik.yml:/traefik.yml:ro
+    networks:
+      - traefik
+    labels:
+      # Enable Traefik
+      - traefik.enable=true
+      - traefik.docker.network=traefik
+      # Redirect any HTTP to HTTPS
+      - traefik.http.middlewares.redirect-to-https.redirectscheme.scheme=https
+      - traefik.http.routers.web.rule=PathPrefix(`/`)
+      - traefik.http.routers.web.entrypoints=web
+      - traefik.http.routers.web.middlewares=redirect-to-https
+      - traefik.http.routers.web.tls=false
+
+  anubis:
+    image: ghcr.io/techarohq/anubis:main
+    environment:
+      # Telling Anubis, where to listen for Traefik
+      - BIND=:8080
+      # Telling Anubis to point to Traefik via the Docker network
+      - TARGET=http://traefik:3923
+    networks:
+      - traefik
+    labels:
+      - traefik.enable=true # Enabling Traefik
+      - traefik.docker.network=traefik # Telling Traefik which network to use
+      - traefik.http.routers.anubis.priority=1 # Setting Anubis to the lowest priority, so it only takes the slack
+      - traefik.http.routers.anubis.rule=PathRegexp(`.*`) # wildcard match anything
+      - traefik.http.routers.anubis.entrypoints=websecure # Listen on HTTPS
+      - traefik.http.services.anubis.loadbalancer.server.port=8080 # Telling Traefik to which port it should route requests
+      - traefik.http.routers.anubis.service=anubis # Telling Traefik to use the above specified port
+      - traefik.http.routers.anubis.tls.certresolver=le # Telling Traefik to resolve a Cert for Anubis
+
+  # Protected by Anubis
+  target:
+    image: traefik/whoami:latest
+    networks:
+      - traefik
+    labels:
+      - traefik.enable=true # Enabling Traefik
+      - traefik.docker.network=traefik # Telling Traefik which network to use
+      - traefik.http.routers.target.rule=Host(`example.com`) # Only Matching Requests for example.com
+      - traefik.http.routers.target.entrypoints=anubis # Listening on the exclusive Anubis Network
+      - traefik.http.services.target.loadbalancer.server.port=80 # Telling Traefik where to receive requests
+      - traefik.http.routers.target.service=target # Telling Traefik to use the above specified port
+
+  # Not Protected by Anubis
+  target2:
+    image: traefik/whoami:latest
+    networks:
+      - traefik
+    labels:
+      - traefik.enable=true # Eneabling Traefik
+      - traefik.docker.network=traefik # Telling Traefik which network to use
+      - traefik.http.routers.target2.rule=Host(`another.com`) # Only Matching Requests for example.com
+      - traefik.http.routers.target2.entrypoints=websecure # Listening on the exclusive Anubis Network
+      - traefik.http.services.target2.loadbalancer.server.port=80 # Telling Traefik where to receive requests
+      - traefik.http.routers.target2.service=target2 # Telling Traefik to use the above specified port
+      - traefik.http.routers.target2.tls.certresolver=le # Telling Traefik to resolve a Cert for this Target
+
+networks:
+  traefik:
+    name: traefik
+```
+
+**traefik.yml**
+
+```yml
+api:
+  insecure: false # shouldn't be enabled in prod
+
+entryPoints:
+  # Web
+  web:
+    address: ":80"
+  websecure:
+    address: ":443"
+  # Anubis
+  anubis:
+    address: ":3923"
+
+certificatesResolvers:
+  le:
+    acme:
+      tlsChallenge: {}
+      email: "admin@example.com"
+      storage: "/letsencrypt/acme.json"
+
+providers:
+  docker: {}
+```

docs/docs/admin/installation.mdx

@@ -2,8 +2,33 @@
 title: Setting up Anubis
 ---
 
+import RandomKey from "@site/src/components/RandomKey";
+
+import Tabs from "@theme/Tabs";
+import TabItem from "@theme/TabItem";
+
 Anubis is meant to sit between your reverse proxy (such as Nginx or Caddy) and your target service. One instance of Anubis must be used per service you are protecting.
 
+<center>
+
+```mermaid
+---
+title: With Anubis installed
+---
+
+flowchart LR
+    LB(Load balancer /
+TLS terminator)
+    Anubis(Anubis)
+    App(App)
+
+    LB --> Anubis --> App
+```
+
+</center>
+
+## Docker image conventions
+
 Anubis is shipped in the Docker repo [`ghcr.io/techarohq/anubis`](https://github.com/TecharoHQ/anubis/pkgs/container/anubis). The following tags exist for your convenience:
 
 | Tag                 | Meaning                                                                                                                            |
@@ -11,129 +36,103 @@ Anubis is shipped in the Docker repo [`ghcr.io/techarohq/anubis`](https://github
 | `latest`            | The latest [tagged release](https://github.com/TecharoHQ/anubis/releases), if you are in doubt, start here.                        |
 | `v<version number>` | The Anubis image for [any given tagged release](https://github.com/TecharoHQ/anubis/tags)                                          |
 | `main`              | The current build on the `main` branch. Only use this if you need the latest and greatest features as they are merged into `main`. |
-| `pr-<number>`       | The build associated with PR `#<number>`. Only use this for debugging issues fixed by a PR.                                        |
-
-Other methods to install Anubis may exist, but the Docker image is currently the only supported method.
 
 The Docker image runs Anubis as user ID 1000 and group ID 1000. If you are mounting external volumes into Anubis' container, please be sure they are owned by or writable to this user/group.
 
 Anubis has very minimal system requirements. I suspect that 128Mi of ram may be sufficient for a large number of concurrent clients. Anubis may be a poor fit for apps that use WebSockets and maintain open connections, but I don't have enough real-world experience to know one way or another.
 
+## Native packages
+
+For more detailed information on installing Anubis with native packages, please read [the native install directions](./native-install.mdx).
+
+## Environment variables
+
 Anubis uses these environment variables for configuration:
 
-| Environment Variable   | Default value              | Explanation                                                                                                                                                                                                                                                                              |
-| :--------------------- | :------------------------- | :--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- |
-| `BIND`                 | `:8923`                    | The network address that Anubis listens on. For `unix`, set this to a path: `/run/anubis/instance.sock`                                                                                                                                                                                  |
-| `BIND_NETWORK`         | `tcp`                      | The address family that Anubis listens on. Accepts `tcp`, `unix` and anything Go's [`net.Listen`](https://pkg.go.dev/net#Listen) supports.                                                                                                                                               |
-| `DIFFICULTY`           | `5`                        | The difficulty of the challenge, or the number of leading zeroes that must be in successful responses.                                                                                                                                                                                   |
-| `METRICS_BIND`         | `:9090`                    | The network address that Anubis serves Prometheus metrics on. See `BIND` for more information.                                                                                                                                                                                           |
-| `METRICS_BIND_NETWORK` | `tcp`                      | The address family that the Anubis metrics server listens on. See `BIND_NETWORK` for more information.                                                                                                                                                                                   |
-| `SOCKET_MODE`          | `0770`                     | *Only used when at least one of the `*_BIND_NETWORK` variables are set to `unix`.* The socket mode (permissions) for Unix domain sockets.                                                                                                                                                |
-| `POLICY_FNAME`         | unset                      | The file containing [bot policy configuration](./policies.md). See the bot policy documentation for more details. If unset, the default bot policy configuration is used.                                                                                                                |
-| `SERVE_ROBOTS_TXT`     | `false`                    | If set `true`, Anubis will serve a default `robots.txt` file that disallows all known AI scrapers by name and then additionally disallows every scraper. This is useful if facts and circumstances make it difficult to change the underlying service to serve such a `robots.txt` file. |
-| `TARGET`               | `http://localhost:3923`    | The URL of the service that Anubis should forward valid requests to. Supports Unix domain sockets, set this to a URI like so: `unix:///path/to/socket.sock`.                                                                                                                             |
+| Environment Variable           | Default value           | Explanation                                                                                                                                                                                                                                                                                                          |
+| :----------------------------- | :---------------------- | :------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- |
+| `BASE_PREFIX`                  | unset                   | If set, adds a global prefix to all Anubis endpoints. For example, setting this to `/myapp` would make Anubis accessible at `/myapp/` instead of `/`. This is useful when running Anubis behind a reverse proxy that routes based on path prefixes.                                                                  |
+| `BIND`                         | `:8923`                 | The network address that Anubis listens on. For `unix`, set this to a path: `/run/anubis/instance.sock`                                                                                                                                                                                                              |
+| `BIND_NETWORK`                 | `tcp`                   | The address family that Anubis listens on. Accepts `tcp`, `unix` and anything Go's [`net.Listen`](https://pkg.go.dev/net#Listen) supports.                                                                                                                                                                           |
+| `COOKIE_DOMAIN`                | unset                   | The domain the Anubis challenge pass cookie should be set to. This should be set to the domain you bought from your registrar (EG: `techaro.lol` if your webapp is running on `anubis.techaro.lol`). See [here](https://stackoverflow.com/a/1063760) for more information.                                           |
+| `COOKIE_PARTITIONED`           | `false`                 | If set to `true`, enables the [partitioned (CHIPS) flag](https://developers.google.com/privacy-sandbox/cookies/chips), meaning that Anubis inside an iframe has a different set of cookies than the domain hosting the iframe.                                                                                       |
+| `DIFFICULTY`                   | `4`                     | The difficulty of the challenge, or the number of leading zeroes that must be in successful responses.                                                                                                                                                                                                               |
+| `ED25519_PRIVATE_KEY_HEX`      | unset                   | The hex-encoded ed25519 private key used to sign Anubis responses. If this is not set, Anubis will generate one for you. This should be exactly 64 characters long. See below for details.                                                                                                                           |
+| `ED25519_PRIVATE_KEY_HEX_FILE` | unset                   | Path to a file containing the hex-encoded ed25519 private key. Only one of this or its sister option may be set.                                                                                                                                                                                                     |
+| `METRICS_BIND`                 | `:9090`                 | The network address that Anubis serves Prometheus metrics on. See `BIND` for more information.                                                                                                                                                                                                                       |
+| `METRICS_BIND_NETWORK`         | `tcp`                   | The address family that the Anubis metrics server listens on. See `BIND_NETWORK` for more information.                                                                                                                                                                                                               |
+| `OG_EXPIRY_TIME`               | `24h`                   | The expiration time for the Open Graph tag cache.                                                                                                                                                                                                                                                                    |
+| `OG_PASSTHROUGH`               | `false`                 | If set to `true`, Anubis will enable Open Graph tag passthrough.                                                                                                                                                                                                                                                     |
+| `POLICY_FNAME`                 | unset                   | The file containing [bot policy configuration](./policies.mdx). See the bot policy documentation for more details. If unset, the default bot policy configuration is used.                                                                                                                                           |
+| `REDIRECT_DOMAINS`             | unset                   | If set, restrict the domains that Anubis can redirect to when passing a challenge.<br/><br/>If this is unset, Anubis may redirect to any domain which could cause security issues in the unlikely case that an attacker passes a challenge for your browser and then tricks you into clicking a link to your domain. |
+| `SERVE_ROBOTS_TXT`             | `false`                 | If set `true`, Anubis will serve a default `robots.txt` file that disallows all known AI scrapers by name and then additionally disallows every scraper. This is useful if facts and circumstances make it difficult to change the underlying service to serve such a `robots.txt` file.                             |
+| `SOCKET_MODE`                  | `0770`                  | _Only used when at least one of the `*_BIND_NETWORK` variables are set to `unix`._ The socket mode (permissions) for Unix domain sockets.                                                                                                                                                                            |
+| `TARGET`                       | `http://localhost:3923` | The URL of the service that Anubis should forward valid requests to. Supports Unix domain sockets, set this to a URI like so: `unix:///path/to/socket.sock`.                                                                                                                                                         |
+| `USE_REMOTE_ADDRESS`           | unset                   | If set to `true`, Anubis will take the client's IP from the network socket. For production deployments, it is expected that a reverse proxy is used in front of Anubis, which pass the IP using headers, instead.                                                                                                    |
+| `WEBMASTER_EMAIL`              | unset                   | If set, shows a contact email address when rendering error pages. This email address will be how users can get in contact with administrators.                                                                                                                                                                       |
 
-## Docker compose
+For more detailed information on configuring Open Graph tags, please refer to the [Open Graph Configuration](./configuration/open-graph.mdx) page.
 
-Add Anubis to your compose file pointed at your service:
+### Using Base Prefix
 
-```yaml
-services:
-  anubis-nginx:
-    image: ghcr.io/techarohq/anubis:latest
-    environment:
-      BIND: ":8080"
-      DIFFICULTY: "5"
-      METRICS_BIND: ":9090"
-      SERVE_ROBOTS_TXT: "true"
-      TARGET: "http://nginx"
-      POLICY_FNAME: "/data/cfg/botPolicy.json"
-    ports:
-      - 8080:8080
-    volumes:
-      - "./botPolicy.json:/data/cfg/botPolicy.json:ro"
-  nginx:
-    image: nginx
-    volumes:
-      - "./www:/usr/share/nginx/html"
+The `BASE_PREFIX` environment variable allows you to run Anubis behind a path prefix. This is useful when:
+
+- You want to host multiple services on the same domain
+- You're using a reverse proxy that routes based on path prefixes
+- You need to integrate Anubis with an existing application structure
+
+For example, if you set `BASE_PREFIX=/myapp`, Anubis will:
+
+- Serve its challenge page at `/myapp/` instead of `/`
+- Serve its API endpoints at `/myapp/.within.website/x/cmd/anubis/api/` instead of `/.within.website/x/cmd/anubis/api/`
+- Serve its static assets at `/myapp/.within.website/x/cmd/anubis/` instead of `/.within.website/x/cmd/anubis/`
+
+When using this feature with a reverse proxy:
+
+1. Configure your reverse proxy to route requests for the specified path prefix to Anubis
+2. Set the `BASE_PREFIX` environment variable to match the path prefix in your reverse proxy configuration
+3. Ensure that your reverse proxy preserves the path when forwarding requests to Anubis
+
+Example with Nginx:
+
+```nginx
+location /myapp/ {
+    proxy_pass http://anubis:8923/myapp;
+    proxy_set_header Host $host;
+    proxy_set_header X-Real-IP $remote_addr;
+}
 ```
 
-## Kubernetes
+With corresponding Anubis configuration:
 
-This example makes the following assumptions:
-
-- Your target service is listening on TCP port `5000`.
-- Anubis will be listening on port `8080`.
-
-Attach Anubis to your Deployment:
-
-```yaml
-containers:
-  # ...
-  - name: anubis
-    image: ghcr.io/techarohq/anubis:latest
-    imagePullPolicy: Always
-    env:
-      - name: "BIND"
-        value: ":8080"
-      - name: "DIFFICULTY"
-        value: "5"
-      - name: "METRICS_BIND"
-        value: ":9090"
-      - name: "SERVE_ROBOTS_TXT"
-        value: "true"
-      - name: "TARGET"
-        value: "http://localhost:5000"
-    resources:
-      limits:
-        cpu: 500m
-        memory: 128Mi
-      requests:
-        cpu: 250m
-        memory: 128Mi
-    securityContext:
-      runAsUser: 1000
-      runAsGroup: 1000
-      runAsNonRoot: true
-      allowPrivilegeEscalation: false
-      capabilities:
-        drop:
-          - ALL
-      seccompProfile:
-        type: RuntimeDefault
+```
+BASE_PREFIX=/myapp
 ```
 
-Then add a Service entry for Anubis:
+### Key generation
 
-```yaml
-# ...
-spec:
-  ports:
-    # diff-add
-    - protocol: TCP
-      # diff-add
-      port: 8080
-      # diff-add
-      targetPort: 8080
-      # diff-add
-      name: anubis
+To generate an ed25519 private key, you can use this command:
+
+```text
+openssl rand -hex 32
 ```
 
-Then point your Ingress to the Anubis port:
+Alternatively here is a key generated by your browser:
 
-```yaml
-  rules:
-  - host: git.xeserv.us
-    http:
-      paths:
-      - pathType: Prefix
-        path: "/"
-        backend:
-          service:
-            name: git
-            port:
-              # diff-remove
-              name: http
-              # diff-add
-              name: anubis
-```
+<RandomKey />
+
+## Next steps
+
+To get Anubis filtering your traffic, you need to make sure it's added to your HTTP load balancer or platform configuration. See the [environments category](/docs/category/environments) for detailed information on individual environments.
+
+- [Apache](./environments/apache.mdx)
+- [Docker compose](./environments/docker-compose.mdx)
+- [Kubernetes](./environments/kubernetes.mdx)
+- [Nginx](./environments/nginx.mdx)
+- [Traefik](./environments/traefik.mdx)
+
+:::note
+
+Anubis loads its assets from `/.within.website/x/xess/` and `/.within.website/x/cmd/anubis`. If you do not reverse proxy these in your server config, Anubis won't work.
+
+:::

docs/docs/admin/native-install.mdx

@@ -0,0 +1,138 @@
+---
+title: Installing Anubis with a native package
+---
+
+import Tabs from "@theme/Tabs";
+import TabItem from "@theme/TabItem";
+
+Download the package for your system from [the most recent release on GitHub](https://github.com/TecharoHQ/anubis/releases).
+
+Install the Anubis package using your package manager of choice:
+
+<Tabs>
+  <TabItem value="deb" label="Debian-based (apt)" default>
+  
+Install Anubis with `apt`:
+
+```text
+sudo apt install ./anubis-$VERSION-$ARCH.deb
+```
+
+  </TabItem>
+  <TabItem value="tarball" label="Tarball">
+  
+Extract the tarball to a folder:
+
+```text
+tar zxf ./anubis-$VERSION-$OS-$ARCH.tar.gz
+cd anubis-$VERSION-$OS-$ARCH
+```
+
+Install the binary to your system:
+
+```text
+sudo install -D ./bin/anubis /usr/local/bin
+```
+
+Edit the systemd unit to point to `/usr/local/bin/anubis` instead of `/usr/bin/anubis`:
+
+```text
+perl -pi -e 's$/usr/bin/anubis$/usr/local/bin/anubis$g' ./run/anubis@.service
+```
+
+Install the systemd unit to your system:
+
+```text
+sudo install -D ./run/anubis@.service /etc/systemd/system
+```
+
+Install the default configuration file to your system:
+
+```text
+sudo install -D ./run/default.env /etc/anubis
+```
+
+  </TabItem>
+  <TabItem value="rpm" label="Red Hat-based (rpm)">
+  
+Install Anubis with `dnf`:
+
+```text
+sudo dnf -y install ./anubis-$VERSION.$ARCH.rpm
+```
+
+OR
+
+Install Anubis with `yum`:
+
+```text
+sudo yum -y install ./anubis-$VERSION.$ARCH.rpm
+```
+
+OR
+
+Install Anubis with `rpm`:
+
+```
+sudo rpm -ivh ./anubis-$VERSION.$ARCH.rpm
+```
+
+  </TabItem>
+</Tabs>
+
+Once it's installed, make a copy of the default configuration file `/etc/anubis/default.env` based on which service you want to protect. For example, to protect a `gitea` server:
+
+```text
+sudo cp /etc/anubis/default.env /etc/anubis/gitea.env
+```
+
+Copy the default bot policies file to `/etc/anubis/gitea.botPolicies.yaml`:
+
+<Tabs>
+<TabItem value="debrpm" label="Debian or Red Hat" default>
+
+```text
+sudo cp /usr/share/doc/anubis/botPolicies.yaml /etc/anubis/gitea.botPolicies.yaml
+```
+
+</TabItem>
+<TabItem value="tarball" label="Tarball">
+
+```text
+sudo cp ./doc/botPolicies.yaml /etc/anubis/gitea.botPolicies.yaml
+```
+
+</TabItem>
+
+</Tabs>
+
+Then open `gitea.env` in your favorite text editor and customize [the environment variables](./installation.mdx#environment-variables) as needed. Here's an example configuration for a Gitea server:
+
+```sh
+BIND=[::1]:8239
+BIND_NETWORK=tcp
+DIFFICULTY=4
+METRICS_BIND=[::1]:8240
+METRICS_BIND_NETWORK=tcp
+POLICY_FNAME=/etc/anubis/gitea.botPolicies.yaml
+TARGET=http://localhost:3000
+```
+
+Then start Anubis with `systemctl enable --now`:
+
+```text
+sudo systemctl enable --now anubis@gitea.service
+```
+
+Test to make sure it's running with `curl`:
+
+```text
+curl http://localhost:8240/metrics
+```
+
+Then set up your reverse proxy (Nginx, Caddy, etc.) to point to the Anubis port. Anubis will then reverse proxy all requests that meet the policies in `/etc/anubis/gitea.botPolicies.json` to the target service.
+
+For more details on particular reverse proxies, see here:
+
+- [Apache](./environments/apache.mdx)
+- [Nginx](./environments/nginx.mdx)

docs/docs/admin/policies.mdx

@@ -2,15 +2,25 @@
 title: Policy Definitions
 ---
 
+import Tabs from "@theme/Tabs";
+import TabItem from "@theme/TabItem";
+
 Out of the box, Anubis is pretty heavy-handed. It will aggressively challenge everything that might be a browser (usually indicated by having `Mozilla` in its user agent). However, some bots are smart enough to get past the challenge. Some things that look like bots may actually be fine (IE: RSS readers). Some resources need to be visible no matter what. Some resources and remotes are fine to begin with.
 
 Bot policies let you customize the rules that Anubis uses to allow, deny, or challenge incoming requests. Currently you can set policies by the following matches:
 
 - Request path
 - User agent string
+- HTTP request header values
+- [Importing other configuration snippets](./configuration/import.mdx)
+
+As of version v1.17.0 or later, configuration can be written in either JSON or YAML.
 
 Here's an example rule that denies [Amazonbot](https://developer.amazon.com/en/amazonbot):
 
+<Tabs>
+<TabItem value="json" label="JSON" default>
+
 ```json
 {
   "name": "amazonbot",
@@ -19,15 +29,37 @@ Here's an example rule that denies [Amazonbot](https://developer.amazon.com/en/a
 }
 ```
 
+</TabItem>
+<TabItem value="yaml" label="YAML">
+
+```yaml
+- name: amazonbot
+  user_agent_regex: Amazonbot
+  action: DENY
+```
+
+</TabItem>
+</Tabs>
+
 When this rule is evaluated, Anubis will check the `User-Agent` string of the request. If it contains `Amazonbot`, Anubis will send an error page to the user saying that access is denied, but in such a way that makes scrapers think they have correctly loaded the webpage.
 
 Right now the only kinds of policies you can write are bot policies. Other forms of policies will be added in the future.
 
 Here is a minimal policy file that will protect against most scraper bots:
 
+<Tabs>
+<TabItem value="json" label="JSON" default>
+
 ```json
 {
   "bots": [
+    {
+      "name": "cloudflare-workers",
+      "headers_regex": {
+        "CF-Worker": ".*"
+      },
+      "action": "DENY"
+    },
     {
       "name": "well-known",
       "path_regex": "^/.well-known/.*$",
@@ -52,9 +84,35 @@ Here is a minimal policy file that will protect against most scraper bots:
 }
 ```
 
-This allows requests to [`/.well-known`](https://en.wikipedia.org/wiki/Well-known_URI), `/favicon.ico`, `/robots.txt`, and challenges any request that has the word `Mozilla` in its User-Agent string. The [default policy file](https://github.com/TecharoHQ/anubis/blob/main/cmd/anubis/botPolicies.json) is a bit more cohesive, but this should be more than enough for most users.
+</TabItem>
+<TabItem value="yaml" label="YAML">
 
-If no rules match the request, it is allowed through.
+```yaml
+bots:
+  - name: cloudflare-workers
+    headers_regex:
+      CF-Worker: .*
+    action: DENY
+  - name: well-known
+    path_regex: ^/.well-known/.*$
+    action: ALLOW
+  - name: favicon
+    path_regex: ^/favicon.ico$
+    action: ALLOW
+  - name: robots-txt
+    path_regex: ^/robots.txt$
+    action: ALLOW
+  - name: generic-browser
+    user_agent_regex: Mozilla
+    action: CHALLENGE
+```
+
+</TabItem>
+</Tabs>
+
+This allows requests to [`/.well-known`](https://en.wikipedia.org/wiki/Well-known_URI), `/favicon.ico`, `/robots.txt`, and challenges any request that has the word `Mozilla` in its User-Agent string. The [default policy file](https://github.com/TecharoHQ/anubis/blob/main/data/botPolicies.json) is a bit more cohesive, but this should be more than enough for most users.
+
+If no rules match the request, it is allowed through. For more details on this default behavior and its implications, see [Default allow behavior](./default-allow-behavior.mdx).
 
 ## Writing your own rules
 
@@ -72,6 +130,11 @@ Name your rules in lower case using kebab-case. Rule names will be exposed in Pr
 
 Rules can also have their own challenge settings. These are customized using the `"challenge"` key. For example, here is a rule that makes challenges artificially hard for connections with the substring "bot" in their user agent:
 
+<Tabs>
+<TabItem value="json" label="JSON" default>
+
+This rule has been known to have a high false positive rate in testing. Please use this with care.
+
 ```json
 {
   "name": "generic-bot-catchall",
@@ -85,6 +148,25 @@ Rules can also have their own challenge settings. These are customized using the
 }
 ```
 
+</TabItem>
+<TabItem value="yaml" label="YAML">
+
+This rule has been known to have a high false positive rate in testing. Please use this with care.
+
+```yaml
+# Punish any bot with "bot" in the user-agent string
+- name: generic-bot-catchall
+  user_agent_regex: (?i:bot|crawler)
+  action: CHALLENGE
+  challenge:
+    difficulty: 16 # impossible
+    report_as: 4 # lie to the operator
+    algorithm: slow # intentionally waste CPU cycles and time
+```
+
+</TabItem>
+</Tabs>
+
 Challenges can be configured with these settings:
 
 | Key          | Example  | Description                                                                                                                                                                                    |
@@ -99,6 +181,9 @@ The `remote_addresses` field of a Bot rule allows you to set the IP range that t
 
 For example, you can allow a search engine to connect if and only if its IP address matches the ones they published:
 
+<Tabs>
+<TabItem value="json" label="JSON" default>
+
 ```json
 {
   "name": "qwantbot",
@@ -108,8 +193,25 @@ For example, you can allow a search engine to connect if and only if its IP addr
 }
 ```
 
+</TabItem>
+<TabItem value="yaml" label="YAML">
+
+```yaml
+- name: qwantbot
+  user_agent_regex: \+https\://help\.qwant\.com/bot/
+  action: ALLOW
+  # https://help.qwant.com/wp-content/uploads/sites/2/2025/01/qwantbot.json
+  remote_addresses: ["91.242.162.0/24"]
+```
+
+</TabItem>
+</Tabs>
+
 This also works at an IP range level without any other checks:
 
+<Tabs>
+<TabItem value="json" label="JSON" default>
+
 ```json
 {
   "name": "internal-network",
@@ -118,6 +220,19 @@ This also works at an IP range level without any other checks:
 }
 ```
 
+</TabItem>
+<TabItem value="yaml" label="YAML">
+
+```yaml
+name: internal-network
+action: ALLOW
+remote_addresses:
+  - 100.64.0.0/10
+```
+
+</TabItem>
+</Tabs>
+
 ## Risk calculation for downstream services
 
 In case your service needs it for risk calculation reasons, Anubis exposes information about the rules that any requests match using a few headers:
@@ -126,6 +241,6 @@ In case your service needs it for risk calculation reasons, Anubis exposes infor
 | :---------------- | :--------------------------------------------------- | :--------------- |
 | `X-Anubis-Rule`   | The name of the rule that was matched                | `bot/lightpanda` |
 | `X-Anubis-Action` | The action that Anubis took in response to that rule | `CHALLENGE`      |
-| `X-Anubis-Status` | The status and how strict Anubis was in its checks   | `PASS-FULL`      |
+| `X-Anubis-Status` | The status and how strict Anubis was in its checks   | `PASS`           |
 
 Policy rules are matched using [Go's standard library regular expressions package](https://pkg.go.dev/regexp). You can mess around with the syntax at [regex101.com](https://regex101.com), make sure to select the Golang option.

docs/docs/developer/_category_.json

@@ -0,0 +1,8 @@
+{
+  "label": "Developer guides",
+  "position": 50,
+  "link": {
+    "type": "generated-index",
+    "description": "Guides and suggestions to make Anubis development go smoothly for everyone."
+  }
+}

docs/docs/developer/building-anubis.md

@@ -0,0 +1,84 @@
+---
+title: Building Anubis without Docker
+---
+
+:::note
+
+These instructions may work, but for right now they are informative for downstream packagers more than they are ready-made instructions for administrators wanting to run Anubis on their servers. Pre-made binary package support is being tracked in [#156](https://github.com/TecharoHQ/anubis/issues/156).
+
+:::
+
+## Entirely from source
+
+If you are doing a build entirely from source, here's what you need to do:
+
+:::info
+
+If you maintain a package for Anubis v1.15.x or older, you will need to update your package build. You may want to use one of the half-baked tarballs if your distro/environment of choice makes it difficult to use npm.
+
+:::
+
+### Tools needed
+
+In order to build a production-ready binary of Anubis, you need the following packages in your environment:
+
+- [Go](https://go.dev) at least version 1.24 - the programming language that Anubis is written in
+- [esbuild](https://esbuild.github.io/) - the JavaScript bundler Anubis uses for its production JS assets
+- [Node.JS & NPM](https://nodejs.org/en) - manages some build dependencies
+- `gzip` - compresses production JS (part of coreutils)
+- `zstd` - compresses production JS
+- `brotli` - compresses production JS
+
+To upgrade your version of Go without system package manager support, install `golang.org/dl/go1.24.2` (this can be done from any version of Go):
+
+```text
+go install golang.org/dl/go1.24.2@latest
+go1.24.2 download
+```
+
+### Install dependencies
+
+```text
+make deps
+```
+
+This will download Go and NPM dependencies.
+
+### Building static assets
+
+```text
+make assets
+```
+
+This will build all static assets (CSS, JavaScript) for distribution.
+
+### Building Anubis to the `./var` folder
+
+```text
+make build
+```
+
+From this point it is up to you to make sure that `./var/anubis` ends up in the right place. You may want to consult the `./run` folder for useful files such as a systemd unit and `anubis.env.default` file.
+
+## "Pre-baked" tarball
+
+The `anubis-src-with-vendor` tarball has many pre-build steps already done, including:
+
+- Go module dependencies are present in `./vendor`
+- Static assets (JS, CSS, etc.) are already built in CI
+
+This means you do not have to manage Go, NPM, or other ecosystem dependencies.
+
+When using this tarball, all you need to do is build `./cmd/anubis`:
+
+```text
+make prebaked-build
+```
+
+Anubis will be built to `./var/anubis`.
+
+## Development dependencies
+
+Optionally, you can install the following dependencies for development:
+
+- [Staticcheck](https://staticcheck.dev/docs/getting-started/) (optional, not required due to [`go tool staticcheck`](https://www.alexedwards.net/blog/how-to-manage-tool-dependencies-in-go-1.24-plus), but required if you are using any version of Go older than 1.24)

docs/docs/developer/code-quality.md

@@ -0,0 +1,31 @@
+---
+title: Code quality guidelines
+---
+
+When submitting code to Anubis, please take the time to consider the fact that this project is security software. If things go bad, bots can pummel sites into oblivion. This is not ideal for uptime.
+
+As such, code reviews will be a bit more strict than you have seen in other projects. This is not people trying to be mean, this is a side effect of taking the problem seriously.
+
+When making code changes, try to do the following:
+
+- If you're submitting a bugfix, add a test case for it
+- If you're changing the JavaScript, make sure the integration tests pass (`npm run test:integration`)
+
+## Commit messages
+
+Anubis follows the Go project's conventions for commit messages. In general, an ideal commit message should read like this:
+
+```text
+path/to/folder: brief description of the change
+
+If the change is subtle, has implementation consequences, or is otherwise
+not entirely self-describing: take the time to spell out why. If things
+are very subtle, please also amend the documentation accordingly
+```
+
+The subject of a commit message should be the second half of the sentence "This commit changes the Anubis project to:". Here's a few examples:
+
+- `disable DroneBL by default`
+- `port the challenge to WebAssembly`
+
+The extended commit message is also your place to give rationale for a new feature. When maintainers are reviewing your code, they will use this to figure out if the burden from feature maintainership is worth the merge.

docs/docs/developer/local-dev.md

@@ -0,0 +1,86 @@
+---
+title: Local development
+---
+
+:::note
+
+TL;DR: `npm ci && npm run dev`
+
+:::
+
+Anubis requires the following tools to be installed to do local development:
+
+- [Go](https://go.dev) - the programming language that Anubis is written in
+- [esbuild](https://esbuild.github.io/) - the JavaScript bundler Anubis uses for its production JS assets
+- [Node.JS & NPM](https://nodejs.org/en) - manages some build dependencies
+- `gzip` - compresses production JS (part of coreutils)
+- `zstd` - compresses production JS
+- `brotli` - compresses production JS
+
+If you have [Homebrew](https://brew.sh) installed, you can install all the dependencies with one command:
+
+```text
+brew bundle
+```
+
+If you don't, you may need to figure out equivalents to the packages in Homebrew.
+
+## Running Anubis locally
+
+```text
+npm run dev
+```
+
+Or to do it manually:
+
+- Run `npm run assets` every time you change the CSS/JavaScript
+- `go run ./cmd/anubis` with any CLI flags you want
+
+## Building JS/CSS assets
+
+```text
+npm run assets
+```
+
+If you change the build process, make sure to update `build.sh` accordingly.
+
+## Production-ready builds
+
+```text
+npm run container
+```
+
+This builds a prod-ready container image with [ko](https://ko.build). If you want to change where the container image is pushed, you need to use environment variables:
+
+```text
+DOCKER_REPO=registry.host/org/repo DOCKER_METADATA_OUTPUT_TAGS=registry.host/org/repo:latest npm run container
+```
+
+## Building packages
+
+For more information, see [Building native packages is complicated](https://xeiaso.net/blog/2025/anubis-packaging/) and [#156: Debian, RPM, and binary tarball packages](https://github.com/TecharoHQ/anubis/issues/156).
+
+Install `yeet`:
+
+:::note
+
+`yeet` will soon be moved to a dedicated TecharoHQ repository. This is currently done in a hacky way in order to get this ready for user feedback.
+
+:::
+
+```text
+go install within.website/x/cmd/yeet@v1.13.4
+```
+
+Install the dependencies for Anubis:
+
+```text
+npm ci
+go mod download
+```
+
+Build the packages into `./var`:
+
+```text
+yeet
+```

docs/docs/developer/signed-commits.md

@@ -0,0 +1,7 @@
+---
+title: Signed commits
+---
+
+Anubis requires developers to sign their commits. This is done so that we can have a better chain of custody from contribution to owner. For more information about commit signing, [read here](https://www.freecodecamp.org/news/what-is-commit-signing-in-git/).
+
+We do not require GPG. SSH signed commits are fine. For an overview on how to set up commit signing with your SSH key, [read here](https://dev.to/ccoveille/git-the-complete-guide-to-sign-your-commits-with-an-ssh-key-35bg).

docs/docs/funding.md

@@ -7,4 +7,4 @@ Anubis is provided to the public for free in order to help advance the common go
 
 If you want to run an unbranded or white-label version of Anubis, please [contact Xe](https://xeiaso.net/contact) to arrange a contract. This is not meant to be "contact us" pricing, I am still evaluating the market for this solution and figuring out what makes sense.
 
-You can donate to the project [on Patreon](https://patreon.com/cadey).
+You can donate to the project [on Patreon](https://patreon.com/cadey) or via [GitHub Sponsors](https://github.com/sponsors/Xe).

docs/docs/index.mdx

@@ -15,14 +15,55 @@ title: Anubis
 ![language count](https://img.shields.io/github/languages/count/TecharoHQ/anubis)
 ![repo size](https://img.shields.io/github/repo-size/TecharoHQ/anubis)
 
-Anubis [weighs the soul of your connection](https://en.wikipedia.org/wiki/Weighing_of_souls) using a sha256 proof-of-work challenge in order to protect upstream resources from scraper bots.
+## Sponsors
+
+Anubis is brought to you by sponsors and donors like:
+
+[![Distrust](/img/sponsors/distrust-logo.webp)](https://distrust.co)
+
+## Overview
+
+Anubis [weighs the soul of your connection](https://en.wikipedia.org/wiki/Weighing_of_souls) using a proof-of-work challenge in order to protect upstream resources from scraper bots.
 
 This program is designed to help protect the small internet from the endless storm of requests that flood in from AI companies. Anubis is as lightweight as possible to ensure that everyone can afford to protect the communities closest to them.
 
-Anubis is a bit of a nuclear response. This will result in your website being blocked from smaller scrapers and may inhibit "good bots" like the Internet Archive. You can configure [bot policy definitions](./admin/policies.md) to explicitly allowlist them and we are working on a curated set of "known good" bots to allow for a compromise between discoverability and uptime.
+Anubis is a bit of a nuclear response. This will result in your website being blocked from smaller scrapers and may inhibit "good bots" like the Internet Archive. You can configure [bot policy definitions](https://anubis.techaro.lol/docs/admin/policies) to explicitly allowlist them and we are working on a curated set of "known good" bots to allow for a compromise between discoverability and uptime.
+
+In most cases, you should not need this and can probably get by using Cloudflare to protect a given origin. However, for circumstances where you can't or won't use Cloudflare, Anubis is there for you.
 
 ## Support
 
-If you run into any issues running Anubis, please [open an issue](https://github.com/TecharoHQ/anubis/issues/new?template=Blank+issue) and tag it with the Anubis tag. Please include all the information I would need to diagnose your issue.
+If you run into any issues running Anubis, please [open an issue](https://github.com/TecharoHQ/anubis/issues/new?template=Blank+issue) and include all the information I would need to diagnose your issue.
 
-For live chat, please join the [Patreon](https://patreon.com/cadey) and ask in the Patron discord in the channel `#anubis`.
+For live chat, please join the [Patreon](https://patreon.com/cadey) or join [GitHub Sponsors](https://github.com/sponsors/Xe) and ask in the Patron discord in the channel `#anubis`.
+
+## Star History
+
+<a href="https://www.star-history.com/#TecharoHQ/anubis&Date">
+  <picture>
+    <source
+      media="(prefers-color-scheme: dark)"
+      srcSet="https://api.star-history.com/svg?repos=TecharoHQ/anubis&type=Date&theme=dark"
+    />
+    <source
+      media="(prefers-color-scheme: light)"
+      srcSet="https://api.star-history.com/svg?repos=TecharoHQ/anubis&type=Date"
+    />
+    <img
+      alt="Star History Chart"
+      src="https://api.star-history.com/svg?repos=TecharoHQ/anubis&type=Date"
+    />
+  </picture>
+</a>
+
+## Packaging Status
+
+[![Packaging status](https://repology.org/badge/vertical-allrepos/anubis-anti-crawler.svg?columns=3)](https://repology.org/project/anubis-anti-crawler/versions)
+
+## Contributors
+
+<a href="https://github.com/TecharoHQ/anubis/graphs/contributors">
+  <img src="https://contrib.rocks/image?repo=TecharoHQ/anubis" />
+</a>
+
+Made with [contrib.rocks](https://contrib.rocks).

docs/docs/user/known-instances.md

@@ -0,0 +1,37 @@
+---
+title: List of known websites using Anubis
+---
+
+This page contains a non-exhaustive list with all websites using Anubis.
+
+- <details>
+  <summary>The Linux Foundation</summary>
+  - https://git.kernel.org/
+  - https://lore.kernel.org/
+  </details>
+- https://gitlab.gnome.org/
+- https://scioly.org/
+- https://bugs.winehq.org/
+- https://svnweb.freebsd.org/
+- https://trac.ffmpeg.org/
+- https://git.sr.ht/
+- https://xeiaso.net/
+- https://source.puri.sm/
+- https://git.enlightenment.org/
+- https://superlove.sayitditto.net/
+- https://linktaco.com/
+- https://jaredallard.dev/
+- https://dev.sanctum.geek.nz/
+- https://canine.tools/
+- https://git.lupancham.net/
+- https://dev.haiku-os.org
+- http://code.hackerspace.pl/
+- https://wiki.archlinux.org/
+- https://git.devuan.org/
+- https://hydra.nixos.org/
+- https://hydra.nixos.org/
+- https://codeberg.org/
+- <details>
+  <summary>The United Nations</summary>
+  - https://policytoolbox.iiep.unesco.org/
+  </details>

docs/docusaurus.config.ts

@@ -45,7 +45,7 @@ const config: Config = {
           // Please change this to your repo.
           // Remove this to remove the "edit this page" links.
           editUrl:
-            'https://github.com/facebook/docusaurus/tree/main/packages/create-docusaurus/templates/shared/',
+            'https://github.com/TecharoHQ/anubis/tree/main/docs/',
         },
         // blog: {
         //   showReadingTime: true,
@@ -70,13 +70,16 @@ const config: Config = {
   ],
 
   themeConfig: {
+    colorMode: {
+      respectPrefersColorScheme: true,
+    },
     // Replace with your project's social card
     image: 'img/docusaurus-social-card.jpg',
     navbar: {
       title: 'Anubis',
       logo: {
         alt: 'A happy jackal woman with brown hair and red eyes',
-        src: 'img/happy.webp',
+        src: 'img/favicon.webp',
       },
       items: [
         {
@@ -125,10 +128,6 @@ const config: Config = {
         {
           title: 'More',
           items: [
-            {
-              label: 'Blog',
-              to: '/blog',
-            },
             {
               label: 'GitHub',
               href: 'https://github.com/TecharoHQ/anubis',

docs/manifest/deployment.yaml

@@ -22,7 +22,7 @@ spec:
         ports:
         - containerPort: 80
       - name: anubis
-        image: ghcr.io/techarohq/anubis:latest
+        image: ghcr.io/techarohq/anubis:main
         imagePullPolicy: Always
         env:
           - name: "BIND"

docs/package-lock.json

@@ -8512,9 +8512,9 @@
       }
     },
     "node_modules/estree-util-value-to-estree": {
-      "version": "3.3.2",
-      "resolved": "https://registry.npmjs.org/estree-util-value-to-estree/-/estree-util-value-to-estree-3.3.2.tgz",
-      "integrity": "sha512-hYH1aSvQI63Cvq3T3loaem6LW4u72F187zW4FHpTrReJSm6W66vYTFNO1vH/chmcOulp1HlAj1pxn8Ag0oXI5Q==",
+      "version": "3.3.3",
+      "resolved": "https://registry.npmjs.org/estree-util-value-to-estree/-/estree-util-value-to-estree-3.3.3.tgz",
+      "integrity": "sha512-Db+m1WSD4+mUO7UgMeKkAwdbfNWwIxLt48XF2oFU9emPfXkIu+k5/nlOj313v7wqtAPo0f9REhUvznFrPkG8CQ==",
       "license": "MIT",
       "dependencies": {
         "@types/estree": "^1.0.0"
@@ -10093,9 +10093,9 @@
       }
     },
     "node_modules/http-proxy-middleware": {
-      "version": "2.0.7",
-      "resolved": "https://registry.npmjs.org/http-proxy-middleware/-/http-proxy-middleware-2.0.7.tgz",
-      "integrity": "sha512-fgVY8AV7qU7z/MmXJ/rxwbrtQH4jBQ9m7kp3llF0liB7glmFeVZFBepQb32T3y8n8k2+AEYuMPCpinYW+/CuRA==",
+      "version": "2.0.9",
+      "resolved": "https://registry.npmjs.org/http-proxy-middleware/-/http-proxy-middleware-2.0.9.tgz",
+      "integrity": "sha512-c1IyJYLYppU574+YI7R4QyX2ystMtVXZwIdzazUIPIJsHuWNd+mho2j+bKoHftndicGj9yh+xjd+l0yj7VeT1Q==",
       "license": "MIT",
       "dependencies": {
         "@types/http-proxy": "^1.17.8",
@@ -10184,9 +10184,9 @@
       }
     },
     "node_modules/image-size": {
-      "version": "1.2.0",
-      "resolved": "https://registry.npmjs.org/image-size/-/image-size-1.2.0.tgz",
-      "integrity": "sha512-4S8fwbO6w3GeCVN6OPtA9I5IGKkcDMPcKndtUlpJuCwu7JLjtj7JZpwqLuyY2nrmQT3AWsCJLSKPsc2mPBSl3w==",
+      "version": "1.2.1",
+      "resolved": "https://registry.npmjs.org/image-size/-/image-size-1.2.1.tgz",
+      "integrity": "sha512-rH+46sQJ2dlwfjfhCyNx5thzrv+dtmBIhPHk0zgRUukHzZ/kRueTJXoYYsclBaKcSMBWuGbOFXtioLpzTb5euw==",
       "license": "MIT",
       "dependencies": {
         "queue": "6.0.2"

docs/src/components/RandomKey/index.tsx

@@ -0,0 +1,42 @@
+import { useState, useCallback } from "react";
+import Code from "@theme/CodeInline";
+import BrowserOnly from "@docusaurus/BrowserOnly";
+
+// https://www.xaymar.com/articles/2020/12/08/fastest-uint8array-to-hex-string-conversion-in-javascript/
+function toHex(buffer) {
+  return Array.prototype.map
+    .call(buffer, (x) => ("00" + x.toString(16)).slice(-2))
+    .join("");
+}
+
+export const genRandomKey = (): String => {
+  const array = new Uint8Array(32);
+  self.crypto.getRandomValues(array);
+  return toHex(array);
+};
+
+export default function RandomKey() {
+  return (
+    <BrowserOnly fallback={<div>Loading...</div>}>
+      {() => {
+        const [key, setKey] = useState<String>(genRandomKey());
+        const genRandomKeyCb = useCallback(() => {
+          setKey(genRandomKey());
+        });
+        return (
+          <span>
+            <Code>{key}</Code>
+            <span style={{ marginLeft: "0.25rem", marginRight: "0.25rem" }} />
+            <button
+              onClick={() => {
+                genRandomKeyCb();
+              }}
+            >
+              ♻️
+            </button>
+          </span>
+        );
+      }}
+    </BrowserOnly>
+  );
+}

docs/src/css/custom.css

@@ -6,31 +6,36 @@
 
 /* You can override the default Infima variables here. */
 :root {
-  --ifm-color-primary: #2e8555;
-  --ifm-color-primary-dark: #29784c;
-  --ifm-color-primary-darker: #277148;
-  --ifm-color-primary-darkest: #205d3b;
-  --ifm-color-primary-light: #33925d;
-  --ifm-color-primary-lighter: #359962;
-  --ifm-color-primary-lightest: #3cad6e;
+  --ifm-color-primary: #ff5630;
+  --ifm-color-primary-dark: #ad422a;
+  --ifm-color-primary-darker: #8f3521;
+  --ifm-color-primary-darkest: #592115;
+  --ifm-color-primary-light: #ff7152;
+  --ifm-color-primary-lighter: #ff9178;
+  --ifm-color-primary-lightest: #ffb09e;
   --ifm-code-font-size: 95%;
   --docusaurus-highlighted-code-line-bg: rgba(0, 0, 0, 0.1);
+  --code-block-diff-add-line-color: #ccffd8;
+  --code-block-diff-remove-line-color: #ffebe9;
 }
 
 /* For readability concerns, you should choose a lighter palette in dark mode. */
 [data-theme="dark"] {
-  --ifm-color-primary: #25c2a0;
-  --ifm-color-primary-dark: #21af90;
-  --ifm-color-primary-darker: #1fa588;
-  --ifm-color-primary-darkest: #1a8870;
-  --ifm-color-primary-light: #29d5b0;
-  --ifm-color-primary-lighter: #32d8b4;
-  --ifm-color-primary-lightest: #4fddbf;
-  --docusaurus-highlighted-code-line-bg: rgba(0, 0, 0, 0.3);
+  --ifm-color-primary: #e64a19;
+  --ifm-color-primary-dark: #b73a12;
+  --ifm-color-primary-darker: #8c2c0e;
+  --ifm-color-primary-darkest: #5a1e0a;
+  --ifm-color-primary-light: #eb6d45;
+  --ifm-color-primary-lighter: #f09178;
+  --ifm-color-primary-lightest: #f5b5a6;
+  --ifm-code-font-size: 95%;
+  --docusaurus-highlighted-code-line-bg: rgba(0, 0, 0, 0.25);
+  --code-block-diff-add-line-color: #2d5a2c;
+  --code-block-diff-remove-line-color: #5a2d2c;
 }
 
 .code-block-diff-add-line {
-  background-color: #ccffd8;
+  background-color: var(--code-block-diff-add-line-color);
   display: block;
   margin: 0 -40px;
   padding: 0 40px;
@@ -44,7 +49,7 @@
 }
 
 .code-block-diff-remove-line {
-  background-color: #ffebe9;
+  background-color: var(--code-block-diff-remove-line-color);
   display: block;
   margin: 0 -40px;
   padding: 0 40px;