Compare commits

..

1 Commits

Author SHA1 Message Date
Xe Iaso f19a5f7eb8 docs(k8s): document that Kubernetes support needs a non-default storage backend
Closes: #1602
Signed-off-by: Xe Iaso <me@xeiaso.net>
2026-06-01 10:29:23 -04:00
24 changed files with 66 additions and 93 deletions
-3
View File
@@ -44,6 +44,3 @@ xou
AWOO
firewalls
bindhosts
handrolled
xai
gitlab
+1 -1
View File
@@ -22,7 +22,7 @@ jobs:
sudo apt-get update
sudo apt-get install -y build-essential
- uses: actions/setup-node@48b55a011bda9f5d6aeb4c2d9c7362e8dae4041e # v6.4.0
- uses: actions/setup-node@53b83947a5a98c8d113130e565377fae1a50d02f # v6.3.0
with:
node-version: "24.11.0"
- uses: actions/setup-go@4a3601121dd01d1626a1e23e37211e3254c1c06c # v6.4.0
+2 -2
View File
@@ -26,7 +26,7 @@ jobs:
sudo apt-get update
sudo apt-get install -y build-essential
- uses: actions/setup-node@48b55a011bda9f5d6aeb4c2d9c7362e8dae4041e # v6.4.0
- uses: actions/setup-node@53b83947a5a98c8d113130e565377fae1a50d02f # v6.3.0
with:
node-version: "24.11.0"
- uses: actions/setup-go@4a3601121dd01d1626a1e23e37211e3254c1c06c # v6.4.0
@@ -37,7 +37,7 @@ jobs:
- name: Docker meta
id: meta
uses: docker/metadata-action@80c7e94dd9b9319bd5eb7a0e0fe9291e23a2a2e9 # v6.1.0
uses: docker/metadata-action@030e881283bb7a6894de51c315a6bfe6a94e05cf # v6.0.0
with:
images: ghcr.io/${{ github.repository }}
+3 -3
View File
@@ -36,7 +36,7 @@ jobs:
run: |
echo "IMAGE=ghcr.io/${GITHUB_REPOSITORY,,}" >> $GITHUB_ENV
- uses: actions/setup-node@48b55a011bda9f5d6aeb4c2d9c7362e8dae4041e # v6.4.0
- uses: actions/setup-node@53b83947a5a98c8d113130e565377fae1a50d02f # v6.3.0
with:
node-version: "24.11.0"
- uses: actions/setup-go@4a3601121dd01d1626a1e23e37211e3254c1c06c # v6.4.0
@@ -46,7 +46,7 @@ jobs:
- uses: ko-build/setup-ko@d006021bd0c28d1ce33a07e7943d48b079944c8d # v0.9
- name: Log into registry
uses: docker/login-action@650006c6eb7dba73a995cc03b0b2d7f5ca915bee # v4.2.0
uses: docker/login-action@4907a6ddec9925e35a0a9e82d7399ccc52663121 # v4.1.0
with:
registry: ghcr.io
username: ${{ github.repository_owner }}
@@ -54,7 +54,7 @@ jobs:
- name: Docker meta
id: meta
uses: docker/metadata-action@80c7e94dd9b9319bd5eb7a0e0fe9291e23a2a2e9 # v6.1.0
uses: docker/metadata-action@030e881283bb7a6894de51c315a6bfe6a94e05cf # v6.0.0
with:
images: ${{ env.IMAGE }}
+6 -6
View File
@@ -22,10 +22,10 @@ jobs:
persist-credentials: false
- name: Set up Docker Buildx
uses: docker/setup-buildx-action@d7f5e7f509e45cec5c76c4d5afdd7de93d0b3df5 # v4.1.0
uses: docker/setup-buildx-action@4d04d5d9486b7bd6fa91e7baf45bbb4f8b9deedd # v4.0.0
- name: Log into registry
uses: docker/login-action@650006c6eb7dba73a995cc03b0b2d7f5ca915bee # v4.2.0
uses: docker/login-action@4907a6ddec9925e35a0a9e82d7399ccc52663121 # v4.1.0
with:
registry: ghcr.io
username: techarohq
@@ -33,7 +33,7 @@ jobs:
- name: Docker meta
id: meta
uses: docker/metadata-action@80c7e94dd9b9319bd5eb7a0e0fe9291e23a2a2e9 # v6.1.0
uses: docker/metadata-action@030e881283bb7a6894de51c315a6bfe6a94e05cf # v6.0.0
with:
images: ghcr.io/techarohq/anubis/docs
tags: |
@@ -42,7 +42,7 @@ jobs:
- name: Build and push
id: build
uses: docker/build-push-action@f9f3042f7e2789586610d6e8b85c8f03e5195baf # v7.2.0
uses: docker/build-push-action@d08e5c354a6adb9ed34480a06d141179aa583294 # v7.0.0
with:
context: ./docs
cache-to: type=gha
@@ -53,14 +53,14 @@ jobs:
push: true
- name: Apply k8s manifests to limsa lominsa
uses: actions-hub/kubectl@af0e87aaa9cd98820383c16cba28c644ba85c067 # v1.36.1
uses: actions-hub/kubectl@934aaa4354bbbc3d2176ae8d7cae92d515032dff # v1.35.3
env:
KUBE_CONFIG: ${{ secrets.LIMSA_LOMINSA_KUBECONFIG }}
with:
args: apply -k docs/manifest
- name: Apply k8s manifests to limsa lominsa
uses: actions-hub/kubectl@af0e87aaa9cd98820383c16cba28c644ba85c067 # v1.36.1
uses: actions-hub/kubectl@934aaa4354bbbc3d2176ae8d7cae92d515032dff # v1.35.3
env:
KUBE_CONFIG: ${{ secrets.LIMSA_LOMINSA_KUBECONFIG }}
with:
+3 -3
View File
@@ -18,11 +18,11 @@ jobs:
persist-credentials: false
- name: Set up Docker Buildx
uses: docker/setup-buildx-action@d7f5e7f509e45cec5c76c4d5afdd7de93d0b3df5 # v4.1.0
uses: docker/setup-buildx-action@4d04d5d9486b7bd6fa91e7baf45bbb4f8b9deedd # v4.0.0
- name: Docker meta
id: meta
uses: docker/metadata-action@80c7e94dd9b9319bd5eb7a0e0fe9291e23a2a2e9 # v6.1.0
uses: docker/metadata-action@030e881283bb7a6894de51c315a6bfe6a94e05cf # v6.0.0
with:
images: ghcr.io/techarohq/anubis/docs
tags: |
@@ -31,7 +31,7 @@ jobs:
- name: Build and push
id: build
uses: docker/build-push-action@f9f3042f7e2789586610d6e8b85c8f03e5195baf # v7.2.0
uses: docker/build-push-action@d08e5c354a6adb9ed34480a06d141179aa583294 # v7.0.0
with:
context: ./docs
cache-to: type=gha
+2 -2
View File
@@ -24,7 +24,7 @@ jobs:
sudo apt-get update
sudo apt-get install -y build-essential
- uses: actions/setup-node@48b55a011bda9f5d6aeb4c2d9c7362e8dae4041e # v6.4.0
- uses: actions/setup-node@53b83947a5a98c8d113130e565377fae1a50d02f # v6.3.0
with:
node-version: "24.11.0"
- uses: actions/setup-go@4a3601121dd01d1626a1e23e37211e3254c1c06c # v6.4.0
@@ -32,7 +32,7 @@ jobs:
go-version: "stable"
- name: Cache playwright binaries
uses: actions/cache@27d5ce7f107fe9357f9df03efb73ab90386fccae # v5.0.5
uses: actions/cache@668228422ae6a00e4ad889ee87cd7109ec5666a7 # v5.0.4
id: playwright-cache
with:
path: |
+1 -1
View File
@@ -25,7 +25,7 @@ jobs:
sudo apt-get update
sudo apt-get install -y build-essential
- uses: actions/setup-node@48b55a011bda9f5d6aeb4c2d9c7362e8dae4041e # v6.4.0
- uses: actions/setup-node@53b83947a5a98c8d113130e565377fae1a50d02f # v6.3.0
with:
node-version: "24.11.0"
- uses: actions/setup-go@4a3601121dd01d1626a1e23e37211e3254c1c06c # v6.4.0
@@ -26,7 +26,7 @@ jobs:
sudo apt-get update
sudo apt-get install -y build-essential
- uses: actions/setup-node@48b55a011bda9f5d6aeb4c2d9c7362e8dae4041e # v6.4.0
- uses: actions/setup-node@53b83947a5a98c8d113130e565377fae1a50d02f # v6.3.0
with:
node-version: "24.11.0"
- uses: actions/setup-go@4a3601121dd01d1626a1e23e37211e3254c1c06c # v6.4.0
@@ -41,7 +41,7 @@ jobs:
run: |
go tool yeet
- uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
- uses: actions/upload-artifact@bbbca2ddaa5d8feaa63e36b76fdaad77386f024f # v7.0.0
with:
name: packages
path: var/*
+2 -2
View File
@@ -35,7 +35,7 @@ jobs:
with:
persist-credentials: false
- uses: actions/setup-node@48b55a011bda9f5d6aeb4c2d9c7362e8dae4041e # v6.4.0
- uses: actions/setup-node@53b83947a5a98c8d113130e565377fae1a50d02f # v6.3.0
with:
node-version: "24.11.0"
- uses: actions/setup-go@4a3601121dd01d1626a1e23e37211e3254c1c06c # v6.4.0
@@ -58,7 +58,7 @@ jobs:
run: echo "ARTIFACT_NAME=${{ matrix.test }}" | sed 's|/|-|g' >> $GITHUB_ENV
- name: Upload artifact
uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a
uses: actions/upload-artifact@bbbca2ddaa5d8feaa63e36b76fdaad77386f024f
if: always()
with:
name: ${{ env.ARTIFACT_NAME }}
+2 -2
View File
@@ -24,13 +24,13 @@ jobs:
fetch-depth: 0
persist-credentials: false
- name: Log into registry
uses: docker/login-action@650006c6eb7dba73a995cc03b0b2d7f5ca915bee # v4.2.0
uses: docker/login-action@4907a6ddec9925e35a0a9e82d7399ccc52663121 # v4.1.0
with:
registry: ghcr.io
username: ${{ github.repository_owner }}
password: ${{ secrets.GITHUB_TOKEN }}
- name: Set up Docker Buildx
uses: docker/setup-buildx-action@d7f5e7f509e45cec5c76c4d5afdd7de93d0b3df5 # v4.1.0
uses: docker/setup-buildx-action@4d04d5d9486b7bd6fa91e7baf45bbb4f8b9deedd # v4.0.0
- name: Build and push
run: |
cd ./test/ssh-ci
+2 -2
View File
@@ -21,7 +21,7 @@ jobs:
persist-credentials: false
- name: Install the latest version of uv
uses: astral-sh/setup-uv@08807647e7069bb48b6ef5acd8ec9567f424441b # v8.1.0
uses: astral-sh/setup-uv@cec208311dfd045dd5311c1add060b2062131d57 # v8.0.0
- name: Run zizmor 🌈
run: uvx zizmor --format sarif . > results.sarif
@@ -29,7 +29,7 @@ jobs:
GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
- name: Upload SARIF file
uses: github/codeql-action/upload-sarif@7211b7c8077ea37d8641b6271f6a365a22a5fbfa # v4.36.0
uses: github/codeql-action/upload-sarif@c10b8064de6f491fea524254123dbe5e09572f13 # v4.35.1
with:
sarif_file: results.sarif
category: zizmor
+1
View File
@@ -10,3 +10,4 @@ builds:
ldflags:
- -s -w
- -extldflags "-static"
- -X github.com/TecharoHQ/anubis.Version={{.Env.VERSION}}
+3 -18
View File
@@ -1,27 +1,12 @@
// Package anubis contains the version number of Anubis.
package anubis
import (
"runtime/debug"
"time"
)
func init() {
bi, ok := debug.ReadBuildInfo()
if !ok {
return
}
// XXX(Xe): many things in this repo assume that the development version
// of anubis is `devel` and ReadBuildInfo returns `(devel)`. Shim the gap.
if bi.Main.Version != "(devel)" {
Version = bi.Main.Version
}
}
import "time"
// Version is the current version of Anubis.
//
// This is set from the Go module runtime version.
// This variable is set at build time using the -X linker flag. If not set,
// it defaults to "devel".
var Version = "devel"
// CookieName is the name of the cookie that Anubis uses in order to validate
-4
View File
@@ -36,7 +36,6 @@ import (
"github.com/TecharoHQ/anubis/lib/thoth"
"github.com/TecharoHQ/anubis/web"
"github.com/facebookgo/flagenv"
"github.com/google/uuid"
_ "github.com/joho/godotenv/autoload"
healthv1 "google.golang.org/grpc/health/grpc_health_v1"
)
@@ -194,9 +193,6 @@ func main() {
flagenv.Parse()
flag.Parse()
// Must be set before any concurrent UUID call.
uuid.EnableRandPool()
if *versionFlag {
fmt.Println("Anubis", anubis.Version)
return
-3
View File
@@ -41,9 +41,6 @@ bots:
# Challenge Firefox AI previews
- import: (data)/clients/x-firefox-ai.yaml
# x.ai has a scraper that is killing gitlab instances
- import: (data)/crawlers/xai.yaml
# Allow common "keeping the internet working" routes (well-known, favicon, robots.txt)
- import: (data)/common/keep-internet-working.yaml
-8
View File
@@ -1,8 +0,0 @@
- name: xai-crawler-and-asn
action: DENY
user_agent_regex: code-review-sourcing.*\+xai-research
remote_addresses:
- 69.12.56.0/21
- name: xai-crawler-user-agent
action: DENY
user_agent_regex: code-review-sourcing.*\+xai-research
-3
View File
@@ -25,9 +25,6 @@
# Challenge Firefox AI previews
- import: (data)/clients/x-firefox-ai.yaml
# x.ai has a scraper that is killing gitlab instances
- import: (data)/crawlers/xai.yaml
# Allow common "keeping the internet working" routes (well-known, favicon, robots.txt)
- import: (data)/common/keep-internet-working.yaml
-4
View File
@@ -23,14 +23,12 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
- Improve error messages and fix broken REDIRECT_DOMAINS link in docs ([#1193](https://github.com/TecharoHQ/anubis/issues/1193))
- Add Bulgarian locale ([#1394](https://github.com/TecharoHQ/anubis/pull/1394))
- Fixed case-sensitivity mismatch in geoipchecker.go
- Use [Go's native version stamping](https://michael.stapelberg.ch/posts/2026-04-05-stamp-it-all-programs-must-report-their-version/) instead of a handrolled variant.
- Fix CEL internal errors when iterating `headers`/`query` map wrappers by implementing map iterators for `HTTPHeaders` and `URLValues` ([#1465](https://github.com/TecharoHQ/anubis/pull/1465)).
- Enable [metrics serving via TLS](./admin/policies.mdx#tls), including [mutual TLS (mTLS)](./admin/policies.mdx#mtls).
- Enable [HTTP basic auth](./admin/policies.mdx#http-basic-authentication) for the metrics server.
- Fix a bug in the dataset poisoning maze that could allow denial of service [#1580](https://github.com/TecharoHQ/anubis/issues/1580).
- Add config option to add ASN to logs/metrics.
- Log weight when issuing challenge.
- Block x.ai's crawler for code review training.
- Gate pprof endpoints behind `metrics.debug` in the policy file.
- Limit naive honeypot r9k delay to one second.
- Fix an obscure case where adding query values to a subrequest match could cause an invalid rule match when using path based matching for protected resources.
@@ -41,9 +39,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
- Fix a race in the bbolt store where the asynchronous cleanup scheduled by an expired read could delete a value that had just been refreshed; the delete now only fires when the key still carries the same expired generation it observed.
- Marginally increase the performances of requests processing
- Marginally improve the performances of PoW validation
- Marginally improve the performances of challenges generation/display
- Significantly improve the performances of the gzip middleware
- Significantly improve the performances of the PoW validation
## v1.25.0: Necron
@@ -131,11 +131,27 @@ Then point your Ingress to the Anubis port:
name: anubis
```
## Storage
By default, Anubis stores all of its data in memory. This memory is not shared between pods. If you have multiple instances of Anubis without the data being [stored outside of memory](../policies.mdx#storage-backends) and a [shared cookie key](../installation.mdx#key-generation), you will run into [unexpected behaviour](https://github.com/TecharoHQ/anubis/issues/1602) when user traffic traverses between pods.
Based on the deployment of your Kubernetes cluster, here are the preferable storage backends to pick from:
| Backend | Pro | Con |
| :------- | :-------------------------------------------------------------- | :------------------------------------------------------------------------------------------- |
| `bbolt` | Only requires a ReadWriteOnce PVC. | Does not support more than one Anubis pod. |
| `memory` | Requires no configuration. | Process memory is not shared between pods. |
| `s3api` | Great if your cluster includes Rook/Ceph to use RADOS directly. | Potentially higher latency unless you use a store like [Tigris](https://www.tigrisdata.com). |
| `valkey` | Trivial to configure in your cluster. | If your Redis/Valkey server is down, Anubis is going to have issues. |
Pick your poison accordingly. Many production deployments use the `s3api` and `valkey` backends without issue. Single node deployments can get away with either `memory` or `bbolt` depending on the facts and circumstances of the deployment.
## Envoy Gateway
If you are using envoy-gateway, the `X-Real-Ip` header is not set by default, but Anubis does require it. You can resolve this by adding the header, either on the specific `HTTPRoute` where Anubis is listening, or on the `ClientTrafficPolicy` to apply it to any number of Gateways:
HTTPRoute:
```yaml
apiVersion: gateway.networking.k8s.io/v1
kind: HTTPRoute
@@ -160,6 +176,7 @@ spec:
```
Applying to any number of Gateways:
```yaml
apiVersion: gateway.envoyproxy.io/v1alpha1
kind: ClientTrafficPolicy
+7
View File
@@ -219,8 +219,11 @@ Anubis offers the following storage backends:
- [`memory`](#memory) -- A simple in-memory hashmap
- [`bbolt`](#bbolt) -- An on-disk key/value store backed by [bbolt](https://github.com/etcd-io/bbolt), an embedded key/value database for Go programs
- [`s3api`](#s3api) -- Amazon S3 based storage or another compatible object store
- [`valkey`](#valkey) -- A remote in-memory key/value database backed by [Valkey](https://valkey.io/) (or another database compatible with the [RESP](https://redis.io/docs/latest/develop/reference/protocol-spec/) protocol)
:::warning
If no storage backend is set in the policy file, Anubis will use the [`memory`](#memory) backend by default. This is equivalent to the following in the policy file:
```yaml
@@ -229,6 +232,10 @@ store:
parameters: {}
```
This means that all session data that is required for the challenge mechanism to work is stored **IN PROCESS MEMORY** that is **NOT** shared between instances of Anubis. If you set up Anubis with multiple instances using the `memory` storage backend, your users will sometimes get "Administrator has misconfigured Anubis" error messages when it cannot look up the aforementioned session data.
:::
### `memory`
The memory backend is an in-memory cache. This backend works best if you don't use multiple instances of Anubis or don't have mutable storage in the environment you're running Anubis in.
+5 -7
View File
@@ -4,7 +4,6 @@ import (
"context"
"crypto/ed25519"
"crypto/rand"
"encoding/hex"
"encoding/json"
"errors"
"fmt"
@@ -163,7 +162,6 @@ func (s *Server) issueChallenge(ctx context.Context, r *http.Request, lg *slog.L
if err != nil {
return nil, err
}
idStr := id.String()
var randomData = make([]byte, 64)
if _, err := rand.Read(randomData); err != nil {
@@ -171,9 +169,9 @@ func (s *Server) issueChallenge(ctx context.Context, r *http.Request, lg *slog.L
}
chall := challenge.Challenge{
ID: idStr,
ID: id.String(),
Method: rule.Challenge.Algorithm,
RandomData: hex.EncodeToString(randomData),
RandomData: fmt.Sprintf("%x", randomData),
IssuedAt: time.Now(),
Difficulty: rule.Challenge.Difficulty,
PolicyRuleHash: rule.Hash(),
@@ -184,11 +182,11 @@ func (s *Server) issueChallenge(ctx context.Context, r *http.Request, lg *slog.L
}
j := store.JSON[challenge.Challenge]{Underlying: s.store}
if err := j.Set(ctx, "challenge:"+idStr, chall, 30*time.Minute); err != nil {
if err := j.Set(ctx, "challenge:"+id.String(), chall, 30*time.Minute); err != nil {
return nil, err
}
lg.Info("new challenge issued", "challenge", idStr, "weight", cr.Weight)
lg.Info("new challenge issued", "challenge", id.String(), "weight", cr.Weight)
return &chall, err
}
@@ -242,7 +240,7 @@ func (s *Server) maybeReverseProxyOrPage(w http.ResponseWriter, r *http.Request)
func (s *Server) maybeReverseProxy(w http.ResponseWriter, r *http.Request, httpStatusOnly bool) {
lg, r := s.getRequestLogger(r)
if val, _ := s.store.Get(r.Context(), "ogtags:allow:"+r.Host+r.URL.String()); val != nil {
if val, _ := s.store.Get(r.Context(), fmt.Sprintf("ogtags:allow:%s%s", r.Host, r.URL.String())); val != nil {
lg.Debug("serving opengraph tag asset")
s.ServeHTTPNext(w, r)
return
+5 -15
View File
@@ -1,15 +1,14 @@
package proofofwork
import (
"crypto/sha256"
"crypto/subtle"
"encoding/hex"
"fmt"
"log/slog"
"net/http"
"strconv"
"strings"
"github.com/TecharoHQ/anubis/internal"
chall "github.com/TecharoHQ/anubis/lib/challenge"
"github.com/TecharoHQ/anubis/lib/localization"
"github.com/a-h/templ"
@@ -67,20 +66,11 @@ func (i *Impl) Validate(r *http.Request, lg *slog.Logger, in *chall.ValidateInpu
return chall.NewError("validate", "invalid response", fmt.Errorf("%w response", chall.ErrMissingField))
}
// Stream the challenge and nonce into a single sha256 hasher to avoid
// the intermediate "challenge + nonceStr" concatenation. Hex-encode
// the digest into a stack buffer so the comparison runs without
// allocating a heap string.
h := sha256.New()
h.Write([]byte(challenge))
h.Write([]byte(nonceStr))
var sumBuf [sha256.Size]byte
sum := h.Sum(sumBuf[:0])
var hexBuf [sha256.Size * 2]byte
hex.Encode(hexBuf[:], sum)
calcString := challenge + nonceStr
calculated := internal.SHA256sum(calcString)
if subtle.ConstantTimeCompare([]byte(response), hexBuf[:]) != 1 {
return chall.NewError("validate", "invalid response", fmt.Errorf("%w: wanted response %s but got %s", chall.ErrFailed, string(hexBuf[:]), response))
if subtle.ConstantTimeCompare([]byte(response), []byte(calculated)) != 1 {
return chall.NewError("validate", "invalid response", fmt.Errorf("%w: wanted response %s but got %s", chall.ErrFailed, calculated, response))
}
// compare the leading zeroes
+2 -2
View File
@@ -17,8 +17,8 @@ $`npm run assets`;
},
build: ({ bin, etc, systemd, doc }) => {
$`go build -o ${bin}/anubis -ldflags '-s -w -extldflags "-static" ./cmd/anubis`;
$`go build -o ${bin}/anubis-robots2policy -ldflags '-s -w -extldflags "-static"' ./cmd/robots2policy`;
$`go build -o ${bin}/anubis -ldflags '-s -w -extldflags "-static" -X "github.com/TecharoHQ/anubis.Version=${git.tag()}"' ./cmd/anubis`;
$`go build -o ${bin}/anubis-robots2policy -ldflags '-s -w -extldflags "-static" -X "github.com/TecharoHQ/anubis.Version=${git.tag()}"' ./cmd/robots2policy`;
file.install("./run/anubis@.service", `${systemd}/anubis@.service`);
file.install("./run/default.env", `${etc}/default.env`);