chore: spelling

Signed-off-by: Xe Iaso <me@xeiaso.net>

.github/actions/spelling/allow.txt

@@ -37,3 +37,4 @@ unipromos
 Samsung
 wenet
 qwertiko
+setuplistener

.github/actions/spelling/expect.txt

@@ -47,6 +47,7 @@ cachediptoasn
 Caddyfile
 caninetools
 Cardyb
+CAs
 celchecker
 celphase
 cerr
@@ -203,8 +204,10 @@ kagi
 kagibot
 Keyfunc
 keypair
+keypairreloader
 KHTML
 kinda
+kpr
 KUBECONFIG
 lcj
 ldflags
@@ -229,6 +232,7 @@ metarefresh
 metrix
 mimi
 Minfilia
+minica
 mistralai
 mnt
 Mojeek
@@ -313,6 +317,7 @@ searchbot
 searx
 sebest
 secretplans
+selfsigned
 Semrush
 Seo
 setsebool

cmd/anubis/main.go

@@ -17,12 +17,10 @@ import (
 	"net"
 	"net/http"
 	"net/http/httputil"
-	"net/http/pprof"
 	"net/url"
 	"os"
 	"os/signal"
 	"path/filepath"
-	"strconv"
 	"strings"
 	"sync"
 	"syscall"
@@ -33,12 +31,12 @@ import (
 	"github.com/TecharoHQ/anubis/internal"
 	libanubis "github.com/TecharoHQ/anubis/lib"
 	"github.com/TecharoHQ/anubis/lib/config"
+	"github.com/TecharoHQ/anubis/lib/metrics"
 	botPolicy "github.com/TecharoHQ/anubis/lib/policy"
 	"github.com/TecharoHQ/anubis/lib/thoth"
 	"github.com/TecharoHQ/anubis/web"
 	"github.com/facebookgo/flagenv"
 	_ "github.com/joho/godotenv/autoload"
-	"github.com/prometheus/client_golang/prometheus/promhttp"
 	healthv1 "google.golang.org/grpc/health/grpc_health_v1"
 )
 
@@ -119,33 +117,6 @@ func doHealthCheck() error {
 	return nil
 }
 
-// parseBindNetFromAddr determine bind network and address based on the given network and address.
-func parseBindNetFromAddr(address string) (string, string) {
-	defaultScheme := "http://"
-	if !strings.Contains(address, "://") {
-		if strings.HasPrefix(address, ":") {
-			address = defaultScheme + "localhost" + address
-		} else {
-			address = defaultScheme + address
-		}
-	}
-
-	bindUri, err := url.Parse(address)
-	if err != nil {
-		log.Fatal(fmt.Errorf("failed to parse bind URL: %w", err))
-	}
-
-	switch bindUri.Scheme {
-	case "unix":
-		return "unix", bindUri.Path
-	case "tcp", "http", "https":
-		return "tcp", bindUri.Host
-	default:
-		log.Fatal(fmt.Errorf("unsupported network scheme %s in address %s", bindUri.Scheme, address))
-	}
-	return "", address
-}
-
 func parseSameSite(s string) http.SameSite {
 	switch strings.ToLower(s) {
 	case "none":
@@ -162,53 +133,6 @@ func parseSameSite(s string) http.SameSite {
 	return http.SameSiteDefaultMode
 }
 
-func setupListener(network string, address string) (net.Listener, string) {
-	formattedAddress := ""
-
-	if network == "" {
-		// keep compatibility
-		network, address = parseBindNetFromAddr(address)
-	}
-
-	switch network {
-	case "unix":
-		formattedAddress = "unix:" + address
-	case "tcp":
-		if strings.HasPrefix(address, ":") { // assume it's just a port e.g. :4259
-			formattedAddress = "http://localhost" + address
-		} else {
-			formattedAddress = "http://" + address
-		}
-	default:
-		formattedAddress = fmt.Sprintf(`(%s) %s`, network, address)
-	}
-
-	listener, err := net.Listen(network, address)
-	if err != nil {
-		log.Fatal(fmt.Errorf("failed to bind to %s: %w", formattedAddress, err))
-	}
-
-	// additional permission handling for unix sockets
-	if network == "unix" {
-		mode, err := strconv.ParseUint(*socketMode, 8, 0)
-		if err != nil {
-			listener.Close()
-			log.Fatal(fmt.Errorf("could not parse socket mode %s: %w", *socketMode, err))
-		}
-
-		err = os.Chmod(address, os.FileMode(mode))
-		if err != nil {
-			err := listener.Close()
-			if err != nil {
-				log.Printf("failed to close listener: %v", err)
-			}
-			log.Fatal(fmt.Errorf("could not change socket mode: %w", err))
-		}
-	}
-
-	return listener, formattedAddress
-}
-
 func makeReverseProxy(target string, targetSNI string, targetHost string, insecureSkipVerify bool, targetDisableKeepAlive bool) (http.Handler, error) {
 	targetUri, err := url.Parse(target)
 	if err != nil {
@@ -304,11 +228,6 @@ func main() {
 
 	wg := new(sync.WaitGroup)
 
-	if *metricsBind != "" {
-		wg.Add(1)
-		go metricsServer(ctx, *lg.With("subsystem", "metrics"), wg.Done)
-	}
-
 	var rp http.Handler
 	// when using anubis via Systemd and environment variables, then it is not possible to set targe to an empty string but only to space
 	if strings.TrimSpace(*target) != "" {
@@ -348,6 +267,26 @@ func main() {
 	lg.Debug("swapped to new logger")
 	slog.SetDefault(lg)
 
+	if *metricsBind != "" || policy.Metrics != nil {
+		wg.Add(1)
+
+		ms := &metrics.Server{
+			Config: policy.Metrics,
+			Log:    lg,
+		}
+
+		if policy.Metrics == nil {
+			lg.Debug("migrating flags to metrics config", "bind", *metricsBind, "network", *metricsBindNetwork, "socket-mode", *socketMode)
+			ms.Config = &config.Metrics{
+				Bind:       *metricsBind,
+				Network:    *metricsBindNetwork,
+				SocketMode: *socketMode,
+			}
+		}
+
+		go ms.Run(ctx, wg.Done)
+	}
+
 	// Warn if persistent storage is used without a configured signing key
 	if policy.Store.IsPersistent() {
 		if *hs512Secret == "" && *ed25519PrivateKeyHex == "" && *ed25519PrivateKeyHexFile == "" {
@@ -484,7 +423,11 @@ func main() {
 	h = internal.JA4H(h)
 
 	srv := http.Server{Handler: h, ErrorLog: internal.GetFilteredHTTPLogger()}
-	listener, listenerUrl := setupListener(*bindNetwork, *bind)
+	listener, listenerUrl, err := internal.SetupListener(*bindNetwork, *bind, *socketMode)
+	if err != nil {
+		log.Fatalf("SetupListener(%q, %q, %q): %v", *bindNetwork, *bind, *socketMode, err)
+	}
+
 	lg.Info(
 		"listening",
 		"url", listenerUrl,
@@ -519,53 +462,6 @@ func main() {
 	wg.Wait()
 }
 
-func metricsServer(ctx context.Context, lg slog.Logger, done func()) {
-	defer done()
-
-	mux := http.NewServeMux()
-	mux.HandleFunc("GET /debug/pprof/", pprof.Index)
-	mux.HandleFunc("GET /debug/pprof/cmdline", pprof.Cmdline)
-	mux.HandleFunc("GET /debug/pprof/profile", pprof.Profile)
-	mux.HandleFunc("GET /debug/pprof/symbol", pprof.Symbol)
-	mux.HandleFunc("GET /debug/pprof/trace", pprof.Trace)
-	mux.Handle("/metrics", promhttp.Handler())
-	mux.HandleFunc("/healthz", func(w http.ResponseWriter, r *http.Request) {
-		st, ok := internal.GetHealth("anubis")
-		if !ok {
-			slog.Error("health service anubis does not exist, file a bug")
-		}
-
-		switch st {
-		case healthv1.HealthCheckResponse_NOT_SERVING:
-			http.Error(w, "NOT OK", http.StatusInternalServerError)
-			return
-		case healthv1.HealthCheckResponse_SERVING:
-			fmt.Fprintln(w, "OK")
-			return
-		default:
-			http.Error(w, "UNKNOWN", http.StatusFailedDependency)
-			return
-		}
-	})
-
-	srv := http.Server{Handler: mux, ErrorLog: internal.GetFilteredHTTPLogger()}
-	listener, metricsUrl := setupListener(*metricsBindNetwork, *metricsBind)
-	lg.Debug("listening for metrics", "url", metricsUrl)
-
-	go func() {
-		<-ctx.Done()
-		c, cancel := context.WithTimeout(context.Background(), 5*time.Second)
-		defer cancel()
-		if err := srv.Shutdown(c); err != nil {
-			log.Printf("cannot shut down: %v", err)
-		}
-	}()
-
-	if err := srv.Serve(listener); !errors.Is(err, http.ErrServerClosed) {
-		log.Fatal(err)
-	}
-}
-
 func extractEmbedFS(fsys embed.FS, root string, destDir string) error {
 	return fs.WalkDir(fsys, root, func(path string, d fs.DirEntry, err error) error {
 		if err != nil {

data/botPolicies.yaml

@@ -166,6 +166,29 @@ status_codes:
   CHALLENGE: 200
   DENY: 200
 
+# # Configuration for the metrics server. See the docs for more information:
+# #
+# # https://anubis.techaro.lol/docs/admin/policies#metrics-server
+# #
+# # This is commented out by default so that command line flags take precedence.
+# metrics:
+#   bind: ":9090"
+#   network: "tcp"
+#
+# # To serve metrics over TLS, set the path to the right TLS certificate and key
+# # here. When the files change on disk, they will automatically be reloaded.
+# #
+# # https://anubis.techaro.lol/docs/admin/policies#tls
+#   tls:
+#     certificate: /path/to/tls.crt
+#     key: /path/to/tls.key
+#
+#     # If you want to secure your metrics endpoint using mutual TLS (mTLS), set
+#     # the path to a certificate authority public certificate here.
+#     #
+#     # https://anubis.techaro.lol/docs/admin/policies#mtls
+#     ca: /path/to/ca.crt
+
 # Anubis can store temporary data in one of a few backends. See the storage
 # backends section of the docs for more information:
 #

data/crawlers/ai-search.yaml

@@ -4,5 +4,5 @@
 #  - Claude-SearchBot: No published IP allowlist
 - name: "ai-crawlers-search"
   user_agent_regex: >-
-    OAI-SearchBot|Claude-SearchBot|PerplexityBot
+    OAI-SearchBot|Claude-SearchBot|PerplexityBot|meta-webindexer
   action: DENY

docs/docs/CHANGELOG.md

@@ -11,15 +11,17 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 
 ## [Unreleased]
 
+<!-- This changes the project to: -->
+
+- Move metrics server configuration to [the policy file](./admin/policies.mdx#metrics-server).
 - Expose [pprof endpoints](https://pkg.go.dev/net/http/pprof) on the metrics listener to enable profiling Anubis in production.
 - fix: prevent nil pointer panic in challenge validation when threshold rules match during PassChallenge (#1463)
 - Instruct reverse proxies to not cache error pages.
 - Fixed mixed tab/space indentation in Caddy documentation code block
 - Improve error messages and fix broken REDIRECT_DOMAINS link in docs ([#1193](https://github.com/TecharoHQ/anubis/issues/1193))
 - Add Bulgarian locale ([#1394](https://github.com/TecharoHQ/anubis/pull/1394))
-
-<!-- This changes the project to: -->
 - Fix CEL internal errors when iterating `headers`/`query` map wrappers by implementing map iterators for `HTTPHeaders` and `URLValues` ([#1465](https://github.com/TecharoHQ/anubis/pull/1465)).
+- Enable [metrics serving via TLS](./admin/policies.mdx#tls), including [mutual TLS (mTLS)](./admin/policies.mdx#mtls).
 
 ## v1.25.0: Necron
 

docs/docs/admin/installation.mdx

@@ -87,15 +87,15 @@ Anubis uses these environment variables for configuration:
 | `ED25519_PRIVATE_KEY_HEX_FILE` | unset                   | Path to a file containing the hex-encoded ed25519 private key. Only one of this or its sister option may be set. **Required when using persistent storage backends** (like bbolt) to ensure challenges survive service restarts. When running multiple instances on the same base domain, the key must be the same across all instances.                                                                                                                                                                                                       |
 | `ERROR_TITLE`                  | unset                   | <EO /> If set, override the translation stack to show a custom title for error pages such as "Something went wrong!". See [Customizing messages](./botstopper.mdx#customizing-messages) for more details.                                                                                                                                                                                                                                                                                                                                      |
 | `JWT_RESTRICTION_HEADER`       | `X-Real-IP`             | If set, the JWT is only valid if the current value of this header matches the value when the JWT was created. You can use it e.g. to restrict a JWT to the source IP of the user using `X-Real-IP`.                                                                                                                                                                                                                                                                                                                                            |
-| `METRICS_BIND`                 | `:9090`                 | The network address that Anubis serves Prometheus metrics on. See `BIND` for more information.                                                                                                                                                                                                                                                                                                                                                                                                                                                 |
-| `METRICS_BIND_NETWORK`         | `tcp`                   | The address family that the Anubis metrics server listens on. See `BIND_NETWORK` for more information.                                                                                                                                                                                                                                                                                                                                                                                                                                         |
+| `METRICS_BIND`                 | `:9090`                 | The legacy configuration value for the network address that Anubis serves Prometheus metrics on. Please migrate this to [the policy file](./policies.mdx#metrics-server) as soon as possible.                                                                                                                                                                                                                                                                                                                                                  |
+| `METRICS_BIND_NETWORK`         | `tcp`                   | The legacy configuration value for the address family that Anubis serves Prometheus metrics on. Please migrate this to [the policy file](./policies.mdx#metrics-server) as soon as possible.                                                                                                                                                                                                                                                                                                                                                   |
 | `OG_EXPIRY_TIME`               | `24h`                   | The expiration time for the Open Graph tag cache. Prefer using [the policy file](./configuration/open-graph.mdx) to configure the Open Graph subsystem.                                                                                                                                                                                                                                                                                                                                                                                        |
 | `OG_PASSTHROUGH`               | `false`                 | If set to `true`, Anubis will enable Open Graph tag passthrough. Prefer using [the policy file](./configuration/open-graph.mdx) to configure the Open Graph subsystem.                                                                                                                                                                                                                                                                                                                                                                         |
 | `OG_CACHE_CONSIDER_HOST`       | `false`                 | If set to `true`, Anubis will consider the host in the Open Graph tag cache key. Prefer using [the policy file](./configuration/open-graph.mdx) to configure the Open Graph subsystem.                                                                                                                                                                                                                                                                                                                                                         |
 | `OVERLAY_FOLDER`               | unset                   | <EO /> If set, treat the given path as an [overlay folder](./botstopper.mdx#custom-images-and-css), allowing you to customize CSS, fonts, images, and add other assets to BotStopper deployments.                                                                                                                                                                                                                                                                                                                                              |
 | `POLICY_FNAME`                 | unset                   | The file containing [bot policy configuration](./policies.mdx). See the bot policy documentation for more details. If unset, the default bot policy configuration is used.                                                                                                                                                                                                                                                                                                                                                                     |
 | `PUBLIC_URL`                   | unset                   | The externally accessible URL for this Anubis instance, used for constructing redirect URLs (e.g., for Traefik forwardAuth). Leave it unset when Anubis terminates traffic directly (sidecar/standalone deployments) or redirect building will fail with `redir=null`.                                                                                                                                                                                                                                                                         |
-| `REDIRECT_DOMAINS`             | unset                   | Comma-separated list of domain names that Anubis should allow redirects to when passing a challenge. See [Redirect Domain Configuration](./configuration/redirect-domains.mdx) for more details.                                                                                                                                                                                                                                                                                                                                                   |
+| `REDIRECT_DOMAINS`             | unset                   | Comma-separated list of domain names that Anubis should allow redirects to when passing a challenge. See [Redirect Domain Configuration](./configuration/redirect-domains.mdx) for more details.                                                                                                                                                                                                                                                                                                                                               |
 | `SERVE_ROBOTS_TXT`             | `false`                 | If set `true`, Anubis will serve a default `robots.txt` file that disallows all known AI scrapers by name and then additionally disallows every scraper. This is useful if facts and circumstances make it difficult to change the underlying service to serve such a `robots.txt` file.                                                                                                                                                                                                                                                       |
 | `SLOG_LEVEL`                   | `INFO`                  | The log level for structured logging. Valid values are `DEBUG`, `INFO`, `WARN`, and `ERROR`. Set to `DEBUG` to see all requests, evaluations, and detailed diagnostic information.                                                                                                                                                                                                                                                                                                                                                             |
 | `SOCKET_MODE`                  | `0770`                  | _Only used when at least one of the `*_BIND_NETWORK` variables are set to `unix`._ The socket mode (permissions) for Unix domain sockets.                                                                                                                                                                                                                                                                                                                                                                                                      |

docs/docs/admin/policies.mdx

@@ -117,6 +117,60 @@ remote_addresses:
   - 100.64.0.0/10
 ```
 
+## Metrics server
+
+Anubis includes support for [Prometheus-style metrics](https://prometheus.io/docs/introduction/overview/), allowing systems administrators to monitor Anubis' performance and effectiveness. This is a separate HTTP server with metrics, health checking, and debug routes.
+
+Anubis' metrics server is configured with the `metrics` block in the configuration file:
+
+```yaml
+metrics:
+  bind: ":9090"
+  network: "tcp"
+```
+
+If you want to bind metrics to a Unix socket, make sure to set the network to `unix` and add a socket mode:
+
+```yaml
+metrics:
+  bind: "/tmp/anubis_metrics.sock"
+  network: unix
+  socketMode: "0700" # must be a string
+```
+
+### TLS
+
+If you want to serve the metrics server over TLS, use the `tls` block:
+
+```yaml
+metrics:
+  bind: ":9090"
+  network: "tcp"
+
+  tls:
+    certificate: /path/to/tls.crt
+    key: /path/to/tls.key
+```
+
+The certificate and key will automatically be reloaded when the respective files change.
+
+### mTLS
+
+If you want to validate requests to ensure that they use a client certificate signed by a certificate authority (mutual TLS or mTLS), set the `ca` value in the `tls` block:
+
+```yaml
+metrics:
+  bind: ":9090"
+  network: "tcp"
+
+  tls:
+    certificate: /path/to/tls.crt
+    key: /path/to/tls.key
+    ca: /path/to/ca.crt
+```
+
+As it is not expected for certificate authority certificates to change often, the CA certificate will NOT be automatically reloaded when the respective file changes.
+
 ## Imprint / Impressum support
 
 Anubis has support for showing imprint / impressum information. This is defined in the `impressum` block of your configuration. See [Imprint / Impressum configuration](./configuration/impressum.mdx) for more information.

internal/setuplistener.go

@@ -0,0 +1,92 @@
+package internal
+
+import (
+	"errors"
+	"fmt"
+	"net"
+	"net/url"
+	"os"
+	"strconv"
+	"strings"
+)
+
+// parseBindNetFromAddr determine bind network and address based on the given network and address.
+func parseBindNetFromAddr(address string) (string, string, error) {
+	defaultScheme := "http://"
+	if !strings.Contains(address, "://") {
+		if strings.HasPrefix(address, ":") {
+			address = defaultScheme + "localhost" + address
+		} else {
+			address = defaultScheme + address
+		}
+	}
+
+	bindUri, err := url.Parse(address)
+	if err != nil {
+		return "", "", fmt.Errorf("failed to parse bind URL: %w", err)
+	}
+
+	switch bindUri.Scheme {
+	case "unix":
+		return "unix", bindUri.Path, nil
+	case "tcp", "http", "https":
+		return "tcp", bindUri.Host, nil
+	default:
+		return "", "", fmt.Errorf("unsupported network scheme %s in address %s", bindUri.Scheme, address)
+	}
+}
+
+// SetupListener sets up a network listener based on the input from configuration
+// envvars. It returns a network listener and the URL to that listener or an error.
+func SetupListener(network, address, socketMode string) (net.Listener, string, error) {
+	formattedAddress := ""
+	var err error
+
+	if network == "" {
+		// keep compatibility
+		network, address, err = parseBindNetFromAddr(address)
+	}
+
+	if err != nil {
+		return nil, "", fmt.Errorf("can't parse bind and network: %w", err)
+	}
+
+	switch network {
+	case "unix":
+		formattedAddress = "unix:" + address
+	case "tcp":
+		if strings.HasPrefix(address, ":") { // assume it's just a port e.g. :4259
+			formattedAddress = "http://localhost" + address
+		} else {
+			formattedAddress = "http://" + address
+		}
+	default:
+		formattedAddress = fmt.Sprintf(`(%s) %s`, network, address)
+	}
+
+	ln, err := net.Listen(network, address)
+	if err != nil {
+		return nil, "", fmt.Errorf("failed to bind to %s: %w", formattedAddress, err)
+	}
+
+	// additional permission handling for unix sockets
+	if network == "unix" {
+		mode, err := strconv.ParseUint(socketMode, 8, 0)
+		if err != nil {
+			ln.Close()
+			return nil, "", fmt.Errorf("could not parse socket mode %s: %w", socketMode, err)
+		}
+
+		err = os.Chmod(address, os.FileMode(mode))
+		if err != nil {
+			err := fmt.Errorf("could not change socket mode: %w", err)
+			clErr := ln.Close()
+			if clErr != nil {
+				return nil, "", errors.Join(err, clErr)
+			}
+			return nil, "", err
+		}
+	}
+
+	return ln, formattedAddress, nil
+}

internal/setuplistener_test.go

@@ -0,0 +1,180 @@
+package internal
+
+import (
+	"io/fs"
+	"os"
+	"path/filepath"
+	"strconv"
+	"strings"
+	"testing"
+)
+
+func TestParseBindNetFromAddr(t *testing.T) {
+	for _, tt := range []struct {
+		name    string
+		address string
+		wantErr bool
+		network string
+		bind    string
+	}{
+		{
+			name:    "simple tcp",
+			address: "localhost:9090",
+			wantErr: false,
+			network: "tcp",
+			bind:    "localhost:9090",
+		},
+		{
+			name:    "simple unix",
+			address: "unix:///tmp/foo.sock",
+			wantErr: false,
+			network: "unix",
+			bind:    "/tmp/foo.sock",
+		},
+		{
+			name:    "invalid network",
+			address: "foo:///tmp/bar.sock",
+			wantErr: true,
+		},
+		{
+			name:    "tcp uri",
+			address: "tcp://[::]:9090",
+			wantErr: false,
+			network: "tcp",
+			bind:    "[::]:9090",
+		},
+		{
+			name:    "http uri",
+			address: "http://[::]:9090",
+			wantErr: false,
+			network: "tcp",
+			bind:    "[::]:9090",
+		},
+		{
+			name:    "https uri",
+			address: "https://[::]:9090",
+			wantErr: false,
+			network: "tcp",
+			bind:    "[::]:9090",
+		},
+	} {
+		t.Run(tt.name, func(t *testing.T) {
+			network, bind, err := parseBindNetFromAddr(tt.address)
+
+			switch {
+			case tt.wantErr && err == nil:
+				t.Errorf("parseBindNetFromAddr(%q) should have errored but did not", tt.address)
+			case !tt.wantErr && err != nil:
+				t.Errorf("parseBindNetFromAddr(%q) threw an error: %v", tt.address, err)
+			}
+
+			if network != tt.network {
+				t.Errorf("parseBindNetFromAddr(%q) wanted network: %q, got: %q", tt.address, tt.network, network)
+			}
+
+			if bind != tt.bind {
+				t.Errorf("parseBindNetFromAddr(%q) wanted bind: %q, got: %q", tt.address, tt.bind, bind)
+			}
+		})
+	}
+}
+
+func TestSetupListener(t *testing.T) {
+	td := t.TempDir()
+
+	for _, tt := range []struct {
+		name                         string
+		network, address, socketMode string
+		wantErr                      bool
+		socketURLPrefix              string
+	}{
+		{
+			name:            "simple tcp",
+			network:         "",
+			address:         ":0",
+			wantErr:         false,
+			socketURLPrefix: "http://localhost:",
+		},
+		{
+			name:            "simple unix",
+			network:         "",
+			address:         "unix://" + filepath.Join(td, "a"),
+			socketMode:      "0770",
+			wantErr:         false,
+			socketURLPrefix: "unix:" + filepath.Join(td, "a"),
+		},
+		{
+			name:            "tcp",
+			network:         "tcp",
+			address:         ":0",
+			wantErr:         false,
+			socketURLPrefix: "http://localhost:",
+		},
+		{
+			name:            "udp",
+			network:         "udp",
+			address:         ":0",
+			wantErr:         true,
+			socketURLPrefix: "http://localhost:",
+		},
+		{
+			name:            "unix socket",
+			network:         "unix",
+			socketMode:      "0770",
+			address:         filepath.Join(td, "a"),
+			wantErr:         false,
+			socketURLPrefix: "unix:" + filepath.Join(td, "a"),
+		},
+		{
+			name:            "invalid socket mode",
+			network:         "unix",
+			socketMode:      "taco bell",
+			address:         filepath.Join(td, "a"),
+			wantErr:         true,
+			socketURLPrefix: "unix:" + filepath.Join(td, "a"),
+		},
+		{
+			name:            "empty socket mode",
+			network:         "unix",
+			socketMode:      "",
+			address:         filepath.Join(td, "a"),
+			wantErr:         true,
+			socketURLPrefix: "unix:" + filepath.Join(td, "a"),
+		},
+	} {
+		t.Run(tt.name, func(t *testing.T) {
+			ln, socketURL, err := SetupListener(tt.network, tt.address, tt.socketMode)
+			switch {
+			case tt.wantErr && err == nil:
+				t.Errorf("SetupListener(%q, %q, %q) should have errored but did not", tt.network, tt.address, tt.socketMode)
+			case !tt.wantErr && err != nil:
+				t.Fatalf("SetupListener(%q, %q, %q) threw an error: %v", tt.network, tt.address, tt.socketMode, err)
+			}
+
+			if ln != nil {
+				defer ln.Close()
+			}
+
+			if !tt.wantErr && !strings.HasPrefix(socketURL, tt.socketURLPrefix) {
+				t.Errorf("SetupListener(%q, %q, %q) should have returned a URL with prefix %q but got: %q", tt.network, tt.address, tt.socketMode, tt.socketURLPrefix, socketURL)
+			}
+
+			if tt.socketMode != "" {
+				mode, err := strconv.ParseUint(tt.socketMode, 8, 0)
+				if err != nil {
+					return
+				}
+
+				sockPath := strings.TrimPrefix(socketURL, "unix:")
+				st, err := os.Stat(sockPath)
+				if err != nil {
+					t.Fatalf("can't os.Stat(%q): %v", sockPath, err)
+				}
+
+				if st.Mode().Perm() != fs.FileMode(mode) {
+					t.Errorf("file mode of %q should be %s but is actually %s", sockPath, strconv.FormatUint(mode, 8), strconv.FormatUint(uint64(st.Mode()), 8))
+				}
+			}
+		})
+	}
+}

lib/config/config.go

@@ -334,6 +334,7 @@ type fileConfig struct {
 	DNSBL       bool                `json:"dnsbl"`
 	DNSTTL      DnsTTL              `json:"dns_ttl"`
 	Logging     *Logging            `json:"logging"`
+	Metrics     *Metrics            `json:"metrics,omitempty"`
 }
 
 func (c *fileConfig) Valid() error {
@@ -375,6 +376,12 @@ func (c *fileConfig) Valid() error {
 		}
 	}
 
+	if c.Metrics != nil {
+		if err := c.Metrics.Valid(); err != nil {
+			errs = append(errs, err)
+		}
+	}
+
 	if len(errs) != 0 {
 		return fmt.Errorf("config is not valid:\n%w", errors.Join(errs...))
 	}
@@ -417,6 +424,7 @@ func Load(fin io.Reader, fname string) (*Config, error) {
 		StatusCodes: c.StatusCodes,
 		Store:       c.Store,
 		Logging:     c.Logging,
+		Metrics:     c.Metrics,
 	}
 
 	if c.OpenGraph.TimeToLive != "" {
@@ -508,6 +516,7 @@ type Config struct {
 	Logging     *Logging
 	DNSBL       bool
 	DNSTTL      DnsTTL
+	Metrics     *Metrics
 }
 
 func (c Config) Valid() error {

lib/config/metrics.go

@@ -0,0 +1,133 @@
+package config
+
+import (
+	"crypto/tls"
+	"crypto/x509"
+	"errors"
+	"fmt"
+	"os"
+	"strconv"
+)
+
+var (
+	ErrInvalidMetricsConfig        = errors.New("config: invalid metrics configuration")
+	ErrInvalidMetricsTLSConfig     = errors.New("config: invalid metrics TLS configuration")
+	ErrNoMetricsBind               = errors.New("config.Metrics: must define bind")
+	ErrNoMetricsNetwork            = errors.New("config.Metrics: must define network")
+	ErrNoMetricsSocketMode         = errors.New("config.Metrics: must define socket mode when using unix sockets")
+	ErrInvalidMetricsSocketMode    = errors.New("config.Metrics: invalid unix socket mode")
+	ErrInvalidMetricsNetwork       = errors.New("config.Metrics: invalid metrics network")
+	ErrNoMetricsTLSCertificate     = errors.New("config.Metrics.TLS: must define certificate file")
+	ErrNoMetricsTLSKey             = errors.New("config.Metrics.TLS: must define key file")
+	ErrInvalidMetricsTLSKeypair    = errors.New("config.Metrics.TLS: keypair is invalid")
+	ErrInvalidMetricsCACertificate = errors.New("config.Metrics.TLS: invalid CA certificate")
+	ErrCantReadFile                = errors.New("config: can't read required file")
+)
+
+type Metrics struct {
+	Bind       string      `json:"bind" yaml:"bind"`
+	Network    string      `json:"network" yaml:"network"`
+	SocketMode string      `json:"socketMode" yaml:"socketMode"`
+	TLS        *MetricsTLS `json:"tls" yaml:"tls"`
+}
+
+func (m *Metrics) Valid() error {
+	var errs []error
+
+	if m.Bind == "" {
+		errs = append(errs, ErrNoMetricsBind)
+	}
+
+	if m.Network == "" {
+		errs = append(errs, ErrNoMetricsNetwork)
+	}
+
+	switch m.Network {
+	case "tcp", "tcp4", "tcp6": // https://pkg.go.dev/net#Listen
+	case "unix":
+		if m.SocketMode == "" {
+			errs = append(errs, ErrNoMetricsSocketMode)
+		}
+
+		if _, err := strconv.ParseUint(m.SocketMode, 8, 0); err != nil {
+			errs = append(errs, fmt.Errorf("%w: %w", ErrInvalidMetricsSocketMode, err))
+		}
+	default:
+		errs = append(errs, ErrInvalidMetricsNetwork)
+	}
+
+	if m.TLS != nil {
+		if err := m.TLS.Valid(); err != nil {
+			errs = append(errs, err)
+		}
+	}
+
+	if len(errs) != 0 {
+		return errors.Join(ErrInvalidMetricsConfig, errors.Join(errs...))
+	}
+
+	return nil
+}
+
+type MetricsTLS struct {
+	Certificate string `json:"certificate" yaml:"certificate"`
+	Key         string `json:"key" yaml:"key"`
+	CA          string `json:"ca" yaml:"ca"`
+}
+
+func (mt *MetricsTLS) Valid() error {
+	var errs []error
+
+	if mt.Certificate == "" {
+		errs = append(errs, ErrNoMetricsTLSCertificate)
+	}
+
+	if err := canReadFile(mt.Certificate); err != nil {
+		errs = append(errs, fmt.Errorf("%w %s: %w", ErrCantReadFile, mt.Certificate, err))
+	}
+
+	if mt.Key == "" {
+		errs = append(errs, ErrNoMetricsTLSKey)
+	}
+
+	if err := canReadFile(mt.Key); err != nil {
+		errs = append(errs, fmt.Errorf("%w %s: %w", ErrCantReadFile, mt.Key, err))
+	}
+
+	if _, err := tls.LoadX509KeyPair(mt.Certificate, mt.Key); err != nil {
+		errs = append(errs, fmt.Errorf("%w: %w", ErrInvalidMetricsTLSKeypair, err))
+	}
+
+	if mt.CA != "" {
+		caCert, err := os.ReadFile(mt.CA)
+		if err != nil {
+			errs = append(errs, fmt.Errorf("%w %s: %w", ErrCantReadFile, mt.CA, err))
+		}
+
+		certPool := x509.NewCertPool()
+		if !certPool.AppendCertsFromPEM(caCert) {
+			errs = append(errs, fmt.Errorf("%w %s", ErrInvalidMetricsCACertificate, mt.CA))
+		}
+	}
+
+	if len(errs) != 0 {
+		return errors.Join(ErrInvalidMetricsTLSConfig, errors.Join(errs...))
+	}
+
+	return nil
+}
+
+func canReadFile(fname string) error {
+	fin, err := os.Open(fname)
+	if err != nil {
+		return err
+	}
+	defer fin.Close()
+
+	data := make([]byte, 64)
+	if _, err := fin.Read(data); err != nil {
+		return fmt.Errorf("can't read %s: %w", fname, err)
+	}
+
+	return nil
+}

lib/config/metrics_test.go

@@ -0,0 +1,169 @@
+package config
+
+import (
+	"errors"
+	"testing"
+)
+
+func TestMetricsValid(t *testing.T) {
+	for _, tt := range []struct {
+		name  string
+		input *Metrics
+		err   error
+	}{
+		{
+			name: "basic TCP",
+			input: &Metrics{
+				Bind:    ":9090",
+				Network: "tcp",
+			},
+		},
+		{
+			name: "basic TCP4",
+			input: &Metrics{
+				Bind:    ":9090",
+				Network: "tcp4",
+			},
+		},
+		{
+			name: "basic TCP6",
+			input: &Metrics{
+				Bind:    ":9090",
+				Network: "tcp6",
+			},
+		},
+		{
+			name: "basic unix",
+			input: &Metrics{
+				Bind:       "/tmp/anubis-metrics.sock",
+				Network:    "unix",
+				SocketMode: "0770",
+			},
+		},
+		{
+			name:  "no bind",
+			input: &Metrics{},
+			err:   ErrNoMetricsBind,
+		},
+		{
+			name:  "no network",
+			input: &Metrics{},
+			err:   ErrNoMetricsNetwork,
+		},
+		{
+			name: "no unix socket mode",
+			input: &Metrics{
+				Bind:    "/tmp/anubis-metrics.sock",
+				Network: "unix",
+			},
+			err: ErrNoMetricsSocketMode,
+		},
+		{
+			name: "invalid unix socket mode",
+			input: &Metrics{
+				Bind:       "/tmp/anubis-metrics.sock",
+				Network:    "unix",
+				SocketMode: "taco bell",
+			},
+			err: ErrInvalidMetricsSocketMode,
+		},
+		{
+			name: "invalid network",
+			input: &Metrics{
+				Bind:    ":9090",
+				Network: "taco",
+			},
+			err: ErrInvalidMetricsNetwork,
+		},
+		{
+			name: "invalid TLS config",
+			input: &Metrics{
+				Bind:    ":9090",
+				Network: "tcp",
+				TLS:     &MetricsTLS{},
+			},
+			err: ErrInvalidMetricsTLSConfig,
+		},
+		{
+			name: "selfsigned TLS cert",
+			input: &Metrics{
+				Bind:    ":9090",
+				Network: "tcp",
+				TLS: &MetricsTLS{
+					Certificate: "./testdata/tls/selfsigned.crt",
+					Key:         "./testdata/tls/selfsigned.key",
+				},
+			},
+		},
+		{
+			name: "wrong path to selfsigned TLS cert",
+			input: &Metrics{
+				Bind:    ":9090",
+				Network: "tcp",
+				TLS: &MetricsTLS{
+					Certificate: "./testdata/tls2/selfsigned.crt",
+					Key:         "./testdata/tls2/selfsigned.key",
+				},
+			},
+			err: ErrCantReadFile,
+		},
+		{
+			name: "unparseable TLS cert",
+			input: &Metrics{
+				Bind:    ":9090",
+				Network: "tcp",
+				TLS: &MetricsTLS{
+					Certificate: "./testdata/tls/invalid.crt",
+					Key:         "./testdata/tls/invalid.key",
+				},
+			},
+			err: ErrInvalidMetricsTLSKeypair,
+		},
+		{
+			name: "mTLS with CA",
+			input: &Metrics{
+				Bind:    ":9090",
+				Network: "tcp",
+				TLS: &MetricsTLS{
+					Certificate: "./testdata/tls/selfsigned.crt",
+					Key:         "./testdata/tls/selfsigned.key",
+					CA:          "./testdata/tls/minica.pem",
+				},
+			},
+		},
+		{
+			name: "mTLS with nonexistent CA",
+			input: &Metrics{
+				Bind:    ":9090",
+				Network: "tcp",
+				TLS: &MetricsTLS{
+					Certificate: "./testdata/tls/selfsigned.crt",
+					Key:         "./testdata/tls/selfsigned.key",
+					CA:          "./testdata/tls/nonexistent.crt",
+				},
+			},
+			err: ErrCantReadFile,
+		},
+		{
+			name: "mTLS with invalid CA",
+			input: &Metrics{
+				Bind:    ":9090",
+				Network: "tcp",
+				TLS: &MetricsTLS{
+					Certificate: "./testdata/tls/selfsigned.crt",
+					Key:         "./testdata/tls/selfsigned.key",
+					CA:          "./testdata/tls/invalid.crt",
+				},
+			},
+			err: ErrInvalidMetricsCACertificate,
+		},
+	} {
+		t.Run(tt.name, func(t *testing.T) {
+			if err := tt.input.Valid(); !errors.Is(err, tt.err) {
+				t.Logf("wanted error: %v", tt.err)
+				t.Logf("got error:    %v", err)
+				t.Error("validation failed")
+			}
+		})
+	}
+}

lib/config/testdata/bad/metrics-invalid-net.yaml

@@ -0,0 +1,3 @@
+metrics:
+  bind: ":9090"
+  network: taco

lib/config/testdata/good/allow_everyone.json

@@ -5,5 +5,9 @@
       "remote_addresses": ["0.0.0.0/0", "::/0"],
       "action": "ALLOW"
     }
-  ]
+  ],
+  "metrics": {
+    "bind": ":9090",
+    "network": "tcp"
+  }
 }

lib/config/testdata/good/allow_everyone.yaml

@@ -4,3 +4,7 @@ bots:
       - "0.0.0.0/0"
       - "::/0"
     action: ALLOW
+
+metrics:
+  bind: ":9090"
+  network: "tcp"

lib/config/testdata/good/block_cf_workers.json

@@ -8,5 +8,9 @@
       "action": "DENY"
     }
   ],
-  "dnsbl": false
+  "dnsbl": false,
+  "metrics": {
+    "bind": ":9090",
+    "network": "tcp"
+  }
 }

lib/config/testdata/good/block_cf_workers.yaml

@@ -3,3 +3,7 @@ bots:
     headers_regex:
       CF-Worker: .*
     action: DENY
+
+metrics:
+  bind: ":9090"
+  network: "tcp"

lib/config/testdata/good/challenge_cloudflare.yaml

@@ -4,3 +4,7 @@ bots:
     asns:
       match:
         - 13335 # Cloudflare
+
+metrics:
+  bind: ":9090"
+  network: "tcp"

lib/config/testdata/good/challengemozilla.json

@@ -5,5 +5,9 @@
       "user_agent_regex": "Mozilla",
       "action": "CHALLENGE"
     }
-  ]
+  ],
+  "metrics": {
+    "bind": ":9090",
+    "network": "tcp"
+  }
 }

lib/config/testdata/good/challengemozilla.yaml

@@ -2,3 +2,7 @@ bots:
   - name: generic-browser
     user_agent_regex: Mozilla
     action: CHALLENGE
+
+metrics:
+  bind: ":9090"
+  network: "tcp"

lib/config/testdata/good/dns-ttl-custom.yaml

@@ -6,3 +6,7 @@ bots:
   - name: "test"
     user_agent_regex: ".*"
     action: "DENY"
+
+metrics:
+  bind: ":9090"
+  network: "tcp"

lib/config/testdata/good/entropy.yaml

@@ -6,3 +6,7 @@ bots:
         - '"Accept" in headers'
         - headers["Accept"].contains("text/html")
         - randInt(1) == 0
+
+metrics:
+  bind: ":9090"
+  network: "tcp"

lib/config/testdata/good/everything_blocked.json

@@ -6,5 +6,9 @@
       "action": "DENY"
     }
   ],
-  "dnsbl": false
+  "dnsbl": false,
+  "metrics": {
+    "bind": ":9090",
+    "network": "tcp"
+  }
 }

lib/config/testdata/good/everything_blocked.yaml

@@ -2,3 +2,7 @@ bots:
   - name: everything
     user_agent_regex: .*
     action: DENY
+
+metrics:
+  bind: ":9090"
+  network: "tcp"

lib/config/testdata/good/geoip_us.yaml

@@ -4,3 +4,7 @@ bots:
     geoip:
       countries:
         - US
+
+metrics:
+  bind: ":9090"
+  network: "tcp"

lib/config/testdata/good/git_client.json

@@ -10,5 +10,9 @@
         ]
       }
     }
-  ]
+  ],
+  "metrics": {
+    "bind": ":9090",
+    "network": "tcp"
+  }
 }

lib/config/testdata/good/git_client.yaml

@@ -6,3 +6,7 @@ bots:
         - userAgent.startsWith("git/") || userAgent.contains("libgit")
         - >
           "Git-Protocol" in headers && headers["Git-Protocol"] == "version=2"
+
+metrics:
+  bind: ":9090"
+  network: "tcp"

lib/config/testdata/good/import_filesystem.json

@@ -3,5 +3,9 @@
     {
       "import": "./testdata/hack-test.json"
     }
-  ]
+  ],
+  "metrics": {
+    "bind": ":9090",
+    "network": "tcp"
+  }
 }

lib/config/testdata/good/import_filesystem.yaml

@@ -1,2 +1,6 @@
 bots:
   - import: ./testdata/hack-test.yaml
+
+metrics:
+  bind: ":9090"
+  network: "tcp"

lib/config/testdata/good/import_keep_internet_working.json

@@ -3,5 +3,9 @@
     {
       "import": "(data)/common/keep-internet-working.yaml"
     }
-  ]
+  ],
+  "metrics": {
+    "bind": ":9090",
+    "network": "tcp"
+  }
 }

lib/config/testdata/good/import_keep_internet_working.yaml

@@ -1,2 +1,6 @@
 bots:
   - import: (data)/common/keep-internet-working.yaml
+
+metrics:
+  bind: ":9090"
+  network: "tcp"

lib/config/testdata/good/impressum.yaml

@@ -8,3 +8,7 @@ impressum:
   page:
     title: Test
     body: <p>This is a test</p>
+
+metrics:
+  bind: ":9090"
+  network: "tcp"

lib/config/testdata/good/logging-file.yaml

@@ -13,3 +13,7 @@ logs:
     oldFileTimeFormat: 2006-01-02T15-04-05 # RFC 3339-ish
     compress: true
     useLocalTime: false # timezone for rotated files is UTC
+
+metrics:
+  bind: ":9090"
+  network: "tcp"

lib/config/testdata/good/logging-stdio.yaml

@@ -5,3 +5,7 @@ bots:
 
 logging:
   sink: "stdio"
+
+metrics:
+  bind: ":9090"
+  network: "tcp"

lib/config/testdata/good/no-thresholds.yaml

@@ -6,3 +6,7 @@ bots:
       adjust: 5
 
 thresholds: []
+
+metrics:
+  bind: ":9090"
+  network: "tcp"

lib/config/testdata/good/old_xesite.json

@@ -75,5 +75,9 @@
       "user_agent_regex": "Mozilla",
       "action": "CHALLENGE"
     }
-  ]
+  ],
+  "metrics": {
+    "bind": ":9090",
+    "network": "tcp"
+  }
 }

lib/config/testdata/good/opengraph_all_good.yaml

@@ -10,3 +10,7 @@ openGraph:
   default:
     "og:title": "Xe's magic land of fun"
     "og:description": "We're no strangers to love, you know the rules and so do I"
+
+metrics:
+  bind: ":9090"
+  network: "tcp"

lib/config/testdata/good/simple-weight.yaml

@@ -4,3 +4,7 @@ bots:
     user_agent_regex: Mozilla
     weight:
       adjust: 5
+
+metrics:
+  bind: ":9090"
+  network: "tcp"

lib/config/testdata/good/status-codes-paranoid.json

@@ -9,5 +9,9 @@
   "status_codes": {
     "CHALLENGE": 200,
     "DENY": 200
+  },
+  "metrics": {
+    "bind": ":9090",
+    "network": "tcp"
   }
 }

lib/config/testdata/good/status-codes-paranoid.yaml

@@ -6,3 +6,7 @@ bots:
 status_codes:
   CHALLENGE: 200
   DENY: 200
+
+metrics:
+  bind: ":9090"
+  network: "tcp"

lib/config/testdata/good/status-codes-rfc.json

@@ -9,5 +9,9 @@
   "status_codes": {
     "CHALLENGE": 403,
     "DENY": 403
+  },
+  "metrics": {
+    "bind": ":9090",
+    "network": "tcp"
   }
 }

lib/config/testdata/good/status-codes-rfc.yaml

@@ -6,3 +6,7 @@ bots:
 status_codes:
   CHALLENGE: 403
   DENY: 403
+
+metrics:
+  bind: ":9090"
+  network: "tcp"

lib/config/testdata/good/thresholds.yaml

@@ -33,3 +33,7 @@ thresholds:
     challenge:
       algorithm: fast
       difficulty: 4
+
+metrics:
+  bind: ":9090"
+  network: "tcp"

lib/config/testdata/good/weight-no-weight.yaml

@@ -2,3 +2,7 @@ bots:
   - name: weight
     action: WEIGH
     user_agent_regex: Mozilla
+
+metrics:
+  bind: ":9090"
+  network: "tcp"

lib/config/testdata/tls/1.1.1.1/cert.pem

@@ -0,0 +1,12 @@
+-----BEGIN CERTIFICATE-----
+MIIB1zCCAVygAwIBAgIIYO0SAFtXlVgwCgYIKoZIzj0EAwMwIDEeMBwGA1UEAxMV
+bWluaWNhIHJvb3QgY2EgNDE2MmMwMB4XDTI2MDQyMjIzMjUwMVoXDTI4MDUyMjIz
+MjUwMVowEjEQMA4GA1UEAxMHMS4xLjEuMTB2MBAGByqGSM49AgEGBSuBBAAiA2IA
+BLsuA2LKGbEBuSA4LTm1KaKc7/QCkUOsipXR4+D5/3sWBZiAH7iWUgHwpx5YZf2q
+kZn6oRda+ks0JLTQ6VhteQedmb7l86bMeDMR8p4Lg2b38l/xEr7S25UfUDKudXrO
+AqNxMG8wDgYDVR0PAQH/BAQDAgWgMB0GA1UdJQQWMBQGCCsGAQUFBwMBBggrBgEF
+BQcDAjAMBgNVHRMBAf8EAjAAMB8GA1UdIwQYMBaAFE/7VDxF2+cUs9bu0pJM3xoC
+L1TSMA8GA1UdEQQIMAaHBAEBAQEwCgYIKoZIzj0EAwMDaQAwZgIxAPLXds9MMH4K
+F5FxTf9i0PKPsLQARsABVTgwB94hMR70rqW8Pwbjl7ZGNaYlaeRHUwIxAPMQ8zoF
+nim+YS1xLqQek/LXuJto8jxcfkQQBsboVzcTa5uaNRhNd5YwrpomGl3lKA==
+-----END CERTIFICATE-----

lib/config/testdata/tls/1.1.1.1/key.pem

@@ -0,0 +1,6 @@
+-----BEGIN PRIVATE KEY-----
+MIG2AgEAMBAGByqGSM49AgEGBSuBBAAiBIGeMIGbAgEBBDBN8QsHxxHGJpStu8K7
+D/FmaBBNo6c514KGFSIfqGFuREF5aOL3gN/W11yk2OIibdWhZANiAAS7LgNiyhmx
+AbkgOC05tSminO/0ApFDrIqV0ePg+f97FgWYgB+4llIB8KceWGX9qpGZ+qEXWvpL
+NCS00OlYbXkHnZm+5fOmzHgzEfKeC4Nm9/Jf8RK+0tuVH1AyrnV6zgI=
+-----END PRIVATE KEY-----

lib/config/testdata/tls/minica-key.pem

@@ -0,0 +1,6 @@
+-----BEGIN PRIVATE KEY-----
+MIG2AgEAMBAGByqGSM49AgEGBSuBBAAiBIGeMIGbAgEBBDDr9QQo7ZaTgUL6d73G
+2BG7+YRTFJHAZa0FogRglfc+jYttL1J4/xTig3RmHoqSgrehZANiAASDhijM9Xe0
+G9Vam6AJMeKC6aWDNSLwrxNVmPxemsY/yJ1urBgnxRd9GFH6YW1ki/B8rS+Xl1UX
+NnhBrukLaXvgAQQq782/5IUYGsvK5jw8+dSscYVMCQJwGfmQuaNeczQ=
+-----END PRIVATE KEY-----

lib/config/testdata/tls/minica.pem

@@ -0,0 +1,13 @@
+-----BEGIN CERTIFICATE-----
+MIIB+zCCAYKgAwIBAgIIQWLAtv4ijQ0wCgYIKoZIzj0EAwMwIDEeMBwGA1UEAxMV
+bWluaWNhIHJvb3QgY2EgNDE2MmMwMCAXDTI2MDQyMjIzMjUwMVoYDzIxMjYwNDIy
+MjMyNTAxWjAgMR4wHAYDVQQDExVtaW5pY2Egcm9vdCBjYSA0MTYyYzAwdjAQBgcq
+hkjOPQIBBgUrgQQAIgNiAASDhijM9Xe0G9Vam6AJMeKC6aWDNSLwrxNVmPxemsY/
+yJ1urBgnxRd9GFH6YW1ki/B8rS+Xl1UXNnhBrukLaXvgAQQq782/5IUYGsvK5jw8
++dSscYVMCQJwGfmQuaNeczSjgYYwgYMwDgYDVR0PAQH/BAQDAgKEMB0GA1UdJQQW
+MBQGCCsGAQUFBwMBBggrBgEFBQcDAjASBgNVHRMBAf8ECDAGAQH/AgEAMB0GA1Ud
+DgQWBBRP+1Q8RdvnFLPW7tKSTN8aAi9U0jAfBgNVHSMEGDAWgBRP+1Q8RdvnFLPW
+7tKSTN8aAi9U0jAKBggqhkjOPQQDAwNnADBkAjBfY7vb7cuLTjg7uoe+kl07FMYT
+BGMSnWdhN3yXqMUS3A6XZxD/LntXT6V7yFOlAJYCMH7w8/ATYaTqbk2jBRyQt9/x
+ajN+kZ6ZK+fKttqE8CD62mbHg09xoNxRq+K2I3PVyQ==
+-----END CERTIFICATE-----

lib/config/testdata/tls/selfsigned.crt

@@ -0,0 +1,11 @@
+-----BEGIN CERTIFICATE-----
+MIIBnzCCAVGgAwIBAgIUK39B3Ft+kU5o81IuISs79O4u1ncwBQYDK2VwMEUxCzAJ
+BgNVBAYTAkFVMRMwEQYDVQQIDApTb21lLVN0YXRlMSEwHwYDVQQKDBhJbnRlcm5l
+dCBXaWRnaXRzIFB0eSBMdGQwHhcNMjYwNDIyMTQyNjE4WhcNMjYwNTIyMTQyNjE4
+WjBFMQswCQYDVQQGEwJBVTETMBEGA1UECAwKU29tZS1TdGF0ZTEhMB8GA1UECgwY
+SW50ZXJuZXQgV2lkZ2l0cyBQdHkgTHRkMCowBQYDK2VwAyEAfgpAUpp8MIOOdQpH
+fxaw3R7mFKQRMR6Kmxzk1Xn/2VujUzBRMB0GA1UdDgQWBBSmkBmzo0RiZ2iocMR8
+uIIpz9cZyTAfBgNVHSMEGDAWgBSmkBmzo0RiZ2iocMR8uIIpz9cZyTAPBgNVHRMB
+Af8EBTADAQH/MAUGAytlcANBAG37XXZrVUUzGyy3T9qsPIzvJQAGpGhdjJ7bt9O6
+sBhzrliTONPrudYuyUggWsHgFb0JlN2xs4/2HhKU+PY7AAQ=
+-----END CERTIFICATE-----

lib/config/testdata/tls/selfsigned.key

@@ -0,0 +1,3 @@
+-----BEGIN PRIVATE KEY-----
+MC4CAQAwBQYDK2VwBCIEIL0HxjjfVlg6zQPB9/zTLq0IBzfp8gEoifEYzQZYIj+T
+-----END PRIVATE KEY-----

lib/metrics/keypairreloader.go

@@ -0,0 +1,81 @@
+package metrics
+
+import (
+	"crypto/tls"
+	"fmt"
+	"log/slog"
+	"os"
+	"sync"
+	"time"
+)
+
+type KeypairReloader struct {
+	certMu   sync.RWMutex
+	cert     *tls.Certificate
+	certPath string
+	keyPath  string
+	modTime  time.Time
+	lg       *slog.Logger
+}
+
+func NewKeypairReloader(certPath, keyPath string, lg *slog.Logger) (*KeypairReloader, error) {
+	result := &KeypairReloader{
+		certPath: certPath,
+		keyPath:  keyPath,
+		lg:       lg,
+	}
+	cert, err := tls.LoadX509KeyPair(certPath, keyPath)
+	if err != nil {
+		return nil, err
+	}
+	result.cert = &cert
+
+	st, err := os.Stat(certPath)
+	if err != nil {
+		return nil, err
+	}
+	result.modTime = st.ModTime()
+
+	return result, nil
+}
+
+func (kpr *KeypairReloader) maybeReload() error {
+	kpr.lg.Debug("loading new keypair", "cert", kpr.certPath, "key", kpr.keyPath)
+	newCert, err := tls.LoadX509KeyPair(kpr.certPath, kpr.keyPath)
+	if err != nil {
+		return err
+	}
+
+	st, err := os.Stat(kpr.certPath)
+	if err != nil {
+		return err
+	}
+
+	kpr.certMu.Lock()
+	defer kpr.certMu.Unlock()
+	kpr.cert = &newCert
+	kpr.modTime = st.ModTime()
+
+	return nil
+}
+
+func (kpr *KeypairReloader) GetCertificate(*tls.ClientHelloInfo) (*tls.Certificate, error) {
+	st, err := os.Stat(kpr.certPath)
+	if err != nil {
+		return nil, fmt.Errorf("stat(%q): %w", kpr.certPath, err)
+	}
+
+	kpr.certMu.RLock()
+	needsReload := st.ModTime().After(kpr.modTime)
+	kpr.certMu.RUnlock()
+
+	if needsReload {
+		if err := kpr.maybeReload(); err != nil {
+			return nil, fmt.Errorf("reload cert: %w", err)
+		}
+	}
+
+	kpr.certMu.RLock()
+	defer kpr.certMu.RUnlock()
+	return kpr.cert, nil
+}

lib/metrics/keypairreloader_test.go

@@ -0,0 +1,265 @@
+package metrics
+
+import (
+	"bytes"
+	"crypto/rand"
+	"crypto/rsa"
+	"crypto/x509"
+	"crypto/x509/pkix"
+	"encoding/pem"
+	"errors"
+	"log/slog"
+	"math/big"
+	"os"
+	"path/filepath"
+	"testing"
+	"time"
+)
+
+func discardLogger() *slog.Logger {
+	return slog.New(slog.DiscardHandler)
+}
+
+// writeKeypair generates a fresh self-signed cert + RSA key and writes them
+// as PEM files in dir. Returns the paths and the cert's DER bytes so callers
+// can identify which pair was loaded.
+func writeKeypair(t *testing.T, dir, prefix string) (certPath, keyPath string, certDER []byte) {
+	t.Helper()
+
+	priv, err := rsa.GenerateKey(rand.Reader, 2048)
+	if err != nil {
+		t.Fatalf("rsa.GenerateKey: %v", err)
+	}
+
+	tmpl := &x509.Certificate{
+		SerialNumber:          big.NewInt(time.Now().UnixNano()),
+		Subject:               pkix.Name{CommonName: "keypairreloader-test"},
+		NotBefore:             time.Now().Add(-time.Hour),
+		NotAfter:              time.Now().Add(time.Hour),
+		KeyUsage:              x509.KeyUsageDigitalSignature | x509.KeyUsageKeyEncipherment,
+		ExtKeyUsage:           []x509.ExtKeyUsage{x509.ExtKeyUsageServerAuth},
+		BasicConstraintsValid: true,
+		DNSNames:              []string{"keypairreloader-test"},
+	}
+
+	der, err := x509.CreateCertificate(rand.Reader, tmpl, tmpl, &priv.PublicKey, priv)
+	if err != nil {
+		t.Fatalf("x509.CreateCertificate: %v", err)
+	}
+
+	certPath = filepath.Join(dir, prefix+"cert.pem")
+	keyPath = filepath.Join(dir, prefix+"key.pem")
+
+	certPEM := pem.EncodeToMemory(&pem.Block{Type: "CERTIFICATE", Bytes: der})
+	keyPEM := pem.EncodeToMemory(&pem.Block{Type: "RSA PRIVATE KEY", Bytes: x509.MarshalPKCS1PrivateKey(priv)})
+
+	if err := os.WriteFile(certPath, certPEM, 0o600); err != nil {
+		t.Fatalf("write cert: %v", err)
+	}
+	if err := os.WriteFile(keyPath, keyPEM, 0o600); err != nil {
+		t.Fatalf("write key: %v", err)
+	}
+
+	return certPath, keyPath, der
+}
+
+func TestNewKeypairReloader(t *testing.T) {
+	dir := t.TempDir()
+	goodCert, goodKey, _ := writeKeypair(t, dir, "good-")
+
+	garbagePath := filepath.Join(dir, "garbage.pem")
+	if err := os.WriteFile(garbagePath, []byte("not a pem file"), 0o600); err != nil {
+		t.Fatalf("write garbage: %v", err)
+	}
+
+	tests := []struct {
+		name     string
+		certPath string
+		keyPath  string
+		wantErr  error
+		wantNil  bool
+	}{
+		{
+			name:     "valid cert and key",
+			certPath: goodCert,
+			keyPath:  goodKey,
+		},
+		{
+			name:     "missing cert file",
+			certPath: filepath.Join(dir, "does-not-exist.pem"),
+			keyPath:  goodKey,
+			wantErr:  os.ErrNotExist,
+			wantNil:  true,
+		},
+		{
+			name:     "missing key file",
+			certPath: goodCert,
+			keyPath:  filepath.Join(dir, "does-not-exist-key.pem"),
+			wantErr:  os.ErrNotExist,
+			wantNil:  true,
+		},
+		{
+			name:     "cert file is garbage",
+			certPath: garbagePath,
+			keyPath:  goodKey,
+			wantNil:  true,
+		},
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			kpr, err := NewKeypairReloader(tt.certPath, tt.keyPath, discardLogger())
+
+			if tt.wantErr != nil && !errors.Is(err, tt.wantErr) {
+				t.Errorf("err = %v, want errors.Is(..., %v)", err, tt.wantErr)
+			}
+			if tt.wantErr == nil && !tt.wantNil && err != nil {
+				t.Errorf("unexpected err: %v", err)
+			}
+			if tt.wantNil && kpr != nil {
+				t.Errorf("kpr = %+v, want nil", kpr)
+			}
+			if !tt.wantNil && kpr == nil {
+				t.Errorf("kpr is nil, want non-nil")
+			}
+		})
+	}
+}
+
+func TestKeypairReloader_GetCertificate(t *testing.T) {
+	tests := []struct {
+		name string
+		run  func(t *testing.T)
+	}{
+		{
+			name: "returns loaded cert",
+			run: func(t *testing.T) {
+				dir := t.TempDir()
+				certPath, keyPath, wantDER := writeKeypair(t, dir, "a-")
+
+				kpr, err := NewKeypairReloader(certPath, keyPath, discardLogger())
+				if err != nil {
+					t.Fatalf("NewKeypairReloader: %v", err)
+				}
+
+				got, err := kpr.GetCertificate(nil)
+				if err != nil {
+					t.Fatalf("GetCertificate: %v", err)
+				}
+				if len(got.Certificate) == 0 || !bytes.Equal(got.Certificate[0], wantDER) {
+					t.Errorf("GetCertificate returned wrong cert bytes")
+				}
+			},
+		},
+		{
+			name: "reloads when mtime advances",
+			run: func(t *testing.T) {
+				dir := t.TempDir()
+				certPath, keyPath, _ := writeKeypair(t, dir, "a-")
+
+				kpr, err := NewKeypairReloader(certPath, keyPath, discardLogger())
+				if err != nil {
+					t.Fatalf("NewKeypairReloader: %v", err)
+				}
+
+				// Overwrite with a new pair at the same paths and bump mtime.
+				newCertPath, newKeyPath, newDER := writeKeypair(t, dir, "b-")
+				mustRename(t, newCertPath, certPath)
+				mustRename(t, newKeyPath, keyPath)
+				future := time.Now().Add(time.Hour)
+				if err := os.Chtimes(certPath, future, future); err != nil {
+					t.Fatalf("Chtimes: %v", err)
+				}
+
+				got, err := kpr.GetCertificate(nil)
+				if err != nil {
+					t.Fatalf("GetCertificate: %v", err)
+				}
+				if len(got.Certificate) == 0 || !bytes.Equal(got.Certificate[0], newDER) {
+					t.Errorf("GetCertificate did not return reloaded cert")
+				}
+			},
+		},
+		{
+			name: "does not reload when mtime unchanged",
+			run: func(t *testing.T) {
+				dir := t.TempDir()
+				certPath, keyPath, originalDER := writeKeypair(t, dir, "a-")
+
+				kpr, err := NewKeypairReloader(certPath, keyPath, discardLogger())
+				if err != nil {
+					t.Fatalf("NewKeypairReloader: %v", err)
+				}
+
+				// Overwrite the cert/key files with a *different* keypair, then
+				// rewind mtime so the reloader must not pick up the change.
+				newCertPath, newKeyPath, newDER := writeKeypair(t, dir, "b-")
+				mustRename(t, newCertPath, certPath)
+				mustRename(t, newKeyPath, keyPath)
+				past := time.Unix(0, 0)
+				if err := os.Chtimes(certPath, past, past); err != nil {
+					t.Fatalf("Chtimes: %v", err)
+				}
+
+				got, err := kpr.GetCertificate(nil)
+				if err != nil {
+					t.Fatalf("GetCertificate: %v", err)
+				}
+				if len(got.Certificate) == 0 {
+					t.Fatal("empty cert chain")
+				}
+				if bytes.Equal(got.Certificate[0], newDER) {
+					t.Errorf("GetCertificate reloaded despite unchanged mtime")
+				}
+				if !bytes.Equal(got.Certificate[0], originalDER) {
+					t.Errorf("GetCertificate did not return original cert")
+				}
+			},
+		},
+		{
+			name: "does not panic when reload fails after mtime bump",
+			run: func(t *testing.T) {
+				dir := t.TempDir()
+				certPath, keyPath, _ := writeKeypair(t, dir, "a-")
+
+				kpr, err := NewKeypairReloader(certPath, keyPath, discardLogger())
+				if err != nil {
+					t.Fatalf("NewKeypairReloader: %v", err)
+				}
+
+				// Corrupt the cert file and bump mtime. maybeReload will fail.
+				if err := os.WriteFile(certPath, []byte("not a pem file"), 0o600); err != nil {
+					t.Fatalf("corrupt cert: %v", err)
+				}
+				future := time.Now().Add(time.Hour)
+				if err := os.Chtimes(certPath, future, future); err != nil {
+					t.Fatalf("Chtimes: %v", err)
+				}
+
+				defer func() {
+					if r := recover(); r != nil {
+						t.Fatalf("GetCertificate panicked on reload failure: %v", r)
+					}
+				}()
+
+				got, err := kpr.GetCertificate(nil)
+				if err == nil {
+					t.Errorf("GetCertificate returned nil err for corrupt cert; got %+v", got)
+				}
+			},
+		},
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			tt.run(t)
+		})
+	}
+}
+
+func mustRename(t *testing.T, from, to string) {
+	t.Helper()
+	if err := os.Rename(from, to); err != nil {
+		t.Fatalf("rename %q -> %q: %v", from, to, err)
+	}
+}

lib/metrics/metrics.go

@@ -0,0 +1,123 @@
+package metrics
+
+import (
+	"context"
+	"crypto/tls"
+	"crypto/x509"
+	"errors"
+	"fmt"
+	"log/slog"
+	"net/http"
+	"net/http/pprof"
+	"os"
+	"time"
+
+	"github.com/TecharoHQ/anubis/internal"
+	"github.com/TecharoHQ/anubis/lib/config"
+	"github.com/prometheus/client_golang/prometheus/promhttp"
+	healthv1 "google.golang.org/grpc/health/grpc_health_v1"
+)
+
+type Server struct {
+	Config *config.Metrics
+	Log    *slog.Logger
+}
+
+func (s *Server) Run(ctx context.Context, done func()) {
+	defer done()
+	lg := s.Log.With("subsystem", "metrics")
+
+	if err := s.run(ctx, lg); err != nil {
+		lg.Error("can't serve metrics server", "err", err)
+	}
+}
+
+func (s *Server) run(ctx context.Context, lg *slog.Logger) error {
+	mux := http.NewServeMux()
+	mux.HandleFunc("GET /debug/pprof/", pprof.Index)
+	mux.HandleFunc("GET /debug/pprof/cmdline", pprof.Cmdline)
+	mux.HandleFunc("GET /debug/pprof/profile", pprof.Profile)
+	mux.HandleFunc("GET /debug/pprof/symbol", pprof.Symbol)
+	mux.HandleFunc("GET /debug/pprof/trace", pprof.Trace)
+	mux.Handle("/metrics", promhttp.Handler())
+	mux.HandleFunc("/healthz", func(w http.ResponseWriter, r *http.Request) {
+		st, ok := internal.GetHealth("anubis")
+		if !ok {
+			slog.Error("health service anubis does not exist, file a bug")
+		}
+
+		switch st {
+		case healthv1.HealthCheckResponse_NOT_SERVING:
+			http.Error(w, "NOT OK", http.StatusInternalServerError)
+			return
+		case healthv1.HealthCheckResponse_SERVING:
+			fmt.Fprintln(w, "OK")
+			return
+		default:
+			http.Error(w, "UNKNOWN", http.StatusFailedDependency)
+			return
+		}
+	})
+
+	srv := http.Server{
+		Handler:  mux,
+		ErrorLog: internal.GetFilteredHTTPLogger(),
+	}
+
+	ln, metricsURL, err := internal.SetupListener(s.Config.Bind, s.Config.Network, s.Config.SocketMode)
+	if err != nil {
+		return fmt.Errorf("can't setup listener: %w", err)
+	}
+
+	defer ln.Close()
+
+	if s.Config.TLS != nil {
+		kpr, err := NewKeypairReloader(s.Config.TLS.Certificate, s.Config.TLS.Key, lg)
+		if err != nil {
+			return fmt.Errorf("can't setup keypair reloader: %w", err)
+		}
+
+		srv.TLSConfig = &tls.Config{
+			GetCertificate: kpr.GetCertificate,
+		}
+
+		if s.Config.TLS.CA != "" {
+			caCert, err := os.ReadFile(s.Config.TLS.CA)
+			if err != nil {
+				return fmt.Errorf("%w %s: %w", config.ErrCantReadFile, s.Config.TLS.CA, err)
+			}
+
+			certPool := x509.NewCertPool()
+			if !certPool.AppendCertsFromPEM(caCert) {
+				return fmt.Errorf("%w %s", config.ErrInvalidMetricsCACertificate, s.Config.TLS.CA)
+			}
+
+			srv.TLSConfig.ClientCAs = certPool
+			srv.TLSConfig.ClientAuth = tls.RequireAndVerifyClientCert
+		}
+	}
+
+	lg.Debug("listening for metrics", "url", metricsURL)
+
+	go func() {
+		<-ctx.Done()
+		c, cancel := context.WithTimeout(context.Background(), 5*time.Second)
+		defer cancel()
+		if err := srv.Shutdown(c); err != nil {
+			lg.Error("can't shut down metrics server", "err", err)
+		}
+	}()
+
+	switch s.Config.TLS != nil {
+	case true:
+		if err := srv.ServeTLS(ln, "", ""); !errors.Is(err, http.ErrServerClosed) {
+			return fmt.Errorf("can't serve TLS metrics server: %w", err)
+		}
+	case false:
+		if err := srv.Serve(ln); !errors.Is(err, http.ErrServerClosed) {
+			return fmt.Errorf("can't serve metrics server: %w", err)
+		}
+	}
+
+	return nil
+}

lib/policy/policy.go

@@ -46,6 +46,7 @@ type ParsedConfig struct {
 	DnsCache          *dns.DnsCache
 	Dns               *dns.Dns
 	Logger            *slog.Logger
+	Metrics           *config.Metrics
 }
 
 func newParsedConfig(orig *config.Config) *ParsedConfig {
@@ -53,6 +54,7 @@ func newParsedConfig(orig *config.Config) *ParsedConfig {
 		orig:        orig,
 		OpenGraph:   orig.OpenGraph,
 		StatusCodes: orig.StatusCodes,
+		Metrics:     orig.Metrics,
 	}
 }
 

web/static/robots.txt

@@ -54,6 +54,7 @@ User-agent: meta-externalagent
 User-agent: Meta-ExternalAgent
 User-agent: meta-externalfetcher
 User-agent: Meta-ExternalFetcher
+User-agent: meta-webindexer
 User-agent: MistralAI-User
 User-agent: MistralAI-User/1.0
 User-agent: MyCentralAIScraperBot