feat(metrics): add naive TLS serving for metrics

Signed-off-by: Xe Iaso <me@xeiaso.net>

.github/actions/spelling/allow.txt

@@ -37,3 +37,4 @@ unipromos
 Samsung
 wenet
 qwertiko
+setuplistener

cmd/anubis/main.go

@@ -17,12 +17,10 @@ import (
 	"net"
 	"net/http"
 	"net/http/httputil"
-	"net/http/pprof"
 	"net/url"
 	"os"
 	"os/signal"
 	"path/filepath"
-	"strconv"
 	"strings"
 	"sync"
 	"syscall"
@@ -33,12 +31,12 @@ import (
 	"github.com/TecharoHQ/anubis/internal"
 	libanubis "github.com/TecharoHQ/anubis/lib"
 	"github.com/TecharoHQ/anubis/lib/config"
+	"github.com/TecharoHQ/anubis/lib/metrics"
 	botPolicy "github.com/TecharoHQ/anubis/lib/policy"
 	"github.com/TecharoHQ/anubis/lib/thoth"
 	"github.com/TecharoHQ/anubis/web"
 	"github.com/facebookgo/flagenv"
 	_ "github.com/joho/godotenv/autoload"
-	"github.com/prometheus/client_golang/prometheus/promhttp"
 	healthv1 "google.golang.org/grpc/health/grpc_health_v1"
 )
 
@@ -119,33 +117,6 @@ func doHealthCheck() error {
 	return nil
 }
 
-// parseBindNetFromAddr determine bind network and address based on the given network and address.
-func parseBindNetFromAddr(address string) (string, string) {
-	defaultScheme := "http://"
-	if !strings.Contains(address, "://") {
-		if strings.HasPrefix(address, ":") {
-			address = defaultScheme + "localhost" + address
-		} else {
-			address = defaultScheme + address
-		}
-	}
-
-	bindUri, err := url.Parse(address)
-	if err != nil {
-		log.Fatal(fmt.Errorf("failed to parse bind URL: %w", err))
-	}
-
-	switch bindUri.Scheme {
-	case "unix":
-		return "unix", bindUri.Path
-	case "tcp", "http", "https":
-		return "tcp", bindUri.Host
-	default:
-		log.Fatal(fmt.Errorf("unsupported network scheme %s in address %s", bindUri.Scheme, address))
-	}
-	return "", address
-}
-
 func parseSameSite(s string) http.SameSite {
 	switch strings.ToLower(s) {
 	case "none":
@@ -162,53 +133,6 @@ func parseSameSite(s string) http.SameSite {
 	return http.SameSiteDefaultMode
 }
 
-func setupListener(network string, address string) (net.Listener, string) {
-	formattedAddress := ""
-
-	if network == "" {
-		// keep compatibility
-		network, address = parseBindNetFromAddr(address)
-	}
-
-	switch network {
-	case "unix":
-		formattedAddress = "unix:" + address
-	case "tcp":
-		if strings.HasPrefix(address, ":") { // assume it's just a port e.g. :4259
-			formattedAddress = "http://localhost" + address
-		} else {
-			formattedAddress = "http://" + address
-		}
-	default:
-		formattedAddress = fmt.Sprintf(`(%s) %s`, network, address)
-	}
-
-	listener, err := net.Listen(network, address)
-	if err != nil {
-		log.Fatal(fmt.Errorf("failed to bind to %s: %w", formattedAddress, err))
-	}
-
-	// additional permission handling for unix sockets
-	if network == "unix" {
-		mode, err := strconv.ParseUint(*socketMode, 8, 0)
-		if err != nil {
-			listener.Close()
-			log.Fatal(fmt.Errorf("could not parse socket mode %s: %w", *socketMode, err))
-		}
-
-		err = os.Chmod(address, os.FileMode(mode))
-		if err != nil {
-			err := listener.Close()
-			if err != nil {
-				log.Printf("failed to close listener: %v", err)
-			}
-			log.Fatal(fmt.Errorf("could not change socket mode: %w", err))
-		}
-	}
-
-	return listener, formattedAddress
-}
-
 func makeReverseProxy(target string, targetSNI string, targetHost string, insecureSkipVerify bool, targetDisableKeepAlive bool) (http.Handler, error) {
 	targetUri, err := url.Parse(target)
 	if err != nil {
@@ -304,11 +228,6 @@ func main() {
 
 	wg := new(sync.WaitGroup)
 
-	if *metricsBind != "" {
-		wg.Add(1)
-		go metricsServer(ctx, *lg.With("subsystem", "metrics"), wg.Done)
-	}
-
 	var rp http.Handler
 	// when using anubis via Systemd and environment variables, then it is not possible to set targe to an empty string but only to space
 	if strings.TrimSpace(*target) != "" {
@@ -348,6 +267,26 @@ func main() {
 	lg.Debug("swapped to new logger")
 	slog.SetDefault(lg)
 
+	if *metricsBind != "" || policy.Metrics != nil {
+		wg.Add(1)
+
+		ms := &metrics.Server{
+			Config: policy.Metrics,
+			Log:    lg,
+		}
+
+		if policy.Metrics == nil {
+			lg.Debug("migrating flags to metrics config", "bind", *metricsBind, "network", *metricsBindNetwork, "socket-mode", *socketMode)
+			ms.Config = &config.Metrics{
+				Bind:       *metricsBind,
+				Network:    *metricsBindNetwork,
+				SocketMode: *socketMode,
+			}
+		}
+
+		go ms.Run(ctx, wg.Done)
+	}
+
 	// Warn if persistent storage is used without a configured signing key
 	if policy.Store.IsPersistent() {
 		if *hs512Secret == "" && *ed25519PrivateKeyHex == "" && *ed25519PrivateKeyHexFile == "" {
@@ -484,7 +423,11 @@ func main() {
 	h = internal.JA4H(h)
 
 	srv := http.Server{Handler: h, ErrorLog: internal.GetFilteredHTTPLogger()}
-	listener, listenerUrl := setupListener(*bindNetwork, *bind)
+	listener, listenerUrl, err := internal.SetupListener(*bindNetwork, *bind, *socketMode)
+	if err != nil {
+		log.Fatalf("SetupListener(%q, %q, %q): %v", *bindNetwork, *bind, *socketMode, err)
+	}
+
 	lg.Info(
 		"listening",
 		"url", listenerUrl,
@@ -519,53 +462,6 @@ func main() {
 	wg.Wait()
 }
 
-func metricsServer(ctx context.Context, lg slog.Logger, done func()) {
-	defer done()
-
-	mux := http.NewServeMux()
-	mux.HandleFunc("GET /debug/pprof/", pprof.Index)
-	mux.HandleFunc("GET /debug/pprof/cmdline", pprof.Cmdline)
-	mux.HandleFunc("GET /debug/pprof/profile", pprof.Profile)
-	mux.HandleFunc("GET /debug/pprof/symbol", pprof.Symbol)
-	mux.HandleFunc("GET /debug/pprof/trace", pprof.Trace)
-	mux.Handle("/metrics", promhttp.Handler())
-	mux.HandleFunc("/healthz", func(w http.ResponseWriter, r *http.Request) {
-		st, ok := internal.GetHealth("anubis")
-		if !ok {
-			slog.Error("health service anubis does not exist, file a bug")
-		}
-
-		switch st {
-		case healthv1.HealthCheckResponse_NOT_SERVING:
-			http.Error(w, "NOT OK", http.StatusInternalServerError)
-			return
-		case healthv1.HealthCheckResponse_SERVING:
-			fmt.Fprintln(w, "OK")
-			return
-		default:
-			http.Error(w, "UNKNOWN", http.StatusFailedDependency)
-			return
-		}
-	})
-
-	srv := http.Server{Handler: mux, ErrorLog: internal.GetFilteredHTTPLogger()}
-	listener, metricsUrl := setupListener(*metricsBindNetwork, *metricsBind)
-	lg.Debug("listening for metrics", "url", metricsUrl)
-
-	go func() {
-		<-ctx.Done()
-		c, cancel := context.WithTimeout(context.Background(), 5*time.Second)
-		defer cancel()
-		if err := srv.Shutdown(c); err != nil {
-			log.Printf("cannot shut down: %v", err)
-		}
-	}()
-
-	if err := srv.Serve(listener); !errors.Is(err, http.ErrServerClosed) {
-		log.Fatal(err)
-	}
-}
-
 func extractEmbedFS(fsys embed.FS, root string, destDir string) error {
 	return fs.WalkDir(fsys, root, func(path string, d fs.DirEntry, err error) error {
 		if err != nil {

data/botPolicies.yaml

@@ -166,6 +166,15 @@ status_codes:
   CHALLENGE: 200
   DENY: 200
 
+# # Configuration for the metrics server. See the docs for more information:
+# #
+# # https://anubis.techaro.lol/docs/admin/policies#metrics-server
+# #
+# # This is commented out by default so that command line flags take precedence.
+# metrics:
+#   bind: ":9090"
+#   network: "tcp"
+
 # Anubis can store temporary data in one of a few backends. See the storage
 # backends section of the docs for more information:
 #

data/crawlers/ai-search.yaml

@@ -4,5 +4,5 @@
 #  - Claude-SearchBot: No published IP allowlist
 - name: "ai-crawlers-search"
   user_agent_regex: >-
-    OAI-SearchBot|Claude-SearchBot|PerplexityBot
+    OAI-SearchBot|Claude-SearchBot|PerplexityBot|meta-webindexer
   action: DENY

docs/docs/CHANGELOG.md

@@ -11,14 +11,15 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 
 ## [Unreleased]
 
+<!-- This changes the project to: -->
+
+- Move metrics server configuration to [the policy file](./admin/policies.mdx#metrics-server).
 - Expose [pprof endpoints](https://pkg.go.dev/net/http/pprof) on the metrics listener to enable profiling Anubis in production.
 - fix: prevent nil pointer panic in challenge validation when threshold rules match during PassChallenge (#1463)
 - Instruct reverse proxies to not cache error pages.
 - Fixed mixed tab/space indentation in Caddy documentation code block
 - Improve error messages and fix broken REDIRECT_DOMAINS link in docs ([#1193](https://github.com/TecharoHQ/anubis/issues/1193))
 - Add Bulgarian locale ([#1394](https://github.com/TecharoHQ/anubis/pull/1394))
-
-<!-- This changes the project to: -->
 - Fix CEL internal errors when iterating `headers`/`query` map wrappers by implementing map iterators for `HTTPHeaders` and `URLValues` ([#1465](https://github.com/TecharoHQ/anubis/pull/1465)).
 
 ## v1.25.0: Necron

docs/docs/admin/installation.mdx

@@ -87,15 +87,15 @@ Anubis uses these environment variables for configuration:
 | `ED25519_PRIVATE_KEY_HEX_FILE` | unset                   | Path to a file containing the hex-encoded ed25519 private key. Only one of this or its sister option may be set. **Required when using persistent storage backends** (like bbolt) to ensure challenges survive service restarts. When running multiple instances on the same base domain, the key must be the same across all instances.                                                                                                                                                                                                       |
 | `ERROR_TITLE`                  | unset                   | <EO /> If set, override the translation stack to show a custom title for error pages such as "Something went wrong!". See [Customizing messages](./botstopper.mdx#customizing-messages) for more details.                                                                                                                                                                                                                                                                                                                                      |
 | `JWT_RESTRICTION_HEADER`       | `X-Real-IP`             | If set, the JWT is only valid if the current value of this header matches the value when the JWT was created. You can use it e.g. to restrict a JWT to the source IP of the user using `X-Real-IP`.                                                                                                                                                                                                                                                                                                                                            |
-| `METRICS_BIND`                 | `:9090`                 | The network address that Anubis serves Prometheus metrics on. See `BIND` for more information.                                                                                                                                                                                                                                                                                                                                                                                                                                                 |
-| `METRICS_BIND_NETWORK`         | `tcp`                   | The address family that the Anubis metrics server listens on. See `BIND_NETWORK` for more information.                                                                                                                                                                                                                                                                                                                                                                                                                                         |
+| `METRICS_BIND`                 | `:9090`                 | The legacy configuration value for the network address that Anubis serves Prometheus metrics on. Please migrate this to [the policy file](./policies.mdx#metrics-server) as soon as possible.                                                                                                                                                                                                                                                                                                                                                  |
+| `METRICS_BIND_NETWORK`         | `tcp`                   | The legacy configuration value for the address family that Anubis serves Prometheus metrics on. Please migrate this to [the policy file](./policies.mdx#metrics-server) as soon as possible.                                                                                                                                                                                                                                                                                                                                                   |
 | `OG_EXPIRY_TIME`               | `24h`                   | The expiration time for the Open Graph tag cache. Prefer using [the policy file](./configuration/open-graph.mdx) to configure the Open Graph subsystem.                                                                                                                                                                                                                                                                                                                                                                                        |
 | `OG_PASSTHROUGH`               | `false`                 | If set to `true`, Anubis will enable Open Graph tag passthrough. Prefer using [the policy file](./configuration/open-graph.mdx) to configure the Open Graph subsystem.                                                                                                                                                                                                                                                                                                                                                                         |
 | `OG_CACHE_CONSIDER_HOST`       | `false`                 | If set to `true`, Anubis will consider the host in the Open Graph tag cache key. Prefer using [the policy file](./configuration/open-graph.mdx) to configure the Open Graph subsystem.                                                                                                                                                                                                                                                                                                                                                         |
 | `OVERLAY_FOLDER`               | unset                   | <EO /> If set, treat the given path as an [overlay folder](./botstopper.mdx#custom-images-and-css), allowing you to customize CSS, fonts, images, and add other assets to BotStopper deployments.                                                                                                                                                                                                                                                                                                                                              |
 | `POLICY_FNAME`                 | unset                   | The file containing [bot policy configuration](./policies.mdx). See the bot policy documentation for more details. If unset, the default bot policy configuration is used.                                                                                                                                                                                                                                                                                                                                                                     |
 | `PUBLIC_URL`                   | unset                   | The externally accessible URL for this Anubis instance, used for constructing redirect URLs (e.g., for Traefik forwardAuth). Leave it unset when Anubis terminates traffic directly (sidecar/standalone deployments) or redirect building will fail with `redir=null`.                                                                                                                                                                                                                                                                         |
-| `REDIRECT_DOMAINS`             | unset                   | Comma-separated list of domain names that Anubis should allow redirects to when passing a challenge. See [Redirect Domain Configuration](./configuration/redirect-domains.mdx) for more details.                                                                                                                                                                                                                                                                                                                                                   |
+| `REDIRECT_DOMAINS`             | unset                   | Comma-separated list of domain names that Anubis should allow redirects to when passing a challenge. See [Redirect Domain Configuration](./configuration/redirect-domains.mdx) for more details.                                                                                                                                                                                                                                                                                                                                               |
 | `SERVE_ROBOTS_TXT`             | `false`                 | If set `true`, Anubis will serve a default `robots.txt` file that disallows all known AI scrapers by name and then additionally disallows every scraper. This is useful if facts and circumstances make it difficult to change the underlying service to serve such a `robots.txt` file.                                                                                                                                                                                                                                                       |
 | `SLOG_LEVEL`                   | `INFO`                  | The log level for structured logging. Valid values are `DEBUG`, `INFO`, `WARN`, and `ERROR`. Set to `DEBUG` to see all requests, evaluations, and detailed diagnostic information.                                                                                                                                                                                                                                                                                                                                                             |
 | `SOCKET_MODE`                  | `0770`                  | _Only used when at least one of the `*_BIND_NETWORK` variables are set to `unix`._ The socket mode (permissions) for Unix domain sockets.                                                                                                                                                                                                                                                                                                                                                                                                      |

docs/docs/admin/policies.mdx

@@ -117,6 +117,27 @@ remote_addresses:
   - 100.64.0.0/10
 ```
 
+## Metrics server
+
+Anubis includes support for [Prometheus-style metrics](https://prometheus.io/docs/introduction/overview/), allowing systems administrators to monitor Anubis' performance and effectiveness. This is a separate HTTP server with metrics, health checking, and debug routes.
+
+Anubis' metrics server is configured with the `metrics` block in the configuration file:
+
+```yaml
+metrics:
+  bind: ":9090"
+  network: "tcp"
+```
+
+If you want to bind metrics to a Unix socket, make sure to set the network to `unix` and add a socket mode:
+
+```yaml
+metrics:
+  bind: "/tmp/anubis_metrics.sock"
+  network: unix
+  socketMode: "0700" # must be a string
+```
+
 ## Imprint / Impressum support
 
 Anubis has support for showing imprint / impressum information. This is defined in the `impressum` block of your configuration. See [Imprint / Impressum configuration](./configuration/impressum.mdx) for more information.

internal/setuplistener.go

@@ -0,0 +1,92 @@
+package internal
+
+import (
+	"errors"
+	"fmt"
+	"net"
+	"net/url"
+	"os"
+	"strconv"
+	"strings"
+)
+
+// parseBindNetFromAddr determine bind network and address based on the given network and address.
+func parseBindNetFromAddr(address string) (string, string, error) {
+	defaultScheme := "http://"
+	if !strings.Contains(address, "://") {
+		if strings.HasPrefix(address, ":") {
+			address = defaultScheme + "localhost" + address
+		} else {
+			address = defaultScheme + address
+		}
+	}
+
+	bindUri, err := url.Parse(address)
+	if err != nil {
+		return "", "", fmt.Errorf("failed to parse bind URL: %w", err)
+	}
+
+	switch bindUri.Scheme {
+	case "unix":
+		return "unix", bindUri.Path, nil
+	case "tcp", "http", "https":
+		return "tcp", bindUri.Host, nil
+	default:
+		return "", "", fmt.Errorf("unsupported network scheme %s in address %s", bindUri.Scheme, address)
+	}
+}
+
+// SetupListener sets up a network listener based on the input from configuration
+// envvars. It returns a network listener and the URL to that listener or an error.
+func SetupListener(network, address, socketMode string) (net.Listener, string, error) {
+	formattedAddress := ""
+	var err error
+
+	if network == "" {
+		// keep compatibility
+		network, address, err = parseBindNetFromAddr(address)
+	}
+
+	if err != nil {
+		return nil, "", fmt.Errorf("can't parse bind and network: %w", err)
+	}
+
+	switch network {
+	case "unix":
+		formattedAddress = "unix:" + address
+	case "tcp":
+		if strings.HasPrefix(address, ":") { // assume it's just a port e.g. :4259
+			formattedAddress = "http://localhost" + address
+		} else {
+			formattedAddress = "http://" + address
+		}
+	default:
+		formattedAddress = fmt.Sprintf(`(%s) %s`, network, address)
+	}
+
+	ln, err := net.Listen(network, address)
+	if err != nil {
+		return nil, "", fmt.Errorf("failed to bind to %s: %w", formattedAddress, err)
+	}
+
+	// additional permission handling for unix sockets
+	if network == "unix" {
+		mode, err := strconv.ParseUint(socketMode, 8, 0)
+		if err != nil {
+			ln.Close()
+			return nil, "", fmt.Errorf("could not parse socket mode %s: %w", socketMode, err)
+		}
+
+		err = os.Chmod(address, os.FileMode(mode))
+		if err != nil {
+			err := fmt.Errorf("could not change socket mode: %w", err)
+			clErr := ln.Close()
+			if clErr != nil {
+				return nil, "", errors.Join(err, clErr)
+			}
+			return nil, "", err
+		}
+	}
+
+	return ln, formattedAddress, nil
+}

internal/setuplistener_test.go

@@ -0,0 +1,180 @@
+package internal
+
+import (
+	"io/fs"
+	"os"
+	"path/filepath"
+	"strconv"
+	"strings"
+	"testing"
+)
+
+func TestParseBindNetFromAddr(t *testing.T) {
+	for _, tt := range []struct {
+		name    string
+		address string
+		wantErr bool
+		network string
+		bind    string
+	}{
+		{
+			name:    "simple tcp",
+			address: "localhost:9090",
+			wantErr: false,
+			network: "tcp",
+			bind:    "localhost:9090",
+		},
+		{
+			name:    "simple unix",
+			address: "unix:///tmp/foo.sock",
+			wantErr: false,
+			network: "unix",
+			bind:    "/tmp/foo.sock",
+		},
+		{
+			name:    "invalid network",
+			address: "foo:///tmp/bar.sock",
+			wantErr: true,
+		},
+		{
+			name:    "tcp uri",
+			address: "tcp://[::]:9090",
+			wantErr: false,
+			network: "tcp",
+			bind:    "[::]:9090",
+		},
+		{
+			name:    "http uri",
+			address: "http://[::]:9090",
+			wantErr: false,
+			network: "tcp",
+			bind:    "[::]:9090",
+		},
+		{
+			name:    "https uri",
+			address: "https://[::]:9090",
+			wantErr: false,
+			network: "tcp",
+			bind:    "[::]:9090",
+		},
+	} {
+		t.Run(tt.name, func(t *testing.T) {
+			network, bind, err := parseBindNetFromAddr(tt.address)
+
+			switch {
+			case tt.wantErr && err == nil:
+				t.Errorf("parseBindNetFromAddr(%q) should have errored but did not", tt.address)
+			case !tt.wantErr && err != nil:
+				t.Errorf("parseBindNetFromAddr(%q) threw an error: %v", tt.address, err)
+			}
+
+			if network != tt.network {
+				t.Errorf("parseBindNetFromAddr(%q) wanted network: %q, got: %q", tt.address, tt.network, network)
+			}
+
+			if bind != tt.bind {
+				t.Errorf("parseBindNetFromAddr(%q) wanted bind: %q, got: %q", tt.address, tt.bind, bind)
+			}
+		})
+	}
+}
+
+func TestSetupListener(t *testing.T) {
+	td := t.TempDir()
+
+	for _, tt := range []struct {
+		name                         string
+		network, address, socketMode string
+		wantErr                      bool
+		socketURLPrefix              string
+	}{
+		{
+			name:            "simple tcp",
+			network:         "",
+			address:         ":0",
+			wantErr:         false,
+			socketURLPrefix: "http://localhost:",
+		},
+		{
+			name:            "simple unix",
+			network:         "",
+			address:         "unix://" + filepath.Join(td, "a"),
+			socketMode:      "0770",
+			wantErr:         false,
+			socketURLPrefix: "unix:" + filepath.Join(td, "a"),
+		},
+		{
+			name:            "tcp",
+			network:         "tcp",
+			address:         ":0",
+			wantErr:         false,
+			socketURLPrefix: "http://localhost:",
+		},
+		{
+			name:            "udp",
+			network:         "udp",
+			address:         ":0",
+			wantErr:         true,
+			socketURLPrefix: "http://localhost:",
+		},
+		{
+			name:            "unix socket",
+			network:         "unix",
+			socketMode:      "0770",
+			address:         filepath.Join(td, "a"),
+			wantErr:         false,
+			socketURLPrefix: "unix:" + filepath.Join(td, "a"),
+		},
+		{
+			name:            "invalid socket mode",
+			network:         "unix",
+			socketMode:      "taco bell",
+			address:         filepath.Join(td, "a"),
+			wantErr:         true,
+			socketURLPrefix: "unix:" + filepath.Join(td, "a"),
+		},
+		{
+			name:            "empty socket mode",
+			network:         "unix",
+			socketMode:      "",
+			address:         filepath.Join(td, "a"),
+			wantErr:         true,
+			socketURLPrefix: "unix:" + filepath.Join(td, "a"),
+		},
+	} {
+		t.Run(tt.name, func(t *testing.T) {
+			ln, socketURL, err := SetupListener(tt.network, tt.address, tt.socketMode)
+			switch {
+			case tt.wantErr && err == nil:
+				t.Errorf("SetupListener(%q, %q, %q) should have errored but did not", tt.network, tt.address, tt.socketMode)
+			case !tt.wantErr && err != nil:
+				t.Fatalf("SetupListener(%q, %q, %q) threw an error: %v", tt.network, tt.address, tt.socketMode, err)
+			}
+
+			if ln != nil {
+				defer ln.Close()
+			}
+
+			if !tt.wantErr && !strings.HasPrefix(socketURL, tt.socketURLPrefix) {
+				t.Errorf("SetupListener(%q, %q, %q) should have returned a URL with prefix %q but got: %q", tt.network, tt.address, tt.socketMode, tt.socketURLPrefix, socketURL)
+			}
+
+			if tt.socketMode != "" {
+				mode, err := strconv.ParseUint(tt.socketMode, 8, 0)
+				if err != nil {
+					return
+				}
+
+				sockPath := strings.TrimPrefix(socketURL, "unix:")
+				st, err := os.Stat(sockPath)
+				if err != nil {
+					t.Fatalf("can't os.Stat(%q): %v", sockPath, err)
+				}
+
+				if st.Mode().Perm() != fs.FileMode(mode) {
+					t.Errorf("file mode of %q should be %s but is actually %s", sockPath, strconv.FormatUint(mode, 8), strconv.FormatUint(uint64(st.Mode()), 8))
+				}
+			}
+		})
+	}
+}

lib/config/config.go

@@ -334,6 +334,7 @@ type fileConfig struct {
 	DNSBL       bool                `json:"dnsbl"`
 	DNSTTL      DnsTTL              `json:"dns_ttl"`
 	Logging     *Logging            `json:"logging"`
+	Metrics     *Metrics            `json:"metrics,omitempty"`
 }
 
 func (c *fileConfig) Valid() error {
@@ -375,6 +376,12 @@ func (c *fileConfig) Valid() error {
 		}
 	}
 
+	if c.Metrics != nil {
+		if err := c.Metrics.Valid(); err != nil {
+			errs = append(errs, err)
+		}
+	}
+
 	if len(errs) != 0 {
 		return fmt.Errorf("config is not valid:\n%w", errors.Join(errs...))
 	}
@@ -417,6 +424,7 @@ func Load(fin io.Reader, fname string) (*Config, error) {
 		StatusCodes: c.StatusCodes,
 		Store:       c.Store,
 		Logging:     c.Logging,
+		Metrics:     c.Metrics,
 	}
 
 	if c.OpenGraph.TimeToLive != "" {
@@ -508,6 +516,7 @@ type Config struct {
 	Logging     *Logging
 	DNSBL       bool
 	DNSTTL      DnsTTL
+	Metrics     *Metrics
 }
 
 func (c Config) Valid() error {

lib/config/metrics.go

@@ -0,0 +1,118 @@
+package config
+
+import (
+	"crypto/tls"
+	"errors"
+	"fmt"
+	"os"
+	"strconv"
+)
+
+var (
+	ErrInvalidMetricsConfig     = errors.New("config: invalid metrics configuration")
+	ErrInvalidMetricsTLSConfig  = errors.New("config: invalid metrics TLS configuration")
+	ErrNoMetricsBind            = errors.New("config.Metrics: must define bind")
+	ErrNoMetricsNetwork         = errors.New("config.Metrics: must define network")
+	ErrNoMetricsSocketMode      = errors.New("config.Metrics: must define socket mode when using unix sockets")
+	ErrInvalidMetricsSocketMode = errors.New("config.Metrics: invalid unix socket mode")
+	ErrInvalidMetricsNetwork    = errors.New("config.Metrics: invalid metrics network")
+	ErrNoMetricsTLSCertificate  = errors.New("config.Metrics.TLS: must define certificate file")
+	ErrNoMetricsTLSKey          = errors.New("config.Metrics.TLS: must define key file")
+	ErrInvalidMetricsTLSKeypair = errors.New("config.Metrics.TLS: keypair is invalid")
+	ErrCantReadFile             = errors.New("config: can't read required file")
+)
+
+type Metrics struct {
+	Bind       string      `json:"bind" yaml:"bind"`
+	Network    string      `json:"network" yaml:"network"`
+	SocketMode string      `json:"socketMode" yaml:"socketMode"`
+	TLS        *MetricsTLS `json:"tls" yaml:"tls"`
+}
+
+func (m *Metrics) Valid() error {
+	var errs []error
+
+	if m.Bind == "" {
+		errs = append(errs, ErrNoMetricsBind)
+	}
+
+	if m.Network == "" {
+		errs = append(errs, ErrNoMetricsNetwork)
+	}
+
+	switch m.Network {
+	case "tcp", "tcp4", "tcp6": // https://pkg.go.dev/net#Listen
+	case "unix":
+		if m.SocketMode == "" {
+			errs = append(errs, ErrNoMetricsSocketMode)
+		}
+
+		if _, err := strconv.ParseUint(m.SocketMode, 8, 0); err != nil {
+			errs = append(errs, fmt.Errorf("%w: %w", ErrInvalidMetricsSocketMode, err))
+		}
+	default:
+		errs = append(errs, ErrInvalidMetricsNetwork)
+	}
+
+	if m.TLS != nil {
+		if err := m.TLS.Valid(); err != nil {
+			errs = append(errs, err)
+		}
+	}
+
+	if len(errs) != 0 {
+		return errors.Join(ErrInvalidMetricsConfig, errors.Join(errs...))
+	}
+
+	return nil
+}
+
+type MetricsTLS struct {
+	Certificate string `json:"certificate" yaml:"certificate"`
+	Key         string `json:"key" yaml:"key"`
+}
+
+func (mt *MetricsTLS) Valid() error {
+	var errs []error
+
+	if mt.Certificate == "" {
+		errs = append(errs, ErrNoMetricsTLSCertificate)
+	}
+
+	if err := canReadFile(mt.Certificate); err != nil {
+		errs = append(errs, fmt.Errorf("%w %s: %w", ErrCantReadFile, mt.Certificate, err))
+	}
+
+	if mt.Key == "" {
+		errs = append(errs, ErrNoMetricsTLSKey)
+	}
+
+	if err := canReadFile(mt.Key); err != nil {
+		errs = append(errs, fmt.Errorf("%w %s: %w", ErrCantReadFile, mt.Key, err))
+	}
+
+	if _, err := tls.LoadX509KeyPair(mt.Certificate, mt.Key); err != nil {
+		errs = append(errs, fmt.Errorf("%w: %w", ErrInvalidMetricsTLSKeypair, err))
+	}
+
+	if len(errs) != 0 {
+		return errors.Join(ErrInvalidMetricsTLSConfig, errors.Join(errs...))
+	}
+
+	return nil
+}
+
+func canReadFile(fname string) error {
+	fin, err := os.Open(fname)
+	if err != nil {
+		return err
+	}
+	defer fin.Close()
+
+	data := make([]byte, 64)
+	if _, err := fin.Read(data); err != nil {
+		return fmt.Errorf("can't read %s: %w", fname, err)
+	}
+
+	return nil
+}

lib/config/metrics_test.go

@@ -0,0 +1,131 @@
+package config
+
+import (
+	"errors"
+	"testing"
+)
+
+func TestMetricsValid(t *testing.T) {
+	for _, tt := range []struct {
+		name  string
+		input *Metrics
+		err   error
+	}{
+		{
+			name: "basic TCP",
+			input: &Metrics{
+				Bind:    ":9090",
+				Network: "tcp",
+			},
+		},
+		{
+			name: "basic TCP4",
+			input: &Metrics{
+				Bind:    ":9090",
+				Network: "tcp4",
+			},
+		},
+		{
+			name: "basic TCP6",
+			input: &Metrics{
+				Bind:    ":9090",
+				Network: "tcp6",
+			},
+		},
+		{
+			name: "basic unix",
+			input: &Metrics{
+				Bind:       "/tmp/anubis-metrics.sock",
+				Network:    "unix",
+				SocketMode: "0770",
+			},
+		},
+		{
+			name:  "no bind",
+			input: &Metrics{},
+			err:   ErrNoMetricsBind,
+		},
+		{
+			name:  "no network",
+			input: &Metrics{},
+			err:   ErrNoMetricsNetwork,
+		},
+		{
+			name: "no unix socket mode",
+			input: &Metrics{
+				Bind:    "/tmp/anubis-metrics.sock",
+				Network: "unix",
+			},
+			err: ErrNoMetricsSocketMode,
+		},
+		{
+			name: "invalid unix socket mode",
+			input: &Metrics{
+				Bind:       "/tmp/anubis-metrics.sock",
+				Network:    "unix",
+				SocketMode: "taco bell",
+			},
+			err: ErrInvalidMetricsSocketMode,
+		},
+		{
+			name: "invalid network",
+			input: &Metrics{
+				Bind:    ":9090",
+				Network: "taco",
+			},
+			err: ErrInvalidMetricsNetwork,
+		},
+		{
+			name: "invalid TLS config",
+			input: &Metrics{
+				Bind:    ":9090",
+				Network: "tcp",
+				TLS:     &MetricsTLS{},
+			},
+			err: ErrInvalidMetricsTLSConfig,
+		},
+		{
+			name: "selfsigned TLS cert",
+			input: &Metrics{
+				Bind:    ":9090",
+				Network: "tcp",
+				TLS: &MetricsTLS{
+					Certificate: "./testdata/tls/selfsigned.crt",
+					Key:         "./testdata/tls/selfsigned.key",
+				},
+			},
+		},
+		{
+			name: "wrong path to selfsigned TLS cert",
+			input: &Metrics{
+				Bind:    ":9090",
+				Network: "tcp",
+				TLS: &MetricsTLS{
+					Certificate: "./testdata/tls2/selfsigned.crt",
+					Key:         "./testdata/tls2/selfsigned.key",
+				},
+			},
+			err: ErrCantReadFile,
+		},
+		{
+			name: "unparseable TLS cert",
+			input: &Metrics{
+				Bind:    ":9090",
+				Network: "tcp",
+				TLS: &MetricsTLS{
+					Certificate: "./testdata/tls/invalid.crt",
+					Key:         "./testdata/tls/invalid.key",
+				},
+			},
+			err: ErrInvalidMetricsTLSKeypair,
+		},
+	} {
+		t.Run(tt.name, func(t *testing.T) {
+			if err := tt.input.Valid(); !errors.Is(err, tt.err) {
+				t.Logf("wanted error: %v", tt.err)
+				t.Logf("got error:    %v", err)
+				t.Error("validation failed")
+			}
+		})
+	}
+}

lib/config/testdata/bad/metrics-invalid-net.yaml

@@ -0,0 +1,3 @@
+metrics:
+  bind: ":9090"
+  network: taco

lib/config/testdata/good/allow_everyone.json

@@ -5,5 +5,9 @@
       "remote_addresses": ["0.0.0.0/0", "::/0"],
       "action": "ALLOW"
     }
-  ]
+  ],
+  "metrics": {
+    "bind": ":9090",
+    "network": "tcp"
+  }
 }

lib/config/testdata/good/allow_everyone.yaml

@@ -4,3 +4,7 @@ bots:
       - "0.0.0.0/0"
       - "::/0"
     action: ALLOW
+
+metrics:
+  bind: ":9090"
+  network: "tcp"

lib/config/testdata/good/block_cf_workers.json

@@ -8,5 +8,9 @@
       "action": "DENY"
     }
   ],
-  "dnsbl": false
+  "dnsbl": false,
+  "metrics": {
+    "bind": ":9090",
+    "network": "tcp"
+  }
 }

lib/config/testdata/good/block_cf_workers.yaml

@@ -3,3 +3,7 @@ bots:
     headers_regex:
       CF-Worker: .*
     action: DENY
+
+metrics:
+  bind: ":9090"
+  network: "tcp"

lib/config/testdata/good/challenge_cloudflare.yaml

@@ -4,3 +4,7 @@ bots:
     asns:
       match:
         - 13335 # Cloudflare
+
+metrics:
+  bind: ":9090"
+  network: "tcp"

lib/config/testdata/good/challengemozilla.json

@@ -5,5 +5,9 @@
       "user_agent_regex": "Mozilla",
       "action": "CHALLENGE"
     }
-  ]
+  ],
+  "metrics": {
+    "bind": ":9090",
+    "network": "tcp"
+  }
 }

lib/config/testdata/good/challengemozilla.yaml

@@ -2,3 +2,7 @@ bots:
   - name: generic-browser
     user_agent_regex: Mozilla
     action: CHALLENGE
+
+metrics:
+  bind: ":9090"
+  network: "tcp"

lib/config/testdata/good/dns-ttl-custom.yaml

@@ -6,3 +6,7 @@ bots:
   - name: "test"
     user_agent_regex: ".*"
     action: "DENY"
+
+metrics:
+  bind: ":9090"
+  network: "tcp"

lib/config/testdata/good/entropy.yaml

@@ -6,3 +6,7 @@ bots:
         - '"Accept" in headers'
         - headers["Accept"].contains("text/html")
         - randInt(1) == 0
+
+metrics:
+  bind: ":9090"
+  network: "tcp"

lib/config/testdata/good/everything_blocked.json

@@ -6,5 +6,9 @@
       "action": "DENY"
     }
   ],
-  "dnsbl": false
+  "dnsbl": false,
+  "metrics": {
+    "bind": ":9090",
+    "network": "tcp"
+  }
 }

lib/config/testdata/good/everything_blocked.yaml

@@ -2,3 +2,7 @@ bots:
   - name: everything
     user_agent_regex: .*
     action: DENY
+
+metrics:
+  bind: ":9090"
+  network: "tcp"

lib/config/testdata/good/geoip_us.yaml

@@ -4,3 +4,7 @@ bots:
     geoip:
       countries:
         - US
+
+metrics:
+  bind: ":9090"
+  network: "tcp"

lib/config/testdata/good/git_client.json

@@ -10,5 +10,9 @@
         ]
       }
     }
-  ]
+  ],
+  "metrics": {
+    "bind": ":9090",
+    "network": "tcp"
+  }
 }

lib/config/testdata/good/git_client.yaml

@@ -6,3 +6,7 @@ bots:
         - userAgent.startsWith("git/") || userAgent.contains("libgit")
         - >
           "Git-Protocol" in headers && headers["Git-Protocol"] == "version=2"
+
+metrics:
+  bind: ":9090"
+  network: "tcp"

lib/config/testdata/good/import_filesystem.json

@@ -3,5 +3,9 @@
     {
       "import": "./testdata/hack-test.json"
     }
-  ]
+  ],
+  "metrics": {
+    "bind": ":9090",
+    "network": "tcp"
+  }
 }

lib/config/testdata/good/import_filesystem.yaml

@@ -1,2 +1,6 @@
 bots:
   - import: ./testdata/hack-test.yaml
+
+metrics:
+  bind: ":9090"
+  network: "tcp"

lib/config/testdata/good/import_keep_internet_working.json

@@ -3,5 +3,9 @@
     {
       "import": "(data)/common/keep-internet-working.yaml"
     }
-  ]
+  ],
+  "metrics": {
+    "bind": ":9090",
+    "network": "tcp"
+  }
 }

lib/config/testdata/good/import_keep_internet_working.yaml

@@ -1,2 +1,6 @@
 bots:
   - import: (data)/common/keep-internet-working.yaml
+
+metrics:
+  bind: ":9090"
+  network: "tcp"

lib/config/testdata/good/impressum.yaml

@@ -8,3 +8,7 @@ impressum:
   page:
     title: Test
     body: <p>This is a test</p>
+
+metrics:
+  bind: ":9090"
+  network: "tcp"

lib/config/testdata/good/logging-file.yaml

@@ -13,3 +13,7 @@ logs:
     oldFileTimeFormat: 2006-01-02T15-04-05 # RFC 3339-ish
     compress: true
     useLocalTime: false # timezone for rotated files is UTC
+
+metrics:
+  bind: ":9090"
+  network: "tcp"

lib/config/testdata/good/logging-stdio.yaml

@@ -5,3 +5,7 @@ bots:
 
 logging:
   sink: "stdio"
+
+metrics:
+  bind: ":9090"
+  network: "tcp"

lib/config/testdata/good/no-thresholds.yaml

@@ -6,3 +6,7 @@ bots:
       adjust: 5
 
 thresholds: []
+
+metrics:
+  bind: ":9090"
+  network: "tcp"

lib/config/testdata/good/old_xesite.json

@@ -75,5 +75,9 @@
       "user_agent_regex": "Mozilla",
       "action": "CHALLENGE"
     }
-  ]
+  ],
+  "metrics": {
+    "bind": ":9090",
+    "network": "tcp"
+  }
 }

lib/config/testdata/good/opengraph_all_good.yaml

@@ -10,3 +10,7 @@ openGraph:
   default:
     "og:title": "Xe's magic land of fun"
     "og:description": "We're no strangers to love, you know the rules and so do I"
+
+metrics:
+  bind: ":9090"
+  network: "tcp"

lib/config/testdata/good/simple-weight.yaml

@@ -4,3 +4,7 @@ bots:
     user_agent_regex: Mozilla
     weight:
       adjust: 5
+
+metrics:
+  bind: ":9090"
+  network: "tcp"

lib/config/testdata/good/status-codes-paranoid.json

@@ -9,5 +9,9 @@
   "status_codes": {
     "CHALLENGE": 200,
     "DENY": 200
+  },
+  "metrics": {
+    "bind": ":9090",
+    "network": "tcp"
   }
 }

lib/config/testdata/good/status-codes-paranoid.yaml

@@ -6,3 +6,7 @@ bots:
 status_codes:
   CHALLENGE: 200
   DENY: 200
+
+metrics:
+  bind: ":9090"
+  network: "tcp"

lib/config/testdata/good/status-codes-rfc.json

@@ -9,5 +9,9 @@
   "status_codes": {
     "CHALLENGE": 403,
     "DENY": 403
+  },
+  "metrics": {
+    "bind": ":9090",
+    "network": "tcp"
   }
 }

lib/config/testdata/good/status-codes-rfc.yaml

@@ -6,3 +6,7 @@ bots:
 status_codes:
   CHALLENGE: 403
   DENY: 403
+
+metrics:
+  bind: ":9090"
+  network: "tcp"

lib/config/testdata/good/thresholds.yaml

@@ -33,3 +33,7 @@ thresholds:
     challenge:
       algorithm: fast
       difficulty: 4
+
+metrics:
+  bind: ":9090"
+  network: "tcp"

lib/config/testdata/good/weight-no-weight.yaml

@@ -2,3 +2,7 @@ bots:
   - name: weight
     action: WEIGH
     user_agent_regex: Mozilla
+
+metrics:
+  bind: ":9090"
+  network: "tcp"

lib/config/testdata/tls/selfsigned.crt

@@ -0,0 +1,11 @@
+-----BEGIN CERTIFICATE-----
+MIIBnzCCAVGgAwIBAgIUK39B3Ft+kU5o81IuISs79O4u1ncwBQYDK2VwMEUxCzAJ
+BgNVBAYTAkFVMRMwEQYDVQQIDApTb21lLVN0YXRlMSEwHwYDVQQKDBhJbnRlcm5l
+dCBXaWRnaXRzIFB0eSBMdGQwHhcNMjYwNDIyMTQyNjE4WhcNMjYwNTIyMTQyNjE4
+WjBFMQswCQYDVQQGEwJBVTETMBEGA1UECAwKU29tZS1TdGF0ZTEhMB8GA1UECgwY
+SW50ZXJuZXQgV2lkZ2l0cyBQdHkgTHRkMCowBQYDK2VwAyEAfgpAUpp8MIOOdQpH
+fxaw3R7mFKQRMR6Kmxzk1Xn/2VujUzBRMB0GA1UdDgQWBBSmkBmzo0RiZ2iocMR8
+uIIpz9cZyTAfBgNVHSMEGDAWgBSmkBmzo0RiZ2iocMR8uIIpz9cZyTAPBgNVHRMB
+Af8EBTADAQH/MAUGAytlcANBAG37XXZrVUUzGyy3T9qsPIzvJQAGpGhdjJ7bt9O6
+sBhzrliTONPrudYuyUggWsHgFb0JlN2xs4/2HhKU+PY7AAQ=
+-----END CERTIFICATE-----

lib/config/testdata/tls/selfsigned.key

@@ -0,0 +1,3 @@
+-----BEGIN PRIVATE KEY-----
+MC4CAQAwBQYDK2VwBCIEIL0HxjjfVlg6zQPB9/zTLq0IBzfp8gEoifEYzQZYIj+T
+-----END PRIVATE KEY-----

lib/metrics/metrics.go

@@ -0,0 +1,88 @@
+package metrics
+
+import (
+	"context"
+	"errors"
+	"fmt"
+	"log/slog"
+	"net/http"
+	"net/http/pprof"
+	"time"
+
+	"github.com/TecharoHQ/anubis/internal"
+	"github.com/TecharoHQ/anubis/lib/config"
+	"github.com/prometheus/client_golang/prometheus/promhttp"
+	healthv1 "google.golang.org/grpc/health/grpc_health_v1"
+)
+
+type Server struct {
+	Config *config.Metrics
+	Log    *slog.Logger
+}
+
+func (s *Server) Run(ctx context.Context, done func()) error {
+	defer done()
+	lg := s.Log.With("subsystem", "metrics")
+
+	mux := http.NewServeMux()
+	mux.HandleFunc("GET /debug/pprof/", pprof.Index)
+	mux.HandleFunc("GET /debug/pprof/cmdline", pprof.Cmdline)
+	mux.HandleFunc("GET /debug/pprof/profile", pprof.Profile)
+	mux.HandleFunc("GET /debug/pprof/symbol", pprof.Symbol)
+	mux.HandleFunc("GET /debug/pprof/trace", pprof.Trace)
+	mux.Handle("/metrics", promhttp.Handler())
+	mux.HandleFunc("/healthz", func(w http.ResponseWriter, r *http.Request) {
+		st, ok := internal.GetHealth("anubis")
+		if !ok {
+			slog.Error("health service anubis does not exist, file a bug")
+		}
+
+		switch st {
+		case healthv1.HealthCheckResponse_NOT_SERVING:
+			http.Error(w, "NOT OK", http.StatusInternalServerError)
+			return
+		case healthv1.HealthCheckResponse_SERVING:
+			fmt.Fprintln(w, "OK")
+			return
+		default:
+			http.Error(w, "UNKNOWN", http.StatusFailedDependency)
+			return
+		}
+	})
+
+	srv := http.Server{
+		Handler:  mux,
+		ErrorLog: internal.GetFilteredHTTPLogger(),
+	}
+
+	ln, metricsURL, err := internal.SetupListener(s.Config.Bind, s.Config.Network, s.Config.SocketMode)
+	if err != nil {
+		return fmt.Errorf("can't setup listener: %w", err)
+	}
+
+	defer ln.Close()
+
+	lg.Debug("listening for metrics", "url", metricsURL)
+
+	go func() {
+		<-ctx.Done()
+		c, cancel := context.WithTimeout(context.Background(), 5*time.Second)
+		defer cancel()
+		if err := srv.Shutdown(c); err != nil {
+			lg.Error("can't shut down metrics server", "err", err)
+		}
+	}()
+
+	switch s.Config.TLS != nil {
+	case true:
+		if err := srv.ServeTLS(ln, s.Config.TLS.Certificate, s.Config.TLS.Key); !errors.Is(err, http.ErrServerClosed) {
+			return fmt.Errorf("can't serve TLS metrics server: %w", err)
+		}
+	case false:
+		if err := srv.Serve(ln); !errors.Is(err, http.ErrServerClosed) {
+			return fmt.Errorf("can't serve metrics server: %w", err)
+		}
+	}
+
+	return nil
+}

lib/policy/policy.go

@@ -46,6 +46,7 @@ type ParsedConfig struct {
 	DnsCache          *dns.DnsCache
 	Dns               *dns.Dns
 	Logger            *slog.Logger
+	Metrics           *config.Metrics
 }
 
 func newParsedConfig(orig *config.Config) *ParsedConfig {
@@ -53,6 +54,7 @@ func newParsedConfig(orig *config.Config) *ParsedConfig {
 		orig:        orig,
 		OpenGraph:   orig.OpenGraph,
 		StatusCodes: orig.StatusCodes,
+		Metrics:     orig.Metrics,
 	}
 }
 

web/static/robots.txt

@@ -54,6 +54,7 @@ User-agent: meta-externalagent
 User-agent: Meta-ExternalAgent
 User-agent: meta-externalfetcher
 User-agent: Meta-ExternalFetcher
+User-agent: meta-webindexer
 User-agent: MistralAI-User
 User-agent: MistralAI-User/1.0
 User-agent: MyCentralAIScraperBot