feat(web): waste headless chrome bandwidth

Most of the worst of the worst scrapers run Headless Chrome. Headless
Chrome is difficult for Anubis to combat because it follows all the
rules that browsers do. The worst of the worst scrapers also use
residential proxy services. Those residental proxy services charge
upwards of $1 per GB of data egressed or ingressed. The Prompt API makes
Chrome download a 4Gi or 16Gi machine learning model. When you ask it to
start downloading, it will _continue_ downloading even when you leave
the Anubis challenge page.

This will make the local model answer "why is the sky blue?" in an
absurt amount of detail, which wastes both bandwidth and scraper CPU
(some scraping companies charge via Chrome CPU too).

Signed-off-by: Xe Iaso <me@xeiaso.net>

.github/actions/spelling/allow.txt

@@ -38,3 +38,4 @@ Samsung
 wenet
 qwertiko
 setuplistener
+mba

data/botPolicies.yaml

@@ -175,6 +175,13 @@ status_codes:
 #   bind: ":9090"
 #   network: "tcp"
 #
+# # To protect your metrics server with basic auth, set credentials below:
+# #
+# # https://anubis.techaro.lol/docs/admin/policies#http-basic-authentication
+#   basicAuth:
+#     username: ""
+#     password: ""
+#
 # # To serve metrics over TLS, set the path to the right TLS certificate and key
 # # here. When the files change on disk, they will automatically be reloaded.
 # #

docs/docs/CHANGELOG.md

@@ -20,8 +20,14 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 - Fixed mixed tab/space indentation in Caddy documentation code block
 - Improve error messages and fix broken REDIRECT_DOMAINS link in docs ([#1193](https://github.com/TecharoHQ/anubis/issues/1193))
 - Add Bulgarian locale ([#1394](https://github.com/TecharoHQ/anubis/pull/1394))
+- Fixed case-sensitivity mismatch in geoipchecker.go
 - Fix CEL internal errors when iterating `headers`/`query` map wrappers by implementing map iterators for `HTTPHeaders` and `URLValues` ([#1465](https://github.com/TecharoHQ/anubis/pull/1465)).
 - Enable [metrics serving via TLS](./admin/policies.mdx#tls), including [mutual TLS (mTLS)](./admin/policies.mdx#mtls).
+- Enable [HTTP basic auth](./admin/policies.mdx#http-basic-authentication) for the metrics server.
+- Fix a bug in the dataset poisoning maze that could allow denial of service .[#1580](https://github.com/TecharoHQ/anubis/issues/1580).
+- Add config option to add ASN to logs/metrics.
+- Log weight when issuing challenge.
+- Waste bandwidth for headless chrome using the [Prompt API](https://developer.chrome.com/docs/ai/prompt-api).
 
 ## v1.25.0: Necron
 

docs/docs/admin/policies.mdx

@@ -171,6 +171,24 @@ metrics:
 
 As it is not expected for certificate authority certificates to change often, the CA certificate will NOT be automatically reloaded when the respective file changes.
 
+### HTTP basic authentication
+
+Anubis' metrics server also supports setting HTTP basic auth as a lightweight protection against unauthorized users viewing metrics data. As the basic auth credentials are hardcoded in the configuration file, administrators SHOULD use randomly generated credentials, such as type-4 UUIDs or other high entropy strings. These credentials MUST NOT be sensitive or used to protect other high value systems.
+
+Configure it with the `basicAuth` block under `metrics`:
+
+```yaml
+metrics:
+  bind: ":9090"
+  network: "tcp"
+
+  basicAuth:
+    username: azurediamond
+    password: hunter2
+```
+
+If you have Python installed, you can generate a high entropy password with `python -c 'import secrets; print(secrets.token_urlsafe(32))'`.
+
 ## Imprint / Impressum support
 
 Anubis has support for showing imprint / impressum information. This is defined in the `impressum` block of your configuration. See [Imprint / Impressum configuration](./configuration/impressum.mdx) for more information.
@@ -393,6 +411,7 @@ Anubis exposes the following logging settings in the policy file:
 | `level`      | [log level](#log-levels) | `info`          | The logging level threshold. Any logs that are at or above this threshold will be drained to the sink. Any other logs will be discarded. |
 | `sink`       | string                   | `stdio`, `file` | The sink where the logs drain to as they are being recorded in Anubis.                                                                   |
 | `parameters` | object                   |                 | Parameters for the given logging sink. This will vary based on the logging sink of choice. See below for more information.               |
+| `asn`        | bool                     | `true`, `false` | Add ASN information to logs/metrics. (Requires a Thoth client configured)                                                                |
 
 Anubis supports the following logging sinks:
 

internal/basicauth.go

@@ -0,0 +1,52 @@
+package internal
+
+import (
+	"crypto/sha256"
+	"crypto/subtle"
+	"fmt"
+	"log/slog"
+	"net/http"
+)
+
+// BasicAuth wraps next in HTTP Basic authentication using the provided
+// credentials. If either username or password is empty, next is returned
+// unchanged and a debug log line is emitted.
+//
+// Credentials are compared in constant time to avoid leaking information
+// through timing side channels.
+func BasicAuth(realm, username, password string, next http.Handler) http.Handler {
+	if username == "" || password == "" {
+		slog.Debug("skipping middleware, basic auth credentials are empty")
+		return next
+	}
+
+	expectedUser := sha256.Sum256([]byte(username))
+	expectedPass := sha256.Sum256([]byte(password))
+	challenge := fmt.Sprintf("Basic realm=%q, charset=\"UTF-8\"", realm)
+
+	return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		user, pass, ok := r.BasicAuth()
+		if !ok {
+			unauthorized(w, challenge)
+			return
+		}
+
+		gotUser := sha256.Sum256([]byte(user))
+		gotPass := sha256.Sum256([]byte(pass))
+
+		userMatch := subtle.ConstantTimeCompare(gotUser[:], expectedUser[:])
+		passMatch := subtle.ConstantTimeCompare(gotPass[:], expectedPass[:])
+
+		if userMatch&passMatch != 1 {
+			unauthorized(w, challenge)
+			return
+		}
+
+		next.ServeHTTP(w, r)
+	})
+}
+
+func unauthorized(w http.ResponseWriter, challenge string) {
+	w.Header().Set("WWW-Authenticate", challenge)
+	http.Error(w, "Unauthorized", http.StatusUnauthorized)
+}

internal/basicauth_test.go

@@ -0,0 +1,138 @@
+package internal
+
+import (
+	"net/http"
+	"net/http/httptest"
+	"strings"
+	"testing"
+)
+
+func okHandler() http.Handler {
+	return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		w.WriteHeader(http.StatusOK)
+		_, _ = w.Write([]byte("ok"))
+	})
+}
+
+func TestBasicAuth(t *testing.T) {
+	t.Parallel()
+
+	const (
+		realm    = "test-realm"
+		username = "admin"
+		password = "hunter2"
+	)
+
+	for _, tt := range []struct {
+		name       string
+		setAuth    bool
+		user       string
+		pass       string
+		wantStatus int
+		wantBody   string
+		wantChall  bool
+	}{
+		{
+			name:       "valid credentials",
+			setAuth:    true,
+			user:       username,
+			pass:       password,
+			wantStatus: http.StatusOK,
+			wantBody:   "ok",
+		},
+		{
+			name:       "missing credentials",
+			setAuth:    false,
+			wantStatus: http.StatusUnauthorized,
+			wantChall:  true,
+		},
+		{
+			name:       "wrong username",
+			setAuth:    true,
+			user:       "nobody",
+			pass:       password,
+			wantStatus: http.StatusUnauthorized,
+			wantChall:  true,
+		},
+		{
+			name:       "wrong password",
+			setAuth:    true,
+			user:       username,
+			pass:       "wrong",
+			wantStatus: http.StatusUnauthorized,
+			wantChall:  true,
+		},
+		{
+			name:       "empty supplied credentials",
+			setAuth:    true,
+			user:       "",
+			pass:       "",
+			wantStatus: http.StatusUnauthorized,
+			wantChall:  true,
+		},
+	} {
+		t.Run(tt.name, func(t *testing.T) {
+			t.Parallel()
+
+			h := BasicAuth(realm, username, password, okHandler())
+
+			req := httptest.NewRequest(http.MethodGet, "/", nil)
+			if tt.setAuth {
+				req.SetBasicAuth(tt.user, tt.pass)
+			}
+			rec := httptest.NewRecorder()
+			h.ServeHTTP(rec, req)
+
+			if rec.Code != tt.wantStatus {
+				t.Errorf("status = %d, want %d", rec.Code, tt.wantStatus)
+			}
+
+			if tt.wantBody != "" && rec.Body.String() != tt.wantBody {
+				t.Errorf("body = %q, want %q", rec.Body.String(), tt.wantBody)
+			}
+
+			chall := rec.Header().Get("WWW-Authenticate")
+			if tt.wantChall {
+				if chall == "" {
+					t.Error("WWW-Authenticate header missing on 401")
+				}
+				if !strings.Contains(chall, realm) {
+					t.Errorf("WWW-Authenticate = %q, want realm %q", chall, realm)
+				}
+			} else if chall != "" {
+				t.Errorf("unexpected WWW-Authenticate header: %q", chall)
+			}
+		})
+	}
+}
+
+func TestBasicAuthPassthrough(t *testing.T) {
+	t.Parallel()
+
+	for _, tt := range []struct {
+		name     string
+		username string
+		password string
+	}{
+		{name: "empty username", username: "", password: "hunter2"},
+		{name: "empty password", username: "admin", password: ""},
+		{name: "both empty", username: "", password: ""},
+	} {
+		t.Run(tt.name, func(t *testing.T) {
+			t.Parallel()
+
+			h := BasicAuth("realm", tt.username, tt.password, okHandler())
+
+			req := httptest.NewRequest(http.MethodGet, "/", nil)
+			rec := httptest.NewRecorder()
+			h.ServeHTTP(rec, req)
+
+			if rec.Code != http.StatusOK {
+				t.Errorf("status = %d, want %d (passthrough expected)", rec.Code, http.StatusOK)
+			}
+			if rec.Body.String() != "ok" {
+				t.Errorf("body = %q, want %q", rec.Body.String(), "ok")
+			}
+		})
+	}
+}

internal/honeypot/naive/naive.go

@@ -76,13 +76,6 @@ type Impl struct {
 	affirmation, body, title spintax.Spintax
 }
 
-func (i *Impl) incrementUA(ctx context.Context, userAgent string) int {
-	result, _ := i.uaWeight.Get(ctx, internal.SHA256sum(userAgent))
-	result++
-	i.uaWeight.Set(ctx, internal.SHA256sum(userAgent), result, time.Hour)
-	return result
-}
-
 func (i *Impl) incrementNetwork(ctx context.Context, network string) int {
 	result, _ := i.networkWeight.Get(ctx, internal.SHA256sum(network))
 	result++
@@ -90,20 +83,19 @@ func (i *Impl) incrementNetwork(ctx context.Context, network string) int {
 	return result
 }
 
-func (i *Impl) CheckUA() checker.Impl {
-	return checker.Func(func(r *http.Request) (bool, error) {
-		result, _ := i.uaWeight.Get(r.Context(), internal.SHA256sum(r.UserAgent()))
-		if result >= 25 {
-			return true, nil
-		}
-
-		return false, nil
-	})
-}
-
 func (i *Impl) CheckNetwork() checker.Impl {
 	return checker.Func(func(r *http.Request) (bool, error) {
-		result, _ := i.uaWeight.Get(r.Context(), internal.SHA256sum(r.UserAgent()))
+		realIP, _ := internal.RealIP(r)
+		if !realIP.IsValid() {
+			realIP = netip.MustParseAddr(r.Header.Get("X-Real-Ip"))
+		}
+
+		network, ok := internal.ClampIP(realIP)
+		if !ok {
+			return false, nil
+		}
+
+		result, _ := i.networkWeight.Get(r.Context(), internal.SHA256sum(network.String()))
 		if result >= 25 {
 			return true, nil
 		}
@@ -164,7 +156,6 @@ func (i *Impl) ServeHTTP(w http.ResponseWriter, r *http.Request) {
 	}
 
 	networkCount := i.incrementNetwork(r.Context(), network.String())
-	uaCount := i.incrementUA(r.Context(), r.UserAgent())
 
 	stage := r.PathValue("stage")
 
@@ -172,8 +163,8 @@ func (i *Impl) ServeHTTP(w http.ResponseWriter, r *http.Request) {
 		lg.Debug("found new entrance point", "id", id, "stage", stage, "userAgent", r.UserAgent(), "clampedIP", network)
 	} else {
 		switch {
-		case networkCount%256 == 0, uaCount%256 == 0:
-			lg.Warn("found possible crawler", "id", id, "network", network)
+		case networkCount%256 == 0:
+			lg.Warn("found possible crawler", "id", id, "network", network, "userAgent", r.UserAgent())
 		}
 	}
 

lib/anubis.go

@@ -11,6 +11,7 @@ import (
 	"net"
 	"net/http"
 	"net/url"
+	"strconv"
 	"strings"
 	"time"
 
@@ -32,6 +33,7 @@ import (
 	"github.com/TecharoHQ/anubis/lib/policy"
 	"github.com/TecharoHQ/anubis/lib/policy/checker"
 	"github.com/TecharoHQ/anubis/lib/store"
+	iptoasnv1 "github.com/TecharoHQ/thoth-proto/gen/techaro/thoth/iptoasn/v1"
 
 	// challenge implementations
 	_ "github.com/TecharoHQ/anubis/lib/challenge/metarefresh"
@@ -39,31 +41,52 @@ import (
 	_ "github.com/TecharoHQ/anubis/lib/challenge/proofofwork"
 )
 
+type contextKey int
+
+const asnContextKey contextKey = iota
+
+type asnInfo struct {
+	ASN         string
+	Description string
+}
+
+func asnFromContext(ctx context.Context) (string, string) {
+	if v, ok := ctx.Value(asnContextKey).(asnInfo); ok {
+		return v.ASN, v.Description
+	}
+	return "", ""
+}
+
 var (
 	challengesIssued = promauto.NewCounterVec(prometheus.CounterOpts{
 		Name: "anubis_challenges_issued",
 		Help: "The total number of challenges issued",
-	}, []string{"method"})
+	}, []string{"method", "asn", "asn_description"})
 
 	challengesValidated = promauto.NewCounterVec(prometheus.CounterOpts{
 		Name: "anubis_challenges_validated",
 		Help: "The total number of challenges validated",
-	}, []string{"method"})
+	}, []string{"method", "asn", "asn_description"})
 
 	droneBLHits = promauto.NewCounterVec(prometheus.CounterOpts{
 		Name: "anubis_dronebl_hits",
 		Help: "The total number of hits from DroneBL",
-	}, []string{"status"})
+	}, []string{"status", "asn", "asn_description"})
 
 	failedValidations = promauto.NewCounterVec(prometheus.CounterOpts{
 		Name: "anubis_failed_validations",
 		Help: "The total number of failed validations",
-	}, []string{"method"})
+	}, []string{"method", "asn", "asn_description"})
 
 	requestsProxied = promauto.NewCounterVec(prometheus.CounterOpts{
 		Name: "anubis_proxied_requests_total",
 		Help: "Number of requests proxied through Anubis to upstream targets",
-	}, []string{"host"})
+	}, []string{"host", "asn", "asn_description"})
+
+	requestsByASN = promauto.NewCounterVec(prometheus.CounterOpts{
+		Name: "anubis_requests_by_asn_total",
+		Help: "Number of requests by ASN",
+	}, []string{"asn", "asn_description"})
 )
 
 type Server struct {
@@ -78,6 +101,28 @@ type Server struct {
 	hs512Secret []byte
 }
 
+func (s *Server) getRequestLogger(r *http.Request) (*slog.Logger, *http.Request) {
+	lg := internal.GetRequestLogger(s.logger, r)
+
+	if s.policy.LogASN && s.policy.ThothClient != nil {
+		ctx, cancel := context.WithTimeout(r.Context(), 500*time.Millisecond)
+		defer cancel()
+
+		ip := r.Header.Get("X-Real-Ip")
+		if info, err := s.policy.ThothClient.IPToASN.Lookup(ctx, &iptoasnv1.LookupRequest{IpAddress: ip}); err == nil && info.GetAnnounced() {
+			asn := strconv.FormatUint(uint64(info.GetAsNumber()), 10)
+			lg = lg.With("asn", info.GetAsNumber(), "asn_description", info.GetDescription())
+			requestsByASN.WithLabelValues(asn, info.GetDescription()).Inc()
+			r = r.WithContext(context.WithValue(r.Context(), asnContextKey, asnInfo{
+				ASN:         asn,
+				Description: info.GetDescription(),
+			}))
+		}
+	}
+
+	return lg, r
+}
+
 func (s *Server) getTokenKeyfunc() jwt.Keyfunc {
 	// return ED25519 key if HS512 is not set
 	if len(s.hs512Secret) == 0 {
@@ -141,7 +186,7 @@ func (s *Server) issueChallenge(ctx context.Context, r *http.Request, lg *slog.L
 		return nil, err
 	}
 
-	lg.Info("new challenge issued", "challenge", id.String())
+	lg.Info("new challenge issued", "challenge", id.String(), "weight", cr.Weight)
 
 	return &chall, err
 }
@@ -193,7 +238,7 @@ func (s *Server) maybeReverseProxyOrPage(w http.ResponseWriter, r *http.Request)
 }
 
 func (s *Server) maybeReverseProxy(w http.ResponseWriter, r *http.Request, httpStatusOnly bool) {
-	lg := internal.GetRequestLogger(s.logger, r)
+	lg, r := s.getRequestLogger(r)
 
 	if val, _ := s.store.Get(r.Context(), fmt.Sprintf("ogtags:allow:%s%s", r.Host, r.URL.String())); val != nil {
 		lg.Debug("serving opengraph tag asset")
@@ -218,7 +263,10 @@ func (s *Server) maybeReverseProxy(w http.ResponseWriter, r *http.Request, httpS
 	r.Header.Add("X-Anubis-Rule", cr.Name)
 	r.Header.Add("X-Anubis-Action", string(cr.Rule))
 	lg = lg.With("check_result", cr)
-	policy.Applications.WithLabelValues(cr.Name, string(cr.Rule)).Add(1)
+	{
+		asn, asnDesc := asnFromContext(r.Context())
+		policy.Applications.WithLabelValues(cr.Name, string(cr.Rule), asn, asnDesc).Add(1)
+	}
 
 	ip := r.Header.Get("X-Real-Ip")
 
@@ -348,7 +396,8 @@ func (s *Server) handleDNSBL(w http.ResponseWriter, r *http.Request, ip string,
 				lg.Error("can't look up ip in dnsbl", "err", err)
 			}
 			db.Set(r.Context(), ip, resp, 24*time.Hour)
-			droneBLHits.WithLabelValues(resp.String()).Inc()
+			asn, asnDesc := asnFromContext(r.Context())
+			droneBLHits.WithLabelValues(resp.String(), asn, asnDesc).Inc()
 		}
 
 		if resp != dnsbl.AllGood {
@@ -366,7 +415,7 @@ func (s *Server) handleDNSBL(w http.ResponseWriter, r *http.Request, ip string,
 }
 
 func (s *Server) MakeChallenge(w http.ResponseWriter, r *http.Request) {
-	lg := internal.GetRequestLogger(s.logger, r)
+	lg, r := s.getRequestLogger(r)
 	localizer := localization.GetLocalizer(r)
 
 	redir := r.FormValue("redir")
@@ -435,11 +484,14 @@ func (s *Server) MakeChallenge(w http.ResponseWriter, r *http.Request) {
 		return
 	}
 	lg.Debug("made challenge", "challenge", chall, "rules", rule.Challenge, "cr", cr)
-	challengesIssued.WithLabelValues("api").Inc()
+	{
+		asn, asnDesc := asnFromContext(r.Context())
+		challengesIssued.WithLabelValues("api", asn, asnDesc).Inc()
+	}
 }
 
 func (s *Server) PassChallenge(w http.ResponseWriter, r *http.Request) {
-	lg := internal.GetRequestLogger(s.logger, r)
+	lg, r := s.getRequestLogger(r)
 	localizer := localization.GetLocalizer(r)
 
 	redir := r.FormValue("redir")
@@ -530,7 +582,8 @@ func (s *Server) PassChallenge(w http.ResponseWriter, r *http.Request) {
 	}
 
 	if err := impl.Validate(r, lg, in); err != nil {
-		failedValidations.WithLabelValues(rule.Challenge.Algorithm).Inc()
+		asn, asnDesc := asnFromContext(r.Context())
+		failedValidations.WithLabelValues(rule.Challenge.Algorithm, asn, asnDesc).Inc()
 		var cerr *challenge.Error
 		s.ClearCookie(w, CookieOpts{Path: cookiePath, Host: r.Host})
 		lg.Debug("challenge validate call failed", "err", err)
@@ -590,7 +643,10 @@ func (s *Server) PassChallenge(w http.ResponseWriter, r *http.Request) {
 		lg.Debug("can't update information about challenge", "err", err)
 	}
 
-	challengesValidated.WithLabelValues(rule.Challenge.Algorithm).Inc()
+	{
+		asn, asnDesc := asnFromContext(r.Context())
+		challengesValidated.WithLabelValues(rule.Challenge.Algorithm, asn, asnDesc).Inc()
+	}
 	lg.Debug("challenge passed, redirecting to app")
 	http.Redirect(w, r, redir, http.StatusFound)
 }
@@ -629,7 +685,8 @@ func (s *Server) check(r *http.Request, lg *slog.Logger) (policy.CheckResult, *p
 				return cr("bot/"+b.Name, b.Action, weight), &b, nil
 			case config.RuleWeigh:
 				lg.Debug("adjusting weight", "name", b.Name, "delta", b.Weight.Adjust)
-				policy.Applications.WithLabelValues("bot/"+b.Name, "WEIGH").Add(1)
+				asn, asnDesc := asnFromContext(r.Context())
+				policy.Applications.WithLabelValues("bot/"+b.Name, "WEIGH", asn, asnDesc).Add(1)
 				weight += b.Weight.Adjust
 			}
 		}

lib/config.go

@@ -190,14 +190,6 @@ func New(opts Options) (*Server, error) {
 				},
 				Name: "honeypot/network",
 			},
-			policy.Bot{
-				Rules:  mazeGen.CheckUA(),
-				Action: config.RuleWeigh,
-				Weight: &config.Weight{
-					Adjust: 30,
-				},
-				Name: "honeypot/user-agent",
-			},
 		)
 	} else {
 		result.logger.Error("can't init honeypot subsystem", "err", err)

lib/config/logging.go

@@ -17,6 +17,7 @@ type Logging struct {
 	Sink       string             `json:"sink"`       // Logging sink, either "stdio" or "file"
 	Level      *slog.Level        `json:"level"`      // Log level, if set supersedes the level in flags
 	Parameters *LoggingFileConfig `json:"parameters"` // Logging parameters, to be dynamic in the future
+	LogASN     bool               `json:"asn" yaml:"asn"`
 }
 
 const (

lib/config/metrics.go

@@ -10,25 +10,29 @@ import (
 )
 
 var (
-	ErrInvalidMetricsConfig        = errors.New("config: invalid metrics configuration")
-	ErrInvalidMetricsTLSConfig     = errors.New("config: invalid metrics TLS configuration")
-	ErrNoMetricsBind               = errors.New("config.Metrics: must define bind")
-	ErrNoMetricsNetwork            = errors.New("config.Metrics: must define network")
-	ErrNoMetricsSocketMode         = errors.New("config.Metrics: must define socket mode when using unix sockets")
-	ErrInvalidMetricsSocketMode    = errors.New("config.Metrics: invalid unix socket mode")
-	ErrInvalidMetricsNetwork       = errors.New("config.Metrics: invalid metrics network")
-	ErrNoMetricsTLSCertificate     = errors.New("config.Metrics.TLS: must define certificate file")
-	ErrNoMetricsTLSKey             = errors.New("config.Metrics.TLS: must define key file")
-	ErrInvalidMetricsTLSKeypair    = errors.New("config.Metrics.TLS: keypair is invalid")
-	ErrInvalidMetricsCACertificate = errors.New("config.Metrics.TLS: invalid CA certificate")
-	ErrCantReadFile                = errors.New("config: can't read required file")
+	ErrInvalidMetricsConfig          = errors.New("config: invalid metrics configuration")
+	ErrInvalidMetricsTLSConfig       = errors.New("config: invalid metrics TLS configuration")
+	ErrInvalidMetricsBasicAuthConfig = errors.New("config: invalid metrics basic auth configuration")
+	ErrNoMetricsBind                 = errors.New("config.Metrics: must define bind")
+	ErrNoMetricsNetwork              = errors.New("config.Metrics: must define network")
+	ErrNoMetricsSocketMode           = errors.New("config.Metrics: must define socket mode when using unix sockets")
+	ErrInvalidMetricsSocketMode      = errors.New("config.Metrics: invalid unix socket mode")
+	ErrInvalidMetricsNetwork         = errors.New("config.Metrics: invalid metrics network")
+	ErrNoMetricsTLSCertificate       = errors.New("config.Metrics.TLS: must define certificate file")
+	ErrNoMetricsTLSKey               = errors.New("config.Metrics.TLS: must define key file")
+	ErrInvalidMetricsTLSKeypair      = errors.New("config.Metrics.TLS: keypair is invalid")
+	ErrInvalidMetricsCACertificate   = errors.New("config.Metrics.TLS: invalid CA certificate")
+	ErrCantReadFile                  = errors.New("config: can't read required file")
+	ErrNoMetricsBasicAuthUsername    = errors.New("config.Metrics.BasicAuth: must define username")
+	ErrNoMetricsBasicAuthPassword    = errors.New("config.Metrics.BasicAuth: must define password")
 )
 
 type Metrics struct {
-	Bind       string      `json:"bind" yaml:"bind"`
-	Network    string      `json:"network" yaml:"network"`
-	SocketMode string      `json:"socketMode" yaml:"socketMode"`
-	TLS        *MetricsTLS `json:"tls" yaml:"tls"`
+	Bind       string            `json:"bind" yaml:"bind"`
+	Network    string            `json:"network" yaml:"network"`
+	SocketMode string            `json:"socketMode" yaml:"socketMode"`
+	TLS        *MetricsTLS       `json:"tls" yaml:"tls"`
+	BasicAuth  *MetricsBasicAuth `json:"basicAuth" yaml:"basicAuth"`
 }
 
 func (m *Metrics) Valid() error {
@@ -62,6 +66,12 @@ func (m *Metrics) Valid() error {
 		}
 	}
 
+	if m.BasicAuth != nil {
+		if err := m.BasicAuth.Valid(); err != nil {
+			errs = append(errs, err)
+		}
+	}
+
 	if len(errs) != 0 {
 		return errors.Join(ErrInvalidMetricsConfig, errors.Join(errs...))
 	}
@@ -131,3 +141,26 @@ func canReadFile(fname string) error {
 
 	return nil
 }
+
+type MetricsBasicAuth struct {
+	Username string `json:"username" yaml:"username"`
+	Password string `json:"password" yaml:"password"`
+}
+
+func (mba *MetricsBasicAuth) Valid() error {
+	var errs []error
+
+	if mba.Username == "" {
+		errs = append(errs, ErrNoMetricsBasicAuthUsername)
+	}
+
+	if mba.Password == "" {
+		errs = append(errs, ErrNoMetricsBasicAuthPassword)
+	}
+
+	if len(errs) != 0 {
+		return errors.Join(ErrInvalidMetricsBasicAuthConfig, errors.Join(errs...))
+	}
+
+	return nil
+}

lib/config/metrics_test.go

@@ -157,6 +157,79 @@ func TestMetricsValid(t *testing.T) {
 			},
 			err: ErrInvalidMetricsCACertificate,
 		},
+		{
+			name: "basic auth credentials set",
+			input: &Metrics{
+				Bind:    ":9090",
+				Network: "tcp",
+				BasicAuth: &MetricsBasicAuth{
+					Username: "admin",
+					Password: "hunter2",
+				},
+			},
+		},
+		{
+			name: "invalid basic auth config",
+			input: &Metrics{
+				Bind:      ":9090",
+				Network:   "tcp",
+				BasicAuth: &MetricsBasicAuth{},
+			},
+			err: ErrInvalidMetricsBasicAuthConfig,
+		},
+	} {
+		t.Run(tt.name, func(t *testing.T) {
+			if err := tt.input.Valid(); !errors.Is(err, tt.err) {
+				t.Logf("wanted error: %v", tt.err)
+				t.Logf("got error:    %v", err)
+				t.Error("validation failed")
+			}
+		})
+	}
+}
+
+func TestMetricsBasicAuthValid(t *testing.T) {
+	for _, tt := range []struct {
+		name  string
+		input *MetricsBasicAuth
+		err   error
+	}{
+		{
+			name: "both set",
+			input: &MetricsBasicAuth{
+				Username: "admin",
+				Password: "hunter2",
+			},
+		},
+		{
+			name:  "empty username and password",
+			input: &MetricsBasicAuth{},
+			err:   ErrInvalidMetricsBasicAuthConfig,
+		},
+		{
+			name: "missing username",
+			input: &MetricsBasicAuth{
+				Password: "hunter2",
+			},
+			err: ErrNoMetricsBasicAuthUsername,
+		},
+		{
+			name: "missing password",
+			input: &MetricsBasicAuth{
+				Username: "admin",
+			},
+			err: ErrNoMetricsBasicAuthPassword,
+		},
+		{
+			name:  "missing both surfaces wrapper error",
+			input: &MetricsBasicAuth{},
+			err:   ErrNoMetricsBasicAuthUsername,
+		},
+		{
+			name:  "missing both surfaces password error",
+			input: &MetricsBasicAuth{},
+			err:   ErrNoMetricsBasicAuthPassword,
+		},
 	} {
 		t.Run(tt.name, func(t *testing.T) {
 			if err := tt.input.Valid(); !errors.Is(err, tt.err) {

lib/http.go

@@ -207,7 +207,7 @@ func (s *Server) RenderIndex(w http.ResponseWriter, r *http.Request, cr policy.C
 		return
 	}
 
-	lg := internal.GetRequestLogger(s.logger, r)
+	lg, r := s.getRequestLogger(r)
 
 	if !strings.Contains(r.Header.Get("Accept-Encoding"), "gzip") && randomChance(64) {
 		lg.Error("client was given a challenge but does not in fact support gzip compression")
@@ -215,7 +215,10 @@ func (s *Server) RenderIndex(w http.ResponseWriter, r *http.Request, cr policy.C
 		return
 	}
 
-	challengesIssued.WithLabelValues("embedded").Add(1)
+	{
+		asn, asnDesc := asnFromContext(r.Context())
+		challengesIssued.WithLabelValues("embedded", asn, asnDesc).Add(1)
+	}
 	chall, err := s.issueChallenge(r.Context(), r, lg, cr, rule)
 	if err != nil {
 		lg.Error("can't get challenge", "err", err)
@@ -306,14 +309,14 @@ func (s *Server) constructRedirectURL(r *http.Request) (string, error) {
 	case "http", "https":
 		// allowed
 	default:
-		lg := internal.GetRequestLogger(s.logger, r)
+		lg, _ := s.getRequestLogger(r)
 		lg.Warn("invalid protocol in X-Forwarded-Proto", "proto", proto)
 		return "", errors.New(localizer.T("invalid_redirect"))
 	}
 
 	// Check if host is allowed in RedirectDomains (supports '*' via glob)
 	if len(s.opts.RedirectDomains) > 0 && !matchRedirectDomain(s.opts.RedirectDomains, host) {
-		lg := internal.GetRequestLogger(s.logger, r)
+		lg, _ := s.getRequestLogger(r)
 		lg.Debug("domain not allowed", "domain", host)
 		return "", errors.New(localizer.T("redirect_domain_not_allowed"))
 	}
@@ -415,7 +418,7 @@ func (s *Server) ServeHTTPNext(w http.ResponseWriter, r *http.Request) {
 		case "", "http", "https":
 			// allowed: empty scheme means relative URL
 		default:
-			lg := internal.GetRequestLogger(s.logger, r)
+			lg, _ := s.getRequestLogger(r)
 			lg.Warn("XSS attempt blocked, invalid redirect scheme", "scheme", urlParsed.Scheme, "redir", redir)
 			s.respondWithStatus(w, r, localizer.T("invalid_redirect"), "", http.StatusBadRequest)
 			return
@@ -427,7 +430,7 @@ func (s *Server) ServeHTTPNext(w http.ResponseWriter, r *http.Request) {
 		hostMismatch := r.URL.Host != "" && urlParsed.Host != "" && urlParsed.Host != r.URL.Host
 
 		if hostNotAllowed || hostMismatch {
-			lg := internal.GetRequestLogger(s.logger, r)
+			lg, _ := s.getRequestLogger(r)
 			lg.Debug("domain not allowed", "domain", urlParsed.Host)
 			s.respondWithStatus(w, r, localizer.T("redirect_domain_not_allowed"), makeCode(err), http.StatusBadRequest)
 			return
@@ -442,7 +445,8 @@ func (s *Server) ServeHTTPNext(w http.ResponseWriter, r *http.Request) {
 			web.Base(localizer.T("you_are_not_a_bot"), web.StaticHappy(localizer), s.policy.Impressum, localizer),
 		).ServeHTTP(w, r)
 	} else {
-		requestsProxied.WithLabelValues(r.Host).Inc()
+		asn, asnDesc := asnFromContext(r.Context())
+		requestsProxied.WithLabelValues(r.Host, asn, asnDesc).Inc()
 		r = s.stripBasePrefixFromRequest(r)
 		s.next.ServeHTTP(w, r)
 	}

lib/metrics/metrics.go

@@ -64,7 +64,7 @@ func (s *Server) run(ctx context.Context, lg *slog.Logger) error {
 		ErrorLog: internal.GetFilteredHTTPLogger(),
 	}
 
-	ln, metricsURL, err := internal.SetupListener(s.Config.Bind, s.Config.Network, s.Config.SocketMode)
+	ln, metricsURL, err := internal.SetupListener(s.Config.Network, s.Config.Bind, s.Config.SocketMode)
 	if err != nil {
 		return fmt.Errorf("can't setup listener: %w", err)
 	}
@@ -97,6 +97,13 @@ func (s *Server) run(ctx context.Context, lg *slog.Logger) error {
 		}
 	}
 
+	if s.Config.BasicAuth != nil {
+		var h http.Handler = mux
+		h = internal.BasicAuth("anubis-metrics", s.Config.BasicAuth.Username, s.Config.BasicAuth.Password, mux)
+
+		srv.Handler = h
+	}
+
 	lg.Debug("listening for metrics", "url", metricsURL)
 
 	go func() {

lib/policy/policy.go

@@ -27,7 +27,7 @@ var (
 	Applications = promauto.NewCounterVec(prometheus.CounterOpts{
 		Name: "anubis_policy_results",
 		Help: "The results of each policy rule",
-	}, []string{"rule", "action"})
+	}, []string{"rule", "action", "asn", "asn_description"})
 
 	ErrChallengeRuleHasWrongAlgorithm = errors.New("config.Bot.ChallengeRules: algorithm is invalid")
 	warnedAboutThresholds             = &atomic.Bool{}
@@ -47,6 +47,8 @@ type ParsedConfig struct {
 	Dns               *dns.Dns
 	Logger            *slog.Logger
 	Metrics           *config.Metrics
+	ThothClient       *thoth.Client
+	LogASN            bool
 }
 
 func newParsedConfig(orig *config.Config) *ParsedConfig {
@@ -70,6 +72,10 @@ func ParseConfig(ctx context.Context, fin io.Reader, fname string, defaultDiffic
 
 	result := newParsedConfig(c)
 	result.DefaultDifficulty = defaultDifficulty
+	result.LogASN = c.Logging.LogASN
+	if hasThothClient {
+		result.ThothClient = tc
+	}
 
 	if c.Logging.Level != nil {
 		logLevel = c.Logging.Level.String()
@@ -94,6 +100,10 @@ func ParseConfig(ctx context.Context, fin io.Reader, fname string, defaultDiffic
 
 	lg := result.Logger.With("at", "config-validate")
 
+	if result.LogASN && !hasThothClient {
+		lg.Warn("logging.asn is enabled but no Thoth client is configured; ASN logging and metrics will be skipped. Please read https://anubis.techaro.lol/docs/admin/thoth for more information")
+	}
+
 	stFac, ok := store.Get(c.Store.Backend)
 	switch ok {
 	case true:

lib/thoth/geoipchecker.go

@@ -18,7 +18,7 @@ func (c *Client) GeoIPCheckerFor(countries []string) checker.Impl {
 	var sb strings.Builder
 	fmt.Fprintln(&sb, "GeoIPChecker")
 	for _, cc := range countries {
-		countryMap[cc] = struct{}{}
+		countryMap[strings.ToLower(cc)] = struct{}{}
 		fmt.Fprintln(&sb, cc)
 	}
 

web/js/main.ts

@@ -93,12 +93,31 @@ const initTranslations = async () => {
   translations = await loadTranslations(currentLang);
 };
 
+const wasteHeadlessChromeDisk = async () => {
+  if (window.LanguageModel !== undefined) {
+    const session = await window.LanguageModel.create({
+      initialPrompts: [
+        {
+          role: "system",
+          content: "You are a helpful assistant that responds in as many words as possible. Be verbose and answer questions fully with as much detail as possible."
+        },
+        {
+          role: "user",
+          content: "Why is the sky blue?",
+        },
+      ],
+    })
+  }
+};
+
 const t = (key) => translations[`js_${key}`] || translations[key] || key;
 
 (async () => {
   // Initialize translations first
   await initTranslations();
 
+  wasteHeadlessChromeDisk();
+
   const dependencies = [
     {
       name: "Web Workers",