feat(metrics): add naive TLS serving for metrics

Signed-off-by: Xe Iaso <me@xeiaso.net>

data/crawlers/ai-search.yaml

@@ -4,5 +4,5 @@
 #  - Claude-SearchBot: No published IP allowlist
 - name: "ai-crawlers-search"
   user_agent_regex: >-
-    OAI-SearchBot|Claude-SearchBot|PerplexityBot
+    OAI-SearchBot|Claude-SearchBot|PerplexityBot|meta-webindexer
   action: DENY

lib/config/metrics.go

@@ -1,24 +1,32 @@
 package config
 
 import (
+	"crypto/tls"
 	"errors"
 	"fmt"
+	"os"
 	"strconv"
 )
 
 var (
 	ErrInvalidMetricsConfig     = errors.New("config: invalid metrics configuration")
+	ErrInvalidMetricsTLSConfig  = errors.New("config: invalid metrics TLS configuration")
 	ErrNoMetricsBind            = errors.New("config.Metrics: must define bind")
 	ErrNoMetricsNetwork         = errors.New("config.Metrics: must define network")
 	ErrNoMetricsSocketMode      = errors.New("config.Metrics: must define socket mode when using unix sockets")
 	ErrInvalidMetricsSocketMode = errors.New("config.Metrics: invalid unix socket mode")
 	ErrInvalidMetricsNetwork    = errors.New("config.Metrics: invalid metrics network")
+	ErrNoMetricsTLSCertificate  = errors.New("config.Metrics.TLS: must define certificate file")
+	ErrNoMetricsTLSKey          = errors.New("config.Metrics.TLS: must define key file")
+	ErrInvalidMetricsTLSKeypair = errors.New("config.Metrics.TLS: keypair is invalid")
+	ErrCantReadFile             = errors.New("config: can't read required file")
 )
 
 type Metrics struct {
-	Bind       string `json:"bind" yaml:"bind"`
-	Network    string `json:"network" yaml:"network"`
-	SocketMode string `json:"socketMode" yaml:"socketMode"`
+	Bind       string      `json:"bind" yaml:"bind"`
+	Network    string      `json:"network" yaml:"network"`
+	SocketMode string      `json:"socketMode" yaml:"socketMode"`
+	TLS        *MetricsTLS `json:"tls" yaml:"tls"`
 }
 
 func (m *Metrics) Valid() error {
@@ -46,9 +54,65 @@ func (m *Metrics) Valid() error {
 		errs = append(errs, ErrInvalidMetricsNetwork)
 	}
 
+	if m.TLS != nil {
+		if err := m.TLS.Valid(); err != nil {
+			errs = append(errs, err)
+		}
+	}
+
 	if len(errs) != 0 {
 		return errors.Join(ErrInvalidMetricsConfig, errors.Join(errs...))
 	}
 
 	return nil
 }
+
+type MetricsTLS struct {
+	Certificate string `json:"certificate" yaml:"certificate"`
+	Key         string `json:"key" yaml:"key"`
+}
+
+func (mt *MetricsTLS) Valid() error {
+	var errs []error
+
+	if mt.Certificate == "" {
+		errs = append(errs, ErrNoMetricsTLSCertificate)
+	}
+
+	if err := canReadFile(mt.Certificate); err != nil {
+		errs = append(errs, fmt.Errorf("%w %s: %w", ErrCantReadFile, mt.Certificate, err))
+	}
+
+	if mt.Key == "" {
+		errs = append(errs, ErrNoMetricsTLSKey)
+	}
+
+	if err := canReadFile(mt.Key); err != nil {
+		errs = append(errs, fmt.Errorf("%w %s: %w", ErrCantReadFile, mt.Key, err))
+	}
+
+	if _, err := tls.LoadX509KeyPair(mt.Certificate, mt.Key); err != nil {
+		errs = append(errs, fmt.Errorf("%w: %w", ErrInvalidMetricsTLSKeypair, err))
+	}
+
+	if len(errs) != 0 {
+		return errors.Join(ErrInvalidMetricsTLSConfig, errors.Join(errs...))
+	}
+
+	return nil
+}
+
+func canReadFile(fname string) error {
+	fin, err := os.Open(fname)
+	if err != nil {
+		return err
+	}
+	defer fin.Close()
+
+	data := make([]byte, 64)
+	if _, err := fin.Read(data); err != nil {
+		return fmt.Errorf("can't read %s: %w", fname, err)
+	}
+
+	return nil
+}

lib/config/metrics_test.go

@@ -75,6 +75,50 @@ func TestMetricsValid(t *testing.T) {
 			},
 			err: ErrInvalidMetricsNetwork,
 		},
+		{
+			name: "invalid TLS config",
+			input: &Metrics{
+				Bind:    ":9090",
+				Network: "tcp",
+				TLS:     &MetricsTLS{},
+			},
+			err: ErrInvalidMetricsTLSConfig,
+		},
+		{
+			name: "selfsigned TLS cert",
+			input: &Metrics{
+				Bind:    ":9090",
+				Network: "tcp",
+				TLS: &MetricsTLS{
+					Certificate: "./testdata/tls/selfsigned.crt",
+					Key:         "./testdata/tls/selfsigned.key",
+				},
+			},
+		},
+		{
+			name: "wrong path to selfsigned TLS cert",
+			input: &Metrics{
+				Bind:    ":9090",
+				Network: "tcp",
+				TLS: &MetricsTLS{
+					Certificate: "./testdata/tls2/selfsigned.crt",
+					Key:         "./testdata/tls2/selfsigned.key",
+				},
+			},
+			err: ErrCantReadFile,
+		},
+		{
+			name: "unparseable TLS cert",
+			input: &Metrics{
+				Bind:    ":9090",
+				Network: "tcp",
+				TLS: &MetricsTLS{
+					Certificate: "./testdata/tls/invalid.crt",
+					Key:         "./testdata/tls/invalid.key",
+				},
+			},
+			err: ErrInvalidMetricsTLSKeypair,
+		},
 	} {
 		t.Run(tt.name, func(t *testing.T) {
 			if err := tt.input.Valid(); !errors.Is(err, tt.err) {

lib/config/testdata/tls/selfsigned.crt

@@ -0,0 +1,11 @@
+-----BEGIN CERTIFICATE-----
+MIIBnzCCAVGgAwIBAgIUK39B3Ft+kU5o81IuISs79O4u1ncwBQYDK2VwMEUxCzAJ
+BgNVBAYTAkFVMRMwEQYDVQQIDApTb21lLVN0YXRlMSEwHwYDVQQKDBhJbnRlcm5l
+dCBXaWRnaXRzIFB0eSBMdGQwHhcNMjYwNDIyMTQyNjE4WhcNMjYwNTIyMTQyNjE4
+WjBFMQswCQYDVQQGEwJBVTETMBEGA1UECAwKU29tZS1TdGF0ZTEhMB8GA1UECgwY
+SW50ZXJuZXQgV2lkZ2l0cyBQdHkgTHRkMCowBQYDK2VwAyEAfgpAUpp8MIOOdQpH
+fxaw3R7mFKQRMR6Kmxzk1Xn/2VujUzBRMB0GA1UdDgQWBBSmkBmzo0RiZ2iocMR8
+uIIpz9cZyTAfBgNVHSMEGDAWgBSmkBmzo0RiZ2iocMR8uIIpz9cZyTAPBgNVHRMB
+Af8EBTADAQH/MAUGAytlcANBAG37XXZrVUUzGyy3T9qsPIzvJQAGpGhdjJ7bt9O6
+sBhzrliTONPrudYuyUggWsHgFb0JlN2xs4/2HhKU+PY7AAQ=
+-----END CERTIFICATE-----

lib/config/testdata/tls/selfsigned.key

@@ -0,0 +1,3 @@
+-----BEGIN PRIVATE KEY-----
+MC4CAQAwBQYDK2VwBCIEIL0HxjjfVlg6zQPB9/zTLq0IBzfp8gEoifEYzQZYIj+T
+-----END PRIVATE KEY-----

lib/metrics/metrics.go

@@ -73,8 +73,15 @@ func (s *Server) Run(ctx context.Context, done func()) error {
 		}
 	}()
 
-	if err := srv.Serve(ln); !errors.Is(err, http.ErrServerClosed) {
-		return fmt.Errorf("can't serve metrics server: %w", err)
+	switch s.Config.TLS != nil {
+	case true:
+		if err := srv.ServeTLS(ln, s.Config.TLS.Certificate, s.Config.TLS.Key); !errors.Is(err, http.ErrServerClosed) {
+			return fmt.Errorf("can't serve TLS metrics server: %w", err)
+		}
+	case false:
+		if err := srv.Serve(ln); !errors.Is(err, http.ErrServerClosed) {
+			return fmt.Errorf("can't serve metrics server: %w", err)
+		}
 	}
 
 	return nil

web/static/robots.txt

@@ -54,6 +54,7 @@ User-agent: meta-externalagent
 User-agent: Meta-ExternalAgent
 User-agent: meta-externalfetcher
 User-agent: Meta-ExternalFetcher
+User-agent: meta-webindexer
 User-agent: MistralAI-User
 User-agent: MistralAI-User/1.0
 User-agent: MyCentralAIScraperBot