feat(prometheus): add metrics to Subsonic API and Plugins (#4266)

* Add prometheus metrics to subsonic and plugins * address feedback, do not log error if operation is not supported * add missing timestamp and client to stats * remove .view from subsonic route * directly inject DataStore in Prometheus, to avoid having to pass it in every call Signed-off-by: Deluan <deluan@navidrome.org> --------- Signed-off-by: Deluan <deluan@navidrome.org> Co-authored-by: Deluan <deluan@navidrome.org>
2025-06-28 02:13:57 +00:00
parent 709714cfc0
commit 0cd15c1ddc
22 changed files with 246 additions and 89 deletions
@@ -2,7 +2,6 @@ package metrics

 import (
 	"context"
-	"fmt"
 	"net/http"
 	"strconv"
 	"sync"
@@ -13,6 +12,7 @@ import (
 	"github.com/navidrome/navidrome/consts"
 	"github.com/navidrome/navidrome/log"
 	"github.com/navidrome/navidrome/model"
+	"github.com/navidrome/navidrome/utils/singleton"
 	"github.com/prometheus/client_golang/prometheus"
 	"github.com/prometheus/client_golang/prometheus/promhttp"
 )
@@ -20,6 +20,8 @@ import (
 type Metrics interface {
 	WriteInitialMetrics(ctx context.Context)
 	WriteAfterScanMetrics(ctx context.Context, success bool)
+	RecordRequest(ctx context.Context, endpoint, method, client string, status int, elapsed int64)
+	RecordPluginRequest(ctx context.Context, plugin, method string, ok bool, elapsed int64)
 	GetHandler() http.Handler
 }

@@ -27,11 +29,14 @@ type metrics struct {
 	ds model.DataStore
 }

-func NewPrometheusInstance(ds model.DataStore) Metrics {
-	if conf.Server.Prometheus.Enabled {
-		return &metrics{ds: ds}
+func GetPrometheusInstance(ds model.DataStore) Metrics {
+	if !conf.Server.Prometheus.Enabled {
+		return noopMetrics{}
 	}
-	return noopMetrics{}
+
+	return singleton.GetInstance(func() *metrics {
+		return &metrics{ds: ds}
+	})
 }

 func NewNoopInstance() Metrics {
@@ -51,6 +56,38 @@ func (m *metrics) WriteAfterScanMetrics(ctx context.Context, success bool) {
 	getPrometheusMetrics().mediaScansCounter.With(scanLabels).Inc()
 }

+func (m *metrics) RecordRequest(_ context.Context, endpoint, method, client string, status int, elapsed int64) {
+	httpLabel := prometheus.Labels{
+		"endpoint": endpoint,
+		"method":   method,
+		"client":   client,
+		"status":   strconv.FormatInt(int64(status), 10),
+	}
+	getPrometheusMetrics().httpRequestCounter.With(httpLabel).Inc()
+
+	httpLatencyLabel := prometheus.Labels{
+		"endpoint": endpoint,
+		"method":   method,
+		"client":   client,
+	}
+	getPrometheusMetrics().httpRequestDuration.With(httpLatencyLabel).Observe(float64(elapsed))
+}
+
+func (m *metrics) RecordPluginRequest(_ context.Context, plugin, method string, ok bool, elapsed int64) {
+	pluginLabel := prometheus.Labels{
+		"plugin": plugin,
+		"method": method,
+		"ok":     strconv.FormatBool(ok),
+	}
+	getPrometheusMetrics().pluginRequestCounter.With(pluginLabel).Inc()
+
+	pluginLatencyLabel := prometheus.Labels{
+		"plugin": plugin,
+		"method": method,
+	}
+	getPrometheusMetrics().pluginRequestDuration.With(pluginLatencyLabel).Observe(float64(elapsed))
+}
+
 func (m *metrics) GetHandler() http.Handler {
 	r := chi.NewRouter()

@@ -59,20 +96,31 @@ func (m *metrics) GetHandler() http.Handler {
 			consts.PrometheusAuthUser: conf.Server.Prometheus.Password,
 		}))
 	}
-	r.Handle("/", promhttp.Handler())

+	// Enable created at timestamp to handle zero counter on create.
+	// This requires --enable-feature=created-timestamp-zero-ingestion to be passed in Prometheus
+	r.Handle("/", promhttp.HandlerFor(prometheus.DefaultGatherer, promhttp.HandlerOpts{
+		EnableOpenMetrics:                   true,
+		EnableOpenMetricsTextCreatedSamples: true,
+	}))
 	return r
 }

 type prometheusMetrics struct {
-	dbTotal           *prometheus.GaugeVec
-	versionInfo       *prometheus.GaugeVec
-	lastMediaScan     *prometheus.GaugeVec
-	mediaScansCounter *prometheus.CounterVec
+	dbTotal               *prometheus.GaugeVec
+	versionInfo           *prometheus.GaugeVec
+	lastMediaScan         *prometheus.GaugeVec
+	mediaScansCounter     *prometheus.CounterVec
+	httpRequestCounter    *prometheus.CounterVec
+	httpRequestDuration   *prometheus.SummaryVec
+	pluginRequestCounter  *prometheus.CounterVec
+	pluginRequestDuration *prometheus.SummaryVec
 }

 // Prometheus' metrics requires initialization. But not more than once
 var getPrometheusMetrics = sync.OnceValue(func() *prometheusMetrics {
+	quartilesToEstimate := map[float64]float64{0.5: 0.05, 0.75: 0.025, 0.9: 0.01, 0.99: 0.001}
+
 	instance := &prometheusMetrics{
 		dbTotal: prometheus.NewGaugeVec(
 			prometheus.GaugeOpts{
@@ -102,23 +150,49 @@ var getPrometheusMetrics = sync.OnceValue(func() *prometheusMetrics {
 			},
 			[]string{"success"},
 		),
+		httpRequestCounter: prometheus.NewCounterVec(
+			prometheus.CounterOpts{
+				Name: "http_request_count",
+				Help: "Request types by status",
+			},
+			[]string{"endpoint", "method", "client", "status"},
+		),
+		httpRequestDuration: prometheus.NewSummaryVec(
+			prometheus.SummaryOpts{
+				Name:       "http_request_latency",
+				Help:       "Latency (in ms) of HTTP requests",
+				Objectives: quartilesToEstimate,
+			},
+			[]string{"endpoint", "method", "client"},
+		),
+		pluginRequestCounter: prometheus.NewCounterVec(
+			prometheus.CounterOpts{
+				Name: "plugin_request_count",
+				Help: "Plugin requests by method/status",
+			},
+			[]string{"plugin", "method", "ok"},
+		),
+		pluginRequestDuration: prometheus.NewSummaryVec(
+			prometheus.SummaryOpts{
+				Name:       "plugin_request_latency",
+				Help:       "Latency (in ms) of plugin requests",
+				Objectives: quartilesToEstimate,
+			},
+			[]string{"plugin", "method"},
+		),
 	}
-	err := prometheus.DefaultRegisterer.Register(instance.dbTotal)
-	if err != nil {
-		log.Fatal("Unable to create Prometheus metric instance", fmt.Errorf("unable to register db_model_totals metrics: %w", err))
-	}
-	err = prometheus.DefaultRegisterer.Register(instance.versionInfo)
-	if err != nil {
-		log.Fatal("Unable to create Prometheus metric instance", fmt.Errorf("unable to register navidrome_info metrics: %w", err))
-	}
-	err = prometheus.DefaultRegisterer.Register(instance.lastMediaScan)
-	if err != nil {
-		log.Fatal("Unable to create Prometheus metric instance", fmt.Errorf("unable to register media_scan_last metrics: %w", err))
-	}
-	err = prometheus.DefaultRegisterer.Register(instance.mediaScansCounter)
-	if err != nil {
-		log.Fatal("Unable to create Prometheus metric instance", fmt.Errorf("unable to register media_scans metrics: %w", err))
-	}
+
+	prometheus.DefaultRegisterer.MustRegister(
+		instance.dbTotal,
+		instance.versionInfo,
+		instance.lastMediaScan,
+		instance.mediaScansCounter,
+		instance.httpRequestCounter,
+		instance.httpRequestDuration,
+		instance.pluginRequestCounter,
+		instance.pluginRequestDuration,
+	)
+
 	return instance
 })

@@ -159,4 +233,8 @@ func (n noopMetrics) WriteInitialMetrics(context.Context) {}

 func (n noopMetrics) WriteAfterScanMetrics(context.Context, bool) {}

+func (n noopMetrics) RecordRequest(context.Context, string, string, string, int, int64) {}
+
+func (n noopMetrics) RecordPluginRequest(context.Context, string, string, bool, int64) {}
+
 func (n noopMetrics) GetHandler() http.Handler { return nil }