Optimize search3, by removing OFFSET when paginating (#2655)

* Optimize pagination, removing offset

* For search, don't add `where` clause for empty queries

* Revert "Replace `COUNT(DISTINCT primary_key)` with `COUNT(*)`"

Genres are required as part of the count queries, so filter by genres work

* Optimize search3 query, using order by id if it is a "" query.

Also fix the optimizePagination query logic

* Allow offset optimizer threshold to be configured

conf/configuration.go

@@ -96,6 +96,7 @@ type configOptions struct {
 	DevSidebarPlaylists              bool
 	DevEnableBufferedScrobble        bool
 	DevShowArtistPage                bool
+	DevOffsetOptimize                int
 	DevArtworkMaxRequests            int
 	DevArtworkThrottleBacklogLimit   int
 	DevArtworkThrottleBacklogTimeout time.Duration
@@ -352,6 +353,7 @@ func init() {
 	viper.SetDefault("devenablebufferedscrobble", true)
 	viper.SetDefault("devsidebarplaylists", true)
 	viper.SetDefault("devshowartistpage", true)
+	viper.SetDefault("devoffsetoptimize", 50000)
 	viper.SetDefault("devartworkmaxrequests", number.Max(2, runtime.NumCPU()/3))
 	viper.SetDefault("devartworkthrottlebackloglimit", consts.RequestThrottleBacklogLimit)
 	viper.SetDefault("devartworkthrottlebacklogtimeout", consts.RequestThrottleBacklogTimeout)

persistence/mediafile_repository.go

@@ -91,7 +91,7 @@ func (r *mediaFileRepository) Get(id string) (*model.MediaFile, error) {
 func (r *mediaFileRepository) GetAll(options ...model.QueryOptions) (model.MediaFiles, error) {
 	sq := r.selectMediaFile(options...)
 	res := model.MediaFiles{}
-	err := r.queryAll(sq, &res)
+	err := r.queryAll(sq, &res, options...)
 	if err != nil {
 		return nil, err
 	}

persistence/sql_base_repository.go

@@ -10,6 +10,7 @@ import (
 	. "github.com/Masterminds/squirrel"
 	"github.com/beego/beego/v2/client/orm"
 	"github.com/google/uuid"
+	"github.com/navidrome/navidrome/conf"
 	"github.com/navidrome/navidrome/log"
 	"github.com/navidrome/navidrome/model"
 	"github.com/navidrome/navidrome/model/request"
@@ -157,7 +158,10 @@ func (r sqlRepository) queryOne(sq Sqlizer, response interface{}) error {
 	return err
 }
 
-func (r sqlRepository) queryAll(sq Sqlizer, response interface{}) error {
+func (r sqlRepository) queryAll(sq SelectBuilder, response interface{}, options ...model.QueryOptions) error {
+	if len(options) > 0 && options[0].Offset > 0 {
+		sq = r.optimizePagination(sq, options[0])
+	}
 	query, args, err := sq.ToSql()
 	if err != nil {
 		return err
@@ -172,6 +176,19 @@ func (r sqlRepository) queryAll(sq Sqlizer, response interface{}) error {
 	return err
 }
 
+// optimizePagination uses a less inefficient pagination, by not using OFFSET.
+// See https://gist.github.com/ssokolow/262503
+func (r sqlRepository) optimizePagination(sq SelectBuilder, options model.QueryOptions) SelectBuilder {
+	if options.Offset > conf.Server.DevOffsetOptimize {
+		sq = sq.RemoveOffset()
+		oidSq := sq.RemoveColumns().Columns(r.tableName + ".oid")
+		oidSq = oidSq.Limit(uint64(options.Offset))
+		oidSql, args, _ := oidSq.ToSql()
+		sq = sq.Where(r.tableName+".oid not in ("+oidSql+")", args...)
+	}
+	return sq
+}
+
 func (r sqlRepository) exists(existsQuery SelectBuilder) (bool, error) {
 	existsQuery = existsQuery.Columns("count(*) as exist").From(r.tableName)
 	var res struct{ Exist int64 }

persistence/sql_search.go

@@ -5,6 +5,7 @@ import (
 
 	. "github.com/Masterminds/squirrel"
 	"github.com/navidrome/navidrome/conf"
+	"github.com/navidrome/navidrome/model"
 	"github.com/navidrome/navidrome/utils"
 )
 
@@ -21,21 +22,31 @@ func (r sqlRepository) doSearch(q string, offset, size int, results interface{},
 	}
 
 	sq := r.newSelectWithAnnotation(r.tableName + ".id").Columns("*")
-	sq = sq.Limit(uint64(size)).Offset(uint64(offset))
+	filter := fullTextExpr(q)
-	if len(orderBys) > 0 {
+	if filter != nil {
-		sq = sq.OrderBy(orderBys...)
+		sq = sq.Where(filter)
+		if len(orderBys) > 0 {
+			sq = sq.OrderBy(orderBys...)
+		}
+	} else {
+		// If the filter is empty, we sort by id.
+		// This is to speed up the results of `search3?query=""`, for OpenSubsonic
+		sq = sq.OrderBy("id")
 	}
-	sq = sq.Where(fullTextExpr(q))
+	sq = sq.Limit(uint64(size)).Offset(uint64(offset))
-	err := r.queryAll(sq, results)
+	err := r.queryAll(sq, results, model.QueryOptions{Offset: offset})
 	return err
 }
 
 func fullTextExpr(value string) Sqlizer {
+	q := utils.SanitizeStrings(value)
+	if q == "" {
+		return nil
+	}
 	var sep string
 	if !conf.Server.SearchFullString {
 		sep = " "
 	}
-	q := utils.SanitizeStrings(value)
 	parts := strings.Split(q, " ")
 	filters := And{}
 	for _, part := range parts {