From e7c6e78dd0ab7d9ade7f487240a3bd165b086e81 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Deluan=20Quint=C3=A3o?= Date: Mon, 16 Mar 2026 07:55:22 -0400 Subject: [PATCH] fix(db): normalize timestamps and fix recently added album sorting (#5176) * fix(db): normalize timestamps and fix recently added album sorting SQLite stores timestamps as TEXT and uses string comparison for ORDER BY. Timestamps in RFC3339 T-format ('2024-01-01T10:00:00Z') sort incorrectly against space-format ('2024-01-01 10:00:00+00:00') because 'T' (ASCII 84) > ' ' (ASCII 32), causing albums with T-format timestamps to appear as newer than they are in the "Recently Added" list. This adds a migration to normalize all T-format timestamps across all tables to the space-format expected by go-sqlite3, wraps the recently_added sort with datetime() to make it format-agnostic, and replaces the plain album timestamp indexes with expression indexes to maintain query performance. * fix(test): improve recently_added sort test robustness Use same-date timestamps (2024-01-15T08:00:00Z vs 2024-01-15 20:00:00) so the T-vs-space character difference at position 10 actually triggers the sorting bug. Initialize index variables to -1 and assert both test albums are found before comparing positions. * chore(db): update migration timestamp to 2026-03-16 --- .../20260316000000_normalize_timestamps.sql | 74 +++++++++++++++++++ persistence/album_repository.go | 4 +- persistence/album_repository_test.go | 47 ++++++++++++ 3 files changed, 123 insertions(+), 2 deletions(-) create mode 100644 db/migrations/20260316000000_normalize_timestamps.sql diff --git a/db/migrations/20260316000000_normalize_timestamps.sql b/db/migrations/20260316000000_normalize_timestamps.sql new file mode 100644 index 00000000..a2e1183e --- /dev/null +++ b/db/migrations/20260316000000_normalize_timestamps.sql @@ -0,0 +1,74 @@ +-- +goose Up + +-- Normalize T-format timestamps (RFC3339Nano with 'T' separator) to SQLite-compatible format. +-- SQLite uses string comparison for ORDER BY on TEXT columns, so 'T' (ASCII 84) > ' ' (ASCII 32) +-- causes T-format timestamps to sort after space-format ones, breaking "Recently Added" ordering. + +UPDATE album SET created_at = replace(replace(created_at, 'T', ' '), 'Z', '+00:00') WHERE created_at LIKE '%T%'; +UPDATE album SET updated_at = replace(replace(updated_at, 'T', ' '), 'Z', '+00:00') WHERE updated_at LIKE '%T%'; +UPDATE album SET imported_at = replace(replace(imported_at, 'T', ' '), 'Z', '+00:00') WHERE imported_at LIKE '%T%'; +UPDATE album SET external_info_updated_at = replace(replace(external_info_updated_at, 'T', ' '), 'Z', '+00:00') WHERE external_info_updated_at LIKE '%T%'; + +UPDATE media_file SET created_at = replace(replace(created_at, 'T', ' '), 'Z', '+00:00') WHERE created_at LIKE '%T%'; +UPDATE media_file SET updated_at = replace(replace(updated_at, 'T', ' '), 'Z', '+00:00') WHERE updated_at LIKE '%T%'; +UPDATE media_file SET birth_time = replace(replace(birth_time, 'T', ' '), 'Z', '+00:00') WHERE birth_time LIKE '%T%'; + +UPDATE artist SET created_at = replace(replace(created_at, 'T', ' '), 'Z', '+00:00') WHERE created_at LIKE '%T%'; +UPDATE artist SET updated_at = replace(replace(updated_at, 'T', ' '), 'Z', '+00:00') WHERE updated_at LIKE '%T%'; +UPDATE artist SET external_info_updated_at = replace(replace(external_info_updated_at, 'T', ' '), 'Z', '+00:00') WHERE external_info_updated_at LIKE '%T%'; + +UPDATE annotation SET play_date = replace(replace(play_date, 'T', ' '), 'Z', '+00:00') WHERE play_date LIKE '%T%'; +UPDATE annotation SET starred_at = replace(replace(starred_at, 'T', ' '), 'Z', '+00:00') WHERE starred_at LIKE '%T%'; +UPDATE annotation SET rated_at = replace(replace(rated_at, 'T', ' '), 'Z', '+00:00') WHERE rated_at LIKE '%T%'; + +UPDATE playlist SET created_at = replace(replace(created_at, 'T', ' '), 'Z', '+00:00') WHERE created_at LIKE '%T%'; +UPDATE playlist SET updated_at = replace(replace(updated_at, 'T', ' '), 'Z', '+00:00') WHERE updated_at LIKE '%T%'; +UPDATE playlist SET evaluated_at = replace(replace(evaluated_at, 'T', ' '), 'Z', '+00:00') WHERE evaluated_at LIKE '%T%'; + +UPDATE user SET created_at = replace(replace(created_at, 'T', ' '), 'Z', '+00:00') WHERE created_at LIKE '%T%'; +UPDATE user SET updated_at = replace(replace(updated_at, 'T', ' '), 'Z', '+00:00') WHERE updated_at LIKE '%T%'; +UPDATE user SET last_login_at = replace(replace(last_login_at, 'T', ' '), 'Z', '+00:00') WHERE last_login_at LIKE '%T%'; +UPDATE user SET last_access_at = replace(replace(last_access_at, 'T', ' '), 'Z', '+00:00') WHERE last_access_at LIKE '%T%'; + +UPDATE player SET last_seen = replace(replace(last_seen, 'T', ' '), 'Z', '+00:00') WHERE last_seen LIKE '%T%'; + +UPDATE playqueue SET created_at = replace(replace(created_at, 'T', ' '), 'Z', '+00:00') WHERE created_at LIKE '%T%'; +UPDATE playqueue SET updated_at = replace(replace(updated_at, 'T', ' '), 'Z', '+00:00') WHERE updated_at LIKE '%T%'; + +UPDATE bookmark SET created_at = replace(replace(created_at, 'T', ' '), 'Z', '+00:00') WHERE created_at LIKE '%T%'; +UPDATE bookmark SET updated_at = replace(replace(updated_at, 'T', ' '), 'Z', '+00:00') WHERE updated_at LIKE '%T%'; + +UPDATE share SET created_at = replace(replace(created_at, 'T', ' '), 'Z', '+00:00') WHERE created_at LIKE '%T%'; +UPDATE share SET updated_at = replace(replace(updated_at, 'T', ' '), 'Z', '+00:00') WHERE updated_at LIKE '%T%'; +UPDATE share SET expires_at = replace(replace(expires_at, 'T', ' '), 'Z', '+00:00') WHERE expires_at LIKE '%T%'; +UPDATE share SET last_visited_at = replace(replace(last_visited_at, 'T', ' '), 'Z', '+00:00') WHERE last_visited_at LIKE '%T%'; + +UPDATE radio SET created_at = replace(replace(created_at, 'T', ' '), 'Z', '+00:00') WHERE created_at LIKE '%T%'; +UPDATE radio SET updated_at = replace(replace(updated_at, 'T', ' '), 'Z', '+00:00') WHERE updated_at LIKE '%T%'; + +UPDATE folder SET created_at = replace(replace(created_at, 'T', ' '), 'Z', '+00:00') WHERE created_at LIKE '%T%'; +UPDATE folder SET updated_at = replace(replace(updated_at, 'T', ' '), 'Z', '+00:00') WHERE updated_at LIKE '%T%'; +UPDATE folder SET images_updated_at = replace(replace(images_updated_at, 'T', ' '), 'Z', '+00:00') WHERE images_updated_at LIKE '%T%'; + +UPDATE library SET created_at = replace(replace(created_at, 'T', ' '), 'Z', '+00:00') WHERE created_at LIKE '%T%'; +UPDATE library SET updated_at = replace(replace(updated_at, 'T', ' '), 'Z', '+00:00') WHERE updated_at LIKE '%T%'; +UPDATE library SET last_scan_at = replace(replace(last_scan_at, 'T', ' '), 'Z', '+00:00') WHERE last_scan_at LIKE '%T%'; +UPDATE library SET last_scan_started_at = replace(replace(last_scan_started_at, 'T', ' '), 'Z', '+00:00') WHERE last_scan_started_at LIKE '%T%'; + +UPDATE scrobble_buffer SET play_time = replace(replace(play_time, 'T', ' '), 'Z', '+00:00') WHERE play_time LIKE '%T%'; +UPDATE scrobble_buffer SET enqueue_time = replace(replace(enqueue_time, 'T', ' '), 'Z', '+00:00') WHERE enqueue_time LIKE '%T%'; + +UPDATE plugin SET created_at = replace(replace(created_at, 'T', ' '), 'Z', '+00:00') WHERE created_at LIKE '%T%'; +UPDATE plugin SET updated_at = replace(replace(updated_at, 'T', ' '), 'Z', '+00:00') WHERE updated_at LIKE '%T%'; + +-- Replace plain indexes with expression indexes for datetime()-based sorting +DROP INDEX IF EXISTS album_created_at; +CREATE INDEX album_created_at ON album(datetime(created_at)); +DROP INDEX IF EXISTS album_updated_at; +CREATE INDEX album_updated_at ON album(datetime(updated_at)); + +-- +goose Down +DROP INDEX IF EXISTS album_created_at; +CREATE INDEX album_created_at ON album(created_at); +DROP INDEX IF EXISTS album_updated_at; +CREATE INDEX album_updated_at ON album(updated_at); diff --git a/persistence/album_repository.go b/persistence/album_repository.go index 7207bf5a..c51a5beb 100644 --- a/persistence/album_repository.go +++ b/persistence/album_repository.go @@ -143,9 +143,9 @@ var albumFilters = sync.OnceValue(func() map[string]filterFunc { func recentlyAddedSort() string { if conf.Server.RecentlyAddedByModTime { - return "updated_at" + return "datetime(album.updated_at)" } - return "created_at" + return "datetime(album.created_at)" } func recentlyPlayedFilter(string, any) Sqlizer { diff --git a/persistence/album_repository_test.go b/persistence/album_repository_test.go index 66b6eba9..2792cec9 100644 --- a/persistence/album_repository_test.go +++ b/persistence/album_repository_test.go @@ -85,6 +85,53 @@ var _ = Describe("AlbumRepository", func() { }) }) + Describe("recently_added sort", func() { + It("sorts correctly regardless of timestamp format (T-format vs space-format)", func() { + // Both timestamps share the same date prefix "2024-01-15" so the T vs space + // character at position 10 determines sort order in raw string comparison. + // Without normalization, 'T' (ASCII 84) > ' ' (ASCII 32) makes the older + // T-format timestamp sort AFTER the newer space-format one. + + // Older album: morning of Jan 15, stored in T-format + olderAlbum := &model.Album{LibraryID: 1, ID: "ts-older", Name: "Older Album"} + Expect(albumRepo.Put(olderAlbum)).To(Succeed()) + _, err := albumRepo.executeSQL(squirrel.Update("album"). + Set("created_at", "2024-01-15T08:00:00Z"). + Where(squirrel.Eq{"id": "ts-older"})) + Expect(err).ToNot(HaveOccurred()) + + // Newer album: evening of Jan 15, stored in space-format + newerAlbum := &model.Album{LibraryID: 1, ID: "ts-newer", Name: "Newer Album"} + Expect(albumRepo.Put(newerAlbum)).To(Succeed()) + _, err = albumRepo.executeSQL(squirrel.Update("album"). + Set("created_at", "2024-01-15 20:00:00+00:00"). + Where(squirrel.Eq{"id": "ts-newer"})) + Expect(err).ToNot(HaveOccurred()) + + albums, err := albumRepo.GetAll(model.QueryOptions{Sort: "recently_added", Order: "desc"}) + Expect(err).ToNot(HaveOccurred()) + + // Find positions of our test albums + olderIdx, newerIdx := -1, -1 + for i, a := range albums { + switch a.ID { + case "ts-older": + olderIdx = i + case "ts-newer": + newerIdx = i + } + } + Expect(olderIdx).To(BeNumerically(">=", 0), "older album not found in results") + Expect(newerIdx).To(BeNumerically(">=", 0), "newer album not found in results") + // Newer album (evening, space-format) should come before older album (morning, T-format) in desc order + Expect(newerIdx).To(BeNumerically("<", olderIdx), + "Newer album (20:00 space-format) should sort before older album (08:00 T-format) in desc order") + + // Clean up + _, _ = albumRepo.executeSQL(squirrel.Delete("album").Where(squirrel.Eq{"id": []string{"ts-older", "ts-newer"}})) + }) + }) + Context("Filters", func() { var albumWithoutAnnotation model.Album