perf: optimize cross-library move detection for single-library setups (#4888)

* feat: skip cross-library detection for single library setup

When only one library is configured, skip the cross-library move detection stage entirely as there are no other libraries to search in. This eliminates unnecessary database queries - the primary performance issue reported by users (5-6 hour scans with 13.5k missing files).

Implementation:
- Added library count check in processCrossLibraryMoves
- Returns input unchanged when len(state.libraries) == 1
- Logs debug message for troubleshooting

* refactor: use lightweight queries for cross-library move detection

Replace selectMediaFile() with newSelect() in FindRecentFilesByMBZTrackID and FindRecentFilesByProperties. These queries only need basic media file columns for hash and path comparisons, not annotations/bookmarks.

Benefits:
- Removes unnecessary LEFT JOINs with annotation and bookmark tables
- Reduces query overhead for cross-library file matching
- Follows existing pattern used by GetMissingAndMatching

The annotation/bookmark joins are user-specific (using loggedUser context) and unused in cross-library matching logic where only Equals() and IsEquivalent() checks are performed.

* test: add coverage for single-library and multi-library cross-library detection

Add test cases to verify:
1. Single-library setup correctly skips cross-library move detection
2. Multi-library setup continues to process cross-library moves

Implementation:
- New test verifies processCrossLibraryMoves returns input unchanged for single library
- Wrapped existing multi-library tests in Context with multiple libraries setup
- Ensures no regressions in multi-library matching behavior

Tests verify:
- Single-library: no database queries, input passed through unchanged
- Multi-library: cross-library matching still works correctly
- Reduces the likelihood of introducing single-library skip bugs in future

* fix: enhance cross-library detection by introducing totalLibraryCount

Signed-off-by: Deluan <deluan@navidrome.org>

---------

Signed-off-by: Deluan <deluan@navidrome.org>
This commit is contained in:
Deluan Quintão
2026-01-15 17:22:46 -05:00
committed by GitHub
parent 14efb13cd4
commit 6d47a6ebd9
4 changed files with 410 additions and 352 deletions
+8 -2
View File
@@ -332,8 +332,11 @@ func (r *mediaFileRepository) GetMissingAndMatching(libId int) (model.MediaFileC
} }
// FindRecentFilesByMBZTrackID finds recently added files by MusicBrainz Track ID in other libraries // FindRecentFilesByMBZTrackID finds recently added files by MusicBrainz Track ID in other libraries
// It uses a lightweight query without annotation/bookmark joins since those are not needed for matching
func (r *mediaFileRepository) FindRecentFilesByMBZTrackID(missing model.MediaFile, since time.Time) (model.MediaFiles, error) { func (r *mediaFileRepository) FindRecentFilesByMBZTrackID(missing model.MediaFile, since time.Time) (model.MediaFiles, error) {
sel := r.selectMediaFile().Where(And{ sel := r.newSelect().Columns("media_file.*", "library.path as library_path", "library.name as library_name").
LeftJoin("library on media_file.library_id = library.id").
Where(And{
NotEq{"media_file.library_id": missing.LibraryID}, NotEq{"media_file.library_id": missing.LibraryID},
Eq{"media_file.mbz_release_track_id": missing.MbzReleaseTrackID}, Eq{"media_file.mbz_release_track_id": missing.MbzReleaseTrackID},
NotEq{"media_file.mbz_release_track_id": ""}, // Exclude empty MBZ Track IDs NotEq{"media_file.mbz_release_track_id": ""}, // Exclude empty MBZ Track IDs
@@ -351,8 +354,11 @@ func (r *mediaFileRepository) FindRecentFilesByMBZTrackID(missing model.MediaFil
} }
// FindRecentFilesByProperties finds recently added files by intrinsic properties in other libraries // FindRecentFilesByProperties finds recently added files by intrinsic properties in other libraries
// It uses a lightweight query without annotation/bookmark joins since those are not needed for matching
func (r *mediaFileRepository) FindRecentFilesByProperties(missing model.MediaFile, since time.Time) (model.MediaFiles, error) { func (r *mediaFileRepository) FindRecentFilesByProperties(missing model.MediaFile, since time.Time) (model.MediaFiles, error) {
sel := r.selectMediaFile().Where(And{ sel := r.newSelect().Columns("media_file.*", "library.path as library_path", "library.name as library_name").
LeftJoin("library on media_file.library_id = library.id").
Where(And{
NotEq{"media_file.library_id": missing.LibraryID}, NotEq{"media_file.library_id": missing.LibraryID},
Eq{"media_file.title": missing.Title}, Eq{"media_file.title": missing.Title},
Eq{"media_file.size": missing.Size}, Eq{"media_file.size": missing.Size},
+7
View File
@@ -187,6 +187,13 @@ func (p *phaseMissingTracks) processCrossLibraryMoves(in *missingTracks) (*missi
return nil, nil return nil, nil
} }
// Skip cross-library move detection when only one library is configured
// since there are no other libraries to search in.
if p.state.totalLibraryCount == 1 {
log.Debug(p.ctx, "Scanner: Skipping cross-library move detection (single library)")
return in, nil
}
log.Debug(p.ctx, "Scanner: Processing cross-library moves", "pid", in.pid, "missing", len(in.missing), "lib", in.lib.Name) log.Debug(p.ctx, "Scanner: Processing cross-library moves", "pid", in.pid, "missing", len(in.missing), "lib", in.lib.Name)
for _, missing := range in.missing { for _, missing := range in.missing {
+43
View File
@@ -30,6 +30,7 @@ var _ = Describe("phaseMissingTracks", func() {
ds = &tests.MockDataStore{MockedMediaFile: mr, MockedLibrary: lr} ds = &tests.MockDataStore{MockedMediaFile: mr, MockedLibrary: lr}
state = &scanState{ state = &scanState{
libraries: model.Libraries{{ID: 1, LastScanStartedAt: time.Date(2021, 1, 1, 0, 0, 0, 0, time.UTC)}}, libraries: model.Libraries{{ID: 1, LastScanStartedAt: time.Date(2021, 1, 1, 0, 0, 0, 0, time.UTC)}},
totalLibraryCount: 1,
} }
phase = createPhaseMissingTracks(ctx, state, ds) phase = createPhaseMissingTracks(ctx, state, ds)
}) })
@@ -330,6 +331,47 @@ var _ = Describe("phaseMissingTracks", func() {
Expect(result).To(BeNil()) Expect(result).To(BeNil())
}) })
It("should skip cross-library move detection when only one library is configured", func() {
// Default BeforeEach sets up single library, so we just need to verify skip behavior
Expect(state.totalLibraryCount).To(Equal(1))
missingTrack := model.MediaFile{
ID: "missing1",
LibraryID: 1,
MbzReleaseTrackID: "mbz-track-123",
Title: "Test Track",
Size: 1000,
Suffix: "mp3",
Path: "/lib1/track.mp3",
Missing: true,
CreatedAt: time.Now().Add(-30 * time.Minute),
}
in := &missingTracks{
lib: model.Library{ID: 1, Name: "Library 1"},
missing: []model.MediaFile{missingTrack},
}
result, err := phase.processCrossLibraryMoves(in)
Expect(err).ToNot(HaveOccurred())
// Should return input unchanged (no processing done)
Expect(result).To(Equal(in))
// No matches should be found since cross-library search was skipped
Expect(phase.totalMatched.Load()).To(Equal(uint32(0)))
// No changes should be detected
Expect(state.changesDetected.Load()).To(BeFalse())
})
Context("with multiple libraries", func() {
BeforeEach(func() {
// Set up multiple libraries for cross-library move tests
state.libraries = model.Libraries{
{ID: 1, LastScanStartedAt: time.Date(2021, 1, 1, 0, 0, 0, 0, time.UTC)},
{ID: 2, LastScanStartedAt: time.Date(2021, 1, 1, 0, 0, 0, 0, time.UTC)},
}
state.totalLibraryCount = 2
})
It("should process cross-library moves using MusicBrainz Track ID", func() { It("should process cross-library moves using MusicBrainz Track ID", func() {
scanStartTime := time.Now().Add(-1 * time.Hour) scanStartTime := time.Now().Add(-1 * time.Hour)
missingTrack := model.MediaFile{ missingTrack := model.MediaFile{
@@ -679,6 +721,7 @@ var _ = Describe("phaseMissingTracks", func() {
Expect(phase.totalMatched.Load()).To(Equal(uint32(0))) Expect(phase.totalMatched.Load()).To(Equal(uint32(0)))
Expect(state.changesDetected.Load()).To(BeFalse()) Expect(state.changesDetected.Load()).To(BeFalse())
}) })
}) // End of Context "with multiple libraries"
}) })
Describe("Album Annotation Reassignment", func() { Describe("Album Annotation Reassignment", func() {
+2
View File
@@ -33,6 +33,7 @@ type scanState struct {
changesDetected atomic.Bool changesDetected atomic.Bool
libraries model.Libraries // Store libraries list for consistency across phases libraries model.Libraries // Store libraries list for consistency across phases
targets map[int][]string // Optional: map[libraryID][]folderPaths for selective scans targets map[int][]string // Optional: map[libraryID][]folderPaths for selective scans
totalLibraryCount int // Total number of libraries (unfiltered), for cross-library move detection
} }
func (s *scanState) sendProgress(info *ProgressInfo) { func (s *scanState) sendProgress(info *ProgressInfo) {
@@ -73,6 +74,7 @@ func (s *scannerImpl) scanFolders(ctx context.Context, fullScan bool, targets []
state.sendWarning(fmt.Sprintf("getting libraries: %s", err)) state.sendWarning(fmt.Sprintf("getting libraries: %s", err))
return return
} }
state.totalLibraryCount = len(allLibs)
if len(targets) > 0 { if len(targets) > 0 {
// Selective scan: filter libraries and build targets map // Selective scan: filter libraries and build targets map