perf: optimize cross-library move detection for single-library setups (#4888)
* feat: skip cross-library detection for single library setup When only one library is configured, skip the cross-library move detection stage entirely as there are no other libraries to search in. This eliminates unnecessary database queries - the primary performance issue reported by users (5-6 hour scans with 13.5k missing files). Implementation: - Added library count check in processCrossLibraryMoves - Returns input unchanged when len(state.libraries) == 1 - Logs debug message for troubleshooting * refactor: use lightweight queries for cross-library move detection Replace selectMediaFile() with newSelect() in FindRecentFilesByMBZTrackID and FindRecentFilesByProperties. These queries only need basic media file columns for hash and path comparisons, not annotations/bookmarks. Benefits: - Removes unnecessary LEFT JOINs with annotation and bookmark tables - Reduces query overhead for cross-library file matching - Follows existing pattern used by GetMissingAndMatching The annotation/bookmark joins are user-specific (using loggedUser context) and unused in cross-library matching logic where only Equals() and IsEquivalent() checks are performed. * test: add coverage for single-library and multi-library cross-library detection Add test cases to verify: 1. Single-library setup correctly skips cross-library move detection 2. Multi-library setup continues to process cross-library moves Implementation: - New test verifies processCrossLibraryMoves returns input unchanged for single library - Wrapped existing multi-library tests in Context with multiple libraries setup - Ensures no regressions in multi-library matching behavior Tests verify: - Single-library: no database queries, input passed through unchanged - Multi-library: cross-library matching still works correctly - Reduces the likelihood of introducing single-library skip bugs in future * fix: enhance cross-library detection by introducing totalLibraryCount Signed-off-by: Deluan <deluan@navidrome.org> --------- Signed-off-by: Deluan <deluan@navidrome.org>
This commit is contained in:
Deluan Quintão
@@ -332,8 +332,11 @@ func (r *mediaFileRepository) GetMissingAndMatching(libId int) (model.MediaFileC
|
||||
}
|
||||
|
||||
// FindRecentFilesByMBZTrackID finds recently added files by MusicBrainz Track ID in other libraries
|
||||
// It uses a lightweight query without annotation/bookmark joins since those are not needed for matching
|
||||
func (r *mediaFileRepository) FindRecentFilesByMBZTrackID(missing model.MediaFile, since time.Time) (model.MediaFiles, error) {
|
||||
sel := r.selectMediaFile().Where(And{
|
||||
sel := r.newSelect().Columns("media_file.*", "library.path as library_path", "library.name as library_name").
|
||||
LeftJoin("library on media_file.library_id = library.id").
|
||||
Where(And{
|
||||
NotEq{"media_file.library_id": missing.LibraryID},
|
||||
Eq{"media_file.mbz_release_track_id": missing.MbzReleaseTrackID},
|
||||
NotEq{"media_file.mbz_release_track_id": ""}, // Exclude empty MBZ Track IDs
|
||||
@@ -351,8 +354,11 @@ func (r *mediaFileRepository) FindRecentFilesByMBZTrackID(missing model.MediaFil
|
||||
}
|
||||
|
||||
// FindRecentFilesByProperties finds recently added files by intrinsic properties in other libraries
|
||||
// It uses a lightweight query without annotation/bookmark joins since those are not needed for matching
|
||||
func (r *mediaFileRepository) FindRecentFilesByProperties(missing model.MediaFile, since time.Time) (model.MediaFiles, error) {
|
||||
sel := r.selectMediaFile().Where(And{
|
||||
sel := r.newSelect().Columns("media_file.*", "library.path as library_path", "library.name as library_name").
|
||||
LeftJoin("library on media_file.library_id = library.id").
|
||||
Where(And{
|
||||
NotEq{"media_file.library_id": missing.LibraryID},
|
||||
Eq{"media_file.title": missing.Title},
|
||||
Eq{"media_file.size": missing.Size},
|
||||
|
||||
@@ -187,6 +187,13 @@ func (p *phaseMissingTracks) processCrossLibraryMoves(in *missingTracks) (*missi
|
||||
return nil, nil
|
||||
}
|
||||
|
||||
// Skip cross-library move detection when only one library is configured
|
||||
// since there are no other libraries to search in.
|
||||
if p.state.totalLibraryCount == 1 {
|
||||
log.Debug(p.ctx, "Scanner: Skipping cross-library move detection (single library)")
|
||||
return in, nil
|
||||
}
|
||||
|
||||
log.Debug(p.ctx, "Scanner: Processing cross-library moves", "pid", in.pid, "missing", len(in.missing), "lib", in.lib.Name)
|
||||
|
||||
for _, missing := range in.missing {
|
||||
|
||||
@@ -30,6 +30,7 @@ var _ = Describe("phaseMissingTracks", func() {
|
||||
ds = &tests.MockDataStore{MockedMediaFile: mr, MockedLibrary: lr}
|
||||
state = &scanState{
|
||||
libraries: model.Libraries{{ID: 1, LastScanStartedAt: time.Date(2021, 1, 1, 0, 0, 0, 0, time.UTC)}},
|
||||
totalLibraryCount: 1,
|
||||
}
|
||||
phase = createPhaseMissingTracks(ctx, state, ds)
|
||||
})
|
||||
@@ -330,6 +331,47 @@ var _ = Describe("phaseMissingTracks", func() {
|
||||
Expect(result).To(BeNil())
|
||||
})
|
||||
|
||||
It("should skip cross-library move detection when only one library is configured", func() {
|
||||
// Default BeforeEach sets up single library, so we just need to verify skip behavior
|
||||
Expect(state.totalLibraryCount).To(Equal(1))
|
||||
|
||||
missingTrack := model.MediaFile{
|
||||
ID: "missing1",
|
||||
LibraryID: 1,
|
||||
MbzReleaseTrackID: "mbz-track-123",
|
||||
Title: "Test Track",
|
||||
Size: 1000,
|
||||
Suffix: "mp3",
|
||||
Path: "/lib1/track.mp3",
|
||||
Missing: true,
|
||||
CreatedAt: time.Now().Add(-30 * time.Minute),
|
||||
}
|
||||
|
||||
in := &missingTracks{
|
||||
lib: model.Library{ID: 1, Name: "Library 1"},
|
||||
missing: []model.MediaFile{missingTrack},
|
||||
}
|
||||
|
||||
result, err := phase.processCrossLibraryMoves(in)
|
||||
Expect(err).ToNot(HaveOccurred())
|
||||
// Should return input unchanged (no processing done)
|
||||
Expect(result).To(Equal(in))
|
||||
// No matches should be found since cross-library search was skipped
|
||||
Expect(phase.totalMatched.Load()).To(Equal(uint32(0)))
|
||||
// No changes should be detected
|
||||
Expect(state.changesDetected.Load()).To(BeFalse())
|
||||
})
|
||||
|
||||
Context("with multiple libraries", func() {
|
||||
BeforeEach(func() {
|
||||
// Set up multiple libraries for cross-library move tests
|
||||
state.libraries = model.Libraries{
|
||||
{ID: 1, LastScanStartedAt: time.Date(2021, 1, 1, 0, 0, 0, 0, time.UTC)},
|
||||
{ID: 2, LastScanStartedAt: time.Date(2021, 1, 1, 0, 0, 0, 0, time.UTC)},
|
||||
}
|
||||
state.totalLibraryCount = 2
|
||||
})
|
||||
|
||||
It("should process cross-library moves using MusicBrainz Track ID", func() {
|
||||
scanStartTime := time.Now().Add(-1 * time.Hour)
|
||||
missingTrack := model.MediaFile{
|
||||
@@ -679,6 +721,7 @@ var _ = Describe("phaseMissingTracks", func() {
|
||||
Expect(phase.totalMatched.Load()).To(Equal(uint32(0)))
|
||||
Expect(state.changesDetected.Load()).To(BeFalse())
|
||||
})
|
||||
}) // End of Context "with multiple libraries"
|
||||
})
|
||||
|
||||
Describe("Album Annotation Reassignment", func() {
|
||||
|
||||
@@ -33,6 +33,7 @@ type scanState struct {
|
||||
changesDetected atomic.Bool
|
||||
libraries model.Libraries // Store libraries list for consistency across phases
|
||||
targets map[int][]string // Optional: map[libraryID][]folderPaths for selective scans
|
||||
totalLibraryCount int // Total number of libraries (unfiltered), for cross-library move detection
|
||||
}
|
||||
|
||||
func (s *scanState) sendProgress(info *ProgressInfo) {
|
||||
@@ -73,6 +74,7 @@ func (s *scannerImpl) scanFolders(ctx context.Context, fullScan bool, targets []
|
||||
state.sendWarning(fmt.Sprintf("getting libraries: %s", err))
|
||||
return
|
||||
}
|
||||
state.totalLibraryCount = len(allLibs)
|
||||
|
||||
if len(targets) > 0 {
|
||||
// Selective scan: filter libraries and build targets map
|
||||
|
||||
Reference in New Issue
Block a user