perf: optimize cross-library move detection for single-library setups (#4888)
* feat: skip cross-library detection for single library setup When only one library is configured, skip the cross-library move detection stage entirely as there are no other libraries to search in. This eliminates unnecessary database queries - the primary performance issue reported by users (5-6 hour scans with 13.5k missing files). Implementation: - Added library count check in processCrossLibraryMoves - Returns input unchanged when len(state.libraries) == 1 - Logs debug message for troubleshooting * refactor: use lightweight queries for cross-library move detection Replace selectMediaFile() with newSelect() in FindRecentFilesByMBZTrackID and FindRecentFilesByProperties. These queries only need basic media file columns for hash and path comparisons, not annotations/bookmarks. Benefits: - Removes unnecessary LEFT JOINs with annotation and bookmark tables - Reduces query overhead for cross-library file matching - Follows existing pattern used by GetMissingAndMatching The annotation/bookmark joins are user-specific (using loggedUser context) and unused in cross-library matching logic where only Equals() and IsEquivalent() checks are performed. * test: add coverage for single-library and multi-library cross-library detection Add test cases to verify: 1. Single-library setup correctly skips cross-library move detection 2. Multi-library setup continues to process cross-library moves Implementation: - New test verifies processCrossLibraryMoves returns input unchanged for single library - Wrapped existing multi-library tests in Context with multiple libraries setup - Ensures no regressions in multi-library matching behavior Tests verify: - Single-library: no database queries, input passed through unchanged - Multi-library: cross-library matching still works correctly - Reduces the likelihood of introducing single-library skip bugs in future * fix: enhance cross-library detection by introducing totalLibraryCount Signed-off-by: Deluan <deluan@navidrome.org> --------- Signed-off-by: Deluan <deluan@navidrome.org>
This commit is contained in:
@@ -332,8 +332,11 @@ func (r *mediaFileRepository) GetMissingAndMatching(libId int) (model.MediaFileC
|
|||||||
}
|
}
|
||||||
|
|
||||||
// FindRecentFilesByMBZTrackID finds recently added files by MusicBrainz Track ID in other libraries
|
// FindRecentFilesByMBZTrackID finds recently added files by MusicBrainz Track ID in other libraries
|
||||||
|
// It uses a lightweight query without annotation/bookmark joins since those are not needed for matching
|
||||||
func (r *mediaFileRepository) FindRecentFilesByMBZTrackID(missing model.MediaFile, since time.Time) (model.MediaFiles, error) {
|
func (r *mediaFileRepository) FindRecentFilesByMBZTrackID(missing model.MediaFile, since time.Time) (model.MediaFiles, error) {
|
||||||
sel := r.selectMediaFile().Where(And{
|
sel := r.newSelect().Columns("media_file.*", "library.path as library_path", "library.name as library_name").
|
||||||
|
LeftJoin("library on media_file.library_id = library.id").
|
||||||
|
Where(And{
|
||||||
NotEq{"media_file.library_id": missing.LibraryID},
|
NotEq{"media_file.library_id": missing.LibraryID},
|
||||||
Eq{"media_file.mbz_release_track_id": missing.MbzReleaseTrackID},
|
Eq{"media_file.mbz_release_track_id": missing.MbzReleaseTrackID},
|
||||||
NotEq{"media_file.mbz_release_track_id": ""}, // Exclude empty MBZ Track IDs
|
NotEq{"media_file.mbz_release_track_id": ""}, // Exclude empty MBZ Track IDs
|
||||||
@@ -351,8 +354,11 @@ func (r *mediaFileRepository) FindRecentFilesByMBZTrackID(missing model.MediaFil
|
|||||||
}
|
}
|
||||||
|
|
||||||
// FindRecentFilesByProperties finds recently added files by intrinsic properties in other libraries
|
// FindRecentFilesByProperties finds recently added files by intrinsic properties in other libraries
|
||||||
|
// It uses a lightweight query without annotation/bookmark joins since those are not needed for matching
|
||||||
func (r *mediaFileRepository) FindRecentFilesByProperties(missing model.MediaFile, since time.Time) (model.MediaFiles, error) {
|
func (r *mediaFileRepository) FindRecentFilesByProperties(missing model.MediaFile, since time.Time) (model.MediaFiles, error) {
|
||||||
sel := r.selectMediaFile().Where(And{
|
sel := r.newSelect().Columns("media_file.*", "library.path as library_path", "library.name as library_name").
|
||||||
|
LeftJoin("library on media_file.library_id = library.id").
|
||||||
|
Where(And{
|
||||||
NotEq{"media_file.library_id": missing.LibraryID},
|
NotEq{"media_file.library_id": missing.LibraryID},
|
||||||
Eq{"media_file.title": missing.Title},
|
Eq{"media_file.title": missing.Title},
|
||||||
Eq{"media_file.size": missing.Size},
|
Eq{"media_file.size": missing.Size},
|
||||||
|
|||||||
@@ -187,6 +187,13 @@ func (p *phaseMissingTracks) processCrossLibraryMoves(in *missingTracks) (*missi
|
|||||||
return nil, nil
|
return nil, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// Skip cross-library move detection when only one library is configured
|
||||||
|
// since there are no other libraries to search in.
|
||||||
|
if p.state.totalLibraryCount == 1 {
|
||||||
|
log.Debug(p.ctx, "Scanner: Skipping cross-library move detection (single library)")
|
||||||
|
return in, nil
|
||||||
|
}
|
||||||
|
|
||||||
log.Debug(p.ctx, "Scanner: Processing cross-library moves", "pid", in.pid, "missing", len(in.missing), "lib", in.lib.Name)
|
log.Debug(p.ctx, "Scanner: Processing cross-library moves", "pid", in.pid, "missing", len(in.missing), "lib", in.lib.Name)
|
||||||
|
|
||||||
for _, missing := range in.missing {
|
for _, missing := range in.missing {
|
||||||
|
|||||||
@@ -30,6 +30,7 @@ var _ = Describe("phaseMissingTracks", func() {
|
|||||||
ds = &tests.MockDataStore{MockedMediaFile: mr, MockedLibrary: lr}
|
ds = &tests.MockDataStore{MockedMediaFile: mr, MockedLibrary: lr}
|
||||||
state = &scanState{
|
state = &scanState{
|
||||||
libraries: model.Libraries{{ID: 1, LastScanStartedAt: time.Date(2021, 1, 1, 0, 0, 0, 0, time.UTC)}},
|
libraries: model.Libraries{{ID: 1, LastScanStartedAt: time.Date(2021, 1, 1, 0, 0, 0, 0, time.UTC)}},
|
||||||
|
totalLibraryCount: 1,
|
||||||
}
|
}
|
||||||
phase = createPhaseMissingTracks(ctx, state, ds)
|
phase = createPhaseMissingTracks(ctx, state, ds)
|
||||||
})
|
})
|
||||||
@@ -330,6 +331,47 @@ var _ = Describe("phaseMissingTracks", func() {
|
|||||||
Expect(result).To(BeNil())
|
Expect(result).To(BeNil())
|
||||||
})
|
})
|
||||||
|
|
||||||
|
It("should skip cross-library move detection when only one library is configured", func() {
|
||||||
|
// Default BeforeEach sets up single library, so we just need to verify skip behavior
|
||||||
|
Expect(state.totalLibraryCount).To(Equal(1))
|
||||||
|
|
||||||
|
missingTrack := model.MediaFile{
|
||||||
|
ID: "missing1",
|
||||||
|
LibraryID: 1,
|
||||||
|
MbzReleaseTrackID: "mbz-track-123",
|
||||||
|
Title: "Test Track",
|
||||||
|
Size: 1000,
|
||||||
|
Suffix: "mp3",
|
||||||
|
Path: "/lib1/track.mp3",
|
||||||
|
Missing: true,
|
||||||
|
CreatedAt: time.Now().Add(-30 * time.Minute),
|
||||||
|
}
|
||||||
|
|
||||||
|
in := &missingTracks{
|
||||||
|
lib: model.Library{ID: 1, Name: "Library 1"},
|
||||||
|
missing: []model.MediaFile{missingTrack},
|
||||||
|
}
|
||||||
|
|
||||||
|
result, err := phase.processCrossLibraryMoves(in)
|
||||||
|
Expect(err).ToNot(HaveOccurred())
|
||||||
|
// Should return input unchanged (no processing done)
|
||||||
|
Expect(result).To(Equal(in))
|
||||||
|
// No matches should be found since cross-library search was skipped
|
||||||
|
Expect(phase.totalMatched.Load()).To(Equal(uint32(0)))
|
||||||
|
// No changes should be detected
|
||||||
|
Expect(state.changesDetected.Load()).To(BeFalse())
|
||||||
|
})
|
||||||
|
|
||||||
|
Context("with multiple libraries", func() {
|
||||||
|
BeforeEach(func() {
|
||||||
|
// Set up multiple libraries for cross-library move tests
|
||||||
|
state.libraries = model.Libraries{
|
||||||
|
{ID: 1, LastScanStartedAt: time.Date(2021, 1, 1, 0, 0, 0, 0, time.UTC)},
|
||||||
|
{ID: 2, LastScanStartedAt: time.Date(2021, 1, 1, 0, 0, 0, 0, time.UTC)},
|
||||||
|
}
|
||||||
|
state.totalLibraryCount = 2
|
||||||
|
})
|
||||||
|
|
||||||
It("should process cross-library moves using MusicBrainz Track ID", func() {
|
It("should process cross-library moves using MusicBrainz Track ID", func() {
|
||||||
scanStartTime := time.Now().Add(-1 * time.Hour)
|
scanStartTime := time.Now().Add(-1 * time.Hour)
|
||||||
missingTrack := model.MediaFile{
|
missingTrack := model.MediaFile{
|
||||||
@@ -679,6 +721,7 @@ var _ = Describe("phaseMissingTracks", func() {
|
|||||||
Expect(phase.totalMatched.Load()).To(Equal(uint32(0)))
|
Expect(phase.totalMatched.Load()).To(Equal(uint32(0)))
|
||||||
Expect(state.changesDetected.Load()).To(BeFalse())
|
Expect(state.changesDetected.Load()).To(BeFalse())
|
||||||
})
|
})
|
||||||
|
}) // End of Context "with multiple libraries"
|
||||||
})
|
})
|
||||||
|
|
||||||
Describe("Album Annotation Reassignment", func() {
|
Describe("Album Annotation Reassignment", func() {
|
||||||
|
|||||||
@@ -33,6 +33,7 @@ type scanState struct {
|
|||||||
changesDetected atomic.Bool
|
changesDetected atomic.Bool
|
||||||
libraries model.Libraries // Store libraries list for consistency across phases
|
libraries model.Libraries // Store libraries list for consistency across phases
|
||||||
targets map[int][]string // Optional: map[libraryID][]folderPaths for selective scans
|
targets map[int][]string // Optional: map[libraryID][]folderPaths for selective scans
|
||||||
|
totalLibraryCount int // Total number of libraries (unfiltered), for cross-library move detection
|
||||||
}
|
}
|
||||||
|
|
||||||
func (s *scanState) sendProgress(info *ProgressInfo) {
|
func (s *scanState) sendProgress(info *ProgressInfo) {
|
||||||
@@ -73,6 +74,7 @@ func (s *scannerImpl) scanFolders(ctx context.Context, fullScan bool, targets []
|
|||||||
state.sendWarning(fmt.Sprintf("getting libraries: %s", err))
|
state.sendWarning(fmt.Sprintf("getting libraries: %s", err))
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
|
state.totalLibraryCount = len(allLibs)
|
||||||
|
|
||||||
if len(targets) > 0 {
|
if len(targets) > 0 {
|
||||||
// Selective scan: filter libraries and build targets map
|
// Selective scan: filter libraries and build targets map
|
||||||
|
|||||||
Reference in New Issue
Block a user