feat(scanner): implement selective folder scanning and file system watcher improvements (#4674)
* feat: Add selective folder scanning capability Implement targeted scanning of specific library/folder pairs without full recursion. This enables efficient rescanning of individual folders when changes are detected, significantly reducing scan time for large libraries. Key changes: - Add ScanTarget struct and ScanFolders API to Scanner interface - Implement CLI flag --targets for specifying libraryID:folderPath pairs - Add FolderRepository.GetByPaths() for batch folder info retrieval - Create loadSpecificFolders() for non-recursive directory loading - Scope GC operations to affected libraries only (with TODO for full impl) - Add comprehensive tests for selective scanning behavior The selective scan: - Only processes specified folders (no subdirectory recursion) - Maintains library isolation - Runs full maintenance pipeline scoped to affected libraries - Supports both full and quick scan modes Examples: navidrome scan --targets "1:Music/Rock,1:Music/Jazz" navidrome scan --full --targets "2:Classical" * feat(folder): replace GetByPaths with GetFolderUpdateInfo for improved folder updates retrieval Signed-off-by: Deluan <deluan@navidrome.org> * test: update parseTargets test to handle folder names with spaces Signed-off-by: Deluan <deluan@navidrome.org> * refactor(folder): remove unused LibraryPath struct and update GC logging message Signed-off-by: Deluan <deluan@navidrome.org> * refactor(folder): enhance external scanner to support target-specific scanning Signed-off-by: Deluan <deluan@navidrome.org> * refactor(scanner): simplify scanner methods Signed-off-by: Deluan <deluan@navidrome.org> * feat(watcher): implement folder scanning notifications with deduplication Signed-off-by: Deluan <deluan@navidrome.org> * refactor(watcher): add resolveFolderPath function for testability Signed-off-by: Deluan <deluan@navidrome.org> * feat(watcher): implement path ignoring based on .ndignore patterns Signed-off-by: Deluan <deluan@navidrome.org> * refactor(scanner): implement IgnoreChecker for managing .ndignore patterns Signed-off-by: Deluan <deluan@navidrome.org> * refactor(ignore_checker): rename scanner to lineScanner for clarity Signed-off-by: Deluan <deluan@navidrome.org> * refactor(scanner): enhance ScanTarget struct with String method for better target representation Signed-off-by: Deluan <deluan@navidrome.org> * fix(scanner): validate library ID to prevent negative values Signed-off-by: Deluan <deluan@navidrome.org> * refactor(scanner): simplify GC method by removing library ID parameter Signed-off-by: Deluan <deluan@navidrome.org> * feat(scanner): update folder scanning to include all descendants of specified folders Signed-off-by: Deluan <deluan@navidrome.org> * feat(subsonic): allow selective scan in the /startScan endpoint Signed-off-by: Deluan <deluan@navidrome.org> * refactor(scanner): update CallScan to handle specific library/folder pairs Signed-off-by: Deluan <deluan@navidrome.org> * refactor(scanner): streamline scanning logic by removing scanAll method Signed-off-by: Deluan <deluan@navidrome.org> * test: enhance mockScanner for thread safety and improve test reliability Signed-off-by: Deluan <deluan@navidrome.org> * refactor(scanner): move scanner.ScanTarget to model.ScanTarget Signed-off-by: Deluan <deluan@navidrome.org> * refactor: move scanner types to model,implement MockScanner Signed-off-by: Deluan <deluan@navidrome.org> * refactor(scanner): update scanner interface and implementations to use model.Scanner Signed-off-by: Deluan <deluan@navidrome.org> * refactor(folder_repository): normalize target path handling by using filepath.Clean Signed-off-by: Deluan <deluan@navidrome.org> * test(folder_repository): add comprehensive tests for folder retrieval and child exclusion Signed-off-by: Deluan <deluan@navidrome.org> * refactor(scanner): simplify selective scan logic using slice.Filter Signed-off-by: Deluan <deluan@navidrome.org> * refactor(scanner): streamline phase folder and album creation by removing unnecessary library parameter Signed-off-by: Deluan <deluan@navidrome.org> * refactor(scanner): move initialization logic from phase_1 to the scanner itself Signed-off-by: Deluan <deluan@navidrome.org> * refactor(tests): rename selective scan test file to scanner_selective_test.go Signed-off-by: Deluan <deluan@navidrome.org> * feat(configuration): add DevSelectiveWatcher configuration option Signed-off-by: Deluan <deluan@navidrome.org> * feat(watcher): enhance .ndignore handling for folder deletions and file changes Signed-off-by: Deluan <deluan@navidrome.org> * docs(scanner): comments Signed-off-by: Deluan <deluan@navidrome.org> * refactor(scanner): enhance walkDirTree to support target folder scanning Signed-off-by: Deluan <deluan@navidrome.org> * fix(scanner, watcher): handle errors when pushing ignore patterns for folders Signed-off-by: Deluan <deluan@navidrome.org> * Update scanner/phase_1_folders.go Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com> * refactor(scanner): replace parseTargets function with direct call to scanner.ParseTargets Signed-off-by: Deluan <deluan@navidrome.org> * test(scanner): add tests for ScanBegin and ScanEnd functionality Signed-off-by: Deluan <deluan@navidrome.org> * fix(library): update PRAGMA optimize to check table sizes without ANALYZE Signed-off-by: Deluan <deluan@navidrome.org> * test(scanner): refactor tests Signed-off-by: Deluan <deluan@navidrome.org> * feat(ui): add selective scan options and update translations Signed-off-by: Deluan <deluan@navidrome.org> * feat(ui): add quick and full scan options for individual libraries Signed-off-by: Deluan <deluan@navidrome.org> * feat(ui): add Scan buttonsto the LibraryList Signed-off-by: Deluan <deluan@navidrome.org> * feat(scan): update scanning parameters from 'path' to 'target' for selective scans. * refactor(scan): move ParseTargets function to model package * test(scan): suppress unused return value from SetUserLibraries in tests * feat(gc): enhance garbage collection to support selective library purging Signed-off-by: Deluan <deluan@navidrome.org> * fix(scanner): prevent race condition when scanning deleted folders When the watcher detects changes in a folder that gets deleted before the scanner runs (due to the 10-second delay), the scanner was prematurely removing these folders from the tracking map, preventing them from being marked as missing. The issue occurred because `newFolderEntry` was calling `popLastUpdate` before verifying the folder actually exists on the filesystem. Changes: - Move fs.Stat check before newFolderEntry creation in loadDir to ensure deleted folders remain in lastUpdates for finalize() to handle - Add early existence check in walkDirTree to skip non-existent target folders with a warning log - Add unit test verifying non-existent folders aren't removed from lastUpdates prematurely - Add integration test for deleted folder scenario with ScanFolders Fixes the issue where deleting entire folders (e.g., /music/AC_DC) wouldn't mark tracks as missing when using selective folder scanning. * refactor(scan): streamline folder entry creation and update handling Signed-off-by: Deluan <deluan@navidrome.org> * feat(scan): add '@Recycle' (QNAP) to ignored directories list Signed-off-by: Deluan <deluan@navidrome.org> * fix(log): improve thread safety in logging level management * test(scan): move unit tests for ParseTargets function Signed-off-by: Deluan <deluan@navidrome.org> * review Signed-off-by: Deluan <deluan@navidrome.org> --------- Signed-off-by: Deluan <deluan@navidrome.org> Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com> Co-authored-by: deluan <deluan.quintao@mechanical-orchard.com>
This commit is contained in:
Deluan Quintão
+179
-65
@@ -25,82 +25,196 @@ var _ = Describe("walk_dir_tree", func() {
|
||||
ctx context.Context
|
||||
)
|
||||
|
||||
BeforeEach(func() {
|
||||
DeferCleanup(configtest.SetupConfig())
|
||||
ctx = GinkgoT().Context()
|
||||
fsys = &mockMusicFS{
|
||||
FS: fstest.MapFS{
|
||||
"root/a/.ndignore": {Data: []byte("ignored/*")},
|
||||
"root/a/f1.mp3": {},
|
||||
"root/a/f2.mp3": {},
|
||||
"root/a/ignored/bad.mp3": {},
|
||||
"root/b/cover.jpg": {},
|
||||
"root/c/f3": {},
|
||||
"root/d": {},
|
||||
"root/d/.ndignore": {},
|
||||
"root/d/f1.mp3": {},
|
||||
"root/d/f2.mp3": {},
|
||||
"root/d/f3.mp3": {},
|
||||
"root/e/original/f1.mp3": {},
|
||||
"root/e/symlink": {Mode: fs.ModeSymlink, Data: []byte("original")},
|
||||
Context("full library", func() {
|
||||
BeforeEach(func() {
|
||||
DeferCleanup(configtest.SetupConfig())
|
||||
ctx = GinkgoT().Context()
|
||||
fsys = &mockMusicFS{
|
||||
FS: fstest.MapFS{
|
||||
"root/a/.ndignore": {Data: []byte("ignored/*")},
|
||||
"root/a/f1.mp3": {},
|
||||
"root/a/f2.mp3": {},
|
||||
"root/a/ignored/bad.mp3": {},
|
||||
"root/b/cover.jpg": {},
|
||||
"root/c/f3": {},
|
||||
"root/d": {},
|
||||
"root/d/.ndignore": {},
|
||||
"root/d/f1.mp3": {},
|
||||
"root/d/f2.mp3": {},
|
||||
"root/d/f3.mp3": {},
|
||||
"root/e/original/f1.mp3": {},
|
||||
"root/e/symlink": {Mode: fs.ModeSymlink, Data: []byte("original")},
|
||||
},
|
||||
}
|
||||
job = &scanJob{
|
||||
fs: fsys,
|
||||
lib: model.Library{Path: "/music"},
|
||||
}
|
||||
})
|
||||
|
||||
// Helper function to call walkDirTree and collect folders from the results channel
|
||||
getFolders := func() map[string]*folderEntry {
|
||||
results, err := walkDirTree(ctx, job)
|
||||
Expect(err).ToNot(HaveOccurred())
|
||||
|
||||
folders := map[string]*folderEntry{}
|
||||
g := errgroup.Group{}
|
||||
g.Go(func() error {
|
||||
for folder := range results {
|
||||
folders[folder.path] = folder
|
||||
}
|
||||
return nil
|
||||
})
|
||||
_ = g.Wait()
|
||||
return folders
|
||||
}
|
||||
|
||||
DescribeTable("symlink handling",
|
||||
func(followSymlinks bool, expectedFolderCount int) {
|
||||
conf.Server.Scanner.FollowSymlinks = followSymlinks
|
||||
folders := getFolders()
|
||||
|
||||
Expect(folders).To(HaveLen(expectedFolderCount + 2)) // +2 for `.` and `root`
|
||||
|
||||
// Basic folder structure checks
|
||||
Expect(folders["root/a"].audioFiles).To(SatisfyAll(
|
||||
HaveLen(2),
|
||||
HaveKey("f1.mp3"),
|
||||
HaveKey("f2.mp3"),
|
||||
))
|
||||
Expect(folders["root/a"].imageFiles).To(BeEmpty())
|
||||
Expect(folders["root/b"].audioFiles).To(BeEmpty())
|
||||
Expect(folders["root/b"].imageFiles).To(SatisfyAll(
|
||||
HaveLen(1),
|
||||
HaveKey("cover.jpg"),
|
||||
))
|
||||
Expect(folders["root/c"].audioFiles).To(BeEmpty())
|
||||
Expect(folders["root/c"].imageFiles).To(BeEmpty())
|
||||
Expect(folders).ToNot(HaveKey("root/d"))
|
||||
|
||||
// Symlink specific checks
|
||||
if followSymlinks {
|
||||
Expect(folders["root/e/symlink"].audioFiles).To(HaveLen(1))
|
||||
} else {
|
||||
Expect(folders).ToNot(HaveKey("root/e/symlink"))
|
||||
}
|
||||
},
|
||||
}
|
||||
job = &scanJob{
|
||||
fs: fsys,
|
||||
lib: model.Library{Path: "/music"},
|
||||
}
|
||||
Entry("with symlinks enabled", true, 7),
|
||||
Entry("with symlinks disabled", false, 6),
|
||||
)
|
||||
})
|
||||
|
||||
// Helper function to call walkDirTree and collect folders from the results channel
|
||||
getFolders := func() map[string]*folderEntry {
|
||||
results, err := walkDirTree(ctx, job)
|
||||
Expect(err).ToNot(HaveOccurred())
|
||||
|
||||
folders := map[string]*folderEntry{}
|
||||
g := errgroup.Group{}
|
||||
g.Go(func() error {
|
||||
for folder := range results {
|
||||
folders[folder.path] = folder
|
||||
Context("with target folders", func() {
|
||||
BeforeEach(func() {
|
||||
DeferCleanup(configtest.SetupConfig())
|
||||
ctx = GinkgoT().Context()
|
||||
fsys = &mockMusicFS{
|
||||
FS: fstest.MapFS{
|
||||
"Artist/Album1/track1.mp3": {},
|
||||
"Artist/Album1/track2.mp3": {},
|
||||
"Artist/Album2/track1.mp3": {},
|
||||
"Artist/Album2/track2.mp3": {},
|
||||
"Artist/Album2/Sub/track3.mp3": {},
|
||||
"OtherArtist/Album3/track1.mp3": {},
|
||||
},
|
||||
}
|
||||
job = &scanJob{
|
||||
fs: fsys,
|
||||
lib: model.Library{Path: "/music"},
|
||||
}
|
||||
return nil
|
||||
})
|
||||
_ = g.Wait()
|
||||
return folders
|
||||
}
|
||||
|
||||
DescribeTable("symlink handling",
|
||||
func(followSymlinks bool, expectedFolderCount int) {
|
||||
conf.Server.Scanner.FollowSymlinks = followSymlinks
|
||||
folders := getFolders()
|
||||
It("should recursively walk all subdirectories of target folders", func() {
|
||||
results, err := walkDirTree(ctx, job, "Artist")
|
||||
Expect(err).ToNot(HaveOccurred())
|
||||
|
||||
Expect(folders).To(HaveLen(expectedFolderCount + 2)) // +2 for `.` and `root`
|
||||
folders := map[string]*folderEntry{}
|
||||
g := errgroup.Group{}
|
||||
g.Go(func() error {
|
||||
for folder := range results {
|
||||
folders[folder.path] = folder
|
||||
}
|
||||
return nil
|
||||
})
|
||||
_ = g.Wait()
|
||||
|
||||
// Basic folder structure checks
|
||||
Expect(folders["root/a"].audioFiles).To(SatisfyAll(
|
||||
HaveLen(2),
|
||||
HaveKey("f1.mp3"),
|
||||
HaveKey("f2.mp3"),
|
||||
// Should include the target folder and all its descendants
|
||||
Expect(folders).To(SatisfyAll(
|
||||
HaveKey("Artist"),
|
||||
HaveKey("Artist/Album1"),
|
||||
HaveKey("Artist/Album2"),
|
||||
HaveKey("Artist/Album2/Sub"),
|
||||
))
|
||||
Expect(folders["root/a"].imageFiles).To(BeEmpty())
|
||||
Expect(folders["root/b"].audioFiles).To(BeEmpty())
|
||||
Expect(folders["root/b"].imageFiles).To(SatisfyAll(
|
||||
HaveLen(1),
|
||||
HaveKey("cover.jpg"),
|
||||
))
|
||||
Expect(folders["root/c"].audioFiles).To(BeEmpty())
|
||||
Expect(folders["root/c"].imageFiles).To(BeEmpty())
|
||||
Expect(folders).ToNot(HaveKey("root/d"))
|
||||
|
||||
// Symlink specific checks
|
||||
if followSymlinks {
|
||||
Expect(folders["root/e/symlink"].audioFiles).To(HaveLen(1))
|
||||
} else {
|
||||
Expect(folders).ToNot(HaveKey("root/e/symlink"))
|
||||
// Should not include folders outside the target
|
||||
Expect(folders).ToNot(HaveKey("OtherArtist"))
|
||||
Expect(folders).ToNot(HaveKey("OtherArtist/Album3"))
|
||||
|
||||
// Verify audio files are present
|
||||
Expect(folders["Artist/Album1"].audioFiles).To(HaveLen(2))
|
||||
Expect(folders["Artist/Album2"].audioFiles).To(HaveLen(2))
|
||||
Expect(folders["Artist/Album2/Sub"].audioFiles).To(HaveLen(1))
|
||||
})
|
||||
|
||||
It("should handle multiple target folders", func() {
|
||||
results, err := walkDirTree(ctx, job, "Artist/Album1", "OtherArtist")
|
||||
Expect(err).ToNot(HaveOccurred())
|
||||
|
||||
folders := map[string]*folderEntry{}
|
||||
g := errgroup.Group{}
|
||||
g.Go(func() error {
|
||||
for folder := range results {
|
||||
folders[folder.path] = folder
|
||||
}
|
||||
return nil
|
||||
})
|
||||
_ = g.Wait()
|
||||
|
||||
// Should include both target folders and their descendants
|
||||
Expect(folders).To(SatisfyAll(
|
||||
HaveKey("Artist/Album1"),
|
||||
HaveKey("OtherArtist"),
|
||||
HaveKey("OtherArtist/Album3"),
|
||||
))
|
||||
|
||||
// Should not include other folders
|
||||
Expect(folders).ToNot(HaveKey("Artist"))
|
||||
Expect(folders).ToNot(HaveKey("Artist/Album2"))
|
||||
Expect(folders).ToNot(HaveKey("Artist/Album2/Sub"))
|
||||
})
|
||||
|
||||
It("should skip non-existent target folders and preserve them in lastUpdates", func() {
|
||||
// Setup job with lastUpdates for both existing and non-existing folders
|
||||
job.lastUpdates = map[string]model.FolderUpdateInfo{
|
||||
model.FolderID(job.lib, "Artist/Album1"): {},
|
||||
model.FolderID(job.lib, "NonExistent/DeletedFolder"): {},
|
||||
model.FolderID(job.lib, "OtherArtist/Album3"): {},
|
||||
}
|
||||
},
|
||||
Entry("with symlinks enabled", true, 7),
|
||||
Entry("with symlinks disabled", false, 6),
|
||||
)
|
||||
|
||||
// Try to scan existing folder and non-existing folder
|
||||
results, err := walkDirTree(ctx, job, "Artist/Album1", "NonExistent/DeletedFolder")
|
||||
Expect(err).ToNot(HaveOccurred())
|
||||
|
||||
// Collect results
|
||||
folders := map[string]struct{}{}
|
||||
for folder := range results {
|
||||
folders[folder.path] = struct{}{}
|
||||
}
|
||||
|
||||
// Should only include the existing folder
|
||||
Expect(folders).To(HaveKey("Artist/Album1"))
|
||||
Expect(folders).ToNot(HaveKey("NonExistent/DeletedFolder"))
|
||||
|
||||
// The non-existent folder should still be in lastUpdates (not removed by popLastUpdate)
|
||||
Expect(job.lastUpdates).To(HaveKey(model.FolderID(job.lib, "NonExistent/DeletedFolder")))
|
||||
|
||||
// The existing folder should have been removed from lastUpdates
|
||||
Expect(job.lastUpdates).ToNot(HaveKey(model.FolderID(job.lib, "Artist/Album1")))
|
||||
|
||||
// Folders not in targets should remain in lastUpdates
|
||||
Expect(job.lastUpdates).To(HaveKey(model.FolderID(job.lib, "OtherArtist/Album3")))
|
||||
})
|
||||
})
|
||||
})
|
||||
|
||||
Describe("helper functions", func() {
|
||||
|
||||
Reference in New Issue
Block a user