feat(scanner): implement selective folder scanning and file system watcher improvements (#4674)
* feat: Add selective folder scanning capability Implement targeted scanning of specific library/folder pairs without full recursion. This enables efficient rescanning of individual folders when changes are detected, significantly reducing scan time for large libraries. Key changes: - Add ScanTarget struct and ScanFolders API to Scanner interface - Implement CLI flag --targets for specifying libraryID:folderPath pairs - Add FolderRepository.GetByPaths() for batch folder info retrieval - Create loadSpecificFolders() for non-recursive directory loading - Scope GC operations to affected libraries only (with TODO for full impl) - Add comprehensive tests for selective scanning behavior The selective scan: - Only processes specified folders (no subdirectory recursion) - Maintains library isolation - Runs full maintenance pipeline scoped to affected libraries - Supports both full and quick scan modes Examples: navidrome scan --targets "1:Music/Rock,1:Music/Jazz" navidrome scan --full --targets "2:Classical" * feat(folder): replace GetByPaths with GetFolderUpdateInfo for improved folder updates retrieval Signed-off-by: Deluan <deluan@navidrome.org> * test: update parseTargets test to handle folder names with spaces Signed-off-by: Deluan <deluan@navidrome.org> * refactor(folder): remove unused LibraryPath struct and update GC logging message Signed-off-by: Deluan <deluan@navidrome.org> * refactor(folder): enhance external scanner to support target-specific scanning Signed-off-by: Deluan <deluan@navidrome.org> * refactor(scanner): simplify scanner methods Signed-off-by: Deluan <deluan@navidrome.org> * feat(watcher): implement folder scanning notifications with deduplication Signed-off-by: Deluan <deluan@navidrome.org> * refactor(watcher): add resolveFolderPath function for testability Signed-off-by: Deluan <deluan@navidrome.org> * feat(watcher): implement path ignoring based on .ndignore patterns Signed-off-by: Deluan <deluan@navidrome.org> * refactor(scanner): implement IgnoreChecker for managing .ndignore patterns Signed-off-by: Deluan <deluan@navidrome.org> * refactor(ignore_checker): rename scanner to lineScanner for clarity Signed-off-by: Deluan <deluan@navidrome.org> * refactor(scanner): enhance ScanTarget struct with String method for better target representation Signed-off-by: Deluan <deluan@navidrome.org> * fix(scanner): validate library ID to prevent negative values Signed-off-by: Deluan <deluan@navidrome.org> * refactor(scanner): simplify GC method by removing library ID parameter Signed-off-by: Deluan <deluan@navidrome.org> * feat(scanner): update folder scanning to include all descendants of specified folders Signed-off-by: Deluan <deluan@navidrome.org> * feat(subsonic): allow selective scan in the /startScan endpoint Signed-off-by: Deluan <deluan@navidrome.org> * refactor(scanner): update CallScan to handle specific library/folder pairs Signed-off-by: Deluan <deluan@navidrome.org> * refactor(scanner): streamline scanning logic by removing scanAll method Signed-off-by: Deluan <deluan@navidrome.org> * test: enhance mockScanner for thread safety and improve test reliability Signed-off-by: Deluan <deluan@navidrome.org> * refactor(scanner): move scanner.ScanTarget to model.ScanTarget Signed-off-by: Deluan <deluan@navidrome.org> * refactor: move scanner types to model,implement MockScanner Signed-off-by: Deluan <deluan@navidrome.org> * refactor(scanner): update scanner interface and implementations to use model.Scanner Signed-off-by: Deluan <deluan@navidrome.org> * refactor(folder_repository): normalize target path handling by using filepath.Clean Signed-off-by: Deluan <deluan@navidrome.org> * test(folder_repository): add comprehensive tests for folder retrieval and child exclusion Signed-off-by: Deluan <deluan@navidrome.org> * refactor(scanner): simplify selective scan logic using slice.Filter Signed-off-by: Deluan <deluan@navidrome.org> * refactor(scanner): streamline phase folder and album creation by removing unnecessary library parameter Signed-off-by: Deluan <deluan@navidrome.org> * refactor(scanner): move initialization logic from phase_1 to the scanner itself Signed-off-by: Deluan <deluan@navidrome.org> * refactor(tests): rename selective scan test file to scanner_selective_test.go Signed-off-by: Deluan <deluan@navidrome.org> * feat(configuration): add DevSelectiveWatcher configuration option Signed-off-by: Deluan <deluan@navidrome.org> * feat(watcher): enhance .ndignore handling for folder deletions and file changes Signed-off-by: Deluan <deluan@navidrome.org> * docs(scanner): comments Signed-off-by: Deluan <deluan@navidrome.org> * refactor(scanner): enhance walkDirTree to support target folder scanning Signed-off-by: Deluan <deluan@navidrome.org> * fix(scanner, watcher): handle errors when pushing ignore patterns for folders Signed-off-by: Deluan <deluan@navidrome.org> * Update scanner/phase_1_folders.go Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com> * refactor(scanner): replace parseTargets function with direct call to scanner.ParseTargets Signed-off-by: Deluan <deluan@navidrome.org> * test(scanner): add tests for ScanBegin and ScanEnd functionality Signed-off-by: Deluan <deluan@navidrome.org> * fix(library): update PRAGMA optimize to check table sizes without ANALYZE Signed-off-by: Deluan <deluan@navidrome.org> * test(scanner): refactor tests Signed-off-by: Deluan <deluan@navidrome.org> * feat(ui): add selective scan options and update translations Signed-off-by: Deluan <deluan@navidrome.org> * feat(ui): add quick and full scan options for individual libraries Signed-off-by: Deluan <deluan@navidrome.org> * feat(ui): add Scan buttonsto the LibraryList Signed-off-by: Deluan <deluan@navidrome.org> * feat(scan): update scanning parameters from 'path' to 'target' for selective scans. * refactor(scan): move ParseTargets function to model package * test(scan): suppress unused return value from SetUserLibraries in tests * feat(gc): enhance garbage collection to support selective library purging Signed-off-by: Deluan <deluan@navidrome.org> * fix(scanner): prevent race condition when scanning deleted folders When the watcher detects changes in a folder that gets deleted before the scanner runs (due to the 10-second delay), the scanner was prematurely removing these folders from the tracking map, preventing them from being marked as missing. The issue occurred because `newFolderEntry` was calling `popLastUpdate` before verifying the folder actually exists on the filesystem. Changes: - Move fs.Stat check before newFolderEntry creation in loadDir to ensure deleted folders remain in lastUpdates for finalize() to handle - Add early existence check in walkDirTree to skip non-existent target folders with a warning log - Add unit test verifying non-existent folders aren't removed from lastUpdates prematurely - Add integration test for deleted folder scenario with ScanFolders Fixes the issue where deleting entire folders (e.g., /music/AC_DC) wouldn't mark tracks as missing when using selective folder scanning. * refactor(scan): streamline folder entry creation and update handling Signed-off-by: Deluan <deluan@navidrome.org> * feat(scan): add '@Recycle' (QNAP) to ignored directories list Signed-off-by: Deluan <deluan@navidrome.org> * fix(log): improve thread safety in logging level management * test(scan): move unit tests for ParseTargets function Signed-off-by: Deluan <deluan@navidrome.org> * review Signed-off-by: Deluan <deluan@navidrome.org> --------- Signed-off-by: Deluan <deluan@navidrome.org> Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com> Co-authored-by: deluan <deluan.quintao@mechanical-orchard.com>
This commit is contained in:
Deluan Quintão
+39
-39
@@ -26,58 +26,46 @@ import (
|
||||
"github.com/navidrome/navidrome/utils/slice"
|
||||
)
|
||||
|
||||
func createPhaseFolders(ctx context.Context, state *scanState, ds model.DataStore, cw artwork.CacheWarmer, libs []model.Library) *phaseFolders {
|
||||
func createPhaseFolders(ctx context.Context, state *scanState, ds model.DataStore, cw artwork.CacheWarmer) *phaseFolders {
|
||||
var jobs []*scanJob
|
||||
var updatedLibs []model.Library
|
||||
for _, lib := range libs {
|
||||
if lib.LastScanStartedAt.IsZero() {
|
||||
err := ds.Library(ctx).ScanBegin(lib.ID, state.fullScan)
|
||||
if err != nil {
|
||||
log.Error(ctx, "Scanner: Error updating last scan started at", "lib", lib.Name, err)
|
||||
state.sendWarning(err.Error())
|
||||
continue
|
||||
}
|
||||
// Reload library to get updated state
|
||||
l, err := ds.Library(ctx).Get(lib.ID)
|
||||
if err != nil {
|
||||
log.Error(ctx, "Scanner: Error reloading library", "lib", lib.Name, err)
|
||||
state.sendWarning(err.Error())
|
||||
continue
|
||||
}
|
||||
lib = *l
|
||||
} else {
|
||||
log.Debug(ctx, "Scanner: Resuming previous scan", "lib", lib.Name, "lastScanStartedAt", lib.LastScanStartedAt, "fullScan", lib.FullScanInProgress)
|
||||
|
||||
// Create scan jobs for all libraries
|
||||
for _, lib := range state.libraries {
|
||||
// Get target folders for this library if selective scan
|
||||
var targetFolders []string
|
||||
if state.isSelectiveScan() {
|
||||
targetFolders = state.targets[lib.ID]
|
||||
}
|
||||
job, err := newScanJob(ctx, ds, cw, lib, state.fullScan)
|
||||
|
||||
job, err := newScanJob(ctx, ds, cw, lib, state.fullScan, targetFolders)
|
||||
if err != nil {
|
||||
log.Error(ctx, "Scanner: Error creating scan context", "lib", lib.Name, err)
|
||||
state.sendWarning(err.Error())
|
||||
continue
|
||||
}
|
||||
jobs = append(jobs, job)
|
||||
updatedLibs = append(updatedLibs, lib)
|
||||
}
|
||||
|
||||
// Update the state with the libraries that have been processed and have their scan timestamps set
|
||||
state.libraries = updatedLibs
|
||||
|
||||
return &phaseFolders{jobs: jobs, ctx: ctx, ds: ds, state: state}
|
||||
}
|
||||
|
||||
type scanJob struct {
|
||||
lib model.Library
|
||||
fs storage.MusicFS
|
||||
cw artwork.CacheWarmer
|
||||
lastUpdates map[string]model.FolderUpdateInfo
|
||||
lock sync.Mutex
|
||||
numFolders atomic.Int64
|
||||
lib model.Library
|
||||
fs storage.MusicFS
|
||||
cw artwork.CacheWarmer
|
||||
lastUpdates map[string]model.FolderUpdateInfo // Holds last update info for all (DB) folders in this library
|
||||
targetFolders []string // Specific folders to scan (including all descendants)
|
||||
lock sync.Mutex
|
||||
numFolders atomic.Int64
|
||||
}
|
||||
|
||||
func newScanJob(ctx context.Context, ds model.DataStore, cw artwork.CacheWarmer, lib model.Library, fullScan bool) (*scanJob, error) {
|
||||
lastUpdates, err := ds.Folder(ctx).GetLastUpdates(lib)
|
||||
func newScanJob(ctx context.Context, ds model.DataStore, cw artwork.CacheWarmer, lib model.Library, fullScan bool, targetFolders []string) (*scanJob, error) {
|
||||
// Get folder updates, optionally filtered to specific target folders
|
||||
lastUpdates, err := ds.Folder(ctx).GetFolderUpdateInfo(lib, targetFolders...)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("getting last updates: %w", err)
|
||||
}
|
||||
|
||||
fileStore, err := storage.For(lib.Path)
|
||||
if err != nil {
|
||||
log.Error(ctx, "Error getting storage for library", "library", lib.Name, "path", lib.Path, err)
|
||||
@@ -88,15 +76,17 @@ func newScanJob(ctx context.Context, ds model.DataStore, cw artwork.CacheWarmer,
|
||||
log.Error(ctx, "Error getting fs for library", "library", lib.Name, "path", lib.Path, err)
|
||||
return nil, fmt.Errorf("getting fs for library: %w", err)
|
||||
}
|
||||
lib.FullScanInProgress = lib.FullScanInProgress || fullScan
|
||||
return &scanJob{
|
||||
lib: lib,
|
||||
fs: fsys,
|
||||
cw: cw,
|
||||
lastUpdates: lastUpdates,
|
||||
lib: lib,
|
||||
fs: fsys,
|
||||
cw: cw,
|
||||
lastUpdates: lastUpdates,
|
||||
targetFolders: targetFolders,
|
||||
}, nil
|
||||
}
|
||||
|
||||
// popLastUpdate retrieves and removes the last update info for the given folder ID
|
||||
// This is used to track which folders have been found during the walk_dir_tree
|
||||
func (j *scanJob) popLastUpdate(folderID string) model.FolderUpdateInfo {
|
||||
j.lock.Lock()
|
||||
defer j.lock.Unlock()
|
||||
@@ -106,6 +96,15 @@ func (j *scanJob) popLastUpdate(folderID string) model.FolderUpdateInfo {
|
||||
return lastUpdate
|
||||
}
|
||||
|
||||
// createFolderEntry creates a new folderEntry for the given path, using the last update info from the job
|
||||
// to populate the previous update time and hash. It also removes the folder from the job's lastUpdates map.
|
||||
// This is used to track which folders have been found during the walk_dir_tree.
|
||||
func (j *scanJob) createFolderEntry(path string) *folderEntry {
|
||||
id := model.FolderID(j.lib, path)
|
||||
info := j.popLastUpdate(id)
|
||||
return newFolderEntry(j, id, path, info.UpdatedAt, info.Hash)
|
||||
}
|
||||
|
||||
// phaseFolders represents the first phase of the scanning process, which is responsible
|
||||
// for scanning all libraries and importing new or updated files. This phase involves
|
||||
// traversing the directory tree of each library, identifying new or modified media files,
|
||||
@@ -144,7 +143,8 @@ func (p *phaseFolders) producer() ppl.Producer[*folderEntry] {
|
||||
if utils.IsCtxDone(p.ctx) {
|
||||
break
|
||||
}
|
||||
outputChan, err := walkDirTree(p.ctx, job)
|
||||
|
||||
outputChan, err := walkDirTree(p.ctx, job, job.targetFolders...)
|
||||
if err != nil {
|
||||
log.Warn(p.ctx, "Scanner: Error scanning library", "lib", job.lib.Name, err)
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user