feat(scanner): implement selective folder scanning and file system watcher improvements (#4674)

* feat: Add selective folder scanning capability Implement targeted scanning of specific library/folder pairs without full recursion. This enables efficient rescanning of individual folders when changes are detected, significantly reducing scan time for large libraries. Key changes: - Add ScanTarget struct and ScanFolders API to Scanner interface - Implement CLI flag --targets for specifying libraryID:folderPath pairs - Add FolderRepository.GetByPaths() for batch folder info retrieval - Create loadSpecificFolders() for non-recursive directory loading - Scope GC operations to affected libraries only (with TODO for full impl) - Add comprehensive tests for selective scanning behavior The selective scan: - Only processes specified folders (no subdirectory recursion) - Maintains library isolation - Runs full maintenance pipeline scoped to affected libraries - Supports both full and quick scan modes Examples: navidrome scan --targets "1:Music/Rock,1:Music/Jazz" navidrome scan --full --targets "2:Classical" * feat(folder): replace GetByPaths with GetFolderUpdateInfo for improved folder updates retrieval Signed-off-by: Deluan <deluan@navidrome.org> * test: update parseTargets test to handle folder names with spaces Signed-off-by: Deluan <deluan@navidrome.org> * refactor(folder): remove unused LibraryPath struct and update GC logging message Signed-off-by: Deluan <deluan@navidrome.org> * refactor(folder): enhance external scanner to support target-specific scanning Signed-off-by: Deluan <deluan@navidrome.org> * refactor(scanner): simplify scanner methods Signed-off-by: Deluan <deluan@navidrome.org> * feat(watcher): implement folder scanning notifications with deduplication Signed-off-by: Deluan <deluan@navidrome.org> * refactor(watcher): add resolveFolderPath function for testability Signed-off-by: Deluan <deluan@navidrome.org> * feat(watcher): implement path ignoring based on .ndignore patterns Signed-off-by: Deluan <deluan@navidrome.org> * refactor(scanner): implement IgnoreChecker for managing .ndignore patterns Signed-off-by: Deluan <deluan@navidrome.org> * refactor(ignore_checker): rename scanner to lineScanner for clarity Signed-off-by: Deluan <deluan@navidrome.org> * refactor(scanner): enhance ScanTarget struct with String method for better target representation Signed-off-by: Deluan <deluan@navidrome.org> * fix(scanner): validate library ID to prevent negative values Signed-off-by: Deluan <deluan@navidrome.org> * refactor(scanner): simplify GC method by removing library ID parameter Signed-off-by: Deluan <deluan@navidrome.org> * feat(scanner): update folder scanning to include all descendants of specified folders Signed-off-by: Deluan <deluan@navidrome.org> * feat(subsonic): allow selective scan in the /startScan endpoint Signed-off-by: Deluan <deluan@navidrome.org> * refactor(scanner): update CallScan to handle specific library/folder pairs Signed-off-by: Deluan <deluan@navidrome.org> * refactor(scanner): streamline scanning logic by removing scanAll method Signed-off-by: Deluan <deluan@navidrome.org> * test: enhance mockScanner for thread safety and improve test reliability Signed-off-by: Deluan <deluan@navidrome.org> * refactor(scanner): move scanner.ScanTarget to model.ScanTarget Signed-off-by: Deluan <deluan@navidrome.org> * refactor: move scanner types to model,implement MockScanner Signed-off-by: Deluan <deluan@navidrome.org> * refactor(scanner): update scanner interface and implementations to use model.Scanner Signed-off-by: Deluan <deluan@navidrome.org> * refactor(folder_repository): normalize target path handling by using filepath.Clean Signed-off-by: Deluan <deluan@navidrome.org> * test(folder_repository): add comprehensive tests for folder retrieval and child exclusion Signed-off-by: Deluan <deluan@navidrome.org> * refactor(scanner): simplify selective scan logic using slice.Filter Signed-off-by: Deluan <deluan@navidrome.org> * refactor(scanner): streamline phase folder and album creation by removing unnecessary library parameter Signed-off-by: Deluan <deluan@navidrome.org> * refactor(scanner): move initialization logic from phase_1 to the scanner itself Signed-off-by: Deluan <deluan@navidrome.org> * refactor(tests): rename selective scan test file to scanner_selective_test.go Signed-off-by: Deluan <deluan@navidrome.org> * feat(configuration): add DevSelectiveWatcher configuration option Signed-off-by: Deluan <deluan@navidrome.org> * feat(watcher): enhance .ndignore handling for folder deletions and file changes Signed-off-by: Deluan <deluan@navidrome.org> * docs(scanner): comments Signed-off-by: Deluan <deluan@navidrome.org> * refactor(scanner): enhance walkDirTree to support target folder scanning Signed-off-by: Deluan <deluan@navidrome.org> * fix(scanner, watcher): handle errors when pushing ignore patterns for folders Signed-off-by: Deluan <deluan@navidrome.org> * Update scanner/phase_1_folders.go Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com> * refactor(scanner): replace parseTargets function with direct call to scanner.ParseTargets Signed-off-by: Deluan <deluan@navidrome.org> * test(scanner): add tests for ScanBegin and ScanEnd functionality Signed-off-by: Deluan <deluan@navidrome.org> * fix(library): update PRAGMA optimize to check table sizes without ANALYZE Signed-off-by: Deluan <deluan@navidrome.org> * test(scanner): refactor tests Signed-off-by: Deluan <deluan@navidrome.org> * feat(ui): add selective scan options and update translations Signed-off-by: Deluan <deluan@navidrome.org> * feat(ui): add quick and full scan options for individual libraries Signed-off-by: Deluan <deluan@navidrome.org> * feat(ui): add Scan buttonsto the LibraryList Signed-off-by: Deluan <deluan@navidrome.org> * feat(scan): update scanning parameters from 'path' to 'target' for selective scans. * refactor(scan): move ParseTargets function to model package * test(scan): suppress unused return value from SetUserLibraries in tests * feat(gc): enhance garbage collection to support selective library purging Signed-off-by: Deluan <deluan@navidrome.org> * fix(scanner): prevent race condition when scanning deleted folders When the watcher detects changes in a folder that gets deleted before the scanner runs (due to the 10-second delay), the scanner was prematurely removing these folders from the tracking map, preventing them from being marked as missing. The issue occurred because `newFolderEntry` was calling `popLastUpdate` before verifying the folder actually exists on the filesystem. Changes: - Move fs.Stat check before newFolderEntry creation in loadDir to ensure deleted folders remain in lastUpdates for finalize() to handle - Add early existence check in walkDirTree to skip non-existent target folders with a warning log - Add unit test verifying non-existent folders aren't removed from lastUpdates prematurely - Add integration test for deleted folder scenario with ScanFolders Fixes the issue where deleting entire folders (e.g., /music/AC_DC) wouldn't mark tracks as missing when using selective folder scanning. * refactor(scan): streamline folder entry creation and update handling Signed-off-by: Deluan <deluan@navidrome.org> * feat(scan): add '@Recycle' (QNAP) to ignored directories list Signed-off-by: Deluan <deluan@navidrome.org> * fix(log): improve thread safety in logging level management * test(scan): move unit tests for ParseTargets function Signed-off-by: Deluan <deluan@navidrome.org> * review Signed-off-by: Deluan <deluan@navidrome.org> --------- Signed-off-by: Deluan <deluan@navidrome.org> Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com> Co-authored-by: deluan <deluan.quintao@mechanical-orchard.com>
2025-11-14 22:15:43 -05:00
parent bca76069c3
commit 28d5299ffc
52 changed files with 3221 additions and 374 deletions
@@ -24,9 +24,9 @@ type Watcher interface {
 type watcher struct {
 	mainCtx         context.Context
 	ds              model.DataStore
-	scanner         Scanner
+	scanner         model.Scanner
 	triggerWait     time.Duration
-	watcherNotify   chan model.Library
+	watcherNotify   chan scanNotification
 	libraryWatchers map[int]*libraryWatcherInstance
 	mu              sync.RWMutex
 }
@@ -36,14 +36,19 @@ type libraryWatcherInstance struct {
 	cancel  context.CancelFunc
 }

+type scanNotification struct {
+	Library    *model.Library
+	FolderPath string
+}
+
 // GetWatcher returns the watcher singleton
-func GetWatcher(ds model.DataStore, s Scanner) Watcher {
+func GetWatcher(ds model.DataStore, s model.Scanner) Watcher {
 	return singleton.GetInstance(func() *watcher {
 		return &watcher{
 			ds:              ds,
 			scanner:         s,
 			triggerWait:     conf.Server.Scanner.WatcherWait,
-			watcherNotify:   make(chan model.Library, 1),
+			watcherNotify:   make(chan scanNotification, 1),
 			libraryWatchers: make(map[int]*libraryWatcherInstance),
 		}
 	})
@@ -68,11 +73,11 @@ func (w *watcher) Run(ctx context.Context) error {
 	// Main scan triggering loop
 	trigger := time.NewTimer(w.triggerWait)
 	trigger.Stop()
-	waiting := false
+	targets := make(map[model.ScanTarget]struct{})
 	for {
 		select {
 		case <-trigger.C:
-			log.Info("Watcher: Triggering scan")
+			log.Info("Watcher: Triggering scan for changed folders", "numTargets", len(targets))
 			status, err := w.scanner.Status(ctx)
 			if err != nil {
 				log.Error(ctx, "Watcher: Error retrieving Scanner status", err)
@@ -83,9 +88,23 @@ func (w *watcher) Run(ctx context.Context) error {
 				trigger.Reset(w.triggerWait * 3)
 				continue
 			}
-			waiting = false
+
+			// Convert targets map to slice
+			targetSlice := make([]model.ScanTarget, 0, len(targets))
+			for target := range targets {
+				targetSlice = append(targetSlice, target)
+			}
+
+			// Clear targets for next batch
+			targets = make(map[model.ScanTarget]struct{})
+
 			go func() {
-				_, err := w.scanner.ScanAll(ctx, false)
+				var err error
+				if conf.Server.DevSelectiveWatcher {
+					_, err = w.scanner.ScanFolders(ctx, false, targetSlice)
+				} else {
+					_, err = w.scanner.ScanAll(ctx, false)
+				}
 				if err != nil {
 					log.Error(ctx, "Watcher: Error scanning", err)
 				} else {
@@ -102,13 +121,20 @@ func (w *watcher) Run(ctx context.Context) error {
 			w.libraryWatchers = make(map[int]*libraryWatcherInstance)
 			w.mu.Unlock()
 			return nil
-		case lib := <-w.watcherNotify:
-			if !waiting {
-				log.Debug(ctx, "Watcher: Detected changes. Waiting for more changes before triggering scan",
-					"libraryID", lib.ID, "name", lib.Name, "path", lib.Path)
-				waiting = true
+		case notification := <-w.watcherNotify:
+			lib := notification.Library
+			folderPath := notification.FolderPath
+
+			// If already scheduled for scan, skip
+			target := model.ScanTarget{LibraryID: lib.ID, FolderPath: folderPath}
+			if _, exists := targets[target]; exists {
+				continue
 			}
+			targets[target] = struct{}{}
 			trigger.Reset(w.triggerWait)
+
+			log.Debug(ctx, "Watcher: Detected changes. Waiting for more changes before triggering scan",
+				"libraryID", lib.ID, "name", lib.Name, "path", lib.Path, "folderPath", folderPath)
 		}
 	}
 }
@@ -199,13 +225,18 @@ func (w *watcher) watchLibrary(ctx context.Context, lib *model.Library) error {

 	log.Info(ctx, "Watcher started for library", "libraryID", lib.ID, "name", lib.Name, "path", lib.Path, "absoluteLibPath", absLibPath)

+	return w.processLibraryEvents(ctx, lib, fsys, c, absLibPath)
+}
+
+// processLibraryEvents processes filesystem events for a library.
+func (w *watcher) processLibraryEvents(ctx context.Context, lib *model.Library, fsys storage.MusicFS, events <-chan string, absLibPath string) error {
 	for {
 		select {
 		case <-ctx.Done():
 			log.Debug(ctx, "Watcher stopped due to context cancellation", "libraryID", lib.ID, "name", lib.Name)
 			return nil
-		case path := <-c:
-			path, err = filepath.Rel(absLibPath, path)
+		case path := <-events:
+			path, err := filepath.Rel(absLibPath, path)
 			if err != nil {
 				log.Error(ctx, "Error getting relative path", "libraryID", lib.ID, "absolutePath", absLibPath, "path", path, err)
 				continue
@@ -215,12 +246,27 @@ func (w *watcher) watchLibrary(ctx context.Context, lib *model.Library) error {
 				log.Trace(ctx, "Ignoring change", "libraryID", lib.ID, "path", path)
 				continue
 			}
-
 			log.Trace(ctx, "Detected change", "libraryID", lib.ID, "path", path, "absoluteLibPath", absLibPath)

+			// Check if the original path (before resolution) matches .ndignore patterns
+			// This is crucial for deleted folders - if a deleted folder matches .ndignore,
+			// we should ignore it BEFORE resolveFolderPath walks up to the parent
+			if w.shouldIgnoreFolderPath(ctx, fsys, path) {
+				log.Debug(ctx, "Ignoring change matching .ndignore pattern", "libraryID", lib.ID, "path", path)
+				continue
+			}
+
+			// Find the folder to scan - validate path exists as directory, walk up if needed
+			folderPath := resolveFolderPath(fsys, path)
+			// Double-check after resolution in case the resolved path is different and also matches patterns
+			if folderPath != path && w.shouldIgnoreFolderPath(ctx, fsys, folderPath) {
+				log.Trace(ctx, "Ignoring change in folder matching .ndignore pattern", "libraryID", lib.ID, "folderPath", folderPath)
+				continue
+			}
+
 			// Notify the main watcher of changes
 			select {
-			case w.watcherNotify <- *lib:
+			case w.watcherNotify <- scanNotification{Library: lib, FolderPath: folderPath}:
 			default:
 				// Channel is full, notification already pending
 			}
@@ -228,6 +274,47 @@ func (w *watcher) watchLibrary(ctx context.Context, lib *model.Library) error {
 	}
 }

+// resolveFolderPath takes a path (which may be a file or directory) and returns
+// the folder path to scan. If the path is a file, it walks up to find the parent
+// directory. Returns empty string if the path should scan the library root.
+func resolveFolderPath(fsys fs.FS, path string) string {
+	// Handle root paths immediately
+	if path == "." || path == "" {
+		return ""
+	}
+
+	folderPath := path
+	for {
+		info, err := fs.Stat(fsys, folderPath)
+		if err == nil && info.IsDir() {
+			// Found a valid directory
+			return folderPath
+		}
+		if folderPath == "." || folderPath == "" {
+			// Reached root, scan entire library
+			return ""
+		}
+		// Walk up the tree
+		dir, _ := filepath.Split(folderPath)
+		if dir == "" || dir == "." {
+			return ""
+		}
+		// Remove trailing slash
+		folderPath = filepath.Clean(dir)
+	}
+}
+
+// shouldIgnoreFolderPath checks if the given folderPath should be ignored based on .ndignore patterns
+// in the library. It pushes all parent folders onto the IgnoreChecker stack before checking.
+func (w *watcher) shouldIgnoreFolderPath(ctx context.Context, fsys storage.MusicFS, folderPath string) bool {
+	checker := newIgnoreChecker(fsys)
+	err := checker.PushAllParents(ctx, folderPath)
+	if err != nil {
+		log.Warn(ctx, "Watcher: Error pushing ignore patterns for folder", "path", folderPath, err)
+	}
+	return checker.ShouldIgnore(ctx, folderPath)
+}
+
 func isIgnoredPath(_ context.Context, _ fs.FS, path string) bool {
 	baseDir, name := filepath.Split(path)
 	switch {