feat(scanner): implement selective folder scanning and file system watcher improvements (#4674)

* feat: Add selective folder scanning capability

Implement targeted scanning of specific library/folder pairs without
full recursion. This enables efficient rescanning of individual folders
when changes are detected, significantly reducing scan time for large
libraries.

Key changes:
- Add ScanTarget struct and ScanFolders API to Scanner interface
- Implement CLI flag --targets for specifying libraryID:folderPath pairs
- Add FolderRepository.GetByPaths() for batch folder info retrieval
- Create loadSpecificFolders() for non-recursive directory loading
- Scope GC operations to affected libraries only (with TODO for full impl)
- Add comprehensive tests for selective scanning behavior

The selective scan:
- Only processes specified folders (no subdirectory recursion)
- Maintains library isolation
- Runs full maintenance pipeline scoped to affected libraries
- Supports both full and quick scan modes

Examples:
  navidrome scan --targets "1:Music/Rock,1:Music/Jazz"
  navidrome scan --full --targets "2:Classical"

* feat(folder): replace GetByPaths with GetFolderUpdateInfo for improved folder updates retrieval

Signed-off-by: Deluan <deluan@navidrome.org>

* test: update parseTargets test to handle folder names with spaces

Signed-off-by: Deluan <deluan@navidrome.org>

* refactor(folder): remove unused LibraryPath struct and update GC logging message

Signed-off-by: Deluan <deluan@navidrome.org>

* refactor(folder): enhance external scanner to support target-specific scanning

Signed-off-by: Deluan <deluan@navidrome.org>

* refactor(scanner): simplify scanner methods

Signed-off-by: Deluan <deluan@navidrome.org>

* feat(watcher): implement folder scanning notifications with deduplication

Signed-off-by: Deluan <deluan@navidrome.org>

* refactor(watcher): add resolveFolderPath function for testability

Signed-off-by: Deluan <deluan@navidrome.org>

* feat(watcher): implement path ignoring based on .ndignore patterns

Signed-off-by: Deluan <deluan@navidrome.org>

* refactor(scanner): implement IgnoreChecker for managing .ndignore patterns

Signed-off-by: Deluan <deluan@navidrome.org>

* refactor(ignore_checker): rename scanner to lineScanner for clarity

Signed-off-by: Deluan <deluan@navidrome.org>

* refactor(scanner): enhance ScanTarget struct with String method for better target representation

Signed-off-by: Deluan <deluan@navidrome.org>

* fix(scanner): validate library ID to prevent negative values

Signed-off-by: Deluan <deluan@navidrome.org>

* refactor(scanner): simplify GC method by removing library ID parameter

Signed-off-by: Deluan <deluan@navidrome.org>

* feat(scanner): update folder scanning to include all descendants of specified folders

Signed-off-by: Deluan <deluan@navidrome.org>

* feat(subsonic): allow selective scan in the /startScan endpoint

Signed-off-by: Deluan <deluan@navidrome.org>

* refactor(scanner): update CallScan to handle specific library/folder pairs

Signed-off-by: Deluan <deluan@navidrome.org>

* refactor(scanner): streamline scanning logic by removing scanAll method

Signed-off-by: Deluan <deluan@navidrome.org>

* test: enhance mockScanner for thread safety and improve test reliability

Signed-off-by: Deluan <deluan@navidrome.org>

* refactor(scanner): move scanner.ScanTarget to model.ScanTarget

Signed-off-by: Deluan <deluan@navidrome.org>

* refactor: move scanner types to model,implement MockScanner

Signed-off-by: Deluan <deluan@navidrome.org>

* refactor(scanner): update scanner interface and implementations to use model.Scanner

Signed-off-by: Deluan <deluan@navidrome.org>

* refactor(folder_repository): normalize target path handling by using filepath.Clean

Signed-off-by: Deluan <deluan@navidrome.org>

* test(folder_repository): add comprehensive tests for folder retrieval and child exclusion

Signed-off-by: Deluan <deluan@navidrome.org>

* refactor(scanner): simplify selective scan logic using slice.Filter

Signed-off-by: Deluan <deluan@navidrome.org>

* refactor(scanner): streamline phase folder and album creation by removing unnecessary library parameter

Signed-off-by: Deluan <deluan@navidrome.org>

* refactor(scanner): move initialization logic from phase_1 to the scanner itself

Signed-off-by: Deluan <deluan@navidrome.org>

* refactor(tests): rename selective scan test file to scanner_selective_test.go

Signed-off-by: Deluan <deluan@navidrome.org>

* feat(configuration): add DevSelectiveWatcher configuration option

Signed-off-by: Deluan <deluan@navidrome.org>

* feat(watcher): enhance .ndignore handling for folder deletions and file changes

Signed-off-by: Deluan <deluan@navidrome.org>

* docs(scanner): comments

Signed-off-by: Deluan <deluan@navidrome.org>

* refactor(scanner): enhance walkDirTree to support target folder scanning

Signed-off-by: Deluan <deluan@navidrome.org>

* fix(scanner, watcher): handle errors when pushing ignore patterns for folders

Signed-off-by: Deluan <deluan@navidrome.org>

* Update scanner/phase_1_folders.go

Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com>

* refactor(scanner): replace parseTargets function with direct call to scanner.ParseTargets

Signed-off-by: Deluan <deluan@navidrome.org>

* test(scanner): add tests for ScanBegin and ScanEnd functionality

Signed-off-by: Deluan <deluan@navidrome.org>

* fix(library): update PRAGMA optimize to check table sizes without ANALYZE

Signed-off-by: Deluan <deluan@navidrome.org>

* test(scanner): refactor tests

Signed-off-by: Deluan <deluan@navidrome.org>

* feat(ui): add selective scan options and update translations

Signed-off-by: Deluan <deluan@navidrome.org>

* feat(ui): add quick and full scan options for individual libraries

Signed-off-by: Deluan <deluan@navidrome.org>

* feat(ui): add Scan buttonsto the LibraryList

Signed-off-by: Deluan <deluan@navidrome.org>

* feat(scan): update scanning parameters from 'path' to 'target' for selective scans.

* refactor(scan): move ParseTargets function to model package

* test(scan): suppress unused return value from SetUserLibraries in tests

* feat(gc): enhance garbage collection to support selective library purging

Signed-off-by: Deluan <deluan@navidrome.org>

* fix(scanner): prevent race condition when scanning deleted folders

When the watcher detects changes in a folder that gets deleted before
the scanner runs (due to the 10-second delay), the scanner was
prematurely removing these folders from the tracking map, preventing
them from being marked as missing.

The issue occurred because `newFolderEntry` was calling `popLastUpdate`
before verifying the folder actually exists on the filesystem.

Changes:
- Move fs.Stat check before newFolderEntry creation in loadDir to
  ensure deleted folders remain in lastUpdates for finalize() to handle
- Add early existence check in walkDirTree to skip non-existent target
  folders with a warning log
- Add unit test verifying non-existent folders aren't removed from
  lastUpdates prematurely
- Add integration test for deleted folder scenario with ScanFolders

Fixes the issue where deleting entire folders (e.g., /music/AC_DC)
wouldn't mark tracks as missing when using selective folder scanning.

* refactor(scan): streamline folder entry creation and update handling

Signed-off-by: Deluan <deluan@navidrome.org>

* feat(scan): add '@Recycle' (QNAP) to ignored directories list

Signed-off-by: Deluan <deluan@navidrome.org>

* fix(log): improve thread safety in logging level management

* test(scan): move unit tests for ParseTargets function

Signed-off-by: Deluan <deluan@navidrome.org>

* review

Signed-off-by: Deluan <deluan@navidrome.org>

---------

Signed-off-by: Deluan <deluan@navidrome.org>
Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com>
Co-authored-by: deluan <deluan.quintao@mechanical-orchard.com>
This commit is contained in:
Deluan Quintão
2025-11-14 22:15:43 -05:00
committed by GitHub
parent bca76069c3
commit 28d5299ffc
52 changed files with 3221 additions and 374 deletions
+2 -3
View File
@@ -18,7 +18,6 @@ import (
"github.com/navidrome/navidrome/core/scrobbler"
"github.com/navidrome/navidrome/log"
"github.com/navidrome/navidrome/model"
"github.com/navidrome/navidrome/scanner"
"github.com/navidrome/navidrome/server"
"github.com/navidrome/navidrome/server/events"
"github.com/navidrome/navidrome/server/subsonic/responses"
@@ -39,7 +38,7 @@ type Router struct {
players core.Players
provider external.Provider
playlists core.Playlists
scanner scanner.Scanner
scanner model.Scanner
broker events.Broker
scrobbler scrobbler.PlayTracker
share core.Share
@@ -48,7 +47,7 @@ type Router struct {
}
func New(ds model.DataStore, artwork artwork.Artwork, streamer core.MediaStreamer, archiver core.Archiver,
players core.Players, provider external.Provider, scanner scanner.Scanner, broker events.Broker,
players core.Players, provider external.Provider, scanner model.Scanner, broker events.Broker,
playlists core.Playlists, scrobbler scrobbler.PlayTracker, share core.Share, playback playback.PlaybackServer,
metrics metrics.Metrics,
) *Router {
+47 -3
View File
@@ -1,10 +1,13 @@
package subsonic
import (
"fmt"
"net/http"
"slices"
"time"
"github.com/navidrome/navidrome/log"
"github.com/navidrome/navidrome/model"
"github.com/navidrome/navidrome/model/request"
"github.com/navidrome/navidrome/server/subsonic/responses"
"github.com/navidrome/navidrome/utils/req"
@@ -44,15 +47,56 @@ func (api *Router) StartScan(r *http.Request) (*responses.Subsonic, error) {
p := req.Params(r)
fullScan := p.BoolOr("fullScan", false)
// Parse optional target parameters for selective scanning
var targets []model.ScanTarget
if targetParams, err := p.Strings("target"); err == nil && len(targetParams) > 0 {
targets, err = model.ParseTargets(targetParams)
if err != nil {
return nil, newError(responses.ErrorGeneric, fmt.Sprintf("Invalid target parameter: %v", err))
}
// Validate all libraries in targets exist and user has access to them
userLibraries, err := api.ds.User(ctx).GetUserLibraries(loggedUser.ID)
if err != nil {
return nil, newError(responses.ErrorGeneric, "Internal error")
}
// Check each target library
for _, target := range targets {
if !slices.ContainsFunc(userLibraries, func(lib model.Library) bool { return lib.ID == target.LibraryID }) {
return nil, newError(responses.ErrorDataNotFound, fmt.Sprintf("Library with ID %d not found", target.LibraryID))
}
}
// Special case: if single library with empty path and it's the only library in DB, call ScanAll
if len(targets) == 1 && targets[0].FolderPath == "" {
allLibs, err := api.ds.Library(ctx).GetAll()
if err != nil {
return nil, newError(responses.ErrorGeneric, "Internal error")
}
if len(allLibs) == 1 {
targets = nil // This will trigger ScanAll below
}
}
}
go func() {
start := time.Now()
log.Info(ctx, "Triggering manual scan", "fullScan", fullScan, "user", loggedUser.UserName)
_, err := api.scanner.ScanAll(ctx, fullScan)
var err error
if len(targets) > 0 {
log.Info(ctx, "Triggering on-demand scan", "fullScan", fullScan, "targets", len(targets), "user", loggedUser.UserName)
_, err = api.scanner.ScanFolders(ctx, fullScan, targets)
} else {
log.Info(ctx, "Triggering on-demand scan", "fullScan", fullScan, "user", loggedUser.UserName)
_, err = api.scanner.ScanAll(ctx, fullScan)
}
if err != nil {
log.Error(ctx, "Error scanning", err)
return
}
log.Info(ctx, "Manual scan complete", "user", loggedUser.UserName, "elapsed", time.Since(start))
log.Info(ctx, "On-demand scan complete", "user", loggedUser.UserName, "elapsed", time.Since(start))
}()
return api.GetScanStatus(r)
+396
View File
@@ -0,0 +1,396 @@
package subsonic
import (
"context"
"errors"
"net/http/httptest"
"github.com/navidrome/navidrome/model"
"github.com/navidrome/navidrome/model/request"
"github.com/navidrome/navidrome/server/subsonic/responses"
"github.com/navidrome/navidrome/tests"
. "github.com/onsi/ginkgo/v2"
. "github.com/onsi/gomega"
)
var _ = Describe("LibraryScanning", func() {
var api *Router
var ms *tests.MockScanner
BeforeEach(func() {
ms = tests.NewMockScanner()
api = &Router{scanner: ms}
})
Describe("StartScan", func() {
It("requires admin authentication", func() {
// Create non-admin user
ctx := request.WithUser(context.Background(), model.User{
ID: "user-id",
IsAdmin: false,
})
// Create request
r := httptest.NewRequest("GET", "/rest/startScan", nil)
r = r.WithContext(ctx)
// Call endpoint
response, err := api.StartScan(r)
// Should return authorization error
Expect(err).To(HaveOccurred())
Expect(response).To(BeNil())
var subErr subError
ok := errors.As(err, &subErr)
Expect(ok).To(BeTrue())
Expect(subErr.code).To(Equal(responses.ErrorAuthorizationFail))
})
It("triggers a full scan with no parameters", func() {
// Create admin user
ctx := request.WithUser(context.Background(), model.User{
ID: "admin-id",
IsAdmin: true,
})
// Create request with no parameters
r := httptest.NewRequest("GET", "/rest/startScan", nil)
r = r.WithContext(ctx)
// Call endpoint
response, err := api.StartScan(r)
// Should succeed
Expect(err).ToNot(HaveOccurred())
Expect(response).ToNot(BeNil())
// Verify ScanAll was called (eventually, since it's in a goroutine)
Eventually(func() int {
return ms.GetScanAllCallCount()
}).Should(BeNumerically(">", 0))
calls := ms.GetScanAllCalls()
Expect(calls).To(HaveLen(1))
Expect(calls[0].FullScan).To(BeFalse())
})
It("triggers a full scan with fullScan=true", func() {
// Create admin user
ctx := request.WithUser(context.Background(), model.User{
ID: "admin-id",
IsAdmin: true,
})
// Create request with fullScan parameter
r := httptest.NewRequest("GET", "/rest/startScan?fullScan=true", nil)
r = r.WithContext(ctx)
// Call endpoint
response, err := api.StartScan(r)
// Should succeed
Expect(err).ToNot(HaveOccurred())
Expect(response).ToNot(BeNil())
// Verify ScanAll was called with fullScan=true
Eventually(func() int {
return ms.GetScanAllCallCount()
}).Should(BeNumerically(">", 0))
calls := ms.GetScanAllCalls()
Expect(calls).To(HaveLen(1))
Expect(calls[0].FullScan).To(BeTrue())
})
It("triggers a selective scan with single target parameter", func() {
// Setup mocks
mockUserRepo := tests.CreateMockUserRepo()
_ = mockUserRepo.SetUserLibraries("admin-id", []int{1, 2})
mockDS := &tests.MockDataStore{MockedUser: mockUserRepo}
api.ds = mockDS
// Create admin user
ctx := request.WithUser(context.Background(), model.User{
ID: "admin-id",
IsAdmin: true,
})
// Create request with single target parameter
r := httptest.NewRequest("GET", "/rest/startScan?target=1:Music/Rock", nil)
r = r.WithContext(ctx)
// Call endpoint
response, err := api.StartScan(r)
// Should succeed
Expect(err).ToNot(HaveOccurred())
Expect(response).ToNot(BeNil())
// Verify ScanFolders was called with correct targets
Eventually(func() int {
return ms.GetScanFoldersCallCount()
}).Should(BeNumerically(">", 0))
calls := ms.GetScanFoldersCalls()
Expect(calls).To(HaveLen(1))
targets := calls[0].Targets
Expect(targets).To(HaveLen(1))
Expect(targets[0].LibraryID).To(Equal(1))
Expect(targets[0].FolderPath).To(Equal("Music/Rock"))
})
It("triggers a selective scan with multiple target parameters", func() {
// Setup mocks
mockUserRepo := tests.CreateMockUserRepo()
_ = mockUserRepo.SetUserLibraries("admin-id", []int{1, 2})
mockDS := &tests.MockDataStore{MockedUser: mockUserRepo}
api.ds = mockDS
// Create admin user
ctx := request.WithUser(context.Background(), model.User{
ID: "admin-id",
IsAdmin: true,
})
// Create request with multiple target parameters
r := httptest.NewRequest("GET", "/rest/startScan?target=1:Music/Reggae&target=2:Classical/Bach", nil)
r = r.WithContext(ctx)
// Call endpoint
response, err := api.StartScan(r)
// Should succeed
Expect(err).ToNot(HaveOccurred())
Expect(response).ToNot(BeNil())
// Verify ScanFolders was called with correct targets
Eventually(func() int {
return ms.GetScanFoldersCallCount()
}).Should(BeNumerically(">", 0))
calls := ms.GetScanFoldersCalls()
Expect(calls).To(HaveLen(1))
targets := calls[0].Targets
Expect(targets).To(HaveLen(2))
Expect(targets[0].LibraryID).To(Equal(1))
Expect(targets[0].FolderPath).To(Equal("Music/Reggae"))
Expect(targets[1].LibraryID).To(Equal(2))
Expect(targets[1].FolderPath).To(Equal("Classical/Bach"))
})
It("triggers a selective full scan with target and fullScan parameters", func() {
// Setup mocks
mockUserRepo := tests.CreateMockUserRepo()
_ = mockUserRepo.SetUserLibraries("admin-id", []int{1})
mockDS := &tests.MockDataStore{MockedUser: mockUserRepo}
api.ds = mockDS
// Create admin user
ctx := request.WithUser(context.Background(), model.User{
ID: "admin-id",
IsAdmin: true,
})
// Create request with target and fullScan parameters
r := httptest.NewRequest("GET", "/rest/startScan?target=1:Music/Jazz&fullScan=true", nil)
r = r.WithContext(ctx)
// Call endpoint
response, err := api.StartScan(r)
// Should succeed
Expect(err).ToNot(HaveOccurred())
Expect(response).ToNot(BeNil())
// Verify ScanFolders was called with fullScan=true
Eventually(func() int {
return ms.GetScanFoldersCallCount()
}).Should(BeNumerically(">", 0))
calls := ms.GetScanFoldersCalls()
Expect(calls).To(HaveLen(1))
Expect(calls[0].FullScan).To(BeTrue())
targets := calls[0].Targets
Expect(targets).To(HaveLen(1))
})
It("returns error for invalid target format", func() {
// Create admin user
ctx := request.WithUser(context.Background(), model.User{
ID: "admin-id",
IsAdmin: true,
})
// Create request with invalid target format (missing colon)
r := httptest.NewRequest("GET", "/rest/startScan?target=1MusicRock", nil)
r = r.WithContext(ctx)
// Call endpoint
response, err := api.StartScan(r)
// Should return error
Expect(err).To(HaveOccurred())
Expect(response).To(BeNil())
var subErr subError
ok := errors.As(err, &subErr)
Expect(ok).To(BeTrue())
Expect(subErr.code).To(Equal(responses.ErrorGeneric))
})
It("returns error for invalid library ID in target", func() {
// Create admin user
ctx := request.WithUser(context.Background(), model.User{
ID: "admin-id",
IsAdmin: true,
})
// Create request with invalid library ID
r := httptest.NewRequest("GET", "/rest/startScan?target=0:Music/Rock", nil)
r = r.WithContext(ctx)
// Call endpoint
response, err := api.StartScan(r)
// Should return error
Expect(err).To(HaveOccurred())
Expect(response).To(BeNil())
var subErr subError
ok := errors.As(err, &subErr)
Expect(ok).To(BeTrue())
Expect(subErr.code).To(Equal(responses.ErrorGeneric))
})
It("returns error when library does not exist", func() {
// Setup mocks - user has access to library 1 and 2 only
mockUserRepo := tests.CreateMockUserRepo()
_ = mockUserRepo.SetUserLibraries("admin-id", []int{1, 2})
mockDS := &tests.MockDataStore{MockedUser: mockUserRepo}
api.ds = mockDS
// Create admin user
ctx := request.WithUser(context.Background(), model.User{
ID: "admin-id",
IsAdmin: true,
})
// Create request with library ID that doesn't exist
r := httptest.NewRequest("GET", "/rest/startScan?target=999:Music/Rock", nil)
r = r.WithContext(ctx)
// Call endpoint
response, err := api.StartScan(r)
// Should return ErrorDataNotFound
Expect(err).To(HaveOccurred())
Expect(response).To(BeNil())
var subErr subError
ok := errors.As(err, &subErr)
Expect(ok).To(BeTrue())
Expect(subErr.code).To(Equal(responses.ErrorDataNotFound))
})
It("calls ScanAll when single library with empty path and only one library exists", func() {
// Setup mocks - single library in DB
mockUserRepo := tests.CreateMockUserRepo()
_ = mockUserRepo.SetUserLibraries("admin-id", []int{1})
mockLibraryRepo := &tests.MockLibraryRepo{}
mockLibraryRepo.SetData(model.Libraries{
{ID: 1, Name: "Music Library", Path: "/music"},
})
mockDS := &tests.MockDataStore{
MockedUser: mockUserRepo,
MockedLibrary: mockLibraryRepo,
}
api.ds = mockDS
// Create admin user
ctx := request.WithUser(context.Background(), model.User{
ID: "admin-id",
IsAdmin: true,
})
// Create request with single library and empty path
r := httptest.NewRequest("GET", "/rest/startScan?target=1:", nil)
r = r.WithContext(ctx)
// Call endpoint
response, err := api.StartScan(r)
// Should succeed
Expect(err).ToNot(HaveOccurred())
Expect(response).ToNot(BeNil())
// Verify ScanAll was called instead of ScanFolders
Eventually(func() int {
return ms.GetScanAllCallCount()
}).Should(BeNumerically(">", 0))
Expect(ms.GetScanFoldersCallCount()).To(Equal(0))
})
It("calls ScanFolders when single library with empty path but multiple libraries exist", func() {
// Setup mocks - multiple libraries in DB
mockUserRepo := tests.CreateMockUserRepo()
_ = mockUserRepo.SetUserLibraries("admin-id", []int{1, 2})
mockLibraryRepo := &tests.MockLibraryRepo{}
mockLibraryRepo.SetData(model.Libraries{
{ID: 1, Name: "Music Library", Path: "/music"},
{ID: 2, Name: "Audiobooks", Path: "/audiobooks"},
})
mockDS := &tests.MockDataStore{
MockedUser: mockUserRepo,
MockedLibrary: mockLibraryRepo,
}
api.ds = mockDS
// Create admin user
ctx := request.WithUser(context.Background(), model.User{
ID: "admin-id",
IsAdmin: true,
})
// Create request with single library and empty path
r := httptest.NewRequest("GET", "/rest/startScan?target=1:", nil)
r = r.WithContext(ctx)
// Call endpoint
response, err := api.StartScan(r)
// Should succeed
Expect(err).ToNot(HaveOccurred())
Expect(response).ToNot(BeNil())
// Verify ScanFolders was called (not ScanAll)
Eventually(func() int {
return ms.GetScanFoldersCallCount()
}).Should(BeNumerically(">", 0))
calls := ms.GetScanFoldersCalls()
Expect(calls).To(HaveLen(1))
targets := calls[0].Targets
Expect(targets).To(HaveLen(1))
Expect(targets[0].LibraryID).To(Equal(1))
Expect(targets[0].FolderPath).To(Equal(""))
})
})
Describe("GetScanStatus", func() {
It("returns scan status", func() {
// Setup mock scanner status
ms.SetStatusResponse(&model.ScannerStatus{
Scanning: false,
Count: 100,
FolderCount: 10,
})
// Create request
ctx := context.Background()
r := httptest.NewRequest("GET", "/rest/getScanStatus", nil)
r = r.WithContext(ctx)
// Call endpoint
response, err := api.GetScanStatus(r)
// Should succeed
Expect(err).ToNot(HaveOccurred())
Expect(response).ToNot(BeNil())
Expect(response.ScanStatus).ToNot(BeNil())
Expect(response.ScanStatus.Scanning).To(BeFalse())
Expect(response.ScanStatus.Count).To(Equal(int64(100)))
Expect(response.ScanStatus.FolderCount).To(Equal(int64(10)))
})
})
})