arian/navidrome
1
0
Fork 0
Code Issues Pull Requests 1 Packages Projects Releases Wiki Activity

Add new scanner algorithm, can be enabled with DevNewScanner config option

Browse Source
This commit is contained in:
Deluan
2020-07-17 10:27:30 -04:00
parent de0cc1f268
commit 51c295d1de
11 changed files with 750 additions and 160 deletions
Download Patch File Download Diff File Expand all files Collapse all files
scanner/change_detector.go
+8 -47
View File
@@ -7,9 +7,7 @@ import (
"path/filepath"
"time"
"github.com/deluan/navidrome/consts"
"github.com/deluan/navidrome/log"
"github.com/deluan/navidrome/utils"
)
type dirInfo struct {
@@ -18,19 +16,19 @@ type dirInfo struct {
}
type dirInfoMap map[string]dirInfo
type ChangeDetector struct {
type changeDetector struct {
rootFolder string
dirMap dirInfoMap
}
func NewChangeDetector(rootFolder string) *ChangeDetector {
return &ChangeDetector{
func newChangeDetector(rootFolder string) *changeDetector {
return &changeDetector{
rootFolder: rootFolder,
dirMap: dirInfoMap{},
}
}
func (s *ChangeDetector) Scan(ctx context.Context, lastModifiedSince time.Time) (changed []string, deleted []string, err error) {
func (s *changeDetector) Scan(ctx context.Context, lastModifiedSince time.Time) (changed []string, deleted []string, err error) {
start := time.Now()
newMap := make(dirInfoMap)
err = s.loadMap(ctx, newMap, s.rootFolder, lastModifiedSince, false)
@@ -48,7 +46,7 @@ func (s *ChangeDetector) Scan(ctx context.Context, lastModifiedSince time.Time)
return
}
func (s *ChangeDetector) loadDir(ctx context.Context, dirPath string) (children []string, lastUpdated time.Time, err error) {
func (s *changeDetector) loadDir(ctx context.Context, dirPath string) (children []string, lastUpdated time.Time, err error) {
dirInfo, err := os.Stat(dirPath)
if err != nil {
log.Error(ctx, "Error stating dir", "path", dirPath, err)
@@ -78,44 +76,7 @@ func (s *ChangeDetector) loadDir(ctx context.Context, dirPath string) (children
return
}
// isDirOrSymlinkToDir returns true if and only if the dirInfo represents a file
// system directory, or a symbolic link to a directory. Note that if the dirInfo
// is not a directory but is a symbolic link, this method will resolve by
// sending a request to the operating system to follow the symbolic link.
// Copied from github.com/karrick/godirwalk
func isDirOrSymlinkToDir(baseDir string, dirInfo os.FileInfo) (bool, error) {
if dirInfo.IsDir() {
return true, nil
}
if dirInfo.Mode()&os.ModeSymlink == 0 {
return false, nil
}
// Does this symlink point to a directory?
dirInfo, err := os.Stat(filepath.Join(baseDir, dirInfo.Name()))
if err != nil {
return false, err
}
return dirInfo.IsDir(), nil
}
// isDirIgnored returns true if the directory represented by dirInfo contains an
// `ignore` file (named after consts.SkipScanFile)
func isDirIgnored(baseDir string, dirInfo os.FileInfo) bool {
_, err := os.Stat(filepath.Join(baseDir, dirInfo.Name(), consts.SkipScanFile))
return err == nil
}
// isDirReadable returns true if the directory represented by dirInfo is readable
func isDirReadable(baseDir string, dirInfo os.FileInfo) bool {
path := filepath.Join(baseDir, dirInfo.Name())
res, err := utils.IsDirReadable(path)
if !res {
log.Debug("Warning: Skipping unreadable directory", "path", path, err)
}
return res
}
func (s *ChangeDetector) loadMap(ctx context.Context, dirMap dirInfoMap, path string, since time.Time, maybe bool) error {
func (s *changeDetector) loadMap(ctx context.Context, dirMap dirInfoMap, path string, since time.Time, maybe bool) error {
children, lastUpdated, err := s.loadDir(ctx, path)
if err != nil {
return err
@@ -134,7 +95,7 @@ func (s *ChangeDetector) loadMap(ctx context.Context, dirMap dirInfoMap, path st
return nil
}
func (s *ChangeDetector) getRelativePath(subFolder string) string {
func (s *changeDetector) getRelativePath(subFolder string) string {
dir, _ := filepath.Rel(s.rootFolder, subFolder)
if dir == "" {
dir = "."
@@ -142,7 +103,7 @@ func (s *ChangeDetector) getRelativePath(subFolder string) string {
return dir
}
func (s *ChangeDetector) checkForUpdates(lastModifiedSince time.Time, newMap dirInfoMap) (changed []string, deleted []string, err error) {
func (s *changeDetector) checkForUpdates(lastModifiedSince time.Time, newMap dirInfoMap) (changed []string, deleted []string, err error) {
for dir, newEntry := range newMap {
lastUpdated := newEntry.mdate
oldLastUpdated := lastModifiedSince
scanner/change_detector_test.go
+4 -4
View File
@@ -11,9 +11,9 @@ import (
. "github.com/onsi/gomega"
)
var _ = Describe("ChangeDetector", func() {
var _ = Describe("changeDetector", func() {
var testFolder string
var scanner *ChangeDetector
var scanner *changeDetector
lastModifiedSince := time.Time{}
@@ -23,7 +23,7 @@ var _ = Describe("ChangeDetector", func() {
if err != nil {
panic(err)
}
scanner = NewChangeDetector(testFolder)
scanner = newChangeDetector(testFolder)
})
It("detects changes recursively", func() {
@@ -97,7 +97,7 @@ var _ = Describe("ChangeDetector", func() {
// Only returns changes after lastModifiedSince
lastModifiedSince = nowWithDelay()
newScanner := NewChangeDetector(testFolder)
newScanner := newChangeDetector(testFolder)
changed, deleted, err = newScanner.Scan(context.TODO(), lastModifiedSince)
Expect(err).To(BeNil())
Expect(deleted).To(BeEmpty())
scanner/flushable_map.go
+58
View File
@@ -0,0 +1,58 @@
package scanner
import (
"context"
"fmt"
"github.com/deluan/navidrome/log"
)
const (
// batchSize used for albums/artists updates
batchSize = 5
)
type refreshCallbackFunc = func(ids ...string) error
type flushableMap struct {
ctx context.Context
flushFunc refreshCallbackFunc
entity string
m map[string]struct{}
}
func newFlushableMap(ctx context.Context, entity string, flushFunc refreshCallbackFunc) *flushableMap {
return &flushableMap{
ctx: ctx,
flushFunc: flushFunc,
entity: entity,
m: map[string]struct{}{},
}
}
func (f *flushableMap) update(id string) error {
f.m[id] = struct{}{}
if len(f.m) >= batchSize {
err := f.flush()
if err != nil {
return err
}
}
return nil
}
func (f *flushableMap) flush() error {
if len(f.m) == 0 {
return nil
}
var ids []string
for id := range f.m {
ids = append(ids, id)
delete(f.m, id)
}
if err := f.flushFunc(ids...); err != nil {
log.Error(f.ctx, fmt.Sprintf("Error writing %ss to the DB", f.entity), err)
return err
}
return nil
}
scanner/load_tree.go
+109
View File
@@ -0,0 +1,109 @@
package scanner
import (
"context"
"io/ioutil"
"os"
"path/filepath"
"time"
"github.com/deluan/navidrome/consts"
"github.com/deluan/navidrome/log"
"github.com/deluan/navidrome/utils"
)
type dirMap = map[string]time.Time
func loadDirTree(ctx context.Context, rootFolder string) (dirMap, error) {
newMap := make(map[string]time.Time)
err := loadMap(ctx, rootFolder, rootFolder, newMap)
if err != nil {
log.Error(ctx, "Error loading directory tree", err)
}
return newMap, err
}
func loadMap(ctx context.Context, rootPath string, currentFolder string, dirMap dirMap) error {
children, lastUpdated, err := loadDir(ctx, currentFolder)
if err != nil {
return err
}
for _, c := range children {
err := loadMap(ctx, rootPath, c, dirMap)
if err != nil {
return err
}
}
dir := filepath.Clean(currentFolder)
dirMap[dir] = lastUpdated
return nil
}
func loadDir(ctx context.Context, dirPath string) (children []string, lastUpdated time.Time, err error) {
dirInfo, err := os.Stat(dirPath)
if err != nil {
log.Error(ctx, "Error stating dir", "path", dirPath, err)
return
}
lastUpdated = dirInfo.ModTime()
files, err := ioutil.ReadDir(dirPath)
if err != nil {
log.Error(ctx, "Error reading dir", "path", dirPath, err)
return
}
for _, f := range files {
isDir, err := isDirOrSymlinkToDir(dirPath, f)
// Skip invalid symlinks
if err != nil {
continue
}
if isDir && !isDirIgnored(dirPath, f) && isDirReadable(dirPath, f) {
children = append(children, filepath.Join(dirPath, f.Name()))
} else {
if f.ModTime().After(lastUpdated) {
lastUpdated = f.ModTime()
}
}
}
return
}
// isDirOrSymlinkToDir returns true if and only if the dirInfo represents a file
// system directory, or a symbolic link to a directory. Note that if the dirInfo
// is not a directory but is a symbolic link, this method will resolve by
// sending a request to the operating system to follow the symbolic link.
// Copied from github.com/karrick/godirwalk
func isDirOrSymlinkToDir(baseDir string, dirInfo os.FileInfo) (bool, error) {
if dirInfo.IsDir() {
return true, nil
}
if dirInfo.Mode()&os.ModeSymlink == 0 {
return false, nil
}
// Does this symlink point to a directory?
dirInfo, err := os.Stat(filepath.Join(baseDir, dirInfo.Name()))
if err != nil {
return false, err
}
return dirInfo.IsDir(), nil
}
// isDirIgnored returns true if the directory represented by dirInfo contains an
// `ignore` file (named after consts.SkipScanFile)
func isDirIgnored(baseDir string, dirInfo os.FileInfo) bool {
_, err := os.Stat(filepath.Join(baseDir, dirInfo.Name(), consts.SkipScanFile))
return err == nil
}
// isDirReadable returns true if the directory represented by dirInfo is readable
func isDirReadable(baseDir string, dirInfo os.FileInfo) bool {
path := filepath.Join(baseDir, dirInfo.Name())
res, err := utils.IsDirReadable(path)
if !res {
log.Debug("Warning: Skipping unreadable directory", "path", path, err)
}
return res
}
scanner/mapping.go
+120
View File
@@ -0,0 +1,120 @@
package scanner
import (
"crypto/md5"
"fmt"
"os"
"path/filepath"
"strings"
"github.com/deluan/navidrome/consts"
"github.com/deluan/navidrome/model"
"github.com/deluan/navidrome/utils"
"github.com/kennygrant/sanitize"
)
type mediaFileMapper struct {
rootFolder string
}
func newMediaFileMapper(rootFolder string) *mediaFileMapper {
return &mediaFileMapper{rootFolder: rootFolder}
}
func (s *mediaFileMapper) toMediaFile(md *Metadata) model.MediaFile {
mf := &model.MediaFile{}
mf.ID = s.trackID(md)
mf.Title = s.mapTrackTitle(md)
mf.Album = md.Album()
mf.AlbumID = s.albumID(md)
mf.Album = s.mapAlbumName(md)
mf.ArtistID = s.artistID(md)
mf.Artist = s.mapArtistName(md)
mf.AlbumArtistID = s.albumArtistID(md)
mf.AlbumArtist = s.mapAlbumArtistName(md)
mf.Genre = md.Genre()
mf.Compilation = md.Compilation()
mf.Year = md.Year()
mf.TrackNumber, _ = md.TrackNumber()
mf.DiscNumber, _ = md.DiscNumber()
mf.DiscSubtitle = md.DiscSubtitle()
mf.Duration = md.Duration()
mf.BitRate = md.BitRate()
mf.Path = md.FilePath()
mf.Suffix = md.Suffix()
mf.Size = md.Size()
mf.HasCoverArt = md.HasPicture()
mf.SortTitle = md.SortTitle()
mf.SortAlbumName = md.SortAlbum()
mf.SortArtistName = md.SortArtist()
mf.SortAlbumArtistName = md.SortAlbumArtist()
mf.OrderAlbumName = sanitizeFieldForSorting(mf.Album)
mf.OrderArtistName = sanitizeFieldForSorting(mf.Artist)
mf.OrderAlbumArtistName = sanitizeFieldForSorting(mf.AlbumArtist)
// TODO Get Creation time. https://github.com/djherbis/times ?
mf.CreatedAt = md.ModificationTime()
mf.UpdatedAt = md.ModificationTime()
return *mf
}
func sanitizeFieldForSorting(originalValue string) string {
v := utils.NoArticle(originalValue)
v = strings.TrimSpace(sanitize.Accents(v))
return utils.NoArticle(v)
}
func (s *mediaFileMapper) mapTrackTitle(md *Metadata) string {
if md.Title() == "" {
s := strings.TrimPrefix(md.FilePath(), s.rootFolder+string(os.PathSeparator))
e := filepath.Ext(s)
return strings.TrimSuffix(s, e)
}
return md.Title()
}
func (s *mediaFileMapper) mapAlbumArtistName(md *Metadata) string {
switch {
case md.Compilation():
return consts.VariousArtists
case md.AlbumArtist() != "":
return md.AlbumArtist()
case md.Artist() != "":
return md.Artist()
default:
return consts.UnknownArtist
}
}
func (s *mediaFileMapper) mapArtistName(md *Metadata) string {
if md.Artist() != "" {
return md.Artist()
}
return consts.UnknownArtist
}
func (s *mediaFileMapper) mapAlbumName(md *Metadata) string {
name := md.Album()
if name == "" {
return "[Unknown Album]"
}
return name
}
func (s *mediaFileMapper) trackID(md *Metadata) string {
return fmt.Sprintf("%x", md5.Sum([]byte(md.FilePath())))
}
func (s *mediaFileMapper) albumID(md *Metadata) string {
albumPath := strings.ToLower(fmt.Sprintf("%s\\%s", s.mapAlbumArtistName(md), s.mapAlbumName(md)))
return fmt.Sprintf("%x", md5.Sum([]byte(albumPath)))
}
func (s *mediaFileMapper) artistID(md *Metadata) string {
return fmt.Sprintf("%x", md5.Sum([]byte(strings.ToLower(s.mapArtistName(md)))))
}
func (s *mediaFileMapper) albumArtistID(md *Metadata) string {
return fmt.Sprintf("%x", md5.Sum([]byte(strings.ToLower(s.mapAlbumArtistName(md)))))
}
scanner/scanner.go
+9 -1
View File
@@ -7,6 +7,7 @@ import (
"strconv"
"time"
"github.com/deluan/navidrome/conf"
"github.com/deluan/navidrome/log"
"github.com/deluan/navidrome/model"
)
@@ -81,10 +82,17 @@ func (s *Scanner) loadFolders() {
fs, _ := s.ds.MediaFolder(context.TODO()).GetAll()
for _, f := range fs {
log.Info("Configuring Media Folder", "name", f.Name, "path", f.Path)
s.folders[f.Path] = NewTagScanner(f.Path, s.ds)
s.folders[f.Path] = s.newScanner(f)
}
}
func (s *Scanner) newScanner(f model.MediaFolder) FolderScanner {
if conf.Server.DevNewScanner {
return NewTagScanner2(f.Path, s.ds)
}
return NewTagScanner(f.Path, s.ds)
}
type Status int
type StatusInfo struct {
scanner/tag_scanner.go
+5 -108
View File
@@ -2,8 +2,6 @@ package scanner
import (
"context"
"crypto/md5"
"fmt"
"os"
"path/filepath"
"sort"
@@ -11,17 +9,16 @@ import (
"sync"
"time"
"github.com/deluan/navidrome/consts"
"github.com/deluan/navidrome/log"
"github.com/deluan/navidrome/model"
"github.com/deluan/navidrome/utils"
"github.com/kennygrant/sanitize"
)
type TagScanner struct {
rootFolder string
ds model.DataStore
detector *ChangeDetector
detector *changeDetector
mapper *mediaFileMapper
firstRun sync.Once
}
@@ -29,7 +26,8 @@ func NewTagScanner(rootFolder string, ds model.DataStore) *TagScanner {
return &TagScanner{
rootFolder: rootFolder,
ds: ds,
detector: NewChangeDetector(rootFolder),
detector: newChangeDetector(rootFolder),
mapper: newMediaFileMapper(rootFolder),
firstRun: sync.Once{},
}
}
@@ -46,9 +44,6 @@ type (
)
const (
// batchSize used for albums/artists updates
batchSize = 5
// filesBatchSize used for extract file metadata
filesBatchSize = 100
)
@@ -339,110 +334,12 @@ func (s *TagScanner) loadTracks(filePaths []string) (model.MediaFiles, error) {
var mfs model.MediaFiles
for _, md := range mds {
mf := s.toMediaFile(md)
mf := s.mapper.toMediaFile(md)
mfs = append(mfs, mf)
}
return mfs, nil
}
func (s *TagScanner) toMediaFile(md *Metadata) model.MediaFile {
mf := &model.MediaFile{}
mf.ID = s.trackID(md)
mf.Title = s.mapTrackTitle(md)
mf.Album = md.Album()
mf.AlbumID = s.albumID(md)
mf.Album = s.mapAlbumName(md)
mf.ArtistID = s.artistID(md)
mf.Artist = s.mapArtistName(md)
mf.AlbumArtistID = s.albumArtistID(md)
mf.AlbumArtist = s.mapAlbumArtistName(md)
mf.Genre = md.Genre()
mf.Compilation = md.Compilation()
mf.Year = md.Year()
mf.TrackNumber, _ = md.TrackNumber()
mf.DiscNumber, _ = md.DiscNumber()
mf.DiscSubtitle = md.DiscSubtitle()
mf.Duration = md.Duration()
mf.BitRate = md.BitRate()
mf.Path = md.FilePath()
mf.Suffix = md.Suffix()
mf.Size = md.Size()
mf.HasCoverArt = md.HasPicture()
mf.SortTitle = md.SortTitle()
mf.SortAlbumName = md.SortAlbum()
mf.SortArtistName = md.SortArtist()
mf.SortAlbumArtistName = md.SortAlbumArtist()
mf.OrderAlbumName = sanitizeFieldForSorting(mf.Album)
mf.OrderArtistName = sanitizeFieldForSorting(mf.Artist)
mf.OrderAlbumArtistName = sanitizeFieldForSorting(mf.AlbumArtist)
// TODO Get Creation time. https://github.com/djherbis/times ?
mf.CreatedAt = md.ModificationTime()
mf.UpdatedAt = md.ModificationTime()
return *mf
}
func sanitizeFieldForSorting(originalValue string) string {
v := utils.NoArticle(originalValue)
v = strings.TrimSpace(sanitize.Accents(v))
return utils.NoArticle(v)
}
func (s *TagScanner) mapTrackTitle(md *Metadata) string {
if md.Title() == "" {
s := strings.TrimPrefix(md.FilePath(), s.rootFolder+string(os.PathSeparator))
e := filepath.Ext(s)
return strings.TrimSuffix(s, e)
}
return md.Title()
}
func (s *TagScanner) mapAlbumArtistName(md *Metadata) string {
switch {
case md.Compilation():
return consts.VariousArtists
case md.AlbumArtist() != "":
return md.AlbumArtist()
case md.Artist() != "":
return md.Artist()
default:
return consts.UnknownArtist
}
}
func (s *TagScanner) mapArtistName(md *Metadata) string {
if md.Artist() != "" {
return md.Artist()
}
return consts.UnknownArtist
}
func (s *TagScanner) mapAlbumName(md *Metadata) string {
name := md.Album()
if name == "" {
return "[Unknown Album]"
}
return name
}
func (s *TagScanner) trackID(md *Metadata) string {
return fmt.Sprintf("%x", md5.Sum([]byte(md.FilePath())))
}
func (s *TagScanner) albumID(md *Metadata) string {
albumPath := strings.ToLower(fmt.Sprintf("%s\\%s", s.mapAlbumArtistName(md), s.mapAlbumName(md)))
return fmt.Sprintf("%x", md5.Sum([]byte(albumPath)))
}
func (s *TagScanner) artistID(md *Metadata) string {
return fmt.Sprintf("%x", md5.Sum([]byte(strings.ToLower(s.mapArtistName(md)))))
}
func (s *TagScanner) albumArtistID(md *Metadata) string {
return fmt.Sprintf("%x", md5.Sum([]byte(strings.ToLower(s.mapAlbumArtistName(md)))))
}
func LoadAllAudioFiles(dirPath string) (map[string]os.FileInfo, error) {
dir, err := os.Open(dirPath)
if err != nil {
scanner/tag_scanner_2.go
+297
View File
@@ -0,0 +1,297 @@
package scanner
import (
"context"
"path/filepath"
"sort"
"strings"
"time"
"github.com/deluan/navidrome/log"
"github.com/deluan/navidrome/model"
"github.com/deluan/navidrome/utils"
)
type TagScanner2 struct {
rootFolder string
ds model.DataStore
mapper *mediaFileMapper
albumMap *flushableMap
artistMap *flushableMap
cnt *counters
}
func NewTagScanner2(rootFolder string, ds model.DataStore) *TagScanner2 {
return &TagScanner2{
rootFolder: rootFolder,
mapper: newMediaFileMapper(rootFolder),
ds: ds,
}
}
// Scan algorithm overview:
// Load all directories under the music folder, with their ModTime (self or any non-dir children)
// Find changed folders (based on lastModifiedSince) and deletes folders (comparing to the DB)
// For each deleted folder: delete all files from DB whose path starts with the delete folder path
// For each changed folder: Get all files from DB whose path starts with the changed folder, scan each file:
// if file in folder is newer, update the one in DB
// if file in folder does not exists in DB, add
// for each file in the DB that is not found in the folder, delete from DB
// Create new albums/artists, update counters:
// collect all albumIDs and artistIDs from previous steps
// refresh the collected albums and artists with the metadata from the mediafiles
// Delete all empty albums, delete all empty Artists
func (s *TagScanner2) Scan(ctx context.Context, lastModifiedSince time.Time) error {
start := time.Now()
allDirs, err := s.getDirTree(ctx)
if err != nil {
return err
}
changedDirs := s.getChangedDirs(ctx, allDirs, lastModifiedSince)
if len(changedDirs) == 0 {
log.Debug(ctx, "No changes found in Music Folder", "folder", s.rootFolder)
return nil
}
deletedDirs, _ := s.getDeletedDirs(ctx, allDirs, changedDirs)
if log.CurrentLevel() >= log.LevelTrace {
log.Info(ctx, "Folder changes found", "changedFolders", len(changedDirs), "deletedFolders", len(deletedDirs),
"changed", strings.Join(changedDirs, ";"), "deleted", strings.Join(deletedDirs, ";"))
} else {
log.Info(ctx, "Folder changes found", "changedFolders", len(changedDirs), "deletedFolders", len(deletedDirs))
}
s.albumMap = newFlushableMap(ctx, "album", s.ds.Album(ctx).Refresh)
s.artistMap = newFlushableMap(ctx, "artist", s.ds.Artist(ctx).Refresh)
s.cnt = &counters{}
for _, dir := range deletedDirs {
err := s.processDeletedDir(ctx, dir)
if err != nil {
log.Error("Error removing deleted folder from DB", "path", dir, err)
continue
}
}
for _, dir := range changedDirs {
err := s.processChangedDir(ctx, dir)
if err != nil {
log.Error("Error updating folder in the DB", "path", dir, err)
continue
}
}
_ = s.albumMap.flush()
_ = s.artistMap.flush()
err = s.ds.GC(log.NewContext(ctx))
log.Info("Finished processing Music Folder", "folder", s.rootFolder, "elapsed", time.Since(start),
"added", s.cnt.added, "updated", s.cnt.updated, "deleted", s.cnt.deleted)
return err
}
func (s *TagScanner2) getDirTree(ctx context.Context) (dirMap, error) {
start := time.Now()
log.Trace(ctx, "Loading directory tree from music folder", "folder", s.rootFolder)
dirs, err := loadDirTree(ctx, s.rootFolder)
if err != nil {
return nil, err
}
log.Trace("Directory tree loaded", "total", len(dirs), "elapsed", time.Since(start))
return dirs, nil
}
func (s *TagScanner2) getChangedDirs(ctx context.Context, dirs dirMap, lastModified time.Time) []string {
start := time.Now()
log.Trace(ctx, "Checking for changed folders")
var changed []string
for d, t := range dirs {
if t.After(lastModified) {
changed = append(changed, d)
}
}
sort.Strings(changed)
log.Trace(ctx, "Finished changed folders check", "total", len(changed), "elapsed", time.Since(start))
return changed
}
func (s *TagScanner2) getDeletedDirs(ctx context.Context, allDirs dirMap, changedDirs []string) ([]string, error) {
start := time.Now()
log.Trace(ctx, "Checking for deleted folders")
var deleted []string
repo := s.ds.MediaFile(ctx)
// If rootFolder is in the list of changedDirs, optimize and only do one query to the DB
var foldersToCheck []string
if utils.StringInSlice(s.rootFolder, changedDirs) {
foldersToCheck = []string{s.rootFolder}
} else {
foldersToCheck = changedDirs
}
for _, changedDir := range foldersToCheck {
dirs, err := repo.FindPathsRecursively(changedDir)
if err != nil {
log.Error("Error getting subfolders from DB", "path", changedDir, err)
continue
}
for _, d := range dirs {
d := filepath.Clean(d)
if _, ok := allDirs[d]; !ok {
deleted = append(deleted, d)
}
}
}
sort.Strings(deleted)
log.Trace(ctx, "Finished deleted folders check", "total", len(deleted), "elapsed", time.Since(start))
return deleted, nil
}
func (s *TagScanner2) processDeletedDir(ctx context.Context, dir string) error {
start := time.Now()
mfs, err := s.ds.MediaFile(ctx).FindByPath(dir)
if err != nil {
return err
}
for _, t := range mfs {
err = s.albumMap.update(t.AlbumID)
if err != nil {
return err
}
err = s.artistMap.update(t.AlbumArtistID)
if err != nil {
return err
}
}
log.Info(ctx, "Finished processing deleted folder", "path", dir, "purged", len(mfs), "elapsed", time.Since(start))
c, err := s.ds.MediaFile(ctx).DeleteByPath(dir)
s.cnt.deleted += c
return err
}
func (s *TagScanner2) processChangedDir(ctx context.Context, dir string) error {
start := time.Now()
// Load folder's current tracks from DB into a map
currentTracks := map[string]model.MediaFile{}
ct, err := s.ds.MediaFile(ctx).FindByPath(dir)
if err != nil {
return err
}
for _, t := range ct {
currentTracks[t.Path] = t
}
// Load tracks FileInfo from the folder
files, err := LoadAllAudioFiles(dir)
if err != nil {
return err
}
// If no files to process, return
if len(files)+len(currentTracks) == 0 {
return nil
}
// If track from folder is newer than the one in DB, select for update/insert in DB and delete from the current tracks
log.Trace(ctx, "Processing changed folder", "dir", dir, "tracksInDB", len(currentTracks), "tracksInFolder", len(files))
var filesToUpdate []string
for filePath, info := range files {
c, ok := currentTracks[filePath]
if !ok {
filesToUpdate = append(filesToUpdate, filePath)
s.cnt.added++
}
if ok && info.ModTime().After(c.UpdatedAt) {
filesToUpdate = append(filesToUpdate, filePath)
s.cnt.updated++
}
delete(currentTracks, filePath)
// Force a refresh of the album and artist, to cater for cover art files
err = s.albumMap.update(c.AlbumID)
if err != nil {
return err
}
err = s.artistMap.update(c.AlbumArtistID)
if err != nil {
return err
}
}
numUpdatedTracks := 0
numPurgedTracks := 0
if len(filesToUpdate) > 0 {
// Break the file list in chunks to avoid calling ffmpeg with too many parameters
chunks := utils.BreakUpStringSlice(filesToUpdate, filesBatchSize)
for _, chunk := range chunks {
// Load tracks Metadata from the folder
newTracks, err := s.loadTracks(chunk)
if err != nil {
return err
}
// If track from folder is newer than the one in DB, update/insert in DB
log.Trace(ctx, "Updating mediaFiles in DB", "dir", dir, "files", chunk, "numFiles", len(chunk))
for i := range newTracks {
n := newTracks[i]
err := s.ds.MediaFile(ctx).Put(&n)
if err != nil {
return err
}
err = s.albumMap.update(n.AlbumID)
if err != nil {
return err
}
err = s.artistMap.update(n.AlbumArtistID)
if err != nil {
return err
}
numUpdatedTracks++
}
}
}
if len(currentTracks) > 0 {
log.Trace(ctx, "Deleting dangling tracks from DB", "dir", dir, "numTracks", len(currentTracks))
// Remaining tracks from DB that are not in the folder are deleted
for _, ct := range currentTracks {
numPurgedTracks++
err = s.albumMap.update(ct.AlbumID)
if err != nil {
return err
}
err = s.artistMap.update(ct.AlbumArtistID)
if err != nil {
return err
}
if err := s.ds.MediaFile(ctx).Delete(ct.ID); err != nil {
return err
}
s.cnt.deleted++
}
}
log.Info(ctx, "Finished processing changed folder", "dir", dir, "updated", numUpdatedTracks, "purged", numPurgedTracks, "elapsed", time.Since(start))
return nil
}
func (s *TagScanner2) loadTracks(filePaths []string) (model.MediaFiles, error) {
mds, err := ExtractAllMetadata(filePaths)
if err != nil {
return nil, err
}
var mfs model.MediaFiles
for _, md := range mds {
mf := s.mapper.toMediaFile(md)
mfs = append(mfs, mf)
}
return mfs, nil
}