feat: make rescan faster, only loading metadata from changed files

scanner/metadata_ffmpeg.go

@@ -43,7 +43,7 @@ func (m *Metadata) FilePath() string            { return m.filePath }
 func (m *Metadata) Suffix() string              { return m.suffix }
 func (m *Metadata) Size() int                   { return int(m.fileInfo.Size()) }
 
-func LoadAllAudioFiles(dirPath string) ([]os.FileInfo, error) {
+func LoadAllAudioFiles(dirPath string) (map[string]os.FileInfo, error) {
 	dir, err := os.Open(dirPath)
 	if err != nil {
 		return nil, err
@@ -52,7 +52,7 @@ func LoadAllAudioFiles(dirPath string) ([]os.FileInfo, error) {
 	if err != nil {
 		return nil, err
 	}
-	var audioFiles []os.FileInfo
+	audioFiles := make(map[string]os.FileInfo)
 	for _, f := range files {
 		if f.IsDir() {
 			continue
@@ -66,7 +66,7 @@ func LoadAllAudioFiles(dirPath string) ([]os.FileInfo, error) {
 		if err != nil {
 			log.Error("Could not stat file", "filePath", filePath, err)
 		} else {
-			audioFiles = append(audioFiles, fi)
+			audioFiles[filePath] = fi
 		}
 	}
 

scanner/metadata_test.go

@@ -7,7 +7,7 @@ import (
 
 var _ = Describe("Metadata", func() {
 	// TODO Need to mock `ffmpeg`
-	XContext("ExtractAllMetadata", func() {
+	Context("ExtractAllMetadata", func() {
 		It("correctly parses metadata from all files in folder", func() {
 			mds, err := ExtractAllMetadata([]string{"tests/fixtures/test.mp3", "tests/fixtures/test.ogg"})
 			Expect(err).NotTo(HaveOccurred())
@@ -52,6 +52,9 @@ var _ = Describe("Metadata", func() {
 			files, err := LoadAllAudioFiles("tests/fixtures")
 			Expect(err).ToNot(HaveOccurred())
 			Expect(files).To(HaveLen(3))
+			Expect(files).To(HaveKey("tests/fixtures/test.ogg"))
+			Expect(files).To(HaveKey("tests/fixtures/test.mp3"))
+			Expect(files).To(HaveKey("tests/fixtures/01 Invisible (RED) Edit Version.mp3"))
 		})
 		It("returns error if path does not exist", func() {
 			_, err := LoadAllAudioFiles("./INVALID/PATH")

scanner/tag_scanner.go

@@ -143,27 +143,42 @@ func (s *TagScanner) processChangedDir(ctx context.Context, dir string, updatedA
 		return err
 	}
 	for _, t := range ct {
-		currentTracks[t.ID] = t
+		currentTracks[t.Path] = t
 	}
 
-	// Load tracks from the folder
+	// Load tracks FileInfo from the folder
-	newTracks, err := s.loadTracks(dir)
+	files, err := LoadAllAudioFiles(dir)
 	if err != nil {
 		return err
 	}
 
-	// If no tracks to process, return
+	// If no files to process, return
-	if len(newTracks)+len(currentTracks) == 0 {
+	if len(files)+len(currentTracks) == 0 {
 		return nil
 	}
 
+	// If track from folder is newer than the one in DB, select for update/insert in DB and delete from the current tracks
+	log.Trace("Processing changed folder", "dir", dir, "tracksInDB", len(currentTracks), "tracksInFolder", len(files))
+	var filesToUpdate []string
+	for filePath, info := range files {
+		c, ok := currentTracks[filePath]
+		if !ok || (ok && info.ModTime().After(c.UpdatedAt)) {
+			filesToUpdate = append(filesToUpdate, filePath)
+		}
+		delete(currentTracks, filePath)
+	}
+
+	// Load tracks Metadata from the folder
+	newTracks, err := s.loadTracks(filesToUpdate)
+	if err != nil {
+		return err
+	}
+
 	// If track from folder is newer than the one in DB, update/insert in DB and delete from the current tracks
-	log.Trace("Processing changed folder", "dir", dir, "tracksInDB", len(currentTracks), "tracksInFolder", len(newTracks))
+	log.Trace("Updating mediaFiles in DB", "dir", dir, "files", filesToUpdate, "numFiles", len(filesToUpdate))
 	numUpdatedTracks := 0
 	numPurgedTracks := 0
 	for _, n := range newTracks {
-		c, ok := currentTracks[n.ID]
-		if !ok || (ok && n.UpdatedAt.After(c.UpdatedAt)) {
 		err := s.ds.MediaFile(ctx).Put(&n)
 		updatedArtists[n.ArtistID] = true
 		updatedAlbums[n.AlbumID] = true
@@ -172,15 +187,13 @@ func (s *TagScanner) processChangedDir(ctx context.Context, dir string, updatedA
 			return err
 		}
 	}
-		delete(currentTracks, n.ID)
-	}
 
 	// Remaining tracks from DB that are not in the folder are deleted
-	for id, ct := range currentTracks {
+	for _, ct := range currentTracks {
 		numPurgedTracks++
 		updatedArtists[ct.ArtistID] = true
 		updatedAlbums[ct.AlbumID] = true
-		if err := s.ds.MediaFile(ctx).Delete(id); err != nil {
+		if err := s.ds.MediaFile(ctx).Delete(ct.ID); err != nil {
 			return err
 		}
 	}
@@ -206,17 +219,7 @@ func (s *TagScanner) processDeletedDir(ctx context.Context, dir string, updatedA
 	return s.ds.MediaFile(ctx).DeleteByPath(dir)
 }
 
-func (s *TagScanner) loadTracks(dirPath string) (model.MediaFiles, error) {
+func (s *TagScanner) loadTracks(filePaths []string) (model.MediaFiles, error) {
-	audioFiles, err := LoadAllAudioFiles(dirPath)
-	if err != nil {
-		return nil, err
-	}
-
-	filePaths := make([]string, len(audioFiles))
-	for i, _ := range audioFiles {
-		filePaths[i] = filepath.Join(dirPath, audioFiles[i].Name())
-	}
-
 	mds, err := ExtractAllMetadata(filePaths)
 	if err != nil {
 		return nil, err