Reduced size of batches, to not break SQLite

persistence/album_repository.go

@@ -129,7 +129,7 @@ group by album_id order by f.id`, strings.Join(ids, "','"))
 		}
 	}
 	if len(toInsert) > 0 {
-		n, err := o.InsertMulti(100, toInsert)
+		n, err := o.InsertMulti(10, toInsert)
 		if err != nil {
 			return err
 		}

persistence/artist_repository.go

@@ -64,6 +64,7 @@ func (r *artistRepository) Get(id string) (*model.Artist, error) {
 // TODO Cache the index (recalculate when there are changes to the DB)
 func (r *artistRepository) GetIndex() (model.ArtistIndexes, error) {
 	var all []artist
+	// TODO Paginate
 	_, err := r.newQuery(Db()).OrderBy("name").All(&all)
 	if err != nil {
 		return nil, err
@@ -133,7 +134,7 @@ where f.artist_id in ('%s') group by f.artist_id order by f.id`, strings.Join(id
 		}
 	}
 	if len(toInsert) > 0 {
-		n, err := o.InsertMulti(100, toInsert)
+		n, err := o.InsertMulti(10, toInsert)
 		if err != nil {
 			return err
 		}

persistence/mediafile_repository.go

@@ -105,6 +105,7 @@ func (r *mediaFileRepository) FindByPath(path string) (model.MediaFiles, error)
 func (r *mediaFileRepository) DeleteByPath(path string) error {
 	o := Db()
 	var mfs []mediaFile
+	// TODO Paginate this (and all other situations similar)
 	_, err := r.newQuery(o).Filter("path__istartswith", path).OrderBy("disc_number", "track_number").All(&mfs)
 	if err != nil {
 		return err

scanner/tag_scanner.go

@@ -7,6 +7,7 @@ import (
 	"os"
 	"path"
 	"path/filepath"
+	"sort"
 	"strconv"
 	"strings"
 	"time"
@@ -51,6 +52,9 @@ func (s *TagScanner) Scan(ctx context.Context, lastModifiedSince time.Time) erro
 
 	log.Info("Folder changes found", "changed", len(changed), "deleted", len(deleted))
 
+	sort.Strings(changed)
+	sort.Strings(deleted)
+
 	updatedArtists := map[string]bool{}
 	updatedAlbums := map[string]bool{}
 
@@ -59,12 +63,36 @@ func (s *TagScanner) Scan(ctx context.Context, lastModifiedSince time.Time) erro
 		if err != nil {
 			return err
 		}
+		if len(updatedAlbums)+len(updatedArtists) > 100 {
+			err = s.refreshAlbums(updatedAlbums)
+			if err != nil {
+				return err
+			}
+			err = s.refreshArtists(updatedArtists)
+			if err != nil {
+				return err
+			}
+			updatedAlbums = map[string]bool{}
+			updatedArtists = map[string]bool{}
+		}
 	}
 	for _, c := range deleted {
 		err := s.processDeletedDir(c, updatedArtists, updatedAlbums)
 		if err != nil {
 			return err
 		}
+		if len(updatedAlbums)+len(updatedArtists) > 100 {
+			err = s.refreshAlbums(updatedAlbums)
+			if err != nil {
+				return err
+			}
+			err = s.refreshArtists(updatedArtists)
+			if err != nil {
+				return err
+			}
+			updatedAlbums = map[string]bool{}
+			updatedArtists = map[string]bool{}
+		}
 	}
 
 	err = s.refreshAlbums(updatedAlbums)
@@ -129,6 +157,11 @@ func (s *TagScanner) processChangedDir(dir string, updatedArtists map[string]boo
 		return err
 	}
 
+	// If no tracks to process, return
+	if len(newTracks)+len(currentTracks) == 0 {
+		return nil
+	}
+
 	// If track from folder is newer than the one in DB, update/insert in DB and delete from the current tracks
 	log.Trace("Processing changed folder", "dir", dir, "tracksInDB", len(currentTracks), "tracksInFolder", len(newTracks))
 	numUpdatedTracks := 0