Fix some performance/volume issues

persistence/db_sql/album_repository.go

@@ -35,7 +35,7 @@ type albumRepository struct {
 
 func NewAlbumRepository() domain.AlbumRepository {
 	r := &albumRepository{}
-	r.entityName = "album"
+	r.tableName = "album"
 	return r
 }
 

persistence/db_sql/artist_repository.go

@@ -18,7 +18,7 @@ type artistRepository struct {
 
 func NewArtistRepository() domain.ArtistRepository {
 	r := &artistRepository{}
-	r.entityName = "artist"
+	r.tableName = "artist"
 	return r
 }
 

persistence/db_sql/checksum_repository.go

@@ -1,8 +1,6 @@
 package db_sql
 
 import (
-	"encoding/json"
-
 	"github.com/astaxie/beego/orm"
 	"github.com/cloudsonic/sonic-server/log"
 	"github.com/cloudsonic/sonic-server/scanner"
@@ -15,8 +13,8 @@ type checkSumRepository struct {
 const checkSumId = "1"
 
 type CheckSums struct {
-	ID   string `orm:"pk;column(id)"`
+	ID    string `orm:"pk;column(id)"`
-	Data string `orm:"type(text)"`
+	Value string
 }
 
 func NewCheckSumRepository() scanner.CheckSumRepository {
@@ -26,20 +24,20 @@ func NewCheckSumRepository() scanner.CheckSumRepository {
 
 func (r *checkSumRepository) loadData() error {
 	loadedData := make(map[string]string)
-	r.data = loadedData
 
-	cks := CheckSums{ID: checkSumId}
+	var all []CheckSums
-	err := Db().Read(&cks)
+	_, err := Db().QueryTable(&CheckSums{}).All(&all)
-	if err == orm.ErrNoRows {
-		_, err = Db().Insert(&cks)
-		return err
-	}
 	if err != nil {
 		return err
 	}
-	_ = json.Unmarshal([]byte(cks.Data), &loadedData)
+
+	for _, cks := range all {
+		loadedData[cks.ID] = cks.Value
+	}
+
+	r.data = loadedData
 	log.Debug("Loaded checksums", "total", len(loadedData))
-	return err
+	return nil
 }
 
 func (r *checkSumRepository) Get(id string) (string, error) {
@@ -53,14 +51,21 @@ func (r *checkSumRepository) Get(id string) (string, error) {
 }
 
 func (r *checkSumRepository) SetData(newSums map[string]string) error {
-	data, _ := json.Marshal(&newSums)
+	err := WithTx(func(o orm.Ormer) error {
-	cks := CheckSums{ID: checkSumId, Data: string(data)}
+		_, err := Db().Raw("delete from check_sums").Exec()
-	var err error
+		if err != nil {
-	if Db().QueryTable(&CheckSums{}).Filter("id", checkSumId).Exist() {
+			return err
-		_, err = Db().Update(&cks)
+		}
-	} else {
+
-		_, err = Db().Insert(&cks)
+		for k, v := range newSums {
-	}
+			cks := CheckSums{ID: k, Value: v}
+			_, err := Db().Insert(&cks)
+			if err != nil {
+				return err
+			}
+		}
+		return nil
+	})
 	if err != nil {
 		return err
 	}

persistence/db_sql/index_repository.go

@@ -21,7 +21,7 @@ type artistIndexRepository struct {
 
 func NewArtistIndexRepository() domain.ArtistIndexRepository {
 	r := &artistIndexRepository{}
-	r.entityName = "artist_info"
+	r.tableName = "artist_info"
 	return r
 }
 
@@ -106,7 +106,7 @@ func (r *artistIndexRepository) GetAll() (domain.ArtistIndexes, error) {
 }
 
 func (r *artistIndexRepository) DeleteAll() error {
-	_, err := r.newQuery(Db()).Exclude("idx", ".PHONY").Delete()
+	_, err := Db().Raw("delete from artist_info").Exec()
 	return err
 }
 

persistence/db_sql/mediafile_repository.go

@@ -41,7 +41,7 @@ type mediaFileRepository struct {
 
 func NewMediaFileRepository() domain.MediaFileRepository {
 	r := &mediaFileRepository{}
-	r.entityName = "media_file"
+	r.tableName = "media_file"
 	return r
 }
 

persistence/db_sql/playlist_repository.go

@@ -24,7 +24,7 @@ type playlistRepository struct {
 
 func NewPlaylistRepository() domain.PlaylistRepository {
 	r := &playlistRepository{}
-	r.entityName = "playlist"
+	r.tableName = "playlist"
 	return r
 }
 

persistence/db_sql/property_repository.go

@@ -16,7 +16,7 @@ type propertyRepository struct {
 
 func NewPropertyRepository() domain.PropertyRepository {
 	r := &propertyRepository{}
-	r.entityName = "property"
+	r.tableName = "property"
 	return r
 }
 

persistence/db_sql/property_repository_test.go

@@ -10,7 +10,7 @@ var _ = Describe("PropertyRepository", func() {
 	var repo domain.PropertyRepository
 
 	BeforeEach(func() {
-		Db().QueryTable("property").Exclude("id", ".PHONY").Delete()
+		Db().Raw("delete from property").Exec()
 		repo = NewPropertyRepository()
 	})
 

persistence/db_sql/sql_repository.go

@@ -3,15 +3,16 @@ package db_sql
 import (
 	"github.com/astaxie/beego/orm"
 	"github.com/cloudsonic/sonic-server/domain"
+	"github.com/cloudsonic/sonic-server/log"
 	"github.com/cloudsonic/sonic-server/persistence"
 )
 
 type sqlRepository struct {
-	entityName string
+	tableName string
 }
 
 func (r *sqlRepository) newQuery(o orm.Ormer, options ...domain.QueryOptions) orm.QuerySeter {
-	q := o.QueryTable(r.entityName)
+	q := o.QueryTable(r.tableName)
 	if len(options) > 0 {
 		opts := options[0]
 		q = q.Offset(opts.Offset)
@@ -69,23 +70,66 @@ func (r *sqlRepository) put(id string, a interface{}) error {
 	})
 }
 
-func (r *sqlRepository) purgeInactive(activeList interface{}, getId func(item interface{}) string) ([]string, error) {
+func paginateSlice(slice []string, skip int, size int) []string {
-	ids := persistence.CollectValue(activeList, getId)
+	if skip > len(slice) {
-	var values []orm.Params
+		skip = len(slice)
-	err := WithTx(func(o orm.Ormer) error {
+	}
-		qs := r.newQuery(o).Exclude("id__in", ids)
+
-		num, err := qs.Values(&values, "id")
+	end := skip + size
-		if num > 0 {
+	if end > len(slice) {
-			_, err = qs.Delete()
+		end = len(slice)
+	}
+
+	return slice[skip:end]
+}
+
+func difference(slice1 []string, slice2 []string) []string {
+	var diffStr []string
+	m := map[string]int{}
+
+	for _, s1Val := range slice1 {
+		m[s1Val] = 1
+	}
+	for _, s2Val := range slice2 {
+		m[s2Val] = m[s2Val] + 1
+	}
+
+	for mKey, mVal := range m {
+		if mVal == 1 {
+			diffStr = append(diffStr, mKey)
 		}
-		return err
+	}
-	})
+
+	return diffStr
+}
+
+func (r *sqlRepository) purgeInactive(activeList interface{}, getId func(item interface{}) string) ([]string, error) {
+	allIds, err := r.GetAllIds()
 	if err != nil {
 		return nil, err
 	}
-	result := make([]string, len(values))
+	activeIds := persistence.CollectValue(activeList, getId)
-	for i, v := range values {
+	idsToDelete := difference(allIds, activeIds)
-		result[i] = v["ID"].(string)
+	if len(idsToDelete) == 0 {
+		return nil, nil
 	}
-	return result, nil
+	log.Debug("Purging inactive records", "table", r.tableName, "total", len(idsToDelete))
+
+	err = WithTx(func(o orm.Ormer) error {
+		var offset int
+		for {
+			var subset = paginateSlice(idsToDelete, offset, 100)
+			if len(subset) == 0 {
+				break
+			}
+			log.Trace("-- Purging inactive records", "table", r.tableName, "num", len(subset), "from", offset)
+			offset += len(subset)
+			_, err := r.newQuery(o).Filter("id__in", subset).Delete()
+			if err != nil {
+				return err
+			}
+		}
+		return nil
+	})
+	return idsToDelete, err
 }