From fddd0dc353d41d54d9c8d769de192c11878bc8e0 Mon Sep 17 00:00:00 2001 From: Deluan Date: Mon, 13 Jan 2020 00:04:11 -0500 Subject: [PATCH] Fix some performance/volume issues --- persistence/db_sql/album_repository.go | 2 +- persistence/db_sql/artist_repository.go | 2 +- persistence/db_sql/checksum_repository.go | 47 +++++++----- persistence/db_sql/index_repository.go | 4 +- persistence/db_sql/mediafile_repository.go | 2 +- persistence/db_sql/playlist_repository.go | 2 +- persistence/db_sql/property_repository.go | 2 +- .../db_sql/property_repository_test.go | 2 +- persistence/db_sql/sql_repository.go | 76 +++++++++++++++---- 9 files changed, 94 insertions(+), 45 deletions(-) diff --git a/persistence/db_sql/album_repository.go b/persistence/db_sql/album_repository.go index 92104d99..37d1c436 100644 --- a/persistence/db_sql/album_repository.go +++ b/persistence/db_sql/album_repository.go @@ -35,7 +35,7 @@ type albumRepository struct { func NewAlbumRepository() domain.AlbumRepository { r := &albumRepository{} - r.entityName = "album" + r.tableName = "album" return r } diff --git a/persistence/db_sql/artist_repository.go b/persistence/db_sql/artist_repository.go index 5ec2062a..43854f2d 100644 --- a/persistence/db_sql/artist_repository.go +++ b/persistence/db_sql/artist_repository.go @@ -18,7 +18,7 @@ type artistRepository struct { func NewArtistRepository() domain.ArtistRepository { r := &artistRepository{} - r.entityName = "artist" + r.tableName = "artist" return r } diff --git a/persistence/db_sql/checksum_repository.go b/persistence/db_sql/checksum_repository.go index 32137070..b4956b4d 100644 --- a/persistence/db_sql/checksum_repository.go +++ b/persistence/db_sql/checksum_repository.go @@ -1,8 +1,6 @@ package db_sql import ( - "encoding/json" - "github.com/astaxie/beego/orm" "github.com/cloudsonic/sonic-server/log" "github.com/cloudsonic/sonic-server/scanner" @@ -15,8 +13,8 @@ type checkSumRepository struct { const checkSumId = "1" type CheckSums struct { - ID string `orm:"pk;column(id)"` - Data string `orm:"type(text)"` + ID string `orm:"pk;column(id)"` + Value string } func NewCheckSumRepository() scanner.CheckSumRepository { @@ -26,20 +24,20 @@ func NewCheckSumRepository() scanner.CheckSumRepository { func (r *checkSumRepository) loadData() error { loadedData := make(map[string]string) - r.data = loadedData - cks := CheckSums{ID: checkSumId} - err := Db().Read(&cks) - if err == orm.ErrNoRows { - _, err = Db().Insert(&cks) - return err - } + var all []CheckSums + _, err := Db().QueryTable(&CheckSums{}).All(&all) if err != nil { return err } - _ = json.Unmarshal([]byte(cks.Data), &loadedData) + + for _, cks := range all { + loadedData[cks.ID] = cks.Value + } + + r.data = loadedData log.Debug("Loaded checksums", "total", len(loadedData)) - return err + return nil } func (r *checkSumRepository) Get(id string) (string, error) { @@ -53,14 +51,21 @@ func (r *checkSumRepository) Get(id string) (string, error) { } func (r *checkSumRepository) SetData(newSums map[string]string) error { - data, _ := json.Marshal(&newSums) - cks := CheckSums{ID: checkSumId, Data: string(data)} - var err error - if Db().QueryTable(&CheckSums{}).Filter("id", checkSumId).Exist() { - _, err = Db().Update(&cks) - } else { - _, err = Db().Insert(&cks) - } + err := WithTx(func(o orm.Ormer) error { + _, err := Db().Raw("delete from check_sums").Exec() + if err != nil { + return err + } + + for k, v := range newSums { + cks := CheckSums{ID: k, Value: v} + _, err := Db().Insert(&cks) + if err != nil { + return err + } + } + return nil + }) if err != nil { return err } diff --git a/persistence/db_sql/index_repository.go b/persistence/db_sql/index_repository.go index f4b343de..f0f17b56 100644 --- a/persistence/db_sql/index_repository.go +++ b/persistence/db_sql/index_repository.go @@ -21,7 +21,7 @@ type artistIndexRepository struct { func NewArtistIndexRepository() domain.ArtistIndexRepository { r := &artistIndexRepository{} - r.entityName = "artist_info" + r.tableName = "artist_info" return r } @@ -106,7 +106,7 @@ func (r *artistIndexRepository) GetAll() (domain.ArtistIndexes, error) { } func (r *artistIndexRepository) DeleteAll() error { - _, err := r.newQuery(Db()).Exclude("idx", ".PHONY").Delete() + _, err := Db().Raw("delete from artist_info").Exec() return err } diff --git a/persistence/db_sql/mediafile_repository.go b/persistence/db_sql/mediafile_repository.go index f258f633..b40ee266 100644 --- a/persistence/db_sql/mediafile_repository.go +++ b/persistence/db_sql/mediafile_repository.go @@ -41,7 +41,7 @@ type mediaFileRepository struct { func NewMediaFileRepository() domain.MediaFileRepository { r := &mediaFileRepository{} - r.entityName = "media_file" + r.tableName = "media_file" return r } diff --git a/persistence/db_sql/playlist_repository.go b/persistence/db_sql/playlist_repository.go index 6a8dc066..2cd4a1ab 100644 --- a/persistence/db_sql/playlist_repository.go +++ b/persistence/db_sql/playlist_repository.go @@ -24,7 +24,7 @@ type playlistRepository struct { func NewPlaylistRepository() domain.PlaylistRepository { r := &playlistRepository{} - r.entityName = "playlist" + r.tableName = "playlist" return r } diff --git a/persistence/db_sql/property_repository.go b/persistence/db_sql/property_repository.go index b831290a..64af618e 100644 --- a/persistence/db_sql/property_repository.go +++ b/persistence/db_sql/property_repository.go @@ -16,7 +16,7 @@ type propertyRepository struct { func NewPropertyRepository() domain.PropertyRepository { r := &propertyRepository{} - r.entityName = "property" + r.tableName = "property" return r } diff --git a/persistence/db_sql/property_repository_test.go b/persistence/db_sql/property_repository_test.go index 8d4a455d..b0bfd14d 100644 --- a/persistence/db_sql/property_repository_test.go +++ b/persistence/db_sql/property_repository_test.go @@ -10,7 +10,7 @@ var _ = Describe("PropertyRepository", func() { var repo domain.PropertyRepository BeforeEach(func() { - Db().QueryTable("property").Exclude("id", ".PHONY").Delete() + Db().Raw("delete from property").Exec() repo = NewPropertyRepository() }) diff --git a/persistence/db_sql/sql_repository.go b/persistence/db_sql/sql_repository.go index b7853370..a561ad04 100644 --- a/persistence/db_sql/sql_repository.go +++ b/persistence/db_sql/sql_repository.go @@ -3,15 +3,16 @@ package db_sql import ( "github.com/astaxie/beego/orm" "github.com/cloudsonic/sonic-server/domain" + "github.com/cloudsonic/sonic-server/log" "github.com/cloudsonic/sonic-server/persistence" ) type sqlRepository struct { - entityName string + tableName string } func (r *sqlRepository) newQuery(o orm.Ormer, options ...domain.QueryOptions) orm.QuerySeter { - q := o.QueryTable(r.entityName) + q := o.QueryTable(r.tableName) if len(options) > 0 { opts := options[0] q = q.Offset(opts.Offset) @@ -69,23 +70,66 @@ func (r *sqlRepository) put(id string, a interface{}) error { }) } -func (r *sqlRepository) purgeInactive(activeList interface{}, getId func(item interface{}) string) ([]string, error) { - ids := persistence.CollectValue(activeList, getId) - var values []orm.Params - err := WithTx(func(o orm.Ormer) error { - qs := r.newQuery(o).Exclude("id__in", ids) - num, err := qs.Values(&values, "id") - if num > 0 { - _, err = qs.Delete() +func paginateSlice(slice []string, skip int, size int) []string { + if skip > len(slice) { + skip = len(slice) + } + + end := skip + size + if end > len(slice) { + end = len(slice) + } + + return slice[skip:end] +} + +func difference(slice1 []string, slice2 []string) []string { + var diffStr []string + m := map[string]int{} + + for _, s1Val := range slice1 { + m[s1Val] = 1 + } + for _, s2Val := range slice2 { + m[s2Val] = m[s2Val] + 1 + } + + for mKey, mVal := range m { + if mVal == 1 { + diffStr = append(diffStr, mKey) } - return err - }) + } + + return diffStr +} + +func (r *sqlRepository) purgeInactive(activeList interface{}, getId func(item interface{}) string) ([]string, error) { + allIds, err := r.GetAllIds() if err != nil { return nil, err } - result := make([]string, len(values)) - for i, v := range values { - result[i] = v["ID"].(string) + activeIds := persistence.CollectValue(activeList, getId) + idsToDelete := difference(allIds, activeIds) + if len(idsToDelete) == 0 { + return nil, nil } - return result, nil + log.Debug("Purging inactive records", "table", r.tableName, "total", len(idsToDelete)) + + err = WithTx(func(o orm.Ormer) error { + var offset int + for { + var subset = paginateSlice(idsToDelete, offset, 100) + if len(subset) == 0 { + break + } + log.Trace("-- Purging inactive records", "table", r.tableName, "num", len(subset), "from", offset) + offset += len(subset) + _, err := r.newQuery(o).Filter("id__in", subset).Delete() + if err != nil { + return err + } + } + return nil + }) + return idsToDelete, err }