arian/navidrome
1
0
Fork 0
Code Issues Pull Requests 1 Packages Projects Releases Wiki Activity

Using squirrel to generalize SQL search

Browse Source
This commit is contained in:
Deluan
2020-01-13 15:28:55 -05:00
committed by Deluan Quintão
parent d3af7e689d
commit c2448d3880
10 changed files with 134 additions and 34 deletions
Download Patch File Download Diff File Expand all files Collapse all files
persistence/db_sql/album_repository.go
+3 -1
View File
@@ -41,7 +41,9 @@ func NewAlbumRepository() domain.AlbumRepository {
func (r *albumRepository) Put(a *domain.Album) error {
ta := Album(*a)
return r.put(a.ID, &ta)
return WithTx(func(o orm.Ormer) error {
return r.put(o, a.ID, &ta)
})
}
func (r *albumRepository) Get(id string) (*domain.Album, error) {
persistence/db_sql/artist_repository.go
+3 -1
View File
@@ -24,7 +24,9 @@ func NewArtistRepository() domain.ArtistRepository {
func (r *artistRepository) Put(a *domain.Artist) error {
ta := Artist(*a)
return r.put(a.ID, &ta)
return WithTx(func(o orm.Ormer) error {
return r.put(o, a.ID, &ta)
})
}
func (r *artistRepository) Get(id string) (*domain.Artist, error) {
persistence/db_sql/checksum_repository.go
+1 -1
View File
@@ -62,7 +62,7 @@ func (r *checkSumRepository) SetData(newSums map[string]string) error {
cks := Checksum{ID: k, Sum: v}
checksums = append(checksums, cks)
}
_, err = Db().InsertMulti(100, &checksums)
_, err = Db().InsertMulti(batchSize, &checksums)
if err != nil {
return err
}
persistence/db_sql/mediafile_repository.go
+12 -14
View File
@@ -1,7 +1,6 @@
package db_sql
import (
"strings"
"time"
"github.com/astaxie/beego/orm"
@@ -27,9 +26,9 @@ type MediaFile struct {
BitRate int ``
Genre string ``
Compilation bool ``
PlayCount int ``
PlayCount int `orm:"index"`
PlayDate time.Time `orm:"null"`
Rating int ``
Rating int `orm:"index"`
Starred bool `orm:"index"`
StarredAt time.Time `orm:"null"`
CreatedAt time.Time `orm:"null"`
@@ -48,7 +47,13 @@ func NewMediaFileRepository() domain.MediaFileRepository {
func (r *mediaFileRepository) Put(m *domain.MediaFile) error {
tm := MediaFile(*m)
return r.put(m.ID, &tm)
return WithTx(func(o orm.Ormer) error {
err := r.put(o, m.ID, &tm)
if err != nil {
return err
}
return r.searcher.Index(o, r.tableName, m.ID, m.Title)
})
}
func (r *mediaFileRepository) Get(id string) (*domain.MediaFile, error) {
@@ -97,19 +102,12 @@ func (r *mediaFileRepository) PurgeInactive(activeList domain.MediaFiles) ([]str
}
func (r *mediaFileRepository) Search(q string, offset int, size int) (domain.MediaFiles, error) {
parts := strings.Split(q, " ")
if len(parts) == 0 {
if len(q) <= 2 {
return nil, nil
}
qs := r.newQuery(Db(), domain.QueryOptions{Offset: offset, Size: size})
cond := orm.NewCondition()
for _, part := range parts {
c := orm.NewCondition()
cond = cond.AndCond(c.Or("title__istartswith", part).Or("title__icontains", " "+part))
}
qs = qs.SetCond(cond).OrderBy("-rating", "-starred", "-play_count")
var results []MediaFile
_, err := qs.All(&results)
err := r.searcher.Search(r.tableName, q, offset, size, &results, "rating desc", "starred desc", "play_count desc", "title")
if err != nil {
return nil, err
}
persistence/db_sql/playlist_repository.go
+3 -1
View File
@@ -30,7 +30,9 @@ func NewPlaylistRepository() domain.PlaylistRepository {
func (r *playlistRepository) Put(p *domain.Playlist) error {
tp := r.fromDomain(p)
return r.put(p.ID, &tp)
return WithTx(func(o orm.Ormer) error {
return r.put(o, p.ID, &tp)
})
}
func (r *playlistRepository) Get(id string) (*domain.Playlist, error) {
persistence/db_sql/sql.go
+3
View File
@@ -11,6 +11,8 @@ import (
_ "github.com/mattn/go-sqlite3"
)
const batchSize = 100
var once sync.Once
func Db() orm.Ormer {
@@ -66,6 +68,7 @@ func initORM(dbPath string) error {
orm.RegisterModel(new(Checksum))
orm.RegisterModel(new(Property))
orm.RegisterModel(new(Playlist))
orm.RegisterModel(new(Search))
err := orm.RegisterDataBase("default", "sqlite3", dbPath)
if err != nil {
panic(err)
persistence/db_sql/sql_repository.go
+20 -16
View File
@@ -9,6 +9,7 @@ import (
type sqlRepository struct {
tableName string
searcher sqlSearcher
}
func (r *sqlRepository) newQuery(o orm.Ormer, options ...domain.QueryOptions) orm.QuerySeter {
@@ -55,19 +56,17 @@ func (r *sqlRepository) GetAllIds() ([]string, error) {
return result, nil
}
func (r *sqlRepository) put(id string, a interface{}) error {
return WithTx(func(o orm.Ormer) error {
c, err := r.newQuery(o).Filter("id", id).Count()
if err != nil {
return err
}
if c == 0 {
_, err = o.Insert(a)
return err
}
_, err = o.Update(a)
func (r *sqlRepository) put(o orm.Ormer, id string, a interface{}) error {
c, err := r.newQuery(o).Filter("id", id).Count()
if err != nil {
return err
})
}
if c == 0 {
_, err = o.Insert(a)
return err
}
_, err = o.Update(a)
return err
}
func paginateSlice(slice []string, skip int, size int) []string {
@@ -104,8 +103,13 @@ func difference(slice1 []string, slice2 []string) []string {
}
func (r *sqlRepository) DeleteAll() error {
_, err := r.newQuery(Db()).Filter("id__isnull", false).Delete()
return err
return WithTx(func(o orm.Ormer) error {
_, err := r.newQuery(Db()).Filter("id__isnull", false).Delete()
if err != nil {
return err
}
return r.searcher.DeleteAll(o, r.tableName)
})
}
func (r *sqlRepository) purgeInactive(activeList interface{}, getId func(item interface{}) string) ([]string, error) {
@@ -123,7 +127,7 @@ func (r *sqlRepository) purgeInactive(activeList interface{}, getId func(item in
err = WithTx(func(o orm.Ormer) error {
var offset int
for {
var subset = paginateSlice(idsToDelete, offset, 100)
var subset = paginateSlice(idsToDelete, offset, batchSize)
if len(subset) == 0 {
break
}
@@ -134,7 +138,7 @@ func (r *sqlRepository) purgeInactive(activeList interface{}, getId func(item in
return err
}
}
return nil
return r.searcher.Remove(o, r.tableName, idsToDelete)
})
return idsToDelete, err
}
persistence/db_sql/sql_searcher.go
+82
View File
@@ -0,0 +1,82 @@
package db_sql
import (
"strings"
"github.com/Masterminds/squirrel"
"github.com/astaxie/beego/orm"
"github.com/cloudsonic/sonic-server/log"
"github.com/kennygrant/sanitize"
)
type Search struct {
ID string `orm:"pk;column(id)"`
Table string `orm:"index"`
FullText string `orm:"type(text)"`
}
type sqlSearcher struct{}
func (s *sqlSearcher) Index(o orm.Ormer, table, id, text string) error {
item := Search{ID: id, Table: table}
err := o.Read(&item)
if err != nil && err != orm.ErrNoRows {
return err
}
sanitizedText := strings.TrimSpace(sanitize.Accents(strings.ToLower(text)))
item = Search{ID: id, Table: table, FullText: sanitizedText}
if err == orm.ErrNoRows {
_, err = o.Insert(&item)
} else {
_, err = o.Update(&item)
}
return err
}
func (s *sqlSearcher) Remove(o orm.Ormer, table string, ids []string) error {
var offset int
for {
var subset = paginateSlice(ids, offset, batchSize)
if len(subset) == 0 {
break
}
log.Trace("Deleting searchable items", "table", table, "num", len(subset), "from", offset)
offset += len(subset)
_, err := o.QueryTable(&Search{}).Filter("table", table).Filter("id__in", subset).Delete()
if err != nil {
return err
}
}
return nil
}
func (s *sqlSearcher) DeleteAll(o orm.Ormer, table string) error {
_, err := o.QueryTable(&Search{}).Filter("table", table).Delete()
return err
}
func (s *sqlSearcher) Search(table string, q string, offset, size int, results interface{}, orderBys ...string) error {
q = strings.TrimSpace(sanitize.Accents(strings.ToLower(strings.TrimSuffix(q, "*"))))
if len(q) <= 2 {
return nil
}
sq := squirrel.Select("*").From(table).OrderBy()
sq = sq.Limit(uint64(size)).Offset(uint64(offset))
if len(orderBys) > 0 {
sq = sq.OrderBy(orderBys...)
}
sq = sq.Join("search").Where("search.id = " + table + ".id")
parts := strings.Split(q, " ")
for _, part := range parts {
sq = sq.Where(squirrel.Or{
squirrel.Like{"full_text": part + "%"},
squirrel.Like{"full_text": "%" + part + "%"},
})
}
sql, args, err := sq.ToSql()
if err != nil {
return err
}
_, err = Db().Raw(sql, args...).QueryRows(results)
return err
}