Big LedisDB refactoring

conf/app.conf

@@ -8,7 +8,7 @@ apiVersion = 1.0.0
 ignoredArticles="The El La Los Las Le Les Os As O A"
 indexGroups=A B C D E F G H I J K L M N O P Q R S T U V W X-Z(XYZ)
 
-musicFolder=./iTunesFull.xml
+musicFolder=./iTunes.xml
 user=deluan
 password=wordpass
 dbPath = ./devDb

models/album.go

@@ -3,32 +3,9 @@ package models
 type Album struct {
 	Id           string
 	Name         string
-	ArtistId     string
+	ArtistId     string `parent:"artist"`
 	CoverArtPath string
 	Year         int
 	Compilation  bool
 	Rating       int
-	MediaFiles   map[string]bool
-}
-
-func (a *Album) AdMediaFiles(mfs ...*MediaFile) {
-	for _, mf := range mfs {
-		a.MediaFiles[mf.Id] = true
-	}
-}
-
-func (a *Album) AddMediaFiles(mfs  ...interface{}) {
-	if a.MediaFiles == nil {
-		a.MediaFiles = make(map[string]bool)
-	}
-	for _, v := range mfs {
-		switch v := v.(type) {
-		case *MediaFile:
-			a.MediaFiles[v.Id] = true
-		case map[string]bool:
-			for k, _ := range v {
-				a.MediaFiles[k] = true
-			}
-		}
-	}
 }

models/artist.go

@@ -20,20 +20,4 @@ func NoArticle(name string) string {
 		}
 	}
 	return name
-}
-
-func (a *Artist) AddAlbums(albums ...interface{}) {
-	if a.Albums == nil {
-		a.Albums = make(map[string]bool)
-	}
-	for _, v := range albums {
-		switch v := v.(type) {
-		case *Album:
-			a.Albums[v.Id] = true
-		case map[string]bool:
-			for k, _ := range v {
-				a.Albums[k] = true
-			}
-		}
-	}
 }

models/media_file.go

@@ -11,6 +11,7 @@ type MediaFile struct {
 	Album       string
 	Artist      string
 	AlbumArtist string
+	AlbumId	    string `parent:"album"`
 	Compilation bool
 	CreatedAt   time.Time
 	UpdatedAt   time.Time

repositories/album_repository.go

@@ -10,20 +10,20 @@ type Album struct {
 
 func NewAlbumRepository() *Album {
 	r := &Album{}
-	r.key = "album"
+	r.table = "album"
 	return r
 }
 
-func (r *Album) Put(m *models.Album) (*models.Album, error) {
+func (r *Album) Put(m *models.Album) error {
 	if m.Id == "" {
 		m.Id = r.NewId(m.Name)
 	}
-	return m, r.saveOrUpdate(m.Id, m)
+	return r.saveOrUpdate(m.Id, m)
 }
 
 func (r *Album) Get(id string) (*models.Album, error) {
 	rec := &models.Album{}
-	err := readStruct(r.key, id, rec)
+	err := r.loadEntity(id, rec)
 	return rec, err
 }
 

repositories/artist_repository.go

@@ -10,20 +10,20 @@ type Artist struct {
 
 func NewArtistRepository() *Artist {
 	r := &Artist{}
-	r.key = "artist"
+	r.table = "artist"
 	return r
 }
 
-func (r *Artist) Put(m *models.Artist) (*models.Artist, error) { // TODO Return only error
+func (r *Artist) Put(m *models.Artist) error {
 	if m.Id == "" {
 		m.Id = r.NewId(m.Name)
 	}
-	return m, r.saveOrUpdate(m.Id, m)
+	return r.saveOrUpdate(m.Id, m)
 }
 
 func (r *Artist) Get(id string) (*models.Artist, error) {
 	rec := &models.Artist{}
-	err := readStruct(r.key, id, rec)
+	err := r.loadEntity(id, rec)
 	return rec, err
 }
 

repositories/base_repository.go

@@ -4,26 +4,100 @@ import (
 	"fmt"
 	"crypto/md5"
 	"strings"
+	"github.com/deluan/gosonic/utils"
+	"encoding/json"
+	"reflect"
 )
 
+
 type BaseRepository struct {
-	key string // TODO Rename to 'table'
+	table string
 }
 
 func (r *BaseRepository) NewId(fields ...string) string {
-	s := fmt.Sprintf("%s\\%s", strings.ToUpper(r.key), strings.Join(fields, ""))
+	s := fmt.Sprintf("%s\\%s", strings.ToUpper(r.table), strings.Join(fields, ""))
 	return fmt.Sprintf("%x", md5.Sum([]byte(s)))
 }
 
 func (r *BaseRepository) CountAll() (int, error) {
-	return count(r.key)
+	ids, err := db().SMembers([]byte(r.table + "s:all"))
+	return len(ids), err
 }
 
 func (r *BaseRepository) saveOrUpdate(id string, rec interface{}) error {
-	return saveStruct(r.key, id, rec)
+	return r.saveEntity(id, rec)
 }
 
 func (r *BaseRepository) Dump() {
 }
 
+func (r *BaseRepository) saveEntity(id string, entity interface{}) error {
+	recordPrefix := fmt.Sprintf("%s:%s:", r.table, id)
+	allKey := r.table + "s:all"
 
+	h, err := utils.ToMap(entity)
+	if err != nil {
+		return err
+	}
+	for f, v := range h {
+		key := recordPrefix + f
+		value, _ := json.Marshal(v)
+		if err := db().Set([]byte(key), value); err != nil {
+			return err
+		}
+
+	}
+
+	if _, err = db().SAdd([]byte(allKey), []byte(id)); err != nil {
+		return err
+	}
+
+	if parentTable, parentId := r.getParent(entity); parentTable != "" {
+		parentCollectionKey := fmt.Sprintf("%s:%s:%ss", parentTable, parentId, r.table)
+		_, err = db().SAdd([]byte(parentCollectionKey), []byte(id))
+	}
+	return nil
+}
+
+// TODO Optimize
+func (r *BaseRepository) getParent(entity interface{}) (table string, id string) {
+	dt := reflect.TypeOf(entity).Elem()
+	for i := 0; i < dt.NumField(); i++ {
+		f := dt.Field(i)
+		table := f.Tag.Get("parent")
+		if table != "" {
+			dv := reflect.ValueOf(entity).Elem()
+			return table, dv.FieldByName(f.Name).String()
+		}
+	}
+	return "", ""
+}
+
+func (r *BaseRepository) loadEntity(id string, entity interface{}) error {
+	recordPrefix := fmt.Sprintf("%s:%s:", r.table, id)
+
+	h, _ := utils.ToMap(entity)
+	var fieldKeys = make([][]byte, len(h))
+	var fieldNames = make([]string, len(h))
+	i := 0
+	for k, _ := range h {
+		fieldNames[i] = k
+		fieldKeys[i] = []byte(recordPrefix + k)
+		i++
+	}
+
+	res, err := db().MGet(fieldKeys...)
+	if err != nil {
+		return err
+	}
+	var record = make(map[string]interface{}, len(res))
+	for i, v := range res {
+		var value interface{}
+		if err := json.Unmarshal(v, &value); err != nil {
+			return err
+		}
+		record[string(fieldNames[i])] = value
+	}
+
+	return utils.ToStruct(record, entity)
+}

repositories/index_repository.go

@@ -11,7 +11,7 @@ type ArtistIndex struct {
 
 func NewArtistIndexRepository() *ArtistIndex {
 	r := &ArtistIndex{}
-	r.key = "index"
+	r.table = "index"
 	return r
 }
 

repositories/ledis_utils.go

@@ -2,11 +2,9 @@ package repositories
 
 import (
 	"sync"
-	"encoding/json"
 	"github.com/astaxie/beego"
 	"github.com/siddontang/ledisdb/ledis"
 	"github.com/siddontang/ledisdb/config"
-	"github.com/deluan/gosonic/utils"
 )
 
 var (
@@ -26,40 +24,4 @@ func db() *ledis.DB {
 		_dbInstance = instance
 	})
 	return _dbInstance
-}
-
-func saveStruct(key, id string, data interface{}) error {
-	h, err := utils.ToMap(data)
-	if err != nil {
-		return err
-	}
-	var fvList = make([]ledis.FVPair, len(h))
-	i := 0
-	for f, v := range h {
-		fvList[i].Field = []byte(f)
-		fvList[i].Value, _ = json.Marshal(v)
-		i++
-	}
-	kh := key + "_id_" + id
-	ks := key + "_ids"
-	db().SAdd([]byte(ks), []byte(id))
-	return db().HMset([]byte(kh), fvList...)
-}
-
-func readStruct(key, id string, rec interface{}) error {
-	kh := key + "_id_" + id
-	fvs, _ := db().HGetAll([]byte(kh))
-	var m = make(map[string]interface{}, len(fvs))
-	for _, fv := range fvs {
-		var v interface{}
-		json.Unmarshal(fv.Value, &v)
-		m[string(fv.Field)] = v
-	}
-
-	return utils.ToStruct(m, rec)
-}
-
-func count(key string) (int, error) {
-	ids, err := db().SMembers([]byte(key + "_ids"))
-	return len(ids), err
 }

repositories/media_file_repository.go

@@ -10,7 +10,7 @@ type MediaFile struct {
 
 func NewMediaFileRepository() *MediaFile {
 	r := &MediaFile{}
-	r.key = "mediafile"
+	r.table = "mediafile"
 	return r
 }
 

scanner/scanner.go

@@ -5,8 +5,6 @@ import (
 	"github.com/deluan/gosonic/repositories"
 	"github.com/deluan/gosonic/models"
 	"strings"
-	"fmt"
-	"encoding/json"
 )
 
 type Scanner interface {
@@ -33,10 +31,8 @@ func importLibrary(files []Track) {
 	var artistIndex = make(map[string]tempIndex)
 
 	for _, t := range files {
-		mf, album, artist := processTrack(&t)
+		mf, album, artist := parseTrack(&t)
-		mergeInfo(mfRepo, mf, albumRepo, album, artistRepo, artist)
+		persist(mfRepo, mf, albumRepo, album, artistRepo, artist)
-		fmt.Printf("%#v\n", album)
-		fmt.Printf("%#v\n\n", artist)
 		collectIndex(artist, artistIndex)
 	}
 
@@ -44,14 +40,15 @@ func importLibrary(files []Track) {
 		beego.Error(err)
 	}
 
-	j,_ := json.MarshalIndent(artistIndex, "", "    ")
+	c, _ := artistRepo.CountAll()
-	fmt.Println(string(j))
+	beego.Info("Total Artists in database:", c)
-
+	c, _ = albumRepo.CountAll()
-	c, _ := mfRepo.CountAll()
+	beego.Info("Total Albums in database:", c)
-	beego.Info("Total mediafiles in database:", c)
+	c, _ = mfRepo.CountAll()
+	beego.Info("Total MediaFiles in database:", c)
 }
 
-func processTrack(t *Track) (*models.MediaFile, *models.Album, *models.Artist) {
+func parseTrack(t *Track) (*models.MediaFile, *models.Album, *models.Artist) {
 	mf := &models.MediaFile{
 		Id: t.Id,
 		Album: t.Album,
@@ -77,30 +74,17 @@ func processTrack(t *Track) (*models.MediaFile, *models.Album, *models.Artist) {
 	return mf, album, artist
 }
 
-func mergeInfo(mfRepo *repositories.MediaFile, mf *models.MediaFile, albumRepo *repositories.Album, album *models.Album, artistRepo *repositories.Artist, artist *models.Artist) {
+func persist(mfRepo *repositories.MediaFile, mf *models.MediaFile, albumRepo *repositories.Album, album *models.Album, artistRepo *repositories.Artist, artist *models.Artist) {
-	artist.Id = artistRepo.NewId(artist.Name)
+	if err := artistRepo.Put(artist); err != nil {
-
-	sAlbum, err := albumRepo.GetByName(album.Name)
-	if err != nil {
 		beego.Error(err)
 	}
+
 	album.ArtistId = artist.Id
-	album.AddMediaFiles(mf, sAlbum.MediaFiles)
+	if err := albumRepo.Put(album); err != nil {
-	sAlbum, err = albumRepo.Put(album)
-	if err != nil {
-		beego.Error(err)
-	}
-
-	sArtist, err := artistRepo.GetByName(artist.Name)
-	if err != nil {
-		beego.Error(err)
-	}
-	artist.AddAlbums(sAlbum, sArtist.Albums)
-	_, err = artistRepo.Put(artist)
-	if err != nil {
 		beego.Error(err)
 	}
 
+	mf.AlbumId = album.Id
 	if err := mfRepo.Put(mf); err != nil {
 		beego.Error(err)
 	}