Cleanup, add Placeholder agent

core/external_info.go

@@ -2,7 +2,6 @@ package core
 
 import (
 	"context"
-	"fmt"
 	"sort"
 	"strings"
 	"sync"
@@ -10,17 +9,14 @@ import (
 
 	"github.com/Masterminds/squirrel"
 	"github.com/microcosm-cc/bluemonday"
+	"github.com/navidrome/navidrome/conf"
 	"github.com/navidrome/navidrome/consts"
-	"github.com/navidrome/navidrome/core/lastfm"
-	"github.com/navidrome/navidrome/core/spotify"
+	"github.com/navidrome/navidrome/core/agents"
 	"github.com/navidrome/navidrome/log"
 	"github.com/navidrome/navidrome/model"
-	"github.com/xrash/smetrics"
 )
 
-const placeholderArtistImageSmallUrl = "https://lastfm.freetls.fastly.net/i/u/64s/2a96cbd8b46e442fc41c2b86b821562f.png"
-const placeholderArtistImageMediumUrl = "https://lastfm.freetls.fastly.net/i/u/174s/2a96cbd8b46e442fc41c2b86b821562f.png"
-const placeholderArtistImageLargeUrl = "https://lastfm.freetls.fastly.net/i/u/300x300/2a96cbd8b46e442fc41c2b86b821562f.png"
+const unavailableArtistID = "-1"
 
 type ExternalInfo interface {
 	UpdateArtistInfo(ctx context.Context, id string, count int, includeNotPresent bool) (*model.Artist, error)
@@ -28,50 +24,106 @@ type ExternalInfo interface {
 	TopSongs(ctx context.Context, artist string, count int) (model.MediaFiles, error)
 }
 
-func NewExternalInfo(ds model.DataStore, lfm *lastfm.Client, spf *spotify.Client) ExternalInfo {
-	return &externalInfo{ds: ds, lfm: lfm, spf: spf}
-}
-
 type externalInfo struct {
-	ds  model.DataStore
-	lfm *lastfm.Client
-	spf *spotify.Client
+	ds model.DataStore
 }
 
-const unavailableArtistID = "-1"
+type auxArtist struct {
+	model.Artist
+	Name string
+}
 
-func (e *externalInfo) UpdateArtistInfo(ctx context.Context, id string, count int, includeNotPresent bool) (*model.Artist, error) {
+func NewExternalInfo2(ds model.DataStore) ExternalInfo {
+	return &externalInfo{ds: ds}
+}
+
+func (e *externalInfo) initAgents(ctx context.Context) []agents.Interface {
+	order := strings.Split(conf.Server.Agents, ",")
+	order = append(order, agents.PlaceholderAgentName)
+	var res []agents.Interface
+	for _, name := range order {
+		init, ok := agents.Map[name]
+		if !ok {
+			log.Error(ctx, "Agent not available. Check configuration", "name", name)
+			continue
+		}
+
+		res = append(res, init(ctx))
+	}
+
+	return res
+}
+
+func (e *externalInfo) getArtist(ctx context.Context, id string) (*auxArtist, error) {
+	var entity interface{}
+	entity, err := GetEntityByID(ctx, e.ds, id)
+	if err != nil {
+		return nil, err
+	}
+
+	var artist auxArtist
+	switch v := entity.(type) {
+	case *model.Artist:
+		artist.Artist = *v
+		artist.Name = clearName(v.Name)
+	case *model.MediaFile:
+		return e.getArtist(ctx, v.ArtistID)
+	case *model.Album:
+		return e.getArtist(ctx, v.AlbumArtistID)
+	default:
+		return nil, model.ErrNotFound
+	}
+	return &artist, nil
+}
+
+// Replace some Unicode chars with their equivalent ASCII
+func clearName(name string) string {
+	name = strings.ReplaceAll(name, "–", "-")
+	name = strings.ReplaceAll(name, "‐", "-")
+	name = strings.ReplaceAll(name, "“", `"`)
+	name = strings.ReplaceAll(name, "”", `"`)
+	name = strings.ReplaceAll(name, "‘", `'`)
+	name = strings.ReplaceAll(name, "’", `'`)
+	return name
+}
+
+func (e *externalInfo) UpdateArtistInfo(ctx context.Context, id string, similarCount int, includeNotPresent bool) (*model.Artist, error) {
+	allAgents := e.initAgents(ctx)
 	artist, err := e.getArtist(ctx, id)
 	if err != nil {
 		return nil, err
 	}
 
-	// If we have updated info, just return it
+	// If we have fresh info, just return it
 	if time.Since(artist.ExternalInfoUpdatedAt) < consts.ArtistInfoTimeToLive {
 		log.Debug("Found cached ArtistInfo", "updatedAt", artist.ExternalInfoUpdatedAt, "name", artist.Name)
 		err := e.loadSimilar(ctx, artist, includeNotPresent)
-		return artist, err
+		return &artist.Artist, err
 	}
-	log.Debug("ArtistInfo not cached", "updatedAt", artist.ExternalInfoUpdatedAt, "id", id)
+	log.Debug(ctx, "ArtistInfo not cached or expired", "updatedAt", artist.ExternalInfoUpdatedAt, "id", id, "name", artist.Name)
 
-	// TODO Load from local: artist.jpg/png/webp, artist.json (with the remaining info)
+	// Get MBID first, if it is not yet available
+	if artist.MbzArtistID == "" {
+		e.callGetMBID(ctx, allAgents, artist)
+	}
 
-	var wg sync.WaitGroup
-	e.callArtistInfo(ctx, artist, &wg)
-	e.callArtistImages(ctx, artist, &wg)
-	e.callSimilarArtists(ctx, artist, count, &wg)
+	// Call all registered agents and collect information
+	wg := &sync.WaitGroup{}
+	e.callGetBiography(ctx, allAgents, artist, wg)
+	e.callGetURL(ctx, allAgents, artist, wg)
+	e.callGetImage(ctx, allAgents, artist, wg)
+	e.callGetSimilar(ctx, allAgents, artist, similarCount, wg)
 	wg.Wait()
 
-	// Use placeholders if could not get from external sources
-	e.setBio(artist, "Biography not available")
-	e.setSmallImageUrl(artist, placeholderArtistImageSmallUrl)
-	e.setMediumImageUrl(artist, placeholderArtistImageMediumUrl)
-	e.setLargeImageUrl(artist, placeholderArtistImageLargeUrl)
+	if isDone(ctx) {
+		log.Warn(ctx, "ArtistInfo update canceled", ctx.Err())
+		return nil, ctx.Err()
+	}
 
 	artist.ExternalInfoUpdatedAt = time.Now()
-	err = e.ds.Artist(ctx).Put(artist)
+	err = e.ds.Artist(ctx).Put(&artist.Artist)
 	if err != nil {
-		log.Error(ctx, "Error trying to update artistImageUrl", "id", id, err)
+		log.Error(ctx, "Error trying to update artist external information", "id", id, "name", artist.Name, err)
 	}
 
 	if !includeNotPresent {
@@ -86,47 +138,34 @@ func (e *externalInfo) UpdateArtistInfo(ctx context.Context, id string, count in
 	}
 
 	log.Trace(ctx, "ArtistInfo collected", "artist", artist)
-
-	return artist, nil
-}
-
-func (e *externalInfo) getArtist(ctx context.Context, id string) (*model.Artist, error) {
-	var entity interface{}
-	entity, err := GetEntityByID(ctx, e.ds, id)
-	if err != nil {
-		return nil, err
-	}
-
-	switch v := entity.(type) {
-	case *model.Artist:
-		return v, nil
-	case *model.MediaFile:
-		return e.ds.Artist(ctx).Get(v.ArtistID)
-	case *model.Album:
-		return e.ds.Artist(ctx).Get(v.AlbumArtistID)
-	}
-	return nil, model.ErrNotFound
+	return &artist.Artist, nil
 }
 
 func (e *externalInfo) SimilarSongs(ctx context.Context, id string, count int) (model.MediaFiles, error) {
-	if e.lfm == nil {
-		log.Warn(ctx, "Last.FM client not configured")
-		return nil, model.ErrNotAvailable
-	}
-
+	allAgents := e.initAgents(ctx)
 	artist, err := e.getArtist(ctx, id)
 	if err != nil {
 		return nil, err
 	}
 
-	artists, err := e.similarArtists(ctx, clearName(artist.Name), count, false)
-	if err != nil {
-		return nil, err
+	wg := &sync.WaitGroup{}
+	e.callGetSimilar(ctx, allAgents, artist, count, wg)
+	wg.Wait()
+
+	if isDone(ctx) {
+		log.Warn(ctx, "SimilarSongs call canceled", ctx.Err())
+		return nil, ctx.Err()
 	}
-	ids := make([]string, len(artists)+1)
-	ids[0] = artist.ID
-	for i, a := range artists {
-		ids[i+1] = a.ID
+
+	if len(artist.SimilarArtists) == 0 {
+		return nil, nil
+	}
+
+	var ids = []string{artist.ID}
+	for _, a := range artist.SimilarArtists {
+		if a.ID != unavailableArtistID {
+			ids = append(ids, a.ID)
+		}
 	}
 
 	return e.ds.MediaFile(ctx).GetAll(model.QueryOptions{
@@ -136,77 +175,28 @@ func (e *externalInfo) SimilarSongs(ctx context.Context, id string, count int) (
 	})
 }
 
-func (e *externalInfo) similarArtists(ctx context.Context, artistName string, count int, includeNotPresent bool) (model.Artists, error) {
-	var result model.Artists
-	var notPresent []string
-
-	log.Debug(ctx, "Calling Last.FM ArtistGetSimilar", "artist", artistName)
-	similar, err := e.lfm.ArtistGetSimilar(ctx, artistName, "", count)
-	if err != nil {
-		return nil, err
-	}
-
-	// First select artists that are present.
-	for _, s := range similar {
-		sa, err := e.findArtistByName(ctx, s.Name)
-		if err != nil {
-			notPresent = append(notPresent, s.Name)
-			continue
-		}
-		result = append(result, *sa)
-	}
-
-	// Then fill up with non-present artists
-	if includeNotPresent {
-		for _, s := range notPresent {
-			sa := model.Artist{ID: unavailableArtistID, Name: s}
-			result = append(result, sa)
-		}
-	}
-
-	return result, nil
-}
-
-func (e *externalInfo) findArtistByName(ctx context.Context, artistName string) (*model.Artist, error) {
-	artists, err := e.ds.Artist(ctx).GetAll(model.QueryOptions{
-		Filters: squirrel.Like{"name": artistName},
-		Max:     1,
-	})
-	if err != nil {
-		return nil, err
-	}
-	if len(artists) == 0 {
-		return nil, model.ErrNotFound
-	}
-	return &artists[0], nil
-}
-
 func (e *externalInfo) TopSongs(ctx context.Context, artistName string, count int) (model.MediaFiles, error) {
-	if e.lfm == nil {
-		log.Warn(ctx, "Last.FM client not configured")
-		return nil, model.ErrNotAvailable
-	}
+	allAgents := e.initAgents(ctx)
 	artist, err := e.findArtistByName(ctx, artistName)
 	if err != nil {
 		log.Error(ctx, "Artist not found", "name", artistName, err)
 		return nil, nil
 	}
-	artistName = clearName(artistName)
 
-	log.Debug(ctx, "Calling Last.FM ArtistGetTopTracks", "artist", artistName, "id", artist.ID)
-	tracks, err := e.lfm.ArtistGetTopTracks(ctx, artistName, artist.MbzArtistID, count)
+	songs, err := e.callGetTopSongs(ctx, allAgents, artist, count)
 	if err != nil {
 		return nil, err
 	}
-	var songs model.MediaFiles
-	for _, t := range tracks {
+
+	var mfs model.MediaFiles
+	for _, t := range songs {
 		mf, err := e.findMatchingTrack(ctx, t.MBID, artist.ID, t.Name)
 		if err != nil {
 			continue
 		}
-		songs = append(songs, *mf)
+		mfs = append(mfs, *mf)
 	}
-	return songs, nil
+	return mfs, nil
 }
 
 func (e *externalInfo) findMatchingTrack(ctx context.Context, mbid string, artistID, title string) (*model.MediaFile, error) {
@@ -234,141 +224,208 @@ func (e *externalInfo) findMatchingTrack(ctx context.Context, mbid string, artis
 	return &mfs[0], nil
 }
 
-func (e *externalInfo) callArtistInfo(ctx context.Context, artist *model.Artist, wg *sync.WaitGroup) {
-	if e.lfm != nil {
-		name := clearName(artist.Name)
-		log.Debug(ctx, "Calling Last.FM ArtistGetInfo", "artist", name)
-		wg.Add(1)
-		go func() {
-			start := time.Now()
-			defer wg.Done()
-			lfmArtist, err := e.lfm.ArtistGetInfo(ctx, name, artist.MbzArtistID)
-			if err != nil {
-				log.Error(ctx, "Error calling Last.FM", "artist", name, err)
-			} else {
-				log.Debug(ctx, "Got info from Last.FM", "artist", name, "info", lfmArtist.Bio.Summary, "elapsed", time.Since(start))
-			}
-			e.setBio(artist, lfmArtist.Bio.Summary)
-			e.setExternalUrl(artist, lfmArtist.URL)
-			e.setMbzID(artist, lfmArtist.MBID)
-		}()
+func isDone(ctx context.Context) bool {
+	select {
+	case <-ctx.Done():
+		return true
+	default:
+		return false
 	}
 }
 
-func (e *externalInfo) searchArtist(ctx context.Context, name string) (*spotify.Artist, error) {
-	artists, err := e.spf.SearchArtists(ctx, name, 40)
-	if err != nil || len(artists) == 0 {
-		return nil, model.ErrNotFound
+func (e *externalInfo) callGetMBID(ctx context.Context, allAgents []agents.Interface, artist *auxArtist) {
+	start := time.Now()
+	for _, a := range allAgents {
+		if isDone(ctx) {
+			break
+		}
+		agent, ok := a.(agents.ArtistMBIDRetriever)
+		if !ok {
+			continue
+		}
+		mbid, err := agent.GetMBID(artist.Name)
+		if mbid != "" && err == nil {
+			artist.MbzArtistID = mbid
+			log.Debug(ctx, "Got MBID", "agent", a.AgentName(), "artist", artist.Name, "mbid", mbid, "elapsed", time.Since(start))
+			break
+		}
 	}
-	name = strings.ToLower(name)
+}
 
-	// Sort results, prioritizing artists with images, with similar names and with high popularity, in this order
-	sort.Slice(artists, func(i, j int) bool {
-		ai := fmt.Sprintf("%-5t-%03d-%04d", len(artists[i].Images) == 0, smetrics.WagnerFischer(name, strings.ToLower(artists[i].Name), 1, 1, 2), 1000-artists[i].Popularity)
-		aj := fmt.Sprintf("%-5t-%03d-%04d", len(artists[j].Images) == 0, smetrics.WagnerFischer(name, strings.ToLower(artists[j].Name), 1, 1, 2), 1000-artists[j].Popularity)
-		return strings.Compare(ai, aj) < 0
+func (e *externalInfo) callGetTopSongs(ctx context.Context, allAgents []agents.Interface, artist *auxArtist,
+	count int) ([]agents.Song, error) {
+	start := time.Now()
+	for _, a := range allAgents {
+		if isDone(ctx) {
+			break
+		}
+		agent, ok := a.(agents.ArtistTopSongsRetriever)
+		if !ok {
+			continue
+		}
+		songs, err := agent.GetTopSongs(artist.Name, artist.MbzArtistID, count)
+		if len(songs) > 0 && err == nil {
+			log.Debug(ctx, "Got Top Songs", "agent", a.AgentName(), "artist", artist.Name, "songs", songs, "elapsed", time.Since(start))
+			return songs, err
+		}
+	}
+	return nil, nil
+}
+
+func (e *externalInfo) callGetURL(ctx context.Context, allAgents []agents.Interface, artist *auxArtist, wg *sync.WaitGroup) {
+	wg.Add(1)
+	go func() {
+		defer wg.Done()
+		start := time.Now()
+		for _, a := range allAgents {
+			if isDone(ctx) {
+				break
+			}
+			agent, ok := a.(agents.ArtistURLRetriever)
+			if !ok {
+				continue
+			}
+			url, err := agent.GetURL(artist.Name, artist.MbzArtistID)
+			if url != "" && err == nil {
+				artist.ExternalUrl = url
+				log.Debug(ctx, "Got External Url", "agent", a.AgentName(), "artist", artist.Name, "url", url, "elapsed", time.Since(start))
+				break
+			}
+		}
+	}()
+}
+
+func (e *externalInfo) callGetBiography(ctx context.Context, allAgents []agents.Interface, artist *auxArtist, wg *sync.WaitGroup) {
+	wg.Add(1)
+	go func() {
+		defer wg.Done()
+		start := time.Now()
+		for _, a := range allAgents {
+			if isDone(ctx) {
+				break
+			}
+			agent, ok := a.(agents.ArtistBiographyRetriever)
+			if !ok {
+				continue
+			}
+			bio, err := agent.GetBiography(clearName(artist.Name), artist.MbzArtistID)
+			if bio != "" && err == nil {
+				policy := bluemonday.UGCPolicy()
+				bio = policy.Sanitize(bio)
+				bio = strings.ReplaceAll(bio, "\n", " ")
+				artist.Biography = strings.ReplaceAll(bio, "<a ", "<a target='_blank' ")
+				log.Debug(ctx, "Got Biography", "agent", a.AgentName(), "artist", artist.Name, "len", len(bio), "elapsed", time.Since(start))
+				break
+			}
+		}
+	}()
+}
+
+func (e *externalInfo) callGetImage(ctx context.Context, allAgents []agents.Interface, artist *auxArtist, wg *sync.WaitGroup) {
+	wg.Add(1)
+	go func() {
+		defer wg.Done()
+		start := time.Now()
+		for _, a := range allAgents {
+			if isDone(ctx) {
+				break
+			}
+			agent, ok := a.(agents.ArtistImageRetriever)
+			if !ok {
+				continue
+			}
+			images, err := agent.GetImages(artist.Name, artist.MbzArtistID)
+			if len(images) == 0 || err != nil {
+				continue
+			}
+			log.Debug(ctx, "Got Images", "agent", a.AgentName(), "artist", artist.Name, "images", images, "elapsed", time.Since(start))
+			sort.Slice(images, func(i, j int) bool { return images[i].Size > images[j].Size })
+			if len(images) >= 1 {
+				artist.LargeImageUrl = images[0].URL
+			}
+			if len(images) >= 2 {
+				artist.MediumImageUrl = images[1].URL
+			}
+			if len(images) >= 3 {
+				artist.SmallImageUrl = images[2].URL
+			}
+			break
+		}
+	}()
+}
+
+func (e *externalInfo) callGetSimilar(ctx context.Context, allAgents []agents.Interface, artist *auxArtist, limit int, wg *sync.WaitGroup) {
+	wg.Add(1)
+	go func() {
+		defer wg.Done()
+		start := time.Now()
+		for _, a := range allAgents {
+			if isDone(ctx) {
+				break
+			}
+			agent, ok := a.(agents.ArtistSimilarRetriever)
+			if !ok {
+				continue
+			}
+			similar, err := agent.GetSimilar(artist.Name, artist.MbzArtistID, limit)
+			if len(similar) == 0 || err != nil {
+				continue
+			}
+			sa, err := e.mapSimilarArtists(ctx, similar, true)
+			if err != nil {
+				continue
+			}
+			log.Debug(ctx, "Got Similar Artists", "agent", a.AgentName(), "artist", artist.Name, "similar", similar, "elapsed", time.Since(start))
+			artist.SimilarArtists = sa
+			break
+		}
+	}()
+}
+
+func (e *externalInfo) mapSimilarArtists(ctx context.Context, similar []agents.Artist, includeNotPresent bool) (model.Artists, error) {
+	var result model.Artists
+	var notPresent []string
+
+	// First select artists that are present.
+	for _, s := range similar {
+		sa, err := e.findArtistByName(ctx, s.Name)
+		if err != nil {
+			notPresent = append(notPresent, s.Name)
+			continue
+		}
+		result = append(result, sa.Artist)
+	}
+
+	// Then fill up with non-present artists
+	if includeNotPresent {
+		for _, s := range notPresent {
+			sa := model.Artist{ID: unavailableArtistID, Name: s}
+			result = append(result, sa)
+		}
+	}
+
+	return result, nil
+}
+
+func (e *externalInfo) findArtistByName(ctx context.Context, artistName string) (*auxArtist, error) {
+	artists, err := e.ds.Artist(ctx).GetAll(model.QueryOptions{
+		Filters: squirrel.Like{"name": artistName},
+		Max:     1,
 	})
-
-	// If the first one has the same name, that's the one
-	if strings.ToLower(artists[0].Name) != name {
+	if err != nil {
+		return nil, err
+	}
+	if len(artists) == 0 {
 		return nil, model.ErrNotFound
 	}
-	return &artists[0], err
-}
-
-func (e *externalInfo) callSimilarArtists(ctx context.Context, artist *model.Artist, count int, wg *sync.WaitGroup) {
-	if e.lfm != nil {
-		name := clearName(artist.Name)
-		wg.Add(1)
-		go func() {
-			start := time.Now()
-			defer wg.Done()
-			similar, err := e.similarArtists(ctx, name, count, true)
-			if err != nil {
-				log.Error(ctx, "Error calling Last.FM", "artist", name, err)
-				return
-			}
-			log.Debug(ctx, "Got similar artists from Last.FM", "artist", name, "info", "elapsed", time.Since(start))
-			artist.SimilarArtists = similar
-		}()
+	artist := &auxArtist{
+		Artist: artists[0],
+		Name:   clearName(artists[0].Name),
 	}
+	return artist, nil
 }
 
-func (e *externalInfo) callArtistImages(ctx context.Context, artist *model.Artist, wg *sync.WaitGroup) {
-	if e.spf != nil {
-		name := clearName(artist.Name)
-		log.Debug(ctx, "Calling Spotify SearchArtist", "artist", name)
-		wg.Add(1)
-		go func() {
-			start := time.Now()
-			defer wg.Done()
-
-			a, err := e.searchArtist(ctx, name)
-			if err != nil {
-				if err == model.ErrNotFound {
-					log.Warn(ctx, "Artist not found in Spotify", "artist", name)
-				} else {
-					log.Error(ctx, "Error calling Spotify", "artist", name, err)
-				}
-				return
-			}
-			spfImages := a.Images
-			log.Debug(ctx, "Got images from Spotify", "artist", name, "images", spfImages, "elapsed", time.Since(start))
-
-			sort.Slice(spfImages, func(i, j int) bool { return spfImages[i].Width > spfImages[j].Width })
-			if len(spfImages) >= 1 {
-				e.setLargeImageUrl(artist, spfImages[0].URL)
-			}
-			if len(spfImages) >= 2 {
-				e.setMediumImageUrl(artist, spfImages[1].URL)
-			}
-			if len(spfImages) >= 3 {
-				e.setSmallImageUrl(artist, spfImages[2].URL)
-			}
-		}()
-	}
-}
-
-func (e *externalInfo) setBio(artist *model.Artist, bio string) {
-	policy := bluemonday.UGCPolicy()
-	if artist.Biography == "" {
-		bio = policy.Sanitize(bio)
-		bio = strings.ReplaceAll(bio, "\n", " ")
-		artist.Biography = strings.ReplaceAll(bio, "<a ", "<a target='_blank' ")
-	}
-}
-
-func (e *externalInfo) setExternalUrl(artist *model.Artist, url string) {
-	if artist.ExternalUrl == "" {
-		artist.ExternalUrl = url
-	}
-}
-
-func (e *externalInfo) setMbzID(artist *model.Artist, mbID string) {
-	if artist.MbzArtistID == "" {
-		artist.MbzArtistID = mbID
-	}
-}
-
-func (e *externalInfo) setSmallImageUrl(artist *model.Artist, url string) {
-	if artist.SmallImageUrl == "" {
-		artist.SmallImageUrl = url
-	}
-}
-
-func (e *externalInfo) setMediumImageUrl(artist *model.Artist, url string) {
-	if artist.MediumImageUrl == "" {
-		artist.MediumImageUrl = url
-	}
-}
-
-func (e *externalInfo) setLargeImageUrl(artist *model.Artist, url string) {
-	if artist.LargeImageUrl == "" {
-		artist.LargeImageUrl = url
-	}
-}
-
-func (e *externalInfo) loadSimilar(ctx context.Context, artist *model.Artist, includeNotPresent bool) error {
+func (e *externalInfo) loadSimilar(ctx context.Context, artist *auxArtist, includeNotPresent bool) error {
 	var ids []string
 	for _, sa := range artist.SimilarArtists {
 		if sa.ID == unavailableArtistID {