feat(bfr): Big Refactor: new scanner, lots of new fields and tags, improvements and DB schema changes (#2709)
* fix(server): more race conditions when updating artist/album from external sources Signed-off-by: Deluan <deluan@navidrome.org> * feat(scanner): add .gitignore syntax to .ndignore. Resolves #1394 Signed-off-by: Deluan <deluan@navidrome.org> * fix(ui): null Signed-off-by: Deluan <deluan@navidrome.org> * fix(scanner): pass configfile option to child process Signed-off-by: Deluan <deluan@navidrome.org> * fix(scanner): resume interrupted fullScans Signed-off-by: Deluan <deluan@navidrome.org> * fix(scanner): remove old scanner code Signed-off-by: Deluan <deluan@navidrome.org> * fix(scanner): rename old metadata package Signed-off-by: Deluan <deluan@navidrome.org> * fix(scanner): move old metadata package Signed-off-by: Deluan <deluan@navidrome.org> * fix: tests Signed-off-by: Deluan <deluan@navidrome.org> * chore(deps): update Go to 1.23.4 Signed-off-by: Deluan <deluan@navidrome.org> * fix: logs Signed-off-by: Deluan <deluan@navidrome.org> * fix(test): Signed-off-by: Deluan <deluan@navidrome.org> * fix: log level Signed-off-by: Deluan <deluan@navidrome.org> * fix: remove log message Signed-off-by: Deluan <deluan@navidrome.org> * feat: add config for scanner watcher Signed-off-by: Deluan <deluan@navidrome.org> * refactor: children playlists Signed-off-by: Deluan <deluan@navidrome.org> * refactor: replace `interface{}` with `any` Signed-off-by: Deluan <deluan@navidrome.org> * fix: smart playlists with genres Signed-off-by: Deluan <deluan@navidrome.org> * fix: allow any tags in smart playlists Signed-off-by: Deluan <deluan@navidrome.org> * fix: artist names in playlists Signed-off-by: Deluan <deluan@navidrome.org> * fix: smart playlist's sort by tags Signed-off-by: Deluan <deluan@navidrome.org> * feat(subsonic): add moods to child Signed-off-by: Deluan <deluan@navidrome.org> * feat(subsonic): add moods to AlbumID3 Signed-off-by: Deluan <deluan@navidrome.org> * refactor(subsonic): use generic JSONArray for OS arrays Signed-off-by: Deluan <deluan@navidrome.org> * refactor(subsonic): use https in test Signed-off-by: Deluan <deluan@navidrome.org> * feat(subsonic): add releaseTypes to AlbumID3 Signed-off-by: Deluan <deluan@navidrome.org> * feat(subsonic): add recordLabels to AlbumID3 Signed-off-by: Deluan <deluan@navidrome.org> * refactor(subsonic): rename JSONArray to Array Signed-off-by: Deluan <deluan@navidrome.org> * feat(subsonic): add artists to AlbumID3 Signed-off-by: Deluan <deluan@navidrome.org> * feat(subsonic): add artists to Child Signed-off-by: Deluan <deluan@navidrome.org> * fix(scanner): do not pre-populate smart playlists Signed-off-by: Deluan <deluan@navidrome.org> * feat(subsonic): implement a simplified version of ArtistID3. See https://github.com/opensubsonic/open-subsonic-api/discussions/120 Signed-off-by: Deluan <deluan@navidrome.org> * feat(subsonic): add artists to album child Signed-off-by: Deluan <deluan@navidrome.org> * feat(subsonic): add contributors to mediafile Child Signed-off-by: Deluan <deluan@navidrome.org> * feat(subsonic): add albumArtists to mediafile Child Signed-off-by: Deluan <deluan@navidrome.org> * feat(subsonic): add displayArtist and displayAlbumArtist Signed-off-by: Deluan <deluan@navidrome.org> * feat(subsonic): add displayComposer to Child Signed-off-by: Deluan <deluan@navidrome.org> * feat(subsonic): add roles to ArtistID3 Signed-off-by: Deluan <deluan@navidrome.org> * fix(subsonic): use " • " separator for displayComposer Signed-off-by: Deluan <deluan@navidrome.org> * refactor: Signed-off-by: Deluan <deluan@navidrome.org> * fix(subsonic): Signed-off-by: Deluan <deluan@navidrome.org> * fix(subsonic): respect `PreferSortTags` config option Signed-off-by: Deluan <deluan@navidrome.org> * refactor(subsonic): Signed-off-by: Deluan <deluan@navidrome.org> * refactor: optimize purging non-unused tags Signed-off-by: Deluan <deluan@navidrome.org> * refactor: don't run 'refresh artist stats' concurrently with other transactions Signed-off-by: Deluan <deluan@navidrome.org> * refactor: Signed-off-by: Deluan <deluan@navidrome.org> * fix: log message Signed-off-by: Deluan <deluan@navidrome.org> * feat: add Scanner.ScanOnStartup config option, default true Signed-off-by: Deluan <deluan@navidrome.org> * feat: better json parsing error msg when importing NSPs Signed-off-by: Deluan <deluan@navidrome.org> * fix: don't update album's imported_time when updating external_metadata Signed-off-by: Deluan <deluan@navidrome.org> * fix: handle interrupted scans and full scans after migrations Signed-off-by: Deluan <deluan@navidrome.org> * feat: run `analyze` when migration requires a full rescan Signed-off-by: Deluan <deluan@navidrome.org> * feat: run `PRAGMA optimize` at the end of the scan Signed-off-by: Deluan <deluan@navidrome.org> * fix: don't update artist's updated_at when updating external_metadata Signed-off-by: Deluan <deluan@navidrome.org> * feat: handle multiple artists and roles in smart playlists Signed-off-by: Deluan <deluan@navidrome.org> * feat(ui): dim missing tracks Signed-off-by: Deluan <deluan@navidrome.org> * fix: album missing logic Signed-off-by: Deluan <deluan@navidrome.org> * fix: error encoding in gob Signed-off-by: Deluan <deluan@navidrome.org> * feat: separate warnings from errors Signed-off-by: Deluan <deluan@navidrome.org> * fix: mark albums as missing if they were contained in a deleted folder Signed-off-by: Deluan <deluan@navidrome.org> * refactor: add participant names to media_file and album tables Signed-off-by: Deluan <deluan@navidrome.org> * refactor: use participations in criteria, instead of m2m relationship Signed-off-by: Deluan <deluan@navidrome.org> * refactor: rename participations to participants Signed-off-by: Deluan <deluan@navidrome.org> * feat(subsonic): add moods to album child Signed-off-by: Deluan <deluan@navidrome.org> * fix: albumartist role case Signed-off-by: Deluan <deluan@navidrome.org> * feat(scanner): run scanner as an external process by default Signed-off-by: Deluan <deluan@navidrome.org> * fix(ui): show albumArtist names Signed-off-by: Deluan <deluan@navidrome.org> * fix(ui): dim out missing albums Signed-off-by: Deluan <deluan@navidrome.org> * fix: flaky test Signed-off-by: Deluan <deluan@navidrome.org> * fix(server): scrobble buffer mapping. fix #3583 Signed-off-by: Deluan <deluan@navidrome.org> * refactor: more participations renaming Signed-off-by: Deluan <deluan@navidrome.org> * fix: listenbrainz scrobbling Signed-off-by: Deluan <deluan@navidrome.org> * feat: send release_group_mbid to listenbrainz Signed-off-by: Deluan <deluan@navidrome.org> * feat(subsonic): implement OpenSubsonic explicitStatus field (#3597) * feat: implement OpenSubsonic explicitStatus field * fix(subsonic): fix failing snapshot tests * refactor: create helper for setting explicitStatus * fix: store smaller values for explicit-status on database * test: ToAlbum explicitStatus * refactor: rename explicitStatus helper function --------- Co-authored-by: Deluan Quintão <deluan@navidrome.org> * fix: handle album and track tags in the DB based on the mappings.yaml file Signed-off-by: Deluan <deluan@navidrome.org> * save similar artists as JSONB Signed-off-by: Deluan <deluan@navidrome.org> * fix: getAlbumList byGenre Signed-off-by: Deluan <deluan@navidrome.org> * detect changes in PID configuration Signed-off-by: Deluan <deluan@navidrome.org> * set default album PID to legacy_pid Signed-off-by: Deluan <deluan@navidrome.org> * fix tests Signed-off-by: Deluan <deluan@navidrome.org> * fix SIGSEGV Signed-off-by: Deluan <deluan@navidrome.org> * fix: don't lose album stars/ratings when migrating Signed-off-by: Deluan <deluan@navidrome.org> * store full PID conf in properties Signed-off-by: Deluan <deluan@navidrome.org> * fix: keep album annotations when changing PID.Album config Signed-off-by: Deluan <deluan@navidrome.org> * fix: reassign album annotations Signed-off-by: Deluan <deluan@navidrome.org> * feat: use (display) albumArtist and add links to each artist Signed-off-by: Deluan <deluan@navidrome.org> * fix: not showing albums by albumartist Signed-off-by: Deluan <deluan@navidrome.org> * fix: error msgs Signed-off-by: Deluan <deluan@navidrome.org> * fix: hide PID from Native API Signed-off-by: Deluan <deluan@navidrome.org> * fix: album cover art resolution Signed-off-by: Deluan <deluan@navidrome.org> * fix: trim participant names Signed-off-by: Deluan <deluan@navidrome.org> * fix: reduce watcher log spam Signed-off-by: Deluan <deluan@navidrome.org> * fix: panic when initializing the watcher Signed-off-by: Deluan <deluan@navidrome.org> * fix: various artists Signed-off-by: Deluan <deluan@navidrome.org> * fix: don't store empty lyrics in the DB Signed-off-by: Deluan <deluan@navidrome.org> * remove unused methods Signed-off-by: Deluan <deluan@navidrome.org> * drop full_text indexes, as they are not being used by SQLite Signed-off-by: Deluan <deluan@navidrome.org> * keep album created_at when upgrading Signed-off-by: Deluan <deluan@navidrome.org> * fix(ui): null pointer Signed-off-by: Deluan <deluan@navidrome.org> * fix: album artwork cache Signed-off-by: Deluan <deluan@navidrome.org> * fix: don't expose missing files in Subsonic API Signed-off-by: Deluan <deluan@navidrome.org> * refactor: searchable interface Signed-off-by: Deluan <deluan@navidrome.org> * fix: filter out missing items from subsonic search * fix: filter out missing items from playlists * fix: filter out missing items from shares Signed-off-by: Deluan <deluan@navidrome.org> * feat(ui): add filter by artist role Signed-off-by: Deluan <deluan@navidrome.org> * feat(subsonic): only return albumartists in getIndexes and getArtists endpoints Signed-off-by: Deluan <deluan@navidrome.org> * sort roles alphabetically Signed-off-by: Deluan <deluan@navidrome.org> * fix: artist playcounts Signed-off-by: Deluan <deluan@navidrome.org> * change default Album PID conf Signed-off-by: Deluan <deluan@navidrome.org> * fix albumartist link when it does not match any albumartists values Signed-off-by: Deluan <deluan@navidrome.org> * fix `Ignoring filter not whitelisted` (role) message Signed-off-by: Deluan <deluan@navidrome.org> * fix: trim any names/titles being imported Signed-off-by: Deluan <deluan@navidrome.org> * remove unused genre code Signed-off-by: Deluan <deluan@navidrome.org> * serialize calls to Last.fm's getArtist Signed-off-by: Deluan <deluan@navidrome.org> xxx Signed-off-by: Deluan <deluan@navidrome.org> * add counters to genres Signed-off-by: Deluan <deluan@navidrome.org> * nit: fix migration `notice` message Signed-off-by: Deluan <deluan@navidrome.org> * optimize similar artists query Signed-off-by: Deluan <deluan@navidrome.org> * fix: last.fm.getInfo when mbid does not exist Signed-off-by: Deluan <deluan@navidrome.org> * ui only show missing items for admins Signed-off-by: Deluan <deluan@navidrome.org> * don't allow interaction with missing items Signed-off-by: Deluan <deluan@navidrome.org> * Add Missing Files view (WIP) Signed-off-by: Deluan <deluan@navidrome.org> * refactor: merged tag_counts into tag table Signed-off-by: Deluan <deluan@navidrome.org> * add option to completely disable automatic scanner Signed-off-by: Deluan <deluan@navidrome.org> * add delete missing files functionality Signed-off-by: Deluan <deluan@navidrome.org> * fix: playlists not showing for regular users Signed-off-by: Deluan <deluan@navidrome.org> * reduce updateLastAccess frequency to once every minute Signed-off-by: Deluan <deluan@navidrome.org> * reduce update player frequency to once every minute Signed-off-by: Deluan <deluan@navidrome.org> * add timeout when updating player Signed-off-by: Deluan <deluan@navidrome.org> * remove dead code Signed-off-by: Deluan <deluan@navidrome.org> * fix duplicated roles in stats Signed-off-by: Deluan <deluan@navidrome.org> * add `; ` to artist splitters Signed-off-by: Deluan <deluan@navidrome.org> * fix stats query Signed-off-by: Deluan <deluan@navidrome.org> * more logs Signed-off-by: Deluan <deluan@navidrome.org> * fix: support legacy clients (DSub) by removing OpenSubsonic extra fields - WIP Signed-off-by: Deluan <deluan@navidrome.org> * fix: support legacy clients (DSub) by removing OpenSubsonic extra fields - WIP Signed-off-by: Deluan <deluan@navidrome.org> * fix: support legacy clients (DSub) by removing OpenSubsonic extra fields - WIP Signed-off-by: Deluan <deluan@navidrome.org> * fix: support legacy clients (DSub) by removing OpenSubsonic extra fields - WIP Signed-off-by: Deluan <deluan@navidrome.org> * add record label filter Signed-off-by: Deluan <deluan@navidrome.org> * add release type filter Signed-off-by: Deluan <deluan@navidrome.org> * fix purgeUnused tags Signed-off-by: Deluan <deluan@navidrome.org> * add grouping filter to albums Signed-off-by: Deluan <deluan@navidrome.org> * allow any album tags to be used in as filters in the API Signed-off-by: Deluan <deluan@navidrome.org> * remove empty tags from album info Signed-off-by: Deluan <deluan@navidrome.org> * comments in the migration Signed-off-by: Deluan <deluan@navidrome.org> * fix: Cannot read properties of undefined Signed-off-by: Deluan <deluan@navidrome.org> * fix: listenbrainz scrobbling (#3640) Signed-off-by: Deluan <deluan@navidrome.org> * fix: remove duplicated tag values Signed-off-by: Deluan <deluan@navidrome.org> * fix: don't ignore the taglib folder! Signed-off-by: Deluan <deluan@navidrome.org> * feat: show track subtitle tag Signed-off-by: Deluan <deluan@navidrome.org> * fix: show artists stats based on selected role Signed-off-by: Deluan <deluan@navidrome.org> * fix: inspect Signed-off-by: Deluan <deluan@navidrome.org> * add media type to album info/filters Signed-off-by: Deluan <deluan@navidrome.org> * fix: change format of subtitle in the UI Signed-off-by: Deluan <deluan@navidrome.org> * fix: subtitle in Subsonic API and search Signed-off-by: Deluan <deluan@navidrome.org> * fix: subtitle in UI's player Signed-off-by: Deluan <deluan@navidrome.org> * fix: split strings should be case-insensitive Signed-off-by: Deluan <deluan@navidrome.org> * disable ScanSchedule Signed-off-by: Deluan <deluan@navidrome.org> * increase default sessiontimeout Signed-off-by: Deluan <deluan@navidrome.org> * add sqlite command line tool to docker image Signed-off-by: Deluan <deluan@navidrome.org> * fix: resources override Signed-off-by: Deluan <deluan@navidrome.org> * fix: album PID conf Signed-off-by: Deluan <deluan@navidrome.org> * change migration to mark current artists as albumArtists Signed-off-by: Deluan <deluan@navidrome.org> * feat(ui): Allow filtering on multiple genres (#3679) * feat(ui): Allow filtering on multiple genres Signed-off-by: Henrik Nordvik <henrikno@gmail.com> Signed-off-by: Deluan <deluan@navidrome.org> * add multi-genre filter in Album list Signed-off-by: Deluan <deluan@navidrome.org> --------- Signed-off-by: Henrik Nordvik <henrikno@gmail.com> Signed-off-by: Deluan <deluan@navidrome.org> Co-authored-by: Henrik Nordvik <henrikno@gmail.com> * add more multi-valued tag filters to Album and Song views Signed-off-by: Deluan <deluan@navidrome.org> * fix(ui): unselect missing files after removing Signed-off-by: Deluan <deluan@navidrome.org> * fix(ui): song filter Signed-off-by: Deluan <deluan@navidrome.org> * fix sharing tracks. fix #3687 Signed-off-by: Deluan <deluan@navidrome.org> * use rowids when using search for sync (ex: Symfonium) Signed-off-by: Deluan <deluan@navidrome.org> * fix "Report Real Paths" option for subsonic clients Signed-off-by: Deluan <deluan@navidrome.org> * fix "Report Real Paths" option for subsonic clients for search Signed-off-by: Deluan <deluan@navidrome.org> * add libraryPath to Native API /songs endpoint Signed-off-by: Deluan <deluan@navidrome.org> * feat(subsonic): add album version Signed-off-by: Deluan <deluan@navidrome.org> * made all tags lowercase as they are case-insensitive anyways. Signed-off-by: Deluan <deluan@navidrome.org> * feat(ui): Show full paths, extended properties for album/song (#3691) * feat(ui): Show full paths, extended properties for album/song - uses library path + os separator + path - show participants (album/song) and tags (song) - make album/participant clickable in show info * add source to path * fix pathSeparator in UI Signed-off-by: Deluan <deluan@navidrome.org> * fix local artist artwork (#3695) Signed-off-by: Deluan <deluan@navidrome.org> * fix: parse vorbis performers Signed-off-by: Deluan <deluan@navidrome.org> * refactor: clean function into smaller functions Signed-off-by: Deluan <deluan@navidrome.org> * fix translations for en and pt Signed-off-by: Deluan <deluan@navidrome.org> * add trace log to show annotations reassignment Signed-off-by: Deluan <deluan@navidrome.org> * add trace log to show annotations reassignment Signed-off-by: Deluan <deluan@navidrome.org> * fix: allow performers without instrument/subrole Signed-off-by: Deluan <deluan@navidrome.org> * refactor: metadata clean function again Signed-off-by: Deluan <deluan@navidrome.org> * refactor: optimize split function Signed-off-by: Deluan <deluan@navidrome.org> * refactor: split function is now a method of TagConf Signed-off-by: Deluan <deluan@navidrome.org> * fix: humanize Artist total size Signed-off-by: Deluan <deluan@navidrome.org> * add album version to album details Signed-off-by: Deluan <deluan@navidrome.org> * don't display album-level tags in SongInfo Signed-off-by: Deluan <deluan@navidrome.org> * fix genre clicking in Album Page Signed-off-by: Deluan <deluan@navidrome.org> * don't use mbids in Last.fm api calls. From https://discord.com/channels/671335427726114836/704303730660737113/1337574018143879248: With MBID: ``` GET https://ws.audioscrobbler.com/2.0/?api_key=XXXX&artist=Van+Morrison&format=json&lang=en&mbid=a41ac10f-0a56-4672-9161-b83f9b223559&method=artist.getInfo { artist: { name: "Bee Gees", mbid: "bf0f7e29-dfe1-416c-b5c6-f9ebc19ea810", url: "https://www.last.fm/music/Bee+Gees", } ``` Without MBID: ``` GET https://ws.audioscrobbler.com/2.0/?api_key=XXXX&artist=Van+Morrison&format=json&lang=en&method=artist.getInfo { artist: { name: "Van Morrison", mbid: "a41ac10f-0a56-4672-9161-b83f9b223559", url: "https://www.last.fm/music/Van+Morrison", } ``` Signed-off-by: Deluan <deluan@navidrome.org> * better logging for when the artist folder is not found Signed-off-by: Deluan <deluan@navidrome.org> * fix various issues with artist image resolution Signed-off-by: Deluan <deluan@navidrome.org> * hide "Additional Tags" header if there are none. Signed-off-by: Deluan <deluan@navidrome.org> * simplify tag rendering Signed-off-by: Deluan <deluan@navidrome.org> * enhance logging for artist folder detection Signed-off-by: Deluan <deluan@navidrome.org> * make folderID consistent for relative and absolute folderPaths Signed-off-by: Deluan <deluan@navidrome.org> * handle more folder paths scenarios Signed-off-by: Deluan <deluan@navidrome.org> * filter out other roles when SubsonicArtistParticipations = true Signed-off-by: Deluan <deluan@navidrome.org> * fix "Cannot read properties of undefined" Signed-off-by: Deluan <deluan@navidrome.org> * fix lyrics and comments being truncated (#3701) * fix lyrics and comments being truncated * specifically test for lyrics and comment length * reorder assertions Signed-off-by: Deluan <deluan@navidrome.org> --------- Signed-off-by: Deluan <deluan@navidrome.org> Co-authored-by: Deluan <deluan@navidrome.org> * fix(server): Expose library_path for playlist (#3705) Allows showing absolute path for UI, and makes "report real path" work for playlists (Subsonic) * fix BFR on Windows (#3704) * fix potential reflected cross-site scripting vulnerability Signed-off-by: Deluan <deluan@navidrome.org> * hack to make it work on Windows * ignore windows executables * try fixing the pipeline Signed-off-by: Deluan <deluan@navidrome.org> * allow MusicFolder in other drives * move windows local drive logic to local storage implementation --------- Signed-off-by: Deluan <deluan@navidrome.org> * increase pagination sizes for missing files Signed-off-by: Deluan <deluan@navidrome.org> * reduce level of "already scanning" watcher log message Signed-off-by: Deluan <deluan@navidrome.org> * only count folders with audio files in it See https://github.com/navidrome/navidrome/discussions/3676#discussioncomment-11990930 Signed-off-by: Deluan <deluan@navidrome.org> * add album version and catalog number to search Signed-off-by: Deluan <deluan@navidrome.org> * add `organization` alias for `recordlabel` Signed-off-by: Deluan <deluan@navidrome.org> * remove mbid from Last.fm agent Signed-off-by: Deluan <deluan@navidrome.org> * feat: support inspect in ui (#3726) * inspect in ui * address round 1 * add catalogNum to AlbumInfo Signed-off-by: Deluan <deluan@navidrome.org> * remove dependency on metadata_old (deprecated) package Signed-off-by: Deluan <deluan@navidrome.org> * add `RawTags` to model Signed-off-by: Deluan <deluan@navidrome.org> * support parsing MBIDs for roles (from the https://github.com/kgarner7/picard-all-mbids plugin) (#3698) * parse standard roles, vorbis/m4a work for now * fix djmixer * working roles, use DJ-mix * add performers to file * map mbids * add a few more tests * add test Signed-off-by: Deluan <deluan@navidrome.org> * try to simplify the performers logic Signed-off-by: Deluan <deluan@navidrome.org> * stylistic changes --------- Signed-off-by: Deluan <deluan@navidrome.org> Co-authored-by: Deluan <deluan@navidrome.org> * remove param mutation Signed-off-by: Deluan <deluan@navidrome.org> * run automated SQLite optimizations Signed-off-by: Deluan <deluan@navidrome.org> * fix playlists import/export on Windows * fix import playlists * fix export playlists * better handling of Windows volumes Signed-off-by: Deluan <deluan@navidrome.org> * handle more album ID reassignments Signed-off-by: Deluan <deluan@navidrome.org> * allow adding/overriding tags in the config file Signed-off-by: Deluan <deluan@navidrome.org> * fix(ui): Fix playlist track id, handle missing tracks better (#3734) - Use `mediaFileId` instead of `id` for playlist tracks - Only fetch if the file is not missing - If extractor fails to get the file, also error (rather than panic) * optimize DB after each scan. Signed-off-by: Deluan <deluan@navidrome.org> * remove sortable from AlbumSongs columns Signed-off-by: Deluan <deluan@navidrome.org> * simplify query to get missing tracks Signed-off-by: Deluan <deluan@navidrome.org> * mark Scanner.Extractor as deprecated Signed-off-by: Deluan <deluan@navidrome.org> --------- Signed-off-by: Deluan <deluan@navidrome.org> Signed-off-by: Henrik Nordvik <henrikno@gmail.com> Co-authored-by: Caio Cotts <caio@cotts.com.br> Co-authored-by: Henrik Nordvik <henrikno@gmail.com> Co-authored-by: Kendall Garner <17521368+kgarner7@users.noreply.github.com>
This commit is contained in:
Deluan Quintão
@@ -3,18 +3,19 @@ package persistence
|
||||
import (
|
||||
"cmp"
|
||||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"net/url"
|
||||
"slices"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
. "github.com/Masterminds/squirrel"
|
||||
"github.com/deluan/rest"
|
||||
"github.com/navidrome/navidrome/conf"
|
||||
"github.com/navidrome/navidrome/consts"
|
||||
"github.com/navidrome/navidrome/log"
|
||||
"github.com/navidrome/navidrome/model"
|
||||
"github.com/navidrome/navidrome/utils"
|
||||
. "github.com/navidrome/navidrome/utils/gg"
|
||||
"github.com/navidrome/navidrome/utils/slice"
|
||||
"github.com/pocketbase/dbx"
|
||||
)
|
||||
@@ -26,35 +27,84 @@ type artistRepository struct {
|
||||
|
||||
type dbArtist struct {
|
||||
*model.Artist `structs:",flatten"`
|
||||
SimilarArtists string `structs:"-" json:"similarArtists"`
|
||||
SimilarArtists string `structs:"-" json:"-"`
|
||||
Stats string `structs:"-" json:"-"`
|
||||
}
|
||||
|
||||
type dbSimilarArtist struct {
|
||||
ID string `json:"id,omitempty"`
|
||||
Name string `json:"name,omitempty"`
|
||||
}
|
||||
|
||||
func (a *dbArtist) PostScan() error {
|
||||
var stats map[string]map[string]int64
|
||||
if err := json.Unmarshal([]byte(a.Stats), &stats); err != nil {
|
||||
return fmt.Errorf("parsing artist stats from db: %w", err)
|
||||
}
|
||||
a.Artist.Stats = make(map[model.Role]model.ArtistStats)
|
||||
for key, c := range stats {
|
||||
if key == "total" {
|
||||
a.Artist.Size = c["s"]
|
||||
a.Artist.SongCount = int(c["m"])
|
||||
a.Artist.AlbumCount = int(c["a"])
|
||||
}
|
||||
role := model.RoleFromString(key)
|
||||
if role == model.RoleInvalid {
|
||||
continue
|
||||
}
|
||||
a.Artist.Stats[role] = model.ArtistStats{
|
||||
SongCount: int(c["m"]),
|
||||
AlbumCount: int(c["a"]),
|
||||
Size: c["s"],
|
||||
}
|
||||
}
|
||||
a.Artist.SimilarArtists = nil
|
||||
if a.SimilarArtists == "" {
|
||||
return nil
|
||||
}
|
||||
for _, s := range strings.Split(a.SimilarArtists, ";") {
|
||||
fields := strings.Split(s, ":")
|
||||
if len(fields) != 2 {
|
||||
continue
|
||||
}
|
||||
name, _ := url.QueryUnescape(fields[1])
|
||||
var sa []dbSimilarArtist
|
||||
if err := json.Unmarshal([]byte(a.SimilarArtists), &sa); err != nil {
|
||||
return fmt.Errorf("parsing similar artists from db: %w", err)
|
||||
}
|
||||
for _, s := range sa {
|
||||
a.Artist.SimilarArtists = append(a.Artist.SimilarArtists, model.Artist{
|
||||
ID: fields[0],
|
||||
Name: name,
|
||||
ID: s.ID,
|
||||
Name: s.Name,
|
||||
})
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (a *dbArtist) PostMapArgs(m map[string]any) error {
|
||||
var sa []string
|
||||
sa := make([]dbSimilarArtist, 0)
|
||||
for _, s := range a.Artist.SimilarArtists {
|
||||
sa = append(sa, fmt.Sprintf("%s:%s", s.ID, url.QueryEscape(s.Name)))
|
||||
sa = append(sa, dbSimilarArtist{ID: s.ID, Name: s.Name})
|
||||
}
|
||||
similarArtists, _ := json.Marshal(sa)
|
||||
m["similar_artists"] = string(similarArtists)
|
||||
m["full_text"] = formatFullText(a.Name, a.SortArtistName)
|
||||
|
||||
// Do not override the sort_artist_name and mbz_artist_id fields if they are empty
|
||||
// BFR: Better way to handle this?
|
||||
if v, ok := m["sort_artist_name"]; !ok || v.(string) == "" {
|
||||
delete(m, "sort_artist_name")
|
||||
}
|
||||
if v, ok := m["mbz_artist_id"]; !ok || v.(string) == "" {
|
||||
delete(m, "mbz_artist_id")
|
||||
}
|
||||
m["similar_artists"] = strings.Join(sa, ";")
|
||||
return nil
|
||||
}
|
||||
|
||||
type dbArtists []dbArtist
|
||||
|
||||
func (dba dbArtists) toModels() model.Artists {
|
||||
res := make(model.Artists, len(dba))
|
||||
for i := range dba {
|
||||
res[i] = *dba[i].Artist
|
||||
}
|
||||
return res
|
||||
}
|
||||
|
||||
func NewArtistRepository(ctx context.Context, db dbx.Builder) model.ArtistRepository {
|
||||
r := &artistRepository{}
|
||||
r.ctx = ctx
|
||||
@@ -62,80 +112,82 @@ func NewArtistRepository(ctx context.Context, db dbx.Builder) model.ArtistReposi
|
||||
r.indexGroups = utils.ParseIndexGroups(conf.Server.IndexGroups)
|
||||
r.tableName = "artist" // To be used by the idFilter below
|
||||
r.registerModel(&model.Artist{}, map[string]filterFunc{
|
||||
"id": idFilter(r.tableName),
|
||||
"name": fullTextFilter,
|
||||
"starred": booleanFilter,
|
||||
"genre_id": eqFilter,
|
||||
"id": idFilter(r.tableName),
|
||||
"name": fullTextFilter(r.tableName),
|
||||
"starred": booleanFilter,
|
||||
"role": roleFilter,
|
||||
})
|
||||
r.setSortMappings(map[string]string{
|
||||
"name": "order_artist_name",
|
||||
"starred_at": "starred, starred_at",
|
||||
"name": "order_artist_name",
|
||||
"starred_at": "starred, starred_at",
|
||||
"song_count": "stats->>'total'->>'m'",
|
||||
"album_count": "stats->>'total'->>'a'",
|
||||
"size": "stats->>'total'->>'s'",
|
||||
})
|
||||
return r
|
||||
}
|
||||
|
||||
func roleFilter(_ string, role any) Sqlizer {
|
||||
return NotEq{fmt.Sprintf("stats ->> '$.%v'", role): nil}
|
||||
}
|
||||
|
||||
func (r *artistRepository) selectArtist(options ...model.QueryOptions) SelectBuilder {
|
||||
sql := r.newSelectWithAnnotation("artist.id", options...).Columns("artist.*")
|
||||
return r.withGenres(sql).GroupBy("artist.id")
|
||||
query := r.newSelect(options...).Columns("artist.*")
|
||||
query = r.withAnnotation(query, "artist.id")
|
||||
// BFR How to handle counts and sizes (per role)?
|
||||
return query
|
||||
}
|
||||
|
||||
func (r *artistRepository) CountAll(options ...model.QueryOptions) (int64, error) {
|
||||
sql := r.newSelectWithAnnotation("artist.id")
|
||||
sql = r.withGenres(sql) // Required for filtering by genre
|
||||
return r.count(sql, options...)
|
||||
query := r.newSelect()
|
||||
query = r.withAnnotation(query, "artist.id")
|
||||
return r.count(query, options...)
|
||||
}
|
||||
|
||||
func (r *artistRepository) Exists(id string) (bool, error) {
|
||||
return r.exists(Select().Where(Eq{"artist.id": id}))
|
||||
return r.exists(Eq{"artist.id": id})
|
||||
}
|
||||
|
||||
func (r *artistRepository) Put(a *model.Artist, colsToUpdate ...string) error {
|
||||
a.FullText = getFullText(a.Name, a.SortArtistName)
|
||||
dba := &dbArtist{Artist: a}
|
||||
dba.CreatedAt = P(time.Now())
|
||||
dba.UpdatedAt = dba.CreatedAt
|
||||
_, err := r.put(dba.ID, dba, colsToUpdate...)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if a.ID == consts.VariousArtistsID {
|
||||
return r.updateGenres(a.ID, nil)
|
||||
}
|
||||
return r.updateGenres(a.ID, a.Genres)
|
||||
return err
|
||||
}
|
||||
|
||||
func (r *artistRepository) UpdateExternalInfo(a *model.Artist) error {
|
||||
dba := &dbArtist{Artist: a}
|
||||
_, err := r.put(a.ID, dba,
|
||||
"biography", "small_image_url", "medium_image_url", "large_image_url",
|
||||
"similar_artists", "external_url", "external_info_updated_at")
|
||||
return err
|
||||
}
|
||||
|
||||
func (r *artistRepository) Get(id string) (*model.Artist, error) {
|
||||
sel := r.selectArtist().Where(Eq{"artist.id": id})
|
||||
var dba []dbArtist
|
||||
var dba dbArtists
|
||||
if err := r.queryAll(sel, &dba); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if len(dba) == 0 {
|
||||
return nil, model.ErrNotFound
|
||||
}
|
||||
res := r.toModels(dba)
|
||||
err := loadAllGenres(r, res)
|
||||
return &res[0], err
|
||||
res := dba.toModels()
|
||||
return &res[0], nil
|
||||
}
|
||||
|
||||
func (r *artistRepository) GetAll(options ...model.QueryOptions) (model.Artists, error) {
|
||||
sel := r.selectArtist(options...)
|
||||
var dba []dbArtist
|
||||
var dba dbArtists
|
||||
err := r.queryAll(sel, &dba)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
res := r.toModels(dba)
|
||||
err = loadAllGenres(r, res)
|
||||
res := dba.toModels()
|
||||
return res, err
|
||||
}
|
||||
|
||||
func (r *artistRepository) toModels(dba []dbArtist) model.Artists {
|
||||
res := model.Artists{}
|
||||
for i := range dba {
|
||||
res = append(res, *dba[i].Artist)
|
||||
}
|
||||
return res
|
||||
}
|
||||
|
||||
func (r *artistRepository) getIndexKey(a model.Artist) string {
|
||||
source := a.OrderArtistName
|
||||
if conf.Server.PreferSortTags {
|
||||
@@ -151,8 +203,15 @@ func (r *artistRepository) getIndexKey(a model.Artist) string {
|
||||
}
|
||||
|
||||
// TODO Cache the index (recalculate when there are changes to the DB)
|
||||
func (r *artistRepository) GetIndex() (model.ArtistIndexes, error) {
|
||||
artists, err := r.GetAll(model.QueryOptions{Sort: "name"})
|
||||
func (r *artistRepository) GetIndex(roles ...model.Role) (model.ArtistIndexes, error) {
|
||||
options := model.QueryOptions{Sort: "name"}
|
||||
if len(roles) > 0 {
|
||||
roleFilters := slice.Map(roles, func(r model.Role) Sqlizer {
|
||||
return roleFilter("role", r)
|
||||
})
|
||||
options.Filters = And(roleFilters)
|
||||
}
|
||||
artists, err := r.GetAll(options)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
@@ -167,23 +226,119 @@ func (r *artistRepository) GetIndex() (model.ArtistIndexes, error) {
|
||||
}
|
||||
|
||||
func (r *artistRepository) purgeEmpty() error {
|
||||
del := Delete(r.tableName).Where("id not in (select distinct(album_artist_id) from album)")
|
||||
del := Delete(r.tableName).Where("id not in (select artist_id from album_artists)")
|
||||
c, err := r.executeSQL(del)
|
||||
if err == nil {
|
||||
if c > 0 {
|
||||
log.Debug(r.ctx, "Purged empty artists", "totalDeleted", c)
|
||||
}
|
||||
if err != nil {
|
||||
return fmt.Errorf("purging empty artists: %w", err)
|
||||
}
|
||||
return err
|
||||
if c > 0 {
|
||||
log.Debug(r.ctx, "Purged empty artists", "totalDeleted", c)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (r *artistRepository) Search(q string, offset int, size int) (model.Artists, error) {
|
||||
var dba []dbArtist
|
||||
err := r.doSearch(q, offset, size, &dba, "name")
|
||||
// RefreshPlayCounts updates the play count and last play date annotations for all artists, based
|
||||
// on the media files associated with them.
|
||||
func (r *artistRepository) RefreshPlayCounts() (int64, error) {
|
||||
query := rawSQL(`
|
||||
with play_counts as (
|
||||
select user_id, atom as artist_id, sum(play_count) as total_play_count, max(play_date) as last_play_date
|
||||
from media_file
|
||||
join annotation on item_id = media_file.id
|
||||
left join json_tree(participants, '$.artist') as jt
|
||||
where atom is not null and key = 'id'
|
||||
group by user_id, atom
|
||||
)
|
||||
insert into annotation (user_id, item_id, item_type, play_count, play_date)
|
||||
select user_id, artist_id, 'artist', total_play_count, last_play_date
|
||||
from play_counts
|
||||
where total_play_count > 0
|
||||
on conflict (user_id, item_id, item_type) do update
|
||||
set play_count = excluded.play_count,
|
||||
play_date = excluded.play_date;
|
||||
`)
|
||||
return r.executeSQL(query)
|
||||
}
|
||||
|
||||
// RefreshStats updates the stats field for all artists, based on the media files associated with them.
|
||||
// BFR Maybe filter by "touched" artists?
|
||||
func (r *artistRepository) RefreshStats() (int64, error) {
|
||||
// First get all counters, one query groups by artist/role, and another with totals per artist.
|
||||
// Union both queries and group by artist to get a single row of counters per artist/role.
|
||||
// Then format the counters in a JSON object, one key for each role.
|
||||
// Finally update the artist table with the new counters
|
||||
// In all queries, atom is the artist ID and path is the role (or "total" for the totals)
|
||||
query := rawSQL(`
|
||||
-- CTE to get counters for each artist, grouped by role
|
||||
with artist_role_counters as (
|
||||
-- Get counters for each artist, grouped by role
|
||||
-- (remove the index from the role: composer[0] => composer
|
||||
select atom as artist_id,
|
||||
substr(
|
||||
replace(jt.path, '$.', ''),
|
||||
1,
|
||||
case when instr(replace(jt.path, '$.', ''), '[') > 0
|
||||
then instr(replace(jt.path, '$.', ''), '[') - 1
|
||||
else length(replace(jt.path, '$.', ''))
|
||||
end
|
||||
) as role,
|
||||
count(distinct album_id) as album_count,
|
||||
count(mf.id) as count,
|
||||
sum(size) as size
|
||||
from media_file mf
|
||||
left join json_tree(participants) jt
|
||||
where atom is not null and key = 'id'
|
||||
group by atom, role
|
||||
),
|
||||
|
||||
-- CTE to get the totals for each artist
|
||||
artist_total_counters as (
|
||||
select mfa.artist_id,
|
||||
'total' as role,
|
||||
count(distinct mf.album) as album_count,
|
||||
count(distinct mf.id) as count,
|
||||
sum(mf.size) as size
|
||||
from (select distinct artist_id, media_file_id
|
||||
from main.media_file_artists) as mfa
|
||||
join main.media_file mf on mfa.media_file_id = mf.id
|
||||
group by mfa.artist_id
|
||||
),
|
||||
|
||||
-- CTE to combine role and total counters
|
||||
combined_counters as (
|
||||
select artist_id, role, album_count, count, size
|
||||
from artist_role_counters
|
||||
union
|
||||
select artist_id, role, album_count, count, size
|
||||
from artist_total_counters
|
||||
),
|
||||
|
||||
-- CTE to format the counters in a JSON object
|
||||
artist_counters as (
|
||||
select artist_id as id,
|
||||
json_group_object(
|
||||
replace(role, '"', ''),
|
||||
json_object('a', album_count, 'm', count, 's', size)
|
||||
) as counters
|
||||
from combined_counters
|
||||
group by artist_id
|
||||
)
|
||||
|
||||
-- Update the artist table with the new counters
|
||||
update artist
|
||||
set stats = coalesce((select counters from artist_counters where artist_counters.id = artist.id), '{}'),
|
||||
updated_at = datetime(current_timestamp, 'localtime')
|
||||
where id <> ''; -- always true, to avoid warnings`)
|
||||
return r.executeSQL(query)
|
||||
}
|
||||
|
||||
func (r *artistRepository) Search(q string, offset int, size int, includeMissing bool) (model.Artists, error) {
|
||||
var dba dbArtists
|
||||
err := r.doSearch(r.selectArtist(), q, offset, size, includeMissing, &dba, "json_extract(stats, '$.total.m') desc", "name")
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return r.toModels(dba), nil
|
||||
return dba.toModels(), nil
|
||||
}
|
||||
|
||||
func (r *artistRepository) Count(options ...rest.QueryOptions) (int64, error) {
|
||||
@@ -195,6 +350,15 @@ func (r *artistRepository) Read(id string) (interface{}, error) {
|
||||
}
|
||||
|
||||
func (r *artistRepository) ReadAll(options ...rest.QueryOptions) (interface{}, error) {
|
||||
role := "total"
|
||||
if len(options) > 0 {
|
||||
if v, ok := options[0].Filters["role"].(string); ok {
|
||||
role = v
|
||||
}
|
||||
}
|
||||
r.sortMappings["song_count"] = "stats->>'" + role + "'->>'m'"
|
||||
r.sortMappings["album_count"] = "stats->>'" + role + "'->>'a'"
|
||||
r.sortMappings["size"] = "stats->>'" + role + "'->>'s'"
|
||||
return r.GetAll(r.parseRestOptions(r.ctx, options...))
|
||||
}
|
||||
|
||||
|
||||
Reference in New Issue
Block a user