feat(bfr): Big Refactor: new scanner, lots of new fields and tags, improvements and DB schema changes (#2709)
* fix(server): more race conditions when updating artist/album from external sources Signed-off-by: Deluan <deluan@navidrome.org> * feat(scanner): add .gitignore syntax to .ndignore. Resolves #1394 Signed-off-by: Deluan <deluan@navidrome.org> * fix(ui): null Signed-off-by: Deluan <deluan@navidrome.org> * fix(scanner): pass configfile option to child process Signed-off-by: Deluan <deluan@navidrome.org> * fix(scanner): resume interrupted fullScans Signed-off-by: Deluan <deluan@navidrome.org> * fix(scanner): remove old scanner code Signed-off-by: Deluan <deluan@navidrome.org> * fix(scanner): rename old metadata package Signed-off-by: Deluan <deluan@navidrome.org> * fix(scanner): move old metadata package Signed-off-by: Deluan <deluan@navidrome.org> * fix: tests Signed-off-by: Deluan <deluan@navidrome.org> * chore(deps): update Go to 1.23.4 Signed-off-by: Deluan <deluan@navidrome.org> * fix: logs Signed-off-by: Deluan <deluan@navidrome.org> * fix(test): Signed-off-by: Deluan <deluan@navidrome.org> * fix: log level Signed-off-by: Deluan <deluan@navidrome.org> * fix: remove log message Signed-off-by: Deluan <deluan@navidrome.org> * feat: add config for scanner watcher Signed-off-by: Deluan <deluan@navidrome.org> * refactor: children playlists Signed-off-by: Deluan <deluan@navidrome.org> * refactor: replace `interface{}` with `any` Signed-off-by: Deluan <deluan@navidrome.org> * fix: smart playlists with genres Signed-off-by: Deluan <deluan@navidrome.org> * fix: allow any tags in smart playlists Signed-off-by: Deluan <deluan@navidrome.org> * fix: artist names in playlists Signed-off-by: Deluan <deluan@navidrome.org> * fix: smart playlist's sort by tags Signed-off-by: Deluan <deluan@navidrome.org> * feat(subsonic): add moods to child Signed-off-by: Deluan <deluan@navidrome.org> * feat(subsonic): add moods to AlbumID3 Signed-off-by: Deluan <deluan@navidrome.org> * refactor(subsonic): use generic JSONArray for OS arrays Signed-off-by: Deluan <deluan@navidrome.org> * refactor(subsonic): use https in test Signed-off-by: Deluan <deluan@navidrome.org> * feat(subsonic): add releaseTypes to AlbumID3 Signed-off-by: Deluan <deluan@navidrome.org> * feat(subsonic): add recordLabels to AlbumID3 Signed-off-by: Deluan <deluan@navidrome.org> * refactor(subsonic): rename JSONArray to Array Signed-off-by: Deluan <deluan@navidrome.org> * feat(subsonic): add artists to AlbumID3 Signed-off-by: Deluan <deluan@navidrome.org> * feat(subsonic): add artists to Child Signed-off-by: Deluan <deluan@navidrome.org> * fix(scanner): do not pre-populate smart playlists Signed-off-by: Deluan <deluan@navidrome.org> * feat(subsonic): implement a simplified version of ArtistID3. See https://github.com/opensubsonic/open-subsonic-api/discussions/120 Signed-off-by: Deluan <deluan@navidrome.org> * feat(subsonic): add artists to album child Signed-off-by: Deluan <deluan@navidrome.org> * feat(subsonic): add contributors to mediafile Child Signed-off-by: Deluan <deluan@navidrome.org> * feat(subsonic): add albumArtists to mediafile Child Signed-off-by: Deluan <deluan@navidrome.org> * feat(subsonic): add displayArtist and displayAlbumArtist Signed-off-by: Deluan <deluan@navidrome.org> * feat(subsonic): add displayComposer to Child Signed-off-by: Deluan <deluan@navidrome.org> * feat(subsonic): add roles to ArtistID3 Signed-off-by: Deluan <deluan@navidrome.org> * fix(subsonic): use " • " separator for displayComposer Signed-off-by: Deluan <deluan@navidrome.org> * refactor: Signed-off-by: Deluan <deluan@navidrome.org> * fix(subsonic): Signed-off-by: Deluan <deluan@navidrome.org> * fix(subsonic): respect `PreferSortTags` config option Signed-off-by: Deluan <deluan@navidrome.org> * refactor(subsonic): Signed-off-by: Deluan <deluan@navidrome.org> * refactor: optimize purging non-unused tags Signed-off-by: Deluan <deluan@navidrome.org> * refactor: don't run 'refresh artist stats' concurrently with other transactions Signed-off-by: Deluan <deluan@navidrome.org> * refactor: Signed-off-by: Deluan <deluan@navidrome.org> * fix: log message Signed-off-by: Deluan <deluan@navidrome.org> * feat: add Scanner.ScanOnStartup config option, default true Signed-off-by: Deluan <deluan@navidrome.org> * feat: better json parsing error msg when importing NSPs Signed-off-by: Deluan <deluan@navidrome.org> * fix: don't update album's imported_time when updating external_metadata Signed-off-by: Deluan <deluan@navidrome.org> * fix: handle interrupted scans and full scans after migrations Signed-off-by: Deluan <deluan@navidrome.org> * feat: run `analyze` when migration requires a full rescan Signed-off-by: Deluan <deluan@navidrome.org> * feat: run `PRAGMA optimize` at the end of the scan Signed-off-by: Deluan <deluan@navidrome.org> * fix: don't update artist's updated_at when updating external_metadata Signed-off-by: Deluan <deluan@navidrome.org> * feat: handle multiple artists and roles in smart playlists Signed-off-by: Deluan <deluan@navidrome.org> * feat(ui): dim missing tracks Signed-off-by: Deluan <deluan@navidrome.org> * fix: album missing logic Signed-off-by: Deluan <deluan@navidrome.org> * fix: error encoding in gob Signed-off-by: Deluan <deluan@navidrome.org> * feat: separate warnings from errors Signed-off-by: Deluan <deluan@navidrome.org> * fix: mark albums as missing if they were contained in a deleted folder Signed-off-by: Deluan <deluan@navidrome.org> * refactor: add participant names to media_file and album tables Signed-off-by: Deluan <deluan@navidrome.org> * refactor: use participations in criteria, instead of m2m relationship Signed-off-by: Deluan <deluan@navidrome.org> * refactor: rename participations to participants Signed-off-by: Deluan <deluan@navidrome.org> * feat(subsonic): add moods to album child Signed-off-by: Deluan <deluan@navidrome.org> * fix: albumartist role case Signed-off-by: Deluan <deluan@navidrome.org> * feat(scanner): run scanner as an external process by default Signed-off-by: Deluan <deluan@navidrome.org> * fix(ui): show albumArtist names Signed-off-by: Deluan <deluan@navidrome.org> * fix(ui): dim out missing albums Signed-off-by: Deluan <deluan@navidrome.org> * fix: flaky test Signed-off-by: Deluan <deluan@navidrome.org> * fix(server): scrobble buffer mapping. fix #3583 Signed-off-by: Deluan <deluan@navidrome.org> * refactor: more participations renaming Signed-off-by: Deluan <deluan@navidrome.org> * fix: listenbrainz scrobbling Signed-off-by: Deluan <deluan@navidrome.org> * feat: send release_group_mbid to listenbrainz Signed-off-by: Deluan <deluan@navidrome.org> * feat(subsonic): implement OpenSubsonic explicitStatus field (#3597) * feat: implement OpenSubsonic explicitStatus field * fix(subsonic): fix failing snapshot tests * refactor: create helper for setting explicitStatus * fix: store smaller values for explicit-status on database * test: ToAlbum explicitStatus * refactor: rename explicitStatus helper function --------- Co-authored-by: Deluan Quintão <deluan@navidrome.org> * fix: handle album and track tags in the DB based on the mappings.yaml file Signed-off-by: Deluan <deluan@navidrome.org> * save similar artists as JSONB Signed-off-by: Deluan <deluan@navidrome.org> * fix: getAlbumList byGenre Signed-off-by: Deluan <deluan@navidrome.org> * detect changes in PID configuration Signed-off-by: Deluan <deluan@navidrome.org> * set default album PID to legacy_pid Signed-off-by: Deluan <deluan@navidrome.org> * fix tests Signed-off-by: Deluan <deluan@navidrome.org> * fix SIGSEGV Signed-off-by: Deluan <deluan@navidrome.org> * fix: don't lose album stars/ratings when migrating Signed-off-by: Deluan <deluan@navidrome.org> * store full PID conf in properties Signed-off-by: Deluan <deluan@navidrome.org> * fix: keep album annotations when changing PID.Album config Signed-off-by: Deluan <deluan@navidrome.org> * fix: reassign album annotations Signed-off-by: Deluan <deluan@navidrome.org> * feat: use (display) albumArtist and add links to each artist Signed-off-by: Deluan <deluan@navidrome.org> * fix: not showing albums by albumartist Signed-off-by: Deluan <deluan@navidrome.org> * fix: error msgs Signed-off-by: Deluan <deluan@navidrome.org> * fix: hide PID from Native API Signed-off-by: Deluan <deluan@navidrome.org> * fix: album cover art resolution Signed-off-by: Deluan <deluan@navidrome.org> * fix: trim participant names Signed-off-by: Deluan <deluan@navidrome.org> * fix: reduce watcher log spam Signed-off-by: Deluan <deluan@navidrome.org> * fix: panic when initializing the watcher Signed-off-by: Deluan <deluan@navidrome.org> * fix: various artists Signed-off-by: Deluan <deluan@navidrome.org> * fix: don't store empty lyrics in the DB Signed-off-by: Deluan <deluan@navidrome.org> * remove unused methods Signed-off-by: Deluan <deluan@navidrome.org> * drop full_text indexes, as they are not being used by SQLite Signed-off-by: Deluan <deluan@navidrome.org> * keep album created_at when upgrading Signed-off-by: Deluan <deluan@navidrome.org> * fix(ui): null pointer Signed-off-by: Deluan <deluan@navidrome.org> * fix: album artwork cache Signed-off-by: Deluan <deluan@navidrome.org> * fix: don't expose missing files in Subsonic API Signed-off-by: Deluan <deluan@navidrome.org> * refactor: searchable interface Signed-off-by: Deluan <deluan@navidrome.org> * fix: filter out missing items from subsonic search * fix: filter out missing items from playlists * fix: filter out missing items from shares Signed-off-by: Deluan <deluan@navidrome.org> * feat(ui): add filter by artist role Signed-off-by: Deluan <deluan@navidrome.org> * feat(subsonic): only return albumartists in getIndexes and getArtists endpoints Signed-off-by: Deluan <deluan@navidrome.org> * sort roles alphabetically Signed-off-by: Deluan <deluan@navidrome.org> * fix: artist playcounts Signed-off-by: Deluan <deluan@navidrome.org> * change default Album PID conf Signed-off-by: Deluan <deluan@navidrome.org> * fix albumartist link when it does not match any albumartists values Signed-off-by: Deluan <deluan@navidrome.org> * fix `Ignoring filter not whitelisted` (role) message Signed-off-by: Deluan <deluan@navidrome.org> * fix: trim any names/titles being imported Signed-off-by: Deluan <deluan@navidrome.org> * remove unused genre code Signed-off-by: Deluan <deluan@navidrome.org> * serialize calls to Last.fm's getArtist Signed-off-by: Deluan <deluan@navidrome.org> xxx Signed-off-by: Deluan <deluan@navidrome.org> * add counters to genres Signed-off-by: Deluan <deluan@navidrome.org> * nit: fix migration `notice` message Signed-off-by: Deluan <deluan@navidrome.org> * optimize similar artists query Signed-off-by: Deluan <deluan@navidrome.org> * fix: last.fm.getInfo when mbid does not exist Signed-off-by: Deluan <deluan@navidrome.org> * ui only show missing items for admins Signed-off-by: Deluan <deluan@navidrome.org> * don't allow interaction with missing items Signed-off-by: Deluan <deluan@navidrome.org> * Add Missing Files view (WIP) Signed-off-by: Deluan <deluan@navidrome.org> * refactor: merged tag_counts into tag table Signed-off-by: Deluan <deluan@navidrome.org> * add option to completely disable automatic scanner Signed-off-by: Deluan <deluan@navidrome.org> * add delete missing files functionality Signed-off-by: Deluan <deluan@navidrome.org> * fix: playlists not showing for regular users Signed-off-by: Deluan <deluan@navidrome.org> * reduce updateLastAccess frequency to once every minute Signed-off-by: Deluan <deluan@navidrome.org> * reduce update player frequency to once every minute Signed-off-by: Deluan <deluan@navidrome.org> * add timeout when updating player Signed-off-by: Deluan <deluan@navidrome.org> * remove dead code Signed-off-by: Deluan <deluan@navidrome.org> * fix duplicated roles in stats Signed-off-by: Deluan <deluan@navidrome.org> * add `; ` to artist splitters Signed-off-by: Deluan <deluan@navidrome.org> * fix stats query Signed-off-by: Deluan <deluan@navidrome.org> * more logs Signed-off-by: Deluan <deluan@navidrome.org> * fix: support legacy clients (DSub) by removing OpenSubsonic extra fields - WIP Signed-off-by: Deluan <deluan@navidrome.org> * fix: support legacy clients (DSub) by removing OpenSubsonic extra fields - WIP Signed-off-by: Deluan <deluan@navidrome.org> * fix: support legacy clients (DSub) by removing OpenSubsonic extra fields - WIP Signed-off-by: Deluan <deluan@navidrome.org> * fix: support legacy clients (DSub) by removing OpenSubsonic extra fields - WIP Signed-off-by: Deluan <deluan@navidrome.org> * add record label filter Signed-off-by: Deluan <deluan@navidrome.org> * add release type filter Signed-off-by: Deluan <deluan@navidrome.org> * fix purgeUnused tags Signed-off-by: Deluan <deluan@navidrome.org> * add grouping filter to albums Signed-off-by: Deluan <deluan@navidrome.org> * allow any album tags to be used in as filters in the API Signed-off-by: Deluan <deluan@navidrome.org> * remove empty tags from album info Signed-off-by: Deluan <deluan@navidrome.org> * comments in the migration Signed-off-by: Deluan <deluan@navidrome.org> * fix: Cannot read properties of undefined Signed-off-by: Deluan <deluan@navidrome.org> * fix: listenbrainz scrobbling (#3640) Signed-off-by: Deluan <deluan@navidrome.org> * fix: remove duplicated tag values Signed-off-by: Deluan <deluan@navidrome.org> * fix: don't ignore the taglib folder! Signed-off-by: Deluan <deluan@navidrome.org> * feat: show track subtitle tag Signed-off-by: Deluan <deluan@navidrome.org> * fix: show artists stats based on selected role Signed-off-by: Deluan <deluan@navidrome.org> * fix: inspect Signed-off-by: Deluan <deluan@navidrome.org> * add media type to album info/filters Signed-off-by: Deluan <deluan@navidrome.org> * fix: change format of subtitle in the UI Signed-off-by: Deluan <deluan@navidrome.org> * fix: subtitle in Subsonic API and search Signed-off-by: Deluan <deluan@navidrome.org> * fix: subtitle in UI's player Signed-off-by: Deluan <deluan@navidrome.org> * fix: split strings should be case-insensitive Signed-off-by: Deluan <deluan@navidrome.org> * disable ScanSchedule Signed-off-by: Deluan <deluan@navidrome.org> * increase default sessiontimeout Signed-off-by: Deluan <deluan@navidrome.org> * add sqlite command line tool to docker image Signed-off-by: Deluan <deluan@navidrome.org> * fix: resources override Signed-off-by: Deluan <deluan@navidrome.org> * fix: album PID conf Signed-off-by: Deluan <deluan@navidrome.org> * change migration to mark current artists as albumArtists Signed-off-by: Deluan <deluan@navidrome.org> * feat(ui): Allow filtering on multiple genres (#3679) * feat(ui): Allow filtering on multiple genres Signed-off-by: Henrik Nordvik <henrikno@gmail.com> Signed-off-by: Deluan <deluan@navidrome.org> * add multi-genre filter in Album list Signed-off-by: Deluan <deluan@navidrome.org> --------- Signed-off-by: Henrik Nordvik <henrikno@gmail.com> Signed-off-by: Deluan <deluan@navidrome.org> Co-authored-by: Henrik Nordvik <henrikno@gmail.com> * add more multi-valued tag filters to Album and Song views Signed-off-by: Deluan <deluan@navidrome.org> * fix(ui): unselect missing files after removing Signed-off-by: Deluan <deluan@navidrome.org> * fix(ui): song filter Signed-off-by: Deluan <deluan@navidrome.org> * fix sharing tracks. fix #3687 Signed-off-by: Deluan <deluan@navidrome.org> * use rowids when using search for sync (ex: Symfonium) Signed-off-by: Deluan <deluan@navidrome.org> * fix "Report Real Paths" option for subsonic clients Signed-off-by: Deluan <deluan@navidrome.org> * fix "Report Real Paths" option for subsonic clients for search Signed-off-by: Deluan <deluan@navidrome.org> * add libraryPath to Native API /songs endpoint Signed-off-by: Deluan <deluan@navidrome.org> * feat(subsonic): add album version Signed-off-by: Deluan <deluan@navidrome.org> * made all tags lowercase as they are case-insensitive anyways. Signed-off-by: Deluan <deluan@navidrome.org> * feat(ui): Show full paths, extended properties for album/song (#3691) * feat(ui): Show full paths, extended properties for album/song - uses library path + os separator + path - show participants (album/song) and tags (song) - make album/participant clickable in show info * add source to path * fix pathSeparator in UI Signed-off-by: Deluan <deluan@navidrome.org> * fix local artist artwork (#3695) Signed-off-by: Deluan <deluan@navidrome.org> * fix: parse vorbis performers Signed-off-by: Deluan <deluan@navidrome.org> * refactor: clean function into smaller functions Signed-off-by: Deluan <deluan@navidrome.org> * fix translations for en and pt Signed-off-by: Deluan <deluan@navidrome.org> * add trace log to show annotations reassignment Signed-off-by: Deluan <deluan@navidrome.org> * add trace log to show annotations reassignment Signed-off-by: Deluan <deluan@navidrome.org> * fix: allow performers without instrument/subrole Signed-off-by: Deluan <deluan@navidrome.org> * refactor: metadata clean function again Signed-off-by: Deluan <deluan@navidrome.org> * refactor: optimize split function Signed-off-by: Deluan <deluan@navidrome.org> * refactor: split function is now a method of TagConf Signed-off-by: Deluan <deluan@navidrome.org> * fix: humanize Artist total size Signed-off-by: Deluan <deluan@navidrome.org> * add album version to album details Signed-off-by: Deluan <deluan@navidrome.org> * don't display album-level tags in SongInfo Signed-off-by: Deluan <deluan@navidrome.org> * fix genre clicking in Album Page Signed-off-by: Deluan <deluan@navidrome.org> * don't use mbids in Last.fm api calls. From https://discord.com/channels/671335427726114836/704303730660737113/1337574018143879248: With MBID: ``` GET https://ws.audioscrobbler.com/2.0/?api_key=XXXX&artist=Van+Morrison&format=json&lang=en&mbid=a41ac10f-0a56-4672-9161-b83f9b223559&method=artist.getInfo { artist: { name: "Bee Gees", mbid: "bf0f7e29-dfe1-416c-b5c6-f9ebc19ea810", url: "https://www.last.fm/music/Bee+Gees", } ``` Without MBID: ``` GET https://ws.audioscrobbler.com/2.0/?api_key=XXXX&artist=Van+Morrison&format=json&lang=en&method=artist.getInfo { artist: { name: "Van Morrison", mbid: "a41ac10f-0a56-4672-9161-b83f9b223559", url: "https://www.last.fm/music/Van+Morrison", } ``` Signed-off-by: Deluan <deluan@navidrome.org> * better logging for when the artist folder is not found Signed-off-by: Deluan <deluan@navidrome.org> * fix various issues with artist image resolution Signed-off-by: Deluan <deluan@navidrome.org> * hide "Additional Tags" header if there are none. Signed-off-by: Deluan <deluan@navidrome.org> * simplify tag rendering Signed-off-by: Deluan <deluan@navidrome.org> * enhance logging for artist folder detection Signed-off-by: Deluan <deluan@navidrome.org> * make folderID consistent for relative and absolute folderPaths Signed-off-by: Deluan <deluan@navidrome.org> * handle more folder paths scenarios Signed-off-by: Deluan <deluan@navidrome.org> * filter out other roles when SubsonicArtistParticipations = true Signed-off-by: Deluan <deluan@navidrome.org> * fix "Cannot read properties of undefined" Signed-off-by: Deluan <deluan@navidrome.org> * fix lyrics and comments being truncated (#3701) * fix lyrics and comments being truncated * specifically test for lyrics and comment length * reorder assertions Signed-off-by: Deluan <deluan@navidrome.org> --------- Signed-off-by: Deluan <deluan@navidrome.org> Co-authored-by: Deluan <deluan@navidrome.org> * fix(server): Expose library_path for playlist (#3705) Allows showing absolute path for UI, and makes "report real path" work for playlists (Subsonic) * fix BFR on Windows (#3704) * fix potential reflected cross-site scripting vulnerability Signed-off-by: Deluan <deluan@navidrome.org> * hack to make it work on Windows * ignore windows executables * try fixing the pipeline Signed-off-by: Deluan <deluan@navidrome.org> * allow MusicFolder in other drives * move windows local drive logic to local storage implementation --------- Signed-off-by: Deluan <deluan@navidrome.org> * increase pagination sizes for missing files Signed-off-by: Deluan <deluan@navidrome.org> * reduce level of "already scanning" watcher log message Signed-off-by: Deluan <deluan@navidrome.org> * only count folders with audio files in it See https://github.com/navidrome/navidrome/discussions/3676#discussioncomment-11990930 Signed-off-by: Deluan <deluan@navidrome.org> * add album version and catalog number to search Signed-off-by: Deluan <deluan@navidrome.org> * add `organization` alias for `recordlabel` Signed-off-by: Deluan <deluan@navidrome.org> * remove mbid from Last.fm agent Signed-off-by: Deluan <deluan@navidrome.org> * feat: support inspect in ui (#3726) * inspect in ui * address round 1 * add catalogNum to AlbumInfo Signed-off-by: Deluan <deluan@navidrome.org> * remove dependency on metadata_old (deprecated) package Signed-off-by: Deluan <deluan@navidrome.org> * add `RawTags` to model Signed-off-by: Deluan <deluan@navidrome.org> * support parsing MBIDs for roles (from the https://github.com/kgarner7/picard-all-mbids plugin) (#3698) * parse standard roles, vorbis/m4a work for now * fix djmixer * working roles, use DJ-mix * add performers to file * map mbids * add a few more tests * add test Signed-off-by: Deluan <deluan@navidrome.org> * try to simplify the performers logic Signed-off-by: Deluan <deluan@navidrome.org> * stylistic changes --------- Signed-off-by: Deluan <deluan@navidrome.org> Co-authored-by: Deluan <deluan@navidrome.org> * remove param mutation Signed-off-by: Deluan <deluan@navidrome.org> * run automated SQLite optimizations Signed-off-by: Deluan <deluan@navidrome.org> * fix playlists import/export on Windows * fix import playlists * fix export playlists * better handling of Windows volumes Signed-off-by: Deluan <deluan@navidrome.org> * handle more album ID reassignments Signed-off-by: Deluan <deluan@navidrome.org> * allow adding/overriding tags in the config file Signed-off-by: Deluan <deluan@navidrome.org> * fix(ui): Fix playlist track id, handle missing tracks better (#3734) - Use `mediaFileId` instead of `id` for playlist tracks - Only fetch if the file is not missing - If extractor fails to get the file, also error (rather than panic) * optimize DB after each scan. Signed-off-by: Deluan <deluan@navidrome.org> * remove sortable from AlbumSongs columns Signed-off-by: Deluan <deluan@navidrome.org> * simplify query to get missing tracks Signed-off-by: Deluan <deluan@navidrome.org> * mark Scanner.Extractor as deprecated Signed-off-by: Deluan <deluan@navidrome.org> --------- Signed-off-by: Deluan <deluan@navidrome.org> Signed-off-by: Henrik Nordvik <henrikno@gmail.com> Co-authored-by: Caio Cotts <caio@cotts.com.br> Co-authored-by: Henrik Nordvik <henrikno@gmail.com> Co-authored-by: Kendall Garner <17521368+kgarner7@users.noreply.github.com>
This commit is contained in:
@@ -0,0 +1,25 @@
|
||||
package storage
|
||||
|
||||
import (
|
||||
"context"
|
||||
"io/fs"
|
||||
|
||||
"github.com/navidrome/navidrome/model/metadata"
|
||||
)
|
||||
|
||||
type Storage interface {
|
||||
FS() (MusicFS, error)
|
||||
}
|
||||
|
||||
// MusicFS is an interface that extends the fs.FS interface with the ability to read tags from files
|
||||
type MusicFS interface {
|
||||
fs.FS
|
||||
ReadTags(path ...string) (map[string]metadata.Info, error)
|
||||
}
|
||||
|
||||
// Watcher is a storage with the ability watch the FS and notify changes
|
||||
type Watcher interface {
|
||||
// Start starts a watcher on the whole FS and returns a channel to send detected changes.
|
||||
// The watcher must be stopped when the context is done.
|
||||
Start(context.Context) (<-chan string, error)
|
||||
}
|
||||
@@ -0,0 +1,29 @@
|
||||
package local
|
||||
|
||||
import (
|
||||
"io/fs"
|
||||
"sync"
|
||||
|
||||
"github.com/navidrome/navidrome/model/metadata"
|
||||
)
|
||||
|
||||
// Extractor is an interface that defines the methods that a tag/metadata extractor must implement
|
||||
type Extractor interface {
|
||||
Parse(files ...string) (map[string]metadata.Info, error)
|
||||
Version() string
|
||||
}
|
||||
|
||||
type extractorConstructor func(fs.FS, string) Extractor
|
||||
|
||||
var (
|
||||
extractors = map[string]extractorConstructor{}
|
||||
lock sync.RWMutex
|
||||
)
|
||||
|
||||
// RegisterExtractor registers a new extractor, so it can be used by the local storage. The one to be used is
|
||||
// defined with the configuration option Scanner.Extractor.
|
||||
func RegisterExtractor(id string, f extractorConstructor) {
|
||||
lock.Lock()
|
||||
defer lock.Unlock()
|
||||
extractors[id] = f
|
||||
}
|
||||
@@ -0,0 +1,91 @@
|
||||
package local
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"io/fs"
|
||||
"net/url"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"sync/atomic"
|
||||
"time"
|
||||
|
||||
"github.com/djherbis/times"
|
||||
"github.com/navidrome/navidrome/conf"
|
||||
"github.com/navidrome/navidrome/core/storage"
|
||||
"github.com/navidrome/navidrome/log"
|
||||
"github.com/navidrome/navidrome/model/metadata"
|
||||
)
|
||||
|
||||
// localStorage implements a Storage that reads the files from the local filesystem and uses registered extractors
|
||||
// to extract the metadata and tags from the files.
|
||||
type localStorage struct {
|
||||
u url.URL
|
||||
extractor Extractor
|
||||
resolvedPath string
|
||||
watching atomic.Bool
|
||||
}
|
||||
|
||||
func newLocalStorage(u url.URL) storage.Storage {
|
||||
newExtractor, ok := extractors[conf.Server.Scanner.Extractor]
|
||||
if !ok || newExtractor == nil {
|
||||
log.Fatal("Extractor not found", "path", conf.Server.Scanner.Extractor)
|
||||
}
|
||||
isWindowsPath := filepath.VolumeName(u.Host) != ""
|
||||
if u.Scheme == storage.LocalSchemaID && isWindowsPath {
|
||||
u.Path = filepath.Join(u.Host, u.Path)
|
||||
}
|
||||
resolvedPath, err := filepath.EvalSymlinks(u.Path)
|
||||
if err != nil {
|
||||
log.Warn("Error resolving path", "path", u.Path, "err", err)
|
||||
resolvedPath = u.Path
|
||||
}
|
||||
return &localStorage{u: u, extractor: newExtractor(os.DirFS(u.Path), u.Path), resolvedPath: resolvedPath}
|
||||
}
|
||||
|
||||
func (s *localStorage) FS() (storage.MusicFS, error) {
|
||||
path := s.u.Path
|
||||
if _, err := os.Stat(path); err != nil {
|
||||
return nil, fmt.Errorf("%w: %s", err, path)
|
||||
}
|
||||
return &localFS{FS: os.DirFS(path), extractor: s.extractor}, nil
|
||||
}
|
||||
|
||||
type localFS struct {
|
||||
fs.FS
|
||||
extractor Extractor
|
||||
}
|
||||
|
||||
func (lfs *localFS) ReadTags(path ...string) (map[string]metadata.Info, error) {
|
||||
res, err := lfs.extractor.Parse(path...)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
for path, v := range res {
|
||||
if v.FileInfo == nil {
|
||||
info, err := fs.Stat(lfs, path)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
v.FileInfo = localFileInfo{info}
|
||||
res[path] = v
|
||||
}
|
||||
}
|
||||
return res, nil
|
||||
}
|
||||
|
||||
// localFileInfo is a wrapper around fs.FileInfo that adds a BirthTime method, to make it compatible
|
||||
// with metadata.FileInfo
|
||||
type localFileInfo struct {
|
||||
fs.FileInfo
|
||||
}
|
||||
|
||||
func (lfi localFileInfo) BirthTime() time.Time {
|
||||
if ts := times.Get(lfi.FileInfo); ts.HasBirthTime() {
|
||||
return ts.BirthTime()
|
||||
}
|
||||
return time.Now()
|
||||
}
|
||||
|
||||
func init() {
|
||||
storage.Register(storage.LocalSchemaID, newLocalStorage)
|
||||
}
|
||||
@@ -0,0 +1,13 @@
|
||||
package local
|
||||
|
||||
import (
|
||||
"testing"
|
||||
|
||||
. "github.com/onsi/ginkgo/v2"
|
||||
. "github.com/onsi/gomega"
|
||||
)
|
||||
|
||||
func TestLocal(t *testing.T) {
|
||||
RegisterFailHandler(Fail)
|
||||
RunSpecs(t, "Local Storage Test Suite")
|
||||
}
|
||||
@@ -0,0 +1,5 @@
|
||||
package local
|
||||
|
||||
import "github.com/rjeczalik/notify"
|
||||
|
||||
const WatchEvents = notify.All | notify.FSEventsInodeMetaMod
|
||||
@@ -0,0 +1,7 @@
|
||||
//go:build !linux && !darwin && !windows
|
||||
|
||||
package local
|
||||
|
||||
import "github.com/rjeczalik/notify"
|
||||
|
||||
const WatchEvents = notify.All
|
||||
@@ -0,0 +1,5 @@
|
||||
package local
|
||||
|
||||
import "github.com/rjeczalik/notify"
|
||||
|
||||
const WatchEvents = notify.All | notify.InModify | notify.InAttrib
|
||||
@@ -0,0 +1,5 @@
|
||||
package local
|
||||
|
||||
import "github.com/rjeczalik/notify"
|
||||
|
||||
const WatchEvents = notify.All | notify.FileNotifyChangeAttributes
|
||||
@@ -0,0 +1,57 @@
|
||||
package local
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
|
||||
"github.com/navidrome/navidrome/log"
|
||||
"github.com/rjeczalik/notify"
|
||||
)
|
||||
|
||||
// Start starts a watcher on the whole FS and returns a channel to send detected changes.
|
||||
// It uses `notify` to detect changes in the filesystem, so it may not work on all platforms/use-cases.
|
||||
// Notoriously, it does not work on some networked mounts and Windows with WSL2.
|
||||
func (s *localStorage) Start(ctx context.Context) (<-chan string, error) {
|
||||
if !s.watching.CompareAndSwap(false, true) {
|
||||
return nil, errors.New("watcher already started")
|
||||
}
|
||||
input := make(chan notify.EventInfo, 1)
|
||||
output := make(chan string, 1)
|
||||
|
||||
started := make(chan struct{})
|
||||
go func() {
|
||||
defer close(input)
|
||||
defer close(output)
|
||||
|
||||
libPath := filepath.Join(s.u.Path, "...")
|
||||
log.Debug(ctx, "Starting watcher", "lib", libPath)
|
||||
err := notify.Watch(libPath, input, WatchEvents)
|
||||
if err != nil {
|
||||
log.Error("Error starting watcher", "lib", libPath, err)
|
||||
return
|
||||
}
|
||||
defer notify.Stop(input)
|
||||
close(started) // signals the main goroutine we have started
|
||||
|
||||
for {
|
||||
select {
|
||||
case event := <-input:
|
||||
log.Trace(ctx, "Detected change", "event", event, "lib", s.u.Path)
|
||||
name := event.Path()
|
||||
name = strings.Replace(name, s.resolvedPath, s.u.Path, 1)
|
||||
output <- name
|
||||
case <-ctx.Done():
|
||||
log.Debug(ctx, "Stopping watcher", "path", s.u.Path)
|
||||
s.watching.Store(false)
|
||||
return
|
||||
}
|
||||
}
|
||||
}()
|
||||
select {
|
||||
case <-started:
|
||||
case <-ctx.Done():
|
||||
}
|
||||
return output, nil
|
||||
}
|
||||
@@ -0,0 +1,139 @@
|
||||
package local_test
|
||||
|
||||
import (
|
||||
"context"
|
||||
"io/fs"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"time"
|
||||
|
||||
"github.com/navidrome/navidrome/conf"
|
||||
"github.com/navidrome/navidrome/core/storage"
|
||||
"github.com/navidrome/navidrome/core/storage/local"
|
||||
_ "github.com/navidrome/navidrome/core/storage/local"
|
||||
"github.com/navidrome/navidrome/model/metadata"
|
||||
. "github.com/onsi/ginkgo/v2"
|
||||
. "github.com/onsi/gomega"
|
||||
)
|
||||
|
||||
var _ = XDescribe("Watcher", func() {
|
||||
var lsw storage.Watcher
|
||||
var tmpFolder string
|
||||
|
||||
BeforeEach(func() {
|
||||
tmpFolder = GinkgoT().TempDir()
|
||||
|
||||
local.RegisterExtractor("noop", func(fs fs.FS, path string) local.Extractor { return noopExtractor{} })
|
||||
conf.Server.Scanner.Extractor = "noop"
|
||||
|
||||
ls, err := storage.For(tmpFolder)
|
||||
Expect(err).ToNot(HaveOccurred())
|
||||
|
||||
// It should implement Watcher
|
||||
var ok bool
|
||||
lsw, ok = ls.(storage.Watcher)
|
||||
Expect(ok).To(BeTrue())
|
||||
|
||||
// Make sure temp folder is created
|
||||
Eventually(func() error {
|
||||
_, err := os.Stat(tmpFolder)
|
||||
return err
|
||||
}).Should(Succeed())
|
||||
})
|
||||
|
||||
It("should start and stop watcher", func() {
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
defer cancel()
|
||||
|
||||
w, err := lsw.Start(ctx)
|
||||
Expect(err).ToNot(HaveOccurred())
|
||||
cancel()
|
||||
Eventually(w).Should(BeClosed())
|
||||
})
|
||||
|
||||
It("should return error if watcher is already started", func() {
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
defer cancel()
|
||||
|
||||
_, err := lsw.Start(ctx)
|
||||
Expect(err).ToNot(HaveOccurred())
|
||||
_, err = lsw.Start(ctx)
|
||||
Expect(err).To(HaveOccurred())
|
||||
})
|
||||
|
||||
It("should detect new files", func() {
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
defer cancel()
|
||||
|
||||
changes, err := lsw.Start(ctx)
|
||||
Expect(err).ToNot(HaveOccurred())
|
||||
|
||||
_, err = os.Create(filepath.Join(tmpFolder, "test.txt"))
|
||||
Expect(err).ToNot(HaveOccurred())
|
||||
|
||||
Eventually(changes).WithTimeout(2 * time.Second).Should(Receive(Equal(tmpFolder)))
|
||||
})
|
||||
|
||||
It("should detect new subfolders", func() {
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
defer cancel()
|
||||
|
||||
changes, err := lsw.Start(ctx)
|
||||
Expect(err).ToNot(HaveOccurred())
|
||||
|
||||
Expect(os.Mkdir(filepath.Join(tmpFolder, "subfolder"), 0755)).To(Succeed())
|
||||
|
||||
Eventually(changes).WithTimeout(2 * time.Second).Should(Receive(Equal(filepath.Join(tmpFolder, "subfolder"))))
|
||||
})
|
||||
|
||||
It("should detect changes in subfolders recursively", func() {
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
defer cancel()
|
||||
|
||||
subfolder := filepath.Join(tmpFolder, "subfolder1/subfolder2")
|
||||
Expect(os.MkdirAll(subfolder, 0755)).To(Succeed())
|
||||
|
||||
changes, err := lsw.Start(ctx)
|
||||
Expect(err).ToNot(HaveOccurred())
|
||||
|
||||
filePath := filepath.Join(subfolder, "test.txt")
|
||||
Expect(os.WriteFile(filePath, []byte("test"), 0600)).To(Succeed())
|
||||
|
||||
Eventually(changes).WithTimeout(2 * time.Second).Should(Receive(Equal(filePath)))
|
||||
})
|
||||
|
||||
It("should detect removed in files", func() {
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
defer cancel()
|
||||
changes, err := lsw.Start(ctx)
|
||||
Expect(err).ToNot(HaveOccurred())
|
||||
|
||||
filePath := filepath.Join(tmpFolder, "test.txt")
|
||||
Expect(os.WriteFile(filePath, []byte("test"), 0600)).To(Succeed())
|
||||
|
||||
Eventually(changes).WithTimeout(2 * time.Second).Should(Receive(Equal(filePath)))
|
||||
|
||||
Expect(os.Remove(filePath)).To(Succeed())
|
||||
Eventually(changes).WithTimeout(2 * time.Second).Should(Receive(Equal(filePath)))
|
||||
})
|
||||
|
||||
It("should detect file moves", func() {
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
defer cancel()
|
||||
|
||||
filePath := filepath.Join(tmpFolder, "test.txt")
|
||||
Expect(os.WriteFile(filePath, []byte("test"), 0600)).To(Succeed())
|
||||
|
||||
changes, err := lsw.Start(ctx)
|
||||
Expect(err).ToNot(HaveOccurred())
|
||||
|
||||
newPath := filepath.Join(tmpFolder, "test2.txt")
|
||||
Expect(os.Rename(filePath, newPath)).To(Succeed())
|
||||
Eventually(changes).WithTimeout(2 * time.Second).Should(Receive(Equal(newPath)))
|
||||
})
|
||||
})
|
||||
|
||||
type noopExtractor struct{}
|
||||
|
||||
func (s noopExtractor) Parse(files ...string) (map[string]metadata.Info, error) { return nil, nil }
|
||||
func (s noopExtractor) Version() string { return "0" }
|
||||
@@ -0,0 +1,51 @@
|
||||
package storage
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"net/url"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"sync"
|
||||
)
|
||||
|
||||
const LocalSchemaID = "file"
|
||||
|
||||
type constructor func(url.URL) Storage
|
||||
|
||||
var (
|
||||
registry = map[string]constructor{}
|
||||
lock sync.RWMutex
|
||||
)
|
||||
|
||||
func Register(schema string, c constructor) {
|
||||
lock.Lock()
|
||||
defer lock.Unlock()
|
||||
registry[schema] = c
|
||||
}
|
||||
|
||||
// For returns a Storage implementation for the given URI.
|
||||
// It uses the schema part of the URI to find the correct registered
|
||||
// Storage constructor.
|
||||
// If the URI does not contain a schema, it is treated as a file:// URI.
|
||||
func For(uri string) (Storage, error) {
|
||||
lock.RLock()
|
||||
defer lock.RUnlock()
|
||||
parts := strings.Split(uri, "://")
|
||||
|
||||
// Paths without schema are treated as file:// and use the default LocalStorage implementation
|
||||
if len(parts) < 2 {
|
||||
uri, _ = filepath.Abs(uri)
|
||||
uri = filepath.ToSlash(uri)
|
||||
uri = LocalSchemaID + "://" + uri
|
||||
}
|
||||
|
||||
u, err := url.Parse(uri)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
c, ok := registry[u.Scheme]
|
||||
if !ok {
|
||||
return nil, errors.New("schema '" + u.Scheme + "' not registered")
|
||||
}
|
||||
return c(*u), nil
|
||||
}
|
||||
@@ -0,0 +1,78 @@
|
||||
package storage
|
||||
|
||||
import (
|
||||
"net/url"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"testing"
|
||||
|
||||
. "github.com/onsi/ginkgo/v2"
|
||||
. "github.com/onsi/gomega"
|
||||
)
|
||||
|
||||
func TestApp(t *testing.T) {
|
||||
RegisterFailHandler(Fail)
|
||||
RunSpecs(t, "Storage Test Suite")
|
||||
}
|
||||
|
||||
var _ = Describe("Storage", func() {
|
||||
When("schema is not registered", func() {
|
||||
BeforeEach(func() {
|
||||
registry = map[string]constructor{}
|
||||
})
|
||||
|
||||
It("should return error", func() {
|
||||
_, err := For("file:///tmp")
|
||||
Expect(err).To(HaveOccurred())
|
||||
})
|
||||
})
|
||||
When("schema is registered", func() {
|
||||
BeforeEach(func() {
|
||||
registry = map[string]constructor{}
|
||||
Register("file", func(url url.URL) Storage { return &fakeLocalStorage{u: url} })
|
||||
Register("s3", func(url url.URL) Storage { return &fakeS3Storage{u: url} })
|
||||
})
|
||||
|
||||
It("should return correct implementation", func() {
|
||||
s, err := For("file:///tmp")
|
||||
Expect(err).ToNot(HaveOccurred())
|
||||
Expect(s).To(BeAssignableToTypeOf(&fakeLocalStorage{}))
|
||||
Expect(s.(*fakeLocalStorage).u.Scheme).To(Equal("file"))
|
||||
Expect(s.(*fakeLocalStorage).u.Path).To(Equal("/tmp"))
|
||||
|
||||
s, err = For("s3:///bucket")
|
||||
Expect(err).ToNot(HaveOccurred())
|
||||
Expect(s).To(BeAssignableToTypeOf(&fakeS3Storage{}))
|
||||
Expect(s.(*fakeS3Storage).u.Scheme).To(Equal("s3"))
|
||||
Expect(s.(*fakeS3Storage).u.Path).To(Equal("/bucket"))
|
||||
})
|
||||
It("should return a file implementation when schema is not specified", func() {
|
||||
s, err := For("/tmp")
|
||||
Expect(err).ToNot(HaveOccurred())
|
||||
Expect(s).To(BeAssignableToTypeOf(&fakeLocalStorage{}))
|
||||
Expect(s.(*fakeLocalStorage).u.Scheme).To(Equal("file"))
|
||||
Expect(s.(*fakeLocalStorage).u.Path).To(Equal("/tmp"))
|
||||
})
|
||||
It("should return a file implementation for a relative folder", func() {
|
||||
s, err := For("tmp")
|
||||
Expect(err).ToNot(HaveOccurred())
|
||||
cwd, _ := os.Getwd()
|
||||
Expect(s).To(BeAssignableToTypeOf(&fakeLocalStorage{}))
|
||||
Expect(s.(*fakeLocalStorage).u.Scheme).To(Equal("file"))
|
||||
Expect(s.(*fakeLocalStorage).u.Path).To(Equal(filepath.Join(cwd, "tmp")))
|
||||
})
|
||||
It("should return error if schema is unregistered", func() {
|
||||
_, err := For("webdav:///tmp")
|
||||
Expect(err).To(HaveOccurred())
|
||||
})
|
||||
})
|
||||
})
|
||||
|
||||
type fakeLocalStorage struct {
|
||||
Storage
|
||||
u url.URL
|
||||
}
|
||||
type fakeS3Storage struct {
|
||||
Storage
|
||||
u url.URL
|
||||
}
|
||||
@@ -0,0 +1,323 @@
|
||||
//nolint:unused
|
||||
package storagetest
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io/fs"
|
||||
"net/url"
|
||||
"path"
|
||||
"testing/fstest"
|
||||
"time"
|
||||
|
||||
"github.com/navidrome/navidrome/core/storage"
|
||||
"github.com/navidrome/navidrome/log"
|
||||
"github.com/navidrome/navidrome/model/metadata"
|
||||
"github.com/navidrome/navidrome/utils/random"
|
||||
)
|
||||
|
||||
// FakeStorage is a fake storage that provides a FakeFS.
|
||||
// It is used for testing purposes.
|
||||
type FakeStorage struct{ fs *FakeFS }
|
||||
|
||||
// Register registers the FakeStorage for the given scheme. To use it, set the model.Library's Path to "fake:///music",
|
||||
// and register a FakeFS with schema = "fake". The storage registered will always return the same FakeFS instance.
|
||||
func Register(schema string, fs *FakeFS) {
|
||||
storage.Register(schema, func(url url.URL) storage.Storage { return &FakeStorage{fs: fs} })
|
||||
}
|
||||
|
||||
func (s FakeStorage) FS() (storage.MusicFS, error) {
|
||||
return s.fs, nil
|
||||
}
|
||||
|
||||
// FakeFS is a fake filesystem that can be used for testing purposes.
|
||||
// It implements the storage.MusicFS interface and keeps all files in memory, by using a fstest.MapFS internally.
|
||||
// You must NOT add files directly in the MapFS property, but use SetFiles and its other methods instead.
|
||||
// This is because the FakeFS keeps track of the latest modification time of directories, simulating the
|
||||
// behavior of a real filesystem, and you should not bypass this logic.
|
||||
type FakeFS struct {
|
||||
fstest.MapFS
|
||||
properInit bool
|
||||
}
|
||||
|
||||
func (ffs *FakeFS) SetFiles(files fstest.MapFS) {
|
||||
ffs.properInit = true
|
||||
ffs.MapFS = files
|
||||
ffs.createDirTimestamps()
|
||||
}
|
||||
|
||||
func (ffs *FakeFS) Add(filePath string, file *fstest.MapFile, when ...time.Time) {
|
||||
if len(when) == 0 {
|
||||
when = append(when, time.Now())
|
||||
}
|
||||
ffs.MapFS[filePath] = file
|
||||
ffs.touchContainingFolder(filePath, when[0])
|
||||
ffs.createDirTimestamps()
|
||||
}
|
||||
|
||||
func (ffs *FakeFS) Remove(filePath string, when ...time.Time) *fstest.MapFile {
|
||||
filePath = path.Clean(filePath)
|
||||
if len(when) == 0 {
|
||||
when = append(when, time.Now())
|
||||
}
|
||||
if f, ok := ffs.MapFS[filePath]; ok {
|
||||
ffs.touchContainingFolder(filePath, when[0])
|
||||
delete(ffs.MapFS, filePath)
|
||||
return f
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (ffs *FakeFS) Move(srcPath string, destPath string, when ...time.Time) {
|
||||
if len(when) == 0 {
|
||||
when = append(when, time.Now())
|
||||
}
|
||||
srcPath = path.Clean(srcPath)
|
||||
destPath = path.Clean(destPath)
|
||||
ffs.MapFS[destPath] = ffs.MapFS[srcPath]
|
||||
ffs.touchContainingFolder(destPath, when[0])
|
||||
ffs.Remove(srcPath, when...)
|
||||
}
|
||||
|
||||
// Touch sets the modification time of a file.
|
||||
func (ffs *FakeFS) Touch(filePath string, when ...time.Time) {
|
||||
if len(when) == 0 {
|
||||
when = append(when, time.Now())
|
||||
}
|
||||
filePath = path.Clean(filePath)
|
||||
file, ok := ffs.MapFS[filePath]
|
||||
if ok {
|
||||
file.ModTime = when[0]
|
||||
} else {
|
||||
ffs.MapFS[filePath] = &fstest.MapFile{ModTime: when[0]}
|
||||
}
|
||||
ffs.touchContainingFolder(filePath, file.ModTime)
|
||||
}
|
||||
|
||||
func (ffs *FakeFS) touchContainingFolder(filePath string, ts time.Time) {
|
||||
dir := path.Dir(filePath)
|
||||
dirFile, ok := ffs.MapFS[dir]
|
||||
if !ok {
|
||||
log.Fatal("Directory not found. Forgot to call SetFiles?", "file", filePath)
|
||||
}
|
||||
if dirFile.ModTime.Before(ts) {
|
||||
dirFile.ModTime = ts
|
||||
}
|
||||
}
|
||||
|
||||
// SetError sets an error that will be returned when trying to read the file.
|
||||
func (ffs *FakeFS) SetError(filePath string, err error) {
|
||||
filePath = path.Clean(filePath)
|
||||
if ffs.MapFS[filePath] == nil {
|
||||
ffs.MapFS[filePath] = &fstest.MapFile{Data: []byte{}}
|
||||
}
|
||||
ffs.MapFS[filePath].Sys = err
|
||||
ffs.Touch(filePath)
|
||||
}
|
||||
|
||||
// ClearError clears the error set by SetError.
|
||||
func (ffs *FakeFS) ClearError(filePath string) {
|
||||
filePath = path.Clean(filePath)
|
||||
if file := ffs.MapFS[filePath]; file != nil {
|
||||
file.Sys = nil
|
||||
}
|
||||
ffs.Touch(filePath)
|
||||
}
|
||||
|
||||
func (ffs *FakeFS) UpdateTags(filePath string, newTags map[string]any, when ...time.Time) {
|
||||
f, ok := ffs.MapFS[filePath]
|
||||
if !ok {
|
||||
panic(fmt.Errorf("file %s not found", filePath))
|
||||
}
|
||||
var tags map[string]any
|
||||
err := json.Unmarshal(f.Data, &tags)
|
||||
if err != nil {
|
||||
panic(err)
|
||||
}
|
||||
for k, v := range newTags {
|
||||
tags[k] = v
|
||||
}
|
||||
data, _ := json.Marshal(tags)
|
||||
f.Data = data
|
||||
ffs.Touch(filePath, when...)
|
||||
}
|
||||
|
||||
// createDirTimestamps loops through all entries and create/updates directories entries in the map with the
|
||||
// latest ModTime from any children of that directory.
|
||||
func (ffs *FakeFS) createDirTimestamps() bool {
|
||||
var changed bool
|
||||
for filePath, file := range ffs.MapFS {
|
||||
dir := path.Dir(filePath)
|
||||
dirFile, ok := ffs.MapFS[dir]
|
||||
if !ok {
|
||||
dirFile = &fstest.MapFile{Mode: fs.ModeDir}
|
||||
ffs.MapFS[dir] = dirFile
|
||||
}
|
||||
if dirFile.ModTime.IsZero() {
|
||||
dirFile.ModTime = file.ModTime
|
||||
changed = true
|
||||
}
|
||||
}
|
||||
if changed {
|
||||
// If we updated any directory, we need to re-run the loop to create any parent directories
|
||||
ffs.createDirTimestamps()
|
||||
}
|
||||
return changed
|
||||
}
|
||||
|
||||
func ModTime(ts string) map[string]any { return map[string]any{fakeFileInfoModTime: ts} }
|
||||
func BirthTime(ts string) map[string]any { return map[string]any{fakeFileInfoBirthTime: ts} }
|
||||
|
||||
func Template(t ...map[string]any) func(...map[string]any) *fstest.MapFile {
|
||||
return func(tags ...map[string]any) *fstest.MapFile {
|
||||
return MP3(append(t, tags...)...)
|
||||
}
|
||||
}
|
||||
|
||||
func Track(num int, title string, tags ...map[string]any) map[string]any {
|
||||
ts := audioProperties("mp3", 320)
|
||||
ts["title"] = title
|
||||
ts["track"] = num
|
||||
for _, t := range tags {
|
||||
for k, v := range t {
|
||||
ts[k] = v
|
||||
}
|
||||
}
|
||||
return ts
|
||||
}
|
||||
|
||||
func MP3(tags ...map[string]any) *fstest.MapFile {
|
||||
ts := audioProperties("mp3", 320)
|
||||
if _, ok := ts[fakeFileInfoSize]; !ok {
|
||||
duration := ts["duration"].(int64)
|
||||
bitrate := ts["bitrate"].(int)
|
||||
ts[fakeFileInfoSize] = duration * int64(bitrate) / 8 * 1000
|
||||
}
|
||||
return File(append([]map[string]any{ts}, tags...)...)
|
||||
}
|
||||
|
||||
func File(tags ...map[string]any) *fstest.MapFile {
|
||||
ts := map[string]any{}
|
||||
for _, t := range tags {
|
||||
for k, v := range t {
|
||||
ts[k] = v
|
||||
}
|
||||
}
|
||||
modTime := time.Now()
|
||||
if mt, ok := ts[fakeFileInfoModTime]; !ok {
|
||||
ts[fakeFileInfoModTime] = time.Now().Format(time.RFC3339)
|
||||
} else {
|
||||
modTime, _ = time.Parse(time.RFC3339, mt.(string))
|
||||
}
|
||||
if _, ok := ts[fakeFileInfoBirthTime]; !ok {
|
||||
ts[fakeFileInfoBirthTime] = time.Now().Format(time.RFC3339)
|
||||
}
|
||||
if _, ok := ts[fakeFileInfoMode]; !ok {
|
||||
ts[fakeFileInfoMode] = fs.ModePerm
|
||||
}
|
||||
data, _ := json.Marshal(ts)
|
||||
if _, ok := ts[fakeFileInfoSize]; !ok {
|
||||
ts[fakeFileInfoSize] = int64(len(data))
|
||||
}
|
||||
return &fstest.MapFile{Data: data, ModTime: modTime, Mode: ts[fakeFileInfoMode].(fs.FileMode)}
|
||||
}
|
||||
|
||||
func audioProperties(suffix string, bitrate int) map[string]any {
|
||||
duration := random.Int64N(300) + 120
|
||||
return map[string]any{
|
||||
"suffix": suffix,
|
||||
"bitrate": bitrate,
|
||||
"duration": duration,
|
||||
"samplerate": 44100,
|
||||
"bitdepth": 16,
|
||||
"channels": 2,
|
||||
}
|
||||
}
|
||||
|
||||
func (ffs *FakeFS) ReadTags(paths ...string) (map[string]metadata.Info, error) {
|
||||
if !ffs.properInit {
|
||||
log.Fatal("FakeFS not initialized properly. Use SetFiles")
|
||||
}
|
||||
result := make(map[string]metadata.Info)
|
||||
var errs []error
|
||||
for _, file := range paths {
|
||||
p, err := ffs.parseFile(file)
|
||||
if err != nil {
|
||||
log.Warn("Error reading metadata from file", "file", file, "err", err)
|
||||
errs = append(errs, err)
|
||||
} else {
|
||||
result[file] = *p
|
||||
}
|
||||
}
|
||||
if len(errs) > 0 {
|
||||
return result, fmt.Errorf("errors reading metadata: %w", errors.Join(errs...))
|
||||
}
|
||||
return result, nil
|
||||
}
|
||||
|
||||
func (ffs *FakeFS) parseFile(filePath string) (*metadata.Info, error) {
|
||||
// Check if it should throw an error when reading this file
|
||||
stat, err := ffs.Stat(filePath)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if stat.Sys() != nil {
|
||||
return nil, stat.Sys().(error)
|
||||
}
|
||||
|
||||
// Read the file contents and parse the tags
|
||||
contents, err := fs.ReadFile(ffs, filePath)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
data := map[string]any{}
|
||||
err = json.Unmarshal(contents, &data)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
p := metadata.Info{
|
||||
Tags: map[string][]string{},
|
||||
AudioProperties: metadata.AudioProperties{},
|
||||
HasPicture: data["has_picture"] == "true",
|
||||
}
|
||||
if d, ok := data["duration"].(float64); ok {
|
||||
p.AudioProperties.Duration = time.Duration(d) * time.Second
|
||||
}
|
||||
getInt := func(key string) int { v, _ := data[key].(float64); return int(v) }
|
||||
p.AudioProperties.BitRate = getInt("bitrate")
|
||||
p.AudioProperties.BitDepth = getInt("bitdepth")
|
||||
p.AudioProperties.SampleRate = getInt("samplerate")
|
||||
p.AudioProperties.Channels = getInt("channels")
|
||||
for k, v := range data {
|
||||
p.Tags[k] = []string{fmt.Sprintf("%v", v)}
|
||||
}
|
||||
file := ffs.MapFS[filePath]
|
||||
p.FileInfo = &fakeFileInfo{path: filePath, tags: data, file: file}
|
||||
return &p, nil
|
||||
}
|
||||
|
||||
const (
|
||||
fakeFileInfoMode = "_mode"
|
||||
fakeFileInfoSize = "_size"
|
||||
fakeFileInfoModTime = "_modtime"
|
||||
fakeFileInfoBirthTime = "_birthtime"
|
||||
)
|
||||
|
||||
type fakeFileInfo struct {
|
||||
path string
|
||||
file *fstest.MapFile
|
||||
tags map[string]any
|
||||
}
|
||||
|
||||
func (ffi *fakeFileInfo) Name() string { return path.Base(ffi.path) }
|
||||
func (ffi *fakeFileInfo) Size() int64 { v, _ := ffi.tags[fakeFileInfoSize].(float64); return int64(v) }
|
||||
func (ffi *fakeFileInfo) Mode() fs.FileMode { return ffi.file.Mode }
|
||||
func (ffi *fakeFileInfo) IsDir() bool { return false }
|
||||
func (ffi *fakeFileInfo) Sys() any { return nil }
|
||||
func (ffi *fakeFileInfo) ModTime() time.Time { return ffi.file.ModTime }
|
||||
func (ffi *fakeFileInfo) BirthTime() time.Time { return ffi.parseTime(fakeFileInfoBirthTime) }
|
||||
func (ffi *fakeFileInfo) parseTime(key string) time.Time {
|
||||
t, _ := time.Parse(time.RFC3339, ffi.tags[key].(string))
|
||||
return t
|
||||
}
|
||||
@@ -0,0 +1,139 @@
|
||||
//nolint:unused
|
||||
package storagetest_test
|
||||
|
||||
import (
|
||||
"io/fs"
|
||||
"testing"
|
||||
"testing/fstest"
|
||||
"time"
|
||||
|
||||
. "github.com/navidrome/navidrome/core/storage/storagetest"
|
||||
. "github.com/onsi/ginkgo/v2"
|
||||
. "github.com/onsi/gomega"
|
||||
)
|
||||
|
||||
type _t = map[string]any
|
||||
|
||||
func TestFakeStorage(t *testing.T) {
|
||||
RegisterFailHandler(Fail)
|
||||
RunSpecs(t, "Fake Storage Test Suite")
|
||||
}
|
||||
|
||||
var _ = Describe("FakeFS", func() {
|
||||
var ffs FakeFS
|
||||
var startTime time.Time
|
||||
|
||||
BeforeEach(func() {
|
||||
startTime = time.Now().Add(-time.Hour)
|
||||
boy := Template(_t{"albumartist": "U2", "album": "Boy", "year": 1980, "genre": "Rock"})
|
||||
files := fstest.MapFS{
|
||||
"U2/Boy/I Will Follow.mp3": boy(Track(1, "I Will Follow")),
|
||||
"U2/Boy/Twilight.mp3": boy(Track(2, "Twilight")),
|
||||
"U2/Boy/An Cat Dubh.mp3": boy(Track(3, "An Cat Dubh")),
|
||||
}
|
||||
ffs.SetFiles(files)
|
||||
})
|
||||
|
||||
It("should implement a fs.FS", func() {
|
||||
Expect(fstest.TestFS(ffs, "U2/Boy/I Will Follow.mp3")).To(Succeed())
|
||||
})
|
||||
|
||||
It("should read file info", func() {
|
||||
props, err := ffs.ReadTags("U2/Boy/I Will Follow.mp3", "U2/Boy/Twilight.mp3")
|
||||
Expect(err).ToNot(HaveOccurred())
|
||||
|
||||
prop := props["U2/Boy/Twilight.mp3"]
|
||||
Expect(prop).ToNot(BeNil())
|
||||
Expect(prop.AudioProperties.Channels).To(Equal(2))
|
||||
Expect(prop.AudioProperties.BitRate).To(Equal(320))
|
||||
Expect(prop.FileInfo.Name()).To(Equal("Twilight.mp3"))
|
||||
Expect(prop.Tags["albumartist"]).To(ConsistOf("U2"))
|
||||
Expect(prop.FileInfo.ModTime()).To(BeTemporally(">=", startTime))
|
||||
|
||||
prop = props["U2/Boy/I Will Follow.mp3"]
|
||||
Expect(prop).ToNot(BeNil())
|
||||
Expect(prop.FileInfo.Name()).To(Equal("I Will Follow.mp3"))
|
||||
})
|
||||
|
||||
It("should return ModTime for directories", func() {
|
||||
root := ffs.MapFS["."]
|
||||
dirInfo1, err := ffs.Stat("U2")
|
||||
Expect(err).ToNot(HaveOccurred())
|
||||
dirInfo2, err := ffs.Stat("U2/Boy")
|
||||
Expect(err).ToNot(HaveOccurred())
|
||||
Expect(dirInfo1.ModTime()).To(Equal(root.ModTime))
|
||||
Expect(dirInfo1.ModTime()).To(BeTemporally(">=", startTime))
|
||||
Expect(dirInfo1.ModTime()).To(Equal(dirInfo2.ModTime()))
|
||||
})
|
||||
|
||||
When("the file is touched", func() {
|
||||
It("should only update the file and the file's directory ModTime", func() {
|
||||
root, _ := ffs.Stat(".")
|
||||
u2Dir, _ := ffs.Stat("U2")
|
||||
boyDir, _ := ffs.Stat("U2/Boy")
|
||||
previousTime := root.ModTime()
|
||||
|
||||
aTimeStamp := previousTime.Add(time.Hour)
|
||||
ffs.Touch("U2/./Boy/Twilight.mp3", aTimeStamp)
|
||||
|
||||
twilightFile, err := ffs.Stat("U2/Boy/Twilight.mp3")
|
||||
Expect(err).ToNot(HaveOccurred())
|
||||
Expect(twilightFile.ModTime()).To(Equal(aTimeStamp))
|
||||
|
||||
Expect(root.ModTime()).To(Equal(previousTime))
|
||||
Expect(u2Dir.ModTime()).To(Equal(previousTime))
|
||||
Expect(boyDir.ModTime()).To(Equal(aTimeStamp))
|
||||
})
|
||||
})
|
||||
|
||||
When("adding/removing files", func() {
|
||||
It("should keep the timestamps correct", func() {
|
||||
root, _ := ffs.Stat(".")
|
||||
u2Dir, _ := ffs.Stat("U2")
|
||||
boyDir, _ := ffs.Stat("U2/Boy")
|
||||
previousTime := root.ModTime()
|
||||
aTimeStamp := previousTime.Add(time.Hour)
|
||||
|
||||
ffs.Add("U2/Boy/../Boy/Another.mp3", &fstest.MapFile{ModTime: aTimeStamp}, aTimeStamp)
|
||||
Expect(u2Dir.ModTime()).To(Equal(previousTime))
|
||||
Expect(boyDir.ModTime()).To(Equal(aTimeStamp))
|
||||
|
||||
aTimeStamp = aTimeStamp.Add(time.Hour)
|
||||
ffs.Remove("U2/./Boy/Twilight.mp3", aTimeStamp)
|
||||
|
||||
_, err := ffs.Stat("U2/Boy/Twilight.mp3")
|
||||
Expect(err).To(MatchError(fs.ErrNotExist))
|
||||
Expect(u2Dir.ModTime()).To(Equal(previousTime))
|
||||
Expect(boyDir.ModTime()).To(Equal(aTimeStamp))
|
||||
})
|
||||
})
|
||||
|
||||
When("moving files", func() {
|
||||
It("should allow relative paths", func() {
|
||||
ffs.Move("U2/../U2/Boy/Twilight.mp3", "./Twilight.mp3")
|
||||
Expect(ffs.MapFS).To(HaveKey("Twilight.mp3"))
|
||||
file, err := ffs.Stat("Twilight.mp3")
|
||||
Expect(err).ToNot(HaveOccurred())
|
||||
Expect(file.Name()).To(Equal("Twilight.mp3"))
|
||||
})
|
||||
It("should keep the timestamps correct", func() {
|
||||
root, _ := ffs.Stat(".")
|
||||
u2Dir, _ := ffs.Stat("U2")
|
||||
boyDir, _ := ffs.Stat("U2/Boy")
|
||||
previousTime := root.ModTime()
|
||||
twilightFile, _ := ffs.Stat("U2/Boy/Twilight.mp3")
|
||||
filePreviousTime := twilightFile.ModTime()
|
||||
aTimeStamp := previousTime.Add(time.Hour)
|
||||
|
||||
ffs.Move("U2/Boy/Twilight.mp3", "Twilight.mp3", aTimeStamp)
|
||||
|
||||
Expect(root.ModTime()).To(Equal(aTimeStamp))
|
||||
Expect(u2Dir.ModTime()).To(Equal(previousTime))
|
||||
Expect(boyDir.ModTime()).To(Equal(aTimeStamp))
|
||||
|
||||
Expect(ffs.MapFS).ToNot(HaveKey("U2/Boy/Twilight.mp3"))
|
||||
twilight := ffs.MapFS["Twilight.mp3"]
|
||||
Expect(twilight.ModTime).To(Equal(filePreviousTime))
|
||||
})
|
||||
})
|
||||
})
|
||||
Reference in New Issue
Block a user