feat(bfr): Big Refactor: new scanner, lots of new fields and tags, improvements and DB schema changes (#2709)

* fix(server): more race conditions when updating artist/album from external sources Signed-off-by: Deluan <deluan@navidrome.org> * feat(scanner): add .gitignore syntax to .ndignore. Resolves #1394 Signed-off-by: Deluan <deluan@navidrome.org> * fix(ui): null Signed-off-by: Deluan <deluan@navidrome.org> * fix(scanner): pass configfile option to child process Signed-off-by: Deluan <deluan@navidrome.org> * fix(scanner): resume interrupted fullScans Signed-off-by: Deluan <deluan@navidrome.org> * fix(scanner): remove old scanner code Signed-off-by: Deluan <deluan@navidrome.org> * fix(scanner): rename old metadata package Signed-off-by: Deluan <deluan@navidrome.org> * fix(scanner): move old metadata package Signed-off-by: Deluan <deluan@navidrome.org> * fix: tests Signed-off-by: Deluan <deluan@navidrome.org> * chore(deps): update Go to 1.23.4 Signed-off-by: Deluan <deluan@navidrome.org> * fix: logs Signed-off-by: Deluan <deluan@navidrome.org> * fix(test): Signed-off-by: Deluan <deluan@navidrome.org> * fix: log level Signed-off-by: Deluan <deluan@navidrome.org> * fix: remove log message Signed-off-by: Deluan <deluan@navidrome.org> * feat: add config for scanner watcher Signed-off-by: Deluan <deluan@navidrome.org> * refactor: children playlists Signed-off-by: Deluan <deluan@navidrome.org> * refactor: replace `interface{}` with `any` Signed-off-by: Deluan <deluan@navidrome.org> * fix: smart playlists with genres Signed-off-by: Deluan <deluan@navidrome.org> * fix: allow any tags in smart playlists Signed-off-by: Deluan <deluan@navidrome.org> * fix: artist names in playlists Signed-off-by: Deluan <deluan@navidrome.org> * fix: smart playlist's sort by tags Signed-off-by: Deluan <deluan@navidrome.org> * feat(subsonic): add moods to child Signed-off-by: Deluan <deluan@navidrome.org> * feat(subsonic): add moods to AlbumID3 Signed-off-by: Deluan <deluan@navidrome.org> * refactor(subsonic): use generic JSONArray for OS arrays Signed-off-by: Deluan <deluan@navidrome.org> * refactor(subsonic): use https in test Signed-off-by: Deluan <deluan@navidrome.org> * feat(subsonic): add releaseTypes to AlbumID3 Signed-off-by: Deluan <deluan@navidrome.org> * feat(subsonic): add recordLabels to AlbumID3 Signed-off-by: Deluan <deluan@navidrome.org> * refactor(subsonic): rename JSONArray to Array Signed-off-by: Deluan <deluan@navidrome.org> * feat(subsonic): add artists to AlbumID3 Signed-off-by: Deluan <deluan@navidrome.org> * feat(subsonic): add artists to Child Signed-off-by: Deluan <deluan@navidrome.org> * fix(scanner): do not pre-populate smart playlists Signed-off-by: Deluan <deluan@navidrome.org> * feat(subsonic): implement a simplified version of ArtistID3. See https://github.com/opensubsonic/open-subsonic-api/discussions/120 Signed-off-by: Deluan <deluan@navidrome.org> * feat(subsonic): add artists to album child Signed-off-by: Deluan <deluan@navidrome.org> * feat(subsonic): add contributors to mediafile Child Signed-off-by: Deluan <deluan@navidrome.org> * feat(subsonic): add albumArtists to mediafile Child Signed-off-by: Deluan <deluan@navidrome.org> * feat(subsonic): add displayArtist and displayAlbumArtist Signed-off-by: Deluan <deluan@navidrome.org> * feat(subsonic): add displayComposer to Child Signed-off-by: Deluan <deluan@navidrome.org> * feat(subsonic): add roles to ArtistID3 Signed-off-by: Deluan <deluan@navidrome.org> * fix(subsonic): use " • " separator for displayComposer Signed-off-by: Deluan <deluan@navidrome.org> * refactor: Signed-off-by: Deluan <deluan@navidrome.org> * fix(subsonic): Signed-off-by: Deluan <deluan@navidrome.org> * fix(subsonic): respect `PreferSortTags` config option Signed-off-by: Deluan <deluan@navidrome.org> * refactor(subsonic): Signed-off-by: Deluan <deluan@navidrome.org> * refactor: optimize purging non-unused tags Signed-off-by: Deluan <deluan@navidrome.org> * refactor: don't run 'refresh artist stats' concurrently with other transactions Signed-off-by: Deluan <deluan@navidrome.org> * refactor: Signed-off-by: Deluan <deluan@navidrome.org> * fix: log message Signed-off-by: Deluan <deluan@navidrome.org> * feat: add Scanner.ScanOnStartup config option, default true Signed-off-by: Deluan <deluan@navidrome.org> * feat: better json parsing error msg when importing NSPs Signed-off-by: Deluan <deluan@navidrome.org> * fix: don't update album's imported_time when updating external_metadata Signed-off-by: Deluan <deluan@navidrome.org> * fix: handle interrupted scans and full scans after migrations Signed-off-by: Deluan <deluan@navidrome.org> * feat: run `analyze` when migration requires a full rescan Signed-off-by: Deluan <deluan@navidrome.org> * feat: run `PRAGMA optimize` at the end of the scan Signed-off-by: Deluan <deluan@navidrome.org> * fix: don't update artist's updated_at when updating external_metadata Signed-off-by: Deluan <deluan@navidrome.org> * feat: handle multiple artists and roles in smart playlists Signed-off-by: Deluan <deluan@navidrome.org> * feat(ui): dim missing tracks Signed-off-by: Deluan <deluan@navidrome.org> * fix: album missing logic Signed-off-by: Deluan <deluan@navidrome.org> * fix: error encoding in gob Signed-off-by: Deluan <deluan@navidrome.org> * feat: separate warnings from errors Signed-off-by: Deluan <deluan@navidrome.org> * fix: mark albums as missing if they were contained in a deleted folder Signed-off-by: Deluan <deluan@navidrome.org> * refactor: add participant names to media_file and album tables Signed-off-by: Deluan <deluan@navidrome.org> * refactor: use participations in criteria, instead of m2m relationship Signed-off-by: Deluan <deluan@navidrome.org> * refactor: rename participations to participants Signed-off-by: Deluan <deluan@navidrome.org> * feat(subsonic): add moods to album child Signed-off-by: Deluan <deluan@navidrome.org> * fix: albumartist role case Signed-off-by: Deluan <deluan@navidrome.org> * feat(scanner): run scanner as an external process by default Signed-off-by: Deluan <deluan@navidrome.org> * fix(ui): show albumArtist names Signed-off-by: Deluan <deluan@navidrome.org> * fix(ui): dim out missing albums Signed-off-by: Deluan <deluan@navidrome.org> * fix: flaky test Signed-off-by: Deluan <deluan@navidrome.org> * fix(server): scrobble buffer mapping. fix #3583 Signed-off-by: Deluan <deluan@navidrome.org> * refactor: more participations renaming Signed-off-by: Deluan <deluan@navidrome.org> * fix: listenbrainz scrobbling Signed-off-by: Deluan <deluan@navidrome.org> * feat: send release_group_mbid to listenbrainz Signed-off-by: Deluan <deluan@navidrome.org> * feat(subsonic): implement OpenSubsonic explicitStatus field (#3597) * feat: implement OpenSubsonic explicitStatus field * fix(subsonic): fix failing snapshot tests * refactor: create helper for setting explicitStatus * fix: store smaller values for explicit-status on database * test: ToAlbum explicitStatus * refactor: rename explicitStatus helper function --------- Co-authored-by: Deluan Quintão <deluan@navidrome.org> * fix: handle album and track tags in the DB based on the mappings.yaml file Signed-off-by: Deluan <deluan@navidrome.org> * save similar artists as JSONB Signed-off-by: Deluan <deluan@navidrome.org> * fix: getAlbumList byGenre Signed-off-by: Deluan <deluan@navidrome.org> * detect changes in PID configuration Signed-off-by: Deluan <deluan@navidrome.org> * set default album PID to legacy_pid Signed-off-by: Deluan <deluan@navidrome.org> * fix tests Signed-off-by: Deluan <deluan@navidrome.org> * fix SIGSEGV Signed-off-by: Deluan <deluan@navidrome.org> * fix: don't lose album stars/ratings when migrating Signed-off-by: Deluan <deluan@navidrome.org> * store full PID conf in properties Signed-off-by: Deluan <deluan@navidrome.org> * fix: keep album annotations when changing PID.Album config Signed-off-by: Deluan <deluan@navidrome.org> * fix: reassign album annotations Signed-off-by: Deluan <deluan@navidrome.org> * feat: use (display) albumArtist and add links to each artist Signed-off-by: Deluan <deluan@navidrome.org> * fix: not showing albums by albumartist Signed-off-by: Deluan <deluan@navidrome.org> * fix: error msgs Signed-off-by: Deluan <deluan@navidrome.org> * fix: hide PID from Native API Signed-off-by: Deluan <deluan@navidrome.org> * fix: album cover art resolution Signed-off-by: Deluan <deluan@navidrome.org> * fix: trim participant names Signed-off-by: Deluan <deluan@navidrome.org> * fix: reduce watcher log spam Signed-off-by: Deluan <deluan@navidrome.org> * fix: panic when initializing the watcher Signed-off-by: Deluan <deluan@navidrome.org> * fix: various artists Signed-off-by: Deluan <deluan@navidrome.org> * fix: don't store empty lyrics in the DB Signed-off-by: Deluan <deluan@navidrome.org> * remove unused methods Signed-off-by: Deluan <deluan@navidrome.org> * drop full_text indexes, as they are not being used by SQLite Signed-off-by: Deluan <deluan@navidrome.org> * keep album created_at when upgrading Signed-off-by: Deluan <deluan@navidrome.org> * fix(ui): null pointer Signed-off-by: Deluan <deluan@navidrome.org> * fix: album artwork cache Signed-off-by: Deluan <deluan@navidrome.org> * fix: don't expose missing files in Subsonic API Signed-off-by: Deluan <deluan@navidrome.org> * refactor: searchable interface Signed-off-by: Deluan <deluan@navidrome.org> * fix: filter out missing items from subsonic search * fix: filter out missing items from playlists * fix: filter out missing items from shares Signed-off-by: Deluan <deluan@navidrome.org> * feat(ui): add filter by artist role Signed-off-by: Deluan <deluan@navidrome.org> * feat(subsonic): only return albumartists in getIndexes and getArtists endpoints Signed-off-by: Deluan <deluan@navidrome.org> * sort roles alphabetically Signed-off-by: Deluan <deluan@navidrome.org> * fix: artist playcounts Signed-off-by: Deluan <deluan@navidrome.org> * change default Album PID conf Signed-off-by: Deluan <deluan@navidrome.org> * fix albumartist link when it does not match any albumartists values Signed-off-by: Deluan <deluan@navidrome.org> * fix `Ignoring filter not whitelisted` (role) message Signed-off-by: Deluan <deluan@navidrome.org> * fix: trim any names/titles being imported Signed-off-by: Deluan <deluan@navidrome.org> * remove unused genre code Signed-off-by: Deluan <deluan@navidrome.org> * serialize calls to Last.fm's getArtist Signed-off-by: Deluan <deluan@navidrome.org> xxx Signed-off-by: Deluan <deluan@navidrome.org> * add counters to genres Signed-off-by: Deluan <deluan@navidrome.org> * nit: fix migration `notice` message Signed-off-by: Deluan <deluan@navidrome.org> * optimize similar artists query Signed-off-by: Deluan <deluan@navidrome.org> * fix: last.fm.getInfo when mbid does not exist Signed-off-by: Deluan <deluan@navidrome.org> * ui only show missing items for admins Signed-off-by: Deluan <deluan@navidrome.org> * don't allow interaction with missing items Signed-off-by: Deluan <deluan@navidrome.org> * Add Missing Files view (WIP) Signed-off-by: Deluan <deluan@navidrome.org> * refactor: merged tag_counts into tag table Signed-off-by: Deluan <deluan@navidrome.org> * add option to completely disable automatic scanner Signed-off-by: Deluan <deluan@navidrome.org> * add delete missing files functionality Signed-off-by: Deluan <deluan@navidrome.org> * fix: playlists not showing for regular users Signed-off-by: Deluan <deluan@navidrome.org> * reduce updateLastAccess frequency to once every minute Signed-off-by: Deluan <deluan@navidrome.org> * reduce update player frequency to once every minute Signed-off-by: Deluan <deluan@navidrome.org> * add timeout when updating player Signed-off-by: Deluan <deluan@navidrome.org> * remove dead code Signed-off-by: Deluan <deluan@navidrome.org> * fix duplicated roles in stats Signed-off-by: Deluan <deluan@navidrome.org> * add `; ` to artist splitters Signed-off-by: Deluan <deluan@navidrome.org> * fix stats query Signed-off-by: Deluan <deluan@navidrome.org> * more logs Signed-off-by: Deluan <deluan@navidrome.org> * fix: support legacy clients (DSub) by removing OpenSubsonic extra fields - WIP Signed-off-by: Deluan <deluan@navidrome.org> * fix: support legacy clients (DSub) by removing OpenSubsonic extra fields - WIP Signed-off-by: Deluan <deluan@navidrome.org> * fix: support legacy clients (DSub) by removing OpenSubsonic extra fields - WIP Signed-off-by: Deluan <deluan@navidrome.org> * fix: support legacy clients (DSub) by removing OpenSubsonic extra fields - WIP Signed-off-by: Deluan <deluan@navidrome.org> * add record label filter Signed-off-by: Deluan <deluan@navidrome.org> * add release type filter Signed-off-by: Deluan <deluan@navidrome.org> * fix purgeUnused tags Signed-off-by: Deluan <deluan@navidrome.org> * add grouping filter to albums Signed-off-by: Deluan <deluan@navidrome.org> * allow any album tags to be used in as filters in the API Signed-off-by: Deluan <deluan@navidrome.org> * remove empty tags from album info Signed-off-by: Deluan <deluan@navidrome.org> * comments in the migration Signed-off-by: Deluan <deluan@navidrome.org> * fix: Cannot read properties of undefined Signed-off-by: Deluan <deluan@navidrome.org> * fix: listenbrainz scrobbling (#3640) Signed-off-by: Deluan <deluan@navidrome.org> * fix: remove duplicated tag values Signed-off-by: Deluan <deluan@navidrome.org> * fix: don't ignore the taglib folder! Signed-off-by: Deluan <deluan@navidrome.org> * feat: show track subtitle tag Signed-off-by: Deluan <deluan@navidrome.org> * fix: show artists stats based on selected role Signed-off-by: Deluan <deluan@navidrome.org> * fix: inspect Signed-off-by: Deluan <deluan@navidrome.org> * add media type to album info/filters Signed-off-by: Deluan <deluan@navidrome.org> * fix: change format of subtitle in the UI Signed-off-by: Deluan <deluan@navidrome.org> * fix: subtitle in Subsonic API and search Signed-off-by: Deluan <deluan@navidrome.org> * fix: subtitle in UI's player Signed-off-by: Deluan <deluan@navidrome.org> * fix: split strings should be case-insensitive Signed-off-by: Deluan <deluan@navidrome.org> * disable ScanSchedule Signed-off-by: Deluan <deluan@navidrome.org> * increase default sessiontimeout Signed-off-by: Deluan <deluan@navidrome.org> * add sqlite command line tool to docker image Signed-off-by: Deluan <deluan@navidrome.org> * fix: resources override Signed-off-by: Deluan <deluan@navidrome.org> * fix: album PID conf Signed-off-by: Deluan <deluan@navidrome.org> * change migration to mark current artists as albumArtists Signed-off-by: Deluan <deluan@navidrome.org> * feat(ui): Allow filtering on multiple genres (#3679) * feat(ui): Allow filtering on multiple genres Signed-off-by: Henrik Nordvik <henrikno@gmail.com> Signed-off-by: Deluan <deluan@navidrome.org> * add multi-genre filter in Album list Signed-off-by: Deluan <deluan@navidrome.org> --------- Signed-off-by: Henrik Nordvik <henrikno@gmail.com> Signed-off-by: Deluan <deluan@navidrome.org> Co-authored-by: Henrik Nordvik <henrikno@gmail.com> * add more multi-valued tag filters to Album and Song views Signed-off-by: Deluan <deluan@navidrome.org> * fix(ui): unselect missing files after removing Signed-off-by: Deluan <deluan@navidrome.org> * fix(ui): song filter Signed-off-by: Deluan <deluan@navidrome.org> * fix sharing tracks. fix #3687 Signed-off-by: Deluan <deluan@navidrome.org> * use rowids when using search for sync (ex: Symfonium) Signed-off-by: Deluan <deluan@navidrome.org> * fix "Report Real Paths" option for subsonic clients Signed-off-by: Deluan <deluan@navidrome.org> * fix "Report Real Paths" option for subsonic clients for search Signed-off-by: Deluan <deluan@navidrome.org> * add libraryPath to Native API /songs endpoint Signed-off-by: Deluan <deluan@navidrome.org> * feat(subsonic): add album version Signed-off-by: Deluan <deluan@navidrome.org> * made all tags lowercase as they are case-insensitive anyways. Signed-off-by: Deluan <deluan@navidrome.org> * feat(ui): Show full paths, extended properties for album/song (#3691) * feat(ui): Show full paths, extended properties for album/song - uses library path + os separator + path - show participants (album/song) and tags (song) - make album/participant clickable in show info * add source to path * fix pathSeparator in UI Signed-off-by: Deluan <deluan@navidrome.org> * fix local artist artwork (#3695) Signed-off-by: Deluan <deluan@navidrome.org> * fix: parse vorbis performers Signed-off-by: Deluan <deluan@navidrome.org> * refactor: clean function into smaller functions Signed-off-by: Deluan <deluan@navidrome.org> * fix translations for en and pt Signed-off-by: Deluan <deluan@navidrome.org> * add trace log to show annotations reassignment Signed-off-by: Deluan <deluan@navidrome.org> * add trace log to show annotations reassignment Signed-off-by: Deluan <deluan@navidrome.org> * fix: allow performers without instrument/subrole Signed-off-by: Deluan <deluan@navidrome.org> * refactor: metadata clean function again Signed-off-by: Deluan <deluan@navidrome.org> * refactor: optimize split function Signed-off-by: Deluan <deluan@navidrome.org> * refactor: split function is now a method of TagConf Signed-off-by: Deluan <deluan@navidrome.org> * fix: humanize Artist total size Signed-off-by: Deluan <deluan@navidrome.org> * add album version to album details Signed-off-by: Deluan <deluan@navidrome.org> * don't display album-level tags in SongInfo Signed-off-by: Deluan <deluan@navidrome.org> * fix genre clicking in Album Page Signed-off-by: Deluan <deluan@navidrome.org> * don't use mbids in Last.fm api calls. From https://discord.com/channels/671335427726114836/704303730660737113/1337574018143879248: With MBID: ``` GET https://ws.audioscrobbler.com/2.0/?api_key=XXXX&artist=Van+Morrison&format=json&lang=en&mbid=a41ac10f-0a56-4672-9161-b83f9b223559&method=artist.getInfo { artist: { name: "Bee Gees", mbid: "bf0f7e29-dfe1-416c-b5c6-f9ebc19ea810", url: "https://www.last.fm/music/Bee+Gees", } ``` Without MBID: ``` GET https://ws.audioscrobbler.com/2.0/?api_key=XXXX&artist=Van+Morrison&format=json&lang=en&method=artist.getInfo { artist: { name: "Van Morrison", mbid: "a41ac10f-0a56-4672-9161-b83f9b223559", url: "https://www.last.fm/music/Van+Morrison", } ``` Signed-off-by: Deluan <deluan@navidrome.org> * better logging for when the artist folder is not found Signed-off-by: Deluan <deluan@navidrome.org> * fix various issues with artist image resolution Signed-off-by: Deluan <deluan@navidrome.org> * hide "Additional Tags" header if there are none. Signed-off-by: Deluan <deluan@navidrome.org> * simplify tag rendering Signed-off-by: Deluan <deluan@navidrome.org> * enhance logging for artist folder detection Signed-off-by: Deluan <deluan@navidrome.org> * make folderID consistent for relative and absolute folderPaths Signed-off-by: Deluan <deluan@navidrome.org> * handle more folder paths scenarios Signed-off-by: Deluan <deluan@navidrome.org> * filter out other roles when SubsonicArtistParticipations = true Signed-off-by: Deluan <deluan@navidrome.org> * fix "Cannot read properties of undefined" Signed-off-by: Deluan <deluan@navidrome.org> * fix lyrics and comments being truncated (#3701) * fix lyrics and comments being truncated * specifically test for lyrics and comment length * reorder assertions Signed-off-by: Deluan <deluan@navidrome.org> --------- Signed-off-by: Deluan <deluan@navidrome.org> Co-authored-by: Deluan <deluan@navidrome.org> * fix(server): Expose library_path for playlist (#3705) Allows showing absolute path for UI, and makes "report real path" work for playlists (Subsonic) * fix BFR on Windows (#3704) * fix potential reflected cross-site scripting vulnerability Signed-off-by: Deluan <deluan@navidrome.org> * hack to make it work on Windows * ignore windows executables * try fixing the pipeline Signed-off-by: Deluan <deluan@navidrome.org> * allow MusicFolder in other drives * move windows local drive logic to local storage implementation --------- Signed-off-by: Deluan <deluan@navidrome.org> * increase pagination sizes for missing files Signed-off-by: Deluan <deluan@navidrome.org> * reduce level of "already scanning" watcher log message Signed-off-by: Deluan <deluan@navidrome.org> * only count folders with audio files in it See https://github.com/navidrome/navidrome/discussions/3676#discussioncomment-11990930 Signed-off-by: Deluan <deluan@navidrome.org> * add album version and catalog number to search Signed-off-by: Deluan <deluan@navidrome.org> * add `organization` alias for `recordlabel` Signed-off-by: Deluan <deluan@navidrome.org> * remove mbid from Last.fm agent Signed-off-by: Deluan <deluan@navidrome.org> * feat: support inspect in ui (#3726) * inspect in ui * address round 1 * add catalogNum to AlbumInfo Signed-off-by: Deluan <deluan@navidrome.org> * remove dependency on metadata_old (deprecated) package Signed-off-by: Deluan <deluan@navidrome.org> * add `RawTags` to model Signed-off-by: Deluan <deluan@navidrome.org> * support parsing MBIDs for roles (from the https://github.com/kgarner7/picard-all-mbids plugin) (#3698) * parse standard roles, vorbis/m4a work for now * fix djmixer * working roles, use DJ-mix * add performers to file * map mbids * add a few more tests * add test Signed-off-by: Deluan <deluan@navidrome.org> * try to simplify the performers logic Signed-off-by: Deluan <deluan@navidrome.org> * stylistic changes --------- Signed-off-by: Deluan <deluan@navidrome.org> Co-authored-by: Deluan <deluan@navidrome.org> * remove param mutation Signed-off-by: Deluan <deluan@navidrome.org> * run automated SQLite optimizations Signed-off-by: Deluan <deluan@navidrome.org> * fix playlists import/export on Windows * fix import playlists * fix export playlists * better handling of Windows volumes Signed-off-by: Deluan <deluan@navidrome.org> * handle more album ID reassignments Signed-off-by: Deluan <deluan@navidrome.org> * allow adding/overriding tags in the config file Signed-off-by: Deluan <deluan@navidrome.org> * fix(ui): Fix playlist track id, handle missing tracks better (#3734) - Use `mediaFileId` instead of `id` for playlist tracks - Only fetch if the file is not missing - If extractor fails to get the file, also error (rather than panic) * optimize DB after each scan. Signed-off-by: Deluan <deluan@navidrome.org> * remove sortable from AlbumSongs columns Signed-off-by: Deluan <deluan@navidrome.org> * simplify query to get missing tracks Signed-off-by: Deluan <deluan@navidrome.org> * mark Scanner.Extractor as deprecated Signed-off-by: Deluan <deluan@navidrome.org> --------- Signed-off-by: Deluan <deluan@navidrome.org> Signed-off-by: Henrik Nordvik <henrikno@gmail.com> Co-authored-by: Caio Cotts <caio@cotts.com.br> Co-authored-by: Henrik Nordvik <henrikno@gmail.com> Co-authored-by: Kendall Garner <17521368+kgarner7@users.noreply.github.com>
2025-02-19 17:35:17 -08:00
parent 46a963a02a
commit c795bcfcf7
329 changed files with 16586 additions and 5852 deletions
@@ -1,4 +1,4 @@
-package db
+package db_test

 import (
 	"context"
@@ -9,6 +9,8 @@ import (

 	"github.com/navidrome/navidrome/conf"
 	"github.com/navidrome/navidrome/conf/configtest"
+	. "github.com/navidrome/navidrome/db"
+	"github.com/navidrome/navidrome/tests"
 	. "github.com/onsi/ginkgo/v2"
 	. "github.com/onsi/gomega"
 )
@@ -71,7 +73,7 @@ var _ = Describe("database backups", func() {
 			})

 			for _, time := range timesShuffled {
-				path := backupPath(time)
+				path := BackupPath(time)
 				file, err := os.Create(path)
 				Expect(err).ToNot(HaveOccurred())
 				_ = file.Close()
@@ -85,7 +87,7 @@ var _ = Describe("database backups", func() {
 			pruneCount, err := Prune(ctx)
 			Expect(err).ToNot(HaveOccurred())
 			for idx, time := range timesDecreasingChronologically {
-				_, err := os.Stat(backupPath(time))
+				_, err := os.Stat(BackupPath(time))
 				shouldExist := idx < conf.Server.Backup.Count
 				if shouldExist {
 					Expect(err).ToNot(HaveOccurred())
@@ -110,7 +112,7 @@ var _ = Describe("database backups", func() {
 			DeferCleanup(configtest.SetupConfig())

 			conf.Server.DbPath = "file::memory:?cache=shared&_foreign_keys=on"
-			DeferCleanup(Init())
+			DeferCleanup(Init(ctx))
 		})

 		BeforeEach(func() {
@@ -129,25 +131,20 @@ var _ = Describe("database backups", func() {

 			backup, err := sql.Open(Driver, path)
 			Expect(err).ToNot(HaveOccurred())
-			Expect(isSchemaEmpty(backup)).To(BeFalse())
+			Expect(IsSchemaEmpty(ctx, backup)).To(BeFalse())
 		})

 		It("successfully restores the database", func() {
 			path, err := Backup(ctx)
 			Expect(err).ToNot(HaveOccurred())

-			// https://stackoverflow.com/questions/525512/drop-all-tables-command
-			_, err = Db().ExecContext(ctx, `
-PRAGMA writable_schema = 1;
-DELETE FROM sqlite_master WHERE type in ('table', 'index', 'trigger');
-PRAGMA writable_schema = 0;
-			`)
+			err = tests.ClearDB()
 			Expect(err).ToNot(HaveOccurred())
-			Expect(isSchemaEmpty(Db())).To(BeTrue())
+			Expect(IsSchemaEmpty(ctx, Db())).To(BeTrue())

 			err = Restore(ctx, path)
 			Expect(err).ToNot(HaveOccurred())
-			Expect(isSchemaEmpty(Db())).To(BeFalse())
+			Expect(IsSchemaEmpty(ctx, Db())).To(BeFalse())
 		})
 	})
 })
@@ -1,9 +1,11 @@
 package db

 import (
+	"context"
 	"database/sql"
 	"embed"
 	"fmt"
+	"runtime"

 	"github.com/mattn/go-sqlite3"
 	"github.com/navidrome/navidrome/conf"
@@ -32,61 +34,110 @@ func Db() *sql.DB {
 				return conn.RegisterFunc("SEEDEDRAND", hasher.HashFunc(), false)
 			},
 		})
-
 		Path = conf.Server.DbPath
 		if Path == ":memory:" {
 			Path = "file::memory:?cache=shared&_foreign_keys=on"
 			conf.Server.DbPath = Path
 		}
 		log.Debug("Opening DataBase", "dbPath", Path, "driver", Driver)
-		instance, err := sql.Open(Driver, Path)
+		db, err := sql.Open(Driver, Path)
+		db.SetMaxOpenConns(max(4, runtime.NumCPU()))
 		if err != nil {
-			panic(err)
+			log.Fatal("Error opening database", err)
 		}
-		return instance
+		_, err = db.Exec("PRAGMA optimize=0x10002")
+		if err != nil {
+			log.Error("Error applying PRAGMA optimize", err)
+			return nil
+		}
+		return db
 	})
 }

-func Close() {
-	log.Info("Closing Database")
+func Close(ctx context.Context) {
+	// Ignore cancellations when closing the DB
+	ctx = context.WithoutCancel(ctx)
+
+	// Run optimize before closing
+	Optimize(ctx)
+
+	log.Info(ctx, "Closing Database")
 	err := Db().Close()
 	if err != nil {
-		log.Error("Error closing Database", err)
+		log.Error(ctx, "Error closing Database", err)
 	}
 }

-func Init() func() {
+func Init(ctx context.Context) func() {
 	db := Db()

 	// Disable foreign_keys to allow re-creating tables in migrations
-	_, err := db.Exec("PRAGMA foreign_keys=off")
+	_, err := db.ExecContext(ctx, "PRAGMA foreign_keys=off")
 	defer func() {
-		_, err := db.Exec("PRAGMA foreign_keys=on")
+		_, err := db.ExecContext(ctx, "PRAGMA foreign_keys=on")
 		if err != nil {
-			log.Error("Error re-enabling foreign_keys", err)
+			log.Error(ctx, "Error re-enabling foreign_keys", err)
 		}
 	}()
 	if err != nil {
-		log.Error("Error disabling foreign_keys", err)
+		log.Error(ctx, "Error disabling foreign_keys", err)
 	}

-	gooseLogger := &logAdapter{silent: isSchemaEmpty(db)}
 	goose.SetBaseFS(embedMigrations)
-
 	err = goose.SetDialect(Dialect)
 	if err != nil {
-		log.Fatal("Invalid DB driver", "driver", Driver, err)
+		log.Fatal(ctx, "Invalid DB driver", "driver", Driver, err)
 	}
-	if !isSchemaEmpty(db) && hasPendingMigrations(db, migrationsFolder) {
-		log.Info("Upgrading DB Schema to latest version")
+	schemaEmpty := isSchemaEmpty(ctx, db)
+	hasSchemaChanges := hasPendingMigrations(ctx, db, migrationsFolder)
+	if !schemaEmpty && hasSchemaChanges {
+		log.Info(ctx, "Upgrading DB Schema to latest version")
 	}
-	goose.SetLogger(gooseLogger)
-	err = goose.Up(db, migrationsFolder)
+	goose.SetLogger(&logAdapter{ctx: ctx, silent: schemaEmpty})
+	err = goose.UpContext(ctx, db, migrationsFolder)
 	if err != nil {
-		log.Fatal("Failed to apply new migrations", err)
+		log.Fatal(ctx, "Failed to apply new migrations", err)
 	}

-	return Close
+	if hasSchemaChanges {
+		log.Debug(ctx, "Applying PRAGMA optimize after schema changes")
+		_, err = db.ExecContext(ctx, "PRAGMA optimize")
+		if err != nil {
+			log.Error(ctx, "Error applying PRAGMA optimize", err)
+		}
+	}
+
+	return func() {
+		Close(ctx)
+	}
+}
+
+// Optimize runs PRAGMA optimize on each connection in the pool
+func Optimize(ctx context.Context) {
+	numConns := Db().Stats().OpenConnections
+	if numConns == 0 {
+		log.Debug(ctx, "No open connections to optimize")
+		return
+	}
+	log.Debug(ctx, "Optimizing open connections", "numConns", numConns)
+	var conns []*sql.Conn
+	for i := 0; i < numConns; i++ {
+		conn, err := Db().Conn(ctx)
+		conns = append(conns, conn)
+		if err != nil {
+			log.Error(ctx, "Error getting connection from pool", err)
+			continue
+		}
+		_, err = conn.ExecContext(ctx, "PRAGMA optimize;")
+		if err != nil {
+			log.Error(ctx, "Error running PRAGMA optimize", err)
+		}
+	}
+
+	// Return all connections to the Connection Pool
+	for _, conn := range conns {
+		conn.Close()
+	}
 }

 type statusLogger struct{ numPending int }
@@ -103,51 +154,52 @@ func (l *statusLogger) Printf(format string, v ...interface{}) {
 	}
 }

-func hasPendingMigrations(db *sql.DB, folder string) bool {
+func hasPendingMigrations(ctx context.Context, db *sql.DB, folder string) bool {
 	l := &statusLogger{}
 	goose.SetLogger(l)
-	err := goose.Status(db, folder)
+	err := goose.StatusContext(ctx, db, folder)
 	if err != nil {
-		log.Fatal("Failed to check for pending migrations", err)
+		log.Fatal(ctx, "Failed to check for pending migrations", err)
 	}
 	return l.numPending > 0
 }

-func isSchemaEmpty(db *sql.DB) bool {
-	rows, err := db.Query("SELECT name FROM sqlite_master WHERE type='table' AND name='goose_db_version';") // nolint:rowserrcheck
+func isSchemaEmpty(ctx context.Context, db *sql.DB) bool {
+	rows, err := db.QueryContext(ctx, "SELECT name FROM sqlite_master WHERE type='table' AND name='goose_db_version';") // nolint:rowserrcheck
 	if err != nil {
-		log.Fatal("Database could not be opened!", err)
+		log.Fatal(ctx, "Database could not be opened!", err)
 	}
 	defer rows.Close()
 	return !rows.Next()
 }

 type logAdapter struct {
+	ctx    context.Context
 	silent bool
 }

 func (l *logAdapter) Fatal(v ...interface{}) {
-	log.Fatal(fmt.Sprint(v...))
+	log.Fatal(l.ctx, fmt.Sprint(v...))
 }

 func (l *logAdapter) Fatalf(format string, v ...interface{}) {
-	log.Fatal(fmt.Sprintf(format, v...))
+	log.Fatal(l.ctx, fmt.Sprintf(format, v...))
 }

 func (l *logAdapter) Print(v ...interface{}) {
 	if !l.silent {
-		log.Info(fmt.Sprint(v...))
+		log.Info(l.ctx, fmt.Sprint(v...))
 	}
 }

 func (l *logAdapter) Println(v ...interface{}) {
 	if !l.silent {
-		log.Info(fmt.Sprintln(v...))
+		log.Info(l.ctx, fmt.Sprintln(v...))
 	}
 }

 func (l *logAdapter) Printf(format string, v ...interface{}) {
 	if !l.silent {
-		log.Info(fmt.Sprintf(format, v...))
+		log.Info(l.ctx, fmt.Sprintf(format, v...))
 	}
 }
@@ -1,9 +1,11 @@
-package db
+package db_test

 import (
+	"context"
 	"database/sql"
 	"testing"

+	"github.com/navidrome/navidrome/db"
 	"github.com/navidrome/navidrome/log"
 	"github.com/navidrome/navidrome/tests"
 	. "github.com/onsi/ginkgo/v2"
@@ -17,20 +19,22 @@ func TestDB(t *testing.T) {
 	RunSpecs(t, "DB Suite")
 }

-var _ = Describe("isSchemaEmpty", func() {
-	var db *sql.DB
+var _ = Describe("IsSchemaEmpty", func() {
+	var database *sql.DB
+	var ctx context.Context
 	BeforeEach(func() {
+		ctx = context.Background()
 		path := "file::memory:"
-		db, _ = sql.Open(Dialect, path)
+		database, _ = sql.Open(db.Dialect, path)
 	})

 	It("returns false if the goose metadata table is found", func() {
-		_, err := db.Exec("create table goose_db_version (id primary key);")
+		_, err := database.Exec("create table goose_db_version (id primary key);")
 		Expect(err).ToNot(HaveOccurred())
-		Expect(isSchemaEmpty(db)).To(BeFalse())
+		Expect(db.IsSchemaEmpty(ctx, database)).To(BeFalse())
 	})

 	It("returns true if the schema is brand new", func() {
-		Expect(isSchemaEmpty(db)).To(BeTrue())
+		Expect(db.IsSchemaEmpty(ctx, database)).To(BeTrue())
 	})
 })
@@ -0,0 +1,7 @@
+package db
+
+// Definitions for testing private methods
+var (
+	IsSchemaEmpty = isSchemaEmpty
+	BackupPath    = backupPath
+)
@@ -4,8 +4,8 @@ import (
 	"context"
 	"database/sql"

-	"github.com/google/uuid"
 	"github.com/navidrome/navidrome/consts"
+	"github.com/navidrome/navidrome/model/id"
 	"github.com/pressly/goose/v3"
 )

@@ -30,7 +30,7 @@ func upAddDefaultTranscodings(_ context.Context, tx *sql.Tx) error {
 	}

 	for _, t := range consts.DefaultTranscodings {
-		_, err := stmt.Exec(uuid.NewString(), t.Name, t.TargetFormat, t.DefaultBitRate, t.Command)
+		_, err := stmt.Exec(id.NewRandom(), t.Name, t.TargetFormat, t.DefaultBitRate, t.Command)
 		if err != nil {
 			return err
 		}
@@ -29,7 +29,7 @@ func upAddLibraryTable(ctx context.Context, tx *sql.Tx) error {
 	}

 	_, err = tx.ExecContext(ctx, fmt.Sprintf(`
-		insert into library(id, name, path, last_scan_at) values(1, 'Music Library', '%s', current_timestamp);
+		insert into library(id, name, path) values(1, 'Music Library', '%s');
 		delete from property where id like 'LastScan-%%';
 `, conf.Server.MusicFolder))
 	if err != nil {
@@ -0,0 +1,307 @@
+package migrations
+
+import (
+	"context"
+	"database/sql"
+	"fmt"
+	"io/fs"
+	"os"
+	"path/filepath"
+	"testing/fstest"
+	"unicode/utf8"
+
+	"github.com/navidrome/navidrome/log"
+	"github.com/navidrome/navidrome/model"
+	"github.com/navidrome/navidrome/utils/chain"
+	"github.com/pressly/goose/v3"
+)
+
+func init() {
+	goose.AddMigrationContext(upSupportNewScanner, downSupportNewScanner)
+}
+
+func upSupportNewScanner(ctx context.Context, tx *sql.Tx) error {
+	execute := createExecuteFunc(ctx, tx)
+	addColumn := createAddColumnFunc(ctx, tx)
+
+	return chain.RunSequentially(
+		upSupportNewScanner_CreateTableFolder(ctx, execute),
+		upSupportNewScanner_PopulateTableFolder(ctx, tx),
+		upSupportNewScanner_UpdateTableMediaFile(ctx, execute, addColumn),
+		upSupportNewScanner_UpdateTableAlbum(ctx, execute),
+		upSupportNewScanner_UpdateTableArtist(ctx, execute, addColumn),
+		execute(`
+alter table library
+	add column last_scan_started_at datetime default '0000-00-00 00:00:00' not null;
+alter table library
+	add column full_scan_in_progress boolean default false not null;
+
+create table if not exists media_file_artists(
+    	media_file_id varchar not null
+				references media_file (id)
+	    		 	on delete cascade,
+    	artist_id varchar not null
+				references artist (id)
+	    		 	on delete cascade,
+    	role varchar default '' not null,
+    	sub_role varchar default '' not null,
+    	constraint artist_tracks
+    	    			unique (artist_id, media_file_id, role, sub_role)
+);
+create index if not exists media_file_artists_media_file_id
+    on media_file_artists (media_file_id);
+create index if not exists media_file_artists_role
+	on media_file_artists (role);
+
+create table if not exists album_artists(
+    	album_id varchar not null
+				references album (id)
+	    		 	on delete cascade,
+    	artist_id varchar not null
+				references artist (id)
+	    		 	on delete cascade,
+    	role varchar default '' not null,
+    	sub_role varchar default '' not null,
+    	constraint album_artists
+    	    			unique (album_id, artist_id, role, sub_role)
+);
+create index if not exists album_artists_album_id
+    on album_artists (album_id);
+create index if not exists album_artists_role
+	on album_artists (role);
+
+create table if not exists tag(
+  	id varchar not null primary key,
+  	tag_name varchar default '' not null,
+  	tag_value varchar default '' not null,
+  	album_count integer default 0 not null,
+  	media_file_count integer default 0 not null,
+  	constraint tags_name_value
+		unique (tag_name, tag_value)
+);
+
+-- Genres are now stored in the tag table
+drop table if exists media_file_genres;
+drop table if exists album_genres;
+drop table if exists artist_genres;
+drop table if exists genre;
+
+-- Drop full_text indexes, as they are not being used by SQLite
+drop index if exists media_file_full_text;
+drop index if exists album_full_text;
+drop index if exists artist_full_text;
+
+-- Add PID config to properties
+insert into property (id, value) values ('PIDTrack', 'track_legacy') on conflict do nothing;
+insert into property (id, value) values ('PIDAlbum', 'album_legacy') on conflict do nothing;
+`),
+		func() error {
+			notice(tx, "A full scan will be triggered to populate the new tables. This may take a while.")
+			return forceFullRescan(tx)
+		},
+	)
+}
+
+func upSupportNewScanner_CreateTableFolder(_ context.Context, execute execStmtFunc) execFunc {
+	return execute(`
+create table if not exists folder(
+	id varchar not null
+		primary key,
+	library_id integer not null
+	    		references library (id)
+	    		 	on delete cascade,
+	path varchar default '' not null,
+	name varchar default '' not null,
+	missing boolean default false not null,
+	parent_id varchar default '' not null,
+	num_audio_files integer default 0 not null,
+	num_playlists integer default 0 not null,
+	image_files jsonb default '[]' not null,
+	images_updated_at datetime default '0000-00-00 00:00:00' not null,
+	updated_at datetime default (datetime(current_timestamp, 'localtime')) not null,
+	created_at datetime default (datetime(current_timestamp, 'localtime')) not null
+);
+create index folder_parent_id on folder(parent_id);
+`)
+}
+
+// Use paths from `media_file` table to populate `folder` table. The `folder` table must contain all paths, including
+// the ones that do not contain any media_file. We can get all paths from the media_file table to populate a
+// fstest.MapFS{}, and then walk the filesystem to insert all folders into the DB, including empty parent ones.
+func upSupportNewScanner_PopulateTableFolder(ctx context.Context, tx *sql.Tx) execFunc {
+	return func() error {
+		// First, get all folder paths from media_file table
+		rows, err := tx.QueryContext(ctx, fmt.Sprintf(`
+select distinct rtrim(media_file.path, replace(media_file.path, '%s', '')), library_id, library.path
+from media_file
+join library on media_file.library_id = library.id`, string(os.PathSeparator)))
+		if err != nil {
+			return err
+		}
+		defer rows.Close()
+
+		// Then create an in-memory filesystem with all paths
+		var path string
+		var lib model.Library
+		var f *model.Folder
+		fsys := fstest.MapFS{}
+
+		for rows.Next() {
+			err = rows.Scan(&path, &lib.ID, &lib.Path)
+			if err != nil {
+				return err
+			}
+
+			// BFR Windows!!
+			path = filepath.Clean(path)
+			path, _ = filepath.Rel("/", path)
+			fsys[path] = &fstest.MapFile{Mode: fs.ModeDir}
+		}
+		if err = rows.Err(); err != nil {
+			return fmt.Errorf("error loading folders from media_file table: %w", err)
+		}
+		if len(fsys) == 0 {
+			return nil
+		}
+
+		// Finally, walk the in-mem filesystem and insert all folders into the DB.
+		stmt, err := tx.PrepareContext(ctx, "insert into folder (id, library_id, path, name, parent_id) values (?, ?, ?, ?, ?)")
+		if err != nil {
+			return err
+		}
+		root, _ := filepath.Rel("/", lib.Path)
+		err = fs.WalkDir(fsys, root, func(path string, d fs.DirEntry, err error) error {
+			if err != nil {
+				return err
+			}
+			if d.IsDir() {
+				path, _ = filepath.Rel(root, path)
+				f = model.NewFolder(lib, path)
+				_, err = stmt.ExecContext(ctx, f.ID, lib.ID, f.Path, f.Name, f.ParentID)
+				if err != nil {
+					log.Error("Error writing folder to DB", "path", path, err)
+				}
+			}
+			return err
+		})
+		if err != nil {
+			return fmt.Errorf("error populating folder table: %w", err)
+		}
+
+		libPathLen := utf8.RuneCountInString(lib.Path)
+		_, err = tx.ExecContext(ctx, fmt.Sprintf(`
+update media_file set path = substr(path,%d);`, libPathLen+2))
+		if err != nil {
+			return fmt.Errorf("error updating media_file path: %w", err)
+		}
+
+		return nil
+	}
+}
+
+func upSupportNewScanner_UpdateTableMediaFile(_ context.Context, execute execStmtFunc, addColumn addColumnFunc) execFunc {
+	return func() error {
+		return chain.RunSequentially(
+			execute(`	
+alter table media_file 
+    add column folder_id varchar default '' not null;
+alter table media_file 
+    add column pid varchar default '' not null;
+alter table media_file
+	add column missing boolean default false not null;
+alter table media_file
+	add column mbz_release_group_id varchar default '' not null;
+alter table media_file
+	add column tags jsonb default '{}' not null;
+alter table media_file
+	add column participants jsonb default '{}' not null;
+alter table media_file 
+    add column bit_depth integer default 0 not null;
+alter table media_file
+	add column explicit_status varchar default '' not null;
+`),
+			addColumn("media_file", "birth_time", "datetime", "current_timestamp", "created_at"),
+			execute(`	
+update media_file 
+	set pid = id where pid = '';
+create index if not exists media_file_birth_time
+	on media_file (birth_time);
+create index if not exists media_file_folder_id
+ 	on media_file (folder_id);
+create index if not exists media_file_pid
+	on media_file (pid);
+create index if not exists media_file_missing
+	on media_file (missing);
+`),
+		)
+	}
+}
+
+func upSupportNewScanner_UpdateTableAlbum(_ context.Context, execute execStmtFunc) execFunc {
+	return execute(`
+drop index if exists album_all_artist_ids;
+alter table album
+	drop column all_artist_ids;
+drop index if exists album_artist;
+drop index if exists album_artist_album;
+alter table album
+	drop column artist;
+drop index if exists album_artist_id;
+alter table album
+	drop column artist_id;
+alter table album
+	add column imported_at datetime default '0000-00-00 00:00:00' not null;
+alter table album
+	add column missing boolean default false not null;
+alter table album
+	add column mbz_release_group_id varchar default '' not null;
+alter table album
+	add column tags jsonb default '{}' not null;
+alter table album
+	add column participants jsonb default '{}' not null;
+alter table album
+	drop column paths;
+alter table album
+	drop column image_files;
+alter table album
+	add column folder_ids jsonb default '[]' not null;
+alter table album
+	add column explicit_status varchar default '' not null; 
+create index if not exists album_imported_at
+	on album (imported_at);
+create index if not exists album_mbz_release_group_id
+	on album (mbz_release_group_id);
+`)
+}
+
+func upSupportNewScanner_UpdateTableArtist(_ context.Context, execute execStmtFunc, addColumn addColumnFunc) execFunc {
+	return func() error {
+		return chain.RunSequentially(
+			execute(`
+alter table artist
+	drop column album_count;
+alter table artist
+	drop column song_count;
+drop index if exists artist_size;
+alter table artist
+	drop column size;
+alter table artist
+	add column missing boolean default false not null;
+alter table artist
+	add column stats jsonb default '{"albumartist":{}}' not null;
+alter table artist
+	drop column similar_artists;
+alter table artist
+	add column similar_artists jsonb default '[]' not null;
+`),
+			addColumn("artist", "updated_at", "datetime", "current_time", "(select min(album.updated_at) from album where album_artist_id = artist.id)"),
+			addColumn("artist", "created_at", "datetime", "current_time", "(select min(album.created_at) from album where album_artist_id = artist.id)"),
+			execute(`create index if not exists artist_updated_at on artist (updated_at);`),
+			execute(`update artist set external_info_updated_at = '0000-00-00 00:00:00';`),
+		)
+	}
+}
+
+func downSupportNewScanner(context.Context, *sql.Tx) error {
+	return nil
+}
@@ -1,8 +1,10 @@
 package migrations

 import (
+	"context"
 	"database/sql"
 	"fmt"
+	"strings"
 	"sync"

 	"github.com/navidrome/navidrome/consts"
@@ -11,24 +13,29 @@ import (
 // Use this in migrations that need to communicate something important (breaking changes, forced reindexes, etc...)
 func notice(tx *sql.Tx, msg string) {
 	if isDBInitialized(tx) {
-		fmt.Printf(`
-*************************************************************************************
-NOTICE: %s
-*************************************************************************************
-
-`, msg)
+		line := strings.Repeat("*", len(msg)+8)
+		fmt.Printf("\n%s\nNOTICE: %s\n%s\n\n", line, msg, line)
 	}
 }

 // Call this in migrations that requires a full rescan
 func forceFullRescan(tx *sql.Tx) error {
-	_, err := tx.Exec(`
-delete from property where id like 'LastScan%';
-update media_file set updated_at = '0001-01-01';
-`)
+	// If a full scan is required, most probably the query optimizer is outdated, so we run `analyze`.
+	_, err := tx.Exec(`ANALYZE;`)
+	if err != nil {
+		return err
+	}
+	_, err = tx.Exec(fmt.Sprintf(`
+INSERT OR REPLACE into property (id, value) values ('%s', '1');
+`, consts.FullScanAfterMigrationFlagKey))
 	return err
 }

+// 	sq := Update(r.tableName).
+//		Set("last_scan_started_at", time.Now()).
+//		Set("full_scan_in_progress", fullScan).
+//		Where(Eq{"id": id})
+
 var (
 	once        sync.Once
 	initialized bool
@@ -56,3 +63,58 @@ func checkErr(err error) {
 		panic(err)
 	}
 }
+
+type (
+	execFunc      func() error
+	execStmtFunc  func(stmt string) execFunc
+	addColumnFunc func(tableName, columnName, columnType, defaultValue, initialValue string) execFunc
+)
+
+func createExecuteFunc(ctx context.Context, tx *sql.Tx) execStmtFunc {
+	return func(stmt string) execFunc {
+		return func() error {
+			_, err := tx.ExecContext(ctx, stmt)
+			return err
+		}
+	}
+}
+
+// Hack way to add a new `not null` column to a table, setting the initial value for existing rows based on a
+// SQL expression. It is done in 3 steps:
+//  1. Add the column as nullable. Due to the way SQLite manipulates the DDL in memory, we need to add extra padding
+//     to the default value to avoid truncating it when changing the column to not null
+//  2. Update the column with the initial value
+//  3. Change the column to not null with the default value
+//
+// Based on https://stackoverflow.com/a/25917323
+func createAddColumnFunc(ctx context.Context, tx *sql.Tx) addColumnFunc {
+	return func(tableName, columnName, columnType, defaultValue, initialValue string) execFunc {
+		return func() error {
+			// Format the `default null` value to have the same length as the final defaultValue
+			finalLen := len(fmt.Sprintf(`%s not`, defaultValue))
+			tempDefault := fmt.Sprintf(`default %s null`, strings.Repeat(" ", finalLen))
+			_, err := tx.ExecContext(ctx, fmt.Sprintf(`
+alter table %s add column %s %s %s;`, tableName, columnName, columnType, tempDefault))
+			if err != nil {
+				return err
+			}
+			_, err = tx.ExecContext(ctx, fmt.Sprintf(`
+update %s set %s = %s where %[2]s is null;`, tableName, columnName, initialValue))
+			if err != nil {
+				return err
+			}
+			_, err = tx.ExecContext(ctx, fmt.Sprintf(`
+PRAGMA writable_schema = on;
+UPDATE sqlite_master
+SET sql = replace(sql, '%[1]s %[2]s %[5]s', '%[1]s %[2]s default %[3]s not null')
+WHERE type = 'table'
+  AND name = '%[4]s';
+PRAGMA writable_schema = off;
+`, columnName, columnType, defaultValue, tableName, tempDefault))
+			if err != nil {
+				return err
+			}
+			return err
+		}
+	}
+}