2471bb9cf6
* feat(plugins): add expires_at column to kvstore schema * feat(plugins): filter expired keys in kvstore Get, Has, List * feat(plugins): add periodic cleanup of expired kvstore keys * feat(plugins): add SetWithTTL, DeleteByPrefix, and GetMany to kvstore Add three new methods to the KVStore host service: - SetWithTTL: store key-value pairs with automatic expiration - DeleteByPrefix: remove all keys matching a prefix in one operation - GetMany: retrieve multiple values in a single call All methods include comprehensive unit tests covering edge cases, expiration behavior, size tracking, and LIKE-special characters. * feat(plugins): regenerate code and update test plugin for new kvstore methods Regenerate host function wrappers and PDK bindings for Go, Python, and Rust. Update the test-kvstore plugin to exercise SetWithTTL, DeleteByPrefix, and GetMany. * feat(plugins): add integration tests for new kvstore methods Add WASM integration tests for SetWithTTL, DeleteByPrefix, and GetMany operations through the plugin boundary, verifying end-to-end behavior including TTL expiration, prefix deletion, and batch retrieval. * fix(plugins): address lint issues in kvstore implementation Handle tx.Rollback error return and suppress gosec false positive for parameterized SQL query construction in GetMany. * fix(plugins): Set clears expires_at when overwriting a TTL'd key Previously, calling Set() on a key that was stored with SetWithTTL() would leave the expires_at value intact, causing the key to silently expire even though Set implies permanent storage. Also excludes expired keys from currentSize calculation at startup. * refactor(plugins): simplify kvstore by removing in-memory size cache Replaced the in-memory currentSize cache (atomic.Int64), periodic cleanup timer, and mutex with direct database queries for storage accounting. This eliminates race conditions and cache drift issues at negligible performance cost for plugin-sized datasets. Also unified Set and SetWithTTL into a shared setValue method, simplified DeleteByPrefix to use RowsAffected instead of a transaction, and added an index on expires_at for efficient expiration filtering. * feat(plugins): add generic SQLite migration helper and refactor kvstore schema Add a reusable migrateDB helper that tracks schema versions via SQLite's PRAGMA user_version and applies pending migrations transactionally. Replace the ad-hoc createKVStoreSchema function in kvstore with a declarative migrations slice, making it easy to add future schema changes. Remove the now-redundant schema migration test since migrateDB has its own test suite and every kvstore test exercises the migrations implicitly. Signed-off-by: Deluan <deluan@navidrome.org> * fix(plugins): harden kvstore with explicit NULL handling, prefix validation, and cleanup timeout - Use sql.NullString for expires_at to explicitly send NULL instead of relying on datetime('now', '') returning NULL by accident - Reject empty prefix in DeleteByPrefix to prevent accidental data wipe - Add 5s timeout context to cleanupExpired on Close - Replace time.Sleep in unit tests with pre-expired timestamps Signed-off-by: Deluan <deluan@navidrome.org> * refactor(plugins): use batch processing in GetMany Process keys in chunks of 200 using slice.CollectChunks to avoid hitting SQLite's SQLITE_MAX_VARIABLE_NUMBER limit with large key sets. * feat(plugins): add periodic cleanup goroutine for expired kvstore keys Use the manager's context to control a background goroutine that purges expired keys every hour, stopping naturally on shutdown when the context is cancelled. --------- Signed-off-by: Deluan <deluan@navidrome.org>
482 lines
12 KiB
Go
482 lines
12 KiB
Go
// Code generated by ndpgen. DO NOT EDIT.
|
|
//
|
|
// This file contains client wrappers for the KVStore host service.
|
|
// It is intended for use in Navidrome plugins built with TinyGo.
|
|
//
|
|
//go:build wasip1
|
|
|
|
package host
|
|
|
|
import (
|
|
"encoding/json"
|
|
"errors"
|
|
|
|
"github.com/navidrome/navidrome/plugins/pdk/go/pdk"
|
|
)
|
|
|
|
// kvstore_set is the host function provided by Navidrome.
|
|
//
|
|
//go:wasmimport extism:host/user kvstore_set
|
|
func kvstore_set(uint64) uint64
|
|
|
|
// kvstore_setwithttl is the host function provided by Navidrome.
|
|
//
|
|
//go:wasmimport extism:host/user kvstore_setwithttl
|
|
func kvstore_setwithttl(uint64) uint64
|
|
|
|
// kvstore_get is the host function provided by Navidrome.
|
|
//
|
|
//go:wasmimport extism:host/user kvstore_get
|
|
func kvstore_get(uint64) uint64
|
|
|
|
// kvstore_getmany is the host function provided by Navidrome.
|
|
//
|
|
//go:wasmimport extism:host/user kvstore_getmany
|
|
func kvstore_getmany(uint64) uint64
|
|
|
|
// kvstore_has is the host function provided by Navidrome.
|
|
//
|
|
//go:wasmimport extism:host/user kvstore_has
|
|
func kvstore_has(uint64) uint64
|
|
|
|
// kvstore_list is the host function provided by Navidrome.
|
|
//
|
|
//go:wasmimport extism:host/user kvstore_list
|
|
func kvstore_list(uint64) uint64
|
|
|
|
// kvstore_delete is the host function provided by Navidrome.
|
|
//
|
|
//go:wasmimport extism:host/user kvstore_delete
|
|
func kvstore_delete(uint64) uint64
|
|
|
|
// kvstore_deletebyprefix is the host function provided by Navidrome.
|
|
//
|
|
//go:wasmimport extism:host/user kvstore_deletebyprefix
|
|
func kvstore_deletebyprefix(uint64) uint64
|
|
|
|
// kvstore_getstorageused is the host function provided by Navidrome.
|
|
//
|
|
//go:wasmimport extism:host/user kvstore_getstorageused
|
|
func kvstore_getstorageused(uint64) uint64
|
|
|
|
type kVStoreSetRequest struct {
|
|
Key string `json:"key"`
|
|
Value []byte `json:"value"`
|
|
}
|
|
|
|
type kVStoreSetWithTTLRequest struct {
|
|
Key string `json:"key"`
|
|
Value []byte `json:"value"`
|
|
TtlSeconds int64 `json:"ttlSeconds"`
|
|
}
|
|
|
|
type kVStoreGetRequest struct {
|
|
Key string `json:"key"`
|
|
}
|
|
|
|
type kVStoreGetResponse struct {
|
|
Value []byte `json:"value,omitempty"`
|
|
Exists bool `json:"exists,omitempty"`
|
|
Error string `json:"error,omitempty"`
|
|
}
|
|
|
|
type kVStoreGetManyRequest struct {
|
|
Keys []string `json:"keys"`
|
|
}
|
|
|
|
type kVStoreGetManyResponse struct {
|
|
Values map[string][]byte `json:"values,omitempty"`
|
|
Error string `json:"error,omitempty"`
|
|
}
|
|
|
|
type kVStoreHasRequest struct {
|
|
Key string `json:"key"`
|
|
}
|
|
|
|
type kVStoreHasResponse struct {
|
|
Exists bool `json:"exists,omitempty"`
|
|
Error string `json:"error,omitempty"`
|
|
}
|
|
|
|
type kVStoreListRequest struct {
|
|
Prefix string `json:"prefix"`
|
|
}
|
|
|
|
type kVStoreListResponse struct {
|
|
Keys []string `json:"keys,omitempty"`
|
|
Error string `json:"error,omitempty"`
|
|
}
|
|
|
|
type kVStoreDeleteRequest struct {
|
|
Key string `json:"key"`
|
|
}
|
|
|
|
type kVStoreDeleteByPrefixRequest struct {
|
|
Prefix string `json:"prefix"`
|
|
}
|
|
|
|
type kVStoreDeleteByPrefixResponse struct {
|
|
DeletedCount int64 `json:"deletedCount,omitempty"`
|
|
Error string `json:"error,omitempty"`
|
|
}
|
|
|
|
type kVStoreGetStorageUsedResponse struct {
|
|
Bytes int64 `json:"bytes,omitempty"`
|
|
Error string `json:"error,omitempty"`
|
|
}
|
|
|
|
// KVStoreSet calls the kvstore_set host function.
|
|
// Set stores a byte value with the given key.
|
|
//
|
|
// Parameters:
|
|
// - key: The storage key (max 256 bytes, UTF-8)
|
|
// - value: The byte slice to store
|
|
//
|
|
// Returns an error if the storage limit would be exceeded or the operation fails.
|
|
func KVStoreSet(key string, value []byte) error {
|
|
// Marshal request to JSON
|
|
req := kVStoreSetRequest{
|
|
Key: key,
|
|
Value: value,
|
|
}
|
|
reqBytes, err := json.Marshal(req)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
reqMem := pdk.AllocateBytes(reqBytes)
|
|
defer reqMem.Free()
|
|
|
|
// Call the host function
|
|
responsePtr := kvstore_set(reqMem.Offset())
|
|
|
|
// Read the response from memory
|
|
responseMem := pdk.FindMemory(responsePtr)
|
|
responseBytes := responseMem.ReadBytes()
|
|
|
|
// Parse error-only response
|
|
var response struct {
|
|
Error string `json:"error,omitempty"`
|
|
}
|
|
if err := json.Unmarshal(responseBytes, &response); err != nil {
|
|
return err
|
|
}
|
|
if response.Error != "" {
|
|
return errors.New(response.Error)
|
|
}
|
|
return nil
|
|
}
|
|
|
|
// KVStoreSetWithTTL calls the kvstore_setwithttl host function.
|
|
// SetWithTTL stores a byte value with the given key and a time-to-live.
|
|
//
|
|
// After ttlSeconds, the key is treated as non-existent and will be
|
|
// cleaned up lazily. ttlSeconds must be greater than 0.
|
|
//
|
|
// Parameters:
|
|
// - key: The storage key (max 256 bytes, UTF-8)
|
|
// - value: The byte slice to store
|
|
// - ttlSeconds: Time-to-live in seconds (must be > 0)
|
|
//
|
|
// Returns an error if the storage limit would be exceeded or the operation fails.
|
|
func KVStoreSetWithTTL(key string, value []byte, ttlSeconds int64) error {
|
|
// Marshal request to JSON
|
|
req := kVStoreSetWithTTLRequest{
|
|
Key: key,
|
|
Value: value,
|
|
TtlSeconds: ttlSeconds,
|
|
}
|
|
reqBytes, err := json.Marshal(req)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
reqMem := pdk.AllocateBytes(reqBytes)
|
|
defer reqMem.Free()
|
|
|
|
// Call the host function
|
|
responsePtr := kvstore_setwithttl(reqMem.Offset())
|
|
|
|
// Read the response from memory
|
|
responseMem := pdk.FindMemory(responsePtr)
|
|
responseBytes := responseMem.ReadBytes()
|
|
|
|
// Parse error-only response
|
|
var response struct {
|
|
Error string `json:"error,omitempty"`
|
|
}
|
|
if err := json.Unmarshal(responseBytes, &response); err != nil {
|
|
return err
|
|
}
|
|
if response.Error != "" {
|
|
return errors.New(response.Error)
|
|
}
|
|
return nil
|
|
}
|
|
|
|
// KVStoreGet calls the kvstore_get host function.
|
|
// Get retrieves a byte value from storage.
|
|
//
|
|
// Parameters:
|
|
// - key: The storage key
|
|
//
|
|
// Returns the value and whether the key exists.
|
|
func KVStoreGet(key string) ([]byte, bool, error) {
|
|
// Marshal request to JSON
|
|
req := kVStoreGetRequest{
|
|
Key: key,
|
|
}
|
|
reqBytes, err := json.Marshal(req)
|
|
if err != nil {
|
|
return nil, false, err
|
|
}
|
|
reqMem := pdk.AllocateBytes(reqBytes)
|
|
defer reqMem.Free()
|
|
|
|
// Call the host function
|
|
responsePtr := kvstore_get(reqMem.Offset())
|
|
|
|
// Read the response from memory
|
|
responseMem := pdk.FindMemory(responsePtr)
|
|
responseBytes := responseMem.ReadBytes()
|
|
|
|
// Parse the response
|
|
var response kVStoreGetResponse
|
|
if err := json.Unmarshal(responseBytes, &response); err != nil {
|
|
return nil, false, err
|
|
}
|
|
|
|
// Convert Error field to Go error
|
|
if response.Error != "" {
|
|
return nil, false, errors.New(response.Error)
|
|
}
|
|
|
|
return response.Value, response.Exists, nil
|
|
}
|
|
|
|
// KVStoreGetMany calls the kvstore_getmany host function.
|
|
// GetMany retrieves multiple values in a single call.
|
|
//
|
|
// Parameters:
|
|
// - keys: The storage keys to retrieve
|
|
//
|
|
// Returns a map of key to value for keys that exist and have not expired.
|
|
// Missing or expired keys are omitted from the result.
|
|
func KVStoreGetMany(keys []string) (map[string][]byte, error) {
|
|
// Marshal request to JSON
|
|
req := kVStoreGetManyRequest{
|
|
Keys: keys,
|
|
}
|
|
reqBytes, err := json.Marshal(req)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
reqMem := pdk.AllocateBytes(reqBytes)
|
|
defer reqMem.Free()
|
|
|
|
// Call the host function
|
|
responsePtr := kvstore_getmany(reqMem.Offset())
|
|
|
|
// Read the response from memory
|
|
responseMem := pdk.FindMemory(responsePtr)
|
|
responseBytes := responseMem.ReadBytes()
|
|
|
|
// Parse the response
|
|
var response kVStoreGetManyResponse
|
|
if err := json.Unmarshal(responseBytes, &response); err != nil {
|
|
return nil, err
|
|
}
|
|
|
|
// Convert Error field to Go error
|
|
if response.Error != "" {
|
|
return nil, errors.New(response.Error)
|
|
}
|
|
|
|
return response.Values, nil
|
|
}
|
|
|
|
// KVStoreHas calls the kvstore_has host function.
|
|
// Has checks if a key exists in storage.
|
|
//
|
|
// Parameters:
|
|
// - key: The storage key
|
|
//
|
|
// Returns true if the key exists.
|
|
func KVStoreHas(key string) (bool, error) {
|
|
// Marshal request to JSON
|
|
req := kVStoreHasRequest{
|
|
Key: key,
|
|
}
|
|
reqBytes, err := json.Marshal(req)
|
|
if err != nil {
|
|
return false, err
|
|
}
|
|
reqMem := pdk.AllocateBytes(reqBytes)
|
|
defer reqMem.Free()
|
|
|
|
// Call the host function
|
|
responsePtr := kvstore_has(reqMem.Offset())
|
|
|
|
// Read the response from memory
|
|
responseMem := pdk.FindMemory(responsePtr)
|
|
responseBytes := responseMem.ReadBytes()
|
|
|
|
// Parse the response
|
|
var response kVStoreHasResponse
|
|
if err := json.Unmarshal(responseBytes, &response); err != nil {
|
|
return false, err
|
|
}
|
|
|
|
// Convert Error field to Go error
|
|
if response.Error != "" {
|
|
return false, errors.New(response.Error)
|
|
}
|
|
|
|
return response.Exists, nil
|
|
}
|
|
|
|
// KVStoreList calls the kvstore_list host function.
|
|
// List returns all keys matching the given prefix.
|
|
//
|
|
// Parameters:
|
|
// - prefix: Key prefix to filter by (empty string returns all keys)
|
|
//
|
|
// Returns a slice of matching keys.
|
|
func KVStoreList(prefix string) ([]string, error) {
|
|
// Marshal request to JSON
|
|
req := kVStoreListRequest{
|
|
Prefix: prefix,
|
|
}
|
|
reqBytes, err := json.Marshal(req)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
reqMem := pdk.AllocateBytes(reqBytes)
|
|
defer reqMem.Free()
|
|
|
|
// Call the host function
|
|
responsePtr := kvstore_list(reqMem.Offset())
|
|
|
|
// Read the response from memory
|
|
responseMem := pdk.FindMemory(responsePtr)
|
|
responseBytes := responseMem.ReadBytes()
|
|
|
|
// Parse the response
|
|
var response kVStoreListResponse
|
|
if err := json.Unmarshal(responseBytes, &response); err != nil {
|
|
return nil, err
|
|
}
|
|
|
|
// Convert Error field to Go error
|
|
if response.Error != "" {
|
|
return nil, errors.New(response.Error)
|
|
}
|
|
|
|
return response.Keys, nil
|
|
}
|
|
|
|
// KVStoreDelete calls the kvstore_delete host function.
|
|
// Delete removes a value from storage.
|
|
//
|
|
// Parameters:
|
|
// - key: The storage key
|
|
//
|
|
// Returns an error if the operation fails. Does not return an error if the key doesn't exist.
|
|
func KVStoreDelete(key string) error {
|
|
// Marshal request to JSON
|
|
req := kVStoreDeleteRequest{
|
|
Key: key,
|
|
}
|
|
reqBytes, err := json.Marshal(req)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
reqMem := pdk.AllocateBytes(reqBytes)
|
|
defer reqMem.Free()
|
|
|
|
// Call the host function
|
|
responsePtr := kvstore_delete(reqMem.Offset())
|
|
|
|
// Read the response from memory
|
|
responseMem := pdk.FindMemory(responsePtr)
|
|
responseBytes := responseMem.ReadBytes()
|
|
|
|
// Parse error-only response
|
|
var response struct {
|
|
Error string `json:"error,omitempty"`
|
|
}
|
|
if err := json.Unmarshal(responseBytes, &response); err != nil {
|
|
return err
|
|
}
|
|
if response.Error != "" {
|
|
return errors.New(response.Error)
|
|
}
|
|
return nil
|
|
}
|
|
|
|
// KVStoreDeleteByPrefix calls the kvstore_deletebyprefix host function.
|
|
// DeleteByPrefix removes all keys matching the given prefix.
|
|
//
|
|
// Parameters:
|
|
// - prefix: Key prefix to match (must not be empty)
|
|
//
|
|
// Returns the number of keys deleted. Includes expired keys.
|
|
func KVStoreDeleteByPrefix(prefix string) (int64, error) {
|
|
// Marshal request to JSON
|
|
req := kVStoreDeleteByPrefixRequest{
|
|
Prefix: prefix,
|
|
}
|
|
reqBytes, err := json.Marshal(req)
|
|
if err != nil {
|
|
return 0, err
|
|
}
|
|
reqMem := pdk.AllocateBytes(reqBytes)
|
|
defer reqMem.Free()
|
|
|
|
// Call the host function
|
|
responsePtr := kvstore_deletebyprefix(reqMem.Offset())
|
|
|
|
// Read the response from memory
|
|
responseMem := pdk.FindMemory(responsePtr)
|
|
responseBytes := responseMem.ReadBytes()
|
|
|
|
// Parse the response
|
|
var response kVStoreDeleteByPrefixResponse
|
|
if err := json.Unmarshal(responseBytes, &response); err != nil {
|
|
return 0, err
|
|
}
|
|
|
|
// Convert Error field to Go error
|
|
if response.Error != "" {
|
|
return 0, errors.New(response.Error)
|
|
}
|
|
|
|
return response.DeletedCount, nil
|
|
}
|
|
|
|
// KVStoreGetStorageUsed calls the kvstore_getstorageused host function.
|
|
// GetStorageUsed returns the total storage used by this plugin in bytes.
|
|
func KVStoreGetStorageUsed() (int64, error) {
|
|
// No parameters - allocate empty JSON object
|
|
reqMem := pdk.AllocateBytes([]byte("{}"))
|
|
defer reqMem.Free()
|
|
|
|
// Call the host function
|
|
responsePtr := kvstore_getstorageused(reqMem.Offset())
|
|
|
|
// Read the response from memory
|
|
responseMem := pdk.FindMemory(responsePtr)
|
|
responseBytes := responseMem.ReadBytes()
|
|
|
|
// Parse the response
|
|
var response kVStoreGetStorageUsedResponse
|
|
if err := json.Unmarshal(responseBytes, &response); err != nil {
|
|
return 0, err
|
|
}
|
|
|
|
// Convert Error field to Go error
|
|
if response.Error != "" {
|
|
return 0, errors.New(response.Error)
|
|
}
|
|
|
|
return response.Bytes, nil
|
|
}
|