feat(plugins): add TTL support, batch operations, and hardening to kvstore (#5127)
* feat(plugins): add expires_at column to kvstore schema * feat(plugins): filter expired keys in kvstore Get, Has, List * feat(plugins): add periodic cleanup of expired kvstore keys * feat(plugins): add SetWithTTL, DeleteByPrefix, and GetMany to kvstore Add three new methods to the KVStore host service: - SetWithTTL: store key-value pairs with automatic expiration - DeleteByPrefix: remove all keys matching a prefix in one operation - GetMany: retrieve multiple values in a single call All methods include comprehensive unit tests covering edge cases, expiration behavior, size tracking, and LIKE-special characters. * feat(plugins): regenerate code and update test plugin for new kvstore methods Regenerate host function wrappers and PDK bindings for Go, Python, and Rust. Update the test-kvstore plugin to exercise SetWithTTL, DeleteByPrefix, and GetMany. * feat(plugins): add integration tests for new kvstore methods Add WASM integration tests for SetWithTTL, DeleteByPrefix, and GetMany operations through the plugin boundary, verifying end-to-end behavior including TTL expiration, prefix deletion, and batch retrieval. * fix(plugins): address lint issues in kvstore implementation Handle tx.Rollback error return and suppress gosec false positive for parameterized SQL query construction in GetMany. * fix(plugins): Set clears expires_at when overwriting a TTL'd key Previously, calling Set() on a key that was stored with SetWithTTL() would leave the expires_at value intact, causing the key to silently expire even though Set implies permanent storage. Also excludes expired keys from currentSize calculation at startup. * refactor(plugins): simplify kvstore by removing in-memory size cache Replaced the in-memory currentSize cache (atomic.Int64), periodic cleanup timer, and mutex with direct database queries for storage accounting. This eliminates race conditions and cache drift issues at negligible performance cost for plugin-sized datasets. Also unified Set and SetWithTTL into a shared setValue method, simplified DeleteByPrefix to use RowsAffected instead of a transaction, and added an index on expires_at for efficient expiration filtering. * feat(plugins): add generic SQLite migration helper and refactor kvstore schema Add a reusable migrateDB helper that tracks schema versions via SQLite's PRAGMA user_version and applies pending migrations transactionally. Replace the ad-hoc createKVStoreSchema function in kvstore with a declarative migrations slice, making it easy to add future schema changes. Remove the now-redundant schema migration test since migrateDB has its own test suite and every kvstore test exercises the migrations implicitly. Signed-off-by: Deluan <deluan@navidrome.org> * fix(plugins): harden kvstore with explicit NULL handling, prefix validation, and cleanup timeout - Use sql.NullString for expires_at to explicitly send NULL instead of relying on datetime('now', '') returning NULL by accident - Reject empty prefix in DeleteByPrefix to prevent accidental data wipe - Add 5s timeout context to cleanupExpired on Close - Replace time.Sleep in unit tests with pre-expired timestamps Signed-off-by: Deluan <deluan@navidrome.org> * refactor(plugins): use batch processing in GetMany Process keys in chunks of 200 using slice.CollectChunks to avoid hitting SQLite's SQLITE_MAX_VARIABLE_NUMBER limit with large key sets. * feat(plugins): add periodic cleanup goroutine for expired kvstore keys Use the manager's context to control a background goroutine that purges expired keys every hour, stopping naturally on shutdown when the context is cancelled. --------- Signed-off-by: Deluan <deluan@navidrome.org>
This commit is contained in:
+43
-9
@@ -9,19 +9,23 @@ import (
|
||||
|
||||
// TestKVStoreInput is the input for nd_test_kvstore callback.
|
||||
type TestKVStoreInput struct {
|
||||
Operation string `json:"operation"` // "set", "get", "delete", "has", "list", "get_storage_used"
|
||||
Key string `json:"key"` // Storage key
|
||||
Value []byte `json:"value"` // For set operations
|
||||
Prefix string `json:"prefix"` // For list operation
|
||||
Operation string `json:"operation"` // "set", "get", "delete", "has", "list", "get_storage_used", "set_with_ttl", "delete_by_prefix", "get_many"
|
||||
Key string `json:"key"` // Storage key
|
||||
Value []byte `json:"value"` // For set operations
|
||||
Prefix string `json:"prefix"` // For list/delete_by_prefix operations
|
||||
TTLSeconds int64 `json:"ttl_seconds,omitempty"` // For set_with_ttl
|
||||
Keys []string `json:"keys,omitempty"` // For get_many
|
||||
}
|
||||
|
||||
// TestKVStoreOutput is the output from nd_test_kvstore callback.
|
||||
type TestKVStoreOutput struct {
|
||||
Value []byte `json:"value,omitempty"`
|
||||
Exists bool `json:"exists,omitempty"`
|
||||
Keys []string `json:"keys,omitempty"`
|
||||
StorageUsed int64 `json:"storage_used,omitempty"`
|
||||
Error *string `json:"error,omitempty"`
|
||||
Value []byte `json:"value,omitempty"`
|
||||
Values map[string][]byte `json:"values,omitempty"`
|
||||
Exists bool `json:"exists,omitempty"`
|
||||
Keys []string `json:"keys,omitempty"`
|
||||
StorageUsed int64 `json:"storage_used,omitempty"`
|
||||
DeletedCount int64 `json:"deleted_count,omitempty"`
|
||||
Error *string `json:"error,omitempty"`
|
||||
}
|
||||
|
||||
// nd_test_kvstore is the test callback that tests the kvstore host functions.
|
||||
@@ -96,6 +100,36 @@ func ndTestKVStore() int32 {
|
||||
pdk.OutputJSON(TestKVStoreOutput{StorageUsed: bytesUsed})
|
||||
return 0
|
||||
|
||||
case "set_with_ttl":
|
||||
err := host.KVStoreSetWithTTL(input.Key, input.Value, input.TTLSeconds)
|
||||
if err != nil {
|
||||
errStr := err.Error()
|
||||
pdk.OutputJSON(TestKVStoreOutput{Error: &errStr})
|
||||
return 0
|
||||
}
|
||||
pdk.OutputJSON(TestKVStoreOutput{})
|
||||
return 0
|
||||
|
||||
case "delete_by_prefix":
|
||||
deletedCount, err := host.KVStoreDeleteByPrefix(input.Prefix)
|
||||
if err != nil {
|
||||
errStr := err.Error()
|
||||
pdk.OutputJSON(TestKVStoreOutput{Error: &errStr})
|
||||
return 0
|
||||
}
|
||||
pdk.OutputJSON(TestKVStoreOutput{DeletedCount: deletedCount})
|
||||
return 0
|
||||
|
||||
case "get_many":
|
||||
values, err := host.KVStoreGetMany(input.Keys)
|
||||
if err != nil {
|
||||
errStr := err.Error()
|
||||
pdk.OutputJSON(TestKVStoreOutput{Error: &errStr})
|
||||
return 0
|
||||
}
|
||||
pdk.OutputJSON(TestKVStoreOutput{Values: values})
|
||||
return 0
|
||||
|
||||
default:
|
||||
errStr := "unknown operation: " + input.Operation
|
||||
pdk.OutputJSON(TestKVStoreOutput{Error: &errStr})
|
||||
|
||||
Reference in New Issue
Block a user