feat(plugins): add TTL support, batch operations, and hardening to kvstore (#5127)

* feat(plugins): add expires_at column to kvstore schema * feat(plugins): filter expired keys in kvstore Get, Has, List * feat(plugins): add periodic cleanup of expired kvstore keys * feat(plugins): add SetWithTTL, DeleteByPrefix, and GetMany to kvstore Add three new methods to the KVStore host service: - SetWithTTL: store key-value pairs with automatic expiration - DeleteByPrefix: remove all keys matching a prefix in one operation - GetMany: retrieve multiple values in a single call All methods include comprehensive unit tests covering edge cases, expiration behavior, size tracking, and LIKE-special characters. * feat(plugins): regenerate code and update test plugin for new kvstore methods Regenerate host function wrappers and PDK bindings for Go, Python, and Rust. Update the test-kvstore plugin to exercise SetWithTTL, DeleteByPrefix, and GetMany. * feat(plugins): add integration tests for new kvstore methods Add WASM integration tests for SetWithTTL, DeleteByPrefix, and GetMany operations through the plugin boundary, verifying end-to-end behavior including TTL expiration, prefix deletion, and batch retrieval. * fix(plugins): address lint issues in kvstore implementation Handle tx.Rollback error return and suppress gosec false positive for parameterized SQL query construction in GetMany. * fix(plugins): Set clears expires_at when overwriting a TTL'd key Previously, calling Set() on a key that was stored with SetWithTTL() would leave the expires_at value intact, causing the key to silently expire even though Set implies permanent storage. Also excludes expired keys from currentSize calculation at startup. * refactor(plugins): simplify kvstore by removing in-memory size cache Replaced the in-memory currentSize cache (atomic.Int64), periodic cleanup timer, and mutex with direct database queries for storage accounting. This eliminates race conditions and cache drift issues at negligible performance cost for plugin-sized datasets. Also unified Set and SetWithTTL into a shared setValue method, simplified DeleteByPrefix to use RowsAffected instead of a transaction, and added an index on expires_at for efficient expiration filtering. * feat(plugins): add generic SQLite migration helper and refactor kvstore schema Add a reusable migrateDB helper that tracks schema versions via SQLite's PRAGMA user_version and applies pending migrations transactionally. Replace the ad-hoc createKVStoreSchema function in kvstore with a declarative migrations slice, making it easy to add future schema changes. Remove the now-redundant schema migration test since migrateDB has its own test suite and every kvstore test exercises the migrations implicitly. Signed-off-by: Deluan <deluan@navidrome.org> * fix(plugins): harden kvstore with explicit NULL handling, prefix validation, and cleanup timeout - Use sql.NullString for expires_at to explicitly send NULL instead of relying on datetime('now', '') returning NULL by accident - Reject empty prefix in DeleteByPrefix to prevent accidental data wipe - Add 5s timeout context to cleanupExpired on Close - Replace time.Sleep in unit tests with pre-expired timestamps Signed-off-by: Deluan <deluan@navidrome.org> * refactor(plugins): use batch processing in GetMany Process keys in chunks of 200 using slice.CollectChunks to avoid hitting SQLite's SQLITE_MAX_VARIABLE_NUMBER limit with large key sets. * feat(plugins): add periodic cleanup goroutine for expired kvstore keys Use the manager's context to control a background goroutine that purges expired keys every hour, stopping naturally on shutdown when the context is cancelled. --------- Signed-off-by: Deluan <deluan@navidrome.org>
2026-02-28 23:12:17 -05:00
parent d9a215e1e3
commit 2471bb9cf6
12 changed files with 1528 additions and 158 deletions
@@ -23,6 +23,20 @@ type KVStoreService interface {
 	//nd:hostfunc
 	Set(ctx context.Context, key string, value []byte) error

+	// SetWithTTL stores a byte value with the given key and a time-to-live.
+	//
+	// After ttlSeconds, the key is treated as non-existent and will be
+	// cleaned up lazily. ttlSeconds must be greater than 0.
+	//
+	// Parameters:
+	//   - key: The storage key (max 256 bytes, UTF-8)
+	//   - value: The byte slice to store
+	//   - ttlSeconds: Time-to-live in seconds (must be > 0)
+	//
+	// Returns an error if the storage limit would be exceeded or the operation fails.
+	//nd:hostfunc
+	SetWithTTL(ctx context.Context, key string, value []byte, ttlSeconds int64) error
+
 	// Get retrieves a byte value from storage.
 	//
 	// Parameters:
@@ -32,14 +46,15 @@ type KVStoreService interface {
 	//nd:hostfunc
 	Get(ctx context.Context, key string) (value []byte, exists bool, err error)

-	// Delete removes a value from storage.
+	// GetMany retrieves multiple values in a single call.
 	//
 	// Parameters:
-	//   - key: The storage key
+	//   - keys: The storage keys to retrieve
 	//
-	// Returns an error if the operation fails. Does not return an error if the key doesn't exist.
+	// Returns a map of key to value for keys that exist and have not expired.
+	// Missing or expired keys are omitted from the result.
 	//nd:hostfunc
-	Delete(ctx context.Context, key string) error
+	GetMany(ctx context.Context, keys []string) (values map[string][]byte, err error)

 	// Has checks if a key exists in storage.
 	//
@@ -59,6 +74,24 @@ type KVStoreService interface {
 	//nd:hostfunc
 	List(ctx context.Context, prefix string) (keys []string, err error)

+	// Delete removes a value from storage.
+	//
+	// Parameters:
+	//   - key: The storage key
+	//
+	// Returns an error if the operation fails. Does not return an error if the key doesn't exist.
+	//nd:hostfunc
+	Delete(ctx context.Context, key string) error
+
+	// DeleteByPrefix removes all keys matching the given prefix.
+	//
+	// Parameters:
+	//   - prefix: Key prefix to match (must not be empty)
+	//
+	// Returns the number of keys deleted. Includes expired keys.
+	//nd:hostfunc
+	DeleteByPrefix(ctx context.Context, prefix string) (deletedCount int64, err error)
+
 	// GetStorageUsed returns the total storage used by this plugin in bytes.
 	//nd:hostfunc
 	GetStorageUsed(ctx context.Context) (bytes int64, err error)
@@ -20,6 +20,18 @@ type KVStoreSetResponse struct {
 	Error string `json:"error,omitempty"`
 }

+// KVStoreSetWithTTLRequest is the request type for KVStore.SetWithTTL.
+type KVStoreSetWithTTLRequest struct {
+	Key        string `json:"key"`
+	Value      []byte `json:"value"`
+	TtlSeconds int64  `json:"ttlSeconds"`
+}
+
+// KVStoreSetWithTTLResponse is the response type for KVStore.SetWithTTL.
+type KVStoreSetWithTTLResponse struct {
+	Error string `json:"error,omitempty"`
+}
+
 // KVStoreGetRequest is the request type for KVStore.Get.
 type KVStoreGetRequest struct {
 	Key string `json:"key"`
@@ -32,14 +44,15 @@ type KVStoreGetResponse struct {
 	Error  string `json:"error,omitempty"`
 }

-// KVStoreDeleteRequest is the request type for KVStore.Delete.
-type KVStoreDeleteRequest struct {
-	Key string `json:"key"`
+// KVStoreGetManyRequest is the request type for KVStore.GetMany.
+type KVStoreGetManyRequest struct {
+	Keys []string `json:"keys"`
 }

-// KVStoreDeleteResponse is the response type for KVStore.Delete.
-type KVStoreDeleteResponse struct {
-	Error string `json:"error,omitempty"`
+// KVStoreGetManyResponse is the response type for KVStore.GetMany.
+type KVStoreGetManyResponse struct {
+	Values map[string][]byte `json:"values,omitempty"`
+	Error  string            `json:"error,omitempty"`
 }

 // KVStoreHasRequest is the request type for KVStore.Has.
@@ -64,6 +77,27 @@ type KVStoreListResponse struct {
 	Error string   `json:"error,omitempty"`
 }

+// KVStoreDeleteRequest is the request type for KVStore.Delete.
+type KVStoreDeleteRequest struct {
+	Key string `json:"key"`
+}
+
+// KVStoreDeleteResponse is the response type for KVStore.Delete.
+type KVStoreDeleteResponse struct {
+	Error string `json:"error,omitempty"`
+}
+
+// KVStoreDeleteByPrefixRequest is the request type for KVStore.DeleteByPrefix.
+type KVStoreDeleteByPrefixRequest struct {
+	Prefix string `json:"prefix"`
+}
+
+// KVStoreDeleteByPrefixResponse is the response type for KVStore.DeleteByPrefix.
+type KVStoreDeleteByPrefixResponse struct {
+	DeletedCount int64  `json:"deletedCount,omitempty"`
+	Error        string `json:"error,omitempty"`
+}
+
 // KVStoreGetStorageUsedResponse is the response type for KVStore.GetStorageUsed.
 type KVStoreGetStorageUsedResponse struct {
 	Bytes int64  `json:"bytes,omitempty"`
@@ -75,10 +109,13 @@ type KVStoreGetStorageUsedResponse struct {
 func RegisterKVStoreHostFunctions(service KVStoreService) []extism.HostFunction {
 	return []extism.HostFunction{
 		newKVStoreSetHostFunction(service),
+		newKVStoreSetWithTTLHostFunction(service),
 		newKVStoreGetHostFunction(service),
-		newKVStoreDeleteHostFunction(service),
+		newKVStoreGetManyHostFunction(service),
 		newKVStoreHasHostFunction(service),
 		newKVStoreListHostFunction(service),
+		newKVStoreDeleteHostFunction(service),
+		newKVStoreDeleteByPrefixHostFunction(service),
 		newKVStoreGetStorageUsedHostFunction(service),
 	}
 }
@@ -114,6 +151,37 @@ func newKVStoreSetHostFunction(service KVStoreService) extism.HostFunction {
 	)
 }

+func newKVStoreSetWithTTLHostFunction(service KVStoreService) extism.HostFunction {
+	return extism.NewHostFunctionWithStack(
+		"kvstore_setwithttl",
+		func(ctx context.Context, p *extism.CurrentPlugin, stack []uint64) {
+			// Read JSON request from plugin memory
+			reqBytes, err := p.ReadBytes(stack[0])
+			if err != nil {
+				kvstoreWriteError(p, stack, err)
+				return
+			}
+			var req KVStoreSetWithTTLRequest
+			if err := json.Unmarshal(reqBytes, &req); err != nil {
+				kvstoreWriteError(p, stack, err)
+				return
+			}
+
+			// Call the service method
+			if svcErr := service.SetWithTTL(ctx, req.Key, req.Value, req.TtlSeconds); svcErr != nil {
+				kvstoreWriteError(p, stack, svcErr)
+				return
+			}
+
+			// Write JSON response to plugin memory
+			resp := KVStoreSetWithTTLResponse{}
+			kvstoreWriteResponse(p, stack, resp)
+		},
+		[]extism.ValueType{extism.ValueTypePTR},
+		[]extism.ValueType{extism.ValueTypePTR},
+	)
+}
+
 func newKVStoreGetHostFunction(service KVStoreService) extism.HostFunction {
 	return extism.NewHostFunctionWithStack(
 		"kvstore_get",
@@ -149,9 +217,9 @@ func newKVStoreGetHostFunction(service KVStoreService) extism.HostFunction {
 	)
 }

-func newKVStoreDeleteHostFunction(service KVStoreService) extism.HostFunction {
+func newKVStoreGetManyHostFunction(service KVStoreService) extism.HostFunction {
 	return extism.NewHostFunctionWithStack(
-		"kvstore_delete",
+		"kvstore_getmany",
 		func(ctx context.Context, p *extism.CurrentPlugin, stack []uint64) {
 			// Read JSON request from plugin memory
 			reqBytes, err := p.ReadBytes(stack[0])
@@ -159,20 +227,23 @@ func newKVStoreDeleteHostFunction(service KVStoreService) extism.HostFunction {
 				kvstoreWriteError(p, stack, err)
 				return
 			}
-			var req KVStoreDeleteRequest
+			var req KVStoreGetManyRequest
 			if err := json.Unmarshal(reqBytes, &req); err != nil {
 				kvstoreWriteError(p, stack, err)
 				return
 			}

 			// Call the service method
-			if svcErr := service.Delete(ctx, req.Key); svcErr != nil {
+			values, svcErr := service.GetMany(ctx, req.Keys)
+			if svcErr != nil {
 				kvstoreWriteError(p, stack, svcErr)
 				return
 			}

 			// Write JSON response to plugin memory
-			resp := KVStoreDeleteResponse{}
+			resp := KVStoreGetManyResponse{
+				Values: values,
+			}
 			kvstoreWriteResponse(p, stack, resp)
 		},
 		[]extism.ValueType{extism.ValueTypePTR},
@@ -248,6 +319,71 @@ func newKVStoreListHostFunction(service KVStoreService) extism.HostFunction {
 	)
 }

+func newKVStoreDeleteHostFunction(service KVStoreService) extism.HostFunction {
+	return extism.NewHostFunctionWithStack(
+		"kvstore_delete",
+		func(ctx context.Context, p *extism.CurrentPlugin, stack []uint64) {
+			// Read JSON request from plugin memory
+			reqBytes, err := p.ReadBytes(stack[0])
+			if err != nil {
+				kvstoreWriteError(p, stack, err)
+				return
+			}
+			var req KVStoreDeleteRequest
+			if err := json.Unmarshal(reqBytes, &req); err != nil {
+				kvstoreWriteError(p, stack, err)
+				return
+			}
+
+			// Call the service method
+			if svcErr := service.Delete(ctx, req.Key); svcErr != nil {
+				kvstoreWriteError(p, stack, svcErr)
+				return
+			}
+
+			// Write JSON response to plugin memory
+			resp := KVStoreDeleteResponse{}
+			kvstoreWriteResponse(p, stack, resp)
+		},
+		[]extism.ValueType{extism.ValueTypePTR},
+		[]extism.ValueType{extism.ValueTypePTR},
+	)
+}
+
+func newKVStoreDeleteByPrefixHostFunction(service KVStoreService) extism.HostFunction {
+	return extism.NewHostFunctionWithStack(
+		"kvstore_deletebyprefix",
+		func(ctx context.Context, p *extism.CurrentPlugin, stack []uint64) {
+			// Read JSON request from plugin memory
+			reqBytes, err := p.ReadBytes(stack[0])
+			if err != nil {
+				kvstoreWriteError(p, stack, err)
+				return
+			}
+			var req KVStoreDeleteByPrefixRequest
+			if err := json.Unmarshal(reqBytes, &req); err != nil {
+				kvstoreWriteError(p, stack, err)
+				return
+			}
+
+			// Call the service method
+			deletedcount, svcErr := service.DeleteByPrefix(ctx, req.Prefix)
+			if svcErr != nil {
+				kvstoreWriteError(p, stack, svcErr)
+				return
+			}
+
+			// Write JSON response to plugin memory
+			resp := KVStoreDeleteByPrefixResponse{
+				DeletedCount: deletedcount,
+			}
+			kvstoreWriteResponse(p, stack, resp)
+		},
+		[]extism.ValueType{extism.ValueTypePTR},
+		[]extism.ValueType{extism.ValueTypePTR},
+	)
+}
+
 func newKVStoreGetStorageUsedHostFunction(service KVStoreService) extism.HostFunction {
 	return extism.NewHostFunctionWithStack(
 		"kvstore_getstorageused",