feat(plugins): add TTL support, batch operations, and hardening to kvstore (#5127)

* feat(plugins): add expires_at column to kvstore schema * feat(plugins): filter expired keys in kvstore Get, Has, List * feat(plugins): add periodic cleanup of expired kvstore keys * feat(plugins): add SetWithTTL, DeleteByPrefix, and GetMany to kvstore Add three new methods to the KVStore host service: - SetWithTTL: store key-value pairs with automatic expiration - DeleteByPrefix: remove all keys matching a prefix in one operation - GetMany: retrieve multiple values in a single call All methods include comprehensive unit tests covering edge cases, expiration behavior, size tracking, and LIKE-special characters. * feat(plugins): regenerate code and update test plugin for new kvstore methods Regenerate host function wrappers and PDK bindings for Go, Python, and Rust. Update the test-kvstore plugin to exercise SetWithTTL, DeleteByPrefix, and GetMany. * feat(plugins): add integration tests for new kvstore methods Add WASM integration tests for SetWithTTL, DeleteByPrefix, and GetMany operations through the plugin boundary, verifying end-to-end behavior including TTL expiration, prefix deletion, and batch retrieval. * fix(plugins): address lint issues in kvstore implementation Handle tx.Rollback error return and suppress gosec false positive for parameterized SQL query construction in GetMany. * fix(plugins): Set clears expires_at when overwriting a TTL'd key Previously, calling Set() on a key that was stored with SetWithTTL() would leave the expires_at value intact, causing the key to silently expire even though Set implies permanent storage. Also excludes expired keys from currentSize calculation at startup. * refactor(plugins): simplify kvstore by removing in-memory size cache Replaced the in-memory currentSize cache (atomic.Int64), periodic cleanup timer, and mutex with direct database queries for storage accounting. This eliminates race conditions and cache drift issues at negligible performance cost for plugin-sized datasets. Also unified Set and SetWithTTL into a shared setValue method, simplified DeleteByPrefix to use RowsAffected instead of a transaction, and added an index on expires_at for efficient expiration filtering. * feat(plugins): add generic SQLite migration helper and refactor kvstore schema Add a reusable migrateDB helper that tracks schema versions via SQLite's PRAGMA user_version and applies pending migrations transactionally. Replace the ad-hoc createKVStoreSchema function in kvstore with a declarative migrations slice, making it easy to add future schema changes. Remove the now-redundant schema migration test since migrateDB has its own test suite and every kvstore test exercises the migrations implicitly. Signed-off-by: Deluan <deluan@navidrome.org> * fix(plugins): harden kvstore with explicit NULL handling, prefix validation, and cleanup timeout - Use sql.NullString for expires_at to explicitly send NULL instead of relying on datetime('now', '') returning NULL by accident - Reject empty prefix in DeleteByPrefix to prevent accidental data wipe - Add 5s timeout context to cleanupExpired on Close - Replace time.Sleep in unit tests with pre-expired timestamps Signed-off-by: Deluan <deluan@navidrome.org> * refactor(plugins): use batch processing in GetMany Process keys in chunks of 200 using slice.CollectChunks to avoid hitting SQLite's SQLITE_MAX_VARIABLE_NUMBER limit with large key sets. * feat(plugins): add periodic cleanup goroutine for expired kvstore keys Use the manager's context to control a background goroutine that purges expired keys every hour, stopping naturally on shutdown when the context is cancelled. --------- Signed-off-by: Deluan <deluan@navidrome.org>
2026-02-28 23:12:17 -05:00
parent d9a215e1e3
commit 2471bb9cf6
12 changed files with 1528 additions and 158 deletions
@@ -44,6 +44,22 @@ struct KVStoreSetResponse {
    error: Option<String>,
 }

+#[derive(Debug, Clone, Serialize)]
+#[serde(rename_all = "camelCase")]
+struct KVStoreSetWithTTLRequest {
+    key: String,
+    #[serde(with = "base64_bytes")]
+    value: Vec<u8>,
+    ttl_seconds: i64,
+}
+
+#[derive(Debug, Clone, Deserialize)]
+#[serde(rename_all = "camelCase")]
+struct KVStoreSetWithTTLResponse {
+    #[serde(default)]
+    error: Option<String>,
+}
+
 #[derive(Debug, Clone, Serialize)]
 #[serde(rename_all = "camelCase")]
 struct KVStoreGetRequest {
@@ -64,13 +80,15 @@ struct KVStoreGetResponse {

 #[derive(Debug, Clone, Serialize)]
 #[serde(rename_all = "camelCase")]
-struct KVStoreDeleteRequest {
-    key: String,
+struct KVStoreGetManyRequest {
+    keys: Vec<String>,
 }

 #[derive(Debug, Clone, Deserialize)]
 #[serde(rename_all = "camelCase")]
-struct KVStoreDeleteResponse {
+struct KVStoreGetManyResponse {
+    #[serde(default)]
+    values: std::collections::HashMap<String, Vec<u8>>,
    #[serde(default)]
    error: Option<String>,
 }
@@ -105,6 +123,34 @@ struct KVStoreListResponse {
    error: Option<String>,
 }

+#[derive(Debug, Clone, Serialize)]
+#[serde(rename_all = "camelCase")]
+struct KVStoreDeleteRequest {
+    key: String,
+}
+
+#[derive(Debug, Clone, Deserialize)]
+#[serde(rename_all = "camelCase")]
+struct KVStoreDeleteResponse {
+    #[serde(default)]
+    error: Option<String>,
+}
+
+#[derive(Debug, Clone, Serialize)]
+#[serde(rename_all = "camelCase")]
+struct KVStoreDeleteByPrefixRequest {
+    prefix: String,
+}
+
+#[derive(Debug, Clone, Deserialize)]
+#[serde(rename_all = "camelCase")]
+struct KVStoreDeleteByPrefixResponse {
+    #[serde(default)]
+    deleted_count: i64,
+    #[serde(default)]
+    error: Option<String>,
+}
+
 #[derive(Debug, Clone, Deserialize)]
 #[serde(rename_all = "camelCase")]
 struct KVStoreGetStorageUsedResponse {
@@ -117,10 +163,13 @@ struct KVStoreGetStorageUsedResponse {
 #[host_fn]
 extern "ExtismHost" {
    fn kvstore_set(input: Json<KVStoreSetRequest>) -> Json<KVStoreSetResponse>;
+    fn kvstore_setwithttl(input: Json<KVStoreSetWithTTLRequest>) -> Json<KVStoreSetWithTTLResponse>;
    fn kvstore_get(input: Json<KVStoreGetRequest>) -> Json<KVStoreGetResponse>;
-    fn kvstore_delete(input: Json<KVStoreDeleteRequest>) -> Json<KVStoreDeleteResponse>;
+    fn kvstore_getmany(input: Json<KVStoreGetManyRequest>) -> Json<KVStoreGetManyResponse>;
    fn kvstore_has(input: Json<KVStoreHasRequest>) -> Json<KVStoreHasResponse>;
    fn kvstore_list(input: Json<KVStoreListRequest>) -> Json<KVStoreListResponse>;
+    fn kvstore_delete(input: Json<KVStoreDeleteRequest>) -> Json<KVStoreDeleteResponse>;
+    fn kvstore_deletebyprefix(input: Json<KVStoreDeleteByPrefixRequest>) -> Json<KVStoreDeleteByPrefixResponse>;
    fn kvstore_getstorageused(input: Json<serde_json::Value>) -> Json<KVStoreGetStorageUsedResponse>;
 }

@@ -153,6 +202,41 @@ pub fn set(key: &str, value: Vec<u8>) -> Result<(), Error> {
    Ok(())
 }

+/// SetWithTTL stores a byte value with the given key and a time-to-live.
+/// 
+/// After ttlSeconds, the key is treated as non-existent and will be
+/// cleaned up lazily. ttlSeconds must be greater than 0.
+/// 
+/// Parameters:
+///   - key: The storage key (max 256 bytes, UTF-8)
+///   - value: The byte slice to store
+///   - ttlSeconds: Time-to-live in seconds (must be > 0)
+/// 
+/// Returns an error if the storage limit would be exceeded or the operation fails.
+///
+/// # Arguments
+/// * `key` - String parameter.
+/// * `value` - Vec<u8> parameter.
+/// * `ttl_seconds` - i64 parameter.
+///
+/// # Errors
+/// Returns an error if the host function call fails.
+pub fn set_with_ttl(key: &str, value: Vec<u8>, ttl_seconds: i64) -> Result<(), Error> {
+    let response = unsafe {
+        kvstore_setwithttl(Json(KVStoreSetWithTTLRequest {
+            key: key.to_owned(),
+            value: value,
+            ttl_seconds: ttl_seconds,
+        }))?
+    };
+
+    if let Some(err) = response.0.error {
+        return Err(Error::msg(err));
+    }
+
+    Ok(())
+}
+
 /// Get retrieves a byte value from storage.
 /// 
 /// Parameters:
@@ -186,22 +270,26 @@ pub fn get(key: &str) -> Result<Option<Vec<u8>>, Error> {
    }
 }

-/// Delete removes a value from storage.
+/// GetMany retrieves multiple values in a single call.
 /// 
 /// Parameters:
-///   - key: The storage key
+///   - keys: The storage keys to retrieve
 /// 
-/// Returns an error if the operation fails. Does not return an error if the key doesn't exist.
+/// Returns a map of key to value for keys that exist and have not expired.
+/// Missing or expired keys are omitted from the result.
 ///
 /// # Arguments
-/// * `key` - String parameter.
+/// * `keys` - Vec<String> parameter.
+///
+/// # Returns
+/// The values value.
 ///
 /// # Errors
 /// Returns an error if the host function call fails.
-pub fn delete(key: &str) -> Result<(), Error> {
+pub fn get_many(keys: Vec<String>) -> Result<std::collections::HashMap<String, Vec<u8>>, Error> {
    let response = unsafe {
-        kvstore_delete(Json(KVStoreDeleteRequest {
-            key: key.to_owned(),
+        kvstore_getmany(Json(KVStoreGetManyRequest {
+            keys: keys,
        }))?
    };

@@ -209,7 +297,7 @@ pub fn delete(key: &str) -> Result<(), Error> {
        return Err(Error::msg(err));
    }

-    Ok(())
+    Ok(response.0.values)
 }

 /// Has checks if a key exists in storage.
@@ -270,6 +358,61 @@ pub fn list(prefix: &str) -> Result<Vec<String>, Error> {
    Ok(response.0.keys)
 }

+/// Delete removes a value from storage.
+/// 
+/// Parameters:
+///   - key: The storage key
+/// 
+/// Returns an error if the operation fails. Does not return an error if the key doesn't exist.
+///
+/// # Arguments
+/// * `key` - String parameter.
+///
+/// # Errors
+/// Returns an error if the host function call fails.
+pub fn delete(key: &str) -> Result<(), Error> {
+    let response = unsafe {
+        kvstore_delete(Json(KVStoreDeleteRequest {
+            key: key.to_owned(),
+        }))?
+    };
+
+    if let Some(err) = response.0.error {
+        return Err(Error::msg(err));
+    }
+
+    Ok(())
+}
+
+/// DeleteByPrefix removes all keys matching the given prefix.
+/// 
+/// Parameters:
+///   - prefix: Key prefix to match (must not be empty)
+/// 
+/// Returns the number of keys deleted. Includes expired keys.
+///
+/// # Arguments
+/// * `prefix` - String parameter.
+///
+/// # Returns
+/// The deleted_count value.
+///
+/// # Errors
+/// Returns an error if the host function call fails.
+pub fn delete_by_prefix(prefix: &str) -> Result<i64, Error> {
+    let response = unsafe {
+        kvstore_deletebyprefix(Json(KVStoreDeleteByPrefixRequest {
+            prefix: prefix.to_owned(),
+        }))?
+    };
+
+    if let Some(err) = response.0.error {
+        return Err(Error::msg(err));
+    }
+
+    Ok(response.0.deleted_count)
+}
+
 /// GetStorageUsed returns the total storage used by this plugin in bytes.
 ///
 /// # Returns