feat(plugins): add TTL support, batch operations, and hardening to kvstore (#5127)
* feat(plugins): add expires_at column to kvstore schema * feat(plugins): filter expired keys in kvstore Get, Has, List * feat(plugins): add periodic cleanup of expired kvstore keys * feat(plugins): add SetWithTTL, DeleteByPrefix, and GetMany to kvstore Add three new methods to the KVStore host service: - SetWithTTL: store key-value pairs with automatic expiration - DeleteByPrefix: remove all keys matching a prefix in one operation - GetMany: retrieve multiple values in a single call All methods include comprehensive unit tests covering edge cases, expiration behavior, size tracking, and LIKE-special characters. * feat(plugins): regenerate code and update test plugin for new kvstore methods Regenerate host function wrappers and PDK bindings for Go, Python, and Rust. Update the test-kvstore plugin to exercise SetWithTTL, DeleteByPrefix, and GetMany. * feat(plugins): add integration tests for new kvstore methods Add WASM integration tests for SetWithTTL, DeleteByPrefix, and GetMany operations through the plugin boundary, verifying end-to-end behavior including TTL expiration, prefix deletion, and batch retrieval. * fix(plugins): address lint issues in kvstore implementation Handle tx.Rollback error return and suppress gosec false positive for parameterized SQL query construction in GetMany. * fix(plugins): Set clears expires_at when overwriting a TTL'd key Previously, calling Set() on a key that was stored with SetWithTTL() would leave the expires_at value intact, causing the key to silently expire even though Set implies permanent storage. Also excludes expired keys from currentSize calculation at startup. * refactor(plugins): simplify kvstore by removing in-memory size cache Replaced the in-memory currentSize cache (atomic.Int64), periodic cleanup timer, and mutex with direct database queries for storage accounting. This eliminates race conditions and cache drift issues at negligible performance cost for plugin-sized datasets. Also unified Set and SetWithTTL into a shared setValue method, simplified DeleteByPrefix to use RowsAffected instead of a transaction, and added an index on expires_at for efficient expiration filtering. * feat(plugins): add generic SQLite migration helper and refactor kvstore schema Add a reusable migrateDB helper that tracks schema versions via SQLite's PRAGMA user_version and applies pending migrations transactionally. Replace the ad-hoc createKVStoreSchema function in kvstore with a declarative migrations slice, making it easy to add future schema changes. Remove the now-redundant schema migration test since migrateDB has its own test suite and every kvstore test exercises the migrations implicitly. Signed-off-by: Deluan <deluan@navidrome.org> * fix(plugins): harden kvstore with explicit NULL handling, prefix validation, and cleanup timeout - Use sql.NullString for expires_at to explicitly send NULL instead of relying on datetime('now', '') returning NULL by accident - Reject empty prefix in DeleteByPrefix to prevent accidental data wipe - Add 5s timeout context to cleanupExpired on Close - Replace time.Sleep in unit tests with pre-expired timestamps Signed-off-by: Deluan <deluan@navidrome.org> * refactor(plugins): use batch processing in GetMany Process keys in chunks of 200 using slice.CollectChunks to avoid hitting SQLite's SQLITE_MAX_VARIABLE_NUMBER limit with large key sets. * feat(plugins): add periodic cleanup goroutine for expired kvstore keys Use the manager's context to control a background goroutine that purges expired keys every hour, stopping naturally on shutdown when the context is cancelled. --------- Signed-off-by: Deluan <deluan@navidrome.org>
This commit is contained in:
@@ -19,15 +19,20 @@ import (
|
||||
//go:wasmimport extism:host/user kvstore_set
|
||||
func kvstore_set(uint64) uint64
|
||||
|
||||
// kvstore_setwithttl is the host function provided by Navidrome.
|
||||
//
|
||||
//go:wasmimport extism:host/user kvstore_setwithttl
|
||||
func kvstore_setwithttl(uint64) uint64
|
||||
|
||||
// kvstore_get is the host function provided by Navidrome.
|
||||
//
|
||||
//go:wasmimport extism:host/user kvstore_get
|
||||
func kvstore_get(uint64) uint64
|
||||
|
||||
// kvstore_delete is the host function provided by Navidrome.
|
||||
// kvstore_getmany is the host function provided by Navidrome.
|
||||
//
|
||||
//go:wasmimport extism:host/user kvstore_delete
|
||||
func kvstore_delete(uint64) uint64
|
||||
//go:wasmimport extism:host/user kvstore_getmany
|
||||
func kvstore_getmany(uint64) uint64
|
||||
|
||||
// kvstore_has is the host function provided by Navidrome.
|
||||
//
|
||||
@@ -39,6 +44,16 @@ func kvstore_has(uint64) uint64
|
||||
//go:wasmimport extism:host/user kvstore_list
|
||||
func kvstore_list(uint64) uint64
|
||||
|
||||
// kvstore_delete is the host function provided by Navidrome.
|
||||
//
|
||||
//go:wasmimport extism:host/user kvstore_delete
|
||||
func kvstore_delete(uint64) uint64
|
||||
|
||||
// kvstore_deletebyprefix is the host function provided by Navidrome.
|
||||
//
|
||||
//go:wasmimport extism:host/user kvstore_deletebyprefix
|
||||
func kvstore_deletebyprefix(uint64) uint64
|
||||
|
||||
// kvstore_getstorageused is the host function provided by Navidrome.
|
||||
//
|
||||
//go:wasmimport extism:host/user kvstore_getstorageused
|
||||
@@ -49,6 +64,12 @@ type kVStoreSetRequest struct {
|
||||
Value []byte `json:"value"`
|
||||
}
|
||||
|
||||
type kVStoreSetWithTTLRequest struct {
|
||||
Key string `json:"key"`
|
||||
Value []byte `json:"value"`
|
||||
TtlSeconds int64 `json:"ttlSeconds"`
|
||||
}
|
||||
|
||||
type kVStoreGetRequest struct {
|
||||
Key string `json:"key"`
|
||||
}
|
||||
@@ -59,8 +80,13 @@ type kVStoreGetResponse struct {
|
||||
Error string `json:"error,omitempty"`
|
||||
}
|
||||
|
||||
type kVStoreDeleteRequest struct {
|
||||
Key string `json:"key"`
|
||||
type kVStoreGetManyRequest struct {
|
||||
Keys []string `json:"keys"`
|
||||
}
|
||||
|
||||
type kVStoreGetManyResponse struct {
|
||||
Values map[string][]byte `json:"values,omitempty"`
|
||||
Error string `json:"error,omitempty"`
|
||||
}
|
||||
|
||||
type kVStoreHasRequest struct {
|
||||
@@ -81,6 +107,19 @@ type kVStoreListResponse struct {
|
||||
Error string `json:"error,omitempty"`
|
||||
}
|
||||
|
||||
type kVStoreDeleteRequest struct {
|
||||
Key string `json:"key"`
|
||||
}
|
||||
|
||||
type kVStoreDeleteByPrefixRequest struct {
|
||||
Prefix string `json:"prefix"`
|
||||
}
|
||||
|
||||
type kVStoreDeleteByPrefixResponse struct {
|
||||
DeletedCount int64 `json:"deletedCount,omitempty"`
|
||||
Error string `json:"error,omitempty"`
|
||||
}
|
||||
|
||||
type kVStoreGetStorageUsedResponse struct {
|
||||
Bytes int64 `json:"bytes,omitempty"`
|
||||
Error string `json:"error,omitempty"`
|
||||
@@ -127,6 +166,52 @@ func KVStoreSet(key string, value []byte) error {
|
||||
return nil
|
||||
}
|
||||
|
||||
// KVStoreSetWithTTL calls the kvstore_setwithttl host function.
|
||||
// SetWithTTL stores a byte value with the given key and a time-to-live.
|
||||
//
|
||||
// After ttlSeconds, the key is treated as non-existent and will be
|
||||
// cleaned up lazily. ttlSeconds must be greater than 0.
|
||||
//
|
||||
// Parameters:
|
||||
// - key: The storage key (max 256 bytes, UTF-8)
|
||||
// - value: The byte slice to store
|
||||
// - ttlSeconds: Time-to-live in seconds (must be > 0)
|
||||
//
|
||||
// Returns an error if the storage limit would be exceeded or the operation fails.
|
||||
func KVStoreSetWithTTL(key string, value []byte, ttlSeconds int64) error {
|
||||
// Marshal request to JSON
|
||||
req := kVStoreSetWithTTLRequest{
|
||||
Key: key,
|
||||
Value: value,
|
||||
TtlSeconds: ttlSeconds,
|
||||
}
|
||||
reqBytes, err := json.Marshal(req)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
reqMem := pdk.AllocateBytes(reqBytes)
|
||||
defer reqMem.Free()
|
||||
|
||||
// Call the host function
|
||||
responsePtr := kvstore_setwithttl(reqMem.Offset())
|
||||
|
||||
// Read the response from memory
|
||||
responseMem := pdk.FindMemory(responsePtr)
|
||||
responseBytes := responseMem.ReadBytes()
|
||||
|
||||
// Parse error-only response
|
||||
var response struct {
|
||||
Error string `json:"error,omitempty"`
|
||||
}
|
||||
if err := json.Unmarshal(responseBytes, &response); err != nil {
|
||||
return err
|
||||
}
|
||||
if response.Error != "" {
|
||||
return errors.New(response.Error)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// KVStoreGet calls the kvstore_get host function.
|
||||
// Get retrieves a byte value from storage.
|
||||
//
|
||||
@@ -167,43 +252,45 @@ func KVStoreGet(key string) ([]byte, bool, error) {
|
||||
return response.Value, response.Exists, nil
|
||||
}
|
||||
|
||||
// KVStoreDelete calls the kvstore_delete host function.
|
||||
// Delete removes a value from storage.
|
||||
// KVStoreGetMany calls the kvstore_getmany host function.
|
||||
// GetMany retrieves multiple values in a single call.
|
||||
//
|
||||
// Parameters:
|
||||
// - key: The storage key
|
||||
// - keys: The storage keys to retrieve
|
||||
//
|
||||
// Returns an error if the operation fails. Does not return an error if the key doesn't exist.
|
||||
func KVStoreDelete(key string) error {
|
||||
// Returns a map of key to value for keys that exist and have not expired.
|
||||
// Missing or expired keys are omitted from the result.
|
||||
func KVStoreGetMany(keys []string) (map[string][]byte, error) {
|
||||
// Marshal request to JSON
|
||||
req := kVStoreDeleteRequest{
|
||||
Key: key,
|
||||
req := kVStoreGetManyRequest{
|
||||
Keys: keys,
|
||||
}
|
||||
reqBytes, err := json.Marshal(req)
|
||||
if err != nil {
|
||||
return err
|
||||
return nil, err
|
||||
}
|
||||
reqMem := pdk.AllocateBytes(reqBytes)
|
||||
defer reqMem.Free()
|
||||
|
||||
// Call the host function
|
||||
responsePtr := kvstore_delete(reqMem.Offset())
|
||||
responsePtr := kvstore_getmany(reqMem.Offset())
|
||||
|
||||
// Read the response from memory
|
||||
responseMem := pdk.FindMemory(responsePtr)
|
||||
responseBytes := responseMem.ReadBytes()
|
||||
|
||||
// Parse error-only response
|
||||
var response struct {
|
||||
Error string `json:"error,omitempty"`
|
||||
}
|
||||
// Parse the response
|
||||
var response kVStoreGetManyResponse
|
||||
if err := json.Unmarshal(responseBytes, &response); err != nil {
|
||||
return err
|
||||
return nil, err
|
||||
}
|
||||
|
||||
// Convert Error field to Go error
|
||||
if response.Error != "" {
|
||||
return errors.New(response.Error)
|
||||
return nil, errors.New(response.Error)
|
||||
}
|
||||
return nil
|
||||
|
||||
return response.Values, nil
|
||||
}
|
||||
|
||||
// KVStoreHas calls the kvstore_has host function.
|
||||
@@ -286,6 +373,85 @@ func KVStoreList(prefix string) ([]string, error) {
|
||||
return response.Keys, nil
|
||||
}
|
||||
|
||||
// KVStoreDelete calls the kvstore_delete host function.
|
||||
// Delete removes a value from storage.
|
||||
//
|
||||
// Parameters:
|
||||
// - key: The storage key
|
||||
//
|
||||
// Returns an error if the operation fails. Does not return an error if the key doesn't exist.
|
||||
func KVStoreDelete(key string) error {
|
||||
// Marshal request to JSON
|
||||
req := kVStoreDeleteRequest{
|
||||
Key: key,
|
||||
}
|
||||
reqBytes, err := json.Marshal(req)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
reqMem := pdk.AllocateBytes(reqBytes)
|
||||
defer reqMem.Free()
|
||||
|
||||
// Call the host function
|
||||
responsePtr := kvstore_delete(reqMem.Offset())
|
||||
|
||||
// Read the response from memory
|
||||
responseMem := pdk.FindMemory(responsePtr)
|
||||
responseBytes := responseMem.ReadBytes()
|
||||
|
||||
// Parse error-only response
|
||||
var response struct {
|
||||
Error string `json:"error,omitempty"`
|
||||
}
|
||||
if err := json.Unmarshal(responseBytes, &response); err != nil {
|
||||
return err
|
||||
}
|
||||
if response.Error != "" {
|
||||
return errors.New(response.Error)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// KVStoreDeleteByPrefix calls the kvstore_deletebyprefix host function.
|
||||
// DeleteByPrefix removes all keys matching the given prefix.
|
||||
//
|
||||
// Parameters:
|
||||
// - prefix: Key prefix to match (must not be empty)
|
||||
//
|
||||
// Returns the number of keys deleted. Includes expired keys.
|
||||
func KVStoreDeleteByPrefix(prefix string) (int64, error) {
|
||||
// Marshal request to JSON
|
||||
req := kVStoreDeleteByPrefixRequest{
|
||||
Prefix: prefix,
|
||||
}
|
||||
reqBytes, err := json.Marshal(req)
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
reqMem := pdk.AllocateBytes(reqBytes)
|
||||
defer reqMem.Free()
|
||||
|
||||
// Call the host function
|
||||
responsePtr := kvstore_deletebyprefix(reqMem.Offset())
|
||||
|
||||
// Read the response from memory
|
||||
responseMem := pdk.FindMemory(responsePtr)
|
||||
responseBytes := responseMem.ReadBytes()
|
||||
|
||||
// Parse the response
|
||||
var response kVStoreDeleteByPrefixResponse
|
||||
if err := json.Unmarshal(responseBytes, &response); err != nil {
|
||||
return 0, err
|
||||
}
|
||||
|
||||
// Convert Error field to Go error
|
||||
if response.Error != "" {
|
||||
return 0, errors.New(response.Error)
|
||||
}
|
||||
|
||||
return response.DeletedCount, nil
|
||||
}
|
||||
|
||||
// KVStoreGetStorageUsed calls the kvstore_getstorageused host function.
|
||||
// GetStorageUsed returns the total storage used by this plugin in bytes.
|
||||
func KVStoreGetStorageUsed() (int64, error) {
|
||||
|
||||
@@ -37,6 +37,28 @@ func KVStoreSet(key string, value []byte) error {
|
||||
return KVStoreMock.Set(key, value)
|
||||
}
|
||||
|
||||
// SetWithTTL is the mock method for KVStoreSetWithTTL.
|
||||
func (m *mockKVStoreService) SetWithTTL(key string, value []byte, ttlSeconds int64) error {
|
||||
args := m.Called(key, value, ttlSeconds)
|
||||
return args.Error(0)
|
||||
}
|
||||
|
||||
// KVStoreSetWithTTL delegates to the mock instance.
|
||||
// SetWithTTL stores a byte value with the given key and a time-to-live.
|
||||
//
|
||||
// After ttlSeconds, the key is treated as non-existent and will be
|
||||
// cleaned up lazily. ttlSeconds must be greater than 0.
|
||||
//
|
||||
// Parameters:
|
||||
// - key: The storage key (max 256 bytes, UTF-8)
|
||||
// - value: The byte slice to store
|
||||
// - ttlSeconds: Time-to-live in seconds (must be > 0)
|
||||
//
|
||||
// Returns an error if the storage limit would be exceeded or the operation fails.
|
||||
func KVStoreSetWithTTL(key string, value []byte, ttlSeconds int64) error {
|
||||
return KVStoreMock.SetWithTTL(key, value, ttlSeconds)
|
||||
}
|
||||
|
||||
// Get is the mock method for KVStoreGet.
|
||||
func (m *mockKVStoreService) Get(key string) ([]byte, bool, error) {
|
||||
args := m.Called(key)
|
||||
@@ -54,21 +76,22 @@ func KVStoreGet(key string) ([]byte, bool, error) {
|
||||
return KVStoreMock.Get(key)
|
||||
}
|
||||
|
||||
// Delete is the mock method for KVStoreDelete.
|
||||
func (m *mockKVStoreService) Delete(key string) error {
|
||||
args := m.Called(key)
|
||||
return args.Error(0)
|
||||
// GetMany is the mock method for KVStoreGetMany.
|
||||
func (m *mockKVStoreService) GetMany(keys []string) (map[string][]byte, error) {
|
||||
args := m.Called(keys)
|
||||
return args.Get(0).(map[string][]byte), args.Error(1)
|
||||
}
|
||||
|
||||
// KVStoreDelete delegates to the mock instance.
|
||||
// Delete removes a value from storage.
|
||||
// KVStoreGetMany delegates to the mock instance.
|
||||
// GetMany retrieves multiple values in a single call.
|
||||
//
|
||||
// Parameters:
|
||||
// - key: The storage key
|
||||
// - keys: The storage keys to retrieve
|
||||
//
|
||||
// Returns an error if the operation fails. Does not return an error if the key doesn't exist.
|
||||
func KVStoreDelete(key string) error {
|
||||
return KVStoreMock.Delete(key)
|
||||
// Returns a map of key to value for keys that exist and have not expired.
|
||||
// Missing or expired keys are omitted from the result.
|
||||
func KVStoreGetMany(keys []string) (map[string][]byte, error) {
|
||||
return KVStoreMock.GetMany(keys)
|
||||
}
|
||||
|
||||
// Has is the mock method for KVStoreHas.
|
||||
@@ -105,6 +128,40 @@ func KVStoreList(prefix string) ([]string, error) {
|
||||
return KVStoreMock.List(prefix)
|
||||
}
|
||||
|
||||
// Delete is the mock method for KVStoreDelete.
|
||||
func (m *mockKVStoreService) Delete(key string) error {
|
||||
args := m.Called(key)
|
||||
return args.Error(0)
|
||||
}
|
||||
|
||||
// KVStoreDelete delegates to the mock instance.
|
||||
// Delete removes a value from storage.
|
||||
//
|
||||
// Parameters:
|
||||
// - key: The storage key
|
||||
//
|
||||
// Returns an error if the operation fails. Does not return an error if the key doesn't exist.
|
||||
func KVStoreDelete(key string) error {
|
||||
return KVStoreMock.Delete(key)
|
||||
}
|
||||
|
||||
// DeleteByPrefix is the mock method for KVStoreDeleteByPrefix.
|
||||
func (m *mockKVStoreService) DeleteByPrefix(prefix string) (int64, error) {
|
||||
args := m.Called(prefix)
|
||||
return args.Get(0).(int64), args.Error(1)
|
||||
}
|
||||
|
||||
// KVStoreDeleteByPrefix delegates to the mock instance.
|
||||
// DeleteByPrefix removes all keys matching the given prefix.
|
||||
//
|
||||
// Parameters:
|
||||
// - prefix: Key prefix to match (must not be empty)
|
||||
//
|
||||
// Returns the number of keys deleted. Includes expired keys.
|
||||
func KVStoreDeleteByPrefix(prefix string) (int64, error) {
|
||||
return KVStoreMock.DeleteByPrefix(prefix)
|
||||
}
|
||||
|
||||
// GetStorageUsed is the mock method for KVStoreGetStorageUsed.
|
||||
func (m *mockKVStoreService) GetStorageUsed() (int64, error) {
|
||||
args := m.Called()
|
||||
|
||||
@@ -26,14 +26,20 @@ def _kvstore_set(offset: int) -> int:
|
||||
...
|
||||
|
||||
|
||||
@extism.import_fn("extism:host/user", "kvstore_setwithttl")
|
||||
def _kvstore_setwithttl(offset: int) -> int:
|
||||
"""Raw host function - do not call directly."""
|
||||
...
|
||||
|
||||
|
||||
@extism.import_fn("extism:host/user", "kvstore_get")
|
||||
def _kvstore_get(offset: int) -> int:
|
||||
"""Raw host function - do not call directly."""
|
||||
...
|
||||
|
||||
|
||||
@extism.import_fn("extism:host/user", "kvstore_delete")
|
||||
def _kvstore_delete(offset: int) -> int:
|
||||
@extism.import_fn("extism:host/user", "kvstore_getmany")
|
||||
def _kvstore_getmany(offset: int) -> int:
|
||||
"""Raw host function - do not call directly."""
|
||||
...
|
||||
|
||||
@@ -50,6 +56,18 @@ def _kvstore_list(offset: int) -> int:
|
||||
...
|
||||
|
||||
|
||||
@extism.import_fn("extism:host/user", "kvstore_delete")
|
||||
def _kvstore_delete(offset: int) -> int:
|
||||
"""Raw host function - do not call directly."""
|
||||
...
|
||||
|
||||
|
||||
@extism.import_fn("extism:host/user", "kvstore_deletebyprefix")
|
||||
def _kvstore_deletebyprefix(offset: int) -> int:
|
||||
"""Raw host function - do not call directly."""
|
||||
...
|
||||
|
||||
|
||||
@extism.import_fn("extism:host/user", "kvstore_getstorageused")
|
||||
def _kvstore_getstorageused(offset: int) -> int:
|
||||
"""Raw host function - do not call directly."""
|
||||
@@ -94,6 +112,43 @@ Returns an error if the storage limit would be exceeded or the operation fails.
|
||||
|
||||
|
||||
|
||||
def kvstore_set_with_ttl(key: str, value: bytes, ttl_seconds: int) -> None:
|
||||
"""SetWithTTL stores a byte value with the given key and a time-to-live.
|
||||
|
||||
After ttlSeconds, the key is treated as non-existent and will be
|
||||
cleaned up lazily. ttlSeconds must be greater than 0.
|
||||
|
||||
Parameters:
|
||||
- key: The storage key (max 256 bytes, UTF-8)
|
||||
- value: The byte slice to store
|
||||
- ttlSeconds: Time-to-live in seconds (must be > 0)
|
||||
|
||||
Returns an error if the storage limit would be exceeded or the operation fails.
|
||||
|
||||
Args:
|
||||
key: str parameter.
|
||||
value: bytes parameter.
|
||||
ttl_seconds: int parameter.
|
||||
|
||||
Raises:
|
||||
HostFunctionError: If the host function returns an error.
|
||||
"""
|
||||
request = {
|
||||
"key": key,
|
||||
"value": base64.b64encode(value).decode("ascii"),
|
||||
"ttlSeconds": ttl_seconds,
|
||||
}
|
||||
request_bytes = json.dumps(request).encode("utf-8")
|
||||
request_mem = extism.memory.alloc(request_bytes)
|
||||
response_offset = _kvstore_setwithttl(request_mem.offset)
|
||||
response_mem = extism.memory.find(response_offset)
|
||||
response = json.loads(extism.memory.string(response_mem))
|
||||
|
||||
if response.get("error"):
|
||||
raise HostFunctionError(response["error"])
|
||||
|
||||
|
||||
|
||||
def kvstore_get(key: str) -> KVStoreGetResult:
|
||||
"""Get retrieves a byte value from storage.
|
||||
|
||||
@@ -129,32 +184,37 @@ Returns the value and whether the key exists.
|
||||
)
|
||||
|
||||
|
||||
def kvstore_delete(key: str) -> None:
|
||||
"""Delete removes a value from storage.
|
||||
def kvstore_get_many(keys: Any) -> Any:
|
||||
"""GetMany retrieves multiple values in a single call.
|
||||
|
||||
Parameters:
|
||||
- key: The storage key
|
||||
- keys: The storage keys to retrieve
|
||||
|
||||
Returns an error if the operation fails. Does not return an error if the key doesn't exist.
|
||||
Returns a map of key to value for keys that exist and have not expired.
|
||||
Missing or expired keys are omitted from the result.
|
||||
|
||||
Args:
|
||||
key: str parameter.
|
||||
keys: Any parameter.
|
||||
|
||||
Returns:
|
||||
Any: The result value.
|
||||
|
||||
Raises:
|
||||
HostFunctionError: If the host function returns an error.
|
||||
"""
|
||||
request = {
|
||||
"key": key,
|
||||
"keys": keys,
|
||||
}
|
||||
request_bytes = json.dumps(request).encode("utf-8")
|
||||
request_mem = extism.memory.alloc(request_bytes)
|
||||
response_offset = _kvstore_delete(request_mem.offset)
|
||||
response_offset = _kvstore_getmany(request_mem.offset)
|
||||
response_mem = extism.memory.find(response_offset)
|
||||
response = json.loads(extism.memory.string(response_mem))
|
||||
|
||||
if response.get("error"):
|
||||
raise HostFunctionError(response["error"])
|
||||
|
||||
return response.get("values", None)
|
||||
|
||||
|
||||
def kvstore_has(key: str) -> bool:
|
||||
@@ -221,6 +281,66 @@ Returns a slice of matching keys.
|
||||
return response.get("keys", None)
|
||||
|
||||
|
||||
def kvstore_delete(key: str) -> None:
|
||||
"""Delete removes a value from storage.
|
||||
|
||||
Parameters:
|
||||
- key: The storage key
|
||||
|
||||
Returns an error if the operation fails. Does not return an error if the key doesn't exist.
|
||||
|
||||
Args:
|
||||
key: str parameter.
|
||||
|
||||
Raises:
|
||||
HostFunctionError: If the host function returns an error.
|
||||
"""
|
||||
request = {
|
||||
"key": key,
|
||||
}
|
||||
request_bytes = json.dumps(request).encode("utf-8")
|
||||
request_mem = extism.memory.alloc(request_bytes)
|
||||
response_offset = _kvstore_delete(request_mem.offset)
|
||||
response_mem = extism.memory.find(response_offset)
|
||||
response = json.loads(extism.memory.string(response_mem))
|
||||
|
||||
if response.get("error"):
|
||||
raise HostFunctionError(response["error"])
|
||||
|
||||
|
||||
|
||||
def kvstore_delete_by_prefix(prefix: str) -> int:
|
||||
"""DeleteByPrefix removes all keys matching the given prefix.
|
||||
|
||||
Parameters:
|
||||
- prefix: Key prefix to match (must not be empty)
|
||||
|
||||
Returns the number of keys deleted. Includes expired keys.
|
||||
|
||||
Args:
|
||||
prefix: str parameter.
|
||||
|
||||
Returns:
|
||||
int: The result value.
|
||||
|
||||
Raises:
|
||||
HostFunctionError: If the host function returns an error.
|
||||
"""
|
||||
request = {
|
||||
"prefix": prefix,
|
||||
}
|
||||
request_bytes = json.dumps(request).encode("utf-8")
|
||||
request_mem = extism.memory.alloc(request_bytes)
|
||||
response_offset = _kvstore_deletebyprefix(request_mem.offset)
|
||||
response_mem = extism.memory.find(response_offset)
|
||||
response = json.loads(extism.memory.string(response_mem))
|
||||
|
||||
if response.get("error"):
|
||||
raise HostFunctionError(response["error"])
|
||||
|
||||
return response.get("deletedCount", 0)
|
||||
|
||||
|
||||
def kvstore_get_storage_used() -> int:
|
||||
"""GetStorageUsed returns the total storage used by this plugin in bytes.
|
||||
|
||||
|
||||
@@ -44,6 +44,22 @@ struct KVStoreSetResponse {
|
||||
error: Option<String>,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Serialize)]
|
||||
#[serde(rename_all = "camelCase")]
|
||||
struct KVStoreSetWithTTLRequest {
|
||||
key: String,
|
||||
#[serde(with = "base64_bytes")]
|
||||
value: Vec<u8>,
|
||||
ttl_seconds: i64,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Deserialize)]
|
||||
#[serde(rename_all = "camelCase")]
|
||||
struct KVStoreSetWithTTLResponse {
|
||||
#[serde(default)]
|
||||
error: Option<String>,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Serialize)]
|
||||
#[serde(rename_all = "camelCase")]
|
||||
struct KVStoreGetRequest {
|
||||
@@ -64,13 +80,15 @@ struct KVStoreGetResponse {
|
||||
|
||||
#[derive(Debug, Clone, Serialize)]
|
||||
#[serde(rename_all = "camelCase")]
|
||||
struct KVStoreDeleteRequest {
|
||||
key: String,
|
||||
struct KVStoreGetManyRequest {
|
||||
keys: Vec<String>,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Deserialize)]
|
||||
#[serde(rename_all = "camelCase")]
|
||||
struct KVStoreDeleteResponse {
|
||||
struct KVStoreGetManyResponse {
|
||||
#[serde(default)]
|
||||
values: std::collections::HashMap<String, Vec<u8>>,
|
||||
#[serde(default)]
|
||||
error: Option<String>,
|
||||
}
|
||||
@@ -105,6 +123,34 @@ struct KVStoreListResponse {
|
||||
error: Option<String>,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Serialize)]
|
||||
#[serde(rename_all = "camelCase")]
|
||||
struct KVStoreDeleteRequest {
|
||||
key: String,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Deserialize)]
|
||||
#[serde(rename_all = "camelCase")]
|
||||
struct KVStoreDeleteResponse {
|
||||
#[serde(default)]
|
||||
error: Option<String>,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Serialize)]
|
||||
#[serde(rename_all = "camelCase")]
|
||||
struct KVStoreDeleteByPrefixRequest {
|
||||
prefix: String,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Deserialize)]
|
||||
#[serde(rename_all = "camelCase")]
|
||||
struct KVStoreDeleteByPrefixResponse {
|
||||
#[serde(default)]
|
||||
deleted_count: i64,
|
||||
#[serde(default)]
|
||||
error: Option<String>,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Deserialize)]
|
||||
#[serde(rename_all = "camelCase")]
|
||||
struct KVStoreGetStorageUsedResponse {
|
||||
@@ -117,10 +163,13 @@ struct KVStoreGetStorageUsedResponse {
|
||||
#[host_fn]
|
||||
extern "ExtismHost" {
|
||||
fn kvstore_set(input: Json<KVStoreSetRequest>) -> Json<KVStoreSetResponse>;
|
||||
fn kvstore_setwithttl(input: Json<KVStoreSetWithTTLRequest>) -> Json<KVStoreSetWithTTLResponse>;
|
||||
fn kvstore_get(input: Json<KVStoreGetRequest>) -> Json<KVStoreGetResponse>;
|
||||
fn kvstore_delete(input: Json<KVStoreDeleteRequest>) -> Json<KVStoreDeleteResponse>;
|
||||
fn kvstore_getmany(input: Json<KVStoreGetManyRequest>) -> Json<KVStoreGetManyResponse>;
|
||||
fn kvstore_has(input: Json<KVStoreHasRequest>) -> Json<KVStoreHasResponse>;
|
||||
fn kvstore_list(input: Json<KVStoreListRequest>) -> Json<KVStoreListResponse>;
|
||||
fn kvstore_delete(input: Json<KVStoreDeleteRequest>) -> Json<KVStoreDeleteResponse>;
|
||||
fn kvstore_deletebyprefix(input: Json<KVStoreDeleteByPrefixRequest>) -> Json<KVStoreDeleteByPrefixResponse>;
|
||||
fn kvstore_getstorageused(input: Json<serde_json::Value>) -> Json<KVStoreGetStorageUsedResponse>;
|
||||
}
|
||||
|
||||
@@ -153,6 +202,41 @@ pub fn set(key: &str, value: Vec<u8>) -> Result<(), Error> {
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// SetWithTTL stores a byte value with the given key and a time-to-live.
|
||||
///
|
||||
/// After ttlSeconds, the key is treated as non-existent and will be
|
||||
/// cleaned up lazily. ttlSeconds must be greater than 0.
|
||||
///
|
||||
/// Parameters:
|
||||
/// - key: The storage key (max 256 bytes, UTF-8)
|
||||
/// - value: The byte slice to store
|
||||
/// - ttlSeconds: Time-to-live in seconds (must be > 0)
|
||||
///
|
||||
/// Returns an error if the storage limit would be exceeded or the operation fails.
|
||||
///
|
||||
/// # Arguments
|
||||
/// * `key` - String parameter.
|
||||
/// * `value` - Vec<u8> parameter.
|
||||
/// * `ttl_seconds` - i64 parameter.
|
||||
///
|
||||
/// # Errors
|
||||
/// Returns an error if the host function call fails.
|
||||
pub fn set_with_ttl(key: &str, value: Vec<u8>, ttl_seconds: i64) -> Result<(), Error> {
|
||||
let response = unsafe {
|
||||
kvstore_setwithttl(Json(KVStoreSetWithTTLRequest {
|
||||
key: key.to_owned(),
|
||||
value: value,
|
||||
ttl_seconds: ttl_seconds,
|
||||
}))?
|
||||
};
|
||||
|
||||
if let Some(err) = response.0.error {
|
||||
return Err(Error::msg(err));
|
||||
}
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Get retrieves a byte value from storage.
|
||||
///
|
||||
/// Parameters:
|
||||
@@ -186,22 +270,26 @@ pub fn get(key: &str) -> Result<Option<Vec<u8>>, Error> {
|
||||
}
|
||||
}
|
||||
|
||||
/// Delete removes a value from storage.
|
||||
/// GetMany retrieves multiple values in a single call.
|
||||
///
|
||||
/// Parameters:
|
||||
/// - key: The storage key
|
||||
/// - keys: The storage keys to retrieve
|
||||
///
|
||||
/// Returns an error if the operation fails. Does not return an error if the key doesn't exist.
|
||||
/// Returns a map of key to value for keys that exist and have not expired.
|
||||
/// Missing or expired keys are omitted from the result.
|
||||
///
|
||||
/// # Arguments
|
||||
/// * `key` - String parameter.
|
||||
/// * `keys` - Vec<String> parameter.
|
||||
///
|
||||
/// # Returns
|
||||
/// The values value.
|
||||
///
|
||||
/// # Errors
|
||||
/// Returns an error if the host function call fails.
|
||||
pub fn delete(key: &str) -> Result<(), Error> {
|
||||
pub fn get_many(keys: Vec<String>) -> Result<std::collections::HashMap<String, Vec<u8>>, Error> {
|
||||
let response = unsafe {
|
||||
kvstore_delete(Json(KVStoreDeleteRequest {
|
||||
key: key.to_owned(),
|
||||
kvstore_getmany(Json(KVStoreGetManyRequest {
|
||||
keys: keys,
|
||||
}))?
|
||||
};
|
||||
|
||||
@@ -209,7 +297,7 @@ pub fn delete(key: &str) -> Result<(), Error> {
|
||||
return Err(Error::msg(err));
|
||||
}
|
||||
|
||||
Ok(())
|
||||
Ok(response.0.values)
|
||||
}
|
||||
|
||||
/// Has checks if a key exists in storage.
|
||||
@@ -270,6 +358,61 @@ pub fn list(prefix: &str) -> Result<Vec<String>, Error> {
|
||||
Ok(response.0.keys)
|
||||
}
|
||||
|
||||
/// Delete removes a value from storage.
|
||||
///
|
||||
/// Parameters:
|
||||
/// - key: The storage key
|
||||
///
|
||||
/// Returns an error if the operation fails. Does not return an error if the key doesn't exist.
|
||||
///
|
||||
/// # Arguments
|
||||
/// * `key` - String parameter.
|
||||
///
|
||||
/// # Errors
|
||||
/// Returns an error if the host function call fails.
|
||||
pub fn delete(key: &str) -> Result<(), Error> {
|
||||
let response = unsafe {
|
||||
kvstore_delete(Json(KVStoreDeleteRequest {
|
||||
key: key.to_owned(),
|
||||
}))?
|
||||
};
|
||||
|
||||
if let Some(err) = response.0.error {
|
||||
return Err(Error::msg(err));
|
||||
}
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// DeleteByPrefix removes all keys matching the given prefix.
|
||||
///
|
||||
/// Parameters:
|
||||
/// - prefix: Key prefix to match (must not be empty)
|
||||
///
|
||||
/// Returns the number of keys deleted. Includes expired keys.
|
||||
///
|
||||
/// # Arguments
|
||||
/// * `prefix` - String parameter.
|
||||
///
|
||||
/// # Returns
|
||||
/// The deleted_count value.
|
||||
///
|
||||
/// # Errors
|
||||
/// Returns an error if the host function call fails.
|
||||
pub fn delete_by_prefix(prefix: &str) -> Result<i64, Error> {
|
||||
let response = unsafe {
|
||||
kvstore_deletebyprefix(Json(KVStoreDeleteByPrefixRequest {
|
||||
prefix: prefix.to_owned(),
|
||||
}))?
|
||||
};
|
||||
|
||||
if let Some(err) = response.0.error {
|
||||
return Err(Error::msg(err));
|
||||
}
|
||||
|
||||
Ok(response.0.deleted_count)
|
||||
}
|
||||
|
||||
/// GetStorageUsed returns the total storage used by this plugin in bytes.
|
||||
///
|
||||
/// # Returns
|
||||
|
||||
Reference in New Issue
Block a user