feat(plugins): add TaskQueue host service for persistent background task queues (#5116)

* feat(plugins): define TaskQueue host service interface Add the TaskQueueService interface with CreateQueue, Enqueue, GetTaskStatus, and CancelTask methods plus QueueConfig struct. * feat(plugins): define TaskWorker capability for task execution callbacks * feat(plugins): add taskqueue permission to manifest schema Add TaskQueuePermission with maxConcurrency option. * feat(plugins): implement TaskQueue service with SQLite persistence and workers Per-plugin SQLite database with queues and tasks tables. Worker goroutines dequeue tasks and invoke nd_task_execute callback. Exponential backoff retries, rate limiting via delayMs, automatic cleanup of terminal tasks. * feat(plugins): require TaskWorker capability for taskqueue permission * feat(plugins): register TaskQueue host service in manager * feat(plugins): add test-taskqueue plugin for integration testing * feat(plugins): add integration tests for TaskQueue host service * docs: document TaskQueue module for persistent task queues Signed-off-by: Deluan <deluan@navidrome.org> * fix(plugins): harden TaskQueue host service with validation and safety improvements Add input validation (queue name length, payload size limits), extract status string constants to eliminate raw SQL literals, make CreateQueue idempotent via upsert for crash recovery, fix RetentionMs default check for negative values, cap exponential backoff at 1 hour to prevent overflow, and replace manual mutex-based delay enforcement with rate.Limiter from golang.org/x/time/rate for correct concurrent worker serialization. * refactor(plugins): remove capability check for TaskWorker in TaskQueue host service Signed-off-by: Deluan <deluan@navidrome.org> * fix(plugins): use context-aware database execution in TaskQueue host service Signed-off-by: Deluan <deluan@navidrome.org> * refactor(plugins): streamline task queue configuration and error handling Signed-off-by: Deluan <deluan@navidrome.org> * feat(plugins): increase maxConcurrency for task queue and handle budget exhaustion Signed-off-by: Deluan <deluan@navidrome.org> * refactor(plugins): simplify goroutine management in task queue service Signed-off-by: Deluan <deluan@navidrome.org> * feat(plugins): update TaskWorker interface to return status messages and refactor task queue service Signed-off-by: Deluan <deluan@navidrome.org> * feat(plugins): add ClearQueue function to remove pending tasks from a specified queue Signed-off-by: Deluan <deluan@navidrome.org> * refactor(plugins): use migrateDB for task queue schema and fix constant name collision Replaced the raw db.Exec call in createTaskQueueSchema with migrateDB, matching the pattern used by createKVStoreSchema. This enables version-tracked schema migrations via SQLite's PRAGMA user_version, allowing future schema changes to be appended incrementally. Also renamed cleanupInterval to taskCleanupInterval to resolve a redeclaration conflict with host_kvstore.go. * regenerate PDKs Signed-off-by: Deluan <deluan@navidrome.org> --------- Signed-off-by: Deluan <deluan@navidrome.org>
2026-03-03 13:48:49 -05:00
parent 24ba655dc3
commit 668869b6c7
24 changed files with 3513 additions and 0 deletions
@@ -9,4 +9,5 @@ pub mod lifecycle;
 pub mod metadata;
 pub mod scheduler;
 pub mod scrobbler;
+pub mod taskworker;
 pub mod websocket;
@@ -0,0 +1,102 @@
+// Code generated by ndpgen. DO NOT EDIT.
+//
+// This file contains export wrappers for the TaskWorker capability.
+// It is intended for use in Navidrome plugins built with extism-pdk.
+
+use serde::{Deserialize, Serialize};
+use base64::Engine as _;
+use base64::engine::general_purpose::STANDARD as BASE64;
+
+mod base64_bytes {
+    use serde::{self, Deserialize, Deserializer, Serializer};
+    use base64::Engine as _;
+    use base64::engine::general_purpose::STANDARD as BASE64;
+
+    pub fn serialize<S>(bytes: &Vec<u8>, serializer: S) -> Result<S::Ok, S::Error>
+    where
+        S: Serializer,
+    {
+        serializer.serialize_str(&BASE64.encode(bytes))
+    }
+
+    pub fn deserialize<'de, D>(deserializer: D) -> Result<Vec<u8>, D::Error>
+    where
+        D: Deserializer<'de>,
+    {
+        let s = String::deserialize(deserializer)?;
+        BASE64.decode(&s).map_err(serde::de::Error::custom)
+    }
+}
+
+// Helper functions for skip_serializing_if with numeric types
+#[allow(dead_code)]
+fn is_zero_i32(value: &i32) -> bool { *value == 0 }
+#[allow(dead_code)]
+fn is_zero_u32(value: &u32) -> bool { *value == 0 }
+#[allow(dead_code)]
+fn is_zero_i64(value: &i64) -> bool { *value == 0 }
+#[allow(dead_code)]
+fn is_zero_u64(value: &u64) -> bool { *value == 0 }
+#[allow(dead_code)]
+fn is_zero_f32(value: &f32) -> bool { *value == 0.0 }
+#[allow(dead_code)]
+fn is_zero_f64(value: &f64) -> bool { *value == 0.0 }
+/// TaskExecuteRequest is the request provided when a task is ready to execute.
+#[derive(Debug, Clone, Default, Serialize, Deserialize)]
+#[serde(rename_all = "camelCase")]
+pub struct TaskExecuteRequest {
+    /// QueueName is the name of the queue this task belongs to.
+    #[serde(default)]
+    pub queue_name: String,
+    /// TaskID is the unique identifier for this task.
+    #[serde(default)]
+    pub task_id: String,
+    /// Payload is the opaque data provided when the task was enqueued.
+    #[serde(default)]
+    #[serde(with = "base64_bytes")]
+    pub payload: Vec<u8>,
+    /// Attempt is the current attempt number (1-based: first attempt = 1).
+    #[serde(default)]
+    pub attempt: i32,
+}
+
+/// Error represents an error from a capability method.
+#[derive(Debug)]
+pub struct Error {
+    pub message: String,
+}
+
+impl std::fmt::Display for Error {
+    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+        write!(f, "{}", self.message)
+    }
+}
+
+impl std::error::Error for Error {}
+
+impl Error {
+    pub fn new(message: impl Into<String>) -> Self {
+        Self { message: message.into() }
+    }
+}
+
+/// TaskExecuteProvider provides the OnTaskExecute function.
+pub trait TaskExecuteProvider {
+    fn on_task_execute(&self, req: TaskExecuteRequest) -> Result<String, Error>;
+}
+
+/// Register the on_task_execute export.
+/// This macro generates the WASM export function for this method.
+#[macro_export]
+macro_rules! register_taskworker_task_execute {
+    ($plugin_type:ty) => {
+        #[extism_pdk::plugin_fn]
+        pub fn nd_task_execute(
+            req: extism_pdk::Json<$crate::taskworker::TaskExecuteRequest>
+        ) -> extism_pdk::FnResult<extism_pdk::Json<String>> {
+            let plugin = <$plugin_type>::default();
+            let result = $crate::taskworker::TaskExecuteProvider::on_task_execute(&plugin, req.into_inner())?;
+            Ok(extism_pdk::Json(result))
+        }
+    };
+}