fix(plugins): add base64 handling for []byte and remove raw=true (#5121)

* fix(plugins): add base64 handling for []byte and remove raw=true

Go's json.Marshal automatically base64-encodes []byte fields, but Rust's
serde_json serializes Vec<u8> as a JSON array and Python's json.dumps
raises TypeError on bytes. This fixes both directions of plugin
communication by adding proper base64 encoding/decoding in generated
client code.

For Rust templates (client and capability): adds a base64_bytes serde
helper module with #[serde(with = "base64_bytes")] on all Vec<u8> fields,
and adds base64 as a dependency. For Python templates: wraps bytes params
with base64.b64encode() and responses with base64.b64decode().

Also removes the raw=true binary framing protocol from all templates,
the parser, and the Method type. The raw mechanism added complexity that
is no longer needed once []byte works properly over JSON.

* fix(plugins): update production code and tests for base64 migration

Remove raw=true annotation from SubsonicAPI.CallRaw, delete all raw
test fixtures, remove raw-related test cases from parser, generator, and
integration tests, and add new test cases validating base64 handling
for Rust and Python templates.

* fix(plugins): update golden files and regenerate production code

Update golden test fixtures for codec and comprehensive services to
include base64 handling for []byte fields. Regenerate all production
PDK code (Go, Rust, Python) and host wrappers to use standard JSON
with base64-encoded byte fields instead of binary framing protocol.

* refactor: remove base64 helper duplication from rust template

Signed-off-by: Deluan <deluan@navidrome.org>

* fix(plugins): add base64 dependency to capabilities' Cargo.toml

Signed-off-by: Deluan <deluan@navidrome.org>

---------

Signed-off-by: Deluan <deluan@navidrome.org>

plugins/pdk/go/host/nd_host_subsonicapi.go

@@ -8,7 +8,6 @@
 package host
 
 import (
-	"encoding/binary"
 	"encoding/json"
 	"errors"
 
@@ -38,6 +37,12 @@ type subsonicAPICallRawRequest struct {
 	Uri string `json:"uri"`
 }
 
+type subsonicAPICallRawResponse struct {
+	ContentType string `json:"contentType,omitempty"`
+	Data        []byte `json:"data,omitempty"`
+	Error       string `json:"error,omitempty"`
+}
+
 // SubsonicAPICall calls the subsonicapi_call host function.
 // Call executes a Subsonic API request and returns the JSON response.
 //
@@ -78,8 +83,8 @@ func SubsonicAPICall(uri string) (string, error) {
 
 // SubsonicAPICallRaw calls the subsonicapi_callraw host function.
 // CallRaw executes a Subsonic API request and returns the raw binary response.
-// Optimized for binary endpoints like getCoverArt and stream that return
-// non-JSON data. The response is returned as raw bytes without JSON encoding overhead.
+// Designed for binary endpoints like getCoverArt and stream that return
+// non-JSON data. The data is base64-encoded over JSON on the wire.
 func SubsonicAPICallRaw(uri string) (string, []byte, error) {
 	// Marshal request to JSON
 	req := subsonicAPICallRawRequest{
@@ -99,22 +104,16 @@ func SubsonicAPICallRaw(uri string) (string, []byte, error) {
 	responseMem := pdk.FindMemory(responsePtr)
 	responseBytes := responseMem.ReadBytes()
 
-	// Parse binary-framed response
-	if len(responseBytes) == 0 {
-		return "", nil, errors.New("empty response from host")
+	// Parse the response
+	var response subsonicAPICallRawResponse
+	if err := json.Unmarshal(responseBytes, &response); err != nil {
+		return "", nil, err
 	}
-	if responseBytes[0] == 0x01 { // error
-		return "", nil, errors.New(string(responseBytes[1:]))
+
+	// Convert Error field to Go error
+	if response.Error != "" {
+		return "", nil, errors.New(response.Error)
 	}
-	if responseBytes[0] != 0x00 {
-		return "", nil, errors.New("unknown response status")
-	}
-	if len(responseBytes) < 5 {
-		return "", nil, errors.New("malformed raw response: incomplete header")
-	}
-	ctLen := binary.BigEndian.Uint32(responseBytes[1:5])
-	if uint32(len(responseBytes)) < 5+ctLen {
-		return "", nil, errors.New("malformed raw response: content-type overflow")
-	}
-	return string(responseBytes[5 : 5+ctLen]), responseBytes[5+ctLen:], nil
+
+	return response.ContentType, response.Data, nil
 }

plugins/pdk/go/host/nd_host_subsonicapi_stub.go

@@ -42,8 +42,8 @@ func (m *mockSubsonicAPIService) CallRaw(uri string) (string, []byte, error) {
 
 // SubsonicAPICallRaw delegates to the mock instance.
 // CallRaw executes a Subsonic API request and returns the raw binary response.
-// Optimized for binary endpoints like getCoverArt and stream that return
-// non-JSON data. The response is returned as raw bytes without JSON encoding overhead.
+// Designed for binary endpoints like getCoverArt and stream that return
+// non-JSON data. The data is base64-encoded over JSON on the wire.
 func SubsonicAPICallRaw(uri string) (string, []byte, error) {
 	return SubsonicAPIMock.CallRaw(uri)
 }

plugins/pdk/python/host/nd_host_cache.py

@@ -12,6 +12,7 @@ from typing import Any
 
 import extism
 import json
+import base64
 
 
 class HostFunctionError(Exception):
@@ -337,7 +338,7 @@ Returns an error if the operation fails.
     """
     request = {
         "key": key,
-        "value": value,
+        "value": base64.b64encode(value).decode("ascii"),
         "ttlSeconds": ttl_seconds,
     }
     request_bytes = json.dumps(request).encode("utf-8")
@@ -382,7 +383,7 @@ or the stored value is not a byte slice, exists will be false.
         raise HostFunctionError(response["error"])
 
     return CacheGetBytesResult(
-        value=response.get("value", b""),
+        value=base64.b64decode(response.get("value", "")),
         exists=response.get("exists", False),
     )
 

plugins/pdk/python/host/nd_host_http.py

@@ -12,6 +12,7 @@ from typing import Any
 
 import extism
 import json
+import base64
 
 
 class HostFunctionError(Exception):

plugins/pdk/python/host/nd_host_kvstore.py

@@ -12,6 +12,7 @@ from typing import Any
 
 import extism
 import json
+import base64
 
 
 class HostFunctionError(Exception):
@@ -80,7 +81,7 @@ Returns an error if the storage limit would be exceeded or the operation fails.
     """
     request = {
         "key": key,
-        "value": value,
+        "value": base64.b64encode(value).decode("ascii"),
     }
     request_bytes = json.dumps(request).encode("utf-8")
     request_mem = extism.memory.alloc(request_bytes)
@@ -123,7 +124,7 @@ Returns the value and whether the key exists.
         raise HostFunctionError(response["error"])
 
     return KVStoreGetResult(
-        value=response.get("value", b""),
+        value=base64.b64decode(response.get("value", "")),
         exists=response.get("exists", False),
     )
 

plugins/pdk/python/host/nd_host_subsonicapi.py

@@ -8,11 +8,11 @@
 # main __init__.py file. Copy the needed functions from this file into your plugin.
 
 from dataclasses import dataclass
-from typing import Any, Tuple
+from typing import Any
 
 import extism
 import json
-import struct
+import base64
 
 
 class HostFunctionError(Exception):
@@ -32,6 +32,13 @@ def _subsonicapi_callraw(offset: int) -> int:
     ...
 
 
+@dataclass
+class SubsonicAPICallRawResult:
+    """Result type for subsonicapi_call_raw."""
+    content_type: str
+    data: bytes
+
+
 def subsonicapi_call(uri: str) -> str:
     """Call executes a Subsonic API request and returns the JSON response.
 
@@ -62,16 +69,16 @@ e.g., "getAlbumList2?type=random&size=10". The response is returned as raw JSON.
     return response.get("responseJson", "")
 
 
-def subsonicapi_call_raw(uri: str) -> Tuple[str, bytes]:
+def subsonicapi_call_raw(uri: str) -> SubsonicAPICallRawResult:
     """CallRaw executes a Subsonic API request and returns the raw binary response.
-Optimized for binary endpoints like getCoverArt and stream that return
-non-JSON data. The response is returned as raw bytes without JSON encoding overhead.
+Designed for binary endpoints like getCoverArt and stream that return
+non-JSON data. The data is base64-encoded over JSON on the wire.
 
     Args:
         uri: str parameter.
 
     Returns:
-        Tuple of (content_type, data) with the raw binary response.
+        SubsonicAPICallRawResult containing content_type, data,.
 
     Raises:
         HostFunctionError: If the host function returns an error.
@@ -83,19 +90,12 @@ non-JSON data. The response is returned as raw bytes without JSON encoding overh
     request_mem = extism.memory.alloc(request_bytes)
     response_offset = _subsonicapi_callraw(request_mem.offset)
     response_mem = extism.memory.find(response_offset)
-    response_bytes = response_mem.bytes()
+    response = json.loads(extism.memory.string(response_mem))
 
-    if len(response_bytes) == 0:
-        raise HostFunctionError("empty response from host")
-    if response_bytes[0] == 0x01:
-        raise HostFunctionError(response_bytes[1:].decode("utf-8"))
-    if response_bytes[0] != 0x00:
-        raise HostFunctionError("unknown response status")
-    if len(response_bytes) < 5:
-        raise HostFunctionError("malformed raw response: incomplete header")
-    ct_len = struct.unpack(">I", response_bytes[1:5])[0]
-    if len(response_bytes) < 5 + ct_len:
-        raise HostFunctionError("malformed raw response: content-type overflow")
-    content_type = response_bytes[5:5 + ct_len].decode("utf-8")
-    data = response_bytes[5 + ct_len:]
-    return content_type, data
+    if response.get("error"):
+        raise HostFunctionError(response["error"])
+
+    return SubsonicAPICallRawResult(
+        content_type=response.get("contentType", ""),
+        data=base64.b64decode(response.get("data", "")),
+    )

plugins/pdk/python/host/nd_host_websocket.py

@@ -12,6 +12,7 @@ from typing import Any
 
 import extism
 import json
+import base64
 
 
 class HostFunctionError(Exception):
@@ -134,7 +135,7 @@ Returns an error if the connection is not found or if sending fails.
     """
     request = {
         "connectionId": connection_id,
-        "data": data,
+        "data": base64.b64encode(data).decode("ascii"),
     }
     request_bytes = json.dumps(request).encode("utf-8")
     request_mem = extism.memory.alloc(request_bytes)

plugins/pdk/rust/nd-pdk-capabilities/Cargo.toml

@@ -11,6 +11,7 @@ path = "src/lib.rs"
 crate-type = ["rlib"]
 
 [dependencies]
+base64 = "0.22"
 extism-pdk = "1.2"
 serde = { version = "1.0", features = ["derive"] }
 serde_json = "1.0"

plugins/pdk/rust/nd-pdk-host/Cargo.toml

@@ -11,6 +11,7 @@ readme = "README.md"
 crate-type = ["rlib"]
 
 [dependencies]
+base64 = "0.22"
 extism-pdk = "1.2"
 serde = { version = "1.0", features = ["derive"] }
 serde_json = "1.0"

plugins/pdk/rust/nd-pdk-host/src/nd_host_cache.rs

@@ -5,6 +5,29 @@
 
 use extism_pdk::*;
 use serde::{Deserialize, Serialize};
+use base64::Engine as _;
+use base64::engine::general_purpose::STANDARD as BASE64;
+
+mod base64_bytes {
+    use serde::{self, Deserialize, Deserializer, Serializer};
+    use base64::Engine as _;
+    use base64::engine::general_purpose::STANDARD as BASE64;
+
+    pub fn serialize<S>(bytes: &Vec<u8>, serializer: S) -> Result<S::Ok, S::Error>
+    where
+        S: Serializer,
+    {
+        serializer.serialize_str(&BASE64.encode(bytes))
+    }
+
+    pub fn deserialize<'de, D>(deserializer: D) -> Result<Vec<u8>, D::Error>
+    where
+        D: Deserializer<'de>,
+    {
+        let s = String::deserialize(deserializer)?;
+        BASE64.decode(&s).map_err(serde::de::Error::custom)
+    }
+}
 
 #[derive(Debug, Clone, Serialize)]
 #[serde(rename_all = "camelCase")]
@@ -106,6 +129,7 @@ struct CacheGetFloatResponse {
 #[serde(rename_all = "camelCase")]
 struct CacheSetBytesRequest {
     key: String,
+    #[serde(with = "base64_bytes")]
     value: Vec<u8>,
     ttl_seconds: i64,
 }
@@ -127,6 +151,7 @@ struct CacheGetBytesRequest {
 #[serde(rename_all = "camelCase")]
 struct CacheGetBytesResponse {
     #[serde(default)]
+    #[serde(with = "base64_bytes")]
     value: Vec<u8>,
     #[serde(default)]
     exists: bool,

plugins/pdk/rust/nd-pdk-host/src/nd_host_http.rs

@@ -5,6 +5,29 @@
 
 use extism_pdk::*;
 use serde::{Deserialize, Serialize};
+use base64::Engine as _;
+use base64::engine::general_purpose::STANDARD as BASE64;
+
+mod base64_bytes {
+    use serde::{self, Deserialize, Deserializer, Serializer};
+    use base64::Engine as _;
+    use base64::engine::general_purpose::STANDARD as BASE64;
+
+    pub fn serialize<S>(bytes: &Vec<u8>, serializer: S) -> Result<S::Ok, S::Error>
+    where
+        S: Serializer,
+    {
+        serializer.serialize_str(&BASE64.encode(bytes))
+    }
+
+    pub fn deserialize<'de, D>(deserializer: D) -> Result<Vec<u8>, D::Error>
+    where
+        D: Deserializer<'de>,
+    {
+        let s = String::deserialize(deserializer)?;
+        BASE64.decode(&s).map_err(serde::de::Error::custom)
+    }
+}
 
 /// HTTPRequest represents an outbound HTTP request from a plugin.
 #[derive(Debug, Clone, Serialize, Deserialize)]
@@ -15,6 +38,7 @@ pub struct HTTPRequest {
     #[serde(default)]
     pub headers: std::collections::HashMap<String, String>,
     #[serde(default)]
+    #[serde(with = "base64_bytes")]
     pub body: Vec<u8>,
     #[serde(default)]
     pub timeout_ms: i32,
@@ -28,6 +52,7 @@ pub struct HTTPResponse {
     #[serde(default)]
     pub headers: std::collections::HashMap<String, String>,
     #[serde(default)]
+    #[serde(with = "base64_bytes")]
     pub body: Vec<u8>,
 }
 

plugins/pdk/rust/nd-pdk-host/src/nd_host_kvstore.rs

@@ -5,11 +5,35 @@
 
 use extism_pdk::*;
 use serde::{Deserialize, Serialize};
+use base64::Engine as _;
+use base64::engine::general_purpose::STANDARD as BASE64;
+
+mod base64_bytes {
+    use serde::{self, Deserialize, Deserializer, Serializer};
+    use base64::Engine as _;
+    use base64::engine::general_purpose::STANDARD as BASE64;
+
+    pub fn serialize<S>(bytes: &Vec<u8>, serializer: S) -> Result<S::Ok, S::Error>
+    where
+        S: Serializer,
+    {
+        serializer.serialize_str(&BASE64.encode(bytes))
+    }
+
+    pub fn deserialize<'de, D>(deserializer: D) -> Result<Vec<u8>, D::Error>
+    where
+        D: Deserializer<'de>,
+    {
+        let s = String::deserialize(deserializer)?;
+        BASE64.decode(&s).map_err(serde::de::Error::custom)
+    }
+}
 
 #[derive(Debug, Clone, Serialize)]
 #[serde(rename_all = "camelCase")]
 struct KVStoreSetRequest {
     key: String,
+    #[serde(with = "base64_bytes")]
     value: Vec<u8>,
 }
 
@@ -30,6 +54,7 @@ struct KVStoreGetRequest {
 #[serde(rename_all = "camelCase")]
 struct KVStoreGetResponse {
     #[serde(default)]
+    #[serde(with = "base64_bytes")]
     value: Vec<u8>,
     #[serde(default)]
     exists: bool,

plugins/pdk/rust/nd-pdk-host/src/nd_host_subsonicapi.rs

@@ -5,6 +5,29 @@
 
 use extism_pdk::*;
 use serde::{Deserialize, Serialize};
+use base64::Engine as _;
+use base64::engine::general_purpose::STANDARD as BASE64;
+
+mod base64_bytes {
+    use serde::{self, Deserialize, Deserializer, Serializer};
+    use base64::Engine as _;
+    use base64::engine::general_purpose::STANDARD as BASE64;
+
+    pub fn serialize<S>(bytes: &Vec<u8>, serializer: S) -> Result<S::Ok, S::Error>
+    where
+        S: Serializer,
+    {
+        serializer.serialize_str(&BASE64.encode(bytes))
+    }
+
+    pub fn deserialize<'de, D>(deserializer: D) -> Result<Vec<u8>, D::Error>
+    where
+        D: Deserializer<'de>,
+    {
+        let s = String::deserialize(deserializer)?;
+        BASE64.decode(&s).map_err(serde::de::Error::custom)
+    }
+}
 
 #[derive(Debug, Clone, Serialize)]
 #[serde(rename_all = "camelCase")]
@@ -27,14 +50,22 @@ struct SubsonicAPICallRawRequest {
     uri: String,
 }
 
+#[derive(Debug, Clone, Deserialize)]
+#[serde(rename_all = "camelCase")]
+struct SubsonicAPICallRawResponse {
+    #[serde(default)]
+    content_type: String,
+    #[serde(default)]
+    #[serde(with = "base64_bytes")]
+    data: Vec<u8>,
+    #[serde(default)]
+    error: Option<String>,
+}
+
 #[host_fn]
 extern "ExtismHost" {
     fn subsonicapi_call(input: Json<SubsonicAPICallRequest>) -> Json<SubsonicAPICallResponse>;
-}
-
-#[link(wasm_import_module = "extism:host/user")]
-extern "C" {
-    fn subsonicapi_callraw(offset: u64) -> u64;
+    fn subsonicapi_callraw(input: Json<SubsonicAPICallRawRequest>) -> Json<SubsonicAPICallRawResponse>;
 }
 
 /// Call executes a Subsonic API request and returns the JSON response.
@@ -65,54 +96,27 @@ pub fn call(uri: &str) -> Result<String, Error> {
 }
 
 /// CallRaw executes a Subsonic API request and returns the raw binary response.
-/// Optimized for binary endpoints like getCoverArt and stream that return
-/// non-JSON data. The response is returned as raw bytes without JSON encoding overhead.
+/// Designed for binary endpoints like getCoverArt and stream that return
+/// non-JSON data. The data is base64-encoded over JSON on the wire.
 ///
 /// # Arguments
 /// * `uri` - String parameter.
 ///
 /// # Returns
-/// A tuple of (content_type, data) with the raw binary response.
+/// A tuple of (content_type, data).
 ///
 /// # Errors
 /// Returns an error if the host function call fails.
 pub fn call_raw(uri: &str) -> Result<(String, Vec<u8>), Error> {
-    let req = SubsonicAPICallRawRequest {
-        uri: uri.to_owned(),
+    let response = unsafe {
+        subsonicapi_callraw(Json(SubsonicAPICallRawRequest {
+            uri: uri.to_owned(),
+        }))?
     };
-    let input_bytes = serde_json::to_vec(&req).map_err(|e| Error::msg(e.to_string()))?;
-    let input_mem = Memory::from_bytes(&input_bytes).map_err(|e| Error::msg(e.to_string()))?;
 
-    let response_offset = unsafe { subsonicapi_callraw(input_mem.offset()) };
+    if let Some(err) = response.0.error {
+        return Err(Error::msg(err));
+    }
 
-    let response_mem = Memory::find(response_offset)
-        .ok_or_else(|| Error::msg("empty response from host"))?;
-    let response_bytes = response_mem.to_vec();
-
-    if response_bytes.is_empty() {
-        return Err(Error::msg("empty response from host"));
-    }
-    if response_bytes[0] == 0x01 {
-        let msg = String::from_utf8_lossy(&response_bytes[1..]).to_string();
-        return Err(Error::msg(msg));
-    }
-    if response_bytes[0] != 0x00 {
-        return Err(Error::msg("unknown response status"));
-    }
-    if response_bytes.len() < 5 {
-        return Err(Error::msg("malformed raw response: incomplete header"));
-    }
-    let ct_len = u32::from_be_bytes([
-        response_bytes[1],
-        response_bytes[2],
-        response_bytes[3],
-        response_bytes[4],
-    ]) as usize;
-    if ct_len > response_bytes.len() - 5 {
-        return Err(Error::msg("malformed raw response: content-type overflow"));
-    }
-    let ct_end = 5 + ct_len;
-    let content_type = String::from_utf8_lossy(&response_bytes[5..ct_end]).to_string();
-    let data = response_bytes[ct_end..].to_vec();
-    Ok((content_type, data))
+    Ok((response.0.content_type, response.0.data))
 }

plugins/pdk/rust/nd-pdk-host/src/nd_host_websocket.rs

@@ -5,6 +5,29 @@
 
 use extism_pdk::*;
 use serde::{Deserialize, Serialize};
+use base64::Engine as _;
+use base64::engine::general_purpose::STANDARD as BASE64;
+
+mod base64_bytes {
+    use serde::{self, Deserialize, Deserializer, Serializer};
+    use base64::Engine as _;
+    use base64::engine::general_purpose::STANDARD as BASE64;
+
+    pub fn serialize<S>(bytes: &Vec<u8>, serializer: S) -> Result<S::Ok, S::Error>
+    where
+        S: Serializer,
+    {
+        serializer.serialize_str(&BASE64.encode(bytes))
+    }
+
+    pub fn deserialize<'de, D>(deserializer: D) -> Result<Vec<u8>, D::Error>
+    where
+        D: Deserializer<'de>,
+    {
+        let s = String::deserialize(deserializer)?;
+        BASE64.decode(&s).map_err(serde::de::Error::custom)
+    }
+}
 
 #[derive(Debug, Clone, Serialize)]
 #[serde(rename_all = "camelCase")]
@@ -41,6 +64,7 @@ struct WebSocketSendTextResponse {
 #[serde(rename_all = "camelCase")]
 struct WebSocketSendBinaryRequest {
     connection_id: String,
+    #[serde(with = "base64_bytes")]
     data: Vec<u8>,
 }