fix(plugins): add base64 handling for []byte and remove raw=true (#5121)
* fix(plugins): add base64 handling for []byte and remove raw=true Go's json.Marshal automatically base64-encodes []byte fields, but Rust's serde_json serializes Vec<u8> as a JSON array and Python's json.dumps raises TypeError on bytes. This fixes both directions of plugin communication by adding proper base64 encoding/decoding in generated client code. For Rust templates (client and capability): adds a base64_bytes serde helper module with #[serde(with = "base64_bytes")] on all Vec<u8> fields, and adds base64 as a dependency. For Python templates: wraps bytes params with base64.b64encode() and responses with base64.b64decode(). Also removes the raw=true binary framing protocol from all templates, the parser, and the Method type. The raw mechanism added complexity that is no longer needed once []byte works properly over JSON. * fix(plugins): update production code and tests for base64 migration Remove raw=true annotation from SubsonicAPI.CallRaw, delete all raw test fixtures, remove raw-related test cases from parser, generator, and integration tests, and add new test cases validating base64 handling for Rust and Python templates. * fix(plugins): update golden files and regenerate production code Update golden test fixtures for codec and comprehensive services to include base64 handling for []byte fields. Regenerate all production PDK code (Go, Rust, Python) and host wrappers to use standard JSON with base64-encoded byte fields instead of binary framing protocol. * refactor: remove base64 helper duplication from rust template Signed-off-by: Deluan <deluan@navidrome.org> * fix(plugins): add base64 dependency to capabilities' Cargo.toml Signed-off-by: Deluan <deluan@navidrome.org> --------- Signed-off-by: Deluan <deluan@navidrome.org>
This commit is contained in:
Deluan Quintão
@@ -0,0 +1,25 @@
|
||||
{{define "base64_bytes_module"}}
|
||||
use base64::Engine as _;
|
||||
use base64::engine::general_purpose::STANDARD as BASE64;
|
||||
|
||||
mod base64_bytes {
|
||||
use serde::{self, Deserialize, Deserializer, Serializer};
|
||||
use base64::Engine as _;
|
||||
use base64::engine::general_purpose::STANDARD as BASE64;
|
||||
|
||||
pub fn serialize<S>(bytes: &Vec<u8>, serializer: S) -> Result<S::Ok, S::Error>
|
||||
where
|
||||
S: Serializer,
|
||||
{
|
||||
serializer.serialize_str(&BASE64.encode(bytes))
|
||||
}
|
||||
|
||||
pub fn deserialize<'de, D>(deserializer: D) -> Result<Vec<u8>, D::Error>
|
||||
where
|
||||
D: Deserializer<'de>,
|
||||
{
|
||||
let s = String::deserialize(deserializer)?;
|
||||
BASE64.decode(&s).map_err(serde::de::Error::custom)
|
||||
}
|
||||
}
|
||||
{{- end}}
|
||||
@@ -7,6 +7,7 @@ use serde::{Deserialize, Serialize};
|
||||
{{- if hasHashMap .Capability}}
|
||||
use std::collections::HashMap;
|
||||
{{- end}}
|
||||
{{- if .Capability.HasByteFields}}{{template "base64_bytes_module" .}}{{- end}}
|
||||
|
||||
// Helper functions for skip_serializing_if with numeric types
|
||||
#[allow(dead_code)]
|
||||
@@ -70,6 +71,9 @@ pub struct {{.Name}} {
|
||||
#[serde(default, skip_serializing_if = "{{skipSerializingFunc .Type}}")]
|
||||
{{- else}}
|
||||
#[serde(default)]
|
||||
{{- end}}
|
||||
{{- if .IsByteSlice}}
|
||||
#[serde(with = "base64_bytes")]
|
||||
{{- end}}
|
||||
pub {{rustFieldName .Name}}: {{fieldRustType .}},
|
||||
{{- end}}
|
||||
|
||||
@@ -8,9 +8,6 @@
|
||||
package {{.Package}}
|
||||
|
||||
import (
|
||||
{{- if .Service.HasRawMethods}}
|
||||
"encoding/binary"
|
||||
{{- end}}
|
||||
"encoding/json"
|
||||
{{- if .Service.HasErrors}}
|
||||
"errors"
|
||||
@@ -52,7 +49,7 @@ type {{requestType .}} struct {
|
||||
{{- end}}
|
||||
}
|
||||
{{- end}}
|
||||
{{- if and (not .IsErrorOnly) (not .Raw)}}
|
||||
{{- if not .IsErrorOnly}}
|
||||
|
||||
type {{responseType .}} struct {
|
||||
{{- range .Returns}}
|
||||
@@ -98,27 +95,7 @@ func {{$.Service.Name}}{{.Name}}({{range $i, $p := .Params}}{{if $i}}, {{end}}{{
|
||||
// Read the response from memory
|
||||
responseMem := pdk.FindMemory(responsePtr)
|
||||
responseBytes := responseMem.ReadBytes()
|
||||
{{- if .Raw}}
|
||||
|
||||
// Parse binary-framed response
|
||||
if len(responseBytes) == 0 {
|
||||
return "", nil, errors.New("empty response from host")
|
||||
}
|
||||
if responseBytes[0] == 0x01 { // error
|
||||
return "", nil, errors.New(string(responseBytes[1:]))
|
||||
}
|
||||
if responseBytes[0] != 0x00 {
|
||||
return "", nil, errors.New("unknown response status")
|
||||
}
|
||||
if len(responseBytes) < 5 {
|
||||
return "", nil, errors.New("malformed raw response: incomplete header")
|
||||
}
|
||||
ctLen := binary.BigEndian.Uint32(responseBytes[1:5])
|
||||
if uint32(len(responseBytes)) < 5+ctLen {
|
||||
return "", nil, errors.New("malformed raw response: content-type overflow")
|
||||
}
|
||||
return string(responseBytes[5 : 5+ctLen]), responseBytes[5+ctLen:], nil
|
||||
{{- else if .IsErrorOnly}}
|
||||
{{- if .IsErrorOnly}}
|
||||
|
||||
// Parse error-only response
|
||||
var response struct {
|
||||
|
||||
@@ -8,12 +8,12 @@
|
||||
# main __init__.py file. Copy the needed functions from this file into your plugin.
|
||||
|
||||
from dataclasses import dataclass
|
||||
from typing import Any{{- if .Service.HasRawMethods}}, Tuple{{end}}
|
||||
from typing import Any
|
||||
|
||||
import extism
|
||||
import json
|
||||
{{- if .Service.HasRawMethods}}
|
||||
import struct
|
||||
{{- if .Service.HasByteFields}}
|
||||
import base64
|
||||
{{- end}}
|
||||
|
||||
|
||||
@@ -32,7 +32,7 @@ def _{{exportName .}}(offset: int) -> int:
|
||||
{{- end}}
|
||||
{{- /* Generate dataclasses for multi-value returns */ -}}
|
||||
{{range .Service.Methods}}
|
||||
{{- if and .NeedsResultClass (not .Raw)}}
|
||||
{{- if .NeedsResultClass}}
|
||||
|
||||
|
||||
@dataclass
|
||||
@@ -47,7 +47,7 @@ class {{pythonResultType .}}:
|
||||
{{range .Service.Methods}}
|
||||
|
||||
|
||||
def {{pythonFunc .}}({{range $i, $p := .Params}}{{if $i}}, {{end}}{{$p.PythonName}}: {{$p.PythonType}}{{end}}){{if .Raw}} -> Tuple[str, bytes]{{else if .NeedsResultClass}} -> {{pythonResultType .}}{{else if .HasReturns}} -> {{(index .Returns 0).PythonType}}{{else}} -> None{{end}}:
|
||||
def {{pythonFunc .}}({{range $i, $p := .Params}}{{if $i}}, {{end}}{{$p.PythonName}}: {{$p.PythonType}}{{end}}){{if .NeedsResultClass}} -> {{pythonResultType .}}{{else if .HasReturns}} -> {{(index .Returns 0).PythonType}}{{else}} -> None{{end}}:
|
||||
"""{{if .Doc}}{{.Doc}}{{else}}Call the {{exportName .}} host function.{{end}}
|
||||
{{- if .HasParams}}
|
||||
|
||||
@@ -56,11 +56,7 @@ def {{pythonFunc .}}({{range $i, $p := .Params}}{{if $i}}, {{end}}{{$p.PythonNam
|
||||
{{.PythonName}}: {{.PythonType}} parameter.
|
||||
{{- end}}
|
||||
{{- end}}
|
||||
{{- if .Raw}}
|
||||
|
||||
Returns:
|
||||
Tuple of (content_type, data) with the raw binary response.
|
||||
{{- else if .HasReturns}}
|
||||
{{- if .HasReturns}}
|
||||
|
||||
Returns:
|
||||
{{- if .NeedsResultClass}}
|
||||
@@ -76,7 +72,11 @@ def {{pythonFunc .}}({{range $i, $p := .Params}}{{if $i}}, {{end}}{{$p.PythonNam
|
||||
{{- if .HasParams}}
|
||||
request = {
|
||||
{{- range .Params}}
|
||||
{{- if .IsByteSlice}}
|
||||
"{{.JSONName}}": base64.b64encode({{.PythonName}}).decode("ascii"),
|
||||
{{- else}}
|
||||
"{{.JSONName}}": {{.PythonName}},
|
||||
{{- end}}
|
||||
{{- end}}
|
||||
}
|
||||
request_bytes = json.dumps(request).encode("utf-8")
|
||||
@@ -86,24 +86,6 @@ def {{pythonFunc .}}({{range $i, $p := .Params}}{{if $i}}, {{end}}{{$p.PythonNam
|
||||
request_mem = extism.memory.alloc(request_bytes)
|
||||
response_offset = _{{exportName .}}(request_mem.offset)
|
||||
response_mem = extism.memory.find(response_offset)
|
||||
{{- if .Raw}}
|
||||
response_bytes = response_mem.bytes()
|
||||
|
||||
if len(response_bytes) == 0:
|
||||
raise HostFunctionError("empty response from host")
|
||||
if response_bytes[0] == 0x01:
|
||||
raise HostFunctionError(response_bytes[1:].decode("utf-8"))
|
||||
if response_bytes[0] != 0x00:
|
||||
raise HostFunctionError("unknown response status")
|
||||
if len(response_bytes) < 5:
|
||||
raise HostFunctionError("malformed raw response: incomplete header")
|
||||
ct_len = struct.unpack(">I", response_bytes[1:5])[0]
|
||||
if len(response_bytes) < 5 + ct_len:
|
||||
raise HostFunctionError("malformed raw response: content-type overflow")
|
||||
content_type = response_bytes[5:5 + ct_len].decode("utf-8")
|
||||
data = response_bytes[5 + ct_len:]
|
||||
return content_type, data
|
||||
{{- else}}
|
||||
response = json.loads(extism.memory.string(response_mem))
|
||||
{{if .HasError}}
|
||||
if response.get("error"):
|
||||
@@ -112,10 +94,17 @@ def {{pythonFunc .}}({{range $i, $p := .Params}}{{if $i}}, {{end}}{{$p.PythonNam
|
||||
{{- if .NeedsResultClass}}
|
||||
return {{pythonResultType .}}(
|
||||
{{- range .Returns}}
|
||||
{{- if .IsByteSlice}}
|
||||
{{.PythonName}}=base64.b64decode(response.get("{{.JSONName}}", "")),
|
||||
{{- else}}
|
||||
{{.PythonName}}=response.get("{{.JSONName}}"{{pythonDefault .}}),
|
||||
{{- end}}
|
||||
{{- end}}
|
||||
)
|
||||
{{- else if .HasReturns}}
|
||||
{{- if (index .Returns 0).IsByteSlice}}
|
||||
return base64.b64decode(response.get("{{(index .Returns 0).JSONName}}", ""))
|
||||
{{- else}}
|
||||
return response.get("{{(index .Returns 0).JSONName}}"{{pythonDefault (index .Returns 0)}})
|
||||
{{- end}}
|
||||
{{- end}}
|
||||
|
||||
@@ -5,6 +5,7 @@
|
||||
|
||||
use extism_pdk::*;
|
||||
use serde::{Deserialize, Serialize};
|
||||
{{- if .Service.HasByteFields}}{{template "base64_bytes_module" .}}{{- end}}
|
||||
{{- /* Generate struct definitions */ -}}
|
||||
{{- range .Service.Structs}}
|
||||
{{if .Doc}}
|
||||
@@ -16,6 +17,9 @@ pub struct {{.Name}} {
|
||||
{{- range .Fields}}
|
||||
{{- if .NeedsDefault}}
|
||||
#[serde(default)]
|
||||
{{- end}}
|
||||
{{- if .IsByteSlice}}
|
||||
#[serde(with = "base64_bytes")]
|
||||
{{- end}}
|
||||
pub {{.RustName}}: {{fieldRustType .}},
|
||||
{{- end}}
|
||||
@@ -29,17 +33,22 @@ pub struct {{.Name}} {
|
||||
#[serde(rename_all = "camelCase")]
|
||||
struct {{requestType .}} {
|
||||
{{- range .Params}}
|
||||
{{- if .IsByteSlice}}
|
||||
#[serde(with = "base64_bytes")]
|
||||
{{- end}}
|
||||
{{.RustName}}: {{rustType .}},
|
||||
{{- end}}
|
||||
}
|
||||
{{- end}}
|
||||
{{- if not .Raw}}
|
||||
|
||||
#[derive(Debug, Clone, Deserialize)]
|
||||
#[serde(rename_all = "camelCase")]
|
||||
struct {{responseType .}} {
|
||||
{{- range .Returns}}
|
||||
#[serde(default)]
|
||||
{{- if .IsByteSlice}}
|
||||
#[serde(with = "base64_bytes")]
|
||||
{{- end}}
|
||||
{{.RustName}}: {{rustType .}},
|
||||
{{- end}}
|
||||
{{- if .HasError}}
|
||||
@@ -48,92 +57,16 @@ struct {{responseType .}} {
|
||||
{{- end}}
|
||||
}
|
||||
{{- end}}
|
||||
{{- end}}
|
||||
|
||||
#[host_fn]
|
||||
extern "ExtismHost" {
|
||||
{{- range .Service.Methods}}
|
||||
{{- if not .Raw}}
|
||||
fn {{exportName .}}(input: Json<{{if .HasParams}}{{requestType .}}{{else}}serde_json::Value{{end}}>) -> Json<{{responseType .}}>;
|
||||
{{- end}}
|
||||
{{- end}}
|
||||
}
|
||||
{{- /* Declare raw extern "C" imports for raw methods */ -}}
|
||||
{{- range .Service.Methods}}
|
||||
{{- if .Raw}}
|
||||
|
||||
#[link(wasm_import_module = "extism:host/user")]
|
||||
extern "C" {
|
||||
fn {{exportName .}}(offset: u64) -> u64;
|
||||
}
|
||||
{{- end}}
|
||||
{{- end}}
|
||||
|
||||
{{- /* Generate wrapper functions */ -}}
|
||||
{{range .Service.Methods}}
|
||||
{{- if .Raw}}
|
||||
|
||||
{{if .Doc}}{{rustDocComment .Doc}}{{else}}/// Calls the {{exportName .}} host function.{{end}}
|
||||
{{- if .HasParams}}
|
||||
///
|
||||
/// # Arguments
|
||||
{{- range .Params}}
|
||||
/// * `{{.RustName}}` - {{rustType .}} parameter.
|
||||
{{- end}}
|
||||
{{- end}}
|
||||
///
|
||||
/// # Returns
|
||||
/// A tuple of (content_type, data) with the raw binary response.
|
||||
///
|
||||
/// # Errors
|
||||
/// Returns an error if the host function call fails.
|
||||
pub fn {{rustFunc .}}({{range $i, $p := .Params}}{{if $i}}, {{end}}{{$p.RustName}}: {{rustParamType $p}}{{end}}) -> Result<(String, Vec<u8>), Error> {
|
||||
{{- if .HasParams}}
|
||||
let req = {{requestType .}} {
|
||||
{{- range .Params}}
|
||||
{{.RustName}}: {{.RustName}}{{if .NeedsToOwned}}.to_owned(){{end}},
|
||||
{{- end}}
|
||||
};
|
||||
let input_bytes = serde_json::to_vec(&req).map_err(|e| Error::msg(e.to_string()))?;
|
||||
{{- else}}
|
||||
let input_bytes = b"{}".to_vec();
|
||||
{{- end}}
|
||||
let input_mem = Memory::from_bytes(&input_bytes).map_err(|e| Error::msg(e.to_string()))?;
|
||||
|
||||
let response_offset = unsafe { {{exportName .}}(input_mem.offset()) };
|
||||
|
||||
let response_mem = Memory::find(response_offset)
|
||||
.ok_or_else(|| Error::msg("empty response from host"))?;
|
||||
let response_bytes = response_mem.to_vec();
|
||||
|
||||
if response_bytes.is_empty() {
|
||||
return Err(Error::msg("empty response from host"));
|
||||
}
|
||||
if response_bytes[0] == 0x01 {
|
||||
let msg = String::from_utf8_lossy(&response_bytes[1..]).to_string();
|
||||
return Err(Error::msg(msg));
|
||||
}
|
||||
if response_bytes[0] != 0x00 {
|
||||
return Err(Error::msg("unknown response status"));
|
||||
}
|
||||
if response_bytes.len() < 5 {
|
||||
return Err(Error::msg("malformed raw response: incomplete header"));
|
||||
}
|
||||
let ct_len = u32::from_be_bytes([
|
||||
response_bytes[1],
|
||||
response_bytes[2],
|
||||
response_bytes[3],
|
||||
response_bytes[4],
|
||||
]) as usize;
|
||||
if ct_len > response_bytes.len() - 5 {
|
||||
return Err(Error::msg("malformed raw response: content-type overflow"));
|
||||
}
|
||||
let ct_end = 5 + ct_len;
|
||||
let content_type = String::from_utf8_lossy(&response_bytes[5..ct_end]).to_string();
|
||||
let data = response_bytes[ct_end..].to_vec();
|
||||
Ok((content_type, data))
|
||||
}
|
||||
{{- else}}
|
||||
|
||||
{{if .Doc}}{{rustDocComment .Doc}}{{else}}/// Calls the {{exportName .}} host function.{{end}}
|
||||
{{- if .HasParams}}
|
||||
@@ -209,4 +142,3 @@ pub fn {{rustFunc .}}({{range $i, $p := .Params}}{{if $i}}, {{end}}{{$p.RustName
|
||||
}
|
||||
{{- end}}
|
||||
{{- end}}
|
||||
{{- end}}
|
||||
|
||||
@@ -4,9 +4,6 @@ package {{.Package}}
|
||||
|
||||
import (
|
||||
"context"
|
||||
{{- if .Service.HasRawMethods}}
|
||||
"encoding/binary"
|
||||
{{- end}}
|
||||
"encoding/json"
|
||||
|
||||
extism "github.com/extism/go-sdk"
|
||||
@@ -23,7 +20,6 @@ type {{requestType .}} struct {
|
||||
{{- end}}
|
||||
}
|
||||
{{- end}}
|
||||
{{- if not .Raw}}
|
||||
|
||||
// {{responseType .}} is the response type for {{$.Service.Name}}.{{.Name}}.
|
||||
type {{responseType .}} struct {
|
||||
@@ -34,7 +30,6 @@ type {{responseType .}} struct {
|
||||
Error string `json:"error,omitempty"`
|
||||
{{- end}}
|
||||
}
|
||||
{{- end}}
|
||||
{{end}}
|
||||
|
||||
// Register{{.Service.Name}}HostFunctions registers {{.Service.Name}} service host functions.
|
||||
@@ -56,48 +51,18 @@ func new{{$.Service.Name}}{{.Name}}HostFunction(service {{$.Service.Interface}})
|
||||
// Read JSON request from plugin memory
|
||||
reqBytes, err := p.ReadBytes(stack[0])
|
||||
if err != nil {
|
||||
{{- if .Raw}}
|
||||
{{$.Service.Name | lower}}WriteRawError(p, stack, err)
|
||||
{{- else}}
|
||||
{{$.Service.Name | lower}}WriteError(p, stack, err)
|
||||
{{- end}}
|
||||
return
|
||||
}
|
||||
var req {{requestType .}}
|
||||
if err := json.Unmarshal(reqBytes, &req); err != nil {
|
||||
{{- if .Raw}}
|
||||
{{$.Service.Name | lower}}WriteRawError(p, stack, err)
|
||||
{{- else}}
|
||||
{{$.Service.Name | lower}}WriteError(p, stack, err)
|
||||
{{- end}}
|
||||
return
|
||||
}
|
||||
{{- end}}
|
||||
|
||||
// Call the service method
|
||||
{{- if .Raw}}
|
||||
{{range $i, $r := .Returns}}{{if $i}}, {{end}}{{lower $r.Name}}{{end}}, svcErr := service.{{.Name}}(ctx{{range .Params}}, req.{{title .Name}}{{end}})
|
||||
if svcErr != nil {
|
||||
{{$.Service.Name | lower}}WriteRawError(p, stack, svcErr)
|
||||
return
|
||||
}
|
||||
|
||||
// Write binary-framed response to plugin memory:
|
||||
// [0x00][4-byte content-type length (big-endian)][content-type string][raw data]
|
||||
ctBytes := []byte({{lower (index .Returns 0).Name}})
|
||||
frame := make([]byte, 1+4+len(ctBytes)+len({{lower (index .Returns 1).Name}}))
|
||||
frame[0] = 0x00 // success
|
||||
binary.BigEndian.PutUint32(frame[1:5], uint32(len(ctBytes)))
|
||||
copy(frame[5:5+len(ctBytes)], ctBytes)
|
||||
copy(frame[5+len(ctBytes):], {{lower (index .Returns 1).Name}})
|
||||
|
||||
respPtr, err := p.WriteBytes(frame)
|
||||
if err != nil {
|
||||
stack[0] = 0
|
||||
return
|
||||
}
|
||||
stack[0] = respPtr
|
||||
{{- else if .HasReturns}}
|
||||
{{- if .HasReturns}}
|
||||
{{- if .HasError}}
|
||||
{{range $i, $r := .Returns}}{{if $i}}, {{end}}{{lower $r.Name}}{{end}}, svcErr := service.{{.Name}}(ctx{{range .Params}}, req.{{title .Name}}{{end}})
|
||||
if svcErr != nil {
|
||||
@@ -162,16 +127,3 @@ func {{.Service.Name | lower}}WriteError(p *extism.CurrentPlugin, stack []uint64
|
||||
respPtr, _ := p.WriteBytes(respBytes)
|
||||
stack[0] = respPtr
|
||||
}
|
||||
{{- if .Service.HasRawMethods}}
|
||||
|
||||
// {{.Service.Name | lower}}WriteRawError writes a binary-framed error response to plugin memory.
|
||||
// Format: [0x01][UTF-8 error message]
|
||||
func {{.Service.Name | lower}}WriteRawError(p *extism.CurrentPlugin, stack []uint64, err error) {
|
||||
errMsg := []byte(err.Error())
|
||||
frame := make([]byte, 1+len(errMsg))
|
||||
frame[0] = 0x01 // error
|
||||
copy(frame[1:], errMsg)
|
||||
respPtr, _ := p.WriteBytes(frame)
|
||||
stack[0] = respPtr
|
||||
}
|
||||
{{- end}}
|
||||
|
||||
Reference in New Issue
Block a user