From b13529767bf842ac2ce32b1e13620f4c0fcdd635 Mon Sep 17 00:00:00 2001 From: Vincent Koc Date: Sat, 30 May 2026 09:56:20 +0200 Subject: [PATCH] refactor: share inline image data URL sanitizer --- .../.generated/plugin-sdk-api-baseline.sha256 | 4 +- docs/plugins/sdk-subpaths.md | 1 + .../src/app-server/image-payload-sanitizer.ts | 136 +----------------- package.json | 4 + scripts/lib/plugin-sdk-doc-metadata.ts | 3 + scripts/lib/plugin-sdk-entrypoints.json | 1 + .../responses-image-payload-sanitizer.ts | 76 +--------- src/media/inline-image-data-url.test.ts | 32 +++++ src/media/inline-image-data-url.ts | 103 +++++++++++++ .../inline-image-data-url-runtime.ts | 5 + 10 files changed, 160 insertions(+), 205 deletions(-) create mode 100644 src/media/inline-image-data-url.test.ts create mode 100644 src/media/inline-image-data-url.ts create mode 100644 src/plugin-sdk/inline-image-data-url-runtime.ts diff --git a/docs/.generated/plugin-sdk-api-baseline.sha256 b/docs/.generated/plugin-sdk-api-baseline.sha256 index 7e12ac48bbdb..c6ec9403e146 100644 --- a/docs/.generated/plugin-sdk-api-baseline.sha256 +++ b/docs/.generated/plugin-sdk-api-baseline.sha256 @@ -1,2 +1,2 @@ -a5351cfeb95830c99fed9fb86c4c5aff13a7685906e746ea832e7f987134a05b plugin-sdk-api-baseline.json -47e3ff6e9fb76025421bd0fca569ec061a5d71fcad276f78db1ccf5956a07231 plugin-sdk-api-baseline.jsonl +cf29066e9465cb5ac1387d1d482d0939b9176220ecc69964da9af1a471939269 plugin-sdk-api-baseline.json +ab43993cf713a96b191c55cf89bb215c18ecdc2d8edf50f31369ce3b162c56e3 plugin-sdk-api-baseline.jsonl diff --git a/docs/plugins/sdk-subpaths.md b/docs/plugins/sdk-subpaths.md index 43e1ade0751c..8cd5de917457 100644 --- a/docs/plugins/sdk-subpaths.md +++ b/docs/plugins/sdk-subpaths.md @@ -292,6 +292,7 @@ and pairing-path families. | `plugin-sdk/error-runtime` | Error graph, formatting, shared error classification helpers, `isApprovalNotFoundError` | | `plugin-sdk/fetch-runtime` | Wrapped fetch, proxy, EnvHttpProxyAgent option, and pinned lookup helpers | | `plugin-sdk/runtime-fetch` | Dispatcher-aware runtime fetch without proxy/guarded-fetch imports | + | `plugin-sdk/inline-image-data-url-runtime` | Inline image data URL sanitizer and signature sniffing helpers without the broad media runtime surface | | `plugin-sdk/response-limit-runtime` | Bounded response-body reader without the broad media runtime surface | | `plugin-sdk/session-binding-runtime` | Current conversation binding state without configured binding routing or pairing stores | | `plugin-sdk/session-store-runtime` | Session-store helpers without broad config writes/maintenance imports | diff --git a/extensions/codex/src/app-server/image-payload-sanitizer.ts b/extensions/codex/src/app-server/image-payload-sanitizer.ts index 4a88eb839937..04733889f5d1 100644 --- a/extensions/codex/src/app-server/image-payload-sanitizer.ts +++ b/extensions/codex/src/app-server/image-payload-sanitizer.ts @@ -1,137 +1,13 @@ +import { + INLINE_IMAGE_DATA_URL_PREFIX, + sanitizeInlineImageDataUrl as sanitizeSharedInlineImageDataUrl, +} from "openclaw/plugin-sdk/inline-image-data-url-runtime"; import { isRecord } from "openclaw/plugin-sdk/string-coerce-runtime"; -const DATA_URL_PREFIX = "data:"; const IMAGE_OMITTED_TEXT = "omitted image payload: invalid inline image data"; -const IMAGE_SIGNATURES: Array<{ - mime: string; - matches: (buffer: Buffer) => boolean; -}> = [ - { - mime: "image/png", - matches: (buffer) => - buffer.length >= 8 && - buffer[0] === 0x89 && - buffer[1] === 0x50 && - buffer[2] === 0x4e && - buffer[3] === 0x47 && - buffer[4] === 0x0d && - buffer[5] === 0x0a && - buffer[6] === 0x1a && - buffer[7] === 0x0a, - }, - { - mime: "image/jpeg", - matches: (buffer) => - buffer.length >= 3 && buffer[0] === 0xff && buffer[1] === 0xd8 && buffer[2] === 0xff, - }, - { - mime: "image/webp", - matches: (buffer) => - buffer.length >= 12 && - buffer.subarray(0, 4).toString("ascii") === "RIFF" && - buffer.subarray(8, 12).toString("ascii") === "WEBP", - }, - { - mime: "image/gif", - matches: (buffer) => - buffer.length >= 6 && - (buffer.subarray(0, 6).toString("ascii") === "GIF87a" || - buffer.subarray(0, 6).toString("ascii") === "GIF89a"), - }, -]; - -function startsWithDataUrl(value: string): boolean { - return value.slice(0, DATA_URL_PREFIX.length).toLowerCase() === DATA_URL_PREFIX; -} - -function canonicalizeBase64(base64: string): string | undefined { - let cleaned = ""; - let padding = 0; - let sawPadding = false; - for (let i = 0; i < base64.length; i += 1) { - const code = base64.charCodeAt(i); - if (code <= 0x20) { - continue; - } - if (code === 0x3d) { - padding += 1; - if (padding > 2) { - return undefined; - } - sawPadding = true; - cleaned += "="; - continue; - } - const isBase64DataChar = - (code >= 0x41 && code <= 0x5a) || - (code >= 0x61 && code <= 0x7a) || - (code >= 0x30 && code <= 0x39) || - code === 0x2b || - code === 0x2f; - if (sawPadding || !isBase64DataChar) { - return undefined; - } - cleaned += base64[i]; - } - if (!cleaned || cleaned.length % 4 !== 0) { - return undefined; - } - return cleaned; -} - -function sniffImageMime(buffer: Buffer): string | undefined { - return IMAGE_SIGNATURES.find((signature) => signature.matches(buffer))?.mime; -} - -function parseImageDataUrl(value: string): - | { - metadata: string[]; - payload: string; - } - | undefined { - if (!startsWithDataUrl(value)) { - return { metadata: [], payload: value }; - } - const commaIndex = value.indexOf(","); - if (commaIndex < 0) { - return undefined; - } - return { - metadata: value - .slice(DATA_URL_PREFIX.length, commaIndex) - .split(";") - .map((part) => part.trim()), - payload: value.slice(commaIndex + 1), - }; -} - -function metadataAllowsImageBase64(metadata: string[]): boolean { - const [mimeType, ...options] = metadata; - const isImageMimeType = mimeType !== undefined && mimeType.toLowerCase().startsWith("image/"); - return isImageMimeType && options.some((part) => part.toLowerCase() === "base64"); -} export function sanitizeInlineImageDataUrl(imageUrl: string): string | undefined { - const parsed = parseImageDataUrl(imageUrl); - if (!parsed) { - return undefined; - } - if (parsed.metadata.length === 0) { - return imageUrl; - } - if (!metadataAllowsImageBase64(parsed.metadata)) { - return undefined; - } - - const canonicalPayload = canonicalizeBase64(parsed.payload); - if (!canonicalPayload) { - return undefined; - } - const sniffedMimeType = sniffImageMime(Buffer.from(canonicalPayload, "base64")); - if (!sniffedMimeType) { - return undefined; - } - return `data:${sniffedMimeType};base64,${canonicalPayload}`; + return sanitizeSharedInlineImageDataUrl(imageUrl); } export function invalidInlineImageText(label: string): string { @@ -149,7 +25,7 @@ function sanitizeImageContentRecord( return { type: "text", text: invalidInlineImageText(label) }; } const commaIndex = imageUrl.indexOf(","); - const metadata = imageUrl.slice(DATA_URL_PREFIX.length, commaIndex); + const metadata = imageUrl.slice(INLINE_IMAGE_DATA_URL_PREFIX.length, commaIndex); const mime = metadata.split(";")[0] ?? mimeType; return { ...record, mimeType: mime, data: imageUrl.slice(commaIndex + 1) }; } diff --git a/package.json b/package.json index 15c36eb10183..188a1c4693ff 100644 --- a/package.json +++ b/package.json @@ -925,6 +925,10 @@ "types": "./dist/plugin-sdk/runtime-fetch.d.ts", "default": "./dist/plugin-sdk/runtime-fetch.js" }, + "./plugin-sdk/inline-image-data-url-runtime": { + "types": "./dist/plugin-sdk/inline-image-data-url-runtime.d.ts", + "default": "./dist/plugin-sdk/inline-image-data-url-runtime.js" + }, "./plugin-sdk/response-limit-runtime": { "types": "./dist/plugin-sdk/response-limit-runtime.d.ts", "default": "./dist/plugin-sdk/response-limit-runtime.js" diff --git a/scripts/lib/plugin-sdk-doc-metadata.ts b/scripts/lib/plugin-sdk-doc-metadata.ts index 53803f2ce198..6356a6fc2cca 100644 --- a/scripts/lib/plugin-sdk-doc-metadata.ts +++ b/scripts/lib/plugin-sdk-doc-metadata.ts @@ -119,6 +119,9 @@ export const pluginSdkDocMetadata = { "tts-runtime": { category: "runtime", }, + "inline-image-data-url-runtime": { + category: "runtime", + }, "allow-from": { category: "utilities", }, diff --git a/scripts/lib/plugin-sdk-entrypoints.json b/scripts/lib/plugin-sdk-entrypoints.json index ea6c8e39ae3e..4c99bc66a681 100644 --- a/scripts/lib/plugin-sdk-entrypoints.json +++ b/scripts/lib/plugin-sdk-entrypoints.json @@ -210,6 +210,7 @@ "file-lock", "fetch-runtime", "runtime-fetch", + "inline-image-data-url-runtime", "response-limit-runtime", "session-binding-runtime", "session-key-runtime", diff --git a/src/agents/responses-image-payload-sanitizer.ts b/src/agents/responses-image-payload-sanitizer.ts index 27f5978a4c11..9ed1e8f3fa84 100644 --- a/src/agents/responses-image-payload-sanitizer.ts +++ b/src/agents/responses-image-payload-sanitizer.ts @@ -1,80 +1,10 @@ -import { canonicalizeBase64 } from "../media/base64.js"; +import { sanitizeInlineImageDataUrl as sanitizeSharedInlineImageDataUrl } from "../media/inline-image-data-url.js"; import { isRecord } from "../shared/record-coerce.js"; -const DATA_URL_PREFIX = "data:"; const IMAGE_OMITTED_TEXT = "omitted image payload: invalid inline image data"; type JsonRecord = Record; -function startsWithDataUrl(value: string): boolean { - return value.slice(0, DATA_URL_PREFIX.length).toLowerCase() === DATA_URL_PREFIX; -} - -function sniffImageMime(buffer: Buffer): string | undefined { - if ( - buffer.length >= 8 && - buffer[0] === 0x89 && - buffer[1] === 0x50 && - buffer[2] === 0x4e && - buffer[3] === 0x47 && - buffer[4] === 0x0d && - buffer[5] === 0x0a && - buffer[6] === 0x1a && - buffer[7] === 0x0a - ) { - return "image/png"; - } - if (buffer.length >= 3 && buffer[0] === 0xff && buffer[1] === 0xd8 && buffer[2] === 0xff) { - return "image/jpeg"; - } - if ( - buffer.length >= 12 && - buffer.subarray(0, 4).toString("ascii") === "RIFF" && - buffer.subarray(8, 12).toString("ascii") === "WEBP" - ) { - return "image/webp"; - } - if ( - buffer.length >= 6 && - (buffer.subarray(0, 6).toString("ascii") === "GIF87a" || - buffer.subarray(0, 6).toString("ascii") === "GIF89a") - ) { - return "image/gif"; - } - return undefined; -} - -function sanitizeImageUrl(imageUrl: string): string | undefined { - if (!startsWithDataUrl(imageUrl)) { - return imageUrl; - } - const commaIndex = imageUrl.indexOf(","); - if (commaIndex < 0) { - return undefined; - } - - const metadata = imageUrl.slice(DATA_URL_PREFIX.length, commaIndex); - const payload = imageUrl.slice(commaIndex + 1); - const metadataParts = metadata.split(";").map((part) => part.trim()); - const declaredMimeType = metadataParts[0]?.toLowerCase(); - if (!declaredMimeType?.startsWith("image/")) { - return undefined; - } - if (!metadataParts.slice(1).some((part) => part.toLowerCase() === "base64")) { - return undefined; - } - - const canonicalPayload = canonicalizeBase64(payload); - if (!canonicalPayload) { - return undefined; - } - const sniffedMimeType = sniffImageMime(Buffer.from(canonicalPayload, "base64")); - if (!sniffedMimeType) { - return undefined; - } - return `data:${sniffedMimeType};base64,${canonicalPayload}`; -} - function invalidSnakeImage(): JsonRecord { return { type: "input_text", text: `[${IMAGE_OMITTED_TEXT}]` }; } @@ -88,7 +18,7 @@ function sanitizeValue(value: unknown): unknown { } if (value.type === "input_image" && typeof value.image_url === "string") { - const imageUrl = sanitizeImageUrl(value.image_url); + const imageUrl = sanitizeSharedInlineImageDataUrl(value.image_url); return imageUrl ? { ...value, image_url: imageUrl } : invalidSnakeImage(); } @@ -110,7 +40,7 @@ export function sanitizeResponsesImagePayload> } export function sanitizeInlineImageDataUrl(imageUrl: string): string | undefined { - return sanitizeImageUrl(imageUrl); + return sanitizeSharedInlineImageDataUrl(imageUrl); } export function invalidInlineImageText(label: string): string { diff --git a/src/media/inline-image-data-url.test.ts b/src/media/inline-image-data-url.test.ts new file mode 100644 index 000000000000..ee4a4625dc60 --- /dev/null +++ b/src/media/inline-image-data-url.test.ts @@ -0,0 +1,32 @@ +import { describe, expect, it } from "vitest"; +import { sanitizeInlineImageDataUrl, sniffInlineImageMime } from "./inline-image-data-url.js"; + +const PNG_1X1 = + "iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAQAAAC1HAwCAAAAC0lEQVR4nGNgYAAAAAMAASsJTYQAAAAASUVORK5CYII="; + +describe("inline image data URL sanitizer", () => { + it("keeps non-data image references unchanged", () => { + expect(sanitizeInlineImageDataUrl("https://example.test/image.png")).toBe( + "https://example.test/image.png", + ); + }); + + it("rejects malformed and non-image data URLs", () => { + expect(sanitizeInlineImageDataUrl("data:image/png;base64")).toBeUndefined(); + expect(sanitizeInlineImageDataUrl("data:text/plain;base64,SGVsbG8=")).toBeUndefined(); + expect(sanitizeInlineImageDataUrl("data:image/png,SGVsbG8=")).toBeUndefined(); + expect(sanitizeInlineImageDataUrl("data:image/png;base64,not base64!")).toBeUndefined(); + expect(sanitizeInlineImageDataUrl("data:image/png;base64,SGVsbG8=")).toBeUndefined(); + }); + + it("canonicalizes valid data URLs with sniffed MIME type", () => { + expect(sanitizeInlineImageDataUrl(`data:image/jpeg;base64,\n${PNG_1X1}`)).toBe( + `data:image/png;base64,${PNG_1X1}`, + ); + }); + + it("sniffs supported inline image signatures", () => { + expect(sniffInlineImageMime(Buffer.from("GIF89a", "ascii"))).toBe("image/gif"); + expect(sniffInlineImageMime(Buffer.from([0xff, 0xd8, 0xff]))).toBe("image/jpeg"); + }); +}); diff --git a/src/media/inline-image-data-url.ts b/src/media/inline-image-data-url.ts new file mode 100644 index 000000000000..fb3f686fb2af --- /dev/null +++ b/src/media/inline-image-data-url.ts @@ -0,0 +1,103 @@ +import { canonicalizeBase64 } from "./base64.js"; + +export const INLINE_IMAGE_DATA_URL_PREFIX = "data:"; + +const IMAGE_SIGNATURES: Array<{ + mime: string; + matches: (buffer: Buffer) => boolean; +}> = [ + { + mime: "image/png", + matches: (buffer) => + buffer.length >= 8 && + buffer[0] === 0x89 && + buffer[1] === 0x50 && + buffer[2] === 0x4e && + buffer[3] === 0x47 && + buffer[4] === 0x0d && + buffer[5] === 0x0a && + buffer[6] === 0x1a && + buffer[7] === 0x0a, + }, + { + mime: "image/jpeg", + matches: (buffer) => + buffer.length >= 3 && buffer[0] === 0xff && buffer[1] === 0xd8 && buffer[2] === 0xff, + }, + { + mime: "image/webp", + matches: (buffer) => + buffer.length >= 12 && + buffer.subarray(0, 4).toString("ascii") === "RIFF" && + buffer.subarray(8, 12).toString("ascii") === "WEBP", + }, + { + mime: "image/gif", + matches: (buffer) => + buffer.length >= 6 && + (buffer.subarray(0, 6).toString("ascii") === "GIF87a" || + buffer.subarray(0, 6).toString("ascii") === "GIF89a"), + }, +]; + +function startsWithDataUrl(value: string): boolean { + return ( + value.slice(0, INLINE_IMAGE_DATA_URL_PREFIX.length).toLowerCase() === + INLINE_IMAGE_DATA_URL_PREFIX + ); +} + +export function sniffInlineImageMime(buffer: Buffer): string | undefined { + return IMAGE_SIGNATURES.find((signature) => signature.matches(buffer))?.mime; +} + +function parseInlineImageDataUrl(value: string): + | { + metadata: string[]; + payload: string; + } + | undefined { + if (!startsWithDataUrl(value)) { + return { metadata: [], payload: value }; + } + const commaIndex = value.indexOf(","); + if (commaIndex < 0) { + return undefined; + } + return { + metadata: value + .slice(INLINE_IMAGE_DATA_URL_PREFIX.length, commaIndex) + .split(";") + .map((part) => part.trim()), + payload: value.slice(commaIndex + 1), + }; +} + +function metadataAllowsImageBase64(metadata: string[]): boolean { + const [mimeType, ...options] = metadata; + const isImageMimeType = mimeType !== undefined && mimeType.toLowerCase().startsWith("image/"); + return isImageMimeType && options.some((part) => part.toLowerCase() === "base64"); +} + +export function sanitizeInlineImageDataUrl(imageUrl: string): string | undefined { + const parsed = parseInlineImageDataUrl(imageUrl); + if (!parsed) { + return undefined; + } + if (parsed.metadata.length === 0) { + return imageUrl; + } + if (!metadataAllowsImageBase64(parsed.metadata)) { + return undefined; + } + + const canonicalPayload = canonicalizeBase64(parsed.payload); + if (!canonicalPayload) { + return undefined; + } + const sniffedMimeType = sniffInlineImageMime(Buffer.from(canonicalPayload, "base64")); + if (!sniffedMimeType) { + return undefined; + } + return `data:${sniffedMimeType};base64,${canonicalPayload}`; +} diff --git a/src/plugin-sdk/inline-image-data-url-runtime.ts b/src/plugin-sdk/inline-image-data-url-runtime.ts new file mode 100644 index 000000000000..45c8ad9e8750 --- /dev/null +++ b/src/plugin-sdk/inline-image-data-url-runtime.ts @@ -0,0 +1,5 @@ +export { + INLINE_IMAGE_DATA_URL_PREFIX, + sanitizeInlineImageDataUrl, + sniffInlineImageMime, +} from "../media/inline-image-data-url.js";