From 6f57286678614bbdfb9c5c88bdee6d4e1e42d116 Mon Sep 17 00:00:00 2001 From: Peter Steinberger Date: Tue, 26 May 2026 02:54:49 +0100 Subject: [PATCH] refactor: use Rastermill for image processing (#86621) * refactor: use Rastermill for image processing * docs: clarify autoreview heartbeat patience * refactor: use simplified rastermill api * fix: preserve rastermill media safety boundaries * build: update rastermill api pin * build: use published rastermill package --- .agents/skills/autoreview/SKILL.md | 5 +- npm-shrinkwrap.json | 14 +- package.json | 2 +- pnpm-lock.yaml | 14 +- pnpm-workspace.yaml | 2 + scripts/check-dependency-pins.mjs | 5 + scripts/generate-npm-shrinkwrap.mjs | 21 +- scripts/lib/dependency-ownership.json | 9 +- scripts/transitive-manifest-risk-report.mjs | 5 + src/media/image-ops.input-guard.test.ts | 21 + .../image-ops.rastermill-adapter.test.ts | 118 ++ src/media/image-ops.security.test.ts | 54 +- src/media/image-ops.tempdir.test.ts | 83 +- src/media/image-ops.ts | 1592 +++-------------- test/package-manager-config.test.ts | 22 +- test/scripts/check-dependency-pins.test.ts | 2 + 16 files changed, 582 insertions(+), 1387 deletions(-) create mode 100644 src/media/image-ops.rastermill-adapter.test.ts diff --git a/.agents/skills/autoreview/SKILL.md b/.agents/skills/autoreview/SKILL.md index 13a8c6e2631d..e207a961d10a 100644 --- a/.agents/skills/autoreview/SKILL.md +++ b/.agents/skills/autoreview/SKILL.md @@ -26,8 +26,9 @@ Use when: - If a review-triggered fix changes code, rerun focused tests and rerun the structured review helper. - For security-audit suppression changes, verify accepted findings remain auditable: suppressed findings stay in structured output, active output keeps an unsuppressible suppression notice, and aggregate findings cannot hide unrelated active risk. - Never switch or override the requested review engine/model. If the review hits model capacity, retry the same command a few times with the same engine/model. -- Be patient with large bundles. Structured review can be silent for several minutes while the model call is active, especially with Codex tools or web search. Treat `review still running: ... elapsed=... pid=...` as healthy progress, not a hang. -- Do not kill a review just because it has been quiet for 2-5 minutes. Inspect the process only after multiple heartbeat intervals or an obviously failed subprocess; prefer letting the same helper command finish. +- Be patient with large bundles. Structured review can take up to 30 minutes while the model call is active, especially with Codex tools or web search. +- Treat heartbeat lines like `review still running: ... elapsed=... pid=...` as healthy progress, not a hang. Let the helper continue while heartbeats are advancing. +- Do not kill a review just because it has been quiet for 2-5 minutes, or because it is still running under the 30-minute window. Inspect the process only after missing multiple expected heartbeats, after 30 minutes, or after an obviously failed subprocess; prefer letting the same helper command finish. - Tools are useful in review mode. The helper allows read-only inspection tools and web search by default so reviewers can check dependency contracts, upstream docs, and current behavior. - Security perspective is always included, but it should not cripple legitimate functionality. Report security findings only when the change creates a concrete, actionable risk or removes an important safety check. - Do not invoke built-in `codex review`, nested reviewers, or reviewer panels from inside the review. The helper builds one bundle, calls one selected engine, validates one structured result, and stops. diff --git a/npm-shrinkwrap.json b/npm-shrinkwrap.json index 9106408ac6b3..59ae55989590 100644 --- a/npm-shrinkwrap.json +++ b/npm-shrinkwrap.json @@ -26,7 +26,6 @@ "@mozilla/readability": "0.6.0", "@openclaw/fs-safe": "0.3.0", "@openclaw/proxyline": "0.3.3", - "@silvia-odwyer/photon-node": "0.3.4", "ajv": "8.20.0", "chalk": "5.6.2", "chokidar": "5.0.0", @@ -49,6 +48,7 @@ "playwright-core": "1.60.0", "qrcode": "1.5.4", "quickjs-wasi": "2.2.0", + "rastermill": "0.2.0", "tar": "7.5.15", "tokenjuice": "0.7.1", "tree-sitter-bash": "0.25.1", @@ -3842,6 +3842,18 @@ "node": ">= 0.6" } }, + "node_modules/rastermill": { + "version": "0.2.0", + "resolved": "https://registry.npmjs.org/rastermill/-/rastermill-0.2.0.tgz", + "integrity": "sha512-Uc3oKCF4v+sh8OznkG5P6wzQQUstZI/dEGubjNXfWqmCn6iMEAVz8V8S3SVJf+lfQsGpL9iIjBOECmt2ernOAA==", + "license": "MIT", + "dependencies": { + "@silvia-odwyer/photon-node": "0.3.4" + }, + "engines": { + "node": ">=20" + } + }, "node_modules/raw-body": { "version": "3.0.2", "resolved": "https://registry.npmjs.org/raw-body/-/raw-body-3.0.2.tgz", diff --git a/package.json b/package.json index a9941980233c..8a65b5327fe8 100644 --- a/package.json +++ b/package.json @@ -1817,7 +1817,6 @@ "@mozilla/readability": "0.6.0", "@openclaw/fs-safe": "0.3.0", "@openclaw/proxyline": "0.3.3", - "@silvia-odwyer/photon-node": "0.3.4", "ajv": "8.20.0", "chalk": "5.6.2", "chokidar": "5.0.0", @@ -1840,6 +1839,7 @@ "playwright-core": "1.60.0", "qrcode": "1.5.4", "quickjs-wasi": "2.2.0", + "rastermill": "0.2.0", "tar": "7.5.15", "tokenjuice": "0.7.1", "tree-sitter-bash": "0.25.1", diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 1b093007e6e1..e167072742fe 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -88,9 +88,6 @@ importers: '@openclaw/proxyline': specifier: 0.3.3 version: 0.3.3(undici@8.3.0) - '@silvia-odwyer/photon-node': - specifier: 0.3.4 - version: 0.3.4 ajv: specifier: 8.20.0 version: 8.20.0 @@ -157,6 +154,9 @@ importers: quickjs-wasi: specifier: 2.2.0 version: 2.2.0 + rastermill: + specifier: 0.2.0 + version: 0.2.0 tar: specifier: 7.5.15 version: 7.5.15 @@ -6333,6 +6333,10 @@ packages: resolution: {integrity: sha512-Hrgsx+orqoygnmhFbKaHE6c296J+HTAQXoxEF6gNupROmmGJRoyzfG3ccAveqCBrwr/2yxQ5BVd/GTl5agOwSg==} engines: {node: '>= 0.6'} + rastermill@0.2.0: + resolution: {integrity: sha512-Uc3oKCF4v+sh8OznkG5P6wzQQUstZI/dEGubjNXfWqmCn6iMEAVz8V8S3SVJf+lfQsGpL9iIjBOECmt2ernOAA==} + engines: {node: '>=20'} + raw-body@3.0.2: resolution: {integrity: sha512-K5zQjDllxWkf7Z5xJdV0/B0WTNqx6vxG70zJE4N0kBs4LovmEYWJzQGxC9bS9RAKu3bgM40lrd5zoLJ12MQ5BA==} engines: {node: '>= 0.10'} @@ -12322,6 +12326,10 @@ snapshots: range-parser@1.2.1: {} + rastermill@0.2.0: + dependencies: + '@silvia-odwyer/photon-node': 0.3.4 + raw-body@3.0.2: dependencies: bytes: 3.1.2 diff --git a/pnpm-workspace.yaml b/pnpm-workspace.yaml index 59d01847679a..f523226c1af2 100644 --- a/pnpm-workspace.yaml +++ b/pnpm-workspace.yaml @@ -56,6 +56,7 @@ minimumReleaseAgeExclude: - "rolldown" - "sqlite-vec" - "sqlite-vec-*" + - "rastermill" nodeLinker: hoisted blockExoticSubdeps: true @@ -105,6 +106,7 @@ allowBuilds: tree-sitter-bash: false openclaw: true "@openclaw/proxyline": true + rastermill: true packageExtensions: baileys: diff --git a/scripts/check-dependency-pins.mjs b/scripts/check-dependency-pins.mjs index 7b99909975e5..1a874fac8313 100644 --- a/scripts/check-dependency-pins.mjs +++ b/scripts/check-dependency-pins.mjs @@ -12,6 +12,8 @@ const EXACT_SEMVER_PATTERN = /^\d+\.\d+\.\d+(?:-[0-9A-Za-z.-]+)?(?:\+[0-9A-Za-z. const EXACT_NPM_ALIAS_PATTERN = /^npm:(?:@[^/\s]+\/)?[^@\s]+@\d+\.\d+\.\d+(?:-[0-9A-Za-z.-]+)?(?:\+[0-9A-Za-z.-]+)?$/u; const PINNED_GIT_PATTERN = /(?:#|\/commit\/)[0-9a-f]{40}$/iu; +const PINNED_GITHUB_TARBALL_PATTERN = + /^https:\/\/codeload\.github\.com\/[^/\s]+\/[^/\s]+\/tar\.gz\/[0-9a-f]{40}$/iu; function listTrackedPackageJsonFiles(cwd) { return execFileSync("git", ["ls-files", "-z", "--", "*package.json"], { @@ -53,6 +55,9 @@ function isAllowedPinnedSpec(spec) { if (/^(?:git\+|github:|gitlab:|bitbucket:)/u.test(spec)) { return PINNED_GIT_PATTERN.test(spec); } + if (PINNED_GITHUB_TARBALL_PATTERN.test(spec)) { + return true; + } return false; } diff --git a/scripts/generate-npm-shrinkwrap.mjs b/scripts/generate-npm-shrinkwrap.mjs index 0f8cb0bfe05c..f64e9ee2551a 100644 --- a/scripts/generate-npm-shrinkwrap.mjs +++ b/scripts/generate-npm-shrinkwrap.mjs @@ -76,14 +76,18 @@ function readPnpmLockPackages() { if (!packages || typeof packages !== "object" || Array.isArray(packages)) { throw new Error("pnpm-lock.yaml is missing package resolution data."); } - return new Set( - Object.keys(packages) - .map((packageKey) => { - const parsed = parsePnpmPackageKey(packageKey); - return parsed ? `${parsed.name}@${parsed.version}` : null; - }) - .filter((packageKey) => packageKey !== null), - ); + const lockPackages = new Set(); + for (const [packageKey, metadata] of Object.entries(packages)) { + const parsed = parsePnpmPackageKey(packageKey); + if (!parsed) { + continue; + } + lockPackages.add(`${parsed.name}@${parsed.version}`); + if (metadata && typeof metadata === "object" && typeof metadata.version === "string") { + lockPackages.add(`${parsed.name}@${metadata.version}`); + } + } + return lockPackages; } function collectPnpmLockPackageVersions(lockfile) { @@ -114,6 +118,7 @@ function readPnpmLockSingleVersionOverrides() { [...versionsByName.entries()] .filter(([, versions]) => versions.size === 1) .map(([name, versions]) => [name, [...versions][0]]) + .filter(([, version]) => exactVersionFromOverrideSpec(version) !== null) .toSorted(([left], [right]) => left.localeCompare(right)), ); } diff --git a/scripts/lib/dependency-ownership.json b/scripts/lib/dependency-ownership.json index 12808766c93f..d5eeac323640 100644 --- a/scripts/lib/dependency-ownership.json +++ b/scripts/lib/dependency-ownership.json @@ -168,11 +168,10 @@ "class": "default-runtime-initially", "risk": ["terminal-rendering", "png-encoding"] }, - "@silvia-odwyer/photon-node": { - "owner": "plugin:media-understanding-core", - "class": "plugin-runtime", - "activation": ["media-understanding-core.image-ops"], - "risk": ["wasm", "parser", "untrusted-files"] + "rastermill": { + "owner": "core:media", + "class": "core-runtime", + "risk": ["wasm", "parser", "untrusted-files", "native-tool-fallback"] }, "sqlite-vec": { "owner": "capability:memory-sqlite-vec", diff --git a/scripts/transitive-manifest-risk-report.mjs b/scripts/transitive-manifest-risk-report.mjs index 3831e081cf4d..7b477fb6cffc 100644 --- a/scripts/transitive-manifest-risk-report.mjs +++ b/scripts/transitive-manifest-risk-report.mjs @@ -15,6 +15,8 @@ const EXACT_SEMVER_PATTERN = /^\d+\.\d+\.\d+(?:-[0-9A-Za-z.-]+)?(?:\+[0-9A-Za-z. const EXACT_NPM_ALIAS_PATTERN = /^npm:(?:@[^/\s]+\/)?[^@\s]+@\d+\.\d+\.\d+(?:-[0-9A-Za-z.-]+)?(?:\+[0-9A-Za-z.-]+)?$/u; const PINNED_GIT_PATTERN = /(?:#|\/commit\/)[0-9a-f]{40}$/iu; +const PINNED_GITHUB_TARBALL_PATTERN = + /^https:\/\/codeload\.github\.com\/[^/\s]+\/[^/\s]+\/tar\.gz\/[0-9a-f]{40}$/iu; const EXOTIC_SPEC_PATTERN = /^(?:git\+|github:|gitlab:|bitbucket:|https?:)/iu; const RECENTLY_PUBLISHED_VERSION_TYPE = "recently-published-version"; @@ -31,6 +33,9 @@ function isAllowedPinnedSpec(spec) { if (/^(?:git\+|github:|gitlab:|bitbucket:)/u.test(spec)) { return PINNED_GIT_PATTERN.test(spec); } + if (PINNED_GITHUB_TARBALL_PATTERN.test(spec)) { + return true; + } return false; } diff --git a/src/media/image-ops.input-guard.test.ts b/src/media/image-ops.input-guard.test.ts index 42e46982d94d..c4d9813a5c74 100644 --- a/src/media/image-ops.input-guard.test.ts +++ b/src/media/image-ops.input-guard.test.ts @@ -12,6 +12,7 @@ import { ImageProcessorUnavailableError, isImageProcessorUnavailableError, MAX_IMAGE_INPUT_PIXELS, + normalizeExifOrientation, resizeToJpeg, resizeToPng, } from "./image-ops.js"; @@ -91,6 +92,16 @@ describe("image input pixel guard", () => { ).rejects.toThrow(/pixel input limit/i); }); + it("rejects oversized images before EXIF normalization returns unchanged bytes", async () => { + await expect(normalizeExifOrientation(oversizedPng)).rejects.toThrow(/pixel input limit/i); + }); + + it("rejects unreadable images before EXIF normalization returns unchanged bytes", async () => { + await expect(normalizeExifOrientation(Buffer.from("not-an-image"))).rejects.toThrow( + /unable to determine image dimensions/i, + ); + }); + it("rejects overflowed pixel counts before resize work starts", async () => { await expect( resizeToJpeg({ @@ -176,6 +187,16 @@ describe("image input pixel guard", () => { await expect(hasAlphaChannel(opaquePng)).resolves.toBe(false); }); + it("returns opaque when header-unknown alpha cannot be decoded", async () => { + await expect(hasAlphaChannel(createHeifLikeBuffer({ width: 1, height: 1 }))).resolves.toBe( + false, + ); + }); + + it("rejects oversized alpha checks before returning a safe default", async () => { + await expect(hasAlphaChannel(oversizedPng)).rejects.toThrow(/pixel input limit/i); + }); + it("resizes grayscale alpha PNGs through the Photon backend", async () => { const source = createGrayscaleAlphaPngBuffer(64, 32); diff --git a/src/media/image-ops.rastermill-adapter.test.ts b/src/media/image-ops.rastermill-adapter.test.ts new file mode 100644 index 000000000000..36dd5cfebf88 --- /dev/null +++ b/src/media/image-ops.rastermill-adapter.test.ts @@ -0,0 +1,118 @@ +import { afterEach, describe, expect, it, vi } from "vitest"; +import { encodePngRgba } from "./png-encode.js"; + +function rgbaPng(alpha: number): Buffer { + return encodePngRgba(Buffer.from([0x20, 0x80, 0xe0, alpha]), 1, 1); +} + +describe("image ops Rastermill adapter", () => { + afterEach(() => { + vi.doUnmock("rastermill"); + vi.resetModules(); + }); + + it("falls back to decoding PNG pixels when header alpha is unknown", async () => { + const encode = vi.fn(async () => ({ + data: rgbaPng(64), + format: "png", + width: 1, + height: 1, + bytes: 11, + })); + + vi.doMock("rastermill", () => ({ + RastermillUnavailableError: class RastermillUnavailableError extends Error {}, + createRastermill: () => ({ + probe: vi.fn(async () => ({ + orientation: null, + width: 1, + height: 1, + hasAlpha: null, + format: "gif", + })), + encode, + }), + isRastermillUnavailableError: () => false, + readImageMetadataFromHeader: vi.fn(() => ({ width: 1, height: 1 })), + readImageProbeFromHeader: vi.fn(() => ({ + width: 1, + height: 1, + format: "gif", + hasAlpha: null, + orientation: null, + })), + })); + + const { hasAlphaChannel } = await import("./image-ops.js"); + + await expect(hasAlphaChannel(Buffer.from("maybe-alpha"))).resolves.toBe(true); + expect(encode).toHaveBeenCalledWith(Buffer.from("maybe-alpha"), { + format: "png", + autoOrient: false, + }); + }); + + it("does not report opaque decoded fallback PNG pixels as transparent", async () => { + vi.doMock("rastermill", () => ({ + RastermillUnavailableError: class RastermillUnavailableError extends Error {}, + createRastermill: () => ({ + probe: vi.fn(async () => ({ + orientation: null, + width: 1, + height: 1, + hasAlpha: null, + format: "gif", + })), + encode: vi.fn(async () => ({ + data: rgbaPng(255), + format: "png", + width: 1, + height: 1, + bytes: 70, + })), + }), + isRastermillUnavailableError: () => false, + readImageMetadataFromHeader: vi.fn(() => ({ width: 1, height: 1 })), + readImageProbeFromHeader: vi.fn(() => ({ + width: 1, + height: 1, + format: "gif", + hasAlpha: null, + orientation: null, + })), + })); + + const { hasAlphaChannel } = await import("./image-ops.js"); + + await expect(hasAlphaChannel(Buffer.from("opaque-maybe-alpha"))).resolves.toBe(false); + }); + + it("uses Photon only for header-valid Photon-owned inputs unless a backend is forced", async () => { + const createRastermill = vi.fn(() => ({ + encode: vi.fn(async () => { + throw new Error("cannot decode corrupt payload"); + }), + })); + + vi.doMock("rastermill", () => ({ + RastermillUnavailableError: class RastermillUnavailableError extends Error {}, + createRastermill, + isRastermillUnavailableError: () => false, + readImageMetadataFromHeader: vi.fn(() => ({ width: 1, height: 1 })), + readImageProbeFromHeader: vi.fn(() => ({ + width: 1, + height: 1, + format: "png", + hasAlpha: true, + orientation: null, + })), + })); + + const { resizeToJpeg } = await import("./image-ops.js"); + + await expect( + resizeToJpeg({ buffer: Buffer.from("corrupt-png"), maxSide: 1, quality: 80 }), + ).rejects.toThrow("cannot decode corrupt payload"); + expect(createRastermill).toHaveBeenCalledWith(expect.objectContaining({ backend: "photon" })); + }); +}); diff --git a/src/media/image-ops.security.test.ts b/src/media/image-ops.security.test.ts index 34851890b79e..bcee365a8b37 100644 --- a/src/media/image-ops.security.test.ts +++ b/src/media/image-ops.security.test.ts @@ -1,39 +1,41 @@ -import { afterEach, describe, expect, it, vi } from "vitest"; +import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; import { createPngBufferWithDimensions } from "./test-helpers.js"; -const { loadBundledPluginPublicArtifactModuleSyncMock, resolveSystemBinMock, runExecMock } = - vi.hoisted(() => ({ - loadBundledPluginPublicArtifactModuleSyncMock: vi.fn(), - resolveSystemBinMock: vi.fn(), - runExecMock: vi.fn(), - })); - -vi.mock("../plugins/public-surface-loader.js", () => ({ - loadBundledPluginPublicArtifactModuleSync: loadBundledPluginPublicArtifactModuleSyncMock, +const { createRastermillMock, resolveSystemBinMock } = vi.hoisted(() => ({ + createRastermillMock: vi.fn(), + resolveSystemBinMock: vi.fn(), })); +vi.mock("rastermill", async (importOriginal) => { + const actual = await importOriginal(); + return { + ...actual, + createRastermill: createRastermillMock, + }; +}); + vi.mock("../infra/resolve-system-bin.js", () => ({ resolveSystemBin: resolveSystemBinMock, })); -vi.mock("../process/exec.js", () => ({ - runExec: runExecMock, -})); - import { getImageMetadata, resizeToJpeg } from "./image-ops.js"; describe("image ops external backend security", () => { const previousBackend = process.env.OPENCLAW_IMAGE_BACKEND; + beforeEach(async () => { + const actual = await vi.importActual("rastermill"); + createRastermillMock.mockImplementation(actual.createRastermill); + }); + afterEach(() => { if (previousBackend === undefined) { delete process.env.OPENCLAW_IMAGE_BACKEND; } else { process.env.OPENCLAW_IMAGE_BACKEND = previousBackend; } - loadBundledPluginPublicArtifactModuleSyncMock.mockReset(); + createRastermillMock.mockReset(); resolveSystemBinMock.mockReset(); - runExecMock.mockReset(); }); it("does not use external metadata tools for unrecognized image bytes", async () => { @@ -46,23 +48,15 @@ describe("image ops external backend security", () => { await expect(getImageMetadata(svgWithExternalReference)).resolves.toBeNull(); - expect(runExecMock).not.toHaveBeenCalled(); - expect(loadBundledPluginPublicArtifactModuleSyncMock).not.toHaveBeenCalled(); + expect(resolveSystemBinMock).not.toHaveBeenCalled(); }); - it("stops backend fallback after a real processing error", async () => { + it("propagates Rastermill processing errors without OpenClaw-side backend fallback", async () => { delete process.env.OPENCLAW_IMAGE_BACKEND; resolveSystemBinMock.mockReturnValue("/usr/bin/magick"); - loadBundledPluginPublicArtifactModuleSyncMock.mockReturnValue({ - createMediaAttachmentImageOps: () => ({ - getImageMetadata: vi.fn(), - normalizeExifOrientation: vi.fn(), - resizeToJpeg: vi.fn(async () => { - throw new Error("corrupt image payload"); - }), - convertHeicToJpeg: vi.fn(), - hasAlphaChannel: vi.fn(), - resizeToPng: vi.fn(), + createRastermillMock.mockReturnValue({ + encode: vi.fn(async () => { + throw new Error("corrupt image payload"); }), }); @@ -74,6 +68,6 @@ describe("image ops external backend security", () => { }), ).rejects.toThrow(/corrupt image payload/); - expect(runExecMock).not.toHaveBeenCalled(); + expect(resolveSystemBinMock).not.toHaveBeenCalled(); }); }); diff --git a/src/media/image-ops.tempdir.test.ts b/src/media/image-ops.tempdir.test.ts index fa8cef8e061c..e3665bd49629 100644 --- a/src/media/image-ops.tempdir.test.ts +++ b/src/media/image-ops.tempdir.test.ts @@ -1,59 +1,82 @@ import fs from "node:fs/promises"; +import os from "node:os"; import path from "node:path"; import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; +import { createSolidPngBuffer, createTinyJpegBuffer } from "../../test/helpers/image-fixtures.js"; import { resolvePreferredOpenClawTmpDir } from "../infra/tmp-openclaw-dir.js"; -import { resizeToJpeg } from "./image-ops.js"; -const PNG_1X1_BASE64 = - "iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAQAAAC1HAwCAAAAC0lEQVR42mP8/x8AAwMCAO+/p9sAAAAASUVORK5CYII="; +const fakeSips = vi.hoisted(() => ({ + logPath: "", + path: "", +})); + +vi.mock("../infra/resolve-system-bin.js", () => ({ + resolveSystemBin: (command: string) => (command === "sips" ? fakeSips.path : null), +})); describe("image-ops temp dir", () => { - let createdTempDir = ""; + let fakeRoot = ""; - beforeEach(() => { + beforeEach(async () => { + vi.resetModules(); process.env.OPENCLAW_IMAGE_BACKEND = "sips"; - const originalMkdtemp = fs.mkdtemp.bind(fs); - vi.spyOn(fs, "mkdtemp").mockImplementation(async (prefix) => { - createdTempDir = await originalMkdtemp(prefix); - return createdTempDir; - }); + fakeRoot = await fs.mkdtemp(path.join(os.tmpdir(), "openclaw-fake-sips-")); + fakeSips.path = path.join(fakeRoot, "sips.js"); + fakeSips.logPath = path.join(fakeRoot, "args.json"); + const outputJpeg = createTinyJpegBuffer().toString("base64"); + await fs.writeFile( + fakeSips.path, + [ + "#!/usr/bin/env node", + "const fs = require('node:fs');", + "const args = process.argv.slice(2);", + `fs.writeFileSync(${JSON.stringify(fakeSips.logPath)}, JSON.stringify(args));`, + "const outIndex = args.indexOf('--out');", + "const output = outIndex >= 0 ? args[outIndex + 1] : args.at(-1);", + `fs.writeFileSync(output, Buffer.from(${JSON.stringify(outputJpeg)}, 'base64'));`, + ].join("\n"), + "utf8", + ); + await fs.chmod(fakeSips.path, 0o755); }); - afterEach(() => { + afterEach(async () => { delete process.env.OPENCLAW_IMAGE_BACKEND; - vi.restoreAllMocks(); + fakeSips.logPath = ""; + fakeSips.path = ""; + await fs.rm(fakeRoot, { recursive: true, force: true }); + fakeRoot = ""; }); it.skipIf(process.platform !== "darwin")( "creates sips temp dirs under the secured OpenClaw tmp root", async () => { - const secureRoot = await fs.realpath(resolvePreferredOpenClawTmpDir()); + const { resizeToJpeg } = await import("./image-ops.js"); + const secureRoot = path.resolve(resolvePreferredOpenClawTmpDir()); await resizeToJpeg({ - buffer: Buffer.from(PNG_1X1_BASE64, "base64"), - maxSide: 1, + buffer: createSolidPngBuffer(2, 2, { r: 255, g: 255, b: 255 }), + maxSide: 2, quality: 80, }); - expect(fs.mkdtemp).toHaveBeenCalledTimes(1); - const [mkdtempCall] = vi.mocked(fs.mkdtemp).mock.calls; - if (!mkdtempCall) { - throw new Error("expected mkdtemp call"); + const args = JSON.parse(await fs.readFile(fakeSips.logPath, "utf8")) as string[]; + const outIndex = args.indexOf("--out"); + if (outIndex < 1) { + throw new Error("expected sips input before --out"); } - const [prefix] = mkdtempCall; - expect(typeof prefix).toBe("string"); - const uuidPrefix = path.join(secureRoot, "openclaw-img-"); - expect(prefix?.startsWith(uuidPrefix)).toBe(true); - expect(prefix?.endsWith("-")).toBe(true); - const uuid = prefix?.slice(uuidPrefix.length, -1) ?? ""; - expect(uuid).toHaveLength(36); - expect(/^[0-9a-f-]+$/u.test(uuid)).toBe(true); + const inputPath = args[outIndex - 1] ?? ""; + const tempDir = path.dirname(inputPath); + const relative = path.relative(secureRoot, tempDir); + expect(relative.startsWith("openclaw-img-")).toBe(true); + expect(relative.includes("..")).toBe(false); + const match = /^openclaw-img-([0-9a-f-]{36})-[A-Za-z0-9]+$/u.exec(path.basename(tempDir)); + expect(match).not.toBeNull(); + const uuid = match?.[1] ?? ""; expect([8, 13, 18, 23].map((index) => uuid[index])).toEqual(["-", "-", "-", "-"]); - expect(path.dirname(prefix ?? "")).toBe(secureRoot); - expect(createdTempDir.startsWith(prefix ?? "")).toBe(true); let accessError: unknown; try { - await fs.access(createdTempDir); + await fs.access(tempDir); } catch (error) { accessError = error; } diff --git a/src/media/image-ops.ts b/src/media/image-ops.ts index 472e9205ff0d..7aaf2f406105 100644 --- a/src/media/image-ops.ts +++ b/src/media/image-ops.ts @@ -1,70 +1,36 @@ -import { withTempWorkspace, type TempWorkspace } from "../infra/private-temp-workspace.js"; +import { randomUUID } from "node:crypto"; +import { inflateSync } from "node:zlib"; +import { + createRastermill, + isRastermillUnavailableError, + RastermillUnavailableError, + readImageMetadataFromHeader as readRastermillImageMetadataFromHeader, + readImageProbeFromHeader as readRastermillImageProbeFromHeader, + type ImageMetadata, +} from "rastermill"; import { resolveSystemBin } from "../infra/resolve-system-bin.js"; import { resolvePreferredOpenClawTmpDir } from "../infra/tmp-openclaw-dir.js"; -import { runExec } from "../process/exec.js"; -import { createLazyPromiseLoader } from "../shared/lazy-promise.js"; -import { uniqueValues } from "../shared/string-normalization.js"; -export type ImageMetadata = { - width: number; - height: number; -}; +export type { ImageMetadata }; -type MediaAttachmentImageOps = { - getImageMetadata(buffer: Buffer): Promise; - normalizeExifOrientation(buffer: Buffer): Promise; - resizeToJpeg(params: ResizeToJpegParams): Promise; - convertHeicToJpeg(buffer: Buffer): Promise; - hasAlphaChannel(buffer: Buffer): Promise; - resizeToPng(params: ResizeToPngParams): Promise; -}; - -type MediaAttachmentImageOpsModule = { - createMediaAttachmentImageOps?: (options: { maxInputPixels: number }) => MediaAttachmentImageOps; -}; - -type ResizeToJpegParams = { +export type ResizeToJpegParams = { buffer: Buffer; maxSide: number; quality: number; withoutEnlargement?: boolean; }; -type ResizeToPngParams = { +export type ResizeToPngParams = { buffer: Buffer; maxSide: number; compressionLevel?: number; withoutEnlargement?: boolean; }; -type ImageBackend = - | "photon" - | "sips" - | "windows-native" - | "imagemagick" - | "graphicsmagick" - | "ffmpeg"; -type ImageBackendPreference = ImageBackend | "auto"; -type ImageOperation = - | "metadata" - | "normalizeExifOrientation" - | "resizeToJpeg" - | "convertHeicToJpeg" - | "resizeToPng"; - -type ExternalImageTool = - | { backend: "imagemagick"; flavor: "magick" | "convert"; command: string } - | { backend: "graphicsmagick"; flavor: "gm"; command: string } - | { backend: "ffmpeg"; flavor: "ffmpeg"; command: string } - | { backend: "windows-native"; flavor: "powershell"; command: string } - | { backend: "sips"; flavor: "sips"; command: string }; - export const IMAGE_REDUCE_QUALITY_STEPS = [85, 75, 65, 55, 45, 35] as const; export const MAX_IMAGE_INPUT_PIXELS = 25_000_000; -const IMAGE_PROCESS_TIMEOUT_MS = 20_000; -const IMAGE_TOOL_MAX_BUFFER = 1024 * 1024; -const MEDIA_UNDERSTANDING_CORE_PLUGIN_ID = "media-understanding-core"; -const MEDIA_UNDERSTANDING_CORE_IMAGE_OPS_ARTIFACT = "image-ops.js"; +const PHOTON_OWNED_FORMATS = new Set(["png", "gif", "webp", "jpeg"]); +const PNG_SIGNATURE = Buffer.from([0x89, 0x50, 0x4e, 0x47, 0x0d, 0x0a, 0x1a, 0x0a]); export class ImageProcessorUnavailableError extends Error { readonly code = "IMAGE_PROCESSOR_UNAVAILABLE"; @@ -81,1256 +47,341 @@ export class ImageProcessorUnavailableError extends Error { } } +function createOpenClawRastermill(options: { backend?: "photon" } = {}) { + return createRastermill({ + ...(options.backend === undefined ? {} : { backend: options.backend }), + limits: { + inputPixels: MAX_IMAGE_INPUT_PIXELS, + outputPixels: MAX_IMAGE_INPUT_PIXELS, + }, + env: { + backendVar: "OPENCLAW_IMAGE_BACKEND", + }, + temp: { + rootDir: resolvePreferredOpenClawTmpDir(), + prefix: () => `openclaw-img-${randomUUID()}-`, + }, + commandResolver: (command) => + resolveSystemBin(command, { trust: command === "powershell" ? "strict" : "standard" }), + }); +} + +function hasExplicitImageBackendPreference(): boolean { + const raw = process.env.OPENCLAW_IMAGE_BACKEND?.trim().toLowerCase(); + return raw !== undefined && raw.length > 0 && raw !== "auto"; +} + +function shouldForcePhotonForInput(buffer: Buffer): boolean { + if (hasExplicitImageBackendPreference()) { + return false; + } + const format = readRastermillImageProbeFromHeader(buffer)?.format; + return format !== undefined && PHOTON_OWNED_FORMATS.has(format); +} + +function createOpenClawRastermillForInput(buffer: Buffer) { + return createOpenClawRastermill({ + backend: shouldForcePhotonForInput(buffer) ? "photon" : undefined, + }); +} + export function isImageProcessorUnavailableError(err: unknown): boolean { + if (err instanceof ImageProcessorUnavailableError || isRastermillUnavailableError(err)) { + return true; + } + const messages: string[] = []; let current: unknown = err; while (current instanceof Error) { - if (current instanceof ImageProcessorUnavailableError) { - return true; - } messages.push(current.message); current = current.cause; } const detail = messages.join("\n").toLowerCase(); return ( detail.includes("image processor unavailable") || - detail.includes("photon did not expose") || - detail.includes("photon backend skipped") || - detail.includes("cannot find package '@silvia-odwyer/photon-node'") || - detail.includes('cannot find package "@silvia-odwyer/photon-node"') || - detail.includes("cannot find module '@silvia-odwyer/photon-node'") || - detail.includes('cannot find module "@silvia-odwyer/photon-node"') + detail.includes("required image processor api") || + detail.includes("rastermill_image_processor_unavailable") ); } export function buildImageResizeSideGrid(maxSide: number, sideStart: number): number[] { - return uniqueValues( - [sideStart, 1800, 1600, 1400, 1200, 1000, 800] - .map((value) => Math.min(maxSide, value)) - .filter((value) => value > 0), - ).toSorted((a, b) => b - a); + return [sideStart, 1800, 1600, 1400, 1200, 1000, 800] + .map((value) => Math.min(maxSide, value)) + .filter((value, idx, arr) => value > 0 && arr.indexOf(value) === idx) + .toSorted((a, b) => b - a); } -function getImageBackendPreference(): ImageBackendPreference { - const raw = process.env.OPENCLAW_IMAGE_BACKEND?.trim().toLowerCase(); - switch (raw) { - case "photon": - case "sips": - case "windows-native": - case "imagemagick": - case "graphicsmagick": - case "ffmpeg": - return raw; - case "windows": - case "powershell": - case "system.drawing": - case "systemdrawing": - return "windows-native"; - case "magick": - case "convert": - return "imagemagick"; - case "gm": - return "graphicsmagick"; - default: - return "auto"; +function wrapRastermillUnavailable(operation: string, error: unknown): never { + if (error instanceof RastermillUnavailableError) { + throw new ImageProcessorUnavailableError(operation, error.message, error.causes); + } + throw error; +} + +function assertImageInputWithinPixelBudget(buffer: Buffer): void { + const metadata = readRastermillImageMetadataFromHeader(buffer); + if (!metadata) { + throw new Error("Unable to determine image dimensions; refusing to process"); + } + if (metadata.width > Math.floor(MAX_IMAGE_INPUT_PIXELS / metadata.height)) { + const pixels = Number.isSafeInteger(metadata.width * metadata.height) + ? ` (${metadata.width * metadata.height} pixels)` + : ""; + throw new Error( + `Image dimensions exceed the ${MAX_IMAGE_INPUT_PIXELS.toLocaleString("en-US")} pixel input limit: ${metadata.width}x${metadata.height}${pixels}`, + ); } } -function shouldFailClosedOnUnknownMetadata(): boolean { - return true; +function paethPredictor(left: number, up: number, upperLeft: number): number { + const prediction = left + up - upperLeft; + const distanceLeft = Math.abs(prediction - left); + const distanceUp = Math.abs(prediction - up); + const distanceUpperLeft = Math.abs(prediction - upperLeft); + if (distanceLeft <= distanceUp && distanceLeft <= distanceUpperLeft) { + return left; + } + return distanceUp <= distanceUpperLeft ? up : upperLeft; } -function imageBackendsForOperation(operation: ImageOperation): ImageBackend[] { - const preference = getImageBackendPreference(); - if (preference !== "auto") { - return [preference]; +function unfilterPngScanlines( + inflated: Buffer, + width: number, + height: number, + bytesPerPixel: number, +): Buffer | null { + const stride = width * bytesPerPixel; + if (inflated.length !== (stride + 1) * height) { + return null; } - - if (operation === "resizeToPng") { - if (process.platform === "win32") { - return ["photon", "windows-native", "imagemagick", "graphicsmagick"]; - } - return ["photon", "imagemagick", "graphicsmagick"]; - } - - if (operation === "normalizeExifOrientation") { - if (process.platform === "win32") { - return ["photon", "imagemagick", "graphicsmagick"]; - } - return process.platform === "darwin" - ? ["photon", "sips", "imagemagick", "graphicsmagick"] - : ["photon", "imagemagick", "graphicsmagick"]; - } - - if (process.platform === "win32") { - if (operation === "convertHeicToJpeg") { - return ["imagemagick", "graphicsmagick", "ffmpeg"]; - } - return ["photon", "windows-native", "imagemagick", "graphicsmagick", "ffmpeg"]; - } - - const fallbacks = - process.platform === "darwin" - ? (["sips", "imagemagick", "graphicsmagick", "ffmpeg"] as const) - : (["imagemagick", "graphicsmagick", "ffmpeg"] as const); - if (operation === "convertHeicToJpeg") { - return [...fallbacks]; - } - return ["photon", ...fallbacks]; -} - -function createImageProcessorUnavailableError( - operation: ImageOperation, - causes: unknown[], -): ImageProcessorUnavailableError { - const backends = imageBackendsForOperation(operation).join(", "); - const hint = - process.platform === "win32" - ? "Install ImageMagick, GraphicsMagick, or ffmpeg; Windows native image resizing is tried automatically when available." - : process.platform === "darwin" - ? "Install a system image tool such as sips, ImageMagick, GraphicsMagick, or ffmpeg." - : "Install ImageMagick, GraphicsMagick, or ffmpeg."; - return new ImageProcessorUnavailableError( - operation, - `Image processor unavailable for ${operation}; tried: ${backends}. ${hint}`, - causes, - ); -} - -function isImageBackendUnavailableCause(error: unknown): boolean { - const messages: string[] = []; - let current: unknown = error; - while (current instanceof Error) { - messages.push(current.message); - current = current.cause; - } - const detail = messages.join("\n").toLowerCase(); - return ( - detail.includes("photon did not expose") || - detail.includes("photon backend skipped") || - detail.includes("cannot find package '@silvia-odwyer/photon-node'") || - detail.includes('cannot find package "@silvia-odwyer/photon-node"') || - detail.includes("cannot find module '@silvia-odwyer/photon-node'") || - detail.includes('cannot find module "@silvia-odwyer/photon-node"') || - detail.includes("support for this compression format has not been built in") || - detail.includes("is not available") || - detail.includes("command not found") || - detail.includes("enoent") - ); -} - -async function runWithImageBackends( - operation: ImageOperation, - fn: (backend: ImageBackend) => Promise, -): Promise { - const errors: unknown[] = []; - for (const backend of imageBackendsForOperation(operation)) { - try { - return await fn(backend); - } catch (error) { - errors.push(error); - if (!isImageBackendUnavailableCause(error)) { - throw error; + const out = Buffer.alloc(stride * height); + for (let row = 0; row < height; row += 1) { + const filter = inflated[row * (stride + 1)]; + const sourceOffset = row * (stride + 1) + 1; + const targetOffset = row * stride; + for (let column = 0; column < stride; column += 1) { + const raw = inflated[sourceOffset + column] ?? 0; + const left = column >= bytesPerPixel ? (out[targetOffset + column - bytesPerPixel] ?? 0) : 0; + const up = row > 0 ? (out[targetOffset + column - stride] ?? 0) : 0; + const upperLeft = + row > 0 && column >= bytesPerPixel + ? (out[targetOffset + column - stride - bytesPerPixel] ?? 0) + : 0; + let value: number; + switch (filter) { + case 0: + value = raw; + break; + case 1: + value = raw + left; + break; + case 2: + value = raw + up; + break; + case 3: + value = raw + Math.floor((left + up) / 2); + break; + case 4: + value = raw + paethPredictor(left, up, upperLeft); + break; + default: + return null; } + out[targetOffset + column] = value & 0xff; } } - throw createImageProcessorUnavailableError(operation, errors); + return out; } -function isMediaAttachmentImageOps(value: unknown): value is MediaAttachmentImageOps { - if (!value || typeof value !== "object") { - return false; - } - const candidate = value as Partial>; - return ( - typeof candidate.getImageMetadata === "function" && - typeof candidate.normalizeExifOrientation === "function" && - typeof candidate.resizeToJpeg === "function" && - typeof candidate.convertHeicToJpeg === "function" && - typeof candidate.hasAlphaChannel === "function" && - typeof candidate.resizeToPng === "function" - ); -} - -const mediaAttachmentImageOpsLoader = createLazyPromiseLoader(async () => { - const { loadBundledPluginPublicArtifactModuleSync } = - await import("../plugins/public-surface-loader.js"); - const mod = loadBundledPluginPublicArtifactModuleSync({ - dirName: MEDIA_UNDERSTANDING_CORE_PLUGIN_ID, - artifactBasename: MEDIA_UNDERSTANDING_CORE_IMAGE_OPS_ARTIFACT, - }); - const ops = mod.createMediaAttachmentImageOps?.({ - maxInputPixels: MAX_IMAGE_INPUT_PIXELS, - }); - if (!isMediaAttachmentImageOps(ops)) { - throw new Error("Media understanding core did not expose image ops"); - } - return ops; -}); - -async function loadMediaAttachmentImageOps(): Promise { - return await mediaAttachmentImageOpsLoader.load(); -} - -function isPositiveImageDimension(value: number): boolean { - return Number.isInteger(value) && value > 0; -} - -function buildImageMetadata(width: number, height: number): ImageMetadata | null { - if (!isPositiveImageDimension(width) || !isPositiveImageDimension(height)) { +function decodedPngHasTransparentPixel(buffer: Buffer): boolean | null { + if (buffer.length < 33 || !buffer.subarray(0, PNG_SIGNATURE.length).equals(PNG_SIGNATURE)) { return null; } - return { width, height }; -} -function readPngMetadata(buffer: Buffer): ImageMetadata | null { - if (buffer.length < 24) { - return null; + let width = 0; + let height = 0; + let bitDepth = 0; + let colorType = 0; + let interlace = 0; + let transparency: Buffer | null = null; + const idatChunks: Buffer[] = []; + for (let offset = 8; offset + 12 <= buffer.length; ) { + const length = buffer.readUInt32BE(offset); + const type = buffer.toString("ascii", offset + 4, offset + 8); + const dataStart = offset + 8; + const dataEnd = dataStart + length; + if (dataEnd + 4 > buffer.length) { + return null; + } + const data = buffer.subarray(dataStart, dataEnd); + if (type === "IHDR") { + if (length !== 13 || data[10] !== 0 || data[11] !== 0) { + return null; + } + width = data.readUInt32BE(0); + height = data.readUInt32BE(4); + bitDepth = data[8] ?? 0; + colorType = data[9] ?? 0; + interlace = data[12] ?? 0; + } else if (type === "tRNS") { + transparency = data; + } else if (type === "IDAT") { + idatChunks.push(data); + } else if (type === "IEND") { + break; + } + offset = dataEnd + 4; } + + const bytesPerPixel = + colorType === 6 + ? 4 + : colorType === 4 + ? 2 + : colorType === 2 + ? 3 + : colorType === 0 || colorType === 3 + ? 1 + : null; if ( - buffer[0] !== 0x89 || - buffer[1] !== 0x50 || - buffer[2] !== 0x4e || - buffer[3] !== 0x47 || - buffer[4] !== 0x0d || - buffer[5] !== 0x0a || - buffer[6] !== 0x1a || - buffer[7] !== 0x0a || - buffer.toString("ascii", 12, 16) !== "IHDR" + width <= 0 || + height <= 0 || + bitDepth !== 8 || + interlace !== 0 || + bytesPerPixel === null || + idatChunks.length === 0 ) { return null; } - return buildImageMetadata(buffer.readUInt32BE(16), buffer.readUInt32BE(20)); -} + if (colorType === 0 || colorType === 2) { + return transparency !== null && transparency.length > 0; + } -function readPngAlphaChannel(buffer: Buffer): boolean | null { - if (buffer.length < 29 || readPngMetadata(buffer) === null) { + const stride = width * bytesPerPixel; + const inflated = inflateSync(Buffer.concat(idatChunks), { + maxOutputLength: (stride + 1) * height, + }); + const pixels = unfilterPngScanlines(inflated, width, height, bytesPerPixel); + if (!pixels) { return null; } - - const colorType = buffer[25]; - if (colorType === 4 || colorType === 6) { - return true; - } - if (colorType !== 0 && colorType !== 2 && colorType !== 3) { - return null; - } - - let offset = 8; - while (offset + 8 <= buffer.length) { - const chunkLength = buffer.readUInt32BE(offset); - const typeStart = offset + 4; - const dataStart = offset + 8; - const dataEnd = dataStart + chunkLength; - const nextOffset = dataEnd + 4; - if (dataEnd > buffer.length || nextOffset > buffer.length) { - return null; - } - const chunkType = buffer.toString("ascii", typeStart, typeStart + 4); - if (chunkType === "tRNS") { - return chunkLength > 0; - } - if (chunkType === "IDAT" || chunkType === "IEND") { + if (colorType === 3) { + if (!transparency) { return false; } - offset = nextOffset; - } - - return false; -} - -function readGifMetadata(buffer: Buffer): ImageMetadata | null { - if (buffer.length < 10) { - return null; - } - const signature = buffer.toString("ascii", 0, 6); - if (signature !== "GIF87a" && signature !== "GIF89a") { - return null; - } - return buildImageMetadata(buffer.readUInt16LE(6), buffer.readUInt16LE(8)); -} - -function readWebpMetadata(buffer: Buffer): ImageMetadata | null { - if ( - buffer.length < 30 || - buffer.toString("ascii", 0, 4) !== "RIFF" || - buffer.toString("ascii", 8, 12) !== "WEBP" - ) { - return null; - } - const chunkType = buffer.toString("ascii", 12, 16); - if (chunkType === "VP8X") { - if (buffer.length < 30) { - return null; + for (const paletteIndex of pixels) { + if ((transparency[paletteIndex] ?? 255) < 255) { + return true; + } } - return buildImageMetadata(1 + buffer.readUIntLE(24, 3), 1 + buffer.readUIntLE(27, 3)); - } - if (chunkType === "VP8 ") { - if (buffer.length < 30) { - return null; - } - return buildImageMetadata(buffer.readUInt16LE(26) & 0x3fff, buffer.readUInt16LE(28) & 0x3fff); - } - if (chunkType === "VP8L") { - if (buffer.length < 25 || buffer[20] !== 0x2f) { - return null; - } - const bits = buffer[21] | (buffer[22] << 8) | (buffer[23] << 16) | (buffer[24] << 24); - return buildImageMetadata((bits & 0x3fff) + 1, ((bits >> 14) & 0x3fff) + 1); - } - return null; -} - -function readBmpMetadata(buffer: Buffer): ImageMetadata | null { - if (buffer.length < 26 || buffer.toString("ascii", 0, 2) !== "BM") { - return null; - } - - const dibHeaderSize = buffer.readUInt32LE(14); - if (dibHeaderSize === 12) { - return buildImageMetadata(buffer.readUInt16LE(18), buffer.readUInt16LE(20)); - } - if (dibHeaderSize < 40 || buffer.length < 26) { - return null; - } - - return buildImageMetadata(buffer.readInt32LE(18), Math.abs(buffer.readInt32LE(22))); -} - -function readTiffUnsignedInteger(buffer: Buffer, offset: number, littleEndian: boolean): number { - return littleEndian ? buffer.readUInt16LE(offset) : buffer.readUInt16BE(offset); -} - -function readTiffUnsignedLong(buffer: Buffer, offset: number, littleEndian: boolean): number { - return littleEndian ? buffer.readUInt32LE(offset) : buffer.readUInt32BE(offset); -} - -function readTiffMetadata(buffer: Buffer): ImageMetadata | null { - if (buffer.length < 8) { - return null; - } - const byteOrder = buffer.toString("ascii", 0, 2); - const littleEndian = byteOrder === "II"; - if (!littleEndian && byteOrder !== "MM") { - return null; - } - if (readTiffUnsignedInteger(buffer, 2, littleEndian) !== 42) { - return null; - } - - const ifdOffset = readTiffUnsignedLong(buffer, 4, littleEndian); - if (ifdOffset + 2 > buffer.length) { - return null; - } - const entryCount = readTiffUnsignedInteger(buffer, ifdOffset, littleEndian); - let width: number | null = null; - let height: number | null = null; - for (let index = 0; index < entryCount; index += 1) { - const entryOffset = ifdOffset + 2 + index * 12; - if (entryOffset + 12 > buffer.length) { - return null; - } - - const tag = readTiffUnsignedInteger(buffer, entryOffset, littleEndian); - if (tag !== 256 && tag !== 257) { - continue; - } - const type = readTiffUnsignedInteger(buffer, entryOffset + 2, littleEndian); - const count = readTiffUnsignedLong(buffer, entryOffset + 4, littleEndian); - if (count !== 1 || (type !== 3 && type !== 4)) { - continue; - } - - const value = - type === 3 - ? readTiffUnsignedInteger(buffer, entryOffset + 8, littleEndian) - : readTiffUnsignedLong(buffer, entryOffset + 8, littleEndian); - if (tag === 256) { - width = value; - } else { - height = value; - } - } - - return width === null || height === null ? null : buildImageMetadata(width, height); -} - -const ISO_BMFF_IMAGE_BRANDS = new Set([ - "avif", - "avis", - "heic", - "heix", - "hevc", - "hevx", - "heif", - "mif1", - "msf1", -]); - -const ISO_BMFF_CONTAINER_BOXES = new Set([ - "edts", - "ipco", - "iprp", - "mdia", - "meta", - "minf", - "moov", - "stbl", - "trak", -]); - -function readIsoBmffBoxSize(buffer: Buffer, offset: number, end: number): number | null { - if (offset + 8 > end) { - return null; - } - const size32 = buffer.readUInt32BE(offset); - if (size32 === 0) { - return end - offset; - } - if (size32 === 1) { - if (offset + 16 > end) { - return null; - } - const size64 = buffer.readBigUInt64BE(offset + 8); - return size64 <= BigInt(Number.MAX_SAFE_INTEGER) ? Number(size64) : null; - } - return size32; -} - -function isIsoBmffImage(buffer: Buffer): boolean { - if (buffer.length < 16 || buffer.toString("ascii", 4, 8) !== "ftyp") { return false; } - const ftypSize = readIsoBmffBoxSize(buffer, 0, buffer.length); - if (!ftypSize || ftypSize < 16 || ftypSize > buffer.length) { - return false; - } - for (let offset = 8; offset + 4 <= ftypSize; offset += 4) { - if (ISO_BMFF_IMAGE_BRANDS.has(buffer.toString("ascii", offset, offset + 4))) { + + const alphaOffset = colorType === 6 ? 3 : 1; + for (let offset = alphaOffset; offset < pixels.length; offset += bytesPerPixel) { + if ((pixels[offset] ?? 255) < 255) { return true; } } return false; } -function pickLargerImageMetadata( - current: ImageMetadata | null, - candidate: ImageMetadata | null, -): ImageMetadata | null { - if (!candidate) { - return current; - } - if (!current) { - return candidate; - } - const currentPixels = BigInt(current.width) * BigInt(current.height); - const candidatePixels = BigInt(candidate.width) * BigInt(candidate.height); - return candidatePixels > currentPixels ? candidate : current; -} - -function findIsoBmffIspeMetadata( - buffer: Buffer, - start: number, - end: number, - depth: number, -): ImageMetadata | null { - if (depth > 8) { - return null; - } - let offset = start; - let largest: ImageMetadata | null = null; - while (offset + 8 <= end) { - const boxSize = readIsoBmffBoxSize(buffer, offset, end); - if (!boxSize || boxSize < 8 || offset + boxSize > end) { - return null; - } - const type = buffer.toString("ascii", offset + 4, offset + 8); - const headerSize = buffer.readUInt32BE(offset) === 1 ? 16 : 8; - const dataStart = offset + headerSize; - const boxEnd = offset + boxSize; - if (type === "ispe" && dataStart + 12 <= boxEnd) { - largest = pickLargerImageMetadata( - largest, - buildImageMetadata(buffer.readUInt32BE(dataStart + 4), buffer.readUInt32BE(dataStart + 8)), - ); - } - if (ISO_BMFF_CONTAINER_BOXES.has(type)) { - const childStart = type === "meta" ? dataStart + 4 : dataStart; - const meta = findIsoBmffIspeMetadata(buffer, childStart, boxEnd, depth + 1); - largest = pickLargerImageMetadata(largest, meta); - } - offset = boxEnd; - } - return largest; -} - -function readIsoBmffImageMetadata(buffer: Buffer): ImageMetadata | null { - if (!isIsoBmffImage(buffer)) { - return null; - } - return findIsoBmffIspeMetadata(buffer, 0, buffer.length, 0); -} - -function readJpegMetadata(buffer: Buffer): ImageMetadata | null { - if (buffer.length < 4 || buffer[0] !== 0xff || buffer[1] !== 0xd8) { - return null; - } - - let offset = 2; - while (offset + 8 < buffer.length) { - while (offset < buffer.length && buffer[offset] === 0xff) { - offset++; - } - if (offset >= buffer.length) { - return null; - } - - const marker = buffer[offset]; - offset++; - if (marker === 0xd8 || marker === 0xd9) { - continue; - } - if (marker === 0x01 || (marker >= 0xd0 && marker <= 0xd7)) { - continue; - } - if (offset + 1 >= buffer.length) { - return null; - } - - const segmentLength = buffer.readUInt16BE(offset); - if (segmentLength < 2 || offset + segmentLength > buffer.length) { - return null; - } - - const isStartOfFrame = - marker >= 0xc0 && marker <= 0xcf && marker !== 0xc4 && marker !== 0xc8 && marker !== 0xcc; - if (isStartOfFrame) { - if (segmentLength < 7 || offset + 6 >= buffer.length) { - return null; - } - return buildImageMetadata(buffer.readUInt16BE(offset + 5), buffer.readUInt16BE(offset + 3)); - } - - offset += segmentLength; - } - - return null; -} - export function readImageMetadataFromHeader(buffer: Buffer): ImageMetadata | null { - return ( - readPngMetadata(buffer) ?? - readGifMetadata(buffer) ?? - readWebpMetadata(buffer) ?? - readBmpMetadata(buffer) ?? - readTiffMetadata(buffer) ?? - readIsoBmffImageMetadata(buffer) ?? - readJpegMetadata(buffer) - ); -} - -function hasPhotonDecodableHeader(buffer: Buffer): boolean { - return ( - readPngMetadata(buffer) !== null || - readGifMetadata(buffer) !== null || - readWebpMetadata(buffer) !== null || - readJpegMetadata(buffer) !== null - ); -} - -function assertPhotonDecodableHeader(buffer: Buffer): void { - if (!hasPhotonDecodableHeader(buffer)) { - throw new Error("Photon backend skipped for image format handled by external tools"); - } -} - -function countImagePixels(meta: ImageMetadata): number | null { - const pixels = meta.width * meta.height; - return Number.isSafeInteger(pixels) ? pixels : null; -} - -function exceedsImagePixelLimit(meta: ImageMetadata): boolean { - return meta.width > Math.floor(MAX_IMAGE_INPUT_PIXELS / meta.height); -} - -function createImagePixelLimitError(meta: ImageMetadata): Error { - const pixelCount = countImagePixels(meta); - const detail = - pixelCount === null - ? `${meta.width}x${meta.height}` - : `${meta.width}x${meta.height} (${pixelCount} pixels)`; - return new Error( - `Image dimensions exceed the ${MAX_IMAGE_INPUT_PIXELS.toLocaleString("en-US")} pixel input limit: ${detail}`, - ); -} - -function validateImagePixelLimit(meta: ImageMetadata): ImageMetadata { - if (exceedsImagePixelLimit(meta)) { - throw createImagePixelLimitError(meta); - } - return meta; -} - -async function readImageMetadataForLimit(buffer: Buffer): Promise { - return readImageMetadataFromHeader(buffer); -} - -async function assertImagePixelLimit(buffer: Buffer): Promise { - const meta = await readImageMetadataForLimit(buffer); - if (!meta) { - if (shouldFailClosedOnUnknownMetadata()) { - throw new Error("Unable to determine image dimensions; refusing to process"); - } - return; - } - validateImagePixelLimit(meta); -} - -function assertKnownImagePixelLimitBeforeExternalFallback(buffer: Buffer): void { - const meta = readImageMetadataFromHeader(buffer); - if (!meta) { - throw new Error("Unable to determine image dimensions; refusing to process"); - } - validateImagePixelLimit(meta); -} - -/** - * Reads EXIF orientation from JPEG buffer. - * Returns orientation value 1-8, or null if not found/not JPEG. - * - * EXIF orientation values: - * 1 = Normal, 2 = Flip H, 3 = Rotate 180, 4 = Flip V, - * 5 = Rotate 270 CW + Flip H, 6 = Rotate 90 CW, 7 = Rotate 90 CW + Flip H, 8 = Rotate 270 CW - */ -function readJpegExifOrientation(buffer: Buffer): number | null { - // Check JPEG magic bytes - if (buffer.length < 2 || buffer[0] !== 0xff || buffer[1] !== 0xd8) { - return null; - } - - let offset = 2; - while (offset < buffer.length - 4) { - // Look for marker - if (buffer[offset] !== 0xff) { - offset++; - continue; - } - - const marker = buffer[offset + 1]; - // Skip padding FF bytes - if (marker === 0xff) { - offset++; - continue; - } - - // APP1 marker (EXIF) - if (marker === 0xe1) { - const exifStart = offset + 4; - - // Check for "Exif\0\0" header - if ( - buffer.length > exifStart + 6 && - buffer.toString("ascii", exifStart, exifStart + 4) === "Exif" && - buffer[exifStart + 4] === 0 && - buffer[exifStart + 5] === 0 - ) { - const tiffStart = exifStart + 6; - if (buffer.length < tiffStart + 8) { - return null; - } - - // Check byte order (II = little-endian, MM = big-endian) - const byteOrder = buffer.toString("ascii", tiffStart, tiffStart + 2); - const isLittleEndian = byteOrder === "II"; - - const readU16 = (pos: number) => - isLittleEndian ? buffer.readUInt16LE(pos) : buffer.readUInt16BE(pos); - const readU32 = (pos: number) => - isLittleEndian ? buffer.readUInt32LE(pos) : buffer.readUInt32BE(pos); - - // Read IFD0 offset - const ifd0Offset = readU32(tiffStart + 4); - const ifd0Start = tiffStart + ifd0Offset; - if (buffer.length < ifd0Start + 2) { - return null; - } - - const numEntries = readU16(ifd0Start); - for (let i = 0; i < numEntries; i++) { - const entryOffset = ifd0Start + 2 + i * 12; - if (buffer.length < entryOffset + 12) { - break; - } - - const tag = readU16(entryOffset); - // Orientation tag = 0x0112 - if (tag === 0x0112) { - const value = readU16(entryOffset + 8); - return value >= 1 && value <= 8 ? value : null; - } - } - } - return null; - } - - // Skip other segments - if (marker >= 0xe0 && marker <= 0xef) { - const segmentLength = buffer.readUInt16BE(offset + 2); - offset += 2 + segmentLength; - continue; - } - - // SOF, SOS, or other marker - stop searching - if (marker === 0xc0 || marker === 0xda) { - break; - } - - offset++; - } - - return null; -} - -async function withImageTemp(fn: (workspace: TempWorkspace) => Promise): Promise { - return await withTempWorkspace( - { rootDir: resolvePreferredOpenClawTmpDir(), prefix: "openclaw-img-" }, - fn, - ); -} - -function clampInteger(value: number, min: number, max: number): number { - return Math.max(min, Math.min(max, Math.round(value))); -} - -function resolveImageTool(backend: Exclude): ExternalImageTool | null { - if (backend === "sips") { - return process.platform === "darwin" - ? { backend, flavor: "sips", command: "/usr/bin/sips" } - : null; - } - if (backend === "windows-native") { - const powershell = resolveSystemBin("powershell", { trust: "strict" }); - return powershell && process.platform === "win32" - ? { backend, flavor: "powershell", command: powershell } - : null; - } - if (backend === "imagemagick") { - const magick = resolveSystemBin("magick", { trust: "standard" }); - if (magick) { - return { backend, flavor: "magick", command: magick }; - } - if (process.platform !== "win32") { - const convert = resolveSystemBin("convert", { trust: "standard" }); - if (convert) { - return { backend, flavor: "convert", command: convert }; - } - } - return null; - } - if (backend === "graphicsmagick") { - const gm = resolveSystemBin("gm", { trust: "standard" }); - return gm ? { backend, flavor: "gm", command: gm } : null; - } - const ffmpeg = resolveSystemBin("ffmpeg", { trust: "standard" }); - return ffmpeg ? { backend, flavor: "ffmpeg", command: ffmpeg } : null; -} - -function convertToolArgs( - tool: Extract, - args: string[], -): string[] { - return tool.flavor === "gm" ? ["convert", ...args] : args; -} - -async function runPowerShellImageScript( - scriptName: string, - script: string, - args: readonly string[], -): Promise<{ stdout: string }> { - const tool = resolveImageTool("windows-native"); - if (!tool || tool.flavor !== "powershell") { - throw new Error("Windows native image backend is not available"); - } - return await withImageTemp(async (workspace) => { - const scriptPath = await workspace.write(scriptName, Buffer.from(script, "utf8")); - return await runExec( - tool.command, - ["-NoProfile", "-NonInteractive", "-ExecutionPolicy", "Bypass", "-File", scriptPath, ...args], - { - timeoutMs: IMAGE_PROCESS_TIMEOUT_MS, - maxBuffer: IMAGE_TOOL_MAX_BUFFER, - }, - ); - }); -} - -const WINDOWS_NATIVE_RESIZE_SCRIPT = ` -param( - [string]$InputPath, - [string]$OutputPath, - [int]$MaxSide, - [int]$Quality, - [int]$WithoutEnlargement, - [string]$Format -) -$ErrorActionPreference = 'Stop' -Add-Type -AssemblyName System.Drawing -$source = [System.Drawing.Image]::FromFile($InputPath) -$bitmap = $null -$graphics = $null -try { - try { - if ($source.PropertyIdList -contains 274) { - $orientation = [BitConverter]::ToUInt16($source.GetPropertyItem(274).Value, 0) - switch ($orientation) { - 2 { $source.RotateFlip([System.Drawing.RotateFlipType]::RotateNoneFlipX) } - 3 { $source.RotateFlip([System.Drawing.RotateFlipType]::Rotate180FlipNone) } - 4 { $source.RotateFlip([System.Drawing.RotateFlipType]::Rotate180FlipX) } - 5 { $source.RotateFlip([System.Drawing.RotateFlipType]::Rotate90FlipX) } - 6 { $source.RotateFlip([System.Drawing.RotateFlipType]::Rotate90FlipNone) } - 7 { $source.RotateFlip([System.Drawing.RotateFlipType]::Rotate270FlipX) } - 8 { $source.RotateFlip([System.Drawing.RotateFlipType]::Rotate270FlipNone) } - } - try { $source.RemovePropertyItem(274) } catch {} - } - } catch {} - $maxDim = [Math]::Max($source.Width, $source.Height) - if ($maxDim -le 0) { throw 'Invalid image dimensions' } - $scale = $MaxSide / [double]$maxDim - if ($WithoutEnlargement -eq 1) { - $scale = [Math]::Min(1.0, $scale) - } - $width = [Math]::Max(1, [int][Math]::Round($source.Width * $scale)) - $height = [Math]::Max(1, [int][Math]::Round($source.Height * $scale)) - $pixelFormat = [System.Drawing.Imaging.PixelFormat]::Format24bppRgb - if ($Format -eq 'png') { - $pixelFormat = [System.Drawing.Imaging.PixelFormat]::Format32bppArgb - } - $bitmap = New-Object System.Drawing.Bitmap($width, $height, $pixelFormat) - $graphics = [System.Drawing.Graphics]::FromImage($bitmap) - $graphics.CompositingQuality = [System.Drawing.Drawing2D.CompositingQuality]::HighQuality - $graphics.InterpolationMode = [System.Drawing.Drawing2D.InterpolationMode]::HighQualityBicubic - $graphics.SmoothingMode = [System.Drawing.Drawing2D.SmoothingMode]::HighQuality - if ($Format -eq 'png') { - $graphics.Clear([System.Drawing.Color]::Transparent) - } else { - $graphics.Clear([System.Drawing.Color]::White) - } - $graphics.DrawImage($source, 0, 0, $width, $height) - if ($Format -eq 'png') { - $bitmap.Save($OutputPath, [System.Drawing.Imaging.ImageFormat]::Png) - } else { - $codec = [System.Drawing.Imaging.ImageCodecInfo]::GetImageEncoders() | - Where-Object { $_.MimeType -eq 'image/jpeg' } | - Select-Object -First 1 - if ($null -eq $codec) { throw 'JPEG encoder not available' } - $encoder = [System.Drawing.Imaging.Encoder]::Quality - $encoderParam = New-Object System.Drawing.Imaging.EncoderParameter($encoder, [int64]$Quality) - $encoderParams = New-Object System.Drawing.Imaging.EncoderParameters(1) - try { - $encoderParams.Param[0] = $encoderParam - $bitmap.Save($OutputPath, $codec, $encoderParams) - } finally { - $encoderParam.Dispose() - $encoderParams.Dispose() - } - } -} finally { - if ($null -ne $graphics) { $graphics.Dispose() } - if ($null -ne $bitmap) { $bitmap.Dispose() } - $source.Dispose() -} -`; - -async function windowsNativeResize( - params: ResizeToJpegParams | ResizeToPngParams, - format: "jpeg" | "png", -): Promise { - return await withImageTemp(async (workspace) => { - const input = await workspace.write("in.img", params.buffer); - const outputName = format === "png" ? "out.png" : "out.jpg"; - const output = workspace.path(outputName); - await runPowerShellImageScript("resize.ps1", WINDOWS_NATIVE_RESIZE_SCRIPT, [ - input, - output, - String(clampInteger(params.maxSide, 1, Number.MAX_SAFE_INTEGER)), - String(clampInteger("quality" in params ? params.quality : 90, 1, 100)), - params.withoutEnlargement === false ? "0" : "1", - format === "png" ? "png" : "jpeg", - ]); - return await workspace.read(outputName); - }); -} - -async function runConvertTool( - tool: Extract, - args: string[], -): Promise { - await runExec(tool.command, convertToolArgs(tool, args), { - timeoutMs: IMAGE_PROCESS_TIMEOUT_MS, - maxBuffer: IMAGE_TOOL_MAX_BUFFER, - }); -} - -function buildResizeGeometry(maxSide: number, withoutEnlargement?: boolean): string { - const side = clampInteger(maxSide, 1, Number.MAX_SAFE_INTEGER); - return `${side}x${side}${withoutEnlargement === false ? "" : ">"}`; -} - -function buildFfmpegResizeFilter(maxSide: number, withoutEnlargement?: boolean): string { - const side = clampInteger(maxSide, 1, Number.MAX_SAFE_INTEGER); - if (withoutEnlargement === false) { - return `scale=w=${side}:h=${side}:force_original_aspect_ratio=decrease`; - } - return `scale=w='min(${side},iw)':h='min(${side},ih)':force_original_aspect_ratio=decrease`; -} - -async function externalResizeToJpeg( - backend: Exclude, - params: ResizeToJpegParams, -): Promise { - const tool = resolveImageTool(backend); - if (!tool) { - throw new Error(`Image backend ${backend} is not available`); - } - if (tool.flavor === "sips") { - const normalized = await normalizeExifOrientationSips(params.buffer); - if (params.withoutEnlargement !== false) { - const meta = await getImageMetadata(normalized); - if (meta) { - const maxDim = Math.max(meta.width, meta.height); - if (maxDim > 0 && maxDim <= params.maxSide) { - return await sipsResizeToJpeg({ - buffer: normalized, - maxSide: maxDim, - quality: params.quality, - }); - } - } - } - return await sipsResizeToJpeg({ - buffer: normalized, - maxSide: params.maxSide, - quality: params.quality, - }); - } - if (tool.flavor === "powershell") { - return await windowsNativeResize(params, "jpeg"); - } - - return await withImageTemp(async (workspace) => { - const input = await workspace.write("in.img", params.buffer); - const output = workspace.path("out.jpg"); - if (tool.flavor === "ffmpeg") { - const side = clampInteger(params.maxSide, 1, Number.MAX_SAFE_INTEGER); - const qv = clampInteger(31 - params.quality * 0.29, 2, 31); - await runExec( - tool.command, - [ - "-y", - "-i", - input, - "-vf", - buildFfmpegResizeFilter(side, params.withoutEnlargement), - "-frames:v", - "1", - "-q:v", - String(qv), - output, - ], - { timeoutMs: IMAGE_PROCESS_TIMEOUT_MS, maxBuffer: IMAGE_TOOL_MAX_BUFFER }, - ); - return await workspace.read("out.jpg"); - } - - await runConvertTool(tool, [ - input, - "-auto-orient", - "-resize", - buildResizeGeometry(params.maxSide, params.withoutEnlargement), - "-quality", - String(clampInteger(params.quality, 1, 100)), - output, - ]); - return await workspace.read("out.jpg"); - }); -} - -async function externalConvertToJpeg( - backend: Exclude, - buffer: Buffer, -): Promise { - const tool = resolveImageTool(backend); - if (!tool) { - throw new Error(`Image backend ${backend} is not available`); - } - if (tool.flavor === "sips") { - return await sipsConvertToJpeg(buffer); - } - if (tool.flavor === "powershell") { - throw new Error("Windows native image backend does not convert HEIC to JPEG"); - } - return await withImageTemp(async (workspace) => { - const input = await workspace.write("in.img", buffer); - const output = workspace.path("out.jpg"); - if (tool.flavor === "ffmpeg") { - await runExec(tool.command, ["-y", "-i", input, "-frames:v", "1", "-q:v", "3", output], { - timeoutMs: IMAGE_PROCESS_TIMEOUT_MS, - maxBuffer: IMAGE_TOOL_MAX_BUFFER, - }); - } else { - await runConvertTool(tool, [input, "-auto-orient", "-quality", "90", output]); - } - return await workspace.read("out.jpg"); - }); -} - -async function externalNormalizeExifOrientation( - backend: Exclude, - buffer: Buffer, -): Promise { - if (backend === "sips") { - return await normalizeExifOrientationSips(buffer); - } - const tool = resolveImageTool(backend); - if (!tool || tool.flavor === "ffmpeg" || tool.flavor === "sips" || tool.flavor === "powershell") { - throw new Error(`Image backend ${backend} is not available`); - } - if (!readJpegExifOrientation(buffer)) { - return buffer; - } - return await withImageTemp(async (workspace) => { - const input = await workspace.write("in.jpg", buffer); - const output = workspace.path("out.jpg"); - await runConvertTool(tool, [input, "-auto-orient", output]); - return await workspace.read("out.jpg"); - }); -} - -async function externalResizeToPng( - backend: Exclude, - params: ResizeToPngParams, -): Promise { - const tool = resolveImageTool(backend); - if (!tool || tool.flavor === "ffmpeg" || tool.flavor === "sips") { - throw new Error(`Image backend ${backend} is not available`); - } - if (tool.flavor === "powershell") { - return await windowsNativeResize(params, "png"); - } - return await withImageTemp(async (workspace) => { - const input = await workspace.write("in.img", params.buffer); - const output = workspace.path("out.png"); - const args = [ - input, - "-auto-orient", - "-resize", - buildResizeGeometry(params.maxSide, params.withoutEnlargement), - ]; - const compressionLevel = params.compressionLevel; - if (compressionLevel !== undefined && tool.flavor !== "gm") { - args.push("-define", `png:compression-level=${clampInteger(compressionLevel, 0, 9)}`); - } - args.push(output); - await runConvertTool(tool, args); - return await workspace.read("out.png"); - }); -} - -async function sipsResizeToJpeg(params: { - buffer: Buffer; - maxSide: number; - quality: number; -}): Promise { - return await withImageTemp(async (workspace) => { - const input = await workspace.write("in.img", params.buffer); - const output = workspace.path("out.jpg"); - await runExec( - "/usr/bin/sips", - [ - "-Z", - String(Math.max(1, Math.round(params.maxSide))), - "-s", - "format", - "jpeg", - "-s", - "formatOptions", - String(Math.max(1, Math.min(100, Math.round(params.quality)))), - input, - "--out", - output, - ], - { timeoutMs: IMAGE_PROCESS_TIMEOUT_MS, maxBuffer: IMAGE_TOOL_MAX_BUFFER }, - ); - return await workspace.read("out.jpg"); - }); -} - -async function sipsConvertToJpeg(buffer: Buffer): Promise { - return await withImageTemp(async (workspace) => { - const input = await workspace.write("in.heic", buffer); - const output = workspace.path("out.jpg"); - await runExec("/usr/bin/sips", ["-s", "format", "jpeg", input, "--out", output], { - timeoutMs: IMAGE_PROCESS_TIMEOUT_MS, - maxBuffer: IMAGE_TOOL_MAX_BUFFER, - }); - return await workspace.read("out.jpg"); - }); + return readRastermillImageMetadataFromHeader(buffer); } export async function getImageMetadata(buffer: Buffer): Promise { - const metadataForLimit = await readImageMetadataForLimit(buffer).catch(() => null); - if (metadataForLimit) { - try { - return validateImagePixelLimit(metadataForLimit); - } catch { - return null; - } - } - - const preference = getImageBackendPreference(); - if (preference !== "auto" && preference !== "photon") { - return null; - } - - return await (async () => { - const meta = await (await loadMediaAttachmentImageOps()).getImageMetadata(buffer); - return meta ? validateImagePixelLimit(meta) : null; - })().catch(() => null); + const info = await createOpenClawRastermill().probe(buffer); + return info ? { width: info.width, height: info.height } : null; } -/** - * Applies rotation/flip to image buffer using sips based on EXIF orientation. - */ -async function sipsApplyOrientation(buffer: Buffer, orientation: number): Promise { - // Map EXIF orientation to sips operations - // sips -r rotates clockwise, -f flips (horizontal/vertical) - const ops: string[] = []; - switch (orientation) { - case 2: // Flip horizontal - ops.push("-f", "horizontal"); - break; - case 3: // Rotate 180 - ops.push("-r", "180"); - break; - case 4: // Flip vertical - ops.push("-f", "vertical"); - break; - case 5: // Rotate 270 CW + flip horizontal - ops.push("-r", "270", "-f", "horizontal"); - break; - case 6: // Rotate 90 CW - ops.push("-r", "90"); - break; - case 7: // Rotate 90 CW + flip horizontal - ops.push("-r", "90", "-f", "horizontal"); - break; - case 8: // Rotate 270 CW - ops.push("-r", "270"); - break; - default: - // Orientation 1 or unknown - no change needed - return buffer; - } - - return await withImageTemp(async (workspace) => { - const input = await workspace.write("in.jpg", buffer); - const output = workspace.path("out.jpg"); - await runExec("/usr/bin/sips", [...ops, input, "--out", output], { - timeoutMs: IMAGE_PROCESS_TIMEOUT_MS, - maxBuffer: IMAGE_TOOL_MAX_BUFFER, - }); - return await workspace.read("out.jpg"); - }); -} - -/** - * Normalizes EXIF orientation in an image buffer. - * Returns the buffer with correct pixel orientation (rotated if needed). - * Falls back to original buffer if normalization fails. - */ export async function normalizeExifOrientation(buffer: Buffer): Promise { - await assertImagePixelLimit(buffer); - - for (const backend of imageBackendsForOperation("normalizeExifOrientation")) { - try { - if (backend === "photon") { - assertPhotonDecodableHeader(buffer); - const ops = await loadMediaAttachmentImageOps(); - return await ops.normalizeExifOrientation(buffer); - } - if (backend !== "ffmpeg") { - assertKnownImagePixelLimitBeforeExternalFallback(buffer); - return await externalNormalizeExifOrientation(backend, buffer); - } - } catch { - // Orientation normalization is best-effort; resizing still handles raw buffers. + try { + assertImageInputWithinPixelBudget(buffer); + const rastermill = createOpenClawRastermillForInput(buffer); + const info = await rastermill.probe(buffer); + if (!info?.orientation || info.orientation === 1) { + return buffer; } + return (await rastermill.encode(buffer, { format: "jpeg", autoOrient: true })).data; + } catch (error) { + if (isImageProcessorUnavailableError(error)) { + return buffer; + } + return wrapRastermillUnavailable("normalizeExifOrientation", error); } - - return buffer; } export async function resizeToJpeg(params: ResizeToJpegParams): Promise { - await assertImagePixelLimit(params.buffer); - return await runWithImageBackends("resizeToJpeg", async (backend) => { - if (backend === "photon") { - assertPhotonDecodableHeader(params.buffer); - return await (await loadMediaAttachmentImageOps()).resizeToJpeg(params); - } - assertKnownImagePixelLimitBeforeExternalFallback(params.buffer); - return await externalResizeToJpeg(backend, params); - }); + try { + return ( + await createOpenClawRastermillForInput(params.buffer).encode(params.buffer, { + format: "jpeg", + resize: { + maxSide: params.maxSide, + enlarge: params.withoutEnlargement === false, + }, + quality: params.quality, + }) + ).data; + } catch (error) { + return wrapRastermillUnavailable("resizeToJpeg", error); + } } export async function convertHeicToJpeg(buffer: Buffer): Promise { - await assertImagePixelLimit(buffer); - return await runWithImageBackends("convertHeicToJpeg", async (backend) => { - if (backend === "photon") { - throw new Error("Photon does not support HEIC/AVIF conversion"); - } - assertKnownImagePixelLimitBeforeExternalFallback(buffer); - return await externalConvertToJpeg(backend, buffer); - }); -} - -/** - * Checks if an image has an alpha channel (transparency). - * Returns true if the image has alpha, false otherwise. - */ -export async function hasAlphaChannel(buffer: Buffer): Promise { - await assertImagePixelLimit(buffer); - - const pngAlphaChannel = readPngAlphaChannel(buffer); - if (pngAlphaChannel !== null) { - return pngAlphaChannel; - } - try { - const ops = await loadMediaAttachmentImageOps(); - return await ops.hasAlphaChannel(buffer); - } catch { - return false; + return (await createOpenClawRastermill().encode(buffer, { format: "jpeg" })).data; + } catch (error) { + return wrapRastermillUnavailable("convertHeicToJpeg", error); + } +} + +export async function hasAlphaChannel(buffer: Buffer): Promise { + try { + assertImageInputWithinPixelBudget(buffer); + const rastermill = createOpenClawRastermillForInput(buffer); + const info = await rastermill.probe(buffer); + if (!info) { + return false; + } + if (info.hasAlpha !== null) { + return info.hasAlpha; + } + try { + const png = await rastermill.encode(buffer, { + format: "png", + autoOrient: false, + }); + return decodedPngHasTransparentPixel(png.data) ?? false; + } catch { + return false; + } + } catch (error) { + if (isImageProcessorUnavailableError(error)) { + return false; + } + throw error; } } -/** - * Resizes an image to PNG format, preserving alpha channel (transparency). - * Falls back to the media attachments plugin only (no sips fallback for PNG with alpha). - */ export async function resizeToPng(params: ResizeToPngParams): Promise { - await assertImagePixelLimit(params.buffer); - return await runWithImageBackends("resizeToPng", async (backend) => { - if (backend === "photon") { - assertPhotonDecodableHeader(params.buffer); - return await (await loadMediaAttachmentImageOps()).resizeToPng(params); - } - if (backend === "windows-native" || backend === "imagemagick" || backend === "graphicsmagick") { - assertKnownImagePixelLimitBeforeExternalFallback(params.buffer); - return await externalResizeToPng(backend, params); - } - throw new Error(`Image backend ${backend} is not available for PNG resizing`); - }); + try { + return ( + await createOpenClawRastermillForInput(params.buffer).encode(params.buffer, { + format: "png", + resize: { + maxSide: params.maxSide, + enlarge: params.withoutEnlargement === false, + }, + ...(params.compressionLevel === undefined + ? {} + : { compressionLevel: params.compressionLevel }), + }) + ).data; + } catch (error) { + return wrapRastermillUnavailable("resizeToPng", error); + } } export async function optimizeImageToPng( @@ -1343,74 +394,19 @@ export async function optimizeImageToPng( resizeSide: number; compressionLevel: number; }> { - // Try a grid of sizes/compression levels until under the limit. - // PNG uses compression levels 0-9 (higher = smaller but slower). - const sides = options?.sides?.length ? [...options.sides] : [2048, 1536, 1280, 1024, 800]; - const compressionLevels = [6, 7, 8, 9]; - let smallest: { - buffer: Buffer; - size: number; - resizeSide: number; - compressionLevel: number; - } | null = null; - let firstResizeError: unknown; - - for (const side of sides) { - for (const compressionLevel of compressionLevels) { - try { - const out = await resizeToPng({ - buffer, - maxSide: side, - compressionLevel, - withoutEnlargement: true, - }); - const size = out.length; - if (!smallest || size < smallest.size) { - smallest = { buffer: out, size, resizeSide: side, compressionLevel }; - } - if (size <= maxBytes) { - return { - buffer: out, - optimizedSize: size, - resizeSide: side, - compressionLevel, - }; - } - } catch (err) { - firstResizeError ??= err; - // Continue trying other size/compression combinations. - } - } - } - - if (smallest) { - return { - buffer: smallest.buffer, - optimizedSize: smallest.size, - resizeSide: smallest.resizeSide, - compressionLevel: smallest.compressionLevel, - }; - } - - if (firstResizeError) { - throw firstResizeError; - } - - throw new Error("Failed to optimize PNG image"); -} - -/** - * Internal sips-only EXIF normalization (no Photon fallback). - * Used by resizeToJpeg to normalize before sips resize. - */ -async function normalizeExifOrientationSips(buffer: Buffer): Promise { try { - const orientation = readJpegExifOrientation(buffer); - if (!orientation || orientation === 1) { - return buffer; - } - return await sipsApplyOrientation(buffer, orientation); - } catch { - return buffer; + const out = await createOpenClawRastermillForInput(buffer).encodeWithinBytes(buffer, { + format: "png", + maxBytes, + search: options?.sides === undefined ? {} : { maxSide: options.sides }, + }); + return { + buffer: out.data, + optimizedSize: out.bytes, + resizeSide: out.chosen.maxSide ?? out.width, + compressionLevel: out.chosen.compressionLevel ?? 6, + }; + } catch (error) { + return wrapRastermillUnavailable("optimizeImageToPng", error); } } diff --git a/test/package-manager-config.test.ts b/test/package-manager-config.test.ts index cbb76fb2c446..2f09ae10acab 100644 --- a/test/package-manager-config.test.ts +++ b/test/package-manager-config.test.ts @@ -33,16 +33,20 @@ function readJson(filePath: string): unknown { function collectPnpmLockPackages(): Set { const lockfile = parse(fs.readFileSync("pnpm-lock.yaml", "utf8")) as { - packages?: Record; + packages?: Record; }; - return new Set( - Object.keys(lockfile.packages ?? {}) - .map((packageKey) => { - const parsed = parsePnpmPackageKey(packageKey); - return parsed ? `${parsed.name}@${parsed.version}` : null; - }) - .filter((packageKey) => packageKey !== null), - ); + const packages = new Set(); + for (const [packageKey, metadata] of Object.entries(lockfile.packages ?? {})) { + const parsed = parsePnpmPackageKey(packageKey); + if (!parsed) { + continue; + } + packages.add(`${parsed.name}@${parsed.version}`); + if (typeof metadata.version === "string") { + packages.add(`${parsed.name}@${metadata.version}`); + } + } + return packages; } describe("package manager build policy", () => { diff --git a/test/scripts/check-dependency-pins.test.ts b/test/scripts/check-dependency-pins.test.ts index d1c3605cb56b..a02f7634327a 100644 --- a/test/scripts/check-dependency-pins.test.ts +++ b/test/scripts/check-dependency-pins.test.ts @@ -62,6 +62,8 @@ describe("check-dependency-pins", () => { linked: "link:../linked", local: "file:../local", gitPinned: "github:owner/repo#0123456789abcdef0123456789abcdef01234567", + tarballPinned: + "https://codeload.github.com/owner/repo/tar.gz/0123456789abcdef0123456789abcdef01234567", }, devDependencies: { devExact: "4.5.6",