diff --git a/npm-shrinkwrap.json b/npm-shrinkwrap.json index 98ebd55e2968..5c8ebb3c01b6 100644 --- a/npm-shrinkwrap.json +++ b/npm-shrinkwrap.json @@ -47,7 +47,7 @@ "playwright-core": "1.60.0", "qrcode": "1.5.4", "quickjs-wasi": "2.2.0", - "rastermill": "https://codeload.github.com/openclaw/rastermill/tar.gz/48457f54bbbe8061cdf358069eee202e27b553be", + "rastermill": "https://codeload.github.com/openclaw/rastermill/tar.gz/89405b23b10e32bcbedbd0e2aa56ba790e94d148", "tar": "7.5.15", "tokenjuice": "0.7.1", "tree-sitter-bash": "0.25.1", @@ -3843,8 +3843,8 @@ }, "node_modules/rastermill": { "version": "0.2.0", - "resolved": "https://codeload.github.com/openclaw/rastermill/tar.gz/48457f54bbbe8061cdf358069eee202e27b553be", - "integrity": "sha512-wGbsJuFVkrwgWcaLESPi9OXFIV9MoJQtO8OwKUitKz6Q+XAsSvBhg08lE7JlBmutYFZofzMv6/GyxMl1ITwMqg==", + "resolved": "https://codeload.github.com/openclaw/rastermill/tar.gz/89405b23b10e32bcbedbd0e2aa56ba790e94d148", + "integrity": "sha512-Rzs9xELc0+t90r0kMon/AqBwkxT4R7En4OT04C4wBpDpQO4M+QLxHYaBBJWSTrEUKn+eunArcnwx+O2GGW/mGQ==", "license": "MIT", "dependencies": { "@silvia-odwyer/photon-node": "0.3.4" diff --git a/package.json b/package.json index 6afc7faa3080..120937fd887d 100644 --- a/package.json +++ b/package.json @@ -1838,7 +1838,7 @@ "playwright-core": "1.60.0", "qrcode": "1.5.4", "quickjs-wasi": "2.2.0", - "rastermill": "https://codeload.github.com/openclaw/rastermill/tar.gz/48457f54bbbe8061cdf358069eee202e27b553be", + "rastermill": "https://codeload.github.com/openclaw/rastermill/tar.gz/89405b23b10e32bcbedbd0e2aa56ba790e94d148", "tar": "7.5.15", "tokenjuice": "0.7.1", "tree-sitter-bash": "0.25.1", diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 3875382e0613..5909fa7fc169 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -152,8 +152,8 @@ importers: specifier: 2.2.0 version: 2.2.0 rastermill: - specifier: https://codeload.github.com/openclaw/rastermill/tar.gz/48457f54bbbe8061cdf358069eee202e27b553be - version: https://codeload.github.com/openclaw/rastermill/tar.gz/48457f54bbbe8061cdf358069eee202e27b553be + specifier: https://codeload.github.com/openclaw/rastermill/tar.gz/89405b23b10e32bcbedbd0e2aa56ba790e94d148 + version: https://codeload.github.com/openclaw/rastermill/tar.gz/89405b23b10e32bcbedbd0e2aa56ba790e94d148 tar: specifier: 7.5.15 version: 7.5.15 @@ -6310,8 +6310,8 @@ packages: resolution: {integrity: sha512-Hrgsx+orqoygnmhFbKaHE6c296J+HTAQXoxEF6gNupROmmGJRoyzfG3ccAveqCBrwr/2yxQ5BVd/GTl5agOwSg==} engines: {node: '>= 0.6'} - rastermill@https://codeload.github.com/openclaw/rastermill/tar.gz/48457f54bbbe8061cdf358069eee202e27b553be: - resolution: {gitHosted: true, integrity: sha512-wGbsJuFVkrwgWcaLESPi9OXFIV9MoJQtO8OwKUitKz6Q+XAsSvBhg08lE7JlBmutYFZofzMv6/GyxMl1ITwMqg==, tarball: https://codeload.github.com/openclaw/rastermill/tar.gz/48457f54bbbe8061cdf358069eee202e27b553be} + rastermill@https://codeload.github.com/openclaw/rastermill/tar.gz/89405b23b10e32bcbedbd0e2aa56ba790e94d148: + resolution: {gitHosted: true, integrity: sha512-Rzs9xELc0+t90r0kMon/AqBwkxT4R7En4OT04C4wBpDpQO4M+QLxHYaBBJWSTrEUKn+eunArcnwx+O2GGW/mGQ==, tarball: https://codeload.github.com/openclaw/rastermill/tar.gz/89405b23b10e32bcbedbd0e2aa56ba790e94d148} version: 0.2.0 engines: {node: '>=20'} @@ -12304,7 +12304,7 @@ snapshots: range-parser@1.2.1: {} - rastermill@https://codeload.github.com/openclaw/rastermill/tar.gz/48457f54bbbe8061cdf358069eee202e27b553be: + rastermill@https://codeload.github.com/openclaw/rastermill/tar.gz/89405b23b10e32bcbedbd0e2aa56ba790e94d148: dependencies: '@silvia-odwyer/photon-node': 0.3.4 diff --git a/src/gateway/managed-image-attachments.ts b/src/gateway/managed-image-attachments.ts index b0cd05faa2fe..cdfbba8dce4c 100644 --- a/src/gateway/managed-image-attachments.ts +++ b/src/gateway/managed-image-attachments.ts @@ -10,7 +10,7 @@ import { assertLocalMediaAllowed } from "../media/local-media-access.js"; import { createImageProcessor, getImageMetadata, - hasAlphaChannel, + readImageProbeFromHeader, } from "../media/media-services.js"; import { isPassThroughRemoteMediaSource } from "../media/media-source-url.js"; import { MEDIA_MAX_BYTES, saveMediaBuffer, saveMediaSource } from "../media/store.js"; @@ -189,70 +189,37 @@ function getManagedImageMetadataLimitError( return null; } -function computeManagedImageResizeTarget( - metadata: { width: number; height: number }, - limits: ManagedImageAttachmentLimits, +function orientManagedImageMetadata( + buffer: Buffer, + metadata: { width: number; height: number } | null, ): { width: number; height: number } | null { - const scale = Math.min( - 1, - limits.maxWidth / metadata.width, - limits.maxHeight / metadata.height, - Math.sqrt(limits.maxPixels / (metadata.width * metadata.height)), - ); - if (!Number.isFinite(scale) || scale >= 1) { + if (!metadata) { return null; } - - let width = Math.max(1, Math.floor(metadata.width * scale)); - let height = Math.max(1, Math.floor(metadata.height * scale)); - while ( - width > limits.maxWidth || - height > limits.maxHeight || - width * height > limits.maxPixels - ) { - if (width >= height && width > 1) { - width -= 1; - } else if (height > 1) { - height -= 1; - } else { - break; - } - } - return { width, height }; + const orientation = readImageProbeFromHeader(buffer)?.orientation; + return orientation === 5 || orientation === 6 || orientation === 7 || orientation === 8 + ? { width: metadata.height, height: metadata.width } + : metadata; } async function resizeManagedImageBufferToLimits(params: { buffer: Buffer; - contentType: string; - metadata: { width: number; height: number }; limits: ManagedImageAttachmentLimits; }): Promise<{ buffer: Buffer; contentType: string; width: number; height: number }> { - const target = computeManagedImageResizeTarget(params.metadata, params.limits); - if (!target) { - return { - buffer: params.buffer, - contentType: params.contentType, - width: params.metadata.width, - height: params.metadata.height, - }; - } - - const preserveAlpha = await hasAlphaChannel(params.buffer); - const resized = await createImageProcessor().encodeBest(params.buffer, { - resize: { - width: target.width, - height: target.height, - fit: "inside", - enlarge: false, + const resized = await createImageProcessor().encodeToLimits(params.buffer, { + limits: { + maxWidth: params.limits.maxWidth, + maxHeight: params.limits.maxHeight, + maxPixels: params.limits.maxPixels, }, opaque: { format: "jpeg", quality: 92 }, transparent: { format: "png", compressionLevel: 9 }, - transparency: preserveAlpha ? "preserve" : "flatten", + transparency: "auto", }); return { buffer: resized.data, - contentType: resized.format === "png" ? "image/png" : "image/jpeg", + contentType: resized.mimeType, width: resized.width, height: resized.height, }; @@ -851,7 +818,8 @@ export async function createManagedOutgoingImageBlocks(params: { originalStats.width != null && originalStats.height != null ? { width: originalStats.width, height: originalStats.height } : await getImageMetadata(originalBuffer); - let effectiveMetadata = originalMetadata; + const originalDisplayMetadata = orientManagedImageMetadata(originalBuffer, originalMetadata); + let effectiveMetadata = originalDisplayMetadata; let metadataLimitError = getManagedImageMetadataLimitError(effectiveMetadata, alt, limits); for (let resizeAttempt = 0; metadataLimitError; resizeAttempt += 1) { if (!effectiveMetadata) { @@ -862,8 +830,6 @@ export async function createManagedOutgoingImageBlocks(params: { } const resized = await resizeManagedImageBufferToLimits({ buffer: originalBuffer, - contentType: savedOriginalContentType, - metadata: effectiveMetadata, limits, }); validateManagedImageBuffer(resized.buffer, alt, limits); @@ -880,16 +846,20 @@ export async function createManagedOutgoingImageBlocks(params: { savedOriginalPath = savedOriginal.path; originalBuffer = resized.buffer; originalStats = await getVariantStats(savedOriginal.path); - effectiveMetadata = + effectiveMetadata = orientManagedImageMetadata( + originalBuffer, originalStats.width != null && originalStats.height != null ? { width: originalStats.width, height: originalStats.height } - : await getImageMetadata(originalBuffer); + : await getImageMetadata(originalBuffer), + ); metadataLimitError = getManagedImageMetadataLimitError(effectiveMetadata, alt, limits); if (!metadataLimitError) { resizeWarning = buildManagedImageResizeWarningBlock({ alt, - originalWidth: originalMetadata?.width ?? effectiveMetadata?.width ?? resized.width, - originalHeight: originalMetadata?.height ?? effectiveMetadata?.height ?? resized.height, + originalWidth: + originalDisplayMetadata?.width ?? effectiveMetadata?.width ?? resized.width, + originalHeight: + originalDisplayMetadata?.height ?? effectiveMetadata?.height ?? resized.height, resizedWidth: effectiveMetadata?.width ?? resized.width, resizedHeight: effectiveMetadata?.height ?? resized.height, }); diff --git a/src/media/image-ops.ts b/src/media/image-ops.ts index c85f32deaa90..fe37c884d863 100644 --- a/src/media/image-ops.ts +++ b/src/media/image-ops.ts @@ -80,14 +80,6 @@ export function readImageProbeFromHeader(buffer: Buffer): ImageProbe | null { return readRastermillImageProbeFromHeader(buffer); } -export function shouldAttemptTransparencyPreservingEncode(buffer: Buffer): boolean { - const probe = readRastermillImageProbeFromHeader(buffer); - if (probe?.hasAlpha !== true) { - return false; - } - return probe.format === "png" || probe.format === "gif" || probe.format === "webp"; -} - function wrapRastermillUnavailable(operation: string, error: unknown): never { if (error instanceof RastermillUnavailableError) { throw new ImageProcessorUnavailableError(operation, error.message, error.causes); diff --git a/src/media/web-media.ts b/src/media/web-media.ts index 208c7c2b9dfa..174a27d8916a 100644 --- a/src/media/web-media.ts +++ b/src/media/web-media.ts @@ -21,7 +21,6 @@ import { createImageProcessor, readImageMetadataFromHeader, readImageProbeFromHeader, - shouldAttemptTransparencyPreservingEncode, } from "./media-services.js"; import { detectMime, @@ -357,7 +356,8 @@ type OptimizedImage = { buffer: Buffer; optimizedSize: number; resizeSide: number; - format: "jpeg" | "png"; + format: "jpeg" | "png" | "webp"; + mimeType: string; quality?: number; compressionLevel?: number; }; @@ -654,7 +654,7 @@ async function optimizeImageWithFallback(params: { maxSide: grid.sides, quality: grid.qualities, }, - transparency: shouldAttemptTransparencyPreservingEncode(buffer) ? "prefer" : "flatten", + transparency: "auto", }); if (optimized.chosen.transparency === "flattened" && shouldLogVerbose()) { logVerbose(`Image transparency flattened to fit ${formatMb(cap, 0)}MB optimization budget`); @@ -663,7 +663,8 @@ async function optimizeImageWithFallback(params: { buffer: optimized.data, optimizedSize: optimized.bytes, resizeSide: optimized.chosen.maxSide ?? Math.max(optimized.width, optimized.height), - format: optimized.format === "png" ? "png" : "jpeg", + format: optimized.format, + mimeType: optimized.mimeType, ...(optimized.chosen.quality === undefined ? {} : { quality: optimized.chosen.quality }), ...(optimized.chosen.compressionLevel === undefined ? {} @@ -720,7 +721,7 @@ export async function optimizeImageBufferForWebMedia(params: { } return { buffer: optimized.buffer, - contentType: optimized.format === "png" ? "image/png" : "image/jpeg", + contentType: optimized.mimeType, kind: "image", fileName: optimized.format === "jpeg" && isHeicSource(params) @@ -784,7 +785,6 @@ async function loadWebMediaInternal( throw new Error(formatCapReduce("Media", cap, optimized.buffer.length)); } - const contentType = optimized.format === "png" ? "image/png" : "image/jpeg"; const fileName = optimized.format === "jpeg" && meta && isHeicSource(meta) ? toJpegFileName(meta.fileName) @@ -792,7 +792,7 @@ async function loadWebMediaInternal( return { buffer: optimized.buffer, - contentType, + contentType: optimized.mimeType, kind: "image" as const, fileName, };