refactor: delegate image limits to Rastermill

npm-shrinkwrap.json

@@ -47,7 +47,7 @@
         "playwright-core": "1.60.0",
         "qrcode": "1.5.4",
         "quickjs-wasi": "2.2.0",
-        "rastermill": "https://codeload.github.com/openclaw/rastermill/tar.gz/48457f54bbbe8061cdf358069eee202e27b553be",
+        "rastermill": "https://codeload.github.com/openclaw/rastermill/tar.gz/89405b23b10e32bcbedbd0e2aa56ba790e94d148",
         "tar": "7.5.15",
         "tokenjuice": "0.7.1",
         "tree-sitter-bash": "0.25.1",
@@ -3843,8 +3843,8 @@
     },
     "node_modules/rastermill": {
       "version": "0.2.0",
-      "resolved": "https://codeload.github.com/openclaw/rastermill/tar.gz/48457f54bbbe8061cdf358069eee202e27b553be",
-      "integrity": "sha512-wGbsJuFVkrwgWcaLESPi9OXFIV9MoJQtO8OwKUitKz6Q+XAsSvBhg08lE7JlBmutYFZofzMv6/GyxMl1ITwMqg==",
+      "resolved": "https://codeload.github.com/openclaw/rastermill/tar.gz/89405b23b10e32bcbedbd0e2aa56ba790e94d148",
+      "integrity": "sha512-Rzs9xELc0+t90r0kMon/AqBwkxT4R7En4OT04C4wBpDpQO4M+QLxHYaBBJWSTrEUKn+eunArcnwx+O2GGW/mGQ==",
       "license": "MIT",
       "dependencies": {
         "@silvia-odwyer/photon-node": "0.3.4"

package.json

@@ -1838,7 +1838,7 @@
     "playwright-core": "1.60.0",
     "qrcode": "1.5.4",
     "quickjs-wasi": "2.2.0",
-    "rastermill": "https://codeload.github.com/openclaw/rastermill/tar.gz/48457f54bbbe8061cdf358069eee202e27b553be",
+    "rastermill": "https://codeload.github.com/openclaw/rastermill/tar.gz/89405b23b10e32bcbedbd0e2aa56ba790e94d148",
     "tar": "7.5.15",
     "tokenjuice": "0.7.1",
     "tree-sitter-bash": "0.25.1",

pnpm-lock.yaml

@@ -152,8 +152,8 @@ importers:
         specifier: 2.2.0
         version: 2.2.0
       rastermill:
-        specifier: https://codeload.github.com/openclaw/rastermill/tar.gz/48457f54bbbe8061cdf358069eee202e27b553be
-        version: https://codeload.github.com/openclaw/rastermill/tar.gz/48457f54bbbe8061cdf358069eee202e27b553be
+        specifier: https://codeload.github.com/openclaw/rastermill/tar.gz/89405b23b10e32bcbedbd0e2aa56ba790e94d148
+        version: https://codeload.github.com/openclaw/rastermill/tar.gz/89405b23b10e32bcbedbd0e2aa56ba790e94d148
       tar:
         specifier: 7.5.15
         version: 7.5.15
@@ -6310,8 +6310,8 @@ packages:
     resolution: {integrity: sha512-Hrgsx+orqoygnmhFbKaHE6c296J+HTAQXoxEF6gNupROmmGJRoyzfG3ccAveqCBrwr/2yxQ5BVd/GTl5agOwSg==}
     engines: {node: '>= 0.6'}
 
-  rastermill@https://codeload.github.com/openclaw/rastermill/tar.gz/48457f54bbbe8061cdf358069eee202e27b553be:
-    resolution: {gitHosted: true, integrity: sha512-wGbsJuFVkrwgWcaLESPi9OXFIV9MoJQtO8OwKUitKz6Q+XAsSvBhg08lE7JlBmutYFZofzMv6/GyxMl1ITwMqg==, tarball: https://codeload.github.com/openclaw/rastermill/tar.gz/48457f54bbbe8061cdf358069eee202e27b553be}
+  rastermill@https://codeload.github.com/openclaw/rastermill/tar.gz/89405b23b10e32bcbedbd0e2aa56ba790e94d148:
+    resolution: {gitHosted: true, integrity: sha512-Rzs9xELc0+t90r0kMon/AqBwkxT4R7En4OT04C4wBpDpQO4M+QLxHYaBBJWSTrEUKn+eunArcnwx+O2GGW/mGQ==, tarball: https://codeload.github.com/openclaw/rastermill/tar.gz/89405b23b10e32bcbedbd0e2aa56ba790e94d148}
     version: 0.2.0
     engines: {node: '>=20'}
 
@@ -12304,7 +12304,7 @@ snapshots:
 
   range-parser@1.2.1: {}
 
-  rastermill@https://codeload.github.com/openclaw/rastermill/tar.gz/48457f54bbbe8061cdf358069eee202e27b553be:
+  rastermill@https://codeload.github.com/openclaw/rastermill/tar.gz/89405b23b10e32bcbedbd0e2aa56ba790e94d148:
     dependencies:
       '@silvia-odwyer/photon-node': 0.3.4
 

src/gateway/managed-image-attachments.ts

@@ -10,7 +10,7 @@ import { assertLocalMediaAllowed } from "../media/local-media-access.js";
 import {
   createImageProcessor,
   getImageMetadata,
-  hasAlphaChannel,
+  readImageProbeFromHeader,
 } from "../media/media-services.js";
 import { isPassThroughRemoteMediaSource } from "../media/media-source-url.js";
 import { MEDIA_MAX_BYTES, saveMediaBuffer, saveMediaSource } from "../media/store.js";
@@ -189,70 +189,37 @@ function getManagedImageMetadataLimitError(
   return null;
 }
 
-function computeManagedImageResizeTarget(
-  metadata: { width: number; height: number },
-  limits: ManagedImageAttachmentLimits,
+function orientManagedImageMetadata(
+  buffer: Buffer,
+  metadata: { width: number; height: number } | null,
 ): { width: number; height: number } | null {
-  const scale = Math.min(
-    1,
-    limits.maxWidth / metadata.width,
-    limits.maxHeight / metadata.height,
-    Math.sqrt(limits.maxPixels / (metadata.width * metadata.height)),
-  );
-  if (!Number.isFinite(scale) || scale >= 1) {
+  if (!metadata) {
     return null;
   }
-
-  let width = Math.max(1, Math.floor(metadata.width * scale));
-  let height = Math.max(1, Math.floor(metadata.height * scale));
-  while (
-    width > limits.maxWidth ||
-    height > limits.maxHeight ||
-    width * height > limits.maxPixels
-  ) {
-    if (width >= height && width > 1) {
-      width -= 1;
-    } else if (height > 1) {
-      height -= 1;
-    } else {
-      break;
-    }
-  }
-  return { width, height };
+  const orientation = readImageProbeFromHeader(buffer)?.orientation;
+  return orientation === 5 || orientation === 6 || orientation === 7 || orientation === 8
+    ? { width: metadata.height, height: metadata.width }
+    : metadata;
 }
 
 async function resizeManagedImageBufferToLimits(params: {
   buffer: Buffer;
-  contentType: string;
-  metadata: { width: number; height: number };
   limits: ManagedImageAttachmentLimits;
 }): Promise<{ buffer: Buffer; contentType: string; width: number; height: number }> {
-  const target = computeManagedImageResizeTarget(params.metadata, params.limits);
-  if (!target) {
-    return {
-      buffer: params.buffer,
-      contentType: params.contentType,
-      width: params.metadata.width,
-      height: params.metadata.height,
-    };
-  }
-
-  const preserveAlpha = await hasAlphaChannel(params.buffer);
-  const resized = await createImageProcessor().encodeBest(params.buffer, {
-    resize: {
-      width: target.width,
-      height: target.height,
-      fit: "inside",
-      enlarge: false,
+  const resized = await createImageProcessor().encodeToLimits(params.buffer, {
+    limits: {
+      maxWidth: params.limits.maxWidth,
+      maxHeight: params.limits.maxHeight,
+      maxPixels: params.limits.maxPixels,
     },
     opaque: { format: "jpeg", quality: 92 },
     transparent: { format: "png", compressionLevel: 9 },
-    transparency: preserveAlpha ? "preserve" : "flatten",
+    transparency: "auto",
   });
 
   return {
     buffer: resized.data,
-    contentType: resized.format === "png" ? "image/png" : "image/jpeg",
+    contentType: resized.mimeType,
     width: resized.width,
     height: resized.height,
   };
@@ -851,7 +818,8 @@ export async function createManagedOutgoingImageBlocks(params: {
         originalStats.width != null && originalStats.height != null
           ? { width: originalStats.width, height: originalStats.height }
           : await getImageMetadata(originalBuffer);
-      let effectiveMetadata = originalMetadata;
+      const originalDisplayMetadata = orientManagedImageMetadata(originalBuffer, originalMetadata);
+      let effectiveMetadata = originalDisplayMetadata;
       let metadataLimitError = getManagedImageMetadataLimitError(effectiveMetadata, alt, limits);
       for (let resizeAttempt = 0; metadataLimitError; resizeAttempt += 1) {
         if (!effectiveMetadata) {
@@ -862,8 +830,6 @@ export async function createManagedOutgoingImageBlocks(params: {
         }
         const resized = await resizeManagedImageBufferToLimits({
           buffer: originalBuffer,
-          contentType: savedOriginalContentType,
-          metadata: effectiveMetadata,
           limits,
         });
         validateManagedImageBuffer(resized.buffer, alt, limits);
@@ -880,16 +846,20 @@ export async function createManagedOutgoingImageBlocks(params: {
         savedOriginalPath = savedOriginal.path;
         originalBuffer = resized.buffer;
         originalStats = await getVariantStats(savedOriginal.path);
-        effectiveMetadata =
+        effectiveMetadata = orientManagedImageMetadata(
+          originalBuffer,
           originalStats.width != null && originalStats.height != null
             ? { width: originalStats.width, height: originalStats.height }
-            : await getImageMetadata(originalBuffer);
+            : await getImageMetadata(originalBuffer),
+        );
         metadataLimitError = getManagedImageMetadataLimitError(effectiveMetadata, alt, limits);
         if (!metadataLimitError) {
           resizeWarning = buildManagedImageResizeWarningBlock({
             alt,
-            originalWidth: originalMetadata?.width ?? effectiveMetadata?.width ?? resized.width,
-            originalHeight: originalMetadata?.height ?? effectiveMetadata?.height ?? resized.height,
+            originalWidth:
+              originalDisplayMetadata?.width ?? effectiveMetadata?.width ?? resized.width,
+            originalHeight:
+              originalDisplayMetadata?.height ?? effectiveMetadata?.height ?? resized.height,
             resizedWidth: effectiveMetadata?.width ?? resized.width,
             resizedHeight: effectiveMetadata?.height ?? resized.height,
           });

src/media/image-ops.ts

@@ -80,14 +80,6 @@ export function readImageProbeFromHeader(buffer: Buffer): ImageProbe | null {
   return readRastermillImageProbeFromHeader(buffer);
 }
 
-export function shouldAttemptTransparencyPreservingEncode(buffer: Buffer): boolean {
-  const probe = readRastermillImageProbeFromHeader(buffer);
-  if (probe?.hasAlpha !== true) {
-    return false;
-  }
-  return probe.format === "png" || probe.format === "gif" || probe.format === "webp";
-}
-
 function wrapRastermillUnavailable(operation: string, error: unknown): never {
   if (error instanceof RastermillUnavailableError) {
     throw new ImageProcessorUnavailableError(operation, error.message, error.causes);

src/media/web-media.ts

@@ -21,7 +21,6 @@ import {
   createImageProcessor,
   readImageMetadataFromHeader,
   readImageProbeFromHeader,
-  shouldAttemptTransparencyPreservingEncode,
 } from "./media-services.js";
 import {
   detectMime,
@@ -357,7 +356,8 @@ type OptimizedImage = {
   buffer: Buffer;
   optimizedSize: number;
   resizeSide: number;
-  format: "jpeg" | "png";
+  format: "jpeg" | "png" | "webp";
+  mimeType: string;
   quality?: number;
   compressionLevel?: number;
 };
@@ -654,7 +654,7 @@ async function optimizeImageWithFallback(params: {
       maxSide: grid.sides,
       quality: grid.qualities,
     },
-    transparency: shouldAttemptTransparencyPreservingEncode(buffer) ? "prefer" : "flatten",
+    transparency: "auto",
   });
   if (optimized.chosen.transparency === "flattened" && shouldLogVerbose()) {
     logVerbose(`Image transparency flattened to fit ${formatMb(cap, 0)}MB optimization budget`);
@@ -663,7 +663,8 @@ async function optimizeImageWithFallback(params: {
     buffer: optimized.data,
     optimizedSize: optimized.bytes,
     resizeSide: optimized.chosen.maxSide ?? Math.max(optimized.width, optimized.height),
-    format: optimized.format === "png" ? "png" : "jpeg",
+    format: optimized.format,
+    mimeType: optimized.mimeType,
     ...(optimized.chosen.quality === undefined ? {} : { quality: optimized.chosen.quality }),
     ...(optimized.chosen.compressionLevel === undefined
       ? {}
@@ -720,7 +721,7 @@ export async function optimizeImageBufferForWebMedia(params: {
   }
   return {
     buffer: optimized.buffer,
-    contentType: optimized.format === "png" ? "image/png" : "image/jpeg",
+    contentType: optimized.mimeType,
     kind: "image",
     fileName:
       optimized.format === "jpeg" && isHeicSource(params)
@@ -784,7 +785,6 @@ async function loadWebMediaInternal(
       throw new Error(formatCapReduce("Media", cap, optimized.buffer.length));
     }
 
-    const contentType = optimized.format === "png" ? "image/png" : "image/jpeg";
     const fileName =
       optimized.format === "jpeg" && meta && isHeicSource(meta)
         ? toJpegFileName(meta.fileName)
@@ -792,7 +792,7 @@ async function loadWebMediaInternal(
 
     return {
       buffer: optimized.buffer,
-      contentType,
+      contentType: optimized.mimeType,
       kind: "image" as const,
       fileName,
     };