fix(test): assert e2e agent reply payloads

scripts/e2e/lib/agent-turn-output.mjs

@@ -0,0 +1,126 @@
+import fs from "node:fs";
+
+function readTextFile(file) {
+  return fs.readFileSync(file, "utf8");
+}
+
+function parseJson(text) {
+  try {
+    return JSON.parse(text);
+  } catch {
+    return undefined;
+  }
+}
+
+function parseJsonObjectsFromText(text) {
+  const payloads = [];
+  let start = -1;
+  let depth = 0;
+  let inString = false;
+  let escaped = false;
+
+  for (let index = 0; index < text.length; index += 1) {
+    const char = text[index];
+    if (start === -1) {
+      if (char === "{") {
+        start = index;
+        depth = 1;
+        inString = false;
+        escaped = false;
+      }
+      continue;
+    }
+
+    if (inString) {
+      if (escaped) {
+        escaped = false;
+      } else if (char === "\\") {
+        escaped = true;
+      } else if (char === '"') {
+        inString = false;
+      }
+      continue;
+    }
+
+    if (char === '"') {
+      inString = true;
+      continue;
+    }
+    if (char === "{") {
+      depth += 1;
+      continue;
+    }
+    if (char !== "}") {
+      continue;
+    }
+
+    depth -= 1;
+    if (depth === 0) {
+      const parsed = parseJson(text.slice(start, index + 1));
+      if (parsed !== undefined) {
+        payloads.push(parsed);
+      }
+      start = -1;
+    }
+  }
+  return payloads;
+}
+
+function parseJsonPayloads(text) {
+  const trimmed = text.trim();
+  if (!trimmed) {
+    return [];
+  }
+  const parsed = parseJson(trimmed);
+  if (parsed !== undefined) {
+    return [parsed];
+  }
+  return parseJsonObjectsFromText(trimmed);
+}
+
+function textValues(values) {
+  return values.filter((value) => typeof value === "string" && value.length > 0);
+}
+
+export function extractAgentReplyTexts(text) {
+  return parseJsonPayloads(text).flatMap((payload) => {
+    const directTexts = textValues([
+      payload?.finalAssistantVisibleText,
+      payload?.finalAssistantRawText,
+      payload?.meta?.finalAssistantVisibleText,
+      payload?.meta?.finalAssistantRawText,
+      payload?.result?.finalAssistantVisibleText,
+      payload?.result?.finalAssistantRawText,
+      payload?.result?.meta?.finalAssistantVisibleText,
+      payload?.result?.meta?.finalAssistantRawText,
+    ]);
+    const payloadEntries = Array.isArray(payload?.payloads)
+      ? payload.payloads
+      : Array.isArray(payload?.result?.payloads)
+        ? payload.result.payloads
+        : [];
+    const payloadTexts = payloadEntries.flatMap((entry) =>
+      typeof entry?.text === "string" && entry.text.length > 0 ? [entry.text] : [],
+    );
+    return directTexts.concat(payloadTexts);
+  });
+}
+
+export function assertAgentReplyContainsMarker(marker, outputPath) {
+  const output = readTextFile(outputPath);
+  const replyTexts = extractAgentReplyTexts(output);
+  if (replyTexts.some((text) => text.includes(marker))) {
+    return;
+  }
+  throw new Error(
+    `agent reply payload did not contain marker ${marker}. Reply payloads: ${JSON.stringify(replyTexts)}. Output: ${output}`,
+  );
+}
+
+export function assertOpenAiRequestLogUsed(requestLogPath, label = "mock OpenAI server") {
+  const requestLog = fs.existsSync(requestLogPath) ? readTextFile(requestLogPath) : "";
+  if (/\/v1\/(responses|chat\/completions)/u.test(requestLog)) {
+    return;
+  }
+  throw new Error(`${label} was not used. Requests: ${requestLog}`);
+}

scripts/e2e/lib/npm-onboard-channel-agent/assertions.mjs

@@ -1,5 +1,9 @@
 import fs from "node:fs";
 import path from "node:path";
+import {
+  assertAgentReplyContainsMarker,
+  assertOpenAiRequestLogUsed,
+} from "../agent-turn-output.mjs";
 import { applyMockOpenAiModelConfig } from "../fixtures/mock-openai-config.mjs";
 
 const command = process.argv[2];
@@ -83,14 +87,8 @@ function assertStatusSurfaces() {
 function assertAgentTurn() {
   const marker = process.argv[3];
   const logPath = process.argv[4];
-  const output = fs.readFileSync("/tmp/openclaw-agent.combined", "utf8");
-  if (!output.includes(marker)) {
-    throw new Error(`agent JSON did not contain success marker. Output: ${output}`);
-  }
-  const requestLog = fs.existsSync(logPath) ? fs.readFileSync(logPath, "utf8") : "";
-  if (!/\/v1\/(responses|chat\/completions)/u.test(requestLog)) {
-    throw new Error(`mock OpenAI server was not used. Requests: ${requestLog}`);
-  }
+  assertAgentReplyContainsMarker(marker, "/tmp/openclaw-agent.combined");
+  assertOpenAiRequestLogUsed(logPath);
 }
 
 const commands = {

scripts/e2e/lib/release-scenarios/assertions.mjs

@@ -1,5 +1,9 @@
 import fs from "node:fs";
 import path from "node:path";
+import {
+  assertAgentReplyContainsMarker,
+  assertOpenAiRequestLogUsed,
+} from "../agent-turn-output.mjs";
 import { applyMockOpenAiModelConfig } from "../fixtures/mock-openai-config.mjs";
 
 const command = process.argv[2];
@@ -60,10 +64,8 @@ function assertAgentTurn() {
   const marker = process.argv[3];
   const outputPath = process.argv[4];
   const requestLogPath = process.argv[5];
-  const output = fs.readFileSync(outputPath, "utf8");
-  assert(output.includes(marker), `agent output did not contain ${marker}. Output: ${output}`);
-  const requestLog = fs.existsSync(requestLogPath) ? fs.readFileSync(requestLogPath, "utf8") : "";
-  assert(/\/v1\/(responses|chat\/completions)/u.test(requestLog), "mock OpenAI was not used");
+  assertAgentReplyContainsMarker(marker, outputPath);
+  assertOpenAiRequestLogUsed(requestLogPath, "mock OpenAI");
 }
 
 function assertFileContains() {

scripts/e2e/lib/release-user-journey/assertions.mjs

@@ -1,5 +1,9 @@
 import fs from "node:fs";
 import path from "node:path";
+import {
+  assertAgentReplyContainsMarker,
+  assertOpenAiRequestLogUsed,
+} from "../agent-turn-output.mjs";
 import { applyMockOpenAiModelConfig } from "../fixtures/mock-openai-config.mjs";
 
 const command = process.argv[2];
@@ -50,13 +54,8 @@ function assertAgentTurn() {
   const marker = process.argv[3];
   const outputPath = process.argv[4];
   const requestLogPath = process.argv[5];
-  const output = fs.readFileSync(outputPath, "utf8");
-  assert(output.includes(marker), `agent output did not contain marker. Output: ${output}`);
-  const requestLog = fs.existsSync(requestLogPath) ? fs.readFileSync(requestLogPath, "utf8") : "";
-  assert(
-    /\/v1\/(responses|chat\/completions)/u.test(requestLog),
-    "mock OpenAI server was not used",
-  );
+  assertAgentReplyContainsMarker(marker, outputPath);
+  assertOpenAiRequestLogUsed(requestLogPath);
 }
 
 function assertFileContains() {

test/scripts/e2e-agent-turn-output.test.ts

@@ -0,0 +1,96 @@
+import { mkdirSync, mkdtempSync, rmSync, writeFileSync } from "node:fs";
+import { tmpdir } from "node:os";
+import { join } from "node:path";
+import { describe, expect, it } from "vitest";
+import {
+  assertAgentReplyContainsMarker,
+  assertOpenAiRequestLogUsed,
+  extractAgentReplyTexts,
+} from "../../scripts/e2e/lib/agent-turn-output.mjs";
+
+describe("scripts/e2e/lib/agent-turn-output", () => {
+  it("extracts local and gateway agent reply payload text", () => {
+    expect(
+      extractAgentReplyTexts(
+        JSON.stringify({
+          payloads: [{ text: "OPENCLAW_E2E_OK_LOCAL" }],
+          meta: { finalAssistantVisibleText: "visible" },
+        }),
+      ),
+    ).toEqual(["visible", "OPENCLAW_E2E_OK_LOCAL"]);
+
+    expect(
+      extractAgentReplyTexts(
+        JSON.stringify({
+          result: {
+            payloads: [{ text: "OPENCLAW_E2E_OK_GATEWAY" }],
+            meta: { finalAssistantRawText: "raw" },
+          },
+        }),
+      ),
+    ).toEqual(["raw", "OPENCLAW_E2E_OK_GATEWAY"]);
+  });
+
+  it("reads compact JSON replies from combined stdout and stderr logs", () => {
+    expect(
+      extractAgentReplyTexts(
+        [
+          "warning: diagnostic on stderr",
+          JSON.stringify({ payloads: [{ text: "OPENCLAW_E2E_OK_COMBINED" }] }),
+        ].join("\n"),
+      ),
+    ).toEqual(["OPENCLAW_E2E_OK_COMBINED"]);
+  });
+
+  it("reads pretty JSON replies from combined stdout and stderr logs", () => {
+    expect(
+      extractAgentReplyTexts(
+        [
+          "warning: diagnostic on stderr",
+          JSON.stringify(
+            {
+              payloads: [{ text: "OPENCLAW_E2E_OK_PRETTY" }],
+            },
+            null,
+            2,
+          ),
+        ].join("\n"),
+      ),
+    ).toEqual(["OPENCLAW_E2E_OK_PRETTY"]);
+  });
+
+  it("does not accept markers that only appear outside reply payloads", () => {
+    const dir = mkdtempSync(join(tmpdir(), "openclaw-e2e-agent-output-"));
+    try {
+      const outputPath = join(dir, "agent.log");
+      writeFileSync(
+        outputPath,
+        [
+          "Return marker OPENCLAW_E2E_OK_PROMPT_ECHO",
+          JSON.stringify({ payloads: [{ text: "wrong reply" }] }),
+        ].join("\n"),
+      );
+
+      expect(() =>
+        assertAgentReplyContainsMarker("OPENCLAW_E2E_OK_PROMPT_ECHO", outputPath),
+      ).toThrow(/agent reply payload did not contain marker/u);
+    } finally {
+      rmSync(dir, { recursive: true, force: true });
+    }
+  });
+
+  it("checks that the mock OpenAI endpoint was actually hit", () => {
+    const dir = mkdtempSync(join(tmpdir(), "openclaw-e2e-request-log-"));
+    try {
+      mkdirSync(dir, { recursive: true });
+      const logPath = join(dir, "requests.jsonl");
+      writeFileSync(logPath, `${JSON.stringify({ path: "/v1/responses" })}\n`);
+      expect(() => assertOpenAiRequestLogUsed(logPath)).not.toThrow();
+
+      writeFileSync(logPath, `${JSON.stringify({ path: "/health" })}\n`);
+      expect(() => assertOpenAiRequestLogUsed(logPath)).toThrow(/was not used/u);
+    } finally {
+      rmSync(dir, { recursive: true, force: true });
+    }
+  });
+});