diff --git a/scripts/e2e/lib/agent-turn-output.mjs b/scripts/e2e/lib/agent-turn-output.mjs new file mode 100644 index 000000000000..834b8eb950bb --- /dev/null +++ b/scripts/e2e/lib/agent-turn-output.mjs @@ -0,0 +1,126 @@ +import fs from "node:fs"; + +function readTextFile(file) { + return fs.readFileSync(file, "utf8"); +} + +function parseJson(text) { + try { + return JSON.parse(text); + } catch { + return undefined; + } +} + +function parseJsonObjectsFromText(text) { + const payloads = []; + let start = -1; + let depth = 0; + let inString = false; + let escaped = false; + + for (let index = 0; index < text.length; index += 1) { + const char = text[index]; + if (start === -1) { + if (char === "{") { + start = index; + depth = 1; + inString = false; + escaped = false; + } + continue; + } + + if (inString) { + if (escaped) { + escaped = false; + } else if (char === "\\") { + escaped = true; + } else if (char === '"') { + inString = false; + } + continue; + } + + if (char === '"') { + inString = true; + continue; + } + if (char === "{") { + depth += 1; + continue; + } + if (char !== "}") { + continue; + } + + depth -= 1; + if (depth === 0) { + const parsed = parseJson(text.slice(start, index + 1)); + if (parsed !== undefined) { + payloads.push(parsed); + } + start = -1; + } + } + return payloads; +} + +function parseJsonPayloads(text) { + const trimmed = text.trim(); + if (!trimmed) { + return []; + } + const parsed = parseJson(trimmed); + if (parsed !== undefined) { + return [parsed]; + } + return parseJsonObjectsFromText(trimmed); +} + +function textValues(values) { + return values.filter((value) => typeof value === "string" && value.length > 0); +} + +export function extractAgentReplyTexts(text) { + return parseJsonPayloads(text).flatMap((payload) => { + const directTexts = textValues([ + payload?.finalAssistantVisibleText, + payload?.finalAssistantRawText, + payload?.meta?.finalAssistantVisibleText, + payload?.meta?.finalAssistantRawText, + payload?.result?.finalAssistantVisibleText, + payload?.result?.finalAssistantRawText, + payload?.result?.meta?.finalAssistantVisibleText, + payload?.result?.meta?.finalAssistantRawText, + ]); + const payloadEntries = Array.isArray(payload?.payloads) + ? payload.payloads + : Array.isArray(payload?.result?.payloads) + ? payload.result.payloads + : []; + const payloadTexts = payloadEntries.flatMap((entry) => + typeof entry?.text === "string" && entry.text.length > 0 ? [entry.text] : [], + ); + return directTexts.concat(payloadTexts); + }); +} + +export function assertAgentReplyContainsMarker(marker, outputPath) { + const output = readTextFile(outputPath); + const replyTexts = extractAgentReplyTexts(output); + if (replyTexts.some((text) => text.includes(marker))) { + return; + } + throw new Error( + `agent reply payload did not contain marker ${marker}. Reply payloads: ${JSON.stringify(replyTexts)}. Output: ${output}`, + ); +} + +export function assertOpenAiRequestLogUsed(requestLogPath, label = "mock OpenAI server") { + const requestLog = fs.existsSync(requestLogPath) ? readTextFile(requestLogPath) : ""; + if (/\/v1\/(responses|chat\/completions)/u.test(requestLog)) { + return; + } + throw new Error(`${label} was not used. Requests: ${requestLog}`); +} diff --git a/scripts/e2e/lib/npm-onboard-channel-agent/assertions.mjs b/scripts/e2e/lib/npm-onboard-channel-agent/assertions.mjs index 7d6943ef1f78..934aecc648e9 100644 --- a/scripts/e2e/lib/npm-onboard-channel-agent/assertions.mjs +++ b/scripts/e2e/lib/npm-onboard-channel-agent/assertions.mjs @@ -1,5 +1,9 @@ import fs from "node:fs"; import path from "node:path"; +import { + assertAgentReplyContainsMarker, + assertOpenAiRequestLogUsed, +} from "../agent-turn-output.mjs"; import { applyMockOpenAiModelConfig } from "../fixtures/mock-openai-config.mjs"; const command = process.argv[2]; @@ -83,14 +87,8 @@ function assertStatusSurfaces() { function assertAgentTurn() { const marker = process.argv[3]; const logPath = process.argv[4]; - const output = fs.readFileSync("/tmp/openclaw-agent.combined", "utf8"); - if (!output.includes(marker)) { - throw new Error(`agent JSON did not contain success marker. Output: ${output}`); - } - const requestLog = fs.existsSync(logPath) ? fs.readFileSync(logPath, "utf8") : ""; - if (!/\/v1\/(responses|chat\/completions)/u.test(requestLog)) { - throw new Error(`mock OpenAI server was not used. Requests: ${requestLog}`); - } + assertAgentReplyContainsMarker(marker, "/tmp/openclaw-agent.combined"); + assertOpenAiRequestLogUsed(logPath); } const commands = { diff --git a/scripts/e2e/lib/release-scenarios/assertions.mjs b/scripts/e2e/lib/release-scenarios/assertions.mjs index 050da3cba599..52d2a87eaaaa 100644 --- a/scripts/e2e/lib/release-scenarios/assertions.mjs +++ b/scripts/e2e/lib/release-scenarios/assertions.mjs @@ -1,5 +1,9 @@ import fs from "node:fs"; import path from "node:path"; +import { + assertAgentReplyContainsMarker, + assertOpenAiRequestLogUsed, +} from "../agent-turn-output.mjs"; import { applyMockOpenAiModelConfig } from "../fixtures/mock-openai-config.mjs"; const command = process.argv[2]; @@ -60,10 +64,8 @@ function assertAgentTurn() { const marker = process.argv[3]; const outputPath = process.argv[4]; const requestLogPath = process.argv[5]; - const output = fs.readFileSync(outputPath, "utf8"); - assert(output.includes(marker), `agent output did not contain ${marker}. Output: ${output}`); - const requestLog = fs.existsSync(requestLogPath) ? fs.readFileSync(requestLogPath, "utf8") : ""; - assert(/\/v1\/(responses|chat\/completions)/u.test(requestLog), "mock OpenAI was not used"); + assertAgentReplyContainsMarker(marker, outputPath); + assertOpenAiRequestLogUsed(requestLogPath, "mock OpenAI"); } function assertFileContains() { diff --git a/scripts/e2e/lib/release-user-journey/assertions.mjs b/scripts/e2e/lib/release-user-journey/assertions.mjs index c9f0f0e5bf83..fac605a7ce69 100644 --- a/scripts/e2e/lib/release-user-journey/assertions.mjs +++ b/scripts/e2e/lib/release-user-journey/assertions.mjs @@ -1,5 +1,9 @@ import fs from "node:fs"; import path from "node:path"; +import { + assertAgentReplyContainsMarker, + assertOpenAiRequestLogUsed, +} from "../agent-turn-output.mjs"; import { applyMockOpenAiModelConfig } from "../fixtures/mock-openai-config.mjs"; const command = process.argv[2]; @@ -50,13 +54,8 @@ function assertAgentTurn() { const marker = process.argv[3]; const outputPath = process.argv[4]; const requestLogPath = process.argv[5]; - const output = fs.readFileSync(outputPath, "utf8"); - assert(output.includes(marker), `agent output did not contain marker. Output: ${output}`); - const requestLog = fs.existsSync(requestLogPath) ? fs.readFileSync(requestLogPath, "utf8") : ""; - assert( - /\/v1\/(responses|chat\/completions)/u.test(requestLog), - "mock OpenAI server was not used", - ); + assertAgentReplyContainsMarker(marker, outputPath); + assertOpenAiRequestLogUsed(requestLogPath); } function assertFileContains() { diff --git a/test/scripts/e2e-agent-turn-output.test.ts b/test/scripts/e2e-agent-turn-output.test.ts new file mode 100644 index 000000000000..e52d41dcec9b --- /dev/null +++ b/test/scripts/e2e-agent-turn-output.test.ts @@ -0,0 +1,96 @@ +import { mkdirSync, mkdtempSync, rmSync, writeFileSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { describe, expect, it } from "vitest"; +import { + assertAgentReplyContainsMarker, + assertOpenAiRequestLogUsed, + extractAgentReplyTexts, +} from "../../scripts/e2e/lib/agent-turn-output.mjs"; + +describe("scripts/e2e/lib/agent-turn-output", () => { + it("extracts local and gateway agent reply payload text", () => { + expect( + extractAgentReplyTexts( + JSON.stringify({ + payloads: [{ text: "OPENCLAW_E2E_OK_LOCAL" }], + meta: { finalAssistantVisibleText: "visible" }, + }), + ), + ).toEqual(["visible", "OPENCLAW_E2E_OK_LOCAL"]); + + expect( + extractAgentReplyTexts( + JSON.stringify({ + result: { + payloads: [{ text: "OPENCLAW_E2E_OK_GATEWAY" }], + meta: { finalAssistantRawText: "raw" }, + }, + }), + ), + ).toEqual(["raw", "OPENCLAW_E2E_OK_GATEWAY"]); + }); + + it("reads compact JSON replies from combined stdout and stderr logs", () => { + expect( + extractAgentReplyTexts( + [ + "warning: diagnostic on stderr", + JSON.stringify({ payloads: [{ text: "OPENCLAW_E2E_OK_COMBINED" }] }), + ].join("\n"), + ), + ).toEqual(["OPENCLAW_E2E_OK_COMBINED"]); + }); + + it("reads pretty JSON replies from combined stdout and stderr logs", () => { + expect( + extractAgentReplyTexts( + [ + "warning: diagnostic on stderr", + JSON.stringify( + { + payloads: [{ text: "OPENCLAW_E2E_OK_PRETTY" }], + }, + null, + 2, + ), + ].join("\n"), + ), + ).toEqual(["OPENCLAW_E2E_OK_PRETTY"]); + }); + + it("does not accept markers that only appear outside reply payloads", () => { + const dir = mkdtempSync(join(tmpdir(), "openclaw-e2e-agent-output-")); + try { + const outputPath = join(dir, "agent.log"); + writeFileSync( + outputPath, + [ + "Return marker OPENCLAW_E2E_OK_PROMPT_ECHO", + JSON.stringify({ payloads: [{ text: "wrong reply" }] }), + ].join("\n"), + ); + + expect(() => + assertAgentReplyContainsMarker("OPENCLAW_E2E_OK_PROMPT_ECHO", outputPath), + ).toThrow(/agent reply payload did not contain marker/u); + } finally { + rmSync(dir, { recursive: true, force: true }); + } + }); + + it("checks that the mock OpenAI endpoint was actually hit", () => { + const dir = mkdtempSync(join(tmpdir(), "openclaw-e2e-request-log-")); + try { + mkdirSync(dir, { recursive: true }); + const logPath = join(dir, "requests.jsonl"); + writeFileSync(logPath, `${JSON.stringify({ path: "/v1/responses" })}\n`); + expect(() => assertOpenAiRequestLogUsed(logPath)).not.toThrow(); + + writeFileSync(logPath, `${JSON.stringify({ path: "/health" })}\n`); + expect(() => assertOpenAiRequestLogUsed(logPath)).toThrow(/was not used/u); + } finally { + rmSync(dir, { recursive: true, force: true }); + } + }); +});