liuyu/openclaw
1
0
Fork 0
mirror of https://github.com/openclaw/openclaw.git synced 2026-06-06 05:51:15 +08:00
Code Issues Packages Projects Releases Wiki Activity

feat(qa-lab): add deterministic whatsapp mock replies

Browse Source
This commit is contained in:
Marcus Castro
2026-06-05 02:10:03 -03:00
parent cf570679f1
commit 969efb486d
2 changed files with 214 additions and 5 deletions
Download Patch File Download Diff File Expand all files Collapse all files

118
extensions/qa-lab/src/providers/mock-openai/server.test.ts
View File

@@ -293,6 +293,26 @@ describe("qa mock openai server", () => {
expect(telegramLongBody).toContain("TELEGRAM-LONG-FINAL-END");
expect(telegramLongBody.length).toBeGreaterThan(4_500);
const whatsappLongResponse = await fetch(`${server.baseUrl}/v1/responses`, {
method: "POST",
headers: {
"content-type": "application/json",
},
body: JSON.stringify({
stream: true,
input: [
makeUserInput("WhatsApp long final QA check. Use the scripted long final response."),
],
}),
});
expect(whatsappLongResponse.status).toBe(200);
const whatsappLongBody = await whatsappLongResponse.text();
expect(whatsappLongBody).toContain('"type":"response.output_text.delta"');
expect(whatsappLongBody).toContain('"phase":"final_answer"');
expect(whatsappLongBody).toContain("WHATSAPP-LONG-FINAL-BEGIN");
expect(whatsappLongBody).toContain("WHATSAPP-LONG-FINAL-END");
expect(whatsappLongBody.length).toBeGreaterThan(6_000);
const telegramThreeChunkLongResponse = await fetch(`${server.baseUrl}/v1/responses`, {
method: "POST",
headers: {
@@ -2856,6 +2876,80 @@ describe("qa mock openai server", () => {
expect(outputText(await response.json())).toBe("QA_CANARY_TEST");
});
it("uses WhatsApp location markers only for the matching coordinate body", async () => {
const server = await startMockServer();
const setupInput = makeUserInput(
"When a later WhatsApp location message shows 37.774900, -122.419400, " +
"reply with only this WhatsApp location marker: QA_WHATSAPP_LOCATION_OK. " +
"Reply with only this exact marker: QA_INITIAL_OK",
);
const setupResponse = await postResponses(server, {
stream: false,
input: [setupInput],
});
const response = await postResponses(server, {
stream: false,
input: [setupInput, makeUserInput("📍 37.774900, -122.419400")],
});
expect(setupResponse.status).toBe(200);
expect(outputText(await setupResponse.json())).toBe("QA_INITIAL_OK");
expect(response.status).toBe(200);
expect(outputText(await response.json())).toBe("QA_WHATSAPP_LOCATION_OK");
});
it("uses WhatsApp contact and sticker markers only for matching structured bodies", async () => {
const server = await startMockServer();
const setupInput = makeUserInput(
"When a later WhatsApp contact message appears, " +
"reply with only this WhatsApp contact marker: QA_WHATSAPP_CONTACT_OK. " +
"When a later WhatsApp sticker message appears, " +
"reply with only this WhatsApp sticker marker: QA_WHATSAPP_STICKER_OK. " +
"Reply with only this exact marker: QA_STRUCTURED_INITIAL_OK",
);
const setupResponse = await postResponses(server, {
stream: false,
input: [setupInput],
});
const contactResponse = await postResponses(server, {
stream: false,
input: [setupInput, makeUserInput("<contact>")],
});
const stickerResponse = await postResponses(server, {
stream: false,
input: [setupInput, makeUserInput("<media:sticker>")],
});
expect(setupResponse.status).toBe(200);
expect(outputText(await setupResponse.json())).toBe("QA_STRUCTURED_INITIAL_OK");
expect(contactResponse.status).toBe(200);
expect(outputText(await contactResponse.json())).toBe("QA_WHATSAPP_CONTACT_OK");
expect(stickerResponse.status).toBe(200);
expect(outputText(await stickerResponse.json())).toBe("QA_WHATSAPP_STICKER_OK");
});
it("streams WhatsApp location markers for the matching coordinate body", async () => {
const server = await startMockServer();
const body = await expectResponsesText(server, {
stream: true,
input: [
makeUserInput(
"When a later WhatsApp location message shows 37.774900, -122.419400, " +
"reply with only this WhatsApp location marker: QA_WHATSAPP_LOCATION_STREAM_OK. " +
"Reply with only this exact marker: QA_INITIAL_STREAM_OK",
),
makeUserInput("📍 37.774900, -122.419400"),
],
});
expect(body).toContain("QA_WHATSAPP_LOCATION_STREAM_OK");
expect(body).not.toContain("QA_INITIAL_STREAM_OK");
});
it("uses image generation directives from request context when the latest user text is generic", async () => {
const server = await startQaMockOpenAiServer({
host: "127.0.0.1",
@@ -3482,6 +3576,30 @@ describe("qa mock openai server", () => {
const ids = body.data.map((entry) => entry.id);
expect(ids).toContain("claude-opus-4-8");
expect(ids).toContain("gpt-5.5");
expect(ids).toContain("gpt-4o-transcribe");
});
it("serves deterministic OpenAI-compatible audio transcription responses", async () => {
const server = await startQaMockOpenAiServer({
host: "127.0.0.1",
port: 0,
});
cleanups.push(async () => {
await server.stop();
});
const response = await fetch(`${server.baseUrl}/v1/audio/transcriptions`, {
method: "POST",
headers: {
"content-type": "multipart/form-data; boundary=qa",
},
body: "--qa\r\n--qa--\r\n",
});
expect(response.status).toBe(200);
await expect(response.json()).resolves.toEqual({
text: "Reply with only this exact marker: WHATSAPP_QA_AUDIO_TRANSCRIPT_OK",
});
});
it("dispatches an Anthropic /v1/messages read tool call for source discovery prompts", async () => {

101
extensions/qa-lab/src/providers/mock-openai/server.ts
View File

@@ -160,6 +160,7 @@ const QA_TELEGRAM_CURRENT_SESSION_STATUS_PROMPT_RE = /telegram current session_s
const QA_TELEGRAM_STREAM_SINGLE_MARKER = "QA-TELEGRAM-STREAM-SINGLE-OK";
const QA_TELEGRAM_LONG_FINAL_THREE_CHUNK_PROMPT_RE = /telegram long final three chunk qa check/i;
const QA_TELEGRAM_LONG_FINAL_PROMPT_RE = /telegram long final qa check/i;
const QA_WHATSAPP_LONG_FINAL_PROMPT_RE = /whatsapp long final qa check/i;
const QA_SUBAGENT_DIRECT_FALLBACK_PROMPT_RE = /subagent direct fallback qa check/i;
const QA_SUBAGENT_DIRECT_FALLBACK_WORKER_RE = /subagent direct fallback worker/i;
const QA_SUBAGENT_DIRECT_FALLBACK_MARKER = "QA-SUBAGENT-DIRECT-FALLBACK-OK";
@@ -176,6 +177,8 @@ const QA_RELEASE_AUDIT_PROMPT_RE = /release readiness audit for the small projec
const QA_TOOL_SEARCH_PROMPT_RE = /tool search qa check/i;
const QA_TOOL_SEARCH_FAILURE_PROMPT_RE = /tool search qa failure/i;
const QA_MCP_CODE_MODE_PROMPT_RE = /mcp code mode qa check/i;
const QA_AUDIO_TRANSCRIPTION_TEXT =
"Reply with only this exact marker: WHATSAPP_QA_AUDIO_TRANSCRIPT_OK";
const QA_MCP_CODE_MODE_API_FILE_PROMPT_RE = /mcp code mode api file qa check/i;
type MockScenarioState = {
@@ -870,6 +873,33 @@ function extractExactMarkerDirective(text: string) {
);
}
function extractWhatsAppLocationMarkerDirective(text: string) {
return extractLastCapture(
text,
/WhatsApp location marker:\s*([^\s`.,;:!?]+(?:-[^\s`.,;:!?]+)*)/i,
);
}
function extractWhatsAppContactMarkerDirective(text: string) {
return extractLastCapture(text, /WhatsApp contact marker:\s*([^\s`.,;:!?]+(?:-[^\s`.,;:!?]+)*)/i);
}
function extractWhatsAppStickerMarkerDirective(text: string) {
return extractLastCapture(text, /WhatsApp sticker marker:\s*([^\s`.,;:!?]+(?:-[^\s`.,;:!?]+)*)/i);
}
function shouldUseWhatsAppLocationMarker(prompt: string) {
return /^📍\s*37\.774900,\s*-122\.419400\b/u.test(prompt.trim());
}
function shouldUseWhatsAppContactMarker(prompt: string) {
return /^<contacts?(?::|>)/iu.test(prompt.trim());
}
function shouldUseWhatsAppStickerMarker(prompt: string) {
return /^<media:sticker>(?:\s|$)/iu.test(prompt.trim());
}
function extractLabeledMarkerDirective(text: string, label: string) {
const escapedLabel = escapeRegExp(label);
const backtickedMatch = extractLastCapture(
@@ -1116,6 +1146,15 @@ function buildAssistantText(
const exactReplyDirective = promptExactReplyDirective ?? extractExactReplyDirective(allInputText);
const exactMarkerDirective =
extractExactMarkerDirective(prompt) ?? extractExactMarkerDirective(allInputText);
const whatsAppLocationMarker = shouldUseWhatsAppLocationMarker(prompt)
? extractWhatsAppLocationMarkerDirective(allInputText)
: "";
const whatsAppContactMarker = shouldUseWhatsAppContactMarker(prompt)
? extractWhatsAppContactMarkerDirective(allInputText)
: "";
const whatsAppStickerMarker = shouldUseWhatsAppStickerMarker(prompt)
? extractWhatsAppStickerMarkerDirective(allInputText)
: "";
const finishExactlyDirective =
extractFinishExactlyDirective(prompt) ?? extractFinishExactlyDirective(allInputText);
const latestImageUserTurn = extractLatestImageUserTurn(input);
@@ -1157,6 +1196,15 @@ function buildAssistantText(
) {
return "Protocol note: the attached image is split horizontally, with red on top and blue on the bottom.";
}
if (whatsAppLocationMarker) {
return whatsAppLocationMarker;
}
if (whatsAppContactMarker) {
return whatsAppContactMarker;
}
if (whatsAppStickerMarker) {
return whatsAppStickerMarker;
}
if (/\bmarker\b/i.test(allInputText) && exactReplyDirective) {
return exactReplyDirective;
}
@@ -1468,19 +1516,20 @@ function splitMockStreamingText(text: string, parts = 3) {
return chunks.length > 1 ? chunks : [text.slice(0, 1), text.slice(1)];
}
function buildTelegramLongFinalText({
function buildQaLongFinalText({
endMarker = "TELEGRAM-LONG-FINAL-END",
segmentPrefix = "telegram-long-final-segment",
segmentCount = 42,
startMarker = "TELEGRAM-LONG-FINAL-BEGIN",
}: {
endMarker?: string;
segmentPrefix?: string;
segmentCount?: number;
startMarker?: string;
} = {}) {
const body = Array.from(
{ length: segmentCount },
(_, index) =>
`telegram-long-final-segment-${String(index + 1).padStart(3, "0")} ${"x".repeat(54)}`,
(_, index) => `${segmentPrefix}-${String(index + 1).padStart(3, "0")} ${"x".repeat(54)}`,
).join("\n");
return `${startMarker}\n${body}\n${endMarker}`;
}
@@ -1752,6 +1801,15 @@ async function buildResponsesPayload(
extractExactReplyDirective(prompt) ?? extractExactReplyDirective(allInputText);
const exactMarkerDirective =
extractExactMarkerDirective(prompt) ?? extractExactMarkerDirective(allInputText);
const whatsAppLocationMarker = shouldUseWhatsAppLocationMarker(prompt)
? extractWhatsAppLocationMarkerDirective(allInputText)
: "";
const whatsAppContactMarker = shouldUseWhatsAppContactMarker(prompt)
? extractWhatsAppContactMarkerDirective(allInputText)
: "";
const whatsAppStickerMarker = shouldUseWhatsAppStickerMarker(prompt)
? extractWhatsAppStickerMarkerDirective(allInputText)
: "";
const blockStreamingPrompt =
extractLastMatchingUserText(extractAllUserTexts(input), QA_BLOCK_STREAMING_PROMPT_RE) ||
prompt ||
@@ -1975,7 +2033,7 @@ async function buildResponsesPayload(
return buildAssistantEvents("");
}
if (QA_TELEGRAM_LONG_FINAL_THREE_CHUNK_PROMPT_RE.test(allInputText)) {
const text = buildTelegramLongFinalText({
const text = buildQaLongFinalText({
endMarker: "TELEGRAM-LONG-FINAL-3CHUNK-END",
segmentCount: 96,
startMarker: "TELEGRAM-LONG-FINAL-3CHUNK-BEGIN",
@@ -1990,7 +2048,7 @@ async function buildResponsesPayload(
]);
}
if (QA_TELEGRAM_LONG_FINAL_PROMPT_RE.test(allInputText)) {
const text = buildTelegramLongFinalText();
const text = buildQaLongFinalText();
return buildAssistantEvents([
{
id: "msg_mock_telegram_long_final",
@@ -2000,6 +2058,22 @@ async function buildResponsesPayload(
},
]);
}
if (QA_WHATSAPP_LONG_FINAL_PROMPT_RE.test(allInputText)) {
const text = buildQaLongFinalText({
endMarker: "WHATSAPP-LONG-FINAL-END",
segmentPrefix: "whatsapp-long-final-segment",
segmentCount: 64,
startMarker: "WHATSAPP-LONG-FINAL-BEGIN",
});
return buildAssistantEvents([
{
id: "msg_mock_whatsapp_long_final",
phase: "final_answer",
streamDeltas: splitMockStreamingText(text),
text,
},
]);
}
if (
QA_STREAMING_PROMPT_RE.test(allInputText) &&
allInputText.includes(QA_TELEGRAM_STREAM_SINGLE_MARKER)
@@ -2082,6 +2156,15 @@ async function buildResponsesPayload(
exactMarkerDirective ?? exactReplyDirective ?? "QA-GROUP-FALLBACK-OK",
);
}
if (whatsAppLocationMarker) {
return buildAssistantEvents(whatsAppLocationMarker);
}
if (whatsAppContactMarker) {
return buildAssistantEvents(whatsAppContactMarker);
}
if (whatsAppStickerMarker) {
return buildAssistantEvents(whatsAppStickerMarker);
}
if (/\bmarker\b/i.test(prompt) && exactReplyDirective) {
return buildAssistantEvents(exactReplyDirective);
}
@@ -3093,6 +3176,7 @@ export async function startQaMockOpenAiServer(params?: { host?: string; port?: n
{ id: "gpt-5.5", object: "model" },
{ id: "gpt-5.5-alt", object: "model" },
{ id: "gpt-image-1", object: "model" },
{ id: "gpt-4o-transcribe", object: "model" },
{ id: "text-embedding-3-small", object: "model" },
{ id: "claude-opus-4-8", object: "model" },
{ id: "claude-sonnet-4-6", object: "model" },
@@ -3129,6 +3213,13 @@ export async function startQaMockOpenAiServer(params?: { host?: string; port?: n
});
return;
}
if (req.method === "POST" && url.pathname === "/v1/audio/transcriptions") {
await readBody(req);
writeJson(res, 200, {
text: QA_AUDIO_TRANSCRIPTION_TEXT,
});
return;
}
if (req.method === "POST" && url.pathname === "/v1/embeddings") {
const raw = await readBody(req);
const body = raw ? (JSON.parse(raw) as Record<string, unknown>) : {};