diff --git a/extensions/qa-lab/src/providers/mock-openai/server.test.ts b/extensions/qa-lab/src/providers/mock-openai/server.test.ts index c8f64af6a43e..6282e6af0cac 100644 --- a/extensions/qa-lab/src/providers/mock-openai/server.test.ts +++ b/extensions/qa-lab/src/providers/mock-openai/server.test.ts @@ -293,6 +293,26 @@ describe("qa mock openai server", () => { expect(telegramLongBody).toContain("TELEGRAM-LONG-FINAL-END"); expect(telegramLongBody.length).toBeGreaterThan(4_500); + const whatsappLongResponse = await fetch(`${server.baseUrl}/v1/responses`, { + method: "POST", + headers: { + "content-type": "application/json", + }, + body: JSON.stringify({ + stream: true, + input: [ + makeUserInput("WhatsApp long final QA check. Use the scripted long final response."), + ], + }), + }); + expect(whatsappLongResponse.status).toBe(200); + const whatsappLongBody = await whatsappLongResponse.text(); + expect(whatsappLongBody).toContain('"type":"response.output_text.delta"'); + expect(whatsappLongBody).toContain('"phase":"final_answer"'); + expect(whatsappLongBody).toContain("WHATSAPP-LONG-FINAL-BEGIN"); + expect(whatsappLongBody).toContain("WHATSAPP-LONG-FINAL-END"); + expect(whatsappLongBody.length).toBeGreaterThan(6_000); + const telegramThreeChunkLongResponse = await fetch(`${server.baseUrl}/v1/responses`, { method: "POST", headers: { @@ -2856,6 +2876,80 @@ describe("qa mock openai server", () => { expect(outputText(await response.json())).toBe("QA_CANARY_TEST"); }); + it("uses WhatsApp location markers only for the matching coordinate body", async () => { + const server = await startMockServer(); + const setupInput = makeUserInput( + "When a later WhatsApp location message shows 37.774900, -122.419400, " + + "reply with only this WhatsApp location marker: QA_WHATSAPP_LOCATION_OK. " + + "Reply with only this exact marker: QA_INITIAL_OK", + ); + + const setupResponse = await postResponses(server, { + stream: false, + input: [setupInput], + }); + + const response = await postResponses(server, { + stream: false, + input: [setupInput, makeUserInput("📍 37.774900, -122.419400")], + }); + + expect(setupResponse.status).toBe(200); + expect(outputText(await setupResponse.json())).toBe("QA_INITIAL_OK"); + expect(response.status).toBe(200); + expect(outputText(await response.json())).toBe("QA_WHATSAPP_LOCATION_OK"); + }); + + it("uses WhatsApp contact and sticker markers only for matching structured bodies", async () => { + const server = await startMockServer(); + const setupInput = makeUserInput( + "When a later WhatsApp contact message appears, " + + "reply with only this WhatsApp contact marker: QA_WHATSAPP_CONTACT_OK. " + + "When a later WhatsApp sticker message appears, " + + "reply with only this WhatsApp sticker marker: QA_WHATSAPP_STICKER_OK. " + + "Reply with only this exact marker: QA_STRUCTURED_INITIAL_OK", + ); + + const setupResponse = await postResponses(server, { + stream: false, + input: [setupInput], + }); + const contactResponse = await postResponses(server, { + stream: false, + input: [setupInput, makeUserInput("")], + }); + const stickerResponse = await postResponses(server, { + stream: false, + input: [setupInput, makeUserInput("")], + }); + + expect(setupResponse.status).toBe(200); + expect(outputText(await setupResponse.json())).toBe("QA_STRUCTURED_INITIAL_OK"); + expect(contactResponse.status).toBe(200); + expect(outputText(await contactResponse.json())).toBe("QA_WHATSAPP_CONTACT_OK"); + expect(stickerResponse.status).toBe(200); + expect(outputText(await stickerResponse.json())).toBe("QA_WHATSAPP_STICKER_OK"); + }); + + it("streams WhatsApp location markers for the matching coordinate body", async () => { + const server = await startMockServer(); + + const body = await expectResponsesText(server, { + stream: true, + input: [ + makeUserInput( + "When a later WhatsApp location message shows 37.774900, -122.419400, " + + "reply with only this WhatsApp location marker: QA_WHATSAPP_LOCATION_STREAM_OK. " + + "Reply with only this exact marker: QA_INITIAL_STREAM_OK", + ), + makeUserInput("📍 37.774900, -122.419400"), + ], + }); + + expect(body).toContain("QA_WHATSAPP_LOCATION_STREAM_OK"); + expect(body).not.toContain("QA_INITIAL_STREAM_OK"); + }); + it("uses image generation directives from request context when the latest user text is generic", async () => { const server = await startQaMockOpenAiServer({ host: "127.0.0.1", @@ -3482,6 +3576,30 @@ describe("qa mock openai server", () => { const ids = body.data.map((entry) => entry.id); expect(ids).toContain("claude-opus-4-8"); expect(ids).toContain("gpt-5.5"); + expect(ids).toContain("gpt-4o-transcribe"); + }); + + it("serves deterministic OpenAI-compatible audio transcription responses", async () => { + const server = await startQaMockOpenAiServer({ + host: "127.0.0.1", + port: 0, + }); + cleanups.push(async () => { + await server.stop(); + }); + + const response = await fetch(`${server.baseUrl}/v1/audio/transcriptions`, { + method: "POST", + headers: { + "content-type": "multipart/form-data; boundary=qa", + }, + body: "--qa\r\n--qa--\r\n", + }); + + expect(response.status).toBe(200); + await expect(response.json()).resolves.toEqual({ + text: "Reply with only this exact marker: WHATSAPP_QA_AUDIO_TRANSCRIPT_OK", + }); }); it("dispatches an Anthropic /v1/messages read tool call for source discovery prompts", async () => { diff --git a/extensions/qa-lab/src/providers/mock-openai/server.ts b/extensions/qa-lab/src/providers/mock-openai/server.ts index dc1378c94f6a..cc88938eac42 100644 --- a/extensions/qa-lab/src/providers/mock-openai/server.ts +++ b/extensions/qa-lab/src/providers/mock-openai/server.ts @@ -160,6 +160,7 @@ const QA_TELEGRAM_CURRENT_SESSION_STATUS_PROMPT_RE = /telegram current session_s const QA_TELEGRAM_STREAM_SINGLE_MARKER = "QA-TELEGRAM-STREAM-SINGLE-OK"; const QA_TELEGRAM_LONG_FINAL_THREE_CHUNK_PROMPT_RE = /telegram long final three chunk qa check/i; const QA_TELEGRAM_LONG_FINAL_PROMPT_RE = /telegram long final qa check/i; +const QA_WHATSAPP_LONG_FINAL_PROMPT_RE = /whatsapp long final qa check/i; const QA_SUBAGENT_DIRECT_FALLBACK_PROMPT_RE = /subagent direct fallback qa check/i; const QA_SUBAGENT_DIRECT_FALLBACK_WORKER_RE = /subagent direct fallback worker/i; const QA_SUBAGENT_DIRECT_FALLBACK_MARKER = "QA-SUBAGENT-DIRECT-FALLBACK-OK"; @@ -176,6 +177,8 @@ const QA_RELEASE_AUDIT_PROMPT_RE = /release readiness audit for the small projec const QA_TOOL_SEARCH_PROMPT_RE = /tool search qa check/i; const QA_TOOL_SEARCH_FAILURE_PROMPT_RE = /tool search qa failure/i; const QA_MCP_CODE_MODE_PROMPT_RE = /mcp code mode qa check/i; +const QA_AUDIO_TRANSCRIPTION_TEXT = + "Reply with only this exact marker: WHATSAPP_QA_AUDIO_TRANSCRIPT_OK"; const QA_MCP_CODE_MODE_API_FILE_PROMPT_RE = /mcp code mode api file qa check/i; type MockScenarioState = { @@ -870,6 +873,33 @@ function extractExactMarkerDirective(text: string) { ); } +function extractWhatsAppLocationMarkerDirective(text: string) { + return extractLastCapture( + text, + /WhatsApp location marker:\s*([^\s`.,;:!?]+(?:-[^\s`.,;:!?]+)*)/i, + ); +} + +function extractWhatsAppContactMarkerDirective(text: string) { + return extractLastCapture(text, /WhatsApp contact marker:\s*([^\s`.,;:!?]+(?:-[^\s`.,;:!?]+)*)/i); +} + +function extractWhatsAppStickerMarkerDirective(text: string) { + return extractLastCapture(text, /WhatsApp sticker marker:\s*([^\s`.,;:!?]+(?:-[^\s`.,;:!?]+)*)/i); +} + +function shouldUseWhatsAppLocationMarker(prompt: string) { + return /^📍\s*37\.774900,\s*-122\.419400\b/u.test(prompt.trim()); +} + +function shouldUseWhatsAppContactMarker(prompt: string) { + return /^)/iu.test(prompt.trim()); +} + +function shouldUseWhatsAppStickerMarker(prompt: string) { + return /^(?:\s|$)/iu.test(prompt.trim()); +} + function extractLabeledMarkerDirective(text: string, label: string) { const escapedLabel = escapeRegExp(label); const backtickedMatch = extractLastCapture( @@ -1116,6 +1146,15 @@ function buildAssistantText( const exactReplyDirective = promptExactReplyDirective ?? extractExactReplyDirective(allInputText); const exactMarkerDirective = extractExactMarkerDirective(prompt) ?? extractExactMarkerDirective(allInputText); + const whatsAppLocationMarker = shouldUseWhatsAppLocationMarker(prompt) + ? extractWhatsAppLocationMarkerDirective(allInputText) + : ""; + const whatsAppContactMarker = shouldUseWhatsAppContactMarker(prompt) + ? extractWhatsAppContactMarkerDirective(allInputText) + : ""; + const whatsAppStickerMarker = shouldUseWhatsAppStickerMarker(prompt) + ? extractWhatsAppStickerMarkerDirective(allInputText) + : ""; const finishExactlyDirective = extractFinishExactlyDirective(prompt) ?? extractFinishExactlyDirective(allInputText); const latestImageUserTurn = extractLatestImageUserTurn(input); @@ -1157,6 +1196,15 @@ function buildAssistantText( ) { return "Protocol note: the attached image is split horizontally, with red on top and blue on the bottom."; } + if (whatsAppLocationMarker) { + return whatsAppLocationMarker; + } + if (whatsAppContactMarker) { + return whatsAppContactMarker; + } + if (whatsAppStickerMarker) { + return whatsAppStickerMarker; + } if (/\bmarker\b/i.test(allInputText) && exactReplyDirective) { return exactReplyDirective; } @@ -1468,19 +1516,20 @@ function splitMockStreamingText(text: string, parts = 3) { return chunks.length > 1 ? chunks : [text.slice(0, 1), text.slice(1)]; } -function buildTelegramLongFinalText({ +function buildQaLongFinalText({ endMarker = "TELEGRAM-LONG-FINAL-END", + segmentPrefix = "telegram-long-final-segment", segmentCount = 42, startMarker = "TELEGRAM-LONG-FINAL-BEGIN", }: { endMarker?: string; + segmentPrefix?: string; segmentCount?: number; startMarker?: string; } = {}) { const body = Array.from( { length: segmentCount }, - (_, index) => - `telegram-long-final-segment-${String(index + 1).padStart(3, "0")} ${"x".repeat(54)}`, + (_, index) => `${segmentPrefix}-${String(index + 1).padStart(3, "0")} ${"x".repeat(54)}`, ).join("\n"); return `${startMarker}\n${body}\n${endMarker}`; } @@ -1752,6 +1801,15 @@ async function buildResponsesPayload( extractExactReplyDirective(prompt) ?? extractExactReplyDirective(allInputText); const exactMarkerDirective = extractExactMarkerDirective(prompt) ?? extractExactMarkerDirective(allInputText); + const whatsAppLocationMarker = shouldUseWhatsAppLocationMarker(prompt) + ? extractWhatsAppLocationMarkerDirective(allInputText) + : ""; + const whatsAppContactMarker = shouldUseWhatsAppContactMarker(prompt) + ? extractWhatsAppContactMarkerDirective(allInputText) + : ""; + const whatsAppStickerMarker = shouldUseWhatsAppStickerMarker(prompt) + ? extractWhatsAppStickerMarkerDirective(allInputText) + : ""; const blockStreamingPrompt = extractLastMatchingUserText(extractAllUserTexts(input), QA_BLOCK_STREAMING_PROMPT_RE) || prompt || @@ -1975,7 +2033,7 @@ async function buildResponsesPayload( return buildAssistantEvents(""); } if (QA_TELEGRAM_LONG_FINAL_THREE_CHUNK_PROMPT_RE.test(allInputText)) { - const text = buildTelegramLongFinalText({ + const text = buildQaLongFinalText({ endMarker: "TELEGRAM-LONG-FINAL-3CHUNK-END", segmentCount: 96, startMarker: "TELEGRAM-LONG-FINAL-3CHUNK-BEGIN", @@ -1990,7 +2048,7 @@ async function buildResponsesPayload( ]); } if (QA_TELEGRAM_LONG_FINAL_PROMPT_RE.test(allInputText)) { - const text = buildTelegramLongFinalText(); + const text = buildQaLongFinalText(); return buildAssistantEvents([ { id: "msg_mock_telegram_long_final", @@ -2000,6 +2058,22 @@ async function buildResponsesPayload( }, ]); } + if (QA_WHATSAPP_LONG_FINAL_PROMPT_RE.test(allInputText)) { + const text = buildQaLongFinalText({ + endMarker: "WHATSAPP-LONG-FINAL-END", + segmentPrefix: "whatsapp-long-final-segment", + segmentCount: 64, + startMarker: "WHATSAPP-LONG-FINAL-BEGIN", + }); + return buildAssistantEvents([ + { + id: "msg_mock_whatsapp_long_final", + phase: "final_answer", + streamDeltas: splitMockStreamingText(text), + text, + }, + ]); + } if ( QA_STREAMING_PROMPT_RE.test(allInputText) && allInputText.includes(QA_TELEGRAM_STREAM_SINGLE_MARKER) @@ -2082,6 +2156,15 @@ async function buildResponsesPayload( exactMarkerDirective ?? exactReplyDirective ?? "QA-GROUP-FALLBACK-OK", ); } + if (whatsAppLocationMarker) { + return buildAssistantEvents(whatsAppLocationMarker); + } + if (whatsAppContactMarker) { + return buildAssistantEvents(whatsAppContactMarker); + } + if (whatsAppStickerMarker) { + return buildAssistantEvents(whatsAppStickerMarker); + } if (/\bmarker\b/i.test(prompt) && exactReplyDirective) { return buildAssistantEvents(exactReplyDirective); } @@ -3093,6 +3176,7 @@ export async function startQaMockOpenAiServer(params?: { host?: string; port?: n { id: "gpt-5.5", object: "model" }, { id: "gpt-5.5-alt", object: "model" }, { id: "gpt-image-1", object: "model" }, + { id: "gpt-4o-transcribe", object: "model" }, { id: "text-embedding-3-small", object: "model" }, { id: "claude-opus-4-8", object: "model" }, { id: "claude-sonnet-4-6", object: "model" }, @@ -3129,6 +3213,13 @@ export async function startQaMockOpenAiServer(params?: { host?: string; port?: n }); return; } + if (req.method === "POST" && url.pathname === "/v1/audio/transcriptions") { + await readBody(req); + writeJson(res, 200, { + text: QA_AUDIO_TRANSCRIPTION_TEXT, + }); + return; + } if (req.method === "POST" && url.pathname === "/v1/embeddings") { const raw = await readBody(req); const body = raw ? (JSON.parse(raw) as Record) : {};