From 2b396131e4177e3b5cd99e9efae32c5b95a53d71 Mon Sep 17 00:00:00 2001 From: Vincent Koc Date: Fri, 22 May 2026 20:00:20 +0800 Subject: [PATCH] test(qa-lab): add bus tool trace scenario --- CHANGELOG.md | 1 + .../qa-lab/src/scenario-catalog.test.ts | 21 ++++++ .../runtime/qa-bus-tool-trace-visibility.md | 69 +++++++++++++++++++ 3 files changed, 91 insertions(+) create mode 100644 qa/scenarios/runtime/qa-bus-tool-trace-visibility.md diff --git a/CHANGELOG.md b/CHANGELOG.md index e35ef21b9959..10f4a33103b5 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -19,6 +19,7 @@ Docs: https://docs.openclaw.ai - Dependencies: refresh provider, plugin, UI, and tooling packages, update `protobufjs` to 8.4.0 to clear the current npm advisory, and carry the Claude ACP completion patch forward to `@agentclientprotocol/claude-agent-acp` 0.36.1. - Agents/tools: remove the old sender-owner tool gating path so configured tools stay visible for trusted sessions while command and channel-action auth still carry real sender identity. - QA-Lab: add curated mock JSONL replay fixtures and first-drift reporting for runtime-parity audits. (#80323, refs #80176) Thanks @100yenadmin. +- QA-Lab: add a QA bus tool-trace visibility scenario for sanitized tool-call assertions. - QA-Lab: replace generic evidence framing in seeded scenario prompts with concrete observed QA behavior. - QA-Lab: list live transport lane membership in the coverage report so real transport checks stay separate from seeded qa-channel scenarios. - QA-Lab: include the optional 100-turn runtime parity soak in release-soak artifacts so long-run Codex/Pi transcript drift stays visible outside the default gate. (#80395) Thanks @100yenadmin. diff --git a/extensions/qa-lab/src/scenario-catalog.test.ts b/extensions/qa-lab/src/scenario-catalog.test.ts index 78bb051b05f9..f91fe7e485d2 100644 --- a/extensions/qa-lab/src/scenario-catalog.test.ts +++ b/extensions/qa-lab/src/scenario-catalog.test.ts @@ -202,6 +202,27 @@ describe("qa scenario catalog", () => { expect(readQaScenarioById("long-context-progress-watchdog").gatewayConfigPatch).toBeUndefined(); }); + it("loads the QA bus tool trace visibility harness scenario", () => { + const scenario = readQaScenarioById("qa-bus-tool-trace-visibility"); + const config = readQaScenarioExecutionConfig(scenario.id) as + | { + expectedToolName?: string; + expectedRedaction?: string; + searchQuery?: string; + } + | undefined; + + expect(scenario.sourcePath).toBe("qa/scenarios/runtime/qa-bus-tool-trace-visibility.md"); + expect(scenario.coverage?.primary).toContain("harness.tool-trace-visibility"); + expect(scenario.coverage?.secondary).toContain("runtime.qa-bus"); + expect(config?.expectedToolName).toBe("exec"); + expect(config?.expectedRedaction).toBe("[redacted]"); + expect(config?.searchQuery).toBe("exec"); + expect(scenario.execution.flow?.steps.map((step) => step.name)).toEqual([ + "preserves searchable sanitized tool-call traces", + ]); + }); + it("loads the opt-in update.run package self-upgrade sentinel", () => { const scenario = readQaScenarioById("update-run-package-self-upgrade"); const config = readQaScenarioExecutionConfig(scenario.id) as diff --git a/qa/scenarios/runtime/qa-bus-tool-trace-visibility.md b/qa/scenarios/runtime/qa-bus-tool-trace-visibility.md new file mode 100644 index 000000000000..0c2f670f6286 --- /dev/null +++ b/qa/scenarios/runtime/qa-bus-tool-trace-visibility.md @@ -0,0 +1,69 @@ +# QA bus tool trace visibility + +```yaml qa-scenario +id: qa-bus-tool-trace-visibility +title: QA bus tool trace visibility +surface: harness +coverage: + primary: + - harness.tool-trace-visibility + secondary: + - runtime.qa-bus + - tools.trace +objective: Verify QA-Lab can assert sanitized tool-call traces directly on bus messages. +successCriteria: + - QA bus messages can carry a toolCalls array. + - Readback preserves the tool name while redacting sensitive argument values. + - QA bus search can locate the message by tool name. +docsRefs: + - docs/help/testing.md + - qa/scenarios/index.md +codeRefs: + - extensions/qa-lab/src/bus-state.ts + - extensions/qa-lab/src/bus-queries.ts + - extensions/qa-lab/src/runtime-api.ts +execution: + kind: flow + summary: Add a synthetic tool-backed bus message and verify sanitized trace assertions. + config: + expectedToolName: exec + expectedRedaction: "[redacted]" + searchQuery: exec +``` + +```yaml qa-flow +steps: + - name: preserves searchable sanitized tool-call traces + actions: + - call: reset + - call: state.addOutboundMessage + saveAs: outbound + args: + - to: dm:qa-operator + text: qa bus tool trace check + toolCalls: + - name: + expr: config.expectedToolName + arguments: + command: pwd + apiToken: qa-secret-token + - set: readback + value: + expr: "state.readMessage({ messageId: outbound.id })" + - assert: + expr: "readback.toolCalls?.[0]?.name === config.expectedToolName" + message: + expr: "`expected tool name ${config.expectedToolName}, got ${String(readback.toolCalls?.[0]?.name ?? '')}`" + - assert: + expr: "readback.toolCalls?.[0]?.arguments?.command === config.expectedRedaction && readback.toolCalls?.[0]?.arguments?.apiToken === config.expectedRedaction" + message: + expr: "`expected redacted tool arguments, got ${JSON.stringify(readback.toolCalls?.[0]?.arguments ?? null)}`" + - set: searchMatches + value: + expr: "state.searchMessages({ query: config.searchQuery })" + - assert: + expr: "searchMatches.some((message) => message.id === outbound.id)" + message: + expr: "`expected search query ${config.searchQuery} to find ${outbound.id}, got ${JSON.stringify(searchMatches.map((message) => message.id))}`" + detailsExpr: "`${readback.toolCalls?.[0]?.name}:${String(readback.toolCalls?.[0]?.arguments?.command ?? '')}`" +```