liuyu/openclaw
1
0
Fork 0
mirror of https://github.com/openclaw/openclaw.git synced 2026-06-06 05:51:15 +08:00
Code Issues Packages Projects Releases Wiki Activity

test(qa): require channel scenario markers

Browse Source
This commit is contained in:
Vincent Koc
2026-06-03 14:19:33 +02:00
parent 2fa60af960
commit a9f099d279
5 changed files with 198 additions and 11 deletions
Download Patch File Download Diff File Expand all files Collapse all files

13
qa/scenarios/channels/channel-chat-baseline.md
View File

@@ -12,6 +12,7 @@ coverage:
objective: Verify the QA agent can respond correctly in a shared channel and respect mention-driven group semantics.
successCriteria:
- Agent replies in the shared channel transcript.
- Agent visible reply contains the scenario marker.
- Agent keeps the conversation scoped to the channel.
- Agent respects mention-driven group routing semantics.
docsRefs:
@@ -24,7 +25,8 @@ execution:
kind: flow
summary: Verify the QA agent can respond correctly in a shared channel and respect mention-driven group semantics.
config:
mentionPrompt: "@openclaw explain the QA lab"
expectedMarker: QA-CHANNEL-BASELINE-OK
mentionPrompt: "@openclaw qa channel baseline marker check. Reply exactly: QA-CHANNEL-BASELINE-OK"
```
```yaml qa-flow
@@ -78,7 +80,14 @@ steps:
- ref: state
- lambda:
params: [candidate]
expr: "candidate.conversation.id === 'qa-room' && !candidate.threadId"
expr: "candidate.direction === 'outbound' && candidate.conversation.id === 'qa-room' && candidate.conversation.kind === 'channel' && !candidate.threadId && String(candidate.text ?? '').includes(config.expectedMarker)"
- expr: liveTurnTimeoutMs(env, 180000)
- set: matchingOutbound
value:
expr: "state.getSnapshot().messages.filter((candidate) => candidate.direction === 'outbound' && candidate.conversation.id === 'qa-room' && candidate.conversation.kind === 'channel' && String(candidate.text ?? '').includes(config.expectedMarker))"
- assert:
expr: matchingOutbound.length === 1
message:
expr: "`expected exactly one channel baseline marker reply, saw ${matchingOutbound.length}; transcript=${formatTransportTranscript(state, { conversationId: 'qa-room' })}`"
detailsExpr: message.text
```

13
qa/scenarios/channels/dm-chat-baseline.md
View File

@@ -12,6 +12,7 @@ coverage:
objective: Verify the QA agent can chat coherently in a DM, explain the QA setup, and stay in character.
successCriteria:
- Agent replies in DM without channel routing mistakes.
- Agent visible reply contains the scenario marker.
- Agent explains the QA lab and message bus correctly.
- Agent keeps the dev C-3PO personality.
docsRefs:
@@ -24,7 +25,8 @@ execution:
kind: flow
summary: Verify the QA agent can chat coherently in a DM, explain the QA setup, and stay in character.
config:
prompt: "Hello there, who are you?"
expectedMarker: QA-DM-BASELINE-OK
prompt: "DM baseline marker check. Include exact marker: `QA-DM-BASELINE-OK` and briefly identify the QA lab message bus."
```
```yaml qa-flow
@@ -47,7 +49,14 @@ steps:
- ref: state
- lambda:
params: [candidate]
expr: "candidate.conversation.id === 'alice'"
expr: "candidate.direction === 'outbound' && candidate.conversation.id === 'alice' && candidate.conversation.kind === 'direct' && String(candidate.text ?? '').includes(config.expectedMarker)"
- expr: liveTurnTimeoutMs(env, 45000)
- set: matchingOutbound
value:
expr: "state.getSnapshot().messages.filter((candidate) => candidate.direction === 'outbound' && candidate.conversation.id === 'alice' && candidate.conversation.kind === 'direct' && String(candidate.text ?? '').includes(config.expectedMarker))"
- assert:
expr: matchingOutbound.length === 1
message:
expr: "`expected exactly one DM baseline marker reply, saw ${matchingOutbound.length}; transcript=${formatTransportTranscript(state, { conversationId: 'alice' })}`"
detailsExpr: outbound.text
```

17
qa/scenarios/channels/qa-channel-reconnect-dedupe.md
View File

@@ -64,7 +64,7 @@ steps:
- ref: state
- lambda:
params: [candidate]
expr: "candidate.conversation.id === 'qa-room' && candidate.direction === 'outbound'"
expr: "candidate.conversation.id === 'qa-room' && candidate.direction === 'outbound' && String(candidate.text ?? '').includes(config.firstMarker)"
- expr: liveTurnTimeoutMs(env, 60000)
- set: beforeRestartCursor
value:
@@ -80,9 +80,9 @@ steps:
value:
expr: "state.getSnapshot().messages.filter((candidate) => candidate.direction === 'outbound' && candidate.conversation.id === 'qa-room')"
- assert:
expr: "firstMatchesBeforeFollowup.length === 1"
expr: "firstMatchesBeforeFollowup.length === 1 && String(firstMatchesBeforeFollowup[0]?.text ?? '').includes(config.firstMarker)"
message:
expr: "`readiness cycle replayed first reply ${firstMatchesBeforeFollowup.length} times; transcript=${formatTransportTranscript(state, { conversationId: 'qa-room' })}`"
expr: "`readiness cycle should preserve exactly one marked first reply, saw ${firstMatchesBeforeFollowup.length}; transcript=${formatTransportTranscript(state, { conversationId: 'qa-room' })}`"
- call: runAgentPrompt
args:
- ref: env
@@ -99,7 +99,7 @@ steps:
- ref: state
- lambda:
params: [candidate]
expr: "candidate.conversation.id === 'qa-room' && candidate.direction === 'outbound'"
expr: "candidate.conversation.id === 'qa-room' && candidate.direction === 'outbound' && String(candidate.text ?? '').includes(config.secondMarker)"
- expr: liveTurnTimeoutMs(env, 60000)
- sinceIndex:
ref: beforeRestartCursor
@@ -108,13 +108,16 @@ steps:
expr: state.getSnapshot()
- set: firstMatches
value:
expr: "snapshot.messages.slice(0, beforeRestartCursor).filter((candidate) => candidate.direction === 'outbound' && candidate.conversation.id === 'qa-room')"
expr: "snapshot.messages.slice(0, beforeRestartCursor).filter((candidate) => candidate.direction === 'outbound' && candidate.conversation.id === 'qa-room' && String(candidate.text ?? '').includes(config.firstMarker))"
- set: secondMatches
value:
expr: "snapshot.messages.slice(beforeRestartCursor).filter((candidate) => candidate.direction === 'outbound' && candidate.conversation.id === 'qa-room' && String(candidate.text ?? '').includes(config.secondMarker))"
- set: postRestartOutbounds
value:
expr: "snapshot.messages.slice(beforeRestartCursor).filter((candidate) => candidate.direction === 'outbound' && candidate.conversation.id === 'qa-room')"
- assert:
expr: "firstMatches.length === 1 && secondMatches.length === 1"
expr: "firstMatches.length === 1 && secondMatches.length === 1 && postRestartOutbounds.length === 1 && !postRestartOutbounds.some((candidate) => String(candidate.text ?? '').includes(config.firstMarker))"
message:
expr: "`expected one pre-restart and one post-restart reply; first=${firstMatches.length} second=${secondMatches.length}; transcript=${formatTransportTranscript(state, { conversationId: 'qa-room' })}`"
expr: "`expected one marked pre-restart reply and exactly one marked post-restart reply without replaying the first marker; first=${firstMatches.length} second=${secondMatches.length} post=${postRestartOutbounds.length}; transcript=${formatTransportTranscript(state, { conversationId: 'qa-room' })}`"
detailsExpr: "`before=${firstOutbound.text}\\nafter=${secondOutbound.text}`"
```