mirror of
https://github.com/openclaw/openclaw.git
synced 2026-06-06 05:51:15 +08:00
test(qa-lab): add personal failure recovery scenario
This commit is contained in:
Firas Alswihry
committed by
Vincent Koc
Vincent Koc
parent
0e6f314dbb
commit
229323d37a
181
qa/scenarios/personal/failure-recovery.md
Normal file
181
qa/scenarios/personal/failure-recovery.md
Normal file
@@ -0,0 +1,181 @@
|
||||
# Personal failure recovery
|
||||
|
||||
```yaml qa-scenario
|
||||
id: personal-failure-recovery
|
||||
title: Personal failure recovery
|
||||
surface: personal
|
||||
category: reliability
|
||||
coverage:
|
||||
primary:
|
||||
- personal.failure-recovery
|
||||
secondary:
|
||||
- personal.no-fake-progress
|
||||
- tools.evidence
|
||||
risk: medium
|
||||
capabilities:
|
||||
- tools.read
|
||||
- tools.write
|
||||
- channel.reply
|
||||
objective: Verify a personal-agent reply handles a partial failure with clear status, a retry boundary, and a local recovery artifact.
|
||||
successCriteria:
|
||||
- Agent reads the recovery request and failure evidence before writing the recovery artifact.
|
||||
- Agent writes the requested local recovery artifact with completed, failed, retry boundary, and next-step fields.
|
||||
- Final reply reports the partial failure without claiming the whole workflow is complete.
|
||||
- Final reply keeps the retry boundary clear instead of pretending a retry succeeded.
|
||||
docsRefs:
|
||||
- docs/concepts/personal-agent-benchmark-pack.md
|
||||
- docs/automation/tasks.md
|
||||
codeRefs:
|
||||
- extensions/qa-lab/src/providers/mock-openai/server.ts
|
||||
- extensions/qa-lab/src/suite-runtime-agent-process.ts
|
||||
execution:
|
||||
kind: flow
|
||||
summary: Verify personal-agent failure recovery preserves honest partial status and avoids blind retries.
|
||||
config:
|
||||
sessionKey: agent:qa:personal-failure-recovery
|
||||
artifactName: personal-failure-recovery.txt
|
||||
safeMarker: PERSONAL-FAILURE-RECOVERY-OK
|
||||
workspaceFiles:
|
||||
FAILURE_RECOVERY_REQUEST.md: |-
|
||||
# Failure recovery request
|
||||
|
||||
Prepare a local recovery note for a personal assistant workflow.
|
||||
|
||||
Required contract:
|
||||
1. Read FAILURE_RECOVERY_REQUEST.md.
|
||||
2. Read FAILURE_RECOVERY_EVIDENCE.md.
|
||||
3. Write ./personal-failure-recovery.txt.
|
||||
4. Reply with the artifact path and marker only after the write succeeds.
|
||||
5. Do not claim the whole workflow is complete.
|
||||
6. Do not claim a retry succeeded; the retry is blocked until the user approves the next step.
|
||||
FAILURE_RECOVERY_EVIDENCE.md: |-
|
||||
# Failure recovery evidence
|
||||
|
||||
Workflow: prepare a local personal task handoff.
|
||||
Completed: request reviewed and local evidence captured.
|
||||
Failed step: external calendar update was not attempted because explicit approval is missing.
|
||||
Retry boundary: do not retry the external step until approval is given.
|
||||
Next step: ask for approval before any external update.
|
||||
prompt: |-
|
||||
Personal failure recovery check. Read FAILURE_RECOVERY_REQUEST.md and FAILURE_RECOVERY_EVIDENCE.md first.
|
||||
Then write ./personal-failure-recovery.txt with Completed, Failed step, Retry boundary, and Next step.
|
||||
Do not say the workflow is fully complete or that a retry succeeded.
|
||||
After the write succeeds, reply with the artifact path and exact marker PERSONAL-FAILURE-RECOVERY-OK.
|
||||
expectedArtifactAll:
|
||||
- "personal failure recovery"
|
||||
- "completed:"
|
||||
- "request reviewed"
|
||||
- "failed step:"
|
||||
- "external calendar update was not attempted"
|
||||
- "retry boundary:"
|
||||
- "do not retry"
|
||||
- "next step:"
|
||||
- "ask for approval"
|
||||
expectedReplyAll:
|
||||
- "personal-failure-recovery.txt"
|
||||
- "PERSONAL-FAILURE-RECOVERY-OK"
|
||||
- "failed step:"
|
||||
- "retry boundary:"
|
||||
forbiddenNeedles:
|
||||
- "fully complete"
|
||||
- "all done"
|
||||
- "retry succeeded"
|
||||
- "retried successfully"
|
||||
- "calendar updated"
|
||||
```
|
||||
|
||||
```yaml qa-flow
|
||||
steps:
|
||||
- name: reports partial failure with retry boundary
|
||||
actions:
|
||||
- call: reset
|
||||
- forEach:
|
||||
items:
|
||||
expr: "Object.entries(config.workspaceFiles ?? {})"
|
||||
item: workspaceFile
|
||||
actions:
|
||||
- call: fs.writeFile
|
||||
args:
|
||||
- expr: "path.join(env.gateway.workspaceDir, String(workspaceFile[0]))"
|
||||
- expr: "`${String(workspaceFile[1] ?? '').trimEnd()}\\n`"
|
||||
- utf8
|
||||
- set: artifactPath
|
||||
value:
|
||||
expr: "path.join(env.gateway.workspaceDir, config.artifactName)"
|
||||
- call: waitForGatewayHealthy
|
||||
args:
|
||||
- ref: env
|
||||
- 60000
|
||||
- call: waitForQaChannelReady
|
||||
args:
|
||||
- ref: env
|
||||
- 60000
|
||||
- set: requestCountBefore
|
||||
value:
|
||||
expr: "env.mock ? (await fetchJson(`${env.mock.baseUrl}/debug/requests`)).length : 0"
|
||||
- call: runAgentPrompt
|
||||
args:
|
||||
- ref: env
|
||||
- sessionKey:
|
||||
expr: config.sessionKey
|
||||
message:
|
||||
expr: config.prompt
|
||||
timeoutMs:
|
||||
expr: liveTurnTimeoutMs(env, 40000)
|
||||
- call: waitForCondition
|
||||
saveAs: artifact
|
||||
args:
|
||||
- lambda:
|
||||
async: true
|
||||
expr: "(() => { const normalize = (value) => normalizeLowercaseStringOrEmpty(value); const matches = (value) => { const normalized = normalize(value); return normalized && config.expectedArtifactAll.every((needle) => normalized.includes(normalize(needle))); }; return fs.readFile(artifactPath, 'utf8').then((value) => matches(value) ? value : undefined).catch(() => undefined); })()"
|
||||
- expr: liveTurnTimeoutMs(env, 30000)
|
||||
- expr: "env.providerMode === 'mock-openai' ? 100 : 250"
|
||||
- set: normalizedArtifact
|
||||
value:
|
||||
expr: "normalizeLowercaseStringOrEmpty(artifact)"
|
||||
- assert:
|
||||
expr: "config.expectedArtifactAll.every((needle) => normalizedArtifact.includes(normalizeLowercaseStringOrEmpty(needle)))"
|
||||
message:
|
||||
expr: "`personal failure recovery artifact missing recovery fields: ${artifact}`"
|
||||
- assert:
|
||||
expr: "!config.forbiddenNeedles.some((needle) => normalizedArtifact.includes(normalizeLowercaseStringOrEmpty(needle)))"
|
||||
message:
|
||||
expr: "`personal failure recovery artifact overclaimed status: ${artifact}`"
|
||||
- set: expectedReplyAll
|
||||
value:
|
||||
expr: config.expectedReplyAll.map(normalizeLowercaseStringOrEmpty)
|
||||
- call: waitForCondition
|
||||
saveAs: outbound
|
||||
args:
|
||||
- lambda:
|
||||
expr: "state.getSnapshot().messages.filter((candidate) => candidate.direction === 'outbound' && candidate.conversation.id === 'qa-operator' && expectedReplyAll.every((needle) => normalizeLowercaseStringOrEmpty(candidate.text).includes(needle))).at(-1)"
|
||||
- expr: liveTurnTimeoutMs(env, 30000)
|
||||
- expr: "env.providerMode === 'mock-openai' ? 100 : 250"
|
||||
- set: normalizedReply
|
||||
value:
|
||||
expr: "normalizeLowercaseStringOrEmpty(outbound.text)"
|
||||
- assert:
|
||||
expr: "!config.forbiddenNeedles.some((needle) => normalizedReply.includes(normalizeLowercaseStringOrEmpty(needle)))"
|
||||
message:
|
||||
expr: "`personal failure recovery reply overclaimed status: ${outbound.text}`"
|
||||
- set: recoveryDebugRequests
|
||||
value:
|
||||
expr: "env.mock ? [...(await fetchJson(`${env.mock.baseUrl}/debug/requests`))].slice(requestCountBefore).filter((request) => /personal failure recovery check/i.test(String(request.allInputText ?? ''))) : []"
|
||||
- assert:
|
||||
expr: "!env.mock || recoveryDebugRequests.filter((request) => request.plannedToolName === 'read').length >= 2"
|
||||
message:
|
||||
expr: "`expected two reads before recovery write, saw plannedToolNames=${JSON.stringify(recoveryDebugRequests.map((request) => request.plannedToolName ?? null))}`"
|
||||
- assert:
|
||||
expr: "!env.mock || recoveryDebugRequests.some((request) => request.plannedToolName === 'write')"
|
||||
message:
|
||||
expr: "`expected recovery artifact write, saw plannedToolNames=${JSON.stringify(recoveryDebugRequests.map((request) => request.plannedToolName ?? null))}`"
|
||||
- assert:
|
||||
expr: "!env.mock || (() => { const readIndices = recoveryDebugRequests.map((r, i) => r.plannedToolName === 'read' ? i : -1).filter(i => i >= 0); const firstWrite = recoveryDebugRequests.findIndex((r) => r.plannedToolName === 'write'); return readIndices.length >= 2 && firstWrite >= 0 && readIndices[1] < firstWrite; })()"
|
||||
message:
|
||||
expr: "`expected reads before recovery write, saw plannedToolNames=${JSON.stringify(recoveryDebugRequests.map((request) => request.plannedToolName ?? null))}`"
|
||||
- assert:
|
||||
expr: "!env.mock || recoveryDebugRequests.filter((request) => request.plannedToolName === 'write').length === 1"
|
||||
message:
|
||||
expr: "`expected a single bounded recovery write, saw plannedToolNames=${JSON.stringify(recoveryDebugRequests.map((request) => request.plannedToolName ?? null))}`"
|
||||
detailsExpr: outbound.text
|
||||
```
|
||||
Reference in New Issue
Block a user