mirror of
https://github.com/openclaw/openclaw.git
synced 2026-06-14 01:58:47 +08:00
Compare commits
69 Commits
ak/memory-
...
codex/secu
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
93fc591af9 | ||
|
|
4c23d1d597 | ||
|
|
8eb1fa09c6 | ||
|
|
2d2c1e63f0 | ||
|
|
6cdbccaa9e | ||
|
|
9f522ee7df | ||
|
|
7404b2b5b4 | ||
|
|
73aabcceda | ||
|
|
b1fc8673df | ||
|
|
4cf4e54179 | ||
|
|
84519f7e3c | ||
|
|
6314c377bb | ||
|
|
d3e7e03669 | ||
|
|
64f9f3c278 | ||
|
|
cd3eb438f0 | ||
|
|
26281a8a11 | ||
|
|
4208c89ec4 | ||
|
|
c9c19a1106 | ||
|
|
f78d7b52d8 | ||
|
|
ff6940036b | ||
|
|
b477bfe84b | ||
|
|
d4237cb14d | ||
|
|
20bc546d94 | ||
|
|
069cb8d636 | ||
|
|
8cc5d2d85c | ||
|
|
690f27749c | ||
|
|
7af8153388 | ||
|
|
a66a065ffb | ||
|
|
64d0fc8336 | ||
|
|
f3df863aff | ||
|
|
26b9736922 | ||
|
|
44f45d8729 | ||
|
|
1c655008cd | ||
|
|
618d78144e | ||
|
|
56f2102c28 | ||
|
|
99db98a7ce | ||
|
|
0dbfa1f6be | ||
|
|
f06f2f17c2 | ||
|
|
7bd533a80e | ||
|
|
561b293c7a | ||
|
|
21aa8faf8a | ||
|
|
b0bd9c8ed8 | ||
|
|
c5d599c8c4 | ||
|
|
3a1a5c0dac | ||
|
|
0849cac106 | ||
|
|
32ce06daf8 | ||
|
|
751d3db1cc | ||
|
|
4640baa299 | ||
|
|
c9f0bfd476 | ||
|
|
ab559a7257 | ||
|
|
7c08804541 | ||
|
|
991471b8ec | ||
|
|
7190fc4de8 | ||
|
|
9d9389bc6b | ||
|
|
6c88811b4b | ||
|
|
4a6666796f | ||
|
|
68222ba5e3 | ||
|
|
1bd783045b | ||
|
|
6cf06e8e7e | ||
|
|
ded3a93058 | ||
|
|
0063f3076c | ||
|
|
8c7e5c6918 | ||
|
|
e338037034 | ||
|
|
05796759ad | ||
|
|
d8b3e523ff | ||
|
|
4809ac70fa | ||
|
|
777edadb36 | ||
|
|
8d9ce35b92 | ||
|
|
69bf333dde |
61
.github/codeql/codeql-process-exec-boundary-critical-security.yml
vendored
Normal file
61
.github/codeql/codeql-process-exec-boundary-critical-security.yml
vendored
Normal file
@@ -0,0 +1,61 @@
|
||||
name: openclaw-codeql-process-exec-boundary-critical-security
|
||||
|
||||
disable-default-queries: true
|
||||
|
||||
queries:
|
||||
- uses: security-extended
|
||||
|
||||
query-filters:
|
||||
- include:
|
||||
precision:
|
||||
- high
|
||||
- very-high
|
||||
tags contain: security
|
||||
security-severity: /([7-9]|10)\.(\d)+/
|
||||
|
||||
paths:
|
||||
- src/process
|
||||
- src/tui/tui-local-shell.ts
|
||||
- src/tui/tui.ts
|
||||
- src/plugin-sdk/windows-spawn.ts
|
||||
- packages/agent-core/src/harness/env
|
||||
- packages/memory-host-sdk/src/host
|
||||
- extensions/acpx/src
|
||||
- extensions/bonjour/src/advertiser.ts
|
||||
- extensions/browser/src/browser/chrome-mcp.ts
|
||||
- extensions/browser/src/browser/chrome.executables.ts
|
||||
- extensions/browser/src/browser/chrome.ts
|
||||
- extensions/codex/src/app-server/sandbox-exec-server
|
||||
- extensions/codex/src/app-server/transport-stdio.ts
|
||||
- extensions/codex/src/node-cli-sessions.ts
|
||||
- extensions/codex-supervisor/src/json-rpc-client.ts
|
||||
- extensions/file-transfer/src
|
||||
- extensions/google-meet/src
|
||||
- extensions/imessage/src
|
||||
- extensions/memory-core/src/memory/qmd-manager.ts
|
||||
- extensions/memory-wiki/src/obsidian.ts
|
||||
- extensions/microsoft-foundry/cli.ts
|
||||
- extensions/ollama/src/wsl2-crash-loop-check.ts
|
||||
- extensions/qa-lab/src
|
||||
- extensions/signal/src/daemon.ts
|
||||
- extensions/tts-local-cli/speech-provider.ts
|
||||
- extensions/voice-call/src
|
||||
- scripts
|
||||
|
||||
paths-ignore:
|
||||
- "**/node_modules"
|
||||
- "**/coverage"
|
||||
- "**/*.generated.ts"
|
||||
- "**/*.bundle.js"
|
||||
- "**/*-runtime.js"
|
||||
- "**/*.test.ts"
|
||||
- "**/*.test.tsx"
|
||||
- "**/*.spec.ts"
|
||||
- "**/*.spec.tsx"
|
||||
- "**/*.e2e.test.ts"
|
||||
- "**/*.e2e.test.tsx"
|
||||
- "**/*test-support*"
|
||||
- "**/*test-helper*"
|
||||
- "**/*mock*"
|
||||
- "**/*fixture*"
|
||||
- "**/*bench*"
|
||||
47
.github/workflows/codeql.yml
vendored
47
.github/workflows/codeql.yml
vendored
@@ -17,7 +17,28 @@ on:
|
||||
- ".github/actions/**"
|
||||
- ".github/codeql/**"
|
||||
- ".github/workflows/**"
|
||||
- "extensions/acpx/src/**"
|
||||
- "extensions/bonjour/src/advertiser.ts"
|
||||
- "extensions/browser/src/browser/chrome-mcp.ts"
|
||||
- "extensions/browser/src/browser/chrome.executables.ts"
|
||||
- "extensions/browser/src/browser/chrome.ts"
|
||||
- "extensions/codex/src/app-server/sandbox-exec-server/**"
|
||||
- "extensions/codex/src/app-server/transport-stdio.ts"
|
||||
- "extensions/codex/src/node-cli-sessions.ts"
|
||||
- "extensions/codex-supervisor/src/json-rpc-client.ts"
|
||||
- "extensions/file-transfer/src/**"
|
||||
- "extensions/google-meet/src/**"
|
||||
- "extensions/imessage/src/**"
|
||||
- "extensions/memory-core/src/memory/qmd-manager.ts"
|
||||
- "extensions/memory-wiki/src/obsidian.ts"
|
||||
- "extensions/microsoft-foundry/cli.ts"
|
||||
- "extensions/ollama/src/wsl2-crash-loop-check.ts"
|
||||
- "extensions/qa-lab/src/**"
|
||||
- "extensions/signal/src/daemon.ts"
|
||||
- "extensions/tts-local-cli/speech-provider.ts"
|
||||
- "extensions/voice-call/src/**"
|
||||
- "packages/**"
|
||||
- "scripts/**"
|
||||
- "src/**"
|
||||
push:
|
||||
branches:
|
||||
@@ -26,7 +47,28 @@ on:
|
||||
- ".github/actions/**"
|
||||
- ".github/codeql/**"
|
||||
- ".github/workflows/**"
|
||||
- "extensions/acpx/src/**"
|
||||
- "extensions/bonjour/src/advertiser.ts"
|
||||
- "extensions/browser/src/browser/chrome-mcp.ts"
|
||||
- "extensions/browser/src/browser/chrome.executables.ts"
|
||||
- "extensions/browser/src/browser/chrome.ts"
|
||||
- "extensions/codex/src/app-server/sandbox-exec-server/**"
|
||||
- "extensions/codex/src/app-server/transport-stdio.ts"
|
||||
- "extensions/codex/src/node-cli-sessions.ts"
|
||||
- "extensions/codex-supervisor/src/json-rpc-client.ts"
|
||||
- "extensions/file-transfer/src/**"
|
||||
- "extensions/google-meet/src/**"
|
||||
- "extensions/imessage/src/**"
|
||||
- "extensions/memory-core/src/memory/qmd-manager.ts"
|
||||
- "extensions/memory-wiki/src/obsidian.ts"
|
||||
- "extensions/microsoft-foundry/cli.ts"
|
||||
- "extensions/ollama/src/wsl2-crash-loop-check.ts"
|
||||
- "extensions/qa-lab/src/**"
|
||||
- "extensions/signal/src/daemon.ts"
|
||||
- "extensions/tts-local-cli/speech-provider.ts"
|
||||
- "extensions/voice-call/src/**"
|
||||
- "packages/**"
|
||||
- "scripts/**"
|
||||
- "src/**"
|
||||
schedule:
|
||||
- cron: "0 6 * * *"
|
||||
@@ -73,6 +115,11 @@ jobs:
|
||||
runs_on: blacksmith-4vcpu-ubuntu-2404
|
||||
timeout_minutes: 25
|
||||
config_file: ./.github/codeql/codeql-mcp-process-tool-boundary-critical-security.yml
|
||||
- language: javascript-typescript
|
||||
category: process-exec-boundary
|
||||
runs_on: blacksmith-4vcpu-ubuntu-2404
|
||||
timeout_minutes: 25
|
||||
config_file: ./.github/codeql/codeql-process-exec-boundary-critical-security.yml
|
||||
- language: javascript-typescript
|
||||
category: plugin-trust-boundary
|
||||
runs_on: blacksmith-4vcpu-ubuntu-2404
|
||||
|
||||
@@ -437,8 +437,17 @@ jobs:
|
||||
echo "::warning::Could not generate motion-trimmed desktop previews; continuing with screenshots and full MP4 links."
|
||||
fi
|
||||
|
||||
baseline_status="$(jq -r '.scenarios[0].status' "$root/baseline/discord-qa-summary.json")"
|
||||
candidate_status="$(jq -r '.scenarios[0].status' "$root/candidate/discord-qa-summary.json")"
|
||||
read_discord_status_reaction_status() {
|
||||
local lane="$1"
|
||||
if [[ -f "$root/$lane/qa-evidence.json" ]]; then
|
||||
jq -r '.entries[0].result.status' "$root/$lane/qa-evidence.json"
|
||||
return
|
||||
fi
|
||||
jq -r '.scenarios[0].status' "$root/$lane/discord-qa-summary.json"
|
||||
}
|
||||
|
||||
baseline_status="$(read_discord_status_reaction_status baseline)"
|
||||
candidate_status="$(read_discord_status_reaction_status candidate)"
|
||||
|
||||
jq -n \
|
||||
--arg baseline_status "$baseline_status" \
|
||||
|
||||
@@ -451,8 +451,17 @@ jobs:
|
||||
|
||||
capture_candidate_discord_web
|
||||
|
||||
baseline_status="$(jq -r '.scenarios[] | select(.id == "discord-thread-reply-filepath-attachment") | .status' "$root/baseline/discord-qa-summary.json")"
|
||||
candidate_status="$(jq -r '.scenarios[] | select(.id == "discord-thread-reply-filepath-attachment") | .status' "$root/candidate/discord-qa-summary.json")"
|
||||
read_discord_thread_attachment_status() {
|
||||
local lane="$1"
|
||||
if [[ -f "$root/$lane/qa-evidence.json" ]]; then
|
||||
jq -r '.entries[] | select(.test.id == "discord-thread-reply-filepath-attachment") | .result.status' "$root/$lane/qa-evidence.json"
|
||||
return
|
||||
fi
|
||||
jq -r '.scenarios[] | select(.id == "discord-thread-reply-filepath-attachment") | .status' "$root/$lane/discord-qa-summary.json"
|
||||
}
|
||||
|
||||
baseline_status="$(read_discord_thread_attachment_status baseline)"
|
||||
candidate_status="$(read_discord_thread_attachment_status candidate)"
|
||||
comparison_status="fail"
|
||||
if [[ "$baseline_status" == "fail" && "$candidate_status" == "pass" ]]; then
|
||||
comparison_status="pass"
|
||||
|
||||
4
.github/workflows/mantis-telegram-live.yml
vendored
4
.github/workflows/mantis-telegram-live.yml
vendored
@@ -445,8 +445,8 @@ jobs:
|
||||
telegram_exit=$?
|
||||
set -e
|
||||
|
||||
if [[ ! -f "$root/telegram-qa-summary.json" ]]; then
|
||||
echo "Telegram live QA did not produce a summary." >&2
|
||||
if [[ ! -f "$root/qa-evidence.json" && ! -f "$root/telegram-qa-summary.json" ]]; then
|
||||
echo "Telegram live QA did not produce an evidence summary." >&2
|
||||
exit "$telegram_exit"
|
||||
fi
|
||||
echo "telegram_exit=${telegram_exit}" >> "$GITHUB_OUTPUT"
|
||||
|
||||
@@ -1748,6 +1748,7 @@ jobs:
|
||||
anthropic) require_any Anthropic ANTHROPIC_API_KEY ANTHROPIC_API_KEY_OLD ANTHROPIC_API_TOKEN ;;
|
||||
google) require_any Google GEMINI_API_KEY GOOGLE_API_KEY ;;
|
||||
minimax) require_any MiniMax MINIMAX_API_KEY ;;
|
||||
moonshot) require_any Moonshot MOONSHOT_API_KEY KIMI_API_KEY ;;
|
||||
openai) require_any OpenAI OPENAI_API_KEY ;;
|
||||
opencode-go) require_any OpenCode OPENCODE_API_KEY OPENCODE_ZEN_API_KEY ;;
|
||||
openrouter) require_any OpenRouter OPENROUTER_API_KEY ;;
|
||||
@@ -1836,7 +1837,7 @@ jobs:
|
||||
run: |
|
||||
set -euo pipefail
|
||||
|
||||
all_providers=(anthropic google minimax openai opencode-go openrouter xai zai fireworks)
|
||||
all_providers=(anthropic google minimax moonshot openai opencode-go openrouter xai zai fireworks)
|
||||
|
||||
normalize_provider() {
|
||||
local value="${1,,}"
|
||||
@@ -1922,6 +1923,7 @@ jobs:
|
||||
anthropic) require_any Anthropic ANTHROPIC_API_KEY ANTHROPIC_API_KEY_OLD ANTHROPIC_API_TOKEN ;;
|
||||
google) require_any Google GEMINI_API_KEY GOOGLE_API_KEY ;;
|
||||
minimax) require_any MiniMax MINIMAX_API_KEY ;;
|
||||
moonshot) require_any Moonshot MOONSHOT_API_KEY KIMI_API_KEY ;;
|
||||
openai) require_any OpenAI OPENAI_API_KEY ;;
|
||||
opencode-go) require_any OpenCode OPENCODE_API_KEY OPENCODE_ZEN_API_KEY ;;
|
||||
openrouter) require_any OpenRouter OPENROUTER_API_KEY ;;
|
||||
|
||||
214
.github/workflows/openclaw-release-publish.yml
vendored
214
.github/workflows/openclaw-release-publish.yml
vendored
@@ -387,7 +387,9 @@ jobs:
|
||||
run: |
|
||||
set -euo pipefail
|
||||
|
||||
dispatch_workflow() {
|
||||
dispatch_workflow_at_ref() {
|
||||
local workflow_ref="$1"
|
||||
shift
|
||||
local workflow="$1"
|
||||
shift
|
||||
|
||||
@@ -397,7 +399,7 @@ jobs:
|
||||
-F per_page=100 \
|
||||
--jq '[.workflow_runs[].id]')"
|
||||
|
||||
dispatch_output="$(gh workflow run --repo "$GITHUB_REPOSITORY" "$workflow" --ref "$CHILD_WORKFLOW_REF" "$@" 2>&1)"
|
||||
dispatch_output="$(gh workflow run --repo "$GITHUB_REPOSITORY" "$workflow" --ref "$workflow_ref" "$@" 2>&1)"
|
||||
printf '%s\n' "$dispatch_output" >&2
|
||||
run_id="$(
|
||||
printf '%s\n' "$dispatch_output" |
|
||||
@@ -432,6 +434,10 @@ jobs:
|
||||
printf '%s\n' "${run_id}"
|
||||
}
|
||||
|
||||
dispatch_workflow() {
|
||||
dispatch_workflow_at_ref "$CHILD_WORKFLOW_REF" "$@"
|
||||
}
|
||||
|
||||
print_pending_deployments() {
|
||||
local workflow="$1"
|
||||
local run_id="$2"
|
||||
@@ -710,6 +716,71 @@ jobs:
|
||||
exit 1
|
||||
}
|
||||
|
||||
resolve_clawhub_release_plan() {
|
||||
local -a plan_args
|
||||
|
||||
clawhub_plan_path="${RUNNER_TEMP}/openclaw-release-clawhub-plan.json"
|
||||
plan_args=(
|
||||
--release-tag "${RELEASE_TAG}"
|
||||
--release-publish-branch "${CHILD_WORKFLOW_REF}"
|
||||
--release-publish-run-id "${GITHUB_RUN_ID}"
|
||||
--plugin-publish-scope "${PLUGIN_PUBLISH_SCOPE}"
|
||||
)
|
||||
if [[ -n "${PLUGINS// }" ]]; then
|
||||
plan_args+=(--plugins "${PLUGINS}")
|
||||
fi
|
||||
|
||||
CLAWHUB_REGISTRY="${CLAWHUB_REGISTRY:-https://clawhub.ai}" \
|
||||
node --import tsx scripts/openclaw-release-clawhub-plan.ts "${plan_args[@]}" > "${clawhub_plan_path}"
|
||||
|
||||
echo "Resolved OpenClaw release ClawHub dispatch plan:"
|
||||
cat "${clawhub_plan_path}"
|
||||
|
||||
clawhub_workflow_ref="$(jq -r '.clawHubWorkflowRef' "${clawhub_plan_path}")"
|
||||
normal_plugins="$(jq -r '.summary.normalPlugins' "${clawhub_plan_path}")"
|
||||
bootstrap_plugins="$(jq -r '.summary.bootstrapPlugins' "${clawhub_plan_path}")"
|
||||
missing_trusted_plugins="$(jq -r '.summary.missingTrustedPlugins' "${clawhub_plan_path}")"
|
||||
normal_plugin_count="$(jq -r '.summary.normalCount' "${clawhub_plan_path}")"
|
||||
bootstrap_plugin_count="$(jq -r '.summary.bootstrapCount' "${clawhub_plan_path}")"
|
||||
missing_trusted_plugin_count="$(jq -r '.summary.missingTrustedPublisherCount' "${clawhub_plan_path}")"
|
||||
|
||||
{
|
||||
echo "### ClawHub release plan"
|
||||
echo
|
||||
echo "- Normal OIDC candidates: \`${normal_plugin_count}\`"
|
||||
echo "- Bootstrap/repair candidates: \`${bootstrap_plugin_count}\`"
|
||||
echo "- Existing-package trusted-publisher repairs: \`${missing_trusted_plugin_count}\`"
|
||||
if [[ -n "${normal_plugins}" ]]; then
|
||||
echo "- Normal plugins: \`${normal_plugins}\`"
|
||||
fi
|
||||
if [[ -n "${bootstrap_plugins}" ]]; then
|
||||
echo "- Bootstrap/repair plugins: \`${bootstrap_plugins}\`"
|
||||
fi
|
||||
if [[ -n "${missing_trusted_plugins}" ]]; then
|
||||
echo "- Trusted-publisher repair plugins: \`${missing_trusted_plugins}\`"
|
||||
fi
|
||||
} >> "$GITHUB_STEP_SUMMARY"
|
||||
}
|
||||
|
||||
append_clawhub_dispatch_args() {
|
||||
local target="$1"
|
||||
while IFS=$'\t' read -r key value; do
|
||||
clawhub_dispatch_args+=(-f "${key}=${value}")
|
||||
done < <(jq -r --arg target "${target}" '.[$target].inputs | to_entries[] | [.key, .value] | @tsv' "${clawhub_plan_path}")
|
||||
}
|
||||
|
||||
write_clawhub_runtime_state() {
|
||||
local force_skip_clawhub="$1"
|
||||
local output_path="$2"
|
||||
node --import tsx scripts/openclaw-release-clawhub-runtime-state.ts \
|
||||
--repository "${GITHUB_REPOSITORY}" \
|
||||
--wait-for-clawhub "${WAIT_FOR_CLAWHUB}" \
|
||||
--force-skip-clawhub "${force_skip_clawhub}" \
|
||||
--normal-run-id "${plugin_clawhub_run_id:-}" \
|
||||
--bootstrap-run-id "${plugin_clawhub_bootstrap_run_id:-}" \
|
||||
--bootstrap-completed "${plugin_clawhub_bootstrap_completed:-false}" > "${output_path}"
|
||||
}
|
||||
|
||||
create_or_update_github_release() {
|
||||
local release_version notes_version title notes_file changelog_file latest_arg prerelease_args
|
||||
release_version="${RELEASE_TAG#v}"
|
||||
@@ -798,7 +869,7 @@ jobs:
|
||||
}
|
||||
|
||||
verify_published_release() {
|
||||
local release_version evidence_path skip_clawhub
|
||||
local release_version evidence_path skip_clawhub clawhub_runtime_state_path
|
||||
local -a verify_args
|
||||
|
||||
skip_clawhub="${1:-false}"
|
||||
@@ -815,17 +886,18 @@ jobs:
|
||||
--dist-tag "${RELEASE_NPM_DIST_TAG}"
|
||||
--repo "${GITHUB_REPOSITORY}"
|
||||
--workflow-ref "${CHILD_WORKFLOW_REF}"
|
||||
--clawhub-workflow-ref "${clawhub_workflow_ref}"
|
||||
--full-release-validation-run "${FULL_RELEASE_VALIDATION_RUN_ID}"
|
||||
--plugin-npm-run "${plugin_npm_run_id}"
|
||||
--openclaw-npm-run "${openclaw_npm_run_id}"
|
||||
--evidence-out "${evidence_path}"
|
||||
--skip-github-release
|
||||
)
|
||||
if [[ "${skip_clawhub}" == "true" || "${WAIT_FOR_CLAWHUB}" != "true" ]]; then
|
||||
verify_args+=(--skip-clawhub)
|
||||
else
|
||||
verify_args+=(--plugin-clawhub-run "${plugin_clawhub_run_id}")
|
||||
fi
|
||||
clawhub_runtime_state_path="${RUNNER_TEMP}/openclaw-release-clawhub-runtime-state-verify.json"
|
||||
write_clawhub_runtime_state "${skip_clawhub}" "${clawhub_runtime_state_path}"
|
||||
while IFS= read -r arg; do
|
||||
verify_args+=("${arg}")
|
||||
done < <(jq -r '.verifierArgs[]' "${clawhub_runtime_state_path}")
|
||||
if [[ -n "${PLUGINS// }" ]]; then
|
||||
verify_args+=(--plugins "${PLUGINS}")
|
||||
fi
|
||||
@@ -841,7 +913,7 @@ jobs:
|
||||
}
|
||||
|
||||
append_release_proof_to_github_release() {
|
||||
local release_version body_file notes_file tarball integrity telegram_line clawhub_line
|
||||
local release_version body_file notes_file tarball integrity telegram_line clawhub_line clawhub_bootstrap_line clawhub_runtime_state_path
|
||||
|
||||
release_version="${RELEASE_TAG#v}"
|
||||
body_file="${RUNNER_TEMP}/release-body.md"
|
||||
@@ -855,11 +927,10 @@ jobs:
|
||||
else
|
||||
telegram_line="- npm Telegram beta E2E: not supplied"
|
||||
fi
|
||||
if [[ "${WAIT_FOR_CLAWHUB}" == "true" ]]; then
|
||||
clawhub_line="- plugin ClawHub publish: https://github.com/${GITHUB_REPOSITORY}/actions/runs/${plugin_clawhub_run_id}"
|
||||
else
|
||||
clawhub_line="- plugin ClawHub publish: dispatched separately, not awaited by this proof: https://github.com/${GITHUB_REPOSITORY}/actions/runs/${plugin_clawhub_run_id}"
|
||||
fi
|
||||
clawhub_runtime_state_path="${RUNNER_TEMP}/openclaw-release-clawhub-runtime-state-proof.json"
|
||||
write_clawhub_runtime_state false "${clawhub_runtime_state_path}"
|
||||
clawhub_line="$(jq -r '.proofLines.normal' "${clawhub_runtime_state_path}")"
|
||||
clawhub_bootstrap_line="$(jq -r '.proofLines.bootstrap' "${clawhub_runtime_state_path}")"
|
||||
|
||||
RELEASE_BODY_FILE="${body_file}" \
|
||||
RELEASE_NOTES_FILE="${notes_file}" \
|
||||
@@ -875,6 +946,7 @@ jobs:
|
||||
PLUGIN_NPM_RUN_ID="${plugin_npm_run_id}" \
|
||||
OPENCLAW_NPM_RUN_ID="${openclaw_npm_run_id}" \
|
||||
CLAWHUB_LINE="${clawhub_line}" \
|
||||
CLAWHUB_BOOTSTRAP_LINE="${clawhub_bootstrap_line}" \
|
||||
TELEGRAM_LINE="${telegram_line}" \
|
||||
node --input-type=module <<'NODE'
|
||||
import { readFileSync, writeFileSync } from "node:fs";
|
||||
@@ -899,6 +971,7 @@ jobs:
|
||||
`- full release validation: https://github.com/${process.env.RELEASE_REPO}/actions/runs/${process.env.FULL_RELEASE_VALIDATION_RUN_ID}`,
|
||||
`- plugin npm publish: https://github.com/${process.env.RELEASE_REPO}/actions/runs/${process.env.PLUGIN_NPM_RUN_ID}`,
|
||||
process.env.CLAWHUB_LINE,
|
||||
process.env.CLAWHUB_BOOTSTRAP_LINE,
|
||||
`- OpenClaw npm publish: https://github.com/${process.env.RELEASE_REPO}/actions/runs/${process.env.OPENCLAW_NPM_RUN_ID}`,
|
||||
process.env.TELEGRAM_LINE,
|
||||
].join("\n");
|
||||
@@ -915,6 +988,7 @@ jobs:
|
||||
echo "### Publish sequence"
|
||||
echo
|
||||
echo "- Workflow ref: \`${CHILD_WORKFLOW_REF}\`"
|
||||
echo "- ClawHub workflow ref: release tag \`${RELEASE_TAG}\`"
|
||||
echo "- Release tag: \`${RELEASE_TAG}\`"
|
||||
echo "- Release SHA: \`${TARGET_SHA}\`"
|
||||
echo "- Release approval: this workflow job"
|
||||
@@ -933,27 +1007,66 @@ jobs:
|
||||
|
||||
guard_existing_public_release
|
||||
guard_openclaw_npm_not_already_published
|
||||
resolve_clawhub_release_plan
|
||||
|
||||
npm_args=(-f publish_scope="${PLUGIN_PUBLISH_SCOPE}" -f ref="${TARGET_SHA}" -f release_publish_run_id="${GITHUB_RUN_ID}")
|
||||
clawhub_args=(-f publish_scope="${PLUGIN_PUBLISH_SCOPE}" -f ref="${TARGET_SHA}" -f release_publish_run_id="${GITHUB_RUN_ID}")
|
||||
if [[ -n "${PLUGINS}" ]]; then
|
||||
npm_args+=(-f plugins="${PLUGINS}")
|
||||
clawhub_args+=(-f plugins="${PLUGINS}")
|
||||
fi
|
||||
|
||||
plugin_npm_run_id="$(dispatch_workflow plugin-npm-release.yml "${npm_args[@]}")"
|
||||
plugin_clawhub_run_id="$(dispatch_workflow plugin-clawhub-release.yml "${clawhub_args[@]}")"
|
||||
plugin_clawhub_run_id=""
|
||||
if [[ "$(jq -r '.normal.shouldDispatch' "${clawhub_plan_path}")" == "true" ]]; then
|
||||
clawhub_dispatch_args=()
|
||||
append_clawhub_dispatch_args normal
|
||||
plugin_clawhub_run_id="$(dispatch_workflow_at_ref \
|
||||
"$(jq -r '.normal.ref' "${clawhub_plan_path}")" \
|
||||
"$(jq -r '.normal.workflow' "${clawhub_plan_path}")" \
|
||||
"${clawhub_dispatch_args[@]}")"
|
||||
else
|
||||
echo "- plugin-clawhub-release.yml: no normal OIDC candidates" >> "$GITHUB_STEP_SUMMARY"
|
||||
fi
|
||||
plugin_clawhub_bootstrap_run_id=""
|
||||
plugin_clawhub_bootstrap_completed="false"
|
||||
if [[ "$(jq -r '.bootstrap.shouldDispatch' "${clawhub_plan_path}")" == "true" ]]; then
|
||||
clawhub_dispatch_args=()
|
||||
append_clawhub_dispatch_args bootstrap
|
||||
plugin_clawhub_bootstrap_run_id="$(dispatch_workflow_at_ref \
|
||||
"$(jq -r '.bootstrap.ref' "${clawhub_plan_path}")" \
|
||||
"$(jq -r '.bootstrap.workflow' "${clawhub_plan_path}")" \
|
||||
"${clawhub_dispatch_args[@]}")"
|
||||
else
|
||||
echo "- plugin-clawhub-new.yml: no bootstrap candidates" >> "$GITHUB_STEP_SUMMARY"
|
||||
fi
|
||||
{
|
||||
echo "- Plugin npm run ID: \`${plugin_npm_run_id}\`"
|
||||
echo "- Plugin ClawHub run ID: \`${plugin_clawhub_run_id}\`"
|
||||
echo "- Plugin ClawHub run ID: \`${plugin_clawhub_run_id:-none}\`"
|
||||
echo "- Plugin ClawHub bootstrap run ID: \`${plugin_clawhub_bootstrap_run_id:-none}\`"
|
||||
} >> "$GITHUB_STEP_SUMMARY"
|
||||
|
||||
if ! wait_for_run plugin-npm-release.yml "${plugin_npm_run_id}"; then
|
||||
echo "Plugin npm publish failed; cancelling ClawHub publish child ${plugin_clawhub_run_id}." >&2
|
||||
gh run cancel --repo "$GITHUB_REPOSITORY" "${plugin_clawhub_run_id}" >/dev/null 2>&1 || true
|
||||
echo "Plugin npm publish failed; cancelling dispatched ClawHub child workflows." >&2
|
||||
if [[ -n "${plugin_clawhub_run_id}" ]]; then
|
||||
gh run cancel --repo "$GITHUB_REPOSITORY" "${plugin_clawhub_run_id}" >/dev/null 2>&1 || true
|
||||
fi
|
||||
if [[ -n "${plugin_clawhub_bootstrap_run_id}" ]]; then
|
||||
gh run cancel --repo "$GITHUB_REPOSITORY" "${plugin_clawhub_bootstrap_run_id}" >/dev/null 2>&1 || true
|
||||
fi
|
||||
exit 1
|
||||
fi
|
||||
|
||||
if [[ -n "${plugin_clawhub_bootstrap_run_id}" && "${WAIT_FOR_CLAWHUB}" == "true" ]]; then
|
||||
echo "Waiting for plugin-clawhub-new.yml bootstrap to finish before continuing release publish."
|
||||
if wait_for_run plugin-clawhub-new.yml "${plugin_clawhub_bootstrap_run_id}"; then
|
||||
plugin_clawhub_bootstrap_completed="true"
|
||||
else
|
||||
if [[ -n "${plugin_clawhub_run_id}" ]]; then
|
||||
gh run cancel --repo "$GITHUB_REPOSITORY" "${plugin_clawhub_run_id}" >/dev/null 2>&1 || true
|
||||
fi
|
||||
exit 1
|
||||
fi
|
||||
fi
|
||||
|
||||
openclaw_npm_run_id=""
|
||||
if [[ "${PUBLISH_OPENCLAW_NPM}" == "true" ]]; then
|
||||
openclaw_npm_run_id="$(dispatch_workflow openclaw-npm-release.yml \
|
||||
@@ -970,19 +1083,52 @@ jobs:
|
||||
|
||||
clawhub_result=""
|
||||
clawhub_pid=""
|
||||
clawhub_bootstrap_result=""
|
||||
clawhub_bootstrap_pid=""
|
||||
if [[ "${WAIT_FOR_CLAWHUB}" == "true" ]]; then
|
||||
clawhub_result="$RUNNER_TEMP/clawhub-result.txt"
|
||||
wait_run_pid=""
|
||||
wait_for_run_background plugin-clawhub-release.yml "${plugin_clawhub_run_id}" "${clawhub_result}"
|
||||
clawhub_pid="${wait_run_pid}"
|
||||
else
|
||||
wait_for_job_success plugin-clawhub-release.yml "${plugin_clawhub_run_id}" "Validate release publish approval"
|
||||
if approve_child_publish_environment plugin-clawhub-release.yml "${plugin_clawhub_run_id}"; then
|
||||
:
|
||||
else
|
||||
echo "- plugin-clawhub-release.yml: child environment gate not ready; publish was left dispatched (${plugin_clawhub_run_id})" >> "$GITHUB_STEP_SUMMARY"
|
||||
if [[ -n "${plugin_clawhub_run_id}" ]]; then
|
||||
clawhub_result="$RUNNER_TEMP/clawhub-result.txt"
|
||||
wait_run_pid=""
|
||||
wait_for_run_background plugin-clawhub-release.yml "${plugin_clawhub_run_id}" "${clawhub_result}"
|
||||
clawhub_pid="${wait_run_pid}"
|
||||
fi
|
||||
if [[ -n "${plugin_clawhub_bootstrap_run_id}" ]]; then
|
||||
if [[ "${plugin_clawhub_bootstrap_completed}" == "true" ]]; then
|
||||
echo "- plugin-clawhub-new.yml: bootstrap already completed before continuing" >> "$GITHUB_STEP_SUMMARY"
|
||||
else
|
||||
clawhub_bootstrap_result="$RUNNER_TEMP/clawhub-bootstrap-result.txt"
|
||||
wait_run_pid=""
|
||||
wait_for_run_background plugin-clawhub-new.yml "${plugin_clawhub_bootstrap_run_id}" "${clawhub_bootstrap_result}"
|
||||
clawhub_bootstrap_pid="${wait_run_pid}"
|
||||
fi
|
||||
fi
|
||||
else
|
||||
if [[ -n "${plugin_clawhub_run_id}" ]]; then
|
||||
wait_for_job_success plugin-clawhub-release.yml "${plugin_clawhub_run_id}" "Validate release publish approval"
|
||||
if approve_child_publish_environment plugin-clawhub-release.yml "${plugin_clawhub_run_id}"; then
|
||||
:
|
||||
else
|
||||
echo "- plugin-clawhub-release.yml: child environment gate not ready; publish was left dispatched (${plugin_clawhub_run_id})" >> "$GITHUB_STEP_SUMMARY"
|
||||
fi
|
||||
echo "- plugin-clawhub-release.yml: publish not awaited (${plugin_clawhub_run_id})" >> "$GITHUB_STEP_SUMMARY"
|
||||
else
|
||||
echo "- plugin-clawhub-release.yml: no normal OIDC publish to await" >> "$GITHUB_STEP_SUMMARY"
|
||||
fi
|
||||
if [[ -n "${plugin_clawhub_bootstrap_run_id}" ]]; then
|
||||
if [[ "${plugin_clawhub_bootstrap_completed}" == "true" ]]; then
|
||||
echo "- plugin-clawhub-new.yml: bootstrap already completed before continuing" >> "$GITHUB_STEP_SUMMARY"
|
||||
else
|
||||
wait_for_job_success plugin-clawhub-new.yml "${plugin_clawhub_bootstrap_run_id}" "Validate release publish approval"
|
||||
if approve_child_publish_environment plugin-clawhub-new.yml "${plugin_clawhub_bootstrap_run_id}"; then
|
||||
:
|
||||
else
|
||||
echo "- plugin-clawhub-new.yml: child environment gate not ready; bootstrap was left dispatched (${plugin_clawhub_bootstrap_run_id})" >> "$GITHUB_STEP_SUMMARY"
|
||||
fi
|
||||
echo "- plugin-clawhub-new.yml: bootstrap not awaited (${plugin_clawhub_bootstrap_run_id})" >> "$GITHUB_STEP_SUMMARY"
|
||||
fi
|
||||
else
|
||||
echo "- plugin-clawhub-new.yml: no bootstrap publish to await" >> "$GITHUB_STEP_SUMMARY"
|
||||
fi
|
||||
echo "- plugin-clawhub-release.yml: publish not awaited (${plugin_clawhub_run_id})" >> "$GITHUB_STEP_SUMMARY"
|
||||
fi
|
||||
|
||||
openclaw_result=""
|
||||
@@ -1011,6 +1157,12 @@ jobs:
|
||||
if [[ -f "${clawhub_result}" && "$(cat "${clawhub_result}")" != "success" ]]; then
|
||||
failed=1
|
||||
fi
|
||||
if [[ -n "${clawhub_bootstrap_pid}" ]] && ! wait "${clawhub_bootstrap_pid}"; then
|
||||
failed=1
|
||||
fi
|
||||
if [[ -f "${clawhub_bootstrap_result}" && "$(cat "${clawhub_bootstrap_result}")" != "success" ]]; then
|
||||
failed=1
|
||||
fi
|
||||
|
||||
if [[ -n "${openclaw_npm_run_id}" && "${openclaw_failed}" == "0" ]]; then
|
||||
if [[ "${failed}" == "0" ]]; then
|
||||
|
||||
504
.github/workflows/plugin-clawhub-new.yml
vendored
Normal file
504
.github/workflows/plugin-clawhub-new.yml
vendored
Normal file
@@ -0,0 +1,504 @@
|
||||
name: Plugin ClawHub New
|
||||
|
||||
on:
|
||||
workflow_dispatch:
|
||||
inputs:
|
||||
plugins:
|
||||
description: Comma-separated plugin package names to bootstrap on ClawHub
|
||||
required: true
|
||||
type: string
|
||||
ref:
|
||||
description: Commit SHA on main, a release branch, or the matching Tideclaw alpha branch to publish from; defaults to the workflow ref
|
||||
required: false
|
||||
default: ""
|
||||
type: string
|
||||
release_publish_run_id:
|
||||
description: Approved OpenClaw Release Publish workflow run id
|
||||
required: false
|
||||
type: string
|
||||
release_publish_branch:
|
||||
description: Branch name of the approving OpenClaw Release Publish workflow run
|
||||
required: false
|
||||
type: string
|
||||
dry_run:
|
||||
description: Validate the token-gated ClawHub bootstrap handoff without publishing.
|
||||
required: false
|
||||
default: false
|
||||
type: boolean
|
||||
|
||||
concurrency:
|
||||
group: plugin-clawhub-new-${{ github.event_name == 'workflow_dispatch' && inputs.ref || github.sha }}
|
||||
cancel-in-progress: false
|
||||
|
||||
env:
|
||||
FORCE_JAVASCRIPT_ACTIONS_TO_NODE24: "true"
|
||||
NODE_VERSION: "24.15.0"
|
||||
CLAWHUB_REGISTRY: "https://clawhub.ai"
|
||||
CLAWHUB_CLI_PACKAGE: "clawhub@0.21.0"
|
||||
|
||||
jobs:
|
||||
resolve_bootstrap_plan:
|
||||
runs-on: ubuntu-latest
|
||||
permissions:
|
||||
contents: read
|
||||
outputs:
|
||||
ref_revision: ${{ steps.ref.outputs.sha }}
|
||||
has_bootstrap_candidates: ${{ steps.plan.outputs.has_bootstrap_candidates }}
|
||||
bootstrap_candidate_count: ${{ steps.plan.outputs.bootstrap_candidate_count }}
|
||||
matrix: ${{ steps.plan.outputs.matrix }}
|
||||
steps:
|
||||
- name: Checkout
|
||||
uses: actions/checkout@v6
|
||||
with:
|
||||
persist-credentials: false
|
||||
ref: ${{ github.ref }}
|
||||
fetch-depth: 0
|
||||
|
||||
- name: Resolve checked-out ref
|
||||
id: ref
|
||||
env:
|
||||
TARGET_REF: ${{ github.event_name == 'workflow_dispatch' && inputs.ref || '' }}
|
||||
run: |
|
||||
set -euo pipefail
|
||||
git fetch --no-tags origin \
|
||||
+refs/heads/main:refs/remotes/origin/main \
|
||||
'+refs/heads/release/*:refs/remotes/origin/release/*'
|
||||
if [[ -n "${TARGET_REF}" ]]; then
|
||||
if git rev-parse --verify --quiet "${TARGET_REF}^{commit}" >/dev/null; then
|
||||
target_sha="$(git rev-parse "${TARGET_REF}^{commit}")"
|
||||
elif git rev-parse --verify --quiet "origin/${TARGET_REF}^{commit}" >/dev/null; then
|
||||
target_sha="$(git rev-parse "origin/${TARGET_REF}^{commit}")"
|
||||
else
|
||||
echo "Unable to resolve requested publish ref: ${TARGET_REF}" >&2
|
||||
exit 1
|
||||
fi
|
||||
git checkout --detach "${target_sha}"
|
||||
fi
|
||||
echo "sha=$(git rev-parse HEAD)" >> "$GITHUB_OUTPUT"
|
||||
|
||||
- name: Validate ref is on a trusted publish branch
|
||||
env:
|
||||
TRUSTED_PUBLISH_BRANCH: ${{ inputs.release_publish_branch || github.ref_name }}
|
||||
run: |
|
||||
set -euo pipefail
|
||||
if git merge-base --is-ancestor HEAD origin/main; then
|
||||
exit 0
|
||||
fi
|
||||
while IFS= read -r release_ref; do
|
||||
if git merge-base --is-ancestor HEAD "${release_ref}"; then
|
||||
exit 0
|
||||
fi
|
||||
done < <(git for-each-ref --format='%(refname)' refs/remotes/origin/release)
|
||||
if [[ "${TRUSTED_PUBLISH_BRANCH}" =~ ^tideclaw/alpha/[0-9]{4}-[0-9]{2}-[0-9]{2}-[0-9]{4}Z$ ]]; then
|
||||
alpha_branch="${TRUSTED_PUBLISH_BRANCH}"
|
||||
git fetch --no-tags origin "+refs/heads/${alpha_branch}:refs/remotes/origin/${alpha_branch}"
|
||||
if git merge-base --is-ancestor HEAD "refs/remotes/origin/${alpha_branch}"; then
|
||||
exit 0
|
||||
fi
|
||||
fi
|
||||
echo "Plugin ClawHub bootstraps must target a commit reachable from main, release/*, or the matching Tideclaw alpha branch." >&2
|
||||
exit 1
|
||||
|
||||
- name: Setup Node environment
|
||||
uses: ./.github/actions/setup-node-env
|
||||
with:
|
||||
node-version: ${{ env.NODE_VERSION }}
|
||||
install-bun: "false"
|
||||
|
||||
- name: Validate publishable plugin metadata
|
||||
env:
|
||||
RELEASE_PLUGINS: ${{ inputs.plugins }}
|
||||
run: |
|
||||
set -euo pipefail
|
||||
if [[ -z "${RELEASE_PLUGINS// }" ]]; then
|
||||
echo "Plugin ClawHub bootstrap requires at least one package name in plugins." >&2
|
||||
exit 1
|
||||
fi
|
||||
pnpm release:plugins:clawhub:check -- --selection-mode selected --plugins "${RELEASE_PLUGINS}"
|
||||
|
||||
- name: Resolve plugin bootstrap plan
|
||||
id: plan
|
||||
env:
|
||||
RELEASE_PLUGINS: ${{ inputs.plugins }}
|
||||
CLAWHUB_REGISTRY: ${{ env.CLAWHUB_REGISTRY }}
|
||||
run: |
|
||||
set -euo pipefail
|
||||
mkdir -p .local
|
||||
node --import tsx scripts/plugin-clawhub-release-plan.ts \
|
||||
--selection-mode selected \
|
||||
--plugins "${RELEASE_PLUGINS}" > .local/plugin-clawhub-release-plan.json
|
||||
|
||||
cat .local/plugin-clawhub-release-plan.json
|
||||
|
||||
bootstrap_candidate_count="$(jq -r '(.bootstrapCandidates | length) + (.missingTrustedPublisher | length)' .local/plugin-clawhub-release-plan.json)"
|
||||
selected_count="$(jq -r '.all | length' .local/plugin-clawhub-release-plan.json)"
|
||||
matrix_json="$(
|
||||
jq -c '
|
||||
[
|
||||
.bootstrapCandidates[]? + {
|
||||
bootstrapMode: "publish",
|
||||
requiresManualOverride: false
|
||||
},
|
||||
.missingTrustedPublisher[]? + {
|
||||
bootstrapMode: (if .alreadyPublished then "configure-only" else "publish" end),
|
||||
requiresManualOverride: true
|
||||
}
|
||||
]
|
||||
' .local/plugin-clawhub-release-plan.json
|
||||
)"
|
||||
has_bootstrap_candidates="false"
|
||||
if [[ "${bootstrap_candidate_count}" != "0" ]]; then
|
||||
has_bootstrap_candidates="true"
|
||||
fi
|
||||
|
||||
invalid_scope="$(
|
||||
jq -r '
|
||||
(.bootstrapCandidates[]?, .missingTrustedPublisher[]?)
|
||||
| select(.packageName | startswith("@openclaw/") | not)
|
||||
| "- \(.packageName)@\(.version)"
|
||||
' .local/plugin-clawhub-release-plan.json
|
||||
)"
|
||||
if [[ -n "${invalid_scope}" ]]; then
|
||||
echo "Plugin ClawHub bootstrap only supports @openclaw/* packages." >&2
|
||||
printf '%s\n' "${invalid_scope}" >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
not_bootstrap="$(
|
||||
jq -r '
|
||||
(.bootstrapCandidates | map(.packageName)) as $bootstrapNames
|
||||
| (.missingTrustedPublisher | map(.packageName)) as $repairNames
|
||||
| .all[]?
|
||||
| select(.packageName as $name | ($bootstrapNames + $repairNames | index($name) | not))
|
||||
| "- \(.packageName)@\(.version)"
|
||||
' .local/plugin-clawhub-release-plan.json
|
||||
)"
|
||||
if [[ -n "${not_bootstrap}" ]]; then
|
||||
echo "Selected packages must all be first-publish bootstrap candidates or trusted-publisher repair candidates." >&2
|
||||
printf '%s\n' "${not_bootstrap}" >&2
|
||||
exit 1
|
||||
fi
|
||||
if [[ "${selected_count}" == "0" || "${bootstrap_candidate_count}" == "0" ]]; then
|
||||
echo "No selected packages require ClawHub bootstrap." >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
{
|
||||
echo "bootstrap_candidate_count=${bootstrap_candidate_count}"
|
||||
echo "has_bootstrap_candidates=${has_bootstrap_candidates}"
|
||||
echo "matrix=${matrix_json}"
|
||||
} >> "$GITHUB_OUTPUT"
|
||||
|
||||
echo "ClawHub bootstrap candidates:"
|
||||
jq -r '
|
||||
.bootstrapCandidates[]? | "- \(.packageName)@\(.version) [\(.publishTag)] from \(.packageDir)"
|
||||
' .local/plugin-clawhub-release-plan.json
|
||||
echo "ClawHub trusted-publisher repair candidates:"
|
||||
jq -r '
|
||||
.missingTrustedPublisher[]? | "- \(.packageName)@\(.version) [\(.publishTag)] from \(.packageDir), alreadyPublished=\(.alreadyPublished)"
|
||||
' .local/plugin-clawhub-release-plan.json
|
||||
|
||||
- name: Validate Tideclaw alpha plugin channels
|
||||
env:
|
||||
TRUSTED_PUBLISH_BRANCH: ${{ inputs.release_publish_branch || github.ref_name }}
|
||||
run: |
|
||||
set -euo pipefail
|
||||
if [[ ! "${TRUSTED_PUBLISH_BRANCH}" =~ ^tideclaw/alpha/[0-9]{4}-[0-9]{2}-[0-9]{2}-[0-9]{4}Z$ ]]; then
|
||||
exit 0
|
||||
fi
|
||||
invalid="$(
|
||||
jq -r '
|
||||
(.bootstrapCandidates[]?, .missingTrustedPublisher[]?)
|
||||
| select(.publishTag != "alpha" or .channel != "alpha")
|
||||
| "- \(.packageName)@\(.version) [\(.publishTag)]"
|
||||
' .local/plugin-clawhub-release-plan.json
|
||||
)"
|
||||
if [[ -n "${invalid}" ]]; then
|
||||
echo "Tideclaw alpha ClawHub bootstraps may only publish alpha plugin versions." >&2
|
||||
printf '%s\n' "${invalid}" >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
validate_release_publish_approval:
|
||||
name: Validate release publish approval
|
||||
needs: resolve_bootstrap_plan
|
||||
if: github.event_name == 'workflow_dispatch' && needs.resolve_bootstrap_plan.outputs.has_bootstrap_candidates == 'true'
|
||||
runs-on: ubuntu-latest
|
||||
permissions:
|
||||
actions: read
|
||||
contents: read
|
||||
steps:
|
||||
- name: Checkout
|
||||
uses: actions/checkout@v6
|
||||
with:
|
||||
persist-credentials: false
|
||||
|
||||
- name: Validate release publish approval run
|
||||
env:
|
||||
GH_TOKEN: ${{ github.token }}
|
||||
RELEASE_PUBLISH_RUN_ID: ${{ inputs.release_publish_run_id }}
|
||||
EXPECTED_WORKFLOW_BRANCH: ${{ inputs.release_publish_branch || github.ref_name }}
|
||||
run: |
|
||||
set -euo pipefail
|
||||
if [[ -z "${RELEASE_PUBLISH_RUN_ID// }" ]]; then
|
||||
if [[ "${GITHUB_ACTOR}" == "github-actions[bot]" ]]; then
|
||||
echo "Plugin ClawHub bootstrap dispatched by another workflow must include release_publish_run_id." >&2
|
||||
exit 1
|
||||
fi
|
||||
echo "Direct Plugin ClawHub New dispatch; relying on this workflow's clawhub-plugin-bootstrap environment approval."
|
||||
exit 0
|
||||
fi
|
||||
direct_recovery=false
|
||||
if [[ "${GITHUB_ACTOR}" != "github-actions[bot]" ]]; then
|
||||
direct_recovery=true
|
||||
echo "Direct Plugin ClawHub New recovery with release_publish_run_id; relying on this workflow's clawhub-plugin-bootstrap environment approval."
|
||||
fi
|
||||
RUN_JSON="$(gh run view "$RELEASE_PUBLISH_RUN_ID" --repo "$GITHUB_REPOSITORY" --json workflowName,headBranch,event,status,conclusion,url)"
|
||||
printf '%s' "$RUN_JSON" | DIRECT_RELEASE_RECOVERY="${direct_recovery}" node scripts/validate-release-publish-approval.mjs
|
||||
|
||||
validate_bootstrap_trusted_publisher_cli:
|
||||
needs: [resolve_bootstrap_plan, validate_release_publish_approval]
|
||||
if: always() && github.event_name == 'workflow_dispatch' && inputs.dry_run != true && needs.resolve_bootstrap_plan.outputs.has_bootstrap_candidates == 'true' && needs.validate_release_publish_approval.result == 'success'
|
||||
runs-on: ubuntu-latest
|
||||
permissions:
|
||||
contents: read
|
||||
steps:
|
||||
- name: Validate pinned ClawHub trusted publisher CLI support
|
||||
env:
|
||||
CLAWHUB_CLI_PACKAGE: ${{ env.CLAWHUB_CLI_PACKAGE }}
|
||||
run: |
|
||||
set -euo pipefail
|
||||
help_output="$(
|
||||
npm exec --yes --package "${CLAWHUB_CLI_PACKAGE}" -- \
|
||||
clawhub package trusted-publisher set --help 2>&1 || true
|
||||
)"
|
||||
printf '%s\n' "${help_output}"
|
||||
if ! grep -Fq "Usage: clawhub package trusted-publisher set" <<<"${help_output}"; then
|
||||
echo "::error::CLAW-277 03 - Split OpenClaw plugin ClawHub publishing into OIDC release and token bootstrap workflows requires ${CLAWHUB_CLI_PACKAGE} to expose 'package trusted-publisher set' before token bootstrap publish can run. The pinned CLI returned parent help or no set command, so this workflow is stopping before creating a ClawHub package row."
|
||||
exit 1
|
||||
fi
|
||||
for required_flag in --repository --workflow-filename; do
|
||||
if ! grep -Fq -- "${required_flag}" <<<"${help_output}"; then
|
||||
echo "::error::CLAW-277 03 - Split OpenClaw plugin ClawHub publishing into OIDC release and token bootstrap workflows requires ${CLAWHUB_CLI_PACKAGE} trusted-publisher set help to include ${required_flag}."
|
||||
exit 1
|
||||
fi
|
||||
done
|
||||
|
||||
publish_bootstrap_plugins:
|
||||
needs:
|
||||
[
|
||||
resolve_bootstrap_plan,
|
||||
validate_release_publish_approval,
|
||||
validate_bootstrap_trusted_publisher_cli,
|
||||
]
|
||||
if: always() && github.event_name == 'workflow_dispatch' && needs.resolve_bootstrap_plan.outputs.has_bootstrap_candidates == 'true' && needs.validate_release_publish_approval.result == 'success' && (inputs.dry_run == true || needs.validate_bootstrap_trusted_publisher_cli.result == 'success')
|
||||
runs-on: ubuntu-latest
|
||||
environment: clawhub-plugin-bootstrap
|
||||
permissions:
|
||||
contents: read
|
||||
strategy:
|
||||
fail-fast: false
|
||||
max-parallel: 8
|
||||
matrix:
|
||||
plugin: ${{ fromJson(needs.resolve_bootstrap_plan.outputs.matrix) }}
|
||||
steps:
|
||||
- name: Checkout
|
||||
uses: actions/checkout@v6
|
||||
with:
|
||||
persist-credentials: false
|
||||
ref: ${{ github.ref }}
|
||||
fetch-depth: 0
|
||||
|
||||
- name: Checkout target revision
|
||||
env:
|
||||
TARGET_SHA: ${{ needs.resolve_bootstrap_plan.outputs.ref_revision }}
|
||||
run: |
|
||||
set -euo pipefail
|
||||
git fetch --no-tags origin \
|
||||
+refs/heads/main:refs/remotes/origin/main \
|
||||
'+refs/heads/release/*:refs/remotes/origin/release/*'
|
||||
git checkout --detach "${TARGET_SHA}"
|
||||
|
||||
- name: Setup Node environment
|
||||
uses: ./.github/actions/setup-node-env
|
||||
with:
|
||||
node-version: ${{ env.NODE_VERSION }}
|
||||
install-bun: "true"
|
||||
install-deps: "true"
|
||||
|
||||
- name: Verify package-local runtime build
|
||||
run: node scripts/check-plugin-npm-runtime-builds.mjs --package "${{ matrix.plugin.packageDir }}"
|
||||
|
||||
- name: Install pinned ClawHub CLI wrapper
|
||||
run: |
|
||||
set -euo pipefail
|
||||
cat > "${RUNNER_TEMP}/clawhub" <<'EOF'
|
||||
#!/usr/bin/env bash
|
||||
set -euo pipefail
|
||||
exec npm exec --yes --package "${CLAWHUB_CLI_PACKAGE}" -- clawhub "$@"
|
||||
EOF
|
||||
chmod +x "${RUNNER_TEMP}/clawhub"
|
||||
echo "${RUNNER_TEMP}" >> "${GITHUB_PATH}"
|
||||
|
||||
- name: Write ClawHub token config
|
||||
if: inputs.dry_run != true
|
||||
env:
|
||||
CLAWHUB_REGISTRY: ${{ env.CLAWHUB_REGISTRY }}
|
||||
CLAWHUB_TOKEN: ${{ secrets.CLAWHUB_TOKEN }}
|
||||
run: |
|
||||
set -euo pipefail
|
||||
config_path="${RUNNER_TEMP}/clawhub-config.json"
|
||||
CONFIG_PATH="${config_path}" node --input-type=module <<'NODE'
|
||||
import { writeFileSync } from "node:fs";
|
||||
|
||||
const registry = process.env.CLAWHUB_REGISTRY?.trim();
|
||||
const token = process.env.CLAWHUB_TOKEN?.trim();
|
||||
const configPath = process.env.CONFIG_PATH;
|
||||
if (!registry) {
|
||||
throw new Error("CLAWHUB_REGISTRY is required for token-gated ClawHub bootstrap.");
|
||||
}
|
||||
if (!token) {
|
||||
throw new Error("CLAWHUB_TOKEN is required for token-gated ClawHub bootstrap.");
|
||||
}
|
||||
if (!configPath) {
|
||||
throw new Error("CONFIG_PATH is required.");
|
||||
}
|
||||
|
||||
writeFileSync(configPath, `${JSON.stringify({ registry, token }, null, 2)}\n`, {
|
||||
encoding: "utf8",
|
||||
mode: 0o600,
|
||||
});
|
||||
NODE
|
||||
echo "CLAWHUB_CONFIG_PATH=${config_path}" >> "${GITHUB_ENV}"
|
||||
|
||||
- name: Publish ClawHub bootstrap package
|
||||
env:
|
||||
CLAWHUB_REGISTRY: ${{ env.CLAWHUB_REGISTRY }}
|
||||
SOURCE_REPO: ${{ github.repository }}
|
||||
SOURCE_COMMIT: ${{ needs.resolve_bootstrap_plan.outputs.ref_revision }}
|
||||
SOURCE_REF: ${{ github.ref }}
|
||||
PACKAGE_TAG: ${{ matrix.plugin.publishTag }}
|
||||
PACKAGE_DIR: ${{ matrix.plugin.packageDir }}
|
||||
BOOTSTRAP_MODE: ${{ matrix.plugin.bootstrapMode }}
|
||||
REQUIRES_MANUAL_OVERRIDE: ${{ matrix.plugin.requiresManualOverride && 'true' || 'false' }}
|
||||
DRY_RUN: ${{ inputs.dry_run && 'true' || 'false' }}
|
||||
OPENCLAW_PLUGIN_NPM_RUNTIME_BUILD: "0"
|
||||
run: |
|
||||
set -euo pipefail
|
||||
if [[ "${BOOTSTRAP_MODE}" == "configure-only" ]]; then
|
||||
echo "Skipping bootstrap publish because ${PACKAGE_DIR} version is already present on ClawHub; configuring trusted publisher only."
|
||||
elif [[ "${DRY_RUN}" == "true" ]]; then
|
||||
bash scripts/plugin-clawhub-publish.sh --dry-run "${PACKAGE_DIR}"
|
||||
else
|
||||
if [[ "${REQUIRES_MANUAL_OVERRIDE}" == "true" ]]; then
|
||||
export OPENCLAW_CLAWHUB_MANUAL_OVERRIDE_REASON="GitHub Actions trusted publisher repair before OIDC migration"
|
||||
fi
|
||||
bash scripts/plugin-clawhub-publish.sh --publish "${PACKAGE_DIR}"
|
||||
fi
|
||||
|
||||
- name: Configure trusted publisher for normal OIDC releases
|
||||
if: inputs.dry_run != true
|
||||
env:
|
||||
CLAWHUB_CLI_PACKAGE: ${{ env.CLAWHUB_CLI_PACKAGE }}
|
||||
PACKAGE_NAME: ${{ matrix.plugin.packageName }}
|
||||
run: |
|
||||
set -euo pipefail
|
||||
npm exec --yes --package "${CLAWHUB_CLI_PACKAGE}" -- \
|
||||
clawhub package trusted-publisher set "${PACKAGE_NAME}" \
|
||||
--repository openclaw/openclaw \
|
||||
--workflow-filename plugin-clawhub-release.yml
|
||||
|
||||
verify_bootstrap_clawhub_package:
|
||||
needs: [resolve_bootstrap_plan, publish_bootstrap_plugins]
|
||||
if: github.event_name == 'workflow_dispatch' && inputs.dry_run != true && needs.resolve_bootstrap_plan.outputs.has_bootstrap_candidates == 'true'
|
||||
runs-on: ubuntu-latest
|
||||
permissions:
|
||||
contents: read
|
||||
strategy:
|
||||
fail-fast: false
|
||||
max-parallel: 8
|
||||
matrix:
|
||||
plugin: ${{ fromJson(needs.resolve_bootstrap_plan.outputs.matrix) }}
|
||||
steps:
|
||||
- name: Verify bootstrap ClawHub package and trusted publisher
|
||||
env:
|
||||
CLAWHUB_REGISTRY: ${{ env.CLAWHUB_REGISTRY }}
|
||||
PACKAGE_NAME: ${{ matrix.plugin.packageName }}
|
||||
PACKAGE_VERSION: ${{ matrix.plugin.version }}
|
||||
PACKAGE_TAG: ${{ matrix.plugin.publishTag }}
|
||||
run: |
|
||||
set -euo pipefail
|
||||
node --input-type=module <<'EOF'
|
||||
const registry = (process.env.CLAWHUB_REGISTRY ?? "https://clawhub.ai").replace(/\/+$/, "");
|
||||
const packageName = process.env.PACKAGE_NAME;
|
||||
const packageVersion = process.env.PACKAGE_VERSION;
|
||||
const packageTag = process.env.PACKAGE_TAG;
|
||||
if (!packageName || !packageVersion || !packageTag) {
|
||||
throw new Error("Missing ClawHub bootstrap verification env.");
|
||||
}
|
||||
const encodedName = encodeURIComponent(packageName);
|
||||
const encodedVersion = encodeURIComponent(packageVersion);
|
||||
const detailUrl = `${registry}/api/v1/packages/${encodedName}`;
|
||||
const trustedPublisherUrl = `${detailUrl}/trusted-publisher`;
|
||||
const versionUrl = `${detailUrl}/versions/${encodedVersion}`;
|
||||
const artifactUrl = `${versionUrl}/artifact/download`;
|
||||
|
||||
async function fetchWithRetry(url, options = {}) {
|
||||
let lastStatus = "unknown";
|
||||
for (let attempt = 1; attempt <= 12; attempt += 1) {
|
||||
try {
|
||||
const response = await fetch(url, { redirect: "manual", ...options });
|
||||
lastStatus = response.status;
|
||||
if (response.status !== 429 && response.status < 500) {
|
||||
return response;
|
||||
}
|
||||
} catch (error) {
|
||||
lastStatus = error instanceof Error ? error.message : String(error);
|
||||
}
|
||||
await new Promise((resolve) => setTimeout(resolve, attempt * 5000));
|
||||
}
|
||||
throw new Error(`${url} did not stabilize; last status ${lastStatus}.`);
|
||||
}
|
||||
|
||||
const detailResponse = await fetchWithRetry(detailUrl, {
|
||||
headers: { accept: "application/json" },
|
||||
});
|
||||
if (!detailResponse.ok) {
|
||||
throw new Error(`${detailUrl} returned HTTP ${detailResponse.status}.`);
|
||||
}
|
||||
const detail = await detailResponse.json();
|
||||
const tags = detail?.package?.tags ?? {};
|
||||
if (tags[packageTag] !== packageVersion) {
|
||||
throw new Error(
|
||||
`${packageName}: ClawHub tag ${packageTag} points to ${tags[packageTag] ?? "<missing>"}, expected ${packageVersion}.`,
|
||||
);
|
||||
}
|
||||
|
||||
const trustedPublisherResponse = await fetchWithRetry(trustedPublisherUrl, {
|
||||
headers: { accept: "application/json" },
|
||||
});
|
||||
if (!trustedPublisherResponse.ok) {
|
||||
throw new Error(`${trustedPublisherUrl} returned HTTP ${trustedPublisherResponse.status}.`);
|
||||
}
|
||||
const trustedPublisherDetail = await trustedPublisherResponse.json();
|
||||
const trustedPublisher = trustedPublisherDetail?.trustedPublisher;
|
||||
if (
|
||||
trustedPublisher?.repository !== "openclaw/openclaw" ||
|
||||
trustedPublisher?.workflowFilename !== "plugin-clawhub-release.yml" ||
|
||||
trustedPublisher?.environment != null
|
||||
) {
|
||||
throw new Error(
|
||||
`${packageName}: trusted publisher config did not match openclaw/openclaw plugin-clawhub-release.yml without an environment pin.`,
|
||||
);
|
||||
}
|
||||
|
||||
const versionResponse = await fetchWithRetry(versionUrl);
|
||||
if (!versionResponse.ok) {
|
||||
throw new Error(`${versionUrl} returned HTTP ${versionResponse.status}.`);
|
||||
}
|
||||
const artifactResponse = await fetchWithRetry(artifactUrl, { method: "HEAD" });
|
||||
if (artifactResponse.status < 200 || artifactResponse.status >= 400) {
|
||||
throw new Error(`${artifactUrl} returned HTTP ${artifactResponse.status}.`);
|
||||
}
|
||||
console.log(`${packageName}@${packageVersion} bootstrap verified on ClawHub.`);
|
||||
EOF
|
||||
252
.github/workflows/plugin-clawhub-release.yml
vendored
252
.github/workflows/plugin-clawhub-release.yml
vendored
@@ -16,7 +16,7 @@ on:
|
||||
required: false
|
||||
type: string
|
||||
ref:
|
||||
description: Commit SHA on main, a release branch, or the matching Tideclaw alpha branch to publish from; defaults to the workflow ref
|
||||
description: Dry-run target ref to validate; real OIDC publishes must dispatch the workflow with --ref set to the target release tag/ref
|
||||
required: false
|
||||
default: ""
|
||||
type: string
|
||||
@@ -24,6 +24,10 @@ on:
|
||||
description: Approved OpenClaw Release Publish workflow run id
|
||||
required: false
|
||||
type: string
|
||||
release_publish_branch:
|
||||
description: Branch name of the approving OpenClaw Release Publish workflow run
|
||||
required: false
|
||||
type: string
|
||||
dry_run:
|
||||
description: Validate the full ClawHub artifact handoff without publishing.
|
||||
required: false
|
||||
@@ -38,9 +42,7 @@ env:
|
||||
FORCE_JAVASCRIPT_ACTIONS_TO_NODE24: "true"
|
||||
NODE_VERSION: "24.15.0"
|
||||
CLAWHUB_REGISTRY: "https://clawhub.ai"
|
||||
CLAWHUB_REPOSITORY: "openclaw/clawhub"
|
||||
# Pinned to a reviewed ClawHub commit so release behavior stays reproducible.
|
||||
CLAWHUB_REF: "c9bb13023598dcc547fdf4a93b9d42512b8c8854"
|
||||
CLAWHUB_CLI_PACKAGE: "clawhub@0.21.0"
|
||||
|
||||
jobs:
|
||||
preview_plugins_clawhub:
|
||||
@@ -50,9 +52,15 @@ jobs:
|
||||
outputs:
|
||||
ref_revision: ${{ steps.ref.outputs.sha }}
|
||||
has_candidates: ${{ steps.plan.outputs.has_candidates }}
|
||||
has_bootstrap_candidates: ${{ steps.plan.outputs.has_bootstrap_candidates }}
|
||||
has_missing_trusted_publisher: ${{ steps.plan.outputs.has_missing_trusted_publisher }}
|
||||
candidate_count: ${{ steps.plan.outputs.candidate_count }}
|
||||
bootstrap_candidate_count: ${{ steps.plan.outputs.bootstrap_candidate_count }}
|
||||
missing_trusted_publisher_count: ${{ steps.plan.outputs.missing_trusted_publisher_count }}
|
||||
skipped_published_count: ${{ steps.plan.outputs.skipped_published_count }}
|
||||
matrix: ${{ steps.plan.outputs.matrix }}
|
||||
bootstrap_matrix: ${{ steps.plan.outputs.bootstrap_matrix }}
|
||||
missing_trusted_publisher_matrix: ${{ steps.plan.outputs.missing_trusted_publisher_matrix }}
|
||||
steps:
|
||||
- name: Checkout
|
||||
uses: actions/checkout@v6
|
||||
@@ -83,9 +91,27 @@ jobs:
|
||||
fi
|
||||
echo "sha=$(git rev-parse HEAD)" >> "$GITHUB_OUTPUT"
|
||||
|
||||
- name: Validate OIDC source matches workflow ref
|
||||
env:
|
||||
TARGET_SHA: ${{ steps.ref.outputs.sha }}
|
||||
WORKFLOW_SHA: ${{ github.sha }}
|
||||
DRY_RUN: ${{ inputs.dry_run && 'true' || 'false' }}
|
||||
run: |
|
||||
set -euo pipefail
|
||||
if [[ "${TARGET_SHA}" != "${WORKFLOW_SHA}" ]]; then
|
||||
if [[ "${DRY_RUN}" == "true" ]]; then
|
||||
echo "Dry-run publish target differs from workflow ref; allowing validation-only dispatch."
|
||||
exit 0
|
||||
fi
|
||||
echo "Plugin ClawHub OIDC publishes must run from the same ref that is being published." >&2
|
||||
echo "The ref input is only supported for dry_run=true." >&2
|
||||
echo "For real publishes, dispatch this workflow with --ref pointing at the target release tag/ref and omit the ref input." >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
- name: Validate ref is on a trusted publish branch
|
||||
env:
|
||||
WORKFLOW_REF: ${{ github.ref }}
|
||||
TRUSTED_PUBLISH_BRANCH: ${{ inputs.release_publish_branch || github.ref_name }}
|
||||
run: |
|
||||
set -euo pipefail
|
||||
if git merge-base --is-ancestor HEAD origin/main; then
|
||||
@@ -96,8 +122,8 @@ jobs:
|
||||
exit 0
|
||||
fi
|
||||
done < <(git for-each-ref --format='%(refname)' refs/remotes/origin/release)
|
||||
if [[ "${WORKFLOW_REF}" =~ ^refs/heads/tideclaw/alpha/[0-9]{4}-[0-9]{2}-[0-9]{2}-[0-9]{4}Z$ ]]; then
|
||||
alpha_branch="${WORKFLOW_REF#refs/heads/}"
|
||||
if [[ "${TRUSTED_PUBLISH_BRANCH}" =~ ^tideclaw/alpha/[0-9]{4}-[0-9]{2}-[0-9]{2}-[0-9]{4}Z$ ]]; then
|
||||
alpha_branch="${TRUSTED_PUBLISH_BRANCH}"
|
||||
git fetch --no-tags origin "+refs/heads/${alpha_branch}:refs/remotes/origin/${alpha_branch}"
|
||||
if git merge-base --is-ancestor HEAD "refs/remotes/origin/${alpha_branch}"; then
|
||||
exit 0
|
||||
@@ -158,36 +184,78 @@ jobs:
|
||||
cat .local/plugin-clawhub-release-plan.json
|
||||
|
||||
candidate_count="$(jq -r '.candidates | length' .local/plugin-clawhub-release-plan.json)"
|
||||
bootstrap_candidate_count="$(jq -r '.bootstrapCandidates | length' .local/plugin-clawhub-release-plan.json)"
|
||||
missing_trusted_publisher_count="$(jq -r '.missingTrustedPublisher | length' .local/plugin-clawhub-release-plan.json)"
|
||||
skipped_published_count="$(jq -r '.skippedPublished | length' .local/plugin-clawhub-release-plan.json)"
|
||||
has_candidates="false"
|
||||
if [[ "${candidate_count}" != "0" ]]; then
|
||||
has_candidates="true"
|
||||
fi
|
||||
has_bootstrap_candidates="false"
|
||||
if [[ "${bootstrap_candidate_count}" != "0" ]]; then
|
||||
has_bootstrap_candidates="true"
|
||||
fi
|
||||
has_missing_trusted_publisher="false"
|
||||
if [[ "${missing_trusted_publisher_count}" != "0" ]]; then
|
||||
has_missing_trusted_publisher="true"
|
||||
fi
|
||||
matrix_json="$(jq -c '.candidates' .local/plugin-clawhub-release-plan.json)"
|
||||
bootstrap_matrix_json="$(jq -c '.bootstrapCandidates' .local/plugin-clawhub-release-plan.json)"
|
||||
missing_trusted_publisher_matrix_json="$(jq -c '.missingTrustedPublisher' .local/plugin-clawhub-release-plan.json)"
|
||||
|
||||
{
|
||||
echo "candidate_count=${candidate_count}"
|
||||
echo "bootstrap_candidate_count=${bootstrap_candidate_count}"
|
||||
echo "missing_trusted_publisher_count=${missing_trusted_publisher_count}"
|
||||
echo "skipped_published_count=${skipped_published_count}"
|
||||
echo "has_candidates=${has_candidates}"
|
||||
echo "has_bootstrap_candidates=${has_bootstrap_candidates}"
|
||||
echo "has_missing_trusted_publisher=${has_missing_trusted_publisher}"
|
||||
echo "matrix=${matrix_json}"
|
||||
echo "bootstrap_matrix=${bootstrap_matrix_json}"
|
||||
echo "missing_trusted_publisher_matrix=${missing_trusted_publisher_matrix_json}"
|
||||
} >> "$GITHUB_OUTPUT"
|
||||
|
||||
echo "Plugin release candidates:"
|
||||
jq -r '.candidates[]? | "- \(.packageName)@\(.version) [\(.publishTag)] from \(.packageDir)"' .local/plugin-clawhub-release-plan.json
|
||||
|
||||
echo "Bootstrap candidates requiring token bootstrap:"
|
||||
jq -r '.bootstrapCandidates[]? | "- \(.packageName)@\(.version) [\(.publishTag)] from \(.packageDir)"' .local/plugin-clawhub-release-plan.json
|
||||
|
||||
echo "Missing trusted publisher candidates:"
|
||||
jq -r '.missingTrustedPublisher[]? | "- \(.packageName)@\(.version) [\(.publishTag)] from \(.packageDir)"' .local/plugin-clawhub-release-plan.json
|
||||
|
||||
echo "Already published / skipped:"
|
||||
jq -r '.skippedPublished[]? | "- \(.packageName)@\(.version)"' .local/plugin-clawhub-release-plan.json
|
||||
|
||||
- name: Fail when trusted publisher is missing
|
||||
if: steps.plan.outputs.missing_trusted_publisher_count != '0'
|
||||
run: |
|
||||
echo "::error::One or more ClawHub packages exist but do not have trusted publishing configured. Configure trusted publishing before running the normal OIDC publish workflow."
|
||||
jq -r '.missingTrustedPublisher[]? | "::error::Missing trusted publisher: \(.packageName)@\(.version). Configure trusted publishing for openclaw/openclaw, workflow plugin-clawhub-release.yml."' .local/plugin-clawhub-release-plan.json
|
||||
exit 1
|
||||
|
||||
- name: Fail normal publish when bootstrap is required
|
||||
if: steps.plan.outputs.bootstrap_candidate_count != '0'
|
||||
run: |
|
||||
echo "::error::One or more ClawHub packages do not exist yet and require the token-gated Plugin ClawHub New bootstrap workflow before normal OIDC publish can run."
|
||||
jq -r '.bootstrapCandidates[]? | "::error::Bootstrap required: \(.packageName)@\(.version). Dispatch plugin-clawhub-new.yml for this package, then rerun the normal release."' .local/plugin-clawhub-release-plan.json
|
||||
exit 1
|
||||
|
||||
- name: Fail manual publish when target versions already exist
|
||||
if: github.event_name == 'workflow_dispatch' && inputs.publish_scope == 'selected' && steps.plan.outputs.skipped_published_count != '0'
|
||||
if: github.event_name == 'workflow_dispatch' && inputs.dry_run != true && inputs.publish_scope == 'selected' && steps.plan.outputs.skipped_published_count != '0'
|
||||
run: |
|
||||
echo "::error::One or more selected plugin versions already exist on ClawHub. Bump the version before running a real publish."
|
||||
exit 1
|
||||
|
||||
- name: Validate Tideclaw alpha plugin channels
|
||||
if: startsWith(github.ref, 'refs/heads/tideclaw/alpha/')
|
||||
env:
|
||||
TRUSTED_PUBLISH_BRANCH: ${{ inputs.release_publish_branch || github.ref_name }}
|
||||
run: |
|
||||
set -euo pipefail
|
||||
if [[ ! "${TRUSTED_PUBLISH_BRANCH}" =~ ^tideclaw/alpha/[0-9]{4}-[0-9]{2}-[0-9]{2}-[0-9]{4}Z$ ]]; then
|
||||
exit 0
|
||||
fi
|
||||
invalid="$(
|
||||
jq -r '.candidates[]? | select(.publishTag != "alpha" or .channel != "alpha") | "- \(.packageName)@\(.version) [\(.publishTag)]"' .local/plugin-clawhub-release-plan.json
|
||||
)"
|
||||
@@ -215,7 +283,7 @@ jobs:
|
||||
env:
|
||||
GH_TOKEN: ${{ github.token }}
|
||||
RELEASE_PUBLISH_RUN_ID: ${{ inputs.release_publish_run_id }}
|
||||
EXPECTED_WORKFLOW_BRANCH: ${{ github.ref_name }}
|
||||
EXPECTED_WORKFLOW_BRANCH: ${{ inputs.release_publish_branch || github.ref_name }}
|
||||
run: |
|
||||
set -euo pipefail
|
||||
if [[ -z "${RELEASE_PUBLISH_RUN_ID// }" ]]; then
|
||||
@@ -234,99 +302,8 @@ jobs:
|
||||
RUN_JSON="$(gh run view "$RELEASE_PUBLISH_RUN_ID" --repo "$GITHUB_REPOSITORY" --json workflowName,headBranch,event,status,conclusion,url)"
|
||||
printf '%s' "$RUN_JSON" | DIRECT_RELEASE_RECOVERY="${direct_recovery}" node scripts/validate-release-publish-approval.mjs
|
||||
|
||||
preview_plugin_pack:
|
||||
needs: preview_plugins_clawhub
|
||||
if: needs.preview_plugins_clawhub.outputs.has_candidates == 'true'
|
||||
runs-on: ubuntu-latest
|
||||
permissions:
|
||||
contents: read
|
||||
strategy:
|
||||
fail-fast: false
|
||||
max-parallel: 12
|
||||
matrix:
|
||||
plugin: ${{ fromJson(needs.preview_plugins_clawhub.outputs.matrix) }}
|
||||
steps:
|
||||
- name: Checkout
|
||||
uses: actions/checkout@v6
|
||||
with:
|
||||
persist-credentials: false
|
||||
ref: ${{ github.ref }}
|
||||
fetch-depth: 0
|
||||
|
||||
- name: Checkout target revision
|
||||
env:
|
||||
TARGET_SHA: ${{ needs.preview_plugins_clawhub.outputs.ref_revision }}
|
||||
run: |
|
||||
set -euo pipefail
|
||||
git fetch --no-tags origin \
|
||||
+refs/heads/main:refs/remotes/origin/main \
|
||||
'+refs/heads/release/*:refs/remotes/origin/release/*'
|
||||
git checkout --detach "${TARGET_SHA}"
|
||||
|
||||
- name: Setup Node environment
|
||||
uses: ./.github/actions/setup-node-env
|
||||
with:
|
||||
node-version: ${{ env.NODE_VERSION }}
|
||||
install-bun: "true"
|
||||
install-deps: "true"
|
||||
|
||||
- name: Checkout ClawHub CLI source
|
||||
uses: actions/checkout@v6
|
||||
with:
|
||||
persist-credentials: false
|
||||
repository: ${{ env.CLAWHUB_REPOSITORY }}
|
||||
ref: main
|
||||
path: clawhub-source
|
||||
fetch-depth: 0
|
||||
|
||||
- name: Checkout pinned ClawHub CLI revision
|
||||
working-directory: clawhub-source
|
||||
env:
|
||||
CLAWHUB_REF: ${{ env.CLAWHUB_REF }}
|
||||
run: git checkout --detach "${CLAWHUB_REF}"
|
||||
|
||||
- name: Install ClawHub CLI dependencies
|
||||
working-directory: clawhub-source
|
||||
run: |
|
||||
set -euo pipefail
|
||||
for attempt in 1 2 3; do
|
||||
if bun install --frozen-lockfile; then
|
||||
exit 0
|
||||
fi
|
||||
status="$?"
|
||||
if [[ "${attempt}" == "3" ]]; then
|
||||
exit "${status}"
|
||||
fi
|
||||
echo "bun install failed while preparing ClawHub CLI; retrying (${attempt}/3)."
|
||||
rm -rf node_modules "${RUNNER_TEMP}/bun-install-cache" || true
|
||||
sleep $((attempt * 15))
|
||||
done
|
||||
|
||||
- name: Bootstrap ClawHub CLI
|
||||
run: |
|
||||
cat > "$RUNNER_TEMP/clawhub" <<'EOF'
|
||||
#!/usr/bin/env bash
|
||||
set -euo pipefail
|
||||
exec bun "$GITHUB_WORKSPACE/clawhub-source/packages/clawhub/src/cli.ts" "$@"
|
||||
EOF
|
||||
chmod +x "$RUNNER_TEMP/clawhub"
|
||||
echo "$RUNNER_TEMP" >> "$GITHUB_PATH"
|
||||
|
||||
- name: Verify package-local runtime build
|
||||
run: node scripts/check-plugin-npm-runtime-builds.mjs --package "${{ matrix.plugin.packageDir }}"
|
||||
|
||||
- name: Preview publish command
|
||||
env:
|
||||
CLAWHUB_REGISTRY: ${{ env.CLAWHUB_REGISTRY }}
|
||||
SOURCE_REPO: ${{ github.repository }}
|
||||
SOURCE_COMMIT: ${{ needs.preview_plugins_clawhub.outputs.ref_revision }}
|
||||
SOURCE_REF: ${{ github.ref }}
|
||||
PACKAGE_TAG: ${{ matrix.plugin.publishTag }}
|
||||
PACKAGE_DIR: ${{ matrix.plugin.packageDir }}
|
||||
run: bash scripts/plugin-clawhub-publish.sh --dry-run "${PACKAGE_DIR}"
|
||||
|
||||
pack_plugins_clawhub_artifacts:
|
||||
needs: [preview_plugins_clawhub, preview_plugin_pack, validate_release_publish_approval]
|
||||
needs: [preview_plugins_clawhub, validate_release_publish_approval]
|
||||
if: github.event_name == 'workflow_dispatch' && needs.preview_plugins_clawhub.outputs.has_candidates == 'true'
|
||||
runs-on: ubuntu-latest
|
||||
permissions:
|
||||
@@ -361,47 +338,19 @@ jobs:
|
||||
install-bun: "true"
|
||||
install-deps: "true"
|
||||
|
||||
- name: Checkout ClawHub CLI source
|
||||
uses: actions/checkout@v6
|
||||
with:
|
||||
persist-credentials: false
|
||||
repository: ${{ env.CLAWHUB_REPOSITORY }}
|
||||
ref: main
|
||||
path: clawhub-source
|
||||
fetch-depth: 0
|
||||
- name: Verify package-local runtime build
|
||||
run: node scripts/check-plugin-npm-runtime-builds.mjs --package "${{ matrix.plugin.packageDir }}"
|
||||
|
||||
- name: Checkout pinned ClawHub CLI revision
|
||||
working-directory: clawhub-source
|
||||
env:
|
||||
CLAWHUB_REF: ${{ env.CLAWHUB_REF }}
|
||||
run: git checkout --detach "${CLAWHUB_REF}"
|
||||
|
||||
- name: Install ClawHub CLI dependencies
|
||||
working-directory: clawhub-source
|
||||
- name: Install pinned ClawHub CLI wrapper
|
||||
run: |
|
||||
set -euo pipefail
|
||||
for attempt in 1 2 3; do
|
||||
if bun install --frozen-lockfile; then
|
||||
exit 0
|
||||
fi
|
||||
status="$?"
|
||||
if [[ "${attempt}" == "3" ]]; then
|
||||
exit "${status}"
|
||||
fi
|
||||
echo "bun install failed while preparing ClawHub CLI; retrying (${attempt}/3)."
|
||||
rm -rf node_modules "${RUNNER_TEMP}/bun-install-cache" || true
|
||||
sleep $((attempt * 15))
|
||||
done
|
||||
|
||||
- name: Bootstrap ClawHub CLI
|
||||
run: |
|
||||
cat > "$RUNNER_TEMP/clawhub" <<'EOF'
|
||||
cat > "${RUNNER_TEMP}/clawhub" <<'EOF'
|
||||
#!/usr/bin/env bash
|
||||
set -euo pipefail
|
||||
exec bun "$GITHUB_WORKSPACE/clawhub-source/packages/clawhub/src/cli.ts" "$@"
|
||||
exec npm exec --yes --package "${CLAWHUB_CLI_PACKAGE}" -- clawhub "$@"
|
||||
EOF
|
||||
chmod +x "$RUNNER_TEMP/clawhub"
|
||||
echo "$RUNNER_TEMP" >> "$GITHUB_PATH"
|
||||
chmod +x "${RUNNER_TEMP}/clawhub"
|
||||
echo "${RUNNER_TEMP}" >> "${GITHUB_PATH}"
|
||||
|
||||
- name: Pack ClawHub package artifact
|
||||
env:
|
||||
@@ -422,19 +371,23 @@ jobs:
|
||||
if-no-files-found: error
|
||||
retention-days: 7
|
||||
|
||||
approve_plugin_clawhub_release:
|
||||
approve_plugins_clawhub_release:
|
||||
needs: [preview_plugins_clawhub, pack_plugins_clawhub_artifacts]
|
||||
if: github.event_name == 'workflow_dispatch' && inputs.dry_run != true && needs.preview_plugins_clawhub.outputs.has_candidates == 'true'
|
||||
if: always() && github.event_name == 'workflow_dispatch' && inputs.dry_run != true && needs.preview_plugins_clawhub.outputs.has_candidates == 'true' && needs.pack_plugins_clawhub_artifacts.result == 'success'
|
||||
runs-on: ubuntu-latest
|
||||
environment: clawhub-plugin-release
|
||||
permissions: {}
|
||||
permissions:
|
||||
contents: read
|
||||
steps:
|
||||
- name: Approve ClawHub package publish
|
||||
run: echo "ClawHub package publish approved."
|
||||
- name: Approve Plugin ClawHub release publish
|
||||
run: |
|
||||
echo "Approved CLAW-277 03 - Split OpenClaw plugin ClawHub publishing into OIDC release and token bootstrap workflows release publish gate."
|
||||
|
||||
publish_plugins_clawhub:
|
||||
needs: [preview_plugins_clawhub, pack_plugins_clawhub_artifacts, approve_plugin_clawhub_release]
|
||||
if: always() && github.event_name == 'workflow_dispatch' && needs.preview_plugins_clawhub.outputs.has_candidates == 'true' && needs.pack_plugins_clawhub_artifacts.result == 'success' && (inputs.dry_run == true || needs.approve_plugin_clawhub_release.result == 'success')
|
||||
needs:
|
||||
[preview_plugins_clawhub, pack_plugins_clawhub_artifacts, approve_plugins_clawhub_release]
|
||||
if: always() && github.event_name == 'workflow_dispatch' && needs.preview_plugins_clawhub.outputs.has_candidates == 'true' && needs.pack_plugins_clawhub_artifacts.result == 'success' && (inputs.dry_run == true || needs.approve_plugins_clawhub_release.result == 'success')
|
||||
uses: openclaw/clawhub/.github/workflows/package-publish.yml@9d49df109d4ad3dc8a6ecf05d26b39f46d294721
|
||||
permissions:
|
||||
actions: read
|
||||
contents: read
|
||||
@@ -444,19 +397,18 @@ jobs:
|
||||
max-parallel: 32
|
||||
matrix:
|
||||
plugin: ${{ fromJson(needs.preview_plugins_clawhub.outputs.matrix) }}
|
||||
uses: openclaw/clawhub/.github/workflows/package-publish.yml@c9bb13023598dcc547fdf4a93b9d42512b8c8854
|
||||
with:
|
||||
dry_run: ${{ inputs.dry_run }}
|
||||
json: true
|
||||
package_artifact_name: ${{ matrix.plugin.artifactName }}
|
||||
dry_run: ${{ inputs.dry_run }}
|
||||
registry: https://clawhub.ai
|
||||
site: https://clawhub.ai
|
||||
tags: ${{ matrix.plugin.publishTag }}
|
||||
source_repo: ${{ github.repository }}
|
||||
source_commit: ${{ needs.preview_plugins_clawhub.outputs.ref_revision }}
|
||||
source_ref: ${{ github.ref }}
|
||||
tags: ${{ matrix.plugin.publishTag }}
|
||||
secrets:
|
||||
clawhub_token: ${{ secrets.CLAWHUB_TOKEN }}
|
||||
source_path: ${{ matrix.plugin.packageDir }}
|
||||
inspector_artifact_name: ${{ matrix.plugin.artifactName }}-inspector
|
||||
publish_json_artifact_name: ${{ matrix.plugin.artifactName }}-publish-json
|
||||
|
||||
verify_published_clawhub_package:
|
||||
needs: [preview_plugins_clawhub, publish_plugins_clawhub]
|
||||
|
||||
@@ -1,2 +1,2 @@
|
||||
ff7bd86cb1b243e0c94fdf9a74e7f985a7d73685b2b0cd0a8761972d145ca7a5 plugin-sdk-api-baseline.json
|
||||
a65283a99e28a300adffa26ed171a3e8b215d9c95e8a1656fc5ae8fd7fc011c6 plugin-sdk-api-baseline.jsonl
|
||||
2c783beea6b3cda3d79060739a923f9f39e7e8b5942123dd6b08a09143a587ca plugin-sdk-api-baseline.json
|
||||
0b33af2cffb42abb46682fb71c8f214da220793f13d10a34d332e75ff99e8ce9 plugin-sdk-api-baseline.jsonl
|
||||
|
||||
@@ -452,7 +452,7 @@ For normal PRs, follow scoped CI/check evidence instead of treating parity as a
|
||||
|
||||
The `CodeQL` workflow is intentionally a narrow first-pass security scanner, not the full repository sweep. Daily, manual, and non-draft pull request guard runs scan Actions workflow code plus the highest-risk JavaScript/TypeScript surfaces with high-confidence security queries filtered to high/critical `security-severity`.
|
||||
|
||||
The pull request guard stays light: it only starts for changes under `.github/actions`, `.github/codeql`, `.github/workflows`, `packages`, or `src`, and it runs the same high-confidence security matrix as the scheduled workflow. Android and macOS CodeQL stay out of PR defaults.
|
||||
The pull request guard stays light: it only starts for changes under `.github/actions`, `.github/codeql`, `.github/workflows`, `packages`, `scripts`, `src`, or process-owning bundled plugin runtime paths, and it runs the same high-confidence security matrix as the scheduled workflow. Android and macOS CodeQL stay out of PR defaults.
|
||||
|
||||
### Security categories
|
||||
|
||||
@@ -462,6 +462,7 @@ The pull request guard stays light: it only starts for changes under `.github/ac
|
||||
| `/codeql-security-high/channel-runtime-boundary` | Core channel implementation contracts plus the channel plugin runtime, gateway, Plugin SDK, secrets, audit touchpoints |
|
||||
| `/codeql-security-high/network-ssrf-boundary` | Core SSRF, IP parsing, network guard, web-fetch, and Plugin SDK SSRF policy surfaces |
|
||||
| `/codeql-security-high/mcp-process-tool-boundary` | MCP servers, process execution helpers, outbound delivery, and agent tool-execution gates |
|
||||
| `/codeql-security-high/process-exec-boundary` | Local shell, process spawn helpers, subprocess-owning bundled plugin runtimes, and workflow script glue |
|
||||
| `/codeql-security-high/plugin-trust-boundary` | Plugin install, loader, manifest, registry, package-manager install, source-loading, and Plugin SDK package contract trust surfaces |
|
||||
|
||||
### Platform-specific security shards
|
||||
|
||||
@@ -35,6 +35,7 @@ openclaw wiki status
|
||||
openclaw wiki doctor
|
||||
openclaw wiki init
|
||||
openclaw wiki ingest ./notes/alpha.md
|
||||
openclaw wiki okf import ./knowledge-catalog/okf/bundles/ga4
|
||||
openclaw wiki compile
|
||||
openclaw wiki lint
|
||||
openclaw wiki search "alpha"
|
||||
@@ -104,6 +105,31 @@ Notes:
|
||||
- imported source pages keep provenance in frontmatter
|
||||
- auto-compile can run after ingest when enabled
|
||||
|
||||
### `wiki okf import <path>`
|
||||
|
||||
Import an unpacked Open Knowledge Format bundle into wiki concept pages.
|
||||
|
||||
The importer reads every non-reserved `.md` concept document in the OKF
|
||||
directory tree, requires a non-empty `type` field, and treats unknown OKF
|
||||
`type` values as generic concepts. Reserved OKF `index.md` and `log.md` files
|
||||
are not imported as concepts.
|
||||
|
||||
Imported pages are flattened under `concepts/` so existing wiki compile,
|
||||
search, get, digest, and dashboard flows see them immediately. The original OKF
|
||||
concept ID, `type`, `resource`, `tags`, timestamp, source path, and full
|
||||
frontmatter are preserved in the page frontmatter. Internal OKF markdown links
|
||||
are rewritten to the generated wiki pages; broken or external links are left
|
||||
unchanged.
|
||||
|
||||
Examples:
|
||||
|
||||
```bash
|
||||
openclaw wiki okf import ./bundles/ga4
|
||||
openclaw wiki okf import ./bundles/ga4 --json
|
||||
openclaw wiki search "BigQuery Table" --mode source-evidence --json
|
||||
openclaw wiki get <path-from-json-result>
|
||||
```
|
||||
|
||||
### `wiki compile`
|
||||
|
||||
Rebuild indexes, related blocks, dashboards, and compiled digests.
|
||||
@@ -233,6 +259,8 @@ These require the official `obsidian` CLI on `PATH` when
|
||||
- Use `wiki lint` before trusting contradictory or low-confidence content.
|
||||
- Use `wiki compile` after bulk imports or source changes when you want fresh
|
||||
dashboards and compiled digests immediately.
|
||||
- Use `wiki okf import` when a data catalog, documentation export, or agent
|
||||
enrichment pipeline already emits OKF markdown bundles.
|
||||
- Use `wiki bridge import` when bridge mode depends on newly exported memory
|
||||
artifacts.
|
||||
|
||||
|
||||
@@ -368,6 +368,7 @@ Kimi K2 model IDs:
|
||||
[//]: # "moonshot-kimi-k2-model-refs:start"
|
||||
|
||||
- `moonshot/kimi-k2.6`
|
||||
- `moonshot/kimi-k2.7-code`
|
||||
- `moonshot/kimi-k2.5`
|
||||
- `moonshot/kimi-k2-thinking`
|
||||
- `moonshot/kimi-k2-thinking-turbo`
|
||||
|
||||
@@ -374,7 +374,7 @@ The implicit default set always covers canary, mention gating, native command re
|
||||
Output artifacts:
|
||||
|
||||
- `telegram-qa-report.md`
|
||||
- `telegram-qa-summary.json` - includes per-reply RTT (driver send → observed SUT reply) starting with the canary.
|
||||
- `qa-evidence.json` - evidence entries for the live transport checks, including profile, coverage, provider, channel, artifacts, result, and RTT fields.
|
||||
- `telegram-qa-observed-messages.json` - bodies redacted unless `OPENCLAW_QA_TELEGRAM_CAPTURE_CONTENT=1`.
|
||||
|
||||
Package RTT comparison uses the same Telegram credential contract while keeping
|
||||
@@ -447,7 +447,7 @@ pnpm openclaw qa discord \
|
||||
Output artifacts:
|
||||
|
||||
- `discord-qa-report.md`
|
||||
- `discord-qa-summary.json`
|
||||
- `qa-evidence.json` - evidence entries for the live transport checks.
|
||||
- `discord-qa-observed-messages.json` - bodies redacted unless `OPENCLAW_QA_DISCORD_CAPTURE_CONTENT=1`.
|
||||
- `discord-qa-reaction-timelines.json` and `discord-status-reactions-tool-only-timeline.png` when the status-reaction scenario runs.
|
||||
|
||||
@@ -495,7 +495,7 @@ Scenarios (`extensions/qa-lab/src/live-transports/slack/slack-live.runtime.ts`):
|
||||
Output artifacts:
|
||||
|
||||
- `slack-qa-report.md`
|
||||
- `slack-qa-summary.json`
|
||||
- `qa-evidence.json` - evidence entries for the live transport checks.
|
||||
- `slack-qa-observed-messages.json` - bodies redacted unless `OPENCLAW_QA_SLACK_CAPTURE_CONTENT=1`.
|
||||
- `approval-checkpoints/` - only when Mantis sets
|
||||
`OPENCLAW_QA_SLACK_APPROVAL_CHECKPOINT_DIR`; contains checkpoint JSON,
|
||||
@@ -740,7 +740,7 @@ poll and upload-file coverage run through deterministic gateway `poll` and
|
||||
Output artifacts:
|
||||
|
||||
- `whatsapp-qa-report.md`
|
||||
- `whatsapp-qa-summary.json`
|
||||
- `qa-evidence.json` - evidence entries for the live transport checks.
|
||||
- `whatsapp-qa-observed-messages.json` - bodies redacted unless `OPENCLAW_QA_WHATSAPP_CAPTURE_CONTENT=1`.
|
||||
|
||||
### Convex credential pool
|
||||
@@ -787,9 +787,10 @@ the source of truth for one test run and should define:
|
||||
- docs and code refs
|
||||
- optional plugin requirements
|
||||
- optional gateway config patch
|
||||
- the executable `qa-flow`
|
||||
- an executable `qa-flow` block for flow scenarios, or `execution.kind`/`execution.path`
|
||||
for Vitest and Playwright scenarios
|
||||
|
||||
The reusable runtime surface that backs `qa-flow` is allowed to stay generic
|
||||
The reusable runtime surface that backs `qa-flow` blocks is allowed to stay generic
|
||||
and cross-cutting. For example, markdown scenarios can combine transport-side
|
||||
helpers with browser-side helpers that drive the embedded Control UI through the
|
||||
Gateway `browser.request` seam without adding a special-case runner.
|
||||
@@ -915,6 +916,7 @@ The report should answer:
|
||||
For the inventory of available scenarios - useful when sizing follow-up work or wiring a new transport - run `pnpm openclaw qa coverage` (add `--json` for machine-readable output).
|
||||
When choosing focused proof for a touched behavior or file path, run `pnpm openclaw qa coverage --match <query>`.
|
||||
The match report searches scenario metadata, docs refs, code refs, coverage IDs, plugins, and provider requirements, then prints matching `qa suite --scenario ...` targets.
|
||||
Every `qa suite` scenario execution writes a `qa-evidence.json` artifact. Flow scenarios also write `qa-suite-summary.json` for existing suite/report tooling; scenarios that declare `execution.kind: vitest` or `execution.kind: playwright` run the matching test path and write `qa-vitest-report.md` or `qa-playwright-report.md` plus per-scenario logs.
|
||||
Treat it as a discovery aid, not a gate replacement; the selected scenario still needs the right provider mode, live transport, Multipass, Testbox, or release lane for the behavior under test.
|
||||
|
||||
For character and style checks, run the same scenario across multiple live model
|
||||
|
||||
@@ -30,6 +30,23 @@ title: "Usage tracking"
|
||||
- CLI: `openclaw channels list` prints the same usage snapshot alongside provider config (use `--no-usage` to skip).
|
||||
- macOS menu bar: "Usage" section under Context (only if available).
|
||||
|
||||
## Custom `/usage full` footer
|
||||
|
||||
Set `messages.usageTemplate` to customize the per-response `/usage full`
|
||||
footer. The value can be an inline template object or a JSON file path:
|
||||
|
||||
```json
|
||||
{
|
||||
"messages": {
|
||||
"usageTemplate": "~/.openclaw/usage-footer.json"
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
Templates read the `openclaw.usageLine.v1` contract and can use `scales`,
|
||||
`aliases`, and `output.surfaces` to render channel-specific footers. Missing,
|
||||
unreadable, invalid, or empty templates fall back to the built-in usage line.
|
||||
|
||||
## Providers + credentials
|
||||
|
||||
- **Anthropic (Claude)**: OAuth tokens in auth profiles.
|
||||
|
||||
@@ -42,6 +42,21 @@ health commands above for live connectivity checks.
|
||||
- `channels.<provider>.accounts.<accountId>.healthMonitor.enabled`: multi-account override that wins over the channel-level setting.
|
||||
- These per-channel overrides apply to the built-in channel monitors that expose them today: Discord, Google Chat, iMessage, Microsoft Teams, Signal, Slack, Telegram, and WhatsApp.
|
||||
|
||||
## Uptime monitoring
|
||||
|
||||
External uptime monitoring services should use the dedicated `/health` endpoint, not `/v1/chat/completions`.
|
||||
|
||||
- **DO use:** `GET /health` — instant response, no session created, no LLM call, returns `{"ok":true,"status":"live"}`
|
||||
- **DON'T use:** `/v1/chat/completions` for health checks — each request creates a full agent session with skill snapshot, context assembly, and LLM calls
|
||||
|
||||
When no `x-openclaw-session-key` header or `user` field is provided, `/v1/chat/completions` generates a new random session for each request. Monitoring services that ping every 15 minutes create ~96 sessions/day, each consuming 4–22KB. Over time this causes session store bloat and can lead to context window overflow.
|
||||
|
||||
### Monitoring service setup examples
|
||||
|
||||
- **BetterStack:** Set health check URL to `https://<your-gateway-host>:<port>/health`
|
||||
- **UptimeRobot:** Add a new HTTP monitor with URL `https://<your-gateway-host>:<port>/health`
|
||||
- **Generic:** Any HTTP GET to `/health` returns 200 with `{"ok":true}` when the gateway is healthy
|
||||
|
||||
## When something fails
|
||||
|
||||
- `logged out` or status 409–515 → relink with `openclaw channels logout` then `openclaw channels login`.
|
||||
|
||||
@@ -75,6 +75,7 @@ Auth matrix:
|
||||
- honor `x-openclaw-scopes` when the header is present
|
||||
- fall back to the normal operator default scope set when the header is absent
|
||||
- only lose owner semantics when the caller explicitly narrows scopes and omits `operator.admin`
|
||||
- require `operator.admin` for owner-level request controls such as `x-openclaw-model`
|
||||
|
||||
See [Security](/gateway/security) and [Remote access](/gateway/remote).
|
||||
|
||||
@@ -96,7 +97,7 @@ OpenClaw treats the OpenAI `model` field as an **agent target**, not a raw provi
|
||||
|
||||
Optional request headers:
|
||||
|
||||
- `x-openclaw-model: <provider/model-or-bare-id>` overrides the backend model for the selected agent.
|
||||
- `x-openclaw-model: <provider/model-or-bare-id>` overrides the backend model for the selected agent. Shared-secret bearer callers can use this header. Identity-bearing callers, such as trusted-proxy or private no-auth ingress requests with `x-openclaw-scopes`, need `operator.admin`; write-only callers get `403 missing scope: operator.admin`.
|
||||
- `x-openclaw-agent-id: <agentId>` remains supported as a compatibility override.
|
||||
- `x-openclaw-session-key: <sessionKey>` fully controls session routing.
|
||||
- `x-openclaw-message-channel: <channel>` sets the synthetic ingress channel context for channel-aware prompts and policies.
|
||||
@@ -178,7 +179,7 @@ This is the highest-leverage compatibility set for self-hosted frontends and too
|
||||
|
||||
</Accordion>
|
||||
<Accordion title="How do I override the backend model?">
|
||||
Use `x-openclaw-model`.
|
||||
Use `x-openclaw-model`. This is an owner-level override: it works with the Gateway shared-secret bearer token/password path, and it requires `operator.admin` on identity-bearing HTTP paths such as trusted proxy auth.
|
||||
|
||||
Examples:
|
||||
`x-openclaw-model: openai/gpt-5.4`
|
||||
@@ -191,7 +192,7 @@ This is the highest-leverage compatibility set for self-hosted frontends and too
|
||||
`/v1/embeddings` uses the same agent-target `model` ids.
|
||||
|
||||
Use `model: "openclaw/default"` or `model: "openclaw/<agentId>"`.
|
||||
When you need a specific embedding model, send it in `x-openclaw-model`.
|
||||
When you need a specific embedding model, send it in `x-openclaw-model` from a shared-secret caller or an identity-bearing caller with `operator.admin`.
|
||||
Without that header, the request passes through to the selected agent's normal embedding setup.
|
||||
|
||||
</Accordion>
|
||||
@@ -285,7 +286,7 @@ Expected behavior:
|
||||
|
||||
- `GET /v1/models` should list `openclaw/default`
|
||||
- Open WebUI should use `openclaw/default` as the chat model id
|
||||
- If you want a specific backend provider/model for that agent, set the agent's normal default model or send `x-openclaw-model`
|
||||
- If you want a specific backend provider/model for that agent, set the agent's normal default model or send `x-openclaw-model` from a shared-secret caller or an identity-bearing caller with `operator.admin`
|
||||
|
||||
Quick smoke:
|
||||
|
||||
@@ -370,7 +371,7 @@ Notes:
|
||||
|
||||
- `/v1/models` returns OpenClaw agent targets, not raw provider catalogs.
|
||||
- `openclaw/default` is always present so one stable id works across environments.
|
||||
- Backend provider/model overrides belong in `x-openclaw-model`, not the OpenAI `model` field.
|
||||
- Backend provider/model overrides belong in `x-openclaw-model`, not the OpenAI `model` field. On identity-bearing HTTP auth paths, this header requires `operator.admin`.
|
||||
- `/v1/embeddings` supports `input` as a string or array of strings.
|
||||
|
||||
## Related
|
||||
|
||||
@@ -951,7 +951,7 @@ Important boundary note:
|
||||
- Treat credentials that can call `/v1/chat/completions`, `/v1/responses`, plugin routes such as `/api/v1/admin/rpc`, or `/api/channels/*` as full-access operator secrets for that gateway.
|
||||
- On the OpenAI-compatible HTTP surface, shared-secret bearer auth restores the full default operator scopes (`operator.admin`, `operator.approvals`, `operator.pairing`, `operator.read`, `operator.talk.secrets`, `operator.write`) and owner semantics for agent turns; narrower `x-openclaw-scopes` values do not reduce that shared-secret path.
|
||||
- Per-request scope semantics on HTTP only apply when the request comes from an identity-bearing mode such as trusted proxy auth, or from an explicitly no-auth private ingress.
|
||||
- In those identity-bearing modes, omitting `x-openclaw-scopes` falls back to the normal operator default scope set; send the header explicitly when you want a narrower scope set.
|
||||
- In those identity-bearing modes, omitting `x-openclaw-scopes` falls back to the normal operator default scope set; send the header explicitly when you want a narrower scope set. Owner-level OpenAI-compatible headers such as `x-openclaw-model` require `operator.admin` when scopes are narrowed.
|
||||
- `/tools/invoke` and HTTP session history endpoints follow the same shared-secret rule: token/password bearer auth is treated as full operator access there too, while identity-bearing modes still honor declared scopes.
|
||||
- Do not share these credentials with untrusted callers; prefer separate gateways per trust boundary.
|
||||
|
||||
|
||||
@@ -18,11 +18,13 @@ most Linux-compatible Gateway runtime.
|
||||
Windows Hub is the native WinUI companion app for Windows 10 20H2+ and Windows 11. It installs without administrator privileges and is published with signed
|
||||
x64 and ARM64 installers on OpenClaw releases.
|
||||
|
||||
Download the latest stable installer:
|
||||
Download the latest stable installer from the [OpenClaw releases page](https://github.com/openclaw/openclaw/releases):
|
||||
|
||||
- [OpenClawCompanion-Setup-x64.exe](https://github.com/openclaw/openclaw/releases/latest/download/OpenClawCompanion-Setup-x64.exe)
|
||||
- [OpenClawCompanion-Setup-arm64.exe](https://github.com/openclaw/openclaw/releases/latest/download/OpenClawCompanion-Setup-arm64.exe)
|
||||
- [Checksums](https://github.com/openclaw/openclaw/releases/latest/download/OpenClawCompanion-SHA256SUMS.txt)
|
||||
- [OpenClawCompanion-Setup-x64.exe](https://github.com/openclaw/openclaw/releases/download/v2026.6.5/OpenClawCompanion-Setup-x64.exe)
|
||||
- [OpenClawCompanion-Setup-arm64.exe](https://github.com/openclaw/openclaw/releases/download/v2026.6.5/OpenClawCompanion-Setup-arm64.exe)
|
||||
- [Checksums](https://github.com/openclaw/openclaw/releases/download/v2026.6.5/OpenClawCompanion-SHA256SUMS.txt)
|
||||
|
||||
If a download link above returns a 404, visit the [releases page](https://github.com/openclaw/openclaw/releases) and look for the `OpenClawCompanion-Setup-*` assets on the latest release.
|
||||
|
||||
After install, launch **OpenClaw Companion** from the Start menu or the system
|
||||
tray. The installer also adds shortcuts for Gateway Setup, Chat, Settings,
|
||||
|
||||
@@ -425,6 +425,10 @@ even when the channel payload has no visible text/caption. Rewriting that
|
||||
`content` updates the hook-visible transcript only; it is not rendered as a
|
||||
media caption.
|
||||
|
||||
`reply_payload_sending` events may include `usageState`, a best-effort live
|
||||
per-turn model/usage/context snapshot. Durable delivery, recovered replay, and
|
||||
replies without exact run correlation omit it.
|
||||
|
||||
Message hook contexts expose stable correlation fields when available:
|
||||
`ctx.sessionKey`, `ctx.runId`, `ctx.messageId`, `ctx.senderId`, `ctx.trace`,
|
||||
`ctx.traceId`, `ctx.spanId`, `ctx.parentSpanId`, and `ctx.callDepth`. Inbound
|
||||
|
||||
@@ -25,6 +25,7 @@ less like a pile of Markdown files.
|
||||
- Page-level provenance, confidence, contradictions, and open questions
|
||||
- Compiled digests for agent/runtime consumers
|
||||
- Wiki-native search/get/apply/lint tools
|
||||
- Open Knowledge Format imports into compiled wiki concepts
|
||||
- Optional bridge mode that imports public artifacts from the active memory plugin
|
||||
- Optional Obsidian-friendly render mode and CLI integration
|
||||
|
||||
@@ -135,6 +136,34 @@ The main page groups are:
|
||||
- `syntheses/` for compiled summaries and maintained rollups
|
||||
- `reports/` for generated dashboards
|
||||
|
||||
## Open Knowledge Format imports
|
||||
|
||||
`memory-wiki` can import unpacked Open Knowledge Format bundles with:
|
||||
|
||||
```bash
|
||||
openclaw wiki okf import ./bundles/ga4
|
||||
```
|
||||
|
||||
This is the cleanest fit when a data catalog, documentation crawler, or
|
||||
enrichment agent already produces OKF: keep OKF as the portable exchange
|
||||
artifact, then let `memory-wiki` turn it into OpenClaw-native concept pages and
|
||||
compiled digests.
|
||||
|
||||
The importer follows the OKF v0.1 shape:
|
||||
|
||||
- non-reserved `.md` files are concept documents
|
||||
- each imported concept needs a non-empty `type` frontmatter field
|
||||
- unknown OKF `type` values are accepted
|
||||
- reserved `index.md` and `log.md` files are not imported as concepts
|
||||
- broken or external markdown links are preserved
|
||||
|
||||
Imported concept pages are flattened under `concepts/` so the existing compile,
|
||||
search, get, dashboard, and prompt-digest paths see them without adding a second
|
||||
wiki tree. Each page keeps the original OKF concept ID, source path, `type`,
|
||||
`resource`, `tags`, timestamp, and full producer frontmatter. Internal OKF links
|
||||
are rewritten to the generated wiki concept pages and also emitted as structured
|
||||
`relationships` entries with `kind: okf-link`.
|
||||
|
||||
## Structured claims and evidence
|
||||
|
||||
Pages can carry structured `claims` frontmatter, not just freeform text.
|
||||
|
||||
@@ -22,6 +22,7 @@ Moonshot and Kimi Coding are **separate providers**. Keys are not interchangeabl
|
||||
| Model ref | Name | Reasoning | Input | Context | Max output |
|
||||
| --------------------------------- | ---------------------- | --------- | ----------- | ------- | ---------- |
|
||||
| `moonshot/kimi-k2.6` | Kimi K2.6 | No | text, image | 262,144 | 262,144 |
|
||||
| `moonshot/kimi-k2.7-code` | Kimi K2.7 Code | Always on | text, image | 262,144 | 262,144 |
|
||||
| `moonshot/kimi-k2.5` | Kimi K2.5 | No | text, image | 262,144 | 262,144 |
|
||||
| `moonshot/kimi-k2-thinking` | Kimi K2 Thinking | Yes | text | 262,144 | 262,144 |
|
||||
| `moonshot/kimi-k2-thinking-turbo` | Kimi K2 Thinking Turbo | Yes | text | 262,144 | 262,144 |
|
||||
@@ -30,11 +31,18 @@ Moonshot and Kimi Coding are **separate providers**. Keys are not interchangeabl
|
||||
[//]: # "moonshot-kimi-k2-ids:end"
|
||||
|
||||
Bundled cost estimates for current Moonshot-hosted K2 models use Moonshot's
|
||||
published pay-as-you-go rates: Kimi K2.6 is $0.16/MTok cache hit,
|
||||
published pay-as-you-go rates: Kimi K2.7 Code is $0.19/MTok cache hit,
|
||||
$0.95/MTok input, and $4.00/MTok output; Kimi K2.6 is $0.16/MTok cache hit,
|
||||
$0.95/MTok input, and $4.00/MTok output; Kimi K2.5 is $0.10/MTok cache hit,
|
||||
$0.60/MTok input, and $3.00/MTok output. Other legacy catalog entries keep
|
||||
zero-cost placeholders unless you override them in config.
|
||||
|
||||
Kimi K2.7 Code always uses native thinking. OpenClaw exposes only the `on`
|
||||
thinking state for this model and omits outbound `thinking` and
|
||||
`reasoning_effort` controls, as required by Moonshot. OpenClaw also omits
|
||||
sampling overrides that K2.7 fixes to provider defaults. Kimi K2.6 remains the
|
||||
onboarding default.
|
||||
|
||||
## Getting started
|
||||
|
||||
Choose your provider and follow the setup steps.
|
||||
@@ -109,6 +117,7 @@ Choose your provider and follow the setup steps.
|
||||
models: {
|
||||
// moonshot-kimi-k2-aliases:start
|
||||
"moonshot/kimi-k2.6": { alias: "Kimi K2.6" },
|
||||
"moonshot/kimi-k2.7-code": { alias: "Kimi K2.7 Code" },
|
||||
"moonshot/kimi-k2.5": { alias: "Kimi K2.5" },
|
||||
"moonshot/kimi-k2-thinking": { alias: "Kimi K2 Thinking" },
|
||||
"moonshot/kimi-k2-thinking-turbo": { alias: "Kimi K2 Thinking Turbo" },
|
||||
@@ -135,6 +144,15 @@ Choose your provider and follow the setup steps.
|
||||
contextWindow: 262144,
|
||||
maxTokens: 262144,
|
||||
},
|
||||
{
|
||||
id: "kimi-k2.7-code",
|
||||
name: "Kimi K2.7 Code",
|
||||
reasoning: true,
|
||||
input: ["text", "image"],
|
||||
cost: { input: 0.95, output: 4, cacheRead: 0.19, cacheWrite: 0 },
|
||||
contextWindow: 262144,
|
||||
maxTokens: 262144,
|
||||
},
|
||||
{
|
||||
id: "kimi-k2.5",
|
||||
name: "Kimi K2.5",
|
||||
@@ -288,7 +306,13 @@ Config lives under `plugins.entries.moonshot.config.webSearch`:
|
||||
|
||||
<AccordionGroup>
|
||||
<Accordion title="Native thinking mode">
|
||||
Moonshot Kimi supports binary native thinking:
|
||||
Kimi K2.7 Code always uses native thinking. Moonshot requires clients to
|
||||
omit the `thinking` field for this model, so OpenClaw exposes only `on` and
|
||||
ignores stale `off` settings. K2.7 also fixes `temperature`, `top_p`, `n`,
|
||||
`presence_penalty`, and `frequency_penalty`; OpenClaw omits configured
|
||||
overrides for those fields.
|
||||
|
||||
Other Moonshot Kimi models support binary native thinking:
|
||||
|
||||
- `thinking: { type: "enabled" }`
|
||||
- `thinking: { type: "disabled" }`
|
||||
@@ -311,7 +335,7 @@ Config lives under `plugins.entries.moonshot.config.webSearch`:
|
||||
}
|
||||
```
|
||||
|
||||
OpenClaw also maps runtime `/think` levels for Moonshot:
|
||||
OpenClaw maps runtime `/think` levels for those models:
|
||||
|
||||
| `/think` level | Moonshot behavior |
|
||||
| -------------------- | -------------------------- |
|
||||
@@ -319,14 +343,16 @@ Config lives under `plugins.entries.moonshot.config.webSearch`:
|
||||
| Any non-off level | `thinking.type=enabled` |
|
||||
|
||||
<Warning>
|
||||
When Moonshot thinking is enabled, `tool_choice` must be `auto` or `none`. OpenClaw normalizes incompatible `tool_choice` values to `auto` for compatibility.
|
||||
When Moonshot thinking is enabled, `tool_choice` must be `auto` or `none`. OpenClaw normalizes incompatible values to `auto`. This includes Kimi K2.7 Code, whose thinking mode cannot be disabled to preserve a pinned tool choice.
|
||||
</Warning>
|
||||
|
||||
Kimi K2.6 also accepts an optional `thinking.keep` field that controls
|
||||
multi-turn retention of `reasoning_content`. Set it to `"all"` to keep full
|
||||
reasoning across turns; omit it (or leave it `null`) to use the server
|
||||
default strategy. OpenClaw only forwards `thinking.keep` for
|
||||
`moonshot/kimi-k2.6` and strips it from other models.
|
||||
`moonshot/kimi-k2.6` and strips it from other models. Kimi K2.7 Code
|
||||
preserves full reasoning history by default while OpenClaw omits the entire
|
||||
`thinking` field.
|
||||
|
||||
```json5
|
||||
{
|
||||
@@ -347,7 +373,7 @@ Config lives under `plugins.entries.moonshot.config.webSearch`:
|
||||
</Accordion>
|
||||
|
||||
<Accordion title="Tool call id sanitization">
|
||||
Moonshot Kimi serves tool_call ids shaped like `functions.<name>:<index>`. OpenClaw preserves them unchanged so multi-turn tool calls keep working.
|
||||
Moonshot Kimi serves native tool_call ids shaped like `functions.<name>:<index>`. For the OpenAI-completions transport, OpenClaw preserves the first occurrence of each native Kimi id and rewrites later duplicates to deterministic OpenAI-style `call_*` ids. Matching tool results are remapped with the same id so replay remains unique without stripping Kimi's first native id.
|
||||
|
||||
To force strict sanitization on a custom OpenAI-compatible provider, set `sanitizeToolCallIds: true`:
|
||||
|
||||
|
||||
@@ -35,7 +35,7 @@ title: "Thinking levels"
|
||||
- Google Gemini maps `/think adaptive` to Gemini's provider-owned dynamic thinking. Gemini 3 requests omit a fixed `thinkingLevel`, while Gemini 2.5 requests send `thinkingBudget: -1`; fixed levels still map to the closest Gemini `thinkingLevel` or budget for that model family.
|
||||
- MiniMax M2.x (`minimax/MiniMax-M2*`) on the Anthropic-compatible streaming path defaults to `thinking: { type: "disabled" }` unless you explicitly set thinking in model params or request params. This avoids leaked `reasoning_content` deltas from M2.x's non-native Anthropic stream format. MiniMax-M3 (and M3.x) is exempt: M3 emits proper Anthropic thinking blocks and returns empty content when thinking is disabled, so OpenClaw keeps M3 on the provider's omitted/adaptive thinking path.
|
||||
- Z.AI (`zai/*`) only supports binary thinking (`on`/`off`). Any non-`off` level is treated as `on` (mapped to `low`).
|
||||
- Moonshot (`moonshot/*`) maps `/think off` to `thinking: { type: "disabled" }` and any non-`off` level to `thinking: { type: "enabled" }`. When thinking is enabled, Moonshot only accepts `tool_choice` `auto|none`; OpenClaw normalizes incompatible values to `auto`.
|
||||
- Moonshot Kimi K2.7 Code (`moonshot/kimi-k2.7-code`) always thinks. Its profile exposes only `on`, and OpenClaw omits the outbound `thinking` field as required by Moonshot. Other `moonshot/*` models map `/think off` to `thinking: { type: "disabled" }` and any non-`off` level to `thinking: { type: "enabled" }`. When thinking is enabled, Moonshot only accepts `tool_choice` `auto|none`; OpenClaw normalizes incompatible values to `auto`.
|
||||
|
||||
## Resolution order
|
||||
|
||||
|
||||
214
extensions/acpx/npm-shrinkwrap.json
generated
214
extensions/acpx/npm-shrinkwrap.json
generated
@@ -224,9 +224,9 @@
|
||||
}
|
||||
},
|
||||
"node_modules/@esbuild/aix-ppc64": {
|
||||
"version": "0.28.0",
|
||||
"resolved": "https://registry.npmjs.org/@esbuild/aix-ppc64/-/aix-ppc64-0.28.0.tgz",
|
||||
"integrity": "sha512-lhRUCeuOyJQURhTxl4WkpFTjIsbDayJHih5kZC1giwE+MhIzAb7mEsQMqMf18rHLsrb5qI1tafG20mLxEWcWlA==",
|
||||
"version": "0.28.1",
|
||||
"resolved": "https://registry.npmjs.org/@esbuild/aix-ppc64/-/aix-ppc64-0.28.1.tgz",
|
||||
"integrity": "sha512-Svl7tq8k/08+p6CXPpRjQ1fKX+1odH/BQbb48fV6fj3CWHhsoIOoY87w1oHXm0qEpkIK3ZfVgp0hed3XBXzXMQ==",
|
||||
"cpu": [
|
||||
"ppc64"
|
||||
],
|
||||
@@ -240,9 +240,9 @@
|
||||
}
|
||||
},
|
||||
"node_modules/@esbuild/android-arm": {
|
||||
"version": "0.28.0",
|
||||
"resolved": "https://registry.npmjs.org/@esbuild/android-arm/-/android-arm-0.28.0.tgz",
|
||||
"integrity": "sha512-wqh0ByljabXLKHeWXYLqoJ5jKC4XBaw6Hk08OfMrCRd2nP2ZQ5eleDZC41XHyCNgktBGYMbqnrJKq/K/lzPMSQ==",
|
||||
"version": "0.28.1",
|
||||
"resolved": "https://registry.npmjs.org/@esbuild/android-arm/-/android-arm-0.28.1.tgz",
|
||||
"integrity": "sha512-0k2F129Xdio1TdJfzJ8sy1Q47vUD2NnwdhiAf7drUN1EBTfPf4hsFCtmMgu/6m8JSzsBrlmVjudMBQqOfG8usQ==",
|
||||
"cpu": [
|
||||
"arm"
|
||||
],
|
||||
@@ -256,9 +256,9 @@
|
||||
}
|
||||
},
|
||||
"node_modules/@esbuild/android-arm64": {
|
||||
"version": "0.28.0",
|
||||
"resolved": "https://registry.npmjs.org/@esbuild/android-arm64/-/android-arm64-0.28.0.tgz",
|
||||
"integrity": "sha512-+WzIXQOSaGs33tLEgYPYe/yQHf0WTU0X42Jca3y8NWMbUVhp7rUnw+vAsRC/QiDrdD31IszMrZy+qwPOPjd+rw==",
|
||||
"version": "0.28.1",
|
||||
"resolved": "https://registry.npmjs.org/@esbuild/android-arm64/-/android-arm64-0.28.1.tgz",
|
||||
"integrity": "sha512-34EGEbCIAgosYz6goLcopX6Mo7NyGv9tfwEM2/7Ce2VcVRk568iSvniGWcUXIy7wEDR1wzolcxcriFVrWYcwBg==",
|
||||
"cpu": [
|
||||
"arm64"
|
||||
],
|
||||
@@ -272,9 +272,9 @@
|
||||
}
|
||||
},
|
||||
"node_modules/@esbuild/android-x64": {
|
||||
"version": "0.28.0",
|
||||
"resolved": "https://registry.npmjs.org/@esbuild/android-x64/-/android-x64-0.28.0.tgz",
|
||||
"integrity": "sha512-+VJggoaKhk2VNNqVL7f6S189UzShHC/mR9EE8rDdSkdpN0KflSwWY/gWjDrNxxisg8Fp1ZCD9jLMo4m0OUfeUA==",
|
||||
"version": "0.28.1",
|
||||
"resolved": "https://registry.npmjs.org/@esbuild/android-x64/-/android-x64-0.28.1.tgz",
|
||||
"integrity": "sha512-dbwY7ltSMDWsRatcRpCnES4F+im88OCUgGZjy52shC7GqHRE/cYlxNbB4Z4UpJswpcc4Qxd2oE/ufM0p61IKng==",
|
||||
"cpu": [
|
||||
"x64"
|
||||
],
|
||||
@@ -288,9 +288,9 @@
|
||||
}
|
||||
},
|
||||
"node_modules/@esbuild/darwin-arm64": {
|
||||
"version": "0.28.0",
|
||||
"resolved": "https://registry.npmjs.org/@esbuild/darwin-arm64/-/darwin-arm64-0.28.0.tgz",
|
||||
"integrity": "sha512-0T+A9WZm+bZ84nZBtk1ckYsOvyA3x7e2Acj1KdVfV4/2tdG4fzUp91YHx+GArWLtwqp77pBXVCPn2We7Letr0Q==",
|
||||
"version": "0.28.1",
|
||||
"resolved": "https://registry.npmjs.org/@esbuild/darwin-arm64/-/darwin-arm64-0.28.1.tgz",
|
||||
"integrity": "sha512-TZbWkQY7kvTAXbXUT7uVACR5cMHsDiSz9z7ZKAX/RTq/WJEk3QyRr0wZpNhBDX+/0CtdqUIJlOiodQcta6tY3Q==",
|
||||
"cpu": [
|
||||
"arm64"
|
||||
],
|
||||
@@ -304,9 +304,9 @@
|
||||
}
|
||||
},
|
||||
"node_modules/@esbuild/darwin-x64": {
|
||||
"version": "0.28.0",
|
||||
"resolved": "https://registry.npmjs.org/@esbuild/darwin-x64/-/darwin-x64-0.28.0.tgz",
|
||||
"integrity": "sha512-fyzLm/DLDl/84OCfp2f/XQ4flmORsjU7VKt8HLjvIXChJoFFOIL6pLJPH4Yhd1n1gGFF9mPwtlN5Wf82DZs+LQ==",
|
||||
"version": "0.28.1",
|
||||
"resolved": "https://registry.npmjs.org/@esbuild/darwin-x64/-/darwin-x64-0.28.1.tgz",
|
||||
"integrity": "sha512-zfdzgK9ACBNZLI/CyHTOx81SyNbM6YXn7rxSgX97VjyiPl9W1i4Ka4fgKECEoFCKGpvBj5qArWIGgQjOwkgskQ==",
|
||||
"cpu": [
|
||||
"x64"
|
||||
],
|
||||
@@ -320,9 +320,9 @@
|
||||
}
|
||||
},
|
||||
"node_modules/@esbuild/freebsd-arm64": {
|
||||
"version": "0.28.0",
|
||||
"resolved": "https://registry.npmjs.org/@esbuild/freebsd-arm64/-/freebsd-arm64-0.28.0.tgz",
|
||||
"integrity": "sha512-l9GeW5UZBT9k9brBYI+0WDffcRxgHQD8ShN2Ur4xWq/NFzUKm3k5lsH4PdaRgb2w7mI9u61nr2gI2mLI27Nh3Q==",
|
||||
"version": "0.28.1",
|
||||
"resolved": "https://registry.npmjs.org/@esbuild/freebsd-arm64/-/freebsd-arm64-0.28.1.tgz",
|
||||
"integrity": "sha512-wG2EA8ENdEI0qhkSZMjfqrdY+ziCYCPMmtZjjIwOmXFjmyzEHn+UUxk5of+SYsjtfs3VpnlC7QLzSI5hY/rOAw==",
|
||||
"cpu": [
|
||||
"arm64"
|
||||
],
|
||||
@@ -336,9 +336,9 @@
|
||||
}
|
||||
},
|
||||
"node_modules/@esbuild/freebsd-x64": {
|
||||
"version": "0.28.0",
|
||||
"resolved": "https://registry.npmjs.org/@esbuild/freebsd-x64/-/freebsd-x64-0.28.0.tgz",
|
||||
"integrity": "sha512-BXoQai/A0wPO6Es3yFJ7APCiKGc1tdAEOgeTNy3SsB491S3aHn4S4r3e976eUnPdU+NbdtmBuLncYir2tMU9Nw==",
|
||||
"version": "0.28.1",
|
||||
"resolved": "https://registry.npmjs.org/@esbuild/freebsd-x64/-/freebsd-x64-0.28.1.tgz",
|
||||
"integrity": "sha512-i7dZ9vQgnvSCzi/rYCXNgtF/U+eKZNJBzu3eTQbRgHnM7tNSizLOkRFAl3qzVc/Op/u5YkHHa4pf/3DOYHthLQ==",
|
||||
"cpu": [
|
||||
"x64"
|
||||
],
|
||||
@@ -352,9 +352,9 @@
|
||||
}
|
||||
},
|
||||
"node_modules/@esbuild/linux-arm": {
|
||||
"version": "0.28.0",
|
||||
"resolved": "https://registry.npmjs.org/@esbuild/linux-arm/-/linux-arm-0.28.0.tgz",
|
||||
"integrity": "sha512-CjaaREJagqJp7iTaNQjjidaNbCKYcd4IDkzbwwxtSvjI7NZm79qiHc8HqciMddQ6CKvJT6aBd8lO9kN/ZudLlw==",
|
||||
"version": "0.28.1",
|
||||
"resolved": "https://registry.npmjs.org/@esbuild/linux-arm/-/linux-arm-0.28.1.tgz",
|
||||
"integrity": "sha512-qVXBOHQS+d5Y722GwJzJUtOLlX7km3CraOaGormF1pDtPd2C/l1SHRPgjLunLGe51Sh5YYWKMFDyV4SxgMQYTQ==",
|
||||
"cpu": [
|
||||
"arm"
|
||||
],
|
||||
@@ -368,9 +368,9 @@
|
||||
}
|
||||
},
|
||||
"node_modules/@esbuild/linux-arm64": {
|
||||
"version": "0.28.0",
|
||||
"resolved": "https://registry.npmjs.org/@esbuild/linux-arm64/-/linux-arm64-0.28.0.tgz",
|
||||
"integrity": "sha512-RVyzfb3FWsGA55n6WY0MEIEPURL1FcbhFE6BffZEMEekfCzCIMtB5yyDcFnVbTnwk+CLAgTujmV/Lgvih56W+A==",
|
||||
"version": "0.28.1",
|
||||
"resolved": "https://registry.npmjs.org/@esbuild/linux-arm64/-/linux-arm64-0.28.1.tgz",
|
||||
"integrity": "sha512-yHs+0uc8+nvEAfAfxrWQKK5peSNzBc4PegcMO0EJ2hT71uA7vB8Ihg2e77R2P7SG5uYjPbHlLLmve4LLLRCf0g==",
|
||||
"cpu": [
|
||||
"arm64"
|
||||
],
|
||||
@@ -384,9 +384,9 @@
|
||||
}
|
||||
},
|
||||
"node_modules/@esbuild/linux-ia32": {
|
||||
"version": "0.28.0",
|
||||
"resolved": "https://registry.npmjs.org/@esbuild/linux-ia32/-/linux-ia32-0.28.0.tgz",
|
||||
"integrity": "sha512-KBnSTt1kxl9x70q+ydterVdl+Cn0H18ngRMRCEQfrbqdUuntQQ0LoMZv47uB97NljZFzY6HcfqEZ2SAyIUTQBQ==",
|
||||
"version": "0.28.1",
|
||||
"resolved": "https://registry.npmjs.org/@esbuild/linux-ia32/-/linux-ia32-0.28.1.tgz",
|
||||
"integrity": "sha512-d1z4ZuP0ajrfz/FhGT4vv278rX8KnPPJx8i5+AtK7TYbx9Le9F1hyzurZpkEyjkGa9dUGhQow4C1NmeGvqxN2w==",
|
||||
"cpu": [
|
||||
"ia32"
|
||||
],
|
||||
@@ -400,9 +400,9 @@
|
||||
}
|
||||
},
|
||||
"node_modules/@esbuild/linux-loong64": {
|
||||
"version": "0.28.0",
|
||||
"resolved": "https://registry.npmjs.org/@esbuild/linux-loong64/-/linux-loong64-0.28.0.tgz",
|
||||
"integrity": "sha512-zpSlUce1mnxzgBADvxKXX5sl8aYQHo2ezvMNI8I0lbblJtp8V4odlm3Yzlj7gPyt3T8ReksE6bK+pT3WD+aJRg==",
|
||||
"version": "0.28.1",
|
||||
"resolved": "https://registry.npmjs.org/@esbuild/linux-loong64/-/linux-loong64-0.28.1.tgz",
|
||||
"integrity": "sha512-M5sRjUVZrkm1OAPR3dlOYzNmN+loZKGVi1VUQGrwuqLcbR6qeAz+famMhjASeH3YVKvZz+zT1jlh/keC3Rj/lg==",
|
||||
"cpu": [
|
||||
"loong64"
|
||||
],
|
||||
@@ -416,9 +416,9 @@
|
||||
}
|
||||
},
|
||||
"node_modules/@esbuild/linux-mips64el": {
|
||||
"version": "0.28.0",
|
||||
"resolved": "https://registry.npmjs.org/@esbuild/linux-mips64el/-/linux-mips64el-0.28.0.tgz",
|
||||
"integrity": "sha512-2jIfP6mmjkdmeTlsX/9vmdmhBmKADrWqN7zcdtHIeNSCH1SqIoNI63cYsjQR8J+wGa4Y5izRcSHSm8K3QWmk3w==",
|
||||
"version": "0.28.1",
|
||||
"resolved": "https://registry.npmjs.org/@esbuild/linux-mips64el/-/linux-mips64el-0.28.1.tgz",
|
||||
"integrity": "sha512-mRObBZeHh2OxcBFPWE/FjylkRgZdYuiTR3vaTozquCGOH14iP9oN4x4Ge81CoIDYQrXmIxpFumJBu5MtZpnQJQ==",
|
||||
"cpu": [
|
||||
"mips64el"
|
||||
],
|
||||
@@ -432,9 +432,9 @@
|
||||
}
|
||||
},
|
||||
"node_modules/@esbuild/linux-ppc64": {
|
||||
"version": "0.28.0",
|
||||
"resolved": "https://registry.npmjs.org/@esbuild/linux-ppc64/-/linux-ppc64-0.28.0.tgz",
|
||||
"integrity": "sha512-bc0FE9wWeC0WBm49IQMPSPILRocGTQt3j5KPCA8os6VprfuJ7KD+5PzESSrJ6GmPIPJK965ZJHTUlSA6GNYEhg==",
|
||||
"version": "0.28.1",
|
||||
"resolved": "https://registry.npmjs.org/@esbuild/linux-ppc64/-/linux-ppc64-0.28.1.tgz",
|
||||
"integrity": "sha512-slScBsMAb3GFDcdrCgLwZtPYRoH2H/youv10QiZyRjmsP48fznoveWytSgCI/R0ZcUgpc0ZhIUEx6LHts8yrfQ==",
|
||||
"cpu": [
|
||||
"ppc64"
|
||||
],
|
||||
@@ -448,9 +448,9 @@
|
||||
}
|
||||
},
|
||||
"node_modules/@esbuild/linux-riscv64": {
|
||||
"version": "0.28.0",
|
||||
"resolved": "https://registry.npmjs.org/@esbuild/linux-riscv64/-/linux-riscv64-0.28.0.tgz",
|
||||
"integrity": "sha512-SQPZOwoTTT/HXFXQJG/vBX8sOFagGqvZyXcgLA3NhIqcBv1BJU1d46c0rGcrij2B56Z2rNiSLaZOYW5cUk7yLQ==",
|
||||
"version": "0.28.1",
|
||||
"resolved": "https://registry.npmjs.org/@esbuild/linux-riscv64/-/linux-riscv64-0.28.1.tgz",
|
||||
"integrity": "sha512-kw0owk1o0GFETUJyW0jc0G4Yzs0BHZn0JDZ8JRT088vjJYX777BAs1fDGxAC+q831qOs2DTC96mNsG2opdfyyQ==",
|
||||
"cpu": [
|
||||
"riscv64"
|
||||
],
|
||||
@@ -464,9 +464,9 @@
|
||||
}
|
||||
},
|
||||
"node_modules/@esbuild/linux-s390x": {
|
||||
"version": "0.28.0",
|
||||
"resolved": "https://registry.npmjs.org/@esbuild/linux-s390x/-/linux-s390x-0.28.0.tgz",
|
||||
"integrity": "sha512-SCfR0HN8CEEjnYnySJTd2cw0k9OHB/YFzt5zgJEwa+wL/T/raGWYMBqwDNAC6dqFKmJYZoQBRfHjgwLHGSrn3Q==",
|
||||
"version": "0.28.1",
|
||||
"resolved": "https://registry.npmjs.org/@esbuild/linux-s390x/-/linux-s390x-0.28.1.tgz",
|
||||
"integrity": "sha512-/lAIjX8aYFRByhh6L5rYtPEDRqa9de/4V/juOXcta5frjvzXO4/sqEtyytse0g3zZFuWu5cDN0MkLz2qRDD2Ag==",
|
||||
"cpu": [
|
||||
"s390x"
|
||||
],
|
||||
@@ -480,9 +480,9 @@
|
||||
}
|
||||
},
|
||||
"node_modules/@esbuild/linux-x64": {
|
||||
"version": "0.28.0",
|
||||
"resolved": "https://registry.npmjs.org/@esbuild/linux-x64/-/linux-x64-0.28.0.tgz",
|
||||
"integrity": "sha512-us0dSb9iFxIi8srnpl931Nvs65it/Jd2a2K3qs7fz2WfGPHqzfzZTfec7oxZJRNPXPnNYZtanmRc4AL/JwVzHQ==",
|
||||
"version": "0.28.1",
|
||||
"resolved": "https://registry.npmjs.org/@esbuild/linux-x64/-/linux-x64-0.28.1.tgz",
|
||||
"integrity": "sha512-u/anNYF2mmVOEDwLtnQ1wOr3EZ9sTNGLWrsYGYwHWzGA3Si84IOkHXlbWTD1NB+9/1lcnweYKO54uhxZydNzfA==",
|
||||
"cpu": [
|
||||
"x64"
|
||||
],
|
||||
@@ -496,9 +496,9 @@
|
||||
}
|
||||
},
|
||||
"node_modules/@esbuild/netbsd-arm64": {
|
||||
"version": "0.28.0",
|
||||
"resolved": "https://registry.npmjs.org/@esbuild/netbsd-arm64/-/netbsd-arm64-0.28.0.tgz",
|
||||
"integrity": "sha512-CR/RYotgtCKwtftMwJlUU7xCVNg3lMYZ0RzTmAHSfLCXw3NtZtNpswLEj/Kkf6kEL3Gw+BpOekRX0BYCtklhUw==",
|
||||
"version": "0.28.1",
|
||||
"resolved": "https://registry.npmjs.org/@esbuild/netbsd-arm64/-/netbsd-arm64-0.28.1.tgz",
|
||||
"integrity": "sha512-oks0DYbLwWMmaakTsCb+zL4E+aHRVLom9IJZOAthMQEPiQmydXHkziYEsGYRx0uNV/IjEKGAV941JzH02pflqw==",
|
||||
"cpu": [
|
||||
"arm64"
|
||||
],
|
||||
@@ -512,9 +512,9 @@
|
||||
}
|
||||
},
|
||||
"node_modules/@esbuild/netbsd-x64": {
|
||||
"version": "0.28.0",
|
||||
"resolved": "https://registry.npmjs.org/@esbuild/netbsd-x64/-/netbsd-x64-0.28.0.tgz",
|
||||
"integrity": "sha512-nU1yhmYutL+fQ71Kxnhg8uEOdC0pwEW9entHykTgEbna2pw2dkbFSMeqjjyHZoCmt8SBkOSvV+yNmm94aUrrqw==",
|
||||
"version": "0.28.1",
|
||||
"resolved": "https://registry.npmjs.org/@esbuild/netbsd-x64/-/netbsd-x64-0.28.1.tgz",
|
||||
"integrity": "sha512-aeL6lAnN89Hz43Mlh1G8ARasbuoYvSITDEx0tHh5b7jJnHcssqgjy9Yx430GDpmCa6OyrKoS0aNRjKundRizGg==",
|
||||
"cpu": [
|
||||
"x64"
|
||||
],
|
||||
@@ -528,9 +528,9 @@
|
||||
}
|
||||
},
|
||||
"node_modules/@esbuild/openbsd-arm64": {
|
||||
"version": "0.28.0",
|
||||
"resolved": "https://registry.npmjs.org/@esbuild/openbsd-arm64/-/openbsd-arm64-0.28.0.tgz",
|
||||
"integrity": "sha512-cXb5vApOsRsxsEl4mcZ1XY3D4DzcoMxR/nnc4IyqYs0rTI8ZKmW6kyyg+11Z8yvgMfAEldKzP7AdP64HnSC/6g==",
|
||||
"version": "0.28.1",
|
||||
"resolved": "https://registry.npmjs.org/@esbuild/openbsd-arm64/-/openbsd-arm64-0.28.1.tgz",
|
||||
"integrity": "sha512-MEFJe5C3R8pwXdZ5Y21oo6m7ePiS0d9pWucn99O/wvyJZChoIQKrQDxKrGeW8F5+T0okTHesAmDeiHDTIq0V/Q==",
|
||||
"cpu": [
|
||||
"arm64"
|
||||
],
|
||||
@@ -544,9 +544,9 @@
|
||||
}
|
||||
},
|
||||
"node_modules/@esbuild/openbsd-x64": {
|
||||
"version": "0.28.0",
|
||||
"resolved": "https://registry.npmjs.org/@esbuild/openbsd-x64/-/openbsd-x64-0.28.0.tgz",
|
||||
"integrity": "sha512-8wZM2qqtv9UP3mzy7HiGYNH/zjTA355mpeuA+859TyR+e+Tc08IHYpLJuMsfpDJwoLo1ikIJI8jC3GFjnRClzA==",
|
||||
"version": "0.28.1",
|
||||
"resolved": "https://registry.npmjs.org/@esbuild/openbsd-x64/-/openbsd-x64-0.28.1.tgz",
|
||||
"integrity": "sha512-i/ZLIOafE0Z8cI/XANJAixoJL/uRAoS2xOA3rb0xN+KK0K177cMAsQYkzHtBrtMXAKuAc7HGgcWiZ/sRC1Nxgw==",
|
||||
"cpu": [
|
||||
"x64"
|
||||
],
|
||||
@@ -560,9 +560,9 @@
|
||||
}
|
||||
},
|
||||
"node_modules/@esbuild/openharmony-arm64": {
|
||||
"version": "0.28.0",
|
||||
"resolved": "https://registry.npmjs.org/@esbuild/openharmony-arm64/-/openharmony-arm64-0.28.0.tgz",
|
||||
"integrity": "sha512-FLGfyizszcef5C3YtoyQDACyg95+dndv79i2EekILBofh5wpCa1KuBqOWKrEHZg3zrL3t5ouE5jgr94vA+Wb2w==",
|
||||
"version": "0.28.1",
|
||||
"resolved": "https://registry.npmjs.org/@esbuild/openharmony-arm64/-/openharmony-arm64-0.28.1.tgz",
|
||||
"integrity": "sha512-ge+Z7EXFNt2BO1oAMsVpiQ8EwndV9i1xXerAeTIK7AtPs3bKFXQM7nlRxDSIUIMeueR1CNXxqztLzdNeReKBJg==",
|
||||
"cpu": [
|
||||
"arm64"
|
||||
],
|
||||
@@ -576,9 +576,9 @@
|
||||
}
|
||||
},
|
||||
"node_modules/@esbuild/sunos-x64": {
|
||||
"version": "0.28.0",
|
||||
"resolved": "https://registry.npmjs.org/@esbuild/sunos-x64/-/sunos-x64-0.28.0.tgz",
|
||||
"integrity": "sha512-1ZgjUoEdHZZl/YlV76TSCz9Hqj9h9YmMGAgAPYd+q4SicWNX3G5GCyx9uhQWSLcbvPW8Ni7lj4gDa1T40akdlw==",
|
||||
"version": "0.28.1",
|
||||
"resolved": "https://registry.npmjs.org/@esbuild/sunos-x64/-/sunos-x64-0.28.1.tgz",
|
||||
"integrity": "sha512-BEjgtECkL3vY+SaSQ6nzVfiALUeFxpawyp8Jmf5PtYhf1Ug40N1h/hxlhts+f1FvSvarEigdxS3BlSMI2PJLcQ==",
|
||||
"cpu": [
|
||||
"x64"
|
||||
],
|
||||
@@ -592,9 +592,9 @@
|
||||
}
|
||||
},
|
||||
"node_modules/@esbuild/win32-arm64": {
|
||||
"version": "0.28.0",
|
||||
"resolved": "https://registry.npmjs.org/@esbuild/win32-arm64/-/win32-arm64-0.28.0.tgz",
|
||||
"integrity": "sha512-Q9StnDmQ/enxnpxCCLSg0oo4+34B9TdXpuyPeTedN/6+iXBJ4J+zwfQI28u/Jl40nOYAxGoNi7mFP40RUtkmUA==",
|
||||
"version": "0.28.1",
|
||||
"resolved": "https://registry.npmjs.org/@esbuild/win32-arm64/-/win32-arm64-0.28.1.tgz",
|
||||
"integrity": "sha512-lCv9eK/H6ZJWbE7bh2nw54CZ9M2nupBxJcTsdk/QQnWkdSjKGuxmmH8/GWrlT1eMmZfn4dGcCjRte397WqfQXA==",
|
||||
"cpu": [
|
||||
"arm64"
|
||||
],
|
||||
@@ -608,9 +608,9 @@
|
||||
}
|
||||
},
|
||||
"node_modules/@esbuild/win32-ia32": {
|
||||
"version": "0.28.0",
|
||||
"resolved": "https://registry.npmjs.org/@esbuild/win32-ia32/-/win32-ia32-0.28.0.tgz",
|
||||
"integrity": "sha512-zF3ag/gfiCe6U2iczcRzSYJKH1DCI+ByzSENHlM2FcDbEeo5Zd2C86Aq0tKUYAJJ1obRP84ymxIAksZUcdztHA==",
|
||||
"version": "0.28.1",
|
||||
"resolved": "https://registry.npmjs.org/@esbuild/win32-ia32/-/win32-ia32-0.28.1.tgz",
|
||||
"integrity": "sha512-zvb/mB2bSCoJOpoCBgYKKpX6YM6mJBlBUVUtVj41DlZJVEB6/0CKlRYxP5wWl1C1ILiCoAU5wZZ4q1P3qeS6Eg==",
|
||||
"cpu": [
|
||||
"ia32"
|
||||
],
|
||||
@@ -624,9 +624,9 @@
|
||||
}
|
||||
},
|
||||
"node_modules/@esbuild/win32-x64": {
|
||||
"version": "0.28.0",
|
||||
"resolved": "https://registry.npmjs.org/@esbuild/win32-x64/-/win32-x64-0.28.0.tgz",
|
||||
"integrity": "sha512-pEl1bO9mfAmIC+tW5btTmrKaujg3zGtUmWNdCw/xs70FBjwAL3o9OEKNHvNmnyylD6ubxUERiEhdsL0xBQ9efw==",
|
||||
"version": "0.28.1",
|
||||
"resolved": "https://registry.npmjs.org/@esbuild/win32-x64/-/win32-x64-0.28.1.tgz",
|
||||
"integrity": "sha512-bm4Mowrv+GXMlpWX++EcXw/iLyd1o3+bJkC2DkWXYVvgZCqD/bSj9ctZeAMC3cIxgjRVR2Dufaiu4YPxr5gW1A==",
|
||||
"cpu": [
|
||||
"x64"
|
||||
],
|
||||
@@ -1208,9 +1208,9 @@
|
||||
}
|
||||
},
|
||||
"node_modules/esbuild": {
|
||||
"version": "0.28.0",
|
||||
"resolved": "https://registry.npmjs.org/esbuild/-/esbuild-0.28.0.tgz",
|
||||
"integrity": "sha512-sNR9MHpXSUV/XB4zmsFKN+QgVG82Cc7+/aaxJ8Adi8hyOac+EXptIp45QBPaVyX3N70664wRbTcLTOemCAnyqw==",
|
||||
"version": "0.28.1",
|
||||
"resolved": "https://registry.npmjs.org/esbuild/-/esbuild-0.28.1.tgz",
|
||||
"integrity": "sha512-HrJrvZv5ayxBzPfwphOoNzkzOIIlifzk0KJrGK2c8R4+LKpMtpYLQeUdjnwjWv/LZlkH2laZk+4w78pi99D4Vw==",
|
||||
"hasInstallScript": true,
|
||||
"license": "MIT",
|
||||
"bin": {
|
||||
@@ -1220,32 +1220,32 @@
|
||||
"node": ">=18"
|
||||
},
|
||||
"optionalDependencies": {
|
||||
"@esbuild/aix-ppc64": "0.28.0",
|
||||
"@esbuild/android-arm": "0.28.0",
|
||||
"@esbuild/android-arm64": "0.28.0",
|
||||
"@esbuild/android-x64": "0.28.0",
|
||||
"@esbuild/darwin-arm64": "0.28.0",
|
||||
"@esbuild/darwin-x64": "0.28.0",
|
||||
"@esbuild/freebsd-arm64": "0.28.0",
|
||||
"@esbuild/freebsd-x64": "0.28.0",
|
||||
"@esbuild/linux-arm": "0.28.0",
|
||||
"@esbuild/linux-arm64": "0.28.0",
|
||||
"@esbuild/linux-ia32": "0.28.0",
|
||||
"@esbuild/linux-loong64": "0.28.0",
|
||||
"@esbuild/linux-mips64el": "0.28.0",
|
||||
"@esbuild/linux-ppc64": "0.28.0",
|
||||
"@esbuild/linux-riscv64": "0.28.0",
|
||||
"@esbuild/linux-s390x": "0.28.0",
|
||||
"@esbuild/linux-x64": "0.28.0",
|
||||
"@esbuild/netbsd-arm64": "0.28.0",
|
||||
"@esbuild/netbsd-x64": "0.28.0",
|
||||
"@esbuild/openbsd-arm64": "0.28.0",
|
||||
"@esbuild/openbsd-x64": "0.28.0",
|
||||
"@esbuild/openharmony-arm64": "0.28.0",
|
||||
"@esbuild/sunos-x64": "0.28.0",
|
||||
"@esbuild/win32-arm64": "0.28.0",
|
||||
"@esbuild/win32-ia32": "0.28.0",
|
||||
"@esbuild/win32-x64": "0.28.0"
|
||||
"@esbuild/aix-ppc64": "0.28.1",
|
||||
"@esbuild/android-arm": "0.28.1",
|
||||
"@esbuild/android-arm64": "0.28.1",
|
||||
"@esbuild/android-x64": "0.28.1",
|
||||
"@esbuild/darwin-arm64": "0.28.1",
|
||||
"@esbuild/darwin-x64": "0.28.1",
|
||||
"@esbuild/freebsd-arm64": "0.28.1",
|
||||
"@esbuild/freebsd-x64": "0.28.1",
|
||||
"@esbuild/linux-arm": "0.28.1",
|
||||
"@esbuild/linux-arm64": "0.28.1",
|
||||
"@esbuild/linux-ia32": "0.28.1",
|
||||
"@esbuild/linux-loong64": "0.28.1",
|
||||
"@esbuild/linux-mips64el": "0.28.1",
|
||||
"@esbuild/linux-ppc64": "0.28.1",
|
||||
"@esbuild/linux-riscv64": "0.28.1",
|
||||
"@esbuild/linux-s390x": "0.28.1",
|
||||
"@esbuild/linux-x64": "0.28.1",
|
||||
"@esbuild/netbsd-arm64": "0.28.1",
|
||||
"@esbuild/netbsd-x64": "0.28.1",
|
||||
"@esbuild/openbsd-arm64": "0.28.1",
|
||||
"@esbuild/openbsd-x64": "0.28.1",
|
||||
"@esbuild/openharmony-arm64": "0.28.1",
|
||||
"@esbuild/sunos-x64": "0.28.1",
|
||||
"@esbuild/win32-arm64": "0.28.1",
|
||||
"@esbuild/win32-ia32": "0.28.1",
|
||||
"@esbuild/win32-x64": "0.28.1"
|
||||
}
|
||||
},
|
||||
"node_modules/escape-html": {
|
||||
|
||||
@@ -101,6 +101,28 @@
|
||||
"contextWindow": 200000,
|
||||
"maxTokens": 64000
|
||||
},
|
||||
{
|
||||
"id": "claude-haiku-4-5",
|
||||
"name": "Claude Haiku 4.5",
|
||||
"reasoning": true,
|
||||
"input": ["text", "image"],
|
||||
"mediaInput": {
|
||||
"image": { "maxSidePx": 1568, "preferredSidePx": 1568, "tokenMode": "provider" }
|
||||
},
|
||||
"contextWindow": 200000,
|
||||
"maxTokens": 64000
|
||||
},
|
||||
{
|
||||
"id": "claude-haiku-4-5-20251001",
|
||||
"name": "Claude Haiku 4.5",
|
||||
"reasoning": true,
|
||||
"input": ["text", "image"],
|
||||
"mediaInput": {
|
||||
"image": { "maxSidePx": 1568, "preferredSidePx": 1568, "tokenMode": "provider" }
|
||||
},
|
||||
"contextWindow": 200000,
|
||||
"maxTokens": 64000
|
||||
},
|
||||
{
|
||||
"id": "claude-sonnet-4-6",
|
||||
"name": "Claude Sonnet 4.6",
|
||||
|
||||
57
extensions/anthropic/openclaw.plugin.test.ts
Normal file
57
extensions/anthropic/openclaw.plugin.test.ts
Normal file
@@ -0,0 +1,57 @@
|
||||
// Anthropic tests cover provider manifest model catalog behavior.
|
||||
import { readFileSync } from "node:fs";
|
||||
import { describe, expect, it } from "vitest";
|
||||
|
||||
type AnthropicManifest = {
|
||||
modelCatalog?: {
|
||||
providers?: {
|
||||
anthropic?: {
|
||||
models?: Array<{
|
||||
id?: string;
|
||||
name?: string;
|
||||
reasoning?: boolean;
|
||||
input?: string[];
|
||||
mediaInput?: {
|
||||
image?: {
|
||||
maxSidePx?: number;
|
||||
preferredSidePx?: number;
|
||||
tokenMode?: string;
|
||||
};
|
||||
};
|
||||
contextWindow?: number;
|
||||
maxTokens?: number;
|
||||
}>;
|
||||
};
|
||||
};
|
||||
discovery?: Record<string, string>;
|
||||
};
|
||||
};
|
||||
|
||||
const manifest = JSON.parse(
|
||||
readFileSync(new URL("./openclaw.plugin.json", import.meta.url), "utf8"),
|
||||
) as AnthropicManifest;
|
||||
|
||||
describe("Anthropic plugin manifest", () => {
|
||||
it("resolves both official Claude Haiku 4.5 API identifiers from the static catalog", () => {
|
||||
expect(manifest.modelCatalog?.discovery?.anthropic).toBe("static");
|
||||
|
||||
const models = manifest.modelCatalog?.providers?.anthropic?.models ?? [];
|
||||
for (const id of ["claude-haiku-4-5", "claude-haiku-4-5-20251001"]) {
|
||||
expect(models.find((model) => model.id === id)).toEqual({
|
||||
id,
|
||||
name: "Claude Haiku 4.5",
|
||||
reasoning: true,
|
||||
input: ["text", "image"],
|
||||
mediaInput: {
|
||||
image: {
|
||||
maxSidePx: 1568,
|
||||
preferredSidePx: 1568,
|
||||
tokenMode: "provider",
|
||||
},
|
||||
},
|
||||
contextWindow: 200000,
|
||||
maxTokens: 64000,
|
||||
});
|
||||
}
|
||||
});
|
||||
});
|
||||
@@ -120,6 +120,7 @@ describe("canvas host", () => {
|
||||
};
|
||||
let createCanvasHostHandler: typeof import("./server.js").createCanvasHostHandler;
|
||||
let startCanvasHost: typeof import("./server.js").startCanvasHost;
|
||||
let canvasLiveReloadMaxInboundMessageBytes = 0;
|
||||
let WebSocketServerClass: typeof import("ws").WebSocketServer;
|
||||
let watcherState: ReturnType<typeof createMockWatcherState>;
|
||||
let fixtureRoot = "";
|
||||
@@ -162,7 +163,10 @@ describe("canvas host", () => {
|
||||
};
|
||||
});
|
||||
vi.resetModules();
|
||||
({ createCanvasHostHandler, startCanvasHost } = await import("./server.js"));
|
||||
const serverModule = await import("./server.js");
|
||||
({ createCanvasHostHandler, startCanvasHost } = serverModule);
|
||||
canvasLiveReloadMaxInboundMessageBytes =
|
||||
serverModule.CANVAS_LIVE_RELOAD_MAX_INBOUND_MESSAGE_BYTES;
|
||||
const wsModule = await vi.importActual<typeof import("ws")>("ws");
|
||||
WebSocketServerClass = wsModule.WebSocketServer;
|
||||
fixtureRoot = await fs.mkdtemp(path.join(os.tmpdir(), "openclaw-canvas-fixtures-"));
|
||||
@@ -221,6 +225,54 @@ describe("canvas host", () => {
|
||||
}
|
||||
});
|
||||
|
||||
it("caps live reload WebSocket inbound payloads", async () => {
|
||||
const dir = await createCaseDir();
|
||||
const constructorOptions: unknown[] = [];
|
||||
let connectionHandler: ((socket: TrackingWebSocket) => void) | undefined;
|
||||
class CapturingWebSocketServer {
|
||||
on(event: string, cb: (socket: TrackingWebSocket) => void) {
|
||||
if (event === "connection") {
|
||||
connectionHandler = cb;
|
||||
}
|
||||
return this;
|
||||
}
|
||||
|
||||
close(cb?: () => void) {
|
||||
cb?.();
|
||||
}
|
||||
|
||||
constructor(options: unknown) {
|
||||
constructorOptions.push(options);
|
||||
}
|
||||
}
|
||||
|
||||
const handler = await createTestCanvasHostHandler(dir, {
|
||||
webSocketServerClass:
|
||||
CapturingWebSocketServer as unknown as typeof import("ws").WebSocketServer,
|
||||
});
|
||||
|
||||
try {
|
||||
expect(constructorOptions[0]).toMatchObject({
|
||||
noServer: true,
|
||||
maxPayload: canvasLiveReloadMaxInboundMessageBytes,
|
||||
});
|
||||
const socketHandlers: string[] = [];
|
||||
const socket: TrackingWebSocket = {
|
||||
sent: [],
|
||||
on: (event) => {
|
||||
socketHandlers.push(event);
|
||||
return socket;
|
||||
},
|
||||
send: vi.fn(),
|
||||
};
|
||||
expect(connectionHandler).toBeDefined();
|
||||
connectionHandler?.(socket);
|
||||
expect(socketHandlers).toEqual(expect.arrayContaining(["error", "close"]));
|
||||
} finally {
|
||||
await handler.close();
|
||||
}
|
||||
});
|
||||
|
||||
it("falls back to the default mount when the configured base path is malformed", async () => {
|
||||
const dir = await createCaseDir();
|
||||
await fs.writeFile(path.join(dir, "index.html"), "<html><body>fallback</body></html>", "utf8");
|
||||
|
||||
@@ -30,6 +30,8 @@ import {
|
||||
} from "./a2ui-shared.js";
|
||||
import { normalizeUrlPath, resolveFileWithinRoot } from "./file-resolver.js";
|
||||
|
||||
export const CANVAS_LIVE_RELOAD_MAX_INBOUND_MESSAGE_BYTES = 64 * 1024;
|
||||
|
||||
type ChokidarWatch = typeof import("chokidar").watch;
|
||||
|
||||
/** Options for Canvas host creation. */
|
||||
@@ -276,11 +278,22 @@ export async function createCanvasHostHandler(
|
||||
const writeStabilityThresholdMs = testMode ? 12 : 75;
|
||||
const writePollIntervalMs = testMode ? 5 : 10;
|
||||
const WebSocketServerClass = opts.webSocketServerClass ?? WebSocketServer;
|
||||
const wss = liveReload ? new WebSocketServerClass({ noServer: true }) : null;
|
||||
const wss = liveReload
|
||||
? new WebSocketServerClass({
|
||||
noServer: true,
|
||||
// Live reload clients never need to send application payloads; cap frames
|
||||
// before ws buffers oversized input on this long-lived upgrade route.
|
||||
maxPayload: CANVAS_LIVE_RELOAD_MAX_INBOUND_MESSAGE_BYTES,
|
||||
})
|
||||
: null;
|
||||
const sockets = new Set<WebSocket>();
|
||||
if (wss) {
|
||||
wss.on("connection", (ws) => {
|
||||
sockets.add(ws);
|
||||
// ws emits error for maxPayload rejections; close handles final cleanup.
|
||||
ws.on("error", () => {
|
||||
sockets.delete(ws);
|
||||
});
|
||||
ws.on("close", () => sockets.delete(ws));
|
||||
});
|
||||
}
|
||||
|
||||
@@ -1,6 +1,7 @@
|
||||
// Codex tests cover sandbox exec server plugin behavior.
|
||||
import { afterEach, describe, expect, it, vi } from "vitest";
|
||||
import {
|
||||
CODEX_SANDBOX_EXEC_SERVER_MAX_INBOUND_MESSAGE_BYTES,
|
||||
closeCodexSandboxExecServersForTests,
|
||||
ensureCodexSandboxExecServerEnvironment,
|
||||
releaseCodexSandboxExecServerEnvironment,
|
||||
@@ -191,6 +192,22 @@ describe("OpenClaw Codex sandbox exec-server", () => {
|
||||
socket.close();
|
||||
});
|
||||
|
||||
it("closes oversized sandbox exec-server frames before JSON-RPC parsing", async () => {
|
||||
const sandbox = createSandboxContext({});
|
||||
const client = createClient();
|
||||
|
||||
await ensureCodexSandboxExecServerEnvironment({
|
||||
client: client as never,
|
||||
sandbox,
|
||||
});
|
||||
const socket = await openSocket(execServerUrlFromClient(client));
|
||||
const closed = waitForSocketClose(socket);
|
||||
|
||||
socket.send(Buffer.alloc(CODEX_SANDBOX_EXEC_SERVER_MAX_INBOUND_MESSAGE_BYTES + 1));
|
||||
|
||||
await expect(closed).resolves.toEqual({ code: 1009 });
|
||||
});
|
||||
|
||||
it("rejects unsupported arg0 overrides instead of dropping them", async () => {
|
||||
const buildExecSpec = vi.fn(async () => ({
|
||||
argv: [process.execPath, "-e", ""],
|
||||
@@ -441,6 +458,26 @@ describe("OpenClaw Codex sandbox exec-server", () => {
|
||||
await expect(waitForSocketClose(socket)).resolves.toEqual({ code: 1008 });
|
||||
});
|
||||
|
||||
it("handles oversized frames from unauthorized WebSocket clients", async () => {
|
||||
const sandbox = createSandboxContext({});
|
||||
const client = createClient();
|
||||
await ensureCodexSandboxExecServerEnvironment({
|
||||
client: client as never,
|
||||
sandbox,
|
||||
});
|
||||
const unauthorizedUrl = execServerUrlFromClient(client).replace(
|
||||
/\/openclaw-[^/?#]+/u,
|
||||
"/wrong",
|
||||
);
|
||||
const socket = await openSocket(unauthorizedUrl);
|
||||
const closed = waitForSocketClose(socket);
|
||||
|
||||
socket.send(Buffer.alloc(CODEX_SANDBOX_EXEC_SERVER_MAX_INBOUND_MESSAGE_BYTES + 1));
|
||||
|
||||
const closeResult = await closed;
|
||||
expect([1008, 1009]).toContain(closeResult.code);
|
||||
});
|
||||
|
||||
it("closes the exec-server when its sandbox environment is released", async () => {
|
||||
const sandbox = createSandboxContext({});
|
||||
const client = createClient();
|
||||
|
||||
@@ -48,6 +48,7 @@ export type CodexSandboxExecEnvironment = {
|
||||
};
|
||||
|
||||
const SANDBOX_EXEC_SERVERS = new Map<string, Promise<OpenClawExecServer>>();
|
||||
export const CODEX_SANDBOX_EXEC_SERVER_MAX_INBOUND_MESSAGE_BYTES = 100 * 1024 * 1024;
|
||||
|
||||
/** Closes all cached sandbox exec-server instances for deterministic tests. */
|
||||
export async function closeCodexSandboxExecServersForTests(): Promise<void> {
|
||||
@@ -193,7 +194,13 @@ function startAndRememberOpenClawExecServer(sandbox: SandboxContext): Promise<Op
|
||||
}
|
||||
|
||||
async function startOpenClawExecServer(sandbox: SandboxContext): Promise<OpenClawExecServer> {
|
||||
const server = new WebSocketServer({ host: "127.0.0.1", port: 0 });
|
||||
const server = new WebSocketServer({
|
||||
host: "127.0.0.1",
|
||||
port: 0,
|
||||
// Match ws' historical default: Codex fs/writeFile sends one base64 JSON-RPC
|
||||
// frame, while the socket error handler below makes oversize frames nonfatal.
|
||||
maxPayload: CODEX_SANDBOX_EXEC_SERVER_MAX_INBOUND_MESSAGE_BYTES,
|
||||
});
|
||||
await once(server, "listening");
|
||||
const address = server.address();
|
||||
if (!address || typeof address === "string") {
|
||||
@@ -212,6 +219,8 @@ async function startOpenClawExecServer(sandbox: SandboxContext): Promise<OpenCla
|
||||
server,
|
||||
};
|
||||
server.on("connection", (socket, request) => {
|
||||
// ws emits error for maxPayload rejections before auth or JSON-RPC sees the frame.
|
||||
socket.on("error", handleExecServerSocketError);
|
||||
if (!isAuthorizedExecServerRequest(execServer, request)) {
|
||||
socket.close(1008, "unauthorized");
|
||||
return;
|
||||
@@ -286,6 +295,10 @@ function handleConnection(execServer: OpenClawExecServer, socket: WebSocket): vo
|
||||
});
|
||||
}
|
||||
|
||||
function handleExecServerSocketError(error: unknown): void {
|
||||
embeddedAgentLog.debug("codex sandbox exec-server websocket failed", { error });
|
||||
}
|
||||
|
||||
async function handleMessage(
|
||||
execServer: OpenClawExecServer,
|
||||
processes: Map<string, ManagedProcess>,
|
||||
|
||||
@@ -24,6 +24,75 @@ function hasDiscordComponentObjectKeys(value: unknown): value is Record<string,
|
||||
);
|
||||
}
|
||||
|
||||
function readDiscordThreadArchiveTimestamp(thread: unknown): string | undefined {
|
||||
if (!thread || typeof thread !== "object" || Array.isArray(thread)) {
|
||||
return undefined;
|
||||
}
|
||||
const record = thread as Record<string, unknown>;
|
||||
const metadata = record.thread_metadata;
|
||||
if (metadata && typeof metadata === "object" && !Array.isArray(metadata)) {
|
||||
const archiveTimestamp = (metadata as Record<string, unknown>).archive_timestamp;
|
||||
if (typeof archiveTimestamp === "string" && archiveTimestamp.trim()) {
|
||||
return archiveTimestamp;
|
||||
}
|
||||
}
|
||||
return undefined;
|
||||
}
|
||||
|
||||
type DiscordThreadListActionResult = {
|
||||
ok: true;
|
||||
threads: unknown;
|
||||
complete: boolean;
|
||||
hasMore: boolean;
|
||||
returnedCount: number;
|
||||
source: "discord.threadList.archived" | "discord.threadList.active";
|
||||
query: {
|
||||
guildId: string;
|
||||
channelId?: string;
|
||||
includeArchived: boolean;
|
||||
before?: string;
|
||||
limit?: number;
|
||||
};
|
||||
nextBefore?: string;
|
||||
};
|
||||
|
||||
function normalizeDiscordThreadListActionResult(params: {
|
||||
value: unknown;
|
||||
includeArchived: boolean;
|
||||
channelId?: string;
|
||||
guildId: string;
|
||||
limit?: number;
|
||||
before?: string;
|
||||
}): DiscordThreadListActionResult {
|
||||
const record =
|
||||
params.value && typeof params.value === "object" && !Array.isArray(params.value)
|
||||
? (params.value as Record<string, unknown>)
|
||||
: undefined;
|
||||
const threadItems = Array.isArray(record?.threads) ? record.threads : [];
|
||||
const hasMore = record?.has_more === true;
|
||||
const nextBefore =
|
||||
params.includeArchived && hasMore
|
||||
? readDiscordThreadArchiveTimestamp(threadItems[threadItems.length - 1])
|
||||
: undefined;
|
||||
|
||||
return {
|
||||
ok: true,
|
||||
threads: params.value,
|
||||
complete: !hasMore,
|
||||
hasMore,
|
||||
returnedCount: threadItems.length,
|
||||
source: params.includeArchived ? "discord.threadList.archived" : "discord.threadList.active",
|
||||
query: {
|
||||
guildId: params.guildId,
|
||||
...(params.channelId ? { channelId: params.channelId } : {}),
|
||||
includeArchived: params.includeArchived,
|
||||
...(params.before ? { before: params.before } : {}),
|
||||
...(params.limit !== undefined ? { limit: params.limit } : {}),
|
||||
},
|
||||
...(nextBefore ? { nextBefore } : {}),
|
||||
};
|
||||
}
|
||||
|
||||
async function appendDiscordThreadRenameResult(
|
||||
ctx: DiscordMessagingActionContext,
|
||||
params: {
|
||||
@@ -306,7 +375,16 @@ export async function handleDiscordMessageSendAction(ctx: DiscordMessagingAction
|
||||
},
|
||||
ctx.withOpts(),
|
||||
);
|
||||
return jsonResult({ ok: true, threads });
|
||||
return jsonResult(
|
||||
normalizeDiscordThreadListActionResult({
|
||||
value: threads,
|
||||
guildId,
|
||||
channelId,
|
||||
includeArchived: includeArchived === true,
|
||||
before,
|
||||
limit,
|
||||
}),
|
||||
);
|
||||
}
|
||||
case "threadReply": {
|
||||
if (!ctx.isActionEnabled("threads")) {
|
||||
|
||||
@@ -101,6 +101,7 @@ const {
|
||||
kickMemberDiscord,
|
||||
listGuildChannelsDiscord,
|
||||
listPinsDiscord,
|
||||
listThreadsDiscord,
|
||||
moveChannelDiscord,
|
||||
reactMessageDiscord,
|
||||
readMessagesDiscord,
|
||||
@@ -271,6 +272,138 @@ describe("handleDiscordMessagingAction", () => {
|
||||
]);
|
||||
});
|
||||
|
||||
it("surfaces incomplete archived thread pages at the action boundary", async () => {
|
||||
listThreadsDiscord.mockResolvedValueOnce({
|
||||
threads: [
|
||||
{
|
||||
id: "thread-1",
|
||||
name: "Old project",
|
||||
thread_metadata: {
|
||||
archive_timestamp: "2026-05-25T17:00:00.000Z",
|
||||
},
|
||||
},
|
||||
],
|
||||
members: [],
|
||||
has_more: true,
|
||||
});
|
||||
|
||||
const result = await handleMessagingAction(
|
||||
"threadList",
|
||||
{
|
||||
guildId: "G1",
|
||||
channelId: "C1",
|
||||
includeArchived: true,
|
||||
before: "2026-05-26T17:00:00.000Z",
|
||||
limit: 1,
|
||||
},
|
||||
enableAllActions,
|
||||
);
|
||||
|
||||
expect(mockCall(listThreadsDiscord, "listThreadsDiscord")).toEqual([
|
||||
{
|
||||
guildId: "G1",
|
||||
channelId: "C1",
|
||||
includeArchived: true,
|
||||
before: "2026-05-26T17:00:00.000Z",
|
||||
limit: 1,
|
||||
},
|
||||
{ cfg: DISCORD_TEST_CFG },
|
||||
]);
|
||||
expect(result.details).toMatchObject({
|
||||
ok: true,
|
||||
complete: false,
|
||||
hasMore: true,
|
||||
returnedCount: 1,
|
||||
source: "discord.threadList.archived",
|
||||
nextBefore: "2026-05-25T17:00:00.000Z",
|
||||
query: {
|
||||
guildId: "G1",
|
||||
channelId: "C1",
|
||||
includeArchived: true,
|
||||
before: "2026-05-26T17:00:00.000Z",
|
||||
limit: 1,
|
||||
},
|
||||
});
|
||||
expect((result.details as { threads?: unknown }).threads).toEqual({
|
||||
threads: [
|
||||
{
|
||||
id: "thread-1",
|
||||
name: "Old project",
|
||||
thread_metadata: {
|
||||
archive_timestamp: "2026-05-25T17:00:00.000Z",
|
||||
},
|
||||
},
|
||||
],
|
||||
members: [],
|
||||
has_more: true,
|
||||
});
|
||||
});
|
||||
|
||||
it("omits archived thread pagination cursors when Discord omits archive timestamps", async () => {
|
||||
listThreadsDiscord.mockResolvedValueOnce({
|
||||
threads: [
|
||||
{
|
||||
id: "thread-without-archive-timestamp",
|
||||
name: "Legacy project",
|
||||
},
|
||||
],
|
||||
members: [],
|
||||
has_more: true,
|
||||
});
|
||||
|
||||
const result = await handleMessagingAction(
|
||||
"threadList",
|
||||
{
|
||||
guildId: "G1",
|
||||
channelId: "C1",
|
||||
includeArchived: true,
|
||||
limit: 1,
|
||||
},
|
||||
enableAllActions,
|
||||
);
|
||||
|
||||
expect(result.details).toMatchObject({
|
||||
ok: true,
|
||||
complete: false,
|
||||
hasMore: true,
|
||||
returnedCount: 1,
|
||||
source: "discord.threadList.archived",
|
||||
});
|
||||
expect(result.details).not.toHaveProperty("nextBefore");
|
||||
});
|
||||
|
||||
it("marks active thread results complete when Discord returns no pagination state", async () => {
|
||||
listThreadsDiscord.mockResolvedValueOnce({
|
||||
threads: [{ id: "thread-active", name: "Current project" }],
|
||||
members: [{ id: "member-1" }],
|
||||
});
|
||||
|
||||
const result = await handleMessagingAction(
|
||||
"threadList",
|
||||
{
|
||||
guildId: "G1",
|
||||
},
|
||||
enableAllActions,
|
||||
);
|
||||
|
||||
expect(result.details).toMatchObject({
|
||||
ok: true,
|
||||
complete: true,
|
||||
hasMore: false,
|
||||
returnedCount: 1,
|
||||
source: "discord.threadList.active",
|
||||
query: {
|
||||
guildId: "G1",
|
||||
includeArchived: false,
|
||||
},
|
||||
});
|
||||
expect((result.details as { threads?: unknown }).threads).toEqual({
|
||||
threads: [{ id: "thread-active", name: "Current project" }],
|
||||
members: [{ id: "member-1" }],
|
||||
});
|
||||
expect(result.details).not.toHaveProperty("nextBefore");
|
||||
});
|
||||
|
||||
it("resolves Discord DM targets for reaction adds", async () => {
|
||||
const resolveReactionTarget = vi.fn(async () => "DM1");
|
||||
discordMessagingActionRuntime.resolveDiscordReactionTargetChannelId = resolveReactionTarget;
|
||||
|
||||
@@ -144,19 +144,19 @@ describe("fireworks provider plugin", () => {
|
||||
expect(resolved?.reasoning).toBe(false);
|
||||
});
|
||||
|
||||
it("disables reasoning metadata for Fireworks Kimi k2.6 dynamic models", async () => {
|
||||
it("defers manifest catalog models to core static-catalog resolution", async () => {
|
||||
const provider = await registerSingleProviderPlugin(fireworksPlugin);
|
||||
const resolved = provider.resolveDynamicModel?.(
|
||||
createProviderDynamicModelContext({
|
||||
provider: "fireworks",
|
||||
modelId: "accounts/fireworks/models/kimi-k2p6",
|
||||
models: [createFireworksDefaultRuntimeModel({ reasoning: false })],
|
||||
}),
|
||||
);
|
||||
for (const modelId of [FIREWORKS_K2_6_MODEL_ID, FIREWORKS_DEFAULT_MODEL_ID]) {
|
||||
const resolved = provider.resolveDynamicModel?.(
|
||||
createProviderDynamicModelContext({
|
||||
provider: "fireworks",
|
||||
modelId,
|
||||
models: [createFireworksDefaultRuntimeModel({ reasoning: false })],
|
||||
}),
|
||||
);
|
||||
|
||||
expect(resolved?.provider).toBe("fireworks");
|
||||
expect(resolved?.id).toBe("accounts/fireworks/models/kimi-k2p6");
|
||||
expect(resolved?.reasoning).toBe(false);
|
||||
expect(resolved).toBeUndefined();
|
||||
}
|
||||
});
|
||||
|
||||
it("exposes off-only thinking policy for Fireworks Kimi models", async () => {
|
||||
|
||||
@@ -15,11 +15,13 @@ import {
|
||||
FIREWORKS_DEFAULT_CONTEXT_WINDOW,
|
||||
FIREWORKS_DEFAULT_MAX_TOKENS,
|
||||
FIREWORKS_DEFAULT_MODEL_ID,
|
||||
isFireworksCatalogModelId,
|
||||
} from "./provider-catalog.js";
|
||||
import { wrapFireworksProviderStream } from "./stream.js";
|
||||
import { resolveFireworksThinkingProfile } from "./thinking-policy.js";
|
||||
|
||||
const PROVIDER_ID = "fireworks";
|
||||
|
||||
function isFireworksGlmModelId(modelId: string): boolean {
|
||||
const normalized = modelId.trim().toLowerCase();
|
||||
const lastSegment = normalized.split("/").pop() ?? normalized;
|
||||
@@ -35,6 +37,11 @@ function resolveFireworksDynamicModel(ctx: ProviderResolveDynamicModelContext) {
|
||||
if (!modelId) {
|
||||
return undefined;
|
||||
}
|
||||
|
||||
if (isFireworksCatalogModelId(modelId)) {
|
||||
return undefined;
|
||||
}
|
||||
|
||||
const isKimiModel = isFireworksKimiModelId(modelId);
|
||||
const input = resolveFireworksDynamicInput(modelId);
|
||||
|
||||
|
||||
@@ -37,6 +37,7 @@
|
||||
{
|
||||
"id": "accounts/fireworks/models/kimi-k2p6",
|
||||
"name": "Kimi K2.6",
|
||||
"reasoning": false,
|
||||
"input": ["text", "image"],
|
||||
"contextWindow": 262144,
|
||||
"maxTokens": 262144,
|
||||
@@ -50,6 +51,7 @@
|
||||
{
|
||||
"id": "accounts/fireworks/routers/kimi-k2p5-turbo",
|
||||
"name": "Kimi K2.5 Turbo (Fire Pass)",
|
||||
"reasoning": false,
|
||||
"input": ["text", "image"],
|
||||
"contextWindow": 256000,
|
||||
"maxTokens": 256000,
|
||||
|
||||
@@ -31,16 +31,12 @@ export const FIREWORKS_DEFAULT_MAX_TOKENS = FIREWORKS_DEFAULT_MODEL.maxTokens;
|
||||
export const FIREWORKS_K2_6_CONTEXT_WINDOW = FIREWORKS_K2_6_MODEL.contextWindow;
|
||||
export const FIREWORKS_K2_6_MAX_TOKENS = FIREWORKS_K2_6_MODEL.maxTokens;
|
||||
|
||||
function cloneFireworksCatalogModel(model: ModelDefinitionConfig): ModelDefinitionConfig {
|
||||
return {
|
||||
...model,
|
||||
input: [...model.input],
|
||||
cost: { ...model.cost },
|
||||
};
|
||||
export function isFireworksCatalogModelId(modelId: string): boolean {
|
||||
return FIREWORKS_MANIFEST_PROVIDER.models.some((model) => model.id === modelId);
|
||||
}
|
||||
|
||||
export function buildFireworksCatalogModels(): ModelDefinitionConfig[] {
|
||||
return FIREWORKS_MANIFEST_PROVIDER.models.map(cloneFireworksCatalogModel);
|
||||
return FIREWORKS_MANIFEST_PROVIDER.models.map((model) => structuredClone(model));
|
||||
}
|
||||
|
||||
export function buildFireworksProvider(): ModelProviderConfig {
|
||||
|
||||
@@ -56,6 +56,10 @@ function isCopilotGeminiModelId(modelId: string): boolean {
|
||||
return /(?:^|[-_.])gemini(?:$|[-_.])/.test(modelId);
|
||||
}
|
||||
|
||||
function isCopilotClaude45ModelId(modelId: string): boolean {
|
||||
return /^claude-(?:haiku|opus|sonnet)-4[.-]5(?:$|[-.])/.test(modelId);
|
||||
}
|
||||
|
||||
export function resolveCopilotTransportApi(modelId: string): CopilotRuntimeApi {
|
||||
const normalized = normalizeOptionalLowercaseString(modelId) ?? "";
|
||||
if (normalized.includes("claude")) {
|
||||
@@ -71,7 +75,15 @@ export function resolveCopilotModelCompat(
|
||||
modelId: string,
|
||||
): ModelDefinitionConfig["compat"] | undefined {
|
||||
const normalized = normalizeOptionalLowercaseString(modelId) ?? "";
|
||||
return isCopilotGeminiModelId(normalized) ? { ...COPILOT_CHAT_COMPLETIONS_COMPAT } : undefined;
|
||||
if (isCopilotGeminiModelId(normalized)) {
|
||||
return { ...COPILOT_CHAT_COMPLETIONS_COMPAT };
|
||||
}
|
||||
// Copilot's Claude 4.5 endpoints reject Anthropic's eager tool extension,
|
||||
// while current Claude 4.6+ endpoints accept it.
|
||||
if (isCopilotClaude45ModelId(normalized)) {
|
||||
return { supportsEagerToolInputStreaming: false };
|
||||
}
|
||||
return undefined;
|
||||
}
|
||||
|
||||
function compatSupportsEffort(
|
||||
|
||||
@@ -90,8 +90,18 @@ describe("github-copilot model defaults", () => {
|
||||
const def = buildCopilotModelDefinition("claude-sonnet-4.6");
|
||||
expect(def.id).toBe("claude-sonnet-4.6");
|
||||
expect(def.api).toBe("anthropic-messages");
|
||||
expect(def.compat).toBeUndefined();
|
||||
});
|
||||
|
||||
it.each(["claude-haiku-4.5", "claude-sonnet-4-5"])(
|
||||
"disables eager tool streaming for Copilot Claude 4.5 model %s",
|
||||
(modelId) => {
|
||||
expect(buildCopilotModelDefinition(modelId).compat).toEqual({
|
||||
supportsEagerToolInputStreaming: false,
|
||||
});
|
||||
},
|
||||
);
|
||||
|
||||
it("uses static metadata overrides for gpt-5.5 fallback rows", () => {
|
||||
const def = buildCopilotModelDefinition("gpt-5.5");
|
||||
expect(def).toEqual({
|
||||
@@ -243,6 +253,12 @@ describe("resolveCopilotForwardCompatModel", () => {
|
||||
expect((result as unknown as Record<string, unknown>).input).toEqual(["text", "image"]);
|
||||
});
|
||||
|
||||
it("disables eager tool streaming for synthetic Copilot Claude 4.5 models", () => {
|
||||
const result = requireResolvedModel(createMockCtx("claude-haiku-4.5"));
|
||||
expect(result.api).toBe("anthropic-messages");
|
||||
expect(result.compat).toEqual({ supportsEagerToolInputStreaming: false });
|
||||
});
|
||||
|
||||
it("creates synthetic Gemini models with Chat Completions compatibility", () => {
|
||||
const result = requireResolvedModel(createMockCtx("gemini-3.1-pro-preview"));
|
||||
expect((result as unknown as Record<string, unknown>).api).toBe("openai-completions");
|
||||
@@ -620,6 +636,7 @@ describe("fetchCopilotModelCatalog", () => {
|
||||
const opus45 = out.find((m) => m.id === "claude-opus-4-5");
|
||||
expect(opus45?.thinkingLevelMap).toEqual({ xhigh: null, max: null });
|
||||
expect(opus45?.compat).toEqual({
|
||||
supportsEagerToolInputStreaming: false,
|
||||
supportedReasoningEfforts: ["low", "medium", "high", "max"],
|
||||
});
|
||||
});
|
||||
|
||||
@@ -520,7 +520,7 @@ async function checkReadableFile(pathname: string): Promise<{ exists: boolean; i
|
||||
}
|
||||
}
|
||||
|
||||
async function diagnoseSessionFiles(agentId: string): Promise<SourceScan> {
|
||||
async function scanSessionFiles(agentId: string): Promise<SourceScan> {
|
||||
const issues: string[] = [];
|
||||
const sessionsDir = resolveSessionTranscriptsDirForAgent(agentId);
|
||||
try {
|
||||
@@ -542,7 +542,7 @@ async function diagnoseSessionFiles(agentId: string): Promise<SourceScan> {
|
||||
}
|
||||
}
|
||||
|
||||
async function diagnoseMemoryFiles(
|
||||
async function scanMemoryFiles(
|
||||
workspaceDir: string,
|
||||
extraPaths: string[] = [],
|
||||
): Promise<SourceScan> {
|
||||
@@ -668,10 +668,10 @@ async function scanMemorySources(params: {
|
||||
const extraPaths = params.extraPaths ?? [];
|
||||
for (const source of params.sources) {
|
||||
if (source === "memory") {
|
||||
scans.push(await diagnoseMemoryFiles(params.workspaceDir, extraPaths));
|
||||
scans.push(await scanMemoryFiles(params.workspaceDir, extraPaths));
|
||||
}
|
||||
if (source === "sessions") {
|
||||
scans.push(await diagnoseSessionFiles(params.agentId));
|
||||
scans.push(await scanSessionFiles(params.agentId));
|
||||
}
|
||||
}
|
||||
const issues = scans.flatMap((scan) => scan.issues);
|
||||
|
||||
@@ -1168,13 +1168,12 @@ describe("memory index", () => {
|
||||
try {
|
||||
const fields = nextManager as unknown as {
|
||||
dirty: boolean;
|
||||
syncSessionFiles: (params: unknown) => Promise<unknown>;
|
||||
syncSessionFiles: (params: unknown) => Promise<void>;
|
||||
};
|
||||
const syncSessionFiles = fields.syncSessionFiles.bind(nextManager);
|
||||
fields.syncSessionFiles = async (params) => {
|
||||
fields.dirty = true;
|
||||
// Forward the sync plan: runSync reads scanOk from the return value.
|
||||
return await syncSessionFiles(params);
|
||||
await syncSessionFiles(params);
|
||||
};
|
||||
|
||||
await nextManager.sync({ reason: "test", force: true });
|
||||
|
||||
@@ -17,7 +17,6 @@ describe("memory session sync state", () => {
|
||||
{ path: "sessions/b.jsonl", hash: "hash-b" },
|
||||
],
|
||||
sessionPathForFile: (file) => `sessions/${file.split("/").at(-1)}`,
|
||||
scanOk: true,
|
||||
});
|
||||
|
||||
expect(plan.indexAll).toBe(true);
|
||||
@@ -32,46 +31,6 @@ describe("memory session sync state", () => {
|
||||
["sessions/b.jsonl", "hash-b"],
|
||||
]),
|
||||
);
|
||||
// A full enumeration that read the directory may prune stale rows.
|
||||
expect(plan.pruneStaleRows).toBe(true);
|
||||
});
|
||||
|
||||
it("skips pruning when the full directory scan failed", () => {
|
||||
const plan = resolveMemorySessionSyncPlan({
|
||||
needsFullReindex: false,
|
||||
// A failed scan surfaces an empty listing even though the index still
|
||||
// holds rows; pruning here would wipe the whole session index.
|
||||
files: [],
|
||||
targetSessionFiles: null,
|
||||
sessionsDirtyFiles: new Set(),
|
||||
existingRows: [
|
||||
{ path: "sessions/a.jsonl", hash: "hash-a" },
|
||||
{ path: "sessions/b.jsonl", hash: "hash-b" },
|
||||
],
|
||||
sessionPathForFile: (file) => `sessions/${file.split("/").at(-1)}`,
|
||||
scanOk: false,
|
||||
});
|
||||
|
||||
expect(plan.activePaths).toEqual(new Set());
|
||||
expect(plan.pruneStaleRows).toBe(false);
|
||||
});
|
||||
|
||||
it("still prunes when an authoritatively empty directory leaves orphaned rows", () => {
|
||||
const plan = resolveMemorySessionSyncPlan({
|
||||
needsFullReindex: false,
|
||||
// The directory was read successfully and genuinely has no session files
|
||||
// (e.g. disk-budget removed the last archive); orphaned rows should be
|
||||
// pruned rather than lingering in search.
|
||||
files: [],
|
||||
targetSessionFiles: null,
|
||||
sessionsDirtyFiles: new Set(),
|
||||
existingRows: [{ path: "sessions/a.jsonl", hash: "hash-a" }],
|
||||
sessionPathForFile: (file) => `sessions/${file.split("/").at(-1)}`,
|
||||
scanOk: true,
|
||||
});
|
||||
|
||||
expect(plan.activePaths).toEqual(new Set());
|
||||
expect(plan.pruneStaleRows).toBe(true);
|
||||
});
|
||||
|
||||
it("treats targeted session syncs as refresh-only and skips unrelated pruning", () => {
|
||||
@@ -85,15 +44,12 @@ describe("memory session sync state", () => {
|
||||
{ path: "sessions/targeted-second.jsonl", hash: "hash-second" },
|
||||
],
|
||||
sessionPathForFile: (file) => `sessions/${file.split("/").at(-1)}`,
|
||||
scanOk: true,
|
||||
});
|
||||
|
||||
expect(plan.indexAll).toBe(true);
|
||||
expect(plan.activePaths).toBeNull();
|
||||
expect(plan.existingRows).toBeNull();
|
||||
expect(plan.existingHashes).toBeNull();
|
||||
// Targeted syncs never hold a full enumeration, so they must not prune.
|
||||
expect(plan.pruneStaleRows).toBe(false);
|
||||
});
|
||||
|
||||
it("keeps dirty-only incremental mode when no targeted sync is requested", () => {
|
||||
@@ -104,7 +60,6 @@ describe("memory session sync state", () => {
|
||||
sessionsDirtyFiles: new Set(["/tmp/incremental.jsonl"]),
|
||||
existingRows: [],
|
||||
sessionPathForFile: (file) => `sessions/${file.split("/").at(-1)}`,
|
||||
scanOk: true,
|
||||
});
|
||||
|
||||
expect(plan.indexAll).toBe(false);
|
||||
|
||||
@@ -40,25 +40,11 @@ export function resolveMemorySessionSyncPlan(params: {
|
||||
sessionsDirtyFiles: Set<string>;
|
||||
existingRows?: MemorySourceFileStateRow[] | null;
|
||||
sessionPathForFile: (file: string) => string;
|
||||
// Whether the enumeration that produced `files` was authoritative (the
|
||||
// sessions dir was read or does not exist). Required so a new caller cannot
|
||||
// silently default a failed scan to authoritative: an empty `files` array
|
||||
// from a failed scan must not drive the destructive stale-row prune. See
|
||||
// `pruneStaleRows`. Targeted syncs pass true (no scan involved).
|
||||
scanOk: boolean;
|
||||
}): {
|
||||
activePaths: Set<string> | null;
|
||||
existingRows: MemorySourceFileStateRow[] | null;
|
||||
existingHashes: Map<string, string> | null;
|
||||
indexAll: boolean;
|
||||
// True only when the stale-row prune is safe to run: a full enumeration
|
||||
// (activePaths !== null) that authoritatively read the directory. When the
|
||||
// scan failed, `files` is empty for reasons unrelated to the on-disk state,
|
||||
// so pruning every row not in the (empty) listing would wipe the session
|
||||
// index on a single transient NFS error. An authoritatively empty directory
|
||||
// still prunes, so legitimately removed sessions (e.g. disk-budget removing
|
||||
// the last archive) do not leave orphaned rows.
|
||||
pruneStaleRows: boolean;
|
||||
} {
|
||||
const activePaths = params.targetSessionFiles
|
||||
? null
|
||||
@@ -72,6 +58,5 @@ export function resolveMemorySessionSyncPlan(params: {
|
||||
params.needsFullReindex ||
|
||||
Boolean(params.targetSessionFiles) ||
|
||||
params.sessionsDirtyFiles.size === 0,
|
||||
pruneStaleRows: activePaths !== null && params.scanOk,
|
||||
};
|
||||
}
|
||||
|
||||
@@ -1,225 +0,0 @@
|
||||
// Memory Core tests cover stale-row prune safety in manager sync ops: a full
|
||||
// enumeration only prunes indexed rows when it actually read the source roots,
|
||||
// so a transient scan failure cannot wipe the session or memory-file index.
|
||||
import fs from "node:fs/promises";
|
||||
import os from "node:os";
|
||||
import path from "node:path";
|
||||
import type { DatabaseSync } from "node:sqlite";
|
||||
import {
|
||||
resolveSessionTranscriptsDirForAgent,
|
||||
type OpenClawConfig,
|
||||
type ResolvedMemorySearchConfig,
|
||||
} from "openclaw/plugin-sdk/memory-core-host-engine-foundation";
|
||||
import { afterEach, beforeEach, describe, expect, it, vi } from "vitest";
|
||||
import { MemoryManagerSyncOps } from "./manager-sync-ops.js";
|
||||
|
||||
type SourceStateRow = { path: string; hash: string; mtime: number; size: number };
|
||||
type SourceSyncPlanResult = { scanOk: boolean };
|
||||
|
||||
class PruneGuardHarness extends MemoryManagerSyncOps {
|
||||
protected readonly cfg = {} as OpenClawConfig;
|
||||
protected readonly agentId = "main";
|
||||
protected readonly settings = {
|
||||
sync: {
|
||||
sessions: {
|
||||
deltaBytes: 100_000,
|
||||
deltaMessages: 50,
|
||||
postCompactionForce: true,
|
||||
},
|
||||
},
|
||||
} as ResolvedMemorySearchConfig;
|
||||
protected readonly batch = {
|
||||
enabled: false,
|
||||
wait: false,
|
||||
concurrency: 1,
|
||||
pollIntervalMs: 0,
|
||||
timeoutMs: 0,
|
||||
};
|
||||
protected readonly vector = { enabled: false, available: false };
|
||||
protected readonly cache = { enabled: false };
|
||||
protected providerUnavailableReason?: string;
|
||||
protected providerLifecycle = { mode: "active" as const, providerId: "test" };
|
||||
protected db: DatabaseSync;
|
||||
|
||||
readonly deletedRows: Array<{ path: string; source: string }> = [];
|
||||
|
||||
constructor(
|
||||
sourceRows: SourceStateRow[],
|
||||
protected readonly workspaceDir = "/tmp/openclaw-test-workspace",
|
||||
) {
|
||||
super();
|
||||
this.sources.add("sessions");
|
||||
this.sources.add("memory");
|
||||
this.db = {
|
||||
prepare: (sql: string) => ({
|
||||
all: () => sourceRows,
|
||||
get: () => undefined,
|
||||
run: (...args: unknown[]) => {
|
||||
if (sql.startsWith("DELETE FROM files")) {
|
||||
this.deletedRows.push({ path: String(args[0]), source: String(args[1]) });
|
||||
}
|
||||
return undefined;
|
||||
},
|
||||
}),
|
||||
} as unknown as DatabaseSync;
|
||||
}
|
||||
|
||||
async runFullSessionSync(): Promise<SourceSyncPlanResult> {
|
||||
return await (
|
||||
this as unknown as {
|
||||
syncSessionFiles: (p: unknown) => Promise<SourceSyncPlanResult>;
|
||||
}
|
||||
).syncSessionFiles({ needsFullReindex: true });
|
||||
}
|
||||
|
||||
async runFullMemorySync(): Promise<SourceSyncPlanResult> {
|
||||
return await (
|
||||
this as unknown as {
|
||||
syncMemoryFiles: (p: unknown) => Promise<SourceSyncPlanResult>;
|
||||
}
|
||||
).syncMemoryFiles({ needsFullReindex: true });
|
||||
}
|
||||
|
||||
deletedPathsFor(source: string): string[] {
|
||||
return this.deletedRows.filter((row) => row.source === source).map((row) => row.path);
|
||||
}
|
||||
|
||||
protected computeProviderKey(): string {
|
||||
return "test";
|
||||
}
|
||||
|
||||
protected async sync(): Promise<void> {}
|
||||
|
||||
protected async withTimeout<T>(promise: Promise<T>): Promise<T> {
|
||||
return await promise;
|
||||
}
|
||||
|
||||
protected getIndexConcurrency(): number {
|
||||
return 1;
|
||||
}
|
||||
|
||||
protected pruneEmbeddingCacheIfNeeded(): void {}
|
||||
|
||||
protected resetProviderInitializationForRetry(): void {}
|
||||
|
||||
protected assertRequiredProviderAvailable(): void {}
|
||||
|
||||
protected async indexFile(): Promise<void> {}
|
||||
}
|
||||
|
||||
describe("session prune safety", () => {
|
||||
let stateDir = "";
|
||||
|
||||
beforeEach(async () => {
|
||||
stateDir = await fs.mkdtemp(path.join(os.tmpdir(), "openclaw-session-prune-"));
|
||||
vi.stubEnv("OPENCLAW_STATE_DIR", stateDir);
|
||||
});
|
||||
|
||||
afterEach(async () => {
|
||||
vi.restoreAllMocks();
|
||||
vi.unstubAllEnvs();
|
||||
await fs.rm(stateDir, { recursive: true, force: true });
|
||||
});
|
||||
|
||||
it("does not prune indexed session rows when the directory scan fails", async () => {
|
||||
// The index holds a session row; a transient readdir failure surfaces an
|
||||
// empty listing. Without the guard the empty listing would prune the row.
|
||||
await fs.mkdir(resolveSessionTranscriptsDirForAgent("main"), { recursive: true });
|
||||
const harness = new PruneGuardHarness([
|
||||
{ path: "sessions/main/thread.jsonl", hash: "hash-a", mtime: 1, size: 1 },
|
||||
]);
|
||||
vi.spyOn(fs, "readdir").mockRejectedValueOnce(
|
||||
Object.assign(new Error("nfs blip"), { code: "EIO" }),
|
||||
);
|
||||
|
||||
const plan = await harness.runFullSessionSync();
|
||||
|
||||
expect(plan.scanOk).toBe(false);
|
||||
expect(harness.deletedPathsFor("sessions")).toEqual([]);
|
||||
});
|
||||
|
||||
it("prunes orphaned session rows when the directory is authoritatively empty", async () => {
|
||||
// The directory is read successfully and genuinely holds no session files
|
||||
// (e.g. disk-budget removed the last archive). The orphaned row must be
|
||||
// pruned rather than lingering in search.
|
||||
await fs.mkdir(resolveSessionTranscriptsDirForAgent("main"), { recursive: true });
|
||||
const harness = new PruneGuardHarness([
|
||||
{ path: "sessions/main/gone.jsonl", hash: "hash-gone", mtime: 1, size: 1 },
|
||||
]);
|
||||
|
||||
const plan = await harness.runFullSessionSync();
|
||||
|
||||
expect(plan.scanOk).toBe(true);
|
||||
expect(harness.deletedPathsFor("sessions")).toEqual(["sessions/main/gone.jsonl"]);
|
||||
});
|
||||
|
||||
it("prunes orphaned session rows when the sessions directory is absent", async () => {
|
||||
// The dir only exists once a transcript is written, so ENOENT under an
|
||||
// existing agent dir is an authoritative empty enumeration (fresh agent,
|
||||
// or the dir was removed wholesale) and orphaned rows must still be
|
||||
// pruned.
|
||||
await fs.mkdir(path.join(stateDir, "agents", "main"), { recursive: true });
|
||||
const harness = new PruneGuardHarness([
|
||||
{ path: "sessions/main/gone.jsonl", hash: "hash-gone", mtime: 1, size: 1 },
|
||||
]);
|
||||
|
||||
const plan = await harness.runFullSessionSync();
|
||||
|
||||
expect(plan.scanOk).toBe(true);
|
||||
expect(harness.deletedPathsFor("sessions")).toEqual(["sessions/main/gone.jsonl"]);
|
||||
});
|
||||
});
|
||||
|
||||
describe("memory-file prune safety", () => {
|
||||
let stateDir = "";
|
||||
|
||||
beforeEach(async () => {
|
||||
stateDir = await fs.mkdtemp(path.join(os.tmpdir(), "openclaw-memory-prune-"));
|
||||
vi.stubEnv("OPENCLAW_STATE_DIR", stateDir);
|
||||
});
|
||||
|
||||
afterEach(async () => {
|
||||
vi.restoreAllMocks();
|
||||
vi.unstubAllEnvs();
|
||||
await fs.rm(stateDir, { recursive: true, force: true });
|
||||
});
|
||||
|
||||
it("does not prune indexed memory rows when the enumeration fails", async () => {
|
||||
const workspaceDir = path.join(stateDir, "workspace");
|
||||
const memoryDir = path.join(workspaceDir, "memory");
|
||||
await fs.mkdir(memoryDir, { recursive: true });
|
||||
const harness = new PruneGuardHarness(
|
||||
[{ path: "memory/notes.md", hash: "hash-notes", mtime: 1, size: 1 }],
|
||||
workspaceDir,
|
||||
);
|
||||
const realLstat = fs.lstat;
|
||||
vi.spyOn(fs, "lstat").mockImplementation(async (...args: Parameters<typeof realLstat>) => {
|
||||
const [target] = args;
|
||||
if (typeof target === "string" && path.resolve(target) === memoryDir) {
|
||||
throw Object.assign(new Error("nfs blip"), { code: "EIO" });
|
||||
}
|
||||
return await realLstat(...args);
|
||||
});
|
||||
|
||||
const plan = await harness.runFullMemorySync();
|
||||
|
||||
expect(plan.scanOk).toBe(false);
|
||||
expect(harness.deletedPathsFor("memory")).toEqual([]);
|
||||
});
|
||||
|
||||
it("prunes orphaned memory rows when the workspace holds no memory files", async () => {
|
||||
// The workspace exists but has no root memory file and no memory dir; the
|
||||
// enumeration is authoritative, so the orphaned row must be pruned.
|
||||
const workspaceDir = path.join(stateDir, "workspace-empty");
|
||||
await fs.mkdir(workspaceDir, { recursive: true });
|
||||
const harness = new PruneGuardHarness(
|
||||
[{ path: "memory/gone.md", hash: "hash-gone", mtime: 1, size: 1 }],
|
||||
workspaceDir,
|
||||
);
|
||||
|
||||
const plan = await harness.runFullMemorySync();
|
||||
|
||||
expect(plan.scanOk).toBe(true);
|
||||
expect(harness.deletedPathsFor("memory")).toEqual(["memory/gone.md"]);
|
||||
});
|
||||
});
|
||||
@@ -1,183 +0,0 @@
|
||||
// Memory Core tests cover session delta sync targeting: event-driven delta
|
||||
// syncs pass the dirty set as explicit targets (skipping the sessions-dir
|
||||
// enumeration), and fall back to a full enumeration at most once per
|
||||
// reconcile window so out-of-band deletions still get pruned.
|
||||
import fs from "node:fs/promises";
|
||||
import os from "node:os";
|
||||
import path from "node:path";
|
||||
import type { DatabaseSync } from "node:sqlite";
|
||||
import {
|
||||
resolveSessionTranscriptsDirForAgent,
|
||||
type OpenClawConfig,
|
||||
type ResolvedMemorySearchConfig,
|
||||
} from "openclaw/plugin-sdk/memory-core-host-engine-foundation";
|
||||
import { afterEach, beforeEach, describe, expect, it, vi } from "vitest";
|
||||
import { MemoryManagerSyncOps } from "./manager-sync-ops.js";
|
||||
|
||||
type RecordedSync = { reason?: string; force?: boolean; sessionFiles?: string[] };
|
||||
|
||||
class TargetedDeltaHarness extends MemoryManagerSyncOps {
|
||||
protected readonly cfg = {} as OpenClawConfig;
|
||||
protected readonly agentId = "main";
|
||||
protected readonly settings = {
|
||||
sync: {
|
||||
sessions: {
|
||||
deltaBytes: 100_000,
|
||||
deltaMessages: 50,
|
||||
postCompactionForce: true,
|
||||
},
|
||||
},
|
||||
} as ResolvedMemorySearchConfig;
|
||||
protected readonly batch = {
|
||||
enabled: false,
|
||||
wait: false,
|
||||
concurrency: 1,
|
||||
pollIntervalMs: 0,
|
||||
timeoutMs: 0,
|
||||
};
|
||||
protected readonly vector = { enabled: false, available: false };
|
||||
protected readonly cache = { enabled: false };
|
||||
protected providerUnavailableReason?: string;
|
||||
protected providerLifecycle = { mode: "active" as const, providerId: "test" };
|
||||
protected db: DatabaseSync;
|
||||
|
||||
readonly syncCalls: RecordedSync[] = [];
|
||||
|
||||
constructor(protected readonly workspaceDir = "/tmp/openclaw-test-workspace") {
|
||||
super();
|
||||
this.sources.add("sessions");
|
||||
this.db = {
|
||||
prepare: () => ({ all: () => [], get: () => undefined, run: () => undefined }),
|
||||
} as unknown as DatabaseSync;
|
||||
}
|
||||
|
||||
async runDeltaBatch(pendingFiles: string[]): Promise<void> {
|
||||
for (const file of pendingFiles) {
|
||||
this.sessionPendingFiles.add(file);
|
||||
}
|
||||
await (
|
||||
this as unknown as { processSessionDeltaBatch: () => Promise<void> }
|
||||
).processSessionDeltaBatch();
|
||||
}
|
||||
|
||||
addDirtyFile(sessionFile: string): void {
|
||||
this.sessionsDirtyFiles.add(sessionFile);
|
||||
}
|
||||
|
||||
setLastReconcileAt(ms: number): void {
|
||||
(this as unknown as { lastSessionPruneReconcileAt: number }).lastSessionPruneReconcileAt = ms;
|
||||
}
|
||||
|
||||
lastReconcileAt(): number {
|
||||
return (this as unknown as { lastSessionPruneReconcileAt: number }).lastSessionPruneReconcileAt;
|
||||
}
|
||||
|
||||
async runFullSessionSync(): Promise<void> {
|
||||
await (
|
||||
this as unknown as { syncSessionFiles: (p: unknown) => Promise<unknown> }
|
||||
).syncSessionFiles({ needsFullReindex: true });
|
||||
}
|
||||
|
||||
protected computeProviderKey(): string {
|
||||
return "test";
|
||||
}
|
||||
|
||||
protected async sync(params?: RecordedSync): Promise<void> {
|
||||
this.syncCalls.push(params ?? {});
|
||||
}
|
||||
|
||||
protected async withTimeout<T>(promise: Promise<T>): Promise<T> {
|
||||
return await promise;
|
||||
}
|
||||
|
||||
protected getIndexConcurrency(): number {
|
||||
return 1;
|
||||
}
|
||||
|
||||
protected pruneEmbeddingCacheIfNeeded(): void {}
|
||||
|
||||
protected resetProviderInitializationForRetry(): void {}
|
||||
|
||||
protected assertRequiredProviderAvailable(): void {}
|
||||
|
||||
protected async indexFile(): Promise<void> {}
|
||||
}
|
||||
|
||||
// Archive artifacts route through the direct mark-dirty branch of the delta
|
||||
// batch, so tests need no transcript fixtures or delta accounting.
|
||||
function archiveSessionFile(name: string): string {
|
||||
return path.join(
|
||||
resolveSessionTranscriptsDirForAgent("main"),
|
||||
`${name}.jsonl.reset.2026-01-01T00-00-00.000Z`,
|
||||
);
|
||||
}
|
||||
|
||||
describe("session delta sync targeting", () => {
|
||||
let stateDir = "";
|
||||
|
||||
beforeEach(async () => {
|
||||
stateDir = await fs.mkdtemp(path.join(os.tmpdir(), "openclaw-targeted-delta-"));
|
||||
vi.stubEnv("OPENCLAW_STATE_DIR", stateDir);
|
||||
});
|
||||
|
||||
afterEach(async () => {
|
||||
vi.restoreAllMocks();
|
||||
vi.unstubAllEnvs();
|
||||
await fs.rm(stateDir, { recursive: true, force: true });
|
||||
});
|
||||
|
||||
it("passes the dirty set as sync targets inside the reconcile window", async () => {
|
||||
const harness = new TargetedDeltaHarness();
|
||||
harness.setLastReconcileAt(Date.now());
|
||||
const archived = archiveSessionFile("thread-a");
|
||||
|
||||
await harness.runDeltaBatch([archived]);
|
||||
|
||||
expect(harness.syncCalls).toEqual([{ reason: "session-delta", sessionFiles: [archived] }]);
|
||||
});
|
||||
|
||||
it("includes leftover dirty files from earlier syncs in the targets", async () => {
|
||||
const harness = new TargetedDeltaHarness();
|
||||
harness.setLastReconcileAt(Date.now());
|
||||
const leftover = archiveSessionFile("thread-leftover");
|
||||
const fresh = archiveSessionFile("thread-fresh");
|
||||
harness.addDirtyFile(leftover);
|
||||
|
||||
await harness.runDeltaBatch([fresh]);
|
||||
|
||||
expect(harness.syncCalls).toHaveLength(1);
|
||||
expect(harness.syncCalls[0]?.reason).toBe("session-delta");
|
||||
expect(new Set(harness.syncCalls[0]?.sessionFiles)).toEqual(new Set([leftover, fresh]));
|
||||
});
|
||||
|
||||
it("falls back to a full enumeration once the reconcile window elapses", async () => {
|
||||
// lastSessionPruneReconcileAt starts at 0, so the first active delta sync
|
||||
// reconciles with a full enumeration unless a prior prune already ran.
|
||||
const harness = new TargetedDeltaHarness();
|
||||
|
||||
await harness.runDeltaBatch([archiveSessionFile("thread-b")]);
|
||||
|
||||
expect(harness.syncCalls).toEqual([{ reason: "session-delta" }]);
|
||||
});
|
||||
|
||||
it("advances the reconcile timestamp after an authoritative prune", async () => {
|
||||
await fs.mkdir(resolveSessionTranscriptsDirForAgent("main"), { recursive: true });
|
||||
const harness = new TargetedDeltaHarness();
|
||||
|
||||
await harness.runFullSessionSync();
|
||||
|
||||
expect(harness.lastReconcileAt()).toBeGreaterThan(0);
|
||||
});
|
||||
|
||||
it("does not advance the reconcile timestamp when the scan fails", async () => {
|
||||
await fs.mkdir(resolveSessionTranscriptsDirForAgent("main"), { recursive: true });
|
||||
const harness = new TargetedDeltaHarness();
|
||||
vi.spyOn(fs, "readdir").mockRejectedValueOnce(
|
||||
Object.assign(new Error("nfs blip"), { code: "EIO" }),
|
||||
);
|
||||
|
||||
await harness.runFullSessionSync();
|
||||
|
||||
expect(harness.lastReconcileAt()).toBe(0);
|
||||
});
|
||||
});
|
||||
@@ -21,18 +21,17 @@ import {
|
||||
isSessionArchiveArtifactName,
|
||||
isUsageCountedSessionTranscriptFileName,
|
||||
listSessionFilesForAgent,
|
||||
scanSessionFilesForAgent,
|
||||
sessionPathForFile,
|
||||
} from "openclaw/plugin-sdk/memory-core-host-engine-qmd";
|
||||
import {
|
||||
buildFileEntry,
|
||||
ensureMemoryIndexSchema,
|
||||
isFileMissingError,
|
||||
listMemoryFiles,
|
||||
loadSqliteVecExtension,
|
||||
normalizeExtraMemoryPaths,
|
||||
retryTransientMemoryRead,
|
||||
runWithConcurrency,
|
||||
scanMemoryFiles,
|
||||
type MemorySource,
|
||||
type MemorySyncProgressUpdate,
|
||||
} from "openclaw/plugin-sdk/memory-core-host-engine-storage";
|
||||
@@ -117,10 +116,6 @@ export type MemoryIndexWorkItem = {
|
||||
type MemorySourceSyncPlan = {
|
||||
indexItems: MemoryIndexWorkItem[];
|
||||
finalize: () => Promise<void> | void;
|
||||
// False when the source enumeration failed (non-authoritative listing).
|
||||
// runSync then keeps the source's dirty state so the next eligible sync
|
||||
// retries, instead of reporting a clean sync that indexed nothing.
|
||||
scanOk: boolean;
|
||||
};
|
||||
|
||||
const META_KEY = "memory_index_meta_v1";
|
||||
@@ -128,11 +123,6 @@ const VECTOR_TABLE = "chunks_vec";
|
||||
const FTS_TABLE = "chunks_fts";
|
||||
const EMBEDDING_CACHE_TABLE = "embedding_cache";
|
||||
const SESSION_DIRTY_DEBOUNCE_MS = 5000;
|
||||
// Targeted delta syncs skip the sessions-dir enumeration entirely, so rows for
|
||||
// files deleted out-of-band (another process/node on shared storage) are only
|
||||
// pruned by full enumerations. Force one full-enumeration delta sync at most
|
||||
// this often to bound how long such stale rows can linger in search.
|
||||
const SESSION_PRUNE_RECONCILE_INTERVAL_MS = 15 * 60 * 1000;
|
||||
const SESSION_DELTA_READ_CHUNK_BYTES = 64 * 1024;
|
||||
const SESSION_SYNC_YIELD_EVERY = 10;
|
||||
const SOURCE_WIDE_SESSION_INDEX_FLUSH_FILES = 128;
|
||||
@@ -279,10 +269,6 @@ export abstract class MemoryManagerSyncOps {
|
||||
protected sessionsDirty = false;
|
||||
private readonly memoryWatchPressureWarning: MemoryWatchPressureWarningState = { shown: false };
|
||||
protected sessionsDirtyFiles = new Set<string>();
|
||||
// Epoch ms of the last completed session stale-row prune (authoritative full
|
||||
// enumeration). Starts at 0 so the first active delta sync reconciles unless
|
||||
// the startup catch-up sync already did.
|
||||
private lastSessionPruneReconcileAt = 0;
|
||||
protected sessionPendingFiles = new Set<string>();
|
||||
protected sessionDeltas = new Map<
|
||||
string,
|
||||
@@ -321,20 +307,7 @@ export abstract class MemoryManagerSyncOps {
|
||||
}
|
||||
|
||||
private emptySourceSyncPlan(): MemorySourceSyncPlan {
|
||||
return { indexItems: [], finalize: () => {}, scanOk: true };
|
||||
}
|
||||
|
||||
// A full reindex builds a fresh index from the enumeration alone, so a
|
||||
// failed scan would persist an empty index over existing rows. Abort: the
|
||||
// safe path restores the original DB on throw, and the unsafe path skips
|
||||
// writeMeta so the next sync retries the full rebuild.
|
||||
private assertReindexScanOk(source: MemorySource, scanOk: boolean): void {
|
||||
if (scanOk) {
|
||||
return;
|
||||
}
|
||||
throw new Error(
|
||||
`Memory reindex aborted: ${source} enumeration failed; keeping the existing index.`,
|
||||
);
|
||||
return { indexItems: [], finalize: () => {} };
|
||||
}
|
||||
|
||||
private shouldDeferSourceWideBatch(): boolean {
|
||||
@@ -396,7 +369,7 @@ export abstract class MemoryManagerSyncOps {
|
||||
needsFullReindex: boolean;
|
||||
targetSessionFiles?: string[];
|
||||
progress?: MemorySyncProgressState;
|
||||
}): Promise<{ memoryScanOk: boolean; sessionsScanOk: boolean }> {
|
||||
}): Promise<void> {
|
||||
const memoryPlan = params.shouldSyncMemory
|
||||
? await this.syncMemoryFiles({
|
||||
needsFullReindex: params.needsFullReindex,
|
||||
@@ -405,7 +378,7 @@ export abstract class MemoryManagerSyncOps {
|
||||
})
|
||||
: this.emptySourceSyncPlan();
|
||||
if (params.shouldSyncSessions) {
|
||||
const sessionPlan = await this.syncSessionFiles({
|
||||
await this.syncSessionFiles({
|
||||
needsFullReindex: params.needsFullReindex,
|
||||
targetSessionFiles: params.targetSessionFiles,
|
||||
progress: params.progress,
|
||||
@@ -413,10 +386,9 @@ export abstract class MemoryManagerSyncOps {
|
||||
prefixIndexItems: memoryPlan.indexItems,
|
||||
});
|
||||
await memoryPlan.finalize();
|
||||
return { memoryScanOk: memoryPlan.scanOk, sessionsScanOk: sessionPlan.scanOk };
|
||||
return;
|
||||
}
|
||||
await this.executeSourceSyncPlans([memoryPlan], params.progress);
|
||||
return { memoryScanOk: memoryPlan.scanOk, sessionsScanOk: true };
|
||||
}
|
||||
|
||||
protected hasIndexedChunks(): boolean {
|
||||
@@ -1442,16 +1414,7 @@ export abstract class MemoryManagerSyncOps {
|
||||
shouldSync = true;
|
||||
}
|
||||
if (shouldSync) {
|
||||
// The dirty set already names every transcript this sync must index, so
|
||||
// pass it through and skip the sessions-dir enumeration (expensive on
|
||||
// networked filesystems). Periodically fall back to a full enumeration
|
||||
// so out-of-band deletions still get their stale index rows pruned.
|
||||
const reconcileDue =
|
||||
Date.now() - this.lastSessionPruneReconcileAt >= SESSION_PRUNE_RECONCILE_INTERVAL_MS;
|
||||
const syncParams = reconcileDue
|
||||
? { reason: "session-delta" }
|
||||
: { reason: "session-delta", sessionFiles: Array.from(this.sessionsDirtyFiles) };
|
||||
void this.sync(syncParams).catch((err: unknown) => {
|
||||
void this.sync({ reason: "session-delta" }).catch((err: unknown) => {
|
||||
log.warn(`memory sync failed (session-delta): ${String(err)}`);
|
||||
});
|
||||
}
|
||||
@@ -1672,15 +1635,11 @@ export abstract class MemoryManagerSyncOps {
|
||||
? this.db.prepare(`DELETE FROM ${FTS_TABLE} WHERE path = ? AND source = ?`)
|
||||
: null;
|
||||
|
||||
// Track whether the enumeration actually read every root, so a transient
|
||||
// failure cannot surface an empty/partial listing that the stale-row prune
|
||||
// below would treat as mass deletion.
|
||||
const scan = await scanMemoryFiles(
|
||||
const files = await listMemoryFiles(
|
||||
this.workspaceDir,
|
||||
this.settings.extraPaths,
|
||||
this.settings.multimodal,
|
||||
);
|
||||
const files = scan.files;
|
||||
const fileEntries = (
|
||||
await runWithConcurrency(
|
||||
files.map(
|
||||
@@ -1713,15 +1672,6 @@ export abstract class MemoryManagerSyncOps {
|
||||
}
|
||||
|
||||
const deleteStaleRows = async () => {
|
||||
if (!scan.ok) {
|
||||
// A failed enumeration is non-authoritative: pruning against it would
|
||||
// delete index rows for files that still exist on disk. Keep the rows
|
||||
// until the next successful scan reconciles.
|
||||
log.warn(
|
||||
`memory sync: skipping memory-file prune after a failed enumeration; retaining ${existingRows.length} indexed rows until the next successful scan`,
|
||||
);
|
||||
return;
|
||||
}
|
||||
for (const stale of existingRows) {
|
||||
if (activePaths.has(stale.path)) {
|
||||
continue;
|
||||
@@ -1760,7 +1710,7 @@ export abstract class MemoryManagerSyncOps {
|
||||
(entry): MemoryIndexWorkItem => ({ entry, source: "memory" }),
|
||||
);
|
||||
if (params.deferIndex) {
|
||||
return { indexItems, finalize: deleteStaleRows, scanOk: scan.ok };
|
||||
return { indexItems, finalize: deleteStaleRows };
|
||||
}
|
||||
await this.indexQueuedFiles(indexItems, params.progress);
|
||||
} else {
|
||||
@@ -1788,7 +1738,7 @@ export abstract class MemoryManagerSyncOps {
|
||||
}
|
||||
|
||||
await deleteStaleRows();
|
||||
return { ...this.emptySourceSyncPlan(), scanOk: scan.ok };
|
||||
return this.emptySourceSyncPlan();
|
||||
}
|
||||
|
||||
private async syncSessionFiles(params: {
|
||||
@@ -1818,14 +1768,9 @@ export abstract class MemoryManagerSyncOps {
|
||||
const targetSessionFiles = params.needsFullReindex
|
||||
? null
|
||||
: this.normalizeTargetSessionFiles(params.targetSessionFiles);
|
||||
// Targeted syncs use the requested set directly (no directory scan). Full
|
||||
// enumerations scan the sessions dir and track whether the scan actually
|
||||
// succeeded, so a transient NFS failure cannot masquerade as an empty dir
|
||||
// and drive the destructive stale-row prune below.
|
||||
const scan = targetSessionFiles
|
||||
? { ok: true, files: Array.from(targetSessionFiles) }
|
||||
: await scanSessionFilesForAgent(this.agentId);
|
||||
const files = scan.files;
|
||||
const files = targetSessionFiles
|
||||
? Array.from(targetSessionFiles)
|
||||
: await listSessionFilesForAgent(this.agentId);
|
||||
const sessionPlan = resolveMemorySessionSyncPlan({
|
||||
needsFullReindex: params.needsFullReindex,
|
||||
files,
|
||||
@@ -1838,9 +1783,8 @@ export abstract class MemoryManagerSyncOps {
|
||||
source: "sessions",
|
||||
}).rows,
|
||||
sessionPathForFile,
|
||||
scanOk: scan.ok,
|
||||
});
|
||||
const { activePaths, existingRows, existingHashes, indexAll, pruneStaleRows } = sessionPlan;
|
||||
const { activePaths, existingRows, existingHashes, indexAll } = sessionPlan;
|
||||
log.debug("memory sync: indexing session files", {
|
||||
files: files.length,
|
||||
indexAll,
|
||||
@@ -1860,21 +1804,7 @@ export abstract class MemoryManagerSyncOps {
|
||||
|
||||
const yieldAfterSessionFile = createSessionSyncYield(files.length);
|
||||
const deleteStaleRows = async () => {
|
||||
if (!pruneStaleRows || activePaths === null) {
|
||||
// Skip the stale-row prune unless we hold an authoritative full
|
||||
// enumeration. Targeted syncs (activePaths === null) only refresh the
|
||||
// requested transcripts. A failed directory scan also lands here: it
|
||||
// surfaces an empty listing for reasons unrelated to on-disk state
|
||||
// (e.g. transient NFS EIO/ESTALE), so pruning would wipe the entire
|
||||
// session index on one blip. Leave the index intact and let the next
|
||||
// successful enumeration reconcile.
|
||||
if (activePaths !== null && !scan.ok) {
|
||||
log.warn(
|
||||
`memory sync: skipping session prune after a failed directory scan; retaining ${
|
||||
(existingRows ?? []).length
|
||||
} indexed session rows until the next successful enumeration`,
|
||||
);
|
||||
}
|
||||
if (activePaths === null) {
|
||||
return;
|
||||
}
|
||||
|
||||
@@ -1901,9 +1831,6 @@ export abstract class MemoryManagerSyncOps {
|
||||
await yieldAfterStaleSessionRow();
|
||||
}
|
||||
}
|
||||
// A completed authoritative prune is the reconciliation point targeted
|
||||
// delta syncs rely on; record it so they keep skipping enumerations.
|
||||
this.lastSessionPruneReconcileAt = Date.now();
|
||||
};
|
||||
|
||||
if (params.deferIndex) {
|
||||
@@ -1991,7 +1918,7 @@ export abstract class MemoryManagerSyncOps {
|
||||
|
||||
await flushPendingIndexItems();
|
||||
await deleteStaleRows();
|
||||
return { ...this.emptySourceSyncPlan(), scanOk: scan.ok };
|
||||
return this.emptySourceSyncPlan();
|
||||
}
|
||||
if ((params.prefixIndexItems?.length ?? 0) > 0) {
|
||||
throw new Error("Memory session sync prefix requires deferred source-wide indexing.");
|
||||
@@ -2053,7 +1980,7 @@ export abstract class MemoryManagerSyncOps {
|
||||
await runWithConcurrency(tasks, this.getIndexConcurrency());
|
||||
|
||||
await deleteStaleRows();
|
||||
return { ...this.emptySourceSyncPlan(), scanOk: scan.ok };
|
||||
return this.emptySourceSyncPlan();
|
||||
}
|
||||
|
||||
private createSyncProgress(
|
||||
@@ -2236,25 +2163,20 @@ export abstract class MemoryManagerSyncOps {
|
||||
((!hasTargetSessionFiles && params?.force) || needsFullReindex || this.dirty);
|
||||
const shouldSyncSessions = this.shouldSyncSessions(params, needsFullReindex);
|
||||
|
||||
// A sync whose enumeration failed indexed nothing for that source; keep
|
||||
// its dirty state so the next eligible sync retries instead of treating
|
||||
// the source as clean until something marks it dirty again.
|
||||
if (this.shouldDeferSourceWideBatch()) {
|
||||
const outcome = await this.executeSourceWideSync({
|
||||
await this.executeSourceWideSync({
|
||||
shouldSyncMemory,
|
||||
shouldSyncSessions,
|
||||
needsFullReindex,
|
||||
targetSessionFiles: targetSessionFiles ? Array.from(targetSessionFiles) : undefined,
|
||||
progress: progress ?? undefined,
|
||||
});
|
||||
if (shouldSyncMemory && outcome.memoryScanOk) {
|
||||
if (shouldSyncMemory) {
|
||||
this.dirty = false;
|
||||
}
|
||||
if (shouldSyncSessions) {
|
||||
if (outcome.sessionsScanOk) {
|
||||
this.sessionsDirty = false;
|
||||
this.sessionsDirtyFiles.clear();
|
||||
}
|
||||
this.sessionsDirty = false;
|
||||
this.sessionsDirtyFiles.clear();
|
||||
} else if (this.sessionsDirtyFiles.size > 0) {
|
||||
this.sessionsDirty = true;
|
||||
} else {
|
||||
@@ -2262,25 +2184,18 @@ export abstract class MemoryManagerSyncOps {
|
||||
}
|
||||
} else {
|
||||
if (shouldSyncMemory) {
|
||||
const memoryPlan = await this.syncMemoryFiles({
|
||||
needsFullReindex,
|
||||
progress: progress ?? undefined,
|
||||
});
|
||||
if (memoryPlan.scanOk) {
|
||||
this.dirty = false;
|
||||
}
|
||||
await this.syncMemoryFiles({ needsFullReindex, progress: progress ?? undefined });
|
||||
this.dirty = false;
|
||||
}
|
||||
|
||||
if (shouldSyncSessions) {
|
||||
const sessionPlan = await this.syncSessionFiles({
|
||||
await this.syncSessionFiles({
|
||||
needsFullReindex,
|
||||
targetSessionFiles: targetSessionFiles ? Array.from(targetSessionFiles) : undefined,
|
||||
progress: progress ?? undefined,
|
||||
});
|
||||
if (sessionPlan.scanOk) {
|
||||
this.sessionsDirty = false;
|
||||
this.sessionsDirtyFiles.clear();
|
||||
}
|
||||
this.sessionsDirty = false;
|
||||
this.sessionsDirtyFiles.clear();
|
||||
} else if (this.sessionsDirtyFiles.size > 0) {
|
||||
this.sessionsDirty = true;
|
||||
} else {
|
||||
@@ -2448,14 +2363,12 @@ export abstract class MemoryManagerSyncOps {
|
||||
);
|
||||
|
||||
if (this.shouldDeferSourceWideBatch()) {
|
||||
const outcome = await this.executeSourceWideSync({
|
||||
await this.executeSourceWideSync({
|
||||
shouldSyncMemory,
|
||||
shouldSyncSessions,
|
||||
needsFullReindex: true,
|
||||
progress: params.progress,
|
||||
});
|
||||
this.assertReindexScanOk("memory", outcome.memoryScanOk);
|
||||
this.assertReindexScanOk("sessions", outcome.sessionsScanOk);
|
||||
if (shouldSyncMemory) {
|
||||
this.dirty = false;
|
||||
}
|
||||
@@ -2469,20 +2382,12 @@ export abstract class MemoryManagerSyncOps {
|
||||
}
|
||||
} else {
|
||||
if (shouldSyncMemory) {
|
||||
const memoryPlan = await this.syncMemoryFiles({
|
||||
needsFullReindex: true,
|
||||
progress: params.progress,
|
||||
});
|
||||
this.assertReindexScanOk("memory", memoryPlan.scanOk);
|
||||
await this.syncMemoryFiles({ needsFullReindex: true, progress: params.progress });
|
||||
this.dirty = false;
|
||||
}
|
||||
|
||||
if (shouldSyncSessions) {
|
||||
const sessionPlan = await this.syncSessionFiles({
|
||||
needsFullReindex: true,
|
||||
progress: params.progress,
|
||||
});
|
||||
this.assertReindexScanOk("sessions", sessionPlan.scanOk);
|
||||
await this.syncSessionFiles({ needsFullReindex: true, progress: params.progress });
|
||||
this.sessionsDirty = false;
|
||||
this.sessionsDirtyFiles.clear();
|
||||
} else if (this.sessionsDirtyFiles.size > 0) {
|
||||
@@ -2561,14 +2466,12 @@ export abstract class MemoryManagerSyncOps {
|
||||
);
|
||||
|
||||
if (this.shouldDeferSourceWideBatch()) {
|
||||
const outcome = await this.executeSourceWideSync({
|
||||
await this.executeSourceWideSync({
|
||||
shouldSyncMemory,
|
||||
shouldSyncSessions,
|
||||
needsFullReindex: true,
|
||||
progress: params.progress,
|
||||
});
|
||||
this.assertReindexScanOk("memory", outcome.memoryScanOk);
|
||||
this.assertReindexScanOk("sessions", outcome.sessionsScanOk);
|
||||
if (shouldSyncMemory) {
|
||||
this.dirty = false;
|
||||
}
|
||||
@@ -2582,20 +2485,12 @@ export abstract class MemoryManagerSyncOps {
|
||||
}
|
||||
} else {
|
||||
if (shouldSyncMemory) {
|
||||
const memoryPlan = await this.syncMemoryFiles({
|
||||
needsFullReindex: true,
|
||||
progress: params.progress,
|
||||
});
|
||||
this.assertReindexScanOk("memory", memoryPlan.scanOk);
|
||||
await this.syncMemoryFiles({ needsFullReindex: true, progress: params.progress });
|
||||
this.dirty = false;
|
||||
}
|
||||
|
||||
if (shouldSyncSessions) {
|
||||
const sessionPlan = await this.syncSessionFiles({
|
||||
needsFullReindex: true,
|
||||
progress: params.progress,
|
||||
});
|
||||
this.assertReindexScanOk("sessions", sessionPlan.scanOk);
|
||||
await this.syncSessionFiles({ needsFullReindex: true, progress: params.progress });
|
||||
this.sessionsDirty = false;
|
||||
this.sessionsDirtyFiles.clear();
|
||||
} else if (this.sessionsDirtyFiles.size > 0) {
|
||||
|
||||
@@ -24,10 +24,10 @@ import {
|
||||
deriveQmdScopeChannel,
|
||||
deriveQmdScopeChatType,
|
||||
isQmdScopeAllowed,
|
||||
listSessionFilesForAgent,
|
||||
parseQmdQueryJson,
|
||||
resolveCliSpawnInvocation,
|
||||
runCliCommand,
|
||||
scanSessionFilesForAgent,
|
||||
type QmdQueryResult,
|
||||
type SessionFileEntry,
|
||||
} from "openclaw/plugin-sdk/memory-core-host-engine-qmd";
|
||||
@@ -2466,8 +2466,7 @@ export class QmdMemoryManager implements MemorySearchManager {
|
||||
const exportDir = this.sessionExporter.dir;
|
||||
await fs.mkdir(exportDir, { recursive: true });
|
||||
const exportRoot = await root(exportDir);
|
||||
const scan = await scanSessionFilesForAgent(this.agentId);
|
||||
const files = scan.files;
|
||||
const files = await listSessionFilesForAgent(this.agentId);
|
||||
const keep = new Set<string>();
|
||||
const tracked = new Set<string>();
|
||||
const cutoff = this.sessionExporter.retentionMs
|
||||
@@ -2497,12 +2496,6 @@ export class QmdMemoryManager implements MemorySearchManager {
|
||||
});
|
||||
keep.add(target);
|
||||
}
|
||||
if (!scan.ok) {
|
||||
// A failed listing is non-authoritative: reconciling against it would
|
||||
// delete the whole exported corpus and force a full re-export on the
|
||||
// next pass. Keep existing exports and state until a real listing.
|
||||
return;
|
||||
}
|
||||
const exported = await exportRoot.list(".").catch(() => []);
|
||||
for (const name of exported) {
|
||||
if (!name.endsWith(".md")) {
|
||||
|
||||
@@ -368,6 +368,30 @@ describe("getMemorySearchManager caching", () => {
|
||||
expect(searchResults).toHaveLength(1);
|
||||
});
|
||||
|
||||
it("returns the qmd startup failure when builtin fallback is unavailable", async () => {
|
||||
const cfg = createQmdCfg("missing-qmd-no-builtin");
|
||||
checkQmdBinaryAvailability.mockResolvedValueOnce({
|
||||
available: false,
|
||||
reason: "binary",
|
||||
error: "spawn qmd ENOENT",
|
||||
});
|
||||
mockMemoryIndexGet.mockRejectedValueOnce(
|
||||
new Error(
|
||||
'Memory search unavailable: embedding provider "openai" is configured but unavailable.',
|
||||
),
|
||||
);
|
||||
|
||||
const result = await getMemorySearchManager({ cfg, agentId: "missing-qmd-no-builtin" });
|
||||
|
||||
expect(result.manager).toBeNull();
|
||||
expect(result.error).toContain("qmd binary unavailable (qmd): spawn qmd ENOENT");
|
||||
expect(result.error).toContain(
|
||||
'builtin fallback unavailable: Memory search unavailable: embedding provider "openai" is configured but unavailable.',
|
||||
);
|
||||
expect(createQmdManagerMock).not.toHaveBeenCalled();
|
||||
expect(mockMemoryIndexGet).toHaveBeenCalledTimes(1);
|
||||
});
|
||||
|
||||
it("treats legacy qmd unavailable results without a reason as binary failures", async () => {
|
||||
const cfg = createQmdCfg("missing-qmd-legacy");
|
||||
checkQmdBinaryAvailability.mockResolvedValueOnce({
|
||||
|
||||
@@ -262,16 +262,18 @@ export async function getMemorySearchManager(params: {
|
||||
}
|
||||
|
||||
if (transient) {
|
||||
const { manager } = await createPrimaryQmdManager(
|
||||
const { manager, failureReason } = await createPrimaryQmdManager(
|
||||
params.purpose === "cli" ? "cli" : "status",
|
||||
);
|
||||
return manager ? { manager } : await getBuiltinMemorySearchManager(params);
|
||||
return manager
|
||||
? { manager }
|
||||
: await getBuiltinMemorySearchManagerAfterQmdFailure(params, failureReason);
|
||||
}
|
||||
|
||||
const recentFailure = getActiveQmdManagerOpenFailure(scopeKey, identityKey);
|
||||
if (recentFailure) {
|
||||
log.debug?.(`qmd memory unavailable; using builtin during cooldown: ${recentFailure.reason}`);
|
||||
return await getBuiltinMemorySearchManager(params);
|
||||
return await getBuiltinMemorySearchManagerAfterQmdFailure(params, recentFailure.reason);
|
||||
}
|
||||
|
||||
const pending = PENDING_QMD_MANAGER_CREATES.get(scopeKey);
|
||||
@@ -280,16 +282,14 @@ export async function getMemorySearchManager(params: {
|
||||
return await getMemorySearchManager(params);
|
||||
}
|
||||
|
||||
let pendingFailureReason: string | undefined;
|
||||
const pendingCreate: PendingQmdManagerCreate = {
|
||||
identityKey,
|
||||
promise: (async () => {
|
||||
const created = await createFullQmdManager(identityKey);
|
||||
if (!created.entry) {
|
||||
recordQmdManagerOpenFailure(
|
||||
scopeKey,
|
||||
identityKey,
|
||||
created.failureReason ?? "qmd memory unavailable",
|
||||
);
|
||||
pendingFailureReason = created.failureReason ?? "qmd memory unavailable";
|
||||
recordQmdManagerOpenFailure(scopeKey, identityKey, pendingFailureReason);
|
||||
return null;
|
||||
}
|
||||
QMD_MANAGER_CACHE.set(scopeKey, created.entry);
|
||||
@@ -308,12 +308,35 @@ export async function getMemorySearchManager(params: {
|
||||
};
|
||||
PENDING_QMD_MANAGER_CREATES.set(scopeKey, pendingCreate);
|
||||
const manager = await pendingCreate.promise;
|
||||
return manager ? { manager } : await getBuiltinMemorySearchManager(params);
|
||||
return manager
|
||||
? { manager }
|
||||
: await getBuiltinMemorySearchManagerAfterQmdFailure(params, pendingFailureReason);
|
||||
}
|
||||
|
||||
return await getBuiltinMemorySearchManager(params);
|
||||
}
|
||||
|
||||
async function getBuiltinMemorySearchManagerAfterQmdFailure(
|
||||
params: {
|
||||
cfg: OpenClawConfig;
|
||||
agentId: string;
|
||||
purpose?: MemorySearchManagerPurpose;
|
||||
},
|
||||
qmdFailureReason: string | undefined,
|
||||
): Promise<MemorySearchManagerResult> {
|
||||
const fallback = await getBuiltinMemorySearchManager(params);
|
||||
if (fallback.manager || !qmdFailureReason) {
|
||||
return fallback;
|
||||
}
|
||||
const fallbackError = fallback.error?.trim();
|
||||
return {
|
||||
manager: null,
|
||||
error: fallbackError
|
||||
? `${qmdFailureReason}; builtin fallback unavailable: ${fallbackError}`
|
||||
: qmdFailureReason,
|
||||
};
|
||||
}
|
||||
|
||||
async function getBuiltinMemorySearchManager(params: {
|
||||
cfg: OpenClawConfig;
|
||||
agentId: string;
|
||||
|
||||
@@ -10,6 +10,7 @@ import {
|
||||
runWikiChatGptImport,
|
||||
runWikiChatGptRollback,
|
||||
runWikiDoctor,
|
||||
runWikiOkfImport,
|
||||
runWikiStatus,
|
||||
} from "./cli.js";
|
||||
import type { MemoryWikiPluginConfig } from "./config.js";
|
||||
@@ -27,6 +28,7 @@ vi.mock("openclaw/plugin-sdk/gateway-runtime", () => ({
|
||||
const { createVault } = createMemoryWikiTestHarness();
|
||||
let suiteRoot = "";
|
||||
let caseIndex = 0;
|
||||
let stdoutWriteMock: ReturnType<typeof vi.fn>;
|
||||
|
||||
describe("memory-wiki cli", () => {
|
||||
beforeAll(async () => {
|
||||
@@ -41,8 +43,9 @@ describe("memory-wiki cli", () => {
|
||||
|
||||
beforeEach(() => {
|
||||
callGatewayFromCliMock.mockReset();
|
||||
stdoutWriteMock = vi.fn(() => true);
|
||||
vi.spyOn(process.stdout, "write").mockImplementation(
|
||||
(() => true) as typeof process.stdout.write,
|
||||
stdoutWriteMock as unknown as typeof process.stdout.write,
|
||||
);
|
||||
process.exitCode = undefined;
|
||||
});
|
||||
@@ -174,6 +177,65 @@ describe("memory-wiki cli", () => {
|
||||
);
|
||||
});
|
||||
|
||||
it("registers OKF import and searches imported concepts", async () => {
|
||||
const { rootDir, config } = await createCliVault();
|
||||
const bundlePath = path.join(rootDir, "okf-bundle");
|
||||
await fs.mkdir(path.join(bundlePath, "tables"), { recursive: true });
|
||||
await fs.writeFile(path.join(bundlePath, "index.md"), "# Sales OKF\n", "utf8");
|
||||
await fs.writeFile(
|
||||
path.join(bundlePath, "tables", "customers.md"),
|
||||
`---
|
||||
type: BigQuery Table
|
||||
title: Customers
|
||||
---
|
||||
|
||||
Customer rows.
|
||||
`,
|
||||
"utf8",
|
||||
);
|
||||
await fs.writeFile(
|
||||
path.join(bundlePath, "tables", "orders.md"),
|
||||
`---
|
||||
type: BigQuery Table
|
||||
title: Orders
|
||||
description: One row per completed order.
|
||||
---
|
||||
|
||||
Orders join to [customers](/tables/customers.md).
|
||||
`,
|
||||
"utf8",
|
||||
);
|
||||
|
||||
const program = new Command();
|
||||
program.name("test");
|
||||
registerWikiCli(program, config);
|
||||
|
||||
await program.parseAsync(["wiki", "okf", "import", bundlePath, "--json"], { from: "user" });
|
||||
|
||||
const importOutput = String(stdoutWriteMock.mock.calls.at(-1)?.[0] ?? "");
|
||||
const importResult = JSON.parse(importOutput) as Awaited<ReturnType<typeof runWikiOkfImport>>;
|
||||
expect(importResult.importedCount).toBe(2);
|
||||
expect(importResult.pagePaths).toEqual(
|
||||
expect.arrayContaining([
|
||||
expect.stringMatching(/^concepts\/okf-okf-bundle-[0-9a-f]{8}-tables-orders-/),
|
||||
]),
|
||||
);
|
||||
|
||||
stdoutWriteMock.mockClear();
|
||||
await program.parseAsync(["wiki", "search", "completed order", "--json"], { from: "user" });
|
||||
|
||||
const searchOutput = String(stdoutWriteMock.mock.calls.at(-1)?.[0] ?? "");
|
||||
const searchResults = JSON.parse(searchOutput) as Array<{ path: string; title: string }>;
|
||||
expect(searchResults).toEqual(
|
||||
expect.arrayContaining([
|
||||
expect.objectContaining({
|
||||
title: "Orders",
|
||||
path: expect.stringMatching(/^concepts\/okf-okf-bundle-[0-9a-f]{8}-tables-orders-/),
|
||||
}),
|
||||
]),
|
||||
);
|
||||
});
|
||||
|
||||
it("rejects apply confidence values outside the documented range", async () => {
|
||||
const { config } = await createCliVault();
|
||||
const program = new Command();
|
||||
|
||||
@@ -33,6 +33,7 @@ import {
|
||||
runObsidianOpen,
|
||||
runObsidianSearch,
|
||||
} from "./obsidian.js";
|
||||
import { formatOkfImportSummary, importMemoryWikiOkfBundle } from "./okf.js";
|
||||
import {
|
||||
getMemoryWikiPage,
|
||||
searchMemoryWiki,
|
||||
@@ -88,6 +89,10 @@ type WikiIngestCommandOptions = {
|
||||
title?: string;
|
||||
};
|
||||
|
||||
type WikiOkfImportCommandOptions = {
|
||||
json?: boolean;
|
||||
};
|
||||
|
||||
type WikiSearchCommandOptions = {
|
||||
json?: boolean;
|
||||
maxResults?: number;
|
||||
@@ -590,6 +595,24 @@ export async function runWikiIngest(params: {
|
||||
});
|
||||
}
|
||||
|
||||
export async function runWikiOkfImport(params: {
|
||||
config: ResolvedMemoryWikiConfig;
|
||||
bundlePath: string;
|
||||
json?: boolean;
|
||||
stdout?: Pick<NodeJS.WriteStream, "write">;
|
||||
}) {
|
||||
return runWikiCommandWithSummary({
|
||||
json: params.json,
|
||||
stdout: params.stdout,
|
||||
run: () =>
|
||||
importMemoryWikiOkfBundle({
|
||||
config: params.config,
|
||||
bundlePath: params.bundlePath,
|
||||
}),
|
||||
render: formatOkfImportSummary,
|
||||
});
|
||||
}
|
||||
|
||||
export async function runWikiSearch(params: {
|
||||
config: ResolvedMemoryWikiConfig;
|
||||
appConfig?: OpenClawConfig;
|
||||
@@ -965,6 +988,16 @@ export function registerWikiCli(
|
||||
await runWikiIngest({ config, inputPath, title: opts.title, json: opts.json });
|
||||
});
|
||||
|
||||
const okf = wiki.command("okf").description("Import Open Knowledge Format bundles");
|
||||
okf
|
||||
.command("import")
|
||||
.description("Import an unpacked OKF bundle into wiki concept pages")
|
||||
.argument("<path>", "OKF bundle directory")
|
||||
.option("--json", "Print JSON")
|
||||
.action(async (bundlePath: string, opts: WikiOkfImportCommandOptions) => {
|
||||
await runWikiOkfImport({ config, bundlePath, json: opts.json });
|
||||
});
|
||||
|
||||
addWikiSearchConfigOptions(
|
||||
wiki
|
||||
.command("search")
|
||||
|
||||
@@ -4,7 +4,7 @@ import path from "node:path";
|
||||
import { appendRegularFile } from "openclaw/plugin-sdk/security-runtime";
|
||||
|
||||
type MemoryWikiLogEntry = {
|
||||
type: "init" | "ingest" | "compile" | "lint";
|
||||
type: "init" | "ingest" | "okf-import" | "compile" | "lint";
|
||||
timestamp: string;
|
||||
details?: Record<string, unknown>;
|
||||
};
|
||||
|
||||
609
extensions/memory-wiki/src/okf.test.ts
Normal file
609
extensions/memory-wiki/src/okf.test.ts
Normal file
@@ -0,0 +1,609 @@
|
||||
// Memory Wiki tests cover Open Knowledge Format import behavior.
|
||||
import fs from "node:fs/promises";
|
||||
import path from "node:path";
|
||||
import { describe, expect, it } from "vitest";
|
||||
import { parseWikiMarkdown } from "./markdown.js";
|
||||
import { importMemoryWikiOkfBundle } from "./okf.js";
|
||||
import { searchMemoryWiki } from "./query.js";
|
||||
import { createMemoryWikiTestHarness } from "./test-helpers.js";
|
||||
|
||||
const { createTempDir, createVault } = createMemoryWikiTestHarness();
|
||||
|
||||
function getOnlyPagePath(paths: string[]): string {
|
||||
expect(paths).toHaveLength(1);
|
||||
const [pagePath] = paths;
|
||||
if (!pagePath) {
|
||||
throw new Error("Expected OKF import to produce one page path.");
|
||||
}
|
||||
return pagePath;
|
||||
}
|
||||
|
||||
async function writeOkfBundle(rootDir: string) {
|
||||
const bundlePath = path.join(rootDir, "sales-okf");
|
||||
await fs.mkdir(path.join(bundlePath, "tables"), { recursive: true });
|
||||
await fs.mkdir(path.join(bundlePath, "metrics"), { recursive: true });
|
||||
await fs.writeFile(
|
||||
path.join(bundlePath, "index.md"),
|
||||
`---
|
||||
id: sales-okf
|
||||
okf_version: "0.1"
|
||||
---
|
||||
|
||||
# Sales Bundle
|
||||
`,
|
||||
"utf8",
|
||||
);
|
||||
await fs.writeFile(path.join(bundlePath, "log.md"), "# Directory Update Log\n", "utf8");
|
||||
await fs.writeFile(
|
||||
path.join(bundlePath, "tables", "customers.md"),
|
||||
`---
|
||||
type: BigQuery Table
|
||||
title: Customers
|
||||
description: Customer table.
|
||||
resource: https://console.cloud.google.com/bigquery?p=acme&d=sales&t=customers
|
||||
tags: [sales, customers]
|
||||
timestamp: 2026-05-28T00:00:00Z
|
||||
producer_field:
|
||||
owner: data
|
||||
---
|
||||
|
||||
# Schema
|
||||
|
||||
Customer rows.
|
||||
`,
|
||||
"utf8",
|
||||
);
|
||||
await fs.writeFile(
|
||||
path.join(bundlePath, "tables", "orders.md"),
|
||||
`---
|
||||
type: BigQuery Table
|
||||
title: Orders
|
||||
description: One row per completed order.
|
||||
tags:
|
||||
- sales
|
||||
- orders
|
||||
---
|
||||
|
||||
# Schema
|
||||
|
||||
Joined with [Customers](/tables/customers.md) and the [weekly metric](../metrics/weekly-active-users.md).
|
||||
Titled link to [weekly metric](../metrics/weekly-active-users.md "metric docs").
|
||||
|
||||
Inline code keeps \`[customers](/tables/customers.md)\` unchanged.
|
||||
|
||||
\`\`\`markdown
|
||||
[customers](/tables/customers.md)
|
||||
\`\`\`
|
||||
|
||||
External citation stays as [BigQuery](https://cloud.google.com/bigquery).
|
||||
`,
|
||||
"utf8",
|
||||
);
|
||||
await fs.writeFile(
|
||||
path.join(bundlePath, "metrics", "weekly-active-users.md"),
|
||||
`---
|
||||
type: Metric
|
||||
title: Weekly Active Users
|
||||
---
|
||||
|
||||
Computed from [orders](../tables/orders.md).
|
||||
`,
|
||||
"utf8",
|
||||
);
|
||||
await fs.writeFile(
|
||||
path.join(bundlePath, "tables", "draft.md"),
|
||||
`---
|
||||
title: Draft
|
||||
---
|
||||
|
||||
Missing type.
|
||||
`,
|
||||
"utf8",
|
||||
);
|
||||
return bundlePath;
|
||||
}
|
||||
|
||||
describe("importMemoryWikiOkfBundle", () => {
|
||||
it("imports OKF concept documents as searchable wiki concept pages", async () => {
|
||||
const rootDir = await createTempDir("memory-wiki-okf-");
|
||||
const bundlePath = await writeOkfBundle(rootDir);
|
||||
const { config } = await createVault({
|
||||
rootDir: path.join(rootDir, "vault"),
|
||||
});
|
||||
|
||||
const result = await importMemoryWikiOkfBundle({
|
||||
config,
|
||||
bundlePath,
|
||||
nowMs: Date.UTC(2026, 5, 12, 10, 0, 0),
|
||||
});
|
||||
|
||||
expect(result.okfVersion).toBe("0.1");
|
||||
expect(result.importedCount).toBe(3);
|
||||
expect(result.skippedCount).toBe(1);
|
||||
expect(result.warnings[0]).toMatchObject({
|
||||
code: "missing-type",
|
||||
path: "tables/draft.md",
|
||||
});
|
||||
expect(result.pagePaths).toHaveLength(3);
|
||||
const repeat = await importMemoryWikiOkfBundle({
|
||||
config,
|
||||
bundlePath,
|
||||
nowMs: Date.UTC(2026, 5, 12, 10, 5, 0),
|
||||
});
|
||||
expect(repeat.importedCount).toBe(3);
|
||||
expect(repeat.updatedCount).toBe(0);
|
||||
|
||||
const ordersPath = result.pagePaths.find((pagePath) => pagePath.includes("orders"));
|
||||
expect(ordersPath).toBeTruthy();
|
||||
const ordersRaw = await fs.readFile(path.join(config.vault.path, ordersPath!), "utf8");
|
||||
const orders = parseWikiMarkdown(ordersRaw);
|
||||
expect(orders.frontmatter).toMatchObject({
|
||||
pageType: "concept",
|
||||
title: "Orders",
|
||||
sourceType: "okf",
|
||||
provenanceMode: "okf-import",
|
||||
okfConceptId: "tables/orders",
|
||||
okfType: "BigQuery Table",
|
||||
});
|
||||
expect(orders.frontmatter.sourceIds).toEqual([
|
||||
expect.stringMatching(/^source\.okf\.sales-okf$/),
|
||||
]);
|
||||
expect(orders.body).toMatch(/\]\(okf-sales-okf-tables-customers-/);
|
||||
expect(orders.body).toMatch(/\]\(okf-sales-okf-metrics-weekly-active-users-/);
|
||||
expect(orders.body).toContain('"metric docs"');
|
||||
expect(orders.body).toContain("`[customers](/tables/customers.md)`");
|
||||
expect(orders.body).toContain("```markdown\n[customers](/tables/customers.md)\n```");
|
||||
expect(orders.body).toContain("https://cloud.google.com/bigquery");
|
||||
|
||||
const okf = orders.frontmatter.okf as Record<string, unknown>;
|
||||
expect(okf).toMatchObject({
|
||||
version: "0.1",
|
||||
bundleName: "sales-okf",
|
||||
conceptId: "tables/orders",
|
||||
sourceRelativePath: "tables/orders.md",
|
||||
});
|
||||
expect(orders.frontmatter.relationships).toEqual(
|
||||
expect.arrayContaining([
|
||||
expect.objectContaining({
|
||||
targetPath: expect.stringMatching(/^concepts\/okf-sales-okf-tables-customers-/),
|
||||
kind: "okf-link",
|
||||
}),
|
||||
expect.objectContaining({
|
||||
targetPath: expect.stringMatching(
|
||||
/^concepts\/okf-sales-okf-metrics-weekly-active-users-/,
|
||||
),
|
||||
kind: "okf-link",
|
||||
}),
|
||||
]),
|
||||
);
|
||||
|
||||
const customersPath = result.pagePaths.find((pagePath) => pagePath.includes("customers"));
|
||||
const customersRaw = await fs.readFile(path.join(config.vault.path, customersPath!), "utf8");
|
||||
const customers = parseWikiMarkdown(customersRaw);
|
||||
const customersOkf = customers.frontmatter.okf as Record<string, unknown>;
|
||||
expect(customersOkf.frontmatter).toMatchObject({
|
||||
producer_field: { owner: "data" },
|
||||
});
|
||||
|
||||
const searchResults = await searchMemoryWiki({
|
||||
config,
|
||||
query: "completed order",
|
||||
searchCorpus: "wiki",
|
||||
});
|
||||
expect(searchResults.map((searchResult) => searchResult.path)).toContain(ordersPath);
|
||||
});
|
||||
|
||||
it("caps generated concept filenames for long OKF concept paths", async () => {
|
||||
const rootDir = await createTempDir("memory-wiki-okf-long-");
|
||||
const bundlePath = path.join(rootDir, "long-okf");
|
||||
const deepSegments = Array.from({ length: 40 }, (_, index) => `segment-${index}`);
|
||||
const deepDir = path.join(bundlePath, ...deepSegments);
|
||||
await fs.mkdir(deepDir, { recursive: true });
|
||||
await fs.writeFile(
|
||||
path.join(deepDir, "orders.md"),
|
||||
`---
|
||||
type: BigQuery Table
|
||||
title: Long Orders
|
||||
---
|
||||
|
||||
Long concept body.
|
||||
`,
|
||||
"utf8",
|
||||
);
|
||||
const { config } = await createVault({
|
||||
rootDir: path.join(rootDir, "vault"),
|
||||
});
|
||||
|
||||
const result = await importMemoryWikiOkfBundle({
|
||||
config,
|
||||
bundlePath,
|
||||
nowMs: Date.UTC(2026, 5, 12, 10, 0, 0),
|
||||
});
|
||||
|
||||
expect(result.importedCount).toBe(1);
|
||||
const [pagePath] = result.pagePaths;
|
||||
expect(pagePath).toBeDefined();
|
||||
if (!pagePath) {
|
||||
throw new Error("Expected OKF import to produce a page path.");
|
||||
}
|
||||
const fileName = path.basename(pagePath);
|
||||
expect(Buffer.byteLength(fileName)).toBeLessThanOrEqual(255);
|
||||
await expect(fs.readFile(path.join(config.vault.path, pagePath), "utf8")).resolves.toContain(
|
||||
"Long concept body.",
|
||||
);
|
||||
});
|
||||
|
||||
it("namespaces concept pages by bundle so repeated OKF paths do not overwrite", async () => {
|
||||
const rootDir = await createTempDir("memory-wiki-okf-bundles-");
|
||||
const firstBundle = path.join(rootDir, "first-bundle");
|
||||
const secondBundle = path.join(rootDir, "second-bundle");
|
||||
for (const [bundlePath, title] of [
|
||||
[firstBundle, "First Customers"],
|
||||
[secondBundle, "Second Customers"],
|
||||
] as const) {
|
||||
await fs.mkdir(path.join(bundlePath, "tables"), { recursive: true });
|
||||
await fs.writeFile(
|
||||
path.join(bundlePath, "tables", "customers.md"),
|
||||
`---
|
||||
type: BigQuery Table
|
||||
title: ${title}
|
||||
---
|
||||
|
||||
${title} body.
|
||||
`,
|
||||
"utf8",
|
||||
);
|
||||
}
|
||||
const { config } = await createVault({
|
||||
rootDir: path.join(rootDir, "vault"),
|
||||
});
|
||||
|
||||
const first = await importMemoryWikiOkfBundle({
|
||||
config,
|
||||
bundlePath: firstBundle,
|
||||
nowMs: Date.UTC(2026, 5, 12, 10, 0, 0),
|
||||
});
|
||||
const second = await importMemoryWikiOkfBundle({
|
||||
config,
|
||||
bundlePath: secondBundle,
|
||||
nowMs: Date.UTC(2026, 5, 12, 10, 0, 0),
|
||||
});
|
||||
|
||||
const firstPath = getOnlyPagePath(first.pagePaths);
|
||||
const secondPath = getOnlyPagePath(second.pagePaths);
|
||||
expect(firstPath).not.toBe(secondPath);
|
||||
await expect(fs.readFile(path.join(config.vault.path, firstPath), "utf8")).resolves.toContain(
|
||||
"First Customers body.",
|
||||
);
|
||||
await expect(fs.readFile(path.join(config.vault.path, secondPath), "utf8")).resolves.toContain(
|
||||
"Second Customers body.",
|
||||
);
|
||||
});
|
||||
|
||||
it("removes stale concept pages when an OKF bundle drops a concept", async () => {
|
||||
const rootDir = await createTempDir("memory-wiki-okf-remove-");
|
||||
const bundlePath = path.join(rootDir, "removing-okf");
|
||||
await fs.mkdir(path.join(bundlePath, "tables"), { recursive: true });
|
||||
const customersPath = path.join(bundlePath, "tables", "customers.md");
|
||||
const ordersPath = path.join(bundlePath, "tables", "orders.md");
|
||||
await fs.writeFile(
|
||||
customersPath,
|
||||
`---
|
||||
type: BigQuery Table
|
||||
title: Customers
|
||||
---
|
||||
|
||||
Customer body.
|
||||
`,
|
||||
"utf8",
|
||||
);
|
||||
await fs.writeFile(
|
||||
ordersPath,
|
||||
`---
|
||||
type: BigQuery Table
|
||||
title: Orders
|
||||
---
|
||||
|
||||
Order body.
|
||||
`,
|
||||
"utf8",
|
||||
);
|
||||
const { config } = await createVault({
|
||||
rootDir: path.join(rootDir, "vault"),
|
||||
});
|
||||
const first = await importMemoryWikiOkfBundle({
|
||||
config,
|
||||
bundlePath,
|
||||
nowMs: Date.UTC(2026, 5, 12, 10, 0, 0),
|
||||
});
|
||||
const stalePagePath = first.pagePaths.find((pagePath) => pagePath.includes("orders"));
|
||||
expect(stalePagePath).toBeDefined();
|
||||
if (!stalePagePath) {
|
||||
throw new Error("Expected initial OKF import to include orders.");
|
||||
}
|
||||
|
||||
await fs.rm(ordersPath);
|
||||
const second = await importMemoryWikiOkfBundle({
|
||||
config,
|
||||
bundlePath,
|
||||
nowMs: Date.UTC(2026, 5, 12, 10, 0, 0),
|
||||
});
|
||||
|
||||
expect(second.importedCount).toBe(1);
|
||||
expect(second.removedCount).toBe(1);
|
||||
expect(second.removedPagePaths).toEqual([stalePagePath]);
|
||||
await expect(fs.stat(path.join(config.vault.path, stalePagePath))).rejects.toThrow();
|
||||
const results = await searchMemoryWiki({
|
||||
config,
|
||||
query: "Order body",
|
||||
searchCorpus: "wiki",
|
||||
});
|
||||
expect(results).toHaveLength(0);
|
||||
});
|
||||
|
||||
it("does not prune existing pages when current OKF scan has invalid concepts", async () => {
|
||||
const rootDir = await createTempDir("memory-wiki-okf-invalid-");
|
||||
const bundlePath = path.join(rootDir, "invalid-okf");
|
||||
await fs.mkdir(path.join(bundlePath, "tables"), { recursive: true });
|
||||
const customersPath = path.join(bundlePath, "tables", "customers.md");
|
||||
await fs.writeFile(
|
||||
customersPath,
|
||||
`---
|
||||
type: BigQuery Table
|
||||
title: Customers
|
||||
---
|
||||
|
||||
Customer body.
|
||||
`,
|
||||
"utf8",
|
||||
);
|
||||
const { config } = await createVault({
|
||||
rootDir: path.join(rootDir, "vault"),
|
||||
});
|
||||
const first = await importMemoryWikiOkfBundle({
|
||||
config,
|
||||
bundlePath,
|
||||
nowMs: Date.UTC(2026, 5, 12, 10, 0, 0),
|
||||
});
|
||||
const pagePath = getOnlyPagePath(first.pagePaths);
|
||||
await fs.writeFile(
|
||||
customersPath,
|
||||
`---
|
||||
title: Customers
|
||||
---
|
||||
|
||||
Temporarily invalid body.
|
||||
`,
|
||||
"utf8",
|
||||
);
|
||||
|
||||
const second = await importMemoryWikiOkfBundle({
|
||||
config,
|
||||
bundlePath,
|
||||
nowMs: Date.UTC(2026, 5, 12, 10, 0, 0),
|
||||
});
|
||||
|
||||
expect(second.importedCount).toBe(0);
|
||||
expect(second.skippedCount).toBe(1);
|
||||
expect(second.removedCount).toBe(0);
|
||||
await expect(fs.readFile(path.join(config.vault.path, pagePath), "utf8")).resolves.toContain(
|
||||
"Customer body.",
|
||||
);
|
||||
});
|
||||
|
||||
it("detects body-only changes on timestamp-shaped markdown lines", async () => {
|
||||
const rootDir = await createTempDir("memory-wiki-okf-body-timestamp-");
|
||||
const bundlePath = path.join(rootDir, "body-timestamp-okf");
|
||||
await fs.mkdir(path.join(bundlePath, "tables"), { recursive: true });
|
||||
const conceptPath = path.join(bundlePath, "tables", "events.md");
|
||||
await fs.writeFile(
|
||||
conceptPath,
|
||||
`---
|
||||
type: BigQuery Table
|
||||
title: Events
|
||||
---
|
||||
|
||||
updatedAt: 2026-06-12
|
||||
`,
|
||||
"utf8",
|
||||
);
|
||||
const { config } = await createVault({
|
||||
rootDir: path.join(rootDir, "vault"),
|
||||
});
|
||||
const first = await importMemoryWikiOkfBundle({
|
||||
config,
|
||||
bundlePath,
|
||||
nowMs: Date.UTC(2026, 5, 12, 10, 0, 0),
|
||||
});
|
||||
const pagePath = getOnlyPagePath(first.pagePaths);
|
||||
await fs.writeFile(
|
||||
conceptPath,
|
||||
`---
|
||||
type: BigQuery Table
|
||||
title: Events
|
||||
---
|
||||
|
||||
updatedAt: 2026-06-13
|
||||
`,
|
||||
"utf8",
|
||||
);
|
||||
|
||||
const second = await importMemoryWikiOkfBundle({
|
||||
config,
|
||||
bundlePath,
|
||||
nowMs: Date.UTC(2026, 5, 13, 10, 0, 0),
|
||||
});
|
||||
|
||||
expect(second.updatedCount).toBe(1);
|
||||
await expect(fs.readFile(path.join(config.vault.path, pagePath), "utf8")).resolves.toContain(
|
||||
"updatedAt: 2026-06-13",
|
||||
);
|
||||
});
|
||||
|
||||
it("rewrites percent-encoded OKF markdown links and preserves suffixes", async () => {
|
||||
const rootDir = await createTempDir("memory-wiki-okf-encoded-link-");
|
||||
const bundlePath = path.join(rootDir, "encoded-okf");
|
||||
await fs.mkdir(bundlePath, { recursive: true });
|
||||
await fs.writeFile(
|
||||
path.join(bundlePath, "BigQuery Table.md"),
|
||||
`---
|
||||
type: BigQuery Table
|
||||
title: BigQuery Table
|
||||
---
|
||||
|
||||
Table body.
|
||||
`,
|
||||
"utf8",
|
||||
);
|
||||
await fs.writeFile(
|
||||
path.join(bundlePath, "links.md"),
|
||||
`---
|
||||
type: Concept
|
||||
title: Links
|
||||
---
|
||||
|
||||
See [table](BigQuery%20Table.md?view=compact#columns).
|
||||
`,
|
||||
"utf8",
|
||||
);
|
||||
const { config } = await createVault({
|
||||
rootDir: path.join(rootDir, "vault"),
|
||||
});
|
||||
|
||||
const result = await importMemoryWikiOkfBundle({
|
||||
config,
|
||||
bundlePath,
|
||||
nowMs: Date.UTC(2026, 5, 12, 10, 0, 0),
|
||||
});
|
||||
|
||||
const linksPath = result.pagePaths.find((pagePath) => pagePath.includes("links"));
|
||||
expect(linksPath).toBeDefined();
|
||||
if (!linksPath) {
|
||||
throw new Error("Expected links page to be imported.");
|
||||
}
|
||||
await expect(fs.readFile(path.join(config.vault.path, linksPath), "utf8")).resolves.toMatch(
|
||||
/\[table\]\(okf-encoded-okf-[0-9a-f]{8}-bigquery-table-[^)]+\.md\?view=compact#columns\)/,
|
||||
);
|
||||
});
|
||||
|
||||
it("imports OKF concept frontmatter with CRLF line endings", async () => {
|
||||
const rootDir = await createTempDir("memory-wiki-okf-crlf-");
|
||||
const bundlePath = path.join(rootDir, "crlf-okf");
|
||||
await fs.mkdir(path.join(bundlePath, "tables"), { recursive: true });
|
||||
await fs.writeFile(
|
||||
path.join(bundlePath, "tables", "events.md"),
|
||||
[
|
||||
"---",
|
||||
"type: BigQuery Table",
|
||||
"title: Events",
|
||||
"---",
|
||||
"",
|
||||
"Windows-flavored frontmatter.",
|
||||
"",
|
||||
].join("\r\n"),
|
||||
"utf8",
|
||||
);
|
||||
const { config } = await createVault({
|
||||
rootDir: path.join(rootDir, "vault"),
|
||||
});
|
||||
|
||||
const result = await importMemoryWikiOkfBundle({
|
||||
config,
|
||||
bundlePath,
|
||||
nowMs: Date.UTC(2026, 5, 12, 10, 0, 0),
|
||||
});
|
||||
|
||||
expect(result.importedCount).toBe(1);
|
||||
expect(result.skippedCount).toBe(0);
|
||||
await expect(
|
||||
fs.readFile(path.join(config.vault.path, getOnlyPagePath(result.pagePaths)), "utf8"),
|
||||
).resolves.toContain("Windows-flavored frontmatter.");
|
||||
});
|
||||
|
||||
it("refuses to write imported OKF concept pages through symlinks", async () => {
|
||||
const rootDir = await createTempDir("memory-wiki-okf-symlink-");
|
||||
const bundlePath = path.join(rootDir, "safe-okf");
|
||||
await fs.mkdir(path.join(bundlePath, "tables"), { recursive: true });
|
||||
const conceptPath = path.join(bundlePath, "tables", "customers.md");
|
||||
await fs.writeFile(
|
||||
conceptPath,
|
||||
`---
|
||||
type: BigQuery Table
|
||||
title: Customers
|
||||
---
|
||||
|
||||
Original body.
|
||||
`,
|
||||
"utf8",
|
||||
);
|
||||
const { config } = await createVault({
|
||||
rootDir: path.join(rootDir, "vault"),
|
||||
});
|
||||
const first = await importMemoryWikiOkfBundle({
|
||||
config,
|
||||
bundlePath,
|
||||
nowMs: Date.UTC(2026, 5, 12, 10, 0, 0),
|
||||
});
|
||||
const pagePath = getOnlyPagePath(first.pagePaths);
|
||||
const pageAbsolutePath = path.join(config.vault.path, pagePath);
|
||||
const externalTarget = path.join(rootDir, "outside.md");
|
||||
await fs.writeFile(externalTarget, "external target\n", "utf8");
|
||||
await fs.rm(pageAbsolutePath);
|
||||
await fs.symlink(externalTarget, pageAbsolutePath);
|
||||
await fs.writeFile(
|
||||
conceptPath,
|
||||
`---
|
||||
type: BigQuery Table
|
||||
title: Customers
|
||||
---
|
||||
|
||||
Updated body.
|
||||
`,
|
||||
"utf8",
|
||||
);
|
||||
|
||||
await expect(
|
||||
importMemoryWikiOkfBundle({
|
||||
config,
|
||||
bundlePath,
|
||||
nowMs: Date.UTC(2026, 5, 12, 11, 0, 0),
|
||||
}),
|
||||
).rejects.toThrow("through symlink");
|
||||
await expect(fs.readFile(externalTarget, "utf8")).resolves.toBe("external target\n");
|
||||
});
|
||||
|
||||
it("refuses to import OKF concept files through hardlinks", async () => {
|
||||
const rootDir = await createTempDir("memory-wiki-okf-hardlink-");
|
||||
const bundlePath = path.join(rootDir, "hardlink-okf");
|
||||
await fs.mkdir(path.join(bundlePath, "tables"), { recursive: true });
|
||||
const externalSource = path.join(rootDir, "outside.md");
|
||||
await fs.writeFile(
|
||||
externalSource,
|
||||
`---
|
||||
type: BigQuery Table
|
||||
title: Private
|
||||
---
|
||||
|
||||
private body
|
||||
`,
|
||||
"utf8",
|
||||
);
|
||||
await fs.link(externalSource, path.join(bundlePath, "tables", "private.md"));
|
||||
const { config } = await createVault({
|
||||
rootDir: path.join(rootDir, "vault"),
|
||||
});
|
||||
|
||||
const result = await importMemoryWikiOkfBundle({
|
||||
config,
|
||||
bundlePath,
|
||||
nowMs: Date.UTC(2026, 5, 12, 10, 0, 0),
|
||||
});
|
||||
|
||||
expect(result.importedCount).toBe(0);
|
||||
expect(result.skippedCount).toBe(1);
|
||||
expect(result.warnings[0]).toMatchObject({
|
||||
code: "unreadable-entry",
|
||||
path: "tables/private.md",
|
||||
});
|
||||
});
|
||||
});
|
||||
746
extensions/memory-wiki/src/okf.ts
Normal file
746
extensions/memory-wiki/src/okf.ts
Normal file
@@ -0,0 +1,746 @@
|
||||
// Memory Wiki plugin module implements Open Knowledge Format import behavior.
|
||||
import { createHash } from "node:crypto";
|
||||
import fs from "node:fs/promises";
|
||||
import path from "node:path";
|
||||
import { FsSafeError, root as fsRoot } from "openclaw/plugin-sdk/security-runtime";
|
||||
import {
|
||||
normalizeOptionalString,
|
||||
normalizeSingleOrTrimmedStringList,
|
||||
uniqueStrings,
|
||||
} from "openclaw/plugin-sdk/string-coerce-runtime";
|
||||
import { compileMemoryWikiVault } from "./compile.js";
|
||||
import type { ResolvedMemoryWikiConfig } from "./config.js";
|
||||
import { appendMemoryWikiLog } from "./log.js";
|
||||
import {
|
||||
createWikiPageFilename,
|
||||
parseWikiMarkdown,
|
||||
renderWikiMarkdown,
|
||||
slugifyWikiSegment,
|
||||
WIKI_RELATED_END_MARKER,
|
||||
WIKI_RELATED_START_MARKER,
|
||||
} from "./markdown.js";
|
||||
import { resolveMemoryWikiTimestamp } from "./time.js";
|
||||
import { initializeMemoryWikiVault } from "./vault.js";
|
||||
|
||||
const OKF_RESERVED_FILENAMES = new Set(["index.md", "log.md"]);
|
||||
const OKF_MARKDOWN_LINK_PATTERN = /(!?)\[([^\]]*)\]\(([^)]+)\)/g;
|
||||
const OKF_FENCE_PATTERN = /^ {0,3}(`{3,}|~{3,})/;
|
||||
const OKF_RELATED_SECTION_PATTERN = new RegExp(
|
||||
`\\n+## Related\\n${WIKI_RELATED_START_MARKER}[\\s\\S]*?${WIKI_RELATED_END_MARKER}\\n?`,
|
||||
"g",
|
||||
);
|
||||
const OKF_VOLATILE_TIMESTAMP_LINE_PATTERN = /^(?:importedAt|updatedAt): .*\n/gm;
|
||||
const OKF_HASH_CHARS = 8;
|
||||
|
||||
type FileStatLike = {
|
||||
isFile?: unknown;
|
||||
nlink?: unknown;
|
||||
};
|
||||
|
||||
type OkfConceptDocument = {
|
||||
conceptId: string;
|
||||
relativePath: string;
|
||||
absolutePath: string;
|
||||
frontmatter: Record<string, unknown>;
|
||||
body: string;
|
||||
type: string;
|
||||
title: string;
|
||||
description?: string;
|
||||
resource?: string;
|
||||
tags: string[];
|
||||
timestamp?: string;
|
||||
};
|
||||
|
||||
type OkfImportedPage = {
|
||||
conceptId: string;
|
||||
sourcePath: string;
|
||||
pageId: string;
|
||||
pagePath: string;
|
||||
title: string;
|
||||
created: boolean;
|
||||
};
|
||||
|
||||
export type ImportMemoryWikiOkfWarning = {
|
||||
code: "invalid-concept" | "missing-type" | "unreadable-entry";
|
||||
path: string;
|
||||
message: string;
|
||||
};
|
||||
|
||||
export type ImportMemoryWikiOkfResult = {
|
||||
bundlePath: string;
|
||||
bundleName: string;
|
||||
okfVersion?: string;
|
||||
importedCount: number;
|
||||
updatedCount: number;
|
||||
removedCount: number;
|
||||
skippedCount: number;
|
||||
pagePaths: string[];
|
||||
removedPagePaths: string[];
|
||||
warnings: ImportMemoryWikiOkfWarning[];
|
||||
indexUpdatedFiles: string[];
|
||||
};
|
||||
|
||||
function toPosixPath(value: string): string {
|
||||
return value.split(path.sep).join("/");
|
||||
}
|
||||
|
||||
function trimMarkdownExtension(value: string): string {
|
||||
return value.replace(/\.md$/i, "");
|
||||
}
|
||||
|
||||
function isRegularFileStat(value: unknown): value is FileStatLike & { nlink: number } {
|
||||
if (!value || typeof value !== "object") {
|
||||
return false;
|
||||
}
|
||||
const stat = value as FileStatLike;
|
||||
const isFile =
|
||||
typeof stat.isFile === "function"
|
||||
? (stat.isFile as () => boolean).call(stat)
|
||||
: stat.isFile === true;
|
||||
return isFile && typeof stat.nlink === "number";
|
||||
}
|
||||
|
||||
type OkfBundleMetadata = {
|
||||
key: string;
|
||||
version?: string;
|
||||
};
|
||||
|
||||
function createOkfBundleKey(params: {
|
||||
rootFrontmatter: Record<string, unknown>;
|
||||
bundleName: string;
|
||||
bundlePath: string;
|
||||
}): string {
|
||||
const producerId =
|
||||
normalizeOptionalString(params.rootFrontmatter.id) ??
|
||||
normalizeOptionalString(params.rootFrontmatter.okf_id);
|
||||
if (producerId) {
|
||||
return slugifyWikiSegment(producerId);
|
||||
}
|
||||
const label =
|
||||
normalizeOptionalString(params.rootFrontmatter.name) ??
|
||||
normalizeOptionalString(params.rootFrontmatter.title) ??
|
||||
params.bundleName;
|
||||
const hash = createHash("sha1").update(params.bundlePath).digest("hex").slice(0, OKF_HASH_CHARS);
|
||||
return `${slugifyWikiSegment(label)}-${hash}`;
|
||||
}
|
||||
|
||||
function createOkfPageStem(bundleKey: string, conceptId: string): string {
|
||||
const slug = slugifyWikiSegment(conceptId.replace(/\//g, "-"));
|
||||
const hash = createHash("sha1").update(conceptId).digest("hex").slice(0, OKF_HASH_CHARS);
|
||||
return `okf-${bundleKey}-${slug}-${hash}`;
|
||||
}
|
||||
|
||||
function createOkfPageIdentity(
|
||||
bundleKey: string,
|
||||
conceptId: string,
|
||||
): { pageId: string; pagePath: string } {
|
||||
const fileName = createWikiPageFilename(createOkfPageStem(bundleKey, conceptId));
|
||||
const stem = trimMarkdownExtension(fileName);
|
||||
return {
|
||||
pageId: `concept.${stem}`,
|
||||
pagePath: `concepts/${fileName}`,
|
||||
};
|
||||
}
|
||||
|
||||
async function collectOkfMarkdownFiles(
|
||||
rootDir: string,
|
||||
warnings: ImportMemoryWikiOkfWarning[],
|
||||
): Promise<string[]> {
|
||||
async function walk(relativeDir: string): Promise<string[]> {
|
||||
const absoluteDir = path.join(rootDir, relativeDir);
|
||||
const entries = await fs.readdir(absoluteDir, { withFileTypes: true }).catch((err: unknown) => {
|
||||
warnings.push({
|
||||
code: "unreadable-entry",
|
||||
path: toPosixPath(relativeDir) || ".",
|
||||
message: err instanceof Error ? err.message : "Unable to read OKF directory.",
|
||||
});
|
||||
return [];
|
||||
});
|
||||
const files: string[] = [];
|
||||
for (const entry of entries.toSorted((left, right) => left.name.localeCompare(right.name))) {
|
||||
if (entry.name === ".git" || entry.name === "node_modules") {
|
||||
continue;
|
||||
}
|
||||
const relativePath = path.join(relativeDir, entry.name);
|
||||
if (entry.isDirectory()) {
|
||||
files.push(...(await walk(relativePath)));
|
||||
continue;
|
||||
}
|
||||
if (entry.isFile() && entry.name.endsWith(".md")) {
|
||||
files.push(relativePath);
|
||||
}
|
||||
}
|
||||
return files;
|
||||
}
|
||||
|
||||
return (await walk("")).map(toPosixPath).toSorted((left, right) => left.localeCompare(right));
|
||||
}
|
||||
|
||||
function parseOkfMarkdown(
|
||||
content: string,
|
||||
relativePath: string,
|
||||
): {
|
||||
frontmatter: Record<string, unknown>;
|
||||
body: string;
|
||||
warning?: ImportMemoryWikiOkfWarning;
|
||||
} {
|
||||
const normalizedContent = content.replace(/\r\n/g, "\n");
|
||||
try {
|
||||
return parseWikiMarkdown(normalizedContent);
|
||||
} catch (err) {
|
||||
return {
|
||||
frontmatter: {},
|
||||
body: normalizedContent,
|
||||
warning: {
|
||||
code: "invalid-concept",
|
||||
path: relativePath,
|
||||
message: err instanceof Error ? err.message : "Unable to parse OKF frontmatter.",
|
||||
},
|
||||
};
|
||||
}
|
||||
}
|
||||
|
||||
async function readOkfTextFile(params: {
|
||||
bundlePath: string;
|
||||
relativePath: string;
|
||||
warnings: ImportMemoryWikiOkfWarning[];
|
||||
}): Promise<string | null> {
|
||||
const root = await fsRoot(params.bundlePath);
|
||||
const stat = await root.stat(params.relativePath).catch((err: unknown) => {
|
||||
params.warnings.push({
|
||||
code: "unreadable-entry",
|
||||
path: params.relativePath,
|
||||
message: err instanceof Error ? err.message : "Unable to read OKF concept.",
|
||||
});
|
||||
return null;
|
||||
});
|
||||
if (!stat) {
|
||||
return null;
|
||||
}
|
||||
if (!isRegularFileStat(stat)) {
|
||||
params.warnings.push({
|
||||
code: "unreadable-entry",
|
||||
path: params.relativePath,
|
||||
message: "Refusing to import OKF concept through non-regular or hardlinked file.",
|
||||
});
|
||||
return null;
|
||||
}
|
||||
return await root.readText(params.relativePath).catch((err: unknown) => {
|
||||
params.warnings.push({
|
||||
code: "unreadable-entry",
|
||||
path: params.relativePath,
|
||||
message: err instanceof Error ? err.message : "Unable to read OKF concept.",
|
||||
});
|
||||
return null;
|
||||
});
|
||||
}
|
||||
|
||||
function deriveOkfTitle(relativePath: string, frontmatter: Record<string, unknown>): string {
|
||||
return (
|
||||
normalizeOptionalString(frontmatter.title) ??
|
||||
path.posix.basename(relativePath, ".md").replace(/[-_]+/g, " ").trim() ??
|
||||
trimMarkdownExtension(relativePath)
|
||||
);
|
||||
}
|
||||
|
||||
function normalizeOkfConcept(params: {
|
||||
bundlePath: string;
|
||||
relativePath: string;
|
||||
content: string;
|
||||
}): { concept?: OkfConceptDocument; warning?: ImportMemoryWikiOkfWarning } {
|
||||
const parsed = parseOkfMarkdown(params.content, params.relativePath);
|
||||
if (parsed.warning) {
|
||||
return { warning: parsed.warning };
|
||||
}
|
||||
|
||||
const type = normalizeOptionalString(parsed.frontmatter.type);
|
||||
if (!type) {
|
||||
return {
|
||||
warning: {
|
||||
code: "missing-type",
|
||||
path: params.relativePath,
|
||||
message: "OKF concept is missing required non-empty type frontmatter.",
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
const conceptId = trimMarkdownExtension(params.relativePath);
|
||||
const timestamp = normalizeOptionalString(parsed.frontmatter.timestamp);
|
||||
return {
|
||||
concept: {
|
||||
conceptId,
|
||||
relativePath: params.relativePath,
|
||||
absolutePath: path.join(params.bundlePath, params.relativePath),
|
||||
frontmatter: parsed.frontmatter,
|
||||
body: parsed.body,
|
||||
type,
|
||||
title: deriveOkfTitle(params.relativePath, parsed.frontmatter),
|
||||
...(normalizeOptionalString(parsed.frontmatter.description)
|
||||
? { description: normalizeOptionalString(parsed.frontmatter.description) }
|
||||
: {}),
|
||||
...(normalizeOptionalString(parsed.frontmatter.resource)
|
||||
? { resource: normalizeOptionalString(parsed.frontmatter.resource) }
|
||||
: {}),
|
||||
tags: normalizeSingleOrTrimmedStringList(parsed.frontmatter.tags),
|
||||
...(timestamp ? { timestamp } : {}),
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
function splitMarkdownLinkDestination(target: string): {
|
||||
destination: string;
|
||||
titleSuffix: string;
|
||||
} {
|
||||
const trimmed = target.trim();
|
||||
if (trimmed.startsWith("<")) {
|
||||
const end = trimmed.indexOf(">");
|
||||
if (end > 0) {
|
||||
return {
|
||||
destination: trimmed.slice(1, end),
|
||||
titleSuffix: trimmed.slice(end + 1),
|
||||
};
|
||||
}
|
||||
}
|
||||
const match = trimmed.match(/^(\S+)(\s+[\s\S]+)?$/);
|
||||
return {
|
||||
destination: match?.[1] ?? trimmed,
|
||||
titleSuffix: match?.[2] ?? "",
|
||||
};
|
||||
}
|
||||
|
||||
function resolveOkfMarkdownTarget(sourceRelativePath: string, target: string): string | null {
|
||||
const { destination } = splitMarkdownLinkDestination(target);
|
||||
const trimmed = destination.trim();
|
||||
if (!trimmed || trimmed.startsWith("#") || /^[a-z][a-z0-9+.-]*:/i.test(trimmed)) {
|
||||
return null;
|
||||
}
|
||||
|
||||
const rawTargetWithoutSuffix = trimmed.split("#")[0]?.split("?")[0]?.replace(/\\/g, "/").trim();
|
||||
const targetWithoutSuffix = safeDecodeOkfLinkPath(rawTargetWithoutSuffix);
|
||||
if (!targetWithoutSuffix || !targetWithoutSuffix.endsWith(".md")) {
|
||||
return null;
|
||||
}
|
||||
|
||||
const normalized = targetWithoutSuffix.startsWith("/")
|
||||
? path.posix.normalize(targetWithoutSuffix.slice(1))
|
||||
: path.posix.normalize(
|
||||
path.posix.join(path.posix.dirname(sourceRelativePath), targetWithoutSuffix),
|
||||
);
|
||||
const conceptId = trimMarkdownExtension(normalized);
|
||||
return conceptId.startsWith("../") ? null : conceptId;
|
||||
}
|
||||
|
||||
function safeDecodeOkfLinkPath(value: string | undefined): string {
|
||||
if (!value) {
|
||||
return "";
|
||||
}
|
||||
try {
|
||||
return decodeURIComponent(value);
|
||||
} catch {
|
||||
return value;
|
||||
}
|
||||
}
|
||||
|
||||
function getMarkdownDestinationSuffix(destination: string): string {
|
||||
const queryIndex = destination.indexOf("?");
|
||||
const fragmentIndex = destination.indexOf("#");
|
||||
const suffixIndex = queryIndex === -1
|
||||
? fragmentIndex
|
||||
: fragmentIndex === -1
|
||||
? queryIndex
|
||||
: Math.min(queryIndex, fragmentIndex);
|
||||
return suffixIndex === -1 ? "" : destination.slice(suffixIndex);
|
||||
}
|
||||
|
||||
function rewriteOkfMarkdownLinks(params: {
|
||||
body: string;
|
||||
sourcePagePath: string;
|
||||
sourceRelativePath: string;
|
||||
pageByConceptId: Map<string, { pageId: string; pagePath: string; title: string }>;
|
||||
}): { body: string; linkedConceptIds: string[] } {
|
||||
const linkedConceptIds: string[] = [];
|
||||
const rewriteLinks = (markdown: string) =>
|
||||
markdown.replace(
|
||||
OKF_MARKDOWN_LINK_PATTERN,
|
||||
(match, imagePrefix: string, label: string, rawTarget: string) => {
|
||||
const conceptId = resolveOkfMarkdownTarget(params.sourceRelativePath, rawTarget);
|
||||
if (!conceptId) {
|
||||
return match;
|
||||
}
|
||||
const target = params.pageByConceptId.get(conceptId);
|
||||
if (!target) {
|
||||
return match;
|
||||
}
|
||||
linkedConceptIds.push(conceptId);
|
||||
const { destination, titleSuffix } = splitMarkdownLinkDestination(rawTarget);
|
||||
const relativeTarget = path.posix.relative(
|
||||
path.posix.dirname(params.sourcePagePath),
|
||||
target.pagePath,
|
||||
);
|
||||
const suffix = getMarkdownDestinationSuffix(destination);
|
||||
return `${imagePrefix}[${label}](${relativeTarget}${suffix}${titleSuffix})`;
|
||||
},
|
||||
);
|
||||
const body = rewriteMarkdownOutsideCode(params.body, rewriteLinks);
|
||||
return { body, linkedConceptIds: uniqueStrings(linkedConceptIds) };
|
||||
}
|
||||
|
||||
function rewriteMarkdownLineOutsideInlineCode(
|
||||
line: string,
|
||||
rewriteLinks: (markdown: string) => string,
|
||||
): string {
|
||||
let result = "";
|
||||
let cursor = 0;
|
||||
while (cursor < line.length) {
|
||||
const codeStart = line.indexOf("`", cursor);
|
||||
if (codeStart === -1) {
|
||||
result += rewriteLinks(line.slice(cursor));
|
||||
break;
|
||||
}
|
||||
result += rewriteLinks(line.slice(cursor, codeStart));
|
||||
const delimiter = line.slice(codeStart).match(/^`+/)?.[0] ?? "`";
|
||||
const codeEnd = line.indexOf(delimiter, codeStart + delimiter.length);
|
||||
if (codeEnd === -1) {
|
||||
result += line.slice(codeStart);
|
||||
break;
|
||||
}
|
||||
result += line.slice(codeStart, codeEnd + delimiter.length);
|
||||
cursor = codeEnd + delimiter.length;
|
||||
}
|
||||
return result;
|
||||
}
|
||||
|
||||
function rewriteMarkdownOutsideCode(
|
||||
markdown: string,
|
||||
rewriteLinks: (markdown: string) => string,
|
||||
): string {
|
||||
const lines = markdown.split(/(\n)/);
|
||||
let inFence = false;
|
||||
let fenceDelimiter = "";
|
||||
return lines
|
||||
.map((line) => {
|
||||
if (line === "\n") {
|
||||
return line;
|
||||
}
|
||||
const fenceMatch = line.match(OKF_FENCE_PATTERN);
|
||||
if (fenceMatch) {
|
||||
const delimiter = fenceMatch[1] ?? "";
|
||||
const closesFence =
|
||||
inFence &&
|
||||
delimiter.startsWith(fenceDelimiter[0] ?? "") &&
|
||||
delimiter.length >= fenceDelimiter.length;
|
||||
const opensFence = !inFence;
|
||||
if (opensFence) {
|
||||
inFence = true;
|
||||
fenceDelimiter = delimiter;
|
||||
} else if (closesFence) {
|
||||
inFence = false;
|
||||
fenceDelimiter = "";
|
||||
}
|
||||
return line;
|
||||
}
|
||||
return inFence ? line : rewriteMarkdownLineOutsideInlineCode(line, rewriteLinks);
|
||||
})
|
||||
.join("");
|
||||
}
|
||||
|
||||
function normalizeOkfRenderedPageForComparison(content: string): string {
|
||||
const withoutRelated = content.replace(OKF_RELATED_SECTION_PATTERN, "\n");
|
||||
const frontmatterMatch = withoutRelated.match(/^---\n([\s\S]*?)\n---\n?/);
|
||||
if (!frontmatterMatch) {
|
||||
return withoutRelated.trimEnd();
|
||||
}
|
||||
const normalizedFrontmatter =
|
||||
frontmatterMatch[1]?.replace(OKF_VOLATILE_TIMESTAMP_LINE_PATTERN, "") ?? "";
|
||||
const frontmatterBody = normalizedFrontmatter.endsWith("\n")
|
||||
? normalizedFrontmatter
|
||||
: `${normalizedFrontmatter}\n`;
|
||||
return `---\n${frontmatterBody}---\n${withoutRelated.slice(frontmatterMatch[0].length)}`.trimEnd();
|
||||
}
|
||||
|
||||
async function writeOkfConceptPage(params: {
|
||||
vaultRoot: string;
|
||||
pagePath: string;
|
||||
content: string;
|
||||
}): Promise<{ changed: boolean; created: boolean }> {
|
||||
const vault = await fsRoot(params.vaultRoot);
|
||||
const pageStat = await vault.stat(params.pagePath).catch((error: unknown) => {
|
||||
if (
|
||||
error instanceof FsSafeError &&
|
||||
(error.code === "not-found" || error.code === "path-alias")
|
||||
) {
|
||||
return null;
|
||||
}
|
||||
throw error;
|
||||
});
|
||||
const existing = pageStat ? await vault.readText(params.pagePath).catch(() => "") : "";
|
||||
if (
|
||||
existing === params.content ||
|
||||
normalizeOkfRenderedPageForComparison(existing) ===
|
||||
normalizeOkfRenderedPageForComparison(params.content)
|
||||
) {
|
||||
return { changed: false, created: !pageStat };
|
||||
}
|
||||
try {
|
||||
if (isRegularFileStat(pageStat) && pageStat.nlink > 1) {
|
||||
await vault.remove(params.pagePath);
|
||||
}
|
||||
await vault.write(params.pagePath, params.content);
|
||||
} catch (error) {
|
||||
if (error instanceof FsSafeError) {
|
||||
if (error.code !== "symlink" && error.code !== "path-alias") {
|
||||
throw new Error(
|
||||
`Refusing to write OKF concept page (${error.code}): ${params.pagePath}: ${error.message}`,
|
||||
{ cause: error },
|
||||
);
|
||||
}
|
||||
throw new Error(`Refusing to write OKF concept page through symlink: ${params.pagePath}`, {
|
||||
cause: error,
|
||||
});
|
||||
}
|
||||
throw error;
|
||||
}
|
||||
return { changed: true, created: !pageStat };
|
||||
}
|
||||
|
||||
async function removeStaleOkfConceptPages(params: {
|
||||
vaultRoot: string;
|
||||
bundleKey: string;
|
||||
currentPagePaths: Set<string>;
|
||||
}): Promise<string[]> {
|
||||
const vault = await fsRoot(params.vaultRoot);
|
||||
const conceptsDir = path.join(params.vaultRoot, "concepts");
|
||||
const entries = await fs.readdir(conceptsDir, { withFileTypes: true }).catch(() => []);
|
||||
const removedPagePaths: string[] = [];
|
||||
for (const entry of entries) {
|
||||
if (!entry.isFile() || !entry.name.endsWith(".md") || entry.name === "index.md") {
|
||||
continue;
|
||||
}
|
||||
const pagePath = `concepts/${entry.name}`;
|
||||
if (params.currentPagePaths.has(pagePath)) {
|
||||
continue;
|
||||
}
|
||||
const raw = await vault.readText(pagePath).catch(() => "");
|
||||
const parsed = parseWikiMarkdown(raw);
|
||||
const okf = parsed.frontmatter.okf;
|
||||
if (
|
||||
okf &&
|
||||
typeof okf === "object" &&
|
||||
!Array.isArray(okf) &&
|
||||
(okf as Record<string, unknown>).bundleKey === params.bundleKey
|
||||
) {
|
||||
await vault.remove(pagePath);
|
||||
removedPagePaths.push(pagePath);
|
||||
}
|
||||
}
|
||||
return removedPagePaths;
|
||||
}
|
||||
|
||||
function readRootOkfMetadata(params: {
|
||||
rootIndex: string | undefined;
|
||||
bundleName: string;
|
||||
bundlePath: string;
|
||||
}): OkfBundleMetadata {
|
||||
if (!params.rootIndex) {
|
||||
return {
|
||||
key: createOkfBundleKey({
|
||||
rootFrontmatter: {},
|
||||
bundleName: params.bundleName,
|
||||
bundlePath: params.bundlePath,
|
||||
}),
|
||||
};
|
||||
}
|
||||
const parsed = parseOkfMarkdown(params.rootIndex, "index.md");
|
||||
return {
|
||||
key: createOkfBundleKey({
|
||||
rootFrontmatter: parsed.frontmatter,
|
||||
bundleName: params.bundleName,
|
||||
bundlePath: params.bundlePath,
|
||||
}),
|
||||
...(normalizeOptionalString(parsed.frontmatter.okf_version)
|
||||
? { version: normalizeOptionalString(parsed.frontmatter.okf_version) }
|
||||
: {}),
|
||||
};
|
||||
}
|
||||
|
||||
function formatOkfImportSummary(result: ImportMemoryWikiOkfResult): string {
|
||||
return `Imported ${result.importedCount} OKF concept${result.importedCount === 1 ? "" : "s"} from ${result.bundlePath} into memory wiki. Updated ${result.updatedCount}; removed ${result.removedCount}; skipped ${result.skippedCount}; refreshed ${result.indexUpdatedFiles.length} index file${result.indexUpdatedFiles.length === 1 ? "" : "s"}.`;
|
||||
}
|
||||
|
||||
export { formatOkfImportSummary };
|
||||
|
||||
export async function importMemoryWikiOkfBundle(params: {
|
||||
config: ResolvedMemoryWikiConfig;
|
||||
bundlePath: string;
|
||||
nowMs?: number;
|
||||
}): Promise<ImportMemoryWikiOkfResult> {
|
||||
await initializeMemoryWikiVault(params.config, { nowMs: params.nowMs });
|
||||
const bundlePath = path.resolve(params.bundlePath);
|
||||
const stat = await fs.stat(bundlePath);
|
||||
if (!stat.isDirectory()) {
|
||||
throw new Error("wiki okf import expects an unpacked OKF bundle directory.");
|
||||
}
|
||||
|
||||
const warnings: ImportMemoryWikiOkfWarning[] = [];
|
||||
const markdownFiles = await collectOkfMarkdownFiles(bundlePath, warnings);
|
||||
const concepts: OkfConceptDocument[] = [];
|
||||
let rootIndexContent: string | undefined;
|
||||
|
||||
for (const relativePath of markdownFiles) {
|
||||
if (relativePath === "index.md") {
|
||||
rootIndexContent =
|
||||
(await readOkfTextFile({ bundlePath, relativePath, warnings })) ?? undefined;
|
||||
}
|
||||
if (OKF_RESERVED_FILENAMES.has(path.posix.basename(relativePath))) {
|
||||
continue;
|
||||
}
|
||||
const content = await readOkfTextFile({ bundlePath, relativePath, warnings });
|
||||
if (content === null) {
|
||||
continue;
|
||||
}
|
||||
const normalized = normalizeOkfConcept({ bundlePath, relativePath, content });
|
||||
if (normalized.warning) {
|
||||
warnings.push(normalized.warning);
|
||||
continue;
|
||||
}
|
||||
if (normalized.concept) {
|
||||
concepts.push(normalized.concept);
|
||||
}
|
||||
}
|
||||
|
||||
const timestamp = resolveMemoryWikiTimestamp(params.nowMs);
|
||||
const bundleName = path.basename(bundlePath);
|
||||
const bundleMetadata = readRootOkfMetadata({
|
||||
rootIndex: rootIndexContent,
|
||||
bundleName,
|
||||
bundlePath,
|
||||
});
|
||||
const bundleKey = bundleMetadata.key;
|
||||
const pageByConceptId = new Map<string, { pageId: string; pagePath: string; title: string }>();
|
||||
for (const concept of concepts) {
|
||||
pageByConceptId.set(concept.conceptId, {
|
||||
...createOkfPageIdentity(bundleKey, concept.conceptId),
|
||||
title: concept.title,
|
||||
});
|
||||
}
|
||||
|
||||
const importedPages: OkfImportedPage[] = [];
|
||||
let updatedCount = 0;
|
||||
|
||||
await fs.mkdir(path.join(params.config.vault.path, "concepts"), { recursive: true });
|
||||
for (const concept of concepts.toSorted((left, right) =>
|
||||
left.conceptId.localeCompare(right.conceptId),
|
||||
)) {
|
||||
const page = pageByConceptId.get(concept.conceptId);
|
||||
if (!page) {
|
||||
continue;
|
||||
}
|
||||
const rewritten = rewriteOkfMarkdownLinks({
|
||||
body: concept.body,
|
||||
sourcePagePath: page.pagePath,
|
||||
sourceRelativePath: concept.relativePath,
|
||||
pageByConceptId,
|
||||
});
|
||||
const relationships = rewritten.linkedConceptIds.flatMap((conceptId) => {
|
||||
const target = pageByConceptId.get(conceptId);
|
||||
return target
|
||||
? [
|
||||
{
|
||||
targetId: target.pageId,
|
||||
targetPath: target.pagePath,
|
||||
targetTitle: target.title,
|
||||
kind: "okf-link",
|
||||
evidenceKind: "okf-markdown-link",
|
||||
},
|
||||
]
|
||||
: [];
|
||||
});
|
||||
|
||||
const frontmatter = {
|
||||
pageType: "concept",
|
||||
id: page.pageId,
|
||||
title: concept.title,
|
||||
sourceType: "okf",
|
||||
provenanceMode: "okf-import",
|
||||
sourcePath: concept.absolutePath,
|
||||
okfConceptId: concept.conceptId,
|
||||
okfType: concept.type,
|
||||
sourceIds: [`source.okf.${bundleKey}`],
|
||||
importedAt: timestamp,
|
||||
updatedAt: concept.timestamp ?? timestamp,
|
||||
status: "active",
|
||||
...(concept.description ? { description: concept.description } : {}),
|
||||
...(concept.resource ? { resource: concept.resource } : {}),
|
||||
...(concept.tags.length > 0 ? { tags: concept.tags } : {}),
|
||||
...(concept.timestamp ? { okfTimestamp: concept.timestamp } : {}),
|
||||
...(relationships.length > 0 ? { relationships } : {}),
|
||||
okf: {
|
||||
...(bundleMetadata.version ? { version: bundleMetadata.version } : {}),
|
||||
bundleName,
|
||||
bundleKey,
|
||||
conceptId: concept.conceptId,
|
||||
sourceRelativePath: concept.relativePath,
|
||||
frontmatter: concept.frontmatter,
|
||||
},
|
||||
};
|
||||
|
||||
const writeResult = await writeOkfConceptPage({
|
||||
vaultRoot: params.config.vault.path,
|
||||
pagePath: page.pagePath,
|
||||
content: renderWikiMarkdown({
|
||||
frontmatter,
|
||||
body: rewritten.body,
|
||||
}),
|
||||
});
|
||||
if (!writeResult.created && writeResult.changed) {
|
||||
updatedCount++;
|
||||
}
|
||||
importedPages.push({
|
||||
conceptId: concept.conceptId,
|
||||
sourcePath: concept.absolutePath,
|
||||
pageId: page.pageId,
|
||||
pagePath: page.pagePath,
|
||||
title: concept.title,
|
||||
created: writeResult.created,
|
||||
});
|
||||
}
|
||||
const currentPagePaths = new Set(importedPages.map((page) => page.pagePath));
|
||||
const removedPagePaths =
|
||||
warnings.length === 0
|
||||
? await removeStaleOkfConceptPages({
|
||||
vaultRoot: params.config.vault.path,
|
||||
bundleKey,
|
||||
currentPagePaths,
|
||||
})
|
||||
: [];
|
||||
|
||||
await appendMemoryWikiLog(params.config.vault.path, {
|
||||
type: "okf-import",
|
||||
timestamp,
|
||||
details: {
|
||||
bundlePath,
|
||||
bundleName,
|
||||
importedCount: importedPages.length,
|
||||
updatedCount,
|
||||
removedCount: removedPagePaths.length,
|
||||
skippedCount: warnings.length,
|
||||
pagePaths: importedPages.map((page) => page.pagePath),
|
||||
removedPagePaths,
|
||||
},
|
||||
});
|
||||
|
||||
const compile = await compileMemoryWikiVault(params.config);
|
||||
return {
|
||||
bundlePath,
|
||||
bundleName,
|
||||
...(bundleMetadata.version ? { okfVersion: bundleMetadata.version } : {}),
|
||||
importedCount: importedPages.length,
|
||||
updatedCount,
|
||||
removedCount: removedPagePaths.length,
|
||||
skippedCount: warnings.length,
|
||||
pagePaths: importedPages.map((page) => page.pagePath),
|
||||
removedPagePaths,
|
||||
warnings,
|
||||
indexUpdatedFiles: compile.updatedFiles,
|
||||
};
|
||||
}
|
||||
@@ -44,7 +44,7 @@ describe("moonshot provider plugin", () => {
|
||||
});
|
||||
});
|
||||
|
||||
it("owns replay policy for OpenAI-compatible Moonshot transports without mangling native Kimi tool_call IDs", async () => {
|
||||
it("rewrites duplicate tool-call ids with OpenAI-style ids for Moonshot replay", async () => {
|
||||
const provider = await registerSingleProviderPlugin(plugin);
|
||||
|
||||
const policy = provider.buildReplayPolicy?.({
|
||||
@@ -57,10 +57,28 @@ describe("moonshot provider plugin", () => {
|
||||
applyAssistantFirstOrderingFix: true,
|
||||
validateGeminiTurns: true,
|
||||
validateAnthropicTurns: true,
|
||||
sanitizeToolCallIds: true,
|
||||
toolCallIdMode: "strict",
|
||||
duplicateToolCallIdStyle: "openai",
|
||||
});
|
||||
expect(policy).not.toHaveProperty("dropReasoningFromHistory");
|
||||
expect(policy).not.toHaveProperty("sanitizeToolCallIds");
|
||||
expect(policy).not.toHaveProperty("toolCallIdMode");
|
||||
});
|
||||
|
||||
it("preserves responses-family replay behavior", async () => {
|
||||
const provider = await registerSingleProviderPlugin(plugin);
|
||||
|
||||
const policy = provider.buildReplayPolicy?.({
|
||||
provider: "moonshot",
|
||||
modelApi: "openai-responses",
|
||||
modelId: "kimi-k2.6",
|
||||
} as never);
|
||||
|
||||
expect(policy).toEqual({
|
||||
applyAssistantFirstOrderingFix: false,
|
||||
validateGeminiTurns: false,
|
||||
validateAnthropicTurns: false,
|
||||
allowSyntheticToolResults: true,
|
||||
});
|
||||
});
|
||||
|
||||
it("wires moonshot-thinking stream hooks", async () => {
|
||||
@@ -89,4 +107,60 @@ describe("moonshot provider plugin", () => {
|
||||
thinking: { type: "disabled" },
|
||||
});
|
||||
});
|
||||
|
||||
it("keeps Kimi K2.7 Code thinking always on without sending a thinking field", async () => {
|
||||
const provider = await registerSingleProviderPlugin(plugin);
|
||||
const capturedStream = createCapturedThinkingConfigStream();
|
||||
|
||||
const wrapped = provider.wrapSimpleCompletionStreamFn?.({
|
||||
provider: "moonshot",
|
||||
modelId: "kimi-k2.7-code",
|
||||
thinkingLevel: "off",
|
||||
streamFn: capturedStream.streamFn,
|
||||
} as never);
|
||||
|
||||
void wrapped?.(
|
||||
{
|
||||
api: "openai-completions",
|
||||
provider: "moonshot",
|
||||
id: "kimi-k2.7-code",
|
||||
} as Model<"openai-completions">,
|
||||
{ messages: [] } as Context,
|
||||
{},
|
||||
);
|
||||
|
||||
expect(capturedStream.getCapturedPayload()).toEqual({
|
||||
config: { thinkingConfig: { thinkingBudget: -1 } },
|
||||
});
|
||||
expect(
|
||||
provider.wrapSimpleCompletionStreamFn?.({
|
||||
provider: "moonshot",
|
||||
modelId: "kimi-k2.6",
|
||||
streamFn: capturedStream.streamFn,
|
||||
} as never),
|
||||
).toBe(capturedStream.streamFn);
|
||||
expect(
|
||||
provider.resolveThinkingProfile?.({
|
||||
provider: "moonshot",
|
||||
modelId: "kimi-k2.7-code",
|
||||
reasoning: true,
|
||||
} as never),
|
||||
).toEqual({
|
||||
levels: [{ id: "low", label: "on" }],
|
||||
defaultLevel: "low",
|
||||
preserveWhenCatalogReasoningFalse: true,
|
||||
});
|
||||
expect(
|
||||
provider.isModernModelRef?.({
|
||||
provider: "moonshot",
|
||||
modelId: "kimi-k2.7-code",
|
||||
}),
|
||||
).toBe(true);
|
||||
expect(
|
||||
provider.isModernModelRef?.({
|
||||
provider: "moonshot",
|
||||
modelId: "kimi-k2.6",
|
||||
}),
|
||||
).toBe(false);
|
||||
});
|
||||
});
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
// Moonshot plugin entrypoint registers its OpenClaw integration.
|
||||
import { defineSingleProviderPluginEntry } from "openclaw/plugin-sdk/provider-entry";
|
||||
import { buildProviderReplayFamilyHooks } from "openclaw/plugin-sdk/provider-model-shared";
|
||||
import { buildOpenAICompatibleReplayPolicy } from "openclaw/plugin-sdk/provider-model-shared";
|
||||
import { MOONSHOT_THINKING_STREAM_HOOKS } from "openclaw/plugin-sdk/provider-stream-family";
|
||||
import { applyMoonshotNativeStreamingUsageCompat } from "./api.js";
|
||||
import { moonshotMediaUnderstandingProvider } from "./media-understanding-provider.js";
|
||||
@@ -10,9 +10,11 @@ import {
|
||||
MOONSHOT_DEFAULT_MODEL_REF,
|
||||
} from "./onboard.js";
|
||||
import { buildMoonshotProvider } from "./provider-catalog.js";
|
||||
import { KIMI_K2_7_CODE_MODEL_ID, resolveThinkingProfile } from "./provider-policy-api.js";
|
||||
import { createKimiWebSearchProvider } from "./src/kimi-web-search-provider.js";
|
||||
|
||||
const PROVIDER_ID = "moonshot";
|
||||
const moonshotThinkingStreamHooks = MOONSHOT_THINKING_STREAM_HOOKS;
|
||||
|
||||
export default defineSingleProviderPluginEntry({
|
||||
id: PROVIDER_ID,
|
||||
@@ -59,22 +61,20 @@ export default defineSingleProviderPluginEntry({
|
||||
},
|
||||
applyNativeStreamingUsageCompat: ({ providerConfig }) =>
|
||||
applyMoonshotNativeStreamingUsageCompat(providerConfig),
|
||||
// Kimi K2+ returns native tool_call IDs shaped like `functions.<name>:<index>`.
|
||||
// Sanitizing them to alphanumeric-only breaks Kimi's serving-layer matching in
|
||||
// multi-turn replay. See openclaw/openclaw#62319.
|
||||
...buildProviderReplayFamilyHooks({
|
||||
family: "openai-compatible",
|
||||
sanitizeToolCallIds: false,
|
||||
dropReasoningFromHistory: false,
|
||||
}),
|
||||
...MOONSHOT_THINKING_STREAM_HOOKS,
|
||||
resolveThinkingProfile: () => ({
|
||||
levels: [
|
||||
{ id: "off", label: "off" },
|
||||
{ id: "low", label: "on" },
|
||||
],
|
||||
defaultLevel: "off",
|
||||
}),
|
||||
buildReplayPolicy: ({ modelApi, modelId }) =>
|
||||
buildOpenAICompatibleReplayPolicy(modelApi, {
|
||||
modelId,
|
||||
sanitizeToolCallIds: modelApi === "openai-completions",
|
||||
duplicateToolCallIdStyle: "openai",
|
||||
dropReasoningFromHistory: false,
|
||||
}),
|
||||
...moonshotThinkingStreamHooks,
|
||||
wrapSimpleCompletionStreamFn: (ctx) =>
|
||||
ctx.modelId.trim().toLowerCase() === KIMI_K2_7_CODE_MODEL_ID
|
||||
? moonshotThinkingStreamHooks.wrapStreamFn?.(ctx)
|
||||
: ctx.streamFn,
|
||||
resolveThinkingProfile,
|
||||
isModernModelRef: ({ modelId }) => modelId.trim().toLowerCase() === KIMI_K2_7_CODE_MODEL_ID,
|
||||
},
|
||||
register(api) {
|
||||
api.registerMediaUnderstandingProvider(moonshotMediaUnderstandingProvider);
|
||||
|
||||
@@ -1,11 +1,25 @@
|
||||
// Moonshot tests cover moonshot plugin behavior.
|
||||
import {
|
||||
streamSimple,
|
||||
type AssistantMessage,
|
||||
type Context,
|
||||
type Model,
|
||||
type Tool,
|
||||
} from "openclaw/plugin-sdk/llm";
|
||||
import { registerSingleProviderPlugin } from "openclaw/plugin-sdk/plugin-test-runtime";
|
||||
import { isLiveTestEnabled } from "openclaw/plugin-sdk/test-env";
|
||||
import { Type } from "typebox";
|
||||
import { describe, expect, it } from "vitest";
|
||||
import plugin from "./index.js";
|
||||
import { buildMoonshotProvider, MOONSHOT_CN_BASE_URL } from "./provider-catalog.js";
|
||||
import { createKimiWebSearchProvider } from "./src/kimi-web-search-provider.js";
|
||||
|
||||
const KIMI_SEARCH_KEY =
|
||||
process.env.KIMI_API_KEY?.trim() || process.env.MOONSHOT_API_KEY?.trim() || "";
|
||||
const MOONSHOT_API_KEY = process.env.MOONSHOT_API_KEY?.trim() || "";
|
||||
const describeLive = isLiveTestEnabled() && KIMI_SEARCH_KEY.length > 0 ? describe : describe.skip;
|
||||
const describeModelLive =
|
||||
isLiveTestEnabled() && MOONSHOT_API_KEY.length > 0 ? describe : describe.skip;
|
||||
const KIMI_LIVE_SEARCH_TIMEOUT_SECONDS = 60;
|
||||
|
||||
function isTransientKimiSearchError(error: unknown): boolean {
|
||||
@@ -19,17 +33,31 @@ function isTransientKimiSearchError(error: unknown): boolean {
|
||||
return message.includes("timeout") || message.includes("aborted");
|
||||
}
|
||||
|
||||
function isKimiAuthDrift(error: unknown): boolean {
|
||||
function isMoonshotAuthDrift(error: unknown): boolean {
|
||||
if (!(error instanceof Error)) {
|
||||
return false;
|
||||
}
|
||||
const message = error.message.toLowerCase();
|
||||
return (
|
||||
message.includes("kimi api error (401)") &&
|
||||
(message.includes("incorrect api key") || message.includes("incorrect_api_key"))
|
||||
message.includes("401") &&
|
||||
(message.includes("incorrect api key") ||
|
||||
message.includes("incorrect_api_key") ||
|
||||
message.includes("invalid authentication") ||
|
||||
message.includes("invalid_authentication_error"))
|
||||
);
|
||||
}
|
||||
|
||||
describe("moonshot live auth drift detection", () => {
|
||||
it.each([
|
||||
["401 Incorrect API key provided", true],
|
||||
["401 invalid_authentication_error: Invalid Authentication", true],
|
||||
["401 Permission denied", false],
|
||||
["400 Incorrect API key provided", false],
|
||||
])("classifies %s", (message, expected) => {
|
||||
expect(isMoonshotAuthDrift(new Error(message))).toBe(expected);
|
||||
});
|
||||
});
|
||||
|
||||
describeLive("moonshot plugin live", () => {
|
||||
it("runs Kimi web search through the provider tool", async () => {
|
||||
const provider = createKimiWebSearchProvider();
|
||||
@@ -51,7 +79,7 @@ describeLive("moonshot plugin live", () => {
|
||||
break;
|
||||
} catch (error) {
|
||||
lastError = error;
|
||||
if (isKimiAuthDrift(error)) {
|
||||
if (isMoonshotAuthDrift(error)) {
|
||||
console.warn("[moonshot:live] skip Kimi web search: auth drift");
|
||||
return;
|
||||
}
|
||||
@@ -71,6 +99,256 @@ describeLive("moonshot plugin live", () => {
|
||||
}, 180_000);
|
||||
});
|
||||
|
||||
function resolveMoonshotModels(modelId: string): Model<"openai-completions">[] {
|
||||
const provider = buildMoonshotProvider();
|
||||
const model = provider.models.find((entry) => entry.id === modelId);
|
||||
if (!model) {
|
||||
throw new Error(`Moonshot catalog does not include ${modelId}`);
|
||||
}
|
||||
const defaultModel = {
|
||||
provider: "moonshot",
|
||||
baseUrl: provider.baseUrl,
|
||||
...model,
|
||||
api: "openai-completions",
|
||||
} as Model<"openai-completions">;
|
||||
return [defaultModel, { ...defaultModel, baseUrl: MOONSHOT_CN_BASE_URL }];
|
||||
}
|
||||
|
||||
function createNoopTool(): Tool {
|
||||
return {
|
||||
name: "noop",
|
||||
description: "Return ok.",
|
||||
parameters: Type.Object({}, { additionalProperties: false }),
|
||||
};
|
||||
}
|
||||
|
||||
async function collectDoneMessage(
|
||||
stream: AsyncIterable<{ type: string; message?: AssistantMessage; error?: AssistantMessage }>,
|
||||
): Promise<AssistantMessage> {
|
||||
let doneMessage: AssistantMessage | undefined;
|
||||
for await (const event of stream) {
|
||||
if (event.type === "error") {
|
||||
throw new Error(event.error?.errorMessage || "Moonshot live request failed");
|
||||
}
|
||||
if (event.type === "done") {
|
||||
doneMessage = event.message;
|
||||
}
|
||||
}
|
||||
if (!doneMessage) {
|
||||
throw new Error("Moonshot live stream ended without a done message");
|
||||
}
|
||||
return doneMessage;
|
||||
}
|
||||
|
||||
describeModelLive("moonshot K2.6 replay live", () => {
|
||||
it("accepts a cross-model tool-call replay after backfilling reasoning_content", async () => {
|
||||
const provider = await registerSingleProviderPlugin(plugin);
|
||||
const wrappedStream = provider.wrapStreamFn?.({
|
||||
provider: "moonshot",
|
||||
modelId: "kimi-k2.6",
|
||||
thinkingLevel: "low",
|
||||
streamFn: streamSimple,
|
||||
} as never);
|
||||
if (!wrappedStream) {
|
||||
throw new Error("Moonshot provider did not register a stream wrapper");
|
||||
}
|
||||
|
||||
const tool = createNoopTool();
|
||||
const replayContext: Context = {
|
||||
messages: [
|
||||
{
|
||||
role: "user",
|
||||
content: "Call the noop tool.",
|
||||
timestamp: Date.now(),
|
||||
},
|
||||
{
|
||||
role: "assistant",
|
||||
api: "openai-responses",
|
||||
provider: "openai",
|
||||
model: "gpt-5.5",
|
||||
stopReason: "toolUse",
|
||||
content: [{ type: "toolCall", id: "call_cross_model", name: "noop", arguments: {} }],
|
||||
timestamp: Date.now(),
|
||||
} as AssistantMessage,
|
||||
{
|
||||
role: "toolResult",
|
||||
toolCallId: "call_cross_model",
|
||||
toolName: "noop",
|
||||
content: [{ type: "text", text: "ok" }],
|
||||
isError: false,
|
||||
timestamp: Date.now(),
|
||||
},
|
||||
{
|
||||
role: "user",
|
||||
content: "The tool returned ok. Reply with exactly: ok",
|
||||
timestamp: Date.now(),
|
||||
},
|
||||
],
|
||||
tools: [tool],
|
||||
};
|
||||
|
||||
const runScenario = async (model: Model<"openai-completions">) => {
|
||||
let payload: Record<string, unknown> | undefined;
|
||||
const response = await collectDoneMessage(
|
||||
wrappedStream(model, replayContext, {
|
||||
apiKey: MOONSHOT_API_KEY,
|
||||
maxTokens: 256,
|
||||
onPayload: (value) => {
|
||||
payload = value as Record<string, unknown>;
|
||||
},
|
||||
}) as AsyncIterable<{
|
||||
type: string;
|
||||
message?: AssistantMessage;
|
||||
error?: AssistantMessage;
|
||||
}>,
|
||||
);
|
||||
|
||||
const messages = payload?.messages as Array<Record<string, unknown>> | undefined;
|
||||
const replayedAssistant = messages?.find(
|
||||
(message) => message.role === "assistant" && Array.isArray(message.tool_calls),
|
||||
);
|
||||
expect(replayedAssistant?.reasoning_content).toBe("");
|
||||
expect(response.stopReason).not.toBe("error");
|
||||
};
|
||||
|
||||
let lastAuthError: unknown;
|
||||
for (const model of resolveMoonshotModels("kimi-k2.6")) {
|
||||
try {
|
||||
await runScenario(model);
|
||||
return;
|
||||
} catch (error) {
|
||||
if (!isMoonshotAuthDrift(error)) {
|
||||
throw error;
|
||||
}
|
||||
lastAuthError = error;
|
||||
}
|
||||
}
|
||||
throw toLintErrorObject(lastAuthError, "Moonshot K2.6 rejected the API key in both regions");
|
||||
}, 180_000);
|
||||
});
|
||||
|
||||
describeModelLive("moonshot K2.7 Code live", () => {
|
||||
it("omits thinking controls and completes a replayed tool turn", async () => {
|
||||
const provider = await registerSingleProviderPlugin(plugin);
|
||||
const wrappedStream = provider.wrapStreamFn?.({
|
||||
provider: "moonshot",
|
||||
modelId: "kimi-k2.7-code",
|
||||
thinkingLevel: "off",
|
||||
extraParams: { thinking: { type: "disabled", keep: "all" } },
|
||||
streamFn: streamSimple,
|
||||
} as never);
|
||||
if (!wrappedStream) {
|
||||
throw new Error("Moonshot provider did not register a stream wrapper");
|
||||
}
|
||||
|
||||
const tool = createNoopTool();
|
||||
const firstUser = {
|
||||
role: "user" as const,
|
||||
content: "Call the noop tool with {}. Do not answer directly.",
|
||||
timestamp: Date.now(),
|
||||
};
|
||||
|
||||
const runScenario = async (model: Model<"openai-completions">) => {
|
||||
let firstPayload: Record<string, unknown> | undefined;
|
||||
const first = await collectDoneMessage(
|
||||
wrappedStream(
|
||||
model,
|
||||
{ messages: [firstUser], tools: [tool] },
|
||||
{
|
||||
apiKey: MOONSHOT_API_KEY,
|
||||
maxTokens: 16_000,
|
||||
temperature: 0,
|
||||
onPayload: (payload) => {
|
||||
firstPayload = payload as Record<string, unknown>;
|
||||
},
|
||||
},
|
||||
) as AsyncIterable<{
|
||||
type: string;
|
||||
message?: AssistantMessage;
|
||||
error?: AssistantMessage;
|
||||
}>,
|
||||
);
|
||||
|
||||
expect(firstPayload).toBeDefined();
|
||||
expect(firstPayload).not.toHaveProperty("thinking");
|
||||
expect(firstPayload).not.toHaveProperty("reasoning_effort");
|
||||
expect(firstPayload).not.toHaveProperty("temperature");
|
||||
const reasoning = first.content.find((block) => block.type === "thinking");
|
||||
if (!reasoning || reasoning.type !== "thinking" || reasoning.thinking.length === 0) {
|
||||
throw new Error("Moonshot K2.7 Code did not return captured reasoning");
|
||||
}
|
||||
const toolCall = first.content.find((block) => block.type === "toolCall");
|
||||
if (!toolCall || toolCall.type !== "toolCall") {
|
||||
throw new Error(`Moonshot K2.7 Code did not call noop: ${first.stopReason}`);
|
||||
}
|
||||
expect(toolCall.name).toBe("noop");
|
||||
|
||||
let secondPayload: Record<string, unknown> | undefined;
|
||||
const replayContext: Context = {
|
||||
messages: [
|
||||
firstUser,
|
||||
first,
|
||||
{
|
||||
role: "toolResult",
|
||||
toolCallId: toolCall.id,
|
||||
toolName: toolCall.name,
|
||||
content: [{ type: "text", text: "ok" }],
|
||||
isError: false,
|
||||
timestamp: Date.now(),
|
||||
},
|
||||
{
|
||||
role: "user",
|
||||
content: "Reply with exactly: ok",
|
||||
timestamp: Date.now(),
|
||||
},
|
||||
],
|
||||
tools: [tool],
|
||||
};
|
||||
const second = await collectDoneMessage(
|
||||
wrappedStream(model, replayContext, {
|
||||
apiKey: MOONSHOT_API_KEY,
|
||||
maxTokens: 16_000,
|
||||
temperature: 0,
|
||||
onPayload: (payload) => {
|
||||
secondPayload = payload as Record<string, unknown>;
|
||||
},
|
||||
}) as AsyncIterable<{
|
||||
type: string;
|
||||
message?: AssistantMessage;
|
||||
error?: AssistantMessage;
|
||||
}>,
|
||||
);
|
||||
|
||||
expect(secondPayload).toBeDefined();
|
||||
expect(secondPayload).not.toHaveProperty("thinking");
|
||||
expect(secondPayload).not.toHaveProperty("reasoning_effort");
|
||||
expect(secondPayload).not.toHaveProperty("temperature");
|
||||
const text = second.content
|
||||
.filter((block) => block.type === "text")
|
||||
.map((block) => block.text.trim())
|
||||
.join(" ");
|
||||
expect(text).toMatch(/^ok[.!]?$/i);
|
||||
};
|
||||
|
||||
let lastAuthError: unknown;
|
||||
for (const model of resolveMoonshotModels("kimi-k2.7-code")) {
|
||||
try {
|
||||
await runScenario(model);
|
||||
return;
|
||||
} catch (error) {
|
||||
if (!isMoonshotAuthDrift(error)) {
|
||||
throw error;
|
||||
}
|
||||
lastAuthError = error;
|
||||
}
|
||||
}
|
||||
throw toLintErrorObject(
|
||||
lastAuthError,
|
||||
"Moonshot K2.7 Code rejected the API key in both regions",
|
||||
);
|
||||
}, 180_000);
|
||||
});
|
||||
|
||||
function toLintErrorObject(value: unknown, fallbackMessage: string): Error {
|
||||
if (value instanceof Error) {
|
||||
return value;
|
||||
|
||||
@@ -63,6 +63,20 @@
|
||||
"cacheWrite": 0
|
||||
}
|
||||
},
|
||||
{
|
||||
"id": "kimi-k2.7-code",
|
||||
"name": "Kimi K2.7 Code",
|
||||
"reasoning": true,
|
||||
"input": ["text", "image"],
|
||||
"contextWindow": 262144,
|
||||
"maxTokens": 262144,
|
||||
"cost": {
|
||||
"input": 0.95,
|
||||
"output": 4,
|
||||
"cacheRead": 0.19,
|
||||
"cacheWrite": 0
|
||||
}
|
||||
},
|
||||
{
|
||||
"id": "kimi-k2.5",
|
||||
"name": "Kimi K2.5",
|
||||
|
||||
@@ -41,6 +41,7 @@ describe("moonshot provider catalog", () => {
|
||||
expect(provider.api).toBe("openai-completions");
|
||||
expect(provider.models.map((model) => model.id)).toEqual([
|
||||
"kimi-k2.6",
|
||||
"kimi-k2.7-code",
|
||||
"kimi-k2.5",
|
||||
"kimi-k2-thinking",
|
||||
"kimi-k2-thinking-turbo",
|
||||
@@ -52,6 +53,18 @@ describe("moonshot provider catalog", () => {
|
||||
cacheRead: 0.16,
|
||||
cacheWrite: 0,
|
||||
});
|
||||
expect(requireMoonshotModel(provider, "kimi-k2.7-code")).toMatchObject({
|
||||
reasoning: true,
|
||||
input: ["text", "image"],
|
||||
contextWindow: 262144,
|
||||
maxTokens: 262144,
|
||||
cost: {
|
||||
input: 0.95,
|
||||
output: 4,
|
||||
cacheRead: 0.19,
|
||||
cacheWrite: 0,
|
||||
},
|
||||
});
|
||||
expect(requireMoonshotModel(provider, "kimi-k2.5").cost).toEqual({
|
||||
input: 0.6,
|
||||
output: 3,
|
||||
|
||||
21
extensions/moonshot/provider-policy-api.ts
Normal file
21
extensions/moonshot/provider-policy-api.ts
Normal file
@@ -0,0 +1,21 @@
|
||||
// Moonshot policy module exposes model-specific thinking controls before runtime registration.
|
||||
import type { ProviderDefaultThinkingPolicyContext } from "openclaw/plugin-sdk/core";
|
||||
|
||||
export const KIMI_K2_7_CODE_MODEL_ID = "kimi-k2.7-code";
|
||||
|
||||
export function resolveThinkingProfile(context: ProviderDefaultThinkingPolicyContext) {
|
||||
if (context.modelId.trim().toLowerCase() === KIMI_K2_7_CODE_MODEL_ID) {
|
||||
return {
|
||||
levels: [{ id: "low" as const, label: "on" }],
|
||||
defaultLevel: "low" as const,
|
||||
preserveWhenCatalogReasoningFalse: true,
|
||||
};
|
||||
}
|
||||
return {
|
||||
levels: [
|
||||
{ id: "off" as const, label: "off" },
|
||||
{ id: "low" as const, label: "on" },
|
||||
],
|
||||
defaultLevel: "off" as const,
|
||||
};
|
||||
}
|
||||
@@ -11,6 +11,24 @@ import {
|
||||
expectUnifiedModelCatalogProviderRegistration,
|
||||
} from "openclaw/plugin-sdk/provider-test-contracts";
|
||||
import { describe, expect, it, vi } from "vitest";
|
||||
|
||||
const { getOpenRouterModelCapabilitiesMock, loadOpenRouterModelCapabilitiesMock } = vi.hoisted(
|
||||
() => ({
|
||||
getOpenRouterModelCapabilitiesMock: vi.fn(),
|
||||
loadOpenRouterModelCapabilitiesMock: vi.fn(async () => {}),
|
||||
}),
|
||||
);
|
||||
|
||||
vi.mock("openclaw/plugin-sdk/provider-stream-family", async (importOriginal) => {
|
||||
const actual =
|
||||
await importOriginal<typeof import("openclaw/plugin-sdk/provider-stream-family")>();
|
||||
return {
|
||||
...actual,
|
||||
getOpenRouterModelCapabilities: getOpenRouterModelCapabilitiesMock,
|
||||
loadOpenRouterModelCapabilities: loadOpenRouterModelCapabilitiesMock,
|
||||
};
|
||||
});
|
||||
|
||||
import openrouterPlugin from "./index.js";
|
||||
import {
|
||||
buildOpenrouterProvider,
|
||||
@@ -204,6 +222,59 @@ describe("openrouter provider hooks", () => {
|
||||
expect(buildOpenrouterProvider().models?.map((model) => model.id)).not.toContain("auto");
|
||||
});
|
||||
|
||||
it("normalizes OpenRouter API ids before capability loading and lookup", async () => {
|
||||
getOpenRouterModelCapabilitiesMock.mockReset();
|
||||
loadOpenRouterModelCapabilitiesMock.mockClear();
|
||||
getOpenRouterModelCapabilitiesMock.mockReturnValue({
|
||||
name: "Claude Sonnet 4.6",
|
||||
reasoning: true,
|
||||
input: ["text", "image"],
|
||||
supportsTools: true,
|
||||
cost: { input: 1, output: 2, cacheRead: 0, cacheWrite: 0 },
|
||||
contextWindow: 200_000,
|
||||
maxTokens: 64_000,
|
||||
});
|
||||
const provider = await registerSingleProviderPlugin(openrouterPlugin);
|
||||
const modelId = "openrouter/anthropic/claude-sonnet-4.6";
|
||||
const context = {
|
||||
provider: "openrouter",
|
||||
modelId,
|
||||
modelRegistry: { find: vi.fn(() => null) },
|
||||
} as never;
|
||||
|
||||
await provider.prepareDynamicModel?.(context);
|
||||
const model = provider.resolveDynamicModel?.(context);
|
||||
|
||||
expect(loadOpenRouterModelCapabilitiesMock).toHaveBeenCalledWith("anthropic/claude-sonnet-4.6");
|
||||
expect(getOpenRouterModelCapabilitiesMock).toHaveBeenCalledWith("anthropic/claude-sonnet-4.6");
|
||||
expect(model).toMatchObject({
|
||||
id: modelId,
|
||||
name: "Claude Sonnet 4.6",
|
||||
reasoning: true,
|
||||
input: ["text", "image"],
|
||||
compat: { supportsTools: true },
|
||||
contextWindow: 200_000,
|
||||
maxTokens: 64_000,
|
||||
});
|
||||
});
|
||||
|
||||
it("keeps native OpenRouter namespace ids for capability lookup", async () => {
|
||||
getOpenRouterModelCapabilitiesMock.mockReset();
|
||||
loadOpenRouterModelCapabilitiesMock.mockClear();
|
||||
const provider = await registerSingleProviderPlugin(openrouterPlugin);
|
||||
const context = {
|
||||
provider: "openrouter",
|
||||
modelId: "openrouter/auto",
|
||||
modelRegistry: { find: vi.fn(() => null) },
|
||||
} as never;
|
||||
|
||||
await provider.prepareDynamicModel?.(context);
|
||||
provider.resolveDynamicModel?.(context);
|
||||
|
||||
expect(loadOpenRouterModelCapabilitiesMock).toHaveBeenCalledWith("openrouter/auto");
|
||||
expect(getOpenRouterModelCapabilitiesMock).toHaveBeenCalledWith("openrouter/auto");
|
||||
});
|
||||
|
||||
it("does not include retired stealth models in the bundled catalog", () => {
|
||||
const modelIds = buildOpenrouterProvider().models?.map((model) => model.id) ?? [];
|
||||
expect(modelIds).not.toContain("openrouter/hunter-alpha");
|
||||
@@ -389,6 +460,61 @@ describe("openrouter provider hooks", () => {
|
||||
},
|
||||
} as never);
|
||||
expect(normalizedHunterModel?.reasoning).toBe(false);
|
||||
expect(normalizedHunterModel?.id).toBe("openrouter/hunter-alpha");
|
||||
|
||||
const normalizedAnthropicModel = provider.normalizeResolvedModel?.({
|
||||
provider: "openrouter",
|
||||
model: {
|
||||
provider: "openrouter",
|
||||
id: "openrouter/anthropic/claude-sonnet-4.6",
|
||||
name: "anthropic/claude-sonnet-4.6",
|
||||
api: "openai-completions",
|
||||
baseUrl: "https://openrouter.ai/api/v1",
|
||||
reasoning: true,
|
||||
input: ["text", "image"],
|
||||
cost: { input: 0, output: 0, cacheRead: 0, cacheWrite: 0 },
|
||||
contextWindow: 200_000,
|
||||
maxTokens: 8192,
|
||||
},
|
||||
} as never);
|
||||
expect(normalizedAnthropicModel?.id).toBe("anthropic/claude-sonnet-4.6");
|
||||
|
||||
expect(
|
||||
provider.normalizeResolvedModel?.({
|
||||
provider: "openrouter",
|
||||
modelId: "openrouter/auto",
|
||||
model: {
|
||||
provider: "openrouter",
|
||||
id: "openrouter/auto",
|
||||
name: "OpenRouter Auto",
|
||||
api: "openai-completions",
|
||||
baseUrl: "https://openrouter.ai/api/v1",
|
||||
reasoning: false,
|
||||
input: ["text", "image"],
|
||||
cost: { input: 0, output: 0, cacheRead: 0, cacheWrite: 0 },
|
||||
contextWindow: 200_000,
|
||||
maxTokens: 8192,
|
||||
},
|
||||
} as never),
|
||||
).toBeUndefined();
|
||||
|
||||
const normalizedDuplicatedAutoModel = provider.normalizeResolvedModel?.({
|
||||
provider: "openrouter",
|
||||
modelId: "openrouter/openrouter/auto",
|
||||
model: {
|
||||
provider: "openrouter",
|
||||
id: "openrouter/openrouter/auto",
|
||||
name: "OpenRouter Auto",
|
||||
api: "openai-completions",
|
||||
baseUrl: "https://openrouter.ai/api/v1",
|
||||
reasoning: false,
|
||||
input: ["text", "image"],
|
||||
cost: { input: 0, output: 0, cacheRead: 0, cacheWrite: 0 },
|
||||
contextWindow: 200_000,
|
||||
maxTokens: 8192,
|
||||
},
|
||||
} as never);
|
||||
expect(normalizedDuplicatedAutoModel?.id).toBe("openrouter/auto");
|
||||
|
||||
expect(
|
||||
provider.normalizeTransport?.({
|
||||
|
||||
@@ -17,7 +17,7 @@ import {
|
||||
} from "openclaw/plugin-sdk/provider-stream-family";
|
||||
import { buildOpenRouterImageGenerationProvider } from "./image-generation-provider.js";
|
||||
import { openrouterMediaUnderstandingProvider } from "./media-understanding-provider.js";
|
||||
import { isOpenRouterMistralModelId } from "./models.js";
|
||||
import { isOpenRouterMistralModelId, normalizeOpenRouterApiModelId } from "./models.js";
|
||||
import { buildOpenRouterMusicGenerationProvider } from "./music-generation-provider.js";
|
||||
import { createOpenRouterOAuthAuthMethod } from "./oauth.js";
|
||||
import { applyOpenrouterConfig, OPENROUTER_DEFAULT_MODEL_REF } from "./onboard.js";
|
||||
@@ -51,15 +51,18 @@ const OPENROUTER_CACHE_TTL_MODEL_PREFIXES = [
|
||||
|
||||
function normalizeOpenRouterResolvedModel<T extends ProviderRuntimeModel>(model: T): T | undefined {
|
||||
const normalizedBaseUrl = normalizeOpenRouterBaseUrl(model.baseUrl);
|
||||
const normalizedId = normalizeOpenRouterApiModelId(model.id);
|
||||
const reasoning = isOpenRouterProxyReasoningUnsupportedModel(model.id) ? false : model.reasoning;
|
||||
if (
|
||||
(!normalizedBaseUrl || normalizedBaseUrl === model.baseUrl) &&
|
||||
(!normalizedId || normalizedId === model.id) &&
|
||||
reasoning === model.reasoning
|
||||
) {
|
||||
return undefined;
|
||||
}
|
||||
return {
|
||||
...model,
|
||||
...(normalizedId ? { id: normalizedId } : {}),
|
||||
...(normalizedBaseUrl ? { baseUrl: normalizedBaseUrl } : {}),
|
||||
reasoning,
|
||||
};
|
||||
@@ -73,7 +76,8 @@ export default definePluginEntry({
|
||||
function buildDynamicOpenRouterModel(
|
||||
ctx: ProviderResolveDynamicModelContext,
|
||||
): ProviderRuntimeModel {
|
||||
const capabilities = getOpenRouterModelCapabilities(ctx.modelId);
|
||||
const apiModelId = normalizeOpenRouterApiModelId(ctx.modelId) ?? ctx.modelId;
|
||||
const capabilities = getOpenRouterModelCapabilities(apiModelId);
|
||||
return {
|
||||
id: ctx.modelId,
|
||||
name: capabilities?.name ?? ctx.modelId,
|
||||
@@ -166,7 +170,9 @@ export default definePluginEntry({
|
||||
},
|
||||
resolveDynamicModel: (ctx) => buildDynamicOpenRouterModel(ctx),
|
||||
prepareDynamicModel: async (ctx) => {
|
||||
await loadOpenRouterModelCapabilities(ctx.modelId);
|
||||
await loadOpenRouterModelCapabilities(
|
||||
normalizeOpenRouterApiModelId(ctx.modelId) ?? ctx.modelId,
|
||||
);
|
||||
},
|
||||
normalizeConfig: ({ providerConfig }) => {
|
||||
const normalizedBaseUrl = normalizeOpenRouterBaseUrl(providerConfig.baseUrl);
|
||||
|
||||
@@ -12,13 +12,30 @@ const OPENROUTER_MISTRAL_MODEL_PREFIXES = [
|
||||
"pixtral-",
|
||||
"voxtral-",
|
||||
] as const;
|
||||
const OPENROUTER_MODEL_PREFIX = "openrouter/";
|
||||
|
||||
export function normalizeOpenRouterModelId(modelId: unknown): string | undefined {
|
||||
if (typeof modelId !== "string") {
|
||||
return undefined;
|
||||
}
|
||||
const normalized = normalizeLowercaseStringOrEmpty(modelId);
|
||||
return normalized.startsWith("openrouter/") ? normalized.slice("openrouter/".length) : normalized;
|
||||
return normalized.startsWith(OPENROUTER_MODEL_PREFIX)
|
||||
? normalized.slice(OPENROUTER_MODEL_PREFIX.length)
|
||||
: normalized;
|
||||
}
|
||||
|
||||
export function normalizeOpenRouterApiModelId(modelId: unknown): string | undefined {
|
||||
if (typeof modelId !== "string") {
|
||||
return undefined;
|
||||
}
|
||||
const normalized = normalizeLowercaseStringOrEmpty(modelId);
|
||||
if (!normalized.startsWith(OPENROUTER_MODEL_PREFIX)) {
|
||||
return normalized;
|
||||
}
|
||||
const unprefixed = normalized.slice(OPENROUTER_MODEL_PREFIX.length);
|
||||
// `openrouter/` is both a provider qualifier and an upstream namespace.
|
||||
// Strip it only when the remainder is still a namespaced API model id.
|
||||
return unprefixed.includes("/") ? unprefixed : normalized;
|
||||
}
|
||||
|
||||
export function isOpenRouterMistralModelId(modelId: unknown): boolean {
|
||||
|
||||
@@ -7,12 +7,17 @@ import {
|
||||
} from "openclaw/plugin-sdk/plugin-test-runtime";
|
||||
import { describe, expect, it } from "vitest";
|
||||
import plugin from "./index.js";
|
||||
import { normalizeOpenRouterApiModelId } from "./models.js";
|
||||
|
||||
const OPENROUTER_MODELS_URL = "https://openrouter.ai/api/v1/models";
|
||||
const OPENROUTER_MISTRAL_PROVIDER_PREFIX = "mistralai/";
|
||||
const OPENROUTER_API_KEY = process.env.OPENROUTER_API_KEY ?? "";
|
||||
const LIVE_MODEL_ID =
|
||||
process.env.OPENCLAW_LIVE_OPENROUTER_PLUGIN_MODEL?.trim() || "openai/gpt-5.4-nano";
|
||||
const LIVE_MODEL_REF =
|
||||
process.env.OPENCLAW_LIVE_OPENROUTER_PLUGIN_MODEL?.trim() ||
|
||||
"openrouter/anthropic/claude-sonnet-4.6";
|
||||
const LIVE_MODEL_ID = LIVE_MODEL_REF.startsWith("openrouter/")
|
||||
? LIVE_MODEL_REF
|
||||
: `openrouter/${LIVE_MODEL_REF}`;
|
||||
const LIVE_CACHE_MODEL_ID =
|
||||
process.env.OPENCLAW_LIVE_OPENROUTER_CACHE_MODEL?.trim() || "deepseek/deepseek-v3.2";
|
||||
const liveEnabled = OPENROUTER_API_KEY.trim().length > 0 && process.env.OPENCLAW_LIVE_TEST === "1";
|
||||
@@ -57,6 +62,40 @@ async function completeOpenRouterChat(params: {
|
||||
});
|
||||
}
|
||||
|
||||
async function expectWeatherToolCall(client: OpenAI, model: string): Promise<void> {
|
||||
const response = await client.chat.completions.create({
|
||||
model,
|
||||
messages: [{ role: "user", content: "Call get_weather for Paris." }],
|
||||
tools: [
|
||||
{
|
||||
type: "function",
|
||||
function: {
|
||||
name: "get_weather",
|
||||
description: "Get the weather for a city.",
|
||||
parameters: {
|
||||
type: "object",
|
||||
properties: { city: { type: "string" } },
|
||||
required: ["city"],
|
||||
additionalProperties: false,
|
||||
},
|
||||
},
|
||||
},
|
||||
],
|
||||
tool_choice: {
|
||||
type: "function",
|
||||
function: { name: "get_weather" },
|
||||
},
|
||||
max_tokens: 64,
|
||||
});
|
||||
|
||||
const toolCall = response.choices[0]?.message?.tool_calls?.find(
|
||||
(call) => call.type === "function",
|
||||
);
|
||||
expect(toolCall?.type).toBe("function");
|
||||
expect(toolCall?.function.name).toBe("get_weather");
|
||||
expect(JSON.parse(toolCall?.function.arguments ?? "{}")).toMatchObject({ city: "Paris" });
|
||||
}
|
||||
|
||||
async function fetchOpenRouterModelIds(): Promise<string[]> {
|
||||
const response = await fetch(OPENROUTER_MODELS_URL, {
|
||||
headers: { "accept-encoding": "identity" },
|
||||
@@ -69,7 +108,7 @@ async function fetchOpenRouterModelIds(): Promise<string[]> {
|
||||
}
|
||||
|
||||
describeLive("openrouter plugin live", () => {
|
||||
it("registers an OpenRouter provider that can complete a live request", async () => {
|
||||
it("normalizes a prefixed OpenRouter model and completes a live tool call", async () => {
|
||||
const { providers } = await registerOpenRouterPlugin();
|
||||
const provider = requireRegisteredProvider(providers, "openrouter");
|
||||
|
||||
@@ -87,17 +126,35 @@ describeLive("openrouter plugin live", () => {
|
||||
expect(resolved.api).toBe("openai-completions");
|
||||
expect(resolved.baseUrl).toBe("https://openrouter.ai/api/v1");
|
||||
|
||||
const normalized =
|
||||
provider.normalizeResolvedModel?.({
|
||||
provider: "openrouter",
|
||||
modelId: resolved.id,
|
||||
model: resolved,
|
||||
}) ?? resolved;
|
||||
expect(normalized.id).toBe(normalizeOpenRouterApiModelId(LIVE_MODEL_ID));
|
||||
|
||||
const client = new OpenAI({
|
||||
apiKey: OPENROUTER_API_KEY,
|
||||
baseURL: resolved.baseUrl,
|
||||
baseURL: normalized.baseUrl,
|
||||
});
|
||||
const response = await client.chat.completions.create({
|
||||
model: resolved.id,
|
||||
messages: [{ role: "user", content: "Reply with exactly OK." }],
|
||||
max_tokens: 16,
|
||||
const autoResolved = provider.resolveDynamicModel?.({
|
||||
provider: "openrouter",
|
||||
modelId: "openrouter/auto",
|
||||
modelRegistry: new ModelRegistryCtor(AuthStorage.inMemory()),
|
||||
});
|
||||
|
||||
expect(response.choices[0]?.message?.content?.trim()).toMatch(/^OK[.!]?$/);
|
||||
if (!autoResolved) {
|
||||
throw new Error("openrouter provider did not resolve openrouter/auto");
|
||||
}
|
||||
const autoModel =
|
||||
provider.normalizeResolvedModel?.({
|
||||
provider: "openrouter",
|
||||
modelId: autoResolved.id,
|
||||
model: autoResolved,
|
||||
}) ?? autoResolved;
|
||||
expect(autoModel.id).toBe("openrouter/auto");
|
||||
await expectWeatherToolCall(client, autoModel.id);
|
||||
await expectWeatherToolCall(client, normalized.id);
|
||||
}, 30_000);
|
||||
});
|
||||
|
||||
|
||||
@@ -106,5 +106,6 @@ export {
|
||||
type QaSuiteStartLabFn,
|
||||
type QaSuiteSummaryJson,
|
||||
type QaSuiteSummaryJsonParams,
|
||||
runQaSuite,
|
||||
runQaFlowSuite,
|
||||
} from "./src/suite.js";
|
||||
export { runQaSuite, type QaSuiteRuntimeResult } from "./src/suite-launch.runtime.js";
|
||||
|
||||
@@ -127,6 +127,7 @@ async function makeSuiteResult(params: {
|
||||
);
|
||||
return {
|
||||
outputDir: params.outputDir,
|
||||
evidencePath: path.join(params.outputDir, "qa-evidence.json"),
|
||||
reportPath: path.join(params.outputDir, "qa-suite-report.md"),
|
||||
summaryPath,
|
||||
report: "# report",
|
||||
|
||||
@@ -426,8 +426,8 @@ async function defaultRunJudge(params: {
|
||||
}
|
||||
|
||||
async function defaultRunSuite(params: Parameters<RunSuiteFn>[0]) {
|
||||
const { runQaSuiteFromRuntime } = await import("./suite-launch.runtime.js");
|
||||
return await runQaSuiteFromRuntime(params);
|
||||
const { runQaFlowSuiteFromRuntime } = await import("./suite-launch.runtime.js");
|
||||
return await runQaFlowSuiteFromRuntime(params);
|
||||
}
|
||||
|
||||
function renderCharacterEvalReport(params: {
|
||||
|
||||
@@ -3,6 +3,18 @@ import fs from "node:fs/promises";
|
||||
import path from "node:path";
|
||||
import { assertNoSymlinkParents, pathScope } from "openclaw/plugin-sdk/security-runtime";
|
||||
|
||||
export function toRepoPath(filePath: string): string {
|
||||
return filePath.split(path.sep).join("/");
|
||||
}
|
||||
|
||||
export function toRepoRelativePath(repoRoot: string, filePath: string): string {
|
||||
return toRepoPath(path.relative(repoRoot, filePath));
|
||||
}
|
||||
|
||||
export function isRepoRootRelativeRef(value: string) {
|
||||
return !path.isAbsolute(value) && value.split(/[\\/]+/u).every((part) => part !== "..");
|
||||
}
|
||||
|
||||
export function resolveRepoRelativeOutputDir(repoRoot: string, outputDir?: string) {
|
||||
if (!outputDir) {
|
||||
return undefined;
|
||||
|
||||
@@ -6,7 +6,8 @@ import { afterEach, beforeEach, describe, expect, it, vi } from "vitest";
|
||||
|
||||
const {
|
||||
runQaManualLane,
|
||||
runQaSuiteFromRuntime,
|
||||
runQaFlowSuiteFromRuntime,
|
||||
runQaSuite,
|
||||
runQaCharacterEval,
|
||||
runQaMultipass,
|
||||
listTelegramQaScenarioCatalog,
|
||||
@@ -18,7 +19,8 @@ const {
|
||||
defaultQaRuntimeModelForMode,
|
||||
} = vi.hoisted(() => ({
|
||||
runQaManualLane: vi.fn(),
|
||||
runQaSuiteFromRuntime: vi.fn(),
|
||||
runQaFlowSuiteFromRuntime: vi.fn(),
|
||||
runQaSuite: vi.fn(),
|
||||
runQaCharacterEval: vi.fn(),
|
||||
runQaMultipass: vi.fn(),
|
||||
listTelegramQaScenarioCatalog: vi.fn(),
|
||||
@@ -36,7 +38,8 @@ vi.mock("./manual-lane.runtime.js", () => ({
|
||||
}));
|
||||
|
||||
vi.mock("./suite-launch.runtime.js", () => ({
|
||||
runQaSuiteFromRuntime,
|
||||
runQaFlowSuiteFromRuntime,
|
||||
runQaSuite,
|
||||
}));
|
||||
|
||||
vi.mock("./character-eval.js", () => ({
|
||||
@@ -82,6 +85,8 @@ import {
|
||||
runQaParityReportCommand,
|
||||
runQaSuiteCommand,
|
||||
} from "./cli.runtime.js";
|
||||
import { QaSuiteInfraError } from "./errors.js";
|
||||
import { QA_EVIDENCE_FILENAME } from "./evidence-summary.js";
|
||||
import { runQaTelegramCommand } from "./live-transports/telegram/cli.runtime.js";
|
||||
import { defaultQaModelForMode as defaultQaProviderModelForMode } from "./model-selection.js";
|
||||
import type { QaProviderModeInput } from "./run-config.js";
|
||||
@@ -113,10 +118,51 @@ function expectWriteContains(mock: unknown, fragment: string): void {
|
||||
).toBe(true);
|
||||
}
|
||||
|
||||
function flowSuiteRuntimeResult(params: {
|
||||
evidencePath?: string;
|
||||
reportPath: string;
|
||||
summaryPath: string;
|
||||
scenarios?: unknown[];
|
||||
}) {
|
||||
return {
|
||||
executionKind: "flow",
|
||||
result: {
|
||||
outputDir: path.dirname(params.reportPath),
|
||||
evidencePath:
|
||||
params.evidencePath ?? path.join(path.dirname(params.reportPath), "qa-evidence.json"),
|
||||
reportPath: params.reportPath,
|
||||
summaryPath: params.summaryPath,
|
||||
report: "# QA Suite Report\n",
|
||||
scenarios: params.scenarios ?? [],
|
||||
watchUrl: "http://127.0.0.1:43124",
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
function testFileSuiteRuntimeResult(params: {
|
||||
evidencePath: string;
|
||||
executionKind?: "vitest" | "playwright";
|
||||
outputDir: string;
|
||||
reportPath: string;
|
||||
results?: unknown[];
|
||||
}) {
|
||||
return {
|
||||
executionKind: params.executionKind ?? "playwright",
|
||||
result: {
|
||||
outputDir: params.outputDir,
|
||||
executionKind: params.executionKind ?? "playwright",
|
||||
reportPath: params.reportPath,
|
||||
evidencePath: params.evidencePath,
|
||||
results: params.results ?? [{ status: "pass" }],
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
describe("qa cli runtime", () => {
|
||||
let stdoutWrite: ReturnType<typeof vi.spyOn>;
|
||||
let stderrWrite: ReturnType<typeof vi.spyOn>;
|
||||
let suiteArtifactsDir: string;
|
||||
let suiteEvidencePath: string;
|
||||
let suiteReportPath: string;
|
||||
let suiteSummaryPath: string;
|
||||
let telegramArtifactsDir: string;
|
||||
@@ -124,11 +170,13 @@ describe("qa cli runtime", () => {
|
||||
|
||||
beforeEach(async () => {
|
||||
suiteArtifactsDir = await fs.mkdtemp(path.join(os.tmpdir(), "qa-suite-runtime-"));
|
||||
suiteEvidencePath = path.join(suiteArtifactsDir, "qa-evidence.json");
|
||||
suiteReportPath = path.join(suiteArtifactsDir, "qa-suite-report.md");
|
||||
suiteSummaryPath = path.join(suiteArtifactsDir, "qa-suite-summary.json");
|
||||
telegramArtifactsDir = await fs.mkdtemp(path.join(os.tmpdir(), "qa-telegram-runtime-"));
|
||||
telegramSummaryPath = path.join(telegramArtifactsDir, "telegram-qa-summary.json");
|
||||
telegramSummaryPath = path.join(telegramArtifactsDir, QA_EVIDENCE_FILENAME);
|
||||
await fs.writeFile(suiteReportPath, "# QA Suite Report\n", "utf8");
|
||||
await fs.writeFile(suiteEvidencePath, JSON.stringify({ entries: [] }), "utf8");
|
||||
await fs.writeFile(
|
||||
suiteSummaryPath,
|
||||
JSON.stringify({
|
||||
@@ -155,7 +203,8 @@ describe("qa cli runtime", () => {
|
||||
);
|
||||
stdoutWrite = vi.spyOn(process.stdout, "write").mockReturnValue(true);
|
||||
stderrWrite = vi.spyOn(process.stderr, "write").mockReturnValue(true);
|
||||
runQaSuiteFromRuntime.mockReset();
|
||||
runQaFlowSuiteFromRuntime.mockReset();
|
||||
runQaSuite.mockReset();
|
||||
runQaCharacterEval.mockReset();
|
||||
runQaManualLane.mockReset();
|
||||
runQaMultipass.mockReset();
|
||||
@@ -169,7 +218,15 @@ describe("qa cli runtime", () => {
|
||||
(mode: string, options?: { alternate?: boolean }) =>
|
||||
defaultQaProviderModelForMode(mode as QaProviderModeInput, options),
|
||||
);
|
||||
runQaSuiteFromRuntime.mockResolvedValue({
|
||||
runQaSuite.mockResolvedValue(
|
||||
flowSuiteRuntimeResult({
|
||||
reportPath: suiteReportPath,
|
||||
summaryPath: suiteSummaryPath,
|
||||
}),
|
||||
);
|
||||
runQaFlowSuiteFromRuntime.mockResolvedValue({
|
||||
outputDir: suiteArtifactsDir,
|
||||
evidencePath: suiteEvidencePath,
|
||||
watchUrl: "http://127.0.0.1:43124",
|
||||
reportPath: suiteReportPath,
|
||||
summaryPath: suiteSummaryPath,
|
||||
@@ -240,6 +297,48 @@ describe("qa cli runtime", () => {
|
||||
await fs.rm(telegramArtifactsDir, { recursive: true, force: true });
|
||||
});
|
||||
|
||||
it("runs selected Playwright scenarios through the suite command", async () => {
|
||||
const evidencePath = path.join(suiteArtifactsDir, "qa-evidence.json");
|
||||
await fs.writeFile(evidencePath, JSON.stringify({ entries: [] }), "utf8");
|
||||
runQaSuite.mockResolvedValueOnce(
|
||||
testFileSuiteRuntimeResult({
|
||||
outputDir: suiteArtifactsDir,
|
||||
reportPath: suiteReportPath,
|
||||
evidencePath,
|
||||
}),
|
||||
);
|
||||
|
||||
await runQaSuiteCommand({
|
||||
repoRoot: process.cwd(),
|
||||
outputDir: ".artifacts/qa-e2e/scenario-test",
|
||||
primaryModel: "mock-openai/gpt-5.5",
|
||||
scenarioIds: ["control-ui-chat-flow-playwright"],
|
||||
});
|
||||
|
||||
expect(runQaSuite).toHaveBeenCalledWith({
|
||||
repoRoot: process.cwd(),
|
||||
outputDir: path.join(process.cwd(), ".artifacts", "qa-e2e", "scenario-test"),
|
||||
transportId: "qa-channel",
|
||||
primaryModel: "mock-openai/gpt-5.5",
|
||||
alternateModel: undefined,
|
||||
fastMode: undefined,
|
||||
scenarioIds: ["control-ui-chat-flow-playwright"],
|
||||
});
|
||||
expectWriteContains(stdoutWrite, `QA suite evidence: ${evidencePath}`);
|
||||
});
|
||||
|
||||
it("rejects host-only resource options for Playwright scenarios", async () => {
|
||||
await expect(
|
||||
runQaSuiteCommand({
|
||||
repoRoot: process.cwd(),
|
||||
image: "lts",
|
||||
scenarioIds: ["control-ui-chat-flow-playwright"],
|
||||
}),
|
||||
).rejects.toThrow("--image, --cpus, --memory, and --disk require --runner multipass");
|
||||
|
||||
expect(runQaSuite).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it("resolves suite repo-root-relative paths before dispatching", async () => {
|
||||
await runQaSuiteCommand({
|
||||
repoRoot: "/tmp/openclaw-repo",
|
||||
@@ -252,7 +351,7 @@ describe("qa cli runtime", () => {
|
||||
scenarioIds: ["approval-turn-tool-followthrough"],
|
||||
});
|
||||
|
||||
expect(runQaSuiteFromRuntime).toHaveBeenCalledWith({
|
||||
expect(runQaSuite).toHaveBeenCalledWith({
|
||||
repoRoot: path.resolve("/tmp/openclaw-repo"),
|
||||
outputDir: path.resolve("/tmp/openclaw-repo", ".artifacts/qa/frontier"),
|
||||
transportId: "qa-channel",
|
||||
@@ -273,7 +372,7 @@ describe("qa cli runtime", () => {
|
||||
enabledPluginIds: ["browser", "memory-core"],
|
||||
});
|
||||
|
||||
expect(runQaSuiteFromRuntime).toHaveBeenCalledWith({
|
||||
expect(runQaSuite).toHaveBeenCalledWith({
|
||||
repoRoot: path.resolve("/tmp/openclaw-repo"),
|
||||
outputDir: undefined,
|
||||
transportId: "qa-channel",
|
||||
@@ -294,7 +393,7 @@ describe("qa cli runtime", () => {
|
||||
runtimePair: "openclaw,codex",
|
||||
});
|
||||
|
||||
expect(runQaSuiteFromRuntime).toHaveBeenCalledWith({
|
||||
expect(runQaSuite).toHaveBeenCalledWith({
|
||||
repoRoot: path.resolve("/tmp/openclaw-repo"),
|
||||
outputDir: undefined,
|
||||
transportId: "qa-channel",
|
||||
@@ -316,7 +415,7 @@ describe("qa cli runtime", () => {
|
||||
runtimePair: "legacy-runtime,codex",
|
||||
}),
|
||||
).rejects.toThrow('--runtime-pair only supports "openclaw" and "codex".');
|
||||
expect(runQaSuiteFromRuntime).not.toHaveBeenCalled();
|
||||
expect(runQaSuite).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it("accepts legacy pi as a runtime-pair suite alias", async () => {
|
||||
@@ -327,7 +426,7 @@ describe("qa cli runtime", () => {
|
||||
runtimePair: "pi,codex",
|
||||
});
|
||||
|
||||
expect(runQaSuiteFromRuntime).toHaveBeenCalledWith(
|
||||
expect(runQaSuite).toHaveBeenCalledWith(
|
||||
expect.objectContaining({
|
||||
repoRoot: path.resolve("/tmp/openclaw-repo"),
|
||||
runtimePair: ["openclaw", "codex"],
|
||||
@@ -344,7 +443,7 @@ describe("qa cli runtime", () => {
|
||||
scenarioIds: ["thread-memory-isolation"],
|
||||
});
|
||||
|
||||
expect(runQaSuiteFromRuntime).toHaveBeenCalledWith({
|
||||
expect(runQaSuite).toHaveBeenCalledWith({
|
||||
repoRoot: path.resolve("/tmp/openclaw-repo"),
|
||||
outputDir: undefined,
|
||||
transportId: "qa-channel",
|
||||
@@ -491,12 +590,13 @@ describe("qa cli runtime", () => {
|
||||
concurrency: 3,
|
||||
});
|
||||
|
||||
expectFields(mockFirstObjectArg(runQaSuiteFromRuntime), {
|
||||
expectFields(mockFirstObjectArg(runQaSuite), {
|
||||
repoRoot: path.resolve("/tmp/openclaw-repo"),
|
||||
transportId: "qa-channel",
|
||||
scenarioIds: ["channel-chat-baseline", "thread-follow-up"],
|
||||
concurrency: 3,
|
||||
});
|
||||
expectWriteContains(stdoutWrite, `QA suite evidence: ${suiteEvidencePath}`);
|
||||
});
|
||||
|
||||
it("rejects fractional suite concurrency from programmatic callers", async () => {
|
||||
@@ -507,7 +607,7 @@ describe("qa cli runtime", () => {
|
||||
concurrency: 1.5,
|
||||
}),
|
||||
).rejects.toThrow("--concurrency must be a positive integer");
|
||||
expect(runQaSuiteFromRuntime).not.toHaveBeenCalled();
|
||||
expect(runQaSuite).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it("sets a failing exit code when host suite scenarios fail", async () => {
|
||||
@@ -525,12 +625,12 @@ describe("qa cli runtime", () => {
|
||||
}),
|
||||
"utf8",
|
||||
);
|
||||
runQaSuiteFromRuntime.mockResolvedValueOnce({
|
||||
watchUrl: "http://127.0.0.1:43124",
|
||||
reportPath: suiteReportPath,
|
||||
summaryPath: suiteSummaryPath,
|
||||
scenarios: [],
|
||||
});
|
||||
runQaSuite.mockResolvedValueOnce(
|
||||
flowSuiteRuntimeResult({
|
||||
reportPath: suiteReportPath,
|
||||
summaryPath: suiteSummaryPath,
|
||||
}),
|
||||
);
|
||||
|
||||
try {
|
||||
await runQaSuiteCommand({
|
||||
@@ -558,12 +658,12 @@ describe("qa cli runtime", () => {
|
||||
}),
|
||||
"utf8",
|
||||
);
|
||||
runQaSuiteFromRuntime.mockResolvedValueOnce({
|
||||
watchUrl: "http://127.0.0.1:43124",
|
||||
reportPath: suiteReportPath,
|
||||
summaryPath: suiteSummaryPath,
|
||||
scenarios: [],
|
||||
});
|
||||
runQaSuite.mockResolvedValueOnce(
|
||||
flowSuiteRuntimeResult({
|
||||
reportPath: suiteReportPath,
|
||||
summaryPath: suiteSummaryPath,
|
||||
}),
|
||||
);
|
||||
|
||||
try {
|
||||
await runQaSuiteCommand({
|
||||
@@ -590,18 +690,19 @@ describe("qa cli runtime", () => {
|
||||
}),
|
||||
"utf8",
|
||||
);
|
||||
runQaSuiteFromRuntime.mockResolvedValueOnce({
|
||||
watchUrl: "http://127.0.0.1:43124",
|
||||
reportPath: suiteReportPath,
|
||||
summaryPath: suiteSummaryPath,
|
||||
scenarios: [
|
||||
{
|
||||
name: "channel chat baseline",
|
||||
status: "fail",
|
||||
steps: [],
|
||||
},
|
||||
],
|
||||
});
|
||||
runQaSuite.mockResolvedValueOnce(
|
||||
flowSuiteRuntimeResult({
|
||||
reportPath: suiteReportPath,
|
||||
summaryPath: suiteSummaryPath,
|
||||
scenarios: [
|
||||
{
|
||||
name: "channel chat baseline",
|
||||
status: "fail",
|
||||
steps: [],
|
||||
},
|
||||
],
|
||||
}),
|
||||
);
|
||||
|
||||
try {
|
||||
await runQaSuiteCommand({
|
||||
@@ -615,42 +716,45 @@ describe("qa cli runtime", () => {
|
||||
});
|
||||
|
||||
it("retries host suite runs once for retryable infra failures", async () => {
|
||||
runQaSuiteFromRuntime
|
||||
.mockRejectedValueOnce(new Error("agent.wait timeout while waiting for transport ready"))
|
||||
.mockResolvedValueOnce({
|
||||
watchUrl: "http://127.0.0.1:43124",
|
||||
reportPath: suiteReportPath,
|
||||
summaryPath: suiteSummaryPath,
|
||||
scenarios: [],
|
||||
});
|
||||
runQaSuite
|
||||
.mockRejectedValueOnce(
|
||||
new QaSuiteInfraError("agent_wait_failed", "agent.wait failed: gateway call timed out"),
|
||||
)
|
||||
.mockResolvedValueOnce(
|
||||
flowSuiteRuntimeResult({
|
||||
reportPath: suiteReportPath,
|
||||
summaryPath: suiteSummaryPath,
|
||||
}),
|
||||
);
|
||||
|
||||
await runQaSuiteCommand({
|
||||
repoRoot: "/tmp/openclaw-repo",
|
||||
});
|
||||
|
||||
expect(runQaSuiteFromRuntime).toHaveBeenCalledTimes(2);
|
||||
expectWriteContains(stderrWrite, "[qa-suite] infra retry 1/1: agent.wait timeout");
|
||||
expect(runQaSuite).toHaveBeenCalledTimes(2);
|
||||
expectWriteContains(stderrWrite, "[qa-suite] infra retry 1/1: agent.wait failed");
|
||||
});
|
||||
|
||||
it("retries host suite runs once for qa-channel readiness timeouts", async () => {
|
||||
runQaSuiteFromRuntime
|
||||
runQaSuite
|
||||
.mockRejectedValueOnce(
|
||||
new Error(
|
||||
new QaSuiteInfraError(
|
||||
"transport_ready_timeout",
|
||||
"timed out after 180000ms waiting for qa-channel ready; last status: no qa-channel accounts reported",
|
||||
),
|
||||
)
|
||||
.mockResolvedValueOnce({
|
||||
watchUrl: "http://127.0.0.1:43124",
|
||||
reportPath: suiteReportPath,
|
||||
summaryPath: suiteSummaryPath,
|
||||
scenarios: [],
|
||||
});
|
||||
.mockResolvedValueOnce(
|
||||
flowSuiteRuntimeResult({
|
||||
reportPath: suiteReportPath,
|
||||
summaryPath: suiteSummaryPath,
|
||||
}),
|
||||
);
|
||||
|
||||
await runQaSuiteCommand({
|
||||
repoRoot: "/tmp/openclaw-repo",
|
||||
});
|
||||
|
||||
expect(runQaSuiteFromRuntime).toHaveBeenCalledTimes(2);
|
||||
expect(runQaSuite).toHaveBeenCalledTimes(2);
|
||||
expectWriteContains(
|
||||
stderrWrite,
|
||||
"[qa-suite] infra retry 1/1: timed out after 180000ms waiting for qa-channel ready",
|
||||
@@ -658,7 +762,7 @@ describe("qa cli runtime", () => {
|
||||
});
|
||||
|
||||
it("does not retry host suite runs for generic timeout wording", async () => {
|
||||
runQaSuiteFromRuntime.mockRejectedValueOnce(
|
||||
runQaSuite.mockRejectedValueOnce(
|
||||
new Error("approval-turn timed out waiting for post-approval read"),
|
||||
);
|
||||
|
||||
@@ -668,7 +772,7 @@ describe("qa cli runtime", () => {
|
||||
}),
|
||||
).rejects.toThrow("approval-turn timed out waiting for post-approval read");
|
||||
|
||||
expect(runQaSuiteFromRuntime).toHaveBeenCalledTimes(1);
|
||||
expect(runQaSuite).toHaveBeenCalledTimes(1);
|
||||
});
|
||||
|
||||
it("does not retry host suite runs for semantic failures", async () => {
|
||||
@@ -686,24 +790,25 @@ describe("qa cli runtime", () => {
|
||||
}),
|
||||
"utf8",
|
||||
);
|
||||
runQaSuiteFromRuntime.mockResolvedValueOnce({
|
||||
watchUrl: "http://127.0.0.1:43124",
|
||||
reportPath: suiteReportPath,
|
||||
summaryPath: suiteSummaryPath,
|
||||
scenarios: [
|
||||
{
|
||||
name: "channel chat baseline",
|
||||
status: "fail",
|
||||
steps: [],
|
||||
},
|
||||
],
|
||||
});
|
||||
runQaSuite.mockResolvedValueOnce(
|
||||
flowSuiteRuntimeResult({
|
||||
reportPath: suiteReportPath,
|
||||
summaryPath: suiteSummaryPath,
|
||||
scenarios: [
|
||||
{
|
||||
name: "channel chat baseline",
|
||||
status: "fail",
|
||||
steps: [],
|
||||
},
|
||||
],
|
||||
}),
|
||||
);
|
||||
|
||||
try {
|
||||
await runQaSuiteCommand({
|
||||
repoRoot: "/tmp/openclaw-repo",
|
||||
});
|
||||
expect(runQaSuiteFromRuntime).toHaveBeenCalledTimes(1);
|
||||
expect(runQaSuite).toHaveBeenCalledTimes(1);
|
||||
expect(process.exitCode).toBe(1);
|
||||
} finally {
|
||||
process.exitCode = priorExitCode;
|
||||
@@ -720,7 +825,7 @@ describe("qa cli runtime", () => {
|
||||
preflight: true,
|
||||
});
|
||||
|
||||
const preflightArgs = mockFirstObjectArg(runQaSuiteFromRuntime);
|
||||
const preflightArgs = mockFirstObjectArg(runQaFlowSuiteFromRuntime);
|
||||
expectFields(preflightArgs, {
|
||||
repoRoot,
|
||||
transportId: "qa-channel",
|
||||
@@ -749,7 +854,9 @@ describe("qa cli runtime", () => {
|
||||
}),
|
||||
"utf8",
|
||||
);
|
||||
runQaSuiteFromRuntime.mockResolvedValueOnce({
|
||||
runQaFlowSuiteFromRuntime.mockResolvedValueOnce({
|
||||
outputDir: suiteArtifactsDir,
|
||||
evidencePath: suiteEvidencePath,
|
||||
watchUrl: "http://127.0.0.1:43124",
|
||||
reportPath: suiteReportPath,
|
||||
summaryPath: suiteSummaryPath,
|
||||
@@ -779,7 +886,9 @@ describe("qa cli runtime", () => {
|
||||
}),
|
||||
"utf8",
|
||||
);
|
||||
runQaSuiteFromRuntime.mockResolvedValueOnce({
|
||||
runQaFlowSuiteFromRuntime.mockResolvedValueOnce({
|
||||
outputDir: suiteArtifactsDir,
|
||||
evidencePath: suiteEvidencePath,
|
||||
watchUrl: "http://127.0.0.1:43124",
|
||||
reportPath: suiteReportPath,
|
||||
summaryPath: suiteSummaryPath,
|
||||
@@ -818,7 +927,7 @@ describe("qa cli runtime", () => {
|
||||
scenarioIds: ["claude-cli-provider-capabilities-subscription"],
|
||||
});
|
||||
|
||||
expectFields(mockFirstObjectArg(runQaSuiteFromRuntime), {
|
||||
expectFields(mockFirstObjectArg(runQaSuite), {
|
||||
repoRoot: path.resolve("/tmp/openclaw-repo"),
|
||||
providerMode: "live-frontier",
|
||||
primaryModel: "claude-cli/claude-sonnet-4-6",
|
||||
@@ -835,7 +944,7 @@ describe("qa cli runtime", () => {
|
||||
scenarioIds: ["channel-chat-baseline"],
|
||||
});
|
||||
|
||||
expectFields(mockFirstObjectArg(runQaSuiteFromRuntime), {
|
||||
expectFields(mockFirstObjectArg(runQaSuite), {
|
||||
repoRoot: path.resolve("/tmp/openclaw-repo"),
|
||||
scenarioIds: [
|
||||
"channel-chat-baseline",
|
||||
@@ -862,7 +971,7 @@ describe("qa cli runtime", () => {
|
||||
scenarioIds: ["channel-chat-baseline"],
|
||||
});
|
||||
|
||||
expectFields(mockFirstObjectArg(runQaSuiteFromRuntime), {
|
||||
expectFields(mockFirstObjectArg(runQaSuite), {
|
||||
repoRoot: path.resolve("/tmp/openclaw-repo"),
|
||||
scenarioIds: [
|
||||
"channel-chat-baseline",
|
||||
@@ -887,7 +996,7 @@ describe("qa cli runtime", () => {
|
||||
scenarioIds: ["channel-chat-baseline", "runtime-tool-bash"],
|
||||
});
|
||||
|
||||
expectFields(mockFirstObjectArg(runQaSuiteFromRuntime), {
|
||||
expectFields(mockFirstObjectArg(runQaSuite), {
|
||||
repoRoot: path.resolve("/tmp/openclaw-repo"),
|
||||
scenarioIds: [
|
||||
"channel-chat-baseline",
|
||||
@@ -920,7 +1029,7 @@ describe("qa cli runtime", () => {
|
||||
runtimeParityTier: ["optional,soak"],
|
||||
});
|
||||
|
||||
expectFields(mockFirstObjectArg(runQaSuiteFromRuntime), {
|
||||
expectFields(mockFirstObjectArg(runQaSuite), {
|
||||
scenarioIds: [
|
||||
"runtime-soak-100-turn",
|
||||
"runtime-tool-image-generate",
|
||||
@@ -1460,7 +1569,21 @@ describe("qa cli runtime", () => {
|
||||
memory: "4G",
|
||||
disk: "24G",
|
||||
});
|
||||
expect(runQaSuiteFromRuntime).not.toHaveBeenCalled();
|
||||
expect(runQaSuite).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it("rejects Vitest and Playwright scenarios on the multipass runner", async () => {
|
||||
await expect(
|
||||
runQaSuiteCommand({
|
||||
repoRoot: "/tmp/openclaw-repo",
|
||||
runner: "multipass",
|
||||
scenarioIds: ["control-ui-chat-flow-playwright"],
|
||||
}),
|
||||
).rejects.toThrow(
|
||||
"--runner multipass requires execution.kind: flow scenarios; unsupported scenario(s): control-ui-chat-flow-playwright (playwright)",
|
||||
);
|
||||
|
||||
expect(runQaMultipass).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it("passes runtime-pair suite selection through to the multipass runner", async () => {
|
||||
@@ -1658,7 +1781,9 @@ describe("qa cli runtime", () => {
|
||||
repoRoot: "/tmp/openclaw-repo",
|
||||
runner: "multipass",
|
||||
}),
|
||||
).rejects.toThrow("did not include counts.failed, counts.skipped, or scenarios[].status");
|
||||
).rejects.toThrow(
|
||||
"did not include counts.failed, counts.skipped, scenarios[].status, or entries[].result.status",
|
||||
);
|
||||
} finally {
|
||||
await fs.rm(repoRoot, { recursive: true, force: true });
|
||||
}
|
||||
@@ -1713,7 +1838,7 @@ describe("qa cli runtime", () => {
|
||||
alternateModel: "anthropic/claude-opus-4-8",
|
||||
});
|
||||
|
||||
expect(runQaSuiteFromRuntime).toHaveBeenCalledWith({
|
||||
expect(runQaSuite).toHaveBeenCalledWith({
|
||||
repoRoot: path.resolve("/tmp/openclaw-repo"),
|
||||
outputDir: undefined,
|
||||
transportId: "qa-channel",
|
||||
|
||||
@@ -28,6 +28,7 @@ import {
|
||||
} from "./coverage-report.js";
|
||||
import { buildQaDockerHarnessImage, writeQaDockerHarnessFiles } from "./docker-harness.js";
|
||||
import { runQaDockerUp } from "./docker-up.runtime.js";
|
||||
import { QaSuiteArtifactError, QaSuiteInfraError } from "./errors.js";
|
||||
import type { QaCliBackendAuthMode } from "./gateway-child.js";
|
||||
import {
|
||||
createMockJsonlReplayCellRunner,
|
||||
@@ -67,7 +68,7 @@ import {
|
||||
type QaRuntimeParityTier,
|
||||
} from "./scenario-catalog.js";
|
||||
import { resolveQaScenarioPackScenarioIds } from "./scenario-packs.js";
|
||||
import { runQaSuiteFromRuntime } from "./suite-launch.runtime.js";
|
||||
import { runQaFlowSuiteFromRuntime, runQaSuite } from "./suite-launch.runtime.js";
|
||||
import { readQaSuiteFailedOrSkippedScenarioCountFromFile } from "./suite-summary.js";
|
||||
import {
|
||||
buildTokenEfficiencyReport,
|
||||
@@ -81,6 +82,13 @@ import {
|
||||
} from "./tool-coverage-report.js";
|
||||
|
||||
const QA_SUITE_INFRA_RETRY_LIMIT = 1;
|
||||
const QA_SUITE_INFRA_RETRY_NETWORK_ERROR_CODES = new Set([
|
||||
"ECONNRESET",
|
||||
"ECONNREFUSED",
|
||||
"EPIPE",
|
||||
"ETIMEDOUT",
|
||||
"UND_ERR_SOCKET",
|
||||
]);
|
||||
|
||||
type InterruptibleServer = {
|
||||
baseUrl: string;
|
||||
@@ -243,46 +251,58 @@ function resolveQaRuntimeParityTierScenarioIds(params: {
|
||||
return uniqueStrings([...params.scenarioIds, ...matchingScenarioIds]);
|
||||
}
|
||||
|
||||
function isQaSuiteInfraRetryableError(error: unknown) {
|
||||
const message = formatErrorMessage(error).toLowerCase();
|
||||
return (
|
||||
message.includes("agent.wait timeout") ||
|
||||
message.includes("qa cli timed out") ||
|
||||
message.includes("readyz") ||
|
||||
message.includes("gateway healthy") ||
|
||||
message.includes("transport ready") ||
|
||||
message.includes("waiting for qa-channel ready") ||
|
||||
message.includes("econnreset") ||
|
||||
message.includes("econnrefused") ||
|
||||
message.includes("socket hang up") ||
|
||||
message.includes("could not read qa summary json") ||
|
||||
message.includes("could not parse qa summary json") ||
|
||||
message.includes("did not include counts.failed, counts.skipped, or scenarios[].status") ||
|
||||
message.includes("did not produce report artifact")
|
||||
function rejectNonFlowScenarioIdsForMultipass(scenarioIds: readonly string[]) {
|
||||
if (scenarioIds.length === 0) {
|
||||
return;
|
||||
}
|
||||
const scenarioById = new Map(
|
||||
readQaScenarioPack().scenarios.map((scenario) => [scenario.id, scenario]),
|
||||
);
|
||||
}
|
||||
|
||||
async function assertQaSuiteArtifacts(result: { reportPath: string; summaryPath: string }) {
|
||||
try {
|
||||
await fs.access(result.reportPath);
|
||||
} catch (error) {
|
||||
const nonFlowScenarios = scenarioIds.flatMap((scenarioId) => {
|
||||
const scenario = scenarioById.get(scenarioId);
|
||||
return scenario && scenario.execution.kind !== "flow"
|
||||
? [`${scenario.id} (${scenario.execution.kind})`]
|
||||
: [];
|
||||
});
|
||||
if (nonFlowScenarios.length > 0) {
|
||||
throw new Error(
|
||||
`QA suite did not produce report artifact at ${result.reportPath}: ${formatErrorMessage(error)}`,
|
||||
{ cause: error },
|
||||
`--runner multipass requires execution.kind: flow scenarios; unsupported scenario(s): ${nonFlowScenarios.join(", ")}`,
|
||||
);
|
||||
}
|
||||
await readQaSuiteFailedOrSkippedScenarioCountFromFile(result.summaryPath);
|
||||
}
|
||||
|
||||
async function runQaSuiteFromRuntimeWithInfraRetry(
|
||||
params: Parameters<typeof runQaSuiteFromRuntime>[0],
|
||||
function isQaSuiteInfraRetryableError(error: unknown) {
|
||||
if (error instanceof QaSuiteArtifactError || error instanceof QaSuiteInfraError) {
|
||||
return true;
|
||||
}
|
||||
return hasQaSuiteRetryableNetworkCode(error);
|
||||
}
|
||||
|
||||
function hasQaSuiteRetryableNetworkCode(error: unknown) {
|
||||
let current: unknown = error;
|
||||
for (let depth = 0; depth < 4 && current; depth += 1) {
|
||||
if (typeof current !== "object") {
|
||||
return false;
|
||||
}
|
||||
const record = current as { cause?: unknown; code?: unknown };
|
||||
if (
|
||||
typeof record.code === "string" &&
|
||||
QA_SUITE_INFRA_RETRY_NETWORK_ERROR_CODES.has(record.code.toUpperCase())
|
||||
) {
|
||||
return true;
|
||||
}
|
||||
current = record.cause;
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
async function runQaSuiteWithInfraRetry<Result>(
|
||||
run: () => Promise<Result>,
|
||||
maxRetries = QA_SUITE_INFRA_RETRY_LIMIT,
|
||||
) {
|
||||
for (let attempt = 0; attempt <= maxRetries; attempt += 1) {
|
||||
try {
|
||||
const result = await runQaSuiteFromRuntime(params);
|
||||
await assertQaSuiteArtifacts(result);
|
||||
return result;
|
||||
return await run();
|
||||
} catch (error) {
|
||||
const retryable = isQaSuiteInfraRetryableError(error);
|
||||
if (!retryable || attempt >= maxRetries) {
|
||||
@@ -311,16 +331,18 @@ async function runQaParityPreflight(params: {
|
||||
"preflight",
|
||||
`suite-${Date.now().toString(36)}`,
|
||||
);
|
||||
const result = await runQaSuiteFromRuntimeWithInfraRetry({
|
||||
repoRoot: params.repoRoot,
|
||||
outputDir,
|
||||
transportId: params.transportId,
|
||||
providerMode: params.providerMode,
|
||||
primaryModel: params.primaryModel,
|
||||
alternateModel: params.alternateModel,
|
||||
scenarioIds: ["approval-turn-tool-followthrough"],
|
||||
concurrency: 1,
|
||||
});
|
||||
const result = await runQaSuiteWithInfraRetry(() =>
|
||||
runQaFlowSuiteFromRuntime({
|
||||
repoRoot: params.repoRoot,
|
||||
outputDir,
|
||||
transportId: params.transportId,
|
||||
providerMode: params.providerMode,
|
||||
primaryModel: params.primaryModel,
|
||||
alternateModel: params.alternateModel,
|
||||
scenarioIds: ["approval-turn-tool-followthrough"],
|
||||
concurrency: 1,
|
||||
}),
|
||||
);
|
||||
process.stdout.write(`QA parity preflight watch: ${result.watchUrl}\n`);
|
||||
process.stdout.write(`QA parity preflight report: ${result.reportPath}\n`);
|
||||
process.stdout.write(`QA parity preflight summary: ${result.summaryPath}\n`);
|
||||
@@ -590,14 +612,14 @@ export async function runQaSuiteCommand(opts: {
|
||||
runtimeParityTiers,
|
||||
});
|
||||
const allowFailures = opts.allowFailures === true;
|
||||
if (runner !== "host" && runner !== "multipass") {
|
||||
throw new Error(`--runner must be one of host or multipass, got "${opts.runner}".`);
|
||||
}
|
||||
const providerMode = normalizeQaProviderMode(opts.providerMode);
|
||||
const runtimePair = parseQaRuntimePair(opts.runtimePair);
|
||||
const claudeCliAuthMode = parseQaCliBackendAuthMode(opts.cliAuthMode);
|
||||
const primaryModel = normalizeQaOptionalModelRef(opts.primaryModel);
|
||||
const alternateModel = normalizeQaOptionalModelRef(opts.alternateModel);
|
||||
if (runner !== "host" && runner !== "multipass") {
|
||||
throw new Error(`--runner must be one of host or multipass, got "${opts.runner}".`);
|
||||
}
|
||||
if (opts.preflight === true && runner !== "host") {
|
||||
throw new Error("--preflight requires --runner host.");
|
||||
}
|
||||
@@ -614,12 +636,13 @@ export async function runQaSuiteCommand(opts: {
|
||||
throw new Error("--cli-auth-mode requires --runner host.");
|
||||
}
|
||||
if (runner === "multipass") {
|
||||
rejectNonFlowScenarioIdsForMultipass(scenarioIds);
|
||||
const thinkingDefault = parseQaThinkingLevel("--thinking", opts.thinking);
|
||||
const result = await runQaMultipass({
|
||||
repoRoot,
|
||||
outputDir: resolveRepoRelativeOutputDir(repoRoot, opts.outputDir),
|
||||
transportId,
|
||||
providerMode,
|
||||
...(opts.providerMode !== undefined ? { providerMode } : {}),
|
||||
primaryModel,
|
||||
alternateModel,
|
||||
fastMode: opts.fastMode,
|
||||
@@ -662,31 +685,49 @@ export async function runQaSuiteCommand(opts: {
|
||||
return;
|
||||
}
|
||||
const thinkingDefault = parseQaThinkingLevel("--thinking", opts.thinking);
|
||||
const result = await runQaSuiteFromRuntimeWithInfraRetry({
|
||||
repoRoot,
|
||||
outputDir: resolveRepoRelativeOutputDir(repoRoot, opts.outputDir),
|
||||
transportId,
|
||||
providerMode,
|
||||
primaryModel,
|
||||
alternateModel,
|
||||
fastMode: opts.fastMode,
|
||||
...(thinkingDefault ? { thinkingDefault } : {}),
|
||||
...(claudeCliAuthMode ? { claudeCliAuthMode } : {}),
|
||||
scenarioIds,
|
||||
...(opts.enabledPluginIds !== undefined ? { enabledPluginIds: opts.enabledPluginIds } : {}),
|
||||
...(opts.concurrency !== undefined
|
||||
? { concurrency: parseQaPositiveIntegerOption("--concurrency", opts.concurrency) }
|
||||
: {}),
|
||||
...(runtimePair ? { runtimePair } : {}),
|
||||
});
|
||||
process.stdout.write(`QA suite watch: ${result.watchUrl}\n`);
|
||||
process.stdout.write(`QA suite report: ${result.reportPath}\n`);
|
||||
process.stdout.write(`QA suite summary: ${result.summaryPath}\n`);
|
||||
const blockingScenarioCount = await readQaSuiteFailedOrSkippedScenarioCountFromFile(
|
||||
result.summaryPath,
|
||||
const runtimeResult = await runQaSuiteWithInfraRetry(() =>
|
||||
runQaSuite({
|
||||
repoRoot,
|
||||
outputDir: resolveRepoRelativeOutputDir(repoRoot, opts.outputDir),
|
||||
transportId,
|
||||
...(opts.providerMode !== undefined ? { providerMode } : {}),
|
||||
primaryModel,
|
||||
alternateModel,
|
||||
fastMode: opts.fastMode,
|
||||
...(thinkingDefault ? { thinkingDefault } : {}),
|
||||
...(claudeCliAuthMode ? { claudeCliAuthMode } : {}),
|
||||
scenarioIds,
|
||||
...(opts.enabledPluginIds !== undefined ? { enabledPluginIds: opts.enabledPluginIds } : {}),
|
||||
...(opts.concurrency !== undefined
|
||||
? { concurrency: parseQaPositiveIntegerOption("--concurrency", opts.concurrency) }
|
||||
: {}),
|
||||
...(runtimePair ? { runtimePair } : {}),
|
||||
}),
|
||||
);
|
||||
if (!allowFailures && blockingScenarioCount > 0) {
|
||||
process.exitCode = 1;
|
||||
switch (runtimeResult.executionKind) {
|
||||
case "vitest":
|
||||
case "playwright": {
|
||||
const result = runtimeResult.result;
|
||||
process.stdout.write(`QA suite report: ${result.reportPath}\n`);
|
||||
process.stdout.write(`QA suite evidence: ${result.evidencePath}\n`);
|
||||
if (!allowFailures && result.results.some((scenario) => scenario.status !== "pass")) {
|
||||
process.exitCode = 1;
|
||||
}
|
||||
return;
|
||||
}
|
||||
case "flow": {
|
||||
const result = runtimeResult.result;
|
||||
process.stdout.write(`QA suite watch: ${result.watchUrl}\n`);
|
||||
process.stdout.write(`QA suite report: ${result.reportPath}\n`);
|
||||
process.stdout.write(`QA suite evidence: ${result.evidencePath}\n`);
|
||||
process.stdout.write(`QA suite summary: ${result.summaryPath}\n`);
|
||||
const blockingScenarioCount = await readQaSuiteFailedOrSkippedScenarioCountFromFile(
|
||||
result.summaryPath,
|
||||
);
|
||||
if (!allowFailures && blockingScenarioCount > 0) {
|
||||
process.exitCode = 1;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -662,6 +662,7 @@ describe("qa cli registration", () => {
|
||||
|
||||
const options = requireQaSuiteOptions();
|
||||
expect(options.allowFailures).toBe(true);
|
||||
expect(options.providerMode).toBeUndefined();
|
||||
});
|
||||
|
||||
it("forwards --pack for suite runs", async () => {
|
||||
|
||||
@@ -46,7 +46,7 @@ async function runQaSelfCheck(opts: { repoRoot?: string; output?: string }) {
|
||||
await runtime.runQaLabSelfCheckCommand(opts);
|
||||
}
|
||||
|
||||
async function runQaSuite(opts: {
|
||||
async function runQaSuiteCliCommand(opts: {
|
||||
repoRoot?: string;
|
||||
outputDir?: string;
|
||||
transportId?: string;
|
||||
@@ -300,7 +300,7 @@ export function registerQaLabCli(program: Command) {
|
||||
.option("--output-dir <path>", "Suite artifact directory")
|
||||
.option("--runner <kind>", "Execution runner: host or multipass", "host")
|
||||
.option("--transport <id>", "QA transport id", "qa-channel")
|
||||
.option("--provider-mode <mode>", formatQaProviderModeHelp(), DEFAULT_QA_LIVE_PROVIDER_MODE)
|
||||
.option("--provider-mode <mode>", formatQaProviderModeHelp())
|
||||
.option("--model <ref>", "Primary provider/model ref")
|
||||
.option("--alt-model <ref>", "Alternate provider/model ref")
|
||||
.option(
|
||||
@@ -372,7 +372,7 @@ export function registerQaLabCli(program: Command) {
|
||||
runtimePair?: string;
|
||||
runtimeParityTier?: string[];
|
||||
}) => {
|
||||
await runQaSuite({
|
||||
await runQaSuiteCliCommand({
|
||||
repoRoot: opts.repoRoot,
|
||||
outputDir: opts.outputDir,
|
||||
transportId: opts.transport,
|
||||
|
||||
@@ -1,7 +1,37 @@
|
||||
// Qa Lab tests cover coverage report plugin behavior.
|
||||
import { describe, expect, it } from "vitest";
|
||||
import { buildQaCoverageInventory, renderQaCoverageMarkdownReport } from "./coverage-report.js";
|
||||
import {
|
||||
buildQaCoverageInventory,
|
||||
findQaScenarioMatches,
|
||||
renderQaCoverageMarkdownReport,
|
||||
renderQaScenarioMatchesMarkdownReport,
|
||||
} from "./coverage-report.js";
|
||||
import { readQaScenarioPack } from "./scenario-catalog.js";
|
||||
import { buildQaScorecardTaxonomyReport, parseQaScorecardTaxonomy } from "./scorecard-taxonomy.js";
|
||||
|
||||
const TEST_EXECUTABLE_CATEGORY_ID = "agent-runtime-and-provider-execution.agent-turn-execution";
|
||||
const TEST_TAXONOMY_REF = {
|
||||
sourcePath: "taxonomy.yaml",
|
||||
version: 1,
|
||||
processVersion: 3,
|
||||
snapshotDate: "2026-05-26",
|
||||
sourceRef: "origin/main@41eef4a7965",
|
||||
};
|
||||
|
||||
function testScorecardProfiles(categoryId = TEST_EXECUTABLE_CATEGORY_ID, profileId = "release") {
|
||||
return [
|
||||
{
|
||||
id: "smoke-ci",
|
||||
description: "Test smoke profile.",
|
||||
categoryIds: profileId === "smoke-ci" ? [categoryId] : [],
|
||||
},
|
||||
{
|
||||
id: "release",
|
||||
description: "Test release profile.",
|
||||
categoryIds: profileId === "release" ? [categoryId] : [],
|
||||
},
|
||||
];
|
||||
}
|
||||
|
||||
describe("qa coverage report", () => {
|
||||
it("groups scenario coverage metadata by theme and surface", () => {
|
||||
@@ -19,6 +49,43 @@ describe("qa coverage report", () => {
|
||||
"telegram",
|
||||
"whatsapp",
|
||||
]);
|
||||
expect(inventory.scorecardTaxonomy.taxonomyId).toBe("stable-lts-initial");
|
||||
expect(inventory.scorecardTaxonomy.profileCount).toBe(2);
|
||||
expect(inventory.scorecardTaxonomy.categoryCount).toBe(16);
|
||||
expect(inventory.scorecardTaxonomy.ltsIncludedCategoryCount).toBe(7);
|
||||
expect(inventory.scorecardTaxonomy.deferredCategoryCount).toBe(8);
|
||||
expect(inventory.scorecardTaxonomy.advisoryCategoryCount).toBe(1);
|
||||
expect(inventory.scorecardTaxonomy.releaseBlockingCategoryCount).toBe(7);
|
||||
expect(inventory.scorecardTaxonomy.mappedCoverageIdCount).toBeGreaterThan(0);
|
||||
expect(inventory.scorecardTaxonomy.mappedScenarioCount).toBeGreaterThan(0);
|
||||
expect(inventory.scorecardTaxonomy.unmappedCoverageIdCount).toBeGreaterThan(0);
|
||||
expect(inventory.scorecardTaxonomy.validationIssues).toStrictEqual([]);
|
||||
expect(
|
||||
inventory.scorecardTaxonomy.profiles
|
||||
.find((profile) => profile.id === "release")
|
||||
?.categoryIds.toSorted(),
|
||||
).toEqual([
|
||||
"agent-runtime-and-provider-execution.agent-turn-execution",
|
||||
"automation-cron-hooks-tasks-polling.cron-jobs",
|
||||
"browser-automation-and-exec-sandbox-tools.tool-invocation-and-execution",
|
||||
"browser-control-ui-and-webchat.browser-ui",
|
||||
"media-understanding-and-media-generation.media-generation",
|
||||
"media-understanding-and-media-generation.media-understanding",
|
||||
"openai-codex-provider-path.responses-and-tool-compatibility",
|
||||
"plugin-sdk-and-bundled-plugin-architecture.installing-and-running-plugins",
|
||||
"security-auth-pairing-and-secrets.approval-policy-and-tool-safeguards",
|
||||
"security-auth-pairing-and-secrets.credential-and-secret-hygiene",
|
||||
"session-memory-and-context-engine.diagnostics-maintenance-and-recovery",
|
||||
"session-memory-and-context-engine.memory",
|
||||
"session-memory-and-context-engine.token-management",
|
||||
"telemetry-diagnostics-and-observability.telemetry-export",
|
||||
]);
|
||||
expect(
|
||||
inventory.scorecardTaxonomy.categories.find(
|
||||
(category) =>
|
||||
category.id === "clawhub-and-external-plugin-distribution.compatibility-and-trust",
|
||||
)?.profiles,
|
||||
).toStrictEqual([]);
|
||||
expect(inventory.scenarioPacks.map((pack) => pack.id)).toEqual([
|
||||
"observability",
|
||||
"personal-agent",
|
||||
@@ -60,5 +127,441 @@ describe("qa coverage report", () => {
|
||||
"- telegram (telegram): canary: always-on, help-command: telegram-help-command, mention-gating: telegram-mention-gating; missing baseline: allowlist-block, top-level-reply-shape, restart-resume",
|
||||
);
|
||||
expect(report).toContain("thread-follow-up: slack-thread-follow-up");
|
||||
expect(report).toContain("## Scorecard Taxonomy");
|
||||
expect(report).toContain("- Mapping ID: stable-lts-initial");
|
||||
expect(report).toContain("- Maturity taxonomy: taxonomy.yaml");
|
||||
expect(report).toContain("- Maturity score snapshot: docs/maturity-scores.yaml");
|
||||
expect(report).toContain("- Categories: 16 (7 LTS-included, 8 deferred, 1 advisory)");
|
||||
expect(report).toContain("- Profiles: 2");
|
||||
expect(report).toContain(
|
||||
"- smoke-ci: 14 categories; agent-runtime-and-provider-execution.agent-turn-execution,",
|
||||
);
|
||||
expect(report).toContain(
|
||||
"- browser-automation-and-exec-sandbox-tools.tool-invocation-and-execution (browser-automation-and-exec-sandbox-tools / Tool Invocation and Execution; lts-included, release-blocking, mapped): profiles: release, smoke-ci; coverage: tools.apply-patch, tools.exec, tools.fs.read, tools.fs.write, tools.web-search;",
|
||||
);
|
||||
expect(report).toContain("### Unmapped Coverage IDs");
|
||||
expect(report).toContain("agents.subagents");
|
||||
});
|
||||
|
||||
it("renders Playwright matches as qa suite targets", () => {
|
||||
const matches = findQaScenarioMatches(readQaScenarioPack().scenarios, "chat-flow.e2e");
|
||||
const report = renderQaScenarioMatchesMarkdownReport({
|
||||
query: "chat-flow.e2e",
|
||||
matches,
|
||||
});
|
||||
|
||||
expect(report).toContain(
|
||||
"- Suite command: `pnpm openclaw qa suite --scenario control-ui-chat-flow-playwright`",
|
||||
);
|
||||
expect(report).toContain(" - execution: playwright ui/src/ui/e2e/chat-flow.e2e.test.ts");
|
||||
});
|
||||
|
||||
it("splits qa suite targets when matches mix execution kinds", () => {
|
||||
const matches = findQaScenarioMatches(readQaScenarioPack().scenarios, "control-ui");
|
||||
const report = renderQaScenarioMatchesMarkdownReport({
|
||||
query: "control-ui",
|
||||
matches,
|
||||
});
|
||||
|
||||
expect(report).toContain("- Suite commands:");
|
||||
expect(report).toContain(" - flow: `pnpm openclaw qa suite --scenario");
|
||||
expect(report).toContain(
|
||||
" - playwright: `pnpm openclaw qa suite --scenario control-ui-chat-flow-playwright`",
|
||||
);
|
||||
});
|
||||
|
||||
it("reports taxonomy mapping gaps as scorecard signals", () => {
|
||||
const taxonomy = parseQaScorecardTaxonomy({
|
||||
version: 1,
|
||||
id: "test-taxonomy",
|
||||
title: "Test taxonomy",
|
||||
taxonomy: TEST_TAXONOMY_REF,
|
||||
scoreSnapshotRef: "docs/maturity-scores.yaml",
|
||||
status: "initial",
|
||||
profiles: testScorecardProfiles(),
|
||||
categories: [
|
||||
{
|
||||
id: TEST_EXECUTABLE_CATEGORY_ID,
|
||||
taxonomySurfaceId: "agent-runtime-and-provider-execution",
|
||||
taxonomyCategoryName: "Agent Turn Execution",
|
||||
supportStatus: "lts-included",
|
||||
releaseBlocking: true,
|
||||
requirement: "Exercise a missing mapping.",
|
||||
evidenceRequired: "A real scenario mapping before promotion.",
|
||||
evidence: {
|
||||
profiles: ["release"],
|
||||
liveProofRequired: false,
|
||||
freshness: "target-ref",
|
||||
coverageIds: ["runtime.missing-coverage"],
|
||||
scenarioRefs: ["qa/scenarios/runtime/missing-scorecard-scenario.md"],
|
||||
docsRefs: ["docs/missing-scorecard-doc.md"],
|
||||
codeRefs: ["src/missing-scorecard-code.ts"],
|
||||
},
|
||||
},
|
||||
],
|
||||
});
|
||||
|
||||
const report = buildQaScorecardTaxonomyReport({
|
||||
taxonomy,
|
||||
repoRoot: process.cwd(),
|
||||
scenarios: readQaScenarioPack().scenarios,
|
||||
});
|
||||
|
||||
expect(report.categories[0]?.mappingStatus).toBe("partial");
|
||||
expect(report.validationIssues.map((issue) => issue.code)).toEqual([
|
||||
"coverage-id-not-found",
|
||||
"scenario-ref-not-found",
|
||||
"docs-ref-not-found",
|
||||
"code-ref-not-found",
|
||||
]);
|
||||
});
|
||||
|
||||
it("reports release-blocking categories missing release profile membership", () => {
|
||||
const taxonomy = parseQaScorecardTaxonomy({
|
||||
version: 1,
|
||||
id: "test-taxonomy",
|
||||
title: "Test taxonomy",
|
||||
taxonomy: TEST_TAXONOMY_REF,
|
||||
scoreSnapshotRef: "docs/maturity-scores.yaml",
|
||||
status: "initial",
|
||||
profiles: testScorecardProfiles(TEST_EXECUTABLE_CATEGORY_ID, "smoke-ci"),
|
||||
categories: [
|
||||
{
|
||||
id: TEST_EXECUTABLE_CATEGORY_ID,
|
||||
taxonomySurfaceId: "agent-runtime-and-provider-execution",
|
||||
taxonomyCategoryName: "Agent Turn Execution",
|
||||
supportStatus: "lts-included",
|
||||
releaseBlocking: true,
|
||||
requirement: "Release-blocking rows must be selected by the release profile.",
|
||||
evidenceRequired: "Release profile membership before promotion.",
|
||||
evidence: {
|
||||
profiles: ["smoke-ci"],
|
||||
liveProofRequired: false,
|
||||
freshness: "target-ref",
|
||||
coverageIds: ["channels.dm"],
|
||||
scenarioRefs: ["qa/scenarios/channels/dm-chat-baseline.md"],
|
||||
docsRefs: ["docs/concepts/qa-e2e-automation.md"],
|
||||
codeRefs: ["extensions/qa-lab/src/suite.ts"],
|
||||
},
|
||||
},
|
||||
],
|
||||
});
|
||||
|
||||
const report = buildQaScorecardTaxonomyReport({
|
||||
taxonomy,
|
||||
repoRoot: process.cwd(),
|
||||
scenarios: readQaScenarioPack().scenarios,
|
||||
});
|
||||
|
||||
expect(report.validationIssues.map((issue) => issue.code)).toEqual([
|
||||
"release-blocking-category-missing-release-profile",
|
||||
]);
|
||||
});
|
||||
|
||||
it("reports advisory categories that are accidentally assigned to a runnable profile", () => {
|
||||
const taxonomy = parseQaScorecardTaxonomy({
|
||||
version: 1,
|
||||
id: "test-taxonomy",
|
||||
title: "Test taxonomy",
|
||||
taxonomy: TEST_TAXONOMY_REF,
|
||||
scoreSnapshotRef: "docs/maturity-scores.yaml",
|
||||
status: "initial",
|
||||
profiles: testScorecardProfiles(
|
||||
"clawhub-and-external-plugin-distribution.compatibility-and-trust",
|
||||
"smoke-ci",
|
||||
),
|
||||
categories: [
|
||||
{
|
||||
id: "clawhub-and-external-plugin-distribution.compatibility-and-trust",
|
||||
taxonomySurfaceId: "clawhub-and-external-plugin-distribution",
|
||||
taxonomyCategoryName: "Compatibility and Trust",
|
||||
supportStatus: "advisory",
|
||||
releaseBlocking: false,
|
||||
requirement: "Keep advisory compatibility out of runnable profiles.",
|
||||
evidenceRequired: "Advisory report metadata only.",
|
||||
evidence: {
|
||||
profiles: [],
|
||||
liveProofRequired: false,
|
||||
freshness: "latest-advisory-run",
|
||||
coverageIds: [],
|
||||
scenarioRefs: [],
|
||||
docsRefs: ["docs/plugins/architecture.md"],
|
||||
codeRefs: [],
|
||||
},
|
||||
},
|
||||
],
|
||||
});
|
||||
|
||||
const report = buildQaScorecardTaxonomyReport({
|
||||
taxonomy,
|
||||
repoRoot: process.cwd(),
|
||||
scenarios: readQaScenarioPack().scenarios,
|
||||
});
|
||||
|
||||
expect(report.validationIssues.map((issue) => issue.code)).toEqual([
|
||||
"profile-membership-missing-category-profile",
|
||||
"advisory-category-has-profile-membership",
|
||||
]);
|
||||
});
|
||||
|
||||
it("reports non-advisory categories with no runnable profile membership", () => {
|
||||
const taxonomy = parseQaScorecardTaxonomy({
|
||||
version: 1,
|
||||
id: "test-taxonomy",
|
||||
title: "Test taxonomy",
|
||||
taxonomy: TEST_TAXONOMY_REF,
|
||||
scoreSnapshotRef: "docs/maturity-scores.yaml",
|
||||
status: "initial",
|
||||
profiles: testScorecardProfiles(TEST_EXECUTABLE_CATEGORY_ID, "none"),
|
||||
categories: [
|
||||
{
|
||||
id: TEST_EXECUTABLE_CATEGORY_ID,
|
||||
taxonomySurfaceId: "agent-runtime-and-provider-execution",
|
||||
taxonomyCategoryName: "Agent Turn Execution",
|
||||
supportStatus: "deferred",
|
||||
releaseBlocking: false,
|
||||
requirement: "Non-advisory rows must stay visible to runnable profiles.",
|
||||
evidenceRequired: "At least one smoke-ci or release membership before promotion.",
|
||||
evidence: {
|
||||
profiles: [],
|
||||
liveProofRequired: false,
|
||||
freshness: "target-ref",
|
||||
coverageIds: ["channels.dm"],
|
||||
scenarioRefs: ["qa/scenarios/channels/dm-chat-baseline.md"],
|
||||
docsRefs: ["docs/concepts/qa-e2e-automation.md"],
|
||||
codeRefs: ["extensions/qa-lab/src/suite.ts"],
|
||||
},
|
||||
},
|
||||
],
|
||||
});
|
||||
|
||||
const report = buildQaScorecardTaxonomyReport({
|
||||
taxonomy,
|
||||
repoRoot: process.cwd(),
|
||||
scenarios: readQaScenarioPack().scenarios,
|
||||
});
|
||||
|
||||
expect(report.validationIssues.map((issue) => issue.code)).toEqual([
|
||||
"non-advisory-category-missing-profile-membership",
|
||||
]);
|
||||
});
|
||||
|
||||
it("reports executable category refs missing from taxonomy.yaml", () => {
|
||||
const taxonomy = parseQaScorecardTaxonomy({
|
||||
version: 1,
|
||||
id: "test-taxonomy",
|
||||
title: "Test taxonomy",
|
||||
taxonomy: TEST_TAXONOMY_REF,
|
||||
scoreSnapshotRef: "docs/maturity-scores.yaml",
|
||||
status: "initial",
|
||||
profiles: testScorecardProfiles(TEST_EXECUTABLE_CATEGORY_ID, "release"),
|
||||
categories: [
|
||||
{
|
||||
id: TEST_EXECUTABLE_CATEGORY_ID,
|
||||
taxonomySurfaceId: "agent-runtime-and-provider-execution",
|
||||
taxonomyCategoryName: "Missing Taxonomy Category",
|
||||
supportStatus: "lts-included",
|
||||
releaseBlocking: true,
|
||||
requirement: "Executable refs must resolve against taxonomy.yaml.",
|
||||
evidenceRequired: "A valid taxonomy surface/category ref.",
|
||||
evidence: {
|
||||
profiles: ["release"],
|
||||
liveProofRequired: false,
|
||||
freshness: "target-ref",
|
||||
coverageIds: ["channels.dm"],
|
||||
scenarioRefs: ["qa/scenarios/channels/dm-chat-baseline.md"],
|
||||
docsRefs: ["docs/concepts/qa-e2e-automation.md"],
|
||||
codeRefs: ["extensions/qa-lab/src/suite.ts"],
|
||||
},
|
||||
},
|
||||
],
|
||||
});
|
||||
|
||||
const report = buildQaScorecardTaxonomyReport({
|
||||
taxonomy,
|
||||
repoRoot: process.cwd(),
|
||||
scenarios: readQaScenarioPack().scenarios,
|
||||
});
|
||||
|
||||
expect(report.validationIssues.map((issue) => issue.code)).toEqual([
|
||||
"taxonomy-category-ref-not-found",
|
||||
]);
|
||||
});
|
||||
|
||||
it("reports profile membership refs missing from executable categories", () => {
|
||||
const taxonomy = parseQaScorecardTaxonomy({
|
||||
version: 1,
|
||||
id: "test-taxonomy",
|
||||
title: "Test taxonomy",
|
||||
taxonomy: TEST_TAXONOMY_REF,
|
||||
scoreSnapshotRef: "docs/maturity-scores.yaml",
|
||||
status: "initial",
|
||||
profiles: [
|
||||
{
|
||||
id: "smoke-ci",
|
||||
description: "Test smoke profile.",
|
||||
categoryIds: ["missing.category"],
|
||||
},
|
||||
{
|
||||
id: "release",
|
||||
description: "Test release profile.",
|
||||
categoryIds: [],
|
||||
},
|
||||
],
|
||||
categories: [
|
||||
{
|
||||
id: TEST_EXECUTABLE_CATEGORY_ID,
|
||||
taxonomySurfaceId: "agent-runtime-and-provider-execution",
|
||||
taxonomyCategoryName: "Agent Turn Execution",
|
||||
supportStatus: "advisory",
|
||||
releaseBlocking: false,
|
||||
requirement: "Profile selectors must reference executable category IDs.",
|
||||
evidenceRequired: "Invalid selector refs should be reported.",
|
||||
evidence: {
|
||||
profiles: [],
|
||||
liveProofRequired: false,
|
||||
freshness: "latest-advisory-run",
|
||||
coverageIds: [],
|
||||
scenarioRefs: [],
|
||||
docsRefs: ["docs/concepts/qa-e2e-automation.md"],
|
||||
codeRefs: ["extensions/qa-lab/src/suite.ts"],
|
||||
},
|
||||
},
|
||||
],
|
||||
});
|
||||
|
||||
const report = buildQaScorecardTaxonomyReport({
|
||||
taxonomy,
|
||||
repoRoot: process.cwd(),
|
||||
scenarios: readQaScenarioPack().scenarios,
|
||||
});
|
||||
|
||||
expect(report.validationIssues.map((issue) => issue.code)).toEqual([
|
||||
"profile-category-ref-not-found",
|
||||
]);
|
||||
});
|
||||
|
||||
it("reports category profile refs missing from top-level mapping profiles", () => {
|
||||
const taxonomy = parseQaScorecardTaxonomy({
|
||||
version: 1,
|
||||
id: "test-taxonomy",
|
||||
title: "Test taxonomy",
|
||||
taxonomy: TEST_TAXONOMY_REF,
|
||||
scoreSnapshotRef: "docs/maturity-scores.yaml",
|
||||
status: "initial",
|
||||
profiles: [...testScorecardProfiles(TEST_EXECUTABLE_CATEGORY_ID, "release")],
|
||||
categories: [
|
||||
{
|
||||
id: TEST_EXECUTABLE_CATEGORY_ID,
|
||||
taxonomySurfaceId: "agent-runtime-and-provider-execution",
|
||||
taxonomyCategoryName: "Agent Turn Execution",
|
||||
supportStatus: "lts-included",
|
||||
releaseBlocking: true,
|
||||
requirement: "Category profile refs must resolve to top-level mapping profiles.",
|
||||
evidenceRequired: "Unknown profile refs should be reported.",
|
||||
evidence: {
|
||||
profiles: ["release", "nightly"],
|
||||
liveProofRequired: false,
|
||||
freshness: "target-ref",
|
||||
coverageIds: ["channels.dm"],
|
||||
scenarioRefs: ["qa/scenarios/channels/dm-chat-baseline.md"],
|
||||
docsRefs: ["docs/concepts/qa-e2e-automation.md"],
|
||||
codeRefs: ["extensions/qa-lab/src/suite.ts"],
|
||||
},
|
||||
},
|
||||
],
|
||||
});
|
||||
|
||||
const report = buildQaScorecardTaxonomyReport({
|
||||
taxonomy,
|
||||
repoRoot: process.cwd(),
|
||||
scenarios: readQaScenarioPack().scenarios,
|
||||
});
|
||||
|
||||
expect(report.validationIssues.map((issue) => issue.code)).toEqual(["profile-ref-not-found"]);
|
||||
});
|
||||
|
||||
it("counts declared custom profiles as runnable category membership", () => {
|
||||
const taxonomy = parseQaScorecardTaxonomy({
|
||||
version: 1,
|
||||
id: "test-taxonomy",
|
||||
title: "Test taxonomy",
|
||||
taxonomy: TEST_TAXONOMY_REF,
|
||||
scoreSnapshotRef: "docs/maturity-scores.yaml",
|
||||
status: "initial",
|
||||
profiles: [
|
||||
...testScorecardProfiles(TEST_EXECUTABLE_CATEGORY_ID, "none"),
|
||||
{
|
||||
id: "nightly",
|
||||
description: "Nightly mapped profile.",
|
||||
categoryIds: [TEST_EXECUTABLE_CATEGORY_ID],
|
||||
},
|
||||
],
|
||||
categories: [
|
||||
{
|
||||
id: TEST_EXECUTABLE_CATEGORY_ID,
|
||||
taxonomySurfaceId: "agent-runtime-and-provider-execution",
|
||||
taxonomyCategoryName: "Agent Turn Execution",
|
||||
supportStatus: "deferred",
|
||||
releaseBlocking: false,
|
||||
requirement: "Declared profile names can satisfy runnable coverage.",
|
||||
evidenceRequired: "Profile names come from taxonomy-mappings.yaml.",
|
||||
evidence: {
|
||||
profiles: ["nightly"],
|
||||
liveProofRequired: false,
|
||||
freshness: "target-ref",
|
||||
coverageIds: ["channels.dm"],
|
||||
scenarioRefs: ["qa/scenarios/channels/dm-chat-baseline.md"],
|
||||
docsRefs: ["docs/concepts/qa-e2e-automation.md"],
|
||||
codeRefs: ["extensions/qa-lab/src/suite.ts"],
|
||||
},
|
||||
},
|
||||
],
|
||||
});
|
||||
|
||||
const report = buildQaScorecardTaxonomyReport({
|
||||
taxonomy,
|
||||
repoRoot: process.cwd(),
|
||||
scenarios: readQaScenarioPack().scenarios,
|
||||
});
|
||||
|
||||
expect(report.validationIssues).toStrictEqual([]);
|
||||
});
|
||||
|
||||
it("rejects taxonomy refs outside the repository", () => {
|
||||
expect(() =>
|
||||
parseQaScorecardTaxonomy({
|
||||
version: 1,
|
||||
id: "bad-taxonomy",
|
||||
title: "Bad taxonomy",
|
||||
taxonomy: {
|
||||
...TEST_TAXONOMY_REF,
|
||||
sourcePath: "../rfcs/rfcs/0007-e2e-qa-lab-scorecard-consolidation.md",
|
||||
},
|
||||
scoreSnapshotRef: "docs/maturity-scores.yaml",
|
||||
status: "initial",
|
||||
profiles: testScorecardProfiles(TEST_EXECUTABLE_CATEGORY_ID, "smoke-ci"),
|
||||
categories: [
|
||||
{
|
||||
id: TEST_EXECUTABLE_CATEGORY_ID,
|
||||
taxonomySurfaceId: "agent-runtime-and-provider-execution",
|
||||
taxonomyCategoryName: "Agent Turn Execution",
|
||||
supportStatus: "deferred",
|
||||
releaseBlocking: false,
|
||||
requirement: "Reject escaped refs.",
|
||||
evidenceRequired: "Parser rejects refs outside the repository.",
|
||||
evidence: {
|
||||
profiles: ["smoke-ci"],
|
||||
liveProofRequired: false,
|
||||
freshness: "target-ref",
|
||||
coverageIds: ["runtime.delivery"],
|
||||
scenarioRefs: ["qa/scenarios/channels/dm-chat-baseline.md"],
|
||||
docsRefs: ["/tmp/outside-openclaw.md"],
|
||||
codeRefs: ["src/agents/../agents/agent-tools.ts"],
|
||||
},
|
||||
},
|
||||
],
|
||||
}),
|
||||
).toThrow("repo refs must not be absolute or contain parent-directory segments");
|
||||
});
|
||||
});
|
||||
|
||||
@@ -5,6 +5,10 @@ import {
|
||||
type LiveTransportCoverageLaneSummary,
|
||||
} from "./live-transports/shared/live-transport-scenarios.js";
|
||||
import { QA_SCENARIO_PACKS, type QaSeedScenarioWithSource } from "./scenario-catalog.js";
|
||||
import {
|
||||
readQaScorecardTaxonomyReport,
|
||||
type QaScorecardTaxonomyReport,
|
||||
} from "./scorecard-taxonomy.js";
|
||||
|
||||
type QaCoverageScenarioSummary = {
|
||||
id: string;
|
||||
@@ -19,6 +23,8 @@ type QaScenarioSearchMatch = QaCoverageScenarioSummary & {
|
||||
coverageIds: string[];
|
||||
docsRefs: string[];
|
||||
codeRefs: string[];
|
||||
executionKind: QaSeedScenarioWithSource["execution"]["kind"];
|
||||
executionPath?: string;
|
||||
runtimeParityTier?: string;
|
||||
requiredProviderMode?: string;
|
||||
requiredProvider?: string;
|
||||
@@ -56,6 +62,7 @@ type QaCoverageInventory = {
|
||||
bySurface: Record<string, QaCoverageFeatureSummary[]>;
|
||||
scenarioPacks: QaCoverageScenarioPackSummary[];
|
||||
liveTransportLanes: LiveTransportCoverageLaneSummary[];
|
||||
scorecardTaxonomy: QaScorecardTaxonomyReport;
|
||||
};
|
||||
|
||||
function scenarioTheme(sourcePath: string) {
|
||||
@@ -133,6 +140,8 @@ function summarizeScenarioSearchMatch(scenario: QaSeedScenarioWithSource): QaSce
|
||||
].toSorted((left, right) => left.localeCompare(right)),
|
||||
docsRefs: [...(scenario.docsRefs ?? [])],
|
||||
codeRefs: [...(scenario.codeRefs ?? [])],
|
||||
executionKind: scenario.execution.kind,
|
||||
...(scenario.execution.kind !== "flow" ? { executionPath: scenario.execution.path } : {}),
|
||||
runtimeParityTier: scenario.runtimeParityTier,
|
||||
requiredProviderMode: stringifyConfigValue(config.requiredProviderMode),
|
||||
requiredProvider: stringifyConfigValue(config.requiredProvider),
|
||||
@@ -265,6 +274,7 @@ export function buildQaCoverageInventory(
|
||||
bySurface,
|
||||
scenarioPacks: buildScenarioPackSummaries(scenarios),
|
||||
liveTransportLanes: buildLiveTransportCoverageLaneSummaries(),
|
||||
scorecardTaxonomy: readQaScorecardTaxonomyReport(scenarios),
|
||||
};
|
||||
}
|
||||
|
||||
@@ -310,6 +320,64 @@ function pushScenarioPackLines(lines: string[], packs: readonly QaCoverageScenar
|
||||
}
|
||||
}
|
||||
|
||||
function pushScorecardTaxonomyLines(lines: string[], report: QaScorecardTaxonomyReport) {
|
||||
lines.push("## Scorecard Taxonomy", "");
|
||||
lines.push(`- Mapping: ${report.taxonomyPath ?? "missing"}`);
|
||||
lines.push(`- Mapping ID: ${report.taxonomyId ?? "missing"}`);
|
||||
lines.push(`- Maturity taxonomy: ${report.taxonomy?.sourcePath ?? "missing"}`);
|
||||
if (report.scoreSnapshotRef) {
|
||||
lines.push(`- Maturity score snapshot: ${report.scoreSnapshotRef}`);
|
||||
}
|
||||
lines.push(
|
||||
`- Categories: ${report.categoryCount} (${report.ltsIncludedCategoryCount} LTS-included, ${report.deferredCategoryCount} deferred, ${report.advisoryCategoryCount} advisory)`,
|
||||
);
|
||||
lines.push(`- Profiles: ${report.profileCount}`);
|
||||
lines.push(`- Release-blocking categories: ${report.releaseBlockingCategoryCount}`);
|
||||
lines.push(`- Mapped coverage IDs: ${report.mappedCoverageIdCount}`);
|
||||
lines.push(`- Mapped scenarios: ${report.mappedScenarioCount}`);
|
||||
lines.push(`- Unmapped coverage IDs: ${report.unmappedCoverageIdCount}`);
|
||||
lines.push(`- Validation warnings: ${report.validationIssueCount}`, "");
|
||||
|
||||
if (report.profiles.length > 0) {
|
||||
lines.push("### Profiles", "");
|
||||
for (const profile of report.profiles) {
|
||||
const categories = profile.categoryIds.length > 0 ? profile.categoryIds.join(", ") : "none";
|
||||
lines.push(`- ${profile.id}: ${profile.categoryIds.length} categories; ${categories}`);
|
||||
}
|
||||
lines.push("");
|
||||
}
|
||||
|
||||
if (report.categories.length > 0) {
|
||||
lines.push("### Category Mapping", "");
|
||||
for (const category of report.categories) {
|
||||
const blocking = category.releaseBlocking ? "release-blocking" : "non-blocking";
|
||||
const coverage = category.coverageIds.length > 0 ? category.coverageIds.join(", ") : "none";
|
||||
const scenarios =
|
||||
category.scenarioRefs.length > 0 ? category.scenarioRefs.join(", ") : "none";
|
||||
const profiles = category.profiles.length > 0 ? category.profiles.join(", ") : "none";
|
||||
lines.push(
|
||||
`- ${category.id} (${category.taxonomySurfaceId} / ${category.taxonomyCategoryName}; ${category.supportStatus}, ${blocking}, ${category.mappingStatus}): profiles: ${profiles}; coverage: ${coverage}; scenarios: ${scenarios}`,
|
||||
);
|
||||
}
|
||||
lines.push("");
|
||||
}
|
||||
|
||||
if (report.validationIssues.length > 0) {
|
||||
lines.push("### Validation Warnings", "");
|
||||
for (const issue of report.validationIssues) {
|
||||
const category = issue.categoryId ? `${issue.categoryId}: ` : "";
|
||||
lines.push(`- ${issue.code}: ${category}${issue.message}`);
|
||||
}
|
||||
lines.push("");
|
||||
}
|
||||
|
||||
if (report.unmappedCoverageIds.length > 0) {
|
||||
lines.push("### Unmapped Coverage IDs", "");
|
||||
lines.push(report.unmappedCoverageIds.join(", "));
|
||||
lines.push("");
|
||||
}
|
||||
}
|
||||
|
||||
export function renderQaCoverageMarkdownReport(inventory: QaCoverageInventory): string {
|
||||
const lines: string[] = [
|
||||
"# QA Coverage Inventory",
|
||||
@@ -349,6 +417,8 @@ export function renderQaCoverageMarkdownReport(inventory: QaCoverageInventory):
|
||||
lines.push("");
|
||||
}
|
||||
|
||||
pushScorecardTaxonomyLines(lines, inventory.scorecardTaxonomy);
|
||||
|
||||
if (inventory.overlappingCoverage.length > 0) {
|
||||
lines.push("## Overlap", "");
|
||||
pushFeatureLines(lines, inventory.overlappingCoverage);
|
||||
@@ -378,11 +448,31 @@ function formatOptionalScenarioMetadata(match: QaScenarioSearchMatch) {
|
||||
return metadata.length > 0 ? metadata.join("; ") : "none";
|
||||
}
|
||||
|
||||
function formatSuiteCommand(matches: readonly QaScenarioSearchMatch[]) {
|
||||
const scenarioArgs = matches.map((match) => `--scenario ${match.id}`).join(" ");
|
||||
return `pnpm openclaw qa suite ${scenarioArgs}`;
|
||||
}
|
||||
|
||||
function scenarioMatchCommandGroups(matches: readonly QaScenarioSearchMatch[]) {
|
||||
const groups = new Map<QaScenarioSearchMatch["executionKind"], QaScenarioSearchMatch[]>();
|
||||
for (const match of matches) {
|
||||
const existing = groups.get(match.executionKind) ?? [];
|
||||
existing.push(match);
|
||||
groups.set(match.executionKind, existing);
|
||||
}
|
||||
const executionOrder: QaScenarioSearchMatch["executionKind"][] = ["flow", "vitest", "playwright"];
|
||||
return executionOrder
|
||||
.map((executionKind) => ({
|
||||
executionKind,
|
||||
matches: groups.get(executionKind) ?? [],
|
||||
}))
|
||||
.filter((group) => group.matches.length > 0);
|
||||
}
|
||||
|
||||
export function renderQaScenarioMatchesMarkdownReport(params: {
|
||||
query: string;
|
||||
matches: readonly QaScenarioSearchMatch[];
|
||||
}) {
|
||||
const scenarioArgs = params.matches.map((match) => `--scenario ${match.id}`).join(" ");
|
||||
const lines = [
|
||||
"# QA Scenario Matches",
|
||||
"",
|
||||
@@ -390,8 +480,14 @@ export function renderQaScenarioMatchesMarkdownReport(params: {
|
||||
`- Matches: ${params.matches.length}`,
|
||||
];
|
||||
|
||||
if (scenarioArgs) {
|
||||
lines.push(`- Suite command: \`pnpm openclaw qa suite ${scenarioArgs}\``);
|
||||
const commandGroups = scenarioMatchCommandGroups(params.matches);
|
||||
if (commandGroups.length === 1) {
|
||||
lines.push(`- Suite command: \`${formatSuiteCommand(commandGroups[0].matches)}\``);
|
||||
} else if (commandGroups.length > 1) {
|
||||
lines.push("- Suite commands:");
|
||||
for (const group of commandGroups) {
|
||||
lines.push(` - ${group.executionKind}: \`${formatSuiteCommand(group.matches)}\``);
|
||||
}
|
||||
}
|
||||
lines.push("");
|
||||
|
||||
@@ -404,6 +500,11 @@ export function renderQaScenarioMatchesMarkdownReport(params: {
|
||||
lines.push(`- ${match.id}: ${match.title}`);
|
||||
lines.push(` - source: ${match.sourcePath}`);
|
||||
lines.push(` - surface: ${match.surfaces.join(", ")}`);
|
||||
lines.push(
|
||||
match.executionKind === "flow"
|
||||
? " - execution: flow (qa-flow block)"
|
||||
: ` - execution: ${match.executionKind} ${match.executionPath ?? "missing"}`,
|
||||
);
|
||||
lines.push(` - coverage: ${match.coverageIds.join(", ") || "none"}`);
|
||||
lines.push(` - live requirements: ${formatOptionalScenarioMetadata(match)}`);
|
||||
if (match.codeRefs.length > 0) {
|
||||
|
||||
36
extensions/qa-lab/src/errors.ts
Normal file
36
extensions/qa-lab/src/errors.ts
Normal file
@@ -0,0 +1,36 @@
|
||||
// Qa Lab plugin module defines shared suite errors.
|
||||
export type QaSuiteArtifactErrorCode =
|
||||
| "evidence_missing"
|
||||
| "report_missing"
|
||||
| "summary_missing"
|
||||
| "summary_read_failed"
|
||||
| "summary_parse_failed"
|
||||
| "summary_failure_count_missing"
|
||||
| "summary_blocking_count_missing";
|
||||
|
||||
export class QaSuiteArtifactError extends Error {
|
||||
readonly code: QaSuiteArtifactErrorCode;
|
||||
|
||||
constructor(code: QaSuiteArtifactErrorCode, message: string, options?: { cause?: unknown }) {
|
||||
super(message, options);
|
||||
this.name = "QaSuiteArtifactError";
|
||||
this.code = code;
|
||||
}
|
||||
}
|
||||
|
||||
export type QaSuiteInfraErrorCode =
|
||||
| "agent_wait_failed"
|
||||
| "gateway_startup_unhealthy"
|
||||
| "gateway_ready_timeout"
|
||||
| "qa_cli_timeout"
|
||||
| "transport_ready_timeout";
|
||||
|
||||
export class QaSuiteInfraError extends Error {
|
||||
readonly code: QaSuiteInfraErrorCode;
|
||||
|
||||
constructor(code: QaSuiteInfraErrorCode, message: string, options?: { cause?: unknown }) {
|
||||
super(message, options);
|
||||
this.name = "QaSuiteInfraError";
|
||||
this.code = code;
|
||||
}
|
||||
}
|
||||
599
extensions/qa-lab/src/evidence-summary.test.ts
Normal file
599
extensions/qa-lab/src/evidence-summary.test.ts
Normal file
@@ -0,0 +1,599 @@
|
||||
// Qa Lab tests cover QA evidence summary behavior.
|
||||
import { describe, expect, it } from "vitest";
|
||||
import {
|
||||
QA_EVIDENCE_SUMMARY_KIND,
|
||||
QA_EVIDENCE_FILENAME,
|
||||
QA_EVIDENCE_SUMMARY_SCHEMA_VERSION,
|
||||
buildLiveTransportEvidenceSummary,
|
||||
buildPlaywrightEvidenceSummary,
|
||||
buildQaSuiteEvidenceSummary,
|
||||
buildVitestEvidenceSummary,
|
||||
validateQaEvidenceSummaryJson,
|
||||
} from "./evidence-summary.js";
|
||||
|
||||
describe("evidence summary", () => {
|
||||
it("builds taxonomy-mapped QA suite evidence entries from catalog metadata", () => {
|
||||
const evidence = buildQaSuiteEvidenceSummary({
|
||||
artifactPaths: [
|
||||
{ kind: "summary", path: "qa-suite-summary.json" },
|
||||
{ kind: "report", path: "qa-suite-report.md" },
|
||||
],
|
||||
scenarioDefinitions: [
|
||||
{
|
||||
id: "dm-chat-baseline",
|
||||
title: "DM baseline conversation",
|
||||
sourcePath: "qa/scenarios/channels/dm-chat-baseline.md",
|
||||
surface: "dm",
|
||||
coverage: {
|
||||
primary: ["channels.dm"],
|
||||
secondary: ["channels.qa-channel"],
|
||||
},
|
||||
runtimeParityTier: "standard",
|
||||
docsRefs: ["docs/channels/qa-channel.md"],
|
||||
codeRefs: ["extensions/qa-channel/src/gateway.ts"],
|
||||
},
|
||||
],
|
||||
channelId: "qa-channel",
|
||||
env: {
|
||||
OPENCLAW_QA_CHANNEL_DRIVER: "local-shim",
|
||||
OPENCLAW_QA_REF: "abc123",
|
||||
} as NodeJS.ProcessEnv,
|
||||
generatedAt: "2026-06-07T12:00:00.000Z",
|
||||
primaryModel: "mock-openai/gpt-5.5",
|
||||
providerMode: "mock-openai",
|
||||
scenarioResults: [{ name: "DM baseline conversation", status: "pass" }],
|
||||
});
|
||||
|
||||
expect(validateQaEvidenceSummaryJson(evidence)).toEqual(evidence);
|
||||
expect(evidence.kind).toBe(QA_EVIDENCE_SUMMARY_KIND);
|
||||
expect(evidence.schemaVersion).toBe(QA_EVIDENCE_SUMMARY_SCHEMA_VERSION);
|
||||
expect(evidence.entries).toHaveLength(1);
|
||||
expect(evidence.entries[0]).toMatchObject({
|
||||
test: {
|
||||
kind: "qa-scenario",
|
||||
id: "dm-chat-baseline",
|
||||
title: "DM baseline conversation",
|
||||
source: {
|
||||
path: "qa/scenarios/channels/dm-chat-baseline.md",
|
||||
},
|
||||
},
|
||||
mapping: {
|
||||
profile: "smoke-ci",
|
||||
coverage: [
|
||||
{
|
||||
id: "channels.dm",
|
||||
role: "primary",
|
||||
surfaceIds: ["dm"],
|
||||
categoryIds: ["channels.dm"],
|
||||
},
|
||||
{
|
||||
id: "channels.qa-channel",
|
||||
role: "secondary",
|
||||
surfaceIds: ["dm"],
|
||||
categoryIds: [],
|
||||
},
|
||||
],
|
||||
refs: [
|
||||
{
|
||||
kind: "docs",
|
||||
path: "docs/channels/qa-channel.md",
|
||||
},
|
||||
{
|
||||
kind: "code",
|
||||
path: "extensions/qa-channel/src/gateway.ts",
|
||||
},
|
||||
],
|
||||
runtimeParityTier: "standard",
|
||||
},
|
||||
execution: {
|
||||
runner: "host",
|
||||
provider: {
|
||||
id: "openai",
|
||||
live: false,
|
||||
model: {
|
||||
name: "gpt-5.5",
|
||||
ref: "mock-openai/gpt-5.5",
|
||||
},
|
||||
fixture: "mock-openai",
|
||||
},
|
||||
channel: {
|
||||
id: "qa-channel",
|
||||
live: false,
|
||||
driver: "local-shim",
|
||||
},
|
||||
packageSource: {
|
||||
kind: "source-checkout",
|
||||
},
|
||||
environment: {
|
||||
ref: "abc123",
|
||||
os: process.platform,
|
||||
nodeVersion: process.version,
|
||||
},
|
||||
artifacts: [
|
||||
{
|
||||
kind: "summary",
|
||||
path: "qa-suite-summary.json",
|
||||
source: "qa-suite",
|
||||
},
|
||||
{
|
||||
kind: "report",
|
||||
path: "qa-suite-report.md",
|
||||
source: "qa-suite",
|
||||
},
|
||||
],
|
||||
},
|
||||
result: {
|
||||
status: "pass",
|
||||
},
|
||||
});
|
||||
});
|
||||
|
||||
it("builds Telegram live transport evidence entries", () => {
|
||||
const evidence = buildLiveTransportEvidenceSummary({
|
||||
artifactPaths: [
|
||||
{ kind: "summary", path: QA_EVIDENCE_FILENAME },
|
||||
{ kind: "report", path: "telegram-qa-report.md" },
|
||||
{ kind: "transport-observations", path: "telegram-qa-observed-messages.json" },
|
||||
],
|
||||
env: {
|
||||
OPENCLAW_QA_RUNNER: "crabbox",
|
||||
} as NodeJS.ProcessEnv,
|
||||
generatedAt: "2026-06-07T12:05:00.000Z",
|
||||
primaryModel: "openai/gpt-5.5",
|
||||
providerMode: "live-frontier",
|
||||
checks: [
|
||||
{
|
||||
id: "telegram-canary",
|
||||
standardId: "canary",
|
||||
title: "Telegram canary",
|
||||
status: "fail",
|
||||
details: "timed out waiting for SUT reply",
|
||||
rttMs: 4321,
|
||||
},
|
||||
],
|
||||
transportId: "telegram",
|
||||
});
|
||||
|
||||
expect(validateQaEvidenceSummaryJson(evidence)).toEqual(evidence);
|
||||
expect(evidence.entries).toEqual([
|
||||
expect.objectContaining({
|
||||
test: {
|
||||
kind: "live-transport-check",
|
||||
id: "telegram-canary",
|
||||
title: "Telegram canary",
|
||||
},
|
||||
mapping: {
|
||||
profile: "release",
|
||||
coverage: [
|
||||
{
|
||||
id: "channels.telegram.live",
|
||||
role: "live-transport",
|
||||
surfaceIds: ["channels.telegram"],
|
||||
categoryIds: ["channels.telegram.live"],
|
||||
},
|
||||
{
|
||||
id: "channels.telegram.canary",
|
||||
role: "live-transport-standard",
|
||||
surfaceIds: ["channels.telegram"],
|
||||
categoryIds: ["channels.telegram.live"],
|
||||
},
|
||||
],
|
||||
},
|
||||
execution: expect.objectContaining({
|
||||
runner: "crabbox",
|
||||
provider: {
|
||||
id: "openai",
|
||||
live: true,
|
||||
model: {
|
||||
name: "gpt-5.5",
|
||||
ref: "openai/gpt-5.5",
|
||||
},
|
||||
auth: "live-frontier",
|
||||
},
|
||||
channel: {
|
||||
id: "telegram",
|
||||
live: true,
|
||||
driver: "native",
|
||||
},
|
||||
artifacts: [
|
||||
{
|
||||
kind: "summary",
|
||||
path: QA_EVIDENCE_FILENAME,
|
||||
source: "telegram-live-transport",
|
||||
},
|
||||
{
|
||||
kind: "report",
|
||||
path: "telegram-qa-report.md",
|
||||
source: "telegram-live-transport",
|
||||
},
|
||||
{
|
||||
kind: "transport-observations",
|
||||
path: "telegram-qa-observed-messages.json",
|
||||
source: "telegram-live-transport",
|
||||
},
|
||||
],
|
||||
}),
|
||||
result: {
|
||||
status: "fail",
|
||||
failure: {
|
||||
reason: "timed out waiting for SUT reply",
|
||||
},
|
||||
timing: {
|
||||
rttMs: 4321,
|
||||
},
|
||||
},
|
||||
}),
|
||||
]);
|
||||
});
|
||||
|
||||
it("builds Vitest runner evidence entries", () => {
|
||||
const evidence = buildVitestEvidenceSummary({
|
||||
artifactPaths: [
|
||||
{ kind: "runner-result", path: "vitest-results/runtime-boundary.vitest.json" },
|
||||
],
|
||||
env: {
|
||||
OPENCLAW_QA_REF: "abc123",
|
||||
} as NodeJS.ProcessEnv,
|
||||
generatedAt: "2026-06-07T12:06:00.000Z",
|
||||
primaryModel: "mock-openai/gpt-5.5",
|
||||
providerMode: "mock-openai",
|
||||
targets: [
|
||||
{
|
||||
id: "runtime.agent-runner-boundary",
|
||||
title: "Agent runner boundary integration tests",
|
||||
sourcePath: "src/agents/agent-runner.e2e.test.ts",
|
||||
primaryCoverageIds: ["runtime.agent-runner", "runtime.delivery"],
|
||||
surfaceIds: ["agent-runtime-and-provider-execution"],
|
||||
categoryIds: ["agent-runtime-and-provider-execution.agent-turn-execution"],
|
||||
codeRefs: ["src/agents/agent-runner.ts"],
|
||||
},
|
||||
],
|
||||
results: [
|
||||
{
|
||||
id: "runtime.agent-runner-boundary",
|
||||
status: "pass",
|
||||
durationMs: 1234,
|
||||
},
|
||||
],
|
||||
});
|
||||
|
||||
expect(validateQaEvidenceSummaryJson(evidence)).toEqual(evidence);
|
||||
expect(evidence.entries).toEqual([
|
||||
expect.objectContaining({
|
||||
test: {
|
||||
kind: "vitest-test",
|
||||
id: "runtime.agent-runner-boundary",
|
||||
title: "Agent runner boundary integration tests",
|
||||
source: {
|
||||
path: "src/agents/agent-runner.e2e.test.ts",
|
||||
},
|
||||
},
|
||||
mapping: {
|
||||
profile: "smoke-ci",
|
||||
coverage: [
|
||||
{
|
||||
id: "runtime.agent-runner",
|
||||
role: "primary",
|
||||
surfaceIds: ["agent-runtime-and-provider-execution"],
|
||||
categoryIds: ["agent-runtime-and-provider-execution.agent-turn-execution"],
|
||||
},
|
||||
{
|
||||
id: "runtime.delivery",
|
||||
role: "primary",
|
||||
surfaceIds: ["agent-runtime-and-provider-execution"],
|
||||
categoryIds: ["agent-runtime-and-provider-execution.agent-turn-execution"],
|
||||
},
|
||||
],
|
||||
refs: [
|
||||
{
|
||||
kind: "code",
|
||||
path: "src/agents/agent-runner.ts",
|
||||
},
|
||||
],
|
||||
},
|
||||
execution: expect.objectContaining({
|
||||
runner: "vitest",
|
||||
provider: expect.objectContaining({
|
||||
live: false,
|
||||
fixture: "mock-openai",
|
||||
}),
|
||||
artifacts: [
|
||||
{
|
||||
kind: "runner-result",
|
||||
path: "vitest-results/runtime-boundary.vitest.json",
|
||||
source: "vitest",
|
||||
},
|
||||
],
|
||||
}),
|
||||
result: {
|
||||
status: "pass",
|
||||
timing: {
|
||||
wallMs: 1234,
|
||||
},
|
||||
},
|
||||
}),
|
||||
]);
|
||||
});
|
||||
|
||||
it("builds Playwright runner evidence entries", () => {
|
||||
const evidence = buildPlaywrightEvidenceSummary({
|
||||
artifactPaths: [
|
||||
{ kind: "runner-result", path: "playwright-results/control-ui.json" },
|
||||
{ kind: "report", path: "playwright-report/index.html" },
|
||||
],
|
||||
env: {
|
||||
GITHUB_SHA: "def456",
|
||||
} as NodeJS.ProcessEnv,
|
||||
generatedAt: "2026-06-07T12:07:00.000Z",
|
||||
primaryModel: "mock-openai/gpt-5.5",
|
||||
providerMode: "mock-openai",
|
||||
targets: [
|
||||
{
|
||||
id: "control-ui.browser-run",
|
||||
title: "Control UI browser workflow",
|
||||
sourcePath: "ui/control-ui.e2e.test.ts",
|
||||
primaryCoverageIds: ["control-ui.browser"],
|
||||
surfaceIds: ["browser-control-ui-and-webchat"],
|
||||
categoryIds: ["browser-control-ui-and-webchat.browser-ui"],
|
||||
docsRefs: ["docs/concepts/qa-e2e-automation.md"],
|
||||
codeRefs: ["ui/"],
|
||||
},
|
||||
],
|
||||
results: [
|
||||
{
|
||||
id: "control-ui.browser-run",
|
||||
status: "fail",
|
||||
durationMs: 2300,
|
||||
failureMessage: "locator timed out",
|
||||
},
|
||||
],
|
||||
});
|
||||
|
||||
expect(validateQaEvidenceSummaryJson(evidence)).toEqual(evidence);
|
||||
expect(evidence.entries[0]).toMatchObject({
|
||||
test: {
|
||||
kind: "playwright-test",
|
||||
id: "control-ui.browser-run",
|
||||
title: "Control UI browser workflow",
|
||||
source: {
|
||||
path: "ui/control-ui.e2e.test.ts",
|
||||
},
|
||||
},
|
||||
mapping: {
|
||||
coverage: [
|
||||
{
|
||||
id: "control-ui.browser",
|
||||
role: "primary",
|
||||
surfaceIds: ["browser-control-ui-and-webchat"],
|
||||
categoryIds: ["browser-control-ui-and-webchat.browser-ui"],
|
||||
},
|
||||
],
|
||||
refs: [
|
||||
{
|
||||
kind: "docs",
|
||||
path: "docs/concepts/qa-e2e-automation.md",
|
||||
},
|
||||
{
|
||||
kind: "code",
|
||||
path: "ui/",
|
||||
},
|
||||
],
|
||||
},
|
||||
execution: {
|
||||
runner: "playwright",
|
||||
artifacts: [
|
||||
{
|
||||
kind: "runner-result",
|
||||
path: "playwright-results/control-ui.json",
|
||||
source: "playwright",
|
||||
},
|
||||
{
|
||||
kind: "report",
|
||||
path: "playwright-report/index.html",
|
||||
source: "playwright",
|
||||
},
|
||||
],
|
||||
},
|
||||
result: {
|
||||
status: "fail",
|
||||
failure: {
|
||||
reason: "locator timed out",
|
||||
},
|
||||
timing: {
|
||||
wallMs: 2300,
|
||||
},
|
||||
},
|
||||
});
|
||||
});
|
||||
|
||||
it("carries profile env values without hardcoding taxonomy mapping ids", () => {
|
||||
const evidence = buildQaSuiteEvidenceSummary({
|
||||
artifactPaths: [{ kind: "summary", path: "qa-suite-summary.json" }],
|
||||
scenarioDefinitions: [
|
||||
{
|
||||
id: "dm-chat-baseline",
|
||||
title: "DM baseline conversation",
|
||||
surface: "dm",
|
||||
coverage: {
|
||||
primary: ["channels.dm"],
|
||||
},
|
||||
},
|
||||
],
|
||||
channelId: "qa-channel",
|
||||
env: {
|
||||
OPENCLAW_QA_PROFILE: "experimental-profile",
|
||||
} as NodeJS.ProcessEnv,
|
||||
generatedAt: "2026-06-07T12:09:00.000Z",
|
||||
primaryModel: "mock-openai/gpt-5.5",
|
||||
providerMode: "mock-openai",
|
||||
scenarioResults: [{ name: "DM baseline conversation", status: "pass" }],
|
||||
});
|
||||
|
||||
expect(evidence.entries[0]?.mapping.profile).toBe("experimental-profile");
|
||||
});
|
||||
|
||||
it("keeps mock non-OpenAI model refs attributed to their model provider", () => {
|
||||
const evidence = buildQaSuiteEvidenceSummary({
|
||||
artifactPaths: [{ kind: "summary", path: "qa-suite-summary.json" }],
|
||||
scenarioDefinitions: [
|
||||
{
|
||||
id: "anthropic-parity",
|
||||
title: "Anthropic parity",
|
||||
surface: "runtime",
|
||||
coverage: {
|
||||
primary: ["providers.anthropic"],
|
||||
},
|
||||
},
|
||||
],
|
||||
channelId: "qa-channel",
|
||||
generatedAt: "2026-06-07T12:10:00.000Z",
|
||||
primaryModel: "anthropic/claude-opus-4-8",
|
||||
providerMode: "mock-openai",
|
||||
scenarioResults: [{ name: "Anthropic parity", status: "pass" }],
|
||||
});
|
||||
|
||||
expect(evidence.entries[0]?.execution.provider).toMatchObject({
|
||||
id: "anthropic",
|
||||
model: {
|
||||
name: "claude-opus-4-8",
|
||||
ref: "anthropic/claude-opus-4-8",
|
||||
},
|
||||
});
|
||||
expect(evidence.entries[0]).toMatchObject({
|
||||
execution: {
|
||||
provider: {
|
||||
live: false,
|
||||
fixture: "mock-openai",
|
||||
},
|
||||
},
|
||||
});
|
||||
});
|
||||
|
||||
it("uses explicit package provenance from package runners", () => {
|
||||
const evidence = buildLiveTransportEvidenceSummary({
|
||||
artifactPaths: [{ kind: "summary", path: QA_EVIDENCE_FILENAME }],
|
||||
generatedAt: "2026-06-07T12:15:00.000Z",
|
||||
packageSource: {
|
||||
kind: "packed-tarball",
|
||||
spec: "/tmp/openclaw.tgz",
|
||||
sha: "abc123",
|
||||
},
|
||||
primaryModel: "openai/gpt-5.5",
|
||||
providerMode: "live-frontier",
|
||||
checks: [
|
||||
{
|
||||
id: "telegram-canary",
|
||||
title: "Telegram canary",
|
||||
details: "Canary passed.",
|
||||
standardId: "canary",
|
||||
status: "pass",
|
||||
},
|
||||
],
|
||||
transportId: "telegram",
|
||||
});
|
||||
|
||||
expect(evidence.entries[0]?.execution.packageSource).toEqual({
|
||||
kind: "packed-tarball",
|
||||
spec: "/tmp/openclaw.tgz",
|
||||
sha: "abc123",
|
||||
});
|
||||
});
|
||||
|
||||
it("derives package provenance from generic QA evidence env", () => {
|
||||
const evidence = buildLiveTransportEvidenceSummary({
|
||||
artifactPaths: [{ kind: "summary", path: QA_EVIDENCE_FILENAME }],
|
||||
env: {
|
||||
OPENCLAW_QA_PACKAGE_SOURCE: "openclaw@beta",
|
||||
OPENCLAW_QA_PACKAGE_SOURCE_KIND: "npm-package",
|
||||
OPENCLAW_QA_PACKAGE_SOURCE_SHA: "def456",
|
||||
} as NodeJS.ProcessEnv,
|
||||
generatedAt: "2026-06-07T12:15:00.000Z",
|
||||
primaryModel: "openai/gpt-5.5",
|
||||
providerMode: "live-frontier",
|
||||
checks: [
|
||||
{
|
||||
id: "telegram-canary",
|
||||
title: "Telegram canary",
|
||||
details: "Canary passed.",
|
||||
standardId: "canary",
|
||||
status: "pass",
|
||||
},
|
||||
],
|
||||
transportId: "telegram",
|
||||
});
|
||||
|
||||
expect(evidence.entries[0]?.execution.packageSource).toEqual({
|
||||
kind: "npm-package",
|
||||
spec: "openclaw@beta",
|
||||
sha: "def456",
|
||||
});
|
||||
});
|
||||
|
||||
it("does not infer package provenance from runner-specific env", () => {
|
||||
const evidence = buildLiveTransportEvidenceSummary({
|
||||
artifactPaths: [{ kind: "summary", path: QA_EVIDENCE_FILENAME }],
|
||||
env: {
|
||||
OPENCLAW_NPM_TELEGRAM_INSTALL_SOURCE: "openclaw@beta",
|
||||
} as NodeJS.ProcessEnv,
|
||||
generatedAt: "2026-06-07T12:16:00.000Z",
|
||||
primaryModel: "openai/gpt-5.5",
|
||||
providerMode: "live-frontier",
|
||||
checks: [
|
||||
{
|
||||
id: "telegram-canary",
|
||||
title: "Telegram canary",
|
||||
details: "Canary passed.",
|
||||
standardId: "canary",
|
||||
status: "pass",
|
||||
},
|
||||
],
|
||||
transportId: "telegram",
|
||||
});
|
||||
|
||||
expect(evidence.entries[0]?.execution.packageSource).toEqual({
|
||||
kind: "source-checkout",
|
||||
spec: undefined,
|
||||
sha: undefined,
|
||||
});
|
||||
});
|
||||
|
||||
it("keeps live transport check artifacts on the owning entry", () => {
|
||||
const evidence = buildLiveTransportEvidenceSummary({
|
||||
artifactPaths: [
|
||||
{ kind: "summary", path: QA_EVIDENCE_FILENAME },
|
||||
{ kind: "report", path: "discord-qa-report.md" },
|
||||
],
|
||||
generatedAt: "2026-06-07T12:20:00.000Z",
|
||||
primaryModel: "openai/gpt-5.5",
|
||||
providerMode: "live-frontier",
|
||||
checks: [
|
||||
{
|
||||
artifactPaths: {
|
||||
screenshot: ".artifacts/discord/status.png",
|
||||
video: ".artifacts/discord/status.mp4",
|
||||
},
|
||||
id: "discord-status-reactions-tool-only",
|
||||
title: "Discord status reactions",
|
||||
details: "Status reaction observed.",
|
||||
status: "pass",
|
||||
},
|
||||
],
|
||||
transportId: "discord",
|
||||
});
|
||||
|
||||
expect(evidence.entries[0]?.execution.artifacts).toEqual(
|
||||
expect.arrayContaining([
|
||||
{
|
||||
kind: "screenshot",
|
||||
path: ".artifacts/discord/status.png",
|
||||
source: "discord-live-transport:discord-status-reactions-tool-only",
|
||||
},
|
||||
{
|
||||
kind: "video",
|
||||
path: ".artifacts/discord/status.mp4",
|
||||
source: "discord-live-transport:discord-status-reactions-tool-only",
|
||||
},
|
||||
]),
|
||||
);
|
||||
});
|
||||
});
|
||||
703
extensions/qa-lab/src/evidence-summary.ts
Normal file
703
extensions/qa-lab/src/evidence-summary.ts
Normal file
@@ -0,0 +1,703 @@
|
||||
// Qa Lab plugin module implements QA evidence summary behavior.
|
||||
import { z } from "zod";
|
||||
import { splitQaModelRef } from "./model-selection.js";
|
||||
import { getQaProvider, type QaProviderMode } from "./providers/index.js";
|
||||
|
||||
export const QA_EVIDENCE_SUMMARY_KIND = "openclaw.qa.evidence-summary";
|
||||
export const QA_EVIDENCE_FILENAME = "qa-evidence.json";
|
||||
export const QA_EVIDENCE_SUMMARY_SCHEMA_VERSION = 2;
|
||||
|
||||
const qaEvidenceStatusSchema = z.enum(["pass", "fail", "blocked", "skipped"]);
|
||||
const nonEmptyStringSchema = z.string().trim().min(1);
|
||||
const nullableStringSchema = nonEmptyStringSchema.nullable();
|
||||
const qaEvidenceProfileIdSchema = nonEmptyStringSchema;
|
||||
const qaEvidenceIdSchema = z.object({ id: nonEmptyStringSchema });
|
||||
|
||||
const qaEvidenceProviderSchema = z
|
||||
.object({
|
||||
id: nonEmptyStringSchema,
|
||||
live: z.boolean(),
|
||||
model: z
|
||||
.object({
|
||||
name: nullableStringSchema,
|
||||
ref: nullableStringSchema,
|
||||
})
|
||||
.strict(),
|
||||
fixture: nonEmptyStringSchema.optional(),
|
||||
auth: nonEmptyStringSchema.optional(),
|
||||
})
|
||||
.strict();
|
||||
|
||||
const qaEvidenceChannelSchema = z
|
||||
.object({
|
||||
id: nonEmptyStringSchema,
|
||||
live: z.boolean(),
|
||||
driver: nonEmptyStringSchema.optional(),
|
||||
})
|
||||
.strict();
|
||||
|
||||
const qaEvidenceEnvironmentSchema = z
|
||||
.object({
|
||||
ref: nullableStringSchema,
|
||||
os: nonEmptyStringSchema,
|
||||
nodeVersion: nonEmptyStringSchema,
|
||||
})
|
||||
.strict();
|
||||
|
||||
const qaEvidencePackageSourceSchema = z
|
||||
.object({
|
||||
kind: nonEmptyStringSchema,
|
||||
spec: nonEmptyStringSchema.optional(),
|
||||
sha: nonEmptyStringSchema.optional(),
|
||||
})
|
||||
.strict();
|
||||
|
||||
const qaEvidenceFailureSchema = z
|
||||
.object({
|
||||
class: nonEmptyStringSchema.optional(),
|
||||
reason: nonEmptyStringSchema,
|
||||
})
|
||||
.strict();
|
||||
|
||||
const qaEvidenceTimingSchema = z
|
||||
.object({
|
||||
wallMs: z.number().finite().positive().optional(),
|
||||
rttMs: z.number().finite().positive().optional(),
|
||||
avgMs: z.number().finite().positive().optional(),
|
||||
p50Ms: z.number().finite().positive().optional(),
|
||||
p95Ms: z.number().finite().positive().optional(),
|
||||
maxMs: z.number().finite().positive().optional(),
|
||||
samples: z.number().int().positive().optional(),
|
||||
failedSamples: z.number().int().nonnegative().optional(),
|
||||
})
|
||||
.strict();
|
||||
|
||||
const qaEvidenceTestSchema = z
|
||||
.object({
|
||||
kind: nonEmptyStringSchema,
|
||||
id: nonEmptyStringSchema,
|
||||
title: nonEmptyStringSchema,
|
||||
source: z
|
||||
.object({
|
||||
path: nonEmptyStringSchema,
|
||||
})
|
||||
.strict()
|
||||
.optional(),
|
||||
})
|
||||
.strict();
|
||||
|
||||
const qaEvidenceRefSchema = z
|
||||
.object({
|
||||
kind: nonEmptyStringSchema,
|
||||
path: nonEmptyStringSchema,
|
||||
})
|
||||
.strict();
|
||||
|
||||
const qaEvidenceCoverageSchema = qaEvidenceIdSchema
|
||||
.extend({
|
||||
role: nonEmptyStringSchema,
|
||||
surfaceIds: z.array(nonEmptyStringSchema),
|
||||
categoryIds: z.array(nonEmptyStringSchema),
|
||||
})
|
||||
.strict();
|
||||
|
||||
const qaEvidenceMappingSchema = z
|
||||
.object({
|
||||
profile: qaEvidenceProfileIdSchema,
|
||||
coverage: z.array(qaEvidenceCoverageSchema),
|
||||
refs: z.array(qaEvidenceRefSchema).optional(),
|
||||
runtimeParityTier: nonEmptyStringSchema.optional(),
|
||||
})
|
||||
.strict();
|
||||
|
||||
const qaEvidenceArtifactSchema = z
|
||||
.object({
|
||||
kind: nonEmptyStringSchema,
|
||||
path: nonEmptyStringSchema,
|
||||
source: nonEmptyStringSchema,
|
||||
})
|
||||
.strict();
|
||||
|
||||
const qaEvidenceExecutionSchema = z
|
||||
.object({
|
||||
runner: nonEmptyStringSchema,
|
||||
environment: qaEvidenceEnvironmentSchema,
|
||||
provider: qaEvidenceProviderSchema,
|
||||
channel: qaEvidenceChannelSchema.optional(),
|
||||
packageSource: qaEvidencePackageSourceSchema,
|
||||
artifacts: z.array(qaEvidenceArtifactSchema),
|
||||
})
|
||||
.strict();
|
||||
|
||||
const qaEvidenceResultSchema = z
|
||||
.object({
|
||||
status: qaEvidenceStatusSchema,
|
||||
failure: qaEvidenceFailureSchema.optional(),
|
||||
timing: qaEvidenceTimingSchema.optional(),
|
||||
})
|
||||
.strict();
|
||||
|
||||
export const qaEvidenceSummaryEntrySchema = z
|
||||
.object({
|
||||
test: qaEvidenceTestSchema,
|
||||
mapping: qaEvidenceMappingSchema,
|
||||
execution: qaEvidenceExecutionSchema,
|
||||
result: qaEvidenceResultSchema,
|
||||
})
|
||||
.strict();
|
||||
|
||||
export const qaEvidenceSummarySchema = z
|
||||
.object({
|
||||
kind: z.literal(QA_EVIDENCE_SUMMARY_KIND),
|
||||
schemaVersion: z.literal(QA_EVIDENCE_SUMMARY_SCHEMA_VERSION),
|
||||
generatedAt: nonEmptyStringSchema,
|
||||
entries: z.array(qaEvidenceSummaryEntrySchema),
|
||||
})
|
||||
.strict();
|
||||
|
||||
export type QaEvidenceProfile = z.infer<typeof qaEvidenceProfileIdSchema>;
|
||||
export type QaEvidenceStatus = z.infer<typeof qaEvidenceStatusSchema>;
|
||||
export type QaEvidenceTiming = z.infer<typeof qaEvidenceTimingSchema>;
|
||||
export type QaEvidencePackageSource = z.infer<typeof qaEvidencePackageSourceSchema>;
|
||||
export type QaEvidenceSummaryEntry = z.infer<typeof qaEvidenceSummaryEntrySchema>;
|
||||
export type QaEvidenceSummaryJson = z.infer<typeof qaEvidenceSummarySchema>;
|
||||
|
||||
type QaEvidenceStatusInput = QaEvidenceStatus | "skip";
|
||||
|
||||
type QaEvidenceScenarioDefinitionInput = {
|
||||
id: string;
|
||||
title: string;
|
||||
sourcePath?: string;
|
||||
surface?: string;
|
||||
surfaces?: readonly string[];
|
||||
category?: string;
|
||||
coverage?: {
|
||||
primary?: readonly string[];
|
||||
secondary?: readonly string[];
|
||||
};
|
||||
runtimeParityTier?: string;
|
||||
docsRefs?: readonly string[];
|
||||
codeRefs?: readonly string[];
|
||||
};
|
||||
|
||||
type QaEvidenceScenarioResultInput = {
|
||||
name: string;
|
||||
status: QaEvidenceStatusInput;
|
||||
details?: string;
|
||||
rttMs?: number;
|
||||
rttMeasurement?: {
|
||||
finalMatchedReplyRttMs?: number;
|
||||
};
|
||||
};
|
||||
|
||||
type QaEvidenceLiveTransportCheckInput = {
|
||||
id: string;
|
||||
title: string;
|
||||
status: QaEvidenceStatusInput;
|
||||
details: string;
|
||||
rttMs?: number;
|
||||
rttMeasurement?: {
|
||||
finalMatchedReplyRttMs?: number;
|
||||
};
|
||||
// Here "standard" means a taxonomy-backed requirement standard, not the default lane.
|
||||
standardId?: string;
|
||||
artifactPaths?: Readonly<Record<string, string>>;
|
||||
};
|
||||
|
||||
type QaEvidenceRttInput = Pick<QaEvidenceScenarioResultInput, "rttMeasurement" | "rttMs">;
|
||||
|
||||
type QaEvidenceTestTargetInput = {
|
||||
id: string;
|
||||
title: string;
|
||||
sourcePath: string;
|
||||
primaryCoverageIds?: readonly string[];
|
||||
secondaryCoverageIds?: readonly string[];
|
||||
surfaceIds: readonly string[];
|
||||
categoryIds: readonly string[];
|
||||
docsRefs?: readonly string[];
|
||||
codeRefs?: readonly string[];
|
||||
};
|
||||
|
||||
type QaEvidenceTestResultInput = {
|
||||
id?: string;
|
||||
title?: string;
|
||||
sourcePath?: string;
|
||||
status: QaEvidenceStatusInput;
|
||||
durationMs?: number;
|
||||
failureMessage?: string;
|
||||
};
|
||||
|
||||
type QaEvidenceArtifactInput = {
|
||||
kind: string;
|
||||
path: string;
|
||||
};
|
||||
|
||||
type QaEvidenceBuildBase = {
|
||||
artifactPaths: readonly QaEvidenceArtifactInput[];
|
||||
env?: NodeJS.ProcessEnv;
|
||||
generatedAt: string;
|
||||
primaryModel: string;
|
||||
providerMode: QaProviderMode;
|
||||
channelDriver?: string;
|
||||
packageSource?: QaEvidencePackageSource;
|
||||
profile?: QaEvidenceProfile;
|
||||
runner?: string;
|
||||
};
|
||||
|
||||
function buildQaEvidenceRefs(params: {
|
||||
docsRefs?: readonly string[];
|
||||
codeRefs?: readonly string[];
|
||||
}) {
|
||||
const buildRef = (kind: "docs" | "code", refPath: string) => {
|
||||
const ref = {
|
||||
kind,
|
||||
path: refPath,
|
||||
};
|
||||
return ref;
|
||||
};
|
||||
const refs = [
|
||||
...(params.docsRefs ?? []).map((path) => buildRef("docs", path)),
|
||||
...(params.codeRefs ?? []).map((path) => buildRef("code", path)),
|
||||
];
|
||||
return [...new Map(refs.map((ref) => [`${ref.kind}:${ref.path}`, ref])).values()];
|
||||
}
|
||||
|
||||
function buildQaEvidenceCoverage(params: {
|
||||
primaryIds?: readonly string[];
|
||||
secondaryIds?: readonly string[];
|
||||
surfaceIds?: readonly string[];
|
||||
categoryIds?: readonly string[];
|
||||
}) {
|
||||
const surfaceIds = uniqueSortedStrings(params.surfaceIds ?? []);
|
||||
const categoryIds = uniqueSortedStrings(params.categoryIds ?? []);
|
||||
const buildCoverage = (id: string, role: "primary" | "secondary") => ({
|
||||
id,
|
||||
role,
|
||||
surfaceIds,
|
||||
categoryIds: role === "primary" ? categoryIds : [],
|
||||
});
|
||||
return [
|
||||
...uniqueSortedStrings(params.primaryIds ?? []).map((id) => buildCoverage(id, "primary")),
|
||||
...uniqueSortedStrings(params.secondaryIds ?? []).map((id) => buildCoverage(id, "secondary")),
|
||||
];
|
||||
}
|
||||
|
||||
function buildQaEvidenceArtifacts(paths: readonly QaEvidenceArtifactInput[], source: string) {
|
||||
return paths.map((artifact) => ({
|
||||
kind: artifact.kind,
|
||||
path: artifact.path,
|
||||
source,
|
||||
}));
|
||||
}
|
||||
|
||||
function buildQaEvidenceNamedArtifacts(paths: Readonly<Record<string, string>>, source: string) {
|
||||
return Object.entries(paths).map(([kind, artifactPath]) => ({
|
||||
kind,
|
||||
path: artifactPath,
|
||||
source,
|
||||
}));
|
||||
}
|
||||
|
||||
function uniqueSortedStrings(values: readonly (string | undefined)[]) {
|
||||
return [...new Set(values.map((value) => value?.trim()).filter(Boolean) as string[])].toSorted(
|
||||
(left, right) => left.localeCompare(right),
|
||||
);
|
||||
}
|
||||
|
||||
function resolveQaEvidenceProfile(params: {
|
||||
env?: NodeJS.ProcessEnv;
|
||||
fallback: QaEvidenceProfile;
|
||||
explicit?: QaEvidenceProfile;
|
||||
}) {
|
||||
if (params.explicit) {
|
||||
const explicit = params.explicit.trim();
|
||||
if (!explicit) {
|
||||
throw new Error("evidence profile must be a non-empty string.");
|
||||
}
|
||||
return explicit;
|
||||
}
|
||||
|
||||
const envProfiles = [
|
||||
["OPENCLAW_E2E_PROFILE", params.env?.OPENCLAW_E2E_PROFILE],
|
||||
["OPENCLAW_QA_PROFILE", params.env?.OPENCLAW_QA_PROFILE],
|
||||
] as const;
|
||||
for (const [, value] of envProfiles) {
|
||||
const normalized = value?.trim();
|
||||
if (!normalized) {
|
||||
continue;
|
||||
}
|
||||
return normalized;
|
||||
}
|
||||
|
||||
return params.fallback;
|
||||
}
|
||||
|
||||
function resolveQaEvidenceRunner(params: { env?: NodeJS.ProcessEnv; fallback?: string }) {
|
||||
return params.env?.OPENCLAW_QA_RUNNER?.trim() || params.fallback || "host";
|
||||
}
|
||||
|
||||
function resolveQaEvidenceChannelDriver(params: { env?: NodeJS.ProcessEnv; fallback?: string }) {
|
||||
const id =
|
||||
params.fallback?.trim() ||
|
||||
params.env?.OPENCLAW_QA_CHANNEL_DRIVER?.trim() ||
|
||||
params.env?.OPENCLAW_E2E_CHANNEL_DRIVER?.trim();
|
||||
return id ? { id } : undefined;
|
||||
}
|
||||
|
||||
function resolveQaEvidenceEnvironment(env: NodeJS.ProcessEnv | undefined) {
|
||||
return {
|
||||
ref: env?.OPENCLAW_QA_REF?.trim() || env?.GITHUB_SHA?.trim() || null,
|
||||
os: process.platform,
|
||||
nodeVersion: process.version,
|
||||
};
|
||||
}
|
||||
|
||||
function resolveQaEvidencePackageSource(env: NodeJS.ProcessEnv | undefined) {
|
||||
const spec = env?.OPENCLAW_QA_PACKAGE_SOURCE?.trim() || undefined;
|
||||
const sha = env?.OPENCLAW_QA_PACKAGE_SOURCE_SHA?.trim() || undefined;
|
||||
const explicitKind = env?.OPENCLAW_QA_PACKAGE_SOURCE_KIND?.trim();
|
||||
const kind =
|
||||
explicitKind ||
|
||||
(spec && spec.endsWith(".tgz") ? "packed-tarball" : spec ? "npm-package" : "source-checkout");
|
||||
return {
|
||||
kind,
|
||||
spec,
|
||||
sha,
|
||||
};
|
||||
}
|
||||
|
||||
function resolveQaEvidenceBuildPackageSource(params: QaEvidenceBuildBase) {
|
||||
return params.packageSource ?? resolveQaEvidencePackageSource(params.env);
|
||||
}
|
||||
|
||||
function buildQaEvidenceProvider(params: { providerMode: QaProviderMode; primaryModel: string }) {
|
||||
const provider = getQaProvider(params.providerMode);
|
||||
const split = splitQaModelRef(params.primaryModel);
|
||||
const providerShape = {
|
||||
id: split?.provider ?? params.providerMode,
|
||||
model: {
|
||||
name: split?.model ?? null,
|
||||
ref: params.primaryModel || null,
|
||||
},
|
||||
};
|
||||
if (provider.kind === "live") {
|
||||
return {
|
||||
...providerShape,
|
||||
live: true,
|
||||
auth: params.providerMode,
|
||||
};
|
||||
}
|
||||
const mockProviderId =
|
||||
split?.provider && split.provider !== params.providerMode
|
||||
? split.provider
|
||||
: params.providerMode === "mock-openai"
|
||||
? "openai"
|
||||
: (split?.provider ?? params.providerMode);
|
||||
return {
|
||||
...providerShape,
|
||||
id: mockProviderId,
|
||||
live: false,
|
||||
fixture: params.providerMode,
|
||||
};
|
||||
}
|
||||
|
||||
function normalizeQaEvidenceStatus(status: QaEvidenceStatusInput): QaEvidenceStatus {
|
||||
return status === "skip" ? "skipped" : status;
|
||||
}
|
||||
|
||||
function failureForResult(result: {
|
||||
details?: string;
|
||||
failureMessage?: string;
|
||||
status: QaEvidenceStatusInput;
|
||||
}) {
|
||||
const status = normalizeQaEvidenceStatus(result.status);
|
||||
if (status === "pass") {
|
||||
return undefined;
|
||||
}
|
||||
return {
|
||||
reason: result.details?.trim() || result.failureMessage?.trim() || `${status} test`,
|
||||
};
|
||||
}
|
||||
|
||||
function timingForRttResult(check: QaEvidenceRttInput) {
|
||||
const rttMs = check.rttMeasurement?.finalMatchedReplyRttMs ?? check.rttMs;
|
||||
return typeof rttMs === "number" && Number.isFinite(rttMs) && rttMs > 0 ? { rttMs } : undefined;
|
||||
}
|
||||
|
||||
function timingForTestResult(result: QaEvidenceTestResultInput) {
|
||||
return typeof result.durationMs === "number" &&
|
||||
Number.isFinite(result.durationMs) &&
|
||||
result.durationMs > 0
|
||||
? { wallMs: result.durationMs }
|
||||
: undefined;
|
||||
}
|
||||
|
||||
function resultForEvidence(
|
||||
result: { details?: string; failureMessage?: string; status: QaEvidenceStatusInput },
|
||||
timing?: QaEvidenceTiming,
|
||||
) {
|
||||
return {
|
||||
status: normalizeQaEvidenceStatus(result.status),
|
||||
failure: failureForResult(result),
|
||||
timing,
|
||||
};
|
||||
}
|
||||
|
||||
function buildQaEvidenceSummary(params: {
|
||||
entries: QaEvidenceSummaryEntry[];
|
||||
generatedAt: string;
|
||||
}): QaEvidenceSummaryJson {
|
||||
return qaEvidenceSummarySchema.parse({
|
||||
kind: QA_EVIDENCE_SUMMARY_KIND,
|
||||
schemaVersion: QA_EVIDENCE_SUMMARY_SCHEMA_VERSION,
|
||||
generatedAt: params.generatedAt,
|
||||
entries: params.entries,
|
||||
});
|
||||
}
|
||||
|
||||
export function validateQaEvidenceSummaryJson(summary: unknown): QaEvidenceSummaryJson {
|
||||
return qaEvidenceSummarySchema.parse(summary);
|
||||
}
|
||||
|
||||
export function buildQaSuiteEvidenceSummary(
|
||||
params: QaEvidenceBuildBase & {
|
||||
channelId: string;
|
||||
scenarioDefinitions: readonly QaEvidenceScenarioDefinitionInput[];
|
||||
scenarioResults: readonly QaEvidenceScenarioResultInput[];
|
||||
},
|
||||
): QaEvidenceSummaryJson {
|
||||
const provider = buildQaEvidenceProvider(params);
|
||||
const environment = resolveQaEvidenceEnvironment(params.env);
|
||||
const packageSource = resolveQaEvidenceBuildPackageSource(params);
|
||||
const runner = resolveQaEvidenceRunner({ env: params.env, fallback: params.runner });
|
||||
const profile = resolveQaEvidenceProfile({
|
||||
env: params.env,
|
||||
fallback: provider.live ? "release" : "smoke-ci",
|
||||
explicit: params.profile,
|
||||
});
|
||||
const channelDriver = resolveQaEvidenceChannelDriver({
|
||||
env: params.env,
|
||||
fallback: params.channelDriver,
|
||||
});
|
||||
const entries = params.scenarioResults.map((result, index): QaEvidenceSummaryEntry => {
|
||||
const scenario = params.scenarioDefinitions[index];
|
||||
const primaryCoverageIds = uniqueSortedStrings(scenario?.coverage?.primary ?? []);
|
||||
const coverageIds = uniqueSortedStrings([
|
||||
...(scenario?.coverage?.primary ?? []),
|
||||
...(scenario?.coverage?.secondary ?? []),
|
||||
]);
|
||||
const surfaceIds = uniqueSortedStrings(
|
||||
scenario?.surfaces && scenario.surfaces.length > 0 ? scenario.surfaces : [scenario?.surface],
|
||||
);
|
||||
const runtimeParityTier = scenario?.runtimeParityTier;
|
||||
const testId = scenario?.id ?? `scenario-${index + 1}`;
|
||||
const refs = buildQaEvidenceRefs({
|
||||
docsRefs: scenario?.docsRefs,
|
||||
codeRefs: scenario?.codeRefs,
|
||||
});
|
||||
const timing = timingForRttResult(result);
|
||||
return {
|
||||
test: {
|
||||
kind: "qa-scenario",
|
||||
id: testId,
|
||||
title: scenario?.title ?? result.name,
|
||||
source: scenario?.sourcePath ? { path: scenario.sourcePath } : undefined,
|
||||
},
|
||||
mapping: {
|
||||
profile,
|
||||
coverage: buildQaEvidenceCoverage({
|
||||
primaryIds: primaryCoverageIds,
|
||||
secondaryIds: coverageIds.filter(
|
||||
(coverageId) => !primaryCoverageIds.includes(coverageId),
|
||||
),
|
||||
surfaceIds,
|
||||
categoryIds: uniqueSortedStrings([scenario?.category, ...primaryCoverageIds]),
|
||||
}),
|
||||
refs: refs.length > 0 ? refs : undefined,
|
||||
runtimeParityTier,
|
||||
},
|
||||
execution: {
|
||||
runner,
|
||||
environment,
|
||||
provider,
|
||||
channel: {
|
||||
id: params.channelId,
|
||||
live: false,
|
||||
driver: channelDriver?.id,
|
||||
},
|
||||
packageSource,
|
||||
artifacts: buildQaEvidenceArtifacts(params.artifactPaths, "qa-suite"),
|
||||
},
|
||||
result: resultForEvidence(result, timing),
|
||||
};
|
||||
});
|
||||
return buildQaEvidenceSummary({ generatedAt: params.generatedAt, entries });
|
||||
}
|
||||
|
||||
function buildTestRunnerEvidenceSummary(
|
||||
params: QaEvidenceBuildBase & {
|
||||
defaultRunner: string;
|
||||
testKind: string;
|
||||
targets: readonly QaEvidenceTestTargetInput[];
|
||||
results: readonly QaEvidenceTestResultInput[];
|
||||
},
|
||||
): QaEvidenceSummaryJson {
|
||||
const provider = buildQaEvidenceProvider(params);
|
||||
const environment = resolveQaEvidenceEnvironment(params.env);
|
||||
const packageSource = resolveQaEvidenceBuildPackageSource(params);
|
||||
const runner = resolveQaEvidenceRunner({
|
||||
env: params.env,
|
||||
fallback: params.runner ?? params.defaultRunner,
|
||||
});
|
||||
const profile = resolveQaEvidenceProfile({
|
||||
env: params.env,
|
||||
fallback: provider.live ? "release" : "smoke-ci",
|
||||
explicit: params.profile,
|
||||
});
|
||||
const targetById = new Map(params.targets.map((target) => [target.id, target]));
|
||||
const targetByPath = new Map(params.targets.map((target) => [target.sourcePath, target]));
|
||||
const entries = params.results.map((result, index): QaEvidenceSummaryEntry => {
|
||||
const target = result.id
|
||||
? targetById.get(result.id)
|
||||
: result.sourcePath
|
||||
? targetByPath.get(result.sourcePath)
|
||||
: undefined;
|
||||
const fallbackId = result.id ?? result.sourcePath ?? `test-${index + 1}`;
|
||||
const sourcePath = target?.sourcePath ?? result.sourcePath;
|
||||
const refs = buildQaEvidenceRefs({
|
||||
docsRefs: target?.docsRefs,
|
||||
codeRefs: target?.codeRefs,
|
||||
});
|
||||
const timing = timingForTestResult(result);
|
||||
return {
|
||||
test: {
|
||||
kind: params.testKind,
|
||||
id: target?.id ?? fallbackId,
|
||||
title: target?.title ?? result.title ?? fallbackId,
|
||||
source: sourcePath ? { path: sourcePath } : undefined,
|
||||
},
|
||||
mapping: {
|
||||
profile,
|
||||
coverage: buildQaEvidenceCoverage({
|
||||
primaryIds: target?.primaryCoverageIds ?? [],
|
||||
secondaryIds: target?.secondaryCoverageIds ?? [],
|
||||
surfaceIds: target?.surfaceIds ?? [],
|
||||
categoryIds: target?.categoryIds ?? [],
|
||||
}),
|
||||
refs: refs.length > 0 ? refs : undefined,
|
||||
},
|
||||
execution: {
|
||||
runner,
|
||||
environment,
|
||||
provider,
|
||||
packageSource,
|
||||
artifacts: buildQaEvidenceArtifacts(params.artifactPaths, runner),
|
||||
},
|
||||
result: resultForEvidence(result, timing),
|
||||
};
|
||||
});
|
||||
return buildQaEvidenceSummary({ generatedAt: params.generatedAt, entries });
|
||||
}
|
||||
|
||||
export function buildVitestEvidenceSummary(
|
||||
params: QaEvidenceBuildBase & {
|
||||
targets: readonly QaEvidenceTestTargetInput[];
|
||||
results: readonly QaEvidenceTestResultInput[];
|
||||
},
|
||||
): QaEvidenceSummaryJson {
|
||||
return buildTestRunnerEvidenceSummary({
|
||||
...params,
|
||||
defaultRunner: "vitest",
|
||||
testKind: "vitest-test",
|
||||
runner: params.runner ?? "vitest",
|
||||
});
|
||||
}
|
||||
|
||||
export function buildPlaywrightEvidenceSummary(
|
||||
params: QaEvidenceBuildBase & {
|
||||
targets: readonly QaEvidenceTestTargetInput[];
|
||||
results: readonly QaEvidenceTestResultInput[];
|
||||
},
|
||||
): QaEvidenceSummaryJson {
|
||||
return buildTestRunnerEvidenceSummary({
|
||||
...params,
|
||||
defaultRunner: "playwright",
|
||||
testKind: "playwright-test",
|
||||
runner: params.runner ?? "playwright",
|
||||
});
|
||||
}
|
||||
|
||||
export function buildLiveTransportEvidenceSummary(
|
||||
params: QaEvidenceBuildBase & {
|
||||
checks: readonly QaEvidenceLiveTransportCheckInput[];
|
||||
transportId: string;
|
||||
},
|
||||
): QaEvidenceSummaryJson {
|
||||
const provider = buildQaEvidenceProvider(params);
|
||||
const environment = resolveQaEvidenceEnvironment(params.env);
|
||||
const packageSource = resolveQaEvidenceBuildPackageSource(params);
|
||||
const runner = resolveQaEvidenceRunner({ env: params.env, fallback: params.runner });
|
||||
const profile = resolveQaEvidenceProfile({
|
||||
env: params.env,
|
||||
fallback: "release",
|
||||
explicit: params.profile,
|
||||
});
|
||||
const channelDriver = resolveQaEvidenceChannelDriver({
|
||||
env: params.env,
|
||||
fallback: params.channelDriver ?? "native",
|
||||
}) ?? { id: "native" };
|
||||
const entries = params.checks.map((check): QaEvidenceSummaryEntry => {
|
||||
const testId = check.id;
|
||||
const standardCoverageId = check.standardId
|
||||
? `channels.${params.transportId}.${check.standardId}`
|
||||
: undefined;
|
||||
const coverage = [
|
||||
{
|
||||
id: `channels.${params.transportId}.live`,
|
||||
role: "live-transport",
|
||||
surfaceIds: [`channels.${params.transportId}`],
|
||||
categoryIds: [`channels.${params.transportId}.live`],
|
||||
},
|
||||
];
|
||||
if (standardCoverageId) {
|
||||
coverage.push({
|
||||
id: standardCoverageId,
|
||||
role: "live-transport-standard",
|
||||
surfaceIds: [`channels.${params.transportId}`],
|
||||
categoryIds: [`channels.${params.transportId}.live`],
|
||||
});
|
||||
}
|
||||
const timing = timingForRttResult(check);
|
||||
return {
|
||||
test: {
|
||||
kind: "live-transport-check",
|
||||
id: testId,
|
||||
title: check.title,
|
||||
},
|
||||
mapping: {
|
||||
profile,
|
||||
coverage,
|
||||
},
|
||||
execution: {
|
||||
runner,
|
||||
environment,
|
||||
provider,
|
||||
channel: {
|
||||
id: params.transportId,
|
||||
live: true,
|
||||
driver: channelDriver.id,
|
||||
},
|
||||
packageSource,
|
||||
artifacts: [
|
||||
...buildQaEvidenceArtifacts(params.artifactPaths, `${params.transportId}-live-transport`),
|
||||
...buildQaEvidenceNamedArtifacts(
|
||||
check.artifactPaths ?? {},
|
||||
`${params.transportId}-live-transport:${testId}`,
|
||||
),
|
||||
],
|
||||
},
|
||||
result: resultForEvidence(check, timing),
|
||||
};
|
||||
});
|
||||
return buildQaEvidenceSummary({ generatedAt: params.generatedAt, entries });
|
||||
}
|
||||
@@ -25,6 +25,7 @@ import {
|
||||
resolveQaRuntimeHostVersion,
|
||||
} from "./bundled-plugin-staging.js";
|
||||
import { assertRepoBoundPath, ensureRepoBoundDirectory } from "./cli-paths.js";
|
||||
import { QaSuiteInfraError } from "./errors.js";
|
||||
import { formatQaGatewayLogsForError, redactQaGatewayDebugText } from "./gateway-log-redaction.js";
|
||||
import { startQaGatewayRpcClient } from "./gateway-rpc-client.js";
|
||||
import { splitQaModelRef, type QaProviderMode } from "./model-selection.js";
|
||||
@@ -470,7 +471,8 @@ async function waitForGatewayReady(params: {
|
||||
const startedAt = Date.now();
|
||||
while (Date.now() - startedAt < (params.timeoutMs ?? 60_000)) {
|
||||
if (params.child.exitCode !== null || params.child.signalCode !== null) {
|
||||
throw new Error(
|
||||
throw new QaSuiteInfraError(
|
||||
"gateway_startup_unhealthy",
|
||||
`gateway exited before becoming healthy (exitCode=${String(params.child.exitCode)}, signal=${String(params.child.signalCode)}):\n${params.logs()}`,
|
||||
);
|
||||
}
|
||||
@@ -485,7 +487,10 @@ async function waitForGatewayReady(params: {
|
||||
}
|
||||
await sleep(250);
|
||||
}
|
||||
throw new Error(`gateway failed to become healthy:\n${params.logs()}`);
|
||||
throw new QaSuiteInfraError(
|
||||
"gateway_startup_unhealthy",
|
||||
`gateway failed to become healthy:\n${params.logs()}`,
|
||||
);
|
||||
}
|
||||
|
||||
function isRetryableRpcStartupError(error: unknown) {
|
||||
@@ -1031,14 +1036,13 @@ export async function startQaGatewayChild(params: {
|
||||
stagedBundledPluginsRoot,
|
||||
});
|
||||
}
|
||||
throw new Error(
|
||||
keepTemp
|
||||
? appendQaGatewayTempRoot(formatErrorMessage(error), tempRoot)
|
||||
: formatErrorMessage(error),
|
||||
{
|
||||
cause: error,
|
||||
},
|
||||
);
|
||||
const message = keepTemp
|
||||
? appendQaGatewayTempRoot(formatErrorMessage(error), tempRoot)
|
||||
: formatErrorMessage(error);
|
||||
if (error instanceof QaSuiteInfraError) {
|
||||
throw new QaSuiteInfraError(error.code, message, { cause: error });
|
||||
}
|
||||
throw new Error(message, { cause: error });
|
||||
}
|
||||
}
|
||||
export { testing as __testing };
|
||||
|
||||
@@ -315,7 +315,7 @@ describe("qa-lab server", () => {
|
||||
controlUiUrl: string | null;
|
||||
controlUiEmbeddedUrl: string | null;
|
||||
kickoffTask: string;
|
||||
scenarios: Array<{ id: string; title: string }>;
|
||||
scenarios: Array<{ id: string; title: string; execution?: { kind?: string } }>;
|
||||
defaults: { conversationId: string; senderId: string };
|
||||
runner: { status: string; selection: { providerMode: string; scenarioIds: string[] } };
|
||||
};
|
||||
@@ -328,7 +328,12 @@ describe("qa-lab server", () => {
|
||||
expect(bootstrap.scenarios.map((scenario) => scenario.id)).toContain("dm-chat-baseline");
|
||||
expect(bootstrap.runner.status).toBe("idle");
|
||||
expect(bootstrap.runner.selection.providerMode).toBe("live-frontier");
|
||||
expect(bootstrap.runner.selection.scenarioIds).toHaveLength(bootstrap.scenarios.length);
|
||||
const flowScenarioIds = bootstrap.scenarios
|
||||
.filter(
|
||||
(scenario) => scenario.execution?.kind === undefined || scenario.execution.kind === "flow",
|
||||
)
|
||||
.map((scenario) => scenario.id);
|
||||
expect(bootstrap.runner.selection.scenarioIds).toEqual(flowScenarioIds);
|
||||
|
||||
const startupStatus = (await (
|
||||
await fetchWithRetry(`${lab.baseUrl}/api/capture/startup-status`)
|
||||
|
||||
@@ -548,8 +548,8 @@ export async function startQaLabServer(
|
||||
};
|
||||
activeSuiteRun = (async () => {
|
||||
try {
|
||||
const { runQaSuite } = await import("./suite.js");
|
||||
const result = await runQaSuite({
|
||||
const { runQaFlowSuite } = await import("./suite.js");
|
||||
const result = await runQaFlowSuite({
|
||||
lab: labHandle ?? undefined,
|
||||
startLab: startQaLabServer,
|
||||
outputDir: createQaRunOutputDir(repoRoot),
|
||||
@@ -565,6 +565,7 @@ export async function startQaLabServer(
|
||||
finishedAt: new Date().toISOString(),
|
||||
artifacts: {
|
||||
outputDir: result.outputDir,
|
||||
evidencePath: result.evidencePath,
|
||||
reportPath: result.reportPath,
|
||||
summaryPath: result.summaryPath,
|
||||
watchUrl: result.watchUrl,
|
||||
|
||||
@@ -15,6 +15,7 @@ import { writeExternalFileWithinRoot } from "openclaw/plugin-sdk/security-runtim
|
||||
import { uniqueStrings } from "openclaw/plugin-sdk/string-coerce-runtime";
|
||||
import { chromium } from "playwright-core";
|
||||
import { z } from "zod";
|
||||
import { QA_EVIDENCE_FILENAME, buildLiveTransportEvidenceSummary } from "../../evidence-summary.js";
|
||||
import { startQaGatewayChild } from "../../gateway-child.js";
|
||||
import { DEFAULT_QA_LIVE_PROVIDER_MODE } from "../../providers/index.js";
|
||||
import {
|
||||
@@ -25,7 +26,6 @@ import {
|
||||
import {
|
||||
acquireQaCredentialLease,
|
||||
startQaCredentialLeaseHeartbeat,
|
||||
type QaCredentialRole,
|
||||
} from "../shared/credential-lease.runtime.js";
|
||||
import {
|
||||
appendQaLiveLaneIssue as appendLiveLaneIssue,
|
||||
@@ -185,6 +185,7 @@ type DiscordQaScenarioResult = {
|
||||
artifactPaths?: Record<string, string>;
|
||||
id: string;
|
||||
title: string;
|
||||
standardId?: string;
|
||||
status: "pass" | "fail";
|
||||
details: string;
|
||||
requestStartedAt?: string;
|
||||
@@ -208,33 +209,6 @@ type DiscordQaRunResult = {
|
||||
scenarios: DiscordQaScenarioResult[];
|
||||
};
|
||||
|
||||
type DiscordQaSummary = {
|
||||
artifacts: {
|
||||
observedMessagesPath: string;
|
||||
reactionTimelinesPath?: string;
|
||||
reportPath: string;
|
||||
summaryPath: string;
|
||||
};
|
||||
credentials: {
|
||||
credentialId?: string;
|
||||
kind: string;
|
||||
ownerId?: string;
|
||||
role?: QaCredentialRole;
|
||||
source: "convex" | "env";
|
||||
};
|
||||
guildId: string;
|
||||
channelId: string;
|
||||
startedAt: string;
|
||||
finishedAt: string;
|
||||
cleanupIssues: string[];
|
||||
counts: {
|
||||
total: number;
|
||||
passed: number;
|
||||
failed: number;
|
||||
};
|
||||
scenarios: DiscordQaScenarioResult[];
|
||||
};
|
||||
|
||||
type DiscordReactionSnapshot = {
|
||||
elapsedMs: number;
|
||||
observedAt: string;
|
||||
@@ -1302,6 +1276,7 @@ async function runDiscordThreadReplyFilePathAttachmentScenario(params: {
|
||||
return {
|
||||
id: params.scenario.id,
|
||||
title: params.scenario.title,
|
||||
standardId: params.scenario.standardId,
|
||||
status,
|
||||
details:
|
||||
status === "pass"
|
||||
@@ -1691,6 +1666,7 @@ export async function runDiscordQaLive(params: {
|
||||
scenarioResults.push({
|
||||
id: scenario.id,
|
||||
title: scenario.title,
|
||||
standardId: scenario.standardId,
|
||||
status: "pass",
|
||||
details: redactPublicMetadata
|
||||
? "native command registered"
|
||||
@@ -1712,6 +1688,7 @@ export async function runDiscordQaLive(params: {
|
||||
scenarioResults.push({
|
||||
id: scenario.id,
|
||||
title: scenario.title,
|
||||
standardId: scenario.standardId,
|
||||
status: "pass",
|
||||
details: redactPublicMetadata
|
||||
? "SUT bot joined voice channel"
|
||||
@@ -1766,6 +1743,7 @@ export async function runDiscordQaLive(params: {
|
||||
scenarioResults.push({
|
||||
id: scenario.id,
|
||||
title: scenario.title,
|
||||
standardId: scenario.standardId,
|
||||
status: missing.length === 0 ? "pass" : "fail",
|
||||
details:
|
||||
missing.length === 0
|
||||
@@ -1811,6 +1789,7 @@ export async function runDiscordQaLive(params: {
|
||||
scenarioResults.push({
|
||||
id: scenario.id,
|
||||
title: scenario.title,
|
||||
standardId: scenario.standardId,
|
||||
status: "pass",
|
||||
details: redactPublicMetadata
|
||||
? "reply matched"
|
||||
@@ -1838,6 +1817,7 @@ export async function runDiscordQaLive(params: {
|
||||
scenarioResults.push({
|
||||
id: scenario.id,
|
||||
title: scenario.title,
|
||||
standardId: scenario.standardId,
|
||||
status: "pass",
|
||||
details: "no reply",
|
||||
});
|
||||
@@ -1847,6 +1827,7 @@ export async function runDiscordQaLive(params: {
|
||||
scenarioResults.push({
|
||||
id: scenario.id,
|
||||
title: scenario.title,
|
||||
standardId: scenario.standardId,
|
||||
status: "fail",
|
||||
details: formatErrorMessage(error),
|
||||
});
|
||||
@@ -1879,40 +1860,26 @@ export async function runDiscordQaLive(params: {
|
||||
const publishedCleanupIssues = redactPublicMetadata
|
||||
? redactQaLiveLaneIssues(cleanupIssues)
|
||||
: cleanupIssues;
|
||||
const passedCount = scenarioResults.filter((entry) => entry.status === "pass").length;
|
||||
const failedCount = scenarioResults.filter((entry) => entry.status === "fail").length;
|
||||
const summary: DiscordQaSummary = {
|
||||
artifacts: {
|
||||
reportPath: path.join(outputDir, "discord-qa-report.md"),
|
||||
summaryPath: path.join(outputDir, "discord-qa-summary.json"),
|
||||
observedMessagesPath: path.join(outputDir, "discord-qa-observed-messages.json"),
|
||||
...(reactionTimelines.length > 0
|
||||
? { reactionTimelinesPath: path.join(outputDir, "discord-qa-reaction-timelines.json") }
|
||||
: {}),
|
||||
},
|
||||
credentials: {
|
||||
source: credentialLease.source,
|
||||
kind: credentialLease.kind,
|
||||
role: credentialLease.role,
|
||||
ownerId: redactPublicMetadata ? undefined : credentialLease.ownerId,
|
||||
credentialId: redactPublicMetadata ? undefined : credentialLease.credentialId,
|
||||
},
|
||||
guildId: redactPublicMetadata ? "<redacted>" : runtimeEnv.guildId,
|
||||
channelId: redactPublicMetadata ? "<redacted>" : runtimeEnv.channelId,
|
||||
startedAt,
|
||||
finishedAt,
|
||||
cleanupIssues: publishedCleanupIssues,
|
||||
counts: {
|
||||
total: scenarioResults.length,
|
||||
passed: passedCount,
|
||||
failed: failedCount,
|
||||
},
|
||||
scenarios: scenarioResults,
|
||||
};
|
||||
const reportPath = path.join(outputDir, "discord-qa-report.md");
|
||||
const summaryPath = path.join(outputDir, "discord-qa-summary.json");
|
||||
const summaryPath = path.join(outputDir, QA_EVIDENCE_FILENAME);
|
||||
const observedMessagesPath = path.join(outputDir, "discord-qa-observed-messages.json");
|
||||
const reactionTimelinesPath = path.join(outputDir, "discord-qa-reaction-timelines.json");
|
||||
const evidence = buildLiveTransportEvidenceSummary({
|
||||
artifactPaths: [
|
||||
{ kind: "summary", path: path.basename(summaryPath) },
|
||||
{ kind: "report", path: path.basename(reportPath) },
|
||||
{ kind: "transport-observations", path: path.basename(observedMessagesPath) },
|
||||
...(reactionTimelines.length > 0
|
||||
? [{ kind: "reaction-timelines", path: path.basename(reactionTimelinesPath) }]
|
||||
: []),
|
||||
],
|
||||
checks: scenarioResults,
|
||||
env: process.env,
|
||||
generatedAt: finishedAt,
|
||||
primaryModel,
|
||||
providerMode,
|
||||
transportId: "discord",
|
||||
});
|
||||
await fs.writeFile(
|
||||
reportPath,
|
||||
`${renderDiscordQaMarkdown({
|
||||
@@ -1928,7 +1895,7 @@ export async function runDiscordQaLive(params: {
|
||||
})}\n`,
|
||||
{ encoding: "utf8", mode: 0o600 },
|
||||
);
|
||||
await fs.writeFile(summaryPath, `${JSON.stringify(summary, null, 2)}\n`, {
|
||||
await fs.writeFile(summaryPath, `${JSON.stringify(evidence, null, 2)}\n`, {
|
||||
encoding: "utf8",
|
||||
mode: 0o600,
|
||||
});
|
||||
|
||||
@@ -3,6 +3,7 @@ import fs from "node:fs/promises";
|
||||
import { tmpdir } from "node:os";
|
||||
import path from "node:path";
|
||||
import { beforeEach, describe, expect, it, vi } from "vitest";
|
||||
import { QA_EVIDENCE_FILENAME, QA_EVIDENCE_SUMMARY_KIND } from "../../evidence-summary.js";
|
||||
import { testing, runSlackQaLive } from "./slack-live.runtime.js";
|
||||
|
||||
describe("Slack live QA runtime helpers", () => {
|
||||
@@ -53,7 +54,7 @@ describe("Slack live QA runtime helpers", () => {
|
||||
});
|
||||
});
|
||||
|
||||
it("reports standard live transport scenario coverage", () => {
|
||||
it("reports live transport standard scenario coverage", () => {
|
||||
expect(testing.SLACK_QA_STANDARD_SCENARIO_IDS).toEqual([
|
||||
"canary",
|
||||
"mention-gating",
|
||||
@@ -71,7 +72,7 @@ describe("Slack live QA runtime helpers", () => {
|
||||
]);
|
||||
});
|
||||
|
||||
it("selects native approval scenarios by id without changing standard coverage", () => {
|
||||
it("selects native approval scenarios by id without changing standard scenario coverage", () => {
|
||||
expect(
|
||||
testing
|
||||
.findScenario(["slack-approval-exec-native", "slack-approval-plugin-native"])
|
||||
@@ -436,15 +437,25 @@ describe("Slack live QA runtime helpers", () => {
|
||||
expect(result.scenarios[0]?.status).toBe("fail");
|
||||
expect(result.scenarios[0]?.details).toContain("Missing OPENCLAW_QA_CONVEX_SITE_URL");
|
||||
await expect(fs.stat(result.reportPath).then((stats) => stats.isFile())).resolves.toBe(true);
|
||||
expect(path.basename(result.summaryPath)).toBe(QA_EVIDENCE_FILENAME);
|
||||
const summary = JSON.parse(await fs.readFile(result.summaryPath, "utf8")) as {
|
||||
channelId: string;
|
||||
credentials: { kind: string; role?: string; source: string };
|
||||
entries: Array<{
|
||||
result: { failure?: { reason?: string }; status: string };
|
||||
test: { id: string };
|
||||
}>;
|
||||
kind: string;
|
||||
};
|
||||
expect(summary.channelId).toBe("<unavailable>");
|
||||
expect(summary.credentials).toEqual({
|
||||
kind: "slack",
|
||||
role: "ci",
|
||||
source: "convex",
|
||||
expect(summary.kind).toBe(QA_EVIDENCE_SUMMARY_KIND);
|
||||
expect(summary.entries[0]).toMatchObject({
|
||||
test: {
|
||||
id: "slack-canary",
|
||||
},
|
||||
result: {
|
||||
status: "fail",
|
||||
failure: {
|
||||
reason: expect.stringContaining("Missing OPENCLAW_QA_CONVEX_SITE_URL"),
|
||||
},
|
||||
},
|
||||
});
|
||||
});
|
||||
});
|
||||
|
||||
@@ -9,6 +9,7 @@ import { formatErrorMessage } from "openclaw/plugin-sdk/error-runtime";
|
||||
import { parseStrictPositiveInteger } from "openclaw/plugin-sdk/number-runtime";
|
||||
import { uniqueStrings } from "openclaw/plugin-sdk/string-coerce-runtime";
|
||||
import { z } from "zod";
|
||||
import { QA_EVIDENCE_FILENAME, buildLiveTransportEvidenceSummary } from "../../evidence-summary.js";
|
||||
import { startQaGatewayChild } from "../../gateway-child.js";
|
||||
import { DEFAULT_QA_LIVE_PROVIDER_MODE } from "../../providers/index.js";
|
||||
import {
|
||||
@@ -19,7 +20,6 @@ import {
|
||||
import {
|
||||
acquireQaCredentialLease,
|
||||
startQaCredentialLeaseHeartbeat,
|
||||
type QaCredentialRole,
|
||||
} from "../shared/credential-lease.runtime.js";
|
||||
import {
|
||||
appendQaLiveLaneIssue as appendLiveLaneIssue,
|
||||
@@ -215,6 +215,7 @@ type SlackQaScenarioResult = {
|
||||
responseObservedAt: string;
|
||||
source: "approval-request-to-resolution" | "request-to-observed-message";
|
||||
};
|
||||
standardId?: string;
|
||||
status: "fail" | "pass";
|
||||
title: string;
|
||||
};
|
||||
@@ -228,26 +229,6 @@ export type SlackQaRunResult = {
|
||||
summaryPath: string;
|
||||
};
|
||||
|
||||
type SlackQaSummary = {
|
||||
channelId: string;
|
||||
cleanupIssues: string[];
|
||||
counts: {
|
||||
failed: number;
|
||||
passed: number;
|
||||
total: number;
|
||||
};
|
||||
credentials: {
|
||||
credentialId?: string;
|
||||
kind: string;
|
||||
ownerId?: string;
|
||||
role?: QaCredentialRole;
|
||||
source: "convex" | "env";
|
||||
};
|
||||
finishedAt: string;
|
||||
scenarios: SlackQaScenarioResult[];
|
||||
startedAt: string;
|
||||
};
|
||||
|
||||
type SlackCredentialLease = Awaited<ReturnType<typeof acquireQaCredentialLease<SlackQaRuntimeEnv>>>;
|
||||
type SlackCredentialHeartbeat = ReturnType<typeof startQaCredentialLeaseHeartbeat>;
|
||||
|
||||
@@ -537,14 +518,6 @@ function inferSlackCredentialSource(
|
||||
return normalized === "convex" ? "convex" : "env";
|
||||
}
|
||||
|
||||
function inferSlackCredentialRole(value: string | undefined): QaCredentialRole | undefined {
|
||||
const normalized = value?.trim().toLowerCase();
|
||||
if (normalized === "ci" || normalized === "maintainer") {
|
||||
return normalized;
|
||||
}
|
||||
return undefined;
|
||||
}
|
||||
|
||||
function normalizeSlackId(value: string, label: string) {
|
||||
const normalized = value.trim();
|
||||
if (!/^[A-Z][A-Z0-9]+$/.test(normalized)) {
|
||||
@@ -1804,7 +1777,6 @@ export async function runSlackQaLive(params: {
|
||||
const sutAccountId = params.sutAccountId?.trim() || "sut";
|
||||
const scenarios = findScenario(params.scenarioIds);
|
||||
const requestedCredentialSource = inferSlackCredentialSource(params.credentialSource);
|
||||
const requestedCredentialRole = inferSlackCredentialRole(params.credentialRole);
|
||||
const redactPublicMetadata = isTruthyOptIn(process.env[QA_REDACT_PUBLIC_METADATA_ENV]);
|
||||
const includeObservedMessageContent = isTruthyOptIn(process.env[SLACK_QA_CAPTURE_CONTENT_ENV]);
|
||||
const startedAt = new Date().toISOString();
|
||||
@@ -1916,6 +1888,7 @@ export async function runSlackQaLive(params: {
|
||||
approval: approval.artifact,
|
||||
id: scenario.id,
|
||||
title: scenario.title,
|
||||
standardId: scenario.standardId,
|
||||
status: "pass",
|
||||
details: [
|
||||
`${scenarioRun.approvalKind} approval resolved ${scenarioRun.decision} in ${approval.rttMs}ms`,
|
||||
@@ -1972,6 +1945,7 @@ export async function runSlackQaLive(params: {
|
||||
scenarioResults.push({
|
||||
id: scenario.id,
|
||||
title: scenario.title,
|
||||
standardId: scenario.standardId,
|
||||
status: "pass",
|
||||
details: [
|
||||
`reply matched in ${rttMs}ms`,
|
||||
@@ -2006,6 +1980,7 @@ export async function runSlackQaLive(params: {
|
||||
scenarioResults.push({
|
||||
id: scenario.id,
|
||||
title: scenario.title,
|
||||
standardId: scenario.standardId,
|
||||
status: "pass",
|
||||
details:
|
||||
scenarioAttempt > 1 ? `no reply; retried ${scenarioAttempt - 1}x` : "no reply",
|
||||
@@ -2023,6 +1998,7 @@ export async function runSlackQaLive(params: {
|
||||
scenarioResults.push({
|
||||
id: scenario.id,
|
||||
title: scenario.title,
|
||||
standardId: scenario.standardId,
|
||||
status: "fail",
|
||||
details:
|
||||
scenarioAttempt > 1
|
||||
@@ -2071,6 +2047,7 @@ export async function runSlackQaLive(params: {
|
||||
scenarioResults.push({
|
||||
id: "slack-canary",
|
||||
title: "Slack canary echo",
|
||||
standardId: "canary",
|
||||
status: "fail",
|
||||
details: formatErrorMessage(error),
|
||||
});
|
||||
@@ -2093,44 +2070,26 @@ export async function runSlackQaLive(params: {
|
||||
|
||||
const finishedAt = new Date().toISOString();
|
||||
const reportPath = path.join(outputDir, "slack-qa-report.md");
|
||||
const summaryPath = path.join(outputDir, "slack-qa-summary.json");
|
||||
const summaryPath = path.join(outputDir, QA_EVIDENCE_FILENAME);
|
||||
const observedMessagesPath = path.join(outputDir, "slack-qa-observed-messages.json");
|
||||
const passed = scenarioResults.filter((entry) => entry.status === "pass").length;
|
||||
const failed = scenarioResults.filter((entry) => entry.status === "fail").length;
|
||||
const artifactScenarioResults = toSlackQaScenarioArtifactResults({
|
||||
scenarios: scenarioResults,
|
||||
includeContent: includeObservedMessageContent,
|
||||
redactMetadata: redactPublicMetadata,
|
||||
});
|
||||
const summary: SlackQaSummary = {
|
||||
credentials: credentialLease
|
||||
? {
|
||||
source: credentialLease.source,
|
||||
kind: credentialLease.kind,
|
||||
role: credentialLease.role,
|
||||
credentialId: redactPublicMetadata ? undefined : credentialLease.credentialId,
|
||||
ownerId: redactPublicMetadata ? undefined : credentialLease.ownerId,
|
||||
}
|
||||
: {
|
||||
source: requestedCredentialSource,
|
||||
kind: "slack",
|
||||
role: requestedCredentialRole,
|
||||
},
|
||||
channelId: runtimeEnv
|
||||
? redactPublicMetadata
|
||||
? "<redacted>"
|
||||
: runtimeEnv.channelId
|
||||
: "<unavailable>",
|
||||
startedAt,
|
||||
finishedAt,
|
||||
cleanupIssues,
|
||||
counts: {
|
||||
total: scenarioResults.length,
|
||||
passed,
|
||||
failed,
|
||||
},
|
||||
scenarios: artifactScenarioResults,
|
||||
};
|
||||
const evidence = buildLiveTransportEvidenceSummary({
|
||||
artifactPaths: [
|
||||
{ kind: "summary", path: path.basename(summaryPath) },
|
||||
{ kind: "report", path: path.basename(reportPath) },
|
||||
{ kind: "transport-observations", path: path.basename(observedMessagesPath) },
|
||||
],
|
||||
checks: artifactScenarioResults,
|
||||
env: process.env,
|
||||
generatedAt: finishedAt,
|
||||
primaryModel,
|
||||
providerMode,
|
||||
transportId: "slack",
|
||||
});
|
||||
await fs.writeFile(
|
||||
observedMessagesPath,
|
||||
`${JSON.stringify(
|
||||
@@ -2143,7 +2102,7 @@ export async function runSlackQaLive(params: {
|
||||
2,
|
||||
)}\n`,
|
||||
);
|
||||
await fs.writeFile(summaryPath, `${JSON.stringify(summary, null, 2)}\n`);
|
||||
await fs.writeFile(summaryPath, `${JSON.stringify(evidence, null, 2)}\n`);
|
||||
await fs.writeFile(
|
||||
reportPath,
|
||||
`${renderSlackQaMarkdown({
|
||||
|
||||
@@ -14,6 +14,7 @@ import { fetchWithSsrFGuard } from "openclaw/plugin-sdk/ssrf-runtime";
|
||||
import { isRecord, uniqueStrings } from "openclaw/plugin-sdk/string-coerce-runtime";
|
||||
import { resolvePreferredOpenClawTmpDir } from "openclaw/plugin-sdk/temp-path";
|
||||
import { z } from "zod";
|
||||
import { QA_EVIDENCE_FILENAME, buildLiveTransportEvidenceSummary } from "../../evidence-summary.js";
|
||||
import { startQaGatewayChild } from "../../gateway-child.js";
|
||||
import { DEFAULT_QA_LIVE_PROVIDER_MODE } from "../../providers/index.js";
|
||||
import {
|
||||
@@ -25,7 +26,6 @@ import {
|
||||
import {
|
||||
acquireQaCredentialLease,
|
||||
startQaCredentialLeaseHeartbeat,
|
||||
type QaCredentialRole,
|
||||
} from "../shared/credential-lease.runtime.js";
|
||||
import {
|
||||
appendQaLiveLaneIssue as appendLiveLaneIssue,
|
||||
@@ -138,6 +138,7 @@ const DEFAULT_TELEGRAM_QA_CANARY_TIMEOUT_MS = 30_000;
|
||||
|
||||
type TelegramQaScenarioResult = {
|
||||
id: string;
|
||||
standardId?: string;
|
||||
title: string;
|
||||
status: "pass" | "fail";
|
||||
details: string;
|
||||
@@ -165,26 +166,6 @@ type TelegramQaRunResult = {
|
||||
scenarios: TelegramQaScenarioResult[];
|
||||
};
|
||||
|
||||
type TelegramQaSummary = {
|
||||
credentials: {
|
||||
credentialId?: string;
|
||||
kind: string;
|
||||
ownerId?: string;
|
||||
role?: QaCredentialRole;
|
||||
source: "convex" | "env";
|
||||
};
|
||||
groupId: string;
|
||||
startedAt: string;
|
||||
finishedAt: string;
|
||||
cleanupIssues: string[];
|
||||
counts: {
|
||||
total: number;
|
||||
passed: number;
|
||||
failed: number;
|
||||
};
|
||||
scenarios: TelegramQaScenarioResult[];
|
||||
};
|
||||
|
||||
class TelegramQaCanaryError extends Error {
|
||||
phase: TelegramQaCanaryPhase;
|
||||
context: Record<string, string | number | undefined>;
|
||||
@@ -1785,6 +1766,7 @@ export async function runTelegramQaLive(params: {
|
||||
latestSutMessageId = canaryTiming.responseMessageId;
|
||||
scenarioResults.push({
|
||||
id: "telegram-canary",
|
||||
standardId: "canary",
|
||||
title: "Telegram canary",
|
||||
status: "pass",
|
||||
details: redactPublicMetadata
|
||||
@@ -1815,6 +1797,7 @@ export async function runTelegramQaLive(params: {
|
||||
});
|
||||
scenarioResults.push({
|
||||
id: "telegram-canary",
|
||||
standardId: "canary",
|
||||
title: "Telegram canary",
|
||||
status: "fail",
|
||||
details: canaryFailure,
|
||||
@@ -1910,6 +1893,7 @@ export async function runTelegramQaLive(params: {
|
||||
if (!lastMatched || !firstRequestStartedAt || lastSentMessageId === undefined) {
|
||||
const result = {
|
||||
id: scenario.id,
|
||||
standardId: scenario.standardId,
|
||||
title: scenario.title,
|
||||
status: "pass",
|
||||
details: "no reply",
|
||||
@@ -1933,6 +1917,7 @@ export async function runTelegramQaLive(params: {
|
||||
: `; ${scenarioSteps.filter((step) => step.expectReply).length} command replies matched`;
|
||||
const result = {
|
||||
id: scenario.id,
|
||||
standardId: scenario.standardId,
|
||||
title: scenario.title,
|
||||
status: "pass",
|
||||
details: redactPublicMetadata
|
||||
@@ -1958,6 +1943,7 @@ export async function runTelegramQaLive(params: {
|
||||
} catch (error) {
|
||||
const result = {
|
||||
id: scenario.id,
|
||||
standardId: scenario.standardId,
|
||||
title: scenario.title,
|
||||
status: "fail",
|
||||
details: formatErrorMessage(error),
|
||||
@@ -2006,28 +1992,22 @@ export async function runTelegramQaLive(params: {
|
||||
if (cleanupIssues.length > 0) {
|
||||
writeTelegramQaProgress(progressEnabled, `cleanup issues: count=${cleanupIssues.length}`);
|
||||
}
|
||||
const summary: TelegramQaSummary = {
|
||||
credentials: {
|
||||
source: credentialLease.source,
|
||||
kind: credentialLease.kind,
|
||||
role: credentialLease.role,
|
||||
ownerId: redactPublicMetadata ? undefined : credentialLease.ownerId,
|
||||
credentialId: redactPublicMetadata ? undefined : credentialLease.credentialId,
|
||||
},
|
||||
groupId: redactPublicMetadata ? "<redacted>" : runtimeEnv.groupId,
|
||||
startedAt,
|
||||
finishedAt,
|
||||
cleanupIssues: publishedCleanupIssues,
|
||||
counts: {
|
||||
total: scenarioResults.length,
|
||||
passed: passedCount,
|
||||
failed: failedCount,
|
||||
},
|
||||
scenarios: scenarioResults,
|
||||
};
|
||||
const reportPath = path.join(outputDir, "telegram-qa-report.md");
|
||||
const summaryPath = path.join(outputDir, "telegram-qa-summary.json");
|
||||
const summaryPath = path.join(outputDir, QA_EVIDENCE_FILENAME);
|
||||
const observedMessagesPath = path.join(outputDir, "telegram-qa-observed-messages.json");
|
||||
const evidence = buildLiveTransportEvidenceSummary({
|
||||
artifactPaths: [
|
||||
{ kind: "summary", path: path.basename(summaryPath) },
|
||||
{ kind: "report", path: path.basename(reportPath) },
|
||||
{ kind: "transport-observations", path: path.basename(observedMessagesPath) },
|
||||
],
|
||||
env: process.env,
|
||||
generatedAt: finishedAt,
|
||||
primaryModel,
|
||||
providerMode,
|
||||
checks: scenarioResults,
|
||||
transportId: "telegram",
|
||||
});
|
||||
await fs.writeFile(
|
||||
reportPath,
|
||||
`${renderTelegramQaMarkdown({
|
||||
@@ -2042,7 +2022,7 @@ export async function runTelegramQaLive(params: {
|
||||
})}\n`,
|
||||
{ encoding: "utf8", mode: 0o600 },
|
||||
);
|
||||
await fs.writeFile(summaryPath, `${JSON.stringify(summary, null, 2)}\n`, {
|
||||
await fs.writeFile(summaryPath, `${JSON.stringify(evidence, null, 2)}\n`, {
|
||||
encoding: "utf8",
|
||||
mode: 0o600,
|
||||
});
|
||||
|
||||
@@ -3,6 +3,7 @@ import fs from "node:fs/promises";
|
||||
import os from "node:os";
|
||||
import path from "node:path";
|
||||
import { afterEach, describe, expect, it, vi } from "vitest";
|
||||
import { QA_EVIDENCE_FILENAME } from "../../evidence-summary.js";
|
||||
import { runQaWhatsAppCommand } from "./cli.runtime.js";
|
||||
|
||||
const runWhatsAppQaLiveMock = vi.hoisted(() => vi.fn());
|
||||
@@ -38,24 +39,48 @@ afterEach(async () => {
|
||||
async function writeSummary(summary: unknown) {
|
||||
const outputDir = await fs.mkdtemp(path.join(os.tmpdir(), "openclaw-whatsapp-cli-"));
|
||||
tempDirs.push(outputDir);
|
||||
const summaryPath = path.join(outputDir, "whatsapp-qa-summary.json");
|
||||
const summaryPath = path.join(outputDir, QA_EVIDENCE_FILENAME);
|
||||
await fs.writeFile(summaryPath, `${JSON.stringify(summary, null, 2)}\n`, "utf8");
|
||||
return { outputDir, summaryPath };
|
||||
}
|
||||
|
||||
function makeEvidenceSummary(status: "pass" | "fail" | "blocked" | "skipped") {
|
||||
return {
|
||||
kind: "openclaw.qa.evidence-summary",
|
||||
schemaVersion: 2,
|
||||
generatedAt: "2026-05-01T00:00:00.000Z",
|
||||
entries: [
|
||||
{
|
||||
test: {
|
||||
kind: "live-transport-check",
|
||||
id: "whatsapp-mention-gating",
|
||||
title: "WhatsApp mention gating",
|
||||
},
|
||||
mapping: { profile: "release", coverage: [] },
|
||||
execution: {
|
||||
runner: "host",
|
||||
environment: { ref: null, os: "darwin", nodeVersion: "v24.0.0" },
|
||||
provider: {
|
||||
id: "openai",
|
||||
live: false,
|
||||
model: { name: null, ref: null },
|
||||
fixture: "mock-openai",
|
||||
},
|
||||
channel: { id: "whatsapp", live: true, driver: "native" },
|
||||
packageSource: { kind: "source-checkout" },
|
||||
artifacts: [],
|
||||
},
|
||||
result: { status },
|
||||
},
|
||||
],
|
||||
};
|
||||
}
|
||||
|
||||
describe("WhatsApp QA CLI runtime", () => {
|
||||
it("fails when a standard scenario is skipped by default", async () => {
|
||||
it("fails when a requirement is skipped by default", async () => {
|
||||
originalExitCode = process.exitCode;
|
||||
process.exitCode = undefined;
|
||||
const { outputDir, summaryPath } = await writeSummary({
|
||||
counts: { total: 1, passed: 0, failed: 0, skipped: 1 },
|
||||
scenarios: [
|
||||
{
|
||||
id: "whatsapp-mention-gating",
|
||||
status: "skip",
|
||||
},
|
||||
],
|
||||
});
|
||||
const { outputDir, summaryPath } = await writeSummary(makeEvidenceSummary("skipped"));
|
||||
runWhatsAppQaLiveMock.mockResolvedValueOnce({
|
||||
observedMessagesPath: path.join(outputDir, "observed.json"),
|
||||
reportPath: path.join(outputDir, "report.md"),
|
||||
@@ -71,10 +96,7 @@ describe("WhatsApp QA CLI runtime", () => {
|
||||
it("allows skipped scenarios when failures are explicitly allowed", async () => {
|
||||
originalExitCode = process.exitCode;
|
||||
process.exitCode = undefined;
|
||||
const { outputDir, summaryPath } = await writeSummary({
|
||||
counts: { total: 1, passed: 0, failed: 0, skipped: 1 },
|
||||
scenarios: [{ id: "whatsapp-mention-gating", status: "skip" }],
|
||||
});
|
||||
const { outputDir, summaryPath } = await writeSummary(makeEvidenceSummary("skipped"));
|
||||
runWhatsAppQaLiveMock.mockResolvedValueOnce({
|
||||
observedMessagesPath: path.join(outputDir, "observed.json"),
|
||||
reportPath: path.join(outputDir, "report.md"),
|
||||
|
||||
@@ -332,7 +332,7 @@ describe("WhatsApp QA live runtime", () => {
|
||||
expect(scenarios.map(({ id }) => id)).toEqual(["whatsapp-canary", "whatsapp-pairing-block"]);
|
||||
});
|
||||
|
||||
it("reports standard WhatsApp live transport scenario coverage", () => {
|
||||
it("reports WhatsApp live transport standard scenario coverage", () => {
|
||||
expect(testing.WHATSAPP_QA_STANDARD_SCENARIO_IDS).toEqual([
|
||||
"canary",
|
||||
"mention-gating",
|
||||
@@ -904,7 +904,7 @@ describe("WhatsApp QA live runtime", () => {
|
||||
expect(waitCallCount).toBe(2);
|
||||
});
|
||||
|
||||
it("selects native approval scenarios by id without changing standard coverage", () => {
|
||||
it("selects native approval scenarios by id without changing standard scenario coverage", () => {
|
||||
const scenarios = testing.findScenarios([
|
||||
"whatsapp-approval-exec-native",
|
||||
"whatsapp-approval-exec-reaction-native",
|
||||
@@ -1181,6 +1181,7 @@ describe("WhatsApp QA live runtime", () => {
|
||||
{
|
||||
id: "whatsapp-mention-gating",
|
||||
title: "WhatsApp group mention gating",
|
||||
standardId: "mention-gating",
|
||||
status: "fail",
|
||||
details: "setup exploded",
|
||||
},
|
||||
|
||||
@@ -15,6 +15,7 @@ import { formatErrorMessage } from "openclaw/plugin-sdk/error-runtime";
|
||||
import { normalizeStringEntries, uniqueStrings } from "openclaw/plugin-sdk/string-coerce-runtime";
|
||||
import { resolvePreferredOpenClawTmpDir } from "openclaw/plugin-sdk/temp-path";
|
||||
import { z } from "zod";
|
||||
import { QA_EVIDENCE_FILENAME, buildLiveTransportEvidenceSummary } from "../../evidence-summary.js";
|
||||
import { startQaGatewayChild } from "../../gateway-child.js";
|
||||
import { DEFAULT_QA_LIVE_PROVIDER_MODE } from "../../providers/index.js";
|
||||
import { fingerprintQaCredentialId } from "../../qa-credentials-fingerprint.runtime.js";
|
||||
@@ -27,7 +28,6 @@ import {
|
||||
import {
|
||||
acquireQaCredentialLease,
|
||||
startQaCredentialLeaseHeartbeat,
|
||||
type QaCredentialRole,
|
||||
} from "../shared/credential-lease.runtime.js";
|
||||
import {
|
||||
appendQaLiveLaneIssue as appendLiveLaneIssue,
|
||||
@@ -258,6 +258,7 @@ type WhatsAppQaScenarioResult = {
|
||||
responseObservedAt: string;
|
||||
source: "approval-request-to-resolution" | "request-to-observed-message";
|
||||
};
|
||||
standardId?: string;
|
||||
status: "fail" | "pass" | "skip";
|
||||
title: string;
|
||||
};
|
||||
@@ -271,29 +272,6 @@ export type WhatsAppQaRunResult = {
|
||||
summaryPath: string;
|
||||
};
|
||||
|
||||
type WhatsAppQaSummary = {
|
||||
cleanupIssues: string[];
|
||||
counts: {
|
||||
failed: number;
|
||||
passed: number;
|
||||
skipped: number;
|
||||
total: number;
|
||||
};
|
||||
credentials: {
|
||||
credentialFingerprint?: string;
|
||||
credentialId?: string;
|
||||
kind: string;
|
||||
ownerId?: string;
|
||||
role?: QaCredentialRole;
|
||||
source: "convex" | "env";
|
||||
};
|
||||
finishedAt: string;
|
||||
scenarios: WhatsAppQaScenarioResult[];
|
||||
startedAt: string;
|
||||
sutAccountId: string;
|
||||
sutPhoneE164: string;
|
||||
};
|
||||
|
||||
type WhatsAppCredentialLease = Awaited<
|
||||
ReturnType<typeof acquireQaCredentialLease<WhatsAppQaRuntimeEnv>>
|
||||
>;
|
||||
@@ -1427,14 +1405,6 @@ function inferWhatsAppCredentialSource(
|
||||
return normalized === "convex" ? "convex" : "env";
|
||||
}
|
||||
|
||||
function inferWhatsAppCredentialRole(value: string | undefined): QaCredentialRole | undefined {
|
||||
const normalized = value?.trim().toLowerCase();
|
||||
if (normalized === "ci" || normalized === "maintainer") {
|
||||
return normalized;
|
||||
}
|
||||
return undefined;
|
||||
}
|
||||
|
||||
function resolveWhatsAppMetadataRedaction(env: NodeJS.ProcessEnv = process.env) {
|
||||
const raw = env[QA_REDACT_PUBLIC_METADATA_ENV];
|
||||
return raw === undefined ? true : isTruthyOptIn(raw);
|
||||
@@ -2630,6 +2600,7 @@ async function runWhatsAppScenario(params: {
|
||||
return {
|
||||
id: params.scenario.id,
|
||||
title: params.scenario.title,
|
||||
standardId: params.scenario.standardId,
|
||||
status: "pass" as const,
|
||||
details: `${scenarioRun.approvalKind} approval ${approval.approvalId} resolved ${scenarioRun.decision} in ${approval.rttMs}ms`,
|
||||
rttMs: approval.rttMs,
|
||||
@@ -2733,6 +2704,7 @@ async function runWhatsAppScenario(params: {
|
||||
return {
|
||||
id: params.scenario.id,
|
||||
title: params.scenario.title,
|
||||
standardId: params.scenario.standardId,
|
||||
status: "pass" as const,
|
||||
details: "no reply",
|
||||
};
|
||||
@@ -2766,6 +2738,7 @@ async function runWhatsAppScenario(params: {
|
||||
return {
|
||||
id: params.scenario.id,
|
||||
title: params.scenario.title,
|
||||
standardId: params.scenario.standardId,
|
||||
status: "pass" as const,
|
||||
details: [`reply matched in ${rttMs}ms`, afterSendDetails, afterReplyDetails, batchDetails]
|
||||
.filter(Boolean)
|
||||
@@ -2945,6 +2918,7 @@ function createMissingGroupJidScenarioResult(params: {
|
||||
return {
|
||||
id: params.scenario.id,
|
||||
title: params.scenario.title,
|
||||
standardId: params.scenario.standardId,
|
||||
status: params.explicitScenarioSelection ? "fail" : "skip",
|
||||
details: params.explicitScenarioSelection
|
||||
? "requested scenario requires groupJid in the WhatsApp QA credential payload"
|
||||
@@ -2967,6 +2941,7 @@ function appendPreScenarioFailureResults(params: {
|
||||
params.scenarioResults.push({
|
||||
id: scenario.id,
|
||||
title: scenario.title,
|
||||
standardId: scenario.standardId,
|
||||
status: "fail",
|
||||
details: params.details,
|
||||
});
|
||||
@@ -3025,7 +3000,6 @@ export async function runWhatsAppQaLive(params: {
|
||||
const scenarios = findScenarios(params.scenarioIds, providerMode);
|
||||
const explicitScenarioSelection = (params.scenarioIds?.length ?? 0) > 0;
|
||||
const requestedCredentialSource = inferWhatsAppCredentialSource(params.credentialSource);
|
||||
const requestedCredentialRole = inferWhatsAppCredentialRole(params.credentialRole);
|
||||
const redactPublicMetadata = resolveWhatsAppMetadataRedaction();
|
||||
const includeObservedMessageContent = isTruthyOptIn(process.env[WHATSAPP_QA_CAPTURE_CONTENT_ENV]);
|
||||
const startedAt = new Date().toISOString();
|
||||
@@ -3156,6 +3130,7 @@ export async function runWhatsAppQaLive(params: {
|
||||
const result: WhatsAppQaScenarioResult = {
|
||||
id: scenario.id,
|
||||
title: scenario.title,
|
||||
standardId: scenario.standardId,
|
||||
status: "fail",
|
||||
details:
|
||||
driverAttempt > 1
|
||||
@@ -3228,11 +3203,8 @@ export async function runWhatsAppQaLive(params: {
|
||||
|
||||
const finishedAt = new Date().toISOString();
|
||||
const reportPath = path.join(outputDir, "whatsapp-qa-report.md");
|
||||
const summaryPath = path.join(outputDir, "whatsapp-qa-summary.json");
|
||||
const summaryPath = path.join(outputDir, QA_EVIDENCE_FILENAME);
|
||||
const observedMessagesPath = path.join(outputDir, "whatsapp-qa-observed-messages.json");
|
||||
const passed = scenarioResults.filter((entry) => entry.status === "pass").length;
|
||||
const failed = scenarioResults.filter((entry) => entry.status === "fail").length;
|
||||
const skipped = scenarioResults.filter((entry) => entry.status === "skip").length;
|
||||
const credentialFingerprint = fingerprintQaCredentialId(credentialLease?.credentialId);
|
||||
const publishedCleanupIssues = redactPublicMetadata
|
||||
? redactQaLiveLaneIssues(cleanupIssues)
|
||||
@@ -3240,36 +3212,19 @@ export async function runWhatsAppQaLive(params: {
|
||||
const publishedScenarioResults = redactPublicMetadata
|
||||
? redactWhatsAppQaScenarioResults(scenarioResults)
|
||||
: scenarioResults;
|
||||
const summary: WhatsAppQaSummary = {
|
||||
credentials: credentialLease
|
||||
? {
|
||||
source: credentialLease.source,
|
||||
kind: credentialLease.kind,
|
||||
role: credentialLease.role,
|
||||
credentialFingerprint,
|
||||
credentialId: redactPublicMetadata ? undefined : credentialLease.credentialId,
|
||||
ownerId: redactPublicMetadata ? undefined : credentialLease.ownerId,
|
||||
}
|
||||
: {
|
||||
source: requestedCredentialSource,
|
||||
kind: "whatsapp",
|
||||
role: requestedCredentialRole,
|
||||
},
|
||||
sutAccountId,
|
||||
sutPhoneE164: redactPublicMetadata
|
||||
? "<redacted>"
|
||||
: (runtimeEnv?.sutPhoneE164 ?? "<unavailable>"),
|
||||
startedAt,
|
||||
finishedAt,
|
||||
cleanupIssues: publishedCleanupIssues,
|
||||
counts: {
|
||||
total: scenarioResults.length,
|
||||
passed,
|
||||
failed,
|
||||
skipped,
|
||||
},
|
||||
scenarios: publishedScenarioResults,
|
||||
};
|
||||
const evidence = buildLiveTransportEvidenceSummary({
|
||||
artifactPaths: [
|
||||
{ kind: "summary", path: path.basename(summaryPath) },
|
||||
{ kind: "report", path: path.basename(reportPath) },
|
||||
{ kind: "transport-observations", path: path.basename(observedMessagesPath) },
|
||||
],
|
||||
checks: publishedScenarioResults,
|
||||
env: process.env,
|
||||
generatedAt: finishedAt,
|
||||
primaryModel,
|
||||
providerMode,
|
||||
transportId: "whatsapp",
|
||||
});
|
||||
await fs.writeFile(
|
||||
observedMessagesPath,
|
||||
`${JSON.stringify(
|
||||
@@ -3282,7 +3237,7 @@ export async function runWhatsAppQaLive(params: {
|
||||
2,
|
||||
)}\n`,
|
||||
);
|
||||
await fs.writeFile(summaryPath, `${JSON.stringify(summary, null, 2)}\n`);
|
||||
await fs.writeFile(summaryPath, `${JSON.stringify(evidence, null, 2)}\n`);
|
||||
await fs.writeFile(
|
||||
reportPath,
|
||||
`${renderWhatsAppQaMarkdown({
|
||||
|
||||
@@ -3,6 +3,7 @@ import fs from "node:fs/promises";
|
||||
import os from "node:os";
|
||||
import path from "node:path";
|
||||
import { afterEach, beforeEach, describe, expect, it, vi } from "vitest";
|
||||
import { QA_EVIDENCE_FILENAME, buildLiveTransportEvidenceSummary } from "../evidence-summary.js";
|
||||
import { runMantisBeforeAfter } from "./run.runtime.js";
|
||||
|
||||
describe("mantis before/after runtime", () => {
|
||||
@@ -32,25 +33,31 @@ describe("mantis before/after runtime", () => {
|
||||
const videoPath = path.join(outputDir, `${lane}-timeline.mp4`);
|
||||
await fs.writeFile(screenshotPath, `${lane} screenshot`);
|
||||
await fs.writeFile(videoPath, `${lane} video`);
|
||||
await fs.writeFile(
|
||||
path.join(outputDir, "discord-qa-summary.json"),
|
||||
`${JSON.stringify(
|
||||
const summary = buildLiveTransportEvidenceSummary({
|
||||
artifactPaths: [
|
||||
{ kind: "summary", path: QA_EVIDENCE_FILENAME },
|
||||
{ kind: "report", path: "discord-qa-report.md" },
|
||||
],
|
||||
checks: [
|
||||
{
|
||||
scenarios: [
|
||||
{
|
||||
artifactPaths: { screenshot: screenshotPath, video: videoPath },
|
||||
details:
|
||||
lane === "baseline"
|
||||
? "reaction timeline missing thinking/done"
|
||||
: "reaction timeline matched queued -> thinking -> done",
|
||||
id: "discord-status-reactions-tool-only",
|
||||
status: lane === "baseline" ? "fail" : "pass",
|
||||
},
|
||||
],
|
||||
artifactPaths: { screenshot: screenshotPath, video: videoPath },
|
||||
details:
|
||||
lane === "baseline"
|
||||
? "reaction timeline missing thinking/done"
|
||||
: "reaction timeline matched queued -> thinking -> done",
|
||||
id: "discord-status-reactions-tool-only",
|
||||
status: lane === "baseline" ? "fail" : "pass",
|
||||
title: "Discord explicit status reactions run in tool-only reply mode",
|
||||
},
|
||||
null,
|
||||
2,
|
||||
)}\n`,
|
||||
],
|
||||
generatedAt: "2026-05-03T12:00:00.000Z",
|
||||
primaryModel: "openai/gpt-5.4",
|
||||
providerMode: "live-frontier",
|
||||
transportId: "discord",
|
||||
});
|
||||
await fs.writeFile(
|
||||
path.join(outputDir, QA_EVIDENCE_FILENAME),
|
||||
`${JSON.stringify(summary, null, 2)}\n`,
|
||||
);
|
||||
});
|
||||
|
||||
|
||||
@@ -4,6 +4,7 @@ import fs from "node:fs/promises";
|
||||
import path from "node:path";
|
||||
import { formatErrorMessage } from "openclaw/plugin-sdk/error-runtime";
|
||||
import { ensureRepoBoundDirectory, resolveRepoRelativeOutputDir } from "../cli-paths.js";
|
||||
import { QA_EVIDENCE_FILENAME, validateQaEvidenceSummaryJson } from "../evidence-summary.js";
|
||||
|
||||
export type MantisBeforeAfterOptions = {
|
||||
allowFailures?: boolean;
|
||||
@@ -47,6 +48,14 @@ type DiscordQaSummary = {
|
||||
}[];
|
||||
};
|
||||
|
||||
type NormalizedScenarioSummary = {
|
||||
details?: string;
|
||||
screenshotPath?: string;
|
||||
status: string;
|
||||
summaryPath: string;
|
||||
videoPath?: string;
|
||||
};
|
||||
|
||||
type LaneResult = {
|
||||
outputDir: string;
|
||||
scenarioDetails?: string;
|
||||
@@ -194,6 +203,18 @@ async function readLaneResult(params: {
|
||||
publishedLaneDir: string;
|
||||
scenario: string;
|
||||
}) {
|
||||
const normalized = await readNormalizedLaneResult(params);
|
||||
if (normalized) {
|
||||
return {
|
||||
outputDir: params.publishedLaneDir,
|
||||
scenarioDetails: normalized.details,
|
||||
screenshotPath: normalized.screenshotPath,
|
||||
status: normalized.status,
|
||||
summaryPath: normalized.summaryPath,
|
||||
videoPath: normalized.videoPath,
|
||||
} satisfies LaneResult;
|
||||
}
|
||||
|
||||
const summaryPath = path.join(params.publishedLaneDir, "discord-qa-summary.json");
|
||||
const summary = JSON.parse(await fs.readFile(summaryPath, "utf8")) as DiscordQaSummary;
|
||||
const scenarioSummary =
|
||||
@@ -211,6 +232,35 @@ async function readLaneResult(params: {
|
||||
} satisfies LaneResult;
|
||||
}
|
||||
|
||||
async function readNormalizedLaneResult(params: {
|
||||
publishedLaneDir: string;
|
||||
scenario: string;
|
||||
}): Promise<NormalizedScenarioSummary | undefined> {
|
||||
const summaryPath = path.join(params.publishedLaneDir, QA_EVIDENCE_FILENAME);
|
||||
let rawSummary: string;
|
||||
try {
|
||||
rawSummary = await fs.readFile(summaryPath, "utf8");
|
||||
} catch (error) {
|
||||
if ((error as NodeJS.ErrnoException).code === "ENOENT") {
|
||||
return undefined;
|
||||
}
|
||||
throw error;
|
||||
}
|
||||
|
||||
const summary = validateQaEvidenceSummaryJson(JSON.parse(rawSummary));
|
||||
const entry =
|
||||
summary.entries.find((candidate) => candidate.test.id === params.scenario) ??
|
||||
summary.entries[0];
|
||||
const artifacts = entry?.execution.artifacts ?? [];
|
||||
return {
|
||||
details: entry?.result.failure?.reason,
|
||||
screenshotPath: artifacts.find((artifact) => artifact.kind === "screenshot")?.path,
|
||||
status: entry?.result.status ?? "fail",
|
||||
summaryPath,
|
||||
videoPath: artifacts.find((artifact) => artifact.kind === "video")?.path,
|
||||
};
|
||||
}
|
||||
|
||||
function renderReport(params: {
|
||||
baseline: LaneResult;
|
||||
candidate: LaneResult;
|
||||
|
||||
@@ -12,6 +12,7 @@ import type { QaProviderMode } from "./model-selection.js";
|
||||
import { resolveQaForwardedLiveEnv, resolveQaLiveProviderConfigPath } from "./providers/env.js";
|
||||
import { DEFAULT_QA_LIVE_PROVIDER_MODE, getQaProvider } from "./providers/index.js";
|
||||
import type { RuntimeId } from "./runtime-parity.js";
|
||||
import { shellQuote } from "./shell-quote.js";
|
||||
|
||||
const MULTIPASS_MOUNTED_REPO_PATH = "/workspace/openclaw-host";
|
||||
const MULTIPASS_GUEST_REPO_PATH = "/workspace/openclaw";
|
||||
@@ -107,10 +108,6 @@ type RenderGuestScriptOptions = {
|
||||
redactSecrets?: boolean;
|
||||
};
|
||||
|
||||
function shellQuote(value: string) {
|
||||
return `'${value.replaceAll("'", `'"'"'`)}'`;
|
||||
}
|
||||
|
||||
function createOutputStamp() {
|
||||
return new Date().toISOString().replaceAll(":", "").replaceAll(".", "").replace("T", "-");
|
||||
}
|
||||
|
||||
@@ -3,6 +3,7 @@ import { setTimeout as sleep } from "node:timers/promises";
|
||||
import type { OpenClawConfig } from "openclaw/plugin-sdk/config-contracts";
|
||||
import { formatErrorMessage } from "openclaw/plugin-sdk/error-runtime";
|
||||
import type { QaBusState } from "./bus-state.js";
|
||||
import { QaSuiteInfraError } from "./errors.js";
|
||||
import { getQaProvider } from "./providers/index.js";
|
||||
import { QaStateBackedTransportAdapter } from "./qa-transport.js";
|
||||
import type {
|
||||
@@ -65,7 +66,8 @@ async function waitForQaChannelReady(params: {
|
||||
await sleep(pollIntervalMs);
|
||||
}
|
||||
|
||||
throw new Error(
|
||||
throw new QaSuiteInfraError(
|
||||
"transport_ready_timeout",
|
||||
[
|
||||
`timed out after ${timeoutMs}ms waiting for qa-channel ready`,
|
||||
`last status: ${lastAccountStatus}`,
|
||||
|
||||
@@ -26,6 +26,7 @@ const scenarios = [
|
||||
surface: "dm",
|
||||
objective: "test DM",
|
||||
successCriteria: ["reply"],
|
||||
execution: { kind: "flow" as const },
|
||||
},
|
||||
{
|
||||
id: "thread-lifecycle",
|
||||
@@ -33,6 +34,18 @@ const scenarios = [
|
||||
surface: "thread",
|
||||
objective: "test thread",
|
||||
successCriteria: ["thread reply"],
|
||||
execution: { kind: "flow" as const },
|
||||
},
|
||||
{
|
||||
id: "control-ui-chat-flow-playwright",
|
||||
title: "Control UI Playwright",
|
||||
surface: "control-ui",
|
||||
objective: "test Control UI",
|
||||
successCriteria: ["playwright pass"],
|
||||
execution: {
|
||||
kind: "playwright" as const,
|
||||
path: "ui/src/ui/e2e/chat-flow.e2e.test.ts",
|
||||
},
|
||||
},
|
||||
];
|
||||
|
||||
@@ -44,7 +57,7 @@ describe("qa run config", () => {
|
||||
);
|
||||
});
|
||||
|
||||
it("creates a live-by-default selection that arms every scenario", () => {
|
||||
it("creates a live-by-default selection that arms flow scenarios", () => {
|
||||
expect(createDefaultQaRunSelection(scenarios)).toEqual({
|
||||
providerMode: "live-frontier",
|
||||
primaryModel: "openai/gpt-5.5",
|
||||
@@ -100,6 +113,17 @@ describe("qa run config", () => {
|
||||
).toEqual(["dm-chat-baseline", "thread-lifecycle"]);
|
||||
});
|
||||
|
||||
it("filters non-flow scenarios from lab runner selections", () => {
|
||||
expect(
|
||||
normalizeQaRunSelection(
|
||||
{
|
||||
scenarioIds: ["control-ui-chat-flow-playwright", "thread-lifecycle"],
|
||||
},
|
||||
scenarios,
|
||||
).scenarioIds,
|
||||
).toEqual(["thread-lifecycle"]);
|
||||
});
|
||||
|
||||
it("keeps idle snapshots on static defaults so startup does not inspect auth profiles", () => {
|
||||
defaultQaRuntimeModelForMode.mockReturnValue("openai/gpt-5.5");
|
||||
defaultQaRuntimeModelForMode.mockClear();
|
||||
|
||||
@@ -25,6 +25,7 @@ type QaLabRunSelection = {
|
||||
|
||||
type QaLabRunArtifacts = {
|
||||
outputDir: string;
|
||||
evidencePath: string;
|
||||
reportPath: string;
|
||||
summaryPath: string;
|
||||
watchUrl: string;
|
||||
@@ -49,6 +50,14 @@ function defaultStaticModelForMode(mode: QaProviderMode, alternate = false) {
|
||||
return defaultStaticQaModelForMode(mode, alternate ? { alternate: true } : undefined);
|
||||
}
|
||||
|
||||
function qaLabFlowScenarioIds(scenarios: QaSeedScenario[]) {
|
||||
return scenarios
|
||||
.filter(
|
||||
(scenario) => scenario.execution?.kind === undefined || scenario.execution.kind === "flow",
|
||||
)
|
||||
.map((scenario) => scenario.id);
|
||||
}
|
||||
|
||||
export function createDefaultQaRunSelection(
|
||||
scenarios: QaSeedScenario[],
|
||||
options?: { resolveDefaultModel?: QaDefaultModelResolver },
|
||||
@@ -60,7 +69,7 @@ export function createDefaultQaRunSelection(
|
||||
primaryModel: resolveDefaultModel(providerMode),
|
||||
alternateModel: resolveDefaultModel(providerMode, true),
|
||||
fastMode: true,
|
||||
scenarioIds: scenarios.map((scenario) => scenario.id),
|
||||
scenarioIds: qaLabFlowScenarioIds(scenarios),
|
||||
};
|
||||
}
|
||||
|
||||
@@ -81,14 +90,15 @@ function normalizeModel(input: unknown, fallback: string) {
|
||||
}
|
||||
|
||||
function normalizeScenarioIds(input: unknown, scenarios: QaSeedScenario[]) {
|
||||
const availableIds = new Set(scenarios.map((scenario) => scenario.id));
|
||||
const defaultScenarioIds = qaLabFlowScenarioIds(scenarios);
|
||||
const availableIds = new Set(defaultScenarioIds);
|
||||
const requestedIds = Array.isArray(input)
|
||||
? input
|
||||
.map((value) => (typeof value === "string" ? value.trim() : ""))
|
||||
.filter((value) => value.length > 0)
|
||||
: [];
|
||||
const selectedIds = uniqueStrings(requestedIds.filter((id) => availableIds.has(id)));
|
||||
return selectedIds.length > 0 ? selectedIds : scenarios.map((scenario) => scenario.id);
|
||||
return selectedIds.length > 0 ? selectedIds : defaultScenarioIds;
|
||||
}
|
||||
|
||||
export function normalizeQaRunSelection(
|
||||
|
||||
@@ -34,10 +34,12 @@ describe("qa scenario catalog", () => {
|
||||
pack.scenarios
|
||||
.filter((scenario) => scenario.execution?.kind !== "flow")
|
||||
.map((scenario) => scenario.id),
|
||||
).toStrictEqual([]);
|
||||
).toStrictEqual(["control-ui-chat-flow-playwright"]);
|
||||
expect(
|
||||
pack.scenarios.filter((scenario) => (scenario.execution.flow?.steps.length ?? 0) > 0),
|
||||
).not.toStrictEqual([]);
|
||||
pack.scenarios
|
||||
.filter((scenario) => scenario.execution.kind === "flow")
|
||||
.every((scenario) => (scenario.execution.flow?.steps.length ?? 0) > 0),
|
||||
).toBe(true);
|
||||
expect(
|
||||
pack.scenarios
|
||||
.filter((scenario) => !(scenario.coverage?.primary.length ?? 0))
|
||||
@@ -109,6 +111,18 @@ describe("qa scenario catalog", () => {
|
||||
expect(scenario.gatewayRuntime?.forwardHostHome).toBe(true);
|
||||
});
|
||||
|
||||
it("loads Playwright execution scenarios from markdown", () => {
|
||||
const scenario = readQaScenarioById("control-ui-chat-flow-playwright");
|
||||
|
||||
expect(scenario.execution.kind).toBe("playwright");
|
||||
if (scenario.execution.kind !== "playwright") {
|
||||
throw new Error("expected Playwright scenario execution");
|
||||
}
|
||||
expect(scenario.execution.path).toBe("ui/src/ui/e2e/chat-flow.e2e.test.ts");
|
||||
expect(scenario.execution.flow).toBeUndefined();
|
||||
expect(scenario.coverage?.primary).toContain("ui.control");
|
||||
});
|
||||
|
||||
it("loads runtime parity tier metadata for first-hour and soak lanes", () => {
|
||||
const firstHour = readQaScenarioById("runtime-first-hour-20-turn");
|
||||
const soak = readQaScenarioById("runtime-soak-100-turn");
|
||||
|
||||
@@ -3,6 +3,7 @@ import fs from "node:fs";
|
||||
import path from "node:path";
|
||||
import YAML from "yaml";
|
||||
import { z } from "zod";
|
||||
import { isRepoRootRelativeRef } from "./cli-paths.js";
|
||||
|
||||
export const DEFAULT_QA_AGENT_IDENTITY_MARKDOWN = `# Dev C-3PO
|
||||
|
||||
@@ -46,12 +47,39 @@ const qaScenarioConfigSchema = z.record(z.string(), z.unknown()).superRefine((co
|
||||
}
|
||||
});
|
||||
|
||||
const qaScenarioExecutionSchema = z.object({
|
||||
const qaScenarioRepoRefSchema = z
|
||||
.string()
|
||||
.trim()
|
||||
.min(1)
|
||||
.regex(/^[A-Za-z0-9._/-]+$/, {
|
||||
message: "repo refs must be repo-root relative paths",
|
||||
})
|
||||
.refine(isRepoRootRelativeRef, {
|
||||
message: "repo refs must not be absolute or contain parent-directory segments",
|
||||
});
|
||||
|
||||
const qaFlowScenarioExecutionSchema = z.object({
|
||||
kind: z.literal("flow").default("flow"),
|
||||
summary: z.string().trim().min(1).optional(),
|
||||
config: qaScenarioConfigSchema.optional(),
|
||||
});
|
||||
|
||||
const qaTestFileScenarioExecutionBaseSchema = z.object({
|
||||
summary: z.string().trim().min(1).optional(),
|
||||
path: qaScenarioRepoRefSchema,
|
||||
config: qaScenarioConfigSchema.optional(),
|
||||
});
|
||||
|
||||
const qaTestFileScenarioExecutionSchema = z.discriminatedUnion("kind", [
|
||||
qaTestFileScenarioExecutionBaseSchema.extend({ kind: z.literal("vitest") }),
|
||||
qaTestFileScenarioExecutionBaseSchema.extend({ kind: z.literal("playwright") }),
|
||||
]);
|
||||
|
||||
const qaScenarioExecutionSchema = z.union([
|
||||
qaFlowScenarioExecutionSchema,
|
||||
qaTestFileScenarioExecutionSchema,
|
||||
]);
|
||||
|
||||
const qaCoverageIdSchema = z
|
||||
.string()
|
||||
.trim()
|
||||
@@ -377,13 +405,14 @@ export function readQaScenarioPack(): QaScenarioPack {
|
||||
parsedScenario.execution ?? {},
|
||||
relativePath,
|
||||
);
|
||||
const flow = extractQaScenarioFlow(content, relativePath);
|
||||
const flow =
|
||||
execution.kind === "flow" ? extractQaScenarioFlow(content, relativePath) : undefined;
|
||||
return {
|
||||
...parsedScenario,
|
||||
sourcePath: relativePath,
|
||||
execution: {
|
||||
...execution,
|
||||
flow,
|
||||
...(flow ? { flow } : {}),
|
||||
},
|
||||
} satisfies QaSeedScenarioWithSource;
|
||||
})(),
|
||||
|
||||
734
extensions/qa-lab/src/scorecard-taxonomy.ts
Normal file
734
extensions/qa-lab/src/scorecard-taxonomy.ts
Normal file
@@ -0,0 +1,734 @@
|
||||
// Qa Lab plugin module validates the scorecard evidence mapping overlay.
|
||||
import fs from "node:fs";
|
||||
import path from "node:path";
|
||||
import YAML from "yaml";
|
||||
import { z } from "zod";
|
||||
import { isRepoRootRelativeRef } from "./cli-paths.js";
|
||||
import type { QaSeedScenarioWithSource } from "./scenario-catalog.js";
|
||||
|
||||
export const QA_SCORECARD_TAXONOMY_PATH = "taxonomy-mappings.yaml";
|
||||
export const QA_MATURITY_TAXONOMY_PATH = "taxonomy.yaml";
|
||||
|
||||
const qaScorecardIdSchema = z
|
||||
.string()
|
||||
.trim()
|
||||
.regex(/^[a-z0-9]+(?:[.-][a-z0-9]+)*$/, {
|
||||
message: "scorecard and coverage ids must use lowercase dotted or dashed tokens",
|
||||
});
|
||||
|
||||
const qaScorecardRepoRefSchema = z
|
||||
.string()
|
||||
.trim()
|
||||
.min(1)
|
||||
.regex(/^[A-Za-z0-9._/-]+$/, {
|
||||
message: "repo refs must be repo-root relative paths",
|
||||
})
|
||||
.refine(isRepoRootRelativeRef, {
|
||||
message: "repo refs must not be absolute or contain parent-directory segments",
|
||||
});
|
||||
|
||||
const qaScorecardFreshnessRuleSchema = z.enum([
|
||||
"target-ref",
|
||||
"target-ref-and-release-package",
|
||||
"release-candidate",
|
||||
"latest-advisory-run",
|
||||
]);
|
||||
|
||||
const qaScorecardSupportStatusSchema = z.enum(["lts-included", "deferred", "advisory"]);
|
||||
|
||||
const qaScorecardTaxonomyRefSchema = z
|
||||
.object({
|
||||
sourcePath: qaScorecardRepoRefSchema,
|
||||
version: z.number().int().positive(),
|
||||
processVersion: z.number().int().positive(),
|
||||
snapshotDate: z.string().trim().min(1),
|
||||
sourceRef: z.string().trim().min(1),
|
||||
})
|
||||
.strict();
|
||||
|
||||
const qaScorecardProfileSchema = z.object({
|
||||
id: qaScorecardIdSchema,
|
||||
description: z.string().trim().min(1),
|
||||
categoryIds: z.array(qaScorecardIdSchema).default([]),
|
||||
});
|
||||
|
||||
const qaScorecardCategorySchema = z.object({
|
||||
id: qaScorecardIdSchema,
|
||||
taxonomySurfaceId: qaScorecardIdSchema,
|
||||
taxonomyCategoryName: z.string().trim().min(1),
|
||||
supportStatus: qaScorecardSupportStatusSchema,
|
||||
releaseBlocking: z.boolean(),
|
||||
requirement: z.string().trim().min(1),
|
||||
evidenceRequired: z.string().trim().min(1),
|
||||
evidence: z.object({
|
||||
profiles: z.array(qaScorecardIdSchema).default([]),
|
||||
liveProofRequired: z.boolean(),
|
||||
freshness: qaScorecardFreshnessRuleSchema,
|
||||
coverageIds: z.array(qaScorecardIdSchema).default([]),
|
||||
scenarioRefs: z.array(qaScorecardRepoRefSchema).default([]),
|
||||
docsRefs: z.array(qaScorecardRepoRefSchema).default([]),
|
||||
codeRefs: z.array(qaScorecardRepoRefSchema).default([]),
|
||||
notes: z.string().trim().min(1).optional(),
|
||||
}),
|
||||
});
|
||||
|
||||
const qaScorecardTaxonomySchema = z
|
||||
.object({
|
||||
version: z.literal(1),
|
||||
id: qaScorecardIdSchema,
|
||||
title: z.string().trim().min(1),
|
||||
taxonomy: qaScorecardTaxonomyRefSchema,
|
||||
scoreSnapshotRef: qaScorecardRepoRefSchema.optional(),
|
||||
status: z.enum(["initial", "candidate", "active"]),
|
||||
notes: z.string().trim().min(1).optional(),
|
||||
profiles: z.array(qaScorecardProfileSchema).min(1),
|
||||
categories: z.array(qaScorecardCategorySchema).min(1),
|
||||
})
|
||||
.superRefine((taxonomy, ctx) => {
|
||||
const seenProfileIds = new Set<string>();
|
||||
for (const [profileIndex, profile] of taxonomy.profiles.entries()) {
|
||||
if (seenProfileIds.has(profile.id)) {
|
||||
ctx.addIssue({
|
||||
code: z.ZodIssueCode.custom,
|
||||
path: ["profiles", profileIndex, "id"],
|
||||
message: `duplicate scorecard profile id: ${profile.id}`,
|
||||
});
|
||||
}
|
||||
seenProfileIds.add(profile.id);
|
||||
|
||||
const seenProfileCategoryIds = new Set<string>();
|
||||
for (const [categoryIndex, categoryId] of profile.categoryIds.entries()) {
|
||||
if (seenProfileCategoryIds.has(categoryId)) {
|
||||
ctx.addIssue({
|
||||
code: z.ZodIssueCode.custom,
|
||||
path: ["profiles", profileIndex, "categoryIds", categoryIndex],
|
||||
message: `duplicate category id in profile ${profile.id}: ${categoryId}`,
|
||||
});
|
||||
}
|
||||
seenProfileCategoryIds.add(categoryId);
|
||||
}
|
||||
}
|
||||
|
||||
const seenCategoryIds = new Set<string>();
|
||||
for (const [categoryIndex, category] of taxonomy.categories.entries()) {
|
||||
if (seenCategoryIds.has(category.id)) {
|
||||
ctx.addIssue({
|
||||
code: z.ZodIssueCode.custom,
|
||||
path: ["categories", categoryIndex, "id"],
|
||||
message: `duplicate scorecard category id: ${category.id}`,
|
||||
});
|
||||
}
|
||||
seenCategoryIds.add(category.id);
|
||||
|
||||
if (category.supportStatus === "lts-included" && !category.releaseBlocking) {
|
||||
ctx.addIssue({
|
||||
code: z.ZodIssueCode.custom,
|
||||
path: ["categories", categoryIndex, "releaseBlocking"],
|
||||
message: `LTS-included category ${category.id} must be release-blocking`,
|
||||
});
|
||||
}
|
||||
if (category.supportStatus !== "lts-included" && category.releaseBlocking) {
|
||||
ctx.addIssue({
|
||||
code: z.ZodIssueCode.custom,
|
||||
path: ["categories", categoryIndex, "releaseBlocking"],
|
||||
message: `${category.supportStatus} category ${category.id} must not be release-blocking`,
|
||||
});
|
||||
}
|
||||
|
||||
const seenCoverageIds = new Set<string>();
|
||||
for (const [coverageIndex, coverageId] of category.evidence.coverageIds.entries()) {
|
||||
if (seenCoverageIds.has(coverageId)) {
|
||||
ctx.addIssue({
|
||||
code: z.ZodIssueCode.custom,
|
||||
path: ["categories", categoryIndex, "evidence", "coverageIds", coverageIndex],
|
||||
message: `duplicate coverage id in category ${category.id}: ${coverageId}`,
|
||||
});
|
||||
}
|
||||
seenCoverageIds.add(coverageId);
|
||||
}
|
||||
}
|
||||
});
|
||||
|
||||
const qaMaturityCategorySchema = z.object({
|
||||
name: z.string().trim().min(1),
|
||||
});
|
||||
|
||||
const qaMaturitySurfaceSchema = z.object({
|
||||
id: qaScorecardIdSchema,
|
||||
name: z.string().trim().min(1),
|
||||
level: z.string().trim().min(1).optional(),
|
||||
level_code: z.string().trim().min(1).optional(),
|
||||
categories: z.array(qaMaturityCategorySchema).default([]),
|
||||
});
|
||||
|
||||
const qaMaturityTaxonomySchema = z.object({
|
||||
version: z.number(),
|
||||
title: z.string().trim().min(1),
|
||||
surfaces: z.array(qaMaturitySurfaceSchema).default([]),
|
||||
});
|
||||
|
||||
export type QaScorecardTaxonomy = z.infer<typeof qaScorecardTaxonomySchema>;
|
||||
export type QaScorecardTaxonomyCategory = QaScorecardTaxonomy["categories"][number];
|
||||
type QaMaturityTaxonomy = z.infer<typeof qaMaturityTaxonomySchema>;
|
||||
|
||||
export type QaScorecardValidationIssueCode =
|
||||
| "coverage-id-not-found"
|
||||
| "scenario-ref-not-found"
|
||||
| "scenario-ref-not-covered-by-category"
|
||||
| "docs-ref-not-found"
|
||||
| "code-ref-not-found"
|
||||
| "taxonomy-ref-not-found"
|
||||
| "taxonomy-category-ref-not-found"
|
||||
| "profile-category-ref-not-found"
|
||||
| "score-snapshot-ref-not-found"
|
||||
| "blocking-category-without-evidence-mapping"
|
||||
| "non-advisory-category-missing-profile-membership"
|
||||
| "release-blocking-category-missing-release-profile"
|
||||
| "advisory-category-has-profile-membership"
|
||||
| "profile-ref-not-found"
|
||||
| "category-profile-missing-top-level-membership"
|
||||
| "profile-membership-missing-category-profile"
|
||||
| "taxonomy-fixture-not-found";
|
||||
|
||||
export type QaScorecardValidationIssue = {
|
||||
code: QaScorecardValidationIssueCode;
|
||||
severity: "warning";
|
||||
categoryId?: string;
|
||||
ref?: string;
|
||||
message: string;
|
||||
};
|
||||
|
||||
export type QaScorecardCategoryMappingReport = {
|
||||
id: string;
|
||||
taxonomySurfaceId: string;
|
||||
taxonomyCategoryName: string;
|
||||
supportStatus: string;
|
||||
releaseBlocking: boolean;
|
||||
mappingStatus: "mapped" | "partial" | "missing";
|
||||
profiles: string[];
|
||||
liveProofRequired: boolean;
|
||||
freshness: string;
|
||||
coverageIds: string[];
|
||||
scenarioRefs: string[];
|
||||
missingCoverageIds: string[];
|
||||
missingScenarioRefs: string[];
|
||||
};
|
||||
|
||||
export type QaScorecardProfileReport = {
|
||||
id: string;
|
||||
categoryIds: string[];
|
||||
};
|
||||
|
||||
export type QaScorecardTaxonomyReport = {
|
||||
taxonomyPath: string | null;
|
||||
taxonomyId: string | null;
|
||||
title: string | null;
|
||||
taxonomy: {
|
||||
sourcePath: string;
|
||||
version: number;
|
||||
processVersion: number;
|
||||
snapshotDate: string;
|
||||
sourceRef: string;
|
||||
} | null;
|
||||
scoreSnapshotRef: string | null;
|
||||
status: string | null;
|
||||
profileCount: number;
|
||||
profiles: QaScorecardProfileReport[];
|
||||
categoryCount: number;
|
||||
releaseBlockingCategoryCount: number;
|
||||
advisoryCategoryCount: number;
|
||||
ltsIncludedCategoryCount: number;
|
||||
deferredCategoryCount: number;
|
||||
mappedCoverageIdCount: number;
|
||||
mappedScenarioCount: number;
|
||||
unmappedCoverageIdCount: number;
|
||||
unmappedCoverageIds: string[];
|
||||
validationIssueCount: number;
|
||||
validationIssues: QaScorecardValidationIssue[];
|
||||
categories: QaScorecardCategoryMappingReport[];
|
||||
};
|
||||
|
||||
function walkUpDirectories(start: string): string[] {
|
||||
const roots: string[] = [];
|
||||
let current = path.resolve(start);
|
||||
while (true) {
|
||||
roots.push(current);
|
||||
const parent = path.dirname(current);
|
||||
if (parent === current) {
|
||||
return roots;
|
||||
}
|
||||
current = parent;
|
||||
}
|
||||
}
|
||||
|
||||
function resolveRepoPath(relativePath: string, kind: "file" | "directory" = "file") {
|
||||
for (const dir of walkUpDirectories(import.meta.dirname)) {
|
||||
const candidate = path.join(dir, relativePath);
|
||||
if (!fs.existsSync(candidate)) {
|
||||
continue;
|
||||
}
|
||||
const stat = fs.statSync(candidate);
|
||||
if ((kind === "file" && stat.isFile()) || (kind === "directory" && stat.isDirectory())) {
|
||||
return candidate;
|
||||
}
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
function repoRootFromMappingPath(mappingPath: string) {
|
||||
return path.dirname(mappingPath);
|
||||
}
|
||||
|
||||
function formatZodIssuePath(pathLocal: PropertyKey[]) {
|
||||
return pathLocal.length ? pathLocal.map(String).join(".") : "<root>";
|
||||
}
|
||||
|
||||
export function parseQaScorecardTaxonomy(value: unknown, label = QA_SCORECARD_TAXONOMY_PATH) {
|
||||
const parsed = qaScorecardTaxonomySchema.safeParse(value);
|
||||
if (parsed.success) {
|
||||
return parsed.data;
|
||||
}
|
||||
const issues = parsed.error.issues
|
||||
.map((issue) => `${formatZodIssuePath(issue.path)}: ${issue.message}`)
|
||||
.join("; ");
|
||||
throw new Error(`${label}: ${issues}`);
|
||||
}
|
||||
|
||||
export function readQaScorecardTaxonomy(): QaScorecardTaxonomy | null {
|
||||
const taxonomyPath = resolveRepoPath(QA_SCORECARD_TAXONOMY_PATH, "file");
|
||||
if (!taxonomyPath) {
|
||||
return null;
|
||||
}
|
||||
return parseQaScorecardTaxonomy(
|
||||
YAML.parse(fs.readFileSync(taxonomyPath, "utf8")) as unknown,
|
||||
QA_SCORECARD_TAXONOMY_PATH,
|
||||
);
|
||||
}
|
||||
|
||||
function parseQaMaturityTaxonomy(value: unknown, label = QA_MATURITY_TAXONOMY_PATH) {
|
||||
const parsed = qaMaturityTaxonomySchema.safeParse(value);
|
||||
if (parsed.success) {
|
||||
return parsed.data;
|
||||
}
|
||||
const issues = parsed.error.issues
|
||||
.map((issue) => `${formatZodIssuePath(issue.path)}: ${issue.message}`)
|
||||
.join("; ");
|
||||
throw new Error(`${label}: ${issues}`);
|
||||
}
|
||||
|
||||
function readQaMaturityTaxonomy(repoRoot: string | undefined, taxonomySourcePath: string) {
|
||||
const taxonomyPath = repoRoot
|
||||
? path.join(repoRoot, taxonomySourcePath)
|
||||
: resolveRepoPath(taxonomySourcePath);
|
||||
if (!taxonomyPath || !fs.existsSync(taxonomyPath)) {
|
||||
return null;
|
||||
}
|
||||
return parseQaMaturityTaxonomy(
|
||||
YAML.parse(fs.readFileSync(taxonomyPath, "utf8")) as unknown,
|
||||
taxonomySourcePath,
|
||||
);
|
||||
}
|
||||
|
||||
function maturityCategoryKey(surfaceId: string, categoryName: string) {
|
||||
return `${surfaceId}\0${categoryName}`;
|
||||
}
|
||||
|
||||
function buildMaturityCategoryKeys(taxonomy: QaMaturityTaxonomy | null) {
|
||||
const categoryKeys = new Set<string>();
|
||||
if (!taxonomy) {
|
||||
return categoryKeys;
|
||||
}
|
||||
for (const surface of taxonomy.surfaces) {
|
||||
for (const category of surface.categories) {
|
||||
categoryKeys.add(maturityCategoryKey(surface.id, category.name));
|
||||
}
|
||||
}
|
||||
return categoryKeys;
|
||||
}
|
||||
|
||||
function scenarioCoverageIds(scenario: QaSeedScenarioWithSource) {
|
||||
return [...(scenario.coverage?.primary ?? []), ...(scenario.coverage?.secondary ?? [])];
|
||||
}
|
||||
|
||||
function pathExists(repoRoot: string | undefined, relativePath: string) {
|
||||
if (!isRepoRootRelativeRef(relativePath)) {
|
||||
return false;
|
||||
}
|
||||
return repoRoot ? fs.existsSync(path.join(repoRoot, relativePath)) : true;
|
||||
}
|
||||
|
||||
function reportMissingRepoRefs(params: {
|
||||
repoRoot: string | undefined;
|
||||
categoryId: string;
|
||||
refs: readonly string[];
|
||||
code: "docs-ref-not-found" | "code-ref-not-found";
|
||||
label: "docs" | "code";
|
||||
issues: QaScorecardValidationIssue[];
|
||||
}) {
|
||||
for (const ref of params.refs) {
|
||||
if (pathExists(params.repoRoot, ref)) {
|
||||
continue;
|
||||
}
|
||||
params.issues.push({
|
||||
code: params.code,
|
||||
severity: "warning",
|
||||
categoryId: params.categoryId,
|
||||
ref,
|
||||
message: `${params.categoryId} references missing ${params.label} ref ${ref}`,
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
export function buildQaScorecardTaxonomyReport(params: {
|
||||
taxonomy: QaScorecardTaxonomy | null;
|
||||
taxonomyPath?: string | null;
|
||||
repoRoot?: string;
|
||||
scenarios: readonly QaSeedScenarioWithSource[];
|
||||
}): QaScorecardTaxonomyReport {
|
||||
if (!params.taxonomy) {
|
||||
const issue = {
|
||||
code: "taxonomy-fixture-not-found",
|
||||
severity: "warning",
|
||||
ref: QA_SCORECARD_TAXONOMY_PATH,
|
||||
message: `Scorecard evidence mapping not found at ${QA_SCORECARD_TAXONOMY_PATH}`,
|
||||
} satisfies QaScorecardValidationIssue;
|
||||
return {
|
||||
taxonomyPath: params.taxonomyPath ?? null,
|
||||
taxonomyId: null,
|
||||
title: null,
|
||||
taxonomy: null,
|
||||
scoreSnapshotRef: null,
|
||||
status: null,
|
||||
profileCount: 0,
|
||||
profiles: [],
|
||||
categoryCount: 0,
|
||||
releaseBlockingCategoryCount: 0,
|
||||
advisoryCategoryCount: 0,
|
||||
ltsIncludedCategoryCount: 0,
|
||||
deferredCategoryCount: 0,
|
||||
mappedCoverageIdCount: 0,
|
||||
mappedScenarioCount: 0,
|
||||
unmappedCoverageIdCount: 0,
|
||||
unmappedCoverageIds: [],
|
||||
validationIssueCount: 1,
|
||||
validationIssues: [issue],
|
||||
categories: [],
|
||||
};
|
||||
}
|
||||
|
||||
const coverageIdsByScenarioRef = new Map(
|
||||
params.scenarios.map((scenario) => [
|
||||
scenario.sourcePath,
|
||||
new Set(scenarioCoverageIds(scenario)),
|
||||
]),
|
||||
);
|
||||
const scenarioRefsByCoverageId = new Map<string, Set<string>>();
|
||||
for (const scenario of params.scenarios) {
|
||||
for (const coverageId of scenarioCoverageIds(scenario)) {
|
||||
const refs = scenarioRefsByCoverageId.get(coverageId) ?? new Set<string>();
|
||||
refs.add(scenario.sourcePath);
|
||||
scenarioRefsByCoverageId.set(coverageId, refs);
|
||||
}
|
||||
}
|
||||
|
||||
const issues: QaScorecardValidationIssue[] = [];
|
||||
const categories: QaScorecardCategoryMappingReport[] = [];
|
||||
const mappedCoverageIds = new Set<string>();
|
||||
const mappedScenarioRefs = new Set<string>();
|
||||
const categoryIds = new Set(params.taxonomy.categories.map((category) => category.id));
|
||||
const profileIds = new Set(params.taxonomy.profiles.map((profile) => profile.id));
|
||||
const maturityTaxonomy = readQaMaturityTaxonomy(
|
||||
params.repoRoot,
|
||||
params.taxonomy.taxonomy.sourcePath,
|
||||
);
|
||||
const maturityCategoryKeys = buildMaturityCategoryKeys(maturityTaxonomy);
|
||||
const profileCategoryIdsByCategoryId = new Map<string, Set<string>>();
|
||||
const profiles = params.taxonomy.profiles.map((profile) => {
|
||||
for (const categoryId of profile.categoryIds) {
|
||||
if (!categoryIds.has(categoryId)) {
|
||||
issues.push({
|
||||
code: "profile-category-ref-not-found",
|
||||
severity: "warning",
|
||||
ref: categoryId,
|
||||
message: `${profile.id} profile references missing executable scorecard category ${categoryId}`,
|
||||
});
|
||||
continue;
|
||||
}
|
||||
const categoryProfileIds =
|
||||
profileCategoryIdsByCategoryId.get(categoryId) ?? new Set<string>();
|
||||
categoryProfileIds.add(profile.id);
|
||||
profileCategoryIdsByCategoryId.set(categoryId, categoryProfileIds);
|
||||
}
|
||||
|
||||
return {
|
||||
id: profile.id,
|
||||
categoryIds: profile.categoryIds.filter((categoryId) => categoryIds.has(categoryId)),
|
||||
};
|
||||
});
|
||||
|
||||
if (!pathExists(params.repoRoot, params.taxonomy.taxonomy.sourcePath) || !maturityTaxonomy) {
|
||||
issues.push({
|
||||
code: "taxonomy-ref-not-found",
|
||||
severity: "warning",
|
||||
ref: params.taxonomy.taxonomy.sourcePath,
|
||||
message: `Scorecard executable mapping references missing maturity taxonomy ${params.taxonomy.taxonomy.sourcePath}`,
|
||||
});
|
||||
}
|
||||
if (
|
||||
params.taxonomy.scoreSnapshotRef &&
|
||||
!pathExists(params.repoRoot, params.taxonomy.scoreSnapshotRef)
|
||||
) {
|
||||
issues.push({
|
||||
code: "score-snapshot-ref-not-found",
|
||||
severity: "warning",
|
||||
ref: params.taxonomy.scoreSnapshotRef,
|
||||
message: `Scorecard executable mapping references missing score snapshot ${params.taxonomy.scoreSnapshotRef}`,
|
||||
});
|
||||
}
|
||||
|
||||
for (const category of params.taxonomy.categories) {
|
||||
const missingCoverageIds: string[] = [];
|
||||
const missingScenarioRefs: string[] = [];
|
||||
const declaredProfileIds = new Set(category.evidence.profiles);
|
||||
const declaredKnownProfileIds = new Set(
|
||||
[...declaredProfileIds].filter((profileId) => profileIds.has(profileId)),
|
||||
);
|
||||
const membershipProfileIds =
|
||||
profileCategoryIdsByCategoryId.get(category.id) ?? new Set<string>();
|
||||
const sortedMembershipProfileIds = [...membershipProfileIds].toSorted();
|
||||
const maturityKey = maturityCategoryKey(
|
||||
category.taxonomySurfaceId,
|
||||
category.taxonomyCategoryName,
|
||||
);
|
||||
|
||||
if (maturityTaxonomy && !maturityCategoryKeys.has(maturityKey)) {
|
||||
issues.push({
|
||||
code: "taxonomy-category-ref-not-found",
|
||||
severity: "warning",
|
||||
categoryId: category.id,
|
||||
ref: `${category.taxonomySurfaceId}/${category.taxonomyCategoryName}`,
|
||||
message: `${category.id} references missing maturity taxonomy category ${category.taxonomySurfaceId}/${category.taxonomyCategoryName}`,
|
||||
});
|
||||
}
|
||||
|
||||
for (const profileId of declaredProfileIds) {
|
||||
if (!profileIds.has(profileId)) {
|
||||
issues.push({
|
||||
code: "profile-ref-not-found",
|
||||
severity: "warning",
|
||||
categoryId: category.id,
|
||||
ref: profileId,
|
||||
message: `${category.id} declares profile ${profileId}, but taxonomy-mappings.yaml has no matching top-level profile`,
|
||||
});
|
||||
continue;
|
||||
}
|
||||
|
||||
if (!membershipProfileIds.has(profileId)) {
|
||||
issues.push({
|
||||
code: "category-profile-missing-top-level-membership",
|
||||
severity: "warning",
|
||||
categoryId: category.id,
|
||||
ref: profileId,
|
||||
message: `${category.id} declares ${profileId} evidence, but the taxonomy profile does not include the category`,
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
for (const profileId of membershipProfileIds) {
|
||||
if (!declaredProfileIds.has(profileId)) {
|
||||
issues.push({
|
||||
code: "profile-membership-missing-category-profile",
|
||||
severity: "warning",
|
||||
categoryId: category.id,
|
||||
ref: profileId,
|
||||
message: `${category.id} belongs to the ${profileId} taxonomy profile, but its evidence profiles do not declare that selector`,
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
if (category.releaseBlocking && !membershipProfileIds.has("release")) {
|
||||
issues.push({
|
||||
code: "release-blocking-category-missing-release-profile",
|
||||
severity: "warning",
|
||||
categoryId: category.id,
|
||||
ref: "release",
|
||||
message: `${category.id} is release-blocking but is not selected by the release profile`,
|
||||
});
|
||||
}
|
||||
|
||||
if (
|
||||
category.supportStatus === "advisory" &&
|
||||
(membershipProfileIds.size > 0 || declaredProfileIds.size > 0)
|
||||
) {
|
||||
const runnableProfiles = [
|
||||
...new Set([...membershipProfileIds, ...declaredProfileIds]),
|
||||
].toSorted();
|
||||
issues.push({
|
||||
code: "advisory-category-has-profile-membership",
|
||||
severity: "warning",
|
||||
categoryId: category.id,
|
||||
message: `${category.id} is advisory metadata but belongs to runnable profile(s): ${runnableProfiles.join(", ")}`,
|
||||
});
|
||||
}
|
||||
|
||||
if (
|
||||
category.supportStatus !== "advisory" &&
|
||||
membershipProfileIds.size === 0 &&
|
||||
declaredKnownProfileIds.size === 0
|
||||
) {
|
||||
issues.push({
|
||||
code: "non-advisory-category-missing-profile-membership",
|
||||
severity: "warning",
|
||||
categoryId: category.id,
|
||||
message: `${category.id} is ${category.supportStatus} but has no runnable profile membership`,
|
||||
});
|
||||
}
|
||||
|
||||
for (const coverageId of category.evidence.coverageIds) {
|
||||
const scenarioRefs = scenarioRefsByCoverageId.get(coverageId);
|
||||
if (!scenarioRefs) {
|
||||
missingCoverageIds.push(coverageId);
|
||||
issues.push({
|
||||
code: "coverage-id-not-found",
|
||||
severity: "warning",
|
||||
categoryId: category.id,
|
||||
ref: coverageId,
|
||||
message: `${category.id} maps missing coverage id ${coverageId}`,
|
||||
});
|
||||
continue;
|
||||
}
|
||||
mappedCoverageIds.add(coverageId);
|
||||
for (const scenarioRef of scenarioRefs) {
|
||||
mappedScenarioRefs.add(scenarioRef);
|
||||
}
|
||||
}
|
||||
|
||||
const categoryCoverageIds = new Set(category.evidence.coverageIds);
|
||||
for (const scenarioRef of category.evidence.scenarioRefs) {
|
||||
const scenarioCoverage = coverageIdsByScenarioRef.get(scenarioRef);
|
||||
if (!scenarioCoverage) {
|
||||
missingScenarioRefs.push(scenarioRef);
|
||||
issues.push({
|
||||
code: "scenario-ref-not-found",
|
||||
severity: "warning",
|
||||
categoryId: category.id,
|
||||
ref: scenarioRef,
|
||||
message: `${category.id} references missing scenario ${scenarioRef}`,
|
||||
});
|
||||
continue;
|
||||
}
|
||||
mappedScenarioRefs.add(scenarioRef);
|
||||
if (
|
||||
categoryCoverageIds.size > 0 &&
|
||||
![...scenarioCoverage].some((coverageId) => categoryCoverageIds.has(coverageId))
|
||||
) {
|
||||
issues.push({
|
||||
code: "scenario-ref-not-covered-by-category",
|
||||
severity: "warning",
|
||||
categoryId: category.id,
|
||||
ref: scenarioRef,
|
||||
message: `${category.id} references ${scenarioRef} without one of the category coverage IDs`,
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
reportMissingRepoRefs({
|
||||
repoRoot: params.repoRoot,
|
||||
categoryId: category.id,
|
||||
refs: category.evidence.docsRefs,
|
||||
code: "docs-ref-not-found",
|
||||
label: "docs",
|
||||
issues,
|
||||
});
|
||||
reportMissingRepoRefs({
|
||||
repoRoot: params.repoRoot,
|
||||
categoryId: category.id,
|
||||
refs: category.evidence.codeRefs,
|
||||
code: "code-ref-not-found",
|
||||
label: "code",
|
||||
issues,
|
||||
});
|
||||
|
||||
if (
|
||||
category.releaseBlocking &&
|
||||
category.evidence.coverageIds.length === 0 &&
|
||||
category.evidence.scenarioRefs.length === 0
|
||||
) {
|
||||
issues.push({
|
||||
code: "blocking-category-without-evidence-mapping",
|
||||
severity: "warning",
|
||||
categoryId: category.id,
|
||||
message: `${category.id} is release-blocking but has no coverage IDs or scenario refs`,
|
||||
});
|
||||
}
|
||||
|
||||
const mappingStatus =
|
||||
category.evidence.coverageIds.length === 0 && category.evidence.scenarioRefs.length === 0
|
||||
? "missing"
|
||||
: missingCoverageIds.length > 0 || missingScenarioRefs.length > 0
|
||||
? "partial"
|
||||
: "mapped";
|
||||
categories.push({
|
||||
id: category.id,
|
||||
taxonomySurfaceId: category.taxonomySurfaceId,
|
||||
taxonomyCategoryName: category.taxonomyCategoryName,
|
||||
supportStatus: category.supportStatus,
|
||||
releaseBlocking: category.releaseBlocking,
|
||||
mappingStatus,
|
||||
profiles: sortedMembershipProfileIds,
|
||||
liveProofRequired: category.evidence.liveProofRequired,
|
||||
freshness: category.evidence.freshness,
|
||||
coverageIds: [...category.evidence.coverageIds],
|
||||
scenarioRefs: [...category.evidence.scenarioRefs],
|
||||
missingCoverageIds,
|
||||
missingScenarioRefs,
|
||||
});
|
||||
}
|
||||
|
||||
const allCoverageIds = [...scenarioRefsByCoverageId.keys()].toSorted();
|
||||
const unmappedCoverageIds = allCoverageIds.filter(
|
||||
(coverageId) => !mappedCoverageIds.has(coverageId),
|
||||
);
|
||||
|
||||
return {
|
||||
taxonomyPath: params.taxonomyPath ?? QA_SCORECARD_TAXONOMY_PATH,
|
||||
taxonomyId: params.taxonomy.id,
|
||||
title: params.taxonomy.title,
|
||||
taxonomy: params.taxonomy.taxonomy,
|
||||
scoreSnapshotRef: params.taxonomy.scoreSnapshotRef ?? null,
|
||||
status: params.taxonomy.status,
|
||||
profileCount: params.taxonomy.profiles.length,
|
||||
profiles,
|
||||
categoryCount: params.taxonomy.categories.length,
|
||||
releaseBlockingCategoryCount: params.taxonomy.categories.filter(
|
||||
(category) => category.releaseBlocking,
|
||||
).length,
|
||||
advisoryCategoryCount: params.taxonomy.categories.filter(
|
||||
(category) => category.supportStatus === "advisory",
|
||||
).length,
|
||||
ltsIncludedCategoryCount: params.taxonomy.categories.filter(
|
||||
(category) => category.supportStatus === "lts-included",
|
||||
).length,
|
||||
deferredCategoryCount: params.taxonomy.categories.filter(
|
||||
(category) => category.supportStatus === "deferred",
|
||||
).length,
|
||||
mappedCoverageIdCount: mappedCoverageIds.size,
|
||||
mappedScenarioCount: mappedScenarioRefs.size,
|
||||
unmappedCoverageIdCount: unmappedCoverageIds.length,
|
||||
unmappedCoverageIds,
|
||||
validationIssueCount: issues.length,
|
||||
validationIssues: issues,
|
||||
categories: categories.toSorted((left, right) => left.id.localeCompare(right.id)),
|
||||
};
|
||||
}
|
||||
|
||||
export function readQaScorecardTaxonomyReport(scenarios: readonly QaSeedScenarioWithSource[]) {
|
||||
const taxonomyPath = resolveRepoPath(QA_SCORECARD_TAXONOMY_PATH, "file");
|
||||
const taxonomy = readQaScorecardTaxonomy();
|
||||
return buildQaScorecardTaxonomyReport({
|
||||
taxonomy,
|
||||
taxonomyPath: taxonomyPath ? QA_SCORECARD_TAXONOMY_PATH : null,
|
||||
repoRoot: taxonomyPath ? repoRootFromMappingPath(taxonomyPath) : undefined,
|
||||
scenarios,
|
||||
});
|
||||
}
|
||||
4
extensions/qa-lab/src/shell-quote.ts
Normal file
4
extensions/qa-lab/src/shell-quote.ts
Normal file
@@ -0,0 +1,4 @@
|
||||
// POSIX shell quoting for generated QA command previews and guest scripts.
|
||||
export function shellQuote(value: string) {
|
||||
return `'${value.replaceAll("'", `'"'"'`)}'`;
|
||||
}
|
||||
154
extensions/qa-lab/src/suite-launch.runtime.test.ts
Normal file
154
extensions/qa-lab/src/suite-launch.runtime.test.ts
Normal file
@@ -0,0 +1,154 @@
|
||||
import fs from "node:fs/promises";
|
||||
import os from "node:os";
|
||||
import path from "node:path";
|
||||
import { afterEach, beforeEach, describe, expect, it, vi } from "vitest";
|
||||
|
||||
const { runQaFlowSuite, runQaTestFileScenarios } = vi.hoisted(() => ({
|
||||
runQaFlowSuite: vi.fn(),
|
||||
runQaTestFileScenarios: vi.fn(),
|
||||
}));
|
||||
|
||||
vi.mock("./suite.js", () => ({
|
||||
runQaFlowSuite,
|
||||
}));
|
||||
|
||||
vi.mock("./test-file-scenario-runner.js", async (importOriginal) => ({
|
||||
...(await importOriginal<typeof import("./test-file-scenario-runner.js")>()),
|
||||
runQaTestFileScenarios,
|
||||
}));
|
||||
|
||||
import { runQaSuite } from "./suite-launch.runtime.js";
|
||||
|
||||
const tempRoots: string[] = [];
|
||||
|
||||
async function makeTempRepo(prefix: string) {
|
||||
const repoRoot = await fs.mkdtemp(path.join(os.tmpdir(), prefix));
|
||||
tempRoots.push(repoRoot);
|
||||
return repoRoot;
|
||||
}
|
||||
|
||||
describe("qa suite runtime launcher", () => {
|
||||
beforeEach(() => {
|
||||
runQaFlowSuite.mockReset();
|
||||
runQaTestFileScenarios.mockReset();
|
||||
runQaFlowSuite.mockResolvedValue({
|
||||
outputDir: "/tmp/qa-flow",
|
||||
evidencePath: "/tmp/qa-flow/qa-evidence.json",
|
||||
reportPath: "/tmp/qa-flow/qa-suite-report.md",
|
||||
summaryPath: "/tmp/qa-flow/qa-suite-summary.json",
|
||||
report: "# QA Suite Report\n",
|
||||
scenarios: [],
|
||||
watchUrl: "http://127.0.0.1:43124",
|
||||
});
|
||||
runQaTestFileScenarios.mockResolvedValue({
|
||||
outputDir: "/tmp/qa-test-file",
|
||||
executionKind: "playwright",
|
||||
reportPath: "/tmp/qa-test-file/qa-playwright-report.md",
|
||||
evidencePath: "/tmp/qa-test-file/qa-evidence.json",
|
||||
results: [{ status: "pass" }],
|
||||
});
|
||||
});
|
||||
|
||||
afterEach(async () => {
|
||||
await Promise.all(
|
||||
tempRoots.splice(0).map((root) => fs.rm(root, { recursive: true, force: true })),
|
||||
);
|
||||
});
|
||||
|
||||
it("routes selected flow scenarios to the flow suite engine", async () => {
|
||||
const result = await runQaSuite({
|
||||
repoRoot: process.cwd(),
|
||||
providerMode: "mock-openai",
|
||||
scenarioIds: ["channel-chat-baseline"],
|
||||
});
|
||||
|
||||
expect(result).toMatchObject({
|
||||
executionKind: "flow",
|
||||
result: {
|
||||
summaryPath: "/tmp/qa-flow/qa-suite-summary.json",
|
||||
},
|
||||
});
|
||||
expect(runQaFlowSuite).toHaveBeenCalledTimes(1);
|
||||
expect(runQaFlowSuite).toHaveBeenCalledWith(
|
||||
expect.objectContaining({
|
||||
repoRoot: process.cwd(),
|
||||
providerMode: "mock-openai",
|
||||
scenarioIds: ["channel-chat-baseline"],
|
||||
}),
|
||||
);
|
||||
expect(runQaTestFileScenarios).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it("routes selected Playwright scenarios to the Playwright scenario runner", async () => {
|
||||
const repoRoot = await makeTempRepo("qa-suite-launch-");
|
||||
const result = await runQaSuite({
|
||||
repoRoot,
|
||||
outputDir: ".artifacts/qa-e2e/scenario-test",
|
||||
scenarioIds: ["control-ui-chat-flow-playwright"],
|
||||
});
|
||||
|
||||
expect(result).toMatchObject({
|
||||
executionKind: "playwright",
|
||||
result: {
|
||||
evidencePath: "/tmp/qa-test-file/qa-evidence.json",
|
||||
},
|
||||
});
|
||||
expect(runQaFlowSuite).not.toHaveBeenCalled();
|
||||
expect(runQaTestFileScenarios).toHaveBeenCalledTimes(1);
|
||||
const [call] = runQaTestFileScenarios.mock.calls[0] ?? [];
|
||||
expect(call).toMatchObject({
|
||||
repoRoot,
|
||||
outputDir: path.join(repoRoot, ".artifacts", "qa-e2e", "scenario-test"),
|
||||
providerMode: "mock-openai",
|
||||
primaryModel: "mock-openai/gpt-5.5",
|
||||
});
|
||||
expect(
|
||||
call.scenarios.map((scenario: { id: string; execution: { kind: string } }) => ({
|
||||
id: scenario.id,
|
||||
kind: scenario.execution.kind,
|
||||
})),
|
||||
).toEqual([{ id: "control-ui-chat-flow-playwright", kind: "playwright" }]);
|
||||
});
|
||||
|
||||
it("rejects mixed flow and Vitest/Playwright scenarios", async () => {
|
||||
await expect(
|
||||
runQaSuite({
|
||||
repoRoot: process.cwd(),
|
||||
scenarioIds: ["channel-chat-baseline", "control-ui-chat-flow-playwright"],
|
||||
}),
|
||||
).rejects.toThrow("qa suite cannot mix execution.kind: flow with Vitest/Playwright scenarios");
|
||||
|
||||
expect(runQaFlowSuite).not.toHaveBeenCalled();
|
||||
expect(runQaTestFileScenarios).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it("rejects runtime-pair requests for Vitest/Playwright scenarios", async () => {
|
||||
await expect(
|
||||
runQaSuite({
|
||||
repoRoot: process.cwd(),
|
||||
runtimePair: ["openclaw", "codex"],
|
||||
scenarioIds: ["control-ui-chat-flow-playwright"],
|
||||
}),
|
||||
).rejects.toThrow("--runtime-pair requires execution.kind: flow scenarios");
|
||||
|
||||
expect(runQaFlowSuite).not.toHaveBeenCalled();
|
||||
expect(runQaTestFileScenarios).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it("rejects repo-local symlink output directories before running Vitest/Playwright scenarios", async () => {
|
||||
const repoRoot = await makeTempRepo("qa-suite-symlink-root-");
|
||||
const outsideRoot = await makeTempRepo("qa-suite-symlink-outside-");
|
||||
await fs.symlink(outsideRoot, path.join(repoRoot, "artifacts-link"));
|
||||
|
||||
await expect(
|
||||
runQaSuite({
|
||||
repoRoot,
|
||||
outputDir: "artifacts-link/qa-out",
|
||||
scenarioIds: ["control-ui-chat-flow-playwright"],
|
||||
}),
|
||||
).rejects.toThrow("QA suite outputDir must not traverse symlinks");
|
||||
|
||||
expect(runQaFlowSuite).not.toHaveBeenCalled();
|
||||
expect(runQaTestFileScenarios).not.toHaveBeenCalled();
|
||||
});
|
||||
});
|
||||
@@ -1,15 +1,120 @@
|
||||
// Qa Lab plugin module implements suite launch behavior.
|
||||
import type { QaSuiteRunParams } from "./suite.js";
|
||||
import path from "node:path";
|
||||
import { DEFAULT_QA_PROVIDER_MODE } from "./providers/index.js";
|
||||
import { defaultQaModelForMode, normalizeQaProviderMode } from "./run-config.js";
|
||||
import { readQaBootstrapScenarioCatalog } from "./scenario-catalog.js";
|
||||
import { resolveQaSuiteOutputDir } from "./suite-planning.js";
|
||||
import type { QaSuiteResult, QaSuiteRunParams } from "./suite.js";
|
||||
import {
|
||||
isQaTestFileScenario,
|
||||
runQaTestFileScenarios,
|
||||
type QaTestFileExecutionKind,
|
||||
type QaTestFileScenario,
|
||||
type QaTestFileScenarioRunResult,
|
||||
} from "./test-file-scenario-runner.js";
|
||||
|
||||
export type QaSuiteRuntimeResult =
|
||||
| {
|
||||
executionKind: "flow";
|
||||
result: QaSuiteResult;
|
||||
}
|
||||
| {
|
||||
executionKind: QaTestFileExecutionKind;
|
||||
result: QaTestFileScenarioRunResult;
|
||||
};
|
||||
|
||||
async function loadQaLabServerRuntime() {
|
||||
const { startQaLabServer } = await import("./lab-server.js");
|
||||
return startQaLabServer;
|
||||
}
|
||||
|
||||
export async function runQaSuiteFromRuntime(...args: [QaSuiteRunParams?]) {
|
||||
const { runQaSuite } = await import("./suite.js");
|
||||
function resolveRequestedScenarios(params: {
|
||||
scenarioIds: readonly string[];
|
||||
scenarios: ReturnType<typeof readQaBootstrapScenarioCatalog>["scenarios"];
|
||||
}) {
|
||||
const scenarioById = new Map(params.scenarios.map((scenario) => [scenario.id, scenario]));
|
||||
return params.scenarioIds.map((scenarioId) => {
|
||||
const scenario = scenarioById.get(scenarioId);
|
||||
if (!scenario) {
|
||||
throw new Error(`unknown QA scenario id(s): ${scenarioId}`);
|
||||
}
|
||||
return scenario;
|
||||
});
|
||||
}
|
||||
|
||||
function resolveTestFileScenariosForSuiteDispatch(
|
||||
params: QaSuiteRunParams | undefined,
|
||||
): QaTestFileScenario[] | null {
|
||||
const scenarioIds = params?.scenarioIds ?? [];
|
||||
if (scenarioIds.length === 0) {
|
||||
return null;
|
||||
}
|
||||
const selectedScenarios = resolveRequestedScenarios({
|
||||
scenarioIds,
|
||||
scenarios: readQaBootstrapScenarioCatalog().scenarios,
|
||||
});
|
||||
const testFileScenarios = selectedScenarios.filter(isQaTestFileScenario);
|
||||
if (testFileScenarios.length === 0) {
|
||||
return null;
|
||||
}
|
||||
if (testFileScenarios.length !== selectedScenarios.length) {
|
||||
throw new Error("qa suite cannot mix execution.kind: flow with Vitest/Playwright scenarios.");
|
||||
}
|
||||
return testFileScenarios;
|
||||
}
|
||||
|
||||
async function runQaTestFileSuiteFromRuntime(params: {
|
||||
runParams: QaSuiteRunParams | undefined;
|
||||
scenarios: readonly QaTestFileScenario[];
|
||||
}): Promise<QaTestFileScenarioRunResult> {
|
||||
const runParams = params.runParams;
|
||||
if (runParams?.runtimePair) {
|
||||
throw new Error("--runtime-pair requires execution.kind: flow scenarios.");
|
||||
}
|
||||
if (runParams?.forcedRuntime) {
|
||||
throw new Error("forced runtime execution requires execution.kind: flow scenarios.");
|
||||
}
|
||||
if (runParams?.captureRuntimeParityCell) {
|
||||
throw new Error("runtime parity capture requires execution.kind: flow scenarios.");
|
||||
}
|
||||
const repoRoot = path.resolve(runParams?.repoRoot ?? process.cwd());
|
||||
const outputDir = await resolveQaSuiteOutputDir(repoRoot, runParams?.outputDir);
|
||||
const providerMode = normalizeQaProviderMode(runParams?.providerMode ?? DEFAULT_QA_PROVIDER_MODE);
|
||||
const primaryModel = runParams?.primaryModel?.trim() || defaultQaModelForMode(providerMode);
|
||||
return await runQaTestFileScenarios({
|
||||
repoRoot,
|
||||
outputDir,
|
||||
providerMode,
|
||||
primaryModel,
|
||||
scenarios: params.scenarios,
|
||||
});
|
||||
}
|
||||
|
||||
export async function runQaSuite(...args: [QaSuiteRunParams?]): Promise<QaSuiteRuntimeResult> {
|
||||
const runParams = args[0];
|
||||
const testFileScenarios = resolveTestFileScenariosForSuiteDispatch(runParams);
|
||||
if (testFileScenarios) {
|
||||
const result = await runQaTestFileSuiteFromRuntime({
|
||||
runParams,
|
||||
scenarios: testFileScenarios,
|
||||
});
|
||||
return {
|
||||
executionKind: result.executionKind,
|
||||
result,
|
||||
};
|
||||
}
|
||||
return {
|
||||
executionKind: "flow",
|
||||
result: await runQaFlowSuiteFromRuntime(...args),
|
||||
};
|
||||
}
|
||||
|
||||
export async function runQaFlowSuiteFromRuntime(
|
||||
...args: [QaSuiteRunParams?]
|
||||
): Promise<QaSuiteResult> {
|
||||
const { runQaFlowSuite } = await import("./suite.js");
|
||||
const params = args[0];
|
||||
return await runQaSuite({
|
||||
return await runQaFlowSuite({
|
||||
...params,
|
||||
startLab: params?.startLab ?? (await loadQaLabServerRuntime()),
|
||||
});
|
||||
|
||||
@@ -13,11 +13,21 @@ import {
|
||||
resolveQaSuiteWorkerStartStaggerMs,
|
||||
resolveQaSuiteOutputDir,
|
||||
scenarioRequiresControlUi,
|
||||
selectQaSuiteScenarios,
|
||||
selectQaFlowSuiteScenarios,
|
||||
shouldUseIsolatedQaSuiteScenarioWorkers,
|
||||
} from "./suite-planning.js";
|
||||
import { makeQaSuiteTestScenario } from "./suite-test-helpers.js";
|
||||
|
||||
function makePlaywrightQaSuiteTestScenario(id: string): ReturnType<typeof makeQaSuiteTestScenario> {
|
||||
return {
|
||||
...makeQaSuiteTestScenario(id),
|
||||
execution: {
|
||||
kind: "playwright",
|
||||
path: `ui/src/ui/e2e/${id}.e2e.test.ts`,
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
describe("qa suite planning helpers", () => {
|
||||
it("normalizes suite concurrency to a bounded integer", () => {
|
||||
const previous = process.env.OPENCLAW_QA_SUITE_CONCURRENCY;
|
||||
@@ -205,7 +215,7 @@ describe("qa suite planning helpers", () => {
|
||||
];
|
||||
|
||||
expect(
|
||||
selectQaSuiteScenarios({
|
||||
selectQaFlowSuiteScenarios({
|
||||
scenarios,
|
||||
scenarioIds: ["anthropic-only"],
|
||||
providerMode: "live-frontier",
|
||||
@@ -222,7 +232,7 @@ describe("qa suite planning helpers", () => {
|
||||
];
|
||||
|
||||
expect(
|
||||
selectQaSuiteScenarios({
|
||||
selectQaFlowSuiteScenarios({
|
||||
scenarios,
|
||||
scenarioIds: ["third", "first"],
|
||||
providerMode: "live-frontier",
|
||||
@@ -393,7 +403,7 @@ describe("qa suite planning helpers", () => {
|
||||
];
|
||||
|
||||
expect(
|
||||
selectQaSuiteScenarios({
|
||||
selectQaFlowSuiteScenarios({
|
||||
scenarios,
|
||||
providerMode: "live-frontier",
|
||||
primaryModel: "openai/gpt-5.5",
|
||||
@@ -401,7 +411,7 @@ describe("qa suite planning helpers", () => {
|
||||
).toEqual(["generic", "openai-only"]);
|
||||
|
||||
expect(
|
||||
selectQaSuiteScenarios({
|
||||
selectQaFlowSuiteScenarios({
|
||||
scenarios,
|
||||
providerMode: "live-frontier",
|
||||
primaryModel: "claude-cli/claude-sonnet-4-6",
|
||||
@@ -410,6 +420,39 @@ describe("qa suite planning helpers", () => {
|
||||
).toEqual(["generic", "claude-subscription"]);
|
||||
});
|
||||
|
||||
it("keeps Playwright scenarios out of implicit flow suite selections", () => {
|
||||
const scenarios = [
|
||||
makeQaSuiteTestScenario("flow"),
|
||||
makePlaywrightQaSuiteTestScenario("playwright"),
|
||||
];
|
||||
|
||||
expect(
|
||||
selectQaFlowSuiteScenarios({
|
||||
scenarios,
|
||||
providerMode: "mock-openai",
|
||||
primaryModel: "mock-openai/gpt-5.5",
|
||||
}).map((scenario) => scenario.id),
|
||||
).toEqual(["flow"]);
|
||||
});
|
||||
|
||||
it("rejects explicit Playwright scenarios in the flow suite selector", () => {
|
||||
const scenarios = [
|
||||
makeQaSuiteTestScenario("flow"),
|
||||
makePlaywrightQaSuiteTestScenario("playwright"),
|
||||
];
|
||||
|
||||
expect(() =>
|
||||
selectQaFlowSuiteScenarios({
|
||||
scenarios,
|
||||
scenarioIds: ["playwright"],
|
||||
providerMode: "mock-openai",
|
||||
primaryModel: "mock-openai/gpt-5.5",
|
||||
}),
|
||||
).toThrow(
|
||||
"flow execution requires execution.kind: flow; unsupported scenario(s): playwright (playwright)",
|
||||
);
|
||||
});
|
||||
|
||||
it("filters provider-mode-specific scenarios from implicit suite selections", () => {
|
||||
const scenarios = [
|
||||
makeQaSuiteTestScenario("generic"),
|
||||
@@ -422,7 +465,7 @@ describe("qa suite planning helpers", () => {
|
||||
];
|
||||
|
||||
expect(
|
||||
selectQaSuiteScenarios({
|
||||
selectQaFlowSuiteScenarios({
|
||||
scenarios,
|
||||
providerMode: "mock-openai",
|
||||
primaryModel: "mock-openai/gpt-5.5",
|
||||
@@ -430,7 +473,7 @@ describe("qa suite planning helpers", () => {
|
||||
).toEqual(["generic", "mock-only"]);
|
||||
|
||||
expect(
|
||||
selectQaSuiteScenarios({
|
||||
selectQaFlowSuiteScenarios({
|
||||
scenarios,
|
||||
providerMode: "live-frontier",
|
||||
primaryModel: "openai/gpt-5.5",
|
||||
@@ -447,7 +490,7 @@ describe("qa suite planning helpers", () => {
|
||||
];
|
||||
|
||||
expect(
|
||||
selectQaSuiteScenarios({
|
||||
selectQaFlowSuiteScenarios({
|
||||
scenarios,
|
||||
providerMode: "mock-openai",
|
||||
primaryModel: "mock-openai/gpt-5.5",
|
||||
@@ -455,7 +498,7 @@ describe("qa suite planning helpers", () => {
|
||||
).toEqual(["generic"]);
|
||||
|
||||
expect(
|
||||
selectQaSuiteScenarios({
|
||||
selectQaFlowSuiteScenarios({
|
||||
scenarios,
|
||||
scenarioIds: ["live-runtime"],
|
||||
providerMode: "mock-openai",
|
||||
|
||||
@@ -63,7 +63,7 @@ function scenarioMatchesLiveLane(params: {
|
||||
return true;
|
||||
}
|
||||
|
||||
function selectQaSuiteScenarios(params: {
|
||||
function selectQaFlowSuiteScenarios(params: {
|
||||
scenarios: ReturnType<typeof readQaBootstrapScenarioCatalog>["scenarios"];
|
||||
scenarioIds?: string[];
|
||||
providerMode: QaProviderMode;
|
||||
@@ -80,15 +80,31 @@ function selectQaSuiteScenarios(params: {
|
||||
if (missingScenarioIds.length > 0) {
|
||||
throw new Error(`unknown QA scenario id(s): ${missingScenarioIds.join(", ")}`);
|
||||
}
|
||||
return [...requestedScenarioIds].map((scenarioId) => scenarioById.get(scenarioId)!);
|
||||
const selectedScenarios = [...requestedScenarioIds].map(
|
||||
(scenarioId) => scenarioById.get(scenarioId)!,
|
||||
);
|
||||
const nonFlowScenarios = selectedScenarios.filter(
|
||||
(scenario) => scenario.execution.kind !== "flow",
|
||||
);
|
||||
if (nonFlowScenarios.length > 0) {
|
||||
const scenarioList = nonFlowScenarios
|
||||
.map((scenario) => `${scenario.id} (${scenario.execution.kind})`)
|
||||
.join(", ");
|
||||
throw new Error(
|
||||
`flow execution requires execution.kind: flow; unsupported scenario(s): ${scenarioList}`,
|
||||
);
|
||||
}
|
||||
return selectedScenarios;
|
||||
}
|
||||
return params.scenarios.filter((scenario) =>
|
||||
scenarioMatchesLiveLane({
|
||||
scenario,
|
||||
providerMode: params.providerMode,
|
||||
primaryModel: params.primaryModel,
|
||||
claudeCliAuthMode: params.claudeCliAuthMode,
|
||||
}),
|
||||
return params.scenarios.filter(
|
||||
(scenario) =>
|
||||
scenario.execution.kind === "flow" &&
|
||||
scenarioMatchesLiveLane({
|
||||
scenario,
|
||||
providerMode: params.providerMode,
|
||||
primaryModel: params.primaryModel,
|
||||
claudeCliAuthMode: params.claudeCliAuthMode,
|
||||
}),
|
||||
);
|
||||
}
|
||||
|
||||
@@ -266,7 +282,7 @@ export {
|
||||
resolveQaSuiteWorkerStartStaggerMs,
|
||||
resolveQaSuiteOutputDir,
|
||||
scenarioRequiresControlUi,
|
||||
selectQaSuiteScenarios,
|
||||
selectQaFlowSuiteScenarios,
|
||||
shouldUseIsolatedQaSuiteScenarioWorkers,
|
||||
splitModelRef,
|
||||
};
|
||||
|
||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user