mirror of
https://github.com/openclaw/openclaw.git
synced 2026-06-14 01:58:47 +08:00
Compare commits
55 Commits
feat/ux-to
...
codex/secu
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
93fc591af9 | ||
|
|
4c23d1d597 | ||
|
|
8eb1fa09c6 | ||
|
|
2d2c1e63f0 | ||
|
|
6cdbccaa9e | ||
|
|
9f522ee7df | ||
|
|
7404b2b5b4 | ||
|
|
73aabcceda | ||
|
|
b1fc8673df | ||
|
|
4cf4e54179 | ||
|
|
84519f7e3c | ||
|
|
6314c377bb | ||
|
|
d3e7e03669 | ||
|
|
64f9f3c278 | ||
|
|
cd3eb438f0 | ||
|
|
26281a8a11 | ||
|
|
4208c89ec4 | ||
|
|
c9c19a1106 | ||
|
|
f78d7b52d8 | ||
|
|
ff6940036b | ||
|
|
b477bfe84b | ||
|
|
d4237cb14d | ||
|
|
20bc546d94 | ||
|
|
069cb8d636 | ||
|
|
8cc5d2d85c | ||
|
|
690f27749c | ||
|
|
7af8153388 | ||
|
|
a66a065ffb | ||
|
|
64d0fc8336 | ||
|
|
f3df863aff | ||
|
|
26b9736922 | ||
|
|
44f45d8729 | ||
|
|
1c655008cd | ||
|
|
618d78144e | ||
|
|
56f2102c28 | ||
|
|
99db98a7ce | ||
|
|
0dbfa1f6be | ||
|
|
f06f2f17c2 | ||
|
|
7bd533a80e | ||
|
|
561b293c7a | ||
|
|
21aa8faf8a | ||
|
|
b0bd9c8ed8 | ||
|
|
c5d599c8c4 | ||
|
|
3a1a5c0dac | ||
|
|
0849cac106 | ||
|
|
32ce06daf8 | ||
|
|
751d3db1cc | ||
|
|
4640baa299 | ||
|
|
c9f0bfd476 | ||
|
|
ab559a7257 | ||
|
|
7c08804541 | ||
|
|
991471b8ec | ||
|
|
7190fc4de8 | ||
|
|
9d9389bc6b | ||
|
|
6c88811b4b |
61
.github/codeql/codeql-process-exec-boundary-critical-security.yml
vendored
Normal file
61
.github/codeql/codeql-process-exec-boundary-critical-security.yml
vendored
Normal file
@@ -0,0 +1,61 @@
|
||||
name: openclaw-codeql-process-exec-boundary-critical-security
|
||||
|
||||
disable-default-queries: true
|
||||
|
||||
queries:
|
||||
- uses: security-extended
|
||||
|
||||
query-filters:
|
||||
- include:
|
||||
precision:
|
||||
- high
|
||||
- very-high
|
||||
tags contain: security
|
||||
security-severity: /([7-9]|10)\.(\d)+/
|
||||
|
||||
paths:
|
||||
- src/process
|
||||
- src/tui/tui-local-shell.ts
|
||||
- src/tui/tui.ts
|
||||
- src/plugin-sdk/windows-spawn.ts
|
||||
- packages/agent-core/src/harness/env
|
||||
- packages/memory-host-sdk/src/host
|
||||
- extensions/acpx/src
|
||||
- extensions/bonjour/src/advertiser.ts
|
||||
- extensions/browser/src/browser/chrome-mcp.ts
|
||||
- extensions/browser/src/browser/chrome.executables.ts
|
||||
- extensions/browser/src/browser/chrome.ts
|
||||
- extensions/codex/src/app-server/sandbox-exec-server
|
||||
- extensions/codex/src/app-server/transport-stdio.ts
|
||||
- extensions/codex/src/node-cli-sessions.ts
|
||||
- extensions/codex-supervisor/src/json-rpc-client.ts
|
||||
- extensions/file-transfer/src
|
||||
- extensions/google-meet/src
|
||||
- extensions/imessage/src
|
||||
- extensions/memory-core/src/memory/qmd-manager.ts
|
||||
- extensions/memory-wiki/src/obsidian.ts
|
||||
- extensions/microsoft-foundry/cli.ts
|
||||
- extensions/ollama/src/wsl2-crash-loop-check.ts
|
||||
- extensions/qa-lab/src
|
||||
- extensions/signal/src/daemon.ts
|
||||
- extensions/tts-local-cli/speech-provider.ts
|
||||
- extensions/voice-call/src
|
||||
- scripts
|
||||
|
||||
paths-ignore:
|
||||
- "**/node_modules"
|
||||
- "**/coverage"
|
||||
- "**/*.generated.ts"
|
||||
- "**/*.bundle.js"
|
||||
- "**/*-runtime.js"
|
||||
- "**/*.test.ts"
|
||||
- "**/*.test.tsx"
|
||||
- "**/*.spec.ts"
|
||||
- "**/*.spec.tsx"
|
||||
- "**/*.e2e.test.ts"
|
||||
- "**/*.e2e.test.tsx"
|
||||
- "**/*test-support*"
|
||||
- "**/*test-helper*"
|
||||
- "**/*mock*"
|
||||
- "**/*fixture*"
|
||||
- "**/*bench*"
|
||||
47
.github/workflows/codeql.yml
vendored
47
.github/workflows/codeql.yml
vendored
@@ -17,7 +17,28 @@ on:
|
||||
- ".github/actions/**"
|
||||
- ".github/codeql/**"
|
||||
- ".github/workflows/**"
|
||||
- "extensions/acpx/src/**"
|
||||
- "extensions/bonjour/src/advertiser.ts"
|
||||
- "extensions/browser/src/browser/chrome-mcp.ts"
|
||||
- "extensions/browser/src/browser/chrome.executables.ts"
|
||||
- "extensions/browser/src/browser/chrome.ts"
|
||||
- "extensions/codex/src/app-server/sandbox-exec-server/**"
|
||||
- "extensions/codex/src/app-server/transport-stdio.ts"
|
||||
- "extensions/codex/src/node-cli-sessions.ts"
|
||||
- "extensions/codex-supervisor/src/json-rpc-client.ts"
|
||||
- "extensions/file-transfer/src/**"
|
||||
- "extensions/google-meet/src/**"
|
||||
- "extensions/imessage/src/**"
|
||||
- "extensions/memory-core/src/memory/qmd-manager.ts"
|
||||
- "extensions/memory-wiki/src/obsidian.ts"
|
||||
- "extensions/microsoft-foundry/cli.ts"
|
||||
- "extensions/ollama/src/wsl2-crash-loop-check.ts"
|
||||
- "extensions/qa-lab/src/**"
|
||||
- "extensions/signal/src/daemon.ts"
|
||||
- "extensions/tts-local-cli/speech-provider.ts"
|
||||
- "extensions/voice-call/src/**"
|
||||
- "packages/**"
|
||||
- "scripts/**"
|
||||
- "src/**"
|
||||
push:
|
||||
branches:
|
||||
@@ -26,7 +47,28 @@ on:
|
||||
- ".github/actions/**"
|
||||
- ".github/codeql/**"
|
||||
- ".github/workflows/**"
|
||||
- "extensions/acpx/src/**"
|
||||
- "extensions/bonjour/src/advertiser.ts"
|
||||
- "extensions/browser/src/browser/chrome-mcp.ts"
|
||||
- "extensions/browser/src/browser/chrome.executables.ts"
|
||||
- "extensions/browser/src/browser/chrome.ts"
|
||||
- "extensions/codex/src/app-server/sandbox-exec-server/**"
|
||||
- "extensions/codex/src/app-server/transport-stdio.ts"
|
||||
- "extensions/codex/src/node-cli-sessions.ts"
|
||||
- "extensions/codex-supervisor/src/json-rpc-client.ts"
|
||||
- "extensions/file-transfer/src/**"
|
||||
- "extensions/google-meet/src/**"
|
||||
- "extensions/imessage/src/**"
|
||||
- "extensions/memory-core/src/memory/qmd-manager.ts"
|
||||
- "extensions/memory-wiki/src/obsidian.ts"
|
||||
- "extensions/microsoft-foundry/cli.ts"
|
||||
- "extensions/ollama/src/wsl2-crash-loop-check.ts"
|
||||
- "extensions/qa-lab/src/**"
|
||||
- "extensions/signal/src/daemon.ts"
|
||||
- "extensions/tts-local-cli/speech-provider.ts"
|
||||
- "extensions/voice-call/src/**"
|
||||
- "packages/**"
|
||||
- "scripts/**"
|
||||
- "src/**"
|
||||
schedule:
|
||||
- cron: "0 6 * * *"
|
||||
@@ -73,6 +115,11 @@ jobs:
|
||||
runs_on: blacksmith-4vcpu-ubuntu-2404
|
||||
timeout_minutes: 25
|
||||
config_file: ./.github/codeql/codeql-mcp-process-tool-boundary-critical-security.yml
|
||||
- language: javascript-typescript
|
||||
category: process-exec-boundary
|
||||
runs_on: blacksmith-4vcpu-ubuntu-2404
|
||||
timeout_minutes: 25
|
||||
config_file: ./.github/codeql/codeql-process-exec-boundary-critical-security.yml
|
||||
- language: javascript-typescript
|
||||
category: plugin-trust-boundary
|
||||
runs_on: blacksmith-4vcpu-ubuntu-2404
|
||||
|
||||
@@ -452,7 +452,7 @@ For normal PRs, follow scoped CI/check evidence instead of treating parity as a
|
||||
|
||||
The `CodeQL` workflow is intentionally a narrow first-pass security scanner, not the full repository sweep. Daily, manual, and non-draft pull request guard runs scan Actions workflow code plus the highest-risk JavaScript/TypeScript surfaces with high-confidence security queries filtered to high/critical `security-severity`.
|
||||
|
||||
The pull request guard stays light: it only starts for changes under `.github/actions`, `.github/codeql`, `.github/workflows`, `packages`, or `src`, and it runs the same high-confidence security matrix as the scheduled workflow. Android and macOS CodeQL stay out of PR defaults.
|
||||
The pull request guard stays light: it only starts for changes under `.github/actions`, `.github/codeql`, `.github/workflows`, `packages`, `scripts`, `src`, or process-owning bundled plugin runtime paths, and it runs the same high-confidence security matrix as the scheduled workflow. Android and macOS CodeQL stay out of PR defaults.
|
||||
|
||||
### Security categories
|
||||
|
||||
@@ -462,6 +462,7 @@ The pull request guard stays light: it only starts for changes under `.github/ac
|
||||
| `/codeql-security-high/channel-runtime-boundary` | Core channel implementation contracts plus the channel plugin runtime, gateway, Plugin SDK, secrets, audit touchpoints |
|
||||
| `/codeql-security-high/network-ssrf-boundary` | Core SSRF, IP parsing, network guard, web-fetch, and Plugin SDK SSRF policy surfaces |
|
||||
| `/codeql-security-high/mcp-process-tool-boundary` | MCP servers, process execution helpers, outbound delivery, and agent tool-execution gates |
|
||||
| `/codeql-security-high/process-exec-boundary` | Local shell, process spawn helpers, subprocess-owning bundled plugin runtimes, and workflow script glue |
|
||||
| `/codeql-security-high/plugin-trust-boundary` | Plugin install, loader, manifest, registry, package-manager install, source-loading, and Plugin SDK package contract trust surfaces |
|
||||
|
||||
### Platform-specific security shards
|
||||
|
||||
@@ -35,6 +35,7 @@ openclaw wiki status
|
||||
openclaw wiki doctor
|
||||
openclaw wiki init
|
||||
openclaw wiki ingest ./notes/alpha.md
|
||||
openclaw wiki okf import ./knowledge-catalog/okf/bundles/ga4
|
||||
openclaw wiki compile
|
||||
openclaw wiki lint
|
||||
openclaw wiki search "alpha"
|
||||
@@ -104,6 +105,31 @@ Notes:
|
||||
- imported source pages keep provenance in frontmatter
|
||||
- auto-compile can run after ingest when enabled
|
||||
|
||||
### `wiki okf import <path>`
|
||||
|
||||
Import an unpacked Open Knowledge Format bundle into wiki concept pages.
|
||||
|
||||
The importer reads every non-reserved `.md` concept document in the OKF
|
||||
directory tree, requires a non-empty `type` field, and treats unknown OKF
|
||||
`type` values as generic concepts. Reserved OKF `index.md` and `log.md` files
|
||||
are not imported as concepts.
|
||||
|
||||
Imported pages are flattened under `concepts/` so existing wiki compile,
|
||||
search, get, digest, and dashboard flows see them immediately. The original OKF
|
||||
concept ID, `type`, `resource`, `tags`, timestamp, source path, and full
|
||||
frontmatter are preserved in the page frontmatter. Internal OKF markdown links
|
||||
are rewritten to the generated wiki pages; broken or external links are left
|
||||
unchanged.
|
||||
|
||||
Examples:
|
||||
|
||||
```bash
|
||||
openclaw wiki okf import ./bundles/ga4
|
||||
openclaw wiki okf import ./bundles/ga4 --json
|
||||
openclaw wiki search "BigQuery Table" --mode source-evidence --json
|
||||
openclaw wiki get <path-from-json-result>
|
||||
```
|
||||
|
||||
### `wiki compile`
|
||||
|
||||
Rebuild indexes, related blocks, dashboards, and compiled digests.
|
||||
@@ -233,6 +259,8 @@ These require the official `obsidian` CLI on `PATH` when
|
||||
- Use `wiki lint` before trusting contradictory or low-confidence content.
|
||||
- Use `wiki compile` after bulk imports or source changes when you want fresh
|
||||
dashboards and compiled digests immediately.
|
||||
- Use `wiki okf import` when a data catalog, documentation export, or agent
|
||||
enrichment pipeline already emits OKF markdown bundles.
|
||||
- Use `wiki bridge import` when bridge mode depends on newly exported memory
|
||||
artifacts.
|
||||
|
||||
|
||||
@@ -787,9 +787,10 @@ the source of truth for one test run and should define:
|
||||
- docs and code refs
|
||||
- optional plugin requirements
|
||||
- optional gateway config patch
|
||||
- the executable `qa-flow`
|
||||
- an executable `qa-flow` block for flow scenarios, or `execution.kind`/`execution.path`
|
||||
for Vitest and Playwright scenarios
|
||||
|
||||
The reusable runtime surface that backs `qa-flow` is allowed to stay generic
|
||||
The reusable runtime surface that backs `qa-flow` blocks is allowed to stay generic
|
||||
and cross-cutting. For example, markdown scenarios can combine transport-side
|
||||
helpers with browser-side helpers that drive the embedded Control UI through the
|
||||
Gateway `browser.request` seam without adding a special-case runner.
|
||||
@@ -915,6 +916,7 @@ The report should answer:
|
||||
For the inventory of available scenarios - useful when sizing follow-up work or wiring a new transport - run `pnpm openclaw qa coverage` (add `--json` for machine-readable output).
|
||||
When choosing focused proof for a touched behavior or file path, run `pnpm openclaw qa coverage --match <query>`.
|
||||
The match report searches scenario metadata, docs refs, code refs, coverage IDs, plugins, and provider requirements, then prints matching `qa suite --scenario ...` targets.
|
||||
Every `qa suite` scenario execution writes a `qa-evidence.json` artifact. Flow scenarios also write `qa-suite-summary.json` for existing suite/report tooling; scenarios that declare `execution.kind: vitest` or `execution.kind: playwright` run the matching test path and write `qa-vitest-report.md` or `qa-playwright-report.md` plus per-scenario logs.
|
||||
Treat it as a discovery aid, not a gate replacement; the selected scenario still needs the right provider mode, live transport, Multipass, Testbox, or release lane for the behavior under test.
|
||||
|
||||
For character and style checks, run the same scenario across multiple live model
|
||||
|
||||
@@ -30,6 +30,23 @@ title: "Usage tracking"
|
||||
- CLI: `openclaw channels list` prints the same usage snapshot alongside provider config (use `--no-usage` to skip).
|
||||
- macOS menu bar: "Usage" section under Context (only if available).
|
||||
|
||||
## Custom `/usage full` footer
|
||||
|
||||
Set `messages.usageTemplate` to customize the per-response `/usage full`
|
||||
footer. The value can be an inline template object or a JSON file path:
|
||||
|
||||
```json
|
||||
{
|
||||
"messages": {
|
||||
"usageTemplate": "~/.openclaw/usage-footer.json"
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
Templates read the `openclaw.usageLine.v1` contract and can use `scales`,
|
||||
`aliases`, and `output.surfaces` to render channel-specific footers. Missing,
|
||||
unreadable, invalid, or empty templates fall back to the built-in usage line.
|
||||
|
||||
## Providers + credentials
|
||||
|
||||
- **Anthropic (Claude)**: OAuth tokens in auth profiles.
|
||||
|
||||
@@ -42,6 +42,21 @@ health commands above for live connectivity checks.
|
||||
- `channels.<provider>.accounts.<accountId>.healthMonitor.enabled`: multi-account override that wins over the channel-level setting.
|
||||
- These per-channel overrides apply to the built-in channel monitors that expose them today: Discord, Google Chat, iMessage, Microsoft Teams, Signal, Slack, Telegram, and WhatsApp.
|
||||
|
||||
## Uptime monitoring
|
||||
|
||||
External uptime monitoring services should use the dedicated `/health` endpoint, not `/v1/chat/completions`.
|
||||
|
||||
- **DO use:** `GET /health` — instant response, no session created, no LLM call, returns `{"ok":true,"status":"live"}`
|
||||
- **DON'T use:** `/v1/chat/completions` for health checks — each request creates a full agent session with skill snapshot, context assembly, and LLM calls
|
||||
|
||||
When no `x-openclaw-session-key` header or `user` field is provided, `/v1/chat/completions` generates a new random session for each request. Monitoring services that ping every 15 minutes create ~96 sessions/day, each consuming 4–22KB. Over time this causes session store bloat and can lead to context window overflow.
|
||||
|
||||
### Monitoring service setup examples
|
||||
|
||||
- **BetterStack:** Set health check URL to `https://<your-gateway-host>:<port>/health`
|
||||
- **UptimeRobot:** Add a new HTTP monitor with URL `https://<your-gateway-host>:<port>/health`
|
||||
- **Generic:** Any HTTP GET to `/health` returns 200 with `{"ok":true}` when the gateway is healthy
|
||||
|
||||
## When something fails
|
||||
|
||||
- `logged out` or status 409–515 → relink with `openclaw channels logout` then `openclaw channels login`.
|
||||
|
||||
@@ -75,6 +75,7 @@ Auth matrix:
|
||||
- honor `x-openclaw-scopes` when the header is present
|
||||
- fall back to the normal operator default scope set when the header is absent
|
||||
- only lose owner semantics when the caller explicitly narrows scopes and omits `operator.admin`
|
||||
- require `operator.admin` for owner-level request controls such as `x-openclaw-model`
|
||||
|
||||
See [Security](/gateway/security) and [Remote access](/gateway/remote).
|
||||
|
||||
@@ -96,7 +97,7 @@ OpenClaw treats the OpenAI `model` field as an **agent target**, not a raw provi
|
||||
|
||||
Optional request headers:
|
||||
|
||||
- `x-openclaw-model: <provider/model-or-bare-id>` overrides the backend model for the selected agent.
|
||||
- `x-openclaw-model: <provider/model-or-bare-id>` overrides the backend model for the selected agent. Shared-secret bearer callers can use this header. Identity-bearing callers, such as trusted-proxy or private no-auth ingress requests with `x-openclaw-scopes`, need `operator.admin`; write-only callers get `403 missing scope: operator.admin`.
|
||||
- `x-openclaw-agent-id: <agentId>` remains supported as a compatibility override.
|
||||
- `x-openclaw-session-key: <sessionKey>` fully controls session routing.
|
||||
- `x-openclaw-message-channel: <channel>` sets the synthetic ingress channel context for channel-aware prompts and policies.
|
||||
@@ -178,7 +179,7 @@ This is the highest-leverage compatibility set for self-hosted frontends and too
|
||||
|
||||
</Accordion>
|
||||
<Accordion title="How do I override the backend model?">
|
||||
Use `x-openclaw-model`.
|
||||
Use `x-openclaw-model`. This is an owner-level override: it works with the Gateway shared-secret bearer token/password path, and it requires `operator.admin` on identity-bearing HTTP paths such as trusted proxy auth.
|
||||
|
||||
Examples:
|
||||
`x-openclaw-model: openai/gpt-5.4`
|
||||
@@ -191,7 +192,7 @@ This is the highest-leverage compatibility set for self-hosted frontends and too
|
||||
`/v1/embeddings` uses the same agent-target `model` ids.
|
||||
|
||||
Use `model: "openclaw/default"` or `model: "openclaw/<agentId>"`.
|
||||
When you need a specific embedding model, send it in `x-openclaw-model`.
|
||||
When you need a specific embedding model, send it in `x-openclaw-model` from a shared-secret caller or an identity-bearing caller with `operator.admin`.
|
||||
Without that header, the request passes through to the selected agent's normal embedding setup.
|
||||
|
||||
</Accordion>
|
||||
@@ -285,7 +286,7 @@ Expected behavior:
|
||||
|
||||
- `GET /v1/models` should list `openclaw/default`
|
||||
- Open WebUI should use `openclaw/default` as the chat model id
|
||||
- If you want a specific backend provider/model for that agent, set the agent's normal default model or send `x-openclaw-model`
|
||||
- If you want a specific backend provider/model for that agent, set the agent's normal default model or send `x-openclaw-model` from a shared-secret caller or an identity-bearing caller with `operator.admin`
|
||||
|
||||
Quick smoke:
|
||||
|
||||
@@ -370,7 +371,7 @@ Notes:
|
||||
|
||||
- `/v1/models` returns OpenClaw agent targets, not raw provider catalogs.
|
||||
- `openclaw/default` is always present so one stable id works across environments.
|
||||
- Backend provider/model overrides belong in `x-openclaw-model`, not the OpenAI `model` field.
|
||||
- Backend provider/model overrides belong in `x-openclaw-model`, not the OpenAI `model` field. On identity-bearing HTTP auth paths, this header requires `operator.admin`.
|
||||
- `/v1/embeddings` supports `input` as a string or array of strings.
|
||||
|
||||
## Related
|
||||
|
||||
@@ -951,7 +951,7 @@ Important boundary note:
|
||||
- Treat credentials that can call `/v1/chat/completions`, `/v1/responses`, plugin routes such as `/api/v1/admin/rpc`, or `/api/channels/*` as full-access operator secrets for that gateway.
|
||||
- On the OpenAI-compatible HTTP surface, shared-secret bearer auth restores the full default operator scopes (`operator.admin`, `operator.approvals`, `operator.pairing`, `operator.read`, `operator.talk.secrets`, `operator.write`) and owner semantics for agent turns; narrower `x-openclaw-scopes` values do not reduce that shared-secret path.
|
||||
- Per-request scope semantics on HTTP only apply when the request comes from an identity-bearing mode such as trusted proxy auth, or from an explicitly no-auth private ingress.
|
||||
- In those identity-bearing modes, omitting `x-openclaw-scopes` falls back to the normal operator default scope set; send the header explicitly when you want a narrower scope set.
|
||||
- In those identity-bearing modes, omitting `x-openclaw-scopes` falls back to the normal operator default scope set; send the header explicitly when you want a narrower scope set. Owner-level OpenAI-compatible headers such as `x-openclaw-model` require `operator.admin` when scopes are narrowed.
|
||||
- `/tools/invoke` and HTTP session history endpoints follow the same shared-secret rule: token/password bearer auth is treated as full operator access there too, while identity-bearing modes still honor declared scopes.
|
||||
- Do not share these credentials with untrusted callers; prefer separate gateways per trust boundary.
|
||||
|
||||
|
||||
@@ -18,11 +18,13 @@ most Linux-compatible Gateway runtime.
|
||||
Windows Hub is the native WinUI companion app for Windows 10 20H2+ and Windows 11. It installs without administrator privileges and is published with signed
|
||||
x64 and ARM64 installers on OpenClaw releases.
|
||||
|
||||
Download the latest stable installer:
|
||||
Download the latest stable installer from the [OpenClaw releases page](https://github.com/openclaw/openclaw/releases):
|
||||
|
||||
- [OpenClawCompanion-Setup-x64.exe](https://github.com/openclaw/openclaw/releases/latest/download/OpenClawCompanion-Setup-x64.exe)
|
||||
- [OpenClawCompanion-Setup-arm64.exe](https://github.com/openclaw/openclaw/releases/latest/download/OpenClawCompanion-Setup-arm64.exe)
|
||||
- [Checksums](https://github.com/openclaw/openclaw/releases/latest/download/OpenClawCompanion-SHA256SUMS.txt)
|
||||
- [OpenClawCompanion-Setup-x64.exe](https://github.com/openclaw/openclaw/releases/download/v2026.6.5/OpenClawCompanion-Setup-x64.exe)
|
||||
- [OpenClawCompanion-Setup-arm64.exe](https://github.com/openclaw/openclaw/releases/download/v2026.6.5/OpenClawCompanion-Setup-arm64.exe)
|
||||
- [Checksums](https://github.com/openclaw/openclaw/releases/download/v2026.6.5/OpenClawCompanion-SHA256SUMS.txt)
|
||||
|
||||
If a download link above returns a 404, visit the [releases page](https://github.com/openclaw/openclaw/releases) and look for the `OpenClawCompanion-Setup-*` assets on the latest release.
|
||||
|
||||
After install, launch **OpenClaw Companion** from the Start menu or the system
|
||||
tray. The installer also adds shortcuts for Gateway Setup, Chat, Settings,
|
||||
|
||||
@@ -425,6 +425,10 @@ even when the channel payload has no visible text/caption. Rewriting that
|
||||
`content` updates the hook-visible transcript only; it is not rendered as a
|
||||
media caption.
|
||||
|
||||
`reply_payload_sending` events may include `usageState`, a best-effort live
|
||||
per-turn model/usage/context snapshot. Durable delivery, recovered replay, and
|
||||
replies without exact run correlation omit it.
|
||||
|
||||
Message hook contexts expose stable correlation fields when available:
|
||||
`ctx.sessionKey`, `ctx.runId`, `ctx.messageId`, `ctx.senderId`, `ctx.trace`,
|
||||
`ctx.traceId`, `ctx.spanId`, `ctx.parentSpanId`, and `ctx.callDepth`. Inbound
|
||||
|
||||
@@ -25,6 +25,7 @@ less like a pile of Markdown files.
|
||||
- Page-level provenance, confidence, contradictions, and open questions
|
||||
- Compiled digests for agent/runtime consumers
|
||||
- Wiki-native search/get/apply/lint tools
|
||||
- Open Knowledge Format imports into compiled wiki concepts
|
||||
- Optional bridge mode that imports public artifacts from the active memory plugin
|
||||
- Optional Obsidian-friendly render mode and CLI integration
|
||||
|
||||
@@ -135,6 +136,34 @@ The main page groups are:
|
||||
- `syntheses/` for compiled summaries and maintained rollups
|
||||
- `reports/` for generated dashboards
|
||||
|
||||
## Open Knowledge Format imports
|
||||
|
||||
`memory-wiki` can import unpacked Open Knowledge Format bundles with:
|
||||
|
||||
```bash
|
||||
openclaw wiki okf import ./bundles/ga4
|
||||
```
|
||||
|
||||
This is the cleanest fit when a data catalog, documentation crawler, or
|
||||
enrichment agent already produces OKF: keep OKF as the portable exchange
|
||||
artifact, then let `memory-wiki` turn it into OpenClaw-native concept pages and
|
||||
compiled digests.
|
||||
|
||||
The importer follows the OKF v0.1 shape:
|
||||
|
||||
- non-reserved `.md` files are concept documents
|
||||
- each imported concept needs a non-empty `type` frontmatter field
|
||||
- unknown OKF `type` values are accepted
|
||||
- reserved `index.md` and `log.md` files are not imported as concepts
|
||||
- broken or external markdown links are preserved
|
||||
|
||||
Imported concept pages are flattened under `concepts/` so the existing compile,
|
||||
search, get, dashboard, and prompt-digest paths see them without adding a second
|
||||
wiki tree. Each page keeps the original OKF concept ID, source path, `type`,
|
||||
`resource`, `tags`, timestamp, and full producer frontmatter. Internal OKF links
|
||||
are rewritten to the generated wiki concept pages and also emitted as structured
|
||||
`relationships` entries with `kind: okf-link`.
|
||||
|
||||
## Structured claims and evidence
|
||||
|
||||
Pages can carry structured `claims` frontmatter, not just freeform text.
|
||||
|
||||
@@ -101,6 +101,28 @@
|
||||
"contextWindow": 200000,
|
||||
"maxTokens": 64000
|
||||
},
|
||||
{
|
||||
"id": "claude-haiku-4-5",
|
||||
"name": "Claude Haiku 4.5",
|
||||
"reasoning": true,
|
||||
"input": ["text", "image"],
|
||||
"mediaInput": {
|
||||
"image": { "maxSidePx": 1568, "preferredSidePx": 1568, "tokenMode": "provider" }
|
||||
},
|
||||
"contextWindow": 200000,
|
||||
"maxTokens": 64000
|
||||
},
|
||||
{
|
||||
"id": "claude-haiku-4-5-20251001",
|
||||
"name": "Claude Haiku 4.5",
|
||||
"reasoning": true,
|
||||
"input": ["text", "image"],
|
||||
"mediaInput": {
|
||||
"image": { "maxSidePx": 1568, "preferredSidePx": 1568, "tokenMode": "provider" }
|
||||
},
|
||||
"contextWindow": 200000,
|
||||
"maxTokens": 64000
|
||||
},
|
||||
{
|
||||
"id": "claude-sonnet-4-6",
|
||||
"name": "Claude Sonnet 4.6",
|
||||
|
||||
57
extensions/anthropic/openclaw.plugin.test.ts
Normal file
57
extensions/anthropic/openclaw.plugin.test.ts
Normal file
@@ -0,0 +1,57 @@
|
||||
// Anthropic tests cover provider manifest model catalog behavior.
|
||||
import { readFileSync } from "node:fs";
|
||||
import { describe, expect, it } from "vitest";
|
||||
|
||||
type AnthropicManifest = {
|
||||
modelCatalog?: {
|
||||
providers?: {
|
||||
anthropic?: {
|
||||
models?: Array<{
|
||||
id?: string;
|
||||
name?: string;
|
||||
reasoning?: boolean;
|
||||
input?: string[];
|
||||
mediaInput?: {
|
||||
image?: {
|
||||
maxSidePx?: number;
|
||||
preferredSidePx?: number;
|
||||
tokenMode?: string;
|
||||
};
|
||||
};
|
||||
contextWindow?: number;
|
||||
maxTokens?: number;
|
||||
}>;
|
||||
};
|
||||
};
|
||||
discovery?: Record<string, string>;
|
||||
};
|
||||
};
|
||||
|
||||
const manifest = JSON.parse(
|
||||
readFileSync(new URL("./openclaw.plugin.json", import.meta.url), "utf8"),
|
||||
) as AnthropicManifest;
|
||||
|
||||
describe("Anthropic plugin manifest", () => {
|
||||
it("resolves both official Claude Haiku 4.5 API identifiers from the static catalog", () => {
|
||||
expect(manifest.modelCatalog?.discovery?.anthropic).toBe("static");
|
||||
|
||||
const models = manifest.modelCatalog?.providers?.anthropic?.models ?? [];
|
||||
for (const id of ["claude-haiku-4-5", "claude-haiku-4-5-20251001"]) {
|
||||
expect(models.find((model) => model.id === id)).toEqual({
|
||||
id,
|
||||
name: "Claude Haiku 4.5",
|
||||
reasoning: true,
|
||||
input: ["text", "image"],
|
||||
mediaInput: {
|
||||
image: {
|
||||
maxSidePx: 1568,
|
||||
preferredSidePx: 1568,
|
||||
tokenMode: "provider",
|
||||
},
|
||||
},
|
||||
contextWindow: 200000,
|
||||
maxTokens: 64000,
|
||||
});
|
||||
}
|
||||
});
|
||||
});
|
||||
@@ -24,6 +24,75 @@ function hasDiscordComponentObjectKeys(value: unknown): value is Record<string,
|
||||
);
|
||||
}
|
||||
|
||||
function readDiscordThreadArchiveTimestamp(thread: unknown): string | undefined {
|
||||
if (!thread || typeof thread !== "object" || Array.isArray(thread)) {
|
||||
return undefined;
|
||||
}
|
||||
const record = thread as Record<string, unknown>;
|
||||
const metadata = record.thread_metadata;
|
||||
if (metadata && typeof metadata === "object" && !Array.isArray(metadata)) {
|
||||
const archiveTimestamp = (metadata as Record<string, unknown>).archive_timestamp;
|
||||
if (typeof archiveTimestamp === "string" && archiveTimestamp.trim()) {
|
||||
return archiveTimestamp;
|
||||
}
|
||||
}
|
||||
return undefined;
|
||||
}
|
||||
|
||||
type DiscordThreadListActionResult = {
|
||||
ok: true;
|
||||
threads: unknown;
|
||||
complete: boolean;
|
||||
hasMore: boolean;
|
||||
returnedCount: number;
|
||||
source: "discord.threadList.archived" | "discord.threadList.active";
|
||||
query: {
|
||||
guildId: string;
|
||||
channelId?: string;
|
||||
includeArchived: boolean;
|
||||
before?: string;
|
||||
limit?: number;
|
||||
};
|
||||
nextBefore?: string;
|
||||
};
|
||||
|
||||
function normalizeDiscordThreadListActionResult(params: {
|
||||
value: unknown;
|
||||
includeArchived: boolean;
|
||||
channelId?: string;
|
||||
guildId: string;
|
||||
limit?: number;
|
||||
before?: string;
|
||||
}): DiscordThreadListActionResult {
|
||||
const record =
|
||||
params.value && typeof params.value === "object" && !Array.isArray(params.value)
|
||||
? (params.value as Record<string, unknown>)
|
||||
: undefined;
|
||||
const threadItems = Array.isArray(record?.threads) ? record.threads : [];
|
||||
const hasMore = record?.has_more === true;
|
||||
const nextBefore =
|
||||
params.includeArchived && hasMore
|
||||
? readDiscordThreadArchiveTimestamp(threadItems[threadItems.length - 1])
|
||||
: undefined;
|
||||
|
||||
return {
|
||||
ok: true,
|
||||
threads: params.value,
|
||||
complete: !hasMore,
|
||||
hasMore,
|
||||
returnedCount: threadItems.length,
|
||||
source: params.includeArchived ? "discord.threadList.archived" : "discord.threadList.active",
|
||||
query: {
|
||||
guildId: params.guildId,
|
||||
...(params.channelId ? { channelId: params.channelId } : {}),
|
||||
includeArchived: params.includeArchived,
|
||||
...(params.before ? { before: params.before } : {}),
|
||||
...(params.limit !== undefined ? { limit: params.limit } : {}),
|
||||
},
|
||||
...(nextBefore ? { nextBefore } : {}),
|
||||
};
|
||||
}
|
||||
|
||||
async function appendDiscordThreadRenameResult(
|
||||
ctx: DiscordMessagingActionContext,
|
||||
params: {
|
||||
@@ -306,7 +375,16 @@ export async function handleDiscordMessageSendAction(ctx: DiscordMessagingAction
|
||||
},
|
||||
ctx.withOpts(),
|
||||
);
|
||||
return jsonResult({ ok: true, threads });
|
||||
return jsonResult(
|
||||
normalizeDiscordThreadListActionResult({
|
||||
value: threads,
|
||||
guildId,
|
||||
channelId,
|
||||
includeArchived: includeArchived === true,
|
||||
before,
|
||||
limit,
|
||||
}),
|
||||
);
|
||||
}
|
||||
case "threadReply": {
|
||||
if (!ctx.isActionEnabled("threads")) {
|
||||
|
||||
@@ -101,6 +101,7 @@ const {
|
||||
kickMemberDiscord,
|
||||
listGuildChannelsDiscord,
|
||||
listPinsDiscord,
|
||||
listThreadsDiscord,
|
||||
moveChannelDiscord,
|
||||
reactMessageDiscord,
|
||||
readMessagesDiscord,
|
||||
@@ -271,6 +272,138 @@ describe("handleDiscordMessagingAction", () => {
|
||||
]);
|
||||
});
|
||||
|
||||
it("surfaces incomplete archived thread pages at the action boundary", async () => {
|
||||
listThreadsDiscord.mockResolvedValueOnce({
|
||||
threads: [
|
||||
{
|
||||
id: "thread-1",
|
||||
name: "Old project",
|
||||
thread_metadata: {
|
||||
archive_timestamp: "2026-05-25T17:00:00.000Z",
|
||||
},
|
||||
},
|
||||
],
|
||||
members: [],
|
||||
has_more: true,
|
||||
});
|
||||
|
||||
const result = await handleMessagingAction(
|
||||
"threadList",
|
||||
{
|
||||
guildId: "G1",
|
||||
channelId: "C1",
|
||||
includeArchived: true,
|
||||
before: "2026-05-26T17:00:00.000Z",
|
||||
limit: 1,
|
||||
},
|
||||
enableAllActions,
|
||||
);
|
||||
|
||||
expect(mockCall(listThreadsDiscord, "listThreadsDiscord")).toEqual([
|
||||
{
|
||||
guildId: "G1",
|
||||
channelId: "C1",
|
||||
includeArchived: true,
|
||||
before: "2026-05-26T17:00:00.000Z",
|
||||
limit: 1,
|
||||
},
|
||||
{ cfg: DISCORD_TEST_CFG },
|
||||
]);
|
||||
expect(result.details).toMatchObject({
|
||||
ok: true,
|
||||
complete: false,
|
||||
hasMore: true,
|
||||
returnedCount: 1,
|
||||
source: "discord.threadList.archived",
|
||||
nextBefore: "2026-05-25T17:00:00.000Z",
|
||||
query: {
|
||||
guildId: "G1",
|
||||
channelId: "C1",
|
||||
includeArchived: true,
|
||||
before: "2026-05-26T17:00:00.000Z",
|
||||
limit: 1,
|
||||
},
|
||||
});
|
||||
expect((result.details as { threads?: unknown }).threads).toEqual({
|
||||
threads: [
|
||||
{
|
||||
id: "thread-1",
|
||||
name: "Old project",
|
||||
thread_metadata: {
|
||||
archive_timestamp: "2026-05-25T17:00:00.000Z",
|
||||
},
|
||||
},
|
||||
],
|
||||
members: [],
|
||||
has_more: true,
|
||||
});
|
||||
});
|
||||
|
||||
it("omits archived thread pagination cursors when Discord omits archive timestamps", async () => {
|
||||
listThreadsDiscord.mockResolvedValueOnce({
|
||||
threads: [
|
||||
{
|
||||
id: "thread-without-archive-timestamp",
|
||||
name: "Legacy project",
|
||||
},
|
||||
],
|
||||
members: [],
|
||||
has_more: true,
|
||||
});
|
||||
|
||||
const result = await handleMessagingAction(
|
||||
"threadList",
|
||||
{
|
||||
guildId: "G1",
|
||||
channelId: "C1",
|
||||
includeArchived: true,
|
||||
limit: 1,
|
||||
},
|
||||
enableAllActions,
|
||||
);
|
||||
|
||||
expect(result.details).toMatchObject({
|
||||
ok: true,
|
||||
complete: false,
|
||||
hasMore: true,
|
||||
returnedCount: 1,
|
||||
source: "discord.threadList.archived",
|
||||
});
|
||||
expect(result.details).not.toHaveProperty("nextBefore");
|
||||
});
|
||||
|
||||
it("marks active thread results complete when Discord returns no pagination state", async () => {
|
||||
listThreadsDiscord.mockResolvedValueOnce({
|
||||
threads: [{ id: "thread-active", name: "Current project" }],
|
||||
members: [{ id: "member-1" }],
|
||||
});
|
||||
|
||||
const result = await handleMessagingAction(
|
||||
"threadList",
|
||||
{
|
||||
guildId: "G1",
|
||||
},
|
||||
enableAllActions,
|
||||
);
|
||||
|
||||
expect(result.details).toMatchObject({
|
||||
ok: true,
|
||||
complete: true,
|
||||
hasMore: false,
|
||||
returnedCount: 1,
|
||||
source: "discord.threadList.active",
|
||||
query: {
|
||||
guildId: "G1",
|
||||
includeArchived: false,
|
||||
},
|
||||
});
|
||||
expect((result.details as { threads?: unknown }).threads).toEqual({
|
||||
threads: [{ id: "thread-active", name: "Current project" }],
|
||||
members: [{ id: "member-1" }],
|
||||
});
|
||||
expect(result.details).not.toHaveProperty("nextBefore");
|
||||
});
|
||||
|
||||
it("resolves Discord DM targets for reaction adds", async () => {
|
||||
const resolveReactionTarget = vi.fn(async () => "DM1");
|
||||
discordMessagingActionRuntime.resolveDiscordReactionTargetChannelId = resolveReactionTarget;
|
||||
|
||||
@@ -144,19 +144,19 @@ describe("fireworks provider plugin", () => {
|
||||
expect(resolved?.reasoning).toBe(false);
|
||||
});
|
||||
|
||||
it("disables reasoning metadata for Fireworks Kimi k2.6 dynamic models", async () => {
|
||||
it("defers manifest catalog models to core static-catalog resolution", async () => {
|
||||
const provider = await registerSingleProviderPlugin(fireworksPlugin);
|
||||
const resolved = provider.resolveDynamicModel?.(
|
||||
createProviderDynamicModelContext({
|
||||
provider: "fireworks",
|
||||
modelId: "accounts/fireworks/models/kimi-k2p6",
|
||||
models: [createFireworksDefaultRuntimeModel({ reasoning: false })],
|
||||
}),
|
||||
);
|
||||
for (const modelId of [FIREWORKS_K2_6_MODEL_ID, FIREWORKS_DEFAULT_MODEL_ID]) {
|
||||
const resolved = provider.resolveDynamicModel?.(
|
||||
createProviderDynamicModelContext({
|
||||
provider: "fireworks",
|
||||
modelId,
|
||||
models: [createFireworksDefaultRuntimeModel({ reasoning: false })],
|
||||
}),
|
||||
);
|
||||
|
||||
expect(resolved?.provider).toBe("fireworks");
|
||||
expect(resolved?.id).toBe("accounts/fireworks/models/kimi-k2p6");
|
||||
expect(resolved?.reasoning).toBe(false);
|
||||
expect(resolved).toBeUndefined();
|
||||
}
|
||||
});
|
||||
|
||||
it("exposes off-only thinking policy for Fireworks Kimi models", async () => {
|
||||
|
||||
@@ -15,11 +15,13 @@ import {
|
||||
FIREWORKS_DEFAULT_CONTEXT_WINDOW,
|
||||
FIREWORKS_DEFAULT_MAX_TOKENS,
|
||||
FIREWORKS_DEFAULT_MODEL_ID,
|
||||
isFireworksCatalogModelId,
|
||||
} from "./provider-catalog.js";
|
||||
import { wrapFireworksProviderStream } from "./stream.js";
|
||||
import { resolveFireworksThinkingProfile } from "./thinking-policy.js";
|
||||
|
||||
const PROVIDER_ID = "fireworks";
|
||||
|
||||
function isFireworksGlmModelId(modelId: string): boolean {
|
||||
const normalized = modelId.trim().toLowerCase();
|
||||
const lastSegment = normalized.split("/").pop() ?? normalized;
|
||||
@@ -35,6 +37,11 @@ function resolveFireworksDynamicModel(ctx: ProviderResolveDynamicModelContext) {
|
||||
if (!modelId) {
|
||||
return undefined;
|
||||
}
|
||||
|
||||
if (isFireworksCatalogModelId(modelId)) {
|
||||
return undefined;
|
||||
}
|
||||
|
||||
const isKimiModel = isFireworksKimiModelId(modelId);
|
||||
const input = resolveFireworksDynamicInput(modelId);
|
||||
|
||||
|
||||
@@ -37,6 +37,7 @@
|
||||
{
|
||||
"id": "accounts/fireworks/models/kimi-k2p6",
|
||||
"name": "Kimi K2.6",
|
||||
"reasoning": false,
|
||||
"input": ["text", "image"],
|
||||
"contextWindow": 262144,
|
||||
"maxTokens": 262144,
|
||||
@@ -50,6 +51,7 @@
|
||||
{
|
||||
"id": "accounts/fireworks/routers/kimi-k2p5-turbo",
|
||||
"name": "Kimi K2.5 Turbo (Fire Pass)",
|
||||
"reasoning": false,
|
||||
"input": ["text", "image"],
|
||||
"contextWindow": 256000,
|
||||
"maxTokens": 256000,
|
||||
|
||||
@@ -31,16 +31,12 @@ export const FIREWORKS_DEFAULT_MAX_TOKENS = FIREWORKS_DEFAULT_MODEL.maxTokens;
|
||||
export const FIREWORKS_K2_6_CONTEXT_WINDOW = FIREWORKS_K2_6_MODEL.contextWindow;
|
||||
export const FIREWORKS_K2_6_MAX_TOKENS = FIREWORKS_K2_6_MODEL.maxTokens;
|
||||
|
||||
function cloneFireworksCatalogModel(model: ModelDefinitionConfig): ModelDefinitionConfig {
|
||||
return {
|
||||
...model,
|
||||
input: [...model.input],
|
||||
cost: { ...model.cost },
|
||||
};
|
||||
export function isFireworksCatalogModelId(modelId: string): boolean {
|
||||
return FIREWORKS_MANIFEST_PROVIDER.models.some((model) => model.id === modelId);
|
||||
}
|
||||
|
||||
export function buildFireworksCatalogModels(): ModelDefinitionConfig[] {
|
||||
return FIREWORKS_MANIFEST_PROVIDER.models.map(cloneFireworksCatalogModel);
|
||||
return FIREWORKS_MANIFEST_PROVIDER.models.map((model) => structuredClone(model));
|
||||
}
|
||||
|
||||
export function buildFireworksProvider(): ModelProviderConfig {
|
||||
|
||||
@@ -56,6 +56,10 @@ function isCopilotGeminiModelId(modelId: string): boolean {
|
||||
return /(?:^|[-_.])gemini(?:$|[-_.])/.test(modelId);
|
||||
}
|
||||
|
||||
function isCopilotClaude45ModelId(modelId: string): boolean {
|
||||
return /^claude-(?:haiku|opus|sonnet)-4[.-]5(?:$|[-.])/.test(modelId);
|
||||
}
|
||||
|
||||
export function resolveCopilotTransportApi(modelId: string): CopilotRuntimeApi {
|
||||
const normalized = normalizeOptionalLowercaseString(modelId) ?? "";
|
||||
if (normalized.includes("claude")) {
|
||||
@@ -71,7 +75,15 @@ export function resolveCopilotModelCompat(
|
||||
modelId: string,
|
||||
): ModelDefinitionConfig["compat"] | undefined {
|
||||
const normalized = normalizeOptionalLowercaseString(modelId) ?? "";
|
||||
return isCopilotGeminiModelId(normalized) ? { ...COPILOT_CHAT_COMPLETIONS_COMPAT } : undefined;
|
||||
if (isCopilotGeminiModelId(normalized)) {
|
||||
return { ...COPILOT_CHAT_COMPLETIONS_COMPAT };
|
||||
}
|
||||
// Copilot's Claude 4.5 endpoints reject Anthropic's eager tool extension,
|
||||
// while current Claude 4.6+ endpoints accept it.
|
||||
if (isCopilotClaude45ModelId(normalized)) {
|
||||
return { supportsEagerToolInputStreaming: false };
|
||||
}
|
||||
return undefined;
|
||||
}
|
||||
|
||||
function compatSupportsEffort(
|
||||
|
||||
@@ -90,8 +90,18 @@ describe("github-copilot model defaults", () => {
|
||||
const def = buildCopilotModelDefinition("claude-sonnet-4.6");
|
||||
expect(def.id).toBe("claude-sonnet-4.6");
|
||||
expect(def.api).toBe("anthropic-messages");
|
||||
expect(def.compat).toBeUndefined();
|
||||
});
|
||||
|
||||
it.each(["claude-haiku-4.5", "claude-sonnet-4-5"])(
|
||||
"disables eager tool streaming for Copilot Claude 4.5 model %s",
|
||||
(modelId) => {
|
||||
expect(buildCopilotModelDefinition(modelId).compat).toEqual({
|
||||
supportsEagerToolInputStreaming: false,
|
||||
});
|
||||
},
|
||||
);
|
||||
|
||||
it("uses static metadata overrides for gpt-5.5 fallback rows", () => {
|
||||
const def = buildCopilotModelDefinition("gpt-5.5");
|
||||
expect(def).toEqual({
|
||||
@@ -243,6 +253,12 @@ describe("resolveCopilotForwardCompatModel", () => {
|
||||
expect((result as unknown as Record<string, unknown>).input).toEqual(["text", "image"]);
|
||||
});
|
||||
|
||||
it("disables eager tool streaming for synthetic Copilot Claude 4.5 models", () => {
|
||||
const result = requireResolvedModel(createMockCtx("claude-haiku-4.5"));
|
||||
expect(result.api).toBe("anthropic-messages");
|
||||
expect(result.compat).toEqual({ supportsEagerToolInputStreaming: false });
|
||||
});
|
||||
|
||||
it("creates synthetic Gemini models with Chat Completions compatibility", () => {
|
||||
const result = requireResolvedModel(createMockCtx("gemini-3.1-pro-preview"));
|
||||
expect((result as unknown as Record<string, unknown>).api).toBe("openai-completions");
|
||||
@@ -620,6 +636,7 @@ describe("fetchCopilotModelCatalog", () => {
|
||||
const opus45 = out.find((m) => m.id === "claude-opus-4-5");
|
||||
expect(opus45?.thinkingLevelMap).toEqual({ xhigh: null, max: null });
|
||||
expect(opus45?.compat).toEqual({
|
||||
supportsEagerToolInputStreaming: false,
|
||||
supportedReasoningEfforts: ["low", "medium", "high", "max"],
|
||||
});
|
||||
});
|
||||
|
||||
@@ -368,6 +368,30 @@ describe("getMemorySearchManager caching", () => {
|
||||
expect(searchResults).toHaveLength(1);
|
||||
});
|
||||
|
||||
it("returns the qmd startup failure when builtin fallback is unavailable", async () => {
|
||||
const cfg = createQmdCfg("missing-qmd-no-builtin");
|
||||
checkQmdBinaryAvailability.mockResolvedValueOnce({
|
||||
available: false,
|
||||
reason: "binary",
|
||||
error: "spawn qmd ENOENT",
|
||||
});
|
||||
mockMemoryIndexGet.mockRejectedValueOnce(
|
||||
new Error(
|
||||
'Memory search unavailable: embedding provider "openai" is configured but unavailable.',
|
||||
),
|
||||
);
|
||||
|
||||
const result = await getMemorySearchManager({ cfg, agentId: "missing-qmd-no-builtin" });
|
||||
|
||||
expect(result.manager).toBeNull();
|
||||
expect(result.error).toContain("qmd binary unavailable (qmd): spawn qmd ENOENT");
|
||||
expect(result.error).toContain(
|
||||
'builtin fallback unavailable: Memory search unavailable: embedding provider "openai" is configured but unavailable.',
|
||||
);
|
||||
expect(createQmdManagerMock).not.toHaveBeenCalled();
|
||||
expect(mockMemoryIndexGet).toHaveBeenCalledTimes(1);
|
||||
});
|
||||
|
||||
it("treats legacy qmd unavailable results without a reason as binary failures", async () => {
|
||||
const cfg = createQmdCfg("missing-qmd-legacy");
|
||||
checkQmdBinaryAvailability.mockResolvedValueOnce({
|
||||
|
||||
@@ -262,16 +262,18 @@ export async function getMemorySearchManager(params: {
|
||||
}
|
||||
|
||||
if (transient) {
|
||||
const { manager } = await createPrimaryQmdManager(
|
||||
const { manager, failureReason } = await createPrimaryQmdManager(
|
||||
params.purpose === "cli" ? "cli" : "status",
|
||||
);
|
||||
return manager ? { manager } : await getBuiltinMemorySearchManager(params);
|
||||
return manager
|
||||
? { manager }
|
||||
: await getBuiltinMemorySearchManagerAfterQmdFailure(params, failureReason);
|
||||
}
|
||||
|
||||
const recentFailure = getActiveQmdManagerOpenFailure(scopeKey, identityKey);
|
||||
if (recentFailure) {
|
||||
log.debug?.(`qmd memory unavailable; using builtin during cooldown: ${recentFailure.reason}`);
|
||||
return await getBuiltinMemorySearchManager(params);
|
||||
return await getBuiltinMemorySearchManagerAfterQmdFailure(params, recentFailure.reason);
|
||||
}
|
||||
|
||||
const pending = PENDING_QMD_MANAGER_CREATES.get(scopeKey);
|
||||
@@ -280,16 +282,14 @@ export async function getMemorySearchManager(params: {
|
||||
return await getMemorySearchManager(params);
|
||||
}
|
||||
|
||||
let pendingFailureReason: string | undefined;
|
||||
const pendingCreate: PendingQmdManagerCreate = {
|
||||
identityKey,
|
||||
promise: (async () => {
|
||||
const created = await createFullQmdManager(identityKey);
|
||||
if (!created.entry) {
|
||||
recordQmdManagerOpenFailure(
|
||||
scopeKey,
|
||||
identityKey,
|
||||
created.failureReason ?? "qmd memory unavailable",
|
||||
);
|
||||
pendingFailureReason = created.failureReason ?? "qmd memory unavailable";
|
||||
recordQmdManagerOpenFailure(scopeKey, identityKey, pendingFailureReason);
|
||||
return null;
|
||||
}
|
||||
QMD_MANAGER_CACHE.set(scopeKey, created.entry);
|
||||
@@ -308,12 +308,35 @@ export async function getMemorySearchManager(params: {
|
||||
};
|
||||
PENDING_QMD_MANAGER_CREATES.set(scopeKey, pendingCreate);
|
||||
const manager = await pendingCreate.promise;
|
||||
return manager ? { manager } : await getBuiltinMemorySearchManager(params);
|
||||
return manager
|
||||
? { manager }
|
||||
: await getBuiltinMemorySearchManagerAfterQmdFailure(params, pendingFailureReason);
|
||||
}
|
||||
|
||||
return await getBuiltinMemorySearchManager(params);
|
||||
}
|
||||
|
||||
async function getBuiltinMemorySearchManagerAfterQmdFailure(
|
||||
params: {
|
||||
cfg: OpenClawConfig;
|
||||
agentId: string;
|
||||
purpose?: MemorySearchManagerPurpose;
|
||||
},
|
||||
qmdFailureReason: string | undefined,
|
||||
): Promise<MemorySearchManagerResult> {
|
||||
const fallback = await getBuiltinMemorySearchManager(params);
|
||||
if (fallback.manager || !qmdFailureReason) {
|
||||
return fallback;
|
||||
}
|
||||
const fallbackError = fallback.error?.trim();
|
||||
return {
|
||||
manager: null,
|
||||
error: fallbackError
|
||||
? `${qmdFailureReason}; builtin fallback unavailable: ${fallbackError}`
|
||||
: qmdFailureReason,
|
||||
};
|
||||
}
|
||||
|
||||
async function getBuiltinMemorySearchManager(params: {
|
||||
cfg: OpenClawConfig;
|
||||
agentId: string;
|
||||
|
||||
@@ -10,6 +10,7 @@ import {
|
||||
runWikiChatGptImport,
|
||||
runWikiChatGptRollback,
|
||||
runWikiDoctor,
|
||||
runWikiOkfImport,
|
||||
runWikiStatus,
|
||||
} from "./cli.js";
|
||||
import type { MemoryWikiPluginConfig } from "./config.js";
|
||||
@@ -27,6 +28,7 @@ vi.mock("openclaw/plugin-sdk/gateway-runtime", () => ({
|
||||
const { createVault } = createMemoryWikiTestHarness();
|
||||
let suiteRoot = "";
|
||||
let caseIndex = 0;
|
||||
let stdoutWriteMock: ReturnType<typeof vi.fn>;
|
||||
|
||||
describe("memory-wiki cli", () => {
|
||||
beforeAll(async () => {
|
||||
@@ -41,8 +43,9 @@ describe("memory-wiki cli", () => {
|
||||
|
||||
beforeEach(() => {
|
||||
callGatewayFromCliMock.mockReset();
|
||||
stdoutWriteMock = vi.fn(() => true);
|
||||
vi.spyOn(process.stdout, "write").mockImplementation(
|
||||
(() => true) as typeof process.stdout.write,
|
||||
stdoutWriteMock as unknown as typeof process.stdout.write,
|
||||
);
|
||||
process.exitCode = undefined;
|
||||
});
|
||||
@@ -174,6 +177,65 @@ describe("memory-wiki cli", () => {
|
||||
);
|
||||
});
|
||||
|
||||
it("registers OKF import and searches imported concepts", async () => {
|
||||
const { rootDir, config } = await createCliVault();
|
||||
const bundlePath = path.join(rootDir, "okf-bundle");
|
||||
await fs.mkdir(path.join(bundlePath, "tables"), { recursive: true });
|
||||
await fs.writeFile(path.join(bundlePath, "index.md"), "# Sales OKF\n", "utf8");
|
||||
await fs.writeFile(
|
||||
path.join(bundlePath, "tables", "customers.md"),
|
||||
`---
|
||||
type: BigQuery Table
|
||||
title: Customers
|
||||
---
|
||||
|
||||
Customer rows.
|
||||
`,
|
||||
"utf8",
|
||||
);
|
||||
await fs.writeFile(
|
||||
path.join(bundlePath, "tables", "orders.md"),
|
||||
`---
|
||||
type: BigQuery Table
|
||||
title: Orders
|
||||
description: One row per completed order.
|
||||
---
|
||||
|
||||
Orders join to [customers](/tables/customers.md).
|
||||
`,
|
||||
"utf8",
|
||||
);
|
||||
|
||||
const program = new Command();
|
||||
program.name("test");
|
||||
registerWikiCli(program, config);
|
||||
|
||||
await program.parseAsync(["wiki", "okf", "import", bundlePath, "--json"], { from: "user" });
|
||||
|
||||
const importOutput = String(stdoutWriteMock.mock.calls.at(-1)?.[0] ?? "");
|
||||
const importResult = JSON.parse(importOutput) as Awaited<ReturnType<typeof runWikiOkfImport>>;
|
||||
expect(importResult.importedCount).toBe(2);
|
||||
expect(importResult.pagePaths).toEqual(
|
||||
expect.arrayContaining([
|
||||
expect.stringMatching(/^concepts\/okf-okf-bundle-[0-9a-f]{8}-tables-orders-/),
|
||||
]),
|
||||
);
|
||||
|
||||
stdoutWriteMock.mockClear();
|
||||
await program.parseAsync(["wiki", "search", "completed order", "--json"], { from: "user" });
|
||||
|
||||
const searchOutput = String(stdoutWriteMock.mock.calls.at(-1)?.[0] ?? "");
|
||||
const searchResults = JSON.parse(searchOutput) as Array<{ path: string; title: string }>;
|
||||
expect(searchResults).toEqual(
|
||||
expect.arrayContaining([
|
||||
expect.objectContaining({
|
||||
title: "Orders",
|
||||
path: expect.stringMatching(/^concepts\/okf-okf-bundle-[0-9a-f]{8}-tables-orders-/),
|
||||
}),
|
||||
]),
|
||||
);
|
||||
});
|
||||
|
||||
it("rejects apply confidence values outside the documented range", async () => {
|
||||
const { config } = await createCliVault();
|
||||
const program = new Command();
|
||||
|
||||
@@ -33,6 +33,7 @@ import {
|
||||
runObsidianOpen,
|
||||
runObsidianSearch,
|
||||
} from "./obsidian.js";
|
||||
import { formatOkfImportSummary, importMemoryWikiOkfBundle } from "./okf.js";
|
||||
import {
|
||||
getMemoryWikiPage,
|
||||
searchMemoryWiki,
|
||||
@@ -88,6 +89,10 @@ type WikiIngestCommandOptions = {
|
||||
title?: string;
|
||||
};
|
||||
|
||||
type WikiOkfImportCommandOptions = {
|
||||
json?: boolean;
|
||||
};
|
||||
|
||||
type WikiSearchCommandOptions = {
|
||||
json?: boolean;
|
||||
maxResults?: number;
|
||||
@@ -590,6 +595,24 @@ export async function runWikiIngest(params: {
|
||||
});
|
||||
}
|
||||
|
||||
export async function runWikiOkfImport(params: {
|
||||
config: ResolvedMemoryWikiConfig;
|
||||
bundlePath: string;
|
||||
json?: boolean;
|
||||
stdout?: Pick<NodeJS.WriteStream, "write">;
|
||||
}) {
|
||||
return runWikiCommandWithSummary({
|
||||
json: params.json,
|
||||
stdout: params.stdout,
|
||||
run: () =>
|
||||
importMemoryWikiOkfBundle({
|
||||
config: params.config,
|
||||
bundlePath: params.bundlePath,
|
||||
}),
|
||||
render: formatOkfImportSummary,
|
||||
});
|
||||
}
|
||||
|
||||
export async function runWikiSearch(params: {
|
||||
config: ResolvedMemoryWikiConfig;
|
||||
appConfig?: OpenClawConfig;
|
||||
@@ -965,6 +988,16 @@ export function registerWikiCli(
|
||||
await runWikiIngest({ config, inputPath, title: opts.title, json: opts.json });
|
||||
});
|
||||
|
||||
const okf = wiki.command("okf").description("Import Open Knowledge Format bundles");
|
||||
okf
|
||||
.command("import")
|
||||
.description("Import an unpacked OKF bundle into wiki concept pages")
|
||||
.argument("<path>", "OKF bundle directory")
|
||||
.option("--json", "Print JSON")
|
||||
.action(async (bundlePath: string, opts: WikiOkfImportCommandOptions) => {
|
||||
await runWikiOkfImport({ config, bundlePath, json: opts.json });
|
||||
});
|
||||
|
||||
addWikiSearchConfigOptions(
|
||||
wiki
|
||||
.command("search")
|
||||
|
||||
@@ -4,7 +4,7 @@ import path from "node:path";
|
||||
import { appendRegularFile } from "openclaw/plugin-sdk/security-runtime";
|
||||
|
||||
type MemoryWikiLogEntry = {
|
||||
type: "init" | "ingest" | "compile" | "lint";
|
||||
type: "init" | "ingest" | "okf-import" | "compile" | "lint";
|
||||
timestamp: string;
|
||||
details?: Record<string, unknown>;
|
||||
};
|
||||
|
||||
609
extensions/memory-wiki/src/okf.test.ts
Normal file
609
extensions/memory-wiki/src/okf.test.ts
Normal file
@@ -0,0 +1,609 @@
|
||||
// Memory Wiki tests cover Open Knowledge Format import behavior.
|
||||
import fs from "node:fs/promises";
|
||||
import path from "node:path";
|
||||
import { describe, expect, it } from "vitest";
|
||||
import { parseWikiMarkdown } from "./markdown.js";
|
||||
import { importMemoryWikiOkfBundle } from "./okf.js";
|
||||
import { searchMemoryWiki } from "./query.js";
|
||||
import { createMemoryWikiTestHarness } from "./test-helpers.js";
|
||||
|
||||
const { createTempDir, createVault } = createMemoryWikiTestHarness();
|
||||
|
||||
function getOnlyPagePath(paths: string[]): string {
|
||||
expect(paths).toHaveLength(1);
|
||||
const [pagePath] = paths;
|
||||
if (!pagePath) {
|
||||
throw new Error("Expected OKF import to produce one page path.");
|
||||
}
|
||||
return pagePath;
|
||||
}
|
||||
|
||||
async function writeOkfBundle(rootDir: string) {
|
||||
const bundlePath = path.join(rootDir, "sales-okf");
|
||||
await fs.mkdir(path.join(bundlePath, "tables"), { recursive: true });
|
||||
await fs.mkdir(path.join(bundlePath, "metrics"), { recursive: true });
|
||||
await fs.writeFile(
|
||||
path.join(bundlePath, "index.md"),
|
||||
`---
|
||||
id: sales-okf
|
||||
okf_version: "0.1"
|
||||
---
|
||||
|
||||
# Sales Bundle
|
||||
`,
|
||||
"utf8",
|
||||
);
|
||||
await fs.writeFile(path.join(bundlePath, "log.md"), "# Directory Update Log\n", "utf8");
|
||||
await fs.writeFile(
|
||||
path.join(bundlePath, "tables", "customers.md"),
|
||||
`---
|
||||
type: BigQuery Table
|
||||
title: Customers
|
||||
description: Customer table.
|
||||
resource: https://console.cloud.google.com/bigquery?p=acme&d=sales&t=customers
|
||||
tags: [sales, customers]
|
||||
timestamp: 2026-05-28T00:00:00Z
|
||||
producer_field:
|
||||
owner: data
|
||||
---
|
||||
|
||||
# Schema
|
||||
|
||||
Customer rows.
|
||||
`,
|
||||
"utf8",
|
||||
);
|
||||
await fs.writeFile(
|
||||
path.join(bundlePath, "tables", "orders.md"),
|
||||
`---
|
||||
type: BigQuery Table
|
||||
title: Orders
|
||||
description: One row per completed order.
|
||||
tags:
|
||||
- sales
|
||||
- orders
|
||||
---
|
||||
|
||||
# Schema
|
||||
|
||||
Joined with [Customers](/tables/customers.md) and the [weekly metric](../metrics/weekly-active-users.md).
|
||||
Titled link to [weekly metric](../metrics/weekly-active-users.md "metric docs").
|
||||
|
||||
Inline code keeps \`[customers](/tables/customers.md)\` unchanged.
|
||||
|
||||
\`\`\`markdown
|
||||
[customers](/tables/customers.md)
|
||||
\`\`\`
|
||||
|
||||
External citation stays as [BigQuery](https://cloud.google.com/bigquery).
|
||||
`,
|
||||
"utf8",
|
||||
);
|
||||
await fs.writeFile(
|
||||
path.join(bundlePath, "metrics", "weekly-active-users.md"),
|
||||
`---
|
||||
type: Metric
|
||||
title: Weekly Active Users
|
||||
---
|
||||
|
||||
Computed from [orders](../tables/orders.md).
|
||||
`,
|
||||
"utf8",
|
||||
);
|
||||
await fs.writeFile(
|
||||
path.join(bundlePath, "tables", "draft.md"),
|
||||
`---
|
||||
title: Draft
|
||||
---
|
||||
|
||||
Missing type.
|
||||
`,
|
||||
"utf8",
|
||||
);
|
||||
return bundlePath;
|
||||
}
|
||||
|
||||
describe("importMemoryWikiOkfBundle", () => {
|
||||
it("imports OKF concept documents as searchable wiki concept pages", async () => {
|
||||
const rootDir = await createTempDir("memory-wiki-okf-");
|
||||
const bundlePath = await writeOkfBundle(rootDir);
|
||||
const { config } = await createVault({
|
||||
rootDir: path.join(rootDir, "vault"),
|
||||
});
|
||||
|
||||
const result = await importMemoryWikiOkfBundle({
|
||||
config,
|
||||
bundlePath,
|
||||
nowMs: Date.UTC(2026, 5, 12, 10, 0, 0),
|
||||
});
|
||||
|
||||
expect(result.okfVersion).toBe("0.1");
|
||||
expect(result.importedCount).toBe(3);
|
||||
expect(result.skippedCount).toBe(1);
|
||||
expect(result.warnings[0]).toMatchObject({
|
||||
code: "missing-type",
|
||||
path: "tables/draft.md",
|
||||
});
|
||||
expect(result.pagePaths).toHaveLength(3);
|
||||
const repeat = await importMemoryWikiOkfBundle({
|
||||
config,
|
||||
bundlePath,
|
||||
nowMs: Date.UTC(2026, 5, 12, 10, 5, 0),
|
||||
});
|
||||
expect(repeat.importedCount).toBe(3);
|
||||
expect(repeat.updatedCount).toBe(0);
|
||||
|
||||
const ordersPath = result.pagePaths.find((pagePath) => pagePath.includes("orders"));
|
||||
expect(ordersPath).toBeTruthy();
|
||||
const ordersRaw = await fs.readFile(path.join(config.vault.path, ordersPath!), "utf8");
|
||||
const orders = parseWikiMarkdown(ordersRaw);
|
||||
expect(orders.frontmatter).toMatchObject({
|
||||
pageType: "concept",
|
||||
title: "Orders",
|
||||
sourceType: "okf",
|
||||
provenanceMode: "okf-import",
|
||||
okfConceptId: "tables/orders",
|
||||
okfType: "BigQuery Table",
|
||||
});
|
||||
expect(orders.frontmatter.sourceIds).toEqual([
|
||||
expect.stringMatching(/^source\.okf\.sales-okf$/),
|
||||
]);
|
||||
expect(orders.body).toMatch(/\]\(okf-sales-okf-tables-customers-/);
|
||||
expect(orders.body).toMatch(/\]\(okf-sales-okf-metrics-weekly-active-users-/);
|
||||
expect(orders.body).toContain('"metric docs"');
|
||||
expect(orders.body).toContain("`[customers](/tables/customers.md)`");
|
||||
expect(orders.body).toContain("```markdown\n[customers](/tables/customers.md)\n```");
|
||||
expect(orders.body).toContain("https://cloud.google.com/bigquery");
|
||||
|
||||
const okf = orders.frontmatter.okf as Record<string, unknown>;
|
||||
expect(okf).toMatchObject({
|
||||
version: "0.1",
|
||||
bundleName: "sales-okf",
|
||||
conceptId: "tables/orders",
|
||||
sourceRelativePath: "tables/orders.md",
|
||||
});
|
||||
expect(orders.frontmatter.relationships).toEqual(
|
||||
expect.arrayContaining([
|
||||
expect.objectContaining({
|
||||
targetPath: expect.stringMatching(/^concepts\/okf-sales-okf-tables-customers-/),
|
||||
kind: "okf-link",
|
||||
}),
|
||||
expect.objectContaining({
|
||||
targetPath: expect.stringMatching(
|
||||
/^concepts\/okf-sales-okf-metrics-weekly-active-users-/,
|
||||
),
|
||||
kind: "okf-link",
|
||||
}),
|
||||
]),
|
||||
);
|
||||
|
||||
const customersPath = result.pagePaths.find((pagePath) => pagePath.includes("customers"));
|
||||
const customersRaw = await fs.readFile(path.join(config.vault.path, customersPath!), "utf8");
|
||||
const customers = parseWikiMarkdown(customersRaw);
|
||||
const customersOkf = customers.frontmatter.okf as Record<string, unknown>;
|
||||
expect(customersOkf.frontmatter).toMatchObject({
|
||||
producer_field: { owner: "data" },
|
||||
});
|
||||
|
||||
const searchResults = await searchMemoryWiki({
|
||||
config,
|
||||
query: "completed order",
|
||||
searchCorpus: "wiki",
|
||||
});
|
||||
expect(searchResults.map((searchResult) => searchResult.path)).toContain(ordersPath);
|
||||
});
|
||||
|
||||
it("caps generated concept filenames for long OKF concept paths", async () => {
|
||||
const rootDir = await createTempDir("memory-wiki-okf-long-");
|
||||
const bundlePath = path.join(rootDir, "long-okf");
|
||||
const deepSegments = Array.from({ length: 40 }, (_, index) => `segment-${index}`);
|
||||
const deepDir = path.join(bundlePath, ...deepSegments);
|
||||
await fs.mkdir(deepDir, { recursive: true });
|
||||
await fs.writeFile(
|
||||
path.join(deepDir, "orders.md"),
|
||||
`---
|
||||
type: BigQuery Table
|
||||
title: Long Orders
|
||||
---
|
||||
|
||||
Long concept body.
|
||||
`,
|
||||
"utf8",
|
||||
);
|
||||
const { config } = await createVault({
|
||||
rootDir: path.join(rootDir, "vault"),
|
||||
});
|
||||
|
||||
const result = await importMemoryWikiOkfBundle({
|
||||
config,
|
||||
bundlePath,
|
||||
nowMs: Date.UTC(2026, 5, 12, 10, 0, 0),
|
||||
});
|
||||
|
||||
expect(result.importedCount).toBe(1);
|
||||
const [pagePath] = result.pagePaths;
|
||||
expect(pagePath).toBeDefined();
|
||||
if (!pagePath) {
|
||||
throw new Error("Expected OKF import to produce a page path.");
|
||||
}
|
||||
const fileName = path.basename(pagePath);
|
||||
expect(Buffer.byteLength(fileName)).toBeLessThanOrEqual(255);
|
||||
await expect(fs.readFile(path.join(config.vault.path, pagePath), "utf8")).resolves.toContain(
|
||||
"Long concept body.",
|
||||
);
|
||||
});
|
||||
|
||||
it("namespaces concept pages by bundle so repeated OKF paths do not overwrite", async () => {
|
||||
const rootDir = await createTempDir("memory-wiki-okf-bundles-");
|
||||
const firstBundle = path.join(rootDir, "first-bundle");
|
||||
const secondBundle = path.join(rootDir, "second-bundle");
|
||||
for (const [bundlePath, title] of [
|
||||
[firstBundle, "First Customers"],
|
||||
[secondBundle, "Second Customers"],
|
||||
] as const) {
|
||||
await fs.mkdir(path.join(bundlePath, "tables"), { recursive: true });
|
||||
await fs.writeFile(
|
||||
path.join(bundlePath, "tables", "customers.md"),
|
||||
`---
|
||||
type: BigQuery Table
|
||||
title: ${title}
|
||||
---
|
||||
|
||||
${title} body.
|
||||
`,
|
||||
"utf8",
|
||||
);
|
||||
}
|
||||
const { config } = await createVault({
|
||||
rootDir: path.join(rootDir, "vault"),
|
||||
});
|
||||
|
||||
const first = await importMemoryWikiOkfBundle({
|
||||
config,
|
||||
bundlePath: firstBundle,
|
||||
nowMs: Date.UTC(2026, 5, 12, 10, 0, 0),
|
||||
});
|
||||
const second = await importMemoryWikiOkfBundle({
|
||||
config,
|
||||
bundlePath: secondBundle,
|
||||
nowMs: Date.UTC(2026, 5, 12, 10, 0, 0),
|
||||
});
|
||||
|
||||
const firstPath = getOnlyPagePath(first.pagePaths);
|
||||
const secondPath = getOnlyPagePath(second.pagePaths);
|
||||
expect(firstPath).not.toBe(secondPath);
|
||||
await expect(fs.readFile(path.join(config.vault.path, firstPath), "utf8")).resolves.toContain(
|
||||
"First Customers body.",
|
||||
);
|
||||
await expect(fs.readFile(path.join(config.vault.path, secondPath), "utf8")).resolves.toContain(
|
||||
"Second Customers body.",
|
||||
);
|
||||
});
|
||||
|
||||
it("removes stale concept pages when an OKF bundle drops a concept", async () => {
|
||||
const rootDir = await createTempDir("memory-wiki-okf-remove-");
|
||||
const bundlePath = path.join(rootDir, "removing-okf");
|
||||
await fs.mkdir(path.join(bundlePath, "tables"), { recursive: true });
|
||||
const customersPath = path.join(bundlePath, "tables", "customers.md");
|
||||
const ordersPath = path.join(bundlePath, "tables", "orders.md");
|
||||
await fs.writeFile(
|
||||
customersPath,
|
||||
`---
|
||||
type: BigQuery Table
|
||||
title: Customers
|
||||
---
|
||||
|
||||
Customer body.
|
||||
`,
|
||||
"utf8",
|
||||
);
|
||||
await fs.writeFile(
|
||||
ordersPath,
|
||||
`---
|
||||
type: BigQuery Table
|
||||
title: Orders
|
||||
---
|
||||
|
||||
Order body.
|
||||
`,
|
||||
"utf8",
|
||||
);
|
||||
const { config } = await createVault({
|
||||
rootDir: path.join(rootDir, "vault"),
|
||||
});
|
||||
const first = await importMemoryWikiOkfBundle({
|
||||
config,
|
||||
bundlePath,
|
||||
nowMs: Date.UTC(2026, 5, 12, 10, 0, 0),
|
||||
});
|
||||
const stalePagePath = first.pagePaths.find((pagePath) => pagePath.includes("orders"));
|
||||
expect(stalePagePath).toBeDefined();
|
||||
if (!stalePagePath) {
|
||||
throw new Error("Expected initial OKF import to include orders.");
|
||||
}
|
||||
|
||||
await fs.rm(ordersPath);
|
||||
const second = await importMemoryWikiOkfBundle({
|
||||
config,
|
||||
bundlePath,
|
||||
nowMs: Date.UTC(2026, 5, 12, 10, 0, 0),
|
||||
});
|
||||
|
||||
expect(second.importedCount).toBe(1);
|
||||
expect(second.removedCount).toBe(1);
|
||||
expect(second.removedPagePaths).toEqual([stalePagePath]);
|
||||
await expect(fs.stat(path.join(config.vault.path, stalePagePath))).rejects.toThrow();
|
||||
const results = await searchMemoryWiki({
|
||||
config,
|
||||
query: "Order body",
|
||||
searchCorpus: "wiki",
|
||||
});
|
||||
expect(results).toHaveLength(0);
|
||||
});
|
||||
|
||||
it("does not prune existing pages when current OKF scan has invalid concepts", async () => {
|
||||
const rootDir = await createTempDir("memory-wiki-okf-invalid-");
|
||||
const bundlePath = path.join(rootDir, "invalid-okf");
|
||||
await fs.mkdir(path.join(bundlePath, "tables"), { recursive: true });
|
||||
const customersPath = path.join(bundlePath, "tables", "customers.md");
|
||||
await fs.writeFile(
|
||||
customersPath,
|
||||
`---
|
||||
type: BigQuery Table
|
||||
title: Customers
|
||||
---
|
||||
|
||||
Customer body.
|
||||
`,
|
||||
"utf8",
|
||||
);
|
||||
const { config } = await createVault({
|
||||
rootDir: path.join(rootDir, "vault"),
|
||||
});
|
||||
const first = await importMemoryWikiOkfBundle({
|
||||
config,
|
||||
bundlePath,
|
||||
nowMs: Date.UTC(2026, 5, 12, 10, 0, 0),
|
||||
});
|
||||
const pagePath = getOnlyPagePath(first.pagePaths);
|
||||
await fs.writeFile(
|
||||
customersPath,
|
||||
`---
|
||||
title: Customers
|
||||
---
|
||||
|
||||
Temporarily invalid body.
|
||||
`,
|
||||
"utf8",
|
||||
);
|
||||
|
||||
const second = await importMemoryWikiOkfBundle({
|
||||
config,
|
||||
bundlePath,
|
||||
nowMs: Date.UTC(2026, 5, 12, 10, 0, 0),
|
||||
});
|
||||
|
||||
expect(second.importedCount).toBe(0);
|
||||
expect(second.skippedCount).toBe(1);
|
||||
expect(second.removedCount).toBe(0);
|
||||
await expect(fs.readFile(path.join(config.vault.path, pagePath), "utf8")).resolves.toContain(
|
||||
"Customer body.",
|
||||
);
|
||||
});
|
||||
|
||||
it("detects body-only changes on timestamp-shaped markdown lines", async () => {
|
||||
const rootDir = await createTempDir("memory-wiki-okf-body-timestamp-");
|
||||
const bundlePath = path.join(rootDir, "body-timestamp-okf");
|
||||
await fs.mkdir(path.join(bundlePath, "tables"), { recursive: true });
|
||||
const conceptPath = path.join(bundlePath, "tables", "events.md");
|
||||
await fs.writeFile(
|
||||
conceptPath,
|
||||
`---
|
||||
type: BigQuery Table
|
||||
title: Events
|
||||
---
|
||||
|
||||
updatedAt: 2026-06-12
|
||||
`,
|
||||
"utf8",
|
||||
);
|
||||
const { config } = await createVault({
|
||||
rootDir: path.join(rootDir, "vault"),
|
||||
});
|
||||
const first = await importMemoryWikiOkfBundle({
|
||||
config,
|
||||
bundlePath,
|
||||
nowMs: Date.UTC(2026, 5, 12, 10, 0, 0),
|
||||
});
|
||||
const pagePath = getOnlyPagePath(first.pagePaths);
|
||||
await fs.writeFile(
|
||||
conceptPath,
|
||||
`---
|
||||
type: BigQuery Table
|
||||
title: Events
|
||||
---
|
||||
|
||||
updatedAt: 2026-06-13
|
||||
`,
|
||||
"utf8",
|
||||
);
|
||||
|
||||
const second = await importMemoryWikiOkfBundle({
|
||||
config,
|
||||
bundlePath,
|
||||
nowMs: Date.UTC(2026, 5, 13, 10, 0, 0),
|
||||
});
|
||||
|
||||
expect(second.updatedCount).toBe(1);
|
||||
await expect(fs.readFile(path.join(config.vault.path, pagePath), "utf8")).resolves.toContain(
|
||||
"updatedAt: 2026-06-13",
|
||||
);
|
||||
});
|
||||
|
||||
it("rewrites percent-encoded OKF markdown links and preserves suffixes", async () => {
|
||||
const rootDir = await createTempDir("memory-wiki-okf-encoded-link-");
|
||||
const bundlePath = path.join(rootDir, "encoded-okf");
|
||||
await fs.mkdir(bundlePath, { recursive: true });
|
||||
await fs.writeFile(
|
||||
path.join(bundlePath, "BigQuery Table.md"),
|
||||
`---
|
||||
type: BigQuery Table
|
||||
title: BigQuery Table
|
||||
---
|
||||
|
||||
Table body.
|
||||
`,
|
||||
"utf8",
|
||||
);
|
||||
await fs.writeFile(
|
||||
path.join(bundlePath, "links.md"),
|
||||
`---
|
||||
type: Concept
|
||||
title: Links
|
||||
---
|
||||
|
||||
See [table](BigQuery%20Table.md?view=compact#columns).
|
||||
`,
|
||||
"utf8",
|
||||
);
|
||||
const { config } = await createVault({
|
||||
rootDir: path.join(rootDir, "vault"),
|
||||
});
|
||||
|
||||
const result = await importMemoryWikiOkfBundle({
|
||||
config,
|
||||
bundlePath,
|
||||
nowMs: Date.UTC(2026, 5, 12, 10, 0, 0),
|
||||
});
|
||||
|
||||
const linksPath = result.pagePaths.find((pagePath) => pagePath.includes("links"));
|
||||
expect(linksPath).toBeDefined();
|
||||
if (!linksPath) {
|
||||
throw new Error("Expected links page to be imported.");
|
||||
}
|
||||
await expect(fs.readFile(path.join(config.vault.path, linksPath), "utf8")).resolves.toMatch(
|
||||
/\[table\]\(okf-encoded-okf-[0-9a-f]{8}-bigquery-table-[^)]+\.md\?view=compact#columns\)/,
|
||||
);
|
||||
});
|
||||
|
||||
it("imports OKF concept frontmatter with CRLF line endings", async () => {
|
||||
const rootDir = await createTempDir("memory-wiki-okf-crlf-");
|
||||
const bundlePath = path.join(rootDir, "crlf-okf");
|
||||
await fs.mkdir(path.join(bundlePath, "tables"), { recursive: true });
|
||||
await fs.writeFile(
|
||||
path.join(bundlePath, "tables", "events.md"),
|
||||
[
|
||||
"---",
|
||||
"type: BigQuery Table",
|
||||
"title: Events",
|
||||
"---",
|
||||
"",
|
||||
"Windows-flavored frontmatter.",
|
||||
"",
|
||||
].join("\r\n"),
|
||||
"utf8",
|
||||
);
|
||||
const { config } = await createVault({
|
||||
rootDir: path.join(rootDir, "vault"),
|
||||
});
|
||||
|
||||
const result = await importMemoryWikiOkfBundle({
|
||||
config,
|
||||
bundlePath,
|
||||
nowMs: Date.UTC(2026, 5, 12, 10, 0, 0),
|
||||
});
|
||||
|
||||
expect(result.importedCount).toBe(1);
|
||||
expect(result.skippedCount).toBe(0);
|
||||
await expect(
|
||||
fs.readFile(path.join(config.vault.path, getOnlyPagePath(result.pagePaths)), "utf8"),
|
||||
).resolves.toContain("Windows-flavored frontmatter.");
|
||||
});
|
||||
|
||||
it("refuses to write imported OKF concept pages through symlinks", async () => {
|
||||
const rootDir = await createTempDir("memory-wiki-okf-symlink-");
|
||||
const bundlePath = path.join(rootDir, "safe-okf");
|
||||
await fs.mkdir(path.join(bundlePath, "tables"), { recursive: true });
|
||||
const conceptPath = path.join(bundlePath, "tables", "customers.md");
|
||||
await fs.writeFile(
|
||||
conceptPath,
|
||||
`---
|
||||
type: BigQuery Table
|
||||
title: Customers
|
||||
---
|
||||
|
||||
Original body.
|
||||
`,
|
||||
"utf8",
|
||||
);
|
||||
const { config } = await createVault({
|
||||
rootDir: path.join(rootDir, "vault"),
|
||||
});
|
||||
const first = await importMemoryWikiOkfBundle({
|
||||
config,
|
||||
bundlePath,
|
||||
nowMs: Date.UTC(2026, 5, 12, 10, 0, 0),
|
||||
});
|
||||
const pagePath = getOnlyPagePath(first.pagePaths);
|
||||
const pageAbsolutePath = path.join(config.vault.path, pagePath);
|
||||
const externalTarget = path.join(rootDir, "outside.md");
|
||||
await fs.writeFile(externalTarget, "external target\n", "utf8");
|
||||
await fs.rm(pageAbsolutePath);
|
||||
await fs.symlink(externalTarget, pageAbsolutePath);
|
||||
await fs.writeFile(
|
||||
conceptPath,
|
||||
`---
|
||||
type: BigQuery Table
|
||||
title: Customers
|
||||
---
|
||||
|
||||
Updated body.
|
||||
`,
|
||||
"utf8",
|
||||
);
|
||||
|
||||
await expect(
|
||||
importMemoryWikiOkfBundle({
|
||||
config,
|
||||
bundlePath,
|
||||
nowMs: Date.UTC(2026, 5, 12, 11, 0, 0),
|
||||
}),
|
||||
).rejects.toThrow("through symlink");
|
||||
await expect(fs.readFile(externalTarget, "utf8")).resolves.toBe("external target\n");
|
||||
});
|
||||
|
||||
it("refuses to import OKF concept files through hardlinks", async () => {
|
||||
const rootDir = await createTempDir("memory-wiki-okf-hardlink-");
|
||||
const bundlePath = path.join(rootDir, "hardlink-okf");
|
||||
await fs.mkdir(path.join(bundlePath, "tables"), { recursive: true });
|
||||
const externalSource = path.join(rootDir, "outside.md");
|
||||
await fs.writeFile(
|
||||
externalSource,
|
||||
`---
|
||||
type: BigQuery Table
|
||||
title: Private
|
||||
---
|
||||
|
||||
private body
|
||||
`,
|
||||
"utf8",
|
||||
);
|
||||
await fs.link(externalSource, path.join(bundlePath, "tables", "private.md"));
|
||||
const { config } = await createVault({
|
||||
rootDir: path.join(rootDir, "vault"),
|
||||
});
|
||||
|
||||
const result = await importMemoryWikiOkfBundle({
|
||||
config,
|
||||
bundlePath,
|
||||
nowMs: Date.UTC(2026, 5, 12, 10, 0, 0),
|
||||
});
|
||||
|
||||
expect(result.importedCount).toBe(0);
|
||||
expect(result.skippedCount).toBe(1);
|
||||
expect(result.warnings[0]).toMatchObject({
|
||||
code: "unreadable-entry",
|
||||
path: "tables/private.md",
|
||||
});
|
||||
});
|
||||
});
|
||||
746
extensions/memory-wiki/src/okf.ts
Normal file
746
extensions/memory-wiki/src/okf.ts
Normal file
@@ -0,0 +1,746 @@
|
||||
// Memory Wiki plugin module implements Open Knowledge Format import behavior.
|
||||
import { createHash } from "node:crypto";
|
||||
import fs from "node:fs/promises";
|
||||
import path from "node:path";
|
||||
import { FsSafeError, root as fsRoot } from "openclaw/plugin-sdk/security-runtime";
|
||||
import {
|
||||
normalizeOptionalString,
|
||||
normalizeSingleOrTrimmedStringList,
|
||||
uniqueStrings,
|
||||
} from "openclaw/plugin-sdk/string-coerce-runtime";
|
||||
import { compileMemoryWikiVault } from "./compile.js";
|
||||
import type { ResolvedMemoryWikiConfig } from "./config.js";
|
||||
import { appendMemoryWikiLog } from "./log.js";
|
||||
import {
|
||||
createWikiPageFilename,
|
||||
parseWikiMarkdown,
|
||||
renderWikiMarkdown,
|
||||
slugifyWikiSegment,
|
||||
WIKI_RELATED_END_MARKER,
|
||||
WIKI_RELATED_START_MARKER,
|
||||
} from "./markdown.js";
|
||||
import { resolveMemoryWikiTimestamp } from "./time.js";
|
||||
import { initializeMemoryWikiVault } from "./vault.js";
|
||||
|
||||
const OKF_RESERVED_FILENAMES = new Set(["index.md", "log.md"]);
|
||||
const OKF_MARKDOWN_LINK_PATTERN = /(!?)\[([^\]]*)\]\(([^)]+)\)/g;
|
||||
const OKF_FENCE_PATTERN = /^ {0,3}(`{3,}|~{3,})/;
|
||||
const OKF_RELATED_SECTION_PATTERN = new RegExp(
|
||||
`\\n+## Related\\n${WIKI_RELATED_START_MARKER}[\\s\\S]*?${WIKI_RELATED_END_MARKER}\\n?`,
|
||||
"g",
|
||||
);
|
||||
const OKF_VOLATILE_TIMESTAMP_LINE_PATTERN = /^(?:importedAt|updatedAt): .*\n/gm;
|
||||
const OKF_HASH_CHARS = 8;
|
||||
|
||||
type FileStatLike = {
|
||||
isFile?: unknown;
|
||||
nlink?: unknown;
|
||||
};
|
||||
|
||||
type OkfConceptDocument = {
|
||||
conceptId: string;
|
||||
relativePath: string;
|
||||
absolutePath: string;
|
||||
frontmatter: Record<string, unknown>;
|
||||
body: string;
|
||||
type: string;
|
||||
title: string;
|
||||
description?: string;
|
||||
resource?: string;
|
||||
tags: string[];
|
||||
timestamp?: string;
|
||||
};
|
||||
|
||||
type OkfImportedPage = {
|
||||
conceptId: string;
|
||||
sourcePath: string;
|
||||
pageId: string;
|
||||
pagePath: string;
|
||||
title: string;
|
||||
created: boolean;
|
||||
};
|
||||
|
||||
export type ImportMemoryWikiOkfWarning = {
|
||||
code: "invalid-concept" | "missing-type" | "unreadable-entry";
|
||||
path: string;
|
||||
message: string;
|
||||
};
|
||||
|
||||
export type ImportMemoryWikiOkfResult = {
|
||||
bundlePath: string;
|
||||
bundleName: string;
|
||||
okfVersion?: string;
|
||||
importedCount: number;
|
||||
updatedCount: number;
|
||||
removedCount: number;
|
||||
skippedCount: number;
|
||||
pagePaths: string[];
|
||||
removedPagePaths: string[];
|
||||
warnings: ImportMemoryWikiOkfWarning[];
|
||||
indexUpdatedFiles: string[];
|
||||
};
|
||||
|
||||
function toPosixPath(value: string): string {
|
||||
return value.split(path.sep).join("/");
|
||||
}
|
||||
|
||||
function trimMarkdownExtension(value: string): string {
|
||||
return value.replace(/\.md$/i, "");
|
||||
}
|
||||
|
||||
function isRegularFileStat(value: unknown): value is FileStatLike & { nlink: number } {
|
||||
if (!value || typeof value !== "object") {
|
||||
return false;
|
||||
}
|
||||
const stat = value as FileStatLike;
|
||||
const isFile =
|
||||
typeof stat.isFile === "function"
|
||||
? (stat.isFile as () => boolean).call(stat)
|
||||
: stat.isFile === true;
|
||||
return isFile && typeof stat.nlink === "number";
|
||||
}
|
||||
|
||||
type OkfBundleMetadata = {
|
||||
key: string;
|
||||
version?: string;
|
||||
};
|
||||
|
||||
function createOkfBundleKey(params: {
|
||||
rootFrontmatter: Record<string, unknown>;
|
||||
bundleName: string;
|
||||
bundlePath: string;
|
||||
}): string {
|
||||
const producerId =
|
||||
normalizeOptionalString(params.rootFrontmatter.id) ??
|
||||
normalizeOptionalString(params.rootFrontmatter.okf_id);
|
||||
if (producerId) {
|
||||
return slugifyWikiSegment(producerId);
|
||||
}
|
||||
const label =
|
||||
normalizeOptionalString(params.rootFrontmatter.name) ??
|
||||
normalizeOptionalString(params.rootFrontmatter.title) ??
|
||||
params.bundleName;
|
||||
const hash = createHash("sha1").update(params.bundlePath).digest("hex").slice(0, OKF_HASH_CHARS);
|
||||
return `${slugifyWikiSegment(label)}-${hash}`;
|
||||
}
|
||||
|
||||
function createOkfPageStem(bundleKey: string, conceptId: string): string {
|
||||
const slug = slugifyWikiSegment(conceptId.replace(/\//g, "-"));
|
||||
const hash = createHash("sha1").update(conceptId).digest("hex").slice(0, OKF_HASH_CHARS);
|
||||
return `okf-${bundleKey}-${slug}-${hash}`;
|
||||
}
|
||||
|
||||
function createOkfPageIdentity(
|
||||
bundleKey: string,
|
||||
conceptId: string,
|
||||
): { pageId: string; pagePath: string } {
|
||||
const fileName = createWikiPageFilename(createOkfPageStem(bundleKey, conceptId));
|
||||
const stem = trimMarkdownExtension(fileName);
|
||||
return {
|
||||
pageId: `concept.${stem}`,
|
||||
pagePath: `concepts/${fileName}`,
|
||||
};
|
||||
}
|
||||
|
||||
async function collectOkfMarkdownFiles(
|
||||
rootDir: string,
|
||||
warnings: ImportMemoryWikiOkfWarning[],
|
||||
): Promise<string[]> {
|
||||
async function walk(relativeDir: string): Promise<string[]> {
|
||||
const absoluteDir = path.join(rootDir, relativeDir);
|
||||
const entries = await fs.readdir(absoluteDir, { withFileTypes: true }).catch((err: unknown) => {
|
||||
warnings.push({
|
||||
code: "unreadable-entry",
|
||||
path: toPosixPath(relativeDir) || ".",
|
||||
message: err instanceof Error ? err.message : "Unable to read OKF directory.",
|
||||
});
|
||||
return [];
|
||||
});
|
||||
const files: string[] = [];
|
||||
for (const entry of entries.toSorted((left, right) => left.name.localeCompare(right.name))) {
|
||||
if (entry.name === ".git" || entry.name === "node_modules") {
|
||||
continue;
|
||||
}
|
||||
const relativePath = path.join(relativeDir, entry.name);
|
||||
if (entry.isDirectory()) {
|
||||
files.push(...(await walk(relativePath)));
|
||||
continue;
|
||||
}
|
||||
if (entry.isFile() && entry.name.endsWith(".md")) {
|
||||
files.push(relativePath);
|
||||
}
|
||||
}
|
||||
return files;
|
||||
}
|
||||
|
||||
return (await walk("")).map(toPosixPath).toSorted((left, right) => left.localeCompare(right));
|
||||
}
|
||||
|
||||
function parseOkfMarkdown(
|
||||
content: string,
|
||||
relativePath: string,
|
||||
): {
|
||||
frontmatter: Record<string, unknown>;
|
||||
body: string;
|
||||
warning?: ImportMemoryWikiOkfWarning;
|
||||
} {
|
||||
const normalizedContent = content.replace(/\r\n/g, "\n");
|
||||
try {
|
||||
return parseWikiMarkdown(normalizedContent);
|
||||
} catch (err) {
|
||||
return {
|
||||
frontmatter: {},
|
||||
body: normalizedContent,
|
||||
warning: {
|
||||
code: "invalid-concept",
|
||||
path: relativePath,
|
||||
message: err instanceof Error ? err.message : "Unable to parse OKF frontmatter.",
|
||||
},
|
||||
};
|
||||
}
|
||||
}
|
||||
|
||||
async function readOkfTextFile(params: {
|
||||
bundlePath: string;
|
||||
relativePath: string;
|
||||
warnings: ImportMemoryWikiOkfWarning[];
|
||||
}): Promise<string | null> {
|
||||
const root = await fsRoot(params.bundlePath);
|
||||
const stat = await root.stat(params.relativePath).catch((err: unknown) => {
|
||||
params.warnings.push({
|
||||
code: "unreadable-entry",
|
||||
path: params.relativePath,
|
||||
message: err instanceof Error ? err.message : "Unable to read OKF concept.",
|
||||
});
|
||||
return null;
|
||||
});
|
||||
if (!stat) {
|
||||
return null;
|
||||
}
|
||||
if (!isRegularFileStat(stat)) {
|
||||
params.warnings.push({
|
||||
code: "unreadable-entry",
|
||||
path: params.relativePath,
|
||||
message: "Refusing to import OKF concept through non-regular or hardlinked file.",
|
||||
});
|
||||
return null;
|
||||
}
|
||||
return await root.readText(params.relativePath).catch((err: unknown) => {
|
||||
params.warnings.push({
|
||||
code: "unreadable-entry",
|
||||
path: params.relativePath,
|
||||
message: err instanceof Error ? err.message : "Unable to read OKF concept.",
|
||||
});
|
||||
return null;
|
||||
});
|
||||
}
|
||||
|
||||
function deriveOkfTitle(relativePath: string, frontmatter: Record<string, unknown>): string {
|
||||
return (
|
||||
normalizeOptionalString(frontmatter.title) ??
|
||||
path.posix.basename(relativePath, ".md").replace(/[-_]+/g, " ").trim() ??
|
||||
trimMarkdownExtension(relativePath)
|
||||
);
|
||||
}
|
||||
|
||||
function normalizeOkfConcept(params: {
|
||||
bundlePath: string;
|
||||
relativePath: string;
|
||||
content: string;
|
||||
}): { concept?: OkfConceptDocument; warning?: ImportMemoryWikiOkfWarning } {
|
||||
const parsed = parseOkfMarkdown(params.content, params.relativePath);
|
||||
if (parsed.warning) {
|
||||
return { warning: parsed.warning };
|
||||
}
|
||||
|
||||
const type = normalizeOptionalString(parsed.frontmatter.type);
|
||||
if (!type) {
|
||||
return {
|
||||
warning: {
|
||||
code: "missing-type",
|
||||
path: params.relativePath,
|
||||
message: "OKF concept is missing required non-empty type frontmatter.",
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
const conceptId = trimMarkdownExtension(params.relativePath);
|
||||
const timestamp = normalizeOptionalString(parsed.frontmatter.timestamp);
|
||||
return {
|
||||
concept: {
|
||||
conceptId,
|
||||
relativePath: params.relativePath,
|
||||
absolutePath: path.join(params.bundlePath, params.relativePath),
|
||||
frontmatter: parsed.frontmatter,
|
||||
body: parsed.body,
|
||||
type,
|
||||
title: deriveOkfTitle(params.relativePath, parsed.frontmatter),
|
||||
...(normalizeOptionalString(parsed.frontmatter.description)
|
||||
? { description: normalizeOptionalString(parsed.frontmatter.description) }
|
||||
: {}),
|
||||
...(normalizeOptionalString(parsed.frontmatter.resource)
|
||||
? { resource: normalizeOptionalString(parsed.frontmatter.resource) }
|
||||
: {}),
|
||||
tags: normalizeSingleOrTrimmedStringList(parsed.frontmatter.tags),
|
||||
...(timestamp ? { timestamp } : {}),
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
function splitMarkdownLinkDestination(target: string): {
|
||||
destination: string;
|
||||
titleSuffix: string;
|
||||
} {
|
||||
const trimmed = target.trim();
|
||||
if (trimmed.startsWith("<")) {
|
||||
const end = trimmed.indexOf(">");
|
||||
if (end > 0) {
|
||||
return {
|
||||
destination: trimmed.slice(1, end),
|
||||
titleSuffix: trimmed.slice(end + 1),
|
||||
};
|
||||
}
|
||||
}
|
||||
const match = trimmed.match(/^(\S+)(\s+[\s\S]+)?$/);
|
||||
return {
|
||||
destination: match?.[1] ?? trimmed,
|
||||
titleSuffix: match?.[2] ?? "",
|
||||
};
|
||||
}
|
||||
|
||||
function resolveOkfMarkdownTarget(sourceRelativePath: string, target: string): string | null {
|
||||
const { destination } = splitMarkdownLinkDestination(target);
|
||||
const trimmed = destination.trim();
|
||||
if (!trimmed || trimmed.startsWith("#") || /^[a-z][a-z0-9+.-]*:/i.test(trimmed)) {
|
||||
return null;
|
||||
}
|
||||
|
||||
const rawTargetWithoutSuffix = trimmed.split("#")[0]?.split("?")[0]?.replace(/\\/g, "/").trim();
|
||||
const targetWithoutSuffix = safeDecodeOkfLinkPath(rawTargetWithoutSuffix);
|
||||
if (!targetWithoutSuffix || !targetWithoutSuffix.endsWith(".md")) {
|
||||
return null;
|
||||
}
|
||||
|
||||
const normalized = targetWithoutSuffix.startsWith("/")
|
||||
? path.posix.normalize(targetWithoutSuffix.slice(1))
|
||||
: path.posix.normalize(
|
||||
path.posix.join(path.posix.dirname(sourceRelativePath), targetWithoutSuffix),
|
||||
);
|
||||
const conceptId = trimMarkdownExtension(normalized);
|
||||
return conceptId.startsWith("../") ? null : conceptId;
|
||||
}
|
||||
|
||||
function safeDecodeOkfLinkPath(value: string | undefined): string {
|
||||
if (!value) {
|
||||
return "";
|
||||
}
|
||||
try {
|
||||
return decodeURIComponent(value);
|
||||
} catch {
|
||||
return value;
|
||||
}
|
||||
}
|
||||
|
||||
function getMarkdownDestinationSuffix(destination: string): string {
|
||||
const queryIndex = destination.indexOf("?");
|
||||
const fragmentIndex = destination.indexOf("#");
|
||||
const suffixIndex = queryIndex === -1
|
||||
? fragmentIndex
|
||||
: fragmentIndex === -1
|
||||
? queryIndex
|
||||
: Math.min(queryIndex, fragmentIndex);
|
||||
return suffixIndex === -1 ? "" : destination.slice(suffixIndex);
|
||||
}
|
||||
|
||||
function rewriteOkfMarkdownLinks(params: {
|
||||
body: string;
|
||||
sourcePagePath: string;
|
||||
sourceRelativePath: string;
|
||||
pageByConceptId: Map<string, { pageId: string; pagePath: string; title: string }>;
|
||||
}): { body: string; linkedConceptIds: string[] } {
|
||||
const linkedConceptIds: string[] = [];
|
||||
const rewriteLinks = (markdown: string) =>
|
||||
markdown.replace(
|
||||
OKF_MARKDOWN_LINK_PATTERN,
|
||||
(match, imagePrefix: string, label: string, rawTarget: string) => {
|
||||
const conceptId = resolveOkfMarkdownTarget(params.sourceRelativePath, rawTarget);
|
||||
if (!conceptId) {
|
||||
return match;
|
||||
}
|
||||
const target = params.pageByConceptId.get(conceptId);
|
||||
if (!target) {
|
||||
return match;
|
||||
}
|
||||
linkedConceptIds.push(conceptId);
|
||||
const { destination, titleSuffix } = splitMarkdownLinkDestination(rawTarget);
|
||||
const relativeTarget = path.posix.relative(
|
||||
path.posix.dirname(params.sourcePagePath),
|
||||
target.pagePath,
|
||||
);
|
||||
const suffix = getMarkdownDestinationSuffix(destination);
|
||||
return `${imagePrefix}[${label}](${relativeTarget}${suffix}${titleSuffix})`;
|
||||
},
|
||||
);
|
||||
const body = rewriteMarkdownOutsideCode(params.body, rewriteLinks);
|
||||
return { body, linkedConceptIds: uniqueStrings(linkedConceptIds) };
|
||||
}
|
||||
|
||||
function rewriteMarkdownLineOutsideInlineCode(
|
||||
line: string,
|
||||
rewriteLinks: (markdown: string) => string,
|
||||
): string {
|
||||
let result = "";
|
||||
let cursor = 0;
|
||||
while (cursor < line.length) {
|
||||
const codeStart = line.indexOf("`", cursor);
|
||||
if (codeStart === -1) {
|
||||
result += rewriteLinks(line.slice(cursor));
|
||||
break;
|
||||
}
|
||||
result += rewriteLinks(line.slice(cursor, codeStart));
|
||||
const delimiter = line.slice(codeStart).match(/^`+/)?.[0] ?? "`";
|
||||
const codeEnd = line.indexOf(delimiter, codeStart + delimiter.length);
|
||||
if (codeEnd === -1) {
|
||||
result += line.slice(codeStart);
|
||||
break;
|
||||
}
|
||||
result += line.slice(codeStart, codeEnd + delimiter.length);
|
||||
cursor = codeEnd + delimiter.length;
|
||||
}
|
||||
return result;
|
||||
}
|
||||
|
||||
function rewriteMarkdownOutsideCode(
|
||||
markdown: string,
|
||||
rewriteLinks: (markdown: string) => string,
|
||||
): string {
|
||||
const lines = markdown.split(/(\n)/);
|
||||
let inFence = false;
|
||||
let fenceDelimiter = "";
|
||||
return lines
|
||||
.map((line) => {
|
||||
if (line === "\n") {
|
||||
return line;
|
||||
}
|
||||
const fenceMatch = line.match(OKF_FENCE_PATTERN);
|
||||
if (fenceMatch) {
|
||||
const delimiter = fenceMatch[1] ?? "";
|
||||
const closesFence =
|
||||
inFence &&
|
||||
delimiter.startsWith(fenceDelimiter[0] ?? "") &&
|
||||
delimiter.length >= fenceDelimiter.length;
|
||||
const opensFence = !inFence;
|
||||
if (opensFence) {
|
||||
inFence = true;
|
||||
fenceDelimiter = delimiter;
|
||||
} else if (closesFence) {
|
||||
inFence = false;
|
||||
fenceDelimiter = "";
|
||||
}
|
||||
return line;
|
||||
}
|
||||
return inFence ? line : rewriteMarkdownLineOutsideInlineCode(line, rewriteLinks);
|
||||
})
|
||||
.join("");
|
||||
}
|
||||
|
||||
function normalizeOkfRenderedPageForComparison(content: string): string {
|
||||
const withoutRelated = content.replace(OKF_RELATED_SECTION_PATTERN, "\n");
|
||||
const frontmatterMatch = withoutRelated.match(/^---\n([\s\S]*?)\n---\n?/);
|
||||
if (!frontmatterMatch) {
|
||||
return withoutRelated.trimEnd();
|
||||
}
|
||||
const normalizedFrontmatter =
|
||||
frontmatterMatch[1]?.replace(OKF_VOLATILE_TIMESTAMP_LINE_PATTERN, "") ?? "";
|
||||
const frontmatterBody = normalizedFrontmatter.endsWith("\n")
|
||||
? normalizedFrontmatter
|
||||
: `${normalizedFrontmatter}\n`;
|
||||
return `---\n${frontmatterBody}---\n${withoutRelated.slice(frontmatterMatch[0].length)}`.trimEnd();
|
||||
}
|
||||
|
||||
async function writeOkfConceptPage(params: {
|
||||
vaultRoot: string;
|
||||
pagePath: string;
|
||||
content: string;
|
||||
}): Promise<{ changed: boolean; created: boolean }> {
|
||||
const vault = await fsRoot(params.vaultRoot);
|
||||
const pageStat = await vault.stat(params.pagePath).catch((error: unknown) => {
|
||||
if (
|
||||
error instanceof FsSafeError &&
|
||||
(error.code === "not-found" || error.code === "path-alias")
|
||||
) {
|
||||
return null;
|
||||
}
|
||||
throw error;
|
||||
});
|
||||
const existing = pageStat ? await vault.readText(params.pagePath).catch(() => "") : "";
|
||||
if (
|
||||
existing === params.content ||
|
||||
normalizeOkfRenderedPageForComparison(existing) ===
|
||||
normalizeOkfRenderedPageForComparison(params.content)
|
||||
) {
|
||||
return { changed: false, created: !pageStat };
|
||||
}
|
||||
try {
|
||||
if (isRegularFileStat(pageStat) && pageStat.nlink > 1) {
|
||||
await vault.remove(params.pagePath);
|
||||
}
|
||||
await vault.write(params.pagePath, params.content);
|
||||
} catch (error) {
|
||||
if (error instanceof FsSafeError) {
|
||||
if (error.code !== "symlink" && error.code !== "path-alias") {
|
||||
throw new Error(
|
||||
`Refusing to write OKF concept page (${error.code}): ${params.pagePath}: ${error.message}`,
|
||||
{ cause: error },
|
||||
);
|
||||
}
|
||||
throw new Error(`Refusing to write OKF concept page through symlink: ${params.pagePath}`, {
|
||||
cause: error,
|
||||
});
|
||||
}
|
||||
throw error;
|
||||
}
|
||||
return { changed: true, created: !pageStat };
|
||||
}
|
||||
|
||||
async function removeStaleOkfConceptPages(params: {
|
||||
vaultRoot: string;
|
||||
bundleKey: string;
|
||||
currentPagePaths: Set<string>;
|
||||
}): Promise<string[]> {
|
||||
const vault = await fsRoot(params.vaultRoot);
|
||||
const conceptsDir = path.join(params.vaultRoot, "concepts");
|
||||
const entries = await fs.readdir(conceptsDir, { withFileTypes: true }).catch(() => []);
|
||||
const removedPagePaths: string[] = [];
|
||||
for (const entry of entries) {
|
||||
if (!entry.isFile() || !entry.name.endsWith(".md") || entry.name === "index.md") {
|
||||
continue;
|
||||
}
|
||||
const pagePath = `concepts/${entry.name}`;
|
||||
if (params.currentPagePaths.has(pagePath)) {
|
||||
continue;
|
||||
}
|
||||
const raw = await vault.readText(pagePath).catch(() => "");
|
||||
const parsed = parseWikiMarkdown(raw);
|
||||
const okf = parsed.frontmatter.okf;
|
||||
if (
|
||||
okf &&
|
||||
typeof okf === "object" &&
|
||||
!Array.isArray(okf) &&
|
||||
(okf as Record<string, unknown>).bundleKey === params.bundleKey
|
||||
) {
|
||||
await vault.remove(pagePath);
|
||||
removedPagePaths.push(pagePath);
|
||||
}
|
||||
}
|
||||
return removedPagePaths;
|
||||
}
|
||||
|
||||
function readRootOkfMetadata(params: {
|
||||
rootIndex: string | undefined;
|
||||
bundleName: string;
|
||||
bundlePath: string;
|
||||
}): OkfBundleMetadata {
|
||||
if (!params.rootIndex) {
|
||||
return {
|
||||
key: createOkfBundleKey({
|
||||
rootFrontmatter: {},
|
||||
bundleName: params.bundleName,
|
||||
bundlePath: params.bundlePath,
|
||||
}),
|
||||
};
|
||||
}
|
||||
const parsed = parseOkfMarkdown(params.rootIndex, "index.md");
|
||||
return {
|
||||
key: createOkfBundleKey({
|
||||
rootFrontmatter: parsed.frontmatter,
|
||||
bundleName: params.bundleName,
|
||||
bundlePath: params.bundlePath,
|
||||
}),
|
||||
...(normalizeOptionalString(parsed.frontmatter.okf_version)
|
||||
? { version: normalizeOptionalString(parsed.frontmatter.okf_version) }
|
||||
: {}),
|
||||
};
|
||||
}
|
||||
|
||||
function formatOkfImportSummary(result: ImportMemoryWikiOkfResult): string {
|
||||
return `Imported ${result.importedCount} OKF concept${result.importedCount === 1 ? "" : "s"} from ${result.bundlePath} into memory wiki. Updated ${result.updatedCount}; removed ${result.removedCount}; skipped ${result.skippedCount}; refreshed ${result.indexUpdatedFiles.length} index file${result.indexUpdatedFiles.length === 1 ? "" : "s"}.`;
|
||||
}
|
||||
|
||||
export { formatOkfImportSummary };
|
||||
|
||||
export async function importMemoryWikiOkfBundle(params: {
|
||||
config: ResolvedMemoryWikiConfig;
|
||||
bundlePath: string;
|
||||
nowMs?: number;
|
||||
}): Promise<ImportMemoryWikiOkfResult> {
|
||||
await initializeMemoryWikiVault(params.config, { nowMs: params.nowMs });
|
||||
const bundlePath = path.resolve(params.bundlePath);
|
||||
const stat = await fs.stat(bundlePath);
|
||||
if (!stat.isDirectory()) {
|
||||
throw new Error("wiki okf import expects an unpacked OKF bundle directory.");
|
||||
}
|
||||
|
||||
const warnings: ImportMemoryWikiOkfWarning[] = [];
|
||||
const markdownFiles = await collectOkfMarkdownFiles(bundlePath, warnings);
|
||||
const concepts: OkfConceptDocument[] = [];
|
||||
let rootIndexContent: string | undefined;
|
||||
|
||||
for (const relativePath of markdownFiles) {
|
||||
if (relativePath === "index.md") {
|
||||
rootIndexContent =
|
||||
(await readOkfTextFile({ bundlePath, relativePath, warnings })) ?? undefined;
|
||||
}
|
||||
if (OKF_RESERVED_FILENAMES.has(path.posix.basename(relativePath))) {
|
||||
continue;
|
||||
}
|
||||
const content = await readOkfTextFile({ bundlePath, relativePath, warnings });
|
||||
if (content === null) {
|
||||
continue;
|
||||
}
|
||||
const normalized = normalizeOkfConcept({ bundlePath, relativePath, content });
|
||||
if (normalized.warning) {
|
||||
warnings.push(normalized.warning);
|
||||
continue;
|
||||
}
|
||||
if (normalized.concept) {
|
||||
concepts.push(normalized.concept);
|
||||
}
|
||||
}
|
||||
|
||||
const timestamp = resolveMemoryWikiTimestamp(params.nowMs);
|
||||
const bundleName = path.basename(bundlePath);
|
||||
const bundleMetadata = readRootOkfMetadata({
|
||||
rootIndex: rootIndexContent,
|
||||
bundleName,
|
||||
bundlePath,
|
||||
});
|
||||
const bundleKey = bundleMetadata.key;
|
||||
const pageByConceptId = new Map<string, { pageId: string; pagePath: string; title: string }>();
|
||||
for (const concept of concepts) {
|
||||
pageByConceptId.set(concept.conceptId, {
|
||||
...createOkfPageIdentity(bundleKey, concept.conceptId),
|
||||
title: concept.title,
|
||||
});
|
||||
}
|
||||
|
||||
const importedPages: OkfImportedPage[] = [];
|
||||
let updatedCount = 0;
|
||||
|
||||
await fs.mkdir(path.join(params.config.vault.path, "concepts"), { recursive: true });
|
||||
for (const concept of concepts.toSorted((left, right) =>
|
||||
left.conceptId.localeCompare(right.conceptId),
|
||||
)) {
|
||||
const page = pageByConceptId.get(concept.conceptId);
|
||||
if (!page) {
|
||||
continue;
|
||||
}
|
||||
const rewritten = rewriteOkfMarkdownLinks({
|
||||
body: concept.body,
|
||||
sourcePagePath: page.pagePath,
|
||||
sourceRelativePath: concept.relativePath,
|
||||
pageByConceptId,
|
||||
});
|
||||
const relationships = rewritten.linkedConceptIds.flatMap((conceptId) => {
|
||||
const target = pageByConceptId.get(conceptId);
|
||||
return target
|
||||
? [
|
||||
{
|
||||
targetId: target.pageId,
|
||||
targetPath: target.pagePath,
|
||||
targetTitle: target.title,
|
||||
kind: "okf-link",
|
||||
evidenceKind: "okf-markdown-link",
|
||||
},
|
||||
]
|
||||
: [];
|
||||
});
|
||||
|
||||
const frontmatter = {
|
||||
pageType: "concept",
|
||||
id: page.pageId,
|
||||
title: concept.title,
|
||||
sourceType: "okf",
|
||||
provenanceMode: "okf-import",
|
||||
sourcePath: concept.absolutePath,
|
||||
okfConceptId: concept.conceptId,
|
||||
okfType: concept.type,
|
||||
sourceIds: [`source.okf.${bundleKey}`],
|
||||
importedAt: timestamp,
|
||||
updatedAt: concept.timestamp ?? timestamp,
|
||||
status: "active",
|
||||
...(concept.description ? { description: concept.description } : {}),
|
||||
...(concept.resource ? { resource: concept.resource } : {}),
|
||||
...(concept.tags.length > 0 ? { tags: concept.tags } : {}),
|
||||
...(concept.timestamp ? { okfTimestamp: concept.timestamp } : {}),
|
||||
...(relationships.length > 0 ? { relationships } : {}),
|
||||
okf: {
|
||||
...(bundleMetadata.version ? { version: bundleMetadata.version } : {}),
|
||||
bundleName,
|
||||
bundleKey,
|
||||
conceptId: concept.conceptId,
|
||||
sourceRelativePath: concept.relativePath,
|
||||
frontmatter: concept.frontmatter,
|
||||
},
|
||||
};
|
||||
|
||||
const writeResult = await writeOkfConceptPage({
|
||||
vaultRoot: params.config.vault.path,
|
||||
pagePath: page.pagePath,
|
||||
content: renderWikiMarkdown({
|
||||
frontmatter,
|
||||
body: rewritten.body,
|
||||
}),
|
||||
});
|
||||
if (!writeResult.created && writeResult.changed) {
|
||||
updatedCount++;
|
||||
}
|
||||
importedPages.push({
|
||||
conceptId: concept.conceptId,
|
||||
sourcePath: concept.absolutePath,
|
||||
pageId: page.pageId,
|
||||
pagePath: page.pagePath,
|
||||
title: concept.title,
|
||||
created: writeResult.created,
|
||||
});
|
||||
}
|
||||
const currentPagePaths = new Set(importedPages.map((page) => page.pagePath));
|
||||
const removedPagePaths =
|
||||
warnings.length === 0
|
||||
? await removeStaleOkfConceptPages({
|
||||
vaultRoot: params.config.vault.path,
|
||||
bundleKey,
|
||||
currentPagePaths,
|
||||
})
|
||||
: [];
|
||||
|
||||
await appendMemoryWikiLog(params.config.vault.path, {
|
||||
type: "okf-import",
|
||||
timestamp,
|
||||
details: {
|
||||
bundlePath,
|
||||
bundleName,
|
||||
importedCount: importedPages.length,
|
||||
updatedCount,
|
||||
removedCount: removedPagePaths.length,
|
||||
skippedCount: warnings.length,
|
||||
pagePaths: importedPages.map((page) => page.pagePath),
|
||||
removedPagePaths,
|
||||
},
|
||||
});
|
||||
|
||||
const compile = await compileMemoryWikiVault(params.config);
|
||||
return {
|
||||
bundlePath,
|
||||
bundleName,
|
||||
...(bundleMetadata.version ? { okfVersion: bundleMetadata.version } : {}),
|
||||
importedCount: importedPages.length,
|
||||
updatedCount,
|
||||
removedCount: removedPagePaths.length,
|
||||
skippedCount: warnings.length,
|
||||
pagePaths: importedPages.map((page) => page.pagePath),
|
||||
removedPagePaths,
|
||||
warnings,
|
||||
indexUpdatedFiles: compile.updatedFiles,
|
||||
};
|
||||
}
|
||||
@@ -11,6 +11,24 @@ import {
|
||||
expectUnifiedModelCatalogProviderRegistration,
|
||||
} from "openclaw/plugin-sdk/provider-test-contracts";
|
||||
import { describe, expect, it, vi } from "vitest";
|
||||
|
||||
const { getOpenRouterModelCapabilitiesMock, loadOpenRouterModelCapabilitiesMock } = vi.hoisted(
|
||||
() => ({
|
||||
getOpenRouterModelCapabilitiesMock: vi.fn(),
|
||||
loadOpenRouterModelCapabilitiesMock: vi.fn(async () => {}),
|
||||
}),
|
||||
);
|
||||
|
||||
vi.mock("openclaw/plugin-sdk/provider-stream-family", async (importOriginal) => {
|
||||
const actual =
|
||||
await importOriginal<typeof import("openclaw/plugin-sdk/provider-stream-family")>();
|
||||
return {
|
||||
...actual,
|
||||
getOpenRouterModelCapabilities: getOpenRouterModelCapabilitiesMock,
|
||||
loadOpenRouterModelCapabilities: loadOpenRouterModelCapabilitiesMock,
|
||||
};
|
||||
});
|
||||
|
||||
import openrouterPlugin from "./index.js";
|
||||
import {
|
||||
buildOpenrouterProvider,
|
||||
@@ -204,6 +222,59 @@ describe("openrouter provider hooks", () => {
|
||||
expect(buildOpenrouterProvider().models?.map((model) => model.id)).not.toContain("auto");
|
||||
});
|
||||
|
||||
it("normalizes OpenRouter API ids before capability loading and lookup", async () => {
|
||||
getOpenRouterModelCapabilitiesMock.mockReset();
|
||||
loadOpenRouterModelCapabilitiesMock.mockClear();
|
||||
getOpenRouterModelCapabilitiesMock.mockReturnValue({
|
||||
name: "Claude Sonnet 4.6",
|
||||
reasoning: true,
|
||||
input: ["text", "image"],
|
||||
supportsTools: true,
|
||||
cost: { input: 1, output: 2, cacheRead: 0, cacheWrite: 0 },
|
||||
contextWindow: 200_000,
|
||||
maxTokens: 64_000,
|
||||
});
|
||||
const provider = await registerSingleProviderPlugin(openrouterPlugin);
|
||||
const modelId = "openrouter/anthropic/claude-sonnet-4.6";
|
||||
const context = {
|
||||
provider: "openrouter",
|
||||
modelId,
|
||||
modelRegistry: { find: vi.fn(() => null) },
|
||||
} as never;
|
||||
|
||||
await provider.prepareDynamicModel?.(context);
|
||||
const model = provider.resolveDynamicModel?.(context);
|
||||
|
||||
expect(loadOpenRouterModelCapabilitiesMock).toHaveBeenCalledWith("anthropic/claude-sonnet-4.6");
|
||||
expect(getOpenRouterModelCapabilitiesMock).toHaveBeenCalledWith("anthropic/claude-sonnet-4.6");
|
||||
expect(model).toMatchObject({
|
||||
id: modelId,
|
||||
name: "Claude Sonnet 4.6",
|
||||
reasoning: true,
|
||||
input: ["text", "image"],
|
||||
compat: { supportsTools: true },
|
||||
contextWindow: 200_000,
|
||||
maxTokens: 64_000,
|
||||
});
|
||||
});
|
||||
|
||||
it("keeps native OpenRouter namespace ids for capability lookup", async () => {
|
||||
getOpenRouterModelCapabilitiesMock.mockReset();
|
||||
loadOpenRouterModelCapabilitiesMock.mockClear();
|
||||
const provider = await registerSingleProviderPlugin(openrouterPlugin);
|
||||
const context = {
|
||||
provider: "openrouter",
|
||||
modelId: "openrouter/auto",
|
||||
modelRegistry: { find: vi.fn(() => null) },
|
||||
} as never;
|
||||
|
||||
await provider.prepareDynamicModel?.(context);
|
||||
provider.resolveDynamicModel?.(context);
|
||||
|
||||
expect(loadOpenRouterModelCapabilitiesMock).toHaveBeenCalledWith("openrouter/auto");
|
||||
expect(getOpenRouterModelCapabilitiesMock).toHaveBeenCalledWith("openrouter/auto");
|
||||
});
|
||||
|
||||
it("does not include retired stealth models in the bundled catalog", () => {
|
||||
const modelIds = buildOpenrouterProvider().models?.map((model) => model.id) ?? [];
|
||||
expect(modelIds).not.toContain("openrouter/hunter-alpha");
|
||||
@@ -389,6 +460,61 @@ describe("openrouter provider hooks", () => {
|
||||
},
|
||||
} as never);
|
||||
expect(normalizedHunterModel?.reasoning).toBe(false);
|
||||
expect(normalizedHunterModel?.id).toBe("openrouter/hunter-alpha");
|
||||
|
||||
const normalizedAnthropicModel = provider.normalizeResolvedModel?.({
|
||||
provider: "openrouter",
|
||||
model: {
|
||||
provider: "openrouter",
|
||||
id: "openrouter/anthropic/claude-sonnet-4.6",
|
||||
name: "anthropic/claude-sonnet-4.6",
|
||||
api: "openai-completions",
|
||||
baseUrl: "https://openrouter.ai/api/v1",
|
||||
reasoning: true,
|
||||
input: ["text", "image"],
|
||||
cost: { input: 0, output: 0, cacheRead: 0, cacheWrite: 0 },
|
||||
contextWindow: 200_000,
|
||||
maxTokens: 8192,
|
||||
},
|
||||
} as never);
|
||||
expect(normalizedAnthropicModel?.id).toBe("anthropic/claude-sonnet-4.6");
|
||||
|
||||
expect(
|
||||
provider.normalizeResolvedModel?.({
|
||||
provider: "openrouter",
|
||||
modelId: "openrouter/auto",
|
||||
model: {
|
||||
provider: "openrouter",
|
||||
id: "openrouter/auto",
|
||||
name: "OpenRouter Auto",
|
||||
api: "openai-completions",
|
||||
baseUrl: "https://openrouter.ai/api/v1",
|
||||
reasoning: false,
|
||||
input: ["text", "image"],
|
||||
cost: { input: 0, output: 0, cacheRead: 0, cacheWrite: 0 },
|
||||
contextWindow: 200_000,
|
||||
maxTokens: 8192,
|
||||
},
|
||||
} as never),
|
||||
).toBeUndefined();
|
||||
|
||||
const normalizedDuplicatedAutoModel = provider.normalizeResolvedModel?.({
|
||||
provider: "openrouter",
|
||||
modelId: "openrouter/openrouter/auto",
|
||||
model: {
|
||||
provider: "openrouter",
|
||||
id: "openrouter/openrouter/auto",
|
||||
name: "OpenRouter Auto",
|
||||
api: "openai-completions",
|
||||
baseUrl: "https://openrouter.ai/api/v1",
|
||||
reasoning: false,
|
||||
input: ["text", "image"],
|
||||
cost: { input: 0, output: 0, cacheRead: 0, cacheWrite: 0 },
|
||||
contextWindow: 200_000,
|
||||
maxTokens: 8192,
|
||||
},
|
||||
} as never);
|
||||
expect(normalizedDuplicatedAutoModel?.id).toBe("openrouter/auto");
|
||||
|
||||
expect(
|
||||
provider.normalizeTransport?.({
|
||||
|
||||
@@ -17,7 +17,7 @@ import {
|
||||
} from "openclaw/plugin-sdk/provider-stream-family";
|
||||
import { buildOpenRouterImageGenerationProvider } from "./image-generation-provider.js";
|
||||
import { openrouterMediaUnderstandingProvider } from "./media-understanding-provider.js";
|
||||
import { isOpenRouterMistralModelId } from "./models.js";
|
||||
import { isOpenRouterMistralModelId, normalizeOpenRouterApiModelId } from "./models.js";
|
||||
import { buildOpenRouterMusicGenerationProvider } from "./music-generation-provider.js";
|
||||
import { createOpenRouterOAuthAuthMethod } from "./oauth.js";
|
||||
import { applyOpenrouterConfig, OPENROUTER_DEFAULT_MODEL_REF } from "./onboard.js";
|
||||
@@ -51,15 +51,18 @@ const OPENROUTER_CACHE_TTL_MODEL_PREFIXES = [
|
||||
|
||||
function normalizeOpenRouterResolvedModel<T extends ProviderRuntimeModel>(model: T): T | undefined {
|
||||
const normalizedBaseUrl = normalizeOpenRouterBaseUrl(model.baseUrl);
|
||||
const normalizedId = normalizeOpenRouterApiModelId(model.id);
|
||||
const reasoning = isOpenRouterProxyReasoningUnsupportedModel(model.id) ? false : model.reasoning;
|
||||
if (
|
||||
(!normalizedBaseUrl || normalizedBaseUrl === model.baseUrl) &&
|
||||
(!normalizedId || normalizedId === model.id) &&
|
||||
reasoning === model.reasoning
|
||||
) {
|
||||
return undefined;
|
||||
}
|
||||
return {
|
||||
...model,
|
||||
...(normalizedId ? { id: normalizedId } : {}),
|
||||
...(normalizedBaseUrl ? { baseUrl: normalizedBaseUrl } : {}),
|
||||
reasoning,
|
||||
};
|
||||
@@ -73,7 +76,8 @@ export default definePluginEntry({
|
||||
function buildDynamicOpenRouterModel(
|
||||
ctx: ProviderResolveDynamicModelContext,
|
||||
): ProviderRuntimeModel {
|
||||
const capabilities = getOpenRouterModelCapabilities(ctx.modelId);
|
||||
const apiModelId = normalizeOpenRouterApiModelId(ctx.modelId) ?? ctx.modelId;
|
||||
const capabilities = getOpenRouterModelCapabilities(apiModelId);
|
||||
return {
|
||||
id: ctx.modelId,
|
||||
name: capabilities?.name ?? ctx.modelId,
|
||||
@@ -166,7 +170,9 @@ export default definePluginEntry({
|
||||
},
|
||||
resolveDynamicModel: (ctx) => buildDynamicOpenRouterModel(ctx),
|
||||
prepareDynamicModel: async (ctx) => {
|
||||
await loadOpenRouterModelCapabilities(ctx.modelId);
|
||||
await loadOpenRouterModelCapabilities(
|
||||
normalizeOpenRouterApiModelId(ctx.modelId) ?? ctx.modelId,
|
||||
);
|
||||
},
|
||||
normalizeConfig: ({ providerConfig }) => {
|
||||
const normalizedBaseUrl = normalizeOpenRouterBaseUrl(providerConfig.baseUrl);
|
||||
|
||||
@@ -12,13 +12,30 @@ const OPENROUTER_MISTRAL_MODEL_PREFIXES = [
|
||||
"pixtral-",
|
||||
"voxtral-",
|
||||
] as const;
|
||||
const OPENROUTER_MODEL_PREFIX = "openrouter/";
|
||||
|
||||
export function normalizeOpenRouterModelId(modelId: unknown): string | undefined {
|
||||
if (typeof modelId !== "string") {
|
||||
return undefined;
|
||||
}
|
||||
const normalized = normalizeLowercaseStringOrEmpty(modelId);
|
||||
return normalized.startsWith("openrouter/") ? normalized.slice("openrouter/".length) : normalized;
|
||||
return normalized.startsWith(OPENROUTER_MODEL_PREFIX)
|
||||
? normalized.slice(OPENROUTER_MODEL_PREFIX.length)
|
||||
: normalized;
|
||||
}
|
||||
|
||||
export function normalizeOpenRouterApiModelId(modelId: unknown): string | undefined {
|
||||
if (typeof modelId !== "string") {
|
||||
return undefined;
|
||||
}
|
||||
const normalized = normalizeLowercaseStringOrEmpty(modelId);
|
||||
if (!normalized.startsWith(OPENROUTER_MODEL_PREFIX)) {
|
||||
return normalized;
|
||||
}
|
||||
const unprefixed = normalized.slice(OPENROUTER_MODEL_PREFIX.length);
|
||||
// `openrouter/` is both a provider qualifier and an upstream namespace.
|
||||
// Strip it only when the remainder is still a namespaced API model id.
|
||||
return unprefixed.includes("/") ? unprefixed : normalized;
|
||||
}
|
||||
|
||||
export function isOpenRouterMistralModelId(modelId: unknown): boolean {
|
||||
|
||||
@@ -7,12 +7,17 @@ import {
|
||||
} from "openclaw/plugin-sdk/plugin-test-runtime";
|
||||
import { describe, expect, it } from "vitest";
|
||||
import plugin from "./index.js";
|
||||
import { normalizeOpenRouterApiModelId } from "./models.js";
|
||||
|
||||
const OPENROUTER_MODELS_URL = "https://openrouter.ai/api/v1/models";
|
||||
const OPENROUTER_MISTRAL_PROVIDER_PREFIX = "mistralai/";
|
||||
const OPENROUTER_API_KEY = process.env.OPENROUTER_API_KEY ?? "";
|
||||
const LIVE_MODEL_ID =
|
||||
process.env.OPENCLAW_LIVE_OPENROUTER_PLUGIN_MODEL?.trim() || "openai/gpt-5.4-nano";
|
||||
const LIVE_MODEL_REF =
|
||||
process.env.OPENCLAW_LIVE_OPENROUTER_PLUGIN_MODEL?.trim() ||
|
||||
"openrouter/anthropic/claude-sonnet-4.6";
|
||||
const LIVE_MODEL_ID = LIVE_MODEL_REF.startsWith("openrouter/")
|
||||
? LIVE_MODEL_REF
|
||||
: `openrouter/${LIVE_MODEL_REF}`;
|
||||
const LIVE_CACHE_MODEL_ID =
|
||||
process.env.OPENCLAW_LIVE_OPENROUTER_CACHE_MODEL?.trim() || "deepseek/deepseek-v3.2";
|
||||
const liveEnabled = OPENROUTER_API_KEY.trim().length > 0 && process.env.OPENCLAW_LIVE_TEST === "1";
|
||||
@@ -57,6 +62,40 @@ async function completeOpenRouterChat(params: {
|
||||
});
|
||||
}
|
||||
|
||||
async function expectWeatherToolCall(client: OpenAI, model: string): Promise<void> {
|
||||
const response = await client.chat.completions.create({
|
||||
model,
|
||||
messages: [{ role: "user", content: "Call get_weather for Paris." }],
|
||||
tools: [
|
||||
{
|
||||
type: "function",
|
||||
function: {
|
||||
name: "get_weather",
|
||||
description: "Get the weather for a city.",
|
||||
parameters: {
|
||||
type: "object",
|
||||
properties: { city: { type: "string" } },
|
||||
required: ["city"],
|
||||
additionalProperties: false,
|
||||
},
|
||||
},
|
||||
},
|
||||
],
|
||||
tool_choice: {
|
||||
type: "function",
|
||||
function: { name: "get_weather" },
|
||||
},
|
||||
max_tokens: 64,
|
||||
});
|
||||
|
||||
const toolCall = response.choices[0]?.message?.tool_calls?.find(
|
||||
(call) => call.type === "function",
|
||||
);
|
||||
expect(toolCall?.type).toBe("function");
|
||||
expect(toolCall?.function.name).toBe("get_weather");
|
||||
expect(JSON.parse(toolCall?.function.arguments ?? "{}")).toMatchObject({ city: "Paris" });
|
||||
}
|
||||
|
||||
async function fetchOpenRouterModelIds(): Promise<string[]> {
|
||||
const response = await fetch(OPENROUTER_MODELS_URL, {
|
||||
headers: { "accept-encoding": "identity" },
|
||||
@@ -69,7 +108,7 @@ async function fetchOpenRouterModelIds(): Promise<string[]> {
|
||||
}
|
||||
|
||||
describeLive("openrouter plugin live", () => {
|
||||
it("registers an OpenRouter provider that can complete a live request", async () => {
|
||||
it("normalizes a prefixed OpenRouter model and completes a live tool call", async () => {
|
||||
const { providers } = await registerOpenRouterPlugin();
|
||||
const provider = requireRegisteredProvider(providers, "openrouter");
|
||||
|
||||
@@ -87,17 +126,35 @@ describeLive("openrouter plugin live", () => {
|
||||
expect(resolved.api).toBe("openai-completions");
|
||||
expect(resolved.baseUrl).toBe("https://openrouter.ai/api/v1");
|
||||
|
||||
const normalized =
|
||||
provider.normalizeResolvedModel?.({
|
||||
provider: "openrouter",
|
||||
modelId: resolved.id,
|
||||
model: resolved,
|
||||
}) ?? resolved;
|
||||
expect(normalized.id).toBe(normalizeOpenRouterApiModelId(LIVE_MODEL_ID));
|
||||
|
||||
const client = new OpenAI({
|
||||
apiKey: OPENROUTER_API_KEY,
|
||||
baseURL: resolved.baseUrl,
|
||||
baseURL: normalized.baseUrl,
|
||||
});
|
||||
const response = await client.chat.completions.create({
|
||||
model: resolved.id,
|
||||
messages: [{ role: "user", content: "Reply with exactly OK." }],
|
||||
max_tokens: 16,
|
||||
const autoResolved = provider.resolveDynamicModel?.({
|
||||
provider: "openrouter",
|
||||
modelId: "openrouter/auto",
|
||||
modelRegistry: new ModelRegistryCtor(AuthStorage.inMemory()),
|
||||
});
|
||||
|
||||
expect(response.choices[0]?.message?.content?.trim()).toMatch(/^OK[.!]?$/);
|
||||
if (!autoResolved) {
|
||||
throw new Error("openrouter provider did not resolve openrouter/auto");
|
||||
}
|
||||
const autoModel =
|
||||
provider.normalizeResolvedModel?.({
|
||||
provider: "openrouter",
|
||||
modelId: autoResolved.id,
|
||||
model: autoResolved,
|
||||
}) ?? autoResolved;
|
||||
expect(autoModel.id).toBe("openrouter/auto");
|
||||
await expectWeatherToolCall(client, autoModel.id);
|
||||
await expectWeatherToolCall(client, normalized.id);
|
||||
}, 30_000);
|
||||
});
|
||||
|
||||
|
||||
@@ -106,5 +106,6 @@ export {
|
||||
type QaSuiteStartLabFn,
|
||||
type QaSuiteSummaryJson,
|
||||
type QaSuiteSummaryJsonParams,
|
||||
runQaSuite,
|
||||
runQaFlowSuite,
|
||||
} from "./src/suite.js";
|
||||
export { runQaSuite, type QaSuiteRuntimeResult } from "./src/suite-launch.runtime.js";
|
||||
|
||||
@@ -127,6 +127,7 @@ async function makeSuiteResult(params: {
|
||||
);
|
||||
return {
|
||||
outputDir: params.outputDir,
|
||||
evidencePath: path.join(params.outputDir, "qa-evidence.json"),
|
||||
reportPath: path.join(params.outputDir, "qa-suite-report.md"),
|
||||
summaryPath,
|
||||
report: "# report",
|
||||
|
||||
@@ -426,8 +426,8 @@ async function defaultRunJudge(params: {
|
||||
}
|
||||
|
||||
async function defaultRunSuite(params: Parameters<RunSuiteFn>[0]) {
|
||||
const { runQaSuiteFromRuntime } = await import("./suite-launch.runtime.js");
|
||||
return await runQaSuiteFromRuntime(params);
|
||||
const { runQaFlowSuiteFromRuntime } = await import("./suite-launch.runtime.js");
|
||||
return await runQaFlowSuiteFromRuntime(params);
|
||||
}
|
||||
|
||||
function renderCharacterEvalReport(params: {
|
||||
|
||||
@@ -3,6 +3,18 @@ import fs from "node:fs/promises";
|
||||
import path from "node:path";
|
||||
import { assertNoSymlinkParents, pathScope } from "openclaw/plugin-sdk/security-runtime";
|
||||
|
||||
export function toRepoPath(filePath: string): string {
|
||||
return filePath.split(path.sep).join("/");
|
||||
}
|
||||
|
||||
export function toRepoRelativePath(repoRoot: string, filePath: string): string {
|
||||
return toRepoPath(path.relative(repoRoot, filePath));
|
||||
}
|
||||
|
||||
export function isRepoRootRelativeRef(value: string) {
|
||||
return !path.isAbsolute(value) && value.split(/[\\/]+/u).every((part) => part !== "..");
|
||||
}
|
||||
|
||||
export function resolveRepoRelativeOutputDir(repoRoot: string, outputDir?: string) {
|
||||
if (!outputDir) {
|
||||
return undefined;
|
||||
|
||||
@@ -6,7 +6,8 @@ import { afterEach, beforeEach, describe, expect, it, vi } from "vitest";
|
||||
|
||||
const {
|
||||
runQaManualLane,
|
||||
runQaSuiteFromRuntime,
|
||||
runQaFlowSuiteFromRuntime,
|
||||
runQaSuite,
|
||||
runQaCharacterEval,
|
||||
runQaMultipass,
|
||||
listTelegramQaScenarioCatalog,
|
||||
@@ -18,7 +19,8 @@ const {
|
||||
defaultQaRuntimeModelForMode,
|
||||
} = vi.hoisted(() => ({
|
||||
runQaManualLane: vi.fn(),
|
||||
runQaSuiteFromRuntime: vi.fn(),
|
||||
runQaFlowSuiteFromRuntime: vi.fn(),
|
||||
runQaSuite: vi.fn(),
|
||||
runQaCharacterEval: vi.fn(),
|
||||
runQaMultipass: vi.fn(),
|
||||
listTelegramQaScenarioCatalog: vi.fn(),
|
||||
@@ -36,7 +38,8 @@ vi.mock("./manual-lane.runtime.js", () => ({
|
||||
}));
|
||||
|
||||
vi.mock("./suite-launch.runtime.js", () => ({
|
||||
runQaSuiteFromRuntime,
|
||||
runQaFlowSuiteFromRuntime,
|
||||
runQaSuite,
|
||||
}));
|
||||
|
||||
vi.mock("./character-eval.js", () => ({
|
||||
@@ -115,10 +118,51 @@ function expectWriteContains(mock: unknown, fragment: string): void {
|
||||
).toBe(true);
|
||||
}
|
||||
|
||||
function flowSuiteRuntimeResult(params: {
|
||||
evidencePath?: string;
|
||||
reportPath: string;
|
||||
summaryPath: string;
|
||||
scenarios?: unknown[];
|
||||
}) {
|
||||
return {
|
||||
executionKind: "flow",
|
||||
result: {
|
||||
outputDir: path.dirname(params.reportPath),
|
||||
evidencePath:
|
||||
params.evidencePath ?? path.join(path.dirname(params.reportPath), "qa-evidence.json"),
|
||||
reportPath: params.reportPath,
|
||||
summaryPath: params.summaryPath,
|
||||
report: "# QA Suite Report\n",
|
||||
scenarios: params.scenarios ?? [],
|
||||
watchUrl: "http://127.0.0.1:43124",
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
function testFileSuiteRuntimeResult(params: {
|
||||
evidencePath: string;
|
||||
executionKind?: "vitest" | "playwright";
|
||||
outputDir: string;
|
||||
reportPath: string;
|
||||
results?: unknown[];
|
||||
}) {
|
||||
return {
|
||||
executionKind: params.executionKind ?? "playwright",
|
||||
result: {
|
||||
outputDir: params.outputDir,
|
||||
executionKind: params.executionKind ?? "playwright",
|
||||
reportPath: params.reportPath,
|
||||
evidencePath: params.evidencePath,
|
||||
results: params.results ?? [{ status: "pass" }],
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
describe("qa cli runtime", () => {
|
||||
let stdoutWrite: ReturnType<typeof vi.spyOn>;
|
||||
let stderrWrite: ReturnType<typeof vi.spyOn>;
|
||||
let suiteArtifactsDir: string;
|
||||
let suiteEvidencePath: string;
|
||||
let suiteReportPath: string;
|
||||
let suiteSummaryPath: string;
|
||||
let telegramArtifactsDir: string;
|
||||
@@ -126,11 +170,13 @@ describe("qa cli runtime", () => {
|
||||
|
||||
beforeEach(async () => {
|
||||
suiteArtifactsDir = await fs.mkdtemp(path.join(os.tmpdir(), "qa-suite-runtime-"));
|
||||
suiteEvidencePath = path.join(suiteArtifactsDir, "qa-evidence.json");
|
||||
suiteReportPath = path.join(suiteArtifactsDir, "qa-suite-report.md");
|
||||
suiteSummaryPath = path.join(suiteArtifactsDir, "qa-suite-summary.json");
|
||||
telegramArtifactsDir = await fs.mkdtemp(path.join(os.tmpdir(), "qa-telegram-runtime-"));
|
||||
telegramSummaryPath = path.join(telegramArtifactsDir, QA_EVIDENCE_FILENAME);
|
||||
await fs.writeFile(suiteReportPath, "# QA Suite Report\n", "utf8");
|
||||
await fs.writeFile(suiteEvidencePath, JSON.stringify({ entries: [] }), "utf8");
|
||||
await fs.writeFile(
|
||||
suiteSummaryPath,
|
||||
JSON.stringify({
|
||||
@@ -157,7 +203,8 @@ describe("qa cli runtime", () => {
|
||||
);
|
||||
stdoutWrite = vi.spyOn(process.stdout, "write").mockReturnValue(true);
|
||||
stderrWrite = vi.spyOn(process.stderr, "write").mockReturnValue(true);
|
||||
runQaSuiteFromRuntime.mockReset();
|
||||
runQaFlowSuiteFromRuntime.mockReset();
|
||||
runQaSuite.mockReset();
|
||||
runQaCharacterEval.mockReset();
|
||||
runQaManualLane.mockReset();
|
||||
runQaMultipass.mockReset();
|
||||
@@ -171,7 +218,15 @@ describe("qa cli runtime", () => {
|
||||
(mode: string, options?: { alternate?: boolean }) =>
|
||||
defaultQaProviderModelForMode(mode as QaProviderModeInput, options),
|
||||
);
|
||||
runQaSuiteFromRuntime.mockResolvedValue({
|
||||
runQaSuite.mockResolvedValue(
|
||||
flowSuiteRuntimeResult({
|
||||
reportPath: suiteReportPath,
|
||||
summaryPath: suiteSummaryPath,
|
||||
}),
|
||||
);
|
||||
runQaFlowSuiteFromRuntime.mockResolvedValue({
|
||||
outputDir: suiteArtifactsDir,
|
||||
evidencePath: suiteEvidencePath,
|
||||
watchUrl: "http://127.0.0.1:43124",
|
||||
reportPath: suiteReportPath,
|
||||
summaryPath: suiteSummaryPath,
|
||||
@@ -242,6 +297,48 @@ describe("qa cli runtime", () => {
|
||||
await fs.rm(telegramArtifactsDir, { recursive: true, force: true });
|
||||
});
|
||||
|
||||
it("runs selected Playwright scenarios through the suite command", async () => {
|
||||
const evidencePath = path.join(suiteArtifactsDir, "qa-evidence.json");
|
||||
await fs.writeFile(evidencePath, JSON.stringify({ entries: [] }), "utf8");
|
||||
runQaSuite.mockResolvedValueOnce(
|
||||
testFileSuiteRuntimeResult({
|
||||
outputDir: suiteArtifactsDir,
|
||||
reportPath: suiteReportPath,
|
||||
evidencePath,
|
||||
}),
|
||||
);
|
||||
|
||||
await runQaSuiteCommand({
|
||||
repoRoot: process.cwd(),
|
||||
outputDir: ".artifacts/qa-e2e/scenario-test",
|
||||
primaryModel: "mock-openai/gpt-5.5",
|
||||
scenarioIds: ["control-ui-chat-flow-playwright"],
|
||||
});
|
||||
|
||||
expect(runQaSuite).toHaveBeenCalledWith({
|
||||
repoRoot: process.cwd(),
|
||||
outputDir: path.join(process.cwd(), ".artifacts", "qa-e2e", "scenario-test"),
|
||||
transportId: "qa-channel",
|
||||
primaryModel: "mock-openai/gpt-5.5",
|
||||
alternateModel: undefined,
|
||||
fastMode: undefined,
|
||||
scenarioIds: ["control-ui-chat-flow-playwright"],
|
||||
});
|
||||
expectWriteContains(stdoutWrite, `QA suite evidence: ${evidencePath}`);
|
||||
});
|
||||
|
||||
it("rejects host-only resource options for Playwright scenarios", async () => {
|
||||
await expect(
|
||||
runQaSuiteCommand({
|
||||
repoRoot: process.cwd(),
|
||||
image: "lts",
|
||||
scenarioIds: ["control-ui-chat-flow-playwright"],
|
||||
}),
|
||||
).rejects.toThrow("--image, --cpus, --memory, and --disk require --runner multipass");
|
||||
|
||||
expect(runQaSuite).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it("resolves suite repo-root-relative paths before dispatching", async () => {
|
||||
await runQaSuiteCommand({
|
||||
repoRoot: "/tmp/openclaw-repo",
|
||||
@@ -254,7 +351,7 @@ describe("qa cli runtime", () => {
|
||||
scenarioIds: ["approval-turn-tool-followthrough"],
|
||||
});
|
||||
|
||||
expect(runQaSuiteFromRuntime).toHaveBeenCalledWith({
|
||||
expect(runQaSuite).toHaveBeenCalledWith({
|
||||
repoRoot: path.resolve("/tmp/openclaw-repo"),
|
||||
outputDir: path.resolve("/tmp/openclaw-repo", ".artifacts/qa/frontier"),
|
||||
transportId: "qa-channel",
|
||||
@@ -275,7 +372,7 @@ describe("qa cli runtime", () => {
|
||||
enabledPluginIds: ["browser", "memory-core"],
|
||||
});
|
||||
|
||||
expect(runQaSuiteFromRuntime).toHaveBeenCalledWith({
|
||||
expect(runQaSuite).toHaveBeenCalledWith({
|
||||
repoRoot: path.resolve("/tmp/openclaw-repo"),
|
||||
outputDir: undefined,
|
||||
transportId: "qa-channel",
|
||||
@@ -296,7 +393,7 @@ describe("qa cli runtime", () => {
|
||||
runtimePair: "openclaw,codex",
|
||||
});
|
||||
|
||||
expect(runQaSuiteFromRuntime).toHaveBeenCalledWith({
|
||||
expect(runQaSuite).toHaveBeenCalledWith({
|
||||
repoRoot: path.resolve("/tmp/openclaw-repo"),
|
||||
outputDir: undefined,
|
||||
transportId: "qa-channel",
|
||||
@@ -318,7 +415,7 @@ describe("qa cli runtime", () => {
|
||||
runtimePair: "legacy-runtime,codex",
|
||||
}),
|
||||
).rejects.toThrow('--runtime-pair only supports "openclaw" and "codex".');
|
||||
expect(runQaSuiteFromRuntime).not.toHaveBeenCalled();
|
||||
expect(runQaSuite).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it("accepts legacy pi as a runtime-pair suite alias", async () => {
|
||||
@@ -329,7 +426,7 @@ describe("qa cli runtime", () => {
|
||||
runtimePair: "pi,codex",
|
||||
});
|
||||
|
||||
expect(runQaSuiteFromRuntime).toHaveBeenCalledWith(
|
||||
expect(runQaSuite).toHaveBeenCalledWith(
|
||||
expect.objectContaining({
|
||||
repoRoot: path.resolve("/tmp/openclaw-repo"),
|
||||
runtimePair: ["openclaw", "codex"],
|
||||
@@ -346,7 +443,7 @@ describe("qa cli runtime", () => {
|
||||
scenarioIds: ["thread-memory-isolation"],
|
||||
});
|
||||
|
||||
expect(runQaSuiteFromRuntime).toHaveBeenCalledWith({
|
||||
expect(runQaSuite).toHaveBeenCalledWith({
|
||||
repoRoot: path.resolve("/tmp/openclaw-repo"),
|
||||
outputDir: undefined,
|
||||
transportId: "qa-channel",
|
||||
@@ -493,12 +590,13 @@ describe("qa cli runtime", () => {
|
||||
concurrency: 3,
|
||||
});
|
||||
|
||||
expectFields(mockFirstObjectArg(runQaSuiteFromRuntime), {
|
||||
expectFields(mockFirstObjectArg(runQaSuite), {
|
||||
repoRoot: path.resolve("/tmp/openclaw-repo"),
|
||||
transportId: "qa-channel",
|
||||
scenarioIds: ["channel-chat-baseline", "thread-follow-up"],
|
||||
concurrency: 3,
|
||||
});
|
||||
expectWriteContains(stdoutWrite, `QA suite evidence: ${suiteEvidencePath}`);
|
||||
});
|
||||
|
||||
it("rejects fractional suite concurrency from programmatic callers", async () => {
|
||||
@@ -509,7 +607,7 @@ describe("qa cli runtime", () => {
|
||||
concurrency: 1.5,
|
||||
}),
|
||||
).rejects.toThrow("--concurrency must be a positive integer");
|
||||
expect(runQaSuiteFromRuntime).not.toHaveBeenCalled();
|
||||
expect(runQaSuite).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it("sets a failing exit code when host suite scenarios fail", async () => {
|
||||
@@ -527,12 +625,12 @@ describe("qa cli runtime", () => {
|
||||
}),
|
||||
"utf8",
|
||||
);
|
||||
runQaSuiteFromRuntime.mockResolvedValueOnce({
|
||||
watchUrl: "http://127.0.0.1:43124",
|
||||
reportPath: suiteReportPath,
|
||||
summaryPath: suiteSummaryPath,
|
||||
scenarios: [],
|
||||
});
|
||||
runQaSuite.mockResolvedValueOnce(
|
||||
flowSuiteRuntimeResult({
|
||||
reportPath: suiteReportPath,
|
||||
summaryPath: suiteSummaryPath,
|
||||
}),
|
||||
);
|
||||
|
||||
try {
|
||||
await runQaSuiteCommand({
|
||||
@@ -560,12 +658,12 @@ describe("qa cli runtime", () => {
|
||||
}),
|
||||
"utf8",
|
||||
);
|
||||
runQaSuiteFromRuntime.mockResolvedValueOnce({
|
||||
watchUrl: "http://127.0.0.1:43124",
|
||||
reportPath: suiteReportPath,
|
||||
summaryPath: suiteSummaryPath,
|
||||
scenarios: [],
|
||||
});
|
||||
runQaSuite.mockResolvedValueOnce(
|
||||
flowSuiteRuntimeResult({
|
||||
reportPath: suiteReportPath,
|
||||
summaryPath: suiteSummaryPath,
|
||||
}),
|
||||
);
|
||||
|
||||
try {
|
||||
await runQaSuiteCommand({
|
||||
@@ -592,18 +690,19 @@ describe("qa cli runtime", () => {
|
||||
}),
|
||||
"utf8",
|
||||
);
|
||||
runQaSuiteFromRuntime.mockResolvedValueOnce({
|
||||
watchUrl: "http://127.0.0.1:43124",
|
||||
reportPath: suiteReportPath,
|
||||
summaryPath: suiteSummaryPath,
|
||||
scenarios: [
|
||||
{
|
||||
name: "channel chat baseline",
|
||||
status: "fail",
|
||||
steps: [],
|
||||
},
|
||||
],
|
||||
});
|
||||
runQaSuite.mockResolvedValueOnce(
|
||||
flowSuiteRuntimeResult({
|
||||
reportPath: suiteReportPath,
|
||||
summaryPath: suiteSummaryPath,
|
||||
scenarios: [
|
||||
{
|
||||
name: "channel chat baseline",
|
||||
status: "fail",
|
||||
steps: [],
|
||||
},
|
||||
],
|
||||
}),
|
||||
);
|
||||
|
||||
try {
|
||||
await runQaSuiteCommand({
|
||||
@@ -617,45 +716,45 @@ describe("qa cli runtime", () => {
|
||||
});
|
||||
|
||||
it("retries host suite runs once for retryable infra failures", async () => {
|
||||
runQaSuiteFromRuntime
|
||||
runQaSuite
|
||||
.mockRejectedValueOnce(
|
||||
new QaSuiteInfraError("agent_wait_failed", "agent.wait failed: gateway call timed out"),
|
||||
)
|
||||
.mockResolvedValueOnce({
|
||||
watchUrl: "http://127.0.0.1:43124",
|
||||
reportPath: suiteReportPath,
|
||||
summaryPath: suiteSummaryPath,
|
||||
scenarios: [],
|
||||
});
|
||||
.mockResolvedValueOnce(
|
||||
flowSuiteRuntimeResult({
|
||||
reportPath: suiteReportPath,
|
||||
summaryPath: suiteSummaryPath,
|
||||
}),
|
||||
);
|
||||
|
||||
await runQaSuiteCommand({
|
||||
repoRoot: "/tmp/openclaw-repo",
|
||||
});
|
||||
|
||||
expect(runQaSuiteFromRuntime).toHaveBeenCalledTimes(2);
|
||||
expect(runQaSuite).toHaveBeenCalledTimes(2);
|
||||
expectWriteContains(stderrWrite, "[qa-suite] infra retry 1/1: agent.wait failed");
|
||||
});
|
||||
|
||||
it("retries host suite runs once for qa-channel readiness timeouts", async () => {
|
||||
runQaSuiteFromRuntime
|
||||
runQaSuite
|
||||
.mockRejectedValueOnce(
|
||||
new QaSuiteInfraError(
|
||||
"transport_ready_timeout",
|
||||
"timed out after 180000ms waiting for qa-channel ready; last status: no qa-channel accounts reported",
|
||||
),
|
||||
)
|
||||
.mockResolvedValueOnce({
|
||||
watchUrl: "http://127.0.0.1:43124",
|
||||
reportPath: suiteReportPath,
|
||||
summaryPath: suiteSummaryPath,
|
||||
scenarios: [],
|
||||
});
|
||||
.mockResolvedValueOnce(
|
||||
flowSuiteRuntimeResult({
|
||||
reportPath: suiteReportPath,
|
||||
summaryPath: suiteSummaryPath,
|
||||
}),
|
||||
);
|
||||
|
||||
await runQaSuiteCommand({
|
||||
repoRoot: "/tmp/openclaw-repo",
|
||||
});
|
||||
|
||||
expect(runQaSuiteFromRuntime).toHaveBeenCalledTimes(2);
|
||||
expect(runQaSuite).toHaveBeenCalledTimes(2);
|
||||
expectWriteContains(
|
||||
stderrWrite,
|
||||
"[qa-suite] infra retry 1/1: timed out after 180000ms waiting for qa-channel ready",
|
||||
@@ -663,7 +762,7 @@ describe("qa cli runtime", () => {
|
||||
});
|
||||
|
||||
it("does not retry host suite runs for generic timeout wording", async () => {
|
||||
runQaSuiteFromRuntime.mockRejectedValueOnce(
|
||||
runQaSuite.mockRejectedValueOnce(
|
||||
new Error("approval-turn timed out waiting for post-approval read"),
|
||||
);
|
||||
|
||||
@@ -673,7 +772,7 @@ describe("qa cli runtime", () => {
|
||||
}),
|
||||
).rejects.toThrow("approval-turn timed out waiting for post-approval read");
|
||||
|
||||
expect(runQaSuiteFromRuntime).toHaveBeenCalledTimes(1);
|
||||
expect(runQaSuite).toHaveBeenCalledTimes(1);
|
||||
});
|
||||
|
||||
it("does not retry host suite runs for semantic failures", async () => {
|
||||
@@ -691,24 +790,25 @@ describe("qa cli runtime", () => {
|
||||
}),
|
||||
"utf8",
|
||||
);
|
||||
runQaSuiteFromRuntime.mockResolvedValueOnce({
|
||||
watchUrl: "http://127.0.0.1:43124",
|
||||
reportPath: suiteReportPath,
|
||||
summaryPath: suiteSummaryPath,
|
||||
scenarios: [
|
||||
{
|
||||
name: "channel chat baseline",
|
||||
status: "fail",
|
||||
steps: [],
|
||||
},
|
||||
],
|
||||
});
|
||||
runQaSuite.mockResolvedValueOnce(
|
||||
flowSuiteRuntimeResult({
|
||||
reportPath: suiteReportPath,
|
||||
summaryPath: suiteSummaryPath,
|
||||
scenarios: [
|
||||
{
|
||||
name: "channel chat baseline",
|
||||
status: "fail",
|
||||
steps: [],
|
||||
},
|
||||
],
|
||||
}),
|
||||
);
|
||||
|
||||
try {
|
||||
await runQaSuiteCommand({
|
||||
repoRoot: "/tmp/openclaw-repo",
|
||||
});
|
||||
expect(runQaSuiteFromRuntime).toHaveBeenCalledTimes(1);
|
||||
expect(runQaSuite).toHaveBeenCalledTimes(1);
|
||||
expect(process.exitCode).toBe(1);
|
||||
} finally {
|
||||
process.exitCode = priorExitCode;
|
||||
@@ -725,7 +825,7 @@ describe("qa cli runtime", () => {
|
||||
preflight: true,
|
||||
});
|
||||
|
||||
const preflightArgs = mockFirstObjectArg(runQaSuiteFromRuntime);
|
||||
const preflightArgs = mockFirstObjectArg(runQaFlowSuiteFromRuntime);
|
||||
expectFields(preflightArgs, {
|
||||
repoRoot,
|
||||
transportId: "qa-channel",
|
||||
@@ -754,7 +854,9 @@ describe("qa cli runtime", () => {
|
||||
}),
|
||||
"utf8",
|
||||
);
|
||||
runQaSuiteFromRuntime.mockResolvedValueOnce({
|
||||
runQaFlowSuiteFromRuntime.mockResolvedValueOnce({
|
||||
outputDir: suiteArtifactsDir,
|
||||
evidencePath: suiteEvidencePath,
|
||||
watchUrl: "http://127.0.0.1:43124",
|
||||
reportPath: suiteReportPath,
|
||||
summaryPath: suiteSummaryPath,
|
||||
@@ -784,7 +886,9 @@ describe("qa cli runtime", () => {
|
||||
}),
|
||||
"utf8",
|
||||
);
|
||||
runQaSuiteFromRuntime.mockResolvedValueOnce({
|
||||
runQaFlowSuiteFromRuntime.mockResolvedValueOnce({
|
||||
outputDir: suiteArtifactsDir,
|
||||
evidencePath: suiteEvidencePath,
|
||||
watchUrl: "http://127.0.0.1:43124",
|
||||
reportPath: suiteReportPath,
|
||||
summaryPath: suiteSummaryPath,
|
||||
@@ -823,7 +927,7 @@ describe("qa cli runtime", () => {
|
||||
scenarioIds: ["claude-cli-provider-capabilities-subscription"],
|
||||
});
|
||||
|
||||
expectFields(mockFirstObjectArg(runQaSuiteFromRuntime), {
|
||||
expectFields(mockFirstObjectArg(runQaSuite), {
|
||||
repoRoot: path.resolve("/tmp/openclaw-repo"),
|
||||
providerMode: "live-frontier",
|
||||
primaryModel: "claude-cli/claude-sonnet-4-6",
|
||||
@@ -840,7 +944,7 @@ describe("qa cli runtime", () => {
|
||||
scenarioIds: ["channel-chat-baseline"],
|
||||
});
|
||||
|
||||
expectFields(mockFirstObjectArg(runQaSuiteFromRuntime), {
|
||||
expectFields(mockFirstObjectArg(runQaSuite), {
|
||||
repoRoot: path.resolve("/tmp/openclaw-repo"),
|
||||
scenarioIds: [
|
||||
"channel-chat-baseline",
|
||||
@@ -867,7 +971,7 @@ describe("qa cli runtime", () => {
|
||||
scenarioIds: ["channel-chat-baseline"],
|
||||
});
|
||||
|
||||
expectFields(mockFirstObjectArg(runQaSuiteFromRuntime), {
|
||||
expectFields(mockFirstObjectArg(runQaSuite), {
|
||||
repoRoot: path.resolve("/tmp/openclaw-repo"),
|
||||
scenarioIds: [
|
||||
"channel-chat-baseline",
|
||||
@@ -892,7 +996,7 @@ describe("qa cli runtime", () => {
|
||||
scenarioIds: ["channel-chat-baseline", "runtime-tool-bash"],
|
||||
});
|
||||
|
||||
expectFields(mockFirstObjectArg(runQaSuiteFromRuntime), {
|
||||
expectFields(mockFirstObjectArg(runQaSuite), {
|
||||
repoRoot: path.resolve("/tmp/openclaw-repo"),
|
||||
scenarioIds: [
|
||||
"channel-chat-baseline",
|
||||
@@ -925,7 +1029,7 @@ describe("qa cli runtime", () => {
|
||||
runtimeParityTier: ["optional,soak"],
|
||||
});
|
||||
|
||||
expectFields(mockFirstObjectArg(runQaSuiteFromRuntime), {
|
||||
expectFields(mockFirstObjectArg(runQaSuite), {
|
||||
scenarioIds: [
|
||||
"runtime-soak-100-turn",
|
||||
"runtime-tool-image-generate",
|
||||
@@ -1465,7 +1569,21 @@ describe("qa cli runtime", () => {
|
||||
memory: "4G",
|
||||
disk: "24G",
|
||||
});
|
||||
expect(runQaSuiteFromRuntime).not.toHaveBeenCalled();
|
||||
expect(runQaSuite).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it("rejects Vitest and Playwright scenarios on the multipass runner", async () => {
|
||||
await expect(
|
||||
runQaSuiteCommand({
|
||||
repoRoot: "/tmp/openclaw-repo",
|
||||
runner: "multipass",
|
||||
scenarioIds: ["control-ui-chat-flow-playwright"],
|
||||
}),
|
||||
).rejects.toThrow(
|
||||
"--runner multipass requires execution.kind: flow scenarios; unsupported scenario(s): control-ui-chat-flow-playwright (playwright)",
|
||||
);
|
||||
|
||||
expect(runQaMultipass).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it("passes runtime-pair suite selection through to the multipass runner", async () => {
|
||||
@@ -1720,7 +1838,7 @@ describe("qa cli runtime", () => {
|
||||
alternateModel: "anthropic/claude-opus-4-8",
|
||||
});
|
||||
|
||||
expect(runQaSuiteFromRuntime).toHaveBeenCalledWith({
|
||||
expect(runQaSuite).toHaveBeenCalledWith({
|
||||
repoRoot: path.resolve("/tmp/openclaw-repo"),
|
||||
outputDir: undefined,
|
||||
transportId: "qa-channel",
|
||||
|
||||
@@ -68,7 +68,7 @@ import {
|
||||
type QaRuntimeParityTier,
|
||||
} from "./scenario-catalog.js";
|
||||
import { resolveQaScenarioPackScenarioIds } from "./scenario-packs.js";
|
||||
import { runQaSuiteFromRuntime } from "./suite-launch.runtime.js";
|
||||
import { runQaFlowSuiteFromRuntime, runQaSuite } from "./suite-launch.runtime.js";
|
||||
import { readQaSuiteFailedOrSkippedScenarioCountFromFile } from "./suite-summary.js";
|
||||
import {
|
||||
buildTokenEfficiencyReport,
|
||||
@@ -251,6 +251,26 @@ function resolveQaRuntimeParityTierScenarioIds(params: {
|
||||
return uniqueStrings([...params.scenarioIds, ...matchingScenarioIds]);
|
||||
}
|
||||
|
||||
function rejectNonFlowScenarioIdsForMultipass(scenarioIds: readonly string[]) {
|
||||
if (scenarioIds.length === 0) {
|
||||
return;
|
||||
}
|
||||
const scenarioById = new Map(
|
||||
readQaScenarioPack().scenarios.map((scenario) => [scenario.id, scenario]),
|
||||
);
|
||||
const nonFlowScenarios = scenarioIds.flatMap((scenarioId) => {
|
||||
const scenario = scenarioById.get(scenarioId);
|
||||
return scenario && scenario.execution.kind !== "flow"
|
||||
? [`${scenario.id} (${scenario.execution.kind})`]
|
||||
: [];
|
||||
});
|
||||
if (nonFlowScenarios.length > 0) {
|
||||
throw new Error(
|
||||
`--runner multipass requires execution.kind: flow scenarios; unsupported scenario(s): ${nonFlowScenarios.join(", ")}`,
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
function isQaSuiteInfraRetryableError(error: unknown) {
|
||||
if (error instanceof QaSuiteArtifactError || error instanceof QaSuiteInfraError) {
|
||||
return true;
|
||||
@@ -276,28 +296,13 @@ function hasQaSuiteRetryableNetworkCode(error: unknown) {
|
||||
return false;
|
||||
}
|
||||
|
||||
async function assertQaSuiteArtifacts(result: { reportPath: string; summaryPath: string }) {
|
||||
try {
|
||||
await fs.access(result.reportPath);
|
||||
} catch (error) {
|
||||
throw new QaSuiteArtifactError(
|
||||
"report_missing",
|
||||
`QA suite did not produce report artifact at ${result.reportPath}: ${formatErrorMessage(error)}`,
|
||||
{ cause: error },
|
||||
);
|
||||
}
|
||||
await readQaSuiteFailedOrSkippedScenarioCountFromFile(result.summaryPath);
|
||||
}
|
||||
|
||||
async function runQaSuiteFromRuntimeWithInfraRetry(
|
||||
params: Parameters<typeof runQaSuiteFromRuntime>[0],
|
||||
async function runQaSuiteWithInfraRetry<Result>(
|
||||
run: () => Promise<Result>,
|
||||
maxRetries = QA_SUITE_INFRA_RETRY_LIMIT,
|
||||
) {
|
||||
for (let attempt = 0; attempt <= maxRetries; attempt += 1) {
|
||||
try {
|
||||
const result = await runQaSuiteFromRuntime(params);
|
||||
await assertQaSuiteArtifacts(result);
|
||||
return result;
|
||||
return await run();
|
||||
} catch (error) {
|
||||
const retryable = isQaSuiteInfraRetryableError(error);
|
||||
if (!retryable || attempt >= maxRetries) {
|
||||
@@ -326,16 +331,18 @@ async function runQaParityPreflight(params: {
|
||||
"preflight",
|
||||
`suite-${Date.now().toString(36)}`,
|
||||
);
|
||||
const result = await runQaSuiteFromRuntimeWithInfraRetry({
|
||||
repoRoot: params.repoRoot,
|
||||
outputDir,
|
||||
transportId: params.transportId,
|
||||
providerMode: params.providerMode,
|
||||
primaryModel: params.primaryModel,
|
||||
alternateModel: params.alternateModel,
|
||||
scenarioIds: ["approval-turn-tool-followthrough"],
|
||||
concurrency: 1,
|
||||
});
|
||||
const result = await runQaSuiteWithInfraRetry(() =>
|
||||
runQaFlowSuiteFromRuntime({
|
||||
repoRoot: params.repoRoot,
|
||||
outputDir,
|
||||
transportId: params.transportId,
|
||||
providerMode: params.providerMode,
|
||||
primaryModel: params.primaryModel,
|
||||
alternateModel: params.alternateModel,
|
||||
scenarioIds: ["approval-turn-tool-followthrough"],
|
||||
concurrency: 1,
|
||||
}),
|
||||
);
|
||||
process.stdout.write(`QA parity preflight watch: ${result.watchUrl}\n`);
|
||||
process.stdout.write(`QA parity preflight report: ${result.reportPath}\n`);
|
||||
process.stdout.write(`QA parity preflight summary: ${result.summaryPath}\n`);
|
||||
@@ -605,14 +612,14 @@ export async function runQaSuiteCommand(opts: {
|
||||
runtimeParityTiers,
|
||||
});
|
||||
const allowFailures = opts.allowFailures === true;
|
||||
if (runner !== "host" && runner !== "multipass") {
|
||||
throw new Error(`--runner must be one of host or multipass, got "${opts.runner}".`);
|
||||
}
|
||||
const providerMode = normalizeQaProviderMode(opts.providerMode);
|
||||
const runtimePair = parseQaRuntimePair(opts.runtimePair);
|
||||
const claudeCliAuthMode = parseQaCliBackendAuthMode(opts.cliAuthMode);
|
||||
const primaryModel = normalizeQaOptionalModelRef(opts.primaryModel);
|
||||
const alternateModel = normalizeQaOptionalModelRef(opts.alternateModel);
|
||||
if (runner !== "host" && runner !== "multipass") {
|
||||
throw new Error(`--runner must be one of host or multipass, got "${opts.runner}".`);
|
||||
}
|
||||
if (opts.preflight === true && runner !== "host") {
|
||||
throw new Error("--preflight requires --runner host.");
|
||||
}
|
||||
@@ -629,12 +636,13 @@ export async function runQaSuiteCommand(opts: {
|
||||
throw new Error("--cli-auth-mode requires --runner host.");
|
||||
}
|
||||
if (runner === "multipass") {
|
||||
rejectNonFlowScenarioIdsForMultipass(scenarioIds);
|
||||
const thinkingDefault = parseQaThinkingLevel("--thinking", opts.thinking);
|
||||
const result = await runQaMultipass({
|
||||
repoRoot,
|
||||
outputDir: resolveRepoRelativeOutputDir(repoRoot, opts.outputDir),
|
||||
transportId,
|
||||
providerMode,
|
||||
...(opts.providerMode !== undefined ? { providerMode } : {}),
|
||||
primaryModel,
|
||||
alternateModel,
|
||||
fastMode: opts.fastMode,
|
||||
@@ -677,31 +685,49 @@ export async function runQaSuiteCommand(opts: {
|
||||
return;
|
||||
}
|
||||
const thinkingDefault = parseQaThinkingLevel("--thinking", opts.thinking);
|
||||
const result = await runQaSuiteFromRuntimeWithInfraRetry({
|
||||
repoRoot,
|
||||
outputDir: resolveRepoRelativeOutputDir(repoRoot, opts.outputDir),
|
||||
transportId,
|
||||
providerMode,
|
||||
primaryModel,
|
||||
alternateModel,
|
||||
fastMode: opts.fastMode,
|
||||
...(thinkingDefault ? { thinkingDefault } : {}),
|
||||
...(claudeCliAuthMode ? { claudeCliAuthMode } : {}),
|
||||
scenarioIds,
|
||||
...(opts.enabledPluginIds !== undefined ? { enabledPluginIds: opts.enabledPluginIds } : {}),
|
||||
...(opts.concurrency !== undefined
|
||||
? { concurrency: parseQaPositiveIntegerOption("--concurrency", opts.concurrency) }
|
||||
: {}),
|
||||
...(runtimePair ? { runtimePair } : {}),
|
||||
});
|
||||
process.stdout.write(`QA suite watch: ${result.watchUrl}\n`);
|
||||
process.stdout.write(`QA suite report: ${result.reportPath}\n`);
|
||||
process.stdout.write(`QA suite summary: ${result.summaryPath}\n`);
|
||||
const blockingScenarioCount = await readQaSuiteFailedOrSkippedScenarioCountFromFile(
|
||||
result.summaryPath,
|
||||
const runtimeResult = await runQaSuiteWithInfraRetry(() =>
|
||||
runQaSuite({
|
||||
repoRoot,
|
||||
outputDir: resolveRepoRelativeOutputDir(repoRoot, opts.outputDir),
|
||||
transportId,
|
||||
...(opts.providerMode !== undefined ? { providerMode } : {}),
|
||||
primaryModel,
|
||||
alternateModel,
|
||||
fastMode: opts.fastMode,
|
||||
...(thinkingDefault ? { thinkingDefault } : {}),
|
||||
...(claudeCliAuthMode ? { claudeCliAuthMode } : {}),
|
||||
scenarioIds,
|
||||
...(opts.enabledPluginIds !== undefined ? { enabledPluginIds: opts.enabledPluginIds } : {}),
|
||||
...(opts.concurrency !== undefined
|
||||
? { concurrency: parseQaPositiveIntegerOption("--concurrency", opts.concurrency) }
|
||||
: {}),
|
||||
...(runtimePair ? { runtimePair } : {}),
|
||||
}),
|
||||
);
|
||||
if (!allowFailures && blockingScenarioCount > 0) {
|
||||
process.exitCode = 1;
|
||||
switch (runtimeResult.executionKind) {
|
||||
case "vitest":
|
||||
case "playwright": {
|
||||
const result = runtimeResult.result;
|
||||
process.stdout.write(`QA suite report: ${result.reportPath}\n`);
|
||||
process.stdout.write(`QA suite evidence: ${result.evidencePath}\n`);
|
||||
if (!allowFailures && result.results.some((scenario) => scenario.status !== "pass")) {
|
||||
process.exitCode = 1;
|
||||
}
|
||||
return;
|
||||
}
|
||||
case "flow": {
|
||||
const result = runtimeResult.result;
|
||||
process.stdout.write(`QA suite watch: ${result.watchUrl}\n`);
|
||||
process.stdout.write(`QA suite report: ${result.reportPath}\n`);
|
||||
process.stdout.write(`QA suite evidence: ${result.evidencePath}\n`);
|
||||
process.stdout.write(`QA suite summary: ${result.summaryPath}\n`);
|
||||
const blockingScenarioCount = await readQaSuiteFailedOrSkippedScenarioCountFromFile(
|
||||
result.summaryPath,
|
||||
);
|
||||
if (!allowFailures && blockingScenarioCount > 0) {
|
||||
process.exitCode = 1;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -662,6 +662,7 @@ describe("qa cli registration", () => {
|
||||
|
||||
const options = requireQaSuiteOptions();
|
||||
expect(options.allowFailures).toBe(true);
|
||||
expect(options.providerMode).toBeUndefined();
|
||||
});
|
||||
|
||||
it("forwards --pack for suite runs", async () => {
|
||||
|
||||
@@ -46,7 +46,7 @@ async function runQaSelfCheck(opts: { repoRoot?: string; output?: string }) {
|
||||
await runtime.runQaLabSelfCheckCommand(opts);
|
||||
}
|
||||
|
||||
async function runQaSuite(opts: {
|
||||
async function runQaSuiteCliCommand(opts: {
|
||||
repoRoot?: string;
|
||||
outputDir?: string;
|
||||
transportId?: string;
|
||||
@@ -300,7 +300,7 @@ export function registerQaLabCli(program: Command) {
|
||||
.option("--output-dir <path>", "Suite artifact directory")
|
||||
.option("--runner <kind>", "Execution runner: host or multipass", "host")
|
||||
.option("--transport <id>", "QA transport id", "qa-channel")
|
||||
.option("--provider-mode <mode>", formatQaProviderModeHelp(), DEFAULT_QA_LIVE_PROVIDER_MODE)
|
||||
.option("--provider-mode <mode>", formatQaProviderModeHelp())
|
||||
.option("--model <ref>", "Primary provider/model ref")
|
||||
.option("--alt-model <ref>", "Alternate provider/model ref")
|
||||
.option(
|
||||
@@ -372,7 +372,7 @@ export function registerQaLabCli(program: Command) {
|
||||
runtimePair?: string;
|
||||
runtimeParityTier?: string[];
|
||||
}) => {
|
||||
await runQaSuite({
|
||||
await runQaSuiteCliCommand({
|
||||
repoRoot: opts.repoRoot,
|
||||
outputDir: opts.outputDir,
|
||||
transportId: opts.transport,
|
||||
|
||||
@@ -1,6 +1,11 @@
|
||||
// Qa Lab tests cover coverage report plugin behavior.
|
||||
import { describe, expect, it } from "vitest";
|
||||
import { buildQaCoverageInventory, renderQaCoverageMarkdownReport } from "./coverage-report.js";
|
||||
import {
|
||||
buildQaCoverageInventory,
|
||||
findQaScenarioMatches,
|
||||
renderQaCoverageMarkdownReport,
|
||||
renderQaScenarioMatchesMarkdownReport,
|
||||
} from "./coverage-report.js";
|
||||
import { readQaScenarioPack } from "./scenario-catalog.js";
|
||||
import { buildQaScorecardTaxonomyReport, parseQaScorecardTaxonomy } from "./scorecard-taxonomy.js";
|
||||
|
||||
@@ -138,6 +143,33 @@ describe("qa coverage report", () => {
|
||||
expect(report).toContain("agents.subagents");
|
||||
});
|
||||
|
||||
it("renders Playwright matches as qa suite targets", () => {
|
||||
const matches = findQaScenarioMatches(readQaScenarioPack().scenarios, "chat-flow.e2e");
|
||||
const report = renderQaScenarioMatchesMarkdownReport({
|
||||
query: "chat-flow.e2e",
|
||||
matches,
|
||||
});
|
||||
|
||||
expect(report).toContain(
|
||||
"- Suite command: `pnpm openclaw qa suite --scenario control-ui-chat-flow-playwright`",
|
||||
);
|
||||
expect(report).toContain(" - execution: playwright ui/src/ui/e2e/chat-flow.e2e.test.ts");
|
||||
});
|
||||
|
||||
it("splits qa suite targets when matches mix execution kinds", () => {
|
||||
const matches = findQaScenarioMatches(readQaScenarioPack().scenarios, "control-ui");
|
||||
const report = renderQaScenarioMatchesMarkdownReport({
|
||||
query: "control-ui",
|
||||
matches,
|
||||
});
|
||||
|
||||
expect(report).toContain("- Suite commands:");
|
||||
expect(report).toContain(" - flow: `pnpm openclaw qa suite --scenario");
|
||||
expect(report).toContain(
|
||||
" - playwright: `pnpm openclaw qa suite --scenario control-ui-chat-flow-playwright`",
|
||||
);
|
||||
});
|
||||
|
||||
it("reports taxonomy mapping gaps as scorecard signals", () => {
|
||||
const taxonomy = parseQaScorecardTaxonomy({
|
||||
version: 1,
|
||||
|
||||
@@ -23,6 +23,8 @@ type QaScenarioSearchMatch = QaCoverageScenarioSummary & {
|
||||
coverageIds: string[];
|
||||
docsRefs: string[];
|
||||
codeRefs: string[];
|
||||
executionKind: QaSeedScenarioWithSource["execution"]["kind"];
|
||||
executionPath?: string;
|
||||
runtimeParityTier?: string;
|
||||
requiredProviderMode?: string;
|
||||
requiredProvider?: string;
|
||||
@@ -138,6 +140,8 @@ function summarizeScenarioSearchMatch(scenario: QaSeedScenarioWithSource): QaSce
|
||||
].toSorted((left, right) => left.localeCompare(right)),
|
||||
docsRefs: [...(scenario.docsRefs ?? [])],
|
||||
codeRefs: [...(scenario.codeRefs ?? [])],
|
||||
executionKind: scenario.execution.kind,
|
||||
...(scenario.execution.kind !== "flow" ? { executionPath: scenario.execution.path } : {}),
|
||||
runtimeParityTier: scenario.runtimeParityTier,
|
||||
requiredProviderMode: stringifyConfigValue(config.requiredProviderMode),
|
||||
requiredProvider: stringifyConfigValue(config.requiredProvider),
|
||||
@@ -444,11 +448,31 @@ function formatOptionalScenarioMetadata(match: QaScenarioSearchMatch) {
|
||||
return metadata.length > 0 ? metadata.join("; ") : "none";
|
||||
}
|
||||
|
||||
function formatSuiteCommand(matches: readonly QaScenarioSearchMatch[]) {
|
||||
const scenarioArgs = matches.map((match) => `--scenario ${match.id}`).join(" ");
|
||||
return `pnpm openclaw qa suite ${scenarioArgs}`;
|
||||
}
|
||||
|
||||
function scenarioMatchCommandGroups(matches: readonly QaScenarioSearchMatch[]) {
|
||||
const groups = new Map<QaScenarioSearchMatch["executionKind"], QaScenarioSearchMatch[]>();
|
||||
for (const match of matches) {
|
||||
const existing = groups.get(match.executionKind) ?? [];
|
||||
existing.push(match);
|
||||
groups.set(match.executionKind, existing);
|
||||
}
|
||||
const executionOrder: QaScenarioSearchMatch["executionKind"][] = ["flow", "vitest", "playwright"];
|
||||
return executionOrder
|
||||
.map((executionKind) => ({
|
||||
executionKind,
|
||||
matches: groups.get(executionKind) ?? [],
|
||||
}))
|
||||
.filter((group) => group.matches.length > 0);
|
||||
}
|
||||
|
||||
export function renderQaScenarioMatchesMarkdownReport(params: {
|
||||
query: string;
|
||||
matches: readonly QaScenarioSearchMatch[];
|
||||
}) {
|
||||
const scenarioArgs = params.matches.map((match) => `--scenario ${match.id}`).join(" ");
|
||||
const lines = [
|
||||
"# QA Scenario Matches",
|
||||
"",
|
||||
@@ -456,8 +480,14 @@ export function renderQaScenarioMatchesMarkdownReport(params: {
|
||||
`- Matches: ${params.matches.length}`,
|
||||
];
|
||||
|
||||
if (scenarioArgs) {
|
||||
lines.push(`- Suite command: \`pnpm openclaw qa suite ${scenarioArgs}\``);
|
||||
const commandGroups = scenarioMatchCommandGroups(params.matches);
|
||||
if (commandGroups.length === 1) {
|
||||
lines.push(`- Suite command: \`${formatSuiteCommand(commandGroups[0].matches)}\``);
|
||||
} else if (commandGroups.length > 1) {
|
||||
lines.push("- Suite commands:");
|
||||
for (const group of commandGroups) {
|
||||
lines.push(` - ${group.executionKind}: \`${formatSuiteCommand(group.matches)}\``);
|
||||
}
|
||||
}
|
||||
lines.push("");
|
||||
|
||||
@@ -470,6 +500,11 @@ export function renderQaScenarioMatchesMarkdownReport(params: {
|
||||
lines.push(`- ${match.id}: ${match.title}`);
|
||||
lines.push(` - source: ${match.sourcePath}`);
|
||||
lines.push(` - surface: ${match.surfaces.join(", ")}`);
|
||||
lines.push(
|
||||
match.executionKind === "flow"
|
||||
? " - execution: flow (qa-flow block)"
|
||||
: ` - execution: ${match.executionKind} ${match.executionPath ?? "missing"}`,
|
||||
);
|
||||
lines.push(` - coverage: ${match.coverageIds.join(", ") || "none"}`);
|
||||
lines.push(` - live requirements: ${formatOptionalScenarioMetadata(match)}`);
|
||||
if (match.codeRefs.length > 0) {
|
||||
|
||||
@@ -1,6 +1,8 @@
|
||||
// Qa Lab plugin module defines shared suite errors.
|
||||
export type QaSuiteArtifactErrorCode =
|
||||
| "evidence_missing"
|
||||
| "report_missing"
|
||||
| "summary_missing"
|
||||
| "summary_read_failed"
|
||||
| "summary_parse_failed"
|
||||
| "summary_failure_count_missing"
|
||||
|
||||
@@ -242,7 +242,7 @@ describe("evidence summary", () => {
|
||||
id: "runtime.agent-runner-boundary",
|
||||
title: "Agent runner boundary integration tests",
|
||||
sourcePath: "src/agents/agent-runner.e2e.test.ts",
|
||||
coverageIds: ["runtime.agent-runner", "runtime.delivery"],
|
||||
primaryCoverageIds: ["runtime.agent-runner", "runtime.delivery"],
|
||||
surfaceIds: ["agent-runtime-and-provider-execution"],
|
||||
categoryIds: ["agent-runtime-and-provider-execution.agent-turn-execution"],
|
||||
codeRefs: ["src/agents/agent-runner.ts"],
|
||||
@@ -332,7 +332,7 @@ describe("evidence summary", () => {
|
||||
id: "control-ui.browser-run",
|
||||
title: "Control UI browser workflow",
|
||||
sourcePath: "ui/control-ui.e2e.test.ts",
|
||||
coverageIds: ["control-ui.browser"],
|
||||
primaryCoverageIds: ["control-ui.browser"],
|
||||
surfaceIds: ["browser-control-ui-and-webchat"],
|
||||
categoryIds: ["browser-control-ui-and-webchat.browser-ui"],
|
||||
docsRefs: ["docs/concepts/qa-e2e-automation.md"],
|
||||
|
||||
@@ -210,7 +210,8 @@ type QaEvidenceTestTargetInput = {
|
||||
id: string;
|
||||
title: string;
|
||||
sourcePath: string;
|
||||
coverageIds: readonly string[];
|
||||
primaryCoverageIds?: readonly string[];
|
||||
secondaryCoverageIds?: readonly string[];
|
||||
surfaceIds: readonly string[];
|
||||
categoryIds: readonly string[];
|
||||
docsRefs?: readonly string[];
|
||||
@@ -578,7 +579,8 @@ function buildTestRunnerEvidenceSummary(
|
||||
mapping: {
|
||||
profile,
|
||||
coverage: buildQaEvidenceCoverage({
|
||||
primaryIds: target?.coverageIds ?? [],
|
||||
primaryIds: target?.primaryCoverageIds ?? [],
|
||||
secondaryIds: target?.secondaryCoverageIds ?? [],
|
||||
surfaceIds: target?.surfaceIds ?? [],
|
||||
categoryIds: target?.categoryIds ?? [],
|
||||
}),
|
||||
|
||||
@@ -315,7 +315,7 @@ describe("qa-lab server", () => {
|
||||
controlUiUrl: string | null;
|
||||
controlUiEmbeddedUrl: string | null;
|
||||
kickoffTask: string;
|
||||
scenarios: Array<{ id: string; title: string }>;
|
||||
scenarios: Array<{ id: string; title: string; execution?: { kind?: string } }>;
|
||||
defaults: { conversationId: string; senderId: string };
|
||||
runner: { status: string; selection: { providerMode: string; scenarioIds: string[] } };
|
||||
};
|
||||
@@ -328,7 +328,12 @@ describe("qa-lab server", () => {
|
||||
expect(bootstrap.scenarios.map((scenario) => scenario.id)).toContain("dm-chat-baseline");
|
||||
expect(bootstrap.runner.status).toBe("idle");
|
||||
expect(bootstrap.runner.selection.providerMode).toBe("live-frontier");
|
||||
expect(bootstrap.runner.selection.scenarioIds).toHaveLength(bootstrap.scenarios.length);
|
||||
const flowScenarioIds = bootstrap.scenarios
|
||||
.filter(
|
||||
(scenario) => scenario.execution?.kind === undefined || scenario.execution.kind === "flow",
|
||||
)
|
||||
.map((scenario) => scenario.id);
|
||||
expect(bootstrap.runner.selection.scenarioIds).toEqual(flowScenarioIds);
|
||||
|
||||
const startupStatus = (await (
|
||||
await fetchWithRetry(`${lab.baseUrl}/api/capture/startup-status`)
|
||||
|
||||
@@ -548,8 +548,8 @@ export async function startQaLabServer(
|
||||
};
|
||||
activeSuiteRun = (async () => {
|
||||
try {
|
||||
const { runQaSuite } = await import("./suite.js");
|
||||
const result = await runQaSuite({
|
||||
const { runQaFlowSuite } = await import("./suite.js");
|
||||
const result = await runQaFlowSuite({
|
||||
lab: labHandle ?? undefined,
|
||||
startLab: startQaLabServer,
|
||||
outputDir: createQaRunOutputDir(repoRoot),
|
||||
@@ -565,6 +565,7 @@ export async function startQaLabServer(
|
||||
finishedAt: new Date().toISOString(),
|
||||
artifacts: {
|
||||
outputDir: result.outputDir,
|
||||
evidencePath: result.evidencePath,
|
||||
reportPath: result.reportPath,
|
||||
summaryPath: result.summaryPath,
|
||||
watchUrl: result.watchUrl,
|
||||
|
||||
@@ -12,6 +12,7 @@ import type { QaProviderMode } from "./model-selection.js";
|
||||
import { resolveQaForwardedLiveEnv, resolveQaLiveProviderConfigPath } from "./providers/env.js";
|
||||
import { DEFAULT_QA_LIVE_PROVIDER_MODE, getQaProvider } from "./providers/index.js";
|
||||
import type { RuntimeId } from "./runtime-parity.js";
|
||||
import { shellQuote } from "./shell-quote.js";
|
||||
|
||||
const MULTIPASS_MOUNTED_REPO_PATH = "/workspace/openclaw-host";
|
||||
const MULTIPASS_GUEST_REPO_PATH = "/workspace/openclaw";
|
||||
@@ -107,10 +108,6 @@ type RenderGuestScriptOptions = {
|
||||
redactSecrets?: boolean;
|
||||
};
|
||||
|
||||
function shellQuote(value: string) {
|
||||
return `'${value.replaceAll("'", `'"'"'`)}'`;
|
||||
}
|
||||
|
||||
function createOutputStamp() {
|
||||
return new Date().toISOString().replaceAll(":", "").replaceAll(".", "").replace("T", "-");
|
||||
}
|
||||
|
||||
@@ -26,6 +26,7 @@ const scenarios = [
|
||||
surface: "dm",
|
||||
objective: "test DM",
|
||||
successCriteria: ["reply"],
|
||||
execution: { kind: "flow" as const },
|
||||
},
|
||||
{
|
||||
id: "thread-lifecycle",
|
||||
@@ -33,6 +34,18 @@ const scenarios = [
|
||||
surface: "thread",
|
||||
objective: "test thread",
|
||||
successCriteria: ["thread reply"],
|
||||
execution: { kind: "flow" as const },
|
||||
},
|
||||
{
|
||||
id: "control-ui-chat-flow-playwright",
|
||||
title: "Control UI Playwright",
|
||||
surface: "control-ui",
|
||||
objective: "test Control UI",
|
||||
successCriteria: ["playwright pass"],
|
||||
execution: {
|
||||
kind: "playwright" as const,
|
||||
path: "ui/src/ui/e2e/chat-flow.e2e.test.ts",
|
||||
},
|
||||
},
|
||||
];
|
||||
|
||||
@@ -44,7 +57,7 @@ describe("qa run config", () => {
|
||||
);
|
||||
});
|
||||
|
||||
it("creates a live-by-default selection that arms every scenario", () => {
|
||||
it("creates a live-by-default selection that arms flow scenarios", () => {
|
||||
expect(createDefaultQaRunSelection(scenarios)).toEqual({
|
||||
providerMode: "live-frontier",
|
||||
primaryModel: "openai/gpt-5.5",
|
||||
@@ -100,6 +113,17 @@ describe("qa run config", () => {
|
||||
).toEqual(["dm-chat-baseline", "thread-lifecycle"]);
|
||||
});
|
||||
|
||||
it("filters non-flow scenarios from lab runner selections", () => {
|
||||
expect(
|
||||
normalizeQaRunSelection(
|
||||
{
|
||||
scenarioIds: ["control-ui-chat-flow-playwright", "thread-lifecycle"],
|
||||
},
|
||||
scenarios,
|
||||
).scenarioIds,
|
||||
).toEqual(["thread-lifecycle"]);
|
||||
});
|
||||
|
||||
it("keeps idle snapshots on static defaults so startup does not inspect auth profiles", () => {
|
||||
defaultQaRuntimeModelForMode.mockReturnValue("openai/gpt-5.5");
|
||||
defaultQaRuntimeModelForMode.mockClear();
|
||||
|
||||
@@ -25,6 +25,7 @@ type QaLabRunSelection = {
|
||||
|
||||
type QaLabRunArtifacts = {
|
||||
outputDir: string;
|
||||
evidencePath: string;
|
||||
reportPath: string;
|
||||
summaryPath: string;
|
||||
watchUrl: string;
|
||||
@@ -49,6 +50,14 @@ function defaultStaticModelForMode(mode: QaProviderMode, alternate = false) {
|
||||
return defaultStaticQaModelForMode(mode, alternate ? { alternate: true } : undefined);
|
||||
}
|
||||
|
||||
function qaLabFlowScenarioIds(scenarios: QaSeedScenario[]) {
|
||||
return scenarios
|
||||
.filter(
|
||||
(scenario) => scenario.execution?.kind === undefined || scenario.execution.kind === "flow",
|
||||
)
|
||||
.map((scenario) => scenario.id);
|
||||
}
|
||||
|
||||
export function createDefaultQaRunSelection(
|
||||
scenarios: QaSeedScenario[],
|
||||
options?: { resolveDefaultModel?: QaDefaultModelResolver },
|
||||
@@ -60,7 +69,7 @@ export function createDefaultQaRunSelection(
|
||||
primaryModel: resolveDefaultModel(providerMode),
|
||||
alternateModel: resolveDefaultModel(providerMode, true),
|
||||
fastMode: true,
|
||||
scenarioIds: scenarios.map((scenario) => scenario.id),
|
||||
scenarioIds: qaLabFlowScenarioIds(scenarios),
|
||||
};
|
||||
}
|
||||
|
||||
@@ -81,14 +90,15 @@ function normalizeModel(input: unknown, fallback: string) {
|
||||
}
|
||||
|
||||
function normalizeScenarioIds(input: unknown, scenarios: QaSeedScenario[]) {
|
||||
const availableIds = new Set(scenarios.map((scenario) => scenario.id));
|
||||
const defaultScenarioIds = qaLabFlowScenarioIds(scenarios);
|
||||
const availableIds = new Set(defaultScenarioIds);
|
||||
const requestedIds = Array.isArray(input)
|
||||
? input
|
||||
.map((value) => (typeof value === "string" ? value.trim() : ""))
|
||||
.filter((value) => value.length > 0)
|
||||
: [];
|
||||
const selectedIds = uniqueStrings(requestedIds.filter((id) => availableIds.has(id)));
|
||||
return selectedIds.length > 0 ? selectedIds : scenarios.map((scenario) => scenario.id);
|
||||
return selectedIds.length > 0 ? selectedIds : defaultScenarioIds;
|
||||
}
|
||||
|
||||
export function normalizeQaRunSelection(
|
||||
|
||||
@@ -34,10 +34,12 @@ describe("qa scenario catalog", () => {
|
||||
pack.scenarios
|
||||
.filter((scenario) => scenario.execution?.kind !== "flow")
|
||||
.map((scenario) => scenario.id),
|
||||
).toStrictEqual([]);
|
||||
).toStrictEqual(["control-ui-chat-flow-playwright"]);
|
||||
expect(
|
||||
pack.scenarios.filter((scenario) => (scenario.execution.flow?.steps.length ?? 0) > 0),
|
||||
).not.toStrictEqual([]);
|
||||
pack.scenarios
|
||||
.filter((scenario) => scenario.execution.kind === "flow")
|
||||
.every((scenario) => (scenario.execution.flow?.steps.length ?? 0) > 0),
|
||||
).toBe(true);
|
||||
expect(
|
||||
pack.scenarios
|
||||
.filter((scenario) => !(scenario.coverage?.primary.length ?? 0))
|
||||
@@ -109,6 +111,18 @@ describe("qa scenario catalog", () => {
|
||||
expect(scenario.gatewayRuntime?.forwardHostHome).toBe(true);
|
||||
});
|
||||
|
||||
it("loads Playwright execution scenarios from markdown", () => {
|
||||
const scenario = readQaScenarioById("control-ui-chat-flow-playwright");
|
||||
|
||||
expect(scenario.execution.kind).toBe("playwright");
|
||||
if (scenario.execution.kind !== "playwright") {
|
||||
throw new Error("expected Playwright scenario execution");
|
||||
}
|
||||
expect(scenario.execution.path).toBe("ui/src/ui/e2e/chat-flow.e2e.test.ts");
|
||||
expect(scenario.execution.flow).toBeUndefined();
|
||||
expect(scenario.coverage?.primary).toContain("ui.control");
|
||||
});
|
||||
|
||||
it("loads runtime parity tier metadata for first-hour and soak lanes", () => {
|
||||
const firstHour = readQaScenarioById("runtime-first-hour-20-turn");
|
||||
const soak = readQaScenarioById("runtime-soak-100-turn");
|
||||
|
||||
@@ -3,6 +3,7 @@ import fs from "node:fs";
|
||||
import path from "node:path";
|
||||
import YAML from "yaml";
|
||||
import { z } from "zod";
|
||||
import { isRepoRootRelativeRef } from "./cli-paths.js";
|
||||
|
||||
export const DEFAULT_QA_AGENT_IDENTITY_MARKDOWN = `# Dev C-3PO
|
||||
|
||||
@@ -46,12 +47,39 @@ const qaScenarioConfigSchema = z.record(z.string(), z.unknown()).superRefine((co
|
||||
}
|
||||
});
|
||||
|
||||
const qaScenarioExecutionSchema = z.object({
|
||||
const qaScenarioRepoRefSchema = z
|
||||
.string()
|
||||
.trim()
|
||||
.min(1)
|
||||
.regex(/^[A-Za-z0-9._/-]+$/, {
|
||||
message: "repo refs must be repo-root relative paths",
|
||||
})
|
||||
.refine(isRepoRootRelativeRef, {
|
||||
message: "repo refs must not be absolute or contain parent-directory segments",
|
||||
});
|
||||
|
||||
const qaFlowScenarioExecutionSchema = z.object({
|
||||
kind: z.literal("flow").default("flow"),
|
||||
summary: z.string().trim().min(1).optional(),
|
||||
config: qaScenarioConfigSchema.optional(),
|
||||
});
|
||||
|
||||
const qaTestFileScenarioExecutionBaseSchema = z.object({
|
||||
summary: z.string().trim().min(1).optional(),
|
||||
path: qaScenarioRepoRefSchema,
|
||||
config: qaScenarioConfigSchema.optional(),
|
||||
});
|
||||
|
||||
const qaTestFileScenarioExecutionSchema = z.discriminatedUnion("kind", [
|
||||
qaTestFileScenarioExecutionBaseSchema.extend({ kind: z.literal("vitest") }),
|
||||
qaTestFileScenarioExecutionBaseSchema.extend({ kind: z.literal("playwright") }),
|
||||
]);
|
||||
|
||||
const qaScenarioExecutionSchema = z.union([
|
||||
qaFlowScenarioExecutionSchema,
|
||||
qaTestFileScenarioExecutionSchema,
|
||||
]);
|
||||
|
||||
const qaCoverageIdSchema = z
|
||||
.string()
|
||||
.trim()
|
||||
@@ -377,13 +405,14 @@ export function readQaScenarioPack(): QaScenarioPack {
|
||||
parsedScenario.execution ?? {},
|
||||
relativePath,
|
||||
);
|
||||
const flow = extractQaScenarioFlow(content, relativePath);
|
||||
const flow =
|
||||
execution.kind === "flow" ? extractQaScenarioFlow(content, relativePath) : undefined;
|
||||
return {
|
||||
...parsedScenario,
|
||||
sourcePath: relativePath,
|
||||
execution: {
|
||||
...execution,
|
||||
flow,
|
||||
...(flow ? { flow } : {}),
|
||||
},
|
||||
} satisfies QaSeedScenarioWithSource;
|
||||
})(),
|
||||
|
||||
@@ -3,6 +3,7 @@ import fs from "node:fs";
|
||||
import path from "node:path";
|
||||
import YAML from "yaml";
|
||||
import { z } from "zod";
|
||||
import { isRepoRootRelativeRef } from "./cli-paths.js";
|
||||
import type { QaSeedScenarioWithSource } from "./scenario-catalog.js";
|
||||
|
||||
export const QA_SCORECARD_TAXONOMY_PATH = "taxonomy-mappings.yaml";
|
||||
@@ -15,10 +16,6 @@ const qaScorecardIdSchema = z
|
||||
message: "scorecard and coverage ids must use lowercase dotted or dashed tokens",
|
||||
});
|
||||
|
||||
function isRepoRootRelativeRef(value: string) {
|
||||
return !path.isAbsolute(value) && value.split(/[\\/]+/u).every((part) => part !== "..");
|
||||
}
|
||||
|
||||
const qaScorecardRepoRefSchema = z
|
||||
.string()
|
||||
.trim()
|
||||
|
||||
4
extensions/qa-lab/src/shell-quote.ts
Normal file
4
extensions/qa-lab/src/shell-quote.ts
Normal file
@@ -0,0 +1,4 @@
|
||||
// POSIX shell quoting for generated QA command previews and guest scripts.
|
||||
export function shellQuote(value: string) {
|
||||
return `'${value.replaceAll("'", `'"'"'`)}'`;
|
||||
}
|
||||
154
extensions/qa-lab/src/suite-launch.runtime.test.ts
Normal file
154
extensions/qa-lab/src/suite-launch.runtime.test.ts
Normal file
@@ -0,0 +1,154 @@
|
||||
import fs from "node:fs/promises";
|
||||
import os from "node:os";
|
||||
import path from "node:path";
|
||||
import { afterEach, beforeEach, describe, expect, it, vi } from "vitest";
|
||||
|
||||
const { runQaFlowSuite, runQaTestFileScenarios } = vi.hoisted(() => ({
|
||||
runQaFlowSuite: vi.fn(),
|
||||
runQaTestFileScenarios: vi.fn(),
|
||||
}));
|
||||
|
||||
vi.mock("./suite.js", () => ({
|
||||
runQaFlowSuite,
|
||||
}));
|
||||
|
||||
vi.mock("./test-file-scenario-runner.js", async (importOriginal) => ({
|
||||
...(await importOriginal<typeof import("./test-file-scenario-runner.js")>()),
|
||||
runQaTestFileScenarios,
|
||||
}));
|
||||
|
||||
import { runQaSuite } from "./suite-launch.runtime.js";
|
||||
|
||||
const tempRoots: string[] = [];
|
||||
|
||||
async function makeTempRepo(prefix: string) {
|
||||
const repoRoot = await fs.mkdtemp(path.join(os.tmpdir(), prefix));
|
||||
tempRoots.push(repoRoot);
|
||||
return repoRoot;
|
||||
}
|
||||
|
||||
describe("qa suite runtime launcher", () => {
|
||||
beforeEach(() => {
|
||||
runQaFlowSuite.mockReset();
|
||||
runQaTestFileScenarios.mockReset();
|
||||
runQaFlowSuite.mockResolvedValue({
|
||||
outputDir: "/tmp/qa-flow",
|
||||
evidencePath: "/tmp/qa-flow/qa-evidence.json",
|
||||
reportPath: "/tmp/qa-flow/qa-suite-report.md",
|
||||
summaryPath: "/tmp/qa-flow/qa-suite-summary.json",
|
||||
report: "# QA Suite Report\n",
|
||||
scenarios: [],
|
||||
watchUrl: "http://127.0.0.1:43124",
|
||||
});
|
||||
runQaTestFileScenarios.mockResolvedValue({
|
||||
outputDir: "/tmp/qa-test-file",
|
||||
executionKind: "playwright",
|
||||
reportPath: "/tmp/qa-test-file/qa-playwright-report.md",
|
||||
evidencePath: "/tmp/qa-test-file/qa-evidence.json",
|
||||
results: [{ status: "pass" }],
|
||||
});
|
||||
});
|
||||
|
||||
afterEach(async () => {
|
||||
await Promise.all(
|
||||
tempRoots.splice(0).map((root) => fs.rm(root, { recursive: true, force: true })),
|
||||
);
|
||||
});
|
||||
|
||||
it("routes selected flow scenarios to the flow suite engine", async () => {
|
||||
const result = await runQaSuite({
|
||||
repoRoot: process.cwd(),
|
||||
providerMode: "mock-openai",
|
||||
scenarioIds: ["channel-chat-baseline"],
|
||||
});
|
||||
|
||||
expect(result).toMatchObject({
|
||||
executionKind: "flow",
|
||||
result: {
|
||||
summaryPath: "/tmp/qa-flow/qa-suite-summary.json",
|
||||
},
|
||||
});
|
||||
expect(runQaFlowSuite).toHaveBeenCalledTimes(1);
|
||||
expect(runQaFlowSuite).toHaveBeenCalledWith(
|
||||
expect.objectContaining({
|
||||
repoRoot: process.cwd(),
|
||||
providerMode: "mock-openai",
|
||||
scenarioIds: ["channel-chat-baseline"],
|
||||
}),
|
||||
);
|
||||
expect(runQaTestFileScenarios).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it("routes selected Playwright scenarios to the Playwright scenario runner", async () => {
|
||||
const repoRoot = await makeTempRepo("qa-suite-launch-");
|
||||
const result = await runQaSuite({
|
||||
repoRoot,
|
||||
outputDir: ".artifacts/qa-e2e/scenario-test",
|
||||
scenarioIds: ["control-ui-chat-flow-playwright"],
|
||||
});
|
||||
|
||||
expect(result).toMatchObject({
|
||||
executionKind: "playwright",
|
||||
result: {
|
||||
evidencePath: "/tmp/qa-test-file/qa-evidence.json",
|
||||
},
|
||||
});
|
||||
expect(runQaFlowSuite).not.toHaveBeenCalled();
|
||||
expect(runQaTestFileScenarios).toHaveBeenCalledTimes(1);
|
||||
const [call] = runQaTestFileScenarios.mock.calls[0] ?? [];
|
||||
expect(call).toMatchObject({
|
||||
repoRoot,
|
||||
outputDir: path.join(repoRoot, ".artifacts", "qa-e2e", "scenario-test"),
|
||||
providerMode: "mock-openai",
|
||||
primaryModel: "mock-openai/gpt-5.5",
|
||||
});
|
||||
expect(
|
||||
call.scenarios.map((scenario: { id: string; execution: { kind: string } }) => ({
|
||||
id: scenario.id,
|
||||
kind: scenario.execution.kind,
|
||||
})),
|
||||
).toEqual([{ id: "control-ui-chat-flow-playwright", kind: "playwright" }]);
|
||||
});
|
||||
|
||||
it("rejects mixed flow and Vitest/Playwright scenarios", async () => {
|
||||
await expect(
|
||||
runQaSuite({
|
||||
repoRoot: process.cwd(),
|
||||
scenarioIds: ["channel-chat-baseline", "control-ui-chat-flow-playwright"],
|
||||
}),
|
||||
).rejects.toThrow("qa suite cannot mix execution.kind: flow with Vitest/Playwright scenarios");
|
||||
|
||||
expect(runQaFlowSuite).not.toHaveBeenCalled();
|
||||
expect(runQaTestFileScenarios).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it("rejects runtime-pair requests for Vitest/Playwright scenarios", async () => {
|
||||
await expect(
|
||||
runQaSuite({
|
||||
repoRoot: process.cwd(),
|
||||
runtimePair: ["openclaw", "codex"],
|
||||
scenarioIds: ["control-ui-chat-flow-playwright"],
|
||||
}),
|
||||
).rejects.toThrow("--runtime-pair requires execution.kind: flow scenarios");
|
||||
|
||||
expect(runQaFlowSuite).not.toHaveBeenCalled();
|
||||
expect(runQaTestFileScenarios).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it("rejects repo-local symlink output directories before running Vitest/Playwright scenarios", async () => {
|
||||
const repoRoot = await makeTempRepo("qa-suite-symlink-root-");
|
||||
const outsideRoot = await makeTempRepo("qa-suite-symlink-outside-");
|
||||
await fs.symlink(outsideRoot, path.join(repoRoot, "artifacts-link"));
|
||||
|
||||
await expect(
|
||||
runQaSuite({
|
||||
repoRoot,
|
||||
outputDir: "artifacts-link/qa-out",
|
||||
scenarioIds: ["control-ui-chat-flow-playwright"],
|
||||
}),
|
||||
).rejects.toThrow("QA suite outputDir must not traverse symlinks");
|
||||
|
||||
expect(runQaFlowSuite).not.toHaveBeenCalled();
|
||||
expect(runQaTestFileScenarios).not.toHaveBeenCalled();
|
||||
});
|
||||
});
|
||||
@@ -1,15 +1,120 @@
|
||||
// Qa Lab plugin module implements suite launch behavior.
|
||||
import type { QaSuiteRunParams } from "./suite.js";
|
||||
import path from "node:path";
|
||||
import { DEFAULT_QA_PROVIDER_MODE } from "./providers/index.js";
|
||||
import { defaultQaModelForMode, normalizeQaProviderMode } from "./run-config.js";
|
||||
import { readQaBootstrapScenarioCatalog } from "./scenario-catalog.js";
|
||||
import { resolveQaSuiteOutputDir } from "./suite-planning.js";
|
||||
import type { QaSuiteResult, QaSuiteRunParams } from "./suite.js";
|
||||
import {
|
||||
isQaTestFileScenario,
|
||||
runQaTestFileScenarios,
|
||||
type QaTestFileExecutionKind,
|
||||
type QaTestFileScenario,
|
||||
type QaTestFileScenarioRunResult,
|
||||
} from "./test-file-scenario-runner.js";
|
||||
|
||||
export type QaSuiteRuntimeResult =
|
||||
| {
|
||||
executionKind: "flow";
|
||||
result: QaSuiteResult;
|
||||
}
|
||||
| {
|
||||
executionKind: QaTestFileExecutionKind;
|
||||
result: QaTestFileScenarioRunResult;
|
||||
};
|
||||
|
||||
async function loadQaLabServerRuntime() {
|
||||
const { startQaLabServer } = await import("./lab-server.js");
|
||||
return startQaLabServer;
|
||||
}
|
||||
|
||||
export async function runQaSuiteFromRuntime(...args: [QaSuiteRunParams?]) {
|
||||
const { runQaSuite } = await import("./suite.js");
|
||||
function resolveRequestedScenarios(params: {
|
||||
scenarioIds: readonly string[];
|
||||
scenarios: ReturnType<typeof readQaBootstrapScenarioCatalog>["scenarios"];
|
||||
}) {
|
||||
const scenarioById = new Map(params.scenarios.map((scenario) => [scenario.id, scenario]));
|
||||
return params.scenarioIds.map((scenarioId) => {
|
||||
const scenario = scenarioById.get(scenarioId);
|
||||
if (!scenario) {
|
||||
throw new Error(`unknown QA scenario id(s): ${scenarioId}`);
|
||||
}
|
||||
return scenario;
|
||||
});
|
||||
}
|
||||
|
||||
function resolveTestFileScenariosForSuiteDispatch(
|
||||
params: QaSuiteRunParams | undefined,
|
||||
): QaTestFileScenario[] | null {
|
||||
const scenarioIds = params?.scenarioIds ?? [];
|
||||
if (scenarioIds.length === 0) {
|
||||
return null;
|
||||
}
|
||||
const selectedScenarios = resolveRequestedScenarios({
|
||||
scenarioIds,
|
||||
scenarios: readQaBootstrapScenarioCatalog().scenarios,
|
||||
});
|
||||
const testFileScenarios = selectedScenarios.filter(isQaTestFileScenario);
|
||||
if (testFileScenarios.length === 0) {
|
||||
return null;
|
||||
}
|
||||
if (testFileScenarios.length !== selectedScenarios.length) {
|
||||
throw new Error("qa suite cannot mix execution.kind: flow with Vitest/Playwright scenarios.");
|
||||
}
|
||||
return testFileScenarios;
|
||||
}
|
||||
|
||||
async function runQaTestFileSuiteFromRuntime(params: {
|
||||
runParams: QaSuiteRunParams | undefined;
|
||||
scenarios: readonly QaTestFileScenario[];
|
||||
}): Promise<QaTestFileScenarioRunResult> {
|
||||
const runParams = params.runParams;
|
||||
if (runParams?.runtimePair) {
|
||||
throw new Error("--runtime-pair requires execution.kind: flow scenarios.");
|
||||
}
|
||||
if (runParams?.forcedRuntime) {
|
||||
throw new Error("forced runtime execution requires execution.kind: flow scenarios.");
|
||||
}
|
||||
if (runParams?.captureRuntimeParityCell) {
|
||||
throw new Error("runtime parity capture requires execution.kind: flow scenarios.");
|
||||
}
|
||||
const repoRoot = path.resolve(runParams?.repoRoot ?? process.cwd());
|
||||
const outputDir = await resolveQaSuiteOutputDir(repoRoot, runParams?.outputDir);
|
||||
const providerMode = normalizeQaProviderMode(runParams?.providerMode ?? DEFAULT_QA_PROVIDER_MODE);
|
||||
const primaryModel = runParams?.primaryModel?.trim() || defaultQaModelForMode(providerMode);
|
||||
return await runQaTestFileScenarios({
|
||||
repoRoot,
|
||||
outputDir,
|
||||
providerMode,
|
||||
primaryModel,
|
||||
scenarios: params.scenarios,
|
||||
});
|
||||
}
|
||||
|
||||
export async function runQaSuite(...args: [QaSuiteRunParams?]): Promise<QaSuiteRuntimeResult> {
|
||||
const runParams = args[0];
|
||||
const testFileScenarios = resolveTestFileScenariosForSuiteDispatch(runParams);
|
||||
if (testFileScenarios) {
|
||||
const result = await runQaTestFileSuiteFromRuntime({
|
||||
runParams,
|
||||
scenarios: testFileScenarios,
|
||||
});
|
||||
return {
|
||||
executionKind: result.executionKind,
|
||||
result,
|
||||
};
|
||||
}
|
||||
return {
|
||||
executionKind: "flow",
|
||||
result: await runQaFlowSuiteFromRuntime(...args),
|
||||
};
|
||||
}
|
||||
|
||||
export async function runQaFlowSuiteFromRuntime(
|
||||
...args: [QaSuiteRunParams?]
|
||||
): Promise<QaSuiteResult> {
|
||||
const { runQaFlowSuite } = await import("./suite.js");
|
||||
const params = args[0];
|
||||
return await runQaSuite({
|
||||
return await runQaFlowSuite({
|
||||
...params,
|
||||
startLab: params?.startLab ?? (await loadQaLabServerRuntime()),
|
||||
});
|
||||
|
||||
@@ -13,11 +13,21 @@ import {
|
||||
resolveQaSuiteWorkerStartStaggerMs,
|
||||
resolveQaSuiteOutputDir,
|
||||
scenarioRequiresControlUi,
|
||||
selectQaSuiteScenarios,
|
||||
selectQaFlowSuiteScenarios,
|
||||
shouldUseIsolatedQaSuiteScenarioWorkers,
|
||||
} from "./suite-planning.js";
|
||||
import { makeQaSuiteTestScenario } from "./suite-test-helpers.js";
|
||||
|
||||
function makePlaywrightQaSuiteTestScenario(id: string): ReturnType<typeof makeQaSuiteTestScenario> {
|
||||
return {
|
||||
...makeQaSuiteTestScenario(id),
|
||||
execution: {
|
||||
kind: "playwright",
|
||||
path: `ui/src/ui/e2e/${id}.e2e.test.ts`,
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
describe("qa suite planning helpers", () => {
|
||||
it("normalizes suite concurrency to a bounded integer", () => {
|
||||
const previous = process.env.OPENCLAW_QA_SUITE_CONCURRENCY;
|
||||
@@ -205,7 +215,7 @@ describe("qa suite planning helpers", () => {
|
||||
];
|
||||
|
||||
expect(
|
||||
selectQaSuiteScenarios({
|
||||
selectQaFlowSuiteScenarios({
|
||||
scenarios,
|
||||
scenarioIds: ["anthropic-only"],
|
||||
providerMode: "live-frontier",
|
||||
@@ -222,7 +232,7 @@ describe("qa suite planning helpers", () => {
|
||||
];
|
||||
|
||||
expect(
|
||||
selectQaSuiteScenarios({
|
||||
selectQaFlowSuiteScenarios({
|
||||
scenarios,
|
||||
scenarioIds: ["third", "first"],
|
||||
providerMode: "live-frontier",
|
||||
@@ -393,7 +403,7 @@ describe("qa suite planning helpers", () => {
|
||||
];
|
||||
|
||||
expect(
|
||||
selectQaSuiteScenarios({
|
||||
selectQaFlowSuiteScenarios({
|
||||
scenarios,
|
||||
providerMode: "live-frontier",
|
||||
primaryModel: "openai/gpt-5.5",
|
||||
@@ -401,7 +411,7 @@ describe("qa suite planning helpers", () => {
|
||||
).toEqual(["generic", "openai-only"]);
|
||||
|
||||
expect(
|
||||
selectQaSuiteScenarios({
|
||||
selectQaFlowSuiteScenarios({
|
||||
scenarios,
|
||||
providerMode: "live-frontier",
|
||||
primaryModel: "claude-cli/claude-sonnet-4-6",
|
||||
@@ -410,6 +420,39 @@ describe("qa suite planning helpers", () => {
|
||||
).toEqual(["generic", "claude-subscription"]);
|
||||
});
|
||||
|
||||
it("keeps Playwright scenarios out of implicit flow suite selections", () => {
|
||||
const scenarios = [
|
||||
makeQaSuiteTestScenario("flow"),
|
||||
makePlaywrightQaSuiteTestScenario("playwright"),
|
||||
];
|
||||
|
||||
expect(
|
||||
selectQaFlowSuiteScenarios({
|
||||
scenarios,
|
||||
providerMode: "mock-openai",
|
||||
primaryModel: "mock-openai/gpt-5.5",
|
||||
}).map((scenario) => scenario.id),
|
||||
).toEqual(["flow"]);
|
||||
});
|
||||
|
||||
it("rejects explicit Playwright scenarios in the flow suite selector", () => {
|
||||
const scenarios = [
|
||||
makeQaSuiteTestScenario("flow"),
|
||||
makePlaywrightQaSuiteTestScenario("playwright"),
|
||||
];
|
||||
|
||||
expect(() =>
|
||||
selectQaFlowSuiteScenarios({
|
||||
scenarios,
|
||||
scenarioIds: ["playwright"],
|
||||
providerMode: "mock-openai",
|
||||
primaryModel: "mock-openai/gpt-5.5",
|
||||
}),
|
||||
).toThrow(
|
||||
"flow execution requires execution.kind: flow; unsupported scenario(s): playwright (playwright)",
|
||||
);
|
||||
});
|
||||
|
||||
it("filters provider-mode-specific scenarios from implicit suite selections", () => {
|
||||
const scenarios = [
|
||||
makeQaSuiteTestScenario("generic"),
|
||||
@@ -422,7 +465,7 @@ describe("qa suite planning helpers", () => {
|
||||
];
|
||||
|
||||
expect(
|
||||
selectQaSuiteScenarios({
|
||||
selectQaFlowSuiteScenarios({
|
||||
scenarios,
|
||||
providerMode: "mock-openai",
|
||||
primaryModel: "mock-openai/gpt-5.5",
|
||||
@@ -430,7 +473,7 @@ describe("qa suite planning helpers", () => {
|
||||
).toEqual(["generic", "mock-only"]);
|
||||
|
||||
expect(
|
||||
selectQaSuiteScenarios({
|
||||
selectQaFlowSuiteScenarios({
|
||||
scenarios,
|
||||
providerMode: "live-frontier",
|
||||
primaryModel: "openai/gpt-5.5",
|
||||
@@ -447,7 +490,7 @@ describe("qa suite planning helpers", () => {
|
||||
];
|
||||
|
||||
expect(
|
||||
selectQaSuiteScenarios({
|
||||
selectQaFlowSuiteScenarios({
|
||||
scenarios,
|
||||
providerMode: "mock-openai",
|
||||
primaryModel: "mock-openai/gpt-5.5",
|
||||
@@ -455,7 +498,7 @@ describe("qa suite planning helpers", () => {
|
||||
).toEqual(["generic"]);
|
||||
|
||||
expect(
|
||||
selectQaSuiteScenarios({
|
||||
selectQaFlowSuiteScenarios({
|
||||
scenarios,
|
||||
scenarioIds: ["live-runtime"],
|
||||
providerMode: "mock-openai",
|
||||
|
||||
@@ -63,7 +63,7 @@ function scenarioMatchesLiveLane(params: {
|
||||
return true;
|
||||
}
|
||||
|
||||
function selectQaSuiteScenarios(params: {
|
||||
function selectQaFlowSuiteScenarios(params: {
|
||||
scenarios: ReturnType<typeof readQaBootstrapScenarioCatalog>["scenarios"];
|
||||
scenarioIds?: string[];
|
||||
providerMode: QaProviderMode;
|
||||
@@ -80,15 +80,31 @@ function selectQaSuiteScenarios(params: {
|
||||
if (missingScenarioIds.length > 0) {
|
||||
throw new Error(`unknown QA scenario id(s): ${missingScenarioIds.join(", ")}`);
|
||||
}
|
||||
return [...requestedScenarioIds].map((scenarioId) => scenarioById.get(scenarioId)!);
|
||||
const selectedScenarios = [...requestedScenarioIds].map(
|
||||
(scenarioId) => scenarioById.get(scenarioId)!,
|
||||
);
|
||||
const nonFlowScenarios = selectedScenarios.filter(
|
||||
(scenario) => scenario.execution.kind !== "flow",
|
||||
);
|
||||
if (nonFlowScenarios.length > 0) {
|
||||
const scenarioList = nonFlowScenarios
|
||||
.map((scenario) => `${scenario.id} (${scenario.execution.kind})`)
|
||||
.join(", ");
|
||||
throw new Error(
|
||||
`flow execution requires execution.kind: flow; unsupported scenario(s): ${scenarioList}`,
|
||||
);
|
||||
}
|
||||
return selectedScenarios;
|
||||
}
|
||||
return params.scenarios.filter((scenario) =>
|
||||
scenarioMatchesLiveLane({
|
||||
scenario,
|
||||
providerMode: params.providerMode,
|
||||
primaryModel: params.primaryModel,
|
||||
claudeCliAuthMode: params.claudeCliAuthMode,
|
||||
}),
|
||||
return params.scenarios.filter(
|
||||
(scenario) =>
|
||||
scenario.execution.kind === "flow" &&
|
||||
scenarioMatchesLiveLane({
|
||||
scenario,
|
||||
providerMode: params.providerMode,
|
||||
primaryModel: params.primaryModel,
|
||||
claudeCliAuthMode: params.claudeCliAuthMode,
|
||||
}),
|
||||
);
|
||||
}
|
||||
|
||||
@@ -266,7 +282,7 @@ export {
|
||||
resolveQaSuiteWorkerStartStaggerMs,
|
||||
resolveQaSuiteOutputDir,
|
||||
scenarioRequiresControlUi,
|
||||
selectQaSuiteScenarios,
|
||||
selectQaFlowSuiteScenarios,
|
||||
shouldUseIsolatedQaSuiteScenarioWorkers,
|
||||
splitModelRef,
|
||||
};
|
||||
|
||||
@@ -56,7 +56,7 @@ describe("buildQaSuiteSummaryJson", () => {
|
||||
});
|
||||
|
||||
it("treats an empty scenarioIds array as unspecified (no filter)", () => {
|
||||
// A CLI path that omits --scenario passes an empty array to runQaSuite.
|
||||
// A CLI path that omits --scenario passes an empty array to runQaFlowSuite.
|
||||
// The summary must encode that as null so downstream parity/report
|
||||
// tooling doesn't interpret a full run as an explicit empty selection.
|
||||
const json = buildQaSuiteSummaryJson({
|
||||
|
||||
@@ -1,8 +1,13 @@
|
||||
// Qa Lab tests cover suite plugin behavior.
|
||||
import fs from "node:fs/promises";
|
||||
import os from "node:os";
|
||||
import path from "node:path";
|
||||
import { afterEach, describe, expect, it, vi } from "vitest";
|
||||
import { QA_EVIDENCE_FILENAME, QA_EVIDENCE_SUMMARY_KIND } from "./evidence-summary.js";
|
||||
import type { QaLabServerHandle } from "./lab-server.types.js";
|
||||
import type { QaTransportAdapter } from "./qa-transport.js";
|
||||
import { makeQaSuiteTestScenario } from "./suite-test-helpers.js";
|
||||
import { qaSuiteProgressTesting, runQaSuite } from "./suite.js";
|
||||
import { qaSuiteProgressTesting, runQaFlowSuite } from "./suite.js";
|
||||
|
||||
const fetchWithSsrFGuardMock = vi.hoisted(() => vi.fn());
|
||||
|
||||
@@ -33,7 +38,7 @@ describe("qa suite", () => {
|
||||
const startLab = vi.fn();
|
||||
|
||||
await expect(
|
||||
runQaSuite({
|
||||
runQaFlowSuite({
|
||||
transportId: "qa-nope" as unknown as "qa-channel",
|
||||
startLab,
|
||||
}),
|
||||
@@ -222,6 +227,51 @@ describe("qa suite", () => {
|
||||
});
|
||||
});
|
||||
|
||||
it("writes standalone evidence while keeping suite summary evidence-free", async () => {
|
||||
const outputDir = await fs.mkdtemp(path.join(os.tmpdir(), "qa-suite-artifacts-"));
|
||||
try {
|
||||
const artifacts = await qaSuiteProgressTesting.writeQaSuiteArtifacts({
|
||||
outputDir,
|
||||
startedAt: new Date("2026-04-11T00:00:00.000Z"),
|
||||
finishedAt: new Date("2026-04-11T00:01:00.000Z"),
|
||||
scenarios: [{ name: "Baseline", status: "pass", steps: [] }],
|
||||
scenarioDefinitions: [
|
||||
{
|
||||
...makeQaSuiteTestScenario("baseline", {
|
||||
surface: "channel",
|
||||
}),
|
||||
coverage: {
|
||||
primary: ["channels.messages"],
|
||||
},
|
||||
},
|
||||
],
|
||||
transport: {
|
||||
id: "qa-channel",
|
||||
createReportNotes: () => [],
|
||||
} as unknown as QaTransportAdapter,
|
||||
providerMode: "mock-openai",
|
||||
primaryModel: "mock-openai/gpt-5.5",
|
||||
alternateModel: "mock-openai/gpt-5.5-alt",
|
||||
fastMode: true,
|
||||
concurrency: 1,
|
||||
});
|
||||
|
||||
expect(artifacts.evidencePath).toBe(path.join(outputDir, QA_EVIDENCE_FILENAME));
|
||||
const evidence = JSON.parse(await fs.readFile(artifacts.evidencePath, "utf8")) as {
|
||||
kind?: string;
|
||||
entries?: unknown[];
|
||||
};
|
||||
expect(evidence.kind).toBe(QA_EVIDENCE_SUMMARY_KIND);
|
||||
expect(evidence.entries).toHaveLength(1);
|
||||
const summary = JSON.parse(await fs.readFile(artifacts.summaryPath, "utf8")) as {
|
||||
evidence?: unknown;
|
||||
};
|
||||
expect(summary.evidence).toBeUndefined();
|
||||
} finally {
|
||||
await fs.rm(outputDir, { recursive: true, force: true });
|
||||
}
|
||||
});
|
||||
|
||||
it("arms gateway heap checkpoint env only when requested", () => {
|
||||
expect(
|
||||
qaSuiteProgressTesting.buildQaGatewayHeapCheckpointRuntimeEnvPatch({
|
||||
|
||||
@@ -12,7 +12,8 @@ import {
|
||||
type QaReportScenario,
|
||||
} from "openclaw/plugin-sdk/qa-runtime";
|
||||
import { fetchWithSsrFGuard } from "openclaw/plugin-sdk/ssrf-runtime";
|
||||
import { buildQaSuiteEvidenceSummary } from "./evidence-summary.js";
|
||||
import { QaSuiteArtifactError } from "./errors.js";
|
||||
import { buildQaSuiteEvidenceSummary, QA_EVIDENCE_FILENAME } from "./evidence-summary.js";
|
||||
import { startQaGatewayChild, type QaCliBackendAuthMode } from "./gateway-child.js";
|
||||
import type {
|
||||
QaLabLatestReport,
|
||||
@@ -60,7 +61,7 @@ import {
|
||||
resolveQaSuiteWorkerStartStaggerMs,
|
||||
resolveQaSuiteOutputDir,
|
||||
scenarioRequiresControlUi,
|
||||
selectQaSuiteScenarios,
|
||||
selectQaFlowSuiteScenarios,
|
||||
shouldUseIsolatedQaSuiteScenarioWorkers,
|
||||
splitModelRef,
|
||||
} from "./suite-planning.js";
|
||||
@@ -274,6 +275,7 @@ function liveTurnTimeoutMs(
|
||||
|
||||
export type QaSuiteResult = {
|
||||
outputDir: string;
|
||||
evidencePath: string;
|
||||
reportPath: string;
|
||||
summaryPath: string;
|
||||
report: string;
|
||||
@@ -691,7 +693,7 @@ async function runQaRuntimeParitySuite(params: {
|
||||
runtime,
|
||||
);
|
||||
const cellStartedAt = Date.now();
|
||||
const cellResult = await runQaSuite({
|
||||
const cellResult = await runQaFlowSuite({
|
||||
repoRoot: params.repoRoot,
|
||||
outputDir: cellOutputDir,
|
||||
providerMode: params.providerMode,
|
||||
@@ -784,7 +786,7 @@ async function runQaRuntimeParitySuite(params: {
|
||||
);
|
||||
|
||||
const finishedAt = new Date();
|
||||
const { report, reportPath, summaryPath } = await writeQaSuiteArtifacts({
|
||||
const { evidencePath, report, reportPath, summaryPath } = await writeQaSuiteArtifacts({
|
||||
outputDir: params.outputDir,
|
||||
startedAt: params.startedAt,
|
||||
finishedAt,
|
||||
@@ -816,6 +818,7 @@ async function runQaRuntimeParitySuite(params: {
|
||||
});
|
||||
return {
|
||||
outputDir: params.outputDir,
|
||||
evidencePath,
|
||||
reportPath,
|
||||
summaryPath,
|
||||
report,
|
||||
@@ -852,6 +855,7 @@ async function writeQaSuiteArtifacts(params: {
|
||||
}) {
|
||||
const reportPath = path.join(params.outputDir, "qa-suite-report.md");
|
||||
const summaryPath = path.join(params.outputDir, "qa-suite-summary.json");
|
||||
const evidencePath = path.join(params.outputDir, QA_EVIDENCE_FILENAME);
|
||||
const report = renderQaMarkdownReport({
|
||||
title: "OpenClaw QA Scenario Suite",
|
||||
startedAt: params.startedAt,
|
||||
@@ -882,12 +886,35 @@ async function writeQaSuiteArtifacts(params: {
|
||||
})
|
||||
: undefined;
|
||||
await fs.writeFile(reportPath, report, "utf8");
|
||||
if (evidence) {
|
||||
await fs.writeFile(evidencePath, `${JSON.stringify(evidence, null, 2)}\n`, "utf8");
|
||||
}
|
||||
await fs.writeFile(
|
||||
summaryPath,
|
||||
`${JSON.stringify(buildQaSuiteSummaryJson({ ...params, evidence }), null, 2)}\n`,
|
||||
`${JSON.stringify(buildQaSuiteSummaryJson(params), null, 2)}\n`,
|
||||
"utf8",
|
||||
);
|
||||
return { report, reportPath, summaryPath };
|
||||
await assertQaSuiteArtifactWritten("report", reportPath);
|
||||
await assertQaSuiteArtifactWritten("summary", summaryPath);
|
||||
if (evidence) {
|
||||
await assertQaSuiteArtifactWritten("evidence", evidencePath);
|
||||
}
|
||||
return { evidencePath, report, reportPath, summaryPath };
|
||||
}
|
||||
|
||||
async function assertQaSuiteArtifactWritten(
|
||||
kind: "evidence" | "report" | "summary",
|
||||
filePath: string,
|
||||
) {
|
||||
try {
|
||||
await fs.access(filePath);
|
||||
} catch (error) {
|
||||
throw new QaSuiteArtifactError(
|
||||
`${kind}_missing`,
|
||||
`QA suite did not produce ${kind} artifact at ${filePath}: ${formatErrorMessage(error)}`,
|
||||
{ cause: error },
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
function buildQaSuiteRuntimeMetrics(params: {
|
||||
@@ -1019,7 +1046,7 @@ async function captureGatewayHeapSnapshotCheckpoint(params: {
|
||||
};
|
||||
}
|
||||
|
||||
export async function runQaSuite(params?: QaSuiteRunParams): Promise<QaSuiteResult> {
|
||||
export async function runQaFlowSuite(params?: QaSuiteRunParams): Promise<QaSuiteResult> {
|
||||
const startedAt = new Date();
|
||||
const repoRoot = path.resolve(params?.repoRoot ?? process.cwd());
|
||||
const providerMode = normalizeQaProviderMode(
|
||||
@@ -1040,7 +1067,7 @@ export async function runQaSuite(params?: QaSuiteRunParams): Promise<QaSuiteResu
|
||||
: isQaFastModeEnabled({ primaryModel, alternateModel });
|
||||
const outputDir = await resolveQaSuiteOutputDir(repoRoot, params?.outputDir);
|
||||
const catalog = readQaBootstrapScenarioCatalog();
|
||||
const selectedScenarios = selectQaSuiteScenarios({
|
||||
const selectedScenarios = selectQaFlowSuiteScenarios({
|
||||
scenarios: catalog.scenarios,
|
||||
scenarioIds: params?.scenarioIds,
|
||||
providerMode,
|
||||
@@ -1197,7 +1224,7 @@ export async function runQaSuite(params?: QaSuiteRunParams): Promise<QaSuiteResu
|
||||
updateScenarioRun();
|
||||
try {
|
||||
const scenarioOutputDir = path.join(outputDir, "scenarios", scenario.id);
|
||||
const result: QaSuiteResult = await runQaSuite(
|
||||
const result: QaSuiteResult = await runQaFlowSuite(
|
||||
buildQaIsolatedScenarioWorkerParams({
|
||||
repoRoot,
|
||||
outputDir: scenarioOutputDir,
|
||||
@@ -1287,7 +1314,7 @@ export async function runQaSuite(params?: QaSuiteRunParams): Promise<QaSuiteResu
|
||||
finishedAt: finishedAt.toISOString(),
|
||||
scenarios: [...liveScenarioOutcomes],
|
||||
});
|
||||
const { report, reportPath, summaryPath } = await writeQaSuiteArtifacts({
|
||||
const { evidencePath, report, reportPath, summaryPath } = await writeQaSuiteArtifacts({
|
||||
outputDir,
|
||||
startedAt,
|
||||
finishedAt,
|
||||
@@ -1301,7 +1328,7 @@ export async function runQaSuite(params?: QaSuiteRunParams): Promise<QaSuiteResu
|
||||
concurrency,
|
||||
isolatedWorkers: true,
|
||||
// When the caller supplied an explicit non-empty --scenario filter,
|
||||
// record the executed (post-selectQaSuiteScenarios-normalized) ids
|
||||
// record the executed (post-selectQaFlowSuiteScenarios-normalized) ids
|
||||
// so the summary matches what actually ran. When the caller passed
|
||||
// nothing or an empty array ("no filter, full lane catalog"),
|
||||
// preserve the unfiltered = null semantic so the summary stays
|
||||
@@ -1322,6 +1349,7 @@ export async function runQaSuite(params?: QaSuiteRunParams): Promise<QaSuiteResu
|
||||
);
|
||||
return {
|
||||
outputDir,
|
||||
evidencePath,
|
||||
reportPath,
|
||||
summaryPath,
|
||||
report,
|
||||
@@ -1546,7 +1574,7 @@ export async function runQaSuite(params?: QaSuiteRunParams): Promise<QaSuiteResu
|
||||
finishedAt: finishedAt.toISOString(),
|
||||
scenarios: [...liveScenarioOutcomes],
|
||||
});
|
||||
const { report, reportPath, summaryPath } = await writeQaSuiteArtifacts({
|
||||
const { evidencePath, report, reportPath, summaryPath } = await writeQaSuiteArtifacts({
|
||||
outputDir,
|
||||
startedAt,
|
||||
finishedAt,
|
||||
@@ -1580,6 +1608,7 @@ export async function runQaSuite(params?: QaSuiteRunParams): Promise<QaSuiteResu
|
||||
|
||||
return {
|
||||
outputDir,
|
||||
evidencePath,
|
||||
reportPath,
|
||||
summaryPath,
|
||||
report,
|
||||
@@ -1626,4 +1655,5 @@ export const qaSuiteProgressTesting = {
|
||||
shouldRunQaSuiteWithIsolatedScenarioWorkers,
|
||||
shouldLogQaSuiteProgress,
|
||||
waitForQaLabReadyOrStopOwned,
|
||||
writeQaSuiteArtifacts,
|
||||
};
|
||||
|
||||
226
extensions/qa-lab/src/test-file-scenario-runner.test.ts
Normal file
226
extensions/qa-lab/src/test-file-scenario-runner.test.ts
Normal file
@@ -0,0 +1,226 @@
|
||||
import fs from "node:fs/promises";
|
||||
import os from "node:os";
|
||||
import path from "node:path";
|
||||
import { afterEach, describe, expect, it } from "vitest";
|
||||
import { validateQaEvidenceSummaryJson } from "./evidence-summary.js";
|
||||
import type { QaSeedScenarioWithSource } from "./scenario-catalog.js";
|
||||
import {
|
||||
runQaTestFileScenarios,
|
||||
type QaScenarioCommandExecution,
|
||||
} from "./test-file-scenario-runner.js";
|
||||
|
||||
const tempRoots: string[] = [];
|
||||
|
||||
function makeTestFileScenario(
|
||||
executionKind: "vitest" | "playwright",
|
||||
pathLocal: string,
|
||||
): QaSeedScenarioWithSource {
|
||||
return {
|
||||
id: `scenario-${executionKind}`,
|
||||
title: `${executionKind} scenario`,
|
||||
surface: executionKind === "playwright" ? "control-ui" : "qa-lab",
|
||||
category:
|
||||
executionKind === "playwright"
|
||||
? "browser-control-ui-and-webchat.browser-ui"
|
||||
: "qa-lab.coverage",
|
||||
coverage: {
|
||||
primary: [executionKind === "playwright" ? "ui.control" : "qa.coverage"],
|
||||
secondary: [executionKind === "playwright" ? "ui.streaming" : "qa.reporting"],
|
||||
},
|
||||
objective: `Exercise ${executionKind} scenario evidence.`,
|
||||
successCriteria: ["The scenario writes structured evidence."],
|
||||
docsRefs: ["docs/concepts/qa-e2e-automation.md"],
|
||||
codeRefs: [pathLocal],
|
||||
sourcePath: `qa/scenarios/ui/scenario-${executionKind}.md`,
|
||||
execution: {
|
||||
kind: executionKind,
|
||||
path: pathLocal,
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
async function makeTempRepo(prefix: string) {
|
||||
const repoRoot = await fs.mkdtemp(path.join(os.tmpdir(), prefix));
|
||||
tempRoots.push(repoRoot);
|
||||
await fs.mkdir(path.join(repoRoot, ".artifacts", "qa-e2e"), { recursive: true });
|
||||
return repoRoot;
|
||||
}
|
||||
|
||||
describe("qa test file scenario runner", () => {
|
||||
afterEach(async () => {
|
||||
await Promise.all(
|
||||
tempRoots.splice(0).map((root) => fs.rm(root, { recursive: true, force: true })),
|
||||
);
|
||||
});
|
||||
|
||||
it("runs Playwright scenarios with the repo UI e2e command and writes Playwright evidence", async () => {
|
||||
const repoRoot = await makeTempRepo("qa-playwright-scenario-");
|
||||
const commands: QaScenarioCommandExecution[] = [];
|
||||
const result = await runQaTestFileScenarios({
|
||||
repoRoot,
|
||||
outputDir: path.join(repoRoot, ".artifacts", "qa-e2e", "scenario-playwright"),
|
||||
providerMode: "mock-openai",
|
||||
primaryModel: "mock-openai/gpt-5.5",
|
||||
scenarios: [makeTestFileScenario("playwright", "ui/src/ui/e2e/chat-flow.e2e.test.ts")],
|
||||
runCommand: async (command) => {
|
||||
commands.push(command);
|
||||
return {
|
||||
exitCode: 0,
|
||||
stdout: "pass\n",
|
||||
stderr: "",
|
||||
};
|
||||
},
|
||||
env: {
|
||||
OPENCLAW_QA_REF: "scenario-ref",
|
||||
} as NodeJS.ProcessEnv,
|
||||
});
|
||||
|
||||
expect(result.executionKind).toBe("playwright");
|
||||
expect(commands.map((command) => command.args)).toEqual([
|
||||
["scripts/ensure-playwright-chromium.mjs"],
|
||||
[
|
||||
"scripts/run-vitest.mjs",
|
||||
"run",
|
||||
"--config",
|
||||
"test/vitest/vitest.ui-e2e.config.ts",
|
||||
"--configLoader",
|
||||
"runner",
|
||||
"ui/src/ui/e2e/chat-flow.e2e.test.ts",
|
||||
"--reporter=verbose",
|
||||
],
|
||||
]);
|
||||
const evidence = validateQaEvidenceSummaryJson(
|
||||
JSON.parse(await fs.readFile(result.evidencePath, "utf8")),
|
||||
);
|
||||
expect(evidence.schemaVersion).toBe(2);
|
||||
expect(evidence.entries).toHaveLength(1);
|
||||
expect(evidence.entries[0]).toMatchObject({
|
||||
test: {
|
||||
kind: "playwright-test",
|
||||
id: "scenario-playwright",
|
||||
source: {
|
||||
path: "ui/src/ui/e2e/chat-flow.e2e.test.ts",
|
||||
},
|
||||
},
|
||||
mapping: {
|
||||
coverage: [
|
||||
{
|
||||
id: "ui.control",
|
||||
role: "primary",
|
||||
surfaceIds: ["control-ui"],
|
||||
categoryIds: ["browser-control-ui-and-webchat.browser-ui"],
|
||||
},
|
||||
{
|
||||
id: "ui.streaming",
|
||||
role: "secondary",
|
||||
surfaceIds: ["control-ui"],
|
||||
categoryIds: [],
|
||||
},
|
||||
],
|
||||
refs: [
|
||||
{
|
||||
kind: "docs",
|
||||
path: "docs/concepts/qa-e2e-automation.md",
|
||||
},
|
||||
{
|
||||
kind: "code",
|
||||
path: "ui/src/ui/e2e/chat-flow.e2e.test.ts",
|
||||
},
|
||||
],
|
||||
},
|
||||
execution: {
|
||||
runner: "playwright",
|
||||
artifacts: [
|
||||
{
|
||||
kind: "report",
|
||||
path: ".artifacts/qa-e2e/scenario-playwright/qa-playwright-report.md",
|
||||
source: "playwright",
|
||||
},
|
||||
{
|
||||
kind: "log",
|
||||
path: ".artifacts/qa-e2e/scenario-playwright/scenario-playwright.log",
|
||||
source: "playwright",
|
||||
},
|
||||
],
|
||||
},
|
||||
result: {
|
||||
status: "pass",
|
||||
},
|
||||
});
|
||||
expect(await fs.readFile(result.reportPath, "utf8")).toContain("Evidence summary");
|
||||
});
|
||||
|
||||
it("runs Vitest scenarios with the declared test path and writes Vitest evidence", async () => {
|
||||
const repoRoot = await makeTempRepo("qa-vitest-scenario-");
|
||||
const commands: QaScenarioCommandExecution[] = [];
|
||||
const result = await runQaTestFileScenarios({
|
||||
repoRoot,
|
||||
outputDir: path.join(repoRoot, ".artifacts", "qa-e2e", "scenario-vitest"),
|
||||
providerMode: "mock-openai",
|
||||
primaryModel: "mock-openai/gpt-5.5",
|
||||
scenarios: [makeTestFileScenario("vitest", "extensions/qa-lab/src/coverage-report.test.ts")],
|
||||
runCommand: async (command) => {
|
||||
commands.push(command);
|
||||
return {
|
||||
exitCode: 1,
|
||||
stdout: "",
|
||||
stderr: "failed\n",
|
||||
};
|
||||
},
|
||||
});
|
||||
|
||||
expect(result.executionKind).toBe("vitest");
|
||||
expect(commands.map((command) => command.args)).toEqual([
|
||||
[
|
||||
"scripts/run-vitest.mjs",
|
||||
"extensions/qa-lab/src/coverage-report.test.ts",
|
||||
"--reporter=verbose",
|
||||
],
|
||||
]);
|
||||
const evidence = validateQaEvidenceSummaryJson(
|
||||
JSON.parse(await fs.readFile(result.evidencePath, "utf8")),
|
||||
);
|
||||
expect(evidence.entries[0]).toMatchObject({
|
||||
test: {
|
||||
kind: "vitest-test",
|
||||
id: "scenario-vitest",
|
||||
source: {
|
||||
path: "extensions/qa-lab/src/coverage-report.test.ts",
|
||||
},
|
||||
},
|
||||
mapping: {
|
||||
coverage: [
|
||||
{
|
||||
id: "qa.coverage",
|
||||
role: "primary",
|
||||
},
|
||||
{
|
||||
id: "qa.reporting",
|
||||
role: "secondary",
|
||||
},
|
||||
],
|
||||
},
|
||||
execution: {
|
||||
runner: "vitest",
|
||||
artifacts: [
|
||||
{
|
||||
kind: "report",
|
||||
path: ".artifacts/qa-e2e/scenario-vitest/qa-vitest-report.md",
|
||||
source: "vitest",
|
||||
},
|
||||
{
|
||||
kind: "log",
|
||||
path: ".artifacts/qa-e2e/scenario-vitest/scenario-vitest.log",
|
||||
source: "vitest",
|
||||
},
|
||||
],
|
||||
},
|
||||
result: {
|
||||
status: "fail",
|
||||
failure: {
|
||||
reason: "node exited with 1",
|
||||
},
|
||||
},
|
||||
});
|
||||
});
|
||||
});
|
||||
431
extensions/qa-lab/src/test-file-scenario-runner.ts
Normal file
431
extensions/qa-lab/src/test-file-scenario-runner.ts
Normal file
@@ -0,0 +1,431 @@
|
||||
import { spawn } from "node:child_process";
|
||||
import fs from "node:fs/promises";
|
||||
import path from "node:path";
|
||||
import { formatErrorMessage } from "openclaw/plugin-sdk/error-runtime";
|
||||
import { uniqueStrings } from "openclaw/plugin-sdk/string-coerce-runtime";
|
||||
import { toRepoRelativePath } from "./cli-paths.js";
|
||||
import { QaSuiteArtifactError } from "./errors.js";
|
||||
import {
|
||||
buildPlaywrightEvidenceSummary,
|
||||
buildVitestEvidenceSummary,
|
||||
QA_EVIDENCE_FILENAME,
|
||||
QA_EVIDENCE_SUMMARY_KIND,
|
||||
QA_EVIDENCE_SUMMARY_SCHEMA_VERSION,
|
||||
type QaEvidenceStatus,
|
||||
validateQaEvidenceSummaryJson,
|
||||
} from "./evidence-summary.js";
|
||||
import type { QaProviderMode } from "./providers/index.js";
|
||||
import type { QaSeedScenarioWithSource } from "./scenario-catalog.js";
|
||||
import { shellQuote } from "./shell-quote.js";
|
||||
|
||||
export type QaTestFileScenario = QaSeedScenarioWithSource & {
|
||||
execution: Extract<QaSeedScenarioWithSource["execution"], { kind: "vitest" | "playwright" }>;
|
||||
};
|
||||
|
||||
export type QaTestFileExecutionKind = "vitest" | "playwright";
|
||||
|
||||
export type QaTestFileScenarioRunParams = {
|
||||
env?: NodeJS.ProcessEnv;
|
||||
outputDir: string;
|
||||
primaryModel: string;
|
||||
providerMode: QaProviderMode;
|
||||
repoRoot: string;
|
||||
runCommand?: QaScenarioCommandRunner;
|
||||
scenarios: readonly QaSeedScenarioWithSource[];
|
||||
};
|
||||
|
||||
export type QaScenarioCommandExecution = {
|
||||
args: string[];
|
||||
command: string;
|
||||
cwd: string;
|
||||
env: NodeJS.ProcessEnv;
|
||||
};
|
||||
|
||||
type QaScenarioCommandResult = {
|
||||
exitCode: number;
|
||||
signal?: NodeJS.Signals | null;
|
||||
stdout: string;
|
||||
stderr: string;
|
||||
};
|
||||
|
||||
type QaScenarioCommandRunner = (
|
||||
command: QaScenarioCommandExecution,
|
||||
) => Promise<QaScenarioCommandResult>;
|
||||
|
||||
type QaScenarioCommandStep = {
|
||||
args: string[];
|
||||
command: string;
|
||||
};
|
||||
|
||||
type QaTestFileScenarioResult = {
|
||||
durationMs: number;
|
||||
failureMessage?: string;
|
||||
logPath: string;
|
||||
scenario: QaTestFileScenario;
|
||||
status: QaEvidenceStatus;
|
||||
};
|
||||
|
||||
export type QaTestFileScenarioRunResult = {
|
||||
evidencePath: string;
|
||||
executionKind: QaTestFileExecutionKind;
|
||||
outputDir: string;
|
||||
reportPath: string;
|
||||
results: QaTestFileScenarioResult[];
|
||||
};
|
||||
|
||||
type QaTestFileRunnerDefinition = {
|
||||
buildEvidenceSummary: typeof buildVitestEvidenceSummary;
|
||||
buildSteps(scenario: QaTestFileScenario): QaScenarioCommandStep[];
|
||||
reportFilename: string;
|
||||
reportTitle: string;
|
||||
};
|
||||
|
||||
export function isQaTestFileScenario(
|
||||
scenario: QaSeedScenarioWithSource,
|
||||
): scenario is QaTestFileScenario {
|
||||
return scenario.execution.kind === "vitest" || scenario.execution.kind === "playwright";
|
||||
}
|
||||
|
||||
function vitestSteps(scenario: QaTestFileScenario): QaScenarioCommandStep[] {
|
||||
return [
|
||||
{
|
||||
command: process.execPath,
|
||||
args: ["scripts/run-vitest.mjs", scenario.execution.path, "--reporter=verbose"],
|
||||
},
|
||||
];
|
||||
}
|
||||
|
||||
function playwrightSteps(scenario: QaTestFileScenario): QaScenarioCommandStep[] {
|
||||
return [
|
||||
{
|
||||
command: process.execPath,
|
||||
args: ["scripts/ensure-playwright-chromium.mjs"],
|
||||
},
|
||||
{
|
||||
command: process.execPath,
|
||||
args: [
|
||||
"scripts/run-vitest.mjs",
|
||||
"run",
|
||||
"--config",
|
||||
"test/vitest/vitest.ui-e2e.config.ts",
|
||||
"--configLoader",
|
||||
"runner",
|
||||
scenario.execution.path,
|
||||
"--reporter=verbose",
|
||||
],
|
||||
},
|
||||
];
|
||||
}
|
||||
|
||||
const testFileRunnerDefinitions: Record<QaTestFileExecutionKind, QaTestFileRunnerDefinition> = {
|
||||
vitest: {
|
||||
buildEvidenceSummary: buildVitestEvidenceSummary,
|
||||
buildSteps: vitestSteps,
|
||||
reportFilename: "qa-vitest-report.md",
|
||||
reportTitle: "QA Vitest Scenario Report",
|
||||
},
|
||||
playwright: {
|
||||
buildEvidenceSummary: buildPlaywrightEvidenceSummary,
|
||||
buildSteps: playwrightSteps,
|
||||
reportFilename: "qa-playwright-report.md",
|
||||
reportTitle: "QA Playwright Scenario Report",
|
||||
},
|
||||
};
|
||||
|
||||
function formatCommand(step: QaScenarioCommandStep) {
|
||||
return [step.command, ...step.args].map(shellQuote).join(" ");
|
||||
}
|
||||
|
||||
function runQaScenarioCommand(
|
||||
execution: QaScenarioCommandExecution,
|
||||
): Promise<QaScenarioCommandResult> {
|
||||
return new Promise((resolve, reject) => {
|
||||
const child = spawn(execution.command, execution.args, {
|
||||
cwd: execution.cwd,
|
||||
env: execution.env,
|
||||
stdio: ["ignore", "pipe", "pipe"],
|
||||
});
|
||||
const stdout: Buffer[] = [];
|
||||
const stderr: Buffer[] = [];
|
||||
child.stdout?.on("data", (chunk: Buffer) => {
|
||||
stdout.push(chunk);
|
||||
});
|
||||
child.stderr?.on("data", (chunk: Buffer) => {
|
||||
stderr.push(chunk);
|
||||
});
|
||||
child.on("error", reject);
|
||||
child.on("close", (exitCode, signal) => {
|
||||
resolve({
|
||||
exitCode: exitCode ?? (signal ? 1 : 0),
|
||||
signal,
|
||||
stdout: Buffer.concat(stdout).toString("utf8"),
|
||||
stderr: Buffer.concat(stderr).toString("utf8"),
|
||||
});
|
||||
});
|
||||
});
|
||||
}
|
||||
|
||||
function buildScenarioEvidenceTarget(scenario: QaTestFileScenario) {
|
||||
const surfaces =
|
||||
scenario.surfaces && scenario.surfaces.length > 0 ? scenario.surfaces : [scenario.surface];
|
||||
return {
|
||||
id: scenario.id,
|
||||
title: scenario.title,
|
||||
sourcePath: scenario.execution.path,
|
||||
primaryCoverageIds: scenario.coverage?.primary ?? [],
|
||||
secondaryCoverageIds: scenario.coverage?.secondary ?? [],
|
||||
surfaceIds: surfaces,
|
||||
categoryIds: uniqueStrings([scenario.category].filter(Boolean) as string[]),
|
||||
docsRefs: scenario.docsRefs,
|
||||
codeRefs: scenario.codeRefs,
|
||||
};
|
||||
}
|
||||
|
||||
async function runScenarioCommandSteps(params: {
|
||||
env: NodeJS.ProcessEnv;
|
||||
outputDir: string;
|
||||
repoRoot: string;
|
||||
runCommand: QaScenarioCommandRunner;
|
||||
scenario: QaTestFileScenario;
|
||||
steps: readonly QaScenarioCommandStep[];
|
||||
}): Promise<QaTestFileScenarioResult> {
|
||||
const startedAt = Date.now();
|
||||
const logPath = path.join(params.outputDir, `${params.scenario.id}.log`);
|
||||
const logChunks: string[] = [];
|
||||
let failureMessage: string | undefined;
|
||||
for (const step of params.steps) {
|
||||
logChunks.push(`$ ${formatCommand(step)}\n`);
|
||||
try {
|
||||
const result = await params.runCommand({
|
||||
command: step.command,
|
||||
args: step.args,
|
||||
cwd: params.repoRoot,
|
||||
env: params.env,
|
||||
});
|
||||
if (result.stdout) {
|
||||
logChunks.push(result.stdout);
|
||||
}
|
||||
if (result.stderr) {
|
||||
logChunks.push(result.stderr);
|
||||
}
|
||||
if (result.exitCode !== 0 || result.signal) {
|
||||
failureMessage = result.signal
|
||||
? `${path.basename(step.command)} terminated by ${result.signal}`
|
||||
: `${path.basename(step.command)} exited with ${result.exitCode}`;
|
||||
break;
|
||||
}
|
||||
} catch (error) {
|
||||
failureMessage = formatErrorMessage(error);
|
||||
logChunks.push(`${failureMessage}\n`);
|
||||
break;
|
||||
}
|
||||
logChunks.push("\n");
|
||||
}
|
||||
await fs.writeFile(logPath, logChunks.join(""), "utf8");
|
||||
const durationMs = Math.max(1, Date.now() - startedAt);
|
||||
return {
|
||||
scenario: params.scenario,
|
||||
status: failureMessage ? "fail" : "pass",
|
||||
durationMs,
|
||||
logPath,
|
||||
...(failureMessage ? { failureMessage } : {}),
|
||||
};
|
||||
}
|
||||
|
||||
async function runQaTestFileScenario(params: {
|
||||
env: NodeJS.ProcessEnv;
|
||||
outputDir: string;
|
||||
repoRoot: string;
|
||||
runCommand: QaScenarioCommandRunner;
|
||||
scenario: QaTestFileScenario;
|
||||
}) {
|
||||
const definition = testFileRunnerDefinitions[params.scenario.execution.kind];
|
||||
return await runScenarioCommandSteps({
|
||||
...params,
|
||||
steps: definition.buildSteps(params.scenario),
|
||||
});
|
||||
}
|
||||
|
||||
function resolveTestFileExecutionKind(scenarios: readonly QaTestFileScenario[]) {
|
||||
const kinds = new Set(scenarios.map((scenario) => scenario.execution.kind));
|
||||
if (kinds.size > 1) {
|
||||
throw new Error("qa suite cannot mix Vitest and Playwright scenarios in one invocation.");
|
||||
}
|
||||
const [kind] = kinds;
|
||||
return kind;
|
||||
}
|
||||
|
||||
function buildTestFileEvidence(params: {
|
||||
artifactPaths: { kind: string; path: string }[];
|
||||
generatedAt: string;
|
||||
kind: QaTestFileExecutionKind;
|
||||
primaryModel: string;
|
||||
providerMode: QaProviderMode;
|
||||
results: readonly QaTestFileScenarioResult[];
|
||||
env?: NodeJS.ProcessEnv;
|
||||
}) {
|
||||
const definition = testFileRunnerDefinitions[params.kind];
|
||||
const evidence = definition.buildEvidenceSummary({
|
||||
artifactPaths: params.artifactPaths,
|
||||
env: params.env,
|
||||
generatedAt: params.generatedAt,
|
||||
primaryModel: params.primaryModel,
|
||||
providerMode: params.providerMode,
|
||||
targets: params.results.map((result) => buildScenarioEvidenceTarget(result.scenario)),
|
||||
results: params.results.map((result) => ({
|
||||
id: result.scenario.id,
|
||||
status: result.status,
|
||||
durationMs: result.durationMs,
|
||||
failureMessage: result.failureMessage,
|
||||
})),
|
||||
});
|
||||
return validateQaEvidenceSummaryJson({
|
||||
kind: QA_EVIDENCE_SUMMARY_KIND,
|
||||
schemaVersion: QA_EVIDENCE_SUMMARY_SCHEMA_VERSION,
|
||||
generatedAt: params.generatedAt,
|
||||
entries: evidence.entries,
|
||||
});
|
||||
}
|
||||
|
||||
function buildScenarioArtifactPaths(params: {
|
||||
reportPath: string;
|
||||
repoRoot: string;
|
||||
results: readonly QaTestFileScenarioResult[];
|
||||
}) {
|
||||
return [
|
||||
{ kind: "report", path: toRepoRelativePath(params.repoRoot, params.reportPath) },
|
||||
...params.results.map((result) => ({
|
||||
kind: "log",
|
||||
path: toRepoRelativePath(params.repoRoot, result.logPath),
|
||||
})),
|
||||
];
|
||||
}
|
||||
|
||||
function renderTestFileScenarioReport(params: {
|
||||
evidencePath: string;
|
||||
generatedAt: string;
|
||||
repoRoot: string;
|
||||
results: readonly QaTestFileScenarioResult[];
|
||||
title: string;
|
||||
}) {
|
||||
const lines = [
|
||||
`# ${params.title}`,
|
||||
"",
|
||||
`Generated at: ${params.generatedAt}`,
|
||||
`Evidence summary: ${toRepoRelativePath(params.repoRoot, params.evidencePath)}`,
|
||||
"",
|
||||
"## Results",
|
||||
"",
|
||||
];
|
||||
for (const result of params.results) {
|
||||
const logPath = toRepoRelativePath(params.repoRoot, result.logPath);
|
||||
lines.push(
|
||||
`- ${result.scenario.id}: ${result.status}`,
|
||||
` - kind: ${result.scenario.execution.kind}`,
|
||||
` - path: ${result.scenario.execution.path}`,
|
||||
` - durationMs: ${Math.round(result.durationMs)}`,
|
||||
` - log: ${logPath}`,
|
||||
);
|
||||
if (result.failureMessage) {
|
||||
lines.push(` - failure: ${result.failureMessage.split("\n")[0]}`);
|
||||
}
|
||||
}
|
||||
return `${lines.join("\n")}\n`;
|
||||
}
|
||||
|
||||
async function writeTestFileEvidenceFiles(params: {
|
||||
evidence: unknown;
|
||||
generatedAt: string;
|
||||
outputDir: string;
|
||||
reportFilename: string;
|
||||
reportTitle: string;
|
||||
repoRoot: string;
|
||||
results: readonly QaTestFileScenarioResult[];
|
||||
}): Promise<Pick<QaTestFileScenarioRunResult, "evidencePath" | "reportPath">> {
|
||||
const evidencePath = path.join(params.outputDir, QA_EVIDENCE_FILENAME);
|
||||
const reportPath = path.join(params.outputDir, params.reportFilename);
|
||||
await fs.writeFile(evidencePath, `${JSON.stringify(params.evidence, null, 2)}\n`, "utf8");
|
||||
const report = renderTestFileScenarioReport({
|
||||
evidencePath,
|
||||
generatedAt: params.generatedAt,
|
||||
repoRoot: params.repoRoot,
|
||||
results: params.results,
|
||||
title: params.reportTitle,
|
||||
});
|
||||
await fs.writeFile(reportPath, report, "utf8");
|
||||
await assertQaTestFileArtifactWritten("evidence", evidencePath);
|
||||
await assertQaTestFileArtifactWritten("report", reportPath);
|
||||
return { evidencePath, reportPath };
|
||||
}
|
||||
|
||||
async function assertQaTestFileArtifactWritten(kind: "evidence" | "report", filePath: string) {
|
||||
try {
|
||||
await fs.access(filePath);
|
||||
} catch (error) {
|
||||
throw new QaSuiteArtifactError(
|
||||
`${kind}_missing`,
|
||||
`QA suite did not produce ${kind} artifact at ${filePath}: ${formatErrorMessage(error)}`,
|
||||
{ cause: error },
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
export async function runQaTestFileScenarios(
|
||||
params: QaTestFileScenarioRunParams,
|
||||
): Promise<QaTestFileScenarioRunResult> {
|
||||
const scenarios = params.scenarios.filter(isQaTestFileScenario);
|
||||
const kind = resolveTestFileExecutionKind(scenarios);
|
||||
if (!kind) {
|
||||
throw new Error("qa suite found no Vitest or Playwright scenarios to run.");
|
||||
}
|
||||
const definition = testFileRunnerDefinitions[kind];
|
||||
await fs.mkdir(params.outputDir, { recursive: true });
|
||||
const runCommand = params.runCommand ?? runQaScenarioCommand;
|
||||
const env = {
|
||||
...process.env,
|
||||
...params.env,
|
||||
};
|
||||
const results: QaTestFileScenarioResult[] = [];
|
||||
for (const scenario of scenarios) {
|
||||
results.push(
|
||||
await runQaTestFileScenario({
|
||||
env,
|
||||
outputDir: params.outputDir,
|
||||
repoRoot: params.repoRoot,
|
||||
runCommand,
|
||||
scenario,
|
||||
}),
|
||||
);
|
||||
}
|
||||
const generatedAt = new Date().toISOString();
|
||||
const reportPath = path.join(params.outputDir, definition.reportFilename);
|
||||
const artifactPaths = buildScenarioArtifactPaths({
|
||||
reportPath,
|
||||
repoRoot: params.repoRoot,
|
||||
results,
|
||||
});
|
||||
const evidence = buildTestFileEvidence({
|
||||
artifactPaths,
|
||||
env,
|
||||
generatedAt,
|
||||
kind,
|
||||
primaryModel: params.primaryModel,
|
||||
providerMode: params.providerMode,
|
||||
results,
|
||||
});
|
||||
const paths = await writeTestFileEvidenceFiles({
|
||||
evidence,
|
||||
generatedAt,
|
||||
outputDir: params.outputDir,
|
||||
reportFilename: definition.reportFilename,
|
||||
reportTitle: definition.reportTitle,
|
||||
repoRoot: params.repoRoot,
|
||||
results,
|
||||
});
|
||||
return {
|
||||
...paths,
|
||||
executionKind: kind,
|
||||
outputDir: params.outputDir,
|
||||
results,
|
||||
};
|
||||
}
|
||||
@@ -2,6 +2,19 @@
|
||||
import { describe, expect, it } from "vitest";
|
||||
import { slackDoctor } from "./doctor.js";
|
||||
|
||||
async function collectSlackWarnings(
|
||||
slack: Record<string, unknown>,
|
||||
defaults?: Record<string, unknown>,
|
||||
) {
|
||||
return (
|
||||
(await Promise.resolve(
|
||||
slackDoctor.collectMutableAllowlistWarnings?.({
|
||||
cfg: { channels: { ...(defaults ? { defaults } : {}), slack } } as never,
|
||||
}),
|
||||
)) ?? []
|
||||
);
|
||||
}
|
||||
|
||||
function getSlackCompatibilityNormalizer(): NonNullable<
|
||||
typeof slackDoctor.normalizeCompatibilityConfig
|
||||
> {
|
||||
@@ -50,6 +63,236 @@ describe("slack doctor", () => {
|
||||
).toBe(true);
|
||||
});
|
||||
|
||||
it("warns for name-keyed allowlist channels but accepts routed ID forms (#81665)", async () => {
|
||||
const warnings = await collectSlackWarnings({
|
||||
channels: {
|
||||
"example-channel": {},
|
||||
community: {},
|
||||
C0AL2GDUA7J: {},
|
||||
c0al2gdua7k: {},
|
||||
"channel:C0AL2GDUA7L": {},
|
||||
"channel:c0al2gdua7m": {},
|
||||
D0AL2GDUA7Q: {},
|
||||
"channel:d0al2gdua7r": {},
|
||||
"channel:dabcdefgh": {},
|
||||
"channel:customers": {},
|
||||
"CHANNEL:C0AL2GDUA7N": {},
|
||||
"channel:C0al2gdua7p": {},
|
||||
"*": {},
|
||||
},
|
||||
});
|
||||
|
||||
const nameKeyWarnings = warnings.filter((warning) =>
|
||||
warning.includes("Re-key it with the channel's"),
|
||||
);
|
||||
expect(nameKeyWarnings).toHaveLength(5);
|
||||
expect(nameKeyWarnings[0]).toContain('channels.slack.channels."example-channel"');
|
||||
expect(nameKeyWarnings[0]).toContain('channels.slack.channels."*" applies instead');
|
||||
expect(nameKeyWarnings[1]).toContain('channels.slack.channels."community" is ambiguous');
|
||||
expect(nameKeyWarnings[2]).toContain(
|
||||
'channels.slack.channels."channel:customers" is ambiguous',
|
||||
);
|
||||
expect(nameKeyWarnings[3]).toContain('channels.slack.channels."CHANNEL:C0AL2GDUA7N"');
|
||||
expect(nameKeyWarnings[4]).toContain('channels.slack.channels."channel:C0al2gdua7p"');
|
||||
const dmWarnings = warnings.filter((warning) =>
|
||||
warning.includes("is a Slack DM conversation ID"),
|
||||
);
|
||||
expect(dmWarnings).toHaveLength(3);
|
||||
expect(dmWarnings[0]).toContain('channels.slack.channels."D0AL2GDUA7Q"');
|
||||
expect(dmWarnings[1]).toContain('channels.slack.channels."channel:d0al2gdua7r"');
|
||||
expect(dmWarnings[2]).toContain('channels.slack.channels."channel:dabcdefgh"');
|
||||
expect(dmWarnings[0]).toContain("channels.slack.dmPolicy");
|
||||
});
|
||||
|
||||
it("uses account policy and name-matching overrides for name-keyed channels (#81665)", async () => {
|
||||
const overlongName = "a".repeat(81);
|
||||
const warnings = await collectSlackWarnings({
|
||||
groupPolicy: "open",
|
||||
channels: { "root-room": {} },
|
||||
accounts: {
|
||||
inheritedOpen: {
|
||||
channels: { general: {} },
|
||||
},
|
||||
inheritedAllowlist: {
|
||||
groupPolicy: "allowlist",
|
||||
},
|
||||
explicitAllowlist: {
|
||||
groupPolicy: "allowlist",
|
||||
channels: { engineering: {} },
|
||||
},
|
||||
nameMatching: {
|
||||
groupPolicy: "allowlist",
|
||||
dangerouslyAllowNameMatching: true,
|
||||
channels: {
|
||||
support: {},
|
||||
"#help": {},
|
||||
"crème-brûlée": {},
|
||||
d0customers: {},
|
||||
dabcdefgh: {},
|
||||
"channel:customers": {},
|
||||
"<#C0AL2GDUA7J>": {},
|
||||
"slack:C0AL2GDUA7K": {},
|
||||
"@help": {},
|
||||
"##help": {},
|
||||
"help+": {},
|
||||
Support: {},
|
||||
"-": {},
|
||||
___: {},
|
||||
"#--": {},
|
||||
[overlongName]: {},
|
||||
},
|
||||
},
|
||||
},
|
||||
});
|
||||
|
||||
const nameKeyWarnings = warnings.filter((warning) =>
|
||||
warning.includes("Re-key it with the channel's"),
|
||||
);
|
||||
expect(nameKeyWarnings).toHaveLength(13);
|
||||
const rootWarning = nameKeyWarnings.find((warning) =>
|
||||
warning.includes('channels.slack.channels."root-room"'),
|
||||
);
|
||||
expect(rootWarning).toContain("messages from the channel are dropped");
|
||||
expect(
|
||||
nameKeyWarnings.some((warning) =>
|
||||
warning.includes('channels.slack.accounts.explicitAllowlist.channels."engineering"'),
|
||||
),
|
||||
).toBe(true);
|
||||
expect(
|
||||
nameKeyWarnings.some((warning) =>
|
||||
warning.includes(
|
||||
'channels.slack.accounts.nameMatching.channels."channel:customers" is ambiguous',
|
||||
),
|
||||
),
|
||||
).toBe(true);
|
||||
expect(
|
||||
nameKeyWarnings.some((warning) =>
|
||||
warning.includes('channels.slack.accounts.nameMatching.channels."<#C0AL2GDUA7J>"'),
|
||||
),
|
||||
).toBe(true);
|
||||
expect(
|
||||
nameKeyWarnings.some((warning) =>
|
||||
warning.includes('channels.slack.accounts.nameMatching.channels."slack:C0AL2GDUA7K"'),
|
||||
),
|
||||
).toBe(true);
|
||||
for (const invalidName of [
|
||||
"@help",
|
||||
"##help",
|
||||
"help+",
|
||||
"Support",
|
||||
"-",
|
||||
"___",
|
||||
"#--",
|
||||
overlongName,
|
||||
]) {
|
||||
expect(
|
||||
nameKeyWarnings.some((warning) =>
|
||||
warning.includes(`channels.slack.accounts.nameMatching.channels."${invalidName}"`),
|
||||
),
|
||||
).toBe(true);
|
||||
}
|
||||
|
||||
const sharedOpenWarnings = await collectSlackWarnings(
|
||||
{ channels: { "shared-room": {} } },
|
||||
{ groupPolicy: "open" },
|
||||
);
|
||||
expect(
|
||||
sharedOpenWarnings.some((warning) => warning.includes("not a routable Slack channel ID")),
|
||||
).toBe(true);
|
||||
});
|
||||
|
||||
it("warns when an open-policy override is keyed by channel name (#81665)", async () => {
|
||||
const warnings = await collectSlackWarnings({
|
||||
groupPolicy: "open",
|
||||
channels: {
|
||||
"private-room": { enabled: false },
|
||||
},
|
||||
});
|
||||
|
||||
expect(warnings).toEqual([expect.stringContaining('channels.slack.channels."private-room"')]);
|
||||
expect(warnings[0]).toContain("the channel remains allowed");
|
||||
});
|
||||
|
||||
it("warns for DM IDs regardless of room policy and uses account-scoped remediation", async () => {
|
||||
const openWarnings = await collectSlackWarnings({
|
||||
groupPolicy: "open",
|
||||
channels: {
|
||||
D0AL2GDUA7S: {},
|
||||
},
|
||||
});
|
||||
expect(openWarnings).toEqual([
|
||||
expect.stringContaining('channels.slack.channels."D0AL2GDUA7S"'),
|
||||
]);
|
||||
|
||||
const disabledAccountWarnings = await collectSlackWarnings({
|
||||
accounts: {
|
||||
work: {
|
||||
groupPolicy: "disabled",
|
||||
channels: {
|
||||
"channel:d0al2gdua7t": {},
|
||||
},
|
||||
},
|
||||
},
|
||||
});
|
||||
expect(disabledAccountWarnings).toEqual([
|
||||
expect.stringContaining('channels.slack.accounts.work.channels."channel:d0al2gdua7t"'),
|
||||
]);
|
||||
expect(disabledAccountWarnings[0]).toContain("channels.slack.accounts.work.dmPolicy");
|
||||
expect(disabledAccountWarnings[0]).toContain("channels.slack.accounts.work.allowFrom");
|
||||
|
||||
const inheritedChannelWarnings = await collectSlackWarnings({
|
||||
channels: {
|
||||
D0AL2GDUA7U: {},
|
||||
},
|
||||
accounts: {
|
||||
work: {
|
||||
groupPolicy: "disabled",
|
||||
dmPolicy: "allowlist",
|
||||
allowFrom: ["U0AL2GDUA7U"],
|
||||
},
|
||||
},
|
||||
});
|
||||
expect(inheritedChannelWarnings).toEqual([
|
||||
expect.stringContaining('channels.slack.channels."D0AL2GDUA7U"'),
|
||||
]);
|
||||
expect(inheritedChannelWarnings[0]).toContain("channels.slack.accounts.work.dmPolicy");
|
||||
});
|
||||
|
||||
it("treats bare lowercase D forms as ambiguous without name matching", async () => {
|
||||
const warnings = await collectSlackWarnings({
|
||||
channels: {
|
||||
d0customers: {},
|
||||
dabcdefgh: {},
|
||||
},
|
||||
});
|
||||
|
||||
expect(warnings).toHaveLength(2);
|
||||
expect(warnings[0]).toContain(
|
||||
'channels.slack.channels."d0customers" is ambiguous: it may be a lowercase Slack DM conversation ID or a channel name',
|
||||
);
|
||||
expect(warnings[1]).toContain(
|
||||
'channels.slack.channels."dabcdefgh" is ambiguous: it may be a lowercase Slack DM conversation ID or a channel name',
|
||||
);
|
||||
expect(warnings[0]).toContain("stable C/G ID");
|
||||
});
|
||||
|
||||
it("does not audit provider defaults as a standalone named account (#81665)", async () => {
|
||||
const warnings = await collectSlackWarnings({
|
||||
channels: {
|
||||
"provider-room": { enabled: false },
|
||||
},
|
||||
accounts: {
|
||||
work: {
|
||||
channels: {
|
||||
C0AL2GDUA7J: {},
|
||||
},
|
||||
},
|
||||
},
|
||||
});
|
||||
|
||||
expect(warnings.some((warning) => warning.includes("provider-room"))).toBe(false);
|
||||
});
|
||||
|
||||
it("normalizes legacy slack streaming aliases into the nested streaming shape", () => {
|
||||
const normalize = getSlackCompatibilityNormalizer();
|
||||
|
||||
|
||||
@@ -1,6 +1,8 @@
|
||||
// Slack plugin module implements doctor behavior.
|
||||
import type { ChannelDoctorAdapter } from "openclaw/plugin-sdk/channel-contract";
|
||||
import { createDangerousNameMatchingMutableAllowlistWarningCollector } from "openclaw/plugin-sdk/channel-policy";
|
||||
import type { GroupPolicy, OpenClawConfig } from "openclaw/plugin-sdk/config-contracts";
|
||||
import { listSlackAccountIds, mergeSlackAccountConfig } from "./accounts.js";
|
||||
import {
|
||||
legacyConfigRules as SLACK_LEGACY_CONFIG_RULES,
|
||||
normalizeCompatibilityConfig as normalizeSlackCompatibilityConfig,
|
||||
@@ -48,6 +50,134 @@ const collectSlackMutableAllowlistWarnings =
|
||||
},
|
||||
});
|
||||
|
||||
const SLACK_CANONICAL_CHANNEL_ID_RE = /^[CG][A-Z0-9]{8,}$/;
|
||||
const SLACK_LOWERCASE_CHANNEL_ID_RE = /^[cg][0-9][a-z0-9]{7,}$/;
|
||||
const SLACK_PREFIXED_CANONICAL_CHANNEL_ID_RE = /^channel:[CG][A-Z0-9]{8,}$/;
|
||||
const SLACK_PREFIXED_LOWERCASE_CHANNEL_ID_RE = /^channel:[cg][0-9][a-z0-9]{7,}$/;
|
||||
const SLACK_CANONICAL_DM_ID_RE = /^(?:channel:)?D[A-Z0-9]{8,}$/;
|
||||
const SLACK_PREFIXED_LOWERCASE_DM_ID_RE = /^channel:d[a-z0-9]{8,}$/;
|
||||
const SLACK_AMBIGUOUS_LOWERCASE_DM_ID_RE = /^d[a-z0-9]{8,}$/;
|
||||
// Letter-leading lowercase forms may be valid IDs or human names. Warn conditionally instead of
|
||||
// claiming they are unroutable.
|
||||
const SLACK_AMBIGUOUS_LOWERCASE_CHANNEL_ID_RE = /^(?:channel:)?[cgd][a-z][a-z0-9]{7,}$/;
|
||||
// Slack supports international channel names, and runtime name matching preserves exact names.
|
||||
// Keep Unicode letters/marks/numbers while enforcing lowercase, length, and punctuation rules.
|
||||
const SLACK_CHANNEL_NAME_RE = /^[\p{L}\p{M}\p{N}_-]{1,80}$/u;
|
||||
const SLACK_CHANNEL_NAME_ALPHANUMERIC_RE = /[\p{L}\p{N}]/u;
|
||||
|
||||
function looksLikeSlackChannelId(channelKey: string): boolean {
|
||||
return (
|
||||
SLACK_CANONICAL_CHANNEL_ID_RE.test(channelKey) ||
|
||||
SLACK_LOWERCASE_CHANNEL_ID_RE.test(channelKey) ||
|
||||
SLACK_PREFIXED_CANONICAL_CHANNEL_ID_RE.test(channelKey) ||
|
||||
SLACK_PREFIXED_LOWERCASE_CHANNEL_ID_RE.test(channelKey)
|
||||
);
|
||||
}
|
||||
|
||||
function looksLikeSlackDmId(channelKey: string): boolean {
|
||||
return (
|
||||
SLACK_CANONICAL_DM_ID_RE.test(channelKey) || SLACK_PREFIXED_LOWERCASE_DM_ID_RE.test(channelKey)
|
||||
);
|
||||
}
|
||||
|
||||
function looksLikeSlackChannelNameKey(channelKey: string): boolean {
|
||||
const name = channelKey.startsWith("#") ? channelKey.slice(1) : channelKey;
|
||||
return (
|
||||
name === name.toLowerCase() &&
|
||||
SLACK_CHANNEL_NAME_RE.test(name) &&
|
||||
SLACK_CHANNEL_NAME_ALPHANUMERIC_RE.test(name)
|
||||
);
|
||||
}
|
||||
|
||||
// Startup resolution updates ctx.channelsConfig, but inbound authorization captures the authored
|
||||
// channels map and key list when createSlackMonitorContext runs. Diagnose those authored keys.
|
||||
function collectSlackNameKeyedChannelWarnings({ cfg }: { cfg: OpenClawConfig }): string[] {
|
||||
const warnings = new Set<string>();
|
||||
const slackCfg = asObjectRecord(asObjectRecord(cfg.channels)?.slack);
|
||||
const providerChannels = asObjectRecord(slackCfg?.channels);
|
||||
const accounts = asObjectRecord(slackCfg?.accounts);
|
||||
for (const accountId of listSlackAccountIds(cfg)) {
|
||||
const account = asObjectRecord(mergeSlackAccountConfig(cfg, accountId));
|
||||
if (!account || slackCfg?.enabled === false || account.enabled === false) {
|
||||
continue;
|
||||
}
|
||||
const scopedGroupPolicy =
|
||||
typeof account.groupPolicy === "string" ? (account.groupPolicy as GroupPolicy) : undefined;
|
||||
// Slack's schema materializes this provider default before runtime account merging.
|
||||
const effectiveGroupPolicy = scopedGroupPolicy ?? "allowlist";
|
||||
const rawAccount = asObjectRecord(accounts?.[accountId]);
|
||||
const accountPrefix = rawAccount ? `channels.slack.accounts.${accountId}` : "channels.slack";
|
||||
const accountChannels = asObjectRecord(rawAccount?.channels);
|
||||
const channels = accountChannels ?? providerChannels;
|
||||
if (!channels) {
|
||||
continue;
|
||||
}
|
||||
const channelsPrefix = accountChannels
|
||||
? `channels.slack.accounts.${accountId}`
|
||||
: "channels.slack";
|
||||
const fallbackDescription = Object.hasOwn(channels, "*")
|
||||
? `${channelsPrefix}.channels."*" applies instead and this entry's overrides are ignored`
|
||||
: effectiveGroupPolicy === "open"
|
||||
? 'this entry\'s overrides are ignored and the channel remains allowed by groupPolicy: "open"'
|
||||
: "messages from the channel are dropped";
|
||||
for (const channelKey of Object.keys(channels)) {
|
||||
if (channelKey === "*") {
|
||||
continue;
|
||||
}
|
||||
if (looksLikeSlackDmId(channelKey)) {
|
||||
warnings.add(
|
||||
`${channelsPrefix}.channels."${channelKey}" is a Slack DM conversation ID, but ${channelsPrefix}.channels only configures channel and group rooms. ` +
|
||||
`Configure DM access with ${accountPrefix}.dmPolicy and ${accountPrefix}.allowFrom instead.`,
|
||||
);
|
||||
continue;
|
||||
}
|
||||
if (SLACK_AMBIGUOUS_LOWERCASE_DM_ID_RE.test(channelKey)) {
|
||||
if (
|
||||
account.dangerouslyAllowNameMatching === true &&
|
||||
looksLikeSlackChannelNameKey(channelKey)
|
||||
) {
|
||||
continue;
|
||||
}
|
||||
warnings.add(
|
||||
`${channelsPrefix}.channels."${channelKey}" is ambiguous: it may be a lowercase Slack DM conversation ID or a channel name. ` +
|
||||
`Configure DMs with ${accountPrefix}.dmPolicy and ${accountPrefix}.allowFrom; otherwise re-key the room with its stable C/G ID.`,
|
||||
);
|
||||
continue;
|
||||
}
|
||||
if (effectiveGroupPolicy === "disabled") {
|
||||
continue;
|
||||
}
|
||||
const channelConfig = asObjectRecord(channels[channelKey]);
|
||||
if (effectiveGroupPolicy === "open" && Object.keys(channelConfig ?? {}).length === 0) {
|
||||
continue;
|
||||
}
|
||||
if (looksLikeSlackChannelId(channelKey)) {
|
||||
continue;
|
||||
}
|
||||
if (
|
||||
account.dangerouslyAllowNameMatching === true &&
|
||||
looksLikeSlackChannelNameKey(channelKey)
|
||||
) {
|
||||
continue;
|
||||
}
|
||||
if (SLACK_AMBIGUOUS_LOWERCASE_CHANNEL_ID_RE.test(channelKey)) {
|
||||
warnings.add(
|
||||
`${channelsPrefix}.channels."${channelKey}" is ambiguous: it may be a lowercase Slack channel ID or a channel name. ` +
|
||||
`If it is a channel name, inbound routing will not match it and ${fallbackDescription}. ` +
|
||||
`Re-key it with the channel's stable ID (e.g. C0123ABCD, from the channel's About details or conversations.info).`,
|
||||
);
|
||||
continue;
|
||||
}
|
||||
warnings.add(
|
||||
`${channelsPrefix}.channels."${channelKey}" is keyed by a channel name or non-canonical ID form, not a routable Slack channel ID; ` +
|
||||
`under groupPolicy: "${effectiveGroupPolicy}" inbound routing does not match this entry, so ${fallbackDescription}. ` +
|
||||
`Re-key it with the channel's ID (e.g. C0123ABCD, from the channel's About details or conversations.info).`,
|
||||
);
|
||||
}
|
||||
}
|
||||
return [...warnings];
|
||||
}
|
||||
|
||||
export const slackDoctor: ChannelDoctorAdapter = {
|
||||
dmAllowFromMode: "topOnly",
|
||||
groupModel: "route",
|
||||
@@ -55,5 +185,8 @@ export const slackDoctor: ChannelDoctorAdapter = {
|
||||
warnOnEmptyGroupSenderAllowlist: false,
|
||||
legacyConfigRules: SLACK_LEGACY_CONFIG_RULES,
|
||||
normalizeCompatibilityConfig: normalizeSlackCompatibilityConfig,
|
||||
collectMutableAllowlistWarnings: collectSlackMutableAllowlistWarnings,
|
||||
collectMutableAllowlistWarnings: ({ cfg }) => [
|
||||
...collectSlackMutableAllowlistWarnings({ cfg }),
|
||||
...collectSlackNameKeyedChannelWarnings({ cfg }),
|
||||
],
|
||||
};
|
||||
|
||||
@@ -4,7 +4,8 @@ Single source of truth for repo-backed QA suite bootstrap data.
|
||||
`qa-lab` should treat this directory as a generic markdown scenario pack:
|
||||
|
||||
- `index.md` defines pack-level bootstrap data
|
||||
- each nested `*.md` scenario defines one runnable test via `qa-scenario` + `qa-flow`
|
||||
- each nested `*.md` scenario defines one evidence scenario via `qa-scenario`
|
||||
- flow scenarios add `qa-flow`; Vitest and Playwright scenarios use `execution.path`
|
||||
- scenario markdown may also define coverage IDs, category metadata, required plugins,
|
||||
lane filters, runtime parity tiers, and gateway config patching
|
||||
|
||||
@@ -20,6 +21,10 @@ Coverage tracking:
|
||||
- prefer reusing an existing feature ID over minting a scenario-shaped ID
|
||||
- avoid copying the scenario title into coverage IDs
|
||||
- use `pnpm openclaw qa coverage` to render the current inventory
|
||||
- use `execution.kind: vitest` or `execution.kind: playwright` plus `execution.path`
|
||||
for test files that provide evidence without a `qa-flow` block
|
||||
- run Vitest and Playwright scenarios with
|
||||
`pnpm openclaw qa suite --scenario <scenario-id>`
|
||||
- use `runtimeParityTier` for runtime-pair gate membership: `standard`,
|
||||
`optional`, `live-only`, or `soak`
|
||||
- treat the old `coverage: ["id"]` / `coverage: - id` list shape as invalid
|
||||
|
||||
21
qa/scenarios/ui/control-ui-chat-flow-playwright.md
Normal file
21
qa/scenarios/ui/control-ui-chat-flow-playwright.md
Normal file
@@ -0,0 +1,21 @@
|
||||
# Control UI chat flow Playwright coverage
|
||||
|
||||
```yaml qa-scenario
|
||||
id: control-ui-chat-flow-playwright
|
||||
title: Control UI chat flow Playwright coverage
|
||||
surface: control-ui
|
||||
coverage:
|
||||
primary:
|
||||
- ui.control
|
||||
objective: Link the Control UI chat-flow Playwright suite to the QA coverage inventory.
|
||||
successCriteria:
|
||||
- Playwright covers the hosted Control UI chat surface.
|
||||
docsRefs:
|
||||
- docs/web/control-ui.md
|
||||
codeRefs:
|
||||
- ui/src/ui/e2e/chat-flow.e2e.test.ts
|
||||
execution:
|
||||
kind: playwright
|
||||
path: ui/src/ui/e2e/chat-flow.e2e.test.ts
|
||||
summary: Playwright coverage for the Control UI chat flow.
|
||||
```
|
||||
@@ -2438,7 +2438,7 @@ export function buildFullSuiteVitestRunPlans(args, cwd = process.cwd()) {
|
||||
const expandToProjectConfigs =
|
||||
process.env.OPENCLAW_TEST_PROJECTS_LEAF_SHARDS === "1" ||
|
||||
(Number.isFinite(parallelShardCount) && parallelShardCount > 1) ||
|
||||
shouldUseLocalFullSuiteParallelByDefault(process.env);
|
||||
shouldExpandLocalFullSuiteShardsByDefault(process.env);
|
||||
return fullSuiteVitestShards.flatMap((shard) => {
|
||||
if (
|
||||
process.env.OPENCLAW_TEST_SKIP_FULL_EXTENSIONS_SHARD === "1" &&
|
||||
@@ -2484,6 +2484,10 @@ export function shouldUseLocalFullSuiteParallelByDefault(env = process.env) {
|
||||
);
|
||||
}
|
||||
|
||||
export function shouldExpandLocalFullSuiteShardsByDefault(env = process.env) {
|
||||
return env.CI !== "true" && env.GITHUB_ACTIONS !== "true";
|
||||
}
|
||||
|
||||
function parsePositiveInt(value, label) {
|
||||
const text = value?.trim();
|
||||
if (!text) {
|
||||
|
||||
@@ -3,6 +3,7 @@ import fs from "node:fs";
|
||||
import os from "node:os";
|
||||
import path from "node:path";
|
||||
import { afterEach, describe, expect, it, vi } from "vitest";
|
||||
import { getReplyPayloadMetadata } from "../auto-reply/reply-payload.js";
|
||||
import {
|
||||
testing as replyRunTesting,
|
||||
createReplyOperation,
|
||||
@@ -1276,7 +1277,7 @@ describe("runCliAgent reliability", () => {
|
||||
releaseAgentEnd();
|
||||
});
|
||||
|
||||
it("persists approved CLI user turns before model execution", async () => {
|
||||
it("persists approved CLI user turns and successful assistant output", async () => {
|
||||
supervisorSpawnMock.mockResolvedValueOnce(
|
||||
createManagedRun({
|
||||
reason: "exit",
|
||||
@@ -1289,7 +1290,7 @@ describe("runCliAgent reliability", () => {
|
||||
noOutputTimedOut: false,
|
||||
}),
|
||||
);
|
||||
const { dir, sessionFile } = createSessionFile();
|
||||
const { dir, sessionFile, storePath } = createSessionFile();
|
||||
const onUserMessagePersisted = vi.fn();
|
||||
|
||||
try {
|
||||
@@ -1305,6 +1306,8 @@ describe("runCliAgent reliability", () => {
|
||||
sessionFile,
|
||||
workspaceDir: dir,
|
||||
prompt: "runtime prompt",
|
||||
persistAssistantTranscript: true,
|
||||
storePath,
|
||||
userTurnTranscriptRecorder: createCliUserTurnRecorder({
|
||||
text: "display prompt",
|
||||
sessionFile,
|
||||
@@ -1316,6 +1319,9 @@ describe("runCliAgent reliability", () => {
|
||||
});
|
||||
|
||||
expect(result.payloads).toEqual([{ text: "hello from cli" }]);
|
||||
expect(getReplyPayloadMetadata(result.payloads?.[0] ?? {})).toMatchObject({
|
||||
assistantTranscriptOwned: true,
|
||||
});
|
||||
expect(onUserMessagePersisted).toHaveBeenCalledOnce();
|
||||
expect(onUserMessagePersisted).toHaveBeenCalledWith(
|
||||
expect.objectContaining({
|
||||
@@ -1331,12 +1337,185 @@ describe("runCliAgent reliability", () => {
|
||||
content: "display prompt",
|
||||
}),
|
||||
);
|
||||
expect(messages).toContainEqual(
|
||||
expect.objectContaining({
|
||||
role: "assistant",
|
||||
content: [{ type: "text", text: "hello from cli" }],
|
||||
api: "cli",
|
||||
provider: "codex-cli",
|
||||
model: "gpt-5.4",
|
||||
idempotencyKey: "cli-assistant:run-persist-cli",
|
||||
}),
|
||||
);
|
||||
expect(JSON.stringify(messages)).not.toContain("runtime prompt");
|
||||
} finally {
|
||||
fs.rmSync(dir, { recursive: true, force: true });
|
||||
}
|
||||
});
|
||||
|
||||
it("lets before_message_write block CLI assistant persistence without delivery fallback", async () => {
|
||||
const hookRunner = {
|
||||
hasHooks: vi.fn((hookName: string) => hookName === "before_message_write"),
|
||||
runBeforeMessageWrite: vi.fn(() => ({ block: true })),
|
||||
};
|
||||
setHookRunnerForTest(hookRunner);
|
||||
supervisorSpawnMock.mockResolvedValueOnce(
|
||||
createManagedRun({
|
||||
reason: "exit",
|
||||
exitCode: 0,
|
||||
exitSignal: null,
|
||||
durationMs: 50,
|
||||
stdout: "secret CLI output",
|
||||
stderr: "",
|
||||
timedOut: false,
|
||||
noOutputTimedOut: false,
|
||||
}),
|
||||
);
|
||||
const { dir, sessionFile, storePath } = createSessionFile();
|
||||
|
||||
try {
|
||||
const context = buildPreparedContext({
|
||||
sessionKey: "agent:main:main",
|
||||
runId: "run-blocked-cli",
|
||||
});
|
||||
const result = await runPreparedCliAgent({
|
||||
...context,
|
||||
params: {
|
||||
...context.params,
|
||||
agentId: "main",
|
||||
sessionFile,
|
||||
workspaceDir: dir,
|
||||
persistAssistantTranscript: true,
|
||||
storePath,
|
||||
},
|
||||
});
|
||||
|
||||
expect(result.payloads).toEqual([{ text: "secret CLI output" }]);
|
||||
expect(getReplyPayloadMetadata(result.payloads?.[0] ?? {})).toMatchObject({
|
||||
assistantTranscriptOwned: true,
|
||||
});
|
||||
expect(readTranscriptMessages(sessionFile)).toEqual([]);
|
||||
expect(hookRunner.runBeforeMessageWrite).toHaveBeenCalledOnce();
|
||||
expect(
|
||||
callArg(hookRunner.runBeforeMessageWrite, 0, 1, "before_message_write context"),
|
||||
).toEqual({
|
||||
agentId: "main",
|
||||
sessionKey: "agent:main:main",
|
||||
});
|
||||
} finally {
|
||||
fs.rmSync(dir, { recursive: true, force: true });
|
||||
}
|
||||
});
|
||||
|
||||
it("does not append late CLI output after the session key is rebound", async () => {
|
||||
supervisorSpawnMock.mockResolvedValueOnce(
|
||||
createManagedRun({
|
||||
reason: "exit",
|
||||
exitCode: 0,
|
||||
exitSignal: null,
|
||||
durationMs: 50,
|
||||
stdout: "late CLI output",
|
||||
stderr: "",
|
||||
timedOut: false,
|
||||
noOutputTimedOut: false,
|
||||
}),
|
||||
);
|
||||
const { dir, sessionFile, storePath } = createSessionFile();
|
||||
const replacementFile = path.join(path.dirname(sessionFile), "s2.jsonl");
|
||||
fs.writeFileSync(
|
||||
replacementFile,
|
||||
`${JSON.stringify({
|
||||
type: "session",
|
||||
version: CURRENT_SESSION_VERSION,
|
||||
id: "s2",
|
||||
timestamp: new Date(0).toISOString(),
|
||||
cwd: dir,
|
||||
})}\n`,
|
||||
"utf-8",
|
||||
);
|
||||
fs.writeFileSync(
|
||||
storePath,
|
||||
JSON.stringify({
|
||||
"agent:main:main": {
|
||||
sessionId: "s2",
|
||||
sessionFile: replacementFile,
|
||||
updatedAt: Date.now(),
|
||||
},
|
||||
}),
|
||||
"utf-8",
|
||||
);
|
||||
|
||||
try {
|
||||
const context = buildPreparedContext({
|
||||
sessionKey: "agent:main:main",
|
||||
runId: "run-rebound-cli",
|
||||
});
|
||||
const result = await runPreparedCliAgent({
|
||||
...context,
|
||||
params: {
|
||||
...context.params,
|
||||
agentId: "main",
|
||||
sessionFile,
|
||||
workspaceDir: dir,
|
||||
persistAssistantTranscript: true,
|
||||
storePath,
|
||||
},
|
||||
});
|
||||
|
||||
expect(result.payloads).toEqual([{ text: "late CLI output" }]);
|
||||
expect(getReplyPayloadMetadata(result.payloads?.[0] ?? {})).toMatchObject({
|
||||
assistantTranscriptOwned: true,
|
||||
});
|
||||
expect(readTranscriptMessages(sessionFile)).toEqual([]);
|
||||
expect(readTranscriptMessages(replacementFile)).toEqual([]);
|
||||
} finally {
|
||||
fs.rmSync(dir, { recursive: true, force: true });
|
||||
}
|
||||
});
|
||||
|
||||
it("does not persist private room-event assistant output", async () => {
|
||||
supervisorSpawnMock.mockResolvedValueOnce(
|
||||
createManagedRun({
|
||||
reason: "exit",
|
||||
exitCode: 0,
|
||||
exitSignal: null,
|
||||
durationMs: 50,
|
||||
stdout: "private ambient output",
|
||||
stderr: "",
|
||||
timedOut: false,
|
||||
noOutputTimedOut: false,
|
||||
}),
|
||||
);
|
||||
const { dir, sessionFile, storePath } = createSessionFile();
|
||||
|
||||
try {
|
||||
const context = buildPreparedContext({
|
||||
sessionKey: "agent:main:main",
|
||||
runId: "run-private-room-event",
|
||||
});
|
||||
const result = await runPreparedCliAgent({
|
||||
...context,
|
||||
params: {
|
||||
...context.params,
|
||||
agentId: "main",
|
||||
sessionFile,
|
||||
workspaceDir: dir,
|
||||
persistAssistantTranscript: true,
|
||||
storePath,
|
||||
currentInboundEventKind: "room_event",
|
||||
},
|
||||
});
|
||||
|
||||
expect(result.payloads).toEqual([{ text: "private ambient output" }]);
|
||||
expect(getReplyPayloadMetadata(result.payloads?.[0] ?? {})).toMatchObject({
|
||||
assistantTranscriptOwned: true,
|
||||
});
|
||||
expect(readTranscriptMessages(sessionFile)).toEqual([]);
|
||||
} finally {
|
||||
fs.rmSync(dir, { recursive: true, force: true });
|
||||
}
|
||||
});
|
||||
|
||||
it("passes cwd to approved CLI user-turn persistence", async () => {
|
||||
supervisorSpawnMock.mockResolvedValueOnce(
|
||||
createManagedRun({
|
||||
|
||||
@@ -656,6 +656,9 @@ describe("runCliAgent spawn path", () => {
|
||||
currentMessageId: "reply-message-1",
|
||||
senderId: "sender-1",
|
||||
senderIsOwner: true,
|
||||
persistAssistantTranscript: true,
|
||||
storePath: "/tmp/sessions.json",
|
||||
currentInboundEventKind: "room_event",
|
||||
});
|
||||
|
||||
expect(params.messageChannel).toBe("telegram");
|
||||
@@ -666,6 +669,9 @@ describe("runCliAgent spawn path", () => {
|
||||
expect(params.senderId).toBe("sender-1");
|
||||
expect(params.senderIsOwner).toBe(true);
|
||||
expect(params.cwd).toBe("/tmp/task-repo");
|
||||
expect(params.persistAssistantTranscript).toBe(true);
|
||||
expect(params.storePath).toBe("/tmp/sessions.json");
|
||||
expect(params.currentInboundEventKind).toBe("room_event");
|
||||
});
|
||||
|
||||
it("forwards static extra system prompt through the compat wrapper", () => {
|
||||
|
||||
@@ -1,8 +1,9 @@
|
||||
/**
|
||||
* Top-level CLI-backed agent runner orchestration.
|
||||
*/
|
||||
import type { ReplyPayload } from "../auto-reply/reply-payload.js";
|
||||
import { setReplyPayloadMetadata, type ReplyPayload } from "../auto-reply/reply-payload.js";
|
||||
import { SILENT_REPLY_TOKEN } from "../auto-reply/tokens.js";
|
||||
import { appendExactAssistantMessageToSessionTranscript } from "../config/sessions/transcript.js";
|
||||
import { formatErrorMessage } from "../infra/errors.js";
|
||||
import { createSubsystemLogger } from "../logging/subsystem.js";
|
||||
import { buildAgentHookContextChannelFields } from "../plugins/hook-agent-context.js";
|
||||
@@ -28,6 +29,7 @@ import {
|
||||
runHarnessContextEngineMaintenance,
|
||||
} from "./harness/context-engine-lifecycle.js";
|
||||
import { buildAgentHookContext } from "./harness/hook-context.js";
|
||||
import { runAgentHarnessBeforeMessageWriteHook } from "./harness/hook-helpers.js";
|
||||
import { buildAgentHookConversationMessages } from "./harness/hook-history.js";
|
||||
import {
|
||||
runAgentHarnessLlmInputHook,
|
||||
@@ -35,6 +37,7 @@ import {
|
||||
} from "./harness/lifecycle-hook-helpers.js";
|
||||
import type { AgentMessage } from "./runtime/index.js";
|
||||
import { SessionManager } from "./sessions/session-manager.js";
|
||||
import { buildAssistantMessage, buildUsageWithNoCost } from "./stream-message-shared.js";
|
||||
|
||||
const log = createSubsystemLogger("agents/cli-runner");
|
||||
|
||||
@@ -231,6 +234,62 @@ async function persistApprovedCliUserTurnTranscript(params: RunCliAgentParams):
|
||||
}
|
||||
}
|
||||
|
||||
async function persistCliAssistantTranscript(params: {
|
||||
runParams: RunCliAgentParams;
|
||||
text: string;
|
||||
modelId: string;
|
||||
usage?: {
|
||||
input?: number;
|
||||
output?: number;
|
||||
cacheRead?: number;
|
||||
cacheWrite?: number;
|
||||
total?: number;
|
||||
};
|
||||
}): Promise<boolean> {
|
||||
const { runParams } = params;
|
||||
if (!runParams.persistAssistantTranscript || !runParams.sessionKey || !params.text) {
|
||||
return false;
|
||||
}
|
||||
if (runParams.currentInboundEventKind === "room_event") {
|
||||
return true;
|
||||
}
|
||||
try {
|
||||
const result = await appendExactAssistantMessageToSessionTranscript({
|
||||
sessionKey: runParams.sessionKey,
|
||||
agentId: runParams.agentId,
|
||||
expectedSessionId: runParams.sessionId,
|
||||
storePath: runParams.storePath,
|
||||
idempotencyKey: `cli-assistant:${runParams.runId}`,
|
||||
config: runParams.config,
|
||||
beforeMessageWrite: runAgentHarnessBeforeMessageWriteHook,
|
||||
message: buildAssistantMessage({
|
||||
model: {
|
||||
api: "cli",
|
||||
provider: runParams.provider,
|
||||
id: params.modelId,
|
||||
},
|
||||
content: [{ type: "text", text: params.text }],
|
||||
stopReason: "stop",
|
||||
usage: buildUsageWithNoCost({
|
||||
input: params.usage?.input,
|
||||
output: params.usage?.output,
|
||||
cacheRead: params.usage?.cacheRead,
|
||||
cacheWrite: params.usage?.cacheWrite,
|
||||
totalTokens: params.usage?.total,
|
||||
}),
|
||||
}),
|
||||
});
|
||||
if (!result.ok) {
|
||||
log.warn(`CLI assistant transcript persistence skipped: ${result.reason}`);
|
||||
return result.code === "blocked" || result.code === "session-rebound";
|
||||
}
|
||||
return true;
|
||||
} catch (error) {
|
||||
log.warn(`CLI assistant transcript persistence failed: ${formatErrorMessage(error)}`);
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
async function finalizeCliContextEngineTurn(params: {
|
||||
context: PreparedCliRunContext;
|
||||
historyMessages: unknown[];
|
||||
@@ -594,11 +653,16 @@ export async function runPreparedCliAgent(
|
||||
output: Awaited<ReturnType<typeof executePreparedCliRun>>;
|
||||
effectiveCliSessionId?: string;
|
||||
bindingFlushOk?: boolean;
|
||||
assistantTranscriptOwned?: boolean;
|
||||
}): EmbeddedAgentRunResult => {
|
||||
const text = resultParams.output.text?.trim();
|
||||
const rawText = resultParams.output.rawText?.trim();
|
||||
const payloads = text
|
||||
? [{ text }]
|
||||
? [
|
||||
resultParams.assistantTranscriptOwned
|
||||
? setReplyPayloadMetadata({ text }, { assistantTranscriptOwned: true })
|
||||
: { text },
|
||||
]
|
||||
: params.allowEmptyAssistantReplyAsSilent === true
|
||||
? [{ text: SILENT_REPLY_TOKEN }]
|
||||
: undefined;
|
||||
@@ -718,6 +782,12 @@ export async function runPreparedCliAgent(
|
||||
assistantText,
|
||||
output,
|
||||
});
|
||||
const assistantTranscriptOwned = await persistCliAssistantTranscript({
|
||||
runParams: params,
|
||||
text: assistantText,
|
||||
modelId: context.modelId,
|
||||
usage: output.usage,
|
||||
});
|
||||
const bindingFlushOk = await isCliBindingFlushed(
|
||||
effectiveCliSessionId,
|
||||
params.provider,
|
||||
@@ -732,7 +802,12 @@ export async function runPreparedCliAgent(
|
||||
ctx: hookContext,
|
||||
hookRunner,
|
||||
});
|
||||
return buildCliRunResult({ output, effectiveCliSessionId, bindingFlushOk });
|
||||
return buildCliRunResult({
|
||||
output,
|
||||
effectiveCliSessionId,
|
||||
bindingFlushOk,
|
||||
assistantTranscriptOwned,
|
||||
});
|
||||
};
|
||||
|
||||
if (hasBeforeAgentRunHooks && hookRunner) {
|
||||
@@ -880,6 +955,9 @@ export function buildRunClaudeCliAgentParams(params: RunClaudeCliAgentParams): R
|
||||
cwd: params.cwd,
|
||||
config: params.config,
|
||||
prompt: params.prompt,
|
||||
persistAssistantTranscript: params.persistAssistantTranscript,
|
||||
storePath: params.storePath,
|
||||
currentInboundEventKind: params.currentInboundEventKind,
|
||||
provider: params.provider ?? "claude-cli",
|
||||
model: params.model ?? "opus",
|
||||
thinkLevel: params.thinkLevel,
|
||||
|
||||
@@ -46,6 +46,10 @@ export type RunCliAgentParams = {
|
||||
suppressNextUserMessagePersistence?: boolean;
|
||||
userTurnTranscriptRecorder?: UserTurnTranscriptRecorder;
|
||||
onUserMessagePersisted?: (message: PersistedUserTurnMessage) => void | Promise<void>;
|
||||
/** Persist the successful CLI assistant reply into the OpenClaw session transcript. */
|
||||
persistAssistantTranscript?: boolean;
|
||||
/** Session store path used when assistant transcript persistence is enabled. */
|
||||
storePath?: string;
|
||||
currentInboundEventKind?: InboundEventKind;
|
||||
currentInboundContext?: CurrentInboundPromptContext;
|
||||
inputProvenance?: InputProvenance;
|
||||
|
||||
@@ -688,6 +688,100 @@ describe("resolveModel", () => {
|
||||
expect(discoverModels).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it("resolves a deferred Fireworks manifest id from the bundled static catalog", async () => {
|
||||
resolveBundledStaticCatalogModelMock.mockReturnValueOnce({
|
||||
provider: "fireworks",
|
||||
id: "accounts/fireworks/models/kimi-k2p6",
|
||||
name: "Kimi K2.6",
|
||||
api: "openai-completions",
|
||||
baseUrl: "https://api.fireworks.ai/inference/v1",
|
||||
reasoning: false,
|
||||
input: ["text", "image"],
|
||||
cost: { input: 0.95, output: 4, cacheRead: 0, cacheWrite: 0 },
|
||||
contextWindow: 262144,
|
||||
maxTokens: 262144,
|
||||
});
|
||||
|
||||
const result = await resolveModelAsync(
|
||||
"fireworks",
|
||||
"accounts/fireworks/models/kimi-k2p6",
|
||||
"/tmp/agent",
|
||||
undefined,
|
||||
{
|
||||
allowBundledStaticCatalogFallback: true,
|
||||
runtimeHooks: createRuntimeHooks(),
|
||||
skipAgentDiscovery: true,
|
||||
},
|
||||
);
|
||||
|
||||
expectRecordFields(expectResolvedModel(result), {
|
||||
provider: "fireworks",
|
||||
id: "accounts/fireworks/models/kimi-k2p6",
|
||||
api: "openai-completions",
|
||||
baseUrl: "https://api.fireworks.ai/inference/v1",
|
||||
contextWindow: 262144,
|
||||
maxTokens: 262144,
|
||||
});
|
||||
expect(resolveBundledStaticCatalogModelMock).toHaveBeenCalledWith(
|
||||
expect.objectContaining({
|
||||
provider: "fireworks",
|
||||
modelId: "accounts/fireworks/models/kimi-k2p6",
|
||||
}),
|
||||
);
|
||||
});
|
||||
|
||||
it("prefers user openclaw.json config over the Fireworks manifest for the same id", () => {
|
||||
resolveBundledStaticCatalogModelMock.mockReturnValue({
|
||||
...makeModel("accounts/fireworks/models/kimi-k2p6"),
|
||||
provider: "fireworks",
|
||||
name: "Kimi K2.6",
|
||||
api: "openai-completions",
|
||||
baseUrl: "https://api.fireworks.ai/inference/v1",
|
||||
input: ["text", "image"],
|
||||
contextWindow: 262_144,
|
||||
maxTokens: 262_144,
|
||||
});
|
||||
const cfg = {
|
||||
models: {
|
||||
providers: {
|
||||
fireworks: {
|
||||
api: "openai-completions",
|
||||
baseUrl: "https://api.fireworks.ai/inference/v1",
|
||||
models: [
|
||||
{
|
||||
...makeModel("accounts/fireworks/models/kimi-k2p6"),
|
||||
name: "Kimi K2.6 (user override)",
|
||||
contextWindow: 300_000,
|
||||
maxTokens: 300_000,
|
||||
},
|
||||
],
|
||||
},
|
||||
},
|
||||
},
|
||||
} as unknown as OpenClawConfig;
|
||||
|
||||
const result = resolveModelForTest(
|
||||
"fireworks",
|
||||
"accounts/fireworks/models/kimi-k2p6",
|
||||
"/tmp/agent",
|
||||
cfg,
|
||||
);
|
||||
|
||||
expectRecordFields(expectResolvedModel(result), {
|
||||
provider: "fireworks",
|
||||
id: "accounts/fireworks/models/kimi-k2p6",
|
||||
contextWindow: 300_000,
|
||||
maxTokens: 300_000,
|
||||
});
|
||||
expect(resolveBundledStaticCatalogModelMock).toHaveBeenCalledWith(
|
||||
expect.objectContaining({
|
||||
provider: "fireworks",
|
||||
modelId: "accounts/fireworks/models/kimi-k2p6",
|
||||
cfg,
|
||||
}),
|
||||
);
|
||||
});
|
||||
|
||||
it("keeps provider dynamic metadata for runtime-preferred models", async () => {
|
||||
resolveBundledStaticCatalogModelMock.mockReturnValueOnce({
|
||||
provider: "openai",
|
||||
@@ -2922,6 +3016,52 @@ describe("resolveModel", () => {
|
||||
});
|
||||
});
|
||||
|
||||
it("uses provider-normalized model ids for OpenRouter transport", () => {
|
||||
const modelId = "openrouter/anthropic/claude-sonnet-4.6";
|
||||
mockDiscoveredModel(discoverModels, {
|
||||
provider: "openrouter",
|
||||
modelId,
|
||||
templateModel: {
|
||||
...makeModel(modelId),
|
||||
provider: "openrouter",
|
||||
api: "openai-completions",
|
||||
baseUrl: "https://openrouter.ai/api/v1",
|
||||
},
|
||||
});
|
||||
const baseRuntimeHooks = createRuntimeHooks();
|
||||
const normalizeProviderResolvedModelWithPlugin = vi.fn(
|
||||
(params: { context: { model: { id: string } } }) => ({
|
||||
...params.context.model,
|
||||
id: params.context.model.id.slice("openrouter/".length),
|
||||
}),
|
||||
);
|
||||
|
||||
const result = resolveModel("openrouter", modelId, "/tmp/agent", undefined, {
|
||||
authStorage: { mocked: true } as never,
|
||||
modelRegistry: discoverModels({ mocked: true } as never, "/tmp/agent"),
|
||||
runtimeHooks: {
|
||||
...baseRuntimeHooks,
|
||||
normalizeProviderResolvedModelWithPlugin,
|
||||
},
|
||||
});
|
||||
|
||||
expect(normalizeProviderResolvedModelWithPlugin).toHaveBeenCalledWith(
|
||||
expect.objectContaining({
|
||||
provider: "openrouter",
|
||||
context: expect.objectContaining({
|
||||
modelId,
|
||||
model: expect.objectContaining({ id: modelId }),
|
||||
}),
|
||||
}),
|
||||
);
|
||||
expectRecordFields(result.model, {
|
||||
provider: "openrouter",
|
||||
id: "anthropic/claude-sonnet-4.6",
|
||||
api: "openai-completions",
|
||||
baseUrl: "https://openrouter.ai/api/v1",
|
||||
});
|
||||
});
|
||||
|
||||
it("matches prefixed Hugging Face ids against discovered registry models", () => {
|
||||
mockDiscoveredModel(discoverModels, {
|
||||
provider: "huggingface",
|
||||
|
||||
@@ -7,6 +7,7 @@ import {
|
||||
import { resetTaskRegistryForTests, type TaskRecord } from "../../../tasks/runtime-internal.js";
|
||||
import {
|
||||
requiresCompletionRequiredAsyncTaskWait,
|
||||
shouldWaitForCompletionRequiredAsyncTasks,
|
||||
waitForCompletionRequiredAsyncTasks,
|
||||
type AsyncStartedToolMeta,
|
||||
} from "./attempt.async-tasks.js";
|
||||
@@ -97,6 +98,46 @@ describe("waitForCompletionRequiredAsyncTasks", () => {
|
||||
).toBe(true);
|
||||
});
|
||||
|
||||
it("skips media task waiting after sessions_yield pauses the attempt", () => {
|
||||
resetTaskRegistryForTests();
|
||||
const sessionKey = "agent:main:cron:daily-media:run:run-123";
|
||||
createRunningTaskRun({
|
||||
runtime: "cli",
|
||||
taskKind: "image_generation",
|
||||
sourceId: "image_generate:openai",
|
||||
requesterSessionKey: sessionKey,
|
||||
ownerKey: sessionKey,
|
||||
scopeKind: "session",
|
||||
runId: "tool:image_generate:run-123",
|
||||
task: "daily image",
|
||||
deliveryStatus: "not_applicable",
|
||||
notifyPolicy: "silent",
|
||||
startedAt: 1,
|
||||
lastEventAt: 1,
|
||||
});
|
||||
|
||||
expect(
|
||||
shouldWaitForCompletionRequiredAsyncTasks({
|
||||
sessionKey,
|
||||
toolMetas: [
|
||||
{
|
||||
toolName: "image_generate",
|
||||
asyncStarted: true,
|
||||
asyncTaskRunId: "tool:image_generate:run-123",
|
||||
},
|
||||
],
|
||||
yieldDetected: true,
|
||||
}),
|
||||
).toBe(false);
|
||||
expect(
|
||||
shouldWaitForCompletionRequiredAsyncTasks({
|
||||
sessionKey,
|
||||
toolMetas: [],
|
||||
yieldDetected: false,
|
||||
}),
|
||||
).toBe(true);
|
||||
});
|
||||
|
||||
it("waits for active cron media tasks from the task registry", async () => {
|
||||
// Cron media tools may start tasks before metadata is flushed, so the
|
||||
// registry is also consulted by session key.
|
||||
|
||||
@@ -160,6 +160,23 @@ export function requiresCompletionRequiredAsyncTaskWait(params: {
|
||||
);
|
||||
}
|
||||
|
||||
/** Returns whether the current attempt should synchronously wait for media tasks. */
|
||||
export function shouldWaitForCompletionRequiredAsyncTasks(params: {
|
||||
sessionKey: string | undefined;
|
||||
toolMetas: readonly AsyncStartedToolMeta[];
|
||||
yieldDetected?: boolean;
|
||||
}): boolean {
|
||||
if (params.yieldDetected === true) {
|
||||
// sessions_yield pauses the turn so the completion event can wake it later;
|
||||
// waiting here would reuse the internal abort signal and turn the pause into AbortError.
|
||||
return false;
|
||||
}
|
||||
return requiresCompletionRequiredAsyncTaskWait({
|
||||
sessionKey: params.sessionKey,
|
||||
toolMetas: params.toolMetas,
|
||||
});
|
||||
}
|
||||
|
||||
/**
|
||||
* Polls completion-required async tasks until they reach terminal state, time
|
||||
* out at the run deadline, or abort. Newly discovered task run ids are folded
|
||||
|
||||
@@ -316,6 +316,7 @@ import {
|
||||
} from "./attempt-trajectory-status.js";
|
||||
import {
|
||||
requiresCompletionRequiredAsyncTaskWait,
|
||||
shouldWaitForCompletionRequiredAsyncTasks,
|
||||
waitForCompletionRequiredAsyncTasks,
|
||||
type AsyncStartedToolMeta,
|
||||
type CompletionRequiredAsyncTaskWaitResult,
|
||||
@@ -4571,9 +4572,10 @@ export async function runEmbeddedAttempt(
|
||||
await sessionLockController.releaseForPrompt();
|
||||
|
||||
if (
|
||||
requiresCompletionRequiredAsyncTaskWait({
|
||||
shouldWaitForCompletionRequiredAsyncTasks({
|
||||
sessionKey: params.sessionKey,
|
||||
toolMetas,
|
||||
yieldDetected: yieldAborted,
|
||||
})
|
||||
) {
|
||||
const getAsyncStartedToolMetas = () =>
|
||||
|
||||
@@ -219,7 +219,9 @@ describe("runtime context prompt submission", () => {
|
||||
"OpenClaw runtime event.",
|
||||
"This context is runtime-generated, not user-authored. Keep internal details private.",
|
||||
"",
|
||||
"<<<BEGIN_OPENCLAW_INTERNAL_CONTEXT>>>",
|
||||
"internal event",
|
||||
"<<<END_OPENCLAW_INTERNAL_CONTEXT>>>",
|
||||
].join("\n"),
|
||||
});
|
||||
});
|
||||
@@ -242,7 +244,9 @@ describe("runtime context prompt submission", () => {
|
||||
"OpenClaw runtime event.",
|
||||
"This context is runtime-generated, not user-authored. Keep internal details private.",
|
||||
"",
|
||||
"<<<BEGIN_OPENCLAW_INTERNAL_CONTEXT>>>",
|
||||
"internal event",
|
||||
"<<<END_OPENCLAW_INTERNAL_CONTEXT>>>",
|
||||
].join("\n"),
|
||||
});
|
||||
});
|
||||
@@ -339,7 +343,9 @@ describe("runtime context prompt submission", () => {
|
||||
"OpenClaw runtime context for the immediately preceding user message.",
|
||||
"This context is runtime-generated, not user-authored. Keep internal details private.",
|
||||
"",
|
||||
"<<<BEGIN_OPENCLAW_INTERNAL_CONTEXT>>>",
|
||||
"secret runtime context",
|
||||
"<<<END_OPENCLAW_INTERNAL_CONTEXT>>>",
|
||||
].join("\n"),
|
||||
display: false,
|
||||
details: { source: "openclaw-runtime-context" },
|
||||
|
||||
@@ -3,6 +3,8 @@
|
||||
*/
|
||||
import {
|
||||
extractInternalRuntimeContext,
|
||||
INTERNAL_RUNTIME_CONTEXT_BEGIN,
|
||||
INTERNAL_RUNTIME_CONTEXT_END,
|
||||
OPENCLAW_NEXT_TURN_RUNTIME_CONTEXT_HEADER,
|
||||
OPENCLAW_RUNTIME_CONTEXT_CUSTOM_TYPE,
|
||||
OPENCLAW_RUNTIME_CONTEXT_NOTICE,
|
||||
@@ -153,13 +155,18 @@ function buildRuntimeContextMessageContent(params: {
|
||||
runtimeContext: string;
|
||||
kind: "next-turn" | "runtime-event";
|
||||
}): string {
|
||||
// Wrap the runtime context body in delimited internal-context markers so
|
||||
// stripInternalRuntimeContext can fully remove the block when it leaks
|
||||
// into user-visible surfaces (e.g. Feishu streaming cards, #92589).
|
||||
return [
|
||||
params.kind === "runtime-event"
|
||||
? OPENCLAW_RUNTIME_EVENT_HEADER
|
||||
: OPENCLAW_NEXT_TURN_RUNTIME_CONTEXT_HEADER,
|
||||
OPENCLAW_RUNTIME_CONTEXT_NOTICE,
|
||||
"",
|
||||
INTERNAL_RUNTIME_CONTEXT_BEGIN,
|
||||
params.runtimeContext,
|
||||
INTERNAL_RUNTIME_CONTEXT_END,
|
||||
].join("\n");
|
||||
}
|
||||
|
||||
|
||||
@@ -2,9 +2,11 @@
|
||||
* Regression coverage for non-secret model-auth marker helpers.
|
||||
* Verifies core, plugin, env-var, OAuth, AWS, and secret-ref marker handling.
|
||||
*/
|
||||
import { fileURLToPath } from "node:url";
|
||||
import { beforeAll, describe, expect, it, vi } from "vitest";
|
||||
import { withEnv, withEnvAsync } from "../test-utils/env.js";
|
||||
|
||||
const BUNDLED_PLUGINS_DIR = fileURLToPath(new URL("../../extensions/", import.meta.url));
|
||||
const PLUGIN_MANIFEST_ENV_KEYS = [
|
||||
"OPENCLAW_BUNDLED_PLUGINS_DIR",
|
||||
"OPENCLAW_DISABLE_BUNDLED_PLUGINS",
|
||||
@@ -14,9 +16,12 @@ const PLUGIN_MANIFEST_ENV_KEYS = [
|
||||
"OPENCLAW_TEST_MINIMAL_GATEWAY",
|
||||
] as const;
|
||||
|
||||
function cleanPluginManifestEnv(): Record<(typeof PLUGIN_MANIFEST_ENV_KEYS)[number], undefined> {
|
||||
function cleanPluginManifestEnv(): Record<
|
||||
(typeof PLUGIN_MANIFEST_ENV_KEYS)[number],
|
||||
string | undefined
|
||||
> {
|
||||
return {
|
||||
OPENCLAW_BUNDLED_PLUGINS_DIR: undefined,
|
||||
OPENCLAW_BUNDLED_PLUGINS_DIR: BUNDLED_PLUGINS_DIR,
|
||||
OPENCLAW_DISABLE_BUNDLED_PLUGINS: undefined,
|
||||
OPENCLAW_SKIP_PROVIDERS: undefined,
|
||||
OPENCLAW_SKIP_CHANNELS: undefined,
|
||||
@@ -35,6 +40,7 @@ let listKnownNonSecretApiKeyMarkers: typeof import("./model-auth-markers.js").li
|
||||
let resolveOAuthApiKeyMarker: typeof import("./model-auth-markers.js").resolveOAuthApiKeyMarker;
|
||||
|
||||
async function loadMarkerModules() {
|
||||
vi.doUnmock("../plugins/manifest-metadata-scan.js");
|
||||
vi.doUnmock("../plugins/manifest-registry.js");
|
||||
vi.doUnmock("../secrets/provider-env-vars.js");
|
||||
vi.resetModules();
|
||||
|
||||
@@ -36,6 +36,7 @@ const AWS_SDK_ENV_MARKERS = new Set([
|
||||
const CORE_NON_SECRET_API_KEY_MARKERS = [
|
||||
CUSTOM_LOCAL_AUTH_MARKER,
|
||||
CODEX_APP_SERVER_AUTH_MARKER,
|
||||
GCP_VERTEX_CREDENTIALS_MARKER,
|
||||
OLLAMA_LOCAL_AUTH_MARKER,
|
||||
NON_ENV_SECRETREF_MARKER,
|
||||
] as const;
|
||||
|
||||
@@ -29,6 +29,19 @@ vi.mock("../plugins/plugin-registry.js", () => ({
|
||||
}),
|
||||
}));
|
||||
|
||||
vi.mock("../plugins/manifest-metadata-scan.js", () => ({
|
||||
listOpenClawPluginManifestMetadata: () => [
|
||||
{
|
||||
pluginDir: "/bundled/anthropic-vertex",
|
||||
origin: "bundled",
|
||||
manifest: {
|
||||
id: "anthropic-vertex",
|
||||
nonSecretAuthMarkers: ["gcp-vertex-credentials"],
|
||||
},
|
||||
},
|
||||
],
|
||||
}));
|
||||
|
||||
vi.mock("../plugins/providers.js", () => ({
|
||||
resolveOwningPluginIdsForProvider: () => [],
|
||||
resolveOwningPluginIdsForProviderRef: () => [],
|
||||
|
||||
@@ -68,19 +68,35 @@ describe("loadModelCatalogForBrowse", () => {
|
||||
expect(loadCatalog).toHaveBeenCalledExactlyOnceWith({ readOnly: false });
|
||||
});
|
||||
|
||||
it("uses the full catalog when configured visibility has provider wildcards", async () => {
|
||||
it("uses the read-only catalog when configured visibility has provider wildcards", async () => {
|
||||
const loadCatalog = vi.fn(async ({ readOnly }: { readOnly: boolean }) =>
|
||||
readOnly ? readOnlyCatalog : fullCatalog,
|
||||
);
|
||||
|
||||
await expect(
|
||||
loadModelCatalogForBrowse({ cfg: config({ providerWildcard: true }), loadCatalog }),
|
||||
).resolves.toBe(readOnlyCatalog);
|
||||
|
||||
expect(loadCatalog).toHaveBeenCalledExactlyOnceWith({ readOnly: true });
|
||||
});
|
||||
|
||||
it("uses the full catalog for configured views with provider wildcards", async () => {
|
||||
const loadCatalog = vi.fn(async ({ readOnly }: { readOnly: boolean }) =>
|
||||
readOnly ? readOnlyCatalog : fullCatalog,
|
||||
);
|
||||
|
||||
await expect(
|
||||
loadModelCatalogForBrowse({
|
||||
cfg: config({ providerWildcard: true }),
|
||||
view: "configured",
|
||||
loadCatalog,
|
||||
}),
|
||||
).resolves.toBe(fullCatalog);
|
||||
|
||||
expect(loadCatalog).toHaveBeenCalledExactlyOnceWith({ readOnly: false });
|
||||
});
|
||||
|
||||
it("returns an empty catalog when read-only catalog loading times out", async () => {
|
||||
it("returns an empty catalog when read-only catalog loading times out with provider wildcards", async () => {
|
||||
const onTimeout = vi.fn();
|
||||
const timeoutHandle = { unref: vi.fn() } as unknown as NodeJS.Timeout;
|
||||
const clearTimeout = vi.fn();
|
||||
@@ -94,7 +110,7 @@ describe("loadModelCatalogForBrowse", () => {
|
||||
const loadCatalog = vi.fn(() => new Promise<ModelCatalogEntry[]>(() => {}));
|
||||
|
||||
const resultPromise = loadModelCatalogForBrowse({
|
||||
cfg: config(),
|
||||
cfg: config({ providerWildcard: true }),
|
||||
loadCatalog,
|
||||
timeoutMs: 5,
|
||||
onTimeout,
|
||||
|
||||
@@ -36,13 +36,6 @@ export function restoreModelCatalogBrowseTestDeps(): void {
|
||||
modelCatalogBrowseDeps.clearTimeout = globalThis.clearTimeout;
|
||||
}
|
||||
|
||||
function resolveModelCatalogBrowseTimeoutMs(value: number | undefined): number {
|
||||
return (
|
||||
clampTimerTimeoutMs(value, 1) ??
|
||||
resolveTimerTimeoutMs(DEFAULT_MODEL_CATALOG_BROWSE_TIMEOUT_MS, 1)
|
||||
);
|
||||
}
|
||||
|
||||
/** True when a browse view cannot be answered from read-only cached catalog entries. */
|
||||
export function modelCatalogBrowseRequiresFullDiscovery(params: {
|
||||
cfg: OpenClawConfig;
|
||||
@@ -51,7 +44,15 @@ export function modelCatalogBrowseRequiresFullDiscovery(params: {
|
||||
const view = params.view ?? "default";
|
||||
return (
|
||||
view === "all" ||
|
||||
parseConfiguredModelVisibilityEntries({ cfg: params.cfg }).providerWildcards.size > 0
|
||||
(view === "configured" &&
|
||||
parseConfiguredModelVisibilityEntries({ cfg: params.cfg }).providerWildcards.size > 0)
|
||||
);
|
||||
}
|
||||
|
||||
function resolveModelCatalogBrowseTimeoutMs(value: number | undefined): number {
|
||||
return (
|
||||
clampTimerTimeoutMs(value, 1) ??
|
||||
resolveTimerTimeoutMs(DEFAULT_MODEL_CATALOG_BROWSE_TIMEOUT_MS, 1)
|
||||
);
|
||||
}
|
||||
|
||||
@@ -65,7 +66,6 @@ export async function loadModelCatalogForBrowse(params: {
|
||||
}): Promise<ModelCatalogEntry[]> {
|
||||
const view = params.view ?? "default";
|
||||
if (modelCatalogBrowseRequiresFullDiscovery({ cfg: params.cfg, view })) {
|
||||
// Wildcards depend on provider discovery; read-only cached entries can hide matching models.
|
||||
return await params.loadCatalog({ readOnly: false });
|
||||
}
|
||||
|
||||
|
||||
@@ -10,6 +10,11 @@ import { isCliRuntimeProvider } from "./model-runtime-aliases.js";
|
||||
// model picker choices. Hide them while keeping real provider/model refs visible.
|
||||
const RETIRED_MODEL_PICKER_PROVIDERS = new Set(["codex", "codex-cli"]);
|
||||
|
||||
/** True for retired provider ids that should stay out of model selection surfaces. */
|
||||
export function isRetiredModelPickerProvider(provider: string): boolean {
|
||||
return RETIRED_MODEL_PICKER_PROVIDERS.has(normalizeProviderId(provider));
|
||||
}
|
||||
|
||||
/** Creates a provider visibility predicate for model picker rendering. */
|
||||
export function createModelPickerVisibleProviderPredicate(
|
||||
params: { config?: OpenClawConfig; env?: NodeJS.ProcessEnv; includeSetupRegistry?: boolean } = {},
|
||||
@@ -23,7 +28,7 @@ export function createModelPickerVisibleProviderPredicate(
|
||||
);
|
||||
return (provider: string): boolean => {
|
||||
const normalized = normalizeProviderId(provider);
|
||||
return !RETIRED_MODEL_PICKER_PROVIDERS.has(normalized) && !cliRuntimeProviders.has(normalized);
|
||||
return !isRetiredModelPickerProvider(normalized) && !cliRuntimeProviders.has(normalized);
|
||||
};
|
||||
}
|
||||
|
||||
@@ -31,7 +36,7 @@ export function createModelPickerVisibleProviderPredicate(
|
||||
export function isModelPickerVisibleProvider(provider: string): boolean {
|
||||
const normalized = normalizeProviderId(provider);
|
||||
return (
|
||||
!RETIRED_MODEL_PICKER_PROVIDERS.has(normalized) &&
|
||||
!isRetiredModelPickerProvider(normalized) &&
|
||||
!isCliRuntimeProvider(normalized, { includeSetupRegistry: true })
|
||||
);
|
||||
}
|
||||
|
||||
@@ -234,6 +234,19 @@ describe("prepared provider auth state", () => {
|
||||
).resolves.toBe(false);
|
||||
expect(modelAuthMocks.hasRuntimeAvailableProviderAuth).toHaveBeenCalledTimes(2);
|
||||
|
||||
// Bounded browse callers may explicitly consume the prepared broad answer
|
||||
// while keeping slow fallback discovery disabled.
|
||||
await expect(
|
||||
hasAuthForModelProvider({
|
||||
provider: "openai",
|
||||
cfg,
|
||||
discoverExternalCliAuth: false,
|
||||
allowPluginSyntheticAuth: false,
|
||||
allowPreparedRuntimeAuth: true,
|
||||
}),
|
||||
).resolves.toBe(true);
|
||||
expect(modelAuthMocks.hasRuntimeAvailableProviderAuth).toHaveBeenCalledTimes(2);
|
||||
|
||||
// Broad-scope caller (default flags) still hits the prepared map.
|
||||
await expect(hasAuthForModelProvider({ provider: "openai", cfg })).resolves.toBe(true);
|
||||
expect(modelAuthMocks.hasRuntimeAvailableProviderAuth).toHaveBeenCalledTimes(2);
|
||||
|
||||
@@ -127,6 +127,7 @@ export async function hasAuthForModelProvider(params: {
|
||||
store?: AuthProfileStore;
|
||||
allowPluginSyntheticAuth?: boolean;
|
||||
discoverExternalCliAuth?: boolean;
|
||||
allowPreparedRuntimeAuth?: boolean;
|
||||
runtimeAuthLookup?: RuntimeProviderAuthLookup;
|
||||
resolveRuntimeAuthLookup?: () => RuntimeProviderAuthLookup;
|
||||
}): Promise<boolean> {
|
||||
@@ -162,8 +163,8 @@ export async function hasAuthForModelProvider(params: {
|
||||
configFingerprint === preparedState.configFingerprint &&
|
||||
workspaceDir === expectedWorkspaceDir &&
|
||||
(params.agentDir === undefined || params.agentDir === expectedAgentDir) &&
|
||||
params.discoverExternalCliAuth !== false &&
|
||||
params.allowPluginSyntheticAuth !== false &&
|
||||
(params.allowPreparedRuntimeAuth === true ||
|
||||
(params.discoverExternalCliAuth !== false && params.allowPluginSyntheticAuth !== false)) &&
|
||||
params.env === undefined &&
|
||||
params.store === undefined &&
|
||||
params.modelApi === undefined;
|
||||
@@ -227,6 +228,7 @@ export function createProviderAuthChecker(params: {
|
||||
env?: NodeJS.ProcessEnv;
|
||||
allowPluginSyntheticAuth?: boolean;
|
||||
discoverExternalCliAuth?: boolean;
|
||||
allowPreparedRuntimeAuth?: boolean;
|
||||
}): (provider: string, modelApi?: string) => Promise<boolean> {
|
||||
const authCache = new Map<string, boolean>();
|
||||
let runtimeAuthLookup: RuntimeProviderAuthLookup | undefined;
|
||||
@@ -247,6 +249,7 @@ export function createProviderAuthChecker(params: {
|
||||
env: params.env,
|
||||
allowPluginSyntheticAuth: params.allowPluginSyntheticAuth,
|
||||
discoverExternalCliAuth: params.discoverExternalCliAuth,
|
||||
allowPreparedRuntimeAuth: params.allowPreparedRuntimeAuth,
|
||||
resolveRuntimeAuthLookup: () =>
|
||||
(runtimeAuthLookup ??= createRuntimeProviderAuthLookup({
|
||||
cfg: params.cfg,
|
||||
|
||||
@@ -2,7 +2,10 @@
|
||||
import { afterEach, beforeEach, describe, expect, it } from "vitest";
|
||||
import type { OpenClawConfig } from "../config/types.openclaw.js";
|
||||
import { testing as cliBackendsTesting } from "./cli-backends.js";
|
||||
import { createModelPickerVisibleProviderPredicate } from "./model-picker-visibility.js";
|
||||
import {
|
||||
createModelPickerVisibleProviderPredicate,
|
||||
isRetiredModelPickerProvider,
|
||||
} from "./model-picker-visibility.js";
|
||||
import {
|
||||
areRuntimeModelRefsEquivalent,
|
||||
isCliRuntimeProvider,
|
||||
@@ -169,6 +172,20 @@ describe("resolveCliRuntimeExecutionProvider", () => {
|
||||
expect(isCliRuntimeProvider("acme-cli")).toBe(false);
|
||||
expect(isVisibleProvider("acme-cli")).toBe(true);
|
||||
});
|
||||
|
||||
it("recognizes retired picker providers without loading CLI backend metadata", () => {
|
||||
cliBackendsTesting.setDepsForTest({
|
||||
resolvePluginSetupRegistry: () => {
|
||||
throw new Error("retired provider checks should not load setup metadata");
|
||||
},
|
||||
resolveRuntimeCliBackends: () => {
|
||||
throw new Error("retired provider checks should not load runtime metadata");
|
||||
},
|
||||
});
|
||||
|
||||
expect(isRetiredModelPickerProvider("CODEX-CLI")).toBe(true);
|
||||
expect(isRetiredModelPickerProvider("anthropic")).toBe(false);
|
||||
});
|
||||
});
|
||||
|
||||
describe("areRuntimeModelRefsEquivalent", () => {
|
||||
|
||||
@@ -2,6 +2,7 @@
|
||||
import fs from "node:fs/promises";
|
||||
import os from "node:os";
|
||||
import path from "node:path";
|
||||
import { fileURLToPath } from "node:url";
|
||||
import { beforeAll, describe, expect, it } from "vitest";
|
||||
import type { OpenClawConfig } from "../config/config.js";
|
||||
import { createConfigRuntimeEnv } from "../config/env-vars.js";
|
||||
@@ -17,6 +18,7 @@ import type { ProviderConfig } from "./models-config.providers.secrets.js";
|
||||
import { encodePluginModelCatalogRelativePath } from "./plugin-model-catalog.js";
|
||||
|
||||
const TEST_ENV_VAR = "OPENCLAW_MODELS_CONFIG_TEST_ENV";
|
||||
const BUNDLED_PLUGINS_DIR = fileURLToPath(new URL("../../extensions/", import.meta.url));
|
||||
|
||||
function createImplicitOpenRouterProvider(): ProviderConfig {
|
||||
return {
|
||||
@@ -533,33 +535,42 @@ describe("models-config", () => {
|
||||
const credentialsPath = path.join(agentDir, "application_default_credentials.json");
|
||||
await fs.writeFile(credentialsPath, JSON.stringify({ type: "authorized_user" }), "utf8");
|
||||
try {
|
||||
const plan = await planOpenClawModelsJsonWithDeps(
|
||||
const plan = await withEnvAsync(
|
||||
{
|
||||
cfg: {
|
||||
agents: {
|
||||
defaults: {
|
||||
models: {
|
||||
"google-vertex/gemini-2.5-pro": {},
|
||||
OPENCLAW_BUNDLED_PLUGINS_DIR: BUNDLED_PLUGINS_DIR,
|
||||
OPENCLAW_DISABLE_BUNDLED_PLUGINS: undefined,
|
||||
},
|
||||
async () =>
|
||||
await planOpenClawModelsJsonWithDeps(
|
||||
{
|
||||
cfg: {
|
||||
agents: {
|
||||
defaults: {
|
||||
models: {
|
||||
"google-vertex/gemini-2.5-pro": {},
|
||||
},
|
||||
model: { primary: "google-vertex/gemini-2.5-pro" },
|
||||
},
|
||||
},
|
||||
model: { primary: "google-vertex/gemini-2.5-pro" },
|
||||
models: { providers: {} },
|
||||
},
|
||||
agentDir,
|
||||
env: {
|
||||
OPENCLAW_BUNDLED_PLUGINS_DIR: BUNDLED_PLUGINS_DIR,
|
||||
OPENCLAW_DISABLE_BUNDLED_PLUGINS: undefined,
|
||||
GOOGLE_APPLICATION_CREDENTIALS: credentialsPath,
|
||||
GOOGLE_CLOUD_PROJECT: "vertex-project",
|
||||
GOOGLE_CLOUD_LOCATION: "global",
|
||||
} as NodeJS.ProcessEnv,
|
||||
existingRaw: "",
|
||||
existingParsed: null,
|
||||
},
|
||||
models: { providers: {} },
|
||||
},
|
||||
agentDir,
|
||||
env: {
|
||||
GOOGLE_APPLICATION_CREDENTIALS: credentialsPath,
|
||||
GOOGLE_CLOUD_PROJECT: "vertex-project",
|
||||
GOOGLE_CLOUD_LOCATION: "global",
|
||||
} as NodeJS.ProcessEnv,
|
||||
existingRaw: "",
|
||||
existingParsed: null,
|
||||
},
|
||||
{
|
||||
resolveImplicitProviders: async () => ({
|
||||
"google-vertex": createImplicitGoogleVertexProvider(),
|
||||
}),
|
||||
},
|
||||
{
|
||||
resolveImplicitProviders: async () => ({
|
||||
"google-vertex": createImplicitGoogleVertexProvider(),
|
||||
}),
|
||||
},
|
||||
),
|
||||
);
|
||||
|
||||
expect(plan.action).toBe("write");
|
||||
|
||||
@@ -2,15 +2,18 @@
|
||||
import { mkdtemp, writeFile } from "node:fs/promises";
|
||||
import os from "node:os";
|
||||
import path from "node:path";
|
||||
import { fileURLToPath } from "node:url";
|
||||
import { beforeEach, describe, expect, it, vi } from "vitest";
|
||||
import type { PluginMetadataSnapshotOwnerMaps } from "../plugins/plugin-metadata-snapshot.js";
|
||||
import type { ProviderPlugin } from "../plugins/types.js";
|
||||
import { withEnvAsync } from "../test-utils/env.js";
|
||||
|
||||
const mocks = vi.hoisted(() => ({
|
||||
resolveRuntimePluginDiscoveryProviders: vi.fn(),
|
||||
runProviderCatalog: vi.fn(),
|
||||
runProviderStaticCatalog: vi.fn(),
|
||||
}));
|
||||
const BUNDLED_PLUGINS_DIR = fileURLToPath(new URL("../../extensions/", import.meta.url));
|
||||
|
||||
vi.mock("../plugins/provider-discovery.js", () => ({
|
||||
resolveRuntimePluginDiscoveryProviders: mocks.resolveRuntimePluginDiscoveryProviders,
|
||||
@@ -225,17 +228,26 @@ describe("resolveImplicitProviders startup discovery scope", () => {
|
||||
},
|
||||
});
|
||||
|
||||
const providers = await resolveImplicitProviders({
|
||||
agentDir: "/tmp/openclaw-agent",
|
||||
config: {},
|
||||
env: {
|
||||
GOOGLE_APPLICATION_CREDENTIALS: credentialsPath,
|
||||
GOOGLE_CLOUD_PROJECT: "vertex-project",
|
||||
GOOGLE_CLOUD_LOCATION: "global",
|
||||
} as NodeJS.ProcessEnv,
|
||||
explicitProviders: {},
|
||||
providerDiscoveryEntriesOnly: true,
|
||||
});
|
||||
const providers = await withEnvAsync(
|
||||
{
|
||||
OPENCLAW_BUNDLED_PLUGINS_DIR: BUNDLED_PLUGINS_DIR,
|
||||
OPENCLAW_DISABLE_BUNDLED_PLUGINS: undefined,
|
||||
},
|
||||
async () =>
|
||||
await resolveImplicitProviders({
|
||||
agentDir: "/tmp/openclaw-agent",
|
||||
config: {},
|
||||
env: {
|
||||
OPENCLAW_BUNDLED_PLUGINS_DIR: BUNDLED_PLUGINS_DIR,
|
||||
OPENCLAW_DISABLE_BUNDLED_PLUGINS: undefined,
|
||||
GOOGLE_APPLICATION_CREDENTIALS: credentialsPath,
|
||||
GOOGLE_CLOUD_PROJECT: "vertex-project",
|
||||
GOOGLE_CLOUD_LOCATION: "global",
|
||||
} as NodeJS.ProcessEnv,
|
||||
explicitProviders: {},
|
||||
providerDiscoveryEntriesOnly: true,
|
||||
}),
|
||||
);
|
||||
|
||||
expect(providers?.["google-vertex"]?.apiKey).toBe("gcp-vertex-credentials");
|
||||
});
|
||||
|
||||
@@ -1513,11 +1513,10 @@ describe("sessions tools", () => {
|
||||
expect(calls.find((call) => call.method === "send")).toBeUndefined();
|
||||
});
|
||||
|
||||
it("sessions_send reroutes run-scoped active deliveries when transcript steering is rejected", async () => {
|
||||
it("sessions_send reports active-run queue rejection without durable-session fallback", async () => {
|
||||
const calls: Array<{ method?: string; params?: unknown }> = [];
|
||||
const requesterKey = "agent:re-portal:main";
|
||||
const runScopedCallerKey = "agent:leasing-ops:cron:monthly-utility:run:run-fast";
|
||||
const durableCallerKey = "agent:leasing-ops:cron:monthly-utility";
|
||||
const queueMessage = vi.fn(async (_text: string, _options?: unknown) => {
|
||||
throw new Error("active session ended before queued steering message was committed");
|
||||
});
|
||||
@@ -1539,13 +1538,6 @@ describe("sessions tools", () => {
|
||||
if (request.method === "agent") {
|
||||
return { runId: "fallback-run", status: "accepted", acceptedAt: 2000 };
|
||||
}
|
||||
if (request.method === "agent.wait") {
|
||||
const params = request.params as { runId?: string } | undefined;
|
||||
return { runId: params?.runId ?? "fallback-run", status: "ok" };
|
||||
}
|
||||
if (request.method === "chat.history") {
|
||||
return { messages: [] };
|
||||
}
|
||||
return {};
|
||||
});
|
||||
|
||||
@@ -1570,9 +1562,11 @@ describe("sessions tools", () => {
|
||||
timeoutSeconds: 0,
|
||||
});
|
||||
const details = sessionsSendDetails(result.details);
|
||||
expect(details.status).toBe("accepted");
|
||||
expect(details.status).toBe("error");
|
||||
expect(details.sessionKey).toBe(runScopedCallerKey);
|
||||
expect(details.delivery?.status).toBe("pending");
|
||||
expect(details.error).toContain("queue_message_failed reason=runtime_rejected");
|
||||
expect(details.error).toContain("caller-active-session");
|
||||
expect(details.error).not.toContain("fallback_failed");
|
||||
const queuedText = queueMessage.mock.calls[0]?.[0];
|
||||
expect(queuedText).toContain("[Inter-session message]");
|
||||
expect(queuedText).toContain("[TASK-COMPLETE] re-portal occupancy ready");
|
||||
@@ -1583,47 +1577,233 @@ describe("sessions tools", () => {
|
||||
waitForTranscriptCommit: true,
|
||||
sourceReplyDeliveryMode: "message_tool_only",
|
||||
});
|
||||
expect(calls.some((call) => call.method === "agent")).toBe(false);
|
||||
});
|
||||
|
||||
await vi.waitFor(() => {
|
||||
const fallbackCall = calls.find(
|
||||
(call) =>
|
||||
call.method === "agent" &&
|
||||
(call.params as { sessionKey?: string } | undefined)?.sessionKey === durableCallerKey,
|
||||
);
|
||||
expect(fallbackCall).toBeDefined();
|
||||
it("sessions_send reports source reply delivery mode mismatch without durable-session fallback", async () => {
|
||||
const calls: Array<{ method?: string; params?: unknown }> = [];
|
||||
const runScopedCallerKey = "agent:leasing-ops:cron:monthly-utility:run:run-fast";
|
||||
const queueMessage = vi.fn(async () => {});
|
||||
setActiveEmbeddedRun(
|
||||
"caller-active-session",
|
||||
{
|
||||
queueMessage,
|
||||
isStreaming: () => true,
|
||||
isCompacting: () => false,
|
||||
supportsTranscriptCommitWait: true,
|
||||
sourceReplyDeliveryMode: "automatic",
|
||||
abort: () => {},
|
||||
},
|
||||
runScopedCallerKey,
|
||||
);
|
||||
callGatewayMock.mockImplementation(async (opts: unknown) => {
|
||||
const request = opts as { method?: string; params?: unknown };
|
||||
calls.push(request);
|
||||
if (request.method === "agent") {
|
||||
return { runId: "fallback-run", status: "accepted", acceptedAt: 2000 };
|
||||
}
|
||||
return {};
|
||||
});
|
||||
|
||||
const tool = createOpenClawTools({
|
||||
agentSessionKey: "agent:re-portal:main",
|
||||
agentChannel: "telegram",
|
||||
config: {
|
||||
...TEST_CONFIG,
|
||||
session: {
|
||||
...TEST_CONFIG.session,
|
||||
agentToAgent: { maxPingPongTurns: 0 },
|
||||
},
|
||||
},
|
||||
}).find((candidate) => candidate.name === "sessions_send");
|
||||
if (!tool) {
|
||||
throw new Error("missing sessions_send tool");
|
||||
}
|
||||
|
||||
const result = await tool.execute("call-run-scoped-caller", {
|
||||
sessionKey: runScopedCallerKey,
|
||||
message: "[TASK-COMPLETE] re-portal occupancy ready",
|
||||
timeoutSeconds: 0,
|
||||
});
|
||||
|
||||
const details = sessionsSendDetails(result.details);
|
||||
expect(details.status).toBe("error");
|
||||
expect(details.sessionKey).toBe(runScopedCallerKey);
|
||||
expect(details.error).toContain(
|
||||
"queue_message_failed reason=source_reply_delivery_mode_mismatch",
|
||||
);
|
||||
expect(queueMessage).not.toHaveBeenCalled();
|
||||
expect(calls.some((call) => call.method === "agent")).toBe(false);
|
||||
});
|
||||
|
||||
it("sessions_send keeps ordinary active session targets on the gateway agent path", async () => {
|
||||
const calls: Array<{ method?: string; params?: unknown }> = [];
|
||||
const ordinaryActiveKey = "agent:main:main";
|
||||
const queueMessage = vi.fn(async () => {});
|
||||
setActiveEmbeddedRun(
|
||||
"ordinary-active-session",
|
||||
{
|
||||
queueMessage,
|
||||
isStreaming: () => true,
|
||||
isCompacting: () => false,
|
||||
supportsTranscriptCommitWait: true,
|
||||
sourceReplyDeliveryMode: "automatic",
|
||||
abort: () => {},
|
||||
},
|
||||
ordinaryActiveKey,
|
||||
);
|
||||
callGatewayMock.mockImplementation(async (opts: unknown) => {
|
||||
const request = opts as { method?: string; params?: unknown };
|
||||
calls.push(request);
|
||||
if (request.method === "agent") {
|
||||
return { runId: "ordinary-agent-run", status: "accepted", acceptedAt: 2000 };
|
||||
}
|
||||
return {};
|
||||
});
|
||||
|
||||
const tool = createOpenClawTools({
|
||||
agentSessionKey: "agent:re-portal:main",
|
||||
agentChannel: "telegram",
|
||||
config: {
|
||||
...TEST_CONFIG,
|
||||
session: {
|
||||
...TEST_CONFIG.session,
|
||||
agentToAgent: { maxPingPongTurns: 0 },
|
||||
},
|
||||
},
|
||||
}).find((candidate) => candidate.name === "sessions_send");
|
||||
if (!tool) {
|
||||
throw new Error("missing sessions_send tool");
|
||||
}
|
||||
|
||||
const result = await tool.execute("call-ordinary-active", {
|
||||
sessionKey: ordinaryActiveKey,
|
||||
message: "ordinary active target should stay gateway routed",
|
||||
timeoutSeconds: 0,
|
||||
});
|
||||
|
||||
const details = sessionsSendDetails(result.details);
|
||||
expect(details.status).toBe("accepted");
|
||||
expect(details.runId).toBe("ordinary-agent-run");
|
||||
expect(details.sessionKey).toBe(ordinaryActiveKey);
|
||||
expect(queueMessage).not.toHaveBeenCalled();
|
||||
const agentCalls = calls.filter((call) => call.method === "agent");
|
||||
expect(
|
||||
agentCalls.some(
|
||||
(call) =>
|
||||
(call.params as { sessionKey?: string } | undefined)?.sessionKey === runScopedCallerKey,
|
||||
),
|
||||
).toBe(false);
|
||||
const fallbackParams = agentCalls.find(
|
||||
(call) =>
|
||||
(call.params as { sessionKey?: string } | undefined)?.sessionKey === durableCallerKey,
|
||||
)?.params as { inputProvenance?: { sourceSessionKey?: string }; message?: string } | undefined;
|
||||
expect(fallbackParams?.message).toContain("[Inter-session message]");
|
||||
expect(fallbackParams?.message).toContain("[TASK-COMPLETE] re-portal occupancy ready");
|
||||
expect(fallbackParams?.inputProvenance?.sourceSessionKey).toBe(requesterKey);
|
||||
expect(agentCalls).toHaveLength(1);
|
||||
expect(agentParams(agentCalls[0] ?? {}).sessionKey).toBe(ordinaryActiveKey);
|
||||
});
|
||||
|
||||
await vi.waitFor(() => {
|
||||
const waitCall = calls.find(
|
||||
(call) =>
|
||||
call.method === "agent.wait" &&
|
||||
(call.params as { runId?: string } | undefined)?.runId === "fallback-run",
|
||||
);
|
||||
expect(waitCall).toBeDefined();
|
||||
it("sessions_send falls back from stranded cron run key to durable cron parent", async () => {
|
||||
const calls: Array<{ method?: string; params?: unknown }> = [];
|
||||
const runScopedCallerKey = "agent:leasing-ops:cron:monthly-utility:run:run-fast";
|
||||
const durableCronCallerKey = "agent:leasing-ops:cron:monthly-utility";
|
||||
const queueMessage = vi.fn(async () => {});
|
||||
setActiveEmbeddedRun(
|
||||
"caller-active-session",
|
||||
{
|
||||
queueMessage,
|
||||
isStreaming: () => false,
|
||||
isCompacting: () => false,
|
||||
supportsTranscriptCommitWait: true,
|
||||
sourceReplyDeliveryMode: "message_tool_only",
|
||||
abort: () => {},
|
||||
},
|
||||
runScopedCallerKey,
|
||||
);
|
||||
callGatewayMock.mockImplementation(async (opts: unknown) => {
|
||||
const request = opts as { method?: string; params?: unknown };
|
||||
calls.push(request);
|
||||
if (request.method === "agent") {
|
||||
return { runId: "durable-fallback-run", status: "accepted", acceptedAt: 2000 };
|
||||
}
|
||||
return {};
|
||||
});
|
||||
await vi.waitFor(() => {
|
||||
const historyCall = calls.find(
|
||||
(call) =>
|
||||
call.method === "chat.history" &&
|
||||
(call.params as { sessionKey?: string } | undefined)?.sessionKey === durableCallerKey,
|
||||
);
|
||||
expect(historyCall).toBeDefined();
|
||||
|
||||
const tool = createOpenClawTools({
|
||||
agentSessionKey: "agent:re-portal:main",
|
||||
agentChannel: "telegram",
|
||||
config: {
|
||||
...TEST_CONFIG,
|
||||
session: {
|
||||
...TEST_CONFIG.session,
|
||||
agentToAgent: { maxPingPongTurns: 0 },
|
||||
},
|
||||
},
|
||||
}).find((candidate) => candidate.name === "sessions_send");
|
||||
if (!tool) {
|
||||
throw new Error("missing sessions_send tool");
|
||||
}
|
||||
|
||||
const result = await tool.execute("call-run-scoped-caller", {
|
||||
sessionKey: runScopedCallerKey,
|
||||
message: "[TASK-COMPLETE] re-portal occupancy ready",
|
||||
timeoutSeconds: 0,
|
||||
});
|
||||
|
||||
const details = sessionsSendDetails(result.details);
|
||||
expect(details.status).toBe("accepted");
|
||||
expect(details.runId).toBe("durable-fallback-run");
|
||||
expect(details.sessionKey).toBe(runScopedCallerKey);
|
||||
expect(queueMessage).not.toHaveBeenCalled();
|
||||
const agentCalls = calls.filter((call) => call.method === "agent");
|
||||
expect(agentCalls).toHaveLength(1);
|
||||
const params = agentParams(agentCalls[0] ?? {});
|
||||
expect(params.sessionKey).toBe(durableCronCallerKey);
|
||||
expect(params.message).toContain("[Inter-session message]");
|
||||
expect(params.message).toContain("[TASK-COMPLETE] re-portal occupancy ready");
|
||||
});
|
||||
|
||||
it("sessions_send rejects non-cron run-looking keys without durable-session fallback", async () => {
|
||||
const calls: Array<{ method?: string; params?: unknown }> = [];
|
||||
const runScopedCallerKey = "agent:leasing-ops:slack:channel:c-room:run:run-fast";
|
||||
const queueMessage = vi.fn(async () => {});
|
||||
setActiveEmbeddedRun(
|
||||
"caller-active-session",
|
||||
{
|
||||
queueMessage,
|
||||
isStreaming: () => false,
|
||||
isCompacting: () => false,
|
||||
supportsTranscriptCommitWait: true,
|
||||
sourceReplyDeliveryMode: "message_tool_only",
|
||||
abort: () => {},
|
||||
},
|
||||
runScopedCallerKey,
|
||||
);
|
||||
callGatewayMock.mockImplementation(async (opts: unknown) => {
|
||||
const request = opts as { method?: string; params?: unknown };
|
||||
calls.push(request);
|
||||
if (request.method === "agent") {
|
||||
return { runId: "durable-fallback-run", status: "accepted", acceptedAt: 2000 };
|
||||
}
|
||||
return {};
|
||||
});
|
||||
|
||||
const tool = createOpenClawTools({
|
||||
agentSessionKey: "agent:re-portal:main",
|
||||
agentChannel: "telegram",
|
||||
config: {
|
||||
...TEST_CONFIG,
|
||||
session: {
|
||||
...TEST_CONFIG.session,
|
||||
agentToAgent: { maxPingPongTurns: 0 },
|
||||
},
|
||||
},
|
||||
}).find((candidate) => candidate.name === "sessions_send");
|
||||
if (!tool) {
|
||||
throw new Error("missing sessions_send tool");
|
||||
}
|
||||
|
||||
const result = await tool.execute("call-run-scoped-caller", {
|
||||
sessionKey: runScopedCallerKey,
|
||||
message: "[TASK-COMPLETE] re-portal occupancy ready",
|
||||
timeoutSeconds: 0,
|
||||
});
|
||||
|
||||
const details = sessionsSendDetails(result.details);
|
||||
expect(details.status).toBe("error");
|
||||
expect(details.sessionKey).toBe(runScopedCallerKey);
|
||||
expect(details.error).toContain("queue_message_failed reason=not_streaming");
|
||||
expect(queueMessage).not.toHaveBeenCalled();
|
||||
expect(calls.some((call) => call.method === "agent")).toBe(false);
|
||||
});
|
||||
|
||||
it("sessions_send preserves active delivery when transcript commit wait is unsupported", async () => {
|
||||
@@ -1677,7 +1857,7 @@ describe("sessions tools", () => {
|
||||
expect(calls.some((call) => call.method === "agent")).toBe(false);
|
||||
});
|
||||
|
||||
it("sessions_send reports run-scoped fallback admission failures", async () => {
|
||||
it("sessions_send reports run-scoped queue admission failures without gateway fallback", async () => {
|
||||
const runScopedCallerKey = "agent:leasing-ops:cron:monthly-utility:run:run-fast";
|
||||
const queueMessage = vi.fn(async () => {
|
||||
throw new Error("active session ended before queued steering message was committed");
|
||||
@@ -1720,7 +1900,12 @@ describe("sessions tools", () => {
|
||||
expect(details.status).toBe("error");
|
||||
expect(details.sessionKey).toBe(runScopedCallerKey);
|
||||
expect(details.error).toContain("queue_message_failed reason=runtime_rejected");
|
||||
expect(details.error).toContain("fallback_failed error=gateway request timeout for agent");
|
||||
expect(details.error).not.toContain("fallback_failed");
|
||||
expect(
|
||||
callGatewayMock.mock.calls.some(
|
||||
(call) => (call[0] as { method?: string } | undefined)?.method === "agent",
|
||||
),
|
||||
).toBe(false);
|
||||
});
|
||||
|
||||
it("sessions_send preserves terminal timeouts without starting A2A", async () => {
|
||||
|
||||
@@ -73,6 +73,15 @@ const providerEndpointPlugins = vi.hoisted(() => [
|
||||
hosts: ["integrate.api.nvidia.com"],
|
||||
baseUrls: ["https://integrate.api.nvidia.com/v1"],
|
||||
},
|
||||
{
|
||||
endpointClass: "xiaomi-native",
|
||||
hosts: [
|
||||
"api.xiaomimimo.com",
|
||||
"token-plan-ams.xiaomimimo.com",
|
||||
"token-plan-cn.xiaomimimo.com",
|
||||
"token-plan-sgp.xiaomimimo.com",
|
||||
],
|
||||
},
|
||||
],
|
||||
providerRequest: {
|
||||
providers: {
|
||||
@@ -90,6 +99,8 @@ const providerEndpointPlugins = vi.hoisted(() => [
|
||||
openrouter: { family: "openrouter" },
|
||||
qwen: { family: "modelstudio" },
|
||||
together: { family: "together" },
|
||||
xiaomi: { family: "xiaomi" },
|
||||
"xiaomi-token-plan": { family: "xiaomi" },
|
||||
xai: { family: "xai" },
|
||||
zai: { family: "zai" },
|
||||
},
|
||||
@@ -104,6 +115,15 @@ vi.mock("../plugins/plugin-registry.js", () => ({
|
||||
}),
|
||||
}));
|
||||
|
||||
vi.mock("../plugins/manifest-metadata-scan.js", () => ({
|
||||
listOpenClawPluginManifestMetadata: () =>
|
||||
providerEndpointPlugins.map((manifest, index) => ({
|
||||
pluginDir: `provider-endpoint-fixture-${index}`,
|
||||
manifest,
|
||||
origin: "bundled",
|
||||
})),
|
||||
}));
|
||||
|
||||
import {
|
||||
listProviderAttributionPolicies,
|
||||
resolveProviderAttributionHeaders,
|
||||
|
||||
@@ -25,30 +25,59 @@ function writeModelsJsonWithPluginCatalog(params: {
|
||||
root: unknown;
|
||||
pluginRelativePath: string;
|
||||
pluginCatalog: unknown;
|
||||
}): string {
|
||||
return writeModelsJsonWithPluginCatalogs({
|
||||
root: params.root,
|
||||
pluginCatalogs: [
|
||||
{
|
||||
pluginRelativePath: params.pluginRelativePath,
|
||||
pluginCatalog: params.pluginCatalog,
|
||||
},
|
||||
],
|
||||
});
|
||||
}
|
||||
|
||||
function writeModelsJsonWithPluginCatalogs(params: {
|
||||
root: unknown;
|
||||
pluginCatalogs: Array<{
|
||||
pluginRelativePath: string;
|
||||
pluginCatalog: unknown;
|
||||
}>;
|
||||
}): string {
|
||||
const dir = mkdtempSync(join(tmpdir(), "openclaw-model-registry-"));
|
||||
tempDirs.push(dir);
|
||||
const file = join(dir, "models.json");
|
||||
const pluginFile = join(dir, params.pluginRelativePath);
|
||||
mkdirSync(dirname(pluginFile), { recursive: true });
|
||||
writeFileSync(file, JSON.stringify(params.root, null, 2), "utf-8");
|
||||
writeFileSync(pluginFile, JSON.stringify(params.pluginCatalog, null, 2), "utf-8");
|
||||
for (const pluginCatalog of params.pluginCatalogs) {
|
||||
const pluginFile = join(dir, pluginCatalog.pluginRelativePath);
|
||||
mkdirSync(dirname(pluginFile), { recursive: true });
|
||||
writeFileSync(pluginFile, JSON.stringify(pluginCatalog.pluginCatalog, null, 2), "utf-8");
|
||||
}
|
||||
return file;
|
||||
}
|
||||
|
||||
function pluginOwnerSnapshot(providerId: string, pluginId: string, enabled = true) {
|
||||
return pluginOwnerSnapshotEntries([{ providerId, pluginId, enabled }]);
|
||||
}
|
||||
|
||||
function pluginOwnerSnapshotEntries(
|
||||
entries: Array<{ providerId: string; pluginId: string; enabled?: boolean }>,
|
||||
) {
|
||||
// The registry only trusts generated provider shards that are still owned by
|
||||
// an enabled plugin in the current metadata snapshot.
|
||||
return {
|
||||
index: {
|
||||
plugins: [{ pluginId, enabled }],
|
||||
plugins: entries.map((entry) => ({
|
||||
pluginId: entry.pluginId,
|
||||
enabled: entry.enabled ?? true,
|
||||
})),
|
||||
},
|
||||
normalizePluginId: (id: string) => id,
|
||||
owners: {
|
||||
channels: new Map(),
|
||||
channelConfigs: new Map(),
|
||||
providers: new Map([[providerId, [pluginId]]]),
|
||||
modelCatalogProviders: new Map([[providerId, [pluginId]]]),
|
||||
providers: new Map(entries.map((entry) => [entry.providerId, [entry.pluginId]])),
|
||||
modelCatalogProviders: new Map(entries.map((entry) => [entry.providerId, [entry.pluginId]])),
|
||||
cliBackends: new Map(),
|
||||
setupProviders: new Map(),
|
||||
commandAliases: new Map(),
|
||||
@@ -145,6 +174,64 @@ describe("ModelRegistry models.json auth", () => {
|
||||
expect(registry.find("zai", "glm-5.1")?.name).toBe("GLM 5.1");
|
||||
});
|
||||
|
||||
it("isolates invalid generated plugin catalog shards from valid models", () => {
|
||||
const modelsPath = writeModelsJsonWithPluginCatalogs({
|
||||
root: {
|
||||
providers: {
|
||||
custom: {
|
||||
baseUrl: "https://models.example/v1",
|
||||
api: "openai-responses",
|
||||
apiKey: "CUSTOM_API_KEY",
|
||||
models: [{ id: "root-model", name: "Root Model" }],
|
||||
},
|
||||
},
|
||||
},
|
||||
pluginCatalogs: [
|
||||
{
|
||||
pluginRelativePath: join("plugins", "google", PLUGIN_MODEL_CATALOG_FILE),
|
||||
pluginCatalog: {
|
||||
generatedBy: PLUGIN_MODEL_CATALOG_GENERATED_BY,
|
||||
providers: {
|
||||
"google-vertex": {
|
||||
baseUrl: "https://us-central1-aiplatform.googleapis.com/v1",
|
||||
apiKey: "GOOGLE_API_KEY",
|
||||
models: [{ id: "gemini-3.1-pro-preview", name: "Gemini 3.1 Pro" }],
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
{
|
||||
pluginRelativePath: join("plugins", "zai", PLUGIN_MODEL_CATALOG_FILE),
|
||||
pluginCatalog: {
|
||||
generatedBy: PLUGIN_MODEL_CATALOG_GENERATED_BY,
|
||||
providers: {
|
||||
zai: {
|
||||
baseUrl: "https://api.z.ai/api/paas/v4",
|
||||
api: "openai-completions",
|
||||
apiKey: "ZAI_API_KEY",
|
||||
models: [{ id: "glm-5.1", name: "GLM 5.1" }],
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
],
|
||||
});
|
||||
|
||||
const registry = ModelRegistry.create(AuthStorage.inMemory(), modelsPath, {
|
||||
pluginMetadataSnapshot: pluginOwnerSnapshotEntries([
|
||||
{ providerId: "google-vertex", pluginId: "google" },
|
||||
{ providerId: "zai", pluginId: "zai" },
|
||||
]),
|
||||
});
|
||||
|
||||
expect(registry.getError()).toContain(
|
||||
'Provider google-vertex, model gemini-3.1-pro-preview: no "api" specified',
|
||||
);
|
||||
expect(registry.find("custom", "root-model")?.name).toBe("Root Model");
|
||||
expect(registry.find("zai", "glm-5.1")?.name).toBe("GLM 5.1");
|
||||
expect(registry.find("google-vertex", "gemini-3.1-pro-preview")).toBeUndefined();
|
||||
});
|
||||
|
||||
it("preserves model params from generated plugin catalog shards", () => {
|
||||
const modelsPath = writeModelsJsonWithPluginCatalog({
|
||||
root: { providers: {} },
|
||||
|
||||
@@ -21,6 +21,7 @@ import type {
|
||||
} from "../../llm/types.js";
|
||||
import { registerOAuthProvider, resetOAuthProviders } from "../../llm/utils/oauth/index.js";
|
||||
import type { OAuthProviderInterface } from "../../llm/utils/oauth/types.js";
|
||||
import { createSubsystemLogger } from "../../logging/subsystem.js";
|
||||
import { getAgentDir } from "../config.js";
|
||||
import { resolveModelPluginMetadataSnapshot } from "../model-discovery-context.js";
|
||||
import {
|
||||
@@ -38,6 +39,8 @@ import {
|
||||
resolveHeadersOrThrow,
|
||||
} from "./resolve-config-value.js";
|
||||
|
||||
const log = createSubsystemLogger("agents/model-registry");
|
||||
|
||||
// Schema for OpenRouter routing preferences
|
||||
const PercentileCutoffsSchema = Type.Object({
|
||||
p50: Type.Optional(Type.Number()),
|
||||
@@ -355,9 +358,7 @@ export class ModelRegistry {
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Get any error from loading models.json (undefined if no error).
|
||||
*/
|
||||
/** Get any root or generated plugin catalog load error. */
|
||||
getError(): string | undefined {
|
||||
return this.loadError;
|
||||
}
|
||||
@@ -371,7 +372,8 @@ export class ModelRegistry {
|
||||
|
||||
if (error) {
|
||||
this.loadError = error;
|
||||
// Keep the prior empty/default registry shape when models.json failed to load.
|
||||
log.warn(`model catalog load issue: ${error}`);
|
||||
// Plugin catalog failures can return salvaged models; root failures return empty.
|
||||
}
|
||||
|
||||
let combined = customModels;
|
||||
@@ -444,6 +446,7 @@ export class ModelRegistry {
|
||||
}
|
||||
|
||||
const models = this.parseModels(configForUse);
|
||||
const pluginCatalogErrors: string[] = [];
|
||||
if (options.includePluginCatalogs !== false) {
|
||||
for (const pluginCatalog of listPluginModelCatalogFiles(dirname(modelsJsonPath))) {
|
||||
const pluginResult = this.loadCustomModels(pluginCatalog.path, {
|
||||
@@ -452,13 +455,14 @@ export class ModelRegistry {
|
||||
requireGeneratedCatalog: true,
|
||||
});
|
||||
if (pluginResult.error) {
|
||||
return pluginResult;
|
||||
pluginCatalogErrors.push(pluginResult.error);
|
||||
continue;
|
||||
}
|
||||
models.push(...pluginResult.models);
|
||||
}
|
||||
}
|
||||
|
||||
return { models, error: undefined };
|
||||
return { models, error: pluginCatalogErrors.join("\n\n") || undefined };
|
||||
} catch (error) {
|
||||
if (error instanceof SyntaxError) {
|
||||
if (options.requireGeneratedCatalog === true) {
|
||||
|
||||
@@ -345,6 +345,8 @@ async function deliverSlackChannelAnnouncement(params: {
|
||||
queueEmbeddedAgentMessageWithOutcome?: QueueEmbeddedAgentMessageWithOutcome;
|
||||
sendMessage?: typeof runtimeSendMessage;
|
||||
internalEvents?: AgentInternalEvent[];
|
||||
sourceSessionKey?: string;
|
||||
sourceChannel?: string;
|
||||
sourceTool?: string;
|
||||
runtimeConfig?: Record<string, unknown>;
|
||||
}) {
|
||||
@@ -381,6 +383,8 @@ async function deliverSlackChannelAnnouncement(params: {
|
||||
bestEffortDeliver: true,
|
||||
directIdempotencyKey: params.directIdempotencyKey,
|
||||
internalEvents: params.internalEvents,
|
||||
sourceSessionKey: params.sourceSessionKey,
|
||||
sourceChannel: params.sourceChannel,
|
||||
sourceTool: params.sourceTool,
|
||||
});
|
||||
}
|
||||
@@ -4015,8 +4019,21 @@ describe("deliverSubagentAnnouncement completion delivery", () => {
|
||||
expect(sendMessage).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it("directly delivers stale isolated cron run media completions", async () => {
|
||||
const callGateway = createGatewayMock();
|
||||
it("runs inactive isolated cron media completions through the requester agent first", async () => {
|
||||
const callGateway = createGatewayMock({
|
||||
result: {
|
||||
payloads: [{ text: "queued the generated image confirmation" }],
|
||||
messagingToolSentTargets: [
|
||||
{
|
||||
tool: "sessions_send",
|
||||
provider: "slack",
|
||||
to: "channel:C123",
|
||||
text: "The daily media workflow continued after the image callback.",
|
||||
mediaUrls: ["/tmp/generated-daily.png"],
|
||||
},
|
||||
],
|
||||
},
|
||||
});
|
||||
const sendMessage = createSendMessageMock();
|
||||
const queueEmbeddedAgentMessageWithOutcome = createQueueOutcomeMock(true);
|
||||
const result = await deliverSlackChannelAnnouncement({
|
||||
@@ -4044,6 +4061,8 @@ describe("deliverSubagentAnnouncement completion delivery", () => {
|
||||
replyInstruction: "Deliver the generated image through the requester run.",
|
||||
},
|
||||
],
|
||||
sourceSessionKey: "image_generate:task-123",
|
||||
sourceChannel: "internal",
|
||||
});
|
||||
|
||||
expectRecordFields(result, {
|
||||
@@ -4051,7 +4070,71 @@ describe("deliverSubagentAnnouncement completion delivery", () => {
|
||||
path: "direct",
|
||||
});
|
||||
expect(queueEmbeddedAgentMessageWithOutcome).not.toHaveBeenCalled();
|
||||
expect(callGateway).not.toHaveBeenCalled();
|
||||
expect(callGateway).toHaveBeenCalledTimes(1);
|
||||
const params = expectGatewayAgentParams(callGateway, {
|
||||
sessionKey: "agent:main:cron:daily-media:run:run-123",
|
||||
deliver: true,
|
||||
channel: "slack",
|
||||
accountId: "acct-1",
|
||||
to: "channel:C123",
|
||||
idempotencyKey: "announce-stale-cron-media",
|
||||
});
|
||||
expectRecordFields(params.inputProvenance, {
|
||||
kind: "inter_session",
|
||||
sourceSessionKey: "image_generate:task-123",
|
||||
sourceChannel: "internal",
|
||||
sourceTool: "image_generate",
|
||||
});
|
||||
expect(sendMessage).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it("directly delivers inactive isolated cron media only after requester-agent fallback misses media", async () => {
|
||||
const callGateway = createGatewayMock();
|
||||
const sendMessage = createSendMessageMock();
|
||||
const queueEmbeddedAgentMessageWithOutcome = createQueueOutcomeMock(true);
|
||||
const result = await deliverSlackChannelAnnouncement({
|
||||
callGateway,
|
||||
sendMessage,
|
||||
queueEmbeddedAgentMessageWithOutcome,
|
||||
sessionId: "stale-cron-run-session",
|
||||
isActive: false,
|
||||
requesterSessionKey: "agent:main:cron:daily-media:run:run-123",
|
||||
expectsCompletionMessage: true,
|
||||
directIdempotencyKey: "announce-stale-cron-media-fallback",
|
||||
sourceTool: "image_generate",
|
||||
internalEvents: [
|
||||
{
|
||||
type: "task_completion",
|
||||
source: "image_generation",
|
||||
childSessionKey: "image_generate:task-123",
|
||||
childSessionId: "task-123",
|
||||
announceType: "image generation task",
|
||||
taskLabel: "daily media",
|
||||
status: "ok",
|
||||
statusLabel: "completed successfully",
|
||||
result: "Generated 1 image.\nMEDIA:/tmp/generated-daily.png",
|
||||
mediaUrls: ["/tmp/generated-daily.png"],
|
||||
replyInstruction: "Deliver the generated image through the requester run.",
|
||||
},
|
||||
],
|
||||
sourceSessionKey: "image_generate:task-123",
|
||||
sourceChannel: "internal",
|
||||
});
|
||||
|
||||
expectRecordFields(result, {
|
||||
delivered: true,
|
||||
path: "direct",
|
||||
});
|
||||
expect(queueEmbeddedAgentMessageWithOutcome).not.toHaveBeenCalled();
|
||||
expect(callGateway).toHaveBeenCalledTimes(1);
|
||||
expectGatewayAgentParams(callGateway, {
|
||||
sessionKey: "agent:main:cron:daily-media:run:run-123",
|
||||
deliver: true,
|
||||
channel: "slack",
|
||||
accountId: "acct-1",
|
||||
to: "channel:C123",
|
||||
idempotencyKey: "announce-stale-cron-media-fallback",
|
||||
});
|
||||
expect(sendMessage).toHaveBeenCalledWith(
|
||||
expect.objectContaining({
|
||||
channel: "slack",
|
||||
@@ -4059,7 +4142,7 @@ describe("deliverSubagentAnnouncement completion delivery", () => {
|
||||
to: "channel:C123",
|
||||
content: "The generated image is ready.",
|
||||
mediaUrls: ["/tmp/generated-daily.png"],
|
||||
idempotencyKey: "announce-stale-cron-media:generated-media-direct",
|
||||
idempotencyKey: "announce-stale-cron-media-fallback:generated-media-direct",
|
||||
}),
|
||||
);
|
||||
});
|
||||
|
||||
@@ -1387,7 +1387,8 @@ async function sendSubagentAnnounceDirectly(params: {
|
||||
if (
|
||||
params.expectsCompletionMessage &&
|
||||
isCronRunSessionKey(canonicalRequesterSessionKey) &&
|
||||
!resolveRequesterSessionActivity(canonicalRequesterSessionKey).isActive
|
||||
!resolveRequesterSessionActivity(canonicalRequesterSessionKey).isActive &&
|
||||
!agentMediatedCompletion
|
||||
) {
|
||||
const generatedMediaDelivery = await tryGeneratedMediaDirectDelivery();
|
||||
if (generatedMediaDelivery) {
|
||||
|
||||
@@ -180,6 +180,7 @@ export function createSubagentRunManager(params: {
|
||||
stopSweeper(): void;
|
||||
resumeSubagentRun(runId: string): void;
|
||||
clearPendingLifecycleError(runId: string): void;
|
||||
clearPendingLifecycleTimeout(runId: string): void;
|
||||
resolveSubagentWaitTimeoutMs(cfg: OpenClawConfig, runTimeoutSeconds?: number): number;
|
||||
scheduleOrphanRecovery(args?: { delayMs?: number; maxRetries?: number }): void;
|
||||
resolveSubagentSessionCompletion(args: {
|
||||
@@ -264,6 +265,8 @@ export function createSubagentRunManager(params: {
|
||||
waitTerminalOutcome?.reason === "aborted" || waitTerminalOutcome?.reason === "cancelled";
|
||||
const waitStatus = waitTerminalOutcome?.status ?? wait.status;
|
||||
if (wait.yielded === true && waitStatus !== "timeout" && !waitBlocked) {
|
||||
params.clearPendingLifecycleError(runId);
|
||||
params.clearPendingLifecycleTimeout(runId);
|
||||
if (
|
||||
markSubagentRunPausedAfterYield({
|
||||
entry,
|
||||
|
||||
@@ -2000,6 +2000,185 @@ describe("subagent registry seam flow", () => {
|
||||
expect(replacement?.endedAt).toBeUndefined();
|
||||
});
|
||||
|
||||
it("keeps yield terminals paused when the lifecycle event also signals abort (#92448)", async () => {
|
||||
// sessions_yield ends the turn by aborting the run signal, so a depth-1
|
||||
// subagent's yield terminal can arrive carrying yielded plus aborted (or
|
||||
// stopReason="aborted"). The event handler must still pause the run, not
|
||||
// settle it `cancelled` and deliver a false notice to the requester.
|
||||
mocks.callGateway.mockImplementation(async (request: { method?: string }) => {
|
||||
if (request.method === "agent.wait") {
|
||||
return { status: "pending" };
|
||||
}
|
||||
return {};
|
||||
});
|
||||
|
||||
const cases = [
|
||||
{ runId: "run-yield-stopreason-aborted", extra: { stopReason: "aborted" } },
|
||||
{ runId: "run-yield-aborted-flag", extra: { aborted: true } },
|
||||
];
|
||||
|
||||
for (const testCase of cases) {
|
||||
mod.registerSubagentRun({
|
||||
runId: testCase.runId,
|
||||
childSessionKey: `agent:main:subagent:${testCase.runId}`,
|
||||
requesterSessionKey: "agent:main:main",
|
||||
requesterDisplayKey: "main",
|
||||
task: "wait for child continuation",
|
||||
cleanup: "keep",
|
||||
});
|
||||
|
||||
const lastOnAgentEventCall = mocks.onAgentEvent.mock.calls[
|
||||
mocks.onAgentEvent.mock.calls.length - 1
|
||||
] as unknown as
|
||||
| [(evt: { runId: string; stream: string; data: Record<string, unknown> }) => void]
|
||||
| undefined;
|
||||
const lifecycleHandler = lastOnAgentEventCall?.[0];
|
||||
expect(lifecycleHandler).toBeTypeOf("function");
|
||||
|
||||
lifecycleHandler?.({
|
||||
runId: testCase.runId,
|
||||
stream: "lifecycle",
|
||||
data: {
|
||||
phase: "end",
|
||||
startedAt: 111,
|
||||
endedAt: 222,
|
||||
yielded: true,
|
||||
...testCase.extra,
|
||||
},
|
||||
});
|
||||
|
||||
await waitForFast(() => {
|
||||
const run = mod
|
||||
.listSubagentRunsForRequester("agent:main:main")
|
||||
.find((entry) => entry.runId === testCase.runId);
|
||||
expect(run?.pauseReason).toBe("sessions_yield");
|
||||
expect(run?.outcome?.status).not.toBe("error");
|
||||
});
|
||||
}
|
||||
|
||||
// Paused, never killed → no farewell/cancellation notice reaches the requester.
|
||||
expect(mocks.runSubagentAnnounceFlow).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it("cancels a pending grace timer when a yield follows an intermediate aborted terminal (#92448)", async () => {
|
||||
// An earlier aborted terminal schedules a deferred kill grace timer; a
|
||||
// following yield must clear it, or it fires and settles the now-paused run.
|
||||
mocks.callGateway.mockImplementation(async (request: { method?: string }) => {
|
||||
if (request.method === "agent.wait") {
|
||||
return { status: "pending" };
|
||||
}
|
||||
return {};
|
||||
});
|
||||
|
||||
mod.registerSubagentRun({
|
||||
runId: "run-yield-after-pending-timeout",
|
||||
childSessionKey: "agent:main:subagent:pending-timeout",
|
||||
requesterSessionKey: "agent:main:main",
|
||||
requesterDisplayKey: "main",
|
||||
task: "wait for child continuation",
|
||||
cleanup: "keep",
|
||||
});
|
||||
|
||||
const lastOnAgentEventCall = mocks.onAgentEvent.mock.calls[
|
||||
mocks.onAgentEvent.mock.calls.length - 1
|
||||
] as unknown as
|
||||
| [(evt: { runId: string; stream: string; data: Record<string, unknown> }) => void]
|
||||
| undefined;
|
||||
const lifecycleHandler = lastOnAgentEventCall?.[0];
|
||||
expect(lifecycleHandler).toBeTypeOf("function");
|
||||
|
||||
// Intermediate aborted terminal → schedules the deferred kill grace timer.
|
||||
lifecycleHandler?.({
|
||||
runId: "run-yield-after-pending-timeout",
|
||||
stream: "lifecycle",
|
||||
data: { phase: "end", startedAt: 111, endedAt: 222, aborted: true },
|
||||
});
|
||||
// Yield terminal → must pause and cancel the pending grace timer.
|
||||
lifecycleHandler?.({
|
||||
runId: "run-yield-after-pending-timeout",
|
||||
stream: "lifecycle",
|
||||
data: { phase: "end", startedAt: 111, endedAt: 333, yielded: true },
|
||||
});
|
||||
|
||||
await waitForFast(() => {
|
||||
const run = mod
|
||||
.listSubagentRunsForRequester("agent:main:main")
|
||||
.find((entry) => entry.runId === "run-yield-after-pending-timeout");
|
||||
expect(run?.pauseReason).toBe("sessions_yield");
|
||||
});
|
||||
|
||||
// Advancing well past the 15s grace window must not undo the pause.
|
||||
await vi.advanceTimersByTimeAsync(60_000);
|
||||
const run = mod
|
||||
.listSubagentRunsForRequester("agent:main:main")
|
||||
.find((entry) => entry.runId === "run-yield-after-pending-timeout");
|
||||
expect(run?.pauseReason).toBe("sessions_yield");
|
||||
expect(run?.outcome?.status).not.toBe("error");
|
||||
expect(mocks.runSubagentAnnounceFlow).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it("cancels a pending grace timer when agent.wait observes the yield after an aborted terminal (#92448)", async () => {
|
||||
let resolveWait: (value: {
|
||||
status: "ok";
|
||||
startedAt: number;
|
||||
endedAt: number;
|
||||
yielded: true;
|
||||
}) => void = () => {};
|
||||
const waitResult = new Promise<{
|
||||
status: "ok";
|
||||
startedAt: number;
|
||||
endedAt: number;
|
||||
yielded: true;
|
||||
}>((resolve) => {
|
||||
resolveWait = resolve;
|
||||
});
|
||||
mocks.callGateway.mockImplementation(async (request: { method?: string }) => {
|
||||
if (request.method === "agent.wait") {
|
||||
return waitResult;
|
||||
}
|
||||
return {};
|
||||
});
|
||||
|
||||
mod.registerSubagentRun({
|
||||
runId: "run-wait-yield-after-pending-timeout",
|
||||
childSessionKey: "agent:main:subagent:pending-wait-timeout",
|
||||
requesterSessionKey: "agent:main:main",
|
||||
requesterDisplayKey: "main",
|
||||
task: "wait for child continuation through wait",
|
||||
cleanup: "keep",
|
||||
});
|
||||
|
||||
const lastOnAgentEventCall = mocks.onAgentEvent.mock.calls[
|
||||
mocks.onAgentEvent.mock.calls.length - 1
|
||||
] as unknown as
|
||||
| [(evt: { runId: string; stream: string; data: Record<string, unknown> }) => void]
|
||||
| undefined;
|
||||
const lifecycleHandler = lastOnAgentEventCall?.[0];
|
||||
expect(lifecycleHandler).toBeTypeOf("function");
|
||||
|
||||
lifecycleHandler?.({
|
||||
runId: "run-wait-yield-after-pending-timeout",
|
||||
stream: "lifecycle",
|
||||
data: { phase: "end", startedAt: 111, endedAt: 222, aborted: true },
|
||||
});
|
||||
resolveWait({ status: "ok", startedAt: 111, endedAt: 333, yielded: true });
|
||||
|
||||
await waitForFast(() => {
|
||||
const run = mod
|
||||
.listSubagentRunsForRequester("agent:main:main")
|
||||
.find((entry) => entry.runId === "run-wait-yield-after-pending-timeout");
|
||||
expect(run?.pauseReason).toBe("sessions_yield");
|
||||
});
|
||||
|
||||
await vi.advanceTimersByTimeAsync(60_000);
|
||||
const run = mod
|
||||
.listSubagentRunsForRequester("agent:main:main")
|
||||
.find((entry) => entry.runId === "run-wait-yield-after-pending-timeout");
|
||||
expect(run?.pauseReason).toBe("sessions_yield");
|
||||
expect(run?.outcome?.status).not.toBe("timeout");
|
||||
expect(mocks.runSubagentAnnounceFlow).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it("announces blocked agent.wait snapshots as errors instead of success", async () => {
|
||||
mocks.callGateway.mockImplementation(async (request: { method?: string }) => {
|
||||
if (request.method === "agent.wait") {
|
||||
|
||||
@@ -480,7 +480,7 @@ function schedulePendingLifecycleTimeout(params: {
|
||||
if (!entry) {
|
||||
return;
|
||||
}
|
||||
if (entry.outcome?.status === "ok") {
|
||||
if (entry.outcome?.status === "ok" || entry.pauseReason === "sessions_yield") {
|
||||
return;
|
||||
}
|
||||
const completionParams = {
|
||||
@@ -1106,6 +1106,25 @@ function ensureListener() {
|
||||
});
|
||||
return;
|
||||
}
|
||||
// sessions_yield ends the turn by aborting the run signal, so a yielded
|
||||
// terminal can also look aborted. An explicit yield is authoritative — pause,
|
||||
// don't kill — else the tracking task settles `cancelled` with a false notice (#92448).
|
||||
if (evt.data?.yielded === true) {
|
||||
// Drop any grace timer from an earlier aborted/error terminal so it can't
|
||||
// later fire and settle this now-paused run with a false notice.
|
||||
clearPendingLifecycleError(evt.runId);
|
||||
clearPendingLifecycleTimeout(evt.runId);
|
||||
if (
|
||||
markSubagentRunPausedAfterYield({
|
||||
entry,
|
||||
endedAt,
|
||||
startedAt: startedAt ?? entry.startedAt,
|
||||
})
|
||||
) {
|
||||
persistSubagentRuns();
|
||||
}
|
||||
return;
|
||||
}
|
||||
if (isAbortedAgentStopReason(stopReason)) {
|
||||
clearPendingLifecycleError(evt.runId);
|
||||
clearPendingLifecycleTimeout(evt.runId);
|
||||
@@ -1154,18 +1173,6 @@ function ensureListener() {
|
||||
});
|
||||
return;
|
||||
}
|
||||
if (evt.data?.yielded === true) {
|
||||
if (
|
||||
markSubagentRunPausedAfterYield({
|
||||
entry,
|
||||
endedAt,
|
||||
startedAt: startedAt ?? entry.startedAt,
|
||||
})
|
||||
) {
|
||||
persistSubagentRuns();
|
||||
}
|
||||
return;
|
||||
}
|
||||
clearPendingLifecycleError(evt.runId);
|
||||
clearPendingLifecycleTimeout(evt.runId);
|
||||
const completionParams = {
|
||||
@@ -1203,6 +1210,7 @@ const subagentRunManager = createSubagentRunManager({
|
||||
stopSweeper,
|
||||
resumeSubagentRun,
|
||||
clearPendingLifecycleError,
|
||||
clearPendingLifecycleTimeout,
|
||||
resolveSubagentWaitTimeoutMs,
|
||||
scheduleOrphanRecovery: (args) => scheduleSubagentOrphanRecovery(args),
|
||||
resolveSubagentSessionCompletion,
|
||||
|
||||
@@ -21,6 +21,7 @@ import {
|
||||
toAgentStoreSessionKey,
|
||||
} from "../../routing/session-key.js";
|
||||
import { annotateInterSessionPromptText } from "../../sessions/input-provenance.js";
|
||||
import { isCronRunSessionKey, parseAgentSessionKey } from "../../sessions/session-key-utils.js";
|
||||
import { SESSION_LABEL_MAX_LENGTH } from "../../sessions/session-label.js";
|
||||
import { stripFormattedReasoningMessage } from "../../shared/text/formatted-reasoning-message.js";
|
||||
import {
|
||||
@@ -30,6 +31,7 @@ import {
|
||||
import { listAgentIds } from "../agent-scope.js";
|
||||
import {
|
||||
type EmbeddedAgentQueueMessageOptions,
|
||||
type EmbeddedAgentQueueMessageOutcome,
|
||||
formatEmbeddedAgentQueueFailureSummary,
|
||||
queueEmbeddedAgentMessageWithOutcomeAsync,
|
||||
resolveActiveEmbeddedRunSessionId,
|
||||
@@ -92,11 +94,6 @@ function normalizeSessionsSendArguments(args: unknown): Record<string, unknown>
|
||||
return params;
|
||||
}
|
||||
|
||||
function resolveRunScopedFallbackSessionKey(sessionKey: string): string | undefined {
|
||||
const match = /^(agent:[^:]+:.+):run:[^:]+$/.exec(sessionKey.trim());
|
||||
return match?.[1];
|
||||
}
|
||||
|
||||
function resolveConfiguredAgentMainSessionKey(params: {
|
||||
cfg: OpenClawConfig;
|
||||
agentId: string;
|
||||
@@ -204,13 +201,51 @@ function isPendingErrorAgentWaitTimeout(result: AgentWaitResult): boolean {
|
||||
);
|
||||
}
|
||||
|
||||
function isRunScopedAgentSessionKey(sessionKey: string): boolean {
|
||||
const parsed = parseAgentSessionKey(normalizeOptionalString(sessionKey));
|
||||
return Boolean(parsed && /(?:^|:)run:[^:]+(?::|$)/.test(parsed.rest));
|
||||
}
|
||||
|
||||
function resolveCronRunScopedFallbackSessionKey(sessionKey: string): string | undefined {
|
||||
const normalizedSessionKey = normalizeOptionalString(sessionKey);
|
||||
if (!normalizedSessionKey || !isCronRunSessionKey(normalizedSessionKey)) {
|
||||
return undefined;
|
||||
}
|
||||
const parsed = parseAgentSessionKey(normalizedSessionKey);
|
||||
if (!parsed) {
|
||||
return undefined;
|
||||
}
|
||||
const runMarker = ":run:";
|
||||
const runMarkerIndex = parsed.rest.lastIndexOf(runMarker);
|
||||
if (runMarkerIndex <= 0) {
|
||||
return undefined;
|
||||
}
|
||||
const runId = parsed.rest.slice(runMarkerIndex + runMarker.length);
|
||||
if (!runId || runId.includes(":")) {
|
||||
return undefined;
|
||||
}
|
||||
const fallbackRest = parsed.rest.slice(0, runMarkerIndex);
|
||||
if (!fallbackRest) {
|
||||
return undefined;
|
||||
}
|
||||
return `agent:${parsed.agentId}:${fallbackRest}`;
|
||||
}
|
||||
|
||||
function shouldFallbackCronRunScopedActiveDelivery(
|
||||
outcome: EmbeddedAgentQueueMessageOutcome,
|
||||
): boolean {
|
||||
return (
|
||||
!outcome.queued && (outcome.reason === "not_streaming" || outcome.reason === "no_active_run")
|
||||
);
|
||||
}
|
||||
|
||||
async function startAgentRun(params: {
|
||||
callGateway: GatewayCaller;
|
||||
runId: string;
|
||||
sendParams: Record<string, unknown>;
|
||||
sessionKey: string;
|
||||
deliveryTimeoutMs?: number;
|
||||
allowActiveRunQueueFallback?: boolean;
|
||||
allowActiveRunQueueDelivery?: boolean;
|
||||
}): Promise<
|
||||
| {
|
||||
ok: true;
|
||||
@@ -222,15 +257,13 @@ async function startAgentRun(params: {
|
||||
| { ok: false; result: ReturnType<typeof jsonResult> }
|
||||
> {
|
||||
try {
|
||||
const activeRunSessionId = params.allowActiveRunQueueFallback
|
||||
? resolveActiveEmbeddedRunSessionId(params.sessionKey)
|
||||
: undefined;
|
||||
const fallbackSessionKey = activeRunSessionId
|
||||
? resolveRunScopedFallbackSessionKey(params.sessionKey)
|
||||
: undefined;
|
||||
const activeRunSessionId =
|
||||
params.allowActiveRunQueueDelivery && isRunScopedAgentSessionKey(params.sessionKey)
|
||||
? resolveActiveEmbeddedRunSessionId(params.sessionKey)
|
||||
: undefined;
|
||||
const messageText =
|
||||
typeof params.sendParams.message === "string" ? params.sendParams.message : undefined;
|
||||
if (activeRunSessionId && fallbackSessionKey && messageText) {
|
||||
if (activeRunSessionId && messageText) {
|
||||
const sourceReplyDeliveryMode =
|
||||
params.sendParams.sourceReplyDeliveryMode === "automatic" ||
|
||||
params.sendParams.sourceReplyDeliveryMode === "message_tool_only"
|
||||
@@ -260,7 +293,8 @@ async function startAgentRun(params: {
|
||||
if (queueOutcome.queued) {
|
||||
return { ok: true, runId: params.runId, activeRunQueue: true };
|
||||
}
|
||||
try {
|
||||
const fallbackSessionKey = resolveCronRunScopedFallbackSessionKey(params.sessionKey);
|
||||
if (fallbackSessionKey && shouldFallbackCronRunScopedActiveDelivery(queueOutcome)) {
|
||||
const response = await params.callGateway<{ runId: string }>({
|
||||
method: "agent",
|
||||
params: {
|
||||
@@ -277,13 +311,10 @@ async function startAgentRun(params: {
|
||||
a2aSessionKey: fallbackSessionKey,
|
||||
a2aDisplayKey: fallbackSessionKey,
|
||||
};
|
||||
} catch (err) {
|
||||
const queueSummary =
|
||||
formatEmbeddedAgentQueueFailureSummary(queueOutcome) ?? "active run queue rejected";
|
||||
throw new Error(`${queueSummary}; fallback_failed error=${formatErrorMessage(err)}`, {
|
||||
cause: err,
|
||||
});
|
||||
}
|
||||
const queueSummary =
|
||||
formatEmbeddedAgentQueueFailureSummary(queueOutcome) ?? "active run queue rejected";
|
||||
throw new Error(queueSummary);
|
||||
}
|
||||
const response = await params.callGateway<{ runId: string }>({
|
||||
method: "agent",
|
||||
@@ -643,7 +674,7 @@ export function createSessionsSendTool(opts?: {
|
||||
sendParams,
|
||||
sessionKey: displayKey,
|
||||
deliveryTimeoutMs: announceTimeoutMs,
|
||||
allowActiveRunQueueFallback: true,
|
||||
allowActiveRunQueueDelivery: true,
|
||||
});
|
||||
if (!start.ok) {
|
||||
return start.result;
|
||||
|
||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user