diff --git a/docs/refactor/database-first.md b/docs/refactor/database-first.md new file mode 100644 index 000000000000..259f5aac3cb7 --- /dev/null +++ b/docs/refactor/database-first.md @@ -0,0 +1,2256 @@ +--- +summary: "Migration plan for making SQLite the primary durable state and cache layer while keeping config file-backed" +title: "Database-first state refactor" +read_when: + - Moving OpenClaw runtime data, cache, transcripts, task state, or scratch files into SQLite + - Designing doctor migrations from legacy JSON or JSONL files + - Changing backup, restore, VFS, or worker storage behavior + - Removing session locks, pruning, truncation, or JSON compatibility paths +--- + +# Database-First State Refactor + +## Decision + +Use a two-level SQLite layout: + +- Global database: `~/.openclaw/state/openclaw.sqlite` +- Agent database: one SQLite database per agent for agent-owned workspace, + transcript, VFS, artifact, and large per-agent runtime state +- Configuration stays file-backed: `openclaw.json` remains outside the + database. Runtime auth profiles move to SQLite; external provider or CLI + credential files remain owner-managed outside OpenClaw's database. + +The global database is the control-plane database. It owns agent discovery, +shared gateway state, pairing, device/node state, task and flow ledgers, plugin +state, scheduler runtime state, backup metadata, and migration state. + +The agent database is the data-plane database. It owns the agent's session +metadata, transcript event stream, VFS workspace or scratch namespace, tool +artifacts, run artifacts, and searchable/indexable agent-local cache data. + +This gives one durable global view without forcing large agent workspaces, +transcripts, and binary scratch data into the shared gateway write lane. + +## Hard Contract + +This migration has one canonical runtime shape: + +- Session rows persist session metadata only. They must not persist + `transcriptLocator`, transcript file paths, sibling JSONL paths, lock paths, + pruning metadata, or file-era compatibility pointers. +- Transcript identity is always SQLite identity: `{agentId, sessionId}` plus + optional topic metadata where the protocol needs it. +- `sqlite-transcript://...` is not a runtime or protocol identity. New code must + not derive, persist, pass, parse, or migrate transcript locators. Runtime and + tests should not contain pseudo-locators at all; docs may mention the string + only to ban it. +- Legacy `sessions.json`, transcript JSONL, `.jsonl.lock`, pruning, truncation, + and old session-path logic belong only to the doctor migration/import path. +- Legacy session config aliases belong only to doctor migration. Runtime does + not interpret `session.idleMinutes`, `session.resetByType.dm`, or + cross-agent `agent:main:*` main-session aliases for another configured agent. +- Session routing identity is typed relational state. Hot runtime and UI paths + should read `sessions.session_scope`, `sessions.account_id`, + `sessions.primary_conversation_id`, `conversations`, and + `session_conversations`; they must not parse `session_key` or mine + `session_entries.entry_json` for provider identity except as a compatibility + shadow while old call sites are being deleted. +- Channel-level direct-message markers such as `dm` versus `direct` are routing + vocabulary, not transcript locators or file-store compatibility handles. +- Legacy hook handler config belongs only to doctor warning/migration surfaces. + Runtime must not load `hooks.internal.handlers`; hooks run through discovered + hook directories and `HOOK.md` metadata only. +- Runtime startup, hot reply paths, compaction, reset, recovery, diagnostics, + TTS, memory hooks, subagents, plugin command routing, protocol boundaries, and + hooks must pass `{agentId, sessionId}` through the runtime. +- Tests should seed and assert SQLite transcript rows through + `{agentId, sessionId}`. Tests that only prove JSONL path forwarding, + caller-supplied locator preservation, or transcript-file compatibility should + be deleted unless they cover doctor import, non-session support/debug + materialization, or protocol shape. +- `runEmbeddedPiAgent(...)`, prepared worker runs, and the inner embedded + attempt must not accept transcript locators. They open the SQLite transcript + manager by `{agentId, sessionId}` and pass that manager to the internalized + PI-compatible agent session, so stale callers cannot make the runner write + JSON/JSONL transcripts. +- Runner diagnostics must store runtime/cache/payload trace records in SQLite. + Runtime diagnostics must not expose JSONL file override knobs or generic + transcript JSONL export helpers; user-facing exports can materialize explicit + artifacts from database rows without feeding file names back into runtime. +- Raw stream logging uses `OPENCLAW_RAW_STREAM=1` plus SQLite diagnostics rows. + The old pi-mono `PI_RAW_STREAM`, `PI_RAW_STREAM_PATH`, and + `raw-openai-completions.jsonl` file logger contract is not part of OpenClaw + runtime or tests. +- QMD memory indexing must not export SQLite transcripts to markdown files. + QMD indexes configured memory files only; session transcript search stays + SQLite-backed. +- The QMD SDK subpath is QMD-only for new code. SQLite session transcript + indexing helpers live on `memory-core-host-engine-session-transcripts`; any + QMD re-export is compatibility only and must not be used by runtime code. +- Built-in memory indexes live in the owning agent database. Runtime config and + resolved runtime contracts must not expose `memorySearch.store.path`; doctor + deletes that legacy config key and current code passes the agent + `databasePath` internally. + +Implementation work should keep deleting code until these statements are true +without exceptions outside doctor/import/export/debug boundaries. + +## Goal state and progress + +### Hard goal + +- One global SQLite database owns control-plane state: + `state/openclaw.sqlite`. +- One per-agent SQLite database owns data-plane state: + `agents//agent/openclaw-agent.sqlite`. +- Config remains file-backed. `openclaw.json` is not part of this database + refactor. +- Legacy files are doctor migration inputs only. +- Runtime never writes or reads session or transcript JSONL as active state. + +### Goal states + +- `not-started`: file-era runtime code still writes active state. +- `migrating`: doctor/import code can move file data into SQLite. +- `dual-read`: temporary bridge reads both SQLite and legacy files. This state + is forbidden for this refactor unless it is explicitly documented as + doctor-only. +- `sqlite-runtime`: runtime reads and writes SQLite only. +- `clean`: legacy runtime APIs and tests are removed, and the guard prevents + regressions. +- `done`: docs, tests, backup, doctor migration, and changed checks prove the + clean state. + +### Current state + +- Sessions: `clean` for runtime. Session rows live in the per-agent database, + runtime APIs use `{agentId, sessionId}` or `{agentId, sessionKey}`, and + `sessions.json` is doctor-only legacy input. +- Transcripts: `clean` for runtime. Transcript events, identities, snapshots, + and trajectory runtime events live in the per-agent database. Runtime no + longer accepts transcript locators or JSONL transcript paths. +- PI embedded runner: `clean`. Embedded PI runs, prepared workers, compaction, + and retry loops use SQLite session scope and reject stale transcript handles. +- Cron: `clean` for runtime. Runtime uses `cron_jobs` and `cron_run_logs`; + runtime tests use SQLite `storeKey` naming, and file-era cron paths remain in + doctor legacy migration tests only. +- Task registry: `clean`. Task and Task Flow runtime rows live in + `state/openclaw.sqlite`; unshipped sidecar SQLite importers are deleted. +- Plugin state: `clean`. Plugin state/blob rows live in the shared global + database; old plugin-state sidecar SQLite helpers are guarded against. +- Memory: `sqlite-runtime` for built-in memory and session transcript indexing. + Memory index tables live in the per-agent database, plugin memory state uses + shared plugin-state rows, and legacy memory files are doctor migration inputs + or user workspace content. +- Backup: `sqlite-runtime`. Backup stages compact SQLite snapshots, omits live + WAL/SHM sidecars, verifies SQLite integrity, and records backup runs in the + global database. +- Doctor migration: `migrating`, intentionally. Doctor imports legacy JSON, + JSONL, and retired sidecar stores into SQLite, records migration runs/sources, + and removes successful sources. +- E2E scripts: `clean` for runtime coverage. Docker MCP seeding writes SQLite + rows. The runtime-context Docker script creates legacy JSONL only inside the + doctor migration seed and names the legacy session index path explicitly. + +### Remaining work + +- [x] Rename cron runtime-test store variables away from `storePath` unless + they are doctor legacy inputs. + Files: `src/cron/service.test-harness.ts`, + `src/cron/service.runs-one-shot-main-job-disables-it.test.ts`, + `src/cron/service/timer.regression.test.ts`, + `src/cron/service/ops.test.ts`, `src/cron/service/store.test.ts`, + `src/cron/service.heartbeat-ok-summary-suppressed.test.ts`, + `src/cron/service.main-job-passes-heartbeat-target-last.test.ts`, + `src/cron/store.test.ts`. + Proof: `pnpm check:database-first-legacy-stores`; `rg -n 'storePath' src/cron --glob '!**/commands/doctor/**'`. +- [x] Remove or rename obsolete file-era export test mocks. + File: `src/auto-reply/reply/commands-export-test-mocks.ts`. + Proof: `rg -n 'resolveSessionFilePath|sessionFile|storePath|transcriptLocator' src/auto-reply/reply`. +- [x] Make the Docker runtime-context legacy JSONL seed obviously doctor-only. + File: `scripts/e2e/session-runtime-context-docker-client.ts`. + Proof: `rg -n 'sessions\\.json|sessionFile|\\.jsonl' scripts/e2e/session-runtime-context-docker-client.ts` shows only + `seedBrokenLegacySessionForDoctorMigration`. +- [x] Keep Kysely generated types aligned after any schema change. + Files: `src/state/openclaw-state-schema.sql`, + `src/state/openclaw-agent-schema.sql`, + `src/state/*generated*`. + Proof: no schema change in this pass; `pnpm db:kysely:check`; + `pnpm lint:kysely`. +- [x] Re-run focused tests for touched stores, commands, and scripts. + Proof: `pnpm test src/cron/service/store.test.ts src/cron/store.test.ts src/cron/service.heartbeat-ok-summary-suppressed.test.ts src/cron/service.main-job-passes-heartbeat-target-last.test.ts src/cron/service.every-jobs-fire.test.ts src/cron/service.persists-delivered-status.test.ts src/cron/service.runs-one-shot-main-job-disables-it.test.ts src/cron/service/ops.test.ts src/cron/service/timer.regression.test.ts src/auto-reply/reply/commands-export-trajectory.test.ts extensions/telegram/src/thread-bindings.test.ts extensions/slack/src/monitor/message-handler/prepare.test.ts src/acp/translator.session-lineage-meta.test.ts`; `git diff --check`. +- [x] Before declaring `done`, run the changed gate or remote broad proof. + Proof: `pnpm check:changed --timed -- ` passed on + Hetzner Crabbox run `run_3f1cabf6b25c` after temporary Node 24/pnpm setup and + explicit path routing for the synced no-`.git` workspace. + +### Do not regress + +- No transcript locators. +- No active session files. +- No fake JSONL test fixtures except doctor legacy migration tests. +- No raw SQLite access where Kysely is expected. +- No new legacy DB migrations. This layout has not shipped; keep schema version + at `1` unless there is a strong reason. + +## Code-Read Assumptions + +No follow-up product decisions are blocking this plan. The implementation should +proceed with these assumptions: + +- Use `node:sqlite` directly and require the Node 22+ runtime for this storage + path. +- Keep exactly one normal configuration file. Do not move config, plugin + manifests, or Git workspaces into SQLite in this refactor. +- Runtime compatibility files are not required. Legacy JSON and JSONL files are + migration inputs only. The branch-local SQLite sidecars never shipped and are + deleted instead of imported. +- `openclaw doctor --fix` owns the legacy file-to-database migration step. + Runtime startup and `openclaw migrate` should not carry legacy OpenClaw + database-upgrade paths. +- Credential compatibility follows the same rule: runtime credentials live in + SQLite. Old `auth-profiles.json`, per-agent `auth.json`, and shared + `credentials/oauth.json` files are doctor migration inputs, then removed + after import. +- Generated model catalog state is database-backed. Runtime code must not write + `agents//agent/models.json`; existing `models.json` files are legacy + doctor inputs and are removed after import into `agent_model_catalogs`. +- Runtime must not migrate, normalize, or bridge transcript locators. Active + transcript identity is `{agentId, sessionId}` in SQLite. File paths are + legacy doctor inputs only, and `sqlite-transcript://...` must disappear from + runtime, protocol, hook, and plugin surfaces instead of being treated as a + boundary handle. +- Runtime SQLite transcript reads do not run old JSONL entry-shape migrations or + rewrite whole transcripts for compatibility. Legacy entry normalization stays in + explicit doctor/import utilities. Doctor normalizes legacy JSONL transcript + files before inserting SQLite rows; current runtime rows are + already written in the current transcript schema. Trajectory/session export + reads those rows as-is and must not perform export-time legacy migrations. +- Legacy transcript JSONL parse/migration helpers are doctor-only. Runtime + transcript format code builds current SQLite transcript context only; doctor + owns old JSONL entry upgrades before inserting rows. +- The old runtime-owned JSONL transcript streaming helper was deleted. Doctor + import code owns explicit legacy file reads; runtime session history reads + SQLite rows. +- Codex app-server bindings use the OpenClaw `sessionId` as the canonical + key in the Codex plugin-state namespace. `sessionKey` is metadata for + routing/display and must not replace the durable session id or resurrect + transcript-file identity. +- Context engines receive the current runtime contract directly. The registry + must not wrap engines with retry shims that delete `sessionKey`, + `transcriptScope`, or `prompt`; engines that cannot accept the current + database-first params should fail loudly instead of being bridged. +- Backup output should remain one archive file. Database contents should enter + that archive as compact SQLite snapshots, not raw live WAL sidecars. +- Transcript search is useful but not required for the first database-first + cut. Design the schema so FTS can be added later. +- Worker execution should stay experimental behind settings while the database + boundary settles. + +## Code-Read Findings + +The current branch is already past the proof-of-concept stage. The shared +database exists, Node `node:sqlite` is wired through a small runtime helper, and +former stores now write to `state/openclaw.sqlite` or the owning +`openclaw-agent.sqlite` database. + +The remaining work is not choosing SQLite; it is keeping the new boundary clean +and deleting any compatibility-shaped interfaces that still look like the old +file world: + +- Session `storePath` is no longer a runtime identity, test fixture shape, or + status payload field. Runtime and bridge tests no longer contain the + `storePath` contract name; doctor/migration code owns that legacy vocabulary. +- Session writes no longer pass through the old in-process `store-writer.ts` + queue. SQLite patch writes use conflict detection and bounded retry instead. +- Legacy path discovery still has valid migration uses, but runtime code should + stop treating `sessions.json` and transcript JSONL files as possible write + targets. +- Agent-owned tables live in per-agent SQLite databases. The global DB keeps + registry/control-plane rows; transcript identity is `{agentId, sessionId}` in + the per-agent transcript rows. Runtime code must not persist transcript file + paths or migrate transcript locators. +- Doctor already imports several legacy files. The cleanup is to make that a + single explicit migration implementation that doctor calls, with a durable + migration report. + +No additional product questions are blocking implementation. + +## Current Code Shape + +The branch already has a real shared SQLite base: + +- The runtime floor is now Node 22+: `package.json`, the CLI runtime guard, + installer defaults, macOS runtime locator, CI, and public install docs all + agree. The old Node 22 compatibility lane is removed. +- `src/state/openclaw-state-db.ts` opens `openclaw.sqlite`, sets WAL, + `synchronous=NORMAL`, `busy_timeout=30000`, `foreign_keys=ON`, and applies + the generated schema module derived from + `src/state/openclaw-state-schema.sql`. +- Kysely table types and runtime schema modules are generated from disposable + SQLite databases created from the committed `.sql` files; runtime code no + longer keeps copy-pasted schema strings for global, per-agent, or proxy + capture databases. +- Runtime stores derive selected and inserted row types from those generated + Kysely `DB` interfaces instead of shadowing SQLite row shapes by hand. Raw SQL + remains limited to schema application, pragmas, and migration-only DDL. +- The SQLite schemas are collapsed to `user_version = 1` because this database + layout has not shipped yet. Runtime openers create the current schema only; + file-to-database import remains in doctor code, and branch-local + database upgrade helpers have been deleted. +- Relational ownership is enforced where the ownership boundary is canonical: + source migration rows cascade from `migration_runs`, task delivery state + cascades from `task_runs`, and transcript identity rows cascade from + transcript events. +- Current shared tables include `agent_databases`, + `auth_profile_stores`, `auth_profile_state`, + `plugin_state_entries`, `plugin_blob_entries`, `media_blobs`, + `skill_uploads`, `capture_sessions`, `capture_events`, `capture_blobs`, + `sandbox_registry_entries`, `cron_run_logs`, `cron_jobs`, `commitments`, + `delivery_queue_entries`, `model_capability_cache`, + `workspace_setup_state`, `native_hook_relay_bridges`, + `current_conversation_bindings`, `plugin_binding_approvals`, + `tui_last_sessions`, `task_runs`, `task_delivery_state`, `flow_runs`, + `subagent_runs`, `migration_runs`, and `backup_runs`. +- Arbitrary plugin-owned state does not get host-owned typed tables. Installed + plugins use `plugin_state_entries` for versioned JSON payloads and + `plugin_blob_entries` for bytes, with namespace/key ownership, TTL cleanup, + backup, and plugin migration records. Host-owned plugin orchestration state can + still have typed tables when the host owns the query contract, such as + `plugin_binding_approvals`. +- Plugin migrations are data migrations over plugin-owned namespaces, not host + schema migrations. A plugin can migrate its own versioned state/blob entries + through a migration provider, and the host records source/run status in the + normal migration ledger. New plugin installs do not require changing + `openclaw-state-schema.sql` unless the host itself is taking ownership of a + new cross-plugin contract. +- `src/state/openclaw-agent-db.ts` opens + `agents//agent/openclaw-agent.sqlite`, registers the database in the + global DB, and owns agent-local session, transcript, VFS, artifact, cache, + and memory-index tables. Shared runtime discovery now reads the generated-typed + `agent_databases` registry instead of reimplementing that query at each call + site. +- Global and per-agent databases record a `schema_meta` row with database role, + schema version, timestamps, and agent id for agent databases. The layout still + stays at `user_version = 1` because this SQLite schema has not shipped yet. +- Per-agent session identity now has a canonical `sessions` root table keyed by + `session_id`, with `session_key`, `session_scope`, `account_id`, + `primary_conversation_id`, timestamps, display fields, model metadata, + harness id, and parent/spawn linkage as queryable columns. `session_routes` + is the unique active route index from `session_key` to the current + `session_id`, so a route key can move to a fresh durable session without + making hot reads pick between duplicate `sessions.session_key` rows. The old + `session_entries.entry_json` compatibility-shaped payload hangs off the + durable `session_id` root by foreign key; it is no longer the only + schema-level representation of a session. +- Per-agent external conversation identity is relational too: + `conversations` stores normalized provider/account/conversation identity, and + `session_conversations` links one OpenClaw session to one or more external + conversations. This covers shared-main DM sessions where multiple peers can + intentionally map to one session without lying in `session_key`. SQLite also + enforces uniqueness for the natural provider identity so the same + channel/account/kind/peer/thread tuple cannot fork across conversation ids. + Shared-main direct peers are linked with a `participant` role, so one + OpenClaw session can represent multiple external DM peers without demoting + older peers into vague related rows. `sessions.primary_conversation_id` still + points at the current typed delivery target. Closed routing/status columns + are enforced with SQLite `CHECK` constraints instead of relying only on + TypeScript unions. + Runtime session projection clears compatibility routing shadows from + `session_entries.entry_json` before applying typed session/conversation + columns, so stale JSON payloads cannot resurrect delivery targets. + Subagent announce routing likewise requires the typed SQLite delivery context; + it no longer falls back to compatibility `SessionEntry` route fields. + Gateway `chat.send` explicit delivery inheritance reads the typed SQLite + delivery context instead of `origin`/`last*` compatibility fields. + `tools.effective` likewise derives provider/account/thread context from typed + SQLite delivery/routing rows, not stale `last*` session-entry shadows. + System-event prompt context rebuilds channel/to/account/thread fields from + typed delivery fields instead of `origin` shadows. + The shared `deliveryContextFromSession` helper and session-to-conversation + mapper now ignore `SessionEntry.origin` entirely; only typed delivery fields + and relational conversation rows can create hot route identity. + Runtime session entry normalization strips `origin` before persisting or + projecting `entry_json`, and inbound metadata writes typed channel/chat + fields plus relational conversation rows instead of creating new origin + shadows. +- Transcript events, transcript snapshots, and trajectory runtime events now + reference the canonical per-agent `sessions` root and cascade on session + deletion. Transcript identity/idempotency rows continue to cascade from the + exact transcript event row. +- Memory-core indexes now use explicit agent-database tables + `memory_index_meta`, `memory_index_sources`, `memory_index_chunks`, and + `memory_embedding_cache`; optional FTS/vector side indexes use the same + `memory_index_*` prefix instead of generic `meta`, `files`, `chunks`, or + `chunks_vec` tables. `memory_index_sources` is keyed by + `(source_kind, source_key)` and carries optional `session_id` ownership, so + session-derived sources and chunks cascade when a session is deleted. Cached + chunk embeddings are stored as Float32 SQLite BLOBs, not JSON text arrays. + These tables are derived/search cache, not canonical transcript storage; they + can be deleted and rebuilt from `sessions`, `transcript_events`, and memory + workspace files. +- Subagent run recovery state now lives in typed shared `subagent_runs` rows + with indexed child, requester, and controller session keys. The old + `subagents/runs.json` file is doctor migration input only. +- Current conversation bindings now live in typed shared + `current_conversation_bindings` rows keyed by normalized conversation id, with + target agent/session columns, conversation kind, status, expiry, and metadata + stored as relational columns instead of a duplicated opaque binding record. + The durable binding key includes the normalized conversation kind so + direct/group/channel refs cannot collide, and SQLite rejects invalid binding + kind/status values. The old + `bindings/current-conversations.json` file is doctor migration input only. +- Delivery queue recovery now overlays typed queue columns for channel, target, + account, session, retry, error, platform-send, and recovery state onto the + replay JSON. `entry_json` keeps the replay payloads, hooks, and formatting + payload, but typed columns are authoritative for hot queue routing/state. +- TUI last-session restore pointers now live in typed shared + `tui_last_sessions` rows keyed by the hashed TUI connection/session scope. + The old TUI JSON file is doctor migration input only. +- Default TTS prefs now live in shared plugin-state SQLite rows keyed under the + `speech-core` plugin. The old `settings/tts.json` file is doctor migration + input only; runtime no longer reads or writes TTS prefs JSON files, and the + legacy path resolver lives in the doctor migration module. +- Secret target metadata now talks about stores instead of pretending every + credential target is a config file. `openclaw.json` remains the config store; + auth-profile targets use typed SQLite `auth_profile_stores` rows with + provider-shaped credentials kept as JSON payloads. +- Secret audit no longer scans retired per-agent `auth.json` files. Doctor owns + warning about, importing, and removing that legacy file. +- Legacy auth profile path helpers now live in doctor legacy code. Core auth + profile path helpers expose SQLite auth-store identity and display locations, + not `auth-profiles.json` or `auth-state.json` runtime paths. +- Subagent run recovery and OpenRouter model capability cache runtime modules + now keep SQLite snapshot readers/writers separate from doctor-only legacy JSON + import helpers. OpenRouter capabilities use the typed generic + `model_capability_cache` rows under `provider_id = "openrouter"` instead of + one opaque cache blob or a provider-specific host table. Subagent run + `taskName` is stored in the typed `subagent_runs.task_name` column; the + `payload_json` copy is replay/debug data, not the source for hot display or + lookup fields. +- `src/agents/filesystem/virtual-agent-fs.sqlite.ts` implements a SQLite VFS + over the agent database `vfs_entries` table. Directory reads, recursive + exports, deletes, and renames use indexed `(namespace, path)` prefix ranges + instead of scanning a whole namespace or relying on `LIKE` path matching. +- `src/agents/runtime-worker.entry.ts` creates per-run SQLite VFS, tool artifact, + run artifact, and scoped cache stores for workers. +- Workspace bootstrap completion markers now live in typed shared + `workspace_setup_state` rows keyed by resolved workspace path instead of + `.openclaw/workspace-state.json`; runtime no longer reads or rewrites the + legacy workspace marker, and helper APIs no longer pass around a fake + `.openclaw/setup-state` path just to derive storage identity. +- Exec approvals now live in the typed shared SQLite `exec_approvals_config` + singleton row. Doctor imports legacy `~/.openclaw/exec-approvals.json`; + runtime writes no longer create, rewrite, or report that file as its active + store location. The macOS companion reads and writes the same + `state/openclaw.sqlite` table row; it keeps only the Unix prompt socket on disk + because that is IPC, not durable runtime state. +- Device identity, device auth, and bootstrap runtime modules now keep their + SQLite snapshot readers/writers separate from doctor-only legacy JSON import + helpers. Device identity uses typed `device_identities` rows and device auth + tokens use typed `device_auth_tokens` rows. Device auth writes reconcile rows + by device/role instead of truncating the token table, and runtime no longer + routes single-token updates through the old whole-store adapter. The legacy + version-1 JSON payloads exist only as doctor import/export shapes. +- GitHub Copilot token exchange cache uses the shared SQLite plugin-state table + under `github-copilot/token-cache/default`. It is provider-owned cache state, + so it intentionally does not add a host schema table. +- The shared Swift runtime (`OpenClawKit`) uses the same + `state/openclaw.sqlite` rows for device identity and device auth. macOS app + helpers import the shared SQLite helpers instead of owning a second JSON or + SQLite path. A leftover legacy `identity/device.json` blocks identity creation + until doctor imports it into SQLite, matching the TypeScript and Android + startup gate. +- Android device identity uses the same TypeScript-compatible key material + stored in typed `state/openclaw.sqlite#table/device_identities` rows. It never + reads or writes `openclaw/identity/device.json`; a leftover legacy file blocks + startup until doctor imports it into SQLite. +- Android cached device auth tokens also use typed + `state/openclaw.sqlite#table/device_auth_tokens` rows and share the same + version-1 token semantics as TypeScript and Swift. Runtime no longer reads `SecurePrefs` + `gateway.deviceToken*` compatibility keys; those belong to migration/doctor + logic only. +- Android notification recent-package history uses typed + `android_notification_recent_packages` rows. Runtime no longer migrates or + reads the old SharedPreferences CSV keys. +- Device identity creation fails closed when legacy `identity/device.json` + exists, when the SQLite identity row is invalid, or when the SQLite identity + store cannot be opened. Doctor imports and removes that file first, so runtime + startup cannot silently rotate pairing identity before migration. +- Device identity selection is a SQLite row key, not a JSON file locator. Tests + and gateway helpers pass explicit identity keys; only doctor migration and the + fail-closed startup gate know the retired `identity/device.json` filename. +- Session reset compatibility now lives in doctor config migration: + `session.idleMinutes` is moved into `session.reset.idleMinutes`, + `session.resetByType.dm` is moved into `session.resetByType.direct`, and the + runtime reset policy only reads canonical reset keys. +- Legacy config compatibility now lives under `src/commands/doctor/`. Normal + `readConfigFileSnapshot()` validation does not import doctor legacy detectors + or annotate legacy issues; `runDoctorConfigPreflight()` adds those issues for + doctor repair/reporting. The doctor config flow imports + `src/commands/doctor/legacy-config.ts`, and old OAuth profile-id repair lives + under + `src/commands/doctor/legacy/oauth-profile-ids.ts`. +- Non-doctor commands do not auto-run legacy config repair. For example, + `openclaw update --channel` now fails on invalid legacy config and asks the + user to run doctor, rather than silently importing doctor migration code. +- Web push, APNs, Voice Wake, update checks, and config health now use typed shared SQLite + tables for subscriptions, VAPID keys, node registrations, trigger rows, + routing rows, update-notification state, and config health entries instead of + whole opaque JSON blobs. Web push and APNs snapshot writes now reconcile + subscriptions/registrations by primary key instead of clearing their tables; + config health does the same by config path. + Their runtime modules keep SQLite snapshot readers/writers separate from + doctor-only legacy JSON import helpers. +- Node-host config now uses a typed singleton row in the shared SQLite database; + doctor imports the old `node.json` file before normal runtime use. +- Device/node pairing, channel pairing, channel allowlists, and bootstrap state + now use typed SQLite rows instead of whole opaque JSON blobs. Plugin binding + approvals and cron job state follow the same split: runtime modules expose + SQLite-backed operations and neutral snapshot helpers, and pairing/bootstrap + plus plugin binding approval snapshot writes reconcile rows by primary key + instead of truncating tables, while doctor imports/removes the old JSON files through + `src/commands/doctor/legacy/*` modules. +- Installed plugin records now live in the SQLite installed-plugin index. + Runtime config read/write no longer migrates or preserves old + `plugins.installs` authored-config data; doctor imports that legacy config + shape into SQLite before normal runtime use. +- QQBot credential recovery snapshots now live in SQLite plugin state under + `qqbot/credential-backups`. Runtime no longer writes + `qqbot/data/credential-backup*.json`; doctor imports and removes those + legacy backup files with the other QQBot state inputs. +- Gateway reload planning compares SQLite installed-plugin index snapshots under + an internal `installedPluginIndex.installRecords.*` diff namespace. Runtime + reload decisions no longer wrap those rows in fake `plugins.installs` config + objects. +- Matrix named-account credential upgrade no longer happens during runtime + reads. Doctor owns the old top-level `credentials/matrix/credentials.json` + rename when a single/default Matrix account can be resolved. +- Core pairing and cron runtime modules no longer export legacy JSON path + builders. Doctor-owned legacy modules construct `pending.json`, `paired.json`, + `bootstrap.json`, and `cron/jobs.json` source paths for import tests and + migration only. Legacy cron job-shape normalization and cron run-log import + live under `src/commands/doctor/legacy/cron*.ts`. +- `src/commands/doctor/legacy/runtime-state.ts` imports legacy JSON state + files, including node host config, into SQLite from doctor. New legacy file + importers stay under `src/commands/doctor/legacy/`. +- `src/commands/doctor/state-migrations.ts` imports legacy `sessions.json` and + `*.jsonl` transcripts directly into SQLite and removes successful sources. It + no longer stages root legacy transcripts through + `agents//sessions/*.jsonl` or creates a canonical JSONL target before + import. +- State integrity doctor checks no longer scan legacy session directories or + offer orphan JSONL deletion. Legacy transcript files are migration inputs + only, and the migration step owns import plus source removal. +- Legacy sandbox registry import lives under + `src/commands/doctor/legacy/sandbox-registry.ts`; active sandbox registry + reads and writes remain SQLite-only. +- The legacy session transcript health/import repair lives under + `src/commands/doctor/legacy/session-transcript-health.ts`; runtime command + modules no longer carry JSONL transcript parsing or active-branch repair code. + +Completed consolidation/deletion highlights: + +- Plugin state now uses the shared `state/openclaw.sqlite` database. The old + branch-local `plugin-state/state.sqlite` sidecar importer is removed because + that SQLite layout never shipped. Probe/test helpers report the shared + `databasePath` instead of exposing a plugin-state-specific SQLite path. +- Task and Task Flow runtime tables now live in the shared + `state/openclaw.sqlite` database instead of `tasks/runs.sqlite` and + `tasks/flows/registry.sqlite`; the old sidecar importers are removed for the + same unshipped-layout reason. +- `src/config/sessions/store.ts` no longer needs `storePath` for inbound + metadata, route updates, or updated-at reads. Command persistence, CLI + session cleanup, subagent depth, auth overrides, and transcript session + identity use agent/session row APIs. Writes are applied as SQLite row patches + with optimistic conflict retry. +- Session target resolution now exposes per-agent database targets, not legacy + `sessions.json` paths. Shared gateway, ACP metadata, doctor route repair, and + `openclaw sessions` enumerate `agent_databases` plus configured agents. +- Gateway session routing now uses `resolveGatewaySessionDatabaseTarget`; the + returned target carries `databasePath` and candidate SQLite row keys instead + of a legacy session-store file path. +- Channel session runtime types now expose `{agentId, sessionKey}` for + updated-at reads, inbound metadata, and last-route updates. The old + `saveSessionStore(storePath, store)` compatibility type is gone. +- Plugin runtime, extension API, and `config/sessions` barrel surfaces now steer + plugin code to SQLite-backed session row helpers. Root library compatibility + exports (`loadSessionStore`, `saveSessionStore`, `resolveStorePath`) remain as + deprecated shims for existing consumers. The old + `resolveLegacySessionStorePath` helper is gone; legacy `sessions.json` path + construction is now local to migration and test fixtures. +- `src/config/sessions/session-entries.sqlite.ts` now stores canonical session + entries in the per-agent database and has row-level read/upsert/delete patch + support. Runtime upsert/patch/delete no longer scans for case variants or + prunes legacy alias keys; doctor owns canonicalization. The + standalone JSON import helper is gone, and migration merges upsert newer rows + instead of replacing the whole session table. Public read/list/load helpers + project hot session metadata from typed `sessions` and `conversations` rows; + `entry_json` is a compatibility/debug shadow and can be stale or invalid + without losing typed session identity or delivery context. +- `src/config/sessions/delivery-info.ts` now resolves delivery context from the + typed per-agent `sessions` + `conversations` + `session_conversations` rows. + It no longer reconstructs runtime delivery identity from + `session_entries.entry_json`; a missing typed conversation row is a doctor + migration/repair problem, not a runtime fallback. +- Stored-session reset decisions now prefer typed `sessions.session_scope`, + `sessions.chat_type`, and `sessions.channel` metadata. `sessionKey` parsing + remains only for explicit thread/topic suffixes on command targets; group vs + direct reset classification no longer comes from key shape. +- Session list/status display classification now uses typed chat metadata and + gateway session kind. It no longer treats `:group:` or `:channel:` substrings + inside `session_key` as durable group/direct truth. +- Silent-reply policy selection now uses explicit conversation type or surface + metadata only. It no longer guesses direct/group policy from + `session_key` substrings. +- Session display model resolution now receives the agent id from the SQLite + session database target instead of splitting it out of `session_key`. +- Agent-to-agent announce target hydration now uses typed `sessions.list` + `deliveryContext` only. It no longer recovers channel/account/thread routing + from legacy `origin`, mirrored `last*` fields, or `session_key` shape. +- `sessions_send` thread-target rejection now reads typed SQLite routing + metadata. It no longer rejects or accepts targets by parsing thread suffixes + out of the target key. +- Group-scoped tool policy validation now reads typed SQLite conversation + routing for the current or spawned session. It no longer trusts group/channel + identity by decoding `sessionKey`; caller-provided group ids are dropped when + no typed session row vouches for them. +- Channel model override matching now uses explicit group and parent + conversation metadata. It no longer decodes parent conversation ids from + `parentSessionKey`. +- Stored model override inheritance now requires an explicit parent session key + from typed session context. It no longer derives parent overrides from + `:thread:` or `:topic:` suffixes in `sessionKey`. +- The old session thread-info wrapper and loaded-plugin thread parser are gone; + no runtime code imports `config/sessions/thread-info`. +- The channel conversation helper no longer exposes full-session-key parsing + bridges. Core still normalizes provider-owned raw conversation ids through + `resolveSessionConversation(...)`, but it does not reconstruct route facts + from `sessionKey`. +- Completion delivery, send policy, and task maintenance no longer derive chat + type from `session_key` shape. The old chat-type key parser has been deleted; + these paths require typed session metadata, typed delivery context, or + explicit delivery target vocabulary. +- Session list/status, diagnostics, approval account binding, TUI heartbeat + filtering, and usage summaries no longer mine `SessionEntry.origin` for + provider/account/thread/display routing. The only remaining runtime + `origin` reads are non-session concepts or current-turn delivery objects. +- Approval-request native conversation lookup now reads typed per-agent session + routing rows. It no longer parses channel/group/thread conversation identity + from `sessionKey`; missing typed metadata is a migration/repair issue. +- Gateway session changed/chat/session event payloads no longer echo + `SessionEntry.origin` or `last*` route shadows; clients receive typed + `channel`, `chatType`, and `deliveryContext`. +- Heartbeat delivery resolution can now receive the typed SQLite + `deliveryContext` directly, and heartbeat runtime passes the per-agent + session delivery row instead of relying on compatibility `session_entries` + shadows for current routing. +- Cron isolated-agent delivery target resolution also hydrates its current + route from the typed per-agent session delivery row before falling back to the + compatibility entry payload. +- Subagent announce origin resolution now threads the typed requester-session + delivery context through `loadRequesterSessionEntry` and prefers that row over + compatibility `last*`/`deliveryContext` shadows. +- Inbound session metadata updates now merge against the typed per-agent + delivery row first; old `SessionEntry` delivery fields are only the fallback + when no typed conversation row exists. +- Restart/update delivery extraction now lets the typed SQLite delivery + `threadId` win over topic/thread fragments parsed from `sessionKey`; parsing + is only a fallback for legacy thread-shaped keys. +- Hook agent context channel ids now prefer typed SQLite conversation identity, + then explicit message metadata. They no longer parse provider/group/channel + fragments from `sessionKey`. +- Gateway `chat.send` external-route inheritance now reads typed SQLite session + routing metadata instead of inferring channel/direct/group scope from + `sessionKey` pieces. Channel-scoped sessions inherit only when the typed + session channel and chat type match the stored delivery context; shared-main + sessions keep their stricter CLI/no-client-metadata rule. +- Restart-sentinel wake and continuation routing now reads typed SQLite + delivery/routing rows before queueing heartbeat wakes or routed agent-turn + continuations. It no longer reconstructs delivery context from the + session-entry JSON shadow. +- Gateway `tools.effective` context resolution now reads typed SQLite + delivery/routing rows for provider, account, target, thread, and reply-mode + inputs. It no longer recovers those hot routing fields from stale + `session_entries.entry_json` origin shadows. +- Realtime voice consult routing now resolves parent/call delivery from typed + per-agent SQLite session rows. It no longer falls back to compatibility + `SessionEntry.deliveryContext` shadows when choosing the embedded agent + message route. +- ACP spawn heartbeat relay and parent-stream routing now read parent delivery + from typed SQLite session rows. They no longer reconstruct parent delivery + context from compatibility session-entry shadows. +- Session delivery route preservation now follows typed chat metadata and + persisted delivery columns. It no longer extracts channel hints, direct/main + markers, or thread shape from `sessionKey`; internal webchat routes only + inherit an external target when SQLite already has typed/persisted delivery + identity for the session. +- Generic session delivery extraction now reads only the exact typed SQLite + session delivery row. It no longer parses thread/topic suffixes or falls back + from a thread-shaped key to a base session key. +- Reply dispatch, restart sentinel recovery, and realtime voice consult routing + now use exact typed SQLite session/conversation rows for thread routing. They + no longer recover thread ids or base-session delivery context by parsing + thread-shaped session keys. +- Embedded PI history limiting now uses the typed SQLite session routing + projection (`sessions` + primary `conversations`) for provider, chat type, + and peer identity. It no longer parses provider, DM, group, or thread shape + out of `sessionKey`. +- Cron tool delivery inference now uses explicit delivery or the current typed + delivery context only. It no longer decodes channel, peer, account, or thread + targets from `agentSessionKey`. +- Runtime session rows no longer carry the old `lastProvider` route alias. + Helpers and tests use typed `lastChannel` and `deliveryContext` fields; + doctor migration is the only place that should translate older route aliases + or persisted `origin` shadows. +- Transcript events, VFS rows, and tool artifact rows now write to the per-agent + database. The unshipped global transcript-file mapping table is gone; doctor + records legacy source paths in durable migration rows instead. +- Runtime transcript lookup no longer scans JSONL byte offsets or probes legacy + transcript files. Gateway chat/media/history paths read transcript rows from + SQLite; session JSONL is now only a legacy doctor input, not a runtime state + or export format. +- Transcript parent and branch relationships use structured + `parentTranscriptScope: {agentId, sessionId}` metadata in SQLite transcript + headers, not path-like `agent-db:...transcript_events...` locator strings. +- The transcript manager contract no longer exposes implicit persisted + `create(cwd)` or `continueRecent(cwd)` constructors. Persisted transcript + managers are opened with an explicit `{agentId, sessionId}` scope; only + in-memory managers remain scope-free for tests and pure transcript transforms. +- Runtime transcript store APIs resolve SQLite scope, not filesystem paths. The + old `resolve...ForPath` helper and unused `transcriptPath` write options are + gone from runtime callers. +- Runtime session resolution now uses `{agentId, sessionId}` and must not derive + `sqlite-transcript:///` strings for external boundaries. + Legacy absolute JSONL paths are doctor migration inputs only. +- Native hook relay direct-bridge records now live in typed shared + `native_hook_relay_bridges` rows keyed by relay id. Runtime no longer writes a + `/tmp` JSON registry or opaque generic records for those short-lived bridge + records. +- `runEmbeddedPiAgent(...)` no longer has a transcript-locator parameter. + Prepared worker descriptors also omit transcript locators. Runtime session + state and queued follow-up runs carry `{agentId, sessionId}` instead of + derived transcript handles. +- Embedded compaction now takes SQLite scope from `agentId` and `sessionId`. + Compaction hooks, context-engine calls, CLI delegation, and protocol replies + must not receive derived `sqlite-transcript://...` handles. Export/debug code + can materialize explicit user artifacts from rows, but it does not provide a + generic session JSONL export path or feed file names back into runtime + identity. +- `/export-session` reads transcript rows from SQLite and writes the requested + standalone HTML view only. The embedded viewer no longer reconstructs or + downloads session JSONL from those rows. +- Context-engine delegation no longer parses a transcript locator to recover + agent identity. The prepared runtime context carries the resolved `agentId` + into the built-in compaction adapter. +- Transcript rewrite and live tool-result truncation now read and persist + transcript state by `{agentId, sessionId}` and do not derive temporary + locators for transcript-update event payloads. +- The transcript-state helper surface no longer has locator-based + `readTranscriptState`, `replaceTranscriptStateEvents`, or + `persistTranscriptStateMutation` variants. Runtime callers must use the + `{agentId, sessionId}` APIs. Doctor import reads legacy files by explicit file + path and writes SQLite rows; it does not migrate locator strings. +- The runtime session-manager contract no longer exposes `open(locator)`, + `forkFrom(locator)`, or `setTranscriptLocator(...)`. Persisted session + managers open by `{agentId, sessionId}` only; list/fork helpers live on + row-oriented session and checkpoint APIs instead of the transcript manager + facade. +- Gateway transcript reader APIs are scope-first. They take + `{agentId, sessionId}` and do not accept a positional transcript locator that + could accidentally become runtime identity. Active transcript locator parsing + is gone; legacy source paths are read only by doctor import code. +- Transcript update events are also scope-first. `emitSessionTranscriptUpdate` + no longer accepts a bare locator string, and listeners route by + `{agentId, sessionId}` without parsing a handle. +- Gateway session-message broadcast resolves session keys from agent/session + scope, not from a transcript locator. The old transcript-locator-to-session + key resolver/cache is gone. +- Gateway session-history SSE filters live updates by agent/session scope. It no + longer canonicalizes transcript locator candidates, realpaths, or file-shaped + transcript identities to decide whether a stream should receive an update. +- Session lifecycle hooks no longer derive or expose transcript locators on + `session_end`. Hook consumers get `sessionId`, `sessionKey`, next-session + ids, and agent context; transcript files are not part of the lifecycle + contract. +- Reset hooks no longer derive or expose transcript locators either. The + `before_reset` payload carries recovered SQLite messages plus the reset + reason, while session identity stays in hook context. +- Agent harness reset no longer accepts a transcript locator. Reset dispatch is + scoped by `sessionId`/`sessionKey` plus reason. +- Agent extension session types no longer expose `transcriptLocator`; extensions + should use session context and runtime APIs rather than reaching for a + file-shaped transcript identity. +- Plugin compaction hooks no longer expose transcript locators. Hook context + already carries session identity, and transcript reads must go through SQLite + scope-aware APIs instead of file-shaped handles. +- `before_agent_finalize` hooks no longer expose `transcriptPath`, including + native hook relay payloads. Finalization hooks use session context only. +- Gateway reset responses no longer synthesize a transcript locator on the + returned entry. The reset creates SQLite transcript rows, returns the clean + session entry, and leaves transcript access to scope-aware readers. +- Embedded run and compaction results no longer surface transcript locators for + session accounting. Automatic compaction updates only the active `sessionId`, + compaction counters, and token metadata. +- Embedded attempt results no longer return `transcriptLocatorUsed`, and + context-engine `compact()` results no longer return transcript locators. + Runtime retry loops only accept a successor `sessionId`. +- Delivery-mirror transcript append results no longer return transcript + locators. Callers get the appended `messageId`; transcript update signals use + SQLite scope. +- Parent-session fork helpers return only the forked `sessionId`. Subagent + preparation passes the child agent/session scope to engines. +- CLI runner params and history reseeding no longer accept transcript locators. + CLI history reads resolve the SQLite transcript scope from `{agentId, +sessionId}` and session key context. +- CLI and embedded-runner test fixtures now seed and read SQLite transcript rows + by session id instead of pretending active sessions are `*.jsonl` files or + passing a `sqlite-transcript://...` string through runtime params. +- Session tool-result guard events emit from known session scope even when an + in-memory manager has no derived locator. Its tests no longer fake active + `/tmp/*.jsonl` transcript files. +- BTW and compaction-checkpoint helpers now read and fork transcript rows by + SQLite scope. Checkpoint metadata now stores session ids and leaf/entry ids + only; derived locators are no longer written into checkpoint payloads. +- Gateway transcript-key lookup uses SQLite transcript scope at protocol + boundaries and no longer realpaths or stats transcript filenames. +- Automatic compaction transcript rotation writes successor transcript rows + directly through the SQLite transcript store. Session rows keep only the + successor session identity, not a durable JSONL path or persisted locator. +- Embedded context-engine compaction uses SQLite-named transcript rotation + helpers. The rotation tests no longer construct JSONL successor paths or + model active sessions as files. +- Managed outgoing image retention keys its transcript-message cache from + SQLite transcript stats instead of filesystem stat calls. +- Runtime session locks and the standalone legacy `.jsonl.lock` doctor + lane have been removed. +- The Microsoft Teams runtime barrel and public plugin SDK no longer re-export + the old file-lock helper; durable plugin state paths are SQLite-backed. +- Session age/count pruning and explicit session cleanup have been removed. + Doctor owns legacy import; stale sessions are reset or deleted explicitly. +- Doctor integrity checks no longer count a legacy JSONL file as a valid active + transcript for a SQLite session row. Active transcript health is SQLite-only; + legacy JSONL files are reported as migration/orphan-cleanup inputs. +- Doctor no longer treats `agents//sessions/` as required runtime + state. It only scans that directory when it already exists, as legacy import + or orphan-cleanup input. +- Gateway `sessions.resolve`, session patch/reset/compact paths, subagent + spawning, fast abort, ACP metadata, heartbeat-isolated sessions, and TUI + patching no longer migrate or prune legacy session keys as a side effect of + normal runtime work. +- CLI command session resolution now returns the owning `agentId` instead of a + `storePath`, and it no longer copies legacy main-session rows during normal + `--to` or `--session-id` resolution. Legacy main-row canonicalization belongs + to doctor only. +- Runtime subagent depth resolution no longer reads `sessions.json` or JSON5 + session stores. It reads SQLite `session_entries` by agent id, and legacy + depth/session metadata can only enter through the doctor import path. +- Auth profile session overrides persist through direct `{agentId, sessionKey}` + row upserts instead of lazy-loading a file-shaped session-store runtime. +- Auto-reply verbose gating and session update helpers now read/upsert SQLite + session rows by session identity and no longer require a legacy store path + before touching persisted row state. +- Command-run session metadata helpers now use entry-oriented names and module + paths; the old `session-store` command helper surface has been removed. +- Bootstrap header seeding and manual compaction boundary hardening now mutate + SQLite transcript rows directly. Runtime callers pass session identity, not + writable `.jsonl` paths. +- Silent session-rotation replay copies recent user/assistant turns by + `{agentId, sessionId}` from SQLite transcript rows. It no longer accepts + source or target transcript locators. +- Fresh runtime session rows no longer store transcript locators. Callers use + `{agentId, sessionId}` directly; export/debug commands can choose output file + names when they materialize rows. +- Starting a new persisted transcript session now always opens SQLite rows by + scope. The session manager no longer reuses a previous file-era transcript + path or locator as the identity for the new session. +- Persisted transcript sessions use the explicit + `openTranscriptSessionManagerForSession({agentId, sessionId})` API. The old + static `SessionManager.create/openForSession/list/forkFromSession` facades are + gone so tests and runtime code cannot accidentally recreate file-era session + discovery. +- Plugin runtime no longer exposes `api.runtime.agent.session.resolveTranscriptLocatorPath`; + plugin code uses SQLite row helpers and scope values. +- The public `session-store-runtime` SDK surface now only exports session row + and transcript row helpers. Raw SQLite database open/path and close/reset + helpers live in the focused `sqlite-runtime` SDK surface, so plugin tests no + longer pull the deprecated broad testing barrel for database cleanup. +- Legacy `.jsonl` trajectory/checkpoint filename classifiers now live in the + doctor legacy session-file module. Core session validation no longer imports + file-artifact helpers to decide normal SQLite session ids. +- Active-memory blocking subagent runs use SQLite transcript rows instead of + creating temporary or persisted `session.jsonl` files under plugin state. The + old `transcriptDir` option is removed. +- One-off slug generation and Crestodian planner runs use SQLite transcript rows + instead of creating temporary `session.jsonl` files. +- `llm-task` helper runs and hidden commitment extraction also use SQLite + transcript rows, so these model-only helper sessions no longer create + temporary JSON/JSONL transcript files. +- `TranscriptSessionManager` is only an opened SQLite transcript scope now. + Runtime code opens it with `openTranscriptSessionManagerForSession({agentId, +sessionId})`; create, branch, continue, list, and fork flows live in their + owning SQLite row helpers rather than static manager facades. + Doctor/import/debug code handles explicit legacy source files outside the + runtime session manager. +- The stale `SessionManager.newSession()` and + `SessionManager.createBranchedSession()` facade methods were removed. New + sessions and transcript descendants are created by their owning SQLite + workflow instead of mutating an already-open manager into a different + persisted session. +- Parent transcript fork decisions and fork creation no longer accept + `storePath` or `sessionsDir`; they use `{agentId, sessionId}` SQLite + transcript scope instead of retained filesystem path metadata. +- Memory-host no longer exports no-op session-directory transcript + classification helpers; transcript filtering now derives from SQLite row + metadata during entry construction. +- Memory-host and QMD session-export tests use SQLite transcript scopes. Old + `agents//sessions/*.jsonl` paths stay covered only where a test is + intentionally proving doctor/import/export compatibility. +- QA-lab raw session inspection now uses `sessions.list` through the gateway + instead of reading `agents/qa/sessions/sessions.json`; MSteams feedback + appends directly to SQLite transcripts without fabricating a JSONL path. +- Shared inbound channel turns now carry `{agentId, sessionKey}` rather than a + legacy `storePath`. LINE, WhatsApp, Slack, Discord, Telegram, Matrix, Signal, + iMessage, BlueBubbles, Feishu, Google Chat, IRC, Nextcloud Talk, Zalo, + Zalo Personal, QA Channel, Microsoft Teams, Mattermost, Synology Chat, Tlon, + Twitch, and QQBot recording paths now read updated-at metadata and record + inbound session rows through SQLite identity. +- Transcript locator persistence is removed from active session rows. + `resolveSessionTranscriptTarget` returns `agentId`, `sessionId`, and optional + topic metadata; doctor is the only code that imports legacy transcript file + names. +- Runtime transcript headers start at SQLite version `1`. Old JSONL V1/V2/V3 + shape upgrades live only in doctor import and normalize imported headers to + the current SQLite transcript version before rows are stored. +- The database-first guard now bans `SessionManager.listAll` and + `SessionManager.forkFromSession`; session listing and fork/restore workflows + must stay on row/scoped SQLite APIs. +- The guard also bans legacy transcript JSONL parse/active-branch repair helper + names outside doctor/import code, so runtime cannot grow a second legacy + transcript migration path. +- Embedded PI runs reject incoming transcript handles. They use the SQLite + `{agentId, sessionId}` identity before worker launch and again before the + attempt touches transcript state. A stale `/tmp/*.jsonl` input cannot select a + runtime write target. +- Cache trace, Anthropic payload, raw stream, and diagnostics timeline records + now write to typed SQLite `diagnostic_events` rows. Gateway stability bundles + now write to typed SQLite `diagnostic_stability_bundles` rows. The old + `diagnostics.cacheTrace.filePath`, `OPENCLAW_CACHE_TRACE_FILE`, + `OPENCLAW_ANTHROPIC_PAYLOAD_LOG_FILE`, and + `OPENCLAW_DIAGNOSTICS_TIMELINE_PATH` JSONL override paths are removed, and + normal stability capture no longer writes `logs/stability/*.json` files. +- Cron persistence now reconciles SQLite `cron_jobs` rows instead of + deleting/reinserting the whole job table on each save. Plugin target + writebacks update matching cron rows directly and keep runtime cron state in + the same state-database transaction. +- Cron runtime callers now use a stable SQLite cron store key. Legacy + `cron.store` paths are doctor import inputs only; production gateway, task + maintenance, status, run-log, and Telegram target writeback paths use + `resolveCronStoreKey` and no longer path-normalize the key. Cron status now + reports `storeKey` rather than the old file-shaped `storePath` field. +- Cron runtime load and scheduling no longer normalize legacy persisted job + shapes such as `jobId`, `schedule.cron`, numeric `atMs`, string booleans, or + missing `sessionTarget`. Doctor legacy import owns those repairs before rows + are inserted into SQLite. +- ACP spawn no longer resolves or persists transcript JSONL file paths. Spawn + and thread-bind setup persist the SQLite session row directly and keep the + session id as the retained transcript identity. +- ACP session metadata APIs now read/list/upsert SQLite rows by `agentId` and + no longer expose `storePath` as part of the ACP session entry contract. +- Session usage accounting and gateway usage aggregation now resolve transcripts + by `{agentId, sessionId}` only. The cost/usage cache and discovered-session + summaries no longer synthesize or return transcript locator strings. +- Gateway chat append, abort-partial persistence, `/sessions.send`, and + webchat media transcript writes append directly through SQLite transcript + scope. The gateway transcript-injection helper no longer accepts a + `transcriptLocator` parameter. +- SQLite transcript discovery now lists transcript scopes and stats only: + `{agentId, sessionId, updatedAt, eventCount}`. The dead + `listSqliteSessionTranscriptLocators` compatibility helper and per-row + `locator` field are gone. +- Transcript repair runtime now exposes only + `repairTranscriptSessionStateIfNeeded({agentId, sessionId})`. The old + locator-based repair helper is deleted; doctor/debug code reads explicit + source file paths and never migrates locator strings. +- ACP replay ledger runtime now stores per-session replay rows in the shared + SQLite state database instead of `acp/event-ledger.json`; doctor imports and + removes the legacy file. +- Gateway transcript reader helpers now live in + `src/gateway/session-transcript-readers.ts` instead of the old + `session-utils.fs` module name. The fallback retry history check is named for + SQLite transcript content instead of the old file-helper surface. +- Gateway injected-chat and compaction helpers now pass SQLite transcript scope + through internal helper APIs instead of naming values transcript paths or + source files. +- Bootstrap continuation detection now checks SQLite transcript rows through + `hasCompletedBootstrapTranscriptTurn`; it no longer exposes a file-shaped + helper name. +- Embedded-runner tests now use SQLite transcript identity, and opening a new + transcript manager always requires an explicit `sessionId`. +- Memory indexing helpers now use SQLite transcript terminology end to end: + host exports `listSessionTranscriptScopesForAgent` and + `sessionTranscriptKeyForScope`, targeted sync queues `sessionTranscripts`, + public session-search hits expose opaque `transcript::` paths, + and the internal DB source key is `session:` under + `source_kind='sessions'` instead of a fake file path. +- The generic plugin SDK persistent-dedupe helper no longer exposes file-shaped + options. Callers provide SQLite scope keys and durable dedupe rows live in + shared plugin state. +- Microsoft Teams SSO and delegated OAuth tokens moved from locked JSON files + to SQLite plugin state. Doctor imports `msteams-sso-tokens.json` and + `msteams-delegated.json`, rebuilds canonical SSO token keys from payloads, + and removes the source files. +- Matrix sync cache state moved from `bot-storage.json` to SQLite plugin + state. Doctor imports legacy raw or wrapped sync payloads and removes the + source file. Active Matrix and QA Matrix clients pass a SQLite sync-store root + directory, not a fake `sync-store.json` or `bot-storage.json` path. +- Matrix legacy crypto migration status moved from + `legacy-crypto-migration.json` to SQLite plugin state. Doctor imports the + old status file; Matrix SDK IndexedDB snapshots moved from + `crypto-idb-snapshot.json` to SQLite plugin blobs. Matrix recovery keys and + credentials are SQLite plugin-state rows; their old JSON files are doctor + migration inputs only. +- Memory Wiki activity logs now use SQLite plugin state instead of + `.openclaw-wiki/log.jsonl`. The Memory Wiki migration provider imports old + JSONL logs; wiki markdown and user vault content stay file-backed as + workspace content. +- Memory Wiki no longer creates `.openclaw-wiki/state.json` or the unused + `.openclaw-wiki/locks` directory. The migration provider removes those retired + plugin metadata files if an older vault still has them. +- Crestodian audit entries now use core SQLite plugin state instead of + `audit/crestodian.jsonl`. Doctor imports the legacy JSONL audit log and + removes it after successful import. +- Config write/observe audit entries now use core SQLite plugin state instead + of `logs/config-audit.jsonl`. Doctor imports the legacy JSONL audit log and + removes it after successful import. +- The macOS companion no longer writes app-local `logs/config-audit.jsonl` or + `logs/config-health.json` sidecars while editing `openclaw.json`. The config + file remains file-backed, recovery snapshots stay next to the config file, + and durable config audit/health state belongs to the Gateway SQLite store. +- Crestodian rescue pending approvals now use core SQLite plugin state instead + of `crestodian/rescue-pending/*.json`. Doctor imports legacy pending approval + files and removes them after successful import. +- Phone Control temporary arm state now uses SQLite plugin state instead of + `plugins/phone-control/armed.json`. Doctor imports the legacy armed-state + file into the `phone-control/arm-state` namespace and removes the file. +- Doctor no longer repairs JSONL transcripts in place or creates backup JSONL + files. It imports the active branch into SQLite and removes the legacy source. +- Session-memory hook transcript lookup uses `{agentId, sessionId}` scope-only + SQLite reads. Its helper no longer accepts or derives transcript locators, + legacy file reads, or file-rewrite options. +- Codex app-server conversation bindings now key SQLite plugin state by + OpenClaw session key or explicit `{agentId, sessionId}` scope. They must not + preserve transcript-path fallback bindings. +- Codex app-server mirrored-history reads use the SQLite transcript scope only; + they must not recover identity from transcript file paths. +- Role-ordering and compaction reset paths no longer unlink old transcript + files; reset only rotates the SQLite session row and transcript identity. +- Gateway reset and checkpoint responses return clean session rows plus session + ids. They no longer synthesize SQLite transcript locators for clients. +- Memory-core dreaming no longer prunes session rows by probing for missing + JSONL files. Subagent cleanup goes through the session runtime API instead of + filesystem existence checks. Its transcript-ingestion tests seed SQLite rows + directly instead of creating `agents//sessions` fixtures or locator + placeholders. +- Memory transcript indexing may expose `transcript::` as a + virtual search-hit path for citation/read helpers. The durable index source is + relational (`source_kind='sessions'`, `source_key='session:'`, + `session_id=`), so the value is not a runtime transcript locator, + not a filesystem path, and must never be passed back into session runtime APIs. +- Gateway doctor memory status reads short-term recall and phase-signal counts + from SQLite plugin-state rows instead of `memory/.dreams/*.json`; CLI and + doctor output now label that storage as a SQLite store, not a path. +- Memory-core runtime, CLI status, Gateway doctor methods, and plugin SDK + facades no longer audit or archive legacy `.dreams/session-corpus` files. + Those files are migration inputs only; doctor imports them into SQLite and + deletes the source after verification. Active session-ingestion evidence rows + now use the virtual SQLite path `memory/session-ingestion/.txt`; runtime + never writes or derives state from `.dreams/session-corpus`. +- Memory-core public artifacts expose SQLite host events as the virtual JSON + artifact `memory/events/memory-host-events.json`; they no longer reuse the + legacy `.dreams/events.jsonl` source path. +- Sandbox container/browser registries now use the shared + `sandbox_registry_entries` SQLite table with typed session, image, timestamp, + backend/config, and browser port columns. Doctor imports legacy monolithic and + sharded JSON registry files and removes successful sources. Runtime reads use + the typed row columns as source of truth; `entry_json` is only a replay/debug + copy. +- Commitments now use a typed shared `commitments` table instead of a + whole-store JSON blob. Snapshot saves upsert by commitment id and delete only + missing rows instead of clearing and reinserting the table. Runtime loads + commitments from typed scope, delivery-window, status, attempt, and text + columns; `record_json` is only a replay/debug copy. Doctor imports legacy + `commitments.json` and removes it after a successful import. +- Cron job definitions, schedule state, and run history no longer have runtime + JSON writers or readers. Runtime uses `cron_jobs` rows with typed schedule, + payload, delivery, failure-alert, session, status, and runtime-state columns plus typed + `cron_run_logs` metadata for status, diagnostics summary, delivery status/error, + session/run, model, and token totals. `job_json` is only a replay/debug copy; `state_json` keeps nested + runtime diagnostics that do not yet have hot query fields, while runtime + rehydrates hot state fields from typed columns. Doctor imports + legacy `jobs.json`, `jobs-state.json`, and `runs/*.jsonl` files and removes + the imported sources. Plugin target writebacks update matching `cron_jobs` + rows instead of loading and replacing the whole cron store. +- If doctor cannot safely translate legacy `notify: true` webhook fallback + without replacing an explicit delivery target, it records a warning and leaves + the legacy source in place instead of publishing a lossy SQLite row. +- Outbound and session delivery queues now store queue status, entry kind, + session key, channel, target, account id, retry count, last attempt/error, + recovery state, and platform-send markers as typed columns in the shared + `delivery_queue_entries` table. Runtime recovery reads those hot fields from + the typed columns, and retry/recovery mutations update those columns directly + without rewriting replay JSON. The full JSON payload remains only as the + replay/debug blob for message bodies and other cold replay data. +- Managed outgoing image records now use typed shared + `managed_outgoing_image_records` rows with media bytes still stored in + `media_blobs`. The JSON record remains only as a replay/debug copy. +- Discord model-picker preferences, command-deploy hashes, and thread bindings + now use shared SQLite plugin state. Their legacy JSON import plans live in the + Discord plugin setup/doctor migration surface, not in core migration code. +- Plugin legacy import detectors use doctor-named modules such as + `doctor-legacy-state.ts` or `doctor-state-imports.ts`; normal channel runtime + modules must not import legacy JSON detectors. +- BlueBubbles catchup cursors and inbound dedupe markers now use shared SQLite + plugin state. Their legacy JSON import plans live in the BlueBubbles plugin + setup/doctor migration surface, not in core migration code. +- Telegram update offsets, sticker cache rows, sent-message cache rows, + topic-name cache rows, and thread bindings now use shared SQLite plugin + state. Their legacy JSON import plans live in the Telegram plugin + setup/doctor migration surface, not in core migration code. +- iMessage catchup cursors, reply short-id mappings, and sent-echo dedupe rows + now use shared SQLite plugin state. The old `imessage/catchup/*.json`, + `imessage/reply-cache.jsonl`, and `imessage/sent-echoes.jsonl` files are + doctor inputs only. +- Feishu message dedupe rows now use shared SQLite plugin state instead of + `feishu/dedup/*.json` files. Its legacy JSON import plan lives in the Feishu + plugin setup/doctor migration surface, not in core migration code. +- Microsoft Teams conversations, polls, pending upload buffers, and feedback + learnings now use shared SQLite plugin state/blob tables. The pending upload + path uses `plugin_blob_entries` so media buffers are stored as SQLite BLOBs + instead of base64 JSON. The runtime helper names now use SQLite/state naming + rather than `*-fs` file-store naming, and the old `storePath` shim is gone + from these stores. Its legacy JSON import plan lives in the Microsoft Teams + plugin setup/doctor migration surface. +- Zalo hosted outbound media now uses shared SQLite `plugin_blob_entries` + instead of `openclaw-zalo-outbound-media` JSON/bin temp sidecars. +- Diffs viewer HTML and metadata now use shared SQLite `plugin_blob_entries` + instead of `meta.json`/`viewer.html` temp files. Rendered PNG/PDF outputs stay + temp materializations because channel delivery still needs a file path. +- Canvas managed documents now use shared SQLite `plugin_blob_entries` instead + of a default `state/canvas/documents` directory. The Canvas host serves those + blobs directly; local files are created only for explicit `host.root` + operator content or temporary materialization when a downstream media reader + requires a path. +- File Transfer audit decisions now use shared SQLite `plugin_state_entries` + instead of the unbounded `audit/file-transfer.jsonl` runtime log. Doctor + imports the legacy JSONL audit file into plugin state and removes the source + after a clean import. +- ACPX process leases and gateway instance identity now use shared SQLite plugin + state. Doctor imports the legacy `gateway-instance-id` file into plugin state + and removes the source. +- ACPX generated wrapper scripts and the isolated Codex home are temporary + materialization under the OpenClaw temp root, not durable OpenClaw state. The + durable ACPX runtime records are the SQLite lease and gateway-instance rows; + the old ACPX `stateDir` config surface is removed because no runtime state is + written there anymore. +- Gateway media attachments now use the shared `media_blobs` SQLite table as + the canonical byte store. Local paths returned to channel and sandbox + compatibility surfaces are temp materializations of the database row, not the + durable media store. Runtime media allowlists no longer include legacy + `$OPENCLAW_STATE_DIR/media` or config-dir `media` roots; those directories are + doctor import sources only. +- Shell completion no longer writes `$OPENCLAW_STATE_DIR/completions/*` cache + files. Install, doctor, update, and release smoke paths use generated + completion output or profile sourcing instead of durable completion cache + files. +- Gateway skill-upload staging now uses shared `skill_uploads` rows. Upload + metadata, idempotency keys, and archive bytes live in SQLite; the installer + only receives a temporary materialized archive path while an install is + running. +- Subagent inline attachments no longer materialize under workspace + `.openclaw/attachments/*`. The spawn path prepares SQLite VFS seed entries, + inline runs seed those entries into the per-agent runtime scratch namespace, + and disk-backed tools overlay that SQLite scratch for attachment paths. The + old subagent-run attachment-dir registry columns and cleanup hooks are gone. +- CLI image hydration no longer maintains stable `openclaw-cli-images` cache + files. External CLI backends still receive file paths, but those paths are + per-run temp materializations with cleanup. +- Cache-trace diagnostics, Anthropic payload diagnostics, raw model stream + diagnostics, diagnostics timeline events, and Gateway stability bundles now + write SQLite rows instead of `logs/*.jsonl` or + `logs/stability/*.json` files. + Runtime path override flags and env vars have been removed; export/debug + commands can materialize files explicitly from database rows. +- The macOS companion no longer has a rolling `diagnostics.jsonl` writer. App + logs go to unified logging, and durable Gateway diagnostics stay SQLite-backed. +- The macOS port-guardian record list now uses typed shared SQLite + `macos_port_guardian_records` rows instead of an Application Support JSON file + or opaque singleton blob. +- Gateway singleton locks now use typed shared SQLite `state_leases` rows under + the `gateway_locks` scope instead of temp-dir lock files. Fly and OAuth + troubleshooting docs now point at the SQLite lease/auth refresh lock instead + of stale file-lock cleanup. +- Gateway restart sentinel state now uses typed shared SQLite + `gateway_restart_sentinel` rows instead of `restart-sentinel.json`; runtime + reads sentinel kind, status, routing, message, continuation, and stats from + typed columns. `payload_json` is only a replay/debug copy. Runtime code clears + the SQLite row directly and no longer carries file cleanup plumbing. +- Gateway restart intent and supervisor handoff state now use typed shared + SQLite `gateway_restart_intent` and `gateway_restart_handoff` rows instead of + `gateway-restart-intent.json` and + `gateway-supervisor-restart-handoff.json` sidecars. +- Gateway singleton coordination now uses typed `state_leases` rows under + `gateway_locks` instead of writing `gateway..lock` files. The lease row + owns the lock owner, expiry, heartbeat, and debug payload; SQLite owns the + atomic acquire/release boundary. The retired file-lock directory option is + gone; tests use the SQLite row identity directly. +- The old unreferenced cron usage-report helper that scanned `cron/runs/*.jsonl` + files was deleted. Cron run history reports should read the typed + `cron_run_logs` SQLite rows. +- Main-session restart recovery now discovers candidate agents through the + SQLite `agent_databases` registry instead of scanning `agents/*/sessions` + directories. +- Gemini session-corruption recovery now deletes only the SQLite session row; + it no longer needs a legacy `storePath` gate or tries to unlink a derived + transcript JSONL path. +- Path override handling now treats literal `undefined`/`null` environment + values as unset, preventing accidental repo-root `undefined/state/*.sqlite` + databases during tests or shell handoffs. +- Config health fingerprints now use typed shared SQLite `config_health_entries` + rows instead of `logs/config-health.json`, keeping the normal config file as + the only non-credential configuration document. The macOS companion keeps only + process-local health state and does not recreate the old JSON sidecar. +- Auth profile runtime no longer imports or writes credential JSON files. The + canonical credential store is SQLite; `auth-profiles.json`, per-agent + `auth.json`, and shared `credentials/oauth.json` are doctor migration inputs + that are removed after import. +- Auth profile save/state tests now assert typed SQLite auth tables directly + and only use legacy auth-profile filenames for doctor migration inputs. +- `openclaw secrets apply` scrubs the config file, env file, and SQLite + auth-profile store only. It no longer carries compatibility logic that edits + retired per-agent `auth.json`; doctor owns importing and deleting that file. +- Hermes secret migration plans and applies imported API-key profiles directly + into the SQLite auth-profile store. It no longer writes or verifies + `auth-profiles.json` as an intermediate target. +- User-facing auth docs now describe + `state/openclaw.sqlite#table/auth_profile_stores/` instead of + telling users to inspect or copy `auth-profiles.json`; legacy OAuth/auth JSON + names remain documented only as doctor-import inputs. +- Core state-path helpers no longer expose the retired `credentials/oauth.json` + file. The legacy filename is local to the doctor auth import path. +- Install, security, onboarding, model-auth, and SecretRef docs now describe + SQLite auth-profile rows and whole-state backup/migration instead of + per-agent auth-profile JSON files. +- PI model discovery now passes canonical credentials into in-memory + `pi-coding-agent` auth storage. It no longer creates, scrubs, or writes + per-agent `auth.json` during discovery. +- Voice Wake trigger and routing settings now use typed shared SQLite tables + instead of `settings/voicewake.json`, `settings/voicewake-routing.json`, or + opaque generic rows; doctor imports the legacy JSON files and removes them after a + successful migration. +- Update-check state now uses a typed shared `update_check_state` row instead of + `update-check.json` or an opaque generic blob; doctor imports + the legacy JSON file and removes it after a successful migration. +- Config health state now uses typed shared `config_health_entries` rows instead + of `logs/config-health.json` or an opaque generic blob; doctor + imports the legacy JSON file and removes it after a successful migration. +- Plugin conversation binding approvals now use typed + `plugin_binding_approvals` rows instead of opaque shared SQLite state or + `plugin-binding-approvals.json`; the legacy file is a doctor migration input. +- Generic current-conversation bindings now store typed + `current_conversation_bindings` rows instead of rewriting + `bindings/current-conversations.json`; doctor imports the legacy JSON file and + removes it after a successful migration. +- Memory Wiki imported-source sync ledgers now store one SQLite plugin-state row + per vault/source key instead of rewriting `.openclaw-wiki/source-sync.json`; + the migration provider imports and removes the legacy JSON ledger. +- Memory Wiki ChatGPT import-run records now store one SQLite plugin-state row + per vault/run id instead of writing `.openclaw-wiki/import-runs/*.json`. + Rollback snapshots remain explicit vault files until import-run snapshot + archival is moved into blob storage. +- Memory Wiki compiled digests now store SQLite plugin blob rows instead of + writing `.openclaw-wiki/cache/agent-digest.json` and + `.openclaw-wiki/cache/claims.jsonl`. The migration provider imports old cache + files and removes the cache directory when it becomes empty. +- ClawHub skill install tracking now stores one SQLite plugin-state row per + workspace/skill instead of writing or reading `.clawhub/lock.json` and + `.clawhub/origin.json` sidecars at runtime. Runtime code uses tracked-install + state objects rather than file-shaped lockfile/origin abstractions. Doctor + imports the legacy sidecars from configured agent workspaces and removes them + after a clean import. +- The installed plugin index now reads and writes the typed shared SQLite + `installed_plugin_index` singleton row instead of `plugins/installs.json`; the + legacy JSON file is only a doctor migration input and is removed after import. +- The legacy `plugins/installs.json` path helper now lives in doctor legacy + code. Runtime plugin-index modules expose only SQLite-backed persistence + options, not a JSON file path. +- Gateway restart sentinel, restart intent, and supervisor handoff state now use + typed shared SQLite rows (`gateway_restart_sentinel`, + `gateway_restart_intent`, and `gateway_restart_handoff`) instead of generic + opaque blobs. Runtime restart code has no file-shaped sentinel/intent/handoff + contract. +- Matrix sync cache, storage metadata, thread bindings, inbound dedupe markers, + startup verification cooldown state, SDK IndexedDB crypto snapshots, + credentials, and recovery keys now use shared SQLite plugin state/blob + tables. Runtime path structs no longer expose a `storage-meta.json` metadata + path; that filename is a legacy migration input only. Their legacy JSON import + plan lives in the Matrix plugin setup/doctor migration surface. +- Matrix startup no longer scans, reports, or completes legacy Matrix file + state. Matrix file detection, legacy crypto snapshot creation, room-key + restore migration state, import, and source removal are all doctor-owned. +- Matrix runtime migration barrels were removed. Legacy state/crypto detection + and mutation helpers are imported by Matrix doctor directly instead of being + part of runtime API surface. +- Matrix migration snapshot reuse markers now live in SQLite plugin state + instead of `matrix/migration-snapshot.json`; doctor can still reuse the same + verified pre-migration archive without writing a sidecar state file. +- Nostr bus cursors and profile publish state now use shared SQLite plugin + state. Their legacy JSON import plan lives in the Nostr plugin setup/doctor + migration surface. +- Active Memory session toggles now use shared SQLite plugin state instead of + `session-toggles.json`; toggling memory back on deletes the row instead of + rewriting a JSON object. +- Skill Workshop proposals and review counters now use shared SQLite plugin + state instead of per-workspace `skill-workshop/.json` stores. Each + proposal is a separate row under `skill-workshop/proposals`, and the review + counter is a separate row under `skill-workshop/reviews`. +- Skill Workshop reviewer subagent runs now use the runtime session transcript + resolver instead of creating `skill-workshop/.json` sidecar session + paths. +- ACPX process leases now use shared SQLite plugin state under + `acpx/process-leases` instead of a whole-file `process-leases.json` registry. + Each lease is stored as its own row, preserving startup stale-process reaping + without a runtime JSON rewrite path. +- ACPX wrapper scripts and the isolated Codex home are generated in the + OpenClaw temp root. They are recreated as needed and are not backup or + migration inputs. +- Subagent run registry persistence uses typed shared `subagent_runs` rows. The + old `subagents/runs.json` path is now only a doctor migration input, and + runtime helper names no longer describe the state layer as disk-backed. + Runtime tests no longer create invalid or empty `runs.json` fixtures to prove + registry behavior; they seed/read SQLite rows directly. +- Backup stages the state directory before archiving, copies non-database files, + snapshots `*.sqlite` databases with `VACUUM INTO`, omits live WAL/SHM + sidecars, records snapshot metadata in the archive manifest, and records + completed backup runs in SQLite with the archive manifest. `openclaw backup +create` validates the written archive by default; `--no-verify` is the + explicit fast path. +- `openclaw backup restore` validates the archive before extraction, reuses the + verifier's normalized manifest, and restores verified manifest assets to their + recorded source paths. It requires `--yes` for writes and supports `--dry-run` + for a restore plan. +- The old backup volatile-path filter is deleted. Backup no longer needs a + live-tar skip list for legacy session or cron JSON/JSONL files because SQLite + snapshots are staged before archive creation. +- Plain setup and onboarding workspace preparation no longer create + `agents//sessions/` directories. They create config/workspace only; + SQLite session rows and transcript rows are created on demand in the + per-agent database. +- Security permission repair now targets the global and per-agent SQLite + databases plus WAL/SHM sidecars instead of `sessions.json` and transcript + JSONL files. +- Sandbox registry runtime names now describe SQLite registry kinds directly + instead of carrying legacy JSON registry terminology through the active store. +- `openclaw reset --scope config+creds+sessions` removes per-agent + `openclaw-agent.sqlite` databases plus WAL/SHM sidecars, not only legacy + `sessions/` directories. +- Gateway aggregate session helpers now use entry-oriented names: + `loadCombinedSessionEntriesForGateway` returns `{ databasePath, entries }`. + The old combined-store naming has been removed from runtime callers. +- Docker MCP channel seeding now writes the main session row and transcript + events into the per-agent SQLite database instead of creating + `sessions.json` and a JSONL transcript. +- The bundled session-memory hook now resolves previous-session context from + SQLite by `{agentId, sessionId}`. It no longer scans, stores, or synthesizes + transcript paths or `workspace/sessions` directories. +- The bundled command-logger hook now writes command audit rows to the shared + SQLite `command_log_entries` table instead of appending + `logs/commands.log`. +- Channel pairing allowlists now expose only SQLite-backed read/write helpers at + runtime and in the plugin SDK. The old `*-allowFrom.json` path resolver and + file reader live only under doctor legacy import code. +- `migration_runs` records legacy-state migration executions with status, + timestamps, and JSON reports. +- `migration_sources` records each imported legacy file source with hash, size, + record count, target table, run id, status, and source-removal state. +- `backup_runs` records backup archive paths, status, and JSON manifests. +- The global schema does not keep an unused `agents` registry table. Agent + database discovery is the canonical `agent_databases` registry until runtime + has a real agent-record owner. +- Generated model catalog config is stored in typed global SQLite + `agent_model_catalogs` rows keyed by agent directory. Runtime callers use + `ensureOpenClawModelCatalog`; there is no `models.json` compatibility API in + runtime code. The implementation writes SQLite and the embedded PI registry is + hydrated from that stored payload without creating a `models.json` file. +- QMD session transcript markdown export and `memory.qmd.sessions` config were + removed. There is no QMD transcript collection, no `qmd/sessions*` runtime + path, and no file-backed session memory bridge. +- Memory-core runtime imports SQLite transcript indexing helpers from + `openclaw/plugin-sdk/memory-core-host-engine-session-transcripts`, not the + QMD SDK subpath. The QMD subpath keeps a compatibility re-export only for + external callers until a major SDK cleanup can remove it. +- QMD's own `index.sqlite` is now a temp runtime materialization backed by the + main SQLite `plugin_blob_entries` table. Runtime no longer creates a durable + `~/.openclaw/agents//qmd` sidecar. +- The optional `memory-lancedb` plugin no longer creates + `~/.openclaw/memory/lancedb` as an implicit OpenClaw-managed store. It is an + external LanceDB backend and stays disabled until the operator configures an + explicit `dbPath`. +- `check:database-first-legacy-stores` fails new runtime source that pairs + legacy store names with write-style filesystem APIs. It also fails runtime + source that reintroduces transcript bridge contracts such as + `transcriptLocator`, `sqlite-transcript://...`, `sessionFile`, or + `storePath`, and scans tests for those bridge-contract names too. It also + bans `SessionManager.open(...)` and the old static SessionManager facades so + runtime and tests cannot silently re-create a file-backed session opener or + file-era session discovery. It also bans the old session JSONL downloader + hook/class from export UI. It also bans sidecar-shaped plugin-state/task + SQLite helper names; tests should assert `databasePath` and the shared + `state/openclaw.sqlite` location instead of pretending those features own + separate SQLite files. It also bans the old generic memory index SQL table + names (`meta`, `files`, `chunks`, `chunks_vec`, + `chunks_fts`, `embedding_cache`) in runtime source so the agent database keeps + its explicit `memory_index_*` schema. It also bans embedding TEXT schemas and + embedding JSON-array writes so vectors stay compact SQLite BLOBs. Migration, + doctor, import, and explicit non-session export code remain allowed. The + guard now also covers runtime `cache/*.json` stores, generic + `thread-bindings.json` sidecars, cron state/run-log JSON, config health JSON, + restart and lock sidecars, Voice Wake settings, plugin binding approvals, + installed plugin index JSON, File Transfer audit JSONL, Memory Wiki activity + logs, the old bundled `command-logger` text log, and pi-mono raw-stream JSONL + diagnostics knobs. It also bans old root-level doctor legacy module names so + compatibility code stays under `src/commands/doctor/`. Android debug handlers + also use logcat/in-memory output instead of staging `camera_debug.log` or + `debug_logs.txt` cache files. + +## Target Schema Shape + +Keep schemas explicit. Host-owned runtime state uses typed tables. Plugin-owned +opaque state uses `plugin_state_entries` / `plugin_blob_entries`; there is no +generic host `kv` table. + +Global database: + +```text +state_leases(scope, lease_key, owner, expires_at, heartbeat_at, payload_json, created_at, updated_at) +exec_approvals_config(config_key, raw_json, socket_path, has_socket_token, default_security, default_ask, default_ask_fallback, auto_allow_skills, agent_count, allowlist_count, updated_at_ms) +schema_meta(meta_key, role, schema_version, agent_id, app_version, created_at, updated_at) +agent_databases(agent_id, path, schema_version, last_seen_at, size_bytes) +task_runs(...) +task_delivery_state(...) +flow_runs(...) +subagent_runs(run_id, child_session_key, requester_session_key, controller_session_key, created_at, ended_at, cleanup_handled, payload_json) +current_conversation_bindings(binding_key, binding_id, target_agent_id, target_session_id, target_session_key, channel, account_id, conversation_kind, parent_conversation_id, conversation_id, target_kind, status, bound_at, expires_at, metadata_json, updated_at) +plugin_binding_approvals(plugin_root, channel, account_id, plugin_id, plugin_name, approved_at) +tui_last_sessions(scope_key, session_key, updated_at) +plugin_state_entries(plugin_id, namespace, entry_key, value_json, created_at, expires_at) +plugin_blob_entries(plugin_id, namespace, entry_key, metadata_json, blob, created_at, expires_at) +media_blobs(subdir, id, content_type, size_bytes, blob, created_at, updated_at) +skill_uploads(upload_id, kind, slug, force, size_bytes, sha256, actual_sha256, received_bytes, archive_blob, created_at, expires_at, committed, committed_at, idempotency_key_hash) +web_push_subscriptions(endpoint_hash, subscription_id, endpoint, p256dh, auth, created_at_ms, updated_at_ms) +web_push_vapid_keys(key_id, public_key, private_key, subject, updated_at_ms) +apns_registrations(node_id, transport, token, relay_handle, send_grant, installation_id, topic, environment, distribution, token_debug_suffix, updated_at_ms) +node_host_config(config_key, version, node_id, token, display_name, gateway_host, gateway_port, gateway_tls, gateway_tls_fingerprint, updated_at_ms) +device_identities(identity_key, device_id, public_key_pem, private_key_pem, created_at_ms, updated_at_ms) +device_auth_tokens(device_id, role, token, scopes_json, updated_at_ms) +macos_port_guardian_records(pid, port, command, mode, timestamp) +workspace_setup_state(workspace_key, workspace_path, version, bootstrap_seeded_at, setup_completed_at, updated_at) +native_hook_relay_bridges(relay_id, pid, hostname, port, token, expires_at_ms, updated_at_ms) +model_capability_cache(provider_id, model_id, name, input_text, input_image, reasoning, supports_tools, context_window, max_tokens, cost_input, cost_output, cost_cache_read, cost_cache_write, updated_at_ms) +agent_model_catalogs(catalog_key, agent_dir, raw_json, updated_at) +managed_outgoing_image_records(attachment_id, session_key, message_id, created_at, updated_at, retention_class, alt, original_media_id, original_media_subdir, original_content_type, original_width, original_height, original_size_bytes, original_filename, record_json) +gateway_restart_sentinel(sentinel_key, version, kind, status, ts, session_key, thread_id, delivery_channel, delivery_to, delivery_account_id, message, continuation_json, doctor_hint, stats_json, payload_json, updated_at_ms) +channel_pairing_requests(channel_key, account_id, request_id, code, created_at, last_seen_at, meta_json) +channel_pairing_allow_entries(channel_key, account_id, entry, sort_order, updated_at) +voicewake_triggers(config_key, position, trigger, updated_at_ms) +voicewake_routing_config(config_key, version, default_target_mode, default_target_agent_id, default_target_session_key, updated_at_ms) +voicewake_routing_routes(config_key, position, trigger, target_mode, target_agent_id, target_session_key, updated_at_ms) +update_check_state(state_key, last_checked_at, last_notified_version, last_notified_tag, last_available_version, last_available_tag, auto_install_id, auto_first_seen_version, auto_first_seen_tag, auto_first_seen_at, auto_last_attempt_version, auto_last_attempt_at, auto_last_success_version, auto_last_success_at, updated_at_ms) +config_health_entries(config_path, last_known_good_json, last_promoted_good_json, last_observed_suspicious_signature, updated_at_ms) +sandbox_registry_entries(registry_kind, container_name, session_key, backend_id, runtime_label, image, created_at_ms, last_used_at_ms, config_label_kind, config_hash, cdp_port, no_vnc_port, entry_json, updated_at) +cron_run_logs(store_key, job_id, seq, ts, status, error, summary, diagnostics_summary, delivery_status, delivery_error, delivered, session_id, session_key, run_id, run_at_ms, duration_ms, next_run_at_ms, model, provider, total_tokens, entry_json, created_at) +cron_jobs(store_key, job_id, name, description, enabled, delete_after_run, created_at_ms, agent_id, session_key, schedule_kind, schedule_expr, schedule_tz, every_ms, anchor_ms, at, stagger_ms, session_target, wake_mode, payload_kind, payload_message, payload_model, payload_fallbacks_json, payload_thinking, payload_timeout_seconds, payload_allow_unsafe_external_content, payload_external_content_source_json, payload_light_context, payload_tools_allow_json, delivery_mode, delivery_channel, delivery_to, delivery_thread_id, delivery_account_id, delivery_best_effort, failure_delivery_mode, failure_delivery_channel, failure_delivery_to, failure_delivery_account_id, failure_alert_disabled, failure_alert_after, failure_alert_channel, failure_alert_to, failure_alert_cooldown_ms, failure_alert_include_skipped, failure_alert_mode, failure_alert_account_id, next_run_at_ms, running_at_ms, last_run_at_ms, last_run_status, last_error, last_duration_ms, consecutive_errors, consecutive_skipped, schedule_error_count, last_delivery_status, last_delivery_error, last_delivered, last_failure_alert_at_ms, job_json, state_json, runtime_updated_at_ms, schedule_identity, sort_order, updated_at) +delivery_queue_entries(queue_name, id, status, entry_kind, session_key, channel, target, account_id, retry_count, last_attempt_at, last_error, recovery_state, platform_send_started_at, entry_json, enqueued_at, updated_at, failed_at) +commitments(id, agent_id, session_key, channel, account_id, recipient_id, thread_id, sender_id, kind, sensitivity, source, status, reason, suggested_text, dedupe_key, confidence, due_earliest_ms, due_latest_ms, due_timezone, source_message_id, source_run_id, created_at_ms, updated_at_ms, attempts, last_attempt_at_ms, sent_at_ms, dismissed_at_ms, snoozed_until_ms, expired_at_ms, record_json) +migration_runs(id, started_at, finished_at, status, report_json) +migration_sources(source_key, migration_kind, source_path, target_table, source_sha256, source_size_bytes, source_record_count, last_run_id, status, imported_at, removed_source, report_json) +backup_runs(id, created_at, archive_path, status, manifest_json) +``` + +Agent database: + +```text +schema_meta(meta_key, role, schema_version, agent_id, app_version, created_at, updated_at) +sessions(session_id, session_key, session_scope, created_at, updated_at, started_at, ended_at, status, chat_type, channel, account_id, primary_conversation_id, model_provider, model, agent_harness_id, parent_session_key, spawned_by, display_name) +conversations(conversation_id, channel, account_id, kind, peer_id, parent_conversation_id, thread_id, native_channel_id, native_direct_user_id, label, metadata_json, created_at, updated_at) +session_conversations(session_id, conversation_id, role, first_seen_at, last_seen_at) +session_routes(session_key, session_id, updated_at) +session_entries(session_id, session_key, entry_json, updated_at) +transcript_events(session_id, seq, event_json, created_at) +transcript_event_identities(session_id, event_id, seq, event_type, has_parent, parent_id, message_idempotency_key, created_at) +transcript_snapshots(session_id, snapshot_id, reason, event_count, created_at, metadata_json) +vfs_entries(namespace, path, kind, content_blob, metadata_json, updated_at) +tool_artifacts(run_id, artifact_id, kind, metadata_json, blob, created_at) +run_artifacts(run_id, path, kind, metadata_json, blob, created_at) +trajectory_runtime_events(session_id, run_id, seq, event_json, created_at) +memory_index_meta(meta_key, schema_version, provider, model, provider_key, sources_json, scope_hash, chunk_tokens, chunk_overlap, vector_dims, fts_tokenizer, config_hash, updated_at) +memory_index_sources(source_kind, source_key, path, session_id, hash, mtime, size) +memory_index_chunks(id, source_kind, source_key, path, session_id, start_line, end_line, hash, model, text, embedding, embedding_dims, updated_at) +memory_embedding_cache(provider, model, provider_key, hash, embedding, dims, updated_at) +cache_entries(scope, key, value_json, blob, expires_at, updated_at) +``` + +Future search can add FTS tables without changing the canonical event tables: + +```text +transcript_events_fts(session_id, seq, text) +vfs_entries_fts(namespace, path, text) +``` + +Large values should use `blob` columns, not JSON string encoding. Keep +`value_json` for small structured data that must remain inspectable with plain +SQLite tooling. + +`agent_databases` is the canonical registry for this branch. Do not add an +`agents` table until a real agent-record owner exists; agent config remains in +`openclaw.json`. + +## Doctor Migration Shape + +Doctor should call one explicit migration step that is reportable and safe to +rerun: + +```bash +openclaw doctor --fix +``` + +`openclaw doctor --fix` invokes the state migration implementation after +ordinary config preflight and creates a verified backup before import. Runtime +startup and `openclaw migrate` must not import legacy OpenClaw state files. + +Migration properties: + +- One migration pass discovers all legacy file sources and produces a plan + before mutating anything. +- Doctor creates a verified pre-migration backup archive before importing + legacy files. +- Imports are idempotent and keyed by source path, mtime, size, hash, and target + table. +- Successful source files are removed or archived after the target database has + committed. +- Failed imports leave the source untouched and record a warning in + `migration_runs`. +- Runtime code reads SQLite only after the migration exists. +- No downgrade/export-to-runtime-files path is required. + +## Migration Inventory + +Move these into the global database: + +- Task registry runtime writes now use the shared database; the unshipped + `tasks/runs.sqlite` sidecar importer is deleted. Snapshot saves upsert by task + id and delete only missing task/delivery rows. +- Task Flow runtime writes now use the shared database; the unshipped + `tasks/flows/registry.sqlite` sidecar importer is deleted. Snapshot saves + upsert by flow id and delete only missing flow rows. +- Plugin state runtime writes now use the shared database; the unshipped + `plugin-state/state.sqlite` sidecar importer is deleted. +- Builtin memory search no longer defaults to `memory/.sqlite`; its + index tables live in the owning agent database, and the explicit + `memorySearch.store.path` sidecar opt-in has been retired to doctor config + migration. +- Builtin memory reindex resets only memory-owned tables in the agent database. + It must not replace the whole SQLite file, because the same database owns + sessions, transcripts, VFS rows, artifacts, and runtime caches. +- Sandbox container/browser registries from monolithic and sharded JSON. Runtime + writes now use the shared database; legacy JSON import remains. +- Cron job definitions, schedule state, and run history now use shared SQLite; + doctor imports/removes legacy `jobs.json`, `jobs-state.json`, and + `cron/runs/*.jsonl` files +- Device identity/auth, push, update check, commitments, OpenRouter model + cache, installed plugin index, and app-server bindings +- Device/node pairing and bootstrap records now use typed SQLite tables +- Device-pair notification subscribers and delivered-request markers now use the + shared SQLite plugin-state table instead of `device-pair-notify.json`. +- Voice-call call records now use the shared SQLite plugin-state table under the + `voice-call` / `calls` namespace instead of `calls.jsonl`; the plugin CLI + tails and summarizes SQLite-backed call history. +- QQBot gateway sessions, known-user records, and ref-index quote cache now use + SQLite plugin state under `qqbot` namespaces (`sessions`, `known-users`, + `ref-index`) instead of `session-*.json`, `known-users.json`, and + `ref-index.jsonl`; the QQBot doctor/setup migration imports and removes the + legacy files. +- Discord model-picker preferences, command-deploy hashes, and thread bindings + now use SQLite plugin state under `discord` namespaces + (`model-picker-preferences`, `command-deploy-hashes`, `thread-bindings`) + instead of `model-picker-preferences.json`, `command-deploy-cache.json`, and + `thread-bindings.json`; the Discord doctor/setup migration imports and + removes the legacy files. +- BlueBubbles catchup cursors and inbound dedupe markers now use SQLite plugin + state under `bluebubbles` namespaces (`catchup-cursors`, `inbound-dedupe`) + instead of `bluebubbles/catchup/*.json` and + `bluebubbles/inbound-dedupe/*.json`; the BlueBubbles doctor/setup migration + imports and removes the legacy files. +- Telegram update offsets, sticker cache entries, reply-chain message cache + entries, sent-message cache entries, topic-name cache entries, and thread + bindings now use SQLite plugin state under `telegram` namespaces + (`update-offsets`, `sticker-cache`, `message-cache`, `sent-messages`, + `topic-names`, `thread-bindings`) instead of `update-offset-*.json`, + `sticker-cache.json`, `*.telegram-messages.json`, + `*.telegram-sent-messages.json`, `*.telegram-topic-names.json`, and + `thread-bindings-*.json`; the Telegram doctor/setup migration imports and + removes the legacy files. +- iMessage catchup cursors, reply short-id mappings, and sent-echo dedupe rows + now use SQLite plugin state under `imessage` namespaces (`catchup-cursors`, + `reply-cache`, `sent-echoes`) instead of `imessage/catchup/*.json`, + `imessage/reply-cache.jsonl`, and `imessage/sent-echoes.jsonl`; the iMessage + doctor/setup migration imports and removes the legacy files. +- Microsoft Teams conversations, polls, delegated tokens, pending uploads, and + feedback learnings now use SQLite plugin state/blob namespaces + (`conversations`, `polls`, `delegated-tokens`, `pending-uploads`, + `feedback-learnings`) instead of `msteams-conversations.json`, + `msteams-polls.json`, `msteams-delegated.json`, + `msteams-pending-uploads.json`, and `*.learnings.json`; the Microsoft Teams + doctor/setup migration imports and removes the legacy files. +- Matrix sync cache, storage metadata, thread bindings, inbound dedupe markers, + startup verification cooldown state, credentials, recovery keys, and SDK + IndexedDB crypto snapshots now use SQLite plugin state/blob namespaces under + `matrix` (`sync-store`, `storage-meta`, `thread-bindings`, `inbound-dedupe`, + `startup-verification`, `credentials`, `recovery-key`, `idb-snapshots`) + instead of `bot-storage.json`, `storage-meta.json`, `thread-bindings.json`, + `inbound-dedupe.json`, `startup-verification.json`, `credentials.json`, + `recovery-key.json`, and `crypto-idb-snapshot.json`; the Matrix doctor/setup + migration imports and removes those legacy files from account-scoped Matrix + storage roots. +- Nostr bus cursors and profile publish state now use SQLite plugin state under + `nostr` namespaces (`bus-state`, `profile-state`) instead of + `bus-state-*.json` and `profile-state-*.json`; the Nostr doctor/setup + migration imports and removes the legacy files. +- Active Memory session toggles now use SQLite plugin state under + `active-memory/session-toggles` instead of `session-toggles.json`. +- Skill Workshop proposal queues and review counters now use SQLite plugin state + under `skill-workshop/proposals` and `skill-workshop/reviews` instead of + per-workspace `skill-workshop/.json` files. +- Outbound delivery and session delivery queues now share the global SQLite + `delivery_queue_entries` table under separate queue names + (`outbound-delivery`, `session-delivery`) instead of durable + `delivery-queue/*.json`, `delivery-queue/failed/*.json`, and + `session-delivery-queue/*.json` files. The doctor legacy-state step imports + pending and failed rows, removes stale delivered markers, and deletes the old + JSON files after import. Hot routing and retry fields are typed columns; the + JSON payload is retained only for replay/debug. +- ACPX process leases now use SQLite plugin state under `acpx/process-leases` + instead of `process-leases.json`. +- Backup and migration run metadata + +Move these into agent databases: + +- Agent session roots and compatibility-shaped session-entry payloads. Done for + runtime writes: hot session metadata is queryable in `sessions`, while the + legacy-shaped full `SessionEntry` payload remains in `session_entries`. +- Agent transcript events. Done for runtime writes. +- Compaction checkpoints and transcript snapshots. Done for runtime writes: + checkpoint transcript copies are SQLite transcript rows and checkpoint + metadata is recorded in `transcript_snapshots`. Gateway checkpoint helpers + now name these values as transcript snapshots rather than source files. +- Agent VFS scratch/workspace namespaces. Done for runtime VFS writes. +- Subagent attachment payloads. Done for runtime writes: they are SQLite VFS + seed entries and never durable workspace files. +- Tool artifacts. Done for runtime writes. +- Run artifacts. Done for worker runtime writes through the per-agent + `run_artifacts` table. +- Agent-local runtime caches. Done for worker runtime scoped cache writes through + the per-agent `cache_entries` table. Gateway-wide model caches stay in the + global database unless they become agent-specific. +- ACP parent stream logs. Done for runtime writes. +- ACP replay ledger sessions. Done for runtime writes via + `acp_replay_sessions` and `acp_replay_events`; legacy `acp/event-ledger.json` + remains only as doctor input. +- Trajectory sidecars when they are not explicit export files. Done for runtime + writes: trajectory capture writes agent-database `trajectory_runtime_events` + rows and mirrors run-scoped artifacts into SQLite. Legacy sidecars are doctor + import inputs only; export can materialize fresh JSONL support-bundle outputs + but does not read or migrate old trajectory/transcript sidecars at runtime. + Runtime trajectory capture exposes SQLite scope; JSONL path helpers are + isolated to export/debug support and are not re-exported from the runtime module. + Embedded-runner trajectory metadata records `{agentId, sessionId, sessionKey}` + identity instead of persisting a transcript locator. + +Keep these file-backed for now: + +- `openclaw.json` +- provider or CLI credential files +- plugin/package manifests +- user workspaces and Git repositories when disk mode is selected +- logs intended for operator tailing, unless a specific log surface is moved + +## Migration Plan + +### Phase 0: Freeze The Boundary + +Make the durable-state boundary explicit before moving more rows: + +- Add a `migration_runs` table to the global database. + Done for legacy-state migration execution reports. +- Add a single doctor-owned state migration service for file-to-database import. + Done: `openclaw doctor --fix` uses the legacy-state migration implementation. +- Make `plan` read-only and make `apply` create a backup, import, verify, and + then delete or quarantine old files. + Done: doctor creates a verified pre-migration backup, passes the backup path + into `migration_runs`, and reuses the importer/removal paths. +- Add static bans so new runtime code cannot write legacy state files while + migration code and tests can still seed/read them. + Done for the currently migrated legacy stores; the guard also scans nested + tests for forbidden runtime transcript locator contracts. + +### Phase 1: Finish The Global Control Plane + +Keep shared coordination state in `state/openclaw.sqlite`: + +- Agents and agent database registry +- Task and Task Flow ledgers +- Plugin state +- Sandbox container/browser registry +- Cron/scheduler run history +- Pairing, device, push, update-check, TUI, OpenRouter/model caches, and other + small gateway-scoped runtime state +- Backup and migration metadata +- Gateway media attachment bytes. Done for runtime writes; direct file paths + are temp materializations for compatibility with channel senders and sandbox + staging. Runtime allowlists accept SQLite materialization paths, not legacy + state/config media roots. Doctor imports legacy media files into + `media_blobs` and removes the source files after successful row writes. +- Debug proxy capture sessions, events, and payload blobs. Done: captures live + in the shared state DB and open through the shared state DB bootstrap, schema, + WAL, and busy-timeout settings. There is no debug proxy runtime sidecar DB + override, blob directory, or proxy-capture-only generated schema/codegen + target. + +This phase also deletes duplicate sidecar openers, permission helpers, WAL +setup, filesystem pruning, and compatibility writers from those subsystems. + +### Phase 2: Introduce Per-Agent Databases + +Create one database per agent and register it from the global DB: + +```text +~/.openclaw/state/openclaw.sqlite +~/.openclaw/agents//agent/openclaw-agent.sqlite +``` + +The global `agent_databases` row stores the path, schema version, last-seen +timestamp, and basic size/integrity metadata. Runtime code asks the registry for +the agent DB instead of deriving file paths directly. + +The agent DB owns: + +- `sessions` as the canonical session root, with `session_entries` as the + compatibility-shaped payload table attached to that root, and + `session_routes` as the unique active `session_key` lookup +- `conversations` and `session_conversations` as the normalized provider + routing identity attached to sessions +- `transcript_events` +- transcript snapshots and compaction checkpoints. Done for runtime writes. +- `vfs_entries` +- `tool_artifacts` and run artifacts +- agent-local runtime/cache rows. Done for worker scoped caches. +- ACP parent stream events +- trajectory runtime events when they are not explicit export artifacts + +### Phase 3: Replace Session Store APIs + +Done for runtime. The file-shaped session store surface is not an active +runtime contract: + +- Runtime no longer calls `loadSessionStore(storePath)` or treats `storePath` as + session identity. +- Runtime row operations are `getSessionEntry`, `upsertSessionEntry`, + `patchSessionEntry`, `deleteSessionEntry`, and `listSessionEntries`. +- Whole-store rewrite helpers, file writers, queue tests, alias pruning, and + legacy-key deletion parameters are gone from runtime. +- Deprecated root-package compatibility exports still adapt canonical + `sessions.json` paths onto the SQLite row APIs. +- `sessions.json` parsing remains only in doctor migration/import code and + doctor tests. +- Runtime lifecycle fallback reads SQLite transcript headers, not JSONL first + lines. + +Keep deleting anything that reintroduces file-lock parameters, +pruning/truncation-as-file-maintenance vocabulary, store-path identity, or tests +whose only assertion is JSON persistence. + +### Phase 4: Move Transcripts, ACP Streams, Trajectories, And VFS + +Make every agent data stream database-native: + +- Transcript append writes go through one SQLite transaction that ensures the + session header, checks message idempotency, selects the parent tail, inserts + into `transcript_events`, and records queryable identity metadata in + `transcript_event_identities`. Done for direct transcript message appends and + normal persisted `TranscriptSessionManager` appends; explicit branch + operations keep their explicit parent choice and still write SQLite rows + without deriving any file locator. +- ACP parent stream logs become rows, not `.acp-stream.jsonl` files. Done. +- ACP spawn setup no longer persists transcript JSONL paths. Done. +- Runtime trajectory capture writes event rows/artifacts directly. The explicit + support/export command can still produce support-bundle JSONL artifacts as an + export format, but session export does not recreate session JSONL. Done. +- Disk workspaces stay on disk when configured as disk mode. +- VFS scratch and experimental VFS-only workspace mode use the agent DB. + +The migration imports old JSONL files once, records counts/hashes in +`migration_runs`, and removes imported files after integrity checks. + +### Phase 5: Backup, Restore, Vacuum, And Verify + +Backups remain one archive file: + +- Checkpoint every global and agent database. +- Snapshot each DB with SQLite backup semantics or `VACUUM INTO`. +- Archive compact DB snapshots, config, external credentials, and requested + workspace exports. +- Omit raw live `*.sqlite-wal` and `*.sqlite-shm` files. +- Verify by opening every DB snapshot and running `PRAGMA integrity_check`. + `openclaw backup create` does this archive verification by default; + `--no-verify` skips only the post-write archive pass, not the snapshot + creation integrity check. +- Restore copies snapshots back to their target paths. This branch resets the + unshipped SQLite layout to `user_version = 1`; future shipped schema changes + can add explicit migrations when they are needed. + +### Phase 6: Worker Runtime + +Keep worker mode experimental while the database split lands: + +- Workers receive agent id, run id, filesystem mode, and DB registry identity. +- Each worker opens its own SQLite connection. +- Parent keeps channel delivery, approvals, config, and cancellation authority. +- Start with one worker per active run; add pooling only after lifecycle and DB + connection ownership are stable. + +### Phase 7: Delete The Old World + +Done for runtime session management. The old world is allowed only as explicit +doctor input or support/export output: + +- No runtime `sessions.json`, transcript JSONL, sandbox registry JSON, task + sidecar SQLite, or plugin-state sidecar SQLite writes. +- No JSON/session file pruning, file transcript truncation, session file locks, + or lock-shaped session tests. +- No runtime compatibility exports whose purpose is keeping old session files + current. +- Explicit support exports remain user-requested archive/materialization + formats and must not feed file names back into runtime identity. + +## Backup And Restore + +Backups should be one archive file, but database capture should be +SQLite-native: + +1. Stop long-running write activity or enter a short backup barrier. +2. For every global and agent database, run a checkpoint. +3. Snapshot each database using SQLite backup semantics or `VACUUM INTO` into a + temporary backup directory. +4. Archive the compacted database snapshots, config file, credentials directory, + selected workspaces, and a manifest. +5. Verify the archive by opening every included SQLite snapshot and running + `PRAGMA integrity_check`. + `openclaw backup create` does this by default; `--no-verify` is only for + intentionally skipping the post-write archive pass. + +Do not rely on raw live `*.sqlite`, `*.sqlite-wal`, and `*.sqlite-shm` copies as +the primary backup format. The archive manifest should record database role, +agent id, schema version, source path, snapshot path, byte size, and integrity +status. + +Restore should rebuild the global database and agent database files from the +archive snapshots. Because the SQLite layout has not shipped yet, this refactor +keeps only the version-1 schema plus doctor file-to-database import. The restore +command validates the archive first, then replaces each manifest asset from the +verified extracted payload. + +## Runtime Refactor Plan + +1. Add database registry APIs. + - Resolve global DB and per-agent DB paths. + - Keep the unshipped schemas at `user_version = 1`; do not add schema + migration runner code until a shipped schema needs it. + - Add close/checkpoint/integrity helpers used by tests, backup, and doctor. + +2. Collapse sidecar SQLite stores. + - Move plugin state tables into the global database. Done for runtime + writes; the unshipped legacy sidecar importer is deleted. + - Move task registry tables into the global database. Done for runtime + writes; the unshipped legacy sidecar importer is deleted. + - Move Task Flow tables into the global database. Done for runtime writes; + the unshipped legacy sidecar importer is deleted. + - Move builtin memory-search tables into each agent database. Done; explicit + custom `memorySearch.store.path` is now removed by doctor config migration. + Full reindex runs in place against memory tables only; the old whole-file + swap path and sidecar index swap helper are deleted. + - Delete duplicate database openers, WAL setup, permission helpers, and + close paths from those subsystems. + +3. Move agent-owned tables into per-agent databases. + - Create agent DB on demand through the global database registry. Done. + - Move runtime session entries, transcript events, VFS rows, and tool + artifacts to agent DBs. Done. + - Do not migrate branch-local shared-DB session entries, transcript events, + VFS rows, or tool artifacts; that layout never shipped. Keep only legacy + file-to-database import in doctor. + +4. Replace session store APIs. + - Remove `storePath` as the runtime identity. Done for runtime and guarded + by `check:database-first-legacy-stores`: session metadata, route updates, + command persistence, CLI session cleanup, Feishu reasoning previews, + transcript-state persistence, subagent depth, auth profile session + overrides, parent-fork logic, and QA-lab inspection now resolve the + database from canonical agent/session keys. + Gateway/TUI/UI/macOS session-list responses now expose `databasePath` + instead of legacy `path`; macOS debug surfaces show the per-agent database + as read-only state instead of writing `session.store` config. + `/status`, chat-driven trajectory export, and CLI dependency proxies no + longer propagate legacy store paths; transcript usage fallback reads + SQLite by agent/session identity. Runtime and bridge tests no longer expose + `storePath`; doctor/migration inputs own that legacy field name. + Gateway combined-session loading no longer has a special runtime branch for + non-templated `session.store` values; it aggregates per-agent SQLite rows. + The legacy session-lock doctor lane and its `.jsonl.lock` cleanup helper + were removed; SQLite is the session concurrency boundary now. + Hot runtime call sites use row-oriented helper names such as + `resolveSessionRowEntry`; the old `resolveSessionStoreEntry` compatibility + alias has been removed from runtime and plugin SDK exports. + +- Use `{ agentId, sessionKey }` row operations. + Done: `getSessionEntry`, `upsertSessionEntry`, `deleteSessionEntry`, + `patchSessionEntry`, and `listSessionEntries` are SQLite-first APIs that do + not require a session store path. Status summary, local agent status, health, + and the `openclaw sessions` listing command now read per-agent rows directly + and display per-agent SQLite database paths instead of `sessions.json` paths. +- Replace whole-store delete/insert with `upsertSessionEntry`, + `deleteSessionEntry`, `listSessionEntries`, and SQL cleanup queries. + Done for runtime: hot paths now use row APIs and conflict-retried row patches; + remaining whole-store import/replace helpers are limited to migration import + code and SQLite backend tests. + - Delete `store-writer.ts` and writer-queue tests. Done. + - Delete runtime legacy-key pruning and alias-delete parameters from session + row upserts/patches. Done. + +5. Delete runtime JSON registry behavior. + - Make sandbox registry reads and writes SQLite-only. Done. + - Import monolithic and sharded JSON only from the migration step. Done. + - Remove sharded registry locks and JSON writes. Done. + +- Keep one typed registry table instead of storing registry rows as generic + opaque JSON if the shape remains hot-path operational state. Done. + +6. Delete file-lock-shaped session mutation. + - Done for runtime lock creation and runtime lock APIs. + - The standalone legacy `.jsonl.lock` doctor cleanup lane is removed. + - `session.writeLock` is doctor-migrated legacy config, not a typed runtime + setting. + - State integrity no longer has a separate orphan transcript-file pruning + path; doctor migration imports/removes legacy JSONL sources in one place. + - Gateway singleton coordination uses typed SQLite `state_leases` rows under + `gateway_locks` and no longer exposes a file-lock directory seam. + - Generic plugin SDK dedupe persistence no longer uses file locks or JSON + files; it writes shared SQLite plugin-state rows. Done. + - QMD embed coordination uses a SQLite state lease instead of + `qmd/embed.lock`. Done. + +7. Make workers database-aware. + - Workers open their own SQLite connections. + - Parent owns delivery, channel callbacks, and config. + - Worker receives agent id, run id, filesystem mode, and DB registry + identity, not live handles. + - `vfs-only` stays experimental and uses the agent database as its storage + root. + - Keep one worker per active run first. Pooling can wait until DB connection + lifetime and cancellation behavior are boring. + +8. Backup integration. + - Teach backup to snapshot global and agent databases via SQLite backup or + `VACUUM INTO`. Done for discovered `*.sqlite` files under the state asset. + - Add backup verification for SQLite integrity and schema version. Done for + backup creation and default archive verification integrity checks. + - Record backup run metadata in SQLite. Done via the shared `backup_runs` + table with archive path, status, and manifest JSON. + - Add restore from verified archive snapshots. Done: `openclaw backup +restore` validates before extraction, uses the verifier's normalized + manifest, supports `--dry-run`, and requires `--yes` before replacing + recorded source paths. + - Include VFS/workspace export only when requested; do not export session + internals as JSON or JSONL. + +9. Delete obsolete tests and code. Done for the known runtime session surfaces. + +- Remove tests that assert runtime creation of `sessions.json` or transcript + JSONL files. Done for core session store, chat, gateway transcript events, + preview, lifecycle, command session-entry updates, auto-reply reset/trace, and + memory-core dreaming fixtures, approval target routing, session transcript + repair, security permission repair, trajectory export, and session export. + Active-memory transcript tests now assert SQLite scopes and no temporary or + persisted JSONL file creation. + The old heartbeat transcript-pruning regression was removed because + runtime no longer truncates JSONL transcripts. + Agent session-list tool tests no longer model legacy `sessions.json` paths + as the gateway response shape; app/UI/macOS tests use `databasePath`. + `/status` transcript-usage tests now seed SQLite transcript rows directly + instead of writing JSONL files. + Gateway session lifecycle tests now use SQLite transcript seeding helpers + directly; the old single-line session-file fixture shape is gone from reset + and delete coverage. + `sessions.delete` no longer returns a file-era `archived: []` field; deletion + reports only the row mutation result. The old `deleteTranscript` option is + gone too: deleting a session removes the canonical `sessions` root and lets + SQLite cascade session-owned transcript, snapshot, and trajectory rows, so no + caller can leave transcript orphans behind or forget a cleanup branch. + Context-engine trajectory capture tests now read `trajectory_runtime_events` + rows from an isolated agent database instead of reading + `session.trajectory.jsonl`. + Docker MCP channel seed scripts now seed SQLite rows directly. Direct + `sessions.json` writes are limited to doctor fixtures. + Tool Search Gateway E2E reads tool-call evidence from SQLite transcript rows + instead of scanning `agents//sessions/*.jsonl` files. + Memory-core host events and session-corpus scratch rows now live in shared + SQLite plugin-state; `events.jsonl` and `session-corpus/*.txt` are legacy + doctor migration inputs only. Active rows use `memory/session-ingestion/` + virtual paths, not `.dreams/session-corpus`. The old memory-core dreaming + repair module and its CLI/Gateway tests were removed because runtime no + longer owns file archive repair for that corpus. Memory-core + bridge/public-artifact tests no longer surface `.dreams/events.jsonl`; they + use the SQLite-backed virtual JSON artifact name. + Public SDK/Codex testing docs now say SQLite session state instead of session + files, and the channel-turn example no longer exposes a `storePath` argument. + Matrix sync state now uses the SQLite plugin-state store directly. Active + client/runtime contracts pass an account storage root, not a `bot-storage.json` + path, and doctor imports legacy `bot-storage.json` into SQLite before deleting + the source. QA Matrix restart/destructive scenarios now mutate the SQLite sync + row directly instead of creating or deleting fake `bot-storage.json` files, and + the E2EE substrate passes a sync-store root instead of a fake + `sync-store.json` path. + Matrix storage-root selection no longer scores roots by legacy sync/thread JSON + files; it uses durable root metadata plus real crypto state. + The runtime SQLite session backend test suite no longer fabricates a + `sessions.json`; legacy source fixtures now live in the doctor + tests that import them. + Gateway session tests no longer expose a `createSessionStoreDir` helper or + unused temp session-store path setup; fixture dirs are explicit, and direct + row setup uses SQLite session-row naming. + Doctor-only JSON5 session-store parser coverage moved out of infra tests and + into doctor migration tests, so runtime test suites no longer own legacy + session-file parsing. + Microsoft Teams runtime SSO/pending-upload tests no longer carry JSON sidecar + fixtures or parsers; legacy SSO token parsing lives only in the plugin + migration module. Telegram tests no longer seed fake `/tmp/*.json` store + paths; they reset the SQLite-backed message cache directly. The generic + OpenClaw test-state helper no longer exposes a legacy `auth-profiles.json` + writer; doctor auth migration tests own that fixture locally. + Runtime tests for TUI last-session pointers, exec approvals, active-memory + toggles, Matrix dedupe/startup verification, Memory Wiki source sync, + current-conversation bindings, onboarding auth, and Hermes secret imports no + longer manufacture old sidecar files or assert old filenames are absent. They + prove behavior through SQLite rows and public store APIs; doctor/migration + tests are the only place legacy source filenames belong. + Runtime tests for device/node pairing, channel allowFrom, restart intents, + restart handoff, session delivery queue entries, config health, iMessage + caches, cron jobs, PI transcript headers, subagent registries, and managed + image attachments also no longer create retired JSON/JSONL files just to prove + they are ignored or absent. + PI overflow recovery no longer has a SessionManager rewrite/truncation + fallback: tool-result truncation and context-engine transcript rewrites mutate + SQLite transcript rows, then refresh active prompt state from the database. + Persisted SessionManager message appends delegate to the atomic SQLite + transcript append helper for parent selection and idempotency. Normal + metadata/custom entry appends also select the current parent inside SQLite, so + stale manager instances do not resurrect pre-SQLite parent-chain races. + Synthetic PI tail cleanup for mid-turn prechecks and `sessions_yield` now + trims SQLite transcript state directly; the old SessionManager tail-removal + bridge and its tests are deleted. + Compaction checkpoint capture also snapshots from SQLite only; callers no + longer pass a live SessionManager as an alternate transcript source. +- Keep tests that seed legacy files only for migration. +- JSON-file proof has been replaced with SQL row proof for active runtime + surfaces. + +- Add static bans for runtime writes to legacy session/cache JSON paths. + Done for the repo guard. + +10. Make the migration report auditable. + - Record migration runs in SQLite with started/finished timestamps, source + paths, source hashes, counts, warnings, and backup path. + Done: legacy-state migration executions now persist a `migration_runs` + report with source path/table inventory, source file SHA-256, sizes, + record counts, warnings, and backup path. + Done: legacy-state migration executions also persist `migration_sources` + rows for source-level audit and future skip/backfill decisions. + - Make apply idempotent. Re-running after a partial import should either + skip an already imported source or merge by stable key. + Done: session indexes, transcripts, delivery queues, plugin state, task + ledgers, and agent-owned global SQLite rows import through stable keys or + upsert/replace semantics, so reruns merge without duplicating durable + rows. + - Failed imports must keep the original source file in place. + Done: failed transcript imports now leave the original JSONL source at + its detected path, and `migration_sources` records the source as + `warning` with `removed_source=0` for the next doctor run. + +## Performance Rules + +- One connection per thread/process is fine; do not share handles across + workers. +- Use WAL, `foreign_keys=ON`, a 30s busy timeout, and short `BEGIN IMMEDIATE` + write transactions. +- Keep write transaction helpers synchronous unless/until an async transaction + API adds explicit mutex/backpressure semantics. +- Keep parent delivery writes small and transactional. +- Avoid whole-store rewrites; use row-level upsert/delete. +- Add indexes for list-by-agent, list-by-session, updated-at, run id, and + expiration paths before moving hot code. +- Store large artifacts, media, and vectors as BLOBs or chunked BLOB rows, not + base64 or numeric-array JSON. +- Keep opaque plugin-state entries small and scoped. +- Add SQL cleanup for TTL/expiration instead of filesystem pruning. + Done for database-owned runtime stores: media, plugin state, plugin blobs, + persistent dedupe, and agent cache all expire through SQLite rows. Remaining + filesystem cleanup is limited to temporary materializations or explicit + removal commands. + +## Static Bans + +Add a repo check that fails new runtime writes to legacy state paths: + +- `sessions.json` +- `*.trajectory.jsonl` except materialized support-bundle outputs +- `.acp-stream.jsonl` +- `acp/event-ledger.json` +- `cache/*.json` runtime cache files +- `agents//agent/auth.json` +- `agents//agent/models.json` +- `credentials/oauth.json` +- `github-copilot.token.json` +- `openrouter-models.json` +- `auth-profiles.json` +- `auth-state.json` +- `exec-approvals.json` +- `workspace-state.json` +- Matrix `credentials*.json` and `recovery-key.json` +- `cron/runs/*.jsonl` +- `cron/jobs.json` +- `jobs-state.json` +- `device-pair-notify.json` +- `devices/pending.json` +- `devices/paired.json` +- `devices/bootstrap.json` +- `nodes/pending.json` +- `nodes/paired.json` +- `identity/device.json` +- `identity/device-auth.json` +- `push/web-push-subscriptions.json` +- `push/vapid-keys.json` +- `push/apns-registrations.json` +- `process-leases.json` +- `gateway-instance-id` +- `session-toggles.json` +- Memory-core `.dreams/events.jsonl` +- Memory-core `.dreams/session-corpus/` +- Memory-core `.dreams/daily-ingestion.json` +- Memory-core `.dreams/session-ingestion.json` +- Memory-core `.dreams/short-term-recall.json` +- Memory-core `.dreams/phase-signals.json` +- Memory-core `.dreams/short-term-promotion.lock` +- Skill Workshop `skill-workshop/.json` +- Skill Workshop `skill-workshop/skill-workshop-review-*.json` +- Nostr `bus-state-*.json` +- Nostr `profile-state-*.json` +- `calls.jsonl` +- `known-users.json` +- `ref-index.jsonl` +- QQBot `session-*.json` +- BlueBubbles `bluebubbles/catchup/*.json` +- BlueBubbles `bluebubbles/inbound-dedupe/*.json` +- Telegram `update-offset-*.json` +- Telegram `sticker-cache.json` +- Telegram `*.telegram-messages.json` +- Telegram `*.telegram-sent-messages.json` +- Telegram `*.telegram-topic-names.json` +- Telegram `thread-bindings-*.json` +- iMessage `catchup/*.json` +- iMessage `reply-cache.jsonl` +- iMessage `sent-echoes.jsonl` +- Microsoft Teams `msteams-conversations.json` +- Microsoft Teams `msteams-polls.json` +- Microsoft Teams `msteams-sso-tokens.json` +- Microsoft Teams `msteams-delegated.json` +- Microsoft Teams `msteams-pending-uploads.json` +- Microsoft Teams `*.learnings.json` +- Matrix `bot-storage.json` +- Matrix `sync-store.json` +- Matrix `thread-bindings.json` +- Matrix `inbound-dedupe.json` +- Matrix `startup-verification.json` +- Matrix `storage-meta.json` +- Matrix `crypto-idb-snapshot.json` +- Discord `model-picker-preferences.json` +- Discord `command-deploy-cache.json` +- sandbox registry shard JSON files +- native hook relay `/tmp` bridge JSON files +- `plugin-state/state.sqlite` +- ad-hoc `openclaw-state.sqlite` runtime sidecars +- `tasks/runs.sqlite` +- `tasks/flows/registry.sqlite` +- `bindings/current-conversations.json` +- `restart-sentinel.json` +- `gateway-restart-intent.json` +- `gateway-supervisor-restart-handoff.json` +- `gateway..lock` +- `qmd/embed.lock` +- `commands.log` +- `config-health.json` +- `port-guard.json` +- `settings/voicewake.json` +- `settings/voicewake-routing.json` +- `plugin-binding-approvals.json` +- `plugins/installs.json` +- `audit/file-transfer.jsonl` +- `audit/crestodian.jsonl` +- `crestodian/rescue-pending/*.json` +- `plugins/phone-control/armed.json` +- Memory Wiki `.openclaw-wiki/log.jsonl` +- Memory Wiki `.openclaw-wiki/state.json` +- Memory Wiki `.openclaw-wiki/locks/` +- Memory Wiki `.openclaw-wiki/source-sync.json` +- Memory Wiki `.openclaw-wiki/import-runs/*.json` +- Memory Wiki `.openclaw-wiki/cache/agent-digest.json` +- Memory Wiki `.openclaw-wiki/cache/claims.jsonl` +- ClawHub `.clawhub/lock.json` +- ClawHub `.clawhub/origin.json` +- Browser profile decoration `.openclaw-profile-decorated` +- `SessionManager.open(...)` file-backed session openers +- `SessionManager.listAll(...)` and `TranscriptSessionManager.listAll(...)` + transcript listing facades +- `SessionManager.forkFromSession(...)` and + `TranscriptSessionManager.forkFromSession(...)` transcript fork facades +- `SessionManager.newSession(...)` and `TranscriptSessionManager.newSession(...)` + mutable session replacement facades +- `SessionManager.createBranchedSession(...)` and + `TranscriptSessionManager.createBranchedSession(...)` branch-session facades + +The ban should allow tests to create legacy fixtures and allow migration code to +read/import/remove legacy file sources. Unshipped SQLite sidecars stay banned +and do not get doctor import allowances. + +## Done Criteria + +- Runtime data and cache writes go to the global or agent SQLite database. +- Runtime no longer writes session indexes, transcript JSONL, sandbox registry + JSON, task sidecar SQLite, or plugin-state sidecar SQLite. The unshipped task + and plugin-state sidecar SQLite importers are deleted. +- Legacy file import is doctor-only. +- Backup produces one archive with compact SQLite snapshots and integrity proof. +- Agent workers can run with disk, VFS scratch, or experimental VFS-only + storage. +- Config and explicit credential files remain the only expected persistent + non-database control files. +- Repo checks prevent reintroducing legacy runtime file stores. diff --git a/package.json b/package.json index b4d163f54c3e..03ea31cc3a41 100644 --- a/package.json +++ b/package.json @@ -1420,7 +1420,7 @@ "canvas:a2ui:bundle": "node scripts/bundle-a2ui.mjs", "changed:lanes": "node scripts/changed-lanes.mjs", "check": "node scripts/check.mjs", - "check:architecture": "pnpm check:import-cycles && pnpm check:madge-import-cycles && pnpm check:deprecated-api-usage && pnpm check:deprecated-jsdoc", + "check:architecture": "pnpm check:import-cycles && pnpm check:madge-import-cycles && pnpm check:deprecated-api-usage && pnpm check:deprecated-jsdoc && pnpm db:kysely:check && pnpm lint:kysely", "check:base-config-schema": "node --import tsx scripts/generate-base-config-schema.ts --check", "check:bundled-channel-config-metadata": "node --import tsx scripts/generate-bundled-channel-config-metadata.ts --check", "check:changed": "node scripts/check-changed.mjs", @@ -1486,6 +1486,8 @@ "deps:ownership-surface:report": "node scripts/dependency-ownership-surface-report.mjs", "deps:transitive-risk:report": "node scripts/transitive-manifest-risk-report.mjs", "deps:vuln:gate": "node scripts/dependency-vulnerability-gate.mjs", + "db:kysely:check": "node scripts/generate-kysely-types.mjs --verify", + "db:kysely:gen": "node scripts/generate-kysely-types.mjs", "dev": "node scripts/run-node.mjs", "dev:ui:mock": "node --import tsx scripts/control-ui-mock-dev.ts", "docs:bin": "node scripts/build-docs-list.mjs", @@ -1534,6 +1536,7 @@ "lint:auth:pairing-account-scope": "node scripts/check-pairing-account-scope.mjs", "lint:core": "node scripts/run-oxlint-shards.mjs --only=core --split-core", "lint:docker-e2e": "node scripts/check-docker-e2e-boundaries.mjs", + "lint:kysely": "node scripts/check-kysely-guardrails.mjs", "lint:docs": "pnpm dlx --config.resolution-mode=highest markdownlint-cli2 --config config/markdownlint-cli2.jsonc", "lint:docs:fix": "pnpm dlx --config.resolution-mode=highest markdownlint-cli2 --config config/markdownlint-cli2.jsonc --fix", "lint:extensions:no-deprecated-channel-access": "node --import tsx scripts/check-no-deprecated-channel-access.ts", diff --git a/scripts/check-kysely-guardrails.mjs b/scripts/check-kysely-guardrails.mjs new file mode 100644 index 000000000000..5293ed62381e --- /dev/null +++ b/scripts/check-kysely-guardrails.mjs @@ -0,0 +1,364 @@ +#!/usr/bin/env node + +import { promises as fs } from "node:fs"; +import { createRequire } from "node:module"; +import path from "node:path"; +import { + collectTypeScriptFilesFromRoots, + getPropertyNameText, + resolveRepoRoot, + runAsScript, + toLine, + unwrapExpression, +} from "./lib/ts-guard-utils.mjs"; + +const require = createRequire(import.meta.url); +const ts = require("typescript"); + +const repoRoot = resolveRepoRoot(import.meta.url); +const sourceRoots = [path.join(repoRoot, "src")]; + +const kyselyRawAllowPaths = new Set([ + "src/infra/kysely-node-sqlite.test.ts", + "src/infra/kysely-sync.ts", +]); + +const compiledRawAllowPaths = new Set([ + "src/infra/kysely-node-sqlite.ts", + "src/infra/kysely-node-sqlite.test.ts", +]); + +const rawSqliteAllowPathGroups = { + "native Kysely adapter and sync execution": [ + "src/infra/kysely-node-sqlite.ts", + "src/infra/kysely-sync.ts", + ], + "SQLite database lifecycle, schema, transactions, and pragmas": [ + "src/infra/node-sqlite.ts", + "src/infra/sqlite-integrity.ts", + "src/infra/sqlite-pragma.test-support.ts", + "src/infra/sqlite-transaction.ts", + "src/infra/sqlite-wal.ts", + "src/state/openclaw-state-db.ts", + "src/state/sqlite-schema-shape.test-support.ts", + ], + "backup snapshot maintenance": ["src/commands/backup-verify.ts", "src/infra/backup-create.ts"], + "doctor legacy state migration": ["src/infra/state-migrations.ts"], + "Kysely-backed stores that own a DatabaseSync boundary": [ + "src/acp/event-ledger.ts", + "src/agents/subagent-registry.store.ts", + "src/cron/run-log.ts", + "src/cron/store.ts", + "src/infra/outbound/current-conversation-bindings.ts", + "src/media/store.ts", + "src/plugin-sdk/memory-core-host-engine-storage.ts", + "src/plugin-state/plugin-state-store.sqlite.ts", + "src/proxy-capture/store.sqlite.ts", + "src/tasks/task-flow-registry.store.sqlite.ts", + "src/tasks/task-registry.store.sqlite.ts", + "src/tui/tui-last-session.ts", + ], +}; + +const rawSqliteAllowPathReasons = new Map(); +for (const [reason, paths] of Object.entries(rawSqliteAllowPathGroups)) { + for (const allowedPath of paths) { + if (rawSqliteAllowPathReasons.has(allowedPath)) { + throw new Error(`Duplicate raw SQLite allowlist path: ${allowedPath}`); + } + rawSqliteAllowPathReasons.set(allowedPath, reason); + } +} + +function lineText(sourceFile, node) { + const line = toLine(sourceFile, node); + return sourceFile.text.split("\n")[line - 1] ?? ""; +} + +function hasAllowComment(sourceFile, node, token) { + const line = lineText(sourceFile, node); + if (line.includes(token)) { + return true; + } + const leading = ts.getLeadingCommentRanges(sourceFile.text, node.pos) ?? []; + return leading.some((range) => sourceFile.text.slice(range.pos, range.end).includes(token)); +} + +function importSource(node) { + const moduleSpecifier = node.moduleSpecifier; + return ts.isStringLiteral(moduleSpecifier) ? moduleSpecifier.text : ""; +} + +function collectImports(sourceFile) { + const kyselySqlNames = new Set(); + const compiledQueryNames = new Set(); + const syncHelperNames = new Set(); + let hasKyselyContext = false; + let hasSqliteContext = false; + + for (const statement of sourceFile.statements) { + if (!ts.isImportDeclaration(statement)) { + continue; + } + const source = importSource(statement); + const clause = statement.importClause; + const namedBindings = clause?.namedBindings; + + if (source === "kysely") { + hasKyselyContext = true; + if (namedBindings && ts.isNamedImports(namedBindings)) { + for (const element of namedBindings.elements) { + const importedName = element.propertyName?.text ?? element.name.text; + if (importedName === "sql") { + kyselySqlNames.add(element.name.text); + } + if (importedName === "CompiledQuery") { + compiledQueryNames.add(element.name.text); + } + } + } + } + + if (source.endsWith("kysely-sync.js") || source.endsWith("kysely-node-sqlite.js")) { + hasKyselyContext = true; + if (namedBindings && ts.isNamedImports(namedBindings)) { + for (const element of namedBindings.elements) { + const importedName = element.propertyName?.text ?? element.name.text; + if ( + importedName === "executeSqliteQuerySync" || + importedName === "executeSqliteQueryTakeFirstSync" || + importedName === "executeSqliteQueryTakeFirstOrThrowSync" + ) { + syncHelperNames.add(element.name.text); + } + if (importedName === "getNodeSqliteKysely") { + hasKyselyContext = true; + hasSqliteContext = true; + } + } + } + } + + if ( + source === "node:sqlite" || + source.endsWith("node-sqlite.js") || + source.endsWith("sqlite-transaction.js") || + source.endsWith("sqlite-wal.js") || + source.endsWith("openclaw-state-db.js") + ) { + hasSqliteContext = true; + } + } + + return { + compiledQueryNames, + hasKyselyContext, + hasSqliteContext, + kyselySqlNames, + syncHelperNames, + }; +} + +function addViolation(violations, sourceFile, node, message) { + violations.push({ + line: toLine(sourceFile, node), + message, + }); +} + +function isIdentifierNamed(node, names) { + const unwrapped = unwrapExpression(node); + return ts.isIdentifier(unwrapped) && names.has(unwrapped.text); +} + +function isTestPath(relativePath) { + return ( + /\.(?:test|spec|e2e)\.ts$/u.test(relativePath) || + relativePath.includes(".test-helpers.") || + relativePath.includes(".test-support.") + ); +} + +function isSqliteStorePath(relativePath) { + return relativePath.endsWith(".sqlite.ts") || relativePath.includes(".store.sqlite.ts"); +} + +function isLikelySqliteReceiver(expression) { + const unwrapped = unwrapExpression(expression); + if (ts.isIdentifier(unwrapped)) { + return /^(?:db|database|legacyDb|stateDb|agentDb)$/u.test(unwrapped.text); + } + return ts.isPropertyAccessExpression(unwrapped) && getPropertyNameText(unwrapped.name) === "db"; +} + +function isPersistedRowExpression(expression) { + const unwrapped = unwrapExpression(expression); + if (ts.isPropertyAccessExpression(unwrapped)) { + const owner = unwrapExpression(unwrapped.expression); + return ts.isIdentifier(owner) && /^(?:row|record|entry)$/u.test(owner.text); + } + if (ts.isElementAccessExpression(unwrapped)) { + const owner = unwrapExpression(unwrapped.expression); + return ts.isIdentifier(owner) && /^(?:row|record|entry)$/u.test(owner.text); + } + return false; +} + +function isPersistedStringCastType(typeText) { + return [ + /\bTaskRecord\["(?:runtime|scopeKind|status|deliveryStatus|notifyPolicy|terminalOutcome)"\]/u, + /\bTaskFlowRecord\["(?:status|notifyPolicy)"\]/u, + /\bTaskFlowSyncMode\b/u, + /\bVirtualAgentFsEntryKind\b/u, + /\b[A-Z][A-Za-z0-9]*(?:Status|Kind|Mode|Policy|Runtime|Outcome)\b/u, + ].some((pattern) => pattern.test(typeText)); +} + +export function collectKyselyGuardrailViolations(content, relativePath) { + const sourceFile = ts.createSourceFile(relativePath, content, ts.ScriptTarget.Latest, true); + const imports = collectImports(sourceFile); + const violations = []; + + function visit(node) { + if ( + isSqliteStorePath(relativePath) && + (ts.isAsExpression(node) || ts.isTypeAssertionExpression(node)) && + isPersistedStringCastType(node.type.getText(sourceFile)) && + isPersistedRowExpression(node.expression) && + !hasAllowComment(sourceFile, node, "sqlite-allow-persisted-cast") + ) { + addViolation( + violations, + sourceFile, + node, + "persisted SQLite enum-like values must be parsed through closed validators, not cast", + ); + } + + if ( + ts.isCallExpression(node) && + ts.isIdentifier(node.expression) && + imports.syncHelperNames.has(node.expression.text) && + node.typeArguments?.length && + !hasAllowComment(sourceFile, node, "kysely-allow-raw") + ) { + addViolation( + violations, + sourceFile, + node, + "sync helper row generic at call site; let Kysely infer builder result rows", + ); + } + + if ( + ts.isTaggedTemplateExpression(node) && + node.typeArguments?.length && + isIdentifierNamed(node.tag, imports.kyselySqlNames) && + !kyselyRawAllowPaths.has(relativePath) && + !hasAllowComment(sourceFile, node, "kysely-allow-raw") + ) { + addViolation( + violations, + sourceFile, + node, + "typed raw sql snippet needs a small helper or allowlisted boundary", + ); + } + + if ( + ts.isCallExpression(node) && + ts.isPropertyAccessExpression(node.expression) && + isIdentifierNamed(node.expression.expression, imports.kyselySqlNames) && + ["ref", "table", "id", "raw"].includes(getPropertyNameText(node.expression.name) ?? "") && + !hasAllowComment(sourceFile, node, "kysely-allow-raw") + ) { + addViolation( + violations, + sourceFile, + node, + "raw Kysely identifier helper requires a closed-set validator and local allow comment", + ); + } + + if ( + imports.hasKyselyContext && + ts.isPropertyAccessExpression(node) && + getPropertyNameText(node.name) === "dynamic" && + !hasAllowComment(sourceFile, node, "kysely-allow-raw") + ) { + addViolation( + violations, + sourceFile, + node, + "Kysely dynamic refs bypass literal reference checking; use only behind closed unions", + ); + } + + if ( + ts.isCallExpression(node) && + ts.isPropertyAccessExpression(node.expression) && + isIdentifierNamed(node.expression.expression, imports.compiledQueryNames) && + getPropertyNameText(node.expression.name) === "raw" && + !compiledRawAllowPaths.has(relativePath) && + !hasAllowComment(sourceFile, node, "kysely-allow-raw") + ) { + addViolation( + violations, + sourceFile, + node, + "CompiledQuery.raw is only allowed in the native SQLite dialect/test boundary", + ); + } + + if ( + imports.hasSqliteContext && + !isTestPath(relativePath) && + ts.isCallExpression(node) && + ts.isPropertyAccessExpression(node.expression) && + ["prepare", "exec"].includes(getPropertyNameText(node.expression.name) ?? "") && + isLikelySqliteReceiver(node.expression.expression) && + !rawSqliteAllowPathReasons.has(relativePath) && + !hasAllowComment(sourceFile, node, "sqlite-allow-raw") + ) { + addViolation( + violations, + sourceFile, + node, + "new raw node:sqlite access requires Kysely or an explicit raw SQLite allowlist entry", + ); + } + + ts.forEachChild(node, visit); + } + + visit(sourceFile); + return violations; +} + +export async function collectKyselyGuardrails() { + const files = await collectTypeScriptFilesFromRoots(sourceRoots, { includeTests: true }); + const violations = []; + for (const filePath of files) { + const relativePath = path.relative(repoRoot, filePath).split(path.sep).join("/"); + const content = await fs.readFile(filePath, "utf8"); + for (const violation of collectKyselyGuardrailViolations(content, relativePath)) { + violations.push({ path: relativePath, ...violation }); + } + } + return violations; +} + +export async function main() { + const violations = await collectKyselyGuardrails(); + if (violations.length === 0) { + console.log("Kysely guardrails OK"); + return; + } + console.error("Kysely guardrail violations:"); + for (const violation of violations) { + console.error(`- ${violation.path}:${violation.line}: ${violation.message}`); + } + process.exit(1); +} + +runAsScript(import.meta.url, main); diff --git a/scripts/generate-kysely-types.mjs b/scripts/generate-kysely-types.mjs new file mode 100644 index 000000000000..5d9bbf344ce6 --- /dev/null +++ b/scripts/generate-kysely-types.mjs @@ -0,0 +1,154 @@ +#!/usr/bin/env node + +import fs from "node:fs"; +import process from "node:process"; +import { DatabaseSync } from "node:sqlite"; + +const SCHEMAS = [ + { + name: "openclaw-state", + schema: "src/state/openclaw-state-schema.sql", + outFile: "src/state/openclaw-state-db.generated.d.ts", + schemaOutFile: "src/state/openclaw-state-schema.generated.ts", + schemaExport: "OPENCLAW_STATE_SCHEMA_SQL", + }, +]; + +const verify = process.argv.includes("--verify") || process.argv.includes("--check"); + +function toInterfaceName(tableName) { + return tableName + .split("_") + .map((part) => `${part.slice(0, 1).toUpperCase()}${part.slice(1)}`) + .join(""); +} + +function columnBaseType(columnType) { + const normalized = columnType.toUpperCase(); + if (normalized.includes("BLOB")) { + return "Uint8Array"; + } + if ( + normalized.includes("INT") || + normalized.includes("REAL") || + normalized.includes("FLOA") || + normalized.includes("DOUB") || + normalized.includes("NUM") || + normalized.includes("DEC") + ) { + return "number"; + } + return "string"; +} + +function columnType(column, primaryKeyColumnCount) { + const baseType = columnBaseType(String(column.type ?? "")); + const generated = + column.dflt_value != null || + (primaryKeyColumnCount === 1 && + Number(column.pk) > 0 && + String(column.type ?? "") + .toUpperCase() + .includes("INT")); + const nullable = Number(column.notnull) !== 1 && !generated; + const valueType = nullable ? `${baseType} | null` : baseType; + return generated ? `Generated<${valueType}>` : valueType; +} + +function quoteSqliteIdentifier(identifier) { + return `"${identifier.replaceAll('"', '""')}"`; +} + +function generateTypes(db) { + const tables = db + .prepare( + "SELECT name FROM sqlite_schema WHERE type = 'table' AND name NOT LIKE 'sqlite_%' ORDER BY name;", + ) + .all() + .map((row) => String(row.name)); + const lines = [ + "/**", + " * This file was generated by kysely-codegen.", + " * Please do not edit it manually.", + " */", + "", + 'import type { ColumnType } from "kysely";', + "", + "export type Generated =", + " T extends ColumnType", + " ? ColumnType", + " : ColumnType;", + "", + ]; + + const interfaces = []; + for (const table of tables) { + const interfaceName = toInterfaceName(table); + interfaces.push({ interfaceName, table }); + lines.push(`export interface ${interfaceName} {`); + const columns = db + .prepare(`PRAGMA table_xinfo(${quoteSqliteIdentifier(table)});`) + .all() + .filter((column) => Number(column.hidden) === 0) + .toSorted((left, right) => String(left.name).localeCompare(String(right.name))); + const primaryKeyColumnCount = columns.filter((column) => Number(column.pk) > 0).length; + for (const column of columns) { + lines.push(` ${column.name}: ${columnType(column, primaryKeyColumnCount)};`); + } + lines.push("}", ""); + } + + lines.push("export interface DB {"); + for (const { interfaceName, table } of interfaces) { + lines.push(` ${table}: ${interfaceName};`); + } + lines.push("}", ""); + return lines.join("\n"); +} + +function readUtf8(file) { + return fs.readFileSync(file, "utf8"); +} + +function generatedSchemaModule(schema) { + const source = readUtf8(schema.schema).trimEnd(); + const literal = source.replaceAll("\\", "\\\\").replaceAll("`", "\\`").replaceAll("${", "\\${"); + return [ + "/**", + " * This file was generated from the SQLite schema source.", + " * Please do not edit it manually.", + " */", + "", + `export const ${schema.schemaExport} = \`${literal}\\n\`;`, + "", + ].join("\n"); +} + +function generate(schema) { + const db = new DatabaseSync(":memory:"); + try { + db.exec(readUtf8(schema.schema)); + const typesSource = generateTypes(db); + const schemaSource = generatedSchemaModule(schema); + + if (verify) { + if (typesSource !== readUtf8(schema.outFile)) { + console.error(`${schema.outFile} is out of date. Run pnpm db:kysely:gen.`); + process.exitCode = 1; + } + if (schemaSource !== readUtf8(schema.schemaOutFile)) { + console.error(`${schema.schemaOutFile} is out of date. Run pnpm db:kysely:gen.`); + process.exitCode = 1; + } + } else { + fs.writeFileSync(schema.outFile, typesSource); + fs.writeFileSync(schema.schemaOutFile, schemaSource); + } + } finally { + db.close(); + } +} + +for (const schema of SCHEMAS) { + generate(schema); +} diff --git a/src/commands/doctor-state-migrations.test.ts b/src/commands/doctor-state-migrations.test.ts index 094902d18693..369870c07d4b 100644 --- a/src/commands/doctor-state-migrations.test.ts +++ b/src/commands/doctor-state-migrations.test.ts @@ -3,6 +3,7 @@ import os from "node:os"; import path from "node:path"; import { afterEach, describe, expect, it, vi } from "vitest"; import type { OpenClawConfig } from "../config/config.js"; +import { requireNodeSqlite } from "../infra/node-sqlite.js"; import { MAX_PLUGIN_STATE_ENTRIES_PER_PLUGIN, createPluginStateKeyedStore, @@ -259,6 +260,34 @@ function writeLegacySessionsFixture(params: { return legacySessionsDir; } +function writeLegacyPluginStateSidecar(root: string): string { + const sourcePath = path.join(root, "plugin-state", "state.sqlite"); + fs.mkdirSync(path.dirname(sourcePath), { recursive: true }); + const sqlite = requireNodeSqlite(); + const db = new sqlite.DatabaseSync(sourcePath); + try { + db.exec(` + CREATE TABLE plugin_state_entries ( + plugin_id TEXT NOT NULL, + namespace TEXT NOT NULL, + entry_key TEXT NOT NULL, + value_json TEXT NOT NULL, + created_at INTEGER NOT NULL, + expires_at INTEGER, + PRIMARY KEY (plugin_id, namespace, entry_key) + ); + `); + db.prepare(` + INSERT INTO plugin_state_entries ( + plugin_id, namespace, entry_key, value_json, created_at, expires_at + ) VALUES (?, ?, ?, ?, ?, ?) + `).run("discord", "components", "interaction:1", '{"ok":true}', 1000, null); + } finally { + db.close(); + } + return sourcePath; +} + async function detectAndRunMigrations(params: { root: string; cfg: OpenClawConfig; @@ -778,6 +807,185 @@ describe("doctor legacy state migrations", () => { }); }); + it("imports the shipped plugin-state SQLite sidecar into shared state", async () => { + const root = await makeTempRoot(); + const sourcePath = writeLegacyPluginStateSidecar(root); + + const detected = await detectLegacyStateMigrations({ + cfg: {}, + env: { OPENCLAW_STATE_DIR: root } as NodeJS.ProcessEnv, + }); + expect(detected.pluginStateSidecar).toEqual({ sourcePath, hasLegacy: true }); + expect(detected.preview).toContain( + `- Plugin state sidecar: ${sourcePath} → shared SQLite state`, + ); + + const result = await runLegacyStateMigrations({ detected }); + + expect(result.warnings).toStrictEqual([]); + expect(result.changes).toContain("Migrated 1 plugin-state sidecar entry → shared SQLite state"); + expect(result.changes).toContain( + `Archived plugin-state sidecar legacy source → ${sourcePath}.migrated`, + ); + expect(fs.existsSync(sourcePath)).toBe(false); + expect(fs.existsSync(`${sourcePath}.migrated`)).toBe(true); + + await withStateDir(root, async () => { + const store = createPluginStateKeyedStore<{ ok: boolean }>("discord", { + namespace: "components", + maxEntries: 10, + }); + await expect(store.lookup("interaction:1")).resolves.toEqual({ ok: true }); + }); + }); + + it("auto-migrates the shipped plugin-state SQLite sidecar by itself", async () => { + const root = await makeTempRoot(); + const sourcePath = writeLegacyPluginStateSidecar(root); + + const result = await autoMigrateLegacyState({ + cfg: {}, + env: { OPENCLAW_STATE_DIR: root } as NodeJS.ProcessEnv, + log: { info: vi.fn(), warn: vi.fn() }, + }); + + expect(result.skipped).toBe(false); + expect(result.changes).toContain("Migrated 1 plugin-state sidecar entry → shared SQLite state"); + expect(fs.existsSync(sourcePath)).toBe(false); + expect(fs.existsSync(`${sourcePath}.migrated`)).toBe(true); + + await withStateDir(root, async () => { + const store = createPluginStateKeyedStore<{ ok: boolean }>("discord", { + namespace: "components", + maxEntries: 10, + }); + await expect(store.lookup("interaction:1")).resolves.toEqual({ ok: true }); + }); + }); + + it("auto-migrates the plugin-state sidecar when custom agent dirs skip session migration", async () => { + const root = await makeTempRoot(); + const sourcePath = writeLegacyPluginStateSidecar(root); + + const result = await autoMigrateLegacyState({ + cfg: {}, + env: { + OPENCLAW_STATE_DIR: root, + OPENCLAW_AGENT_DIR: path.join(root, "custom-agent"), + } as NodeJS.ProcessEnv, + log: { info: vi.fn(), warn: vi.fn() }, + }); + + expect(result.skipped).toBe(true); + expect(result.changes).toContain("Migrated 1 plugin-state sidecar entry → shared SQLite state"); + expect(fs.existsSync(sourcePath)).toBe(false); + expect(fs.existsSync(`${sourcePath}.migrated`)).toBe(true); + + await withStateDir(root, async () => { + const store = createPluginStateKeyedStore<{ ok: boolean }>("discord", { + namespace: "components", + maxEntries: 10, + }); + await expect(store.lookup("interaction:1")).resolves.toEqual({ ok: true }); + }); + }); + + it("keeps the plugin-state sidecar when shared state already has a conflicting row", async () => { + const root = await makeTempRoot(); + const sourcePath = writeLegacyPluginStateSidecar(root); + await withStateDir(root, async () => { + const store = createPluginStateKeyedStore<{ ok: boolean }>("discord", { + namespace: "components", + maxEntries: 10, + }); + await store.register("interaction:1", { ok: false }); + }); + resetPluginStateStoreForTests(); + + const detected = await detectLegacyStateMigrations({ + cfg: {}, + env: { OPENCLAW_STATE_DIR: root } as NodeJS.ProcessEnv, + }); + const result = await runLegacyStateMigrations({ detected }); + + expect(result.warnings).toStrictEqual([ + "Left plugin-state sidecar in place because 1 row already existed in shared state: discord/components/interaction:1", + ]); + expect(fs.existsSync(sourcePath)).toBe(true); + expect(fs.existsSync(`${sourcePath}.migrated`)).toBe(false); + + await withStateDir(root, async () => { + const store = createPluginStateKeyedStore<{ ok: boolean }>("discord", { + namespace: "components", + maxEntries: 10, + }); + await expect(store.lookup("interaction:1")).resolves.toEqual({ ok: false }); + }); + }); + + it("archives the plugin-state sidecar when conflicting rows already match", async () => { + const root = await makeTempRoot(); + const sourcePath = writeLegacyPluginStateSidecar(root); + await withStateDir(root, async () => { + seedPluginStateEntriesForTests([ + { + pluginId: "discord", + namespace: "components", + key: "interaction:1", + value: { ok: true }, + createdAt: 1000, + expiresAt: null, + }, + ]); + }); + resetPluginStateStoreForTests(); + + const detected = await detectLegacyStateMigrations({ + cfg: {}, + env: { OPENCLAW_STATE_DIR: root } as NodeJS.ProcessEnv, + }); + const result = await runLegacyStateMigrations({ detected }); + + expect(result.warnings).toStrictEqual([]); + expect(fs.existsSync(sourcePath)).toBe(false); + expect(fs.existsSync(`${sourcePath}.migrated`)).toBe(true); + }); + + it("lets live sidecar rows replace expired shared plugin state during migration", async () => { + const root = await makeTempRoot(); + const sourcePath = writeLegacyPluginStateSidecar(root); + await withStateDir(root, async () => { + seedPluginStateEntriesForTests([ + { + pluginId: "discord", + namespace: "components", + key: "interaction:1", + value: { ok: false }, + expiresAt: 1, + }, + ]); + }); + resetPluginStateStoreForTests(); + + const detected = await detectLegacyStateMigrations({ + cfg: {}, + env: { OPENCLAW_STATE_DIR: root } as NodeJS.ProcessEnv, + }); + const result = await runLegacyStateMigrations({ detected }); + + expect(result.warnings).toStrictEqual([]); + expect(fs.existsSync(sourcePath)).toBe(false); + expect(fs.existsSync(`${sourcePath}.migrated`)).toBe(true); + + await withStateDir(root, async () => { + const store = createPluginStateKeyedStore<{ ok: boolean }>("discord", { + namespace: "components", + maxEntries: 10, + }); + await expect(store.lookup("interaction:1")).resolves.toEqual({ ok: true }); + }); + }); + it("routes legacy state to the default agent entry", async () => { const root = await makeTempRoot(); const cfg: OpenClawConfig = { diff --git a/src/commands/doctor.e2e-harness.ts b/src/commands/doctor.e2e-harness.ts index 8839ab3f01c1..fc34e695723c 100644 --- a/src/commands/doctor.e2e-harness.ts +++ b/src/commands/doctor.e2e-harness.ts @@ -206,6 +206,10 @@ function createLegacyStateMigrationDetectionResult(params?: { targetDir: "/tmp/state/agents/main/agent", hasLegacy: false, }, + pluginStateSidecar: { + sourcePath: "/tmp/state/plugin-state/state.sqlite", + hasLegacy: false, + }, channelPlans: { hasLegacy: false, plans: [], diff --git a/src/infra/kysely-sync.ts b/src/infra/kysely-sync.ts new file mode 100644 index 000000000000..f1d3184530b4 --- /dev/null +++ b/src/infra/kysely-sync.ts @@ -0,0 +1,76 @@ +import type { DatabaseSync, SQLInputValue } from "node:sqlite"; +import type { CompiledQuery, Kysely, QueryResult } from "kysely"; +import { InsertQueryNode, Kysely as KyselyInstance } from "kysely"; +import { NodeSqliteKyselyDialect } from "./kysely-node-sqlite.js"; + +type CompilableQuery = { + compile(): CompiledQuery; +}; + +const kyselyByDatabase = new WeakMap>(); + +export function getNodeSqliteKysely(db: DatabaseSync): Kysely { + const existing = kyselyByDatabase.get(db); + if (existing) { + return existing as Kysely; + } + const kysely = new KyselyInstance({ + dialect: new NodeSqliteKyselyDialect({ database: db }), + }); + kyselyByDatabase.set(db, kysely as Kysely); + return kysely; +} + +export function executeCompiledSqliteQuerySync( + db: DatabaseSync, + compiledQuery: CompiledQuery, +): QueryResult { + const statement = db.prepare(compiledQuery.sql); + const parameters = compiledQuery.parameters as SQLInputValue[]; + + if (statement.columns().length > 0) { + return { rows: statement.all(...parameters) as Row[] }; + } + + const { changes, lastInsertRowid } = statement.run(...parameters); + const result: QueryResult = { + numAffectedRows: BigInt(changes), + rows: [], + }; + if (InsertQueryNode.is(compiledQuery.query) && changes > 0) { + return { + ...result, + insertId: BigInt(lastInsertRowid), + }; + } + return result; +} + +export function executeSqliteQuerySync( + db: DatabaseSync, + query: CompilableQuery, +): QueryResult { + return executeCompiledSqliteQuerySync(db, query.compile()); +} + +export function executeSqliteQueryTakeFirstSync( + db: DatabaseSync, + query: CompilableQuery, +): Row | undefined { + return executeSqliteQuerySync(db, query).rows[0]; +} + +export function executeSqliteQueryTakeFirstOrThrowSync( + db: DatabaseSync, + query: CompilableQuery, +): Row { + const row = executeSqliteQueryTakeFirstSync(db, query); + if (!row) { + throw new Error("Kysely query returned no rows"); + } + return row; +} + +export function clearNodeSqliteKyselyCacheForDatabase(db: DatabaseSync): void { + kyselyByDatabase.delete(db); +} diff --git a/src/infra/kysely-sync.types.test.ts b/src/infra/kysely-sync.types.test.ts new file mode 100644 index 000000000000..6551f1f32a1c --- /dev/null +++ b/src/infra/kysely-sync.types.test.ts @@ -0,0 +1,67 @@ +import type { DatabaseSync } from "node:sqlite"; +import { describe, expect, expectTypeOf, it } from "vitest"; +import { + executeSqliteQuerySync, + executeSqliteQueryTakeFirstSync, + getNodeSqliteKysely, +} from "./kysely-sync.js"; + +type TypeTestDatabase = { + type_test_items: { + id: number; + name: string | null; + data: Uint8Array; + }; +}; + +describe("kysely sync helper types", () => { + it("preserves Kysely builder result rows through sync helpers", () => { + const nativeDb = {} as DatabaseSync; + const db = getNodeSqliteKysely(nativeDb); + const query = db + .selectFrom("type_test_items") + .select((eb) => ["id as itemId", "name", "data", eb.fn.countAll().as("total")]) + .groupBy(["id", "name", "data"]); + + const assertTypes = () => { + const result = executeSqliteQuerySync(nativeDb, query); + expectTypeOf(result.rows).toEqualTypeOf< + Array<{ + itemId: number; + name: string | null; + data: Uint8Array; + total: number; + }> + >(); + + const row = executeSqliteQueryTakeFirstSync(nativeDb, query); + expectTypeOf(row).toEqualTypeOf< + | { + itemId: number; + name: string | null; + data: Uint8Array; + total: number; + } + | undefined + >(); + + // @ts-expect-error Kysely checks selected column string literals. + db.selectFrom("type_test_items").select("missing_column"); + + // @ts-expect-error Kysely checks table string literals. + db.selectFrom("missing_table").selectAll(); + + // @ts-expect-error Kysely checks where-reference string literals. + db.selectFrom("type_test_items").select("id").where("missing_column", "=", 1); + + // @ts-expect-error Kysely checks grouped column string literals. + query.groupBy("missing_column"); + + // @ts-expect-error Kysely checks order references and selected aliases. + query.orderBy("missingAlias"); + }; + void assertTypes; + + expect(query.compile().sql).toContain("select"); + }); +}); diff --git a/src/infra/sqlite-pragma.test-support.ts b/src/infra/sqlite-pragma.test-support.ts new file mode 100644 index 000000000000..1188b17f407f --- /dev/null +++ b/src/infra/sqlite-pragma.test-support.ts @@ -0,0 +1,14 @@ +import type { DatabaseSync } from "node:sqlite"; + +export type SqliteNumberPragma = + | "busy_timeout" + | "foreign_keys" + | "synchronous" + | "user_version" + | "wal_autocheckpoint"; + +export function readSqliteNumberPragma(db: DatabaseSync, pragma: SqliteNumberPragma): number { + const row = db.prepare(`PRAGMA ${pragma}`).get() as Record | undefined; + const value = row?.[pragma] ?? row?.timeout; + return typeof value === "bigint" ? Number(value) : Number(value); +} diff --git a/src/infra/sqlite-transaction.test.ts b/src/infra/sqlite-transaction.test.ts new file mode 100644 index 000000000000..5bd4eb1b84ec --- /dev/null +++ b/src/infra/sqlite-transaction.test.ts @@ -0,0 +1,123 @@ +import { afterEach, describe, expect, it } from "vitest"; +import { requireNodeSqlite } from "./node-sqlite.js"; +import { runSqliteImmediateTransactionSync } from "./sqlite-transaction.js"; + +const openDatabases: Array = []; + +function createDatabase(): import("node:sqlite").DatabaseSync { + const { DatabaseSync } = requireNodeSqlite(); + const db = new DatabaseSync(":memory:"); + db.exec("CREATE TABLE entries (id TEXT NOT NULL PRIMARY KEY, value TEXT NOT NULL);"); + openDatabases.push(db); + return db; +} + +function readEntries(db: import("node:sqlite").DatabaseSync): string[] { + return db + .prepare("SELECT id FROM entries ORDER BY id") + .all() + .map((row) => (row as { id: string }).id); +} + +afterEach(() => { + for (const db of openDatabases.splice(0)) { + db.close(); + } +}); + +describe("runSqliteImmediateTransactionSync", () => { + it("keeps outer writes when a nested savepoint rolls back", () => { + const db = createDatabase(); + + runSqliteImmediateTransactionSync(db, () => { + db.prepare("INSERT INTO entries(id, value) VALUES (?, ?)").run("outer", "kept"); + expect(() => + runSqliteImmediateTransactionSync(db, () => { + db.prepare("INSERT INTO entries(id, value) VALUES (?, ?)").run("inner", "rolled back"); + throw new Error("nested failure"); + }), + ).toThrow("nested failure"); + }); + + expect(readEntries(db)).toEqual(["outer"]); + }); + + it("commits nested savepoint writes with the outer transaction", () => { + const db = createDatabase(); + + runSqliteImmediateTransactionSync(db, () => { + db.prepare("INSERT INTO entries(id, value) VALUES (?, ?)").run("outer", "kept"); + runSqliteImmediateTransactionSync(db, () => { + db.prepare("INSERT INTO entries(id, value) VALUES (?, ?)").run("inner", "kept"); + }); + }); + + expect(readEntries(db)).toEqual(["inner", "outer"]); + }); + + it("rejects Promise-returning operations and rolls back their synchronous writes", () => { + const db = createDatabase(); + + expect(() => + runSqliteImmediateTransactionSync(db, async () => { + db.prepare("INSERT INTO entries(id, value) VALUES (?, ?)").run("async", "rolled back"); + return "done"; + }), + ).toThrow("must be synchronous"); + expect(readEntries(db)).toEqual([]); + + runSqliteImmediateTransactionSync(db, () => { + db.prepare("INSERT INTO entries(id, value) VALUES (?, ?)").run("after", "works"); + }); + expect(readEntries(db)).toEqual(["after"]); + }); + + it("retries retryable commit failures without rolling back successful writes", () => { + const execCalls: string[] = []; + let commitAttempts = 0; + const db = { + exec(sql: string) { + execCalls.push(sql); + if (sql === "COMMIT") { + commitAttempts += 1; + if (commitAttempts === 1) { + throw Object.assign(new Error("database is busy"), { code: "SQLITE_BUSY" }); + } + } + }, + } as import("node:sqlite").DatabaseSync; + + const result = runSqliteImmediateTransactionSync(db, () => "committed"); + + expect(result).toBe("committed"); + expect(execCalls).toEqual(["BEGIN IMMEDIATE", "COMMIT", "COMMIT"]); + }); + + it("rolls back and clears depth after exhausted retryable commit failures", () => { + const execCalls: string[] = []; + let failCommits = true; + const db = { + exec(sql: string) { + execCalls.push(sql); + if (failCommits && sql === "COMMIT") { + throw Object.assign(new Error("database is busy"), { code: "SQLITE_BUSY" }); + } + }, + close() {}, + } as import("node:sqlite").DatabaseSync; + + expect(() => runSqliteImmediateTransactionSync(db, () => "not committed")).toThrow( + "database is busy", + ); + + expect(execCalls.filter((sql) => sql === "COMMIT")).toHaveLength(8); + expect(execCalls.at(-1)).toBe("ROLLBACK"); + + execCalls.length = 0; + failCommits = false; + const result = runSqliteImmediateTransactionSync(db, () => "committed later"); + + expect(result).toBe("committed later"); + expect(execCalls).toEqual(["BEGIN IMMEDIATE", "COMMIT"]); + }); +}); diff --git a/src/infra/sqlite-transaction.ts b/src/infra/sqlite-transaction.ts new file mode 100644 index 000000000000..052e028de1fe --- /dev/null +++ b/src/infra/sqlite-transaction.ts @@ -0,0 +1,132 @@ +import type { DatabaseSync } from "node:sqlite"; + +const transactionDepthByDatabase = new WeakMap(); + +const RETRYABLE_COMMIT_ERROR_CODES = new Set(["SQLITE_BUSY", "SQLITE_LOCKED"]); +const MAX_COMMIT_ATTEMPTS = 8; + +let nextSavepointId = 0; + +function nextSavepointName(): string { + nextSavepointId += 1; + return `openclaw_tx_${nextSavepointId}`; +} + +function isPromiseLike(value: unknown): value is PromiseLike { + return Boolean(value && typeof (value as { then?: unknown }).then === "function"); +} + +function assertSyncTransactionResult(value: unknown): void { + if (isPromiseLike(value)) { + throw new Error( + "SQLite write transactions must be synchronous; Promise returns are not supported.", + ); + } +} + +function isRetryableCommitError(error: unknown): boolean { + const code = error && typeof error === "object" ? (error as { code?: unknown }).code : undefined; + return typeof code === "string" && RETRYABLE_COMMIT_ERROR_CODES.has(code); +} + +function commitImmediateTransaction(db: DatabaseSync): void { + for (let attempt = 1; ; attempt += 1) { + try { + db.exec("COMMIT"); + return; + } catch (error) { + if (!isRetryableCommitError(error) || attempt >= MAX_COMMIT_ATTEMPTS) { + throw error; + } + } + } +} + +function abortImmediateTransaction(db: DatabaseSync): void { + try { + db.exec("ROLLBACK"); + } catch { + // If rollback itself fails, close the handle so callers cannot keep using a + // connection that may still hold an abandoned write transaction. + try { + db.close(); + } catch { + // Preserve the original transaction error; close failure is secondary. + } + } +} + +function getTransactionDepth(db: DatabaseSync): number { + return transactionDepthByDatabase.get(db) ?? 0; +} + +function setTransactionDepth(db: DatabaseSync, depth: number): void { + if (depth <= 0) { + transactionDepthByDatabase.delete(db); + return; + } + transactionDepthByDatabase.set(db, depth); +} + +export function runSqliteImmediateTransactionSync(db: DatabaseSync, operation: () => T): T { + const depth = getTransactionDepth(db); + if (depth > 0) { + const savepointName = nextSavepointName(); + db.exec(`SAVEPOINT ${savepointName}`); + setTransactionDepth(db, depth + 1); + try { + const result = operation(); + assertSyncTransactionResult(result); + db.exec(`RELEASE SAVEPOINT ${savepointName}`); + return result; + } catch (error) { + try { + db.exec(`ROLLBACK TO SAVEPOINT ${savepointName}`); + } finally { + db.exec(`RELEASE SAVEPOINT ${savepointName}`); + } + throw error; + } finally { + setTransactionDepth(db, depth); + } + } + + db.exec("BEGIN IMMEDIATE"); + setTransactionDepth(db, 1); + let transactionStillActive = true; + let result: T; + try { + result = operation(); + assertSyncTransactionResult(result); + } catch (error) { + try { + abortImmediateTransaction(db); + transactionStillActive = false; + } catch { + // Preserve the original error; rollback failure is secondary. + } + throw error; + } finally { + if (!transactionStillActive) { + setTransactionDepth(db, 0); + } + } + + try { + commitImmediateTransaction(db); + transactionStillActive = false; + return result; + } catch (error) { + try { + abortImmediateTransaction(db); + transactionStillActive = false; + } catch { + // Preserve the original error; rollback failure is secondary. + } + throw error; + } finally { + if (!transactionStillActive) { + setTransactionDepth(db, 0); + } + } +} diff --git a/src/infra/sqlite-wal.ts b/src/infra/sqlite-wal.ts index 5f972a2b1ef5..13b5f97cdfc0 100644 --- a/src/infra/sqlite-wal.ts +++ b/src/infra/sqlite-wal.ts @@ -18,6 +18,8 @@ export type SqliteWalMaintenanceOptions = { autoCheckpointPages?: number; checkpointIntervalMs?: number; checkpointMode?: SqliteWalCheckpointMode; + databaseLabel?: string; + databasePath?: string; onCheckpointError?: (error: unknown) => void; }; @@ -41,7 +43,6 @@ export function configureSqliteWalMaintenance( "checkpointIntervalMs", ); const checkpointMode = options.checkpointMode ?? "TRUNCATE"; - db.exec("PRAGMA journal_mode = WAL;"); db.exec(`PRAGMA wal_autocheckpoint = ${autoCheckpointPages};`); diff --git a/src/infra/state-migrations.ts b/src/infra/state-migrations.ts index 30c9b9726425..cc25a47feebc 100644 --- a/src/infra/state-migrations.ts +++ b/src/infra/state-migrations.ts @@ -34,7 +34,15 @@ import { } from "../routing/session-key.js"; import { normalizeSessionKeyPreservingOpaquePeerIds } from "../sessions/session-key-utils.js"; import { normalizeLowercaseStringOrEmpty } from "../shared/string-coerce.js"; +import type { DB as OpenClawStateKyselyDatabase } from "../state/openclaw-state-db.generated.js"; +import { runOpenClawStateWriteTransaction } from "../state/openclaw-state-db.js"; import { expandHomePrefix } from "./home-dir.js"; +import { + executeSqliteQuerySync, + executeSqliteQueryTakeFirstSync, + getNodeSqliteKysely, +} from "./kysely-sync.js"; +import { requireNodeSqlite } from "./node-sqlite.js"; import { isWithinDir } from "./path-safety.js"; import { ensureDir, @@ -69,6 +77,10 @@ export type LegacyStateDetection = { hasLegacy: boolean; plans: ChannelLegacyStateMigrationPlan[]; }; + pluginStateSidecar: { + sourcePath: string; + hasLegacy: boolean; + }; preview: string[]; }; @@ -89,6 +101,25 @@ type LegacySessionSurface = { }) => string | null | undefined; }; +type LegacyPluginStateSidecarRow = { + plugin_id: string; + namespace: string; + entry_key: string; + value_json: string; + created_at: number | bigint; + expires_at: number | bigint | null; +}; + +type LegacyPluginStateImportDatabase = Pick; + +const PLUGIN_STATE_SQLITE_SIDECAR_SUFFIXES = ["", "-shm", "-wal"] as const; + +class LegacyPluginStateSidecarConflictError extends Error { + constructor(readonly conflictedKeys: string[]) { + super("legacy plugin-state sidecar conflicts with shared state"); + } +} + function getLegacySessionSurfaces(): LegacySessionSurface[] { // Legacy migrations run on cold doctor/startup paths. Prefer the narrower // setup plugin surface here so session-key cleanup does not materialize full @@ -125,6 +156,185 @@ function buildLegacyMigrationPreview(plan: ChannelLegacyStateMigrationPlan): str return `- ${plan.label}: ${plan.sourcePath} → ${plan.targetPath}`; } +function resolveLegacyPluginStateSidecarPath(stateDir: string): string { + return path.join(stateDir, "plugin-state", "state.sqlite"); +} + +function readLegacyPluginStateSidecarRows(sourcePath: string): LegacyPluginStateSidecarRow[] { + const sqlite = requireNodeSqlite(); + const db = new sqlite.DatabaseSync(sourcePath, { readOnly: true }); + try { + return db + .prepare( + ` + SELECT plugin_id, namespace, entry_key, value_json, created_at, expires_at + FROM plugin_state_entries + ORDER BY plugin_id ASC, namespace ASC, entry_key ASC + `, + ) + .all() as LegacyPluginStateSidecarRow[]; + } finally { + db.close(); + } +} + +function normalizeLegacySqliteInteger(value: number | bigint | null): number | null { + if (typeof value === "bigint") { + return Number(value); + } + return value; +} + +function legacyPluginStateRowsMatch( + existing: { value_json: string; created_at: number | bigint; expires_at: number | bigint | null }, + legacy: LegacyPluginStateSidecarRow, +): boolean { + return ( + existing.value_json === legacy.value_json && + normalizeLegacySqliteInteger(existing.created_at) === + normalizeLegacySqliteInteger(legacy.created_at) && + normalizeLegacySqliteInteger(existing.expires_at) === + normalizeLegacySqliteInteger(legacy.expires_at) + ); +} + +function archiveLegacyPluginStateSidecar(params: { + sourcePath: string; + changes: string[]; + warnings: string[]; +}): void { + const existingSources = PLUGIN_STATE_SQLITE_SIDECAR_SUFFIXES.map( + (suffix) => `${params.sourcePath}${suffix}`, + ).filter(fileExists); + const existingArchives = existingSources + .map((sourcePath) => `${sourcePath}.migrated`) + .filter(fileExists); + if (existingArchives.length > 0) { + params.warnings.push( + `Left migrated plugin-state sidecar in place because archive already exists: ${existingArchives[0]}`, + ); + return; + } + + for (const sourcePath of existingSources) { + const archivedPath = `${sourcePath}.migrated`; + try { + fs.renameSync(sourcePath, archivedPath); + } catch (err) { + params.warnings.push(`Failed archiving plugin-state sidecar ${sourcePath}: ${String(err)}`); + return; + } + } + params.changes.push( + `Archived plugin-state sidecar legacy source → ${params.sourcePath}.migrated`, + ); +} + +async function migrateLegacyPluginStateSidecar(params: { + stateDir: string; +}): Promise<{ changes: string[]; warnings: string[] }> { + const sourcePath = resolveLegacyPluginStateSidecarPath(params.stateDir); + if (!fileExists(sourcePath)) { + return { changes: [], warnings: [] }; + } + + const changes: string[] = []; + const warnings: string[] = []; + let rows: LegacyPluginStateSidecarRow[]; + try { + rows = readLegacyPluginStateSidecarRows(sourcePath); + } catch (err) { + return { + changes, + warnings: [`Failed reading plugin-state sidecar ${sourcePath}: ${String(err)}`], + }; + } + + try { + const conflictedKeys: string[] = []; + const rowsToInsert: LegacyPluginStateSidecarRow[] = []; + let imported = 0; + const now = Date.now(); + runOpenClawStateWriteTransaction( + ({ db }) => { + const stateDb = getNodeSqliteKysely(db); + for (const row of rows) { + executeSqliteQuerySync( + db, + stateDb + .deleteFrom("plugin_state_entries") + .where("plugin_id", "=", row.plugin_id) + .where("namespace", "=", row.namespace) + .where("entry_key", "=", row.entry_key) + .where("expires_at", "is not", null) + .where("expires_at", "<=", now), + ); + const existing = executeSqliteQueryTakeFirstSync( + db, + stateDb + .selectFrom("plugin_state_entries") + .select(["value_json", "created_at", "expires_at"]) + .where("plugin_id", "=", row.plugin_id) + .where("namespace", "=", row.namespace) + .where("entry_key", "=", row.entry_key), + ); + if (existing) { + if (!legacyPluginStateRowsMatch(existing, row)) { + conflictedKeys.push(`${row.plugin_id}/${row.namespace}/${row.entry_key}`); + } + continue; + } + rowsToInsert.push(row); + } + if (conflictedKeys.length > 0) { + throw new LegacyPluginStateSidecarConflictError(conflictedKeys); + } + for (const row of rowsToInsert) { + executeSqliteQuerySync( + db, + stateDb + .insertInto("plugin_state_entries") + .values({ + plugin_id: row.plugin_id, + namespace: row.namespace, + entry_key: row.entry_key, + value_json: row.value_json, + created_at: normalizeLegacySqliteInteger(row.created_at) ?? 0, + expires_at: normalizeLegacySqliteInteger(row.expires_at), + }) + .onConflict((conflict) => + conflict.columns(["plugin_id", "namespace", "entry_key"]).doNothing(), + ), + ); + imported += 1; + } + }, + { env: { ...process.env, OPENCLAW_STATE_DIR: params.stateDir } }, + ); + if (imported > 0) { + changes.push( + `Migrated ${imported} plugin-state sidecar ${imported === 1 ? "entry" : "entries"} → shared SQLite state`, + ); + } + } catch (err) { + if (err instanceof LegacyPluginStateSidecarConflictError) { + return { + changes, + warnings: [ + `Left plugin-state sidecar in place because ${err.conflictedKeys.length} ${err.conflictedKeys.length === 1 ? "row" : "rows"} already existed in shared state: ${err.conflictedKeys[0]}`, + ], + }; + } + return { + changes, + warnings: [`Failed migrating plugin-state sidecar ${sourcePath}: ${String(err)}`], + }; + } + + archiveLegacyPluginStateSidecar({ sourcePath, changes, warnings }); + return { changes, warnings }; +} + function resolvePluginStateImportTargetKey(scopeKey: string, key: string): string { return scopeKey ? `${scopeKey}:${key}` : key; } @@ -1089,6 +1299,8 @@ export async function detectLegacyStateMigrations(params: { const legacyAgentDir = path.join(stateDir, "agent"); const targetAgentDir = path.join(stateDir, "agents", targetAgentId, "agent"); const hasLegacyAgentDir = existsDir(legacyAgentDir); + const pluginStateSidecarPath = resolveLegacyPluginStateSidecarPath(stateDir); + const hasPluginStateSidecar = fileExists(pluginStateSidecarPath); const channelPlans = await collectChannelLegacyStateMigrationPlans({ cfg: params.cfg, env, @@ -1106,6 +1318,9 @@ export async function detectLegacyStateMigrations(params: { if (hasLegacyAgentDir) { preview.push(`- Agent dir: ${legacyAgentDir} → ${targetAgentDir}`); } + if (hasPluginStateSidecar) { + preview.push(`- Plugin state sidecar: ${pluginStateSidecarPath} → shared SQLite state`); + } if (channelPlans.length > 0) { preview.push(...channelPlans.map(buildLegacyMigrationPreview)); } @@ -1133,6 +1348,10 @@ export async function detectLegacyStateMigrations(params: { hasLegacy: channelPlans.length > 0, plans: channelPlans, }, + pluginStateSidecar: { + sourcePath: pluginStateSidecarPath, + hasLegacy: hasPluginStateSidecar, + }, preview, }; } @@ -1317,6 +1536,9 @@ export async function runLegacyStateMigrations(params: { }): Promise<{ changes: string[]; warnings: string[] }> { const now = params.now ?? (() => Date.now()); const detected = params.detected; + const pluginStateSidecar = await migrateLegacyPluginStateSidecar({ + stateDir: detected.stateDir, + }); const preSessionChannelPlans = await runLegacyMigrationPlans( detected.channelPlans.plans.filter((plan) => plan.kind === "plugin-state-import"), ); @@ -1327,12 +1549,14 @@ export async function runLegacyStateMigrations(params: { ); return { changes: [ + ...pluginStateSidecar.changes, ...preSessionChannelPlans.changes, ...sessions.changes, ...agentDir.changes, ...channelPlans.changes, ], warnings: [ + ...pluginStateSidecar.warnings, ...preSessionChannelPlans.warnings, ...sessions.warnings, ...agentDir.warnings, @@ -1559,24 +1783,49 @@ export async function autoMigrateLegacyState(params: { } }; - if (env.OPENCLAW_AGENT_DIR?.trim() || env.PI_CODING_AGENT_DIR?.trim()) { - const changes = [...stateDirResult.changes, ...orphanKeys.changes]; - const warnings = [...stateDirResult.warnings, ...orphanKeys.warnings]; - logMigrationResults(changes, warnings); - return { - migrated: stateDirResult.migrated || orphanKeys.changes.length > 0, - skipped: true, - changes, - warnings, - }; - } - const detected = await detectLegacyStateMigrations({ cfg: params.cfg, env, homedir: params.homedir, }); - if (!detected.sessions.hasLegacy && !detected.agentDir.hasLegacy) { + const hasCustomAgentDir = env.OPENCLAW_AGENT_DIR?.trim() || env.PI_CODING_AGENT_DIR?.trim(); + if (hasCustomAgentDir) { + const pluginStateSidecar = await migrateLegacyPluginStateSidecar({ + stateDir: detected.stateDir, + }); + const preSessionChannelPlans = await runLegacyMigrationPlans( + detected.channelPlans.plans.filter((plan) => plan.kind === "plugin-state-import"), + ); + const changes = [ + ...stateDirResult.changes, + ...orphanKeys.changes, + ...pluginStateSidecar.changes, + ...preSessionChannelPlans.changes, + ]; + const warnings = [ + ...stateDirResult.warnings, + ...orphanKeys.warnings, + ...pluginStateSidecar.warnings, + ...preSessionChannelPlans.warnings, + ]; + logMigrationResults(changes, warnings); + return { + migrated: + stateDirResult.migrated || + orphanKeys.changes.length > 0 || + pluginStateSidecar.changes.length > 0 || + preSessionChannelPlans.changes.length > 0, + skipped: true, + changes, + warnings, + }; + } + if ( + !detected.sessions.hasLegacy && + !detected.agentDir.hasLegacy && + !detected.channelPlans.hasLegacy && + !detected.pluginStateSidecar.hasLegacy + ) { const changes = [...stateDirResult.changes, ...orphanKeys.changes]; const warnings = [...stateDirResult.warnings, ...orphanKeys.warnings]; logMigrationResults(changes, warnings); @@ -1589,19 +1838,34 @@ export async function autoMigrateLegacyState(params: { } const now = params.now ?? (() => Date.now()); + const pluginStateSidecar = await migrateLegacyPluginStateSidecar({ + stateDir: detected.stateDir, + }); + const preSessionChannelPlans = await runLegacyMigrationPlans( + detected.channelPlans.plans.filter((plan) => plan.kind === "plugin-state-import"), + ); const sessions = await migrateLegacySessions(detected, now); const agentDir = await migrateLegacyAgentDir(detected, now); + const channelPlans = await runLegacyMigrationPlans( + detected.channelPlans.plans.filter((plan) => plan.kind !== "plugin-state-import"), + ); const changes = [ ...stateDirResult.changes, ...orphanKeys.changes, + ...pluginStateSidecar.changes, + ...preSessionChannelPlans.changes, ...sessions.changes, ...agentDir.changes, + ...channelPlans.changes, ]; const warnings = [ ...stateDirResult.warnings, ...orphanKeys.warnings, + ...pluginStateSidecar.warnings, + ...preSessionChannelPlans.warnings, ...sessions.warnings, ...agentDir.warnings, + ...channelPlans.warnings, ]; logMigrationResults(changes, warnings); diff --git a/src/plugin-state/plugin-state-store.e2e.test.ts b/src/plugin-state/plugin-state-store.e2e.test.ts index f4c0927acc38..2aca48e9dd1d 100644 --- a/src/plugin-state/plugin-state-store.e2e.test.ts +++ b/src/plugin-state/plugin-state-store.e2e.test.ts @@ -1,38 +1,19 @@ -import { mkdirSync } from "node:fs"; import { afterEach, describe, expect, it, vi } from "vitest"; -import { requireNodeSqlite } from "../infra/node-sqlite.js"; import { withOpenClawTestState } from "../test-utils/openclaw-test-state.js"; import { - closePluginStateSqliteStore, + closePluginStateDatabase, createPluginStateKeyedStore, PluginStateStoreError, probePluginStateStore, resetPluginStateStoreForTests, sweepExpiredPluginStateEntries, } from "./plugin-state-store.js"; -import { resolvePluginStateDir, resolvePluginStateSqlitePath } from "./plugin-state-store.paths.js"; -import { MAX_PLUGIN_STATE_ENTRIES_PER_PLUGIN } from "./plugin-state-store.sqlite.js"; -import { seedPluginStateEntriesForTests } from "./plugin-state-store.test-helpers.js"; afterEach(() => { vi.useRealTimers(); resetPluginStateStoreForTests(); }); -async function expectPluginStateStoreError( - promise: Promise, - expected: { code: string }, -): Promise { - let storeError: unknown; - try { - await promise; - } catch (error) { - storeError = error; - } - expect(storeError).toBeInstanceOf(PluginStateStoreError); - expect((storeError as PluginStateStoreError | undefined)?.code).toBe(expected.code); -} - // --------------------------------------------------------------------------- // Runtime smoke // --------------------------------------------------------------------------- @@ -188,36 +169,7 @@ describe("limits", () => { }); // 65 535 chars → 65 537 bytes of JSON → over limit. const oversize = "x".repeat(65_535); - await expectPluginStateStoreError(store.register("big", oversize), { - code: "PLUGIN_STATE_LIMIT_EXCEEDED", - }); - }); - }); - - it("enforces the per-plugin live-row cap", async () => { - await withOpenClawTestState({ label: "e2e-limit-plugin" }, async () => { - // Fill the plugin budget outside the namespace that attempts the write. - const nsCount = 10; - const perNs = MAX_PLUGIN_STATE_ENTRIES_PER_PLUGIN / nsCount; - seedPluginStateEntriesForTests( - Array.from({ length: MAX_PLUGIN_STATE_ENTRIES_PER_PLUGIN }, (_, index) => { - const ns = Math.floor(index / perNs); - const k = index % perNs; - return { - pluginId: "fixture-plugin", - namespace: `ns-${ns}`, - key: `k-${k}`, - value: { ns, k }, - }; - }), - ); - const store = createPluginStateKeyedStore("fixture-plugin", { - namespace: "overflow-ns", - maxEntries: 10, - }); - - // One more row tips over the plugin-wide limit. - await expectPluginStateStoreError(store.register("overflow", { boom: true }), { + await expect(store.register("big", oversize)).rejects.toMatchObject({ code: "PLUGIN_STATE_LIMIT_EXCEEDED", }); }); @@ -252,33 +204,14 @@ describe("limits", () => { // Failure safety // --------------------------------------------------------------------------- describe("failure safety", () => { - it("gives a typed error for unsupported schema versions", async () => { - await withOpenClawTestState({ label: "e2e-fail-schema" }, async () => { - // Pre-seed the DB with a future schema version. - mkdirSync(resolvePluginStateDir(), { recursive: true }); - const { DatabaseSync } = requireNodeSqlite(); - const db = new DatabaseSync(resolvePluginStateSqlitePath()); - db.exec("PRAGMA user_version = 99;"); - db.close(); - - const store = createPluginStateKeyedStore("fixture-plugin", { - namespace: "schema", - maxEntries: 10, - }); - const error = await store.register("k", { ok: true }).catch((e: unknown) => e); - expect(error).toBeInstanceOf(PluginStateStoreError); - expect((error as PluginStateStoreError).code).toBe("PLUGIN_STATE_SCHEMA_UNSUPPORTED"); - }); - }); - it("probe returns redacted diagnostics without leaking stored values", async () => { await withOpenClawTestState({ label: "e2e-fail-probe" }, async () => { const result = probePluginStateStore(); expect(result.ok).toBe(true); - expect(result.dbPath).toContain("state.sqlite"); + expect(result.databasePath).toContain("openclaw.sqlite"); expect(result.steps.length).toBeGreaterThanOrEqual(4); const failedSteps = result.steps.filter((step) => !step.ok); - expect(failedSteps).toStrictEqual([]); + expect(failedSteps).toEqual([]); // The probe's temporary stored value must not leak into the result. const serialised = JSON.stringify(result); @@ -295,11 +228,11 @@ describe("failure safety", () => { await store.register("k", { v: 1 }); // First close. - closePluginStateSqliteStore(); + closePluginStateDatabase(); await expect(store.lookup("k")).resolves.toEqual({ v: 1 }); // Second close (idempotent). - closePluginStateSqliteStore(); + closePluginStateDatabase(); await expect(store.lookup("k")).resolves.toEqual({ v: 1 }); // Write after reopen. diff --git a/src/plugin-state/plugin-state-store.paths.ts b/src/plugin-state/plugin-state-store.paths.ts deleted file mode 100644 index e84083635d3e..000000000000 --- a/src/plugin-state/plugin-state-store.paths.ts +++ /dev/null @@ -1,10 +0,0 @@ -import path from "node:path"; -import { resolveStateDir } from "../config/paths.js"; - -export function resolvePluginStateDir(env: NodeJS.ProcessEnv = process.env): string { - return path.join(resolveStateDir(env), "plugin-state"); -} - -export function resolvePluginStateSqlitePath(env: NodeJS.ProcessEnv = process.env): string { - return path.join(resolvePluginStateDir(env), "state.sqlite"); -} diff --git a/src/plugin-state/plugin-state-store.permissions.test.ts b/src/plugin-state/plugin-state-store.permissions.test.ts deleted file mode 100644 index e0818b934523..000000000000 --- a/src/plugin-state/plugin-state-store.permissions.test.ts +++ /dev/null @@ -1,56 +0,0 @@ -import { afterEach, describe, expect, it, vi } from "vitest"; -import { withOpenClawTestState } from "../test-utils/openclaw-test-state.js"; - -afterEach(() => { - vi.doUnmock("node:fs"); - vi.resetModules(); -}); - -describe("plugin state permission hardening", () => { - it("does not reject a committed write when post-commit chmod fails", async () => { - let chmodCalls = 0; - let throwAfter = Number.POSITIVE_INFINITY; - - vi.doMock("node:fs", async (importOriginal) => { - const actual = await importOriginal(); - return { - ...actual, - chmodSync: (target: Parameters[0], mode: number) => { - chmodCalls += 1; - if (chmodCalls > throwAfter) { - throw Object.assign(new Error("chmod denied"), { code: "EACCES" }); - } - return actual.chmodSync(target, mode); - }, - existsSync: (target: Parameters[0]) => { - const pathname = String(target); - if (pathname.endsWith("-shm") || pathname.endsWith("-wal")) { - return false; - } - return actual.existsSync(target); - }, - }; - }); - - const { createPluginStateKeyedStore, resetPluginStateStoreForTests } = - await import("./plugin-state-store.js"); - - try { - await withOpenClawTestState({ label: "plugin-state-post-commit-chmod" }, async () => { - const store = createPluginStateKeyedStore<{ value: number }>("fixture-plugin", { - namespace: "post-commit", - maxEntries: 10, - }); - await store.register("first", { value: 1 }); - - chmodCalls = 0; - throwAfter = 2; - - await expect(store.register("second", { value: 2 })).resolves.toBeUndefined(); - await expect(store.lookup("second")).resolves.toEqual({ value: 2 }); - }); - } finally { - resetPluginStateStoreForTests(); - } - }); -}); diff --git a/src/plugin-state/plugin-state-store.sqlite.ts b/src/plugin-state/plugin-state-store.sqlite.ts index cdd7cc492e2e..eaa7f80ec823 100644 --- a/src/plugin-state/plugin-state-store.sqlite.ts +++ b/src/plugin-state/plugin-state-store.sqlite.ts @@ -1,8 +1,20 @@ -import { chmodSync, existsSync, mkdirSync } from "node:fs"; -import type { DatabaseSync, StatementSync } from "node:sqlite"; +import type { DatabaseSync } from "node:sqlite"; +import type { Insertable, Selectable } from "kysely"; +import { + executeSqliteQuerySync, + executeSqliteQueryTakeFirstSync, + getNodeSqliteKysely, +} from "../infra/kysely-sync.js"; import { requireNodeSqlite } from "../infra/node-sqlite.js"; -import { configureSqliteWalMaintenance, type SqliteWalMaintenance } from "../infra/sqlite-wal.js"; -import { resolvePluginStateDir, resolvePluginStateSqlitePath } from "./plugin-state-store.paths.js"; +import type { DB as OpenClawStateKyselyDatabase } from "../state/openclaw-state-db.generated.js"; +import { + closeOpenClawStateDatabase, + isOpenClawStateDatabaseOpen, + openOpenClawStateDatabase, + type OpenClawStateDatabaseOptions, + runOpenClawStateWriteTransaction, +} from "../state/openclaw-state-db.js"; +import { resolveOpenClawStateSqlitePath } from "../state/openclaw-state-db.paths.js"; import { PluginStateStoreError, type PluginStateEntry, @@ -12,52 +24,22 @@ import { type PluginStateStoreProbeStep, } from "./plugin-state-store.types.js"; -const PLUGIN_STATE_SCHEMA_VERSION = 1; -const PLUGIN_STATE_DIR_MODE = 0o700; -const PLUGIN_STATE_FILE_MODE = 0o600; -const PLUGIN_STATE_SIDECAR_SUFFIXES = ["", "-shm", "-wal"] as const; // Plugin-wide fuse only; namespace maxEntries still owns normal cache eviction. -const MAX_ENTRIES_PER_PLUGIN = 50_000; - export const MAX_PLUGIN_STATE_VALUE_BYTES = 65_536; -export const MAX_PLUGIN_STATE_ENTRIES_PER_PLUGIN = MAX_ENTRIES_PER_PLUGIN; +export const MAX_PLUGIN_STATE_ENTRIES_PER_PLUGIN = 50_000; -type PluginStateRow = { - plugin_id: string; - namespace: string; - entry_key: string; - value_json: string; - created_at: number | bigint; - expires_at: number | bigint | null; -}; +type PluginStateEntriesTable = OpenClawStateKyselyDatabase["plugin_state_entries"]; +type PluginStateStoreDatabase = Pick; + +type PluginStateRow = Selectable; type CountRow = { count: number | bigint; }; -type UserVersionRow = { - user_version?: number | bigint; -}; - -type PluginStateStatements = { - upsertEntry: StatementSync; - insertEntryIfAbsent: StatementSync; - selectEntry: StatementSync; - selectEntries: StatementSync; - deleteEntry: StatementSync; - clearNamespace: StatementSync; - pruneExpiredNamespace: StatementSync; - countLiveNamespace: StatementSync; - countLivePlugin: StatementSync; - deleteOldestNamespace: StatementSync; - sweepExpired: StatementSync; -}; - type PluginStateDatabase = { db: DatabaseSync; path: string; - statements: PluginStateStatements; - walMaintenance: SqliteWalMaintenance; }; type PluginStateSeedEntryForTests = { @@ -98,7 +80,7 @@ function wrapPluginStateError( operation: PluginStateStoreOperation, fallbackCode: PluginStateStoreErrorCode, message: string, - pathname: string = resolvePluginStateSqlitePath(process.env), + pathname = resolveOpenClawStateSqlitePath(process.env), ): PluginStateStoreError { if (error instanceof PluginStateStoreError) { return error; @@ -120,7 +102,7 @@ function parseStoredJson(raw: string, operation: PluginStateStoreOperation): unk code: "PLUGIN_STATE_CORRUPT", operation, message: "Plugin state entry contains corrupt JSON.", - path: resolvePluginStateSqlitePath(process.env), + path: resolveOpenClawStateSqlitePath(process.env), cause: error, }); } @@ -139,223 +121,204 @@ function rowToEntry( }; } -function getUserVersion(db: DatabaseSync): number { - const row = db.prepare("PRAGMA user_version").get() as UserVersionRow | undefined; - const raw = row?.user_version ?? 0; - return typeof raw === "bigint" ? Number(raw) : raw; +function getPluginStateKysely(db: DatabaseSync) { + return getNodeSqliteKysely(db); } -function ensureSchema(db: DatabaseSync, pathname: string) { - const userVersion = getUserVersion(db); - if (userVersion > PLUGIN_STATE_SCHEMA_VERSION) { - throw createPluginStateError({ - code: "PLUGIN_STATE_SCHEMA_UNSUPPORTED", - operation: "ensure-schema", - message: `Plugin state database schema version ${userVersion} is newer than supported version ${PLUGIN_STATE_SCHEMA_VERSION}.`, - path: pathname, - }); - } - - db.exec(` - CREATE TABLE IF NOT EXISTS plugin_state_entries ( - plugin_id TEXT NOT NULL, - namespace TEXT NOT NULL, - entry_key TEXT NOT NULL, - value_json TEXT NOT NULL, - created_at INTEGER NOT NULL, - expires_at INTEGER, - PRIMARY KEY (plugin_id, namespace, entry_key) - ); - - CREATE INDEX IF NOT EXISTS idx_plugin_state_expiry - ON plugin_state_entries(expires_at) - WHERE expires_at IS NOT NULL; - - CREATE INDEX IF NOT EXISTS idx_plugin_state_listing - ON plugin_state_entries(plugin_id, namespace, created_at, entry_key); - - PRAGMA user_version = ${PLUGIN_STATE_SCHEMA_VERSION}; - `); -} - -function createStatements(db: DatabaseSync): PluginStateStatements { +function bindPluginStateEntry(params: { + pluginId: string; + namespace: string; + key: string; + valueJson: string; + createdAt: number; + expiresAt: number | null; +}): Insertable { return { - upsertEntry: db.prepare(` - INSERT INTO plugin_state_entries ( - plugin_id, - namespace, - entry_key, - value_json, - created_at, - expires_at - ) VALUES ( - @plugin_id, - @namespace, - @entry_key, - @value_json, - @created_at, - @expires_at - ) - ON CONFLICT(plugin_id, namespace, entry_key) DO UPDATE SET - value_json = excluded.value_json, - created_at = excluded.created_at, - expires_at = excluded.expires_at - `), - insertEntryIfAbsent: db.prepare(` - INSERT OR IGNORE INTO plugin_state_entries ( - plugin_id, - namespace, - entry_key, - value_json, - created_at, - expires_at - ) VALUES ( - @plugin_id, - @namespace, - @entry_key, - @value_json, - @created_at, - @expires_at - ) - `), - selectEntry: db.prepare(` - SELECT plugin_id, namespace, entry_key, value_json, created_at, expires_at - FROM plugin_state_entries - WHERE plugin_id = ? - AND namespace = ? - AND entry_key = ? - AND (expires_at IS NULL OR expires_at > ?) - `), - selectEntries: db.prepare(` - SELECT plugin_id, namespace, entry_key, value_json, created_at, expires_at - FROM plugin_state_entries - WHERE plugin_id = ? - AND namespace = ? - AND (expires_at IS NULL OR expires_at > ?) - ORDER BY created_at ASC, entry_key ASC - `), - deleteEntry: db.prepare(` - DELETE FROM plugin_state_entries - WHERE plugin_id = ? AND namespace = ? AND entry_key = ? - `), - clearNamespace: db.prepare(` - DELETE FROM plugin_state_entries - WHERE plugin_id = ? AND namespace = ? - `), - pruneExpiredNamespace: db.prepare(` - DELETE FROM plugin_state_entries - WHERE plugin_id = ? - AND namespace = ? - AND expires_at IS NOT NULL - AND expires_at <= ? - `), - countLiveNamespace: db.prepare(` - SELECT COUNT(*) AS count - FROM plugin_state_entries - WHERE plugin_id = ? - AND namespace = ? - AND (expires_at IS NULL OR expires_at > ?) - `), - countLivePlugin: db.prepare(` - SELECT COUNT(*) AS count - FROM plugin_state_entries - WHERE plugin_id = ? - AND (expires_at IS NULL OR expires_at > ?) - `), - deleteOldestNamespace: db.prepare(` - DELETE FROM plugin_state_entries - WHERE rowid IN ( - SELECT rowid - FROM plugin_state_entries - WHERE plugin_id = ? - AND namespace = ? - AND entry_key <> ? - AND (expires_at IS NULL OR expires_at > ?) - ORDER BY created_at ASC, entry_key ASC - LIMIT ? - ) - `), - sweepExpired: db.prepare(` - DELETE FROM plugin_state_entries - WHERE expires_at IS NOT NULL AND expires_at <= ? - `), + plugin_id: params.pluginId, + namespace: params.namespace, + entry_key: params.key, + value_json: params.valueJson, + created_at: params.createdAt, + expires_at: params.expiresAt, }; } -function ensurePluginStatePermissions(pathname: string, env: NodeJS.ProcessEnv = process.env) { - const dir = resolvePluginStateDir(env); - mkdirSync(dir, { recursive: true, mode: PLUGIN_STATE_DIR_MODE }); - chmodSync(dir, PLUGIN_STATE_DIR_MODE); - for (const suffix of PLUGIN_STATE_SIDECAR_SUFFIXES) { - const candidate = `${pathname}${suffix}`; - if (existsSync(candidate)) { - chmodSync(candidate, PLUGIN_STATE_FILE_MODE); - } +function upsertPluginStateEntry(db: DatabaseSync, row: Insertable): void { + executeSqliteQuerySync( + db, + getPluginStateKysely(db) + .insertInto("plugin_state_entries") + .values(row) + .onConflict((conflict) => + conflict.columns(["plugin_id", "namespace", "entry_key"]).doUpdateSet({ + value_json: (eb) => eb.ref("excluded.value_json"), + created_at: (eb) => eb.ref("excluded.created_at"), + expires_at: (eb) => eb.ref("excluded.expires_at"), + }), + ), + ); +} + +function insertPluginStateEntryIfAbsent( + db: DatabaseSync, + row: Insertable, +): boolean { + const result = executeSqliteQuerySync( + db, + getPluginStateKysely(db).insertInto("plugin_state_entries").orIgnore().values(row), + ); + return Number(result.numAffectedRows ?? 0) > 0; +} + +function selectPluginStateEntry( + db: DatabaseSync, + params: { pluginId: string; namespace: string; key: string; now: number }, +): PluginStateRow | undefined { + return executeSqliteQueryTakeFirstSync( + db, + getPluginStateKysely(db) + .selectFrom("plugin_state_entries") + .select(["plugin_id", "namespace", "entry_key", "value_json", "created_at", "expires_at"]) + .where("plugin_id", "=", params.pluginId) + .where("namespace", "=", params.namespace) + .where("entry_key", "=", params.key) + .where((eb) => eb.or([eb("expires_at", "is", null), eb("expires_at", ">", params.now)])), + ); +} + +function selectPluginStateEntries( + db: DatabaseSync, + params: { pluginId: string; namespace: string; now: number }, +): PluginStateRow[] { + return executeSqliteQuerySync( + db, + getPluginStateKysely(db) + .selectFrom("plugin_state_entries") + .select(["plugin_id", "namespace", "entry_key", "value_json", "created_at", "expires_at"]) + .where("plugin_id", "=", params.pluginId) + .where("namespace", "=", params.namespace) + .where((eb) => eb.or([eb("expires_at", "is", null), eb("expires_at", ">", params.now)])) + .orderBy("created_at", "asc") + .orderBy("entry_key", "asc"), + ).rows; +} + +function deletePluginStateEntry( + db: DatabaseSync, + params: { pluginId: string; namespace: string; key: string }, +): number { + const result = executeSqliteQuerySync( + db, + getPluginStateKysely(db) + .deleteFrom("plugin_state_entries") + .where("plugin_id", "=", params.pluginId) + .where("namespace", "=", params.namespace) + .where("entry_key", "=", params.key), + ); + return Number(result.numAffectedRows ?? 0); +} + +function deleteExpiredPluginStateNamespaceEntries( + db: DatabaseSync, + params: { pluginId: string; namespace: string; now: number }, +): void { + executeSqliteQuerySync( + db, + getPluginStateKysely(db) + .deleteFrom("plugin_state_entries") + .where("plugin_id", "=", params.pluginId) + .where("namespace", "=", params.namespace) + .where("expires_at", "is not", null) + .where("expires_at", "<=", params.now), + ); +} + +function countLivePluginStateNamespaceEntries( + db: DatabaseSync, + params: { pluginId: string; namespace: string; now: number }, +): number { + const row = executeSqliteQueryTakeFirstSync( + db, + getPluginStateKysely(db) + .selectFrom("plugin_state_entries") + .select((eb) => eb.fn.countAll().as("count")) + .where("plugin_id", "=", params.pluginId) + .where("namespace", "=", params.namespace) + .where((eb) => eb.or([eb("expires_at", "is", null), eb("expires_at", ">", params.now)])), + ); + return countRow(row); +} + +function countLivePluginStateEntries( + db: DatabaseSync, + params: { pluginId: string; now: number }, +): number { + const row = executeSqliteQueryTakeFirstSync( + db, + getPluginStateKysely(db) + .selectFrom("plugin_state_entries") + .select((eb) => eb.fn.countAll().as("count")) + .where("plugin_id", "=", params.pluginId) + .where((eb) => eb.or([eb("expires_at", "is", null), eb("expires_at", ">", params.now)])), + ); + return countRow(row); +} + +function deleteOldestPluginStateNamespaceEntries( + db: DatabaseSync, + params: { pluginId: string; namespace: string; protectedKey: string; now: number; limit: number }, +): void { + const keys = executeSqliteQuerySync( + db, + getPluginStateKysely(db) + .selectFrom("plugin_state_entries") + .select(["entry_key"]) + .where("plugin_id", "=", params.pluginId) + .where("namespace", "=", params.namespace) + .where("entry_key", "!=", params.protectedKey) + .where((eb) => eb.or([eb("expires_at", "is", null), eb("expires_at", ">", params.now)])) + .orderBy("created_at", "asc") + .orderBy("entry_key", "asc") + .limit(params.limit), + ).rows; + for (const row of keys) { + deletePluginStateEntry(db, { + pluginId: params.pluginId, + namespace: params.namespace, + key: row.entry_key, + }); } } -function ensurePluginStatePermissionsBestEffort( - pathname: string, - env: NodeJS.ProcessEnv = process.env, -): void { - try { - ensurePluginStatePermissions(pathname, env); - } catch { - // The write already committed. Permission hardening is best-effort from here. - } +function sweepExpiredPluginStateEntriesFromDatabase(db: DatabaseSync, now: number): number { + const result = executeSqliteQuerySync( + db, + getPluginStateKysely(db) + .deleteFrom("plugin_state_entries") + .where("expires_at", "is not", null) + .where("expires_at", "<=", now), + ); + return Number(result.numAffectedRows ?? 0); } function openPluginStateDatabase( operation: PluginStateStoreOperation = "open", - env: NodeJS.ProcessEnv = process.env, + options: OpenClawStateDatabaseOptions = {}, ): PluginStateDatabase { - const pathname = resolvePluginStateSqlitePath(env); - if (cachedDatabase && cachedDatabase.path === pathname) { + const env = options.env ?? process.env; + const pathname = resolveOpenClawStateSqlitePath(env); + if (cachedDatabase && cachedDatabase.path === pathname && cachedDatabase.db.isOpen) { return cachedDatabase; } - if (cachedDatabase) { - cachedDatabase.walMaintenance.close(); - cachedDatabase.db.close(); + if (cachedDatabase && !cachedDatabase.db.isOpen) { cachedDatabase = null; } try { - ensurePluginStatePermissions(pathname, env); - } catch (error) { - throw createPluginStateError({ - code: "PLUGIN_STATE_OPEN_FAILED", - operation, - message: "Failed to prepare the plugin state database directory.", - path: pathname, - cause: error, - }); - } - - let sqlite: typeof import("node:sqlite"); - try { - sqlite = requireNodeSqlite(); - } catch (error) { - throw createPluginStateError({ - code: "PLUGIN_STATE_SQLITE_UNAVAILABLE", - operation: "load-sqlite", - message: "SQLite support is unavailable for plugin state storage.", - path: pathname, - cause: error, - }); - } - - try { - const db = new sqlite.DatabaseSync(pathname); - const walMaintenance = configureSqliteWalMaintenance(db); - db.exec("PRAGMA synchronous = NORMAL;"); - db.exec("PRAGMA busy_timeout = 5000;"); - ensureSchema(db, pathname); - ensurePluginStatePermissions(pathname, env); + const database = openOpenClawStateDatabase(options); cachedDatabase = { - db, - path: pathname, - statements: createStatements(db), - walMaintenance, + db: database.db, + path: database.path, }; return cachedDatabase; } catch (error) { @@ -374,27 +337,20 @@ function countRow(row: CountRow | undefined): number { return typeof raw === "bigint" ? Number(raw) : raw; } +function envOptions(env?: NodeJS.ProcessEnv): OpenClawStateDatabaseOptions { + return env ? { env } : {}; +} + function runWriteTransaction( operation: PluginStateStoreOperation, write: (store: PluginStateDatabase) => T, - env: NodeJS.ProcessEnv = process.env, + options: OpenClawStateDatabaseOptions = {}, ): T { - const store = openPluginStateDatabase(operation, env); - ensurePluginStatePermissions(store.path, env); - store.db.exec("BEGIN IMMEDIATE"); - try { + const store = openPluginStateDatabase(operation, options); + return runOpenClawStateWriteTransaction(() => { const result = write(store); - store.db.exec("COMMIT"); - ensurePluginStatePermissionsBestEffort(store.path, env); return result; - } catch (error) { - try { - store.db.exec("ROLLBACK"); - } catch { - // Preserve the original failure; rollback errors are secondary here. - } - throw error; - } + }, options); } function enforcePostRegisterLimits(params: { @@ -405,50 +361,46 @@ function enforcePostRegisterLimits(params: { now: number; protectedKey: string; }): void { - const namespaceCount = countRow( - params.store.statements.countLiveNamespace.get( - params.pluginId, - params.namespace, - params.now, - ) as CountRow | undefined, - ); + const namespaceCount = countLivePluginStateNamespaceEntries(params.store.db, { + pluginId: params.pluginId, + namespace: params.namespace, + now: params.now, + }); if (namespaceCount > params.maxEntries) { - params.store.statements.deleteOldestNamespace.run( - params.pluginId, - params.namespace, - params.protectedKey, - params.now, - namespaceCount - params.maxEntries, - ); + deleteOldestPluginStateNamespaceEntries(params.store.db, { + pluginId: params.pluginId, + namespace: params.namespace, + protectedKey: params.protectedKey, + now: params.now, + limit: namespaceCount - params.maxEntries, + }); } - const pluginCount = countRow( - params.store.statements.countLivePlugin.get(params.pluginId, params.now) as - | CountRow - | undefined, - ); - if (pluginCount <= MAX_ENTRIES_PER_PLUGIN) { + const pluginCount = countLivePluginStateEntries(params.store.db, { + pluginId: params.pluginId, + now: params.now, + }); + if (pluginCount <= MAX_PLUGIN_STATE_ENTRIES_PER_PLUGIN) { return; } // Shed rows from the namespace that grew before failing the plugin write. - params.store.statements.deleteOldestNamespace.run( - params.pluginId, - params.namespace, - params.protectedKey, - params.now, - pluginCount - MAX_ENTRIES_PER_PLUGIN, - ); - const remainingPluginCount = countRow( - params.store.statements.countLivePlugin.get(params.pluginId, params.now) as - | CountRow - | undefined, - ); - if (remainingPluginCount > MAX_ENTRIES_PER_PLUGIN) { + deleteOldestPluginStateNamespaceEntries(params.store.db, { + pluginId: params.pluginId, + namespace: params.namespace, + protectedKey: params.protectedKey, + now: params.now, + limit: pluginCount - MAX_PLUGIN_STATE_ENTRIES_PER_PLUGIN, + }); + const remainingPluginCount = countLivePluginStateEntries(params.store.db, { + pluginId: params.pluginId, + now: params.now, + }); + if (remainingPluginCount > MAX_PLUGIN_STATE_ENTRIES_PER_PLUGIN) { throw createPluginStateError({ code: "PLUGIN_STATE_LIMIT_EXCEEDED", operation: "register", - message: `Plugin state for ${params.pluginId} exceeds the ${MAX_ENTRIES_PER_PLUGIN} live row limit.`, + message: `Plugin state for ${params.pluginId} exceeds the ${MAX_PLUGIN_STATE_ENTRIES_PER_PLUGIN} live row limit.`, path: params.store.path, }); } @@ -469,15 +421,22 @@ export function pluginStateRegister(params: { (store) => { const now = Date.now(); const expiresAt = params.ttlMs == null ? null : now + params.ttlMs; - store.statements.pruneExpiredNamespace.run(params.pluginId, params.namespace, now); - store.statements.upsertEntry.run({ - plugin_id: params.pluginId, + deleteExpiredPluginStateNamespaceEntries(store.db, { + pluginId: params.pluginId, namespace: params.namespace, - entry_key: params.key, - value_json: params.valueJson, - created_at: now, - expires_at: expiresAt, + now, }); + upsertPluginStateEntry( + store.db, + bindPluginStateEntry({ + pluginId: params.pluginId, + namespace: params.namespace, + key: params.key, + valueJson: params.valueJson, + createdAt: now, + expiresAt, + }), + ); enforcePostRegisterLimits({ store, pluginId: params.pluginId, @@ -487,7 +446,7 @@ export function pluginStateRegister(params: { protectedKey: params.key, }); }, - params.env, + envOptions(params.env), ); } catch (error) { throw wrapPluginStateError( @@ -495,7 +454,6 @@ export function pluginStateRegister(params: { "register", "PLUGIN_STATE_WRITE_FAILED", "Failed to register plugin state entry.", - resolvePluginStateSqlitePath(params.env), ); } } @@ -515,16 +473,23 @@ export function pluginStateRegisterIfAbsent(params: { (store) => { const now = Date.now(); const expiresAt = params.ttlMs == null ? null : now + params.ttlMs; - store.statements.pruneExpiredNamespace.run(params.pluginId, params.namespace, now); - const result = store.statements.insertEntryIfAbsent.run({ - plugin_id: params.pluginId, + deleteExpiredPluginStateNamespaceEntries(store.db, { + pluginId: params.pluginId, namespace: params.namespace, - entry_key: params.key, - value_json: params.valueJson, - created_at: now, - expires_at: expiresAt, + now, }); - if (result.changes === 0) { + const inserted = insertPluginStateEntryIfAbsent( + store.db, + bindPluginStateEntry({ + pluginId: params.pluginId, + namespace: params.namespace, + key: params.key, + valueJson: params.valueJson, + createdAt: now, + expiresAt, + }), + ); + if (!inserted) { return false; } enforcePostRegisterLimits({ @@ -537,7 +502,7 @@ export function pluginStateRegisterIfAbsent(params: { }); return true; }, - params.env, + envOptions(params.env), ); } catch (error) { throw wrapPluginStateError( @@ -545,7 +510,6 @@ export function pluginStateRegisterIfAbsent(params: { "register", "PLUGIN_STATE_WRITE_FAILED", "Failed to register plugin state entry.", - resolvePluginStateSqlitePath(params.env), ); } } @@ -557,13 +521,13 @@ export function pluginStateLookup(params: { env?: NodeJS.ProcessEnv; }): unknown { try { - const { statements } = openPluginStateDatabase("lookup", params.env); - const row = statements.selectEntry.get( - params.pluginId, - params.namespace, - params.key, - Date.now(), - ) as PluginStateRow | undefined; + const { db } = openPluginStateDatabase("lookup", envOptions(params.env)); + const row = selectPluginStateEntry(db, { + pluginId: params.pluginId, + namespace: params.namespace, + key: params.key, + now: Date.now(), + }); return row ? parseStoredJson(row.value_json, "lookup") : undefined; } catch (error) { throw wrapPluginStateError( @@ -571,7 +535,6 @@ export function pluginStateLookup(params: { "lookup", "PLUGIN_STATE_READ_FAILED", "Failed to read plugin state entry.", - resolvePluginStateSqlitePath(params.env), ); } } @@ -586,19 +549,19 @@ export function pluginStateConsume(params: { return runWriteTransaction( "consume", (store) => { - const row = store.statements.selectEntry.get( - params.pluginId, - params.namespace, - params.key, - Date.now(), - ) as PluginStateRow | undefined; + const row = selectPluginStateEntry(store.db, { + pluginId: params.pluginId, + namespace: params.namespace, + key: params.key, + now: Date.now(), + }); if (!row) { return undefined; } - store.statements.deleteEntry.run(params.pluginId, params.namespace, params.key); + deletePluginStateEntry(store.db, params); return parseStoredJson(row.value_json, "consume"); }, - params.env, + envOptions(params.env), ); } catch (error) { throw wrapPluginStateError( @@ -606,7 +569,6 @@ export function pluginStateConsume(params: { "consume", "PLUGIN_STATE_READ_FAILED", "Failed to consume plugin state entry.", - resolvePluginStateSqlitePath(params.env), ); } } @@ -618,16 +580,19 @@ export function pluginStateDelete(params: { env?: NodeJS.ProcessEnv; }): boolean { try { - const { statements } = openPluginStateDatabase("delete", params.env); - const result = statements.deleteEntry.run(params.pluginId, params.namespace, params.key); - return result.changes > 0; + return runWriteTransaction( + "delete", + ({ db }) => { + return deletePluginStateEntry(db, params) > 0; + }, + envOptions(params.env), + ); } catch (error) { throw wrapPluginStateError( error, "delete", "PLUGIN_STATE_WRITE_FAILED", "Failed to delete plugin state entry.", - resolvePluginStateSqlitePath(params.env), ); } } @@ -638,12 +603,12 @@ export function pluginStateEntries(params: { env?: NodeJS.ProcessEnv; }): PluginStateEntry[] { try { - const { statements } = openPluginStateDatabase("entries", params.env); - const rows = statements.selectEntries.all( - params.pluginId, - params.namespace, - Date.now(), - ) as PluginStateRow[]; + const { db } = openPluginStateDatabase("entries", envOptions(params.env)); + const rows = selectPluginStateEntries(db, { + pluginId: params.pluginId, + namespace: params.namespace, + now: Date.now(), + }); return rows.map((row) => rowToEntry(row, "entries")); } catch (error) { throw wrapPluginStateError( @@ -651,21 +616,6 @@ export function pluginStateEntries(params: { "entries", "PLUGIN_STATE_READ_FAILED", "Failed to list plugin state entries.", - resolvePluginStateSqlitePath(params.env), - ); - } -} - -export function countPluginStateLiveEntries(pluginId: string): number { - try { - const { statements } = openPluginStateDatabase("entries"); - return countRow(statements.countLivePlugin.get(pluginId, Date.now()) as CountRow | undefined); - } catch (error) { - throw wrapPluginStateError( - error, - "entries", - "PLUGIN_STATE_READ_FAILED", - "Failed to count plugin state entries.", ); } } @@ -676,24 +626,34 @@ export function pluginStateClear(params: { env?: NodeJS.ProcessEnv; }): void { try { - const { statements } = openPluginStateDatabase("clear", params.env); - statements.clearNamespace.run(params.pluginId, params.namespace); + runWriteTransaction( + "clear", + ({ db }) => { + executeSqliteQuerySync( + db, + getPluginStateKysely(db) + .deleteFrom("plugin_state_entries") + .where("plugin_id", "=", params.pluginId) + .where("namespace", "=", params.namespace), + ); + }, + envOptions(params.env), + ); } catch (error) { throw wrapPluginStateError( error, "clear", "PLUGIN_STATE_WRITE_FAILED", "Failed to clear plugin state namespace.", - resolvePluginStateSqlitePath(params.env), ); } } export function sweepExpiredPluginStateEntries(): number { try { - const { statements } = openPluginStateDatabase("sweep"); - const result = statements.sweepExpired.run(Date.now()); - return Number(result.changes); + return runWriteTransaction("sweep", ({ db }) => + sweepExpiredPluginStateEntriesFromDatabase(db, Date.now()), + ); } catch (error) { throw wrapPluginStateError( error, @@ -705,15 +665,32 @@ export function sweepExpiredPluginStateEntries(): number { } export function isPluginStateDatabaseOpen(): boolean { - return cachedDatabase !== null; + return cachedDatabase?.db.isOpen === true; } -export function clearPluginStateSqliteStoreForTests(): void { +export function clearPluginStateDatabaseForTests(): void { const store = openPluginStateDatabase("clear"); - store.db.exec("DELETE FROM plugin_state_entries;"); + executeSqliteQuerySync( + store.db, + getPluginStateKysely(store.db).deleteFrom("plugin_state_entries"), + ); } -export function seedPluginStateSqliteEntriesForTests( +export function countPluginStateLiveEntries(pluginId: string): number { + try { + const { db } = openPluginStateDatabase("entries"); + return countLivePluginStateEntries(db, { pluginId, now: Date.now() }); + } catch (error) { + throw wrapPluginStateError( + error, + "entries", + "PLUGIN_STATE_READ_FAILED", + "Failed to count plugin state entries.", + ); + } +} + +export function seedPluginStateDatabaseEntriesForTests( entries: readonly PluginStateSeedEntryForTests[], ): void { if (entries.length === 0) { @@ -724,22 +701,26 @@ export function seedPluginStateSqliteEntriesForTests( runWriteTransaction("register", (store) => { for (let index = 0; index < entries.length; index += 1) { const entry = entries[index]; - store.statements.upsertEntry.run({ - plugin_id: entry.pluginId, - namespace: entry.namespace, - entry_key: entry.key, - value_json: entry.valueJson, - created_at: entry.createdAt ?? now + index, - expires_at: entry.expiresAt ?? null, - }); + upsertPluginStateEntry( + store.db, + bindPluginStateEntry({ + pluginId: entry.pluginId, + namespace: entry.namespace, + key: entry.key, + valueJson: entry.valueJson, + createdAt: entry.createdAt ?? now + index, + expiresAt: entry.expiresAt ?? null, + }), + ); } }); } export function probePluginStateStore(): PluginStateStoreProbeResult { - const dbPath = resolvePluginStateSqlitePath(process.env); + const databasePath = resolveOpenClawStateSqlitePath(process.env); const steps: PluginStateStoreProbeStep[] = []; const wasOpen = cachedDatabase !== null; + const stateWasOpen = isOpenClawStateDatabaseOpen(); const pushOk = (name: string) => steps.push({ name, ok: true }); const pushFailure = (name: string, error: unknown) => { @@ -750,20 +731,12 @@ export function probePluginStateStore(): PluginStateStoreProbeResult { code: "PLUGIN_STATE_OPEN_FAILED", operation: "probe", message: error instanceof Error ? error.message : String(error), - path: dbPath, + path: databasePath, cause: error, }); steps.push({ name, ok: false, code: wrapped.code, message: wrapped.message }); }; - try { - ensurePluginStatePermissions(dbPath); - pushOk("state-dir"); - } catch (error) { - pushFailure("state-dir", error); - return { ok: false, dbPath, steps }; - } - try { requireNodeSqlite(); pushOk("load-sqlite"); @@ -774,60 +747,59 @@ export function probePluginStateStore(): PluginStateStoreProbeResult { code: "PLUGIN_STATE_SQLITE_UNAVAILABLE", operation: "load-sqlite", message: "SQLite support is unavailable for plugin state storage.", - path: dbPath, + path: databasePath, cause: error, }), ); - return { ok: false, dbPath, steps }; + return { ok: false, databasePath, steps }; } try { - const store = openPluginStateDatabase("probe"); + openPluginStateDatabase("probe"); pushOk("open"); - ensureSchema(store.db, store.path); pushOk("schema"); - runWriteTransaction("probe", ({ statements }) => { + runWriteTransaction("probe", ({ db }) => { const now = Date.now(); - statements.upsertEntry.run({ - plugin_id: "core:plugin-state-probe", + upsertPluginStateEntry( + db, + bindPluginStateEntry({ + pluginId: "core:plugin-state-probe", + namespace: "diagnostics", + key: "probe", + valueJson: JSON.stringify({ ok: true }), + createdAt: now, + expiresAt: now + 60_000, + }), + ); + selectPluginStateEntry(db, { + pluginId: "core:plugin-state-probe", namespace: "diagnostics", - entry_key: "probe", - value_json: JSON.stringify({ ok: true }), - created_at: now, - expires_at: now + 60_000, + key: "probe", + now, + }); + deletePluginStateEntry(db, { + pluginId: "core:plugin-state-probe", + namespace: "diagnostics", + key: "probe", }); - statements.selectEntry.get("core:plugin-state-probe", "diagnostics", "probe", now); - statements.deleteEntry.run("core:plugin-state-probe", "diagnostics", "probe"); }); pushOk("write-read-delete"); - store.walMaintenance.checkpoint(); + openOpenClawStateDatabase().walMaintenance.checkpoint(); pushOk("checkpoint"); } catch (error) { pushFailure("probe", error); } finally { - if (!wasOpen) { - closePluginStateSqliteStore(); + if (!wasOpen && !stateWasOpen) { + closePluginStateDatabase(); } } - return { ok: steps.every((step) => step.ok), dbPath, steps }; + return { ok: steps.every((step) => step.ok), databasePath, steps }; } -export function closePluginStateSqliteStore(): void { - if (!cachedDatabase) { - return; - } - try { - cachedDatabase.walMaintenance.close(); - cachedDatabase.db.close(); - cachedDatabase = null; - } catch (error) { - cachedDatabase = null; - throw wrapPluginStateError( - error, - "close", - "PLUGIN_STATE_WRITE_FAILED", - "Failed to close plugin state database.", - ); - } +export function closePluginStateDatabase(): void { + cachedDatabase = null; + closeOpenClawStateDatabase(); } + +export const closePluginStateSqliteStore = closePluginStateDatabase; diff --git a/src/plugin-state/plugin-state-store.test-helpers.ts b/src/plugin-state/plugin-state-store.test-helpers.ts index 187712b2aa1a..b0d2f85ec2cd 100644 --- a/src/plugin-state/plugin-state-store.test-helpers.ts +++ b/src/plugin-state/plugin-state-store.test-helpers.ts @@ -1,4 +1,4 @@ -import { seedPluginStateSqliteEntriesForTests } from "./plugin-state-store.sqlite.js"; +import { seedPluginStateDatabaseEntriesForTests } from "./plugin-state-store.sqlite.js"; export type PluginStateSeedEntry = { pluginId: string; @@ -14,7 +14,7 @@ export function seedPluginStateEntriesForTests(entries: PluginStateSeedEntry[]): return; } - seedPluginStateSqliteEntriesForTests( + seedPluginStateDatabaseEntriesForTests( entries.map((entry) => { const valueJson = JSON.stringify(entry.value); if (valueJson == null) { diff --git a/src/plugin-state/plugin-state-store.test.ts b/src/plugin-state/plugin-state-store.test.ts index ffa1a43e403b..6266a912dbf7 100644 --- a/src/plugin-state/plugin-state-store.test.ts +++ b/src/plugin-state/plugin-state-store.test.ts @@ -1,14 +1,17 @@ -import { mkdirSync, rmSync, statSync } from "node:fs"; +import { rmSync, statSync } from "node:fs"; +import path from "node:path"; import { afterAll, afterEach, beforeAll, beforeEach, describe, expect, it, vi } from "vitest"; -import { requireNodeSqlite } from "../infra/node-sqlite.js"; +import { openOpenClawStateDatabase } from "../state/openclaw-state-db.js"; +import { resolveOpenClawStateSqlitePath } from "../state/openclaw-state-db.paths.js"; import { createOpenClawTestState, + withOpenClawTestState, type OpenClawTestState, } from "../test-utils/openclaw-test-state.js"; import { MAX_PLUGIN_STATE_ENTRIES_PER_PLUGIN, clearPluginStateStoreForTests, - closePluginStateSqliteStore, + closePluginStateDatabase, createCorePluginStateKeyedStore, createPluginStateKeyedStore, createPluginStateSyncKeyedStore, @@ -17,14 +20,13 @@ import { resetPluginStateStoreForTests, sweepExpiredPluginStateEntries, } from "./plugin-state-store.js"; -import { resolvePluginStateDir, resolvePluginStateSqlitePath } from "./plugin-state-store.paths.js"; import { seedPluginStateEntriesForTests } from "./plugin-state-store.test-helpers.js"; let testState: OpenClawTestState | undefined; beforeAll(async () => { testState = await createOpenClawTestState({ label: "plugin-state-store" }); - rmSync(resolvePluginStateDir(), { recursive: true, force: true }); + rmSync(path.dirname(resolveOpenClawStateSqlitePath()), { recursive: true, force: true }); }); beforeEach(() => { @@ -96,6 +98,38 @@ describe("plugin state keyed store", () => { }); }); + it("honors explicit store env without mutating process state", async () => { + await withOpenClawTestState( + { label: "plugin-state-explicit-env-a", applyEnv: false }, + async (stateA) => { + await withOpenClawTestState( + { label: "plugin-state-explicit-env-b", applyEnv: false }, + async (stateB) => { + const storeA = createPluginStateKeyedStore<{ owner: string }>("discord", { + namespace: "explicit-env", + maxEntries: 10, + env: stateA.env, + }); + const storeB = createPluginStateKeyedStore<{ owner: string }>("discord", { + namespace: "explicit-env", + maxEntries: 10, + env: stateB.env, + }); + + await storeA.register("shared", { owner: "a" }); + await storeB.register("shared", { owner: "b" }); + + await expect(storeA.lookup("shared")).resolves.toEqual({ owner: "a" }); + await expect(storeB.lookup("shared")).resolves.toEqual({ owner: "b" }); + expect(resolveOpenClawStateSqlitePath(stateA.env)).not.toBe( + resolveOpenClawStateSqlitePath(stateB.env), + ); + }, + ); + }, + ); + }); + it("upserts values and refreshes deterministic entry ordering", async () => { await withPluginStateTestState(async () => { vi.useFakeTimers(); @@ -590,18 +624,53 @@ describe("plugin state keyed store", () => { await withPluginStateTestState(async () => { const store = createPluginStateKeyedStore("discord", { namespace: "close", maxEntries: 10 }); await store.register("k", { ok: true }); - closePluginStateSqliteStore(); + const database = openOpenClawStateDatabase(); + closePluginStateDatabase(); + expect(() => database.db.exec("SELECT 1")).toThrow(); await expect(store.lookup("k")).resolves.toEqual({ ok: true }); }); }); + it("does not close a shared state database opened before the plugin-state probe", async () => { + await withPluginStateTestState(async () => { + const database = openOpenClawStateDatabase(); + const result = probePluginStateStore(); + + expect(result.ok).toBe(true); + expect(database.db.isOpen).toBe(true); + }); + }); + + it("reopens after the shared state DB cache closes its handle", async () => { + await withPluginStateTestState(async () => { + const store = createPluginStateKeyedStore("discord", { + namespace: "cache-switch", + maxEntries: 10, + }); + await store.register("k", { ok: true }); + + const secondary = await createOpenClawTestState({ + label: "plugin-state-cache-secondary", + applyEnv: false, + }); + try { + openOpenClawStateDatabase({ env: secondary.env }); + testState?.applyEnv(); + await expect(store.lookup("k")).resolves.toEqual({ ok: true }); + } finally { + await secondary.cleanup(); + } + }); + }); + it.runIf(process.platform !== "win32")("hardens DB directory and file permissions", async () => { await withPluginStateTestState(async () => { const store = createPluginStateKeyedStore("discord", { namespace: "perms", maxEntries: 10 }); await store.register("k", { ok: true }); - expect(statSync(resolvePluginStateDir()).mode & 0o777).toBe(0o700); - expect(statSync(resolvePluginStateSqlitePath()).mode & 0o777).toBe(0o600); + const databasePath = resolveOpenClawStateSqlitePath(); + expect(statSync(path.dirname(databasePath)).mode & 0o777).toBe(0o700); + expect(statSync(databasePath).mode & 0o777).toBe(0o600); }); }); @@ -614,20 +683,4 @@ describe("plugin state keyed store", () => { expect(JSON.stringify(result)).not.toContain("probe-value"); }); }); - - it("throws on unsupported future schema versions", async () => { - await withPluginStateTestState(async () => { - closePluginStateSqliteStore(); - mkdirSync(resolvePluginStateDir(), { recursive: true }); - const { DatabaseSync } = requireNodeSqlite(); - const db = new DatabaseSync(resolvePluginStateSqlitePath()); - db.exec("PRAGMA user_version = 2;"); - db.close(); - - const store = createPluginStateKeyedStore("discord", { namespace: "schema", maxEntries: 10 }); - await expectPluginStateStoreError(store.register("k", { ok: true }), { - code: "PLUGIN_STATE_SCHEMA_UNSUPPORTED", - }); - }); - }); }); diff --git a/src/plugin-state/plugin-state-store.ts b/src/plugin-state/plugin-state-store.ts index 8685c3d4502b..47c92fc23a27 100644 --- a/src/plugin-state/plugin-state-store.ts +++ b/src/plugin-state/plugin-state-store.ts @@ -1,6 +1,7 @@ +import { closeOpenClawStateDatabaseForTest } from "../state/openclaw-state-db.js"; import { - clearPluginStateSqliteStoreForTests, - closePluginStateSqliteStore, + clearPluginStateDatabaseForTests, + closePluginStateDatabase, MAX_PLUGIN_STATE_VALUE_BYTES, pluginStateClear, pluginStateConsume, @@ -31,10 +32,11 @@ export type { } from "./plugin-state-store.types.js"; export { PluginStateStoreError } from "./plugin-state-store.types.js"; export { + closePluginStateDatabase, closePluginStateSqliteStore, countPluginStateLiveEntries, - MAX_PLUGIN_STATE_ENTRIES_PER_PLUGIN, isPluginStateDatabaseOpen, + MAX_PLUGIN_STATE_ENTRIES_PER_PLUGIN, probePluginStateStore, sweepExpiredPluginStateEntries, } from "./plugin-state-store.sqlite.js"; @@ -424,13 +426,14 @@ export function createCorePluginStateSyncKeyedStore( } export function clearPluginStateStoreForTests(): void { - clearPluginStateSqliteStoreForTests(); + clearPluginStateDatabaseForTests(); namespaceOptionSignatures.clear(); } export function resetPluginStateStoreForTests(options: { closeDatabase?: boolean } = {}): void { if (options.closeDatabase !== false) { - closePluginStateSqliteStore(); + closePluginStateDatabase(); + closeOpenClawStateDatabaseForTest(); } namespaceOptionSignatures.clear(); } diff --git a/src/plugin-state/plugin-state-store.types.ts b/src/plugin-state/plugin-state-store.types.ts index 79613aa21486..dad9cd1eb028 100644 --- a/src/plugin-state/plugin-state-store.types.ts +++ b/src/plugin-state/plugin-state-store.types.ts @@ -35,7 +35,6 @@ export type OpenKeyedStoreOptions = { export type PluginStateStoreErrorCode = | "PLUGIN_STATE_SQLITE_UNAVAILABLE" | "PLUGIN_STATE_OPEN_FAILED" - | "PLUGIN_STATE_SCHEMA_UNSUPPORTED" | "PLUGIN_STATE_WRITE_FAILED" | "PLUGIN_STATE_READ_FAILED" | "PLUGIN_STATE_CORRUPT" @@ -88,6 +87,6 @@ export type PluginStateStoreProbeStep = { export type PluginStateStoreProbeResult = { ok: boolean; - dbPath: string; + databasePath: string; steps: PluginStateStoreProbeStep[]; }; diff --git a/src/state/openclaw-state-db.generated.d.ts b/src/state/openclaw-state-db.generated.d.ts new file mode 100644 index 000000000000..ce57a6c95e1b --- /dev/null +++ b/src/state/openclaw-state-db.generated.d.ts @@ -0,0 +1,965 @@ +/** + * This file was generated by kysely-codegen. + * Please do not edit it manually. + */ + +import type { ColumnType } from "kysely"; + +export type Generated = + T extends ColumnType + ? ColumnType + : ColumnType; + +export interface AcpReplayEvents { + at: number; + run_id: string | null; + seq: number; + session_id: string; + session_key: string; + update_json: string; +} + +export interface AcpReplaySessions { + complete: number; + created_at: number; + cwd: string; + next_seq: number; + session_id: string; + session_key: string; + updated_at: number; +} + +export interface AgentDatabases { + agent_id: string; + last_seen_at: number; + path: string; + schema_version: number; + size_bytes: number | null; +} + +export interface AgentModelCatalogs { + agent_dir: string; + catalog_key: string; + raw_json: string; + updated_at: number; +} + +export interface AndroidNotificationRecentPackages { + package_name: string; + sort_order: number; + updated_at_ms: number; +} + +export interface ApnsRegistrations { + distribution: string | null; + environment: string; + installation_id: string | null; + node_id: string; + relay_handle: string | null; + send_grant: string | null; + token: string | null; + token_debug_suffix: string | null; + topic: string; + transport: string; + updated_at_ms: number; +} + +export interface AuthProfileState { + state_json: string; + store_key: string; + updated_at: number; +} + +export interface AuthProfileStores { + store_json: string; + store_key: string; + updated_at: number; +} + +export interface BackupRuns { + archive_path: string; + created_at: number; + id: string; + manifest_json: string; + status: string; +} + +export interface CaptureBlobs { + blob_id: string; + content_type: string | null; + created_at: number; + data: Uint8Array; + encoding: string; + sha256: string; + size_bytes: number; +} + +export interface CaptureEvents { + close_code: number | null; + content_type: string | null; + data_blob_id: string | null; + data_sha256: string | null; + data_text: string | null; + direction: string; + error_text: string | null; + flow_id: string; + headers_json: string | null; + host: string | null; + id: Generated; + kind: string; + meta_json: string | null; + method: string | null; + path: string | null; + protocol: string; + session_id: string; + source_process: string; + source_scope: string; + status: number | null; + ts: number; +} + +export interface CaptureSessions { + ended_at: number | null; + id: string; + mode: string; + proxy_url: string | null; + source_process: string; + source_scope: string; + started_at: number; +} + +export interface ChannelPairingAllowEntries { + account_id: string; + channel_key: string; + entry: string; + sort_order: number; + updated_at: number; +} + +export interface ChannelPairingRequests { + account_id: string; + channel_key: string; + code: string; + created_at: string; + last_seen_at: string; + meta_json: string | null; + request_id: string; +} + +export interface CommandLogEntries { + action: string; + entry_json: string; + id: string; + sender_id: string; + session_key: string; + source: string; + timestamp_ms: number; +} + +export interface Commitments { + account_id: string | null; + agent_id: string; + attempts: number; + channel: string; + confidence: number; + created_at_ms: number; + dedupe_key: string; + dismissed_at_ms: number | null; + due_earliest_ms: number; + due_latest_ms: number; + due_timezone: string; + expired_at_ms: number | null; + id: string; + kind: string; + last_attempt_at_ms: number | null; + reason: string; + recipient_id: string | null; + record_json: string; + sender_id: string | null; + sensitivity: string; + sent_at_ms: number | null; + session_key: string; + snoozed_until_ms: number | null; + source: string; + source_message_id: string | null; + source_run_id: string | null; + status: string; + suggested_text: string; + thread_id: string | null; + updated_at_ms: number; +} + +export interface ConfigHealthEntries { + config_path: string; + last_known_good_json: string | null; + last_observed_suspicious_signature: string | null; + last_promoted_good_json: string | null; + updated_at_ms: number; +} + +export interface CronJobs { + agent_id: string | null; + anchor_ms: number | null; + at: string | null; + consecutive_errors: number | null; + consecutive_skipped: number | null; + created_at_ms: number; + delete_after_run: number | null; + delivery_account_id: string | null; + delivery_best_effort: number | null; + delivery_channel: string | null; + delivery_mode: string | null; + delivery_thread_id: string | null; + delivery_to: string | null; + description: string | null; + enabled: number; + every_ms: number | null; + failure_alert_account_id: string | null; + failure_alert_after: number | null; + failure_alert_channel: string | null; + failure_alert_cooldown_ms: number | null; + failure_alert_disabled: number | null; + failure_alert_include_skipped: number | null; + failure_alert_mode: string | null; + failure_alert_to: string | null; + failure_delivery_account_id: string | null; + failure_delivery_channel: string | null; + failure_delivery_mode: string | null; + failure_delivery_to: string | null; + job_id: string; + job_json: string; + last_delivered: number | null; + last_delivery_error: string | null; + last_delivery_status: string | null; + last_duration_ms: number | null; + last_error: string | null; + last_failure_alert_at_ms: number | null; + last_run_at_ms: number | null; + last_run_status: string | null; + name: string; + next_run_at_ms: number | null; + payload_allow_unsafe_external_content: number | null; + payload_external_content_source_json: string | null; + payload_fallbacks_json: string | null; + payload_kind: string; + payload_light_context: number | null; + payload_message: string | null; + payload_model: string | null; + payload_thinking: string | null; + payload_timeout_seconds: number | null; + payload_tools_allow_json: string | null; + running_at_ms: number | null; + runtime_updated_at_ms: number | null; + schedule_error_count: number | null; + schedule_expr: string | null; + schedule_identity: string | null; + schedule_kind: string; + schedule_tz: string | null; + session_key: string | null; + session_target: string; + sort_order: Generated; + stagger_ms: number | null; + state_json: Generated; + store_key: string; + updated_at: number; + wake_mode: string; +} + +export interface CronRunLogs { + created_at: number; + delivered: number | null; + delivery_error: string | null; + delivery_status: string | null; + diagnostics_summary: string | null; + duration_ms: number | null; + entry_json: string; + error: string | null; + job_id: string; + model: string | null; + next_run_at_ms: number | null; + provider: string | null; + run_at_ms: number | null; + run_id: string | null; + seq: number; + session_id: string | null; + session_key: string | null; + status: string | null; + store_key: string; + summary: string | null; + total_tokens: number | null; + ts: number; +} + +export interface CurrentConversationBindings { + account_id: string; + binding_id: string; + binding_key: string; + bound_at: number; + channel: string; + conversation_id: string; + conversation_kind: string; + expires_at: number | null; + metadata_json: string | null; + parent_conversation_id: string | null; + record_json: string; + status: string; + target_agent_id: string; + target_kind: string; + target_session_id: string | null; + target_session_key: string; + updated_at: number; +} + +export interface DeliveryQueueEntries { + account_id: string | null; + channel: string | null; + enqueued_at: number; + entry_json: string; + entry_kind: string | null; + failed_at: number | null; + id: string; + last_attempt_at: number | null; + last_error: string | null; + platform_send_started_at: number | null; + queue_name: string; + recovery_state: string | null; + retry_count: Generated; + session_key: string | null; + status: string; + target: string | null; + updated_at: number; +} + +export interface DeviceAuthTokens { + device_id: string; + role: string; + scopes_json: string; + token: string; + updated_at_ms: number; +} + +export interface DeviceBootstrapTokens { + device_id: string | null; + issued_at_ms: number; + last_used_at_ms: number | null; + pending_profile_json: string | null; + profile_json: string | null; + public_key: string | null; + redeemed_profile_json: string | null; + token: string; + token_key: string; + ts: number; +} + +export interface DeviceIdentities { + created_at_ms: number; + device_id: string; + identity_key: string; + private_key_pem: string; + public_key_pem: string; + updated_at_ms: number; +} + +export interface DevicePairingPaired { + approved_at_ms: number; + approved_scopes_json: string | null; + client_id: string | null; + client_mode: string | null; + created_at_ms: number; + device_family: string | null; + device_id: string; + display_name: string | null; + last_seen_at_ms: number | null; + last_seen_reason: string | null; + platform: string | null; + public_key: string; + remote_ip: string | null; + role: string | null; + roles_json: string | null; + scopes_json: string | null; + tokens_json: string | null; +} + +export interface DevicePairingPending { + client_id: string | null; + client_mode: string | null; + device_family: string | null; + device_id: string; + display_name: string | null; + is_repair: number | null; + platform: string | null; + public_key: string; + remote_ip: string | null; + request_id: string; + role: string | null; + roles_json: string | null; + scopes_json: string | null; + silent: number | null; + ts: number; +} + +export interface DiagnosticEvents { + created_at: number; + event_key: string; + payload_json: string; + scope: string; +} + +export interface DiagnosticStabilityBundles { + bundle_json: string; + bundle_key: string; + created_at: number; + generated_at: string; + reason: string; +} + +export interface ExecApprovalsConfig { + agent_count: number; + allowlist_count: number; + auto_allow_skills: number | null; + config_key: string; + default_ask: string | null; + default_ask_fallback: string | null; + default_security: string | null; + has_socket_token: number; + raw_json: string; + socket_path: string | null; + updated_at_ms: number; +} + +export interface FlowRuns { + blocked_summary: string | null; + blocked_task_id: string | null; + cancel_requested_at: number | null; + controller_id: string | null; + created_at: number; + current_step: string | null; + ended_at: number | null; + flow_id: string; + goal: string; + notify_policy: string; + owner_key: string; + requester_origin_json: string | null; + revision: Generated; + shape: string | null; + state_json: string | null; + status: string; + sync_mode: Generated; + updated_at: number; + wait_json: string | null; +} + +export interface GatewayRestartHandoff { + created_at: number; + expires_at: number; + handoff_key: string; + intent_id: string; + kind: string; + pid: number; + process_instance_id: string | null; + reason: string | null; + restart_kind: string; + restart_trace_last_at: number | null; + restart_trace_started_at: number | null; + source: string; + supervisor_mode: string; + updated_at_ms: number; + version: number; +} + +export interface GatewayRestartIntent { + created_at: number; + force: number | null; + intent_key: string; + kind: string; + pid: number; + reason: string | null; + updated_at_ms: number; + wait_ms: number | null; +} + +export interface GatewayRestartSentinel { + continuation_json: string | null; + delivery_account_id: string | null; + delivery_channel: string | null; + delivery_to: string | null; + doctor_hint: string | null; + kind: string; + message: string | null; + payload_json: string; + sentinel_key: string; + session_key: string | null; + stats_json: string | null; + status: string; + thread_id: string | null; + ts: number; + updated_at_ms: number; + version: number; +} + +export interface InstalledPluginIndex { + compat_registry_version: string; + diagnostics_json: string; + generated_at_ms: number; + host_contract_version: string; + index_key: string; + install_records_json: string; + migration_version: number; + plugins_json: string; + policy_hash: string; + refresh_reason: string | null; + updated_at_ms: number; + version: number; + warning: string | null; +} + +export interface MacosPortGuardianRecords { + command: string; + mode: string; + pid: Generated; + port: number; + timestamp: number; +} + +export interface ManagedOutgoingImageRecords { + alt: string; + attachment_id: string; + created_at: string; + message_id: string | null; + original_content_type: string; + original_filename: string | null; + original_height: number | null; + original_media_id: string; + original_media_subdir: string; + original_size_bytes: number | null; + original_width: number | null; + record_json: string; + retention_class: string | null; + session_key: string; + updated_at: string | null; +} + +export interface MediaBlobs { + blob: Uint8Array; + content_type: string | null; + created_at: number; + id: string; + size_bytes: number; + subdir: string; + updated_at: number; +} + +export interface MigrationRuns { + finished_at: number | null; + id: string; + report_json: string; + started_at: number; + status: string; +} + +export interface MigrationSources { + imported_at: number; + last_run_id: string; + migration_kind: string; + removed_source: Generated; + report_json: string; + source_key: string; + source_path: string; + source_record_count: number | null; + source_sha256: string | null; + source_size_bytes: number | null; + status: string; + target_table: string; +} + +export interface ModelCapabilityCache { + context_window: number; + cost_cache_read: number; + cost_cache_write: number; + cost_input: number; + cost_output: number; + input_image: number; + input_text: number; + max_tokens: number; + model_id: string; + name: string; + provider_id: string; + reasoning: number; + supports_tools: number | null; + updated_at_ms: number; +} + +export interface NativeHookRelayBridges { + expires_at_ms: number; + hostname: string; + pid: number; + port: number; + relay_id: string; + token: string; + updated_at_ms: number; +} + +export interface NodeHostConfig { + config_key: string; + display_name: string | null; + gateway_host: string | null; + gateway_port: number | null; + gateway_tls: number | null; + gateway_tls_fingerprint: string | null; + node_id: string; + token: string | null; + updated_at_ms: number; + version: number; +} + +export interface NodePairingPaired { + approved_at_ms: number; + bins_json: string | null; + caps_json: string | null; + client_id: string | null; + client_mode: string | null; + commands_json: string | null; + core_version: string | null; + created_at_ms: number; + device_family: string | null; + display_name: string | null; + last_connected_at_ms: number | null; + last_seen_at_ms: number | null; + last_seen_reason: string | null; + model_identifier: string | null; + node_id: string; + permissions_json: string | null; + platform: string | null; + remote_ip: string | null; + token: string; + ui_version: string | null; + version: string | null; +} + +export interface NodePairingPending { + caps_json: string | null; + client_id: string | null; + client_mode: string | null; + commands_json: string | null; + core_version: string | null; + device_family: string | null; + display_name: string | null; + model_identifier: string | null; + node_id: string; + permissions_json: string | null; + platform: string | null; + remote_ip: string | null; + request_id: string; + silent: number | null; + ts: number; + ui_version: string | null; + version: string | null; +} + +export interface PluginBindingApprovals { + account_id: string; + approved_at: number; + channel: string; + plugin_id: string; + plugin_name: string | null; + plugin_root: string; +} + +export interface PluginBlobEntries { + blob: Uint8Array; + created_at: number; + entry_key: string; + expires_at: number | null; + metadata_json: string; + namespace: string; + plugin_id: string; +} + +export interface PluginStateEntries { + created_at: number; + entry_key: string; + expires_at: number | null; + namespace: string; + plugin_id: string; + value_json: string; +} + +export interface SandboxRegistryEntries { + backend_id: string | null; + cdp_port: number | null; + config_hash: string | null; + config_label_kind: string | null; + container_name: string; + created_at_ms: number | null; + entry_json: string; + image: string | null; + last_used_at_ms: number | null; + no_vnc_port: number | null; + registry_kind: string; + runtime_label: string | null; + session_key: string | null; + updated_at: number; +} + +export interface SchemaMeta { + agent_id: string | null; + app_version: string | null; + created_at: number; + meta_key: string; + role: string; + schema_version: number; + updated_at: number; +} + +export interface SkillUploads { + actual_sha256: string | null; + archive_blob: Uint8Array; + committed: number; + committed_at: number | null; + created_at: number; + expires_at: number; + force: number; + idempotency_key_hash: string | null; + kind: string; + received_bytes: number; + sha256: string | null; + size_bytes: number; + slug: string; + upload_id: string; +} + +export interface StateLeases { + created_at: number; + expires_at: number | null; + heartbeat_at: number | null; + lease_key: string; + owner: string; + payload_json: string | null; + scope: string; + updated_at: number; +} + +export interface SubagentRuns { + accumulated_runtime_ms: number | null; + agent_dir: string | null; + announce_retry_count: number | null; + archive_at_ms: number | null; + child_session_key: string; + cleanup: string; + cleanup_completed_at: number | null; + cleanup_handled: number | null; + completion_announced_at: number | null; + controller_session_key: string | null; + created_at: number; + ended_at: number | null; + ended_hook_emitted_at: number | null; + ended_reason: string | null; + expects_completion_message: number | null; + fallback_frozen_result_captured_at: number | null; + fallback_frozen_result_text: string | null; + frozen_result_captured_at: number | null; + frozen_result_text: string | null; + label: string | null; + last_announce_delivery_error: string | null; + last_announce_retry_at: number | null; + model: string | null; + outcome_json: string | null; + pause_reason: string | null; + payload_json: Generated; + pending_final_delivery: number | null; + pending_final_delivery_attempt_count: number | null; + pending_final_delivery_created_at: number | null; + pending_final_delivery_last_attempt_at: number | null; + pending_final_delivery_last_error: string | null; + pending_final_delivery_payload_json: string | null; + requester_display_key: string; + requester_origin_json: string | null; + requester_session_key: string; + run_id: string; + run_timeout_seconds: number | null; + session_started_at: number | null; + spawn_mode: string | null; + started_at: number | null; + suppress_announce_reason: string | null; + task: string; + task_name: string | null; + wake_on_descendant_settle: number | null; + workspace_dir: string | null; +} + +export interface TaskDeliveryState { + last_notified_event_at: number | null; + requester_origin_json: string | null; + task_id: string; +} + +export interface TaskRuns { + agent_id: string | null; + child_session_key: string | null; + cleanup_after: number | null; + created_at: number; + delivery_status: string; + ended_at: number | null; + error: string | null; + label: string | null; + last_event_at: number | null; + notify_policy: string; + owner_key: string; + parent_flow_id: string | null; + parent_task_id: string | null; + progress_summary: string | null; + requester_session_key: string | null; + run_id: string | null; + runtime: string; + scope_kind: string; + source_id: string | null; + started_at: number | null; + status: string; + task: string; + task_id: string; + task_kind: string | null; + terminal_outcome: string | null; + terminal_summary: string | null; +} + +export interface TuiLastSessions { + scope_key: string; + session_key: string; + updated_at: number; +} + +export interface UpdateCheckState { + auto_first_seen_at: string | null; + auto_first_seen_tag: string | null; + auto_first_seen_version: string | null; + auto_install_id: string | null; + auto_last_attempt_at: string | null; + auto_last_attempt_version: string | null; + auto_last_success_at: string | null; + auto_last_success_version: string | null; + last_available_tag: string | null; + last_available_version: string | null; + last_checked_at: string | null; + last_notified_tag: string | null; + last_notified_version: string | null; + state_key: string; + updated_at_ms: number; +} + +export interface VoicewakeRoutingConfig { + config_key: string; + default_target_agent_id: string | null; + default_target_mode: string; + default_target_session_key: string | null; + updated_at_ms: number; + version: number; +} + +export interface VoicewakeRoutingRoutes { + config_key: string; + position: number; + target_agent_id: string | null; + target_mode: string; + target_session_key: string | null; + trigger: string; + updated_at_ms: number; +} + +export interface VoicewakeTriggers { + config_key: string; + position: number; + trigger: string; + updated_at_ms: number; +} + +export interface WebPushSubscriptions { + auth: string; + created_at_ms: number; + endpoint: string; + endpoint_hash: string; + p256dh: string; + subscription_id: string; + updated_at_ms: number; +} + +export interface WebPushVapidKeys { + key_id: string; + private_key: string; + public_key: string; + subject: string; + updated_at_ms: number; +} + +export interface WorkspaceSetupState { + bootstrap_seeded_at: string | null; + setup_completed_at: string | null; + updated_at: number; + version: number; + workspace_key: string; + workspace_path: string; +} + +export interface DB { + acp_replay_events: AcpReplayEvents; + acp_replay_sessions: AcpReplaySessions; + agent_databases: AgentDatabases; + agent_model_catalogs: AgentModelCatalogs; + android_notification_recent_packages: AndroidNotificationRecentPackages; + apns_registrations: ApnsRegistrations; + auth_profile_state: AuthProfileState; + auth_profile_stores: AuthProfileStores; + backup_runs: BackupRuns; + capture_blobs: CaptureBlobs; + capture_events: CaptureEvents; + capture_sessions: CaptureSessions; + channel_pairing_allow_entries: ChannelPairingAllowEntries; + channel_pairing_requests: ChannelPairingRequests; + command_log_entries: CommandLogEntries; + commitments: Commitments; + config_health_entries: ConfigHealthEntries; + cron_jobs: CronJobs; + cron_run_logs: CronRunLogs; + current_conversation_bindings: CurrentConversationBindings; + delivery_queue_entries: DeliveryQueueEntries; + device_auth_tokens: DeviceAuthTokens; + device_bootstrap_tokens: DeviceBootstrapTokens; + device_identities: DeviceIdentities; + device_pairing_paired: DevicePairingPaired; + device_pairing_pending: DevicePairingPending; + diagnostic_events: DiagnosticEvents; + diagnostic_stability_bundles: DiagnosticStabilityBundles; + exec_approvals_config: ExecApprovalsConfig; + flow_runs: FlowRuns; + gateway_restart_handoff: GatewayRestartHandoff; + gateway_restart_intent: GatewayRestartIntent; + gateway_restart_sentinel: GatewayRestartSentinel; + installed_plugin_index: InstalledPluginIndex; + macos_port_guardian_records: MacosPortGuardianRecords; + managed_outgoing_image_records: ManagedOutgoingImageRecords; + media_blobs: MediaBlobs; + migration_runs: MigrationRuns; + migration_sources: MigrationSources; + model_capability_cache: ModelCapabilityCache; + native_hook_relay_bridges: NativeHookRelayBridges; + node_host_config: NodeHostConfig; + node_pairing_paired: NodePairingPaired; + node_pairing_pending: NodePairingPending; + plugin_binding_approvals: PluginBindingApprovals; + plugin_blob_entries: PluginBlobEntries; + plugin_state_entries: PluginStateEntries; + sandbox_registry_entries: SandboxRegistryEntries; + schema_meta: SchemaMeta; + skill_uploads: SkillUploads; + state_leases: StateLeases; + subagent_runs: SubagentRuns; + task_delivery_state: TaskDeliveryState; + task_runs: TaskRuns; + tui_last_sessions: TuiLastSessions; + update_check_state: UpdateCheckState; + voicewake_routing_config: VoicewakeRoutingConfig; + voicewake_routing_routes: VoicewakeRoutingRoutes; + voicewake_triggers: VoicewakeTriggers; + web_push_subscriptions: WebPushSubscriptions; + web_push_vapid_keys: WebPushVapidKeys; + workspace_setup_state: WorkspaceSetupState; +} diff --git a/src/state/openclaw-state-db.paths.ts b/src/state/openclaw-state-db.paths.ts new file mode 100644 index 000000000000..f3b3ceda732b --- /dev/null +++ b/src/state/openclaw-state-db.paths.ts @@ -0,0 +1,10 @@ +import path from "node:path"; +import { resolveStateDir } from "../config/paths.js"; + +export function resolveOpenClawStateSqliteDir(env: NodeJS.ProcessEnv = process.env): string { + return path.join(resolveStateDir(env), "state"); +} + +export function resolveOpenClawStateSqlitePath(env: NodeJS.ProcessEnv = process.env): string { + return path.join(resolveOpenClawStateSqliteDir(env), "openclaw.sqlite"); +} diff --git a/src/state/openclaw-state-db.test.ts b/src/state/openclaw-state-db.test.ts new file mode 100644 index 000000000000..87baebc20482 --- /dev/null +++ b/src/state/openclaw-state-db.test.ts @@ -0,0 +1,205 @@ +import fs from "node:fs"; +import os from "node:os"; +import path from "node:path"; +import { afterEach, describe, expect, it } from "vitest"; +import { + executeSqliteQuerySync, + executeSqliteQueryTakeFirstSync, + getNodeSqliteKysely, +} from "../infra/kysely-sync.js"; +import { requireNodeSqlite } from "../infra/node-sqlite.js"; +import { readSqliteNumberPragma } from "../infra/sqlite-pragma.test-support.js"; +import type { DB as OpenClawStateKyselyDatabase } from "./openclaw-state-db.generated.js"; +import { + closeOpenClawStateDatabaseForTest, + openOpenClawStateDatabase, + runOpenClawStateWriteTransaction, +} from "./openclaw-state-db.js"; +import { resolveOpenClawStateSqlitePath } from "./openclaw-state-db.paths.js"; +import { + collectSqliteSchemaShape, + createSqliteSchemaShapeFromSql, +} from "./sqlite-schema-shape.test-support.js"; + +type StateDbTestDatabase = Pick; + +function createTempStateDir(): string { + return fs.mkdtempSync(path.join(os.tmpdir(), "openclaw-state-db-")); +} + +afterEach(() => { + closeOpenClawStateDatabaseForTest(); +}); + +describe("openclaw state database", () => { + it("resolves under the shared state database directory", () => { + const stateDir = createTempStateDir(); + + expect(resolveOpenClawStateSqlitePath({ OPENCLAW_STATE_DIR: stateDir })).toBe( + path.join(stateDir, "state", "openclaw.sqlite"), + ); + }); + + it("creates the shared state schema from the committed SQL shape", () => { + const stateDir = createTempStateDir(); + const database = openOpenClawStateDatabase({ + env: { OPENCLAW_STATE_DIR: stateDir }, + }); + + expect(collectSqliteSchemaShape(database.db)).toEqual( + createSqliteSchemaShapeFromSql(new URL("./openclaw-state-schema.sql", import.meta.url)), + ); + expect(database.path).toBe(path.join(stateDir, "state", "openclaw.sqlite")); + }); + + it("configures durable SQLite connection pragmas", () => { + const stateDir = createTempStateDir(); + const database = openOpenClawStateDatabase({ + env: { OPENCLAW_STATE_DIR: stateDir }, + }); + + expect(readSqliteNumberPragma(database.db, "busy_timeout")).toBe(30_000); + expect(readSqliteNumberPragma(database.db, "foreign_keys")).toBe(1); + expect(readSqliteNumberPragma(database.db, "synchronous")).toBe(1); + expect(readSqliteNumberPragma(database.db, "user_version")).toBe(1); + expect(readSqliteNumberPragma(database.db, "wal_autocheckpoint")).toBe(1000); + const journalMode = database.db.prepare("PRAGMA journal_mode").get() as + | { journal_mode?: string } + | undefined; + expect(journalMode?.journal_mode?.toLowerCase()).toBe("wal"); + }); + + it("records durable schema metadata", () => { + const stateDir = createTempStateDir(); + const database = openOpenClawStateDatabase({ + env: { OPENCLAW_STATE_DIR: stateDir }, + }); + const stateDb = getNodeSqliteKysely(database.db); + + expect( + executeSqliteQueryTakeFirstSync( + database.db, + stateDb.selectFrom("schema_meta").select(["role", "schema_version"]), + ), + ).toEqual({ role: "global", schema_version: 1 }); + }); + + it("refuses to open newer global schema versions", () => { + const stateDir = createTempStateDir(); + const databasePath = path.join(stateDir, "state", "openclaw.sqlite"); + fs.mkdirSync(path.dirname(databasePath), { recursive: true }); + const { DatabaseSync } = requireNodeSqlite(); + const db = new DatabaseSync(databasePath); + db.exec("PRAGMA user_version = 2;"); + db.close(); + + expect(() => + openOpenClawStateDatabase({ + env: { OPENCLAW_STATE_DIR: stateDir }, + }), + ).toThrow(/newer schema version 2/); + }); + + it("does not chmod shared parent directories for explicit database paths", () => { + const databasePath = path.join( + os.tmpdir(), + `openclaw-explicit-state-${process.pid}-${Date.now()}.sqlite`, + ); + + expect(() => openOpenClawStateDatabase({ path: databasePath })).not.toThrow(); + expect(fs.existsSync(databasePath)).toBe(true); + }); + + it("keeps cached handles open when another state path is opened", () => { + const firstPath = path.join( + createTempStateDir(), + "state", + `first-${process.pid}-${Date.now()}.sqlite`, + ); + const secondPath = path.join( + createTempStateDir(), + "state", + `second-${process.pid}-${Date.now()}.sqlite`, + ); + + const first = openOpenClawStateDatabase({ path: firstPath }); + const second = openOpenClawStateDatabase({ path: secondPath }); + + expect(first.db.isOpen).toBe(true); + expect(second.db.isOpen).toBe(true); + expect(openOpenClawStateDatabase({ path: firstPath })).toBe(first); + expect(readSqliteNumberPragma(first.db, "user_version")).toBe(1); + }); + + it("uses savepoints for nested write transaction rollback", () => { + const stateDir = createTempStateDir(); + const options = { env: { OPENCLAW_STATE_DIR: stateDir } }; + + runOpenClawStateWriteTransaction((database) => { + const stateDb = getNodeSqliteKysely(database.db); + executeSqliteQuerySync( + database.db, + stateDb.insertInto("diagnostic_events").values({ + scope: "transaction-test", + event_key: "outer", + payload_json: "{}", + created_at: 1, + }), + ); + expect(() => + runOpenClawStateWriteTransaction((inner) => { + const innerDb = getNodeSqliteKysely(inner.db); + executeSqliteQuerySync( + inner.db, + innerDb.insertInto("diagnostic_events").values({ + scope: "transaction-test", + event_key: "inner", + payload_json: "{}", + created_at: 2, + }), + ); + throw new Error("rollback nested"); + }, options), + ).toThrow("rollback nested"); + }, options); + + const database = openOpenClawStateDatabase(options); + const stateDb = getNodeSqliteKysely(database.db); + expect( + executeSqliteQuerySync( + database.db, + stateDb + .selectFrom("diagnostic_events") + .select("event_key") + .where("scope", "=", "transaction-test") + .orderBy("event_key"), + ).rows.map((row) => row.event_key), + ).toEqual(["outer"]); + }); + + it("rejects Promise-returning write transactions", () => { + const stateDir = createTempStateDir(); + const options = { env: { OPENCLAW_STATE_DIR: stateDir } }; + + expect(() => + runOpenClawStateWriteTransaction(async () => { + return "not sync"; + }, options), + ).toThrow("must be synchronous"); + + expect(() => + runOpenClawStateWriteTransaction((database) => { + const stateDb = getNodeSqliteKysely(database.db); + executeSqliteQuerySync( + database.db, + stateDb.insertInto("diagnostic_events").values({ + scope: "transaction-test", + event_key: "after", + payload_json: "{}", + created_at: 3, + }), + ); + }, options), + ).not.toThrow(); + }); +}); diff --git a/src/state/openclaw-state-db.ts b/src/state/openclaw-state-db.ts new file mode 100644 index 000000000000..089017b83fba --- /dev/null +++ b/src/state/openclaw-state-db.ts @@ -0,0 +1,317 @@ +import { randomUUID } from "node:crypto"; +import { chmodSync, existsSync, mkdirSync } from "node:fs"; +import path from "node:path"; +import type { DatabaseSync } from "node:sqlite"; +import { + clearNodeSqliteKyselyCacheForDatabase, + executeSqliteQuerySync, + getNodeSqliteKysely, +} from "../infra/kysely-sync.js"; +import { requireNodeSqlite } from "../infra/node-sqlite.js"; +import { runSqliteImmediateTransactionSync } from "../infra/sqlite-transaction.js"; +import { configureSqliteWalMaintenance, type SqliteWalMaintenance } from "../infra/sqlite-wal.js"; +import type { DB as OpenClawStateKyselyDatabase } from "./openclaw-state-db.generated.js"; +import { + resolveOpenClawStateSqliteDir, + resolveOpenClawStateSqlitePath, +} from "./openclaw-state-db.paths.js"; +import { OPENCLAW_STATE_SCHEMA_SQL } from "./openclaw-state-schema.generated.js"; + +const OPENCLAW_STATE_SCHEMA_VERSION = 1; +export const OPENCLAW_SQLITE_BUSY_TIMEOUT_MS = 30_000; +const OPENCLAW_STATE_DIR_MODE = 0o700; +const OPENCLAW_STATE_FILE_MODE = 0o600; +const OPENCLAW_STATE_SIDECAR_SUFFIXES = ["", "-shm", "-wal"] as const; + +export type OpenClawStateDatabase = { + db: DatabaseSync; + path: string; + walMaintenance: SqliteWalMaintenance; +}; + +export type OpenClawStateDatabaseOptions = { + env?: NodeJS.ProcessEnv; + path?: string; +}; + +export type OpenClawMigrationRunStatus = "completed" | "warning" | "failed"; +export type OpenClawBackupRunStatus = "completed" | "failed"; + +export type RecordOpenClawStateMigrationRunOptions = OpenClawStateDatabaseOptions & { + id?: string; + startedAt: number; + finishedAt?: number; + status: OpenClawMigrationRunStatus; + report: Record; +}; + +export type RecordOpenClawStateMigrationSourceOptions = OpenClawStateDatabaseOptions & { + runId: string; + migrationKind: string; + sourceKey: string; + sourcePath: string; + targetTable: string; + status: OpenClawMigrationRunStatus; + importedAt: number; + removedSource: boolean; + sourceSha256?: string; + sourceSizeBytes?: number; + sourceRecordCount?: number; + report: Record; +}; + +export type RecordOpenClawStateBackupRunOptions = OpenClawStateDatabaseOptions & { + id?: string; + createdAt: number; + archivePath: string; + status: OpenClawBackupRunStatus; + manifest: Record; +}; + +const cachedDatabases = new Map(); + +type OpenClawStateMetadataDatabase = Pick< + OpenClawStateKyselyDatabase, + "backup_runs" | "migration_runs" | "migration_sources" | "schema_meta" +>; + +function readSqliteUserVersion(db: DatabaseSync): number { + const row = db.prepare("PRAGMA user_version").get() as { user_version?: unknown } | undefined; + return Number(row?.user_version ?? 0); +} + +function assertSupportedSchemaVersion(db: DatabaseSync, pathname: string): void { + const userVersion = readSqliteUserVersion(db); + if (userVersion > OPENCLAW_STATE_SCHEMA_VERSION) { + throw new Error( + `OpenClaw state database ${pathname} uses newer schema version ${userVersion}; this OpenClaw build supports ${OPENCLAW_STATE_SCHEMA_VERSION}.`, + ); + } +} + +function ensureOpenClawStatePermissions(pathname: string, env: NodeJS.ProcessEnv): void { + const dir = path.dirname(pathname); + const defaultDir = resolveOpenClawStateSqliteDir(env); + const isDefaultStateDatabase = + path.resolve(pathname) === path.resolve(resolveOpenClawStateSqlitePath(env)); + if (isDefaultStateDatabase && dir !== defaultDir) { + throw new Error(`OpenClaw state database path resolved outside its state dir: ${pathname}`); + } + const dirExisted = existsSync(dir); + mkdirSync(dir, { recursive: true, mode: OPENCLAW_STATE_DIR_MODE }); + if (isDefaultStateDatabase || !dirExisted) { + chmodSync(dir, OPENCLAW_STATE_DIR_MODE); + } + for (const suffix of OPENCLAW_STATE_SIDECAR_SUFFIXES) { + const candidate = `${pathname}${suffix}`; + if (existsSync(candidate)) { + chmodSync(candidate, OPENCLAW_STATE_FILE_MODE); + } + } +} + +function tableHasColumn(db: DatabaseSync, tableName: string, columnName: string): boolean { + const rows = db.prepare(`PRAGMA table_info(${tableName})`).all() as Array<{ name?: unknown }>; + return rows.some((row) => row.name === columnName); +} + +function ensureColumn(db: DatabaseSync, tableName: string, columnSql: string): void { + const columnName = columnSql.trim().split(/\s+/, 1)[0]; + if (!columnName || tableHasColumn(db, tableName, columnName)) { + return; + } + db.exec(`ALTER TABLE ${tableName} ADD COLUMN ${columnSql};`); +} + +function ensureAdditiveStateColumns(db: DatabaseSync): void { + ensureColumn(db, "node_pairing_pending", "client_id TEXT"); + ensureColumn(db, "node_pairing_pending", "client_mode TEXT"); + ensureColumn(db, "node_pairing_paired", "client_id TEXT"); + ensureColumn(db, "node_pairing_paired", "client_mode TEXT"); +} + +function ensureSchema(db: DatabaseSync, pathname: string): void { + assertSupportedSchemaVersion(db, pathname); + db.exec(OPENCLAW_STATE_SCHEMA_SQL); + ensureAdditiveStateColumns(db); + db.exec(`PRAGMA user_version = ${OPENCLAW_STATE_SCHEMA_VERSION};`); + const now = Date.now(); + const kysely = getNodeSqliteKysely(db); + executeSqliteQuerySync( + db, + kysely + .insertInto("schema_meta") + .values({ + meta_key: "primary", + role: "global", + schema_version: OPENCLAW_STATE_SCHEMA_VERSION, + agent_id: null, + app_version: null, + created_at: now, + updated_at: now, + }) + .onConflict((conflict) => + conflict.column("meta_key").doUpdateSet({ + role: "global", + schema_version: OPENCLAW_STATE_SCHEMA_VERSION, + agent_id: null, + app_version: null, + updated_at: now, + }), + ), + ); +} + +function resolveDatabasePath(options: OpenClawStateDatabaseOptions = {}): string { + return options.path ?? resolveOpenClawStateSqlitePath(options.env ?? process.env); +} + +export function openOpenClawStateDatabase( + options: OpenClawStateDatabaseOptions = {}, +): OpenClawStateDatabase { + const env = options.env ?? process.env; + const pathname = resolveDatabasePath(options); + const cached = cachedDatabases.get(pathname); + if (cached?.db.isOpen) { + return cached; + } + if (cached) { + cached.walMaintenance.close(); + clearNodeSqliteKyselyCacheForDatabase(cached.db); + cachedDatabases.delete(pathname); + } + + ensureOpenClawStatePermissions(pathname, env); + const sqlite = requireNodeSqlite(); + const db = new sqlite.DatabaseSync(pathname); + const walMaintenance = configureSqliteWalMaintenance(db, { + databaseLabel: "openclaw-state", + databasePath: pathname, + }); + db.exec("PRAGMA synchronous = NORMAL;"); + db.exec(`PRAGMA busy_timeout = ${OPENCLAW_SQLITE_BUSY_TIMEOUT_MS};`); + db.exec("PRAGMA foreign_keys = ON;"); + try { + ensureSchema(db, pathname); + } catch (err) { + walMaintenance.close(); + db.close(); + throw err; + } + ensureOpenClawStatePermissions(pathname, env); + const database = { db, path: pathname, walMaintenance }; + cachedDatabases.set(pathname, database); + return database; +} + +export function runOpenClawStateWriteTransaction( + operation: (database: OpenClawStateDatabase) => T, + options: OpenClawStateDatabaseOptions = {}, +): T { + const database = openOpenClawStateDatabase(options); + const result = runSqliteImmediateTransactionSync(database.db, () => operation(database)); + try { + ensureOpenClawStatePermissions(database.path, options.env ?? process.env); + } catch { + // The write already committed; permission hardening is best-effort here so + // callers never retry an operation that is durable in SQLite. + } + return result; +} + +export function recordOpenClawStateMigrationRun( + options: RecordOpenClawStateMigrationRunOptions, +): string { + const id = options.id ?? randomUUID(); + runOpenClawStateWriteTransaction((database) => { + const db = getNodeSqliteKysely(database.db); + executeSqliteQuerySync( + database.db, + db.insertInto("migration_runs").values({ + id, + started_at: options.startedAt, + finished_at: options.finishedAt ?? null, + status: options.status, + report_json: JSON.stringify(options.report), + }), + ); + }, options); + return id; +} + +export function recordOpenClawStateMigrationSource( + options: RecordOpenClawStateMigrationSourceOptions, +): void { + runOpenClawStateWriteTransaction((database) => { + const db = getNodeSqliteKysely(database.db); + executeSqliteQuerySync( + database.db, + db + .insertInto("migration_sources") + .values({ + source_key: options.sourceKey, + migration_kind: options.migrationKind, + source_path: options.sourcePath, + target_table: options.targetTable, + source_sha256: options.sourceSha256 ?? null, + source_size_bytes: options.sourceSizeBytes ?? null, + source_record_count: options.sourceRecordCount ?? null, + last_run_id: options.runId, + status: options.status, + imported_at: options.importedAt, + removed_source: options.removedSource ? 1 : 0, + report_json: JSON.stringify(options.report), + }) + .onConflict((conflict) => + conflict.column("source_key").doUpdateSet({ + migration_kind: (eb) => eb.ref("excluded.migration_kind"), + source_path: (eb) => eb.ref("excluded.source_path"), + target_table: (eb) => eb.ref("excluded.target_table"), + source_sha256: (eb) => eb.ref("excluded.source_sha256"), + source_size_bytes: (eb) => eb.ref("excluded.source_size_bytes"), + source_record_count: (eb) => eb.ref("excluded.source_record_count"), + last_run_id: (eb) => eb.ref("excluded.last_run_id"), + status: (eb) => eb.ref("excluded.status"), + imported_at: (eb) => eb.ref("excluded.imported_at"), + removed_source: (eb) => eb.ref("excluded.removed_source"), + report_json: (eb) => eb.ref("excluded.report_json"), + }), + ), + ); + }, options); +} + +export function recordOpenClawStateBackupRun(options: RecordOpenClawStateBackupRunOptions): string { + const id = options.id ?? randomUUID(); + runOpenClawStateWriteTransaction((database) => { + const db = getNodeSqliteKysely(database.db); + executeSqliteQuerySync( + database.db, + db.insertInto("backup_runs").values({ + id, + created_at: options.createdAt, + archive_path: options.archivePath, + status: options.status, + manifest_json: JSON.stringify(options.manifest), + }), + ); + }, options); + return id; +} + +export function closeOpenClawStateDatabase(): void { + for (const database of cachedDatabases.values()) { + database.walMaintenance.close(); + clearNodeSqliteKyselyCacheForDatabase(database.db); + if (database.db.isOpen) { + database.db.close(); + } + } + cachedDatabases.clear(); +} + +export function isOpenClawStateDatabaseOpen(): boolean { + return Array.from(cachedDatabases.values()).some((database) => database.db.isOpen); +} + +export const closeOpenClawStateDatabaseForTest = closeOpenClawStateDatabase; diff --git a/src/state/openclaw-state-schema.generated.ts b/src/state/openclaw-state-schema.generated.ts new file mode 100644 index 000000000000..5a5f0e277789 --- /dev/null +++ b/src/state/openclaw-state-schema.generated.ts @@ -0,0 +1,1155 @@ +/** + * This file was generated from the SQLite schema source. + * Please do not edit it manually. + */ + +export const OPENCLAW_STATE_SCHEMA_SQL = `CREATE TABLE IF NOT EXISTS auth_profile_stores ( + store_key TEXT NOT NULL PRIMARY KEY, + store_json TEXT NOT NULL, + updated_at INTEGER NOT NULL +); + +CREATE TABLE IF NOT EXISTS auth_profile_state ( + store_key TEXT NOT NULL PRIMARY KEY, + state_json TEXT NOT NULL, + updated_at INTEGER NOT NULL +); + +CREATE TABLE IF NOT EXISTS diagnostic_events ( + scope TEXT NOT NULL, + event_key TEXT NOT NULL, + payload_json TEXT NOT NULL, + created_at INTEGER NOT NULL, + PRIMARY KEY (scope, event_key) +); + +CREATE INDEX IF NOT EXISTS idx_diagnostic_events_scope_created + ON diagnostic_events(scope, created_at, event_key); + +CREATE TABLE IF NOT EXISTS diagnostic_stability_bundles ( + bundle_key TEXT NOT NULL PRIMARY KEY, + reason TEXT NOT NULL, + generated_at TEXT NOT NULL, + bundle_json TEXT NOT NULL, + created_at INTEGER NOT NULL +); + +CREATE INDEX IF NOT EXISTS idx_diagnostic_stability_bundles_created + ON diagnostic_stability_bundles(created_at DESC, bundle_key); + +CREATE TABLE IF NOT EXISTS state_leases ( + scope TEXT NOT NULL, + lease_key TEXT NOT NULL, + owner TEXT NOT NULL, + expires_at INTEGER, + heartbeat_at INTEGER, + payload_json TEXT, + created_at INTEGER NOT NULL, + updated_at INTEGER NOT NULL, + PRIMARY KEY (scope, lease_key) +); + +CREATE INDEX IF NOT EXISTS idx_state_leases_expiry + ON state_leases(expires_at, scope, lease_key) + WHERE expires_at IS NOT NULL; + +CREATE INDEX IF NOT EXISTS idx_state_leases_owner + ON state_leases(owner, updated_at DESC); + +CREATE TABLE IF NOT EXISTS exec_approvals_config ( + config_key TEXT NOT NULL PRIMARY KEY, + raw_json TEXT NOT NULL, + socket_path TEXT, + has_socket_token INTEGER NOT NULL, + default_security TEXT, + default_ask TEXT, + default_ask_fallback TEXT, + auto_allow_skills INTEGER, + agent_count INTEGER NOT NULL, + allowlist_count INTEGER NOT NULL, + updated_at_ms INTEGER NOT NULL +); + +CREATE TABLE IF NOT EXISTS schema_meta ( + meta_key TEXT NOT NULL PRIMARY KEY, + role TEXT NOT NULL, + schema_version INTEGER NOT NULL, + agent_id TEXT, + app_version TEXT, + created_at INTEGER NOT NULL, + updated_at INTEGER NOT NULL +); + +CREATE TABLE IF NOT EXISTS device_pairing_pending ( + request_id TEXT NOT NULL PRIMARY KEY, + device_id TEXT NOT NULL, + public_key TEXT NOT NULL, + display_name TEXT, + platform TEXT, + device_family TEXT, + client_id TEXT, + client_mode TEXT, + role TEXT, + roles_json TEXT, + scopes_json TEXT, + remote_ip TEXT, + silent INTEGER, + is_repair INTEGER, + ts INTEGER NOT NULL +); + +CREATE INDEX IF NOT EXISTS idx_device_pairing_pending_device + ON device_pairing_pending(device_id, ts DESC); + +CREATE TABLE IF NOT EXISTS device_pairing_paired ( + device_id TEXT NOT NULL PRIMARY KEY, + public_key TEXT NOT NULL, + display_name TEXT, + platform TEXT, + device_family TEXT, + client_id TEXT, + client_mode TEXT, + role TEXT, + roles_json TEXT, + scopes_json TEXT, + approved_scopes_json TEXT, + remote_ip TEXT, + tokens_json TEXT, + created_at_ms INTEGER NOT NULL, + approved_at_ms INTEGER NOT NULL, + last_seen_at_ms INTEGER, + last_seen_reason TEXT +); + +CREATE INDEX IF NOT EXISTS idx_device_pairing_paired_approved + ON device_pairing_paired(approved_at_ms DESC, device_id); + +CREATE TABLE IF NOT EXISTS device_bootstrap_tokens ( + token_key TEXT NOT NULL PRIMARY KEY, + token TEXT NOT NULL, + ts INTEGER NOT NULL, + device_id TEXT, + public_key TEXT, + profile_json TEXT, + redeemed_profile_json TEXT, + pending_profile_json TEXT, + issued_at_ms INTEGER NOT NULL, + last_used_at_ms INTEGER +); + +CREATE INDEX IF NOT EXISTS idx_device_bootstrap_tokens_ts + ON device_bootstrap_tokens(ts); + +CREATE TABLE IF NOT EXISTS node_pairing_pending ( + request_id TEXT NOT NULL PRIMARY KEY, + node_id TEXT NOT NULL, + display_name TEXT, + platform TEXT, + version TEXT, + core_version TEXT, + ui_version TEXT, + device_family TEXT, + model_identifier TEXT, + client_id TEXT, + client_mode TEXT, + caps_json TEXT, + commands_json TEXT, + permissions_json TEXT, + remote_ip TEXT, + silent INTEGER, + ts INTEGER NOT NULL +); + +CREATE INDEX IF NOT EXISTS idx_node_pairing_pending_node + ON node_pairing_pending(node_id, ts DESC); + +CREATE TABLE IF NOT EXISTS node_pairing_paired ( + node_id TEXT NOT NULL PRIMARY KEY, + token TEXT NOT NULL, + display_name TEXT, + platform TEXT, + version TEXT, + core_version TEXT, + ui_version TEXT, + device_family TEXT, + model_identifier TEXT, + client_id TEXT, + client_mode TEXT, + caps_json TEXT, + commands_json TEXT, + permissions_json TEXT, + remote_ip TEXT, + bins_json TEXT, + created_at_ms INTEGER NOT NULL, + approved_at_ms INTEGER NOT NULL, + last_connected_at_ms INTEGER, + last_seen_at_ms INTEGER, + last_seen_reason TEXT +); + +CREATE INDEX IF NOT EXISTS idx_node_pairing_paired_approved + ON node_pairing_paired(approved_at_ms DESC, node_id); + +CREATE TABLE IF NOT EXISTS device_identities ( + identity_key TEXT NOT NULL PRIMARY KEY, + device_id TEXT NOT NULL, + public_key_pem TEXT NOT NULL, + private_key_pem TEXT NOT NULL, + created_at_ms INTEGER NOT NULL, + updated_at_ms INTEGER NOT NULL +); + +CREATE INDEX IF NOT EXISTS idx_device_identities_device + ON device_identities(device_id, updated_at_ms DESC); + +CREATE TABLE IF NOT EXISTS device_auth_tokens ( + device_id TEXT NOT NULL, + role TEXT NOT NULL, + token TEXT NOT NULL, + scopes_json TEXT NOT NULL, + updated_at_ms INTEGER NOT NULL, + PRIMARY KEY (device_id, role) +); + +CREATE INDEX IF NOT EXISTS idx_device_auth_tokens_updated + ON device_auth_tokens(updated_at_ms DESC, device_id, role); + +CREATE TABLE IF NOT EXISTS android_notification_recent_packages ( + package_name TEXT NOT NULL PRIMARY KEY, + sort_order INTEGER NOT NULL, + updated_at_ms INTEGER NOT NULL +); + +CREATE INDEX IF NOT EXISTS idx_android_notification_recent_packages_order + ON android_notification_recent_packages(sort_order, package_name); + +CREATE TABLE IF NOT EXISTS macos_port_guardian_records ( + pid INTEGER NOT NULL PRIMARY KEY, + port INTEGER NOT NULL, + command TEXT NOT NULL, + mode TEXT NOT NULL, + timestamp REAL NOT NULL +); + +CREATE INDEX IF NOT EXISTS idx_macos_port_guardian_records_port + ON macos_port_guardian_records(port, timestamp DESC); + +CREATE TABLE IF NOT EXISTS workspace_setup_state ( + workspace_key TEXT NOT NULL PRIMARY KEY, + workspace_path TEXT NOT NULL, + version INTEGER NOT NULL, + bootstrap_seeded_at TEXT, + setup_completed_at TEXT, + updated_at INTEGER NOT NULL +); + +CREATE INDEX IF NOT EXISTS idx_workspace_setup_state_path + ON workspace_setup_state(workspace_path); + +CREATE TABLE IF NOT EXISTS native_hook_relay_bridges ( + relay_id TEXT NOT NULL PRIMARY KEY, + pid INTEGER NOT NULL, + hostname TEXT NOT NULL, + port INTEGER NOT NULL, + token TEXT NOT NULL, + expires_at_ms INTEGER NOT NULL, + updated_at_ms INTEGER NOT NULL +); + +CREATE INDEX IF NOT EXISTS idx_native_hook_relay_bridges_expires + ON native_hook_relay_bridges(expires_at_ms, relay_id); + +CREATE TABLE IF NOT EXISTS model_capability_cache ( + provider_id TEXT NOT NULL, + model_id TEXT NOT NULL, + name TEXT NOT NULL, + input_text INTEGER NOT NULL, + input_image INTEGER NOT NULL, + reasoning INTEGER NOT NULL, + supports_tools INTEGER, + context_window INTEGER NOT NULL, + max_tokens INTEGER NOT NULL, + cost_input REAL NOT NULL, + cost_output REAL NOT NULL, + cost_cache_read REAL NOT NULL, + cost_cache_write REAL NOT NULL, + updated_at_ms INTEGER NOT NULL, + PRIMARY KEY (provider_id, model_id) +); + +CREATE INDEX IF NOT EXISTS idx_model_capability_cache_provider_updated + ON model_capability_cache(provider_id, updated_at_ms DESC, model_id); + +CREATE TABLE IF NOT EXISTS agent_model_catalogs ( + catalog_key TEXT NOT NULL PRIMARY KEY, + agent_dir TEXT NOT NULL, + raw_json TEXT NOT NULL, + updated_at INTEGER NOT NULL +); + +CREATE INDEX IF NOT EXISTS idx_agent_model_catalogs_agent_dir + ON agent_model_catalogs(agent_dir, updated_at DESC); + +CREATE TABLE IF NOT EXISTS managed_outgoing_image_records ( + attachment_id TEXT NOT NULL PRIMARY KEY, + session_key TEXT NOT NULL, + message_id TEXT, + created_at TEXT NOT NULL, + updated_at TEXT, + retention_class TEXT, + alt TEXT NOT NULL, + original_media_id TEXT NOT NULL, + original_media_subdir TEXT NOT NULL, + original_content_type TEXT NOT NULL, + original_width INTEGER, + original_height INTEGER, + original_size_bytes INTEGER, + original_filename TEXT, + record_json TEXT NOT NULL +); + +CREATE INDEX IF NOT EXISTS idx_managed_outgoing_images_session + ON managed_outgoing_image_records(session_key, created_at DESC, attachment_id); + +CREATE INDEX IF NOT EXISTS idx_managed_outgoing_images_message + ON managed_outgoing_image_records(session_key, message_id, attachment_id) + WHERE message_id IS NOT NULL; + +CREATE TABLE IF NOT EXISTS channel_pairing_requests ( + channel_key TEXT NOT NULL, + account_id TEXT NOT NULL, + request_id TEXT NOT NULL, + code TEXT NOT NULL, + created_at TEXT NOT NULL, + last_seen_at TEXT NOT NULL, + meta_json TEXT, + PRIMARY KEY (channel_key, account_id, request_id) +); + +CREATE INDEX IF NOT EXISTS idx_channel_pairing_requests_code + ON channel_pairing_requests(channel_key, code); + +CREATE INDEX IF NOT EXISTS idx_channel_pairing_requests_created + ON channel_pairing_requests(channel_key, created_at, request_id); + +CREATE TABLE IF NOT EXISTS channel_pairing_allow_entries ( + channel_key TEXT NOT NULL, + account_id TEXT NOT NULL, + entry TEXT NOT NULL, + sort_order INTEGER NOT NULL, + updated_at INTEGER NOT NULL, + PRIMARY KEY (channel_key, account_id, entry) +); + +CREATE INDEX IF NOT EXISTS idx_channel_pairing_allow_account + ON channel_pairing_allow_entries(channel_key, account_id, sort_order, entry); + +CREATE TABLE IF NOT EXISTS web_push_subscriptions ( + endpoint_hash TEXT NOT NULL PRIMARY KEY, + subscription_id TEXT NOT NULL UNIQUE, + endpoint TEXT NOT NULL, + p256dh TEXT NOT NULL, + auth TEXT NOT NULL, + created_at_ms INTEGER NOT NULL, + updated_at_ms INTEGER NOT NULL +); + +CREATE INDEX IF NOT EXISTS idx_web_push_subscriptions_updated + ON web_push_subscriptions(updated_at_ms DESC, subscription_id); + +CREATE TABLE IF NOT EXISTS web_push_vapid_keys ( + key_id TEXT NOT NULL PRIMARY KEY, + public_key TEXT NOT NULL, + private_key TEXT NOT NULL, + subject TEXT NOT NULL, + updated_at_ms INTEGER NOT NULL +); + +CREATE TABLE IF NOT EXISTS apns_registrations ( + node_id TEXT NOT NULL PRIMARY KEY, + transport TEXT NOT NULL, + token TEXT, + relay_handle TEXT, + send_grant TEXT, + installation_id TEXT, + topic TEXT NOT NULL, + environment TEXT NOT NULL, + distribution TEXT, + token_debug_suffix TEXT, + updated_at_ms INTEGER NOT NULL +); + +CREATE INDEX IF NOT EXISTS idx_apns_registrations_updated + ON apns_registrations(updated_at_ms DESC, node_id); + +CREATE TABLE IF NOT EXISTS node_host_config ( + config_key TEXT NOT NULL PRIMARY KEY, + version INTEGER NOT NULL, + node_id TEXT NOT NULL, + token TEXT, + display_name TEXT, + gateway_host TEXT, + gateway_port INTEGER, + gateway_tls INTEGER, + gateway_tls_fingerprint TEXT, + updated_at_ms INTEGER NOT NULL +); + +CREATE TABLE IF NOT EXISTS voicewake_triggers ( + config_key TEXT NOT NULL, + position INTEGER NOT NULL, + trigger TEXT NOT NULL, + updated_at_ms INTEGER NOT NULL, + PRIMARY KEY (config_key, position) +); + +CREATE INDEX IF NOT EXISTS idx_voicewake_triggers_trigger + ON voicewake_triggers(config_key, trigger); + +CREATE TABLE IF NOT EXISTS voicewake_routing_config ( + config_key TEXT NOT NULL PRIMARY KEY, + version INTEGER NOT NULL, + default_target_mode TEXT NOT NULL, + default_target_agent_id TEXT, + default_target_session_key TEXT, + updated_at_ms INTEGER NOT NULL +); + +CREATE TABLE IF NOT EXISTS voicewake_routing_routes ( + config_key TEXT NOT NULL, + position INTEGER NOT NULL, + trigger TEXT NOT NULL, + target_mode TEXT NOT NULL, + target_agent_id TEXT, + target_session_key TEXT, + updated_at_ms INTEGER NOT NULL, + PRIMARY KEY (config_key, position), + FOREIGN KEY (config_key) REFERENCES voicewake_routing_config(config_key) ON DELETE CASCADE +); + +CREATE INDEX IF NOT EXISTS idx_voicewake_routing_routes_trigger + ON voicewake_routing_routes(config_key, trigger); + +CREATE TABLE IF NOT EXISTS update_check_state ( + state_key TEXT NOT NULL PRIMARY KEY, + last_checked_at TEXT, + last_notified_version TEXT, + last_notified_tag TEXT, + last_available_version TEXT, + last_available_tag TEXT, + auto_install_id TEXT, + auto_first_seen_version TEXT, + auto_first_seen_tag TEXT, + auto_first_seen_at TEXT, + auto_last_attempt_version TEXT, + auto_last_attempt_at TEXT, + auto_last_success_version TEXT, + auto_last_success_at TEXT, + updated_at_ms INTEGER NOT NULL +); + +CREATE TABLE IF NOT EXISTS config_health_entries ( + config_path TEXT NOT NULL PRIMARY KEY, + last_known_good_json TEXT, + last_promoted_good_json TEXT, + last_observed_suspicious_signature TEXT, + updated_at_ms INTEGER NOT NULL +); + +CREATE TABLE IF NOT EXISTS installed_plugin_index ( + index_key TEXT NOT NULL PRIMARY KEY, + version INTEGER NOT NULL, + host_contract_version TEXT NOT NULL, + compat_registry_version TEXT NOT NULL, + migration_version INTEGER NOT NULL, + policy_hash TEXT NOT NULL, + generated_at_ms INTEGER NOT NULL, + refresh_reason TEXT, + install_records_json TEXT NOT NULL, + plugins_json TEXT NOT NULL, + diagnostics_json TEXT NOT NULL, + warning TEXT, + updated_at_ms INTEGER NOT NULL +); + +CREATE INDEX IF NOT EXISTS idx_installed_plugin_index_generated + ON installed_plugin_index(generated_at_ms DESC, index_key); + +CREATE TABLE IF NOT EXISTS gateway_restart_sentinel ( + sentinel_key TEXT NOT NULL PRIMARY KEY, + version INTEGER NOT NULL, + kind TEXT NOT NULL, + status TEXT NOT NULL, + ts INTEGER NOT NULL, + session_key TEXT, + thread_id TEXT, + delivery_channel TEXT, + delivery_to TEXT, + delivery_account_id TEXT, + message TEXT, + continuation_json TEXT, + doctor_hint TEXT, + stats_json TEXT, + payload_json TEXT NOT NULL, + updated_at_ms INTEGER NOT NULL +); + +CREATE INDEX IF NOT EXISTS idx_gateway_restart_sentinel_ts + ON gateway_restart_sentinel(ts DESC, sentinel_key); + +CREATE TABLE IF NOT EXISTS gateway_restart_intent ( + intent_key TEXT NOT NULL PRIMARY KEY, + kind TEXT NOT NULL, + pid INTEGER NOT NULL, + created_at INTEGER NOT NULL, + reason TEXT, + force INTEGER, + wait_ms INTEGER, + updated_at_ms INTEGER NOT NULL +); + +CREATE TABLE IF NOT EXISTS gateway_restart_handoff ( + handoff_key TEXT NOT NULL PRIMARY KEY, + kind TEXT NOT NULL, + version INTEGER NOT NULL, + intent_id TEXT NOT NULL, + pid INTEGER NOT NULL, + process_instance_id TEXT, + created_at INTEGER NOT NULL, + expires_at INTEGER NOT NULL, + reason TEXT, + restart_trace_started_at INTEGER, + restart_trace_last_at INTEGER, + source TEXT NOT NULL, + restart_kind TEXT NOT NULL, + supervisor_mode TEXT NOT NULL, + updated_at_ms INTEGER NOT NULL +); + +CREATE INDEX IF NOT EXISTS idx_gateway_restart_handoff_expiry + ON gateway_restart_handoff(expires_at, pid); + +CREATE TABLE IF NOT EXISTS acp_replay_sessions ( + session_id TEXT NOT NULL PRIMARY KEY, + session_key TEXT NOT NULL, + cwd TEXT NOT NULL, + complete INTEGER NOT NULL, + created_at INTEGER NOT NULL, + updated_at INTEGER NOT NULL, + next_seq INTEGER NOT NULL +); + +CREATE INDEX IF NOT EXISTS idx_acp_replay_sessions_key_updated + ON acp_replay_sessions(session_key, complete, updated_at DESC, session_id); + +CREATE INDEX IF NOT EXISTS idx_acp_replay_sessions_updated + ON acp_replay_sessions(updated_at DESC, session_id); + +CREATE TABLE IF NOT EXISTS acp_replay_events ( + session_id TEXT NOT NULL, + seq INTEGER NOT NULL, + at INTEGER NOT NULL, + session_key TEXT NOT NULL, + run_id TEXT, + update_json TEXT NOT NULL, + PRIMARY KEY (session_id, seq), + FOREIGN KEY (session_id) REFERENCES acp_replay_sessions(session_id) ON DELETE CASCADE +); + +CREATE INDEX IF NOT EXISTS idx_acp_replay_events_session_seq + ON acp_replay_events(session_id, seq); + +CREATE TABLE IF NOT EXISTS agent_databases ( + agent_id TEXT NOT NULL, + path TEXT NOT NULL, + schema_version INTEGER NOT NULL, + last_seen_at INTEGER NOT NULL, + size_bytes INTEGER, + PRIMARY KEY (agent_id, path) +); + +CREATE TABLE IF NOT EXISTS plugin_state_entries ( + plugin_id TEXT NOT NULL, + namespace TEXT NOT NULL, + entry_key TEXT NOT NULL, + value_json TEXT NOT NULL, + created_at INTEGER NOT NULL, + expires_at INTEGER, + PRIMARY KEY (plugin_id, namespace, entry_key) +); + +CREATE INDEX IF NOT EXISTS idx_plugin_state_expiry + ON plugin_state_entries(expires_at) + WHERE expires_at IS NOT NULL; + +CREATE INDEX IF NOT EXISTS idx_plugin_state_listing + ON plugin_state_entries(plugin_id, namespace, created_at, entry_key); + +CREATE TABLE IF NOT EXISTS plugin_blob_entries ( + plugin_id TEXT NOT NULL, + namespace TEXT NOT NULL, + entry_key TEXT NOT NULL, + metadata_json TEXT NOT NULL, + blob BLOB NOT NULL, + created_at INTEGER NOT NULL, + expires_at INTEGER, + PRIMARY KEY (plugin_id, namespace, entry_key) +); + +CREATE INDEX IF NOT EXISTS idx_plugin_blob_expiry + ON plugin_blob_entries(expires_at) + WHERE expires_at IS NOT NULL; + +CREATE INDEX IF NOT EXISTS idx_plugin_blob_listing + ON plugin_blob_entries(plugin_id, namespace, created_at, entry_key); + +CREATE TABLE IF NOT EXISTS media_blobs ( + subdir TEXT NOT NULL, + id TEXT NOT NULL, + content_type TEXT, + size_bytes INTEGER NOT NULL, + blob BLOB NOT NULL, + created_at INTEGER NOT NULL, + updated_at INTEGER NOT NULL, + PRIMARY KEY (subdir, id) +); + +CREATE INDEX IF NOT EXISTS idx_media_blobs_created + ON media_blobs(created_at); + +CREATE TABLE IF NOT EXISTS skill_uploads ( + upload_id TEXT NOT NULL PRIMARY KEY, + kind TEXT NOT NULL, + slug TEXT NOT NULL, + force INTEGER NOT NULL, + size_bytes INTEGER NOT NULL, + sha256 TEXT, + actual_sha256 TEXT, + received_bytes INTEGER NOT NULL, + archive_blob BLOB NOT NULL, + created_at INTEGER NOT NULL, + expires_at INTEGER NOT NULL, + committed INTEGER NOT NULL, + committed_at INTEGER, + idempotency_key_hash TEXT UNIQUE +); + +CREATE INDEX IF NOT EXISTS idx_skill_uploads_expiry + ON skill_uploads(expires_at); + +CREATE INDEX IF NOT EXISTS idx_skill_uploads_idempotency + ON skill_uploads(idempotency_key_hash) + WHERE idempotency_key_hash IS NOT NULL; + +CREATE TABLE IF NOT EXISTS capture_sessions ( + id TEXT NOT NULL PRIMARY KEY, + started_at INTEGER NOT NULL, + ended_at INTEGER, + mode TEXT NOT NULL, + source_scope TEXT NOT NULL, + source_process TEXT NOT NULL, + proxy_url TEXT +); + +CREATE TABLE IF NOT EXISTS capture_blobs ( + blob_id TEXT NOT NULL PRIMARY KEY, + content_type TEXT, + encoding TEXT NOT NULL, + size_bytes INTEGER NOT NULL, + sha256 TEXT NOT NULL, + data BLOB NOT NULL, + created_at INTEGER NOT NULL +); + +CREATE TABLE IF NOT EXISTS capture_events ( + id INTEGER NOT NULL PRIMARY KEY, + session_id TEXT NOT NULL, + ts INTEGER NOT NULL, + source_scope TEXT NOT NULL, + source_process TEXT NOT NULL, + protocol TEXT NOT NULL, + direction TEXT NOT NULL, + kind TEXT NOT NULL, + flow_id TEXT NOT NULL, + method TEXT, + host TEXT, + path TEXT, + status INTEGER, + close_code INTEGER, + content_type TEXT, + headers_json TEXT, + data_text TEXT, + data_blob_id TEXT, + data_sha256 TEXT, + error_text TEXT, + meta_json TEXT, + FOREIGN KEY (session_id) REFERENCES capture_sessions(id) ON DELETE CASCADE, + FOREIGN KEY (data_blob_id) REFERENCES capture_blobs(blob_id) ON DELETE SET NULL +); + +CREATE INDEX IF NOT EXISTS capture_events_session_ts_idx + ON capture_events(session_id, ts); + +CREATE INDEX IF NOT EXISTS capture_events_flow_idx + ON capture_events(flow_id, ts); + +CREATE TABLE IF NOT EXISTS sandbox_registry_entries ( + registry_kind TEXT NOT NULL, + container_name TEXT NOT NULL, + session_key TEXT, + backend_id TEXT, + runtime_label TEXT, + image TEXT, + created_at_ms INTEGER, + last_used_at_ms INTEGER, + config_label_kind TEXT, + config_hash TEXT, + cdp_port INTEGER, + no_vnc_port INTEGER, + entry_json TEXT NOT NULL, + updated_at INTEGER NOT NULL, + PRIMARY KEY (registry_kind, container_name) +); + +CREATE INDEX IF NOT EXISTS idx_sandbox_registry_updated + ON sandbox_registry_entries(registry_kind, updated_at DESC, container_name); + +CREATE INDEX IF NOT EXISTS idx_sandbox_registry_session + ON sandbox_registry_entries(registry_kind, session_key, last_used_at_ms DESC, container_name) + WHERE session_key IS NOT NULL; + +CREATE INDEX IF NOT EXISTS idx_sandbox_registry_last_used + ON sandbox_registry_entries(registry_kind, last_used_at_ms DESC, container_name) + WHERE last_used_at_ms IS NOT NULL; + +CREATE TABLE IF NOT EXISTS commitments ( + id TEXT NOT NULL PRIMARY KEY, + agent_id TEXT NOT NULL, + session_key TEXT NOT NULL, + channel TEXT NOT NULL, + account_id TEXT, + recipient_id TEXT, + thread_id TEXT, + sender_id TEXT, + kind TEXT NOT NULL, + sensitivity TEXT NOT NULL, + source TEXT NOT NULL, + status TEXT NOT NULL, + reason TEXT NOT NULL, + suggested_text TEXT NOT NULL, + dedupe_key TEXT NOT NULL, + confidence REAL NOT NULL, + due_earliest_ms INTEGER NOT NULL, + due_latest_ms INTEGER NOT NULL, + due_timezone TEXT NOT NULL, + source_message_id TEXT, + source_run_id TEXT, + created_at_ms INTEGER NOT NULL, + updated_at_ms INTEGER NOT NULL, + attempts INTEGER NOT NULL, + last_attempt_at_ms INTEGER, + sent_at_ms INTEGER, + dismissed_at_ms INTEGER, + snoozed_until_ms INTEGER, + expired_at_ms INTEGER, + record_json TEXT NOT NULL +); + +CREATE INDEX IF NOT EXISTS idx_commitments_scope_due + ON commitments(agent_id, session_key, status, due_earliest_ms, due_latest_ms); + +CREATE INDEX IF NOT EXISTS idx_commitments_status_due + ON commitments(status, due_earliest_ms, due_latest_ms); + +CREATE INDEX IF NOT EXISTS idx_commitments_scope_dedupe + ON commitments(agent_id, session_key, channel, dedupe_key, status); + +CREATE TABLE IF NOT EXISTS cron_run_logs ( + store_key TEXT NOT NULL, + job_id TEXT NOT NULL, + seq INTEGER NOT NULL, + ts INTEGER NOT NULL, + status TEXT, + error TEXT, + summary TEXT, + diagnostics_summary TEXT, + delivery_status TEXT, + delivery_error TEXT, + delivered INTEGER, + session_id TEXT, + session_key TEXT, + run_id TEXT, + run_at_ms INTEGER, + duration_ms INTEGER, + next_run_at_ms INTEGER, + model TEXT, + provider TEXT, + total_tokens INTEGER, + entry_json TEXT NOT NULL, + created_at INTEGER NOT NULL, + PRIMARY KEY (store_key, job_id, seq) +); + +CREATE INDEX IF NOT EXISTS idx_cron_run_logs_store_ts + ON cron_run_logs(store_key, ts DESC, seq DESC); + +CREATE INDEX IF NOT EXISTS idx_cron_run_logs_job_status + ON cron_run_logs(store_key, job_id, status, ts DESC, seq DESC); + +CREATE INDEX IF NOT EXISTS idx_cron_run_logs_delivery + ON cron_run_logs(store_key, delivery_status, ts DESC, seq DESC) + WHERE delivery_status IS NOT NULL; + +CREATE TABLE IF NOT EXISTS cron_jobs ( + store_key TEXT NOT NULL, + job_id TEXT NOT NULL, + name TEXT NOT NULL, + description TEXT, + enabled INTEGER NOT NULL, + delete_after_run INTEGER, + created_at_ms INTEGER NOT NULL, + agent_id TEXT, + session_key TEXT, + schedule_kind TEXT NOT NULL, + schedule_expr TEXT, + schedule_tz TEXT, + every_ms INTEGER, + anchor_ms INTEGER, + at TEXT, + stagger_ms INTEGER, + session_target TEXT NOT NULL, + wake_mode TEXT NOT NULL, + payload_kind TEXT NOT NULL, + payload_message TEXT, + payload_model TEXT, + payload_fallbacks_json TEXT, + payload_thinking TEXT, + payload_timeout_seconds INTEGER, + payload_allow_unsafe_external_content INTEGER, + payload_external_content_source_json TEXT, + payload_light_context INTEGER, + payload_tools_allow_json TEXT, + delivery_mode TEXT, + delivery_channel TEXT, + delivery_to TEXT, + delivery_thread_id TEXT, + delivery_account_id TEXT, + delivery_best_effort INTEGER, + failure_delivery_mode TEXT, + failure_delivery_channel TEXT, + failure_delivery_to TEXT, + failure_delivery_account_id TEXT, + failure_alert_disabled INTEGER, + failure_alert_after INTEGER, + failure_alert_channel TEXT, + failure_alert_to TEXT, + failure_alert_cooldown_ms INTEGER, + failure_alert_include_skipped INTEGER, + failure_alert_mode TEXT, + failure_alert_account_id TEXT, + next_run_at_ms INTEGER, + running_at_ms INTEGER, + last_run_at_ms INTEGER, + last_run_status TEXT, + last_error TEXT, + last_duration_ms INTEGER, + consecutive_errors INTEGER, + consecutive_skipped INTEGER, + schedule_error_count INTEGER, + last_delivery_status TEXT, + last_delivery_error TEXT, + last_delivered INTEGER, + last_failure_alert_at_ms INTEGER, + job_json TEXT NOT NULL, + state_json TEXT NOT NULL DEFAULT '{}', + runtime_updated_at_ms INTEGER, + schedule_identity TEXT, + sort_order INTEGER NOT NULL DEFAULT 0, + updated_at INTEGER NOT NULL, + PRIMARY KEY (store_key, job_id) +); + +CREATE INDEX IF NOT EXISTS idx_cron_jobs_store_updated + ON cron_jobs(store_key, sort_order ASC, updated_at DESC, job_id); + +CREATE INDEX IF NOT EXISTS idx_cron_jobs_enabled_next_run + ON cron_jobs(store_key, enabled, next_run_at_ms, job_id) + WHERE next_run_at_ms IS NOT NULL; + +CREATE INDEX IF NOT EXISTS idx_cron_jobs_agent_session + ON cron_jobs(agent_id, session_key, updated_at DESC, job_id) + WHERE agent_id IS NOT NULL OR session_key IS NOT NULL; + +CREATE TABLE IF NOT EXISTS command_log_entries ( + id TEXT NOT NULL PRIMARY KEY, + timestamp_ms INTEGER NOT NULL, + action TEXT NOT NULL, + session_key TEXT NOT NULL, + sender_id TEXT NOT NULL, + source TEXT NOT NULL, + entry_json TEXT NOT NULL +); + +CREATE INDEX IF NOT EXISTS idx_command_log_entries_timestamp + ON command_log_entries(timestamp_ms DESC, id); + +CREATE INDEX IF NOT EXISTS idx_command_log_entries_session + ON command_log_entries(session_key, timestamp_ms DESC, id); + +CREATE TABLE IF NOT EXISTS delivery_queue_entries ( + queue_name TEXT NOT NULL, + id TEXT NOT NULL, + status TEXT NOT NULL, + entry_kind TEXT, + session_key TEXT, + channel TEXT, + target TEXT, + account_id TEXT, + retry_count INTEGER NOT NULL DEFAULT 0, + last_attempt_at INTEGER, + last_error TEXT, + recovery_state TEXT, + platform_send_started_at INTEGER, + entry_json TEXT NOT NULL, + enqueued_at INTEGER NOT NULL, + updated_at INTEGER NOT NULL, + failed_at INTEGER, + PRIMARY KEY (queue_name, id) +); + +CREATE INDEX IF NOT EXISTS idx_delivery_queue_pending + ON delivery_queue_entries(queue_name, status, enqueued_at, id); + +CREATE INDEX IF NOT EXISTS idx_delivery_queue_failed + ON delivery_queue_entries(queue_name, status, failed_at, id); + +CREATE INDEX IF NOT EXISTS idx_delivery_queue_session + ON delivery_queue_entries(queue_name, status, session_key, enqueued_at, id) + WHERE session_key IS NOT NULL; + +CREATE INDEX IF NOT EXISTS idx_delivery_queue_target + ON delivery_queue_entries(queue_name, status, channel, target, enqueued_at, id) + WHERE channel IS NOT NULL AND target IS NOT NULL; + +CREATE TABLE IF NOT EXISTS task_runs ( + task_id TEXT NOT NULL PRIMARY KEY, + runtime TEXT NOT NULL, + task_kind TEXT, + source_id TEXT, + requester_session_key TEXT, + owner_key TEXT NOT NULL, + scope_kind TEXT NOT NULL, + child_session_key TEXT, + parent_flow_id TEXT, + parent_task_id TEXT, + agent_id TEXT, + run_id TEXT, + label TEXT, + task TEXT NOT NULL, + status TEXT NOT NULL, + delivery_status TEXT NOT NULL, + notify_policy TEXT NOT NULL, + created_at INTEGER NOT NULL, + started_at INTEGER, + ended_at INTEGER, + last_event_at INTEGER, + cleanup_after INTEGER, + error TEXT, + progress_summary TEXT, + terminal_summary TEXT, + terminal_outcome TEXT +); + +CREATE INDEX IF NOT EXISTS idx_task_runs_run_id ON task_runs(run_id); +CREATE INDEX IF NOT EXISTS idx_task_runs_status ON task_runs(status); +CREATE INDEX IF NOT EXISTS idx_task_runs_runtime_status ON task_runs(runtime, status); +CREATE INDEX IF NOT EXISTS idx_task_runs_cleanup_after ON task_runs(cleanup_after); +CREATE INDEX IF NOT EXISTS idx_task_runs_last_event_at ON task_runs(last_event_at); +CREATE INDEX IF NOT EXISTS idx_task_runs_owner_key ON task_runs(owner_key); +CREATE INDEX IF NOT EXISTS idx_task_runs_parent_flow_id ON task_runs(parent_flow_id); +CREATE INDEX IF NOT EXISTS idx_task_runs_child_session_key ON task_runs(child_session_key); + +CREATE TABLE IF NOT EXISTS subagent_runs ( + run_id TEXT NOT NULL PRIMARY KEY, + child_session_key TEXT NOT NULL, + controller_session_key TEXT, + requester_session_key TEXT NOT NULL, + requester_display_key TEXT NOT NULL, + requester_origin_json TEXT, + task TEXT NOT NULL, + task_name TEXT, + cleanup TEXT NOT NULL, + label TEXT, + model TEXT, + agent_dir TEXT, + workspace_dir TEXT, + run_timeout_seconds INTEGER, + spawn_mode TEXT, + created_at INTEGER NOT NULL, + started_at INTEGER, + session_started_at INTEGER, + accumulated_runtime_ms INTEGER, + ended_at INTEGER, + outcome_json TEXT, + archive_at_ms INTEGER, + cleanup_completed_at INTEGER, + cleanup_handled INTEGER, + suppress_announce_reason TEXT, + expects_completion_message INTEGER, + announce_retry_count INTEGER, + last_announce_retry_at INTEGER, + last_announce_delivery_error TEXT, + ended_reason TEXT, + pause_reason TEXT, + wake_on_descendant_settle INTEGER, + frozen_result_text TEXT, + frozen_result_captured_at INTEGER, + fallback_frozen_result_text TEXT, + fallback_frozen_result_captured_at INTEGER, + ended_hook_emitted_at INTEGER, + pending_final_delivery INTEGER, + pending_final_delivery_created_at INTEGER, + pending_final_delivery_last_attempt_at INTEGER, + pending_final_delivery_attempt_count INTEGER, + pending_final_delivery_last_error TEXT, + pending_final_delivery_payload_json TEXT, + completion_announced_at INTEGER, + payload_json TEXT NOT NULL DEFAULT '{}' +); + +CREATE INDEX IF NOT EXISTS idx_subagent_runs_child_session_key + ON subagent_runs(child_session_key, created_at DESC, run_id); +CREATE INDEX IF NOT EXISTS idx_subagent_runs_requester_session_key + ON subagent_runs(requester_session_key, created_at DESC, run_id); +CREATE INDEX IF NOT EXISTS idx_subagent_runs_controller_session_key + ON subagent_runs(controller_session_key, created_at DESC, run_id); +CREATE INDEX IF NOT EXISTS idx_subagent_runs_archive_at + ON subagent_runs(archive_at_ms, cleanup_handled, run_id); +CREATE INDEX IF NOT EXISTS idx_subagent_runs_ended_cleanup + ON subagent_runs(ended_at, cleanup_handled, run_id); + +CREATE TABLE IF NOT EXISTS current_conversation_bindings ( + binding_key TEXT NOT NULL PRIMARY KEY, + binding_id TEXT NOT NULL, + target_agent_id TEXT NOT NULL, + target_session_id TEXT, + target_session_key TEXT NOT NULL, + channel TEXT NOT NULL, + account_id TEXT NOT NULL, + conversation_kind TEXT NOT NULL, + parent_conversation_id TEXT, + conversation_id TEXT NOT NULL, + target_kind TEXT NOT NULL, + status TEXT NOT NULL, + bound_at INTEGER NOT NULL, + expires_at INTEGER, + metadata_json TEXT, + record_json TEXT NOT NULL, + updated_at INTEGER NOT NULL +); + +CREATE INDEX IF NOT EXISTS idx_current_conversation_bindings_target + ON current_conversation_bindings(target_agent_id, target_session_key, updated_at DESC, binding_key); +CREATE INDEX IF NOT EXISTS idx_current_conversation_bindings_conversation + ON current_conversation_bindings(channel, account_id, conversation_kind, conversation_id); +CREATE INDEX IF NOT EXISTS idx_current_conversation_bindings_expires + ON current_conversation_bindings(expires_at, binding_key); + +CREATE TABLE IF NOT EXISTS plugin_binding_approvals ( + plugin_root TEXT NOT NULL, + channel TEXT NOT NULL, + account_id TEXT NOT NULL, + plugin_id TEXT NOT NULL, + plugin_name TEXT, + approved_at INTEGER NOT NULL, + PRIMARY KEY (plugin_root, channel, account_id) +); + +CREATE INDEX IF NOT EXISTS idx_plugin_binding_approvals_plugin + ON plugin_binding_approvals(plugin_id, approved_at DESC); + +CREATE TABLE IF NOT EXISTS tui_last_sessions ( + scope_key TEXT NOT NULL PRIMARY KEY, + session_key TEXT NOT NULL, + updated_at INTEGER NOT NULL +); + +CREATE INDEX IF NOT EXISTS idx_tui_last_sessions_session_key + ON tui_last_sessions(session_key, updated_at DESC, scope_key); + +CREATE TABLE IF NOT EXISTS task_delivery_state ( + task_id TEXT NOT NULL PRIMARY KEY, + requester_origin_json TEXT, + last_notified_event_at INTEGER, + FOREIGN KEY (task_id) REFERENCES task_runs(task_id) ON DELETE CASCADE +); + +CREATE TABLE IF NOT EXISTS flow_runs ( + flow_id TEXT NOT NULL PRIMARY KEY, + shape TEXT, + sync_mode TEXT NOT NULL DEFAULT 'managed', + owner_key TEXT NOT NULL, + requester_origin_json TEXT, + controller_id TEXT, + revision INTEGER NOT NULL DEFAULT 0, + status TEXT NOT NULL, + notify_policy TEXT NOT NULL, + goal TEXT NOT NULL, + current_step TEXT, + blocked_task_id TEXT, + blocked_summary TEXT, + state_json TEXT, + wait_json TEXT, + cancel_requested_at INTEGER, + created_at INTEGER NOT NULL, + updated_at INTEGER NOT NULL, + ended_at INTEGER +); + +CREATE INDEX IF NOT EXISTS idx_flow_runs_status ON flow_runs(status); +CREATE INDEX IF NOT EXISTS idx_flow_runs_owner_key ON flow_runs(owner_key); +CREATE INDEX IF NOT EXISTS idx_flow_runs_updated_at ON flow_runs(updated_at); + +CREATE TABLE IF NOT EXISTS migration_runs ( + id TEXT NOT NULL PRIMARY KEY, + started_at INTEGER NOT NULL, + finished_at INTEGER, + status TEXT NOT NULL, + report_json TEXT NOT NULL +); + +CREATE INDEX IF NOT EXISTS idx_migration_runs_started + ON migration_runs(started_at DESC, id); + +CREATE TABLE IF NOT EXISTS migration_sources ( + source_key TEXT NOT NULL PRIMARY KEY, + migration_kind TEXT NOT NULL, + source_path TEXT NOT NULL, + target_table TEXT NOT NULL, + source_sha256 TEXT, + source_size_bytes INTEGER, + source_record_count INTEGER, + last_run_id TEXT NOT NULL, + status TEXT NOT NULL, + imported_at INTEGER NOT NULL, + removed_source INTEGER NOT NULL DEFAULT 0, + report_json TEXT NOT NULL, + FOREIGN KEY (last_run_id) REFERENCES migration_runs(id) ON DELETE CASCADE +); + +CREATE INDEX IF NOT EXISTS idx_migration_sources_path + ON migration_sources(source_path, migration_kind, target_table); + +CREATE INDEX IF NOT EXISTS idx_migration_sources_run + ON migration_sources(last_run_id, source_path); + +CREATE TABLE IF NOT EXISTS backup_runs ( + id TEXT NOT NULL PRIMARY KEY, + created_at INTEGER NOT NULL, + archive_path TEXT NOT NULL, + status TEXT NOT NULL, + manifest_json TEXT NOT NULL +); + +CREATE INDEX IF NOT EXISTS idx_backup_runs_created + ON backup_runs(created_at DESC, id);\n`; diff --git a/src/state/openclaw-state-schema.sql b/src/state/openclaw-state-schema.sql new file mode 100644 index 000000000000..55101c69cf94 --- /dev/null +++ b/src/state/openclaw-state-schema.sql @@ -0,0 +1,1150 @@ +CREATE TABLE IF NOT EXISTS auth_profile_stores ( + store_key TEXT NOT NULL PRIMARY KEY, + store_json TEXT NOT NULL, + updated_at INTEGER NOT NULL +); + +CREATE TABLE IF NOT EXISTS auth_profile_state ( + store_key TEXT NOT NULL PRIMARY KEY, + state_json TEXT NOT NULL, + updated_at INTEGER NOT NULL +); + +CREATE TABLE IF NOT EXISTS diagnostic_events ( + scope TEXT NOT NULL, + event_key TEXT NOT NULL, + payload_json TEXT NOT NULL, + created_at INTEGER NOT NULL, + PRIMARY KEY (scope, event_key) +); + +CREATE INDEX IF NOT EXISTS idx_diagnostic_events_scope_created + ON diagnostic_events(scope, created_at, event_key); + +CREATE TABLE IF NOT EXISTS diagnostic_stability_bundles ( + bundle_key TEXT NOT NULL PRIMARY KEY, + reason TEXT NOT NULL, + generated_at TEXT NOT NULL, + bundle_json TEXT NOT NULL, + created_at INTEGER NOT NULL +); + +CREATE INDEX IF NOT EXISTS idx_diagnostic_stability_bundles_created + ON diagnostic_stability_bundles(created_at DESC, bundle_key); + +CREATE TABLE IF NOT EXISTS state_leases ( + scope TEXT NOT NULL, + lease_key TEXT NOT NULL, + owner TEXT NOT NULL, + expires_at INTEGER, + heartbeat_at INTEGER, + payload_json TEXT, + created_at INTEGER NOT NULL, + updated_at INTEGER NOT NULL, + PRIMARY KEY (scope, lease_key) +); + +CREATE INDEX IF NOT EXISTS idx_state_leases_expiry + ON state_leases(expires_at, scope, lease_key) + WHERE expires_at IS NOT NULL; + +CREATE INDEX IF NOT EXISTS idx_state_leases_owner + ON state_leases(owner, updated_at DESC); + +CREATE TABLE IF NOT EXISTS exec_approvals_config ( + config_key TEXT NOT NULL PRIMARY KEY, + raw_json TEXT NOT NULL, + socket_path TEXT, + has_socket_token INTEGER NOT NULL, + default_security TEXT, + default_ask TEXT, + default_ask_fallback TEXT, + auto_allow_skills INTEGER, + agent_count INTEGER NOT NULL, + allowlist_count INTEGER NOT NULL, + updated_at_ms INTEGER NOT NULL +); + +CREATE TABLE IF NOT EXISTS schema_meta ( + meta_key TEXT NOT NULL PRIMARY KEY, + role TEXT NOT NULL, + schema_version INTEGER NOT NULL, + agent_id TEXT, + app_version TEXT, + created_at INTEGER NOT NULL, + updated_at INTEGER NOT NULL +); + +CREATE TABLE IF NOT EXISTS device_pairing_pending ( + request_id TEXT NOT NULL PRIMARY KEY, + device_id TEXT NOT NULL, + public_key TEXT NOT NULL, + display_name TEXT, + platform TEXT, + device_family TEXT, + client_id TEXT, + client_mode TEXT, + role TEXT, + roles_json TEXT, + scopes_json TEXT, + remote_ip TEXT, + silent INTEGER, + is_repair INTEGER, + ts INTEGER NOT NULL +); + +CREATE INDEX IF NOT EXISTS idx_device_pairing_pending_device + ON device_pairing_pending(device_id, ts DESC); + +CREATE TABLE IF NOT EXISTS device_pairing_paired ( + device_id TEXT NOT NULL PRIMARY KEY, + public_key TEXT NOT NULL, + display_name TEXT, + platform TEXT, + device_family TEXT, + client_id TEXT, + client_mode TEXT, + role TEXT, + roles_json TEXT, + scopes_json TEXT, + approved_scopes_json TEXT, + remote_ip TEXT, + tokens_json TEXT, + created_at_ms INTEGER NOT NULL, + approved_at_ms INTEGER NOT NULL, + last_seen_at_ms INTEGER, + last_seen_reason TEXT +); + +CREATE INDEX IF NOT EXISTS idx_device_pairing_paired_approved + ON device_pairing_paired(approved_at_ms DESC, device_id); + +CREATE TABLE IF NOT EXISTS device_bootstrap_tokens ( + token_key TEXT NOT NULL PRIMARY KEY, + token TEXT NOT NULL, + ts INTEGER NOT NULL, + device_id TEXT, + public_key TEXT, + profile_json TEXT, + redeemed_profile_json TEXT, + pending_profile_json TEXT, + issued_at_ms INTEGER NOT NULL, + last_used_at_ms INTEGER +); + +CREATE INDEX IF NOT EXISTS idx_device_bootstrap_tokens_ts + ON device_bootstrap_tokens(ts); + +CREATE TABLE IF NOT EXISTS node_pairing_pending ( + request_id TEXT NOT NULL PRIMARY KEY, + node_id TEXT NOT NULL, + display_name TEXT, + platform TEXT, + version TEXT, + core_version TEXT, + ui_version TEXT, + device_family TEXT, + model_identifier TEXT, + client_id TEXT, + client_mode TEXT, + caps_json TEXT, + commands_json TEXT, + permissions_json TEXT, + remote_ip TEXT, + silent INTEGER, + ts INTEGER NOT NULL +); + +CREATE INDEX IF NOT EXISTS idx_node_pairing_pending_node + ON node_pairing_pending(node_id, ts DESC); + +CREATE TABLE IF NOT EXISTS node_pairing_paired ( + node_id TEXT NOT NULL PRIMARY KEY, + token TEXT NOT NULL, + display_name TEXT, + platform TEXT, + version TEXT, + core_version TEXT, + ui_version TEXT, + device_family TEXT, + model_identifier TEXT, + client_id TEXT, + client_mode TEXT, + caps_json TEXT, + commands_json TEXT, + permissions_json TEXT, + remote_ip TEXT, + bins_json TEXT, + created_at_ms INTEGER NOT NULL, + approved_at_ms INTEGER NOT NULL, + last_connected_at_ms INTEGER, + last_seen_at_ms INTEGER, + last_seen_reason TEXT +); + +CREATE INDEX IF NOT EXISTS idx_node_pairing_paired_approved + ON node_pairing_paired(approved_at_ms DESC, node_id); + +CREATE TABLE IF NOT EXISTS device_identities ( + identity_key TEXT NOT NULL PRIMARY KEY, + device_id TEXT NOT NULL, + public_key_pem TEXT NOT NULL, + private_key_pem TEXT NOT NULL, + created_at_ms INTEGER NOT NULL, + updated_at_ms INTEGER NOT NULL +); + +CREATE INDEX IF NOT EXISTS idx_device_identities_device + ON device_identities(device_id, updated_at_ms DESC); + +CREATE TABLE IF NOT EXISTS device_auth_tokens ( + device_id TEXT NOT NULL, + role TEXT NOT NULL, + token TEXT NOT NULL, + scopes_json TEXT NOT NULL, + updated_at_ms INTEGER NOT NULL, + PRIMARY KEY (device_id, role) +); + +CREATE INDEX IF NOT EXISTS idx_device_auth_tokens_updated + ON device_auth_tokens(updated_at_ms DESC, device_id, role); + +CREATE TABLE IF NOT EXISTS android_notification_recent_packages ( + package_name TEXT NOT NULL PRIMARY KEY, + sort_order INTEGER NOT NULL, + updated_at_ms INTEGER NOT NULL +); + +CREATE INDEX IF NOT EXISTS idx_android_notification_recent_packages_order + ON android_notification_recent_packages(sort_order, package_name); + +CREATE TABLE IF NOT EXISTS macos_port_guardian_records ( + pid INTEGER NOT NULL PRIMARY KEY, + port INTEGER NOT NULL, + command TEXT NOT NULL, + mode TEXT NOT NULL, + timestamp REAL NOT NULL +); + +CREATE INDEX IF NOT EXISTS idx_macos_port_guardian_records_port + ON macos_port_guardian_records(port, timestamp DESC); + +CREATE TABLE IF NOT EXISTS workspace_setup_state ( + workspace_key TEXT NOT NULL PRIMARY KEY, + workspace_path TEXT NOT NULL, + version INTEGER NOT NULL, + bootstrap_seeded_at TEXT, + setup_completed_at TEXT, + updated_at INTEGER NOT NULL +); + +CREATE INDEX IF NOT EXISTS idx_workspace_setup_state_path + ON workspace_setup_state(workspace_path); + +CREATE TABLE IF NOT EXISTS native_hook_relay_bridges ( + relay_id TEXT NOT NULL PRIMARY KEY, + pid INTEGER NOT NULL, + hostname TEXT NOT NULL, + port INTEGER NOT NULL, + token TEXT NOT NULL, + expires_at_ms INTEGER NOT NULL, + updated_at_ms INTEGER NOT NULL +); + +CREATE INDEX IF NOT EXISTS idx_native_hook_relay_bridges_expires + ON native_hook_relay_bridges(expires_at_ms, relay_id); + +CREATE TABLE IF NOT EXISTS model_capability_cache ( + provider_id TEXT NOT NULL, + model_id TEXT NOT NULL, + name TEXT NOT NULL, + input_text INTEGER NOT NULL, + input_image INTEGER NOT NULL, + reasoning INTEGER NOT NULL, + supports_tools INTEGER, + context_window INTEGER NOT NULL, + max_tokens INTEGER NOT NULL, + cost_input REAL NOT NULL, + cost_output REAL NOT NULL, + cost_cache_read REAL NOT NULL, + cost_cache_write REAL NOT NULL, + updated_at_ms INTEGER NOT NULL, + PRIMARY KEY (provider_id, model_id) +); + +CREATE INDEX IF NOT EXISTS idx_model_capability_cache_provider_updated + ON model_capability_cache(provider_id, updated_at_ms DESC, model_id); + +CREATE TABLE IF NOT EXISTS agent_model_catalogs ( + catalog_key TEXT NOT NULL PRIMARY KEY, + agent_dir TEXT NOT NULL, + raw_json TEXT NOT NULL, + updated_at INTEGER NOT NULL +); + +CREATE INDEX IF NOT EXISTS idx_agent_model_catalogs_agent_dir + ON agent_model_catalogs(agent_dir, updated_at DESC); + +CREATE TABLE IF NOT EXISTS managed_outgoing_image_records ( + attachment_id TEXT NOT NULL PRIMARY KEY, + session_key TEXT NOT NULL, + message_id TEXT, + created_at TEXT NOT NULL, + updated_at TEXT, + retention_class TEXT, + alt TEXT NOT NULL, + original_media_id TEXT NOT NULL, + original_media_subdir TEXT NOT NULL, + original_content_type TEXT NOT NULL, + original_width INTEGER, + original_height INTEGER, + original_size_bytes INTEGER, + original_filename TEXT, + record_json TEXT NOT NULL +); + +CREATE INDEX IF NOT EXISTS idx_managed_outgoing_images_session + ON managed_outgoing_image_records(session_key, created_at DESC, attachment_id); + +CREATE INDEX IF NOT EXISTS idx_managed_outgoing_images_message + ON managed_outgoing_image_records(session_key, message_id, attachment_id) + WHERE message_id IS NOT NULL; + +CREATE TABLE IF NOT EXISTS channel_pairing_requests ( + channel_key TEXT NOT NULL, + account_id TEXT NOT NULL, + request_id TEXT NOT NULL, + code TEXT NOT NULL, + created_at TEXT NOT NULL, + last_seen_at TEXT NOT NULL, + meta_json TEXT, + PRIMARY KEY (channel_key, account_id, request_id) +); + +CREATE INDEX IF NOT EXISTS idx_channel_pairing_requests_code + ON channel_pairing_requests(channel_key, code); + +CREATE INDEX IF NOT EXISTS idx_channel_pairing_requests_created + ON channel_pairing_requests(channel_key, created_at, request_id); + +CREATE TABLE IF NOT EXISTS channel_pairing_allow_entries ( + channel_key TEXT NOT NULL, + account_id TEXT NOT NULL, + entry TEXT NOT NULL, + sort_order INTEGER NOT NULL, + updated_at INTEGER NOT NULL, + PRIMARY KEY (channel_key, account_id, entry) +); + +CREATE INDEX IF NOT EXISTS idx_channel_pairing_allow_account + ON channel_pairing_allow_entries(channel_key, account_id, sort_order, entry); + +CREATE TABLE IF NOT EXISTS web_push_subscriptions ( + endpoint_hash TEXT NOT NULL PRIMARY KEY, + subscription_id TEXT NOT NULL UNIQUE, + endpoint TEXT NOT NULL, + p256dh TEXT NOT NULL, + auth TEXT NOT NULL, + created_at_ms INTEGER NOT NULL, + updated_at_ms INTEGER NOT NULL +); + +CREATE INDEX IF NOT EXISTS idx_web_push_subscriptions_updated + ON web_push_subscriptions(updated_at_ms DESC, subscription_id); + +CREATE TABLE IF NOT EXISTS web_push_vapid_keys ( + key_id TEXT NOT NULL PRIMARY KEY, + public_key TEXT NOT NULL, + private_key TEXT NOT NULL, + subject TEXT NOT NULL, + updated_at_ms INTEGER NOT NULL +); + +CREATE TABLE IF NOT EXISTS apns_registrations ( + node_id TEXT NOT NULL PRIMARY KEY, + transport TEXT NOT NULL, + token TEXT, + relay_handle TEXT, + send_grant TEXT, + installation_id TEXT, + topic TEXT NOT NULL, + environment TEXT NOT NULL, + distribution TEXT, + token_debug_suffix TEXT, + updated_at_ms INTEGER NOT NULL +); + +CREATE INDEX IF NOT EXISTS idx_apns_registrations_updated + ON apns_registrations(updated_at_ms DESC, node_id); + +CREATE TABLE IF NOT EXISTS node_host_config ( + config_key TEXT NOT NULL PRIMARY KEY, + version INTEGER NOT NULL, + node_id TEXT NOT NULL, + token TEXT, + display_name TEXT, + gateway_host TEXT, + gateway_port INTEGER, + gateway_tls INTEGER, + gateway_tls_fingerprint TEXT, + updated_at_ms INTEGER NOT NULL +); + +CREATE TABLE IF NOT EXISTS voicewake_triggers ( + config_key TEXT NOT NULL, + position INTEGER NOT NULL, + trigger TEXT NOT NULL, + updated_at_ms INTEGER NOT NULL, + PRIMARY KEY (config_key, position) +); + +CREATE INDEX IF NOT EXISTS idx_voicewake_triggers_trigger + ON voicewake_triggers(config_key, trigger); + +CREATE TABLE IF NOT EXISTS voicewake_routing_config ( + config_key TEXT NOT NULL PRIMARY KEY, + version INTEGER NOT NULL, + default_target_mode TEXT NOT NULL, + default_target_agent_id TEXT, + default_target_session_key TEXT, + updated_at_ms INTEGER NOT NULL +); + +CREATE TABLE IF NOT EXISTS voicewake_routing_routes ( + config_key TEXT NOT NULL, + position INTEGER NOT NULL, + trigger TEXT NOT NULL, + target_mode TEXT NOT NULL, + target_agent_id TEXT, + target_session_key TEXT, + updated_at_ms INTEGER NOT NULL, + PRIMARY KEY (config_key, position), + FOREIGN KEY (config_key) REFERENCES voicewake_routing_config(config_key) ON DELETE CASCADE +); + +CREATE INDEX IF NOT EXISTS idx_voicewake_routing_routes_trigger + ON voicewake_routing_routes(config_key, trigger); + +CREATE TABLE IF NOT EXISTS update_check_state ( + state_key TEXT NOT NULL PRIMARY KEY, + last_checked_at TEXT, + last_notified_version TEXT, + last_notified_tag TEXT, + last_available_version TEXT, + last_available_tag TEXT, + auto_install_id TEXT, + auto_first_seen_version TEXT, + auto_first_seen_tag TEXT, + auto_first_seen_at TEXT, + auto_last_attempt_version TEXT, + auto_last_attempt_at TEXT, + auto_last_success_version TEXT, + auto_last_success_at TEXT, + updated_at_ms INTEGER NOT NULL +); + +CREATE TABLE IF NOT EXISTS config_health_entries ( + config_path TEXT NOT NULL PRIMARY KEY, + last_known_good_json TEXT, + last_promoted_good_json TEXT, + last_observed_suspicious_signature TEXT, + updated_at_ms INTEGER NOT NULL +); + +CREATE TABLE IF NOT EXISTS installed_plugin_index ( + index_key TEXT NOT NULL PRIMARY KEY, + version INTEGER NOT NULL, + host_contract_version TEXT NOT NULL, + compat_registry_version TEXT NOT NULL, + migration_version INTEGER NOT NULL, + policy_hash TEXT NOT NULL, + generated_at_ms INTEGER NOT NULL, + refresh_reason TEXT, + install_records_json TEXT NOT NULL, + plugins_json TEXT NOT NULL, + diagnostics_json TEXT NOT NULL, + warning TEXT, + updated_at_ms INTEGER NOT NULL +); + +CREATE INDEX IF NOT EXISTS idx_installed_plugin_index_generated + ON installed_plugin_index(generated_at_ms DESC, index_key); + +CREATE TABLE IF NOT EXISTS gateway_restart_sentinel ( + sentinel_key TEXT NOT NULL PRIMARY KEY, + version INTEGER NOT NULL, + kind TEXT NOT NULL, + status TEXT NOT NULL, + ts INTEGER NOT NULL, + session_key TEXT, + thread_id TEXT, + delivery_channel TEXT, + delivery_to TEXT, + delivery_account_id TEXT, + message TEXT, + continuation_json TEXT, + doctor_hint TEXT, + stats_json TEXT, + payload_json TEXT NOT NULL, + updated_at_ms INTEGER NOT NULL +); + +CREATE INDEX IF NOT EXISTS idx_gateway_restart_sentinel_ts + ON gateway_restart_sentinel(ts DESC, sentinel_key); + +CREATE TABLE IF NOT EXISTS gateway_restart_intent ( + intent_key TEXT NOT NULL PRIMARY KEY, + kind TEXT NOT NULL, + pid INTEGER NOT NULL, + created_at INTEGER NOT NULL, + reason TEXT, + force INTEGER, + wait_ms INTEGER, + updated_at_ms INTEGER NOT NULL +); + +CREATE TABLE IF NOT EXISTS gateway_restart_handoff ( + handoff_key TEXT NOT NULL PRIMARY KEY, + kind TEXT NOT NULL, + version INTEGER NOT NULL, + intent_id TEXT NOT NULL, + pid INTEGER NOT NULL, + process_instance_id TEXT, + created_at INTEGER NOT NULL, + expires_at INTEGER NOT NULL, + reason TEXT, + restart_trace_started_at INTEGER, + restart_trace_last_at INTEGER, + source TEXT NOT NULL, + restart_kind TEXT NOT NULL, + supervisor_mode TEXT NOT NULL, + updated_at_ms INTEGER NOT NULL +); + +CREATE INDEX IF NOT EXISTS idx_gateway_restart_handoff_expiry + ON gateway_restart_handoff(expires_at, pid); + +CREATE TABLE IF NOT EXISTS acp_replay_sessions ( + session_id TEXT NOT NULL PRIMARY KEY, + session_key TEXT NOT NULL, + cwd TEXT NOT NULL, + complete INTEGER NOT NULL, + created_at INTEGER NOT NULL, + updated_at INTEGER NOT NULL, + next_seq INTEGER NOT NULL +); + +CREATE INDEX IF NOT EXISTS idx_acp_replay_sessions_key_updated + ON acp_replay_sessions(session_key, complete, updated_at DESC, session_id); + +CREATE INDEX IF NOT EXISTS idx_acp_replay_sessions_updated + ON acp_replay_sessions(updated_at DESC, session_id); + +CREATE TABLE IF NOT EXISTS acp_replay_events ( + session_id TEXT NOT NULL, + seq INTEGER NOT NULL, + at INTEGER NOT NULL, + session_key TEXT NOT NULL, + run_id TEXT, + update_json TEXT NOT NULL, + PRIMARY KEY (session_id, seq), + FOREIGN KEY (session_id) REFERENCES acp_replay_sessions(session_id) ON DELETE CASCADE +); + +CREATE INDEX IF NOT EXISTS idx_acp_replay_events_session_seq + ON acp_replay_events(session_id, seq); + +CREATE TABLE IF NOT EXISTS agent_databases ( + agent_id TEXT NOT NULL, + path TEXT NOT NULL, + schema_version INTEGER NOT NULL, + last_seen_at INTEGER NOT NULL, + size_bytes INTEGER, + PRIMARY KEY (agent_id, path) +); + +CREATE TABLE IF NOT EXISTS plugin_state_entries ( + plugin_id TEXT NOT NULL, + namespace TEXT NOT NULL, + entry_key TEXT NOT NULL, + value_json TEXT NOT NULL, + created_at INTEGER NOT NULL, + expires_at INTEGER, + PRIMARY KEY (plugin_id, namespace, entry_key) +); + +CREATE INDEX IF NOT EXISTS idx_plugin_state_expiry + ON plugin_state_entries(expires_at) + WHERE expires_at IS NOT NULL; + +CREATE INDEX IF NOT EXISTS idx_plugin_state_listing + ON plugin_state_entries(plugin_id, namespace, created_at, entry_key); + +CREATE TABLE IF NOT EXISTS plugin_blob_entries ( + plugin_id TEXT NOT NULL, + namespace TEXT NOT NULL, + entry_key TEXT NOT NULL, + metadata_json TEXT NOT NULL, + blob BLOB NOT NULL, + created_at INTEGER NOT NULL, + expires_at INTEGER, + PRIMARY KEY (plugin_id, namespace, entry_key) +); + +CREATE INDEX IF NOT EXISTS idx_plugin_blob_expiry + ON plugin_blob_entries(expires_at) + WHERE expires_at IS NOT NULL; + +CREATE INDEX IF NOT EXISTS idx_plugin_blob_listing + ON plugin_blob_entries(plugin_id, namespace, created_at, entry_key); + +CREATE TABLE IF NOT EXISTS media_blobs ( + subdir TEXT NOT NULL, + id TEXT NOT NULL, + content_type TEXT, + size_bytes INTEGER NOT NULL, + blob BLOB NOT NULL, + created_at INTEGER NOT NULL, + updated_at INTEGER NOT NULL, + PRIMARY KEY (subdir, id) +); + +CREATE INDEX IF NOT EXISTS idx_media_blobs_created + ON media_blobs(created_at); + +CREATE TABLE IF NOT EXISTS skill_uploads ( + upload_id TEXT NOT NULL PRIMARY KEY, + kind TEXT NOT NULL, + slug TEXT NOT NULL, + force INTEGER NOT NULL, + size_bytes INTEGER NOT NULL, + sha256 TEXT, + actual_sha256 TEXT, + received_bytes INTEGER NOT NULL, + archive_blob BLOB NOT NULL, + created_at INTEGER NOT NULL, + expires_at INTEGER NOT NULL, + committed INTEGER NOT NULL, + committed_at INTEGER, + idempotency_key_hash TEXT UNIQUE +); + +CREATE INDEX IF NOT EXISTS idx_skill_uploads_expiry + ON skill_uploads(expires_at); + +CREATE INDEX IF NOT EXISTS idx_skill_uploads_idempotency + ON skill_uploads(idempotency_key_hash) + WHERE idempotency_key_hash IS NOT NULL; + +CREATE TABLE IF NOT EXISTS capture_sessions ( + id TEXT NOT NULL PRIMARY KEY, + started_at INTEGER NOT NULL, + ended_at INTEGER, + mode TEXT NOT NULL, + source_scope TEXT NOT NULL, + source_process TEXT NOT NULL, + proxy_url TEXT +); + +CREATE TABLE IF NOT EXISTS capture_blobs ( + blob_id TEXT NOT NULL PRIMARY KEY, + content_type TEXT, + encoding TEXT NOT NULL, + size_bytes INTEGER NOT NULL, + sha256 TEXT NOT NULL, + data BLOB NOT NULL, + created_at INTEGER NOT NULL +); + +CREATE TABLE IF NOT EXISTS capture_events ( + id INTEGER NOT NULL PRIMARY KEY, + session_id TEXT NOT NULL, + ts INTEGER NOT NULL, + source_scope TEXT NOT NULL, + source_process TEXT NOT NULL, + protocol TEXT NOT NULL, + direction TEXT NOT NULL, + kind TEXT NOT NULL, + flow_id TEXT NOT NULL, + method TEXT, + host TEXT, + path TEXT, + status INTEGER, + close_code INTEGER, + content_type TEXT, + headers_json TEXT, + data_text TEXT, + data_blob_id TEXT, + data_sha256 TEXT, + error_text TEXT, + meta_json TEXT, + FOREIGN KEY (session_id) REFERENCES capture_sessions(id) ON DELETE CASCADE, + FOREIGN KEY (data_blob_id) REFERENCES capture_blobs(blob_id) ON DELETE SET NULL +); + +CREATE INDEX IF NOT EXISTS capture_events_session_ts_idx + ON capture_events(session_id, ts); + +CREATE INDEX IF NOT EXISTS capture_events_flow_idx + ON capture_events(flow_id, ts); + +CREATE TABLE IF NOT EXISTS sandbox_registry_entries ( + registry_kind TEXT NOT NULL, + container_name TEXT NOT NULL, + session_key TEXT, + backend_id TEXT, + runtime_label TEXT, + image TEXT, + created_at_ms INTEGER, + last_used_at_ms INTEGER, + config_label_kind TEXT, + config_hash TEXT, + cdp_port INTEGER, + no_vnc_port INTEGER, + entry_json TEXT NOT NULL, + updated_at INTEGER NOT NULL, + PRIMARY KEY (registry_kind, container_name) +); + +CREATE INDEX IF NOT EXISTS idx_sandbox_registry_updated + ON sandbox_registry_entries(registry_kind, updated_at DESC, container_name); + +CREATE INDEX IF NOT EXISTS idx_sandbox_registry_session + ON sandbox_registry_entries(registry_kind, session_key, last_used_at_ms DESC, container_name) + WHERE session_key IS NOT NULL; + +CREATE INDEX IF NOT EXISTS idx_sandbox_registry_last_used + ON sandbox_registry_entries(registry_kind, last_used_at_ms DESC, container_name) + WHERE last_used_at_ms IS NOT NULL; + +CREATE TABLE IF NOT EXISTS commitments ( + id TEXT NOT NULL PRIMARY KEY, + agent_id TEXT NOT NULL, + session_key TEXT NOT NULL, + channel TEXT NOT NULL, + account_id TEXT, + recipient_id TEXT, + thread_id TEXT, + sender_id TEXT, + kind TEXT NOT NULL, + sensitivity TEXT NOT NULL, + source TEXT NOT NULL, + status TEXT NOT NULL, + reason TEXT NOT NULL, + suggested_text TEXT NOT NULL, + dedupe_key TEXT NOT NULL, + confidence REAL NOT NULL, + due_earliest_ms INTEGER NOT NULL, + due_latest_ms INTEGER NOT NULL, + due_timezone TEXT NOT NULL, + source_message_id TEXT, + source_run_id TEXT, + created_at_ms INTEGER NOT NULL, + updated_at_ms INTEGER NOT NULL, + attempts INTEGER NOT NULL, + last_attempt_at_ms INTEGER, + sent_at_ms INTEGER, + dismissed_at_ms INTEGER, + snoozed_until_ms INTEGER, + expired_at_ms INTEGER, + record_json TEXT NOT NULL +); + +CREATE INDEX IF NOT EXISTS idx_commitments_scope_due + ON commitments(agent_id, session_key, status, due_earliest_ms, due_latest_ms); + +CREATE INDEX IF NOT EXISTS idx_commitments_status_due + ON commitments(status, due_earliest_ms, due_latest_ms); + +CREATE INDEX IF NOT EXISTS idx_commitments_scope_dedupe + ON commitments(agent_id, session_key, channel, dedupe_key, status); + +CREATE TABLE IF NOT EXISTS cron_run_logs ( + store_key TEXT NOT NULL, + job_id TEXT NOT NULL, + seq INTEGER NOT NULL, + ts INTEGER NOT NULL, + status TEXT, + error TEXT, + summary TEXT, + diagnostics_summary TEXT, + delivery_status TEXT, + delivery_error TEXT, + delivered INTEGER, + session_id TEXT, + session_key TEXT, + run_id TEXT, + run_at_ms INTEGER, + duration_ms INTEGER, + next_run_at_ms INTEGER, + model TEXT, + provider TEXT, + total_tokens INTEGER, + entry_json TEXT NOT NULL, + created_at INTEGER NOT NULL, + PRIMARY KEY (store_key, job_id, seq) +); + +CREATE INDEX IF NOT EXISTS idx_cron_run_logs_store_ts + ON cron_run_logs(store_key, ts DESC, seq DESC); + +CREATE INDEX IF NOT EXISTS idx_cron_run_logs_job_status + ON cron_run_logs(store_key, job_id, status, ts DESC, seq DESC); + +CREATE INDEX IF NOT EXISTS idx_cron_run_logs_delivery + ON cron_run_logs(store_key, delivery_status, ts DESC, seq DESC) + WHERE delivery_status IS NOT NULL; + +CREATE TABLE IF NOT EXISTS cron_jobs ( + store_key TEXT NOT NULL, + job_id TEXT NOT NULL, + name TEXT NOT NULL, + description TEXT, + enabled INTEGER NOT NULL, + delete_after_run INTEGER, + created_at_ms INTEGER NOT NULL, + agent_id TEXT, + session_key TEXT, + schedule_kind TEXT NOT NULL, + schedule_expr TEXT, + schedule_tz TEXT, + every_ms INTEGER, + anchor_ms INTEGER, + at TEXT, + stagger_ms INTEGER, + session_target TEXT NOT NULL, + wake_mode TEXT NOT NULL, + payload_kind TEXT NOT NULL, + payload_message TEXT, + payload_model TEXT, + payload_fallbacks_json TEXT, + payload_thinking TEXT, + payload_timeout_seconds INTEGER, + payload_allow_unsafe_external_content INTEGER, + payload_external_content_source_json TEXT, + payload_light_context INTEGER, + payload_tools_allow_json TEXT, + delivery_mode TEXT, + delivery_channel TEXT, + delivery_to TEXT, + delivery_thread_id TEXT, + delivery_account_id TEXT, + delivery_best_effort INTEGER, + failure_delivery_mode TEXT, + failure_delivery_channel TEXT, + failure_delivery_to TEXT, + failure_delivery_account_id TEXT, + failure_alert_disabled INTEGER, + failure_alert_after INTEGER, + failure_alert_channel TEXT, + failure_alert_to TEXT, + failure_alert_cooldown_ms INTEGER, + failure_alert_include_skipped INTEGER, + failure_alert_mode TEXT, + failure_alert_account_id TEXT, + next_run_at_ms INTEGER, + running_at_ms INTEGER, + last_run_at_ms INTEGER, + last_run_status TEXT, + last_error TEXT, + last_duration_ms INTEGER, + consecutive_errors INTEGER, + consecutive_skipped INTEGER, + schedule_error_count INTEGER, + last_delivery_status TEXT, + last_delivery_error TEXT, + last_delivered INTEGER, + last_failure_alert_at_ms INTEGER, + job_json TEXT NOT NULL, + state_json TEXT NOT NULL DEFAULT '{}', + runtime_updated_at_ms INTEGER, + schedule_identity TEXT, + sort_order INTEGER NOT NULL DEFAULT 0, + updated_at INTEGER NOT NULL, + PRIMARY KEY (store_key, job_id) +); + +CREATE INDEX IF NOT EXISTS idx_cron_jobs_store_updated + ON cron_jobs(store_key, sort_order ASC, updated_at DESC, job_id); + +CREATE INDEX IF NOT EXISTS idx_cron_jobs_enabled_next_run + ON cron_jobs(store_key, enabled, next_run_at_ms, job_id) + WHERE next_run_at_ms IS NOT NULL; + +CREATE INDEX IF NOT EXISTS idx_cron_jobs_agent_session + ON cron_jobs(agent_id, session_key, updated_at DESC, job_id) + WHERE agent_id IS NOT NULL OR session_key IS NOT NULL; + +CREATE TABLE IF NOT EXISTS command_log_entries ( + id TEXT NOT NULL PRIMARY KEY, + timestamp_ms INTEGER NOT NULL, + action TEXT NOT NULL, + session_key TEXT NOT NULL, + sender_id TEXT NOT NULL, + source TEXT NOT NULL, + entry_json TEXT NOT NULL +); + +CREATE INDEX IF NOT EXISTS idx_command_log_entries_timestamp + ON command_log_entries(timestamp_ms DESC, id); + +CREATE INDEX IF NOT EXISTS idx_command_log_entries_session + ON command_log_entries(session_key, timestamp_ms DESC, id); + +CREATE TABLE IF NOT EXISTS delivery_queue_entries ( + queue_name TEXT NOT NULL, + id TEXT NOT NULL, + status TEXT NOT NULL, + entry_kind TEXT, + session_key TEXT, + channel TEXT, + target TEXT, + account_id TEXT, + retry_count INTEGER NOT NULL DEFAULT 0, + last_attempt_at INTEGER, + last_error TEXT, + recovery_state TEXT, + platform_send_started_at INTEGER, + entry_json TEXT NOT NULL, + enqueued_at INTEGER NOT NULL, + updated_at INTEGER NOT NULL, + failed_at INTEGER, + PRIMARY KEY (queue_name, id) +); + +CREATE INDEX IF NOT EXISTS idx_delivery_queue_pending + ON delivery_queue_entries(queue_name, status, enqueued_at, id); + +CREATE INDEX IF NOT EXISTS idx_delivery_queue_failed + ON delivery_queue_entries(queue_name, status, failed_at, id); + +CREATE INDEX IF NOT EXISTS idx_delivery_queue_session + ON delivery_queue_entries(queue_name, status, session_key, enqueued_at, id) + WHERE session_key IS NOT NULL; + +CREATE INDEX IF NOT EXISTS idx_delivery_queue_target + ON delivery_queue_entries(queue_name, status, channel, target, enqueued_at, id) + WHERE channel IS NOT NULL AND target IS NOT NULL; + +CREATE TABLE IF NOT EXISTS task_runs ( + task_id TEXT NOT NULL PRIMARY KEY, + runtime TEXT NOT NULL, + task_kind TEXT, + source_id TEXT, + requester_session_key TEXT, + owner_key TEXT NOT NULL, + scope_kind TEXT NOT NULL, + child_session_key TEXT, + parent_flow_id TEXT, + parent_task_id TEXT, + agent_id TEXT, + run_id TEXT, + label TEXT, + task TEXT NOT NULL, + status TEXT NOT NULL, + delivery_status TEXT NOT NULL, + notify_policy TEXT NOT NULL, + created_at INTEGER NOT NULL, + started_at INTEGER, + ended_at INTEGER, + last_event_at INTEGER, + cleanup_after INTEGER, + error TEXT, + progress_summary TEXT, + terminal_summary TEXT, + terminal_outcome TEXT +); + +CREATE INDEX IF NOT EXISTS idx_task_runs_run_id ON task_runs(run_id); +CREATE INDEX IF NOT EXISTS idx_task_runs_status ON task_runs(status); +CREATE INDEX IF NOT EXISTS idx_task_runs_runtime_status ON task_runs(runtime, status); +CREATE INDEX IF NOT EXISTS idx_task_runs_cleanup_after ON task_runs(cleanup_after); +CREATE INDEX IF NOT EXISTS idx_task_runs_last_event_at ON task_runs(last_event_at); +CREATE INDEX IF NOT EXISTS idx_task_runs_owner_key ON task_runs(owner_key); +CREATE INDEX IF NOT EXISTS idx_task_runs_parent_flow_id ON task_runs(parent_flow_id); +CREATE INDEX IF NOT EXISTS idx_task_runs_child_session_key ON task_runs(child_session_key); + +CREATE TABLE IF NOT EXISTS subagent_runs ( + run_id TEXT NOT NULL PRIMARY KEY, + child_session_key TEXT NOT NULL, + controller_session_key TEXT, + requester_session_key TEXT NOT NULL, + requester_display_key TEXT NOT NULL, + requester_origin_json TEXT, + task TEXT NOT NULL, + task_name TEXT, + cleanup TEXT NOT NULL, + label TEXT, + model TEXT, + agent_dir TEXT, + workspace_dir TEXT, + run_timeout_seconds INTEGER, + spawn_mode TEXT, + created_at INTEGER NOT NULL, + started_at INTEGER, + session_started_at INTEGER, + accumulated_runtime_ms INTEGER, + ended_at INTEGER, + outcome_json TEXT, + archive_at_ms INTEGER, + cleanup_completed_at INTEGER, + cleanup_handled INTEGER, + suppress_announce_reason TEXT, + expects_completion_message INTEGER, + announce_retry_count INTEGER, + last_announce_retry_at INTEGER, + last_announce_delivery_error TEXT, + ended_reason TEXT, + pause_reason TEXT, + wake_on_descendant_settle INTEGER, + frozen_result_text TEXT, + frozen_result_captured_at INTEGER, + fallback_frozen_result_text TEXT, + fallback_frozen_result_captured_at INTEGER, + ended_hook_emitted_at INTEGER, + pending_final_delivery INTEGER, + pending_final_delivery_created_at INTEGER, + pending_final_delivery_last_attempt_at INTEGER, + pending_final_delivery_attempt_count INTEGER, + pending_final_delivery_last_error TEXT, + pending_final_delivery_payload_json TEXT, + completion_announced_at INTEGER, + payload_json TEXT NOT NULL DEFAULT '{}' +); + +CREATE INDEX IF NOT EXISTS idx_subagent_runs_child_session_key + ON subagent_runs(child_session_key, created_at DESC, run_id); +CREATE INDEX IF NOT EXISTS idx_subagent_runs_requester_session_key + ON subagent_runs(requester_session_key, created_at DESC, run_id); +CREATE INDEX IF NOT EXISTS idx_subagent_runs_controller_session_key + ON subagent_runs(controller_session_key, created_at DESC, run_id); +CREATE INDEX IF NOT EXISTS idx_subagent_runs_archive_at + ON subagent_runs(archive_at_ms, cleanup_handled, run_id); +CREATE INDEX IF NOT EXISTS idx_subagent_runs_ended_cleanup + ON subagent_runs(ended_at, cleanup_handled, run_id); + +CREATE TABLE IF NOT EXISTS current_conversation_bindings ( + binding_key TEXT NOT NULL PRIMARY KEY, + binding_id TEXT NOT NULL, + target_agent_id TEXT NOT NULL, + target_session_id TEXT, + target_session_key TEXT NOT NULL, + channel TEXT NOT NULL, + account_id TEXT NOT NULL, + conversation_kind TEXT NOT NULL, + parent_conversation_id TEXT, + conversation_id TEXT NOT NULL, + target_kind TEXT NOT NULL, + status TEXT NOT NULL, + bound_at INTEGER NOT NULL, + expires_at INTEGER, + metadata_json TEXT, + record_json TEXT NOT NULL, + updated_at INTEGER NOT NULL +); + +CREATE INDEX IF NOT EXISTS idx_current_conversation_bindings_target + ON current_conversation_bindings(target_agent_id, target_session_key, updated_at DESC, binding_key); +CREATE INDEX IF NOT EXISTS idx_current_conversation_bindings_conversation + ON current_conversation_bindings(channel, account_id, conversation_kind, conversation_id); +CREATE INDEX IF NOT EXISTS idx_current_conversation_bindings_expires + ON current_conversation_bindings(expires_at, binding_key); + +CREATE TABLE IF NOT EXISTS plugin_binding_approvals ( + plugin_root TEXT NOT NULL, + channel TEXT NOT NULL, + account_id TEXT NOT NULL, + plugin_id TEXT NOT NULL, + plugin_name TEXT, + approved_at INTEGER NOT NULL, + PRIMARY KEY (plugin_root, channel, account_id) +); + +CREATE INDEX IF NOT EXISTS idx_plugin_binding_approvals_plugin + ON plugin_binding_approvals(plugin_id, approved_at DESC); + +CREATE TABLE IF NOT EXISTS tui_last_sessions ( + scope_key TEXT NOT NULL PRIMARY KEY, + session_key TEXT NOT NULL, + updated_at INTEGER NOT NULL +); + +CREATE INDEX IF NOT EXISTS idx_tui_last_sessions_session_key + ON tui_last_sessions(session_key, updated_at DESC, scope_key); + +CREATE TABLE IF NOT EXISTS task_delivery_state ( + task_id TEXT NOT NULL PRIMARY KEY, + requester_origin_json TEXT, + last_notified_event_at INTEGER, + FOREIGN KEY (task_id) REFERENCES task_runs(task_id) ON DELETE CASCADE +); + +CREATE TABLE IF NOT EXISTS flow_runs ( + flow_id TEXT NOT NULL PRIMARY KEY, + shape TEXT, + sync_mode TEXT NOT NULL DEFAULT 'managed', + owner_key TEXT NOT NULL, + requester_origin_json TEXT, + controller_id TEXT, + revision INTEGER NOT NULL DEFAULT 0, + status TEXT NOT NULL, + notify_policy TEXT NOT NULL, + goal TEXT NOT NULL, + current_step TEXT, + blocked_task_id TEXT, + blocked_summary TEXT, + state_json TEXT, + wait_json TEXT, + cancel_requested_at INTEGER, + created_at INTEGER NOT NULL, + updated_at INTEGER NOT NULL, + ended_at INTEGER +); + +CREATE INDEX IF NOT EXISTS idx_flow_runs_status ON flow_runs(status); +CREATE INDEX IF NOT EXISTS idx_flow_runs_owner_key ON flow_runs(owner_key); +CREATE INDEX IF NOT EXISTS idx_flow_runs_updated_at ON flow_runs(updated_at); + +CREATE TABLE IF NOT EXISTS migration_runs ( + id TEXT NOT NULL PRIMARY KEY, + started_at INTEGER NOT NULL, + finished_at INTEGER, + status TEXT NOT NULL, + report_json TEXT NOT NULL +); + +CREATE INDEX IF NOT EXISTS idx_migration_runs_started + ON migration_runs(started_at DESC, id); + +CREATE TABLE IF NOT EXISTS migration_sources ( + source_key TEXT NOT NULL PRIMARY KEY, + migration_kind TEXT NOT NULL, + source_path TEXT NOT NULL, + target_table TEXT NOT NULL, + source_sha256 TEXT, + source_size_bytes INTEGER, + source_record_count INTEGER, + last_run_id TEXT NOT NULL, + status TEXT NOT NULL, + imported_at INTEGER NOT NULL, + removed_source INTEGER NOT NULL DEFAULT 0, + report_json TEXT NOT NULL, + FOREIGN KEY (last_run_id) REFERENCES migration_runs(id) ON DELETE CASCADE +); + +CREATE INDEX IF NOT EXISTS idx_migration_sources_path + ON migration_sources(source_path, migration_kind, target_table); + +CREATE INDEX IF NOT EXISTS idx_migration_sources_run + ON migration_sources(last_run_id, source_path); + +CREATE TABLE IF NOT EXISTS backup_runs ( + id TEXT NOT NULL PRIMARY KEY, + created_at INTEGER NOT NULL, + archive_path TEXT NOT NULL, + status TEXT NOT NULL, + manifest_json TEXT NOT NULL +); + +CREATE INDEX IF NOT EXISTS idx_backup_runs_created + ON backup_runs(created_at DESC, id); diff --git a/src/state/sqlite-schema-shape.test-support.ts b/src/state/sqlite-schema-shape.test-support.ts new file mode 100644 index 000000000000..d3fa6c29a8b0 --- /dev/null +++ b/src/state/sqlite-schema-shape.test-support.ts @@ -0,0 +1,106 @@ +import { readFileSync } from "node:fs"; +import { DatabaseSync } from "node:sqlite"; + +type ColumnShape = { + name: string; + type: string; + notnull: number; + dflt_value: unknown; + pk: number; +}; + +type IndexShape = { + name: string; + unique: number; + origin: string; + partial: number; +}; + +export type SqliteSchemaShape = Record< + string, + { + columns: ColumnShape[]; + indexes: IndexShape[]; + } +>; + +type TableInfoRow = ColumnShape & { + cid: number; +}; + +type IndexListRow = IndexShape & { + seq: number; +}; + +type SqliteMasterRow = { + name: string; +}; + +export function createSqliteSchemaShapeFromSql(schemaUrl: URL): SqliteSchemaShape { + const db = new DatabaseSync(":memory:"); + try { + db.exec(readFileSync(schemaUrl, "utf8")); + return collectSqliteSchemaShape(db); + } finally { + db.close(); + } +} + +export function collectSqliteSchemaShape(db: DatabaseSync): SqliteSchemaShape { + const tableRows = db + .prepare( + ` + SELECT name + FROM sqlite_master + WHERE type = 'table' + AND name NOT LIKE 'sqlite_%' + ORDER BY name ASC + `, + ) + .all() as SqliteMasterRow[]; + + return Object.fromEntries( + tableRows.map((table) => [ + table.name, + { + columns: collectColumns(db, table.name), + indexes: collectIndexes(db, table.name), + }, + ]), + ); +} + +function collectColumns(db: DatabaseSync, tableName: string): ColumnShape[] { + return ( + db.prepare(`PRAGMA table_info(${quoteSqliteIdentifier(tableName)})`).all() as TableInfoRow[] + ) + .map(({ name, type, notnull, dflt_value, pk }) => ({ + name, + type, + notnull, + dflt_value, + pk, + })) + .toSorted((left, right) => left.name.localeCompare(right.name)); +} + +function collectIndexes(db: DatabaseSync, tableName: string): IndexShape[] { + return ( + db.prepare(`PRAGMA index_list(${quoteSqliteIdentifier(tableName)})`).all() as IndexListRow[] + ) + .map(({ name, unique, origin, partial }) => ({ + name: normalizeAutoIndexName(name), + unique, + origin, + partial, + })) + .toSorted((left, right) => left.name.localeCompare(right.name)); +} + +function normalizeAutoIndexName(name: string): string { + return name.startsWith("sqlite_autoindex_") ? "sqlite_autoindex" : name; +} + +function quoteSqliteIdentifier(identifier: string): string { + return `"${identifier.replaceAll('"', '""')}"`; +}