From 5443baa8527eec4aa4d9fdb285781013c90e4a71 Mon Sep 17 00:00:00 2001 From: Peter Steinberger Date: Sun, 31 May 2026 20:51:33 -0400 Subject: [PATCH] Persist plugin install index in SQLite (#88794) * refactor: persist plugin install index in sqlite * fix: merge legacy plugin index records into sqlite * test: update plugin index sqlite fixtures * fix: migrate custom plugin install indexes * test: update plugin index sentinel * fix: exclude migrated plugin index archives * fix: read post-upgrade plugin index from sqlite * fix: migrate legacy plugin index before agent runs * fix: respect disabled persisted plugin registry reads * test: type plugin install record fixtures * fix: simplify plugin index record reader type * test: fix sqlite plugin index CI fallout * test: mock provider normalization in agent command tests # Conflicts: # src/commands/agent-command.test-mocks.ts * build: remove unused ui three dependency --- docs/cli/plugins.md | 2 +- docs/plugins/architecture-internals.md | 4 +- pnpm-lock.yaml | 53 ---- scripts/check-kysely-guardrails.mjs | 2 + .../probe.mjs | 9 +- scripts/e2e/lib/codex-install-utils.mjs | 5 +- .../lib/kitchen-sink-plugin/assertions.mjs | 13 +- .../e2e/lib/live-plugin-tool/assertions.mjs | 9 +- scripts/e2e/lib/plugin-index-sqlite.mjs | 160 +++++++++++ .../e2e/lib/plugin-lifecycle-matrix/probe.mjs | 4 +- scripts/e2e/lib/plugin-update/probe.mjs | 9 +- scripts/e2e/lib/plugins/assertions.mjs | 56 ++-- .../release-plugin-marketplace/scenario.sh | 3 +- .../e2e/lib/release-scenarios/assertions.mjs | 7 +- .../lib/release-user-journey/assertions.mjs | 5 +- .../e2e/lib/upgrade-survivor/assertions.mjs | 7 +- scripts/lib/live-docker-stage.sh | 38 ++- .../agent-command.compaction-rotation.test.ts | 8 +- .../agent-command.live-model-switch.test.ts | 1 + src/cli/argv.test.ts | 11 +- src/cli/argv.ts | 6 - src/cli/program/config-guard.test.ts | 19 ++ src/cli/program/config-guard.ts | 10 +- src/commands/agent-command.test-mocks.ts | 1 + src/commands/doctor-post-upgrade.test.ts | 62 ++++ src/commands/doctor-post-upgrade.ts | 19 +- src/commands/doctor-state-migrations.test.ts | 169 +++++++++++ src/commands/doctor.e2e-harness.ts | 4 + src/commands/doctor.ts | 4 +- .../doctor/shared/deprecation-compat.ts | 2 +- .../shared/plugin-registry-migration.test.ts | 29 +- src/config/config.plugin-validation.test.ts | 51 ++-- src/config/io.ts | 106 +------ src/config/io.write-config.test.ts | 30 +- src/infra/state-migrations.state-dir.test.ts | 73 +++++ src/infra/state-migrations.ts | 267 +++++++++++++++++- src/plugins/compat/registry.ts | 2 +- .../installed-plugin-index-record-cache.ts | 30 ++ .../installed-plugin-index-record-reader.ts | 108 +++++-- .../installed-plugin-index-records.test.ts | 165 ++++------- .../installed-plugin-index-store-path.ts | 20 +- .../installed-plugin-index-store.test.ts | 69 +++-- src/plugins/installed-plugin-index-store.ts | 232 +++++++++++++-- src/plugins/manifest-metadata-scan.test.ts | 34 ++- src/plugins/manifest-metadata-scan.ts | 16 +- .../manifest-model-id-normalization.test.ts | 85 ++++-- .../plugin-metadata-snapshot.memo.test.ts | 180 +++++++----- src/plugins/plugin-metadata-snapshot.ts | 35 +-- src/plugins/plugin-registry-snapshot.ts | 20 +- src/plugins/plugin-registry.test.ts | 39 ++- src/security/audit-plugins-trust.test.ts | 5 +- test/scripts/docker-build-helper.test.ts | 4 +- test/scripts/live-docker-stage.test.ts | 4 + .../live-plugin-tool-assertions.test.ts | 9 + .../release-scenarios-assertions.test.ts | 12 + .../release-user-journey-assertions.test.ts | 9 + ui/package.json | 4 +- 57 files changed, 1711 insertions(+), 629 deletions(-) create mode 100644 scripts/e2e/lib/plugin-index-sqlite.mjs create mode 100644 src/plugins/installed-plugin-index-record-cache.ts diff --git a/docs/cli/plugins.md b/docs/cli/plugins.md index ccd5027e6012..caff1e28456f 100644 --- a/docs/cli/plugins.md +++ b/docs/cli/plugins.md @@ -336,7 +336,7 @@ Use `--pin` on npm installs to save the resolved exact spec (`name@version`) in ### Plugin index -Plugin install metadata is machine-managed state, not user config. Installs and updates write it to `plugins/installs.json` under the active OpenClaw state directory. Its top-level `installRecords` map is the durable source of install metadata, including records for broken or missing plugin manifests. The `plugins` array is the manifest-derived cold registry cache. The file includes a do-not-edit warning and is used by `openclaw plugins update`, uninstall, diagnostics, and the cold plugin registry. +Plugin install metadata is machine-managed state, not user config. Installs and updates write it to the shared SQLite state database under the active OpenClaw state directory. The `installed_plugin_index` row stores durable `installRecords` metadata, including records for broken or missing plugin manifests, plus a manifest-derived cold registry cache used by `openclaw plugins update`, uninstall, diagnostics, and the cold plugin registry. When OpenClaw sees shipped legacy `plugins.installs` records in config, runtime reads treat them as compatibility input without rewriting `openclaw.json`. Explicit plugin writes and `openclaw doctor --fix` move those records into the plugin index and remove the config key when config writes are allowed; if either write fails, the config records are kept so the install metadata is not lost. diff --git a/docs/plugins/architecture-internals.md b/docs/plugins/architecture-internals.md index 259d7cc12a5d..dc3c5f3419c8 100644 --- a/docs/plugins/architecture-internals.md +++ b/docs/plugins/architecture-internals.md @@ -1021,10 +1021,10 @@ plugin index entry with `source: "path"` and a workspace-relative `plugins.load.paths`; the install record avoids duplicating local workstation paths into long-lived config. This keeps local development installs visible to source-plane diagnostics without adding a second raw filesystem-path disclosure -surface. The persisted `plugins/installs.json` plugin index is the install +surface. The persisted `installed_plugin_index` SQLite row is the install source of truth and can be refreshed without loading plugin runtime modules. Its `installRecords` map is durable even when a plugin manifest is missing or -invalid; its `plugins` array is a rebuildable manifest view. +invalid; its `plugins` payload is a rebuildable manifest view. ## Context engine plugins diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index f8058ce75846..193a0a2f43d2 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -1919,16 +1919,10 @@ importers: marked: specifier: 18.0.4 version: 18.0.4 - three: - specifier: 0.184.0 - version: 0.184.0 devDependencies: '@types/markdown-it': specifier: 14.1.2 version: 14.1.2 - '@types/three': - specifier: 0.184.1 - version: 0.184.1 '@vitest/browser-playwright': specifier: 4.1.7 version: 4.1.7(playwright@1.60.0)(vite@8.0.14(@types/node@25.9.1)(esbuild@0.28.0)(jiti@2.7.0)(tsx@4.22.3)(yaml@2.9.0))(vitest@4.1.7) @@ -2447,9 +2441,6 @@ packages: '@d-fischer/typed-event-emitter@3.3.3': resolution: {integrity: sha512-OvSEOa8icfdWDqcRtjSEZtgJTFOFNgTjje7zaL0+nAtu2/kZtRCSK5wUMrI/aXtCH8o0Qz2vA8UqkhWUTARFQQ==} - '@dimforge/rapier3d-compat@0.12.0': - resolution: {integrity: sha512-uekIGetywIgopfD97oDL5PfeezkFpNhwlzlaEYNOA0N6ghdsOvh/HYjSMek5Q2O1PYvRSDFcqFVJl4r4ZBwOow==} - '@discordjs/voice@0.19.2': resolution: {integrity: sha512-3yJ255e4ag3wfZu/DSxeOZK1UtnqNxnspmLaQetGT0pDkThNZoHs+Zg6dgZZ19JEVomXygvfHn9lNpICZuYtEA==} engines: {node: '>=22.12.0'} @@ -4009,9 +4000,6 @@ packages: '@tokenizer/token@0.3.0': resolution: {integrity: sha512-OvjF+z51L3ov0OyAU0duzsYuvO01PH7x4t6DJx+guahgTnBHkhJdG7soQeTSFLWN3efnHyibZ4Z8l2EuWwJN3A==} - '@tweenjs/tween.js@23.1.3': - resolution: {integrity: sha512-vJmvvwFxYuGnF2axRtPYocag6Clbb5YS7kLL+SO/TeVFzHqDIWrNKYtcsPMibjDx9O+bu+psAy9NKfWklassUA==} - '@twurple/api-call@8.1.4': resolution: {integrity: sha512-qh2TpdxxyiSkwadcCSes6uBHQB6l4Fz8sVfmzk+Brb12asemHMXTEyQAdrMJT7LlgtZq01nr+RASzWM3jmGtkw==} @@ -4139,12 +4127,6 @@ packages: '@types/serve-static@2.2.0': resolution: {integrity: sha512-8mam4H1NHLtu7nmtalF7eyBH14QyOASmcxHhSfEoRyr0nP/YdoesEtU+uSRvMe96TW/HPTtkoKqQLl53N7UXMQ==} - '@types/stats.js@0.17.4': - resolution: {integrity: sha512-jIBvWWShCvlBqBNIZt0KAshWpvSjhkwkEu4ZUcASoAvhmrgAUI2t1dXrjSL4xXVLB4FznPrIsX3nKXFl/Dt4vA==} - - '@types/three@0.184.1': - resolution: {integrity: sha512-6q4VdiqVsrTRqmk62/BnlcAvIrnDM0zf2ZDVKI5kZiniWrSaOHaQzmbp+BNzoggc/8tgW412pL//wZIxu2PPTA==} - '@types/trusted-types@2.0.7': resolution: {integrity: sha512-ScaPdn1dQczgbl0QFTeTOmVHFULt394XJgOQNoyVhZ6r2vLnMLJfBPd53SB52T/3G36VI1/g2MZaX0cwDuXsfw==} @@ -4154,9 +4136,6 @@ packages: '@types/unist@3.0.3': resolution: {integrity: sha512-ko/gIFJRv177XgZsZcBwnqJN5x/Gien8qNOn0D5bQU/zAzVf9Zt3BlcUiLqhV9y4ARk0GbT3tnUiPNgnTXzc/Q==} - '@types/webxr@0.5.24': - resolution: {integrity: sha512-h8fgEd/DpoS9CBrjEQXR+dIDraopAEfu4wYVNY2tEPwk60stPWhvZMf4Foo5FakuQ7HFZoa8WceaWFervK2Ovg==} - '@types/ws@8.18.1': resolution: {integrity: sha512-ThVF6DCVhA8kUGy+aazFQ4kXQ7E1Ty7A3ypFOe0IcJV8O/M511G99AW24irKrW56Wt44yG9+ij8FaqoBGkuBXg==} @@ -5077,9 +5056,6 @@ packages: resolution: {integrity: sha512-7yAQpD2UMJzLi1Dqv7qFYnPbaPx7ZfFK6PiIxQ4PfkGPyNyl2Ugx+a/umUonmKqjhM4DnfbMvdX6otXq83soQQ==} engines: {node: ^12.20 || >= 14.13} - fflate@0.8.3: - resolution: {integrity: sha512-tbZNuJrLwGUp3zshBtdy4W+ORxZuIh8a5ilyIEQDC5rY1f3U20JMry0Ll3WBzU58EZKsEuJFXhb5gwv8CsPvgA==} - file-type@22.0.1: resolution: {integrity: sha512-ww5Mhre0EE+jmBvOXTmXAbEMuZE7uX4a3+oRCQFNj8w++g3ev913N6tXQz0XTXbueQ5TWQfm6BdaViEHHn8bhA==} engines: {node: '>=22'} @@ -5833,9 +5809,6 @@ packages: resolution: {integrity: sha512-8q7VEgMJW4J8tcfVPy8g09NcQwZdbwFEqhe/WZkoIzjn/3TGDwtOCYtXGxA3O8tPzpczCCDgv+P2P5y00ZJOOg==} engines: {node: '>= 8'} - meshoptimizer@1.1.1: - resolution: {integrity: sha512-oRFNWJRDA/WTrVj7NWvqa5HqE1t9MYDj2VaWirQCzCCrAd2GHrqR/sQezCxiWATPNlKTcRaPRHPJwIRoPBAp5g==} - micromark-core-commonmark@2.0.3: resolution: {integrity: sha512-RDBrHEMSxVFLg6xvnXmb1Ayr2WzLAWjeSATAoxwKYJV94TeNavgoIdA0a9ytzDSVzBy2YKFK+emCPOEibLeCrg==} @@ -6809,9 +6782,6 @@ packages: thread-stream@3.1.0: resolution: {integrity: sha512-OqyPZ9u96VohAyMfJykzmivOrY2wfMSf3C5TtFJVgN+Hm6aj+voFhlK+kZEIv2FBh1X6Xp3DlnCOfEQ3B2J86A==} - three@0.184.0: - resolution: {integrity: sha512-wtTRjG92pM5eUg/KuUnHsqSAlPM296brTOcLgMRqEeylYTh/CdtvKUvCyyCQTzFuStieWxvZb8mVTMvdPyUpxg==} - tinybench@2.9.0: resolution: {integrity: sha512-0+DUvqWMValLmha6lr4kD8iAMK1HzV0/aKnCtWb9v9641TnP/MFb7Pc2bxoxQjTXAErryXVgUOfv2YqNllqGeg==} @@ -8048,8 +8018,6 @@ snapshots: dependencies: tslib: 2.8.1 - '@dimforge/rapier3d-compat@0.12.0': {} - '@discordjs/voice@0.19.2(@emnapi/core@1.10.0)(@emnapi/runtime@1.10.0)': dependencies: '@snazzah/davey': 0.1.11(@emnapi/core@1.10.0)(@emnapi/runtime@1.10.0) @@ -9475,8 +9443,6 @@ snapshots: '@tokenizer/token@0.3.0': {} - '@tweenjs/tween.js@23.1.3': {} - '@twurple/api-call@8.1.4': dependencies: '@d-fischer/shared-utils': 3.6.4 @@ -9649,25 +9615,12 @@ snapshots: '@types/http-errors': 2.0.5 '@types/node': 25.9.1 - '@types/stats.js@0.17.4': {} - - '@types/three@0.184.1': - dependencies: - '@dimforge/rapier3d-compat': 0.12.0 - '@tweenjs/tween.js': 23.1.3 - '@types/stats.js': 0.17.4 - '@types/webxr': 0.5.24 - fflate: 0.8.3 - meshoptimizer: 1.1.1 - '@types/trusted-types@2.0.7': {} '@types/unist@2.0.11': {} '@types/unist@3.0.3': {} - '@types/webxr@0.5.24': {} - '@types/ws@8.18.1': dependencies: '@types/node': 25.9.1 @@ -10609,8 +10562,6 @@ snapshots: node-domexception: '@nolyfill/domexception@1.0.28' web-streams-polyfill: 3.3.3 - fflate@0.8.3: {} - file-type@22.0.1: dependencies: '@tokenizer/inflate': 0.4.1 @@ -11507,8 +11458,6 @@ snapshots: merge2@1.4.1: {} - meshoptimizer@1.1.1: {} - micromark-core-commonmark@2.0.3: dependencies: decode-named-character-reference: 1.3.0 @@ -12783,8 +12732,6 @@ snapshots: dependencies: real-require: 0.2.0 - three@0.184.0: {} - tinybench@2.9.0: {} tinyexec@1.2.2: {} diff --git a/scripts/check-kysely-guardrails.mjs b/scripts/check-kysely-guardrails.mjs index 1a11b0d11606..b0d405866e50 100644 --- a/scripts/check-kysely-guardrails.mjs +++ b/scripts/check-kysely-guardrails.mjs @@ -54,6 +54,8 @@ const rawSqliteAllowPathGroups = { "src/infra/outbound/current-conversation-bindings.ts", "src/media/store.ts", "src/plugin-sdk/memory-core-host-engine-storage.ts", + "src/plugins/installed-plugin-index-record-reader.ts", + "src/plugins/installed-plugin-index-store.ts", "src/plugin-state/plugin-state-store.sqlite.ts", "src/proxy-capture/store.sqlite.ts", "src/tasks/task-flow-registry.store.sqlite.ts", diff --git a/scripts/e2e/lib/bundled-plugin-install-uninstall/probe.mjs b/scripts/e2e/lib/bundled-plugin-install-uninstall/probe.mjs index ec935aefdaf1..f0f763e46d48 100644 --- a/scripts/e2e/lib/bundled-plugin-install-uninstall/probe.mjs +++ b/scripts/e2e/lib/bundled-plugin-install-uninstall/probe.mjs @@ -2,6 +2,7 @@ import { spawnSync } from "node:child_process"; import fs from "node:fs"; import os from "node:os"; import path from "node:path"; +import { readPluginInstallRecords } from "../plugin-index-sqlite.mjs"; const readJson = (file) => JSON.parse(fs.readFileSync(file, "utf8")); const normalizePathForProbe = (value) => String(value ?? "").replace(/\\/g, "/"); @@ -174,10 +175,8 @@ async function selectedManifestEntries() { function assertInstalled(pluginId, pluginDir, requiresConfig) { const stateDir = resolveStateDir(); const configPath = path.join(stateDir, "openclaw.json"); - const indexPath = path.join(stateDir, "plugins", "installs.json"); const config = readJson(configPath); - const index = readJson(indexPath); - const records = index.installRecords ?? index.records ?? {}; + const records = readPluginInstallRecords({ stateDir, configPath }); const record = records[pluginId]; if (!record) { throw new Error(`missing install record for ${pluginId}`); @@ -220,10 +219,8 @@ function assertInstalled(pluginId, pluginDir, requiresConfig) { function assertUninstalled(pluginId, pluginDir) { const stateDir = resolveStateDir(); const configPath = path.join(stateDir, "openclaw.json"); - const indexPath = path.join(stateDir, "plugins", "installs.json"); const config = fs.existsSync(configPath) ? readJson(configPath) : {}; - const index = fs.existsSync(indexPath) ? readJson(indexPath) : {}; - const records = index.installRecords ?? index.records ?? {}; + const records = readPluginInstallRecords({ stateDir, configPath }); if (records[pluginId]) { throw new Error(`install record still present after uninstall for ${pluginId}`); } diff --git a/scripts/e2e/lib/codex-install-utils.mjs b/scripts/e2e/lib/codex-install-utils.mjs index 4540aa90c528..9040fa138c50 100644 --- a/scripts/e2e/lib/codex-install-utils.mjs +++ b/scripts/e2e/lib/codex-install-utils.mjs @@ -1,6 +1,7 @@ import fs from "node:fs"; import path from "node:path"; import { readJson } from "./fixtures/common.mjs"; +import { readPluginInstallRecords } from "./plugin-index-sqlite.mjs"; export { readJson }; @@ -34,9 +35,7 @@ export function assertPathInside(parentPath, childPath, label) { } export function readInstallRecords(fallbackRecords = {}) { - const indexPath = path.join(stateDir(), "plugins", "installs.json"); - const index = fs.existsSync(indexPath) ? readJson(indexPath) : {}; - return index.installRecords || index.records || fallbackRecords || {}; + return readPluginInstallRecords({ fallbackRecords }); } export function npmProjectRootForInstalledPackage(installPath, packageName) { diff --git a/scripts/e2e/lib/kitchen-sink-plugin/assertions.mjs b/scripts/e2e/lib/kitchen-sink-plugin/assertions.mjs index 18bfa87afcc6..6562982e1bbb 100644 --- a/scripts/e2e/lib/kitchen-sink-plugin/assertions.mjs +++ b/scripts/e2e/lib/kitchen-sink-plugin/assertions.mjs @@ -1,6 +1,7 @@ import fs from "node:fs"; import os from "node:os"; import path from "node:path"; +import { readPluginInstallRecords } from "../plugin-index-sqlite.mjs"; const command = process.argv[2]; const scratchRoot = process.env.KITCHEN_SINK_TMP_DIR || os.tmpdir(); @@ -329,9 +330,7 @@ function assertCutoverPreinstalled() { throw new Error(`invalid kitchen-sink cutover preinstall spec: ${preinstallSpec}`); } - const indexPath = path.join(process.env.HOME, ".openclaw", "plugins", "installs.json"); - const index = readJson(indexPath); - const record = (index.installRecords ?? index.records ?? {})[pluginId]; + const record = readPluginInstallRecords()[pluginId]; if (!record) { throw new Error(`missing kitchen-sink cutover preinstall record for ${pluginId}`); } @@ -456,9 +455,7 @@ function assertInstalled() { } assertExpectedDiagnostics(surfaceMode, errorMessages); - const indexPath = path.join(process.env.HOME, ".openclaw", "plugins", "installs.json"); - const index = readJson(indexPath); - const record = (index.installRecords ?? index.records ?? {})[pluginId]; + const record = readPluginInstallRecords()[pluginId]; if (!record) { throw new Error(`missing kitchen-sink install record for ${pluginId}`); } @@ -513,9 +510,7 @@ function assertRemoved() { throw new Error(`kitchen-sink plugin still listed after uninstall: ${pluginId}`); } - const indexPath = path.join(process.env.HOME, ".openclaw", "plugins", "installs.json"); - const index = fs.existsSync(indexPath) ? readJson(indexPath) : {}; - const records = index.installRecords ?? index.records ?? {}; + const records = readPluginInstallRecords(); if (records[pluginId]) { throw new Error(`kitchen-sink install record still present after uninstall: ${pluginId}`); } diff --git a/scripts/e2e/lib/live-plugin-tool/assertions.mjs b/scripts/e2e/lib/live-plugin-tool/assertions.mjs index 338027ee751b..9ba6650f6673 100644 --- a/scripts/e2e/lib/live-plugin-tool/assertions.mjs +++ b/scripts/e2e/lib/live-plugin-tool/assertions.mjs @@ -1,5 +1,6 @@ import fs from "node:fs"; import path from "node:path"; +import { readPluginInstallRecords } from "../plugin-index-sqlite.mjs"; import { readTextFileTail, tailText } from "../text-file-utils.mjs"; const command = process.argv[2]; @@ -153,10 +154,12 @@ function writeJson(file, value) { } function installRecords() { - const indexPath = path.join(stateDir(), "plugins", "installs.json"); - const index = fs.existsSync(indexPath) ? readJson(indexPath) : {}; const cfg = fs.existsSync(configPath()) ? readJson(configPath()) : {}; - return index.installRecords || index.records || cfg.plugins?.installs || {}; + return readPluginInstallRecords({ + stateDir: stateDir(), + configPath: configPath(), + fallbackRecords: cfg.plugins?.installs ?? {}, + }); } function pluginInstallPath() { diff --git a/scripts/e2e/lib/plugin-index-sqlite.mjs b/scripts/e2e/lib/plugin-index-sqlite.mjs new file mode 100644 index 000000000000..47e6a45d2cd2 --- /dev/null +++ b/scripts/e2e/lib/plugin-index-sqlite.mjs @@ -0,0 +1,160 @@ +import fs from "node:fs"; +import path from "node:path"; +import { DatabaseSync } from "node:sqlite"; + +const INDEX_KEY = "installed-plugin-index"; + +export function stateDir() { + return process.env.OPENCLAW_STATE_DIR || path.join(process.env.HOME, ".openclaw"); +} + +export function configPath() { + return process.env.OPENCLAW_CONFIG_PATH || path.join(stateDir(), "openclaw.json"); +} + +function readJsonMaybe(file) { + try { + return JSON.parse(fs.readFileSync(file, "utf8")); + } catch { + return {}; + } +} + +function sqlitePath(root = stateDir()) { + return path.join(root, "state", "openclaw.sqlite"); +} + +function legacyIndexPath(root = stateDir()) { + return path.join(root, "plugins", "installs.json"); +} + +function readSqlitePluginIndex(root = stateDir()) { + const dbPath = sqlitePath(root); + if (!fs.existsSync(dbPath)) { + return {}; + } + let db; + try { + db = new DatabaseSync(dbPath, { readOnly: true }); + const row = db + .prepare( + ` + SELECT version, warning, host_contract_version, compat_registry_version, + migration_version, policy_hash, generated_at_ms, refresh_reason, + install_records_json, plugins_json, diagnostics_json + FROM installed_plugin_index + WHERE index_key = ? + `, + ) + .get(INDEX_KEY); + if (!row) { + return {}; + } + return { + version: Number(row.version), + ...(row.warning ? { warning: row.warning } : {}), + hostContractVersion: row.host_contract_version, + compatRegistryVersion: row.compat_registry_version, + migrationVersion: Number(row.migration_version), + policyHash: row.policy_hash, + generatedAtMs: Number(row.generated_at_ms), + ...(row.refresh_reason ? { refreshReason: row.refresh_reason } : {}), + installRecords: JSON.parse(row.install_records_json), + plugins: JSON.parse(row.plugins_json), + diagnostics: JSON.parse(row.diagnostics_json), + }; + } catch { + return {}; + } finally { + db?.close(); + } +} + +export function readPluginInstallIndex(options = {}) { + const root = options.stateDir ?? stateDir(); + const config = readJsonMaybe(options.configPath ?? configPath()); + const sqliteIndex = readSqlitePluginIndex(root); + if (sqliteIndex.installRecords) { + return sqliteIndex; + } + const legacyIndex = readJsonMaybe(legacyIndexPath(root)); + const installRecords = + legacyIndex.installRecords ?? + legacyIndex.records ?? + options.fallbackRecords ?? + config.plugins?.installs ?? + {}; + return { + ...legacyIndex, + installRecords, + }; +} + +export function readPluginInstallRecords(options = {}) { + return readPluginInstallIndex(options).installRecords ?? {}; +} + +export function writePluginInstallIndexForE2E(index, options = {}) { + const root = options.stateDir ?? stateDir(); + const dbPath = sqlitePath(root); + fs.mkdirSync(path.dirname(dbPath), { recursive: true }); + const db = new DatabaseSync(dbPath); + try { + db.exec(` + CREATE TABLE IF NOT EXISTS installed_plugin_index ( + index_key TEXT NOT NULL PRIMARY KEY, + version INTEGER NOT NULL, + host_contract_version TEXT NOT NULL, + compat_registry_version TEXT NOT NULL, + migration_version INTEGER NOT NULL, + policy_hash TEXT NOT NULL, + generated_at_ms INTEGER NOT NULL, + refresh_reason TEXT, + install_records_json TEXT NOT NULL, + plugins_json TEXT NOT NULL, + diagnostics_json TEXT NOT NULL, + warning TEXT, + updated_at_ms INTEGER NOT NULL + ); + `); + const now = Date.now(); + db.prepare( + ` + INSERT INTO installed_plugin_index ( + index_key, version, host_contract_version, compat_registry_version, + migration_version, policy_hash, generated_at_ms, refresh_reason, + install_records_json, plugins_json, diagnostics_json, warning, updated_at_ms + ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?) + ON CONFLICT(index_key) DO UPDATE SET + version = excluded.version, + host_contract_version = excluded.host_contract_version, + compat_registry_version = excluded.compat_registry_version, + migration_version = excluded.migration_version, + policy_hash = excluded.policy_hash, + generated_at_ms = excluded.generated_at_ms, + refresh_reason = excluded.refresh_reason, + install_records_json = excluded.install_records_json, + plugins_json = excluded.plugins_json, + diagnostics_json = excluded.diagnostics_json, + warning = excluded.warning, + updated_at_ms = excluded.updated_at_ms + `, + ).run( + INDEX_KEY, + index.version ?? 1, + index.hostContractVersion ?? "docker-e2e", + index.compatRegistryVersion ?? "docker-e2e", + index.migrationVersion ?? 1, + index.policyHash ?? "docker-e2e", + index.generatedAtMs ?? now, + index.refreshReason ?? null, + JSON.stringify(index.installRecords ?? {}), + JSON.stringify(index.plugins ?? []), + JSON.stringify(index.diagnostics ?? []), + index.warning ?? "DO NOT EDIT. This row is generated by OpenClaw plugin registry commands.", + now, + ); + } finally { + db.close(); + } +} diff --git a/scripts/e2e/lib/plugin-lifecycle-matrix/probe.mjs b/scripts/e2e/lib/plugin-lifecycle-matrix/probe.mjs index e113a991941a..9ac055e1971d 100644 --- a/scripts/e2e/lib/plugin-lifecycle-matrix/probe.mjs +++ b/scripts/e2e/lib/plugin-lifecycle-matrix/probe.mjs @@ -1,6 +1,7 @@ import fs from "node:fs"; import os from "node:os"; import path from "node:path"; +import { readPluginInstallRecords } from "../plugin-index-sqlite.mjs"; const home = os.homedir(); @@ -26,8 +27,7 @@ function readRequiredJson(file) { } function records() { - const index = readJson(openclawPath("plugins", "installs.json")); - return index.installRecords ?? index.records ?? {}; + return readPluginInstallRecords(); } function recordFor(pluginId) { diff --git a/scripts/e2e/lib/plugin-update/probe.mjs b/scripts/e2e/lib/plugin-update/probe.mjs index 4433ef6871ce..7e9c8461b82c 100644 --- a/scripts/e2e/lib/plugin-update/probe.mjs +++ b/scripts/e2e/lib/plugin-update/probe.mjs @@ -3,6 +3,10 @@ import http from "node:http"; import os from "node:os"; import path from "node:path"; import { legacyPackageAcceptanceCompat } from "../package-compat.mjs"; +import { + readPluginInstallRecords, + writePluginInstallIndexForE2E, +} from "../plugin-index-sqlite.mjs"; const home = os.homedir(); @@ -16,8 +20,7 @@ const readJson = (file) => { const pluginRecordSnapshot = () => { const config = readJson(openclawPath("openclaw.json")); - const index = readJson(openclawPath("plugins", "installs.json")); - const records = index.installRecords ?? index.records ?? config.plugins?.installs ?? {}; + const records = readPluginInstallRecords({ fallbackRecords: config.plugins?.installs ?? {} }); const record = records["lossless-claw"] ?? records["@example/lossless-claw"]; if (!record) { throw new Error("missing plugin install record"); @@ -41,7 +44,7 @@ function seedInstallState() { version: "0.9.0", }); writeJson(process.env.OPENCLAW_CONFIG_PATH, { plugins: {} }); - writeJson(openclawPath("plugins", "installs.json"), { + writePluginInstallIndexForE2E({ version: 1, warning: "DO NOT EDIT. This file is generated by OpenClaw plugin registry commands.", hostContractVersion: "docker-e2e", diff --git a/scripts/e2e/lib/plugins/assertions.mjs b/scripts/e2e/lib/plugins/assertions.mjs index 05df10551c7b..b75136b522c5 100644 --- a/scripts/e2e/lib/plugins/assertions.mjs +++ b/scripts/e2e/lib/plugins/assertions.mjs @@ -2,6 +2,11 @@ import fs from "node:fs"; import os from "node:os"; import path from "node:path"; import { readPositiveIntEnv } from "../env-limits.mjs"; +import { + readPluginInstallIndex, + readPluginInstallRecords, + writePluginInstallIndexForE2E, +} from "../plugin-index-sqlite.mjs"; const command = process.argv[2]; const scratchRoot = process.env.OPENCLAW_PLUGINS_TMP_DIR || os.tmpdir(); @@ -14,10 +19,7 @@ function readClawHubPreflightLimits() { "OPENCLAW_PLUGINS_E2E_CLAWHUB_PREFLIGHT_BODY_MAX_BYTES", 1024 * 1024, ), - timeoutMs: readPositiveIntEnv( - "OPENCLAW_PLUGINS_E2E_CLAWHUB_PREFLIGHT_TIMEOUT_MS", - 30_000, - ), + timeoutMs: readPositiveIntEnv("OPENCLAW_PLUGINS_E2E_CLAWHUB_PREFLIGHT_TIMEOUT_MS", 30_000), }; } @@ -112,17 +114,17 @@ function pathsEqual(left, right) { } function getInstallRecords() { - const indexPath = path.join(process.env.HOME, ".openclaw", "plugins", "installs.json"); - const index = fs.existsSync(indexPath) ? readJson(indexPath) : {}; const configPath = path.join(process.env.HOME, ".openclaw", "openclaw.json"); const config = fs.existsSync(configPath) ? readJson(configPath) : {}; const allowLegacyCompat = process.env.OPENCLAW_PACKAGE_ACCEPTANCE_LEGACY_COMPAT === "1"; + const index = readPluginInstallIndex({ + configPath, + fallbackRecords: allowLegacyCompat ? (config.plugins?.installs ?? {}) : {}, + }); if (!allowLegacyCompat && !index.installRecords) { throw new Error("expected modern installRecords in installed plugin index"); } - return allowLegacyCompat - ? (index.installRecords ?? index.records ?? config.plugins?.installs ?? {}) - : (index.installRecords ?? {}); + return index.installRecords ?? {}; } function readOpenClawConfig() { @@ -214,25 +216,14 @@ function recordFixturePluginTrust() { fs.mkdirSync(path.dirname(configPath), { recursive: true }); fs.writeFileSync(configPath, `${JSON.stringify(config, null, 2)}\n`, "utf8"); - const ledgerPath = path.join(process.env.HOME, ".openclaw", "plugins", "installs.json"); - const ledger = fs.existsSync(ledgerPath) - ? readJson(ledgerPath) - : { - version: 1, - warning: - "DO NOT EDIT. This file is generated by OpenClaw plugin install/update/uninstall commands. Use `openclaw plugins install/update/uninstall` instead.", - records: {}, - }; - ledger.updatedAtMs = Date.now(); - ledger.records ??= {}; - ledger.records[pluginId] = { - ...ledger.records[pluginId], + const installRecords = readPluginInstallRecords(); + installRecords[pluginId] = { + ...installRecords[pluginId], source: "path", installPath: pluginRoot, sourcePath: pluginRoot, }; - fs.mkdirSync(path.dirname(ledgerPath), { recursive: true }); - fs.writeFileSync(ledgerPath, `${JSON.stringify(ledger, null, 2)}\n`, "utf8"); + writePluginInstallIndexForE2E({ installRecords }); } function assertDemoPlugin() { @@ -908,17 +899,17 @@ function assertClawHubInstalled() { throw new Error(`unexpected ClawHub inspect plugin id: ${inspect.plugin?.id}`); } - const indexPath = path.join(process.env.HOME, ".openclaw", "plugins", "installs.json"); - const index = readJson(indexPath); const configPath = path.join(process.env.HOME, ".openclaw", "openclaw.json"); const config = fs.existsSync(configPath) ? readJson(configPath) : {}; const allowLegacyCompat = process.env.OPENCLAW_PACKAGE_ACCEPTANCE_LEGACY_COMPAT === "1"; + const index = readPluginInstallIndex({ + configPath, + fallbackRecords: allowLegacyCompat ? (config.plugins?.installs ?? {}) : {}, + }); if (!allowLegacyCompat && !index.installRecords) { throw new Error("expected modern installRecords in installed plugin index"); } - const installRecords = allowLegacyCompat - ? (index.installRecords ?? index.records ?? config.plugins?.installs ?? {}) - : (index.installRecords ?? {}); + const installRecords = index.installRecords ?? {}; const record = installRecords[pluginId]; if (!record) { throw new Error(`missing ClawHub install record for ${pluginId}`); @@ -963,11 +954,12 @@ function assertClawHubRemoved() { throw new Error(`ClawHub plugin still listed after uninstall: ${pluginId}`); } - const indexPath = path.join(process.env.HOME, ".openclaw", "plugins", "installs.json"); - const index = fs.existsSync(indexPath) ? readJson(indexPath) : {}; const configPath = path.join(process.env.HOME, ".openclaw", "openclaw.json"); const config = fs.existsSync(configPath) ? readJson(configPath) : {}; - const installRecords = index.installRecords ?? index.records ?? config.plugins?.installs ?? {}; + const installRecords = readPluginInstallRecords({ + configPath, + fallbackRecords: config.plugins?.installs ?? {}, + }); if (installRecords[pluginId]) { throw new Error(`ClawHub install record still present after uninstall: ${pluginId}`); } diff --git a/scripts/e2e/lib/release-plugin-marketplace/scenario.sh b/scripts/e2e/lib/release-plugin-marketplace/scenario.sh index 4729d208d767..6555ebe2d8f6 100755 --- a/scripts/e2e/lib/release-plugin-marketplace/scenario.sh +++ b/scripts/e2e/lib/release-plugin-marketplace/scenario.sh @@ -28,8 +28,7 @@ dump_debug_logs() { /tmp/openclaw-release-plugin-marketplace-update.log \ /tmp/openclaw-release-plugin-marketplace-cli-v2.log \ /tmp/openclaw-release-plugin-marketplace-uninstall.log \ - /tmp/openclaw-release-plugin-marketplace-cli-after-uninstall.log \ - "$HOME/.openclaw/plugins/installs.json" + /tmp/openclaw-release-plugin-marketplace-cli-after-uninstall.log } trap 'status=$?; dump_debug_logs "$status"; exit "$status"' ERR diff --git a/scripts/e2e/lib/release-scenarios/assertions.mjs b/scripts/e2e/lib/release-scenarios/assertions.mjs index 393ef864c33c..be98e3fecbeb 100644 --- a/scripts/e2e/lib/release-scenarios/assertions.mjs +++ b/scripts/e2e/lib/release-scenarios/assertions.mjs @@ -5,6 +5,7 @@ import { assertOpenAiRequestLogUsed, } from "../agent-turn-output.mjs"; import { applyMockOpenAiModelConfig } from "../fixtures/mock-openai-config.mjs"; +import { readPluginInstallRecords } from "../plugin-index-sqlite.mjs"; import { readTextFileTail } from "../text-file-utils.mjs"; const command = process.argv[2]; @@ -174,9 +175,7 @@ function assertPluginUninstalled() { const pluginId = process.argv[3]; const cliRoot = process.argv[4]; const cfg = readJson(configPath()); - const recordsPath = path.join(process.env.HOME ?? "", ".openclaw", "plugins", "installs.json"); - const records = fs.existsSync(recordsPath) ? readJson(recordsPath) : {}; - const installRecords = records.installRecords ?? records.records ?? {}; + const installRecords = readPluginInstallRecords({ configPath: configPath() }); assert(!installRecords[pluginId], `install record still present for ${pluginId}`); assert(!cfg.plugins?.entries?.[pluginId], `plugin config entry still present for ${pluginId}`); const managedRoot = path.join( @@ -188,7 +187,7 @@ function assertPluginUninstalled() { ); assert(!fs.existsSync(managedRoot), `managed plugin directory still present: ${managedRoot}`); if (cliRoot) { - const list = JSON.stringify(records); + const list = JSON.stringify(installRecords); assert(!list.includes(cliRoot), `install records still mention CLI root ${cliRoot}`); } } diff --git a/scripts/e2e/lib/release-user-journey/assertions.mjs b/scripts/e2e/lib/release-user-journey/assertions.mjs index f51c114cf7b8..1c75648890e5 100644 --- a/scripts/e2e/lib/release-user-journey/assertions.mjs +++ b/scripts/e2e/lib/release-user-journey/assertions.mjs @@ -7,6 +7,7 @@ import { } from "../agent-turn-output.mjs"; import { readBoundedResponseText as readBoundedResponseTextWithLimit } from "../bounded-response-text.mjs"; import { applyMockOpenAiModelConfig } from "../fixtures/mock-openai-config.mjs"; +import { readPluginInstallRecords } from "../plugin-index-sqlite.mjs"; function clickClackHttpTimeoutMs() { return readPositiveInt(process.env.OPENCLAW_RELEASE_USER_JOURNEY_HTTP_TIMEOUT_MS, 5000); @@ -98,9 +99,7 @@ function writeConfig(cfg) { } function installRecords() { - const recordsPath = path.join(process.env.HOME ?? "", ".openclaw", "plugins", "installs.json"); - const records = fs.existsSync(recordsPath) ? readJson(recordsPath) : {}; - return records.installRecords ?? records.records ?? {}; + return readPluginInstallRecords({ configPath: configPath() }); } function assertOnboard() { diff --git a/scripts/e2e/lib/upgrade-survivor/assertions.mjs b/scripts/e2e/lib/upgrade-survivor/assertions.mjs index 0708d0180e0d..33d5155ba32f 100644 --- a/scripts/e2e/lib/upgrade-survivor/assertions.mjs +++ b/scripts/e2e/lib/upgrade-survivor/assertions.mjs @@ -1,5 +1,6 @@ import fs from "node:fs"; import path from "node:path"; +import { readPluginInstallIndex } from "../plugin-index-sqlite.mjs"; const command = process.argv[2]; const SCENARIOS = new Set([ @@ -406,9 +407,9 @@ function assertStateSurvived() { function readInstalledPluginIndex() { const stateDir = requireEnv("OPENCLAW_STATE_DIR"); - const file = path.join(stateDir, "plugins", "installs.json"); - assert(fs.existsSync(file), `installed plugin index missing: ${file}`); - return readJson(file); + const index = readPluginInstallIndex({ stateDir }); + assert(index.installRecords, "installed plugin index missing"); + return index; } function assertExternalPluginInstall(records, pluginId, packageName) { diff --git a/scripts/lib/live-docker-stage.sh b/scripts/lib/live-docker-stage.sh index 7474389750a7..1d6f44620542 100644 --- a/scripts/lib/live-docker-stage.sh +++ b/scripts/lib/live-docker-stage.sh @@ -63,6 +63,36 @@ openclaw_live_stage_node_modules() { mkdir -p "$target_dir/.vite-temp" } +openclaw_live_scrub_staged_plugin_index() { + local dest_dir="${1:?destination directory required}" + local db_path="$dest_dir/state/openclaw.sqlite" + + if [ ! -f "$db_path" ]; then + return 0 + fi + + node - "$db_path" <<'NODE' +const dbPath = process.argv[2]; +let db; +try { + const { DatabaseSync } = await import("node:sqlite"); + db = new DatabaseSync(dbPath); + try { + db.exec("PRAGMA secure_delete = ON;"); + db.prepare("DELETE FROM installed_plugin_index WHERE index_key = ?").run("installed-plugin-index"); + db.exec("PRAGMA wal_checkpoint(TRUNCATE);"); + db.exec("VACUUM;"); + } catch (err) { + if (!String(err?.message ?? err).includes("no such table")) { + throw err; + } + } +} finally { + db?.close(); +} +NODE +} + openclaw_live_stage_state_dir() { local dest_dir="${1:?destination directory required}" local source_dir="${HOME}/.openclaw" @@ -70,9 +100,9 @@ openclaw_live_stage_state_dir() { mkdir -p "$dest_dir" if [ -d "$source_dir" ]; then # Sandbox workspaces can accumulate root-owned artifacts from prior Docker - # runs. The persisted plugin registry contains host-absolute paths that are - # not portable into Linux containers. Neither is needed for live-test - # auth/config staging, so keep them out of the staged state copy. + # runs. Persisted plugin registry state contains host-absolute paths that + # are not portable into Linux containers. Live-test auth/config staging does + # not need the old JSON source or the SQLite installed_plugin_index row. set +e tar -C "$source_dir" \ --warning=no-file-changed \ @@ -80,6 +110,7 @@ openclaw_live_stage_state_dir() { --exclude=workspace \ --exclude=sandboxes \ --exclude=plugins/installs.json \ + --exclude=plugins/installs.json.migrated \ --exclude=relay.sock \ --exclude='*.sock' \ --exclude='*/*.sock' \ @@ -90,6 +121,7 @@ openclaw_live_stage_state_dir() { return "$status" fi chmod -R u+rwX "$dest_dir" || true + openclaw_live_scrub_staged_plugin_index "$dest_dir" if [ -d "$source_dir/workspace" ] && [ ! -e "$dest_dir/workspace" ]; then ln -s "$source_dir/workspace" "$dest_dir/workspace" fi diff --git a/src/agents/agent-command.compaction-rotation.test.ts b/src/agents/agent-command.compaction-rotation.test.ts index 47de6b46368e..eafaeea2c5e7 100644 --- a/src/agents/agent-command.compaction-rotation.test.ts +++ b/src/agents/agent-command.compaction-rotation.test.ts @@ -5,16 +5,18 @@ import { afterEach, beforeAll, beforeEach, describe, expect, it, vi } from "vite import { loadSessionStore, saveSessionStore, type SessionEntry } from "../config/sessions.js"; import { CURRENT_SESSION_VERSION } from "../config/sessions/version.js"; import type { OpenClawConfig } from "../config/types.openclaw.js"; +import type { EmbeddedAgentRunResult } from "./embedded-agent.js"; import type { loadManifestModelCatalog } from "./model-catalog.js"; type ProviderModelNormalizationParams = { provider: string; context: { modelId: string } }; type LoadManifestModelCatalogParams = Parameters[0]; +type RunAgentAttempt = typeof import("./command/attempt-execution.runtime.js").runAgentAttempt; const state = vi.hoisted(() => ({ cfg: undefined as OpenClawConfig | undefined, workspaceDir: undefined as string | undefined, agentDir: undefined as string | undefined, - runAgentAttemptMock: vi.fn(), + runAgentAttemptMock: vi.fn(), loadManifestModelCatalogMock: vi.fn((_params: LoadManifestModelCatalogParams) => []), normalizeProviderModelIdWithRuntimeMock: vi.fn( (_params: ProviderModelNormalizationParams) => undefined, @@ -135,7 +137,7 @@ vi.mock("./command/attempt-execution.runtime.js", async () => { ); return { ...actual, - runAgentAttempt: (...args: unknown[]) => state.runAgentAttemptMock(...args), + runAgentAttempt: (params: Parameters[0]) => state.runAgentAttemptMock(params), }; }); @@ -198,7 +200,7 @@ function makeResult(params: { sessionFile?: string; text: string; compactionCount?: number; -}) { +}): EmbeddedAgentRunResult { return { payloads: [{ text: params.text }], meta: { diff --git a/src/agents/agent-command.live-model-switch.test.ts b/src/agents/agent-command.live-model-switch.test.ts index 817da85791e6..888518579c1b 100644 --- a/src/agents/agent-command.live-model-switch.test.ts +++ b/src/agents/agent-command.live-model-switch.test.ts @@ -539,6 +539,7 @@ vi.mock("./model-selection.js", () => { modelKey: (p: string, m: string) => `${p}/${m}`, normalizeModelRef: (p: string, m: string) => ({ provider: p, model: m }), normalizeProviderId, + normalizeProviderIdForAuth: normalizeProviderId, parseModelRef: (m: string, p: string) => ({ provider: p, model: m }), resolveConfiguredModelRef: ({ cfg }: { cfg?: unknown }) => { const raw = (cfg as { agents?: { defaults?: { model?: string | { primary?: string } } } }) diff --git a/src/cli/argv.test.ts b/src/cli/argv.test.ts index bf73a5dd258b..b582b0e68fef 100644 --- a/src/cli/argv.test.ts +++ b/src/cli/argv.test.ts @@ -586,13 +586,13 @@ describe("argv helpers", () => { { argv: ["node", "openclaw", "health"], expected: false }, { argv: ["node", "openclaw", "sessions"], expected: false }, { argv: ["node", "openclaw", "--profile", "work", "status"], expected: true }, - { argv: ["node", "openclaw", "--log-level=debug", "models", "list"], expected: false }, + { argv: ["node", "openclaw", "--log-level=debug", "models", "list"], expected: true }, { argv: ["node", "openclaw", "config", "get", "update"], expected: false }, { argv: ["node", "openclaw", "config", "unset", "update"], expected: false }, - { argv: ["node", "openclaw", "models", "list"], expected: false }, - { argv: ["node", "openclaw", "models", "status"], expected: false }, + { argv: ["node", "openclaw", "models", "list"], expected: true }, + { argv: ["node", "openclaw", "models", "status"], expected: true }, { argv: ["node", "openclaw", "update", "status", "--json"], expected: false }, - { argv: ["node", "openclaw", "agent", "--message", "hi"], expected: false }, + { argv: ["node", "openclaw", "agent", "--message", "hi"], expected: true }, { argv: ["node", "openclaw", "agents", "list"], expected: true }, { argv: ["node", "openclaw", "message", "send"], expected: true }, ] as const)("decides when to migrate state: $argv", ({ argv, expected }) => { @@ -603,7 +603,8 @@ describe("argv helpers", () => { { path: ["status"], expected: true }, { path: ["update", "status"], expected: false }, { path: ["config", "get"], expected: false }, - { path: ["models", "status"], expected: false }, + { path: ["agent"], expected: true }, + { path: ["models", "status"], expected: true }, { path: ["agents", "list"], expected: true }, ])("reuses command path for migrate state decisions: $path", ({ path, expected }) => { expect(shouldMigrateStateFromPath(path)).toBe(expected); diff --git a/src/cli/argv.ts b/src/cli/argv.ts index 7241d424bb35..57bb50657efa 100644 --- a/src/cli/argv.ts +++ b/src/cli/argv.ts @@ -461,12 +461,6 @@ export function shouldMigrateStateFromPath(path: string[]): boolean { if (primary === "config" && (secondary === "get" || secondary === "unset")) { return false; } - if (primary === "models" && (secondary === "list" || secondary === "status")) { - return false; - } - if (primary === "agent") { - return false; - } return true; } diff --git a/src/cli/program/config-guard.test.ts b/src/cli/program/config-guard.test.ts index 095b8949d959..5fac6991ea09 100644 --- a/src/cli/program/config-guard.test.ts +++ b/src/cli/program/config-guard.test.ts @@ -168,6 +168,11 @@ describe("ensureConfigReady", () => { commandPath: ["update", "status"], expectedDoctorCalls: 0, }, + { + name: "skips doctor flow for agent without legacy state", + commandPath: ["agent"], + expectedDoctorCalls: 0, + }, { name: "runs doctor flow for commands that may mutate state without legacy state", commandPath: ["message"], @@ -207,6 +212,20 @@ describe("ensureConfigReady", () => { expect(loadAndMaybeMigrateDoctorConfigMock).toHaveBeenCalledOnce(); }); + it("runs doctor flow before agent commands when the legacy plugin install index exists", async () => { + const root = useTempOpenClawHome(); + writeStateMarker(root, "plugins/installs.json"); + + await runEnsureConfigReady(["agent"]); + + expect(loadAndMaybeMigrateDoctorConfigMock).toHaveBeenCalledOnce(); + expect(loadAndMaybeMigrateDoctorConfigMock).toHaveBeenCalledWith({ + migrateState: true, + migrateLegacyConfig: false, + invalidConfigNote: false, + }); + }); + it.each([ ["Discord model picker preferences", "discord/model-picker-preferences.json"], ["Feishu dedupe sidecar", "feishu/dedup/default.json"], diff --git a/src/cli/program/config-guard.ts b/src/cli/program/config-guard.ts index 9bc1875004ae..c74a3f596303 100644 --- a/src/cli/program/config-guard.ts +++ b/src/cli/program/config-guard.ts @@ -110,16 +110,18 @@ function hasLegacyStateMigrationInputs(): boolean { path.join(stateDir, "agents"), path.join(stateDir, "flows", "registry.sqlite"), path.join(stateDir, "plugin-state", "state.sqlite"), + path.join(stateDir, "plugins", "installs.json"), path.join(stateDir, "sessions"), path.join(stateDir, "tasks", "runs.sqlite"), ].some(fileOrDirExists) || hasBundledChannelLegacyStateMigrationInputs(stateDir, oauthDir) ); } -function isReadOnlyStateMigrationCommand(commandPath: string[]): boolean { +function shouldRunStateMigrationOnlyWithLegacyInputs(commandPath: string[]): boolean { const commandName = commandPath[0]; const subcommandName = commandPath[1]; return ( + commandName === "agent" || commandName === "status" || (commandName === "tasks" && (subcommandName === undefined || ALLOWED_INVALID_TASK_SUBCOMMANDS.has(subcommandName))) @@ -160,7 +162,7 @@ export async function ensureConfigReady(params: { const commandPath = params.commandPath ?? []; let preflightSnapshot: Awaited> | null = null; const shouldConsiderStateMigration = shouldMigrateStateFromPath(commandPath); - const isReadOnlyMigrationCommand = isReadOnlyStateMigrationCommand(commandPath); + const requiresLegacyStateInput = shouldRunStateMigrationOnlyWithLegacyInputs(commandPath); const runStateMigrationPreflight = async () => { didRunDoctorConfigFlow = true; const runDoctorConfigPreflight = async () => @@ -176,7 +178,7 @@ export async function ensureConfigReady(params: { if ( !didRunDoctorConfigFlow && shouldConsiderStateMigration && - (!isReadOnlyMigrationCommand || hasLegacyStateMigrationInputs()) + (!requiresLegacyStateInput || hasLegacyStateMigrationInputs()) ) { preflightSnapshot = await runStateMigrationPreflight(); } @@ -186,7 +188,7 @@ export async function ensureConfigReady(params: { !preflightSnapshot && !didRunDoctorConfigFlow && shouldConsiderStateMigration && - isReadOnlyMigrationCommand && + requiresLegacyStateInput && snapshot.valid && snapshotHasConfiguredSessionStore(snapshot) ) { diff --git a/src/commands/agent-command.test-mocks.ts b/src/commands/agent-command.test-mocks.ts index ce22cdfec7ba..85ce6da13a21 100644 --- a/src/commands/agent-command.test-mocks.ts +++ b/src/commands/agent-command.test-mocks.ts @@ -186,6 +186,7 @@ vi.mock("../agents/model-selection.js", () => { modelKey, normalizeModelRef, normalizeProviderId, + normalizeProviderIdForAuth: normalizeProviderId, parseModelRef, resolveConfiguredModelRef: vi.fn( ({ cfg }: { cfg?: ConfigWithModels; defaultProvider?: string; defaultModel?: string }) => diff --git a/src/commands/doctor-post-upgrade.test.ts b/src/commands/doctor-post-upgrade.test.ts index bd351f437301..5bd699d1c9b6 100644 --- a/src/commands/doctor-post-upgrade.test.ts +++ b/src/commands/doctor-post-upgrade.test.ts @@ -3,6 +3,8 @@ import fs from "node:fs/promises"; import os from "node:os"; import path from "node:path"; import { describe, expect, it } from "vitest"; +import { writePersistedInstalledPluginIndex } from "../plugins/installed-plugin-index-store.js"; +import type { InstalledPluginIndex } from "../plugins/installed-plugin-index.js"; import { runPostUpgradeProbes } from "./doctor-post-upgrade.js"; async function makeFixtureRoot(prefix: string): Promise { @@ -72,6 +74,66 @@ describe("runPostUpgradeProbes — plugin.index_unavailable", () => { }); describe("runPostUpgradeProbes — plugin.entry_unresolved", () => { + it("reads the canonical SQLite plugin index by default", async () => { + const root = await makeFixtureRoot("entry-sqlite"); + try { + const pluginDir = path.join(root, "user-plugins", "sqlite-ghost"); + await fs.mkdir(pluginDir, { recursive: true }); + await fs.writeFile( + path.join(pluginDir, "package.json"), + JSON.stringify({ + name: "sqlite-ghost", + version: "0.0.1", + type: "module", + openclaw: { extensions: ["./dist/index.js"] }, + }), + "utf-8", + ); + const manifestPath = path.join(pluginDir, "openclaw.plugin.json"); + await fs.writeFile(manifestPath, JSON.stringify({ id: "sqlite-ghost" }), "utf-8"); + const index: InstalledPluginIndex = { + version: 1, + hostContractVersion: "test-host", + compatRegistryVersion: "test-compat", + migrationVersion: 1, + policyHash: "test-policy", + generatedAtMs: 1, + installRecords: {}, + plugins: [ + { + pluginId: "sqlite-ghost", + manifestPath, + manifestHash: "manifest-hash", + rootDir: pluginDir, + origin: "global", + enabled: true, + startup: { + sidecar: false, + memory: false, + deferConfiguredChannelFullLoadUntilAfterListen: false, + agentHarnesses: [], + }, + compat: [], + packageJson: { path: "package.json", hash: "package-hash" }, + }, + ], + diagnostics: [], + }; + await writePersistedInstalledPluginIndex(index, { stateDir: root }); + + const report = await runPostUpgradeProbes({ stateDir: root }); + + expect(report.findings).not.toContainEqual( + expect.objectContaining({ code: "plugin.index_unavailable" }), + ); + const finding = report.findings.find((f) => f.code === "plugin.entry_unresolved"); + expect(finding).toBeDefined(); + expect(finding?.plugin).toBe("sqlite-ghost"); + } finally { + await fs.rm(root, { recursive: true, force: true }); + } + }); + it("flags an enabled plugin whose declared entry does not exist on disk", async () => { const root = await makeFixtureRoot("entry-unresolved"); try { diff --git a/src/commands/doctor-post-upgrade.ts b/src/commands/doctor-post-upgrade.ts index eddd7d4f1bde..8539914a14bd 100644 --- a/src/commands/doctor-post-upgrade.ts +++ b/src/commands/doctor-post-upgrade.ts @@ -1,6 +1,7 @@ import crypto from "node:crypto"; import fs from "node:fs/promises"; import path from "node:path"; +import { readPersistedInstalledPluginIndex } from "../plugins/installed-plugin-index-store.js"; import type { PackageManifest } from "../plugins/manifest.js"; import { validatePackageExtensionEntriesForInstall } from "../plugins/package-entry-resolution.js"; import { @@ -71,6 +72,19 @@ async function readInstallsJson(installsPath: string): Promise { + if (params.installsPath) { + return await readInstallsJson(params.installsPath); + } + const index = await readPersistedInstalledPluginIndex( + params.stateDir ? { stateDir: params.stateDir } : {}, + ); + return index && isInstallsJson(index) ? { plugins: [...index.plugins] } : null; +} + async function readInstalledPackageJson( rootDir: string, packageJsonRelPath: string, @@ -90,10 +104,11 @@ async function sha256OfFile(absPath: string): Promise { } export async function runPostUpgradeProbes(params: { - installsPath: string; + installsPath?: string; + stateDir?: string; }): Promise { const findings: PostUpgradeFinding[] = []; - const installs = await readInstallsJson(params.installsPath); + const installs = await readInstalledPluginIndex(params); if (!installs) { findings.push({ level: "error", diff --git a/src/commands/doctor-state-migrations.test.ts b/src/commands/doctor-state-migrations.test.ts index dd214714557c..d8ef9338058c 100644 --- a/src/commands/doctor-state-migrations.test.ts +++ b/src/commands/doctor-state-migrations.test.ts @@ -11,6 +11,10 @@ import { setMaxPluginStateEntriesPerPluginForTests, } from "../plugin-state/plugin-state-store.js"; import { seedPluginStateEntriesForTests } from "../plugin-state/plugin-state-store.test-helpers.js"; +import { + readPersistedInstalledPluginIndex, + writePersistedInstalledPluginIndex, +} from "../plugins/installed-plugin-index-store.js"; import { closeOpenClawStateDatabaseForTest } from "../state/openclaw-state-db.js"; import { loadTaskFlowRegistryStateFromSqlite } from "../tasks/task-flow-registry.store.sqlite.js"; import { loadTaskRegistryStateFromSqlite } from "../tasks/task-registry.store.sqlite.js"; @@ -1184,6 +1188,171 @@ describe("doctor legacy state migrations", () => { }); }); + it("imports the legacy plugin install index JSON into shared state", async () => { + const root = await makeTempRoot(); + const sourcePath = path.join(root, "plugins", "installs.json"); + fs.mkdirSync(path.dirname(sourcePath), { recursive: true }); + fs.writeFileSync( + sourcePath, + JSON.stringify({ + plugins: [ + { + pluginId: "demo", + installRecord: { + source: "npm", + spec: "demo@1.0.0", + }, + }, + ], + }), + "utf8", + ); + + const detected = await detectLegacyStateMigrations({ + cfg: {}, + env: { OPENCLAW_STATE_DIR: root } as NodeJS.ProcessEnv, + }); + expect(detected.pluginInstallIndex).toEqual({ sourcePath, hasLegacy: true }); + expect(detected.preview).toContain( + `- Plugin install index: ${sourcePath} → shared SQLite state`, + ); + + const result = await runLegacyStateMigrations({ detected }); + + expect(result.warnings).toStrictEqual([]); + expect(result.changes).toContain( + "Migrated plugin install index 1 record → shared SQLite state", + ); + expect(result.changes).toContain( + `Archived plugin install index legacy source → ${sourcePath}.migrated`, + ); + expect(fs.existsSync(sourcePath)).toBe(false); + expect(fs.existsSync(`${sourcePath}.migrated`)).toBe(true); + await expect(readPersistedInstalledPluginIndex({ stateDir: root })).resolves.toMatchObject({ + installRecords: { demo: { source: "npm", spec: "demo@1.0.0" } }, + plugins: [], + }); + }); + + it("imports legacy record-only plugin install index JSON into shared state", async () => { + const root = await makeTempRoot(); + const sourcePath = path.join(root, "plugins", "installs.json"); + fs.mkdirSync(path.dirname(sourcePath), { recursive: true }); + fs.writeFileSync( + sourcePath, + JSON.stringify({ + installRecords: { + demo: { + source: "npm", + spec: "demo@1.0.0", + }, + }, + }), + "utf8", + ); + + const detected = await detectLegacyStateMigrations({ + cfg: {}, + env: { OPENCLAW_STATE_DIR: root } as NodeJS.ProcessEnv, + }); + const result = await runLegacyStateMigrations({ detected }); + + expect(result.warnings).toStrictEqual([]); + expect(result.changes).toContain( + "Migrated plugin install index 1 record → shared SQLite state", + ); + await expect(readPersistedInstalledPluginIndex({ stateDir: root })).resolves.toMatchObject({ + installRecords: { demo: { source: "npm", spec: "demo@1.0.0" } }, + plugins: [], + }); + }); + + it("imports legacy records-only plugin install index JSON into shared state", async () => { + const root = await makeTempRoot(); + const sourcePath = path.join(root, "plugins", "installs.json"); + fs.mkdirSync(path.dirname(sourcePath), { recursive: true }); + fs.writeFileSync( + sourcePath, + JSON.stringify({ + records: { + demo: { + source: "path", + sourcePath: "/tmp/demo", + }, + }, + }), + "utf8", + ); + + const detected = await detectLegacyStateMigrations({ + cfg: {}, + env: { OPENCLAW_STATE_DIR: root } as NodeJS.ProcessEnv, + }); + const result = await runLegacyStateMigrations({ detected }); + + expect(result.warnings).toStrictEqual([]); + expect(result.changes).toContain( + "Migrated plugin install index 1 record → shared SQLite state", + ); + await expect(readPersistedInstalledPluginIndex({ stateDir: root })).resolves.toMatchObject({ + installRecords: { demo: { source: "path", sourcePath: "/tmp/demo" } }, + plugins: [], + }); + }); + + it("merges missing legacy plugin install records into an existing SQLite index", async () => { + const root = await makeTempRoot(); + await writePersistedInstalledPluginIndex( + { + version: 1, + hostContractVersion: "test", + compatRegistryVersion: "test", + migrationVersion: 1, + policyHash: "test", + generatedAtMs: 1, + installRecords: { + existing: { + source: "npm", + spec: "existing@1.0.0", + }, + }, + plugins: [], + diagnostics: [], + }, + { stateDir: root }, + ); + const sourcePath = path.join(root, "plugins", "installs.json"); + fs.mkdirSync(path.dirname(sourcePath), { recursive: true }); + fs.writeFileSync( + sourcePath, + JSON.stringify({ + records: { + legacy: { + source: "git", + spec: "git:file:///tmp/legacy", + }, + }, + }), + "utf8", + ); + + const detected = await detectLegacyStateMigrations({ + cfg: {}, + env: { OPENCLAW_STATE_DIR: root } as NodeJS.ProcessEnv, + }); + const result = await runLegacyStateMigrations({ detected }); + + expect(result.warnings).toStrictEqual([]); + expect(result.changes).toContain("Merged 1 legacy plugin install record → shared SQLite state"); + expect(fs.existsSync(sourcePath)).toBe(false); + await expect(readPersistedInstalledPluginIndex({ stateDir: root })).resolves.toMatchObject({ + installRecords: { + existing: { source: "npm", spec: "existing@1.0.0" }, + legacy: { source: "git", spec: "git:file:///tmp/legacy" }, + }, + }); + }); + it("auto-migrates the shipped plugin-state SQLite sidecar by itself", async () => { const root = await makeTempRoot(); const sourcePath = writeLegacyPluginStateSidecar(root); diff --git a/src/commands/doctor.e2e-harness.ts b/src/commands/doctor.e2e-harness.ts index 5b273af07648..7d0ef02b7821 100644 --- a/src/commands/doctor.e2e-harness.ts +++ b/src/commands/doctor.e2e-harness.ts @@ -222,6 +222,10 @@ function createLegacyStateMigrationDetectionResult(params?: { sourcePath: "/tmp/state/plugin-state/state.sqlite", hasLegacy: false, }, + pluginInstallIndex: { + sourcePath: "/tmp/state/plugins/installs.json", + hasLegacy: false, + }, taskStateSidecars: { taskRunsPath: "/tmp/state/tasks/runs.sqlite", flowRunsPath: "/tmp/state/flows/registry.sqlite", diff --git a/src/commands/doctor.ts b/src/commands/doctor.ts index d711a343bddb..f7b704874965 100644 --- a/src/commands/doctor.ts +++ b/src/commands/doctor.ts @@ -1,12 +1,10 @@ -import { resolveInstalledPluginIndexStorePath } from "../plugins/installed-plugin-index-store-path.js"; import type { RuntimeEnv } from "../runtime.js"; import { runPostUpgradeProbes } from "./doctor-post-upgrade.js"; import type { DoctorOptions } from "./doctor-prompter.js"; export async function doctorCommand(runtime?: RuntimeEnv, options?: DoctorOptions): Promise { if (options?.postUpgrade) { - const installsPath = resolveInstalledPluginIndexStorePath(); - const report = await runPostUpgradeProbes({ installsPath }); + const report = await runPostUpgradeProbes({}); if (options.json) { console.log(JSON.stringify(report, null, 2)); } else { diff --git a/src/commands/doctor/shared/deprecation-compat.ts b/src/commands/doctor/shared/deprecation-compat.ts index 4b8cb3dbc3d6..cb5312d25ba4 100644 --- a/src/commands/doctor/shared/deprecation-compat.ts +++ b/src/commands/doctor/shared/deprecation-compat.ts @@ -249,7 +249,7 @@ const DOCTOR_DEPRECATION_COMPAT_RECORDS = [ introduced: "2026-04-25", source: "plugins.installs in authored config", migration: "src/config/plugin-install-config-migration.ts", - replacement: "state-managed plugins/installs.json install ledger", + replacement: "shared SQLite installed_plugin_index install ledger", docsPath: "/cli/plugins#registry", tests: [ "src/config/io.write-config.test.ts", diff --git a/src/commands/doctor/shared/plugin-registry-migration.test.ts b/src/commands/doctor/shared/plugin-registry-migration.test.ts index 41fc105af1f3..377842f1ab84 100644 --- a/src/commands/doctor/shared/plugin-registry-migration.test.ts +++ b/src/commands/doctor/shared/plugin-registry-migration.test.ts @@ -4,6 +4,7 @@ import { afterEach, describe, expect, it, vi } from "vitest"; import type { PluginCandidate } from "../../../plugins/discovery.js"; import { readPersistedInstalledPluginIndex, + resolveInstalledPluginIndexStorePath, writePersistedInstalledPluginIndex, } from "../../../plugins/installed-plugin-index-store.js"; import type { InstalledPluginIndex } from "../../../plugins/installed-plugin-index.js"; @@ -11,6 +12,7 @@ import { cleanupTrackedTempDirs, makeTrackedTempDir, } from "../../../plugins/test-helpers/fs-fixtures.js"; +import { runOpenClawStateWriteTransaction } from "../../../state/openclaw-state-db.js"; import { DISABLE_PLUGIN_REGISTRY_MIGRATION_ENV, FORCE_PLUGIN_REGISTRY_MIGRATION_ENV, @@ -120,10 +122,31 @@ function requirePlugin(index: InstalledPluginIndex | null | undefined, pluginId: return plugin; } +function insertStalePersistedIndexRow(stateDir: string) { + runOpenClawStateWriteTransaction( + ({ db }) => { + db.prepare( + ` + INSERT OR REPLACE INTO installed_plugin_index ( + index_key, version, host_contract_version, compat_registry_version, + migration_version, policy_hash, generated_at_ms, refresh_reason, + install_records_json, plugins_json, diagnostics_json, warning, updated_at_ms + ) VALUES ( + 'installed-plugin-index', 1, '2026.4.25', 'compat-v1', + 0, 'stale-policy', 123, NULL, + '{}', '[]', '[]', NULL, 123 + ) + `, + ).run(); + }, + { env: { ...process.env, OPENCLAW_STATE_DIR: stateDir } }, + ); +} + describe("plugin registry install migration", () => { it("short-circuits when a current registry file already exists", async () => { const stateDir = makeTempDir(); - const filePath = path.join(stateDir, "plugins", "installs.json"); + const filePath = resolveInstalledPluginIndexStorePath({ stateDir }); await writePersistedInstalledPluginIndex(createCurrentIndex(), { stateDir }); const readConfig = vi.fn(async () => ({})); @@ -145,11 +168,9 @@ describe("plugin registry install migration", () => { it("migrates when an existing registry file is not current", async () => { const stateDir = makeTempDir(); - const filePath = path.join(stateDir, "plugins", "installs.json"); const pluginDir = path.join(stateDir, "plugins", "demo"); - fs.mkdirSync(path.dirname(filePath), { recursive: true }); fs.mkdirSync(pluginDir, { recursive: true }); - fs.writeFileSync(filePath, JSON.stringify({ version: 1, migrationVersion: 0 }), "utf8"); + insertStalePersistedIndexRow(stateDir); const result = await migratePluginRegistryForInstall({ stateDir, diff --git a/src/config/config.plugin-validation.test.ts b/src/config/config.plugin-validation.test.ts index 0e288e1e2bd3..74d5328af4b0 100644 --- a/src/config/config.plugin-validation.test.ts +++ b/src/config/config.plugin-validation.test.ts @@ -3,6 +3,7 @@ import os from "node:os"; import path from "node:path"; import { afterAll, beforeAll, describe, expect, it, vi } from "vitest"; import { clearLoadInstalledPluginIndexInstallRecordsCache } from "../plugins/installed-plugin-index-records.js"; +import { writePersistedInstalledPluginIndex } from "../plugins/installed-plugin-index-store.js"; import { validateConfigObjectWithPlugins } from "./validation.js"; vi.unmock("../version.js"); @@ -832,26 +833,27 @@ describe("config plugin validation", () => { }); it("uses persisted installed-plugin records as stale channel evidence", async () => { - const installedPluginIndexPath = path.join(suiteHome, ".openclaw", "plugins", "installs.json"); - await mkdirSafe(path.dirname(installedPluginIndexPath)); + const stateDir = path.join(suiteHome, ".openclaw"); clearLoadInstalledPluginIndexInstallRecordsCache(); - await fs.writeFile( - installedPluginIndexPath, - JSON.stringify( - { - installRecords: { - "missing-sms": { - source: "npm", - spec: "missing-sms@1.0.0", - installedAt: "2026-04-12T00:00:00.000Z", - }, + await writePersistedInstalledPluginIndex( + { + version: 1, + hostContractVersion: "test", + compatRegistryVersion: "test", + migrationVersion: 1, + policyHash: "test", + generatedAtMs: 1, + installRecords: { + "missing-sms": { + source: "npm", + spec: "missing-sms@1.0.0", + installedAt: "2026-04-12T00:00:00.000Z", }, - plugins: [], }, - null, - 2, - ), - "utf-8", + plugins: [], + diagnostics: [], + }, + { stateDir }, ); clearLoadInstalledPluginIndexInstallRecordsCache(); try { @@ -872,7 +874,20 @@ describe("config plugin validation", () => { "unknown channel id: missing-sms (stale channel plugin config ignored; run openclaw doctor --fix to remove stale config, or install the plugin)", }); } finally { - await fs.rm(installedPluginIndexPath, { force: true }); + await writePersistedInstalledPluginIndex( + { + version: 1, + hostContractVersion: "test", + compatRegistryVersion: "test", + migrationVersion: 1, + policyHash: "test", + generatedAtMs: 2, + installRecords: {}, + plugins: [], + diagnostics: [], + }, + { stateDir }, + ); clearLoadInstalledPluginIndexInstallRecordsCache(); } }); diff --git a/src/config/io.ts b/src/config/io.ts index 8a47a919db02..8ef7da7bd2f0 100644 --- a/src/config/io.ts +++ b/src/config/io.ts @@ -22,7 +22,6 @@ import { import { createConfigValidationMetadataPluginIdScope } from "../plugins/channel-plugin-ids.js"; import { loadInstalledPluginIndexInstallRecordsSync, - resolveInstalledPluginIndexRecordsStorePath, writePersistedInstalledPluginIndexInstallRecordsSync, } from "../plugins/installed-plugin-index-records.js"; import { @@ -138,16 +137,6 @@ type ShippedPluginInstallConfigWriteMigration = } | { migrated: true; - filePath: string; - writtenHash: string; - previousFile: - | { - existed: false; - } - | { - existed: true; - raw: string; - }; }; type ShippedPluginInstallConfigReadMigration = { @@ -1409,49 +1398,6 @@ export function createConfigIO( return applyConfigOverrides(cfgWithOwnerDisplaySecret); } - function captureFileSnapshotSync(filePath: string): - | { - existed: false; - } - | { - existed: true; - raw: string; - } { - return deps.fs.existsSync(filePath) - ? ({ - existed: true, - raw: deps.fs.readFileSync(filePath, "utf-8"), - } as const) - : ({ existed: false } as const); - } - - function restoreFileSnapshotSync( - filePath: string, - previousFile: - | { - existed: false; - } - | { - existed: true; - raw: string; - }, - ): void { - if (previousFile.existed) { - deps.fs.writeFileSync(filePath, previousFile.raw, { - encoding: "utf-8", - mode: 0o600, - }); - return; - } - try { - deps.fs.unlinkSync(filePath); - } catch (err) { - if ((err as NodeJS.ErrnoException)?.code !== "ENOENT") { - throw err; - } - } - } - function replaceConfigFileSync(raw: string): void { replaceFileAtomicSync({ filePath: configPath, @@ -1479,11 +1425,6 @@ export function createConfigIO( try { const stateDir = resolveStateDir(deps.env, deps.homedir); - const filePath = resolveInstalledPluginIndexRecordsStorePath({ - env: deps.env, - stateDir, - }); - const previousFile = captureFileSnapshotSync(filePath); const existingRecords = loadInstalledPluginIndexInstallRecordsSync({ env: deps.env, stateDir, @@ -1508,12 +1449,7 @@ export function createConfigIO( const persistedRootRaw = JSON.stringify(persistedRootParsed, null, 2) .trimEnd() .concat("\n"); - try { - replaceConfigFileSync(persistedRootRaw); - } catch (err) { - restoreFileSnapshotSync(filePath, previousFile); - throw err; - } + replaceConfigFileSync(persistedRootRaw); return { config: stripped, persistedRootParsed, persistedRootRaw }; } } catch (err) { @@ -1572,10 +1508,6 @@ export function createConfigIO( } const stateDir = resolveStateDir(deps.env, deps.homedir); - const filePath = resolveInstalledPluginIndexRecordsStorePath({ - env: deps.env, - stateDir, - }); const existingRecords = loadInstalledPluginIndexInstallRecordsSync({ env: deps.env, stateDir, @@ -1584,14 +1516,8 @@ export function createConfigIO( return { migrated: false }; } - const previousFile = deps.fs.existsSync(filePath) - ? ({ - existed: true, - raw: deps.fs.readFileSync(filePath, "utf-8"), - } as const) - : ({ existed: false } as const); try { - const writtenPath = writePersistedInstalledPluginIndexInstallRecordsSync( + writePersistedInstalledPluginIndexInstallRecordsSync( { ...installRecords, ...existingRecords, @@ -1602,14 +1528,8 @@ export function createConfigIO( stateDir, }, ); - const writtenRaw = deps.fs.existsSync(writtenPath) - ? deps.fs.readFileSync(writtenPath, "utf-8") - : null; return { migrated: true, - filePath: writtenPath, - writtenHash: hashConfigRaw(writtenRaw), - previousFile, }; } catch (err) { throw new Error( @@ -1627,27 +1547,7 @@ export function createConfigIO( if (!migration.migrated) { return false; } - const currentRaw = deps.fs.existsSync(migration.filePath) - ? deps.fs.readFileSync(migration.filePath, "utf-8") - : null; - if (hashConfigRaw(currentRaw) !== migration.writtenHash) { - return false; - } - if (migration.previousFile.existed) { - deps.fs.writeFileSync(migration.filePath, migration.previousFile.raw, { - encoding: "utf-8", - mode: 0o600, - }); - return true; - } - try { - deps.fs.unlinkSync(migration.filePath); - } catch (err) { - if ((err as NodeJS.ErrnoException)?.code !== "ENOENT") { - throw err; - } - } - return true; + return false; } function loadConfigLocal(): OpenClawConfig { diff --git a/src/config/io.write-config.test.ts b/src/config/io.write-config.test.ts index 57b42d361af3..65cc37ef1a15 100644 --- a/src/config/io.write-config.test.ts +++ b/src/config/io.write-config.test.ts @@ -495,7 +495,7 @@ describe("config io write", () => { homedir: () => home, logger: { warn, error: vi.fn() }, }); - await fs.writeFile(path.join(unwritableStatePath, "plugins"), "not a directory", "utf-8"); + await fs.writeFile(path.join(unwritableStatePath, "state"), "not a directory", "utf-8"); const loadedConfig = io.loadConfig(); expectInstallRecord(loadedConfig.plugins?.installs?.demo, { @@ -522,7 +522,7 @@ describe("config io write", () => { }); }); - it("rolls back shipped plugin install index migration when config write fails", async () => { + it("keeps shipped plugin install index migration when config write fails", async () => { await withSuiteHome(async (home) => { const configPath = path.join(home, ".openclaw", "openclaw.json"); const pluginDir = path.join(home, ".openclaw", "plugins", "demo"); @@ -553,11 +553,14 @@ describe("config io write", () => { spec: "demo@1.0.0", installPath: pluginDir, }); - await expect( - readPersistedInstalledPluginIndex({ - stateDir: path.join(home, ".openclaw"), - }), - ).resolves.toBeNull(); + const persistedIndex = await readPersistedInstalledPluginIndex({ + stateDir: path.join(home, ".openclaw"), + }); + expectInstallRecord(persistedIndex?.installRecords.demo, { + source: "npm", + spec: "demo@1.0.0", + installPath: pluginDir, + }); }); }); @@ -769,9 +772,7 @@ describe("config io write", () => { gateway: { mode: "local", port: 18790 }, }); - expect(warn.mock.calls).toContainEqual([ - expect.stringContaining("Config write anomaly:"), - ]); + expect(warn.mock.calls).toContainEqual([expect.stringContaining("Config write anomaly:")]); expect(warn.mock.calls).toContainEqual([ expect.stringContaining("missing-meta-before-write"), ]); @@ -1669,7 +1670,7 @@ describe("config io write", () => { }); }); - it("rolls back plugin install index migration when runtime refresh fails", async () => { + it("keeps plugin install index migration when runtime refresh fails", async () => { await withSuiteHome(async (home) => { const stateDir = path.join(home, ".openclaw"); const configPath = path.join(stateDir, "openclaw.json"); @@ -1706,7 +1707,12 @@ describe("config io write", () => { ).rejects.toThrow(/runtime snapshot refresh failed: synthetic refresh failure/); await expect(fs.readFile(configPath, "utf-8")).resolves.toBe(initialRaw); - await expect(readPersistedInstalledPluginIndex({ stateDir })).resolves.toBeNull(); + const persistedIndex = await readPersistedInstalledPluginIndex({ stateDir }); + expectInstallRecord(persistedIndex?.installRecords.demo, { + source: "npm", + spec: "demo@1.0.0", + installPath: pluginDir, + }); } finally { setRuntimeConfigSnapshotRefreshHandler(null); if (previousConfigPath === undefined) { diff --git a/src/infra/state-migrations.state-dir.test.ts b/src/infra/state-migrations.state-dir.test.ts index 69c0324378e6..47e755c7f422 100644 --- a/src/infra/state-migrations.state-dir.test.ts +++ b/src/infra/state-migrations.state-dir.test.ts @@ -1,6 +1,7 @@ import fs from "node:fs"; import path from "node:path"; import { describe, expect, it } from "vitest"; +import { readPersistedInstalledPluginIndex } from "../plugins/installed-plugin-index-store.js"; import { withTempDir } from "../test-helpers/temp-dir.js"; import { autoMigrateLegacyStateDir, @@ -65,6 +66,44 @@ describe("legacy state dir auto-migration", () => { }); }); + it("migrates the legacy plugin install index from an explicit state dir", async () => { + await withStateDirFixture(async (root) => { + const legacyDir = path.join(root, ".clawdbot"); + const stateDir = path.join(root, "custom-state"); + const sourcePath = path.join(stateDir, "plugins", "installs.json"); + fs.mkdirSync(legacyDir, { recursive: true }); + fs.mkdirSync(path.dirname(sourcePath), { recursive: true }); + fs.writeFileSync( + sourcePath, + JSON.stringify({ + records: { + demo: { + source: "npm", + spec: "demo@1.0.0", + }, + }, + }), + "utf8", + ); + + const result = await autoMigrateLegacyStateDir({ + env: { OPENCLAW_STATE_DIR: stateDir } as NodeJS.ProcessEnv, + homedir: () => root, + }); + + expect(result.migrated).toBe(true); + expect(result.skipped).toBe(false); + expect(result.changes).toContain( + "Migrated plugin install index 1 record → shared SQLite state", + ); + expect(fs.existsSync(legacyDir)).toBe(true); + expect(fs.existsSync(sourcePath)).toBe(false); + await expect(readPersistedInstalledPluginIndex({ stateDir })).resolves.toMatchObject({ + installRecords: { demo: { source: "npm", spec: "demo@1.0.0" } }, + }); + }); + }); + it("only runs once per process until reset", async () => { await withStateDirFixture(async (root) => { const legacyDir = path.join(root, ".clawdbot"); @@ -89,4 +128,38 @@ describe("legacy state dir auto-migration", () => { }); }); }); + + it("migrates the legacy plugin install index before config reads", async () => { + await withStateDirFixture(async (root) => { + const stateDir = path.join(root, ".openclaw"); + const sourcePath = path.join(stateDir, "plugins", "installs.json"); + fs.mkdirSync(path.dirname(sourcePath), { recursive: true }); + fs.writeFileSync( + sourcePath, + JSON.stringify({ + records: { + demo: { + source: "npm", + spec: "demo@1.0.0", + }, + }, + }), + "utf8", + ); + + const result = await autoMigrateLegacyStateDir({ + env: {} as NodeJS.ProcessEnv, + homedir: () => root, + }); + + expect(result.migrated).toBe(true); + expect(result.changes).toContain( + "Migrated plugin install index 1 record → shared SQLite state", + ); + expect(fs.existsSync(sourcePath)).toBe(false); + await expect(readPersistedInstalledPluginIndex({ stateDir })).resolves.toMatchObject({ + installRecords: { demo: { source: "npm", spec: "demo@1.0.0" } }, + }); + }); + }); }); diff --git a/src/infra/state-migrations.ts b/src/infra/state-migrations.ts index a3800599d4ee..aeb6ef068a28 100644 --- a/src/infra/state-migrations.ts +++ b/src/infra/state-migrations.ts @@ -34,6 +34,17 @@ import { type PluginDoctorStateMigrationContext, type PluginDoctorStateMigration, } from "../plugins/doctor-contract-registry.js"; +import { + parseInstalledPluginIndex, + readPersistedInstalledPluginIndexSync, + resolveLegacyInstalledPluginIndexStorePath, + writePersistedInstalledPluginIndexSync, +} from "../plugins/installed-plugin-index-store.js"; +import { + INSTALLED_PLUGIN_INDEX_MIGRATION_VERSION, + INSTALLED_PLUGIN_INDEX_VERSION, + type InstalledPluginIndex, +} from "../plugins/installed-plugin-index.js"; import { buildAgentMainSessionKey, DEFAULT_AGENT_ID, @@ -94,6 +105,10 @@ export type LegacyStateDetection = { sourcePath: string; hasLegacy: boolean; }; + pluginInstallIndex: { + sourcePath: string; + hasLegacy: boolean; + }; taskStateSidecars: { taskRunsPath: string; flowRunsPath: string; @@ -284,6 +299,131 @@ function archiveLegacyPluginStateSidecar(params: { ); } +function readLegacyInstalledPluginIndex(sourcePath: string): InstalledPluginIndex | null { + try { + const parsed = JSON.parse(fs.readFileSync(sourcePath, "utf8")) as unknown; + const current = parseInstalledPluginIndex(parsed); + if (current) { + return current; + } + const installRecords = + readLegacyTopLevelInstallRecords(parsed) ?? readLegacyEmbeddedInstallRecords(parsed); + if (!installRecords || typeof installRecords !== "object" || Array.isArray(installRecords)) { + return null; + } + return parseInstalledPluginIndex({ + version: INSTALLED_PLUGIN_INDEX_VERSION, + hostContractVersion: "legacy", + compatRegistryVersion: "legacy", + migrationVersion: INSTALLED_PLUGIN_INDEX_MIGRATION_VERSION, + policyHash: "legacy", + generatedAtMs: 0, + installRecords, + plugins: [], + diagnostics: [], + }); + } catch { + return null; + } +} + +function readLegacyTopLevelInstallRecords(parsed: unknown): unknown { + if (!parsed || typeof parsed !== "object" || Array.isArray(parsed)) { + return null; + } + const legacy = parsed as { installRecords?: unknown; records?: unknown }; + return legacy.installRecords ?? legacy.records; +} + +function readLegacyEmbeddedInstallRecords(parsed: unknown): Record | null { + if (!parsed || typeof parsed !== "object" || Array.isArray(parsed)) { + return null; + } + const plugins = (parsed as { plugins?: unknown }).plugins; + if (!Array.isArray(plugins)) { + return null; + } + const records: Record = {}; + for (const plugin of plugins) { + if (!plugin || typeof plugin !== "object" || Array.isArray(plugin)) { + continue; + } + const pluginId = (plugin as { pluginId?: unknown }).pluginId; + const installRecord = (plugin as { installRecord?: unknown }).installRecord; + if ( + typeof pluginId === "string" && + pluginId.trim() && + installRecord && + typeof installRecord === "object" && + !Array.isArray(installRecord) + ) { + records[pluginId] = installRecord; + } + } + return Object.keys(records).length > 0 ? records : null; +} + +function legacyInstalledPluginIndexMatches( + current: InstalledPluginIndex, + legacy: InstalledPluginIndex, +): boolean { + return ( + JSON.stringify(current.installRecords) === JSON.stringify(legacy.installRecords) && + JSON.stringify(current.plugins) === JSON.stringify(legacy.plugins) && + JSON.stringify(current.diagnostics) === JSON.stringify(legacy.diagnostics) + ); +} + +function mergeLegacyInstalledPluginIndexRecords( + current: InstalledPluginIndex, + legacy: InstalledPluginIndex, +): { merged: InstalledPluginIndex; addedCount: number; conflicts: string[] } { + const installRecords = { ...current.installRecords }; + const conflicts: string[] = []; + let addedCount = 0; + for (const [pluginId, legacyRecord] of Object.entries(legacy.installRecords)) { + const currentRecord = installRecords[pluginId]; + if (!currentRecord) { + installRecords[pluginId] = legacyRecord; + addedCount += 1; + continue; + } + if (JSON.stringify(currentRecord) !== JSON.stringify(legacyRecord)) { + conflicts.push(pluginId); + } + } + return { + merged: { + ...current, + installRecords, + }, + addedCount, + conflicts, + }; +} + +function archiveLegacyInstalledPluginIndex(params: { + sourcePath: string; + changes: string[]; + warnings: string[]; +}): void { + const archivedPath = `${params.sourcePath}.migrated`; + if (fileExists(archivedPath)) { + params.warnings.push( + `Left migrated plugin install index in place because archive already exists: ${archivedPath}`, + ); + return; + } + try { + fs.renameSync(params.sourcePath, archivedPath); + params.changes.push(`Archived plugin install index legacy source → ${archivedPath}`); + } catch (err) { + params.warnings.push( + `Failed archiving plugin install index ${params.sourcePath}: ${String(err)}`, + ); + } +} + function archiveLegacyTaskStateSidecar(params: { sourcePath: string; label: string; @@ -1256,6 +1396,70 @@ async function migrateLegacyPluginStateSidecar(params: { return { changes, warnings }; } +async function migrateLegacyInstalledPluginIndex(params: { + stateDir: string; +}): Promise<{ changes: string[]; warnings: string[] }> { + const sourcePath = resolveLegacyInstalledPluginIndexStorePath({ stateDir: params.stateDir }); + if (!fileExists(sourcePath)) { + return { changes: [], warnings: [] }; + } + + const changes: string[] = []; + const warnings: string[] = []; + const legacy = readLegacyInstalledPluginIndex(sourcePath); + if (!legacy) { + return { + changes, + warnings: [`Left plugin install index in place because ${sourcePath} is invalid`], + }; + } + + const storeOptions = { stateDir: params.stateDir }; + const current = readPersistedInstalledPluginIndexSync(storeOptions); + if (current && !legacyInstalledPluginIndexMatches(current, legacy)) { + const merged = mergeLegacyInstalledPluginIndexRecords(current, legacy); + if (merged.addedCount > 0) { + try { + writePersistedInstalledPluginIndexSync(merged.merged, storeOptions); + changes.push( + `Merged ${merged.addedCount} legacy plugin install ${merged.addedCount === 1 ? "record" : "records"} → shared SQLite state`, + ); + } catch (err) { + return { + changes, + warnings: [`Failed merging plugin install index ${sourcePath}: ${String(err)}`], + }; + } + } + if (merged.conflicts.length > 0) { + return { + changes, + warnings: [ + `Left plugin install index in place because shared SQLite state has conflicting plugin install metadata for: ${merged.conflicts.join(", ")}`, + ], + }; + } + } + + if (!current) { + try { + writePersistedInstalledPluginIndexSync(legacy, storeOptions); + const recordCount = Object.keys(legacy.installRecords).length; + changes.push( + `Migrated plugin install index ${recordCount} ${recordCount === 1 ? "record" : "records"} → shared SQLite state`, + ); + } catch (err) { + return { + changes, + warnings: [`Failed migrating plugin install index ${sourcePath}: ${String(err)}`], + }; + } + } + + archiveLegacyInstalledPluginIndex({ sourcePath, changes, warnings }); + return { changes, warnings }; +} + function resolvePluginStateImportTargetKey(scopeKey: string, key: string): string { return scopeKey ? `${scopeKey}:${key}` : key; } @@ -2048,13 +2252,27 @@ export async function autoMigrateLegacyStateDir(params: { } autoMigrateStateDirChecked = true; + const homedir = params.homedir ?? os.homedir; const env = params.env ?? process.env; - if (env.OPENCLAW_STATE_DIR?.trim()) { - return { migrated: false, skipped: true, changes: [], warnings: [] }; + const warnings: string[] = []; + const changes: string[] = []; + const hasCustomStateDir = Boolean(env.OPENCLAW_STATE_DIR?.trim()); + const targetDir = hasCustomStateDir ? resolveStateDir(env, homedir) : resolveNewStateDir(homedir); + const migratePluginInstallIndex = async () => { + const result = await migrateLegacyInstalledPluginIndex({ stateDir: targetDir }); + changes.push(...result.changes); + warnings.push(...result.warnings); + }; + if (hasCustomStateDir) { + await migratePluginInstallIndex(); + return { + migrated: changes.length > 0, + skipped: changes.length === 0 && warnings.length === 0, + changes, + warnings, + }; } - const homedir = params.homedir ?? os.homedir; - const targetDir = resolveNewStateDir(homedir); const legacyDirs = resolveLegacyStateDirs(homedir); let legacyDir = legacyDirs.find((dir) => { try { @@ -2063,8 +2281,6 @@ export async function autoMigrateLegacyStateDir(params: { return false; } }); - const warnings: string[] = []; - const changes: string[] = []; let legacyStat: fs.Stats | null; try { @@ -2073,7 +2289,8 @@ export async function autoMigrateLegacyStateDir(params: { legacyStat = null; } if (!legacyStat) { - return { migrated: false, skipped: false, changes, warnings }; + await migratePluginInstallIndex(); + return { migrated: changes.length > 0, skipped: false, changes, warnings }; } if (!legacyStat.isDirectory() && !legacyStat.isSymbolicLink()) { warnings.push(`Legacy state path is not a directory: ${legacyDir}`); @@ -2090,7 +2307,8 @@ export async function autoMigrateLegacyStateDir(params: { return { migrated: false, skipped: false, changes, warnings }; } if (path.resolve(legacyTarget) === path.resolve(targetDir)) { - return { migrated: false, skipped: false, changes, warnings }; + await migratePluginInstallIndex(); + return { migrated: changes.length > 0, skipped: false, changes, warnings }; } if (legacyDirs.some((dir) => path.resolve(dir) === path.resolve(legacyTarget))) { legacyDir = legacyTarget; @@ -2122,12 +2340,14 @@ export async function autoMigrateLegacyStateDir(params: { if (isDirPath(targetDir)) { if (legacyDir && isLegacyDirSymlinkMirror(legacyDir, targetDir)) { - return { migrated: false, skipped: false, changes, warnings }; + await migratePluginInstallIndex(); + return { migrated: changes.length > 0, skipped: false, changes, warnings }; } + await migratePluginInstallIndex(); warnings.push( `State dir migration skipped: target already exists (${targetDir}). Remove or merge manually.`, ); - return { migrated: false, skipped: false, changes, warnings }; + return { migrated: changes.length > 0, skipped: false, changes, warnings }; } try { @@ -2181,6 +2401,7 @@ export async function autoMigrateLegacyStateDir(params: { } } + await migratePluginInstallIndex(); return { migrated: changes.length > 0, skipped: false, changes, warnings }; } @@ -2337,6 +2558,8 @@ export async function detectLegacyStateMigrations(params: { const hasLegacyAgentDir = existsDir(legacyAgentDir); const pluginStateSidecarPath = resolveLegacyPluginStateSidecarPath(stateDir); const hasPluginStateSidecar = fileExists(pluginStateSidecarPath); + const pluginInstallIndexPath = resolveLegacyInstalledPluginIndexStorePath({ stateDir }); + const hasPluginInstallIndex = fileExists(pluginInstallIndexPath); const taskRunsSidecarPath = resolveLegacyTaskRunsSidecarPath(stateDir); const flowRunsSidecarPath = resolveLegacyFlowRunsSidecarPath(stateDir); const hasTaskStateSidecars = fileExists(taskRunsSidecarPath) || fileExists(flowRunsSidecarPath); @@ -2375,6 +2598,9 @@ export async function detectLegacyStateMigrations(params: { if (hasPluginStateSidecar) { preview.push(`- Plugin state sidecar: ${pluginStateSidecarPath} → shared SQLite state`); } + if (hasPluginInstallIndex) { + preview.push(`- Plugin install index: ${pluginInstallIndexPath} → shared SQLite state`); + } if (fileExists(taskRunsSidecarPath)) { preview.push(`- Task registry sidecar: ${taskRunsSidecarPath} → shared SQLite state`); } @@ -2422,6 +2648,10 @@ export async function detectLegacyStateMigrations(params: { sourcePath: pluginStateSidecarPath, hasLegacy: hasPluginStateSidecar, }, + pluginInstallIndex: { + sourcePath: pluginInstallIndexPath, + hasLegacy: hasPluginInstallIndex, + }, taskStateSidecars: { taskRunsPath: taskRunsSidecarPath, flowRunsPath: flowRunsSidecarPath, @@ -2683,6 +2913,9 @@ export async function runLegacyStateMigrations(params: { const pluginStateSidecar = await migrateLegacyPluginStateSidecar({ stateDir: detected.stateDir, }); + const pluginInstallIndex = await migrateLegacyInstalledPluginIndex({ + stateDir: detected.stateDir, + }); const taskStateSidecars = await migrateLegacyTaskStateSidecars({ stateDir: detected.stateDir, }); @@ -2711,6 +2944,7 @@ export async function runLegacyStateMigrations(params: { return { changes: [ ...pluginStateSidecar.changes, + ...pluginInstallIndex.changes, ...taskStateSidecars.changes, ...deliveryQueues.changes, ...preSessionChannelPlans.changes, @@ -2722,6 +2956,7 @@ export async function runLegacyStateMigrations(params: { ], warnings: [ ...pluginStateSidecar.warnings, + ...pluginInstallIndex.warnings, ...taskStateSidecars.warnings, ...deliveryQueues.warnings, ...preSessionChannelPlans.warnings, @@ -3042,6 +3277,9 @@ export async function autoMigrateLegacyState(params: { const pluginStateSidecar = await migrateLegacyPluginStateSidecar({ stateDir: detected.stateDir, }); + const pluginInstallIndex = await migrateLegacyInstalledPluginIndex({ + stateDir: detected.stateDir, + }); const taskStateSidecars = await migrateLegacyTaskStateSidecars({ stateDir: detected.stateDir, }); @@ -3060,6 +3298,7 @@ export async function autoMigrateLegacyState(params: { ...orphanKeys.changes, ...acpSessionMetadata.changes, ...pluginStateSidecar.changes, + ...pluginInstallIndex.changes, ...taskStateSidecars.changes, ...deliveryQueues.changes, ...preSessionChannelPlans.changes, @@ -3070,6 +3309,7 @@ export async function autoMigrateLegacyState(params: { ...orphanKeys.warnings, ...acpSessionMetadata.warnings, ...pluginStateSidecar.warnings, + ...pluginInstallIndex.warnings, ...taskStateSidecars.warnings, ...deliveryQueues.warnings, ...preSessionChannelPlans.warnings, @@ -3082,6 +3322,7 @@ export async function autoMigrateLegacyState(params: { orphanKeys.changes.length > 0 || acpSessionMetadata.changes.length > 0 || pluginStateSidecar.changes.length > 0 || + pluginInstallIndex.changes.length > 0 || taskStateSidecars.changes.length > 0 || deliveryQueues.changes.length > 0 || preSessionChannelPlans.changes.length > 0 || @@ -3097,6 +3338,7 @@ export async function autoMigrateLegacyState(params: { !detected.channelPlans.hasLegacy && !detected.pluginPlans?.hasLegacy && !detected.pluginStateSidecar.hasLegacy && + !detected.pluginInstallIndex.hasLegacy && !detected.taskStateSidecars.hasLegacy && !detected.deliveryQueues.hasLegacy ) { @@ -3126,6 +3368,9 @@ export async function autoMigrateLegacyState(params: { const pluginStateSidecar = await migrateLegacyPluginStateSidecar({ stateDir: detected.stateDir, }); + const pluginInstallIndex = await migrateLegacyInstalledPluginIndex({ + stateDir: detected.stateDir, + }); const taskStateSidecars = await migrateLegacyTaskStateSidecars({ stateDir: detected.stateDir, }); @@ -3156,6 +3401,7 @@ export async function autoMigrateLegacyState(params: { ...orphanKeys.changes, ...acpSessionMetadata.changes, ...pluginStateSidecar.changes, + ...pluginInstallIndex.changes, ...taskStateSidecars.changes, ...deliveryQueues.changes, ...preSessionChannelPlans.changes, @@ -3170,6 +3416,7 @@ export async function autoMigrateLegacyState(params: { ...orphanKeys.warnings, ...acpSessionMetadata.warnings, ...pluginStateSidecar.warnings, + ...pluginInstallIndex.warnings, ...taskStateSidecars.warnings, ...deliveryQueues.warnings, ...preSessionChannelPlans.warnings, diff --git a/src/plugins/compat/registry.ts b/src/plugins/compat/registry.ts index c38765abe09b..30c1d7953c83 100644 --- a/src/plugins/compat/registry.ts +++ b/src/plugins/compat/registry.ts @@ -568,7 +568,7 @@ export const PLUGIN_COMPAT_RECORDS = [ deprecated: "2026-04-26", warningStarts: "2026-04-26", removeAfter: "2026-07-26", - replacement: "state-managed `plugins/installs.json` install ledger", + replacement: "shared SQLite `installed_plugin_index` install ledger", docsPath: "/cli/plugins#registry", surfaces: ["plugins.installs authored config", "plugin install/update migration"], diagnostics: ["config write migration warning", "doctor registry migration"], diff --git a/src/plugins/installed-plugin-index-record-cache.ts b/src/plugins/installed-plugin-index-record-cache.ts new file mode 100644 index 000000000000..4cfbaa5e74f7 --- /dev/null +++ b/src/plugins/installed-plugin-index-record-cache.ts @@ -0,0 +1,30 @@ +import type { PluginInstallRecord } from "../config/types.plugins.js"; + +export type InstallRecordsCacheEntry = { + records: Record; +}; + +const installRecordsCache = new Map(); +let installRecordsCacheGeneration = 0; + +export function getInstalledPluginIndexInstallRecordsCache( + key: string, +): InstallRecordsCacheEntry | undefined { + return installRecordsCache.get(key); +} + +export function setInstalledPluginIndexInstallRecordsCache( + key: string, + entry: InstallRecordsCacheEntry, +): void { + installRecordsCache.set(key, entry); +} + +export function getInstalledPluginIndexInstallRecordsCacheGeneration(): number { + return installRecordsCacheGeneration; +} + +export function clearLoadInstalledPluginIndexInstallRecordsCache(): void { + installRecordsCacheGeneration += 1; + installRecordsCache.clear(); +} diff --git a/src/plugins/installed-plugin-index-record-reader.ts b/src/plugins/installed-plugin-index-record-reader.ts index 42a3f391da59..8b38b8aec2f1 100644 --- a/src/plugins/installed-plugin-index-record-reader.ts +++ b/src/plugins/installed-plugin-index-record-reader.ts @@ -2,14 +2,23 @@ import fs from "node:fs"; import path from "node:path"; import { isRecord } from "@openclaw/normalization-core/record-coerce"; import type { PluginInstallRecord } from "../config/types.plugins.js"; -import { tryReadJson, tryReadJsonSync } from "../infra/json-files.js"; +import { tryReadJsonSync } from "../infra/json-files.js"; +import type { OpenClawStateDatabaseOptions } from "../state/openclaw-state-db.js"; +import { openOpenClawStateDatabase } from "../state/openclaw-state-db.js"; import { resolveDefaultPluginNpmDir, validatePluginId } from "./install-paths.js"; +import { + getInstalledPluginIndexInstallRecordsCache, + getInstalledPluginIndexInstallRecordsCacheGeneration, + setInstalledPluginIndexInstallRecordsCache, +} from "./installed-plugin-index-record-cache.js"; import { resolveInstalledPluginIndexStorePath, type InstalledPluginIndexStoreOptions, } from "./installed-plugin-index-store-path.js"; import { listManagedPluginNpmProjectRootsSync } from "./npm-project-roots.js"; +export { clearLoadInstalledPluginIndexInstallRecordsCache } from "./installed-plugin-index-record-cache.js"; + function cloneInstallRecords( records: Record | undefined, ): Record { @@ -212,6 +221,9 @@ function extractPluginInstallRecordsFromPersistedInstalledPluginIndex( if (Object.hasOwn(index, "installRecords")) { return readRecordMap(index.installRecords) ?? {}; } + if (Object.hasOwn(index, "records")) { + return readRecordMap(index.records) ?? {}; + } if (!Array.isArray(index.plugins)) { return null; } @@ -228,27 +240,86 @@ function extractPluginInstallRecordsFromPersistedInstalledPluginIndex( return records; } +type InstalledPluginIndexRecordRow = { + install_records_json: string; + plugins_json: string; +}; + +function resolveStateDatabaseOptions( + options: InstalledPluginIndexStoreOptions = {}, +): OpenClawStateDatabaseOptions { + if (options.filePath) { + return { + ...(options.env ? { env: options.env } : {}), + path: options.filePath, + }; + } + if (options.stateDir) { + return { + env: { + ...(options.env ?? process.env), + OPENCLAW_STATE_DIR: options.stateDir, + }, + }; + } + return options.env ? { env: options.env } : {}; +} + +function parseJsonColumn(value: string): unknown { + try { + return JSON.parse(value) as unknown; + } catch { + return undefined; + } +} + +function readPersistedInstalledPluginIndexForRecords( + options: InstalledPluginIndexStoreOptions = {}, +): unknown { + const storePath = resolveInstalledPluginIndexStorePath(options); + if (!fs.existsSync(storePath)) { + return null; + } + if (options.filePath?.endsWith(".json")) { + return tryReadJsonSync(options.filePath); + } + try { + const database = openOpenClawStateDatabase(resolveStateDatabaseOptions(options)); + const row = database.db + .prepare( + ` + SELECT install_records_json, plugins_json + FROM installed_plugin_index + WHERE index_key = ? + `, + ) + .get("installed-plugin-index") as InstalledPluginIndexRecordRow | undefined; + if (!row) { + return null; + } + return { + installRecords: parseJsonColumn(row.install_records_json), + plugins: parseJsonColumn(row.plugins_json), + }; + } catch { + return null; + } +} + export async function readPersistedInstalledPluginIndexInstallRecords( options: InstalledPluginIndexStoreOptions = {}, ): Promise | null> { - const parsed = await tryReadJson(resolveInstalledPluginIndexStorePath(options)); + const parsed = readPersistedInstalledPluginIndexForRecords(options); return extractPluginInstallRecordsFromPersistedInstalledPluginIndex(parsed); } export function readPersistedInstalledPluginIndexInstallRecordsSync( options: InstalledPluginIndexStoreOptions = {}, ): Record | null { - const parsed = tryReadJsonSync(resolveInstalledPluginIndexStorePath(options)); + const parsed = readPersistedInstalledPluginIndexForRecords(options); return extractPluginInstallRecordsFromPersistedInstalledPluginIndex(parsed); } -type InstallRecordsCacheEntry = { - records: Record; -}; - -const installRecordsCache = new Map(); -let installRecordsCacheGeneration = 0; - function resolveInstallRecordsCacheKey(options: InstalledPluginIndexStoreOptions): string { return [ path.resolve(resolveInstalledPluginIndexStorePath(options)), @@ -256,30 +327,25 @@ function resolveInstallRecordsCacheKey(options: InstalledPluginIndexStoreOptions ].join("\0"); } -export function clearLoadInstalledPluginIndexInstallRecordsCache(): void { - installRecordsCacheGeneration += 1; - installRecordsCache.clear(); -} - export async function loadInstalledPluginIndexInstallRecords( params: InstalledPluginIndexStoreOptions = {}, ): Promise> { const cacheKey = resolveInstallRecordsCacheKey(params); - const cached = installRecordsCache.get(cacheKey); + const cached = getInstalledPluginIndexInstallRecordsCache(cacheKey); if (cached) { return cloneInstallRecords(cached.records); } - const cacheGeneration = installRecordsCacheGeneration; + const cacheGeneration = getInstalledPluginIndexInstallRecordsCacheGeneration(); const records = cloneInstallRecords( mergeRecoveredManagedNpmInstallRecords( await readPersistedInstalledPluginIndexInstallRecords(params), params, ), ); - if (cacheGeneration !== installRecordsCacheGeneration) { + if (cacheGeneration !== getInstalledPluginIndexInstallRecordsCacheGeneration()) { return await loadInstalledPluginIndexInstallRecords(params); } - installRecordsCache.set(cacheKey, { records }); + setInstalledPluginIndexInstallRecordsCache(cacheKey, { records }); return cloneInstallRecords(records); } @@ -287,7 +353,7 @@ export function loadInstalledPluginIndexInstallRecordsSync( params: InstalledPluginIndexStoreOptions = {}, ): Record { const cacheKey = resolveInstallRecordsCacheKey(params); - const cached = installRecordsCache.get(cacheKey); + const cached = getInstalledPluginIndexInstallRecordsCache(cacheKey); if (cached) { return cloneInstallRecords(cached.records); } @@ -297,6 +363,6 @@ export function loadInstalledPluginIndexInstallRecordsSync( params, ), ); - installRecordsCache.set(cacheKey, { records }); + setInstalledPluginIndexInstallRecordsCache(cacheKey, { records }); return cloneInstallRecords(records); } diff --git a/src/plugins/installed-plugin-index-records.test.ts b/src/plugins/installed-plugin-index-records.test.ts index 5fd4d384f3e7..f547931320e1 100644 --- a/src/plugins/installed-plugin-index-records.test.ts +++ b/src/plugins/installed-plugin-index-records.test.ts @@ -3,6 +3,10 @@ import os from "node:os"; import path from "node:path"; import { afterEach, describe, expect, it, vi } from "vitest"; import type { PluginInstallRecord } from "../config/types.plugins.js"; +import { + closeOpenClawStateDatabaseForTest, + runOpenClawStateWriteTransaction, +} from "../state/openclaw-state-db.js"; import type { PluginCandidate } from "./discovery.js"; import { clearLoadInstalledPluginIndexInstallRecordsCache, @@ -16,6 +20,7 @@ import { writePersistedInstalledPluginIndexInstallRecords, writePersistedInstalledPluginIndexInstallRecordsSync, } from "./installed-plugin-index-records.js"; +import { readPersistedInstalledPluginIndex } from "./installed-plugin-index-store.js"; import { writeManagedNpmPlugin } from "./test-helpers/managed-npm-plugin.js"; const tempDirs: string[] = []; @@ -58,16 +63,28 @@ function expectRecordFields(record: unknown, expected: Record) return actual; } -function createDeferred() { - let resolve: (value: T) => void = () => {}; - const promise = new Promise((done) => { - resolve = done; - }); - return { promise, resolve }; +function updatePersistedInstallRecordsWithoutClearingCache( + stateDir: string, + records: Record, +) { + runOpenClawStateWriteTransaction( + ({ db }) => { + db.prepare( + ` + UPDATE installed_plugin_index + SET install_records_json = ?, + updated_at_ms = ? + WHERE index_key = 'installed-plugin-index' + `, + ).run(JSON.stringify(records), Date.now()); + }, + { env: { ...process.env, OPENCLAW_STATE_DIR: stateDir } }, + ); } afterEach(() => { - vi.doUnmock("../infra/json-files.js"); + closeOpenClawStateDatabaseForTest(); + vi.doUnmock("./installed-plugin-index-store.js"); clearLoadInstalledPluginIndexInstallRecordsCache(); for (const dir of tempDirs.splice(0)) { fs.rmSync(dir, { recursive: true, force: true }); @@ -95,13 +112,11 @@ describe("plugin index install records store", () => { ); const indexPath = resolveInstalledPluginIndexRecordsStorePath({ stateDir }); - expect(indexPath).toBe(path.join(stateDir, "plugins", "installs.json")); - const persisted = JSON.parse(fs.readFileSync(indexPath, "utf8")) as { - version?: number; - generatedAtMs?: number; - installRecords?: Record; - plugins?: Array<{ pluginId?: string; installRecordHash?: string }>; - }; + expect(indexPath).toBe(path.join(stateDir, "state", "openclaw.sqlite")); + const persisted = await readPersistedInstalledPluginIndex({ stateDir }); + if (!persisted) { + throw new Error("Expected persisted plugin index"); + } expect(persisted.version).toBe(1); expect(persisted.generatedAtMs).toBe(1777118400000); expectRecordFields(persisted.installRecords?.twitch, { @@ -139,9 +154,10 @@ describe("plugin index install records store", () => { }, ); - const persisted = JSON.parse( - fs.readFileSync(resolveInstalledPluginIndexRecordsStorePath({ stateDir }), "utf8"), - ) as { installRecords?: Record; plugins?: unknown[] }; + const persisted = await readPersistedInstalledPluginIndex({ stateDir }); + if (!persisted) { + throw new Error("Expected persisted plugin index"); + } expectRecordFields(persisted.installRecords?.missing, { source: "npm", spec: "missing-plugin@1.0.0", @@ -263,21 +279,12 @@ describe("plugin index install records store", () => { }, }); - const indexPath = resolveInstalledPluginIndexRecordsStorePath({ stateDir }); - const persisted = JSON.parse(fs.readFileSync(indexPath, "utf8")) as Record; - fs.writeFileSync( - indexPath, - JSON.stringify({ - ...persisted, - installRecords: { - external: { - source: "npm", - spec: "external-plugin@2.0.0", - }, - }, - }), - "utf8", - ); + updatePersistedInstallRecordsWithoutClearingCache(stateDir, { + external: { + source: "npm", + spec: "external-plugin@2.0.0", + }, + }); expect(loadInstalledPluginIndexInstallRecordsSync({ stateDir })).toEqual({ external: { @@ -296,22 +303,17 @@ describe("plugin index install records store", () => { }); }); - it("reads legacy persisted records when the plugin index has no plugin list", async () => { + it("reads persisted records when the plugin index has no plugin list", async () => { const stateDir = makeStateDir(); - const indexPath = resolveInstalledPluginIndexRecordsStorePath({ stateDir }); - fs.mkdirSync(path.dirname(indexPath), { recursive: true }); - fs.writeFileSync( - indexPath, - JSON.stringify({ - installRecords: { - legacy: { - source: "npm", - spec: "legacy@1.0.0", - installPath: path.join(stateDir, "plugins", "legacy"), - }, + await writePersistedInstalledPluginIndexInstallRecords( + { + legacy: { + source: "npm", + spec: "legacy@1.0.0", + installPath: path.join(stateDir, "plugins", "legacy"), }, - }), - "utf8", + }, + { stateDir, candidates: [] }, ); await expect(loadInstalledPluginIndexInstallRecords({ stateDir })).resolves.toEqual({ @@ -337,10 +339,6 @@ describe("plugin index install records store", () => { pluginId: "codex", version: "2026.5.2", }); - const indexPath = resolveInstalledPluginIndexRecordsStorePath({ stateDir }); - fs.mkdirSync(path.dirname(indexPath), { recursive: true }); - fs.writeFileSync(indexPath, JSON.stringify({ installRecords: {}, plugins: [] }), "utf8"); - const loaded = await loadInstalledPluginIndexInstallRecords({ stateDir }); expectRecordFields(loaded.codex, { source: "npm", @@ -374,10 +372,6 @@ describe("plugin index install records store", () => { version: "2026.5.2", layout: "legacy", }); - const indexPath = resolveInstalledPluginIndexRecordsStorePath({ stateDir }); - fs.mkdirSync(path.dirname(indexPath), { recursive: true }); - fs.writeFileSync(indexPath, JSON.stringify({ installRecords: {}, plugins: [] }), "utf8"); - const loaded = await loadInstalledPluginIndexInstallRecords({ stateDir }); expectRecordFields(loaded.discord, { source: "npm", @@ -475,10 +469,6 @@ describe("plugin index install records store", () => { pluginId: "codex", version: "2026.5.18-beta.1", }); - const indexPath = resolveInstalledPluginIndexRecordsStorePath({ stateDir }); - fs.mkdirSync(path.dirname(indexPath), { recursive: true }); - fs.writeFileSync(indexPath, JSON.stringify({ installRecords: {}, plugins: [] }), "utf8"); - expectRecordFields(loadInstalledPluginIndexInstallRecordsSync({ stateDir }).codex, { source: "npm", spec: "@openclaw/codex@2026.5.18-beta.1", @@ -553,58 +543,6 @@ describe("plugin index install records store", () => { expect(readSpy).not.toHaveBeenCalled(); }); - it("does not cache stale async records when cache clears during load", async () => { - const stateDir = makeStateDir(); - const firstRead = createDeferred(); - const firstReadStarted = createDeferred(); - let reads = 0; - vi.doMock("../infra/json-files.js", async (importOriginal) => { - const actual = await importOriginal(); - return { - ...actual, - tryReadJson: vi.fn(async () => { - reads += 1; - if (reads === 1) { - firstReadStarted.resolve(); - return await firstRead.promise; - } - return { - installRecords: { - demo: { - source: "npm", - spec: "demo@fresh", - }, - }, - }; - }), - tryReadJsonSync: vi.fn(() => null), - }; - }); - const reader = (await import( - "./installed-plugin-index-record-reader.js?cache-race" as string - )) as typeof import("./installed-plugin-index-record-reader.js"); - const load = reader.loadInstalledPluginIndexInstallRecords({ stateDir }); - - await firstReadStarted.promise; - reader.clearLoadInstalledPluginIndexInstallRecordsCache(); - firstRead.resolve({ - installRecords: { - demo: { - source: "npm", - spec: "demo@stale", - }, - }, - }); - - await expect(load).resolves.toEqual({ - demo: { - source: "npm", - spec: "demo@fresh", - }, - }); - expect(reads).toBe(2); - }); - it("preserves git install resolution fields in persisted records", async () => { const stateDir = makeStateDir(); const candidate = createPluginCandidate(stateDir, "git-demo"); @@ -735,13 +673,8 @@ describe("plugin index install records store", () => { }); }); - it("ignores invalid persisted plugin index files", async () => { + it("returns empty records when the persisted plugin index is missing", async () => { const stateDir = makeStateDir(); - fs.mkdirSync(path.join(stateDir, "plugins"), { recursive: true }); - fs.writeFileSync( - resolveInstalledPluginIndexRecordsStorePath({ stateDir }), - JSON.stringify({ version: 999, records: {} }), - ); await expect(readPersistedInstalledPluginIndexInstallRecords({ stateDir })).resolves.toBeNull(); await expect( diff --git a/src/plugins/installed-plugin-index-store-path.ts b/src/plugins/installed-plugin-index-store-path.ts index 72001a41092d..f1d102c84a38 100644 --- a/src/plugins/installed-plugin-index-store-path.ts +++ b/src/plugins/installed-plugin-index-store-path.ts @@ -1,7 +1,8 @@ import path from "node:path"; import { resolveStateDir } from "../config/paths.js"; +import { resolveOpenClawStateSqlitePath } from "../state/openclaw-state-db.paths.js"; -const INSTALLED_PLUGIN_INDEX_STORE_PATH = path.join("plugins", "installs.json"); +const LEGACY_INSTALLED_PLUGIN_INDEX_STORE_PATH = path.join("plugins", "installs.json"); export type InstalledPluginIndexStoreOptions = { env?: NodeJS.ProcessEnv; @@ -9,13 +10,28 @@ export type InstalledPluginIndexStoreOptions = { filePath?: string; }; +function resolveStoreEnv(options: InstalledPluginIndexStoreOptions): NodeJS.ProcessEnv { + return options.stateDir + ? { ...(options.env ?? process.env), OPENCLAW_STATE_DIR: options.stateDir } + : (options.env ?? process.env); +} + export function resolveInstalledPluginIndexStorePath( options: InstalledPluginIndexStoreOptions = {}, +): string { + if (options.filePath) { + return options.filePath; + } + return resolveOpenClawStateSqlitePath(resolveStoreEnv(options)); +} + +export function resolveLegacyInstalledPluginIndexStorePath( + options: InstalledPluginIndexStoreOptions = {}, ): string { if (options.filePath) { return options.filePath; } const env = options.env ?? process.env; const stateDir = options.stateDir ?? resolveStateDir(env); - return path.join(stateDir, INSTALLED_PLUGIN_INDEX_STORE_PATH); + return path.join(stateDir, LEGACY_INSTALLED_PLUGIN_INDEX_STORE_PATH); } diff --git a/src/plugins/installed-plugin-index-store.test.ts b/src/plugins/installed-plugin-index-store.test.ts index 1534babe9cda..7bdf472114ad 100644 --- a/src/plugins/installed-plugin-index-store.test.ts +++ b/src/plugins/installed-plugin-index-store.test.ts @@ -1,6 +1,10 @@ import fs from "node:fs"; import path from "node:path"; import { afterEach, describe, expect, it } from "vitest"; +import { + closeOpenClawStateDatabaseForTest, + runOpenClawStateWriteTransaction, +} from "../state/openclaw-state-db.js"; import type { PluginCandidate } from "./discovery.js"; import { inspectPersistedInstalledPluginIndex, @@ -15,6 +19,7 @@ import { cleanupTrackedTempDirs, makeTrackedTempDir } from "./test-helpers/fs-fi const tempDirs: string[] = []; afterEach(() => { + closeOpenClawStateDatabaseForTest(); cleanupTrackedTempDirs(tempDirs); }); @@ -158,12 +163,48 @@ async function expectPersistedIndex( return persisted; } +function insertPersistedIndexRow( + stateDir: string, + values: { + version?: number; + migrationVersion?: number; + installRecordsJson?: string; + pluginsJson?: string; + diagnosticsJson?: string; + }, +) { + runOpenClawStateWriteTransaction( + ({ db }) => { + db.prepare( + ` + INSERT OR REPLACE INTO installed_plugin_index ( + index_key, version, host_contract_version, compat_registry_version, + migration_version, policy_hash, generated_at_ms, refresh_reason, + install_records_json, plugins_json, diagnostics_json, warning, updated_at_ms + ) VALUES ( + 'installed-plugin-index', @version, '2026.4.25', 'compat-v1', + @migration_version, 'policy-hash', 123, NULL, + @install_records_json, @plugins_json, @diagnostics_json, NULL, 123 + ) + `, + ).run({ + version: values.version ?? 1, + migration_version: values.migrationVersion ?? 1, + install_records_json: values.installRecordsJson ?? "{}", + plugins_json: values.pluginsJson ?? "[]", + diagnostics_json: values.diagnosticsJson ?? "[]", + }); + }, + { env: { ...process.env, OPENCLAW_STATE_DIR: stateDir } }, + ); +} + describe("installed plugin index persistence", () => { - it("resolves the persisted index path under the state plugins directory", () => { + it("resolves the persisted index path to the shared state database", () => { const stateDir = makeTempDir(); expect(resolveInstalledPluginIndexStorePath({ stateDir })).toBe( - path.join(stateDir, "plugins", "installs.json"), + path.join(stateDir, "state", "openclaw.sqlite"), ); }); @@ -174,14 +215,12 @@ describe("installed plugin index persistence", () => { await expect(writePersistedInstalledPluginIndex(index, { stateDir })).resolves.toBe(filePath); - const raw = fs.readFileSync(filePath, "utf8"); - expect(raw).toContain('"warning": "DO NOT EDIT.'); - expect(raw).toContain('"pluginId": "demo"'); if (process.platform !== "win32") { expect(fs.statSync(filePath).mode & 0o777).toBe(0o600); } const persisted = requirePersisted(await readPersistedInstalledPluginIndex({ stateDir })); expect(persisted.version).toBe(index.version); + expect(persisted.warning).toContain("DO NOT EDIT."); expect(persisted.policyHash).toBe(index.policyHash); expectPluginIds(persisted, ["demo"]); }); @@ -290,11 +329,7 @@ describe("installed plugin index persistence", () => { ...current, plugins: current.plugins.map(dropStartupConfigPaths), }; - fs.writeFileSync( - resolveInstalledPluginIndexStorePath({ stateDir }), - JSON.stringify(legacy), - "utf8", - ); + await writePersistedInstalledPluginIndex(legacy, { stateDir }); const inspection = await inspectPersistedInstalledPluginIndex({ stateDir, @@ -317,8 +352,6 @@ describe("installed plugin index persistence", () => { it("does not preserve prototype poison keys from persisted index JSON", async () => { const stateDir = makeTempDir(); - const filePath = resolveInstalledPluginIndexStorePath({ stateDir }); - fs.mkdirSync(path.dirname(filePath), { recursive: true }); const index = createIndex({ installRecords: { demo: { @@ -335,7 +368,7 @@ describe("installed plugin index persistence", () => { enumerable: true, value: { polluted: true }, }); - fs.writeFileSync(filePath, JSON.stringify(index), "utf8"); + await writePersistedInstalledPluginIndex(index, { stateDir }); const persisted = await readPersistedInstalledPluginIndex({ stateDir }); @@ -351,20 +384,14 @@ describe("installed plugin index persistence", () => { const stateDir = makeTempDir(); await expect(readPersistedInstalledPluginIndex({ stateDir })).resolves.toBeNull(); - const filePath = resolveInstalledPluginIndexStorePath({ stateDir }); - fs.mkdirSync(path.dirname(filePath), { recursive: true }); - fs.writeFileSync(filePath, JSON.stringify({ version: 999 }), "utf8"); + insertPersistedIndexRow(stateDir, { version: 999 }); await expect(readPersistedInstalledPluginIndex({ stateDir })).resolves.toBeNull(); }); it("rejects pre-migration persisted indexes so update can rebuild them", async () => { const stateDir = makeTempDir(); - const filePath = resolveInstalledPluginIndexStorePath({ stateDir }); - fs.mkdirSync(path.dirname(filePath), { recursive: true }); - const legacyIndex = createIndex(); - delete (legacyIndex as unknown as Record).migrationVersion; - fs.writeFileSync(filePath, JSON.stringify(legacyIndex), "utf8"); + insertPersistedIndexRow(stateDir, { migrationVersion: 0 }); await expect(readPersistedInstalledPluginIndex({ stateDir })).resolves.toBeNull(); }); diff --git a/src/plugins/installed-plugin-index-store.ts b/src/plugins/installed-plugin-index-store.ts index 0c71439376fb..fdd80b372f75 100644 --- a/src/plugins/installed-plugin-index-store.ts +++ b/src/plugins/installed-plugin-index-store.ts @@ -1,14 +1,18 @@ +import { existsSync, readFileSync } from "node:fs"; import { z } from "zod"; -import { saveJsonFile } from "../infra/json-file.js"; -import { tryReadJson, tryReadJsonSync, writeJson } from "../infra/json-files.js"; import { isBlockedObjectKey } from "../infra/prototype-keys.js"; +import type { OpenClawStateDatabaseOptions } from "../state/openclaw-state-db.js"; +import { + openOpenClawStateDatabase, + runOpenClawStateWriteTransaction, +} from "../state/openclaw-state-db.js"; import { safeParseWithSchema } from "../utils/zod-parse.js"; import { resolveCompatibilityHostVersion } from "../version.js"; import { normalizePluginsConfig, resolveEffectiveEnableState } from "./config-state.js"; import { isPluginEnabledByDefaultForPlatform } from "./default-enablement.js"; import { hashJson } from "./installed-plugin-index-hash.js"; import { resolveCompatRegistryVersion } from "./installed-plugin-index-policy.js"; -import { clearLoadInstalledPluginIndexInstallRecordsCache } from "./installed-plugin-index-record-reader.js"; +import { clearLoadInstalledPluginIndexInstallRecordsCache } from "./installed-plugin-index-record-cache.js"; import { resolveInstalledPluginIndexStorePath, type InstalledPluginIndexStoreOptions, @@ -32,6 +36,7 @@ import { import { clearPluginMetadataLifecycleCaches } from "./plugin-metadata-lifecycle.js"; export { resolveInstalledPluginIndexStorePath, + resolveLegacyInstalledPluginIndexStorePath, type InstalledPluginIndexStoreOptions, } from "./installed-plugin-index-store-path.js"; @@ -45,6 +50,7 @@ export type InstalledPluginIndexStoreInspection = { }; const StringArraySchema = z.array(z.string()); +const INSTALLED_PLUGIN_INDEX_SQLITE_KEY = "installed-plugin-index"; const InstalledPluginIndexStartupSchema = z.object({ sidecar: z.boolean(), @@ -144,7 +150,7 @@ function copySafeInstallRecords( return safeRecords; } -function parseInstalledPluginIndex(value: unknown): InstalledPluginIndex | null { +export function parseInstalledPluginIndex(value: unknown): InstalledPluginIndex | null { const parsed = safeParseWithSchema(InstalledPluginIndexSchema, value) as | (Omit & { installRecords?: InstalledPluginIndex["installRecords"]; @@ -174,18 +180,216 @@ function parseInstalledPluginIndex(value: unknown): InstalledPluginIndex | null }; } +type InstalledPluginIndexSqliteRow = { + version: number | bigint; + warning: string | null; + host_contract_version: string; + compat_registry_version: string; + migration_version: number | bigint; + policy_hash: string; + generated_at_ms: number | bigint; + refresh_reason: string | null; + install_records_json: string; + plugins_json: string; + diagnostics_json: string; +}; + +function resolveStateDatabaseOptions( + options: InstalledPluginIndexStoreOptions = {}, +): OpenClawStateDatabaseOptions { + if (options.filePath) { + return { + ...(options.env ? { env: options.env } : {}), + path: options.filePath, + }; + } + if (options.stateDir) { + return { + env: { + ...(options.env ?? process.env), + OPENCLAW_STATE_DIR: options.stateDir, + }, + }; + } + return options.env ? { env: options.env } : {}; +} + +function isExplicitLegacyJsonStorePath(options: InstalledPluginIndexStoreOptions): boolean { + return Boolean(options.filePath && options.filePath.endsWith(".json")); +} + +function readLegacyRecordContainer(value: unknown): unknown { + if (!value || typeof value !== "object" || Array.isArray(value)) { + return undefined; + } + const legacy = value as { installRecords?: unknown; records?: unknown }; + return legacy.installRecords ?? legacy.records; +} + +function readPersistedInstalledPluginIndexFromLegacyJson( + options: InstalledPluginIndexStoreOptions, +): InstalledPluginIndex | null { + if (!options.filePath || !existsSync(options.filePath)) { + return null; + } + try { + const parsed = JSON.parse(readFileSync(options.filePath, "utf8")) as unknown; + const current = parseInstalledPluginIndex(parsed); + if (current) { + return current; + } + const installRecords = readLegacyRecordContainer(parsed); + if (!installRecords) { + return null; + } + return parseInstalledPluginIndex({ + version: INSTALLED_PLUGIN_INDEX_VERSION, + hostContractVersion: "legacy-file", + compatRegistryVersion: "legacy-file", + migrationVersion: INSTALLED_PLUGIN_INDEX_MIGRATION_VERSION, + policyHash: "legacy-file", + generatedAtMs: 0, + installRecords, + plugins: [], + diagnostics: [], + }); + } catch { + return null; + } +} + +function assertWritableInstalledPluginIndexStoreOptions( + options: InstalledPluginIndexStoreOptions, +): void { + if (isExplicitLegacyJsonStorePath(options)) { + throw new Error( + "Explicit JSON installed plugin index paths are retired. Use the shared SQLite state DB or run openclaw doctor --fix to migrate legacy plugins/installs.json.", + ); + } +} + +function parseJsonColumn(value: string): unknown { + try { + return JSON.parse(value) as unknown; + } catch { + return undefined; + } +} + +function parseInstalledPluginIndexSqliteRow( + row: InstalledPluginIndexSqliteRow | undefined, +): InstalledPluginIndex | null { + if (!row) { + return null; + } + return parseInstalledPluginIndex({ + version: Number(row.version), + ...(row.warning ? { warning: row.warning } : {}), + hostContractVersion: row.host_contract_version, + compatRegistryVersion: row.compat_registry_version, + migrationVersion: Number(row.migration_version), + policyHash: row.policy_hash, + generatedAtMs: Number(row.generated_at_ms), + ...(row.refresh_reason ? { refreshReason: row.refresh_reason } : {}), + installRecords: parseJsonColumn(row.install_records_json), + plugins: parseJsonColumn(row.plugins_json), + diagnostics: parseJsonColumn(row.diagnostics_json), + }); +} + +function readPersistedInstalledPluginIndexFromSqlite( + options: InstalledPluginIndexStoreOptions = {}, +): InstalledPluginIndex | null { + if (isExplicitLegacyJsonStorePath(options)) { + return readPersistedInstalledPluginIndexFromLegacyJson(options); + } + if (!existsSync(resolveInstalledPluginIndexStorePath(options))) { + return null; + } + try { + const database = openOpenClawStateDatabase(resolveStateDatabaseOptions(options)); + const row = database.db + .prepare( + ` + SELECT version, warning, host_contract_version, compat_registry_version, + migration_version, policy_hash, generated_at_ms, refresh_reason, + install_records_json, plugins_json, diagnostics_json + FROM installed_plugin_index + WHERE index_key = ? + `, + ) + .get(INSTALLED_PLUGIN_INDEX_SQLITE_KEY) as InstalledPluginIndexSqliteRow | undefined; + return parseInstalledPluginIndexSqliteRow(row); + } catch { + return null; + } +} + +function writePersistedInstalledPluginIndexToSqlite( + index: InstalledPluginIndex, + options: InstalledPluginIndexStoreOptions = {}, +): void { + assertWritableInstalledPluginIndexStoreOptions(options); + const persisted = { + ...index, + warning: INSTALLED_PLUGIN_INDEX_WARNING, + installRecords: copySafeInstallRecords(index.installRecords) ?? {}, + }; + const now = Date.now(); + runOpenClawStateWriteTransaction(({ db }) => { + db.prepare( + ` + INSERT INTO installed_plugin_index ( + index_key, version, host_contract_version, compat_registry_version, + migration_version, policy_hash, generated_at_ms, refresh_reason, + install_records_json, plugins_json, diagnostics_json, warning, updated_at_ms + ) VALUES ( + @index_key, @version, @host_contract_version, @compat_registry_version, + @migration_version, @policy_hash, @generated_at_ms, @refresh_reason, + @install_records_json, @plugins_json, @diagnostics_json, @warning, @updated_at_ms + ) + ON CONFLICT(index_key) DO UPDATE SET + version = excluded.version, + host_contract_version = excluded.host_contract_version, + compat_registry_version = excluded.compat_registry_version, + migration_version = excluded.migration_version, + policy_hash = excluded.policy_hash, + generated_at_ms = excluded.generated_at_ms, + refresh_reason = excluded.refresh_reason, + install_records_json = excluded.install_records_json, + plugins_json = excluded.plugins_json, + diagnostics_json = excluded.diagnostics_json, + warning = excluded.warning, + updated_at_ms = excluded.updated_at_ms + `, + ).run({ + index_key: INSTALLED_PLUGIN_INDEX_SQLITE_KEY, + version: persisted.version, + host_contract_version: persisted.hostContractVersion, + compat_registry_version: persisted.compatRegistryVersion, + migration_version: persisted.migrationVersion, + policy_hash: persisted.policyHash, + generated_at_ms: persisted.generatedAtMs, + refresh_reason: persisted.refreshReason ?? null, + install_records_json: JSON.stringify(persisted.installRecords), + plugins_json: JSON.stringify(persisted.plugins), + diagnostics_json: JSON.stringify(persisted.diagnostics), + warning: persisted.warning, + updated_at_ms: now, + }); + }, resolveStateDatabaseOptions(options)); +} + export async function readPersistedInstalledPluginIndex( options: InstalledPluginIndexStoreOptions = {}, ): Promise { - const parsed = await tryReadJson(resolveInstalledPluginIndexStorePath(options)); - return parseInstalledPluginIndex(parsed); + return readPersistedInstalledPluginIndexFromSqlite(options); } export function readPersistedInstalledPluginIndexSync( options: InstalledPluginIndexStoreOptions = {}, ): InstalledPluginIndex | null { - const parsed = tryReadJsonSync(resolveInstalledPluginIndexStorePath(options)); - return parseInstalledPluginIndex(parsed); + return readPersistedInstalledPluginIndexFromSqlite(options); } export async function writePersistedInstalledPluginIndex( @@ -193,15 +397,7 @@ export async function writePersistedInstalledPluginIndex( options: InstalledPluginIndexStoreOptions = {}, ): Promise { const filePath = resolveInstalledPluginIndexStorePath(options); - await writeJson( - filePath, - { ...index, warning: INSTALLED_PLUGIN_INDEX_WARNING }, - { - trailingNewline: true, - dirMode: 0o700, - mode: 0o600, - }, - ); + writePersistedInstalledPluginIndexToSqlite(index, options); clearPluginMetadataLifecycleCaches(); clearLoadInstalledPluginIndexInstallRecordsCache(); return filePath; @@ -212,7 +408,7 @@ export function writePersistedInstalledPluginIndexSync( options: InstalledPluginIndexStoreOptions = {}, ): string { const filePath = resolveInstalledPluginIndexStorePath(options); - saveJsonFile(filePath, { ...index, warning: INSTALLED_PLUGIN_INDEX_WARNING }); + writePersistedInstalledPluginIndexToSqlite(index, options); clearPluginMetadataLifecycleCaches(); clearLoadInstalledPluginIndexInstallRecordsCache(); return filePath; diff --git a/src/plugins/manifest-metadata-scan.test.ts b/src/plugins/manifest-metadata-scan.test.ts index 4c1d50027285..a6c4b343d0ac 100644 --- a/src/plugins/manifest-metadata-scan.test.ts +++ b/src/plugins/manifest-metadata-scan.test.ts @@ -2,6 +2,7 @@ import fs from "node:fs"; import os from "node:os"; import path from "node:path"; import { afterEach, describe, expect, it } from "vitest"; +import { writePersistedInstalledPluginIndexSync } from "./installed-plugin-index-store.js"; import { listOpenClawPluginManifestMetadata } from "./manifest-metadata-scan.js"; const tempRoots: string[] = []; @@ -38,9 +39,36 @@ describe("listOpenClawPluginManifestMetadata", () => { id: "openai", providers: ["openai"], }); - writeJson(path.join(home, ".openclaw", "plugins", "installs.json"), { - plugins: [{ rootDir: path.join(staleBundledRoot, "openai"), origin: "bundled" }], - }); + writePersistedInstalledPluginIndexSync( + { + version: 1, + hostContractVersion: "test", + compatRegistryVersion: "test", + migrationVersion: 1, + policyHash: "test", + generatedAtMs: 1, + installRecords: {}, + plugins: [ + { + pluginId: "openai", + manifestPath: path.join(staleBundledRoot, "openai", "openclaw.plugin.json"), + manifestHash: "stale-openai", + rootDir: path.join(staleBundledRoot, "openai"), + origin: "bundled", + enabled: true, + startup: { + sidecar: false, + memory: false, + deferConfiguredChannelFullLoadUntilAfterListen: false, + agentHarnesses: [], + }, + compat: [], + }, + ], + diagnostics: [], + }, + { stateDir: path.join(home, ".openclaw") }, + ); const records = listOpenClawPluginManifestMetadata({ OPENCLAW_HOME: home, diff --git a/src/plugins/manifest-metadata-scan.ts b/src/plugins/manifest-metadata-scan.ts index 869af8fda4ad..80729495b2e8 100644 --- a/src/plugins/manifest-metadata-scan.ts +++ b/src/plugins/manifest-metadata-scan.ts @@ -5,6 +5,7 @@ import { fileURLToPath } from "node:url"; import { isRecord } from "@openclaw/normalization-core/record-coerce"; import { normalizeOptionalString as normalizeTrimmedString } from "@openclaw/normalization-core/string-coerce"; import { parseJsonWithJson5Fallback } from "../utils/parse-json-compat.js"; +import { readPersistedInstalledPluginIndexSync } from "./installed-plugin-index-store.js"; type PluginManifestMetadataRecord = { pluginDir: string; @@ -117,26 +118,23 @@ function manifestFileFingerprint(pluginDir: string): string { } function listPersistedIndexPluginDirs(env: NodeJS.ProcessEnv, startOrder: number): CandidateDir[] { - const index = readJsonObject(path.join(resolveStateDir(env), "plugins", "installs.json")); - if (!index || !Array.isArray(index.plugins)) { + const index = readPersistedInstalledPluginIndexSync({ env }); + if (!index) { return []; } const dirs: CandidateDir[] = []; let order = startOrder; - for (const rawPlugin of index.plugins) { - if (!isRecord(rawPlugin)) { - continue; - } - const rootDir = normalizeTrimmedString(rawPlugin.rootDir); + for (const plugin of index.plugins) { + const rootDir = normalizeTrimmedString(plugin.rootDir); if (!rootDir) { continue; } dirs.push({ pluginDir: resolveUserPath(rootDir, env), - rank: rawPlugin.origin === "bundled" ? 3 : 1, + rank: plugin.origin === "bundled" ? 3 : 1, order: order++, - origin: normalizeTrimmedString(rawPlugin.origin), + origin: normalizeTrimmedString(plugin.origin), }); } return dirs; diff --git a/src/plugins/manifest-model-id-normalization.test.ts b/src/plugins/manifest-model-id-normalization.test.ts index 2039ee361ece..bba4c9567fb5 100644 --- a/src/plugins/manifest-model-id-normalization.test.ts +++ b/src/plugins/manifest-model-id-normalization.test.ts @@ -9,6 +9,7 @@ import { setCurrentPluginMetadataSnapshot, } from "./current-plugin-metadata-snapshot.js"; import { resolveInstalledPluginIndexPolicyHash } from "./installed-plugin-index-policy.js"; +import { writePersistedInstalledPluginIndexSync } from "./installed-plugin-index-store.js"; import type { InstalledPluginIndex } from "./installed-plugin-index.js"; import { listOpenClawPluginManifestMetadata } from "./manifest-metadata-scan.js"; import { normalizeProviderModelIdWithManifest } from "./manifest-model-id-normalization.js"; @@ -26,37 +27,85 @@ const ORIGINAL_ENV = { const tempDirs: string[] = []; +function restoreOpenClawStateDirEnv(): void { + const value = ORIGINAL_ENV.OPENCLAW_STATE_DIR; + if (value === undefined) { + delete process.env.OPENCLAW_STATE_DIR; + } else { + process.env.OPENCLAW_STATE_DIR = value; + } +} + +function restoreOpenClawHomeEnv(): void { + const value = ORIGINAL_ENV.OPENCLAW_HOME; + if (value === undefined) { + delete process.env.OPENCLAW_HOME; + } else { + process.env.OPENCLAW_HOME = value; + } +} + +function restoreOpenClawDisableBundledPluginsEnv(): void { + const value = ORIGINAL_ENV.OPENCLAW_DISABLE_BUNDLED_PLUGINS; + if (value === undefined) { + delete process.env.OPENCLAW_DISABLE_BUNDLED_PLUGINS; + } else { + process.env.OPENCLAW_DISABLE_BUNDLED_PLUGINS = value; + } +} + +function restoreOpenClawBundledPluginsDirEnv(): void { + const value = ORIGINAL_ENV.OPENCLAW_BUNDLED_PLUGINS_DIR; + if (value === undefined) { + delete process.env.OPENCLAW_BUNDLED_PLUGINS_DIR; + } else { + process.env.OPENCLAW_BUNDLED_PLUGINS_DIR = value; + } +} + +function restoreEnv(): void { + restoreOpenClawStateDirEnv(); + restoreOpenClawHomeEnv(); + restoreOpenClawDisableBundledPluginsEnv(); + restoreOpenClawBundledPluginsDirEnv(); +} + function makeTempDir(): string { const dir = fs.mkdtempSync(path.join(os.tmpdir(), "openclaw-model-id-normalization-")); tempDirs.push(dir); return dir; } -function restoreEnv(): void { - for (const [key, value] of Object.entries(ORIGINAL_ENV)) { - if (value === undefined) { - delete process.env[key]; - } else { - process.env[key] = value; - } - } -} - function writeInstallIndex(params: { stateDir: string; pluginDir: string }): void { - const indexPath = path.join(params.stateDir, "plugins", "installs.json"); - fs.mkdirSync(path.dirname(indexPath), { recursive: true }); - fs.writeFileSync( - indexPath, - JSON.stringify({ + writePersistedInstalledPluginIndexSync( + { + version: 1, + hostContractVersion: "test", + compatRegistryVersion: "test", + migrationVersion: 1, + policyHash: "test", + generatedAtMs: 1, + installRecords: {}, plugins: [ { - id: "normalizer", + pluginId: "normalizer", + manifestPath: path.join(params.pluginDir, "openclaw.plugin.json"), + manifestHash: "normalizer-manifest", rootDir: params.pluginDir, origin: "global", + enabled: true, + startup: { + sidecar: false, + memory: false, + deferConfiguredChannelFullLoadUntilAfterListen: false, + agentHarnesses: [], + }, + compat: [], }, ], - }), - "utf-8", + diagnostics: [], + }, + { stateDir: params.stateDir }, ); } diff --git a/src/plugins/plugin-metadata-snapshot.memo.test.ts b/src/plugins/plugin-metadata-snapshot.memo.test.ts index 67da72ae761c..65feefb130f8 100644 --- a/src/plugins/plugin-metadata-snapshot.memo.test.ts +++ b/src/plugins/plugin-metadata-snapshot.memo.test.ts @@ -2,12 +2,17 @@ import fs from "node:fs"; import os from "node:os"; import path from "node:path"; import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; +import { runOpenClawStateWriteTransaction } from "../state/openclaw-state-db.js"; import { clearCurrentPluginMetadataSnapshot, setCurrentPluginMetadataSnapshot, } from "./current-plugin-metadata-snapshot.js"; import { resolveInstalledPluginIndexPolicyHash } from "./installed-plugin-index-policy.js"; -import type { InstalledPluginIndex } from "./installed-plugin-index.js"; +import { writePersistedInstalledPluginIndexSync } from "./installed-plugin-index-store.js"; +import type { + InstalledPluginIndex, + InstalledPluginInstallRecordInfo, +} from "./installed-plugin-index.js"; import type { PluginManifestRecord, PluginManifestRegistry } from "./manifest-registry.js"; import { clearPluginMetadataLifecycleCaches } from "./plugin-metadata-lifecycle.js"; import { @@ -46,9 +51,27 @@ function tempStateDir(): string { } function touchPersistedIndex(stateDir: string, value = 1): void { - const indexPath = path.join(stateDir, "plugins", "installs.json"); - fs.mkdirSync(path.dirname(indexPath), { recursive: true }); - fs.writeFileSync(indexPath, JSON.stringify({ value })); + runOpenClawStateWriteTransaction( + ({ db }) => { + db.prepare( + ` + INSERT OR REPLACE INTO installed_plugin_index ( + index_key, version, host_contract_version, compat_registry_version, + migration_version, policy_hash, generated_at_ms, refresh_reason, + install_records_json, plugins_json, diagnostics_json, warning, updated_at_ms + ) VALUES ( + 'installed-plugin-index', 1, 'test', 'test', + 1, @policy_hash, @generated_at_ms, NULL, + '{}', '[]', '[]', NULL, @generated_at_ms + ) + `, + ).run({ + policy_hash: `test-${value}`, + generated_at_ms: value, + }); + }, + { env: { ...process.env, OPENCLAW_STATE_DIR: stateDir } }, + ); } function writeJson(filePath: string, value: unknown): void { @@ -67,36 +90,39 @@ function writePersistedIndex(params: { const pluginDir = path.join(params.stateDir, "extensions", params.pluginId); const manifestPath = params.manifestPath ?? path.join(pluginDir, "openclaw.plugin.json"); const packageJsonPath = params.packageJsonPath ?? path.join(pluginDir, "package.json"); - writeJson(path.join(params.stateDir, "plugins", "installs.json"), { - version: 1, - hostContractVersion: "test", - compatRegistryVersion: "test", - migrationVersion: 1, - policyHash: "test", - generatedAtMs: 1, - installRecords: {}, - diagnostics: [], - plugins: [ - { - pluginId: params.pluginId, - manifestPath, - manifestHash: `${params.pluginId}-manifest`, - rootDir: pluginDir, - ...(params.source ? { source: params.source } : {}), - ...(params.setupSource ? { setupSource: params.setupSource } : {}), - origin: "global", - enabled: true, - packageJson: { path: "package.json", hash: `${params.pluginId}-package` }, - startup: { - sidecar: false, - memory: false, - deferConfiguredChannelFullLoadUntilAfterListen: false, - agentHarnesses: [], + writePersistedInstalledPluginIndexSync( + { + version: 1, + hostContractVersion: "test", + compatRegistryVersion: "test", + migrationVersion: 1, + policyHash: "test", + generatedAtMs: 1, + installRecords: {}, + diagnostics: [], + plugins: [ + { + pluginId: params.pluginId, + manifestPath, + manifestHash: `${params.pluginId}-manifest`, + rootDir: pluginDir, + ...(params.source ? { source: params.source } : {}), + ...(params.setupSource ? { setupSource: params.setupSource } : {}), + origin: "global", + enabled: true, + packageJson: { path: "package.json", hash: `${params.pluginId}-package` }, + startup: { + sidecar: false, + memory: false, + deferConfiguredChannelFullLoadUntilAfterListen: false, + agentHarnesses: [], + }, + compat: [], }, - compat: [], - }, - ], - }); + ], + }, + { stateDir: params.stateDir }, + ); writeJson(manifestPath, { id: params.pluginId }); writeJson(packageJsonPath, { name: params.pluginId }); } @@ -128,19 +154,22 @@ function writeRecoverableNpmPlugin(params: { function writePersistedInstallRecords( stateDir: string, - installRecords: Record>, + installRecords: Record, ): void { - writeJson(path.join(stateDir, "plugins", "installs.json"), { - version: 1, - hostContractVersion: "test", - compatRegistryVersion: "test", - migrationVersion: 1, - policyHash: "test", - generatedAtMs: 1, - installRecords, - diagnostics: [], - plugins: [], - }); + writePersistedInstalledPluginIndexSync( + { + version: 1, + hostContractVersion: "test", + compatRegistryVersion: "test", + migrationVersion: 1, + policyHash: "test", + generatedAtMs: 1, + installRecords, + diagnostics: [], + plugins: [], + }, + { stateDir }, + ); } function makeIndex( @@ -650,13 +679,15 @@ describe("loadPluginMetadataSnapshot process memo", () => { [ "install path package manifest", "~/tracked-plugin", - (recordPath: string) => ({ source: "path", installPath: recordPath }), + (recordPath: string) => + ({ source: "path", installPath: recordPath }) satisfies InstalledPluginInstallRecordInfo, (homeDir: string) => path.join(homeDir, "tracked-plugin", "package.json"), ], [ "source path package manifest", "~/tracked-plugin", - (recordPath: string) => ({ source: "path", sourcePath: recordPath }), + (recordPath: string) => + ({ source: "path", sourcePath: recordPath }) satisfies InstalledPluginInstallRecordInfo, (homeDir: string) => path.join(homeDir, "tracked-plugin", "package.json"), ], ])( @@ -861,33 +892,36 @@ describe("loadPluginMetadataSnapshot process memo", () => { fs.mkdirSync(pluginDir, { recursive: true }); writeJson(outsideManifestPath, { id: "outside" }); fs.symlinkSync(outsideManifestPath, manifestPath); - writeJson(path.join(stateDir, "plugins", "installs.json"), { - version: 1, - hostContractVersion: "test", - compatRegistryVersion: "test", - migrationVersion: 1, - policyHash: "test", - generatedAtMs: 1, - installRecords: {}, - diagnostics: [], - plugins: [ - { - pluginId: "demo", - manifestPath, - manifestHash: "demo-manifest", - rootDir: pluginDir, - origin: "global", - enabled: true, - startup: { - sidecar: false, - memory: false, - deferConfiguredChannelFullLoadUntilAfterListen: false, - agentHarnesses: [], + writePersistedInstalledPluginIndexSync( + { + version: 1, + hostContractVersion: "test", + compatRegistryVersion: "test", + migrationVersion: 1, + policyHash: "test", + generatedAtMs: 1, + installRecords: {}, + diagnostics: [], + plugins: [ + { + pluginId: "demo", + manifestPath, + manifestHash: "demo-manifest", + rootDir: pluginDir, + origin: "global", + enabled: true, + startup: { + sidecar: false, + memory: false, + deferConfiguredChannelFullLoadUntilAfterListen: false, + agentHarnesses: [], + }, + compat: [], }, - compat: [], - }, - ], - }); + ], + }, + { stateDir }, + ); loadPluginRegistrySnapshotWithMetadata.mockReturnValue({ source: "persisted", snapshot: makeIndex(), diff --git a/src/plugins/plugin-metadata-snapshot.ts b/src/plugins/plugin-metadata-snapshot.ts index a9d5d645e306..860f28a679e8 100644 --- a/src/plugins/plugin-metadata-snapshot.ts +++ b/src/plugins/plugin-metadata-snapshot.ts @@ -14,7 +14,7 @@ import { getCurrentPluginMetadataSnapshot } from "./current-plugin-metadata-snap import { resolveDefaultPluginNpmDir, resolvePluginNpmProjectsDir } from "./install-paths.js"; import { hashJson } from "./installed-plugin-index-hash.js"; import { resolveInstalledPluginIndexPolicyHash } from "./installed-plugin-index-policy.js"; -import { resolveInstalledPluginIndexStorePath } from "./installed-plugin-index-store-path.js"; +import { readPersistedInstalledPluginIndexSync } from "./installed-plugin-index-store.js"; import type { InstalledPluginIndex } from "./installed-plugin-index.js"; import { loadPluginManifestRegistryForInstalledIndex, @@ -111,15 +111,6 @@ function directoryChildPackageJsonFingerprint(directoryPath: string): unknown { ]; } -function readJsonObject(filePath: string): Record | undefined { - try { - const parsed = JSON.parse(fs.readFileSync(filePath, "utf8")); - return isRecord(parsed) ? parsed : undefined; - } catch { - return undefined; - } -} - function stableMemoValue(value: unknown): unknown { if (Array.isArray(value)) { return value.map(stableMemoValue); @@ -207,15 +198,18 @@ function resolvePersistedRegistryFastMemoFingerprint(params: { if (disabled) { return { disabled: true }; } - const indexPath = resolveInstalledPluginIndexStorePath({ - env: params.env, - ...(params.stateDir ? { stateDir: params.stateDir } : {}), - }); const npmRoot = params.stateDir ? path.join(params.stateDir, "npm") : resolveDefaultPluginNpmDir(params.env); return { - index: fileFingerprint(indexPath), + index: hashJson( + stableMemoValue( + readPersistedInstalledPluginIndexSync({ + env: params.env, + ...(params.stateDir ? { stateDir: params.stateDir } : {}), + }), + ) ?? null, + ), npmPackageJson: fileFingerprint(path.join(npmRoot, "package.json")), npmProjectPackageJsons: directoryChildPackageJsonFingerprint( resolvePluginNpmProjectsDir(npmRoot), @@ -268,11 +262,12 @@ function resolvePersistedRegistryMemoState(params: { fingerprint: fastFingerprint, }; } - const indexPath = resolveInstalledPluginIndexStorePath({ - env: params.env, - ...(params.stateDir ? { stateDir: params.stateDir } : {}), - }); - const index = params.index ?? readJsonObject(indexPath); + const index = + params.index ?? + readPersistedInstalledPluginIndexSync({ + env: params.env, + ...(params.stateDir ? { stateDir: params.stateDir } : {}), + }); return { contextHash, fastHash, diff --git a/src/plugins/plugin-registry-snapshot.ts b/src/plugins/plugin-registry-snapshot.ts index ebbc78cb8af4..f15e31983141 100644 --- a/src/plugins/plugin-registry-snapshot.ts +++ b/src/plugins/plugin-registry-snapshot.ts @@ -10,7 +10,6 @@ import type { PluginDiscoveryResult } from "./discovery.js"; import { fileSignatureMatches, hashJson } from "./installed-plugin-index-hash.js"; import { hasOptionalMissingPluginManifestFile } from "./installed-plugin-index-manifest.js"; import { loadInstalledPluginIndexInstallRecordsSync } from "./installed-plugin-index-record-reader.js"; -import { resolveInstalledPluginIndexStorePath } from "./installed-plugin-index-store-path.js"; import { inspectPersistedInstalledPluginIndex, readPersistedInstalledPluginIndexSync, @@ -137,6 +136,16 @@ function resolvePluginRegistrySnapshotMemoKey( if (!canMemoizePluginRegistrySnapshot(params)) { return undefined; } + const persistedReadsEnabled = + params.preferPersisted !== false && !hasEnvFlag(env, DISABLE_PERSISTED_PLUGIN_REGISTRY_ENV); + const persistedRegistryFingerprint = persistedReadsEnabled + ? hashJson( + readPersistedInstalledPluginIndexSync({ + env, + ...(params.stateDir ? { stateDir: params.stateDir } : {}), + }), + ) + : "disabled"; return hashJson({ config: params.config ?? null, cwd: process.cwd(), @@ -145,12 +154,7 @@ function resolvePluginRegistrySnapshotMemoKey( preferPersisted: params.preferPersisted ?? null, // Plugin manifests are process-stable inside the Gateway, while the persisted // registry envelope can change through explicit refresh/install flows. - registryFile: fileFingerprint( - resolveInstalledPluginIndexStorePath({ - env, - ...(params.stateDir ? { stateDir: params.stateDir } : {}), - }), - ), + registry: persistedRegistryFingerprint, pluginRoots: fingerprintPluginSourceRoots(params, env), stateDir: params.stateDir ? resolveUserPath(params.stateDir, env) : null, workspaceDir: params.workspaceDir ? resolveUserPath(params.workspaceDir, env) : null, @@ -441,7 +445,7 @@ export function loadPluginRegistrySnapshotWithMetadata( const disabledByCaller = params.preferPersisted === false; const disabledByEnv = hasEnvFlag(env, DISABLE_PERSISTED_PLUGIN_REGISTRY_ENV); const persistedReadsEnabled = !disabledByCaller && !disabledByEnv; - const persistedInstallRecordReadsEnabled = !disabledByEnv; + const persistedInstallRecordReadsEnabled = persistedReadsEnabled; let persistedIndex: InstalledPluginIndex | null; if (persistedInstallRecordReadsEnabled) { persistedIndex = readPersistedInstalledPluginIndexSync(params); diff --git a/src/plugins/plugin-registry.test.ts b/src/plugins/plugin-registry.test.ts index 344971c27119..021e0c693910 100644 --- a/src/plugins/plugin-registry.test.ts +++ b/src/plugins/plugin-registry.test.ts @@ -2,8 +2,11 @@ import crypto from "node:crypto"; import fs from "node:fs"; import path from "node:path"; import { afterEach, describe, expect, it, vi } from "vitest"; +import { + closeOpenClawStateDatabaseForTest, + runOpenClawStateWriteTransaction, +} from "../state/openclaw-state-db.js"; import type { PluginCandidate } from "./discovery.js"; -import { resolveInstalledPluginIndexStorePath } from "./installed-plugin-index-store-path.js"; import { readPersistedInstalledPluginIndex, writePersistedInstalledPluginIndex, @@ -41,6 +44,7 @@ import { cleanupTrackedTempDirs, makeTrackedTempDir } from "./test-helpers/fs-fi const tempDirs: string[] = []; afterEach(() => { + closeOpenClawStateDatabaseForTest(); clearPluginMetadataLifecycleCaches(); cleanupTrackedTempDirs(tempDirs); }); @@ -820,12 +824,26 @@ describe("plugin registry facade", () => { const env = hermeticEnv(); await writePersistedInstalledPluginIndex(createPersistableIndex("first"), { stateDir }); const first = loadPluginRegistrySnapshotWithMetadata({ stateDir, env }); - const filePath = resolveInstalledPluginIndexStorePath({ stateDir, env }); - - fs.writeFileSync( - filePath, - JSON.stringify(createPersistableIndex("second-external"), null, 2), - "utf8", + const external = createPersistableIndex("second-external"); + runOpenClawStateWriteTransaction( + ({ db }) => { + db.prepare( + ` + UPDATE installed_plugin_index + SET plugins_json = ?, + install_records_json = ?, + diagnostics_json = ?, + updated_at_ms = ? + WHERE index_key = 'installed-plugin-index' + `, + ).run( + JSON.stringify(external.plugins), + JSON.stringify(external.installRecords), + JSON.stringify(external.diagnostics), + Date.now(), + ); + }, + { env: { ...env, OPENCLAW_STATE_DIR: stateDir } }, ); const second = loadPluginRegistrySnapshotWithMetadata({ stateDir, env }); @@ -855,7 +873,7 @@ describe("plugin registry facade", () => { expectSnapshotPluginIds(result.snapshot, ["demo"]); }); - it("derives a fresh registry without dropping persisted install records", async () => { + it("derives a fresh registry without persisted install records when caller disables persisted reads", async () => { const stateDir = makeTempDir(); const rootDir = makeTempDir(); const candidate = createCandidate(rootDir); @@ -881,10 +899,7 @@ describe("plugin registry facade", () => { expect(result.source).toBe("derived"); expectSnapshotPluginIds(result.snapshot, ["demo"]); - expectInstallRecord(result.snapshot.installRecords, "persisted", { - source: "npm", - spec: "persisted-plugin@1.0.0", - }); + expect(result.snapshot.installRecords).not.toHaveProperty("persisted"); }); it("exposes explicit persisted registry inspect and refresh operations", async () => { diff --git a/src/security/audit-plugins-trust.test.ts b/src/security/audit-plugins-trust.test.ts index 4f75262642af..26972e16908c 100644 --- a/src/security/audit-plugins-trust.test.ts +++ b/src/security/audit-plugins-trust.test.ts @@ -4,6 +4,7 @@ import path from "node:path"; import { afterAll, beforeAll, describe, expect, it, vi } from "vitest"; import type { OpenClawConfig } from "../config/config.js"; import type { PluginInstallRecord } from "../config/types.plugins.js"; +import { writePersistedInstalledPluginIndex } from "../plugins/installed-plugin-index-store.js"; import type { InstalledPluginIndex } from "../plugins/installed-plugin-index.js"; import { createPathResolutionEnv, withEnvAsync } from "../test-utils/env.js"; @@ -205,9 +206,7 @@ describe("security audit install metadata findings", () => { })), diagnostics: [], }; - const filePath = path.join(stateDir, "plugins", "installs.json"); - await fs.mkdir(path.dirname(filePath), { recursive: true, mode: 0o700 }); - await fs.writeFile(filePath, `${JSON.stringify(index, null, 2)}\n`, { mode: 0o600 }); + await writePersistedInstalledPluginIndex(index, { stateDir }); }; beforeAll(async () => { diff --git a/test/scripts/docker-build-helper.test.ts b/test/scripts/docker-build-helper.test.ts index c8316cc600c3..7ad79ba30486 100644 --- a/test/scripts/docker-build-helper.test.ts +++ b/test/scripts/docker-build-helper.test.ts @@ -1552,7 +1552,9 @@ test -f "$TMPDIR/docker-cmd-seen" expect(runner).toContain("scripts/e2e/lib/plugin-update/unchanged-scenario.sh"); expect(probe).toContain("plugin install record changed unexpectedly"); - expect(probe).toContain("index.installRecords ?? index.records ?? config.plugins?.installs"); + expect(probe).toContain( + "readPluginInstallRecords({ fallbackRecords: config.plugins?.installs ?? {} })", + ); expect(scenario).toContain("Config changed unexpectedly for modern package"); expect(scenario).not.toContain("before_hash"); }); diff --git a/test/scripts/live-docker-stage.test.ts b/test/scripts/live-docker-stage.test.ts index 3f542be5f983..528070cebe2b 100644 --- a/test/scripts/live-docker-stage.test.ts +++ b/test/scripts/live-docker-stage.test.ts @@ -19,6 +19,10 @@ describe("live Docker state staging", () => { expect(script).toContain("--exclude=workspace"); expect(script).toContain("--exclude=sandboxes"); expect(script).toContain("--exclude=plugins/installs.json"); + expect(script).toContain("--exclude=plugins/installs.json.migrated"); + expect(script).toContain("DELETE FROM installed_plugin_index"); + expect(script).toContain("PRAGMA secure_delete = ON"); + expect(script).toContain("VACUUM"); expect(script).toContain("host-absolute paths"); }); }); diff --git a/test/scripts/live-plugin-tool-assertions.test.ts b/test/scripts/live-plugin-tool-assertions.test.ts index ec1d2319b436..c6f7fd1097f9 100644 --- a/test/scripts/live-plugin-tool-assertions.test.ts +++ b/test/scripts/live-plugin-tool-assertions.test.ts @@ -5,6 +5,14 @@ import path from "node:path"; import { describe, expect, it } from "vitest"; const ASSERTIONS_SCRIPT = "scripts/e2e/lib/live-plugin-tool/assertions.mjs"; +const DISABLE_EXPERIMENTAL_WARNING = "--disable-warning=ExperimentalWarning"; + +function nodeOptionsWithoutExperimentalWarnings(extra?: string): string { + const current = [process.env.NODE_OPTIONS, extra].filter(Boolean).join(" "); + return current.includes(DISABLE_EXPERIMENTAL_WARNING) + ? current + : [current, DISABLE_EXPERIMENTAL_WARNING].filter(Boolean).join(" "); +} function writeJson(filePath: string, value: unknown) { mkdirSync(path.dirname(filePath), { recursive: true }); @@ -32,6 +40,7 @@ function runAssertionCommand(command: string, root: string, env: Record