From e16ac0433092da879e88213a47d6f4dae0432a6e Mon Sep 17 00:00:00 2001 From: Peter Steinberger Date: Wed, 3 Jun 2026 16:14:15 -0700 Subject: [PATCH] refactor(auth): store auth profiles in sqlite (#89102) --- docs/concepts/model-failover.md | 14 +- docs/gateway/authentication.md | 10 +- scripts/deadcode-unused-files.allowlist.mjs | 3 - src/agents/agent-auth-discovery-core.ts | 47 - .../agent-auth-discovery.external-cli.test.ts | 1 - src/agents/agent-auth-discovery.ts | 5 +- src/agents/agent-auth-json.test.ts | 256 --- src/agents/agent-auth-json.ts | 82 - src/agents/agent-dir-registry.ts | 27 + src/agents/agent-model-discovery.auth.test.ts | 73 +- ...ent-model-discovery.synthetic-auth.test.ts | 1 - src/agents/agent-model-discovery.ts | 6 - src/agents/agent-scope-config.ts | 9 +- src/agents/auth-profiles.chutes.test.ts | 15 +- ...th-profiles.ensureauthprofilestore.test.ts | 1201 -------------- ...th-profiles.markauthprofilefailure.test.ts | 52 +- .../auth-profiles.readonly-sync.test.ts | 42 +- src/agents/auth-profiles.sqlite-store.test.ts | 251 +++ src/agents/auth-profiles.store-cache.test.ts | 433 ----- src/agents/auth-profiles.store.save.test.ts | 1421 ----------------- src/agents/auth-profiles.ts | 1 + src/agents/auth-profiles/constants.ts | 2 +- ...-external-auth-passthrough.test-support.ts | 1 + src/agents/auth-profiles/oauth-manager.ts | 339 ++-- src/agents/auth-profiles/oauth-test-utils.ts | 15 + .../oauth.adopt-identity.test.ts | 13 +- .../oauth.fallback-to-main-agent.test.ts | 26 +- .../oauth.mirror-refresh.test.ts | 25 +- ...auth.openai-codex-refresh-fallback.test.ts | 28 +- src/agents/auth-profiles/path-resolve.ts | 5 +- .../auth-profiles/paths-direct-import.test.ts | 56 +- src/agents/auth-profiles/paths.ts | 15 - src/agents/auth-profiles/persisted.ts | 15 +- src/agents/auth-profiles/profiles.test.ts | 138 +- src/agents/auth-profiles/source-check.ts | 31 +- src/agents/auth-profiles/sqlite.ts | 248 +++ src/agents/auth-profiles/state.ts | 34 +- src/agents/auth-profiles/store-cache.ts | 50 - .../store.runtime-external.test.ts | 105 -- src/agents/auth-profiles/store.ts | 312 ++-- src/agents/auth-profiles/upsert-with-lock.ts | 4 - .../command/attempt-execution.cli.test.ts | 36 +- .../model-discovery-cache.ts | 4 +- .../embedded-agent-runner/model.test.ts | 21 +- ...els-config.applies-config-env-vars.test.ts | 26 +- src/agents/models-config.ts | 8 +- src/auto-reply/reply/commands-status.test.ts | 64 +- src/commands/agents.add.test.ts | 127 +- src/commands/agents.commands.add.ts | 37 +- ...octor-auth-canonical-api-key-alias.test.ts | 22 +- .../doctor-auth-flat-profiles.test.ts | 375 ++++- src/commands/doctor-auth-flat-profiles.ts | 455 +++++- .../doctor-auth-oauth-sidecar.test.ts | 18 +- .../doctor-auth.profile-health.test.ts | 50 + src/commands/doctor-auth.ts | 15 +- src/commands/doctor.fast-path-mocks.ts | 4 + src/commands/doctor/repair-sequencing.test.ts | 25 +- src/commands/doctor/repair-sequencing.ts | 25 +- .../stale-oauth-profile-shadows.test.ts | 94 +- .../shared/stale-oauth-profile-shadows.ts | 89 +- src/commands/models/auth-list.test.ts | 10 +- src/commands/models/auth-list.ts | 2 +- src/commands/models/auth-order.ts | 6 +- src/infra/backup-create.test.ts | 69 + src/infra/backup-create.ts | 77 +- src/infra/channel-runtime-context.ts | 7 +- .../runner.local-no-auth.test.ts | 10 +- src/secrets/apply.test.ts | 99 +- src/secrets/apply.ts | 111 +- src/secrets/audit.test.ts | 62 +- src/secrets/audit.ts | 38 +- src/secrets/auth-store-paths.ts | 15 +- src/secrets/runtime-fast-path.ts | 2 + src/secrets/runtime-state.test.ts | 56 - src/secrets/runtime-state.ts | 2 - src/secrets/runtime.fast-path.test.ts | 13 +- src/secrets/storage-scan.ts | 11 +- src/security/audit-extra.async.test.ts | 30 + src/security/audit-extra.async.ts | 79 +- src/security/fix.test.ts | 7 + src/security/fix.ts | 4 + src/state/openclaw-agent-db.generated.d.ts | 14 + src/state/openclaw-agent-schema.generated.ts | 14 +- src/state/openclaw-agent-schema.sql | 12 + src/test-utils/openclaw-test-state.test.ts | 12 +- src/test-utils/openclaw-test-state.ts | 11 +- test/scripts/lint-suppressions.test.ts | 1 - 87 files changed, 2832 insertions(+), 4849 deletions(-) delete mode 100644 src/agents/agent-auth-json.test.ts delete mode 100644 src/agents/agent-auth-json.ts create mode 100644 src/agents/agent-dir-registry.ts delete mode 100644 src/agents/auth-profiles.ensureauthprofilestore.test.ts create mode 100644 src/agents/auth-profiles.sqlite-store.test.ts delete mode 100644 src/agents/auth-profiles.store-cache.test.ts delete mode 100644 src/agents/auth-profiles.store.save.test.ts create mode 100644 src/agents/auth-profiles/sqlite.ts delete mode 100644 src/agents/auth-profiles/store-cache.ts delete mode 100644 src/agents/auth-profiles/store.runtime-external.test.ts diff --git a/docs/concepts/model-failover.md b/docs/concepts/model-failover.md index a27eecfbf894..3a3641979982 100644 --- a/docs/concepts/model-failover.md +++ b/docs/concepts/model-failover.md @@ -108,10 +108,10 @@ These notices are operational messages, not assistant content. They are delivere OpenClaw uses **auth profiles** for both API keys and OAuth tokens. -- Secrets live in `~/.openclaw/agents//agent/auth-profiles.json` (legacy: `~/.openclaw/agent/auth-profiles.json`). -- Runtime auth-routing state lives in `~/.openclaw/agents//agent/auth-state.json`. +- Secrets and runtime auth-routing state live in `~/.openclaw/agents//agent/openclaw-agent.sqlite`. - Config `auth.profiles` / `auth.order` are **metadata + routing only** (no secrets). -- Legacy import-only OAuth file: `~/.openclaw/credentials/oauth.json` (imported into `auth-profiles.json` on first use). +- Legacy import-only OAuth file: `~/.openclaw/credentials/oauth.json` (imported into the per-agent auth store on first use). +- Legacy `auth-profiles.json`, `auth-state.json`, and per-agent `auth.json` files are imported by `openclaw doctor --fix`. More detail: [OAuth](/concepts/oauth) @@ -127,7 +127,7 @@ OAuth logins create distinct profiles so multiple accounts can coexist. - Default: `provider:default` when no email is available. - OAuth with email: `provider:` (for example `google-antigravity:user@gmail.com`). -Profiles live in `~/.openclaw/agents//agent/auth-profiles.json` under `profiles`. +Profiles live in the per-agent `openclaw-agent.sqlite` auth profile store. ## Rotation order @@ -141,7 +141,7 @@ When a provider has multiple profiles, OpenClaw chooses an order like this: `auth.profiles` filtered by provider. - Entries in `auth-profiles.json` for the provider. + Per-agent SQLite auth profile entries for the provider. @@ -229,7 +229,7 @@ Cooldowns use exponential backoff: - 25 minutes - 1 hour (cap) -State is stored in `auth-state.json` under `usageStats`: +State is stored in the per-agent SQLite auth state under `usageStats`: ```json { @@ -253,7 +253,7 @@ Not every billing-shaped response is `402`, and not every HTTP `402` lands here. Meanwhile temporary `402` usage-window and organization/workspace spend-limit errors are classified as `rate_limit` when the message looks retryable (for example `weekly usage limit exhausted`, `daily limit reached, resets tomorrow`, or `organization spending limit exceeded`). Those stay on the short cooldown/failover path instead of the long billing-disable path. -State is stored in `auth-state.json`: +State is stored in the per-agent SQLite auth state: ```json { diff --git a/docs/gateway/authentication.md b/docs/gateway/authentication.md index de855142a6fa..5a5688cd70a6 100644 --- a/docs/gateway/authentication.md +++ b/docs/gateway/authentication.md @@ -87,13 +87,13 @@ This is a two-step setup: If `claude` is not on `PATH`, either install Claude Code first or set `agents.defaults.cliBackends.claude-cli.command` to the real binary path. -Manual token entry (any provider; writes `auth-profiles.json` + updates config): +Manual token entry (any provider; writes the per-agent SQLite auth store + updates config): ```bash openclaw models auth paste-token --provider openrouter ``` -`auth-profiles.json` stores credentials only. The canonical shape is: +The auth profile store keeps credentials only. Legacy `auth-profiles.json` files used this canonical shape: ```json { @@ -108,9 +108,9 @@ openclaw models auth paste-token --provider openrouter } ``` -OpenClaw expects the canonical `version` + `profiles` shape at runtime. If an older install still has a flat file such as `{ "openrouter": { "apiKey": "..." } }`, run `openclaw doctor --fix` to rewrite it as an `openrouter:default` API-key profile; doctor keeps a `.legacy-flat.*.bak` copy beside the original. Endpoint details such as `baseUrl`, `api`, model ids, headers, and timeouts belong under `models.providers.` in `openclaw.json` or `models.json`, not in `auth-profiles.json`. +OpenClaw now reads auth profiles from each agent's `openclaw-agent.sqlite`. If an older install still has `auth-profiles.json`, `auth-state.json`, or a flat auth profile file such as `{ "openrouter": { "apiKey": "..." } }`, run `openclaw doctor --fix` to import it into SQLite; doctor keeps timestamped backups beside the original JSON files. Endpoint details such as `baseUrl`, `api`, model ids, headers, and timeouts belong under `models.providers.` in `openclaw.json` or `models.json`, not in auth profiles. -External auth routes such as Bedrock `auth: "aws-sdk"` are also not credentials. If you want a named Bedrock route, put `auth.profiles..mode: "aws-sdk"` in `openclaw.json`; do not write `type: "aws-sdk"` into `auth-profiles.json`. `openclaw doctor --fix` moves legacy AWS SDK markers from the credential store into config metadata. +External auth routes such as Bedrock `auth: "aws-sdk"` are also not credentials. If you want a named Bedrock route, put `auth.profiles..mode: "aws-sdk"` in `openclaw.json`; do not write `type: "aws-sdk"` into the auth profile store. `openclaw doctor --fix` moves legacy AWS SDK markers from the credential store into config metadata. Auth profile refs are also supported for static credentials: @@ -225,7 +225,7 @@ Use `/model` (or `/model list`) for a compact picker; use `/model status` for th ### Per-agent (CLI override) -Set an explicit auth profile order override for an agent (stored in that agent's `auth-state.json`): +Set an explicit auth profile order override for an agent (stored in that agent's SQLite auth state): ```bash openclaw models auth order get --provider anthropic diff --git a/scripts/deadcode-unused-files.allowlist.mjs b/scripts/deadcode-unused-files.allowlist.mjs index 7cb6ee9b92d8..7c02317053aa 100644 --- a/scripts/deadcode-unused-files.allowlist.mjs +++ b/scripts/deadcode-unused-files.allowlist.mjs @@ -6,9 +6,6 @@ export const KNIP_UNUSED_FILE_ALLOWLIST = [ // The pending SQLite session/runtime branch wires these files into production. "src/agents/cache/agent-cache-store.sqlite.ts", "src/agents/cache/agent-cache-store.ts", - "src/state/openclaw-agent-db.paths.ts", - "src/state/openclaw-agent-db.ts", - "src/state/openclaw-agent-schema.generated.ts", ]; // Knip can disagree across supported local/CI platforms for files that are diff --git a/src/agents/agent-auth-discovery-core.ts b/src/agents/agent-auth-discovery-core.ts index 23ec78ef179d..08b5874aeee0 100644 --- a/src/agents/agent-auth-discovery-core.ts +++ b/src/agents/agent-auth-discovery-core.ts @@ -1,8 +1,4 @@ -import fs from "node:fs"; import type { OpenClawConfig } from "../config/types.openclaw.js"; -import { tryReadJsonSync } from "../infra/json-files.js"; -import { replaceFileAtomicSync } from "../infra/replace-file.js"; -import { isRecord } from "../utils.js"; import type { AgentCredentialMap } from "./agent-auth-credentials.js"; import { listProviderEnvAuthLookupKeys, @@ -56,46 +52,3 @@ export function addEnvBackedAgentCredentials( } return next; } - -export function scrubLegacyStaticAuthJsonEntriesForDiscovery(pathname: string): void { - if (process.env.OPENCLAW_AUTH_STORE_READONLY === "1") { - return; - } - if (!fs.existsSync(pathname)) { - return; - } - - const parsed = tryReadJsonSync(pathname); - if (!isRecord(parsed)) { - return; - } - - let changed = false; - for (const [provider, value] of Object.entries(parsed)) { - if (!isRecord(value)) { - continue; - } - if (value.type !== "api_key") { - continue; - } - delete parsed[provider]; - changed = true; - } - - if (!changed) { - return; - } - - if (Object.keys(parsed).length === 0) { - fs.rmSync(pathname, { force: true }); - return; - } - - replaceFileAtomicSync({ - filePath: pathname, - content: `${JSON.stringify(parsed, null, 2)}\n`, - dirMode: 0o700, - mode: 0o600, - tempPrefix: ".agent-auth", - }); -} diff --git a/src/agents/agent-auth-discovery.external-cli.test.ts b/src/agents/agent-auth-discovery.external-cli.test.ts index 7b4a9daca26e..850acea801f3 100644 --- a/src/agents/agent-auth-discovery.external-cli.test.ts +++ b/src/agents/agent-auth-discovery.external-cli.test.ts @@ -15,7 +15,6 @@ const credentialMocks = vi.hoisted(() => ({ const discoveryCoreMocks = vi.hoisted(() => ({ addEnvBackedAgentCredentials: vi.fn((credentials: unknown) => credentials), - scrubLegacyStaticAuthJsonEntriesForDiscovery: vi.fn(), })); const syntheticAuthMocks = vi.hoisted(() => ({ diff --git a/src/agents/agent-auth-discovery.ts b/src/agents/agent-auth-discovery.ts index 041330b9582f..270e011f3668 100644 --- a/src/agents/agent-auth-discovery.ts +++ b/src/agents/agent-auth-discovery.ts @@ -82,7 +82,4 @@ export function resolveAgentCredentialsForDiscovery( return credentials; } -export { - addEnvBackedAgentCredentials, - scrubLegacyStaticAuthJsonEntriesForDiscovery, -} from "./agent-auth-discovery-core.js"; +export { addEnvBackedAgentCredentials } from "./agent-auth-discovery-core.js"; diff --git a/src/agents/agent-auth-json.test.ts b/src/agents/agent-auth-json.test.ts deleted file mode 100644 index 72306ce75ea9..000000000000 --- a/src/agents/agent-auth-json.test.ts +++ /dev/null @@ -1,256 +0,0 @@ -import fs from "node:fs/promises"; -import os from "node:os"; -import path from "node:path"; -import { describe, expect, it, vi } from "vitest"; -import { ensureAgentAuthJsonFromAuthProfiles } from "./agent-auth-json.js"; -import { saveAuthProfileStore } from "./auth-profiles/store.js"; - -vi.mock("./auth-profiles/external-auth.js", () => ({ - listRuntimeExternalAuthProfiles: () => [], - overlayExternalAuthProfiles: (store: T) => store, - shouldPersistExternalAuthProfile: () => true, - syncPersistedExternalCliAuthProfiles: (store: T) => store, -})); - -type AuthProfileStore = Parameters[0]; - -async function createAgentDir() { - return fs.mkdtemp(path.join(os.tmpdir(), "openclaw-agent-")); -} - -function writeProfiles(agentDir: string, profiles: AuthProfileStore["profiles"]) { - saveAuthProfileStore( - { - version: 1, - profiles, - }, - agentDir, - ); -} - -async function readAuthJson(agentDir: string) { - const authPath = path.join(agentDir, "auth.json"); - return JSON.parse(await fs.readFile(authPath, "utf8")) as Record; -} - -function requireAuthEntry( - auth: Record, - provider: string, -): Record { - const entry = auth[provider]; - if (!entry || typeof entry !== "object") { - throw new Error(`expected auth entry ${provider}`); - } - return entry as Record; -} - -function expectApiKeyAuth(auth: Record, provider: string, key: string): void { - const entry = requireAuthEntry(auth, provider); - expect(entry.type).toBe("api_key"); - expect(entry.key).toBe(key); -} - -function expectOAuthAuth( - auth: Record, - provider: string, - access: string, - refresh?: string, -): void { - const entry = requireAuthEntry(auth, provider); - expect(entry.type).toBe("oauth"); - expect(entry.access).toBe(access); - if (refresh !== undefined) { - expect(entry.refresh).toBe(refresh); - } -} - -describe("ensureAgentAuthJsonFromAuthProfiles", () => { - it("writes openai oauth credentials into auth.json for session runtime discovery", async () => { - const agentDir = await createAgentDir(); - - writeProfiles(agentDir, { - "openai:default": { - type: "oauth", - provider: "openai", - access: "access-token", - refresh: "refresh-token", - expires: Date.now() + 60_000, - }, - }); - - const first = await ensureAgentAuthJsonFromAuthProfiles(agentDir); - expect(first.wrote).toBe(true); - - const auth = await readAuthJson(agentDir); - expectOAuthAuth(auth, "openai", "access-token", "refresh-token"); - - const second = await ensureAgentAuthJsonFromAuthProfiles(agentDir); - expect(second.wrote).toBe(false); - }); - - it("writes api_key credentials into auth.json", async () => { - const agentDir = await createAgentDir(); - - writeProfiles(agentDir, { - "openrouter:default": { - type: "api_key", - provider: "openrouter", - key: "sk-or-v1-test-key", - }, - }); - - const result = await ensureAgentAuthJsonFromAuthProfiles(agentDir); - expect(result.wrote).toBe(true); - - const auth = await readAuthJson(agentDir); - expectApiKeyAuth(auth, "openrouter", "sk-or-v1-test-key"); - }); - - it("writes token credentials as api_key into auth.json", async () => { - const agentDir = await createAgentDir(); - - writeProfiles(agentDir, { - "anthropic:default": { - type: "token", - provider: "anthropic", - token: "sk-ant-test-token", - }, - }); - - const result = await ensureAgentAuthJsonFromAuthProfiles(agentDir); - expect(result.wrote).toBe(true); - - const auth = await readAuthJson(agentDir); - expectApiKeyAuth(auth, "anthropic", "sk-ant-test-token"); - }); - - it("syncs multiple providers at once", async () => { - const agentDir = await createAgentDir(); - - writeProfiles(agentDir, { - "openrouter:default": { - type: "api_key", - provider: "openrouter", - key: "sk-or-key", - }, - "anthropic:default": { - type: "token", - provider: "anthropic", - token: "sk-ant-token", - }, - "openai:default": { - type: "oauth", - provider: "openai", - access: "access", - refresh: "refresh", - expires: Date.now() + 60_000, - }, - }); - - const result = await ensureAgentAuthJsonFromAuthProfiles(agentDir); - expect(result.wrote).toBe(true); - - const auth = await readAuthJson(agentDir); - - expectApiKeyAuth(auth, "openrouter", "sk-or-key"); - expectApiKeyAuth(auth, "anthropic", "sk-ant-token"); - expectOAuthAuth(auth, "openai", "access"); - }); - - it("skips profiles with empty keys", async () => { - const agentDir = await createAgentDir(); - - writeProfiles(agentDir, { - "openrouter:default": { - type: "api_key", - provider: "openrouter", - key: "", - }, - }); - - const result = await ensureAgentAuthJsonFromAuthProfiles(agentDir); - expect(result.wrote).toBe(false); - }); - - it("skips expired token credentials", async () => { - const agentDir = await createAgentDir(); - - writeProfiles(agentDir, { - "anthropic:default": { - type: "token", - provider: "anthropic", - token: "sk-ant-expired", - expires: Date.now() - 60_000, - }, - }); - - const result = await ensureAgentAuthJsonFromAuthProfiles(agentDir); - expect(result.wrote).toBe(false); - }); - - it("preserves provider ids when writing auth.json keys", async () => { - const agentDir = await createAgentDir(); - - writeProfiles(agentDir, { - "z.ai:default": { - type: "api_key", - provider: "z.ai", - key: "sk-zai", - }, - }); - - const result = await ensureAgentAuthJsonFromAuthProfiles(agentDir); - expect(result.wrote).toBe(true); - - const auth = await readAuthJson(agentDir); - expectApiKeyAuth(auth, "z.ai", "sk-zai"); - expect(auth.zai).toBeUndefined(); - }); - - it("preserves existing auth.json entries not in auth-profiles", async () => { - const agentDir = await createAgentDir(); - const authPath = path.join(agentDir, "auth.json"); - - await fs.mkdir(agentDir, { recursive: true }); - await fs.writeFile( - authPath, - JSON.stringify({ "legacy-provider": { type: "api_key", key: "legacy-key" } }), - ); - - writeProfiles(agentDir, { - "openrouter:default": { - type: "api_key", - provider: "openrouter", - key: "new-key", - }, - }); - - await ensureAgentAuthJsonFromAuthProfiles(agentDir); - - const auth = await readAuthJson(agentDir); - expectApiKeyAuth(auth, "legacy-provider", "legacy-key"); - expectApiKeyAuth(auth, "openrouter", "new-key"); - }); - - it("treats malformed existing provider entries as stale and replaces them", async () => { - const agentDir = await createAgentDir(); - const authPath = path.join(agentDir, "auth.json"); - - await fs.mkdir(agentDir, { recursive: true }); - await fs.writeFile(authPath, JSON.stringify({ openrouter: { type: "api_key", key: 123 } })); - - writeProfiles(agentDir, { - "openrouter:default": { - type: "api_key", - provider: "openrouter", - key: "new-key", - }, - }); - - const result = await ensureAgentAuthJsonFromAuthProfiles(agentDir); - expect(result.wrote).toBe(true); - - const auth = await readAuthJson(agentDir); - expectApiKeyAuth(auth, "openrouter", "new-key"); - }); -}); diff --git a/src/agents/agent-auth-json.ts b/src/agents/agent-auth-json.ts deleted file mode 100644 index 3f6fdb703ff7..000000000000 --- a/src/agents/agent-auth-json.ts +++ /dev/null @@ -1,82 +0,0 @@ -import path from "node:path"; -import { z } from "zod"; -import { privateFileStore } from "../infra/private-file-store.js"; -import { safeParseWithSchema } from "../utils/zod-parse.js"; -import { - agentCredentialsEqual, - resolveAgentCredentialMapFromStore, - type AgentCredential, -} from "./agent-auth-credentials.js"; -import { ensureAuthProfileStore } from "./auth-profiles/store.js"; - -type AuthJsonShape = Record; - -const AgentCredentialSchema: z.ZodType = z.discriminatedUnion("type", [ - z.object({ - type: z.literal("api_key"), - key: z.string(), - }), - z.object({ - type: z.literal("oauth"), - access: z.string(), - refresh: z.string(), - expires: z.number(), - }), -]); - -const AuthJsonShapeSchema = z.record(z.string(), z.unknown()); - -async function readAuthJson(rootDir: string, filePath: string): Promise { - try { - const parsed = await privateFileStore(rootDir).readJsonIfExists( - path.relative(rootDir, filePath), - ); - return safeParseWithSchema(AuthJsonShapeSchema, parsed) ?? {}; - } catch { - return {}; - } -} - -/** - * session runtime's ModelRegistry/AuthStorage expects credentials in auth.json. - * - * OpenClaw stores credentials in auth-profiles.json instead. This helper - * bridges all credentials into agentDir/auth.json so session runtime can - * consider provider-owned configured models authenticated. - * - * Syncs all credential types: api_key, token (as api_key), and oauth. - * - * @deprecated Runtime auth now comes from OpenClaw auth-profiles snapshots. - */ -export async function ensureAgentAuthJsonFromAuthProfiles(agentDir: string): Promise<{ - wrote: boolean; - authPath: string; -}> { - const store = ensureAuthProfileStore(agentDir, { allowKeychainPrompt: false }); - const authPath = path.join(agentDir, "auth.json"); - const providerCredentials = resolveAgentCredentialMapFromStore(store); - if (Object.keys(providerCredentials).length === 0) { - return { wrote: false, authPath }; - } - - const existing = await readAuthJson(agentDir, authPath); - let changed = false; - - for (const [provider, cred] of Object.entries(providerCredentials)) { - const current = safeParseWithSchema(AgentCredentialSchema, existing[provider]) ?? undefined; - if (!agentCredentialsEqual(current, cred)) { - existing[provider] = cred; - changed = true; - } - } - - if (!changed) { - return { wrote: false, authPath }; - } - - await privateFileStore(agentDir).writeJson(path.basename(authPath), existing, { - trailingNewline: true, - }); - - return { wrote: true, authPath }; -} diff --git a/src/agents/agent-dir-registry.ts b/src/agents/agent-dir-registry.ts new file mode 100644 index 000000000000..ff3c05627460 --- /dev/null +++ b/src/agents/agent-dir-registry.ts @@ -0,0 +1,27 @@ +import path from "node:path"; +import { normalizeAgentId } from "../routing/session-key.js"; +import { resolveUserPath } from "../utils.js"; + +const agentIdByDir = new Map(); + +function normalizeAgentDirKey(agentDir: string, env: NodeJS.ProcessEnv = process.env): string { + return path.resolve(resolveUserPath(agentDir, env)); +} + +export function registerResolvedAgentDir(params: { + agentId: string; + agentDir: string; + env?: NodeJS.ProcessEnv; +}): void { + agentIdByDir.set( + normalizeAgentDirKey(params.agentDir, params.env), + normalizeAgentId(params.agentId), + ); +} + +export function resolveRegisteredAgentIdForDir( + agentDir: string, + env?: NodeJS.ProcessEnv, +): string | undefined { + return agentIdByDir.get(normalizeAgentDirKey(agentDir, env)); +} diff --git a/src/agents/agent-model-discovery.auth.test.ts b/src/agents/agent-model-discovery.auth.test.ts index a5e63eb9d5e5..7ba85442bb3e 100644 --- a/src/agents/agent-model-discovery.auth.test.ts +++ b/src/agents/agent-model-discovery.auth.test.ts @@ -4,12 +4,10 @@ import path from "node:path"; import { MAX_DATE_TIMESTAMP_MS } from "@openclaw/normalization-core/number-coercion"; import { describe, expect, it, vi } from "vitest"; import { resolveAgentCredentialMapFromStore } from "./agent-auth-credentials.js"; -import { - addEnvBackedAgentCredentials, - scrubLegacyStaticAuthJsonEntriesForDiscovery, -} from "./agent-auth-discovery-core.js"; +import { addEnvBackedAgentCredentials } from "./agent-auth-discovery-core.js"; import { discoverAuthStorage } from "./agent-model-discovery.js"; import type { AuthProfileStore } from "./auth-profiles.js"; +import { writePersistedAuthProfileStoreRaw } from "./auth-profiles/sqlite.js"; vi.mock("./model-auth-env-vars.js", () => ({ listProviderEnvAuthLookupKeys: () => ["mistral", "workspace-cloud"], @@ -74,22 +72,8 @@ async function withAgentDir(run: (agentDir: string) => Promise): Promise, -): Promise { - await fs.writeFile(path.join(agentDir, "auth.json"), JSON.stringify(authEntries, null, 2)); -} - -async function writeAuthProfilesJson(agentDir: string, store: AuthProfileStore): Promise { - await fs.writeFile(path.join(agentDir, "auth-profiles.json"), JSON.stringify(store, null, 2)); -} - -async function readLegacyAuthJson(agentDir: string): Promise> { - return JSON.parse(await fs.readFile(path.join(agentDir, "auth.json"), "utf8")) as Record< - string, - unknown - >; +function writeAuthProfilesSqlite(agentDir: string, store: AuthProfileStore): void { + writePersistedAuthProfileStoreRaw(store, agentDir); } describe("discoverAuthStorage", () => { @@ -213,7 +197,7 @@ describe("discoverAuthStorage", () => { it("marks keyRef-only auth profiles configured for read-only model discovery", async () => { await withAgentDir(async (agentDir) => { - await writeAuthProfilesJson(agentDir, { + writeAuthProfilesSqlite(agentDir, { version: 1, profiles: { "fixture-ref-provider:default": { @@ -239,53 +223,6 @@ describe("discoverAuthStorage", () => { }); }); - it("scrubs static api_key entries from legacy auth.json and keeps oauth entries", async () => { - await withAgentDir(async (agentDir) => { - await writeLegacyAuthJson(agentDir, { - openrouter: { type: "api_key", key: "legacy-static-key" }, - openai: { - type: "oauth", - access: "oauth-access", - refresh: "oauth-refresh", - expires: Date.now() + 60_000, - }, - }); - - scrubLegacyStaticAuthJsonEntriesForDiscovery(path.join(agentDir, "auth.json")); - - const parsed = await readLegacyAuthJson(agentDir); - expect(parsed.openrouter).toBeUndefined(); - const codexEntry = parsed["openai"] as { type?: string; access?: string } | undefined; - expect(codexEntry?.type).toBe("oauth"); - expect(codexEntry?.access).toBe("oauth-access"); - }); - }); - - it("preserves legacy auth.json when auth store is forced read-only", async () => { - await withAgentDir(async (agentDir) => { - const previous = process.env.OPENCLAW_AUTH_STORE_READONLY; - process.env.OPENCLAW_AUTH_STORE_READONLY = "1"; - try { - await writeLegacyAuthJson(agentDir, { - openrouter: { type: "api_key", key: "legacy-static-key" }, - }); - - scrubLegacyStaticAuthJsonEntriesForDiscovery(path.join(agentDir, "auth.json")); - - const parsed = await readLegacyAuthJson(agentDir); - const openrouterEntry = parsed.openrouter as { type?: string; key?: string } | undefined; - expect(openrouterEntry?.type).toBe("api_key"); - expect(openrouterEntry?.key).toBe("legacy-static-key"); - } finally { - if (previous === undefined) { - delete process.env.OPENCLAW_AUTH_STORE_READONLY; - } else { - process.env.OPENCLAW_AUTH_STORE_READONLY = previous; - } - } - }); - }); - it("includes env-backed provider auth when no auth profile exists", () => { const previousMistral = process.env.MISTRAL_API_KEY; const previousBundledPluginsDir = process.env.OPENCLAW_BUNDLED_PLUGINS_DIR; diff --git a/src/agents/agent-model-discovery.synthetic-auth.test.ts b/src/agents/agent-model-discovery.synthetic-auth.test.ts index 2344f4d44305..2932d9a5332a 100644 --- a/src/agents/agent-model-discovery.synthetic-auth.test.ts +++ b/src/agents/agent-model-discovery.synthetic-auth.test.ts @@ -35,7 +35,6 @@ vi.mock("./auth-profiles/store.js", () => ({ vi.mock("./agent-auth-discovery-core.js", () => ({ addEnvBackedAgentCredentials: (credentials: Record) => ({ ...credentials }), - scrubLegacyStaticAuthJsonEntriesForDiscovery: vi.fn(), })); let resolveAgentCredentialsForDiscovery: typeof import("./agent-auth-discovery.js").resolveAgentCredentialsForDiscovery; diff --git a/src/agents/agent-model-discovery.ts b/src/agents/agent-model-discovery.ts index 1515adde746c..023f94c4268e 100644 --- a/src/agents/agent-model-discovery.ts +++ b/src/agents/agent-model-discovery.ts @@ -10,7 +10,6 @@ import { import { isRecord } from "../utils.js"; import { resolveAgentCredentialsForDiscovery, - scrubLegacyStaticAuthJsonEntriesForDiscovery, type DiscoverAuthStorageOptions, } from "./agent-auth-discovery.js"; import { resolveModelPluginMetadataSnapshot } from "./model-discovery-context.js"; @@ -152,10 +151,6 @@ export function discoverAuthStorage( ): AgentAuthStorage { const credentials = options?.skipCredentials === true ? {} : resolveAgentCredentialsForDiscovery(agentDir, options); - const authPath = path.join(agentDir, "auth.json"); - if (options?.readOnly !== true) { - scrubLegacyStaticAuthJsonEntriesForDiscovery(authPath); - } return AuthStorage.inMemory(credentials); } @@ -175,6 +170,5 @@ export function discoverModels( export { addEnvBackedAgentCredentials, resolveAgentCredentialsForDiscovery, - scrubLegacyStaticAuthJsonEntriesForDiscovery, type DiscoverAuthStorageOptions, } from "./agent-auth-discovery.js"; diff --git a/src/agents/agent-scope-config.ts b/src/agents/agent-scope-config.ts index 0941771560ac..e69d8dedb9f8 100644 --- a/src/agents/agent-scope-config.ts +++ b/src/agents/agent-scope-config.ts @@ -8,6 +8,7 @@ import type { import type { OpenClawConfig } from "../config/types.js"; import { DEFAULT_AGENT_ID, normalizeAgentId } from "../routing/session-key.js"; import { resolveUserPath } from "../utils.js"; +import { registerResolvedAgentDir } from "./agent-dir-registry.js"; import { resolveDefaultAgentWorkspaceDir } from "./workspace-default.js"; type AgentEntry = NonNullable["list"]>[number]; @@ -200,10 +201,14 @@ export function resolveAgentDir( const id = normalizeAgentId(agentId); const configured = resolveAgentConfig(cfg, id)?.agentDir?.trim(); if (configured) { - return resolveUserPath(configured, env); + const agentDir = resolveUserPath(configured, env); + registerResolvedAgentDir({ agentId: id, agentDir, env }); + return agentDir; } const root = resolveStateDir(env); - return path.join(root, "agents", id, "agent"); + const agentDir = path.join(root, "agents", id, "agent"); + registerResolvedAgentDir({ agentId: id, agentDir, env }); + return agentDir; } export function resolveDefaultAgentDir( diff --git a/src/agents/auth-profiles.chutes.test.ts b/src/agents/auth-profiles.chutes.test.ts index 07c19b244fc1..08774944f9ed 100644 --- a/src/agents/auth-profiles.chutes.test.ts +++ b/src/agents/auth-profiles.chutes.test.ts @@ -1,4 +1,3 @@ -import fs from "node:fs/promises"; import { afterAll, afterEach, beforeAll, beforeEach, describe, expect, it, vi } from "vitest"; import { withOpenClawTestState } from "../test-utils/openclaw-test-state.js"; import type { AuthProfileStore } from "./auth-profiles.js"; @@ -20,6 +19,7 @@ afterAll(() => { let clearRuntimeAuthProfileStoreSnapshots: typeof import("./auth-profiles.js").clearRuntimeAuthProfileStoreSnapshots; let ensureAuthProfileStore: typeof import("./auth-profiles.js").ensureAuthProfileStore; +let loadPersistedAuthProfileStore: typeof import("./auth-profiles/persisted.js").loadPersistedAuthProfileStore; let resolveApiKeyForProfile: typeof import("./auth-profiles.js").resolveApiKeyForProfile; let resetFileLockStateForTest: typeof import("../infra/file-lock.js").resetFileLockStateForTest; @@ -27,6 +27,7 @@ describe("auth-profiles (chutes)", () => { beforeAll(async () => { ({ clearRuntimeAuthProfileStoreSnapshots, ensureAuthProfileStore, resolveApiKeyForProfile } = await import("./auth-profiles.js")); + ({ loadPersistedAuthProfileStore } = await import("./auth-profiles/persisted.js")); ({ resetFileLockStateForTest } = await import("../infra/file-lock.js")); }); @@ -65,7 +66,7 @@ describe("auth-profiles (chutes)", () => { }, }, }; - const authProfilePath = await state.writeAuthProfiles(store); + await state.writeAuthProfiles(store); const fetchSpy = vi.fn(async (input: string | URL) => { const url = typeof input === "string" ? input : input.toString(); @@ -92,10 +93,12 @@ describe("auth-profiles (chutes)", () => { expect(fetchSpy).toHaveBeenCalledTimes(1); expect(fetchSpy).toHaveBeenCalledWith(CHUTES_TOKEN_ENDPOINT, expect.any(Object)); - const persisted = JSON.parse(await fs.readFile(authProfilePath, "utf8")) as { - profiles?: Record; - }; - expect(persisted.profiles?.["chutes:default"]?.access).toBe("at_new"); + const persisted = loadPersistedAuthProfileStore(state.agentDir()); + const persistedCredential = persisted?.profiles["chutes:default"]; + expect(persistedCredential?.type).toBe("oauth"); + expect(persistedCredential?.type === "oauth" ? persistedCredential.access : undefined).toBe( + "at_new", + ); }, ); }); diff --git a/src/agents/auth-profiles.ensureauthprofilestore.test.ts b/src/agents/auth-profiles.ensureauthprofilestore.test.ts deleted file mode 100644 index 7d66c4815480..000000000000 --- a/src/agents/auth-profiles.ensureauthprofilestore.test.ts +++ /dev/null @@ -1,1201 +0,0 @@ -import fs from "node:fs"; -import os from "node:os"; -import path from "node:path"; -import { afterEach, describe, expect, it, vi } from "vitest"; -import type { ProviderExternalAuthProfile } from "../plugins/provider-external-auth.types.js"; -import { AUTH_STORE_VERSION, log } from "./auth-profiles/constants.js"; -import { - clearRuntimeAuthProfileStoreSnapshots, - ensureAuthProfileStore, - loadAuthProfileStoreForRuntime, - saveAuthProfileStore, -} from "./auth-profiles/store.js"; -import type { AuthProfileCredential } from "./auth-profiles/types.js"; - -const resolveExternalAuthProfilesWithPluginsMock = vi.hoisted(() => - vi.fn<() => ProviderExternalAuthProfile[]>(() => []), -); - -vi.mock("../plugins/provider-runtime.js", () => ({ - resolveExternalAuthProfilesWithPlugins: resolveExternalAuthProfilesWithPluginsMock, -})); - -vi.mock("./cli-credentials.js", () => ({ - readClaudeCliCredentialsCached: () => null, - readCodexCliCredentialsCached: () => { - const codexHome = process.env.CODEX_HOME; - if (!codexHome) { - return null; - } - try { - const raw = JSON.parse(fs.readFileSync(path.join(codexHome, "auth.json"), "utf8")) as { - tokens?: { - access_token?: unknown; - refresh_token?: unknown; - account_id?: unknown; - }; - }; - const access = raw.tokens?.access_token; - const refresh = raw.tokens?.refresh_token; - if (typeof access !== "string" || typeof refresh !== "string") { - return null; - } - return { - type: "oauth", - provider: "openai", - access, - refresh, - expires: Date.now() + 60 * 60 * 1000, - accountId: typeof raw.tokens?.account_id === "string" ? raw.tokens.account_id : undefined, - }; - } catch { - return null; - } - }, - readMiniMaxCliCredentialsCached: () => null, - resetCliCredentialCachesForTest: vi.fn(), -})); - -describe("ensureAuthProfileStore", () => { - afterEach(() => { - clearRuntimeAuthProfileStoreSnapshots(); - resolveExternalAuthProfilesWithPluginsMock.mockReset(); - resolveExternalAuthProfilesWithPluginsMock.mockReturnValue([]); - }); - - function withTempAgentDir(prefix: string, run: (agentDir: string) => T): T { - const agentDir = fs.mkdtempSync(path.join(os.tmpdir(), prefix)); - try { - return run(agentDir); - } finally { - fs.rmSync(agentDir, { recursive: true, force: true }); - } - } - - function writeAuthProfileStore(agentDir: string, profiles: Record): void { - fs.writeFileSync( - path.join(agentDir, "auth-profiles.json"), - `${JSON.stringify({ version: AUTH_STORE_VERSION, profiles }, null, 2)}\n`, - "utf8", - ); - } - - function writeRawAuthProfileStore(agentDir: string, raw: unknown): void { - fs.writeFileSync( - path.join(agentDir, "auth-profiles.json"), - `${JSON.stringify(raw, null, 2)}\n`, - "utf8", - ); - } - - function loadAuthProfile(agentDir: string, profileId: string): AuthProfileCredential { - clearRuntimeAuthProfileStoreSnapshots(); - const store = ensureAuthProfileStore(agentDir); - const profile = store.profiles[profileId]; - if (!profile) { - throw new Error(`expected auth profile ${profileId}`); - } - return profile; - } - - function restoreEnvValue(name: string, previous: string | undefined): void { - if (previous === undefined) { - delete process.env[name]; - } else { - process.env[name] = previous; - } - } - - function restoreAgentDirEnv(params: { - previousStateDir?: string | undefined; - previousAgentDir: string | undefined; - }): void { - if ("previousStateDir" in params) { - restoreEnvValue("OPENCLAW_STATE_DIR", params.previousStateDir); - } - restoreEnvValue("OPENCLAW_AGENT_DIR", params.previousAgentDir); - } - - function configureMainAuthTestDirs(root: string): { - mainDir: string; - agentDir: string; - previousStateDir: string | undefined; - previousAgentDir: string | undefined; - } { - const previousStateDir = process.env.OPENCLAW_STATE_DIR; - const previousAgentDir = process.env.OPENCLAW_AGENT_DIR; - const mainDir = path.join(root, "agents", "main", "agent"); - const agentDir = path.join(root, "agents", "agent-x", "agent"); - fs.mkdirSync(mainDir, { recursive: true }); - fs.mkdirSync(agentDir, { recursive: true }); - - process.env.OPENCLAW_STATE_DIR = root; - process.env.OPENCLAW_AGENT_DIR = mainDir; - clearRuntimeAuthProfileStoreSnapshots(); - return { mainDir, agentDir, previousStateDir, previousAgentDir }; - } - - function expectApiKeyProfile( - profile: AuthProfileCredential, - ): Extract { - expect(profile.type).toBe("api_key"); - if (profile.type !== "api_key") { - throw new Error(`Expected api_key profile, got ${profile.type}`); - } - return profile; - } - - function expectTokenProfile( - profile: AuthProfileCredential, - ): Extract { - expect(profile.type).toBe("token"); - if (profile.type !== "token") { - throw new Error(`Expected token profile, got ${profile.type}`); - } - return profile; - } - - function expectRecordFields( - value: unknown, - expected: Record, - message?: string, - ): void { - const record = value as Record | undefined; - for (const [key, expectedValue] of Object.entries(expected)) { - expect(record?.[key], message ? `${message}:${key}` : key).toEqual(expectedValue); - } - } - - it("migrates legacy auth.json and deletes it (PR #368)", () => { - const agentDir = fs.mkdtempSync(path.join(os.tmpdir(), "openclaw-auth-profiles-")); - try { - const legacyPath = path.join(agentDir, "auth.json"); - fs.writeFileSync( - legacyPath, - `${JSON.stringify( - { - anthropic: { - type: "oauth", - provider: "anthropic", - access: "access-token", - refresh: "refresh-token", - expires: Date.now() + 60_000, - }, - }, - null, - 2, - )}\n`, - "utf8", - ); - - const store = ensureAuthProfileStore(agentDir); - expectRecordFields(store.profiles["anthropic:default"], { - type: "oauth", - provider: "anthropic", - }); - - const migratedPath = path.join(agentDir, "auth-profiles.json"); - expect(fs.existsSync(migratedPath)).toBe(true); - expect(fs.existsSync(legacyPath)).toBe(false); - - // idempotent - const store2 = ensureAuthProfileStore(agentDir); - expect(store2.profiles).toHaveProperty("anthropic:default"); - expect(fs.existsSync(legacyPath)).toBe(false); - } finally { - fs.rmSync(agentDir, { recursive: true, force: true }); - } - }); - - it("ignores array-shaped auth profile stores instead of loading numeric profile ids", () => { - withTempAgentDir("openclaw-auth-profiles-array-", (agentDir) => { - writeRawAuthProfileStore(agentDir, { - version: AUTH_STORE_VERSION, - profiles: [ - { - type: "api_key", - provider: "openai", - key: "test-array-shaped-profile", - }, - ], - }); - - const store = ensureAuthProfileStore(agentDir); - - expect(store.profiles["0"]).toBeUndefined(); - expect(Object.keys(store.profiles)).toEqual([]); - }); - }); - - it("ignores top-level array auth stores instead of treating entries as profiles", () => { - withTempAgentDir("openclaw-auth-top-array-", (agentDir) => { - writeRawAuthProfileStore(agentDir, [ - { - type: "api_key", - provider: "openai", - key: "test-array-shaped-store", - }, - ]); - - const store = ensureAuthProfileStore(agentDir); - - expect(store.profiles["0"]).toBeUndefined(); - expect(Object.keys(store.profiles)).toEqual([]); - }); - }); - - it("merges main auth profiles into agent store and keeps agent overrides", () => { - const root = fs.mkdtempSync(path.join(os.tmpdir(), "openclaw-auth-merge-")); - const { mainDir, agentDir, previousStateDir, previousAgentDir } = - configureMainAuthTestDirs(root); - try { - const mainStore = { - version: AUTH_STORE_VERSION, - profiles: { - "openai:default": { - type: "api_key", - provider: "openai", - key: "main-key", - }, - "anthropic:default": { - type: "api_key", - provider: "anthropic", - key: "main-anthropic-key", - }, - }, - }; - fs.writeFileSync( - path.join(mainDir, "auth-profiles.json"), - `${JSON.stringify(mainStore, null, 2)}\n`, - "utf8", - ); - - const agentStore = { - version: AUTH_STORE_VERSION, - profiles: { - "openai:default": { - type: "api_key", - provider: "openai", - key: "agent-key", - }, - }, - }; - fs.writeFileSync( - path.join(agentDir, "auth-profiles.json"), - `${JSON.stringify(agentStore, null, 2)}\n`, - "utf8", - ); - - const store = ensureAuthProfileStore(agentDir); - expectRecordFields(store.profiles["anthropic:default"], { - type: "api_key", - provider: "anthropic", - key: "main-anthropic-key", - }); - expectRecordFields(store.profiles["openai:default"], { - type: "api_key", - provider: "openai", - key: "agent-key", - }); - } finally { - restoreAgentDirEnv({ previousStateDir, previousAgentDir }); - fs.rmSync(root, { recursive: true, force: true }); - } - }); - - it("uses the main agent's newer OAuth profile when an agent still has a stale default profile", () => { - const root = fs.mkdtempSync(path.join(os.tmpdir(), "openclaw-auth-drift-")); - const { mainDir, agentDir, previousStateDir, previousAgentDir } = - configureMainAuthTestDirs(root); - try { - const freshProfileId = "openai:user@example.com"; - const staleProfileId = "openai:default"; - saveAuthProfileStore( - { - version: AUTH_STORE_VERSION, - profiles: { - [freshProfileId]: { - type: "oauth", - provider: "openai", - access: "main-access", - refresh: "main-refresh", - expires: Date.now() + 60 * 60 * 1000, - email: "user@example.com", - }, - }, - order: { - openai: [freshProfileId], - }, - lastGood: { - openai: freshProfileId, - }, - }, - mainDir, - ); - saveAuthProfileStore( - { - version: AUTH_STORE_VERSION, - profiles: { - [freshProfileId]: { - type: "oauth", - provider: "openai", - access: "stale-identity-access", - refresh: "stale-identity-refresh", - expires: Date.now() - 30 * 60 * 1000, - email: "user@example.com", - }, - [staleProfileId]: { - type: "oauth", - provider: "openai", - access: "stale-access", - refresh: "stale-refresh", - expires: Date.now() - 60 * 60 * 1000, - accountId: "acct-from-old-codex-auth", - }, - }, - order: { - openai: [staleProfileId], - }, - lastGood: { - openai: staleProfileId, - }, - usageStats: { - [staleProfileId]: { - lastUsed: Date.now() - 30_000, - errorCount: 3, - }, - }, - }, - agentDir, - ); - clearRuntimeAuthProfileStoreSnapshots(); - - const store = loadAuthProfileStoreForRuntime(agentDir, { readOnly: true }); - - expectRecordFields(store.profiles[freshProfileId], { - type: "oauth", - provider: "openai", - access: "main-access", - refresh: "main-refresh", - }); - expect(store.profiles[staleProfileId]).toBeUndefined(); - expect(store.order?.["openai"]).toEqual([freshProfileId]); - expect(store.lastGood?.["openai"]).toBe(freshProfileId); - expect(store.usageStats?.[staleProfileId]).toBeUndefined(); - - const persistedAgentStore = JSON.parse( - fs.readFileSync(path.join(agentDir, "auth-profiles.json"), "utf8"), - ) as { profiles: Record }; - expect(persistedAgentStore.profiles).toHaveProperty(staleProfileId); - } finally { - restoreAgentDirEnv({ previousStateDir, previousAgentDir }); - fs.rmSync(root, { recursive: true, force: true }); - } - }); - - it("keeps a newer agent replacement credential while repairing stale default references", () => { - const root = fs.mkdtempSync(path.join(os.tmpdir(), "openclaw-auth-drift-newer-agent-")); - const { mainDir, agentDir, previousStateDir, previousAgentDir } = - configureMainAuthTestDirs(root); - try { - const freshProfileId = "openai:user@example.com"; - const staleProfileId = "openai:default"; - saveAuthProfileStore( - { - version: AUTH_STORE_VERSION, - profiles: { - [freshProfileId]: { - type: "oauth", - provider: "openai", - access: "older-main-access", - refresh: "older-main-refresh", - expires: Date.now() + 30 * 60 * 1000, - email: "user@example.com", - }, - }, - order: { - openai: [freshProfileId], - }, - }, - mainDir, - ); - saveAuthProfileStore( - { - version: AUTH_STORE_VERSION, - profiles: { - [freshProfileId]: { - type: "oauth", - provider: "openai", - access: "newer-agent-access", - refresh: "newer-agent-refresh", - expires: Date.now() + 90 * 60 * 1000, - email: "user@example.com", - }, - [staleProfileId]: { - type: "oauth", - provider: "openai", - access: "stale-access", - refresh: "stale-refresh", - expires: Date.now() - 60 * 60 * 1000, - email: "user@example.com", - }, - }, - order: { - openai: [staleProfileId], - }, - lastGood: { - openai: staleProfileId, - }, - }, - agentDir, - ); - clearRuntimeAuthProfileStoreSnapshots(); - - const store = loadAuthProfileStoreForRuntime(agentDir, { readOnly: true }); - - expectRecordFields(store.profiles[freshProfileId], { - type: "oauth", - provider: "openai", - access: "newer-agent-access", - refresh: "newer-agent-refresh", - }); - expect(store.profiles[staleProfileId]).toBeUndefined(); - expect(store.order?.["openai"]).toEqual([freshProfileId]); - expect(store.lastGood?.["openai"]).toBe(freshProfileId); - } finally { - restoreAgentDirEnv({ previousStateDir, previousAgentDir }); - fs.rmSync(root, { recursive: true, force: true }); - } - }); - - it("preserves a valid main default OAuth profile while replacing a stale agent override", () => { - const root = fs.mkdtempSync(path.join(os.tmpdir(), "openclaw-auth-drift-base-default-")); - const { mainDir, agentDir, previousStateDir, previousAgentDir } = - configureMainAuthTestDirs(root); - try { - const freshProfileId = "openai:user@example.com"; - const defaultProfileId = "openai:default"; - saveAuthProfileStore( - { - version: AUTH_STORE_VERSION, - profiles: { - [freshProfileId]: { - type: "oauth", - provider: "openai", - access: "main-access", - refresh: "main-refresh", - expires: Date.now() + 60 * 60 * 1000, - email: "user@example.com", - }, - [defaultProfileId]: { - type: "oauth", - provider: "openai", - access: "main-default-access", - refresh: "main-default-refresh", - expires: Date.now() + 45 * 60 * 1000, - }, - }, - order: { - openai: [freshProfileId, defaultProfileId], - }, - usageStats: { - [defaultProfileId]: { - lastUsed: 123, - }, - }, - }, - mainDir, - ); - saveAuthProfileStore( - { - version: AUTH_STORE_VERSION, - profiles: { - [defaultProfileId]: { - type: "oauth", - provider: "openai", - access: "stale-agent-default-access", - refresh: "stale-agent-default-refresh", - expires: Date.now() - 60 * 60 * 1000, - }, - }, - order: { - openai: [defaultProfileId], - }, - usageStats: { - [defaultProfileId]: { - lastUsed: 999, - errorCount: 2, - }, - }, - }, - agentDir, - ); - clearRuntimeAuthProfileStoreSnapshots(); - - const store = loadAuthProfileStoreForRuntime(agentDir, { readOnly: true }); - - expect(store.order?.["openai"]).toEqual([freshProfileId, defaultProfileId]); - expectRecordFields(store.profiles[defaultProfileId], { - type: "oauth", - provider: "openai", - access: "main-default-access", - }); - expectRecordFields(store.usageStats?.[defaultProfileId], { - lastUsed: 123, - }); - } finally { - restoreAgentDirEnv({ previousStateDir, previousAgentDir }); - fs.rmSync(root, { recursive: true, force: true }); - } - }); - - it("keeps a stale default OAuth profile when the main profile belongs to a different identity", () => { - const root = fs.mkdtempSync(path.join(os.tmpdir(), "openclaw-auth-drift-mismatch-")); - const { mainDir, agentDir, previousStateDir, previousAgentDir } = - configureMainAuthTestDirs(root); - try { - const freshProfileId = "openai:user@example.com"; - const staleProfileId = "openai:default"; - saveAuthProfileStore( - { - version: AUTH_STORE_VERSION, - profiles: { - [freshProfileId]: { - type: "oauth", - provider: "openai", - access: "main-access", - refresh: "main-refresh", - expires: Date.now() + 60 * 60 * 1000, - email: "user@example.com", - }, - }, - }, - mainDir, - ); - saveAuthProfileStore( - { - version: AUTH_STORE_VERSION, - profiles: { - [staleProfileId]: { - type: "oauth", - provider: "openai", - access: "other-access", - refresh: "other-refresh", - expires: Date.now() - 60 * 60 * 1000, - email: "other@example.com", - }, - }, - order: { - openai: [staleProfileId], - }, - lastGood: { - openai: staleProfileId, - }, - }, - agentDir, - ); - clearRuntimeAuthProfileStoreSnapshots(); - - const store = loadAuthProfileStoreForRuntime(agentDir, { readOnly: true }); - - expect(store.profiles).toHaveProperty(freshProfileId); - expectRecordFields(store.profiles[staleProfileId], { - type: "oauth", - provider: "openai", - access: "other-access", - }); - expect(store.order?.["openai"]).toEqual([staleProfileId]); - expect(store.lastGood?.["openai"]).toBe(staleProfileId); - } finally { - restoreAgentDirEnv({ previousStateDir, previousAgentDir }); - fs.rmSync(root, { recursive: true, force: true }); - } - }); - - it("keeps an invalidated identity-specific agent profile when the main agent has a different identity", () => { - const root = fs.mkdtempSync(path.join(os.tmpdir(), "openclaw-auth-codex-relogin-")); - const { mainDir, agentDir, previousStateDir, previousAgentDir } = - configureMainAuthTestDirs(root); - try { - const now = Date.now(); - const healthyProfileId = "openai:bunsthedev@gmail.com"; - const staleProfileId = "openai:val@viewdue.ai"; - saveAuthProfileStore( - { - version: AUTH_STORE_VERSION, - profiles: { - [healthyProfileId]: { - type: "oauth", - provider: "openai", - access: "healthy-access", - refresh: "healthy-refresh", - expires: now + 60 * 60 * 1000, - email: "bunsthedev@gmail.com", - }, - }, - order: { - openai: [healthyProfileId], - }, - lastGood: { - openai: healthyProfileId, - }, - }, - mainDir, - ); - saveAuthProfileStore( - { - version: AUTH_STORE_VERSION, - profiles: { - [staleProfileId]: { - type: "oauth", - provider: "openai", - access: "stale-access", - refresh: "stale-refresh", - expires: now + 30 * 60 * 1000, - email: "val@viewdue.ai", - }, - }, - order: { - openai: [staleProfileId], - }, - lastGood: { - openai: staleProfileId, - }, - usageStats: { - [staleProfileId]: { - cooldownUntil: now + 60_000, - cooldownReason: "auth", - failureCounts: { auth: 1 }, - errorCount: 1, - lastFailureAt: now - 1_000, - }, - }, - }, - agentDir, - ); - clearRuntimeAuthProfileStoreSnapshots(); - - const store = loadAuthProfileStoreForRuntime(agentDir, { readOnly: true }); - - expectRecordFields(store.profiles[healthyProfileId], { - type: "oauth", - provider: "openai", - access: "healthy-access", - }); - expectRecordFields(store.profiles[staleProfileId], { - type: "oauth", - provider: "openai", - access: "stale-access", - }); - expect(store.order?.["openai"]).toEqual([staleProfileId]); - expect(store.lastGood?.["openai"]).toBe(staleProfileId); - expect(store.usageStats?.[staleProfileId]?.cooldownReason).toBe("auth"); - } finally { - restoreAgentDirEnv({ previousStateDir, previousAgentDir }); - fs.rmSync(root, { recursive: true, force: true }); - } - }); - - it.each([ - { - name: "mode/apiKey aliases map to type/key", - profile: { - provider: "anthropic", - mode: "api_key", - apiKey: "sk-ant-alias", // pragma: allowlist secret - }, - expected: { - type: "api_key", - key: "sk-ant-alias", - }, - }, - { - name: "canonical type overrides conflicting mode alias", - profile: { - provider: "anthropic", - type: "api_key", - mode: "token", - key: "sk-ant-canonical", - }, - expected: { - type: "api_key", - key: "sk-ant-canonical", - }, - }, - { - name: "canonical key overrides conflicting apiKey alias", - profile: { - provider: "anthropic", - type: "api_key", - key: "sk-ant-canonical", - apiKey: "sk-ant-alias", // pragma: allowlist secret - }, - expected: { - type: "api_key", - key: "sk-ant-canonical", - }, - }, - { - name: "canonical profile shape remains unchanged", - profile: { - provider: "anthropic", - type: "api_key", - key: "sk-ant-direct", - }, - expected: { - type: "api_key", - key: "sk-ant-direct", - }, - }, - ] as const)( - "normalizes auth-profiles credential aliases with canonical-field precedence: $name", - ({ name, profile, expected }) => { - withTempAgentDir("openclaw-auth-alias-", (agentDir) => { - const storeData = { - version: AUTH_STORE_VERSION, - profiles: { - "anthropic:work": profile, - }, - }; - fs.writeFileSync( - path.join(agentDir, "auth-profiles.json"), - `${JSON.stringify(storeData, null, 2)}\n`, - "utf8", - ); - - const store = ensureAuthProfileStore(agentDir); - expectRecordFields(store.profiles["anthropic:work"], expected, name); - }); - }, - ); - - it("normalizes mode/apiKey aliases while migrating legacy auth.json", () => { - withTempAgentDir("openclaw-auth-legacy-alias-", (agentDir) => { - fs.writeFileSync( - path.join(agentDir, "auth.json"), - `${JSON.stringify( - { - anthropic: { - provider: "anthropic", - mode: "api_key", - apiKey: "sk-ant-legacy", // pragma: allowlist secret - }, - }, - null, - 2, - )}\n`, - "utf8", - ); - - const store = ensureAuthProfileStore(agentDir); - expectRecordFields(store.profiles["anthropic:default"], { - type: "api_key", - provider: "anthropic", - key: "sk-ant-legacy", - }); - }); - }); - - it("does not load legacy flat auth-profiles.json entries at runtime", () => { - const agentDir = fs.mkdtempSync(path.join(os.tmpdir(), "openclaw-auth-flat-profiles-")); - try { - const authPath = path.join(agentDir, "auth-profiles.json"); - const legacyFlatStore = { - "ollama-windows": { - apiKey: "ollama-local", - baseUrl: "http://10.0.2.2:11434/v1", - }, - }; - fs.writeFileSync(authPath, `${JSON.stringify(legacyFlatStore)}\n`, "utf8"); - - const store = ensureAuthProfileStore(agentDir); - - expect(store.profiles["ollama-windows:default"]).toBeUndefined(); - expect(JSON.parse(fs.readFileSync(authPath, "utf8"))).toEqual(legacyFlatStore); - } finally { - fs.rmSync(agentDir, { recursive: true, force: true }); - } - }); - - it("merges legacy oauth.json into auth-profiles.json", () => { - const root = fs.mkdtempSync(path.join(os.tmpdir(), "openclaw-oauth-migrate-")); - const previousStateDir = process.env.OPENCLAW_STATE_DIR; - const previousAgentDir = process.env.OPENCLAW_AGENT_DIR; - try { - const agentDir = path.join(root, "agent"); - const oauthDir = path.join(root, "credentials"); - fs.mkdirSync(agentDir, { recursive: true }); - fs.mkdirSync(oauthDir, { recursive: true }); - fs.writeFileSync( - path.join(oauthDir, "oauth.json"), - `${JSON.stringify( - { - openai: { - access: "access-token", - refresh: "refresh-token", - expires: Date.now() + 60_000, - accountId: "acct_123", - }, - }, - null, - 2, - )}\n`, - "utf8", - ); - - process.env.OPENCLAW_STATE_DIR = root; - process.env.OPENCLAW_AGENT_DIR = agentDir; - clearRuntimeAuthProfileStoreSnapshots(); - - const store = ensureAuthProfileStore(agentDir); - expectRecordFields(store.profiles["openai:default"], { - type: "oauth", - provider: "openai", - access: "access-token", - refresh: "refresh-token", - }); - - const persisted = JSON.parse( - fs.readFileSync(path.join(agentDir, "auth-profiles.json"), "utf8"), - ) as { - profiles: Record>; - }; - const persistedProfile = persisted.profiles["openai:default"]; - expect(persistedProfile?.type).toBe("oauth"); - expect(persistedProfile?.provider).toBe("openai"); - expect(persistedProfile?.access).toBe("access-token"); - expect(persistedProfile?.refresh).toBe("refresh-token"); - expect(persistedProfile).not.toHaveProperty("oauthRef"); - expect(persistedProfile).not.toHaveProperty("idToken"); - } finally { - clearRuntimeAuthProfileStoreSnapshots(); - restoreEnvValue("OPENCLAW_STATE_DIR", previousStateDir); - restoreAgentDirEnv({ previousAgentDir }); - fs.rmSync(root, { recursive: true, force: true }); - } - }); - - it("exposes provider-managed runtime auth without persisting copied tokens", () => { - const root = fs.mkdtempSync(path.join(os.tmpdir(), "openclaw-external-auth-")); - const previousAgentDir = process.env.OPENCLAW_AGENT_DIR; - try { - const agentDir = path.join(root, "agent"); - fs.mkdirSync(agentDir, { recursive: true }); - resolveExternalAuthProfilesWithPluginsMock.mockReturnValueOnce([ - { - profileId: "demo-provider:external", - credential: { - type: "oauth", - provider: "demo-provider", - access: "external-access-token", - refresh: "external-refresh-token", - expires: Date.now() + 60_000, - accountId: "acct_123", - }, - persistence: "runtime-only", - }, - ]); - - process.env.OPENCLAW_AGENT_DIR = agentDir; - clearRuntimeAuthProfileStoreSnapshots(); - - const store = ensureAuthProfileStore(agentDir); - expectRecordFields(store.profiles["demo-provider:external"], { - type: "oauth", - provider: "demo-provider", - access: "external-access-token", - refresh: "external-refresh-token", - }); - - expect(fs.existsSync(path.join(agentDir, "auth-profiles.json"))).toBe(false); - } finally { - clearRuntimeAuthProfileStoreSnapshots(); - restoreAgentDirEnv({ previousAgentDir }); - fs.rmSync(root, { recursive: true, force: true }); - } - }); - - it("does not write inherited auth stores during secrets runtime reads", () => { - const root = fs.mkdtempSync(path.join(os.tmpdir(), "openclaw-secrets-runtime-")); - const previousStateDir = process.env.OPENCLAW_STATE_DIR; - try { - const stateDir = path.join(root, ".openclaw"); - const mainAgentDir = path.join(stateDir, "agents", "main", "agent"); - const workerAgentDir = path.join(stateDir, "agents", "worker", "agent"); - const workerStorePath = path.join(workerAgentDir, "auth-profiles.json"); - fs.mkdirSync(mainAgentDir, { recursive: true }); - fs.writeFileSync( - path.join(mainAgentDir, "auth-profiles.json"), - `${JSON.stringify( - { - version: AUTH_STORE_VERSION, - profiles: { - "openai:default": { - type: "api_key", - provider: "openai", - keyRef: { source: "env", provider: "default", id: "OPENAI_API_KEY" }, - }, - }, - }, - null, - 2, - )}\n`, - "utf8", - ); - process.env.OPENCLAW_STATE_DIR = stateDir; - clearRuntimeAuthProfileStoreSnapshots(); - - const store = loadAuthProfileStoreForRuntime(workerAgentDir, { readOnly: true }); - - expectRecordFields(store.profiles["openai:default"], { - type: "api_key", - provider: "openai", - }); - expect(fs.existsSync(workerStorePath)).toBe(false); - } finally { - clearRuntimeAuthProfileStoreSnapshots(); - restoreEnvValue("OPENCLAW_STATE_DIR", previousStateDir); - fs.rmSync(root, { recursive: true, force: true }); - } - }); - - it("does not clone inherited auth stores during normal agent reads", () => { - const root = fs.mkdtempSync(path.join(os.tmpdir(), "openclaw-auth-read-through-")); - const previousStateDir = process.env.OPENCLAW_STATE_DIR; - try { - const stateDir = path.join(root, ".openclaw"); - const mainAgentDir = path.join(stateDir, "agents", "main", "agent"); - const workerAgentDir = path.join(stateDir, "agents", "worker", "agent"); - const workerStorePath = path.join(workerAgentDir, "auth-profiles.json"); - fs.mkdirSync(mainAgentDir, { recursive: true }); - fs.writeFileSync( - path.join(mainAgentDir, "auth-profiles.json"), - `${JSON.stringify( - { - version: AUTH_STORE_VERSION, - profiles: { - "openai:default": { - type: "oauth", - provider: "openai", - access: "main-access", - refresh: "main-refresh", - expires: Date.now() + 60_000, - }, - }, - }, - null, - 2, - )}\n`, - "utf8", - ); - process.env.OPENCLAW_STATE_DIR = stateDir; - clearRuntimeAuthProfileStoreSnapshots(); - - const store = ensureAuthProfileStore(workerAgentDir); - - expectRecordFields(store.profiles["openai:default"], { - type: "oauth", - provider: "openai", - access: "main-access", - }); - expect(fs.existsSync(workerStorePath)).toBe(false); - } finally { - clearRuntimeAuthProfileStoreSnapshots(); - restoreEnvValue("OPENCLAW_STATE_DIR", previousStateDir); - fs.rmSync(root, { recursive: true, force: true }); - } - }); - - it("logs one warning with aggregated reasons for rejected auth-profiles entries", () => { - const warnSpy = vi.spyOn(log, "warn").mockImplementation(() => undefined); - try { - withTempAgentDir("openclaw-auth-invalid-", (agentDir) => { - const invalidStore = { - version: AUTH_STORE_VERSION, - profiles: { - "anthropic:missing-type": { - provider: "anthropic", - }, - "openai:missing-provider": { - type: "api_key", - key: "sk-openai", - }, - "qwen:not-object": "broken", - }, - }; - fs.writeFileSync( - path.join(agentDir, "auth-profiles.json"), - `${JSON.stringify(invalidStore, null, 2)}\n`, - "utf8", - ); - const store = ensureAuthProfileStore(agentDir); - expect(store.profiles).toStrictEqual({}); - expect(warnSpy).toHaveBeenCalledTimes(1); - expect(warnSpy).toHaveBeenCalledWith( - "ignored invalid auth profile entries during store load", - { - source: "auth-profiles.json", - dropped: 3, - reasons: { - invalid_type: 1, - missing_provider: 1, - non_object: 1, - }, - validTypes: ["api_key", "oauth", "token"], - keys: ["anthropic:missing-type", "openai:missing-provider", "qwen:not-object"], - }, - ); - }); - } finally { - warnSpy.mockRestore(); - } - }); - - it.each([ - { - name: "migrates SecretRef object in `key` to `keyRef` and clears `key`", - prefix: "openclaw-nonstr-key-ref-", - profileId: "openai:default", - profile: { - type: "api_key", - provider: "openai", - key: { source: "env", provider: "default", id: "OPENAI_API_KEY" }, - }, - assert(profile: AuthProfileCredential) { - const apiKey = expectApiKeyProfile(profile); - expect(apiKey.key).toBeUndefined(); - expect(apiKey.keyRef).toEqual({ - source: "env", - provider: "default", - id: "OPENAI_API_KEY", - }); - }, - }, - { - name: "deletes non-string non-SecretRef `key` without setting keyRef", - prefix: "openclaw-nonstr-key-num-", - profileId: "openai:default", - profile: { - type: "api_key", - provider: "openai", - key: 12345, - }, - assert(profile: AuthProfileCredential) { - const apiKey = expectApiKeyProfile(profile); - expect(apiKey.key).toBeUndefined(); - expect(apiKey.keyRef).toBeUndefined(); - }, - }, - { - name: "does not overwrite existing `keyRef` when `key` contains a SecretRef", - prefix: "openclaw-nonstr-key-dup-", - profileId: "openai:default", - profile: { - type: "api_key", - provider: "openai", - key: { source: "env", provider: "default", id: "WRONG_VAR" }, - keyRef: { source: "env", provider: "default", id: "CORRECT_VAR" }, - }, - assert(profile: AuthProfileCredential) { - const apiKey = expectApiKeyProfile(profile); - expect(apiKey.key).toBeUndefined(); - expect(apiKey.keyRef).toEqual({ - source: "env", - provider: "default", - id: "CORRECT_VAR", - }); - }, - }, - { - name: "overwrites malformed `keyRef` with migrated ref from `key`", - prefix: "openclaw-nonstr-key-malformed-ref-", - profileId: "openai:default", - profile: { - type: "api_key", - provider: "openai", - key: { source: "env", provider: "default", id: "OPENAI_API_KEY" }, - keyRef: null, - }, - assert(profile: AuthProfileCredential) { - const apiKey = expectApiKeyProfile(profile); - expect(apiKey.key).toBeUndefined(); - expect(apiKey.keyRef).toEqual({ - source: "env", - provider: "default", - id: "OPENAI_API_KEY", - }); - }, - }, - { - name: "preserves valid string `key` values unchanged", - prefix: "openclaw-str-key-", - profileId: "openai:default", - profile: { - type: "api_key", - provider: "openai", - key: "sk-valid-plaintext-key", - }, - assert(profile: AuthProfileCredential) { - const apiKey = expectApiKeyProfile(profile); - expect(apiKey.key).toBe("sk-valid-plaintext-key"); - }, - }, - { - name: "migrates SecretRef object in `token` to `tokenRef` and clears `token`", - prefix: "openclaw-nonstr-token-ref-", - profileId: "anthropic:default", - profile: { - type: "token", - provider: "anthropic", - token: { source: "env", provider: "default", id: "ANTHROPIC_TOKEN" }, - }, - assert(profile: AuthProfileCredential) { - const token = expectTokenProfile(profile); - expect(token.token).toBeUndefined(); - expect(token.tokenRef).toEqual({ - source: "env", - provider: "default", - id: "ANTHROPIC_TOKEN", - }); - }, - }, - { - name: "deletes non-string non-SecretRef `token` without setting tokenRef", - prefix: "openclaw-nonstr-token-num-", - profileId: "anthropic:default", - profile: { - type: "token", - provider: "anthropic", - token: 99999, - }, - assert(profile: AuthProfileCredential) { - const token = expectTokenProfile(profile); - expect(token.token).toBeUndefined(); - expect(token.tokenRef).toBeUndefined(); - }, - }, - { - name: "preserves valid string `token` values unchanged", - prefix: "openclaw-str-token-", - profileId: "anthropic:default", - profile: { - type: "token", - provider: "anthropic", - token: "tok-valid-plaintext", - }, - assert(profile: AuthProfileCredential) { - const token = expectTokenProfile(profile); - expect(token.token).toBe("tok-valid-plaintext"); - }, - }, - ] as const)( - "normalizes secret-backed auth profile fields during store load: $name (#58861)", - (testCase) => { - withTempAgentDir(testCase.prefix, (agentDir) => { - writeAuthProfileStore(agentDir, { [testCase.profileId]: testCase.profile }); - const profile = loadAuthProfile(agentDir, testCase.profileId); - testCase.assert(profile); - }); - }, - ); -}); diff --git a/src/agents/auth-profiles.markauthprofilefailure.test.ts b/src/agents/auth-profiles.markauthprofilefailure.test.ts index 433c2c58981f..84ba0325ebda 100644 --- a/src/agents/auth-profiles.markauthprofilefailure.test.ts +++ b/src/agents/auth-profiles.markauthprofilefailure.test.ts @@ -2,6 +2,7 @@ import fs from "node:fs"; import os from "node:os"; import path from "node:path"; import { afterAll, beforeAll, describe, expect, it, vi } from "vitest"; +import { closeOpenClawAgentDatabasesForTest } from "../state/openclaw-agent-db.js"; vi.mock("./cli-credentials.js", () => ({ readClaudeCliCredentialsCached: () => null, @@ -16,6 +17,7 @@ vi.mock("../plugins/provider-runtime.js", () => ({ import { clearRuntimeAuthProfileStoreSnapshots, ensureAuthProfileStore, + saveAuthProfileStore, } from "./auth-profiles/store.js"; import { calculateAuthProfileCooldownMs, @@ -34,6 +36,7 @@ beforeAll(() => { afterAll(() => { clearRuntimeAuthProfileStoreSnapshots(); + closeOpenClawAgentDatabasesForTest(); fs.rmSync(tempRoot, { recursive: true, force: true }); }); @@ -48,10 +51,8 @@ async function withAuthProfileStore( fn: (ctx: { agentDir: string; store: AuthProfileStore }) => Promise, ): Promise { const agentDir = makeAgentDir("store"); - const authPath = path.join(agentDir, "auth-profiles.json"); - fs.writeFileSync( - authPath, - JSON.stringify({ + saveAuthProfileStore( + { version: 1, profiles: { "anthropic:default": { @@ -65,7 +66,9 @@ async function withAuthProfileStore( key: "sk-or-default", }, }, - }), + }, + agentDir, + { filterExternalAuthProfiles: false, syncExternalCli: false }, ); const store = ensureAuthProfileStore(agentDir); @@ -80,10 +83,8 @@ function expectCooldownInRange(remainingMs: number, minMs: number, maxMs: number describe("markAuthProfileFailure", () => { it("does not overwrite fresher on-disk credentials with a stale runtime snapshot", async () => { const agentDir = makeAgentDir("stale-snapshot"); - const authPath = path.join(agentDir, "auth-profiles.json"); - fs.writeFileSync( - authPath, - JSON.stringify({ + saveAuthProfileStore( + { version: 1, profiles: { "openai:default": { @@ -92,7 +93,9 @@ describe("markAuthProfileFailure", () => { key: "sk-expired-old", }, }, - }), + }, + agentDir, + { filterExternalAuthProfiles: false, syncExternalCli: false }, ); const staleRuntimeStore: AuthProfileStore = { @@ -106,9 +109,8 @@ describe("markAuthProfileFailure", () => { }, }; - fs.writeFileSync( - authPath, - JSON.stringify({ + saveAuthProfileStore( + { version: 1, profiles: { "openai:default": { @@ -117,7 +119,9 @@ describe("markAuthProfileFailure", () => { key: "sk-fresh-new", }, }, - }), + }, + agentDir, + { filterExternalAuthProfiles: false, syncExternalCli: false }, ); const staleCredential = staleRuntimeStore.profiles["openai:default"]; @@ -292,11 +296,9 @@ describe("markAuthProfileFailure", () => { }); it("resets backoff counters outside the failure window", async () => { const agentDir = makeAgentDir("reset-window"); - const authPath = path.join(agentDir, "auth-profiles.json"); const now = Date.now(); - fs.writeFileSync( - authPath, - JSON.stringify({ + saveAuthProfileStore( + { version: 1, profiles: { "anthropic:default": { @@ -312,7 +314,9 @@ describe("markAuthProfileFailure", () => { lastFailureAt: now - 48 * 60 * 60 * 1000, }, }, - }), + }, + agentDir, + { filterExternalAuthProfiles: false, syncExternalCli: false }, ); const store = ensureAuthProfileStore(agentDir); @@ -332,14 +336,12 @@ describe("markAuthProfileFailure", () => { it("resets error count when previous cooldown has expired to prevent escalation", async () => { const agentDir = makeAgentDir("expired-cooldown"); - const authPath = path.join(agentDir, "auth-profiles.json"); const now = Date.now(); // Simulate state left on disk after 3 rapid failures within a 1-min cooldown // window. The cooldown has since expired, but clearExpiredCooldowns() only // ran in-memory and never persisted - so disk still carries errorCount: 3. - fs.writeFileSync( - authPath, - JSON.stringify({ + saveAuthProfileStore( + { version: 1, profiles: { "anthropic:default": { @@ -356,7 +358,9 @@ describe("markAuthProfileFailure", () => { cooldownUntil: now - 60_000, // expired 1 minute ago }, }, - }), + }, + agentDir, + { filterExternalAuthProfiles: false, syncExternalCli: false }, ); const store = ensureAuthProfileStore(agentDir); diff --git a/src/agents/auth-profiles.readonly-sync.test.ts b/src/agents/auth-profiles.readonly-sync.test.ts index 491b7ea2c23e..9a0560975757 100644 --- a/src/agents/auth-profiles.readonly-sync.test.ts +++ b/src/agents/auth-profiles.readonly-sync.test.ts @@ -2,22 +2,27 @@ import fs from "node:fs"; import os from "node:os"; import path from "node:path"; import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; +import { closeOpenClawAgentDatabasesForTest } from "../state/openclaw-agent-db.js"; import { AUTH_STORE_VERSION } from "./auth-profiles/constants.js"; +import { loadPersistedAuthProfileStore } from "./auth-profiles/persisted.js"; +import { saveAuthProfileStore } from "./auth-profiles/store.js"; import type { AuthProfileStore } from "./auth-profiles/types.js"; -const resolveExternalAuthProfilesWithPluginsMock = vi.fn(() => [ - { - profileId: "minimax-portal:default", - credential: { - type: "oauth" as const, - provider: "minimax-portal", - access: "access-token", - refresh: "refresh-token", - expires: Date.now() + 60_000, +const { resolveExternalAuthProfilesWithPluginsMock } = vi.hoisted(() => ({ + resolveExternalAuthProfilesWithPluginsMock: vi.fn(() => [ + { + profileId: "minimax-portal:default", + credential: { + type: "oauth" as const, + provider: "minimax-portal", + access: "access-token", + refresh: "refresh-token", + expires: Date.now() + 60_000, + }, + persistence: "runtime-only" as const, }, - persistence: "runtime-only" as const, - }, -]); + ]), +})); vi.mock("../plugins/provider-runtime.js", () => ({ resolveExternalAuthProfilesWithPlugins: resolveExternalAuthProfilesWithPluginsMock, @@ -47,13 +52,13 @@ describe("auth profiles read-only external auth overlay", () => { afterEach(() => { clearRuntimeAuthProfileStoreSnapshots(); + closeOpenClawAgentDatabasesForTest(); vi.clearAllMocks(); }); it("overlays runtime-only external auth without writing auth-profiles.json in read-only mode", () => { const agentDir = fs.mkdtempSync(path.join(os.tmpdir(), "openclaw-auth-readonly-sync-")); try { - const authPath = path.join(agentDir, "auth-profiles.json"); const baseline: AuthProfileStore = { version: AUTH_STORE_VERSION, profiles: { @@ -64,7 +69,10 @@ describe("auth profiles read-only external auth overlay", () => { }, }, }; - fs.writeFileSync(authPath, `${JSON.stringify(baseline, null, 2)}\n`, "utf8"); + saveAuthProfileStore(baseline, agentDir, { + filterExternalAuthProfiles: false, + syncExternalCli: false, + }); const loaded = loadAuthProfileStoreForRuntime(agentDir, { readOnly: true }); @@ -90,7 +98,10 @@ describe("auth profiles read-only external auth overlay", () => { expect(loaded.profiles["minimax-portal:default"]?.type).toBe("oauth"); expect(loaded.profiles["minimax-portal:default"]?.provider).toBe("minimax-portal"); - const persisted = JSON.parse(fs.readFileSync(authPath, "utf8")) as AuthProfileStore; + const persisted = loadPersistedAuthProfileStore(agentDir) ?? { + version: AUTH_STORE_VERSION, + profiles: {}, + }; expect(persisted.profiles["minimax-portal:default"]).toBeUndefined(); const persistedOpenAiProfile = persisted.profiles["openai:default"]; expect(persistedOpenAiProfile?.type).toBe("api_key"); @@ -100,6 +111,7 @@ describe("auth profiles read-only external auth overlay", () => { expect(persistedOpenAiProfile.provider).toBe("openai"); expect(persistedOpenAiProfile.key).toBe("sk-test"); } finally { + closeOpenClawAgentDatabasesForTest(); fs.rmSync(agentDir, { recursive: true, force: true }); } }); diff --git a/src/agents/auth-profiles.sqlite-store.test.ts b/src/agents/auth-profiles.sqlite-store.test.ts new file mode 100644 index 000000000000..e717ad598cbb --- /dev/null +++ b/src/agents/auth-profiles.sqlite-store.test.ts @@ -0,0 +1,251 @@ +import fs from "node:fs"; +import os from "node:os"; +import path from "node:path"; +import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; +import { + closeOpenClawAgentDatabasesForTest, + openOpenClawAgentDatabase, +} from "../state/openclaw-agent-db.js"; +import { closeOpenClawStateDatabaseForTest } from "../state/openclaw-state-db.js"; +import { resolveOpenClawStateSqlitePath } from "../state/openclaw-state-db.paths.js"; +import { resolveAgentDir } from "./agent-scope.js"; +import { loadPersistedAuthProfileStore } from "./auth-profiles/persisted.js"; +import { resolveAuthProfileDatabasePath } from "./auth-profiles/sqlite.js"; +import { + clearRuntimeAuthProfileStoreSnapshots, + ensureAuthProfileStore, + saveAuthProfileStore, +} from "./auth-profiles/store.js"; +import type { AuthProfileStore, OAuthCredential } from "./auth-profiles/types.js"; + +type RuntimeOnlyOverlay = { + profileId: string; + credential: OAuthCredential; + persistence?: "runtime-only" | "persisted"; +}; + +const mocks = vi.hoisted(() => ({ + resolveExternalCliAuthProfiles: vi.fn< + (store?: unknown, options?: unknown) => RuntimeOnlyOverlay[] + >(() => []), +})); + +vi.mock("./auth-profiles/external-cli-sync.js", () => ({ + resolveExternalCliAuthProfiles: mocks.resolveExternalCliAuthProfiles, +})); + +vi.mock("../plugins/provider-runtime.js", () => ({ + resolveExternalAuthProfilesWithPlugins: () => [], +})); + +function apiKeyStore(key: string): AuthProfileStore { + return { + version: 1, + profiles: { + "openai:default": { + type: "api_key", + provider: "openai", + key, + }, + }, + }; +} + +async function withAgentDirEnv(prefix: string, run: (agentDir: string) => void | Promise) { + const root = fs.mkdtempSync(path.join(os.tmpdir(), prefix)); + const previousStateDir = process.env.OPENCLAW_STATE_DIR; + const previousAgentDir = process.env.OPENCLAW_AGENT_DIR; + const agentDir = path.join(root, "agents", "main", "agent"); + try { + process.env.OPENCLAW_STATE_DIR = root; + process.env.OPENCLAW_AGENT_DIR = agentDir; + fs.mkdirSync(agentDir, { recursive: true }); + await run(agentDir); + } finally { + closeOpenClawAgentDatabasesForTest(); + closeOpenClawStateDatabaseForTest(); + if (previousStateDir === undefined) { + delete process.env.OPENCLAW_STATE_DIR; + } else { + process.env.OPENCLAW_STATE_DIR = previousStateDir; + } + if (previousAgentDir === undefined) { + delete process.env.OPENCLAW_AGENT_DIR; + } else { + process.env.OPENCLAW_AGENT_DIR = previousAgentDir; + } + fs.rmSync(root, { recursive: true, force: true }); + } +} + +describe("auth profile sqlite store", () => { + beforeEach(() => { + clearRuntimeAuthProfileStoreSnapshots(); + mocks.resolveExternalCliAuthProfiles.mockReset(); + mocks.resolveExternalCliAuthProfiles.mockReturnValue([]); + }); + + afterEach(() => { + clearRuntimeAuthProfileStoreSnapshots(); + }); + + it("persists auth profiles and runtime scheduling state in the agent sqlite database", async () => { + await withAgentDirEnv("openclaw-auth-sqlite-", (agentDir) => { + saveAuthProfileStore( + { + ...apiKeyStore("sk-test"), + order: { openai: ["openai:default"] }, + lastGood: { openai: "openai:default" }, + usageStats: { "openai:default": { lastUsed: 123 } }, + }, + agentDir, + ); + + const loaded = ensureAuthProfileStore(agentDir, { syncExternalCli: false }); + + expect(loaded.profiles["openai:default"]).toMatchObject({ key: "sk-test" }); + expect(loaded.order?.openai).toEqual(["openai:default"]); + expect(loaded.lastGood?.openai).toBe("openai:default"); + expect(loaded.usageStats?.["openai:default"]?.lastUsed).toBe(123); + expect(fs.existsSync(path.join(agentDir, "auth-profiles.json"))).toBe(false); + expect(fs.existsSync(path.join(agentDir, "auth-state.json"))).toBe(false); + expect(fs.existsSync(path.join(agentDir, "openclaw-agent.sqlite"))).toBe(true); + }); + }); + + it("does not read legacy auth-profiles.json at runtime", async () => { + await withAgentDirEnv("openclaw-auth-no-json-fallback-", (agentDir) => { + fs.writeFileSync( + path.join(agentDir, "auth-profiles.json"), + `${JSON.stringify(apiKeyStore("sk-json"))}\n`, + "utf8", + ); + + const loaded = ensureAuthProfileStore(agentDir, { syncExternalCli: false }); + + expect(loaded.profiles["openai:default"]).toBeUndefined(); + }); + }); + + it("does not create sqlite files for missing-store reads", async () => { + await withAgentDirEnv("openclaw-auth-sqlite-no-create-", (agentDir) => { + expect(loadPersistedAuthProfileStore(agentDir)).toBeNull(); + expect(fs.existsSync(path.join(agentDir, "openclaw-agent.sqlite"))).toBe(false); + }); + }); + + it("reads existing sqlite auth stores without registering shared state", async () => { + await withAgentDirEnv("openclaw-auth-sqlite-readonly-", (agentDir) => { + saveAuthProfileStore(apiKeyStore("sk-test"), agentDir); + closeOpenClawAgentDatabasesForTest(); + closeOpenClawStateDatabaseForTest(); + const stateDbPath = resolveOpenClawStateSqlitePath(); + fs.rmSync(path.dirname(stateDbPath), { recursive: true, force: true }); + + const loaded = loadPersistedAuthProfileStore(agentDir); + + expect(loaded?.profiles["openai:default"]).toMatchObject({ key: "sk-test" }); + expect(fs.existsSync(stateDbPath)).toBe(false); + }); + }); + + it("uses the configured agent id for custom agentDir databases", async () => { + await withAgentDirEnv("openclaw-auth-sqlite-custom-agent-", (envAgentDir) => { + const customAgentDir = path.join(path.dirname(path.dirname(envAgentDir)), "custom-coder"); + const cfg = { + agents: { + list: [{ id: "coder", agentDir: customAgentDir }], + }, + }; + const agentDir = resolveAgentDir(cfg, "coder"); + + saveAuthProfileStore(apiKeyStore("sk-test"), agentDir); + + const database = openOpenClawAgentDatabase({ + agentId: "coder", + path: resolveAuthProfileDatabasePath(agentDir), + }); + expect(database.agentId).toBe("coder"); + }); + }); + + it("keeps SecretRef-backed credentials from persisting duplicate plaintext", async () => { + await withAgentDirEnv("openclaw-auth-sqlite-secret-ref-", (agentDir) => { + saveAuthProfileStore( + { + version: 1, + profiles: { + "openai:default": { + type: "api_key", + provider: "openai", + key: "sk-plaintext", + keyRef: { source: "env", provider: "default", id: "OPENAI_API_KEY" }, + }, + "anthropic:default": { + type: "token", + provider: "anthropic", + token: "token-plaintext", + tokenRef: { source: "env", provider: "default", id: "ANTHROPIC_AUTH_TOKEN" }, + }, + }, + }, + agentDir, + ); + + const loaded = ensureAuthProfileStore(agentDir, { syncExternalCli: false }); + + expect(loaded.profiles["openai:default"]).toEqual({ + type: "api_key", + provider: "openai", + keyRef: { source: "env", provider: "default", id: "OPENAI_API_KEY" }, + }); + expect(loaded.profiles["anthropic:default"]).toEqual({ + type: "token", + provider: "anthropic", + tokenRef: { source: "env", provider: "default", id: "ANTHROPIC_AUTH_TOKEN" }, + }); + }); + }); + + it("recomputes runtime-only external auth overlays from the sqlite base store", async () => { + await withAgentDirEnv("openclaw-auth-sqlite-overlay-", (agentDir) => { + saveAuthProfileStore(apiKeyStore("sk-test"), agentDir); + mocks.resolveExternalCliAuthProfiles + .mockReturnValueOnce([ + { + profileId: "openai:default", + credential: { + type: "oauth", + provider: "openai", + access: "access-1", + refresh: "refresh-1", + expires: Date.now() + 60_000, + }, + }, + ]) + .mockReturnValueOnce([ + { + profileId: "openai:default", + credential: { + type: "oauth", + provider: "openai", + access: "access-2", + refresh: "refresh-2", + expires: Date.now() + 60_000, + }, + }, + ]); + + const first = ensureAuthProfileStore(agentDir); + const second = ensureAuthProfileStore(agentDir); + + expect((first.profiles["openai:default"] as OAuthCredential | undefined)?.access).toBe( + "access-1", + ); + expect((second.profiles["openai:default"] as OAuthCredential | undefined)?.access).toBe( + "access-2", + ); + expect(mocks.resolveExternalCliAuthProfiles).toHaveBeenCalledTimes(2); + }); + }); +}); diff --git a/src/agents/auth-profiles.store-cache.test.ts b/src/agents/auth-profiles.store-cache.test.ts deleted file mode 100644 index 25b4cde3bb1f..000000000000 --- a/src/agents/auth-profiles.store-cache.test.ts +++ /dev/null @@ -1,433 +0,0 @@ -import fs from "node:fs"; -import os from "node:os"; -import path from "node:path"; -import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; -import { AUTH_STORE_LOCK_OPTIONS, AUTH_STORE_VERSION } from "./auth-profiles/constants.js"; -import { - clearRuntimeAuthProfileStoreSnapshots, - ensureAuthProfileStore, - ensureAuthProfileStoreWithoutExternalProfiles, -} from "./auth-profiles/store.js"; -import type { OAuthCredential } from "./auth-profiles/types.js"; - -type RuntimeOnlyOverlay = { - profileId: string; - credential: OAuthCredential; - persistence?: "runtime-only" | "persisted"; -}; - -const mocks = vi.hoisted(() => ({ - resolveExternalCliAuthProfiles: vi.fn< - (store?: unknown, options?: unknown) => RuntimeOnlyOverlay[] - >(() => []), -})); - -vi.mock("./auth-profiles/external-cli-sync.js", () => ({ - resolveExternalCliAuthProfiles: mocks.resolveExternalCliAuthProfiles, -})); - -vi.mock("../plugins/provider-runtime.js", () => ({ - resolveExternalAuthProfilesWithPlugins: () => [], -})); - -async function withAgentDirEnv(prefix: string, run: (agentDir: string) => void | Promise) { - const agentDir = fs.mkdtempSync(path.join(os.tmpdir(), prefix)); - const previousAgentDir = process.env.OPENCLAW_AGENT_DIR; - try { - process.env.OPENCLAW_AGENT_DIR = agentDir; - await run(agentDir); - } finally { - if (previousAgentDir === undefined) { - delete process.env.OPENCLAW_AGENT_DIR; - } else { - process.env.OPENCLAW_AGENT_DIR = previousAgentDir; - } - fs.rmSync(agentDir, { recursive: true, force: true }); - } -} - -function writeAuthStore(agentDir: string, key: string) { - const authPath = path.join(agentDir, "auth-profiles.json"); - fs.writeFileSync( - authPath, - `${JSON.stringify( - { - version: AUTH_STORE_VERSION, - profiles: { - "openai:default": { - type: "api_key", - provider: "openai", - key, - }, - }, - }, - null, - 2, - )}\n`, - "utf8", - ); - return authPath; -} - -function writeOAuthStore(agentDir: string, profileId: string, credential: OAuthCredential) { - const authPath = path.join(agentDir, "auth-profiles.json"); - fs.writeFileSync( - authPath, - `${JSON.stringify( - { - version: AUTH_STORE_VERSION, - profiles: { - [profileId]: credential, - }, - }, - null, - 2, - )}\n`, - "utf8", - ); - return authPath; -} - -describe("auth profile store cache", () => { - beforeEach(() => { - clearRuntimeAuthProfileStoreSnapshots(); - mocks.resolveExternalCliAuthProfiles.mockReset(); - mocks.resolveExternalCliAuthProfiles.mockReturnValue([]); - }); - - afterEach(() => { - vi.useRealTimers(); - clearRuntimeAuthProfileStoreSnapshots(); - }); - - function createRuntimeOnlyOverlay(access: string): RuntimeOnlyOverlay { - return { - profileId: "openai:default", - credential: { - type: "oauth", - provider: "openai", - access, - refresh: `refresh-${access}`, - expires: Date.now() + 60_000, - }, - }; - } - - function createPersistedOverlay( - profileId: string, - credential: OAuthCredential, - ): RuntimeOnlyOverlay { - return { - profileId, - credential, - persistence: "persisted", - }; - } - - it("recomputes runtime-only external auth overlays even while the base store is cached", async () => { - await withAgentDirEnv("openclaw-auth-store-cache-", (agentDir) => { - writeAuthStore(agentDir, "sk-test"); - mocks.resolveExternalCliAuthProfiles - .mockReturnValueOnce([createRuntimeOnlyOverlay("access-1")]) - .mockReturnValueOnce([createRuntimeOnlyOverlay("access-2")]); - - const first = ensureAuthProfileStore(agentDir); - const second = ensureAuthProfileStore(agentDir); - - expect((first.profiles["openai:default"] as OAuthCredential | undefined)?.access).toBe( - "access-1", - ); - expect((second.profiles["openai:default"] as OAuthCredential | undefined)?.access).toBe( - "access-2", - ); - expect(mocks.resolveExternalCliAuthProfiles).toHaveBeenCalledTimes(2); - }); - }); - - it("refreshes the cached auth store after auth-profiles.json changes", async () => { - await withAgentDirEnv("openclaw-auth-store-refresh-", async (agentDir) => { - const authPath = writeAuthStore(agentDir, "sk-test-1"); - - ensureAuthProfileStore(agentDir); - - writeAuthStore(agentDir, "sk-test-2"); - const bumpedMtime = new Date(Date.now() + 2_000); - fs.utimesSync(authPath, bumpedMtime, bumpedMtime); - - const reloaded = ensureAuthProfileStore(agentDir); - - expect((reloaded.profiles["openai:default"] as { key?: string } | undefined)?.key).toBe( - "sk-test-2", - ); - }); - }); - - it("isolates cached auth stores without structuredClone", async () => { - const structuredCloneSpy = vi.spyOn(globalThis, "structuredClone"); - await withAgentDirEnv("openclaw-auth-store-isolated-", (agentDir) => { - writeAuthStore(agentDir, "sk-test"); - - const first = ensureAuthProfileStore(agentDir); - const profile = first.profiles["openai:default"]; - if (profile?.type === "api_key") { - profile.key = "sk-mutated"; - } - first.profiles["anthropic:default"] = { - type: "api_key", - provider: "anthropic", - key: "sk-added", - }; - - const second = ensureAuthProfileStore(agentDir); - expect((second.profiles["openai:default"] as { key?: string } | undefined)?.key).toBe( - "sk-test", - ); - expect(second.profiles["anthropic:default"]).toBeUndefined(); - expect(structuredCloneSpy).not.toHaveBeenCalled(); - }); - structuredCloneSpy.mockRestore(); - }); - - it("keeps runtime-only external auth out of persisted auth-profiles.json files", async () => { - mocks.resolveExternalCliAuthProfiles.mockReturnValue([createRuntimeOnlyOverlay("access-1")]); - - await withAgentDirEnv("openclaw-auth-store-missing-", (agentDir) => { - const store = ensureAuthProfileStore(agentDir); - - expect((store.profiles["openai:default"] as OAuthCredential | undefined)?.access).toBe( - "access-1", - ); - expect(fs.existsSync(path.join(agentDir, "auth-profiles.json"))).toBe(false); - }); - }); - - it("persists fresher external CLI oauth over a stale local managed profile", async () => { - await withAgentDirEnv("openclaw-auth-store-external-cli-persist-", (agentDir) => { - const profileId = "anthropic:claude-cli"; - writeOAuthStore(agentDir, profileId, { - type: "oauth", - provider: "claude-cli", - access: "stale-local-access", - refresh: "stale-local-refresh", - expires: Date.now() - 60_000, - }); - mocks.resolveExternalCliAuthProfiles - .mockReturnValueOnce([ - createPersistedOverlay(profileId, { - type: "oauth", - provider: "claude-cli", - access: "fresh-cli-access", - refresh: "fresh-cli-refresh", - expires: Date.now() + 60_000, - }), - ]) - .mockReturnValue([]); - - const store = ensureAuthProfileStore(agentDir); - const persisted = JSON.parse( - fs.readFileSync(path.join(agentDir, "auth-profiles.json"), "utf8"), - ) as { profiles: Record }; - - expect((store.profiles[profileId] as OAuthCredential | undefined)?.access).toBe( - "fresh-cli-access", - ); - expect(persisted.profiles[profileId]?.access).toBe("fresh-cli-access"); - expect(persisted.profiles[profileId]?.refresh).toBe("fresh-cli-refresh"); - }); - }); - - it("preserves concurrent auth-store updates while persisting external CLI oauth", async () => { - await withAgentDirEnv("openclaw-auth-store-external-cli-concurrent-", (agentDir) => { - const profileId = "anthropic:claude-cli"; - const authPath = writeOAuthStore(agentDir, profileId, { - type: "oauth", - provider: "claude-cli", - access: "stale-local-access", - refresh: "stale-local-refresh", - expires: Date.now() - 60_000, - }); - mocks.resolveExternalCliAuthProfiles.mockImplementationOnce(() => { - const current = JSON.parse(fs.readFileSync(authPath, "utf8")) as { - profiles: Record; - }; - fs.writeFileSync( - authPath, - `${JSON.stringify( - { - ...current, - profiles: { - ...current.profiles, - "openai:default": { - type: "api_key", - provider: "openai", - key: "sk-concurrent", - }, - }, - }, - null, - 2, - )}\n`, - "utf8", - ); - return [ - createPersistedOverlay(profileId, { - type: "oauth", - provider: "claude-cli", - access: "fresh-cli-access", - refresh: "fresh-cli-refresh", - expires: Date.now() + 60_000, - }), - ]; - }); - - ensureAuthProfileStore(agentDir); - const persisted = JSON.parse(fs.readFileSync(authPath, "utf8")) as { - profiles: Record; - }; - const cliProfile = persisted.profiles[profileId] as OAuthCredential | undefined; - const openaiProfile = persisted.profiles["openai:default"] as { key?: string } | undefined; - - expect(cliProfile?.access).toBe("fresh-cli-access"); - expect(openaiProfile?.key).toBe("sk-concurrent"); - }); - }); - - it("returns the reloaded store when the synced CLI profile changed concurrently", async () => { - await withAgentDirEnv("openclaw-auth-store-external-cli-profile-race-", (agentDir) => { - const profileId = "anthropic:claude-cli"; - const authPath = writeOAuthStore(agentDir, profileId, { - type: "oauth", - provider: "claude-cli", - access: "stale-local-access", - refresh: "stale-local-refresh", - expires: Date.now() - 60_000, - }); - mocks.resolveExternalCliAuthProfiles.mockImplementationOnce(() => { - writeOAuthStore(agentDir, profileId, { - type: "oauth", - provider: "claude-cli", - access: "manual-concurrent-access", - refresh: "manual-concurrent-refresh", - expires: Date.now() + 120_000, - }); - return [ - createPersistedOverlay(profileId, { - type: "oauth", - provider: "claude-cli", - access: "fresh-cli-access", - refresh: "fresh-cli-refresh", - expires: Date.now() + 60_000, - }), - ]; - }); - - const first = ensureAuthProfileStore(agentDir); - const second = ensureAuthProfileStore(agentDir); - const persisted = JSON.parse(fs.readFileSync(authPath, "utf8")) as { - profiles: Record; - }; - - expect((first.profiles[profileId] as OAuthCredential | undefined)?.access).toBe( - "manual-concurrent-access", - ); - expect((second.profiles[profileId] as OAuthCredential | undefined)?.access).toBe( - "manual-concurrent-access", - ); - expect(persisted.profiles[profileId]?.access).toBe("manual-concurrent-access"); - }); - }); - - it("does not reclaim an existing auth-store lock while syncing external CLI oauth", async () => { - await withAgentDirEnv("openclaw-auth-store-external-cli-live-lock-", (agentDir) => { - const profileId = "anthropic:claude-cli"; - const authPath = writeOAuthStore(agentDir, profileId, { - type: "oauth", - provider: "claude-cli", - access: "stale-local-access", - refresh: "stale-local-refresh", - expires: Date.now() - 60_000, - }); - const lockPath = `${authPath}.lock`; - const lockRaw = `${JSON.stringify( - { - pid: process.pid, - createdAt: new Date(Date.now() - AUTH_STORE_LOCK_OPTIONS.stale - 1_000).toISOString(), - }, - null, - 2, - )}\n`; - fs.writeFileSync(lockPath, lockRaw, "utf8"); - const oldLockTime = new Date(Date.now() - AUTH_STORE_LOCK_OPTIONS.stale - 1_000); - fs.utimesSync(lockPath, oldLockTime, oldLockTime); - mocks.resolveExternalCliAuthProfiles.mockReturnValue([ - createPersistedOverlay(profileId, { - type: "oauth", - provider: "claude-cli", - access: "fresh-cli-access", - refresh: "fresh-cli-refresh", - expires: Date.now() + 60_000, - }), - ]); - - ensureAuthProfileStore(agentDir); - const persisted = JSON.parse(fs.readFileSync(authPath, "utf8")) as { - profiles: Record; - }; - - expect(fs.readFileSync(lockPath, "utf8")).toBe(lockRaw); - expect(persisted.profiles[profileId]?.access).toBe("stale-local-access"); - expect(persisted.profiles[profileId]?.refresh).toBe("stale-local-refresh"); - }); - }); - - it("does not cache stale auth after external CLI sync lock contention", async () => { - await withAgentDirEnv("openclaw-auth-store-external-cli-locked-cache-", (agentDir) => { - const profileId = "anthropic:claude-cli"; - const authPath = writeOAuthStore(agentDir, profileId, { - type: "oauth", - provider: "claude-cli", - access: "stale-local-access", - refresh: "stale-local-refresh", - expires: Date.now() - 60_000, - }); - const lockPath = `${authPath}.lock`; - fs.writeFileSync( - lockPath, - `${JSON.stringify({ pid: process.pid, createdAt: new Date().toISOString() }, null, 2)}\n`, - "utf8", - ); - mocks.resolveExternalCliAuthProfiles - .mockImplementationOnce(() => { - writeOAuthStore(agentDir, profileId, { - type: "oauth", - provider: "claude-cli", - access: "fresh-disk-access", - refresh: "fresh-disk-refresh", - expires: Date.now() + 120_000, - }); - const bumpedMtime = new Date(Date.now() + 2_000); - fs.utimesSync(authPath, bumpedMtime, bumpedMtime); - return [ - createPersistedOverlay(profileId, { - type: "oauth", - provider: "claude-cli", - access: "fresh-cli-access", - refresh: "fresh-cli-refresh", - expires: Date.now() + 60_000, - }), - ]; - }) - .mockReturnValue([]); - - const first = ensureAuthProfileStoreWithoutExternalProfiles(agentDir); - const second = ensureAuthProfileStoreWithoutExternalProfiles(agentDir); - - expect((first.profiles[profileId] as OAuthCredential | undefined)?.access).toBe( - "stale-local-access", - ); - expect((second.profiles[profileId] as OAuthCredential | undefined)?.access).toBe( - "fresh-disk-access", - ); - }); - }); -}); diff --git a/src/agents/auth-profiles.store.save.test.ts b/src/agents/auth-profiles.store.save.test.ts deleted file mode 100644 index bc48e89c5094..000000000000 --- a/src/agents/auth-profiles.store.save.test.ts +++ /dev/null @@ -1,1421 +0,0 @@ -import fs from "node:fs/promises"; -import os from "node:os"; -import path from "node:path"; -import { beforeEach, describe, expect, it, vi } from "vitest"; -import { MAX_DATE_TIMESTAMP_MS } from "../shared/number-coercion.js"; -import { resolveAuthStatePath, resolveAuthStorePath } from "./auth-profiles/paths.js"; -import { getRuntimeAuthProfileStoreSnapshot } from "./auth-profiles/runtime-snapshots.js"; -import { - clearRuntimeAuthProfileStoreSnapshots, - ensureAuthProfileStoreForLocalUpdate, - ensureAuthProfileStore, - ensureAuthProfileStoreWithoutExternalProfiles, - replaceRuntimeAuthProfileStoreSnapshots, - saveAuthProfileStore, -} from "./auth-profiles/store.js"; -import type { AuthProfileStore } from "./auth-profiles/types.js"; - -const externalAuthMocks = vi.hoisted(() => ({ - listRuntimeExternalAuthProfiles: vi.fn((params?: { store?: unknown }) => { - const store = params?.store as { profiles?: Record } | undefined; - return Object.entries(store?.profiles ?? {}) - .filter(([, credential]) => (credential as { type?: string }).type === "oauth") - .map(([profileId, credential]) => ({ - profileId, - credential, - persistence: externalAuthMocks.shouldPersistExternalAuthProfile({ profileId }) - ? "persisted" - : "runtime-only", - })); - }), - overlayExternalAuthProfiles: vi.fn((store: unknown) => store), - shouldPersistExternalAuthProfile: vi.fn((_params?: { profileId?: string }) => true), -})); - -vi.mock("./auth-profiles/external-auth.js", () => ({ - listRuntimeExternalAuthProfiles: externalAuthMocks.listRuntimeExternalAuthProfiles, - overlayExternalAuthProfiles: externalAuthMocks.overlayExternalAuthProfiles, - shouldPersistExternalAuthProfile: externalAuthMocks.shouldPersistExternalAuthProfile, - syncPersistedExternalCliAuthProfiles: (store: T) => store, -})); - -function requireRecord(value: unknown, label: string): Record { - if (!value || typeof value !== "object") { - throw new Error(`expected ${label} to be an object`); - } - return value as Record; -} - -function expectProfileFields(profile: unknown, expected: Record): void { - const actual = requireRecord(profile, "auth profile"); - for (const [key, value] of Object.entries(expected)) { - expect(actual[key]).toEqual(value); - } -} - -describe("saveAuthProfileStore", () => { - it("resolves external auth profiles once per save", async () => { - const agentDir = await fs.mkdtemp(path.join(os.tmpdir(), "openclaw-auth-save-once-")); - const store: AuthProfileStore = { - version: 1, - profiles: { - "openai:one": { - type: "oauth", - provider: "openai", - access: "access-one", - refresh: "refresh-one", - expires: Date.now() + 60_000, - }, - "openai:two": { - type: "oauth", - provider: "openai", - access: "access-two", - refresh: "refresh-two", - expires: Date.now() + 60_000, - }, - "openai:default": { - type: "api_key", - provider: "openai", - key: "sk-openai", - }, - }, - }; - - try { - externalAuthMocks.listRuntimeExternalAuthProfiles.mockClear(); - - saveAuthProfileStore(store, agentDir); - - expect(externalAuthMocks.listRuntimeExternalAuthProfiles).toHaveBeenCalledTimes(1); - expect(externalAuthMocks.listRuntimeExternalAuthProfiles.mock.calls[0]?.[0]).toMatchObject({ - store, - agentDir, - }); - } finally { - await fs.rm(agentDir, { recursive: true, force: true }); - } - }); - - beforeEach(() => { - externalAuthMocks.listRuntimeExternalAuthProfiles.mockClear(); - externalAuthMocks.overlayExternalAuthProfiles.mockImplementation((store) => store); - externalAuthMocks.shouldPersistExternalAuthProfile.mockReturnValue(true); - }); - - it("strips plaintext when keyRef/tokenRef are present", async () => { - const structuredCloneSpy = vi.spyOn(globalThis, "structuredClone"); - const agentDir = await fs.mkdtemp(path.join(os.tmpdir(), "openclaw-auth-save-")); - try { - const store: AuthProfileStore = { - version: 1, - profiles: { - "openai:default": { - type: "api_key", - provider: "openai", - key: "sk-runtime-value", - keyRef: { source: "env", provider: "default", id: "OPENAI_API_KEY" }, - }, - "github-copilot:default": { - type: "token", - provider: "github-copilot", - token: "gh-runtime-token", - tokenRef: { source: "env", provider: "default", id: "GITHUB_TOKEN" }, - }, - "anthropic:default": { - type: "api_key", - provider: "anthropic", - key: "sk-anthropic-plain", - }, - }, - }; - - saveAuthProfileStore(store, agentDir); - - const parsed = JSON.parse(await fs.readFile(resolveAuthStorePath(agentDir), "utf8")) as { - profiles: Record< - string, - { key?: string; keyRef?: unknown; token?: string; tokenRef?: unknown } - >; - }; - - expect(parsed.profiles["openai:default"]?.key).toBeUndefined(); - expect(parsed.profiles["openai:default"]?.keyRef).toEqual({ - source: "env", - provider: "default", - id: "OPENAI_API_KEY", - }); - - expect(parsed.profiles["github-copilot:default"]?.token).toBeUndefined(); - expect(parsed.profiles["github-copilot:default"]?.tokenRef).toEqual({ - source: "env", - provider: "default", - id: "GITHUB_TOKEN", - }); - - expect(parsed.profiles["anthropic:default"]?.key).toBe("sk-anthropic-plain"); - expect(structuredCloneSpy).not.toHaveBeenCalled(); - } finally { - structuredCloneSpy.mockRestore(); - await fs.rm(agentDir, { recursive: true, force: true }); - } - }); - - it("refreshes the runtime snapshot when a saved store rotates oauth tokens", async () => { - const agentDir = await fs.mkdtemp(path.join(os.tmpdir(), "openclaw-auth-save-runtime-")); - try { - replaceRuntimeAuthProfileStoreSnapshots([ - { - agentDir, - store: { - version: 1, - profiles: { - "anthropic:default": { - type: "oauth", - provider: "anthropic", - access: "access-1", - refresh: "refresh-1", - expires: 1, - }, - }, - }, - }, - ]); - - expectProfileFields(ensureAuthProfileStore(agentDir).profiles["anthropic:default"], { - access: "access-1", - refresh: "refresh-1", - }); - - const rotatedStore: AuthProfileStore = { - version: 1, - profiles: { - "anthropic:default": { - type: "oauth", - provider: "anthropic", - access: "access-2", - refresh: "refresh-2", - expires: 2, - }, - }, - }; - - saveAuthProfileStore(rotatedStore, agentDir); - - expectProfileFields(ensureAuthProfileStore(agentDir).profiles["anthropic:default"], { - access: "access-2", - refresh: "refresh-2", - }); - - const persisted = JSON.parse(await fs.readFile(resolveAuthStorePath(agentDir), "utf8")) as { - profiles: Record; - }; - expectProfileFields(persisted.profiles["anthropic:default"], { - access: "access-2", - refresh: "refresh-2", - }); - } finally { - clearRuntimeAuthProfileStoreSnapshots(); - await fs.rm(agentDir, { recursive: true, force: true }); - } - }); - - it("keeps runtime-only external cli oauth profiles in active runtime snapshots", async () => { - const agentDir = await fs.mkdtemp(path.join(os.tmpdir(), "openclaw-auth-save-external-")); - const externalProfileId = "anthropic:claude-cli"; - const localAnthropicProfileId = "anthropic:local"; - const localProfileId = "openai:default"; - externalAuthMocks.shouldPersistExternalAuthProfile.mockImplementation( - (params?: { profileId?: string }) => params?.profileId !== externalProfileId, - ); - - try { - replaceRuntimeAuthProfileStoreSnapshots([ - { - agentDir, - store: { - version: 1, - profiles: { - [externalProfileId]: { - type: "oauth", - provider: "anthropic", - access: "stale-external-access", - refresh: "stale-external-refresh", - expires: 1, - }, - }, - }, - }, - ]); - - const runtimeStore: AuthProfileStore = { - version: 1, - runtimeExternalProfileIds: [externalProfileId], - runtimeExternalProfileIdsAuthoritative: true, - profiles: { - [externalProfileId]: { - type: "oauth", - provider: "anthropic", - access: "external-access", - refresh: "external-refresh", - expires: 2, - }, - [localProfileId]: { - type: "api_key", - provider: "openai", - key: "sk-local", - }, - [localAnthropicProfileId]: { - type: "api_key", - provider: "anthropic", - key: "sk-anthropic-local", - }, - }, - order: { - anthropic: [externalProfileId], - openai: [localProfileId], - }, - lastGood: { - anthropic: externalProfileId, - openai: localProfileId, - }, - usageStats: { - [externalProfileId]: { - lastUsed: 123, - }, - [localProfileId]: { - lastUsed: 456, - }, - }, - }; - externalAuthMocks.overlayExternalAuthProfiles.mockImplementation((store) => { - const base = store as AuthProfileStore; - const externalUsage = base.usageStats?.[externalProfileId] ?? { lastUsed: 123 }; - return { - ...base, - profiles: { - ...base.profiles, - [externalProfileId]: runtimeStore.profiles[externalProfileId], - }, - order: { - ...base.order, - anthropic: [externalProfileId], - }, - lastGood: { - ...base.lastGood, - anthropic: externalProfileId, - }, - usageStats: { - ...base.usageStats, - [externalProfileId]: externalUsage, - }, - runtimeExternalProfileIds: [externalProfileId], - runtimeExternalProfileIdsAuthoritative: true, - }; - }); - - saveAuthProfileStore(runtimeStore, agentDir); - - const persisted = JSON.parse(await fs.readFile(resolveAuthStorePath(agentDir), "utf8")) as { - profiles: Record; - }; - expect(persisted.profiles[externalProfileId]).toBeUndefined(); - expectProfileFields(persisted.profiles[localProfileId], { - type: "api_key", - provider: "openai", - key: "sk-local", - }); - - const persistedState = JSON.parse( - await fs.readFile(resolveAuthStatePath(agentDir), "utf8"), - ) as { - order?: Record; - lastGood?: Record; - usageStats?: Record; - }; - expect(persistedState.order?.anthropic).toBeUndefined(); - expect(persistedState.lastGood?.anthropic).toBeUndefined(); - expect(persistedState.usageStats?.[externalProfileId]).toBeUndefined(); - expect(persistedState.order?.openai).toEqual([localProfileId]); - - const runtime = ensureAuthProfileStore(agentDir); - expectProfileFields(runtime.profiles[externalProfileId], { - type: "oauth", - provider: "anthropic", - access: "external-access", - refresh: "external-refresh", - }); - expect(runtime.order?.anthropic).toEqual([externalProfileId]); - expect(runtime.lastGood?.anthropic).toBe(externalProfileId); - expect(runtime.usageStats?.[externalProfileId]?.lastUsed).toBe(123); - - const runtimeWithoutExternal = ensureAuthProfileStoreWithoutExternalProfiles(agentDir); - expect(runtimeWithoutExternal.profiles[externalProfileId]).toBeUndefined(); - expect(runtimeWithoutExternal.order?.anthropic).toBeUndefined(); - expect(runtimeWithoutExternal.lastGood?.anthropic).toBeUndefined(); - expect(runtimeWithoutExternal.usageStats?.[externalProfileId]).toBeUndefined(); - - saveAuthProfileStore( - { - ...runtimeStore, - profiles: { - ...runtimeStore.profiles, - [externalProfileId]: { - type: "oauth", - provider: "anthropic", - access: "refreshed-external-access", - refresh: "refreshed-external-refresh", - expires: 3, - }, - }, - usageStats: { - ...runtimeStore.usageStats, - [externalProfileId]: { - lastUsed: 789, - }, - }, - }, - agentDir, - ); - const snapshotAfterRuntimeBackedSave = getRuntimeAuthProfileStoreSnapshot(agentDir); - expectProfileFields(snapshotAfterRuntimeBackedSave?.profiles[externalProfileId], { - type: "oauth", - provider: "anthropic", - access: "refreshed-external-access", - refresh: "refreshed-external-refresh", - }); - expect(snapshotAfterRuntimeBackedSave?.usageStats?.[externalProfileId]?.lastUsed).toBe(789); - - saveAuthProfileStore(runtimeWithoutExternal, agentDir); - const persistedAfterDiskBackedSave = JSON.parse( - await fs.readFile(resolveAuthStorePath(agentDir), "utf8"), - ) as { - profiles: Record; - }; - expect(persistedAfterDiskBackedSave.profiles[externalProfileId]).toBeUndefined(); - const snapshotAfterDiskBackedSave = getRuntimeAuthProfileStoreSnapshot(agentDir); - expect(snapshotAfterDiskBackedSave?.runtimeExternalProfileIds).toEqual([externalProfileId]); - expect(snapshotAfterDiskBackedSave?.runtimeExternalProfileIdsAuthoritative).toBe(true); - expectProfileFields(snapshotAfterDiskBackedSave?.profiles[externalProfileId], { - type: "oauth", - provider: "anthropic", - access: "refreshed-external-access", - refresh: "refreshed-external-refresh", - }); - expectProfileFields(snapshotAfterDiskBackedSave?.profiles[localProfileId], { - type: "api_key", - provider: "openai", - key: "sk-local", - }); - expect(snapshotAfterDiskBackedSave?.order?.anthropic).toEqual([externalProfileId]); - expect(snapshotAfterDiskBackedSave?.lastGood?.anthropic).toBe(externalProfileId); - expect(snapshotAfterDiskBackedSave?.usageStats?.[externalProfileId]?.lastUsed).toBe(789); - const ensuredRuntime = ensureAuthProfileStore(agentDir); - expectProfileFields(ensuredRuntime.profiles[localProfileId], { - type: "api_key", - provider: "openai", - key: "sk-local", - }); - expect(ensuredRuntime.order?.anthropic).toEqual([externalProfileId]); - expect(ensuredRuntime.lastGood?.anthropic).toBe(externalProfileId); - expect(ensuredRuntime.usageStats?.[externalProfileId]?.lastUsed).toBe(789); - - saveAuthProfileStore( - { - ...runtimeWithoutExternal, - order: { - ...runtimeWithoutExternal.order, - anthropic: [localAnthropicProfileId], - }, - lastGood: { - ...runtimeWithoutExternal.lastGood, - anthropic: localAnthropicProfileId, - }, - }, - agentDir, - ); - const snapshotAfterExplicitOrderSave = getRuntimeAuthProfileStoreSnapshot(agentDir); - expectProfileFields(snapshotAfterExplicitOrderSave?.profiles[externalProfileId], { - type: "oauth", - provider: "anthropic", - access: "refreshed-external-access", - refresh: "refreshed-external-refresh", - }); - expect(snapshotAfterExplicitOrderSave?.order?.anthropic).toEqual([localAnthropicProfileId]); - expect(snapshotAfterExplicitOrderSave?.lastGood?.anthropic).toBe(localAnthropicProfileId); - - saveAuthProfileStore( - { - ...runtimeWithoutExternal, - runtimeExternalProfileIds: [], - runtimeExternalProfileIdsAuthoritative: true, - }, - agentDir, - ); - const snapshotAfterAuthoritativeRemoval = getRuntimeAuthProfileStoreSnapshot(agentDir); - expect(snapshotAfterAuthoritativeRemoval?.runtimeExternalProfileIds).toEqual([]); - expect(snapshotAfterAuthoritativeRemoval?.runtimeExternalProfileIdsAuthoritative).toBe(true); - expect(snapshotAfterAuthoritativeRemoval?.profiles[externalProfileId]).toBeUndefined(); - } finally { - clearRuntimeAuthProfileStoreSnapshots(); - await fs.rm(agentDir, { recursive: true, force: true }); - } - }); - - it("preserves unrelated runtime-only external profiles after scoped runtime saves", async () => { - const agentDir = await fs.mkdtemp(path.join(os.tmpdir(), "openclaw-auth-save-scoped-")); - const scopedProfileId = "anthropic:claude-cli"; - const unrelatedProfileId = "minimax:minimax-cli"; - externalAuthMocks.shouldPersistExternalAuthProfile.mockReturnValue(false); - - try { - replaceRuntimeAuthProfileStoreSnapshots([ - { - agentDir, - store: { - version: 1, - runtimeExternalProfileIds: [scopedProfileId, unrelatedProfileId], - runtimeExternalProfileIdsAuthoritative: true, - profiles: { - [scopedProfileId]: { - type: "oauth", - provider: "anthropic", - access: "old-scoped-access", - refresh: "old-scoped-refresh", - expires: 1, - }, - [unrelatedProfileId]: { - type: "oauth", - provider: "minimax-portal", - access: "unrelated-access", - refresh: "unrelated-refresh", - expires: 2, - }, - }, - order: { - anthropic: [scopedProfileId], - "minimax-portal": [unrelatedProfileId], - }, - lastGood: { - anthropic: scopedProfileId, - "minimax-portal": unrelatedProfileId, - }, - usageStats: { - [scopedProfileId]: { lastUsed: 10 }, - [unrelatedProfileId]: { lastUsed: 20 }, - }, - }, - }, - ]); - - saveAuthProfileStore( - { - version: 1, - runtimeExternalProfileIds: [scopedProfileId], - profiles: { - [scopedProfileId]: { - type: "oauth", - provider: "anthropic", - access: "new-scoped-access", - refresh: "new-scoped-refresh", - expires: 3, - }, - }, - order: { - anthropic: [scopedProfileId], - }, - usageStats: { - [scopedProfileId]: { lastUsed: 30 }, - }, - }, - agentDir, - ); - - const snapshot = getRuntimeAuthProfileStoreSnapshot(agentDir); - expect(snapshot?.runtimeExternalProfileIds).toEqual([scopedProfileId, unrelatedProfileId]); - expect(snapshot?.runtimeExternalProfileIdsAuthoritative).toBe(true); - expectProfileFields(snapshot?.profiles[scopedProfileId], { - type: "oauth", - provider: "anthropic", - access: "new-scoped-access", - refresh: "new-scoped-refresh", - }); - expectProfileFields(snapshot?.profiles[unrelatedProfileId], { - type: "oauth", - provider: "minimax-portal", - access: "unrelated-access", - refresh: "unrelated-refresh", - }); - expect(snapshot?.usageStats?.[scopedProfileId]?.lastUsed).toBe(30); - expect(snapshot?.usageStats?.[unrelatedProfileId]?.lastUsed).toBe(20); - expect(snapshot?.order?.anthropic).toEqual([scopedProfileId]); - expect(snapshot?.order?.["minimax-portal"]).toEqual([unrelatedProfileId]); - const scopedRead = ensureAuthProfileStore(agentDir, { - externalCliProviderIds: ["anthropic"], - }); - expect(scopedRead.profiles[unrelatedProfileId]).toBeUndefined(); - - saveAuthProfileStore( - { - version: 1, - runtimeExternalProfileIds: [scopedProfileId], - profiles: { - [scopedProfileId]: { - type: "oauth", - provider: "anthropic", - access: "newer-scoped-access", - refresh: "newer-scoped-refresh", - expires: 4, - }, - [unrelatedProfileId]: { - type: "oauth", - provider: "minimax-portal", - access: "unrelated-access", - refresh: "unrelated-refresh", - expires: 2, - }, - }, - order: { - anthropic: [scopedProfileId], - "minimax-portal": [unrelatedProfileId], - }, - }, - agentDir, - ); - - const snapshotAfterProfileCarryingScopedSave = getRuntimeAuthProfileStoreSnapshot(agentDir); - expect(snapshotAfterProfileCarryingScopedSave?.runtimeExternalProfileIds).toEqual([ - scopedProfileId, - unrelatedProfileId, - ]); - expect(snapshotAfterProfileCarryingScopedSave?.runtimeExternalProfileIdsAuthoritative).toBe( - true, - ); - const runtimeWithoutExternal = ensureAuthProfileStoreWithoutExternalProfiles(agentDir); - expect(runtimeWithoutExternal.profiles[unrelatedProfileId]).toBeUndefined(); - } finally { - clearRuntimeAuthProfileStoreSnapshots(); - await fs.rm(agentDir, { recursive: true, force: true }); - } - }); - - it("does not persist profiles already marked runtime-only external", async () => { - const agentDir = await fs.mkdtemp(path.join(os.tmpdir(), "openclaw-auth-save-runtime-only-")); - const profileId = "anthropic:claude-cli"; - - try { - const store: AuthProfileStore = { - version: 1, - runtimeExternalProfileIds: [profileId], - runtimeExternalProfileIdsAuthoritative: true, - profiles: { - [profileId]: { - type: "oauth", - provider: "anthropic", - access: "external-access", - refresh: "external-refresh", - expires: 1, - }, - }, - order: { - anthropic: [profileId], - }, - lastGood: { - anthropic: profileId, - }, - usageStats: { - [profileId]: { lastUsed: 10 }, - }, - }; - replaceRuntimeAuthProfileStoreSnapshots([{ agentDir, store }]); - - saveAuthProfileStore(store, agentDir); - - const authProfiles = JSON.parse( - await fs.readFile(resolveAuthStorePath(agentDir), "utf8"), - ) as { - profiles: Record; - }; - expect(authProfiles.profiles[profileId]).toBeUndefined(); - - const snapshot = getRuntimeAuthProfileStoreSnapshot(agentDir); - expect(snapshot?.runtimeExternalProfileIds).toEqual([profileId]); - expect(snapshot?.profiles[profileId]).toMatchObject({ - type: "oauth", - provider: "anthropic", - access: "external-access", - refresh: "external-refresh", - }); - } finally { - clearRuntimeAuthProfileStoreSnapshots(); - await fs.rm(agentDir, { recursive: true, force: true }); - } - }); - - it("does not persist runtime-only external profiles without an installed snapshot", async () => { - const agentDir = await fs.mkdtemp(path.join(os.tmpdir(), "openclaw-auth-save-unsnapshotted-")); - const profileId = "openai:oauth"; - - try { - saveAuthProfileStore( - { - version: 1, - runtimeExternalProfileIds: [profileId], - profiles: { - [profileId]: { - type: "oauth", - provider: "openai", - access: "runtime-access", - refresh: "runtime-refresh", - expires: 1, - }, - }, - }, - agentDir, - ); - - const authProfiles = JSON.parse( - await fs.readFile(resolveAuthStorePath(agentDir), "utf8"), - ) as { - profiles: Record; - }; - expect(authProfiles.profiles[profileId]).toBeUndefined(); - } finally { - clearRuntimeAuthProfileStoreSnapshots(); - await fs.rm(agentDir, { recursive: true, force: true }); - } - }); - - it("returns active runtime-only external profiles on unscoped reads", async () => { - const agentDir = await fs.mkdtemp(path.join(os.tmpdir(), "openclaw-auth-read-runtime-only-")); - const profileId = "openai:oauth"; - - try { - replaceRuntimeAuthProfileStoreSnapshots([ - { - agentDir, - store: { - version: 1, - runtimeExternalProfileIds: [profileId], - runtimeExternalProfileIdsAuthoritative: true, - profiles: { - [profileId]: { - type: "oauth", - provider: "openai", - access: "runtime-access", - refresh: "runtime-refresh", - expires: 1, - }, - }, - usageStats: { - [profileId]: { lastUsed: 10 }, - }, - }, - }, - ]); - - const store = ensureAuthProfileStore(agentDir); - - expect(store.runtimeExternalProfileIds).toEqual([profileId]); - expectProfileFields(store.profiles[profileId], { - type: "oauth", - provider: "openai", - access: "runtime-access", - refresh: "runtime-refresh", - }); - expect(store.usageStats?.[profileId]?.lastUsed).toBe(10); - } finally { - clearRuntimeAuthProfileStoreSnapshots(); - await fs.rm(agentDir, { recursive: true, force: true }); - } - }); - - it("does not resurrect runtime-only profiles after authoritative empty overlays", async () => { - const agentDir = await fs.mkdtemp(path.join(os.tmpdir(), "openclaw-auth-read-removed-")); - const profileId = "anthropic:claude-cli"; - externalAuthMocks.overlayExternalAuthProfiles.mockImplementation((store) => ({ - ...(store as AuthProfileStore), - runtimeExternalProfileIds: [], - runtimeExternalProfileIdsAuthoritative: true, - })); - - try { - replaceRuntimeAuthProfileStoreSnapshots([ - { - agentDir, - store: { - version: 1, - runtimeExternalProfileIds: [profileId], - runtimeExternalProfileIdsAuthoritative: true, - profiles: { - [profileId]: { - type: "oauth", - provider: "anthropic", - access: "runtime-access", - refresh: "runtime-refresh", - expires: 1, - }, - }, - }, - }, - ]); - - const store = ensureAuthProfileStore(agentDir); - - expect(store.runtimeExternalProfileIds).toEqual([]); - expect(store.runtimeExternalProfileIdsAuthoritative).toBe(true); - expect(store.profiles[profileId]).toBeUndefined(); - } finally { - clearRuntimeAuthProfileStoreSnapshots(); - await fs.rm(agentDir, { recursive: true, force: true }); - } - }); - - it("persists refreshed runtime-only external OAuth credentials", async () => { - const agentDir = await fs.mkdtemp(path.join(os.tmpdir(), "openclaw-auth-save-refreshed-")); - const profileId = "anthropic:claude-cli"; - - try { - replaceRuntimeAuthProfileStoreSnapshots([ - { - agentDir, - store: { - version: 1, - runtimeExternalProfileIds: [profileId], - runtimeExternalProfileIdsAuthoritative: true, - profiles: { - [profileId]: { - type: "oauth", - provider: "anthropic", - access: "external-access", - refresh: "external-refresh", - expires: 1, - }, - }, - }, - }, - ]); - - saveAuthProfileStore( - { - version: 1, - runtimeExternalProfileIds: [profileId], - runtimeExternalProfileIdsAuthoritative: true, - profiles: { - [profileId]: { - type: "oauth", - provider: "anthropic", - access: "refreshed-access", - refresh: "refreshed-refresh", - expires: 2, - }, - }, - }, - agentDir, - ); - - const authProfiles = JSON.parse( - await fs.readFile(resolveAuthStorePath(agentDir), "utf8"), - ) as { - profiles: Record; - }; - expectProfileFields(authProfiles.profiles[profileId], { - type: "oauth", - provider: "anthropic", - access: "refreshed-access", - refresh: "refreshed-refresh", - }); - - const activeRuntime = getRuntimeAuthProfileStoreSnapshot(agentDir); - if (!activeRuntime) { - throw new Error("expected active runtime auth snapshot"); - } - saveAuthProfileStore( - { - ...activeRuntime, - usageStats: { - [profileId]: { lastUsed: 20 }, - }, - }, - agentDir, - ); - - const authProfilesAfterUsageSave = JSON.parse( - await fs.readFile(resolveAuthStorePath(agentDir), "utf8"), - ) as { - profiles: Record; - }; - expectProfileFields(authProfilesAfterUsageSave.profiles[profileId], { - type: "oauth", - provider: "anthropic", - access: "refreshed-access", - refresh: "refreshed-refresh", - }); - } finally { - clearRuntimeAuthProfileStoreSnapshots(); - await fs.rm(agentDir, { recursive: true, force: true }); - } - }); - - it("writes runtime scheduling state to auth-state.json only", async () => { - const agentDir = await fs.mkdtemp(path.join(os.tmpdir(), "openclaw-auth-save-state-")); - try { - const store: AuthProfileStore = { - version: 1, - profiles: { - "anthropic:default": { - type: "api_key", - provider: "anthropic", - key: "sk-anthropic-plain", - }, - }, - order: { - anthropic: ["anthropic:default"], - }, - lastGood: { - anthropic: "anthropic:default", - }, - usageStats: { - "anthropic:default": { - lastUsed: 123, - }, - }, - }; - - saveAuthProfileStore(store, agentDir); - - const authProfiles = JSON.parse( - await fs.readFile(resolveAuthStorePath(agentDir), "utf8"), - ) as { - profiles: Record; - order?: unknown; - lastGood?: unknown; - usageStats?: unknown; - }; - expect(authProfiles.profiles["anthropic:default"]).toEqual({ - type: "api_key", - provider: "anthropic", - key: "sk-anthropic-plain", - }); - expect(authProfiles.order).toBeUndefined(); - expect(authProfiles.lastGood).toBeUndefined(); - expect(authProfiles.usageStats).toBeUndefined(); - - const authState = JSON.parse(await fs.readFile(resolveAuthStatePath(agentDir), "utf8")) as { - order?: Record; - lastGood?: Record; - usageStats?: Record; - }; - expect(authState.order?.anthropic).toEqual(["anthropic:default"]); - expect(authState.lastGood?.anthropic).toBe("anthropic:default"); - expect(authState.usageStats?.["anthropic:default"]?.lastUsed).toBe(123); - } finally { - await fs.rm(agentDir, { recursive: true, force: true }); - } - }); - - it("does not rewrite auth secrets when only runtime scheduling state changes", async () => { - const agentDir = await fs.mkdtemp(path.join(os.tmpdir(), "openclaw-auth-save-state-only-")); - try { - const profileId = "anthropic:default"; - const store: AuthProfileStore = { - version: 1, - profiles: { - [profileId]: { - type: "api_key", - provider: "anthropic", - key: "sk-anthropic-plain", - }, - }, - usageStats: { - [profileId]: { lastUsed: 1 }, - }, - }; - - saveAuthProfileStore(store, agentDir); - - const authPath = resolveAuthStorePath(agentDir); - const statePath = resolveAuthStatePath(agentDir); - const oldTimestamp = new Date("2001-01-01T00:00:00.000Z"); - await fs.utimes(authPath, oldTimestamp, oldTimestamp); - await fs.utimes(statePath, oldTimestamp, oldTimestamp); - - saveAuthProfileStore( - { - ...store, - usageStats: { - [profileId]: { lastUsed: 2 }, - }, - }, - agentDir, - ); - - expect((await fs.stat(authPath)).mtimeMs).toBe(oldTimestamp.getTime()); - expect((await fs.stat(statePath)).mtimeMs).toBeGreaterThan(oldTimestamp.getTime()); - - const authProfiles = JSON.parse(await fs.readFile(authPath, "utf8")) as { - usageStats?: unknown; - }; - expect(authProfiles.usageStats).toBeUndefined(); - - const authState = JSON.parse(await fs.readFile(statePath, "utf8")) as { - usageStats?: Record; - }; - expect(authState.usageStats?.[profileId]?.lastUsed).toBe(2); - } finally { - await fs.rm(agentDir, { recursive: true, force: true }); - } - }); - - it.runIf(process.platform !== "win32")( - "repairs auth secrets permissions when the payload is unchanged", - async () => { - const agentDir = await fs.mkdtemp(path.join(os.tmpdir(), "openclaw-auth-save-permissions-")); - try { - const store: AuthProfileStore = { - version: 1, - profiles: { - "anthropic:default": { - type: "api_key", - provider: "anthropic", - key: "sk-anthropic-plain", - }, - }, - }; - - saveAuthProfileStore(store, agentDir); - - const authPath = resolveAuthStorePath(agentDir); - await fs.chmod(authPath, 0o644); - - saveAuthProfileStore(store, agentDir); - - expect((await fs.stat(authPath)).mode & 0o777).toBe(0o600); - } finally { - await fs.rm(agentDir, { recursive: true, force: true }); - } - }, - ); - - it("does not rewrite unchanged runtime scheduling state", async () => { - const agentDir = await fs.mkdtemp(path.join(os.tmpdir(), "openclaw-auth-save-state-same-")); - try { - const profileId = "anthropic:default"; - const store: AuthProfileStore = { - version: 1, - profiles: { - [profileId]: { - type: "api_key", - provider: "anthropic", - key: "sk-anthropic-plain", - }, - }, - usageStats: { - [profileId]: { lastUsed: 1 }, - }, - }; - - saveAuthProfileStore(store, agentDir); - - const statePath = resolveAuthStatePath(agentDir); - const oldTimestamp = new Date("2001-01-01T00:00:00.000Z"); - await fs.utimes(statePath, oldTimestamp, oldTimestamp); - - saveAuthProfileStore(store, agentDir); - - expect((await fs.stat(statePath)).mtimeMs).toBe(oldTimestamp.getTime()); - } finally { - await fs.rm(agentDir, { recursive: true, force: true }); - } - }); - - it("does not persist unchanged inherited main OAuth when saving secondary local updates", async () => { - const root = await fs.mkdtemp(path.join(os.tmpdir(), "openclaw-auth-save-inherited-")); - const stateDir = path.join(root, ".openclaw"); - const childAgentDir = path.join(stateDir, "agents", "worker", "agent"); - const childAuthPath = resolveAuthStorePath(childAgentDir); - vi.stubEnv("OPENCLAW_STATE_DIR", stateDir); - vi.stubEnv("OPENCLAW_AGENT_DIR", ""); - try { - saveAuthProfileStore({ - version: 1, - profiles: { - "openai:oauth": { - type: "oauth", - provider: "openai", - access: "main-access-token", - refresh: "main-refresh-token", - expires: Date.now() + 60_000, - }, - }, - }); - - const localUpdateStore = ensureAuthProfileStoreForLocalUpdate(childAgentDir); - expectProfileFields(localUpdateStore.profiles["openai:oauth"], { - type: "oauth", - refresh: "main-refresh-token", - }); - localUpdateStore.profiles["openai:default"] = { - type: "api_key", - provider: "openai", - key: "sk-child-local", - }; - - saveAuthProfileStore(localUpdateStore, childAgentDir, { - filterExternalAuthProfiles: false, - }); - - const child = JSON.parse(await fs.readFile(childAuthPath, "utf8")) as { - profiles: Record; - }; - expectProfileFields(child.profiles["openai:default"], { - type: "api_key", - provider: "openai", - }); - expect(child.profiles["openai:oauth"]).toBeUndefined(); - - saveAuthProfileStore({ - version: 1, - profiles: { - "openai:oauth": { - type: "oauth", - provider: "openai", - access: "main-refreshed-access-token", - refresh: "main-refreshed-refresh-token", - expires: Date.now() + 120_000, - }, - }, - }); - - expectProfileFields(ensureAuthProfileStore(childAgentDir).profiles["openai:oauth"], { - type: "oauth", - access: "main-refreshed-access-token", - refresh: "main-refreshed-refresh-token", - }); - } finally { - clearRuntimeAuthProfileStoreSnapshots(); - vi.unstubAllEnvs(); - await fs.rm(root, { recursive: true, force: true }); - } - }); - - it("does not persist stale inherited main OAuth after main refreshes", async () => { - const root = await fs.mkdtemp(path.join(os.tmpdir(), "openclaw-auth-save-stale-inherited-")); - const stateDir = path.join(root, ".openclaw"); - const childAgentDir = path.join(stateDir, "agents", "worker", "agent"); - const childAuthPath = resolveAuthStorePath(childAgentDir); - vi.stubEnv("OPENCLAW_STATE_DIR", stateDir); - vi.stubEnv("OPENCLAW_AGENT_DIR", ""); - try { - saveAuthProfileStore({ - version: 1, - profiles: { - "openai:oauth": { - type: "oauth", - provider: "openai", - access: "main-old-access-token", - refresh: "main-old-refresh-token", - expires: Date.now() + 60_000, - accountId: "acct-shared", - email: "codex@example.test", - }, - }, - }); - - const localUpdateStore = ensureAuthProfileStoreForLocalUpdate(childAgentDir); - expectProfileFields(localUpdateStore.profiles["openai:oauth"], { - type: "oauth", - refresh: "main-old-refresh-token", - }); - - saveAuthProfileStore({ - version: 1, - profiles: { - "openai:oauth": { - type: "oauth", - provider: "openai", - access: "main-refreshed-access-token", - refresh: "main-refreshed-refresh-token", - expires: Date.now() + 120_000, - accountId: "acct-shared", - email: "codex@example.test", - }, - }, - }); - - localUpdateStore.profiles["openai:default"] = { - type: "api_key", - provider: "openai", - key: "sk-child-local", - }; - saveAuthProfileStore(localUpdateStore, childAgentDir, { - filterExternalAuthProfiles: false, - }); - - const child = JSON.parse(await fs.readFile(childAuthPath, "utf8")) as { - profiles: Record; - }; - expectProfileFields(child.profiles["openai:default"], { - type: "api_key", - provider: "openai", - }); - expect(child.profiles["openai:oauth"]).toBeUndefined(); - expectProfileFields(ensureAuthProfileStore(childAgentDir).profiles["openai:oauth"], { - type: "oauth", - access: "main-refreshed-access-token", - refresh: "main-refreshed-refresh-token", - }); - } finally { - clearRuntimeAuthProfileStoreSnapshots(); - vi.unstubAllEnvs(); - await fs.rm(root, { recursive: true, force: true }); - } - }); - - it("does not persist inherited OAuth with an out-of-range local expiry", async () => { - const root = await fs.mkdtemp(path.join(os.tmpdir(), "openclaw-auth-save-invalid-local-")); - const stateDir = path.join(root, ".openclaw"); - const childAgentDir = path.join(stateDir, "agents", "worker", "agent"); - const childAuthPath = resolveAuthStorePath(childAgentDir); - vi.stubEnv("OPENCLAW_STATE_DIR", stateDir); - vi.stubEnv("OPENCLAW_AGENT_DIR", ""); - try { - saveAuthProfileStore({ - version: 1, - profiles: { - "openai-codex:default": { - type: "oauth", - provider: "openai-codex", - access: "main-access-token", - refresh: "main-refresh-token", - expires: Date.now() + 120_000, - accountId: "acct-shared", - }, - }, - }); - - const localUpdateStore = ensureAuthProfileStoreForLocalUpdate(childAgentDir); - localUpdateStore.profiles["openai-codex:default"] = { - type: "oauth", - provider: "openai-codex", - access: "main-access-token", - refresh: "main-refresh-token", - expires: MAX_DATE_TIMESTAMP_MS + 1, - accountId: "acct-shared", - }; - localUpdateStore.profiles["openai:default"] = { - type: "api_key", - provider: "openai", - key: "sk-child-local", - }; - - saveAuthProfileStore(localUpdateStore, childAgentDir, { - filterExternalAuthProfiles: false, - }); - - const child = JSON.parse(await fs.readFile(childAuthPath, "utf8")) as { - profiles: Record; - }; - expect(child.profiles["openai-codex:default"]).toBeUndefined(); - expectProfileFields(ensureAuthProfileStore(childAgentDir).profiles["openai-codex:default"], { - type: "oauth", - access: "main-access-token", - refresh: "main-refresh-token", - }); - } finally { - clearRuntimeAuthProfileStoreSnapshots(); - vi.unstubAllEnvs(); - await fs.rm(root, { recursive: true, force: true }); - } - }); - - it("preserves inherited main OAuth in active secondary runtime snapshots", async () => { - const root = await fs.mkdtemp(path.join(os.tmpdir(), "openclaw-auth-save-snapshot-")); - const stateDir = path.join(root, ".openclaw"); - const childAgentDir = path.join(stateDir, "agents", "worker", "agent"); - const childAuthPath = resolveAuthStorePath(childAgentDir); - vi.stubEnv("OPENCLAW_STATE_DIR", stateDir); - vi.stubEnv("OPENCLAW_AGENT_DIR", ""); - try { - saveAuthProfileStore({ - version: 1, - profiles: { - "openai:oauth": { - type: "oauth", - provider: "openai", - access: "main-access-token", - refresh: "main-refresh-token", - expires: Date.now() + 60_000, - }, - }, - }); - - const localUpdateStore = ensureAuthProfileStoreForLocalUpdate(childAgentDir); - localUpdateStore.profiles["openai:default"] = { - type: "api_key", - provider: "openai", - key: "sk-child-local", - }; - replaceRuntimeAuthProfileStoreSnapshots([ - { - agentDir: childAgentDir, - store: localUpdateStore, - }, - ]); - - saveAuthProfileStore(localUpdateStore, childAgentDir, { - filterExternalAuthProfiles: false, - }); - - const child = JSON.parse(await fs.readFile(childAuthPath, "utf8")) as { - profiles: Record; - }; - expect(child.profiles["openai:oauth"]).toBeUndefined(); - - const runtime = ensureAuthProfileStore(childAgentDir); - expectProfileFields(runtime.profiles["openai:default"], { - type: "api_key", - provider: "openai", - }); - expectProfileFields(runtime.profiles["openai:oauth"], { - type: "oauth", - access: "main-access-token", - refresh: "main-refresh-token", - }); - - saveAuthProfileStore({ - version: 1, - profiles: { - "openai:oauth": { - type: "oauth", - provider: "openai", - access: "main-refreshed-access-token", - refresh: "main-refreshed-refresh-token", - expires: Date.now() + 120_000, - }, - }, - }); - - expectProfileFields(ensureAuthProfileStore(childAgentDir).profiles["openai:oauth"], { - type: "oauth", - access: "main-refreshed-access-token", - refresh: "main-refreshed-refresh-token", - }); - } finally { - clearRuntimeAuthProfileStoreSnapshots(); - vi.unstubAllEnvs(); - await fs.rm(root, { recursive: true, force: true }); - } - }); - - it("keeps local replacements for old runtime-only profile ids visible", async () => { - const agentDir = await fs.mkdtemp(path.join(os.tmpdir(), "openclaw-auth-save-replace-")); - const profileId = "anthropic:claude-cli"; - - try { - replaceRuntimeAuthProfileStoreSnapshots([ - { - agentDir, - store: { - version: 1, - runtimeExternalProfileIds: [profileId], - runtimeExternalProfileIdsAuthoritative: true, - profiles: { - [profileId]: { - type: "oauth", - provider: "anthropic", - access: "external-access", - refresh: "external-refresh", - expires: 1, - }, - }, - }, - }, - ]); - - saveAuthProfileStore( - { - version: 1, - profiles: { - [profileId]: { - type: "api_key", - provider: "anthropic", - key: "sk-local", - }, - }, - }, - agentDir, - ); - - const snapshot = getRuntimeAuthProfileStoreSnapshot(agentDir); - expect(snapshot?.runtimeExternalProfileIds).toEqual([]); - expect(snapshot?.runtimeExternalProfileIdsAuthoritative).toBe(true); - expect(snapshot?.profiles[profileId]).toMatchObject({ - type: "api_key", - provider: "anthropic", - key: "sk-local", - }); - - const runtimeWithoutExternal = ensureAuthProfileStoreWithoutExternalProfiles(agentDir); - expect(runtimeWithoutExternal.profiles[profileId]).toMatchObject({ - type: "api_key", - provider: "anthropic", - key: "sk-local", - }); - } finally { - clearRuntimeAuthProfileStoreSnapshots(); - await fs.rm(agentDir, { recursive: true, force: true }); - } - }); - - it("clears non-authoritative runtime-only metadata after local replacements", async () => { - const agentDir = await fs.mkdtemp(path.join(os.tmpdir(), "openclaw-auth-save-replace-scoped-")); - const profileId = "anthropic:claude-cli"; - - try { - replaceRuntimeAuthProfileStoreSnapshots([ - { - agentDir, - store: { - version: 1, - runtimeExternalProfileIds: [profileId], - profiles: { - [profileId]: { - type: "oauth", - provider: "anthropic", - access: "external-access", - refresh: "external-refresh", - expires: 1, - }, - }, - }, - }, - ]); - - saveAuthProfileStore( - { - version: 1, - profiles: { - [profileId]: { - type: "api_key", - provider: "anthropic", - key: "sk-local", - }, - }, - }, - agentDir, - ); - - const snapshot = getRuntimeAuthProfileStoreSnapshot(agentDir); - expect(snapshot?.runtimeExternalProfileIds).toBeUndefined(); - expect(snapshot?.runtimeExternalProfileIdsAuthoritative).toBeUndefined(); - expect(snapshot?.profiles[profileId]).toMatchObject({ - type: "api_key", - provider: "anthropic", - key: "sk-local", - }); - } finally { - clearRuntimeAuthProfileStoreSnapshots(); - await fs.rm(agentDir, { recursive: true, force: true }); - } - }); -}); diff --git a/src/agents/auth-profiles.ts b/src/agents/auth-profiles.ts index e293be09e367..d816551fd0db 100644 --- a/src/agents/auth-profiles.ts +++ b/src/agents/auth-profiles.ts @@ -55,6 +55,7 @@ export { ensureAuthProfileStoreWithoutExternalProfiles, getRuntimeAuthProfileStoreSnapshot, hasAnyAuthProfileStoreSource, + hasLocalAuthProfileStoreSource, loadAuthProfileStoreForSecretsRuntime, loadAuthProfileStoreWithoutExternalProfiles, loadAuthProfileStoreForRuntime, diff --git a/src/agents/auth-profiles/constants.ts b/src/agents/auth-profiles/constants.ts index 34940d27a9e7..bedadaa96066 100644 --- a/src/agents/auth-profiles/constants.ts +++ b/src/agents/auth-profiles/constants.ts @@ -28,7 +28,7 @@ export const AUTH_STORE_LOCK_OPTIONS = { // Separate from AUTH_STORE_LOCK_OPTIONS for independent tuning: this lock // serializes the cross-agent OAuth refresh (see issue #26322), whereas -// AUTH_STORE_LOCK_OPTIONS guards per-store file writes. Keeping them +// AUTH_STORE_LOCK_OPTIONS guards per-store refresh updates. Keeping them // distinct lets us widen the refresh lock's timeout/retry budget without // affecting the hot-path auth-store writers. // diff --git a/src/agents/auth-profiles/oauth-external-auth-passthrough.test-support.ts b/src/agents/auth-profiles/oauth-external-auth-passthrough.test-support.ts index 5d989c5962c6..73f64cf13abc 100644 --- a/src/agents/auth-profiles/oauth-external-auth-passthrough.test-support.ts +++ b/src/agents/auth-profiles/oauth-external-auth-passthrough.test-support.ts @@ -4,4 +4,5 @@ vi.mock("./external-auth.js", () => ({ listRuntimeExternalAuthProfiles: () => [], overlayExternalAuthProfiles: (store: T) => store, shouldPersistExternalAuthProfile: () => true, + syncPersistedExternalCliAuthProfiles: (store: T) => store, })); diff --git a/src/agents/auth-profiles/oauth-manager.ts b/src/agents/auth-profiles/oauth-manager.ts index 4e479d5d1d07..6369069125b4 100644 --- a/src/agents/auth-profiles/oauth-manager.ts +++ b/src/agents/auth-profiles/oauth-manager.ts @@ -4,12 +4,7 @@ import { formatErrorMessage } from "../../infra/errors.js"; import { withFileLock } from "../../infra/file-lock.js"; import { redactSensitiveText } from "../../logging/redact.js"; import { asDateTimestampMs } from "../../shared/number-coercion.js"; -import { - AUTH_STORE_LOCK_OPTIONS, - OAUTH_REFRESH_CALL_TIMEOUT_MS, - OAUTH_REFRESH_LOCK_OPTIONS, - log, -} from "./constants.js"; +import { OAUTH_REFRESH_CALL_TIMEOUT_MS, OAUTH_REFRESH_LOCK_OPTIONS, log } from "./constants.js"; import { shouldMirrorRefreshedOAuthCredential } from "./oauth-identity.js"; import { OAuthRefreshFailureError } from "./oauth-refresh-failure.js"; import { @@ -29,11 +24,10 @@ import { shouldReplaceStoredOAuthCredential, type RuntimeExternalOAuthProfile, } from "./oauth-shared.js"; -import { ensureAuthStoreFile, resolveAuthStorePath, resolveOAuthRefreshLockPath } from "./paths.js"; +import { resolveAuthStorePath, resolveOAuthRefreshLockPath } from "./paths.js"; import { ensureAuthProfileStoreWithoutExternalProfiles, loadAuthProfileStoreWithoutExternalProfiles, - saveAuthProfileStore, resolvePersistedAuthProfileOwnerAgentDir, updateAuthProfileStoreWithLock, } from "./store.js"; @@ -387,8 +381,6 @@ export function createOAuthManager(adapter: OAuthManagerAdapter) { refreshed: OAuthCredential; }): Promise { try { - const mainPath = resolveAuthStorePath(undefined); - ensureAuthStoreFile(mainPath); await updateAuthProfileStoreWithLock({ agentDir: undefined, updater: (store) => { @@ -423,6 +415,46 @@ export function createOAuthManager(adapter: OAuthManagerAdapter) { } } + async function saveOAuthCredentialWithStoreLock(params: { + agentDir?: string; + profileId: string; + expected: OAuthCredential | OAuthCredential[]; + credential: OAuthCredential; + }): Promise { + let saved = false; + const result = await updateAuthProfileStoreWithLock({ + agentDir: params.agentDir, + updater: (store) => { + const existing = store.profiles[params.profileId]; + const expectedCredentials = Array.isArray(params.expected) + ? params.expected + : [params.expected]; + if ( + existing?.type !== "oauth" || + !expectedCredentials.some((expected) => areOAuthCredentialsEquivalent(existing, expected)) + ) { + log.debug("skipped OAuth credential write because stored profile changed", { + profileId: params.profileId, + }); + return false; + } + if ( + !isSafeToAdoptBootstrapOAuthIdentity(existing, params.credential) || + !shouldReplaceStoredOAuthCredential(existing, params.credential) + ) { + log.debug("skipped OAuth credential write because stored profile changed", { + profileId: params.profileId, + }); + return false; + } + store.profiles[params.profileId] = { ...params.credential }; + saved = true; + return true; + }, + }); + return result !== null && saved; + } + async function doRefreshOAuthTokenWithLock(params: { profileId: string; provider: string; @@ -433,152 +465,165 @@ export function createOAuthManager(adapter: OAuthManagerAdapter) { }): Promise { const ownerAgentDir = resolvePersistedAuthProfileOwnerAgentDir(params); const authPath = resolveAuthStorePath(ownerAgentDir); - ensureAuthStoreFile(authPath); const globalRefreshLockPath = resolveOAuthRefreshLockPath(params.provider, params.profileId); try { - return await withFileLock(globalRefreshLockPath, OAUTH_REFRESH_LOCK_OPTIONS, async () => - withFileLock(authPath, AUTH_STORE_LOCK_OPTIONS, async () => { - const store = loadStoredOAuthRefreshStore(ownerAgentDir); - const cred = store.profiles[params.profileId]; - if (!cred || cred.type !== "oauth") { - return null; - } - let credentialToRefresh = cred; + return await withFileLock(globalRefreshLockPath, OAUTH_REFRESH_LOCK_OPTIONS, async () => { + const store = loadStoredOAuthRefreshStore(ownerAgentDir); + const cred = store.profiles[params.profileId]; + if (!cred || cred.type !== "oauth") { + return null; + } + let credentialToRefresh = cred; - if (!params.forceRefresh && hasUsableOAuthCredential(cred)) { - return { - apiKey: await adapter.buildApiKey(cred.provider, cred, { - cfg: params.cfg, - agentDir: params.agentDir, - }), - credential: cred, - }; - } - - if (params.agentDir) { - try { - const mainStore = loadStoredOAuthRefreshStore(undefined); - const mainCred = mainStore.profiles[params.profileId]; - if ( - mainCred?.type === "oauth" && - mainCred.provider === cred.provider && - hasUsableOAuthCredential(mainCred) && - !params.forceRefresh && - isSafeToAdoptMainStoreOAuthIdentity(cred, mainCred) - ) { - store.profiles[params.profileId] = { ...mainCred }; - log.info("adopted fresh OAuth credential from main store (under refresh lock)", { - profileId: params.profileId, - agentDir: params.agentDir, - expires: new Date(mainCred.expires).toISOString(), - }); - return { - apiKey: await adapter.buildApiKey(mainCred.provider, mainCred, { - cfg: params.cfg, - agentDir: params.agentDir, - }), - credential: mainCred, - }; - } else if ( - mainCred?.type === "oauth" && - mainCred.provider === cred.provider && - hasUsableOAuthCredential(mainCred) && - !isSafeToAdoptMainStoreOAuthIdentity(cred, mainCred) - ) { - log.warn("refused to adopt fresh main-store OAuth credential: identity mismatch", { - profileId: params.profileId, - agentDir: params.agentDir, - }); - } - } catch (err) { - log.debug("inside-lock main-store adoption failed; proceeding to refresh", { - profileId: params.profileId, - error: formatErrorMessage(err), - }); - } - } - - const externallyManaged = adapter.readBootstrapCredential({ - profileId: params.profileId, - credential: cred, - }); - if (externallyManaged) { - if (externallyManaged.provider !== cred.provider) { - log.warn("refused external oauth bootstrap credential: provider mismatch", { - profileId: params.profileId, - provider: cred.provider, - }); - } else if (!isSafeToAdoptBootstrapOAuthIdentity(cred, externallyManaged)) { - log.warn( - "refused external oauth bootstrap credential: identity mismatch or missing binding", - { - profileId: params.profileId, - provider: cred.provider, - }, - ); - } else { - if ( - shouldReplaceStoredOAuthCredential(cred, externallyManaged) && - !areOAuthCredentialsEquivalent(cred, externallyManaged) - ) { - store.profiles[params.profileId] = { ...externallyManaged }; - saveAuthProfileStore(store, ownerAgentDir); - } - credentialToRefresh = externallyManaged; - if (!params.forceRefresh && hasUsableOAuthCredential(externallyManaged)) { - return { - apiKey: await adapter.buildApiKey(externallyManaged.provider, externallyManaged, { - cfg: params.cfg, - agentDir: params.agentDir, - }), - credential: externallyManaged, - }; - } - } - } - - if (normalizeSecretInputString(credentialToRefresh.refresh) === undefined) { - return null; - } - const refreshedCredentials = await withRefreshCallTimeout( - `refreshOAuthCredential(${cred.provider})`, - OAUTH_REFRESH_CALL_TIMEOUT_MS, - async () => { - params.attemptedCredentials?.push(credentialToRefresh); - const refreshed = await adapter.refreshCredential(credentialToRefresh); - return refreshed - ? ({ - ...credentialToRefresh, - ...refreshed, - type: "oauth", - } satisfies OAuthCredential) - : null; - }, - ); - if (!refreshedCredentials) { - return null; - } - store.profiles[params.profileId] = refreshedCredentials; - saveAuthProfileStore(store, ownerAgentDir); - if (ownerAgentDir) { - const mainPath = resolveAuthStorePath(undefined); - if (mainPath !== authPath) { - await mirrorRefreshedCredentialIntoMainStore({ - profileId: params.profileId, - refreshed: refreshedCredentials, - }); - } - } + if (!params.forceRefresh && hasUsableOAuthCredential(cred)) { return { - apiKey: await adapter.buildApiKey(cred.provider, refreshedCredentials, { + apiKey: await adapter.buildApiKey(cred.provider, cred, { cfg: params.cfg, agentDir: params.agentDir, }), - credential: refreshedCredentials, + credential: cred, }; - }), - ); + } + + if (params.agentDir) { + try { + const mainStore = loadStoredOAuthRefreshStore(undefined); + const mainCred = mainStore.profiles[params.profileId]; + if ( + mainCred?.type === "oauth" && + mainCred.provider === cred.provider && + hasUsableOAuthCredential(mainCred) && + !params.forceRefresh && + isSafeToAdoptMainStoreOAuthIdentity(cred, mainCred) + ) { + store.profiles[params.profileId] = { ...mainCred }; + log.info("adopted fresh OAuth credential from main store (under refresh lock)", { + profileId: params.profileId, + agentDir: params.agentDir, + expires: new Date(mainCred.expires).toISOString(), + }); + return { + apiKey: await adapter.buildApiKey(mainCred.provider, mainCred, { + cfg: params.cfg, + agentDir: params.agentDir, + }), + credential: mainCred, + }; + } else if ( + mainCred?.type === "oauth" && + mainCred.provider === cred.provider && + hasUsableOAuthCredential(mainCred) && + !isSafeToAdoptMainStoreOAuthIdentity(cred, mainCred) + ) { + log.warn("refused to adopt fresh main-store OAuth credential: identity mismatch", { + profileId: params.profileId, + agentDir: params.agentDir, + }); + } + } catch (err) { + log.debug("inside-lock main-store adoption failed; proceeding to refresh", { + profileId: params.profileId, + error: formatErrorMessage(err), + }); + } + } + + const externallyManaged = adapter.readBootstrapCredential({ + profileId: params.profileId, + credential: cred, + }); + if (externallyManaged) { + if (externallyManaged.provider !== cred.provider) { + log.warn("refused external oauth bootstrap credential: provider mismatch", { + profileId: params.profileId, + provider: cred.provider, + }); + } else if (!isSafeToAdoptBootstrapOAuthIdentity(cred, externallyManaged)) { + log.warn( + "refused external oauth bootstrap credential: identity mismatch or missing binding", + { + profileId: params.profileId, + provider: cred.provider, + }, + ); + } else { + if ( + shouldReplaceStoredOAuthCredential(cred, externallyManaged) && + !areOAuthCredentialsEquivalent(cred, externallyManaged) + ) { + store.profiles[params.profileId] = { ...externallyManaged }; + await saveOAuthCredentialWithStoreLock({ + agentDir: ownerAgentDir, + profileId: params.profileId, + expected: cred, + credential: externallyManaged, + }); + } + credentialToRefresh = externallyManaged; + if (!params.forceRefresh && hasUsableOAuthCredential(externallyManaged)) { + return { + apiKey: await adapter.buildApiKey(externallyManaged.provider, externallyManaged, { + cfg: params.cfg, + agentDir: params.agentDir, + }), + credential: externallyManaged, + }; + } + } + } + + if (normalizeSecretInputString(credentialToRefresh.refresh) === undefined) { + return null; + } + const refreshedCredentials = await withRefreshCallTimeout( + `refreshOAuthCredential(${cred.provider})`, + OAUTH_REFRESH_CALL_TIMEOUT_MS, + async () => { + params.attemptedCredentials?.push(credentialToRefresh); + const refreshed = await adapter.refreshCredential(credentialToRefresh); + return refreshed + ? ({ + ...credentialToRefresh, + ...refreshed, + type: "oauth", + } satisfies OAuthCredential) + : null; + }, + ); + if (!refreshedCredentials) { + return null; + } + store.profiles[params.profileId] = refreshedCredentials; + const persisted = await saveOAuthCredentialWithStoreLock({ + agentDir: ownerAgentDir, + profileId: params.profileId, + expected: + credentialToRefresh === cred || areOAuthCredentialsEquivalent(credentialToRefresh, cred) + ? credentialToRefresh + : [credentialToRefresh, cred], + credential: refreshedCredentials, + }); + if (!persisted) { + throw new Error("Failed to persist refreshed OAuth credential"); + } + if (ownerAgentDir) { + const mainPath = resolveAuthStorePath(undefined); + if (mainPath !== authPath) { + await mirrorRefreshedCredentialIntoMainStore({ + profileId: params.profileId, + refreshed: refreshedCredentials, + }); + } + } + return { + apiKey: await adapter.buildApiKey(cred.provider, refreshedCredentials, { + cfg: params.cfg, + agentDir: params.agentDir, + }), + credential: refreshedCredentials, + }; + }); } catch (error) { if (isGlobalRefreshLockTimeoutError(error, globalRefreshLockPath)) { throw buildRefreshContentionError({ diff --git a/src/agents/auth-profiles/oauth-test-utils.ts b/src/agents/auth-profiles/oauth-test-utils.ts index c3fb457d309c..5e5682d295fb 100644 --- a/src/agents/auth-profiles/oauth-test-utils.ts +++ b/src/agents/auth-profiles/oauth-test-utils.ts @@ -1,7 +1,10 @@ import fs from "node:fs/promises"; import os from "node:os"; import path from "node:path"; +import { closeOpenClawAgentDatabasesForTest } from "../../state/openclaw-agent-db.js"; import type { resolveApiKeyForProfile } from "./oauth.js"; +import { loadPersistedAuthProfileStore } from "./persisted.js"; +import { saveAuthProfileStore } from "./store.js"; import type { AuthProfileStore, OAuthCredential } from "./types.js"; export const OAUTH_AGENT_ENV_KEYS = ["OPENCLAW_STATE_DIR", "OPENCLAW_AGENT_DIR"]; @@ -66,10 +69,22 @@ export async function createOAuthMainAgentDir(stateDir: string): Promise export async function removeOAuthTestTempRoot(tempRoot: string): Promise { if (tempRoot) { + closeOpenClawAgentDatabasesForTest(); await fs.rm(tempRoot, { recursive: true, force: true }); } } +export function writeAuthProfileStoreForTest(agentDir: string, store: AuthProfileStore): void { + saveAuthProfileStore(store, agentDir, { + filterExternalAuthProfiles: false, + syncExternalCli: false, + }); +} + +export function readAuthProfileStoreForTest(agentDir: string): AuthProfileStore { + return loadPersistedAuthProfileStore(agentDir) ?? { version: 1, profiles: {} }; +} + type ResettableMock = { mockReset(): unknown; }; diff --git a/src/agents/auth-profiles/oauth.adopt-identity.test.ts b/src/agents/auth-profiles/oauth.adopt-identity.test.ts index c0bb372222e8..ad07989e7377 100644 --- a/src/agents/auth-profiles/oauth.adopt-identity.test.ts +++ b/src/agents/auth-profiles/oauth.adopt-identity.test.ts @@ -11,6 +11,7 @@ import { createOAuthMainAgentDir, createOAuthTestTempRoot, oauthCred, + readAuthProfileStoreForTest, removeOAuthTestTempRoot, resolveApiKeyForProfileInTest, resetOAuthProviderRuntimeMocks, @@ -132,9 +133,7 @@ describe("OAuth credential adoption is identity-gated", () => { expect(result?.apiKey).toBe("sub-own-access"); // Sub-agent store must NOT have been overwritten with main's foreign cred. - const subRaw = JSON.parse( - await fs.readFile(path.join(subAgentDir, "auth-profiles.json"), "utf8"), - ) as AuthProfileStore; + const subRaw = readAuthProfileStoreForTest(subAgentDir); expectPersistedOpenAICodexProfile(subRaw.profiles[profileId], { access: "sub-own-access", refresh: "sub-own-refresh", @@ -207,9 +206,7 @@ describe("OAuth credential adoption is identity-gated", () => { // Main must still hold its foreign cred, untouched (mirror would also // refuse because of identity mismatch). - const mainRaw = JSON.parse( - await fs.readFile(path.join(mainAgentDir, "auth-profiles.json"), "utf8"), - ) as AuthProfileStore; + const mainRaw = readAuthProfileStoreForTest(mainAgentDir); expectPersistedOpenAICodexProfile(mainRaw.profiles[profileId], { access: "main-foreign-access", refresh: "main-foreign-refresh", @@ -285,9 +282,7 @@ describe("OAuth credential adoption is identity-gated", () => { ).rejects.toThrow(/OAuth token refresh failed for openai/); // Sub-agent store must still have its own stale cred \u2014 no leak. - const subRaw = JSON.parse( - await fs.readFile(path.join(subAgentDir, "auth-profiles.json"), "utf8"), - ) as AuthProfileStore; + const subRaw = readAuthProfileStoreForTest(subAgentDir); expectPersistedOpenAICodexProfile(subRaw.profiles[profileId], { access: "sub-stale", refresh: "sub-refresh-token", diff --git a/src/agents/auth-profiles/oauth.fallback-to-main-agent.test.ts b/src/agents/auth-profiles/oauth.fallback-to-main-agent.test.ts index 2525652b1129..0537fae4e128 100644 --- a/src/agents/auth-profiles/oauth.fallback-to-main-agent.test.ts +++ b/src/agents/auth-profiles/oauth.fallback-to-main-agent.test.ts @@ -3,9 +3,15 @@ import os from "node:os"; import path from "node:path"; import { afterAll, afterEach, beforeEach, describe, expect, it, vi } from "vitest"; import { resetFileLockStateForTest } from "../../infra/file-lock.js"; +import { closeOpenClawAgentDatabasesForTest } from "../../state/openclaw-agent-db.js"; import { captureEnv } from "../../test-utils/env.js"; import { resolveApiKeyForProfile } from "./oauth.js"; -import { clearRuntimeAuthProfileStoreSnapshots, ensureAuthProfileStore } from "./store.js"; +import { loadPersistedAuthProfileStore } from "./persisted.js"; +import { + clearRuntimeAuthProfileStoreSnapshots, + ensureAuthProfileStore, + saveAuthProfileStore, +} from "./store.js"; import type { AuthProfileStore } from "./types.js"; const { getOAuthApiKeyMock } = vi.hoisted(() => ({ getOAuthApiKeyMock: vi.fn(async () => { @@ -107,7 +113,14 @@ describe("resolveApiKeyForProfile fallback to main agent", () => { } async function writeAuthProfilesStore(agentDir: string, store: AuthProfileStore) { - await fs.writeFile(path.join(agentDir, "auth-profiles.json"), JSON.stringify(store)); + saveAuthProfileStore(store, agentDir, { + filterExternalAuthProfiles: false, + syncExternalCli: false, + }); + } + + function readAuthProfilesStore(agentDir: string): AuthProfileStore { + return loadPersistedAuthProfileStore(agentDir) ?? { version: 1, profiles: {} }; } async function resolveFromSecondaryAgent(profileId: string) { @@ -122,6 +135,7 @@ describe("resolveApiKeyForProfile fallback to main agent", () => { afterEach(async () => { resetFileLockStateForTest(); clearRuntimeAuthProfileStoreSnapshots(); + closeOpenClawAgentDatabasesForTest(); vi.unstubAllGlobals(); envSnapshot.restore(); @@ -202,9 +216,7 @@ describe("resolveApiKeyForProfile fallback to main agent", () => { expect(result.provider).toBe("anthropic"); // The secondary store keeps its local credential; inherited OAuth is read-through. - const secondaryStore = JSON.parse( - await fs.readFile(path.join(secondaryAgentDir, "auth-profiles.json"), "utf8"), - ) as AuthProfileStore; + const secondaryStore = readAuthProfilesStore(secondaryAgentDir); expectOauthCredentialFields(secondaryStore, profileId, { access: "expired-access-token", expires: expiredTime, @@ -241,9 +253,7 @@ describe("resolveApiKeyForProfile fallback to main agent", () => { expect(result?.apiKey).toBe("main-newer-access-token"); - const secondaryStore = JSON.parse( - await fs.readFile(path.join(secondaryAgentDir, "auth-profiles.json"), "utf8"), - ) as AuthProfileStore; + const secondaryStore = readAuthProfilesStore(secondaryAgentDir); expectOauthCredentialFields(secondaryStore, profileId, { access: "secondary-access-token", expires: secondaryExpiry, diff --git a/src/agents/auth-profiles/oauth.mirror-refresh.test.ts b/src/agents/auth-profiles/oauth.mirror-refresh.test.ts index 50aee50ff79a..08383772bf3b 100644 --- a/src/agents/auth-profiles/oauth.mirror-refresh.test.ts +++ b/src/agents/auth-profiles/oauth.mirror-refresh.test.ts @@ -11,6 +11,7 @@ import { createOAuthMainAgentDir, createOAuthTestTempRoot, createExpiredOauthStore, + readAuthProfileStoreForTest, removeOAuthTestTempRoot, resolveApiKeyForProfileInTest, resetOAuthProviderRuntimeMocks, @@ -129,9 +130,7 @@ describe("resolveApiKeyForProfile OAuth refresh mirror-to-main (#26322)", () => // Main store should now carry refreshed metadata, so a peer agent // starting fresh can resolve the runtime credential without token races. - const mainRaw = JSON.parse( - await fs.readFile(path.join(mainAgentDir, "auth-profiles.json"), "utf8"), - ) as AuthProfileStore; + const mainRaw = readAuthProfileStoreForTest(mainAgentDir); expectPersistedOpenAICodexProfile(mainRaw.profiles[profileId], { access: "sub-refreshed-access", refresh: "sub-refreshed-refresh", @@ -171,9 +170,7 @@ describe("resolveApiKeyForProfile OAuth refresh mirror-to-main (#26322)", () => }); expect(result?.apiKey).toBe("main-refreshed-access"); - const mainRaw = JSON.parse( - await fs.readFile(path.join(mainAgentDir, "auth-profiles.json"), "utf8"), - ) as AuthProfileStore; + const mainRaw = readAuthProfileStoreForTest(mainAgentDir); expectPersistedOpenAICodexProfile(mainRaw.profiles[profileId], { access: "main-refreshed-access", refresh: "main-refreshed-refresh", @@ -343,9 +340,7 @@ describe("resolveApiKeyForProfile OAuth refresh mirror-to-main (#26322)", () => expect(result?.apiKey).toBe("main-owner-refreshed-access"); expect(refreshProviderOAuthCredentialWithPluginMock).toHaveBeenCalledTimes(1); - const subRaw = JSON.parse( - await fs.readFile(path.join(subAgentDir, "auth-profiles.json"), "utf8"), - ) as AuthProfileStore; + const subRaw = readAuthProfileStoreForTest(subAgentDir); expectPersistedOpenAICodexProfile(subRaw.profiles[profileId], { access: "local-stale-access", refresh: "local-stale-refresh", @@ -353,9 +348,7 @@ describe("resolveApiKeyForProfile OAuth refresh mirror-to-main (#26322)", () => accountId, }); - const mainRaw = JSON.parse( - await fs.readFile(path.join(mainAgentDir, "auth-profiles.json"), "utf8"), - ) as AuthProfileStore; + const mainRaw = readAuthProfileStoreForTest(mainAgentDir); expectPersistedOpenAICodexProfile(mainRaw.profiles[profileId], { access: "main-owner-refreshed-access", refresh: "main-owner-refreshed-refresh", @@ -426,9 +419,7 @@ describe("resolveApiKeyForProfile OAuth refresh mirror-to-main (#26322)", () => expect(result?.provider).toBe(provider); // Sub-agent's store keeps its local expired credential; inherited OAuth is read-through. - const subRaw = JSON.parse( - await fs.readFile(path.join(subAgentDir, "auth-profiles.json"), "utf8"), - ) as AuthProfileStore; + const subRaw = readAuthProfileStoreForTest(subAgentDir); expectPersistedOpenAICodexProfile(subRaw.profiles[profileId], { access: "cached-access-token", refresh: "refresh-token", @@ -510,9 +501,7 @@ describe("resolveApiKeyForProfile OAuth refresh mirror-to-main (#26322)", () => expect(result?.apiKey).toBe("plugin-refreshed-access"); // Main store must have been mirrored from the plugin-refresh branch. - const mainRaw = JSON.parse( - await fs.readFile(path.join(mainAgentDir, "auth-profiles.json"), "utf8"), - ) as AuthProfileStore; + const mainRaw = readAuthProfileStoreForTest(mainAgentDir); const mainCredential = requireOAuthCredential(mainRaw, profileId); expect(mainCredential.access).toBe("plugin-refreshed-access"); expect(mainCredential.refresh).toBe("plugin-refreshed-refresh"); diff --git a/src/agents/auth-profiles/oauth.openai-codex-refresh-fallback.test.ts b/src/agents/auth-profiles/oauth.openai-codex-refresh-fallback.test.ts index 461f1ca6eb85..e5366ac4e419 100644 --- a/src/agents/auth-profiles/oauth.openai-codex-refresh-fallback.test.ts +++ b/src/agents/auth-profiles/oauth.openai-codex-refresh-fallback.test.ts @@ -3,8 +3,13 @@ import os from "node:os"; import path from "node:path"; import { afterAll, afterEach, beforeAll, beforeEach, describe, expect, it, vi } from "vitest"; import { resetFileLockStateForTest } from "../../infra/file-lock.js"; +import { closeOpenClawAgentDatabasesForTest } from "../../state/openclaw-agent-db.js"; import { captureEnv } from "../../test-utils/env.js"; -import { OAUTH_AGENT_ENV_KEYS, createExpiredOauthStore } from "./oauth-test-utils.js"; +import { + OAUTH_AGENT_ENV_KEYS, + createExpiredOauthStore, + readAuthProfileStoreForTest, +} from "./oauth-test-utils.js"; import { clearRuntimeAuthProfileStoreSnapshots, ensureAuthProfileStore, @@ -76,9 +81,7 @@ afterAll(() => { }); async function readPersistedStore(agentDir: string): Promise { - return JSON.parse( - await fs.readFile(path.join(agentDir, "auth-profiles.json"), "utf8"), - ) as AuthProfileStore; + return readAuthProfileStoreForTest(agentDir); } function mockRotatedOpenAICodexRefresh() { @@ -168,25 +171,34 @@ describe("resolveApiKeyForProfile openai refresh fallback", () => { afterEach(async () => { resetFileLockStateForTest(); clearRuntimeAuthProfileStoreSnapshots(); + closeOpenClawAgentDatabasesForTest(); envSnapshot.restore(); }); afterAll(async () => { + closeOpenClawAgentDatabasesForTest(); await fs.rm(tempRoot, { recursive: true, force: true }); }); - it("falls back to cached access token when openai refresh fails on accountId extraction", async () => { + it("falls back to matching cached Codex CLI credentials when openai refresh fails", async () => { const profileId = "openai:default"; - refreshProviderOAuthCredentialWithPluginMock.mockImplementationOnce( - async (params?: { context?: unknown }) => params?.context as never, - ); saveAuthProfileStore( createExpiredOauthStore({ profileId, provider: "openai", + accountId: "acct-cached", }), agentDir, + { filterExternalAuthProfiles: false, syncExternalCli: false }, ); + readCodexCliCredentialsCachedMock.mockReturnValue({ + type: "oauth", + provider: "openai", + access: "cached-access-token", + refresh: "cached-refresh-token", + expires: Date.now() + 86_400_000, + accountId: "acct-cached", + }); const result = await resolveApiKeyForProfile({ store: ensureAuthProfileStore(agentDir), diff --git a/src/agents/auth-profiles/path-resolve.ts b/src/agents/auth-profiles/path-resolve.ts index 382b3d98c9af..e1f955235a05 100644 --- a/src/agents/auth-profiles/path-resolve.ts +++ b/src/agents/auth-profiles/path-resolve.ts @@ -8,6 +8,7 @@ import { AUTH_STATE_FILENAME, LEGACY_AUTH_FILENAME, } from "./path-constants.js"; +import { resolveAuthProfileDatabasePath } from "./sqlite.js"; export function resolveAuthStorePath(agentDir?: string): string { const resolved = resolveUserPath(agentDir ?? resolveDefaultAgentDir({})); @@ -25,12 +26,12 @@ export function resolveAuthStatePath(agentDir?: string): string { } export function resolveAuthStorePathForDisplay(agentDir?: string): string { - const pathname = resolveAuthStorePath(agentDir); + const pathname = resolveAuthProfileDatabasePath(agentDir); return pathname.startsWith("~") ? pathname : resolveUserPath(pathname); } export function resolveAuthStatePathForDisplay(agentDir?: string): string { - const pathname = resolveAuthStatePath(agentDir); + const pathname = resolveAuthProfileDatabasePath(agentDir); return pathname.startsWith("~") ? pathname : resolveUserPath(pathname); } diff --git a/src/agents/auth-profiles/paths-direct-import.test.ts b/src/agents/auth-profiles/paths-direct-import.test.ts index 18b7bdc74e92..c2122e420f62 100644 --- a/src/agents/auth-profiles/paths-direct-import.test.ts +++ b/src/agents/auth-profiles/paths-direct-import.test.ts @@ -3,7 +3,6 @@ import os from "node:os"; import path from "node:path"; import { afterEach, beforeEach, describe, expect, it } from "vitest"; import { captureEnv } from "../../test-utils/env.js"; -import { AUTH_STORE_VERSION } from "./constants.js"; import { resolveAuthStatePath, resolveAuthStatePathForDisplay, @@ -11,7 +10,6 @@ import { resolveAuthStorePathForDisplay, resolveLegacyAuthStorePath, } from "./path-resolve.js"; -import { ensureAuthStoreFile } from "./paths.js"; // Direct-import sanity tests. These helpers are exercised transitively by the // wider auth-profile test suite via ESM re-exports through paths.ts, but v8 @@ -77,62 +75,18 @@ describe("path-resolve helpers (direct-import coverage attribution)", () => { const agentDir = path.join(stateDir, "agents", "main", "agent"); const resolved = resolveAuthStorePathForDisplay(agentDir); expect(resolved.startsWith(stateDir)).toBe(true); + expect(path.basename(resolved)).toBe("openclaw-agent.sqlite"); }); - it("resolveAuthStorePathForDisplay preserves a tilde-rooted path unchanged", () => { - // Exercises the `pathname.startsWith(\"~\")` branch. We use a contrived - // agentDir that already starts with `~` so the resolver echoes the - // tilde path back instead of expanding it via resolveUserPath. + it("resolveAuthStorePathForDisplay expands a tilde-rooted agent dir to the sqlite store", () => { const tildeAgentDir = "~fake-openclaw-no-expand"; const resolved = resolveAuthStorePathForDisplay(tildeAgentDir); - expect(resolved).toBe(path.resolve(tildeAgentDir, "auth-profiles.json")); + expect(resolved).toBe(path.resolve(tildeAgentDir, "openclaw-agent.sqlite")); }); - it("resolveAuthStatePathForDisplay returns the auth-state path for a non-tilde input", () => { + it("resolveAuthStatePathForDisplay returns the sqlite auth state store", () => { const agentDir = path.join(stateDir, "agents", "main", "agent"); const resolved = resolveAuthStatePathForDisplay(agentDir); - expect(resolved).toBe(path.join(agentDir, "auth-state.json")); - }); -}); - -describe("ensureAuthStoreFile (direct-import coverage attribution)", () => { - const envSnapshot = captureEnv(["OPENCLAW_STATE_DIR"]); - let stateDir = ""; - - beforeEach(async () => { - stateDir = await fs.mkdtemp(path.join(os.tmpdir(), "openclaw-path-ensure-")); - process.env.OPENCLAW_STATE_DIR = stateDir; - }); - - afterEach(async () => { - envSnapshot.restore(); - await fs.rm(stateDir, { recursive: true, force: true }); - }); - - it("creates a new auth-profiles.json when the file does not yet exist", async () => { - const target = path.join(stateDir, "sub", "auth-profiles.json"); - ensureAuthStoreFile(target); - const raw = await fs.readFile(target, "utf8"); - const parsed = JSON.parse(raw) as { version: number; profiles: Record }; - expect(parsed.version).toBe(AUTH_STORE_VERSION); - expect(parsed.profiles).toStrictEqual({}); - }); - - it("leaves an existing auth-profiles.json unchanged", async () => { - const target = path.join(stateDir, "auth-profiles.json"); - // Seed a file with custom content; ensureAuthStoreFile should bail out - // on the existsSync short-circuit and NOT overwrite. - await fs.writeFile( - target, - JSON.stringify({ - version: 1, - profiles: { canary: { type: "api_key", provider: "x", key: "k" } }, - }), - "utf8", - ); - ensureAuthStoreFile(target); - const raw = await fs.readFile(target, "utf8"); - const parsed = JSON.parse(raw) as { profiles: Record }; - expect(parsed.profiles.canary).toEqual({ type: "api_key", provider: "x", key: "k" }); + expect(resolved).toBe(path.join(agentDir, "openclaw-agent.sqlite")); }); }); diff --git a/src/agents/auth-profiles/paths.ts b/src/agents/auth-profiles/paths.ts index fb05e687c45f..e47678eed8d6 100644 --- a/src/agents/auth-profiles/paths.ts +++ b/src/agents/auth-profiles/paths.ts @@ -1,7 +1,3 @@ -import fs from "node:fs"; -import { saveJsonFile } from "../../infra/json-file.js"; -import { AUTH_STORE_VERSION } from "./constants.js"; -import type { AuthProfileSecretsStore } from "./types.js"; export { resolveAuthStatePath, resolveAuthStatePathForDisplay, @@ -10,14 +6,3 @@ export { resolveLegacyAuthStorePath, resolveOAuthRefreshLockPath, } from "./path-resolve.js"; - -export function ensureAuthStoreFile(pathname: string) { - if (fs.existsSync(pathname)) { - return; - } - const payload: AuthProfileSecretsStore = { - version: AUTH_STORE_VERSION, - profiles: {}, - }; - saveJsonFile(pathname, payload); -} diff --git a/src/agents/auth-profiles/persisted.ts b/src/agents/auth-profiles/persisted.ts index 94fddda85cec..855de6d1d4f3 100644 --- a/src/agents/auth-profiles/persisted.ts +++ b/src/agents/auth-profiles/persisted.ts @@ -4,6 +4,7 @@ import { uniqueStrings } from "@openclaw/normalization-core/string-normalization import { resolveOAuthPath } from "../../config/paths.js"; import { coerceSecretRef } from "../../config/types.secrets.js"; import { loadJsonFile } from "../../infra/json-file.js"; +import type { OpenClawAgentDatabase } from "../../state/openclaw-agent-db.js"; import { asBoolean } from "../../utils/boolean.js"; import { AUTH_STORE_VERSION, log } from "./constants.js"; import { isLegacyOAuthRef } from "./legacy-oauth-ref.js"; @@ -14,7 +15,8 @@ import { normalizeAuthEmailToken, normalizeAuthIdentityToken, } from "./oauth-shared.js"; -import { resolveAuthStorePath, resolveLegacyAuthStorePath } from "./paths.js"; +import { resolveLegacyAuthStorePath } from "./paths.js"; +import { readPersistedAuthProfileStoreRaw } from "./sqlite.js"; import { coerceAuthProfileState, loadPersistedAuthProfileState, @@ -32,6 +34,7 @@ export type LegacyAuthStore = Record; type LoadPersistedAuthProfileStoreOptions = { allowKeychainPrompt?: boolean; + database?: OpenClawAgentDatabase; }; type CredentialRejectReason = "non_object" | "invalid_type" | "missing_provider"; @@ -690,17 +693,19 @@ export function mergeOAuthFileIntoStore(store: AuthProfileStore): boolean { export function loadPersistedAuthProfileStore( agentDir?: string, - _options?: LoadPersistedAuthProfileStoreOptions, + options?: LoadPersistedAuthProfileStoreOptions, ): AuthProfileStore | null { - const authPath = resolveAuthStorePath(agentDir); - const raw = loadJsonFile(authPath); + const raw = readPersistedAuthProfileStoreRaw(agentDir, options?.database); const store = coercePersistedAuthProfileStore(raw); if (!store) { return null; } const merged = { ...store, - ...mergeAuthProfileState(coerceAuthProfileState(raw), loadPersistedAuthProfileState(agentDir)), + ...mergeAuthProfileState( + coerceAuthProfileState(raw), + loadPersistedAuthProfileState(agentDir, options?.database), + ), }; return merged; } diff --git a/src/agents/auth-profiles/profiles.test.ts b/src/agents/auth-profiles/profiles.test.ts index c5991c1d6900..a4f07be0e68b 100644 --- a/src/agents/auth-profiles/profiles.test.ts +++ b/src/agents/auth-profiles/profiles.test.ts @@ -3,8 +3,10 @@ import os from "node:os"; import path from "node:path"; import { describe, expect, it } from "vitest"; import { resolveOAuthDir } from "../../config/paths.js"; +import { closeOpenClawAgentDatabasesForTest } from "../../state/openclaw-agent-db.js"; +import { closeOpenClawStateDatabaseForTest } from "../../state/openclaw-state-db.js"; import { AUTH_STORE_VERSION } from "./constants.js"; -import { resolveAuthStorePath } from "./paths.js"; +import { loadPersistedAuthProfileStore } from "./persisted.js"; import { clearLastGoodProfileWithLock, promoteAuthProfileInOrder, @@ -141,10 +143,7 @@ describe("promoteAuthProfileInOrder", () => { { filterExternalAuthProfiles: false }, ); - const persisted = JSON.parse(fs.readFileSync(resolveAuthStorePath(agentDir), "utf8")) as { - profiles: Record>; - }; - const credential = persisted.profiles[profileId]; + const credential = loadPersistedAuthProfileStore(agentDir)?.profiles[profileId]; expectOAuthCredentialFields(credential, { provider: "openai", @@ -175,6 +174,8 @@ describe("promoteAuthProfileInOrder", () => { } else { process.env.OPENCLAW_STATE_DIR = previousStateDir; } + closeOpenClawAgentDatabasesForTest(); + closeOpenClawStateDatabaseForTest(); fs.rmSync(stateDir, { recursive: true, force: true }); } }); @@ -204,10 +205,7 @@ describe("promoteAuthProfileInOrder", () => { { filterExternalAuthProfiles: false }, ); - const persisted = JSON.parse(fs.readFileSync(resolveAuthStorePath(agentDir), "utf8")) as { - profiles: Record>; - }; - const credential = persisted.profiles[profileId]; + const credential = loadPersistedAuthProfileStore(agentDir)?.profiles[profileId]; expectOAuthCredentialFields(credential, { provider: "openai", access: "access-only-token", @@ -229,114 +227,8 @@ describe("promoteAuthProfileInOrder", () => { } else { process.env.OPENCLAW_STATE_DIR = previousStateDir; } - fs.rmSync(stateDir, { recursive: true, force: true }); - } - }); - - it("preserves legacy OAuth sidecar refs during unrelated auth-store saves", () => { - const stateDir = fs.mkdtempSync(path.join(os.tmpdir(), "openclaw-auth-profile-sidecar-")); - const agentDir = path.join(stateDir, "agents", "main", "agent"); - const previousStateDir = process.env.OPENCLAW_STATE_DIR; - process.env.OPENCLAW_STATE_DIR = stateDir; - try { - fs.mkdirSync(agentDir, { recursive: true }); - const authPath = resolveAuthStorePath(agentDir); - const legacyProvider = "retired-oauth-provider"; - fs.writeFileSync( - authPath, - JSON.stringify({ - version: AUTH_STORE_VERSION, - profiles: { - "openai:default": { - type: "oauth", - provider: "openai", - refresh: "refresh-token", - oauthRef: { - source: "openclaw-credentials", - provider: legacyProvider, - id: "legacy-profile", - }, - }, - }, - }), - ); - - const store = loadAuthProfileStoreWithoutExternalProfiles(agentDir); - saveAuthProfileStore(store, agentDir); - - const persisted = JSON.parse(fs.readFileSync(authPath, "utf8")) as { - profiles: Record>; - }; - expect(persisted.profiles["openai:default"]?.oauthRef).toEqual({ - source: "openclaw-credentials", - provider: legacyProvider, - id: "legacy-profile", - }); - expect(store.profiles["openai:default"]).not.toHaveProperty("oauthRef"); - } finally { - if (previousStateDir === undefined) { - delete process.env.OPENCLAW_STATE_DIR; - } else { - process.env.OPENCLAW_STATE_DIR = previousStateDir; - } - fs.rmSync(stateDir, { recursive: true, force: true }); - } - }); - - it("drops legacy OAuth sidecar refs when inline token material changes", () => { - const stateDir = fs.mkdtempSync(path.join(os.tmpdir(), "openclaw-auth-profile-sidecar-new-")); - const agentDir = path.join(stateDir, "agents", "main", "agent"); - const previousStateDir = process.env.OPENCLAW_STATE_DIR; - process.env.OPENCLAW_STATE_DIR = stateDir; - try { - fs.mkdirSync(agentDir, { recursive: true }); - const authPath = resolveAuthStorePath(agentDir); - fs.writeFileSync( - authPath, - JSON.stringify({ - version: AUTH_STORE_VERSION, - profiles: { - "openai:default": { - type: "oauth", - provider: "openai", - refresh: "old-refresh-token", - oauthRef: { - source: "openclaw-credentials", - provider: "retired-oauth-provider", - id: "legacy-profile", - }, - }, - }, - }), - ); - - saveAuthProfileStore( - { - version: AUTH_STORE_VERSION, - profiles: { - "openai:default": { - type: "oauth", - provider: "openai", - access: "new-access-token", - refresh: "new-refresh-token", - expires: Date.now() + 60 * 60 * 1000, - }, - }, - }, - agentDir, - { filterExternalAuthProfiles: false }, - ); - - const persisted = JSON.parse(fs.readFileSync(authPath, "utf8")) as { - profiles: Record>; - }; - expect(persisted.profiles["openai:default"]).not.toHaveProperty("oauthRef"); - } finally { - if (previousStateDir === undefined) { - delete process.env.OPENCLAW_STATE_DIR; - } else { - process.env.OPENCLAW_STATE_DIR = previousStateDir; - } + closeOpenClawAgentDatabasesForTest(); + closeOpenClawStateDatabaseForTest(); fs.rmSync(stateDir, { recursive: true, force: true }); } }); @@ -405,16 +297,20 @@ describe("promoteAuthProfileInOrder", () => { refresh: "copy-refresh-token", }, ); - const copiedRaw = fs.readFileSync(resolveAuthStorePath(copiedAgentDir), "utf8"); - expect(copiedRaw).toContain("copy-access-token"); - expect(copiedRaw).toContain("copy-refresh-token"); - expect(copiedRaw).not.toContain("oauthRef"); + expect( + loadPersistedAuthProfileStore(copiedAgentDir)?.profiles[copiedProfileId], + ).toMatchObject({ + access: "copy-access-token", + refresh: "copy-refresh-token", + }); } finally { if (previousStateDir === undefined) { delete process.env.OPENCLAW_STATE_DIR; } else { process.env.OPENCLAW_STATE_DIR = previousStateDir; } + closeOpenClawAgentDatabasesForTest(); + closeOpenClawStateDatabaseForTest(); fs.rmSync(stateDir, { recursive: true, force: true }); } }); diff --git a/src/agents/auth-profiles/source-check.ts b/src/agents/auth-profiles/source-check.ts index 9e20baec0c4d..4912d611926a 100644 --- a/src/agents/auth-profiles/source-check.ts +++ b/src/agents/auth-profiles/source-check.ts @@ -4,7 +4,11 @@ import { resolveAuthStorePath, resolveLegacyAuthStorePath, } from "./path-resolve.js"; -import { hasAnyRuntimeAuthProfileStoreSource } from "./runtime-snapshots.js"; +import { + getRuntimeAuthProfileStoreSnapshot, + hasAnyRuntimeAuthProfileStoreSource, +} from "./runtime-snapshots.js"; +import { readPersistedAuthProfileStateRaw, readPersistedAuthProfileStoreRaw } from "./sqlite.js"; function hasStoredAuthProfileFiles(agentDir?: string): boolean { return ( @@ -15,17 +19,36 @@ function hasStoredAuthProfileFiles(agentDir?: string): boolean { } export function hasAnyAuthProfileStoreSource(agentDir?: string): boolean { - if (hasAnyRuntimeAuthProfileStoreSource(agentDir)) { + if (hasLocalAuthProfileStoreSource(agentDir)) { return true; } - if (hasStoredAuthProfileFiles(agentDir)) { + if (hasAnyRuntimeAuthProfileStoreSource(agentDir)) { return true; } const authPath = resolveAuthStorePath(agentDir); const mainAuthPath = resolveAuthStorePath(); - if (agentDir && authPath !== mainAuthPath && hasStoredAuthProfileFiles(undefined)) { + if ( + agentDir && + authPath !== mainAuthPath && + (hasStoredAuthProfileFiles(undefined) || + readPersistedAuthProfileStoreRaw(undefined) || + readPersistedAuthProfileStateRaw(undefined)) + ) { return true; } return false; } + +export function hasLocalAuthProfileStoreSource(agentDir?: string): boolean { + const runtimeStore = getRuntimeAuthProfileStoreSnapshot(agentDir); + if (runtimeStore && Object.keys(runtimeStore.profiles).length > 0) { + return true; + } + if (hasStoredAuthProfileFiles(agentDir)) { + return true; + } + return Boolean( + readPersistedAuthProfileStoreRaw(agentDir) || readPersistedAuthProfileStateRaw(agentDir), + ); +} diff --git a/src/agents/auth-profiles/sqlite.ts b/src/agents/auth-profiles/sqlite.ts new file mode 100644 index 000000000000..85ee9fbccd00 --- /dev/null +++ b/src/agents/auth-profiles/sqlite.ts @@ -0,0 +1,248 @@ +import { createHash } from "node:crypto"; +import fs from "node:fs"; +import path from "node:path"; +import type { DatabaseSync } from "node:sqlite"; +import { + clearNodeSqliteKyselyCacheForDatabase, + executeSqliteQuerySync, + executeSqliteQueryTakeFirstSync, + getNodeSqliteKysely, +} from "../../infra/kysely-sync.js"; +import { requireNodeSqlite } from "../../infra/node-sqlite.js"; +import type { DB as OpenClawAgentKyselyDatabase } from "../../state/openclaw-agent-db.generated.js"; +import { + openOpenClawAgentDatabase, + runOpenClawAgentWriteTransaction, + type OpenClawAgentDatabase, +} from "../../state/openclaw-agent-db.js"; +import { resolveUserPath } from "../../utils.js"; +import { resolveRegisteredAgentIdForDir } from "../agent-dir-registry.js"; +import { resolveDefaultAgentDir } from "../agent-scope-config.js"; + +type AuthProfileDatabase = Pick< + OpenClawAgentKyselyDatabase, + "auth_profile_store" | "auth_profile_state" +>; + +const PRIMARY_ROW_KEY = "primary"; + +function resolveAgentDir(agentDir?: string): string { + return resolveUserPath(agentDir ?? resolveDefaultAgentDir({})); +} + +function inferAgentIdFromDir(agentDir: string): string { + const normalized = path.normalize(agentDir); + if (path.basename(normalized) === "agent") { + const parent = path.basename(path.dirname(normalized)); + if (parent) { + return parent; + } + } + const hash = createHash("sha256").update(normalized).digest("hex").slice(0, 12); + return `custom-${hash}`; +} + +function resolveAuthProfileDatabaseOptions(agentDir?: string) { + const dir = resolveAgentDir(agentDir); + return { + agentId: resolveRegisteredAgentIdForDir(dir) ?? inferAgentIdFromDir(dir), + path: path.join(dir, "openclaw-agent.sqlite"), + }; +} + +export function resolveAuthProfileDatabasePath(agentDir?: string): string { + return resolveAuthProfileDatabaseOptions(agentDir).path; +} + +export function resolveAuthProfileDatabaseFilePaths(agentDir?: string): string[] { + const databasePath = resolveAuthProfileDatabasePath(agentDir); + return [databasePath, `${databasePath}-wal`, `${databasePath}-shm`]; +} + +function parseJsonCell(raw: string | null | undefined): unknown { + if (!raw) { + return null; + } + try { + return JSON.parse(raw) as unknown; + } catch { + return null; + } +} + +function getAuthProfileKysely(db: DatabaseSync) { + return getNodeSqliteKysely(db); +} + +export function openAuthProfileDatabase(agentDir?: string): OpenClawAgentDatabase { + return openOpenClawAgentDatabase(resolveAuthProfileDatabaseOptions(agentDir)); +} + +function readAuthProfileJsonCellReadOnly(pathname: string, target: "store" | "state"): unknown { + const sqlite = requireNodeSqlite(); + const db = new sqlite.DatabaseSync(pathname, { readOnly: true }); + try { + const kysely = getAuthProfileKysely(db); + if (target === "store") { + const row = executeSqliteQueryTakeFirstSync( + db, + kysely + .selectFrom("auth_profile_store") + .select("store_json") + .where("store_key", "=", PRIMARY_ROW_KEY), + ); + return parseJsonCell(row?.store_json); + } + const row = executeSqliteQueryTakeFirstSync( + db, + kysely + .selectFrom("auth_profile_state") + .select("state_json") + .where("state_key", "=", PRIMARY_ROW_KEY), + ); + return parseJsonCell(row?.state_json); + } catch { + return null; + } finally { + clearNodeSqliteKyselyCacheForDatabase(db); + db.close(); + } +} + +export function readPersistedAuthProfileStoreRaw( + agentDir?: string, + database?: OpenClawAgentDatabase, +): unknown { + if (database) { + const db = getAuthProfileKysely(database.db); + const row = executeSqliteQueryTakeFirstSync( + database.db, + db + .selectFrom("auth_profile_store") + .select("store_json") + .where("store_key", "=", PRIMARY_ROW_KEY), + ); + return parseJsonCell(row?.store_json); + } + const databasePath = resolveAuthProfileDatabasePath(agentDir); + if (!fs.existsSync(databasePath)) { + return null; + } + return readAuthProfileJsonCellReadOnly(databasePath, "store"); +} + +export function readPersistedAuthProfileStateRaw( + agentDir?: string, + database?: OpenClawAgentDatabase, +): unknown { + if (database) { + const db = getAuthProfileKysely(database.db); + const row = executeSqliteQueryTakeFirstSync( + database.db, + db + .selectFrom("auth_profile_state") + .select("state_json") + .where("state_key", "=", PRIMARY_ROW_KEY), + ); + return parseJsonCell(row?.state_json); + } + const databasePath = resolveAuthProfileDatabasePath(agentDir); + if (!fs.existsSync(databasePath)) { + return null; + } + return readAuthProfileJsonCellReadOnly(databasePath, "state"); +} + +export function writePersistedAuthProfileStoreRaw( + payload: unknown, + agentDir?: string, + database?: OpenClawAgentDatabase, +): void { + const write = (target: OpenClawAgentDatabase) => { + const db = getAuthProfileKysely(target.db); + executeSqliteQuerySync( + target.db, + db + .insertInto("auth_profile_store") + .values({ + store_key: PRIMARY_ROW_KEY, + store_json: JSON.stringify(payload), + updated_at: Date.now(), + }) + .onConflict((conflict) => + conflict.column("store_key").doUpdateSet({ + store_json: JSON.stringify(payload), + updated_at: Date.now(), + }), + ), + ); + }; + if (database) { + write(database); + return; + } + runOpenClawAgentWriteTransaction(write, resolveAuthProfileDatabaseOptions(agentDir)); +} + +export function deletePersistedAuthProfileStoreRaw( + agentDir?: string, + database?: OpenClawAgentDatabase, +): void { + const remove = (target: OpenClawAgentDatabase) => { + const db = getAuthProfileKysely(target.db); + executeSqliteQuerySync( + target.db, + db.deleteFrom("auth_profile_store").where("store_key", "=", PRIMARY_ROW_KEY), + ); + }; + if (database) { + remove(database); + return; + } + runOpenClawAgentWriteTransaction(remove, resolveAuthProfileDatabaseOptions(agentDir)); +} + +export function writePersistedAuthProfileStateRaw( + payload: unknown, + agentDir?: string, + database?: OpenClawAgentDatabase, +): void { + const write = (target: OpenClawAgentDatabase) => { + const db = getAuthProfileKysely(target.db); + if (!payload) { + executeSqliteQuerySync( + target.db, + db.deleteFrom("auth_profile_state").where("state_key", "=", PRIMARY_ROW_KEY), + ); + return; + } + executeSqliteQuerySync( + target.db, + db + .insertInto("auth_profile_state") + .values({ + state_key: PRIMARY_ROW_KEY, + state_json: JSON.stringify(payload), + updated_at: Date.now(), + }) + .onConflict((conflict) => + conflict.column("state_key").doUpdateSet({ + state_json: JSON.stringify(payload), + updated_at: Date.now(), + }), + ), + ); + }; + if (database) { + write(database); + return; + } + runOpenClawAgentWriteTransaction(write, resolveAuthProfileDatabaseOptions(agentDir)); +} + +export function runAuthProfileWriteTransaction( + agentDir: string | undefined, + operation: (database: OpenClawAgentDatabase) => T, +): T { + return runOpenClawAgentWriteTransaction(operation, resolveAuthProfileDatabaseOptions(agentDir)); +} diff --git a/src/agents/auth-profiles/state.ts b/src/agents/auth-profiles/state.ts index 76f1f6fa372b..008c448d7dc1 100644 --- a/src/agents/auth-profiles/state.ts +++ b/src/agents/auth-profiles/state.ts @@ -1,13 +1,12 @@ -import fs from "node:fs"; import { isDeepStrictEqual } from "node:util"; import { normalizeProviderId } from "@openclaw/model-catalog-core/provider-id"; import { asFiniteNumber } from "@openclaw/normalization-core/number-coercion"; import { isRecord } from "@openclaw/normalization-core/record-coerce"; import { normalizeOptionalString } from "@openclaw/normalization-core/string-coerce"; import { normalizeTrimmedStringList } from "@openclaw/normalization-core/string-normalization"; -import { loadJsonFile, repairJsonFilePermissions, saveJsonFile } from "../../infra/json-file.js"; +import type { OpenClawAgentDatabase } from "../../state/openclaw-agent-db.js"; import { AUTH_STORE_VERSION } from "./constants.js"; -import { resolveAuthStatePath } from "./paths.js"; +import { readPersistedAuthProfileStateRaw, writePersistedAuthProfileStateRaw } from "./sqlite.js"; import type { AuthProfileBlockedReason, AuthProfileBlockedSource, @@ -181,11 +180,16 @@ export function mergeAuthProfileState( }; } -export function loadPersistedAuthProfileState(agentDir?: string): AuthProfileState { - return coerceAuthProfileState(loadJsonFile(resolveAuthStatePath(agentDir))); +export function loadPersistedAuthProfileState( + agentDir?: string, + database?: OpenClawAgentDatabase, +): AuthProfileState { + return coerceAuthProfileState(readPersistedAuthProfileStateRaw(agentDir, database)); } -function buildPersistedAuthProfileState(store: AuthProfileState): AuthProfileStateStore | null { +export function buildPersistedAuthProfileState( + store: AuthProfileState, +): AuthProfileStateStore | null { const state = coerceAuthProfileState(store); if (!state.order && !state.lastGood && !state.usageStats) { return null; @@ -203,21 +207,9 @@ export function savePersistedAuthProfileState( agentDir?: string, ): AuthProfileStateStore | null { const payload = buildPersistedAuthProfileState(store); - const statePath = resolveAuthStatePath(agentDir); - if (!payload) { - try { - fs.unlinkSync(statePath); - } catch (error) { - if ((error as NodeJS.ErrnoException)?.code !== "ENOENT") { - throw error; - } - } - return null; - } - if (isDeepStrictEqual(loadJsonFile(statePath), payload)) { - repairJsonFilePermissions(statePath); - } else { - saveJsonFile(statePath, payload); + const existingRaw = readPersistedAuthProfileStateRaw(agentDir); + if (!payload || !isDeepStrictEqual(existingRaw, payload)) { + writePersistedAuthProfileStateRaw(payload, agentDir); } return payload; } diff --git a/src/agents/auth-profiles/store-cache.ts b/src/agents/auth-profiles/store-cache.ts deleted file mode 100644 index c3100d4daa58..000000000000 --- a/src/agents/auth-profiles/store-cache.ts +++ /dev/null @@ -1,50 +0,0 @@ -import { cloneAuthProfileStore } from "./clone.js"; -import { EXTERNAL_CLI_SYNC_TTL_MS } from "./constants.js"; -import type { AuthProfileStore } from "./types.js"; - -const loadedAuthStoreCache = new Map< - string, - { - authMtimeMs: number | null; - stateMtimeMs: number | null; - syncedAtMs: number; - store: AuthProfileStore; - } ->(); - -export function readCachedAuthProfileStore(params: { - authPath: string; - authMtimeMs: number | null; - stateMtimeMs: number | null; -}): AuthProfileStore | null { - const cached = loadedAuthStoreCache.get(params.authPath); - if ( - !cached || - cached.authMtimeMs !== params.authMtimeMs || - cached.stateMtimeMs !== params.stateMtimeMs - ) { - return null; - } - if (Date.now() - cached.syncedAtMs >= EXTERNAL_CLI_SYNC_TTL_MS) { - return null; - } - return cloneAuthProfileStore(cached.store); -} - -export function writeCachedAuthProfileStore(params: { - authPath: string; - authMtimeMs: number | null; - stateMtimeMs: number | null; - store: AuthProfileStore; -}): void { - loadedAuthStoreCache.set(params.authPath, { - authMtimeMs: params.authMtimeMs, - stateMtimeMs: params.stateMtimeMs, - syncedAtMs: Date.now(), - store: cloneAuthProfileStore(params.store), - }); -} - -export function clearLoadedAuthStoreCache(): void { - loadedAuthStoreCache.clear(); -} diff --git a/src/agents/auth-profiles/store.runtime-external.test.ts b/src/agents/auth-profiles/store.runtime-external.test.ts deleted file mode 100644 index 77593afe630e..000000000000 --- a/src/agents/auth-profiles/store.runtime-external.test.ts +++ /dev/null @@ -1,105 +0,0 @@ -import fs from "node:fs/promises"; -import os from "node:os"; -import path from "node:path"; -import { afterEach, beforeEach, describe, expect, it } from "vitest"; -import type { ProviderExternalAuthProfile } from "../../plugins/types.js"; -import { testing as externalAuthTesting } from "./external-auth.js"; -import { resolveAuthStatePath, resolveAuthStorePath } from "./paths.js"; -import { getRuntimeAuthProfileStoreSnapshot } from "./runtime-snapshots.js"; -import { - clearRuntimeAuthProfileStoreSnapshots, - replaceRuntimeAuthProfileStoreSnapshots, - saveAuthProfileStore, -} from "./store.js"; -import type { AuthProfileStore, OAuthCredential } from "./types.js"; - -const envBackup: Record = {}; -const envKeys = ["OPENCLAW_STATE_DIR"]; -const tempDirs: string[] = []; - -function createRuntimeExternalCredential(): OAuthCredential { - return { - type: "oauth", - provider: "claude-cli", - access: "external-access-token", - refresh: "external-refresh-token", - expires: Date.now() + 60_000, - }; -} - -beforeEach(() => { - for (const key of envKeys) { - envBackup[key] = process.env[key]; - } - externalAuthTesting.setResolveExternalAuthProfilesForTest(() => []); - clearRuntimeAuthProfileStoreSnapshots(); -}); - -afterEach(async () => { - for (const key of envKeys) { - if (envBackup[key] === undefined) { - delete process.env[key]; - } else { - process.env[key] = envBackup[key]; - } - } - externalAuthTesting.resetResolveExternalAuthProfilesForTest(); - clearRuntimeAuthProfileStoreSnapshots(); - await Promise.all(tempDirs.splice(0).map((dir) => fs.rm(dir, { recursive: true, force: true }))); -}); - -describe("auth profile store runtime external snapshots", () => { - it("keeps runtime-only external oauth profiles in active snapshots after save", async () => { - const stateDir = await fs.mkdtemp(path.join(os.tmpdir(), "openclaw-runtime-external-save-")); - tempDirs.push(stateDir); - process.env.OPENCLAW_STATE_DIR = stateDir; - const agentDir = path.join(stateDir, "agents", "main", "agent"); - await fs.mkdir(agentDir, { recursive: true }); - - const externalProfileId = "anthropic:claude-cli"; - const externalCredential = createRuntimeExternalCredential(); - const externalProfiles: ProviderExternalAuthProfile[] = [ - { - profileId: externalProfileId, - credential: externalCredential, - persistence: "runtime-only", - }, - ]; - externalAuthTesting.setResolveExternalAuthProfilesForTest(() => externalProfiles); - - const runtimeStore: AuthProfileStore = { - version: 1, - profiles: { - "openai:static": { - type: "api_key", - provider: "openai", - key: "sk-openai-static", // pragma: allowlist secret - }, - [externalProfileId]: externalCredential, - }, - order: { - openai: ["openai:static"], - "claude-cli": [externalProfileId], - }, - runtimeExternalProfileIds: [externalProfileId], - }; - replaceRuntimeAuthProfileStoreSnapshots([{ agentDir, store: runtimeStore }]); - - saveAuthProfileStore(runtimeStore, agentDir); - - const persisted = JSON.parse( - await fs.readFile(resolveAuthStorePath(agentDir), "utf8"), - ) as AuthProfileStore; - const persistedState = JSON.parse( - await fs.readFile(resolveAuthStatePath(agentDir), "utf8"), - ) as AuthProfileStore; - expect(persisted.profiles[externalProfileId]).toBeUndefined(); - expect(persisted.order?.["claude-cli"]).toBeUndefined(); - expect(persistedState.order?.["claude-cli"]).toBeUndefined(); - - const snapshot = getRuntimeAuthProfileStoreSnapshot(agentDir); - expect(snapshot?.profiles[externalProfileId]).toEqual(externalCredential); - expect(snapshot?.runtimeExternalProfileIds).toEqual([externalProfileId]); - expect(snapshot?.order?.["claude-cli"]).toEqual([externalProfileId]); - }); -}); diff --git a/src/agents/auth-profiles/store.ts b/src/agents/auth-profiles/store.ts index 9b43fa4f1c70..1771e3d05a4e 100644 --- a/src/agents/auth-profiles/store.ts +++ b/src/agents/auth-profiles/store.ts @@ -1,13 +1,10 @@ -import fs from "node:fs"; -import path from "node:path"; import { isDeepStrictEqual } from "node:util"; import type { OpenClawConfig } from "../../config/types.openclaw.js"; -import { withFileLock } from "../../infra/file-lock.js"; -import { loadJsonFile, repairJsonFilePermissions, saveJsonFile } from "../../infra/json-file.js"; import { asDateTimestampMs } from "../../shared/number-coercion.js"; +import type { OpenClawAgentDatabase } from "../../state/openclaw-agent-db.js"; import { isRecord } from "../../utils.js"; import { cloneAuthProfileStore } from "./clone.js"; -import { AUTH_STORE_LOCK_OPTIONS, AUTH_STORE_VERSION, log } from "./constants.js"; +import { AUTH_STORE_VERSION, log } from "./constants.js"; import { listRuntimeExternalAuthProfiles, overlayExternalAuthProfiles, @@ -19,16 +16,9 @@ import { shouldPersistRuntimeExternalOAuthProfile, type RuntimeExternalOAuthProfile, } from "./oauth-shared.js"; +import { resolveAuthStorePath } from "./paths.js"; import { - ensureAuthStoreFile, - resolveAuthStatePath, - resolveAuthStorePath, - resolveLegacyAuthStorePath, -} from "./paths.js"; -import { - applyLegacyAuthStore, buildPersistedAuthProfileSecretsStore, - loadLegacyAuthProfileStore, loadPersistedAuthProfileStore, mergeAuthProfileStores, mergeOAuthFileIntoStore, @@ -40,17 +30,23 @@ import { replaceRuntimeAuthProfileStoreSnapshots as replaceRuntimeAuthProfileStoreSnapshotsImpl, setRuntimeAuthProfileStoreSnapshot, } from "./runtime-snapshots.js"; -import { loadPersistedAuthProfileState, savePersistedAuthProfileState } from "./state.js"; import { - clearLoadedAuthStoreCache, - readCachedAuthProfileStore, - writeCachedAuthProfileStore, -} from "./store-cache.js"; + readPersistedAuthProfileStoreRaw, + writePersistedAuthProfileStateRaw, + runAuthProfileWriteTransaction, + writePersistedAuthProfileStoreRaw, +} from "./sqlite.js"; +import { + buildPersistedAuthProfileState, + loadPersistedAuthProfileState, + savePersistedAuthProfileState, +} from "./state.js"; import type { AuthProfileStore } from "./types.js"; type LoadAuthProfileStoreOptions = { allowKeychainPrompt?: boolean; config?: OpenClawConfig; + database?: OpenClawAgentDatabase; externalCli?: ExternalCliAuthDiscovery; readOnly?: boolean; syncExternalCli?: boolean; @@ -61,6 +57,7 @@ type LoadAuthProfileStoreOptions = { type SaveAuthProfileStoreOptions = { filterExternalAuthProfiles?: boolean; preserveOrderProfileIds?: Iterable; + preserveStateProfileIds?: Iterable; pruneOrderProfileIds?: Iterable; syncExternalCli?: boolean; }; @@ -127,23 +124,20 @@ type ResolvedExternalCliOverlayOptions = { externalCliProfileIds?: Iterable; }; -type SyncLockSnapshot = { - raw: string; - stat: fs.Stats; - payload: Record | null; -}; - type ExternalCliSyncResult = { store: AuthProfileStore; cacheable: boolean; }; function resolvePersistedLoadOptions( - options: Pick | undefined, -): { allowKeychainPrompt?: boolean } { - return options?.allowKeychainPrompt !== undefined - ? { allowKeychainPrompt: options.allowKeychainPrompt } - : {}; + options: Pick | undefined, +): { allowKeychainPrompt?: boolean; database?: OpenClawAgentDatabase } { + return { + ...(options?.allowKeychainPrompt !== undefined + ? { allowKeychainPrompt: options.allowKeychainPrompt } + : {}), + ...(options?.database ? { database: options.database } : {}), + }; } function isInheritedMainOAuthCredential(params: { @@ -235,77 +229,6 @@ function resolveRuntimeAuthProfileStore( return null; } -function readAuthStoreMtimeMs(authPath: string): number | null { - try { - return fs.statSync(authPath).mtimeMs; - } catch { - return null; - } -} - -function readSyncLockSnapshot(lockPath: string): SyncLockSnapshot | null { - try { - const stat = fs.lstatSync(lockPath); - const raw = fs.readFileSync(lockPath, "utf8"); - let payload: Record | null = null; - try { - const parsed = JSON.parse(raw) as unknown; - payload = - parsed && typeof parsed === "object" && !Array.isArray(parsed) - ? (parsed as Record) - : null; - } catch { - payload = null; - } - return { raw, stat, payload }; - } catch { - return null; - } -} - -function syncLockSnapshotMatches(lockPath: string, snapshot: SyncLockSnapshot): boolean { - try { - const stat = fs.lstatSync(lockPath); - return ( - stat.dev === snapshot.stat.dev && - stat.ino === snapshot.stat.ino && - fs.readFileSync(lockPath, "utf8") === snapshot.raw - ); - } catch { - return false; - } -} - -function acquireAuthStoreLockSync(authPath: string): (() => void) | null { - const lockPath = `${authPath}.lock`; - fs.mkdirSync(path.dirname(authPath), { recursive: true }); - - try { - const fd = fs.openSync(lockPath, "wx"); - const raw = `${JSON.stringify( - { pid: process.pid, createdAt: new Date().toISOString() }, - null, - 2, - )}\n`; - try { - fs.writeFileSync(fd, raw, "utf8"); - } finally { - fs.closeSync(fd); - } - const snapshot = readSyncLockSnapshot(lockPath); - return () => { - if (snapshot && syncLockSnapshotMatches(lockPath, snapshot)) { - fs.rmSync(lockPath, { force: true }); - } - }; - } catch (err) { - if ((err as NodeJS.ErrnoException)?.code === "EEXIST") { - return null; - } - throw err; - } -} - function resolveExternalCliOverlayOptions( options: LoadAuthProfileStoreOptions | undefined, ): ResolvedExternalCliOverlayOptions { @@ -384,45 +307,46 @@ function maybeSyncPersistedExternalCliAuthProfiles(params: { return { store: synced, cacheable: true }; } - const authPath = resolveAuthStorePath(params.agentDir); - const release = acquireAuthStoreLockSync(authPath); - if (!release) { - log.warn("skipped persisted external cli auth sync because auth store is locked", { - authPath, + try { + return runAuthProfileWriteTransaction(params.agentDir, (database) => { + const latestStore = loadPersistedAuthProfileStore(params.agentDir, { + ...resolvePersistedLoadOptions(params.options), + database, + }) ?? { + version: AUTH_STORE_VERSION, + profiles: {}, + }; + let changed = false; + for (const [profileId, credential] of changedProfiles) { + const previous = params.store.profiles[profileId]; + const latest = latestStore.profiles[profileId]; + if (!isDeepStrictEqual(latest, previous)) { + log.debug("skipped persisted external cli auth sync for concurrently changed profile", { + profileId, + }); + continue; + } + latestStore.profiles[profileId] = credential; + changed = true; + } + if (changed) { + saveAuthProfileStore( + latestStore, + params.agentDir, + { + filterExternalAuthProfiles: false, + }, + database, + ); + } + return { store: latestStore, cacheable: true }; + }); + } catch (err) { + log.warn("skipped persisted external cli auth sync because auth store write failed", { + err, }); return { store: params.store, cacheable: false }; } - try { - const latestStore = loadPersistedAuthProfileStore( - params.agentDir, - resolvePersistedLoadOptions(params.options), - ) ?? { - version: AUTH_STORE_VERSION, - profiles: {}, - }; - let changed = false; - for (const [profileId, credential] of changedProfiles) { - const previous = params.store.profiles[profileId]; - const latest = latestStore.profiles[profileId]; - if (!isDeepStrictEqual(latest, previous)) { - log.debug("skipped persisted external cli auth sync for concurrently changed profile", { - profileId, - }); - continue; - } - latestStore.profiles[profileId] = credential; - changed = true; - } - if (changed) { - saveAuthProfileStore(latestStore, params.agentDir, { - filterExternalAuthProfiles: false, - }); - return { store: latestStore, cacheable: true }; - } - return { store: latestStore, cacheable: true }; - } finally { - release(); - } } function shouldKeepProfileInLocalStore(params: { @@ -538,6 +462,13 @@ function buildLocalAuthProfileStoreForSave(params: { ); const keptProfileIds = new Set(Object.keys(localStore.profiles)); const keptOrderProfileIds = new Set(keptProfileIds); + for (const profileId of params.options?.preserveStateProfileIds ?? []) { + const normalizedProfileId = profileId.trim(); + if (normalizedProfileId) { + keptProfileIds.add(normalizedProfileId); + keptOrderProfileIds.add(normalizedProfileId); + } + } for (const profileIds of Object.values( loadPersistedAuthProfileState(params.agentDir).order ?? {}, )) { @@ -776,18 +707,16 @@ export async function updateAuthProfileStoreWithLock(params: { saveOptions?: SaveAuthProfileStoreOptions; updater: (store: AuthProfileStore) => boolean; }): Promise { - const authPath = resolveAuthStorePath(params.agentDir); - ensureAuthStoreFile(authPath); - try { - return await withFileLock(authPath, AUTH_STORE_LOCK_OPTIONS, async () => { - // Locked writers must reload from disk, not from any runtime snapshot. - // Otherwise a live gateway can overwrite fresher CLI/config-auth writes - // with stale in-memory auth state during usage/cooldown updates. - const store = loadAuthProfileStoreForAgent(params.agentDir, { syncExternalCli: false }); + return runAuthProfileWriteTransaction(params.agentDir, (database) => { + const store = loadAuthProfileStoreForAgent(params.agentDir, { + database, + readOnly: true, + syncExternalCli: false, + }); const shouldSave = params.updater(store); if (shouldSave) { - saveAuthProfileStore(store, params.agentDir, params.saveOptions); + saveAuthProfileStore(store, params.agentDir, params.saveOptions, database); } return store; }); @@ -801,15 +730,6 @@ export function loadAuthProfileStore(): AuthProfileStore { if (asStore) { return overlayExternalAuthProfiles(asStore); } - const legacy = loadLegacyAuthProfileStore(); - if (legacy) { - const store: AuthProfileStore = { - version: AUTH_STORE_VERSION, - profiles: {}, - }; - applyLegacyAuthStore(store, legacy); - return overlayExternalAuthProfiles(store); - } const store: AuthProfileStore = { version: AUTH_STORE_VERSION, profiles: {} }; return overlayExternalAuthProfiles(store); @@ -820,20 +740,6 @@ function loadAuthProfileStoreForAgent( options?: LoadAuthProfileStoreOptions, ): AuthProfileStore { const readOnly = options?.readOnly === true; - const authPath = resolveAuthStorePath(agentDir); - const statePath = resolveAuthStatePath(agentDir); - const authMtimeMs = readAuthStoreMtimeMs(authPath); - const stateMtimeMs = readAuthStoreMtimeMs(statePath); - if (!readOnly) { - const cached = readCachedAuthProfileStore({ - authPath, - authMtimeMs, - stateMtimeMs, - }); - if (cached) { - return cached; - } - } const asStore = loadPersistedAuthProfileStore(agentDir, resolvePersistedLoadOptions(options)); if (asStore) { const synced = maybeSyncPersistedExternalCliAuthProfiles({ @@ -841,64 +747,26 @@ function loadAuthProfileStoreForAgent( agentDir, options, }); - if (!readOnly && synced.cacheable) { - writeCachedAuthProfileStore({ - authPath, - authMtimeMs: readAuthStoreMtimeMs(authPath), - stateMtimeMs: readAuthStoreMtimeMs(statePath), - store: synced.store, - }); - } return synced.store; } - const legacy = loadLegacyAuthProfileStore(agentDir); const store: AuthProfileStore = { version: AUTH_STORE_VERSION, profiles: {}, }; - if (legacy) { - applyLegacyAuthStore(store, legacy); - } const mergedOAuth = mergeOAuthFileIntoStore(store); const forceReadOnly = process.env.OPENCLAW_AUTH_STORE_READONLY === "1"; - const shouldWrite = !readOnly && !forceReadOnly && (legacy !== null || mergedOAuth); + const shouldWrite = !readOnly && !forceReadOnly && mergedOAuth; if (shouldWrite) { saveAuthProfileStore(store, agentDir); } - // PR #368: legacy auth.json could get re-migrated from other agent dirs, - // overwriting fresh OAuth creds with stale tokens (fixes #363). Delete only - // after we've successfully written auth-profiles.json. - if (shouldWrite && legacy !== null) { - const legacyPath = resolveLegacyAuthStorePath(agentDir); - try { - fs.unlinkSync(legacyPath); - } catch (err) { - if ((err as NodeJS.ErrnoException)?.code !== "ENOENT") { - log.warn("failed to delete legacy auth.json after migration", { - err, - legacyPath, - }); - } - } - } - const synced = maybeSyncPersistedExternalCliAuthProfiles({ store, agentDir, options, }); - - if (!readOnly && synced.cacheable) { - writeCachedAuthProfileStore({ - authPath, - authMtimeMs: readAuthStoreMtimeMs(authPath), - stateMtimeMs: readAuthStoreMtimeMs(statePath), - store: synced.store, - }); - } return synced.store; } @@ -1091,7 +959,7 @@ export function ensureAuthProfileStoreForLocalUpdate(agentDir?: string): AuthPro }); } -export { hasAnyAuthProfileStoreSource } from "./source-check.js"; +export { hasAnyAuthProfileStoreSource, hasLocalAuthProfileStoreSource } from "./source-check.js"; export function getRuntimeAuthProfileStoreSnapshot( agentDir?: string, @@ -1107,34 +975,32 @@ export function replaceRuntimeAuthProfileStoreSnapshots( export function clearRuntimeAuthProfileStoreSnapshots(): void { clearRuntimeAuthProfileStoreSnapshotsImpl(); - clearLoadedAuthStoreCache(); } export function saveAuthProfileStore( store: AuthProfileStore, agentDir?: string, options?: SaveAuthProfileStoreOptions, + database?: OpenClawAgentDatabase, ): void { - const authPath = resolveAuthStorePath(agentDir); - const statePath = resolveAuthStatePath(agentDir); const localStore = buildLocalAuthProfileStoreForSave({ store, agentDir, options }); - const existingRaw = loadJsonFile(authPath); + const existingRaw = readPersistedAuthProfileStoreRaw(agentDir, database); const payload = preserveLegacyOAuthRefsOnSave({ payload: buildPersistedAuthProfileSecretsStore(localStore), existingRaw, }); - if (isDeepStrictEqual(existingRaw, payload)) { - repairJsonFilePermissions(authPath); - } else { - saveJsonFile(authPath, payload); + if (!isDeepStrictEqual(existingRaw, payload)) { + writePersistedAuthProfileStoreRaw(payload, agentDir, database); + } + if (database) { + writePersistedAuthProfileStateRaw( + buildPersistedAuthProfileState(localStore), + agentDir, + database, + ); + } else { + savePersistedAuthProfileState(localStore, agentDir); } - savePersistedAuthProfileState(localStore, agentDir); - writeCachedAuthProfileStore({ - authPath, - authMtimeMs: readAuthStoreMtimeMs(authPath), - stateMtimeMs: readAuthStoreMtimeMs(statePath), - store: localStore, - }); if (hasRuntimeAuthProfileStoreSnapshot(agentDir)) { const existingRuntimeStore = getRuntimeAuthProfileStoreSnapshot(agentDir); const nextRuntimeStore = buildRuntimeAuthProfileStoreForSave({ store, agentDir, options }); diff --git a/src/agents/auth-profiles/upsert-with-lock.ts b/src/agents/auth-profiles/upsert-with-lock.ts index 76e5cd5fa32f..d487389db3f3 100644 --- a/src/agents/auth-profiles/upsert-with-lock.ts +++ b/src/agents/auth-profiles/upsert-with-lock.ts @@ -1,5 +1,4 @@ import { normalizeSecretInput } from "../../utils/normalize-secret-input.js"; -import { ensureAuthStoreFile, resolveAuthStorePath } from "./paths.js"; import { updateAuthProfileStoreWithLock } from "./store.js"; import type { AuthProfileCredential, AuthProfileStore } from "./types.js"; @@ -31,9 +30,6 @@ export async function upsertAuthProfileWithLock(params: { credential: AuthProfileCredential; agentDir?: string; }): Promise { - const authPath = resolveAuthStorePath(params.agentDir); - ensureAuthStoreFile(authPath); - try { const credential = normalizeAuthProfileCredential(params.credential); return await updateAuthProfileStoreWithLock({ diff --git a/src/agents/command/attempt-execution.cli.test.ts b/src/agents/command/attempt-execution.cli.test.ts index ddf98a1a57fd..8d3f5c9b8e29 100644 --- a/src/agents/command/attempt-execution.cli.test.ts +++ b/src/agents/command/attempt-execution.cli.test.ts @@ -5,6 +5,7 @@ import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; import type { SessionEntry } from "../../config/sessions.js"; import { appendSessionTranscriptMessage } from "../../config/sessions/transcript-append.js"; import type { OpenClawConfig } from "../../config/types.openclaw.js"; +import { saveAuthProfileStore } from "../auth-profiles/store.js"; import type { EmbeddedAgentRunResult } from "../embedded-agent.js"; import { FailoverError } from "../failover-error.js"; import { persistCliTurnTranscript, runAgentAttempt } from "./attempt-execution.js"; @@ -1320,23 +1321,19 @@ describe("CLI attempt execution", () => { }; const sessionStore: Record = { [sessionKey]: sessionEntry }; await fs.writeFile(storePath, JSON.stringify(sessionStore, null, 2), "utf-8"); - await fs.writeFile( - path.join(tmpDir, "auth-profiles.json"), - JSON.stringify( - { - version: 1, - profiles: { - "openai:backup": { - type: "api_key", - provider: "openai", - key: "sk-test", - }, + saveAuthProfileStore( + { + version: 1, + profiles: { + "openai:backup": { + type: "api_key", + provider: "openai", + key: "sk-test", }, }, - null, - 2, - ), - "utf-8", + }, + tmpDir, + { filterExternalAuthProfiles: false, syncExternalCli: false }, ); clearAgentHarnesses(); registerAgentHarness({ @@ -1762,9 +1759,8 @@ describe("embedded attempt harness pinning", () => { sessionId: "codex-auth-session", updatedAt: Date.now(), }; - await fs.writeFile( - path.join(tmpDir, "auth-profiles.json"), - JSON.stringify({ + saveAuthProfileStore( + { version: 1, profiles: { "openai:work": { @@ -1775,7 +1771,9 @@ describe("embedded attempt harness pinning", () => { expires: Date.now() + 60_000, }, }, - }), + }, + tmpDir, + { filterExternalAuthProfiles: false, syncExternalCli: false }, ); runEmbeddedAgentMock.mockResolvedValueOnce({ meta: { durationMs: 1 }, diff --git a/src/agents/embedded-agent-runner/model-discovery-cache.ts b/src/agents/embedded-agent-runner/model-discovery-cache.ts index 4f9e31432a06..6885ac19185f 100644 --- a/src/agents/embedded-agent-runner/model-discovery-cache.ts +++ b/src/agents/embedded-agent-runner/model-discovery-cache.ts @@ -48,8 +48,8 @@ function normalizeCacheDir(dirname: string | undefined): string | undefined { function authFingerprint(agentDir: string): object { return { - authJson: fileFingerprint(path.join(agentDir, "auth.json")), - authProfilesJson: fileFingerprint(path.join(agentDir, "auth-profiles.json")), + authProfilesSqlite: fileFingerprint(path.join(agentDir, "openclaw-agent.sqlite")), + authProfilesSqliteWal: fileFingerprint(path.join(agentDir, "openclaw-agent.sqlite-wal")), }; } diff --git a/src/agents/embedded-agent-runner/model.test.ts b/src/agents/embedded-agent-runner/model.test.ts index ff90830e6adb..083d4f6c481f 100644 --- a/src/agents/embedded-agent-runner/model.test.ts +++ b/src/agents/embedded-agent-runner/model.test.ts @@ -7,6 +7,7 @@ import { discoverAuthStorage, discoverModels } from "../agent-model-discovery.js import { clearRuntimeAuthProfileStoreSnapshots, replaceRuntimeAuthProfileStoreSnapshots, + saveAuthProfileStore, } from "../auth-profiles.js"; import { PLUGIN_MODEL_CATALOG_FILE, @@ -355,9 +356,13 @@ describe("resolveModel", () => { const first = await resolveModelAsync("openai", "gpt-5.5", agentDir, cfg, { runtimeHooks: createRuntimeHooks(), }); - fs.writeFileSync( - path.join(defaultAgentDir, "auth-profiles.json"), - JSON.stringify({ version: 1, profiles: { openai: { type: "api_key", key: "one" } } }), + saveAuthProfileStore( + { + version: 1, + profiles: { "openai:default": { type: "api_key", provider: "openai", key: "one" } }, + }, + defaultAgentDir, + { filterExternalAuthProfiles: false, syncExternalCli: false }, ); const second = await resolveModelAsync("openai", "gpt-5.5", agentDir, cfg, { runtimeHooks: createRuntimeHooks(), @@ -419,9 +424,13 @@ describe("resolveModel", () => { const first = await resolveModelAsync("openai", "gpt-5.5", agentDir, undefined, { runtimeHooks: createRuntimeHooks(), }); - fs.writeFileSync( - path.join(mainAgentDir, "auth-profiles.json"), - JSON.stringify({ version: 1, profiles: { openai: { type: "api_key", key: "one" } } }), + saveAuthProfileStore( + { + version: 1, + profiles: { "openai:default": { type: "api_key", provider: "openai", key: "one" } }, + }, + mainAgentDir, + { filterExternalAuthProfiles: false, syncExternalCli: false }, ); const second = await resolveModelAsync("openai", "gpt-5.5", agentDir, undefined, { runtimeHooks: createRuntimeHooks(), diff --git a/src/agents/models-config.applies-config-env-vars.test.ts b/src/agents/models-config.applies-config-env-vars.test.ts index 309a952da484..d4def8328c5e 100644 --- a/src/agents/models-config.applies-config-env-vars.test.ts +++ b/src/agents/models-config.applies-config-env-vars.test.ts @@ -5,6 +5,7 @@ import { beforeAll, describe, expect, it } from "vitest"; import type { OpenClawConfig } from "../config/config.js"; import { createConfigRuntimeEnv } from "../config/env-vars.js"; import type { PluginMetadataSnapshot } from "../plugins/plugin-metadata-snapshot.js"; +import { saveAuthProfileStore } from "./auth-profiles/store.js"; import { unsetEnv, withTempEnv } from "./models-config.e2e-harness.js"; import { planOpenClawModelsJsonWithDeps, @@ -427,22 +428,19 @@ describe("models-config", () => { it("keeps google-vertex static catalog rows when an auth profile supplies the API key", async () => { const agentDir = await fs.mkdtemp(path.join(os.tmpdir(), "openclaw-google-vertex-models-")); try { - await fs.writeFile( - path.join(agentDir, "auth-profiles.json"), - `${JSON.stringify( - { - version: 1, - profiles: { - "google-vertex:default": { - type: "api_key", - provider: "google-vertex", - keyRef: { source: "env", provider: "default", id: "GOOGLE_CLOUD_API_KEY" }, - }, + saveAuthProfileStore( + { + version: 1, + profiles: { + "google-vertex:default": { + type: "api_key", + provider: "google-vertex", + keyRef: { source: "env", provider: "default", id: "GOOGLE_CLOUD_API_KEY" }, }, }, - null, - 2, - )}\n`, + }, + agentDir, + { filterExternalAuthProfiles: false, syncExternalCli: false }, ); const plan = await planOpenClawModelsJsonWithDeps( diff --git a/src/agents/models-config.ts b/src/agents/models-config.ts index 2863e308b913..3df10456c037 100644 --- a/src/agents/models-config.ts +++ b/src/agents/models-config.ts @@ -18,6 +18,7 @@ import { resolveDefaultAgentDir, resolveDefaultAgentId, } from "./agent-scope.js"; +import { resolveAuthProfileDatabasePath } from "./auth-profiles/sqlite.js"; import { MODELS_JSON_STATE } from "./models-config-state.js"; import { planOpenClawModelsJson } from "./models-config.plan.js"; import { @@ -62,9 +63,9 @@ async function buildModelsJsonFingerprint(params: { providerDiscoveryTimeoutMs?: number; providerDiscoveryEntriesOnly?: boolean; }): Promise { - const authProfilesMtimeMs = await readFileMtimeMs( - path.join(params.agentDir, "auth-profiles.json"), - ); + const authProfilesSqlitePath = resolveAuthProfileDatabasePath(params.agentDir); + const authProfilesMtimeMs = await readFileMtimeMs(authProfilesSqlitePath); + const authProfilesWalMtimeMs = await readFileMtimeMs(`${authProfilesSqlitePath}-wal`); const modelsFileMtimeMs = await readFileMtimeMs(path.join(params.agentDir, "models.json")); const pluginCatalogMtimes = await readPluginCatalogMtimes(params.agentDir); const envShape = createConfigRuntimeEnv(params.config, {}); @@ -76,6 +77,7 @@ async function buildModelsJsonFingerprint(params: { sourceConfigForSecrets: params.sourceConfigForSecrets, envShape, authProfilesMtimeMs, + authProfilesWalMtimeMs, modelsFileMtimeMs, pluginCatalogMtimes, workspaceDir: params.workspaceDir, diff --git a/src/auto-reply/reply/commands-status.test.ts b/src/auto-reply/reply/commands-status.test.ts index e3db6993e5ae..077857dd32f8 100644 --- a/src/auto-reply/reply/commands-status.test.ts +++ b/src/auto-reply/reply/commands-status.test.ts @@ -4,6 +4,7 @@ import path from "node:path"; import { withTempHome } from "openclaw/plugin-sdk/test-env"; import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; import { normalizeTestText } from "../../../test/helpers/normalize-text.js"; +import { saveAuthProfileStore } from "../../agents/auth-profiles/store.js"; import { testing as cliBackendsTesting } from "../../agents/cli-backends.js"; import { clearAgentHarnesses, registerAgentHarness } from "../../agents/harness/registry.js"; import type { AgentHarness } from "../../agents/harness/types.js"; @@ -627,18 +628,10 @@ describe("buildStatusReply subagent summary", () => { await withTempHome( async (dir) => { - const authPath = path.join( - dir, - ".openclaw", - "agents", - "main", - "agent", - "auth-profiles.json", - ); - fs.mkdirSync(path.dirname(authPath), { recursive: true }); - fs.writeFileSync( - authPath, - JSON.stringify({ + const agentDir = path.join(dir, ".openclaw", "agents", "main", "agent"); + fs.mkdirSync(agentDir, { recursive: true }); + saveAuthProfileStore( + { version: 1, profiles: { "openai:status": { @@ -649,8 +642,9 @@ describe("buildStatusReply subagent summary", () => { expires: Date.now() + 60 * 60_000, }, }, - }), - "utf8", + }, + agentDir, + { filterExternalAuthProfiles: false, syncExternalCli: false }, ); const usageResetBase = Math.floor(Date.now() / 1000); providerUsageMock.loadProviderUsageSummary.mockResolvedValue({ @@ -744,18 +738,10 @@ describe("buildStatusReply subagent summary", () => { await withTempHome( async (dir) => { - const authPath = path.join( - dir, - ".openclaw", - "agents", - "main", - "agent", - "auth-profiles.json", - ); - fs.mkdirSync(path.dirname(authPath), { recursive: true }); - fs.writeFileSync( - authPath, - JSON.stringify({ + const agentDir = path.join(dir, ".openclaw", "agents", "main", "agent"); + fs.mkdirSync(agentDir, { recursive: true }); + saveAuthProfileStore( + { version: 1, profiles: { "openai:status": { @@ -766,8 +752,9 @@ describe("buildStatusReply subagent summary", () => { expires: Date.now() + 60 * 60_000, }, }, - }), - "utf8", + }, + agentDir, + { filterExternalAuthProfiles: false, syncExternalCli: false }, ); const usageResetBase = Math.floor(Date.now() / 1000); providerUsageMock.loadProviderUsageSummary.mockResolvedValue({ @@ -881,18 +868,10 @@ describe("buildStatusReply subagent summary", () => { it("uses Codex OAuth auth labels for explicit OpenAI OpenClaw auth order", async () => { await withTempHome( async (dir) => { - const authPath = path.join( - dir, - ".openclaw", - "agents", - "main", - "agent", - "auth-profiles.json", - ); - fs.mkdirSync(path.dirname(authPath), { recursive: true }); - fs.writeFileSync( - authPath, - JSON.stringify({ + const agentDir = path.join(dir, ".openclaw", "agents", "main", "agent"); + fs.mkdirSync(agentDir, { recursive: true }); + saveAuthProfileStore( + { version: 1, profiles: { "openai:status": { @@ -908,8 +887,9 @@ describe("buildStatusReply subagent summary", () => { key: "sk-test", }, }, - }), - "utf8", + }, + agentDir, + { filterExternalAuthProfiles: false, syncExternalCli: false }, ); const text = await buildStatusText({ diff --git a/src/commands/agents.add.test.ts b/src/commands/agents.add.test.ts index 1efa6d4a9115..37709aca2630 100644 --- a/src/commands/agents.add.test.ts +++ b/src/commands/agents.add.test.ts @@ -3,8 +3,11 @@ import os from "node:os"; import path from "node:path"; import { beforeEach, describe, expect, it, vi } from "vitest"; import { AUTH_STORE_VERSION } from "../agents/auth-profiles/constants.js"; +import { loadPersistedAuthProfileStore } from "../agents/auth-profiles/persisted.js"; import { saveAuthProfileStore } from "../agents/auth-profiles/store.js"; import { formatCliCommand } from "../cli/command-format.js"; +import { closeOpenClawAgentDatabasesForTest } from "../state/openclaw-agent-db.js"; +import { closeOpenClawStateDatabaseForTest } from "../state/openclaw-state-db.js"; import { baseConfigSnapshot, createTestRuntime } from "./test-runtime-config-helpers.js"; const readConfigFileSnapshotMock = vi.hoisted(() => vi.fn()); @@ -200,56 +203,57 @@ describe("agents add command", () => { it("copies only portable auth profiles when seeding a new agent store", async () => { const root = await fs.mkdtemp(path.join(os.tmpdir(), "openclaw-agents-add-auth-copy-")); + const previousStateDir = process.env.OPENCLAW_STATE_DIR; + process.env.OPENCLAW_STATE_DIR = root; try { const sourceAgentDir = path.join(root, "main", "agent"); const destAgentDir = path.join(root, "work", "agent"); - const destAuthPath = path.join(destAgentDir, "auth-profiles.json"); await fs.mkdir(sourceAgentDir, { recursive: true }); - await fs.writeFile( - path.join(sourceAgentDir, "auth-profiles.json"), - `${JSON.stringify( - { - version: AUTH_STORE_VERSION, - profiles: { - "openai:default": { - type: "api_key", - provider: "openai", - key: "sk-test", - }, - "github-copilot:default": { - type: "token", - provider: "github-copilot", - token: "gho-test", - }, - "openai:oauth": { - type: "oauth", - provider: "openai", - access: "codex-access", - refresh: "codex-refresh", - expires: Date.now() + 60_000, - }, + saveAuthProfileStore( + { + version: AUTH_STORE_VERSION, + profiles: { + "openai:default": { + type: "api_key", + provider: "openai", + key: "sk-test", + }, + "github-copilot:default": { + type: "token", + provider: "github-copilot", + token: "gho-test", + }, + "openai:oauth": { + type: "oauth", + provider: "openai", + access: "codex-access", + refresh: "codex-refresh", + expires: Date.now() + 60_000, }, }, - null, - 2, - )}\n`, - "utf8", + }, + sourceAgentDir, ); const result = await testing.copyPortableAuthProfiles({ sourceAgentDir, - destAuthPath, + destAgentDir, }); expect(result).toEqual({ copied: 2, skipped: 1 }); - const copied = JSON.parse(await fs.readFile(destAuthPath, "utf8")) as { - profiles: Record; - }; - expect(Object.keys(copied.profiles).toSorted()).toEqual([ + const copied = loadPersistedAuthProfileStore(destAgentDir); + expect(Object.keys(copied?.profiles ?? {}).toSorted()).toEqual([ "github-copilot:default", "openai:default", ]); } finally { + closeOpenClawAgentDatabasesForTest(); + closeOpenClawStateDatabaseForTest(); + if (previousStateDir === undefined) { + delete process.env.OPENCLAW_STATE_DIR; + } else { + process.env.OPENCLAW_STATE_DIR = previousStateDir; + } await fs.rm(root, { recursive: true, force: true }); } }); @@ -261,7 +265,6 @@ describe("agents add command", () => { try { const sourceAgentDir = path.join(root, "main", "agent"); const destAgentDir = path.join(root, "work", "agent"); - const destAuthPath = path.join(destAgentDir, "auth-profiles.json"); const expires = Date.now() + 60_000; await fs.mkdir(sourceAgentDir, { recursive: true }); saveAuthProfileStore( @@ -283,17 +286,12 @@ describe("agents add command", () => { const result = await testing.copyPortableAuthProfiles({ sourceAgentDir, - destAuthPath, + destAgentDir, }); expect(result).toEqual({ copied: 1, skipped: 0 }); - const copiedRaw = await fs.readFile(destAuthPath, "utf8"); - expect(copiedRaw).toContain("codex-copy-access-token"); - expect(copiedRaw).toContain("codex-copy-refresh-token"); - const copied = JSON.parse(copiedRaw) as { - profiles: Record>; - }; - const credential = copied.profiles["openai:oauth"]; + const copied = loadPersistedAuthProfileStore(destAgentDir); + const credential = copied?.profiles["openai:oauth"]; expect(credential).toStrictEqual({ type: "oauth", provider: "openai", @@ -303,6 +301,8 @@ describe("agents add command", () => { copyToAgents: true, }); } finally { + closeOpenClawAgentDatabasesForTest(); + closeOpenClawStateDatabaseForTest(); if (previousStateDir === undefined) { delete process.env.OPENCLAW_STATE_DIR; } else { @@ -314,10 +314,11 @@ describe("agents add command", () => { it("skips unresolved OAuth profiles when seeding a new agent store", async () => { const root = await fs.mkdtemp(path.join(os.tmpdir(), "openclaw-agents-add-oauth-ref-skip-")); + const previousStateDir = process.env.OPENCLAW_STATE_DIR; + process.env.OPENCLAW_STATE_DIR = root; try { const sourceAgentDir = path.join(root, "main", "agent"); const destAgentDir = path.join(root, "work", "agent"); - const destAuthPath = path.join(destAgentDir, "auth-profiles.json"); const profileId = "openai:oauth"; const ref = { source: "openclaw-credentials" as const, @@ -325,34 +326,36 @@ describe("agents add command", () => { id: "0123456789abcdef0123456789abcdef", }; await fs.mkdir(sourceAgentDir, { recursive: true }); - await fs.writeFile( - path.join(sourceAgentDir, "auth-profiles.json"), - `${JSON.stringify( - { - version: AUTH_STORE_VERSION, - profiles: { - [profileId]: { - type: "oauth", - provider: "openai", - copyToAgents: true, - expires: Date.now() + 60_000, - oauthRef: ref, - }, + saveAuthProfileStore( + { + version: AUTH_STORE_VERSION, + profiles: { + [profileId]: { + type: "oauth", + provider: "openai", + copyToAgents: true, + expires: Date.now() + 60_000, + oauthRef: ref, }, }, - null, - 2, - )}\n`, - "utf8", + } as never, + sourceAgentDir, ); const result = await testing.copyPortableAuthProfiles({ sourceAgentDir, - destAuthPath, + destAgentDir, }); expect(result).toEqual({ copied: 0, skipped: 1 }); - await expect(fs.stat(destAuthPath)).rejects.toMatchObject({ code: "ENOENT" }); + expect(loadPersistedAuthProfileStore(destAgentDir)).toBeNull(); } finally { + closeOpenClawAgentDatabasesForTest(); + closeOpenClawStateDatabaseForTest(); + if (previousStateDir === undefined) { + delete process.env.OPENCLAW_STATE_DIR; + } else { + process.env.OPENCLAW_STATE_DIR = previousStateDir; + } await fs.rm(root, { recursive: true, force: true }); } }); diff --git a/src/commands/agents.commands.add.ts b/src/commands/agents.commands.add.ts index b8da5ae8ed84..0e935d69f7ae 100644 --- a/src/commands/agents.commands.add.ts +++ b/src/commands/agents.commands.add.ts @@ -14,18 +14,14 @@ import { ensureAuthProfileStore, } from "../agents/auth-profiles.js"; import { resolveAuthStorePath } from "../agents/auth-profiles/paths.js"; -import { - buildPersistedAuthProfileSecretsStore, - loadPersistedAuthProfileStore, -} from "../agents/auth-profiles/persisted.js"; +import { loadPersistedAuthProfileStore } from "../agents/auth-profiles/persisted.js"; +import { saveAuthProfileStore } from "../agents/auth-profiles/store.js"; import { formatCliCommand } from "../cli/command-format.js"; import { commitConfigWithPendingPluginInstalls, transformConfigWithPendingPluginInstalls, } from "../cli/plugins-install-record-commit.js"; import { logConfigUpdated } from "../config/logging.js"; -import { pathExists } from "../infra/fs-safe.js"; -import { saveJsonFile } from "../infra/json-file.js"; import { DEFAULT_AGENT_ID, normalizeAgentId } from "../routing/session-key.js"; import { type RuntimeEnv, writeRuntimeJson } from "../runtime.js"; import { defaultRuntime } from "../runtime.js"; @@ -75,7 +71,7 @@ function emptyBindingResult(config: Parameters[0]): A } async function copyPortableAuthProfiles(params: { - destAuthPath: string; + destAgentDir: string; sourceAgentDir: string; }): Promise<{ copied: number; skipped: number }> { const sourceStore = loadPersistedAuthProfileStore(params.sourceAgentDir); @@ -86,8 +82,11 @@ async function copyPortableAuthProfiles(params: { if (portable.copiedProfileIds.length === 0) { return { copied: 0, skipped: portable.skippedProfileIds.length }; } - await fs.mkdir(path.dirname(params.destAuthPath), { recursive: true }); - saveJsonFile(params.destAuthPath, buildPersistedAuthProfileSecretsStore(portable.store)); + await fs.mkdir(params.destAgentDir, { recursive: true }); + saveAuthProfileStore(portable.store, params.destAgentDir, { + filterExternalAuthProfiles: false, + syncExternalCli: false, + }); return { copied: portable.copiedProfileIds.length, skipped: portable.skippedProfileIds.length, @@ -332,23 +331,27 @@ export async function agentsAddCommand( const sourceIsInheritedMain = normalizeLowercaseStringOrEmpty(path.resolve(sourceAuthPath)) === normalizeLowercaseStringOrEmpty(path.resolve(mainAuthPath)); - if ( - !sameAuthPath && - (await pathExists(sourceAuthPath)) && - !(await pathExists(destAuthPath)) - ) { + if (!sameAuthPath) { const sourceStore = loadPersistedAuthProfileStore(sourceAgentDir); + const destStore = loadPersistedAuthProfileStore(agentDir); const portable = sourceStore ? buildPortableAuthProfileSecretsStoreForAgentCopy(sourceStore) : undefined; - if (portable && portable.copiedProfileIds.length > 0) { + if ( + portable && + portable.copiedProfileIds.length > 0 && + Object.keys(destStore?.profiles ?? {}).length === 0 + ) { const shouldCopy = await prompter.confirm({ message: `Copy portable auth profiles from "${defaultAgentId}"?`, initialValue: false, }); if (shouldCopy) { - await fs.mkdir(path.dirname(destAuthPath), { recursive: true }); - saveJsonFile(destAuthPath, buildPersistedAuthProfileSecretsStore(portable.store)); + await fs.mkdir(agentDir, { recursive: true }); + saveAuthProfileStore(portable.store, agentDir, { + filterExternalAuthProfiles: false, + syncExternalCli: false, + }); const skippedText = portable.skippedProfileIds.length > 0 ? ` ${formatSkippedOAuthProfilesMessage({ diff --git a/src/commands/doctor-auth-canonical-api-key-alias.test.ts b/src/commands/doctor-auth-canonical-api-key-alias.test.ts index f47458fb8223..66262f85e89d 100644 --- a/src/commands/doctor-auth-canonical-api-key-alias.test.ts +++ b/src/commands/doctor-auth-canonical-api-key-alias.test.ts @@ -42,6 +42,16 @@ async function makeTestState(): Promise { return state; } +async function writeLegacyAuthProfilesJson( + state: OpenClawTestState, + value: unknown, +): Promise { + return await state.writeText( + "agents/main/agent/auth-profiles.json", + `${JSON.stringify(value, null, 2)}\n`, + ); +} + afterEach(async () => { clearRuntimeAuthProfileStoreSnapshots(); for (const state of states.splice(0)) { @@ -65,7 +75,7 @@ describe("maybeRepairCanonicalApiKeyFieldAlias", () => { "my-provider": ["my-key"], }, }; - const authPath = await state.writeAuthProfiles(canonical); + const authPath = await writeLegacyAuthProfilesJson(state, canonical); const result = await maybeRepairCanonicalApiKeyFieldAlias({ cfg: {}, @@ -110,7 +120,7 @@ describe("maybeRepairCanonicalApiKeyFieldAlias", () => { }, }, }; - const authPath = await state.writeAuthProfiles(canonical); + const authPath = await writeLegacyAuthProfilesJson(state, canonical); const result = await maybeRepairCanonicalApiKeyFieldAlias({ cfg: {}, @@ -230,7 +240,7 @@ describe("maybeRepairCanonicalApiKeyFieldAlias", () => { }, }, }; - const authPath = await state.writeAuthProfiles(canonical); + const authPath = await writeLegacyAuthProfilesJson(state, canonical); const result = await maybeRepairCanonicalApiKeyFieldAlias({ cfg: {}, @@ -257,7 +267,7 @@ describe("maybeRepairCanonicalApiKeyFieldAlias", () => { }, }, }; - const authPath = await state.writeAuthProfiles(canonical); + const authPath = await writeLegacyAuthProfilesJson(state, canonical); const result = await maybeRepairCanonicalApiKeyFieldAlias({ cfg: {}, @@ -284,7 +294,7 @@ describe("maybeRepairCanonicalApiKeyFieldAlias", () => { }, }, }; - const authPath = await state.writeAuthProfiles(canonical); + const authPath = await writeLegacyAuthProfilesJson(state, canonical); const result = await maybeRepairCanonicalApiKeyFieldAlias({ cfg: {}, @@ -310,7 +320,7 @@ describe("maybeRepairCanonicalApiKeyFieldAlias", () => { }, }, }; - const authPath = await state.writeAuthProfiles(canonical); + const authPath = await writeLegacyAuthProfilesJson(state, canonical); const result = await maybeRepairCanonicalApiKeyFieldAlias({ cfg: {}, diff --git a/src/commands/doctor-auth-flat-profiles.test.ts b/src/commands/doctor-auth-flat-profiles.test.ts index b518d41e2a75..bdf8f6a348cb 100644 --- a/src/commands/doctor-auth-flat-profiles.test.ts +++ b/src/commands/doctor-auth-flat-profiles.test.ts @@ -1,13 +1,20 @@ import fs from "node:fs"; import { afterEach, describe, expect, it, vi } from "vitest"; -import { clearRuntimeAuthProfileStoreSnapshots } from "../agents/auth-profiles/store.js"; +import { loadPersistedAuthProfileStore } from "../agents/auth-profiles/persisted.js"; +import { + clearRuntimeAuthProfileStoreSnapshots, + saveAuthProfileStore, +} from "../agents/auth-profiles/store.js"; import type { OpenClawConfig } from "../config/types.openclaw.js"; +import { closeOpenClawAgentDatabasesForTest } from "../state/openclaw-agent-db.js"; +import { closeOpenClawStateDatabaseForTest } from "../state/openclaw-state-db.js"; import { createOpenClawTestState, type OpenClawTestState, } from "../test-utils/openclaw-test-state.js"; import { collectOpenAICodexAuthProfileStoreIdMap, + maybeMigrateAuthProfileJsonStoresToSqlite, maybeRepairLegacyFlatAuthProfileStores, maybeRepairOpenAICodexAuthConfig, maybeRepairOpenAICodexAuthProfileStores, @@ -47,15 +54,364 @@ async function makeTestState(): Promise { return state; } +async function writeLegacyAuthProfilesJson( + state: OpenClawTestState, + value: unknown, + agentId = "main", +): Promise { + return await state.writeText( + `agents/${agentId}/agent/auth-profiles.json`, + `${JSON.stringify(value, null, 2)}\n`, + ); +} + afterEach(async () => { clearRuntimeAuthProfileStoreSnapshots(); + closeOpenClawAgentDatabasesForTest(); + closeOpenClawStateDatabaseForTest(); for (const state of states.splice(0)) { await state.cleanup(); } }); +describe("maybeMigrateAuthProfileJsonStoresToSqlite", () => { + it("imports legacy JSON auth profiles and state into the agent sqlite database", async () => { + const state = await makeTestState(); + const authPath = await writeLegacyAuthProfilesJson(state, { + version: 1, + profiles: { + "openai:default": { + type: "api_key", + provider: "openai", + key: "sk-migrated", + }, + }, + }); + const statePath = await state.writeText( + "agents/main/agent/auth-state.json", + `${JSON.stringify({ version: 1, lastGood: { openai: "openai:default" } })}\n`, + ); + + const result = await maybeMigrateAuthProfileJsonStoresToSqlite({ + cfg: {}, + prompter: makePrompter(true), + now: () => 456, + }); + + expect(result.detected.toSorted()).toEqual([authPath, statePath].toSorted()); + expect(result.warnings).toStrictEqual([]); + expect(loadPersistedAuthProfileStore(state.agentDir())).toMatchObject({ + profiles: { + "openai:default": { + type: "api_key", + provider: "openai", + key: "sk-migrated", + }, + }, + lastGood: { openai: "openai:default" }, + }); + expect(fs.existsSync(authPath)).toBe(false); + expect(fs.existsSync(statePath)).toBe(false); + expect(fs.existsSync(`${authPath}.sqlite-import.456.bak`)).toBe(true); + expect(fs.existsSync(`${statePath}.sqlite-import.456.bak`)).toBe(true); + }); + + it("moves legacy aws-sdk auth markers to config before removing JSON", async () => { + const state = await makeTestState(); + const cfg = {}; + const authPath = await writeLegacyAuthProfilesJson(state, { + version: 1, + profiles: { + "amazon-bedrock:default": { + type: "aws-sdk", + provider: "amazon-bedrock", + }, + "openrouter:default": { + type: "api_key", + provider: "openrouter", + key: "sk-openrouter", + }, + }, + }); + + const result = await maybeMigrateAuthProfileJsonStoresToSqlite({ + cfg, + prompter: makePrompter(true), + now: () => 457, + }); + + expect(result.detected).toEqual([authPath]); + expect(result.configChanged).toBe(true); + expect(result.warnings).toStrictEqual([]); + expect(cfg).toEqual({ + auth: { + profiles: { + "amazon-bedrock:default": { + provider: "amazon-bedrock", + mode: "aws-sdk", + }, + }, + }, + }); + expect(loadPersistedAuthProfileStore(state.agentDir())?.profiles).toEqual({ + "openrouter:default": { + type: "api_key", + provider: "openrouter", + key: "sk-openrouter", + }, + }); + expect(fs.existsSync(authPath)).toBe(false); + expect(fs.existsSync(`${authPath}.sqlite-import.457.bak`)).toBe(true); + }); + + it("preserves state-only legacy auth state for inherited profiles", async () => { + const state = await makeTestState(); + const statePath = await state.writeText( + "agents/main/agent/auth-state.json", + `${JSON.stringify({ + version: 1, + order: { openai: ["openai:default"] }, + lastGood: { openai: "openai:default" }, + usageStats: { + "openai:default": { + errorCount: 2, + lastFailureAt: 123, + }, + }, + })}\n`, + ); + const authPath = state.path("agents/main/agent/auth-profiles.json"); + + const result = await maybeMigrateAuthProfileJsonStoresToSqlite({ + cfg: {}, + prompter: makePrompter(true), + now: () => 459, + }); + + expect(result.detected).toEqual([statePath]); + expect(loadPersistedAuthProfileStore(state.agentDir())).toMatchObject({ + profiles: {}, + order: { openai: ["openai:default"] }, + lastGood: { openai: "openai:default" }, + usageStats: { + "openai:default": { + errorCount: 2, + lastFailureAt: 123, + }, + }, + }); + expect(fs.existsSync(statePath)).toBe(false); + expect(fs.existsSync(`${statePath}.sqlite-import.459.bak`)).toBe(true); + expect(fs.existsSync(authPath)).toBe(false); + }); + + it("leaves unresolved legacy OAuth sidecar refs in JSON", async () => { + const state = await makeTestState(); + const authPath = await writeLegacyAuthProfilesJson(state, { + version: 1, + profiles: { + "openai:user@example.com": { + type: "oauth", + provider: "openai", + email: "user@example.com", + oauthRef: { + id: "0123456789abcdef0123456789abcdef", + provider: "openai", + }, + }, + }, + }); + + const result = await maybeMigrateAuthProfileJsonStoresToSqlite({ + cfg: {}, + prompter: makePrompter(true), + now: () => 460, + }); + + expect(result.detected).toEqual([authPath]); + expect(result.changes).toEqual([]); + expect(result.warnings).toEqual([ + expect.stringContaining("legacy OAuth sidecar profile"), + expect.stringContaining("no importable auth profiles or state"), + ]); + expect(loadPersistedAuthProfileStore(state.agentDir())).toBeNull(); + expect(fs.existsSync(authPath)).toBe(true); + expect(fs.existsSync(`${authPath}.sqlite-import.460.bak`)).toBe(false); + }); + + it("imports valid profiles when one legacy OAuth sidecar ref is unresolved", async () => { + const state = await makeTestState(); + const authPath = await writeLegacyAuthProfilesJson(state, { + version: 1, + profiles: { + "openai:default": { + type: "api_key", + provider: "openai", + key: "sk-imported", + }, + "openai:user@example.com": { + type: "oauth", + provider: "openai", + email: "user@example.com", + oauthRef: { + id: "0123456789abcdef0123456789abcdef", + provider: "openai", + }, + }, + }, + order: { openai: ["openai:default", "openai:user@example.com"] }, + lastGood: { openai: "openai:default" }, + }); + + const result = await maybeMigrateAuthProfileJsonStoresToSqlite({ + cfg: {}, + prompter: makePrompter(true), + now: () => 463, + }); + + expect(result.changes).toEqual([expect.stringContaining("Migrated auth profile JSON")]); + expect(result.warnings).toEqual([expect.stringContaining("legacy OAuth sidecar profile")]); + expect(loadPersistedAuthProfileStore(state.agentDir())).toMatchObject({ + profiles: { + "openai:default": { + type: "api_key", + provider: "openai", + key: "sk-imported", + }, + }, + order: { openai: ["openai:default"] }, + lastGood: { openai: "openai:default" }, + }); + expect(fs.existsSync(authPath)).toBe(true); + expect(fs.existsSync(`${authPath}.sqlite-import.463.bak`)).toBe(true); + const remaining = JSON.parse(fs.readFileSync(authPath, "utf8")); + expect(remaining.profiles).toEqual({ + "openai:user@example.com": { + type: "oauth", + provider: "openai", + email: "user@example.com", + oauthRef: { + id: "0123456789abcdef0123456789abcdef", + provider: "openai", + }, + }, + }); + expect(remaining.order).toEqual({ openai: ["openai:user@example.com"] }); + expect(remaining.lastGood).toBeUndefined(); + }); + + it("keeps existing SQLite credentials when importing stale JSON", async () => { + const state = await makeTestState(); + saveAuthProfileStore( + { + version: 1, + profiles: { + "openai:default": { + type: "api_key", + provider: "openai", + key: "sk-fresh-sqlite", + }, + }, + }, + state.agentDir(), + { syncExternalCli: false }, + ); + const authPath = await writeLegacyAuthProfilesJson(state, { + version: 1, + profiles: { + "openai:default": { + type: "api_key", + provider: "openai", + key: "sk-stale-json", + }, + }, + }); + + const result = await maybeMigrateAuthProfileJsonStoresToSqlite({ + cfg: {}, + prompter: makePrompter(true), + now: () => 461, + }); + + expect(result.detected).toEqual([authPath]); + expect(loadPersistedAuthProfileStore(state.agentDir())?.profiles["openai:default"]).toEqual({ + type: "api_key", + provider: "openai", + key: "sk-fresh-sqlite", + }); + expect(fs.existsSync(authPath)).toBe(false); + expect(fs.existsSync(`${authPath}.sqlite-import.461.bak`)).toBe(true); + }); + + it("keeps auth-state.json precedence over auth-profiles.json state during import", async () => { + const state = await makeTestState(); + const authPath = await writeLegacyAuthProfilesJson(state, { + version: 1, + profiles: { + "openai:default": { + type: "api_key", + provider: "openai", + key: "sk-openai", + }, + "openai:work": { + type: "api_key", + provider: "openai", + key: "sk-work", + }, + }, + order: { openai: ["openai:default"] }, + }); + const statePath = await state.writeText( + "agents/main/agent/auth-state.json", + `${JSON.stringify({ version: 1, order: { openai: ["openai:work"] } })}\n`, + ); + + await maybeMigrateAuthProfileJsonStoresToSqlite({ + cfg: {}, + prompter: makePrompter(true), + now: () => 462, + }); + + expect(loadPersistedAuthProfileStore(state.agentDir())?.order).toEqual({ + openai: ["openai:work"], + }); + expect(fs.existsSync(authPath)).toBe(false); + expect(fs.existsSync(statePath)).toBe(false); + }); + + it("imports legacy api_key alias fields before removing JSON", async () => { + const state = await makeTestState(); + const authPath = await writeLegacyAuthProfilesJson(state, { + version: 1, + profiles: { + "openrouter:default": { + type: "api_key", + provider: "openrouter", + api_key: "sk-openrouter-legacy", + }, + }, + }); + + await maybeMigrateAuthProfileJsonStoresToSqlite({ + cfg: {}, + prompter: makePrompter(true), + now: () => 458, + }); + + expect(loadPersistedAuthProfileStore(state.agentDir())?.profiles).toEqual({ + "openrouter:default": { + type: "api_key", + provider: "openrouter", + key: "sk-openrouter-legacy", + }, + }); + expect(fs.existsSync(authPath)).toBe(false); + expect(fs.existsSync(`${authPath}.sqlite-import.458.bak`)).toBe(true); + }); +}); + describe("maybeRepairLegacyFlatAuthProfileStores", () => { - it("rewrites legacy flat auth-profiles.json stores with a backup", async () => { + it("migrates legacy flat auth-profiles.json stores with a backup", async () => { const state = await makeTestState(); const legacy = { "ollama-windows": { @@ -63,7 +419,7 @@ describe("maybeRepairLegacyFlatAuthProfileStores", () => { baseUrl: "http://10.0.2.2:11434/v1", }, }; - const authPath = await state.writeAuthProfiles(legacy); + const authPath = await writeLegacyAuthProfilesJson(state, legacy); const result = await maybeRepairLegacyFlatAuthProfileStores({ cfg: {}, @@ -73,10 +429,10 @@ describe("maybeRepairLegacyFlatAuthProfileStores", () => { expect(result.detected).toEqual([authPath]); expect(result.changes).toStrictEqual([ - `Rewrote ${authPath} to the canonical auth profile format (backup: ${authPath}.legacy-flat.123.bak).`, + `Migrated ${authPath} to the SQLite auth profile store (backup: ${authPath}.legacy-flat.123.bak).`, ]); expect(result.warnings).toStrictEqual([]); - expect(JSON.parse(fs.readFileSync(authPath, "utf8"))).toEqual({ + expect(loadPersistedAuthProfileStore(state.agentDir())).toEqual({ version: 1, profiles: { "ollama-windows:default": { @@ -86,6 +442,7 @@ describe("maybeRepairLegacyFlatAuthProfileStores", () => { }, }, }); + expect(fs.existsSync(authPath)).toBe(false); expect(JSON.parse(fs.readFileSync(`${authPath}.legacy-flat.123.bak`, "utf8"))).toEqual(legacy); }); @@ -96,7 +453,7 @@ describe("maybeRepairLegacyFlatAuthProfileStores", () => { apiKey: "sk-openai", }, }; - const authPath = await state.writeAuthProfiles(legacy); + const authPath = await writeLegacyAuthProfilesJson(state, legacy); const result = await maybeRepairLegacyFlatAuthProfileStores({ cfg: {}, @@ -125,7 +482,7 @@ describe("maybeRepairLegacyFlatAuthProfileStores", () => { }, }, }; - const authPath = await state.writeAuthProfiles(legacy); + const authPath = await writeLegacyAuthProfilesJson(state, legacy); const cfg = {}; const result = await maybeRepairLegacyFlatAuthProfileStores({ @@ -481,7 +838,7 @@ describe("maybeRepairOpenAICodexAuthConfig", () => { describe("maybeRepairOpenAICodexAuthProfileStores", () => { it("collects the store-derived legacy OpenAI Codex profile id map", async () => { const state = await makeTestState(); - await state.writeAuthProfiles({ + await writeLegacyAuthProfilesJson(state, { version: 1, profiles: { "openai:default": { @@ -535,7 +892,7 @@ describe("maybeRepairOpenAICodexAuthProfileStores", () => { }, }, }; - const authPath = await state.writeAuthProfiles(legacy); + const authPath = await writeLegacyAuthProfilesJson(state, legacy); const result = await maybeRepairOpenAICodexAuthProfileStores({ cfg: {}, diff --git a/src/commands/doctor-auth-flat-profiles.ts b/src/commands/doctor-auth-flat-profiles.ts index d56341d5ead1..fd2c3d3a2977 100644 --- a/src/commands/doctor-auth-flat-profiles.ts +++ b/src/commands/doctor-auth-flat-profiles.ts @@ -4,12 +4,27 @@ import { isRecord } from "@openclaw/normalization-core/record-coerce"; import { note } from "../../packages/terminal-core/src/note.js"; import { resolveAgentDir, resolveDefaultAgentDir, listAgentIds } from "../agents/agent-scope.js"; import { AUTH_STORE_VERSION } from "../agents/auth-profiles/constants.js"; -import { resolveAuthStorePath } from "../agents/auth-profiles/paths.js"; +import { + resolveAuthStatePath, + resolveAuthStorePath, + resolveLegacyAuthStorePath, +} from "../agents/auth-profiles/paths.js"; +import { + applyLegacyAuthStore, + coercePersistedAuthProfileStore, + loadLegacyAuthProfileStore, + loadPersistedAuthProfileStore, +} from "../agents/auth-profiles/persisted.js"; +import { coerceAuthProfileState } from "../agents/auth-profiles/state.js"; import { clearRuntimeAuthProfileStoreSnapshots, saveAuthProfileStore, } from "../agents/auth-profiles/store.js"; -import type { AuthProfileCredential, AuthProfileStore } from "../agents/auth-profiles/types.js"; +import type { + AuthProfileCredential, + AuthProfileState, + AuthProfileStore, +} from "../agents/auth-profiles/types.js"; import { formatCliCommand } from "../cli/command-format.js"; import { resolveStateDir } from "../config/paths.js"; import type { AuthProfileConfig } from "../config/types.auth.js"; @@ -30,6 +45,11 @@ type LegacyFlatAuthProfileStore = { store: AuthProfileStore; }; +type AuthProfileSqliteMigrationCandidate = AuthProfileRepairCandidate & { + statePath: string; + legacyPath: string; +}; + type AwsSdkProfileMarker = { profileId: string; provider: string; @@ -47,6 +67,7 @@ type AwsSdkAuthProfileMarkerStore = { export type LegacyFlatAuthProfileRepairResult = { detected: string[]; changes: string[]; + configChanged?: boolean; warnings: string[]; }; @@ -214,6 +235,433 @@ function listAuthProfileRepairCandidates( return [...candidates.values()]; } +function listAuthProfileSqliteMigrationCandidates( + cfg: OpenClawConfig, + env: NodeJS.ProcessEnv, +): AuthProfileSqliteMigrationCandidate[] { + const candidates: AuthProfileSqliteMigrationCandidate[] = []; + for (const candidate of listAuthProfileRepairCandidates(cfg, env)) { + candidates.push({ + agentDir: candidate.agentDir, + authPath: candidate.authPath, + statePath: resolveAuthStatePath(candidate.agentDir), + legacyPath: resolveLegacyAuthStorePath(candidate.agentDir), + }); + } + return candidates; +} + +function hasAuthProfileState(state: AuthProfileState): boolean { + return Boolean(state.order || state.lastGood || state.usageStats); +} + +function normalizeLegacyApiKeyAliasesForImport(raw: unknown): void { + if (!isRecord(raw) || !isRecord(raw.profiles)) { + return; + } + for (const profile of Object.values(raw.profiles)) { + if (!isRecord(profile)) { + continue; + } + const type = readNonEmptyString(profile.type) ?? readNonEmptyString(profile.mode); + if (type !== "api_key") { + continue; + } + const hasCanonicalCredential = + readNonEmptyString(profile.key) !== undefined || coerceSecretRef(profile.keyRef) !== null; + if (hasCanonicalCredential || profile["api_key"] === undefined) { + continue; + } + profile.key = profile["api_key"]; + } +} + +function collectAuthProfileStateProfileIds(state: AuthProfileState): string[] { + const profileIds = new Set(); + for (const entries of Object.values(state.order ?? {})) { + for (const profileId of entries) { + profileIds.add(profileId); + } + } + for (const profileId of Object.values(state.lastGood ?? {})) { + profileIds.add(profileId); + } + for (const profileId of Object.keys(state.usageStats ?? {})) { + profileIds.add(profileId); + } + return [...profileIds]; +} + +function mergeImportedAuthProfiles(params: { + store: AuthProfileStore; + profiles: AuthProfileStore["profiles"]; + existingProfileIds: ReadonlySet; +}): AuthProfileStore { + const profiles = { ...params.store.profiles }; + for (const [profileId, credential] of Object.entries(params.profiles)) { + if (!params.existingProfileIds.has(profileId)) { + profiles[profileId] = credential; + } + } + return { ...params.store, profiles }; +} + +function mergeImportedAuthProfileState(params: { + store: AuthProfileStore; + state: AuthProfileState; + existingState: AuthProfileState; +}): AuthProfileStore { + return { + ...params.store, + ...(params.state.order + ? { + order: { + ...params.store.order, + ...Object.fromEntries( + Object.entries(params.state.order).filter( + ([provider]) => !params.existingState.order?.[provider], + ), + ), + }, + } + : {}), + ...(params.state.lastGood + ? { + lastGood: { + ...params.store.lastGood, + ...Object.fromEntries( + Object.entries(params.state.lastGood).filter( + ([provider]) => !params.existingState.lastGood?.[provider], + ), + ), + }, + } + : {}), + ...(params.state.usageStats + ? { + usageStats: { + ...params.store.usageStats, + ...Object.fromEntries( + Object.entries(params.state.usageStats).filter( + ([profileId]) => !params.existingState.usageStats?.[profileId], + ), + ), + }, + } + : {}), + }; +} + +function filterRawAuthProfileState( + raw: Record, + shouldKeepProfileId: (profileId: string) => boolean, +): void { + if (isRecord(raw.order)) { + for (const [provider, profileIds] of Object.entries(raw.order)) { + if (!Array.isArray(profileIds)) { + continue; + } + const kept = profileIds.filter( + (profileId): profileId is string => + typeof profileId === "string" && shouldKeepProfileId(profileId), + ); + if (kept.length > 0) { + raw.order[provider] = kept; + } else { + delete raw.order[provider]; + } + } + if (Object.keys(raw.order).length === 0) { + delete raw.order; + } + } + if (isRecord(raw.lastGood)) { + for (const [provider, profileId] of Object.entries(raw.lastGood)) { + if (typeof profileId !== "string" || !shouldKeepProfileId(profileId)) { + delete raw.lastGood[provider]; + } + } + if (Object.keys(raw.lastGood).length === 0) { + delete raw.lastGood; + } + } + if (isRecord(raw.usageStats)) { + for (const profileId of Object.keys(raw.usageStats)) { + if (!shouldKeepProfileId(profileId)) { + delete raw.usageStats[profileId]; + } + } + if (Object.keys(raw.usageStats).length === 0) { + delete raw.usageStats; + } + } +} + +function pruneRawAuthProfileIds(raw: unknown, profileIds: ReadonlySet): void { + if (!isRecord(raw) || !isRecord(raw.profiles)) { + return; + } + for (const profileId of profileIds) { + delete raw.profiles[profileId]; + } + filterRawAuthProfileState(raw, (profileId) => !profileIds.has(profileId)); +} + +function pickRawAuthProfileIds( + raw: unknown, + profileIds: ReadonlySet, +): Record | null { + if (!isRecord(raw) || !isRecord(raw.profiles)) { + return null; + } + const profiles = Object.fromEntries( + Object.entries(raw.profiles).filter(([profileId]) => profileIds.has(profileId)), + ); + if (Object.keys(profiles).length === 0) { + return null; + } + const next = structuredClone(raw); + next.profiles = profiles; + filterRawAuthProfileState(next, (profileId) => profileIds.has(profileId)); + return next; +} + +function collectUnresolvedLegacyOAuthSidecarProfileIds(raw: unknown): string[] { + if (!isRecord(raw) || !isRecord(raw.profiles)) { + return []; + } + const profileIds: string[] = []; + for (const [profileId, profile] of Object.entries(raw.profiles)) { + if (!isRecord(profile) || profile.type !== "oauth" || !isRecord(profile.oauthRef)) { + continue; + } + if ( + readNonEmptyString(profile.oauthRef.id) && + readNonEmptyString(profile.oauthRef.provider) && + (!readNonEmptyString(profile.access) || !readNonEmptyString(profile.refresh)) + ) { + profileIds.push(profileId); + } + } + return profileIds; +} + +function hasImportableAuthProfileStore(store: AuthProfileStore | null): store is AuthProfileStore { + return Boolean(store && (Object.keys(store.profiles).length > 0 || hasAuthProfileState(store))); +} + +function backupAuthProfileJson(pathname: string, suffix: string, now: () => number): string { + const backupPath = `${pathname}.${suffix}.${now()}.bak`; + fs.copyFileSync(pathname, backupPath); + return backupPath; +} + +function backupAndRemoveAuthProfileJson( + pathname: string, + suffix: string, + now: () => number, +): string { + const backupPath = backupAuthProfileJson(pathname, suffix, now); + fs.unlinkSync(pathname); + return backupPath; +} + +function writeJsonFile(pathname: string, value: unknown): void { + fs.writeFileSync(pathname, `${JSON.stringify(value, null, 2)}\n`, "utf8"); +} + +export async function maybeMigrateAuthProfileJsonStoresToSqlite(params: { + cfg: OpenClawConfig; + prompter: Pick; + now?: () => number; + env?: NodeJS.ProcessEnv; +}): Promise { + const now = params.now ?? Date.now; + const env = params.env ?? process.env; + const candidates = listAuthProfileSqliteMigrationCandidates(params.cfg, env); + const detected = candidates.filter( + (candidate) => + fs.existsSync(candidate.authPath) || + fs.existsSync(candidate.statePath) || + fs.existsSync(candidate.legacyPath), + ); + const result: LegacyFlatAuthProfileRepairResult = { + detected: detected.flatMap((candidate) => + [candidate.authPath, candidate.statePath, candidate.legacyPath].filter((pathname) => + fs.existsSync(pathname), + ), + ), + changes: [], + warnings: [], + }; + if (detected.length === 0) { + return result; + } + + note( + [ + ...detected.map( + (candidate) => + `- ${shortenHomePath(candidate.authPath)} / ${shortenHomePath(candidate.statePath)}`, + ), + `- ${formatCliCommand("openclaw doctor --fix")} imports legacy auth profile JSON into the per-agent SQLite database and removes the old files after backup.`, + ].join("\n"), + "Auth profile SQLite migration", + ); + + const shouldRepair = await params.prompter.confirmAutoFix({ + message: "Migrate auth profile JSON files into SQLite now?", + initialValue: true, + }); + if (!shouldRepair) { + return result; + } + + for (const candidate of detected) { + try { + const rawStore = fs.existsSync(candidate.authPath) ? loadJsonFile(candidate.authPath) : null; + const unresolvedSidecarProfileIds = new Set( + collectUnresolvedLegacyOAuthSidecarProfileIds(rawStore), + ); + const unresolvedSidecarRawStore = + unresolvedSidecarProfileIds.size > 0 + ? pickRawAuthProfileIds(rawStore, unresolvedSidecarProfileIds) + : null; + if (unresolvedSidecarProfileIds.size > 0) { + pruneRawAuthProfileIds(rawStore, unresolvedSidecarProfileIds); + result.warnings.push( + `Left ${unresolvedSidecarProfileIds.size} legacy OAuth sidecar profile${unresolvedSidecarProfileIds.size === 1 ? "" : "s"} in ${shortenHomePath(candidate.authPath)}; rerun ${formatCliCommand("openclaw doctor --fix")} after sidecar migration or re-authenticate those profiles.`, + ); + } + const awsSdkMarkerStore = + isRecord(rawStore) && isRecord(rawStore.profiles) + ? resolveAwsSdkAuthProfileMarkerStore(candidate) + : null; + if (awsSdkMarkerStore && isRecord(rawStore)) { + const configProfiles = ensureConfigAuthProfiles(params.cfg); + for (const marker of awsSdkMarkerStore.profiles) { + configProfiles[marker.profileId] = { + provider: marker.provider, + mode: "aws-sdk", + ...(marker.email ? { email: marker.email } : {}), + ...(marker.displayName ? { displayName: marker.displayName } : {}), + }; + } + removeAwsSdkProfileMarkers( + rawStore, + awsSdkMarkerStore.profiles.map((profile) => profile.profileId), + ); + result.configChanged = true; + } + normalizeLegacyApiKeyAliasesForImport(rawStore); + const maybeCanonicalStore = + coercePersistedAuthProfileStore(rawStore) ?? + coerceLegacyFlatAuthProfileStore(rawStore) ?? + null; + const canonicalStore = hasImportableAuthProfileStore(maybeCanonicalStore) + ? maybeCanonicalStore + : null; + const legacyStore = loadLegacyAuthProfileStore(candidate.agentDir); + const rawState = fs.existsSync(candidate.statePath) + ? loadJsonFile(candidate.statePath) + : null; + const state = coerceAuthProfileState(rawState); + if (!canonicalStore && !legacyStore && !hasAuthProfileState(state) && !awsSdkMarkerStore) { + result.warnings.push( + `Left auth profile JSON in place for ${shortenHomePath(candidate.authPath)} because no importable auth profiles or state were found.`, + ); + continue; + } + + const existing = loadPersistedAuthProfileStore(candidate.agentDir) ?? { + version: AUTH_STORE_VERSION, + profiles: {}, + }; + const existingProfileIds = new Set(Object.keys(existing.profiles)); + const existingState = coerceAuthProfileState(existing); + let next: AuthProfileStore = { ...existing }; + if (legacyStore) { + const legacyAsStore: AuthProfileStore = { version: AUTH_STORE_VERSION, profiles: {} }; + applyLegacyAuthStore(legacyAsStore, legacyStore); + next = mergeImportedAuthProfiles({ + store: next, + profiles: legacyAsStore.profiles, + existingProfileIds, + }); + } + if (canonicalStore) { + next = { + ...next, + version: Math.max(next.version, canonicalStore.version), + }; + next = mergeImportedAuthProfiles({ + store: next, + profiles: canonicalStore.profiles, + existingProfileIds, + }); + next = mergeImportedAuthProfileState({ + store: next, + state: coerceAuthProfileState(canonicalStore), + existingState, + }); + } + if (hasAuthProfileState(state)) { + next = mergeImportedAuthProfileState({ store: next, state, existingState }); + } + + if (canonicalStore || legacyStore || hasAuthProfileState(state)) { + const stateProfileIds = [ + ...collectAuthProfileStateProfileIds(state), + ...(canonicalStore + ? collectAuthProfileStateProfileIds(coerceAuthProfileState(canonicalStore)) + : []), + ]; + saveAuthProfileStore(next, candidate.agentDir, { + filterExternalAuthProfiles: false, + preserveStateProfileIds: stateProfileIds, + syncExternalCli: false, + }); + } + + const backups: string[] = []; + if (fs.existsSync(candidate.authPath)) { + if (unresolvedSidecarRawStore) { + backups.push(backupAuthProfileJson(candidate.authPath, "sqlite-import", now)); + writeJsonFile(candidate.authPath, unresolvedSidecarRawStore); + } else { + backups.push(backupAndRemoveAuthProfileJson(candidate.authPath, "sqlite-import", now)); + } + } + if (fs.existsSync(candidate.statePath)) { + backups.push(backupAndRemoveAuthProfileJson(candidate.statePath, "sqlite-import", now)); + } + if (fs.existsSync(candidate.legacyPath)) { + backups.push(backupAndRemoveAuthProfileJson(candidate.legacyPath, "sqlite-import", now)); + } + result.changes.push( + `Migrated auth profile JSON for ${shortenHomePath(candidate.authPath)} into SQLite (backup${backups.length === 1 ? "" : "s"}: ${backups.map(shortenHomePath).join(", ")}).`, + ); + if (awsSdkMarkerStore) { + result.changes.push( + `Moved aws-sdk profile metadata from ${shortenHomePath(candidate.authPath)} to auth.profiles before removing the legacy auth profile JSON.`, + ); + } + } catch (err) { + result.warnings.push( + `Failed to migrate auth profile JSON for ${shortenHomePath(candidate.authPath)}: ${String(err)}`, + ); + } + } + clearRuntimeAuthProfileStoreSnapshots(); + if (result.changes.length > 0) { + note(result.changes.map((change) => `- ${change}`).join("\n"), "Doctor changes"); + } + if (result.warnings.length > 0) { + note(result.warnings.map((warning) => `- ${warning}`).join("\n"), "Doctor warnings"); + } + return result; +} + function resolveLegacyFlatStore( candidate: AuthProfileRepairCandidate, ): LegacyFlatAuthProfileStore | null { @@ -368,8 +816,9 @@ export async function maybeRepairLegacyFlatAuthProfileStores(params: { try { const backupPath = backupAuthProfileStore(entry.authPath, now); saveAuthProfileStore(entry.store, entry.agentDir, { syncExternalCli: false }); + fs.unlinkSync(entry.authPath); result.changes.push( - `Rewrote ${shortenHomePath(entry.authPath)} to the canonical auth profile format (backup: ${shortenHomePath(backupPath)}).`, + `Migrated ${shortenHomePath(entry.authPath)} to the SQLite auth profile store (backup: ${shortenHomePath(backupPath)}).`, ); } catch (err) { result.warnings.push(`Failed to rewrite ${shortenHomePath(entry.authPath)}: ${String(err)}`); diff --git a/src/commands/doctor-auth-oauth-sidecar.test.ts b/src/commands/doctor-auth-oauth-sidecar.test.ts index 28244e7b5df0..04baa5f89772 100644 --- a/src/commands/doctor-auth-oauth-sidecar.test.ts +++ b/src/commands/doctor-auth-oauth-sidecar.test.ts @@ -44,6 +44,14 @@ async function makeTestState(seed = "legacy-oauth-seed"): Promise { + return state.writeJson(path.join("agents", agentId, "agent", "auth-profiles.json"), store); +} + function encryptLegacySidecarMaterial(params: { ref: { source: "openclaw-credentials"; provider: "openai-codex"; id: string }; profileId: string; @@ -109,7 +117,7 @@ describe("maybeRepairLegacyOAuthSidecarProfiles", () => { "openai-codex": profileId, }, }; - const authPath = await state.writeAuthProfiles(auth); + const authPath = await writeLegacyAuthProfiles(state, auth); const sidecarPath = await state.writeJson( path.join("credentials", "auth-profiles", `${ref.id}.json`), { @@ -183,7 +191,7 @@ describe("maybeRepairLegacyOAuthSidecarProfiles", () => { }, }, }; - const authPath = await state.writeAuthProfiles(auth); + const authPath = await writeLegacyAuthProfiles(state, auth); const result = await maybeRepairLegacyOAuthSidecarProfiles({ cfg: {}, @@ -214,7 +222,7 @@ describe("maybeRepairLegacyOAuthSidecarProfiles", () => { }, }, }; - const authPath = await state.writeAuthProfiles(auth); + const authPath = await writeLegacyAuthProfiles(state, auth); const sidecarPath = await state.writeJson( path.join("credentials", "auth-profiles", `${ref.id}.json`), { @@ -432,8 +440,8 @@ describe("maybeRepairLegacyOAuthSidecarProfiles", () => { }, }, }; - const mainAuthPath = await state.writeAuthProfiles(auth, "main"); - const workerAuthPath = await state.writeAuthProfiles(auth, "worker"); + const mainAuthPath = await writeLegacyAuthProfiles(state, auth, "main"); + const workerAuthPath = await writeLegacyAuthProfiles(state, auth, "worker"); const sidecarPath = await state.writeJson( path.join("credentials", "auth-profiles", `${ref.id}.json`), { diff --git a/src/commands/doctor-auth.profile-health.test.ts b/src/commands/doctor-auth.profile-health.test.ts index 4105717e6852..827d8726836c 100644 --- a/src/commands/doctor-auth.profile-health.test.ts +++ b/src/commands/doctor-auth.profile-health.test.ts @@ -21,6 +21,7 @@ const authProfileMocks = vi.hoisted(() => ({ throw new Error("unexpected auth profile load"); }), hasAnyAuthProfileStoreSource: vi.fn((_agentDir?: string) => false), + hasLocalAuthProfileStoreSource: vi.fn((_agentDir?: string) => false), resolveApiKeyForProfile: vi.fn(), resolveProfileUnusableUntilForDisplay: vi.fn(), })); @@ -28,6 +29,7 @@ const authProfileMocks = vi.hoisted(() => ({ vi.mock("../agents/auth-profiles.js", () => ({ ensureAuthProfileStore: authProfileMocks.ensureAuthProfileStore, hasAnyAuthProfileStoreSource: authProfileMocks.hasAnyAuthProfileStoreSource, + hasLocalAuthProfileStoreSource: authProfileMocks.hasLocalAuthProfileStoreSource, resolveApiKeyForProfile: authProfileMocks.resolveApiKeyForProfile, resolveProfileUnusableUntilForDisplay: authProfileMocks.resolveProfileUnusableUntilForDisplay, })); @@ -47,6 +49,8 @@ describe("noteAuthProfileHealth", () => { authProfileMocks.ensureAuthProfileStore.mockReset(); authProfileMocks.hasAnyAuthProfileStoreSource.mockReset(); authProfileMocks.hasAnyAuthProfileStoreSource.mockReturnValue(false); + authProfileMocks.hasLocalAuthProfileStoreSource.mockReset(); + authProfileMocks.hasLocalAuthProfileStoreSource.mockReturnValue(false); authProfileMocks.resolveApiKeyForProfile.mockReset(); authProfileMocks.resolveProfileUnusableUntilForDisplay.mockReset(); noteMock.mockReset(); @@ -121,6 +125,7 @@ describe("noteAuthProfileHealth", () => { writeAuthStore(mainDir); writeAuthStore(coderDir); authProfileMocks.hasAnyAuthProfileStoreSource.mockReturnValue(true); + authProfileMocks.hasLocalAuthProfileStoreSource.mockReturnValue(true); authProfileMocks.ensureAuthProfileStore.mockImplementation((agentDir) => { if (agentDir === mainDir) { return expiredStore("openai-codex:main", now - 60_000); @@ -156,12 +161,57 @@ describe("noteAuthProfileHealth", () => { ); }); + it("does not treat inherited main auth as a local secondary-agent source", async () => { + const now = 1_700_000_000_000; + vi.spyOn(Date, "now").mockReturnValue(now); + const mainDir = path.join(tempDir, "main-agent"); + const coderDir = path.join(tempDir, "coder-agent"); + authProfileMocks.hasAnyAuthProfileStoreSource.mockReturnValue(true); + authProfileMocks.hasLocalAuthProfileStoreSource.mockImplementation( + (agentDir) => agentDir === mainDir, + ); + authProfileMocks.ensureAuthProfileStore.mockImplementation((agentDir) => { + if (agentDir === mainDir) { + return expiredStore("openai-codex:main", now - 60_000); + } + throw new Error(`unexpected secondary agent dir: ${agentDir ?? ""}`); + }); + + await noteAuthProfileHealth({ + cfg: { + agents: { + list: [ + { id: "main", default: true, agentDir: mainDir }, + { id: "coder", agentDir: coderDir }, + ], + }, + } as OpenClawConfig, + prompter: { + confirmAutoFix: vi.fn(async () => false), + } as unknown as DoctorPrompter, + allowKeychainPrompt: false, + }); + + expect(authProfileMocks.hasLocalAuthProfileStoreSource).toHaveBeenCalledWith(coderDir); + expect(authProfileMocks.ensureAuthProfileStore).toHaveBeenCalledOnce(); + expect(authProfileMocks.ensureAuthProfileStore).toHaveBeenCalledWith(mainDir, { + allowKeychainPrompt: false, + }); + expect(noteMock).toHaveBeenCalledWith( + expect.stringContaining("openai-codex:main"), + "Model auth", + ); + }); + it("passes the target agent dir when refreshing OAuth profiles", async () => { const now = 1_700_000_000_000; vi.spyOn(Date, "now").mockReturnValue(now); const coderDir = path.join(tempDir, "coder-agent"); writeAuthStore(coderDir); authProfileMocks.hasAnyAuthProfileStoreSource.mockReturnValue(false); + authProfileMocks.hasLocalAuthProfileStoreSource.mockImplementation( + (agentDir) => agentDir === coderDir, + ); authProfileMocks.ensureAuthProfileStore.mockImplementation((agentDir) => { if (agentDir === coderDir) { return expiredStore("openai-codex:coder", now - 60_000); diff --git a/src/commands/doctor-auth.ts b/src/commands/doctor-auth.ts index ec32d6dc57c8..906405e92329 100644 --- a/src/commands/doctor-auth.ts +++ b/src/commands/doctor-auth.ts @@ -1,4 +1,3 @@ -import fs from "node:fs"; import path from "node:path"; import { note } from "../../packages/terminal-core/src/note.js"; import { @@ -16,6 +15,7 @@ import { type AuthCredentialReasonCode, ensureAuthProfileStore, hasAnyAuthProfileStoreSource, + hasLocalAuthProfileStoreSource, resolveApiKeyForProfile, resolveProfileUnusableUntilForDisplay, } from "../agents/auth-profiles.js"; @@ -25,11 +25,6 @@ import { classifyOAuthRefreshFailure, type OAuthRefreshFailureReason, } from "../agents/auth-profiles/oauth-refresh-failure.js"; -import { - resolveAuthStatePath, - resolveAuthStorePath, - resolveLegacyAuthStorePath, -} from "../agents/auth-profiles/paths.js"; import { buildProviderAuthRecoveryHint } from "../agents/provider-auth-recovery-hint.js"; import type { OpenClawConfig } from "../config/types.openclaw.js"; import { formatErrorMessage } from "../infra/errors.js"; @@ -146,14 +141,6 @@ type AuthProfileHealthTarget = { isDefault: boolean; }; -function hasLocalAuthProfileStoreSource(agentDir: string): boolean { - return ( - fs.existsSync(resolveAuthStorePath(agentDir)) || - fs.existsSync(resolveAuthStatePath(agentDir)) || - fs.existsSync(resolveLegacyAuthStorePath(agentDir)) - ); -} - function formatAgentNoteTitle(title: string, agentId: string, labelAgents: boolean): string { return labelAgents ? `${title} (agent: ${agentId})` : title; } diff --git a/src/commands/doctor.fast-path-mocks.ts b/src/commands/doctor.fast-path-mocks.ts index 41b89d5f864d..c161d2a06355 100644 --- a/src/commands/doctor.fast-path-mocks.ts +++ b/src/commands/doctor.fast-path-mocks.ts @@ -10,6 +10,10 @@ vi.mock("./doctor-bootstrap-size.js", () => ({ vi.mock("./doctor-auth-flat-profiles.js", () => ({ maybeRepairCanonicalApiKeyFieldAlias: vi.fn(async (params: { cfg: unknown }) => params.cfg), + maybeMigrateAuthProfileJsonStoresToSqlite: vi.fn().mockResolvedValue({ + changes: [], + warnings: [], + }), maybeRepairLegacyFlatAuthProfileStores: vi.fn().mockResolvedValue(undefined), maybeRepairOpenAICodexAuthConfig: vi.fn((cfg: unknown) => cfg), maybeRepairOpenAICodexAuthProfileStores: vi.fn().mockResolvedValue(undefined), diff --git a/src/commands/doctor/repair-sequencing.test.ts b/src/commands/doctor/repair-sequencing.test.ts index ccdb3faeb6f2..a7bce34fd296 100644 --- a/src/commands/doctor/repair-sequencing.test.ts +++ b/src/commands/doctor/repair-sequencing.test.ts @@ -13,6 +13,7 @@ const mocks = vi.hoisted(() => ({ maybeRepairGroupAllowFromFallback: vi.fn(), maybeRepairManagedNpmOpenClawPeerLinks: vi.fn(), maybeRepairLegacyOAuthSidecarProfiles: vi.fn(), + maybeMigrateAuthProfileJsonStoresToSqlite: vi.fn(), maybeRepairOpenAICodexAuthConfig: vi.fn(), maybeRepairOpenAICodexAuthProfileStores: vi.fn(), maybeRepairOpenPolicyAllowFrom: vi.fn(), @@ -38,7 +39,8 @@ vi.mock("../doctor-auth-oauth-sidecar.js", () => ({ })); vi.mock("../doctor-auth-flat-profiles.js", () => ({ - collectOpenAICodexAuthProfileStoreIdMap: () => new Map(), + collectOpenAICodexAuthProfileStoreIdMap: vi.fn(() => new Map()), + maybeMigrateAuthProfileJsonStoresToSqlite: mocks.maybeMigrateAuthProfileJsonStoresToSqlite, maybeRepairOpenAICodexAuthConfig: mocks.maybeRepairOpenAICodexAuthConfig, maybeRepairOpenAICodexAuthProfileStores: mocks.maybeRepairOpenAICodexAuthProfileStores, })); @@ -230,12 +232,18 @@ describe("doctor repair sequencing", () => { changes: [], warnings: [], }); + mocks.maybeMigrateAuthProfileJsonStoresToSqlite.mockResolvedValue({ + detected: [], + changes: [], + warnings: [], + }); mocks.maybeRepairOpenAICodexAuthConfig.mockImplementation((cfg: OpenClawConfig) => ({ changes: [], config: cfg, warnings: [], })); mocks.maybeRepairOpenAICodexAuthProfileStores.mockResolvedValue({ + detected: [], changes: [], warnings: [], }); @@ -376,7 +384,7 @@ describe("doctor repair sequencing", () => { expect(peerLinkCall?.env).toBe(process.env); }); - it("migrates legacy OAuth sidecars before stale OAuth shadow cleanup", async () => { + it("repairs stale OAuth shadows before importing and removing auth JSON", async () => { const events: string[] = []; mocks.maybeRepairLegacyOAuthSidecarProfiles.mockImplementationOnce(async () => { events.push("sidecar-oauth"); @@ -393,6 +401,15 @@ describe("doctor repair sequencing", () => { warnings: [], }; }); + mocks.maybeMigrateAuthProfileJsonStoresToSqlite.mockImplementationOnce(async () => { + events.push("sqlite-migration"); + return { + detected: ["auth-profiles.json"], + changes: ["Migrated auth profile JSON into SQLite."], + configChanged: true, + warnings: [], + }; + }); const result = await runDoctorRepairSequence({ state: { @@ -404,7 +421,7 @@ describe("doctor repair sequencing", () => { doctorFixCommand: "openclaw doctor --fix", }); - expect(events).toEqual(["sidecar-oauth", "stale-oauth-shadows"]); + expect(events).toEqual(["sidecar-oauth", "stale-oauth-shadows", "sqlite-migration"]); expect(mocks.maybeRepairLegacyOAuthSidecarProfiles).toHaveBeenCalledWith({ cfg: {}, prompter: { confirmAutoFix: expect.any(Function) }, @@ -414,7 +431,9 @@ describe("doctor repair sequencing", () => { expect(result.changeNotes).toEqual([ "Migrated 1 legacy Codex OAuth profile.", "Removed stale OAuth auth profile shadow openai-codex.", + "Migrated auth profile JSON into SQLite.", ]); + expect(result.state.pendingChanges).toBe(true); expect(result.warningNotes).toEqual(["Sidecar warning"]); expect(result.authProfilesRepaired).toBe(true); }); diff --git a/src/commands/doctor/repair-sequencing.ts b/src/commands/doctor/repair-sequencing.ts index 2dee42631deb..e5b59f283515 100644 --- a/src/commands/doctor/repair-sequencing.ts +++ b/src/commands/doctor/repair-sequencing.ts @@ -2,6 +2,7 @@ import { sanitizeForLog } from "../../../packages/terminal-core/src/ansi.js"; import { applyPluginAutoEnable } from "../../config/plugin-auto-enable.js"; import { collectOpenAICodexAuthProfileStoreIdMap, + maybeMigrateAuthProfileJsonStoresToSqlite, maybeRepairOpenAICodexAuthConfig, maybeRepairOpenAICodexAuthProfileStores, } from "../doctor-auth-flat-profiles.js"; @@ -189,10 +190,32 @@ export async function runDoctorRepairSequence(params: { if (staleOAuthShadowRepair.warnings.length > 0) { warningNotes.push(sanitizeLines(staleOAuthShadowRepair.warnings)); } + const authProfileSqliteMigration = await maybeMigrateAuthProfileJsonStoresToSqlite({ + cfg: state.candidate, + prompter: { confirmAutoFix: async () => true }, + env, + }); + if (authProfileSqliteMigration.configChanged) { + state = applyDoctorConfigMutation({ + state, + mutation: { + config: state.candidate, + changes: ["Auth profile SQLite migration updated auth.profiles."], + }, + shouldRepair: true, + }); + } + if (authProfileSqliteMigration.changes.length > 0) { + changeNotes.push(sanitizeLines(authProfileSqliteMigration.changes)); + } + if (authProfileSqliteMigration.warnings.length > 0) { + warningNotes.push(sanitizeLines(authProfileSqliteMigration.warnings)); + } const authProfilesRepaired = legacyOAuthSidecarRepair.changes.length > 0 || openAIAuthProviderRepair.changes.length > 0 || - staleOAuthShadowRepair.changes.length > 0; + staleOAuthShadowRepair.changes.length > 0 || + authProfileSqliteMigration.changes.length > 0; const activeToolSchemaWarnings = collectActiveToolSchemaProjectionWarnings({ cfg: state.candidate, diff --git a/src/commands/doctor/shared/stale-oauth-profile-shadows.test.ts b/src/commands/doctor/shared/stale-oauth-profile-shadows.test.ts index 4f69e2a35b5f..976a684c5008 100644 --- a/src/commands/doctor/shared/stale-oauth-profile-shadows.test.ts +++ b/src/commands/doctor/shared/stale-oauth-profile-shadows.test.ts @@ -3,7 +3,11 @@ import os from "node:os"; import path from "node:path"; import { afterEach, beforeEach, describe, expect, it } from "vitest"; import { resolveAuthStorePath } from "../../../agents/auth-profiles/paths.js"; -import { loadPersistedAuthProfileStore } from "../../../agents/auth-profiles/persisted.js"; +import { + coercePersistedAuthProfileStore, + loadPersistedAuthProfileStore, +} from "../../../agents/auth-profiles/persisted.js"; +import { writePersistedAuthProfileStoreRaw } from "../../../agents/auth-profiles/sqlite.js"; import { clearRuntimeAuthProfileStoreSnapshots, saveAuthProfileStore, @@ -40,6 +44,13 @@ async function writeRawAuthStore(agentDir: string, store: unknown): Promise { @@ -104,6 +115,50 @@ describe("stale OAuth profile shadow doctor repair", () => { expect(loadPersistedAuthProfileStore(childAgentDir)?.profiles[profileId]).toBeDefined(); }); + it("scans sqlite-only child auth stores after JSON migration", async () => { + const profileId = "anthropic:default"; + const now = Date.now(); + const childAgentDir = path.join(stateDir, "agents", "telegram", "agent"); + saveAuthProfileStore( + storeWith( + profileId, + oauthCredential({ + access: "child-access", + refresh: "child-refresh", + expires: now - 60_000, + accountId: "acct-shared", + }), + ), + childAgentDir, + ); + saveAuthProfileStore( + storeWith( + profileId, + oauthCredential({ + access: "main-access", + refresh: "main-refresh", + expires: now + 60 * 60 * 1000, + accountId: "acct-shared", + }), + ), + ); + + const hits = await scanStaleOAuthProfileShadows({ + cfg: {} satisfies OpenClawConfig, + now, + }); + + expect(hits).toEqual([ + expect.objectContaining({ + authPath: resolveAuthStorePath(childAgentDir), + profileId, + }), + ]); + await expect(fs.access(resolveAuthStorePath(childAgentDir))).rejects.toMatchObject({ + code: "ENOENT", + }); + }); + it("uses the injected env for the main auth store", async () => { const profileId = "anthropic:default"; const now = Date.now(); @@ -229,25 +284,28 @@ describe("stale OAuth profile shadow doctor repair", () => { ), undefined, ); - await writeRawAuthStore(childAgentDir, { - ...storeWith( - profileId, - oauthCredential({ - access: "child-access", - refresh: "child-refresh", - expires: now - 60_000, - accountId: "acct-shared", - }), - ), - order: { anthropic: [profileId] }, - lastGood: { anthropic: profileId }, - usageStats: { - [profileId]: { - cooldownReason: "auth", - failureCounts: { auth: 2 }, + writePersistedAuthProfileStoreRaw( + { + ...storeWith( + profileId, + oauthCredential({ + access: "child-access", + refresh: "child-refresh", + expires: now - 60_000, + accountId: "acct-shared", + }), + ), + order: { anthropic: [profileId] }, + lastGood: { anthropic: profileId }, + usageStats: { + [profileId]: { + cooldownReason: "auth", + failureCounts: { auth: 2 }, + }, }, }, - }); + childAgentDir, + ); const result = await repairStaleOAuthProfileShadows({ cfg: { agents: { list: [{ id: "telegram" }] } } satisfies OpenClawConfig, diff --git a/src/commands/doctor/shared/stale-oauth-profile-shadows.ts b/src/commands/doctor/shared/stale-oauth-profile-shadows.ts index dace39e527d6..b7ec6f4e7260 100644 --- a/src/commands/doctor/shared/stale-oauth-profile-shadows.ts +++ b/src/commands/doctor/shared/stale-oauth-profile-shadows.ts @@ -6,7 +6,6 @@ import { resolveDefaultAgentDir, listAgentEntries, } from "../../../agents/agent-scope.js"; -import { AUTH_STORE_LOCK_OPTIONS } from "../../../agents/auth-profiles/constants.js"; import { isLegacyOAuthRef, LEGACY_OAUTH_REF_PROVIDER, @@ -18,11 +17,10 @@ import { } from "../../../agents/auth-profiles/oauth-shared.js"; import { resolveAuthStorePath } from "../../../agents/auth-profiles/paths.js"; import { loadPersistedAuthProfileStore } from "../../../agents/auth-profiles/persisted.js"; -import { saveAuthProfileStore } from "../../../agents/auth-profiles/store.js"; +import { updateAuthProfileStoreWithLock } from "../../../agents/auth-profiles/store.js"; import type { AuthProfileStore, OAuthCredential } from "../../../agents/auth-profiles/types.js"; import { resolveStateDir } from "../../../config/paths.js"; import type { OpenClawConfig } from "../../../config/types.openclaw.js"; -import { withFileLock } from "../../../infra/file-lock.js"; import { shortenHomePath } from "../../../utils.js"; type StaleOAuthProfileShadow = { @@ -57,15 +55,6 @@ function hasLegacyOAuthSidecarRef(raw: Record | null, profileId ); } -async function pathExists(targetPath: string): Promise { - try { - await fs.lstat(targetPath); - return true; - } catch { - return false; - } -} - async function collectStateAgentDirs(env: NodeJS.ProcessEnv): Promise { const agentsRoot = path.join(resolveStateDir(env), "agents"); const entries = await fs.readdir(agentsRoot, { withFileTypes: true }).catch(() => []); @@ -133,7 +122,7 @@ export async function scanStaleOAuthProfileShadows(params: { const hits: StaleOAuthProfileShadow[] = []; for (const agentDir of await collectCandidateAgentDirs(params.cfg, env)) { const authPath = path.resolve(resolveAuthStorePath(agentDir)); - if (authPath === mainAuthPath || !(await pathExists(authPath))) { + if (authPath === mainAuthPath) { continue; } const rawLocalStore = await loadRawAuthProfileStore(authPath); @@ -172,6 +161,8 @@ function removeStaleProfilesFromStore(params: { const removedProfileIds: string[] = []; const profiles = { ...params.store.profiles }; const usageStats = params.store.usageStats ? { ...params.store.usageStats } : undefined; + const order = params.store.order ? { ...params.store.order } : undefined; + const lastGood = params.store.lastGood ? { ...params.store.lastGood } : undefined; for (const profileId of params.profileIds) { const local = profiles[profileId]; const main = params.mainStore.profiles[profileId]; @@ -189,6 +180,23 @@ function removeStaleProfilesFromStore(params: { if (usageStats) { delete usageStats[profileId]; } + if (lastGood) { + for (const [provider, lastGoodProfileId] of Object.entries(lastGood)) { + if (lastGoodProfileId === profileId) { + delete lastGood[provider]; + } + } + } + if (order) { + for (const [provider, profileIds] of Object.entries(order)) { + const nextProfileIds = profileIds.filter((entry) => entry !== profileId); + if (nextProfileIds.length > 0) { + order[provider] = nextProfileIds; + } else { + delete order[provider]; + } + } + } removedProfileIds.push(profileId); } return { @@ -198,6 +206,8 @@ function removeStaleProfilesFromStore(params: { ...(usageStats && Object.keys(usageStats).length > 0 ? { usageStats } : { usageStats: undefined }), + ...(lastGood && Object.keys(lastGood).length > 0 ? { lastGood } : { lastGood: undefined }), + ...(order && Object.keys(order).length > 0 ? { order } : { order: undefined }), }, removedProfileIds, }; @@ -215,23 +225,22 @@ async function repairStaleOAuthProfilesForAgent(params: { }): Promise< { status: "changed"; removedProfileIds: string[] } | { status: "missing" | "unchanged" } > { - return await withFileLock( - resolveAuthStorePath(params.agentDir), - AUTH_STORE_LOCK_OPTIONS, - async () => { - const store = loadPersistedAuthProfileStore(params.agentDir); - if (!store) { - return { status: "missing" }; - } - const rawStore = await loadRawAuthProfileStore(resolveAuthStorePath(params.agentDir)); - const profileIds = new Set( - [...params.profileIds].filter( - (profileId) => !hasLegacyOAuthSidecarRef(rawStore, profileId), - ), - ); - if (profileIds.size === 0) { - return { status: "unchanged" }; - } + const rawStore = await loadRawAuthProfileStore(resolveAuthStorePath(params.agentDir)); + const profileIds = new Set( + [...params.profileIds].filter((profileId) => !hasLegacyOAuthSidecarRef(rawStore, profileId)), + ); + if (profileIds.size === 0) { + return { status: "unchanged" }; + } + if (!loadPersistedAuthProfileStore(params.agentDir)) { + return { status: "missing" }; + } + let sawStore = false; + let removedProfileIds: string[] = []; + await updateAuthProfileStoreWithLock({ + agentDir: params.agentDir, + updater: (store) => { + sawStore = true; const result = removeStaleProfilesFromStore({ store, mainStore: params.mainStore, @@ -239,17 +248,19 @@ async function repairStaleOAuthProfilesForAgent(params: { now: params.now, }); if (result.removedProfileIds.length === 0) { - return { status: "unchanged" }; + return false; } - saveAuthProfileStore(result.store, params.agentDir, { - pruneOrderProfileIds: result.removedProfileIds, - }); - return { - status: "changed", - removedProfileIds: result.removedProfileIds, - }; + removedProfileIds = result.removedProfileIds; + Object.assign(store, result.store); + return true; }, - ); + }); + if (!sawStore) { + return { status: "missing" }; + } + return removedProfileIds.length > 0 + ? { status: "changed", removedProfileIds } + : { status: "unchanged" }; } export function collectStaleOAuthProfileShadowWarnings(params: { diff --git a/src/commands/models/auth-list.test.ts b/src/commands/models/auth-list.test.ts index 4ffaabbb8517..2451d0d68596 100644 --- a/src/commands/models/auth-list.test.ts +++ b/src/commands/models/auth-list.test.ts @@ -21,7 +21,7 @@ vi.mock("../../agents/auth-profiles.js", () => ({ ensureAuthProfileStore: mocks.ensureAuthProfileStore, externalCliDiscoveryForProviderAuth: mocks.externalCliDiscoveryForProviderAuth, resolveAuthProfileDisplayLabel: mocks.resolveAuthProfileDisplayLabel, - resolveAuthStatePathForDisplay: (agentDir: string) => `${agentDir}/auth-state.json`, + resolveAuthStatePathForDisplay: (agentDir: string) => `${agentDir}/openclaw-agent.sqlite`, })); vi.mock("./load-config.js", () => ({ @@ -98,7 +98,7 @@ describe("modelsAuthListCommand", () => { { agentDir: "/tmp/openclaw/agents/coder", agentId: "coder", - authStatePath: "/tmp/openclaw/agents/coder/auth-state.json", + authStatePath: "/tmp/openclaw/agents/coder/openclaw-agent.sqlite", profiles: [ { cooldownUntil: "2027-01-15T08:00:10.000Z", @@ -153,7 +153,7 @@ describe("modelsAuthListCommand", () => { { agentDir: "/tmp/openclaw/agents/main", agentId: "main", - authStatePath: "/tmp/openclaw/agents/main/auth-state.json", + authStatePath: "/tmp/openclaw/agents/main/openclaw-agent.sqlite", profiles: [ { id: "openai:api-key-backup", @@ -184,7 +184,7 @@ describe("modelsAuthListCommand", () => { expect(runtime.logs).toEqual([ "Agent: main", - "Auth state file: /tmp/openclaw/agents/main/auth-state.json", + "Auth state store: /tmp/openclaw/agents/main/openclaw-agent.sqlite", "Profiles: (none)", ]); }); @@ -217,7 +217,7 @@ describe("modelsAuthListCommand", () => { { agentDir: "/tmp/openclaw/agents/main", agentId: "main", - authStatePath: "/tmp/openclaw/agents/main/auth-state.json", + authStatePath: "/tmp/openclaw/agents/main/openclaw-agent.sqlite", profiles: [ { email: "user@example.com", diff --git a/src/commands/models/auth-list.ts b/src/commands/models/auth-list.ts index d1cb64a63c42..e73bd250fd34 100644 --- a/src/commands/models/auth-list.ts +++ b/src/commands/models/auth-list.ts @@ -151,7 +151,7 @@ export async function modelsAuthListCommand( } runtime.log(`Agent: ${agentId}`); - runtime.log(`Auth state file: ${shortenHomePath(resolveAuthStatePathForDisplay(agentDir))}`); + runtime.log(`Auth state store: ${shortenHomePath(resolveAuthStatePathForDisplay(agentDir))}`); if (providerFilter.provider) { runtime.log(`Provider: ${providerFilter.provider}`); } diff --git a/src/commands/models/auth-order.ts b/src/commands/models/auth-order.ts index 823603804acf..42281706d56e 100644 --- a/src/commands/models/auth-order.ts +++ b/src/commands/models/auth-order.ts @@ -71,7 +71,7 @@ export async function modelsAuthOrderGetCommand( runtime.log(`Agent: ${agentId}`); runtime.log(`Provider: ${provider}`); - runtime.log(`Auth state file: ${shortenHomePath(resolveAuthStatePathForDisplay(agentDir))}`); + runtime.log(`Auth state store: ${shortenHomePath(resolveAuthStatePathForDisplay(agentDir))}`); runtime.log(order.length > 0 ? `Order override: ${order.join(", ")}` : "Order override: (none)"); } @@ -87,7 +87,7 @@ export async function modelsAuthOrderClearCommand( }); if (!updated) { throw new Error( - `Failed to update auth-state.json; the auth state lock may be busy. Wait a moment and rerun ${formatCliCommand("openclaw models auth order clear --provider " + provider)}.`, + `Failed to update auth state; the auth state lock may be busy. Wait a moment and rerun ${formatCliCommand("openclaw models auth order clear --provider " + provider)}.`, ); } @@ -132,7 +132,7 @@ export async function modelsAuthOrderSetCommand( }); if (!updated) { throw new Error( - `Failed to update auth-state.json; the auth state lock may be busy. Wait a moment and rerun ${formatCliCommand("openclaw models auth order set --provider " + provider + " ")}.`, + `Failed to update auth state; the auth state lock may be busy. Wait a moment and rerun ${formatCliCommand("openclaw models auth order set --provider " + provider + " ")}.`, ); } diff --git a/src/infra/backup-create.test.ts b/src/infra/backup-create.test.ts index 597b190c16a3..60cf63e9080f 100644 --- a/src/infra/backup-create.test.ts +++ b/src/infra/backup-create.test.ts @@ -3,8 +3,10 @@ import os from "node:os"; import path from "node:path"; import * as tar from "tar"; import { describe, expect, it, vi } from "vitest"; +import { saveAuthProfileStore } from "../agents/auth-profiles/store.js"; import { backupVerifyCommand } from "../commands/backup-verify.js"; import type { RuntimeEnv } from "../runtime.js"; +import { closeOpenClawAgentDatabasesForTest } from "../state/openclaw-agent-db.js"; import { closeOpenClawStateDatabase, openOpenClawStateDatabase, @@ -496,6 +498,73 @@ describe("createBackupArchive", () => { ); }); + it("snapshots per-agent SQLite auth stores into the archive", async () => { + await withOpenClawTestState( + { + layout: "state-only", + prefix: "openclaw-backup-agent-sqlite-", + scenario: "minimal", + }, + async (state) => { + const outputDir = state.path("backups"); + const extractDir = state.path("extract"); + await fs.mkdir(outputDir, { recursive: true }); + await fs.mkdir(extractDir, { recursive: true }); + saveAuthProfileStore( + { + version: 1, + profiles: { + "openai:default": { + type: "api_key", + provider: "openai", + key: "sk-backup", + }, + }, + }, + state.agentDir(), + { syncExternalCli: false }, + ); + closeOpenClawAgentDatabasesForTest(); + + const result = await createBackupArchive({ + output: outputDir, + includeWorkspace: false, + nowMs: Date.UTC(2026, 4, 9, 8, 31, 0), + }); + const entries = await listArchiveEntries(result.archivePath); + const archivedDbEntry = entries.find((entry) => + entry.endsWith("/state/agents/main/agent/openclaw-agent.sqlite"), + ); + expect(archivedDbEntry).toBeDefined(); + expect( + entries.some((entry) => + entry.endsWith("/state/agents/main/agent/openclaw-agent.sqlite-wal"), + ), + ).toBe(false); + + await tar.x({ file: result.archivePath, gzip: true, cwd: extractDir }); + const extractedPath = path.join(extractDir, archivedDbEntry!); + expect((await fs.stat(extractedPath)).mode & 0o777).toBe(0o600); + const sqlite = requireNodeSqlite(); + const archivedDb = new sqlite.DatabaseSync(extractedPath, { + readOnly: true, + }); + try { + const row = archivedDb + .prepare("SELECT store_json FROM auth_profile_store WHERE store_key = 'primary'") + .get() as { store_json: string }; + expect(JSON.parse(row.store_json).profiles["openai:default"]).toMatchObject({ + type: "api_key", + provider: "openai", + key: "sk-backup", + }); + } finally { + archivedDb.close(); + } + }, + ); + }); + it("omits installed plugin node_modules from the real archive while keeping plugin files", async () => { await withOpenClawTestState( { diff --git a/src/infra/backup-create.ts b/src/infra/backup-create.ts index 2d1586e6604c..aa570473f61a 100644 --- a/src/infra/backup-create.ts +++ b/src/infra/backup-create.ts @@ -497,6 +497,7 @@ async function createSanitizedStateSqliteBackupAsset(params: { } finally { source.close(); } + await fs.chmod(sourcePath, 0o600); const snapshot = new sqlite.DatabaseSync(sourcePath); try { @@ -519,6 +520,65 @@ async function createSanitizedStateSqliteBackupAsset(params: { }; } +async function listAgentSqlitePaths(stateDir: string): Promise { + const agentsDir = path.join(stateDir, "agents"); + const found: string[] = []; + async function visit(dir: string): Promise { + let entries: import("node:fs").Dirent[]; + try { + entries = await fs.readdir(dir, { withFileTypes: true }); + } catch { + return; + } + for (const entry of entries) { + const entryPath = path.join(dir, entry.name); + if (entry.isDirectory()) { + await visit(entryPath); + } else if (entry.isFile() && entry.name === "openclaw-agent.sqlite") { + found.push(entryPath); + } + } + } + await visit(agentsDir); + return found; +} + +async function createAgentSqliteBackupAssets(params: { + stateDir: string; + tempDir: string; +}): Promise { + const sqlitePaths = await listAgentSqlitePaths(params.stateDir); + if (sqlitePaths.length === 0) { + return []; + } + const sqlite = requireNodeSqlite(); + const snapshots: SanitizedSqliteBackupAsset[] = []; + for (const archiveSourcePath of sqlitePaths) { + const source = new sqlite.DatabaseSync(archiveSourcePath, { readOnly: true }); + const sourcePath = path.join( + params.tempDir, + `openclaw-agent-backup-${snapshots.length}.sqlite`, + ); + try { + source.exec("PRAGMA busy_timeout = 30000;"); + source.prepare("VACUUM INTO ?").run(sourcePath); + } finally { + source.close(); + } + await fs.chmod(sourcePath, 0o600); + snapshots.push({ + sourcePath, + archiveSourcePath, + skippedSourcePaths: new Set([ + path.resolve(archiveSourcePath), + path.resolve(`${archiveSourcePath}-wal`), + path.resolve(`${archiveSourcePath}-shm`), + ]), + }); + } + return snapshots; +} + export async function createBackupArchive( opts: BackupCreateOptions = {}, ): Promise { @@ -588,6 +648,12 @@ export async function createBackupArchive( tempDir, }) : undefined; + const agentSqliteSnapshots = stateAsset + ? await createAgentSqliteBackupAssets({ + stateDir: stateAsset.sourcePath, + tempDir, + }) + : []; const sourcePathRemaps = new Map(); if (sanitizedStateSqlite) { sourcePathRemaps.set( @@ -595,6 +661,9 @@ export async function createBackupArchive( sanitizedStateSqlite.archiveSourcePath, ); } + for (const snapshot of agentSqliteSnapshots) { + sourcePathRemaps.set(path.resolve(snapshot.sourcePath), snapshot.archiveSourcePath); + } const manifest = buildManifest({ createdAt, archiveRoot, @@ -621,7 +690,12 @@ export async function createBackupArchive( if (path.resolve(entryPath) === manifestPath) { return true; } - if (sanitizedStateSqlite?.skippedSourcePaths.has(path.resolve(entryPath))) { + if ( + sanitizedStateSqlite?.skippedSourcePaths.has(path.resolve(entryPath)) || + agentSqliteSnapshots.some((snapshot) => + snapshot.skippedSourcePaths.has(path.resolve(entryPath)), + ) + ) { return false; } if (extensionsFilter && !extensionsFilter(entryPath)) { @@ -661,6 +735,7 @@ export async function createBackupArchive( [ manifestPath, ...(sanitizedStateSqlite ? [sanitizedStateSqlite.sourcePath] : []), + ...agentSqliteSnapshots.map((snapshot) => snapshot.sourcePath), ...result.assets.map((asset) => asset.sourcePath), ], ); diff --git a/src/infra/channel-runtime-context.ts b/src/infra/channel-runtime-context.ts index 3893fda3cfac..c4f53dbc6043 100644 --- a/src/infra/channel-runtime-context.ts +++ b/src/infra/channel-runtime-context.ts @@ -50,17 +50,16 @@ export function registerChannelRuntimeContext( } /** Reads a channel-scoped runtime context from the current runtime registry. */ -// oxlint-disable-next-line typescript/no-unnecessary-type-parameters -- Runtime context values are caller-typed by key. -export function getChannelRuntimeContext( +export function getChannelRuntimeContext( params: ChannelRuntimeContextKey & { channelRuntime?: ChannelRuntimeSurface; }, -): T | undefined { +): unknown { const runtimeContexts = resolveRuntimeContextRegistry(params); if (!runtimeContexts) { return undefined; } - return runtimeContexts.get({ + return runtimeContexts.get({ channelId: params.channelId, accountId: params.accountId, capability: params.capability, diff --git a/src/media-understanding/runner.local-no-auth.test.ts b/src/media-understanding/runner.local-no-auth.test.ts index e4458c190d5a..a585a186fdc3 100644 --- a/src/media-understanding/runner.local-no-auth.test.ts +++ b/src/media-understanding/runner.local-no-auth.test.ts @@ -2,6 +2,7 @@ import fs from "node:fs/promises"; import os from "node:os"; import path from "node:path"; import { describe, expect, it, vi } from "vitest"; +import { saveAuthProfileStore } from "../agents/auth-profiles/store.js"; import { CUSTOM_LOCAL_AUTH_MARKER } from "../agents/model-auth-markers.js"; import type { OpenClawConfig } from "../config/types.js"; import { withEnvAsync } from "../test-utils/env.js"; @@ -234,9 +235,8 @@ describe("runCapability local no-auth audio providers", () => { it("prefers stored auth profile credentials over plugin-only media no-auth", async () => { await withIsolatedAgentDir(async (agentDir) => { await withEnvAsync(AUTH_ENV, async () => { - await fs.writeFile( - path.join(agentDir, "auth-profiles.json"), - JSON.stringify({ + saveAuthProfileStore( + { version: 1, profiles: { "local-audio:default": { @@ -245,7 +245,9 @@ describe("runCapability local no-auth audio providers", () => { key: "stored-local-audio-key", }, }, - }), + }, + agentDir, + { filterExternalAuthProfiles: false, syncExternalCli: false }, ); await withAudioFixture( "openclaw-local-audio-stored-profile", diff --git a/src/secrets/apply.test.ts b/src/secrets/apply.test.ts index 4d15c6a35536..25f67a1e8e8d 100644 --- a/src/secrets/apply.test.ts +++ b/src/secrets/apply.test.ts @@ -2,6 +2,13 @@ import fs from "node:fs/promises"; import os from "node:os"; import path from "node:path"; import { afterEach, beforeAll, beforeEach, describe, expect, it, vi } from "vitest"; +import { resolveAuthProfileDatabasePath } from "../agents/auth-profiles/sqlite.js"; +import { saveAuthProfileStore } from "../agents/auth-profiles/store.js"; +import type { AuthProfileStore } from "../agents/auth-profiles/types.js"; +import { + closeOpenClawAgentDatabasesForTest, + openOpenClawAgentDatabase, +} from "../state/openclaw-agent-db.js"; import { buildTalkTestProviderConfig, TALK_TEST_PROVIDER_API_KEY_PATH, @@ -34,6 +41,7 @@ type ApplyFixture = { rootDir: string; stateDir: string; configPath: string; + agentDir: string; authStorePath: string; authJsonPath: string; envPath: string; @@ -56,9 +64,21 @@ function stripVolatileConfigMeta(input: string): Record { } async function writeJsonFile(filePath: string, value: unknown): Promise { + if (path.basename(filePath) === "openclaw-agent.sqlite") { + saveAuthProfileStore(value as AuthProfileStore, path.dirname(filePath), { + filterExternalAuthProfiles: false, + syncExternalCli: false, + }); + return; + } await fs.writeFile(filePath, `${JSON.stringify(value, null, 2)}\n`, "utf8"); } +async function readAuthStore(fixture: ApplyFixture): Promise { + const { loadPersistedAuthProfileStore } = await import("../agents/auth-profiles/persisted.js"); + return loadPersistedAuthProfileStore(fixture.agentDir) ?? { version: 1, profiles: {} }; +} + function createOpenAiProviderConfig(apiKey: unknown = "sk-openai-plaintext") { return { baseUrl: "https://api.openai.com/v1", @@ -70,12 +90,14 @@ function createOpenAiProviderConfig(apiKey: unknown = "sk-openai-plaintext") { function buildFixturePaths(rootDir: string) { const stateDir = path.join(rootDir, ".openclaw"); + const agentDir = path.join(stateDir, "agents", "main", "agent"); return { rootDir, stateDir, configPath: path.join(stateDir, "openclaw.json"), - authStorePath: path.join(stateDir, "agents", "main", "agent", "auth-profiles.json"), - authJsonPath: path.join(stateDir, "agents", "main", "agent", "auth.json"), + agentDir, + authStorePath: resolveAuthProfileDatabasePath(agentDir), + authJsonPath: path.join(agentDir, "auth.json"), envPath: path.join(stateDir, ".env"), }; } @@ -85,7 +107,7 @@ async function createApplyFixture(): Promise { await fs.mkdtemp(path.join(os.tmpdir(), "openclaw-secrets-apply-")), ); await fs.mkdir(path.dirname(paths.configPath), { recursive: true }); - await fs.mkdir(path.dirname(paths.authStorePath), { recursive: true }); + await fs.mkdir(paths.agentDir, { recursive: true }); return { ...paths, env: { @@ -262,6 +284,7 @@ describe("secrets apply", () => { afterEach(async () => { clearSecretsRuntimeSnapshot(); + closeOpenClawAgentDatabasesForTest(); await fs.rm(fixture.rootDir, { recursive: true, force: true }); }); @@ -287,7 +310,7 @@ describe("secrets apply", () => { }; expect(nextConfig.models.providers.openai.apiKey).toEqual(OPENAI_API_KEY_ENV_REF); - const nextAuthStore = JSON.parse(await fs.readFile(fixture.authStorePath, "utf8")) as { + const nextAuthStore = (await readAuthStore(fixture)) as unknown as { profiles: { "openai:default": { key?: string; keyRef?: unknown } }; }; expect(nextAuthStore.profiles["openai:default"].key).toBeUndefined(); @@ -327,7 +350,7 @@ describe("secrets apply", () => { await runSecretsApply({ plan, env: fixture.env, write: true }); - const nextAuthStore = JSON.parse(await fs.readFile(fixture.authStorePath, "utf8")) as { + const nextAuthStore = (await readAuthStore(fixture)) as unknown as { profiles: { "openai:bot": { token?: string; tokenRef?: unknown } }; }; expect(nextAuthStore.profiles["openai:bot"].token).toBeUndefined(); @@ -353,7 +376,7 @@ describe("secrets apply", () => { await runSecretsApply({ plan, env: fixture.env, write: true }); - const nextAuthStore = JSON.parse(await fs.readFile(fixture.authStorePath, "utf8")) as { + const nextAuthStore = (await readAuthStore(fixture)) as unknown as { profiles: { "openai:default": { key?: string; keyRef?: unknown } }; }; expect(nextAuthStore.profiles["openai:default"].key).toBeUndefined(); @@ -501,6 +524,16 @@ describe("secrets apply", () => { }); it("applies auth-profiles sibling ref targets to the scoped agent store", async () => { + await writeJsonFile(fixture.authStorePath, { + version: 1, + profiles: { + "openai:default": { + type: "api_key", + provider: "openai", + key: "sk-ope...text", // pragma: allowlist secret + }, + }, + }); const plan: SecretsApplyPlan = { version: 1, protocolVersion: 1, @@ -526,7 +559,7 @@ describe("secrets apply", () => { expect(result.changed).toBe(true); expect(result.changedFiles).toContain(fixture.authStorePath); - const nextAuthStore = JSON.parse(await fs.readFile(fixture.authStorePath, "utf8")) as { + const nextAuthStore = (await readAuthStore(fixture)) as unknown as { profiles: { "openai:default": { key?: string; keyRef?: unknown } }; }; expect(nextAuthStore.profiles["openai:default"].key).toBeUndefined(); @@ -537,6 +570,46 @@ describe("secrets apply", () => { }); }); + it("uses the configured agent id for custom auth-profile target agent dirs", async () => { + const coderAgentDir = path.join(fixture.rootDir, "custom-coder-agent"); + const coderStorePath = resolveAuthProfileDatabasePath(coderAgentDir); + await writeJsonFile(fixture.configPath, { + agents: { + list: [{ id: "coder", agentDir: coderAgentDir }], + }, + }); + const plan: SecretsApplyPlan = { + version: 1, + protocolVersion: 1, + generatedAt: new Date().toISOString(), + generatedBy: "manual", + targets: [ + { + type: "auth-profiles.api_key.key", + path: "profiles.openai:default.key", + pathSegments: ["profiles", "openai:default", "key"], + agentId: "coder", + ref: { source: "env", provider: "default", id: "OPENAI_API_KEY" }, + authProfileProvider: "openai", + }, + ], + options: { + scrubEnv: false, + scrubAuthProfilesForProviderTargets: false, + scrubLegacyAuthJson: false, + }, + }; + + const result = await runSecretsApply({ plan, env: fixture.env, write: true }); + + expect(result.changedFiles).toContain(coderStorePath); + const database = openOpenClawAgentDatabase({ + agentId: "coder", + path: coderStorePath, + }); + expect(database.agentId).toBe("coder"); + }); + it("preserves unrelated oauth profiles while applying auth-profile key ref targets", async () => { const codexOAuthRef = { id: "codex-sidecar-ref", @@ -590,7 +663,7 @@ describe("secrets apply", () => { const result = await runSecretsApply({ plan, env: fixture.env, write: true }); expect(result.changed).toBe(true); - const nextAuthStore = JSON.parse(await fs.readFile(fixture.authStorePath, "utf8")) as { + const nextAuthStore = (await readAuthStore(fixture)) as unknown as { profiles: Record< string, { @@ -615,12 +688,11 @@ describe("secrets apply", () => { expect(nextAuthStore.profiles["openai:sidecar"]).toMatchObject({ type: "oauth", provider: "openai", - oauthRef: codexOAuthRef, email: "codex@example.invalid", }); expect(nextAuthStore.profiles["anthropic:claude-cli"]).toEqual({ provider: "claude-cli", - mode: "oauth", + type: "oauth", }); expect(nextAuthStore.order?.["openai"]).toEqual(["openai:sidecar", "openai:static"]); expect(nextAuthStore.lastGood?.["claude-cli"]).toBe("anthropic:claude-cli"); @@ -650,7 +722,7 @@ describe("secrets apply", () => { }; await runSecretsApply({ plan, env: fixture.env, write: true }); - const nextAuthStore = JSON.parse(await fs.readFile(fixture.authStorePath, "utf8")) as { + const nextAuthStore = (await readAuthStore(fixture)) as unknown as { profiles: { "openai:bot": { type: string; @@ -679,12 +751,11 @@ describe("secrets apply", () => { const first = await runSecretsApply({ plan, env: fixture.env, write: true }); expect(first.changed).toBe(true); const configAfterFirst = await fs.readFile(fixture.configPath, "utf8"); - const authStoreAfterFirst = await fs.readFile(fixture.authStorePath, "utf8"); + const authStoreAfterFirst = JSON.stringify(await readAuthStore(fixture)); const authJsonAfterFirst = await fs.readFile(fixture.authJsonPath, "utf8"); const envAfterFirst = await fs.readFile(fixture.envPath, "utf8"); await fs.chmod(fixture.configPath, 0o400); - await fs.chmod(fixture.authStorePath, 0o400); const second = await runSecretsApply({ plan, env: fixture.env, write: true }); expect(second.mode).toBe("write"); @@ -692,7 +763,7 @@ describe("secrets apply", () => { expect(stripVolatileConfigMeta(configAfterSecond)).toEqual( stripVolatileConfigMeta(configAfterFirst), ); - await expect(fs.readFile(fixture.authStorePath, "utf8")).resolves.toBe(authStoreAfterFirst); + expect(JSON.stringify(await readAuthStore(fixture))).toBe(authStoreAfterFirst); await expect(fs.readFile(fixture.authJsonPath, "utf8")).resolves.toBe(authJsonAfterFirst); await expect(fs.readFile(fixture.envPath, "utf8")).resolves.toBe(envAfterFirst); }); diff --git a/src/secrets/apply.ts b/src/secrets/apply.ts index f4f488d942f2..9ce8aa70d98f 100644 --- a/src/secrets/apply.ts +++ b/src/secrets/apply.ts @@ -2,11 +2,19 @@ import fs from "node:fs"; import os from "node:os"; import path from "node:path"; import { isDeepStrictEqual } from "node:util"; +import { registerResolvedAgentDir } from "../agents/agent-dir-registry.js"; import { resolveAgentConfig } from "../agents/agent-scope.js"; import { loadAuthProfileStoreForSecretsRuntime } from "../agents/auth-profiles.js"; import { AUTH_STORE_VERSION } from "../agents/auth-profiles/constants.js"; -import { resolveAuthStorePath } from "../agents/auth-profiles/paths.js"; -import { coercePersistedAuthProfileStore } from "../agents/auth-profiles/persisted.js"; +import { + coercePersistedAuthProfileStore, + loadPersistedAuthProfileStore, +} from "../agents/auth-profiles/persisted.js"; +import { + deletePersistedAuthProfileStoreRaw, + resolveAuthProfileDatabasePath, +} from "../agents/auth-profiles/sqlite.js"; +import { saveAuthProfileStore } from "../agents/auth-profiles/store.js"; import { normalizeProviderId } from "../agents/model-selection.js"; import { replaceConfigFile, @@ -34,7 +42,7 @@ import { prepareSecretsRuntimeSnapshot } from "./runtime.js"; import { assertExpectedResolvedSecretValue } from "./secret-value.js"; import { isNonEmptyString, isRecord, writeTextFileAtomic } from "./shared.js"; import { - listAuthProfileStorePaths, + listAuthProfileStoreAgentDirs, listLegacyAuthJsonPaths, parseEnvAssignmentValue, readJsonObjectIfExists, @@ -52,12 +60,18 @@ type ApplyWrite = { mode: number; }; +type AuthStoreSnapshot = { + agentDir: string; + store: ReturnType; +}; + type ProjectedState = { nextConfig: OpenClawConfig; configSnapshot: ConfigFileSnapshot; configPath: string; configWriteOptions: ConfigWriteOptions; authStoreByPath: Map>; + authStoreAgentDirByPath: Map; authJsonByPath: Map>; envRawByPath: Map; changedFiles: Set; @@ -78,6 +92,7 @@ type ConfigTargetMutationResult = { providerTargets: Set; configChanged: boolean; authStoreByPath: Map>; + authStoreAgentDirByPath: Map; }; type MutableAuthProfileStore = Record & { @@ -234,6 +249,7 @@ async function projectPlanState(params: { nextConfig, stateDir, authStoreByPath: new Map>(), + authStoreAgentDirByPath: new Map(), changedFiles, }); if (targetMutations.configChanged) { @@ -246,6 +262,7 @@ async function projectPlanState(params: { providerTargets: targetMutations.providerTargets, scrubbedValues: targetMutations.scrubbedValues, authStoreByPath: targetMutations.authStoreByPath, + authStoreAgentDirByPath: targetMutations.authStoreAgentDirByPath, changedFiles, warnings, enabled: options.scrubAuthProfilesForProviderTargets, @@ -281,6 +298,7 @@ async function projectPlanState(params: { configPath, configWriteOptions: writeOptions, authStoreByPath, + authStoreAgentDirByPath: targetMutations.authStoreAgentDirByPath, authJsonByPath, envRawByPath, changedFiles, @@ -296,6 +314,7 @@ function applyConfigTargetMutations(params: { nextConfig: OpenClawConfig; stateDir: string; authStoreByPath: Map>; + authStoreAgentDirByPath: Map; changedFiles: Set; }): ConfigTargetMutationResult { const resolvedTargets = params.planTargets.map((target) => ({ @@ -314,6 +333,7 @@ function applyConfigTargetMutations(params: { nextConfig: params.nextConfig, stateDir: params.stateDir, authStoreByPath: params.authStoreByPath, + authStoreAgentDirByPath: params.authStoreAgentDirByPath, scrubbedValues, }); if (authStoreChanged) { @@ -322,11 +342,11 @@ function applyConfigTargetMutations(params: { throw new Error(`Missing required agentId for auth-profiles target ${target.path}.`); } params.changedFiles.add( - resolveAuthStorePathForAgent({ + resolveAuthStoreTargetForAgent({ nextConfig: params.nextConfig, stateDir: params.stateDir, agentId, - }), + }).path, ); } continue; @@ -370,6 +390,7 @@ function applyConfigTargetMutations(params: { providerTargets, configChanged, authStoreByPath: params.authStoreByPath, + authStoreAgentDirByPath: params.authStoreAgentDirByPath, }; } @@ -379,6 +400,7 @@ function scrubAuthStoresForProviderTargets(params: { providerTargets: Set; scrubbedValues: Set; authStoreByPath: Map>; + authStoreAgentDirByPath: Map; changedFiles: Set; warnings: string[]; enabled: boolean; @@ -387,9 +409,13 @@ function scrubAuthStoresForProviderTargets(params: { return params.authStoreByPath; } - for (const authStorePath of listAuthProfileStorePaths(params.nextConfig, params.stateDir)) { + for (const target of listAuthProfileStoreTargets(params.nextConfig, params.stateDir)) { + const { agentDir, path: authStorePath } = target; const existing = params.authStoreByPath.get(authStorePath); - const parsed = existing ?? readJsonObjectIfExists(authStorePath).value; + if (!existing && !fs.existsSync(authStorePath)) { + continue; + } + const parsed = existing ?? loadPersistedAuthProfileStore(agentDir); if (!parsed || !isRecord(parsed.profiles)) { continue; } @@ -429,6 +455,7 @@ function scrubAuthStoresForProviderTargets(params: { } if (mutated) { params.authStoreByPath.set(authStorePath, nextStore); + params.authStoreAgentDirByPath.set(authStorePath, agentDir); params.changedFiles.add(authStorePath); } } @@ -452,43 +479,59 @@ function resolveAuthStoreForTarget(params: { nextConfig: OpenClawConfig; stateDir: string; authStoreByPath: Map>; + authStoreAgentDirByPath: Map; }): { path: string; store: MutableAuthProfileStore } { const agentId = (params.target.agentId ?? "").trim(); if (!agentId) { throw new Error(`Missing required agentId for auth-profiles target ${params.target.path}.`); } - const authStorePath = resolveAuthStorePathForAgent({ + const authStoreTarget = resolveAuthStoreTargetForAgent({ nextConfig: params.nextConfig, stateDir: params.stateDir, agentId, }); + const authStorePath = authStoreTarget.path; const existing = params.authStoreByPath.get(authStorePath); - const loaded = existing ?? readJsonObjectIfExists(authStorePath).value; + const loaded = existing ?? loadPersistedAuthProfileStore(authStoreTarget.agentDir); const store = ensureMutableAuthStore(isRecord(loaded) ? loaded : undefined); params.authStoreByPath.set(authStorePath, store); + params.authStoreAgentDirByPath.set(authStorePath, authStoreTarget.agentDir); return { path: authStorePath, store }; } -function resolveAuthStorePathForAgent(params: { +function resolveAuthStoreTargetForAgent(params: { nextConfig: OpenClawConfig; stateDir: string; agentId: string; -}): string { +}): { agentDir: string; path: string } { const normalizedAgentId = normalizeAgentId(params.agentId); const configuredAgentDir = resolveAgentConfig( params.nextConfig, normalizedAgentId, )?.agentDir?.trim(); if (configuredAgentDir) { - return resolveUserPath(resolveAuthStorePath(configuredAgentDir)); + const agentDir = resolveUserPath(configuredAgentDir); + registerResolvedAgentDir({ agentId: normalizedAgentId, agentDir }); + return { agentDir, path: resolveAuthProfileDatabasePath(agentDir) }; } - return path.join( + const agentDir = path.join( resolveUserPath(params.stateDir), "agents", normalizedAgentId, "agent", - "auth-profiles.json", ); + registerResolvedAgentDir({ agentId: normalizedAgentId, agentDir }); + return { agentDir, path: resolveAuthProfileDatabasePath(agentDir) }; +} + +function listAuthProfileStoreTargets( + config: OpenClawConfig, + stateDir: string, +): Array<{ agentDir: string; path: string }> { + return listAuthProfileStoreAgentDirs(config, stateDir).map((agentDir) => ({ + agentDir, + path: resolveAuthProfileDatabasePath(agentDir), + })); } function ensureAuthProfileContainer(params: { @@ -548,6 +591,7 @@ function applyAuthProfileTargetMutation(params: { nextConfig: OpenClawConfig; stateDir: string; authStoreByPath: Map>; + authStoreAgentDirByPath: Map; scrubbedValues: Set; }): boolean { if (params.resolved.entry.configFile !== "auth-profiles.json") { @@ -558,6 +602,7 @@ function applyAuthProfileTargetMutation(params: { nextConfig: params.nextConfig, stateDir: params.stateDir, authStoreByPath: params.authStoreByPath, + authStoreAgentDirByPath: params.authStoreAgentDirByPath, }); let changed = ensureAuthProfileContainer({ target: params.target, @@ -703,7 +748,7 @@ async function validateProjectedSecretsState(params: { // whole-runtime check. includeAuthStoreRefs: params.write || params.authStoreByPath.size > 0, loadAuthStore: (agentDir?: string) => { - const storePath = resolveUserPath(resolveAuthStorePath(agentDir)); + const storePath = resolveUserPath(resolveAuthProfileDatabasePath(agentDir)); const override = authStoreLookup.get(storePath); if (override) { return ( @@ -809,18 +854,23 @@ export async function runSecretsApply(params: { const io = createSecretsConfigIO({ env }); const snapshots = new Map(); + const authStoreSnapshots = new Map(); const capture = (pathname: string) => { if (!snapshots.has(pathname)) { snapshots.set(pathname, captureFileSnapshot(pathname)); } }; + const captureAuthStore = (pathname: string, agentDir: string) => { + if (!authStoreSnapshots.has(pathname)) { + authStoreSnapshots.set(pathname, { + agentDir, + store: loadPersistedAuthProfileStore(agentDir), + }); + } + }; capture(projected.configPath); const writes: ApplyWrite[] = []; - for (const [pathname, value] of projected.authStoreByPath.entries()) { - capture(pathname); - writes.push(toJsonWrite(pathname, value)); - } for (const [pathname, value] of projected.authJsonByPath.entries()) { capture(pathname); writes.push(toJsonWrite(pathname, value)); @@ -833,6 +883,9 @@ export async function runSecretsApply(params: { mode: 0o600, }); } + for (const [pathname, agentDir] of projected.authStoreAgentDirByPath.entries()) { + captureAuthStore(pathname, agentDir); + } try { await replaceConfigFile({ @@ -845,6 +898,13 @@ export async function runSecretsApply(params: { for (const writeLocal of writes) { writeTextFileAtomic(writeLocal.path, writeLocal.content, writeLocal.mode); } + for (const [pathname, value] of projected.authStoreByPath.entries()) { + const agentDir = projected.authStoreAgentDirByPath.get(pathname); + const store = coercePersistedAuthProfileStore(value); + if (agentDir && store) { + saveAuthProfileStore(store, agentDir); + } + } } catch (err) { // Apply can touch multiple files; restore captured snapshots so partial writes do not leave // config/auth/env stores out of sync when a later write fails. @@ -855,6 +915,19 @@ export async function runSecretsApply(params: { // Best effort only; preserve original error. } } + for (const snapshot of authStoreSnapshots.values()) { + try { + if (snapshot.store) { + saveAuthProfileStore(snapshot.store, snapshot.agentDir, { + syncExternalCli: false, + }); + } else { + deletePersistedAuthProfileStoreRaw(snapshot.agentDir); + } + } catch { + // Best effort only; preserve original error. + } + } throw new Error(`Secrets apply failed: ${String(err)}`, { cause: err }); } diff --git a/src/secrets/audit.test.ts b/src/secrets/audit.test.ts index 6f9bc26e67e2..8b5646f6caf4 100644 --- a/src/secrets/audit.test.ts +++ b/src/secrets/audit.test.ts @@ -2,12 +2,20 @@ import fs from "node:fs/promises"; import os from "node:os"; import path from "node:path"; import { afterEach, beforeEach, describe, expect, it } from "vitest"; +import { + resolveAuthProfileDatabasePath, + writePersistedAuthProfileStoreRaw, +} from "../agents/auth-profiles/sqlite.js"; +import { saveAuthProfileStore } from "../agents/auth-profiles/store.js"; +import type { AuthProfileStore } from "../agents/auth-profiles/types.js"; +import { closeOpenClawAgentDatabasesForTest } from "../state/openclaw-agent-db.js"; import { runSecretsAudit } from "./audit.js"; type AuditFixture = { rootDir: string; stateDir: string; configPath: string; + agentDir: string; authStorePath: string; authJsonPath: string; modelsPath: string; @@ -29,9 +37,21 @@ function countNonEmptyLines(value: string): number { } async function writeJsonFile(filePath: string, value: unknown): Promise { + if (path.basename(filePath) === "openclaw-agent.sqlite") { + saveAuthProfileStore(value as AuthProfileStore, path.dirname(filePath), { + filterExternalAuthProfiles: false, + syncExternalCli: false, + }); + return; + } await fs.writeFile(filePath, `${JSON.stringify(value, null, 2)}\n`, "utf8"); } +async function removeAuthStore(fixture: AuditFixture): Promise { + closeOpenClawAgentDatabasesForTest(); + await fs.rm(fixture.authStorePath, { force: true }); +} + async function writeExecResolverShellScript(params: { scriptPath: string; logPath: string; @@ -128,18 +148,20 @@ async function createAuditFixture(): Promise { const rootDir = await fs.mkdtemp(path.join(os.tmpdir(), "openclaw-secrets-audit-")); const stateDir = path.join(rootDir, ".openclaw"); const configPath = path.join(stateDir, "openclaw.json"); - const authStorePath = path.join(stateDir, "agents", "main", "agent", "auth-profiles.json"); - const authJsonPath = path.join(stateDir, "agents", "main", "agent", "auth.json"); - const modelsPath = path.join(stateDir, "agents", "main", "agent", "models.json"); + const agentDir = path.join(stateDir, "agents", "main", "agent"); + const authStorePath = resolveAuthProfileDatabasePath(agentDir); + const authJsonPath = path.join(agentDir, "auth.json"); + const modelsPath = path.join(agentDir, "models.json"); const envPath = path.join(stateDir, ".env"); await fs.mkdir(path.dirname(configPath), { recursive: true }); - await fs.mkdir(path.dirname(authStorePath), { recursive: true }); + await fs.mkdir(agentDir, { recursive: true }); return { rootDir, stateDir, configPath, + agentDir, authStorePath, authJsonPath, modelsPath, @@ -239,6 +261,7 @@ describe("secrets audit", () => { }); afterEach(async () => { + closeOpenClawAgentDatabasesForTest(); await fs.rm(fixture.rootDir, { recursive: true, force: true }); }); @@ -252,7 +275,7 @@ describe("secrets audit", () => { }); it("does not mutate legacy auth.json during audit", async () => { - await fs.rm(fixture.authStorePath, { force: true }); + await removeAuthStore(fixture); await writeJsonFile(fixture.authJsonPath, { openai: { type: "api_key", @@ -268,11 +291,9 @@ describe("secrets audit", () => { }); it("reports malformed sidecar JSON as findings instead of crashing", async () => { - await fs.writeFile(fixture.authStorePath, "{invalid-json", "utf8"); await fs.writeFile(fixture.authJsonPath, "{invalid-json", "utf8"); const report = await runSecretsAudit({ env: fixture.env }); - expectFindingFile(report, fixture.authStorePath); expectFindingFile(report, fixture.authJsonPath); expectFindingCode(report, "REF_UNRESOLVED"); }); @@ -302,7 +323,7 @@ describe("secrets audit", () => { }, ], }); - await fs.rm(fixture.authStorePath, { force: true }); + await removeAuthStore(fixture); await fs.writeFile(fixture.envPath, "", "utf8"); const report = await runSecretsAudit({ env: fixture.env }); @@ -344,7 +365,7 @@ describe("secrets audit", () => { }, ], }); - await fs.rm(fixture.authStorePath, { force: true }); + await removeAuthStore(fixture); await fs.writeFile(fixture.envPath, "", "utf8"); const report = await runSecretsAudit({ env: fixture.env, allowExec: true }); @@ -408,7 +429,7 @@ describe("secrets audit", () => { )}\n`, "utf8", ); - await fs.rm(fixture.authStorePath, { force: true }); + await removeAuthStore(fixture); await fs.writeFile(fixture.envPath, "", "utf8"); const report = await runSecretsAudit({ env: fixture.env, allowExec: true }); @@ -625,17 +646,20 @@ describe("secrets audit", () => { }); it("still flags auth profile plaintext when an explicit ref is also configured", async () => { - await writeJsonFile(fixture.authStorePath, { - version: 1, - profiles: { - "openai:default": { - type: "api_key", - provider: "openai", - key: "sk-leftover-plaintext", // pragma: allowlist secret - keyRef: { source: "env", id: "OPENAI_API_KEY" }, + writePersistedAuthProfileStoreRaw( + { + version: 1, + profiles: { + "openai:default": { + type: "api_key", + provider: "openai", + key: "sk-leftover-plaintext", // pragma: allowlist secret + keyRef: { source: "env", id: "OPENAI_API_KEY" }, + }, }, }, - }); + fixture.agentDir, + ); const report = await runSecretsAudit({ env: fixture.env }); expect( diff --git a/src/secrets/audit.ts b/src/secrets/audit.ts index c02c0e44eb35..bcc1cf7548ae 100644 --- a/src/secrets/audit.ts +++ b/src/secrets/audit.ts @@ -1,6 +1,10 @@ import fs from "node:fs"; import os from "node:os"; import path from "node:path"; +import { + readPersistedAuthProfileStoreRaw, + resolveAuthProfileDatabasePath, +} from "../agents/auth-profiles/sqlite.js"; import { isNonSecretApiKeyMarker, isSecretRefHeaderValueMarker, @@ -31,7 +35,7 @@ import { import { isNonEmptyString, isRecord } from "./shared.js"; import { listAgentModelsJsonPaths, - listAuthProfileStorePaths, + listAuthProfileStoreAgentDirs, listLegacyAuthJsonPaths, parseEnvAssignmentValue, readJsonObjectIfExists, @@ -231,29 +235,19 @@ function collectConfigSecrets(params: { } function collectAuthStoreSecrets(params: { - authStorePath: string; + agentDir: string; collector: AuditCollector; defaults?: SecretDefaults; }): void { - if (!fs.existsSync(params.authStorePath)) { + const authStorePath = resolveAuthProfileDatabasePath(params.agentDir); + if (!fs.existsSync(authStorePath)) { return; } - params.collector.filesScanned.add(params.authStorePath); - const parsedResult = readJsonObjectIfExists(params.authStorePath); - if (parsedResult.error) { - addFinding(params.collector, { - code: "REF_UNRESOLVED", - severity: "error", - file: params.authStorePath, - jsonPath: "", - message: `Invalid JSON in auth-profiles store: ${parsedResult.error}`, - }); - return; - } - const parsed = parsedResult.value; - if (!parsed || !isRecord(parsed.profiles)) { + const parsed = readPersistedAuthProfileStoreRaw(params.agentDir); + if (!isRecord(parsed) || !isRecord(parsed.profiles)) { return; } + params.collector.filesScanned.add(authStorePath); for (const entry of iterateAuthProfileCredentials(parsed.profiles)) { if (entry.kind === "api_key" || entry.kind === "token") { const { ref } = resolveSecretInputRef({ @@ -264,7 +258,7 @@ function collectAuthStoreSecrets(params: { const authoredValueRef = coerceSecretRef(entry.value, params.defaults); if (ref) { params.collector.refAssignments.push({ - file: params.authStorePath, + file: authStorePath, path: `profiles.${entry.profileId}.${entry.valueField}`, ref, expected: "string", @@ -279,7 +273,7 @@ function collectAuthStoreSecrets(params: { addFinding(params.collector, { code: "PLAINTEXT_FOUND", severity: "warn", - file: params.authStorePath, + file: authStorePath, jsonPath: `profiles.${entry.profileId}.${entry.valueField}`, message: entry.kind === "api_key" @@ -296,7 +290,7 @@ function collectAuthStoreSecrets(params: { addFinding(params.collector, { code: "LEGACY_RESIDUE", severity: "info", - file: params.authStorePath, + file: authStorePath, jsonPath: `profiles.${entry.profileId}`, message: "OAuth credentials are present (out of scope for static SecretRef migration).", provider: entry.provider, @@ -650,9 +644,9 @@ export async function runSecretsAudit( configPath, collector, }); - for (const authStorePath of listAuthProfileStorePaths(config, stateDir)) { + for (const agentDir of listAuthProfileStoreAgentDirs(config, stateDir)) { collectAuthStoreSecrets({ - authStorePath, + agentDir, collector, defaults, }); diff --git a/src/secrets/auth-store-paths.ts b/src/secrets/auth-store-paths.ts index a6f67819b65c..f5648149570c 100644 --- a/src/secrets/auth-store-paths.ts +++ b/src/secrets/auth-store-paths.ts @@ -1,19 +1,18 @@ import fs from "node:fs"; import path from "node:path"; import { listAgentIds, resolveAgentDir } from "../agents/agent-scope.js"; -import { resolveAuthStorePath } from "../agents/auth-profiles/paths.js"; import type { OpenClawConfig } from "../config/types.openclaw.js"; import { resolveUserPath } from "../utils.js"; /** - * Lists deduplicated auth-profile store paths that may contain SecretRefs. + * Lists deduplicated auth-profile store agent dirs that may contain SecretRefs. * Covers implicit main, discovered state-dir agents, and config-declared agent dirs. */ -export function listAuthProfileStorePaths(config: OpenClawConfig, stateDir: string): string[] { +export function listAuthProfileStoreAgentDirs(config: OpenClawConfig, stateDir: string): string[] { const paths = new Set(); // Scope default auth store discovery to the provided stateDir instead of // ambient process env, so scans do not include unrelated host-global stores. - paths.add(path.join(resolveUserPath(stateDir), "agents", "main", "agent", "auth-profiles.json")); + paths.add(path.join(resolveUserPath(stateDir), "agents", "main", "agent")); const agentsRoot = path.join(resolveUserPath(stateDir), "agents"); if (fs.existsSync(agentsRoot)) { @@ -21,20 +20,18 @@ export function listAuthProfileStorePaths(config: OpenClawConfig, stateDir: stri if (!entry.isDirectory()) { continue; } - paths.add(path.join(agentsRoot, entry.name, "agent", "auth-profiles.json")); + paths.add(path.join(agentsRoot, entry.name, "agent")); } } // Configured agent dirs may live outside stateDir; include them after state-dir discovery. for (const agentId of listAgentIds(config)) { if (agentId === "main") { - paths.add( - path.join(resolveUserPath(stateDir), "agents", "main", "agent", "auth-profiles.json"), - ); + paths.add(path.join(resolveUserPath(stateDir), "agents", "main", "agent")); continue; } const agentDir = resolveAgentDir(config, agentId); - paths.add(resolveUserPath(resolveAuthStorePath(agentDir))); + paths.add(resolveUserPath(agentDir)); } return [...paths]; diff --git a/src/secrets/runtime-fast-path.ts b/src/secrets/runtime-fast-path.ts index 809a5def534d..35523bee11d9 100644 --- a/src/secrets/runtime-fast-path.ts +++ b/src/secrets/runtime-fast-path.ts @@ -11,6 +11,7 @@ import { AUTH_STATE_FILENAME, LEGACY_AUTH_FILENAME, } from "../agents/auth-profiles/path-constants.js"; +import { resolveAuthProfileDatabasePath } from "../agents/auth-profiles/sqlite.js"; import type { AuthProfileStore } from "../agents/auth-profiles/types.js"; import { resolveOAuthPath } from "../config/paths.js"; import type { OpenClawConfig } from "../config/types.openclaw.js"; @@ -98,6 +99,7 @@ function resolveCandidateAgentDirs(params: { function hasCandidateAuthProfileStoreSource(agentDir: string): boolean { return ( + existsSync(resolveAuthProfileDatabasePath(agentDir)) || existsSync(path.join(agentDir, AUTH_PROFILE_FILENAME)) || existsSync(path.join(agentDir, AUTH_STATE_FILENAME)) || existsSync(path.join(agentDir, LEGACY_AUTH_FILENAME)) diff --git a/src/secrets/runtime-state.test.ts b/src/secrets/runtime-state.test.ts index d417ffa6ad22..9d6ad6114c44 100644 --- a/src/secrets/runtime-state.test.ts +++ b/src/secrets/runtime-state.test.ts @@ -1,11 +1,4 @@ -import fs from "node:fs"; -import os from "node:os"; -import path from "node:path"; import { afterEach, describe, expect, it } from "vitest"; -import { resolveAuthStatePath, resolveAuthStorePath } from "../agents/auth-profiles/paths.js"; -import { writeCachedAuthProfileStore } from "../agents/auth-profiles/store-cache.js"; -import { loadAuthProfileStoreForRuntime } from "../agents/auth-profiles/store.js"; -import type { AuthProfileStore } from "../agents/auth-profiles/types.js"; import { activateSecretsRuntimeSnapshotState, clearSecretsRuntimeSnapshot, @@ -14,19 +7,6 @@ import { type PreparedSecretsRuntimeSnapshot, } from "./runtime-state.js"; -function authStore(key: string): AuthProfileStore { - return { - version: 1, - profiles: { - "openai:default": { - type: "api_key", - provider: "openai", - key, - }, - }, - }; -} - describe("secrets runtime state", () => { const previousStateDir = process.env.OPENCLAW_STATE_DIR; @@ -39,42 +19,6 @@ describe("secrets runtime state", () => { } }); - it("clears loaded auth-profile cache without importing the full secrets runtime", () => { - const root = fs.mkdtempSync(path.join(os.tmpdir(), "openclaw-runtime-state-cache-")); - process.env.OPENCLAW_STATE_DIR = root; - const agentDir = path.join(root, "agents", "default", "agent"); - - try { - fs.mkdirSync(agentDir, { recursive: true }); - const authPath = resolveAuthStorePath(agentDir); - const statePath = resolveAuthStatePath(agentDir); - fs.writeFileSync(authPath, `${JSON.stringify(authStore("sk-new"))}\n`); - const stat = fs.statSync(authPath); - writeCachedAuthProfileStore({ - authPath, - authMtimeMs: stat.mtimeMs, - stateMtimeMs: fs.existsSync(statePath) ? fs.statSync(statePath).mtimeMs : null, - store: authStore("sk-old"), - }); - - expect( - loadAuthProfileStoreForRuntime(agentDir, { syncExternalCli: false }).profiles[ - "openai:default" - ], - ).toMatchObject({ key: "sk-old" }); - - clearSecretsRuntimeSnapshot(); - - expect( - loadAuthProfileStoreForRuntime(agentDir, { syncExternalCli: false }).profiles[ - "openai:default" - ], - ).toMatchObject({ key: "sk-new" }); - } finally { - fs.rmSync(root, { recursive: true, force: true }); - } - }); - it("exposes the active config pair for hot paths without requiring the full snapshot", () => { const snapshot: PreparedSecretsRuntimeSnapshot = { sourceConfig: { agents: { list: [{ id: "source" }] } }, diff --git a/src/secrets/runtime-state.ts b/src/secrets/runtime-state.ts index 6892682ec10e..84e8ab696101 100644 --- a/src/secrets/runtime-state.ts +++ b/src/secrets/runtime-state.ts @@ -3,7 +3,6 @@ import { getRuntimeAuthProfileStoreSnapshot, replaceRuntimeAuthProfileStoreSnapshots, } from "../agents/auth-profiles/runtime-snapshots.js"; -import { clearLoadedAuthStoreCache } from "../agents/auth-profiles/store-cache.js"; import type { AuthProfileStore } from "../agents/auth-profiles/types.js"; import { clearRuntimeConfigSnapshot, @@ -202,7 +201,6 @@ export function clearSecretsRuntimeSnapshot(): void { setRuntimeConfigSnapshotRefreshHandler(null); clearRuntimeConfigSnapshot(); clearRuntimeAuthProfileStoreSnapshots(); - clearLoadedAuthStoreCache(); for (const clearHook of clearHooks) { clearHook(); } diff --git a/src/secrets/runtime.fast-path.test.ts b/src/secrets/runtime.fast-path.test.ts index ad5968b6ccad..f6a2ff3f66d7 100644 --- a/src/secrets/runtime.fast-path.test.ts +++ b/src/secrets/runtime.fast-path.test.ts @@ -4,11 +4,12 @@ import path from "node:path"; import { afterEach, describe, expect, it, vi } from "vitest"; import { resolveDefaultAgentDir } from "../agents/agent-scope-config.js"; import type { AuthProfileStore } from "../agents/auth-profiles.js"; -import { AUTH_PROFILE_FILENAME } from "../agents/auth-profiles/path-constants.js"; +import { saveAuthProfileStore } from "../agents/auth-profiles/store.js"; import { clearConfigCache, clearRuntimeConfigSnapshot } from "../config/config.js"; import { resolveOAuthPath } from "../config/paths.js"; import { createEmptyPluginRegistry } from "../plugins/registry.js"; import { setActivePluginRegistry } from "../plugins/runtime.js"; +import { closeOpenClawAgentDatabasesForTest } from "../state/openclaw-agent-db.js"; import { clearSecretsRuntimeSnapshot } from "./runtime.js"; import { asConfig } from "./runtime.test-support.js"; @@ -56,9 +57,8 @@ function requireGatewayAuth( function writeAuthProfileStore(agentDir: string): void { mkdirSync(agentDir, { recursive: true }); - writeFileSync( - path.join(agentDir, AUTH_PROFILE_FILENAME), - `${JSON.stringify({ + saveAuthProfileStore( + { version: 1, profiles: { "openai:default": { @@ -67,7 +67,9 @@ function writeAuthProfileStore(agentDir: string): void { key: "sk-test", }, }, - })}\n`, + }, + agentDir, + { filterExternalAuthProfiles: false, syncExternalCli: false }, ); } @@ -79,6 +81,7 @@ describe("secrets runtime fast path", () => { clearSecretsRuntimeSnapshot(); clearRuntimeConfigSnapshot(); clearConfigCache(); + closeOpenClawAgentDatabasesForTest(); vi.resetModules(); }); diff --git a/src/secrets/storage-scan.ts b/src/secrets/storage-scan.ts index 21bf884af533..a31d84856559 100644 --- a/src/secrets/storage-scan.ts +++ b/src/secrets/storage-scan.ts @@ -2,10 +2,11 @@ import fs from "node:fs"; import path from "node:path"; import { isRecord as isJsonObject } from "@openclaw/normalization-core/record-coerce"; import { listAgentIds, resolveAgentDir } from "../agents/agent-scope.js"; +import { resolveAuthProfileDatabasePath } from "../agents/auth-profiles/sqlite.js"; import type { OpenClawConfig } from "../config/types.openclaw.js"; import { formatErrorMessage } from "../infra/errors.js"; import { resolveUserPath } from "../utils.js"; -import { listAuthProfileStorePaths as listAuthProfileStorePathsFromAuthStorePaths } from "./auth-store-paths.js"; +import { listAuthProfileStoreAgentDirs as listAuthProfileStoreAgentDirsFromAuthStorePaths } from "./auth-store-paths.js"; import { parseEnvValue } from "./shared.js"; /** Parses one .env assignment value using the shared shell-ish env parser. */ @@ -15,7 +16,13 @@ export function parseEnvAssignmentValue(raw: string): string { /** Lists canonical auth-profile stores visible to secrets audit/apply storage scanners. */ export function listAuthProfileStorePaths(config: OpenClawConfig, stateDir: string): string[] { - return listAuthProfileStorePathsFromAuthStorePaths(config, stateDir); + return listAuthProfileStoreAgentDirs(config, stateDir).map((agentDir) => + resolveAuthProfileDatabasePath(agentDir), + ); +} + +export function listAuthProfileStoreAgentDirs(config: OpenClawConfig, stateDir: string): string[] { + return listAuthProfileStoreAgentDirsFromAuthStorePaths(config, stateDir); } /** Lists legacy per-agent auth.json stores that can contain static credentials. */ diff --git a/src/security/audit-extra.async.test.ts b/src/security/audit-extra.async.test.ts index 4b2d76c2faea..ea567bee5fcd 100644 --- a/src/security/audit-extra.async.test.ts +++ b/src/security/audit-extra.async.test.ts @@ -7,6 +7,7 @@ import * as skillScanner from "../skills/security/scanner.js"; import { collectInstalledSkillsCodeSafetyFindings, collectPluginsCodeSafetyFindings, + collectStateDeepFilesystemFindings, } from "./audit-extra.async.js"; vi.mock("../skills/loading/workspace.js", () => ({ @@ -270,4 +271,33 @@ description: test skill scanSpy.mockRestore(); } }); + + it("audits canonical auth profile SQLite store permissions", async () => { + const stateDir = await makeTmpDir("audit-auth-sqlite-perms"); + const agentDir = path.join(stateDir, "agents", "main", "agent"); + await fs.mkdir(agentDir, { recursive: true }); + const databasePath = path.join(agentDir, "openclaw-agent.sqlite"); + for (const targetPath of [databasePath, `${databasePath}-wal`, `${databasePath}-shm`]) { + await fs.writeFile(targetPath, "sqlite\n", "utf-8"); + await fs.chmod(targetPath, 0o644); + } + + const findings = await collectStateDeepFilesystemFindings({ + cfg: {} as OpenClawConfig, + env: {}, + stateDir, + platform: "linux", + }); + + const readableAuthTargets = findings + .filter((finding) => finding.checkId === "fs.auth_profiles.perms_readable") + .map((finding) => finding.detail); + expect(readableAuthTargets).toEqual( + expect.arrayContaining([ + expect.stringContaining("openclaw-agent.sqlite"), + expect.stringContaining("openclaw-agent.sqlite-wal"), + expect.stringContaining("openclaw-agent.sqlite-shm"), + ]), + ); + }); }); diff --git a/src/security/audit-extra.async.ts b/src/security/audit-extra.async.ts index d0564b2efe7c..585796c05adb 100644 --- a/src/security/audit-extra.async.ts +++ b/src/security/audit-extra.async.ts @@ -15,6 +15,7 @@ import { normalizeTrimmedStringList, uniqueStrings, } from "@openclaw/normalization-core/string-normalization"; +import { resolveAuthProfileDatabaseFilePaths } from "../agents/auth-profiles/sqlite.js"; import { formatCliCommand } from "../cli/command-format.js"; import { MANIFEST_KEY } from "../compat/legacy-names.js"; import type { OpenClawConfig, ConfigFileSnapshot } from "../config/config.js"; @@ -708,41 +709,49 @@ export async function collectStateDeepFilesystemFindings(params: { for (const agentId of ids) { const agentDir = path.join(params.stateDir, "agents", agentId, "agent"); - const authPath = path.join(agentDir, "auth-profiles.json"); - const authPerms = await inspectPathPermissions(authPath, { - env: params.env, - platform: params.platform, - exec: params.execIcacls, - }); - if (authPerms.ok) { - if (authPerms.worldWritable || authPerms.groupWritable) { - findings.push({ - checkId: "fs.auth_profiles.perms_writable", - severity: "critical", - title: "auth-profiles.json is writable by others", - detail: `${formatPermissionDetail(authPath, authPerms)}; another user could inject credentials.`, - remediation: formatPermissionRemediation({ - targetPath: authPath, - perms: authPerms, - isDir: false, - posixMode: 0o600, - env: params.env, - }), - }); - } else if (authPerms.worldReadable || authPerms.groupReadable) { - findings.push({ - checkId: "fs.auth_profiles.perms_readable", - severity: "warn", - title: "auth-profiles.json is readable by others", - detail: `${formatPermissionDetail(authPath, authPerms)}; auth-profiles.json contains API keys and OAuth tokens.`, - remediation: formatPermissionRemediation({ - targetPath: authPath, - perms: authPerms, - isDir: false, - posixMode: 0o600, - env: params.env, - }), - }); + const authTargets = [ + { path: path.join(agentDir, "auth-profiles.json"), label: "legacy auth-profiles.json" }, + ...resolveAuthProfileDatabaseFilePaths(agentDir).map((targetPath) => ({ + path: targetPath, + label: "auth profile SQLite store", + })), + ]; + for (const authTarget of authTargets) { + const authPerms = await inspectPathPermissions(authTarget.path, { + env: params.env, + platform: params.platform, + exec: params.execIcacls, + }); + if (authPerms.ok) { + if (authPerms.worldWritable || authPerms.groupWritable) { + findings.push({ + checkId: "fs.auth_profiles.perms_writable", + severity: "critical", + title: `${authTarget.label} is writable by others`, + detail: `${formatPermissionDetail(authTarget.path, authPerms)}; another user could inject credentials.`, + remediation: formatPermissionRemediation({ + targetPath: authTarget.path, + perms: authPerms, + isDir: false, + posixMode: 0o600, + env: params.env, + }), + }); + } else if (authPerms.worldReadable || authPerms.groupReadable) { + findings.push({ + checkId: "fs.auth_profiles.perms_readable", + severity: "warn", + title: `${authTarget.label} is readable by others`, + detail: `${formatPermissionDetail(authTarget.path, authPerms)}; auth profile storage contains API keys and OAuth tokens.`, + remediation: formatPermissionRemediation({ + targetPath: authTarget.path, + perms: authPerms, + isDir: false, + posixMode: 0o600, + env: params.env, + }), + }); + } } } diff --git a/src/security/fix.test.ts b/src/security/fix.test.ts index d690e0af7740..b2f1ec7cf03d 100644 --- a/src/security/fix.test.ts +++ b/src/security/fix.test.ts @@ -267,6 +267,10 @@ describe("security fix", () => { const agentDir = path.join(stateDir, "agents", "main", "agent"); await fs.mkdir(agentDir, { recursive: true }); + const authDatabasePath = path.join(agentDir, "openclaw-agent.sqlite"); + await fs.writeFile(authDatabasePath, "sqlite\n", "utf-8"); + await fs.writeFile(`${authDatabasePath}-wal`, "wal\n", "utf-8"); + await fs.writeFile(`${authDatabasePath}-shm`, "shm\n", "utf-8"); const authProfilesPath = path.join(agentDir, "auth-profiles.json"); await fs.writeFile(authProfilesPath, "{}\n", "utf-8"); await fs.chmod(authProfilesPath, 0o644); @@ -298,6 +302,9 @@ describe("security fix", () => { { path: allowFromPath, mode: 0o600, require: "file" }, { path: path.join(stateDir, "agents", "main"), mode: 0o700, require: "dir" }, { path: agentDir, mode: 0o700, require: "dir" }, + { path: authDatabasePath, mode: 0o600, require: "file" }, + { path: `${authDatabasePath}-wal`, mode: 0o600, require: "file" }, + { path: `${authDatabasePath}-shm`, mode: 0o600, require: "file" }, { path: authProfilesPath, mode: 0o600, require: "file" }, { path: sessionsDir, mode: 0o700, require: "dir" }, { path: sessionsStorePath, mode: 0o600, require: "file" }, diff --git a/src/security/fix.ts b/src/security/fix.ts index eccb493a6114..7daa4445eb4a 100644 --- a/src/security/fix.ts +++ b/src/security/fix.ts @@ -1,6 +1,7 @@ import fs from "node:fs/promises"; import path from "node:path"; import { resolveDefaultAgentId } from "../agents/agent-scope.js"; +import { resolveAuthProfileDatabaseFilePaths } from "../agents/auth-profiles/sqlite.js"; import type { ChannelPlugin } from "../channels/plugins/types.plugin.js"; import { createConfigIO, replaceConfigFile } from "../config/config.js"; import { collectIncludePathsRecursive } from "../config/includes-scan.js"; @@ -362,6 +363,9 @@ export async function collectSecurityPermissionTargets(params: { targets.push({ path: agentRoot, mode: 0o700, require: "dir" }); targets.push({ path: agentDir, mode: 0o700, require: "dir" }); + for (const databasePath of resolveAuthProfileDatabaseFilePaths(agentDir)) { + targets.push({ path: databasePath, mode: 0o600, require: "file" }); + } const authPath = path.join(agentDir, "auth-profiles.json"); targets.push({ path: authPath, mode: 0o600, require: "file" }); diff --git a/src/state/openclaw-agent-db.generated.d.ts b/src/state/openclaw-agent-db.generated.d.ts index 5c960be4fd7c..38f5670a6361 100644 --- a/src/state/openclaw-agent-db.generated.d.ts +++ b/src/state/openclaw-agent-db.generated.d.ts @@ -10,6 +10,18 @@ export type Generated = ? ColumnType : ColumnType; +export interface AuthProfileState { + state_json: string; + state_key: string; + updated_at: number; +} + +export interface AuthProfileStore { + store_json: string; + store_key: string; + updated_at: number; +} + export interface CacheEntries { blob: Uint8Array | null; expires_at: number | null; @@ -30,6 +42,8 @@ export interface SchemaMeta { } export interface DB { + auth_profile_state: AuthProfileState; + auth_profile_store: AuthProfileStore; cache_entries: CacheEntries; schema_meta: SchemaMeta; } diff --git a/src/state/openclaw-agent-schema.generated.ts b/src/state/openclaw-agent-schema.generated.ts index 1fced176fb33..5f5a39957a25 100644 --- a/src/state/openclaw-agent-schema.generated.ts +++ b/src/state/openclaw-agent-schema.generated.ts @@ -28,4 +28,16 @@ CREATE INDEX IF NOT EXISTS idx_agent_cache_expiry WHERE expires_at IS NOT NULL; CREATE INDEX IF NOT EXISTS idx_agent_cache_updated - ON cache_entries(scope, updated_at DESC, key);\n`; + ON cache_entries(scope, updated_at DESC, key); + +CREATE TABLE IF NOT EXISTS auth_profile_store ( + store_key TEXT NOT NULL PRIMARY KEY, + store_json TEXT NOT NULL, + updated_at INTEGER NOT NULL +); + +CREATE TABLE IF NOT EXISTS auth_profile_state ( + state_key TEXT NOT NULL PRIMARY KEY, + state_json TEXT NOT NULL, + updated_at INTEGER NOT NULL +);\n`; diff --git a/src/state/openclaw-agent-schema.sql b/src/state/openclaw-agent-schema.sql index d69b4934dc48..e212a1a66a57 100644 --- a/src/state/openclaw-agent-schema.sql +++ b/src/state/openclaw-agent-schema.sql @@ -24,3 +24,15 @@ CREATE INDEX IF NOT EXISTS idx_agent_cache_expiry CREATE INDEX IF NOT EXISTS idx_agent_cache_updated ON cache_entries(scope, updated_at DESC, key); + +CREATE TABLE IF NOT EXISTS auth_profile_store ( + store_key TEXT NOT NULL PRIMARY KEY, + store_json TEXT NOT NULL, + updated_at INTEGER NOT NULL +); + +CREATE TABLE IF NOT EXISTS auth_profile_state ( + state_key TEXT NOT NULL PRIMARY KEY, + state_json TEXT NOT NULL, + updated_at INTEGER NOT NULL +); diff --git a/src/test-utils/openclaw-test-state.test.ts b/src/test-utils/openclaw-test-state.test.ts index 01772548a746..90a98e09cd00 100644 --- a/src/test-utils/openclaw-test-state.test.ts +++ b/src/test-utils/openclaw-test-state.test.ts @@ -1,6 +1,7 @@ import fs from "node:fs/promises"; import path from "node:path"; import { describe, expect, it } from "vitest"; +import { loadPersistedAuthProfileStore } from "../agents/auth-profiles/persisted.js"; import { createOpenClawTestState, withOpenClawTestState } from "./openclaw-test-state.js"; async function expectPathMissing(targetPath: string): Promise { @@ -143,13 +144,10 @@ describe("openclaw test state", () => { }, }); - expect(profilePath).toBe(path.join(state.agentDir(), "auth-profiles.json")); - const profiles = JSON.parse(await fs.readFile(profilePath, "utf8")) as { - version?: unknown; - profiles?: Record; - }; - expect(profiles.version).toBe(1); - expect(profiles.profiles?.["openai:test"]?.provider).toBe("openai"); + expect(profilePath).toBe(path.join(state.agentDir(), "openclaw-agent.sqlite")); + const profiles = loadPersistedAuthProfileStore(state.agentDir()); + expect(profiles?.version).toBe(1); + expect(profiles?.profiles["openai:test"]?.provider).toBe("openai"); }, ); }); diff --git a/src/test-utils/openclaw-test-state.ts b/src/test-utils/openclaw-test-state.ts index a6dc1e5e7db8..45b5a773da2a 100644 --- a/src/test-utils/openclaw-test-state.ts +++ b/src/test-utils/openclaw-test-state.ts @@ -2,6 +2,9 @@ import fs from "node:fs/promises"; import os from "node:os"; import path from "node:path"; import { uniqueStrings } from "@openclaw/normalization-core/string-normalization"; +import { resolveAuthProfileDatabasePath } from "../agents/auth-profiles/sqlite.js"; +import { saveAuthProfileStore } from "../agents/auth-profiles/store.js"; +import type { AuthProfileStore } from "../agents/auth-profiles/types.js"; import { captureEnv } from "./env.js"; import { cleanupSessionStateForTest } from "./session-state-cleanup.js"; @@ -299,8 +302,12 @@ export async function createOpenClawTestState( return filePath; }, writeAuthProfiles: (store, agentId = "main") => { - const filePath = path.join(agentDir(agentId), "auth-profiles.json"); - return writeJsonFile(filePath, store); + const targetAgentDir = agentDir(agentId); + saveAuthProfileStore(store as AuthProfileStore, targetAgentDir, { + filterExternalAuthProfiles: false, + syncExternalCli: false, + }); + return Promise.resolve(resolveAuthProfileDatabasePath(targetAgentDir)); }, applyEnv: () => { for (const [key, value] of Object.entries(envVars)) { diff --git a/test/scripts/lint-suppressions.test.ts b/test/scripts/lint-suppressions.test.ts index 2454999ef86d..4fa9527e0fac 100644 --- a/test/scripts/lint-suppressions.test.ts +++ b/test/scripts/lint-suppressions.test.ts @@ -201,7 +201,6 @@ describe("production lint suppressions", () => { "src/cli/test-runtime-capture.ts|typescript/no-unnecessary-type-parameters|1", "src/gateway/test-helpers.server.ts|typescript/no-unnecessary-type-parameters|1", "src/hooks/module-loader.ts|typescript/no-unnecessary-type-parameters|1", - "src/infra/channel-runtime-context.ts|typescript/no-unnecessary-type-parameters|1", "src/infra/exec-approvals-effective.ts|typescript/no-unnecessary-type-parameters|1", "src/infra/json-file.ts|typescript-eslint/no-unnecessary-type-parameters|1", "src/infra/outbound/send-deps.ts|typescript/no-unnecessary-type-parameters|1",