diff --git a/extensions/qqbot/src/bridge/runtime.ts b/extensions/qqbot/src/bridge/runtime.ts index c50c5d2d4414..8b976a7ba033 100644 --- a/extensions/qqbot/src/bridge/runtime.ts +++ b/extensions/qqbot/src/bridge/runtime.ts @@ -4,11 +4,14 @@ import type { GatewayPluginRuntime } from "../engine/gateway/types.js"; import { setOpenClawVersion } from "../engine/messaging/sender.js"; // Single plugin runtime per process — concurrent multi-tenant qqbot runtimes are not supported. -const { setRuntime: _setRuntime, getRuntime: getQQBotRuntime } = - createPluginRuntimeStore({ - pluginId: "qqbot", - errorMessage: "QQBot runtime not initialized", - }); +const { + setRuntime: _setRuntime, + clearRuntime: resetQQBotRuntimeForTest, + getRuntime: getQQBotRuntime, +} = createPluginRuntimeStore({ + pluginId: "qqbot", + errorMessage: "QQBot runtime not initialized", +}); /** Set the QQBot runtime and inject the framework version into the User-Agent. */ function setQQBotRuntime(runtime: PluginRuntime): void { @@ -17,7 +20,7 @@ function setQQBotRuntime(runtime: PluginRuntime): void { setOpenClawVersion(runtime.version); } -export { getQQBotRuntime, setQQBotRuntime }; +export { getQQBotRuntime, resetQQBotRuntimeForTest, setQQBotRuntime }; /** Type-narrowed getter for engine/ modules that need GatewayPluginRuntime. */ export function getQQBotRuntimeForEngine(): GatewayPluginRuntime { diff --git a/extensions/qqbot/src/engine/config/credential-backup.test.ts b/extensions/qqbot/src/engine/config/credential-backup.test.ts index 49baa8765ba7..5aee71953571 100644 --- a/extensions/qqbot/src/engine/config/credential-backup.test.ts +++ b/extensions/qqbot/src/engine/config/credential-backup.test.ts @@ -1,88 +1,174 @@ import fs from "node:fs"; -import { afterEach, beforeEach, describe, expect, it } from "vitest"; -import { getCredentialBackupFile, getLegacyCredentialBackupFile } from "../utils/data-paths.js"; -import { loadCredentialBackup, saveCredentialBackup } from "./credential-backup.js"; +import os from "node:os"; +import path from "node:path"; +import { + createPluginStateSyncKeyedStoreForTests, + resetPluginStateStoreForTests, +} from "openclaw/plugin-sdk/plugin-state-test-runtime"; +import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; +import { + installQQBotRuntimeForStateTests, + resetQQBotStateTestRuntime, +} from "../../test-support/runtime.js"; + +type CredentialBackup = { + accountId: string; + appId: string; + clientSecret: string; + savedAt: string; +}; + +const createdDirs: string[] = []; + +function createTempDir(prefix: string): string { + const dir = fs.mkdtempSync(path.join(os.tmpdir(), prefix)); + createdDirs.push(dir); + return dir; +} + +async function useMockHome(homeDir: string): Promise { + vi.doMock("node:os", async (importOriginal) => { + const actual = await importOriginal(); + return { + ...actual, + default: { ...actual, homedir: () => homeDir }, + homedir: () => homeDir, + }; + }); +} + +function useStateDir(stateDir: string): void { + vi.stubEnv("OPENCLAW_STATE_DIR", stateDir); + installQQBotRuntimeForStateTests(stateDir); +} + +function legacyOsHomeBackupPath(homeDir: string, accountId = "default"): string { + return path.join(homeDir, ".openclaw", "qqbot", "data", `credential-backup-${accountId}.json`); +} + +function writeJson(filePath: string, value: unknown): void { + fs.mkdirSync(path.dirname(filePath), { recursive: true }); + fs.writeFileSync(filePath, `${JSON.stringify(value, null, 2)}\n`); +} + +function readCredentialRows(stateDir: string): CredentialBackup[] { + const store = createPluginStateSyncKeyedStoreForTests("qqbot", { + namespace: "credential-backups", + maxEntries: 1000, + env: { ...process.env, OPENCLAW_STATE_DIR: stateDir }, + }); + return store.entries().map((entry) => entry.value); +} -/** - * These tests write to `~/.openclaw/qqbot/data` under a test-specific - * accountId prefix and clean up after themselves. Mirrors the approach - * used by `platform.test.ts` in the same package. - */ describe("engine/config/credential-backup", () => { - const acct = `test-cb-${process.pid}-${Date.now()}`; - const legacyPath = getLegacyCredentialBackupFile(); - let legacyBackup: string | null = null; - - beforeEach(() => { - // Preserve any legacy backup that might happen to live in the user's - // real home so we can restore it after the test. - legacyBackup = null; - if (fs.existsSync(legacyPath)) { - legacyBackup = fs.readFileSync(legacyPath, "utf8"); - fs.unlinkSync(legacyPath); - } + beforeEach(async () => { + vi.resetModules(); + const stateDir = createTempDir("qqbot-state-"); + const homeDir = createTempDir("qqbot-home-"); + vi.stubEnv("HOME", homeDir); + await useMockHome(homeDir); + useStateDir(stateDir); }); afterEach(() => { - try { - fs.unlinkSync(getCredentialBackupFile(acct)); - } catch { - /* ignore */ - } - if (fs.existsSync(legacyPath)) { - fs.unlinkSync(legacyPath); - } - if (legacyBackup != null) { - fs.writeFileSync(legacyPath, legacyBackup); + resetQQBotStateTestRuntime(); + resetPluginStateStoreForTests(); + vi.doUnmock("node:os"); + vi.resetModules(); + vi.unstubAllEnvs(); + for (const dir of createdDirs.splice(0)) { + fs.rmSync(dir, { recursive: true, force: true }); } }); - it("round-trips a credential snapshot", () => { - saveCredentialBackup(acct, "app-1", "secret-1"); - const loaded = loadCredentialBackup(acct); - expect(loaded?.appId).toBe("app-1"); - expect(loaded?.clientSecret).toBe("secret-1"); - expect(loaded?.accountId).toBe(acct); - expect(fs.existsSync(getCredentialBackupFile(acct))).toBe(true); + it("round-trips a credential snapshot through SQLite without writing JSON", async () => { + const { getCredentialBackupFile } = await import("../utils/data-paths.js"); + const { loadCredentialBackup, saveCredentialBackup } = await import("./credential-backup.js"); + const stateDir = process.env.OPENCLAW_STATE_DIR!; + + saveCredentialBackup("default", "app-1", "secret-1"); + + const loaded = loadCredentialBackup("default"); + expect(loaded).toMatchObject({ + accountId: "default", + appId: "app-1", + clientSecret: "secret-1", + }); + expect(fs.existsSync(getCredentialBackupFile("default"))).toBe(false); + expect(readCredentialRows(stateDir)).toHaveLength(1); }); - it("returns null when no backup exists", () => { - expect(loadCredentialBackup(acct)).toBeNull(); + it("keeps same account IDs isolated across state directories", async () => { + const { loadCredentialBackup, saveCredentialBackup } = await import("./credential-backup.js"); + const stateDirA = process.env.OPENCLAW_STATE_DIR!; + saveCredentialBackup("default", "app-a", "secret-a"); + + const stateDirB = createTempDir("qqbot-state-b-"); + useStateDir(stateDirB); + expect(loadCredentialBackup("default")).toBeNull(); + saveCredentialBackup("default", "app-b", "secret-b"); + + useStateDir(stateDirA); + expect(loadCredentialBackup("default")?.appId).toBe("app-a"); + + useStateDir(stateDirB); + expect(loadCredentialBackup("default")?.appId).toBe("app-b"); }); - it("returns null when legacy backup belongs to a different accountId", () => { - fs.writeFileSync( - legacyPath, - JSON.stringify({ - accountId: "other-acct", - appId: "app-old", - clientSecret: "secret-old", - savedAt: new Date().toISOString(), - }), - ); - expect(loadCredentialBackup(acct)).toBeNull(); + it("imports the state-dir legacy per-account JSON backup once", async () => { + const { getCredentialBackupFile } = await import("../utils/data-paths.js"); + const { loadCredentialBackup } = await import("./credential-backup.js"); + writeJson(getCredentialBackupFile("default"), { + accountId: "default", + appId: "app-old", + clientSecret: "secret-old", + savedAt: new Date().toISOString(), + }); + + const loaded = loadCredentialBackup("default"); + + expect(loaded?.appId).toBe("app-old"); + expect(fs.existsSync(getCredentialBackupFile("default"))).toBe(false); + expect(loadCredentialBackup("default")?.clientSecret).toBe("secret-old"); }); - it("migrates legacy single-file backup to per-account path on load", () => { - fs.writeFileSync( - legacyPath, - JSON.stringify({ - accountId: acct, - appId: "app-1", - clientSecret: "secret-1", - savedAt: new Date().toISOString(), - }), - ); + it("imports the old OS-home JSON backup once", async () => { + const { loadCredentialBackup } = await import("./credential-backup.js"); + const legacyPath = legacyOsHomeBackupPath(process.env.HOME!); + writeJson(legacyPath, { + accountId: "default", + appId: "app-home", + clientSecret: "secret-home", + savedAt: new Date().toISOString(), + }); - const loaded = loadCredentialBackup(acct); - expect(loaded?.appId).toBe("app-1"); + const loaded = loadCredentialBackup("default"); + + expect(loaded?.appId).toBe("app-home"); expect(fs.existsSync(legacyPath)).toBe(false); - expect(fs.existsSync(getCredentialBackupFile(acct))).toBe(true); + expect(loadCredentialBackup("default")?.clientSecret).toBe("secret-home"); }); - it("ignores empty appId/clientSecret on save", () => { - saveCredentialBackup(acct, "", "secret"); - saveCredentialBackup(acct, "app", ""); - expect(fs.existsSync(getCredentialBackupFile(acct))).toBe(false); + it("returns null when the legacy single-file backup belongs to a different accountId", async () => { + const { getLegacyCredentialBackupFile } = await import("../utils/data-paths.js"); + const { loadCredentialBackup } = await import("./credential-backup.js"); + writeJson(getLegacyCredentialBackupFile(), { + accountId: "other-acct", + appId: "app-old", + clientSecret: "secret-old", + savedAt: new Date().toISOString(), + }); + + expect(loadCredentialBackup("default")).toBeNull(); + expect(fs.existsSync(getLegacyCredentialBackupFile())).toBe(true); + }); + + it("ignores empty appId/clientSecret on save", async () => { + const { loadCredentialBackup, saveCredentialBackup } = await import("./credential-backup.js"); + saveCredentialBackup("default", "", "secret"); + saveCredentialBackup("default", "app", ""); + + expect(loadCredentialBackup("default")).toBeNull(); + expect(readCredentialRows(process.env.OPENCLAW_STATE_DIR!)).toHaveLength(0); }); }); diff --git a/extensions/qqbot/src/engine/config/credential-backup.ts b/extensions/qqbot/src/engine/config/credential-backup.ts index 619bcc8c1de4..a3e721218814 100644 --- a/extensions/qqbot/src/engine/config/credential-backup.ts +++ b/extensions/qqbot/src/engine/config/credential-backup.ts @@ -7,28 +7,26 @@ * * Mechanics: * - After each successful gateway start we snapshot the currently - * resolved `appId` / `clientSecret` to a per-account backup file. + * resolved `appId` / `clientSecret` to a per-account SQLite KV entry. * - During plugin startup, if the live config has an empty appId or * secret, the gateway consults the backup and restores the values * via the config mutation API. - * - Backups live under `~/.openclaw/qqbot/data/` so they survive - * plugin directory replacement. + * - Legacy JSON backups are imported on first read, then removed after + * SQLite has the canonical copy. * * Safety notes: * - Only restore when credentials are **actually empty** — never * overwrite a user's intentional config change. - * - Atomic write (temp file + rename) to avoid torn files. - * - Per-account file: `credential-backup-.json`. We do - * **not** also key by appId because recovery happens precisely - * when appId is unknown. - * - Legacy single `credential-backup.json` is migrated automatically - * when the stored accountId matches the caller. + * - Per-account key only; not keyed by appId because recovery happens + * precisely when appId is unknown. */ import fs from "node:fs"; +import path from "node:path"; import { loadJsonFile } from "openclaw/plugin-sdk/json-store"; -import { replaceFileAtomicSync } from "openclaw/plugin-sdk/security-runtime"; import { getCredentialBackupFile, getLegacyCredentialBackupFile } from "../utils/data-paths.js"; +import { getQQBotDataPath } from "../utils/platform.js"; +import { buildQQBotStateKey, openQQBotSyncKeyedStore } from "../utils/sqlite-state.js"; interface CredentialBackup { accountId: string; @@ -37,6 +35,72 @@ interface CredentialBackup { savedAt: string; } +const CREDENTIAL_BACKUPS_NAMESPACE = "credential-backups"; +const MAX_CREDENTIAL_BACKUPS = 1000; + +function createCredentialBackupStore() { + return openQQBotSyncKeyedStore({ + namespace: CREDENTIAL_BACKUPS_NAMESPACE, + maxEntries: MAX_CREDENTIAL_BACKUPS, + }); +} + +function safeName(id: string): string { + return id.replace(/[^a-zA-Z0-9._-]/g, "_"); +} + +function getLegacyOsHomeCredentialBackupFile(accountId: string): string { + return path.join(getQQBotDataPath("data"), `credential-backup-${safeName(accountId)}.json`); +} + +function getLegacyOsHomeCredentialBackupFileWithoutAccount(): string { + return path.join(getQQBotDataPath("data"), "credential-backup.json"); +} + +function credentialBackupKey(accountId: string): string { + return buildQQBotStateKey("credential-backup", accountId); +} + +function isUsableBackup(data: CredentialBackup | null | undefined): data is CredentialBackup { + return Boolean(data?.accountId && data.appId && data.clientSecret); +} + +function loadUsableBackupFromFile(filePath: string): CredentialBackup | null { + const data = loadJsonFile(filePath); + return isUsableBackup(data) ? data : null; +} + +function removeFileQuietly(filePath: string): void { + try { + fs.unlinkSync(filePath); + } catch { + /* ignore cleanup errors */ + } +} + +function findLegacyBackup(accountId?: string): { data: CredentialBackup; filePath: string } | null { + const candidates = accountId + ? [ + getCredentialBackupFile(accountId), + getLegacyCredentialBackupFile(), + getLegacyOsHomeCredentialBackupFile(accountId), + getLegacyOsHomeCredentialBackupFileWithoutAccount(), + ] + : [getLegacyCredentialBackupFile(), getLegacyOsHomeCredentialBackupFileWithoutAccount()]; + + for (const filePath of candidates) { + const data = loadUsableBackupFromFile(filePath); + if (!data) { + continue; + } + if (accountId && data.accountId !== accountId) { + continue; + } + return { data, filePath }; + } + return null; +} + /** Persist a credential snapshot (called once gateway reaches READY). */ export function saveCredentialBackup(accountId: string, appId: string, clientSecret: string): void { if (!appId || !clientSecret) { @@ -50,11 +114,8 @@ export function saveCredentialBackup(accountId: string, appId: string, clientSec clientSecret, savedAt: new Date().toISOString(), }; - replaceFileAtomicSync({ - filePath: backupPath, - content: `${JSON.stringify(data, null, 2)}\n`, - tempPrefix: ".qqbot-credential-backup", - }); + createCredentialBackupStore().register(credentialBackupKey(accountId), data); + removeFileQuietly(backupPath); } catch { /* best-effort — ignore */ } @@ -63,43 +124,27 @@ export function saveCredentialBackup(accountId: string, appId: string, clientSec /** * Load a credential snapshot for `accountId`. * - * Consults the new per-account file first; falls back to the legacy - * global backup file and migrates it when the embedded `accountId` - * matches the request. Returns `null` when no usable backup exists. + * Consults SQLite first; falls back to shipped JSON backups and imports + * them when the embedded `accountId` matches the request. */ export function loadCredentialBackup(accountId?: string): CredentialBackup | null { try { if (accountId) { - const newPath = getCredentialBackupFile(accountId); - const data = loadJsonFile(newPath); - if (data?.appId && data.clientSecret) { + const store = createCredentialBackupStore(); + const data = store.lookup(credentialBackupKey(accountId)); + if (isUsableBackup(data)) { return data; } } - const legacy = getLegacyCredentialBackupFile(); - const data = loadJsonFile(legacy); - if (data) { - if (!data?.appId || !data?.clientSecret) { - return null; - } - if (accountId && data.accountId !== accountId) { - return null; - } - if (data.accountId) { - try { - const backupPath = getCredentialBackupFile(data.accountId); - replaceFileAtomicSync({ - filePath: backupPath, - content: `${JSON.stringify(data, null, 2)}\n`, - tempPrefix: ".qqbot-credential-backup", - }); - fs.unlinkSync(legacy); - } catch { - /* ignore migration errors */ - } - } - return data; + const legacy = findLegacyBackup(accountId); + if (legacy) { + createCredentialBackupStore().register( + credentialBackupKey(legacy.data.accountId), + legacy.data, + ); + removeFileQuietly(legacy.filePath); + return legacy.data; } } catch { /* corrupt file — ignore */ diff --git a/extensions/qqbot/src/engine/ref/store.test.ts b/extensions/qqbot/src/engine/ref/store.test.ts new file mode 100644 index 000000000000..890ea3bf63af --- /dev/null +++ b/extensions/qqbot/src/engine/ref/store.test.ts @@ -0,0 +1,131 @@ +import fs from "node:fs"; +import os from "node:os"; +import path from "node:path"; +import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; +import { + installQQBotRuntimeForStateTests, + resetQQBotStateTestRuntime, +} from "../../test-support/runtime.js"; +import type { RefIndexEntry } from "./store.js"; + +const createdDirs: string[] = []; + +function createTempDir(prefix: string): string { + const dir = fs.mkdtempSync(path.join(os.tmpdir(), prefix)); + createdDirs.push(dir); + return dir; +} + +function refIndexFile(homeDir: string): string { + return path.join(homeDir, ".openclaw", "qqbot", "data", "ref-index.jsonl"); +} + +async function useMockHome(homeDir: string): Promise { + vi.doMock("node:os", async (importOriginal) => { + const actual = await importOriginal(); + return { + ...actual, + default: { ...actual, homedir: () => homeDir }, + homedir: () => homeDir, + }; + }); +} + +function entry(content = "hello"): RefIndexEntry { + return { + content, + senderId: "user-1", + senderName: "User", + timestamp: Date.now(), + isBot: false, + }; +} + +describe("engine/ref/store", () => { + beforeEach(async () => { + vi.resetModules(); + const stateDir = createTempDir("qqbot-state-"); + const homeDir = createTempDir("qqbot-home-"); + vi.stubEnv("OPENCLAW_STATE_DIR", stateDir); + vi.stubEnv("HOME", homeDir); + await useMockHome(homeDir); + installQQBotRuntimeForStateTests(stateDir); + }); + + afterEach(() => { + resetQQBotStateTestRuntime(); + vi.doUnmock("node:os"); + vi.resetModules(); + vi.unstubAllEnvs(); + for (const dir of createdDirs.splice(0)) { + fs.rmSync(dir, { recursive: true, force: true }); + } + }); + + it("round-trips ref-index rows through SQLite without writing JSONL", async () => { + const { getRefIndex, setRefIndex } = await import("./store.js"); + const homeDir = process.env.HOME!; + + setRefIndex("ref-1", entry("from-sqlite")); + + expect(getRefIndex("ref-1")?.content).toBe("from-sqlite"); + expect(fs.existsSync(refIndexFile(homeDir))).toBe(false); + }); + + it("omits undefined optional fields before writing to SQLite", async () => { + const { getRefIndex, setRefIndex } = await import("./store.js"); + + setRefIndex("ref-optional", { + content: "plain inbound", + senderId: "user-1", + senderName: undefined, + timestamp: Date.now(), + isBot: undefined, + attachments: [ + { + type: "image", + filename: undefined, + contentType: undefined, + transcript: undefined, + localPath: "/tmp/image.png", + }, + ], + }); + + expect(getRefIndex("ref-optional")).toEqual({ + content: "plain inbound", + senderId: "user-1", + timestamp: expect.any(Number), + attachments: [{ type: "image", localPath: "/tmp/image.png" }], + }); + }); + + it("keeps ref-index persistence best-effort when SQLite is unavailable", async () => { + resetQQBotStateTestRuntime(); + const { getRefIndex, setRefIndex } = await import("./store.js"); + + expect(() => setRefIndex("ref-unavailable", entry("ignored"))).not.toThrow(); + expect(getRefIndex("ref-unavailable")).toBeNull(); + }); + + it("imports legacy ref-index JSONL and drops expired rows", async () => { + const { getRefIndex } = await import("./store.js"); + const legacyPath = refIndexFile(process.env.HOME!); + fs.mkdirSync(path.dirname(legacyPath), { recursive: true }); + fs.writeFileSync( + legacyPath, + [ + JSON.stringify({ k: "valid", v: entry("valid-content"), t: Date.now() }), + JSON.stringify({ + k: "expired", + v: entry("expired-content"), + t: Date.now() - 8 * 24 * 60 * 60 * 1000, + }), + ].join("\n"), + ); + + expect(getRefIndex("valid")?.content).toBe("valid-content"); + expect(getRefIndex("expired")).toBeNull(); + expect(fs.existsSync(legacyPath)).toBe(false); + }); +}); diff --git a/extensions/qqbot/src/engine/ref/store.ts b/extensions/qqbot/src/engine/ref/store.ts index 9645f9fc8fb0..866ff7e0963f 100644 --- a/extensions/qqbot/src/engine/ref/store.ts +++ b/extensions/qqbot/src/engine/ref/store.ts @@ -1,17 +1,17 @@ /** - * Ref-index store — JSONL file-based store for message reference index. + * Ref-index store — SQLite KV-backed store for message reference index. * - * Migrated from src/ref-index-store.ts. Dependencies are only Node.js - * built-ins + log + platform (both zero plugin-sdk). + * Legacy JSONL entries are imported once, then deleted after SQLite has the + * canonical ref-index rows. */ import fs from "node:fs"; import path from "node:path"; -import { appendRegularFileSync, replaceFileAtomicSync } from "openclaw/plugin-sdk/security-runtime"; import { formatErrorMessage } from "../utils/format.js"; import { debugLog, debugError } from "../utils/log.js"; -import { getQQBotDataDir, getQQBotDataPath } from "../utils/platform.js"; -import type { RefIndexEntry } from "./types.js"; +import { getQQBotDataPath } from "../utils/platform.js"; +import { buildQQBotStateKey, openQQBotSyncKeyedStore } from "../utils/sqlite-state.js"; +import type { RefAttachmentSummary, RefIndexEntry } from "./types.js"; // Re-export types and format function for convenience. export type { RefIndexEntry, RefAttachmentSummary } from "./types.js"; @@ -19,7 +19,9 @@ export { formatRefEntryForAgent } from "./format-ref-entry.js"; const MAX_ENTRIES = 50000; const TTL_MS = 7 * 24 * 60 * 60 * 1000; -const COMPACT_THRESHOLD_RATIO = 2; +const REF_INDEX_NAMESPACE = "ref-index"; +const REF_INDEX_MIGRATIONS_NAMESPACE = "ref-index-migrations"; +const LEGACY_REF_INDEX_MIGRATION_KEY = "ref-index-jsonl-v1"; interface RefIndexLine { k: string; @@ -27,36 +29,106 @@ interface RefIndexLine { t: number; } -let cache: Map | null = null; -let totalLinesOnDisk = 0; +type StoredRefIndexEntry = RefIndexEntry & { + createdAt: number; +}; + +type RefIndexMigrationMarker = { + importedAt: string; +}; + +let legacyImported = false; function getRefIndexFile(): string { return path.join(getQQBotDataPath("data"), "ref-index.jsonl"); } -function loadFromFile(): Map { - if (cache !== null) { - return cache; - } - cache = new Map(); - totalLinesOnDisk = 0; +function createRefIndexStore() { + return openQQBotSyncKeyedStore({ + namespace: REF_INDEX_NAMESPACE, + maxEntries: MAX_ENTRIES, + defaultTtlMs: TTL_MS, + }); +} +function createRefIndexMigrationStore() { + return openQQBotSyncKeyedStore({ + namespace: REF_INDEX_MIGRATIONS_NAMESPACE, + maxEntries: 100, + }); +} + +function refIndexStateKey(refIdx: string): string { + return buildQQBotStateKey("ref-index", refIdx); +} + +function toStoredAttachment(attachment: RefAttachmentSummary): RefAttachmentSummary { + return { + type: attachment.type, + ...(attachment.filename !== undefined ? { filename: attachment.filename } : {}), + ...(attachment.contentType !== undefined ? { contentType: attachment.contentType } : {}), + ...(attachment.transcript !== undefined ? { transcript: attachment.transcript } : {}), + ...(attachment.transcriptSource !== undefined + ? { transcriptSource: attachment.transcriptSource } + : {}), + ...(attachment.localPath !== undefined ? { localPath: attachment.localPath } : {}), + ...(attachment.url !== undefined ? { url: attachment.url } : {}), + }; +} + +function toStoredRefIndexEntry(entry: RefIndexEntry, createdAt: number): StoredRefIndexEntry { + return { + content: entry.content, + senderId: entry.senderId, + ...(entry.senderName !== undefined ? { senderName: entry.senderName } : {}), + timestamp: entry.timestamp, + ...(entry.isBot !== undefined ? { isBot: entry.isBot } : {}), + ...(entry.attachments ? { attachments: entry.attachments.map(toStoredAttachment) } : {}), + createdAt, + }; +} + +function toRefIndexEntry(entry: StoredRefIndexEntry): RefIndexEntry { + return { + content: entry.content, + senderId: entry.senderId, + ...(entry.senderName !== undefined ? { senderName: entry.senderName } : {}), + timestamp: entry.timestamp, + ...(entry.isBot !== undefined ? { isBot: entry.isBot } : {}), + ...(entry.attachments ? { attachments: entry.attachments.map(toStoredAttachment) } : {}), + }; +} + +function ensureLegacyRefIndexImported(): void { + if (legacyImported) { + return; + } + const migrationStore = createRefIndexMigrationStore(); + if (migrationStore.lookup(LEGACY_REF_INDEX_MIGRATION_KEY)) { + legacyImported = true; + return; + } try { const refIndexFile = getRefIndexFile(); if (!fs.existsSync(refIndexFile)) { - return cache; + migrationStore.register(LEGACY_REF_INDEX_MIGRATION_KEY, { + importedAt: new Date().toISOString(), + }); + legacyImported = true; + return; } const raw = fs.readFileSync(refIndexFile, "utf-8"); const lines = raw.split("\n"); const now = Date.now(); let expired = 0; + let imported = 0; + const store = createRefIndexStore(); for (const line of lines) { const trimmed = line.trim(); if (!trimmed) { continue; } - totalLinesOnDisk++; try { const entry = JSON.parse(trimmed) as RefIndexLine; if (!entry.k || !entry.v || !entry.t) { @@ -66,148 +138,58 @@ function loadFromFile(): Map { expired++; continue; } - cache.set(entry.k, { ...entry.v, createdAt: entry.t }); + store.register(refIndexStateKey(entry.k), toStoredRefIndexEntry(entry.v, entry.t), { + ttlMs: Math.max(1, TTL_MS - (now - entry.t)), + }); + imported++; } catch {} } - debugLog( - `[ref-index-store] Loaded ${cache.size} entries from ${totalLinesOnDisk} lines (${expired} expired)`, - ); - if (shouldCompact()) { - compactFile(); - } - } catch (err) { - debugError(`[ref-index-store] Failed to load: ${formatErrorMessage(err)}`); - cache = new Map(); - } - return cache; -} - -function ensureDir(): void { - getQQBotDataDir("data"); -} - -function appendLine(line: RefIndexLine): void { - try { - ensureDir(); - appendRegularFileSync({ filePath: getRefIndexFile(), content: JSON.stringify(line) + "\n" }); - totalLinesOnDisk++; - } catch (err) { - debugError(`[ref-index-store] Failed to append: ${formatErrorMessage(err)}`); - } -} - -function shouldCompact(): boolean { - return ( - cache !== null && - totalLinesOnDisk > cache.size * COMPACT_THRESHOLD_RATIO && - totalLinesOnDisk > 1000 - ); -} - -function compactFile(): void { - if (!cache) { - return; - } - const before = totalLinesOnDisk; - try { - ensureDir(); - const refIndexFile = getRefIndexFile(); - const lines: string[] = []; - for (const [key, entry] of cache) { - lines.push( - JSON.stringify({ - k: key, - v: { - content: entry.content, - senderId: entry.senderId, - senderName: entry.senderName, - timestamp: entry.timestamp, - isBot: entry.isBot, - attachments: entry.attachments, - }, - t: entry.createdAt, - }), - ); - } - replaceFileAtomicSync({ - filePath: refIndexFile, - content: `${lines.join("\n")}\n`, - tempPrefix: ".qqbot-ref-index", + migrationStore.register(LEGACY_REF_INDEX_MIGRATION_KEY, { + importedAt: new Date().toISOString(), }); - totalLinesOnDisk = cache.size; - debugLog(`[ref-index-store] Compacted: ${before} lines → ${totalLinesOnDisk} lines`); + legacyImported = true; + fs.rmSync(refIndexFile, { force: true }); + debugLog(`[ref-index-store] Migrated ${imported} entries to SQLite (${expired} expired)`); } catch (err) { - debugError(`[ref-index-store] Compact failed: ${formatErrorMessage(err)}`); - } -} - -function evictIfNeeded(): void { - if (!cache || cache.size < MAX_ENTRIES) { - return; - } - const now = Date.now(); - for (const [key, entry] of cache) { - if (now - entry.createdAt > TTL_MS) { - cache.delete(key); - } - } - if (cache.size >= MAX_ENTRIES) { - const sorted = [...cache.entries()].toSorted((a, b) => a[1].createdAt - b[1].createdAt); - const toRemove = sorted.slice(0, cache.size - MAX_ENTRIES + 1000); - for (const [key] of toRemove) { - cache.delete(key); - } - debugLog(`[ref-index-store] Evicted ${toRemove.length} oldest entries`); + debugError(`[ref-index-store] Failed to import legacy JSONL: ${formatErrorMessage(err)}`); } } /** Persist a refIdx mapping for one message. */ export function setRefIndex(refIdx: string, entry: RefIndexEntry): void { - const store = loadFromFile(); - evictIfNeeded(); - const now = Date.now(); - store.set(refIdx, { ...entry, createdAt: now }); - appendLine({ - k: refIdx, - v: { - content: entry.content, - senderId: entry.senderId, - senderName: entry.senderName, - timestamp: entry.timestamp, - isBot: entry.isBot, - attachments: entry.attachments, - }, - t: now, - }); - if (shouldCompact()) { - compactFile(); + try { + ensureLegacyRefIndexImported(); + const now = Date.now(); + createRefIndexStore().register(refIndexStateKey(refIdx), toStoredRefIndexEntry(entry, now), { + ttlMs: TTL_MS, + }); + } catch (err) { + debugError(`[ref-index-store] Failed to persist ref index: ${formatErrorMessage(err)}`); } } /** Look up one quoted message by refIdx. */ export function getRefIndex(refIdx: string): RefIndexEntry | null { - const store = loadFromFile(); - const entry = store.get(refIdx); - if (!entry) { + try { + ensureLegacyRefIndexImported(); + const store = createRefIndexStore(); + const key = refIndexStateKey(refIdx); + const entry = store.lookup(key); + if (!entry) { + return null; + } + if (Date.now() - entry.createdAt > TTL_MS) { + store.delete(key); + return null; + } + return toRefIndexEntry(entry); + } catch (err) { + debugError(`[ref-index-store] Failed to read ref index: ${formatErrorMessage(err)}`); return null; } - if (Date.now() - entry.createdAt > TTL_MS) { - store.delete(refIdx); - return null; - } - return { - content: entry.content, - senderId: entry.senderId, - senderName: entry.senderName, - timestamp: entry.timestamp, - isBot: entry.isBot, - attachments: entry.attachments, - }; } /** Compact the store before process exit when needed. */ export function flushRefIndex(): void { - if (cache && shouldCompact()) { - compactFile(); - } + // SQLite writes are synchronous; no JSONL compaction remains. } diff --git a/extensions/qqbot/src/engine/session/known-users.test.ts b/extensions/qqbot/src/engine/session/known-users.test.ts new file mode 100644 index 000000000000..a15dc4cbcfa2 --- /dev/null +++ b/extensions/qqbot/src/engine/session/known-users.test.ts @@ -0,0 +1,146 @@ +import fs from "node:fs"; +import os from "node:os"; +import path from "node:path"; +import { createPluginStateSyncKeyedStoreForTests } from "openclaw/plugin-sdk/plugin-state-test-runtime"; +import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; +import { + installQQBotRuntimeForStateTests, + resetQQBotStateTestRuntime, +} from "../../test-support/runtime.js"; + +type KnownUser = { + openid: string; + type: "c2c" | "group"; + nickname?: string; + groupOpenid?: string; + accountId: string; + firstSeenAt: number; + lastSeenAt: number; + interactionCount: number; +}; + +const createdDirs: string[] = []; + +function createTempDir(prefix: string): string { + const dir = fs.mkdtempSync(path.join(os.tmpdir(), prefix)); + createdDirs.push(dir); + return dir; +} + +function knownUsersFile(homeDir: string): string { + return path.join(homeDir, ".openclaw", "qqbot", "data", "known-users.json"); +} + +async function useMockHome(homeDir: string): Promise { + vi.doMock("node:os", async (importOriginal) => { + const actual = await importOriginal(); + return { + ...actual, + default: { ...actual, homedir: () => homeDir }, + homedir: () => homeDir, + }; + }); +} + +function knownUserRows(stateDir: string): KnownUser[] { + const store = createPluginStateSyncKeyedStoreForTests("qqbot", { + namespace: "known-users", + maxEntries: 100_000, + env: { ...process.env, OPENCLAW_STATE_DIR: stateDir }, + }); + return store.entries().map((entry) => entry.value); +} + +describe("engine/session/known-users", () => { + beforeEach(async () => { + vi.resetModules(); + const stateDir = createTempDir("qqbot-state-"); + const homeDir = createTempDir("qqbot-home-"); + vi.stubEnv("OPENCLAW_STATE_DIR", stateDir); + vi.stubEnv("HOME", homeDir); + await useMockHome(homeDir); + installQQBotRuntimeForStateTests(stateDir); + }); + + afterEach(() => { + resetQQBotStateTestRuntime(); + vi.doUnmock("node:os"); + vi.resetModules(); + vi.unstubAllEnvs(); + for (const dir of createdDirs.splice(0)) { + fs.rmSync(dir, { recursive: true, force: true }); + } + }); + + it("records known users in SQLite and flushes synchronously", async () => { + const { flushKnownUsers, recordKnownUser } = await import("./known-users.js"); + const stateDir = process.env.OPENCLAW_STATE_DIR!; + + recordKnownUser({ + openid: "user-1", + type: "c2c", + nickname: "First", + accountId: "acct-1", + }); + recordKnownUser({ + openid: "user-1", + type: "c2c", + nickname: "Second", + accountId: "acct-1", + }); + flushKnownUsers(); + + expect(knownUserRows(stateDir)).toMatchObject([ + { + openid: "user-1", + nickname: "Second", + interactionCount: 2, + }, + ]); + expect(fs.existsSync(knownUsersFile(process.env.HOME!))).toBe(false); + }); + + it("imports legacy known-users.json once", async () => { + const { recordKnownUser } = await import("./known-users.js"); + const stateDir = process.env.OPENCLAW_STATE_DIR!; + const legacyPath = knownUsersFile(process.env.HOME!); + fs.mkdirSync(path.dirname(legacyPath), { recursive: true }); + fs.writeFileSync( + legacyPath, + JSON.stringify([ + { + openid: "legacy-user", + type: "group", + groupOpenid: "group-1", + accountId: "acct-1", + firstSeenAt: 1, + lastSeenAt: 2, + interactionCount: 3, + }, + ]), + ); + + recordKnownUser({ + openid: "new-user", + type: "c2c", + accountId: "acct-1", + }); + + const rows = knownUserRows(stateDir); + expect(rows.map((row) => row.openid).toSorted()).toEqual(["legacy-user", "new-user"]); + expect(fs.existsSync(legacyPath)).toBe(false); + }); + + it("keeps known-user tracking best-effort when SQLite is unavailable", async () => { + resetQQBotStateTestRuntime(); + const { recordKnownUser } = await import("./known-users.js"); + + expect(() => + recordKnownUser({ + openid: "user-1", + type: "c2c", + accountId: "acct-1", + }), + ).not.toThrow(); + }); +}); diff --git a/extensions/qqbot/src/engine/session/known-users.ts b/extensions/qqbot/src/engine/session/known-users.ts index 0b94dcf14ace..5a1b3615efc1 100644 --- a/extensions/qqbot/src/engine/session/known-users.ts +++ b/extensions/qqbot/src/engine/session/known-users.ts @@ -1,16 +1,19 @@ /** - * Known user tracking — JSON file-based store. + * Known user tracking — SQLite KV-backed store. * - * Migrated from src/known-users.ts. Dependencies are only Node.js - * built-ins + log + platform (both zero plugin-sdk). + * Legacy `known-users.json` data is imported once, then deleted after SQLite + * has the canonical copy. */ +import crypto from "node:crypto"; +import fs from "node:fs"; import path from "node:path"; import { privateFileStoreSync } from "openclaw/plugin-sdk/security-runtime"; import type { ChatScope } from "../types.js"; import { formatErrorMessage } from "../utils/format.js"; import { debugLog, debugError } from "../utils/log.js"; -import { getQQBotDataDir, getQQBotDataPath } from "../utils/platform.js"; +import { getQQBotDataPath } from "../utils/platform.js"; +import { openQQBotSyncKeyedStore } from "../utils/sqlite-state.js"; /** Persisted record for a user who has interacted with the bot. */ interface KnownUser { @@ -24,14 +27,9 @@ interface KnownUser { interactionCount: number; } -let usersCache: Map | null = null; -const SAVE_THROTTLE_MS = 5000; -let saveTimer: ReturnType | null = null; -let isDirty = false; - -function ensureDir(): void { - getQQBotDataDir("data"); -} +type KnownUsersMigrationMarker = { + importedAt: string; +}; function getKnownUsersFile(): string { return path.join(getQQBotDataPath("data"), "known-users.json"); @@ -42,63 +40,77 @@ function makeUserKey(user: Partial): string { return user.type === "group" && user.groupOpenid ? `${base}:${user.groupOpenid}` : base; } -function loadUsersFromFile(): Map { - if (usersCache !== null) { - return usersCache; +const KNOWN_USERS_NAMESPACE = "known-users"; +const KNOWN_USERS_MIGRATIONS_NAMESPACE = "known-users-migrations"; +const LEGACY_KNOWN_USERS_MIGRATION_KEY = "known-users-json-v1"; +const MAX_KNOWN_USERS = 100_000; +let legacyImported = false; + +function createKnownUsersStore() { + return openQQBotSyncKeyedStore({ + namespace: KNOWN_USERS_NAMESPACE, + maxEntries: MAX_KNOWN_USERS, + }); +} + +function createKnownUsersMigrationStore() { + return openQQBotSyncKeyedStore({ + namespace: KNOWN_USERS_MIGRATIONS_NAMESPACE, + maxEntries: 100, + }); +} + +function knownUserStateKey(key: string): string { + return crypto.createHash("sha256").update(key).digest("hex"); +} + +function toStoredKnownUser(user: KnownUser): KnownUser { + return { + openid: user.openid, + type: user.type, + ...(user.nickname ? { nickname: user.nickname } : {}), + ...(user.groupOpenid ? { groupOpenid: user.groupOpenid } : {}), + accountId: user.accountId, + firstSeenAt: user.firstSeenAt, + lastSeenAt: user.lastSeenAt, + interactionCount: user.interactionCount, + }; +} + +function ensureLegacyKnownUsersImported(): void { + if (legacyImported) { + return; + } + const migrationStore = createKnownUsersMigrationStore(); + if (migrationStore.lookup(LEGACY_KNOWN_USERS_MIGRATION_KEY)) { + legacyImported = true; + return; } - usersCache = new Map(); try { const knownUsersFile = getKnownUsersFile(); const users = privateFileStoreSync(path.dirname(knownUsersFile)).readJsonIfExists( path.basename(knownUsersFile), ); - if (users) { + if (Array.isArray(users)) { + const store = createKnownUsersStore(); for (const user of users) { - usersCache.set(makeUserKey(user), user); + store.registerIfAbsent(knownUserStateKey(makeUserKey(user)), toStoredKnownUser(user)); } - debugLog(`[known-users] Loaded ${usersCache.size} users`); + debugLog(`[known-users] Migrated ${users.length} users to SQLite`); + fs.rmSync(knownUsersFile, { force: true }); } + migrationStore.register(LEGACY_KNOWN_USERS_MIGRATION_KEY, { + importedAt: new Date().toISOString(), + }); + legacyImported = true; } catch (err) { - debugError(`[known-users] Failed to load users: ${formatErrorMessage(err)}`); - usersCache = new Map(); - } - return usersCache; -} - -function saveUsersToFile(): void { - if (!isDirty || saveTimer) { - return; - } - saveTimer = setTimeout(() => { - saveTimer = null; - doSaveUsersToFile(); - }, SAVE_THROTTLE_MS); -} - -function doSaveUsersToFile(): void { - if (!usersCache || !isDirty) { - return; - } - try { - ensureDir(); - const filePath = getKnownUsersFile(); - privateFileStoreSync(path.dirname(filePath)).writeJson( - path.basename(filePath), - Array.from(usersCache.values()), - ); - isDirty = false; - } catch (err) { - debugError(`[known-users] Failed to save users: ${formatErrorMessage(err)}`); + debugError(`[known-users] Failed to import legacy users: ${formatErrorMessage(err)}`); } } /** Flush pending writes immediately, typically during shutdown. */ export function flushKnownUsers(): void { - if (saveTimer) { - clearTimeout(saveTimer); - saveTimer = null; - } - doSaveUsersToFile(); + // SQLite writes are synchronous; no pending JSON flush remains. } /** Record a known user whenever a message is received. */ @@ -109,30 +121,41 @@ export function recordKnownUser(user: { groupOpenid?: string; accountId: string; }): void { - const cache = loadUsersFromFile(); - const key = makeUserKey(user); - const now = Date.now(); - const existing = cache.get(key); + try { + ensureLegacyKnownUsersImported(); + const store = createKnownUsersStore(); + const key = makeUserKey(user); + const stateKey = knownUserStateKey(key); + const now = Date.now(); + const existing = store.lookup(stateKey); - if (existing) { - existing.lastSeenAt = now; - existing.interactionCount++; - if (user.nickname && user.nickname !== existing.nickname) { - existing.nickname = user.nickname; + if (existing) { + const next: KnownUser = { + ...existing, + lastSeenAt: now, + interactionCount: existing.interactionCount + 1, + }; + if (user.nickname && user.nickname !== existing.nickname) { + next.nickname = user.nickname; + } + store.register(stateKey, toStoredKnownUser(next)); + } else { + store.register( + stateKey, + toStoredKnownUser({ + openid: user.openid, + type: user.type, + nickname: user.nickname, + groupOpenid: user.groupOpenid, + accountId: user.accountId, + firstSeenAt: now, + lastSeenAt: now, + interactionCount: 1, + }), + ); + debugLog(`[known-users] New user: ${user.openid} (${user.type})`); } - } else { - cache.set(key, { - openid: user.openid, - type: user.type, - nickname: user.nickname, - groupOpenid: user.groupOpenid, - accountId: user.accountId, - firstSeenAt: now, - lastSeenAt: now, - interactionCount: 1, - }); - debugLog(`[known-users] New user: ${user.openid} (${user.type})`); + } catch (err) { + debugError(`[known-users] Failed to record user: ${formatErrorMessage(err)}`); } - isDirty = true; - saveUsersToFile(); } diff --git a/extensions/qqbot/src/engine/session/session-store.test.ts b/extensions/qqbot/src/engine/session/session-store.test.ts new file mode 100644 index 000000000000..19626e913892 --- /dev/null +++ b/extensions/qqbot/src/engine/session/session-store.test.ts @@ -0,0 +1,122 @@ +import fs from "node:fs"; +import os from "node:os"; +import path from "node:path"; +import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; +import { + installQQBotRuntimeForStateTests, + resetQQBotStateTestRuntime, +} from "../../test-support/runtime.js"; +import type { SessionState } from "./session-store.js"; + +const createdDirs: string[] = []; + +function createTempDir(prefix: string): string { + const dir = fs.mkdtempSync(path.join(os.tmpdir(), prefix)); + createdDirs.push(dir); + return dir; +} + +async function useMockHome(homeDir: string): Promise { + vi.doMock("node:os", async (importOriginal) => { + const actual = await importOriginal(); + return { + ...actual, + default: { ...actual, homedir: () => homeDir }, + homedir: () => homeDir, + }; + }); +} + +async function useStateAndHome(): Promise<{ stateDir: string; homeDir: string }> { + const stateDir = createTempDir("qqbot-state-"); + const homeDir = createTempDir("qqbot-home-"); + vi.stubEnv("OPENCLAW_STATE_DIR", stateDir); + vi.stubEnv("HOME", homeDir); + await useMockHome(homeDir); + installQQBotRuntimeForStateTests(stateDir); + return { stateDir, homeDir }; +} + +function sessionPath(homeDir: string, accountId: string): string { + const encodedId = Buffer.from(accountId, "utf8").toString("base64url"); + return path.join(homeDir, ".openclaw", "qqbot", "sessions", `session-${encodedId}.json`); +} + +function writeLegacySession(homeDir: string, state: SessionState): string { + const filePath = sessionPath(homeDir, state.accountId); + fs.mkdirSync(path.dirname(filePath), { recursive: true }); + fs.writeFileSync(filePath, `${JSON.stringify(state, null, 2)}\n`); + return filePath; +} + +function makeSession(overrides: Partial = {}): SessionState { + return { + sessionId: "session-1", + lastSeq: 42, + lastConnectedAt: Date.now(), + intentLevelIndex: 0, + accountId: "acct-1", + savedAt: Date.now(), + appId: "app-1", + ...overrides, + }; +} + +describe("engine/session/session-store", () => { + beforeEach(async () => { + vi.resetModules(); + await useStateAndHome(); + }); + + afterEach(async () => { + const { clearSession } = await import("./session-store.js"); + clearSession("acct-1"); + resetQQBotStateTestRuntime(); + vi.doUnmock("node:os"); + vi.resetModules(); + vi.unstubAllEnvs(); + for (const dir of createdDirs.splice(0)) { + fs.rmSync(dir, { recursive: true, force: true }); + } + }); + + it("round-trips gateway sessions through SQLite without creating JSON files", async () => { + const { loadSession, saveSession } = await import("./session-store.js"); + const homeDir = process.env.HOME!; + + saveSession(makeSession()); + + expect(loadSession("acct-1", "app-1")?.sessionId).toBe("session-1"); + expect(fs.existsSync(sessionPath(homeDir, "acct-1"))).toBe(false); + }); + + it("imports legacy JSON sessions and removes the old file", async () => { + const { loadSession } = await import("./session-store.js"); + const homeDir = process.env.HOME!; + const legacyPath = writeLegacySession(homeDir, makeSession({ sessionId: "legacy-session" })); + + expect(loadSession("acct-1", "app-1")?.sessionId).toBe("legacy-session"); + expect(fs.existsSync(legacyPath)).toBe(false); + expect(loadSession("acct-1", "app-1")?.sessionId).toBe("legacy-session"); + }); + + it("deletes mismatched appId sessions from SQLite", async () => { + const { loadSession, saveSession } = await import("./session-store.js"); + saveSession(makeSession({ appId: "app-a" })); + + expect(loadSession("acct-1", "app-b")).toBeNull(); + expect(loadSession("acct-1", "app-a")).toBeNull(); + }); + + it("drops expired legacy JSON sessions during import", async () => { + const { loadSession } = await import("./session-store.js"); + const homeDir = process.env.HOME!; + const legacyPath = writeLegacySession( + homeDir, + makeSession({ savedAt: Date.now() - 10 * 60 * 1000 }), + ); + + expect(loadSession("acct-1", "app-1")).toBeNull(); + expect(fs.existsSync(legacyPath)).toBe(false); + }); +}); diff --git a/extensions/qqbot/src/engine/session/session-store.ts b/extensions/qqbot/src/engine/session/session-store.ts index f0798366a2e7..1d0d6ac608a7 100644 --- a/extensions/qqbot/src/engine/session/session-store.ts +++ b/extensions/qqbot/src/engine/session/session-store.ts @@ -1,8 +1,8 @@ /** - * Gateway session persistence — JSONL file-based store. + * Gateway session persistence — SQLite KV-backed store. * - * Migrated from src/session-store.ts. Dependencies are only Node.js - * built-ins + log + platform (both zero plugin-sdk). + * Legacy JSON session files are imported on first account access, then + * removed after SQLite has the canonical short-lived session entry. */ import fs from "node:fs"; @@ -10,7 +10,8 @@ import path from "node:path"; import { privateFileStoreSync } from "openclaw/plugin-sdk/security-runtime"; import { formatErrorMessage } from "../utils/format.js"; import { debugLog, debugError } from "../utils/log.js"; -import { getQQBotDataDir, getQQBotDataPath } from "../utils/platform.js"; +import { getQQBotDataPath } from "../utils/platform.js"; +import { buildQQBotStateKey, openQQBotSyncKeyedStore } from "../utils/sqlite-state.js"; /** Persisted gateway session state. */ export interface SessionState { @@ -25,6 +26,8 @@ export interface SessionState { const SESSION_EXPIRE_TIME = 5 * 60 * 1000; const SAVE_THROTTLE_MS = 1000; +const SESSION_NAMESPACE = "gateway-sessions"; +const MAX_SESSIONS = 1000; const throttleState = new Map< string, @@ -35,10 +38,6 @@ const throttleState = new Map< } >(); -function ensureDir(): void { - getQQBotDataDir("sessions"); -} - function getSessionDir(): string { return getQQBotDataPath("sessions"); } @@ -63,21 +62,80 @@ function getCandidateSessionPaths(accountId: string): string[] { return primaryPath === legacyPath ? [primaryPath] : [primaryPath, legacyPath]; } +function createSessionStore() { + return openQQBotSyncKeyedStore({ + namespace: SESSION_NAMESPACE, + maxEntries: MAX_SESSIONS, + defaultTtlMs: SESSION_EXPIRE_TIME, + }); +} + +function sessionKey(accountId: string): string { + return buildQQBotStateKey("gateway-session", accountId); +} + +function remainingSessionTtlMs(state: SessionState, now = Date.now()): number { + return Math.max(1, SESSION_EXPIRE_TIME - (now - state.savedAt)); +} + +function toStoredSessionState(state: SessionState): SessionState { + return { + sessionId: state.sessionId, + lastSeq: state.lastSeq, + lastConnectedAt: state.lastConnectedAt, + intentLevelIndex: state.intentLevelIndex, + accountId: state.accountId, + savedAt: state.savedAt, + ...(state.appId ? { appId: state.appId } : {}), + }; +} + +function removeFileQuietly(filePath: string): void { + try { + fs.unlinkSync(filePath); + } catch { + /* ignore cleanup errors */ + } +} + +function loadLegacySession(accountId: string): { state: SessionState; filePath: string } | null { + for (const candidatePath of getCandidateSessionPaths(accountId)) { + const state = privateFileStoreSync(path.dirname(candidatePath)).readJsonIfExists( + path.basename(candidatePath), + ); + if (state) { + return { state, filePath: candidatePath }; + } + } + return null; +} + +function migrateLegacySession(accountId: string): SessionState | null { + const legacy = loadLegacySession(accountId); + if (!legacy) { + return null; + } + const now = Date.now(); + if (now - legacy.state.savedAt <= SESSION_EXPIRE_TIME) { + createSessionStore().register(sessionKey(accountId), toStoredSessionState(legacy.state), { + ttlMs: remainingSessionTtlMs(legacy.state, now), + }); + } + for (const filePath of getCandidateSessionPaths(accountId)) { + removeFileQuietly(filePath); + } + return legacy.state; +} + /** Load a saved session, rejecting expired or mismatched appId entries. */ export function loadSession(accountId: string, expectedAppId?: string): SessionState | null { try { - let filePath: string | null = null; - let state: SessionState | null = null; - for (const candidatePath of getCandidateSessionPaths(accountId)) { - state = privateFileStoreSync(path.dirname(candidatePath)).readJsonIfExists( - path.basename(candidatePath), - ); - if (state) { - filePath = candidatePath; - break; - } + const store = createSessionStore(); + let state = store.lookup(sessionKey(accountId)); + if (!state) { + state = migrateLegacySession(accountId) ?? undefined; } - if (!filePath || !state) { + if (!state) { return null; } @@ -87,9 +145,7 @@ export function loadSession(accountId: string, expectedAppId?: string): SessionS debugLog( `[session-store] Session expired for ${accountId}, age: ${Math.round((now - state.savedAt) / 1000)}s`, ); - try { - fs.unlinkSync(filePath); - } catch {} + store.delete(sessionKey(accountId)); return null; } @@ -97,14 +153,13 @@ export function loadSession(accountId: string, expectedAppId?: string): SessionS debugLog( `[session-store] appId mismatch for ${accountId}: saved=${state.appId}, current=${expectedAppId}. Discarding stale session.`, ); - try { - fs.unlinkSync(filePath); - } catch {} + store.delete(sessionKey(accountId)); return null; } if (!state.sessionId || state.lastSeq === null || state.lastSeq === undefined) { debugLog(`[session-store] Invalid session data for ${accountId}`); + store.delete(sessionKey(accountId)); return null; } @@ -163,12 +218,12 @@ function doSaveSession(state: SessionState): void { const filePath = getSessionPath(state.accountId); const legacyPath = getLegacySessionPath(state.accountId); try { - ensureDir(); const stateToSave: SessionState = { ...state, savedAt: Date.now() }; - privateFileStoreSync(path.dirname(filePath)).writeJson(path.basename(filePath), stateToSave); - if (legacyPath !== filePath && fs.existsSync(legacyPath)) { - fs.unlinkSync(legacyPath); - } + createSessionStore().register(sessionKey(state.accountId), toStoredSessionState(stateToSave), { + ttlMs: SESSION_EXPIRE_TIME, + }); + removeFileQuietly(filePath); + removeFileQuietly(legacyPath); debugLog( `[session-store] Saved session for ${state.accountId}: sessionId=${state.sessionId}, lastSeq=${state.lastSeq}`, ); @@ -189,12 +244,9 @@ export function clearSession(accountId: string): void { throttleState.delete(accountId); } try { - let cleared = false; + const cleared = createSessionStore().delete(sessionKey(accountId)); for (const filePath of getCandidateSessionPaths(accountId)) { - if (fs.existsSync(filePath)) { - fs.unlinkSync(filePath); - cleared = true; - } + removeFileQuietly(filePath); } if (cleared) { debugLog(`[session-store] Cleared session for ${accountId}`); diff --git a/extensions/qqbot/src/engine/utils/data-paths.test.ts b/extensions/qqbot/src/engine/utils/data-paths.test.ts index 34cc007b3f5f..8acbf7c54b9f 100644 --- a/extensions/qqbot/src/engine/utils/data-paths.test.ts +++ b/extensions/qqbot/src/engine/utils/data-paths.test.ts @@ -12,7 +12,7 @@ function createTempDir(prefix: string): string { return dir; } -describe("qqbot credential backup paths", () => { +describe("qqbot legacy credential backup paths", () => { afterEach(() => { vi.unstubAllEnvs(); for (const stateDir of createdStateDirs.splice(0)) { @@ -20,7 +20,7 @@ describe("qqbot credential backup paths", () => { } }); - it("scopes credential backups to the active OPENCLAW_STATE_DIR", () => { + it("scopes legacy credential backup imports to the active OPENCLAW_STATE_DIR", () => { const stateDir = createTempDir("qqbot-state-"); vi.stubEnv("OPENCLAW_STATE_DIR", stateDir); @@ -32,7 +32,7 @@ describe("qqbot credential backup paths", () => { ); }); - it("keeps same account IDs isolated across different state directories", () => { + it("keeps legacy account import paths isolated across different state directories", () => { const stateDirA = createTempDir("qqbot-state-a-"); const stateDirB = createTempDir("qqbot-state-b-"); @@ -51,7 +51,7 @@ describe("qqbot credential backup paths", () => { expect(gatewayBPath).not.toBe(gatewayAPath); }); - it("uses OPENCLAW_HOME for default credential backup state", () => { + it("uses OPENCLAW_HOME for default legacy credential backup imports", () => { const homeDir = createTempDir("qqbot-openclaw-home-"); vi.stubEnv("OPENCLAW_STATE_DIR", ""); vi.stubEnv("OPENCLAW_HOME", homeDir); diff --git a/extensions/qqbot/src/engine/utils/platform-storage-laziness.test.ts b/extensions/qqbot/src/engine/utils/platform-storage-laziness.test.ts index 906fcc80599a..60ed8a5ea9fb 100644 --- a/extensions/qqbot/src/engine/utils/platform-storage-laziness.test.ts +++ b/extensions/qqbot/src/engine/utils/platform-storage-laziness.test.ts @@ -2,6 +2,10 @@ import fs from "node:fs"; import os from "node:os"; import path from "node:path"; import { afterEach, describe, expect, it, vi } from "vitest"; +import { + installQQBotRuntimeForStateTests, + resetQQBotStateTestRuntime, +} from "../../test-support/runtime.js"; const createdHomes: string[] = []; @@ -26,6 +30,7 @@ function makeHome(): string { describe("qqbot storage laziness", () => { afterEach(() => { + resetQQBotStateTestRuntime(); vi.doUnmock("node:os"); vi.unstubAllEnvs(); vi.resetModules(); @@ -36,7 +41,10 @@ describe("qqbot storage laziness", () => { it("does not create ~/.openclaw/qqbot from module imports or read-only probes", async () => { const homeDir = makeHome(); + const stateDir = makeHome(); await useMockHome(homeDir); + vi.stubEnv("OPENCLAW_STATE_DIR", stateDir); + installQQBotRuntimeForStateTests(stateDir); const qqbotRoot = path.join(homeDir, ".openclaw", "qqbot"); @@ -51,15 +59,20 @@ describe("qqbot storage laziness", () => { it("creates storage when qqbot persists runtime state", async () => { const homeDir = makeHome(); + const stateDir = makeHome(); await useMockHome(homeDir); + vi.stubEnv("OPENCLAW_STATE_DIR", stateDir); + installQQBotRuntimeForStateTests(stateDir); const qqbotRoot = path.join(homeDir, ".openclaw", "qqbot"); + const sqlitePath = path.join(stateDir, "state", "openclaw.sqlite"); const { saveCredentialBackup } = await import("../config/credential-backup.js"); saveCredentialBackup("default", "123456", "secret"); + expect(fs.existsSync(sqlitePath)).toBe(true); expect(fs.existsSync(path.join(qqbotRoot, "data", "credential-backup-default.json"))).toBe( - true, + false, ); }); }); diff --git a/extensions/qqbot/src/engine/utils/platform.ts b/extensions/qqbot/src/engine/utils/platform.ts index 4ee4f0f5c55b..fa1517fb9248 100644 --- a/extensions/qqbot/src/engine/utils/platform.ts +++ b/extensions/qqbot/src/engine/utils/platform.ts @@ -22,9 +22,8 @@ import { debugLog, debugWarn } from "./log.js"; * 3. PlatformAdapter.getTempDir() as a last resort * * This is the *operating-system* home and intentionally ignores - * `OPENCLAW_HOME`. Persistent QQ Bot data (sessions, known users, refs) is - * keyed on this value to keep upgrades from hiding existing state when an - * operator later sets `OPENCLAW_HOME`. + * `OPENCLAW_HOME`. QQ Bot still checks this tree for legacy state imports and + * media-path remaps from older releases. */ export function getHomeDir(): string { try { @@ -78,11 +77,10 @@ function resolveOpenClawHome(): string { } /** - * Return a path under `~/.openclaw/qqbot` without creating it. + * Return a legacy path under `~/.openclaw/qqbot` without creating it. * - * Anchored on the OS home (not `OPENCLAW_HOME`) so persisted QQ Bot data - * (sessions, known users, ref index, credential backups) does not silently - * disappear when an operator adds `OPENCLAW_HOME` after the fact. + * Current QQ Bot runtime state lives in plugin SQLite KV. This path remains for + * legacy imports and media-path remaps from older releases. */ export function getQQBotDataPath(...subPaths: string[]): string { return path.join(getHomeDir(), ".openclaw", "qqbot", ...subPaths); diff --git a/extensions/qqbot/src/engine/utils/sqlite-state.ts b/extensions/qqbot/src/engine/utils/sqlite-state.ts new file mode 100644 index 000000000000..1a1a4e16e8ed --- /dev/null +++ b/extensions/qqbot/src/engine/utils/sqlite-state.ts @@ -0,0 +1,35 @@ +import crypto from "node:crypto"; +import type { + OpenKeyedStoreOptions, + PluginStateSyncKeyedStore, +} from "openclaw/plugin-sdk/plugin-state-runtime"; +import { getQQBotRuntime } from "../../bridge/runtime.js"; + +type QQBotSyncStoreOptions = OpenKeyedStoreOptions & { + stateDir?: string; +}; + +function resolveStoreEnv(options: QQBotSyncStoreOptions): NodeJS.ProcessEnv | undefined { + if (!options.stateDir) { + return options.env; + } + return { + ...(options.env ?? process.env), + OPENCLAW_STATE_DIR: options.stateDir, + }; +} + +export function openQQBotSyncKeyedStore( + options: QQBotSyncStoreOptions, +): PluginStateSyncKeyedStore { + return getQQBotRuntime().state.openSyncKeyedStore({ + namespace: options.namespace, + maxEntries: options.maxEntries, + ...(options.defaultTtlMs != null ? { defaultTtlMs: options.defaultTtlMs } : {}), + ...(resolveStoreEnv(options) ? { env: resolveStoreEnv(options) } : {}), + }); +} + +export function buildQQBotStateKey(...parts: string[]): string { + return crypto.createHash("sha256").update(JSON.stringify(parts)).digest("hex"); +} diff --git a/extensions/qqbot/src/test-support/runtime.ts b/extensions/qqbot/src/test-support/runtime.ts new file mode 100644 index 000000000000..26ebaecc4efa --- /dev/null +++ b/extensions/qqbot/src/test-support/runtime.ts @@ -0,0 +1,43 @@ +import type { PluginRuntime } from "openclaw/plugin-sdk/core"; +import type { OpenKeyedStoreOptions } from "openclaw/plugin-sdk/plugin-state-runtime"; +import { + createPluginStateKeyedStoreForTests, + createPluginStateSyncKeyedStoreForTests, + resetPluginStateStoreForTests, +} from "openclaw/plugin-sdk/plugin-state-test-runtime"; +import { resetQQBotRuntimeForTest, setQQBotRuntime } from "../bridge/runtime.js"; + +function stateEnv(stateDir: string, env?: NodeJS.ProcessEnv): NodeJS.ProcessEnv { + return { + ...(env ?? process.env), + OPENCLAW_STATE_DIR: stateDir, + }; +} + +export function installQQBotRuntimeForStateTests(stateDir: string): void { + resetPluginStateStoreForTests(); + setQQBotRuntime({ + version: "test", + state: { + resolveStateDir: () => stateDir, + openKeyedStore: (options: OpenKeyedStoreOptions) => + createPluginStateKeyedStoreForTests("qqbot", { + ...options, + env: stateEnv(stateDir, options.env), + }), + openSyncKeyedStore: (options: OpenKeyedStoreOptions) => + createPluginStateSyncKeyedStoreForTests("qqbot", { + ...options, + env: stateEnv(stateDir, options.env), + }), + openChannelIngressQueue: () => { + throw new Error("openChannelIngressQueue is not configured for QQBot state tests"); + }, + }, + } as unknown as PluginRuntime); +} + +export function resetQQBotStateTestRuntime(): void { + resetQQBotRuntimeForTest(); + resetPluginStateStoreForTests(); +}