mirror of
https://github.com/openclaw/openclaw.git
synced 2026-06-06 05:51:15 +08:00
fix(memory): keep FTS-only sync offline
This commit is contained in:
@@ -75,6 +75,7 @@ Docs: https://docs.openclaw.ai
|
||||
- Cron: keep SQLite cron migrations compatible with legacy run-log tables, archived job stores, diagnostic cron names, and legacy one-shot delete-after-run behavior. (#88285)
|
||||
- Cron: keep update delivery validation scoped, harden restart state, and retire MCP runtimes on isolated cron cleanup.
|
||||
- Memory: serialize QMD update/embed writes per store, preserve phase signals on read errors, harden envelope metadata sanitization, and rewrite generated transcript paths on rollover so memory/search state survives concurrent gateway and CLI activity. (#66339, #85931) Thanks @openperf and @amittell.
|
||||
- Memory: keep vector-disabled FTS indexes from resolving embedding providers during sync and search.
|
||||
- Providers: bound generated media downloads from OpenAI, Runway, xAI, MiniMax, BytePlus, DashScope-compatible, FAL, OpenRouter, Google, Vydra, and Comfy providers.
|
||||
- Providers: resolve Google defaults to `google-generative-ai`, register Vertex static catalog rows, align Foundry reasoning metadata, skip DeepSeek V4 thinking params on Foundry fallback, use MiniMax account OAuth endpoints, preserve Copilot Claude 1M capabilities, suppress disabled Ollama reasoning output, keep OpenAI stop-finished tool calls, and avoid replay ids when the Responses store is disabled. (#88480, #88512)
|
||||
- Providers: cap GitHub Copilot OAuth request timeouts before creating abort signals.
|
||||
|
||||
@@ -290,7 +290,7 @@ export abstract class MemoryManagerSyncOps {
|
||||
}): MemoryIndexIdentityState {
|
||||
const hasProviderOverride = params && "provider" in params;
|
||||
const configuredProvider =
|
||||
this.settings.provider === "none"
|
||||
!this.vector.enabled || this.settings.provider === "none"
|
||||
? null
|
||||
: {
|
||||
id:
|
||||
|
||||
@@ -9,12 +9,16 @@ import type { MemoryIndexMeta } from "./manager-reindex-state.js";
|
||||
import type { MemoryIndexManager } from "./manager.js";
|
||||
import "./test-runtime-mocks.js";
|
||||
|
||||
vi.mock("./embeddings.js", () => ({
|
||||
createEmbeddingProvider: async () => ({
|
||||
const createEmbeddingProviderMock = vi.hoisted(() =>
|
||||
vi.fn(async () => ({
|
||||
requestedProvider: "auto",
|
||||
provider: null,
|
||||
providerUnavailableReason: "No embeddings provider available.",
|
||||
}),
|
||||
})),
|
||||
);
|
||||
|
||||
vi.mock("./embeddings.js", () => ({
|
||||
createEmbeddingProvider: createEmbeddingProviderMock,
|
||||
resolveEmbeddingProviderAdapterId: (providerId: string) => providerId,
|
||||
resolveEmbeddingProviderFallbackModel: () => "fts-only",
|
||||
}));
|
||||
@@ -31,6 +35,7 @@ describe("memory manager FTS-only reindex", () => {
|
||||
});
|
||||
|
||||
beforeEach(async () => {
|
||||
createEmbeddingProviderMock.mockClear();
|
||||
workspaceDir = path.join(fixtureRoot, `case-${caseId++}`);
|
||||
await fs.mkdir(path.join(workspaceDir, "memory"), { recursive: true });
|
||||
await fs.writeFile(path.join(workspaceDir, "MEMORY.md"), "Alpha topic\n\nKeep this note.");
|
||||
@@ -52,7 +57,13 @@ describe("memory manager FTS-only reindex", () => {
|
||||
}
|
||||
});
|
||||
|
||||
async function createManager(): Promise<MemoryIndexManager> {
|
||||
async function createManager(
|
||||
params: { provider?: string; vectorEnabled?: boolean } = {},
|
||||
): Promise<MemoryIndexManager> {
|
||||
const store =
|
||||
params.vectorEnabled === undefined
|
||||
? { path: indexPath }
|
||||
: { path: indexPath, vector: { enabled: params.vectorEnabled } };
|
||||
const cfg = {
|
||||
memory: {
|
||||
backend: "builtin",
|
||||
@@ -61,9 +72,9 @@ describe("memory manager FTS-only reindex", () => {
|
||||
defaults: {
|
||||
workspace: workspaceDir,
|
||||
memorySearch: {
|
||||
provider: "auto",
|
||||
provider: params.provider ?? "auto",
|
||||
model: "",
|
||||
store: { path: indexPath },
|
||||
store,
|
||||
cache: { enabled: false },
|
||||
sync: { watch: false, onSessionStart: false, onSearch: false },
|
||||
},
|
||||
@@ -118,6 +129,16 @@ describe("memory manager FTS-only reindex", () => {
|
||||
expect(countChunksContaining("Alpha topic")).toBeGreaterThan(0);
|
||||
});
|
||||
|
||||
it("syncs vector-disabled memory without resolving an embedding provider", async () => {
|
||||
const memoryManager = await createManager({ provider: "none", vectorEnabled: false });
|
||||
|
||||
await memoryManager.sync({ force: true });
|
||||
|
||||
expect(createEmbeddingProviderMock).not.toHaveBeenCalled();
|
||||
expect(countChunksContaining("Alpha topic")).toBeGreaterThan(0);
|
||||
expect(memoryManager.status().custom?.indexIdentity).toEqual({ status: "valid" });
|
||||
});
|
||||
|
||||
it("refreshes FTS-only indexed content after memory file updates", async () => {
|
||||
const memoryManager = await createManager();
|
||||
await memoryManager.sync({ force: true });
|
||||
|
||||
@@ -392,11 +392,13 @@ export class MemoryIndexManager extends MemoryManagerEmbeddingOps implements Mem
|
||||
}
|
||||
|
||||
private refreshIndexIdentityDirty(params?: { providerKeyKnown?: boolean }) {
|
||||
const provider = this.providerInitialized
|
||||
? this.provider
|
||||
? { id: this.provider.id, model: this.provider.model }
|
||||
: null
|
||||
: undefined;
|
||||
const provider = !this.vector.enabled
|
||||
? null
|
||||
: this.providerInitialized
|
||||
? this.provider
|
||||
? { id: this.provider.id, model: this.provider.model }
|
||||
: null
|
||||
: undefined;
|
||||
const state = this.resolveCurrentIndexIdentityState({
|
||||
...(provider !== undefined ? { provider } : {}),
|
||||
providerKeyKnown: params?.providerKeyKnown,
|
||||
@@ -451,7 +453,7 @@ export class MemoryIndexManager extends MemoryManagerEmbeddingOps implements Mem
|
||||
log.warn(`memory sync failed (search): ${String(err)}`);
|
||||
},
|
||||
});
|
||||
if (preflight.shouldInitializeProvider) {
|
||||
if (preflight.shouldInitializeProvider && this.vector.enabled) {
|
||||
await this.ensureProviderInitialized();
|
||||
}
|
||||
if (!this.provider && this.providerLifecycle.mode === "degraded") {
|
||||
@@ -789,7 +791,9 @@ export class MemoryIndexManager extends MemoryManagerEmbeddingOps implements Mem
|
||||
return this.syncing;
|
||||
}
|
||||
this.syncing = (async () => {
|
||||
await this.ensureProviderInitialized();
|
||||
if (this.vector.enabled) {
|
||||
await this.ensureProviderInitialized();
|
||||
}
|
||||
await this.runSyncWithReadonlyRecovery(params);
|
||||
})().finally(() => {
|
||||
this.syncing = null;
|
||||
|
||||
@@ -28,6 +28,7 @@ const DEFAULT_FILE_COUNT = 512;
|
||||
const DEFAULT_MAX_WORKSPACE_REG_FDS = process.platform === "darwin" ? 8 : 64;
|
||||
export const GATEWAY_READY_OUTPUT_MAX_CHARS = 128 * 1024;
|
||||
export const MEMORY_SEARCH_RESPONSE_MAX_BYTES = 256 * 1024;
|
||||
export const MEMORY_SEARCH_PROBE_QUERY = "Top-level memory file";
|
||||
|
||||
const SKIP_GATEWAY_ENV = {
|
||||
NODE_ENV: "test",
|
||||
@@ -277,15 +278,22 @@ function writeSyntheticWorkspace(workspaceDir, fileCount) {
|
||||
}
|
||||
}
|
||||
|
||||
function writeConfig({ homeDir, workspaceDir, port, token }) {
|
||||
export function writeConfig({ homeDir, workspaceDir, port, token }) {
|
||||
const configDir = path.join(homeDir, ".openclaw");
|
||||
fs.mkdirSync(configDir, { recursive: true });
|
||||
const configPath = path.join(configDir, "openclaw.json");
|
||||
const indexPath = path.join(configDir, "memory", "main.sqlite");
|
||||
const config = {
|
||||
agents: {
|
||||
defaults: {
|
||||
workspace: workspaceDir,
|
||||
memorySearch: {
|
||||
provider: "none",
|
||||
model: "",
|
||||
store: {
|
||||
path: indexPath,
|
||||
vector: { enabled: false },
|
||||
},
|
||||
sync: {
|
||||
watch: true,
|
||||
onSessionStart: false,
|
||||
@@ -313,6 +321,37 @@ function writeConfig({ homeDir, workspaceDir, port, token }) {
|
||||
return configPath;
|
||||
}
|
||||
|
||||
function formatTail(text, maxChars = 4096) {
|
||||
return text.length > maxChars ? text.slice(-maxChars) : text;
|
||||
}
|
||||
|
||||
function preindexSyntheticMemory(env) {
|
||||
logStep("preindex start");
|
||||
const result = spawnSync(
|
||||
process.execPath,
|
||||
["scripts/run-node.mjs", "memory", "index", "--force", "--agent", "main"],
|
||||
{
|
||||
cwd: process.cwd(),
|
||||
encoding: "utf8",
|
||||
env,
|
||||
maxBuffer: 10 * 1024 * 1024,
|
||||
stdio: ["ignore", "pipe", "pipe"],
|
||||
},
|
||||
);
|
||||
if (result.status !== 0) {
|
||||
throw new Error(
|
||||
[
|
||||
`memory preindex failed with exit ${result.status ?? result.signal}`,
|
||||
formatTail(result.stdout || ""),
|
||||
formatTail(result.stderr || ""),
|
||||
]
|
||||
.filter(Boolean)
|
||||
.join("\n"),
|
||||
);
|
||||
}
|
||||
logStep("preindex complete");
|
||||
}
|
||||
|
||||
export function updateGatewayReadyOutputState(
|
||||
state,
|
||||
chunk,
|
||||
@@ -591,7 +630,7 @@ async function invokeMemorySearch({ port, token, timeoutMs }) {
|
||||
body: JSON.stringify({
|
||||
tool: "memory_search",
|
||||
args: {
|
||||
query: "FD-leak-probe-sentinel-xyzzy-nomatch",
|
||||
query: MEMORY_SEARCH_PROBE_QUERY,
|
||||
maxResults: 1,
|
||||
corpus: "memory",
|
||||
},
|
||||
@@ -673,6 +712,7 @@ async function main() {
|
||||
OPENCLAW_CONFIG_PATH: configPath,
|
||||
OPENCLAW_GATEWAY_TOKEN: token,
|
||||
};
|
||||
preindexSyntheticMemory(env);
|
||||
const child = spawn(
|
||||
process.execPath,
|
||||
[
|
||||
|
||||
@@ -1,7 +1,11 @@
|
||||
import { EventEmitter } from "node:events";
|
||||
import fs from "node:fs";
|
||||
import os from "node:os";
|
||||
import path from "node:path";
|
||||
import { describe, expect, it, vi } from "vitest";
|
||||
import {
|
||||
GATEWAY_READY_OUTPUT_MAX_CHARS,
|
||||
MEMORY_SEARCH_PROBE_QUERY,
|
||||
MEMORY_SEARCH_RESPONSE_MAX_BYTES,
|
||||
classifyMemorySearchInvokeResponse,
|
||||
hasChildExited,
|
||||
@@ -12,6 +16,7 @@ import {
|
||||
stopGatewayWithRuntime,
|
||||
updateGatewayReadyOutputState,
|
||||
waitForGatewayReady,
|
||||
writeConfig,
|
||||
} from "../../scripts/check-memory-fd-repro.mjs";
|
||||
|
||||
function withEnv<T>(env: Record<string, string | undefined>, callback: () => T): T {
|
||||
@@ -125,6 +130,43 @@ describe("check-memory-fd-repro", () => {
|
||||
});
|
||||
});
|
||||
|
||||
it("uses a fast matching probe query instead of a no-hit stress query", () => {
|
||||
expect(MEMORY_SEARCH_PROBE_QUERY).toBe("Top-level memory file");
|
||||
expect(MEMORY_SEARCH_PROBE_QUERY).not.toContain("nomatch");
|
||||
});
|
||||
|
||||
it("writes an offline FTS-only memory search config for repro indexing", () => {
|
||||
const root = fs.mkdtempSync(path.join(os.tmpdir(), "openclaw-memory-fd-config-"));
|
||||
try {
|
||||
const homeDir = path.join(root, "home");
|
||||
const workspaceDir = path.join(root, "workspace");
|
||||
const configPath = writeConfig({
|
||||
homeDir,
|
||||
workspaceDir,
|
||||
port: 12345,
|
||||
token: "test-token",
|
||||
});
|
||||
const config = JSON.parse(fs.readFileSync(configPath, "utf8"));
|
||||
const memorySearch = config.agents.defaults.memorySearch;
|
||||
|
||||
expect(memorySearch).toMatchObject({
|
||||
provider: "none",
|
||||
model: "",
|
||||
store: {
|
||||
path: path.join(homeDir, ".openclaw", "memory", "main.sqlite"),
|
||||
vector: { enabled: false },
|
||||
},
|
||||
sync: {
|
||||
onSearch: false,
|
||||
onSessionStart: false,
|
||||
watch: true,
|
||||
},
|
||||
});
|
||||
} finally {
|
||||
fs.rmSync(root, { recursive: true, force: true });
|
||||
}
|
||||
});
|
||||
|
||||
it("accepts an available memory_search tool payload", () => {
|
||||
const result = classifyMemorySearchInvokeResponse({
|
||||
httpOk: true,
|
||||
|
||||
Reference in New Issue
Block a user