perf: reduce gateway benchmark filesystem churn

2026-06-06 05:51:15 +08:00 · 2026-05-23 23:58:51 +01:00
parent e2249d8d1e
commit e5534dd2f3
7 changed files with 221 additions and 19 deletions
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -6,6 +6,7 @@ Docs: https://docs.openclaw.ai

 ### Changes

+- Gateway/perf: reuse stat-fingerprinted channel catalog reads, avoid repeated bundled-channel boundary checks, and rotate gateway watch CPU profiles so benchmark runs do not accumulate unbounded artifacts.
 - Gateway/perf: reuse immutable plugin metadata snapshots across startup, config, model, channel, setup, and secret metadata readers so hot paths avoid repeated plugin file stats and manifest registry reloads.
 - Gateway/perf: lazy-load startup-idle plugin work, core gateway method handlers, and the embedded ACPX runtime so Gateway health and ready signals no longer wait on unused handler trees or ACPX probes.
 - Gateway/perf: cache plugin SDK public-surface alias maps and skip irrelevant macOS Linuxbrew PATH probes so Gateway startup avoids repeated filesystem walks and slow missing-directory stats.
--- a/scripts/gateway-watch-tmux.mjs
+++ b/scripts/gateway-watch-tmux.mjs
@@ -8,7 +8,9 @@ const TMUX_ATTACH_DISABLE_VALUES = new Set(["0", "false", "no", "off"]);
 const TMUX_ATTACH_FORCE_VALUES = new Set(["1", "true", "yes", "on"]);
 const DEFAULT_PROFILE_NAME = "main";
 const DEFAULT_BENCHMARK_PROFILE_DIR = ".artifacts/gateway-watch-profiles";
+const DEFAULT_BENCHMARK_PROFILE_MAX_FILES = "40";
 const RUN_NODE_CPU_PROF_DIR_ENV = "OPENCLAW_RUN_NODE_CPU_PROF_DIR";
+const RUN_NODE_CPU_PROF_MAX_FILES_ENV = "OPENCLAW_RUN_NODE_CPU_PROF_MAX_FILES";
 const RUN_NODE_OUTPUT_LOG_ENV = "OPENCLAW_RUN_NODE_OUTPUT_LOG";
 const RUN_NODE_FILTER_SYNC_IO_STDERR_ENV = "OPENCLAW_RUN_NODE_FILTER_SYNC_IO_STDERR";
 const RAW_WATCH_SCRIPT = "scripts/watch-node.mjs";
@@ -21,6 +23,7 @@ const TMUX_CHILD_ENV_KEYS = [
  "OPENCLAW_HOME",
  "OPENCLAW_PROFILE",
  RUN_NODE_CPU_PROF_DIR_ENV,
+  RUN_NODE_CPU_PROF_MAX_FILES_ENV,
  RUN_NODE_FILTER_SYNC_IO_STDERR_ENV,
  RUN_NODE_OUTPUT_LOG_ENV,
  "OPENCLAW_SKIP_CHANNELS",
@@ -106,6 +109,7 @@ const resolveGatewayWatchBenchmarkArgs = ({ args = [], env = process.env } = {})
  if (benchmarkFlagSeen) {
    nextEnv[RUN_NODE_CPU_PROF_DIR_ENV] =
      benchmarkDir || nextEnv[RUN_NODE_CPU_PROF_DIR_ENV] || DEFAULT_BENCHMARK_PROFILE_DIR;
+    nextEnv[RUN_NODE_CPU_PROF_MAX_FILES_ENV] ??= DEFAULT_BENCHMARK_PROFILE_MAX_FILES;
    nextEnv.OPENCLAW_TRACE_SYNC_IO ??= "0";
    if (nextEnv.OPENCLAW_TRACE_SYNC_IO === "1") {
      nextEnv[RUN_NODE_OUTPUT_LOG_ENV] ??= joinArtifactPath(
--- a/scripts/run-node.mjs
+++ b/scripts/run-node.mjs
@@ -613,6 +613,7 @@ const getSignalExitCode = (signal) => (isSignalKey(signal) ? SIGNAL_EXIT_CODES[s

 const RUN_NODE_OUTPUT_LOG_ENV = "OPENCLAW_RUN_NODE_OUTPUT_LOG";
 const RUN_NODE_CPU_PROF_DIR_ENV = "OPENCLAW_RUN_NODE_CPU_PROF_DIR";
+const RUN_NODE_CPU_PROF_MAX_FILES_ENV = "OPENCLAW_RUN_NODE_CPU_PROF_MAX_FILES";
 const RUN_NODE_FILTER_SYNC_IO_STDERR_ENV = "OPENCLAW_RUN_NODE_FILTER_SYNC_IO_STDERR";
 const RUN_NODE_BUILD_LOCK_TIMEOUT_ENV = "OPENCLAW_RUN_NODE_BUILD_LOCK_TIMEOUT_MS";
 const RUN_NODE_BUILD_LOCK_POLL_ENV = "OPENCLAW_RUN_NODE_BUILD_LOCK_POLL_MS";
@@ -774,6 +775,52 @@ const sanitizeCpuProfileNamePart = (value) => {
  return normalized || "command";
 };

+const parsePositiveInteger = (value) => {
+  const parsed = Number(value);
+  return Number.isInteger(parsed) && parsed > 0 ? parsed : null;
+};
+
+const listRunNodeCpuProfiles = (deps, absoluteProfileDir, commandName) => {
+  let entries = [];
+  try {
+    entries = deps.fs.readdirSync(absoluteProfileDir, { withFileTypes: true });
+  } catch {
+    return [];
+  }
+  const prefix = `openclaw-${commandName}-`;
+  return entries
+    .filter(
+      (entry) =>
+        entry.isFile() && entry.name.startsWith(prefix) && entry.name.endsWith(".cpuprofile"),
+    )
+    .flatMap((entry) => {
+      const filePath = path.join(absoluteProfileDir, entry.name);
+      try {
+        const stat = deps.fs.statSync(filePath);
+        return [{ filePath, mtimeMs: stat.mtimeMs }];
+      } catch {
+        return [];
+      }
+    })
+    .toSorted((left, right) => left.mtimeMs - right.mtimeMs);
+};
+
+const pruneRunNodeCpuProfiles = (deps, absoluteProfileDir, commandName) => {
+  const maxFiles = parsePositiveInteger(deps.env[RUN_NODE_CPU_PROF_MAX_FILES_ENV]);
+  if (!maxFiles) {
+    return;
+  }
+  const profiles = listRunNodeCpuProfiles(deps, absoluteProfileDir, commandName);
+  const deleteCount = Math.max(0, profiles.length - maxFiles + 1);
+  for (const profile of profiles.slice(0, deleteCount)) {
+    try {
+      deps.fs.rmSync(profile.filePath, { force: true });
+    } catch {
+      // Best-effort artifact rotation; profiling should not fail the command.
+    }
+  }
+};
+
 const resolveRunNodeCpuProfileArgs = (deps) => {
  const profileDir = deps.env[RUN_NODE_CPU_PROF_DIR_ENV]?.trim();
  if (!profileDir) {
@@ -785,6 +832,7 @@ const resolveRunNodeCpuProfileArgs = (deps) => {
  deps.env[RUN_NODE_CPU_PROF_DIR_ENV] = absoluteProfileDir;

  const commandName = sanitizeCpuProfileNamePart(deps.args[0]);
+  pruneRunNodeCpuProfiles(deps, absoluteProfileDir, commandName);
  const timestamp = new Date().toISOString().replace(/[:.]/g, "-");
  const pid = Number.isInteger(deps.process.pid) && deps.process.pid > 0 ? deps.process.pid : "pid";
  const profileName = `openclaw-${commandName}-${pid}-${timestamp}.cpuprofile`;
--- a/src/channels/plugins/bundled.ts
+++ b/src/channels/plugins/bundled.ts
@@ -94,7 +94,9 @@ type BundledChannelLoadContext = {

 const log = createSubsystemLogger("channels");
 const MAX_BUNDLED_CHANNEL_LOAD_CONTEXTS = 32;
+const MAX_BUNDLED_CHANNEL_BOUNDARY_ROOTS = 256;
 const bundledChannelLoadContextsByRoot = new Map<string, BundledChannelLoadContext>();
+const bundledChannelBoundaryRoots = new Map<string, string>();
 const sourceBundledEntryLoaderCache: PluginModuleLoaderCache = new Map();

 function isSourceModulePath(modulePath: string): boolean {
@@ -161,27 +163,55 @@ function resolveBundledChannelBoundaryRoot(params: {
  metadata: BundledChannelPluginMetadata;
  modulePath: string;
 }): string {
+  const cacheKey = [
+    params.packageRoot,
+    params.pluginsDir ?? "",
+    params.metadata.dirName,
+    params.modulePath,
+  ].join("\0");
+  const cached = bundledChannelBoundaryRoots.get(cacheKey);
+  if (cached) {
+    bundledChannelBoundaryRoots.delete(cacheKey);
+    bundledChannelBoundaryRoots.set(cacheKey, cached);
+    return cached;
+  }
  const isModuleUnderRoot = (root: string) => isPathInside(path.resolve(root), params.modulePath);
  const overrideRoot = params.pluginsDir
    ? path.resolve(params.pluginsDir, params.metadata.dirName)
    : null;
+  let boundaryRoot: string;
  if (overrideRoot && isModuleUnderRoot(overrideRoot)) {
-    return overrideRoot;
+    boundaryRoot = overrideRoot;
+  } else {
+    const distRoot = path.resolve(
+      params.packageRoot,
+      "dist",
+      "extensions",
+      params.metadata.dirName,
+    );
+    if (isModuleUnderRoot(distRoot)) {
+      boundaryRoot = distRoot;
+    } else {
+      const distRuntimeRoot = path.resolve(
+        params.packageRoot,
+        "dist-runtime",
+        "extensions",
+        params.metadata.dirName,
+      );
+      boundaryRoot = isModuleUnderRoot(distRuntimeRoot)
+        ? distRuntimeRoot
+        : path.resolve(params.packageRoot, "extensions", params.metadata.dirName);
+    }
  }
-  const distRoot = path.resolve(params.packageRoot, "dist", "extensions", params.metadata.dirName);
-  if (isModuleUnderRoot(distRoot)) {
-    return distRoot;
+  bundledChannelBoundaryRoots.set(cacheKey, boundaryRoot);
+  while (bundledChannelBoundaryRoots.size > MAX_BUNDLED_CHANNEL_BOUNDARY_ROOTS) {
+    const oldestKey = bundledChannelBoundaryRoots.keys().next().value;
+    if (oldestKey === undefined) {
+      break;
+    }
+    bundledChannelBoundaryRoots.delete(oldestKey);
  }
-  const distRuntimeRoot = path.resolve(
-    params.packageRoot,
-    "dist-runtime",
-    "extensions",
-    params.metadata.dirName,
-  );
-  if (isModuleUnderRoot(distRuntimeRoot)) {
-    return distRuntimeRoot;
-  }
-  return path.resolve(params.packageRoot, "extensions", params.metadata.dirName);
+  return boundaryRoot;
 }

 function resolveBundledChannelScanDir(rootScope: BundledChannelRootScope): string | undefined {
--- a/src/channels/plugins/catalog.ts
+++ b/src/channels/plugins/catalog.ts
@@ -1,9 +1,12 @@
+import fs from "node:fs";
 import path from "node:path";
 import { MANIFEST_KEY } from "../../compat/legacy-names.js";
+import type { PluginInstallRecord } from "../../config/types.plugins.js";
 import { tryReadJsonSync } from "../../infra/json-files.js";
 import { isPrereleaseSemverVersion, parseRegistryNpmSpec } from "../../infra/npm-registry-spec.js";
 import { resolveOpenClawPackageRootSync } from "../../infra/openclaw-root.js";
 import { listChannelCatalogEntries } from "../../plugins/channel-catalog-registry.js";
+import type { PluginDiscoveryResult } from "../../plugins/discovery.js";
 import {
  describePluginInstallSource,
  type PluginInstallSourceInfo,
@@ -52,6 +55,8 @@ type CatalogOptions = {
  officialCatalogPaths?: string[];
  env?: NodeJS.ProcessEnv;
  excludeWorkspace?: boolean;
+  installRecords?: Record<string, PluginInstallRecord>;
+  discovery?: PluginDiscoveryResult;
 };

 const ORIGIN_PRIORITY: Record<PluginOrigin, number> = {
@@ -72,7 +77,13 @@ type ExternalCatalogEntry = {

 const ENV_CATALOG_PATHS = ["OPENCLAW_PLUGIN_CATALOG_PATHS", "OPENCLAW_MPM_CATALOG_PATHS"];
 const OFFICIAL_CHANNEL_CATALOG_RELATIVE_PATH = path.join("dist", "channel-catalog.json");
+type CatalogEntriesCacheEntry = {
+  fingerprint: string;
+  entries: ExternalCatalogEntry[] | null;
+};
+
 const officialCatalogEntriesByPath = new Map<string, ExternalCatalogEntry[] | null>();
+const externalCatalogEntriesByPath = new Map<string, CatalogEntriesCacheEntry>();

 type ManifestKey = typeof MANIFEST_KEY;

@@ -129,17 +140,43 @@ function loadExternalCatalogEntries(options: CatalogOptions): ExternalCatalogEnt
  const paths = resolveExternalCatalogPaths(options).map((rawPath) =>
    resolveUserPath(rawPath, options.env ?? process.env),
  );
-  return loadCatalogEntriesFromPaths(paths);
+  return loadCatalogEntriesFromPaths(paths, externalCatalogEntriesByPath);
 }

-function loadCatalogEntriesFromPaths(paths: Iterable<string>): ExternalCatalogEntry[] {
+function fingerprintCatalogPath(filePath: string): string {
+  try {
+    const stat = fs.statSync(filePath, { bigint: true });
+    const kind = stat.isFile() ? "file" : stat.isDirectory() ? "dir" : "other";
+    return [kind, stat.size.toString(), stat.mtimeNs.toString(), stat.ctimeNs.toString()].join(":");
+  } catch {
+    return "missing";
+  }
+}
+
+function readCatalogEntriesFromPath(resolvedPath: string): ExternalCatalogEntry[] | null {
+  const payload = tryReadJsonSync(resolvedPath);
+  return payload === null ? null : parseCatalogEntries(payload);
+}
+
+function loadCatalogEntriesFromPaths(
+  paths: Iterable<string>,
+  cache?: Map<string, CatalogEntriesCacheEntry>,
+): ExternalCatalogEntry[] {
  const entries: ExternalCatalogEntry[] = [];
  for (const resolvedPath of paths) {
-    const payload = tryReadJsonSync(resolvedPath);
-    if (payload === null) {
+    const fingerprint = cache ? fingerprintCatalogPath(resolvedPath) : undefined;
+    const cached = fingerprint ? cache?.get(resolvedPath) : undefined;
+    const parsed =
+      cached && cached.fingerprint === fingerprint
+        ? cached.entries
+        : readCatalogEntriesFromPath(resolvedPath);
+    if (cache && fingerprint) {
+      cache.set(resolvedPath, { fingerprint, entries: parsed });
+    }
+    if (parsed === null) {
      continue;
    }
-    entries.push(...parseCatalogEntries(payload));
+    entries.push(...parsed);
  }
  return entries;
 }
@@ -399,6 +436,8 @@ export function listChannelPluginCatalogEntries(
  const manifestEntries = listChannelCatalogEntries({
    workspaceDir: options.workspaceDir,
    env: options.env,
+    installRecords: options.installRecords,
+    discovery: options.discovery,
  });
  const resolved = new Map<string, { entry: ChannelPluginCatalogEntry; priority: number }>();

--- a/src/infra/gateway-watch-tmux.test.ts
+++ b/src/infra/gateway-watch-tmux.test.ts
@@ -121,6 +121,7 @@ describe("gateway-watch tmux wrapper", () => {
    expect(code).toBe(0);
    const command = spawnShellCommand(spawnSync, 1);
    expect(command).toContain("'OPENCLAW_RUN_NODE_CPU_PROF_DIR=.artifacts/gateway-watch-profiles'");
+    expect(command).toContain("'OPENCLAW_RUN_NODE_CPU_PROF_MAX_FILES=40'");
    expect(command).toContain("'OPENCLAW_TRACE_SYNC_IO=0'");
    expect(command).not.toContain("--benchmark");
    expect(command).toContain("'gateway'");
@@ -130,6 +131,31 @@ describe("gateway-watch tmux wrapper", () => {
    );
  });

+  it("preserves an explicit benchmark CPU profile retention cap", () => {
+    const stdout = createOutput();
+    const stderr = createOutput();
+    const spawnSync = vi
+      .fn()
+      .mockReturnValueOnce({ status: 1, stdout: "", stderr: "" })
+      .mockReturnValueOnce({ status: 0, stdout: "", stderr: "" })
+      .mockReturnValueOnce({ status: 0, stdout: "", stderr: "" })
+      .mockReturnValueOnce({ status: 0, stdout: "", stderr: "" });
+
+    const code = runGatewayWatchTmuxMain({
+      args: ["gateway", "--force", "--benchmark"],
+      cwd: "/repo",
+      env: { OPENCLAW_RUN_NODE_CPU_PROF_MAX_FILES: "8", SHELL: "/bin/zsh" },
+      nodePath: "/node",
+      spawnSync,
+      stderr: stderr.stream,
+      stdout: stdout.stream,
+    });
+
+    expect(code).toBe(0);
+    const command = spawnShellCommand(spawnSync, 1);
+    expect(command).toContain("'OPENCLAW_RUN_NODE_CPU_PROF_MAX_FILES=8'");
+  });
+
  it("preserves explicit sync I/O tracing in benchmark mode", () => {
    const stdout = createOutput();
    const stderr = createOutput();
--- a/src/infra/run-node.test.ts
+++ b/src/infra/run-node.test.ts
@@ -673,6 +673,60 @@ describe("run-node script", () => {
    });
  });

+  it("rotates old Node CPU profiles when a retention cap is set", async () => {
+    await withTempDir({ prefix: "openclaw-run-node-" }, async (tmp) => {
+      await setupTrackedProject(tmp, {
+        files: {
+          [ROOT_SRC]: "export const value = 1;\n",
+        },
+        oldPaths: [ROOT_SRC, ROOT_TSCONFIG, ROOT_PACKAGE],
+        buildPaths: [DIST_ENTRY, BUILD_STAMP],
+      });
+      const profileDir = path.join(tmp, ".artifacts", "profiles");
+      fsSync.mkdirSync(profileDir, { recursive: true });
+      const oldProfiles = [
+        "openclaw-status-oldest.cpuprofile",
+        "openclaw-status-middle.cpuprofile",
+        "openclaw-status-newest.cpuprofile",
+      ];
+      for (const [index, name] of oldProfiles.entries()) {
+        const filePath = path.join(profileDir, name);
+        fsSync.writeFileSync(filePath, "{}");
+        const mtime = new Date(1_700_000_000_000 + index * 1000);
+        fsSync.utimesSync(filePath, mtime, mtime);
+      }
+      fsSync.writeFileSync(path.join(profileDir, "openclaw-models-old.cpuprofile"), "{}");
+
+      const spawn = () => createExitedProcess(0);
+      const { spawnSync } = createSpawnRecorder({
+        gitHead: "abc123\n",
+        gitStatus: "",
+      });
+
+      const exitCode = await runNodeMain({
+        cwd: tmp,
+        args: ["status"],
+        env: {
+          ...process.env,
+          OPENCLAW_RUNNER_LOG: "0",
+          OPENCLAW_RUN_NODE_CPU_PROF_DIR: ".artifacts/profiles",
+          OPENCLAW_RUN_NODE_CPU_PROF_MAX_FILES: "2",
+        },
+        spawn,
+        spawnSync,
+        execPath: process.execPath,
+        platform: process.platform,
+        process: createFakeProcess(),
+      });
+
+      expect(exitCode).toBe(0);
+      expect(fsSync.existsSync(path.join(profileDir, oldProfiles[0]))).toBe(false);
+      expect(fsSync.existsSync(path.join(profileDir, oldProfiles[1]))).toBe(false);
+      expect(fsSync.existsSync(path.join(profileDir, oldProfiles[2]))).toBe(true);
+      expect(fsSync.existsSync(path.join(profileDir, "openclaw-models-old.cpuprofile"))).toBe(true);
+    });
+  });
+
  it("adds Node sync I/O tracing flag to the launched OpenClaw child when requested", async () => {
    await withTempDir({ prefix: "openclaw-run-node-" }, async (tmp) => {
      await setupTrackedProject(tmp, {