test(startup): make cli startup budgets arch-aware

2026-06-06 05:51:15 +08:00 · 2026-06-03 09:25:57 -07:00
parent 158c4d7540
commit 286e5ffe07
4 changed files with 222 additions and 59 deletions
--- a/scripts/bench-cli-startup.ts
+++ b/scripts/bench-cli-startup.ts
@@ -552,7 +552,12 @@ function collectExitSummary(samples: Sample[]): string {
 }

 function buildConfigFixture(commandCase: CommandCase): Record<string, unknown> | null {
-  if (commandCase.id !== "configGetGatewayPort" && commandCase.id !== "gatewayHealthJson") {
+  if (
+    commandCase.id !== "configGetGatewayPort" &&
+    commandCase.id !== "gatewayHealthJson" &&
+    commandCase.id !== "health" &&
+    commandCase.id !== "healthJson"
+  ) {
    return null;
  }
  const port = parseGatewayPortEnv(process.env.OPENCLAW_GATEWAY_PORT);
--- a/scripts/test-cli-startup-bench-budget.mjs
+++ b/scripts/test-cli-startup-bench-budget.mjs
@@ -35,6 +35,11 @@ if (process.argv.slice(2).includes("--help")) {
      "                                Fail if avg first-output time regresses more than this percent",
      "  --max-rss-regression-pct <n>  Fail if avg RSS regresses more than this percent",
      "  --skip-baseline               Skip fixture regression checks and enforce case contracts only",
+      "  --skip-response-budgets       Skip response first-output and exit budget contracts",
+      "",
+      "Non-x64 runs skip fixture regression checks by default because the",
+      "checked-in startup fixture is a canonical x64 budget. Response contracts still run. Set",
+      "OPENCLAW_STARTUP_BENCH_ENFORCE_NONCANONICAL_ARCH=1 to force them.",
      "  --help                        Show this help text",
      "",
      "Example:",
@@ -63,6 +68,7 @@ try {
      maxRssRegressionPct:
        readBudgetEnvNumber("OPENCLAW_STARTUP_BENCH_MAX_RSS_REGRESSION_PCT") ?? 20,
      skipBaseline: false,
+      skipResponseBudgets: false,
    },
    [
      stringFlag("--baseline", "baseline"),
@@ -76,6 +82,7 @@ try {
      budgetFloatFlag("--max-first-output-regression-pct", "maxFirstOutputRegressionPct"),
      budgetFloatFlag("--max-rss-regression-pct", "maxRssRegressionPct"),
      booleanFlag("--skip-baseline", "skipBaseline"),
+      booleanFlag("--skip-response-budgets", "skipResponseBudgets"),
    ],
  );
 } catch (error) {
@@ -83,6 +90,18 @@ try {
  process.exit(1);
 }

+const shouldAutoSkipNonCanonicalBaselineChecks =
+  process.arch !== "x64" && process.env.OPENCLAW_STARTUP_BENCH_ENFORCE_NONCANONICAL_ARCH !== "1";
+if (shouldAutoSkipNonCanonicalBaselineChecks && !opts.skipBaseline) {
+  console.warn(
+    `[test-cli-startup-bench-budget] skipping x64 startup fixture budgets on ${process.arch}; response contracts and sample output validation still ran. Set OPENCLAW_STARTUP_BENCH_ENFORCE_NONCANONICAL_ARCH=1 to force fixture checks.`,
+  );
+  opts = {
+    ...opts,
+    skipBaseline: true,
+  };
+}
+
 function resolveCurrentReportPath() {
  if (opts.report) {
    return opts.report;
@@ -137,14 +156,24 @@ const shouldRequireEveryBaselineCase = opts.preset === "all";

 let failed = false;

+for (const [id] of baselineCases) {
+  if (shouldRequireEveryBaselineCase && !currentCases.has(id)) {
+    console.error(`[test-cli-startup-bench-budget] missing current case ${String(id)}`);
+    failed = true;
+  }
+}
+
+for (const currentCase of currentCases.values()) {
+  if ((currentCase.samples ?? []).length === 0) {
+    console.error(`[test-cli-startup-bench-budget] ${currentCase.name} has no measured samples.`);
+    failed = true;
+  }
+}
+
 if (!opts.skipBaseline) {
  for (const [id, baselineCase] of baselineCases) {
    const currentCase = currentCases.get(id);
    if (!currentCase) {
-      if (shouldRequireEveryBaselineCase) {
-        console.error(`[test-cli-startup-bench-budget] missing current case ${String(id)}`);
-        failed = true;
-      }
      continue;
    }

@@ -236,36 +265,42 @@ for (const currentCase of currentCases.values()) {
    failed = true;
  }

-  const firstOutputBudgetMs = contract.firstOutputBudgetMs;
-  const firstOutputMax = currentCase.summary?.firstOutputMs?.max;
-  if (Number.isFinite(firstOutputBudgetMs)) {
-    if (!Number.isFinite(firstOutputMax)) {
+  if (!opts.skipResponseBudgets) {
+    const firstOutputBudgetMs = contract.firstOutputBudgetMs;
+    const firstOutputMax = currentCase.summary?.firstOutputMs?.max;
+    if (Number.isFinite(firstOutputBudgetMs)) {
+      if (!Number.isFinite(firstOutputMax)) {
+        console.error(
+          `[test-cli-startup-bench-budget] ${currentCase.name} produced no stdout/stderr before exit; response contract requires first output within ${formatMs(
+            firstOutputBudgetMs,
+          )}.`,
+        );
+        failed = true;
+      } else if (firstOutputMax > firstOutputBudgetMs) {
+        console.error(
+          `[test-cli-startup-bench-budget] ${currentCase.name} first output ${formatMs(
+            firstOutputMax,
+          )} exceeded contract ${formatMs(firstOutputBudgetMs)}.`,
+        );
+        failed = true;
+      }
+    }
+
+    const exitBudgetMs = contract.exitBudgetMs;
+    const durationMax = currentCase.summary?.durationMs?.max;
+    if (
+      Number.isFinite(exitBudgetMs) &&
+      Number.isFinite(durationMax) &&
+      durationMax > exitBudgetMs
+    ) {
      console.error(
-        `[test-cli-startup-bench-budget] ${currentCase.name} produced no stdout/stderr before exit; response contract requires first output within ${formatMs(
-          firstOutputBudgetMs,
-        )}.`,
-      );
-      failed = true;
-    } else if (firstOutputMax > firstOutputBudgetMs) {
-      console.error(
-        `[test-cli-startup-bench-budget] ${currentCase.name} first output ${formatMs(
-          firstOutputMax,
-        )} exceeded contract ${formatMs(firstOutputBudgetMs)}.`,
+        `[test-cli-startup-bench-budget] ${currentCase.name} exit ${formatMs(
+          durationMax,
+        )} exceeded contract ${formatMs(exitBudgetMs)}.`,
      );
      failed = true;
    }
  }
-
-  const exitBudgetMs = contract.exitBudgetMs;
-  const durationMax = currentCase.summary?.durationMs?.max;
-  if (Number.isFinite(exitBudgetMs) && Number.isFinite(durationMax) && durationMax > exitBudgetMs) {
-    console.error(
-      `[test-cli-startup-bench-budget] ${currentCase.name} exit ${formatMs(
-        durationMax,
-      )} exceeded contract ${formatMs(exitBudgetMs)}.`,
-    );
-    failed = true;
-  }
 }

 if (failed) {
--- a/test/scripts/bench-cli-startup.test.ts
+++ b/test/scripts/bench-cli-startup.test.ts
@@ -187,40 +187,41 @@ describe("bench-cli-startup", () => {
  });

  it("writes a config fixture for config get benchmarks", () => {
-    expect(
-      withEnv({ OPENCLAW_GATEWAY_PORT: undefined }, () =>
-        testing.buildConfigFixture({
-          id: "configGetGatewayPort",
-          name: "config get gateway.port",
-          args: ["config", "get", "gateway.port"],
-          presets: ["real"],
-        }),
-      ),
-    ).toEqual({
+    const expectedFixture = {
      gateway: {
        auth: { mode: "none" },
        bind: "loopback",
        mode: "local",
        port: 32123,
      },
-    });
-    expect(
-      withEnv({ OPENCLAW_GATEWAY_PORT: undefined }, () =>
-        testing.buildConfigFixture({
-          id: "gatewayHealthJson",
-          name: "gateway health --json",
-          args: ["gateway", "health", "--json"],
-          presets: ["real"],
-        }),
-      ),
-    ).toEqual({
-      gateway: {
-        auth: { mode: "none" },
-        bind: "loopback",
-        mode: "local",
-        port: 32123,
+    };
+    for (const commandCase of [
+      {
+        id: "configGetGatewayPort",
+        name: "config get gateway.port",
+        args: ["config", "get", "gateway.port"],
+        presets: ["real"],
      },
-    });
+      {
+        id: "gatewayHealthJson",
+        name: "gateway health --json",
+        args: ["gateway", "health", "--json"],
+        presets: ["real"],
+      },
+      { id: "health", name: "health", args: ["health"], presets: ["startup", "real"] },
+      {
+        id: "healthJson",
+        name: "health --json",
+        args: ["health", "--json"],
+        presets: ["startup"],
+      },
+    ]) {
+      expect(
+        withEnv({ OPENCLAW_GATEWAY_PORT: undefined }, () =>
+          testing.buildConfigFixture(commandCase),
+        ),
+      ).toEqual(expectedFixture);
+    }
  });

  it("parses config fixture gateway ports strictly from env", () => {
--- a/test/scripts/cli-startup-bench-spawner.test.ts
+++ b/test/scripts/cli-startup-bench-spawner.test.ts
@@ -41,7 +41,7 @@ describe("CLI startup benchmark script spawners", () => {
      const makeCase = (id: string, name: string) => ({
        id,
        name,
-        samples: [],
+        samples: [{ ms: 10, firstOutputMs: 5, maxRssMb: 10, exitCode: 0, signal: null }],
        summary: {
          durationMs: { avg: 10, p50: 10, p95: 10, min: 10, max: 10 },
          firstOutputMs: null,
@@ -88,7 +88,14 @@ describe("CLI startup benchmark script spawners", () => {
            "--preset",
            "all",
          ],
-          { cwd: process.cwd(), stdio: "pipe" },
+          {
+            cwd: process.cwd(),
+            env: {
+              ...process.env,
+              OPENCLAW_STARTUP_BENCH_ENFORCE_NONCANONICAL_ARCH: "1",
+            },
+            stdio: "pipe",
+          },
        ),
      ).toThrow();
    } finally {
@@ -96,6 +103,121 @@ describe("CLI startup benchmark script spawners", () => {
    }
  });

+  it("skips x64 startup budgets on noncanonical architectures", () => {
+    const tmpDir = fs.mkdtempSync(path.join(os.tmpdir(), "openclaw-bench-budget-arch-test-"));
+    try {
+      const archShimPath = path.join(tmpDir, "arch-shim.mjs");
+      const baselinePath = path.join(tmpDir, "baseline.json");
+      const reportPath = path.join(tmpDir, "current.json");
+      const slowCase = {
+        id: "slow",
+        name: "slow",
+        contract: {
+          firstOutputBudgetMs: 20,
+          exitBudgetMs: 20,
+        },
+        samples: [{ ms: 10, firstOutputMs: 10, maxRssMb: 100, exitCode: 0, signal: null }],
+        summary: {
+          durationMs: { avg: 10, p50: 10, p95: 10, min: 10, max: 10 },
+          firstOutputMs: { avg: 10, p50: 10, p95: 10, min: 10, max: 10 },
+          maxRssMb: { avg: 100, p50: 100, p95: 100, min: 100, max: 100 },
+        },
+      };
+      fs.writeFileSync(
+        archShimPath,
+        'Object.defineProperty(process, "arch", { value: "arm64" });\n',
+      );
+      fs.writeFileSync(
+        baselinePath,
+        JSON.stringify({
+          primary: {
+            cases: [
+              {
+                ...slowCase,
+                summary: {
+                  ...slowCase.summary,
+                  durationMs: { avg: 1, p50: 1, p95: 1, min: 1, max: 1 },
+                },
+              },
+            ],
+          },
+        }),
+      );
+      fs.writeFileSync(reportPath, JSON.stringify({ primary: { cases: [slowCase] } }));
+
+      const result = spawnSync(
+        process.execPath,
+        [
+          "--import",
+          archShimPath,
+          "scripts/test-cli-startup-bench-budget.mjs",
+          "--baseline",
+          baselinePath,
+          "--report",
+          reportPath,
+          "--preset",
+          "all",
+        ],
+        { cwd: process.cwd(), encoding: "utf8" },
+      );
+
+      expect(result.status).toBe(0);
+      expect(result.stderr).toContain("skipping x64 startup fixture budgets on arm64");
+      expect(result.stderr).not.toContain("exceeded");
+
+      const slowResponseCase = {
+        ...slowCase,
+        contract: {
+          firstOutputBudgetMs: 1,
+          exitBudgetMs: 1,
+        },
+      };
+      fs.writeFileSync(
+        baselinePath,
+        JSON.stringify({ primary: { cases: [slowResponseCase] } }),
+      );
+      fs.writeFileSync(reportPath, JSON.stringify({ primary: { cases: [slowResponseCase] } }));
+      const responseBudgetResult = spawnSync(
+        process.execPath,
+        [
+          "--import",
+          archShimPath,
+          "scripts/test-cli-startup-bench-budget.mjs",
+          "--baseline",
+          baselinePath,
+          "--report",
+          reportPath,
+          "--preset",
+          "all",
+        ],
+        { cwd: process.cwd(), encoding: "utf8" },
+      );
+      expect(responseBudgetResult.status).toBe(1);
+      expect(responseBudgetResult.stderr).toContain("first output 10.0ms exceeded contract 1.0ms");
+
+      fs.writeFileSync(reportPath, JSON.stringify({ primary: { cases: [] } }));
+      const missingCaseResult = spawnSync(
+        process.execPath,
+        [
+          "--import",
+          archShimPath,
+          "scripts/test-cli-startup-bench-budget.mjs",
+          "--baseline",
+          baselinePath,
+          "--report",
+          reportPath,
+          "--preset",
+          "all",
+        ],
+        { cwd: process.cwd(), encoding: "utf8" },
+      );
+      expect(missingCaseResult.status).toBe(1);
+      expect(missingCaseResult.stderr).toContain("missing current case slow");
+    } finally {
+      fs.rmSync(tmpDir, { recursive: true, force: true });
+    }
+  });
+
  it("rejects malformed startup budget env vars before reading reports", () => {
    const result = spawnSync(process.execPath, ["scripts/test-cli-startup-bench-budget.mjs"], {
      cwd: process.cwd(),