test(live): keep cache prereq skips provider-aware

src/agents/live-cache-regression-runner.test.ts

@@ -1,5 +1,10 @@
 import { describe, expect, it } from "vitest";
 import { testing } from "./live-cache-regression-runner.js";
+import { ProviderAuthError } from "./model-auth-runtime-shared.js";
+import {
+  LiveCachePrerequisiteSkip,
+  toLiveCachePrerequisiteSkip,
+} from "./live-cache-test-support.js";
 
 describe("live cache regression runner", () => {
   it("keeps OpenAI image cache floors observable without blocking release validation", () => {
@@ -84,6 +89,85 @@ describe("live cache regression runner", () => {
     ).toBe(false);
   });
 
+  it("keeps missing optional live-cache prerequisites non-blocking", async () => {
+    const regressions: string[] = [];
+    const warnings: string[] = [];
+    const summary: Record<string, Record<string, unknown>> = {
+      anthropic: {},
+      openai: {},
+    };
+
+    const resolved = await testing.resolveLiveCacheProviderPool({
+      config: {
+        provider: "openai",
+        api: "openai-responses",
+        envVar: "OPENCLAW_LIVE_OPENAI_CACHE_MODEL",
+        preferredModelIds: ["gpt-5.5"],
+      },
+      resolver: async () => {
+        throw new LiveCachePrerequisiteSkip(
+          "openai",
+          "No openai openai-responses model available in registry.",
+        );
+      },
+      regressions,
+      summary,
+      warnings,
+    });
+
+    expect(resolved).toBeUndefined();
+    expect(regressions).toStrictEqual([]);
+    expect(warnings).toEqual([
+      "openai skipped: No openai openai-responses model available in registry.",
+    ]);
+    expect(summary.openai).toEqual({ skipped: true });
+  });
+
+  it("keeps missing Anthropic live-cache prerequisites blocking", async () => {
+    const regressions: string[] = [];
+    const warnings: string[] = [];
+    const summary: Record<string, Record<string, unknown>> = {
+      anthropic: {},
+      openai: {},
+    };
+
+    const resolved = await testing.resolveLiveCacheProviderPool({
+      config: {
+        provider: "anthropic",
+        api: "anthropic-messages",
+        envVar: "OPENCLAW_LIVE_ANTHROPIC_CACHE_MODEL",
+        preferredModelIds: ["claude-sonnet-4-6"],
+      },
+      resolver: async () => {
+        throw new LiveCachePrerequisiteSkip(
+          "anthropic",
+          "No anthropic anthropic-messages model available in registry.",
+        );
+      },
+      regressions,
+      summary,
+      warnings,
+    });
+
+    expect(resolved).toBeUndefined();
+    expect(regressions).toEqual([
+      "anthropic skipped: No anthropic anthropic-messages model available in registry.",
+    ]);
+    expect(warnings).toStrictEqual([]);
+    expect(summary.anthropic).toEqual({ skipped: true });
+  });
+
+  it("classifies missing provider auth as a live-cache prerequisite", () => {
+    const skip = toLiveCachePrerequisiteSkip(
+      "openai",
+      new ProviderAuthError("missing-provider-auth", "openai", "No API key found."),
+    );
+
+    expect(skip).toBeInstanceOf(LiveCachePrerequisiteSkip);
+    expect(skip?.provider).toBe("openai");
+    expect(skip?.message).toBe("No API key found.");
+  });
+
   it("retries a cache probe twice when provider text misses the sentinel", () => {
     expect(
       testing.shouldRetryCacheProbeText({

src/agents/live-cache-regression-runner.ts

@@ -13,7 +13,9 @@ import {
   completeSimpleWithLiveTimeout,
   computeCacheHitRate,
   extractAssistantText,
+  isLiveCachePrerequisiteSkip,
   type LiveResolvedModel,
+  type LiveResolvedModelPool,
   logLiveCache,
   resolveLiveDirectModelPool,
   withLiveDirectModelApiKey,
@@ -35,6 +37,7 @@ const LIVE_TEST_PNG_URL = new URL(
   import.meta.url,
 );
 
+type LiveCacheProviderConfig = Parameters<typeof resolveLiveDirectModelPool>[0];
 type ProviderKey = keyof typeof LIVE_CACHE_REGRESSION_BASELINE;
 type CacheLane = "image" | "mcp" | "stable" | "tool";
 type CacheUsage = {
@@ -65,6 +68,8 @@ type LiveCacheRegressionResult = {
   summary: Record<string, Record<string, unknown>>;
   warnings: string[];
 };
+type LiveCacheRegressionSummary = LiveCacheRegressionResult["summary"];
+type LiveCacheProviderResolver = (params: LiveCacheProviderConfig) => Promise<LiveResolvedModelPool>;
 
 class CacheProbeTextMismatchError extends Error {
   constructor(
@@ -95,6 +100,31 @@ function makeUserTurn(content: Extract<Message, { role: "user" }>["content"]): M
   };
 }
 
+async function resolveLiveCacheProviderPool(params: {
+  config: LiveCacheProviderConfig;
+  regressions: string[];
+  resolver?: LiveCacheProviderResolver;
+  summary: LiveCacheRegressionSummary;
+  warnings: string[];
+}): Promise<LiveResolvedModelPool | undefined> {
+  try {
+    return await (params.resolver ?? resolveLiveDirectModelPool)(params.config);
+  } catch (error) {
+    if (!isLiveCachePrerequisiteSkip(error)) {
+      throw error;
+    }
+    const warning = `${error.provider} skipped: ${error.message}`;
+    if (error.provider === "openai") {
+      params.warnings.push(warning);
+    } else {
+      params.regressions.push(warning);
+    }
+    params.summary[error.provider].skipped = true;
+    logLiveCache(warning);
+    return undefined;
+  }
+}
+
 function makeImageUserTurn(text: string, pngBase64: string): Message {
   return makeUserTurn([
     { type: "text", text },
@@ -699,6 +729,7 @@ async function runAnthropicDisabledCacheLane(params: {
 export const testing = {
   assertAgainstBaseline,
   evaluateAgainstBaseline,
+  resolveLiveCacheProviderPool,
   resolveCacheProbeMaxTokens,
   isAnthropicToolProbeDrift,
   shouldAcceptEmptyCacheProbe,
@@ -709,49 +740,66 @@ export const testing = {
 export async function runLiveCacheRegression(): Promise<LiveCacheRegressionResult> {
   const pngBase64 = (await fs.readFile(LIVE_TEST_PNG_URL)).toString("base64");
   const runToken = randomUUID().slice(0, 13);
-  const openai = await resolveLiveDirectModelPool({
-    provider: "openai",
-    api: "openai-responses",
-    envVar: "OPENCLAW_LIVE_OPENAI_CACHE_MODEL",
-    preferredModelIds: ["gpt-4.1", "gpt-5.2", "gpt-5.4-mini", "gpt-5.4", "gpt-5.5"],
-  });
-  const anthropic = await resolveLiveDirectModelPool({
-    provider: "anthropic",
-    api: "anthropic-messages",
-    envVar: "OPENCLAW_LIVE_ANTHROPIC_CACHE_MODEL",
-    preferredModelIds: ["claude-sonnet-4-6", "claude-sonnet-4-5", "claude-haiku-3-5"],
-  });
-
   const regressions: string[] = [];
   const warnings: string[] = [];
   const summary: Record<string, Record<string, unknown>> = {
     anthropic: {},
     openai: {},
   };
+  const openai = await resolveLiveCacheProviderPool({
+    config: {
+      provider: "openai",
+      api: "openai-responses",
+      envVar: "OPENCLAW_LIVE_OPENAI_CACHE_MODEL",
+      preferredModelIds: ["gpt-4.1", "gpt-5.2", "gpt-5.4-mini", "gpt-5.4", "gpt-5.5"],
+    },
+    regressions,
+    summary,
+    warnings,
+  });
+  const anthropic = await resolveLiveCacheProviderPool({
+    config: {
+      provider: "anthropic",
+      api: "anthropic-messages",
+      envVar: "OPENCLAW_LIVE_ANTHROPIC_CACHE_MODEL",
+      preferredModelIds: ["claude-sonnet-4-6", "claude-sonnet-4-5", "claude-haiku-3-5"],
+    },
+    regressions,
+    summary,
+    warnings,
+  });
 
   for (const lane of ["stable", "tool", "image", "mcp"] as const) {
-    const openaiAttempt = await runRepeatedLaneWithBaselineRetry({
-      lane,
-      providerTag: "openai",
-      fixture: openai.fixture,
-      runToken,
-      pngBase64,
-    });
-    const openaiResult = openaiAttempt.result;
-    logLiveCache(
-      `openai ${lane} warmup ${formatUsage(openaiResult.warmup?.usage ?? {})} rate=${openaiResult.warmup?.hitRate.toFixed(3) ?? "0.000"}`,
-    );
-    logLiveCache(
-      `openai ${lane} best ${formatUsage(openaiResult.best?.usage ?? {})} rate=${openaiResult.best?.hitRate.toFixed(3) ?? "0.000"}`,
-    );
-    summary.openai[lane] = {
-      best: openaiResult.best?.usage,
-      hitRate: openaiResult.best?.hitRate,
-      attempts: openaiAttempt.attempts,
-      warmup: openaiResult.warmup?.usage,
-    };
-    appendBaselineFindings({ regressions, warnings }, openaiAttempt.findings);
+    if (openai) {
+      const openaiAttempt = await runRepeatedLaneWithBaselineRetry({
+        lane,
+        providerTag: "openai",
+        fixture: openai.fixture,
+        runToken,
+        pngBase64,
+      });
+      const openaiResult = openaiAttempt.result;
+      logLiveCache(
+        `openai ${lane} warmup ${formatUsage(openaiResult.warmup?.usage ?? {})} rate=${openaiResult.warmup?.hitRate.toFixed(3) ?? "0.000"}`,
+      );
+      logLiveCache(
+        `openai ${lane} best ${formatUsage(openaiResult.best?.usage ?? {})} rate=${openaiResult.best?.hitRate.toFixed(3) ?? "0.000"}`,
+      );
+      summary.openai[lane] = {
+        best: openaiResult.best?.usage,
+        hitRate: openaiResult.best?.hitRate,
+        attempts: openaiAttempt.attempts,
+        warmup: openaiResult.warmup?.usage,
+      };
+      appendBaselineFindings({ regressions, warnings }, openaiAttempt.findings);
+    } else {
+      summary.openai[lane] = { skipped: true };
+    }
 
+    if (!anthropic) {
+      summary.anthropic[lane] = { skipped: true };
+      continue;
+    }
     const { attempt: anthropicAttempt } = await runAnthropicCacheLane({
       apiKeys: anthropic.apiKeys,
       lane,
@@ -780,11 +828,13 @@ export async function runLiveCacheRegression(): Promise<LiveCacheRegressionResul
     appendBaselineFindings({ regressions, warnings }, anthropicAttempt.findings);
   }
 
-  const disabled = await runAnthropicDisabledCacheLane({
-    fixture: anthropic.fixture,
-    runToken,
-    warnings,
-  });
+  const disabled = anthropic
+    ? await runAnthropicDisabledCacheLane({
+        fixture: anthropic.fixture,
+        runToken,
+        warnings,
+      })
+    : undefined;
   if (disabled) {
     logLiveCache(`anthropic disabled ${formatUsage(disabled.disabled?.usage ?? {})}`);
     summary.anthropic.disabled = {

src/agents/live-cache-test-support.ts

@@ -7,7 +7,12 @@ import { discoverAuthStorage, discoverModels } from "./agent-model-discovery.js"
 import { resolveDefaultAgentDir } from "./agent-scope.js";
 import { collectProviderApiKeys } from "./live-auth-keys.js";
 import { isLiveTestEnabled } from "./live-test-helpers.js";
-import { getApiKeyForModel, requireApiKey } from "./model-auth.js";
+import {
+  getApiKeyForModel,
+  isMissingProviderAuthError,
+  isProviderAuthError,
+  requireApiKey,
+} from "./model-auth.js";
 import { normalizeProviderId, parseModelRef } from "./model-selection.js";
 import { ensureOpenClawModelsJson } from "./models-config.js";
 import { buildAssistantMessageWithZeroUsage } from "./stream-message-shared.js";
@@ -28,6 +33,30 @@ export type LiveResolvedModelPool = {
   fixture: LiveResolvedModel;
 };
 
+export class LiveCachePrerequisiteSkip extends Error {
+  constructor(
+    readonly provider: "anthropic" | "openai",
+    reason: string,
+  ) {
+    super(reason);
+    this.name = "LiveCachePrerequisiteSkip";
+  }
+}
+
+export function isLiveCachePrerequisiteSkip(error: unknown): error is LiveCachePrerequisiteSkip {
+  return error instanceof LiveCachePrerequisiteSkip;
+}
+
+export function toLiveCachePrerequisiteSkip(
+  provider: "anthropic" | "openai",
+  error: unknown,
+): LiveCachePrerequisiteSkip | undefined {
+  if (isMissingProviderAuthError(error) || isProviderAuthError(error, "missing-provider-auth")) {
+    return new LiveCachePrerequisiteSkip(provider, error.message);
+  }
+  return undefined;
+}
+
 function toInt(value: string | undefined, fallback: number): number {
   const trimmed = value?.trim();
   if (!trimmed) {
@@ -197,11 +226,13 @@ export async function resolveLiveDirectModelPool(params: {
   if (liveKeys.length > 0) {
     const selectedModel = selectModel();
     if (!selectedModel || selectedModel.api !== params.api) {
-      throw new Error(
-        requestedModelId
-          ? `Model not found for ${params.provider}: ${requestedModelId}`
-          : `No built-in ${params.provider} ${params.api} model available.`,
-      );
+      const message = requestedModelId
+        ? `Model not found for ${params.provider}: ${requestedModelId}`
+        : `No built-in ${params.provider} ${params.api} model available.`;
+      if (requestedModelId) {
+        throw new Error(message);
+      }
+      throw new LiveCachePrerequisiteSkip(params.provider, message);
     }
     logLiveCache(`resolved ${params.provider} model ${selectedModel.id} from live env key`);
     return {
@@ -216,21 +247,32 @@ export async function resolveLiveDirectModelPool(params: {
   logLiveCache(`resolving ${params.provider} model from configured auth storage`);
   const resolvedModel = selectModel();
   if (!resolvedModel) {
-    throw new Error(
-      rawModel
-        ? `Model not found for ${params.provider}: ${rawModel}`
-        : `No ${params.provider} ${params.api} model available in registry.`,
-    );
+    const message = rawModel
+      ? `Model not found for ${params.provider}: ${rawModel}`
+      : `No ${params.provider} ${params.api} model available in registry.`;
+    if (rawModel) {
+      throw new Error(message);
+    }
+    throw new LiveCachePrerequisiteSkip(params.provider, message);
   }
 
-  const apiKey = requireApiKey(
-    await getApiKeyForModel({
-      model: resolvedModel,
-      cfg,
-      agentDir,
-    }),
-    resolvedModel.provider,
-  );
+  let apiKey: string;
+  try {
+    apiKey = requireApiKey(
+      await getApiKeyForModel({
+        model: resolvedModel,
+        cfg,
+        agentDir,
+      }),
+      resolvedModel.provider,
+    );
+  } catch (error) {
+    const skip = toLiveCachePrerequisiteSkip(params.provider, error);
+    if (skip) {
+      throw skip;
+    }
+    throw error;
+  }
   logLiveCache(
     `resolved ${params.provider} model ${resolvedModel.id} from configured auth storage`,
   );

src/agents/model-auth.profiles.test.ts

@@ -648,9 +648,11 @@ describe("getApiKeyForModel", () => {
         },
       },
       async () => {
-        await expect(resolveApiKeyForProvider({ provider: "openai" })).rejects.toThrow(
-          'No API key found for provider "openai".',
-        );
+        await expect(resolveApiKeyForProvider({ provider: "openai" })).rejects.toMatchObject({
+          code: "missing-provider-auth",
+          message: expect.stringContaining('No API key found for provider "openai".'),
+          provider: "openai",
+        });
       },
     );
 

src/agents/model-auth.ts

@@ -1262,7 +1262,7 @@ export async function resolveApiKeyForProvider(params: {
       },
     });
     if (pluginMissingAuthMessage) {
-      throw new Error(pluginMissingAuthMessage);
+      throw new ProviderAuthError("missing-provider-auth", provider, pluginMissingAuthMessage);
     }
   }