From 932034f1fc8a3fcccff54ba30cc2f76dd63369f7 Mon Sep 17 00:00:00 2001 From: Vincent Koc Date: Wed, 3 Jun 2026 11:50:46 -0700 Subject: [PATCH] test(live): keep cache prereq skips provider-aware --- .../live-cache-regression-runner.test.ts | 84 ++++++++++++ src/agents/live-cache-regression-runner.ts | 128 ++++++++++++------ src/agents/live-cache-test-support.ts | 80 ++++++++--- src/agents/model-auth.profiles.test.ts | 8 +- src/agents/model-auth.ts | 2 +- 5 files changed, 240 insertions(+), 62 deletions(-) diff --git a/src/agents/live-cache-regression-runner.test.ts b/src/agents/live-cache-regression-runner.test.ts index c5d05d7d9086..5ebd4b6617a0 100644 --- a/src/agents/live-cache-regression-runner.test.ts +++ b/src/agents/live-cache-regression-runner.test.ts @@ -1,5 +1,10 @@ import { describe, expect, it } from "vitest"; import { testing } from "./live-cache-regression-runner.js"; +import { ProviderAuthError } from "./model-auth-runtime-shared.js"; +import { + LiveCachePrerequisiteSkip, + toLiveCachePrerequisiteSkip, +} from "./live-cache-test-support.js"; describe("live cache regression runner", () => { it("keeps OpenAI image cache floors observable without blocking release validation", () => { @@ -84,6 +89,85 @@ describe("live cache regression runner", () => { ).toBe(false); }); + it("keeps missing optional live-cache prerequisites non-blocking", async () => { + const regressions: string[] = []; + const warnings: string[] = []; + const summary: Record> = { + anthropic: {}, + openai: {}, + }; + + const resolved = await testing.resolveLiveCacheProviderPool({ + config: { + provider: "openai", + api: "openai-responses", + envVar: "OPENCLAW_LIVE_OPENAI_CACHE_MODEL", + preferredModelIds: ["gpt-5.5"], + }, + resolver: async () => { + throw new LiveCachePrerequisiteSkip( + "openai", + "No openai openai-responses model available in registry.", + ); + }, + regressions, + summary, + warnings, + }); + + expect(resolved).toBeUndefined(); + expect(regressions).toStrictEqual([]); + expect(warnings).toEqual([ + "openai skipped: No openai openai-responses model available in registry.", + ]); + expect(summary.openai).toEqual({ skipped: true }); + }); + + it("keeps missing Anthropic live-cache prerequisites blocking", async () => { + const regressions: string[] = []; + const warnings: string[] = []; + const summary: Record> = { + anthropic: {}, + openai: {}, + }; + + const resolved = await testing.resolveLiveCacheProviderPool({ + config: { + provider: "anthropic", + api: "anthropic-messages", + envVar: "OPENCLAW_LIVE_ANTHROPIC_CACHE_MODEL", + preferredModelIds: ["claude-sonnet-4-6"], + }, + resolver: async () => { + throw new LiveCachePrerequisiteSkip( + "anthropic", + "No anthropic anthropic-messages model available in registry.", + ); + }, + regressions, + summary, + warnings, + }); + + expect(resolved).toBeUndefined(); + expect(regressions).toEqual([ + "anthropic skipped: No anthropic anthropic-messages model available in registry.", + ]); + expect(warnings).toStrictEqual([]); + expect(summary.anthropic).toEqual({ skipped: true }); + }); + + it("classifies missing provider auth as a live-cache prerequisite", () => { + const skip = toLiveCachePrerequisiteSkip( + "openai", + new ProviderAuthError("missing-provider-auth", "openai", "No API key found."), + ); + + expect(skip).toBeInstanceOf(LiveCachePrerequisiteSkip); + expect(skip?.provider).toBe("openai"); + expect(skip?.message).toBe("No API key found."); + }); + it("retries a cache probe twice when provider text misses the sentinel", () => { expect( testing.shouldRetryCacheProbeText({ diff --git a/src/agents/live-cache-regression-runner.ts b/src/agents/live-cache-regression-runner.ts index de3a83565a5c..71673577acd3 100644 --- a/src/agents/live-cache-regression-runner.ts +++ b/src/agents/live-cache-regression-runner.ts @@ -13,7 +13,9 @@ import { completeSimpleWithLiveTimeout, computeCacheHitRate, extractAssistantText, + isLiveCachePrerequisiteSkip, type LiveResolvedModel, + type LiveResolvedModelPool, logLiveCache, resolveLiveDirectModelPool, withLiveDirectModelApiKey, @@ -35,6 +37,7 @@ const LIVE_TEST_PNG_URL = new URL( import.meta.url, ); +type LiveCacheProviderConfig = Parameters[0]; type ProviderKey = keyof typeof LIVE_CACHE_REGRESSION_BASELINE; type CacheLane = "image" | "mcp" | "stable" | "tool"; type CacheUsage = { @@ -65,6 +68,8 @@ type LiveCacheRegressionResult = { summary: Record>; warnings: string[]; }; +type LiveCacheRegressionSummary = LiveCacheRegressionResult["summary"]; +type LiveCacheProviderResolver = (params: LiveCacheProviderConfig) => Promise; class CacheProbeTextMismatchError extends Error { constructor( @@ -95,6 +100,31 @@ function makeUserTurn(content: Extract["content"]): M }; } +async function resolveLiveCacheProviderPool(params: { + config: LiveCacheProviderConfig; + regressions: string[]; + resolver?: LiveCacheProviderResolver; + summary: LiveCacheRegressionSummary; + warnings: string[]; +}): Promise { + try { + return await (params.resolver ?? resolveLiveDirectModelPool)(params.config); + } catch (error) { + if (!isLiveCachePrerequisiteSkip(error)) { + throw error; + } + const warning = `${error.provider} skipped: ${error.message}`; + if (error.provider === "openai") { + params.warnings.push(warning); + } else { + params.regressions.push(warning); + } + params.summary[error.provider].skipped = true; + logLiveCache(warning); + return undefined; + } +} + function makeImageUserTurn(text: string, pngBase64: string): Message { return makeUserTurn([ { type: "text", text }, @@ -699,6 +729,7 @@ async function runAnthropicDisabledCacheLane(params: { export const testing = { assertAgainstBaseline, evaluateAgainstBaseline, + resolveLiveCacheProviderPool, resolveCacheProbeMaxTokens, isAnthropicToolProbeDrift, shouldAcceptEmptyCacheProbe, @@ -709,49 +740,66 @@ export const testing = { export async function runLiveCacheRegression(): Promise { const pngBase64 = (await fs.readFile(LIVE_TEST_PNG_URL)).toString("base64"); const runToken = randomUUID().slice(0, 13); - const openai = await resolveLiveDirectModelPool({ - provider: "openai", - api: "openai-responses", - envVar: "OPENCLAW_LIVE_OPENAI_CACHE_MODEL", - preferredModelIds: ["gpt-4.1", "gpt-5.2", "gpt-5.4-mini", "gpt-5.4", "gpt-5.5"], - }); - const anthropic = await resolveLiveDirectModelPool({ - provider: "anthropic", - api: "anthropic-messages", - envVar: "OPENCLAW_LIVE_ANTHROPIC_CACHE_MODEL", - preferredModelIds: ["claude-sonnet-4-6", "claude-sonnet-4-5", "claude-haiku-3-5"], - }); - const regressions: string[] = []; const warnings: string[] = []; const summary: Record> = { anthropic: {}, openai: {}, }; + const openai = await resolveLiveCacheProviderPool({ + config: { + provider: "openai", + api: "openai-responses", + envVar: "OPENCLAW_LIVE_OPENAI_CACHE_MODEL", + preferredModelIds: ["gpt-4.1", "gpt-5.2", "gpt-5.4-mini", "gpt-5.4", "gpt-5.5"], + }, + regressions, + summary, + warnings, + }); + const anthropic = await resolveLiveCacheProviderPool({ + config: { + provider: "anthropic", + api: "anthropic-messages", + envVar: "OPENCLAW_LIVE_ANTHROPIC_CACHE_MODEL", + preferredModelIds: ["claude-sonnet-4-6", "claude-sonnet-4-5", "claude-haiku-3-5"], + }, + regressions, + summary, + warnings, + }); for (const lane of ["stable", "tool", "image", "mcp"] as const) { - const openaiAttempt = await runRepeatedLaneWithBaselineRetry({ - lane, - providerTag: "openai", - fixture: openai.fixture, - runToken, - pngBase64, - }); - const openaiResult = openaiAttempt.result; - logLiveCache( - `openai ${lane} warmup ${formatUsage(openaiResult.warmup?.usage ?? {})} rate=${openaiResult.warmup?.hitRate.toFixed(3) ?? "0.000"}`, - ); - logLiveCache( - `openai ${lane} best ${formatUsage(openaiResult.best?.usage ?? {})} rate=${openaiResult.best?.hitRate.toFixed(3) ?? "0.000"}`, - ); - summary.openai[lane] = { - best: openaiResult.best?.usage, - hitRate: openaiResult.best?.hitRate, - attempts: openaiAttempt.attempts, - warmup: openaiResult.warmup?.usage, - }; - appendBaselineFindings({ regressions, warnings }, openaiAttempt.findings); + if (openai) { + const openaiAttempt = await runRepeatedLaneWithBaselineRetry({ + lane, + providerTag: "openai", + fixture: openai.fixture, + runToken, + pngBase64, + }); + const openaiResult = openaiAttempt.result; + logLiveCache( + `openai ${lane} warmup ${formatUsage(openaiResult.warmup?.usage ?? {})} rate=${openaiResult.warmup?.hitRate.toFixed(3) ?? "0.000"}`, + ); + logLiveCache( + `openai ${lane} best ${formatUsage(openaiResult.best?.usage ?? {})} rate=${openaiResult.best?.hitRate.toFixed(3) ?? "0.000"}`, + ); + summary.openai[lane] = { + best: openaiResult.best?.usage, + hitRate: openaiResult.best?.hitRate, + attempts: openaiAttempt.attempts, + warmup: openaiResult.warmup?.usage, + }; + appendBaselineFindings({ regressions, warnings }, openaiAttempt.findings); + } else { + summary.openai[lane] = { skipped: true }; + } + if (!anthropic) { + summary.anthropic[lane] = { skipped: true }; + continue; + } const { attempt: anthropicAttempt } = await runAnthropicCacheLane({ apiKeys: anthropic.apiKeys, lane, @@ -780,11 +828,13 @@ export async function runLiveCacheRegression(): Promise 0) { const selectedModel = selectModel(); if (!selectedModel || selectedModel.api !== params.api) { - throw new Error( - requestedModelId - ? `Model not found for ${params.provider}: ${requestedModelId}` - : `No built-in ${params.provider} ${params.api} model available.`, - ); + const message = requestedModelId + ? `Model not found for ${params.provider}: ${requestedModelId}` + : `No built-in ${params.provider} ${params.api} model available.`; + if (requestedModelId) { + throw new Error(message); + } + throw new LiveCachePrerequisiteSkip(params.provider, message); } logLiveCache(`resolved ${params.provider} model ${selectedModel.id} from live env key`); return { @@ -216,21 +247,32 @@ export async function resolveLiveDirectModelPool(params: { logLiveCache(`resolving ${params.provider} model from configured auth storage`); const resolvedModel = selectModel(); if (!resolvedModel) { - throw new Error( - rawModel - ? `Model not found for ${params.provider}: ${rawModel}` - : `No ${params.provider} ${params.api} model available in registry.`, - ); + const message = rawModel + ? `Model not found for ${params.provider}: ${rawModel}` + : `No ${params.provider} ${params.api} model available in registry.`; + if (rawModel) { + throw new Error(message); + } + throw new LiveCachePrerequisiteSkip(params.provider, message); } - const apiKey = requireApiKey( - await getApiKeyForModel({ - model: resolvedModel, - cfg, - agentDir, - }), - resolvedModel.provider, - ); + let apiKey: string; + try { + apiKey = requireApiKey( + await getApiKeyForModel({ + model: resolvedModel, + cfg, + agentDir, + }), + resolvedModel.provider, + ); + } catch (error) { + const skip = toLiveCachePrerequisiteSkip(params.provider, error); + if (skip) { + throw skip; + } + throw error; + } logLiveCache( `resolved ${params.provider} model ${resolvedModel.id} from configured auth storage`, ); diff --git a/src/agents/model-auth.profiles.test.ts b/src/agents/model-auth.profiles.test.ts index 7a9b651ee5c7..509c6a0a0d74 100644 --- a/src/agents/model-auth.profiles.test.ts +++ b/src/agents/model-auth.profiles.test.ts @@ -648,9 +648,11 @@ describe("getApiKeyForModel", () => { }, }, async () => { - await expect(resolveApiKeyForProvider({ provider: "openai" })).rejects.toThrow( - 'No API key found for provider "openai".', - ); + await expect(resolveApiKeyForProvider({ provider: "openai" })).rejects.toMatchObject({ + code: "missing-provider-auth", + message: expect.stringContaining('No API key found for provider "openai".'), + provider: "openai", + }); }, ); diff --git a/src/agents/model-auth.ts b/src/agents/model-auth.ts index d990b6a2415e..848f09efefd7 100644 --- a/src/agents/model-auth.ts +++ b/src/agents/model-auth.ts @@ -1262,7 +1262,7 @@ export async function resolveApiKeyForProvider(params: { }, }); if (pluginMissingAuthMessage) { - throw new Error(pluginMissingAuthMessage); + throw new ProviderAuthError("missing-provider-auth", provider, pluginMissingAuthMessage); } }