Compare commits

...

1 Commits

Author SHA1 Message Date
Bek
ed45202b70 fix: route media understanding through codex auth 2026-05-27 02:18:02 -04:00
7 changed files with 595 additions and 66 deletions

View File

@@ -2712,6 +2712,20 @@ describe("runWithImageModelFallback", () => {
modelOverride: "gpt-5.4-mini",
expected: [["openai-codex", "gpt-5.4-mini"]],
},
{
name: "bare override inherits bounded Codex image provider",
cfg: makeCfg({
agents: {
defaults: {
imageModel: {
primary: "codex/gpt-5.5",
},
},
},
}),
modelOverride: "gpt-5.5",
expected: [["codex", "gpt-5.5"]],
},
{
name: "qualified override keeps provider",
cfg: makeCfg({

View File

@@ -70,6 +70,50 @@ const imageProviderHarness = vi.hoisted(() => {
};
});
const modelAuthEnvMock = vi.hoisted(() => ({
envVarByProvider: {
anthropic: ["ANTHROPIC_API_KEY", "ANTHROPIC_OAUTH_TOKEN"],
minimax: ["MINIMAX_API_KEY", "MINIMAX_OAUTH_TOKEN"],
"minimax-portal": ["MINIMAX_OAUTH_TOKEN"],
moonshot: ["MOONSHOT_API_KEY"],
openai: ["OPENAI_API_KEY"],
opencode: ["OPENCODE_API_KEY", "OPENCODE_ZEN_API_KEY"],
"opencode-go": ["OPENCODE_API_KEY", "OPENCODE_ZEN_API_KEY"],
openrouter: ["OPENROUTER_API_KEY"],
zai: ["ZAI_API_KEY", "Z_AI_API_KEY"],
} as Record<string, string[]>,
}));
function createDefaultImageModelMap(): Map<string, string> {
return new Map<string, string>([
["anthropic", "claude-opus-4-6"],
["minimax", "MiniMax-VL-01"],
["minimax-cn", "MiniMax-VL-01"],
["minimax-portal", "MiniMax-VL-01"],
["minimax-portal-cn", "MiniMax-VL-01"],
["codex", "gpt-5.5"],
["openai", "gpt-5.4-mini"],
["opencode", "gpt-5-nano"],
["opencode-go", "kimi-k2.6"],
["zai", "glm-4.6v"],
]);
}
function makeOpenAiCodexAuthStore() {
return {
version: 1,
profiles: {
"openai-codex:test": {
type: "oauth",
provider: "openai-codex",
access: "codex-oauth-token",
refresh: "codex-refresh-token",
expires: Date.now() + 60_000,
},
},
} as const;
}
vi.mock("../bash-tools.js", async () => {
const actual = await vi.importActual<typeof import("../bash-tools.js")>("../bash-tools.js");
return {
@@ -125,24 +169,38 @@ vi.mock("../auth-profiles.js", () => ({
}));
vi.mock("../model-auth.js", () => ({
hasUsableCustomProviderApiKey: (cfg?: OpenClawConfig, provider?: string) => {
createRuntimeProviderAuthLookup: () => ({
envApiKey: {
aliasMap: {},
candidateMap: {},
authEvidenceMap: {},
},
syntheticAuthProviderRefs: [],
}),
hasRuntimeAvailableProviderAuth: ({
provider,
cfg,
allowPluginSyntheticAuth,
}: {
provider: string;
cfg?: OpenClawConfig;
allowPluginSyntheticAuth?: boolean;
}) => {
const providerConfig = cfg?.models?.providers?.[provider ?? ""];
const apiKey = providerConfig?.apiKey;
return typeof apiKey === "string" && apiKey.trim().length > 0;
if (typeof apiKey === "string" && apiKey.trim().length > 0) {
return true;
}
if (provider === "codex" && allowPluginSyntheticAuth !== false) {
return true;
}
return (modelAuthEnvMock.envVarByProvider[provider] ?? []).some((key) => {
const value = process.env[key];
return typeof value === "string" && value.length > 0;
});
},
resolveEnvApiKey: (provider: string) => {
const envVarByProvider: Record<string, string[]> = {
anthropic: ["ANTHROPIC_API_KEY", "ANTHROPIC_OAUTH_TOKEN"],
minimax: ["MINIMAX_API_KEY", "MINIMAX_OAUTH_TOKEN"],
"minimax-portal": ["MINIMAX_OAUTH_TOKEN"],
moonshot: ["MOONSHOT_API_KEY"],
openai: ["OPENAI_API_KEY"],
opencode: ["OPENCODE_API_KEY", "OPENCODE_ZEN_API_KEY"],
"opencode-go": ["OPENCODE_API_KEY", "OPENCODE_ZEN_API_KEY"],
openrouter: ["OPENROUTER_API_KEY"],
zai: ["ZAI_API_KEY", "Z_AI_API_KEY"],
};
const envVar = (envVarByProvider[provider] ?? []).find((key) => {
const envVar = (modelAuthEnvMock.envVarByProvider[provider] ?? []).find((key) => {
const value = process.env[key];
return typeof value === "string" && value.length > 0;
});
@@ -190,17 +248,7 @@ async function writeAuthProfiles(agentDir: string, profiles: unknown) {
}
async function createOpenClawCodingToolsWithFreshModules(options?: CreateOpenClawCodingToolsArgs) {
const defaultImageModels = new Map<string, string>([
["anthropic", "claude-opus-4-6"],
["minimax", "MiniMax-VL-01"],
["minimax-cn", "MiniMax-VL-01"],
["minimax-portal", "MiniMax-VL-01"],
["minimax-portal-cn", "MiniMax-VL-01"],
["openai", "gpt-5.4-mini"],
["opencode", "gpt-5-nano"],
["opencode-go", "kimi-k2.6"],
["zai", "glm-4.6v"],
]);
const defaultImageModels = createDefaultImageModelMap();
testing.setProviderDepsForTest({
buildProviderRegistry: (overrides?: Record<string, MediaUnderstandingProvider>) =>
imageProviderHarness.buildProviderRegistry(overrides),
@@ -539,17 +587,7 @@ const moonshotProvider = {
function installImageUnderstandingProviderStubs(...providers: MediaUnderstandingProvider[]) {
imageProviderHarness.setProviders(providers);
const defaultImageModels = new Map<string, string>([
["anthropic", "claude-opus-4-6"],
["minimax", "MiniMax-VL-01"],
["minimax-cn", "MiniMax-VL-01"],
["minimax-portal", "MiniMax-VL-01"],
["minimax-portal-cn", "MiniMax-VL-01"],
["openai", "gpt-5.4-mini"],
["opencode", "gpt-5-nano"],
["opencode-go", "kimi-k2.6"],
["zai", "glm-4.6v"],
]);
const defaultImageModels = createDefaultImageModelMap();
testing.setProviderDepsForTest({
buildProviderRegistry: (overrides?: Record<string, MediaUnderstandingProvider>) =>
imageProviderHarness.buildProviderRegistry(overrides),
@@ -717,6 +755,97 @@ describe("image tool implicit imageModel config", () => {
});
});
it("selects bounded Codex image understanding for OpenAI GPT models with Codex OAuth only", async () => {
await withTempAgentDir(async (agentDir) => {
const defaultImageModels = new Map<string, string>([
["codex", "gpt-5.5"],
["openai", "gpt-5.4-mini"],
]);
testing.setProviderDepsForTest({
buildProviderRegistry: (overrides?: Record<string, MediaUnderstandingProvider>) =>
imageProviderHarness.buildProviderRegistry(overrides),
getMediaUnderstandingProvider: (
id: string,
registry: Map<string, MediaUnderstandingProvider>,
) => imageProviderHarness.getMediaUnderstandingProvider(id, registry),
describeImageWithModel: describeGenericImageWithModel,
describeImagesWithModel: describeGenericImagesWithModel,
resolveAutoMediaKeyProviders: ({ capability }) =>
capability === "image" ? ["openai"] : [],
resolveDefaultMediaModel: ({ providerId, capability }) =>
capability === "image" ? defaultImageModels.get(providerId.toLowerCase()) : undefined,
});
const cfg: OpenClawConfig = {
agents: { defaults: { model: { primary: "openai/gpt-5.5" } } },
};
const authStore = makeOpenAiCodexAuthStore();
expect(resolveImageModelConfigForTool({ cfg, agentDir, authStore })).toEqual({
primary: "codex/gpt-5.5",
});
});
});
it("keeps direct OpenAI image understanding when OpenAI API auth is available", async () => {
await withTempAgentDir(async (agentDir) => {
vi.stubEnv("OPENAI_API_KEY", "openai-test");
testing.setProviderDepsForTest({
buildProviderRegistry: (overrides?: Record<string, MediaUnderstandingProvider>) =>
imageProviderHarness.buildProviderRegistry(overrides),
getMediaUnderstandingProvider: (
id: string,
registry: Map<string, MediaUnderstandingProvider>,
) => imageProviderHarness.getMediaUnderstandingProvider(id, registry),
describeImageWithModel: describeGenericImageWithModel,
describeImagesWithModel: describeGenericImagesWithModel,
resolveAutoMediaKeyProviders: ({ capability }) =>
capability === "image" ? ["openai"] : [],
resolveDefaultMediaModel: ({ providerId, capability }) =>
capability === "image" && providerId === "openai" ? "gpt-5.4-mini" : undefined,
});
const cfg: OpenClawConfig = {
agents: { defaults: { model: { primary: "openai/gpt-5.5" } } },
};
const authStore = makeOpenAiCodexAuthStore();
expect(resolveImageModelConfigForTool({ cfg, agentDir, authStore })).toEqual({
primary: "openai/gpt-5.4-mini",
});
});
});
it("does not preempt other authenticated image providers with Codex", async () => {
await withTempAgentDir(async (agentDir) => {
vi.stubEnv("ANTHROPIC_API_KEY", "anthropic-test");
const defaultImageModels = new Map<string, string>([
["anthropic", "claude-opus-4-6"],
["codex", "gpt-5.5"],
["openai", "gpt-5.4-mini"],
]);
testing.setProviderDepsForTest({
buildProviderRegistry: (overrides?: Record<string, MediaUnderstandingProvider>) =>
imageProviderHarness.buildProviderRegistry(overrides),
getMediaUnderstandingProvider: (
id: string,
registry: Map<string, MediaUnderstandingProvider>,
) => imageProviderHarness.getMediaUnderstandingProvider(id, registry),
describeImageWithModel: describeGenericImageWithModel,
describeImagesWithModel: describeGenericImagesWithModel,
resolveAutoMediaKeyProviders: ({ capability }) =>
capability === "image" ? ["openai", "anthropic"] : [],
resolveDefaultMediaModel: ({ providerId, capability }) =>
capability === "image" ? defaultImageModels.get(providerId.toLowerCase()) : undefined,
});
const cfg: OpenClawConfig = {
agents: { defaults: { model: { primary: "openai/gpt-5.5" } } },
};
expect(resolveImageModelConfigForTool({ cfg, agentDir })).toEqual({
primary: "anthropic/claude-opus-4-6",
});
});
});
it("defers implicit image model discovery during hot-path tool registration", async () => {
await withTempAgentDir(async (agentDir) => {
const resolveDefaultMediaModelSpy = vi.fn(() => "gpt-5.4-mini");
@@ -782,6 +911,56 @@ describe("image tool implicit imageModel config", () => {
});
});
it("lets bare per-call model overrides inherit the bounded Codex image route", async () => {
await withTempAgentDir(async (agentDir) => {
const fetch = vi.fn().mockResolvedValue({
headers: new Headers({ "content-type": "application/json" }),
json: async () => ({ content: "ok codex" }),
});
global.fetch = withFetchPreconnect(fetch);
testing.setProviderDepsForTest({
buildProviderRegistry: (overrides?: Record<string, MediaUnderstandingProvider>) =>
imageProviderHarness.buildProviderRegistry(overrides),
getMediaUnderstandingProvider: (
id: string,
registry: Map<string, MediaUnderstandingProvider>,
) => imageProviderHarness.getMediaUnderstandingProvider(id, registry),
describeImageWithModel: describeGenericImageWithModel,
describeImagesWithModel: describeGenericImagesWithModel,
resolveAutoMediaKeyProviders: ({ capability }) =>
capability === "image" ? ["openai"] : [],
resolveDefaultMediaModel: ({ providerId, capability }) => {
if (capability !== "image") {
return undefined;
}
return providerId === "codex" ? "gpt-5.5" : "gpt-5.4-mini";
},
});
const cfg: OpenClawConfig = {
agents: { defaults: { model: { primary: "openai/gpt-5.5" } } },
};
const authStore = makeOpenAiCodexAuthStore();
const tool = createRequiredImageTool({
config: cfg,
agentDir,
authProfileStore: authStore,
deferAutoModelResolution: true,
});
const result = await tool.execute("t1", {
prompt: "Describe this image.",
image: `data:image/png;base64,${ONE_PIXEL_PNG_B64}`,
model: "gpt-5.4",
});
expectToolText(result, "ok codex");
expect(JSON.parse(String(fetch.mock.calls[0]?.[1]?.body))).toMatchObject({
provider: "codex",
model: "gpt-5.4",
});
});
});
it("pairs minimax primary with MiniMax-VL-01 (and fallbacks) when auth exists", async () => {
await withTempAgentDir(async (agentDir) => {
vi.stubEnv("MINIMAX_API_KEY", "minimax-test");
@@ -1562,7 +1741,11 @@ describe("image tool implicit imageModel config", () => {
type: "array",
items: { type: "string" },
},
model: { type: "string" },
model: {
description:
"Optional provider/model override. Usually omit so OpenClaw uses agents.defaults.imageModel. Use codex/gpt-* for bounded Codex image understanding.",
type: "string",
},
maxBytesMb: { type: "number" },
maxImages: { type: "number" },
},
@@ -2661,18 +2844,44 @@ describe("image compression policy", () => {
});
});
it("resolves providerless overrides before reading compression metadata", async () => {
it("resolves providerless overrides against the configured media route for compression", async () => {
const cfg = {
...cfgWithImageModelMetadata,
models: {
providers: {
...cfgWithImageModelMetadata.models.providers,
codex: {
baseUrl: "https://codex.example.invalid",
models: [
{
id: "gpt-5.5",
name: "Codex GPT-5.5",
reasoning: true,
input: ["text", "image"],
contextWindow: 200_000,
maxTokens: 128_000,
cost: { input: 0, output: 0, cacheRead: 0, cacheWrite: 0 },
mediaInput: {
image: { maxSidePx: 4096, preferredSidePx: 2048, tokenMode: "provider" },
},
},
],
},
},
},
} satisfies OpenClawConfig;
await expect(
testing.resolveImageCompressionPolicy({
cfg: cfgWithImageModelMetadata,
cfg,
imageModelConfig: {
primary: "anthropic/claude-opus-4-6",
primary: "codex/gpt-5.5",
},
modelOverride: "gpt-5.5",
imageCount: 1,
}),
).resolves.toMatchObject({
models: [{ maxSidePx: 6000, preferredSidePx: 2048, tokenMode: "detail" }],
models: [{ maxSidePx: 4096, preferredSidePx: 2048, tokenMode: "provider" }],
});
});
});

View File

@@ -61,6 +61,7 @@ import {
import {
buildToolModelConfigFromCandidates,
hasToolModelConfig,
resolveCodexMediaCandidateForOpenAiCodexRoute,
resolveDefaultModelRef,
} from "./model-config.helpers.js";
import {
@@ -141,6 +142,15 @@ function isCanonicalCandidateShadowedByExecutionAlias(
);
}
function isProviderQualifiedModelRef(ref: string | undefined | null): boolean {
const trimmed = ref?.trim();
if (!trimmed) {
return false;
}
const slash = trimmed.indexOf("/");
return slash > 0 && slash < trimmed.length - 1;
}
export const testing = {
decodeDataUrl,
coerceImageAssistantText,
@@ -274,6 +284,16 @@ export function resolveImageModelConfigForTool(params: {
primaryAliasCandidates.length === 0
? autoCandidates
: autoCandidates.filter((candidate) => !primaryAliasCandidates.includes(candidate));
const codexContextCandidates = [
resolveCodexMediaCandidateForOpenAiCodexRoute({
cfg: params.cfg,
primary,
agentDir: params.agentDir,
workspaceDir: params.workspaceDir,
authStore: params.authStore,
resolveDefaultMediaModel: imageToolProviderDeps.resolveDefaultMediaModel,
}),
];
return buildToolModelConfigFromCandidates({
explicit,
@@ -281,7 +301,12 @@ export function resolveImageModelConfigForTool(params: {
workspaceDir: params.workspaceDir,
agentDir: params.agentDir,
authStore: params.authStore,
candidates: [...primaryAliasCandidates, ...primaryCandidates, ...remainingAutoCandidates],
candidates: [
...primaryAliasCandidates,
...primaryCandidates,
...codexContextCandidates,
...remainingAutoCandidates,
],
});
}
@@ -299,6 +324,36 @@ function resolveImageModelConfigForOverride(params: {
});
}
function resolveImageModelConfigForExecution(params: {
cfg?: OpenClawConfig;
agentDir: string;
workspaceDir?: string;
authStore?: AuthProfileStore;
resolvedImageModelConfig?: ImageModelConfig | null;
modelOverride?: string;
}): ImageModelConfig | null {
const modelOverride = params.modelOverride?.trim();
if (modelOverride && isProviderQualifiedModelRef(modelOverride)) {
return resolveImageModelConfigForOverride({
cfg: params.cfg,
modelOverride,
});
}
return (
params.resolvedImageModelConfig ??
resolveImageModelConfigForTool({
cfg: params.cfg,
agentDir: params.agentDir,
workspaceDir: params.workspaceDir,
authStore: params.authStore,
}) ??
resolveImageModelConfigForOverride({
cfg: params.cfg,
modelOverride,
})
);
}
function pickMaxBytes(cfg?: OpenClawConfig, maxBytesMb?: number): number | undefined {
if (typeof maxBytesMb === "number" && Number.isFinite(maxBytesMb) && maxBytesMb > 0) {
return Math.floor(maxBytesMb * 1024 * 1024);
@@ -315,10 +370,12 @@ function resolveCompressionModelCandidates(params: {
imageModelConfig?: ImageModelConfig | null;
modelOverride?: string;
}): Array<{ provider: string; model: string }> {
const overrideConfig = resolveImageModelConfigForOverride({
cfg: params.cfg,
modelOverride: params.modelOverride,
});
const overrideConfig = isProviderQualifiedModelRef(params.modelOverride)
? resolveImageModelConfigForOverride({
cfg: params.cfg,
modelOverride: params.modelOverride,
})
: null;
const configuredImageModelConfig = params.imageModelConfig
? resolveConfiguredImageModelRefs({
cfg: params.cfg,
@@ -332,6 +389,7 @@ function resolveCompressionModelCandidates(params: {
return resolveImageFallbackCandidates({
cfg: effectiveCfg,
defaultProvider: resolveImageFallbackDefaultProvider(effectiveCfg),
modelOverride: params.modelOverride,
});
}
@@ -722,10 +780,10 @@ export function createImageTool(options?: {
// If model has native vision, images in the prompt are auto-injected
// so this tool is only needed when image wasn't provided in the prompt
const description = options?.modelHasVision
? "Analyze images with vision model. Use image for one path/URL, images for max 20. Only use this tool when images were NOT already provided; prompt images already visible."
? "Analyze images with vision model. Use image for one path/URL, images for max 20. Only use this tool when images were NOT already provided; prompt images already visible. Omit model unless intentionally overriding the configured image route."
: explicitImageModelConfig
? "Analyze images with configured image model. Use image for one path/URL, images for max 20. Prompt says what to inspect."
: "Analyze images with available vision model. Use image for one path/URL, images for max 20. Prompt says what to inspect.";
? "Analyze images with configured image model. Use image for one path/URL, images for max 20. Prompt says what to inspect. Omit model unless intentionally overriding the configured image route."
: "Analyze images with available vision model. Use image for one path/URL, images for max 20. Prompt says what to inspect. Omit model unless intentionally overriding the configured image route.";
return {
label: "Image",
@@ -739,7 +797,12 @@ export function createImageTool(options?: {
description: "Image paths/URLs; maxImages default 20.",
}),
),
model: Type.Optional(Type.String()),
model: Type.Optional(
Type.String({
description:
"Optional provider/model override. Usually omit so OpenClaw uses agents.defaults.imageModel. Use codex/gpt-* for bounded Codex image understanding.",
}),
),
maxBytesMb: Type.Optional(Type.Number()),
maxImages: Type.Optional(Type.Number()),
}),
@@ -796,18 +859,14 @@ export function createImageTool(options?: {
);
const maxBytesMb = typeof record.maxBytesMb === "number" ? record.maxBytesMb : undefined;
const maxBytes = pickMaxBytes(options?.config, maxBytesMb);
const imageModelConfig =
resolvedImageModelConfig ??
resolveImageModelConfigForOverride({
cfg: options?.config,
modelOverride,
}) ??
resolveImageModelConfigForTool({
cfg: options?.config,
agentDir,
workspaceDir: options?.workspaceDir,
authStore: options?.authProfileStore,
});
const imageModelConfig = resolveImageModelConfigForExecution({
cfg: options?.config,
agentDir,
workspaceDir: options?.workspaceDir,
authStore: options?.authProfileStore,
resolvedImageModelConfig,
modelOverride,
});
if (!imageModelConfig) {
throw new Error(
"No image model is configured. Set agents.defaults.imageModel or configure an image-capable provider.",

View File

@@ -1,10 +1,40 @@
import { afterEach, describe, expect, it, vi } from "vitest";
import type { OpenClawConfig } from "../../config/config.js";
import type { RuntimeProviderAuthLookup } from "../model-auth.js";
import { hasProviderAuthForTool } from "./model-config.helpers.js";
function createRuntimeLookup(
syntheticAuthProviderRefs: readonly string[] = [],
): RuntimeProviderAuthLookup {
return {
envApiKey: {
aliasMap: {},
candidateMap: {},
authEvidenceMap: {},
},
syntheticAuthProviderRefs,
};
}
const modelAuthMock = vi.hoisted(() => ({
createRuntimeProviderAuthLookup: vi.fn(() => createRuntimeLookup()),
hasRuntimeAvailableProviderAuth: vi.fn(() => false),
resolveEnvApiKey: vi.fn(() => null),
}));
vi.mock("../model-auth.js", () => ({
createRuntimeProviderAuthLookup: modelAuthMock.createRuntimeProviderAuthLookup,
hasRuntimeAvailableProviderAuth: modelAuthMock.hasRuntimeAvailableProviderAuth,
resolveEnvApiKey: modelAuthMock.resolveEnvApiKey,
}));
describe("hasProviderAuthForTool", () => {
afterEach(() => {
vi.unstubAllEnvs();
vi.clearAllMocks();
modelAuthMock.createRuntimeProviderAuthLookup.mockReturnValue(createRuntimeLookup());
modelAuthMock.hasRuntimeAvailableProviderAuth.mockReturnValue(false);
modelAuthMock.resolveEnvApiKey.mockReturnValue(null);
});
it("accepts config-backed custom provider auth", () => {
@@ -19,6 +49,7 @@ describe("hasProviderAuthForTool", () => {
},
},
} as OpenClawConfig;
modelAuthMock.hasRuntimeAvailableProviderAuth.mockReturnValue(true);
expect(hasProviderAuthForTool({ provider: "hatchery", cfg })).toBe(true);
});
@@ -44,4 +75,29 @@ describe("hasProviderAuthForTool", () => {
it("rejects providers without config, env, or profile auth", () => {
expect(hasProviderAuthForTool({ provider: "unconfigured-provider" })).toBe(false);
});
it("accepts scoped runtime provider auth", () => {
const cfg = {} as OpenClawConfig;
const runtimeLookup = createRuntimeLookup(["codex"]);
modelAuthMock.createRuntimeProviderAuthLookup.mockReturnValue(runtimeLookup);
modelAuthMock.hasRuntimeAvailableProviderAuth.mockReturnValue(true);
expect(
hasProviderAuthForTool({
provider: "codex",
cfg,
workspaceDir: "/tmp/openclaw-workspace",
}),
).toBe(true);
expect(modelAuthMock.createRuntimeProviderAuthLookup).toHaveBeenCalledWith({
cfg,
workspaceDir: "/tmp/openclaw-workspace",
});
expect(modelAuthMock.hasRuntimeAvailableProviderAuth).toHaveBeenCalledWith({
provider: "codex",
cfg,
workspaceDir: "/tmp/openclaw-workspace",
runtimeLookup,
});
});
});

View File

@@ -14,8 +14,14 @@ import {
} from "../auth-profiles.js";
import type { AuthProfileCredential, AuthProfileStore } from "../auth-profiles/types.js";
import { DEFAULT_MODEL, DEFAULT_PROVIDER } from "../defaults.js";
import { hasUsableCustomProviderApiKey, resolveEnvApiKey } from "../model-auth.js";
import {
createRuntimeProviderAuthLookup,
hasRuntimeAvailableProviderAuth,
resolveEnvApiKey,
type RuntimeProviderAuthLookup,
} from "../model-auth.js";
import { resolveConfiguredModelRef } from "../model-selection.js";
import { normalizeProviderId } from "../provider-id.js";
export type ToolModelConfig = { primary?: string; fallbacks?: string[]; timeoutMs?: number };
@@ -85,7 +91,24 @@ export function hasProviderAuthForTool(params: {
workspaceDir?: string;
agentDir?: string;
authStore?: AuthProfileStore;
runtimeAuthLookup?: RuntimeProviderAuthLookup;
}): boolean {
const provider = normalizeProviderId(params.provider);
if (
hasRuntimeAvailableProviderAuth({
provider,
cfg: params.cfg,
workspaceDir: params.workspaceDir,
runtimeLookup:
params.runtimeAuthLookup ??
createRuntimeProviderAuthLookup({
cfg: params.cfg,
workspaceDir: params.workspaceDir,
}),
})
) {
return true;
}
if (
hasAuthForProvider({
provider: params.provider,
@@ -95,7 +118,90 @@ export function hasProviderAuthForTool(params: {
) {
return true;
}
return hasUsableCustomProviderApiKey(params.cfg, params.provider);
return false;
}
function isOpenAiGptModelRef(ref: { provider: string; model: string }): boolean {
return (
normalizeProviderId(ref.provider) === "openai" &&
ref.model.trim().toLowerCase().startsWith("gpt-")
);
}
function hasDirectOpenAiAuthForTool(params: {
cfg?: OpenClawConfig;
agentDir: string;
workspaceDir?: string;
authStore?: AuthProfileStore;
}): boolean {
if (
hasRuntimeAvailableProviderAuth({
provider: "openai",
cfg: params.cfg,
workspaceDir: params.workspaceDir,
allowPluginSyntheticAuth: false,
runtimeLookup: createRuntimeProviderAuthLookup({
cfg: params.cfg,
workspaceDir: params.workspaceDir,
includePluginSyntheticAuth: false,
}),
})
) {
return true;
}
return hasAuthProfileForProvider({
provider: "openai",
agentDir: params.agentDir,
authStore: params.authStore,
});
}
export function resolveCodexMediaCandidateForOpenAiCodexRoute(params: {
cfg?: OpenClawConfig;
primary: { provider: string; model: string };
agentDir: string;
workspaceDir?: string;
authStore?: AuthProfileStore;
resolveDefaultMediaModel: (params: {
cfg?: OpenClawConfig;
workspaceDir?: string;
providerId: string;
capability: "image";
}) => string | undefined;
}): string | null {
if (!isOpenAiGptModelRef(params.primary)) {
return null;
}
if (hasDirectOpenAiAuthForTool(params)) {
return null;
}
if (
!hasAuthProfileForProvider({
provider: "openai-codex",
agentDir: params.agentDir,
authStore: params.authStore,
})
) {
return null;
}
if (
!hasProviderAuthForTool({
provider: "codex",
cfg: params.cfg,
workspaceDir: params.workspaceDir,
agentDir: params.agentDir,
authStore: params.authStore,
})
) {
return null;
}
const modelId = params.resolveDefaultMediaModel({
cfg: params.cfg,
workspaceDir: params.workspaceDir,
providerId: "codex",
capability: "image",
});
return modelId ? `codex/${modelId}` : null;
}
export function coerceToolModelConfig(model?: AgentToolModelConfig): ToolModelConfig {
@@ -122,6 +228,10 @@ export function buildToolModelConfigFromCandidates(params: {
return params.explicit;
}
const runtimeAuthLookup = createRuntimeProviderAuthLookup({
cfg: params.cfg,
workspaceDir: params.workspaceDir,
});
const deduped: string[] = [];
for (const candidate of params.candidates) {
const trimmed = candidate?.trim();
@@ -137,6 +247,7 @@ export function buildToolModelConfigFromCandidates(params: {
workspaceDir: params.workspaceDir,
agentDir: params.agentDir,
authStore: params.authStore,
runtimeAuthLookup,
});
if (!provider || !providerConfigured) {
continue;

View File

@@ -6,6 +6,29 @@ import { resetPdfToolAuthEnv } from "./pdf-tool.test-support.js";
const ANTHROPIC_PDF_MODEL = "anthropic/claude-opus-4-7";
const TEST_AGENT_DIR = "/tmp/openclaw-pdf-model-config";
type MockAuthStore = { profiles?: Record<string, { provider?: string }> };
function hasOpenAiCodexProfile(authStore?: MockAuthStore): boolean {
return Object.values(authStore?.profiles ?? {}).some(
(profile) => profile?.provider === "openai-codex",
);
}
function makeOpenAiCodexAuthStore() {
return {
version: 1,
profiles: {
"openai-codex:test": {
type: "oauth",
provider: "openai-codex",
access: "codex-oauth-token",
refresh: "codex-refresh-token",
expires: Date.now() + 60_000,
},
},
} as const;
}
vi.mock("./model-config.helpers.js", () => ({
coerceToolModelConfig: (model?: unknown) => {
if (typeof model === "string") {
@@ -18,11 +41,22 @@ vi.mock("./model-config.helpers.js", () => ({
...(objectModel?.fallbacks?.length ? { fallbacks: objectModel.fallbacks } : {}),
};
},
hasProviderAuthForTool: ({ provider, cfg }: { provider: string; cfg?: OpenClawConfig }) => {
hasProviderAuthForTool: ({
provider,
cfg,
authStore,
}: {
provider: string;
cfg?: OpenClawConfig;
authStore?: MockAuthStore;
}) => {
const providerCfg = cfg?.models?.providers?.[provider] as { apiKey?: string } | undefined;
if (providerCfg?.apiKey?.trim()) {
return true;
}
if (provider === "codex") {
return hasOpenAiCodexProfile(authStore);
}
if (provider === "anthropic") {
return Boolean(process.env.ANTHROPIC_API_KEY || process.env.ANTHROPIC_OAUTH_TOKEN);
}
@@ -42,6 +76,28 @@ vi.mock("./model-config.helpers.js", () => ({
}
return false;
},
resolveCodexMediaCandidateForOpenAiCodexRoute: ({
primary,
authStore,
resolveDefaultMediaModel,
}: {
primary: { provider: string; model: string };
authStore?: MockAuthStore;
resolveDefaultMediaModel: (params: {
providerId: string;
capability: "image";
}) => string | undefined;
}) => {
if (
primary.provider !== "openai" ||
!primary.model.startsWith("gpt-") ||
!hasOpenAiCodexProfile(authStore)
) {
return null;
}
const model = resolveDefaultMediaModel({ providerId: "codex", capability: "image" });
return model ? `codex/${model}` : null;
},
resolveDefaultModelRef: (cfg?: OpenClawConfig) => {
const modelCfg = cfg?.agents?.defaults?.model;
const primary =
@@ -118,6 +174,15 @@ describe("resolvePdfModelConfigForTool", () => {
);
});
it("selects bounded Codex image understanding for OpenAI GPT models with Codex OAuth only", () => {
const cfg = withDefaultModel("openai/gpt-5.5");
const authStore = makeOpenAiCodexAuthStore();
expect(resolvePdfModelConfigForTool({ cfg, agentDir: TEST_AGENT_DIR, authStore })).toEqual({
primary: "codex/gpt-5.5",
});
});
it("uses configured MiniMax chat models for PDF text extraction fallback", () => {
vi.stubEnv("MINIMAX_API_KEY", "minimax-test");
const cfg = {

View File

@@ -13,7 +13,11 @@ import {
resolveConfiguredImageModelRefs,
resolveProviderVisionModelFromConfig,
} from "./image-tool.helpers.js";
import { hasProviderAuthForTool, resolveDefaultModelRef } from "./model-config.helpers.js";
import {
hasProviderAuthForTool,
resolveCodexMediaCandidateForOpenAiCodexRoute,
resolveDefaultModelRef,
} from "./model-config.helpers.js";
import { coercePdfModelConfig } from "./pdf-tool.helpers.js";
function formatProviderModelRef(providerId: string, modelId: string): string {
@@ -268,6 +272,17 @@ export function resolvePdfModelConfigForTool(params: {
workspaceDir: params.workspaceDir,
authStore: params.authStore,
});
const codexContextCandidate = resolveCodexMediaCandidateForOpenAiCodexRoute({
cfg: params.cfg,
primary,
agentDir: params.agentDir,
workspaceDir: params.workspaceDir,
authStore: params.authStore,
resolveDefaultMediaModel,
});
if (codexContextCandidate && !genericImageCandidates.includes(codexContextCandidate)) {
genericImageCandidates.unshift(codexContextCandidate);
}
const textExtractionCandidates = resolveTextExtractionCandidateRefs({
cfg: params.cfg,
primary,