fix(openai): skip unreadable responses tool schemas

This commit is contained in:
Vincent Koc
2026-06-04 17:27:37 +02:00
parent efda5918b5
commit 1f58a7d90f
4 changed files with 320 additions and 41 deletions

View File

@@ -1,5 +1,5 @@
import { createServer } from "node:http";
import type { Api, Model } from "openclaw/plugin-sdk/llm";
import type { Api, Model, Tool } from "openclaw/plugin-sdk/llm";
import { describe, expect, it, vi } from "vitest";
import {
buildOpenAIResponsesParams,
@@ -4608,6 +4608,69 @@ describe("openai transport stream", () => {
});
});
it("skips unreadable responses transport tools without downgrading healthy strict tools", () => {
const { proxy, revoke } = Proxy.revocable({}, {});
revoke();
const brokenDescriptor = {
get name(): string {
throw new Error("tool name unavailable");
},
description: "Broken",
parameters: {},
} as Tool;
const brokenSchema = {
name: "broken_schema",
description: "Broken",
parameters: proxy,
} as unknown as Tool;
const params = buildOpenAIResponsesParams(
{
id: "gpt-5.4",
name: "GPT-5.4",
api: "openai-responses",
provider: "openai",
baseUrl: "https://api.openai.com/v1",
reasoning: true,
input: ["text"],
cost: { input: 0, output: 0, cacheRead: 0, cacheWrite: 0 },
contextWindow: 200000,
maxTokens: 8192,
} satisfies Model<"openai-responses">,
{
systemPrompt: "system",
messages: [],
tools: [
brokenDescriptor,
brokenSchema,
{
name: "lookup_weather",
description: "Get forecast",
parameters: {},
},
],
} as never,
undefined,
) as {
tools?: Array<{ name?: string; strict?: boolean; parameters?: Record<string, unknown> }>;
};
expect(params.tools).toEqual([
{
type: "function",
name: "lookup_weather",
description: "Get forecast",
strict: true,
parameters: {
type: "object",
properties: {},
required: [],
additionalProperties: false,
},
},
]);
});
it("passes explicit Responses tool_choice when tools are present", () => {
const params = buildOpenAIResponsesParams(
{

View File

@@ -26,6 +26,7 @@ import { calculateCost } from "../llm/model-utils.js";
import { resolveAzureDeploymentNameFromMap } from "../llm/providers/azure-deployment-map.js";
import { convertMessages } from "../llm/providers/openai-completions.js";
import { clampOpenAIPromptCacheKey } from "../llm/providers/openai-prompt-cache.js";
import { convertResponsesTools as convertOpenAIResponsesTools } from "../llm/providers/openai-responses-tools.js";
import type { Api, Context, Model } from "../llm/types.js";
import { createAssistantMessageEventStream } from "../llm/utils/event-stream.js";
import { parseStreamingJson } from "../llm/utils/json-parse.js";
@@ -1274,33 +1275,6 @@ function convertResponsesMessages(
return messages;
}
function convertResponsesTools(
tools: NonNullable<Context["tools"]>,
model: OpenAIModeModel,
options?: { strict?: boolean | null },
): FunctionTool[] {
const strict = resolveOpenAIStrictToolFlagWithDiagnostics(tools, options?.strict, {
transport: "responses",
model,
});
return sortTransportToolsByName(tools).map((tool): FunctionTool => {
const result = {
type: "function" as const,
name: tool.name,
description: tool.description,
parameters: normalizeOpenAIStrictToolParameters(
tool.parameters,
strict === true,
model.compat,
) as Record<string, unknown>,
} as FunctionTool;
if (strict !== undefined) {
result.strict = strict;
}
return result;
});
}
function resolveOpenAIStrictToolFlagWithDiagnostics(
tools: NonNullable<Context["tools"]>,
strictSetting: boolean | null | undefined,
@@ -2257,11 +2231,12 @@ export function buildOpenAIResponsesParams(
params.service_tier = options.serviceTier;
}
if (context.tools) {
params.tools = convertResponsesTools(context.tools, model as OpenAIModeModel, {
params.tools = convertOpenAIResponsesTools(context.tools, {
model: model as OpenAIModeModel,
strict: resolveOpenAIStrictToolSetting(model as OpenAIModeModel, {
transport: "stream",
}),
});
}) as FunctionTool[];
if (options?.toolChoice) {
params.tool_choice = options.toolChoice;
}

View File

@@ -122,6 +122,115 @@ describe("convertResponsesTools", () => {
convertResponsesTools([zeta, alpha]).map((tool) => expectResponsesFunctionTool(tool).name),
).toEqual(["alpha", "zeta"]);
});
it("skips unreadable tool descriptors while preserving healthy tools", () => {
const broken = {
get name(): string {
const error = new Error("tool name unavailable");
error.toString = () => {
throw new Error("unprintable tool error");
};
throw error;
},
description: "Broken",
parameters: {},
} as Tool;
const healthy = {
name: "lookup_weather",
description: "Get forecast",
parameters: {},
} satisfies Tool;
const converted = convertResponsesTools([broken, healthy], { model: nativeOpenAIModel });
expect(converted).toEqual([
{
type: "function",
name: "lookup_weather",
description: "Get forecast",
strict: true,
parameters: {
type: "object",
properties: {},
required: [],
additionalProperties: false,
},
},
]);
});
it("skips unreadable schemas without downgrading healthy strict tools", () => {
const { proxy, revoke } = Proxy.revocable({}, {});
revoke();
const broken = {
name: "broken_schema",
description: "Broken",
parameters: proxy,
} as unknown as Tool;
const healthy = {
name: "lookup_weather",
description: "Get forecast",
parameters: {},
} satisfies Tool;
const converted = convertResponsesTools([broken, healthy], { model: nativeOpenAIModel });
expect(converted).toEqual([
{
type: "function",
name: "lookup_weather",
description: "Get forecast",
strict: true,
parameters: {
type: "object",
properties: {},
required: [],
additionalProperties: false,
},
},
]);
});
it("keeps valid shared schemas while skipping cyclic schemas", () => {
const sharedProperty = { type: "string" };
const sharedSchema = {
type: "object",
properties: {
first: sharedProperty,
second: sharedProperty,
},
required: ["first", "second"],
additionalProperties: false,
} satisfies Tool["parameters"];
const cyclicSchema: Record<string, unknown> = { type: "object" };
cyclicSchema.properties = { self: cyclicSchema };
const converted = convertResponsesTools(
[
{
name: "cyclic_schema",
description: "Broken",
parameters: cyclicSchema,
} as Tool,
{
name: "shared_schema",
description: "Shared",
parameters: sharedSchema,
},
],
{ model: nativeOpenAIModel },
);
expect(converted).toEqual([
{
type: "function",
name: "shared_schema",
description: "Shared",
strict: true,
parameters: sharedSchema,
},
]);
});
});
describe("convertResponsesMessages", () => {

View File

@@ -3,8 +3,8 @@ import type { Tool as OpenAITool } from "openai/resources/responses/responses.js
import { resolveOpenAIStrictToolSetting } from "../../agents/openai-strict-tool-setting.js";
import {
findOpenAIStrictToolSchemaDiagnostics,
isStrictOpenAIJsonSchemaCompatible,
normalizeOpenAIStrictToolParameters,
resolveOpenAIStrictToolFlagForInventory,
} from "../../agents/openai-tool-schema.js";
import { createSubsystemLogger } from "../../logging/subsystem.js";
import type { Model, Tool } from "../types.js";
@@ -24,6 +24,15 @@ type ResponsesFunctionTool = {
parameters: Record<string, unknown>;
strict?: boolean | null;
};
type PreparedResponsesTool = {
index: number;
name: string;
description: string;
parameters: unknown;
looseParameters: Record<string, unknown>;
strictCompatible?: boolean;
strictParameters?: Record<string, unknown>;
};
// Converts OpenClaw tool schemas to OpenAI Responses tools, including strict-mode compatibility.
const log = createSubsystemLogger("llm/openai-responses");
@@ -36,18 +45,17 @@ export function convertResponsesTools(
options?: ConvertResponsesToolsOptions,
): OpenAITool[] {
const strictSetting = resolveResponsesStrictToolSetting(options);
const strict = resolveResponsesStrictToolFlag(tools, strictSetting, options?.model);
const modelCompat = options?.model?.compat as OpenAIToolSchemaCompat;
const preparedTools = prepareResponsesTools(tools, strictSetting, modelCompat);
const strict = resolveResponsesStrictToolFlag(preparedTools, strictSetting, options?.model);
// Sort tools before request construction so prompt-cache bytes stay deterministic.
return sortResponsesToolsByName(tools).map((tool) => {
return sortResponsesToolsByName(preparedTools).map((tool) => {
const result: ResponsesFunctionTool = {
type: "function",
name: tool.name,
description: tool.description,
parameters: normalizeOpenAIStrictToolParameters(
tool.parameters,
strict === true,
options?.model?.compat as OpenAIToolSchemaCompat,
) as Record<string, unknown>,
parameters:
strict === true ? (tool.strictParameters ?? tool.looseParameters) : tool.looseParameters,
};
if (strict !== undefined) {
result.strict = strict;
@@ -71,14 +79,105 @@ function resolveResponsesStrictToolSetting(
return false;
}
function resolveResponsesStrictToolFlag(
function prepareResponsesTools(
tools: Tool[],
strictSetting: boolean | null | undefined,
modelCompat: OpenAIToolSchemaCompat | undefined,
): PreparedResponsesTool[] {
const prepared: PreparedResponsesTool[] = [];
for (const [index, tool] of tools.entries()) {
let name: string;
let description: string;
let parameters: unknown;
try {
name = tool.name;
description = tool.description;
parameters = tool.parameters;
} catch (error) {
warnSkippedResponsesTool({ index, reason: "descriptor was unreadable", error });
continue;
}
let looseParameters: Record<string, unknown>;
try {
looseParameters = normalizeOpenAIStrictToolParameters(
parameters,
false,
modelCompat,
) as Record<string, unknown>;
} catch (error) {
warnSkippedResponsesTool({
index,
name,
reason: "schema could not be normalized",
error,
});
continue;
}
if (strictSetting !== true) {
prepared.push({ index, name, description, parameters, looseParameters });
continue;
}
let strictCompatible: boolean;
try {
strictCompatible = isStrictOpenAIJsonSchemaCompatible(parameters);
} catch (error) {
warnSkippedResponsesTool({
index,
name,
reason: "schema could not be checked for strict mode",
error,
});
continue;
}
let strictParameters: Record<string, unknown> | undefined;
if (strictCompatible) {
try {
strictParameters = normalizeOpenAIStrictToolParameters(
parameters,
true,
modelCompat,
) as Record<string, unknown>;
} catch (error) {
warnSkippedResponsesTool({
index,
name,
reason: "schema could not be normalized for strict mode",
error,
});
continue;
}
}
prepared.push({
index,
name,
description,
parameters,
looseParameters,
strictCompatible,
...(strictParameters ? { strictParameters } : {}),
});
}
return prepared;
}
function resolveResponsesStrictToolFlag(
tools: PreparedResponsesTool[],
strictSetting: boolean | null | undefined,
model: Model | undefined,
): boolean | undefined {
const strict = resolveOpenAIStrictToolFlagForInventory(tools, strictSetting);
const strict =
strictSetting === true
? tools.every((tool) => tool.strictCompatible === true)
: strictSetting === false
? false
: undefined;
if (strictSetting === true && strict === false && model && log.isEnabled("debug", "any")) {
const diagnostics = findOpenAIStrictToolSchemaDiagnostics(tools);
const diagnostics = getStrictToolSchemaDiagnostics(tools);
if (shouldLogStrictToolDowngradeDiagnostic(diagnostics, model)) {
const sample = diagnostics.slice(0, 5).map((entry) => ({
tool: entry.toolName ?? `tool[${entry.toolIndex}]`,
@@ -100,6 +199,19 @@ function resolveResponsesStrictToolFlag(
return strict;
}
function getStrictToolSchemaDiagnostics(
tools: PreparedResponsesTool[],
): ReturnType<typeof findOpenAIStrictToolSchemaDiagnostics> {
try {
return findOpenAIStrictToolSchemaDiagnostics(tools);
} catch (error) {
log.warn(
`failed to inspect OpenAI Responses strict tool schemas: ${formatUnknownError(error)}`,
);
return [];
}
}
function shouldLogStrictToolDowngradeDiagnostic(
diagnostics: ReturnType<typeof findOpenAIStrictToolSchemaDiagnostics>,
model: Model,
@@ -149,3 +261,23 @@ function sortResponsesToolsByName<T extends { name?: string; description?: strin
compareToolText(left.description, right.description),
);
}
function warnSkippedResponsesTool(params: {
index: number;
name?: string;
reason: string;
error: unknown;
}): void {
const label = params.name ? `${params.name} (index ${params.index})` : `index ${params.index}`;
log.warn(
`skipping OpenAI Responses tool ${label}: ${params.reason}: ${formatUnknownError(params.error)}`,
);
}
function formatUnknownError(error: unknown): string {
try {
return String(error);
} catch {
return "<unprintable error>";
}
}