fix(diagnostics): sanitize uncloneable model content

This commit is contained in:
Vincent Koc
2026-06-02 23:11:17 +02:00
parent 20577f0b3b
commit 5baa44d58a
6 changed files with 480 additions and 61 deletions

View File

@@ -4221,6 +4221,70 @@ describe("diagnostics-otel service", () => {
await service.stop?.(ctx);
});
test("exports model spans when captured tool definitions have unreadable schema fields", async () => {
const service = createDiagnosticsOtelService();
const ctx = createOtelContext(OTEL_TEST_ENDPOINT, {
traces: true,
captureContent: {
enabled: true,
toolDefinitions: true,
},
});
await service.start(ctx);
const toolDefinition = {
name: "unstable_tool",
description: "Unstable tool",
};
Object.defineProperty(toolDefinition, "parameters", {
enumerable: true,
get() {
throw new Error("schema unavailable");
},
});
emitTrustedModelCallCompletedWithContent(
{
runId: "run-1",
callId: "call-1",
provider: "openai",
model: "gpt-5.4",
durationMs: 80,
},
{
toolDefinitions: [toolDefinition],
},
);
await flushDiagnosticEvents();
const modelCall = telemetryState.tracer.startSpan.mock.calls.find(
(call) => call[0] === "openclaw.model.call",
);
const attrs = (modelCall?.[1] as { attributes?: Record<string, unknown> } | undefined)
?.attributes;
expect(JSON.parse(stringAttribute(attrs, "gen_ai.tool.definitions"))).toEqual([
{
type: "function",
name: "unstable_tool",
description: "Unstable tool",
parameters: {
truncated: true,
reason: "unreadable_diagnostic_field",
},
},
]);
expect(JSON.parse(stringAttribute(attrs, "openclaw.content.tool_definitions"))).toEqual([
{
name: "unstable_tool",
description: "Unstable tool",
parameters: {
truncated: true,
reason: "unreadable_diagnostic_field",
},
},
]);
await service.stop?.(ctx);
});
test("normalizes snake_case tool_call parts the same as camelCase toolCall parts", async () => {
const service = createDiagnosticsOtelService();
const ctx = createOtelContext(OTEL_TEST_ENDPOINT, {

View File

@@ -609,15 +609,24 @@ function truncateJsonArrayForOtelAttribute(
return [{ truncated: true, reason: "circular_reference" }];
}
options.seen.add(value);
const nextOptions = { ...options, maxDepth: options.maxDepth - 1 };
const items = value
.slice(0, options.maxArrayItems)
.map((item) => truncateJsonValueForOtelAttribute(item, nextOptions));
if (value.length > items.length) {
items.push({ truncated: true, omittedItems: value.length - items.length });
try {
const nextOptions = { ...options, maxDepth: options.maxDepth - 1 };
const items = arrayPrefix(value, options.maxArrayItems).map((item) =>
truncateJsonValueForOtelAttribute(item, nextOptions),
);
try {
if (value.length > items.length) {
items.push({ truncated: true, omittedItems: value.length - items.length });
}
} catch {
items.push({ truncated: true, reason: "unreadable_array_length" });
}
return items;
} catch {
return [{ truncated: true, reason: "unreadable_array" }];
} finally {
options.seen.delete(value);
}
options.seen.delete(value);
return items;
}
function truncateJsonObjectForOtelAttribute(
@@ -628,20 +637,27 @@ function truncateJsonObjectForOtelAttribute(
return { truncated: true, reason: "circular_reference" };
}
options.seen.add(value);
const nextOptions = { ...options, maxDepth: options.maxDepth - 1 };
const result: Record<string, unknown> = {};
const entries = Object.entries(value).filter(
([, field]) => field !== undefined && typeof field !== "function" && typeof field !== "symbol",
);
for (const [key, field] of entries.slice(0, options.maxObjectFields)) {
result[key] = truncateJsonValueForOtelAttribute(field, nextOptions);
try {
const nextOptions = { ...options, maxDepth: options.maxDepth - 1 };
const result: Record<string, unknown> = {};
const keys = Object.keys(value);
for (const key of keys.slice(0, options.maxObjectFields)) {
const field = readRecordValue(value, key);
if (field === undefined || typeof field === "function" || typeof field === "symbol") {
continue;
}
result[key] = truncateJsonValueForOtelAttribute(field, nextOptions);
}
if (keys.length > options.maxObjectFields) {
result.truncated = true;
result.omittedFields = keys.length - options.maxObjectFields;
}
return result;
} catch {
return { truncated: true, reason: "unreadable_object" };
} finally {
options.seen.delete(value);
}
if (entries.length > options.maxObjectFields) {
result.truncated = true;
result.omittedFields = entries.length - options.maxObjectFields;
}
options.seen.delete(value);
return result;
}
function truncateJsonTextForOtelAttribute(value: string, maxChars: number): string {
@@ -670,15 +686,51 @@ function isRecord(value: unknown): value is Record<string, unknown> {
return Boolean(value) && typeof value === "object" && !Array.isArray(value);
}
function readRecordValue(value: Record<string, unknown>, key: string): unknown {
try {
return value[key];
} catch {
return undefined;
}
}
function readRecordString(value: Record<string, unknown>, key: string): string | undefined {
const field = readRecordValue(value, key);
return typeof field === "string" ? field : undefined;
}
function arrayPrefix(value: readonly unknown[], maxItems: number): unknown[] {
try {
return value.slice(0, maxItems);
} catch {
return [];
}
}
function arrayItems(value: readonly unknown[]): unknown[] {
try {
return value.slice();
} catch {
return [];
}
}
function textPart(content: string): Record<string, unknown> {
return { type: "text", content };
}
function toolCallResponsePart(part: Record<string, unknown>): Record<string, unknown> {
const id = readRecordValue(part, "id");
const result =
readRecordValue(part, "result") ??
readRecordValue(part, "response") ??
readRecordValue(part, "content") ??
readRecordValue(part, "details") ??
"";
return {
type: "tool_call_response",
...(typeof part.id === "string" ? { id: part.id } : {}),
result: part.result ?? part.response ?? part.content ?? part.details ?? "",
...(typeof id === "string" ? { id } : {}),
result,
};
}
@@ -697,7 +749,7 @@ function contentParts(value: unknown): Record<string, unknown>[] {
return json ? [textPart(json)] : [];
}
const parts: Record<string, unknown>[] = [];
for (const part of value) {
for (const part of arrayItems(value)) {
if (typeof part === "string") {
if (part.length > 0) {
parts.push(textPart(part));
@@ -707,35 +759,44 @@ function contentParts(value: unknown): Record<string, unknown>[] {
if (!isRecord(part)) {
continue;
}
if (part.type === "text" && typeof part.text === "string") {
parts.push(textPart(part.text));
} else if (part.type === "text" && typeof part.content === "string") {
parts.push(textPart(part.content));
} else if (part.type === "thinking" && typeof part.thinking === "string") {
parts.push({ type: "reasoning", content: part.thinking });
} else if (part.type === "toolCall" && typeof part.name === "string") {
const type = readRecordValue(part, "type");
const text = readRecordString(part, "text");
const content = readRecordString(part, "content");
const thinking = readRecordString(part, "thinking");
const name = readRecordString(part, "name");
const id = readRecordValue(part, "id");
const args = readRecordValue(part, "arguments");
if (type === "text" && text !== undefined) {
parts.push(textPart(text));
} else if (type === "text" && content !== undefined) {
parts.push(textPart(content));
} else if (type === "thinking" && thinking !== undefined) {
parts.push({ type: "reasoning", content: thinking });
} else if (type === "toolCall" && name !== undefined) {
parts.push({
type: "tool_call",
name: part.name,
...(typeof part.id === "string" ? { id: part.id } : {}),
...(part.arguments !== undefined ? { arguments: part.arguments } : {}),
name,
...(typeof id === "string" ? { id } : {}),
...(args !== undefined ? { arguments: args } : {}),
});
} else if (part.type === "tool_call" && typeof part.name === "string") {
} else if (type === "tool_call" && name !== undefined) {
parts.push({
type: "tool_call",
name: part.name,
...(typeof part.id === "string" ? { id: part.id } : {}),
...(part.arguments !== undefined ? { arguments: part.arguments } : {}),
name,
...(typeof id === "string" ? { id } : {}),
...(args !== undefined ? { arguments: args } : {}),
});
} else if (part.type === "tool_call_response") {
} else if (type === "tool_call_response") {
parts.push(toolCallResponsePart(part));
} else if (part.type === "image") {
const data = typeof part.data === "string" ? part.data : undefined;
} else if (type === "image") {
const data = readRecordString(part, "data");
const mimeType = readRecordString(part, "mimeType");
const mimeTypeSnake = readRecordString(part, "mime_type");
parts.push({
type: "blob",
modality: "image",
...(typeof part.mimeType === "string" ? { mime_type: part.mimeType } : {}),
...(typeof part.mime_type === "string" ? { mime_type: part.mime_type } : {}),
...(mimeType !== undefined ? { mime_type: mimeType } : {}),
...(mimeTypeSnake !== undefined ? { mime_type: mimeTypeSnake } : {}),
...(data ? { content: data } : {}),
});
}
@@ -753,39 +814,42 @@ function normalizeGenAiMessage(
if (!isRecord(value)) {
return undefined;
}
const rawRole = typeof value.role === "string" ? value.role : fallbackRole;
const rawRole = readRecordString(value, "role") ?? fallbackRole;
const role = rawRole === "toolResult" ? "tool" : rawRole;
let parts: Record<string, unknown>[];
if (role === "tool") {
const explicitParts = contentParts(value.parts);
const explicitParts = contentParts(readRecordValue(value, "parts"));
parts =
explicitParts.length > 0
? explicitParts
: [
toolCallResponsePart({
id: value.toolCallId,
result: value.content ?? value.details ?? "",
id: readRecordValue(value, "toolCallId"),
result: readRecordValue(value, "content") ?? readRecordValue(value, "details") ?? "",
}),
];
} else {
parts = contentParts(value.parts ?? value.content);
parts = contentParts(readRecordValue(value, "parts") ?? readRecordValue(value, "content"));
}
if (parts.length === 0) {
return undefined;
}
const name = readRecordString(value, "name");
const finishReason = readRecordString(value, "finish_reason");
const stopReason = readRecordString(value, "stopReason");
return {
role,
parts,
...(typeof value.name === "string" ? { name: value.name } : {}),
...(typeof value.finish_reason === "string" ? { finish_reason: value.finish_reason } : {}),
...(typeof value.stopReason === "string" ? { finish_reason: value.stopReason } : {}),
...(name !== undefined ? { name } : {}),
...(finishReason !== undefined ? { finish_reason: finishReason } : {}),
...(stopReason !== undefined ? { finish_reason: stopReason } : {}),
};
}
function normalizeGenAiMessages(value: unknown, fallbackRole: "user" | "assistant") {
const source = Array.isArray(value) ? value : value === undefined ? [] : [value];
const messages: Record<string, unknown>[] = [];
for (const item of source.slice(0, MAX_OTEL_CONTENT_ARRAY_ITEMS)) {
for (const item of arrayPrefix(source, MAX_OTEL_CONTENT_ARRAY_ITEMS)) {
const message = normalizeGenAiMessage(item, fallbackRole);
if (message) {
messages.push(message);
@@ -795,14 +859,21 @@ function normalizeGenAiMessages(value: unknown, fallbackRole: "user" | "assistan
}
function normalizeGenAiToolDefinition(value: unknown): Record<string, unknown> | undefined {
if (!isRecord(value) || typeof value.name !== "string" || value.name.trim().length === 0) {
if (!isRecord(value)) {
return undefined;
}
const name = readRecordString(value, "name");
if (!name || name.trim().length === 0) {
return undefined;
}
const type = readRecordString(value, "type") ?? "function";
const description = readRecordString(value, "description");
const parameters = readRecordValue(value, "parameters");
return {
type: typeof value.type === "string" ? value.type : "function",
name: value.name,
...(typeof value.description === "string" ? { description: value.description } : {}),
...(value.parameters !== undefined ? { parameters: value.parameters } : {}),
type,
name,
...(description !== undefined ? { description } : {}),
...(parameters !== undefined ? { parameters } : {}),
};
}
@@ -811,7 +882,7 @@ function normalizeGenAiToolDefinitions(value: unknown) {
return [];
}
const definitions: Record<string, unknown>[] = [];
for (const item of value.slice(0, MAX_OTEL_CONTENT_ARRAY_ITEMS)) {
for (const item of arrayPrefix(value, MAX_OTEL_CONTENT_ARRAY_ITEMS)) {
const definition = normalizeGenAiToolDefinition(item);
if (definition) {
definitions.push(definition);

View File

@@ -524,6 +524,66 @@ describe("wrapStreamFnWithDiagnosticModelCallEvents", () => {
expect(events[1]?.privateData.modelContent?.outputMessages).toEqual([assistant]);
});
it("keeps model-call events when captured tool definitions cannot be cloned", async () => {
async function* stream() {
yield { type: "text_delta", delta: "ok" };
}
const wrapped = wrapStreamFnWithDiagnosticModelCallEvents(
(() => stream()) as unknown as StreamFn,
{
runId: "run-1",
provider: "openai",
model: "gpt-5.4",
trace: createDiagnosticTraceContext(),
contentCapture: {
inputMessages: false,
outputMessages: false,
toolInputs: false,
toolOutputs: false,
systemPrompt: false,
toolDefinitions: true,
anyModelContent: true,
},
nextCallId: () => "call-hostile-tools",
},
);
const hostileTool: Record<string | symbol, unknown> = {
name: "unstable_tool",
callback: () => undefined,
};
Object.defineProperty(hostileTool, "toJSON", {
value() {
throw new Error("schema json unavailable");
},
});
Object.defineProperty(hostileTool, Symbol.toPrimitive, {
value() {
throw new Error("schema string unavailable");
},
});
const events = await collectTrustedModelCallEvents(async () => {
await drain(
wrapped(
{} as never,
{
tools: [hostileTool],
} as never,
{},
) as unknown as AsyncIterable<unknown>,
);
});
expect(events.map((entry) => entry.event.type)).toEqual([
"model.call.started",
"model.call.completed",
]);
expect(events[0]?.privateData.modelContent?.toolDefinitions).toEqual({
truncated: true,
reason: "unserializable_diagnostic_content",
});
});
it("propagates the trusted model-call traceparent without mutating caller headers", async () => {
async function* stream() {
yield { type: "text", text: "ok" };

View File

@@ -138,7 +138,11 @@ function cloneDiagnosticContentValue(value: unknown): unknown {
const serialized = JSON.stringify(value);
return serialized === undefined ? null : (JSON.parse(serialized) as unknown);
} catch {
return String(value);
try {
return String(value);
} catch {
return { truncated: true, reason: "unserializable_diagnostic_content" };
}
}
}
}

View File

@@ -763,6 +763,110 @@ describe("diagnostic-events", () => {
});
});
it("sanitizes uncloneable trusted private data before dispatching to trusted listeners", async () => {
const trustedEvents: Array<{
event: DiagnosticEventPayload;
privateData: unknown;
}> = [];
onTrustedInternalDiagnosticEvent((event, _metadata, privateData) => {
trustedEvents.push({ event, privateData });
});
const toolDefinition = {
name: "unstable_tool",
callback: () => undefined,
};
Object.defineProperty(toolDefinition, "parameters", {
enumerable: true,
get() {
throw new Error("schema unavailable");
},
});
Object.defineProperty(toolDefinition, "__proto__", {
enumerable: true,
value: { polluted: true },
});
emitTrustedDiagnosticEventWithPrivateData(
{
type: "model.call.completed",
runId: "run-1",
callId: "call-1",
provider: "openai",
model: "gpt-5.4",
durationMs: 5,
},
{
modelContent: {
toolDefinitions: [toolDefinition],
},
},
);
await waitForDiagnosticEventsDrained();
const sanitizedPrivateData = trustedEvents[0]?.privateData as
| { modelContent?: { toolDefinitions?: unknown[] } }
| undefined;
const sanitizedToolDefinition = sanitizedPrivateData?.modelContent?.toolDefinitions?.[0];
expect(sanitizedToolDefinition).toEqual({
name: "unstable_tool",
parameters: {
truncated: true,
reason: "unreadable_diagnostic_field",
},
});
expect(sanitizedToolDefinition).not.toHaveProperty("polluted");
});
it("bounds uncloneable trusted private data object walks before dispatching", async () => {
const trustedEvents: Array<{
event: DiagnosticEventPayload;
privateData: unknown;
}> = [];
onTrustedInternalDiagnosticEvent((event, _metadata, privateData) => {
trustedEvents.push({ event, privateData });
});
const parameters: Record<string, unknown> = {};
for (let index = 0; index < 1005; index += 1) {
parameters[`field_${index}`] = { type: "string" };
}
const toolDefinition = {
name: "wide_tool",
callback: () => undefined,
parameters,
};
emitTrustedDiagnosticEventWithPrivateData(
{
type: "model.call.completed",
runId: "run-1",
callId: "call-1",
provider: "openai",
model: "gpt-5.4",
durationMs: 5,
},
{
modelContent: {
toolDefinitions: [toolDefinition],
},
},
);
await waitForDiagnosticEventsDrained();
const sanitizedPrivateData = trustedEvents[0]?.privateData as
| { modelContent?: { toolDefinitions?: Array<{ parameters?: Record<string, unknown> }> } }
| undefined;
const sanitizedToolParameters =
sanitizedPrivateData?.modelContent?.toolDefinitions?.[0]?.parameters;
expect(sanitizedToolParameters).toMatchObject({
field_0: { type: "string" },
truncated: true,
omittedFields: 5,
});
expect(sanitizedToolParameters).not.toHaveProperty("field_1000");
});
it("skips event enrichment and subscribers when diagnostics are disabled", () => {
const nowSpy = vi.spyOn(Date, "now");
const seen: string[] = [];

View File

@@ -752,6 +752,9 @@ type DiagnosticEventsGlobalState = {
const MAX_ASYNC_DIAGNOSTIC_EVENTS = 10_000;
const MAX_ASYNC_DIAGNOSTIC_EVENTS_PER_TURN = 100;
const MAX_SANITIZED_DIAGNOSTIC_ARRAY_ITEMS = 1000;
const MAX_SANITIZED_DIAGNOSTIC_OBJECT_FIELDS = 1000;
const MAX_SANITIZED_DIAGNOSTIC_DEPTH = 20;
const DIAGNOSTIC_EVENTS_STATE_KEY = Symbol.for("openclaw.diagnosticEvents.state.v1");
const dispatchedTrustedDiagnosticMetadata = new WeakSet<object>();
const ASYNC_DIAGNOSTIC_EVENT_TYPES = new Set<DiagnosticEventPayload["type"]>([
@@ -920,7 +923,9 @@ function createDiagnosticMetadataForListener(
}
function cloneDiagnosticEventForListener(event: DiagnosticEventPayload): DiagnosticEventPayload {
return deepFreezeDiagnosticValue(structuredClone(event)) as DiagnosticEventPayload;
return deepFreezeDiagnosticValue(
cloneDiagnosticValueForListener(event),
) as DiagnosticEventPayload;
}
function cloneDiagnosticPrivateDataForListener(
@@ -929,7 +934,118 @@ function cloneDiagnosticPrivateDataForListener(
if (!privateData) {
return Object.freeze({});
}
return deepFreezeDiagnosticValue(structuredClone(privateData)) as DiagnosticEventPrivateData;
return deepFreezeDiagnosticValue(
cloneDiagnosticValueForListener(privateData),
) as DiagnosticEventPrivateData;
}
function cloneDiagnosticValueForListener(value: unknown): unknown {
try {
return structuredClone(value);
} catch {
return sanitizeUncloneableDiagnosticValue(
value,
new WeakMap<object, unknown>(),
MAX_SANITIZED_DIAGNOSTIC_DEPTH,
);
}
}
function sanitizeUncloneableDiagnosticValue(
value: unknown,
seen: WeakMap<object, unknown>,
depth: number,
): unknown {
if (
value === null ||
typeof value === "string" ||
typeof value === "number" ||
typeof value === "boolean" ||
typeof value === "bigint"
) {
return value;
}
if (value === undefined || typeof value === "function" || typeof value === "symbol") {
return undefined;
}
if (typeof value !== "object") {
return { truncated: true, reason: "uncloneable_diagnostic_value" };
}
const existing = seen.get(value);
if (existing) {
return existing;
}
if (depth <= 0) {
return { truncated: true, reason: "max_diagnostic_depth" };
}
if (Array.isArray(value)) {
return sanitizeUncloneableDiagnosticArray(value, seen, depth);
}
return sanitizeUncloneableDiagnosticObject(value as Record<string, unknown>, seen, depth);
}
function sanitizeUncloneableDiagnosticArray(
value: readonly unknown[],
seen: WeakMap<object, unknown>,
depth: number,
): unknown {
let length: number;
try {
length = value.length;
} catch {
return { truncated: true, reason: "unreadable_diagnostic_array" };
}
if (!Number.isSafeInteger(length) || length < 0) {
return { truncated: true, reason: "invalid_diagnostic_array_length" };
}
const sanitized: unknown[] = [];
seen.set(value, sanitized);
const itemCount = Math.min(length, MAX_SANITIZED_DIAGNOSTIC_ARRAY_ITEMS);
for (let index = 0; index < itemCount; index += 1) {
try {
sanitized.push(sanitizeUncloneableDiagnosticValue(value[index], seen, depth - 1));
} catch {
sanitized.push({ truncated: true, reason: "unreadable_diagnostic_item" });
}
}
if (length > itemCount) {
sanitized.push({ truncated: true, omittedItems: length - itemCount });
}
return sanitized;
}
function sanitizeUncloneableDiagnosticObject(
value: Record<string, unknown>,
seen: WeakMap<object, unknown>,
depth: number,
): unknown {
let keys: string[];
try {
keys = Object.keys(value);
} catch {
return { truncated: true, reason: "unreadable_diagnostic_object" };
}
const sanitized = Object.create(null) as Record<string, unknown>;
seen.set(value, sanitized);
const fieldKeys = keys.slice(0, MAX_SANITIZED_DIAGNOSTIC_OBJECT_FIELDS);
for (const key of fieldKeys) {
if (isBlockedObjectKey(key)) {
continue;
}
try {
const field = value[key];
if (field !== undefined && typeof field !== "function" && typeof field !== "symbol") {
sanitized[key] = sanitizeUncloneableDiagnosticValue(field, seen, depth - 1);
}
} catch {
sanitized[key] = { truncated: true, reason: "unreadable_diagnostic_field" };
}
}
if (keys.length > fieldKeys.length) {
sanitized.truncated = true;
sanitized.omittedFields = keys.length - fieldKeys.length;
}
return sanitized;
}
function isPriorityAsyncDiagnosticEvent(entry: QueuedDiagnosticEvent): boolean {