diff --git a/extensions/diagnostics-otel/src/service.test.ts b/extensions/diagnostics-otel/src/service.test.ts index e427a6d3b5c5..f0f80aa7c83d 100644 --- a/extensions/diagnostics-otel/src/service.test.ts +++ b/extensions/diagnostics-otel/src/service.test.ts @@ -4221,6 +4221,70 @@ describe("diagnostics-otel service", () => { await service.stop?.(ctx); }); + test("exports model spans when captured tool definitions have unreadable schema fields", async () => { + const service = createDiagnosticsOtelService(); + const ctx = createOtelContext(OTEL_TEST_ENDPOINT, { + traces: true, + captureContent: { + enabled: true, + toolDefinitions: true, + }, + }); + await service.start(ctx); + const toolDefinition = { + name: "unstable_tool", + description: "Unstable tool", + }; + Object.defineProperty(toolDefinition, "parameters", { + enumerable: true, + get() { + throw new Error("schema unavailable"); + }, + }); + + emitTrustedModelCallCompletedWithContent( + { + runId: "run-1", + callId: "call-1", + provider: "openai", + model: "gpt-5.4", + durationMs: 80, + }, + { + toolDefinitions: [toolDefinition], + }, + ); + await flushDiagnosticEvents(); + + const modelCall = telemetryState.tracer.startSpan.mock.calls.find( + (call) => call[0] === "openclaw.model.call", + ); + const attrs = (modelCall?.[1] as { attributes?: Record } | undefined) + ?.attributes; + expect(JSON.parse(stringAttribute(attrs, "gen_ai.tool.definitions"))).toEqual([ + { + type: "function", + name: "unstable_tool", + description: "Unstable tool", + parameters: { + truncated: true, + reason: "unreadable_diagnostic_field", + }, + }, + ]); + expect(JSON.parse(stringAttribute(attrs, "openclaw.content.tool_definitions"))).toEqual([ + { + name: "unstable_tool", + description: "Unstable tool", + parameters: { + truncated: true, + reason: "unreadable_diagnostic_field", + }, + }, + ]); + await service.stop?.(ctx); + }); + test("normalizes snake_case tool_call parts the same as camelCase toolCall parts", async () => { const service = createDiagnosticsOtelService(); const ctx = createOtelContext(OTEL_TEST_ENDPOINT, { diff --git a/extensions/diagnostics-otel/src/service.ts b/extensions/diagnostics-otel/src/service.ts index 4fac83490acf..642816de955d 100644 --- a/extensions/diagnostics-otel/src/service.ts +++ b/extensions/diagnostics-otel/src/service.ts @@ -609,15 +609,24 @@ function truncateJsonArrayForOtelAttribute( return [{ truncated: true, reason: "circular_reference" }]; } options.seen.add(value); - const nextOptions = { ...options, maxDepth: options.maxDepth - 1 }; - const items = value - .slice(0, options.maxArrayItems) - .map((item) => truncateJsonValueForOtelAttribute(item, nextOptions)); - if (value.length > items.length) { - items.push({ truncated: true, omittedItems: value.length - items.length }); + try { + const nextOptions = { ...options, maxDepth: options.maxDepth - 1 }; + const items = arrayPrefix(value, options.maxArrayItems).map((item) => + truncateJsonValueForOtelAttribute(item, nextOptions), + ); + try { + if (value.length > items.length) { + items.push({ truncated: true, omittedItems: value.length - items.length }); + } + } catch { + items.push({ truncated: true, reason: "unreadable_array_length" }); + } + return items; + } catch { + return [{ truncated: true, reason: "unreadable_array" }]; + } finally { + options.seen.delete(value); } - options.seen.delete(value); - return items; } function truncateJsonObjectForOtelAttribute( @@ -628,20 +637,27 @@ function truncateJsonObjectForOtelAttribute( return { truncated: true, reason: "circular_reference" }; } options.seen.add(value); - const nextOptions = { ...options, maxDepth: options.maxDepth - 1 }; - const result: Record = {}; - const entries = Object.entries(value).filter( - ([, field]) => field !== undefined && typeof field !== "function" && typeof field !== "symbol", - ); - for (const [key, field] of entries.slice(0, options.maxObjectFields)) { - result[key] = truncateJsonValueForOtelAttribute(field, nextOptions); + try { + const nextOptions = { ...options, maxDepth: options.maxDepth - 1 }; + const result: Record = {}; + const keys = Object.keys(value); + for (const key of keys.slice(0, options.maxObjectFields)) { + const field = readRecordValue(value, key); + if (field === undefined || typeof field === "function" || typeof field === "symbol") { + continue; + } + result[key] = truncateJsonValueForOtelAttribute(field, nextOptions); + } + if (keys.length > options.maxObjectFields) { + result.truncated = true; + result.omittedFields = keys.length - options.maxObjectFields; + } + return result; + } catch { + return { truncated: true, reason: "unreadable_object" }; + } finally { + options.seen.delete(value); } - if (entries.length > options.maxObjectFields) { - result.truncated = true; - result.omittedFields = entries.length - options.maxObjectFields; - } - options.seen.delete(value); - return result; } function truncateJsonTextForOtelAttribute(value: string, maxChars: number): string { @@ -670,15 +686,51 @@ function isRecord(value: unknown): value is Record { return Boolean(value) && typeof value === "object" && !Array.isArray(value); } +function readRecordValue(value: Record, key: string): unknown { + try { + return value[key]; + } catch { + return undefined; + } +} + +function readRecordString(value: Record, key: string): string | undefined { + const field = readRecordValue(value, key); + return typeof field === "string" ? field : undefined; +} + +function arrayPrefix(value: readonly unknown[], maxItems: number): unknown[] { + try { + return value.slice(0, maxItems); + } catch { + return []; + } +} + +function arrayItems(value: readonly unknown[]): unknown[] { + try { + return value.slice(); + } catch { + return []; + } +} + function textPart(content: string): Record { return { type: "text", content }; } function toolCallResponsePart(part: Record): Record { + const id = readRecordValue(part, "id"); + const result = + readRecordValue(part, "result") ?? + readRecordValue(part, "response") ?? + readRecordValue(part, "content") ?? + readRecordValue(part, "details") ?? + ""; return { type: "tool_call_response", - ...(typeof part.id === "string" ? { id: part.id } : {}), - result: part.result ?? part.response ?? part.content ?? part.details ?? "", + ...(typeof id === "string" ? { id } : {}), + result, }; } @@ -697,7 +749,7 @@ function contentParts(value: unknown): Record[] { return json ? [textPart(json)] : []; } const parts: Record[] = []; - for (const part of value) { + for (const part of arrayItems(value)) { if (typeof part === "string") { if (part.length > 0) { parts.push(textPart(part)); @@ -707,35 +759,44 @@ function contentParts(value: unknown): Record[] { if (!isRecord(part)) { continue; } - if (part.type === "text" && typeof part.text === "string") { - parts.push(textPart(part.text)); - } else if (part.type === "text" && typeof part.content === "string") { - parts.push(textPart(part.content)); - } else if (part.type === "thinking" && typeof part.thinking === "string") { - parts.push({ type: "reasoning", content: part.thinking }); - } else if (part.type === "toolCall" && typeof part.name === "string") { + const type = readRecordValue(part, "type"); + const text = readRecordString(part, "text"); + const content = readRecordString(part, "content"); + const thinking = readRecordString(part, "thinking"); + const name = readRecordString(part, "name"); + const id = readRecordValue(part, "id"); + const args = readRecordValue(part, "arguments"); + if (type === "text" && text !== undefined) { + parts.push(textPart(text)); + } else if (type === "text" && content !== undefined) { + parts.push(textPart(content)); + } else if (type === "thinking" && thinking !== undefined) { + parts.push({ type: "reasoning", content: thinking }); + } else if (type === "toolCall" && name !== undefined) { parts.push({ type: "tool_call", - name: part.name, - ...(typeof part.id === "string" ? { id: part.id } : {}), - ...(part.arguments !== undefined ? { arguments: part.arguments } : {}), + name, + ...(typeof id === "string" ? { id } : {}), + ...(args !== undefined ? { arguments: args } : {}), }); - } else if (part.type === "tool_call" && typeof part.name === "string") { + } else if (type === "tool_call" && name !== undefined) { parts.push({ type: "tool_call", - name: part.name, - ...(typeof part.id === "string" ? { id: part.id } : {}), - ...(part.arguments !== undefined ? { arguments: part.arguments } : {}), + name, + ...(typeof id === "string" ? { id } : {}), + ...(args !== undefined ? { arguments: args } : {}), }); - } else if (part.type === "tool_call_response") { + } else if (type === "tool_call_response") { parts.push(toolCallResponsePart(part)); - } else if (part.type === "image") { - const data = typeof part.data === "string" ? part.data : undefined; + } else if (type === "image") { + const data = readRecordString(part, "data"); + const mimeType = readRecordString(part, "mimeType"); + const mimeTypeSnake = readRecordString(part, "mime_type"); parts.push({ type: "blob", modality: "image", - ...(typeof part.mimeType === "string" ? { mime_type: part.mimeType } : {}), - ...(typeof part.mime_type === "string" ? { mime_type: part.mime_type } : {}), + ...(mimeType !== undefined ? { mime_type: mimeType } : {}), + ...(mimeTypeSnake !== undefined ? { mime_type: mimeTypeSnake } : {}), ...(data ? { content: data } : {}), }); } @@ -753,39 +814,42 @@ function normalizeGenAiMessage( if (!isRecord(value)) { return undefined; } - const rawRole = typeof value.role === "string" ? value.role : fallbackRole; + const rawRole = readRecordString(value, "role") ?? fallbackRole; const role = rawRole === "toolResult" ? "tool" : rawRole; let parts: Record[]; if (role === "tool") { - const explicitParts = contentParts(value.parts); + const explicitParts = contentParts(readRecordValue(value, "parts")); parts = explicitParts.length > 0 ? explicitParts : [ toolCallResponsePart({ - id: value.toolCallId, - result: value.content ?? value.details ?? "", + id: readRecordValue(value, "toolCallId"), + result: readRecordValue(value, "content") ?? readRecordValue(value, "details") ?? "", }), ]; } else { - parts = contentParts(value.parts ?? value.content); + parts = contentParts(readRecordValue(value, "parts") ?? readRecordValue(value, "content")); } if (parts.length === 0) { return undefined; } + const name = readRecordString(value, "name"); + const finishReason = readRecordString(value, "finish_reason"); + const stopReason = readRecordString(value, "stopReason"); return { role, parts, - ...(typeof value.name === "string" ? { name: value.name } : {}), - ...(typeof value.finish_reason === "string" ? { finish_reason: value.finish_reason } : {}), - ...(typeof value.stopReason === "string" ? { finish_reason: value.stopReason } : {}), + ...(name !== undefined ? { name } : {}), + ...(finishReason !== undefined ? { finish_reason: finishReason } : {}), + ...(stopReason !== undefined ? { finish_reason: stopReason } : {}), }; } function normalizeGenAiMessages(value: unknown, fallbackRole: "user" | "assistant") { const source = Array.isArray(value) ? value : value === undefined ? [] : [value]; const messages: Record[] = []; - for (const item of source.slice(0, MAX_OTEL_CONTENT_ARRAY_ITEMS)) { + for (const item of arrayPrefix(source, MAX_OTEL_CONTENT_ARRAY_ITEMS)) { const message = normalizeGenAiMessage(item, fallbackRole); if (message) { messages.push(message); @@ -795,14 +859,21 @@ function normalizeGenAiMessages(value: unknown, fallbackRole: "user" | "assistan } function normalizeGenAiToolDefinition(value: unknown): Record | undefined { - if (!isRecord(value) || typeof value.name !== "string" || value.name.trim().length === 0) { + if (!isRecord(value)) { return undefined; } + const name = readRecordString(value, "name"); + if (!name || name.trim().length === 0) { + return undefined; + } + const type = readRecordString(value, "type") ?? "function"; + const description = readRecordString(value, "description"); + const parameters = readRecordValue(value, "parameters"); return { - type: typeof value.type === "string" ? value.type : "function", - name: value.name, - ...(typeof value.description === "string" ? { description: value.description } : {}), - ...(value.parameters !== undefined ? { parameters: value.parameters } : {}), + type, + name, + ...(description !== undefined ? { description } : {}), + ...(parameters !== undefined ? { parameters } : {}), }; } @@ -811,7 +882,7 @@ function normalizeGenAiToolDefinitions(value: unknown) { return []; } const definitions: Record[] = []; - for (const item of value.slice(0, MAX_OTEL_CONTENT_ARRAY_ITEMS)) { + for (const item of arrayPrefix(value, MAX_OTEL_CONTENT_ARRAY_ITEMS)) { const definition = normalizeGenAiToolDefinition(item); if (definition) { definitions.push(definition); diff --git a/src/agents/embedded-agent-runner/run/attempt.model-diagnostic-events.test.ts b/src/agents/embedded-agent-runner/run/attempt.model-diagnostic-events.test.ts index 6d0cd224b241..1c916403ed08 100644 --- a/src/agents/embedded-agent-runner/run/attempt.model-diagnostic-events.test.ts +++ b/src/agents/embedded-agent-runner/run/attempt.model-diagnostic-events.test.ts @@ -524,6 +524,66 @@ describe("wrapStreamFnWithDiagnosticModelCallEvents", () => { expect(events[1]?.privateData.modelContent?.outputMessages).toEqual([assistant]); }); + it("keeps model-call events when captured tool definitions cannot be cloned", async () => { + async function* stream() { + yield { type: "text_delta", delta: "ok" }; + } + const wrapped = wrapStreamFnWithDiagnosticModelCallEvents( + (() => stream()) as unknown as StreamFn, + { + runId: "run-1", + provider: "openai", + model: "gpt-5.4", + trace: createDiagnosticTraceContext(), + contentCapture: { + inputMessages: false, + outputMessages: false, + toolInputs: false, + toolOutputs: false, + systemPrompt: false, + toolDefinitions: true, + anyModelContent: true, + }, + nextCallId: () => "call-hostile-tools", + }, + ); + const hostileTool: Record = { + name: "unstable_tool", + callback: () => undefined, + }; + Object.defineProperty(hostileTool, "toJSON", { + value() { + throw new Error("schema json unavailable"); + }, + }); + Object.defineProperty(hostileTool, Symbol.toPrimitive, { + value() { + throw new Error("schema string unavailable"); + }, + }); + + const events = await collectTrustedModelCallEvents(async () => { + await drain( + wrapped( + {} as never, + { + tools: [hostileTool], + } as never, + {}, + ) as unknown as AsyncIterable, + ); + }); + + expect(events.map((entry) => entry.event.type)).toEqual([ + "model.call.started", + "model.call.completed", + ]); + expect(events[0]?.privateData.modelContent?.toolDefinitions).toEqual({ + truncated: true, + reason: "unserializable_diagnostic_content", + }); + }); + it("propagates the trusted model-call traceparent without mutating caller headers", async () => { async function* stream() { yield { type: "text", text: "ok" }; diff --git a/src/agents/embedded-agent-runner/run/attempt.model-diagnostic-events.ts b/src/agents/embedded-agent-runner/run/attempt.model-diagnostic-events.ts index 26e96633c91b..d03f8f6f031a 100644 --- a/src/agents/embedded-agent-runner/run/attempt.model-diagnostic-events.ts +++ b/src/agents/embedded-agent-runner/run/attempt.model-diagnostic-events.ts @@ -138,7 +138,11 @@ function cloneDiagnosticContentValue(value: unknown): unknown { const serialized = JSON.stringify(value); return serialized === undefined ? null : (JSON.parse(serialized) as unknown); } catch { - return String(value); + try { + return String(value); + } catch { + return { truncated: true, reason: "unserializable_diagnostic_content" }; + } } } } diff --git a/src/infra/diagnostic-events.test.ts b/src/infra/diagnostic-events.test.ts index 5f12a56e0f54..6df9adc177ac 100644 --- a/src/infra/diagnostic-events.test.ts +++ b/src/infra/diagnostic-events.test.ts @@ -763,6 +763,110 @@ describe("diagnostic-events", () => { }); }); + it("sanitizes uncloneable trusted private data before dispatching to trusted listeners", async () => { + const trustedEvents: Array<{ + event: DiagnosticEventPayload; + privateData: unknown; + }> = []; + onTrustedInternalDiagnosticEvent((event, _metadata, privateData) => { + trustedEvents.push({ event, privateData }); + }); + const toolDefinition = { + name: "unstable_tool", + callback: () => undefined, + }; + Object.defineProperty(toolDefinition, "parameters", { + enumerable: true, + get() { + throw new Error("schema unavailable"); + }, + }); + Object.defineProperty(toolDefinition, "__proto__", { + enumerable: true, + value: { polluted: true }, + }); + + emitTrustedDiagnosticEventWithPrivateData( + { + type: "model.call.completed", + runId: "run-1", + callId: "call-1", + provider: "openai", + model: "gpt-5.4", + durationMs: 5, + }, + { + modelContent: { + toolDefinitions: [toolDefinition], + }, + }, + ); + + await waitForDiagnosticEventsDrained(); + + const sanitizedPrivateData = trustedEvents[0]?.privateData as + | { modelContent?: { toolDefinitions?: unknown[] } } + | undefined; + const sanitizedToolDefinition = sanitizedPrivateData?.modelContent?.toolDefinitions?.[0]; + expect(sanitizedToolDefinition).toEqual({ + name: "unstable_tool", + parameters: { + truncated: true, + reason: "unreadable_diagnostic_field", + }, + }); + expect(sanitizedToolDefinition).not.toHaveProperty("polluted"); + }); + + it("bounds uncloneable trusted private data object walks before dispatching", async () => { + const trustedEvents: Array<{ + event: DiagnosticEventPayload; + privateData: unknown; + }> = []; + onTrustedInternalDiagnosticEvent((event, _metadata, privateData) => { + trustedEvents.push({ event, privateData }); + }); + const parameters: Record = {}; + for (let index = 0; index < 1005; index += 1) { + parameters[`field_${index}`] = { type: "string" }; + } + const toolDefinition = { + name: "wide_tool", + callback: () => undefined, + parameters, + }; + + emitTrustedDiagnosticEventWithPrivateData( + { + type: "model.call.completed", + runId: "run-1", + callId: "call-1", + provider: "openai", + model: "gpt-5.4", + durationMs: 5, + }, + { + modelContent: { + toolDefinitions: [toolDefinition], + }, + }, + ); + + await waitForDiagnosticEventsDrained(); + + const sanitizedPrivateData = trustedEvents[0]?.privateData as + | { modelContent?: { toolDefinitions?: Array<{ parameters?: Record }> } } + | undefined; + const sanitizedToolParameters = + sanitizedPrivateData?.modelContent?.toolDefinitions?.[0]?.parameters; + expect(sanitizedToolParameters).toMatchObject({ + field_0: { type: "string" }, + truncated: true, + omittedFields: 5, + }); + expect(sanitizedToolParameters).not.toHaveProperty("field_1000"); + }); + it("skips event enrichment and subscribers when diagnostics are disabled", () => { const nowSpy = vi.spyOn(Date, "now"); const seen: string[] = []; diff --git a/src/infra/diagnostic-events.ts b/src/infra/diagnostic-events.ts index 88f320d430be..ea574da59a77 100644 --- a/src/infra/diagnostic-events.ts +++ b/src/infra/diagnostic-events.ts @@ -752,6 +752,9 @@ type DiagnosticEventsGlobalState = { const MAX_ASYNC_DIAGNOSTIC_EVENTS = 10_000; const MAX_ASYNC_DIAGNOSTIC_EVENTS_PER_TURN = 100; +const MAX_SANITIZED_DIAGNOSTIC_ARRAY_ITEMS = 1000; +const MAX_SANITIZED_DIAGNOSTIC_OBJECT_FIELDS = 1000; +const MAX_SANITIZED_DIAGNOSTIC_DEPTH = 20; const DIAGNOSTIC_EVENTS_STATE_KEY = Symbol.for("openclaw.diagnosticEvents.state.v1"); const dispatchedTrustedDiagnosticMetadata = new WeakSet(); const ASYNC_DIAGNOSTIC_EVENT_TYPES = new Set([ @@ -920,7 +923,9 @@ function createDiagnosticMetadataForListener( } function cloneDiagnosticEventForListener(event: DiagnosticEventPayload): DiagnosticEventPayload { - return deepFreezeDiagnosticValue(structuredClone(event)) as DiagnosticEventPayload; + return deepFreezeDiagnosticValue( + cloneDiagnosticValueForListener(event), + ) as DiagnosticEventPayload; } function cloneDiagnosticPrivateDataForListener( @@ -929,7 +934,118 @@ function cloneDiagnosticPrivateDataForListener( if (!privateData) { return Object.freeze({}); } - return deepFreezeDiagnosticValue(structuredClone(privateData)) as DiagnosticEventPrivateData; + return deepFreezeDiagnosticValue( + cloneDiagnosticValueForListener(privateData), + ) as DiagnosticEventPrivateData; +} + +function cloneDiagnosticValueForListener(value: unknown): unknown { + try { + return structuredClone(value); + } catch { + return sanitizeUncloneableDiagnosticValue( + value, + new WeakMap(), + MAX_SANITIZED_DIAGNOSTIC_DEPTH, + ); + } +} + +function sanitizeUncloneableDiagnosticValue( + value: unknown, + seen: WeakMap, + depth: number, +): unknown { + if ( + value === null || + typeof value === "string" || + typeof value === "number" || + typeof value === "boolean" || + typeof value === "bigint" + ) { + return value; + } + if (value === undefined || typeof value === "function" || typeof value === "symbol") { + return undefined; + } + if (typeof value !== "object") { + return { truncated: true, reason: "uncloneable_diagnostic_value" }; + } + const existing = seen.get(value); + if (existing) { + return existing; + } + if (depth <= 0) { + return { truncated: true, reason: "max_diagnostic_depth" }; + } + if (Array.isArray(value)) { + return sanitizeUncloneableDiagnosticArray(value, seen, depth); + } + return sanitizeUncloneableDiagnosticObject(value as Record, seen, depth); +} + +function sanitizeUncloneableDiagnosticArray( + value: readonly unknown[], + seen: WeakMap, + depth: number, +): unknown { + let length: number; + try { + length = value.length; + } catch { + return { truncated: true, reason: "unreadable_diagnostic_array" }; + } + if (!Number.isSafeInteger(length) || length < 0) { + return { truncated: true, reason: "invalid_diagnostic_array_length" }; + } + const sanitized: unknown[] = []; + seen.set(value, sanitized); + const itemCount = Math.min(length, MAX_SANITIZED_DIAGNOSTIC_ARRAY_ITEMS); + for (let index = 0; index < itemCount; index += 1) { + try { + sanitized.push(sanitizeUncloneableDiagnosticValue(value[index], seen, depth - 1)); + } catch { + sanitized.push({ truncated: true, reason: "unreadable_diagnostic_item" }); + } + } + if (length > itemCount) { + sanitized.push({ truncated: true, omittedItems: length - itemCount }); + } + return sanitized; +} + +function sanitizeUncloneableDiagnosticObject( + value: Record, + seen: WeakMap, + depth: number, +): unknown { + let keys: string[]; + try { + keys = Object.keys(value); + } catch { + return { truncated: true, reason: "unreadable_diagnostic_object" }; + } + const sanitized = Object.create(null) as Record; + seen.set(value, sanitized); + const fieldKeys = keys.slice(0, MAX_SANITIZED_DIAGNOSTIC_OBJECT_FIELDS); + for (const key of fieldKeys) { + if (isBlockedObjectKey(key)) { + continue; + } + try { + const field = value[key]; + if (field !== undefined && typeof field !== "function" && typeof field !== "symbol") { + sanitized[key] = sanitizeUncloneableDiagnosticValue(field, seen, depth - 1); + } + } catch { + sanitized[key] = { truncated: true, reason: "unreadable_diagnostic_field" }; + } + } + if (keys.length > fieldKeys.length) { + sanitized.truncated = true; + sanitized.omittedFields = keys.length - fieldKeys.length; + } + return sanitized; } function isPriorityAsyncDiagnosticEvent(entry: QueuedDiagnosticEvent): boolean {