From 0a43e9cd78130d5e1386ac7570f3f4fb3156d675 Mon Sep 17 00:00:00 2001 From: Vincent Koc Date: Thu, 4 Jun 2026 12:48:06 +0200 Subject: [PATCH] fix(qa-lab): guard parity stable hashes --- extensions/qa-lab/src/harness-parity.test.ts | 110 +++++++++++++++++++ extensions/qa-lab/src/harness-parity.ts | 25 +---- extensions/qa-lab/src/runtime-parity.ts | 25 +---- extensions/qa-lab/src/stable-hash.ts | 91 +++++++++++++++ 4 files changed, 203 insertions(+), 48 deletions(-) create mode 100644 extensions/qa-lab/src/stable-hash.ts diff --git a/extensions/qa-lab/src/harness-parity.test.ts b/extensions/qa-lab/src/harness-parity.test.ts index a332cfd14ba3..219567024bb6 100644 --- a/extensions/qa-lab/src/harness-parity.test.ts +++ b/extensions/qa-lab/src/harness-parity.test.ts @@ -265,6 +265,116 @@ describe("harness parity", () => { ).toBe("tool-description"); }); + it("keeps unreadable tool schemas from crashing parity hashing", () => { + const unreadableSchema: Record = {}; + Object.defineProperty(unreadableSchema, "properties", { + enumerable: true, + get() { + throw new Error("schema getter exploded"); + }, + }); + const left = buildHarnessParityCell({ + variant: LEFT, + cell: makeCell("openclaw", { + systemPromptReport: { + ...BASE_PROMPT_REPORT, + tools: { + ...BASE_PROMPT_REPORT.tools, + entries: [ + { + ...BASE_PROMPT_REPORT.tools.entries[0], + schema: { properties: null }, + }, + ], + }, + }, + }), + tokenUsageSource: "live-usage", + }); + const right = buildHarnessParityCell({ + variant: RIGHT, + cell: makeCell("openclaw", { + systemPromptReport: { + ...BASE_PROMPT_REPORT, + tools: { + ...BASE_PROMPT_REPORT.tools, + entries: [ + { + ...BASE_PROMPT_REPORT.tools.entries[0], + schema: unreadableSchema, + }, + ], + }, + }, + }), + tokenUsageSource: "live-usage", + }); + + expect( + buildHarnessParityResult({ + scenarioId: "scenario", + left, + right, + }).drift, + ).toBe("tool-schema"); + }); + + it("keeps unreadable array tool schema fields from crashing parity hashing", () => { + const unreadableRequired = ["query"]; + Object.defineProperty(unreadableRequired, "0", { + enumerable: true, + get() { + throw new Error("required getter exploded"); + }, + }); + const cyclicAnyOf: unknown[] = []; + cyclicAnyOf.push(cyclicAnyOf); + const left = buildHarnessParityCell({ + variant: LEFT, + cell: makeCell("openclaw", { + systemPromptReport: { + ...BASE_PROMPT_REPORT, + tools: { + ...BASE_PROMPT_REPORT.tools, + entries: [ + { + ...BASE_PROMPT_REPORT.tools.entries[0], + schema: { anyOf: [null], required: [null] }, + }, + ], + }, + }, + }), + tokenUsageSource: "live-usage", + }); + const right = buildHarnessParityCell({ + variant: RIGHT, + cell: makeCell("openclaw", { + systemPromptReport: { + ...BASE_PROMPT_REPORT, + tools: { + ...BASE_PROMPT_REPORT.tools, + entries: [ + { + ...BASE_PROMPT_REPORT.tools.entries[0], + schema: { anyOf: cyclicAnyOf, required: unreadableRequired }, + }, + ], + }, + }, + }), + tokenUsageSource: "live-usage", + }); + + expect( + buildHarnessParityResult({ + scenarioId: "scenario", + left, + right, + }).drift, + ).toBe("tool-schema"); + }); + it("labels mock token estimates separately from live usage", () => { const sourceCell = makeCell("openclaw", { usage: { inputTokens: 0, outputTokens: 0, totalTokens: 0 }, diff --git a/extensions/qa-lab/src/harness-parity.ts b/extensions/qa-lab/src/harness-parity.ts index 449c492aa895..acd928678a7f 100644 --- a/extensions/qa-lab/src/harness-parity.ts +++ b/extensions/qa-lab/src/harness-parity.ts @@ -1,4 +1,3 @@ -import { createHash } from "node:crypto"; import type { RuntimeId, RuntimeParityCell, @@ -7,6 +6,7 @@ import type { RuntimeParityUsage, } from "./runtime-parity.js"; import type { RuntimeParityComparisonMode } from "./runtime-tool-metadata.js"; +import { stableHash } from "./stable-hash.js"; export type HarnessVariant = { id: string; @@ -107,10 +107,6 @@ export type HarnessParityReport = { failures: string[]; }; -function sha256(value: string) { - return createHash("sha256").update(value).digest("hex"); -} - function countComparableTranscriptRecords(transcriptBytes: string) { let count = 0; for (const line of transcriptBytes.split(/\r?\n/u)) { @@ -136,25 +132,6 @@ function countComparableTranscriptRecords(transcriptBytes: string) { return count; } -function normalizeForStableHash(value: unknown): unknown { - if (Array.isArray(value)) { - return value.map((entry) => normalizeForStableHash(entry)); - } - if (value && typeof value === "object") { - const record = value as Record; - return Object.fromEntries( - Object.keys(record) - .toSorted((left, right) => left.localeCompare(right)) - .map((key) => [key, normalizeForStableHash(record[key])]), - ); - } - return value; -} - -function stableHash(value: unknown) { - return sha256(JSON.stringify(normalizeForStableHash(value)) ?? "null"); -} - function readPositiveNumber(value: unknown) { return typeof value === "number" && Number.isFinite(value) && value > 0 ? Math.floor(value) : 0; } diff --git a/extensions/qa-lab/src/runtime-parity.ts b/extensions/qa-lab/src/runtime-parity.ts index 869384c05208..7eed57370945 100644 --- a/extensions/qa-lab/src/runtime-parity.ts +++ b/extensions/qa-lab/src/runtime-parity.ts @@ -1,4 +1,3 @@ -import { createHash } from "node:crypto"; import fs from "node:fs/promises"; import path from "node:path"; import { fetchWithSsrFGuard } from "openclaw/plugin-sdk/ssrf-runtime"; @@ -12,6 +11,7 @@ import { scanGatewayLogSentinels, type GatewayLogSentinelFinding, } from "./gateway-log-sentinel.js"; +import { stableHash } from "./stable-hash.js"; export type RuntimeId = "openclaw" | "codex"; @@ -141,29 +141,6 @@ function normalizeTextForParity(text: string) { return text.replace(/\s+/gu, " ").trim(); } -function sha256(value: string) { - return createHash("sha256").update(value).digest("hex"); -} - -function normalizeForStableHash(value: unknown): unknown { - if (Array.isArray(value)) { - return value.map((entry) => normalizeForStableHash(entry)); - } - if (value && typeof value === "object") { - const record = value as Record; - return Object.fromEntries( - Object.keys(record) - .toSorted((left, right) => left.localeCompare(right)) - .map((key) => [key, normalizeForStableHash(record[key])]), - ); - } - return value; -} - -function stableHash(value: unknown) { - return sha256(JSON.stringify(normalizeForStableHash(value)) ?? "null"); -} - function readUsageTotals(raw: unknown): RuntimeParityUsage { const usage = isMessageRecord(raw) ? raw : {}; const inputTokens = diff --git a/extensions/qa-lab/src/stable-hash.ts b/extensions/qa-lab/src/stable-hash.ts new file mode 100644 index 000000000000..ff78dd213315 --- /dev/null +++ b/extensions/qa-lab/src/stable-hash.ts @@ -0,0 +1,91 @@ +import { createHash } from "node:crypto"; + +type StableHashState = { + circularPaths: string[]; + unreadablePaths: string[]; + seen: WeakSet; +}; + +function sha256(value: string) { + return createHash("sha256").update(value).digest("hex"); +} + +function childPath(path: string, key: string | number) { + return `${path}[${JSON.stringify(key)}]`; +} + +function normalizeForStableHash(value: unknown, state: StableHashState, path: string): unknown { + if (!value || typeof value !== "object") { + return value; + } + if (state.seen.has(value)) { + state.circularPaths.push(path); + return null; + } + + state.seen.add(value); + if (Array.isArray(value)) { + const entries: unknown[] = []; + let length: number; + try { + length = value.length; + } catch { + state.seen.delete(value); + state.unreadablePaths.push(path); + return null; + } + for (let index = 0; index < length; index += 1) { + let entry: unknown; + try { + entry = Reflect.get(value, index); + } catch { + state.unreadablePaths.push(childPath(path, index)); + entry = null; + } + entries.push(normalizeForStableHash(entry, state, childPath(path, index))); + } + state.seen.delete(value); + return entries; + } + + let keys: string[]; + try { + keys = Object.keys(value).toSorted((left, right) => left.localeCompare(right)); + } catch { + state.seen.delete(value); + state.unreadablePaths.push(path); + return null; + } + + const record = value as Record; + const entries = keys.map((key) => { + let entry: unknown; + try { + entry = Reflect.get(record, key); + } catch { + state.unreadablePaths.push(childPath(path, key)); + entry = null; + } + return [key, normalizeForStableHash(entry, state, childPath(path, key))] as const; + }); + state.seen.delete(value); + return Object.fromEntries(entries); +} + +export function stableHash(value: unknown) { + const state: StableHashState = { + circularPaths: [], + unreadablePaths: [], + seen: new WeakSet(), + }; + const normalized = normalizeForStableHash(value, state, "$"); + const payload = + state.circularPaths.length || state.unreadablePaths.length + ? { + normalized, + circularPaths: state.circularPaths, + unreadablePaths: state.unreadablePaths, + } + : normalized; + return sha256(JSON.stringify(payload) ?? "null"); +}