From bd04d2db0d672bf373d2c803a93c1def179db37c Mon Sep 17 00:00:00 2001 From: Dallin Romney Date: Fri, 29 May 2026 17:18:35 -0700 Subject: [PATCH] feat: only include the current changelog section in tarball (#88107) * build: package current changelog section * build: guard packaged changelog section size --- package.json | 1 + scripts/openclaw-prepack.ts | 2 + scripts/package-changelog.mjs | 156 ++++++++++++++++ scripts/package-openclaw-for-docker.mjs | 54 ++++-- test/scripts/package-changelog.test.ts | 170 ++++++++++++++++++ .../package-openclaw-for-docker.test.ts | 51 ++++++ 6 files changed, 420 insertions(+), 14 deletions(-) create mode 100644 scripts/package-changelog.mjs create mode 100644 test/scripts/package-changelog.test.ts diff --git a/package.json b/package.json index e5ac3c535f62..6d7bc6f283e6 100644 --- a/package.json +++ b/package.json @@ -1603,6 +1603,7 @@ "plugins:sync": "node --import tsx scripts/sync-plugin-versions.ts", "plugins:sync:check": "node --import tsx scripts/sync-plugin-versions.ts --check", "postinstall": "node scripts/postinstall-bundled-plugins.mjs", + "postpack": "node scripts/package-changelog.mjs restore", "preinstall": "node scripts/preinstall-package-manager-warning.mjs", "prepack": "node --import tsx scripts/openclaw-prepack.ts", "prepare": "node scripts/prepare-git-hooks.mjs", diff --git a/scripts/openclaw-prepack.ts b/scripts/openclaw-prepack.ts index 8da9f8d7249c..59bb8cc8176c 100644 --- a/scripts/openclaw-prepack.ts +++ b/scripts/openclaw-prepack.ts @@ -5,6 +5,7 @@ import { existsSync, readdirSync } from "node:fs"; import { pathToFileURL } from "node:url"; import { formatErrorMessage } from "../src/infra/errors.ts"; import { writePackageDistInventory } from "../src/infra/package-dist-inventory.ts"; +import { preparePackageChangelog } from "./package-changelog.mjs"; import { createPnpmRunnerSpawnSpec } from "./pnpm-runner.mjs"; const requiredPreparedPathGroups = [ ["dist/index.js", "dist/index.mjs"], @@ -158,6 +159,7 @@ async function main(): Promise { ensurePreparedArtifacts(); await writeDistInventory(); runBuildSmoke(); + await preparePackageChangelog(); } if (import.meta.url === pathToFileURL(process.argv[1] ?? "").href) { diff --git a/scripts/package-changelog.mjs b/scripts/package-changelog.mjs new file mode 100644 index 000000000000..a72c6115e4a2 --- /dev/null +++ b/scripts/package-changelog.mjs @@ -0,0 +1,156 @@ +#!/usr/bin/env node + +import { existsSync } from "node:fs"; +import { mkdir, readFile, rm, writeFile } from "node:fs/promises"; +import path from "node:path"; +import { fileURLToPath } from "node:url"; + +const CHANGELOG_PATH = "CHANGELOG.md"; +const PACKAGE_JSON_PATH = "package.json"; +const BACKUP_PATH = path.join(".artifacts", "package-changelog", "CHANGELOG.md.prepack-backup"); +const MAX_PACKAGED_CHANGELOG_BYTES = 500 * 1024; +const MIN_RELEASE_SECTION_BODY_BYTES = 32; +const UNRELEASED_HEADING = "Unreleased"; +const RELEASE_HEADING_PATTERN = + /^##\s+([0-9]{4}\.[1-9][0-9]*\.[1-9][0-9]*(?:(?:-(?:alpha|beta)\.[1-9][0-9]*)|(?:-[1-9][0-9]*))?)(?:\s+.*)?$/u; +const RELEASE_VERSION_PATTERN = + /^([0-9]{4}\.[1-9][0-9]*\.[1-9][0-9]*)(?:(?:-(?:alpha|beta)\.[1-9][0-9]*)|(?:-[1-9][0-9]*))?$/u; +const PRERELEASE_VERSION_PATTERN = + /^([0-9]{4}\.[1-9][0-9]*\.[1-9][0-9]*)-(?:alpha|beta)\.[1-9][0-9]*$/u; + +export function resolvePackageChangelogVersions(packageVersion) { + const match = RELEASE_VERSION_PATTERN.exec(packageVersion); + if (!match) { + throw new Error( + `Unsupported OpenClaw package version for changelog packaging: ${packageVersion}`, + ); + } + if (PRERELEASE_VERSION_PATTERN.test(packageVersion)) { + return [packageVersion, match[1], UNRELEASED_HEADING]; + } + return [packageVersion]; +} + +function splitLines(content) { + return content.replace(/^\uFEFF/u, "").split(/\r?\n/u); +} + +function parseLevelTwoHeading(line) { + const releaseMatch = RELEASE_HEADING_PATTERN.exec(line); + if (releaseMatch) { + return releaseMatch[1]; + } + return /^##\s+Unreleased(?:\s+.*)?$/u.test(line) ? UNRELEASED_HEADING : null; +} + +function findLevelTwoHeadings(lines) { + return lines.flatMap((line, index) => { + const version = parseLevelTwoHeading(line); + return version ? [{ index, version }] : []; + }); +} + +function extractPreamble(lines, firstHeadingIndex) { + return lines.slice(0, firstHeadingIndex).join("\n").trimEnd(); +} + +export function extractCurrentPackageChangelog(content, packageVersion) { + const targetVersions = resolvePackageChangelogVersions(packageVersion); + const lines = splitLines(content); + const headings = findLevelTwoHeadings(lines); + const heading = targetVersions + .map((version) => headings.find((entry) => entry.version === version)) + .find((entry) => entry !== undefined); + if (!heading) { + throw new Error( + `CHANGELOG.md does not contain a release section for ${targetVersions.join(" or ")}.`, + ); + } + const nextHeading = headings.find((entry) => entry.index > heading.index); + const firstLevelTwoHeadingIndex = lines.findIndex((line) => line.startsWith("## ")); + const preamble = extractPreamble(lines, firstLevelTwoHeadingIndex); + const releaseSection = lines + .slice(heading.index, nextHeading?.index ?? lines.length) + .join("\n") + .trimEnd(); + const releaseBody = releaseSection.split(/\r?\n/u).slice(1).join("\n").trim(); + const releaseBodyBytes = Buffer.byteLength(releaseBody, "utf8"); + if (releaseBodyBytes < MIN_RELEASE_SECTION_BODY_BYTES) { + throw new Error( + `Packaged changelog section for ${heading.version} is only ${releaseBodyBytes} body bytes, which is below the ${MIN_RELEASE_SECTION_BODY_BYTES} byte safety minimum.`, + ); + } + const packaged = `${preamble}\n\n${releaseSection}\n`; + const packagedBytes = Buffer.byteLength(packaged, "utf8"); + if (packagedBytes > MAX_PACKAGED_CHANGELOG_BYTES) { + throw new Error( + `Packaged changelog is ${packagedBytes} bytes, which exceeds the ${MAX_PACKAGED_CHANGELOG_BYTES} byte safety limit.`, + ); + } + return packaged; +} + +async function readPackageVersion(cwd) { + const packageJsonPath = path.join(cwd, PACKAGE_JSON_PATH); + const packageJson = JSON.parse(await readFile(packageJsonPath, "utf8")); + if (typeof packageJson.version !== "string") { + throw new Error("package.json version must be a string."); + } + return packageJson.version; +} + +export async function restorePackageChangelog(cwd = process.cwd()) { + const backupPath = path.join(cwd, BACKUP_PATH); + if (!existsSync(backupPath)) { + return false; + } + const changelogPath = path.join(cwd, CHANGELOG_PATH); + const backup = await readFile(backupPath, "utf8"); + await writeFile(changelogPath, backup, "utf8"); + await rm(backupPath, { force: true }); + return true; +} + +export async function preparePackageChangelog(cwd = process.cwd()) { + await restorePackageChangelog(cwd); + const changelogPath = path.join(cwd, CHANGELOG_PATH); + const backupPath = path.join(cwd, BACKUP_PATH); + const original = await readFile(changelogPath, "utf8"); + const packageVersion = await readPackageVersion(cwd); + const packaged = extractCurrentPackageChangelog(original, packageVersion); + if (packaged === original) { + return false; + } + await mkdir(path.dirname(backupPath), { recursive: true }); + await writeFile(backupPath, original, "utf8"); + await writeFile(changelogPath, packaged, "utf8"); + return true; +} + +async function main(argv = process.argv.slice(2)) { + const command = argv[0]; + if (command === "prepare") { + const changed = await preparePackageChangelog(); + console.error( + changed + ? "package-changelog: wrote current release notes for package tarball." + : "package-changelog: source changelog already matches package notes.", + ); + return; + } + if (command === "restore") { + const restored = await restorePackageChangelog(); + console.error( + restored + ? "package-changelog: restored source CHANGELOG.md." + : "package-changelog: no packaged changelog backup to restore.", + ); + return; + } + console.error("Usage: node scripts/package-changelog.mjs "); + process.exitCode = 1; +} + +if (process.argv[1] && path.resolve(process.argv[1]) === fileURLToPath(import.meta.url)) { + await main(); +} diff --git a/scripts/package-openclaw-for-docker.mjs b/scripts/package-openclaw-for-docker.mjs index 20a7a0c09119..4183a6eb5d30 100644 --- a/scripts/package-openclaw-for-docker.mjs +++ b/scripts/package-openclaw-for-docker.mjs @@ -6,6 +6,7 @@ import { spawn } from "node:child_process"; import fs from "node:fs/promises"; import path from "node:path"; import { fileURLToPath } from "node:url"; +import { preparePackageChangelog, restorePackageChangelog } from "./package-changelog.mjs"; const ROOT_DIR = path.resolve(path.dirname(fileURLToPath(import.meta.url)), ".."); const DEFAULT_PACKAGE_BUILD_TIMEOUT_MS = 45 * 60 * 1000; @@ -22,6 +23,13 @@ const SIGNAL_EXIT_CODES = { }; let forwardedSignalExitCode; +class ForwardedSignalExitError extends Error { + constructor(exitCode) { + super(`forwarded signal requested exit ${exitCode}`); + this.exitCode = exitCode; + } +} + for (const signal of Object.keys(SIGNAL_EXIT_CODES)) { process.on(signal, () => { forwardedSignalExitCode ??= SIGNAL_EXIT_CODES[signal]; @@ -112,6 +120,10 @@ function run(command, args, cwd, options = {}) { } ACTIVE_CHILD_KILLERS.delete(killChild); if (forwardedSignalExitCode !== undefined && ACTIVE_CHILD_KILLERS.size === 0) { + if (options.deferForwardedSignalExit) { + reject(new ForwardedSignalExitError(forwardedSignalExitCode)); + return; + } process.exit(forwardedSignalExitCode); } if (error) { @@ -245,6 +257,32 @@ async function newestOpenClawTarball(outputDir, packOutput) { return path.join(outputDir, packed); } +export async function packOpenClawPackageForDocker(sourceDir, outputDir, options = {}) { + const runCaptureImpl = options.runCaptureImpl ?? runCapture; + const prepareChangelog = options.prepareChangelog ?? preparePackageChangelog; + const restoreChangelog = options.restoreChangelog ?? restorePackageChangelog; + console.error("==> Packing OpenClaw package"); + await prepareChangelog(sourceDir); + let packOutput = ""; + try { + packOutput = await runCaptureImpl( + "npm", + ["pack", "--silent", "--ignore-scripts", "--pack-destination", outputDir], + sourceDir, + { + deferForwardedSignalExit: true, + timeoutMs: resolveTimeoutMs( + "OPENCLAW_DOCKER_PACKAGE_PACK_TIMEOUT_MS", + DEFAULT_PACKAGE_PACK_TIMEOUT_MS, + ), + }, + ); + } finally { + await restoreChangelog(sourceDir); + } + return await newestOpenClawTarball(outputDir, packOutput); +} + async function main() { const options = parseArgs(process.argv.slice(2)); const sourceDir = path.resolve(ROOT_DIR, options.sourceDir || ROOT_DIR); @@ -277,19 +315,7 @@ async function main() { }, ); - console.error("==> Packing OpenClaw package"); - const packOutput = await runCapture( - "npm", - ["pack", "--silent", "--ignore-scripts", "--pack-destination", outputDir], - sourceDir, - { - timeoutMs: resolveTimeoutMs( - "OPENCLAW_DOCKER_PACKAGE_PACK_TIMEOUT_MS", - DEFAULT_PACKAGE_PACK_TIMEOUT_MS, - ), - }, - ); - let tarball = await newestOpenClawTarball(outputDir, packOutput); + let tarball = await packOpenClawPackageForDocker(sourceDir, outputDir); if (options.outputName) { const target = path.join(outputDir, options.outputName); @@ -323,6 +349,6 @@ async function main() { if (process.argv[1] && path.resolve(process.argv[1]) === fileURLToPath(import.meta.url)) { await main().catch((error) => { console.error(error instanceof Error ? error.message : String(error)); - process.exit(1); + process.exit(Number.isInteger(error?.exitCode) ? error.exitCode : 1); }); } diff --git a/test/scripts/package-changelog.test.ts b/test/scripts/package-changelog.test.ts new file mode 100644 index 000000000000..a6b1a0224e09 --- /dev/null +++ b/test/scripts/package-changelog.test.ts @@ -0,0 +1,170 @@ +import { mkdtempSync, readFileSync, rmSync, writeFileSync } from "node:fs"; +import os from "node:os"; +import path from "node:path"; +import { describe, expect, it } from "vitest"; +import { + extractCurrentPackageChangelog, + preparePackageChangelog, + resolvePackageChangelogVersions, + restorePackageChangelog, +} from "../../scripts/package-changelog.mjs"; + +function changelog(strings: TemplateStringsArray, ...values: string[]) { + return `${String.raw({ raw: strings }, ...values) + .replace(/^\n/u, "") + .trimEnd()}\n`; +} + +const cumulativeChangelog = changelog` +# Changelog +Docs: https://docs.openclaw.ai +## Unreleased +### Fixes +- Pending note. +## 2026.5.28 +### Highlights +- Current highlight. +### Changes +- Current change. +### Fixes +- Current fix. +## 2026.5.27 +### Highlights +- Older highlight. +`; + +describe("package-changelog", () => { + it("maps release-channel package versions to package changelog candidate headings", () => { + expect(resolvePackageChangelogVersions("2026.5.28")).toEqual(["2026.5.28"]); + expect(resolvePackageChangelogVersions("2026.5.28-1")).toEqual(["2026.5.28-1"]); + expect(resolvePackageChangelogVersions("2026.5.28-beta.1")).toEqual([ + "2026.5.28-beta.1", + "2026.5.28", + "Unreleased", + ]); + expect(resolvePackageChangelogVersions("2026.5.28-alpha.2")).toEqual([ + "2026.5.28-alpha.2", + "2026.5.28", + "Unreleased", + ]); + }); + + it("extracts only the package version stable release section", () => { + expect(extractCurrentPackageChangelog(cumulativeChangelog, "2026.5.28-beta.1")).toBe( + changelog` +# Changelog +Docs: https://docs.openclaw.ai + +## 2026.5.28 +### Highlights +- Current highlight. +### Changes +- Current change. +### Fixes +- Current fix. +`, + ); + }); + + it("prefers an exact prerelease section when it exists", () => { + const source = changelog` +# Changelog +## 2026.5.28-beta.2 +- Beta 2 package notes with enough release detail. +## 2026.5.28 +- Stable. +`; + + expect(extractCurrentPackageChangelog(source, "2026.5.28-beta.2")).toBe(changelog` +# Changelog + +## 2026.5.28-beta.2 +- Beta 2 package notes with enough release detail. +`); + }); + + it("uses Unreleased only as a prerelease fallback when no release heading exists", () => { + const source = changelog` +# Changelog +## Unreleased +- Pending beta package notes with enough release detail. +## 2026.5.27 +- Older stable. +`; + + expect(extractCurrentPackageChangelog(source, "2026.5.28-beta.1")).toBe(changelog` +# Changelog + +## Unreleased +- Pending beta package notes with enough release detail. +`); + }); + + it("extracts exact correction release sections", () => { + const source = changelog` +# Changelog +## 2026.5.28-1 +- Correction release notes with enough detail. +## 2026.5.28 +- Stable. +`; + + expect(extractCurrentPackageChangelog(source, "2026.5.28-1")).toBe(changelog` +# Changelog + +## 2026.5.28-1 +- Correction release notes with enough detail. +`); + }); + + it("fails closed when package version has no matching release section", () => { + expect(() => extractCurrentPackageChangelog(cumulativeChangelog, "2026.5.29")).toThrow( + "CHANGELOG.md does not contain a release section for 2026.5.29.", + ); + }); + + it("fails closed when the packaged changelog is unexpectedly large", () => { + const source = changelog` +# Changelog +## 2026.5.28 +${"é".repeat(260_000)} +`; + + expect(() => extractCurrentPackageChangelog(source, "2026.5.28")).toThrow( + "exceeds the 512000 byte safety limit", + ); + }); + + it("fails closed when the extracted release section is effectively empty", () => { + const source = changelog` +# Changelog +Docs: https://docs.openclaw.ai +## 2026.5.28 +### Fixes +## 2026.5.27 +- Older stable release notes with enough detail. +`; + + expect(() => extractCurrentPackageChangelog(source, "2026.5.28")).toThrow( + "below the 32 byte safety minimum", + ); + }); + + it("prepares and restores the packaged changelog without changing the source permanently", async () => { + const root = mkdtempSync(path.join(os.tmpdir(), "openclaw-package-changelog-")); + try { + writeFileSync(path.join(root, "package.json"), '{"version":"2026.5.28-beta.1"}\n', "utf8"); + writeFileSync(path.join(root, "CHANGELOG.md"), cumulativeChangelog, "utf8"); + + await expect(preparePackageChangelog(root)).resolves.toBe(true); + expect(readFileSync(path.join(root, "CHANGELOG.md"), "utf8")).not.toContain("## Unreleased"); + expect(readFileSync(path.join(root, "CHANGELOG.md"), "utf8")).not.toContain("## 2026.5.27"); + expect(readFileSync(path.join(root, "CHANGELOG.md"), "utf8")).toContain("## 2026.5.28"); + + await expect(restorePackageChangelog(root)).resolves.toBe(true); + expect(readFileSync(path.join(root, "CHANGELOG.md"), "utf8")).toBe(cumulativeChangelog); + } finally { + rmSync(root, { recursive: true, force: true }); + } + }); +}); diff --git a/test/scripts/package-openclaw-for-docker.test.ts b/test/scripts/package-openclaw-for-docker.test.ts index dc851dd34217..28d24af300c6 100644 --- a/test/scripts/package-openclaw-for-docker.test.ts +++ b/test/scripts/package-openclaw-for-docker.test.ts @@ -6,6 +6,7 @@ import { pathToFileURL } from "node:url"; import { describe, expect, it } from "vitest"; import { buildPackageArtifacts, + packOpenClawPackageForDocker, runCommandForTest, } from "../../scripts/package-openclaw-for-docker.mjs"; @@ -115,6 +116,56 @@ describe("package-openclaw-for-docker", () => { ]); }); + it("trims and restores the changelog around ignore-scripts package artifacts", async () => { + const calls: string[] = []; + const tarball = await packOpenClawPackageForDocker("/repo", "/out", { + prepareChangelog: async (cwd: string) => { + calls.push(`prepare:${cwd}`); + }, + restoreChangelog: async (cwd: string) => { + calls.push(`restore:${cwd}`); + }, + runCaptureImpl: async ( + command: string, + args: string[], + cwd: string, + options: { deferForwardedSignalExit?: boolean }, + ) => { + calls.push(`${command}:${args.join(" ")}:${cwd}`); + expect(options.deferForwardedSignalExit).toBe(true); + return "openclaw-2026.5.28.tgz\n"; + }, + }); + + expect(tarball).toBe(path.join("/out", "openclaw-2026.5.28.tgz")); + expect(calls).toEqual([ + "prepare:/repo", + "npm:pack --silent --ignore-scripts --pack-destination /out:/repo", + "restore:/repo", + ]); + }); + + it("restores the changelog when ignore-scripts packaging fails", async () => { + const calls: string[] = []; + + await expect( + packOpenClawPackageForDocker("/repo", "/out", { + prepareChangelog: async (cwd: string) => { + calls.push(`prepare:${cwd}`); + }, + restoreChangelog: async (cwd: string) => { + calls.push(`restore:${cwd}`); + }, + runCaptureImpl: async () => { + calls.push("pack"); + throw new Error("pack failed"); + }, + }), + ).rejects.toThrow("pack failed"); + + expect(calls).toEqual(["prepare:/repo", "pack", "restore:/repo"]); + }); + it("kills timed-out child process groups", async () => { if (process.platform === "win32") { return;