diff --git a/.github/package-trusted-sources.json b/.github/package-trusted-sources.json new file mode 100644 index 000000000000..18b4090797e0 --- /dev/null +++ b/.github/package-trusted-sources.json @@ -0,0 +1,4 @@ +{ + "schemaVersion": 1, + "sources": {} +} diff --git a/.github/workflows/package-acceptance.yml b/.github/workflows/package-acceptance.yml index c234f9b608fe..2cf6b38da72e 100644 --- a/.github/workflows/package-acceptance.yml +++ b/.github/workflows/package-acceptance.yml @@ -17,6 +17,7 @@ on: - npm - ref - url + - trusted-url - artifact package_ref: description: Trusted package source ref when source=ref @@ -29,12 +30,17 @@ on: default: openclaw@beta type: string package_url: - description: HTTPS .tgz URL when source=url + description: HTTPS .tgz URL when source=url or source=trusted-url required: false default: "" type: string package_sha256: - description: Expected package SHA-256; required for source=url + description: Expected package SHA-256; required for source=url or source=trusted-url + required: false + default: "" + type: string + trusted_source_id: + description: Named trusted source policy when source=trusted-url required: false default: "" type: string @@ -111,7 +117,7 @@ on: default: main type: string source: - description: "Package candidate source: npm, ref, url, or artifact" + description: "Package candidate source: npm, ref, url, trusted-url, or artifact" required: true type: string package_ref: @@ -125,12 +131,17 @@ on: default: openclaw@beta type: string package_url: - description: HTTPS .tgz URL when source=url + description: HTTPS .tgz URL when source=url or source=trusted-url required: false default: "" type: string package_sha256: - description: Expected package SHA-256; required for source=url + description: Expected package SHA-256; required for source=url or source=trusted-url + required: false + default: "" + type: string + trusted_source_id: + description: Named trusted source policy when source=trusted-url required: false default: "" type: string @@ -180,6 +191,8 @@ on: default: "" type: string secrets: + OPENCLAW_TRUSTED_PACKAGE_TOKEN: + required: false OPENAI_API_KEY: required: false OPENAI_BASE_URL: @@ -355,6 +368,8 @@ jobs: PACKAGE_SPEC: ${{ inputs.package_spec }} PACKAGE_URL: ${{ inputs.package_url }} PACKAGE_SHA256: ${{ inputs.package_sha256 }} + TRUSTED_SOURCE_ID: ${{ inputs.trusted_source_id }} + OPENCLAW_TRUSTED_PACKAGE_TOKEN: ${{ secrets.OPENCLAW_TRUSTED_PACKAGE_TOKEN }} shell: bash run: | set -euo pipefail @@ -369,6 +384,7 @@ jobs: --package-spec "$PACKAGE_SPEC" \ --package-url "$PACKAGE_URL" \ --package-sha256 "$PACKAGE_SHA256" \ + --trusted-source-id "$TRUSTED_SOURCE_ID" \ --artifact-dir "${artifact_dir:-.}" \ --output-dir .artifacts/docker-e2e-package \ --output-name openclaw-current.tgz \ @@ -490,6 +506,7 @@ jobs: PACKAGE_SHA256: ${{ steps.resolve.outputs.sha256 }} PACKAGE_VERSION: ${{ steps.resolve.outputs.package_version }} PACKAGE_REF: ${{ inputs.package_ref }} + TRUSTED_SOURCE_ID: ${{ inputs.trusted_source_id }} SOURCE: ${{ inputs.source }} SUITE_PROFILE: ${{ inputs.suite_profile }} WORKFLOW_REF: ${{ inputs.workflow_ref }} @@ -506,6 +523,9 @@ jobs: if [[ "${SOURCE}" == "ref" ]]; then echo "- Package ref: \`${PACKAGE_REF}\`" fi + if [[ "${SOURCE}" == "trusted-url" ]]; then + echo "- Trusted source: \`${TRUSTED_SOURCE_ID}\`" + fi echo "- Version: \`${PACKAGE_VERSION}\`" echo "- SHA-256: \`${PACKAGE_SHA256}\`" echo "- Profile: \`${SUITE_PROFILE}\`" diff --git a/docs/ci.md b/docs/ci.md index 497528ebcd1e..c042d69b747d 100644 --- a/docs/ci.md +++ b/docs/ci.md @@ -278,7 +278,8 @@ Use `Package Acceptance` when the question is "does this installable OpenClaw pa - `source=npm` accepts only `openclaw@beta`, `openclaw@latest`, or an exact OpenClaw release version such as `openclaw@2026.4.27-beta.2`. Use this for published prerelease/stable acceptance. - `source=ref` packs a trusted `package_ref` branch, tag, or full commit SHA. The resolver fetches OpenClaw branches/tags, verifies the selected commit is reachable from repository branch history or a release tag, installs deps in a detached worktree, and packs it with `scripts/package-openclaw-for-docker.mjs`. -- `source=url` downloads an HTTPS `.tgz`; `package_sha256` is required. +- `source=url` downloads a public HTTPS `.tgz`; `package_sha256` is required. This path rejects URL credentials, non-default HTTPS ports, private/internal/special-use hostnames or resolved IPs, and redirects outside the same public safety policy. +- `source=trusted-url` downloads an HTTPS `.tgz` from a named trusted-source policy in `.github/package-trusted-sources.json`; `package_sha256` and `trusted_source_id` are required. Use this only for maintainer-owned enterprise mirrors or private package repositories that need configured hosts, ports, path prefixes, redirect hosts, or private-network resolution. If the policy declares bearer auth, the workflow uses the fixed `OPENCLAW_TRUSTED_PACKAGE_TOKEN` secret; URL-embedded credentials are still rejected. - `source=artifact` downloads one `.tgz` from `artifact_run_id` and `artifact_name`; `package_sha256` is optional but should be supplied for externally shared artifacts. Keep `workflow_ref` and `package_ref` separate. `workflow_ref` is the trusted workflow/harness code that runs the test. `package_ref` is the source commit that gets packed when `source=ref`. This lets the current test harness validate older trusted source commits without running old workflow logic. @@ -341,6 +342,16 @@ gh workflow run package-acceptance.yml \ -f package_sha256=<64-char-sha256> \ -f suite_profile=smoke +# Validate a tarball from a named trusted private mirror policy. +gh workflow run package-acceptance.yml \ + --ref main \ + -f workflow_ref=main \ + -f source=trusted-url \ + -f trusted_source_id=enterprise-artifactory \ + -f package_url=https://packages.example.internal:8443/artifactory/openclaw/openclaw-current.tgz \ + -f package_sha256=<64-char-sha256> \ + -f suite_profile=smoke + # Reuse a tarball uploaded by another Actions run. gh workflow run package-acceptance.yml \ --ref main \ diff --git a/docs/help/testing-updates-plugins.md b/docs/help/testing-updates-plugins.md index dda905dba304..16f898e8fdaa 100644 --- a/docs/help/testing-updates-plugins.md +++ b/docs/help/testing-updates-plugins.md @@ -162,7 +162,15 @@ Candidate sources: published version. - `source=ref`: pack a trusted branch, tag, or commit with the selected current harness. -- `source=url`: validate an HTTPS tarball with required `package_sha256`. +- `source=url`: validate a public HTTPS tarball with required `package_sha256`. + This path rejects URL credentials, non-default HTTPS ports, private/internal + hostnames or DNS/IP results, special-use IP space, and unsafe redirects. +- `source=trusted-url`: validate an HTTPS tarball with required + `package_sha256` and `trusted_source_id` against the maintainer-owned policy + in `.github/package-trusted-sources.json`. Use this for enterprise/private + mirrors instead of weakening `source=url` with an input-level allow-private + switch. Bearer auth, when configured by policy, uses the fixed + `OPENCLAW_TRUSTED_PACKAGE_TOKEN` secret. - `source=artifact`: reuse a tarball uploaded by another Actions run. Full Release Validation uses `source=artifact` by default, built from the diff --git a/docs/help/testing.md b/docs/help/testing.md index 7ba6225fb104..d5b1bb78ec6b 100644 --- a/docs/help/testing.md +++ b/docs/help/testing.md @@ -242,7 +242,7 @@ gh workflow run package-acceptance.yml --ref main \ -f telegram_mode=mock-openai ``` -- Exact tarball URL proof requires a digest: +- Exact tarball URL proof requires a digest and uses the public URL safety policy: ```bash gh workflow run package-acceptance.yml --ref main \ @@ -252,6 +252,19 @@ gh workflow run package-acceptance.yml --ref main \ -f suite_profile=package ``` +- Enterprise/private tarball mirrors use an explicit trusted-source policy: + +```bash +gh workflow run package-acceptance.yml --ref main \ + -f source=trusted-url \ + -f trusted_source_id=enterprise-artifactory \ + -f package_url=https://packages.example.internal:8443/artifactory/openclaw/openclaw-VERSION.tgz \ + -f package_sha256= \ + -f suite_profile=package +``` + +`source=trusted-url` reads `.github/package-trusted-sources.json` from the trusted workflow ref and does not accept URL credentials or a workflow-input private-network bypass. If the named policy declares bearer auth, configure the fixed `OPENCLAW_TRUSTED_PACKAGE_TOKEN` secret. + - Artifact proof downloads a tarball artifact from another Actions run: ```bash diff --git a/docs/reference/RELEASING.md b/docs/reference/RELEASING.md index 1d55d41bc393..ef67ba8ea1a0 100644 --- a/docs/reference/RELEASING.md +++ b/docs/reference/RELEASING.md @@ -164,9 +164,11 @@ vYYYY.M.D-beta.N` from the matching `release/YYYY.M.D` branch. The helper runs for a package candidate while release work continues. Use `source=npm` for `openclaw@beta`, `openclaw@latest`, or an exact release version; `source=ref` to pack a trusted `package_ref` branch/tag/SHA with the current - `workflow_ref` harness; `source=url` for an HTTPS tarball with a required - SHA-256; or `source=artifact` for a tarball uploaded by another GitHub - Actions run. The workflow resolves the candidate to + `workflow_ref` harness; `source=url` for a public HTTPS tarball with a + required SHA-256 and strict public URL policy; `source=trusted-url` for a + named trusted-source policy using required `trusted_source_id` and SHA-256; or + `source=artifact` for a tarball uploaded by another GitHub Actions run. The + workflow resolves the candidate to `package-under-test`, reuses the Docker E2E release scheduler against that tarball, and can run Telegram QA against the same tarball with `telegram_mode=mock-openai` or `telegram_mode=live-frontier`. When the @@ -550,7 +552,14 @@ Supported candidate sources: version - `source=ref`: pack a trusted `package_ref` branch, tag, or full commit SHA with the selected `workflow_ref` harness -- `source=url`: download an HTTPS `.tgz` with required `package_sha256` +- `source=url`: download a public HTTPS `.tgz` with required `package_sha256`; + URL credentials, non-default HTTPS ports, private/internal/special-use + hostnames or resolved addresses, and unsafe redirects are rejected +- `source=trusted-url`: download an HTTPS `.tgz` with required + `package_sha256` and `trusted_source_id` from a named policy in + `.github/package-trusted-sources.json`; use this for maintainer-owned + enterprise mirrors or private package repositories instead of adding an + input-level private-network bypass to `source=url` - `source=artifact`: reuse a `.tgz` uploaded by another GitHub Actions run `OpenClaw Release Checks` runs Package Acceptance with `source=artifact`, the @@ -563,9 +572,11 @@ tarball. Blocking release checks use the default latest published package baseline; `run_release_soak=true` or `release_profile=full` expands to every stable npm-published baseline from `2026.4.23` through `latest` plus reported-issue fixtures. Use -Package Acceptance with `source=npm` for an already shipped candidate, or -`source=ref`/`source=artifact` for a SHA-backed local npm tarball before -publish. It is the GitHub-native +Package Acceptance with `source=npm` for an already shipped candidate, +`source=ref` for a SHA-backed local npm tarball before publish, +`source=trusted-url` for a maintainer-owned enterprise/private mirror, or +`source=artifact` for a prepared tarball uploaded by another GitHub Actions run. +It is the GitHub-native replacement for most of the package/update coverage that previously required Parallels. Cross-OS release checks still matter for OS-specific onboarding, installer, and platform behavior, but package/update product validation should diff --git a/extensions/anthropic/provider-policy-api.test.ts b/extensions/anthropic/provider-policy-api.test.ts index d8f2cf645ac9..7b0226aecd08 100644 --- a/extensions/anthropic/provider-policy-api.test.ts +++ b/extensions/anthropic/provider-policy-api.test.ts @@ -98,6 +98,32 @@ describe("anthropic provider policy public artifact", () => { expect(nextConfig.agents?.defaults?.contextPruning?.ttl).toBe("1h"); }); + it("adds cacheRetention defaults for dated Anthropic primary model refs", () => { + const nextConfig = applyConfigDefaults({ + config: { + auth: { + profiles: { + "anthropic:default": { + provider: "anthropic", + mode: "api_key", + }, + }, + }, + agents: { + defaults: { + model: { primary: "anthropic/claude-sonnet-4-20250514" }, + }, + }, + }, + env: {}, + }); + + expect( + nextConfig.agents?.defaults?.models?.["anthropic/claude-sonnet-4-20250514"]?.params + ?.cacheRetention, + ).toBe("short"); + }); + it("exposes Claude Opus 4.7 thinking levels without loading the full provider plugin", () => { const profile = resolveThinkingProfile({ provider: "anthropic", diff --git a/scripts/resolve-openclaw-package-candidate.mjs b/scripts/resolve-openclaw-package-candidate.mjs index e2184c3918ec..a331a8590914 100644 --- a/scripts/resolve-openclaw-package-candidate.mjs +++ b/scripts/resolve-openclaw-package-candidate.mjs @@ -2,8 +2,12 @@ // Normalizes package-acceptance inputs into the tarball shape consumed by Docker E2E. import { spawn } from "node:child_process"; import { createHash } from "node:crypto"; +import { lookup as dnsLookupCb } from "node:dns"; +import { lookup as dnsLookup } from "node:dns/promises"; import { createWriteStream } from "node:fs"; import fs from "node:fs/promises"; +import { request as httpsRequest } from "node:https"; +import { isIP } from "node:net"; import os from "node:os"; import path from "node:path"; import { pipeline } from "node:stream/promises"; @@ -11,17 +15,30 @@ import { fileURLToPath } from "node:url"; const ROOT_DIR = path.resolve(path.dirname(fileURLToPath(import.meta.url)), ".."); const DEFAULT_OUTPUT_NAME = "openclaw-current.tgz"; +const PACKAGE_URL_DOWNLOAD_TIMEOUT_MS = 60_000; +const PACKAGE_URL_MAX_BYTES = 250 * 1024 * 1024; +const PACKAGE_URL_MAX_REDIRECTS = 5; +const TRUSTED_PACKAGE_SOURCE_POLICY = ".github/package-trusted-sources.json"; +const TRUSTED_PACKAGE_SOURCE_TOKEN_ENV = "OPENCLAW_TRUSTED_PACKAGE_TOKEN"; +const BLOCKED_PACKAGE_HOSTNAMES = new Set([ + "localhost", + "localhost.localdomain", + "metadata.google.internal", +]); export const OPENCLAW_PACKAGE_SPEC_RE = /^openclaw@(alpha|beta|latest|[0-9]{4}\.[1-9][0-9]*\.[1-9][0-9]*(-[1-9][0-9]*|-(alpha|beta)\.[1-9][0-9]*)?)$/u; function usage() { - return `Usage: node scripts/resolve-openclaw-package-candidate.mjs --source --output-dir [options] + return `Usage: node scripts/resolve-openclaw-package-candidate.mjs --source --output-dir [options] Options: --package-spec Published npm spec for source=npm. --package-ref Trusted repo ref for source=ref. - --package-url HTTPS tarball URL for source=url. - --package-sha256 Expected tarball SHA-256 for source=url or source=artifact. + --package-url HTTPS tarball URL for source=url or source=trusted-url. + --package-sha256 Expected tarball SHA-256 for source=url, source=trusted-url, or source=artifact. + --trusted-source-id Named trusted URL policy for source=trusted-url. + --trusted-source-policy + Repo-controlled trusted URL source policy. Default: ${TRUSTED_PACKAGE_SOURCE_POLICY} --artifact-dir Directory containing exactly one .tgz for source=artifact. --output-name Output tarball filename. Default: ${DEFAULT_OUTPUT_NAME} --metadata Write package metadata JSON. @@ -40,6 +57,8 @@ export function parseArgs(argv) { packageSpec: "", packageUrl: "", source: "", + trustedSourceId: "", + trustedSourcePolicy: TRUSTED_PACKAGE_SOURCE_POLICY, }; for (let index = 0; index < argv.length; index += 1) { const arg = argv[index]; @@ -70,6 +89,10 @@ export function parseArgs(argv) { options.packageUrl = readValue(arg); } else if (arg === "--source") { options.source = readValue(arg); + } else if (arg === "--trusted-source-id") { + options.trustedSourceId = readValue(arg); + } else if (arg === "--trusted-source-policy") { + options.trustedSourcePolicy = readValue(arg); } else if (arg === "--help" || arg === "-h") { options.help = true; } else { @@ -340,16 +363,583 @@ async function moveNewestPackedTarball(outputDir, packOutput, outputName) { return target; } -async function downloadUrl(url, target) { - const parsed = new URL(url); +function normalizeUrlHostname(hostname) { + return hostname.replace(/^\[/u, "").replace(/\]$/u, "").replace(/\.+$/u, "").toLowerCase(); +} + +function parseIpv4(address) { + const parts = address.split("."); + if (parts.length !== 4) { + return null; + } + const octets = parts.map((part) => Number(part)); + if (octets.some((octet) => !Number.isInteger(octet) || octet < 0 || octet > 255)) { + return null; + } + return octets; +} + +function ipv4ToInt(octets) { + return ((octets[0] << 24) >>> 0) + (octets[1] << 16) + (octets[2] << 8) + octets[3]; +} + +function ipv4InCidr(octets, base, bits) { + const mask = bits === 0 ? 0 : (0xffffffff << (32 - bits)) >>> 0; + return (ipv4ToInt(octets) & mask) === (ipv4ToInt(base) & mask); +} + +function isUnsafeIpv4(address) { + const octets = Array.isArray(address) ? address : parseIpv4(address); + if (!octets) { + return true; + } + return [ + [[0, 0, 0, 0], 8], + [[10, 0, 0, 0], 8], + [[100, 64, 0, 0], 10], + [[127, 0, 0, 0], 8], + [[169, 254, 0, 0], 16], + [[172, 16, 0, 0], 12], + [[192, 0, 0, 0], 24], + [[192, 0, 2, 0], 24], + [[192, 168, 0, 0], 16], + [[198, 18, 0, 0], 15], + [[198, 51, 100, 0], 24], + [[203, 0, 113, 0], 24], + [[224, 0, 0, 0], 4], + [[240, 0, 0, 0], 4], + ].some(([base, bits]) => ipv4InCidr(octets, base, bits)); +} + +function ipv4FromHextets(high, low) { + return [(high >>> 8) & 0xff, high & 0xff, (low >>> 8) & 0xff, low & 0xff]; +} + +function ipv4OctetsToHextets(octets) { + return [ + ((octets[0] << 8) | octets[1]).toString(16), + ((octets[2] << 8) | octets[3]).toString(16), + ]; +} + +function parseIpv6Parts(address) { + const normalized = address.toLowerCase().replace(/%[0-9a-z_.-]+$/u, ""); + const dottedIpv4 = normalized.match(/^(.*:)(\d{1,3}(?:\.\d{1,3}){3})$/u); + const dottedIpv4Octets = dottedIpv4 ? parseIpv4(dottedIpv4[2]) : null; + if (dottedIpv4 && !dottedIpv4Octets) { + return null; + } + const canonical = dottedIpv4 + ? `${dottedIpv4[1]}${ipv4OctetsToHextets(dottedIpv4Octets)[0]}:${ipv4OctetsToHextets(dottedIpv4Octets)[1]}` + : normalized; + if (canonical.includes(":::") || canonical.split("::").length > 2) { + return null; + } + const [leftRaw = "", rightRaw = ""] = canonical.split("::"); + const parseParts = (value) => { + if (!value) { + return []; + } + return value.split(":").map((part) => { + if (!/^[0-9a-f]{1,4}$/u.test(part)) { + return Number.NaN; + } + return Number.parseInt(part, 16); + }); + }; + const left = parseParts(leftRaw); + const right = parseParts(rightRaw); + if ([...left, ...right].some((part) => !Number.isInteger(part) || part < 0 || part > 0xffff)) { + return null; + } + const zeroCount = canonical.includes("::") ? 8 - left.length - right.length : 0; + if (zeroCount < 0 || (!canonical.includes("::") && left.length !== 8)) { + return null; + } + return [...left, ...Array.from({ length: zeroCount }, () => 0), ...right]; +} + +function extractUnsafeEmbeddedIpv4FromIpv6(address) { + const parts = parseIpv6Parts(address); + if (!parts || parts.length !== 8) { + return null; + } + const candidates = []; + if (parts.slice(0, 5).every((part) => part === 0) && parts[5] === 0xffff) { + candidates.push(ipv4FromHextets(parts[6], parts[7])); + } + if (parts.slice(0, 6).every((part) => part === 0)) { + candidates.push(ipv4FromHextets(parts[6], parts[7])); + } + if (parts[0] === 0x0064 && parts[1] === 0xff9b && parts.slice(2, 6).every((part) => part === 0)) { + candidates.push(ipv4FromHextets(parts[6], parts[7])); + } + if ( + parts[0] === 0x0064 && + parts[1] === 0xff9b && + parts[2] === 0x0001 && + parts.slice(3, 6).every((part) => part === 0) + ) { + candidates.push(ipv4FromHextets(parts[6], parts[7])); + } + if (parts[0] === 0x2002) { + candidates.push(ipv4FromHextets(parts[1], parts[2])); + } + if (parts[0] === 0x2001 && parts[1] === 0x0000) { + candidates.push(ipv4FromHextets(parts[6] ^ 0xffff, parts[7] ^ 0xffff)); + } + if ((parts[4] & 0xfcff) === 0 && parts[5] === 0x5efe) { + candidates.push(ipv4FromHextets(parts[6], parts[7])); + } + return candidates.find((candidate) => isUnsafeIpv4(candidate)) ?? null; +} + +function isUnsafeIpv6(address) { + const normalized = address.toLowerCase(); + if (extractUnsafeEmbeddedIpv4FromIpv6(normalized)) { + return true; + } + return ( + normalized === "::" || + normalized === "::1" || + normalized.startsWith("fc") || + normalized.startsWith("fd") || + /^fe[89ab]/u.test(normalized) || + normalized.startsWith("ff") || + normalized.startsWith("64:ff9b:") || + normalized.startsWith("100:") || + normalized.startsWith("2001:2:") || + normalized.startsWith("2001:db8:") + ); +} + +function isUnsafeIpAddress(address) { + const normalized = normalizeUrlHostname(address); + const family = isIP(normalized); + if (family === 4) { + return isUnsafeIpv4(normalized); + } + if (family === 6) { + return isUnsafeIpv6(normalized); + } + return true; +} + +function isBlockedPackageHostname(hostname) { + const normalized = normalizeUrlHostname(hostname); + return ( + BLOCKED_PACKAGE_HOSTNAMES.has(normalized) || + normalized.endsWith(".localhost") || + normalized.endsWith(".local") || + normalized.endsWith(".internal") || + (isIP(normalized) !== 0 && isUnsafeIpAddress(normalized)) + ); +} + +function packageUrlPort(parsed) { + return parsed.port ? Number(parsed.port) : 443; +} + +function toUniqueNormalizedHostList(value, field, sourceId) { + if (!Array.isArray(value) || value.length === 0) { + throw new Error(`trusted package source ${sourceId} must define non-empty ${field}`); + } + return [...new Set(value.map((entry) => normalizeUrlHostname(String(entry))).filter(Boolean))]; +} + +function toTrustedPorts(value, sourceId) { + const ports = value === undefined ? [443] : value; + if (!Array.isArray(ports) || ports.length === 0) { + throw new Error(`trusted package source ${sourceId} must define non-empty ports`); + } + const normalized = ports.map((port) => Number(port)); + if (normalized.some((port) => !Number.isInteger(port) || port < 1 || port > 65535)) { + throw new Error(`trusted package source ${sourceId} has invalid ports`); + } + return [...new Set(normalized)].toSorted((a, b) => a - b); +} + +function toPathPrefixes(value, sourceId) { + const prefixes = value === undefined ? ["/"] : value; + if (!Array.isArray(prefixes) || prefixes.length === 0) { + throw new Error(`trusted package source ${sourceId} must define non-empty pathPrefixes`); + } + return prefixes.map((prefix) => { + const text = String(prefix); + if (!text.startsWith("/")) { + throw new Error(`trusted package source ${sourceId} pathPrefixes must start with /`); + } + return text; + }); +} + +function normalizeTrustedPackageSource(id, raw) { + if (!/^[A-Za-z0-9][A-Za-z0-9._-]{0,127}$/u.test(id)) { + throw new Error(`Invalid trusted package source id: ${id}`); + } + if (!raw || typeof raw !== "object" || Array.isArray(raw)) { + throw new Error(`trusted package source ${id} must be an object`); + } + const hosts = toUniqueNormalizedHostList(raw.hosts, "hosts", id); + const redirectHosts = raw.redirectHosts + ? toUniqueNormalizedHostList(raw.redirectHosts, "redirectHosts", id) + : hosts; + const auth = raw.auth === undefined ? undefined : raw.auth; + if (auth !== undefined) { + if (!auth || typeof auth !== "object" || Array.isArray(auth) || auth.type !== "bearer") { + throw new Error(`trusted package source ${id} auth must be {"type":"bearer"}`); + } + const authKeys = Object.keys(auth); + if (authKeys.some((key) => key !== "type")) { + throw new Error(`trusted package source ${id} auth only supports type`); + } + } + return { + allowPrivateNetwork: raw.allowPrivateNetwork === true, + auth, + hosts, + id, + pathPrefixes: toPathPrefixes(raw.pathPrefixes, id), + ports: toTrustedPorts(raw.ports, id), + redirectHosts, + }; +} + +export async function loadTrustedPackageSource(id, policyPath = TRUSTED_PACKAGE_SOURCE_POLICY) { + if (!id) { + throw new Error("source=trusted-url requires --trusted-source-id"); + } + const absolutePolicyPath = path.resolve(ROOT_DIR, policyPath); + let policy; + try { + policy = JSON.parse(await fs.readFile(absolutePolicyPath, "utf8")); + } catch (error) { + throw new Error(`Unable to read trusted package source policy: ${policyPath}`, { + cause: error, + }); + } + if (!policy || typeof policy !== "object" || policy.schemaVersion !== 1) { + throw new Error(`Trusted package source policy must use schemaVersion 1: ${policyPath}`); + } + const sources = policy.sources; + if (!sources || typeof sources !== "object" || Array.isArray(sources)) { + throw new Error(`Trusted package source policy must define sources: ${policyPath}`); + } + if (!Object.hasOwn(sources, id)) { + throw new Error(`Unknown trusted package source: ${id}`); + } + return normalizeTrustedPackageSource(id, sources[id]); +} + +function validateTrustedPackageDownloadUrl(parsed, trustedSource, options = {}) { if (parsed.protocol !== "https:") { - throw new Error(`package_url must use https: ${url}`); + throw new Error(`package_url must use https: ${parsed.toString()}`); } - const response = await fetch(parsed); - if (!response.ok || !response.body) { - throw new Error(`failed to download package_url: HTTP ${response.status}`); + if (parsed.username || parsed.password) { + throw new Error(`package_url must not include credentials: ${parsed.origin}`); + } + const hostname = normalizeUrlHostname(parsed.hostname); + const allowedHosts = options.isRedirect ? trustedSource.redirectHosts : trustedSource.hosts; + if (!allowedHosts.includes(hostname)) { + throw new Error( + `package_url host ${parsed.hostname} is not allowed by trusted package source ${trustedSource.id}`, + ); + } + if (!trustedSource.ports.includes(packageUrlPort(parsed))) { + throw new Error( + `package_url port ${packageUrlPort(parsed)} is not allowed by trusted package source ${trustedSource.id}`, + ); + } + if (!trustedSource.pathPrefixes.some((prefix) => parsed.pathname.startsWith(prefix))) { + throw new Error( + `package_url path is not allowed by trusted package source ${trustedSource.id}`, + ); + } + if (!trustedSource.allowPrivateNetwork && isBlockedPackageHostname(parsed.hostname)) { + throw new Error( + `Blocked hostname or private/internal/special-use IP address: ${parsed.hostname}`, + ); + } +} + +function createTrustedPackageAuthHeaders(trustedSource) { + if (!trustedSource?.auth) { + return undefined; + } + const token = process.env[TRUSTED_PACKAGE_SOURCE_TOKEN_ENV]; + if (!token) { + throw new Error( + `trusted package source ${trustedSource.id} requires ${TRUSTED_PACKAGE_SOURCE_TOKEN_ENV}`, + ); + } + return { authorization: `Bearer ${token}` }; +} + +function validatePackageDownloadUrl(parsed) { + if (parsed.protocol !== "https:") { + throw new Error(`package_url must use https: ${parsed.toString()}`); + } + if (parsed.username || parsed.password) { + throw new Error(`package_url must not include credentials: ${parsed.origin}`); + } + if (parsed.port && parsed.port !== "443") { + throw new Error(`package_url must use the default HTTPS port: ${parsed.origin}`); + } + if (isBlockedPackageHostname(parsed.hostname)) { + throw new Error( + `Blocked hostname or private/internal/special-use IP address: ${parsed.hostname}`, + ); + } +} + +async function defaultLookupHost(hostname) { + return await dnsLookup(hostname, { all: true, verbatim: true }); +} + +function normalizeLookupResults(results) { + const entries = Array.isArray(results) ? results : [results]; + return entries + .map((entry) => ({ address: String(entry.address ?? ""), family: Number(entry.family ?? 0) })) + .filter((entry) => entry.address && (entry.family === 4 || entry.family === 6)); +} + +function createPinnedLookup(hostname, addresses) { + const normalizedHost = normalizeUrlHostname(hostname); + const records = addresses.map((address) => ({ + address, + family: isIP(normalizeUrlHostname(address)), + })); + return (host, options, callback) => { + const cb = typeof options === "function" ? options : callback; + if (!cb) { + return; + } + if (normalizeUrlHostname(host) !== normalizedHost) { + if (typeof options === "function") { + dnsLookupCb(host, cb); + return; + } + dnsLookupCb(host, options, cb); + return; + } + const opts = typeof options === "object" && options !== null ? options : {}; + const filtered = opts.family + ? records.filter((record) => record.family === opts.family) + : records; + const usable = filtered.length > 0 ? filtered : records; + if (opts.all) { + cb(null, usable); + return; + } + const chosen = usable[0]; + cb(null, chosen.address, chosen.family); + }; +} + +async function resolvePackageDownloadAddresses(parsed, lookupHost, trustedSource) { + const hostname = normalizeUrlHostname(parsed.hostname); + if (isIP(hostname)) { + if (!trustedSource?.allowPrivateNetwork && isUnsafeIpAddress(hostname)) { + throw new Error( + `Blocked: package_url resolves to private/internal/special-use IP address: ${hostname}`, + ); + } + return [hostname]; + } + const results = normalizeLookupResults(await lookupHost(hostname)); + if (results.length === 0) { + throw new Error(`Unable to resolve package_url hostname: ${parsed.hostname}`); + } + if (!trustedSource?.allowPrivateNetwork) { + const blocked = results.find((entry) => isUnsafeIpAddress(entry.address)); + if (blocked) { + throw new Error( + `Blocked: package_url resolves to private/internal/special-use IP address: ${blocked.address}`, + ); + } + } + return [...new Set(results.map((entry) => entry.address))]; +} + +function responseStatus(response) { + return Number(response.status ?? 0); +} + +function responseOk(response) { + const status = responseStatus(response); + return status >= 200 && status < 300; +} + +function responseHeader(response, name) { + return response.headers?.get?.(name) ?? null; +} + +async function closeResponseBody(body) { + if (!body) { + return; + } + if (typeof body.cancel === "function") { + await body.cancel().catch(() => {}); + return; + } + if (typeof body.destroy === "function") { + body.destroy(); + } +} + +async function openFetchPackageDownloadResponse(parsed, options) { + const controller = new AbortController(); + const timeout = setTimeout(() => controller.abort(), options.timeoutMs); + timeout.unref?.(); + const response = await options.fetchImpl(parsed, { + headers: options.headers, + redirect: "manual", + signal: controller.signal, + }).catch((error) => { + clearTimeout(timeout); + if (error?.name === "AbortError") { + throw new Error(`package_url download timed out after ${options.timeoutMs}ms: ${parsed.toString()}`, { + cause: error, + }); + } + throw error; + }); + return { + close: async () => closeResponseBody(response.body), + response, + timeout, + timeoutMs: options.timeoutMs, + }; +} + +async function openHttpsPackageDownloadResponse(parsed, options) { + const controller = new AbortController(); + const timeout = setTimeout(() => controller.abort(), options.timeoutMs); + timeout.unref?.(); + const lookup = createPinnedLookup(parsed.hostname, options.addresses); + const response = await new Promise((resolve, reject) => { + const request = httpsRequest( + parsed, + { + headers: options.headers, + lookup, + signal: controller.signal, + }, + (message) => { + resolve({ + body: message, + headers: { + get(name) { + const value = message.headers[name.toLowerCase()]; + if (Array.isArray(value)) { + return value[0] ?? null; + } + return value ?? null; + }, + }, + status: message.statusCode ?? 0, + }); + }, + ); + request.on("error", reject); + request.end(); + }).catch((error) => { + clearTimeout(timeout); + if (error?.name === "AbortError" || error?.code === "ABORT_ERR") { + throw new Error(`package_url download timed out after ${options.timeoutMs}ms: ${parsed.toString()}`, { + cause: error, + }); + } + throw error; + }); + return { + close: async () => closeResponseBody(response.body), + response, + timeout, + timeoutMs: options.timeoutMs, + }; +} + +async function openPackageDownloadResponse(url, options) { + const lookupHost = options.lookupHost ?? defaultLookupHost; + const timeoutMs = options.timeoutMs ?? PACKAGE_URL_DOWNLOAD_TIMEOUT_MS; + const maxRedirects = options.maxRedirects ?? PACKAGE_URL_MAX_REDIRECTS; + const trustedSource = options.trustedSource; + const headers = createTrustedPackageAuthHeaders(trustedSource); + let parsed = new URL(url); + for (let redirectCount = 0; redirectCount <= maxRedirects; redirectCount += 1) { + if (trustedSource) { + validateTrustedPackageDownloadUrl(parsed, trustedSource, { isRedirect: redirectCount > 0 }); + } else { + validatePackageDownloadUrl(parsed); + } + const addresses = await resolvePackageDownloadAddresses(parsed, lookupHost, trustedSource); + const opened = options.fetchImpl + ? await openFetchPackageDownloadResponse(parsed, { + fetchImpl: options.fetchImpl, + headers, + timeoutMs, + }) + : await openHttpsPackageDownloadResponse(parsed, { + addresses, + headers, + timeoutMs, + }); + const status = responseStatus(opened.response); + if ([301, 302, 303, 307, 308].includes(status)) { + clearTimeout(opened.timeout); + await opened.close(); + const location = responseHeader(opened.response, "location"); + if (!location) { + throw new Error(`package_url redirect missing Location header: HTTP ${status}`); + } + parsed = new URL(location, parsed); + continue; + } + return opened; + } + throw new Error(`package_url exceeded ${maxRedirects} redirects: ${url}`); +} + +async function* limitResponseBody(body, maxBytes) { + let downloaded = 0; + for await (const chunk of body) { + const size = typeof chunk === "string" ? Buffer.byteLength(chunk) : chunk.byteLength; + downloaded += size; + if (downloaded > maxBytes) { + throw new Error(`package_url exceeds maximum download size of ${maxBytes} bytes`); + } + yield chunk; + } +} + +export async function downloadUrl(url, target, options = {}) { + const maxBytes = options.maxBytes ?? PACKAGE_URL_MAX_BYTES; + const { close, response, timeout, timeoutMs } = await openPackageDownloadResponse(url, options); + const tempTarget = `${target}.tmp`; + try { + if (!responseOk(response) || !response.body) { + throw new Error(`failed to download package_url: HTTP ${responseStatus(response)}`); + } + const contentLength = Number(responseHeader(response, "content-length") ?? ""); + if (Number.isFinite(contentLength) && contentLength > maxBytes) { + throw new Error(`package_url exceeds maximum download size of ${maxBytes} bytes`); + } + await fs.rm(tempTarget, { force: true }); + await pipeline(limitResponseBody(response.body, maxBytes), createWriteStream(tempTarget)); + await fs.rename(tempTarget, target); + } catch (error) { + if (error?.name === "AbortError") { + throw new Error(`package_url download timed out after ${timeoutMs}ms: ${url}`, { + cause: error, + }); + } + throw error; + } finally { + clearTimeout(timeout); + await close(); + await fs.rm(tempTarget, { force: true }); } - await pipeline(response.body, createWriteStream(target)); } async function readPackageJson(tarball) { @@ -394,6 +984,7 @@ async function resolveCandidate(options) { let packageRef = ""; let packageSourceSha = ""; let packageTrustedReason = ""; + let packageTrustedSourceId = ""; let packageWorktreeDir = ""; let artifactMetadata = {}; @@ -433,14 +1024,27 @@ async function resolveCandidate(options) { packOutput, options.outputName || DEFAULT_OUTPUT_NAME, ); - } else if (options.source === "url") { + } else if (options.source === "url" || options.source === "trusted-url") { if (!options.packageUrl) { - throw new Error("source=url requires --package-url"); + throw new Error(`${options.source} requires --package-url`); } if (!options.packageSha256) { - throw new Error("source=url requires --package-sha256"); + throw new Error(`${options.source} requires --package-sha256`); + } + if (options.source === "trusted-url") { + const trustedSource = await loadTrustedPackageSource( + options.trustedSourceId, + options.trustedSourcePolicy, + ); + await downloadUrl(options.packageUrl, target, { trustedSource }); + packageTrustedReason = `trusted-url-policy:${trustedSource.id}`; + packageTrustedSourceId = trustedSource.id; + } else { + if (options.trustedSourceId) { + throw new Error("--trusted-source-id is only allowed with source=trusted-url"); + } + await downloadUrl(options.packageUrl, target); } - await downloadUrl(options.packageUrl, target); } else if (options.source === "artifact") { if (!options.artifactDir) { throw new Error("source=artifact requires --artifact-dir"); @@ -459,7 +1063,9 @@ async function resolveCandidate(options) { const input = await findSingleTarball(options.artifactDir); await fs.copyFile(input, target); } else { - throw new Error(`source must be one of: ref, npm, url, artifact. Got: ${options.source}`); + throw new Error( + `source must be one of: ref, npm, url, trusted-url, artifact. Got: ${options.source}`, + ); } } finally { if (packageWorktreeDir) { @@ -490,6 +1096,7 @@ async function resolveCandidate(options) { packageSpec: options.packageSpec || "", packageSourceSha, packageTrustedReason, + trustedSourceId: packageTrustedSourceId, sha256: digest, source: options.source, tarball: path.relative(ROOT_DIR, target), diff --git a/src/config/config.pruning-defaults.test.ts b/src/config/config.pruning-defaults.test.ts index 3d5e2e214808..66e3aa2d543f 100644 --- a/src/config/config.pruning-defaults.test.ts +++ b/src/config/config.pruning-defaults.test.ts @@ -1,5 +1,7 @@ import path from "node:path"; import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; +import { setBundledPluginsDirOverrideForTest } from "../plugins/bundled-dir.js"; +import { resetBundledPluginPublicArtifactLoaderForTest } from "../plugins/public-surface-loader.js"; import type { OpenClawConfig } from "./config.js"; import { applyProviderConfigDefaultsForConfig } from "./provider-policy.js"; @@ -15,6 +17,8 @@ function applyAnthropicDefaultsForTest(config: OpenClawConfig) { describe("config pruning defaults", () => { beforeEach(() => { + setBundledPluginsDirOverrideForTest(path.resolve(import.meta.dirname, "../../extensions")); + resetBundledPluginPublicArtifactLoaderForTest(); vi.stubEnv( "OPENCLAW_BUNDLED_PLUGINS_DIR", path.resolve(import.meta.dirname, "../../extensions"), @@ -22,6 +26,8 @@ describe("config pruning defaults", () => { }); afterEach(() => { + setBundledPluginsDirOverrideForTest(undefined); + resetBundledPluginPublicArtifactLoaderForTest(); vi.unstubAllEnvs(); }); @@ -91,26 +97,6 @@ describe("config pruning defaults", () => { ).toBe("short"); }); - it("adds cacheRetention defaults for dated Anthropic primary model refs", () => { - const cfg = applyAnthropicDefaultsForTest({ - auth: { - profiles: { - "anthropic:api": { provider: "anthropic", mode: "api_key" }, - }, - }, - agents: { - defaults: { - model: { primary: "anthropic/claude-sonnet-4-20250514" }, - }, - }, - }); - - expectAnthropicPruningDefaults(cfg); - expect( - cfg.agents?.defaults?.models?.["anthropic/claude-sonnet-4-20250514"]?.params?.cacheRetention, - ).toBe("short"); - }); - it("adds default cacheRetention for Anthropic Claude models on Bedrock", () => { const cfg = applyAnthropicDefaultsForTest({ auth: { diff --git a/test/scripts/package-acceptance-workflow.test.ts b/test/scripts/package-acceptance-workflow.test.ts index 5fe662f000f3..e0ad979fb483 100644 --- a/test/scripts/package-acceptance-workflow.test.ts +++ b/test/scripts/package-acceptance-workflow.test.ts @@ -143,7 +143,11 @@ describe("package acceptance workflow", () => { expect(workflow).toContain("- npm"); expect(workflow).toContain("- ref"); expect(workflow).toContain("- url"); + expect(workflow).toContain("- trusted-url"); expect(workflow).toContain("- artifact"); + expect(workflow).toContain("trusted_source_id:"); + expect(workflow).toContain("TRUSTED_SOURCE_ID: ${{ inputs.trusted_source_id }}"); + expect(workflow).toContain('--trusted-source-id "$TRUSTED_SOURCE_ID"'); expect(workflow).toContain("scripts/resolve-openclaw-package-candidate.mjs"); expect(workflow).toContain('--package-ref "$PACKAGE_REF"'); expect(workflow).toContain('gh run download "$ARTIFACT_RUN_ID"'); diff --git a/test/scripts/resolve-openclaw-package-candidate-ip-bypass.test.ts b/test/scripts/resolve-openclaw-package-candidate-ip-bypass.test.ts new file mode 100644 index 000000000000..b7b4497e8c9a --- /dev/null +++ b/test/scripts/resolve-openclaw-package-candidate-ip-bypass.test.ts @@ -0,0 +1,66 @@ +import { mkdtemp, rm } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import path from "node:path"; +import { afterEach, describe, expect, it } from "vitest"; +import { downloadUrl } from "../../scripts/resolve-openclaw-package-candidate.mjs"; + +const tempDirs: string[] = []; +const dotted = (...parts: number[]) => parts.join("."); + +type LookupAddress = { address: string; family: number }; + +function lookupAddresses(addresses: LookupAddress[]) { + return async () => addresses; +} + +function unexpectedFetch(): never { + throw new Error("downloadUrl should reject before fetching"); +} + +afterEach(async () => { + await Promise.all(tempDirs.splice(0).map((dir) => rm(dir, { recursive: true, force: true }))); +}); + +describe("package URL IPv6 transition address blocking", () => { + it.each([ + ["IPv4-mapped loopback dotted", `::ffff:${dotted(127, 0, 0, 1)}`], + ["IPv4-mapped RFC1918 dotted", `::ffff:${dotted(10, 0, 0, 1)}`], + ["IPv4-mapped loopback hex", "::ffff:7f00:1"], + ["IPv4-mapped RFC1918 hex", "::ffff:a00:1"], + ["IPv4-compatible loopback dotted", `::${dotted(127, 0, 0, 1)}`], + ["IPv4-compatible RFC1918 dotted", `::${dotted(10, 0, 0, 1)}`], + ["IPv4-compatible loopback hex", "::7f00:1"], + ["well-known NAT64 to loopback", "64:ff9b::7f00:1"], + ["local-use NAT64 to RFC1918", "64:ff9b:1::a00:1"], + ["6to4 embedded RFC1918", "2002:0a00:0001::"], + ["Teredo embedded loopback", "2001:0:0:0:0:0:80ff:fffe"], + ["ISATAP embedded RFC1918", "fe80::5efe:a00:1"], + ])("rejects %s DNS result before fetch", async (_name, address) => { + const dir = await mkdtemp(path.join(tmpdir(), "openclaw-package-ip-bypass-")); + tempDirs.push(dir); + const target = path.join(dir, "openclaw.tgz"); + + await expect( + downloadUrl("https://packages.example/openclaw.tgz", target, { + fetchImpl: unexpectedFetch, + lookupHost: lookupAddresses([{ address, family: 6 }]), + }), + ).rejects.toThrow(/private\/internal\/special-use/iu); + }); + + it.each([ + ["IPv4-mapped loopback dotted", `https://[::ffff:${dotted(127, 0, 0, 1)}]/openclaw.tgz`], + ["IPv4-compatible loopback dotted", `https://[::${dotted(127, 0, 0, 1)}]/openclaw.tgz`], + ])("rejects %s URL literals before fetch", async (_name, url) => { + const dir = await mkdtemp(path.join(tmpdir(), "openclaw-package-ip-bypass-")); + tempDirs.push(dir); + const target = path.join(dir, "openclaw.tgz"); + + await expect( + downloadUrl(url, target, { + fetchImpl: unexpectedFetch, + lookupHost: lookupAddresses([{ address: dotted(93, 184, 216, 34), family: 4 }]), + }), + ).rejects.toThrow(/private\/internal\/special-use/iu); + }); +}); diff --git a/test/scripts/resolve-openclaw-package-candidate.test.ts b/test/scripts/resolve-openclaw-package-candidate.test.ts index d762d9ad489c..cfb05608c856 100644 --- a/test/scripts/resolve-openclaw-package-candidate.test.ts +++ b/test/scripts/resolve-openclaw-package-candidate.test.ts @@ -1,9 +1,11 @@ import { execFile } from "node:child_process"; -import { mkdir, mkdtemp, rm, writeFile } from "node:fs/promises"; +import { access, mkdir, mkdtemp, readFile, rm, writeFile } from "node:fs/promises"; import { tmpdir } from "node:os"; import path from "node:path"; import { afterEach, describe, expect, it } from "vitest"; import { + downloadUrl, + loadTrustedPackageSource, parseArgs, readArtifactPackageCandidateMetadata, readPackageBuildSourceSha, @@ -12,6 +14,23 @@ import { const tempDirs: string[] = []; +type LookupAddress = { address: string; family: number }; + +function lookupAddresses(addresses: LookupAddress[]) { + return async () => addresses; +} + +function unexpectedFetch(): never { + throw new Error("downloadUrl should reject before fetching"); +} + +async function missing(file: string): Promise { + return await access(file).then( + () => false, + () => true, + ); +} + afterEach(async () => { await Promise.all(tempDirs.splice(0).map((dir) => rm(dir, { recursive: true, force: true }))); }); @@ -76,9 +95,320 @@ describe("resolve-openclaw-package-candidate", () => { packageSpec: "openclaw@beta", packageUrl: "", source: "npm", + trustedSourceId: "", + trustedSourcePolicy: ".github/package-trusted-sources.json", }); }); + it("loads named trusted package URL source policies", async () => { + const dir = await mkdtemp(path.join(tmpdir(), "openclaw-trusted-package-source-")); + tempDirs.push(dir); + const policy = path.join(dir, "trusted-sources.json"); + await writeFile( + policy, + JSON.stringify({ + schemaVersion: 1, + sources: { + "enterprise-artifactory": { + allowPrivateNetwork: true, + hosts: ["packages.internal"], + pathPrefixes: ["/artifactory/openclaw/"], + ports: [443, 8443], + redirectHosts: ["packages.internal", "mirror.internal"], + }, + }, + }), + ); + + await expect(loadTrustedPackageSource("enterprise-artifactory", policy)).resolves.toEqual({ + allowPrivateNetwork: true, + auth: undefined, + hosts: ["packages.internal"], + id: "enterprise-artifactory", + pathPrefixes: ["/artifactory/openclaw/"], + ports: [443, 8443], + redirectHosts: ["packages.internal", "mirror.internal"], + }); + await expect(loadTrustedPackageSource("missing", policy)).rejects.toThrow( + "Unknown trusted package source: missing", + ); + }); + + it("rejects unsafe package_url downloads before fetching private targets", async () => { + const dir = await mkdtemp(path.join(tmpdir(), "openclaw-package-download-")); + tempDirs.push(dir); + const target = path.join(dir, "openclaw.tgz"); + + await expect( + downloadUrl("http://packages.example/openclaw.tgz", target, { + fetchImpl: unexpectedFetch, + lookupHost: lookupAddresses([{ address: "93.184.216.34", family: 4 }]), + }), + ).rejects.toThrow("package_url must use https"); + await expect( + downloadUrl("https://user@packages.example/openclaw.tgz", target, { + fetchImpl: unexpectedFetch, + lookupHost: lookupAddresses([{ address: "93.184.216.34", family: 4 }]), + }), + ).rejects.toThrow("package_url must not include credentials"); + await expect( + downloadUrl("https://localhost/openclaw.tgz", target, { + fetchImpl: unexpectedFetch, + lookupHost: lookupAddresses([{ address: "127.0.0.1", family: 4 }]), + }), + ).rejects.toThrow(/private\/internal\/special-use/iu); + await expect( + downloadUrl("https://packages.example/openclaw.tgz", target, { + fetchImpl: unexpectedFetch, + lookupHost: lookupAddresses([{ address: "10.0.0.8", family: 4 }]), + }), + ).rejects.toThrow(/resolves to private\/internal\/special-use/iu); + await expect( + downloadUrl("https://packages.example/openclaw.tgz", target, { + fetchImpl: unexpectedFetch, + lookupHost: lookupAddresses([{ address: "64:ff9b::a9fe:a9fe", family: 6 }]), + }), + ).rejects.toThrow(/resolves to private\/internal\/special-use/iu); + }); + + it("allows private package_url downloads only through an explicit trusted source policy", async () => { + const dir = await mkdtemp(path.join(tmpdir(), "openclaw-package-download-")); + tempDirs.push(dir); + const target = path.join(dir, "openclaw.tgz"); + const trustedSource = { + allowPrivateNetwork: true, + hosts: ["packages.internal"], + id: "enterprise-artifactory", + pathPrefixes: ["/artifactory/openclaw/"], + ports: [8443], + redirectHosts: ["packages.internal"], + }; + const requestedUrls: string[] = []; + + await downloadUrl("https://packages.internal:8443/artifactory/openclaw/openclaw.tgz", target, { + fetchImpl: async (url: URL) => { + requestedUrls.push(url.toString()); + return new Response(new Uint8Array([4, 5, 6]), { + headers: { "content-length": "3" }, + status: 200, + }); + }, + lookupHost: lookupAddresses([{ address: "10.0.0.8", family: 4 }]), + maxBytes: 3, + trustedSource, + }); + + expect(requestedUrls).toEqual([ + "https://packages.internal:8443/artifactory/openclaw/openclaw.tgz", + ]); + await expect(readFile(target)).resolves.toEqual(Buffer.from([4, 5, 6])); + + await expect( + downloadUrl("https://evil.internal:8443/artifactory/openclaw/openclaw.tgz", target, { + fetchImpl: unexpectedFetch, + lookupHost: lookupAddresses([{ address: "10.0.0.9", family: 4 }]), + trustedSource, + }), + ).rejects.toThrow("is not allowed by trusted package source enterprise-artifactory"); + await expect( + downloadUrl("https://packages.internal:8443/other/openclaw.tgz", target, { + fetchImpl: unexpectedFetch, + lookupHost: lookupAddresses([{ address: "10.0.0.8", family: 4 }]), + trustedSource, + }), + ).rejects.toThrow("path is not allowed by trusted package source enterprise-artifactory"); + }); + + it("keeps trusted package_url redirects inside the named source policy", async () => { + const dir = await mkdtemp(path.join(tmpdir(), "openclaw-package-download-")); + tempDirs.push(dir); + const target = path.join(dir, "openclaw.tgz"); + const trustedSource = { + allowPrivateNetwork: true, + hosts: ["packages.internal"], + id: "enterprise-artifactory", + pathPrefixes: ["/artifactory/openclaw/"], + ports: [8443], + redirectHosts: ["packages.internal"], + }; + + await expect( + downloadUrl("https://packages.internal:8443/artifactory/openclaw/openclaw.tgz", target, { + fetchImpl: async () => + new Response(null, { + headers: { location: "https://metadata.internal:8443/artifactory/openclaw/pwn.tgz" }, + status: 302, + }), + lookupHost: lookupAddresses([{ address: "10.0.0.8", family: 4 }]), + trustedSource, + }), + ).rejects.toThrow("is not allowed by trusted package source enterprise-artifactory"); + }); + + it("validates redirects for package_url downloads", async () => { + const dir = await mkdtemp(path.join(tmpdir(), "openclaw-package-download-")); + tempDirs.push(dir); + const target = path.join(dir, "openclaw.tgz"); + const requestedUrls: string[] = []; + + await expect( + downloadUrl("https://packages.example/openclaw.tgz", target, { + fetchImpl: async (url: URL) => { + requestedUrls.push(url.toString()); + return new Response(null, { + headers: { location: "https://169.254.169.254/latest/meta-data" }, + status: 302, + }); + }, + lookupHost: lookupAddresses([{ address: "93.184.216.34", family: 4 }]), + }), + ).rejects.toThrow(/private\/internal\/special-use/iu); + expect(requestedUrls).toEqual(["https://packages.example/openclaw.tgz"]); + }); + + it("cancels redirect response bodies before following the next hop", async () => { + const dir = await mkdtemp(path.join(tmpdir(), "openclaw-package-download-")); + tempDirs.push(dir); + const target = path.join(dir, "openclaw.tgz"); + const bodyCancelled: string[] = []; + + await expect( + downloadUrl("https://packages.example/openclaw.tgz", target, { + fetchImpl: async (url: URL) => { + let cancelled = false; + const body = new ReadableStream({ + start(controller) { + const timer = setInterval(() => { + if (cancelled) { + clearInterval(timer); + return; + } + try { + controller.enqueue(new Uint8Array([0])); + } catch { + // Controller may already be closed after cancel. + clearInterval(timer); + } + }, 100); + }, + cancel() { + cancelled = true; + bodyCancelled.push(url.toString()); + }, + }); + return new Response(body, { + headers: { location: "https://packages.example/redirected.tgz" }, + status: 302, + }); + }, + lookupHost: lookupAddresses([{ address: "93.184.216.34", family: 4 }]), + timeoutMs: 5000, + }), + ).rejects.toThrow(); + // The redirect body must have been cancelled, not left open + expect(bodyCancelled.length).toBeGreaterThan(0); + }); + + it("cancels response body on HTTP error before closing dispatcher", async () => { + const dir = await mkdtemp(path.join(tmpdir(), "openclaw-package-download-")); + tempDirs.push(dir); + const target = path.join(dir, "openclaw.tgz"); + let bodyCancelled = false; + + await expect( + downloadUrl("https://packages.example/openclaw.tgz", target, { + fetchImpl: async () => { + const body = new ReadableStream({ + start(controller) { + const timer = setInterval(() => { + try { + controller.enqueue(new Uint8Array([0])); + } catch { + clearInterval(timer); + } + }, 100); + }, + cancel() { + bodyCancelled = true; + }, + }); + return new Response(body, { status: 500 }); + }, + lookupHost: lookupAddresses([{ address: "93.184.216.34", family: 4 }]), + timeoutMs: 5000, + }), + ).rejects.toThrow(/failed to download package_url: HTTP 500/u); + expect(bodyCancelled).toBe(true); + }); + + it("cancels response body on declared oversize before closing dispatcher", async () => { + const dir = await mkdtemp(path.join(tmpdir(), "openclaw-package-download-")); + tempDirs.push(dir); + const target = path.join(dir, "openclaw.tgz"); + let bodyCancelled = false; + + await expect( + downloadUrl("https://packages.example/openclaw.tgz", target, { + fetchImpl: async () => { + const body = new ReadableStream({ + start(controller) { + const timer = setInterval(() => { + try { + controller.enqueue(new Uint8Array([0])); + } catch { + clearInterval(timer); + } + }, 100); + }, + cancel() { + bodyCancelled = true; + }, + }); + return new Response(body, { + headers: { "content-length": String(1024 * 1024 * 100) }, + status: 200, + }); + }, + lookupHost: lookupAddresses([{ address: "93.184.216.34", family: 4 }]), + maxBytes: 1024, + timeoutMs: 5000, + }), + ).rejects.toThrow(/exceeds maximum download size/u); + expect(bodyCancelled).toBe(true); + }); + + it("bounds package_url downloads and writes completed files atomically", async () => { + const dir = await mkdtemp(path.join(tmpdir(), "openclaw-package-download-")); + tempDirs.push(dir); + const target = path.join(dir, "openclaw.tgz"); + + await expect( + downloadUrl("https://packages.example/openclaw.tgz", target, { + fetchImpl: async () => + new Response(new Uint8Array([1, 2, 3, 4]), { + headers: { "content-length": "4" }, + status: 200, + }), + lookupHost: lookupAddresses([{ address: "93.184.216.34", family: 4 }]), + maxBytes: 3, + }), + ).rejects.toThrow("package_url exceeds maximum download size"); + await expect(missing(target)).resolves.toBe(true); + await expect(missing(`${target}.tmp`)).resolves.toBe(true); + + await downloadUrl("https://packages.example/openclaw.tgz", target, { + fetchImpl: async () => + new Response(new Uint8Array([1, 2, 3]), { + headers: { "content-length": "3" }, + status: 200, + }), + lookupHost: lookupAddresses([{ address: "93.184.216.34", family: 4 }]), + maxBytes: 3, + }); + await expect(readFile(target)).resolves.toEqual(Buffer.from([1, 2, 3])); + await expect(missing(`${target}.tmp`)).resolves.toBe(true); + }); + it("reads package source metadata from package artifacts", async () => { const dir = await mkdtemp(path.join(tmpdir(), "openclaw-package-candidate-")); tempDirs.push(dir);