diff --git a/.crabbox.yaml b/.crabbox.yaml index 796736b2f694..7fbb9963751e 100644 --- a/.crabbox.yaml +++ b/.crabbox.yaml @@ -20,7 +20,8 @@ actions: workflow: .github/workflows/crabbox-hydrate.yml # Default AWS hydration uses local Actions replay. Use # `crabbox actions hydrate --github-runner --job hydrate-github` when the - # hydrate job needs GitHub secrets. + # hydrate job needs GitHub secrets, or `--github-runner --job + # hydrate-windows-daemon` for focused native Windows daemon proof. job: hydrate ref: main runnerLabels: diff --git a/.github/workflows/crabbox-hydrate.yml b/.github/workflows/crabbox-hydrate.yml index a2538a65ea40..f2cdadcf3096 100644 --- a/.github/workflows/crabbox-hydrate.yml +++ b/.github/workflows/crabbox-hydrate.yml @@ -41,7 +41,7 @@ env: jobs: hydrate: name: hydrate - if: ${{ inputs.crabbox_job != 'hydrate-github' }} + if: ${{ inputs.crabbox_job != 'hydrate-github' && inputs.crabbox_job != 'hydrate-windows-daemon' }} runs-on: [self-hosted, "${{ inputs.crabbox_runner_label }}"] timeout-minutes: 120 steps: @@ -131,7 +131,7 @@ jobs: ln -sfn . "$PNPM_CONFIG_MODULES_DIR/node_modules" fi - - name: Prepare Crabbox shell + - name: Fetch main ref shell: bash run: | set -euo pipefail @@ -140,6 +140,11 @@ jobs: git fetch --no-tags --depth=50 origin "+refs/heads/main:refs/remotes/origin/main" fi + - name: Prepare Crabbox shell + shell: bash + run: | + set -euo pipefail + node_bin="$(dirname "$(node -p 'process.execPath')")" sudo ln -sf "$node_bin/node" /usr/local/bin/node sudo ln -sf "$node_bin/npm" /usr/local/bin/npm @@ -245,7 +250,7 @@ jobs: fi } { - for key in CI GITHUB_ACTIONS GITHUB_WORKSPACE GITHUB_REPOSITORY GITHUB_RUN_ID GITHUB_RUN_NUMBER GITHUB_RUN_ATTEMPT GITHUB_REF GITHUB_REF_NAME GITHUB_SHA GITHUB_EVENT_NAME GITHUB_ACTOR RUNNER_OS RUNNER_ARCH RUNNER_TEMP RUNNER_TOOL_CACHE XDG_CACHE_HOME COREPACK_HOME PNPM_HOME PNPM_CONFIG_CHILD_CONCURRENCY PNPM_CONFIG_MODULES_DIR PNPM_CONFIG_NETWORK_CONCURRENCY PNPM_CONFIG_STORE_DIR PNPM_CONFIG_VERIFY_DEPS_BEFORE_RUN PNPM_CONFIG_VIRTUAL_STORE_DIR; do + for key in CI GITHUB_ACTIONS GITHUB_WORKSPACE GITHUB_REPOSITORY GITHUB_RUN_ID GITHUB_RUN_NUMBER GITHUB_RUN_ATTEMPT GITHUB_REF GITHUB_REF_NAME GITHUB_SHA GITHUB_EVENT_NAME GITHUB_ACTOR RUNNER_OS RUNNER_ARCH RUNNER_TEMP RUNNER_TOOL_CACHE XDG_CACHE_HOME COREPACK_HOME NODE_BIN PNPM_HOME PNPM_CONFIG_CHILD_CONCURRENCY PNPM_CONFIG_MODULES_DIR PNPM_CONFIG_NETWORK_CONCURRENCY PNPM_CONFIG_STORE_DIR PNPM_CONFIG_VERIFY_DEPS_BEFORE_RUN PNPM_CONFIG_VIRTUAL_STORE_DIR PATH; do write_export "$key" done } > "${env_file}.tmp" @@ -292,6 +297,200 @@ jobs: sleep 15 done + hydrate-windows-daemon: + name: hydrate-windows-daemon + if: ${{ inputs.crabbox_job == 'hydrate-windows-daemon' }} + runs-on: [self-hosted, "${{ inputs.crabbox_runner_label }}"] + timeout-minutes: 120 + steps: + - uses: actions/checkout@v6 + with: + ref: ${{ inputs.ref || github.ref }} + + - name: Setup Node.js + uses: actions/setup-node@v6 + with: + node-version: "24" + + - name: Fetch main ref + shell: powershell + run: | + $ErrorActionPreference = "Stop" + + if (git rev-parse --is-inside-work-tree 2>$null) { + git fetch --no-tags --depth=50 origin "+refs/heads/main:refs/remotes/origin/main" + } + + - name: Setup pnpm and dependencies + shell: powershell + env: + CI: "true" + COREPACK_ENABLE_DOWNLOAD_PROMPT: "0" + run: | + $ErrorActionPreference = "Stop" + + $workspace = (Get-Location).Path + $cacheRoot = if ($env:RUNNER_TEMP) { $env:RUNNER_TEMP } else { [System.IO.Path]::GetTempPath() } + $env:XDG_CACHE_HOME = Join-Path $cacheRoot "cache" + $env:COREPACK_HOME = Join-Path $env:XDG_CACHE_HOME "corepack" + $env:PNPM_HOME = Join-Path $cacheRoot "pnpm-home" + $env:PNPM_CONFIG_STORE_DIR = Join-Path $cacheRoot "openclaw-pnpm-store" + $env:PNPM_CONFIG_MODULES_DIR = Join-Path $workspace "node_modules" + $env:PNPM_CONFIG_VIRTUAL_STORE_DIR = Join-Path $workspace "node_modules\.pnpm" + $env:PNPM_CONFIG_CHILD_CONCURRENCY = "4" + $env:PNPM_CONFIG_NETWORK_CONCURRENCY = "8" + $env:PNPM_CONFIG_VERIFY_DEPS_BEFORE_RUN = "false" + $env:PNPM_CONFIG_SIDE_EFFECTS_CACHE = "false" + function Add-GitHubCommandLine([string]$Path, [string]$Value) { + $Value | Out-File -FilePath $Path -Encoding utf8 -Append + } + New-Item -ItemType Directory -Force ` + $env:XDG_CACHE_HOME, ` + $env:COREPACK_HOME, ` + $env:PNPM_HOME, ` + $env:PNPM_CONFIG_STORE_DIR | Out-Null + $env:PATH = "$env:PNPM_HOME;$env:PATH" + @( + "XDG_CACHE_HOME=$env:XDG_CACHE_HOME" + "COREPACK_HOME=$env:COREPACK_HOME" + "PNPM_HOME=$env:PNPM_HOME" + "PNPM_CONFIG_STORE_DIR=$env:PNPM_CONFIG_STORE_DIR" + "PNPM_CONFIG_MODULES_DIR=$env:PNPM_CONFIG_MODULES_DIR" + "PNPM_CONFIG_VIRTUAL_STORE_DIR=$env:PNPM_CONFIG_VIRTUAL_STORE_DIR" + "PNPM_CONFIG_CHILD_CONCURRENCY=$env:PNPM_CONFIG_CHILD_CONCURRENCY" + "PNPM_CONFIG_NETWORK_CONCURRENCY=$env:PNPM_CONFIG_NETWORK_CONCURRENCY" + "PNPM_CONFIG_VERIFY_DEPS_BEFORE_RUN=$env:PNPM_CONFIG_VERIFY_DEPS_BEFORE_RUN" + "PNPM_CONFIG_SIDE_EFFECTS_CACHE=$env:PNPM_CONFIG_SIDE_EFFECTS_CACHE" + ) | ForEach-Object { Add-GitHubCommandLine $env:GITHUB_ENV $_ } + Add-GitHubCommandLine $env:GITHUB_PATH $env:PNPM_HOME + + $packageManager = (Get-Content package.json -Raw | ConvertFrom-Json).packageManager + if (-not $packageManager -or -not $packageManager.StartsWith("pnpm@")) { + Write-Error "Expected packageManager to pin pnpm, got '$packageManager'" + } + corepack enable --install-directory $env:PNPM_HOME + for ($attempt = 1; $attempt -le 3; $attempt++) { + corepack prepare $packageManager --activate + if ($LASTEXITCODE -eq 0) { + break + } + if ($attempt -eq 3) { + exit $LASTEXITCODE + } + Start-Sleep -Seconds ($attempt * 5) + } + $nodeBin = Split-Path -Parent (node -p "process.execPath") + Add-GitHubCommandLine $env:GITHUB_ENV "NODE_BIN=$nodeBin" + Add-GitHubCommandLine $env:GITHUB_PATH $nodeBin + $env:PATH = "$nodeBin;$env:PATH" + + node -v + npm -v + pnpm -v + + $installArgs = @( + "install", + "--filter", + "openclaw", + "--prefer-offline", + "--ignore-scripts=true", + "--config.engine-strict=false", + "--config.enable-pre-post-scripts=false", + "--config.side-effects-cache=false", + "--frozen-lockfile", + "--child-concurrency=$env:PNPM_CONFIG_CHILD_CONCURRENCY", + "--modules-dir=$env:PNPM_CONFIG_MODULES_DIR", + "--network-concurrency=$env:PNPM_CONFIG_NETWORK_CONCURRENCY", + "--store-dir=$env:PNPM_CONFIG_STORE_DIR", + "--virtual-store-dir=$env:PNPM_CONFIG_VIRTUAL_STORE_DIR" + ) + pnpm @installArgs + if ($LASTEXITCODE -ne 0) { + exit $LASTEXITCODE + } + $corepackShimDir = Join-Path $nodeBin "node_modules\corepack\shims" + if (Test-Path $corepackShimDir) { + $env:PNPM_HOME = $corepackShimDir + Add-GitHubCommandLine $env:GITHUB_ENV "PNPM_HOME=$env:PNPM_HOME" + Add-GitHubCommandLine $env:GITHUB_PATH $env:PNPM_HOME + } + + - name: Mark Crabbox ready + shell: powershell + env: + CRABBOX_ID: ${{ inputs.crabbox_id }} + CRABBOX_JOB: ${{ inputs.crabbox_job }} + run: | + $ErrorActionPreference = "Stop" + $job = if ($env:CRABBOX_JOB) { $env:CRABBOX_JOB } else { "hydrate-windows-daemon" } + if (-not $env:CRABBOX_ID -or $env:CRABBOX_ID -notmatch '^[A-Za-z0-9._-]+$') { + Write-Error "Invalid crabbox_id" + } + $actionsRoot = Join-Path $HOME ".crabbox\actions" + New-Item -ItemType Directory -Force $actionsRoot | Out-Null + $state = Join-Path $actionsRoot "$env:CRABBOX_ID.env" + $envFile = Join-Path $actionsRoot "$env:CRABBOX_ID.env.ps1" + $servicesFile = Join-Path $actionsRoot "$env:CRABBOX_ID.services" + $keys = @( + "CI", "GITHUB_ACTIONS", "GITHUB_WORKSPACE", "GITHUB_REPOSITORY", + "GITHUB_RUN_ID", "GITHUB_RUN_NUMBER", "GITHUB_RUN_ATTEMPT", + "GITHUB_REF", "GITHUB_REF_NAME", "GITHUB_SHA", "GITHUB_EVENT_NAME", + "GITHUB_ACTOR", "RUNNER_OS", "RUNNER_ARCH", "RUNNER_TEMP", + "RUNNER_TOOL_CACHE", "XDG_CACHE_HOME", "COREPACK_HOME", "NODE_BIN", + "PNPM_HOME", "PNPM_CONFIG_CHILD_CONCURRENCY", "PNPM_CONFIG_MODULES_DIR", + "PNPM_CONFIG_NETWORK_CONCURRENCY", "PNPM_CONFIG_STORE_DIR", + "PNPM_CONFIG_VERIFY_DEPS_BEFORE_RUN", "PNPM_CONFIG_VIRTUAL_STORE_DIR", + "PNPM_CONFIG_SIDE_EFFECTS_CACHE", "PATH" + ) + $envLines = foreach ($key in $keys) { + $value = [Environment]::GetEnvironmentVariable($key) + if ($value) { + "$key=$value" + } + } + $utf8NoBom = [System.Text.UTF8Encoding]::new($false) + [System.IO.File]::WriteAllLines("$envFile.tmp", $envLines, $utf8NoBom) + Move-Item -Force "$envFile.tmp" $envFile + [System.IO.File]::WriteAllLines( + "$servicesFile.tmp", + @("# Docker containers visible from the hydrated runner", "docker not available on native Windows hydration"), + $utf8NoBom + ) + Move-Item -Force "$servicesFile.tmp" $servicesFile + $stateLines = @( + "WORKSPACE=$env:GITHUB_WORKSPACE", + "RUN_ID=$env:GITHUB_RUN_ID", + "JOB=$job", + "ENV_FILE=$envFile", + "SERVICES_FILE=$servicesFile", + "READY_AT=$((Get-Date).ToUniversalTime().ToString("yyyy-MM-ddTHH:mm:ssZ"))" + ) + [System.IO.File]::WriteAllLines("$state.tmp", $stateLines, $utf8NoBom) + Move-Item -Force "$state.tmp" $state + + - name: Keep Crabbox job alive + shell: powershell + env: + CRABBOX_ID: ${{ inputs.crabbox_id }} + CRABBOX_KEEP_ALIVE_MINUTES: ${{ inputs.crabbox_keep_alive_minutes }} + run: | + $ErrorActionPreference = "Stop" + if (-not $env:CRABBOX_ID -or $env:CRABBOX_ID -notmatch '^[A-Za-z0-9._-]+$') { + Write-Error "Invalid crabbox_id" + } + $minutes = 90 + if ($env:CRABBOX_KEEP_ALIVE_MINUTES -match '^[0-9]+$') { + $minutes = [int]$env:CRABBOX_KEEP_ALIVE_MINUTES + } + $stop = Join-Path $HOME ".crabbox\actions\$env:CRABBOX_ID.stop" + $deadline = (Get-Date).AddMinutes($minutes) + while ((Get-Date) -lt $deadline) { + if (Test-Path $stop) { + exit 0 + } + Start-Sleep -Seconds 15 + } + hydrate-github: name: hydrate-github if: ${{ inputs.crabbox_job == 'hydrate-github' }} @@ -445,7 +644,7 @@ jobs: fi } { - for key in CI GITHUB_ACTIONS GITHUB_WORKSPACE GITHUB_REPOSITORY GITHUB_RUN_ID GITHUB_RUN_NUMBER GITHUB_RUN_ATTEMPT GITHUB_REF GITHUB_REF_NAME GITHUB_SHA GITHUB_EVENT_NAME GITHUB_ACTOR RUNNER_OS RUNNER_ARCH RUNNER_TEMP RUNNER_TOOL_CACHE PNPM_CONFIG_CHILD_CONCURRENCY PNPM_CONFIG_MODULES_DIR PNPM_CONFIG_NETWORK_CONCURRENCY PNPM_CONFIG_STORE_DIR PNPM_CONFIG_VERIFY_DEPS_BEFORE_RUN PNPM_CONFIG_VIRTUAL_STORE_DIR; do + for key in CI GITHUB_ACTIONS GITHUB_WORKSPACE GITHUB_REPOSITORY GITHUB_RUN_ID GITHUB_RUN_NUMBER GITHUB_RUN_ATTEMPT GITHUB_REF GITHUB_REF_NAME GITHUB_SHA GITHUB_EVENT_NAME GITHUB_ACTOR RUNNER_OS RUNNER_ARCH RUNNER_TEMP RUNNER_TOOL_CACHE NODE_BIN PNPM_HOME PNPM_CONFIG_CHILD_CONCURRENCY PNPM_CONFIG_MODULES_DIR PNPM_CONFIG_NETWORK_CONCURRENCY PNPM_CONFIG_STORE_DIR PNPM_CONFIG_VERIFY_DEPS_BEFORE_RUN PNPM_CONFIG_VIRTUAL_STORE_DIR PATH; do write_export "$key" done } > "${env_file}.tmp" diff --git a/test/scripts/package-acceptance-workflow.test.ts b/test/scripts/package-acceptance-workflow.test.ts index 1aa2139d27ea..ee227fb554a3 100644 --- a/test/scripts/package-acceptance-workflow.test.ts +++ b/test/scripts/package-acceptance-workflow.test.ts @@ -123,18 +123,64 @@ describe("package acceptance workflow", () => { const workflow = readWorkflow(CRABBOX_HYDRATE_WORKFLOW); const workflowText = readFileSync(CRABBOX_HYDRATE_WORKFLOW, "utf8"); const hydrate = workflowJob(CRABBOX_HYDRATE_WORKFLOW, "hydrate"); + const hydrateWindowsDaemon = workflowJob(CRABBOX_HYDRATE_WORKFLOW, "hydrate-windows-daemon"); const hydrateGithub = workflowJob(CRABBOX_HYDRATE_WORKFLOW, "hydrate-github"); expect(crabboxConfig.actions?.job).toBe("hydrate"); - expect(hydrate.if).toBe("${{ inputs.crabbox_job != 'hydrate-github' }}"); + expect(hydrate.if).toBe( + "${{ inputs.crabbox_job != 'hydrate-github' && inputs.crabbox_job != 'hydrate-windows-daemon' }}", + ); expect(workflowStep(hydrate, "Setup Node.js").uses).toBe("actions/setup-node@v6"); expect(workflowStep(hydrate, "Setup Node.js").with?.["node-version"]).toBe("24"); - expect(workflowStep(hydrate, "Setup pnpm and dependencies").run).toContain( - 'corepack enable --install-directory "$PNPM_HOME"', + const hydratePnpm = workflowStep(hydrate, "Setup pnpm and dependencies"); + expect(hydratePnpm.if).toBeUndefined(); + expect(hydratePnpm.run).toContain('corepack enable --install-directory "$PNPM_HOME"'); + expect(hydratePnpm.run).toContain("COREPACK_HOME"); + expect(workflowStep(hydrate, "Fetch main ref").run).toContain( + 'git fetch --no-tags --depth=50 origin "+refs/heads/main:refs/remotes/origin/main"', ); - expect(workflowStep(hydrate, "Setup pnpm and dependencies").run).toContain("COREPACK_HOME"); + expect(workflowStep(hydrate, "Prepare Crabbox shell").if).toBeUndefined(); + expect(workflowStep(hydrate, "Ensure Docker is running").if).toBeUndefined(); + expect(workflowStep(hydrate, "Ensure SSH is available").if).toBeUndefined(); + expect(workflowStep(hydrate, "Hydrate provider env helper").if).toBeUndefined(); expect(workflowStep(hydrate, "Mark Crabbox ready").run).toContain("COREPACK_HOME"); expect(workflowStep(hydrate, "Hydrate provider env helper").env).toBeUndefined(); + + expect(hydrateWindowsDaemon.if).toBe("${{ inputs.crabbox_job == 'hydrate-windows-daemon' }}"); + expect(workflowStep(hydrateWindowsDaemon, "Setup Node.js").uses).toBe("actions/setup-node@v6"); + const hydrateWindowsPnpm = workflowStep(hydrateWindowsDaemon, "Setup pnpm and dependencies"); + expect(hydrateWindowsPnpm.shell).toBe("powershell"); + expect(hydrateWindowsPnpm.run).toContain( + '$env:PNPM_CONFIG_MODULES_DIR = Join-Path $workspace "node_modules"', + ); + expect(hydrateWindowsPnpm.run).not.toContain("PNPM_CONFIG_PACKAGE_IMPORT_METHOD"); + expect(hydrateWindowsPnpm.run).toContain("--config.side-effects-cache=false"); + expect(hydrateWindowsPnpm.run).toContain("--ignore-scripts=true"); + expect(hydrateWindowsPnpm.run).toContain('$env:PNPM_CONFIG_CHILD_CONCURRENCY = "4"'); + expect(hydrateWindowsPnpm.run).toContain('$env:PNPM_CONFIG_NETWORK_CONCURRENCY = "8"'); + expect(hydrateWindowsPnpm.run).toContain('$env:PNPM_CONFIG_VERIFY_DEPS_BEFORE_RUN = "false"'); + expect(hydrateWindowsPnpm.run).toContain( + "$Value | Out-File -FilePath $Path -Encoding utf8 -Append", + ); + expect(hydrateWindowsPnpm.run).toContain('"--filter",'); + expect(hydrateWindowsPnpm.run).toContain('"openclaw",'); + expect(hydrateWindowsPnpm.run).not.toContain("Remove-Item -Recurse -Force"); + expect(hydrateWindowsPnpm.run).not.toContain("Add-Content -Path $env:GITHUB_ENV"); + expect(hydrateWindowsPnpm.run).not.toContain("Add-Content -Path $env:GITHUB_PATH"); + expect(hydrateWindowsPnpm.run).toContain("corepack enable --install-directory $env:PNPM_HOME"); + expect(hydrateWindowsPnpm.run).toContain("pnpm @installArgs"); + expect(hydrateWindowsPnpm.run).toContain( + '$corepackShimDir = Join-Path $nodeBin "node_modules\\corepack\\shims"', + ); + const hydrateWindowsFetch = workflowStep(hydrateWindowsDaemon, "Fetch main ref"); + expect(hydrateWindowsFetch.shell).toBe("powershell"); + expect(hydrateWindowsFetch.run).toContain( + 'git fetch --no-tags --depth=50 origin "+refs/heads/main:refs/remotes/origin/main"', + ); + expect(workflowStep(hydrateWindowsDaemon, "Mark Crabbox ready").shell).toBe("powershell"); + expect(workflowStep(hydrateWindowsDaemon, "Mark Crabbox ready").run).toContain('"NODE_BIN"'); + expect(workflowStep(hydrateWindowsDaemon, "Mark Crabbox ready").run).toContain('"PNPM_HOME"'); + expect(workflowStep(hydrateWindowsDaemon, "Mark Crabbox ready").run).toContain('"PATH"'); expect(workflowText).toContain("OPENCLAW_CRABBOX_HYDRATE_DOWNLOAD_TIMEOUT_SECONDS:-300"); expect(workflowText).toContain("OPENCLAW_CRABBOX_HYDRATE_DOWNLOAD_RETRIES:-3"); expect(workflowText).toContain("--retry-all-errors"); @@ -431,7 +477,7 @@ describe("package artifact reuse", () => { expect(pullHelper).toContain("timeout --kill-after=1s 1s true >/dev/null 2>&1"); expect(pullHelper).toContain('timeout "${timeout_seconds}s" docker pull "$image"'); expect(pullHelper).toContain( - 'timeout command not found; cannot bound Docker pull after ${timeout_seconds}s', + "timeout command not found; cannot bound Docker pull after ${timeout_seconds}s", ); expect(dockerE2ePlanAction.match(/bash scripts\/ci-docker-pull-retry\.sh/g)?.length).toBe(2); expect(dockerE2ePlanAction).not.toContain('docker pull "${OPENCLAW_DOCKER_E2E_');