From 32f98d7fe8b6eae9eaaddb93d5fddb8363c943a6 Mon Sep 17 00:00:00 2001 From: Vincent Koc Date: Tue, 2 Jun 2026 00:59:04 +0200 Subject: [PATCH] fix(e2e): forward sighup in node watchdogs --- scripts/lib/docker-e2e-container.sh | 1 + scripts/lib/openclaw-e2e-instance.sh | 1 + test/scripts/docker-build-helper.test.ts | 55 ++++++++-------- test/scripts/openclaw-e2e-instance.test.ts | 74 ++++++++++++---------- 4 files changed, 72 insertions(+), 59 deletions(-) diff --git a/scripts/lib/docker-e2e-container.sh b/scripts/lib/docker-e2e-container.sh index 509ec6594e22..cc041a2034c9 100644 --- a/scripts/lib/docker-e2e-container.sh +++ b/scripts/lib/docker-e2e-container.sh @@ -99,6 +99,7 @@ const forwardSignal = (signal) => { }; process.once("SIGINT", forwardSignal); process.once("SIGTERM", forwardSignal); +process.once("SIGHUP", forwardSignal); child.on("exit", (code, signal) => { clearTimeout(timer); if (parentSignalTimer) { diff --git a/scripts/lib/openclaw-e2e-instance.sh b/scripts/lib/openclaw-e2e-instance.sh index 39debaa1a1a2..cfcab2c40413 100644 --- a/scripts/lib/openclaw-e2e-instance.sh +++ b/scripts/lib/openclaw-e2e-instance.sh @@ -137,6 +137,7 @@ const forwardSignal = (signal) => { }; process.once("SIGINT", forwardSignal); process.once("SIGTERM", forwardSignal); +process.once("SIGHUP", forwardSignal); child.on("close", (code, signal) => { clearTimeout(timer); if (parentSignalTimer) { diff --git a/test/scripts/docker-build-helper.test.ts b/test/scripts/docker-build-helper.test.ts index 2312cbe6bc5c..01de29e8881c 100644 --- a/test/scripts/docker-build-helper.test.ts +++ b/test/scripts/docker-build-helper.test.ts @@ -517,29 +517,33 @@ stderr="$(<"$TMPDIR/stderr")" } }); - it("escalates Docker watchdog children that ignore parent termination", () => { - const workDir = mkdtempSync(join(tmpdir(), "openclaw-docker-node-signal-")); + for (const [shellSignal, expectedStatus] of [ + ["TERM", "143"], + ["HUP", "129"], + ] as const) { + it(`escalates Docker watchdog children that ignore parent SIG${shellSignal}`, () => { + const workDir = mkdtempSync(join(tmpdir(), "openclaw-docker-node-signal-")); - try { - const binDir = join(workDir, "bin"); - mkdirSync(binDir); - writeFileSync( - join(binDir, "node"), - `#!/bin/bash\nexec ${shellQuote(process.execPath)} "$@"\n`, - ); - writeFileSync( - join(binDir, "docker"), - `#!/bin/bash + try { + const binDir = join(workDir, "bin"); + mkdirSync(binDir); + writeFileSync( + join(binDir, "node"), + `#!/bin/bash\nexec ${shellQuote(process.execPath)} "$@"\n`, + ); + writeFileSync( + join(binDir, "docker"), + `#!/bin/bash printf "%s\\n" "$$" >"$TMPDIR/docker-pid" printf "%s\\n" "$PPID" >"$TMPDIR/watchdog-pid" -trap "" TERM +trap "" TERM HUP while true; do /bin/sleep 1; done `, - ); - chmodSync(join(binDir, "node"), 0o755); - chmodSync(join(binDir, "docker"), 0o755); - const rootDir = process.cwd(); - const script = ` + ); + chmodSync(join(binDir, "node"), 0o755); + chmodSync(join(binDir, "docker"), 0o755); + const rootDir = process.cwd(); + const script = ` set -euo pipefail ROOT_DIR=${shellQuote(rootDir)} TMPDIR=${shellQuote(workDir)} @@ -558,12 +562,12 @@ for ((i = 0; i < 100; i += 1)); do done [ -s "$TMPDIR/docker-pid" ] [ -s "$TMPDIR/watchdog-pid" ] -kill -TERM "$(/bin/cat "$TMPDIR/watchdog-pid")" +kill -${shellSignal} "$(/bin/cat "$TMPDIR/watchdog-pid")" set +e wait "$watchdog_pid" status="$?" set -e -[ "$status" = "143" ] +[ "$status" = "${expectedStatus}" ] docker_pid="$(/bin/cat "$TMPDIR/docker-pid")" for ((i = 0; i < 100; i += 1)); do kill -0 "$docker_pid" 2>/dev/null || exit 0 @@ -573,11 +577,12 @@ echo "docker child still alive after watchdog termination" >&2 exit 1 `; - execFileSync("bash", ["-lc", script], { encoding: "utf8" }); - } finally { - rmSync(workDir, { recursive: true, force: true }); - } - }); + execFileSync("bash", ["-lc", script], { encoding: "utf8" }); + } finally { + rmSync(workDir, { recursive: true, force: true }); + } + }); + } it("uses plain timeout when kill-after is unsupported", () => { const workDir = mkdtempSync(join(tmpdir(), "openclaw-docker-plain-timeout-")); diff --git a/test/scripts/openclaw-e2e-instance.test.ts b/test/scripts/openclaw-e2e-instance.test.ts index bcad79557667..2d6b4283bf79 100644 --- a/test/scripts/openclaw-e2e-instance.test.ts +++ b/test/scripts/openclaw-e2e-instance.test.ts @@ -374,28 +374,33 @@ describe("scripts/lib/openclaw-e2e-instance.sh", () => { } }); - it("escalates Node watchdog children that ignore parent termination", () => { - const tempDir = fs.mkdtempSync( - path.join(os.tmpdir(), "openclaw-e2e-instance-node-watchdog-signal-"), - ); - try { - writeNodeShim(tempDir); - const childPath = path.join(tempDir, "ignore-term.cjs"); - const pidPath = path.join(tempDir, "child.pid"); - const watchdogPidPath = path.join(tempDir, "watchdog.pid"); - fs.writeFileSync( - childPath, - [ - "const fs = require('node:fs');", - "fs.writeFileSync(process.argv[2], String(process.pid));", - "fs.writeFileSync(process.argv[3], String(process.ppid));", - "process.on('SIGTERM', () => {});", - "setInterval(() => {}, 1000);", - "", - ].join("\n"), + for (const [shellSignal, expectedStatus] of [ + ["TERM", "143"], + ["HUP", "129"], + ] as const) { + it(`escalates Node watchdog children that ignore parent SIG${shellSignal}`, () => { + const tempDir = fs.mkdtempSync( + path.join(os.tmpdir(), "openclaw-e2e-instance-node-watchdog-signal-"), ); + try { + writeNodeShim(tempDir); + const childPath = path.join(tempDir, "ignore-term.cjs"); + const pidPath = path.join(tempDir, "child.pid"); + const watchdogPidPath = path.join(tempDir, "watchdog.pid"); + fs.writeFileSync( + childPath, + [ + "const fs = require('node:fs');", + "fs.writeFileSync(process.argv[2], String(process.pid));", + "fs.writeFileSync(process.argv[3], String(process.ppid));", + "process.on('SIGTERM', () => {});", + "process.on('SIGHUP', () => {});", + "setInterval(() => {}, 1000);", + "", + ].join("\n"), + ); - const script = ` + const script = ` set -euo pipefail source ${shellQuote(helperPath)} export OPENCLAW_E2E_TIMEOUT_KILL_GRACE_MS=100 @@ -407,12 +412,12 @@ for ((i = 0; i < 100; i += 1)); do done [ -s ${shellQuote(pidPath)} ] [ -s ${shellQuote(watchdogPidPath)} ] -kill -TERM "$(/bin/cat ${shellQuote(watchdogPidPath)})" +kill -${shellSignal} "$(/bin/cat ${shellQuote(watchdogPidPath)})" set +e wait "$wrapper_pid" status="$?" set -e -[ "$status" = "143" ] +[ "$status" = "${expectedStatus}" ] child_pid="$(/bin/cat ${shellQuote(pidPath)})" for ((i = 0; i < 100; i += 1)); do kill -0 "$child_pid" 2>/dev/null || exit 0 @@ -422,19 +427,20 @@ echo "child still alive after watchdog termination" >&2 exit 1 `; - const result = spawnSync("/bin/bash", ["-c", script], { - encoding: "utf8", - env: shellTestEnv({ - PATH: tempDir, - }), - timeout: 5_000, - }); + const result = spawnSync("/bin/bash", ["-c", script], { + encoding: "utf8", + env: shellTestEnv({ + PATH: tempDir, + }), + timeout: 5_000, + }); - expectShellSuccess(result); - } finally { - fs.rmSync(tempDir, { force: true, recursive: true }); - } - }); + expectShellSuccess(result); + } finally { + fs.rmSync(tempDir, { force: true, recursive: true }); + } + }); + } it("terminates only the tracked gateway process", () => { const tempDir = fs.mkdtempSync(path.join(os.tmpdir(), "openclaw-e2e-gateway-terminate-"));