From fbde572491c6e45fc3932d5f45303588984a3d87 Mon Sep 17 00:00:00 2001 From: Vincent Koc Date: Sun, 31 May 2026 15:02:44 +0200 Subject: [PATCH] fix(e2e): heartbeat resource-sampled docker lanes --- scripts/e2e/kitchen-sink-plugin-docker.sh | 13 +++++---- scripts/e2e/kitchen-sink-rpc-docker.sh | 13 +++++---- scripts/e2e/onboard-docker.sh | 13 +++++---- scripts/lib/docker-e2e-image.sh | 34 +++++++++++++++++++++++ test/scripts/docker-build-helper.test.ts | 9 ++++-- 5 files changed, 61 insertions(+), 21 deletions(-) diff --git a/scripts/e2e/kitchen-sink-plugin-docker.sh b/scripts/e2e/kitchen-sink-plugin-docker.sh index a41f9f563d36..08c3ceb37778 100644 --- a/scripts/e2e/kitchen-sink-plugin-docker.sh +++ b/scripts/e2e/kitchen-sink-plugin-docker.sh @@ -64,12 +64,13 @@ DOCKER_COMMAND_TIMEOUT="$DOCKER_RUN_TIMEOUT" docker_e2e_docker_run_cmd run --nam >"$RUN_LOG" 2>&1 & docker_pid="$!" -while kill -0 "$docker_pid" 2>/dev/null; do - if docker_e2e_docker_cmd inspect "$CONTAINER_NAME" >/dev/null 2>&1; then - docker_e2e_docker_cmd stats --no-stream --format '{{json .}}' "$CONTAINER_NAME" >>"$STATS_LOG" 2>/dev/null || true - fi - sleep 2 -done +docker_e2e_sample_stats_until_exit \ + "$CONTAINER_NAME" \ + "$docker_pid" \ + "$STATS_LOG" \ + "$RUN_LOG" \ + "Kitchen-sink plugin Docker E2E" \ + "${OPENCLAW_DOCKER_E2E_STATS_HEARTBEAT_SECONDS:-30}" set +e wait "$docker_pid" diff --git a/scripts/e2e/kitchen-sink-rpc-docker.sh b/scripts/e2e/kitchen-sink-rpc-docker.sh index bbf80c227d51..02db45825316 100755 --- a/scripts/e2e/kitchen-sink-rpc-docker.sh +++ b/scripts/e2e/kitchen-sink-rpc-docker.sh @@ -47,12 +47,13 @@ DOCKER_COMMAND_TIMEOUT="$DOCKER_RUN_TIMEOUT" docker_e2e_docker_run_cmd run --nam node scripts/e2e/kitchen-sink-rpc-walk.mjs >"$RUN_LOG" 2>&1 & docker_pid="$!" -while kill -0 "$docker_pid" 2>/dev/null; do - if docker_e2e_docker_cmd inspect "$CONTAINER_NAME" >/dev/null 2>&1; then - docker_e2e_docker_cmd stats --no-stream --format '{{json .}}' "$CONTAINER_NAME" >>"$STATS_LOG" 2>/dev/null || true - fi - sleep 2 -done +docker_e2e_sample_stats_until_exit \ + "$CONTAINER_NAME" \ + "$docker_pid" \ + "$STATS_LOG" \ + "$RUN_LOG" \ + "Kitchen-sink RPC Docker E2E" \ + "${OPENCLAW_DOCKER_E2E_STATS_HEARTBEAT_SECONDS:-30}" set +e wait "$docker_pid" diff --git a/scripts/e2e/onboard-docker.sh b/scripts/e2e/onboard-docker.sh index 8e176a1fa22a..e170f56debe9 100755 --- a/scripts/e2e/onboard-docker.sh +++ b/scripts/e2e/onboard-docker.sh @@ -30,12 +30,13 @@ DOCKER_COMMAND_TIMEOUT="$DOCKER_RUN_TIMEOUT" docker_e2e_docker_run_cmd run --nam "$IMAGE_NAME" bash scripts/e2e/lib/onboard/scenario.sh >"$RUN_LOG" 2>&1 & docker_pid="$!" -while kill -0 "$docker_pid" 2>/dev/null; do - if docker_e2e_docker_cmd inspect "$CONTAINER_NAME" >/dev/null 2>&1; then - docker_e2e_docker_cmd stats --no-stream --format '{{json .}}' "$CONTAINER_NAME" >>"$STATS_LOG" 2>/dev/null || true - fi - sleep 2 -done +docker_e2e_sample_stats_until_exit \ + "$CONTAINER_NAME" \ + "$docker_pid" \ + "$STATS_LOG" \ + "$RUN_LOG" \ + "Onboarding Docker E2E" \ + "${OPENCLAW_DOCKER_E2E_STATS_HEARTBEAT_SECONDS:-30}" set +e wait "$docker_pid" diff --git a/scripts/lib/docker-e2e-image.sh b/scripts/lib/docker-e2e-image.sh index 0ca375589063..4fa543ccadce 100644 --- a/scripts/lib/docker-e2e-image.sh +++ b/scripts/lib/docker-e2e-image.sh @@ -117,3 +117,37 @@ docker_e2e_test_state_function_b64() { base64 | tr -d '\n' } + +docker_e2e_sample_stats_until_exit() { + local container_name="${1:?missing container name}" + local docker_pid="${2:?missing docker pid}" + local stats_log="${3:?missing stats log}" + local run_log="${4:?missing run log}" + local label="${5:-Docker E2E}" + local heartbeat_seconds="${6:-30}" + local started_at="$SECONDS" + local last_heartbeat="$SECONDS" + + if ! [[ "$heartbeat_seconds" =~ ^[0-9]+$ ]] || [ "$heartbeat_seconds" -lt 1 ]; then + heartbeat_seconds="30" + fi + + while kill -0 "$docker_pid" 2>/dev/null; do + if docker_e2e_docker_cmd inspect "$container_name" >/dev/null 2>&1; then + docker_e2e_docker_cmd stats --no-stream --format '{{json .}}' "$container_name" >>"$stats_log" 2>/dev/null || true + fi + + if ((SECONDS - last_heartbeat >= heartbeat_seconds)); then + local elapsed_seconds=$((SECONDS - started_at)) + local log_bytes="0" + if [ -f "$run_log" ]; then + log_bytes="$(wc -c <"$run_log" 2>/dev/null || echo 0)" + log_bytes="${log_bytes//[[:space:]]/}" + fi + echo "$label still running (${elapsed_seconds}s elapsed, ${log_bytes} log bytes captured)..." + last_heartbeat="$SECONDS" + fi + + sleep 2 + done +} diff --git a/test/scripts/docker-build-helper.test.ts b/test/scripts/docker-build-helper.test.ts index af0b1a204a39..7f3b00f4e747 100644 --- a/test/scripts/docker-build-helper.test.ts +++ b/test/scripts/docker-build-helper.test.ts @@ -1359,7 +1359,9 @@ test -f "$TMPDIR/docker-cmd-seen" ); expect(runner).toContain('-e "OPENCLAW_E2E_COMMAND_TIMEOUT=$COMMAND_TIMEOUT"'); expect(runner).toContain('--name "$CONTAINER_NAME"'); - expect(runner).toContain("docker_e2e_docker_cmd stats --no-stream"); + expect(runner).toContain("docker_e2e_sample_stats_until_exit \\"); + expect(runner).toContain('"$STATS_LOG" \\'); + expect(runner).toContain('"$RUN_LOG" \\'); expect(runner).toContain("assert-resource-ceiling.mjs"); expect(runner).not.toContain("docker_e2e_run_with_harness -t"); }); @@ -1379,8 +1381,9 @@ test -f "$TMPDIR/docker-cmd-seen" 'DOCKER_COMMAND_TIMEOUT="$DOCKER_RUN_TIMEOUT" docker_e2e_docker_run_cmd run --name "$CONTAINER_NAME"', ); expect(runner, path).toContain('DOCKER_RUN_TIMEOUT="${OPENCLAW_'); - expect(runner, path).toContain('docker_e2e_docker_cmd inspect "$CONTAINER_NAME"'); - expect(runner, path).toContain("docker_e2e_docker_cmd stats --no-stream"); + expect(runner, path).toContain("docker_e2e_sample_stats_until_exit \\"); + expect(runner, path).toContain('"$STATS_LOG" \\'); + expect(runner, path).toContain('"$RUN_LOG" \\'); expect(runner, path).not.toMatch(/(^|\n)docker run --name "\$CONTAINER_NAME"/u); expect(runner, path).not.toMatch(/(^|\n)docker (?:inspect|stats) /u); expect(runner, path).toMatch(/cleanup\(\) \{[\s\S]*rm -f "\$RUN_LOG" "\$STATS_LOG"/u);