diff --git a/.github/workflows/openclaw-live-and-e2e-checks-reusable.yml b/.github/workflows/openclaw-live-and-e2e-checks-reusable.yml index d4745362224..9cc5bfd15c6 100644 --- a/.github/workflows/openclaw-live-and-e2e-checks-reusable.yml +++ b/.github/workflows/openclaw-live-and-e2e-checks-reusable.yml @@ -603,14 +603,14 @@ jobs: shell: bash run: | set -euo pipefail - docker pull "${OPENCLAW_DOCKER_E2E_BARE_IMAGE}" + bash .release-harness/scripts/ci-docker-pull-retry.sh "${OPENCLAW_DOCKER_E2E_BARE_IMAGE}" - name: Pull shared functional Docker E2E image if: steps.plan.outputs.needs_functional_image == '1' shell: bash run: | set -euo pipefail - docker pull "${OPENCLAW_DOCKER_E2E_FUNCTIONAL_IMAGE}" + bash .release-harness/scripts/ci-docker-pull-retry.sh "${OPENCLAW_DOCKER_E2E_FUNCTIONAL_IMAGE}" - name: Validate Docker E2E credentials shell: bash @@ -794,7 +794,7 @@ jobs: id: plan shell: bash env: - LANES: ${{ inputs.docker_lanes }} + LANES: ${{ matrix.group.docker_lanes }} INCLUDE_OPENWEBUI: ${{ inputs.include_openwebui }} run: | set -euo pipefail @@ -826,14 +826,14 @@ jobs: shell: bash run: | set -euo pipefail - docker pull "${OPENCLAW_DOCKER_E2E_BARE_IMAGE}" + bash .release-harness/scripts/ci-docker-pull-retry.sh "${OPENCLAW_DOCKER_E2E_BARE_IMAGE}" - name: Pull shared functional Docker E2E image if: steps.plan.outputs.needs_functional_image == '1' shell: bash run: | set -euo pipefail - docker pull "${OPENCLAW_DOCKER_E2E_FUNCTIONAL_IMAGE}" + bash .release-harness/scripts/ci-docker-pull-retry.sh "${OPENCLAW_DOCKER_E2E_FUNCTIONAL_IMAGE}" - name: Validate Docker E2E credentials shell: bash @@ -971,14 +971,14 @@ jobs: shell: bash run: | set -euo pipefail - docker pull "${OPENCLAW_DOCKER_E2E_BARE_IMAGE}" + bash .release-harness/scripts/ci-docker-pull-retry.sh "${OPENCLAW_DOCKER_E2E_BARE_IMAGE}" - name: Pull shared functional Docker E2E image if: steps.plan.outputs.needs_functional_image == '1' shell: bash run: | set -euo pipefail - docker pull "${OPENCLAW_DOCKER_E2E_FUNCTIONAL_IMAGE}" + bash .release-harness/scripts/ci-docker-pull-retry.sh "${OPENCLAW_DOCKER_E2E_FUNCTIONAL_IMAGE}" - name: Run Open WebUI Docker E2E chunk shell: bash diff --git a/scripts/ci-docker-pull-retry.sh b/scripts/ci-docker-pull-retry.sh new file mode 100644 index 00000000000..9b01e43169e --- /dev/null +++ b/scripts/ci-docker-pull-retry.sh @@ -0,0 +1,42 @@ +#!/usr/bin/env bash +set -euo pipefail + +if [[ "$#" -ne 1 || -z "${1// }" ]]; then + echo "usage: $0 " >&2 + exit 2 +fi + +image="$1" +attempts="${OPENCLAW_DOCKER_PULL_ATTEMPTS:-3}" +timeout_seconds="${OPENCLAW_DOCKER_PULL_TIMEOUT_SECONDS:-480}" +retry_delay_seconds="${OPENCLAW_DOCKER_PULL_RETRY_DELAY_SECONDS:-20}" + +if ! [[ "$attempts" =~ ^[1-9][0-9]*$ ]]; then + echo "OPENCLAW_DOCKER_PULL_ATTEMPTS must be a positive integer, got: $attempts" >&2 + exit 2 +fi + +if ! [[ "$timeout_seconds" =~ ^[1-9][0-9]*$ ]]; then + echo "OPENCLAW_DOCKER_PULL_TIMEOUT_SECONDS must be a positive integer, got: $timeout_seconds" >&2 + exit 2 +fi + +if ! [[ "$retry_delay_seconds" =~ ^[0-9]+$ ]]; then + echo "OPENCLAW_DOCKER_PULL_RETRY_DELAY_SECONDS must be a non-negative integer, got: $retry_delay_seconds" >&2 + exit 2 +fi + +last_status=1 +for attempt in $(seq 1 "$attempts"); do + echo "==> Pull Docker image attempt ${attempt}/${attempts}: ${image}" + if timeout --foreground --kill-after=30s "${timeout_seconds}s" docker pull "$image"; then + exit 0 + fi + last_status="$?" + echo "Docker pull failed or timed out after ${timeout_seconds}s: status=${last_status}" >&2 + if [[ "$attempt" -lt "$attempts" && "$retry_delay_seconds" -gt 0 ]]; then + sleep "$retry_delay_seconds" + fi +done + +exit "$last_status" diff --git a/test/scripts/package-acceptance-workflow.test.ts b/test/scripts/package-acceptance-workflow.test.ts index 8528ee8920a..d408f435d48 100644 --- a/test/scripts/package-acceptance-workflow.test.ts +++ b/test/scripts/package-acceptance-workflow.test.ts @@ -83,12 +83,24 @@ describe("package artifact reuse", () => { expect(workflow).toContain("OPENCLAW_DOCKER_E2E_REPO_ROOT:"); expect(workflow).toContain("node .release-harness/scripts/test-docker-all.mjs --plan-json"); expect(workflow).toContain("node .release-harness/scripts/docker-e2e.mjs github-outputs"); + expect(workflow).toContain("bash .release-harness/scripts/ci-docker-pull-retry.sh"); expect(workflow).toContain("plan_docker_lane_groups:"); expect(workflow).toContain("Docker E2E targeted lanes (${{ matrix.group.label }})"); + expect(workflow).toContain("LANES: ${{ matrix.group.docker_lanes }}"); expect(workflow).toContain("DOCKER_E2E_LANES: ${{ matrix.group.docker_lanes }}"); expect(workflow).toContain("name: docker-e2e-${{ steps.plan.outputs.artifact_suffix }}"); }); + it("bounds shared Docker image pulls so package acceptance cannot stall forever", () => { + const pullHelper = readFileSync("scripts/ci-docker-pull-retry.sh", "utf8"); + + expect(pullHelper).toContain("OPENCLAW_DOCKER_PULL_ATTEMPTS"); + expect(pullHelper).toContain("OPENCLAW_DOCKER_PULL_TIMEOUT_SECONDS"); + expect(pullHelper).toContain( + 'timeout --foreground --kill-after=30s "${timeout_seconds}s" docker pull "$image"', + ); + }); + it("uses Blacksmith Docker build caching for prepared E2E images", () => { const workflow = readFileSync(LIVE_E2E_WORKFLOW, "utf8");