From cfa4628f8c2e6b57bb9d4ab9b7384730e5c060b1 Mon Sep 17 00:00:00 2001 From: ekklesia Date: Mon, 5 Oct 2026 20:58:04 -0500 Subject: [PATCH 1/3] test(ci): the integration job must clean up its stacks even when killed The platform leaked a whole CI stack for two days: a job the runner kills never runs its cleanup trap. The platform's rule now applies here: the job that starts Docker stacks (integration) has its own EKKLESIA_CI_JOB_KEY, sweeps stale CI stacks first, removes its own in a final if: always() step, and labels the image it builds. ci/check-cleanup.sh checks that, and the lint job runs it. It fails today on all five counts. Co-Authored-By: Claude Opus 5.5 --- .gitea/workflows/ci.yml | 3 +++ ci/check-cleanup.sh | 45 +++++++++++++++++++++++++++++++++++++++++ 2 files changed, 48 insertions(+) create mode 100755 ci/check-cleanup.sh diff --git a/.gitea/workflows/ci.yml b/.gitea/workflows/ci.yml index a469f0c..5aac322 100644 --- a/.gitea/workflows/ci.yml +++ b/.gitea/workflows/ci.yml @@ -61,6 +61,9 @@ jobs: working-directory: build/workspace run: pnpm --filter @ekklesia/signage lint + - name: Check that CI cleans up every stack it starts + run: bash ci/check-cleanup.sh + typecheck: name: typecheck runs-on: ubuntu-latest diff --git a/ci/check-cleanup.sh b/ci/check-cleanup.sh new file mode 100755 index 0000000..cf50927 --- /dev/null +++ b/ci/check-cleanup.sh @@ -0,0 +1,45 @@ +#!/usr/bin/env bash +# Checks that Signage's CI cleans up every stack it starts, even when the runner kills a job (the +# platform's rule: docs/development/security.md, "Disposable stacks never outlive their job"). +# +# bash ci/check-cleanup.sh +# +# The `integration` job is the one that starts Docker stacks. It must have its own +# EKKLESIA_CI_JOB_KEY, sweep stale CI stacks before it starts anything, and remove its own in a final +# `if: always()` step; and the image ci/integration.sh builds must carry the CI labels the sweep finds. +set -Eeuo pipefail + +root="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")/.." && pwd)" +workflow="$root/.gitea/workflows/ci.yml" +failures=0 +check() { + if eval "$1"; then + echo "ok $2" + else + echo "FAIL $2" + failures=$((failures + 1)) + fi +} + +# The integration job's lines, from its key to the next job (or the end of the file). +job="$(awk '/^ integration:$/ { inside = 1; print; next } inside && /^ [a-z][a-z-]*:$/ { exit } inside { print }' "$workflow")" +line_of() { grep -n -F -- "$1" <<<"$job" | head -1 | cut -d: -f1 || true; } + +stale="$(line_of 'ci-sweep.sh --stale')" +run="$(line_of 'bash ci/integration.sh')" +own="$(line_of 'ci-sweep.sh --job')" +last_step="$(grep -n -- " - name:" <<<"$job" | tail -1 | cut -d: -f1 || true)" + +check '[[ -n "$job" ]]' "the workflow has an integration job" +check 'grep -q "EKKLESIA_CI_JOB_KEY:" <<<"$job"' "the integration job names its stacks with its own key" +check '[[ -n "$stale" && -n "$run" ]] && ((stale < run))' "it sweeps stale CI stacks before it starts anything" +check '[[ -n "$own" && -n "$run" ]] && ((own > run)) && ((own > last_step))' \ + "its last step removes its own stacks" +check 'sed -n "${last_step},\$p" <<<"$job" | grep -q "if: always()"' "that last step always runs" +check 'grep -q "EKKLESIA_CI_LABEL" "$root/ci/integration.sh" && grep -q "EKKLESIA_CI_CREATED_LABEL" "$root/ci/integration.sh"' \ + "the image ci/integration.sh builds carries the CI labels" + +((failures == 0)) || { + echo "check-cleanup: ${failures} check(s) failed." >&2 + exit 1 +} From cbf2a8ee39f3828edf3dbf42b751c9cc0f619d19 Mon Sep 17 00:00:00 2001 From: ekklesia Date: Mon, 5 Oct 2026 21:05:15 -0500 Subject: [PATCH 2/3] ci: the integration job removes its stacks even when the runner kills it The platform's rule (its ci-sweep.sh, docs/development/security.md), applied to the one Signage job that starts Docker stacks: - it has its own EKKLESIA_CI_JOB_KEY; - it sweeps stale CI stacks from the runner before it starts; - it removes its own stacks in a final `if: always()` step, which tolerates a failed assembly, since nothing was started then; - the image ci/integration.sh builds is named and labeled as a CI stack of this run and job, with the time CI built it, so the sweep finds it. ci/platform.env moves to the platform commit that adds the sweep (Ekklesia #60). The assembled lockfile is unchanged, and still passes a frozen install. Verified: SIGKILL of the integration script's whole process group mid-run left 11 resources (2 containers, 4 networks, 4 volumes and the image). `ci-sweep.sh --job` removed exactly those 11. A normal run passes, and the job sweep after it finds nothing. Co-Authored-By: Claude Opus 5.5 --- .gitea/workflows/ci.yml | 17 +++++++++++++++++ ci/integration.sh | 8 +++++++- ci/platform.env | 2 +- 3 files changed, 25 insertions(+), 2 deletions(-) diff --git a/.gitea/workflows/ci.yml b/.gitea/workflows/ci.yml index 5aac322..d8b50f0 100644 --- a/.gitea/workflows/ci.yml +++ b/.gitea/workflows/ci.yml @@ -187,6 +187,9 @@ jobs: timeout-minutes: 45 env: DOCKER_BUILDKIT: "1" + # Names this job's stacks, so its final step removes exactly these (the platform's + # scripts/lib/ci-project.sh). + EKKLESIA_CI_JOB_KEY: signage-integration steps: - name: Check out repository uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 @@ -196,5 +199,19 @@ jobs: - name: Assemble Signage into the pinned platform run: bash ci/assemble.sh + # A job the runner killed (cancelled, or past the runner's own time limit) never ran its + # cleanup. Anything a CI job left on this shared daemon more than three hours ago goes first. + - name: Sweep stale CI stacks from the runner + run: bash build/workspace/scripts/ci-sweep.sh --stale + - name: Build the image and run the display scenarios run: bash ci/integration.sh + + # Whatever happened above, this job's own stacks do not outlive it. If assembly failed, nothing + # was started. + - name: Remove this job's CI stacks + if: always() + run: | + if [ -f build/workspace/scripts/ci-sweep.sh ]; then + bash build/workspace/scripts/ci-sweep.sh --job + fi diff --git a/ci/integration.sh b/ci/integration.sh index a22bd0c..b5ebaa4 100755 --- a/ci/integration.sh +++ b/ci/integration.sh @@ -15,7 +15,12 @@ workspace="$root/build/workspace" exit 1 } -image="signage-ci:${GITHUB_SHA:-local}-${GITHUB_RUN_ID:-0}-${GITHUB_RUN_ATTEMPT:-1}" +# Named and labeled as a CI stack of this run and job (the platform's scripts/lib/ci-project.sh), so +# the workflow's final `ci-sweep.sh --job` removes it even if this script is killed before its trap. +# shellcheck source=/dev/null +source "$workspace/scripts/lib/ci-project.sh" +project="$(ci_project_name signage-image)" +image="${project}:latest" trap 'docker image rm --force "$image" >/dev/null 2>&1 || true' EXIT scenarios="$(sed -e 's/#.*//' -e '/^[[:space:]]*$/d' "$root/ci/display-scenarios" | tr '\n' ' ')" @@ -25,6 +30,7 @@ scenarios="$(sed -e 's/#.*//' -e '/^[[:space:]]*$/d' "$root/ci/display-scenarios } DOCKER_BUILDKIT=1 docker build --quiet --file "$workspace/deploy/Dockerfile" --tag "$image" \ + --label "${EKKLESIA_CI_LABEL}=${project}" --label "${EKKLESIA_CI_CREATED_LABEL}=$(date +%s)" \ "$workspace" >/dev/null echo "integration: built the deployment image with Signage; scenarios: ${scenarios}" diff --git a/ci/platform.env b/ci/platform.env index de7e07f..41cf4b3 100644 --- a/ci/platform.env +++ b/ci/platform.env @@ -3,4 +3,4 @@ # # Move it deliberately: change the commit, run `bash ci/update-lock.sh`, and commit both. EKKLESIA_PLATFORM_REPOSITORY=https://git.riverofglorymn.org/River_of_Glory_Church/Ekklesia.git -EKKLESIA_PLATFORM_COMMIT=f50e8f42018c15698709739e2ba54771e97df08a +EKKLESIA_PLATFORM_COMMIT=00e544125b855372e0f173db33323a57ac357d02 From 6869ae433a31005b587a02491c4d54e3b7c4a7ac Mon Sep 17 00:00:00 2001 From: ekklesia Date: Mon, 5 Oct 2026 21:19:57 -0500 Subject: [PATCH 3/3] ci: pin the platform at the merged sweep, 874deab on main Ekklesia #60 was rebase-merged, so the PR head 00e5441 never became part of main, and ci/assemble.sh rightly refuses it. 874deab is #60's merge on main. ci-sweep.sh, ci-project.sh and smoke-deploy.sh are byte-identical to the old pin, no package file changed, and the assembled lockfile still passes a frozen install. Co-Authored-By: Claude Opus 5.5 --- ci/platform.env | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ci/platform.env b/ci/platform.env index 41cf4b3..eb7fadd 100644 --- a/ci/platform.env +++ b/ci/platform.env @@ -3,4 +3,4 @@ # # Move it deliberately: change the commit, run `bash ci/update-lock.sh`, and commit both. EKKLESIA_PLATFORM_REPOSITORY=https://git.riverofglorymn.org/River_of_Glory_Church/Ekklesia.git -EKKLESIA_PLATFORM_COMMIT=00e544125b855372e0f173db33323a57ac357d02 +EKKLESIA_PLATFORM_COMMIT=874deabea3468eb5c3e907f0821718b077a682c7