.gitea/workflows/ci.yml

@@ -61,6 +61,9 @@ jobs:
         working-directory: build/workspace
         run: pnpm --filter @ekklesia/signage lint
 
+      - name: Check that CI cleans up every stack it starts
+        run: bash ci/check-cleanup.sh
+
   typecheck:
     name: typecheck
     runs-on: ubuntu-latest
@@ -184,6 +187,9 @@ jobs:
     timeout-minutes: 45
     env:
       DOCKER_BUILDKIT: "1"
+      # Names this job's stacks, so its final step removes exactly these (the platform's
+      # scripts/lib/ci-project.sh).
+      EKKLESIA_CI_JOB_KEY: signage-integration
     steps:
       - name: Check out repository
         uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
@@ -193,5 +199,19 @@ jobs:
       - name: Assemble Signage into the pinned platform
         run: bash ci/assemble.sh
 
+      # A job the runner killed (cancelled, or past the runner's own time limit) never ran its
+      # cleanup. Anything a CI job left on this shared daemon more than three hours ago goes first.
+      - name: Sweep stale CI stacks from the runner
+        run: bash build/workspace/scripts/ci-sweep.sh --stale
+
       - name: Build the image and run the display scenarios
         run: bash ci/integration.sh
+
+      # Whatever happened above, this job's own stacks do not outlive it. If assembly failed, nothing
+      # was started.
+      - name: Remove this job's CI stacks
+        if: always()
+        run: |
+          if [ -f build/workspace/scripts/ci-sweep.sh ]; then
+            bash build/workspace/scripts/ci-sweep.sh --job
+          fi

ci/check-cleanup.sh

@@ -0,0 +1,45 @@
+#!/usr/bin/env bash
+# Checks that Signage's CI cleans up every stack it starts, even when the runner kills a job (the
+# platform's rule: docs/development/security.md, "Disposable stacks never outlive their job").
+#
+#   bash ci/check-cleanup.sh
+#
+# The `integration` job is the one that starts Docker stacks. It must have its own
+# EKKLESIA_CI_JOB_KEY, sweep stale CI stacks before it starts anything, and remove its own in a final
+# `if: always()` step; and the image ci/integration.sh builds must carry the CI labels the sweep finds.
+set -Eeuo pipefail
+
+root="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")/.." && pwd)"
+workflow="$root/.gitea/workflows/ci.yml"
+failures=0
+check() {
+  if eval "$1"; then
+    echo "ok   $2"
+  else
+    echo "FAIL $2"
+    failures=$((failures + 1))
+  fi
+}
+
+# The integration job's lines, from its key to the next job (or the end of the file).
+job="$(awk '/^  integration:$/ { inside = 1; print; next } inside && /^  [a-z][a-z-]*:$/ { exit } inside { print }' "$workflow")"
+line_of() { grep -n -F -- "$1" <<<"$job" | head -1 | cut -d: -f1 || true; }
+
+stale="$(line_of 'ci-sweep.sh --stale')"
+run="$(line_of 'bash ci/integration.sh')"
+own="$(line_of 'ci-sweep.sh --job')"
+last_step="$(grep -n -- "      - name:" <<<"$job" | tail -1 | cut -d: -f1 || true)"
+
+check '[[ -n "$job" ]]' "the workflow has an integration job"
+check 'grep -q "EKKLESIA_CI_JOB_KEY:" <<<"$job"' "the integration job names its stacks with its own key"
+check '[[ -n "$stale" && -n "$run" ]] && ((stale < run))' "it sweeps stale CI stacks before it starts anything"
+check '[[ -n "$own" && -n "$run" ]] && ((own > run)) && ((own > last_step))' \
+  "its last step removes its own stacks"
+check 'sed -n "${last_step},\$p" <<<"$job" | grep -q "if: always()"' "that last step always runs"
+check 'grep -q "EKKLESIA_CI_LABEL" "$root/ci/integration.sh" && grep -q "EKKLESIA_CI_CREATED_LABEL" "$root/ci/integration.sh"' \
+  "the image ci/integration.sh builds carries the CI labels"
+
+((failures == 0)) || {
+  echo "check-cleanup: ${failures} check(s) failed." >&2
+  exit 1
+}

ci/integration.sh

@@ -15,7 +15,12 @@ workspace="$root/build/workspace"
   exit 1
 }
 
-image="signage-ci:${GITHUB_SHA:-local}-${GITHUB_RUN_ID:-0}-${GITHUB_RUN_ATTEMPT:-1}"
+# Named and labeled as a CI stack of this run and job (the platform's scripts/lib/ci-project.sh), so
+# the workflow's final `ci-sweep.sh --job` removes it even if this script is killed before its trap.
+# shellcheck source=/dev/null
+source "$workspace/scripts/lib/ci-project.sh"
+project="$(ci_project_name signage-image)"
+image="${project}:latest"
 trap 'docker image rm --force "$image" >/dev/null 2>&1 || true' EXIT
 
 scenarios="$(sed -e 's/#.*//' -e '/^[[:space:]]*$/d' "$root/ci/display-scenarios" | tr '\n' ' ')"
@@ -25,6 +30,7 @@ scenarios="$(sed -e 's/#.*//' -e '/^[[:space:]]*$/d' "$root/ci/display-scenarios
 }
 
 DOCKER_BUILDKIT=1 docker build --quiet --file "$workspace/deploy/Dockerfile" --tag "$image" \
+  --label "${EKKLESIA_CI_LABEL}=${project}" --label "${EKKLESIA_CI_CREATED_LABEL}=$(date +%s)" \
   "$workspace" >/dev/null
 echo "integration: built the deployment image with Signage; scenarios: ${scenarios}"
 

ci/platform.env

@@ -3,4 +3,4 @@
 #
 # Move it deliberately: change the commit, run `bash ci/update-lock.sh`, and commit both.
 EKKLESIA_PLATFORM_REPOSITORY=https://git.riverofglorymn.org/River_of_Glory_Church/Ekklesia.git
-EKKLESIA_PLATFORM_COMMIT=f50e8f42018c15698709739e2ba54771e97df08a
+EKKLESIA_PLATFORM_COMMIT=874deabea3468eb5c3e907f0821718b077a682c7