From 94fa7a17a8b1f06995f2b962e2c662d28e2a5184 Mon Sep 17 00:00:00 2001 From: ekklesia Date: Tue, 13 Oct 2026 22:46:38 -0500 Subject: [PATCH] fix(ci): confine the sweep self-test to its own run scripts/test-ci-sweep.sh proves the stale sweep by running `ci-sweep.sh --stale --max-age-seconds 0` on the runner's shared Docker daemon. With a zero threshold that removes every `ekklesia-ci-` stack on the daemon, not just the test's own: any job running at that moment loses its containers, networks, and volumes. It reported "CI sweep checks passed" while doing it. Core's infrastructure-smoke job runs the self-test on every run, and twice tore down Signage's integration stack mid-scenario (Zayin/Signage run 288: its first attempt, and the second one during connectivity-blip). `ci-sweep.sh --stale` now takes `--prefix P`, which narrows the sweep to stacks whose names begin with P and must itself begin with the CI prefix, so it can only narrow a sweep, never widen one. The self-test confines every stale sweep to its own fake run and proves that another run's stack survives even the zero threshold. The workflow's own sweeps are unchanged. Regression: a bystander stack from another run in the self-test, and a Docker-free guard that every stale sweep in the self-test is confined and the workflow's keep the default threshold. Co-Authored-By: Claude Opus 5.5 --- scripts/ci-sweep.sh | 16 ++++++++++++++-- scripts/test-ci-sweep.sh | 23 ++++++++++++++++++----- tests/ci-scripts.test.ts | 28 ++++++++++++++++++++++++++++ 3 files changed, 60 insertions(+), 7 deletions(-) diff --git a/scripts/ci-sweep.sh b/scripts/ci-sweep.sh index 00f5a9e..9d433c8 100755 --- a/scripts/ci-sweep.sh +++ b/scripts/ci-sweep.sh @@ -5,6 +5,11 @@ # bash scripts/ci-sweep.sh --stale [--max-age-seconds N] any CI stack older than N (default 3 h) # bash scripts/ci-sweep.sh --job this job's own stacks, at any age # +# `--stale --prefix P` narrows the sweep to stacks whose names begin with P, which must itself begin +# with the CI prefix: it can only ever narrow a sweep, never widen one. scripts/test-ci-sweep.sh +# confines its short thresholds with it, because the daemon is shared and another job's live stack +# may be on it right now. +# # Every CI script already removes its own stack in an EXIT trap, but a trap never runs when the runner # kills the job container: a cancelled or superseded job (the server honours `concurrency`), or # the runner's own time limit. So each workflow job that starts stacks ends with `--job` in an @@ -31,9 +36,10 @@ source "$repository_root/scripts/lib/ci-project.sh" compose_label="com.docker.compose.project" mode="" max_age_seconds="${EKKLESIA_CI_SWEEP_MAX_AGE_SECONDS:-10800}" +narrow_prefix="" usage() { - echo "usage: ci-sweep.sh --stale [--max-age-seconds N] | --job" >&2 + echo "usage: ci-sweep.sh --stale [--max-age-seconds N] [--prefix P] | --job" >&2 exit 2 } @@ -48,11 +54,17 @@ while (($# > 0)); do max_age_seconds="$2" shift ;; + --prefix) + [[ "${2:-}" == "$EKKLESIA_CI_PREFIX"?* ]] || usage + narrow_prefix="$2" + shift + ;; *) usage ;; esac shift done [[ -n "$mode" ]] || usage +[[ -z "$narrow_prefix" || "$mode" == stale ]] || usage if [[ "$mode" == job ]]; then [[ -n "${GITHUB_RUN_ID:-}" ]] || { @@ -62,7 +74,7 @@ if [[ "$mode" == job ]]; then prefix="$(ci_job_prefix)" cutoff="" else - prefix="$EKKLESIA_CI_PREFIX" + prefix="${narrow_prefix:-$EKKLESIA_CI_PREFIX}" cutoff="$(($(date +%s) - max_age_seconds))" fi diff --git a/scripts/test-ci-sweep.sh b/scripts/test-ci-sweep.sh index ccb5def..0c31d99 100755 --- a/scripts/test-ci-sweep.sh +++ b/scripts/test-ci-sweep.sh @@ -4,8 +4,10 @@ # # bash scripts/test-ci-sweep.sh # -# Every resource it creates is named and labeled for this run alone, under fake run identifiers, so -# it can never touch a real stack; and it removes all of them on exit, whatever happens. +# Every resource it creates is named and labeled for this run alone, under fake run identifiers, and +# every sweep it runs is confined to those identifiers, so it can never touch a real stack: the +# runner's daemon is shared, and another job's live stack may be on it right now. It removes all of +# its own resources on exit, whatever happens. set -Eeuo pipefail @@ -91,10 +93,14 @@ source "$repository_root/scripts/lib/ci-project.sh" # Fake run identifiers: nothing here can match a real run's stacks. export GITHUB_RUN_ID="sweeptest${nonce}" export GITHUB_RUN_ATTEMPT="1" +# Every stack this test sweeps begins with this; nothing it sweeps reaches past it. +own_run="$(ci_name_safe "ekklesia-ci-${GITHUB_RUN_ID}-")" mine="$(EKKLESIA_CI_JOB_KEY=alpha ci_project_name mine)" sibling="$(EKKLESIA_CI_JOB_KEY=beta ci_project_name sibling)" foreign="ekklesia-sweep-control-${nonce}" -for project in "$mine" "$sibling" "$foreign"; do +# Stands in for another run's live stack on the shared daemon — a module's integration job, say. +bystander="$(GITHUB_RUN_ID="bystander${nonce}" EKKLESIA_CI_JOB_KEY=gamma ci_project_name live)" +for project in "$mine" "$sibling" "$foreign" "$bystander"; do make_stack "$project" done # A job container connected to this job's stack network, exactly as compose-job-network.sh does. @@ -115,12 +121,19 @@ check '! env -u GITHUB_RUN_ID bash scripts/ci-sweep.sh --job >/dev/null 2>&1' \ "--job without GITHUB_RUN_ID fails" echo "--- --stale removes only CI stacks older than its threshold" -bash scripts/ci-sweep.sh --stale +bash scripts/ci-sweep.sh --stale --prefix "$own_run" check 'project_exists "$sibling"' "a young CI stack survives the default threshold" -bash scripts/ci-sweep.sh --stale --max-age-seconds 0 +bash scripts/ci-sweep.sh --stale --max-age-seconds 0 --prefix "$own_run" check '! project_exists "$sibling"' "a CI stack older than the threshold is removed" check '! staging_exists "$sibling"' "its staging directory too" check 'project_exists "$foreign"' "a stack outside the CI prefix is never removed, at any age" +check 'project_exists "$bystander"' \ + "another run's stack is never removed by a narrowed sweep, even past the threshold" +echo "--- --prefix only ever narrows the sweep" +check '! bash scripts/ci-sweep.sh --stale --prefix "ekklesia-" >/dev/null 2>&1' \ + "a prefix outside the CI prefix is refused" +check '! bash scripts/ci-sweep.sh --job --prefix "$own_run" >/dev/null 2>&1' \ + "--prefix with --job is refused" if ((failures > 0)); then echo "test-ci-sweep: ${failures} check(s) failed." >&2 diff --git a/tests/ci-scripts.test.ts b/tests/ci-scripts.test.ts index 33439b9..d93423b 100644 --- a/tests/ci-scripts.test.ts +++ b/tests/ci-scripts.test.ts @@ -128,6 +128,34 @@ describe("CI scripts on a shared Docker daemon", () => { expect(workflow.jobs["deploy-smoke"]?.env?.EKKLESIA_CI_JOB_KEY).toContain("matrix.topology"); }); + it("never sweep another run's live stack early, not even from the sweep's own self-test", async () => { + // The workflow's stale sweeps keep the default threshold, which is far above any job's run time. + const workflow = await source(".github/workflows/ci.yml"); + const workflowSweeps = workflow + .split("\n") + .filter((line) => line.includes("ci-sweep.sh --stale")); + expect(workflowSweeps.length).toBeGreaterThan(0); + for (const line of workflowSweeps) { + expect(line).not.toContain("--max-age-seconds"); + } + + // The self-test does use short thresholds, so every stale sweep it runs is confined to its own + // fake run. Unconfined, `--max-age-seconds 0` removed every CI stack on the shared daemon — + // including a module repository's integration stack mid-run. + const selfTest = await source("scripts/test-ci-sweep.sh"); + expect(selfTest).toMatch(/^own_run="\$\(ci_name_safe "ekklesia-ci-\$\{GITHUB_RUN_ID\}-"\)"$/mu); + expect(selfTest.indexOf('export GITHUB_RUN_ID="sweeptest')).toBeLessThan( + selfTest.indexOf("own_run="), + ); + const selfTestSweeps = selfTest + .split("\n") + .filter((line) => /^\s*bash scripts\/ci-sweep\.sh --stale/u.test(line)); + expect(selfTestSweeps.length).toBeGreaterThan(0); + for (const line of selfTestSweeps) { + expect(line).toContain('--prefix "$own_run"'); + } + }); + it("choose the deploy smoke's edge subnet from those not in use, and choose again on a race", async () => { const deploy = await source("scripts/smoke-deploy.sh"); expect(deploy).not.toMatch(/^octet=\$\(\(RANDOM/mu); -- 2.54.0