Merge pull request 'fix(ci): confine the sweep self-test to its own run' (#96) from fix/ci-sweep-self-test-scope into main

Reviewed-on: #96
Reviewed-by: Zayin_Admin <[email protected]>

scripts/ci-sweep.sh

@@ -5,6 +5,11 @@
 #   bash scripts/ci-sweep.sh --stale [--max-age-seconds N]   any CI stack older than N (default 3 h)
 #   bash scripts/ci-sweep.sh --job                           this job's own stacks, at any age
 #
+# `--stale --prefix P` narrows the sweep to stacks whose names begin with P, which must itself begin
+# with the CI prefix: it can only ever narrow a sweep, never widen one. scripts/test-ci-sweep.sh
+# confines its short thresholds with it, because the daemon is shared and another job's live stack
+# may be on it right now.
+#
 # Every CI script already removes its own stack in an EXIT trap, but a trap never runs when the runner
 # kills the job container: a cancelled or superseded job (the server honours `concurrency`), or
 # the runner's own time limit. So each workflow job that starts stacks ends with `--job` in an
@@ -31,9 +36,10 @@ source "$repository_root/scripts/lib/ci-project.sh"
 compose_label="com.docker.compose.project"
 mode=""
 max_age_seconds="${EKKLESIA_CI_SWEEP_MAX_AGE_SECONDS:-10800}"
+narrow_prefix=""
 
 usage() {
-  echo "usage: ci-sweep.sh --stale [--max-age-seconds N] | --job" >&2
+  echo "usage: ci-sweep.sh --stale [--max-age-seconds N] [--prefix P] | --job" >&2
   exit 2
 }
 
@@ -48,11 +54,17 @@ while (($# > 0)); do
     max_age_seconds="$2"
     shift
     ;;
+  --prefix)
+    [[ "${2:-}" == "$EKKLESIA_CI_PREFIX"?* ]] || usage
+    narrow_prefix="$2"
+    shift
+    ;;
   *) usage ;;
   esac
   shift
 done
 [[ -n "$mode" ]] || usage
+[[ -z "$narrow_prefix" || "$mode" == stale ]] || usage
 
 if [[ "$mode" == job ]]; then
   [[ -n "${GITHUB_RUN_ID:-}" ]] || {
@@ -62,7 +74,7 @@ if [[ "$mode" == job ]]; then
   prefix="$(ci_job_prefix)"
   cutoff=""
 else
-  prefix="$EKKLESIA_CI_PREFIX"
+  prefix="${narrow_prefix:-$EKKLESIA_CI_PREFIX}"
   cutoff="$(($(date +%s) - max_age_seconds))"
 fi
 

scripts/test-ci-sweep.sh

@@ -4,8 +4,10 @@
 #
 #   bash scripts/test-ci-sweep.sh
 #
-# Every resource it creates is named and labeled for this run alone, under fake run identifiers, so
-# it can never touch a real stack; and it removes all of them on exit, whatever happens.
+# Every resource it creates is named and labeled for this run alone, under fake run identifiers, and
+# every sweep it runs is confined to those identifiers, so it can never touch a real stack: the
+# runner's daemon is shared, and another job's live stack may be on it right now. It removes all of
+# its own resources on exit, whatever happens.
 
 set -Eeuo pipefail
 
@@ -91,10 +93,14 @@ source "$repository_root/scripts/lib/ci-project.sh"
 # Fake run identifiers: nothing here can match a real run's stacks.
 export GITHUB_RUN_ID="sweeptest${nonce}"
 export GITHUB_RUN_ATTEMPT="1"
+# Every stack this test sweeps begins with this; nothing it sweeps reaches past it.
+own_run="$(ci_name_safe "ekklesia-ci-${GITHUB_RUN_ID}-")"
 mine="$(EKKLESIA_CI_JOB_KEY=alpha ci_project_name mine)"
 sibling="$(EKKLESIA_CI_JOB_KEY=beta ci_project_name sibling)"
 foreign="ekklesia-sweep-control-${nonce}"
-for project in "$mine" "$sibling" "$foreign"; do
+# Stands in for another run's live stack on the shared daemon — a module's integration job, say.
+bystander="$(GITHUB_RUN_ID="bystander${nonce}" EKKLESIA_CI_JOB_KEY=gamma ci_project_name live)"
+for project in "$mine" "$sibling" "$foreign" "$bystander"; do
   make_stack "$project"
 done
 # A job container connected to this job's stack network, exactly as compose-job-network.sh does.
@@ -115,12 +121,19 @@ check '! env -u GITHUB_RUN_ID bash scripts/ci-sweep.sh --job >/dev/null 2>&1' \
   "--job without GITHUB_RUN_ID fails"
 
 echo "--- --stale removes only CI stacks older than its threshold"
-bash scripts/ci-sweep.sh --stale
+bash scripts/ci-sweep.sh --stale --prefix "$own_run"
 check 'project_exists "$sibling"' "a young CI stack survives the default threshold"
-bash scripts/ci-sweep.sh --stale --max-age-seconds 0
+bash scripts/ci-sweep.sh --stale --max-age-seconds 0 --prefix "$own_run"
 check '! project_exists "$sibling"' "a CI stack older than the threshold is removed"
 check '! staging_exists "$sibling"' "its staging directory too"
 check 'project_exists "$foreign"' "a stack outside the CI prefix is never removed, at any age"
+check 'project_exists "$bystander"' \
+  "another run's stack is never removed by a narrowed sweep, even past the threshold"
+echo "--- --prefix only ever narrows the sweep"
+check '! bash scripts/ci-sweep.sh --stale --prefix "ekklesia-" >/dev/null 2>&1' \
+  "a prefix outside the CI prefix is refused"
+check '! bash scripts/ci-sweep.sh --job --prefix "$own_run" >/dev/null 2>&1' \
+  "--prefix with --job is refused"
 
 if ((failures > 0)); then
   echo "test-ci-sweep: ${failures} check(s) failed." >&2

tests/ci-scripts.test.ts

@@ -128,6 +128,34 @@ describe("CI scripts on a shared Docker daemon", () => {
     expect(workflow.jobs["deploy-smoke"]?.env?.EKKLESIA_CI_JOB_KEY).toContain("matrix.topology");
   });
 
+  it("never sweep another run's live stack early, not even from the sweep's own self-test", async () => {
+    // The workflow's stale sweeps keep the default threshold, which is far above any job's run time.
+    const workflow = await source(".github/workflows/ci.yml");
+    const workflowSweeps = workflow
+      .split("\n")
+      .filter((line) => line.includes("ci-sweep.sh --stale"));
+    expect(workflowSweeps.length).toBeGreaterThan(0);
+    for (const line of workflowSweeps) {
+      expect(line).not.toContain("--max-age-seconds");
+    }
+
+    // The self-test does use short thresholds, so every stale sweep it runs is confined to its own
+    // fake run. Unconfined, `--max-age-seconds 0` removed every CI stack on the shared daemon —
+    // including a module repository's integration stack mid-run.
+    const selfTest = await source("scripts/test-ci-sweep.sh");
+    expect(selfTest).toMatch(/^own_run="\$\(ci_name_safe "ekklesia-ci-\$\{GITHUB_RUN_ID\}-"\)"$/mu);
+    expect(selfTest.indexOf('export GITHUB_RUN_ID="sweeptest')).toBeLessThan(
+      selfTest.indexOf("own_run="),
+    );
+    const selfTestSweeps = selfTest
+      .split("\n")
+      .filter((line) => /^\s*bash scripts\/ci-sweep\.sh --stale/u.test(line));
+    expect(selfTestSweeps.length).toBeGreaterThan(0);
+    for (const line of selfTestSweeps) {
+      expect(line).toContain('--prefix "$own_run"');
+    }
+  });
+
   it("choose the deploy smoke's edge subnet from those not in use, and choose again on a race", async () => {
     const deploy = await source("scripts/smoke-deploy.sh");
     expect(deploy).not.toMatch(/^octet=\$\(\(RANDOM/mu);