diff --git a/scripts/ci-sweep.sh b/scripts/ci-sweep.sh index 00f5a9e..9d433c8 100755 --- a/scripts/ci-sweep.sh +++ b/scripts/ci-sweep.sh @@ -5,6 +5,11 @@ # bash scripts/ci-sweep.sh --stale [--max-age-seconds N] any CI stack older than N (default 3 h) # bash scripts/ci-sweep.sh --job this job's own stacks, at any age # +# `--stale --prefix P` narrows the sweep to stacks whose names begin with P, which must itself begin +# with the CI prefix: it can only ever narrow a sweep, never widen one. scripts/test-ci-sweep.sh +# confines its short thresholds with it, because the daemon is shared and another job's live stack +# may be on it right now. +# # Every CI script already removes its own stack in an EXIT trap, but a trap never runs when the runner # kills the job container: a cancelled or superseded job (the server honours `concurrency`), or # the runner's own time limit. So each workflow job that starts stacks ends with `--job` in an @@ -31,9 +36,10 @@ source "$repository_root/scripts/lib/ci-project.sh" compose_label="com.docker.compose.project" mode="" max_age_seconds="${EKKLESIA_CI_SWEEP_MAX_AGE_SECONDS:-10800}" +narrow_prefix="" usage() { - echo "usage: ci-sweep.sh --stale [--max-age-seconds N] | --job" >&2 + echo "usage: ci-sweep.sh --stale [--max-age-seconds N] [--prefix P] | --job" >&2 exit 2 } @@ -48,11 +54,17 @@ while (($# > 0)); do max_age_seconds="$2" shift ;; + --prefix) + [[ "${2:-}" == "$EKKLESIA_CI_PREFIX"?* ]] || usage + narrow_prefix="$2" + shift + ;; *) usage ;; esac shift done [[ -n "$mode" ]] || usage +[[ -z "$narrow_prefix" || "$mode" == stale ]] || usage if [[ "$mode" == job ]]; then [[ -n "${GITHUB_RUN_ID:-}" ]] || { @@ -62,7 +74,7 @@ if [[ "$mode" == job ]]; then prefix="$(ci_job_prefix)" cutoff="" else - prefix="$EKKLESIA_CI_PREFIX" + prefix="${narrow_prefix:-$EKKLESIA_CI_PREFIX}" cutoff="$(($(date +%s) - max_age_seconds))" fi diff --git a/scripts/test-ci-sweep.sh b/scripts/test-ci-sweep.sh index ccb5def..0c31d99 100755 --- a/scripts/test-ci-sweep.sh +++ b/scripts/test-ci-sweep.sh @@ -4,8 +4,10 @@ # # bash scripts/test-ci-sweep.sh # -# Every resource it creates is named and labeled for this run alone, under fake run identifiers, so -# it can never touch a real stack; and it removes all of them on exit, whatever happens. +# Every resource it creates is named and labeled for this run alone, under fake run identifiers, and +# every sweep it runs is confined to those identifiers, so it can never touch a real stack: the +# runner's daemon is shared, and another job's live stack may be on it right now. It removes all of +# its own resources on exit, whatever happens. set -Eeuo pipefail @@ -91,10 +93,14 @@ source "$repository_root/scripts/lib/ci-project.sh" # Fake run identifiers: nothing here can match a real run's stacks. export GITHUB_RUN_ID="sweeptest${nonce}" export GITHUB_RUN_ATTEMPT="1" +# Every stack this test sweeps begins with this; nothing it sweeps reaches past it. +own_run="$(ci_name_safe "ekklesia-ci-${GITHUB_RUN_ID}-")" mine="$(EKKLESIA_CI_JOB_KEY=alpha ci_project_name mine)" sibling="$(EKKLESIA_CI_JOB_KEY=beta ci_project_name sibling)" foreign="ekklesia-sweep-control-${nonce}" -for project in "$mine" "$sibling" "$foreign"; do +# Stands in for another run's live stack on the shared daemon — a module's integration job, say. +bystander="$(GITHUB_RUN_ID="bystander${nonce}" EKKLESIA_CI_JOB_KEY=gamma ci_project_name live)" +for project in "$mine" "$sibling" "$foreign" "$bystander"; do make_stack "$project" done # A job container connected to this job's stack network, exactly as compose-job-network.sh does. @@ -115,12 +121,19 @@ check '! env -u GITHUB_RUN_ID bash scripts/ci-sweep.sh --job >/dev/null 2>&1' \ "--job without GITHUB_RUN_ID fails" echo "--- --stale removes only CI stacks older than its threshold" -bash scripts/ci-sweep.sh --stale +bash scripts/ci-sweep.sh --stale --prefix "$own_run" check 'project_exists "$sibling"' "a young CI stack survives the default threshold" -bash scripts/ci-sweep.sh --stale --max-age-seconds 0 +bash scripts/ci-sweep.sh --stale --max-age-seconds 0 --prefix "$own_run" check '! project_exists "$sibling"' "a CI stack older than the threshold is removed" check '! staging_exists "$sibling"' "its staging directory too" check 'project_exists "$foreign"' "a stack outside the CI prefix is never removed, at any age" +check 'project_exists "$bystander"' \ + "another run's stack is never removed by a narrowed sweep, even past the threshold" +echo "--- --prefix only ever narrows the sweep" +check '! bash scripts/ci-sweep.sh --stale --prefix "ekklesia-" >/dev/null 2>&1' \ + "a prefix outside the CI prefix is refused" +check '! bash scripts/ci-sweep.sh --job --prefix "$own_run" >/dev/null 2>&1' \ + "--prefix with --job is refused" if ((failures > 0)); then echo "test-ci-sweep: ${failures} check(s) failed." >&2 diff --git a/tests/ci-scripts.test.ts b/tests/ci-scripts.test.ts index 33439b9..d93423b 100644 --- a/tests/ci-scripts.test.ts +++ b/tests/ci-scripts.test.ts @@ -128,6 +128,34 @@ describe("CI scripts on a shared Docker daemon", () => { expect(workflow.jobs["deploy-smoke"]?.env?.EKKLESIA_CI_JOB_KEY).toContain("matrix.topology"); }); + it("never sweep another run's live stack early, not even from the sweep's own self-test", async () => { + // The workflow's stale sweeps keep the default threshold, which is far above any job's run time. + const workflow = await source(".github/workflows/ci.yml"); + const workflowSweeps = workflow + .split("\n") + .filter((line) => line.includes("ci-sweep.sh --stale")); + expect(workflowSweeps.length).toBeGreaterThan(0); + for (const line of workflowSweeps) { + expect(line).not.toContain("--max-age-seconds"); + } + + // The self-test does use short thresholds, so every stale sweep it runs is confined to its own + // fake run. Unconfined, `--max-age-seconds 0` removed every CI stack on the shared daemon — + // including a module repository's integration stack mid-run. + const selfTest = await source("scripts/test-ci-sweep.sh"); + expect(selfTest).toMatch(/^own_run="\$\(ci_name_safe "ekklesia-ci-\$\{GITHUB_RUN_ID\}-"\)"$/mu); + expect(selfTest.indexOf('export GITHUB_RUN_ID="sweeptest')).toBeLessThan( + selfTest.indexOf("own_run="), + ); + const selfTestSweeps = selfTest + .split("\n") + .filter((line) => /^\s*bash scripts\/ci-sweep\.sh --stale/u.test(line)); + expect(selfTestSweeps.length).toBeGreaterThan(0); + for (const line of selfTestSweeps) { + expect(line).toContain('--prefix "$own_run"'); + } + }); + it("choose the deploy smoke's edge subnet from those not in use, and choose again on a race", async () => { const deploy = await source("scripts/smoke-deploy.sh"); expect(deploy).not.toMatch(/^octet=\$\(\(RANDOM/mu);