diff --git a/scripts/test-ci-sweep.sh b/scripts/test-ci-sweep.sh new file mode 100755 index 0000000..ff8b938 --- /dev/null +++ b/scripts/test-ci-sweep.sh @@ -0,0 +1,128 @@ +#!/usr/bin/env bash +# Proves scripts/ci-sweep.sh against real Docker resources: what it removes, and above all what it +# leaves alone. CI runs it before trusting the sweep with a shared runner's daemon. +# +# bash scripts/test-ci-sweep.sh +# +# Every resource it creates is named and labeled for this run alone, under fake run identifiers, so +# it can never touch a real stack; and it removes all of them on exit, whatever happens. + +set -Eeuo pipefail + +repository_root="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")/.." && pwd)" +cd "$repository_root" + +image="$(sed -n 's/^[[:space:]]*image: \(postgres:[^[:space:]]*\)$/\1/p' deploy/compose.yaml | head -1)" +[[ -n "$image" ]] || { + echo "test-ci-sweep: no pinned PostgreSQL image in deploy/compose.yaml." >&2 + exit 1 +} +docker image inspect "$image" >/dev/null 2>&1 || docker pull --quiet "$image" >/dev/null + +nonce="$(openssl rand -hex 4)" +compose_label="com.docker.compose.project" +ci_label="org.riverofglorymn.ekklesia.ci.project" +failures=0 +created=() + +check() { + if eval "$1"; then + echo "ok $2" + else + echo "FAIL $2" + failures=$((failures + 1)) + fi +} + +# project_exists : any container, network, volume, or image still carries its label. +project_exists() { + local project="$1" + [[ -n "$( + docker ps --all --quiet --filter "label=${compose_label}=${project}" + docker ps --all --quiet --filter "label=${ci_label}=${project}" + docker network ls --quiet --filter "label=${compose_label}=${project}" + docker volume ls --quiet --filter "label=${compose_label}=${project}" + docker image ls --quiet --filter "label=${ci_label}=${project}" + )" ]] +} + +staging_exists() { + docker run --rm --entrypoint test -v /tmp:/host-tmp "$image" -d "/host-tmp/$1" +} + +# make_stack : what a CI script leaves behind when it is killed: a Compose-labeled network, +# volume, and running container; a CI-labeled container and image; and a staging directory on the +# Docker host. +make_stack() { + local project="$1" + created+=("$project") + docker network create --label "${compose_label}=${project}" "${project}_default" >/dev/null + docker volume create --label "${compose_label}=${project}" "${project}_data" >/dev/null + docker run --detach --name "${project}-db" --label "${compose_label}=${project}" \ + --network "${project}_default" -v "${project}_data:/data" --entrypoint sleep "$image" 600 >/dev/null + docker run --detach --name "${project}-browser" --label "${ci_label}=${project}" \ + --entrypoint sleep "$image" 600 >/dev/null + printf 'FROM %s\nLABEL %s=%s\n' "$image" "$ci_label" "$project" | + docker build --quiet --tag "${project}-image:test" - >/dev/null + docker run --rm --entrypoint mkdir -v /tmp:/host-tmp "$image" -p "/host-tmp/${project}" +} + +job_container="test-ci-sweep-job-${nonce}" +cleanup() { + local status=$? + trap - EXIT INT TERM + docker rm --force "$job_container" >/dev/null 2>&1 || true + for project in "${created[@]}"; do + docker ps --all --quiet --filter "label=${compose_label}=${project}" | xargs -r docker rm --force >/dev/null 2>&1 || true + docker ps --all --quiet --filter "label=${ci_label}=${project}" | xargs -r docker rm --force >/dev/null 2>&1 || true + docker network rm "${project}_default" >/dev/null 2>&1 || true + docker volume rm --force "${project}_data" >/dev/null 2>&1 || true + docker image rm --force "${project}-image:test" >/dev/null 2>&1 || true + docker run --rm --entrypoint rm -v /tmp:/host-tmp "$image" -rf "/host-tmp/${project}" >/dev/null 2>&1 || true + done + exit "$status" +} +trap cleanup EXIT INT TERM + +# shellcheck source=scripts/lib/ci-project.sh +source "$repository_root/scripts/lib/ci-project.sh" + +# Fake run identifiers: nothing here can match a real run's stacks. +export GITHUB_RUN_ID="sweeptest${nonce}" +export GITHUB_RUN_ATTEMPT="1" +mine="$(EKKLESIA_CI_JOB_KEY=alpha ci_project_name mine)" +sibling="$(EKKLESIA_CI_JOB_KEY=beta ci_project_name sibling)" +foreign="ekklesia-sweep-control-${nonce}" +for project in "$mine" "$sibling" "$foreign"; do + make_stack "$project" +done +# A job container connected to this job's stack network, exactly as compose-job-network.sh does. +docker run --detach --name "$job_container" --entrypoint sleep "$image" 600 >/dev/null +docker network connect "${mine}_default" "$job_container" + +echo "--- --job removes this job's stacks, at any age, and nothing else" +EKKLESIA_CI_JOB_KEY=alpha bash scripts/ci-sweep.sh --job +check '! project_exists "$mine"' "this job's containers, network, volume, and image are gone" +check '! staging_exists "$mine"' "this job's staging directory on the Docker host is gone" +check 'project_exists "$sibling"' "another job's stack in the same run is left alone" +check 'project_exists "$foreign"' "a stack outside the CI prefix is left alone" +check 'docker inspect "$job_container" >/dev/null 2>&1' \ + "a job container connected to the stack's network is disconnected, never removed" + +echo "--- --job refuses to run outside CI, where it could remove a developer's own stacks" +check '! env -u GITHUB_RUN_ID bash scripts/ci-sweep.sh --job >/dev/null 2>&1' \ + "--job without GITHUB_RUN_ID fails" + +echo "--- --stale removes only CI stacks older than its threshold" +bash scripts/ci-sweep.sh --stale +check 'project_exists "$sibling"' "a young CI stack survives the default threshold" +bash scripts/ci-sweep.sh --stale --max-age-seconds 0 +check '! project_exists "$sibling"' "a CI stack older than the threshold is removed" +check '! staging_exists "$sibling"' "its staging directory too" +check 'project_exists "$foreign"' "a stack outside the CI prefix is never removed, at any age" + +if ((failures > 0)); then + echo "test-ci-sweep: ${failures} check(s) failed." >&2 + exit 1 +fi +echo "CI sweep checks passed." diff --git a/tests/ci-scripts.test.ts b/tests/ci-scripts.test.ts index 7203ada..e498d70 100644 --- a/tests/ci-scripts.test.ts +++ b/tests/ci-scripts.test.ts @@ -4,6 +4,7 @@ import path from "node:path"; import { fileURLToPath } from "node:url"; import { promisify } from "node:util"; import { describe, expect, it } from "vitest"; +import { parse } from "yaml"; /** * The scripts CI runs against a shared Docker daemon. Several runs, and several jobs of one run, can @@ -59,21 +60,68 @@ describe("CI scripts on a shared Docker daemon", () => { } }); - it("give every Compose project a name unique to the invocation, not only the run", async () => { - expect(await source("scripts/test-migrations-integration.sh")).toMatch( - /raw_project="ekklesia-migration-[^"\n]*\$\$-\$\{RANDOM\}\$\{RANDOM\}"/u, + it("name every stack through one helper: the reserved CI prefix, the job's key, and a unique suffix", async () => { + const helper = await source("scripts/lib/ci-project.sh"); + // Run, attempt, job key, kind, then a per-invocation suffix: unique even when job containers + // share a PID, and findable later by run and job alone (scripts/ci-sweep.sh --job). + expect(helper).toContain( + "ekklesia-ci-${GITHUB_RUN_ID:-local}-${GITHUB_RUN_ATTEMPT:-0}-${EKKLESIA_CI_JOB_KEY:-${GITHUB_JOB:-local}}-", ); - expect(await source("scripts/smoke-infrastructure.sh")).toMatch( - /raw_project="ekklesia-smoke-[^"\n]*\$\$-\$\{RANDOM\}\$\{RANDOM\}"/u, - ); - const deploy = await source("scripts/smoke-deploy.sh"); - expect(deploy).toContain('run_id="$(openssl rand -hex 4)"'); - expect(deploy).toContain('project="ekklesia-deploy-smoke-${run_id}"'); + expect(helper).toContain("$$-${RANDOM}${RANDOM}"); for (const relativePath of ciScripts) { - expect(await source(relativePath), relativePath).toContain('--project-name "$project"'); + const text = await source(relativePath); + expect(text, relativePath).toContain('source "$repository_root/scripts/lib/ci-project.sh"'); + expect(text, relativePath).toMatch(/^project="\$\(ci_project_name [a-z"${}_-]+\)"$/mu); + expect(text, relativePath).not.toContain("raw_project="); + expect(text, relativePath).toContain('--project-name "$project"'); } }); + it("label everything the deploy smoke creates outside Compose, so a sweep can find it", async () => { + // Continuation lines joined, so each command is checked whole. + const deploy = (await source("scripts/smoke-deploy.sh")).replace(/\\\n\s*/gu, " "); + const creating = deploy + .split("\n") + .filter((line) => /\bdocker (?:run --detach|volume create|build)\b/u.test(line)); + expect(creating.length).toBeGreaterThanOrEqual(4); + for (const line of creating) { + expect(line).toContain('--label "${EKKLESIA_CI_LABEL}=${project}"'); + } + }); + + it("sweep stale stacks before, and this job's own stacks after, every job that starts any", async () => { + const workflow = parse(await source(".github/workflows/ci.yml")) as { + jobs: Record< + string, + { env?: Record; steps: { if?: string; run?: string; name?: string }[] } + >; + }; + const startsStacks = + /smoke-infrastructure\.sh|test:migrations:integration|smoke-deploy\.sh|docker build/u; + const jobs = Object.entries(workflow.jobs).filter(([, job]) => + job.steps.some((step) => startsStacks.test(step.run ?? "")), + ); + expect(jobs.map(([id]) => id).sort()).toEqual( + ["deploy-smoke", "image-build", "infrastructure-smoke", "migration-smoke"].sort(), + ); + for (const [id, job] of jobs) { + expect(job.env?.EKKLESIA_CI_JOB_KEY, id).toBeTruthy(); + const runs = job.steps.map((step) => step.run ?? ""); + const firstStack = runs.findIndex((run) => startsStacks.test(run)); + const stale = runs.findIndex((run) => run.includes("bash scripts/ci-sweep.sh --stale")); + const own = job.steps.findIndex( + (step) => + step.if === "always()" && (step.run ?? "").includes("bash scripts/ci-sweep.sh --job"), + ); + expect(stale, `${id}: stale sweep`).toBeGreaterThanOrEqual(0); + expect(stale, `${id}: stale sweep runs first`).toBeLessThan(firstStack); + expect(own, `${id}: its own cleanup, always`).toBeGreaterThan(firstStack); + expect(own, `${id}: cleanup is the last step`).toBe(job.steps.length - 1); + } + // One matrix leg's cleanup must never remove the other leg's running stack. + expect(workflow.jobs["deploy-smoke"]?.env?.EKKLESIA_CI_JOB_KEY).toContain("matrix.topology"); + }); + it("choose the deploy smoke's edge subnet from those not in use, and choose again on a race", async () => { const deploy = await source("scripts/smoke-deploy.sh"); expect(deploy).not.toMatch(/^octet=\$\(\(RANDOM/mu);