scripts/test-ci-sweep.sh

@@ -0,0 +1,128 @@
+#!/usr/bin/env bash
+# Proves scripts/ci-sweep.sh against real Docker resources: what it removes, and above all what it
+# leaves alone. CI runs it before trusting the sweep with a shared runner's daemon.
+#
+#   bash scripts/test-ci-sweep.sh
+#
+# Every resource it creates is named and labeled for this run alone, under fake run identifiers, so
+# it can never touch a real stack; and it removes all of them on exit, whatever happens.
+
+set -Eeuo pipefail
+
+repository_root="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")/.." && pwd)"
+cd "$repository_root"
+
+image="$(sed -n 's/^[[:space:]]*image: \(postgres:[^[:space:]]*\)$/\1/p' deploy/compose.yaml | head -1)"
+[[ -n "$image" ]] || {
+  echo "test-ci-sweep: no pinned PostgreSQL image in deploy/compose.yaml." >&2
+  exit 1
+}
+docker image inspect "$image" >/dev/null 2>&1 || docker pull --quiet "$image" >/dev/null
+
+nonce="$(openssl rand -hex 4)"
+compose_label="com.docker.compose.project"
+ci_label="org.riverofglorymn.ekklesia.ci.project"
+failures=0
+created=()
+
+check() {
+  if eval "$1"; then
+    echo "ok   $2"
+  else
+    echo "FAIL $2"
+    failures=$((failures + 1))
+  fi
+}
+
+# project_exists <project>: any container, network, volume, or image still carries its label.
+project_exists() {
+  local project="$1"
+  [[ -n "$(
+    docker ps --all --quiet --filter "label=${compose_label}=${project}"
+    docker ps --all --quiet --filter "label=${ci_label}=${project}"
+    docker network ls --quiet --filter "label=${compose_label}=${project}"
+    docker volume ls --quiet --filter "label=${compose_label}=${project}"
+    docker image ls --quiet --filter "label=${ci_label}=${project}"
+  )" ]]
+}
+
+staging_exists() {
+  docker run --rm --entrypoint test -v /tmp:/host-tmp "$image" -d "/host-tmp/$1"
+}
+
+# make_stack <project>: what a CI script leaves behind when it is killed: a Compose-labeled network,
+# volume, and running container; a CI-labeled container and image; and a staging directory on the
+# Docker host.
+make_stack() {
+  local project="$1"
+  created+=("$project")
+  docker network create --label "${compose_label}=${project}" "${project}_default" >/dev/null
+  docker volume create --label "${compose_label}=${project}" "${project}_data" >/dev/null
+  docker run --detach --name "${project}-db" --label "${compose_label}=${project}" \
+    --network "${project}_default" -v "${project}_data:/data" --entrypoint sleep "$image" 600 >/dev/null
+  docker run --detach --name "${project}-browser" --label "${ci_label}=${project}" \
+    --entrypoint sleep "$image" 600 >/dev/null
+  printf 'FROM %s\nLABEL %s=%s\n' "$image" "$ci_label" "$project" |
+    docker build --quiet --tag "${project}-image:test" - >/dev/null
+  docker run --rm --entrypoint mkdir -v /tmp:/host-tmp "$image" -p "/host-tmp/${project}"
+}
+
+job_container="test-ci-sweep-job-${nonce}"
+cleanup() {
+  local status=$?
+  trap - EXIT INT TERM
+  docker rm --force "$job_container" >/dev/null 2>&1 || true
+  for project in "${created[@]}"; do
+    docker ps --all --quiet --filter "label=${compose_label}=${project}" | xargs -r docker rm --force >/dev/null 2>&1 || true
+    docker ps --all --quiet --filter "label=${ci_label}=${project}" | xargs -r docker rm --force >/dev/null 2>&1 || true
+    docker network rm "${project}_default" >/dev/null 2>&1 || true
+    docker volume rm --force "${project}_data" >/dev/null 2>&1 || true
+    docker image rm --force "${project}-image:test" >/dev/null 2>&1 || true
+    docker run --rm --entrypoint rm -v /tmp:/host-tmp "$image" -rf "/host-tmp/${project}" >/dev/null 2>&1 || true
+  done
+  exit "$status"
+}
+trap cleanup EXIT INT TERM
+
+# shellcheck source=scripts/lib/ci-project.sh
+source "$repository_root/scripts/lib/ci-project.sh"
+
+# Fake run identifiers: nothing here can match a real run's stacks.
+export GITHUB_RUN_ID="sweeptest${nonce}"
+export GITHUB_RUN_ATTEMPT="1"
+mine="$(EKKLESIA_CI_JOB_KEY=alpha ci_project_name mine)"
+sibling="$(EKKLESIA_CI_JOB_KEY=beta ci_project_name sibling)"
+foreign="ekklesia-sweep-control-${nonce}"
+for project in "$mine" "$sibling" "$foreign"; do
+  make_stack "$project"
+done
+# A job container connected to this job's stack network, exactly as compose-job-network.sh does.
+docker run --detach --name "$job_container" --entrypoint sleep "$image" 600 >/dev/null
+docker network connect "${mine}_default" "$job_container"
+
+echo "--- --job removes this job's stacks, at any age, and nothing else"
+EKKLESIA_CI_JOB_KEY=alpha bash scripts/ci-sweep.sh --job
+check '! project_exists "$mine"' "this job's containers, network, volume, and image are gone"
+check '! staging_exists "$mine"' "this job's staging directory on the Docker host is gone"
+check 'project_exists "$sibling"' "another job's stack in the same run is left alone"
+check 'project_exists "$foreign"' "a stack outside the CI prefix is left alone"
+check 'docker inspect "$job_container" >/dev/null 2>&1' \
+  "a job container connected to the stack's network is disconnected, never removed"
+
+echo "--- --job refuses to run outside CI, where it could remove a developer's own stacks"
+check '! env -u GITHUB_RUN_ID bash scripts/ci-sweep.sh --job >/dev/null 2>&1' \
+  "--job without GITHUB_RUN_ID fails"
+
+echo "--- --stale removes only CI stacks older than its threshold"
+bash scripts/ci-sweep.sh --stale
+check 'project_exists "$sibling"' "a young CI stack survives the default threshold"
+bash scripts/ci-sweep.sh --stale --max-age-seconds 0
+check '! project_exists "$sibling"' "a CI stack older than the threshold is removed"
+check '! staging_exists "$sibling"' "its staging directory too"
+check 'project_exists "$foreign"' "a stack outside the CI prefix is never removed, at any age"
+
+if ((failures > 0)); then
+  echo "test-ci-sweep: ${failures} check(s) failed." >&2
+  exit 1
+fi
+echo "CI sweep checks passed."

tests/ci-scripts.test.ts

@@ -4,6 +4,7 @@ import path from "node:path";
 import { fileURLToPath } from "node:url";
 import { promisify } from "node:util";
 import { describe, expect, it } from "vitest";
+import { parse } from "yaml";
 
 /**
  * The scripts CI runs against a shared Docker daemon. Several runs, and several jobs of one run, can
@@ -59,21 +60,68 @@ describe("CI scripts on a shared Docker daemon", () => {
     }
   });
 
-  it("give every Compose project a name unique to the invocation, not only the run", async () => {
+  it("name every stack through one helper: the reserved CI prefix, the job's key, and a unique suffix", async () => {
-    expect(await source("scripts/test-migrations-integration.sh")).toMatch(
+    const helper = await source("scripts/lib/ci-project.sh");
-      /raw_project="ekklesia-migration-[^"\n]*\$\$-\$\{RANDOM\}\$\{RANDOM\}"/u,
+    // Run, attempt, job key, kind, then a per-invocation suffix: unique even when job containers
+    // share a PID, and findable later by run and job alone (scripts/ci-sweep.sh --job).
+    expect(helper).toContain(
+      "ekklesia-ci-${GITHUB_RUN_ID:-local}-${GITHUB_RUN_ATTEMPT:-0}-${EKKLESIA_CI_JOB_KEY:-${GITHUB_JOB:-local}}-",
     );
-    expect(await source("scripts/smoke-infrastructure.sh")).toMatch(
+    expect(helper).toContain("$$-${RANDOM}${RANDOM}");
-      /raw_project="ekklesia-smoke-[^"\n]*\$\$-\$\{RANDOM\}\$\{RANDOM\}"/u,
-    );
-    const deploy = await source("scripts/smoke-deploy.sh");
-    expect(deploy).toContain('run_id="$(openssl rand -hex 4)"');
-    expect(deploy).toContain('project="ekklesia-deploy-smoke-${run_id}"');
     for (const relativePath of ciScripts) {
-      expect(await source(relativePath), relativePath).toContain('--project-name "$project"');
+      const text = await source(relativePath);
+      expect(text, relativePath).toContain('source "$repository_root/scripts/lib/ci-project.sh"');
+      expect(text, relativePath).toMatch(/^project="\$\(ci_project_name [a-z"${}_-]+\)"$/mu);
+      expect(text, relativePath).not.toContain("raw_project=");
+      expect(text, relativePath).toContain('--project-name "$project"');
     }
   });
 
+  it("label everything the deploy smoke creates outside Compose, so a sweep can find it", async () => {
+    // Continuation lines joined, so each command is checked whole.
+    const deploy = (await source("scripts/smoke-deploy.sh")).replace(/\\\n\s*/gu, " ");
+    const creating = deploy
+      .split("\n")
+      .filter((line) => /\bdocker (?:run --detach|volume create|build)\b/u.test(line));
+    expect(creating.length).toBeGreaterThanOrEqual(4);
+    for (const line of creating) {
+      expect(line).toContain('--label "${EKKLESIA_CI_LABEL}=${project}"');
+    }
+  });
+
+  it("sweep stale stacks before, and this job's own stacks after, every job that starts any", async () => {
+    const workflow = parse(await source(".github/workflows/ci.yml")) as {
+      jobs: Record<
+        string,
+        { env?: Record<string, string>; steps: { if?: string; run?: string; name?: string }[] }
+      >;
+    };
+    const startsStacks =
+      /smoke-infrastructure\.sh|test:migrations:integration|smoke-deploy\.sh|docker build/u;
+    const jobs = Object.entries(workflow.jobs).filter(([, job]) =>
+      job.steps.some((step) => startsStacks.test(step.run ?? "")),
+    );
+    expect(jobs.map(([id]) => id).sort()).toEqual(
+      ["deploy-smoke", "image-build", "infrastructure-smoke", "migration-smoke"].sort(),
+    );
+    for (const [id, job] of jobs) {
+      expect(job.env?.EKKLESIA_CI_JOB_KEY, id).toBeTruthy();
+      const runs = job.steps.map((step) => step.run ?? "");
+      const firstStack = runs.findIndex((run) => startsStacks.test(run));
+      const stale = runs.findIndex((run) => run.includes("bash scripts/ci-sweep.sh --stale"));
+      const own = job.steps.findIndex(
+        (step) =>
+          step.if === "always()" && (step.run ?? "").includes("bash scripts/ci-sweep.sh --job"),
+      );
+      expect(stale, `${id}: stale sweep`).toBeGreaterThanOrEqual(0);
+      expect(stale, `${id}: stale sweep runs first`).toBeLessThan(firstStack);
+      expect(own, `${id}: its own cleanup, always`).toBeGreaterThan(firstStack);
+      expect(own, `${id}: cleanup is the last step`).toBe(job.steps.length - 1);
+    }
+    // One matrix leg's cleanup must never remove the other leg's running stack.
+    expect(workflow.jobs["deploy-smoke"]?.env?.EKKLESIA_CI_JOB_KEY).toContain("matrix.topology");
+  });
+
   it("choose the deploy smoke's edge subnet from those not in use, and choose again on a race", async () => {
     const deploy = await source("scripts/smoke-deploy.sh");
     expect(deploy).not.toMatch(/^octet=\$\(\(RANDOM/mu);