test(ci): every CI stack is cleaned up, even when its job is killed
A migration-smoke stack (PostgreSQL and SeaweedFS) leaked on the shared runner and ran for two days, until it was removed by hand. Every CI script already tears its stack down in an EXIT/INT/TERM trap, but a trap never runs when the runner kills the job container: a cancelled job, or the runner's own time limit, since Gitea 1.24 ignores timeout-minutes. Static checks (tests/ci-scripts.test.ts): - every stack script names its project through one helper: the reserved ekklesia-ci- prefix, run, attempt, job key, and a unique suffix; - everything the deploy smoke creates outside Compose carries the CI label; - every workflow job that starts stacks sweeps stale CI stacks first, and removes its own in a final if: always() step. Each matrix leg has its own key, so one cannot remove the other's running stack. Behaviour against real Docker (scripts/test-ci-sweep.sh): - --job removes this job's containers, network, volume, image and staging directory, and leaves other jobs' and non-CI stacks alone; - --job disconnects, never removes, a job container on the stack's network, and refuses to run outside CI; - --stale spares young stacks and removes old ones, never anything outside the prefix. Fails today: there is no helper, no sweep, no label and no step. Co-Authored-By: Claude Opus 5.5 <[email protected]>
This commit is contained in:
ekklesia
co-authored by
Claude Opus 5.5
Claude Opus 5.5
parent
2152eb036a
commit
95440b37ea
Executable
+128
@@ -0,0 +1,128 @@
|
|||||||
|
#!/usr/bin/env bash
|
||||||
|
# Proves scripts/ci-sweep.sh against real Docker resources: what it removes, and above all what it
|
||||||
|
# leaves alone. CI runs it before trusting the sweep with a shared runner's daemon.
|
||||||
|
#
|
||||||
|
# bash scripts/test-ci-sweep.sh
|
||||||
|
#
|
||||||
|
# Every resource it creates is named and labeled for this run alone, under fake run identifiers, so
|
||||||
|
# it can never touch a real stack; and it removes all of them on exit, whatever happens.
|
||||||
|
|
||||||
|
set -Eeuo pipefail
|
||||||
|
|
||||||
|
repository_root="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")/.." && pwd)"
|
||||||
|
cd "$repository_root"
|
||||||
|
|
||||||
|
image="$(sed -n 's/^[[:space:]]*image: \(postgres:[^[:space:]]*\)$/\1/p' deploy/compose.yaml | head -1)"
|
||||||
|
[[ -n "$image" ]] || {
|
||||||
|
echo "test-ci-sweep: no pinned PostgreSQL image in deploy/compose.yaml." >&2
|
||||||
|
exit 1
|
||||||
|
}
|
||||||
|
docker image inspect "$image" >/dev/null 2>&1 || docker pull --quiet "$image" >/dev/null
|
||||||
|
|
||||||
|
nonce="$(openssl rand -hex 4)"
|
||||||
|
compose_label="com.docker.compose.project"
|
||||||
|
ci_label="org.riverofglorymn.ekklesia.ci.project"
|
||||||
|
failures=0
|
||||||
|
created=()
|
||||||
|
|
||||||
|
check() {
|
||||||
|
if eval "$1"; then
|
||||||
|
echo "ok $2"
|
||||||
|
else
|
||||||
|
echo "FAIL $2"
|
||||||
|
failures=$((failures + 1))
|
||||||
|
fi
|
||||||
|
}
|
||||||
|
|
||||||
|
# project_exists <project>: any container, network, volume, or image still carries its label.
|
||||||
|
project_exists() {
|
||||||
|
local project="$1"
|
||||||
|
[[ -n "$(
|
||||||
|
docker ps --all --quiet --filter "label=${compose_label}=${project}"
|
||||||
|
docker ps --all --quiet --filter "label=${ci_label}=${project}"
|
||||||
|
docker network ls --quiet --filter "label=${compose_label}=${project}"
|
||||||
|
docker volume ls --quiet --filter "label=${compose_label}=${project}"
|
||||||
|
docker image ls --quiet --filter "label=${ci_label}=${project}"
|
||||||
|
)" ]]
|
||||||
|
}
|
||||||
|
|
||||||
|
staging_exists() {
|
||||||
|
docker run --rm --entrypoint test -v /tmp:/host-tmp "$image" -d "/host-tmp/$1"
|
||||||
|
}
|
||||||
|
|
||||||
|
# make_stack <project>: what a CI script leaves behind when it is killed: a Compose-labeled network,
|
||||||
|
# volume, and running container; a CI-labeled container and image; and a staging directory on the
|
||||||
|
# Docker host.
|
||||||
|
make_stack() {
|
||||||
|
local project="$1"
|
||||||
|
created+=("$project")
|
||||||
|
docker network create --label "${compose_label}=${project}" "${project}_default" >/dev/null
|
||||||
|
docker volume create --label "${compose_label}=${project}" "${project}_data" >/dev/null
|
||||||
|
docker run --detach --name "${project}-db" --label "${compose_label}=${project}" \
|
||||||
|
--network "${project}_default" -v "${project}_data:/data" --entrypoint sleep "$image" 600 >/dev/null
|
||||||
|
docker run --detach --name "${project}-browser" --label "${ci_label}=${project}" \
|
||||||
|
--entrypoint sleep "$image" 600 >/dev/null
|
||||||
|
printf 'FROM %s\nLABEL %s=%s\n' "$image" "$ci_label" "$project" |
|
||||||
|
docker build --quiet --tag "${project}-image:test" - >/dev/null
|
||||||
|
docker run --rm --entrypoint mkdir -v /tmp:/host-tmp "$image" -p "/host-tmp/${project}"
|
||||||
|
}
|
||||||
|
|
||||||
|
job_container="test-ci-sweep-job-${nonce}"
|
||||||
|
cleanup() {
|
||||||
|
local status=$?
|
||||||
|
trap - EXIT INT TERM
|
||||||
|
docker rm --force "$job_container" >/dev/null 2>&1 || true
|
||||||
|
for project in "${created[@]}"; do
|
||||||
|
docker ps --all --quiet --filter "label=${compose_label}=${project}" | xargs -r docker rm --force >/dev/null 2>&1 || true
|
||||||
|
docker ps --all --quiet --filter "label=${ci_label}=${project}" | xargs -r docker rm --force >/dev/null 2>&1 || true
|
||||||
|
docker network rm "${project}_default" >/dev/null 2>&1 || true
|
||||||
|
docker volume rm --force "${project}_data" >/dev/null 2>&1 || true
|
||||||
|
docker image rm --force "${project}-image:test" >/dev/null 2>&1 || true
|
||||||
|
docker run --rm --entrypoint rm -v /tmp:/host-tmp "$image" -rf "/host-tmp/${project}" >/dev/null 2>&1 || true
|
||||||
|
done
|
||||||
|
exit "$status"
|
||||||
|
}
|
||||||
|
trap cleanup EXIT INT TERM
|
||||||
|
|
||||||
|
# shellcheck source=scripts/lib/ci-project.sh
|
||||||
|
source "$repository_root/scripts/lib/ci-project.sh"
|
||||||
|
|
||||||
|
# Fake run identifiers: nothing here can match a real run's stacks.
|
||||||
|
export GITHUB_RUN_ID="sweeptest${nonce}"
|
||||||
|
export GITHUB_RUN_ATTEMPT="1"
|
||||||
|
mine="$(EKKLESIA_CI_JOB_KEY=alpha ci_project_name mine)"
|
||||||
|
sibling="$(EKKLESIA_CI_JOB_KEY=beta ci_project_name sibling)"
|
||||||
|
foreign="ekklesia-sweep-control-${nonce}"
|
||||||
|
for project in "$mine" "$sibling" "$foreign"; do
|
||||||
|
make_stack "$project"
|
||||||
|
done
|
||||||
|
# A job container connected to this job's stack network, exactly as compose-job-network.sh does.
|
||||||
|
docker run --detach --name "$job_container" --entrypoint sleep "$image" 600 >/dev/null
|
||||||
|
docker network connect "${mine}_default" "$job_container"
|
||||||
|
|
||||||
|
echo "--- --job removes this job's stacks, at any age, and nothing else"
|
||||||
|
EKKLESIA_CI_JOB_KEY=alpha bash scripts/ci-sweep.sh --job
|
||||||
|
check '! project_exists "$mine"' "this job's containers, network, volume, and image are gone"
|
||||||
|
check '! staging_exists "$mine"' "this job's staging directory on the Docker host is gone"
|
||||||
|
check 'project_exists "$sibling"' "another job's stack in the same run is left alone"
|
||||||
|
check 'project_exists "$foreign"' "a stack outside the CI prefix is left alone"
|
||||||
|
check 'docker inspect "$job_container" >/dev/null 2>&1' \
|
||||||
|
"a job container connected to the stack's network is disconnected, never removed"
|
||||||
|
|
||||||
|
echo "--- --job refuses to run outside CI, where it could remove a developer's own stacks"
|
||||||
|
check '! env -u GITHUB_RUN_ID bash scripts/ci-sweep.sh --job >/dev/null 2>&1' \
|
||||||
|
"--job without GITHUB_RUN_ID fails"
|
||||||
|
|
||||||
|
echo "--- --stale removes only CI stacks older than its threshold"
|
||||||
|
bash scripts/ci-sweep.sh --stale
|
||||||
|
check 'project_exists "$sibling"' "a young CI stack survives the default threshold"
|
||||||
|
bash scripts/ci-sweep.sh --stale --max-age-seconds 0
|
||||||
|
check '! project_exists "$sibling"' "a CI stack older than the threshold is removed"
|
||||||
|
check '! staging_exists "$sibling"' "its staging directory too"
|
||||||
|
check 'project_exists "$foreign"' "a stack outside the CI prefix is never removed, at any age"
|
||||||
|
|
||||||
|
if ((failures > 0)); then
|
||||||
|
echo "test-ci-sweep: ${failures} check(s) failed." >&2
|
||||||
|
exit 1
|
||||||
|
fi
|
||||||
|
echo "CI sweep checks passed."
|
||||||
+58
-10
@@ -4,6 +4,7 @@ import path from "node:path";
|
|||||||
import { fileURLToPath } from "node:url";
|
import { fileURLToPath } from "node:url";
|
||||||
import { promisify } from "node:util";
|
import { promisify } from "node:util";
|
||||||
import { describe, expect, it } from "vitest";
|
import { describe, expect, it } from "vitest";
|
||||||
|
import { parse } from "yaml";
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* The scripts CI runs against a shared Docker daemon. Several runs, and several jobs of one run, can
|
* The scripts CI runs against a shared Docker daemon. Several runs, and several jobs of one run, can
|
||||||
@@ -59,21 +60,68 @@ describe("CI scripts on a shared Docker daemon", () => {
|
|||||||
}
|
}
|
||||||
});
|
});
|
||||||
|
|
||||||
it("give every Compose project a name unique to the invocation, not only the run", async () => {
|
it("name every stack through one helper: the reserved CI prefix, the job's key, and a unique suffix", async () => {
|
||||||
expect(await source("scripts/test-migrations-integration.sh")).toMatch(
|
const helper = await source("scripts/lib/ci-project.sh");
|
||||||
/raw_project="ekklesia-migration-[^"\n]*\$\$-\$\{RANDOM\}\$\{RANDOM\}"/u,
|
// Run, attempt, job key, kind, then a per-invocation suffix: unique even when job containers
|
||||||
|
// share a PID, and findable later by run and job alone (scripts/ci-sweep.sh --job).
|
||||||
|
expect(helper).toContain(
|
||||||
|
"ekklesia-ci-${GITHUB_RUN_ID:-local}-${GITHUB_RUN_ATTEMPT:-0}-${EKKLESIA_CI_JOB_KEY:-${GITHUB_JOB:-local}}-",
|
||||||
);
|
);
|
||||||
expect(await source("scripts/smoke-infrastructure.sh")).toMatch(
|
expect(helper).toContain("$$-${RANDOM}${RANDOM}");
|
||||||
/raw_project="ekklesia-smoke-[^"\n]*\$\$-\$\{RANDOM\}\$\{RANDOM\}"/u,
|
|
||||||
);
|
|
||||||
const deploy = await source("scripts/smoke-deploy.sh");
|
|
||||||
expect(deploy).toContain('run_id="$(openssl rand -hex 4)"');
|
|
||||||
expect(deploy).toContain('project="ekklesia-deploy-smoke-${run_id}"');
|
|
||||||
for (const relativePath of ciScripts) {
|
for (const relativePath of ciScripts) {
|
||||||
expect(await source(relativePath), relativePath).toContain('--project-name "$project"');
|
const text = await source(relativePath);
|
||||||
|
expect(text, relativePath).toContain('source "$repository_root/scripts/lib/ci-project.sh"');
|
||||||
|
expect(text, relativePath).toMatch(/^project="\$\(ci_project_name [a-z"${}_-]+\)"$/mu);
|
||||||
|
expect(text, relativePath).not.toContain("raw_project=");
|
||||||
|
expect(text, relativePath).toContain('--project-name "$project"');
|
||||||
}
|
}
|
||||||
});
|
});
|
||||||
|
|
||||||
|
it("label everything the deploy smoke creates outside Compose, so a sweep can find it", async () => {
|
||||||
|
// Continuation lines joined, so each command is checked whole.
|
||||||
|
const deploy = (await source("scripts/smoke-deploy.sh")).replace(/\\\n\s*/gu, " ");
|
||||||
|
const creating = deploy
|
||||||
|
.split("\n")
|
||||||
|
.filter((line) => /\bdocker (?:run --detach|volume create|build)\b/u.test(line));
|
||||||
|
expect(creating.length).toBeGreaterThanOrEqual(4);
|
||||||
|
for (const line of creating) {
|
||||||
|
expect(line).toContain('--label "${EKKLESIA_CI_LABEL}=${project}"');
|
||||||
|
}
|
||||||
|
});
|
||||||
|
|
||||||
|
it("sweep stale stacks before, and this job's own stacks after, every job that starts any", async () => {
|
||||||
|
const workflow = parse(await source(".github/workflows/ci.yml")) as {
|
||||||
|
jobs: Record<
|
||||||
|
string,
|
||||||
|
{ env?: Record<string, string>; steps: { if?: string; run?: string; name?: string }[] }
|
||||||
|
>;
|
||||||
|
};
|
||||||
|
const startsStacks =
|
||||||
|
/smoke-infrastructure\.sh|test:migrations:integration|smoke-deploy\.sh|docker build/u;
|
||||||
|
const jobs = Object.entries(workflow.jobs).filter(([, job]) =>
|
||||||
|
job.steps.some((step) => startsStacks.test(step.run ?? "")),
|
||||||
|
);
|
||||||
|
expect(jobs.map(([id]) => id).sort()).toEqual(
|
||||||
|
["deploy-smoke", "image-build", "infrastructure-smoke", "migration-smoke"].sort(),
|
||||||
|
);
|
||||||
|
for (const [id, job] of jobs) {
|
||||||
|
expect(job.env?.EKKLESIA_CI_JOB_KEY, id).toBeTruthy();
|
||||||
|
const runs = job.steps.map((step) => step.run ?? "");
|
||||||
|
const firstStack = runs.findIndex((run) => startsStacks.test(run));
|
||||||
|
const stale = runs.findIndex((run) => run.includes("bash scripts/ci-sweep.sh --stale"));
|
||||||
|
const own = job.steps.findIndex(
|
||||||
|
(step) =>
|
||||||
|
step.if === "always()" && (step.run ?? "").includes("bash scripts/ci-sweep.sh --job"),
|
||||||
|
);
|
||||||
|
expect(stale, `${id}: stale sweep`).toBeGreaterThanOrEqual(0);
|
||||||
|
expect(stale, `${id}: stale sweep runs first`).toBeLessThan(firstStack);
|
||||||
|
expect(own, `${id}: its own cleanup, always`).toBeGreaterThan(firstStack);
|
||||||
|
expect(own, `${id}: cleanup is the last step`).toBe(job.steps.length - 1);
|
||||||
|
}
|
||||||
|
// One matrix leg's cleanup must never remove the other leg's running stack.
|
||||||
|
expect(workflow.jobs["deploy-smoke"]?.env?.EKKLESIA_CI_JOB_KEY).toContain("matrix.topology");
|
||||||
|
});
|
||||||
|
|
||||||
it("choose the deploy smoke's edge subnet from those not in use, and choose again on a race", async () => {
|
it("choose the deploy smoke's edge subnet from those not in use, and choose again on a race", async () => {
|
||||||
const deploy = await source("scripts/smoke-deploy.sh");
|
const deploy = await source("scripts/smoke-deploy.sh");
|
||||||
expect(deploy).not.toMatch(/^octet=\$\(\(RANDOM/mu);
|
expect(deploy).not.toMatch(/^octet=\$\(\(RANDOM/mu);
|
||||||
|
|||||||
Reference in New Issue
Block a user