fix(ci): nothing a CI job starts outlives it, even when the runner kills the job #60
Executable
+128
@@ -0,0 +1,128 @@
|
||||
#!/usr/bin/env bash
|
||||
# Proves scripts/ci-sweep.sh against real Docker resources: what it removes, and above all what it
|
||||
# leaves alone. CI runs it before trusting the sweep with a shared runner's daemon.
|
||||
#
|
||||
# bash scripts/test-ci-sweep.sh
|
||||
#
|
||||
# Every resource it creates is named and labeled for this run alone, under fake run identifiers, so
|
||||
# it can never touch a real stack; and it removes all of them on exit, whatever happens.
|
||||
|
||||
set -Eeuo pipefail
|
||||
|
||||
repository_root="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")/.." && pwd)"
|
||||
cd "$repository_root"
|
||||
|
||||
image="$(sed -n 's/^[[:space:]]*image: \(postgres:[^[:space:]]*\)$/\1/p' deploy/compose.yaml | head -1)"
|
||||
[[ -n "$image" ]] || {
|
||||
echo "test-ci-sweep: no pinned PostgreSQL image in deploy/compose.yaml." >&2
|
||||
exit 1
|
||||
}
|
||||
docker image inspect "$image" >/dev/null 2>&1 || docker pull --quiet "$image" >/dev/null
|
||||
|
||||
nonce="$(openssl rand -hex 4)"
|
||||
compose_label="com.docker.compose.project"
|
||||
ci_label="org.riverofglorymn.ekklesia.ci.project"
|
||||
failures=0
|
||||
created=()
|
||||
|
||||
check() {
|
||||
if eval "$1"; then
|
||||
echo "ok $2"
|
||||
else
|
||||
echo "FAIL $2"
|
||||
failures=$((failures + 1))
|
||||
fi
|
||||
}
|
||||
|
||||
# project_exists <project>: any container, network, volume, or image still carries its label.
|
||||
project_exists() {
|
||||
local project="$1"
|
||||
[[ -n "$(
|
||||
docker ps --all --quiet --filter "label=${compose_label}=${project}"
|
||||
docker ps --all --quiet --filter "label=${ci_label}=${project}"
|
||||
docker network ls --quiet --filter "label=${compose_label}=${project}"
|
||||
docker volume ls --quiet --filter "label=${compose_label}=${project}"
|
||||
docker image ls --quiet --filter "label=${ci_label}=${project}"
|
||||
)" ]]
|
||||
}
|
||||
|
||||
staging_exists() {
|
||||
docker run --rm --entrypoint test -v /tmp:/host-tmp "$image" -d "/host-tmp/$1"
|
||||
}
|
||||
|
||||
# make_stack <project>: what a CI script leaves behind when it is killed: a Compose-labeled network,
|
||||
# volume, and running container; a CI-labeled container and image; and a staging directory on the
|
||||
# Docker host.
|
||||
make_stack() {
|
||||
local project="$1"
|
||||
created+=("$project")
|
||||
docker network create --label "${compose_label}=${project}" "${project}_default" >/dev/null
|
||||
docker volume create --label "${compose_label}=${project}" "${project}_data" >/dev/null
|
||||
docker run --detach --name "${project}-db" --label "${compose_label}=${project}" \
|
||||
--network "${project}_default" -v "${project}_data:/data" --entrypoint sleep "$image" 600 >/dev/null
|
||||
docker run --detach --name "${project}-browser" --label "${ci_label}=${project}" \
|
||||
--entrypoint sleep "$image" 600 >/dev/null
|
||||
printf 'FROM %s\nLABEL %s=%s\n' "$image" "$ci_label" "$project" |
|
||||
docker build --quiet --tag "${project}-image:test" - >/dev/null
|
||||
docker run --rm --entrypoint mkdir -v /tmp:/host-tmp "$image" -p "/host-tmp/${project}"
|
||||
}
|
||||
|
||||
job_container="test-ci-sweep-job-${nonce}"
|
||||
cleanup() {
|
||||
local status=$?
|
||||
trap - EXIT INT TERM
|
||||
docker rm --force "$job_container" >/dev/null 2>&1 || true
|
||||
for project in "${created[@]}"; do
|
||||
docker ps --all --quiet --filter "label=${compose_label}=${project}" | xargs -r docker rm --force >/dev/null 2>&1 || true
|
||||
docker ps --all --quiet --filter "label=${ci_label}=${project}" | xargs -r docker rm --force >/dev/null 2>&1 || true
|
||||
docker network rm "${project}_default" >/dev/null 2>&1 || true
|
||||
docker volume rm --force "${project}_data" >/dev/null 2>&1 || true
|
||||
docker image rm --force "${project}-image:test" >/dev/null 2>&1 || true
|
||||
docker run --rm --entrypoint rm -v /tmp:/host-tmp "$image" -rf "/host-tmp/${project}" >/dev/null 2>&1 || true
|
||||
done
|
||||
exit "$status"
|
||||
}
|
||||
trap cleanup EXIT INT TERM
|
||||
|
||||
# shellcheck source=scripts/lib/ci-project.sh
|
||||
source "$repository_root/scripts/lib/ci-project.sh"
|
||||
|
||||
# Fake run identifiers: nothing here can match a real run's stacks.
|
||||
export GITHUB_RUN_ID="sweeptest${nonce}"
|
||||
export GITHUB_RUN_ATTEMPT="1"
|
||||
mine="$(EKKLESIA_CI_JOB_KEY=alpha ci_project_name mine)"
|
||||
sibling="$(EKKLESIA_CI_JOB_KEY=beta ci_project_name sibling)"
|
||||
foreign="ekklesia-sweep-control-${nonce}"
|
||||
for project in "$mine" "$sibling" "$foreign"; do
|
||||
make_stack "$project"
|
||||
done
|
||||
# A job container connected to this job's stack network, exactly as compose-job-network.sh does.
|
||||
docker run --detach --name "$job_container" --entrypoint sleep "$image" 600 >/dev/null
|
||||
docker network connect "${mine}_default" "$job_container"
|
||||
|
||||
echo "--- --job removes this job's stacks, at any age, and nothing else"
|
||||
EKKLESIA_CI_JOB_KEY=alpha bash scripts/ci-sweep.sh --job
|
||||
check '! project_exists "$mine"' "this job's containers, network, volume, and image are gone"
|
||||
check '! staging_exists "$mine"' "this job's staging directory on the Docker host is gone"
|
||||
check 'project_exists "$sibling"' "another job's stack in the same run is left alone"
|
||||
check 'project_exists "$foreign"' "a stack outside the CI prefix is left alone"
|
||||
check 'docker inspect "$job_container" >/dev/null 2>&1' \
|
||||
"a job container connected to the stack's network is disconnected, never removed"
|
||||
|
||||
echo "--- --job refuses to run outside CI, where it could remove a developer's own stacks"
|
||||
check '! env -u GITHUB_RUN_ID bash scripts/ci-sweep.sh --job >/dev/null 2>&1' \
|
||||
"--job without GITHUB_RUN_ID fails"
|
||||
|
||||
echo "--- --stale removes only CI stacks older than its threshold"
|
||||
bash scripts/ci-sweep.sh --stale
|
||||
check 'project_exists "$sibling"' "a young CI stack survives the default threshold"
|
||||
bash scripts/ci-sweep.sh --stale --max-age-seconds 0
|
||||
check '! project_exists "$sibling"' "a CI stack older than the threshold is removed"
|
||||
check '! staging_exists "$sibling"' "its staging directory too"
|
||||
check 'project_exists "$foreign"' "a stack outside the CI prefix is never removed, at any age"
|
||||
|
||||
if ((failures > 0)); then
|
||||
echo "test-ci-sweep: ${failures} check(s) failed." >&2
|
||||
exit 1
|
||||
fi
|
||||
echo "CI sweep checks passed."
|
||||
+58
-10
@@ -4,6 +4,7 @@ import path from "node:path";
|
||||
import { fileURLToPath } from "node:url";
|
||||
import { promisify } from "node:util";
|
||||
import { describe, expect, it } from "vitest";
|
||||
import { parse } from "yaml";
|
||||
|
||||
/**
|
||||
* The scripts CI runs against a shared Docker daemon. Several runs, and several jobs of one run, can
|
||||
@@ -59,21 +60,68 @@ describe("CI scripts on a shared Docker daemon", () => {
|
||||
}
|
||||
});
|
||||
|
||||
it("give every Compose project a name unique to the invocation, not only the run", async () => {
|
||||
expect(await source("scripts/test-migrations-integration.sh")).toMatch(
|
||||
/raw_project="ekklesia-migration-[^"\n]*\$\$-\$\{RANDOM\}\$\{RANDOM\}"/u,
|
||||
it("name every stack through one helper: the reserved CI prefix, the job's key, and a unique suffix", async () => {
|
||||
const helper = await source("scripts/lib/ci-project.sh");
|
||||
// Run, attempt, job key, kind, then a per-invocation suffix: unique even when job containers
|
||||
// share a PID, and findable later by run and job alone (scripts/ci-sweep.sh --job).
|
||||
expect(helper).toContain(
|
||||
"ekklesia-ci-${GITHUB_RUN_ID:-local}-${GITHUB_RUN_ATTEMPT:-0}-${EKKLESIA_CI_JOB_KEY:-${GITHUB_JOB:-local}}-",
|
||||
);
|
||||
expect(await source("scripts/smoke-infrastructure.sh")).toMatch(
|
||||
/raw_project="ekklesia-smoke-[^"\n]*\$\$-\$\{RANDOM\}\$\{RANDOM\}"/u,
|
||||
);
|
||||
const deploy = await source("scripts/smoke-deploy.sh");
|
||||
expect(deploy).toContain('run_id="$(openssl rand -hex 4)"');
|
||||
expect(deploy).toContain('project="ekklesia-deploy-smoke-${run_id}"');
|
||||
expect(helper).toContain("$$-${RANDOM}${RANDOM}");
|
||||
for (const relativePath of ciScripts) {
|
||||
expect(await source(relativePath), relativePath).toContain('--project-name "$project"');
|
||||
const text = await source(relativePath);
|
||||
expect(text, relativePath).toContain('source "$repository_root/scripts/lib/ci-project.sh"');
|
||||
expect(text, relativePath).toMatch(/^project="\$\(ci_project_name [a-z"${}_-]+\)"$/mu);
|
||||
expect(text, relativePath).not.toContain("raw_project=");
|
||||
expect(text, relativePath).toContain('--project-name "$project"');
|
||||
}
|
||||
});
|
||||
|
||||
it("label everything the deploy smoke creates outside Compose, so a sweep can find it", async () => {
|
||||
// Continuation lines joined, so each command is checked whole.
|
||||
const deploy = (await source("scripts/smoke-deploy.sh")).replace(/\\\n\s*/gu, " ");
|
||||
const creating = deploy
|
||||
.split("\n")
|
||||
.filter((line) => /\bdocker (?:run --detach|volume create|build)\b/u.test(line));
|
||||
expect(creating.length).toBeGreaterThanOrEqual(4);
|
||||
for (const line of creating) {
|
||||
expect(line).toContain('--label "${EKKLESIA_CI_LABEL}=${project}"');
|
||||
}
|
||||
});
|
||||
|
||||
it("sweep stale stacks before, and this job's own stacks after, every job that starts any", async () => {
|
||||
const workflow = parse(await source(".github/workflows/ci.yml")) as {
|
||||
jobs: Record<
|
||||
string,
|
||||
{ env?: Record<string, string>; steps: { if?: string; run?: string; name?: string }[] }
|
||||
>;
|
||||
};
|
||||
const startsStacks =
|
||||
/smoke-infrastructure\.sh|test:migrations:integration|smoke-deploy\.sh|docker build/u;
|
||||
const jobs = Object.entries(workflow.jobs).filter(([, job]) =>
|
||||
job.steps.some((step) => startsStacks.test(step.run ?? "")),
|
||||
);
|
||||
expect(jobs.map(([id]) => id).sort()).toEqual(
|
||||
["deploy-smoke", "image-build", "infrastructure-smoke", "migration-smoke"].sort(),
|
||||
);
|
||||
for (const [id, job] of jobs) {
|
||||
expect(job.env?.EKKLESIA_CI_JOB_KEY, id).toBeTruthy();
|
||||
const runs = job.steps.map((step) => step.run ?? "");
|
||||
const firstStack = runs.findIndex((run) => startsStacks.test(run));
|
||||
const stale = runs.findIndex((run) => run.includes("bash scripts/ci-sweep.sh --stale"));
|
||||
const own = job.steps.findIndex(
|
||||
(step) =>
|
||||
step.if === "always()" && (step.run ?? "").includes("bash scripts/ci-sweep.sh --job"),
|
||||
);
|
||||
expect(stale, `${id}: stale sweep`).toBeGreaterThanOrEqual(0);
|
||||
expect(stale, `${id}: stale sweep runs first`).toBeLessThan(firstStack);
|
||||
expect(own, `${id}: its own cleanup, always`).toBeGreaterThan(firstStack);
|
||||
expect(own, `${id}: cleanup is the last step`).toBe(job.steps.length - 1);
|
||||
}
|
||||
// One matrix leg's cleanup must never remove the other leg's running stack.
|
||||
expect(workflow.jobs["deploy-smoke"]?.env?.EKKLESIA_CI_JOB_KEY).toContain("matrix.topology");
|
||||
});
|
||||
|
||||
it("choose the deploy smoke's edge subnet from those not in use, and choose again on a race", async () => {
|
||||
const deploy = await source("scripts/smoke-deploy.sh");
|
||||
expect(deploy).not.toMatch(/^octet=\$\(\(RANDOM/mu);
|
||||
|
||||
Reference in New Issue
Block a user