diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index b074b4a..0d1f17e 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -63,6 +63,9 @@ jobs: needs: verify runs-on: ubuntu-latest timeout-minutes: 10 + env: + # Names this job's stacks, so its final step removes exactly these (scripts/lib/ci-project.sh). + EKKLESIA_CI_JOB_KEY: infrastructure-smoke steps: - name: Check out repository uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 @@ -72,13 +75,29 @@ jobs: # in this workflow pushes, so the credential is never needed after checkout. persist-credentials: false + # A job the runner killed (cancelled, or past the runner's own time limit) never ran its + # cleanup. Anything a CI job left on this shared daemon more than three hours ago goes first. + - name: Sweep stale CI stacks from the runner + run: bash scripts/ci-sweep.sh --stale + + - name: Prove the sweep removes exactly what it should, against real Docker resources + run: bash scripts/test-ci-sweep.sh + - name: Exercise PostgreSQL and S3 infrastructure run: bash scripts/smoke-infrastructure.sh + # Whatever happened above, this job's own stacks do not outlive it. + - name: Remove this job's CI stacks + if: always() + run: bash scripts/ci-sweep.sh --job + migration-smoke: needs: verify runs-on: ubuntu-latest timeout-minutes: 15 + env: + # Names this job's stacks, so its final step removes exactly these (scripts/lib/ci-project.sh). + EKKLESIA_CI_JOB_KEY: migration-smoke steps: - name: Check out repository uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 @@ -88,6 +107,11 @@ jobs: # in this workflow pushes, so the credential is never needed after checkout. persist-credentials: false + # A job the runner killed (cancelled, or past the runner's own time limit) never ran its + # cleanup. Anything a CI job left on this shared daemon more than three hours ago goes first. + - name: Sweep stale CI stacks from the runner + run: bash scripts/ci-sweep.sh --stale + - name: Set up Node.js uses: actions/setup-node@48b55a011bda9f5d6aeb4c2d9c7362e8dae4041e # v6.4.0 with: @@ -104,11 +128,18 @@ jobs: - name: Exercise transactional migrations against PostgreSQL run: pnpm test:migrations:integration + # Whatever happened above, this job's own stacks do not outlive it. + - name: Remove this job's CI stacks + if: always() + run: bash scripts/ci-sweep.sh --job + image-build: needs: verify runs-on: ubuntu-latest timeout-minutes: 20 env: + # Names this job's stacks, so its final step removes exactly these (scripts/lib/ci-project.sh). + EKKLESIA_CI_JOB_KEY: image-build # Cache mounts in deploy/Dockerfile need BuildKit; this selects it even where the Docker CLI # has no buildx plugin. DOCKER_BUILDKIT: "1" @@ -121,22 +152,33 @@ jobs: # in this workflow pushes, so the credential is never needed after checkout. persist-credentials: false + # A job the runner killed (cancelled, or past the runner's own time limit) never ran its + # cleanup. Anything a CI job left on this shared daemon more than three hours ago goes first. + - name: Sweep stale CI stacks from the runner + run: bash scripts/ci-sweep.sh --stale + # The platform-only image (ADR 0044): a checkout has no `modules/`, so this proves the image # builds, validates, and assembles with zero modules. Build only — no registry login, no # credentials, no push, no tag anywhere but this runner's local image store. # Tagged per run and attempt, not only per commit: a pull request and a push of the same commit # can build at once on one runner's Docker daemon, and neither may remove the other's image. + # Labeled like every CI stack, so this job's final step removes it even if the build is killed. - name: Build the production image without modules - run: >- - docker build - --file deploy/Dockerfile - --build-arg REVISION="$GITHUB_SHA" - --tag ekklesia-ci:"$GITHUB_SHA-${GITHUB_RUN_ID:-0}-${GITHUB_RUN_ATTEMPT:-1}" - . + shell: bash + run: | + source scripts/lib/ci-project.sh + docker build \ + --file deploy/Dockerfile \ + --build-arg REVISION="$GITHUB_SHA" \ + --label "${EKKLESIA_CI_LABEL}=$(ci_project_name image)" \ + --label "${EKKLESIA_CI_CREATED_LABEL}=$(date +%s)" \ + --tag ekklesia-ci:"$GITHUB_SHA-${GITHUB_RUN_ID:-0}-${GITHUB_RUN_ATTEMPT:-1}" \ + . - - name: Remove the image from the runner + # Whatever happened above, this job's own stacks do not outlive it. + - name: Remove this job's CI stacks if: always() - run: docker image rm --force ekklesia-ci:"$GITHUB_SHA-${GITHUB_RUN_ID:-0}-${GITHUB_RUN_ATTEMPT:-1}" || true + run: bash scripts/ci-sweep.sh --job deploy-smoke: needs: verify @@ -151,6 +193,8 @@ jobs: - same-origin - split env: + # Names this job's stacks, so its final step removes exactly these (scripts/lib/ci-project.sh). + EKKLESIA_CI_JOB_KEY: deploy-smoke-${{ matrix.topology }} DOCKER_BUILDKIT: "1" EKKLESIA_SMOKE_TOPOLOGY: ${{ matrix.topology }} steps: @@ -162,6 +206,11 @@ jobs: # in this workflow pushes, so the credential is never needed after checkout. persist-credentials: false + # A job the runner killed (cancelled, or past the runner's own time limit) never ran its + # cleanup. Anything a CI job left on this shared daemon more than three hours ago goes first. + - name: Sweep stale CI stacks from the runner + run: bash scripts/ci-sweep.sh --stale + # The reference deployment end to end (ADR 0044): a platform-only image, base + proxy with # Caddy's internal CA (test-only), an explicit migration apply against this job's own fresh # PostgreSQL, then the API, Dashboard, storage delivery, client-address chain, and verify-full @@ -169,3 +218,8 @@ jobs: # it; nothing is pushed, published, or logged in to. - name: Smoke-test the reference deployment (${{ matrix.topology }}) run: bash scripts/smoke-deploy.sh + + # Whatever happened above, this job's own stacks do not outlive it. + - name: Remove this job's CI stacks + if: always() + run: bash scripts/ci-sweep.sh --job diff --git a/docs/development/security.md b/docs/development/security.md index 56e692b..5291347 100644 --- a/docs/development/security.md +++ b/docs/development/security.md @@ -746,6 +746,33 @@ The declarations are kept deliberately: they are already correct for a supported for GitHub Actions, and keeping them means the upgrade does not require re-deriving the intent. They are documented as ignored, not removed. +### Disposable stacks never outlive their job + +Every CI script removes its own Compose stack in an `EXIT` trap, but a trap never runs when the runner +kills the job container: a cancelled job, or the runner's own time limit. One such stack ran on the +shared runner for two days. So stack cleanup has three layers: + +- **The script's own trap**, for every ordinary failure. +- **A final `if: always()` step** in every job that starts stacks: `scripts/ci-sweep.sh --job` removes + everything that job started, at any age, even after a failed step. +- **A sweep before every such job starts**: `scripts/ci-sweep.sh --stale` removes anything a CI job + left more than three hours ago, which catches whatever a killed job could not clean up itself. + +A stack is found by name and label, never by guesswork (`scripts/lib/ci-project.sh`): + +- every CI Compose project is named `ekklesia-ci----…`, and Compose labels + its containers, networks, and volumes with that name; +- anything a script starts outside Compose carries `org.riverofglorymn.ekklesia.ci.project`; +- images also carry the time CI built them, because an image's own creation time can be its base's. + +The job key is per matrix leg, so one leg's cleanup cannot match another's running stack. Nothing +outside the `ekklesia-ci-` prefix is ever removed. A job container attached to a stack's network is +disconnected, never removed. + +**This is not a substitute for a runner-side reaper.** A stack killed with its job is removed only +when another job that sweeps runs on the same runner. Bounded execution and teardown are still the +runner host's to enforce, as above. + ### Runner topology is not verified **The runner topology is not verified in this repository and is not documented here as though it diff --git a/docs/development/testing.md b/docs/development/testing.md index 218427c..d4a9f18 100644 --- a/docs/development/testing.md +++ b/docs/development/testing.md @@ -159,6 +159,12 @@ also receives a safely named disposable database. Direct host runs use a Docker- port bound only to `127.0.0.1`; a containerized Gitea job joins the isolated Compose network and uses the PostgreSQL service name. +In CI, a stack whose job was killed before its own cleanup ran is still removed: by that job's final +`if: always()` step, or by the stale sweep before a later job starts (`scripts/ci-sweep.sh`, see +[Disposable stacks never outlive their job](security.md#disposable-stacks-never-outlive-their-job)). +`bash scripts/test-ci-sweep.sh` proves what the sweep removes and what it leaves alone, against real +Docker resources. + The live suite proves fresh read-only planning, initial and incremental application, genuine no-op behavior, exact applied checksums and history, edited or corrupt history rejection, whole-batch DDL rollback, Drizzle breakpoint execution, real advisory-lock concurrency and diff --git a/scripts/ci-sweep.sh b/scripts/ci-sweep.sh new file mode 100755 index 0000000..681fa88 --- /dev/null +++ b/scripts/ci-sweep.sh @@ -0,0 +1,205 @@ +#!/usr/bin/env bash +# Removes disposable CI stacks from the Docker daemon: containers, networks, volumes, images, and +# staging directories on the Docker host. +# +# bash scripts/ci-sweep.sh --stale [--max-age-seconds N] any CI stack older than N (default 3 h) +# bash scripts/ci-sweep.sh --job this job's own stacks, at any age +# +# Every CI script already removes its own stack in an EXIT trap, but a trap never runs when the runner +# kills the job container: a cancelled job, or the runner's own time limit (Gitea 1.24 ignores +# `timeout-minutes`). So each workflow job that starts stacks ends with `--job` in an `if: always()` +# step, and begins with `--stale`, which catches whatever a killed job left before it could clean up. +# +# What a stack is, and how it is found, is defined in scripts/lib/ci-project.sh: a Compose project +# named with the `ekklesia-ci-` prefix (Compose labels everything it creates with that name), plus +# anything labeled EKKLESIA_CI_LABEL. Nothing outside that prefix is ever touched, at any age. +# +# --stale's threshold is far above any job's run time, so a stack another job on this daemon is using +# right now is never young enough to match. --job refuses to run outside CI: locally, its prefix would +# match a developer's own stacks. +# +# Best effort by design: a resource that cannot be removed is reported, and the sweep goes on. Output +# names resources only; nothing a stack holds is printed. + +set -Eeuo pipefail + +repository_root="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")/.." && pwd)" +# shellcheck source=scripts/lib/ci-project.sh +source "$repository_root/scripts/lib/ci-project.sh" + +compose_label="com.docker.compose.project" +mode="" +max_age_seconds="${EKKLESIA_CI_SWEEP_MAX_AGE_SECONDS:-10800}" + +usage() { + echo "usage: ci-sweep.sh --stale [--max-age-seconds N] | --job" >&2 + exit 2 +} + +while (($# > 0)); do + case "$1" in + --stale | --job) + [[ -z "$mode" ]] || usage + mode="${1#--}" + ;; + --max-age-seconds) + [[ "${2:-}" =~ ^[0-9]+$ ]] || usage + max_age_seconds="$2" + shift + ;; + *) usage ;; + esac + shift +done +[[ -n "$mode" ]] || usage + +if [[ "$mode" == job ]]; then + [[ -n "${GITHUB_RUN_ID:-}" ]] || { + echo "ci-sweep: --job runs only in CI (GITHUB_RUN_ID is unset); locally it would match your own stacks." >&2 + exit 2 + } + prefix="$(ci_job_prefix)" + cutoff="" +else + prefix="$EKKLESIA_CI_PREFIX" + cutoff="$(($(date +%s) - max_age_seconds))" +fi + +if ! docker info >/dev/null 2>&1; then + echo "ci-sweep: no Docker daemon reachable; nothing to sweep." + exit 0 +fi + +removed=0 +warn() { + echo "ci-sweep: warning: $*" >&2 +} + +# selected : whether a resource belongs to the sweep. +selected() { + local project="$1" created="$2" seconds + [[ -n "$project" && "$project" == "$prefix"* ]] || return 1 + [[ -z "$cutoff" ]] && return 0 + seconds="$(date -d "$created" +%s 2>/dev/null)" || return 1 + ((seconds <= cutoff)) +} + +# --- Containers: Compose's own, and those a script started itself -------------------------------- +mapfile -t containers < <( + { + docker ps --all --quiet --no-trunc --filter "label=${compose_label}" + docker ps --all --quiet --no-trunc --filter "label=${EKKLESIA_CI_LABEL}" + } | sort -u +) +for id in "${containers[@]}"; do + [[ -n "$id" ]] || continue + record="$(docker inspect --format \ + "{{index .Config.Labels \"${compose_label}\"}}|{{index .Config.Labels \"${EKKLESIA_CI_LABEL}\"}}|{{.Created}}|{{.Name}}" \ + "$id" 2>/dev/null)" || continue + IFS='|' read -r compose_project ci_project created name <<<"$record" + project="${compose_project:-$ci_project}" + selected "$project" "$created" || continue + if docker rm --force --volumes "$id" >/dev/null 2>&1; then + echo "ci-sweep: removed container ${name#/} (${project})" + removed=$((removed + 1)) + else + warn "could not remove container ${name#/}" + fi +done + +# --- Networks: anything still attached (a job container) is disconnected, never removed --------- +mapfile -t networks < <(docker network ls --quiet --no-trunc --filter "label=${compose_label}") +for id in "${networks[@]}"; do + [[ -n "$id" ]] || continue + # A network's creation time is a time value, not a string: ask for RFC 3339, which `date` reads. + record="$(docker network inspect --format \ + "{{index .Labels \"${compose_label}\"}}|{{.Created.Format \"2006-01-02T15:04:05Z07:00\"}}|{{.Name}}" \ + "$id" 2>/dev/null)" || continue + IFS='|' read -r project created name <<<"$record" + selected "$project" "$created" || continue + for endpoint in $(docker network inspect --format '{{range $id, $_ := .Containers}}{{$id}} {{end}}' "$id" 2>/dev/null); do + docker network disconnect --force "$id" "$endpoint" >/dev/null 2>&1 || + warn "could not disconnect a container from network ${name}" + done + if docker network rm "$id" >/dev/null 2>&1; then + echo "ci-sweep: removed network ${name} (${project})" + removed=$((removed + 1)) + else + warn "could not remove network ${name}" + fi +done + +# --- Volumes --------------------------------------------------------------------------------------- +mapfile -t volumes < <( + { + docker volume ls --quiet --filter "label=${compose_label}" + docker volume ls --quiet --filter "label=${EKKLESIA_CI_LABEL}" + } | sort -u +) +for name in "${volumes[@]}"; do + [[ -n "$name" ]] || continue + record="$(docker volume inspect --format \ + "{{index .Labels \"${compose_label}\"}}|{{index .Labels \"${EKKLESIA_CI_LABEL}\"}}|{{.CreatedAt}}" \ + "$name" 2>/dev/null)" || continue + IFS='|' read -r compose_project ci_project created <<<"$record" + project="${compose_project:-$ci_project}" + selected "$project" "$created" || continue + if docker volume rm --force "$name" >/dev/null 2>&1; then + echo "ci-sweep: removed volume ${name} (${project})" + removed=$((removed + 1)) + else + warn "could not remove volume ${name}" + fi +done + +# --- Images built for a stack ------------------------------------------------------------------------ +mapfile -t images < <(docker image ls --quiet --no-trunc --filter "label=${EKKLESIA_CI_LABEL}" | sort -u) +for id in "${images[@]}"; do + [[ -n "$id" ]] || continue + # Judged by when CI built it (EKKLESIA_CI_CREATED_LABEL), never by the image's own creation time, + # which BuildKit may have taken from its base. An image without that label is never swept by age. + record="$(docker image inspect --format \ + "{{index .Config.Labels \"${EKKLESIA_CI_LABEL}\"}}|{{index .Config.Labels \"${EKKLESIA_CI_CREATED_LABEL}\"}}" \ + "$id" 2>/dev/null)" || continue + IFS='|' read -r project built <<<"$record" + if [[ -n "$cutoff" ]]; then + [[ "$built" =~ ^[0-9]+$ ]] || continue + selected "$project" "@${built}" || continue + else + selected "$project" "" || continue + fi + if docker image rm --force "$id" >/dev/null 2>&1; then + echo "ci-sweep: removed image ${id#sha256:} (${project})" + removed=$((removed + 1)) + else + warn "could not remove image ${id#sha256:}" + fi +done + +# --- Staging directories on the Docker host ------------------------------------------------------ +# A deploy smoke run inside a job container stages its mounted files under the Docker host's /tmp, +# named for its project, as root-owned files; they hold that run's disposable secrets and keys. They +# are removed with the same pinned image that created them. If that image is not on this daemon, no +# smoke has staged anything here. +image="$(sed -n 's/^[[:space:]]*image: \(postgres:[^[:space:]]*\)$/\1/p' "$repository_root/deploy/compose.yaml" | head -1)" +if [[ -n "$image" ]] && docker image inspect "$image" >/dev/null 2>&1; then + age=() + # `find -mmin +0` does not match something younger than a minute, so a threshold under a minute + # sweeps without an age filter, like the rest of this script. + if [[ -n "$cutoff" ]] && ((max_age_seconds >= 60)); then + age=(-mmin "+$((max_age_seconds / 60))") + fi + if staged="$(docker run --rm --entrypoint find -v /tmp:/host-tmp "$image" \ + /host-tmp -mindepth 1 -maxdepth 1 -type d -name "${prefix}*" "${age[@]}" \ + -print -exec rm -rf {} + 2>/dev/null)"; then + while IFS= read -r directory; do + [[ -n "$directory" ]] || continue + echo "ci-sweep: removed staging directory /tmp/${directory#/host-tmp/}" + removed=$((removed + 1)) + done <<<"$staged" + else + warn "could not sweep staging directories on the Docker host" + fi +fi + +echo "ci-sweep: ${mode} sweep removed ${removed} resource(s)." diff --git a/scripts/lib/ci-project.sh b/scripts/lib/ci-project.sh new file mode 100644 index 0000000..a028dbc --- /dev/null +++ b/scripts/lib/ci-project.sh @@ -0,0 +1,40 @@ +# shellcheck shell=bash +# How CI names, labels, and later finds the disposable stacks it starts. +# +# Every Compose project a CI script starts is named +# +# ekklesia-ci----- +# +# and Compose labels every container, network, and volume it creates with that name. What a script +# creates outside Compose (a browser container, a volume, an image) gets EKKLESIA_CI_LABEL with the +# same name. scripts/ci-sweep.sh finds stacks by those labels: this job's own, by the prefix up to +# the job key (in a final `if: always()` step), and any CI stack by age (before a job starts). +# +# The job key is EKKLESIA_CI_JOB_KEY, which the workflow sets per job, and per matrix leg where a job +# has one, so that one leg's cleanup cannot match another leg's running stack. Outside CI the +# identifiers are "local". + +# shellcheck disable=SC2034 # read by the scripts that source this +EKKLESIA_CI_LABEL="org.riverofglorymn.ekklesia.ci.project" +# On images, when they were built, in Unix seconds. An image's own creation time is not that: BuildKit +# gives an image that only adds metadata its base's time, and a reproducible build a fixed one. A +# stale sweep that trusted it could remove an image another job is using right now. +EKKLESIA_CI_CREATED_LABEL="org.riverofglorymn.ekklesia.ci.created" +EKKLESIA_CI_PREFIX="ekklesia-ci-" + +# ci_name_safe : lower case, and only what a Compose project name may contain. +ci_name_safe() { + local text="${1//[^a-zA-Z0-9_-]/-}" + printf '%s' "${text,,}" +} + +# ci_job_prefix: every project this job starts begins with exactly this. +ci_job_prefix() { + ci_name_safe "ekklesia-ci-${GITHUB_RUN_ID:-local}-${GITHUB_RUN_ATTEMPT:-0}-${EKKLESIA_CI_JOB_KEY:-${GITHUB_JOB:-local}}-" +} + +# ci_project_name : a project name unique to this invocation. Not only per run: several runs, +# and several jobs of one run, share a runner's Docker daemon, and job containers often share a PID. +ci_project_name() { + ci_name_safe "ekklesia-ci-${GITHUB_RUN_ID:-local}-${GITHUB_RUN_ATTEMPT:-0}-${EKKLESIA_CI_JOB_KEY:-${GITHUB_JOB:-local}}-${1:?ci_project_name needs a kind}-$$-${RANDOM}${RANDOM}" +} diff --git a/scripts/smoke-deploy.sh b/scripts/smoke-deploy.sh index 8ac81cc..ece1296 100755 --- a/scripts/smoke-deploy.sh +++ b/scripts/smoke-deploy.sh @@ -39,6 +39,8 @@ cd "$repository_root" # shellcheck source=scripts/lib/compose-diagnostics.sh source "$repository_root/scripts/lib/compose-diagnostics.sh" +# shellcheck source=scripts/lib/ci-project.sh +source "$repository_root/scripts/lib/ci-project.sh" for command in docker openssl awk; do command -v "$command" >/dev/null || { @@ -67,7 +69,10 @@ same-origin | split) ;; esac run_id="$(openssl rand -hex 4)" -project="ekklesia-deploy-smoke-${run_id}" +# Unique per invocation, and findable by run and job if this script is killed before its trap runs +# (scripts/lib/ci-project.sh, scripts/ci-sweep.sh). Everything started outside Compose below carries +# the same name in EKKLESIA_CI_LABEL. +project="$(ci_project_name "deploy-${topology}")" work="$(mktemp -d)" image="${EKKLESIA_SMOKE_IMAGE:-}" built_image="" @@ -186,7 +191,9 @@ fail() { if [[ -z "$image" && "$topology" == same-origin ]]; then step "Building the platform-only image" built_image="ekklesia-deploy-smoke:${run_id}" - DOCKER_BUILDKIT=1 docker build --quiet --file deploy/Dockerfile --tag "$built_image" . >/dev/null + DOCKER_BUILDKIT=1 docker build --quiet --file deploy/Dockerfile --tag "$built_image" \ + --label "${EKKLESIA_CI_LABEL}=${project}" --label "${EKKLESIA_CI_CREATED_LABEL}=$(date +%s)" \ + . >/dev/null image="$built_image" elif [[ -z "$image" ]]; then # TEST-ONLY: the repository's module template, installed as the module `example`, in a copy of the @@ -206,6 +213,7 @@ elif [[ -z "$image" ]]; then sed -i 's|^ templates/module:$| modules/example:|' "$context/pnpm-lock.yaml" built_image="ekklesia-deploy-smoke:${run_id}" DOCKER_BUILDKIT=1 docker build --quiet --file "$context/deploy/Dockerfile" --tag "$built_image" \ + --label "${EKKLESIA_CI_LABEL}=${project}" --label "${EKKLESIA_CI_CREATED_LABEL}=$(date +%s)" \ "$context" >/dev/null || fail "the test-module image did not build." rm -rf "$context" image="$built_image" @@ -362,6 +370,7 @@ client() { local mode="$1" shift docker run --rm --read-only --cap-drop ALL --security-opt no-new-privileges \ + --label "${EKKLESIA_CI_LABEL}=${project}" \ --network "$edge_network" "$@" "${host_mappings[@]}" \ -e NODE_EXTRA_CA_CERTS=/smoke/client-ca.pem -e SMOKE_HOSTNAME="$hostname" \ -e SMOKE_DASHBOARD_HOSTNAME="$dashboard_hostname" \ @@ -401,7 +410,7 @@ if [[ "$topology" == split ]]; then browser_container="${project}-browser" # The image's entry script passes its arguments on unquoted, so none may contain a space: the # hostnames reach Chromium through /etc/hosts, which it reads, rather than a resolver rule. - docker run --detach --rm --name "$browser_container" \ + docker run --detach --rm --name "$browser_container" --label "${EKKLESIA_CI_LABEL}=${project}" \ --cap-drop ALL --security-opt no-new-privileges --tmpfs /tmp \ --network "$edge_network" --ip "$browser_address" "${host_mappings[@]}" \ "$browser_image" \ @@ -410,6 +419,7 @@ if [[ "$topology" == split ]]; then cp scripts/lib/deploy-smoke-browser.mjs "$work/browser.mjs" stage browser.mjs docker run --rm --read-only --cap-drop ALL --security-opt no-new-privileges \ + --label "${EKKLESIA_CI_LABEL}=${project}" \ --network "$edge_network" \ -e SMOKE_CDP_ADDRESS="${browser_address}:9222" \ -e SMOKE_HOSTNAME="$hostname" -e SMOKE_DASHBOARD_HOSTNAME="$dashboard_hostname" \ @@ -423,9 +433,9 @@ if [[ "$topology" == split ]]; then kiosk_container="${project}-kiosk" kiosk_volume="${project}-kiosk-profile" kiosk_address="10.249.${octet}.250" - docker volume create "$kiosk_volume" >/dev/null + docker volume create --label "${EKKLESIA_CI_LABEL}=${project}" "$kiosk_volume" >/dev/null start_kiosk() { - docker run --detach --rm --name "$kiosk_container" \ + docker run --detach --rm --name "$kiosk_container" --label "${EKKLESIA_CI_LABEL}=${project}" \ --cap-drop ALL --security-opt no-new-privileges --tmpfs /tmp \ --network "$edge_network" --ip "$kiosk_address" "${host_mappings[@]}" \ -v "${kiosk_volume}:/profile" \ diff --git a/scripts/smoke-infrastructure.sh b/scripts/smoke-infrastructure.sh index 8cbcebe..1165640 100755 --- a/scripts/smoke-infrastructure.sh +++ b/scripts/smoke-infrastructure.sh @@ -9,6 +9,8 @@ cd "$repository_root" source "$repository_root/scripts/lib/compose-job-network.sh" # shellcheck source=scripts/lib/compose-diagnostics.sh source "$repository_root/scripts/lib/compose-diagnostics.sh" +# shellcheck source=scripts/lib/ci-project.sh +source "$repository_root/scripts/lib/ci-project.sh" command -v docker >/dev/null || { echo "Docker is required for the infrastructure smoke test." >&2 @@ -30,11 +32,9 @@ if [[ "$curl_help" != *"--aws-sigv4"* ]]; then exit 1 fi -# Unique per invocation, not only per run: several runs can share a runner's Docker daemon at once, -# and the shell's PID is often identical across job containers. -raw_project="ekklesia-smoke-${GITHUB_RUN_ID:-local}-${GITHUB_RUN_ATTEMPT:-0}-$$-${RANDOM}${RANDOM}" -project="${raw_project//[^a-zA-Z0-9_-]/-}" -project="${project,,}" +# Unique per invocation, and findable by run and job if this script is killed before its trap runs +# (scripts/lib/ci-project.sh, scripts/ci-sweep.sh). +project="$(ci_project_name infrastructure)" temporary_directory="$(mktemp -d)" payload_file="$temporary_directory/payload.txt" download_file="$temporary_directory/download.txt" diff --git a/scripts/test-ci-sweep.sh b/scripts/test-ci-sweep.sh new file mode 100755 index 0000000..ccb5def --- /dev/null +++ b/scripts/test-ci-sweep.sh @@ -0,0 +1,129 @@ +#!/usr/bin/env bash +# Proves scripts/ci-sweep.sh against real Docker resources: what it removes, and above all what it +# leaves alone. CI runs it before trusting the sweep with a shared runner's daemon. +# +# bash scripts/test-ci-sweep.sh +# +# Every resource it creates is named and labeled for this run alone, under fake run identifiers, so +# it can never touch a real stack; and it removes all of them on exit, whatever happens. + +set -Eeuo pipefail + +repository_root="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")/.." && pwd)" +cd "$repository_root" + +image="$(sed -n 's/^[[:space:]]*image: \(postgres:[^[:space:]]*\)$/\1/p' deploy/compose.yaml | head -1)" +[[ -n "$image" ]] || { + echo "test-ci-sweep: no pinned PostgreSQL image in deploy/compose.yaml." >&2 + exit 1 +} +docker image inspect "$image" >/dev/null 2>&1 || docker pull --quiet "$image" >/dev/null + +nonce="$(openssl rand -hex 4)" +compose_label="com.docker.compose.project" +ci_label="org.riverofglorymn.ekklesia.ci.project" +failures=0 +created=() + +check() { + if eval "$1"; then + echo "ok $2" + else + echo "FAIL $2" + failures=$((failures + 1)) + fi +} + +# project_exists : any container, network, volume, or image still carries its label. +project_exists() { + local project="$1" + [[ -n "$( + docker ps --all --quiet --filter "label=${compose_label}=${project}" + docker ps --all --quiet --filter "label=${ci_label}=${project}" + docker network ls --quiet --filter "label=${compose_label}=${project}" + docker volume ls --quiet --filter "label=${compose_label}=${project}" + docker image ls --quiet --filter "label=${ci_label}=${project}" + )" ]] +} + +staging_exists() { + docker run --rm --entrypoint test -v /tmp:/host-tmp "$image" -d "/host-tmp/$1" +} + +# make_stack : what a CI script leaves behind when it is killed: a Compose-labeled network, +# volume, and running container; a CI-labeled container and image; and a staging directory on the +# Docker host. +make_stack() { + local project="$1" + created+=("$project") + docker network create --label "${compose_label}=${project}" "${project}_default" >/dev/null + docker volume create --label "${compose_label}=${project}" "${project}_data" >/dev/null + docker run --detach --name "${project}-db" --label "${compose_label}=${project}" \ + --network "${project}_default" -v "${project}_data:/data" --entrypoint sleep "$image" 600 >/dev/null + docker run --detach --name "${project}-browser" --label "${ci_label}=${project}" \ + --entrypoint sleep "$image" 600 >/dev/null + printf 'FROM %s\nLABEL %s=%s %s=%s\n' "$image" "$ci_label" "$project" \ + "org.riverofglorymn.ekklesia.ci.created" "$(date +%s)" | + docker build --quiet --tag "${project}-image:test" - >/dev/null + docker run --rm --entrypoint mkdir -v /tmp:/host-tmp "$image" -p "/host-tmp/${project}" +} + +job_container="test-ci-sweep-job-${nonce}" +cleanup() { + local status=$? + trap - EXIT INT TERM + docker rm --force "$job_container" >/dev/null 2>&1 || true + for project in "${created[@]}"; do + docker ps --all --quiet --filter "label=${compose_label}=${project}" | xargs -r docker rm --force >/dev/null 2>&1 || true + docker ps --all --quiet --filter "label=${ci_label}=${project}" | xargs -r docker rm --force >/dev/null 2>&1 || true + docker network rm "${project}_default" >/dev/null 2>&1 || true + docker volume rm --force "${project}_data" >/dev/null 2>&1 || true + docker image rm --force "${project}-image:test" >/dev/null 2>&1 || true + docker run --rm --entrypoint rm -v /tmp:/host-tmp "$image" -rf "/host-tmp/${project}" >/dev/null 2>&1 || true + done + exit "$status" +} +trap cleanup EXIT INT TERM + +# shellcheck source=scripts/lib/ci-project.sh +source "$repository_root/scripts/lib/ci-project.sh" + +# Fake run identifiers: nothing here can match a real run's stacks. +export GITHUB_RUN_ID="sweeptest${nonce}" +export GITHUB_RUN_ATTEMPT="1" +mine="$(EKKLESIA_CI_JOB_KEY=alpha ci_project_name mine)" +sibling="$(EKKLESIA_CI_JOB_KEY=beta ci_project_name sibling)" +foreign="ekklesia-sweep-control-${nonce}" +for project in "$mine" "$sibling" "$foreign"; do + make_stack "$project" +done +# A job container connected to this job's stack network, exactly as compose-job-network.sh does. +docker run --detach --name "$job_container" --entrypoint sleep "$image" 600 >/dev/null +docker network connect "${mine}_default" "$job_container" + +echo "--- --job removes this job's stacks, at any age, and nothing else" +EKKLESIA_CI_JOB_KEY=alpha bash scripts/ci-sweep.sh --job +check '! project_exists "$mine"' "this job's containers, network, volume, and image are gone" +check '! staging_exists "$mine"' "this job's staging directory on the Docker host is gone" +check 'project_exists "$sibling"' "another job's stack in the same run is left alone" +check 'project_exists "$foreign"' "a stack outside the CI prefix is left alone" +check 'docker inspect "$job_container" >/dev/null 2>&1' \ + "a job container connected to the stack's network is disconnected, never removed" + +echo "--- --job refuses to run outside CI, where it could remove a developer's own stacks" +check '! env -u GITHUB_RUN_ID bash scripts/ci-sweep.sh --job >/dev/null 2>&1' \ + "--job without GITHUB_RUN_ID fails" + +echo "--- --stale removes only CI stacks older than its threshold" +bash scripts/ci-sweep.sh --stale +check 'project_exists "$sibling"' "a young CI stack survives the default threshold" +bash scripts/ci-sweep.sh --stale --max-age-seconds 0 +check '! project_exists "$sibling"' "a CI stack older than the threshold is removed" +check '! staging_exists "$sibling"' "its staging directory too" +check 'project_exists "$foreign"' "a stack outside the CI prefix is never removed, at any age" + +if ((failures > 0)); then + echo "test-ci-sweep: ${failures} check(s) failed." >&2 + exit 1 +fi +echo "CI sweep checks passed." diff --git a/scripts/test-migrations-integration.sh b/scripts/test-migrations-integration.sh index 29d4dd1..847caf0 100755 --- a/scripts/test-migrations-integration.sh +++ b/scripts/test-migrations-integration.sh @@ -9,6 +9,8 @@ cd "$repository_root" source "$repository_root/scripts/lib/compose-job-network.sh" # shellcheck source=scripts/lib/compose-diagnostics.sh source "$repository_root/scripts/lib/compose-diagnostics.sh" +# shellcheck source=scripts/lib/ci-project.sh +source "$repository_root/scripts/lib/ci-project.sh" command -v docker >/dev/null || { echo "Docker is required for the PostgreSQL migration integration suite." >&2 @@ -27,11 +29,9 @@ if ! compose_job_network_initialize; then exit 1 fi -# Unique per invocation, not only per run: several runs (and several jobs of one run) can share a -# runner's Docker daemon at once, and the shell's PID is often identical across job containers. -raw_project="ekklesia-migration-${GITHUB_RUN_ID:-local}-${GITHUB_RUN_ATTEMPT:-0}-${GITHUB_JOB:-job}-$$-${RANDOM}${RANDOM}" -project="${raw_project//[^a-zA-Z0-9_-]/-}" -project="${project,,}" +# Unique per invocation, and findable by run and job if this script is killed before its trap runs +# (scripts/lib/ci-project.sh, scripts/ci-sweep.sh). +project="$(ci_project_name migration)" export POSTGRES_DB="ekklesia_migration_test" export POSTGRES_USER="ekklesia_migration_test" diff --git a/tests/ci-scripts.test.ts b/tests/ci-scripts.test.ts index 7203ada..e498d70 100644 --- a/tests/ci-scripts.test.ts +++ b/tests/ci-scripts.test.ts @@ -4,6 +4,7 @@ import path from "node:path"; import { fileURLToPath } from "node:url"; import { promisify } from "node:util"; import { describe, expect, it } from "vitest"; +import { parse } from "yaml"; /** * The scripts CI runs against a shared Docker daemon. Several runs, and several jobs of one run, can @@ -59,21 +60,68 @@ describe("CI scripts on a shared Docker daemon", () => { } }); - it("give every Compose project a name unique to the invocation, not only the run", async () => { - expect(await source("scripts/test-migrations-integration.sh")).toMatch( - /raw_project="ekklesia-migration-[^"\n]*\$\$-\$\{RANDOM\}\$\{RANDOM\}"/u, + it("name every stack through one helper: the reserved CI prefix, the job's key, and a unique suffix", async () => { + const helper = await source("scripts/lib/ci-project.sh"); + // Run, attempt, job key, kind, then a per-invocation suffix: unique even when job containers + // share a PID, and findable later by run and job alone (scripts/ci-sweep.sh --job). + expect(helper).toContain( + "ekklesia-ci-${GITHUB_RUN_ID:-local}-${GITHUB_RUN_ATTEMPT:-0}-${EKKLESIA_CI_JOB_KEY:-${GITHUB_JOB:-local}}-", ); - expect(await source("scripts/smoke-infrastructure.sh")).toMatch( - /raw_project="ekklesia-smoke-[^"\n]*\$\$-\$\{RANDOM\}\$\{RANDOM\}"/u, - ); - const deploy = await source("scripts/smoke-deploy.sh"); - expect(deploy).toContain('run_id="$(openssl rand -hex 4)"'); - expect(deploy).toContain('project="ekklesia-deploy-smoke-${run_id}"'); + expect(helper).toContain("$$-${RANDOM}${RANDOM}"); for (const relativePath of ciScripts) { - expect(await source(relativePath), relativePath).toContain('--project-name "$project"'); + const text = await source(relativePath); + expect(text, relativePath).toContain('source "$repository_root/scripts/lib/ci-project.sh"'); + expect(text, relativePath).toMatch(/^project="\$\(ci_project_name [a-z"${}_-]+\)"$/mu); + expect(text, relativePath).not.toContain("raw_project="); + expect(text, relativePath).toContain('--project-name "$project"'); } }); + it("label everything the deploy smoke creates outside Compose, so a sweep can find it", async () => { + // Continuation lines joined, so each command is checked whole. + const deploy = (await source("scripts/smoke-deploy.sh")).replace(/\\\n\s*/gu, " "); + const creating = deploy + .split("\n") + .filter((line) => /\bdocker (?:run --detach|volume create|build)\b/u.test(line)); + expect(creating.length).toBeGreaterThanOrEqual(4); + for (const line of creating) { + expect(line).toContain('--label "${EKKLESIA_CI_LABEL}=${project}"'); + } + }); + + it("sweep stale stacks before, and this job's own stacks after, every job that starts any", async () => { + const workflow = parse(await source(".github/workflows/ci.yml")) as { + jobs: Record< + string, + { env?: Record; steps: { if?: string; run?: string; name?: string }[] } + >; + }; + const startsStacks = + /smoke-infrastructure\.sh|test:migrations:integration|smoke-deploy\.sh|docker build/u; + const jobs = Object.entries(workflow.jobs).filter(([, job]) => + job.steps.some((step) => startsStacks.test(step.run ?? "")), + ); + expect(jobs.map(([id]) => id).sort()).toEqual( + ["deploy-smoke", "image-build", "infrastructure-smoke", "migration-smoke"].sort(), + ); + for (const [id, job] of jobs) { + expect(job.env?.EKKLESIA_CI_JOB_KEY, id).toBeTruthy(); + const runs = job.steps.map((step) => step.run ?? ""); + const firstStack = runs.findIndex((run) => startsStacks.test(run)); + const stale = runs.findIndex((run) => run.includes("bash scripts/ci-sweep.sh --stale")); + const own = job.steps.findIndex( + (step) => + step.if === "always()" && (step.run ?? "").includes("bash scripts/ci-sweep.sh --job"), + ); + expect(stale, `${id}: stale sweep`).toBeGreaterThanOrEqual(0); + expect(stale, `${id}: stale sweep runs first`).toBeLessThan(firstStack); + expect(own, `${id}: its own cleanup, always`).toBeGreaterThan(firstStack); + expect(own, `${id}: cleanup is the last step`).toBe(job.steps.length - 1); + } + // One matrix leg's cleanup must never remove the other leg's running stack. + expect(workflow.jobs["deploy-smoke"]?.env?.EKKLESIA_CI_JOB_KEY).toContain("matrix.topology"); + }); + it("choose the deploy smoke's edge subnet from those not in use, and choose again on a race", async () => { const deploy = await source("scripts/smoke-deploy.sh"); expect(deploy).not.toMatch(/^octet=\$\(\(RANDOM/mu);