fix(ci): nothing a CI job starts outlives it, even when the runner kills the job #60

Merged
Zayin_Admin merged 2 commits from fix/ci-stack-cleanup into main 2026-10-06 02:11:53 +00:00
9 changed files with 367 additions and 24 deletions
Showing only changes of commit 62951975da - Show all commits
+61 -7
View File
@@ -63,6 +63,9 @@ jobs:
needs: verify needs: verify
runs-on: ubuntu-latest runs-on: ubuntu-latest
timeout-minutes: 10 timeout-minutes: 10
env:
# Names this job's stacks, so its final step removes exactly these (scripts/lib/ci-project.sh).
EKKLESIA_CI_JOB_KEY: infrastructure-smoke
steps: steps:
- name: Check out repository - name: Check out repository
uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
@@ -72,13 +75,29 @@ jobs:
# in this workflow pushes, so the credential is never needed after checkout. # in this workflow pushes, so the credential is never needed after checkout.
persist-credentials: false persist-credentials: false
# A job the runner killed (cancelled, or past the runner's own time limit) never ran its
# cleanup. Anything a CI job left on this shared daemon more than three hours ago goes first.
- name: Sweep stale CI stacks from the runner
run: bash scripts/ci-sweep.sh --stale
- name: Prove the sweep removes exactly what it should, against real Docker resources
run: bash scripts/test-ci-sweep.sh
- name: Exercise PostgreSQL and S3 infrastructure - name: Exercise PostgreSQL and S3 infrastructure
run: bash scripts/smoke-infrastructure.sh run: bash scripts/smoke-infrastructure.sh
# Whatever happened above, this job's own stacks do not outlive it.
- name: Remove this job's CI stacks
if: always()
run: bash scripts/ci-sweep.sh --job
migration-smoke: migration-smoke:
needs: verify needs: verify
runs-on: ubuntu-latest runs-on: ubuntu-latest
timeout-minutes: 15 timeout-minutes: 15
env:
# Names this job's stacks, so its final step removes exactly these (scripts/lib/ci-project.sh).
EKKLESIA_CI_JOB_KEY: migration-smoke
steps: steps:
- name: Check out repository - name: Check out repository
uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
@@ -88,6 +107,11 @@ jobs:
# in this workflow pushes, so the credential is never needed after checkout. # in this workflow pushes, so the credential is never needed after checkout.
persist-credentials: false persist-credentials: false
# A job the runner killed (cancelled, or past the runner's own time limit) never ran its
# cleanup. Anything a CI job left on this shared daemon more than three hours ago goes first.
- name: Sweep stale CI stacks from the runner
run: bash scripts/ci-sweep.sh --stale
- name: Set up Node.js - name: Set up Node.js
uses: actions/setup-node@48b55a011bda9f5d6aeb4c2d9c7362e8dae4041e # v6.4.0 uses: actions/setup-node@48b55a011bda9f5d6aeb4c2d9c7362e8dae4041e # v6.4.0
with: with:
@@ -104,11 +128,18 @@ jobs:
- name: Exercise transactional migrations against PostgreSQL - name: Exercise transactional migrations against PostgreSQL
run: pnpm test:migrations:integration run: pnpm test:migrations:integration
# Whatever happened above, this job's own stacks do not outlive it.
- name: Remove this job's CI stacks
if: always()
run: bash scripts/ci-sweep.sh --job
image-build: image-build:
needs: verify needs: verify
runs-on: ubuntu-latest runs-on: ubuntu-latest
timeout-minutes: 20 timeout-minutes: 20
env: env:
# Names this job's stacks, so its final step removes exactly these (scripts/lib/ci-project.sh).
EKKLESIA_CI_JOB_KEY: image-build
# Cache mounts in deploy/Dockerfile need BuildKit; this selects it even where the Docker CLI # Cache mounts in deploy/Dockerfile need BuildKit; this selects it even where the Docker CLI
# has no buildx plugin. # has no buildx plugin.
DOCKER_BUILDKIT: "1" DOCKER_BUILDKIT: "1"
@@ -121,22 +152,33 @@ jobs:
# in this workflow pushes, so the credential is never needed after checkout. # in this workflow pushes, so the credential is never needed after checkout.
persist-credentials: false persist-credentials: false
# A job the runner killed (cancelled, or past the runner's own time limit) never ran its
# cleanup. Anything a CI job left on this shared daemon more than three hours ago goes first.
- name: Sweep stale CI stacks from the runner
run: bash scripts/ci-sweep.sh --stale
# The platform-only image (ADR 0044): a checkout has no `modules/`, so this proves the image # The platform-only image (ADR 0044): a checkout has no `modules/`, so this proves the image
# builds, validates, and assembles with zero modules. Build only — no registry login, no # builds, validates, and assembles with zero modules. Build only — no registry login, no
# credentials, no push, no tag anywhere but this runner's local image store. # credentials, no push, no tag anywhere but this runner's local image store.
# Tagged per run and attempt, not only per commit: a pull request and a push of the same commit # Tagged per run and attempt, not only per commit: a pull request and a push of the same commit
# can build at once on one runner's Docker daemon, and neither may remove the other's image. # can build at once on one runner's Docker daemon, and neither may remove the other's image.
# Labeled like every CI stack, so this job's final step removes it even if the build is killed.
- name: Build the production image without modules - name: Build the production image without modules
run: >- shell: bash
docker build run: |
--file deploy/Dockerfile source scripts/lib/ci-project.sh
--build-arg REVISION="$GITHUB_SHA" docker build \
--tag ekklesia-ci:"$GITHUB_SHA-${GITHUB_RUN_ID:-0}-${GITHUB_RUN_ATTEMPT:-1}" --file deploy/Dockerfile \
--build-arg REVISION="$GITHUB_SHA" \
--label "${EKKLESIA_CI_LABEL}=$(ci_project_name image)" \
--label "${EKKLESIA_CI_CREATED_LABEL}=$(date +%s)" \
--tag ekklesia-ci:"$GITHUB_SHA-${GITHUB_RUN_ID:-0}-${GITHUB_RUN_ATTEMPT:-1}" \
. .
- name: Remove the image from the runner # Whatever happened above, this job's own stacks do not outlive it.
- name: Remove this job's CI stacks
if: always() if: always()
run: docker image rm --force ekklesia-ci:"$GITHUB_SHA-${GITHUB_RUN_ID:-0}-${GITHUB_RUN_ATTEMPT:-1}" || true run: bash scripts/ci-sweep.sh --job
deploy-smoke: deploy-smoke:
needs: verify needs: verify
@@ -151,6 +193,8 @@ jobs:
- same-origin - same-origin
- split - split
env: env:
# Names this job's stacks, so its final step removes exactly these (scripts/lib/ci-project.sh).
EKKLESIA_CI_JOB_KEY: deploy-smoke-${{ matrix.topology }}
DOCKER_BUILDKIT: "1" DOCKER_BUILDKIT: "1"
EKKLESIA_SMOKE_TOPOLOGY: ${{ matrix.topology }} EKKLESIA_SMOKE_TOPOLOGY: ${{ matrix.topology }}
steps: steps:
@@ -162,6 +206,11 @@ jobs:
# in this workflow pushes, so the credential is never needed after checkout. # in this workflow pushes, so the credential is never needed after checkout.
persist-credentials: false persist-credentials: false
# A job the runner killed (cancelled, or past the runner's own time limit) never ran its
# cleanup. Anything a CI job left on this shared daemon more than three hours ago goes first.
- name: Sweep stale CI stacks from the runner
run: bash scripts/ci-sweep.sh --stale
# The reference deployment end to end (ADR 0044): a platform-only image, base + proxy with # The reference deployment end to end (ADR 0044): a platform-only image, base + proxy with
# Caddy's internal CA (test-only), an explicit migration apply against this job's own fresh # Caddy's internal CA (test-only), an explicit migration apply against this job's own fresh
# PostgreSQL, then the API, Dashboard, storage delivery, client-address chain, and verify-full # PostgreSQL, then the API, Dashboard, storage delivery, client-address chain, and verify-full
@@ -169,3 +218,8 @@ jobs:
# it; nothing is pushed, published, or logged in to. # it; nothing is pushed, published, or logged in to.
- name: Smoke-test the reference deployment (${{ matrix.topology }}) - name: Smoke-test the reference deployment (${{ matrix.topology }})
run: bash scripts/smoke-deploy.sh run: bash scripts/smoke-deploy.sh
# Whatever happened above, this job's own stacks do not outlive it.
- name: Remove this job's CI stacks
if: always()
run: bash scripts/ci-sweep.sh --job
+27
View File
@@ -746,6 +746,33 @@ The declarations are kept deliberately: they are already correct for a supported
for GitHub Actions, and keeping them means the upgrade does not require re-deriving the intent. They for GitHub Actions, and keeping them means the upgrade does not require re-deriving the intent. They
are documented as ignored, not removed. are documented as ignored, not removed.
### Disposable stacks never outlive their job
Every CI script removes its own Compose stack in an `EXIT` trap, but a trap never runs when the runner
kills the job container: a cancelled job, or the runner's own time limit. One such stack ran on the
shared runner for two days. So stack cleanup has three layers:
- **The script's own trap**, for every ordinary failure.
- **A final `if: always()` step** in every job that starts stacks: `scripts/ci-sweep.sh --job` removes
everything that job started, at any age, even after a failed step.
- **A sweep before every such job starts**: `scripts/ci-sweep.sh --stale` removes anything a CI job
left more than three hours ago, which catches whatever a killed job could not clean up itself.
A stack is found by name and label, never by guesswork (`scripts/lib/ci-project.sh`):
- every CI Compose project is named `ekklesia-ci-<run>-<attempt>-<job key>-…`, and Compose labels
its containers, networks, and volumes with that name;
- anything a script starts outside Compose carries `org.riverofglorymn.ekklesia.ci.project`;
- images also carry the time CI built them, because an image's own creation time can be its base's.
The job key is per matrix leg, so one leg's cleanup cannot match another's running stack. Nothing
outside the `ekklesia-ci-` prefix is ever removed. A job container attached to a stack's network is
disconnected, never removed.
**This is not a substitute for a runner-side reaper.** A stack killed with its job is removed only
when another job that sweeps runs on the same runner. Bounded execution and teardown are still the
runner host's to enforce, as above.
### Runner topology is not verified ### Runner topology is not verified
**The runner topology is not verified in this repository and is not documented here as though it **The runner topology is not verified in this repository and is not documented here as though it
+6
View File
@@ -159,6 +159,12 @@ also receives a safely named disposable database. Direct host runs use a Docker-
port bound only to `127.0.0.1`; a containerized Gitea job joins the isolated Compose network and port bound only to `127.0.0.1`; a containerized Gitea job joins the isolated Compose network and
uses the PostgreSQL service name. uses the PostgreSQL service name.
In CI, a stack whose job was killed before its own cleanup ran is still removed: by that job's final
`if: always()` step, or by the stale sweep before a later job starts (`scripts/ci-sweep.sh`, see
[Disposable stacks never outlive their job](security.md#disposable-stacks-never-outlive-their-job)).
`bash scripts/test-ci-sweep.sh` proves what the sweep removes and what it leaves alone, against real
Docker resources.
The live suite proves fresh read-only planning, initial and incremental application, genuine The live suite proves fresh read-only planning, initial and incremental application, genuine
no-op behavior, exact applied checksums and history, edited or corrupt history rejection, no-op behavior, exact applied checksums and history, edited or corrupt history rejection,
whole-batch DDL rollback, Drizzle breakpoint execution, real advisory-lock concurrency and whole-batch DDL rollback, Drizzle breakpoint execution, real advisory-lock concurrency and
+205
View File
@@ -0,0 +1,205 @@
#!/usr/bin/env bash
# Removes disposable CI stacks from the Docker daemon: containers, networks, volumes, images, and
# staging directories on the Docker host.
#
# bash scripts/ci-sweep.sh --stale [--max-age-seconds N] any CI stack older than N (default 3 h)
# bash scripts/ci-sweep.sh --job this job's own stacks, at any age
#
# Every CI script already removes its own stack in an EXIT trap, but a trap never runs when the runner
# kills the job container: a cancelled job, or the runner's own time limit (Gitea 1.24 ignores
# `timeout-minutes`). So each workflow job that starts stacks ends with `--job` in an `if: always()`
# step, and begins with `--stale`, which catches whatever a killed job left before it could clean up.
#
# What a stack is, and how it is found, is defined in scripts/lib/ci-project.sh: a Compose project
# named with the `ekklesia-ci-` prefix (Compose labels everything it creates with that name), plus
# anything labeled EKKLESIA_CI_LABEL. Nothing outside that prefix is ever touched, at any age.
#
# --stale's threshold is far above any job's run time, so a stack another job on this daemon is using
# right now is never young enough to match. --job refuses to run outside CI: locally, its prefix would
# match a developer's own stacks.
#
# Best effort by design: a resource that cannot be removed is reported, and the sweep goes on. Output
# names resources only; nothing a stack holds is printed.
set -Eeuo pipefail
repository_root="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")/.." && pwd)"
# shellcheck source=scripts/lib/ci-project.sh
source "$repository_root/scripts/lib/ci-project.sh"
compose_label="com.docker.compose.project"
mode=""
max_age_seconds="${EKKLESIA_CI_SWEEP_MAX_AGE_SECONDS:-10800}"
usage() {
echo "usage: ci-sweep.sh --stale [--max-age-seconds N] | --job" >&2
exit 2
}
while (($# > 0)); do
case "$1" in
--stale | --job)
[[ -z "$mode" ]] || usage
mode="${1#--}"
;;
--max-age-seconds)
[[ "${2:-}" =~ ^[0-9]+$ ]] || usage
max_age_seconds="$2"
shift
;;
*) usage ;;
esac
shift
done
[[ -n "$mode" ]] || usage
if [[ "$mode" == job ]]; then
[[ -n "${GITHUB_RUN_ID:-}" ]] || {
echo "ci-sweep: --job runs only in CI (GITHUB_RUN_ID is unset); locally it would match your own stacks." >&2
exit 2
}
prefix="$(ci_job_prefix)"
cutoff=""
else
prefix="$EKKLESIA_CI_PREFIX"
cutoff="$(($(date +%s) - max_age_seconds))"
fi
if ! docker info >/dev/null 2>&1; then
echo "ci-sweep: no Docker daemon reachable; nothing to sweep."
exit 0
fi
removed=0
warn() {
echo "ci-sweep: warning: $*" >&2
}
# selected <project> <created>: whether a resource belongs to the sweep.
selected() {
local project="$1" created="$2" seconds
[[ -n "$project" && "$project" == "$prefix"* ]] || return 1
[[ -z "$cutoff" ]] && return 0
seconds="$(date -d "$created" +%s 2>/dev/null)" || return 1
((seconds <= cutoff))
}
# --- Containers: Compose's own, and those a script started itself --------------------------------
mapfile -t containers < <(
{
docker ps --all --quiet --no-trunc --filter "label=${compose_label}"
docker ps --all --quiet --no-trunc --filter "label=${EKKLESIA_CI_LABEL}"
} | sort -u
)
for id in "${containers[@]}"; do
[[ -n "$id" ]] || continue
record="$(docker inspect --format \
"{{index .Config.Labels \"${compose_label}\"}}|{{index .Config.Labels \"${EKKLESIA_CI_LABEL}\"}}|{{.Created}}|{{.Name}}" \
"$id" 2>/dev/null)" || continue
IFS='|' read -r compose_project ci_project created name <<<"$record"
project="${compose_project:-$ci_project}"
selected "$project" "$created" || continue
if docker rm --force --volumes "$id" >/dev/null 2>&1; then
echo "ci-sweep: removed container ${name#/} (${project})"
removed=$((removed + 1))
else
warn "could not remove container ${name#/}"
fi
done
# --- Networks: anything still attached (a job container) is disconnected, never removed ---------
mapfile -t networks < <(docker network ls --quiet --no-trunc --filter "label=${compose_label}")
for id in "${networks[@]}"; do
[[ -n "$id" ]] || continue
# A network's creation time is a time value, not a string: ask for RFC 3339, which `date` reads.
record="$(docker network inspect --format \
"{{index .Labels \"${compose_label}\"}}|{{.Created.Format \"2006-01-02T15:04:05Z07:00\"}}|{{.Name}}" \
"$id" 2>/dev/null)" || continue
IFS='|' read -r project created name <<<"$record"
selected "$project" "$created" || continue
for endpoint in $(docker network inspect --format '{{range $id, $_ := .Containers}}{{$id}} {{end}}' "$id" 2>/dev/null); do
docker network disconnect --force "$id" "$endpoint" >/dev/null 2>&1 ||
warn "could not disconnect a container from network ${name}"
done
if docker network rm "$id" >/dev/null 2>&1; then
echo "ci-sweep: removed network ${name} (${project})"
removed=$((removed + 1))
else
warn "could not remove network ${name}"
fi
done
# --- Volumes ---------------------------------------------------------------------------------------
mapfile -t volumes < <(
{
docker volume ls --quiet --filter "label=${compose_label}"
docker volume ls --quiet --filter "label=${EKKLESIA_CI_LABEL}"
} | sort -u
)
for name in "${volumes[@]}"; do
[[ -n "$name" ]] || continue
record="$(docker volume inspect --format \
"{{index .Labels \"${compose_label}\"}}|{{index .Labels \"${EKKLESIA_CI_LABEL}\"}}|{{.CreatedAt}}" \
"$name" 2>/dev/null)" || continue
IFS='|' read -r compose_project ci_project created <<<"$record"
project="${compose_project:-$ci_project}"
selected "$project" "$created" || continue
if docker volume rm --force "$name" >/dev/null 2>&1; then
echo "ci-sweep: removed volume ${name} (${project})"
removed=$((removed + 1))
else
warn "could not remove volume ${name}"
fi
done
# --- Images built for a stack ------------------------------------------------------------------------
mapfile -t images < <(docker image ls --quiet --no-trunc --filter "label=${EKKLESIA_CI_LABEL}" | sort -u)
for id in "${images[@]}"; do
[[ -n "$id" ]] || continue
# Judged by when CI built it (EKKLESIA_CI_CREATED_LABEL), never by the image's own creation time,
# which BuildKit may have taken from its base. An image without that label is never swept by age.
record="$(docker image inspect --format \
"{{index .Config.Labels \"${EKKLESIA_CI_LABEL}\"}}|{{index .Config.Labels \"${EKKLESIA_CI_CREATED_LABEL}\"}}" \
"$id" 2>/dev/null)" || continue
IFS='|' read -r project built <<<"$record"
if [[ -n "$cutoff" ]]; then
[[ "$built" =~ ^[0-9]+$ ]] || continue
selected "$project" "@${built}" || continue
else
selected "$project" "" || continue
fi
if docker image rm --force "$id" >/dev/null 2>&1; then
echo "ci-sweep: removed image ${id#sha256:} (${project})"
removed=$((removed + 1))
else
warn "could not remove image ${id#sha256:}"
fi
done
# --- Staging directories on the Docker host ------------------------------------------------------
# A deploy smoke run inside a job container stages its mounted files under the Docker host's /tmp,
# named for its project, as root-owned files; they hold that run's disposable secrets and keys. They
# are removed with the same pinned image that created them. If that image is not on this daemon, no
# smoke has staged anything here.
image="$(sed -n 's/^[[:space:]]*image: \(postgres:[^[:space:]]*\)$/\1/p' "$repository_root/deploy/compose.yaml" | head -1)"
if [[ -n "$image" ]] && docker image inspect "$image" >/dev/null 2>&1; then
age=()
# `find -mmin +0` does not match something younger than a minute, so a threshold under a minute
# sweeps without an age filter, like the rest of this script.
if [[ -n "$cutoff" ]] && ((max_age_seconds >= 60)); then
age=(-mmin "+$((max_age_seconds / 60))")
fi
if staged="$(docker run --rm --entrypoint find -v /tmp:/host-tmp "$image" \
/host-tmp -mindepth 1 -maxdepth 1 -type d -name "${prefix}*" "${age[@]}" \
-print -exec rm -rf {} + 2>/dev/null)"; then
while IFS= read -r directory; do
[[ -n "$directory" ]] || continue
echo "ci-sweep: removed staging directory /tmp/${directory#/host-tmp/}"
removed=$((removed + 1))
done <<<"$staged"
else
warn "could not sweep staging directories on the Docker host"
fi
fi
echo "ci-sweep: ${mode} sweep removed ${removed} resource(s)."
+40
View File
@@ -0,0 +1,40 @@
# shellcheck shell=bash
# How CI names, labels, and later finds the disposable stacks it starts.
#
# Every Compose project a CI script starts is named
#
# ekklesia-ci-<run id>-<run attempt>-<job key>-<kind>-<pid><random>
#
# and Compose labels every container, network, and volume it creates with that name. What a script
# creates outside Compose (a browser container, a volume, an image) gets EKKLESIA_CI_LABEL with the
# same name. scripts/ci-sweep.sh finds stacks by those labels: this job's own, by the prefix up to
# the job key (in a final `if: always()` step), and any CI stack by age (before a job starts).
#
# The job key is EKKLESIA_CI_JOB_KEY, which the workflow sets per job, and per matrix leg where a job
# has one, so that one leg's cleanup cannot match another leg's running stack. Outside CI the
# identifiers are "local".
# shellcheck disable=SC2034 # read by the scripts that source this
EKKLESIA_CI_LABEL="org.riverofglorymn.ekklesia.ci.project"
# On images, when they were built, in Unix seconds. An image's own creation time is not that: BuildKit
# gives an image that only adds metadata its base's time, and a reproducible build a fixed one. A
# stale sweep that trusted it could remove an image another job is using right now.
EKKLESIA_CI_CREATED_LABEL="org.riverofglorymn.ekklesia.ci.created"
EKKLESIA_CI_PREFIX="ekklesia-ci-"
# ci_name_safe <text>: lower case, and only what a Compose project name may contain.
ci_name_safe() {
local text="${1//[^a-zA-Z0-9_-]/-}"
printf '%s' "${text,,}"
}
# ci_job_prefix: every project this job starts begins with exactly this.
ci_job_prefix() {
ci_name_safe "ekklesia-ci-${GITHUB_RUN_ID:-local}-${GITHUB_RUN_ATTEMPT:-0}-${EKKLESIA_CI_JOB_KEY:-${GITHUB_JOB:-local}}-"
}
# ci_project_name <kind>: a project name unique to this invocation. Not only per run: several runs,
# and several jobs of one run, share a runner's Docker daemon, and job containers often share a PID.
ci_project_name() {
ci_name_safe "ekklesia-ci-${GITHUB_RUN_ID:-local}-${GITHUB_RUN_ATTEMPT:-0}-${EKKLESIA_CI_JOB_KEY:-${GITHUB_JOB:-local}}-${1:?ci_project_name needs a kind}-$$-${RANDOM}${RANDOM}"
}
+15 -5
View File
@@ -39,6 +39,8 @@ cd "$repository_root"
# shellcheck source=scripts/lib/compose-diagnostics.sh # shellcheck source=scripts/lib/compose-diagnostics.sh
source "$repository_root/scripts/lib/compose-diagnostics.sh" source "$repository_root/scripts/lib/compose-diagnostics.sh"
# shellcheck source=scripts/lib/ci-project.sh
source "$repository_root/scripts/lib/ci-project.sh"
for command in docker openssl awk; do for command in docker openssl awk; do
command -v "$command" >/dev/null || { command -v "$command" >/dev/null || {
@@ -67,7 +69,10 @@ same-origin | split) ;;
esac esac
run_id="$(openssl rand -hex 4)" run_id="$(openssl rand -hex 4)"
project="ekklesia-deploy-smoke-${run_id}" # Unique per invocation, and findable by run and job if this script is killed before its trap runs
# (scripts/lib/ci-project.sh, scripts/ci-sweep.sh). Everything started outside Compose below carries
# the same name in EKKLESIA_CI_LABEL.
project="$(ci_project_name "deploy-${topology}")"
work="$(mktemp -d)" work="$(mktemp -d)"
image="${EKKLESIA_SMOKE_IMAGE:-}" image="${EKKLESIA_SMOKE_IMAGE:-}"
built_image="" built_image=""
@@ -186,7 +191,9 @@ fail() {
if [[ -z "$image" && "$topology" == same-origin ]]; then if [[ -z "$image" && "$topology" == same-origin ]]; then
step "Building the platform-only image" step "Building the platform-only image"
built_image="ekklesia-deploy-smoke:${run_id}" built_image="ekklesia-deploy-smoke:${run_id}"
DOCKER_BUILDKIT=1 docker build --quiet --file deploy/Dockerfile --tag "$built_image" . >/dev/null DOCKER_BUILDKIT=1 docker build --quiet --file deploy/Dockerfile --tag "$built_image" \
--label "${EKKLESIA_CI_LABEL}=${project}" --label "${EKKLESIA_CI_CREATED_LABEL}=$(date +%s)" \
. >/dev/null
image="$built_image" image="$built_image"
elif [[ -z "$image" ]]; then elif [[ -z "$image" ]]; then
# TEST-ONLY: the repository's module template, installed as the module `example`, in a copy of the # TEST-ONLY: the repository's module template, installed as the module `example`, in a copy of the
@@ -206,6 +213,7 @@ elif [[ -z "$image" ]]; then
sed -i 's|^ templates/module:$| modules/example:|' "$context/pnpm-lock.yaml" sed -i 's|^ templates/module:$| modules/example:|' "$context/pnpm-lock.yaml"
built_image="ekklesia-deploy-smoke:${run_id}" built_image="ekklesia-deploy-smoke:${run_id}"
DOCKER_BUILDKIT=1 docker build --quiet --file "$context/deploy/Dockerfile" --tag "$built_image" \ DOCKER_BUILDKIT=1 docker build --quiet --file "$context/deploy/Dockerfile" --tag "$built_image" \
--label "${EKKLESIA_CI_LABEL}=${project}" --label "${EKKLESIA_CI_CREATED_LABEL}=$(date +%s)" \
"$context" >/dev/null || fail "the test-module image did not build." "$context" >/dev/null || fail "the test-module image did not build."
rm -rf "$context" rm -rf "$context"
image="$built_image" image="$built_image"
@@ -362,6 +370,7 @@ client() {
local mode="$1" local mode="$1"
shift shift
docker run --rm --read-only --cap-drop ALL --security-opt no-new-privileges \ docker run --rm --read-only --cap-drop ALL --security-opt no-new-privileges \
--label "${EKKLESIA_CI_LABEL}=${project}" \
--network "$edge_network" "$@" "${host_mappings[@]}" \ --network "$edge_network" "$@" "${host_mappings[@]}" \
-e NODE_EXTRA_CA_CERTS=/smoke/client-ca.pem -e SMOKE_HOSTNAME="$hostname" \ -e NODE_EXTRA_CA_CERTS=/smoke/client-ca.pem -e SMOKE_HOSTNAME="$hostname" \
-e SMOKE_DASHBOARD_HOSTNAME="$dashboard_hostname" \ -e SMOKE_DASHBOARD_HOSTNAME="$dashboard_hostname" \
@@ -401,7 +410,7 @@ if [[ "$topology" == split ]]; then
browser_container="${project}-browser" browser_container="${project}-browser"
# The image's entry script passes its arguments on unquoted, so none may contain a space: the # The image's entry script passes its arguments on unquoted, so none may contain a space: the
# hostnames reach Chromium through /etc/hosts, which it reads, rather than a resolver rule. # hostnames reach Chromium through /etc/hosts, which it reads, rather than a resolver rule.
docker run --detach --rm --name "$browser_container" \ docker run --detach --rm --name "$browser_container" --label "${EKKLESIA_CI_LABEL}=${project}" \
--cap-drop ALL --security-opt no-new-privileges --tmpfs /tmp \ --cap-drop ALL --security-opt no-new-privileges --tmpfs /tmp \
--network "$edge_network" --ip "$browser_address" "${host_mappings[@]}" \ --network "$edge_network" --ip "$browser_address" "${host_mappings[@]}" \
"$browser_image" \ "$browser_image" \
@@ -410,6 +419,7 @@ if [[ "$topology" == split ]]; then
cp scripts/lib/deploy-smoke-browser.mjs "$work/browser.mjs" cp scripts/lib/deploy-smoke-browser.mjs "$work/browser.mjs"
stage browser.mjs stage browser.mjs
docker run --rm --read-only --cap-drop ALL --security-opt no-new-privileges \ docker run --rm --read-only --cap-drop ALL --security-opt no-new-privileges \
--label "${EKKLESIA_CI_LABEL}=${project}" \
--network "$edge_network" \ --network "$edge_network" \
-e SMOKE_CDP_ADDRESS="${browser_address}:9222" \ -e SMOKE_CDP_ADDRESS="${browser_address}:9222" \
-e SMOKE_HOSTNAME="$hostname" -e SMOKE_DASHBOARD_HOSTNAME="$dashboard_hostname" \ -e SMOKE_HOSTNAME="$hostname" -e SMOKE_DASHBOARD_HOSTNAME="$dashboard_hostname" \
@@ -423,9 +433,9 @@ if [[ "$topology" == split ]]; then
kiosk_container="${project}-kiosk" kiosk_container="${project}-kiosk"
kiosk_volume="${project}-kiosk-profile" kiosk_volume="${project}-kiosk-profile"
kiosk_address="10.249.${octet}.250" kiosk_address="10.249.${octet}.250"
docker volume create "$kiosk_volume" >/dev/null docker volume create --label "${EKKLESIA_CI_LABEL}=${project}" "$kiosk_volume" >/dev/null
start_kiosk() { start_kiosk() {
docker run --detach --rm --name "$kiosk_container" \ docker run --detach --rm --name "$kiosk_container" --label "${EKKLESIA_CI_LABEL}=${project}" \
--cap-drop ALL --security-opt no-new-privileges --tmpfs /tmp \ --cap-drop ALL --security-opt no-new-privileges --tmpfs /tmp \
--network "$edge_network" --ip "$kiosk_address" "${host_mappings[@]}" \ --network "$edge_network" --ip "$kiosk_address" "${host_mappings[@]}" \
-v "${kiosk_volume}:/profile" \ -v "${kiosk_volume}:/profile" \
+5 -5
View File
@@ -9,6 +9,8 @@ cd "$repository_root"
source "$repository_root/scripts/lib/compose-job-network.sh" source "$repository_root/scripts/lib/compose-job-network.sh"
# shellcheck source=scripts/lib/compose-diagnostics.sh # shellcheck source=scripts/lib/compose-diagnostics.sh
source "$repository_root/scripts/lib/compose-diagnostics.sh" source "$repository_root/scripts/lib/compose-diagnostics.sh"
# shellcheck source=scripts/lib/ci-project.sh
source "$repository_root/scripts/lib/ci-project.sh"
command -v docker >/dev/null || { command -v docker >/dev/null || {
echo "Docker is required for the infrastructure smoke test." >&2 echo "Docker is required for the infrastructure smoke test." >&2
@@ -30,11 +32,9 @@ if [[ "$curl_help" != *"--aws-sigv4"* ]]; then
exit 1 exit 1
fi fi
# Unique per invocation, not only per run: several runs can share a runner's Docker daemon at once, # Unique per invocation, and findable by run and job if this script is killed before its trap runs
# and the shell's PID is often identical across job containers. # (scripts/lib/ci-project.sh, scripts/ci-sweep.sh).
raw_project="ekklesia-smoke-${GITHUB_RUN_ID:-local}-${GITHUB_RUN_ATTEMPT:-0}-$$-${RANDOM}${RANDOM}" project="$(ci_project_name infrastructure)"
project="${raw_project//[^a-zA-Z0-9_-]/-}"
project="${project,,}"
temporary_directory="$(mktemp -d)" temporary_directory="$(mktemp -d)"
payload_file="$temporary_directory/payload.txt" payload_file="$temporary_directory/payload.txt"
download_file="$temporary_directory/download.txt" download_file="$temporary_directory/download.txt"
+2 -1
View File
@@ -62,7 +62,8 @@ make_stack() {
--network "${project}_default" -v "${project}_data:/data" --entrypoint sleep "$image" 600 >/dev/null --network "${project}_default" -v "${project}_data:/data" --entrypoint sleep "$image" 600 >/dev/null
docker run --detach --name "${project}-browser" --label "${ci_label}=${project}" \ docker run --detach --name "${project}-browser" --label "${ci_label}=${project}" \
--entrypoint sleep "$image" 600 >/dev/null --entrypoint sleep "$image" 600 >/dev/null
printf 'FROM %s\nLABEL %s=%s\n' "$image" "$ci_label" "$project" | printf 'FROM %s\nLABEL %s=%s %s=%s\n' "$image" "$ci_label" "$project" \
"org.riverofglorymn.ekklesia.ci.created" "$(date +%s)" |
docker build --quiet --tag "${project}-image:test" - >/dev/null docker build --quiet --tag "${project}-image:test" - >/dev/null
docker run --rm --entrypoint mkdir -v /tmp:/host-tmp "$image" -p "/host-tmp/${project}" docker run --rm --entrypoint mkdir -v /tmp:/host-tmp "$image" -p "/host-tmp/${project}"
} }
+5 -5
View File
@@ -9,6 +9,8 @@ cd "$repository_root"
source "$repository_root/scripts/lib/compose-job-network.sh" source "$repository_root/scripts/lib/compose-job-network.sh"
# shellcheck source=scripts/lib/compose-diagnostics.sh # shellcheck source=scripts/lib/compose-diagnostics.sh
source "$repository_root/scripts/lib/compose-diagnostics.sh" source "$repository_root/scripts/lib/compose-diagnostics.sh"
# shellcheck source=scripts/lib/ci-project.sh
source "$repository_root/scripts/lib/ci-project.sh"
command -v docker >/dev/null || { command -v docker >/dev/null || {
echo "Docker is required for the PostgreSQL migration integration suite." >&2 echo "Docker is required for the PostgreSQL migration integration suite." >&2
@@ -27,11 +29,9 @@ if ! compose_job_network_initialize; then
exit 1 exit 1
fi fi
# Unique per invocation, not only per run: several runs (and several jobs of one run) can share a # Unique per invocation, and findable by run and job if this script is killed before its trap runs
# runner's Docker daemon at once, and the shell's PID is often identical across job containers. # (scripts/lib/ci-project.sh, scripts/ci-sweep.sh).
raw_project="ekklesia-migration-${GITHUB_RUN_ID:-local}-${GITHUB_RUN_ATTEMPT:-0}-${GITHUB_JOB:-job}-$$-${RANDOM}${RANDOM}" project="$(ci_project_name migration)"
project="${raw_project//[^a-zA-Z0-9_-]/-}"
project="${project,,}"
export POSTGRES_DB="ekklesia_migration_test" export POSTGRES_DB="ekklesia_migration_test"
export POSTGRES_USER="ekklesia_migration_test" export POSTGRES_USER="ekklesia_migration_test"