#!/usr/bin/env bash # Removes disposable CI stacks from the Docker daemon: containers, networks, volumes, images, and # staging directories on the Docker host. # # bash scripts/ci-sweep.sh --stale [--max-age-seconds N] any CI stack older than N (default 3 h) # bash scripts/ci-sweep.sh --job this job's own stacks, at any age # # Every CI script already removes its own stack in an EXIT trap, but a trap never runs when the runner # kills the job container: a cancelled or superseded job (the server honours `concurrency`), or # the runner's own time limit. So each workflow job that starts stacks ends with `--job` in an # `if: always()` step, and begins with `--stale`, which catches whatever a killed job left before # it could clean up. # # What a stack is, and how it is found, is defined in scripts/lib/ci-project.sh: a Compose project # named with the `ekklesia-ci-` prefix (Compose labels everything it creates with that name), plus # anything labeled EKKLESIA_CI_LABEL. Nothing outside that prefix is ever touched, at any age. # # --stale's threshold is far above any job's run time, so a stack another job on this daemon is using # right now is never young enough to match. --job refuses to run outside CI: locally, its prefix would # match a developer's own stacks. # # Best effort by design: a resource that cannot be removed is reported, and the sweep goes on. Output # names resources only; nothing a stack holds is printed. set -Eeuo pipefail repository_root="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")/.." && pwd)" # shellcheck source=scripts/lib/ci-project.sh source "$repository_root/scripts/lib/ci-project.sh" compose_label="com.docker.compose.project" mode="" max_age_seconds="${EKKLESIA_CI_SWEEP_MAX_AGE_SECONDS:-10800}" usage() { echo "usage: ci-sweep.sh --stale [--max-age-seconds N] | --job" >&2 exit 2 } while (($# > 0)); do case "$1" in --stale | --job) [[ -z "$mode" ]] || usage mode="${1#--}" ;; --max-age-seconds) [[ "${2:-}" =~ ^[0-9]+$ ]] || usage max_age_seconds="$2" shift ;; *) usage ;; esac shift done [[ -n "$mode" ]] || usage if [[ "$mode" == job ]]; then [[ -n "${GITHUB_RUN_ID:-}" ]] || { echo "ci-sweep: --job runs only in CI (GITHUB_RUN_ID is unset); locally it would match your own stacks." >&2 exit 2 } prefix="$(ci_job_prefix)" cutoff="" else prefix="$EKKLESIA_CI_PREFIX" cutoff="$(($(date +%s) - max_age_seconds))" fi if ! docker info >/dev/null 2>&1; then echo "ci-sweep: no Docker daemon reachable; nothing to sweep." exit 0 fi removed=0 warn() { echo "ci-sweep: warning: $*" >&2 } # selected : whether a resource belongs to the sweep. selected() { local project="$1" created="$2" seconds [[ -n "$project" && "$project" == "$prefix"* ]] || return 1 [[ -z "$cutoff" ]] && return 0 seconds="$(date -d "$created" +%s 2>/dev/null)" || return 1 ((seconds <= cutoff)) } # --- Containers: Compose's own, and those a script started itself -------------------------------- mapfile -t containers < <( { docker ps --all --quiet --no-trunc --filter "label=${compose_label}" docker ps --all --quiet --no-trunc --filter "label=${EKKLESIA_CI_LABEL}" } | sort -u ) for id in "${containers[@]}"; do [[ -n "$id" ]] || continue record="$(docker inspect --format \ "{{index .Config.Labels \"${compose_label}\"}}|{{index .Config.Labels \"${EKKLESIA_CI_LABEL}\"}}|{{.Created}}|{{.Name}}" \ "$id" 2>/dev/null)" || continue IFS='|' read -r compose_project ci_project created name <<<"$record" project="${compose_project:-$ci_project}" selected "$project" "$created" || continue if docker rm --force --volumes "$id" >/dev/null 2>&1; then echo "ci-sweep: removed container ${name#/} (${project})" removed=$((removed + 1)) else warn "could not remove container ${name#/}" fi done # --- Networks: anything still attached (a job container) is disconnected, never removed --------- mapfile -t networks < <(docker network ls --quiet --no-trunc --filter "label=${compose_label}") for id in "${networks[@]}"; do [[ -n "$id" ]] || continue # A network's creation time is a time value, not a string: ask for RFC 3339, which `date` reads. record="$(docker network inspect --format \ "{{index .Labels \"${compose_label}\"}}|{{.Created.Format \"2006-01-02T15:04:05Z07:00\"}}|{{.Name}}" \ "$id" 2>/dev/null)" || continue IFS='|' read -r project created name <<<"$record" selected "$project" "$created" || continue for endpoint in $(docker network inspect --format '{{range $id, $_ := .Containers}}{{$id}} {{end}}' "$id" 2>/dev/null); do docker network disconnect --force "$id" "$endpoint" >/dev/null 2>&1 || warn "could not disconnect a container from network ${name}" done if docker network rm "$id" >/dev/null 2>&1; then echo "ci-sweep: removed network ${name} (${project})" removed=$((removed + 1)) else warn "could not remove network ${name}" fi done # --- Volumes --------------------------------------------------------------------------------------- mapfile -t volumes < <( { docker volume ls --quiet --filter "label=${compose_label}" docker volume ls --quiet --filter "label=${EKKLESIA_CI_LABEL}" } | sort -u ) for name in "${volumes[@]}"; do [[ -n "$name" ]] || continue record="$(docker volume inspect --format \ "{{index .Labels \"${compose_label}\"}}|{{index .Labels \"${EKKLESIA_CI_LABEL}\"}}|{{.CreatedAt}}" \ "$name" 2>/dev/null)" || continue IFS='|' read -r compose_project ci_project created <<<"$record" project="${compose_project:-$ci_project}" selected "$project" "$created" || continue if docker volume rm --force "$name" >/dev/null 2>&1; then echo "ci-sweep: removed volume ${name} (${project})" removed=$((removed + 1)) else warn "could not remove volume ${name}" fi done # --- Images built for a stack ------------------------------------------------------------------------ mapfile -t images < <(docker image ls --quiet --no-trunc --filter "label=${EKKLESIA_CI_LABEL}" | sort -u) for id in "${images[@]}"; do [[ -n "$id" ]] || continue # Judged by when CI built it (EKKLESIA_CI_CREATED_LABEL), never by the image's own creation time, # which BuildKit may have taken from its base. An image without that label is never swept by age. record="$(docker image inspect --format \ "{{index .Config.Labels \"${EKKLESIA_CI_LABEL}\"}}|{{index .Config.Labels \"${EKKLESIA_CI_CREATED_LABEL}\"}}" \ "$id" 2>/dev/null)" || continue IFS='|' read -r project built <<<"$record" if [[ -n "$cutoff" ]]; then [[ "$built" =~ ^[0-9]+$ ]] || continue selected "$project" "@${built}" || continue else selected "$project" "" || continue fi if docker image rm --force "$id" >/dev/null 2>&1; then echo "ci-sweep: removed image ${id#sha256:} (${project})" removed=$((removed + 1)) else warn "could not remove image ${id#sha256:}" fi done # --- Staging directories on the Docker host ------------------------------------------------------ # A deploy smoke run inside a job container stages its mounted files under the Docker host's /tmp, # named for its project, as root-owned files; they hold that run's disposable secrets and keys. They # are removed with the same pinned image that created them. If that image is not on this daemon, no # smoke has staged anything here. image="$(sed -n 's/^[[:space:]]*image: \(postgres:[^[:space:]]*\)$/\1/p' "$repository_root/deploy/compose.yaml" | head -1)" if [[ -n "$image" ]] && docker image inspect "$image" >/dev/null 2>&1; then age=() # `find -mmin +0` does not match something younger than a minute, so a threshold under a minute # sweeps without an age filter, like the rest of this script. if [[ -n "$cutoff" ]] && ((max_age_seconds >= 60)); then age=(-mmin "+$((max_age_seconds / 60))") fi if staged="$(docker run --rm --entrypoint find -v /tmp:/host-tmp "$image" \ /host-tmp -mindepth 1 -maxdepth 1 -type d -name "${prefix}*" "${age[@]}" \ -print -exec rm -rf {} + 2>/dev/null)"; then while IFS= read -r directory; do [[ -n "$directory" ]] || continue echo "ci-sweep: removed staging directory /tmp/${directory#/host-tmp/}" removed=$((removed + 1)) done <<<"$staged" else warn "could not sweep staging directories on the Docker host" fi fi echo "ci-sweep: ${mode} sweep removed ${removed} resource(s)."