Ekklesia/scripts/ci-sweep.sh

#!/usr/bin/env bash
# Removes disposable CI stacks from the Docker daemon: containers, networks, volumes, images, and
# staging directories on the Docker host.
#
#   bash scripts/ci-sweep.sh --stale [--max-age-seconds N]   any CI stack older than N (default 3 h)
#   bash scripts/ci-sweep.sh --job                           this job's own stacks, at any age
#
# Every CI script already removes its own stack in an EXIT trap, but a trap never runs when the runner
# kills the job container: a cancelled or superseded job (the server honours `concurrency`), or
# the runner's own time limit. So each workflow job that starts stacks ends with `--job` in an
# `if: always()` step, and begins with `--stale`, which catches whatever a killed job left before
# it could clean up.
#
# What a stack is, and how it is found, is defined in scripts/lib/ci-project.sh: a Compose project
# named with the `ekklesia-ci-` prefix (Compose labels everything it creates with that name), plus
# anything labeled EKKLESIA_CI_LABEL. Nothing outside that prefix is ever touched, at any age.
#
# --stale's threshold is far above any job's run time, so a stack another job on this daemon is using
# right now is never young enough to match. --job refuses to run outside CI: locally, its prefix would
# match a developer's own stacks.
#
# Best effort by design: a resource that cannot be removed is reported, and the sweep goes on. Output
# names resources only; nothing a stack holds is printed.

set -Eeuo pipefail

repository_root="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")/.." && pwd)"
# shellcheck source=scripts/lib/ci-project.sh
source "$repository_root/scripts/lib/ci-project.sh"

compose_label="com.docker.compose.project"
mode=""
max_age_seconds="${EKKLESIA_CI_SWEEP_MAX_AGE_SECONDS:-10800}"

usage() {
  echo "usage: ci-sweep.sh --stale [--max-age-seconds N] | --job" >&2
  exit 2
}

while (($# > 0)); do
  case "$1" in
  --stale | --job)
    [[ -z "$mode" ]] || usage
    mode="${1#--}"
    ;;
  --max-age-seconds)
    [[ "${2:-}" =~ ^[0-9]+$ ]] || usage
    max_age_seconds="$2"
    shift
    ;;
  *) usage ;;
  esac
  shift
done
[[ -n "$mode" ]] || usage

if [[ "$mode" == job ]]; then
  [[ -n "${GITHUB_RUN_ID:-}" ]] || {
    echo "ci-sweep: --job runs only in CI (GITHUB_RUN_ID is unset); locally it would match your own stacks." >&2
    exit 2
  }
  prefix="$(ci_job_prefix)"
  cutoff=""
else
  prefix="$EKKLESIA_CI_PREFIX"
  cutoff="$(($(date +%s) - max_age_seconds))"
fi

if ! docker info >/dev/null 2>&1; then
  echo "ci-sweep: no Docker daemon reachable; nothing to sweep."
  exit 0
fi

removed=0
warn() {
  echo "ci-sweep: warning: $*" >&2
}

# selected <project> <created>: whether a resource belongs to the sweep.
selected() {
  local project="$1" created="$2" seconds
  [[ -n "$project" && "$project" == "$prefix"* ]] || return 1
  [[ -z "$cutoff" ]] && return 0
  seconds="$(date -d "$created" +%s 2>/dev/null)" || return 1
  ((seconds <= cutoff))
}

# --- Containers: Compose's own, and those a script started itself --------------------------------
mapfile -t containers < <(
  {
    docker ps --all --quiet --no-trunc --filter "label=${compose_label}"
    docker ps --all --quiet --no-trunc --filter "label=${EKKLESIA_CI_LABEL}"
  } | sort -u
)
for id in "${containers[@]}"; do
  [[ -n "$id" ]] || continue
  record="$(docker inspect --format \
    "{{index .Config.Labels \"${compose_label}\"}}|{{index .Config.Labels \"${EKKLESIA_CI_LABEL}\"}}|{{.Created}}|{{.Name}}" \
    "$id" 2>/dev/null)" || continue
  IFS='|' read -r compose_project ci_project created name <<<"$record"
  project="${compose_project:-$ci_project}"
  selected "$project" "$created" || continue
  if docker rm --force --volumes "$id" >/dev/null 2>&1; then
    echo "ci-sweep: removed container ${name#/} (${project})"
    removed=$((removed + 1))
  else
    warn "could not remove container ${name#/}"
  fi
done

# --- Networks: anything still attached (a job container) is disconnected, never removed ---------
mapfile -t networks < <(docker network ls --quiet --no-trunc --filter "label=${compose_label}")
for id in "${networks[@]}"; do
  [[ -n "$id" ]] || continue
  # A network's creation time is a time value, not a string: ask for RFC 3339, which `date` reads.
  record="$(docker network inspect --format \
    "{{index .Labels \"${compose_label}\"}}|{{.Created.Format \"2006-01-02T15:04:05Z07:00\"}}|{{.Name}}" \
    "$id" 2>/dev/null)" || continue
  IFS='|' read -r project created name <<<"$record"
  selected "$project" "$created" || continue
  for endpoint in $(docker network inspect --format '{{range $id, $_ := .Containers}}{{$id}} {{end}}' "$id" 2>/dev/null); do
    docker network disconnect --force "$id" "$endpoint" >/dev/null 2>&1 ||
      warn "could not disconnect a container from network ${name}"
  done
  if docker network rm "$id" >/dev/null 2>&1; then
    echo "ci-sweep: removed network ${name} (${project})"
    removed=$((removed + 1))
  else
    warn "could not remove network ${name}"
  fi
done

# --- Volumes ---------------------------------------------------------------------------------------
mapfile -t volumes < <(
  {
    docker volume ls --quiet --filter "label=${compose_label}"
    docker volume ls --quiet --filter "label=${EKKLESIA_CI_LABEL}"
  } | sort -u
)
for name in "${volumes[@]}"; do
  [[ -n "$name" ]] || continue
  record="$(docker volume inspect --format \
    "{{index .Labels \"${compose_label}\"}}|{{index .Labels \"${EKKLESIA_CI_LABEL}\"}}|{{.CreatedAt}}" \
    "$name" 2>/dev/null)" || continue
  IFS='|' read -r compose_project ci_project created <<<"$record"
  project="${compose_project:-$ci_project}"
  selected "$project" "$created" || continue
  if docker volume rm --force "$name" >/dev/null 2>&1; then
    echo "ci-sweep: removed volume ${name} (${project})"
    removed=$((removed + 1))
  else
    warn "could not remove volume ${name}"
  fi
done

# --- Images built for a stack ------------------------------------------------------------------------
mapfile -t images < <(docker image ls --quiet --no-trunc --filter "label=${EKKLESIA_CI_LABEL}" | sort -u)
for id in "${images[@]}"; do
  [[ -n "$id" ]] || continue
  # Judged by when CI built it (EKKLESIA_CI_CREATED_LABEL), never by the image's own creation time,
  # which BuildKit may have taken from its base. An image without that label is never swept by age.
  record="$(docker image inspect --format \
    "{{index .Config.Labels \"${EKKLESIA_CI_LABEL}\"}}|{{index .Config.Labels \"${EKKLESIA_CI_CREATED_LABEL}\"}}" \
    "$id" 2>/dev/null)" || continue
  IFS='|' read -r project built <<<"$record"
  if [[ -n "$cutoff" ]]; then
    [[ "$built" =~ ^[0-9]+$ ]] || continue
    selected "$project" "@${built}" || continue
  else
    selected "$project" "" || continue
  fi
  if docker image rm --force "$id" >/dev/null 2>&1; then
    echo "ci-sweep: removed image ${id#sha256:} (${project})"
    removed=$((removed + 1))
  else
    warn "could not remove image ${id#sha256:}"
  fi
done

# --- Staging directories on the Docker host ------------------------------------------------------
# A deploy smoke run inside a job container stages its mounted files under the Docker host's /tmp,
# named for its project, as root-owned files; they hold that run's disposable secrets and keys. They
# are removed with the same pinned image that created them. If that image is not on this daemon, no
# smoke has staged anything here.
image="$(sed -n 's/^[[:space:]]*image: \(postgres:[^[:space:]]*\)$/\1/p' "$repository_root/deploy/compose.yaml" | head -1)"
if [[ -n "$image" ]] && docker image inspect "$image" >/dev/null 2>&1; then
  age=()
  # `find -mmin +0` does not match something younger than a minute, so a threshold under a minute
  # sweeps without an age filter, like the rest of this script.
  if [[ -n "$cutoff" ]] && ((max_age_seconds >= 60)); then
    age=(-mmin "+$((max_age_seconds / 60))")
  fi
  if staged="$(docker run --rm --entrypoint find -v /tmp:/host-tmp "$image" \
    /host-tmp -mindepth 1 -maxdepth 1 -type d -name "${prefix}*" "${age[@]}" \
    -print -exec rm -rf {} + 2>/dev/null)"; then
    while IFS= read -r directory; do
      [[ -n "$directory" ]] || continue
      echo "ci-sweep: removed staging directory /tmp/${directory#/host-tmp/}"
      removed=$((removed + 1))
    done <<<"$staged"
  else
    warn "could not sweep staging directories on the Docker host"
  fi
fi

echo "ci-sweep: ${mode} sweep removed ${removed} resource(s)."