Ekklesia/scripts/smoke-deploy.sh

#!/usr/bin/env bash
# End-to-end smoke test of the reference deployment (ADR 0044): deploy/compose.yaml with
# deploy/compose.proxy.yaml, a platform-only image, and a disposable PostgreSQL and SeaweedFS that
# exist only for this run. CI runs it; it is safe to run on a workstation.
#
#   bash scripts/smoke-deploy.sh                 builds the image first
#   EKKLESIA_SMOKE_IMAGE=<image> bash scripts/smoke-deploy.sh
#   EKKLESIA_SMOKE_TOPOLOGY=split bash scripts/smoke-deploy.sh
#
# Either topology (ADR 0045). `same-origin` (the default) serves the Dashboard, the API, and displays
# from one hostname with Caddyfile.example and a platform-only image. `split` serves the Dashboard on
# its own hostname with Caddyfile.split.example, builds an image with the repository's module
# template installed as a test-only module (so a module UI, its API routes, and a display exist), and
# then drives a real headless Chromium through the Dashboard, the module, the display, and sign-out.
# With EKKLESIA_SMOKE_IMAGE, the given image is used as is in either topology; a split run then
# exercises whichever of its modules publishes UI and a display.
#
# Opt-in, split topology only: EKKLESIA_SMOKE_DISPLAY_DEVICE=1 adds the display-device scenarios in
# scripts/lib/deploy-smoke-display.mjs (pairing a kiosk browser with a persistent profile, a clean
# restart, two pages, injected network failures, revocation), each with its own time budget.
# EKKLESIA_SMOKE_DISPLAY_SCENARIOS selects a subset; EKKLESIA_SMOKE_WATCH_MS and
# EKKLESIA_SMOKE_RECYCLE_WATCH_MS change how long they watch. Several scenarios expect Signage.
#
# TEST-ONLY: the proxy runs with Caddy's internal CA (`local_certs`) instead of ACME, because a test
# run has no public hostname. Never use that option in an installation.
#
# Follows the documented first boot: TLS material, secrets, data services, an explicit migration
# apply against this run's own fresh database, then the API and proxy. Everything it starts, and
# every file it writes, is removed on exit.
#
# Bind mounts are resolved by the Docker daemon, on its host. When this script runs inside a CI job
# container that drives the host's Docker, its own files are not at those paths, so every file a
# container mounts is staged under an identical path in the Docker host's /tmp first.

set -Eeuo pipefail

repository_root="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")/.." && pwd)"
cd "$repository_root"

# shellcheck source=scripts/lib/compose-diagnostics.sh
source "$repository_root/scripts/lib/compose-diagnostics.sh"
# shellcheck source=scripts/lib/ci-project.sh
source "$repository_root/scripts/lib/ci-project.sh"

for command in docker openssl awk; do
  command -v "$command" >/dev/null || {
    echo "smoke-deploy: $command is required." >&2
    exit 1
  }
done
docker compose version >/dev/null 2>&1 || {
  echo "smoke-deploy: Docker Compose is required." >&2
  exit 1
}

# The pinned PostgreSQL image doubles as a root shell for file ownership, so the run needs neither
# root nor sudo on the host.
postgres_image="postgres:18.4-alpine3.24@sha256:9a8afca54e7861fd90fab5fdf4c42477a6b1cb7d293595148e674e0a3181de15"
# The split topology's real browser: headless Chromium, pinned by its multi-architecture index digest.
browser_image="chromedp/headless-shell:151.0.7922.109@sha256:2d349b544a1ea6b5b5fd7c0fe99215ff662339c57407ee2e8c0a11af93516b04"

topology="${EKKLESIA_SMOKE_TOPOLOGY:-same-origin}"
case "$topology" in
same-origin | split) ;;
*)
  echo "smoke-deploy: EKKLESIA_SMOKE_TOPOLOGY must be same-origin or split." >&2
  exit 1
  ;;
esac

run_id="$(openssl rand -hex 4)"
# Unique per invocation, and findable by run and job if this script is killed before its trap runs
# (scripts/lib/ci-project.sh, scripts/ci-sweep.sh). Everything started outside Compose below carries
# the same name in EKKLESIA_CI_LABEL.
project="$(ci_project_name "deploy-${topology}")"
work="$(mktemp -d)"
image="${EKKLESIA_SMOKE_IMAGE:-}"
built_image=""
# `hostname` is always the API's (and the display host's); the Dashboard has its own only when split.
if [[ "$topology" == split ]]; then
  hostname="api.smoke.test"
  dashboard_hostname="dash.smoke.test"
  caddyfile_example="deploy/Caddyfile.split.example"
else
  hostname="ekklesia.smoke.test"
  dashboard_hostname=""
  caddyfile_example="deploy/Caddyfile.example"
fi
dashboard_origin="https://${dashboard_hostname:-$hostname}"
storage_hostname="storage.smoke.test"
browser_container=""

# choose_edge_network: a per-run edge subnet that no network on this Docker daemon already uses, so
# concurrent runs sharing a runner's daemon do not collide. Two runs can still pick the same free
# subnet at the same moment; the first `up` below detects that and chooses again. Fixed addresses come
# from the upper half, outside Docker's dynamic range, exactly as documented.
choose_edge_network() {
  local in_use candidate attempt
  in_use="$(
    docker network ls --quiet |
      xargs -r docker network inspect --format '{{range .IPAM.Config}}{{println .Subnet}}{{end}}' 2>/dev/null |
      sed -n 's#^10\.249\.\([0-9]*\)\.0/24$#\1#p'
  )" || in_use=""
  for attempt in $(seq 1 100); do
    candidate=$((RANDOM % 200 + 20))
    if ! grep -qx "$candidate" <<<"$in_use"; then
      octet="$candidate"
      edge_subnet="10.249.${octet}.0/24"
      edge_range="10.249.${octet}.0/25"
      proxy_address="10.249.${octet}.254"
      tunnel_address="10.249.${octet}.253"
      # A test-only client address, also outside the dynamic range, so its limiter bucket is its own.
      session_client_address="10.249.${octet}.252"
      # The headless browser's fixed address, likewise (split topology only).
      browser_address="10.249.${octet}.251"
      return 0
    fi
  done
  echo "smoke-deploy: no free 10.249.x.0/24 edge subnet on this Docker daemon." >&2
  exit 1
}
choose_edge_network

# daemon_work: where the Docker daemon sees this run's mounted files. The same directory as `work`
# on a workstation; a staged copy on the Docker host when this script runs in a job container.
if [[ -f /.dockerenv ]]; then
  daemon_work="/tmp/${project}"
  docker run --rm --entrypoint mkdir -v /tmp:/host-tmp "$postgres_image" -p "/host-tmp/${project}"
  # stage <path relative to work>: copy it to the same relative path under daemon_work, keeping
  # numeric ownership and modes.
  stage() {
    tar -C "$work" -cf - "$1" |
      docker run --rm -i --entrypoint tar -v "$daemon_work:/w" "$postgres_image" -xpf - -C /w
  }
else
  daemon_work="$work"
  stage() { :; }
fi

compose=(
  docker compose
  --project-name "$project"
  --project-directory "$daemon_work/deploy"
  --env-file "$work/compose.env"
  --file deploy/compose.yaml
  --file deploy/compose.proxy.yaml
)

cleanup() {
  local status=$?
  trap - EXIT INT TERM
  if [[ -n "$browser_container" ]]; then
    docker rm --force "$browser_container" >/dev/null 2>&1 || true
  fi
  if [[ -n "${kiosk_container:-}" ]]; then
    docker ps --all --quiet --filter "name=${project}-display-" | xargs -r docker rm --force \
      >/dev/null 2>&1 || true
    docker rm --force "$kiosk_container" >/dev/null 2>&1 || true
    docker volume rm --force "$kiosk_volume" >/dev/null 2>&1 || true
  fi
  if [[ -f "$work/compose.env" ]]; then
    "${compose[@]}" --profile ops down --volumes --remove-orphans --timeout 10 >/dev/null 2>&1 || status=1
  fi
  # Some files are owned by the PostgreSQL and SeaweedFS uids by now; remove them as root, in the
  # same image that set that ownership. In a job container the daemon cannot see `work` (and the
  # job itself runs as root), so only the staged copy on the Docker host needs this.
  if [[ "$daemon_work" == "$work" ]]; then
    docker run --rm --entrypoint rm -v "$work:/work" "$postgres_image" -rf /work/tls /work/env \
      >/dev/null 2>&1 || true
  else
    docker run --rm --entrypoint rm -v /tmp:/host-tmp "$postgres_image" -rf "/host-tmp/${project}" \
      >/dev/null 2>&1 || true
  fi
  rm -rf "$work" || true
  if [[ -n "$built_image" ]]; then
    docker image rm --force "$built_image" >/dev/null 2>&1 || true
  fi
  exit "$status"
}
trap cleanup EXIT INT TERM

step() {
  echo "==> $*"
}

fail() {
  echo "smoke-deploy: $*" >&2
  exit 1
}

if [[ -z "$image" && "$topology" == same-origin ]]; then
  step "Building the platform-only image"
  built_image="ekklesia-deploy-smoke:${run_id}"
  DOCKER_BUILDKIT=1 docker build --quiet --file deploy/Dockerfile --tag "$built_image" \
    --label "${EKKLESIA_CI_LABEL}=${project}" --label "${EKKLESIA_CI_CREATED_LABEL}=$(date +%s)" \
    . >/dev/null
  image="$built_image"
elif [[ -z "$image" ]]; then
  # TEST-ONLY: the repository's module template, installed as the module `example`, in a copy of the
  # working tree. Its lockfile importer moves with it; the depth is the same, so every relative link
  # in that importer stays correct and the install stays frozen.
  step "Building an image with the module template installed as a test-only module"
  command -v git >/dev/null || fail "git is required to build the split topology's test image."
  context="$work/build-context"
  mkdir -p "$context"
  git ls-files -z --cached --others --exclude-standard |
    tar --null --files-from=- --create --file=- |
    tar -C "$context" -xf -
  mkdir -p "$context/modules"
  mv "$context/templates/module" "$context/modules/example"
  [[ "$(grep -c '^  templates/module:$' "$context/pnpm-lock.yaml")" == 1 ]] ||
    fail "the lockfile has no single templates/module importer to move."
  sed -i 's|^  templates/module:$|  modules/example:|' "$context/pnpm-lock.yaml"
  built_image="ekklesia-deploy-smoke:${run_id}"
  DOCKER_BUILDKIT=1 docker build --quiet --file "$context/deploy/Dockerfile" --tag "$built_image" \
    --label "${EKKLESIA_CI_LABEL}=${project}" --label "${EKKLESIA_CI_CREATED_LABEL}=$(date +%s)" \
    "$context" >/dev/null || fail "the test-module image did not build."
  rm -rf "$context"
  image="$built_image"
fi

# The compose project directory: its relative bind mount (postgres/pg_hba.conf) resolves here.
cp -R deploy "$work/deploy"
stage deploy

step "Creating the installation-private CA and server certificates"
init_status=0
sh deploy/scripts/init-tls.sh "$work/tls" >/dev/null 2>&1 || init_status=$?
if ((init_status != 0 && init_status != 77)); then
  fail "init-tls.sh failed to create TLS material."
fi
stage tls
# The root-only ownership pass, exactly as an operator would re-run the script with root.
ownership_output="$(
  docker run --rm --entrypoint sh -v "$daemon_work/tls:/tls" \
    -v "$daemon_work/deploy/scripts:/scripts:ro" "$postgres_image" /scripts/init-tls.sh /tls
)" || fail "init-tls.sh could not apply ownership."
database_ca_line="$(grep '^EKKLESIA_DATABASE_CA=' <<<"$ownership_output")" ||
  fail "init-tls.sh did not print the EKKLESIA_DATABASE_CA line."
# A second run must create nothing and still succeed.
rerun_output="$(
  docker run --rm --entrypoint sh -v "$daemon_work/tls:/tls" \
    -v "$daemon_work/deploy/scripts:/scripts:ro" "$postgres_image" /scripts/init-tls.sh /tls
)" || fail "init-tls.sh failed on a second run."
grep -q 'nothing to create' <<<"$rerun_output" || fail "init-tls.sh is not idempotent."

step "Writing env files from the examples, with freshly generated secrets"
secrets="$(sh deploy/scripts/gen-secrets.sh)"
secret() {
  awk -F= -v key="$1" '$1 == key { sub(/^[^=]*=/, ""); print; exit }' <<<"$secrets"
}
postgres_password="$(secret POSTGRES_PASSWORD)"
s3_access_key="$(secret S3_ACCESS_KEY)"
s3_secret_key="$(secret S3_SECRET_KEY)"
identity_secret="$(secret EKKLESIA_IDENTITY_SECRET)"
# Masked in any diagnostics below, along with anything else that looks like a credential.
COMPOSE_DIAGNOSTICS_SECRETS=("$postgres_password" "$s3_access_key" "$s3_secret_key" "$identity_secret")

mkdir -p "$work/env"
# set_env <file> <key> <value>: replaces an uncommented KEY= line, keeping the value byte-exact.
set_env() {
  KEY="$2" VALUE="$3" awk '
    BEGIN { found = 0 }
    index($0, ENVIRON["KEY"] "=") == 1 { print ENVIRON["KEY"] "=" ENVIRON["VALUE"]; found = 1; next }
    { print }
    END { if (!found) print ENVIRON["KEY"] "=" ENVIRON["VALUE"] }
  ' "$1" >"$1.next"
  mv "$1.next" "$1"
}
for name in ekklesia postgres object-storage; do
  cp "deploy/env/${name}.env.example" "$work/env/${name}.env"
done
ekklesia_env="$work/env/ekklesia.env"
set_env "$ekklesia_env" DATABASE_URL "postgresql://ekklesia:${postgres_password}@postgres:5432/ekklesia"
set_env "$ekklesia_env" EKKLESIA_DATABASE_CA "${database_ca_line#EKKLESIA_DATABASE_CA=}"
set_env "$ekklesia_env" EKKLESIA_IDENTITY_BASE_URL "https://${hostname}"
set_env "$ekklesia_env" EKKLESIA_IDENTITY_TRUSTED_ORIGINS "$dashboard_origin"
if [[ "$topology" == split ]]; then
  set_env "$ekklesia_env" EKKLESIA_DASHBOARD_ORIGIN "$dashboard_origin"
fi
set_env "$ekklesia_env" EKKLESIA_IDENTITY_SECRET "$identity_secret"
set_env "$ekklesia_env" S3_DELIVERY_ENDPOINT "https://${storage_hostname}"
set_env "$ekklesia_env" S3_ACCESS_KEY "$s3_access_key"
set_env "$ekklesia_env" S3_SECRET_KEY "$s3_secret_key"
set_env "$work/env/postgres.env" POSTGRES_PASSWORD "$postgres_password"
set_env "$work/env/object-storage.env" AWS_ACCESS_KEY_ID "$s3_access_key"
set_env "$work/env/object-storage.env" AWS_SECRET_ACCESS_KEY "$s3_secret_key"
chmod 600 "$work"/env/*.env

# TEST-ONLY: Caddy's internal CA instead of ACME. Inserted as the first global option.
awk '!done && $0 == "{" { print; print "\tlocal_certs"; done = 1; next } { print }' \
  "$caddyfile_example" >"$work/Caddyfile"
grep -q $'^\tlocal_certs$' "$work/Caddyfile" || fail "could not enable local_certs for the test proxy."
stage Caddyfile

cat >"$work/compose.env" <<EOF
EKKLESIA_IMAGE=${image}
EKKLESIA_ENV_DIR=${work}/env
EKKLESIA_TLS_DIR=${daemon_work}/tls
EKKLESIA_HOSTNAME=${hostname}
EKKLESIA_DASHBOARD_HOSTNAME=${dashboard_hostname}
EKKLESIA_STORAGE_HOSTNAME=${storage_hostname}
[email protected]
EKKLESIA_CADDYFILE=${daemon_work}/Caddyfile
EKKLESIA_PUBLISH_ADDRESS=127.0.0.1
EKKLESIA_HTTP_PORT=0
EKKLESIA_HTTPS_PORT=0
EKKLESIA_EDGE_SUBNET=${edge_subnet}
EKKLESIA_EDGE_DYNAMIC_RANGE=${edge_range}
EKKLESIA_PROXY_ADDRESS=${proxy_address}
EKKLESIA_TUNNEL_ADDRESS=${tunnel_address}
EOF

step "Starting PostgreSQL and object storage"
# This `up` creates the edge network. If another run claimed the same subnet since it was chosen,
# Docker refuses with an address-pool overlap: choose another and start again, a bounded number of
# times. Any other failure is reported with the project's containers and logs.
for attempt in 1 2 3 4 5; do
  if up_output="$(
    "${compose[@]}" up --detach --wait --wait-timeout 180 postgres object-storage 2>&1
  )"; then
    break
  fi
  if [[ "$up_output" == *"overlap"* ]] && ((attempt < 5)); then
    echo "     edge subnet ${edge_subnet} was taken by a concurrent run; choosing another"
    "${compose[@]}" down --volumes --remove-orphans --timeout 10 >/dev/null 2>&1 || true
    choose_edge_network
    set_env "$work/compose.env" EKKLESIA_EDGE_SUBNET "$edge_subnet"
    set_env "$work/compose.env" EKKLESIA_EDGE_DYNAMIC_RANGE "$edge_range"
    set_env "$work/compose.env" EKKLESIA_PROXY_ADDRESS "$proxy_address"
    set_env "$work/compose.env" EKKLESIA_TUNNEL_ADDRESS "$tunnel_address"
    continue
  fi
  compose_report_not_ready "$up_output" "${compose[@]}"
  fail "PostgreSQL and object storage did not become healthy."
done

step "Planning, then explicitly applying, migrations against this run's fresh database"
"${compose[@]}" run --rm --no-deps migrate >/dev/null || fail "migrate (plan) failed."
"${compose[@]}" run --rm --no-deps migrate node core/dist/cli/apply-migrations.js >/dev/null ||
  fail "migrate (apply) failed."

step "Starting the API and the proxy"
if ! up_output="$("${compose[@]}" up --detach --wait --wait-timeout 180 2>&1)"; then
  compose_report_not_ready "$up_output" "${compose[@]}"
  fail "the API and proxy did not become healthy."
fi

# Caddy creates its internal CA on first start. The test clients trust it, plus the installation CA
# for their direct connection to object-storage.
for _ in $(seq 1 30); do
  if "${compose[@]}" exec -T proxy cat /data/caddy/pki/authorities/local/root.crt \
    >"$work/caddy-root.crt" 2>/dev/null && [[ -s "$work/caddy-root.crt" ]]; then
    break
  fi
  sleep 1
done
[[ -s "$work/caddy-root.crt" ]] || fail "the test proxy's internal CA did not appear."
cat "$work/caddy-root.crt" "$work/tls/ca.crt" >"$work/client-ca.pem"
cp scripts/lib/deploy-smoke-client.mjs "$work/client.mjs"
stage client-ca.pem
stage client.mjs

edge_network="${project}_edge"
host_mappings=(--add-host "${hostname}:${proxy_address}" --add-host "${storage_hostname}:${proxy_address}")
if [[ -n "$dashboard_hostname" ]]; then
  host_mappings+=(--add-host "${dashboard_hostname}:${proxy_address}")
fi
client() {
  local mode="$1"
  shift
  docker run --rm --read-only --cap-drop ALL --security-opt no-new-privileges \
    --label "${EKKLESIA_CI_LABEL}=${project}" \
    --network "$edge_network" "$@" "${host_mappings[@]}" \
    -e NODE_EXTRA_CA_CERTS=/smoke/client-ca.pem -e SMOKE_HOSTNAME="$hostname" \
    -e SMOKE_DASHBOARD_HOSTNAME="$dashboard_hostname" \
    -e SMOKE_STORAGE_HOSTNAME="$storage_hostname" \
    -e S3_ACCESS_KEY="$s3_access_key" -e S3_SECRET_KEY="$s3_secret_key" \
    -v "$daemon_work/client.mjs:/smoke/client.mjs:ro" \
    -v "$daemon_work/client-ca.pem:/smoke/client-ca.pem:ro" \
    --entrypoint node "$image" /smoke/client.mjs "$mode"
}

step "Checking the API, Dashboard, storage delivery, and client-address chain through the proxy"
client edge || fail "edge checks failed."

step "Checking the tunnel connector's client-address path from its fixed address"
client tunnel --ip "$tunnel_address" || fail "tunnel checks failed."

step "Creating the initial owner with the image's own CLI, then signing in and out through the proxy"
owner_email="[email protected]"
owner_password="$(openssl rand -hex 16)"
printf '%s' "$owner_password" |
  "${compose[@]}" run --rm --no-deps -T migrate node core/dist/cli/create-initial-owner.js \
    --email "$owner_email" >/dev/null || fail "creating the initial owner failed."
echo "ok   the initial owner was created from the image (password on standard input)"
client session --ip "$session_client_address" \
  -e SMOKE_OWNER_EMAIL="$owner_email" -e SMOKE_OWNER_PASSWORD="$owner_password" ||
  fail "session checks failed."

if [[ "$topology" == split ]]; then
  step "Driving a real browser through the split Dashboard, a module, its display, and sign-out"
  # Chromium trusts this run's proxy through the public-key hash of Caddy's internal intermediate CA,
  # which signs every certificate the test proxy serves: no CA is installed anywhere, and nothing but
  # this run's certificates is accepted.
  "${compose[@]}" exec -T proxy cat /data/caddy/pki/authorities/local/intermediate.crt \
    >"$work/caddy-intermediate.crt" || fail "the test proxy's intermediate CA is unavailable."
  spki="$(openssl x509 -in "$work/caddy-intermediate.crt" -pubkey -noout |
    openssl pkey -pubin -outform der | openssl dgst -sha256 -binary | openssl base64 -A)"
  browser_container="${project}-browser"
  # The image's entry script passes its arguments on unquoted, so none may contain a space: the
  # hostnames reach Chromium through /etc/hosts, which it reads, rather than a resolver rule.
  docker run --detach --rm --name "$browser_container" --label "${EKKLESIA_CI_LABEL}=${project}" \
    --cap-drop ALL --security-opt no-new-privileges --tmpfs /tmp \
    --network "$edge_network" --ip "$browser_address" "${host_mappings[@]}" \
    "$browser_image" \
    "--ignore-certificate-errors-spki-list=${spki}" \
    --user-data-dir=/tmp/profile >/dev/null || fail "the headless browser did not start."
  cp scripts/lib/deploy-smoke-browser.mjs "$work/browser.mjs"
  stage browser.mjs
  docker run --rm --read-only --cap-drop ALL --security-opt no-new-privileges \
    --label "${EKKLESIA_CI_LABEL}=${project}" \
    --network "$edge_network" \
    -e SMOKE_CDP_ADDRESS="${browser_address}:9222" \
    -e SMOKE_HOSTNAME="$hostname" -e SMOKE_DASHBOARD_HOSTNAME="$dashboard_hostname" \
    -e SMOKE_OWNER_EMAIL="$owner_email" -e SMOKE_OWNER_PASSWORD="$owner_password" \
    -v "$daemon_work/browser.mjs:/smoke/browser.mjs:ro" \
    --entrypoint node "$image" /smoke/browser.mjs || fail "browser checks failed."

  if [[ -n "${EKKLESIA_SMOKE_DISPLAY_DEVICE:-}" ]]; then
    # A kiosk: a second browser whose profile directory is a volume, so it outlives the container
    # exactly as a kiosk's --user-data-dir outlives a browser restart.
    kiosk_container="${project}-kiosk"
    kiosk_volume="${project}-kiosk-profile"
    kiosk_address="10.249.${octet}.250"
    docker volume create --label "${EKKLESIA_CI_LABEL}=${project}" "$kiosk_volume" >/dev/null
    start_kiosk() {
      docker run --detach --rm --name "$kiosk_container" --label "${EKKLESIA_CI_LABEL}=${project}" \
        --cap-drop ALL --security-opt no-new-privileges --tmpfs /tmp \
        --network "$edge_network" --ip "$kiosk_address" "${host_mappings[@]}" \
        -v "${kiosk_volume}:/profile" \
        "$browser_image" \
        "--ignore-certificate-errors-spki-list=${spki}" \
        --user-data-dir=/profile >/dev/null || fail "the kiosk browser did not start."
    }
    wait_kiosk_exit() {
      for _ in $(seq 1 60); do
        docker inspect "$kiosk_container" >/dev/null 2>&1 || return 0
        sleep 0.5
      done
      fail "the kiosk browser did not exit after a clean close."
    }
    cp scripts/lib/deploy-smoke-display.mjs "$work/display.mjs"
    stage display.mjs
    # display_driver <budget seconds> <step>: one scenario with a hard budget. The driver enforces it
    # itself and names whatever call was still waiting; `timeout` is the backstop if it cannot.
    display_driver() {
      local budget="$1" step_name="$2" driver="${project}-display-${2}"
      timeout --signal=TERM "$((budget + 30))" \
        docker run --rm --name "$driver" --read-only --cap-drop ALL --security-opt no-new-privileges \
        --network "$edge_network" \
        -e SMOKE_CDP_ADDRESS="${browser_address}:9222" \
        -e SMOKE_KIOSK_CDP_ADDRESS="${kiosk_address}:9222" \
        -e SMOKE_STEP_BUDGET_MS="$((budget * 1000))" \
        -e SMOKE_WATCH_MS="${EKKLESIA_SMOKE_WATCH_MS:-120000}" \
        -e SMOKE_RECYCLE_WATCH_MS="${EKKLESIA_SMOKE_RECYCLE_WATCH_MS:-330000}" \
        -e SMOKE_HOSTNAME="$hostname" -e SMOKE_DASHBOARD_HOSTNAME="$dashboard_hostname" \
        -e SMOKE_OWNER_EMAIL="$owner_email" -e SMOKE_OWNER_PASSWORD="$owner_password" \
        -v "$daemon_work/display.mjs:/smoke/display.mjs:ro" \
        --entrypoint node "$image" /smoke/display.mjs "$step_name"
      local status=$?
      if ((status == 124)); then
        echo "FAIL ${step_name}: the driver did not stop within its budget plus 30s; removed it"
        docker rm --force "$driver" >/dev/null 2>&1 || true
      fi
      return "$status"
    }
    display_status=0
    step "Pairing a kiosk display, then closing its browser cleanly"
    start_kiosk
    display_driver 180 pair || display_status=1
    wait_kiosk_exit
    step "Restarting the kiosk browser on the same profile"
    start_kiosk
    display_driver 120 restart || display_status=1
    for scenario in ${EKKLESIA_SMOKE_DISPLAY_SCENARIOS:-two-pages connectivity-blip session-blip stream-failure caching-probe stream-recycle revoke}; do
      case "$scenario" in
      two-pages) budget=240 ;;
      stream-recycle) budget=420 ;;
      revoke) budget=210 ;;
      *) budget=150 ;;
      esac
      step "Display scenario: ${scenario} (budget ${budget}s)"
      display_driver "$budget" "$scenario" || display_status=1
    done
    step "Proxy: streams aborted with an incomplete response (UTC time, path only)"
    # Captured, never printed: only each line's time and path (no query, so no signature) leave it.
    proxy_logs="$("${compose[@]}" logs --no-color proxy 2>/dev/null)" || proxy_logs=""
    grep -o '{.*"aborting with incomplete response".*}' <<<"$proxy_logs" |
      sed -E 's/.*"ts":([0-9]+)[^,]*,.*"uri":"([^"?]*)[^"]*".*/\1 \2/' |
      while read -r ts uri; do echo "     $(date -u -d "@${ts}" +%T) ${uri}"; done || true
    ((display_status == 0)) || fail "display-device checks failed."
  fi
fi

step "Proving the API's database connection is certificate-verified TLS"
api_container="$("${compose[@]}" ps -q api)"
api_data_address="$(
  docker inspect --format "{{(index .NetworkSettings.Networks \"${project}_data\").IPAddress}}" \
    "$api_container"
)"
[[ -n "$api_data_address" ]] || fail "could not find the API's database-network address."
psql_as_superuser() {
  "${compose[@]}" exec -T postgres psql -U ekklesia -d ekklesia -Atc "$1"
}
tls_connections="$(psql_as_superuser "SELECT count(*) FROM pg_stat_ssl s JOIN pg_stat_activity a USING (pid) WHERE a.client_addr = '${api_data_address}' AND s.ssl AND s.version = 'TLSv1.3'")"
plain_connections="$(psql_as_superuser "SELECT count(*) FROM pg_stat_ssl s JOIN pg_stat_activity a USING (pid) WHERE a.client_addr = '${api_data_address}' AND NOT s.ssl")"
((tls_connections > 0)) || fail "the API holds no TLS 1.3 database connection."
((plain_connections == 0)) || fail "the API holds a database connection without TLS."
echo "ok   the API's database connections are TLS 1.3 (${tls_connections}), none plaintext"

# PostgreSQL itself refuses a network connection without TLS.
if "${compose[@]}" exec -T -e PGPASSWORD="$postgres_password" postgres \
  psql "host=postgres user=ekklesia dbname=ekklesia sslmode=disable" -Atc 'SELECT 1' >/dev/null 2>&1; then
  fail "PostgreSQL accepted a plaintext network connection."
fi
echo "ok   PostgreSQL refuses a plaintext network connection"

# The server cannot see whether a client verified it, so verification is proven by refusal: the
# same command fails against a CA that did not issue the certificate, and against a host the
# certificate does not name.
openssl ecparam -name prime256v1 -genkey -noout -out "$work/other-ca.key" 2>/dev/null
openssl req -x509 -new -sha256 -days 1 -key "$work/other-ca.key" -out "$work/other-ca.crt" \
  -subj "/CN=Unrelated CA" 2>/dev/null
# expect_connection_refused <description> <compose run arguments...>: the command must fail, and fail
# at the connection, not on a configuration error that would prove nothing about verification.
expect_connection_refused() {
  local description="$1"
  shift
  local output
  if output="$("${compose[@]}" run --rm --no-deps "$@" 2>&1)"; then
    fail "the database connection accepted ${description}."
  fi
  grep -q 'MIGRATION_CONNECTION_FAILED' <<<"$output" ||
    fail "rejecting ${description} failed for a reason other than the connection."
}
expect_connection_refused "a certificate from a CA it was not given" \
  -e EKKLESIA_DATABASE_CA="$(cat "$work/other-ca.crt")" migrate
echo "ok   the database connection rejects a certificate its CA did not issue"
postgres_address="$(
  docker inspect --format "{{(index .NetworkSettings.Networks \"${project}_data\").IPAddress}}" \
    "$("${compose[@]}" ps -q postgres)"
)"
expect_connection_refused "a certificate for a different host name" \
  -e DATABASE_URL="postgresql://ekklesia:${postgres_password}@${postgres_address}:5432/ekklesia" migrate
echo "ok   the database connection rejects a certificate that does not name the host"
"${compose[@]}" run --rm --no-deps migrate >/dev/null || fail "the control connection failed."
echo "ok   the same connection succeeds with the installation CA and the name postgres"

step "Running Doctor from the image"
doctor_output="$("${compose[@]}" run --rm --no-deps migrate node core/dist/cli/doctor.js 2>&1)" ||
  fail "Doctor reported the installation not ready."
grep -q 'Result: READY' <<<"$doctor_output" || fail "Doctor did not report READY."
grep -q 'TLS required with certificate and hostname verification (explicit private CA supplied)' \
  <<<"$doctor_output" || fail "Doctor did not report verify-full with the private CA."
grep -q '✓ Object storage reachable' <<<"$doctor_output" ||
  fail "Doctor could not reach object storage over HTTPS."
# Doctor runs as `migrate`; it must check the client-address configuration the API runs with, which
# behind the proxy is trusted-proxy mode, not the direct mode an unconfigured process reports.
grep -q 'Trusted-proxy client-address mode' <<<"$doctor_output" ||
  fail "Doctor did not check the API's trusted-proxy client-address configuration."
for value in "$postgres_password" "$s3_secret_key" "$identity_secret"; do
  if grep -qF -- "$value" <<<"$doctor_output"; then
    fail "Doctor output contains a secret."
  fi
done
echo "ok   Doctor: READY, verify-full with the private CA, storage reachable, the API's client-address mode, no secret printed"

api_logs="$("${compose[@]}" logs --no-color api 2>&1)"
for value in "$postgres_password" "$s3_secret_key" "$identity_secret" "$owner_password"; do
  if grep -qF -- "$value" <<<"$api_logs"; then
    fail "the API's logs contain a secret."
  fi
done
restarts="$(docker inspect --format '{{.RestartCount}}' "$api_container")"
((restarts == 0)) || fail "the API restarted ${restarts} time(s)."
echo "ok   the API logged no secret and never restarted"

echo "Reference deployment smoke test passed."