Ekklesia/scripts/smoke-deploy.sh

#!/usr/bin/env bash
# End-to-end smoke test of the reference deployment (ADR 0044, ADR 0048): deploy/compose.yaml with
# deploy/compose.proxy.yaml, the one generic platform image, installation-owned module storage, and
# a disposable PostgreSQL and SeaweedFS that exist only for this run. CI runs it; it is safe to run
# on a workstation.
#
#   bash scripts/smoke-deploy.sh                 builds the image first
#   EKKLESIA_SMOKE_IMAGE=<image> bash scripts/smoke-deploy.sh
#   EKKLESIA_SMOKE_TOPOLOGY=split bash scripts/smoke-deploy.sh
#   EKKLESIA_SMOKE_MODULE_RELEASE_DIR=<dir> EKKLESIA_SMOKE_MODULE_ID=<id> \
#     EKKLESIA_SMOKE_TOPOLOGY=split bash scripts/smoke-deploy.sh
#
# Either topology (ADR 0045), and in both the same generic image, built once from deploy/Dockerfile.
# It contains no module; module storage starts empty, as on a fresh installation. `same-origin` (the
# default) serves the Dashboard, the API, and displays from one hostname with Caddyfile.example.
# `split` serves the Dashboard on its own hostname with Caddyfile.split.example and then exercises
# ADR 0048 end to end with that same image:
#
#   - it boots with no modules;
#   - either the repository's module template is packaged by the canonical packager, or an external
#     canonical release pair is supplied through EKKLESIA_SMOKE_MODULE_RELEASE_DIR; the package is
#     verified by the same image's own verifier, extracted to staging, installed into module storage,
#     its migrations planned and explicitly applied, and the API restarted;
#   - a real headless Chromium drives the Dashboard, the module, its display, and sign-out;
#   - the module is removed and the API restarted: Core still runs, migration planning reports the
#     module's retained history, and its data survives;
#   - the exact same package is verified and installed again, and the API restarted: the module
#     returns with its previous data.
#
# EKKLESIA_SMOKE_IMAGE uses the given image instead of building one. It must be a generic image
# (ADR 0048): modules come from module storage, never from the image.
#
# Opt-in, split topology only: EKKLESIA_SMOKE_DISPLAY_DEVICE=1 adds the display-device scenarios in
# scripts/lib/deploy-smoke-display.mjs (pairing a kiosk browser with a persistent profile, a clean
# restart, two pages, injected network failures, revocation), each with its own time budget.
# EKKLESIA_SMOKE_DISPLAY_SCENARIOS selects a subset; EKKLESIA_SMOKE_WATCH_MS and
# EKKLESIA_SMOKE_RECYCLE_WATCH_MS change how long they watch. Several scenarios expect Signage.
#
# TEST-ONLY: the proxy runs with Caddy's internal CA (`local_certs`) instead of ACME, because a test
# run has no public hostname. Never use that option in an installation.
#
# Follows the documented first boot: TLS material, secrets, data services, an explicit migration
# apply against this run's own fresh database, then the API and proxy. Everything it starts, and
# every file it writes, is removed on exit.
#
# Bind mounts are resolved by the Docker daemon, on its host. When this script runs inside a CI job
# container that drives the host's Docker, its own files are not at those paths, so every file a
# container mounts is staged under an identical path in the Docker host's /tmp first.

set -Eeuo pipefail

repository_root="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")/.." && pwd)"
cd "$repository_root"

# shellcheck source=scripts/lib/compose-diagnostics.sh
source "$repository_root/scripts/lib/compose-diagnostics.sh"
# shellcheck source=scripts/lib/ci-project.sh
source "$repository_root/scripts/lib/ci-project.sh"
# shellcheck source=scripts/lib/module-package-release.sh
source "$repository_root/scripts/lib/module-package-release.sh"

for command in docker openssl awk; do
  command -v "$command" >/dev/null || {
    echo "smoke-deploy: $command is required." >&2
    exit 1
  }
done
docker compose version >/dev/null 2>&1 || {
  echo "smoke-deploy: Docker Compose is required." >&2
  exit 1
}

# The pinned PostgreSQL image doubles as a root shell for file ownership, so the run needs neither
# root nor sudo on the host.
postgres_image="postgres:18.4-alpine3.24@sha256:9a8afca54e7861fd90fab5fdf4c42477a6b1cb7d293595148e674e0a3181de15"
# The split topology's real browser: headless Chromium, pinned by its multi-architecture index digest.
browser_image="chromedp/headless-shell:151.0.7922.109@sha256:2d349b544a1ea6b5b5fd7c0fe99215ff662339c57407ee2e8c0a11af93516b04"

topology="${EKKLESIA_SMOKE_TOPOLOGY:-same-origin}"
case "$topology" in
same-origin | split) ;;
*)
  echo "smoke-deploy: EKKLESIA_SMOKE_TOPOLOGY must be same-origin or split." >&2
  exit 1
  ;;
esac

module_release_directory="${EKKLESIA_SMOKE_MODULE_RELEASE_DIR:-}"
module_id="${EKKLESIA_SMOKE_MODULE_ID:-example}"
module_probe_path="${EKKLESIA_SMOKE_MODULE_PROBE_PATH:-}"
[[ "$module_id" =~ ^[a-z][a-z0-9]*(-[a-z0-9]+)*$ ]] || {
  echo "smoke-deploy: EKKLESIA_SMOKE_MODULE_ID is not a valid module id." >&2
  exit 1
}
if [[ -n "$module_release_directory" ]]; then
  [[ "$topology" == split ]] || {
    echo "smoke-deploy: an external module release requires the split topology." >&2
    exit 1
  }
  [[ -d "$module_release_directory" ]] || {
    echo "smoke-deploy: EKKLESIA_SMOKE_MODULE_RELEASE_DIR is not a directory." >&2
    exit 1
  }
fi
if [[ -n "$module_probe_path" && "$module_probe_path" != "/api/v1/modules/${module_id}/"* ]]; then
  echo "smoke-deploy: EKKLESIA_SMOKE_MODULE_PROBE_PATH must be inside the selected module's API namespace." >&2
  exit 1
fi

run_id="$(openssl rand -hex 4)"
# Unique per invocation, and findable by run and job if this script is killed before its trap runs
# (scripts/lib/ci-project.sh, scripts/ci-sweep.sh). Everything started outside Compose below carries
# the same name in EKKLESIA_CI_LABEL.
project="$(ci_project_name "deploy-${topology}")"
work="$(mktemp -d)"
image="${EKKLESIA_SMOKE_IMAGE:-}"
built_image=""
# `hostname` is always the API's (and the display host's); the Dashboard has its own only when split.
if [[ "$topology" == split ]]; then
  hostname="api.smoke.test"
  dashboard_hostname="dash.smoke.test"
  caddyfile_example="deploy/Caddyfile.split.example"
else
  hostname="ekklesia.smoke.test"
  dashboard_hostname=""
  caddyfile_example="deploy/Caddyfile.example"
fi
dashboard_origin="https://${dashboard_hostname:-$hostname}"
storage_hostname="storage.smoke.test"
browser_container=""

# choose_edge_network: a per-run edge subnet that no network on this Docker daemon already uses, so
# concurrent runs sharing a runner's daemon do not collide. Two runs can still pick the same free
# subnet at the same moment; the first `up` below detects that and chooses again. Fixed addresses come
# from the upper half, outside Docker's dynamic range, exactly as documented.
choose_edge_network() {
  local in_use candidate attempt
  in_use="$(
    docker network ls --quiet |
      xargs -r docker network inspect --format '{{range .IPAM.Config}}{{println .Subnet}}{{end}}' 2>/dev/null |
      sed -n 's#^10\.249\.\([0-9]*\)\.0/24$#\1#p'
  )" || in_use=""
  for attempt in $(seq 1 100); do
    candidate=$((RANDOM % 200 + 20))
    if ! grep -qx "$candidate" <<<"$in_use"; then
      octet="$candidate"
      edge_subnet="10.249.${octet}.0/24"
      edge_range="10.249.${octet}.0/25"
      proxy_address="10.249.${octet}.254"
      tunnel_address="10.249.${octet}.253"
      # A test-only client address, also outside the dynamic range, so its limiter bucket is its own.
      session_client_address="10.249.${octet}.252"
      # The headless browser's fixed address, likewise (split topology only).
      browser_address="10.249.${octet}.251"
      # The installed-module checks' own client address, so their sign-ins have their own bucket.
      module_client_address="10.249.${octet}.249"
      return 0
    fi
  done
  echo "smoke-deploy: no free 10.249.x.0/24 edge subnet on this Docker daemon." >&2
  exit 1
}
choose_edge_network

# daemon_work: where the Docker daemon sees this run's mounted files. The same directory as `work`
# on a workstation; a staged copy on the Docker host when this script runs in a job container.
if [[ -f /.dockerenv ]]; then
  daemon_work="/tmp/${project}"
  docker run --rm --entrypoint mkdir -v /tmp:/host-tmp "$postgres_image" -p "/host-tmp/${project}"
  # stage <path relative to work>: copy it to the same relative path under daemon_work, keeping
  # numeric ownership and modes.
  stage() {
    tar -C "$work" -cf - "$1" |
      docker run --rm -i --entrypoint tar -v "$daemon_work:/w" "$postgres_image" -xpf - -C /w
  }
else
  daemon_work="$work"
  stage() { :; }
fi

compose=(
  docker compose
  --project-name "$project"
  --project-directory "$daemon_work/deploy"
  --env-file "$work/compose.env"
  --file deploy/compose.yaml
  --file deploy/compose.proxy.yaml
)

cleanup() {
  local status=$?
  trap - EXIT INT TERM
  if [[ -n "$browser_container" ]]; then
    docker rm --force "$browser_container" >/dev/null 2>&1 || true
  fi
  if [[ -n "${kiosk_container:-}" ]]; then
    docker ps --all --quiet --filter "name=${project}-display-" | xargs -r docker rm --force \
      >/dev/null 2>&1 || true
    docker rm --force "$kiosk_container" >/dev/null 2>&1 || true
    docker volume rm --force "$kiosk_volume" >/dev/null 2>&1 || true
  fi
  if [[ -f "$work/compose.env" ]]; then
    "${compose[@]}" --profile ops down --volumes --remove-orphans --timeout 10 >/dev/null 2>&1 || status=1
  fi
  # Some files are owned by the PostgreSQL and SeaweedFS uids by now; remove them as root, in the
  # same image that set that ownership. In a job container the daemon cannot see `work` (and the
  # job itself runs as root), so only the staged copy on the Docker host needs this.
  if [[ "$daemon_work" == "$work" ]]; then
    docker run --rm --entrypoint rm -v "$work:/work" "$postgres_image" -rf /work/tls /work/env \
      /work/modules /work/staging >/dev/null 2>&1 || true
  else
    docker run --rm --entrypoint rm -v /tmp:/host-tmp "$postgres_image" -rf "/host-tmp/${project}" \
      >/dev/null 2>&1 || true
  fi
  rm -rf "$work" || true
  if [[ -n "$built_image" ]]; then
    docker image rm --force "$built_image" >/dev/null 2>&1 || true
  fi
  exit "$status"
}
trap cleanup EXIT INT TERM

step() {
  echo "==> $*"
}

fail() {
  echo "smoke-deploy: $*" >&2
  exit 1
}

if [[ -z "$image" ]]; then
  # The one generic platform image, for both topologies. Nothing about either topology, and no
  # module, is built into it.
  step "Building the generic platform image"
  built_image="ekklesia-deploy-smoke:${run_id}"
  DOCKER_BUILDKIT=1 docker build --quiet --file deploy/Dockerfile --tag "$built_image" \
    --label "${EKKLESIA_CI_LABEL}=${project}" --label "${EKKLESIA_CI_CREATED_LABEL}=$(date +%s)" \
    . >/dev/null
  image="$built_image"
fi

if [[ "$topology" == split ]]; then
  # Never a second platform image: the package is installed into module storage below, after the
  # generic image verifies it. The default keeps Core's own template smoke self-contained; downstream
  # module CI hands this harness the exact release pair it built instead.
  if [[ -z "$module_release_directory" ]]; then
    step "Packaging the module template with the canonical Ekklesia packager"
    source_revision="${GITHUB_SHA:-$(git rev-parse HEAD 2>/dev/null || true)}"
    [[ "$source_revision" =~ ^[0-9a-f]{40}$ ]] || fail "no source revision to record in the package."
    DOCKER_BUILDKIT=1 docker build --quiet --file scripts/lib/build-module-package.Dockerfile \
      --label "${EKKLESIA_CI_LABEL}=${project}" --build-arg SOURCE_REVISION="$source_revision" \
      --output "type=local,dest=${work}/module-package" . >/dev/null ||
      fail "the module package did not build."
    package_output="$work/module-package"
  else
    step "Accepting the external module's canonical release pair"
    package_output="$module_release_directory"
  fi
  # The archive and its digest are one release: copied together, from this run's own build output,
  # into a fresh directory the verifier's unprivileged user can read. BuildKit's output directory is
  # mode 0700 and owned by whoever built it, so it is never handed to the verifier itself.
  package_archive="$(module_package_release "$package_output" "$work/package")" ||
    fail "the packager did not produce exactly one archive and its digest."
  if [[ -z "$module_release_directory" ]]; then
    [[ "$package_archive" =~ ^example-[0-9]+\.[0-9]+\.[0-9]+-any\.tar\.gz$ ]] ||
      fail "the packager produced an unexpected archive name."
  fi
  echo "ok   packaged ${package_archive} ($(wc -c <"$work/package/$package_archive") bytes, sha256 $(cut -d' ' -f1 "$work/package/${package_archive}.sha256"))"
fi

# The compose project directory: its relative bind mount (postgres/pg_hba.conf) resolves here.
cp -R deploy "$work/deploy"
stage deploy

# Installation-owned module storage (ADR 0048), empty, as an operator creates it before the first
# `up`: Compose refuses to start `api` or `migrate` without it, and never creates it.
mkdir -m 755 "$work/modules"
stage modules
if [[ "$topology" == split ]]; then
  stage package
  # TEST-ONLY staging the verifier extracts into, as the image's unprivileged user. Nothing here is
  # module storage: the install step below moves a verified tree into module storage as root.
  mkdir -m 777 "$work/staging"
  stage staging
fi

step "Creating the installation-private CA and server certificates"
init_status=0
sh deploy/scripts/init-tls.sh "$work/tls" >/dev/null 2>&1 || init_status=$?
if ((init_status != 0 && init_status != 77)); then
  fail "init-tls.sh failed to create TLS material."
fi
stage tls
# The root-only ownership pass, exactly as an operator would re-run the script with root.
ownership_output="$(
  docker run --rm --entrypoint sh -v "$daemon_work/tls:/tls" \
    -v "$daemon_work/deploy/scripts:/scripts:ro" "$postgres_image" /scripts/init-tls.sh /tls
)" || fail "init-tls.sh could not apply ownership."
database_ca_line="$(grep '^EKKLESIA_DATABASE_CA=' <<<"$ownership_output")" ||
  fail "init-tls.sh did not print the EKKLESIA_DATABASE_CA line."
# A second run must create nothing and still succeed.
rerun_output="$(
  docker run --rm --entrypoint sh -v "$daemon_work/tls:/tls" \
    -v "$daemon_work/deploy/scripts:/scripts:ro" "$postgres_image" /scripts/init-tls.sh /tls
)" || fail "init-tls.sh failed on a second run."
grep -q 'nothing to create' <<<"$rerun_output" || fail "init-tls.sh is not idempotent."

step "Writing env files from the examples, with freshly generated secrets"
secrets="$(sh deploy/scripts/gen-secrets.sh)"
secret() {
  awk -F= -v key="$1" '$1 == key { sub(/^[^=]*=/, ""); print; exit }' <<<"$secrets"
}
postgres_password="$(secret POSTGRES_PASSWORD)"
s3_access_key="$(secret S3_ACCESS_KEY)"
s3_secret_key="$(secret S3_SECRET_KEY)"
identity_secret="$(secret EKKLESIA_IDENTITY_SECRET)"
# Masked in any diagnostics below, along with anything else that looks like a credential.
COMPOSE_DIAGNOSTICS_SECRETS=("$postgres_password" "$s3_access_key" "$s3_secret_key" "$identity_secret")

mkdir -p "$work/env"
# set_env <file> <key> <value>: replaces an uncommented KEY= line, keeping the value byte-exact.
set_env() {
  KEY="$2" VALUE="$3" awk '
    BEGIN { found = 0 }
    index($0, ENVIRON["KEY"] "=") == 1 { print ENVIRON["KEY"] "=" ENVIRON["VALUE"]; found = 1; next }
    { print }
    END { if (!found) print ENVIRON["KEY"] "=" ENVIRON["VALUE"] }
  ' "$1" >"$1.next"
  mv "$1.next" "$1"
}
for name in ekklesia postgres object-storage; do
  cp "deploy/env/${name}.env.example" "$work/env/${name}.env"
done
ekklesia_env="$work/env/ekklesia.env"
set_env "$ekklesia_env" DATABASE_URL "postgresql://ekklesia:${postgres_password}@postgres:5432/ekklesia"
set_env "$ekklesia_env" EKKLESIA_DATABASE_CA "${database_ca_line#EKKLESIA_DATABASE_CA=}"
set_env "$ekklesia_env" EKKLESIA_IDENTITY_BASE_URL "https://${hostname}"
set_env "$ekklesia_env" EKKLESIA_IDENTITY_TRUSTED_ORIGINS "$dashboard_origin"
if [[ "$topology" == split ]]; then
  set_env "$ekklesia_env" EKKLESIA_DASHBOARD_ORIGIN "$dashboard_origin"
fi
set_env "$ekklesia_env" EKKLESIA_IDENTITY_SECRET "$identity_secret"
set_env "$ekklesia_env" S3_DELIVERY_ENDPOINT "https://${storage_hostname}"
set_env "$ekklesia_env" S3_ACCESS_KEY "$s3_access_key"
set_env "$ekklesia_env" S3_SECRET_KEY "$s3_secret_key"
set_env "$work/env/postgres.env" POSTGRES_PASSWORD "$postgres_password"
set_env "$work/env/object-storage.env" AWS_ACCESS_KEY_ID "$s3_access_key"
set_env "$work/env/object-storage.env" AWS_SECRET_ACCESS_KEY "$s3_secret_key"
chmod 600 "$work"/env/*.env

# TEST-ONLY: Caddy's internal CA instead of ACME. Inserted as the first global option.
awk '!done && $0 == "{" { print; print "\tlocal_certs"; done = 1; next } { print }' \
  "$caddyfile_example" >"$work/Caddyfile"
grep -q $'^\tlocal_certs$' "$work/Caddyfile" || fail "could not enable local_certs for the test proxy."
stage Caddyfile

cat >"$work/compose.env" <<EOF
EKKLESIA_IMAGE=${image}
EKKLESIA_ENV_DIR=${work}/env
EKKLESIA_TLS_DIR=${daemon_work}/tls
EKKLESIA_HOSTNAME=${hostname}
EKKLESIA_DASHBOARD_HOSTNAME=${dashboard_hostname}
EKKLESIA_STORAGE_HOSTNAME=${storage_hostname}
[email protected]
EKKLESIA_CADDYFILE=${daemon_work}/Caddyfile
EKKLESIA_PUBLISH_ADDRESS=127.0.0.1
EKKLESIA_HTTP_PORT=0
EKKLESIA_HTTPS_PORT=0
EKKLESIA_EDGE_SUBNET=${edge_subnet}
EKKLESIA_EDGE_DYNAMIC_RANGE=${edge_range}
EKKLESIA_PROXY_ADDRESS=${proxy_address}
EKKLESIA_TUNNEL_ADDRESS=${tunnel_address}
EKKLESIA_MODULES_HOST_DIR=${daemon_work}/modules
EOF

step "Starting PostgreSQL and object storage"
# This `up` creates the edge network. If another run claimed the same subnet since it was chosen,
# Docker refuses with an address-pool overlap: choose another and start again, a bounded number of
# times. Any other failure is reported with the project's containers and logs.
for attempt in 1 2 3 4 5; do
  if up_output="$(
    "${compose[@]}" up --detach --wait --wait-timeout 180 postgres object-storage 2>&1
  )"; then
    break
  fi
  if [[ "$up_output" == *"overlap"* ]] && ((attempt < 5)); then
    echo "     edge subnet ${edge_subnet} was taken by a concurrent run; choosing another"
    "${compose[@]}" down --volumes --remove-orphans --timeout 10 >/dev/null 2>&1 || true
    choose_edge_network
    set_env "$work/compose.env" EKKLESIA_EDGE_SUBNET "$edge_subnet"
    set_env "$work/compose.env" EKKLESIA_EDGE_DYNAMIC_RANGE "$edge_range"
    set_env "$work/compose.env" EKKLESIA_PROXY_ADDRESS "$proxy_address"
    set_env "$work/compose.env" EKKLESIA_TUNNEL_ADDRESS "$tunnel_address"
    continue
  fi
  compose_report_not_ready "$up_output" "${compose[@]}"
  fail "PostgreSQL and object storage did not become healthy."
done

step "Planning, then explicitly applying, migrations against this run's fresh database"
"${compose[@]}" run --rm --no-deps migrate >/dev/null || fail "migrate (plan) failed."
"${compose[@]}" run --rm --no-deps migrate node core/dist/cli/apply-migrations.js >/dev/null ||
  fail "migrate (apply) failed."

step "Starting the API and the proxy"
if ! up_output="$("${compose[@]}" up --detach --wait --wait-timeout 180 2>&1)"; then
  compose_report_not_ready "$up_output" "${compose[@]}"
  fail "the API and proxy did not become healthy."
fi

# Caddy creates its internal CA on first start. The test clients trust it, plus the installation CA
# for their direct connection to object-storage.
for _ in $(seq 1 30); do
  if "${compose[@]}" exec -T proxy cat /data/caddy/pki/authorities/local/root.crt \
    >"$work/caddy-root.crt" 2>/dev/null && [[ -s "$work/caddy-root.crt" ]]; then
    break
  fi
  sleep 1
done
[[ -s "$work/caddy-root.crt" ]] || fail "the test proxy's internal CA did not appear."
cat "$work/caddy-root.crt" "$work/tls/ca.crt" >"$work/client-ca.pem"
cp scripts/lib/deploy-smoke-client.mjs "$work/client.mjs"
stage client-ca.pem
stage client.mjs

edge_network="${project}_edge"
host_mappings=(--add-host "${hostname}:${proxy_address}" --add-host "${storage_hostname}:${proxy_address}")
if [[ -n "$dashboard_hostname" ]]; then
  host_mappings+=(--add-host "${dashboard_hostname}:${proxy_address}")
fi
client() {
  local mode="$1"
  shift
  docker run --rm --read-only --cap-drop ALL --security-opt no-new-privileges \
    --label "${EKKLESIA_CI_LABEL}=${project}" \
    --network "$edge_network" "$@" "${host_mappings[@]}" \
    -e NODE_EXTRA_CA_CERTS=/smoke/client-ca.pem -e SMOKE_HOSTNAME="$hostname" \
    -e SMOKE_DASHBOARD_HOSTNAME="$dashboard_hostname" \
    -e SMOKE_STORAGE_HOSTNAME="$storage_hostname" \
    -e S3_ACCESS_KEY="$s3_access_key" -e S3_SECRET_KEY="$s3_secret_key" \
    -v "$daemon_work/client.mjs:/smoke/client.mjs:ro" \
    -v "$daemon_work/client-ca.pem:/smoke/client-ca.pem:ro" \
    --entrypoint node "$image" /smoke/client.mjs "$mode"
}

step "Checking the API, Dashboard, storage delivery, and client-address chain through the proxy"
client edge || fail "edge checks failed."

step "Checking the tunnel connector's client-address path from its fixed address"
client tunnel --ip "$tunnel_address" || fail "tunnel checks failed."

step "Creating the initial owner with the image's own CLI, then signing in and out through the proxy"
owner_email="[email protected]"
owner_password="$(openssl rand -hex 16)"
printf '%s' "$owner_password" |
  "${compose[@]}" run --rm --no-deps -T migrate node core/dist/cli/create-initial-owner.js \
    --email "$owner_email" >/dev/null || fail "creating the initial owner failed."
echo "ok   the initial owner was created from the image (password on standard input)"
client session --ip "$session_client_address" \
  -e SMOKE_OWNER_EMAIL="$owner_email" -e SMOKE_OWNER_PASSWORD="$owner_password" ||
  fail "session checks failed."

psql_as_superuser() {
  "${compose[@]}" exec -T postgres psql -U ekklesia -d ekklesia -Atc "$1"
}

# module_storage <script>: runs a root shell over this run's module storage, as an operator changing
# installed modules on the host would. The API never writes it; it is mounted read-only.
module_storage() {
  docker run --rm -i --network none --entrypoint sh -v "$daemon_work/modules:/modules" \
    "$postgres_image" -ec "$1"
}

# install_test_module: verifies the package with the generic image's own verifier, which writes the
# verified module directory to staging, then moves it into module storage under a hidden name, which
# discovery ignores, and renames it into place in one step, owned by root and readable by the API's
# unprivileged user. Nothing unverified ever reaches module storage.
install_test_module() {
  local verified
  verified="$(
    docker run --rm --network none --read-only \
      --tmpfs /tmp:rw,nosuid,nodev,noexec,size=64m \
      --tmpfs /run/ekklesia-verifier:rw,nosuid,nodev,exec,size=1100m,mode=0700,uid=1000,gid=1000 \
      --cap-drop ALL --security-opt no-new-privileges \
      -e EKKLESIA_MODULE_VERIFIER_SCRATCH_DIR=/run/ekklesia-verifier \
      -v "$daemon_work/package:/package:ro" -v "$daemon_work/staging:/staging" \
      --entrypoint node "$image" core/dist/cli/verify-module-package.js "/package/${package_archive}" \
      --sha256-file "/package/${package_archive}.sha256" --extract-to "/staging/${module_id}" 2>&1
  )" || fail "the module package did not verify: ${verified}"
  echo "ok   ${verified%%$'\n'*}"
  docker run --rm --network none --entrypoint sh -v "$daemon_work/modules:/modules" \
    -v "$daemon_work/staging:/staging" -e MODULE_ID="$module_id" "$postgres_image" -ec '
      incoming="/modules/.incoming-${MODULE_ID}"
      installed="/modules/${MODULE_ID}"
      staged="/staging/${MODULE_ID}"
      rm -rf "$incoming" && cp -R "$staged" "$incoming" && rm -rf "$staged" &&
      chown -R 0:0 "$incoming" && chmod -R a+rX,go-w "$incoming" && mv "$incoming" "$installed"' ||
    fail "the verified module could not be installed into module storage."
}

# The package and its claimed digest are deliberately made to disagree. The verifier must reject it
# before extraction; the valid release pair is left untouched for the installation that follows.
reject_invalid_package() {
  local rejected zero_digest
  zero_digest="$(printf '0%.0s' {1..64})"
  if rejected="$(
    docker run --rm --network none --read-only \
      --tmpfs /tmp:rw,nosuid,nodev,noexec,size=64m \
      --tmpfs /run/ekklesia-verifier:rw,nosuid,nodev,exec,size=1100m,mode=0700,uid=1000,gid=1000 \
      --cap-drop ALL --security-opt no-new-privileges \
      -e EKKLESIA_MODULE_VERIFIER_SCRATCH_DIR=/run/ekklesia-verifier \
      -v "$daemon_work/package:/package:ro" \
      --entrypoint node "$image" core/dist/cli/verify-module-package.js "/package/${package_archive}" \
      --sha256 "$zero_digest" 2>&1
  )"; then
    fail "the verifier accepted a package whose archive did not match its claimed digest."
  fi
  grep -q '^\[MODULE_PACKAGE_DIGEST_MISMATCH\]' <<<"$rejected" ||
    fail "the invalid package was rejected for an unexpected reason."
  echo "ok   an invalid package is rejected before installation"
}

# restart_api <command...>: stops the API, runs the command, and starts the API again. A controlled
# restart is the only way a module change is picked up; nothing is ever hot loaded.
restart_api() {
  "${compose[@]}" stop api >/dev/null 2>&1 || fail "the API did not stop."
  "$@"
  if ! up_output="$("${compose[@]}" up --detach --wait --wait-timeout 180 api 2>&1)"; then
    compose_report_not_ready "$up_output" "${compose[@]}"
    fail "the API did not become healthy after a restart."
  fi
}

# recreate_api removes the application container, not its installation-owned module bind mount or
# database volumes, then creates a fresh container from the same generic image.
recreate_api() {
  "${compose[@]}" stop api >/dev/null 2>&1 || fail "the API did not stop for recreation."
  "${compose[@]}" rm --force api >/dev/null 2>&1 || fail "the API container was not removed."
  if ! up_output="$("${compose[@]}" up --detach --wait --wait-timeout 180 api 2>&1)"; then
    compose_report_not_ready "$up_output" "${compose[@]}"
    fail "the recreated API did not become healthy."
  fi
}

# module_check <expectation>: "absent", or the exact JSON array of values `example` must return.
module_check() {
  client module --ip "$module_client_address" -e SMOKE_MODULE_EXPECT="$1" \
    -e SMOKE_MODULE_ID="$module_id" -e SMOKE_MODULE_PROBE_PATH="$module_probe_path" \
    -e SMOKE_OWNER_EMAIL="$owner_email" -e SMOKE_OWNER_PASSWORD="$owner_password"
}

# plan_output: the read-only migration plan, as `docker compose run --rm migrate` prints it.
plan_output() {
  "${compose[@]}" run --rm --no-deps migrate 2>&1 || fail "migration planning failed."
}

if [[ "$topology" == split ]]; then
  step "Booting the generic image with empty module storage"
  module_check absent || fail "the image without modules did not run module-free."

  step "Rejecting an invalid package before installation"
  reject_invalid_package

  step "Installing the test module into module storage, migrating explicitly, and restarting the API"
  install_module_and_migrate() {
    install_test_module
    local plan
    plan="$(plan_output)"
    grep -q "^- \\[pending\\] ${module_id}:" <<<"$plan" ||
      fail "the plan does not list the installed module's migration as pending."
    "${compose[@]}" run --rm --no-deps migrate node core/dist/cli/apply-migrations.js >/dev/null ||
      fail "applying the installed module's migration failed."
    plan="$(plan_output)"
    grep -q "^- \\[applied\\] ${module_id}:" <<<"$plan" ||
      fail "the plan does not list the installed module's migration as applied."
    if grep -q "^- \\[pending\\] ${module_id}:" <<<"$plan"; then
      fail "the installed module still has a pending migration after apply."
    fi
    "${compose[@]}" run --rm --no-deps migrate node core/dist/cli/apply-migrations.js >/dev/null ||
      fail "reapplying migrations was not idempotent."
  }
  restart_api install_module_and_migrate
  if [[ "$module_id" == example ]]; then
    module_check '[]' || fail "the installed module did not run from module storage."
    psql_as_superuser "INSERT INTO example_records (id, value) VALUES (gen_random_uuid(), 'kept')" \
      >/dev/null || fail "could not write the module's own data."
    module_check '["Hello, kept"]' || fail "the installed module did not read its own data."
  else
    module_check present || fail "the installed module did not run from module storage."
  fi

  step "Driving a real browser through the split Dashboard, a module, its display, and sign-out"
  # Chromium trusts this run's proxy through the public-key hash of Caddy's internal intermediate CA,
  # which signs every certificate the test proxy serves: no CA is installed anywhere, and nothing but
  # this run's certificates is accepted.
  "${compose[@]}" exec -T proxy cat /data/caddy/pki/authorities/local/intermediate.crt \
    >"$work/caddy-intermediate.crt" || fail "the test proxy's intermediate CA is unavailable."
  spki="$(openssl x509 -in "$work/caddy-intermediate.crt" -pubkey -noout |
    openssl pkey -pubin -outform der | openssl dgst -sha256 -binary | openssl base64 -A)"
  browser_container="${project}-browser"
  start_owner_browser() {
    docker rm --force "$browser_container" >/dev/null 2>&1 || true
    # The image's entry script passes its arguments on unquoted, so none may contain a space: the
    # hostnames reach Chromium through /etc/hosts, which it reads, rather than a resolver rule.
    docker run --detach --rm --name "$browser_container" --label "${EKKLESIA_CI_LABEL}=${project}" \
      --cap-drop ALL --security-opt no-new-privileges --tmpfs /tmp \
      --network "$edge_network" --ip "$browser_address" "${host_mappings[@]}" \
      "$browser_image" \
      "--ignore-certificate-errors-spki-list=${spki}" \
      --user-data-dir=/tmp/profile >/dev/null || fail "the headless browser did not start."
  }
  cp scripts/lib/deploy-smoke-browser.mjs "$work/browser.mjs"
  stage browser.mjs
  run_browser_checks() {
    start_owner_browser
    docker run --rm --read-only --cap-drop ALL --security-opt no-new-privileges \
      --label "${EKKLESIA_CI_LABEL}=${project}" \
      --network "$edge_network" \
      -e SMOKE_CDP_ADDRESS="${browser_address}:9222" \
      -e SMOKE_HOSTNAME="$hostname" -e SMOKE_DASHBOARD_HOSTNAME="$dashboard_hostname" \
      -e SMOKE_OWNER_EMAIL="$owner_email" -e SMOKE_OWNER_PASSWORD="$owner_password" \
      -v "$daemon_work/browser.mjs:/smoke/browser.mjs:ro" \
      --entrypoint node "$image" /smoke/browser.mjs
  }
  run_browser_checks || fail "browser checks failed."

  if [[ -n "${EKKLESIA_SMOKE_DISPLAY_DEVICE:-}" ]]; then
    # A kiosk: a second browser whose profile directory is a volume, so it outlives the container
    # exactly as a kiosk's --user-data-dir outlives a browser restart.
    kiosk_container="${project}-kiosk"
    kiosk_volume="${project}-kiosk-profile"
    kiosk_address="10.249.${octet}.250"
    docker volume create --label "${EKKLESIA_CI_LABEL}=${project}" "$kiosk_volume" >/dev/null
    start_kiosk() {
      docker run --detach --rm --name "$kiosk_container" --label "${EKKLESIA_CI_LABEL}=${project}" \
        --cap-drop ALL --security-opt no-new-privileges --tmpfs /tmp \
        --network "$edge_network" --ip "$kiosk_address" "${host_mappings[@]}" \
        -v "${kiosk_volume}:/profile" \
        "$browser_image" \
        "--ignore-certificate-errors-spki-list=${spki}" \
        --user-data-dir=/profile >/dev/null || fail "the kiosk browser did not start."
    }
    wait_kiosk_exit() {
      for _ in $(seq 1 60); do
        docker inspect "$kiosk_container" >/dev/null 2>&1 || return 0
        sleep 0.5
      done
      fail "the kiosk browser did not exit after a clean close."
    }
    cp scripts/lib/deploy-smoke-display.mjs "$work/display.mjs"
    stage display.mjs
    # display_driver <budget seconds> <step>: one scenario with a hard budget. The driver enforces it
    # itself and names whatever call was still waiting; `timeout` is the backstop if it cannot.
    display_driver() {
      local budget="$1" step_name="$2" driver="${project}-display-${2}"
      timeout --signal=TERM "$((budget + 30))" \
        docker run --rm --name "$driver" --read-only --cap-drop ALL --security-opt no-new-privileges \
        --network "$edge_network" \
        -e SMOKE_CDP_ADDRESS="${browser_address}:9222" \
        -e SMOKE_KIOSK_CDP_ADDRESS="${kiosk_address}:9222" \
        -e SMOKE_STEP_BUDGET_MS="$((budget * 1000))" \
        -e SMOKE_WATCH_MS="${EKKLESIA_SMOKE_WATCH_MS:-120000}" \
        -e SMOKE_RECYCLE_WATCH_MS="${EKKLESIA_SMOKE_RECYCLE_WATCH_MS:-330000}" \
        -e SMOKE_HOSTNAME="$hostname" -e SMOKE_DASHBOARD_HOSTNAME="$dashboard_hostname" \
        -e SMOKE_OWNER_EMAIL="$owner_email" -e SMOKE_OWNER_PASSWORD="$owner_password" \
        -v "$daemon_work/display.mjs:/smoke/display.mjs:ro" \
        --entrypoint node "$image" /smoke/display.mjs "$step_name"
      local status=$?
      if ((status == 124)); then
        echo "FAIL ${step_name}: the driver did not stop within its budget plus 30s; removed it"
        docker rm --force "$driver" >/dev/null 2>&1 || true
      fi
      return "$status"
    }
    display_status=0
    step "Pairing a kiosk display, then closing its browser cleanly"
    start_kiosk
    display_driver 180 pair || display_status=1
    wait_kiosk_exit
    step "Restarting the kiosk browser on the same profile"
    start_kiosk
    display_driver 120 restart || display_status=1
    for scenario in ${EKKLESIA_SMOKE_DISPLAY_SCENARIOS:-two-pages connectivity-blip session-blip stream-failure caching-probe stream-recycle revoke}; do
      case "$scenario" in
      two-pages) budget=240 ;;
      stream-recycle) budget=420 ;;
      revoke) budget=210 ;;
      *) budget=150 ;;
      esac
      step "Display scenario: ${scenario} (budget ${budget}s)"
      display_driver "$budget" "$scenario" || display_status=1
    done
    step "Proxy: streams aborted with an incomplete response (UTC time, path only)"
    # Captured, never printed: only each line's time and path (no query, so no signature) leave it.
    proxy_logs="$("${compose[@]}" logs --no-color proxy 2>/dev/null)" || proxy_logs=""
    grep -o '{.*"aborting with incomplete response".*}' <<<"$proxy_logs" |
      sed -E 's/.*"ts":([0-9]+)[^,]*,.*"uri":"([^"?]*)[^"]*".*/\1 \2/' |
      while read -r ts uri; do echo "     $(date -u -d "@${ts}" +%T) ${uri}"; done || true
    ((display_status == 0)) || fail "display-device checks failed."
  fi

  step "Restarting the API process with the installed module and data intact"
  restart_api
  if [[ "$module_id" == example ]]; then
    module_check '["Hello, kept"]' || fail "the module or its data did not survive an API restart."
  else
    module_check present || fail "the module did not survive an API restart."
  fi
  run_browser_checks || fail "the module UI did not survive an API restart."

  step "Recreating the API container with the same module storage and database volumes"
  recreate_api
  if [[ "$module_id" == example ]]; then
    module_check '["Hello, kept"]' || fail "the module or its data did not survive container recreation."
  else
    module_check present || fail "the module did not survive container recreation."
  fi
  plan="$(plan_output)"
  if grep -q "^- \\[pending\\] ${module_id}:" <<<"$plan"; then
    fail "container recreation made an applied module migration pending again."
  fi
  run_browser_checks || fail "the module UI did not survive container recreation."
  if [[ -n "${EKKLESIA_SMOKE_DISPLAY_DEVICE:-}" ]]; then
    display_driver 120 restart || fail "the display runtime or its data did not survive container recreation."
  fi

  step "Disabling and re-enabling the module through the supported lifecycle command"
  restart_api "${compose[@]}" run --rm --no-deps migrate node \
    core/dist/cli/set-module-enabled.js --disable --module "$module_id"
  module_check absent || fail "the disabled module was still reported as running."
  module_storage "test -d /modules/${module_id}" ||
    fail "disabling the module removed its installed package."
  restart_api "${compose[@]}" run --rm --no-deps migrate node \
    core/dist/cli/set-module-enabled.js --enable --module "$module_id"
  if [[ "$module_id" == example ]]; then
    module_check '["Hello, kept"]' || fail "the re-enabled module did not return with its data."
  else
    module_check present || fail "the re-enabled module did not return."
  fi
  run_browser_checks || fail "the re-enabled module UI did not return."

  step "Removing the module from module storage and restarting: Core runs, its data stays"
  restart_api module_storage "rm -rf /modules/${module_id}"
  module_check absent || fail "the removed module is still running."
  plan="$(plan_output)"
  grep -q "^- \\[retained\\] ${module_id}: .*applied migration.*kept; the module is not installed$" \
    <<<"$plan" || fail "migration planning did not report the removed module's retained history."
  grep -q '^Migration plan (.*0 pending, 0 approval-blocked, 1 retained):$' <<<"$plan" ||
    fail "migration planning did not succeed cleanly with the module absent."
  "${compose[@]}" run --rm --no-deps migrate node core/dist/cli/apply-migrations.js >/dev/null ||
    fail "applying migrations failed with the module absent."
  if [[ "$module_id" == example ]]; then
    [[ "$(psql_as_superuser "SELECT value FROM example_records")" == "kept" ]] ||
      fail "the removed module's data did not survive."
  fi
  echo "ok   the module is gone, Core runs, its history is retained, and its data survives"

  step "Reinstalling the exact same package and restarting: the module returns with its previous data"
  restart_api install_test_module
  plan="$(plan_output)"
  grep -q "^- \\[applied\\] ${module_id}:" <<<"$plan" ||
    fail "the restored module's migration history is not valid."
  if grep -q '\[retained\]\|\[pending\]' <<<"$plan"; then
    fail "the restored module left retained or pending migrations."
  fi
  if [[ "$module_id" == example ]]; then
    module_check '["Hello, kept"]' || fail "the restored module did not return with its data."
  else
    module_check present || fail "the restored module did not return."
  fi
  run_browser_checks || fail "the restored module UI did not return."
  if [[ -n "${EKKLESIA_SMOKE_DISPLAY_DEVICE:-}" ]]; then
    display_driver 120 restart || fail "the restored display runtime did not return with its data."
  fi
  echo "ok   the restored module runs again, with its history strict and its data intact"
fi

step "Proving the API's database connection is certificate-verified TLS"
api_container="$("${compose[@]}" ps -q api)"
api_data_address="$(
  docker inspect --format "{{(index .NetworkSettings.Networks \"${project}_data\").IPAddress}}" \
    "$api_container"
)"
[[ -n "$api_data_address" ]] || fail "could not find the API's database-network address."
tls_connections="$(psql_as_superuser "SELECT count(*) FROM pg_stat_ssl s JOIN pg_stat_activity a USING (pid) WHERE a.client_addr = '${api_data_address}' AND s.ssl AND s.version = 'TLSv1.3'")"
plain_connections="$(psql_as_superuser "SELECT count(*) FROM pg_stat_ssl s JOIN pg_stat_activity a USING (pid) WHERE a.client_addr = '${api_data_address}' AND NOT s.ssl")"
((tls_connections > 0)) || fail "the API holds no TLS 1.3 database connection."
((plain_connections == 0)) || fail "the API holds a database connection without TLS."
echo "ok   the API's database connections are TLS 1.3 (${tls_connections}), none plaintext"

# PostgreSQL itself refuses a network connection without TLS.
if "${compose[@]}" exec -T -e PGPASSWORD="$postgres_password" postgres \
  psql "host=postgres user=ekklesia dbname=ekklesia sslmode=disable" -Atc 'SELECT 1' >/dev/null 2>&1; then
  fail "PostgreSQL accepted a plaintext network connection."
fi
echo "ok   PostgreSQL refuses a plaintext network connection"

# The server cannot see whether a client verified it, so verification is proven by refusal: the
# same command fails against a CA that did not issue the certificate, and against a host the
# certificate does not name.
openssl ecparam -name prime256v1 -genkey -noout -out "$work/other-ca.key" 2>/dev/null
openssl req -x509 -new -sha256 -days 1 -key "$work/other-ca.key" -out "$work/other-ca.crt" \
  -subj "/CN=Unrelated CA" 2>/dev/null
# expect_connection_refused <description> <compose run arguments...>: the command must fail, and fail
# at the connection, not on a configuration error that would prove nothing about verification.
expect_connection_refused() {
  local description="$1"
  shift
  local output
  if output="$("${compose[@]}" run --rm --no-deps "$@" 2>&1)"; then
    fail "the database connection accepted ${description}."
  fi
  grep -q 'MIGRATION_CONNECTION_FAILED' <<<"$output" ||
    fail "rejecting ${description} failed for a reason other than the connection."
}
expect_connection_refused "a certificate from a CA it was not given" \
  -e EKKLESIA_DATABASE_CA="$(cat "$work/other-ca.crt")" migrate
echo "ok   the database connection rejects a certificate its CA did not issue"
postgres_address="$(
  docker inspect --format "{{(index .NetworkSettings.Networks \"${project}_data\").IPAddress}}" \
    "$("${compose[@]}" ps -q postgres)"
)"
expect_connection_refused "a certificate for a different host name" \
  -e DATABASE_URL="postgresql://ekklesia:${postgres_password}@${postgres_address}:5432/ekklesia" migrate
echo "ok   the database connection rejects a certificate that does not name the host"
"${compose[@]}" run --rm --no-deps migrate >/dev/null || fail "the control connection failed."
echo "ok   the same connection succeeds with the installation CA and the name postgres"

step "Running Doctor from the image"
doctor_output="$("${compose[@]}" run --rm --no-deps migrate node core/dist/cli/doctor.js 2>&1)" ||
  fail "Doctor reported the installation not ready."
grep -q 'Result: READY' <<<"$doctor_output" || fail "Doctor did not report READY."
grep -q 'TLS required with certificate and hostname verification (explicit private CA supplied)' \
  <<<"$doctor_output" || fail "Doctor did not report verify-full with the private CA."
grep -q '✓ Object storage reachable' <<<"$doctor_output" ||
  fail "Doctor could not reach object storage over HTTPS."
# Doctor runs as `migrate`; it must check the client-address configuration the API runs with, which
# behind the proxy is trusted-proxy mode, not the direct mode an unconfigured process reports.
grep -q 'Trusted-proxy client-address mode' <<<"$doctor_output" ||
  fail "Doctor did not check the API's trusted-proxy client-address configuration."
for value in "$postgres_password" "$s3_secret_key" "$identity_secret"; do
  if grep -qF -- "$value" <<<"$doctor_output"; then
    fail "Doctor output contains a secret."
  fi
done
echo "ok   Doctor: READY, verify-full with the private CA, storage reachable, the API's client-address mode, no secret printed"

api_logs="$("${compose[@]}" logs --no-color api 2>&1)"
for value in "$postgres_password" "$s3_secret_key" "$identity_secret" "$owner_password"; do
  if grep -qF -- "$value" <<<"$api_logs"; then
    fail "the API's logs contain a secret."
  fi
done
restarts="$(docker inspect --format '{{.RestartCount}}' "$api_container")"
((restarts == 0)) || fail "the API restarted ${restarts} time(s)."
echo "ok   the API logged no secret and never restarted"

echo "Reference deployment smoke test passed."