Ekklesia/scripts/smoke-deploy.sh

#!/usr/bin/env bash
# End-to-end smoke test of the reference deployment (ADR 0044): deploy/compose.yaml with
# deploy/compose.proxy.yaml, a platform-only image, and a disposable PostgreSQL and SeaweedFS that
# exist only for this run. CI runs it; it is safe to run on a workstation.
#
#   bash scripts/smoke-deploy.sh                 builds the image first
#   EKKLESIA_SMOKE_IMAGE=<image> bash scripts/smoke-deploy.sh
#   EKKLESIA_SMOKE_TOPOLOGY=split bash scripts/smoke-deploy.sh
#
# Either topology (ADR 0045). `same-origin` (the default) serves the Dashboard, the API, and displays
# from one hostname with Caddyfile.example and a platform-only image. `split` serves the Dashboard on
# its own hostname with Caddyfile.split.example, builds an image with the repository's module
# template installed as a test-only module (so a module UI, its API routes, and a display exist), and
# then drives a real headless Chromium through the Dashboard, the module, the display, and sign-out.
# With EKKLESIA_SMOKE_IMAGE, the given image is used as is in either topology; a split run then
# exercises whichever of its modules publishes UI and a display.
#
# TEST-ONLY: the proxy runs with Caddy's internal CA (`local_certs`) instead of ACME, because a test
# run has no public hostname. Never use that option in an installation.
#
# Follows the documented first boot: TLS material, secrets, data services, an explicit migration
# apply against this run's own fresh database, then the API and proxy. Everything it starts, and
# every file it writes, is removed on exit.
#
# Bind mounts are resolved by the Docker daemon, on its host. When this script runs inside a CI job
# container that drives the host's Docker, its own files are not at those paths, so every file a
# container mounts is staged under an identical path in the Docker host's /tmp first.

set -Eeuo pipefail

repository_root="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")/.." && pwd)"
cd "$repository_root"

for command in docker openssl awk; do
  command -v "$command" >/dev/null || {
    echo "smoke-deploy: $command is required." >&2
    exit 1
  }
done
docker compose version >/dev/null 2>&1 || {
  echo "smoke-deploy: Docker Compose is required." >&2
  exit 1
}

# The pinned PostgreSQL image doubles as a root shell for file ownership, so the run needs neither
# root nor sudo on the host.
postgres_image="postgres:18.4-alpine3.24@sha256:9a8afca54e7861fd90fab5fdf4c42477a6b1cb7d293595148e674e0a3181de15"
# The split topology's real browser: headless Chromium, pinned by its multi-architecture index digest.
browser_image="chromedp/headless-shell:151.0.7922.109@sha256:2d349b544a1ea6b5b5fd7c0fe99215ff662339c57407ee2e8c0a11af93516b04"

topology="${EKKLESIA_SMOKE_TOPOLOGY:-same-origin}"
case "$topology" in
same-origin | split) ;;
*)
  echo "smoke-deploy: EKKLESIA_SMOKE_TOPOLOGY must be same-origin or split." >&2
  exit 1
  ;;
esac

run_id="$(openssl rand -hex 4)"
project="ekklesia-deploy-smoke-${run_id}"
work="$(mktemp -d)"
image="${EKKLESIA_SMOKE_IMAGE:-}"
built_image=""
# `hostname` is always the API's (and the display host's); the Dashboard has its own only when split.
if [[ "$topology" == split ]]; then
  hostname="api.smoke.test"
  dashboard_hostname="dash.smoke.test"
  caddyfile_example="deploy/Caddyfile.split.example"
else
  hostname="ekklesia.smoke.test"
  dashboard_hostname=""
  caddyfile_example="deploy/Caddyfile.example"
fi
dashboard_origin="https://${dashboard_hostname:-$hostname}"
storage_hostname="storage.smoke.test"
browser_container=""

# A per-run edge subnet, so concurrent runs and the host's own networks do not collide. Fixed
# addresses come from the upper half, outside Docker's dynamic range, exactly as documented.
octet=$((RANDOM % 200 + 20))
edge_subnet="10.249.${octet}.0/24"
edge_range="10.249.${octet}.0/25"
proxy_address="10.249.${octet}.254"
tunnel_address="10.249.${octet}.253"
# A test-only client address, also outside the dynamic range, so its limiter bucket is its own.
session_client_address="10.249.${octet}.252"
# The headless browser's fixed address, likewise (split topology only).
browser_address="10.249.${octet}.251"

# daemon_work: where the Docker daemon sees this run's mounted files. The same directory as `work`
# on a workstation; a staged copy on the Docker host when this script runs in a job container.
if [[ -f /.dockerenv ]]; then
  daemon_work="/tmp/${project}"
  docker run --rm --entrypoint mkdir -v /tmp:/host-tmp "$postgres_image" -p "/host-tmp/${project}"
  # stage <path relative to work>: copy it to the same relative path under daemon_work, keeping
  # numeric ownership and modes.
  stage() {
    tar -C "$work" -cf - "$1" |
      docker run --rm -i --entrypoint tar -v "$daemon_work:/w" "$postgres_image" -xpf - -C /w
  }
else
  daemon_work="$work"
  stage() { :; }
fi

compose=(
  docker compose
  --project-name "$project"
  --project-directory "$daemon_work/deploy"
  --env-file "$work/compose.env"
  --file deploy/compose.yaml
  --file deploy/compose.proxy.yaml
)

cleanup() {
  local status=$?
  trap - EXIT INT TERM
  if [[ -n "$browser_container" ]]; then
    docker rm --force "$browser_container" >/dev/null 2>&1 || true
  fi
  if [[ -f "$work/compose.env" ]]; then
    "${compose[@]}" --profile ops down --volumes --remove-orphans --timeout 10 >/dev/null 2>&1 || status=1
  fi
  # Some files are owned by the PostgreSQL and SeaweedFS uids by now; remove them as root, in the
  # same image that set that ownership. In a job container the daemon cannot see `work` (and the
  # job itself runs as root), so only the staged copy on the Docker host needs this.
  if [[ "$daemon_work" == "$work" ]]; then
    docker run --rm --entrypoint rm -v "$work:/work" "$postgres_image" -rf /work/tls /work/env \
      >/dev/null 2>&1 || true
  else
    docker run --rm --entrypoint rm -v /tmp:/host-tmp "$postgres_image" -rf "/host-tmp/${project}" \
      >/dev/null 2>&1 || true
  fi
  rm -rf "$work" || true
  if [[ -n "$built_image" ]]; then
    docker image rm --force "$built_image" >/dev/null 2>&1 || true
  fi
  exit "$status"
}
trap cleanup EXIT INT TERM

step() {
  echo "==> $*"
}

fail() {
  echo "smoke-deploy: $*" >&2
  exit 1
}

if [[ -z "$image" && "$topology" == same-origin ]]; then
  step "Building the platform-only image"
  built_image="ekklesia-deploy-smoke:${run_id}"
  DOCKER_BUILDKIT=1 docker build --quiet --file deploy/Dockerfile --tag "$built_image" . >/dev/null
  image="$built_image"
elif [[ -z "$image" ]]; then
  # TEST-ONLY: the repository's module template, installed as the module `example`, in a copy of the
  # working tree. Its lockfile importer moves with it; the depth is the same, so every relative link
  # in that importer stays correct and the install stays frozen.
  step "Building an image with the module template installed as a test-only module"
  command -v git >/dev/null || fail "git is required to build the split topology's test image."
  context="$work/build-context"
  mkdir -p "$context"
  git ls-files -z --cached --others --exclude-standard |
    tar --null --files-from=- --create --file=- |
    tar -C "$context" -xf -
  mkdir -p "$context/modules"
  mv "$context/templates/module" "$context/modules/example"
  [[ "$(grep -c '^  templates/module:$' "$context/pnpm-lock.yaml")" == 1 ]] ||
    fail "the lockfile has no single templates/module importer to move."
  sed -i 's|^  templates/module:$|  modules/example:|' "$context/pnpm-lock.yaml"
  built_image="ekklesia-deploy-smoke:${run_id}"
  DOCKER_BUILDKIT=1 docker build --quiet --file "$context/deploy/Dockerfile" --tag "$built_image" \
    "$context" >/dev/null || fail "the test-module image did not build."
  rm -rf "$context"
  image="$built_image"
fi

# The compose project directory: its relative bind mount (postgres/pg_hba.conf) resolves here.
cp -R deploy "$work/deploy"
stage deploy

step "Creating the installation-private CA and server certificates"
init_status=0
sh deploy/scripts/init-tls.sh "$work/tls" >/dev/null 2>&1 || init_status=$?
if ((init_status != 0 && init_status != 77)); then
  fail "init-tls.sh failed to create TLS material."
fi
stage tls
# The root-only ownership pass, exactly as an operator would re-run the script with root.
ownership_output="$(
  docker run --rm --entrypoint sh -v "$daemon_work/tls:/tls" \
    -v "$daemon_work/deploy/scripts:/scripts:ro" "$postgres_image" /scripts/init-tls.sh /tls
)" || fail "init-tls.sh could not apply ownership."
database_ca_line="$(grep '^EKKLESIA_DATABASE_CA=' <<<"$ownership_output")" ||
  fail "init-tls.sh did not print the EKKLESIA_DATABASE_CA line."
# A second run must create nothing and still succeed.
rerun_output="$(
  docker run --rm --entrypoint sh -v "$daemon_work/tls:/tls" \
    -v "$daemon_work/deploy/scripts:/scripts:ro" "$postgres_image" /scripts/init-tls.sh /tls
)" || fail "init-tls.sh failed on a second run."
grep -q 'nothing to create' <<<"$rerun_output" || fail "init-tls.sh is not idempotent."

step "Writing env files from the examples, with freshly generated secrets"
secrets="$(sh deploy/scripts/gen-secrets.sh)"
secret() {
  awk -F= -v key="$1" '$1 == key { sub(/^[^=]*=/, ""); print; exit }' <<<"$secrets"
}
postgres_password="$(secret POSTGRES_PASSWORD)"
s3_access_key="$(secret S3_ACCESS_KEY)"
s3_secret_key="$(secret S3_SECRET_KEY)"
identity_secret="$(secret EKKLESIA_IDENTITY_SECRET)"

mkdir -p "$work/env"
# set_env <file> <key> <value>: replaces an uncommented KEY= line, keeping the value byte-exact.
set_env() {
  KEY="$2" VALUE="$3" awk '
    BEGIN { found = 0 }
    index($0, ENVIRON["KEY"] "=") == 1 { print ENVIRON["KEY"] "=" ENVIRON["VALUE"]; found = 1; next }
    { print }
    END { if (!found) print ENVIRON["KEY"] "=" ENVIRON["VALUE"] }
  ' "$1" >"$1.next"
  mv "$1.next" "$1"
}
for name in ekklesia postgres object-storage; do
  cp "deploy/env/${name}.env.example" "$work/env/${name}.env"
done
ekklesia_env="$work/env/ekklesia.env"
set_env "$ekklesia_env" DATABASE_URL "postgresql://ekklesia:${postgres_password}@postgres:5432/ekklesia"
set_env "$ekklesia_env" EKKLESIA_DATABASE_CA "${database_ca_line#EKKLESIA_DATABASE_CA=}"
set_env "$ekklesia_env" EKKLESIA_IDENTITY_BASE_URL "https://${hostname}"
set_env "$ekklesia_env" EKKLESIA_IDENTITY_TRUSTED_ORIGINS "$dashboard_origin"
if [[ "$topology" == split ]]; then
  set_env "$ekklesia_env" EKKLESIA_DASHBOARD_ORIGIN "$dashboard_origin"
fi
set_env "$ekklesia_env" EKKLESIA_IDENTITY_SECRET "$identity_secret"
set_env "$ekklesia_env" S3_DELIVERY_ENDPOINT "https://${storage_hostname}"
set_env "$ekklesia_env" S3_ACCESS_KEY "$s3_access_key"
set_env "$ekklesia_env" S3_SECRET_KEY "$s3_secret_key"
set_env "$work/env/postgres.env" POSTGRES_PASSWORD "$postgres_password"
set_env "$work/env/object-storage.env" AWS_ACCESS_KEY_ID "$s3_access_key"
set_env "$work/env/object-storage.env" AWS_SECRET_ACCESS_KEY "$s3_secret_key"
chmod 600 "$work"/env/*.env

# TEST-ONLY: Caddy's internal CA instead of ACME. Inserted as the first global option.
awk '!done && $0 == "{" { print; print "\tlocal_certs"; done = 1; next } { print }' \
  "$caddyfile_example" >"$work/Caddyfile"
grep -q $'^\tlocal_certs$' "$work/Caddyfile" || fail "could not enable local_certs for the test proxy."
stage Caddyfile

cat >"$work/compose.env" <<EOF
EKKLESIA_IMAGE=${image}
EKKLESIA_ENV_DIR=${work}/env
EKKLESIA_TLS_DIR=${daemon_work}/tls
EKKLESIA_HOSTNAME=${hostname}
EKKLESIA_DASHBOARD_HOSTNAME=${dashboard_hostname}
EKKLESIA_STORAGE_HOSTNAME=${storage_hostname}
[email protected]
EKKLESIA_CADDYFILE=${daemon_work}/Caddyfile
EKKLESIA_PUBLISH_ADDRESS=127.0.0.1
EKKLESIA_HTTP_PORT=0
EKKLESIA_HTTPS_PORT=0
EKKLESIA_EDGE_SUBNET=${edge_subnet}
EKKLESIA_EDGE_DYNAMIC_RANGE=${edge_range}
EKKLESIA_PROXY_ADDRESS=${proxy_address}
EKKLESIA_TUNNEL_ADDRESS=${tunnel_address}
EOF

step "Starting PostgreSQL and object storage"
"${compose[@]}" up --detach --wait --wait-timeout 180 postgres object-storage >/dev/null 2>&1 ||
  fail "PostgreSQL and object storage did not become healthy."

step "Planning, then explicitly applying, migrations against this run's fresh database"
"${compose[@]}" run --rm --no-deps migrate >/dev/null || fail "migrate (plan) failed."
"${compose[@]}" run --rm --no-deps migrate node core/dist/cli/apply-migrations.js >/dev/null ||
  fail "migrate (apply) failed."

step "Starting the API and the proxy"
"${compose[@]}" up --detach --wait --wait-timeout 180 >/dev/null 2>&1 ||
  fail "the API and proxy did not become healthy."

# Caddy creates its internal CA on first start. The test clients trust it, plus the installation CA
# for their direct connection to object-storage.
for _ in $(seq 1 30); do
  if "${compose[@]}" exec -T proxy cat /data/caddy/pki/authorities/local/root.crt \
    >"$work/caddy-root.crt" 2>/dev/null && [[ -s "$work/caddy-root.crt" ]]; then
    break
  fi
  sleep 1
done
[[ -s "$work/caddy-root.crt" ]] || fail "the test proxy's internal CA did not appear."
cat "$work/caddy-root.crt" "$work/tls/ca.crt" >"$work/client-ca.pem"
cp scripts/lib/deploy-smoke-client.mjs "$work/client.mjs"
stage client-ca.pem
stage client.mjs

edge_network="${project}_edge"
host_mappings=(--add-host "${hostname}:${proxy_address}" --add-host "${storage_hostname}:${proxy_address}")
if [[ -n "$dashboard_hostname" ]]; then
  host_mappings+=(--add-host "${dashboard_hostname}:${proxy_address}")
fi
client() {
  local mode="$1"
  shift
  docker run --rm --read-only --cap-drop ALL --security-opt no-new-privileges \
    --network "$edge_network" "$@" "${host_mappings[@]}" \
    -e NODE_EXTRA_CA_CERTS=/smoke/client-ca.pem -e SMOKE_HOSTNAME="$hostname" \
    -e SMOKE_DASHBOARD_HOSTNAME="$dashboard_hostname" \
    -e SMOKE_STORAGE_HOSTNAME="$storage_hostname" \
    -e S3_ACCESS_KEY="$s3_access_key" -e S3_SECRET_KEY="$s3_secret_key" \
    -v "$daemon_work/client.mjs:/smoke/client.mjs:ro" \
    -v "$daemon_work/client-ca.pem:/smoke/client-ca.pem:ro" \
    --entrypoint node "$image" /smoke/client.mjs "$mode"
}

step "Checking the API, Dashboard, storage delivery, and client-address chain through the proxy"
client edge || fail "edge checks failed."

step "Checking the tunnel connector's client-address path from its fixed address"
client tunnel --ip "$tunnel_address" || fail "tunnel checks failed."

step "Creating the initial owner with the image's own CLI, then signing in and out through the proxy"
owner_email="[email protected]"
owner_password="$(openssl rand -hex 16)"
printf '%s' "$owner_password" |
  "${compose[@]}" run --rm --no-deps -T migrate node core/dist/cli/create-initial-owner.js \
    --email "$owner_email" >/dev/null || fail "creating the initial owner failed."
echo "ok   the initial owner was created from the image (password on standard input)"
client session --ip "$session_client_address" \
  -e SMOKE_OWNER_EMAIL="$owner_email" -e SMOKE_OWNER_PASSWORD="$owner_password" ||
  fail "session checks failed."

if [[ "$topology" == split ]]; then
  step "Driving a real browser through the split Dashboard, a module, its display, and sign-out"
  # Chromium trusts this run's proxy through the public-key hash of Caddy's internal intermediate CA,
  # which signs every certificate the test proxy serves: no CA is installed anywhere, and nothing but
  # this run's certificates is accepted.
  "${compose[@]}" exec -T proxy cat /data/caddy/pki/authorities/local/intermediate.crt \
    >"$work/caddy-intermediate.crt" || fail "the test proxy's intermediate CA is unavailable."
  spki="$(openssl x509 -in "$work/caddy-intermediate.crt" -pubkey -noout |
    openssl pkey -pubin -outform der | openssl dgst -sha256 -binary | openssl base64 -A)"
  browser_container="${project}-browser"
  # The image's entry script passes its arguments on unquoted, so none may contain a space: the
  # hostnames reach Chromium through /etc/hosts, which it reads, rather than a resolver rule.
  docker run --detach --rm --name "$browser_container" \
    --cap-drop ALL --security-opt no-new-privileges --tmpfs /tmp \
    --network "$edge_network" --ip "$browser_address" "${host_mappings[@]}" \
    "$browser_image" \
    "--ignore-certificate-errors-spki-list=${spki}" \
    --user-data-dir=/tmp/profile >/dev/null || fail "the headless browser did not start."
  cp scripts/lib/deploy-smoke-browser.mjs "$work/browser.mjs"
  stage browser.mjs
  docker run --rm --read-only --cap-drop ALL --security-opt no-new-privileges \
    --network "$edge_network" \
    -e SMOKE_CDP_ADDRESS="${browser_address}:9222" \
    -e SMOKE_HOSTNAME="$hostname" -e SMOKE_DASHBOARD_HOSTNAME="$dashboard_hostname" \
    -e SMOKE_OWNER_EMAIL="$owner_email" -e SMOKE_OWNER_PASSWORD="$owner_password" \
    -v "$daemon_work/browser.mjs:/smoke/browser.mjs:ro" \
    --entrypoint node "$image" /smoke/browser.mjs || fail "browser checks failed."
fi

step "Proving the API's database connection is certificate-verified TLS"
api_container="$("${compose[@]}" ps -q api)"
api_data_address="$(
  docker inspect --format "{{(index .NetworkSettings.Networks \"${project}_data\").IPAddress}}" \
    "$api_container"
)"
[[ -n "$api_data_address" ]] || fail "could not find the API's database-network address."
psql_as_superuser() {
  "${compose[@]}" exec -T postgres psql -U ekklesia -d ekklesia -Atc "$1"
}
tls_connections="$(psql_as_superuser "SELECT count(*) FROM pg_stat_ssl s JOIN pg_stat_activity a USING (pid) WHERE a.client_addr = '${api_data_address}' AND s.ssl AND s.version = 'TLSv1.3'")"
plain_connections="$(psql_as_superuser "SELECT count(*) FROM pg_stat_ssl s JOIN pg_stat_activity a USING (pid) WHERE a.client_addr = '${api_data_address}' AND NOT s.ssl")"
((tls_connections > 0)) || fail "the API holds no TLS 1.3 database connection."
((plain_connections == 0)) || fail "the API holds a database connection without TLS."
echo "ok   the API's database connections are TLS 1.3 (${tls_connections}), none plaintext"

# PostgreSQL itself refuses a network connection without TLS.
if "${compose[@]}" exec -T -e PGPASSWORD="$postgres_password" postgres \
  psql "host=postgres user=ekklesia dbname=ekklesia sslmode=disable" -Atc 'SELECT 1' >/dev/null 2>&1; then
  fail "PostgreSQL accepted a plaintext network connection."
fi
echo "ok   PostgreSQL refuses a plaintext network connection"

# The server cannot see whether a client verified it, so verification is proven by refusal: the
# same command fails against a CA that did not issue the certificate, and against a host the
# certificate does not name.
openssl ecparam -name prime256v1 -genkey -noout -out "$work/other-ca.key" 2>/dev/null
openssl req -x509 -new -sha256 -days 1 -key "$work/other-ca.key" -out "$work/other-ca.crt" \
  -subj "/CN=Unrelated CA" 2>/dev/null
# expect_connection_refused <description> <compose run arguments...>: the command must fail, and fail
# at the connection, not on a configuration error that would prove nothing about verification.
expect_connection_refused() {
  local description="$1"
  shift
  local output
  if output="$("${compose[@]}" run --rm --no-deps "$@" 2>&1)"; then
    fail "the database connection accepted ${description}."
  fi
  grep -q 'MIGRATION_CONNECTION_FAILED' <<<"$output" ||
    fail "rejecting ${description} failed for a reason other than the connection."
}
expect_connection_refused "a certificate from a CA it was not given" \
  -e EKKLESIA_DATABASE_CA="$(cat "$work/other-ca.crt")" migrate
echo "ok   the database connection rejects a certificate its CA did not issue"
postgres_address="$(
  docker inspect --format "{{(index .NetworkSettings.Networks \"${project}_data\").IPAddress}}" \
    "$("${compose[@]}" ps -q postgres)"
)"
expect_connection_refused "a certificate for a different host name" \
  -e DATABASE_URL="postgresql://ekklesia:${postgres_password}@${postgres_address}:5432/ekklesia" migrate
echo "ok   the database connection rejects a certificate that does not name the host"
"${compose[@]}" run --rm --no-deps migrate >/dev/null || fail "the control connection failed."
echo "ok   the same connection succeeds with the installation CA and the name postgres"

step "Running Doctor from the image"
doctor_output="$("${compose[@]}" run --rm --no-deps migrate node core/dist/cli/doctor.js 2>&1)" ||
  fail "Doctor reported the installation not ready."
grep -q 'Result: READY' <<<"$doctor_output" || fail "Doctor did not report READY."
grep -q 'TLS required with certificate and hostname verification (explicit private CA supplied)' \
  <<<"$doctor_output" || fail "Doctor did not report verify-full with the private CA."
grep -q '✓ Object storage reachable' <<<"$doctor_output" ||
  fail "Doctor could not reach object storage over HTTPS."
for value in "$postgres_password" "$s3_secret_key" "$identity_secret"; do
  if grep -qF -- "$value" <<<"$doctor_output"; then
    fail "Doctor output contains a secret."
  fi
done
echo "ok   Doctor: READY, verify-full with the private CA, storage reachable, no secret printed"

api_logs="$("${compose[@]}" logs --no-color api 2>&1)"
for value in "$postgres_password" "$s3_secret_key" "$identity_secret" "$owner_password"; do
  if grep -qF -- "$value" <<<"$api_logs"; then
    fail "the API's logs contain a secret."
  fi
done
restarts="$(docker inspect --format '{{.RestartCount}}' "$api_container")"
((restarts == 0)) || fail "the API restarted ${restarts} time(s)."
echo "ok   the API logged no secret and never restarted"

echo "Reference deployment smoke test passed."