Ekklesia/scripts/smoke-deploy.sh

#!/usr/bin/env bash
# End-to-end smoke test of the reference deployment (ADR 0044): deploy/compose.yaml with
# deploy/compose.proxy.yaml, a platform-only image, and a disposable PostgreSQL and SeaweedFS that
# exist only for this run. CI runs it; it is safe to run on a workstation.
#
#   bash scripts/smoke-deploy.sh                 builds the image first
#   EKKLESIA_SMOKE_IMAGE=<image> bash scripts/smoke-deploy.sh
#
# TEST-ONLY: the proxy runs with Caddy's internal CA (`local_certs`) instead of ACME, because a test
# run has no public hostname. Never use that option in an installation.
#
# Follows the documented first boot: TLS material, secrets, data services, an explicit migration
# apply against this run's own fresh database, then the API and proxy. Everything it starts, and
# every file it writes, is removed on exit.
#
# Bind mounts are resolved by the Docker daemon, on its host. When this script runs inside a CI job
# container that drives the host's Docker, its own files are not at those paths, so every file a
# container mounts is staged under an identical path in the Docker host's /tmp first.

set -Eeuo pipefail

repository_root="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")/.." && pwd)"
cd "$repository_root"

# shellcheck source=scripts/lib/compose-diagnostics.sh
source "$repository_root/scripts/lib/compose-diagnostics.sh"

for command in docker openssl awk; do
  command -v "$command" >/dev/null || {
    echo "smoke-deploy: $command is required." >&2
    exit 1
  }
done
docker compose version >/dev/null 2>&1 || {
  echo "smoke-deploy: Docker Compose is required." >&2
  exit 1
}

# The pinned PostgreSQL image doubles as a root shell for file ownership, so the run needs neither
# root nor sudo on the host.
postgres_image="postgres:18.4-alpine3.24@sha256:9a8afca54e7861fd90fab5fdf4c42477a6b1cb7d293595148e674e0a3181de15"

run_id="$(openssl rand -hex 4)"
project="ekklesia-deploy-smoke-${run_id}"
work="$(mktemp -d)"
image="${EKKLESIA_SMOKE_IMAGE:-}"
built_image=""
hostname="ekklesia.smoke.test"
storage_hostname="storage.smoke.test"

# choose_edge_network: a per-run edge subnet that no network on this Docker daemon already uses, so
# concurrent runs sharing a runner's daemon do not collide. Two runs can still pick the same free
# subnet at the same moment; the first `up` below detects that and chooses again. Fixed addresses come
# from the upper half, outside Docker's dynamic range, exactly as documented.
choose_edge_network() {
  local in_use candidate attempt
  in_use="$(
    docker network ls --quiet |
      xargs -r docker network inspect --format '{{range .IPAM.Config}}{{println .Subnet}}{{end}}' 2>/dev/null |
      sed -n 's#^10\.249\.\([0-9]*\)\.0/24$#\1#p'
  )" || in_use=""
  for attempt in $(seq 1 100); do
    candidate=$((RANDOM % 200 + 20))
    if ! grep -qx "$candidate" <<<"$in_use"; then
      octet="$candidate"
      edge_subnet="10.249.${octet}.0/24"
      edge_range="10.249.${octet}.0/25"
      proxy_address="10.249.${octet}.254"
      tunnel_address="10.249.${octet}.253"
      # A test-only client address, also outside the dynamic range, so its limiter bucket is its own.
      session_client_address="10.249.${octet}.252"
      return 0
    fi
  done
  echo "smoke-deploy: no free 10.249.x.0/24 edge subnet on this Docker daemon." >&2
  exit 1
}
choose_edge_network

# daemon_work: where the Docker daemon sees this run's mounted files. The same directory as `work`
# on a workstation; a staged copy on the Docker host when this script runs in a job container.
if [[ -f /.dockerenv ]]; then
  daemon_work="/tmp/${project}"
  docker run --rm --entrypoint mkdir -v /tmp:/host-tmp "$postgres_image" -p "/host-tmp/${project}"
  # stage <path relative to work>: copy it to the same relative path under daemon_work, keeping
  # numeric ownership and modes.
  stage() {
    tar -C "$work" -cf - "$1" |
      docker run --rm -i --entrypoint tar -v "$daemon_work:/w" "$postgres_image" -xpf - -C /w
  }
else
  daemon_work="$work"
  stage() { :; }
fi

compose=(
  docker compose
  --project-name "$project"
  --project-directory "$daemon_work/deploy"
  --env-file "$work/compose.env"
  --file deploy/compose.yaml
  --file deploy/compose.proxy.yaml
)

cleanup() {
  local status=$?
  trap - EXIT INT TERM
  if [[ -f "$work/compose.env" ]]; then
    "${compose[@]}" --profile ops down --volumes --remove-orphans --timeout 10 >/dev/null 2>&1 || status=1
  fi
  # Some files are owned by the PostgreSQL and SeaweedFS uids by now; remove them as root, in the
  # same image that set that ownership. In a job container the daemon cannot see `work` (and the
  # job itself runs as root), so only the staged copy on the Docker host needs this.
  if [[ "$daemon_work" == "$work" ]]; then
    docker run --rm --entrypoint rm -v "$work:/work" "$postgres_image" -rf /work/tls /work/env \
      >/dev/null 2>&1 || true
  else
    docker run --rm --entrypoint rm -v /tmp:/host-tmp "$postgres_image" -rf "/host-tmp/${project}" \
      >/dev/null 2>&1 || true
  fi
  rm -rf "$work" || true
  if [[ -n "$built_image" ]]; then
    docker image rm --force "$built_image" >/dev/null 2>&1 || true
  fi
  exit "$status"
}
trap cleanup EXIT INT TERM

step() {
  echo "==> $*"
}

fail() {
  echo "smoke-deploy: $*" >&2
  exit 1
}

if [[ -z "$image" ]]; then
  step "Building the platform-only image"
  built_image="ekklesia-deploy-smoke:${run_id}"
  DOCKER_BUILDKIT=1 docker build --quiet --file deploy/Dockerfile --tag "$built_image" . >/dev/null
  image="$built_image"
fi

# The compose project directory: its relative bind mount (postgres/pg_hba.conf) resolves here.
cp -R deploy "$work/deploy"
stage deploy

step "Creating the installation-private CA and server certificates"
init_status=0
sh deploy/scripts/init-tls.sh "$work/tls" >/dev/null 2>&1 || init_status=$?
if ((init_status != 0 && init_status != 77)); then
  fail "init-tls.sh failed to create TLS material."
fi
stage tls
# The root-only ownership pass, exactly as an operator would re-run the script with root.
ownership_output="$(
  docker run --rm --entrypoint sh -v "$daemon_work/tls:/tls" \
    -v "$daemon_work/deploy/scripts:/scripts:ro" "$postgres_image" /scripts/init-tls.sh /tls
)" || fail "init-tls.sh could not apply ownership."
database_ca_line="$(grep '^EKKLESIA_DATABASE_CA=' <<<"$ownership_output")" ||
  fail "init-tls.sh did not print the EKKLESIA_DATABASE_CA line."
# A second run must create nothing and still succeed.
rerun_output="$(
  docker run --rm --entrypoint sh -v "$daemon_work/tls:/tls" \
    -v "$daemon_work/deploy/scripts:/scripts:ro" "$postgres_image" /scripts/init-tls.sh /tls
)" || fail "init-tls.sh failed on a second run."
grep -q 'nothing to create' <<<"$rerun_output" || fail "init-tls.sh is not idempotent."

step "Writing env files from the examples, with freshly generated secrets"
secrets="$(sh deploy/scripts/gen-secrets.sh)"
secret() {
  awk -F= -v key="$1" '$1 == key { sub(/^[^=]*=/, ""); print; exit }' <<<"$secrets"
}
postgres_password="$(secret POSTGRES_PASSWORD)"
s3_access_key="$(secret S3_ACCESS_KEY)"
s3_secret_key="$(secret S3_SECRET_KEY)"
identity_secret="$(secret EKKLESIA_IDENTITY_SECRET)"
# Masked in any diagnostics below, along with anything else that looks like a credential.
COMPOSE_DIAGNOSTICS_SECRETS=("$postgres_password" "$s3_access_key" "$s3_secret_key" "$identity_secret")

mkdir -p "$work/env"
# set_env <file> <key> <value>: replaces an uncommented KEY= line, keeping the value byte-exact.
set_env() {
  KEY="$2" VALUE="$3" awk '
    BEGIN { found = 0 }
    index($0, ENVIRON["KEY"] "=") == 1 { print ENVIRON["KEY"] "=" ENVIRON["VALUE"]; found = 1; next }
    { print }
    END { if (!found) print ENVIRON["KEY"] "=" ENVIRON["VALUE"] }
  ' "$1" >"$1.next"
  mv "$1.next" "$1"
}
for name in ekklesia postgres object-storage; do
  cp "deploy/env/${name}.env.example" "$work/env/${name}.env"
done
ekklesia_env="$work/env/ekklesia.env"
set_env "$ekklesia_env" DATABASE_URL "postgresql://ekklesia:${postgres_password}@postgres:5432/ekklesia"
set_env "$ekklesia_env" EKKLESIA_DATABASE_CA "${database_ca_line#EKKLESIA_DATABASE_CA=}"
set_env "$ekklesia_env" EKKLESIA_IDENTITY_BASE_URL "https://${hostname}"
set_env "$ekklesia_env" EKKLESIA_IDENTITY_TRUSTED_ORIGINS "https://${hostname}"
set_env "$ekklesia_env" EKKLESIA_IDENTITY_SECRET "$identity_secret"
set_env "$ekklesia_env" S3_DELIVERY_ENDPOINT "https://${storage_hostname}"
set_env "$ekklesia_env" S3_ACCESS_KEY "$s3_access_key"
set_env "$ekklesia_env" S3_SECRET_KEY "$s3_secret_key"
set_env "$work/env/postgres.env" POSTGRES_PASSWORD "$postgres_password"
set_env "$work/env/object-storage.env" AWS_ACCESS_KEY_ID "$s3_access_key"
set_env "$work/env/object-storage.env" AWS_SECRET_ACCESS_KEY "$s3_secret_key"
chmod 600 "$work"/env/*.env

# TEST-ONLY: Caddy's internal CA instead of ACME. Inserted as the first global option.
awk '!done && $0 == "{" { print; print "\tlocal_certs"; done = 1; next } { print }' \
  deploy/Caddyfile.example >"$work/Caddyfile"
grep -q $'^\tlocal_certs$' "$work/Caddyfile" || fail "could not enable local_certs for the test proxy."
stage Caddyfile

cat >"$work/compose.env" <<EOF
EKKLESIA_IMAGE=${image}
EKKLESIA_ENV_DIR=${work}/env
EKKLESIA_TLS_DIR=${daemon_work}/tls
EKKLESIA_HOSTNAME=${hostname}
EKKLESIA_STORAGE_HOSTNAME=${storage_hostname}
[email protected]
EKKLESIA_CADDYFILE=${daemon_work}/Caddyfile
EKKLESIA_PUBLISH_ADDRESS=127.0.0.1
EKKLESIA_HTTP_PORT=0
EKKLESIA_HTTPS_PORT=0
EKKLESIA_EDGE_SUBNET=${edge_subnet}
EKKLESIA_EDGE_DYNAMIC_RANGE=${edge_range}
EKKLESIA_PROXY_ADDRESS=${proxy_address}
EKKLESIA_TUNNEL_ADDRESS=${tunnel_address}
EOF

step "Starting PostgreSQL and object storage"
# This `up` creates the edge network. If another run claimed the same subnet since it was chosen,
# Docker refuses with an address-pool overlap: choose another and start again, a bounded number of
# times. Any other failure is reported with the project's containers and logs.
for attempt in 1 2 3 4 5; do
  if up_output="$(
    "${compose[@]}" up --detach --wait --wait-timeout 180 postgres object-storage 2>&1
  )"; then
    break
  fi
  if [[ "$up_output" == *"overlap"* ]] && ((attempt < 5)); then
    echo "     edge subnet ${edge_subnet} was taken by a concurrent run; choosing another"
    "${compose[@]}" down --volumes --remove-orphans --timeout 10 >/dev/null 2>&1 || true
    choose_edge_network
    set_env "$work/compose.env" EKKLESIA_EDGE_SUBNET "$edge_subnet"
    set_env "$work/compose.env" EKKLESIA_EDGE_DYNAMIC_RANGE "$edge_range"
    set_env "$work/compose.env" EKKLESIA_PROXY_ADDRESS "$proxy_address"
    set_env "$work/compose.env" EKKLESIA_TUNNEL_ADDRESS "$tunnel_address"
    continue
  fi
  compose_report_not_ready "$up_output" "${compose[@]}"
  fail "PostgreSQL and object storage did not become healthy."
done

step "Planning, then explicitly applying, migrations against this run's fresh database"
"${compose[@]}" run --rm --no-deps migrate >/dev/null || fail "migrate (plan) failed."
"${compose[@]}" run --rm --no-deps migrate node core/dist/cli/apply-migrations.js >/dev/null ||
  fail "migrate (apply) failed."

step "Starting the API and the proxy"
if ! up_output="$("${compose[@]}" up --detach --wait --wait-timeout 180 2>&1)"; then
  compose_report_not_ready "$up_output" "${compose[@]}"
  fail "the API and proxy did not become healthy."
fi

# Caddy creates its internal CA on first start. The test clients trust it, plus the installation CA
# for their direct connection to object-storage.
for _ in $(seq 1 30); do
  if "${compose[@]}" exec -T proxy cat /data/caddy/pki/authorities/local/root.crt \
    >"$work/caddy-root.crt" 2>/dev/null && [[ -s "$work/caddy-root.crt" ]]; then
    break
  fi
  sleep 1
done
[[ -s "$work/caddy-root.crt" ]] || fail "the test proxy's internal CA did not appear."
cat "$work/caddy-root.crt" "$work/tls/ca.crt" >"$work/client-ca.pem"
cp scripts/lib/deploy-smoke-client.mjs "$work/client.mjs"
stage client-ca.pem
stage client.mjs

edge_network="${project}_edge"
client() {
  local mode="$1"
  shift
  docker run --rm --read-only --cap-drop ALL --security-opt no-new-privileges \
    --network "$edge_network" "$@" \
    --add-host "${hostname}:${proxy_address}" --add-host "${storage_hostname}:${proxy_address}" \
    -e NODE_EXTRA_CA_CERTS=/smoke/client-ca.pem -e SMOKE_HOSTNAME="$hostname" \
    -e SMOKE_STORAGE_HOSTNAME="$storage_hostname" \
    -e S3_ACCESS_KEY="$s3_access_key" -e S3_SECRET_KEY="$s3_secret_key" \
    -v "$daemon_work/client.mjs:/smoke/client.mjs:ro" \
    -v "$daemon_work/client-ca.pem:/smoke/client-ca.pem:ro" \
    --entrypoint node "$image" /smoke/client.mjs "$mode"
}

step "Checking the API, Dashboard, storage delivery, and client-address chain through the proxy"
client edge || fail "edge checks failed."

step "Checking the tunnel connector's client-address path from its fixed address"
client tunnel --ip "$tunnel_address" || fail "tunnel checks failed."

step "Creating the initial owner with the image's own CLI, then signing in and out through the proxy"
owner_email="[email protected]"
owner_password="$(openssl rand -hex 16)"
printf '%s' "$owner_password" |
  "${compose[@]}" run --rm --no-deps -T migrate node core/dist/cli/create-initial-owner.js \
    --email "$owner_email" >/dev/null || fail "creating the initial owner failed."
echo "ok   the initial owner was created from the image (password on standard input)"
client session --ip "$session_client_address" \
  -e SMOKE_OWNER_EMAIL="$owner_email" -e SMOKE_OWNER_PASSWORD="$owner_password" ||
  fail "session checks failed."

step "Proving the API's database connection is certificate-verified TLS"
api_container="$("${compose[@]}" ps -q api)"
api_data_address="$(
  docker inspect --format "{{(index .NetworkSettings.Networks \"${project}_data\").IPAddress}}" \
    "$api_container"
)"
[[ -n "$api_data_address" ]] || fail "could not find the API's database-network address."
psql_as_superuser() {
  "${compose[@]}" exec -T postgres psql -U ekklesia -d ekklesia -Atc "$1"
}
tls_connections="$(psql_as_superuser "SELECT count(*) FROM pg_stat_ssl s JOIN pg_stat_activity a USING (pid) WHERE a.client_addr = '${api_data_address}' AND s.ssl AND s.version = 'TLSv1.3'")"
plain_connections="$(psql_as_superuser "SELECT count(*) FROM pg_stat_ssl s JOIN pg_stat_activity a USING (pid) WHERE a.client_addr = '${api_data_address}' AND NOT s.ssl")"
((tls_connections > 0)) || fail "the API holds no TLS 1.3 database connection."
((plain_connections == 0)) || fail "the API holds a database connection without TLS."
echo "ok   the API's database connections are TLS 1.3 (${tls_connections}), none plaintext"

# PostgreSQL itself refuses a network connection without TLS.
if "${compose[@]}" exec -T -e PGPASSWORD="$postgres_password" postgres \
  psql "host=postgres user=ekklesia dbname=ekklesia sslmode=disable" -Atc 'SELECT 1' >/dev/null 2>&1; then
  fail "PostgreSQL accepted a plaintext network connection."
fi
echo "ok   PostgreSQL refuses a plaintext network connection"

# The server cannot see whether a client verified it, so verification is proven by refusal: the
# same command fails against a CA that did not issue the certificate, and against a host the
# certificate does not name.
openssl ecparam -name prime256v1 -genkey -noout -out "$work/other-ca.key" 2>/dev/null
openssl req -x509 -new -sha256 -days 1 -key "$work/other-ca.key" -out "$work/other-ca.crt" \
  -subj "/CN=Unrelated CA" 2>/dev/null
# expect_connection_refused <description> <compose run arguments...>: the command must fail, and fail
# at the connection, not on a configuration error that would prove nothing about verification.
expect_connection_refused() {
  local description="$1"
  shift
  local output
  if output="$("${compose[@]}" run --rm --no-deps "$@" 2>&1)"; then
    fail "the database connection accepted ${description}."
  fi
  grep -q 'MIGRATION_CONNECTION_FAILED' <<<"$output" ||
    fail "rejecting ${description} failed for a reason other than the connection."
}
expect_connection_refused "a certificate from a CA it was not given" \
  -e EKKLESIA_DATABASE_CA="$(cat "$work/other-ca.crt")" migrate
echo "ok   the database connection rejects a certificate its CA did not issue"
postgres_address="$(
  docker inspect --format "{{(index .NetworkSettings.Networks \"${project}_data\").IPAddress}}" \
    "$("${compose[@]}" ps -q postgres)"
)"
expect_connection_refused "a certificate for a different host name" \
  -e DATABASE_URL="postgresql://ekklesia:${postgres_password}@${postgres_address}:5432/ekklesia" migrate
echo "ok   the database connection rejects a certificate that does not name the host"
"${compose[@]}" run --rm --no-deps migrate >/dev/null || fail "the control connection failed."
echo "ok   the same connection succeeds with the installation CA and the name postgres"

step "Running Doctor from the image"
doctor_output="$("${compose[@]}" run --rm --no-deps migrate node core/dist/cli/doctor.js 2>&1)" ||
  fail "Doctor reported the installation not ready."
grep -q 'Result: READY' <<<"$doctor_output" || fail "Doctor did not report READY."
grep -q 'TLS required with certificate and hostname verification (explicit private CA supplied)' \
  <<<"$doctor_output" || fail "Doctor did not report verify-full with the private CA."
grep -q '✓ Object storage reachable' <<<"$doctor_output" ||
  fail "Doctor could not reach object storage over HTTPS."
for value in "$postgres_password" "$s3_secret_key" "$identity_secret"; do
  if grep -qF -- "$value" <<<"$doctor_output"; then
    fail "Doctor output contains a secret."
  fi
done
echo "ok   Doctor: READY, verify-full with the private CA, storage reachable, no secret printed"

api_logs="$("${compose[@]}" logs --no-color api 2>&1)"
for value in "$postgres_password" "$s3_secret_key" "$identity_secret" "$owner_password"; do
  if grep -qF -- "$value" <<<"$api_logs"; then
    fail "the API's logs contain a secret."
  fi
done
restarts="$(docker inspect --format '{{.RestartCount}}' "$api_container")"
((restarts == 0)) || fail "the API restarted ${restarts} time(s)."
echo "ok   the API logged no secret and never restarted"

echo "Reference deployment smoke test passed."