Ekklesia/scripts/smoke-infrastructure.sh

#!/usr/bin/env bash

set -Eeuo pipefail

repository_root="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")/.." && pwd)"
cd "$repository_root"

# shellcheck source=scripts/lib/compose-job-network.sh
source "$repository_root/scripts/lib/compose-job-network.sh"
# shellcheck source=scripts/lib/compose-diagnostics.sh
source "$repository_root/scripts/lib/compose-diagnostics.sh"
# shellcheck source=scripts/lib/ci-project.sh
source "$repository_root/scripts/lib/ci-project.sh"

command -v docker >/dev/null || {
  echo "Docker is required for the infrastructure smoke test." >&2
  exit 1
}
command -v curl >/dev/null || {
  echo "curl is required for the infrastructure smoke test." >&2
  exit 1
}
command -v cmp >/dev/null || {
  echo "cmp is required for the infrastructure smoke test." >&2
  exit 1
}
docker compose version >/dev/null

curl_help="$(curl --help all)"
if [[ "$curl_help" != *"--aws-sigv4"* ]]; then
  echo "curl must support --aws-sigv4 for authenticated S3 checks." >&2
  exit 1
fi

# Unique per invocation, and findable by run and job if this script is killed before its trap runs
# (scripts/lib/ci-project.sh, scripts/ci-sweep.sh).
project="$(ci_project_name infrastructure)"
temporary_directory="$(mktemp -d)"
payload_file="$temporary_directory/payload.txt"
download_file="$temporary_directory/download.txt"
bucket_list_file="$temporary_directory/buckets.xml"
: >"$bucket_list_file"

export POSTGRES_DB="ekklesia_smoke"
export POSTGRES_USER="ekklesia_smoke"
export POSTGRES_PASSWORD="ekklesia-smoke-postgres-password"
export POSTGRES_PORT="${EKKLESIA_SMOKE_POSTGRES_PORT:-0}"
export S3_ACCESS_KEY="ekklesia-smoke-access"
export S3_SECRET_KEY="ekklesia-smoke-secret-key"
export S3_BUCKET="ekklesia-smoke"
export S3_REGION="us-east-1"
export S3_PORT="${EKKLESIA_SMOKE_S3_PORT:-0}"
COMPOSE_DIAGNOSTICS_SECRETS=("$POSTGRES_PASSWORD" "$S3_ACCESS_KEY" "$S3_SECRET_KEY")
up_output=""

compose=(
  docker compose
  --project-name "$project"
  --env-file .env.example
  --file infrastructure/compose.yaml
)
if ! compose_job_network_initialize; then
  echo "The current Docker job container could not be identified." >&2
  exit 1
fi

cleanup() {
  local exit_status=$?
  local cleanup_output
  trap - EXIT INT TERM
  if ((exit_status != 0)); then
    compose_report_not_ready "$up_output" "${compose[@]}" || true
  fi
  if ! compose_job_network_disconnect; then
    echo "Infrastructure smoke network cleanup failed." >&2
    if ((exit_status == 0)); then
      exit_status=1
    fi
  fi
  if ! cleanup_output="$(
    "${compose[@]}" down --volumes --remove-orphans --timeout 10 2>&1
  )"; then
    echo "Infrastructure smoke cleanup failed:" >&2
    echo "$cleanup_output" >&2
    if ((exit_status == 0)); then
      exit_status=1
    fi
  fi
  rm -rf -- "$temporary_directory"
  exit "$exit_status"
}
trap cleanup EXIT
trap 'exit 130' INT
trap 'exit 143' TERM

"${compose[@]}" config --quiet
# Started twice: the second `up` proves the project is idempotent. A failure's own output is kept for
# the cleanup's diagnostics, which also print every container and the tail of its logs.
for _ in 1 2; do
  if ! up_output="$("${compose[@]}" up --detach --wait --wait-timeout 120 2>&1)"; then
    echo "Disposable PostgreSQL and object storage did not become ready." >&2
    exit 1
  fi
done
up_output=""

database_result="$(
  "${compose[@]}" exec -T \
    -e "PGPASSWORD=$POSTGRES_PASSWORD" \
    postgres \
    psql \
    --no-psqlrc \
    --host 127.0.0.1 \
    --username "$POSTGRES_USER" \
    --dbname "$POSTGRES_DB" \
    --tuples-only \
    --no-align \
    --set ON_ERROR_STOP=1 \
    --command "SELECT current_database(), current_user"
)"
if [[ "$database_result" != "$POSTGRES_DB|$POSTGRES_USER" ]]; then
  echo "PostgreSQL returned an unexpected identity: $database_result" >&2
  exit 1
fi

if "${compose[@]}" exec -T \
  -e "PGPASSWORD=incorrect-password" \
  postgres \
  psql \
  --no-psqlrc \
  --host 127.0.0.1 \
  --username "$POSTGRES_USER" \
  --dbname "$POSTGRES_DB" \
  --command "SELECT 1" >/dev/null 2>&1; then
  echo "PostgreSQL unexpectedly accepted an invalid password." >&2
  exit 1
fi

if [[ "$COMPOSE_JOB_NETWORK_RUNNING_IN_CONTAINER" == "true" ]]; then
  if ! compose_job_network_connect object-storage "${compose[@]}"; then
    echo "The infrastructure smoke job could not join the isolated storage network." >&2
    exit 1
  fi
  s3_endpoint="http://object-storage:8333"
else
  s3_binding="$("${compose[@]}" port object-storage 8333)"
  s3_port="${s3_binding##*:}"
  if [[ ! "$s3_port" =~ ^[0-9]+$ ]]; then
    echo "Compose returned an invalid S3 port binding: $s3_binding" >&2
    exit 1
  fi
  s3_endpoint="http://127.0.0.1:$s3_port"
fi
signed_curl=(
  curl
  --silent
  --show-error
  --aws-sigv4 "aws:amz:$S3_REGION:s3"
  --user "$S3_ACCESS_KEY:$S3_SECRET_KEY"
)

s3_ready=false
for _ in {1..30}; do
  status="$(
    "${signed_curl[@]}" \
      --output "$bucket_list_file" \
      --write-out "%{http_code}" \
      "$s3_endpoint/" || true
  )"
  bucket_list="$(<"$bucket_list_file")"
  if [[ "$status" == "200" && "$bucket_list" == *"<Name>$S3_BUCKET</Name>"* ]]; then
    s3_ready=true
    break
  fi
  sleep 2
done
if [[ "$s3_ready" != "true" ]]; then
  echo "The authenticated S3 endpoint and bucket did not become ready." >&2
  exit 1
fi

printf '%s' "ekklesia-storage-smoke" >"$payload_file"
object_key="smoke/$project/object.txt"
object_url="$s3_endpoint/$S3_BUCKET/$object_key"

put_status="$(
  "${signed_curl[@]}" \
    --upload-file "$payload_file" \
    --header "Content-Type: text/plain" \
    --output /dev/null \
    --write-out "%{http_code}" \
    "$object_url"
)"
if [[ "$put_status" != "200" ]]; then
  echo "S3 PUT returned $put_status instead of 200." >&2
  exit 1
fi

get_status="$(
  "${signed_curl[@]}" \
    --output "$download_file" \
    --write-out "%{http_code}" \
    "$object_url"
)"
if [[ "$get_status" != "200" ]] || ! cmp --silent "$payload_file" "$download_file"; then
  echo "S3 GET did not return the uploaded object unchanged." >&2
  exit 1
fi

head_status="$(
  "${signed_curl[@]}" --head --output /dev/null --write-out "%{http_code}" "$object_url"
)"
if [[ "$head_status" != "200" ]]; then
  echo "S3 HEAD returned $head_status instead of 200." >&2
  exit 1
fi

anonymous_status="$(
  curl --silent --show-error --output /dev/null --write-out "%{http_code}" "$object_url"
)"
if [[ "$anonymous_status" != "403" ]]; then
  echo "S3 anonymous GET returned $anonymous_status instead of 403." >&2
  exit 1
fi

bad_signature_status="$(
  curl \
    --silent \
    --show-error \
    --aws-sigv4 "aws:amz:$S3_REGION:s3" \
    --user "$S3_ACCESS_KEY:incorrect-secret" \
    --output /dev/null \
    --write-out "%{http_code}" \
    "$object_url"
)"
if [[ "$bad_signature_status" != "403" ]]; then
  echo "S3 invalid signature returned $bad_signature_status instead of 403." >&2
  exit 1
fi

delete_status="$(
  "${signed_curl[@]}" \
    --request DELETE \
    --output /dev/null \
    --write-out "%{http_code}" \
    "$object_url"
)"
if [[ "$delete_status" != "204" ]]; then
  echo "S3 DELETE returned $delete_status instead of 204." >&2
  exit 1
fi

missing_status="$(
  "${signed_curl[@]}" --head --output /dev/null --write-out "%{http_code}" "$object_url"
)"
if [[ "$missing_status" != "404" ]]; then
  echo "Deleted S3 object returned $missing_status instead of 404." >&2
  exit 1
fi

echo "Infrastructure smoke test passed for PostgreSQL and authenticated S3 operations."