diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index d881d12..4f12846 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -124,17 +124,19 @@ jobs: # The platform-only image (ADR 0044): a checkout has no `modules/`, so this proves the image # builds, validates, and assembles with zero modules. Build only — no registry login, no # credentials, no push, no tag anywhere but this runner's local image store. + # Tagged per run and attempt, not only per commit: a pull request and a push of the same commit + # can build at once on one runner's Docker daemon, and neither may remove the other's image. - name: Build the production image without modules run: >- docker build --file deploy/Dockerfile --build-arg REVISION="$GITHUB_SHA" - --tag ekklesia-ci:"$GITHUB_SHA" + --tag ekklesia-ci:"$GITHUB_SHA-${GITHUB_RUN_ID:-0}-${GITHUB_RUN_ATTEMPT:-1}" . - name: Remove the image from the runner if: always() - run: docker image rm --force ekklesia-ci:"$GITHUB_SHA" || true + run: docker image rm --force ekklesia-ci:"$GITHUB_SHA-${GITHUB_RUN_ID:-0}-${GITHUB_RUN_ATTEMPT:-1}" || true deploy-smoke: needs: verify diff --git a/scripts/lib/compose-diagnostics.sh b/scripts/lib/compose-diagnostics.sh new file mode 100644 index 0000000..9ef2353 --- /dev/null +++ b/scripts/lib/compose-diagnostics.sh @@ -0,0 +1,72 @@ +#!/usr/bin/env bash +# Why a disposable Compose project did not become ready: `docker compose ps` and the last log lines +# of every service, so a CI failure says what went wrong (a port already allocated, a container that +# exited, a health check that never passed) instead of only that something did. +# +# source scripts/lib/compose-diagnostics.sh +# COMPOSE_DIAGNOSTICS_SECRETS=("$password" "$secret_key") # literal values to mask, optional +# compose_report_not_ready "$up_output" "${compose[@]}" +# +# Everything printed passes through `compose_diagnostics_redact` first. The projects these scripts +# start hold only disposable, generated test credentials, but output is redacted anyway: every +# literal in COMPOSE_DIAGNOSTICS_SECRETS, the value of any `NAME=value` or `NAME: value` whose name +# looks like a credential, the user-information part of any URL, and the rest of any Authorization +# header. Terminal control sequences are stripped. Output goes to standard error. + +COMPOSE_DIAGNOSTICS_SECRETS=() +COMPOSE_DIAGNOSTICS_TAIL_LINES=40 + +# compose_diagnostics_redact: standard input to standard output, with credentials masked. +compose_diagnostics_redact() { + local secrets="" + if ((${#COMPOSE_DIAGNOSTICS_SECRETS[@]} > 0)); then + secrets="$(printf '%s\n' "${COMPOSE_DIAGNOSTICS_SECRETS[@]}")" + fi + # Literal values first, by plain substring rather than as patterns. Very short values are skipped: + # masking every occurrence of a two-letter word would only make the output unreadable. + COMPOSE_DIAGNOSTICS_LITERALS="$secrets" awk ' + BEGIN { count = split(ENVIRON["COMPOSE_DIAGNOSTICS_LITERALS"], secret, "\n") } + { + line = $0 + for (i = 1; i <= count; i++) { + if (length(secret[i]) < 6) { + continue + } + masked = "" + while ((at = index(line, secret[i])) > 0) { + masked = masked substr(line, 1, at - 1) "[redacted]" + line = substr(line, at + length(secret[i])) + } + line = masked line + } + print line + } + ' | sed -E \ + -e 's/\x1b\[[0-9;?]*[A-Za-z]//g' \ + -e 's#([Aa][Uu][Tt][Hh][Oo][Rr][Ii][Zz][Aa][Tt][Ii][Oo][Nn][[:space:]]*[:=]).*#\1 [redacted]#' \ + -e 's#([A-Za-z0-9_.-]*([Pp][Aa][Ss][Ss][Ww][Oo][Rr][Dd]|[Pp][Aa][Ss][Ss][Ww][Dd]|[Ss][Ee][Cc][Rr][Ee][Tt]|[Tt][Oo][Kk][Ee][Nn]|[_.-][Kk][Ee][Yy])[A-Za-z0-9_.-]*"?[[:space:]]*[=:][[:space:]]*)("[^"]*"|[^[:space:],;"]+)#\1[redacted]#g' \ + -e 's#([A-Za-z][A-Za-z0-9+.-]*://)[^/@[:space:]]+@#\1[redacted]@#g' +} + +# compose_report_not_ready : the failed command's own output, then +# the project's containers and the tail of each service's logs, all redacted. +compose_report_not_ready() { + local up_output="$1" + shift + local service services + { + if [[ -n "$up_output" ]]; then + echo "--- docker compose up (last 20 lines)" + printf '%s\n' "$up_output" | tail -n 20 | compose_diagnostics_redact + fi + echo "--- docker compose ps" + "$@" ps --all 2>&1 | compose_diagnostics_redact + services="$("$@" config --services 2>/dev/null || true)" + for service in $services; do + echo "--- docker compose logs ${service} (last ${COMPOSE_DIAGNOSTICS_TAIL_LINES} lines)" + "$@" logs --no-color --tail "$COMPOSE_DIAGNOSTICS_TAIL_LINES" "$service" 2>&1 | + compose_diagnostics_redact + done + echo "--- end of diagnostics" + } >&2 +} diff --git a/scripts/smoke-deploy.sh b/scripts/smoke-deploy.sh index 7090761..269e37d 100755 --- a/scripts/smoke-deploy.sh +++ b/scripts/smoke-deploy.sh @@ -22,6 +22,9 @@ set -Eeuo pipefail repository_root="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")/.." && pwd)" cd "$repository_root" +# shellcheck source=scripts/lib/compose-diagnostics.sh +source "$repository_root/scripts/lib/compose-diagnostics.sh" + for command in docker openssl awk; do command -v "$command" >/dev/null || { echo "smoke-deploy: $command is required." >&2 @@ -45,15 +48,34 @@ built_image="" hostname="ekklesia.smoke.test" storage_hostname="storage.smoke.test" -# A per-run edge subnet, so concurrent runs and the host's own networks do not collide. Fixed -# addresses come from the upper half, outside Docker's dynamic range, exactly as documented. -octet=$((RANDOM % 200 + 20)) -edge_subnet="10.249.${octet}.0/24" -edge_range="10.249.${octet}.0/25" -proxy_address="10.249.${octet}.254" -tunnel_address="10.249.${octet}.253" -# A test-only client address, also outside the dynamic range, so its limiter bucket is its own. -session_client_address="10.249.${octet}.252" +# choose_edge_network: a per-run edge subnet that no network on this Docker daemon already uses, so +# concurrent runs sharing a runner's daemon do not collide. Two runs can still pick the same free +# subnet at the same moment; the first `up` below detects that and chooses again. Fixed addresses come +# from the upper half, outside Docker's dynamic range, exactly as documented. +choose_edge_network() { + local in_use candidate attempt + in_use="$( + docker network ls --quiet | + xargs -r docker network inspect --format '{{range .IPAM.Config}}{{println .Subnet}}{{end}}' 2>/dev/null | + sed -n 's#^10\.249\.\([0-9]*\)\.0/24$#\1#p' + )" || in_use="" + for attempt in $(seq 1 100); do + candidate=$((RANDOM % 200 + 20)) + if ! grep -qx "$candidate" <<<"$in_use"; then + octet="$candidate" + edge_subnet="10.249.${octet}.0/24" + edge_range="10.249.${octet}.0/25" + proxy_address="10.249.${octet}.254" + tunnel_address="10.249.${octet}.253" + # A test-only client address, also outside the dynamic range, so its limiter bucket is its own. + session_client_address="10.249.${octet}.252" + return 0 + fi + done + echo "smoke-deploy: no free 10.249.x.0/24 edge subnet on this Docker daemon." >&2 + exit 1 +} +choose_edge_network # daemon_work: where the Docker daemon sees this run's mounted files. The same directory as `work` # on a workstation; a staged copy on the Docker host when this script runs in a job container. @@ -154,6 +176,8 @@ postgres_password="$(secret POSTGRES_PASSWORD)" s3_access_key="$(secret S3_ACCESS_KEY)" s3_secret_key="$(secret S3_SECRET_KEY)" identity_secret="$(secret EKKLESIA_IDENTITY_SECRET)" +# Masked in any diagnostics below, along with anything else that looks like a credential. +COMPOSE_DIAGNOSTICS_SECRETS=("$postgres_password" "$s3_access_key" "$s3_secret_key" "$identity_secret") mkdir -p "$work/env" # set_env : replaces an uncommented KEY= line, keeping the value byte-exact. @@ -207,8 +231,28 @@ EKKLESIA_TUNNEL_ADDRESS=${tunnel_address} EOF step "Starting PostgreSQL and object storage" -"${compose[@]}" up --detach --wait --wait-timeout 180 postgres object-storage >/dev/null 2>&1 || +# This `up` creates the edge network. If another run claimed the same subnet since it was chosen, +# Docker refuses with an address-pool overlap: choose another and start again, a bounded number of +# times. Any other failure is reported with the project's containers and logs. +for attempt in 1 2 3 4 5; do + if up_output="$( + "${compose[@]}" up --detach --wait --wait-timeout 180 postgres object-storage 2>&1 + )"; then + break + fi + if [[ "$up_output" == *"overlap"* ]] && ((attempt < 5)); then + echo " edge subnet ${edge_subnet} was taken by a concurrent run; choosing another" + "${compose[@]}" down --volumes --remove-orphans --timeout 10 >/dev/null 2>&1 || true + choose_edge_network + set_env "$work/compose.env" EKKLESIA_EDGE_SUBNET "$edge_subnet" + set_env "$work/compose.env" EKKLESIA_EDGE_DYNAMIC_RANGE "$edge_range" + set_env "$work/compose.env" EKKLESIA_PROXY_ADDRESS "$proxy_address" + set_env "$work/compose.env" EKKLESIA_TUNNEL_ADDRESS "$tunnel_address" + continue + fi + compose_report_not_ready "$up_output" "${compose[@]}" fail "PostgreSQL and object storage did not become healthy." +done step "Planning, then explicitly applying, migrations against this run's fresh database" "${compose[@]}" run --rm --no-deps migrate >/dev/null || fail "migrate (plan) failed." @@ -216,8 +260,10 @@ step "Planning, then explicitly applying, migrations against this run's fresh da fail "migrate (apply) failed." step "Starting the API and the proxy" -"${compose[@]}" up --detach --wait --wait-timeout 180 >/dev/null 2>&1 || +if ! up_output="$("${compose[@]}" up --detach --wait --wait-timeout 180 2>&1)"; then + compose_report_not_ready "$up_output" "${compose[@]}" fail "the API and proxy did not become healthy." +fi # Caddy creates its internal CA on first start. The test clients trust it, plus the installation CA # for their direct connection to object-storage. diff --git a/scripts/smoke-infrastructure.sh b/scripts/smoke-infrastructure.sh index 4a83a26..8cbcebe 100755 --- a/scripts/smoke-infrastructure.sh +++ b/scripts/smoke-infrastructure.sh @@ -7,6 +7,8 @@ cd "$repository_root" # shellcheck source=scripts/lib/compose-job-network.sh source "$repository_root/scripts/lib/compose-job-network.sh" +# shellcheck source=scripts/lib/compose-diagnostics.sh +source "$repository_root/scripts/lib/compose-diagnostics.sh" command -v docker >/dev/null || { echo "Docker is required for the infrastructure smoke test." >&2 @@ -28,7 +30,9 @@ if [[ "$curl_help" != *"--aws-sigv4"* ]]; then exit 1 fi -raw_project="ekklesia-smoke-${GITHUB_RUN_ID:-local}-${GITHUB_RUN_ATTEMPT:-0}-$$" +# Unique per invocation, not only per run: several runs can share a runner's Docker daemon at once, +# and the shell's PID is often identical across job containers. +raw_project="ekklesia-smoke-${GITHUB_RUN_ID:-local}-${GITHUB_RUN_ATTEMPT:-0}-$$-${RANDOM}${RANDOM}" project="${raw_project//[^a-zA-Z0-9_-]/-}" project="${project,,}" temporary_directory="$(mktemp -d)" @@ -46,6 +50,8 @@ export S3_SECRET_KEY="ekklesia-smoke-secret-key" export S3_BUCKET="ekklesia-smoke" export S3_REGION="us-east-1" export S3_PORT="${EKKLESIA_SMOKE_S3_PORT:-0}" +COMPOSE_DIAGNOSTICS_SECRETS=("$POSTGRES_PASSWORD" "$S3_ACCESS_KEY" "$S3_SECRET_KEY") +up_output="" compose=( docker compose @@ -63,7 +69,7 @@ cleanup() { local cleanup_output trap - EXIT INT TERM if ((exit_status != 0)); then - "${compose[@]}" logs --no-color >&2 || true + compose_report_not_ready "$up_output" "${compose[@]}" || true fi if ! compose_job_network_disconnect; then echo "Infrastructure smoke network cleanup failed." >&2 @@ -88,8 +94,15 @@ trap 'exit 130' INT trap 'exit 143' TERM "${compose[@]}" config --quiet -"${compose[@]}" up --detach --wait --wait-timeout 120 -"${compose[@]}" up --detach --wait --wait-timeout 120 +# Started twice: the second `up` proves the project is idempotent. A failure's own output is kept for +# the cleanup's diagnostics, which also print every container and the tail of its logs. +for _ in 1 2; do + if ! up_output="$("${compose[@]}" up --detach --wait --wait-timeout 120 2>&1)"; then + echo "Disposable PostgreSQL and object storage did not become ready." >&2 + exit 1 + fi +done +up_output="" database_result="$( "${compose[@]}" exec -T \ diff --git a/scripts/test-migrations-integration.sh b/scripts/test-migrations-integration.sh index a476323..29d4dd1 100755 --- a/scripts/test-migrations-integration.sh +++ b/scripts/test-migrations-integration.sh @@ -7,6 +7,8 @@ cd "$repository_root" # shellcheck source=scripts/lib/compose-job-network.sh source "$repository_root/scripts/lib/compose-job-network.sh" +# shellcheck source=scripts/lib/compose-diagnostics.sh +source "$repository_root/scripts/lib/compose-diagnostics.sh" command -v docker >/dev/null || { echo "Docker is required for the PostgreSQL migration integration suite." >&2 @@ -25,14 +27,20 @@ if ! compose_job_network_initialize; then exit 1 fi -raw_project="ekklesia-migration-${GITHUB_RUN_ID:-local}-${GITHUB_RUN_ATTEMPT:-0}-${GITHUB_JOB:-job}-$$" +# Unique per invocation, not only per run: several runs (and several jobs of one run) can share a +# runner's Docker daemon at once, and the shell's PID is often identical across job containers. +raw_project="ekklesia-migration-${GITHUB_RUN_ID:-local}-${GITHUB_RUN_ATTEMPT:-0}-${GITHUB_JOB:-job}-$$-${RANDOM}${RANDOM}" project="${raw_project//[^a-zA-Z0-9_-]/-}" project="${project,,}" export POSTGRES_DB="ekklesia_migration_test" export POSTGRES_USER="ekklesia_migration_test" export POSTGRES_PASSWORD="temporary-$project-${RANDOM}-${RANDOM}" +# Both host ports are ephemeral: concurrent runs on one Docker daemon must never contend for a +# fixed one. `.env.example` pins the storage port for local development, so it is overridden here +# exactly like the database's. The bound ports are discovered below. export POSTGRES_PORT="0" +export S3_PORT="0" compose=( docker compose @@ -72,8 +80,17 @@ fi # and CI already knows how to provision. SeaweedFS is the intended deployment target (ADR 0004), and # proving the storage contract against a mock instead would have hidden the provider quirks this # suite exists to catch. -if ! "${compose[@]}" up --detach --wait --wait-timeout 180 postgres object-storage >/dev/null 2>&1; then +# Masked in any diagnostics below, along with anything else that looks like a credential. +COMPOSE_DIAGNOSTICS_SECRETS=( + "$POSTGRES_PASSWORD" + "$(grep -E '^S3_ACCESS_KEY=' .env.example | cut -d= -f2-)" + "$(grep -E '^S3_SECRET_KEY=' .env.example | cut -d= -f2-)" +) +if ! up_output="$( + "${compose[@]}" up --detach --wait --wait-timeout 180 postgres object-storage 2>&1 +)"; then echo "Disposable PostgreSQL and object storage did not become ready." >&2 + compose_report_not_ready "$up_output" "${compose[@]}" exit 1 fi diff --git a/tests/ci-scripts.test.ts b/tests/ci-scripts.test.ts new file mode 100644 index 0000000..7203ada --- /dev/null +++ b/tests/ci-scripts.test.ts @@ -0,0 +1,165 @@ +import { execFile } from "node:child_process"; +import { readFile } from "node:fs/promises"; +import path from "node:path"; +import { fileURLToPath } from "node:url"; +import { promisify } from "node:util"; +import { describe, expect, it } from "vitest"; + +/** + * The scripts CI runs against a shared Docker daemon. Several runs, and several jobs of one run, can + * be active on one runner at once (and the inventoried Gitea ignores workflow `concurrency`), so + * nothing they start may claim a fixed host port, a fixed project or network name, or a fixed + * subnet. And when a disposable project does not become ready, the failure must say why without + * printing a credential. + */ + +const repositoryRoot = path.resolve(path.dirname(fileURLToPath(import.meta.url)), ".."); +const execFileAsync = promisify(execFile); + +const ciScripts = [ + "scripts/test-migrations-integration.sh", + "scripts/smoke-infrastructure.sh", + "scripts/smoke-deploy.sh", +]; + +async function source(relativePath: string): Promise { + return readFile(path.join(repositoryRoot, relativePath), "utf8"); +} + +/** Runs the real helper under bash with the given literal secrets and input. */ +async function redact(input: string, secrets: readonly string[] = []): Promise { + const helper = path.join(repositoryRoot, "scripts", "lib", "compose-diagnostics.sh"); + const script = `source "$1"; shift; COMPOSE_DIAGNOSTICS_SECRETS=("$@"); printf '%s' "$INPUT" | compose_diagnostics_redact`; + const { stdout } = await execFileAsync("bash", ["-c", script, "redact", helper, ...secrets], { + env: { ...process.env, INPUT: input }, + }); + return stdout; +} + +describe("CI scripts on a shared Docker daemon", () => { + it("publish every host port ephemerally, never a fixed one", async () => { + const migrations = await source("scripts/test-migrations-integration.sh"); + // `.env.example` pins both ports for local development; the suite must override both. + expect(migrations).toContain('export POSTGRES_PORT="0"'); + expect(migrations).toContain('export S3_PORT="0"'); + + const infrastructure = await source("scripts/smoke-infrastructure.sh"); + expect(infrastructure).toContain('export POSTGRES_PORT="${EKKLESIA_SMOKE_POSTGRES_PORT:-0}"'); + expect(infrastructure).toContain('export S3_PORT="${EKKLESIA_SMOKE_S3_PORT:-0}"'); + + const deploy = await source("scripts/smoke-deploy.sh"); + expect(deploy).toContain("EKKLESIA_HTTP_PORT=0"); + expect(deploy).toContain("EKKLESIA_HTTPS_PORT=0"); + + for (const relativePath of ciScripts) { + const text = await source(relativePath); + // No `-p 8333:…`, `127.0.0.1:5432:…`, or `PORT=` anywhere. + expect(text, relativePath).not.toMatch(/\b(?:-p|--publish)\s+[^\s]*\d{2,5}:\d/u); + expect(text, relativePath).not.toMatch(/\b[A-Z_]*PORT="?[1-9]\d*"?\s*$/mu); + } + }); + + it("give every Compose project a name unique to the invocation, not only the run", async () => { + expect(await source("scripts/test-migrations-integration.sh")).toMatch( + /raw_project="ekklesia-migration-[^"\n]*\$\$-\$\{RANDOM\}\$\{RANDOM\}"/u, + ); + expect(await source("scripts/smoke-infrastructure.sh")).toMatch( + /raw_project="ekklesia-smoke-[^"\n]*\$\$-\$\{RANDOM\}\$\{RANDOM\}"/u, + ); + const deploy = await source("scripts/smoke-deploy.sh"); + expect(deploy).toContain('run_id="$(openssl rand -hex 4)"'); + expect(deploy).toContain('project="ekklesia-deploy-smoke-${run_id}"'); + for (const relativePath of ciScripts) { + expect(await source(relativePath), relativePath).toContain('--project-name "$project"'); + } + }); + + it("choose the deploy smoke's edge subnet from those not in use, and choose again on a race", async () => { + const deploy = await source("scripts/smoke-deploy.sh"); + expect(deploy).not.toMatch(/^octet=\$\(\(RANDOM/mu); + expect(deploy).toContain("choose_edge_network() {"); + expect(deploy).toContain("docker network inspect --format '{{range .IPAM.Config}}"); + expect(deploy).toContain('[[ "$up_output" == *"overlap"* ]]'); + }); + + it("tag the CI image per run and attempt, so concurrent runs of one commit cannot remove it", async () => { + const workflow = await source(".github/workflows/ci.yml"); + expect(workflow).toContain( + 'ekklesia-ci:"$GITHUB_SHA-${GITHUB_RUN_ID:-0}-${GITHUB_RUN_ATTEMPT:-1}"', + ); + expect(workflow).not.toContain('ekklesia-ci:"$GITHUB_SHA"'); + }); + + it("report every readiness failure with the project's containers and logs, through the redactor only", async () => { + for (const relativePath of ciScripts) { + const text = await source(relativePath); + expect(text, relativePath).toContain( + 'source "$repository_root/scripts/lib/compose-diagnostics.sh"', + ); + expect(text, relativePath).toContain("compose_report_not_ready"); + expect(text, relativePath).toContain("COMPOSE_DIAGNOSTICS_SECRETS=("); + // Compose output is never printed raw: logs are only ever captured (the deploy smoke scans + // the API's for secrets), and a readiness failure's own output is never discarded. + for (const line of text + .split("\n") + .filter((candidate) => candidate.includes('compose[@]}" logs'))) { + expect(line, relativePath).toMatch(/="\$\("\$\{compose\[@\]\}" logs /u); + } + expect(text, relativePath).not.toMatch(/up --detach --wait[^\n]*>\/dev\/null 2>&1/u); + } + }); +}); + +describe("compose diagnostics redaction", () => { + it("masks credential-looking values, URL user information, and Authorization headers", async () => { + const output = await redact( + [ + "POSTGRES_PASSWORD=hunter2hunter2", + ' "AWS_SECRET_ACCESS_KEY": "abc/def+ghi"', + "S3_ACCESS_KEY=AKIAEXAMPLE other=fine", + "api_key: xyz123", + "connecting to postgresql://ekklesia:s3cr3t@postgres:5432/ekklesia", + "Authorization: Bearer eyJhbGciOi.payload.sig", + "EKKLESIA_IDENTITY_SECRET=0123456789abcdef", + ].join("\n"), + ); + + for (const secret of [ + "hunter2hunter2", + "abc/def+ghi", + "AKIAEXAMPLE", + "xyz123", + "s3cr3t", + "eyJhbGciOi", + "0123456789abcdef", + ]) { + expect(output).not.toContain(secret); + } + expect(output).toContain("POSTGRES_PASSWORD=[redacted]"); + expect(output).toContain("other=fine"); + expect(output).toContain("postgresql://[redacted]@postgres:5432/ekklesia"); + }); + + it("masks every occurrence of a supplied literal, wherever it appears", async () => { + const output = await redact("started with temporary-pw-12345 and again temporary-pw-12345", [ + "temporary-pw-12345", + ]); + expect(output).toBe("started with [redacted] and again [redacted]\n"); + }); + + it("keeps the diagnosis itself readable", async () => { + const diagnosis = + "Error response from daemon: Bind for 127.0.0.1:8333 failed: port is already allocated"; + expect(await redact(diagnosis)).toBe(`${diagnosis}\n`); + expect(await redact("duplicate key value violates unique constraint")).toBe( + "duplicate key value violates unique constraint\n", + ); + expect(await redact('password authentication failed for user "ekklesia"')).toBe( + 'password authentication failed for user "ekklesia"\n', + ); + }); + + it("strips terminal control sequences from container logs", async () => { + expect(await redact("\u001b[2Kpostgres-1 | ready")).toBe("postgres-1 | ready\n"); + }); +});