Merge pull request 'fix(ci): isolate concurrent runs on a shared Docker daemon and explain readiness failures' (#52) from fix/ci-concurrent-runs into main

Reviewed-on: http://git.riverofglorymn.org/River_of_Glory_Church/Ekklesia/pulls/52
Reviewed-by: The_River <[email protected]>

.github/workflows/ci.yml

@@ -124,17 +124,19 @@ jobs:
       # The platform-only image (ADR 0044): a checkout has no `modules/`, so this proves the image
       # builds, validates, and assembles with zero modules. Build only — no registry login, no
       # credentials, no push, no tag anywhere but this runner's local image store.
+      # Tagged per run and attempt, not only per commit: a pull request and a push of the same commit
+      # can build at once on one runner's Docker daemon, and neither may remove the other's image.
       - name: Build the production image without modules
         run: >-
           docker build
           --file deploy/Dockerfile
           --build-arg REVISION="$GITHUB_SHA"
-          --tag ekklesia-ci:"$GITHUB_SHA"
+          --tag ekklesia-ci:"$GITHUB_SHA-${GITHUB_RUN_ID:-0}-${GITHUB_RUN_ATTEMPT:-1}"
           .
 
       - name: Remove the image from the runner
         if: always()
-        run: docker image rm --force ekklesia-ci:"$GITHUB_SHA" || true
+        run: docker image rm --force ekklesia-ci:"$GITHUB_SHA-${GITHUB_RUN_ID:-0}-${GITHUB_RUN_ATTEMPT:-1}" || true
 
   deploy-smoke:
     needs: verify

scripts/lib/compose-diagnostics.sh

@@ -0,0 +1,72 @@
+#!/usr/bin/env bash
+# Why a disposable Compose project did not become ready: `docker compose ps` and the last log lines
+# of every service, so a CI failure says what went wrong (a port already allocated, a container that
+# exited, a health check that never passed) instead of only that something did.
+#
+#   source scripts/lib/compose-diagnostics.sh
+#   COMPOSE_DIAGNOSTICS_SECRETS=("$password" "$secret_key")   # literal values to mask, optional
+#   compose_report_not_ready "$up_output" "${compose[@]}"
+#
+# Everything printed passes through `compose_diagnostics_redact` first. The projects these scripts
+# start hold only disposable, generated test credentials, but output is redacted anyway: every
+# literal in COMPOSE_DIAGNOSTICS_SECRETS, the value of any `NAME=value` or `NAME: value` whose name
+# looks like a credential, the user-information part of any URL, and the rest of any Authorization
+# header. Terminal control sequences are stripped. Output goes to standard error.
+
+COMPOSE_DIAGNOSTICS_SECRETS=()
+COMPOSE_DIAGNOSTICS_TAIL_LINES=40
+
+# compose_diagnostics_redact: standard input to standard output, with credentials masked.
+compose_diagnostics_redact() {
+  local secrets=""
+  if ((${#COMPOSE_DIAGNOSTICS_SECRETS[@]} > 0)); then
+    secrets="$(printf '%s\n' "${COMPOSE_DIAGNOSTICS_SECRETS[@]}")"
+  fi
+  # Literal values first, by plain substring rather than as patterns. Very short values are skipped:
+  # masking every occurrence of a two-letter word would only make the output unreadable.
+  COMPOSE_DIAGNOSTICS_LITERALS="$secrets" awk '
+    BEGIN { count = split(ENVIRON["COMPOSE_DIAGNOSTICS_LITERALS"], secret, "\n") }
+    {
+      line = $0
+      for (i = 1; i <= count; i++) {
+        if (length(secret[i]) < 6) {
+          continue
+        }
+        masked = ""
+        while ((at = index(line, secret[i])) > 0) {
+          masked = masked substr(line, 1, at - 1) "[redacted]"
+          line = substr(line, at + length(secret[i]))
+        }
+        line = masked line
+      }
+      print line
+    }
+  ' | sed -E \
+    -e 's/\x1b\[[0-9;?]*[A-Za-z]//g' \
+    -e 's#([Aa][Uu][Tt][Hh][Oo][Rr][Ii][Zz][Aa][Tt][Ii][Oo][Nn][[:space:]]*[:=]).*#\1 [redacted]#' \
+    -e 's#([A-Za-z0-9_.-]*([Pp][Aa][Ss][Ss][Ww][Oo][Rr][Dd]|[Pp][Aa][Ss][Ss][Ww][Dd]|[Ss][Ee][Cc][Rr][Ee][Tt]|[Tt][Oo][Kk][Ee][Nn]|[_.-][Kk][Ee][Yy])[A-Za-z0-9_.-]*"?[[:space:]]*[=:][[:space:]]*)("[^"]*"|[^[:space:],;"]+)#\1[redacted]#g' \
+    -e 's#([A-Za-z][A-Za-z0-9+.-]*://)[^/@[:space:]]+@#\1[redacted]@#g'
+}
+
+# compose_report_not_ready <up output> <compose command...>: the failed command's own output, then
+# the project's containers and the tail of each service's logs, all redacted.
+compose_report_not_ready() {
+  local up_output="$1"
+  shift
+  local service services
+  {
+    if [[ -n "$up_output" ]]; then
+      echo "--- docker compose up (last 20 lines)"
+      printf '%s\n' "$up_output" | tail -n 20 | compose_diagnostics_redact
+    fi
+    echo "--- docker compose ps"
+    "$@" ps --all 2>&1 | compose_diagnostics_redact
+    services="$("$@" config --services 2>/dev/null || true)"
+    for service in $services; do
+      echo "--- docker compose logs ${service} (last ${COMPOSE_DIAGNOSTICS_TAIL_LINES} lines)"
+      "$@" logs --no-color --tail "$COMPOSE_DIAGNOSTICS_TAIL_LINES" "$service" 2>&1 |
+        compose_diagnostics_redact
+    done
+    echo "--- end of diagnostics"
+  } >&2
+}

scripts/smoke-deploy.sh

@@ -22,6 +22,9 @@ set -Eeuo pipefail
 repository_root="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")/.." && pwd)"
 cd "$repository_root"
 
+# shellcheck source=scripts/lib/compose-diagnostics.sh
+source "$repository_root/scripts/lib/compose-diagnostics.sh"
+
 for command in docker openssl awk; do
   command -v "$command" >/dev/null || {
     echo "smoke-deploy: $command is required." >&2
@@ -45,15 +48,34 @@ built_image=""
 hostname="ekklesia.smoke.test"
 storage_hostname="storage.smoke.test"
 
-# A per-run edge subnet, so concurrent runs and the host's own networks do not collide. Fixed
-# addresses come from the upper half, outside Docker's dynamic range, exactly as documented.
-octet=$((RANDOM % 200 + 20))
+# choose_edge_network: a per-run edge subnet that no network on this Docker daemon already uses, so
+# concurrent runs sharing a runner's daemon do not collide. Two runs can still pick the same free
+# subnet at the same moment; the first `up` below detects that and chooses again. Fixed addresses come
+# from the upper half, outside Docker's dynamic range, exactly as documented.
+choose_edge_network() {
+  local in_use candidate attempt
+  in_use="$(
+    docker network ls --quiet |
+      xargs -r docker network inspect --format '{{range .IPAM.Config}}{{println .Subnet}}{{end}}' 2>/dev/null |
+      sed -n 's#^10\.249\.\([0-9]*\)\.0/24$#\1#p'
+  )" || in_use=""
+  for attempt in $(seq 1 100); do
+    candidate=$((RANDOM % 200 + 20))
+    if ! grep -qx "$candidate" <<<"$in_use"; then
+      octet="$candidate"
       edge_subnet="10.249.${octet}.0/24"
       edge_range="10.249.${octet}.0/25"
       proxy_address="10.249.${octet}.254"
       tunnel_address="10.249.${octet}.253"
       # A test-only client address, also outside the dynamic range, so its limiter bucket is its own.
       session_client_address="10.249.${octet}.252"
+      return 0
+    fi
+  done
+  echo "smoke-deploy: no free 10.249.x.0/24 edge subnet on this Docker daemon." >&2
+  exit 1
+}
+choose_edge_network
 
 # daemon_work: where the Docker daemon sees this run's mounted files. The same directory as `work`
 # on a workstation; a staged copy on the Docker host when this script runs in a job container.
@@ -154,6 +176,8 @@ postgres_password="$(secret POSTGRES_PASSWORD)"
 s3_access_key="$(secret S3_ACCESS_KEY)"
 s3_secret_key="$(secret S3_SECRET_KEY)"
 identity_secret="$(secret EKKLESIA_IDENTITY_SECRET)"
+# Masked in any diagnostics below, along with anything else that looks like a credential.
+COMPOSE_DIAGNOSTICS_SECRETS=("$postgres_password" "$s3_access_key" "$s3_secret_key" "$identity_secret")
 
 mkdir -p "$work/env"
 # set_env <file> <key> <value>: replaces an uncommented KEY= line, keeping the value byte-exact.
@@ -207,8 +231,28 @@ EKKLESIA_TUNNEL_ADDRESS=${tunnel_address}
 EOF
 
 step "Starting PostgreSQL and object storage"
-"${compose[@]}" up --detach --wait --wait-timeout 180 postgres object-storage >/dev/null 2>&1 ||
+# This `up` creates the edge network. If another run claimed the same subnet since it was chosen,
+# Docker refuses with an address-pool overlap: choose another and start again, a bounded number of
+# times. Any other failure is reported with the project's containers and logs.
+for attempt in 1 2 3 4 5; do
+  if up_output="$(
+    "${compose[@]}" up --detach --wait --wait-timeout 180 postgres object-storage 2>&1
+  )"; then
+    break
+  fi
+  if [[ "$up_output" == *"overlap"* ]] && ((attempt < 5)); then
+    echo "     edge subnet ${edge_subnet} was taken by a concurrent run; choosing another"
+    "${compose[@]}" down --volumes --remove-orphans --timeout 10 >/dev/null 2>&1 || true
+    choose_edge_network
+    set_env "$work/compose.env" EKKLESIA_EDGE_SUBNET "$edge_subnet"
+    set_env "$work/compose.env" EKKLESIA_EDGE_DYNAMIC_RANGE "$edge_range"
+    set_env "$work/compose.env" EKKLESIA_PROXY_ADDRESS "$proxy_address"
+    set_env "$work/compose.env" EKKLESIA_TUNNEL_ADDRESS "$tunnel_address"
+    continue
+  fi
+  compose_report_not_ready "$up_output" "${compose[@]}"
   fail "PostgreSQL and object storage did not become healthy."
+done
 
 step "Planning, then explicitly applying, migrations against this run's fresh database"
 "${compose[@]}" run --rm --no-deps migrate >/dev/null || fail "migrate (plan) failed."
@@ -216,8 +260,10 @@ step "Planning, then explicitly applying, migrations against this run's fresh da
   fail "migrate (apply) failed."
 
 step "Starting the API and the proxy"
-"${compose[@]}" up --detach --wait --wait-timeout 180 >/dev/null 2>&1 ||
+if ! up_output="$("${compose[@]}" up --detach --wait --wait-timeout 180 2>&1)"; then
+  compose_report_not_ready "$up_output" "${compose[@]}"
   fail "the API and proxy did not become healthy."
+fi
 
 # Caddy creates its internal CA on first start. The test clients trust it, plus the installation CA
 # for their direct connection to object-storage.

scripts/smoke-infrastructure.sh

@@ -7,6 +7,8 @@ cd "$repository_root"
 
 # shellcheck source=scripts/lib/compose-job-network.sh
 source "$repository_root/scripts/lib/compose-job-network.sh"
+# shellcheck source=scripts/lib/compose-diagnostics.sh
+source "$repository_root/scripts/lib/compose-diagnostics.sh"
 
 command -v docker >/dev/null || {
   echo "Docker is required for the infrastructure smoke test." >&2
@@ -28,7 +30,9 @@ if [[ "$curl_help" != *"--aws-sigv4"* ]]; then
   exit 1
 fi
 
-raw_project="ekklesia-smoke-${GITHUB_RUN_ID:-local}-${GITHUB_RUN_ATTEMPT:-0}-$$"
+# Unique per invocation, not only per run: several runs can share a runner's Docker daemon at once,
+# and the shell's PID is often identical across job containers.
+raw_project="ekklesia-smoke-${GITHUB_RUN_ID:-local}-${GITHUB_RUN_ATTEMPT:-0}-$$-${RANDOM}${RANDOM}"
 project="${raw_project//[^a-zA-Z0-9_-]/-}"
 project="${project,,}"
 temporary_directory="$(mktemp -d)"
@@ -46,6 +50,8 @@ export S3_SECRET_KEY="ekklesia-smoke-secret-key"
 export S3_BUCKET="ekklesia-smoke"
 export S3_REGION="us-east-1"
 export S3_PORT="${EKKLESIA_SMOKE_S3_PORT:-0}"
+COMPOSE_DIAGNOSTICS_SECRETS=("$POSTGRES_PASSWORD" "$S3_ACCESS_KEY" "$S3_SECRET_KEY")
+up_output=""
 
 compose=(
   docker compose
@@ -63,7 +69,7 @@ cleanup() {
   local cleanup_output
   trap - EXIT INT TERM
   if ((exit_status != 0)); then
-    "${compose[@]}" logs --no-color >&2 || true
+    compose_report_not_ready "$up_output" "${compose[@]}" || true
   fi
   if ! compose_job_network_disconnect; then
     echo "Infrastructure smoke network cleanup failed." >&2
@@ -88,8 +94,15 @@ trap 'exit 130' INT
 trap 'exit 143' TERM
 
 "${compose[@]}" config --quiet
-"${compose[@]}" up --detach --wait --wait-timeout 120
-"${compose[@]}" up --detach --wait --wait-timeout 120
+# Started twice: the second `up` proves the project is idempotent. A failure's own output is kept for
+# the cleanup's diagnostics, which also print every container and the tail of its logs.
+for _ in 1 2; do
+  if ! up_output="$("${compose[@]}" up --detach --wait --wait-timeout 120 2>&1)"; then
+    echo "Disposable PostgreSQL and object storage did not become ready." >&2
+    exit 1
+  fi
+done
+up_output=""
 
 database_result="$(
   "${compose[@]}" exec -T \

scripts/test-migrations-integration.sh

@@ -7,6 +7,8 @@ cd "$repository_root"
 
 # shellcheck source=scripts/lib/compose-job-network.sh
 source "$repository_root/scripts/lib/compose-job-network.sh"
+# shellcheck source=scripts/lib/compose-diagnostics.sh
+source "$repository_root/scripts/lib/compose-diagnostics.sh"
 
 command -v docker >/dev/null || {
   echo "Docker is required for the PostgreSQL migration integration suite." >&2
@@ -25,14 +27,20 @@ if ! compose_job_network_initialize; then
   exit 1
 fi
 
-raw_project="ekklesia-migration-${GITHUB_RUN_ID:-local}-${GITHUB_RUN_ATTEMPT:-0}-${GITHUB_JOB:-job}-$$"
+# Unique per invocation, not only per run: several runs (and several jobs of one run) can share a
+# runner's Docker daemon at once, and the shell's PID is often identical across job containers.
+raw_project="ekklesia-migration-${GITHUB_RUN_ID:-local}-${GITHUB_RUN_ATTEMPT:-0}-${GITHUB_JOB:-job}-$$-${RANDOM}${RANDOM}"
 project="${raw_project//[^a-zA-Z0-9_-]/-}"
 project="${project,,}"
 
 export POSTGRES_DB="ekklesia_migration_test"
 export POSTGRES_USER="ekklesia_migration_test"
 export POSTGRES_PASSWORD="temporary-$project-${RANDOM}-${RANDOM}"
+# Both host ports are ephemeral: concurrent runs on one Docker daemon must never contend for a
+# fixed one. `.env.example` pins the storage port for local development, so it is overridden here
+# exactly like the database's. The bound ports are discovered below.
 export POSTGRES_PORT="0"
+export S3_PORT="0"
 
 compose=(
   docker compose
@@ -72,8 +80,17 @@ fi
 # and CI already knows how to provision. SeaweedFS is the intended deployment target (ADR 0004), and
 # proving the storage contract against a mock instead would have hidden the provider quirks this
 # suite exists to catch.
-if ! "${compose[@]}" up --detach --wait --wait-timeout 180 postgres object-storage >/dev/null 2>&1; then
+# Masked in any diagnostics below, along with anything else that looks like a credential.
+COMPOSE_DIAGNOSTICS_SECRETS=(
+  "$POSTGRES_PASSWORD"
+  "$(grep -E '^S3_ACCESS_KEY=' .env.example | cut -d= -f2-)"
+  "$(grep -E '^S3_SECRET_KEY=' .env.example | cut -d= -f2-)"
+)
+if ! up_output="$(
+  "${compose[@]}" up --detach --wait --wait-timeout 180 postgres object-storage 2>&1
+)"; then
   echo "Disposable PostgreSQL and object storage did not become ready." >&2
+  compose_report_not_ready "$up_output" "${compose[@]}"
   exit 1
 fi
 

tests/ci-scripts.test.ts

@@ -0,0 +1,165 @@
+import { execFile } from "node:child_process";
+import { readFile } from "node:fs/promises";
+import path from "node:path";
+import { fileURLToPath } from "node:url";
+import { promisify } from "node:util";
+import { describe, expect, it } from "vitest";
+
+/**
+ * The scripts CI runs against a shared Docker daemon. Several runs, and several jobs of one run, can
+ * be active on one runner at once (and the inventoried Gitea ignores workflow `concurrency`), so
+ * nothing they start may claim a fixed host port, a fixed project or network name, or a fixed
+ * subnet. And when a disposable project does not become ready, the failure must say why without
+ * printing a credential.
+ */
+
+const repositoryRoot = path.resolve(path.dirname(fileURLToPath(import.meta.url)), "..");
+const execFileAsync = promisify(execFile);
+
+const ciScripts = [
+  "scripts/test-migrations-integration.sh",
+  "scripts/smoke-infrastructure.sh",
+  "scripts/smoke-deploy.sh",
+];
+
+async function source(relativePath: string): Promise<string> {
+  return readFile(path.join(repositoryRoot, relativePath), "utf8");
+}
+
+/** Runs the real helper under bash with the given literal secrets and input. */
+async function redact(input: string, secrets: readonly string[] = []): Promise<string> {
+  const helper = path.join(repositoryRoot, "scripts", "lib", "compose-diagnostics.sh");
+  const script = `source "$1"; shift; COMPOSE_DIAGNOSTICS_SECRETS=("$@"); printf '%s' "$INPUT" | compose_diagnostics_redact`;
+  const { stdout } = await execFileAsync("bash", ["-c", script, "redact", helper, ...secrets], {
+    env: { ...process.env, INPUT: input },
+  });
+  return stdout;
+}
+
+describe("CI scripts on a shared Docker daemon", () => {
+  it("publish every host port ephemerally, never a fixed one", async () => {
+    const migrations = await source("scripts/test-migrations-integration.sh");
+    // `.env.example` pins both ports for local development; the suite must override both.
+    expect(migrations).toContain('export POSTGRES_PORT="0"');
+    expect(migrations).toContain('export S3_PORT="0"');
+
+    const infrastructure = await source("scripts/smoke-infrastructure.sh");
+    expect(infrastructure).toContain('export POSTGRES_PORT="${EKKLESIA_SMOKE_POSTGRES_PORT:-0}"');
+    expect(infrastructure).toContain('export S3_PORT="${EKKLESIA_SMOKE_S3_PORT:-0}"');
+
+    const deploy = await source("scripts/smoke-deploy.sh");
+    expect(deploy).toContain("EKKLESIA_HTTP_PORT=0");
+    expect(deploy).toContain("EKKLESIA_HTTPS_PORT=0");
+
+    for (const relativePath of ciScripts) {
+      const text = await source(relativePath);
+      // No `-p 8333:…`, `127.0.0.1:5432:…`, or `PORT=<non-zero>` anywhere.
+      expect(text, relativePath).not.toMatch(/\b(?:-p|--publish)\s+[^\s]*\d{2,5}:\d/u);
+      expect(text, relativePath).not.toMatch(/\b[A-Z_]*PORT="?[1-9]\d*"?\s*$/mu);
+    }
+  });
+
+  it("give every Compose project a name unique to the invocation, not only the run", async () => {
+    expect(await source("scripts/test-migrations-integration.sh")).toMatch(
+      /raw_project="ekklesia-migration-[^"\n]*\$\$-\$\{RANDOM\}\$\{RANDOM\}"/u,
+    );
+    expect(await source("scripts/smoke-infrastructure.sh")).toMatch(
+      /raw_project="ekklesia-smoke-[^"\n]*\$\$-\$\{RANDOM\}\$\{RANDOM\}"/u,
+    );
+    const deploy = await source("scripts/smoke-deploy.sh");
+    expect(deploy).toContain('run_id="$(openssl rand -hex 4)"');
+    expect(deploy).toContain('project="ekklesia-deploy-smoke-${run_id}"');
+    for (const relativePath of ciScripts) {
+      expect(await source(relativePath), relativePath).toContain('--project-name "$project"');
+    }
+  });
+
+  it("choose the deploy smoke's edge subnet from those not in use, and choose again on a race", async () => {
+    const deploy = await source("scripts/smoke-deploy.sh");
+    expect(deploy).not.toMatch(/^octet=\$\(\(RANDOM/mu);
+    expect(deploy).toContain("choose_edge_network() {");
+    expect(deploy).toContain("docker network inspect --format '{{range .IPAM.Config}}");
+    expect(deploy).toContain('[[ "$up_output" == *"overlap"* ]]');
+  });
+
+  it("tag the CI image per run and attempt, so concurrent runs of one commit cannot remove it", async () => {
+    const workflow = await source(".github/workflows/ci.yml");
+    expect(workflow).toContain(
+      'ekklesia-ci:"$GITHUB_SHA-${GITHUB_RUN_ID:-0}-${GITHUB_RUN_ATTEMPT:-1}"',
+    );
+    expect(workflow).not.toContain('ekklesia-ci:"$GITHUB_SHA"');
+  });
+
+  it("report every readiness failure with the project's containers and logs, through the redactor only", async () => {
+    for (const relativePath of ciScripts) {
+      const text = await source(relativePath);
+      expect(text, relativePath).toContain(
+        'source "$repository_root/scripts/lib/compose-diagnostics.sh"',
+      );
+      expect(text, relativePath).toContain("compose_report_not_ready");
+      expect(text, relativePath).toContain("COMPOSE_DIAGNOSTICS_SECRETS=(");
+      // Compose output is never printed raw: logs are only ever captured (the deploy smoke scans
+      // the API's for secrets), and a readiness failure's own output is never discarded.
+      for (const line of text
+        .split("\n")
+        .filter((candidate) => candidate.includes('compose[@]}" logs'))) {
+        expect(line, relativePath).toMatch(/="\$\("\$\{compose\[@\]\}" logs /u);
+      }
+      expect(text, relativePath).not.toMatch(/up --detach --wait[^\n]*>\/dev\/null 2>&1/u);
+    }
+  });
+});
+
+describe("compose diagnostics redaction", () => {
+  it("masks credential-looking values, URL user information, and Authorization headers", async () => {
+    const output = await redact(
+      [
+        "POSTGRES_PASSWORD=hunter2hunter2",
+        '  "AWS_SECRET_ACCESS_KEY": "abc/def+ghi"',
+        "S3_ACCESS_KEY=AKIAEXAMPLE other=fine",
+        "api_key: xyz123",
+        "connecting to postgresql://ekklesia:s3cr3t@postgres:5432/ekklesia",
+        "Authorization: Bearer eyJhbGciOi.payload.sig",
+        "EKKLESIA_IDENTITY_SECRET=0123456789abcdef",
+      ].join("\n"),
+    );
+
+    for (const secret of [
+      "hunter2hunter2",
+      "abc/def+ghi",
+      "AKIAEXAMPLE",
+      "xyz123",
+      "s3cr3t",
+      "eyJhbGciOi",
+      "0123456789abcdef",
+    ]) {
+      expect(output).not.toContain(secret);
+    }
+    expect(output).toContain("POSTGRES_PASSWORD=[redacted]");
+    expect(output).toContain("other=fine");
+    expect(output).toContain("postgresql://[redacted]@postgres:5432/ekklesia");
+  });
+
+  it("masks every occurrence of a supplied literal, wherever it appears", async () => {
+    const output = await redact("started with temporary-pw-12345 and again temporary-pw-12345", [
+      "temporary-pw-12345",
+    ]);
+    expect(output).toBe("started with [redacted] and again [redacted]\n");
+  });
+
+  it("keeps the diagnosis itself readable", async () => {
+    const diagnosis =
+      "Error response from daemon: Bind for 127.0.0.1:8333 failed: port is already allocated";
+    expect(await redact(diagnosis)).toBe(`${diagnosis}\n`);
+    expect(await redact("duplicate key value violates unique constraint")).toBe(
+      "duplicate key value violates unique constraint\n",
+    );
+    expect(await redact('password authentication failed for user "ekklesia"')).toBe(
+      'password authentication failed for user "ekklesia"\n',
+    );
+  });
+
+  it("strips terminal control sequences from container logs", async () => {
+    expect(await redact("\u001b[2Kpostgres-1  | ready")).toBe("postgres-1  | ready\n");
+  });
+});