import { execFile } from "node:child_process"; import { readFile } from "node:fs/promises"; import path from "node:path"; import { fileURLToPath } from "node:url"; import { promisify } from "node:util"; import { describe, expect, it } from "vitest"; import { parse } from "yaml"; /** * The scripts CI runs against a shared Docker daemon. Several runs, and several jobs of one run, can * be active on one runner at once (and the inventoried Gitea ignores workflow `concurrency`), so * nothing they start may claim a fixed host port, a fixed project or network name, or a fixed * subnet. And when a disposable project does not become ready, the failure must say why without * printing a credential. */ const repositoryRoot = path.resolve(path.dirname(fileURLToPath(import.meta.url)), ".."); const execFileAsync = promisify(execFile); const ciScripts = [ "scripts/test-migrations-integration.sh", "scripts/smoke-infrastructure.sh", "scripts/smoke-deploy.sh", ]; async function source(relativePath: string): Promise { return readFile(path.join(repositoryRoot, relativePath), "utf8"); } /** Runs the real helper under bash with the given literal secrets and input. */ async function redact(input: string, secrets: readonly string[] = []): Promise { const helper = path.join(repositoryRoot, "scripts", "lib", "compose-diagnostics.sh"); const script = `source "$1"; shift; COMPOSE_DIAGNOSTICS_SECRETS=("$@"); printf '%s' "$INPUT" | compose_diagnostics_redact`; const { stdout } = await execFileAsync("bash", ["-c", script, "redact", helper, ...secrets], { env: { ...process.env, INPUT: input }, }); return stdout; } describe("CI scripts on a shared Docker daemon", () => { it("publish every host port ephemerally, never a fixed one", async () => { const migrations = await source("scripts/test-migrations-integration.sh"); // `.env.example` pins both ports for local development; the suite must override both. expect(migrations).toContain('export POSTGRES_PORT="0"'); expect(migrations).toContain('export S3_PORT="0"'); const infrastructure = await source("scripts/smoke-infrastructure.sh"); expect(infrastructure).toContain('export POSTGRES_PORT="${EKKLESIA_SMOKE_POSTGRES_PORT:-0}"'); expect(infrastructure).toContain('export S3_PORT="${EKKLESIA_SMOKE_S3_PORT:-0}"'); const deploy = await source("scripts/smoke-deploy.sh"); expect(deploy).toContain("EKKLESIA_HTTP_PORT=0"); expect(deploy).toContain("EKKLESIA_HTTPS_PORT=0"); for (const relativePath of ciScripts) { const text = await source(relativePath); // No `-p 8333:…`, `127.0.0.1:5432:…`, or `PORT=` anywhere. expect(text, relativePath).not.toMatch(/\b(?:-p|--publish)\s+[^\s]*\d{2,5}:\d/u); expect(text, relativePath).not.toMatch(/\b[A-Z_]*PORT="?[1-9]\d*"?\s*$/mu); } }); it("name every stack through one helper: the reserved CI prefix, the job's key, and a unique suffix", async () => { const helper = await source("scripts/lib/ci-project.sh"); // Run, attempt, job key, kind, then a per-invocation suffix: unique even when job containers // share a PID, and findable later by run and job alone (scripts/ci-sweep.sh --job). expect(helper).toContain( "ekklesia-ci-${GITHUB_RUN_ID:-local}-${GITHUB_RUN_ATTEMPT:-0}-${EKKLESIA_CI_JOB_KEY:-${GITHUB_JOB:-local}}-", ); expect(helper).toContain("$$-${RANDOM}${RANDOM}"); for (const relativePath of ciScripts) { const text = await source(relativePath); expect(text, relativePath).toContain('source "$repository_root/scripts/lib/ci-project.sh"'); expect(text, relativePath).toMatch(/^project="\$\(ci_project_name [a-z"${}_-]+\)"$/mu); expect(text, relativePath).not.toContain("raw_project="); expect(text, relativePath).toContain('--project-name "$project"'); } }); it("label everything the deploy smoke creates outside Compose, so a sweep can find it", async () => { // Continuation lines joined, so each command is checked whole. const deploy = (await source("scripts/smoke-deploy.sh")).replace(/\\\n\s*/gu, " "); const creating = deploy .split("\n") .filter((line) => /\bdocker (?:run --detach|volume create|build)\b/u.test(line)); expect(creating.length).toBeGreaterThanOrEqual(4); for (const line of creating) { expect(line).toContain('--label "${EKKLESIA_CI_LABEL}=${project}"'); } }); it("sweep stale stacks before, and this job's own stacks after, every job that starts any", async () => { const workflow = parse(await source(".github/workflows/ci.yml")) as { jobs: Record< string, { env?: Record; steps: { if?: string; run?: string; name?: string }[] } >; }; const startsStacks = /smoke-infrastructure\.sh|test:migrations:integration|smoke-deploy\.sh|check-image\.sh|test-package-publication\.sh|docker build/u; const jobs = Object.entries(workflow.jobs).filter(([, job]) => job.steps.some((step) => startsStacks.test(step.run ?? "")), ); expect(jobs.map(([id]) => id).sort()).toEqual( [ "deploy-smoke", "image-build", "infrastructure-smoke", "migration-smoke", "package-publication", ].sort(), ); for (const [id, job] of jobs) { expect(job.env?.EKKLESIA_CI_JOB_KEY, id).toBeTruthy(); const runs = job.steps.map((step) => step.run ?? ""); const firstStack = runs.findIndex((run) => startsStacks.test(run)); const stale = runs.findIndex((run) => run.includes("bash scripts/ci-sweep.sh --stale")); const own = job.steps.findIndex( (step) => step.if === "always()" && (step.run ?? "").includes("bash scripts/ci-sweep.sh --job"), ); expect(stale, `${id}: stale sweep`).toBeGreaterThanOrEqual(0); expect(stale, `${id}: stale sweep runs first`).toBeLessThan(firstStack); expect(own, `${id}: its own cleanup, always`).toBeGreaterThan(firstStack); expect(own, `${id}: cleanup is the last step`).toBe(job.steps.length - 1); } // One matrix leg's cleanup must never remove the other leg's running stack. expect(workflow.jobs["deploy-smoke"]?.env?.EKKLESIA_CI_JOB_KEY).toContain("matrix.topology"); }); it("choose the deploy smoke's edge subnet from those not in use, and choose again on a race", async () => { const deploy = await source("scripts/smoke-deploy.sh"); expect(deploy).not.toMatch(/^octet=\$\(\(RANDOM/mu); expect(deploy).toContain("choose_edge_network() {"); expect(deploy).toContain("docker network inspect --format '{{range .IPAM.Config}}"); expect(deploy).toContain('[[ "$up_output" == *"overlap"* ]]'); }); it("tag the CI image per run and attempt, so concurrent runs of one commit cannot remove it", async () => { const workflow = await source(".github/workflows/ci.yml"); const imageCheck = await source("scripts/check-image.sh"); expect(workflow).toContain("bash scripts/check-image.sh"); expect(imageCheck).toContain( 'tag_base="ekklesia-ci:${GITHUB_SHA:-local}-${GITHUB_RUN_ID:-0}-${GITHUB_RUN_ATTEMPT:-1}-"', ); expect(workflow).not.toContain('ekklesia-ci:"$GITHUB_SHA"'); }); it("label every image the image check builds, so a sweep can find it", async () => { const imageCheck = (await source("scripts/check-image.sh")).replace(/\\\n\s*/gu, " "); expect(imageCheck).toContain('source "$repository_root/scripts/lib/ci-project.sh"'); expect(imageCheck).toMatch(/^project="\$\(ci_project_name image\)"$/mu); const builds = imageCheck.split("\n").filter((line) => /\bdocker build\b/u.test(line)); expect(builds.length).toBeGreaterThanOrEqual(1); for (const line of builds) { expect(line).toContain('--label "${EKKLESIA_CI_LABEL}=${project}"'); } }); it("exercise a real native package through the hardened verifier container", async () => { const imageCheck = (await source("scripts/check-image.sh")).replace(/\\\n\s*/gu, " "); expect(imageCheck).toContain( "packages/module-pack/tests/support/pack-native-verifier-proof.mjs", ); expect(imageCheck).toContain("--tmpfs /tmp:rw,nosuid,nodev,noexec,size=64m"); expect(imageCheck).toContain( "--tmpfs /run/ekklesia-verifier:rw,nosuid,nodev,exec,size=1100m,mode=0700,uid=1000,gid=1000", ); expect(imageCheck).toContain("const actual=await m.proof()"); expect(imageCheck).toContain("file:///tmp/noexec-installed/server/index.js"); expect(imageCheck).toContain("scratch_failure /tmp"); expect(imageCheck).toContain("scratch_failure /package"); }); it("install the smoke test's module only as a canonical package the image itself verified (ADR 0049)", async () => { const deploy = (await source("scripts/smoke-deploy.sh")).replace(/\\\n\s*/gu, " "); const packaging = await source("scripts/lib/build-module-package.Dockerfile"); expect(packaging).toContain("packages/module-pack/bin/ekklesia-pack-module.mjs"); expect(packaging).not.toMatch(/pnpm[^\n]*\bdeploy\b/u); expect(deploy).toContain("scripts/lib/build-module-package.Dockerfile"); expect(deploy).toMatch( /--entrypoint node "\$image" core\/dist\/cli\/verify-module-package\.js[^\n]*--sha256-file[^\n]*--extract-to \/staging\/example/u, ); expect(deploy).toContain("--tmpfs /tmp:rw,nosuid,nodev,noexec,size=64m"); expect(deploy).toContain( "--tmpfs /run/ekklesia-verifier:rw,nosuid,nodev,exec,size=1100m,mode=0700,uid=1000,gid=1000", ); expect(deploy).toContain("-e EKKLESIA_MODULE_VERIFIER_SCRATCH_DIR=/run/ekklesia-verifier"); // Module storage only ever receives the verified tree from staging. expect(deploy).toContain("cp -R /staging/example /modules/.incoming-example"); }); it("hand the verifier the package's archive and digest together, from a fresh release pair, never from build output", async () => { const deploy = (await source("scripts/smoke-deploy.sh")).replace(/\\\n\s*/gu, " "); expect(deploy).toContain('source "$repository_root/scripts/lib/module-package-release.sh"'); // The pair is made from this run's own build output, into a directory that must not exist yet. expect(deploy).toContain('--output "type=local,dest=${work}/module-package"'); expect(deploy).toContain( 'package_archive="$(module_package_release "$work/module-package" "$work/package")"', ); expect(deploy).toMatch(/^\s*stage package$/mu); // BuildKit's output directory is 0700 and owned by whoever built it: it is never mounted. expect(deploy).not.toContain('"$daemon_work/module-package:'); expect(deploy).not.toMatch(/^\s*stage module-package$/mu); // The verifier reads both files from the one mount, and always checks the digest file. const verifier = deploy .split("\n") .find((line) => line.includes("core/dist/cli/verify-module-package.js")); expect(verifier).toContain('-v "$daemon_work/package:/package:ro"'); expect(verifier).toContain('"/package/${package_archive}"'); expect(verifier).toContain('--sha256-file "/package/${package_archive}.sha256"'); expect(verifier).not.toMatch(/--sha256\s/u); }); it("run every deploy smoke against the one generic image, never a second image with a module in it", async () => { // ADR 0048: the split topology installs its test module into module storage at run time. const deploy = (await source("scripts/smoke-deploy.sh")).replace(/\\\n\s*/gu, " "); const imageBuilds = deploy .split("\n") .filter((line) => /\bdocker build\b/u.test(line) && line.includes("deploy/Dockerfile")); expect(imageBuilds).toHaveLength(1); expect(deploy).not.toContain('mv "$context/templates/module" "$context/modules/'); expect(deploy).not.toMatch(/sed -i 's\|\^ {2}templates\/module:/u); expect(deploy).toContain("EKKLESIA_MODULES_HOST_DIR="); }); it("report every readiness failure with the project's containers and logs, through the redactor only", async () => { for (const relativePath of ciScripts) { const text = await source(relativePath); expect(text, relativePath).toContain( 'source "$repository_root/scripts/lib/compose-diagnostics.sh"', ); expect(text, relativePath).toContain("compose_report_not_ready"); expect(text, relativePath).toContain("COMPOSE_DIAGNOSTICS_SECRETS=("); // Compose output is never printed raw: logs are only ever captured (the deploy smoke scans // the API's for secrets), and a readiness failure's own output is never discarded. for (const line of text .split("\n") .filter((candidate) => candidate.includes('compose[@]}" logs'))) { expect(line, relativePath).toMatch(/="\$\("\$\{compose\[@\]\}" logs /u); } expect(text, relativePath).not.toMatch(/up --detach --wait[^\n]*>\/dev\/null 2>&1/u); } }); }); describe("compose diagnostics redaction", () => { it("masks credential-looking values, URL user information, and Authorization headers", async () => { const output = await redact( [ "POSTGRES_PASSWORD=hunter2hunter2", ' "AWS_SECRET_ACCESS_KEY": "abc/def+ghi"', "S3_ACCESS_KEY=AKIAEXAMPLE other=fine", "api_key: xyz123", "connecting to postgresql://ekklesia:s3cr3t@postgres:5432/ekklesia", "Authorization: Bearer eyJhbGciOi.payload.sig", "EKKLESIA_IDENTITY_SECRET=0123456789abcdef", ].join("\n"), ); for (const secret of [ "hunter2hunter2", "abc/def+ghi", "AKIAEXAMPLE", "xyz123", "s3cr3t", "eyJhbGciOi", "0123456789abcdef", ]) { expect(output).not.toContain(secret); } expect(output).toContain("POSTGRES_PASSWORD=[redacted]"); expect(output).toContain("other=fine"); expect(output).toContain("postgresql://[redacted]@postgres:5432/ekklesia"); }); it("masks every occurrence of a supplied literal, wherever it appears", async () => { const output = await redact("started with temporary-pw-12345 and again temporary-pw-12345", [ "temporary-pw-12345", ]); expect(output).toBe("started with [redacted] and again [redacted]\n"); }); it("keeps the diagnosis itself readable", async () => { const diagnosis = "Error response from daemon: Bind for 127.0.0.1:8333 failed: port is already allocated"; expect(await redact(diagnosis)).toBe(`${diagnosis}\n`); expect(await redact("duplicate key value violates unique constraint")).toBe( "duplicate key value violates unique constraint\n", ); expect(await redact('password authentication failed for user "ekklesia"')).toBe( 'password authentication failed for user "ekklesia"\n', ); }); it("strips terminal control sequences from container logs", async () => { expect(await redact("\u001b[2Kpostgres-1 | ready")).toBe("postgres-1 | ready\n"); }); });