Ekklesia/tests/ci-scripts.test.ts

import { execFile } from "node:child_process";
import { readFile } from "node:fs/promises";
import path from "node:path";
import { fileURLToPath } from "node:url";
import { promisify } from "node:util";
import { describe, expect, it } from "vitest";
import { parse } from "yaml";

/**
 * The scripts CI runs against a shared Docker daemon. Several runs, and several jobs of one run, can
 * be active on one runner at once (and the inventoried Gitea ignores workflow `concurrency`), so
 * nothing they start may claim a fixed host port, a fixed project or network name, or a fixed
 * subnet. And when a disposable project does not become ready, the failure must say why without
 * printing a credential.
 */

const repositoryRoot = path.resolve(path.dirname(fileURLToPath(import.meta.url)), "..");
const execFileAsync = promisify(execFile);

const ciScripts = [
  "scripts/test-migrations-integration.sh",
  "scripts/smoke-infrastructure.sh",
  "scripts/smoke-deploy.sh",
];

async function source(relativePath: string): Promise<string> {
  return readFile(path.join(repositoryRoot, relativePath), "utf8");
}

/** Runs the real helper under bash with the given literal secrets and input. */
async function redact(input: string, secrets: readonly string[] = []): Promise<string> {
  const helper = path.join(repositoryRoot, "scripts", "lib", "compose-diagnostics.sh");
  const script = `source "$1"; shift; COMPOSE_DIAGNOSTICS_SECRETS=("$@"); printf '%s' "$INPUT" | compose_diagnostics_redact`;
  const { stdout } = await execFileAsync("bash", ["-c", script, "redact", helper, ...secrets], {
    env: { ...process.env, INPUT: input },
  });
  return stdout;
}

describe("CI scripts on a shared Docker daemon", () => {
  it("publish every host port ephemerally, never a fixed one", async () => {
    const migrations = await source("scripts/test-migrations-integration.sh");
    // `.env.example` pins both ports for local development; the suite must override both.
    expect(migrations).toContain('export POSTGRES_PORT="0"');
    expect(migrations).toContain('export S3_PORT="0"');

    const infrastructure = await source("scripts/smoke-infrastructure.sh");
    expect(infrastructure).toContain('export POSTGRES_PORT="${EKKLESIA_SMOKE_POSTGRES_PORT:-0}"');
    expect(infrastructure).toContain('export S3_PORT="${EKKLESIA_SMOKE_S3_PORT:-0}"');

    const deploy = await source("scripts/smoke-deploy.sh");
    expect(deploy).toContain("EKKLESIA_HTTP_PORT=0");
    expect(deploy).toContain("EKKLESIA_HTTPS_PORT=0");

    for (const relativePath of ciScripts) {
      const text = await source(relativePath);
      // No `-p 8333:…`, `127.0.0.1:5432:…`, or `PORT=<non-zero>` anywhere.
      expect(text, relativePath).not.toMatch(/\b(?:-p|--publish)\s+[^\s]*\d{2,5}:\d/u);
      expect(text, relativePath).not.toMatch(/\b[A-Z_]*PORT="?[1-9]\d*"?\s*$/mu);
    }
  });

  it("name every stack through one helper: the reserved CI prefix, the job's key, and a unique suffix", async () => {
    const helper = await source("scripts/lib/ci-project.sh");
    // Run, attempt, job key, kind, then a per-invocation suffix: unique even when job containers
    // share a PID, and findable later by run and job alone (scripts/ci-sweep.sh --job).
    expect(helper).toContain(
      "ekklesia-ci-${GITHUB_RUN_ID:-local}-${GITHUB_RUN_ATTEMPT:-0}-${EKKLESIA_CI_JOB_KEY:-${GITHUB_JOB:-local}}-",
    );
    expect(helper).toContain("$$-${RANDOM}${RANDOM}");
    for (const relativePath of ciScripts) {
      const text = await source(relativePath);
      expect(text, relativePath).toContain('source "$repository_root/scripts/lib/ci-project.sh"');
      expect(text, relativePath).toMatch(/^project="\$\(ci_project_name [a-z"${}_-]+\)"$/mu);
      expect(text, relativePath).not.toContain("raw_project=");
      expect(text, relativePath).toContain('--project-name "$project"');
    }
  });

  it("label everything the deploy smoke creates outside Compose, so a sweep can find it", async () => {
    // Continuation lines joined, so each command is checked whole.
    const deploy = (await source("scripts/smoke-deploy.sh")).replace(/\\\n\s*/gu, " ");
    const creating = deploy
      .split("\n")
      .filter((line) => /\bdocker (?:run --detach|volume create|build)\b/u.test(line));
    expect(creating.length).toBeGreaterThanOrEqual(4);
    for (const line of creating) {
      expect(line).toContain('--label "${EKKLESIA_CI_LABEL}=${project}"');
    }
  });

  it("sweep stale stacks before, and this job's own stacks after, every job that starts any", async () => {
    const workflow = parse(await source(".github/workflows/ci.yml")) as {
      jobs: Record<
        string,
        { env?: Record<string, string>; steps: { if?: string; run?: string; name?: string }[] }
      >;
    };
    const startsStacks =
      /smoke-infrastructure\.sh|test:migrations:integration|smoke-deploy\.sh|check-image\.sh|test-package-publication\.sh|docker build/u;
    const jobs = Object.entries(workflow.jobs).filter(([, job]) =>
      job.steps.some((step) => startsStacks.test(step.run ?? "")),
    );
    expect(jobs.map(([id]) => id).sort()).toEqual(
      [
        "deploy-smoke",
        "image-build",
        "infrastructure-smoke",
        "migration-smoke",
        "package-publication",
      ].sort(),
    );
    for (const [id, job] of jobs) {
      expect(job.env?.EKKLESIA_CI_JOB_KEY, id).toBeTruthy();
      const runs = job.steps.map((step) => step.run ?? "");
      const firstStack = runs.findIndex((run) => startsStacks.test(run));
      const stale = runs.findIndex((run) => run.includes("bash scripts/ci-sweep.sh --stale"));
      const own = job.steps.findIndex(
        (step) =>
          step.if === "always()" && (step.run ?? "").includes("bash scripts/ci-sweep.sh --job"),
      );
      expect(stale, `${id}: stale sweep`).toBeGreaterThanOrEqual(0);
      expect(stale, `${id}: stale sweep runs first`).toBeLessThan(firstStack);
      expect(own, `${id}: its own cleanup, always`).toBeGreaterThan(firstStack);
      expect(own, `${id}: cleanup is the last step`).toBe(job.steps.length - 1);
    }
    // One matrix leg's cleanup must never remove the other leg's running stack.
    expect(workflow.jobs["deploy-smoke"]?.env?.EKKLESIA_CI_JOB_KEY).toContain("matrix.topology");
  });

  it("choose the deploy smoke's edge subnet from those not in use, and choose again on a race", async () => {
    const deploy = await source("scripts/smoke-deploy.sh");
    expect(deploy).not.toMatch(/^octet=\$\(\(RANDOM/mu);
    expect(deploy).toContain("choose_edge_network() {");
    expect(deploy).toContain("docker network inspect --format '{{range .IPAM.Config}}");
    expect(deploy).toContain('[[ "$up_output" == *"overlap"* ]]');
  });

  it("tag the CI image per run and attempt, so concurrent runs of one commit cannot remove it", async () => {
    const workflow = await source(".github/workflows/ci.yml");
    const imageCheck = await source("scripts/check-image.sh");
    expect(workflow).toContain("bash scripts/check-image.sh");
    expect(imageCheck).toContain(
      'tag_base="ekklesia-ci:${GITHUB_SHA:-local}-${GITHUB_RUN_ID:-0}-${GITHUB_RUN_ATTEMPT:-1}-"',
    );
    expect(workflow).not.toContain('ekklesia-ci:"$GITHUB_SHA"');
  });

  it("label every image the image check builds, so a sweep can find it", async () => {
    const imageCheck = (await source("scripts/check-image.sh")).replace(/\\\n\s*/gu, " ");
    expect(imageCheck).toContain('source "$repository_root/scripts/lib/ci-project.sh"');
    expect(imageCheck).toMatch(/^project="\$\(ci_project_name image\)"$/mu);
    const builds = imageCheck.split("\n").filter((line) => /\bdocker build\b/u.test(line));
    expect(builds.length).toBeGreaterThanOrEqual(1);
    for (const line of builds) {
      expect(line).toContain('--label "${EKKLESIA_CI_LABEL}=${project}"');
    }
  });

  it("exercise a real native package through the hardened verifier container", async () => {
    const imageCheck = (await source("scripts/check-image.sh")).replace(/\\\n\s*/gu, " ");
    expect(imageCheck).toContain(
      "packages/module-pack/tests/support/pack-native-verifier-proof.mjs",
    );
    expect(imageCheck).toContain("--tmpfs /tmp:rw,nosuid,nodev,noexec,size=64m");
    expect(imageCheck).toContain(
      "--tmpfs /run/ekklesia-verifier:rw,nosuid,nodev,exec,size=1100m,mode=0700,uid=1000,gid=1000",
    );
    expect(imageCheck).toContain("const actual=await m.proof()");
    expect(imageCheck).toContain("file:///tmp/noexec-installed/server/index.js");
    expect(imageCheck).toContain("scratch_failure /tmp");
    expect(imageCheck).toContain("scratch_failure /package");
  });

  it("install the smoke test's module only as a canonical package the image itself verified (ADR 0049)", async () => {
    const deploy = (await source("scripts/smoke-deploy.sh")).replace(/\\\n\s*/gu, " ");
    const packaging = await source("scripts/lib/build-module-package.Dockerfile");
    expect(packaging).toContain("packages/module-pack/bin/ekklesia-pack-module.mjs");
    expect(packaging).not.toMatch(/pnpm[^\n]*\bdeploy\b/u);
    expect(deploy).toContain("scripts/lib/build-module-package.Dockerfile");
    expect(deploy).toMatch(
      /--entrypoint node "\$image" core\/dist\/cli\/verify-module-package\.js[^\n]*--sha256-file[^\n]*--extract-to "\/staging\/\$\{module_id\}"/u,
    );
    expect(deploy).toContain("--tmpfs /tmp:rw,nosuid,nodev,noexec,size=64m");
    expect(deploy).toContain(
      "--tmpfs /run/ekklesia-verifier:rw,nosuid,nodev,exec,size=1100m,mode=0700,uid=1000,gid=1000",
    );
    expect(deploy).toContain("-e EKKLESIA_MODULE_VERIFIER_SCRATCH_DIR=/run/ekklesia-verifier");
    // Module storage only ever receives the verified tree from staging.
    expect(deploy).toContain(`staged="/staging/\${MODULE_ID}"`);
    expect(deploy).toContain('cp -R "$staged" "$incoming"');
    expect(deploy).toContain('mv "$incoming" "$installed"');
  });

  it("hand the verifier the package's archive and digest together, from a fresh release pair, never from build output", async () => {
    const deploy = (await source("scripts/smoke-deploy.sh")).replace(/\\\n\s*/gu, " ");
    expect(deploy).toContain('source "$repository_root/scripts/lib/module-package-release.sh"');
    // The pair is made from this run's own build output, into a directory that must not exist yet.
    expect(deploy).toContain(`--output "type=local,dest=\${work}/module-package"`);
    expect(deploy).toContain('package_output="$work/module-package"');
    expect(deploy).toContain('package_output="$module_release_directory"');
    expect(deploy).toContain(
      'package_archive="$(module_package_release "$package_output" "$work/package")"',
    );
    expect(deploy).toMatch(/^\s*stage package$/mu);
    // BuildKit's output directory is 0700 and owned by whoever built it: it is never mounted.
    expect(deploy).not.toContain('"$daemon_work/module-package:');
    expect(deploy).not.toMatch(/^\s*stage module-package$/mu);
    // The verifier reads both files from the one mount, and always checks the digest file.
    const verifier = deploy
      .split("\n")
      .find((line) => line.includes("core/dist/cli/verify-module-package.js"));
    expect(verifier).toContain('-v "$daemon_work/package:/package:ro"');
    expect(verifier).toContain(`"/package/\${package_archive}"`);
    expect(verifier).toContain(`--sha256-file "/package/\${package_archive}.sha256"`);
    expect(verifier).not.toMatch(/--sha256\s/u);
  });

  it("accepts one exact external module release without putting its source in the platform image", async () => {
    const deploy = await source("scripts/smoke-deploy.sh");
    expect(deploy).toContain(`module_release_directory="\${EKKLESIA_SMOKE_MODULE_RELEASE_DIR:-}"`);
    expect(deploy).toContain(`module_id="\${EKKLESIA_SMOKE_MODULE_ID:-example}"`);
    expect(deploy).toContain('package_output="$module_release_directory"');
    expect(deploy).toContain('-e SMOKE_MODULE_ID="$module_id"');
    expect(deploy).toContain(`--extract-to "/staging/\${module_id}"`);
    expect(deploy).toContain(`installed="/modules/\${MODULE_ID}"`);
  });

  it("proves installed module state across restart, container recreation, and lifecycle changes", async () => {
    const deploy = await source("scripts/smoke-deploy.sh");
    expect(deploy).toContain(`"\${compose[@]}" rm --force api`);
    expect(deploy).toContain("container recreation made an applied module migration pending again");
    expect(deploy).toContain('core/dist/cli/set-module-enabled.js --disable --module "$module_id"');
    expect(deploy).toContain('core/dist/cli/set-module-enabled.js --enable --module "$module_id"');
    expect(deploy).toContain("the restored display runtime did not return with its data");
  });

  it("gives page-side display requests enough time on a constrained shared runner", async () => {
    const deploy = await source("scripts/smoke-deploy.sh");
    expect(deploy).toContain('-e SMOKE_CALL_TIMEOUT_MS="');
    expect(deploy).toContain("EKKLESIA_SMOKE_CALL_TIMEOUT_MS:-30000");
  });

  it("run every deploy smoke against the one generic image, never a second image with a module in it", async () => {
    // ADR 0048: the split topology installs its test module into module storage at run time.
    const deploy = (await source("scripts/smoke-deploy.sh")).replace(/\\\n\s*/gu, " ");
    const imageBuilds = deploy
      .split("\n")
      .filter((line) => /\bdocker build\b/u.test(line) && line.includes("deploy/Dockerfile"));
    expect(imageBuilds).toHaveLength(1);
    expect(deploy).not.toContain('mv "$context/templates/module" "$context/modules/');
    expect(deploy).not.toMatch(/sed -i 's\|\^ {2}templates\/module:/u);
    expect(deploy).toContain("EKKLESIA_MODULES_HOST_DIR=");
  });

  it("report every readiness failure with the project's containers and logs, through the redactor only", async () => {
    for (const relativePath of ciScripts) {
      const text = await source(relativePath);
      expect(text, relativePath).toContain(
        'source "$repository_root/scripts/lib/compose-diagnostics.sh"',
      );
      expect(text, relativePath).toContain("compose_report_not_ready");
      expect(text, relativePath).toContain("COMPOSE_DIAGNOSTICS_SECRETS=(");
      // Compose output is never printed raw: logs are only ever captured (the deploy smoke scans
      // the API's for secrets), and a readiness failure's own output is never discarded.
      for (const line of text
        .split("\n")
        .filter((candidate) => candidate.includes('compose[@]}" logs'))) {
        expect(line, relativePath).toMatch(/="\$\("\$\{compose\[@\]\}" logs /u);
      }
      expect(text, relativePath).not.toMatch(/up --detach --wait[^\n]*>\/dev\/null 2>&1/u);
    }
  });
});

describe("compose diagnostics redaction", () => {
  it("masks credential-looking values, URL user information, and Authorization headers", async () => {
    const output = await redact(
      [
        "POSTGRES_PASSWORD=hunter2hunter2",
        '  "AWS_SECRET_ACCESS_KEY": "abc/def+ghi"',
        "S3_ACCESS_KEY=AKIAEXAMPLE other=fine",
        "api_key: xyz123",
        "connecting to postgresql://ekklesia:s3cr3t@postgres:5432/ekklesia",
        "Authorization: Bearer eyJhbGciOi.payload.sig",
        "EKKLESIA_IDENTITY_SECRET=0123456789abcdef",
      ].join("\n"),
    );

    for (const secret of [
      "hunter2hunter2",
      "abc/def+ghi",
      "AKIAEXAMPLE",
      "xyz123",
      "s3cr3t",
      "eyJhbGciOi",
      "0123456789abcdef",
    ]) {
      expect(output).not.toContain(secret);
    }
    expect(output).toContain("POSTGRES_PASSWORD=[redacted]");
    expect(output).toContain("other=fine");
    expect(output).toContain("postgresql://[redacted]@postgres:5432/ekklesia");
  });

  it("masks every occurrence of a supplied literal, wherever it appears", async () => {
    const output = await redact("started with temporary-pw-12345 and again temporary-pw-12345", [
      "temporary-pw-12345",
    ]);
    expect(output).toBe("started with [redacted] and again [redacted]\n");
  });

  it("keeps the diagnosis itself readable", async () => {
    const diagnosis =
      "Error response from daemon: Bind for 127.0.0.1:8333 failed: port is already allocated";
    expect(await redact(diagnosis)).toBe(`${diagnosis}\n`);
    expect(await redact("duplicate key value violates unique constraint")).toBe(
      "duplicate key value violates unique constraint\n",
    );
    expect(await redact('password authentication failed for user "ekklesia"')).toBe(
      'password authentication failed for user "ekklesia"\n',
    );
  });

  it("strips terminal control sequences from container logs", async () => {
    expect(await redact("\u001b[2Kpostgres-1  | ready")).toBe("postgres-1  | ready\n");
  });
});