From 831a48187ce50c62903e38341dd66221b41aaae1 Mon Sep 17 00:00:00 2001 From: ekklesia Date: Tue, 21 Jul 2026 09:37:15 -0500 Subject: [PATCH] test: add PostgreSQL migration integration coverage --- .github/workflows/ci.yml | 24 + CONTRIBUTING.md | 9 + README.md | 14 +- core/tests/postgres-migration.integration.ts | 1238 +++++++++++++++++ .../0009-transactional-module-migrations.md | 18 +- docs/architecture/platform-architecture.md | 5 +- docs/development/repository-workflow.md | 24 +- docs/development/testing.md | 60 + docs/module-development/README.md | 8 +- package.json | 1 + scripts/lib/compose-job-network.sh | 112 ++ scripts/smoke-infrastructure.sh | 71 +- scripts/test-migrations-integration.sh | 100 ++ tests/infrastructure.test.ts | 84 +- tsconfig.json | 7 +- vitest.migrations.integration.config.ts | 11 + 16 files changed, 1690 insertions(+), 96 deletions(-) create mode 100644 core/tests/postgres-migration.integration.ts create mode 100644 docs/development/testing.md create mode 100644 scripts/lib/compose-job-network.sh create mode 100755 scripts/test-migrations-integration.sh create mode 100644 vitest.migrations.integration.config.ts diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index f42d653..c3b3d39 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -47,3 +47,27 @@ jobs: - name: Exercise PostgreSQL and S3 infrastructure run: bash scripts/smoke-infrastructure.sh + + migration-smoke: + needs: verify + runs-on: ubuntu-latest + timeout-minutes: 15 + steps: + - name: Check out repository + uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + + - name: Set up Node.js + uses: actions/setup-node@48b55a011bda9f5d6aeb4c2d9c7362e8dae4041e # v6.4.0 + with: + node-version-file: .node-version + + - name: Activate pinned pnpm + run: | + corepack enable + corepack prepare pnpm@11.12.0 --activate + + - name: Install dependencies + run: pnpm install --frozen-lockfile + + - name: Exercise transactional migrations against PostgreSQL + run: pnpm test:migrations:integration diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 81c075d..20fb06c 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -9,6 +9,15 @@ Before opening a change, run: pnpm verify ``` +Migration-runner changes also require the disposable live PostgreSQL suite: + +```bash +pnpm test:migrations:integration +``` + +See the [testing guide](docs/development/testing.md) for the distinction between unit and live +evidence and for credential-safe troubleshooting. + When adding or changing a module manifest, also run `pnpm modules:validate` directly for focused, path-aware discovery errors. It is included in `pnpm verify`. diff --git a/README.md b/README.md index f0679cc..680ea7d 100644 --- a/README.md +++ b/README.md @@ -13,9 +13,10 @@ contracts, deterministic schema aggregation, and a cloneable backend-only module artifacts now have module-owned Drizzle contracts, deterministic offline discovery, explicit destructive-change declarations, and exact checksums. Core now provides read-only applied-history planning and a locked, whole-batch transactional PostgreSQL runner with exact external destructive -approval matching. Live PostgreSQL runner integration coverage, production approval-provider -composition, identity, HTTP routing, jobs, physical storage provisioning, surfaces, and feature -modules remain later vertical slices. +approval matching. A hermetic live PostgreSQL suite now proves history, rollback, statement +breakpoints, advisory-lock concurrency and release, exact destructive approvals, and sanitization +against the production adapter. Production approval-provider composition, identity, HTTP routing, +jobs, physical storage provisioning, surfaces, and feature modules remain later vertical slices. ## Quick start @@ -28,6 +29,7 @@ pnpm infrastructure:up pnpm modules:validate pnpm migrations:validate pnpm migrations:plan +pnpm test:migrations:integration pnpm verify ``` @@ -36,8 +38,9 @@ pnpm verify advisory lock and one transaction. Review the plan before applying. Destructive migrations are blocked by default and require an exact externally supplied approval; there is no force flag. -The runner currently has deterministic unit coverage through a database port. Live PostgreSQL -migration-runner tests are deferred and are not yet claimed by `pnpm verify`. +The runner has both deterministic unit coverage through a narrow database port and a separate live +PostgreSQL integration suite. The live command creates and destroys disposable Docker +infrastructure; it remains separate from the Docker-free `pnpm verify` gate. Run `pnpm infrastructure:smoke` for an isolated end-to-end PostgreSQL and authenticated S3 check. @@ -54,6 +57,7 @@ environment. - [Platform architecture](docs/architecture/platform-architecture.md) - [Repository boundaries](docs/architecture/repository-boundaries.md) - [Repository workflow](docs/development/repository-workflow.md) +- [Testing](docs/development/testing.md) - [Module development](docs/module-development/README.md) - [Architecture decisions](docs/architecture/decisions/README.md) diff --git a/core/tests/postgres-migration.integration.ts b/core/tests/postgres-migration.integration.ts new file mode 100644 index 0000000..2b47f66 --- /dev/null +++ b/core/tests/postgres-migration.integration.ts @@ -0,0 +1,1238 @@ +import { createHash, randomBytes } from "node:crypto"; +import { Client, type QueryResult, type QueryResultRow } from "pg"; +import { afterAll, describe, expect, it, vi } from "vitest"; +import { reportMigrationCommandError } from "../src/cli/migration-command-errors.js"; +import { + applyModuleMigrations, + createPostgresMigrationDatabase, + type MigrationApproval, + type MigrationApprovalProvider, + MigrationRunnerError, + type MigrationRunnerErrorCode, + planModuleMigrations, +} from "../src/migrations/index.js"; +import type { + DiscoveredModuleMigration, + ModuleMigrationAggregate, +} from "../src/modules/migration-discovery.js"; + +const integrationDatabaseUrlEnvironment = "EKKLESIA_MIGRATION_TEST_DATABASE_URL"; +const historyTable = "public.ekklesia_core_migrations"; +const migrationLockNamespace = 1_162_562_380; +const migrationLockKey = 1_296_648_018; +const concurrencyBarrierNamespace = 1_162_562_380; +const concurrencyBarrierKey = 1_414_743_380; +const trackedDatabases = new Set(); +let databaseCounter = 0; + +class IntegrationHarnessError extends Error { + constructor(message: string) { + super(message); + this.name = "IntegrationHarnessError"; + this.stack = `${this.name}: ${this.message}`; + } +} + +function harnessError(message: string): IntegrationHarnessError { + return new IntegrationHarnessError(message); +} + +function baseDatabaseUrl(): string { + const value = process.env[integrationDatabaseUrlEnvironment]; + if (typeof value !== "string" || value.trim() === "") { + throw harnessError("The migration integration database is not configured."); + } + + try { + const parsed = new URL(value); + if (parsed.protocol !== "postgres:" && parsed.protocol !== "postgresql:") { + throw new TypeError("Unsupported protocol."); + } + } catch { + throw harnessError("The migration integration database configuration is invalid."); + } + return value; +} + +function quoteDatabaseName(databaseName: string): string { + if (!/^[a-z][a-z0-9_]{0,62}$/u.test(databaseName)) { + throw harnessError("The generated integration database name is invalid."); + } + return `"${databaseName}"`; +} + +function makeDatabaseName(label: string): string { + const safeLabel = label + .toLowerCase() + .replaceAll(/[^a-z0-9]+/gu, "_") + .replaceAll(/^_+|_+$/gu, "") + .slice(0, 18); + databaseCounter += 1; + return `ekk_mig_${process.pid}_${databaseCounter}_${safeLabel || "scenario"}_${randomBytes(4).toString("hex")}`; +} + +function databaseUrl(databaseName: string, applicationName?: string): string { + const parsed = new URL(baseDatabaseUrl()); + parsed.pathname = `/${databaseName}`; + if (applicationName === undefined) { + parsed.searchParams.delete("application_name"); + } else { + parsed.searchParams.set("application_name", applicationName); + } + return parsed.toString(); +} + +async function openAdministrativeClient( + connectionString: string, + applicationName: string, +): Promise { + const client = new Client({ application_name: applicationName, connectionString }); + client.on("error", () => undefined); + try { + await client.connect(); + return client; + } catch { + try { + await client.end(); + } catch { + // The public harness failure remains static when connection cleanup also fails. + } + throw harnessError("The migration integration database connection failed."); + } +} + +async function safeQuery( + client: Client, + text: string, + values: unknown[] = [], + failureMessage = "An integration database administration operation failed.", +): Promise> { + try { + return await client.query(text, values); + } catch { + throw harnessError(failureMessage); + } +} + +async function safeEnd(client: Client): Promise { + try { + await client.end(); + } catch { + throw harnessError("An integration database connection could not be closed."); + } +} + +async function createDatabase(databaseName: string): Promise { + const admin = await openAdministrativeClient(baseDatabaseUrl(), "ekklesia_migration_test_setup"); + let operationError: unknown; + try { + await safeQuery(admin, `CREATE DATABASE ${quoteDatabaseName(databaseName)} TEMPLATE template0`); + } catch (error) { + operationError = error; + } + try { + await safeEnd(admin); + } catch (error) { + if (operationError === undefined) { + operationError = error; + } + } + if (operationError !== undefined) { + throw operationError; + } +} + +async function dropDatabase(databaseName: string): Promise { + const admin = await openAdministrativeClient( + baseDatabaseUrl(), + "ekklesia_migration_test_cleanup", + ); + let operationError: unknown; + try { + await safeQuery( + admin, + "SELECT pg_terminate_backend(pid) FROM pg_stat_activity WHERE datname = $1 AND pid <> pg_backend_pid()", + [databaseName], + ); + await safeQuery( + admin, + `DROP DATABASE IF EXISTS ${quoteDatabaseName(databaseName)} WITH (FORCE)`, + ); + trackedDatabases.delete(databaseName); + } catch (error) { + operationError = error; + } + try { + await safeEnd(admin); + } catch (error) { + if (operationError === undefined) { + operationError = error; + } + } + if (operationError !== undefined) { + throw operationError; + } +} + +interface ScenarioDatabase { + readonly client: Client; + readonly name: string; + readonly url: string; +} + +async function withIsolatedDatabase( + label: string, + operation: (database: ScenarioDatabase) => Promise, +): Promise { + const name = makeDatabaseName(label); + await createDatabase(name); + trackedDatabases.add(name); + + let client: Client | undefined; + let outcome: + | { readonly error: unknown; readonly success: false } + | { readonly success: true; readonly value: T }; + try { + const url = databaseUrl(name); + client = await openAdministrativeClient(url, "ekklesia_migration_test_inspection"); + outcome = { success: true, value: await operation(Object.freeze({ client, name, url })) }; + } catch (error) { + outcome = { error, success: false }; + } + + let cleanupFailed = false; + if (client !== undefined) { + try { + await safeEnd(client); + } catch { + cleanupFailed = true; + } + } + try { + await dropDatabase(name); + } catch { + cleanupFailed = true; + } + + if (!outcome.success) { + throw outcome.error; + } + if (cleanupFailed) { + throw harnessError("An isolated migration integration database could not be removed."); + } + return outcome.value; +} + +afterAll(async () => { + let cleanupFailed = false; + for (const databaseName of [...trackedDatabases]) { + try { + await dropDatabase(databaseName); + } catch { + cleanupFailed = true; + } + } + if (cleanupFailed) { + throw harnessError("Migration integration database cleanup was incomplete."); + } +}); + +interface MigrationOptions { + readonly approvalRequired?: boolean; + readonly sql: string; + readonly tag: string; +} + +interface ModuleFixture { + readonly id: string; + readonly migrations: readonly DiscoveredModuleMigration[]; +} + +function migration( + moduleId: string, + index: number, + options: MigrationOptions, +): DiscoveredModuleMigration { + const approvalRequired = options.approvalRequired ?? false; + return Object.freeze({ + approvalRequired, + breakpoints: true, + checksum: createHash("sha256").update(options.sql).digest("hex"), + destructive: Object.freeze( + approvalRequired + ? { + mode: "approval-required" as const, + reason: "This integration fixture requires an exact external approval.", + } + : { mode: "none" as const }, + ), + file: `${options.tag}.sql`, + identity: `${moduleId}:${options.tag}`, + index, + moduleId, + sql: options.sql, + tag: options.tag, + timestamp: 1_800_000_000_000 + index, + }); +} + +function aggregate(...moduleFixtures: readonly ModuleFixture[]): ModuleMigrationAggregate { + const modules = Object.freeze( + moduleFixtures.map((moduleFixture) => + Object.freeze({ + migrations: Object.freeze([...moduleFixture.migrations]), + moduleId: moduleFixture.id, + }), + ), + ); + return Object.freeze({ + discoveredModules: Object.freeze([]), + migrations: Object.freeze(modules.flatMap((moduleFixture) => moduleFixture.migrations)), + modules, + }); +} + +function moduleFixture( + id: string, + migrations: readonly DiscoveredModuleMigration[], +): ModuleFixture { + return Object.freeze({ id, migrations: Object.freeze([...migrations]) }); +} + +const peopleInitialSql = `CREATE TABLE "people_records" ( + "id" bigint GENERATED ALWAYS AS IDENTITY PRIMARY KEY, + "display_name" text NOT NULL +);\n`; +const calendarInitialSql = `CREATE TABLE "calendar_events" ( + "id" bigint GENERATED ALWAYS AS IDENTITY PRIMARY KEY, + "owner_id" bigint NOT NULL, + "title" text NOT NULL +);\n`; +const peopleLaterSql = `ALTER TABLE "people_records" ADD COLUMN "email" text;\n`; + +function peopleInitial(sql = peopleInitialSql): DiscoveredModuleMigration { + return migration("people", 0, { sql, tag: "0000_initial" }); +} + +function calendarInitial(): DiscoveredModuleMigration { + return migration("calendar", 0, { sql: calendarInitialSql, tag: "0000_initial" }); +} + +function peopleLater(sql = peopleLaterSql): DiscoveredModuleMigration { + return migration("people", 1, { sql, tag: "0001_add_email" }); +} + +interface RelationExistenceRow extends QueryResultRow { + readonly relation_name: string | null; +} + +interface UserObjectRow extends QueryResultRow { + readonly object_kind: string; + readonly object_name: string; + readonly schema_name: string; +} + +interface ColumnRow extends QueryResultRow { + readonly column_default: string | null; + readonly column_name: string; + readonly data_type: string; + readonly is_nullable: string; + readonly ordinal_position: number; + readonly table_name: string; + readonly table_schema: string; +} + +interface ConstraintRow extends QueryResultRow { + readonly constraint_definition: string; + readonly constraint_name: string; + readonly table_name: string; + readonly table_schema: string; +} + +interface SequenceRow extends QueryResultRow { + readonly cache_size: string; + readonly cycle: boolean; + readonly increment_by: string; + readonly last_value: string | null; + readonly sequence_name: string; + readonly sequence_schema: string; +} + +interface HistoryRow extends QueryResultRow { + readonly applied_at: Date; + readonly applied_order: number; + readonly checksum: string; + readonly identity: string; + readonly migration_index: number; + readonly migration_tag: string; + readonly module_id: string; +} + +interface BooleanRow extends QueryResultRow { + readonly value: boolean; +} + +interface CountRow extends QueryResultRow { + readonly value: number; +} + +interface DatabaseState { + readonly columns: readonly ColumnRow[]; + readonly constraints: readonly ConstraintRow[]; + readonly history: readonly HistoryRow[]; + readonly objects: readonly UserObjectRow[]; + readonly sequences: readonly SequenceRow[]; + readonly supplementalObjects: readonly UserObjectRow[]; +} + +async function relationExists(client: Client, qualifiedName: string): Promise { + const result = await safeQuery( + client, + "SELECT to_regclass($1) AS relation_name", + [qualifiedName], + ); + return result.rows[0]?.relation_name !== null && result.rows[0]?.relation_name !== undefined; +} + +async function readHistory(client: Client): Promise { + if (!(await relationExists(client, historyTable))) { + return Object.freeze([]); + } + const result = await safeQuery( + client, + ` + SELECT + applied_order::integer AS applied_order, + identity, + module_id, + migration_tag, + migration_index, + checksum, + applied_at + FROM public.ekklesia_core_migrations + ORDER BY applied_order ASC + `, + ); + return Object.freeze(result.rows.map((row) => Object.freeze({ ...row }))); +} + +async function databaseState(client: Client): Promise { + const objects = await safeQuery( + client, + ` + SELECT + namespace.nspname AS schema_name, + relation.relname AS object_name, + relation.relkind::text AS object_kind + FROM pg_catalog.pg_class AS relation + INNER JOIN pg_catalog.pg_namespace AS namespace + ON namespace.oid = relation.relnamespace + WHERE namespace.nspname NOT LIKE 'pg\\_%' ESCAPE '\\' + AND namespace.nspname <> 'information_schema' + ORDER BY namespace.nspname, relation.relname, relation.relkind + `, + [], + "Integration user-object inspection failed.", + ); + const columns = await safeQuery( + client, + ` + SELECT + table_schema, + table_name, + column_name, + ordinal_position, + data_type, + is_nullable, + column_default + FROM information_schema.columns + WHERE table_schema NOT LIKE 'pg\\_%' ESCAPE '\\' + AND table_schema <> 'information_schema' + ORDER BY table_schema, table_name, ordinal_position + `, + [], + "Integration column inspection failed.", + ); + const constraints = await safeQuery( + client, + ` + SELECT + namespace.nspname AS table_schema, + relation.relname AS table_name, + catalog_constraint.conname AS constraint_name, + pg_catalog.pg_get_constraintdef(catalog_constraint.oid, true) AS constraint_definition + FROM pg_catalog.pg_constraint AS catalog_constraint + INNER JOIN pg_catalog.pg_class AS relation + ON relation.oid = catalog_constraint.conrelid + INNER JOIN pg_catalog.pg_namespace AS namespace ON namespace.oid = relation.relnamespace + WHERE namespace.nspname NOT LIKE 'pg\\_%' ESCAPE '\\' + AND namespace.nspname <> 'information_schema' + ORDER BY namespace.nspname, relation.relname, catalog_constraint.conname + `, + [], + "Integration constraint inspection failed.", + ); + const sequences = await safeQuery( + client, + ` + SELECT + schemaname AS sequence_schema, + sequencename AS sequence_name, + increment_by::text, + cycle, + cache_size::text, + last_value::text + FROM pg_catalog.pg_sequences + WHERE schemaname NOT LIKE 'pg\\_%' ESCAPE '\\' + AND schemaname <> 'information_schema' + ORDER BY schemaname, sequencename + `, + [], + "Integration sequence inspection failed.", + ); + const supplementalObjects = await safeQuery( + client, + ` + SELECT + 'schema' AS object_kind, + namespace.nspname AS object_name, + '' AS schema_name + FROM pg_catalog.pg_namespace AS namespace + WHERE namespace.nspname NOT LIKE 'pg\\_%' ESCAPE '\\' + AND namespace.nspname <> 'information_schema' + UNION ALL + SELECT + 'extension' AS object_kind, + catalog_extension.extname AS object_name, + namespace.nspname AS schema_name + FROM pg_catalog.pg_extension AS catalog_extension + INNER JOIN pg_catalog.pg_namespace AS namespace + ON namespace.oid = catalog_extension.extnamespace + UNION ALL + SELECT + 'routine' AS object_kind, + catalog_routine.oid::regprocedure::text AS object_name, + namespace.nspname AS schema_name + FROM pg_catalog.pg_proc AS catalog_routine + INNER JOIN pg_catalog.pg_namespace AS namespace + ON namespace.oid = catalog_routine.pronamespace + WHERE namespace.nspname NOT LIKE 'pg\\_%' ESCAPE '\\' + AND namespace.nspname <> 'information_schema' + UNION ALL + SELECT + CASE user_type.typtype + WHEN 'd' THEN 'domain' + WHEN 'e' THEN 'enum' + WHEN 'm' THEN 'multirange' + WHEN 'r' THEN 'range' + END AS object_kind, + user_type.typname AS object_name, + namespace.nspname AS schema_name + FROM pg_catalog.pg_type AS user_type + INNER JOIN pg_catalog.pg_namespace AS namespace + ON namespace.oid = user_type.typnamespace + WHERE namespace.nspname NOT LIKE 'pg\\_%' ESCAPE '\\' + AND namespace.nspname <> 'information_schema' + AND user_type.typtype IN ('d', 'e', 'm', 'r') + ORDER BY object_kind, schema_name, object_name + `, + [], + "Integration supplemental-object inspection failed.", + ); + const history = await readHistory(client); + + return Object.freeze({ + columns: Object.freeze(columns.rows.map((row) => Object.freeze({ ...row }))), + constraints: Object.freeze(constraints.rows.map((row) => Object.freeze({ ...row }))), + history, + objects: Object.freeze(objects.rows.map((row) => Object.freeze({ ...row }))), + sequences: Object.freeze(sequences.rows.map((row) => Object.freeze({ ...row }))), + supplementalObjects: Object.freeze( + supplementalObjects.rows.map((row) => Object.freeze({ ...row })), + ), + }); +} + +async function columnExists( + client: Client, + tableName: string, + columnName: string, +): Promise { + const result = await safeQuery( + client, + ` + SELECT EXISTS ( + SELECT 1 + FROM information_schema.columns + WHERE table_schema = 'public' AND table_name = $1 AND column_name = $2 + ) AS value + `, + [tableName, columnName], + ); + return result.rows[0]?.value === true; +} + +async function captureRunnerError( + operation: Promise, + code: MigrationRunnerErrorCode, +): Promise { + try { + await operation; + } catch (error) { + if (!(error instanceof MigrationRunnerError)) { + throw harnessError("A migration operation returned an unsafe error type."); + } + const migrationError = error as MigrationRunnerError; + expect(migrationError.code).toBe(code); + return migrationError; + } + throw harnessError("A migration operation unexpectedly succeeded."); +} + +function approvalFor(migrationFixture: DiscoveredModuleMigration): MigrationApproval { + return Object.freeze({ + checksum: migrationFixture.checksum, + identity: migrationFixture.identity, + moduleId: migrationFixture.moduleId, + }); +} + +function approvalProvider(approvals: readonly MigrationApproval[]): MigrationApprovalProvider { + const frozenApprovals = Object.freeze( + approvals.map((approval) => Object.freeze({ ...approval })), + ); + return Object.freeze({ + getApprovals: () => Promise.resolve(frozenApprovals), + }); +} + +async function acquireAdministrativeLock( + client: Client, + namespace: number, + key: number, +): Promise { + await safeQuery(client, "SELECT pg_advisory_lock($1::integer, $2::integer)", [namespace, key]); +} + +async function releaseAdministrativeLock( + client: Client, + namespace: number, + key: number, +): Promise { + const result = await safeQuery( + client, + "SELECT pg_advisory_unlock($1::integer, $2::integer) AS value", + [namespace, key], + ); + if (result.rows[0]?.value !== true) { + throw harnessError("An integration advisory lock was not released."); + } +} + +async function expectMigrationLockAvailable(client: Client): Promise { + const result = await safeQuery( + client, + "SELECT pg_try_advisory_lock($1::integer, $2::integer) AS value", + [migrationLockNamespace, migrationLockKey], + ); + expect(result.rows[0]?.value).toBe(true); + if (result.rows[0]?.value === true) { + await releaseAdministrativeLock(client, migrationLockNamespace, migrationLockKey); + } +} + +async function waitForDatabaseCondition( + check: () => Promise, + timeoutMs = 5_000, +): Promise { + const deadline = performance.now() + timeoutMs; + while (performance.now() < deadline) { + if (await check()) { + return; + } + await new Promise((resolve) => setTimeout(resolve, 25)); + } + throw harnessError("A bounded integration database condition was not observed."); +} + +describe.sequential("live PostgreSQL module migration execution", () => { + it("keeps fresh planning read-only", async () => { + await withIsolatedDatabase("fresh_plan", async (database) => { + const people = peopleInitial(); + const calendar = calendarInitial(); + const inventory = aggregate( + moduleFixture("people", [people]), + moduleFixture("calendar", [calendar]), + ); + const before = await databaseState(database.client); + + const plan = await planModuleMigrations({ + aggregate: inventory, + database: createPostgresMigrationDatabase(database.url), + }); + + expect(plan.entries.map((entry) => [entry.identity, entry.status])).toEqual([ + [people.identity, "pending"], + [calendar.identity, "pending"], + ]); + expect(plan).toMatchObject({ appliedCount: 0, approvalRequiredCount: 0, pendingCount: 2 }); + expect(await databaseState(database.client)).toEqual(before); + expect(await relationExists(database.client, historyTable)).toBe(false); + expect(await relationExists(database.client, "public.people_records")).toBe(false); + expect(await relationExists(database.client, "public.calendar_events")).toBe(false); + }); + }); + + it("applies a fresh dependency-ordered inventory and records exact history", async () => { + await withIsolatedDatabase("fresh_apply", async (database) => { + const people = peopleInitial(); + const calendar = calendarInitial(); + const inventory = aggregate( + moduleFixture("people", [people]), + moduleFixture("calendar", [calendar]), + ); + + const result = await applyModuleMigrations({ + aggregate: inventory, + database: createPostgresMigrationDatabase(database.url), + }); + + expect(result.applied.map((entry) => entry.identity)).toEqual([ + people.identity, + calendar.identity, + ]); + expect(await relationExists(database.client, "public.people_records")).toBe(true); + expect(await relationExists(database.client, "public.calendar_events")).toBe(true); + const history = await readHistory(database.client); + expect(history.map(({ applied_at: _appliedAt, ...row }) => row)).toEqual([ + { + applied_order: 1, + checksum: people.checksum, + identity: people.identity, + migration_index: 0, + migration_tag: people.tag, + module_id: people.moduleId, + }, + { + applied_order: 2, + checksum: calendar.checksum, + identity: calendar.identity, + migration_index: 0, + migration_tag: calendar.tag, + module_id: calendar.moduleId, + }, + ]); + expect(history).toHaveLength(2); + expect( + history.every( + (row) => row.applied_at instanceof Date && !Number.isNaN(row.applied_at.getTime()), + ), + ).toBe(true); + }); + }); + + it("performs a genuine no-op without changing schema, data, sequences, or history", async () => { + await withIsolatedDatabase("no_op", async (database) => { + const initial = peopleInitial(); + const inventory = aggregate(moduleFixture("people", [initial])); + const migrationDatabase = createPostgresMigrationDatabase(database.url); + await applyModuleMigrations({ aggregate: inventory, database: migrationDatabase }); + await safeQuery( + database.client, + "INSERT INTO public.people_records (display_name) VALUES ($1)", + ["No-op sentinel"], + ); + const before = await databaseState(database.client); + const beforeData = await safeQuery( + database.client, + "SELECT id::integer, display_name FROM public.people_records ORDER BY id", + ); + + const result = await applyModuleMigrations({ + aggregate: inventory, + database: migrationDatabase, + }); + + expect(result).toEqual({ applied: [], appliedCount: 0, noOp: true }); + expect(await databaseState(database.client)).toEqual(before); + const afterData = await safeQuery( + database.client, + "SELECT id::integer, display_name FROM public.people_records ORDER BY id", + ); + expect(afterData.rows).toEqual(beforeData.rows); + }); + }); + + it("executes only a newly presented later migration", async () => { + await withIsolatedDatabase("later", async (database) => { + const initial = peopleInitial(); + const later = peopleLater(); + const initialInventory = aggregate(moduleFixture("people", [initial])); + const laterInventory = aggregate(moduleFixture("people", [initial, later])); + const migrationDatabase = createPostgresMigrationDatabase(database.url); + await applyModuleMigrations({ aggregate: initialInventory, database: migrationDatabase }); + const originalHistory = await readHistory(database.client); + + const result = await applyModuleMigrations({ + aggregate: laterInventory, + database: migrationDatabase, + }); + + expect(result.applied.map((entry) => entry.identity)).toEqual([later.identity]); + expect(await columnExists(database.client, "people_records", "email")).toBe(true); + const history = await readHistory(database.client); + expect(history).toHaveLength(2); + expect(history[0]).toEqual(originalHistory[0]); + expect(history[1]).toMatchObject({ + applied_order: 2, + checksum: later.checksum, + identity: later.identity, + migration_index: 1, + migration_tag: later.tag, + module_id: later.moduleId, + }); + }); + }); + + it("rejects edited applied contents during both planning and applying", async () => { + await withIsolatedDatabase("edited", async (database) => { + const initial = peopleInitial(); + const edited = peopleInitial( + `CREATE TABLE "people_records" ("id" bigint PRIMARY KEY, "edited_marker" text);\n`, + ); + const originalInventory = aggregate(moduleFixture("people", [initial])); + const editedInventory = aggregate(moduleFixture("people", [edited])); + const migrationDatabase = createPostgresMigrationDatabase(database.url); + await applyModuleMigrations({ aggregate: originalInventory, database: migrationDatabase }); + const before = await databaseState(database.client); + + const planError = await captureRunnerError( + planModuleMigrations({ aggregate: editedInventory, database: migrationDatabase }), + "MIGRATION_CHECKSUM_MISMATCH", + ); + expect(planError.migrationIdentity).toBe(initial.identity); + expect(await databaseState(database.client)).toEqual(before); + + const applyError = await captureRunnerError( + applyModuleMigrations({ aggregate: editedInventory, database: migrationDatabase }), + "MIGRATION_CHECKSUM_MISMATCH", + ); + expect(applyError.migrationIdentity).toBe(initial.identity); + expect(await databaseState(database.client)).toEqual(before); + expect(await columnExists(database.client, "people_records", "edited_marker")).toBe(false); + }); + }); + + it("rejects an unknown applied identity without repair or further execution", async () => { + await withIsolatedDatabase("unknown_history", async (database) => { + const initial = peopleInitial(); + const later = peopleLater(); + const initialInventory = aggregate(moduleFixture("people", [initial])); + const completeInventory = aggregate(moduleFixture("people", [initial, later])); + const migrationDatabase = createPostgresMigrationDatabase(database.url); + await applyModuleMigrations({ aggregate: initialInventory, database: migrationDatabase }); + await safeQuery( + database.client, + ` + INSERT INTO public.ekklesia_core_migrations + (identity, module_id, migration_tag, migration_index, checksum) + VALUES ($1, $2, $3, $4, $5) + `, + ["ghost:0000_unknown", "ghost", "0000_unknown", 0, "f".repeat(64)], + ); + const corrupted = await databaseState(database.client); + + await captureRunnerError( + planModuleMigrations({ aggregate: completeInventory, database: migrationDatabase }), + "MIGRATION_UNKNOWN_APPLIED", + ); + await captureRunnerError( + applyModuleMigrations({ aggregate: completeInventory, database: migrationDatabase }), + "MIGRATION_UNKNOWN_APPLIED", + ); + + expect(await databaseState(database.client)).toEqual(corrupted); + expect(await columnExists(database.client, "people_records", "email")).toBe(false); + expect((await readHistory(database.client)).map((row) => row.identity)).toEqual([ + initial.identity, + "ghost:0000_unknown", + ]); + }); + }); + + it("rejects an invalid per-module history prefix without silent repair", async () => { + await withIsolatedDatabase("history_gap", async (database) => { + const initial = peopleInitial(); + const later = peopleLater(); + const unreachable = migration("people", 2, { + sql: `CREATE TABLE "people_unreachable" ("id" bigint PRIMARY KEY);\n`, + tag: "0002_unreachable", + }); + const appliedInventory = aggregate(moduleFixture("people", [initial, later])); + const completeInventory = aggregate(moduleFixture("people", [initial, later, unreachable])); + const migrationDatabase = createPostgresMigrationDatabase(database.url); + await applyModuleMigrations({ aggregate: appliedInventory, database: migrationDatabase }); + await safeQuery( + database.client, + "DELETE FROM public.ekklesia_core_migrations WHERE identity = $1", + [initial.identity], + ); + const corrupted = await databaseState(database.client); + + await captureRunnerError( + planModuleMigrations({ aggregate: completeInventory, database: migrationDatabase }), + "MIGRATION_HISTORY_INVALID", + ); + await captureRunnerError( + applyModuleMigrations({ aggregate: completeInventory, database: migrationDatabase }), + "MIGRATION_HISTORY_INVALID", + ); + + expect(await databaseState(database.client)).toEqual(corrupted); + expect(await relationExists(database.client, "public.people_unreachable")).toBe(false); + expect((await readHistory(database.client)).map((row) => row.identity)).toEqual([ + later.identity, + ]); + }); + }); + + it("rolls back the entire current pending batch after a real PostgreSQL failure", async () => { + await withIsolatedDatabase("rollback", async (database) => { + const committed = migration("rollback", 0, { + sql: `CREATE TABLE "rollback_committed" ("id" bigint PRIMARY KEY);\n`, + tag: "0000_committed", + }); + const pendingFirst = migration("rollback", 1, { + sql: `CREATE TABLE "rollback_pending_one" ("id" bigint PRIMARY KEY);\n`, + tag: "0001_pending_one", + }); + const pendingFailure = migration("rollback", 2, { + sql: `CREATE TABLE "rollback_pending_two" ("id" bigint PRIMARY KEY);\n--> statement-breakpoint\nSELECT 1 / 0;\n`, + tag: "0002_pending_failure", + }); + const committedInventory = aggregate(moduleFixture("rollback", [committed])); + const completeInventory = aggregate( + moduleFixture("rollback", [committed, pendingFirst, pendingFailure]), + ); + const migrationDatabase = createPostgresMigrationDatabase(database.url); + await applyModuleMigrations({ aggregate: committedInventory, database: migrationDatabase }); + + const error = await captureRunnerError( + applyModuleMigrations({ aggregate: completeInventory, database: migrationDatabase }), + "MIGRATION_EXECUTION_FAILED", + ); + + expect(error.migrationIdentity).toBe(pendingFailure.identity); + expect(await relationExists(database.client, "public.rollback_committed")).toBe(true); + expect(await relationExists(database.client, "public.rollback_pending_one")).toBe(false); + expect(await relationExists(database.client, "public.rollback_pending_two")).toBe(false); + expect((await readHistory(database.client)).map((row) => row.identity)).toEqual([ + committed.identity, + ]); + await expectMigrationLockAvailable(database.client); + }); + }); + + it("honors Drizzle statement breakpoints without splitting procedural semicolons", async () => { + await withIsolatedDatabase("breakpoints", async (database) => { + const sql = `CREATE TABLE "breakpoints_steps" ( + "id" integer PRIMARY KEY, + "note" text NOT NULL +); +--> statement-breakpoint +DO $body$ +BEGIN + INSERT INTO "breakpoints_steps" ("id", "note") VALUES (1, 'first;still-one-statement'); +END +$body$; +--> statement-breakpoint +ALTER TABLE "breakpoints_steps" ADD COLUMN "sequence_value" integer; +UPDATE "breakpoints_steps" SET "sequence_value" = 1 WHERE "id" = 1; +INSERT INTO "breakpoints_steps" ("id", "note", "sequence_value") +SELECT 2, 'second', "sequence_value" + 1 FROM "breakpoints_steps" WHERE "id" = 1; +`; + const initial = migration("breakpoints", 0, { sql, tag: "0000_ordered_steps" }); + + await applyModuleMigrations({ + aggregate: aggregate(moduleFixture("breakpoints", [initial])), + database: createPostgresMigrationDatabase(database.url), + }); + + const rows = await safeQuery( + database.client, + `SELECT id, note, sequence_value FROM public.breakpoints_steps ORDER BY id`, + ); + expect(rows.rows).toEqual([ + { id: 1, note: "first;still-one-statement", sequence_value: 1 }, + { id: 2, note: "second", sequence_value: 2 }, + ]); + expect((await readHistory(database.client)).map((row) => row.identity)).toEqual([ + initial.identity, + ]); + }); + }); + + it("serializes two real migrators with the session advisory lock", async () => { + await withIsolatedDatabase("concurrency", async (database) => { + await acquireAdministrativeLock( + database.client, + concurrencyBarrierNamespace, + concurrencyBarrierKey, + ); + let barrierHeld = true; + const sql = `SELECT pg_advisory_xact_lock(${concurrencyBarrierNamespace}::integer, ${concurrencyBarrierKey}::integer);\n--> statement-breakpoint\nCREATE TABLE "concurrency_records" ("id" bigint PRIMARY KEY);\n`; + const initial = migration("concurrency", 0, { sql, tag: "0000_serialized" }); + const inventory = aggregate(moduleFixture("concurrency", [initial])); + const applicationName = "ekklesia_migration_test_concurrency"; + const productionUrl = databaseUrl(database.name, applicationName); + const firstApply = applyModuleMigrations({ + aggregate: inventory, + database: createPostgresMigrationDatabase(productionUrl), + lockTimeoutMs: 5_000, + }); + + try { + await waitForDatabaseCondition(async () => { + const result = await safeQuery( + database.client, + ` + SELECT count(*)::integer AS value + FROM pg_catalog.pg_locks AS lock + INNER JOIN pg_catalog.pg_stat_activity AS activity ON activity.pid = lock.pid + WHERE lock.locktype = 'advisory' + AND lock.classid = $1::oid + AND lock.objid = $2::oid + AND lock.objsubid = 2 + AND lock.granted + AND activity.datname = current_database() + AND activity.application_name = $3 + `, + [migrationLockNamespace, migrationLockKey, applicationName], + ); + return result.rows[0]?.value === 1; + }); + + const secondError = await captureRunnerError( + applyModuleMigrations({ + aggregate: inventory, + database: createPostgresMigrationDatabase(productionUrl), + lockTimeoutMs: 250, + }), + "MIGRATION_LOCK_TIMEOUT", + ); + expect(secondError.migrationIdentity).toBeUndefined(); + expect(await relationExists(database.client, historyTable)).toBe(false); + expect(await relationExists(database.client, "public.concurrency_records")).toBe(false); + + await releaseAdministrativeLock( + database.client, + concurrencyBarrierNamespace, + concurrencyBarrierKey, + ); + barrierHeld = false; + const firstResult = await firstApply; + expect(firstResult.applied.map((entry) => entry.identity)).toEqual([initial.identity]); + } finally { + if (barrierHeld) { + await releaseAdministrativeLock( + database.client, + concurrencyBarrierNamespace, + concurrencyBarrierKey, + ); + } + await Promise.allSettled([firstApply]); + } + + expect(await relationExists(database.client, "public.concurrency_records")).toBe(true); + expect((await readHistory(database.client)).map((row) => row.identity)).toEqual([ + initial.identity, + ]); + }); + }); + + it("times out on the real lock, releases it, and permits a later apply", async () => { + await withIsolatedDatabase("lock_timeout", async (database) => { + const initial = migration("locks", 0, { + sql: `CREATE TABLE "locks_records" ("id" bigint PRIMARY KEY);\n`, + tag: "0000_initial", + }); + const inventory = aggregate(moduleFixture("locks", [initial])); + const migrationDatabase = createPostgresMigrationDatabase(database.url); + await acquireAdministrativeLock(database.client, migrationLockNamespace, migrationLockKey); + let lockHeld = true; + const before = await databaseState(database.client); + const startedAt = performance.now(); + try { + await captureRunnerError( + applyModuleMigrations({ + aggregate: inventory, + database: migrationDatabase, + lockTimeoutMs: 250, + }), + "MIGRATION_LOCK_TIMEOUT", + ); + } finally { + await releaseAdministrativeLock(database.client, migrationLockNamespace, migrationLockKey); + lockHeld = false; + } + const elapsedMs = performance.now() - startedAt; + expect(elapsedMs).toBeGreaterThanOrEqual(150); + expect(elapsedMs).toBeLessThan(2_000); + expect(await databaseState(database.client)).toEqual(before); + expect(lockHeld).toBe(false); + + const result = await applyModuleMigrations({ + aggregate: inventory, + database: migrationDatabase, + }); + expect(result.applied.map((entry) => entry.identity)).toEqual([initial.identity]); + expect(await relationExists(database.client, "public.locks_records")).toBe(true); + }); + }); + + it("does not strand the migration lock after execution failure", async () => { + await withIsolatedDatabase("failed_lock_release", async (database) => { + const failing = migration("lock_failure", 0, { + sql: `CREATE TABLE "lock_failure_records" ("id" bigint PRIMARY KEY);\n--> statement-breakpoint\nSELECT 1 / 0;\n`, + tag: "0000_initial", + }); + const corrected = migration("lock_failure", 0, { + sql: `CREATE TABLE "lock_failure_records" ("id" bigint PRIMARY KEY);\n`, + tag: "0000_initial", + }); + const migrationDatabase = createPostgresMigrationDatabase(database.url); + + await captureRunnerError( + applyModuleMigrations({ + aggregate: aggregate(moduleFixture("lock_failure", [failing])), + database: migrationDatabase, + }), + "MIGRATION_EXECUTION_FAILED", + ); + await expectMigrationLockAvailable(database.client); + expect(await relationExists(database.client, historyTable)).toBe(false); + expect(await relationExists(database.client, "public.lock_failure_records")).toBe(false); + + const result = await applyModuleMigrations({ + aggregate: aggregate(moduleFixture("lock_failure", [corrected])), + database: migrationDatabase, + }); + expect(result.applied.map((entry) => entry.identity)).toEqual([corrected.identity]); + }); + }); + + it("requires an exact destructive approval before any persistent mutation", async () => { + await withIsolatedDatabase("approvals", async (database) => { + const destructive = migration("destructive", 0, { + approvalRequired: true, + sql: `CREATE TABLE "destructive_records" ("id" bigint PRIMARY KEY);\n`, + tag: "0000_replace_records", + }); + const inventory = aggregate(moduleFixture("destructive", [destructive])); + const migrationDatabase = createPostgresMigrationDatabase(database.url); + const exact = approvalFor(destructive); + const rejectedApprovals: readonly (readonly MigrationApproval[] | undefined)[] = [ + undefined, + [Object.freeze({ ...exact, moduleId: "wrong-module" })], + [Object.freeze({ ...exact, identity: "destructive:0000_wrong" })], + [Object.freeze({ ...exact, checksum: "0".repeat(64) })], + ]; + + for (const supplied of rejectedApprovals) { + const before = await databaseState(database.client); + const operation = + supplied === undefined + ? applyModuleMigrations({ aggregate: inventory, database: migrationDatabase }) + : applyModuleMigrations({ + aggregate: inventory, + approvals: approvalProvider(supplied), + database: migrationDatabase, + }); + const error = await captureRunnerError(operation, "MIGRATION_APPROVAL_REQUIRED"); + expect(error.migrationIdentity).toBe(destructive.identity); + expect(await databaseState(database.client)).toEqual(before); + expect(await relationExists(database.client, historyTable)).toBe(false); + expect(await relationExists(database.client, "public.destructive_records")).toBe(false); + } + + const result = await applyModuleMigrations({ + aggregate: inventory, + approvals: approvalProvider([exact]), + database: migrationDatabase, + }); + expect(result.applied.map((entry) => entry.identity)).toEqual([destructive.identity]); + expect(await relationExists(database.client, "public.destructive_records")).toBe(true); + expect((await readHistory(database.client)).map((row) => row.identity)).toEqual([ + destructive.identity, + ]); + }); + }); + + it("sanitizes a real PostgreSQL failure and normal command reporting", async () => { + await withIsolatedDatabase("sanitization", async (database) => { + const hostilePassword = "fake-password-7f7e620d"; + const hostileUrl = `postgresql://hostile:${hostilePassword}@private-db.invalid/secret`; + const hostileHostname = "private-db.invalid"; + const hostilePath = "/srv/ekklesia/private/migration.sql"; + const hostileSqlMarker = "HOSTILE_RAW_SQL_MARKER_45f28a"; + const sql = `DO $body$ +BEGIN + RAISE EXCEPTION '${hostilePassword} ${hostileUrl} ${hostileHostname} ${hostilePath} ${hostileSqlMarker}'; +END +$body$;\n`; + const failing = migration("sanitization", 0, { + sql, + tag: "0000_hostile_failure", + }); + const capturedOutput: string[] = []; + const errorSpy = vi + .spyOn(console, "error") + .mockImplementation((...values: unknown[]) => + capturedOutput.push(values.map(String).join(" ")), + ); + const logSpy = vi + .spyOn(console, "log") + .mockImplementation((...values: unknown[]) => + capturedOutput.push(values.map(String).join(" ")), + ); + let publicError: MigrationRunnerError; + try { + publicError = await captureRunnerError( + applyModuleMigrations({ + aggregate: aggregate(moduleFixture("sanitization", [failing])), + database: createPostgresMigrationDatabase(database.url), + }), + "MIGRATION_EXECUTION_FAILED", + ); + reportMigrationCommandError(publicError); + } finally { + errorSpy.mockRestore(); + logSpy.mockRestore(); + } + + const publicText = [ + String(publicError), + publicError.message, + publicError.stack ?? "", + JSON.stringify(publicError), + ...capturedOutput, + ].join("\n"); + expect(publicError.migrationIdentity).toBe(failing.identity); + expect(publicText.includes("MIGRATION_EXECUTION_FAILED")).toBe(true); + expect(publicText.includes(failing.identity)).toBe(true); + const hostileValues = [ + hostilePassword, + hostileUrl, + hostileHostname, + hostilePath, + hostileSqlMarker, + "RAISE EXCEPTION", + database.url, + new URL(database.url).password, + new URL(database.url).hostname, + ]; + expect(hostileValues.some((hostileValue) => publicText.includes(hostileValue))).toBe(false); + expect(await relationExists(database.client, historyTable)).toBe(false); + }); + }); +}); diff --git a/docs/architecture/decisions/0009-transactional-module-migrations.md b/docs/architecture/decisions/0009-transactional-module-migrations.md index 8296bc9..3c4ceed 100644 --- a/docs/architecture/decisions/0009-transactional-module-migrations.md +++ b/docs/architecture/decisions/0009-transactional-module-migrations.md @@ -122,10 +122,14 @@ messages, machine-specific paths, causes, stacks, or partial execution results. are reported without hiding the primary failure and without copying underlying exception text. The developer commands read `DATABASE_URL` from the environment. A local uncommitted `.env` may -supply it, but commands never print it. This slice verifies database behavior through a narrow -in-memory database port and unit tests. A real PostgreSQL integration suite for lock contention, -DDL rollback, history creation, and driver behavior is deliberately deferred rather than claimed -by the current verification gate. +supply it, but commands never print it. This decision's implementation slice verified database +behavior through a narrow in-memory database port and unit tests; it deliberately deferred live +PostgreSQL evidence to a follow-up slice rather than claiming it at acceptance time. + +Implementation follow-up (2026-07-21): `pnpm test:migrations:integration` now runs the production +adapter and migration services against disposable PostgreSQL for lock contention, DDL rollback, +history creation, approvals, and sanitized driver failures. The Docker-backed command remains +separate from the code-only `pnpm verify` gate. The transaction and single-client choices follow node-postgres's [transaction guidance](https://node-postgres.com/features/transactions). The non-blocking lock and @@ -162,6 +166,6 @@ that invocation's schema or history changes committed. Destructive migrations ar a trusted approval provider is composed, by design. The all-or-nothing batch can hold locks for longer than per-migration transactions and excludes -operations PostgreSQL forbids in a transaction. The current unit-tested database port gives fast, -deterministic coverage but cannot prove real server behavior. The next infrastructure-focused -slice must add live PostgreSQL tests before production operation is considered complete. +operations PostgreSQL forbids in a transaction. The narrow database port continues to provide fast, +deterministic unit coverage; the dated implementation follow-up above records the separate live +PostgreSQL evidence added after this decision was accepted. diff --git a/docs/architecture/platform-architecture.md b/docs/architecture/platform-architecture.md index 59ae00c..761b9d7 100644 --- a/docs/architecture/platform-architecture.md +++ b/docs/architecture/platform-architecture.md @@ -69,8 +69,9 @@ validated metadata, explicit destructive-change declarations, and exact SHA-256 now compare that inventory with its fixed PostgreSQL history table, produce a read-only plan, and apply the complete pending batch under a session advisory lock and one transaction. Destructive migrations require an exact externally supplied approval bound to module, identity, and checksum. +Hermetic integration coverage now exercises those contracts through the production adapter against +the pinned PostgreSQL container, including real DDL rollback and advisory-lock concurrency. -Target architecture, not yet implemented: production approval-provider composition, live -PostgreSQL migration-runner integration coverage, Better Auth, +Target architecture, not yet implemented: production approval-provider composition, Better Auth, permission enforcement, physical storage-bucket provisioning, Surface SDK, richer public resource and action contracts, API server, dashboard, Builder, and reference feature modules. diff --git a/docs/development/repository-workflow.md b/docs/development/repository-workflow.md index 70f433b..eb9223b 100644 --- a/docs/development/repository-workflow.md +++ b/docs/development/repository-workflow.md @@ -24,6 +24,7 @@ pnpm migrations:plan # read-only comparison with PostgreSQL applied history pnpm migrations:apply # locked, transactional application of the pending batch pnpm typecheck # strict TypeScript pnpm test # Vitest +pnpm test:migrations:integration # production runner against disposable PostgreSQL pnpm build # Turborepo package build graph pnpm verify # every code-only gate; never plan or apply ``` @@ -63,13 +64,24 @@ The default command composition supplies no destructive approvals. There is no ` or source-controlled approval escape hatch; deployments must later compose a trusted external provider that supplies an exact module ID, migration identity, and checksum triple. -The migration runner currently has comprehensive unit coverage through its narrow database port. -Live PostgreSQL tests for real DDL rollback, advisory-lock contention, history-table behavior, and -driver cleanup are deliberately deferred and are not part of `pnpm verify` or CI yet. +Migration-runner unit tests cover orchestration through its narrow database port. Run the separate +live suite when changing migration behavior: -CI runs `pnpm ci:verify`, which adds Compose configuration validation to `pnpm verify`. A separate -CI job runs the live infrastructure smoke test so code-only local verification does not require -Docker. The explicit suffix avoids pnpm's built-in `ci` install command. +```bash +pnpm test:migrations:integration +``` + +It requires Docker with Compose, starts the pinned PostgreSQL image with a generated throwaway +credential and a Docker-assigned loopback port, creates an isolated database for each scenario, +and removes its containers, network, databases, and volume afterward. It exercises the production +adapter and real PostgreSQL transactions, history, advisory locks, approvals, and sanitized errors. +It remains outside the Docker-free `pnpm verify` command. + +CI runs `pnpm ci:verify`, which adds Compose configuration validation to `pnpm verify`. Separate +`infrastructure-smoke` and `migration-smoke` jobs run the live service and migration suites. The +exact pull-request context to add to `main` branch protection after merge is +`CI / migration-smoke (pull_request)`. The explicit `ci:verify` suffix avoids pnpm's built-in `ci` +install command. ## Development infrastructure diff --git a/docs/development/testing.md b/docs/development/testing.md new file mode 100644 index 0000000..db45580 --- /dev/null +++ b/docs/development/testing.md @@ -0,0 +1,60 @@ +# Testing + +Ekklesia separates fast code-level evidence from live infrastructure evidence. + +## Unit and repository verification + +```bash +pnpm test +pnpm verify +``` + +These commands exercise package tests, repository boundaries, type checking, offline module and +migration discovery, formatting, linting, and builds. Migration-runner unit tests use a narrow +database/session port to prove deterministic planning, approval, transaction orchestration, +cleanup, and sanitized errors. They do not claim to prove PostgreSQL behavior. + +`pnpm verify` is intentionally Docker-free. It never plans or applies migrations against a live +database. + +## Live PostgreSQL migration evidence + +Docker and the Docker Compose plugin are required for the live suite: + +```bash +pnpm test:migrations:integration +``` + +The command starts the repository's pinned PostgreSQL image in a unique Compose project, waits for +bounded health readiness, runs the production PostgreSQL adapter and migration services, and then +removes the container, network, and volume. Each scenario also receives a safely named disposable +database. Direct host runs use a Docker-assigned random port bound only to `127.0.0.1`; a +containerized Gitea job joins the isolated Compose network and uses the PostgreSQL service name. + +The live suite proves fresh read-only planning, initial and incremental application, genuine +no-op behavior, exact applied checksums and history, edited or corrupt history rejection, +whole-batch DDL rollback, Drizzle breakpoint execution, real advisory-lock concurrency and +timeouts, lock release after failure, exact destructive approvals, and public-error sanitization. +It does not make migrations automatic during Core startup. + +CI runs the same command in the dedicated `migration-smoke` job. After this change is merged, add +the following pull-request status context to `main` branch protection: + +```text +CI / migration-smoke (pull_request) +``` + +## Safe troubleshooting + +Confirm the prerequisites without printing configuration: + +```bash +docker info +docker compose version +``` + +Then rerun `pnpm test:migrations:integration`. The harness emits fixed failure messages and does +not dump PostgreSQL logs, SQL, container inspection output, or the generated database URL. Do not +enable query logging, print the test environment, run `docker compose config` without redaction, +or paste Docker inspection output into an issue. If the Docker daemon itself stopped during +cleanup, restore it before rerunning so the harness can create a fresh uniquely named project. diff --git a/docs/module-development/README.md b/docs/module-development/README.md index 4e4f2ba..bdaf8fc 100644 --- a/docs/module-development/README.md +++ b/docs/module-development/README.md @@ -275,9 +275,11 @@ ID, exact identity, and exact checksum. Module metadata cannot grant approval, a wildcard, force, or approve-all option. The default developer command supplies no approvals and therefore refuses destructive pending migrations. -Runner behavior is unit-tested through a narrow database port. Live PostgreSQL coverage of DDL -rollback, advisory-lock contention, and driver behavior is a deferred infrastructure slice; the -current verification suite does not claim that coverage. +Runner behavior is unit-tested through a narrow database port. The separate +`pnpm test:migrations:integration` command now exercises the production adapter against disposable +PostgreSQL, including DDL rollback, advisory-lock contention, history behavior, approvals, and +sanitized failures. It remains outside the Docker-free `pnpm verify` gate; see the +[testing guide](../development/testing.md). Core completes discovery before importing any server entrypoint. It then loads and verifies every runtime definition before running hooks, registers all modules dependency-first into an unpublished diff --git a/package.json b/package.json index 04a51a8..904b977 100644 --- a/package.json +++ b/package.json @@ -27,6 +27,7 @@ "migrations:validate": "turbo run build && node core/dist/cli/validate-migrations.js", "modules:validate": "turbo run build && node core/dist/cli/validate-modules.js", "test": "turbo run test test:root", + "test:migrations:integration": "bash scripts/test-migrations-integration.sh", "test:root": "vitest run --config vitest.config.ts", "test:watch": "vitest", "typecheck": "turbo run typecheck typecheck:root", diff --git a/scripts/lib/compose-job-network.sh b/scripts/lib/compose-job-network.sh new file mode 100644 index 0000000..890ca1f --- /dev/null +++ b/scripts/lib/compose-job-network.sh @@ -0,0 +1,112 @@ +#!/usr/bin/env bash + +COMPOSE_JOB_NETWORK_RUNNING_IN_CONTAINER=false +COMPOSE_JOB_NETWORK_CURRENT_CONTAINER="" +COMPOSE_JOB_NETWORK_NAME="" +COMPOSE_JOB_NETWORK_CONNECTED=false + +compose_job_network_initialize() { + COMPOSE_JOB_NETWORK_RUNNING_IN_CONTAINER=false + COMPOSE_JOB_NETWORK_CURRENT_CONTAINER="" + COMPOSE_JOB_NETWORK_NAME="" + COMPOSE_JOB_NETWORK_CONNECTED=false + + if [[ ! -f /.dockerenv ]]; then + return 0 + fi + if [[ -z "${HOSTNAME:-}" ]]; then + return 1 + fi + + COMPOSE_JOB_NETWORK_RUNNING_IN_CONTAINER=true + COMPOSE_JOB_NETWORK_CURRENT_CONTAINER="$HOSTNAME" +} + +compose_job_network_connect() { + if [[ "$COMPOSE_JOB_NETWORK_RUNNING_IN_CONTAINER" != "true" ]]; then + return 1 + fi + if (($# == 0)); then + return 1 + fi + local service="$1" + shift + if [[ -z "$service" || $# -eq 0 ]]; then + return 1 + fi + + local service_container_output + if ! service_container_output="$("$@" ps -q "$service" 2>/dev/null)"; then + return 1 + fi + + local service_containers=() + while IFS= read -r container_id; do + if [[ -n "$container_id" ]]; then + service_containers+=("$container_id") + fi + done <<<"$service_container_output" + if ((${#service_containers[@]} != 1)); then + return 1 + fi + + local service_network_output + if ! service_network_output="$( + docker inspect \ + --format '{{range $networkName, $_ := .NetworkSettings.Networks}}{{println $networkName}}{{end}}' \ + "${service_containers[0]}" 2>/dev/null + )"; then + return 1 + fi + + local service_networks=() + while IFS= read -r network_name; do + if [[ -n "$network_name" ]]; then + service_networks+=("$network_name") + fi + done <<<"$service_network_output" + if ((${#service_networks[@]} != 1)); then + return 1 + fi + COMPOSE_JOB_NETWORK_NAME="${service_networks[0]}" + + local current_network_output + if ! current_network_output="$( + docker inspect \ + --format '{{range $networkName, $_ := .NetworkSettings.Networks}}{{println $networkName}}{{end}}' \ + "$COMPOSE_JOB_NETWORK_CURRENT_CONTAINER" 2>/dev/null + )"; then + return 1 + fi + + while IFS= read -r network_name; do + if [[ "$network_name" == "$COMPOSE_JOB_NETWORK_NAME" ]]; then + COMPOSE_JOB_NETWORK_CONNECTED=true + return 0 + fi + done <<<"$current_network_output" + + COMPOSE_JOB_NETWORK_CONNECTED=true + if ! docker network connect \ + "$COMPOSE_JOB_NETWORK_NAME" \ + "$COMPOSE_JOB_NETWORK_CURRENT_CONTAINER" >/dev/null 2>&1; then + docker network disconnect \ + "$COMPOSE_JOB_NETWORK_NAME" \ + "$COMPOSE_JOB_NETWORK_CURRENT_CONTAINER" >/dev/null 2>&1 || true + COMPOSE_JOB_NETWORK_CONNECTED=false + return 1 + fi +} + +compose_job_network_disconnect() { + if [[ "$COMPOSE_JOB_NETWORK_CONNECTED" != "true" ]]; then + return 0 + fi + + if ! docker network disconnect \ + "$COMPOSE_JOB_NETWORK_NAME" \ + "$COMPOSE_JOB_NETWORK_CURRENT_CONTAINER" >/dev/null 2>&1; then + return 1 + fi + COMPOSE_JOB_NETWORK_CONNECTED=false +} diff --git a/scripts/smoke-infrastructure.sh b/scripts/smoke-infrastructure.sh index 2f986aa..4a83a26 100755 --- a/scripts/smoke-infrastructure.sh +++ b/scripts/smoke-infrastructure.sh @@ -5,6 +5,9 @@ set -Eeuo pipefail repository_root="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")/.." && pwd)" cd "$repository_root" +# shellcheck source=scripts/lib/compose-job-network.sh +source "$repository_root/scripts/lib/compose-job-network.sh" + command -v docker >/dev/null || { echo "Docker is required for the infrastructure smoke test." >&2 exit 1 @@ -50,17 +53,9 @@ compose=( --env-file .env.example --file infrastructure/compose.yaml ) -running_in_container=false -current_container="" -compose_network="" -connected_to_compose_network=false -if [[ -f /.dockerenv ]]; then - running_in_container=true - if [[ -z "${HOSTNAME:-}" ]]; then - echo "HOSTNAME is required when the smoke test runs inside a Docker container." >&2 - exit 1 - fi - current_container="$HOSTNAME" +if ! compose_job_network_initialize; then + echo "The current Docker job container could not be identified." >&2 + exit 1 fi cleanup() { @@ -70,15 +65,10 @@ cleanup() { if ((exit_status != 0)); then "${compose[@]}" logs --no-color >&2 || true fi - if [[ "$connected_to_compose_network" == "true" ]]; then - if ! cleanup_output="$( - docker network disconnect "$compose_network" "$current_container" 2>&1 - )"; then - echo "Infrastructure smoke network cleanup failed:" >&2 - echo "$cleanup_output" >&2 - if ((exit_status == 0)); then - exit_status=1 - fi + if ! compose_job_network_disconnect; then + echo "Infrastructure smoke network cleanup failed." >&2 + if ((exit_status == 0)); then + exit_status=1 fi fi if ! cleanup_output="$( @@ -133,46 +123,11 @@ if "${compose[@]}" exec -T \ exit 1 fi -if [[ "$running_in_container" == "true" ]]; then - object_storage_container="$("${compose[@]}" ps -q object-storage)" - if [[ -z "$object_storage_container" ]]; then - echo "Compose did not return the object-storage container ID." >&2 +if [[ "$COMPOSE_JOB_NETWORK_RUNNING_IN_CONTAINER" == "true" ]]; then + if ! compose_job_network_connect object-storage "${compose[@]}"; then + echo "The infrastructure smoke job could not join the isolated storage network." >&2 exit 1 fi - - network_output="$( - docker inspect \ - --format '{{range $networkName, $_ := .NetworkSettings.Networks}}{{println $networkName}}{{end}}' \ - "$object_storage_container" - )" - object_storage_networks=() - while IFS= read -r network_name; do - if [[ -n "$network_name" ]]; then - object_storage_networks+=("$network_name") - fi - done <<<"$network_output" - if ((${#object_storage_networks[@]} != 1)); then - echo "Expected object-storage to use exactly one Compose network." >&2 - exit 1 - fi - compose_network="${object_storage_networks[0]}" - - current_network_output="$( - docker inspect \ - --format '{{range $networkName, $_ := .NetworkSettings.Networks}}{{println $networkName}}{{end}}' \ - "$current_container" - )" - already_connected=false - while IFS= read -r network_name; do - if [[ "$network_name" == "$compose_network" ]]; then - already_connected=true - break - fi - done <<<"$current_network_output" - if [[ "$already_connected" != "true" ]]; then - docker network connect "$compose_network" "$current_container" - connected_to_compose_network=true - fi s3_endpoint="http://object-storage:8333" else s3_binding="$("${compose[@]}" port object-storage 8333)" diff --git a/scripts/test-migrations-integration.sh b/scripts/test-migrations-integration.sh new file mode 100755 index 0000000..1f1cf56 --- /dev/null +++ b/scripts/test-migrations-integration.sh @@ -0,0 +1,100 @@ +#!/usr/bin/env bash + +set -Eeuo pipefail + +repository_root="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")/.." && pwd)" +cd "$repository_root" + +# shellcheck source=scripts/lib/compose-job-network.sh +source "$repository_root/scripts/lib/compose-job-network.sh" + +command -v docker >/dev/null || { + echo "Docker is required for the PostgreSQL migration integration suite." >&2 + exit 1 +} +command -v pnpm >/dev/null || { + echo "pnpm is required for the PostgreSQL migration integration suite." >&2 + exit 1 +} +if ! docker compose version >/dev/null 2>&1; then + echo "Docker Compose is required for the PostgreSQL migration integration suite." >&2 + exit 1 +fi +if ! compose_job_network_initialize; then + echo "The current Docker job container could not be identified." >&2 + exit 1 +fi + +raw_project="ekklesia-migration-${GITHUB_RUN_ID:-local}-${GITHUB_RUN_ATTEMPT:-0}-${GITHUB_JOB:-job}-$$" +project="${raw_project//[^a-zA-Z0-9_-]/-}" +project="${project,,}" + +export POSTGRES_DB="ekklesia_migration_test" +export POSTGRES_USER="ekklesia_migration_test" +export POSTGRES_PASSWORD="temporary-$project-${RANDOM}-${RANDOM}" +export POSTGRES_PORT="0" + +compose=( + docker compose + --project-name "$project" + --env-file .env.example + --file infrastructure/compose.yaml +) + +cleanup() { + local exit_status=$? + trap - EXIT INT TERM + + if ! compose_job_network_disconnect; then + echo "PostgreSQL migration test network cleanup failed." >&2 + if ((exit_status == 0)); then + exit_status=1 + fi + fi + if ! "${compose[@]}" down --volumes --remove-orphans --timeout 10 >/dev/null 2>&1; then + echo "PostgreSQL migration test infrastructure cleanup failed." >&2 + if ((exit_status == 0)); then + exit_status=1 + fi + fi + exit "$exit_status" +} +trap cleanup EXIT +trap 'exit 130' INT +trap 'exit 143' TERM + +if ! "${compose[@]}" config --quiet >/dev/null 2>&1; then + echo "PostgreSQL migration test infrastructure configuration is invalid." >&2 + exit 1 +fi +if ! "${compose[@]}" up --detach --wait --wait-timeout 120 postgres >/dev/null 2>&1; then + echo "Disposable PostgreSQL did not become ready." >&2 + exit 1 +fi + +if [[ "$COMPOSE_JOB_NETWORK_RUNNING_IN_CONTAINER" == "true" ]]; then + if ! compose_job_network_connect postgres "${compose[@]}"; then + echo "The migration test job could not join the isolated PostgreSQL network." >&2 + exit 1 + fi + database_host="postgres" + database_port="5432" +else + if ! postgres_binding="$("${compose[@]}" port postgres 5432 2>/dev/null)"; then + echo "The disposable PostgreSQL loopback port could not be determined." >&2 + exit 1 + fi + if [[ ! "$postgres_binding" =~ ^127\.0\.0\.1:([0-9]+)$ ]]; then + echo "The disposable PostgreSQL port is not bound safely to loopback." >&2 + exit 1 + fi + database_host="127.0.0.1" + database_port="${BASH_REMATCH[1]}" + if ((database_port < 1 || database_port > 65535)); then + echo "The disposable PostgreSQL loopback port is invalid." >&2 + exit 1 + fi +fi + +EKKLESIA_MIGRATION_TEST_DATABASE_URL="postgresql://$POSTGRES_USER:$POSTGRES_PASSWORD@$database_host:$database_port/$POSTGRES_DB" \ + pnpm exec vitest run --config vitest.migrations.integration.config.ts diff --git a/tests/infrastructure.test.ts b/tests/infrastructure.test.ts index 7a5a7d5..bf481fa 100644 --- a/tests/infrastructure.test.ts +++ b/tests/infrastructure.test.ts @@ -21,6 +21,17 @@ interface PackageDocument { scripts?: Record; } +interface WorkflowJob { + needs?: string; + steps?: WorkflowStep[]; + "timeout-minutes"?: number; +} + +interface WorkflowStep { + run?: string; + uses?: string; +} + interface TurboDocument { tasks?: Record; } @@ -64,19 +75,28 @@ describe("development infrastructure", () => { "utf8", ); const workflow = parse(source) as { - jobs?: { - "infrastructure-smoke"?: { steps?: Array<{ run?: string }> }; - verify?: { steps?: Array<{ run?: string }> }; - }; + jobs?: Record; + name?: string; }; - const commands = workflow.jobs?.verify?.steps?.flatMap((step) => step.run ?? []) ?? []; - const infrastructureCommands = - workflow.jobs?.["infrastructure-smoke"]?.steps?.flatMap((step) => step.run ?? []) ?? []; + const verify = workflow.jobs?.verify; + const infrastructure = workflow.jobs?.["infrastructure-smoke"]; + const migration = workflow.jobs?.["migration-smoke"]; + const commands = verify?.steps?.flatMap((step) => step.run ?? []) ?? []; + const infrastructureCommands = infrastructure?.steps?.flatMap((step) => step.run ?? []) ?? []; + const migrationCommands = migration?.steps?.flatMap((step) => step.run ?? []) ?? []; + const verifyActions = verify?.steps?.flatMap((step) => step.uses ?? []) ?? []; + const migrationActions = migration?.steps?.flatMap((step) => step.uses ?? []) ?? []; - expect(workflow.jobs?.verify).toBeDefined(); + expect(workflow.name).toBe("CI"); + expect(verify).toBeDefined(); expect(commands).toContain("pnpm install --frozen-lockfile"); expect(commands).toContain("pnpm ci:verify"); expect(infrastructureCommands).toContain("bash scripts/smoke-infrastructure.sh"); + expect(migration?.needs).toBe("verify"); + expect(migration?.["timeout-minutes"]).toBe(15); + expect(migrationActions).toEqual(verifyActions); + expect(migrationCommands).toContain("pnpm install --frozen-lockfile"); + expect(migrationCommands).toContain("pnpm test:migrations:integration"); }); it("routes containerized S3 smoke checks through the isolated Compose network", async () => { @@ -84,18 +104,54 @@ describe("development infrastructure", () => { path.join(repositoryRoot, "scripts", "smoke-infrastructure.sh"), "utf8", ); + const helper = await readFile( + path.join(repositoryRoot, "scripts", "lib", "compose-job-network.sh"), + "utf8", + ); - expect(source).toContain("[[ -f /.dockerenv ]]"); - expect(source).toContain(`object_storage_container="$("\${compose[@]}" ps -q object-storage)"`); - expect(source).toContain("docker inspect"); - expect(source).toContain('docker network connect "$compose_network" "$current_container"'); - expect(source).toContain('docker network disconnect "$compose_network" "$current_container"'); + expect(source).toContain('source "$repository_root/scripts/lib/compose-job-network.sh"'); + expect(source).toContain(`compose_job_network_connect object-storage "\${compose[@]}"`); expect(source).toContain('s3_endpoint="http://object-storage:8333"'); expect(source).toContain('s3_endpoint="http://127.0.0.1:$s3_port"'); expect(source).not.toContain("0.0.0.0:${S3_PORT"); - expect(source.indexOf("docker network disconnect")).toBeLessThan( + expect(source.indexOf("compose_job_network_disconnect")).toBeLessThan( source.indexOf(`"\${compose[@]}" down`), ); + + expect(helper).toContain("[[ ! -f /.dockerenv ]]"); + expect(helper).toContain('"$@" ps -q "$service"'); + expect(helper).toContain("docker inspect"); + expect(helper).toContain("docker network connect"); + expect(helper).toContain("docker network disconnect"); + }); + + it("provisions isolated PostgreSQL for the live migration suite", async () => { + const packageSource = await readFile(path.join(repositoryRoot, "package.json"), "utf8"); + const packageDocument = JSON.parse(packageSource) as PackageDocument; + const source = await readFile( + path.join(repositoryRoot, "scripts", "test-migrations-integration.sh"), + "utf8", + ); + + expect(packageDocument.scripts?.["test:migrations:integration"]).toBe( + "bash scripts/test-migrations-integration.sh", + ); + expect(source).toContain('source "$repository_root/scripts/lib/compose-job-network.sh"'); + expect(source).toContain('export POSTGRES_PORT="0"'); + expect(source).toContain("up --detach --wait --wait-timeout 120 postgres"); + expect(source).toContain(`compose_job_network_connect postgres "\${compose[@]}"`); + expect(source).toContain('database_host="postgres"'); + expect(source).toContain('database_port="5432"'); + expect(source).toContain('database_host="127.0.0.1"'); + expect(source).toContain( + 'EKKLESIA_MIGRATION_TEST_DATABASE_URL="postgresql://$POSTGRES_USER:$POSTGRES_PASSWORD@$database_host:$database_port/$POSTGRES_DB"', + ); + expect(source).toContain("--config vitest.migrations.integration.config.ts"); + expect(source).not.toContain("0.0.0.0"); + expect(source).not.toContain('compose[@]}" logs'); + expect(source.indexOf("compose_job_network_disconnect")).toBeLessThan( + source.indexOf("down --volumes --remove-orphans"), + ); }); it("includes every official platform layer in the workspace", async () => { diff --git a/tsconfig.json b/tsconfig.json index a8401e6..a33d681 100644 --- a/tsconfig.json +++ b/tsconfig.json @@ -3,6 +3,11 @@ "compilerOptions": { "noEmit": true }, - "include": ["scripts/**/*.ts", "tests/**/*.ts", "vitest.config.ts"], + "include": [ + "scripts/**/*.ts", + "tests/**/*.ts", + "vitest.config.ts", + "vitest.migrations.integration.config.ts" + ], "exclude": ["node_modules", "tests/fixtures"] } diff --git a/vitest.migrations.integration.config.ts b/vitest.migrations.integration.config.ts new file mode 100644 index 0000000..9271684 --- /dev/null +++ b/vitest.migrations.integration.config.ts @@ -0,0 +1,11 @@ +import { defineConfig } from "vitest/config"; + +export default defineConfig({ + test: { + fileParallelism: false, + hookTimeout: 30_000, + include: ["core/tests/postgres-migration.integration.ts"], + passWithNoTests: false, + testTimeout: 30_000, + }, +}); -- 2.54.0