Ekklesia/core/tests/module-migration-discovery.test.ts

import { createHash } from "node:crypto";
import { mkdir, mkdtemp, rm, symlink, writeFile } from "node:fs/promises";
import { tmpdir } from "node:os";
import path from "node:path";
import { afterEach, describe, expect, it } from "vitest";
import { formatMigrationValidationReport } from "../src/cli/migration-report.js";
import {
  discoverModuleMigrations,
  ModuleDiscoveryError,
  ModuleMigrationDiscoveryError,
  type ModuleMigrationDiscoveryIssueCode,
} from "../src/modules/index.js";

const originSnapshotId = "00000000-0000-0000-0000-000000000000";
const temporaryDirectories: string[] = [];

interface MigrationFixture {
  readonly breakpoints?: boolean;
  readonly destructive?:
    | { readonly mode: "none" }
    | { readonly mode: "approval-required"; readonly reason: string };
  readonly snapshotId?: string;
  readonly sql: string | Uint8Array;
  readonly tag: string;
  readonly timestamp?: number;
}

interface MigrationModuleFixture {
  readonly dependencies?: Record<string, unknown>;
  readonly manifest?: Record<string, unknown>;
  readonly migrations?: readonly MigrationFixture[] | false;
  readonly schemaSource?: string;
  readonly serverSource?: string;
}

interface CreatedMigrationModule {
  readonly migrationDirectory?: string;
  readonly moduleDirectory: string;
}

afterEach(async () => {
  await Promise.all(
    temporaryDirectories
      .splice(0)
      .map((directory) => rm(directory, { force: true, recursive: true })),
  );
});

async function createModulesDirectory(): Promise<string> {
  const directory = await mkdtemp(path.join(tmpdir(), "ekklesia-migrations-"));
  temporaryDirectories.push(directory);
  return directory;
}

function snapshotId(index: number): string {
  return `10000000-0000-4000-8000-${(index + 1).toString(16).padStart(12, "0")}`;
}

function manifestFor(
  id: string,
  options: {
    readonly dependencies?: Record<string, unknown>;
    readonly migrations?: boolean;
    readonly schema?: boolean;
  } = {},
): Record<string, unknown> {
  return {
    manifestVersion: 5,
    id,
    name: `${id} module`,
    version: "0.1.0",
    compatibility: { platform: "^0.1.0", moduleSdk: "^0.1.0" },
    ...(options.dependencies ? { dependencies: options.dependencies } : {}),
    capabilities: [],
    permissions: [],
    entrypoints: {
      server: "./dist/server/index.js",
      ...(options.schema ? { schema: "./dist/schema/index.js" } : {}),
      ...(options.migrations === false ? {} : { migrations: "./drizzle" }),
    },
  };
}

function defaultMigrations(id: string): readonly MigrationFixture[] {
  return [
    {
      sql: `CREATE TABLE "${id.replaceAll("-", "_")}_records" ("id" integer);\n`,
      tag: "0000_initial",
    },
  ];
}

async function createMigrationModule(
  modulesDirectory: string,
  id: string,
  fixture: MigrationModuleFixture = {},
): Promise<CreatedMigrationModule> {
  const migrations =
    fixture.migrations === false ? undefined : (fixture.migrations ?? defaultMigrations(id));
  const moduleDirectory = path.join(modulesDirectory, id);
  const migrationDirectory = migrations ? path.join(moduleDirectory, "drizzle") : undefined;
  await mkdir(path.join(moduleDirectory, "dist/server"), { recursive: true });
  if (fixture.schemaSource !== undefined) {
    await mkdir(path.join(moduleDirectory, "dist/schema"), { recursive: true });
  }
  if (migrationDirectory) {
    await mkdir(path.join(migrationDirectory, "meta"), { recursive: true });
  }

  await Promise.all([
    writeFile(
      path.join(moduleDirectory, "package.json"),
      JSON.stringify({
        name: `@test/module-${id}`,
        version: "0.1.0",
        type: "module",
        ekklesia: { layer: "module" },
      }),
    ),
    writeFile(
      path.join(moduleDirectory, "ekklesia.module.json"),
      JSON.stringify(
        fixture.manifest ??
          manifestFor(id, {
            ...(fixture.dependencies ? { dependencies: fixture.dependencies } : {}),
            migrations: migrations !== undefined,
            schema: fixture.schemaSource !== undefined,
          }),
      ),
    ),
    writeFile(
      path.join(moduleDirectory, "dist/server/index.js"),
      fixture.serverSource ?? `export default { id: ${JSON.stringify(id)}, version: "0.1.0" };`,
    ),
    ...(fixture.schemaSource === undefined
      ? []
      : [writeFile(path.join(moduleDirectory, "dist/schema/index.js"), fixture.schemaSource)]),
  ]);

  if (!migrationDirectory || !migrations) {
    return { moduleDirectory };
  }

  const journalEntries = migrations.map((migration, index) => ({
    idx: index,
    version: "7",
    when: migration.timestamp ?? 1_700_000_000_000 + index,
    tag: migration.tag,
    breakpoints: migration.breakpoints ?? true,
  }));
  const policyEntries = migrations.map((migration) => ({
    tag: migration.tag,
    destructive: migration.destructive ?? { mode: "none" },
  }));
  await Promise.all([
    writeFile(
      path.join(migrationDirectory, "meta/_journal.json"),
      JSON.stringify({ version: "7", dialect: "postgresql", entries: journalEntries }),
    ),
    writeFile(
      path.join(migrationDirectory, "meta/_ekklesia.json"),
      JSON.stringify({ version: 1, moduleId: id, entries: policyEntries }),
    ),
    ...migrations.flatMap((migration, index) => {
      const idForSnapshot = migration.snapshotId ?? snapshotId(index);
      const previousId =
        index === 0
          ? originSnapshotId
          : (migrations[index - 1]?.snapshotId ?? snapshotId(index - 1));
      return [
        writeFile(path.join(migrationDirectory, `${migration.tag}.sql`), migration.sql),
        writeFile(
          path.join(migrationDirectory, `meta/${index.toString().padStart(4, "0")}_snapshot.json`),
          JSON.stringify({
            version: "7",
            dialect: "postgresql",
            id: idForSnapshot,
            prevId: previousId,
          }),
        ),
      ];
    }),
  ]);

  return { migrationDirectory, moduleDirectory };
}

function discoveryOptions(modulesDirectory: string) {
  return {
    moduleSdkVersion: "0.1.0",
    modulesDirectory,
    platformVersion: "0.1.0",
  };
}

async function captureMigrationError(
  modulesDirectory: string,
): Promise<ModuleMigrationDiscoveryError> {
  try {
    await discoverModuleMigrations(discoveryOptions(modulesDirectory));
  } catch (error) {
    if (error instanceof ModuleMigrationDiscoveryError) {
      return error;
    }
    throw error;
  }
  throw new Error("Expected module migration discovery to fail.");
}

function issueCodes(error: ModuleMigrationDiscoveryError): ModuleMigrationDiscoveryIssueCode[] {
  return error.issues.map((issue) => issue.code);
}

function sha256(contents: string | Uint8Array): string {
  return createHash("sha256").update(contents).digest("hex");
}

describe("module migration discovery", () => {
  it("aggregates dependency-first modules and journal-ordered migrations with an exact report", async () => {
    const modulesDirectory = await createModulesDirectory();
    const peopleInitial = 'CREATE TABLE "people_records" ("id" integer);\n';
    const peopleArchive = new TextEncoder().encode(
      'ALTER TABLE "people_records" ADD COLUMN "archived" boolean;\r\n',
    );
    const calendarEvents = 'CREATE TABLE "calendar_events" ("id" integer);\n';
    await createMigrationModule(modulesDirectory, "calendar", {
      dependencies: { modules: { required: { people: "^0.1.0" } } },
      migrations: [{ sql: calendarEvents, tag: "0000_events" }],
    });
    await createMigrationModule(modulesDirectory, "people", {
      migrations: [
        { sql: peopleInitial, tag: "0000_initial" },
        {
          destructive: {
            mode: "approval-required",
            reason: "Existing deployments must approve the archive-column rollout.",
          },
          sql: peopleArchive,
          tag: "0001_archive",
        },
      ],
    });

    const aggregate = await discoverModuleMigrations(discoveryOptions(modulesDirectory));

    expect(aggregate.modules.map((module) => module.moduleId)).toEqual(["people", "calendar"]);
    expect(aggregate.migrations.map((migration) => migration.identity)).toEqual([
      "people:0000_initial",
      "people:0001_archive",
      "calendar:0000_events",
    ]);
    expect(aggregate.migrations.map((migration) => migration.checksum)).toEqual([
      sha256(peopleInitial),
      sha256(peopleArchive),
      sha256(calendarEvents),
    ]);
    expect(aggregate.migrations[1]).toMatchObject({
      approvalRequired: true,
      breakpoints: true,
      destructive: {
        mode: "approval-required",
        reason: "Existing deployments must approve the archive-column rollout.",
      },
      file: "0001_archive.sql",
      index: 1,
      timestamp: 1_700_000_000_001,
    });
    expect(formatMigrationValidationReport(aggregate)).toEqual([
      "Migration validation passed (2 modules, 3 migrations):",
      `- people:0000_initial sha256=${sha256(peopleInitial)} destructive=none`,
      `- people:0001_archive sha256=${sha256(peopleArchive)} destructive=approval-required`,
      `- calendar:0000_events sha256=${sha256(calendarEvents)} destructive=none`,
    ]);
    expect(Object.isFrozen(aggregate)).toBe(true);
    expect(Object.isFrozen(aggregate.modules)).toBe(true);
    expect(Object.isFrozen(aggregate.modules[0])).toBe(true);
    expect(Object.isFrozen(aggregate.modules[0]?.migrations)).toBe(true);
    expect(Object.isFrozen(aggregate.migrations)).toBe(true);
    expect(Object.isFrozen(aggregate.migrations[0])).toBe(true);
    expect(Object.isFrozen(aggregate.migrations[1]?.destructive)).toBe(true);
    expect(Object.isFrozen(formatMigrationValidationReport(aggregate))).toBe(true);
  });

  it("hashes exact SQL bytes and changes only the checksum when content changes", async () => {
    const modulesDirectory = await createModulesDirectory();
    const original = new TextEncoder().encode("SELECT 1;\n");
    const changed = new TextEncoder().encode("SELECT 1;\r\n");
    const fixture = await createMigrationModule(modulesDirectory, "exact", {
      migrations: [{ sql: original, tag: "0000_initial" }],
    });

    const first = await discoverModuleMigrations(discoveryOptions(modulesDirectory));
    expect(first.migrations[0]).toMatchObject({
      checksum: sha256(original),
      identity: "exact:0000_initial",
      sql: "SELECT 1;\n",
    });

    await writeFile(path.join(fixture.migrationDirectory ?? "", "0000_initial.sql"), changed);
    const second = await discoverModuleMigrations(discoveryOptions(modulesDirectory));

    expect(second.migrations[0]?.identity).toBe(first.migrations[0]?.identity);
    expect(second.migrations[0]?.checksum).toBe(sha256(changed));
    expect(second.migrations[0]?.checksum).not.toBe(first.migrations[0]?.checksum);
  });

  it("supports modules without migrations without importing server or schema code", async () => {
    const modulesDirectory = await createModulesDirectory();
    await createMigrationModule(modulesDirectory, "schema-only", {
      migrations: false,
      schemaSource: 'throw new Error("schema entrypoint must remain unloaded");',
      serverSource: 'throw new Error("server entrypoint must remain unloaded");',
    });
    await createMigrationModule(modulesDirectory, "server-only", {
      migrations: false,
      serverSource: 'throw new Error("server entrypoint must remain unloaded");',
    });

    const aggregate = await discoverModuleMigrations(discoveryOptions(modulesDirectory));

    expect(aggregate.discoveredModules).toHaveLength(2);
    expect(aggregate.modules).toEqual([
      { migrations: [], moduleId: "schema-only" },
      { migrations: [], moduleId: "server-only" },
    ]);
    expect(aggregate.migrations).toEqual([]);
    expect(formatMigrationValidationReport(aggregate)).toEqual([
      "Migration validation passed (2 modules, 0 migrations):",
      "- schema-only (no migrations)",
      "- server-only (no migrations)",
    ]);
    expect(Object.isFrozen(aggregate.modules[0]?.migrations)).toBe(true);
  });

  it("completes static discovery before inspecting migration contents", async () => {
    const modulesDirectory = await createModulesDirectory();
    const valid = await createMigrationModule(modulesDirectory, "migration-invalid");
    await writeFile(
      path.join(valid.migrationDirectory ?? "", "meta/_journal.json"),
      '{"secret":"postgres://admin:password@database/private"',
    );
    await createMigrationModule(modulesDirectory, "static-invalid", {
      manifest: {
        ...manifestFor("static-invalid", { migrations: false }),
        entrypoints: { server: "./dist/server/missing.js" },
      },
      migrations: false,
    });

    let discoveryError: ModuleDiscoveryError | undefined;
    try {
      await discoverModuleMigrations(discoveryOptions(modulesDirectory));
    } catch (error) {
      if (error instanceof ModuleDiscoveryError) {
        discoveryError = error;
      } else {
        throw error;
      }
    }

    expect(discoveryError?.issues).toContainEqual(
      expect.objectContaining({
        code: "MODULE_ENTRYPOINT_MISSING",
        moduleId: "static-invalid",
        path: "entrypoints.server",
      }),
    );
    expect(JSON.stringify(discoveryError)).not.toContain("postgres://");
  });

  it("rejects malformed journal and policy metadata without leaking their contents", async () => {
    const modulesDirectory = await createModulesDirectory();
    const journalInvalid = await createMigrationModule(modulesDirectory, "journal-invalid");
    const policyInvalid = await createMigrationModule(modulesDirectory, "policy-invalid");
    await Promise.all([
      writeFile(
        path.join(journalInvalid.migrationDirectory ?? "", "meta/_journal.json"),
        '{"url":"postgres://admin:password@database/private","path":"/machine/journal"',
      ),
      writeFile(
        path.join(policyInvalid.migrationDirectory ?? "", "meta/_ekklesia.json"),
        JSON.stringify({
          version: 1,
          moduleId: "policy-invalid",
          entries: [
            {
              tag: "0000_initial",
              destructive: { mode: "none" },
              approved: "token-at-/machine/policy",
            },
          ],
        }),
      ),
    ]);

    const error = await captureMigrationError(modulesDirectory);

    expect(issueCodes(error)).toEqual([
      "MODULE_MIGRATION_JOURNAL_INVALID",
      "MODULE_MIGRATION_METADATA_INVALID",
    ]);
    const report = `${error.message}\n${error.stack}\n${JSON.stringify(error.issues)}`;
    expect(report).not.toContain("postgres://");
    expect(report).not.toContain("password");
    expect(report).not.toContain("/machine/journal");
    expect(report).not.toContain("token-at-/machine/policy");
    expect(report).not.toContain(modulesDirectory);
  });

  it("rejects duplicate identities, invalid ordering, and inconsistent destructive policy", async () => {
    const modulesDirectory = await createModulesDirectory();
    const fixture = await createMigrationModule(modulesDirectory, "inconsistent", {
      migrations: [
        { sql: "SELECT 1;\n", tag: "0000_first" },
        { sql: "SELECT 2;\n", tag: "0001_second" },
      ],
    });
    const root = fixture.migrationDirectory ?? "";
    await Promise.all([
      writeFile(
        path.join(root, "meta/_journal.json"),
        JSON.stringify({
          version: "7",
          dialect: "postgresql",
          entries: [
            {
              idx: 1,
              version: "7",
              when: 200,
              tag: "0001_second",
              breakpoints: true,
            },
            {
              idx: 1,
              version: "7",
              when: 100,
              tag: "0001_second",
              breakpoints: true,
            },
          ],
        }),
      ),
      writeFile(
        path.join(root, "meta/_ekklesia.json"),
        JSON.stringify({
          version: 1,
          moduleId: "wrong-module",
          entries: [
            { tag: "0001_second", destructive: { mode: "none" } },
            { tag: "0001_second", destructive: { mode: "none" } },
          ],
        }),
      ),
    ]);

    const error = await captureMigrationError(modulesDirectory);

    expect(issueCodes(error)).toEqual(
      expect.arrayContaining([
        "MODULE_MIGRATION_ID_DUPLICATE",
        "MODULE_MIGRATION_METADATA_INCONSISTENT",
        "MODULE_MIGRATION_ORDER_INVALID",
      ]),
    );
    expect(
      issueCodes(error).filter((code) => code === "MODULE_MIGRATION_METADATA_INCONSISTENT"),
    ).toHaveLength(2);
    expect(
      issueCodes(error).filter((code) => code === "MODULE_MIGRATION_ORDER_INVALID"),
    ).toHaveLength(2);
  });

  it("rejects unsafe journal tags before deriving or reading a migration path", async () => {
    const modulesDirectory = await createModulesDirectory();
    const fixture = await createMigrationModule(modulesDirectory, "unsafe-tag");
    const root = fixture.migrationDirectory ?? "";
    await Promise.all([
      writeFile(
        path.join(root, "meta/_journal.json"),
        JSON.stringify({
          version: "7",
          dialect: "postgresql",
          entries: [
            {
              idx: 0,
              version: "7",
              when: 1_700_000_000_000,
              tag: "../outside",
              breakpoints: true,
            },
          ],
        }),
      ),
      writeFile(
        path.join(fixture.moduleDirectory, "outside.sql"),
        "SELECT 'database-password at /machine/outside';\n",
      ),
    ]);

    const error = await captureMigrationError(modulesDirectory);

    expect(issueCodes(error)).toContain("MODULE_MIGRATION_ID_INVALID");
    expect(error.issues).not.toContainEqual(expect.objectContaining({ path: "migrations.0.sql" }));
    const report = `${error.stack}\n${JSON.stringify(error.issues)}`;
    expect(report).not.toContain("../outside");
    expect(report).not.toContain("database-password");
    expect(report).not.toContain("/machine/outside");
  });

  it("rejects missing, undeclared, unsupported, empty, and invalid UTF-8 migration files", async () => {
    const modulesDirectory = await createModulesDirectory();
    const missing = await createMigrationModule(modulesDirectory, "missing");
    const inventory = await createMigrationModule(modulesDirectory, "inventory");
    await createMigrationModule(modulesDirectory, "empty", {
      migrations: [{ sql: " \n\t", tag: "0000_initial" }],
    });
    await createMigrationModule(modulesDirectory, "invalid-utf8", {
      migrations: [{ sql: Uint8Array.from([0xff, 0xfe, 0xfd]), tag: "0000_initial" }],
    });
    await Promise.all([
      rm(path.join(missing.migrationDirectory ?? "", "0000_initial.sql")),
      writeFile(path.join(inventory.migrationDirectory ?? "", "0001_orphan.sql"), "SELECT 2;\n"),
      writeFile(path.join(inventory.migrationDirectory ?? "", "run.ts"), "throw new Error();\n"),
    ]);

    const error = await captureMigrationError(modulesDirectory);

    expect(issueCodes(error)).toEqual(
      expect.arrayContaining([
        "MODULE_MIGRATION_EMPTY",
        "MODULE_MIGRATION_FILE_INVALID",
        "MODULE_MIGRATION_FILE_MISSING",
        "MODULE_MIGRATION_FILE_UNDECLARED",
        "MODULE_MIGRATION_UNSUPPORTED_FILE",
      ]),
    );
    expect(error.issues.some((issue) => issue.moduleId === "missing")).toBe(true);
    expect(error.issues.some((issue) => issue.moduleId === "inventory")).toBe(true);
    expect(error.issues.some((issue) => issue.moduleId === "empty")).toBe(true);
    expect(error.issues.some((issue) => issue.moduleId === "invalid-utf8")).toBe(true);
  });

  it("rejects missing, broken, and undeclared Drizzle snapshots", async () => {
    const modulesDirectory = await createModulesDirectory();
    const fixture = await createMigrationModule(modulesDirectory, "snapshots", {
      migrations: [
        { sql: "SELECT 1;\n", tag: "0000_first" },
        { sql: "SELECT 2;\n", tag: "0001_second" },
      ],
    });
    const missing = await createMigrationModule(modulesDirectory, "snapshot-missing");
    const root = fixture.migrationDirectory ?? "";
    await Promise.all([
      rm(path.join(missing.migrationDirectory ?? "", "meta/0000_snapshot.json")),
      writeFile(
        path.join(root, "meta/0001_snapshot.json"),
        JSON.stringify({
          version: "7",
          dialect: "postgresql",
          id: snapshotId(1),
          prevId: originSnapshotId,
        }),
      ),
      writeFile(
        path.join(root, "meta/9999_snapshot.json"),
        JSON.stringify({
          version: "7",
          dialect: "postgresql",
          id: "99999999-9999-4999-8999-999999999999",
          prevId: snapshotId(1),
        }),
      ),
    ]);

    const error = await captureMigrationError(modulesDirectory);

    expect(issueCodes(error)).toEqual(
      expect.arrayContaining([
        "MODULE_MIGRATION_FILE_MISSING",
        "MODULE_MIGRATION_FILE_UNDECLARED",
        "MODULE_MIGRATION_SNAPSHOT_INVALID",
      ]),
    );
    expect(error.issues).toContainEqual(
      expect.objectContaining({
        code: "MODULE_MIGRATION_FILE_MISSING",
        moduleId: "snapshot-missing",
        path: "snapshots.0",
      }),
    );
  });

  it("rejects unsafe migration symlinks and sanitizes the target path", async () => {
    const modulesDirectory = await createModulesDirectory();
    const fixture = await createMigrationModule(modulesDirectory, "linked");
    const outsideDirectory = await createModulesDirectory();
    const outsideSql = path.join(outsideDirectory, "database-password.sql");
    const migrationFile = path.join(fixture.migrationDirectory ?? "", "0000_initial.sql");
    await writeFile(outsideSql, "SELECT 'secret-at-/machine/private';\n");
    await rm(migrationFile);
    await symlink(outsideSql, migrationFile);

    const error = await captureMigrationError(modulesDirectory);

    expect(issueCodes(error)).toContain("MODULE_MIGRATION_PATH_UNSAFE");
    const report = `${error.stack}\n${JSON.stringify(error.issues)}`;
    expect(report).not.toContain("database-password");
    expect(report).not.toContain("secret-at-/machine/private");
    expect(report).not.toContain(outsideDirectory);
  });

  it("produces deterministic sanitized errors and no partial result when any module is invalid", async () => {
    const firstModulesDirectory = await createModulesDirectory();
    const secondModulesDirectory = await createModulesDirectory();
    for (const modulesDirectory of [firstModulesDirectory, secondModulesDirectory]) {
      await createMigrationModule(modulesDirectory, "valid");
      const broken = await createMigrationModule(modulesDirectory, "broken");
      await writeFile(
        path.join(broken.migrationDirectory ?? "", "meta/_journal.json"),
        '{"secret":"https://user:[email protected]/private","sql":"DROP TABLE private"',
      );
    }

    const [firstError, secondError] = await Promise.all([
      captureMigrationError(firstModulesDirectory),
      captureMigrationError(secondModulesDirectory),
    ]);

    expect(firstError.issues).toEqual(secondError.issues);
    expect(firstError.stack).toBe(secondError.stack);
    expect(firstError.issues).toEqual([
      expect.objectContaining({
        code: "MODULE_MIGRATION_JOURNAL_INVALID",
        moduleId: "broken",
        path: "journal",
      }),
    ]);
    const report = `${firstError.message}\n${firstError.stack}\n${JSON.stringify(firstError.issues)}`;
    expect(report).not.toContain("https://");
    expect(report).not.toContain("password");
    expect(report).not.toContain("DROP TABLE");
    expect(report).not.toContain(firstModulesDirectory);
    expect(Object.isFrozen(firstError.issues)).toBe(true);
    expect(Object.isFrozen(firstError.issues[0])).toBe(true);
  });
});