Ekklesia/packages/module-pack/src/pack-module.ts

import { createHash } from "node:crypto";
import { lstat, mkdir, readdir, readFile, realpath, rename, writeFile } from "node:fs/promises";
import { isBuiltin } from "node:module";
import path from "node:path";
import { fileURLToPath } from "node:url";
import {
  MODULE_PACKAGE_ANY_TARGET,
  MODULE_PACKAGE_FORMAT,
  MODULE_PACKAGE_METADATA_FILENAME,
  MODULE_PACKAGE_NOTICES_FILENAME,
  type ModuleManifest,
  modulePackageArchiveFileName,
  modulePackageRuntimePackage,
  safeParseModuleManifest,
  safeParseModulePackageMetadata,
} from "@ekklesia/contracts";
import { analyzeModuleUi, ModuleUiBuildError } from "@ekklesia/module-ui-build";
import { build, type Message, type Plugin } from "esbuild";
import { minVersion, satisfies, validRange } from "semver";
import { NoticeError, type NoticeComponent, type NoticeUse, thirdPartyNotices } from "./notices.js";
import { BundledPackageIndex, isNativeOrPlatformSpecific } from "./packages.js";
import { deterministicTarGz, TarPathError } from "./tar.js";

/**
 * The canonical Ekklesia module packager (ADR 0049): turns a built module directory into one
 * deterministic Module Package v1 archive and its `.sha256` line.
 *
 * It reads the module directory and the dependencies its workspace installed, and writes only the
 * two output files. It never installs, links, or deploys anything, so the workspace it runs in is
 * left exactly as it was. In this release it produces the portable `any` target only.
 */

export type ModulePackErrorCode =
  | "MODULE_PACK_BUNDLE_FAILED"
  | "MODULE_PACK_DATABASE_RUNTIME_CONFLICT"
  | "MODULE_PACK_ENTRYPOINT_MISSING"
  | "MODULE_PACK_ENTRYPOINT_UNSUPPORTED"
  | "MODULE_PACK_EXTERNAL_DEPENDENCY"
  | "MODULE_PACK_FORBIDDEN_FILE"
  | "MODULE_PACK_IDENTITY_MISMATCH"
  | "MODULE_PACK_INCORPORATED_CONFLICT"
  | "MODULE_PACK_LAYOUT_CONFLICT"
  | "MODULE_PACK_LICENCE_TEXT_MISSING"
  | "MODULE_PACK_LICENCE_UNKNOWN"
  | "MODULE_PACK_MANIFEST_INVALID"
  | "MODULE_PACK_METADATA_INVALID"
  | "MODULE_PACK_MIGRATIONS_INVALID"
  | "MODULE_PACK_NATIVE_DEPENDENCY"
  | "MODULE_PACK_OUTPUT_FAILED"
  | "MODULE_PACK_PATH_UNSUPPORTED"
  | "MODULE_PACK_REVISION_INVALID"
  | "MODULE_PACK_RUNTIME_UNKNOWN"
  | "MODULE_PACK_SDK_INCOMPATIBLE"
  | "MODULE_PACK_TARGET_UNSUPPORTED"
  | "MODULE_PACK_UI_STALE";

const messages: Readonly<Record<ModulePackErrorCode, string>> = Object.freeze({
  MODULE_PACK_BUNDLE_FAILED:
    "The backend could not be bundled into self-contained files. Every import must resolve and bundle cleanly.",
  MODULE_PACK_DATABASE_RUNTIME_CONFLICT: "The backend bundles more than one Drizzle version.",
  MODULE_PACK_ENTRYPOINT_MISSING:
    "A declared backend entrypoint does not exist. Build the module before packaging it.",
  MODULE_PACK_ENTRYPOINT_UNSUPPORTED:
    "Backend entrypoints must be compiled .js files to be bundled into a Module Package v1.",
  MODULE_PACK_EXTERNAL_DEPENDENCY:
    "The backend bundle would import a dependency at run time; an any-target package bundles everything.",
  MODULE_PACK_FORBIDDEN_FILE:
    "The package would contain a source, test, declaration, source-map, or dependency file.",
  MODULE_PACK_IDENTITY_MISMATCH:
    "package.json must declare a name and the same version as ekklesia.module.json.",
  MODULE_PACK_INCORPORATED_CONFLICT:
    "The package bundles more than one version of an Ekklesia package.",
  MODULE_PACK_LAYOUT_CONFLICT: "Two package files would have the same path.",
  MODULE_PACK_LICENCE_TEXT_MISSING:
    "A bundled package ships no licence text, and its licence has no complete standard text to supply.",
  MODULE_PACK_LICENCE_UNKNOWN: "A bundled package declares no licence that permits redistribution.",
  MODULE_PACK_MANIFEST_INVALID: "ekklesia.module.json is missing or invalid.",
  MODULE_PACK_METADATA_INVALID: "The generated package metadata failed its own contract.",
  MODULE_PACK_MIGRATIONS_INVALID:
    "The migrations directory must hold only regular files and directories.",
  MODULE_PACK_NATIVE_DEPENDENCY:
    "The backend bundles a native or platform-specific dependency, which an any-target package cannot contain.",
  MODULE_PACK_OUTPUT_FAILED: "The package could not be written to the output directory.",
  MODULE_PACK_PATH_UNSUPPORTED:
    "A package path is not portable printable ASCII that fits the archive format.",
  MODULE_PACK_REVISION_INVALID: "The source revision must be a full 40- or 64-character commit id.",
  MODULE_PACK_RUNTIME_UNKNOWN:
    "The Node.js range could not be determined. Run inside an Ekklesia workspace, or pass one.",
  MODULE_PACK_SDK_INCOMPATIBLE:
    "The bundled Module SDK is outside the manifest's own compatibility.moduleSdk range.",
  MODULE_PACK_TARGET_UNSUPPORTED: "This packager produces the any target only.",
  MODULE_PACK_UI_STALE:
    "A declared browser artifact is missing or out of date. Build the module before packaging it.",
});

export class ModulePackError extends Error {
  readonly code: ModulePackErrorCode;
  /** The package or path the failure concerns, when there is one. A module author's own build
   * input, never a secret. */
  readonly detail?: string;
  /** esbuild's structured diagnostics, for a command line to print. */
  readonly diagnostics: readonly Message[];

  constructor(code: ModulePackErrorCode, detail?: string, diagnostics: readonly Message[] = []) {
    super(detail === undefined ? messages[code] : `${messages[code]} (${detail})`);
    this.name = "ModulePackError";
    this.code = code;
    if (detail !== undefined) {
      this.detail = detail;
    }
    this.diagnostics = diagnostics;
  }
}

export interface PackModuleOptions {
  /** A built module directory: its manifest, package.json, and compiled entrypoints. */
  readonly moduleDirectory: string;
  /** `any` only, in this release. */
  readonly target: string;
  readonly outDirectory: string;
  /** The full commit id the module was built from. */
  readonly sourceRevision: string;
  /** Informational only; recorded as given. */
  readonly repository?: string;
  /** The Node.js range the package is built for. Defaults to the enclosing workspace's
   * `engines.node`, the toolchain the module was built and tested with. */
  readonly nodeRange?: string;
}

export interface ModulePackResult {
  readonly archivePath: string;
  readonly sha256Path: string;
  readonly sha256: string;
  readonly archiveBytes: number;
  readonly moduleId: string;
  readonly version: string;
  readonly target: string;
  /** Payload files, the metadata file not included. */
  readonly fileCount: number;
}

const NATIVE_GUARD = "ekklesia-native-addon-guard";

/** Refuses a compiled addon the moment the bundle reaches one. */
const nativeAddonGuard: Plugin = {
  name: NATIVE_GUARD,
  setup(pluginBuild) {
    pluginBuild.onLoad({ filter: /\.node$/i }, () => ({
      errors: [{ text: "A compiled Node.js addon cannot be bundled into an any-target package." }],
    }));
  },
};

/** Bundled CommonJS code that requires a Node built-in needs a `require` in module scope. Nothing
 * else can be required at run time: every other dependency is inside the bundle, and the packager
 * refuses a bundle that would import anything but a built-in. */
const REQUIRE_BANNER =
  'import { createRequire as __ekklesiaCreateRequire } from "node:module";\nconst require = __ekklesiaCreateRequire(import.meta.url);';

function compare(left: string, right: string): number {
  return left < right ? -1 : left > right ? 1 : 0;
}

function sha256(data: Uint8Array): string {
  return createHash("sha256").update(data).digest("hex");
}

function isRecord(value: unknown): value is Record<string, unknown> {
  return typeof value === "object" && value !== null && !Array.isArray(value);
}

async function readJson(file: string): Promise<unknown> {
  return JSON.parse(await readFile(file, "utf8")) as unknown;
}

const ownManifestPath = path.resolve(
  path.dirname(fileURLToPath(import.meta.url)),
  "..",
  "package.json",
);

async function packagerIdentity(): Promise<{ readonly name: string; readonly version: string }> {
  const manifest = (await readJson(ownManifestPath)) as Record<string, unknown>;
  return { name: String(manifest.name), version: String(manifest.version) };
}

async function workspaceNodeRange(moduleDirectory: string): Promise<string> {
  let directory = moduleDirectory;
  for (;;) {
    try {
      await lstat(path.join(directory, "pnpm-workspace.yaml"));
      const manifest = await readJson(path.join(directory, "package.json"));
      const engines = isRecord(manifest) ? manifest.engines : undefined;
      const range = isRecord(engines) ? engines.node : undefined;
      if (typeof range === "string") {
        return range;
      }
      break;
    } catch {
      const parent = path.dirname(directory);
      if (parent === directory) {
        break;
      }
      directory = parent;
    }
  }
  throw new ModulePackError("MODULE_PACK_RUNTIME_UNKNOWN");
}

/** `./dist/server/index.js` → `dist/server/index.js`, inside the module directory. */
function moduleRelative(declared: string): string {
  return path.posix.normalize(declared.replace(/^\.\//u, ""));
}

function portable(relative: string): boolean {
  const segments = relative.split("/");
  return (
    /^[\x20-\x7e]+$/u.test(relative) &&
    !relative.includes("\\") &&
    segments.every((segment) => segment !== "" && segment !== "." && segment !== "..")
  );
}

function forbidden(relative: string): boolean {
  return (
    /(^|\/)(src|tests?|node_modules)\//u.test(relative) ||
    /\.(map|d\.ts|d\.mts|d\.cts)$/u.test(relative)
  );
}

async function readTree(root: string, relative: string, files: Map<string, Buffer>): Promise<void> {
  const directory = path.join(root, relative);
  for (const entry of (await readdir(directory, { withFileTypes: true })).sort((left, right) =>
    compare(left.name, right.name),
  )) {
    const childRelative = `${relative}/${entry.name}`;
    const stats = await lstat(path.join(directory, entry.name));
    if (stats.isDirectory()) {
      await readTree(root, childRelative, files);
    } else if (stats.isFile()) {
      files.set(childRelative, await readFile(path.join(directory, entry.name)));
    } else {
      throw new ModulePackError("MODULE_PACK_MIGRATIONS_INVALID", childRelative);
    }
  }
}

interface BackendBundle {
  readonly outputs: ReadonlyMap<string, Buffer>;
  readonly inputs: readonly string[];
}

async function bundleBackend(
  moduleDirectory: string,
  manifest: ModuleManifest,
  nodeRange: string,
): Promise<BackendBundle> {
  const declared = [
    manifest.entrypoints.server,
    manifest.entrypoints.schema,
    manifest.entrypoints.jobs,
  ].filter((entrypoint): entrypoint is string => entrypoint !== undefined);
  const entryPoints: { in: string; out: string }[] = [];
  for (const entrypoint of declared) {
    const relative = moduleRelative(entrypoint);
    if (!relative.endsWith(".js")) {
      throw new ModulePackError("MODULE_PACK_ENTRYPOINT_UNSUPPORTED", relative);
    }
    try {
      if (!(await lstat(path.join(moduleDirectory, relative))).isFile()) {
        throw new Error("not a file");
      }
    } catch {
      throw new ModulePackError("MODULE_PACK_ENTRYPOINT_MISSING", relative);
    }
    entryPoints.push({ in: `./${relative}`, out: relative.slice(0, -".js".length) });
  }

  const minimum = minVersion(nodeRange);
  let result: Awaited<ReturnType<typeof build>>;
  try {
    result = await build({
      absWorkingDir: moduleDirectory,
      banner: { js: REQUIRE_BANNER },
      bundle: true,
      chunkNames: "dist/chunks/[name]-[hash]",
      entryPoints,
      format: "esm",
      // Licence comments stay in the code as well; THIRD_PARTY_NOTICES is what satisfies the
      // licences, never these comments alone.
      legalComments: "eof",
      logLevel: "silent",
      metafile: true,
      minify: false,
      outdir: moduleDirectory,
      platform: "node",
      plugins: [nativeAddonGuard],
      sourcemap: false,
      splitting: true,
      target: `node${minimum?.version ?? "24"}`,
      write: false,
    });
  } catch (error) {
    const errors =
      isRecord(error) && Array.isArray(error.errors) ? (error.errors as Message[]) : [];
    const native = errors.some((message) => message.pluginName === NATIVE_GUARD);
    throw new ModulePackError(
      native ? "MODULE_PACK_NATIVE_DEPENDENCY" : "MODULE_PACK_BUNDLE_FAILED",
      undefined,
      errors,
    );
  }
  // A warning is a bundle that might not run: a require it could not follow, an import it could
  // not see. An installed module has nothing to fall back on, so a warning fails packaging.
  if (result.warnings.length > 0) {
    throw new ModulePackError("MODULE_PACK_BUNDLE_FAILED", undefined, result.warnings);
  }
  const { metafile, outputFiles } = result;
  if (metafile === undefined || outputFiles === undefined) {
    throw new ModulePackError("MODULE_PACK_BUNDLE_FAILED");
  }
  for (const output of Object.values(metafile.outputs)) {
    for (const imported of output.imports) {
      if (imported.external && !isBuiltin(imported.path)) {
        throw new ModulePackError("MODULE_PACK_EXTERNAL_DEPENDENCY", imported.path);
      }
    }
  }
  const outputs = new Map<string, Buffer>();
  for (const outputFile of outputFiles) {
    outputs.set(
      path.relative(moduleDirectory, outputFile.path).split(path.sep).join("/"),
      Buffer.from(outputFile.contents),
    );
  }
  return {
    inputs: Object.keys(metafile.inputs).map((input) => path.resolve(moduleDirectory, input)),
    outputs,
  };
}

export async function packModule(options: PackModuleOptions): Promise<ModulePackResult> {
  if (options.target !== MODULE_PACKAGE_ANY_TARGET) {
    throw new ModulePackError("MODULE_PACK_TARGET_UNSUPPORTED", options.target);
  }
  if (!/^(?:[0-9a-f]{40}|[0-9a-f]{64})$/u.test(options.sourceRevision)) {
    throw new ModulePackError("MODULE_PACK_REVISION_INVALID");
  }
  const moduleDirectory = await realpath(path.resolve(options.moduleDirectory)).catch(() => {
    throw new ModulePackError("MODULE_PACK_MANIFEST_INVALID");
  });

  const manifestBytes = await readFile(path.join(moduleDirectory, "ekklesia.module.json")).catch(
    () => {
      throw new ModulePackError("MODULE_PACK_MANIFEST_INVALID");
    },
  );
  let manifestInput: unknown;
  try {
    manifestInput = JSON.parse(manifestBytes.toString("utf8"));
  } catch {
    throw new ModulePackError("MODULE_PACK_MANIFEST_INVALID");
  }
  const parsedManifest = safeParseModuleManifest(manifestInput);
  if (!parsedManifest.success) {
    throw new ModulePackError("MODULE_PACK_MANIFEST_INVALID");
  }
  const manifest = parsedManifest.data;
  const developmentPackage = await readJson(path.join(moduleDirectory, "package.json")).catch(
    () => undefined,
  );
  if (
    !isRecord(developmentPackage) ||
    typeof developmentPackage.name !== "string" ||
    developmentPackage.name === "" ||
    developmentPackage.version !== manifest.version
  ) {
    throw new ModulePackError("MODULE_PACK_IDENTITY_MISMATCH");
  }

  const nodeRange = options.nodeRange ?? (await workspaceNodeRange(moduleDirectory));
  if (validRange(nodeRange) === null) {
    throw new ModulePackError("MODULE_PACK_RUNTIME_UNKNOWN");
  }

  // 1. The backend, bundled with every pure-JavaScript dependency it uses.
  const backend = await bundleBackend(moduleDirectory, manifest, nodeRange);

  // 2. The browser artifacts, exactly as built, with what went into each.
  let ui: Awaited<ReturnType<typeof analyzeModuleUi>>;
  try {
    ui = await analyzeModuleUi({ moduleDirectory });
  } catch (error) {
    throw new ModulePackError(
      "MODULE_PACK_UI_STALE",
      error instanceof ModuleUiBuildError ? error.code : undefined,
    );
  }

  // 3. Every bundled source file traced to its package.
  const index = new BundledPackageIndex();
  const components = new Map<
    string,
    { bundled: Awaited<ReturnType<typeof index.packageOf>>; uses: Set<NoticeUse> }
  >();
  const incorporates = new Map<string, string>();
  const drizzleVersions = new Set<string>();
  const sources: { readonly file: string; readonly use: NoticeUse }[] = [
    ...backend.inputs.map((file) => ({ file, use: "backend" as const })),
    ...ui.artifacts.flatMap((artifact) =>
      artifact.inputs.map((file) => ({ file, use: "browser" as const })),
    ),
  ];
  for (const { file, use } of sources) {
    const bundled = await index.packageOf(file);
    if (bundled === undefined || bundled.root === moduleDirectory) {
      continue;
    }
    if (!bundled.thirdParty) {
      if (bundled.name.startsWith("@ekklesia/")) {
        const known = incorporates.get(bundled.name);
        if (known !== undefined && known !== bundled.version) {
          throw new ModulePackError("MODULE_PACK_INCORPORATED_CONFLICT", bundled.name);
        }
        incorporates.set(bundled.name, bundled.version);
      }
      continue;
    }
    if (use === "backend" && bundled.name === "drizzle-orm") {
      drizzleVersions.add(bundled.version);
    }
    const key = `${bundled.name}@${bundled.version}`;
    const existing = components.get(key);
    if (existing) {
      existing.uses.add(use);
    } else {
      if (use === "backend" && (await isNativeOrPlatformSpecific(bundled))) {
        throw new ModulePackError("MODULE_PACK_NATIVE_DEPENDENCY", key);
      }
      components.set(key, { bundled, uses: new Set([use]) });
    }
  }
  if (drizzleVersions.size > 1) {
    throw new ModulePackError("MODULE_PACK_DATABASE_RUNTIME_CONFLICT");
  }
  const sdk = incorporates.get("@ekklesia/module-sdk");
  if (sdk !== undefined && !satisfies(sdk, manifest.compatibility.moduleSdk)) {
    throw new ModulePackError("MODULE_PACK_SDK_INCOMPATIBLE", sdk);
  }

  // 4. The payload.
  const files = new Map<string, Buffer>();
  const add = (relative: string, data: Buffer): void => {
    if (!portable(relative)) {
      throw new ModulePackError("MODULE_PACK_PATH_UNSUPPORTED", relative);
    }
    if (forbidden(relative)) {
      throw new ModulePackError("MODULE_PACK_FORBIDDEN_FILE", relative);
    }
    if (files.has(relative) || relative === MODULE_PACKAGE_METADATA_FILENAME) {
      throw new ModulePackError("MODULE_PACK_LAYOUT_CONFLICT", relative);
    }
    files.set(relative, data);
  };
  for (const [relative, data] of backend.outputs) {
    add(relative, data);
  }
  for (const artifact of ui.artifacts) {
    if (!artifact.builtArtifactCurrent) {
      throw new ModulePackError(
        "MODULE_PACK_UI_STALE",
        path.relative(moduleDirectory, artifact.outputFile),
      );
    }
    add(
      path.relative(moduleDirectory, artifact.outputFile).split(path.sep).join("/"),
      await readFile(artifact.outputFile),
    );
  }
  if (manifest.entrypoints.migrations !== undefined) {
    const migrations = new Map<string, Buffer>();
    try {
      await readTree(moduleDirectory, moduleRelative(manifest.entrypoints.migrations), migrations);
    } catch (error) {
      if (error instanceof ModulePackError) {
        throw error;
      }
      throw new ModulePackError("MODULE_PACK_MIGRATIONS_INVALID");
    }
    for (const [relative, data] of migrations) {
      add(relative, data);
    }
  }
  add("ekklesia.module.json", manifestBytes);
  add(
    "package.json",
    Buffer.from(
      `${JSON.stringify(
        modulePackageRuntimePackage({ name: developmentPackage.name, version: manifest.version }),
        null,
        2,
      )}\n`,
    ),
  );
  try {
    add(
      MODULE_PACKAGE_NOTICES_FILENAME,
      Buffer.from(await thirdPartyNotices(components as ReadonlyMap<string, NoticeComponent>)),
    );
  } catch (error) {
    if (error instanceof NoticeError) {
      throw new ModulePackError(error.code, error.component);
    }
    throw error;
  }

  // 5. The metadata, checked against its own contract before anything is written.
  const listed = [...files.entries()]
    .sort(([left], [right]) => compare(left, right))
    .map(([relative, data]) => ({ path: relative, sha256: sha256(data), size: data.length }));
  const metadata = {
    packageFormat: MODULE_PACKAGE_FORMAT,
    module: { id: manifest.id, version: manifest.version },
    target: MODULE_PACKAGE_ANY_TARGET,
    runtime: { node: nodeRange },
    provenance: {
      revision: options.sourceRevision,
      ...(options.repository === undefined ? {} : { repository: options.repository }),
      packager: await packagerIdentity(),
    },
    incorporates: Object.fromEntries(
      [...incorporates].sort(([left], [right]) => compare(left, right)),
    ),
    ...(drizzleVersions.size === 1
      ? { transitional: { databaseRuntime: { "drizzle-orm": [...drizzleVersions][0] } } }
      : {}),
    files: listed.map((file) => ({ path: file.path, size: file.size, sha256: file.sha256 })),
  };
  if (!safeParseModulePackageMetadata(metadata).success) {
    throw new ModulePackError("MODULE_PACK_METADATA_INVALID");
  }
  const payload = new Map(files);
  payload.set(
    MODULE_PACKAGE_METADATA_FILENAME,
    Buffer.from(`${JSON.stringify(metadata, null, 2)}\n`),
  );

  // 6. The archive and its digest.
  let archive: Buffer;
  try {
    archive = deterministicTarGz(manifest.id, payload);
  } catch (error) {
    if (error instanceof TarPathError) {
      throw new ModulePackError("MODULE_PACK_PATH_UNSUPPORTED", error.path);
    }
    throw error;
  }
  const digest = sha256(archive);
  const fileName = modulePackageArchiveFileName(
    manifest.id,
    manifest.version,
    MODULE_PACKAGE_ANY_TARGET,
  );
  const outDirectory = path.resolve(options.outDirectory);
  const archivePath = path.join(outDirectory, fileName);
  const sha256Path = `${archivePath}.sha256`;
  try {
    await mkdir(outDirectory, { recursive: true });
    const staging = `${archivePath}.partial-${process.pid}`;
    await writeFile(staging, archive);
    await rename(staging, archivePath);
    await writeFile(`${sha256Path}.partial-${process.pid}`, `${digest}  ${fileName}\n`);
    await rename(`${sha256Path}.partial-${process.pid}`, sha256Path);
  } catch {
    throw new ModulePackError("MODULE_PACK_OUTPUT_FAILED");
  }

  return Object.freeze({
    archiveBytes: archive.length,
    archivePath,
    fileCount: listed.length,
    moduleId: manifest.id,
    sha256: digest,
    sha256Path,
    target: MODULE_PACKAGE_ANY_TARGET,
    version: manifest.version,
  });
}