import { createHash } from "node:crypto"; import { lstat, mkdir, readdir, readFile, realpath, rename, writeFile } from "node:fs/promises"; import { isBuiltin } from "node:module"; import path from "node:path"; import { fileURLToPath } from "node:url"; import { MODULE_PACKAGE_ANY_TARGET, MODULE_PACKAGE_FORMAT, MODULE_PACKAGE_METADATA_FILENAME, MODULE_PACKAGE_NOTICES_FILENAME, type ModuleManifest, modulePackageArchiveFileName, modulePackageRuntimePackage, safeParseModuleManifest, safeParseModulePackageMetadata, } from "@ekklesia/contracts"; import { analyzeModuleUi, ModuleUiBuildError } from "@ekklesia/module-ui-build"; import { build, type Message, type Plugin } from "esbuild"; import { minVersion, satisfies, validRange } from "semver"; import { NoticeError, type NoticeComponent, type NoticeUse, thirdPartyNotices } from "./notices.js"; import { BundledPackageIndex, isNativeOrPlatformSpecific } from "./packages.js"; import { deterministicTarGz, TarPathError } from "./tar.js"; /** * The canonical Ekklesia module packager (ADR 0049): turns a built module directory into one * deterministic Module Package v1 archive and its `.sha256` line. * * It reads the module directory and the dependencies its workspace installed, and writes only the * two output files. It never installs, links, or deploys anything, so the workspace it runs in is * left exactly as it was. In this release it produces the portable `any` target only. */ export type ModulePackErrorCode = | "MODULE_PACK_BUNDLE_FAILED" | "MODULE_PACK_DATABASE_RUNTIME_CONFLICT" | "MODULE_PACK_ENTRYPOINT_MISSING" | "MODULE_PACK_ENTRYPOINT_UNSUPPORTED" | "MODULE_PACK_EXTERNAL_DEPENDENCY" | "MODULE_PACK_FORBIDDEN_FILE" | "MODULE_PACK_IDENTITY_MISMATCH" | "MODULE_PACK_INCORPORATED_CONFLICT" | "MODULE_PACK_LAYOUT_CONFLICT" | "MODULE_PACK_LICENCE_TEXT_MISSING" | "MODULE_PACK_LICENCE_UNKNOWN" | "MODULE_PACK_MANIFEST_INVALID" | "MODULE_PACK_METADATA_INVALID" | "MODULE_PACK_MIGRATIONS_INVALID" | "MODULE_PACK_NATIVE_DEPENDENCY" | "MODULE_PACK_OUTPUT_FAILED" | "MODULE_PACK_PATH_UNSUPPORTED" | "MODULE_PACK_REVISION_INVALID" | "MODULE_PACK_RUNTIME_UNKNOWN" | "MODULE_PACK_SDK_INCOMPATIBLE" | "MODULE_PACK_TARGET_UNSUPPORTED" | "MODULE_PACK_UI_STALE"; const messages: Readonly> = Object.freeze({ MODULE_PACK_BUNDLE_FAILED: "The backend could not be bundled into self-contained files. Every import must resolve and bundle cleanly.", MODULE_PACK_DATABASE_RUNTIME_CONFLICT: "The backend bundles more than one Drizzle version.", MODULE_PACK_ENTRYPOINT_MISSING: "A declared backend entrypoint does not exist. Build the module before packaging it.", MODULE_PACK_ENTRYPOINT_UNSUPPORTED: "Backend entrypoints must be compiled .js files to be bundled into a Module Package v1.", MODULE_PACK_EXTERNAL_DEPENDENCY: "The backend bundle would import a dependency at run time; an any-target package bundles everything.", MODULE_PACK_FORBIDDEN_FILE: "The package would contain a source, test, declaration, source-map, or dependency file.", MODULE_PACK_IDENTITY_MISMATCH: "package.json must declare a name and the same version as ekklesia.module.json.", MODULE_PACK_INCORPORATED_CONFLICT: "The package bundles more than one version of an Ekklesia package.", MODULE_PACK_LAYOUT_CONFLICT: "Two package files would have the same path.", MODULE_PACK_LICENCE_TEXT_MISSING: "A bundled package ships no licence text, and its licence has no complete standard text to supply.", MODULE_PACK_LICENCE_UNKNOWN: "A bundled package declares no licence that permits redistribution.", MODULE_PACK_MANIFEST_INVALID: "ekklesia.module.json is missing or invalid.", MODULE_PACK_METADATA_INVALID: "The generated package metadata failed its own contract.", MODULE_PACK_MIGRATIONS_INVALID: "The migrations directory must hold only regular files and directories.", MODULE_PACK_NATIVE_DEPENDENCY: "The backend bundles a native or platform-specific dependency, which an any-target package cannot contain.", MODULE_PACK_OUTPUT_FAILED: "The package could not be written to the output directory.", MODULE_PACK_PATH_UNSUPPORTED: "A package path is not portable printable ASCII that fits the archive format.", MODULE_PACK_REVISION_INVALID: "The source revision must be a full 40- or 64-character commit id.", MODULE_PACK_RUNTIME_UNKNOWN: "The Node.js range could not be determined. Run inside an Ekklesia workspace, or pass one.", MODULE_PACK_SDK_INCOMPATIBLE: "The bundled Module SDK is outside the manifest's own compatibility.moduleSdk range.", MODULE_PACK_TARGET_UNSUPPORTED: "This packager produces the any target only.", MODULE_PACK_UI_STALE: "A declared browser artifact is missing or out of date. Build the module before packaging it.", }); export class ModulePackError extends Error { readonly code: ModulePackErrorCode; /** The package or path the failure concerns, when there is one. A module author's own build * input, never a secret. */ readonly detail?: string; /** esbuild's structured diagnostics, for a command line to print. */ readonly diagnostics: readonly Message[]; constructor(code: ModulePackErrorCode, detail?: string, diagnostics: readonly Message[] = []) { super(detail === undefined ? messages[code] : `${messages[code]} (${detail})`); this.name = "ModulePackError"; this.code = code; if (detail !== undefined) { this.detail = detail; } this.diagnostics = diagnostics; } } export interface PackModuleOptions { /** A built module directory: its manifest, package.json, and compiled entrypoints. */ readonly moduleDirectory: string; /** `any` only, in this release. */ readonly target: string; readonly outDirectory: string; /** The full commit id the module was built from. */ readonly sourceRevision: string; /** Informational only; recorded as given. */ readonly repository?: string; /** The Node.js range the package is built for. Defaults to the enclosing workspace's * `engines.node`, the toolchain the module was built and tested with. */ readonly nodeRange?: string; } export interface ModulePackResult { readonly archivePath: string; readonly sha256Path: string; readonly sha256: string; readonly archiveBytes: number; readonly moduleId: string; readonly version: string; readonly target: string; /** Payload files, the metadata file not included. */ readonly fileCount: number; } const NATIVE_GUARD = "ekklesia-native-addon-guard"; /** Refuses a compiled addon the moment the bundle reaches one. */ const nativeAddonGuard: Plugin = { name: NATIVE_GUARD, setup(pluginBuild) { pluginBuild.onLoad({ filter: /\.node$/i }, () => ({ errors: [{ text: "A compiled Node.js addon cannot be bundled into an any-target package." }], })); }, }; /** Bundled CommonJS code that requires a Node built-in needs a `require` in module scope. Nothing * else can be required at run time: every other dependency is inside the bundle, and the packager * refuses a bundle that would import anything but a built-in. */ const REQUIRE_BANNER = 'import { createRequire as __ekklesiaCreateRequire } from "node:module";\nconst require = __ekklesiaCreateRequire(import.meta.url);'; function compare(left: string, right: string): number { return left < right ? -1 : left > right ? 1 : 0; } function sha256(data: Uint8Array): string { return createHash("sha256").update(data).digest("hex"); } function isRecord(value: unknown): value is Record { return typeof value === "object" && value !== null && !Array.isArray(value); } async function readJson(file: string): Promise { return JSON.parse(await readFile(file, "utf8")) as unknown; } const ownManifestPath = path.resolve( path.dirname(fileURLToPath(import.meta.url)), "..", "package.json", ); async function packagerIdentity(): Promise<{ readonly name: string; readonly version: string }> { const manifest = (await readJson(ownManifestPath)) as Record; return { name: String(manifest.name), version: String(manifest.version) }; } async function workspaceNodeRange(moduleDirectory: string): Promise { let directory = moduleDirectory; for (;;) { try { await lstat(path.join(directory, "pnpm-workspace.yaml")); const manifest = await readJson(path.join(directory, "package.json")); const engines = isRecord(manifest) ? manifest.engines : undefined; const range = isRecord(engines) ? engines.node : undefined; if (typeof range === "string") { return range; } break; } catch { const parent = path.dirname(directory); if (parent === directory) { break; } directory = parent; } } throw new ModulePackError("MODULE_PACK_RUNTIME_UNKNOWN"); } /** `./dist/server/index.js` → `dist/server/index.js`, inside the module directory. */ function moduleRelative(declared: string): string { return path.posix.normalize(declared.replace(/^\.\//u, "")); } function portable(relative: string): boolean { const segments = relative.split("/"); return ( /^[\x20-\x7e]+$/u.test(relative) && !relative.includes("\\") && segments.every((segment) => segment !== "" && segment !== "." && segment !== "..") ); } function forbidden(relative: string): boolean { return ( /(^|\/)(src|tests?|node_modules)\//u.test(relative) || /\.(map|d\.ts|d\.mts|d\.cts)$/u.test(relative) ); } async function readTree(root: string, relative: string, files: Map): Promise { const directory = path.join(root, relative); for (const entry of (await readdir(directory, { withFileTypes: true })).sort((left, right) => compare(left.name, right.name), )) { const childRelative = `${relative}/${entry.name}`; const stats = await lstat(path.join(directory, entry.name)); if (stats.isDirectory()) { await readTree(root, childRelative, files); } else if (stats.isFile()) { files.set(childRelative, await readFile(path.join(directory, entry.name))); } else { throw new ModulePackError("MODULE_PACK_MIGRATIONS_INVALID", childRelative); } } } interface BackendBundle { readonly outputs: ReadonlyMap; readonly inputs: readonly string[]; } async function bundleBackend( moduleDirectory: string, manifest: ModuleManifest, nodeRange: string, ): Promise { const declared = [ manifest.entrypoints.server, manifest.entrypoints.schema, manifest.entrypoints.jobs, ].filter((entrypoint): entrypoint is string => entrypoint !== undefined); const entryPoints: { in: string; out: string }[] = []; for (const entrypoint of declared) { const relative = moduleRelative(entrypoint); if (!relative.endsWith(".js")) { throw new ModulePackError("MODULE_PACK_ENTRYPOINT_UNSUPPORTED", relative); } try { if (!(await lstat(path.join(moduleDirectory, relative))).isFile()) { throw new Error("not a file"); } } catch { throw new ModulePackError("MODULE_PACK_ENTRYPOINT_MISSING", relative); } entryPoints.push({ in: `./${relative}`, out: relative.slice(0, -".js".length) }); } const minimum = minVersion(nodeRange); let result: Awaited>; try { result = await build({ absWorkingDir: moduleDirectory, banner: { js: REQUIRE_BANNER }, bundle: true, chunkNames: "dist/chunks/[name]-[hash]", entryPoints, format: "esm", // Licence comments stay in the code as well; THIRD_PARTY_NOTICES is what satisfies the // licences, never these comments alone. legalComments: "eof", logLevel: "silent", metafile: true, minify: false, outdir: moduleDirectory, platform: "node", plugins: [nativeAddonGuard], sourcemap: false, splitting: true, target: `node${minimum?.version ?? "24"}`, write: false, }); } catch (error) { const errors = isRecord(error) && Array.isArray(error.errors) ? (error.errors as Message[]) : []; const native = errors.some((message) => message.pluginName === NATIVE_GUARD); throw new ModulePackError( native ? "MODULE_PACK_NATIVE_DEPENDENCY" : "MODULE_PACK_BUNDLE_FAILED", undefined, errors, ); } // A warning is a bundle that might not run: a require it could not follow, an import it could // not see. An installed module has nothing to fall back on, so a warning fails packaging. if (result.warnings.length > 0) { throw new ModulePackError("MODULE_PACK_BUNDLE_FAILED", undefined, result.warnings); } const { metafile, outputFiles } = result; if (metafile === undefined || outputFiles === undefined) { throw new ModulePackError("MODULE_PACK_BUNDLE_FAILED"); } for (const output of Object.values(metafile.outputs)) { for (const imported of output.imports) { if (imported.external && !isBuiltin(imported.path)) { throw new ModulePackError("MODULE_PACK_EXTERNAL_DEPENDENCY", imported.path); } } } const outputs = new Map(); for (const outputFile of outputFiles) { outputs.set( path.relative(moduleDirectory, outputFile.path).split(path.sep).join("/"), Buffer.from(outputFile.contents), ); } return { inputs: Object.keys(metafile.inputs).map((input) => path.resolve(moduleDirectory, input)), outputs, }; } export async function packModule(options: PackModuleOptions): Promise { if (options.target !== MODULE_PACKAGE_ANY_TARGET) { throw new ModulePackError("MODULE_PACK_TARGET_UNSUPPORTED", options.target); } if (!/^(?:[0-9a-f]{40}|[0-9a-f]{64})$/u.test(options.sourceRevision)) { throw new ModulePackError("MODULE_PACK_REVISION_INVALID"); } const moduleDirectory = await realpath(path.resolve(options.moduleDirectory)).catch(() => { throw new ModulePackError("MODULE_PACK_MANIFEST_INVALID"); }); const manifestBytes = await readFile(path.join(moduleDirectory, "ekklesia.module.json")).catch( () => { throw new ModulePackError("MODULE_PACK_MANIFEST_INVALID"); }, ); let manifestInput: unknown; try { manifestInput = JSON.parse(manifestBytes.toString("utf8")); } catch { throw new ModulePackError("MODULE_PACK_MANIFEST_INVALID"); } const parsedManifest = safeParseModuleManifest(manifestInput); if (!parsedManifest.success) { throw new ModulePackError("MODULE_PACK_MANIFEST_INVALID"); } const manifest = parsedManifest.data; const developmentPackage = await readJson(path.join(moduleDirectory, "package.json")).catch( () => undefined, ); if ( !isRecord(developmentPackage) || typeof developmentPackage.name !== "string" || developmentPackage.name === "" || developmentPackage.version !== manifest.version ) { throw new ModulePackError("MODULE_PACK_IDENTITY_MISMATCH"); } const nodeRange = options.nodeRange ?? (await workspaceNodeRange(moduleDirectory)); if (validRange(nodeRange) === null) { throw new ModulePackError("MODULE_PACK_RUNTIME_UNKNOWN"); } // 1. The backend, bundled with every pure-JavaScript dependency it uses. const backend = await bundleBackend(moduleDirectory, manifest, nodeRange); // 2. The browser artifacts, exactly as built, with what went into each. let ui: Awaited>; try { ui = await analyzeModuleUi({ moduleDirectory }); } catch (error) { throw new ModulePackError( "MODULE_PACK_UI_STALE", error instanceof ModuleUiBuildError ? error.code : undefined, ); } // 3. Every bundled source file traced to its package. const index = new BundledPackageIndex(); const components = new Map< string, { bundled: Awaited>; uses: Set } >(); const incorporates = new Map(); const drizzleVersions = new Set(); const sources: { readonly file: string; readonly use: NoticeUse }[] = [ ...backend.inputs.map((file) => ({ file, use: "backend" as const })), ...ui.artifacts.flatMap((artifact) => artifact.inputs.map((file) => ({ file, use: "browser" as const })), ), ]; for (const { file, use } of sources) { const bundled = await index.packageOf(file); if (bundled === undefined || bundled.root === moduleDirectory) { continue; } if (!bundled.thirdParty) { if (bundled.name.startsWith("@ekklesia/")) { const known = incorporates.get(bundled.name); if (known !== undefined && known !== bundled.version) { throw new ModulePackError("MODULE_PACK_INCORPORATED_CONFLICT", bundled.name); } incorporates.set(bundled.name, bundled.version); } continue; } if (use === "backend" && bundled.name === "drizzle-orm") { drizzleVersions.add(bundled.version); } const key = `${bundled.name}@${bundled.version}`; const existing = components.get(key); if (existing) { existing.uses.add(use); } else { if (use === "backend" && (await isNativeOrPlatformSpecific(bundled))) { throw new ModulePackError("MODULE_PACK_NATIVE_DEPENDENCY", key); } components.set(key, { bundled, uses: new Set([use]) }); } } if (drizzleVersions.size > 1) { throw new ModulePackError("MODULE_PACK_DATABASE_RUNTIME_CONFLICT"); } const sdk = incorporates.get("@ekklesia/module-sdk"); if (sdk !== undefined && !satisfies(sdk, manifest.compatibility.moduleSdk)) { throw new ModulePackError("MODULE_PACK_SDK_INCOMPATIBLE", sdk); } // 4. The payload. const files = new Map(); const add = (relative: string, data: Buffer): void => { if (!portable(relative)) { throw new ModulePackError("MODULE_PACK_PATH_UNSUPPORTED", relative); } if (forbidden(relative)) { throw new ModulePackError("MODULE_PACK_FORBIDDEN_FILE", relative); } if (files.has(relative) || relative === MODULE_PACKAGE_METADATA_FILENAME) { throw new ModulePackError("MODULE_PACK_LAYOUT_CONFLICT", relative); } files.set(relative, data); }; for (const [relative, data] of backend.outputs) { add(relative, data); } for (const artifact of ui.artifacts) { if (!artifact.builtArtifactCurrent) { throw new ModulePackError( "MODULE_PACK_UI_STALE", path.relative(moduleDirectory, artifact.outputFile), ); } add( path.relative(moduleDirectory, artifact.outputFile).split(path.sep).join("/"), await readFile(artifact.outputFile), ); } if (manifest.entrypoints.migrations !== undefined) { const migrations = new Map(); try { await readTree(moduleDirectory, moduleRelative(manifest.entrypoints.migrations), migrations); } catch (error) { if (error instanceof ModulePackError) { throw error; } throw new ModulePackError("MODULE_PACK_MIGRATIONS_INVALID"); } for (const [relative, data] of migrations) { add(relative, data); } } add("ekklesia.module.json", manifestBytes); add( "package.json", Buffer.from( `${JSON.stringify( modulePackageRuntimePackage({ name: developmentPackage.name, version: manifest.version }), null, 2, )}\n`, ), ); try { add( MODULE_PACKAGE_NOTICES_FILENAME, Buffer.from(await thirdPartyNotices(components as ReadonlyMap)), ); } catch (error) { if (error instanceof NoticeError) { throw new ModulePackError(error.code, error.component); } throw error; } // 5. The metadata, checked against its own contract before anything is written. const listed = [...files.entries()] .sort(([left], [right]) => compare(left, right)) .map(([relative, data]) => ({ path: relative, sha256: sha256(data), size: data.length })); const metadata = { packageFormat: MODULE_PACKAGE_FORMAT, module: { id: manifest.id, version: manifest.version }, target: MODULE_PACKAGE_ANY_TARGET, runtime: { node: nodeRange }, provenance: { revision: options.sourceRevision, ...(options.repository === undefined ? {} : { repository: options.repository }), packager: await packagerIdentity(), }, incorporates: Object.fromEntries( [...incorporates].sort(([left], [right]) => compare(left, right)), ), ...(drizzleVersions.size === 1 ? { transitional: { databaseRuntime: { "drizzle-orm": [...drizzleVersions][0] } } } : {}), files: listed.map((file) => ({ path: file.path, size: file.size, sha256: file.sha256 })), }; if (!safeParseModulePackageMetadata(metadata).success) { throw new ModulePackError("MODULE_PACK_METADATA_INVALID"); } const payload = new Map(files); payload.set( MODULE_PACKAGE_METADATA_FILENAME, Buffer.from(`${JSON.stringify(metadata, null, 2)}\n`), ); // 6. The archive and its digest. let archive: Buffer; try { archive = deterministicTarGz(manifest.id, payload); } catch (error) { if (error instanceof TarPathError) { throw new ModulePackError("MODULE_PACK_PATH_UNSUPPORTED", error.path); } throw error; } const digest = sha256(archive); const fileName = modulePackageArchiveFileName( manifest.id, manifest.version, MODULE_PACKAGE_ANY_TARGET, ); const outDirectory = path.resolve(options.outDirectory); const archivePath = path.join(outDirectory, fileName); const sha256Path = `${archivePath}.sha256`; try { await mkdir(outDirectory, { recursive: true }); const staging = `${archivePath}.partial-${process.pid}`; await writeFile(staging, archive); await rename(staging, archivePath); await writeFile(`${sha256Path}.partial-${process.pid}`, `${digest} ${fileName}\n`); await rename(`${sha256Path}.partial-${process.pid}`, sha256Path); } catch { throw new ModulePackError("MODULE_PACK_OUTPUT_FAILED"); } return Object.freeze({ archiveBytes: archive.length, archivePath, fileCount: listed.length, moduleId: manifest.id, sha256: digest, sha256Path, target: MODULE_PACKAGE_ANY_TARGET, version: manifest.version, }); }