diff --git a/README.md b/README.md index bd5f06c..175501d 100644 --- a/README.md +++ b/README.md @@ -44,8 +44,12 @@ selected by `EKKLESIA_MODULES_DIR` (`/var/lib/ekklesia/modules` in the image, th self-contained directory carrying its own production dependencies. Installing, updating, or removing a module needs only a controlled restart and, when it has migrations, an explicit plan and apply. Nothing is hot loaded. Removing a module's code keeps its data and its stored state; its applied -migration history is retained and reported, and blocks no other migration. Purge, a package format, -signing, and any installer or marketplace do not exist yet. +migration history is retained and reported, and blocks no other migration. A module is distributed +as one **Ekklesia Module Package v1** +([ADR 0049](docs/architecture/decisions/0049-module-package-v1.md)): a deterministic `.tar.gz` and +its SHA-256, produced by the canonical packager (`@ekklesia/module-pack`) with the module's backend +bundled, and checked by Core's verifier before it is installed. Only the portable `any` target exists +so far. Purge, native-target packages, signing, and any installer or marketplace do not exist yet. Phase 4A added Core-owned identity foundations: a Core-owned Better Auth database schema (`ekklesia_core_user`, `ekklesia_core_session`, `ekklesia_core_account`, diff --git a/core/src/cli/verify-module-package.ts b/core/src/cli/verify-module-package.ts new file mode 100644 index 0000000..6b88808 --- /dev/null +++ b/core/src/cli/verify-module-package.ts @@ -0,0 +1,175 @@ +import { readFile } from "node:fs/promises"; +import path from "node:path"; +import { pathToFileURL } from "node:url"; +import { + type ModulePackagePlatform, + ModulePackageVerificationError, + verifyModulePackage, +} from "../modules/index.js"; + +/** + * `verify-module-package` — verifies an Ekklesia Module Package v1 archive against this platform + * (ADR 0049), and never installs it. + * + * node core/dist/cli/verify-module-package.js + * [--sha256 | --sha256-file ] the archive's expected SHA-256 + * [--extract-to ] write the verified module directory there + * + * Run from the platform's root (`/opt/ekklesia` in the image), like every operator command: the + * platform's own versions are read from there. `--extract-to` writes only after every check passed, + * into a directory that must not exist yet; it is the staging step of an installation, not the + * installation itself. + */ + +class VerifyArgumentsError extends Error {} + +const USAGE = + "Usage: verify-module-package [--sha256 | --sha256-file ] [--extract-to ]"; + +export interface VerifyModulePackageArguments { + readonly archivePath: string; + readonly expectedSha256?: string; + readonly sha256File?: string; + readonly extractTo?: string; +} + +function normalizedDigest(value: string): string { + const digest = value.trim().toLowerCase(); + if (!/^[0-9a-f]{64}$/u.test(digest)) { + throw new VerifyArgumentsError("The SHA-256 must be 64 hexadecimal characters."); + } + return digest; +} + +export function parseVerifyModulePackageArguments( + argv: readonly string[], +): VerifyModulePackageArguments { + const positional: string[] = []; + const options = new Map(); + for (let index = 0; index < argv.length; index += 1) { + const argument = argv[index] ?? ""; + if (argument.startsWith("--")) { + if (!["--sha256", "--sha256-file", "--extract-to"].includes(argument)) { + throw new VerifyArgumentsError(USAGE); + } + const value = argv[index + 1]; + if (value === undefined || value.startsWith("--") || options.has(argument)) { + throw new VerifyArgumentsError(USAGE); + } + options.set(argument, value); + index += 1; + } else { + positional.push(argument); + } + } + const archivePath = positional[0]; + if (positional.length !== 1 || archivePath === undefined || archivePath === "") { + throw new VerifyArgumentsError(USAGE); + } + if (options.has("--sha256") && options.has("--sha256-file")) { + throw new VerifyArgumentsError(USAGE); + } + const sha256 = options.get("--sha256"); + const sha256File = options.get("--sha256-file"); + const extractTo = options.get("--extract-to"); + return { + archivePath, + ...(sha256 === undefined ? {} : { expectedSha256: normalizedDigest(sha256) }), + ...(sha256File === undefined ? {} : { sha256File }), + ...(extractTo === undefined ? {} : { extractTo }), + }; +} + +/** The `sha256sum` format: one line, the digest, whitespace, and a file name that is never trusted. */ +export function parseSha256File(contents: string): string { + const lines = contents.split("\n").filter((line) => line.trim() !== ""); + const match = lines.length === 1 ? /^([0-9a-fA-F]{64})\s+\*?\S/u.exec(lines[0] ?? "") : null; + if (!match?.[1]) { + throw new VerifyArgumentsError("The SHA-256 file must hold one sha256sum line."); + } + return match[1].toLowerCase(); +} + +async function readJson(file: string): Promise> { + return JSON.parse(await readFile(file, "utf8")) as Record; +} + +function requiredString(value: unknown): string { + if (typeof value !== "string" || value === "") { + throw new VerifyArgumentsError("The platform's own package metadata could not be read."); + } + return value; +} + +/** What this platform is, read from where it is installed. The transitional database runtime is + * the Drizzle version Core itself depends on (ADR 0049). */ +export async function readModulePackagePlatform( + repositoryRoot: string, +): Promise { + const [root, sdk, core] = await Promise.all([ + readJson(path.join(repositoryRoot, "package.json")), + readJson(path.join(repositoryRoot, "packages", "module-sdk", "package.json")), + readJson(path.join(repositoryRoot, "core", "package.json")), + ]); + const dependencies = core.dependencies as Record | undefined; + return { + databaseRuntime: { "drizzle-orm": requiredString(dependencies?.["drizzle-orm"]) }, + moduleSdkVersion: requiredString(sdk.version), + platformVersion: requiredString(root.version), + }; +} + +async function main(): Promise { + const parsed = parseVerifyModulePackageArguments(process.argv.slice(2)); + let expectedSha256 = parsed.expectedSha256; + if (parsed.sha256File !== undefined) { + let contents: string; + try { + contents = await readFile(parsed.sha256File, "utf8"); + } catch { + throw new VerifyArgumentsError("The SHA-256 file could not be read."); + } + expectedSha256 = parseSha256File(contents); + } + const platform = await readModulePackagePlatform(process.cwd()).catch(() => { + throw new VerifyArgumentsError("The platform's own package metadata could not be read."); + }); + const result = await verifyModulePackage({ + archivePath: parsed.archivePath, + platform, + ...(expectedSha256 === undefined ? {} : { expectedSha256 }), + ...(parsed.extractTo === undefined ? {} : { extractTo: parsed.extractTo }), + }); + console.log( + `Module package verified: ${result.moduleId}@${result.version} (target ${result.target}), ${result.fileCount} files, sha256 ${result.archiveSha256}.`, + ); + if (expectedSha256 === undefined) { + console.log("The archive's SHA-256 was not checked: no expected digest was given."); + } + if (parsed.extractTo !== undefined) { + console.log("The verified module directory was extracted. Nothing was installed."); + } +} + +function isDirectExecution(): boolean { + const entrypoint = process.argv[1]; + return ( + entrypoint !== undefined && import.meta.url === pathToFileURL(path.resolve(entrypoint)).href + ); +} + +if (isDirectExecution()) { + main().catch((error: unknown) => { + if (error instanceof ModulePackageVerificationError) { + console.error(`[${error.code}] ${error.message}`); + if (error.issueCodes.length > 0) { + console.error(`Module validation issues: ${error.issueCodes.join(", ")}`); + } + } else if (error instanceof VerifyArgumentsError) { + console.error(error.message); + } else { + console.error("Module package verification failed unexpectedly."); + } + process.exitCode = 1; + }); +} diff --git a/core/src/modules/index.ts b/core/src/modules/index.ts index 0350420..020f131 100644 --- a/core/src/modules/index.ts +++ b/core/src/modules/index.ts @@ -8,6 +8,7 @@ export * from "./lifecycle-schema.js"; export * from "./lifecycle-store.js"; export * from "./migration-discovery.js"; export * from "./modules-directory.js"; +export * from "./module-package-verification.js"; export * from "./permissions.js"; export * from "./realtime.js"; export * from "./runtime.js"; diff --git a/core/src/modules/module-package-archive.ts b/core/src/modules/module-package-archive.ts new file mode 100644 index 0000000..5ded35a --- /dev/null +++ b/core/src/modules/module-package-archive.ts @@ -0,0 +1,203 @@ +import { gunzipSync } from "node:zlib"; + +/** + * Reads an Ekklesia Module Package v1 archive (ADR 0049) entirely in memory and enforces the + * archive-safety rules before anything is written anywhere. + * + * Deliberately a strict reader rather than a general tar implementation: v1 packages hold only + * POSIX ustar directories and regular files, so every other entry type is refused instead of + * interpreted. Nothing here touches the filesystem; extraction happens only after the caller has + * verified the whole package. + */ + +export type ModulePackageArchiveErrorCode = + | "MODULE_PACKAGE_LAYOUT_INVALID" + | "MODULE_PACKAGE_LINK" + | "MODULE_PACKAGE_SPECIAL_ENTRY" + | "MODULE_PACKAGE_TOO_LARGE" + | "MODULE_PACKAGE_UNREADABLE" + | "MODULE_PACKAGE_UNSAFE_MODE" + | "MODULE_PACKAGE_UNSAFE_PATH"; + +export class ModulePackageArchiveError extends Error { + readonly code: ModulePackageArchiveErrorCode; + + constructor(code: ModulePackageArchiveErrorCode) { + super(code); + this.name = "ModulePackageArchiveError"; + this.code = code; + } +} + +/** Generous for a module, small enough that a hostile archive cannot exhaust the host. */ +export const MODULE_PACKAGE_MAX_COMPRESSED_BYTES = 256 * 1024 * 1024; +export const MODULE_PACKAGE_MAX_UNCOMPRESSED_BYTES = 1024 * 1024 * 1024; +export const MODULE_PACKAGE_MAX_ENTRIES = 100_000; + +export interface ModulePackageArchiveEntry { + /** Relative to the top-level module directory; empty for that directory itself. */ + readonly path: string; + readonly type: "directory" | "file"; + readonly data: Buffer; +} + +export interface ModulePackageArchive { + /** The single top-level directory's name. Validated as a module id by the caller. */ + readonly topLevel: string; + /** Every entry except the top-level directory itself, in archive order. */ + readonly entries: readonly ModulePackageArchiveEntry[]; +} + +const BLOCK = 512; + +function field(block: Buffer, offset: number, length: number): string { + const raw = block.subarray(offset, offset + length); + const end = raw.indexOf(0); + return raw.subarray(0, end === -1 ? length : end).toString("latin1"); +} + +function octalField(block: Buffer, offset: number, length: number): number { + const text = field(block, offset, length).trim(); + if (!/^[0-7]+$/u.test(text)) { + throw new ModulePackageArchiveError("MODULE_PACKAGE_UNREADABLE"); + } + const value = Number.parseInt(text, 8); + if (!Number.isSafeInteger(value)) { + throw new ModulePackageArchiveError("MODULE_PACKAGE_UNREADABLE"); + } + return value; +} + +function headerChecksumValid(block: Buffer): boolean { + const declared = octalField(block, 148, 8); + let sum = 0; + for (let index = 0; index < BLOCK; index += 1) { + sum += index >= 148 && index < 156 ? 0x20 : (block[index] ?? 0); + } + return sum === declared; +} + +/** Printable ASCII, no backslash, relative, and no empty, `.`, or `..` segment. */ +function safeSegments(value: string): readonly string[] { + if ( + value.length === 0 || + !/^[\x20-\x7e]+$/u.test(value) || + value.includes("\\") || + value.startsWith("/") + ) { + throw new ModulePackageArchiveError("MODULE_PACKAGE_UNSAFE_PATH"); + } + const segments = value.split("/"); + if (segments.some((segment) => segment === "" || segment === "." || segment === "..")) { + throw new ModulePackageArchiveError("MODULE_PACKAGE_UNSAFE_PATH"); + } + return segments; +} + +export function readModulePackageArchive(compressed: Buffer): ModulePackageArchive { + if (compressed.length > MODULE_PACKAGE_MAX_COMPRESSED_BYTES) { + throw new ModulePackageArchiveError("MODULE_PACKAGE_TOO_LARGE"); + } + let raw: Buffer; + try { + raw = gunzipSync(compressed, { maxOutputLength: MODULE_PACKAGE_MAX_UNCOMPRESSED_BYTES }); + } catch (error) { + throw new ModulePackageArchiveError( + error instanceof RangeError ? "MODULE_PACKAGE_TOO_LARGE" : "MODULE_PACKAGE_UNREADABLE", + ); + } + + const entries: ModulePackageArchiveEntry[] = []; + const kinds = new Map(); + /** Every path that is an ancestor of some entry, so a file can never claim one. */ + const ancestors = new Set(); + let topLevel: string | undefined; + let offset = 0; + let ended = false; + + while (offset + BLOCK <= raw.length) { + const block = raw.subarray(offset, offset + BLOCK); + if (block.every((byte) => byte === 0)) { + ended = true; + break; + } + if (field(block, 257, 6) !== "ustar" || field(block, 263, 2) !== "00") { + throw new ModulePackageArchiveError("MODULE_PACKAGE_UNREADABLE"); + } + if (!headerChecksumValid(block)) { + throw new ModulePackageArchiveError("MODULE_PACKAGE_UNREADABLE"); + } + if (entries.length + 1 > MODULE_PACKAGE_MAX_ENTRIES) { + throw new ModulePackageArchiveError("MODULE_PACKAGE_TOO_LARGE"); + } + + const typeflag = field(block, 156, 1); + if (typeflag === "1" || typeflag === "2") { + throw new ModulePackageArchiveError("MODULE_PACKAGE_LINK"); + } + const type = + typeflag === "5" ? "directory" : typeflag === "0" || typeflag === "" ? "file" : undefined; + if (type === undefined) { + // Devices, FIFOs, contiguous files, and every extended or long-name header: v1 packages hold + // directories and regular files, so nothing else is interpreted. + throw new ModulePackageArchiveError("MODULE_PACKAGE_SPECIAL_ENTRY"); + } + + const mode = octalField(block, 100, 8); + if ((mode & 0o7000) !== 0) { + throw new ModulePackageArchiveError("MODULE_PACKAGE_UNSAFE_MODE"); + } + + const name = field(block, 0, 100); + const prefix = field(block, 345, 155); + let fullPath = prefix === "" ? name : `${prefix}/${name}`; + if (type === "directory" && fullPath.endsWith("/")) { + fullPath = fullPath.slice(0, -1); + } + const segments = safeSegments(fullPath); + const top = segments[0] ?? ""; + if (topLevel === undefined) { + topLevel = top; + } else if (top !== topLevel) { + throw new ModulePackageArchiveError("MODULE_PACKAGE_LAYOUT_INVALID"); + } + + const size = type === "file" ? octalField(block, 124, 12) : 0; + const dataStart = offset + BLOCK; + const dataEnd = dataStart + size; + if (dataEnd > raw.length) { + throw new ModulePackageArchiveError("MODULE_PACKAGE_UNREADABLE"); + } + const relative = segments.slice(1).join("/"); + + if (relative === "") { + if (type !== "directory") { + throw new ModulePackageArchiveError("MODULE_PACKAGE_LAYOUT_INVALID"); + } + } else { + if (kinds.has(relative)) { + throw new ModulePackageArchiveError("MODULE_PACKAGE_LAYOUT_INVALID"); + } + // No entry may sit below a file, and no file may claim a path something already sits below. + // Linear in the total number of path segments, whatever order the entries arrive in. + if (type === "file" && ancestors.has(relative)) { + throw new ModulePackageArchiveError("MODULE_PACKAGE_LAYOUT_INVALID"); + } + for (let index = 1; index < segments.length - 1; index += 1) { + const ancestor = segments.slice(1, index + 1).join("/"); + if (kinds.get(ancestor) === "file") { + throw new ModulePackageArchiveError("MODULE_PACKAGE_LAYOUT_INVALID"); + } + ancestors.add(ancestor); + } + kinds.set(relative, type); + entries.push({ data: Buffer.from(raw.subarray(dataStart, dataEnd)), path: relative, type }); + } + offset = dataStart + Math.ceil(size / BLOCK) * BLOCK; + } + + if (!ended || topLevel === undefined) { + throw new ModulePackageArchiveError("MODULE_PACKAGE_UNREADABLE"); + } + return Object.freeze({ entries: Object.freeze(entries), topLevel }); +} diff --git a/core/src/modules/module-package-probe.ts b/core/src/modules/module-package-probe.ts new file mode 100644 index 0000000..66e68a3 --- /dev/null +++ b/core/src/modules/module-package-probe.ts @@ -0,0 +1,89 @@ +// The isolated, short-lived validation process of Module Package v1 verification (ADR 0049). +// +// The verifier forks this file with an empty environment and one JSON argument naming a candidate +// module's backend entrypoints. It imports each one, so a package whose code cannot load, or whose +// bundle still reaches for a dependency it does not contain, is refused before it is ever installed. +// Whatever the module's top-level code does happens here, in a process with no database URL, no +// secret, and no platform state, and that is discarded as soon as it reports. +// +// The result travels over the IPC channel, never stdout, so nothing a module prints can forge it. +// It carries fixed role and reason names only: never an error message, a path, or module output. +// +// Written as erasable TypeScript with no relative imports, so Node can run the source file directly +// in tests as well as the compiled file in the image. +import { pathToFileURL } from "node:url"; +import { parseModuleRuntimeDefinition, parseModuleSchemaDefinition } from "@ekklesia/contracts"; + +interface ProbeRequest { + readonly moduleId: string; + readonly version: string; + readonly entrypoints: Readonly>; +} + +interface ProbeFailure { + readonly role: string; + readonly reason: "export" | "identity" | "load"; +} + +function defaultExport(namespace: unknown): unknown { + return typeof namespace === "object" && namespace !== null && "default" in namespace + ? (namespace as { default: unknown }).default + : undefined; +} + +async function probe(request: ProbeRequest): Promise { + const failures: ProbeFailure[] = []; + for (const role of Object.keys(request.entrypoints).sort()) { + const entrypoint = request.entrypoints[role]; + if (entrypoint === undefined) { + continue; + } + let namespace: unknown; + try { + namespace = await import(pathToFileURL(entrypoint).href); + } catch { + failures.push({ reason: "load", role }); + continue; + } + try { + if (role === "server") { + const definition = parseModuleRuntimeDefinition(defaultExport(namespace)); + if (definition.id !== request.moduleId || definition.version !== request.version) { + failures.push({ reason: "identity", role }); + } + } else if (role === "schema") { + const definition = parseModuleSchemaDefinition(defaultExport(namespace)); + if (definition.moduleId !== request.moduleId) { + failures.push({ reason: "identity", role }); + } + } + } catch { + failures.push({ reason: "export", role }); + } + } + return failures; +} + +function report(message: unknown): void { + if (typeof process.send !== "function") { + process.exitCode = 2; + return; + } + process.send(message, () => { + // A module may leave timers or handles behind; the probe's work is done either way. + process.exit(0); + }); +} + +let request: ProbeRequest | undefined; +try { + request = JSON.parse(process.argv[2] ?? "") as ProbeRequest; +} catch { + report({ failures: [{ reason: "load", role: "request" }] }); +} +if (request !== undefined) { + probe(request).then( + (failures) => report({ failures }), + () => report({ failures: [{ reason: "load", role: "probe" }] }), + ); +} diff --git a/core/src/modules/module-package-verification.ts b/core/src/modules/module-package-verification.ts new file mode 100644 index 0000000..c1570fa --- /dev/null +++ b/core/src/modules/module-package-verification.ts @@ -0,0 +1,479 @@ +import { fork } from "node:child_process"; +import { createHash } from "node:crypto"; +import { chmod, mkdir, mkdtemp, readFile, rm, writeFile } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import path from "node:path"; +import { fileURLToPath } from "node:url"; +import { + isVersionCompatible, + MODULE_PACKAGE_ANY_TARGET, + MODULE_PACKAGE_FORMAT, + MODULE_PACKAGE_METADATA_FILENAME, + type ModuleManifest, + type ModulePackageMetadata, + safeParseModuleManifest, + safeParseModulePackageMetadata, + safeParseModulePackageRuntimePackage, +} from "@ekklesia/contracts"; +import { discoverModules, MODULE_MANIFEST_FILENAME, ModuleDiscoveryError } from "./discovery.js"; +import { discoverModuleMigrations, ModuleMigrationDiscoveryError } from "./migration-discovery.js"; +import { + type ModulePackageArchive, + ModulePackageArchiveError, + type ModulePackageArchiveErrorCode, + readModulePackageArchive, +} from "./module-package-archive.js"; +import { discoverModuleSchemas, ModuleSchemaDiscoveryError } from "./schema-discovery.js"; + +/** + * Verifies an Ekklesia Module Package v1 archive (ADR 0049) against the platform it runs on, without + * ever installing it. + * + * Order matters: everything that can be decided from bytes is decided before anything is written, + * and nothing is executed until the package has proven itself well-formed. The archive is read and + * checked for safety in memory; its metadata, every payload digest, and every identity statement + * are checked next; only then is it written to a fresh temporary directory, validated by the same + * discovery the platform uses, and its backend loaded once in a separate, short-lived process with + * an empty environment. A caller that asks for extraction gets the verified tree written to a new + * directory only after all of that succeeded. + * + * Errors carry a stable code and a fixed message. Paths, archive entry names, file contents, module + * output, and exception messages never appear in them: an archive is untrusted input, and none of + * it is ever echoed. + */ + +export type ModulePackageVerificationErrorCode = + | ModulePackageArchiveErrorCode + | "MODULE_PACKAGE_DATABASE_RUNTIME_INCOMPATIBLE" + | "MODULE_PACKAGE_DIGEST_MISMATCH" + | "MODULE_PACKAGE_ENTRYPOINT_LOAD_FAILED" + | "MODULE_PACKAGE_EXTRACTION_FAILED" + | "MODULE_PACKAGE_FILE_MISMATCH" + | "MODULE_PACKAGE_FILE_MISSING" + | "MODULE_PACKAGE_FILE_UNLISTED" + | "MODULE_PACKAGE_FORMAT_UNSUPPORTED" + | "MODULE_PACKAGE_IDENTITY_MISMATCH" + | "MODULE_PACKAGE_MANIFEST_INVALID" + | "MODULE_PACKAGE_METADATA_INVALID" + | "MODULE_PACKAGE_METADATA_MISSING" + | "MODULE_PACKAGE_MODULE_INVALID" + | "MODULE_PACKAGE_NATIVE_FILE" + | "MODULE_PACKAGE_RUNTIME_INCOMPATIBLE" + | "MODULE_PACKAGE_RUNTIME_PACKAGE_INVALID" + | "MODULE_PACKAGE_SDK_INCOMPATIBLE" + | "MODULE_PACKAGE_TARGET_INCOMPATIBLE" + | "MODULE_PACKAGE_TRANSITIONAL_UNSUPPORTED"; + +const messages: Readonly> = Object.freeze({ + MODULE_PACKAGE_DATABASE_RUNTIME_INCOMPATIBLE: + "The package's bundled database runtime does not match this platform's (a transitional constraint).", + MODULE_PACKAGE_DIGEST_MISMATCH: "The archive does not match the SHA-256 it was given.", + MODULE_PACKAGE_ENTRYPOINT_LOAD_FAILED: + "A backend entrypoint could not be loaded from the package alone.", + MODULE_PACKAGE_EXTRACTION_FAILED: + "The verified package could not be extracted to a new directory.", + MODULE_PACKAGE_FILE_MISMATCH: "A payload file does not match its listed size or SHA-256.", + MODULE_PACKAGE_FILE_MISSING: "A file listed in the package metadata is missing from the archive.", + MODULE_PACKAGE_FILE_UNLISTED: "The archive contains a file the package metadata does not list.", + MODULE_PACKAGE_FORMAT_UNSUPPORTED: "The package format is not one this platform implements.", + MODULE_PACKAGE_IDENTITY_MISMATCH: + "The package metadata, runtime package.json, and manifest disagree about the module's identity.", + MODULE_PACKAGE_LAYOUT_INVALID: + "The archive must contain exactly one top-level module directory, with each path once.", + MODULE_PACKAGE_LINK: "The archive contains a link; module packages may contain none.", + MODULE_PACKAGE_MANIFEST_INVALID: "The package's module manifest is missing or invalid.", + MODULE_PACKAGE_METADATA_INVALID: "The package metadata is invalid.", + MODULE_PACKAGE_METADATA_MISSING: "The archive contains no package metadata.", + MODULE_PACKAGE_MODULE_INVALID: "The packaged module failed module validation.", + MODULE_PACKAGE_NATIVE_FILE: "An any-target package contains a native file.", + MODULE_PACKAGE_RUNTIME_INCOMPATIBLE: + "The package needs a Node.js version this host does not run.", + MODULE_PACKAGE_RUNTIME_PACKAGE_INVALID: + "The package's runtime package.json is not the generated runtime shape.", + MODULE_PACKAGE_SDK_INCOMPATIBLE: + "The Module SDK bundled into the package is outside the manifest's own moduleSdk range.", + MODULE_PACKAGE_SPECIAL_ENTRY: + "The archive contains an entry that is neither a directory nor a regular file.", + MODULE_PACKAGE_TARGET_INCOMPATIBLE: "The package's target is not one this platform can run.", + MODULE_PACKAGE_TOO_LARGE: "The archive exceeds the module package size limits.", + MODULE_PACKAGE_TRANSITIONAL_UNSUPPORTED: + "The package metadata contains transitional facts this platform does not understand.", + MODULE_PACKAGE_UNREADABLE: "The file is not a readable gzip-compressed ustar archive.", + MODULE_PACKAGE_UNSAFE_MODE: "The archive contains a setuid, setgid, or sticky entry.", + MODULE_PACKAGE_UNSAFE_PATH: "The archive contains an absolute, escaping, or non-portable path.", +}); + +export class ModulePackageVerificationError extends Error { + readonly code: ModulePackageVerificationErrorCode; + /** Stable discovery issue codes, when module validation failed; never their messages. */ + readonly issueCodes: readonly string[]; + + constructor(code: ModulePackageVerificationErrorCode, issueCodes: readonly string[] = []) { + super(messages[code]); + this.name = "ModulePackageVerificationError"; + this.code = code; + this.issueCodes = Object.freeze([...new Set(issueCodes)].sort()); + this.stack = `${this.name}: ${this.message}`; + Object.freeze(this); + } +} + +/** What this platform is, for the checks that compare a package against it. */ +export interface ModulePackagePlatform { + readonly platformVersion: string; + readonly moduleSdkVersion: string; + /** The transitional database runtime facts this platform requires a package to agree with. */ + readonly databaseRuntime: Readonly>; + /** Defaults to the running process's Node.js version. */ + readonly nodeVersion?: string; +} + +export interface VerifyModulePackageOptions { + readonly archivePath: string; + /** When given, the archive's SHA-256 must equal it (lowercase hexadecimal). */ + readonly expectedSha256?: string; + readonly platform: ModulePackagePlatform; + /** A directory that must not exist yet; the verified module directory is written there. */ + readonly extractTo?: string; + readonly loadTimeoutMs?: number; +} + +export interface ModulePackageVerificationResult { + readonly archiveSha256: string; + /** Payload files, the metadata file not included. */ + readonly fileCount: number; + readonly moduleId: string; + readonly target: string; + readonly version: string; +} + +function fail(code: ModulePackageVerificationErrorCode, issueCodes?: readonly string[]): never { + throw new ModulePackageVerificationError(code, issueCodes); +} + +function sha256(data: Uint8Array): string { + return createHash("sha256").update(data).digest("hex"); +} + +function isRecord(value: unknown): value is Record { + return typeof value === "object" && value !== null && !Array.isArray(value); +} + +function parseJson(data: Buffer | undefined): unknown { + if (data === undefined) { + return undefined; + } + try { + return JSON.parse(data.toString("utf8")) as unknown; + } catch { + return undefined; + } +} + +/** The transitional facts this verifier understands, and nothing else (ADR 0049). */ +const UNDERSTOOD_TRANSITIONAL: Readonly> = Object.freeze({ + databaseRuntime: Object.freeze(["drizzle-orm"]), +}); + +function checkTransitional(raw: Record): void { + const transitional = raw.transitional; + if (transitional === undefined) { + return; + } + if (!isRecord(transitional)) { + fail("MODULE_PACKAGE_TRANSITIONAL_UNSUPPORTED"); + } + for (const [key, value] of Object.entries(transitional)) { + const understood = UNDERSTOOD_TRANSITIONAL[key]; + if (!understood || !isRecord(value)) { + fail("MODULE_PACKAGE_TRANSITIONAL_UNSUPPORTED"); + } + if (Object.keys(value).some((name) => !understood.includes(name))) { + fail("MODULE_PACKAGE_TRANSITIONAL_UNSUPPORTED"); + } + } +} + +function readMetadata(files: ReadonlyMap): ModulePackageMetadata { + const raw = parseJson(files.get(MODULE_PACKAGE_METADATA_FILENAME)); + if (!files.has(MODULE_PACKAGE_METADATA_FILENAME)) { + fail("MODULE_PACKAGE_METADATA_MISSING"); + } + if (!isRecord(raw)) { + fail("MODULE_PACKAGE_METADATA_INVALID"); + } + if (raw.packageFormat !== MODULE_PACKAGE_FORMAT) { + fail("MODULE_PACKAGE_FORMAT_UNSUPPORTED"); + } + checkTransitional(raw); + const parsed = safeParseModulePackageMetadata(raw); + if (!parsed.success) { + fail("MODULE_PACKAGE_METADATA_INVALID"); + } + return parsed.data; +} + +function checkPayload(metadata: ModulePackageMetadata, files: ReadonlyMap): void { + const listed = new Set(metadata.files.map((file) => file.path)); + for (const filePath of files.keys()) { + if (filePath !== MODULE_PACKAGE_METADATA_FILENAME && !listed.has(filePath)) { + fail("MODULE_PACKAGE_FILE_UNLISTED"); + } + } + for (const file of metadata.files) { + const data = files.get(file.path); + if (data === undefined) { + fail("MODULE_PACKAGE_FILE_MISSING"); + } + if (data.length !== file.size || sha256(data) !== file.sha256) { + fail("MODULE_PACKAGE_FILE_MISMATCH"); + } + } +} + +function checkIdentity( + archive: ModulePackageArchive, + metadata: ModulePackageMetadata, + files: ReadonlyMap, +): ModuleManifest { + const runtimePackage = safeParseModulePackageRuntimePackage(parseJson(files.get("package.json"))); + if (!runtimePackage.success) { + fail("MODULE_PACKAGE_RUNTIME_PACKAGE_INVALID"); + } + const manifest = safeParseModuleManifest(parseJson(files.get(MODULE_MANIFEST_FILENAME))); + if (!manifest.success) { + fail("MODULE_PACKAGE_MANIFEST_INVALID"); + } + if ( + archive.topLevel !== metadata.module.id || + manifest.data.id !== metadata.module.id || + manifest.data.version !== metadata.module.version || + runtimePackage.data.version !== metadata.module.version + ) { + fail("MODULE_PACKAGE_IDENTITY_MISMATCH"); + } + return manifest.data; +} + +function checkCompatibility( + metadata: ModulePackageMetadata, + manifest: ModuleManifest, + files: ReadonlyMap, + platform: ModulePackagePlatform, +): void { + // This verifier implements the portable target only. Native targets are a later slice. + if (metadata.target !== MODULE_PACKAGE_ANY_TARGET) { + fail("MODULE_PACKAGE_TARGET_INCOMPATIBLE"); + } + if ([...files.keys()].some((filePath) => filePath.toLowerCase().endsWith(".node"))) { + fail("MODULE_PACKAGE_NATIVE_FILE"); + } + if (!isVersionCompatible(platform.nodeVersion ?? process.versions.node, metadata.runtime.node)) { + fail("MODULE_PACKAGE_RUNTIME_INCOMPATIBLE"); + } + const sdk = metadata.incorporates["@ekklesia/module-sdk"]; + if (sdk !== undefined && !isVersionCompatible(sdk, manifest.compatibility.moduleSdk)) { + fail("MODULE_PACKAGE_SDK_INCOMPATIBLE"); + } + // Transitional (ADR 0049): today a module's Drizzle copy must be interchangeable with the + // platform's, because Core queries the module's tables with its own Drizzle instance. This is an + // implementation constraint pending a Module Database ABI, not that ABI. + for (const [runtime, version] of Object.entries(metadata.transitional?.databaseRuntime ?? {})) { + if (platform.databaseRuntime[runtime] !== version) { + fail("MODULE_PACKAGE_DATABASE_RUNTIME_INCOMPATIBLE"); + } + } +} + +/** Writes the archive's directories (`0755`) and files (`0644`) under `root`, which must not exist. + * Safe by construction: every path was validated as relative and non-escaping, and only directories + * and regular files exist. Ownership is the writing process's, never the archive's. */ +async function writeTree(root: string, archive: ModulePackageArchive): Promise { + await mkdir(root, { mode: 0o755 }); + await chmod(root, 0o755); + for (const entry of archive.entries) { + const target = path.join(root, ...entry.path.split("/")); + if (entry.type === "directory") { + await mkdir(target, { mode: 0o755, recursive: true }); + await chmod(target, 0o755); + } else { + await mkdir(path.dirname(target), { mode: 0o755, recursive: true }); + await writeFile(target, entry.data, { flag: "wx", mode: 0o644 }); + await chmod(target, 0o644); + } + } +} + +function probeEntrypointPath(): string { + // The probe sits beside this file: compiled `.js` in an installation, source `.ts` under a test + // runner, which Node runs directly. Either way it is a separate process. + const self = fileURLToPath(import.meta.url); + return path.join(path.dirname(self), `module-package-probe${path.extname(self)}`); +} + +/** Loads every backend entrypoint in a short-lived child with an empty environment, and reports + * only whether each loaded and exported a definition naming this module. */ +async function loadEntrypoints( + moduleId: string, + version: string, + entrypoints: Readonly>, + timeoutMs: number, +): Promise { + const outcome = await new Promise<"ok" | "failed">((resolve) => { + let settled = false; + let child: ReturnType | undefined; + let timer: ReturnType | undefined; + const finish = (value: "ok" | "failed"): void => { + if (!settled) { + settled = true; + clearTimeout(timer); + child?.kill("SIGKILL"); + resolve(value); + } + }; + try { + child = fork(probeEntrypointPath(), [JSON.stringify({ entrypoints, moduleId, version })], { + cwd: tmpdir(), + env: {}, + execArgv: [], + serialization: "json", + stdio: ["ignore", "ignore", "ignore", "ipc"], + }); + } catch { + finish("failed"); + return; + } + timer = setTimeout(() => finish("failed"), timeoutMs); + child.on("message", (message: unknown) => { + const failures = isRecord(message) ? message.failures : undefined; + finish(Array.isArray(failures) && failures.length === 0 ? "ok" : "failed"); + }); + child.on("error", () => finish("failed")); + child.on("exit", () => finish("failed")); + }); + if (outcome !== "ok") { + fail("MODULE_PACKAGE_ENTRYPOINT_LOAD_FAILED"); + } +} + +function discoveryIssueCodes(error: unknown): readonly string[] { + if ( + error instanceof ModuleDiscoveryError || + error instanceof ModuleMigrationDiscoveryError || + error instanceof ModuleSchemaDiscoveryError + ) { + return error.issues.map((issue) => issue.code); + } + return []; +} + +export async function verifyModulePackage( + options: VerifyModulePackageOptions, +): Promise { + let compressed: Buffer; + try { + compressed = await readFile(options.archivePath); + } catch { + fail("MODULE_PACKAGE_UNREADABLE"); + } + const archiveSha256 = sha256(compressed); + if (options.expectedSha256 !== undefined && options.expectedSha256 !== archiveSha256) { + fail("MODULE_PACKAGE_DIGEST_MISMATCH"); + } + + let archive: ModulePackageArchive; + try { + archive = readModulePackageArchive(compressed); + } catch (error) { + if (error instanceof ModulePackageArchiveError) { + fail(error.code); + } + fail("MODULE_PACKAGE_UNREADABLE"); + } + const files = new Map( + archive.entries + .filter((entry) => entry.type === "file") + .map((entry) => [entry.path, entry.data] as const), + ); + + const metadata = readMetadata(files); + checkPayload(metadata, files); + const manifest = checkIdentity(archive, metadata, files); + checkCompatibility(metadata, manifest, files, options.platform); + + const scratch = await mkdtemp(path.join(tmpdir(), "ekklesia-package-verify-")); + try { + // The candidate alone, in a scratch inventory of its own: never module storage. + const modulesDirectory = path.join(scratch, "candidates"); + await mkdir(modulesDirectory, { mode: 0o755 }); + await writeTree(path.join(modulesDirectory, metadata.module.id), archive); + + const discoveryOptions = { + moduleSdkVersion: options.platform.moduleSdkVersion, + modulesDirectory, + modulesDirectoryRequired: true, + platformVersion: options.platform.platformVersion, + } as const; + let entrypoints: Record; + try { + // The same static validation the platform runs, against this package alone. A module that + // requires another module is validated with its dependencies when it is installed. + const [discovered] = await discoverModules(discoveryOptions); + if (discovered === undefined) { + fail("MODULE_PACKAGE_MODULE_INVALID"); + } + await discoverModuleMigrations(discoveryOptions); + entrypoints = { + server: discovered.entrypoints.server, + ...(discovered.entrypoints.schema ? { schema: discovered.entrypoints.schema } : {}), + ...(discovered.entrypoints.jobs ? { jobs: discovered.entrypoints.jobs } : {}), + }; + } catch (error) { + if (error instanceof ModulePackageVerificationError) { + throw error; + } + fail("MODULE_PACKAGE_MODULE_INVALID", discoveryIssueCodes(error)); + } + + // Executed only now, and first in isolation: a schema is imported in this process below only + // after the probe proved it loads on its own. + await loadEntrypoints( + metadata.module.id, + metadata.module.version, + entrypoints, + options.loadTimeoutMs ?? 30_000, + ); + try { + await discoverModuleSchemas(discoveryOptions); + } catch (error) { + fail("MODULE_PACKAGE_MODULE_INVALID", discoveryIssueCodes(error)); + } + + if (options.extractTo !== undefined) { + const extractTo = options.extractTo; + let created = false; + try { + await mkdir(extractTo, { mode: 0o755 }); + created = true; + await rm(extractTo, { recursive: true }); + await writeTree(extractTo, archive); + } catch { + // Never leave a partial module directory behind; never remove one this call did not create. + if (created) { + await rm(extractTo, { force: true, recursive: true }).catch(() => undefined); + } + fail("MODULE_PACKAGE_EXTRACTION_FAILED"); + } + } + } finally { + await rm(scratch, { force: true, recursive: true }); + } + + return Object.freeze({ + archiveSha256, + fileCount: metadata.files.length, + moduleId: metadata.module.id, + target: metadata.target, + version: metadata.module.version, + }); +} diff --git a/core/tests/module-package-verification.test.ts b/core/tests/module-package-verification.test.ts new file mode 100644 index 0000000..153cf88 --- /dev/null +++ b/core/tests/module-package-verification.test.ts @@ -0,0 +1,387 @@ +import { mkdtemp, readdir, rm, stat, writeFile } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import path from "node:path"; +import { afterEach, describe, expect, it } from "vitest"; +import { + type ModulePackageVerificationErrorCode, + ModulePackageVerificationError, + verifyModulePackage, +} from "../src/modules/index.js"; +import { + type FixtureEntry, + modulePackageFixture, + type PackageFixtureOptions, + sha256, +} from "./support/module-package-fixture.js"; + +const platform = { + databaseRuntime: { "drizzle-orm": "0.45.2" }, + moduleSdkVersion: "0.1.2", + platformVersion: "0.1.2", +} as const; + +const temporaryDirectories: string[] = []; + +afterEach(async () => { + await Promise.all( + temporaryDirectories + .splice(0) + .map((directory) => rm(directory, { force: true, recursive: true })), + ); +}); + +async function scratch(): Promise { + const directory = await mkdtemp(path.join(tmpdir(), "ekklesia-verify-")); + temporaryDirectories.push(directory); + return directory; +} + +async function archiveFile(options: PackageFixtureOptions = {}): Promise<{ + readonly archivePath: string; + readonly sha256: string; +}> { + const fixture = await modulePackageFixture(options); + const archivePath = path.join(await scratch(), "private-host-name-5e1d.tar.gz"); + await writeFile(archivePath, fixture.archive); + return { archivePath, sha256: fixture.sha256 }; +} + +async function rejection( + options: PackageFixtureOptions, + code: ModulePackageVerificationErrorCode, + extra: { readonly expectedSha256?: string } = {}, +): Promise { + const { archivePath } = await archiveFile(options); + try { + await verifyModulePackage({ archivePath, platform, ...extra }); + } catch (error) { + expect(error).toBeInstanceOf(ModulePackageVerificationError); + const verificationError = error as ModulePackageVerificationError; + expect(verificationError.code).toBe(code); + // Sanitized: never the archive's location, a temporary directory, or text from inside it. + const rendered = JSON.stringify({ ...verificationError, message: verificationError.message }); + expect(rendered).not.toContain("private-host-name-5e1d"); + expect(rendered).not.toContain(tmpdir()); + expect(rendered).not.toContain("attacker-chosen"); + return verificationError; + } + throw new Error(`Expected ${code}.`); +} + +function withEntry(entry: FixtureEntry): PackageFixtureOptions { + return { extraEntries: [entry] }; +} + +describe("Module Package v1 verification (ADR 0049)", () => { + it("verifies a valid any-target package, its digest, and its module", async () => { + const { archivePath, sha256: digest } = await archiveFile(); + const result = await verifyModulePackage({ archivePath, expectedSha256: digest, platform }); + expect(result).toEqual({ + archiveSha256: digest, + fileCount: 8, + moduleId: "example", + target: "any", + version: "0.1.0", + }); + }); + + it("extracts only a verified package, with normalized modes, into a new directory", async () => { + const { archivePath } = await archiveFile(); + const target = path.join(await scratch(), "example"); + await verifyModulePackage({ archivePath, extractTo: target, platform }); + const names = (await readdir(target, { recursive: true })).sort(); + expect(names).toContain("ekklesia.package.json"); + expect(names).toContain("dist/server/index.js"); + expect((await stat(target)).mode & 0o7777).toBe(0o755); + expect((await stat(path.join(target, "dist", "server", "index.js"))).mode & 0o7777).toBe(0o644); + + const { archivePath: broken } = await archiveFile({ runtimePackage: { name: "x" } }); + const refused = path.join(await scratch(), "refused"); + await expect( + verifyModulePackage({ archivePath: broken, extractTo: refused, platform }), + ).rejects.toBeInstanceOf(ModulePackageVerificationError); + await expect(stat(refused)).rejects.toMatchObject({ code: "ENOENT" }); + }); + + it("refuses to extract over an existing directory", async () => { + const { archivePath } = await archiveFile(); + const existing = await scratch(); + await expect( + verifyModulePackage({ archivePath, extractTo: existing, platform }), + ).rejects.toMatchObject({ code: "MODULE_PACKAGE_EXTRACTION_FAILED" }); + }); + + it("fails a wrong archive digest", async () => { + await rejection({}, "MODULE_PACKAGE_DIGEST_MISMATCH", { expectedSha256: sha256("other") }); + }); + + it("fails a payload whose contents no longer match their digest", async () => { + await rejection( + { + tamper: (files) => { + files.set("dist/server/index.js", Buffer.from("export default {}; // attacker-chosen\n")); + }, + }, + "MODULE_PACKAGE_FILE_MISMATCH", + ); + }); + + it("fails a listed file missing from the archive", async () => { + await rejection( + { tamper: (files) => void files.delete("THIRD_PARTY_NOTICES") }, + "MODULE_PACKAGE_FILE_MISSING", + ); + }); + + it("fails an archive file that is not listed", async () => { + await rejection( + withEntry({ path: "example/attacker-chosen.js", data: "extra" }), + "MODULE_PACKAGE_FILE_UNLISTED", + ); + }); + + it("fails when the manifest and the package disagree about the version", async () => { + await rejection( + { manifest: (manifest) => ({ ...manifest, version: "0.2.0" }) }, + "MODULE_PACKAGE_IDENTITY_MISMATCH", + ); + }); + + it("fails when the metadata names another module", async () => { + await rejection( + { metadata: (metadata) => ({ ...metadata, module: { id: "other", version: "0.1.0" } }) }, + "MODULE_PACKAGE_IDENTITY_MISMATCH", + ); + }); + + it("fails a runtime package.json that is not the generated shape", async () => { + await rejection( + { + runtimePackage: { + name: "@test/module-example", + version: "0.1.0", + private: true, + type: "module", + ekklesia: { layer: "module" }, + dependencies: { "@ekklesia/module-sdk": "workspace:*" }, + }, + }, + "MODULE_PACKAGE_RUNTIME_PACKAGE_INVALID", + ); + }); + + it("fails an unsupported package format", async () => { + await rejection( + { metadata: (metadata) => ({ ...metadata, packageFormat: 2 }) }, + "MODULE_PACKAGE_FORMAT_UNSUPPORTED", + ); + }); + + it("fails transitional metadata it does not understand", async () => { + await rejection( + { metadata: (metadata) => ({ ...metadata, transitional: { orm: { prisma: "6.0.0" } } }) }, + "MODULE_PACKAGE_TRANSITIONAL_UNSUPPORTED", + ); + await rejection( + { + metadata: (metadata) => ({ + ...metadata, + transitional: { databaseRuntime: { "drizzle-orm": "0.45.2", prisma: "6.0.0" } }, + }), + }, + "MODULE_PACKAGE_TRANSITIONAL_UNSUPPORTED", + ); + }); + + it("enforces today's database runtime agreement, and accepts it when it holds", async () => { + await rejection( + { + metadata: (metadata) => ({ + ...metadata, + transitional: { databaseRuntime: { "drizzle-orm": "0.44.0" } }, + }), + }, + "MODULE_PACKAGE_DATABASE_RUNTIME_INCOMPATIBLE", + ); + const { archivePath } = await archiveFile({ + metadata: (metadata) => ({ + ...metadata, + transitional: { databaseRuntime: { "drizzle-orm": "0.45.2" } }, + }), + }); + await expect(verifyModulePackage({ archivePath, platform })).resolves.toMatchObject({ + moduleId: "example", + }); + }); + + it("fails a target this platform cannot run", async () => { + await rejection( + { + metadata: (metadata) => ({ + ...metadata, + target: "linux-arm64-musl", + runtime: { node: ">=20", nodeApi: 9 }, + }), + }, + "MODULE_PACKAGE_TARGET_INCOMPATIBLE", + ); + }); + + it("fails a native file inside an any-target package", async () => { + const native = await modulePackageFixture({ + tamper: (files) => { + files.set("dist/addon.node", Buffer.from([0x7f, 0x45, 0x4c, 0x46])); + }, + metadata: (metadata) => { + const files = [ + ...(metadata.files as { path: string; size: number; sha256: string }[]), + { + path: "dist/addon.node", + size: 4, + sha256: sha256(Buffer.from([0x7f, 0x45, 0x4c, 0x46])), + }, + ].sort((left, right) => (left.path < right.path ? -1 : 1)); + return { ...metadata, files }; + }, + }); + const archivePath = path.join(await scratch(), "native.tar.gz"); + await writeFile(archivePath, native.archive); + await expect(verifyModulePackage({ archivePath, platform })).rejects.toMatchObject({ + code: "MODULE_PACKAGE_NATIVE_FILE", + }); + }); + + it("fails a package that needs a newer Node.js", async () => { + await rejection( + { metadata: (metadata) => ({ ...metadata, runtime: { node: ">=99" } }) }, + "MODULE_PACKAGE_RUNTIME_INCOMPATIBLE", + ); + }); + + it("fails an incorporated SDK outside the manifest's own range", async () => { + await rejection( + { + metadata: (metadata) => ({ + ...metadata, + incorporates: { "@ekklesia/module-sdk": "0.9.0" }, + }), + }, + "MODULE_PACKAGE_SDK_INCOMPATIBLE", + ); + }); + + it("fails a module whose declared entrypoint is missing", async () => { + await rejection( + { + manifest: (manifest) => ({ + ...manifest, + version: "0.1.0", + entrypoints: { + ...(manifest.entrypoints as Record), + schema: "./dist/schema/index.js", + }, + }), + }, + "MODULE_PACKAGE_MODULE_INVALID", + ); + }); + + it("fails a backend that imports a dependency the package does not contain", async () => { + const error = await rejection( + { + serverSource: + 'import pad from "ekklesia-dependency-not-in-this-package";\nexport default { id: "example", version: "0.1.0", register() { pad(); } };\n', + }, + "MODULE_PACKAGE_ENTRYPOINT_LOAD_FAILED", + ); + expect(error.message).not.toContain("ekklesia-dependency-not-in-this-package"); + }); + + it("fails a backend whose default export names another module", async () => { + await rejection( + { serverSource: 'export default { id: "other", version: "0.1.0" };\n' }, + "MODULE_PACKAGE_ENTRYPOINT_LOAD_FAILED", + ); + }); + + it.each([ + ["an absolute path", { path: "/etc/attacker-chosen", data: "x" }, "MODULE_PACKAGE_UNSAFE_PATH"], + [ + "a .. segment", + { path: "example/../attacker-chosen", data: "x" }, + "MODULE_PACKAGE_UNSAFE_PATH", + ], + [ + "a symbolic link", + { path: "example/link", type: "symlink", linkname: "/etc/passwd" }, + "MODULE_PACKAGE_LINK", + ], + [ + "a hard link", + { path: "example/hard", type: "hardlink", linkname: "example/package.json" }, + "MODULE_PACKAGE_LINK", + ], + [ + "a device node", + { path: "example/device", type: "character" }, + "MODULE_PACKAGE_SPECIAL_ENTRY", + ], + ["a FIFO", { path: "example/fifo", type: "fifo" }, "MODULE_PACKAGE_SPECIAL_ENTRY"], + [ + "a setuid file", + { path: "example/setuid", data: "x", mode: 0o4755 }, + "MODULE_PACKAGE_UNSAFE_MODE", + ], + [ + "a setgid directory", + { path: "example/setgid", type: "directory", mode: 0o2755 }, + "MODULE_PACKAGE_UNSAFE_MODE", + ], + [ + "a second top-level directory", + { path: "attacker-chosen/file", data: "x" }, + "MODULE_PACKAGE_LAYOUT_INVALID", + ], + [ + "a duplicate entry", + { path: "example/package.json", data: "{}" }, + "MODULE_PACKAGE_LAYOUT_INVALID", + ], + ] as const)("fails %s", async (_label, entry, code) => { + await rejection(withEntry(entry as FixtureEntry), code); + }); + + it("fails a file that is not a gzip-compressed tar archive", async () => { + const archivePath = path.join(await scratch(), "plain.tar.gz"); + await writeFile(archivePath, "not an archive"); + await expect(verifyModulePackage({ archivePath, platform })).rejects.toMatchObject({ + code: "MODULE_PACKAGE_UNREADABLE", + }); + }); + + it("fails an archive without package metadata", async () => { + const fixture = await modulePackageFixture(); + const { gunzipSync, gzipSync } = await import("node:zlib"); + // Rebuild without the metadata entry by filtering the fixture through a fresh archive. + const archivePath = path.join(await scratch(), "no-metadata.tar.gz"); + const raw = gunzipSync(fixture.archive); + const blocks: Buffer[] = []; + for (let offset = 0; offset + 512 <= raw.length; ) { + const headerBlock = raw.subarray(offset, offset + 512); + if (headerBlock.every((byte) => byte === 0)) { + break; + } + const name = headerBlock.subarray(0, 100).toString("latin1").replace(/\0.*$/su, ""); + const size = Number.parseInt(headerBlock.subarray(124, 136).toString("latin1"), 8); + const length = 512 + Math.ceil(size / 512) * 512; + if (!name.endsWith("ekklesia.package.json")) { + blocks.push(raw.subarray(offset, offset + length)); + } + offset += length; + } + await writeFile(archivePath, gzipSync(Buffer.concat([...blocks, Buffer.alloc(1024)]))); + await expect(verifyModulePackage({ archivePath, platform })).rejects.toMatchObject({ + code: "MODULE_PACKAGE_METADATA_MISSING", + }); + }); +}); diff --git a/core/tests/support/module-package-fixture.ts b/core/tests/support/module-package-fixture.ts new file mode 100644 index 0000000..bc051b5 --- /dev/null +++ b/core/tests/support/module-package-fixture.ts @@ -0,0 +1,205 @@ +import { createHash } from "node:crypto"; +import { mkdtemp, readdir, readFile, rm } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import path from "node:path"; +import { gzipSync } from "node:zlib"; +import { createMigrationModule } from "./migration-module-fixture.js"; + +/** + * Test-only builders for Ekklesia Module Package v1 archives (ADR 0049). They write ustar archives + * by hand so a test can produce exactly what the canonical packager never would: links, device + * nodes, traversal, a second top-level directory, a corrupted payload. + */ + +export type FixtureEntryType = "file" | "directory" | "symlink" | "hardlink" | "character" | "fifo"; + +export interface FixtureEntry { + readonly path: string; + readonly type?: FixtureEntryType; + readonly mode?: number; + readonly data?: Uint8Array | string; + readonly linkname?: string; +} + +const typeflags: Readonly> = { + character: "3", + directory: "5", + fifo: "6", + file: "0", + hardlink: "1", + symlink: "2", +}; + +function octal(value: number, width: number): string { + return `${value.toString(8).padStart(width - 1, "0")}\u0000`; +} + +function header(entry: FixtureEntry, size: number): Buffer { + const block = Buffer.alloc(512); + const name = + entry.type === "directory" && !entry.path.endsWith("/") ? `${entry.path}/` : entry.path; + let prefix = ""; + let base = name; + if (Buffer.byteLength(name) > 100) { + const split = name.lastIndexOf("/", name.length - 2); + prefix = name.slice(0, split); + base = name.slice(split + 1); + } + block.write(base, 0, 100, "latin1"); + block.write(octal(entry.mode ?? (entry.type === "directory" ? 0o755 : 0o644), 8), 100, "latin1"); + block.write(octal(0, 8), 108, "latin1"); + block.write(octal(0, 8), 116, "latin1"); + block.write(octal(size, 12), 124, "latin1"); + block.write(octal(0, 12), 136, "latin1"); + block.write(" ", 148, "latin1"); + block.write(typeflags[entry.type ?? "file"], 156, "latin1"); + block.write(entry.linkname ?? "", 157, 100, "latin1"); + block.write("ustar\u000000", 257, "latin1"); + block.write(prefix, 345, 155, "latin1"); + let checksum = 0; + for (const byte of block) { + checksum += byte; + } + block.write(`${checksum.toString(8).padStart(6, "0")}\u0000 `, 148, "latin1"); + return block; +} + +/** A gzip-compressed ustar archive of exactly these entries, in this order. */ +export function tarGz(entries: readonly FixtureEntry[]): Buffer { + const parts: Buffer[] = []; + for (const entry of entries) { + const data = + entry.type === undefined || entry.type === "file" + ? Buffer.from(entry.data ?? "") + : Buffer.alloc(0); + parts.push(header(entry, data.length), data, Buffer.alloc((512 - (data.length % 512)) % 512)); + } + parts.push(Buffer.alloc(1024)); + return gzipSync(Buffer.concat(parts), { level: 9 }); +} + +export function sha256(data: Uint8Array | string): string { + return createHash("sha256").update(data).digest("hex"); +} + +export interface PackageFixtureOptions { + readonly id?: string; + readonly version?: string; + readonly serverSource?: string; + readonly manifest?: (manifest: Record) => Record; + readonly runtimePackage?: Record; + readonly metadata?: (metadata: Record) => Record; + /** Changes the payload after its digests were computed. */ + readonly tamper?: (files: Map) => void; + /** Entries appended after the package's own, under any path. */ + readonly extraEntries?: readonly FixtureEntry[]; +} + +export interface PackageFixture { + readonly archive: Buffer; + readonly sha256: string; +} + +async function readTree(root: string): Promise> { + const files = new Map(); + for (const entry of await readdir(root, { recursive: true, withFileTypes: true })) { + if (entry.isFile()) { + const absolute = path.join(entry.parentPath, entry.name); + files.set(path.relative(root, absolute).split(path.sep).join("/"), await readFile(absolute)); + } + } + return files; +} + +/** + * A valid any-target package of a minimal module: a server stub with no imports, one migration, a + * notices file, the generated runtime package.json, and complete metadata. Every option changes one + * thing, so a test can break exactly the rule it is about. + */ +export async function modulePackageFixture( + options: PackageFixtureOptions = {}, +): Promise { + const id = options.id ?? "example"; + const version = options.version ?? "0.1.0"; + const scratch = await mkdtemp(path.join(tmpdir(), "ekklesia-package-fixture-")); + let files: Map; + try { + const created = await createMigrationModule(scratch, id, { + serverSource: + options.serverSource ?? + `export default { id: ${JSON.stringify(id)}, version: ${JSON.stringify(version)}, register() {} };\n`, + }); + files = await readTree(created.moduleDirectory); + } finally { + await rm(scratch, { force: true, recursive: true }); + } + + const manifest = JSON.parse( + files.get("ekklesia.module.json")?.toString("utf8") ?? "{}", + ) as Record; + files.set( + "ekklesia.module.json", + Buffer.from( + `${JSON.stringify(options.manifest ? options.manifest(manifest) : { ...manifest, version }, null, 2)}\n`, + ), + ); + files.set( + "package.json", + Buffer.from( + `${JSON.stringify( + options.runtimePackage ?? { + name: `@test/module-${id}`, + version, + private: true, + type: "module", + ekklesia: { layer: "module" }, + }, + null, + 2, + )}\n`, + ), + ); + files.set("THIRD_PARTY_NOTICES", Buffer.from("No third-party software is included.\n")); + + const listed = [...files.entries()] + .sort(([left], [right]) => (left < right ? -1 : left > right ? 1 : 0)) + .map(([filePath, data]) => ({ path: filePath, size: data.length, sha256: sha256(data) })); + const baseMetadata: Record = { + packageFormat: 1, + module: { id, version }, + target: "any", + runtime: { node: ">=20" }, + provenance: { + revision: "86ae907f59458508e3277666cdd214f5bd7c1078", + packager: { name: "@ekklesia/module-pack", version: "0.1.2" }, + }, + incorporates: { "@ekklesia/module-sdk": "0.1.2" }, + files: listed, + }; + const metadata = options.metadata ? options.metadata(baseMetadata) : baseMetadata; + options.tamper?.(files); + + const directories = new Set([id]); + for (const filePath of files.keys()) { + const segments = filePath.split("/"); + for (let index = 1; index < segments.length; index += 1) { + directories.add(`${id}/${segments.slice(0, index).join("/")}`); + } + } + const all = new Map(); + for (const directory of directories) { + all.set(directory, { path: directory, type: "directory" }); + } + for (const [filePath, data] of files) { + all.set(`${id}/${filePath}`, { path: `${id}/${filePath}`, data }); + } + all.set(`${id}/ekklesia.package.json`, { + path: `${id}/ekklesia.package.json`, + data: `${JSON.stringify(metadata, null, 2)}\n`, + }); + const ordered = [...all.values()].sort((left, right) => + left.path < right.path ? -1 : left.path > right.path ? 1 : 0, + ); + const archive = tarGz([...ordered, ...(options.extraEntries ?? [])]); + return { archive, sha256: sha256(archive) }; +} diff --git a/core/tests/verify-module-package-cli.test.ts b/core/tests/verify-module-package-cli.test.ts new file mode 100644 index 0000000..853bfd6 --- /dev/null +++ b/core/tests/verify-module-package-cli.test.ts @@ -0,0 +1,75 @@ +import { readFile } from "node:fs/promises"; +import path from "node:path"; +import { fileURLToPath } from "node:url"; +import { describe, expect, it } from "vitest"; +import { + parseSha256File, + parseVerifyModulePackageArguments, + readModulePackagePlatform, +} from "../src/cli/verify-module-package.js"; + +const repositoryRoot = path.resolve(path.dirname(fileURLToPath(import.meta.url)), "..", ".."); +const digest = "ab".repeat(32); + +describe("verify-module-package arguments", () => { + it("takes an archive and, optionally, a digest and an extraction directory", () => { + expect(parseVerifyModulePackageArguments(["example-0.1.0-any.tar.gz"])).toEqual({ + archivePath: "example-0.1.0-any.tar.gz", + }); + expect( + parseVerifyModulePackageArguments([ + "x.tar.gz", + "--sha256", + digest.toUpperCase(), + "--extract-to", + "/staging/example", + ]), + ).toEqual({ archivePath: "x.tar.gz", expectedSha256: digest, extractTo: "/staging/example" }); + expect( + parseVerifyModulePackageArguments(["x.tar.gz", "--sha256-file", "x.tar.gz.sha256"]), + ).toEqual({ archivePath: "x.tar.gz", sha256File: "x.tar.gz.sha256" }); + }); + + it.each([ + ["no archive", []], + ["two archives", ["a.tar.gz", "b.tar.gz"]], + ["an unknown option", ["a.tar.gz", "--force"]], + ["both digest options", ["a.tar.gz", "--sha256", digest, "--sha256-file", "a.sha256"]], + ["a malformed digest", ["a.tar.gz", "--sha256", "abc"]], + ["an option without its value", ["a.tar.gz", "--extract-to"]], + ])("refuses %s", (_label, argv) => { + expect(() => parseVerifyModulePackageArguments(argv)).toThrow(); + }); +}); + +describe("verify-module-package digest files", () => { + it("reads the sha256sum format, ignoring the recorded file name", () => { + expect(parseSha256File(`${digest} example-0.1.0-any.tar.gz\n`)).toBe(digest); + expect(parseSha256File(`${digest.toUpperCase()} *renamed.tar.gz\n`)).toBe(digest); + }); + + it.each([ + "", + "not-a-digest x.tar.gz\n", + `${digest}\n${digest} second\n`, + ])("refuses %j", (contents) => { + expect(() => parseSha256File(contents)).toThrow(); + }); +}); + +describe("verify-module-package platform facts", () => { + it("reads this platform's versions and its transitional database runtime from the repository", async () => { + const root = JSON.parse(await readFile(path.join(repositoryRoot, "package.json"), "utf8")); + const core = JSON.parse( + await readFile(path.join(repositoryRoot, "core", "package.json"), "utf8"), + ); + const sdk = JSON.parse( + await readFile(path.join(repositoryRoot, "packages", "module-sdk", "package.json"), "utf8"), + ); + await expect(readModulePackagePlatform(repositoryRoot)).resolves.toEqual({ + databaseRuntime: { "drizzle-orm": core.dependencies["drizzle-orm"] }, + moduleSdkVersion: sdk.version, + platformVersion: root.version, + }); + }); +}); diff --git a/deploy/README.md b/deploy/README.md index 1ed0d00..14c739d 100644 --- a/deploy/README.md +++ b/deploy/README.md @@ -142,12 +142,16 @@ Installed modules live on the host, in module storage: `/var/lib/ekklesia/module and `migrate`, which is where `EKKLESIA_MODULES_DIR` points. Ekklesia never writes there. Installing, updating, or removing a module never needs a new image, only a restart. -**What belongs there.** One directory per module, each a self-contained installed module: its -`package.json`, `ekklesia.module.json`, built artifacts, migrations, and the production -dependencies its server code imports, in its own `node_modules` +**What belongs there.** One directory per module, each extracted from that module's **Ekklesia +Module Package** +([ADR 0049](../docs/architecture/decisions/0049-module-package-v1.md)): one archive, +`--.tar.gz`, and the SHA-256 published beside it. A package holds the module's +generated `package.json`, its manifest, `ekklesia.package.json`, bundled backend code, built browser +artifacts, migrations, and `THIRD_PARTY_NOTICES` ([Module development](../docs/module-development/README.md#an-installed-module-is-self-contained)). Copying a module's `dist/` alone is not enough. Files are owned by root and readable by everyone (directories 755, files 644), so the API's unprivileged user can read them and never change them. +The digest proves the archive is the one you were given; it does not prove who published it. **Nothing else belongs there.** Every non-hidden directory must be a valid module, or the API does not start. That includes `lost+found`, so if module storage is a file system of its own, mount it @@ -159,12 +163,28 @@ never loads or unloads a module while it runs. Change module storage only while it reads browser artifacts from disk as it serves them, so files changed under a running API would serve one version's browser code beside another's server code. -To **install or update** a module, with `` its module id: +To **install or update** a module, with `` its module id and the package and its `.sha256` in +the current directory, first **verify** it with the image's own verifier. It checks the archive's +digest, its safety (no links, special files, or escaping paths), every file's digest, the package's +identity, target, and Node.js range against this platform, runs the platform's module and migration +validation, and loads the backend once in a short-lived process with no environment. Only then does +it write the verified module directory, to a new directory outside module storage: + +```sh +mkdir -m 777 staging # writable by the image's unprivileged user; nothing else is +docker run --rm --network none --read-only --tmpfs /tmp \ + -v "$PWD:/package:ro" -v "$PWD/staging:/staging" \ + --entrypoint node "$EKKLESIA_IMAGE" core/dist/cli/verify-module-package.js \ + /package/--any.tar.gz --sha256-file /package/--any.tar.gz.sha256 \ + --extract-to /staging/ +``` + +Then install the verified directory: ```sh docker compose stop api # Stage under a hidden name, which discovery ignores, then rename it into place in one step. -sudo cp -R ./ /var/lib/ekklesia/modules/.incoming- +sudo cp -R staging/ /var/lib/ekklesia/modules/.incoming- sudo chown -R root:root /var/lib/ekklesia/modules/.incoming- sudo chmod -R a+rX,go-w /var/lib/ekklesia/modules/.incoming- sudo mv /var/lib/ekklesia/modules/ /var/lib/ekklesia/modules/.previous- # update only @@ -188,7 +208,8 @@ is information, not a problem, and it blocks no other migration. To stop a modul it, disable it instead (`set-module-enabled.js --disable`, [Operator commands](#operator-commands)). There is no purge: deleting a module's data is a separate operation that does not exist yet. -To **restore** a removed module, install it again as above, at the same version or a newer one. It +To **restore** a removed module, verify and install its package again as above, at the same version +or a newer one. It finds its data, its settings, and its history where it left them. Its history is strict again: a package that no longer contains a migration that was applied, or contains one with different contents, is refused by the migration plan, so an older version cannot be installed over newer data. @@ -409,10 +430,9 @@ deployment: instead, on the **same generic image**, and also proves [Installed modules](#installed-modules) end to end: - the image boots with no module; - - the repository's module template, packaged as a self-contained test module - (`scripts/lib/test-module-package.Dockerfile`, test-only and not a package format), is installed - into module storage, its migration is planned and explicitly applied, the API restarts, and the - module reads data it owns; + - the repository's module template, packaged as an Ekklesia Module Package v1 by the canonical + packager, is verified by the image itself and installed into module storage, its migration is + planned and explicitly applied, the API restarts, and the module reads data it owns; - every hostname's served and refused paths, exact CORS for the Dashboard's origin and none for others; - a real headless Chromium (pinned by digest, driven over the DevTools protocol) signs in on the @@ -421,8 +441,8 @@ deployment: API's hostname with its service worker, and signs out; - the module is removed and the API restarted: Core still runs, the module's routes are gone, migration planning and applying succeed and report its retained history, and its data is intact; - - the module is restored and the API restarted: its history validates strictly and it returns its - previous data. + - the exact same package is verified and installed again and the API restarted: its history + validates strictly and it returns its previous data. CI runs both topologies on every change. - **Test-only:** the smoke test runs Caddy with its internal CA (`local_certs`), because a test run @@ -586,6 +606,7 @@ reports direct mode for an API that trusts its proxy. The running API's own valu | Doctor (read-only report) | `docker compose run --rm migrate node core/dist/cli/doctor.js` | | Create the initial owner (interactive) | `docker compose run --rm -it migrate node core/dist/cli/create-initial-owner.js --email owner@example.org` | | Validate the installed modules (read-only) | `docker compose run --rm migrate node core/dist/cli/validate-modules.js` | +| Verify a module package (read-only unless extracting) | `node core/dist/cli/verify-module-package.js --sha256-file .sha256 [--extract-to ]`, in a `docker run` of the image ([Installed modules](#installed-modules)) | | Enable or disable a module | `docker compose run --rm migrate node core/dist/cli/set-module-enabled.js --enable --module ` (or `--disable`) | Every command reads installed modules from the same `EKKLESIA_MODULES_DIR` the API does. The plan, diff --git a/docs/architecture/decisions/0049-module-package-v1.md b/docs/architecture/decisions/0049-module-package-v1.md index c15005c..2a8ecad 100644 --- a/docs/architecture/decisions/0049-module-package-v1.md +++ b/docs/architecture/decisions/0049-module-package-v1.md @@ -1,6 +1,6 @@ # 0049: A module release is one immutable, self-contained package: Ekklesia Module Package v1 -- Status: Proposed +- Status: Accepted - Date: 2026-10-07 - Supersedes: none - Extends: [0048](0048-generic-image-and-installed-module-storage.md) diff --git a/docs/architecture/decisions/README.md b/docs/architecture/decisions/README.md index b7ac19d..4fd6b13 100644 --- a/docs/architecture/decisions/README.md +++ b/docs/architecture/decisions/README.md @@ -57,4 +57,4 @@ ADRs capture durable choices that affect more than one package or phase. - [0046: Ekklesia's branding is never forced on an installation](0046-installation-owned-branding.md) — Proposed - [0047: Every module proves itself inside Ekklesia, with one harness the platform publishes](0047-module-ci-harness.md) — Proposed - [0048: Ekklesia ships one generic image, and installed modules live outside it](0048-generic-image-and-installed-module-storage.md) — Accepted -- [0049: A module release is one immutable, self-contained package: Ekklesia Module Package v1](0049-module-package-v1.md) — Proposed +- [0049: A module release is one immutable, self-contained package: Ekklesia Module Package v1](0049-module-package-v1.md) — Accepted diff --git a/docs/architecture/platform-architecture.md b/docs/architecture/platform-architecture.md index 9463192..06efa2c 100644 --- a/docs/architecture/platform-architecture.md +++ b/docs/architecture/platform-architecture.md @@ -672,3 +672,18 @@ artifact from what an installation installs: module that lost or changed an applied migration is still refused. - **Trust is unchanged.** Installed modules are still trusted code; module storage is mounted read-only, which keeps Ekklesia from modifying installed code but does not sandbox it. + +[ADR 0049](decisions/0049-module-package-v1.md) defines how a module is distributed: one Ekklesia +Module Package v1, a deterministic `.tar.gz` with its SHA-256. + +- **The packager.** `@ekklesia/module-pack` bundles the module's backend with every pure-JavaScript + dependency, the SDK, contracts, and Drizzle included. It copies the built browser artifacts and + migrations, generates the runtime `package.json`, `ekklesia.package.json` with every file's digest, + and `THIRD_PARTY_NOTICES`, and writes the archive. +- **The verifier.** Core's `verifyModulePackage` reads the archive in memory and refuses links, + special files, and escaping paths. It checks every digest and identity, the target and runtime, + and the transitional Drizzle agreement, runs the platform's own module and migration validation, + and loads the backend once in a short-lived process with an empty environment. Only then does it + extract anything. +- **What exists today.** Only the `any` target. Drizzle agreement is recorded as transitional, not a + database ABI. diff --git a/docs/development/security.md b/docs/development/security.md index 316fb97..2616bef 100644 --- a/docs/development/security.md +++ b/docs/development/security.md @@ -479,7 +479,12 @@ Installed modules come from installation-owned module storage, which the referen is an integrity property, not a sandbox: an installed module is still trusted code with the API's database role, exactly as above. The repository's boundary checks prove only source modules in a workspace, and an installed package usually ships none, so installing one means trusting that -module's own CI and whoever placed it in module storage. +module's own CI and whoever placed it in module storage. A Module Package v1 (ADR 0049) is verified +before it is installed: its digest, its archive safety (no links, special files, setuid bits, or +escaping paths), every file's digest, and its identity and compatibility. The digest proves +integrity, not who published the package; signing is not implemented. The verifier loads the backend +in a short-lived process with an empty environment, so module code never sees a database URL or a +secret while it is being checked. Two production defaults are relevant but are **not** by themselves a deployment guarantee: diff --git a/docs/development/testing.md b/docs/development/testing.md index c3e1189..d532911 100644 --- a/docs/development/testing.md +++ b/docs/development/testing.md @@ -564,6 +564,29 @@ the following pull-request status context to `main` branch protection: CI / migration-smoke (pull_request) ``` +## Module packages + +Ekklesia Module Package v1 (ADR 0049) is proven at three levels, all in `pnpm verify`: + +- `packages/contracts/tests/module-package.test.ts`: the `ekklesia.package.json` schema, target + names, and the generated runtime `package.json`. +- `core/tests/module-package-verification.test.ts`: the verifier against hand-built archives, + including what the packager never produces. A valid package verifies and extracts with normalized + modes. A wrong digest, a corrupted or missing or unlisted file, disagreeing identities, an unknown + format, unknown transitional metadata, an incompatible target, Node.js range, SDK, or database + runtime, a missing entrypoint, a backend that imports something the package lacks, absolute and + `..` paths, symbolic and hard links, device nodes and FIFOs, setuid and setgid entries, a second + top-level directory, and a duplicate entry each fail with their own sanitized code. The backend + load check runs in a real child process. +- `packages/module-pack/tests/`: the packager against generated fixture modules (payload contents, + generated `package.json`, metadata, bundled backend with only built-in imports, refusal of native + and platform-specific dependencies and of unknown or textless licences, notices, byte-identical + archives, normalized tar and gzip headers), and end to end against the built module template: + packed twice to identical bytes without changing the workspace's install state, verified by Core's + verifier, extracted elsewhere, and loaded with Node's own resolution hooks proving nothing outside + the package is resolved. Turbo builds the template and Core first, and runs the template's own + tests (which rebuild its UI in place) before these. + ## The generic platform image and the reference deployment These need Docker and build images locally; nothing is pushed or published. @@ -581,13 +604,13 @@ These need Docker and build images locally; nothing is pushed or published. - `pnpm deploy:smoke` (`scripts/smoke-deploy.sh`) runs the reference deployment end to end on the generic image with empty module storage. With `EKKLESIA_SMOKE_TOPOLOGY=split`, the same image also runs the installed-module lifecycle: - - the module template, packaged as a self-contained test module - (`scripts/lib/test-module-package.Dockerfile`, test-only), is installed into module storage; + - the module template is packaged by the canonical packager (Module Package v1, ADR 0049), + verified by the image's own verifier, and installed into module storage; - its migration is planned and explicitly applied, and the API restarted; - a real browser mounts it; - it is removed and the API restarted, and Core still runs, migrations still plan and apply, and its data survives; - - it is restored, with its data. + - the exact same package is verified and installed again, and the module returns with its data. See [deploy/README.md](../../deploy/README.md#testing-this-deployment). diff --git a/docs/module-development/README.md b/docs/module-development/README.md index c84fb03..49ba1f0 100644 --- a/docs/module-development/README.md +++ b/docs/module-development/README.md @@ -78,42 +78,79 @@ names (`/var/lib/ekklesia/modules` in the production image, the checkout's `modu unset). Discovery reads it only when a process starts. Nothing is hot loaded, and an installed module is read-only to the platform. -An installed module directory is **self-contained for runtime use**. Everything the platform reads -or loads from it, and everything those files import at run time, resolves inside it: - -- `package.json`, with no `workspace:` range needed at run time, and `ekklesia.module.json`; -- every built artifact the manifest declares: `dist/server`, `dist/schema`, jobs, and the UI, - display, and contribution bundles; -- the `drizzle/` migrations directory, with its journal and `_ekklesia.json`; -- a `node_modules` holding the production dependencies of your server, schema, and jobs code, - including your own copies of `@ekklesia/module-sdk`, `@ekklesia/contracts`, and `drizzle-orm`. +An installed module directory is **self-contained for runtime use**: everything the platform reads +or loads from it, and everything those files import at run time, resolves inside it. You never +assemble that directory by hand. You build your module, and the canonical packager turns it into an +**Ekklesia Module Package v1** +([ADR 0049](../architecture/decisions/0049-module-package-v1.md)): one deterministic archive and its +SHA-256, which an installation verifies and extracts into module storage. **Copying `dist/` is not an installed module.** Your compiled server code imports `@ekklesia/module-sdk` and `drizzle-orm` by name, and Node resolves those only from your module's own directory upwards. The platform's `node_modules` is never on that path, and Ekklesia adds it -through no link, `NODE_PATH`, loader hook, or alias. A module whose imports do not resolve stops the -API from starting with a sanitized `MODULE_RUNTIME_IMPORT_FAILED`, so test the directory you ship, -not your workspace. +through no link, `NODE_PATH`, loader hook, or alias. The packager therefore bundles your backend. -**Native dependencies** must match the platform image's runtime: Node.js 24.18.0 on Debian 13 -(glibc), on the image's CPU architecture. +#### Packaging your module + +Build your module, then run the packager from the workspace your module is built in. It is a +development dependency like `@ekklesia/module-ui-build`: + +```bash +pnpm exec ekklesia-pack-module --module . --target any --out ../packages \ + --source-revision "$(git rev-parse HEAD)" +``` + +It writes `--any.tar.gz` and its `.sha256` line, and nothing else anywhere: it never +installs, links, or deploys, so your workspace is left exactly as it was. Never run `pnpm deploy` in +a workspace instead: it rewrites that workspace's install state to production-only. + +The archive holds one directory named with your module id: + +```text +/ +├── package.json generated: name, version, private, "type": "module", the layer marker +├── ekklesia.module.json your manifest, byte for byte +├── ekklesia.package.json package metadata: target, runtime, provenance, file digests +├── THIRD_PARTY_NOTICES licence and notice text of every third-party package bundled +├── dist/server/index.js your backend entrypoints, bundled +├── dist/schema/index.js +├── dist/chunks/ code your backend entrypoints share +├── dist/ui/ … dist/display/ … dist/contributions/ your browser artifacts, exactly as built +└── drizzle/ your migrations +``` + +- **Bundled backend.** Your server, schema, and jobs entrypoints are bundled with every + pure-JavaScript dependency they use, the Module SDK, the contracts, and Drizzle included. The only + imports left are Node.js built-ins. A dependency the bundle cannot resolve fails packaging; nothing + is left to be found at run time. +- **Never shipped:** `src/`, tests, TypeScript declarations, source maps, development dependencies, + your development `package.json`, any `workspace:` range, and links of any kind. +- **Browser artifacts must be current.** The packager rebuilds them in memory and refuses an artifact + on disk that differs, so run your build first. +- **`any` is the only target today.** A dependency that is native or platform-specific (a compiled + addon, a package that declares an operating system or CPU, or one that loads a per-platform binary + such as `sharp`) cannot be in an `any` package, and packaging fails. Native targets come later. +- **Licences.** `THIRD_PARTY_NOTICES` credits every third-party package your bundles contain, with + its declared licence and the licence and notice files it ships. A package that declares no licence + fails packaging, and so does one that ships no licence text, unless its licence's standard text is + complete without a copyright line (Apache-2.0, which the packager then supplies and says so). The + Ekklesia platform's own packages are listed in `ekklesia.package.json` under `incorporates`. +- **Deterministic.** The same packager version and the same built inputs produce the same archive, + byte for byte, wherever the checkout is. **Drizzle, for now.** You define tables with your own copy of Drizzle, and Core runs your queries through the platform's Drizzle instance (see [Database access](#database-access)). Objects from two copies interoperate only when the copies are interchangeable, so the current implementation requires -**exactly the Drizzle version the platform uses**: the one `@ekklesia/module-sdk` depends on. This is -a constraint of today's implementation, not a permanent contract. +**exactly the Drizzle version the platform uses**: the one `@ekklesia/module-sdk` depends on. The +package records it under `transitional`, and an installation refuses a package whose Drizzle differs. +This is a constraint of today's implementation, not a permanent contract. -Browser artifacts are already self-contained bundles, so nothing changes for them. +#### Verifying a package -The format modules are delivered in is proposed in -[ADR 0049](../architecture/decisions/0049-module-package-v1.md): one deterministic archive per -target, with bundled backend code, produced by an Ekklesia-owned packager that does not exist yet. -Until it does, the repository's smoke test packages the module template with `pnpm deploy` in a -disposable Docker build (`scripts/lib/test-module-package.Dockerfile`), a test fixture and not a -package format. Never run `pnpm deploy` in your own workspace: it rewrites that workspace's install -state to production-only, and the next build or `pnpm verify` then fails until a normal -`pnpm install` repairs it. +`node core/dist/cli/verify-module-package.js --sha256-file .sha256`, run from an +Ekklesia installation (`/opt/ekklesia` in the image), checks a package against that platform without +installing it. Pass `--extract-to ` to receive the verified module directory. +[deploy/README.md](../../deploy/README.md#installed-modules) shows the whole installation. The repository's boundary checks (`pnpm boundaries`) prove only source modules in a workspace. An installed package normally has no source, so the platform cannot check it the same way. Your diff --git a/package.json b/package.json index 454aaea..ec96ffa 100644 --- a/package.json +++ b/package.json @@ -34,7 +34,9 @@ "migrations:validate": "turbo run build && node core/dist/cli/validate-migrations.js", "modules:disable": "turbo run build && node --env-file-if-exists=.env core/dist/cli/set-module-enabled.js --disable", "modules:enable": "turbo run build && node --env-file-if-exists=.env core/dist/cli/set-module-enabled.js --enable", + "modules:pack": "turbo run build && node packages/module-pack/bin/ekklesia-pack-module.mjs", "modules:validate": "turbo run build && node core/dist/cli/validate-modules.js", + "modules:verify-package": "turbo run build && node core/dist/cli/verify-module-package.js", "test": "turbo run test test:root", "test:migrations:integration": "pnpm build && bash scripts/test-migrations-integration.sh", "test:root": "vitest run --config vitest.config.ts", diff --git a/packages/contracts/src/index.ts b/packages/contracts/src/index.ts index 9de2a52..6bbe580 100644 --- a/packages/contracts/src/index.ts +++ b/packages/contracts/src/index.ts @@ -8,6 +8,7 @@ export * from "./module-http.js"; export * from "./module-lifecycle.js"; export * from "./module-manifest.js"; export * from "./module-migration.js"; +export * from "./module-package.js"; export * from "./module-realtime.js"; export * from "./module-runtime.js"; export * from "./module-schema.js"; diff --git a/packages/contracts/src/module-package.ts b/packages/contracts/src/module-package.ts new file mode 100644 index 0000000..a0b8239 --- /dev/null +++ b/packages/contracts/src/module-package.ts @@ -0,0 +1,203 @@ +import { z } from "zod"; +import { moduleIdSchema } from "./identifiers.js"; +import { semanticVersionRangeSchema, semanticVersionSchema } from "./versions.js"; + +/** + * Ekklesia Module Package v1 (ADR 0049): the format a separately versioned module is distributed + * in. One deterministic `.tar.gz` holding exactly one top-level directory named with the module id, + * whose contents are the installed module directory ADR 0048 requires. This file defines the + * package's metadata (`ekklesia.package.json`), its target names, and the runtime `package.json` + * the packager generates. The archive encoding itself belongs to the packager and the verifier. + */ + +export const MODULE_PACKAGE_FORMAT = 1; +export const MODULE_PACKAGE_METADATA_FILENAME = "ekklesia.package.json"; +export const MODULE_PACKAGE_NOTICES_FILENAME = "THIRD_PARTY_NOTICES"; +/** The portable target: a package with no native or platform-specific file. */ +export const MODULE_PACKAGE_ANY_TARGET = "any"; + +/** + * `any`, or `-[-]` in Node.js's own `process.platform` and `process.arch` + * vocabulary. On Linux the C library is required, because a native addon built against one does + * not load against the other; elsewhere it is meaningless and refused. + */ +export const modulePackageTargetSchema = z.string().superRefine((value, context) => { + if (value === MODULE_PACKAGE_ANY_TARGET) { + return; + } + const match = /^([a-z][a-z0-9]*)-([a-z][a-z0-9]*)(?:-([a-z]+))?$/u.exec(value); + const platform = match?.[1]; + const libc = match?.[3]; + const valid = + match !== null && + platform !== "any" && + (platform === "linux" ? libc === "glibc" || libc === "musl" : libc === undefined); + if (!valid) { + context.addIssue({ + code: "custom", + message: "Must be any, or -, with -glibc or -musl on Linux.", + }); + } +}); + +/** A path inside the module directory: relative, POSIX, printable ASCII, and never escaping. ASCII + * keeps JavaScript's string order identical to the archive's byte-wise order. */ +const payloadPathSchema = z.string().superRefine((value, context) => { + const segments = value.split("/"); + const valid = + /^[\x20-\x7e]+$/u.test(value) && + !value.includes("\\") && + !value.startsWith("/") && + segments.every((segment) => segment !== "" && segment !== "." && segment !== "..") && + value !== MODULE_PACKAGE_METADATA_FILENAME; + if (!valid) { + context.addIssue({ + code: "custom", + message: "Must be a portable path inside the module directory, other than the metadata file.", + }); + } +}); + +const modulePackageFileSchema = z.strictObject({ + path: payloadPathSchema, + size: z.number().int().nonnegative(), + sha256: z.string().regex(/^[0-9a-f]{64}$/u, "Must be a lowercase hexadecimal SHA-256."), +}); + +const modulePackageRuntimeSchema = z.strictObject({ + /** The Node.js versions the package was built for. Always present. */ + node: semanticVersionRangeSchema, + /** Native packages only: the minimum Node-API version their addons need. */ + nodeApi: z.number().int().positive().optional(), + /** Native packages only: the exact `process.versions.modules` an ABI-specific addon needs. */ + nodeAbi: z + .string() + .regex(/^[0-9]+$/u) + .optional(), + /** Native Linux packages only: the minimum C library version. */ + libcVersion: semanticVersionRangeSchema.optional(), +}); + +const ekklesiaPackageNameSchema = z + .string() + .regex(/^@ekklesia\/[a-z0-9][a-z0-9-]*$/u, "Must name an @ekklesia package."); + +/** + * Compatibility facts today's implementation needs and that are not part of the long-term contract + * (ADR 0049). Strict at every level: a reader that finds a fact it does not understand must refuse + * the package rather than ignore it. + * + * `databaseRuntime` records the Drizzle version bundled with the module's backend, because Core runs + * a module's queries through its own Drizzle instance over tables the module defined with its own + * copy. That is an implementation constraint pending a Module Database ABI decision, **not** a + * database contract a module may rely on. + */ +const modulePackageTransitionalSchema = z.strictObject({ + databaseRuntime: z + .strictObject({ + "drizzle-orm": semanticVersionSchema.optional(), + }) + .optional(), +}); + +export const modulePackageMetadataSchema = z + .strictObject({ + packageFormat: z.literal(MODULE_PACKAGE_FORMAT), + module: z.strictObject({ id: moduleIdSchema, version: semanticVersionSchema }), + target: modulePackageTargetSchema, + runtime: modulePackageRuntimeSchema, + provenance: z.strictObject({ + /** The exact source revision the package was built from: a full SHA-1 or SHA-256 commit id. */ + revision: z.string().regex(/^(?:[0-9a-f]{40}|[0-9a-f]{64})$/u), + /** Informational only: nothing trusts it, and it ties the package to no host. */ + repository: z.string().min(1).max(2048).optional(), + packager: z.strictObject({ + name: z.string().min(1).max(214), + version: semanticVersionSchema, + }), + }), + /** The exact versions of the Ekklesia SDK and runtime libraries bundled into the package. */ + incorporates: z.record(ekklesiaPackageNameSchema, semanticVersionSchema), + transitional: modulePackageTransitionalSchema.optional(), + /** Every regular file in the module directory except this metadata file, in byte-wise path + * order. The set is exact: nothing missing, nothing extra. */ + files: z.array(modulePackageFileSchema), + }) + .superRefine((metadata, context) => { + if ( + metadata.target === MODULE_PACKAGE_ANY_TARGET && + (metadata.runtime.nodeApi !== undefined || + metadata.runtime.nodeAbi !== undefined || + metadata.runtime.libcVersion !== undefined) + ) { + context.addIssue({ + code: "custom", + message: "An any-target package carries no native runtime requirement.", + path: ["runtime"], + }); + } + for (let index = 1; index < metadata.files.length; index += 1) { + const previous = metadata.files[index - 1]?.path ?? ""; + const current = metadata.files[index]?.path ?? ""; + if (!(previous < current)) { + context.addIssue({ + code: "custom", + message: "Files must be listed once each, in byte-wise path order.", + path: ["files", index, "path"], + }); + } + } + }); + +export type ModulePackageMetadata = z.output; +export type ModulePackageFile = z.output; +export type ModulePackageTarget = z.output; + +export function safeParseModulePackageMetadata(input: unknown) { + return modulePackageMetadataSchema.safeParse(input); +} + +export function safeParseModulePackageTarget(input: unknown) { + return modulePackageTargetSchema.safeParse(input); +} + +/** + * The runtime `package.json` of an installed module: exactly what module discovery reads, generated + * by the packager and never copied from a development workspace. No dependencies, scripts, exports, + * or imports, because nothing at run time resolves through them. + */ +export const modulePackageRuntimePackageSchema = z.strictObject({ + name: z.string().min(1).max(214), + version: semanticVersionSchema, + private: z.literal(true), + type: z.literal("module"), + ekklesia: z.strictObject({ layer: z.literal("module") }), +}); + +export type ModulePackageRuntimePackage = z.output; + +export function modulePackageRuntimePackage(identity: { + readonly name: string; + readonly version: string; +}): ModulePackageRuntimePackage { + return { + name: identity.name, + version: identity.version, + private: true, + type: "module", + ekklesia: { layer: "module" }, + }; +} + +export function safeParseModulePackageRuntimePackage(input: unknown) { + return modulePackageRuntimePackageSchema.safeParse(input); +} + +/** The recommended archive file name. A convenience only: a file name is never trusted. */ +export function modulePackageArchiveFileName( + moduleId: string, + version: string, + target: string, +): string { + return `${moduleId}-${version}-${target}.tar.gz`; +} diff --git a/packages/contracts/tests/module-package.test.ts b/packages/contracts/tests/module-package.test.ts new file mode 100644 index 0000000..9592a18 --- /dev/null +++ b/packages/contracts/tests/module-package.test.ts @@ -0,0 +1,167 @@ +import { describe, expect, it } from "vitest"; +import { + MODULE_PACKAGE_FORMAT, + MODULE_PACKAGE_METADATA_FILENAME, + modulePackageArchiveFileName, + modulePackageRuntimePackage, + safeParseModulePackageMetadata, + safeParseModulePackageRuntimePackage, + safeParseModulePackageTarget, +} from "../src/index.js"; + +const digest = "a".repeat(64); + +function metadata(overrides: Record = {}): Record { + return { + packageFormat: 1, + module: { id: "example", version: "0.1.0" }, + target: "any", + runtime: { node: ">=24.11.0 <25" }, + provenance: { + revision: "86ae907f59458508e3277666cdd214f5bd7c1078", + packager: { name: "@ekklesia/module-pack", version: "0.1.2" }, + }, + incorporates: { "@ekklesia/contracts": "0.1.2", "@ekklesia/module-sdk": "0.1.2" }, + transitional: { databaseRuntime: { "drizzle-orm": "0.45.2" } }, + files: [ + { path: "THIRD_PARTY_NOTICES", size: 10, sha256: digest }, + { path: "dist/server/index.js", size: 20, sha256: digest }, + { path: "ekklesia.module.json", size: 30, sha256: digest }, + { path: "package.json", size: 40, sha256: digest }, + ], + ...overrides, + }; +} + +describe("Ekklesia Module Package v1 metadata (ADR 0049)", () => { + it("names the format, its metadata file, and the recommended archive file name", () => { + expect(MODULE_PACKAGE_FORMAT).toBe(1); + expect(MODULE_PACKAGE_METADATA_FILENAME).toBe("ekklesia.package.json"); + expect(modulePackageArchiveFileName("example", "0.1.0", "any")).toBe( + "example-0.1.0-any.tar.gz", + ); + }); + + it("accepts a complete any-target package description", () => { + expect(safeParseModulePackageMetadata(metadata()).success).toBe(true); + }); + + it("accepts a package that incorporates nothing and needs no transitional facts", () => { + const { transitional: _transitional, ...rest } = metadata(); + expect(safeParseModulePackageMetadata({ ...rest, incorporates: {} }).success).toBe(true); + }); + + it.each([ + ["an unknown top-level field", { signature: "x" }], + ["format 2", { packageFormat: 2 }], + ["a missing source revision", { provenance: { packager: { name: "p", version: "1.0.0" } } }], + [ + "an abbreviated revision", + { provenance: { revision: "86ae907", packager: { name: "p", version: "1.0.0" } } }, + ], + ["a non-semver module version", { module: { id: "example", version: "1.0" } }], + ["an invalid module id", { module: { id: "Example", version: "0.1.0" } }], + ["an invalid Node range", { runtime: { node: "soon" } }], + ["native runtime facts on an any target", { runtime: { node: ">=24", nodeApi: 9 } }], + ["a non-Ekklesia incorporated package", { incorporates: { zod: "4.4.3" } }], + [ + "an incorporated range instead of a version", + { incorporates: { "@ekklesia/module-sdk": "^0.1.0" } }, + ], + ["an unknown transitional key", { transitional: { orm: {} } }], + ["an unknown database runtime", { transitional: { databaseRuntime: { prisma: "6.0.0" } } }], + ])("rejects %s", (_label, overrides) => { + expect(safeParseModulePackageMetadata(metadata(overrides)).success).toBe(false); + }); + + it.each([ + ["an absolute path", "/etc/passwd"], + ["a parent segment", "dist/../../secret"], + ["a dot segment", "./package.json"], + ["an empty segment", "dist//index.js"], + ["a backslash", "dist\\index.js"], + ["a NUL byte", "dist/\u0000"], + ["the metadata file itself", "ekklesia.package.json"], + ])("rejects a file entry with %s", (_label, path) => { + const files = [{ path, size: 1, sha256: digest }]; + expect(safeParseModulePackageMetadata(metadata({ files })).success).toBe(false); + }); + + it("requires file entries in byte-wise path order, without duplicates", () => { + const entry = (path: string) => ({ path, size: 1, sha256: digest }); + expect( + safeParseModulePackageMetadata(metadata({ files: [entry("b"), entry("a")] })).success, + ).toBe(false); + expect( + safeParseModulePackageMetadata(metadata({ files: [entry("a"), entry("a")] })).success, + ).toBe(false); + // Byte-wise, not locale order: upper case sorts before lower case. + expect( + safeParseModulePackageMetadata(metadata({ files: [entry("THIRD"), entry("dist")] })).success, + ).toBe(true); + }); + + it.each([ + ["an uppercase digest", { sha256: "A".repeat(64) }], + ["a short digest", { sha256: "a".repeat(63) }], + ["a negative size", { size: -1 }], + ["a fractional size", { size: 1.5 }], + ])("rejects a file entry with %s", (_label, override) => { + const files = [{ path: "package.json", size: 1, sha256: digest, ...override }]; + expect(safeParseModulePackageMetadata(metadata({ files })).success).toBe(false); + }); +}); + +describe("module package targets", () => { + it.each([ + "any", + "linux-x64-glibc", + "linux-arm64-glibc", + "linux-x64-musl", + "darwin-arm64", + ])("accepts %s", (target) => { + expect(safeParseModulePackageTarget(target).success).toBe(true); + }); + + it.each([ + "", + "ANY", + "linux-x64", + "linux-x64-gnu", + "darwin-arm64-glibc", + "linux", + "any-x64", + ])("rejects %j", (target) => { + expect(safeParseModulePackageTarget(target).success).toBe(false); + }); +}); + +describe("the generated runtime package.json", () => { + it("holds exactly what module discovery reads", () => { + expect( + modulePackageRuntimePackage({ name: "@ekklesia/module-template", version: "0.1.0" }), + ).toEqual({ + name: "@ekklesia/module-template", + version: "0.1.0", + private: true, + type: "module", + ekklesia: { layer: "module" }, + }); + }); + + it("accepts the generated shape and nothing more", () => { + const generated = modulePackageRuntimePackage({ name: "@scope/example", version: "1.2.3" }); + expect(safeParseModulePackageRuntimePackage(generated).success).toBe(true); + for (const extra of [ + { dependencies: { "@ekklesia/module-sdk": "workspace:*" } }, + { scripts: { build: "tsc" } }, + { exports: { ".": "./dist/server/index.js" } }, + { imports: { "#x": "./x.js" } }, + ]) { + expect(safeParseModulePackageRuntimePackage({ ...generated, ...extra }).success).toBe(false); + } + expect(safeParseModulePackageRuntimePackage({ ...generated, type: "commonjs" }).success).toBe( + false, + ); + }); +}); diff --git a/packages/module-pack/bin/ekklesia-pack-module.mjs b/packages/module-pack/bin/ekklesia-pack-module.mjs new file mode 100755 index 0000000..51766eb --- /dev/null +++ b/packages/module-pack/bin/ekklesia-pack-module.mjs @@ -0,0 +1,25 @@ +#!/usr/bin/env node +// Committed launcher, deliberately not generated. +// +// pnpm creates `.bin` links during `pnpm install`, which runs before Turborepo builds anything. A +// `bin` pointing straight at `dist/cli.js` therefore fails to link on a fresh clone — the compiled +// file does not exist yet — and the packager would not link in CI. This file always exists, +// so the link always succeeds; it defers to the compiled CLI at run time, by which point +// `dependsOn: ["^build"]` has produced it. +import { fileURLToPath } from "node:url"; + +const compiledCli = new URL("../dist/cli.js", import.meta.url); + +try { + await import(compiledCli.href); +} catch (error) { + if (error?.code === "ERR_MODULE_NOT_FOUND") { + process.stderr.write( + `@ekklesia/module-pack is not built yet (${fileURLToPath(compiledCli)}).\n` + + "Run `pnpm build` from the repository root first.\n", + ); + process.exitCode = 1; + } else { + throw error; + } +} diff --git a/packages/module-pack/licenses/Apache-2.0.txt b/packages/module-pack/licenses/Apache-2.0.txt new file mode 100644 index 0000000..d645695 --- /dev/null +++ b/packages/module-pack/licenses/Apache-2.0.txt @@ -0,0 +1,202 @@ + + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright [yyyy] [name of copyright owner] + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. diff --git a/packages/module-pack/package.json b/packages/module-pack/package.json new file mode 100644 index 0000000..cbf3912 --- /dev/null +++ b/packages/module-pack/package.json @@ -0,0 +1,46 @@ +{ + "name": "@ekklesia/module-pack", + "version": "0.1.2", + "type": "module", + "files": [ + "bin", + "dist", + "licenses" + ], + "bin": { + "ekklesia-pack-module": "./bin/ekklesia-pack-module.mjs" + }, + "exports": { + ".": { + "types": "./dist/index.d.ts", + "import": "./dist/index.js" + } + }, + "scripts": { + "build": "tsc --project tsconfig.build.json", + "lint": "biome lint .", + "prepack": "pnpm --workspace-root exec turbo run build --filter=@ekklesia/module-pack", + "test": "vitest run", + "typecheck": "tsc --noEmit --project tsconfig.json" + }, + "dependencies": { + "@ekklesia/contracts": "workspace:*", + "@ekklesia/module-ui-build": "workspace:*", + "esbuild": "0.28.1", + "semver": "7.8.5" + }, + "devDependencies": { + "@biomejs/biome": "2.5.3", + "@ekklesia/core": "workspace:*", + "@ekklesia/module-sdk": "workspace:*", + "@ekklesia/module-template": "workspace:*", + "@types/node": "24.13.3", + "@types/semver": "7.7.1", + "drizzle-orm": "0.45.2", + "typescript": "7.0.2", + "vitest": "4.1.10" + }, + "ekklesia": { + "layer": "tooling" + } +} diff --git a/packages/module-pack/src/cli.ts b/packages/module-pack/src/cli.ts new file mode 100644 index 0000000..0cecf62 --- /dev/null +++ b/packages/module-pack/src/cli.ts @@ -0,0 +1,102 @@ +#!/usr/bin/env node +import path from "node:path"; +import process from "node:process"; +import { pathToFileURL } from "node:url"; +import { formatMessages } from "esbuild"; +import { ModulePackError, type PackModuleOptions, packModule } from "./pack-module.js"; + +/** + * `ekklesia-pack-module` — the canonical Ekklesia module packager (ADR 0049). + * + * ekklesia-pack-module --module --target any --out + * --source-revision [--repository ] + * + * Writes `--any.tar.gz` and its `.sha256` line to the output directory, and nothing + * else anywhere. Run it in the module's CI after building the module. + */ + +class PackArgumentsError extends Error {} + +const USAGE = + "Usage: ekklesia-pack-module --module --target any --out --source-revision [--repository ]"; + +const OPTIONS: Readonly> = Object.freeze({ + "--module": "moduleDirectory", + "--out": "outDirectory", + "--repository": "repository", + "--source-revision": "sourceRevision", + "--target": "target", +}); + +export function parsePackModuleArguments(argv: readonly string[]): PackModuleOptions { + const values = new Map(); + for (let index = 0; index < argv.length; index += 2) { + const option = OPTIONS[argv[index] ?? ""]; + const value = argv[index + 1]; + if ( + option === undefined || + value === undefined || + value.startsWith("--") || + values.has(option) + ) { + throw new PackArgumentsError(USAGE); + } + values.set(option, value); + } + const required = ["moduleDirectory", "outDirectory", "sourceRevision", "target"] as const; + if (required.some((option) => !values.has(option))) { + throw new PackArgumentsError(USAGE); + } + const repository = values.get("repository"); + return { + moduleDirectory: values.get("moduleDirectory") ?? "", + outDirectory: values.get("outDirectory") ?? "", + ...(repository === undefined ? {} : { repository }), + sourceRevision: values.get("sourceRevision") ?? "", + target: values.get("target") ?? "", + }; +} + +async function main(): Promise { + try { + const result = await packModule(parsePackModuleArguments(process.argv.slice(2))); + process.stdout.write( + `Packed ${result.moduleId}@${result.version} (target ${result.target}): ${path.basename(result.archivePath)}, ${result.archiveBytes} bytes, ${result.fileCount} files\n`, + ); + process.stdout.write(`sha256 ${result.sha256}\n`); + return 0; + } catch (error) { + if (error instanceof PackArgumentsError) { + process.stderr.write(`${error.message}\n`); + return 64; + } + if (error instanceof ModulePackError) { + process.stderr.write(`[${error.code}] ${error.message}\n`); + if (error.diagnostics.length > 0) { + const formatted = await formatMessages([...error.diagnostics], { + color: false, + kind: "error", + }); + process.stderr.write(formatted.join("")); + } + return 1; + } + process.stderr.write("Module packaging failed unexpectedly.\n"); + return 1; + } +} + +function isDirectExecution(): boolean { + const entrypoint = process.argv[1]; + // Run through the committed launcher, which imports this file, or directly. + return ( + entrypoint !== undefined && + (import.meta.url === pathToFileURL(path.resolve(entrypoint)).href || + path.basename(entrypoint) === "ekklesia-pack-module.mjs" || + path.basename(entrypoint) === "ekklesia-pack-module") + ); +} + +if (isDirectExecution()) { + process.exitCode = await main(); +} diff --git a/packages/module-pack/src/index.ts b/packages/module-pack/src/index.ts new file mode 100644 index 0000000..51f4e59 --- /dev/null +++ b/packages/module-pack/src/index.ts @@ -0,0 +1 @@ +export * from "./pack-module.js"; diff --git a/packages/module-pack/src/notices.ts b/packages/module-pack/src/notices.ts new file mode 100644 index 0000000..6c081bd --- /dev/null +++ b/packages/module-pack/src/notices.ts @@ -0,0 +1,176 @@ +import { readdir, readFile } from "node:fs/promises"; +import path from "node:path"; +import { fileURLToPath } from "node:url"; +import type { BundledPackage } from "./packages.js"; + +/** + * `THIRD_PARTY_NOTICES` (ADR 0049): the licence and notice text that redistributing a module + * package requires, generated from what was actually bundled. + * + * - **Discovery.** Every source file in every bundle (the backend, and each browser artifact) is + * traced to the package it belongs to. Packages installed under `node_modules` are third-party + * software and are credited here. The Ekklesia platform's own packages, and the module itself, + * are not third-party software: the platform packages appear in `ekklesia.package.json` under + * `incorporates`. + * - **Content.** For each third-party package: its name, version, declared licence, where in the + * package it is used, and the complete text of every licence, copying, copyright, and notice file + * it ships at its root, plus the file a `SEE LICENSE IN` declaration names. + * - **Duplicates.** A package bundled into several artifacts is credited once, listing every + * artifact kind it is used in. Different packages are never merged, even when their texts match, + * so each component keeps its own notice. + * - **Gaps fail.** A package that declares no licence, or whose licence cannot be established, + * stops packaging. When a package ships no licence file, the standard text of its declared licence + * is supplied only if that licence's standard text is complete without project-specific lines + * (Apache-2.0). An MIT, ISC, or BSD package without its file is refused, because its copyright line + * exists only in that file. + * + * Everything is sorted, and line endings are normalized, so the file is byte-identical for identical + * inputs. + */ + +export type NoticeErrorCode = "MODULE_PACK_LICENCE_TEXT_MISSING" | "MODULE_PACK_LICENCE_UNKNOWN"; + +export class NoticeError extends Error { + readonly code: NoticeErrorCode; + readonly component: string; + + constructor(code: NoticeErrorCode, component: string) { + super(code); + this.name = "NoticeError"; + this.code = code; + this.component = component; + } +} + +export type NoticeUse = "backend" | "browser"; + +export interface NoticeComponent { + readonly bundled: BundledPackage; + readonly uses: ReadonlySet; +} + +/** Licences whose standard text is complete without any project-specific line, so supplying it + * for a package that ships none is faithful. Checked into this package (`licenses/`). */ +const STANDARD_TEXTS: Readonly> = Object.freeze({ + "Apache-2.0": "Apache-2.0.txt", +}); + +const LICENCE_FILE = + /^(?:licen[cs]e(?:[-._][a-z0-9.-]+)?|copying(?:[-._][a-z0-9.-]+)?|notice(?:[-._][a-z0-9.-]+)?|copyright(?:[-._][a-z0-9.-]+)?)$/iu; + +const RULE = "-".repeat(80); + +function licenceOf(bundled: BundledPackage): string | undefined { + const declared = bundled.manifest.license; + if (typeof declared === "string" && declared.trim() !== "") { + return declared.trim(); + } + if (typeof declared === "object" && declared !== null && "type" in declared) { + const type = (declared as { type?: unknown }).type; + return typeof type === "string" && type.trim() !== "" ? type.trim() : undefined; + } + return undefined; +} + +function normalized(text: string): string { + return `${text.replaceAll("\r\n", "\n").replaceAll("\r", "\n").replace(/\s+$/u, "")}\n`; +} + +function compare(left: string, right: string): number { + return left < right ? -1 : left > right ? 1 : 0; +} + +async function licenceTexts( + bundled: BundledPackage, + licence: string, +): Promise { + const names = (await readdir(bundled.root, { withFileTypes: true })) + .filter((entry) => entry.isFile() && LICENCE_FILE.test(entry.name)) + .map((entry) => entry.name) + .sort(compare); + const seeFile = /^SEE LICEN[CS]E IN (.+)$/iu.exec(licence)?.[1]?.trim(); + if (seeFile && !names.includes(seeFile)) { + const resolved = path.resolve(bundled.root, seeFile); + if (resolved.startsWith(`${bundled.root}${path.sep}`)) { + names.push(seeFile); + } + } + const texts: { title: string; text: string }[] = []; + for (const name of names) { + texts.push({ + text: normalized(await readFile(path.join(bundled.root, name), "utf8")), + title: name, + }); + } + return texts; +} + +const packageRoot = path.resolve(path.dirname(fileURLToPath(import.meta.url)), ".."); + +async function standardText(licence: string): Promise { + const file = STANDARD_TEXTS[licence]; + return file === undefined + ? undefined + : normalized(await readFile(path.join(packageRoot, "licenses", file), "utf8")); +} + +export async function thirdPartyNotices( + components: ReadonlyMap, +): Promise { + const ordered = [...components.values()].sort( + (left, right) => + compare(left.bundled.name, right.bundled.name) || + compare(left.bundled.version, right.bundled.version), + ); + const sections: string[] = []; + for (const { bundled, uses } of ordered) { + const component = `${bundled.name}@${bundled.version}`; + const licence = licenceOf(bundled); + if (licence === undefined || /^UNLICEN[CS]ED$/iu.test(licence)) { + throw new NoticeError("MODULE_PACK_LICENCE_UNKNOWN", component); + } + let texts = await licenceTexts(bundled, licence); + // A NOTICE file alone is not the licence: the licence text must still come from somewhere. + if (!texts.some((text) => !/^notice/iu.test(text.title))) { + const standard = await standardText(licence); + if (standard === undefined) { + throw new NoticeError("MODULE_PACK_LICENCE_TEXT_MISSING", component); + } + texts = [ + ...texts, + { + text: `The package ships no licence file. This is the standard text of the licence it declares (${licence}).\n\n${standard}`, + title: `Standard ${licence} text`, + }, + ]; + } + sections.push( + [ + RULE, + `${bundled.name} ${bundled.version}`, + `License: ${licence}`, + `Used in: ${[...uses].sort(compare).join(", ")}`, + RULE, + "", + ...texts.flatMap((text) => [`[${text.title}]`, "", text.text]), + ].join("\n"), + ); + } + + const header = [ + "THIRD-PARTY SOFTWARE NOTICES", + "", + "This module package includes the third-party software listed below. Each entry names the", + "component, its version, its declared licence, where in the package it is used, and the licence", + "and notice text that came with it.", + "", + "The Ekklesia platform's own packages are not third-party software. They are listed in", + 'ekklesia.package.json under "incorporates".', + "", + ordered.length === 0 + ? "This module package includes no third-party software." + : `${ordered.length} ${ordered.length === 1 ? "component" : "components"}.`, + "", + ].join("\n"); + return `${[header, ...sections].join("\n").replace(/\n+$/u, "")}\n`; +} diff --git a/packages/module-pack/src/pack-module.ts b/packages/module-pack/src/pack-module.ts new file mode 100644 index 0000000..76b26c4 --- /dev/null +++ b/packages/module-pack/src/pack-module.ts @@ -0,0 +1,588 @@ +import { createHash } from "node:crypto"; +import { lstat, mkdir, readdir, readFile, realpath, rename, writeFile } from "node:fs/promises"; +import { isBuiltin } from "node:module"; +import path from "node:path"; +import { fileURLToPath } from "node:url"; +import { + MODULE_PACKAGE_ANY_TARGET, + MODULE_PACKAGE_FORMAT, + MODULE_PACKAGE_METADATA_FILENAME, + MODULE_PACKAGE_NOTICES_FILENAME, + type ModuleManifest, + modulePackageArchiveFileName, + modulePackageRuntimePackage, + safeParseModuleManifest, + safeParseModulePackageMetadata, +} from "@ekklesia/contracts"; +import { analyzeModuleUi, ModuleUiBuildError } from "@ekklesia/module-ui-build"; +import { build, type Message, type Plugin } from "esbuild"; +import { minVersion, satisfies, validRange } from "semver"; +import { NoticeError, type NoticeComponent, type NoticeUse, thirdPartyNotices } from "./notices.js"; +import { BundledPackageIndex, isNativeOrPlatformSpecific } from "./packages.js"; +import { deterministicTarGz, TarPathError } from "./tar.js"; + +/** + * The canonical Ekklesia module packager (ADR 0049): turns a built module directory into one + * deterministic Module Package v1 archive and its `.sha256` line. + * + * It reads the module directory and the dependencies its workspace installed, and writes only the + * two output files. It never installs, links, or deploys anything, so the workspace it runs in is + * left exactly as it was. In this release it produces the portable `any` target only. + */ + +export type ModulePackErrorCode = + | "MODULE_PACK_BUNDLE_FAILED" + | "MODULE_PACK_DATABASE_RUNTIME_CONFLICT" + | "MODULE_PACK_ENTRYPOINT_MISSING" + | "MODULE_PACK_ENTRYPOINT_UNSUPPORTED" + | "MODULE_PACK_EXTERNAL_DEPENDENCY" + | "MODULE_PACK_FORBIDDEN_FILE" + | "MODULE_PACK_IDENTITY_MISMATCH" + | "MODULE_PACK_INCORPORATED_CONFLICT" + | "MODULE_PACK_LAYOUT_CONFLICT" + | "MODULE_PACK_LICENCE_TEXT_MISSING" + | "MODULE_PACK_LICENCE_UNKNOWN" + | "MODULE_PACK_MANIFEST_INVALID" + | "MODULE_PACK_METADATA_INVALID" + | "MODULE_PACK_MIGRATIONS_INVALID" + | "MODULE_PACK_NATIVE_DEPENDENCY" + | "MODULE_PACK_OUTPUT_FAILED" + | "MODULE_PACK_PATH_UNSUPPORTED" + | "MODULE_PACK_REVISION_INVALID" + | "MODULE_PACK_RUNTIME_UNKNOWN" + | "MODULE_PACK_SDK_INCOMPATIBLE" + | "MODULE_PACK_TARGET_UNSUPPORTED" + | "MODULE_PACK_UI_STALE"; + +const messages: Readonly> = Object.freeze({ + MODULE_PACK_BUNDLE_FAILED: + "The backend could not be bundled into self-contained files. Every import must resolve and bundle cleanly.", + MODULE_PACK_DATABASE_RUNTIME_CONFLICT: "The backend bundles more than one Drizzle version.", + MODULE_PACK_ENTRYPOINT_MISSING: + "A declared backend entrypoint does not exist. Build the module before packaging it.", + MODULE_PACK_ENTRYPOINT_UNSUPPORTED: + "Backend entrypoints must be compiled .js files to be bundled into a Module Package v1.", + MODULE_PACK_EXTERNAL_DEPENDENCY: + "The backend bundle would import a dependency at run time; an any-target package bundles everything.", + MODULE_PACK_FORBIDDEN_FILE: + "The package would contain a source, test, declaration, source-map, or dependency file.", + MODULE_PACK_IDENTITY_MISMATCH: + "package.json must declare a name and the same version as ekklesia.module.json.", + MODULE_PACK_INCORPORATED_CONFLICT: + "The package bundles more than one version of an Ekklesia package.", + MODULE_PACK_LAYOUT_CONFLICT: "Two package files would have the same path.", + MODULE_PACK_LICENCE_TEXT_MISSING: + "A bundled package ships no licence text, and its licence has no complete standard text to supply.", + MODULE_PACK_LICENCE_UNKNOWN: "A bundled package declares no licence that permits redistribution.", + MODULE_PACK_MANIFEST_INVALID: "ekklesia.module.json is missing or invalid.", + MODULE_PACK_METADATA_INVALID: "The generated package metadata failed its own contract.", + MODULE_PACK_MIGRATIONS_INVALID: + "The migrations directory must hold only regular files and directories.", + MODULE_PACK_NATIVE_DEPENDENCY: + "The backend bundles a native or platform-specific dependency, which an any-target package cannot contain.", + MODULE_PACK_OUTPUT_FAILED: "The package could not be written to the output directory.", + MODULE_PACK_PATH_UNSUPPORTED: + "A package path is not portable printable ASCII that fits the archive format.", + MODULE_PACK_REVISION_INVALID: "The source revision must be a full 40- or 64-character commit id.", + MODULE_PACK_RUNTIME_UNKNOWN: + "The Node.js range could not be determined. Run inside an Ekklesia workspace, or pass one.", + MODULE_PACK_SDK_INCOMPATIBLE: + "The bundled Module SDK is outside the manifest's own compatibility.moduleSdk range.", + MODULE_PACK_TARGET_UNSUPPORTED: "This packager produces the any target only.", + MODULE_PACK_UI_STALE: + "A declared browser artifact is missing or out of date. Build the module before packaging it.", +}); + +export class ModulePackError extends Error { + readonly code: ModulePackErrorCode; + /** The package or path the failure concerns, when there is one. A module author's own build + * input, never a secret. */ + readonly detail?: string; + /** esbuild's structured diagnostics, for a command line to print. */ + readonly diagnostics: readonly Message[]; + + constructor(code: ModulePackErrorCode, detail?: string, diagnostics: readonly Message[] = []) { + super(detail === undefined ? messages[code] : `${messages[code]} (${detail})`); + this.name = "ModulePackError"; + this.code = code; + if (detail !== undefined) { + this.detail = detail; + } + this.diagnostics = diagnostics; + } +} + +export interface PackModuleOptions { + /** A built module directory: its manifest, package.json, and compiled entrypoints. */ + readonly moduleDirectory: string; + /** `any` only, in this release. */ + readonly target: string; + readonly outDirectory: string; + /** The full commit id the module was built from. */ + readonly sourceRevision: string; + /** Informational only; recorded as given. */ + readonly repository?: string; + /** The Node.js range the package is built for. Defaults to the enclosing workspace's + * `engines.node`, the toolchain the module was built and tested with. */ + readonly nodeRange?: string; +} + +export interface ModulePackResult { + readonly archivePath: string; + readonly sha256Path: string; + readonly sha256: string; + readonly archiveBytes: number; + readonly moduleId: string; + readonly version: string; + readonly target: string; + /** Payload files, the metadata file not included. */ + readonly fileCount: number; +} + +const NATIVE_GUARD = "ekklesia-native-addon-guard"; + +/** Refuses a compiled addon the moment the bundle reaches one. */ +const nativeAddonGuard: Plugin = { + name: NATIVE_GUARD, + setup(pluginBuild) { + pluginBuild.onLoad({ filter: /\.node$/i }, () => ({ + errors: [{ text: "A compiled Node.js addon cannot be bundled into an any-target package." }], + })); + }, +}; + +/** Bundled CommonJS code that requires a Node built-in needs a `require` in module scope. Nothing + * else can be required at run time: every other dependency is inside the bundle, and the packager + * refuses a bundle that would import anything but a built-in. */ +const REQUIRE_BANNER = + 'import { createRequire as __ekklesiaCreateRequire } from "node:module";\nconst require = __ekklesiaCreateRequire(import.meta.url);'; + +function compare(left: string, right: string): number { + return left < right ? -1 : left > right ? 1 : 0; +} + +function sha256(data: Uint8Array): string { + return createHash("sha256").update(data).digest("hex"); +} + +function isRecord(value: unknown): value is Record { + return typeof value === "object" && value !== null && !Array.isArray(value); +} + +async function readJson(file: string): Promise { + return JSON.parse(await readFile(file, "utf8")) as unknown; +} + +const ownManifestPath = path.resolve( + path.dirname(fileURLToPath(import.meta.url)), + "..", + "package.json", +); + +async function packagerIdentity(): Promise<{ readonly name: string; readonly version: string }> { + const manifest = (await readJson(ownManifestPath)) as Record; + return { name: String(manifest.name), version: String(manifest.version) }; +} + +async function workspaceNodeRange(moduleDirectory: string): Promise { + let directory = moduleDirectory; + for (;;) { + try { + await lstat(path.join(directory, "pnpm-workspace.yaml")); + const manifest = await readJson(path.join(directory, "package.json")); + const engines = isRecord(manifest) ? manifest.engines : undefined; + const range = isRecord(engines) ? engines.node : undefined; + if (typeof range === "string") { + return range; + } + break; + } catch { + const parent = path.dirname(directory); + if (parent === directory) { + break; + } + directory = parent; + } + } + throw new ModulePackError("MODULE_PACK_RUNTIME_UNKNOWN"); +} + +/** `./dist/server/index.js` → `dist/server/index.js`, inside the module directory. */ +function moduleRelative(declared: string): string { + return path.posix.normalize(declared.replace(/^\.\//u, "")); +} + +function portable(relative: string): boolean { + const segments = relative.split("/"); + return ( + /^[\x20-\x7e]+$/u.test(relative) && + !relative.includes("\\") && + segments.every((segment) => segment !== "" && segment !== "." && segment !== "..") + ); +} + +function forbidden(relative: string): boolean { + return ( + /(^|\/)(src|tests?|node_modules)\//u.test(relative) || + /\.(map|d\.ts|d\.mts|d\.cts)$/u.test(relative) + ); +} + +async function readTree(root: string, relative: string, files: Map): Promise { + const directory = path.join(root, relative); + for (const entry of (await readdir(directory, { withFileTypes: true })).sort((left, right) => + compare(left.name, right.name), + )) { + const childRelative = `${relative}/${entry.name}`; + const stats = await lstat(path.join(directory, entry.name)); + if (stats.isDirectory()) { + await readTree(root, childRelative, files); + } else if (stats.isFile()) { + files.set(childRelative, await readFile(path.join(directory, entry.name))); + } else { + throw new ModulePackError("MODULE_PACK_MIGRATIONS_INVALID", childRelative); + } + } +} + +interface BackendBundle { + readonly outputs: ReadonlyMap; + readonly inputs: readonly string[]; +} + +async function bundleBackend( + moduleDirectory: string, + manifest: ModuleManifest, + nodeRange: string, +): Promise { + const declared = [ + manifest.entrypoints.server, + manifest.entrypoints.schema, + manifest.entrypoints.jobs, + ].filter((entrypoint): entrypoint is string => entrypoint !== undefined); + const entryPoints: { in: string; out: string }[] = []; + for (const entrypoint of declared) { + const relative = moduleRelative(entrypoint); + if (!relative.endsWith(".js")) { + throw new ModulePackError("MODULE_PACK_ENTRYPOINT_UNSUPPORTED", relative); + } + try { + if (!(await lstat(path.join(moduleDirectory, relative))).isFile()) { + throw new Error("not a file"); + } + } catch { + throw new ModulePackError("MODULE_PACK_ENTRYPOINT_MISSING", relative); + } + entryPoints.push({ in: `./${relative}`, out: relative.slice(0, -".js".length) }); + } + + const minimum = minVersion(nodeRange); + let result: Awaited>; + try { + result = await build({ + absWorkingDir: moduleDirectory, + banner: { js: REQUIRE_BANNER }, + bundle: true, + chunkNames: "dist/chunks/[name]-[hash]", + entryPoints, + format: "esm", + // Licence comments stay in the code as well; THIRD_PARTY_NOTICES is what satisfies the + // licences, never these comments alone. + legalComments: "eof", + logLevel: "silent", + metafile: true, + minify: false, + outdir: moduleDirectory, + platform: "node", + plugins: [nativeAddonGuard], + sourcemap: false, + splitting: true, + target: `node${minimum?.version ?? "24"}`, + write: false, + }); + } catch (error) { + const errors = + isRecord(error) && Array.isArray(error.errors) ? (error.errors as Message[]) : []; + const native = errors.some((message) => message.pluginName === NATIVE_GUARD); + throw new ModulePackError( + native ? "MODULE_PACK_NATIVE_DEPENDENCY" : "MODULE_PACK_BUNDLE_FAILED", + undefined, + errors, + ); + } + // A warning is a bundle that might not run: a require it could not follow, an import it could + // not see. An installed module has nothing to fall back on, so a warning fails packaging. + if (result.warnings.length > 0) { + throw new ModulePackError("MODULE_PACK_BUNDLE_FAILED", undefined, result.warnings); + } + const { metafile, outputFiles } = result; + if (metafile === undefined || outputFiles === undefined) { + throw new ModulePackError("MODULE_PACK_BUNDLE_FAILED"); + } + for (const output of Object.values(metafile.outputs)) { + for (const imported of output.imports) { + if (imported.external && !isBuiltin(imported.path)) { + throw new ModulePackError("MODULE_PACK_EXTERNAL_DEPENDENCY", imported.path); + } + } + } + const outputs = new Map(); + for (const outputFile of outputFiles) { + outputs.set( + path.relative(moduleDirectory, outputFile.path).split(path.sep).join("/"), + Buffer.from(outputFile.contents), + ); + } + return { + inputs: Object.keys(metafile.inputs).map((input) => path.resolve(moduleDirectory, input)), + outputs, + }; +} + +export async function packModule(options: PackModuleOptions): Promise { + if (options.target !== MODULE_PACKAGE_ANY_TARGET) { + throw new ModulePackError("MODULE_PACK_TARGET_UNSUPPORTED", options.target); + } + if (!/^(?:[0-9a-f]{40}|[0-9a-f]{64})$/u.test(options.sourceRevision)) { + throw new ModulePackError("MODULE_PACK_REVISION_INVALID"); + } + const moduleDirectory = await realpath(path.resolve(options.moduleDirectory)).catch(() => { + throw new ModulePackError("MODULE_PACK_MANIFEST_INVALID"); + }); + + const manifestBytes = await readFile(path.join(moduleDirectory, "ekklesia.module.json")).catch( + () => { + throw new ModulePackError("MODULE_PACK_MANIFEST_INVALID"); + }, + ); + let manifestInput: unknown; + try { + manifestInput = JSON.parse(manifestBytes.toString("utf8")); + } catch { + throw new ModulePackError("MODULE_PACK_MANIFEST_INVALID"); + } + const parsedManifest = safeParseModuleManifest(manifestInput); + if (!parsedManifest.success) { + throw new ModulePackError("MODULE_PACK_MANIFEST_INVALID"); + } + const manifest = parsedManifest.data; + const developmentPackage = await readJson(path.join(moduleDirectory, "package.json")).catch( + () => undefined, + ); + if ( + !isRecord(developmentPackage) || + typeof developmentPackage.name !== "string" || + developmentPackage.name === "" || + developmentPackage.version !== manifest.version + ) { + throw new ModulePackError("MODULE_PACK_IDENTITY_MISMATCH"); + } + + const nodeRange = options.nodeRange ?? (await workspaceNodeRange(moduleDirectory)); + if (validRange(nodeRange) === null) { + throw new ModulePackError("MODULE_PACK_RUNTIME_UNKNOWN"); + } + + // 1. The backend, bundled with every pure-JavaScript dependency it uses. + const backend = await bundleBackend(moduleDirectory, manifest, nodeRange); + + // 2. The browser artifacts, exactly as built, with what went into each. + let ui: Awaited>; + try { + ui = await analyzeModuleUi({ moduleDirectory }); + } catch (error) { + throw new ModulePackError( + "MODULE_PACK_UI_STALE", + error instanceof ModuleUiBuildError ? error.code : undefined, + ); + } + + // 3. Every bundled source file traced to its package. + const index = new BundledPackageIndex(); + const components = new Map< + string, + { bundled: Awaited>; uses: Set } + >(); + const incorporates = new Map(); + const drizzleVersions = new Set(); + const sources: { readonly file: string; readonly use: NoticeUse }[] = [ + ...backend.inputs.map((file) => ({ file, use: "backend" as const })), + ...ui.artifacts.flatMap((artifact) => + artifact.inputs.map((file) => ({ file, use: "browser" as const })), + ), + ]; + for (const { file, use } of sources) { + const bundled = await index.packageOf(file); + if (bundled === undefined || bundled.root === moduleDirectory) { + continue; + } + if (!bundled.thirdParty) { + if (bundled.name.startsWith("@ekklesia/")) { + const known = incorporates.get(bundled.name); + if (known !== undefined && known !== bundled.version) { + throw new ModulePackError("MODULE_PACK_INCORPORATED_CONFLICT", bundled.name); + } + incorporates.set(bundled.name, bundled.version); + } + continue; + } + if (use === "backend" && bundled.name === "drizzle-orm") { + drizzleVersions.add(bundled.version); + } + const key = `${bundled.name}@${bundled.version}`; + const existing = components.get(key); + if (existing) { + existing.uses.add(use); + } else { + if (use === "backend" && (await isNativeOrPlatformSpecific(bundled))) { + throw new ModulePackError("MODULE_PACK_NATIVE_DEPENDENCY", key); + } + components.set(key, { bundled, uses: new Set([use]) }); + } + } + if (drizzleVersions.size > 1) { + throw new ModulePackError("MODULE_PACK_DATABASE_RUNTIME_CONFLICT"); + } + const sdk = incorporates.get("@ekklesia/module-sdk"); + if (sdk !== undefined && !satisfies(sdk, manifest.compatibility.moduleSdk)) { + throw new ModulePackError("MODULE_PACK_SDK_INCOMPATIBLE", sdk); + } + + // 4. The payload. + const files = new Map(); + const add = (relative: string, data: Buffer): void => { + if (!portable(relative)) { + throw new ModulePackError("MODULE_PACK_PATH_UNSUPPORTED", relative); + } + if (forbidden(relative)) { + throw new ModulePackError("MODULE_PACK_FORBIDDEN_FILE", relative); + } + if (files.has(relative) || relative === MODULE_PACKAGE_METADATA_FILENAME) { + throw new ModulePackError("MODULE_PACK_LAYOUT_CONFLICT", relative); + } + files.set(relative, data); + }; + for (const [relative, data] of backend.outputs) { + add(relative, data); + } + for (const artifact of ui.artifacts) { + if (!artifact.builtArtifactCurrent) { + throw new ModulePackError( + "MODULE_PACK_UI_STALE", + path.relative(moduleDirectory, artifact.outputFile), + ); + } + add( + path.relative(moduleDirectory, artifact.outputFile).split(path.sep).join("/"), + await readFile(artifact.outputFile), + ); + } + if (manifest.entrypoints.migrations !== undefined) { + const migrations = new Map(); + try { + await readTree(moduleDirectory, moduleRelative(manifest.entrypoints.migrations), migrations); + } catch (error) { + if (error instanceof ModulePackError) { + throw error; + } + throw new ModulePackError("MODULE_PACK_MIGRATIONS_INVALID"); + } + for (const [relative, data] of migrations) { + add(relative, data); + } + } + add("ekklesia.module.json", manifestBytes); + add( + "package.json", + Buffer.from( + `${JSON.stringify( + modulePackageRuntimePackage({ name: developmentPackage.name, version: manifest.version }), + null, + 2, + )}\n`, + ), + ); + try { + add( + MODULE_PACKAGE_NOTICES_FILENAME, + Buffer.from(await thirdPartyNotices(components as ReadonlyMap)), + ); + } catch (error) { + if (error instanceof NoticeError) { + throw new ModulePackError(error.code, error.component); + } + throw error; + } + + // 5. The metadata, checked against its own contract before anything is written. + const listed = [...files.entries()] + .sort(([left], [right]) => compare(left, right)) + .map(([relative, data]) => ({ path: relative, sha256: sha256(data), size: data.length })); + const metadata = { + packageFormat: MODULE_PACKAGE_FORMAT, + module: { id: manifest.id, version: manifest.version }, + target: MODULE_PACKAGE_ANY_TARGET, + runtime: { node: nodeRange }, + provenance: { + revision: options.sourceRevision, + ...(options.repository === undefined ? {} : { repository: options.repository }), + packager: await packagerIdentity(), + }, + incorporates: Object.fromEntries( + [...incorporates].sort(([left], [right]) => compare(left, right)), + ), + ...(drizzleVersions.size === 1 + ? { transitional: { databaseRuntime: { "drizzle-orm": [...drizzleVersions][0] } } } + : {}), + files: listed.map((file) => ({ path: file.path, size: file.size, sha256: file.sha256 })), + }; + if (!safeParseModulePackageMetadata(metadata).success) { + throw new ModulePackError("MODULE_PACK_METADATA_INVALID"); + } + const payload = new Map(files); + payload.set( + MODULE_PACKAGE_METADATA_FILENAME, + Buffer.from(`${JSON.stringify(metadata, null, 2)}\n`), + ); + + // 6. The archive and its digest. + let archive: Buffer; + try { + archive = deterministicTarGz(manifest.id, payload); + } catch (error) { + if (error instanceof TarPathError) { + throw new ModulePackError("MODULE_PACK_PATH_UNSUPPORTED", error.path); + } + throw error; + } + const digest = sha256(archive); + const fileName = modulePackageArchiveFileName( + manifest.id, + manifest.version, + MODULE_PACKAGE_ANY_TARGET, + ); + const outDirectory = path.resolve(options.outDirectory); + const archivePath = path.join(outDirectory, fileName); + const sha256Path = `${archivePath}.sha256`; + try { + await mkdir(outDirectory, { recursive: true }); + const staging = `${archivePath}.partial-${process.pid}`; + await writeFile(staging, archive); + await rename(staging, archivePath); + await writeFile(`${sha256Path}.partial-${process.pid}`, `${digest} ${fileName}\n`); + await rename(`${sha256Path}.partial-${process.pid}`, sha256Path); + } catch { + throw new ModulePackError("MODULE_PACK_OUTPUT_FAILED"); + } + + return Object.freeze({ + archiveBytes: archive.length, + archivePath, + fileCount: listed.length, + moduleId: manifest.id, + sha256: digest, + sha256Path, + target: MODULE_PACKAGE_ANY_TARGET, + version: manifest.version, + }); +} diff --git a/packages/module-pack/src/packages.ts b/packages/module-pack/src/packages.ts new file mode 100644 index 0000000..be2cb0c --- /dev/null +++ b/packages/module-pack/src/packages.ts @@ -0,0 +1,154 @@ +import { access, readdir, readFile, realpath } from "node:fs/promises"; +import path from "node:path"; + +/** + * What the packager knows about a package whose files were bundled: where it is, what it is called, + * and whether it is third-party software. A package installed under a `node_modules` directory is + * third-party; one that is not (a workspace package, or the module itself) is the platform's or the + * module's own code. + */ +export interface BundledPackage { + readonly root: string; + readonly name: string; + readonly version: string; + readonly thirdParty: boolean; + readonly manifest: Readonly>; +} + +function isRecord(value: unknown): value is Record { + return typeof value === "object" && value !== null && !Array.isArray(value); +} + +async function readManifest(directory: string): Promise | undefined> { + try { + const parsed = JSON.parse( + await readFile(path.join(directory, "package.json"), "utf8"), + ) as unknown; + return isRecord(parsed) ? parsed : undefined; + } catch { + return undefined; + } +} + +/** Finds the package a bundled file belongs to: the nearest enclosing directory whose package.json + * names a package and its version. Nested `package.json` files that only set a module type, as + * some libraries ship in subdirectories, are skipped. */ +export class BundledPackageIndex { + readonly #byDirectory = new Map(); + + async packageOf(file: string): Promise { + let directory = path.dirname(await realpath(file).catch(() => file)); + const visited: string[] = []; + let found: BundledPackage | undefined; + for (;;) { + if (this.#byDirectory.has(directory)) { + found = this.#byDirectory.get(directory); + break; + } + visited.push(directory); + const manifest = await readManifest(directory); + if (manifest && typeof manifest.name === "string" && typeof manifest.version === "string") { + found = Object.freeze({ + manifest: Object.freeze(manifest), + name: manifest.name, + root: directory, + thirdParty: directory.split(path.sep).includes("node_modules"), + version: manifest.version, + }); + break; + } + const parent = path.dirname(directory); + if (parent === directory) { + break; + } + directory = parent; + } + for (const seen of visited) { + this.#byDirectory.set(seen, found); + } + return found; + } +} + +async function exists(file: string): Promise { + try { + await access(file); + return true; + } catch { + return false; + } +} + +function declaresPlatform(manifest: Readonly>): boolean { + return ["os", "cpu", "libc"].some((field) => manifest[field] !== undefined); +} + +async function containsCompiledAddon(directory: string): Promise { + const entries = await readdir(directory, { withFileTypes: true }).catch(() => []); + for (const entry of entries) { + if (entry.isDirectory()) { + if ( + entry.name !== "node_modules" && + (await containsCompiledAddon(path.join(directory, entry.name))) + ) { + return true; + } + } else if (entry.name.toLowerCase().endsWith(".node")) { + return true; + } + } + return false; +} + +/** The installed package a dependency name resolves to from `root`, by Node's own directory walk. */ +async function installedDependency( + root: string, + name: string, +): Promise | undefined> { + let directory = root; + for (;;) { + if (path.basename(directory) !== "node_modules") { + const manifest = await readManifest(path.join(directory, "node_modules", ...name.split("/"))); + if (manifest) { + return manifest; + } + } + const parent = path.dirname(directory); + if (parent === directory) { + return undefined; + } + directory = parent; + } +} + +/** + * Whether a bundled package is native or platform-specific, which an `any` package may never + * contain (ADR 0049): it declares an operating system, CPU, or C library; it is built with node-gyp; + * it ships a compiled `.node` addon; or one of its installed dependencies declares a platform, as + * packages that load a per-platform binary do. + */ +export async function isNativeOrPlatformSpecific(bundled: BundledPackage): Promise { + const manifest = bundled.manifest; + if (declaresPlatform(manifest) || manifest.gypfile === true) { + return true; + } + if (await exists(path.join(bundled.root, "binding.gyp"))) { + return true; + } + if (await containsCompiledAddon(bundled.root)) { + return true; + } + for (const field of ["dependencies", "optionalDependencies"]) { + const dependencies = manifest[field]; + if (!isRecord(dependencies)) { + continue; + } + for (const name of Object.keys(dependencies)) { + const dependency = await installedDependency(bundled.root, name); + if (dependency && declaresPlatform(dependency)) { + return true; + } + } + } + return false; +} diff --git a/packages/module-pack/src/tar.ts b/packages/module-pack/src/tar.ts new file mode 100644 index 0000000..ead1115 --- /dev/null +++ b/packages/module-pack/src/tar.ts @@ -0,0 +1,109 @@ +import { gzipSync } from "node:zlib"; + +/** + * The deterministic archive encoding of Ekklesia Module Package v1 (ADR 0049). + * + * POSIX ustar, written by hand rather than by a general tar library, so every byte is decided here: + * - one top-level directory, then every directory and regular file in byte-wise path order, which + * puts every directory before what it contains; + * - uid and gid 0, empty user and group names, modification time 0, device fields 0; + * - directories 0755 and files 0644; no links, no special files, no extended or long-name headers; + * - exactly two zero blocks at the end; + * - gzip at a fixed level, with Node's header: no file name and a modification time of 0. + * + * The same packager version and the same input bytes therefore always produce the same archive. + */ + +export class TarPathError extends Error { + readonly path: string; + + constructor(entryPath: string) { + super("A path does not fit a portable ustar header."); + this.name = "TarPathError"; + this.path = entryPath; + } +} + +const BLOCK = 512; +const GZIP_LEVEL = 9; + +function octal(value: number, width: number): string { + return `${value.toString(8).padStart(width - 1, "0")}\u0000`; +} + +/** Splits a path into ustar's 155-byte prefix and 100-byte name, at a `/`. */ +function splitPath(entryPath: string): { readonly name: string; readonly prefix: string } { + if (entryPath.length <= 100) { + return { name: entryPath, prefix: "" }; + } + for (let index = entryPath.length - 1; index > 0; index -= 1) { + if (entryPath[index] === "/") { + const prefix = entryPath.slice(0, index); + const name = entryPath.slice(index + 1); + if (prefix.length <= 155 && name.length > 0 && name.length <= 100) { + return { name, prefix }; + } + } + } + throw new TarPathError(entryPath); +} + +function header(entryPath: string, directory: boolean, size: number): Buffer { + const block = Buffer.alloc(BLOCK); + const { name, prefix } = splitPath(directory ? `${entryPath}/` : entryPath); + block.write(name, 0, 100, "latin1"); + block.write(octal(directory ? 0o755 : 0o644, 8), 100, "latin1"); + block.write(octal(0, 8), 108, "latin1"); + block.write(octal(0, 8), 116, "latin1"); + block.write(octal(size, 12), 124, "latin1"); + block.write(octal(0, 12), 136, "latin1"); + block.write(" ", 148, "latin1"); + block.write(directory ? "5" : "0", 156, "latin1"); + block.write("ustar\u000000", 257, "latin1"); + block.write(octal(0, 8), 329, "latin1"); + block.write(octal(0, 8), 337, "latin1"); + block.write(prefix, 345, 155, "latin1"); + let checksum = 0; + for (const byte of block) { + checksum += byte; + } + block.write(`${checksum.toString(8).padStart(6, "0")}\u0000 `, 148, "latin1"); + return block; +} + +function compareBytes(left: string, right: string): number { + return left < right ? -1 : left > right ? 1 : 0; +} + +/** + * Archives `files` (paths relative to the module directory, printable ASCII) under `topLevel`. + * Directories are implied by the file paths. + */ +export function deterministicTarGz(topLevel: string, files: ReadonlyMap): Buffer { + const directories = new Set([topLevel]); + for (const filePath of files.keys()) { + const segments = filePath.split("/"); + for (let index = 1; index < segments.length; index += 1) { + directories.add(`${topLevel}/${segments.slice(0, index).join("/")}`); + } + } + const entries: { readonly path: string; readonly data?: Buffer }[] = [ + ...[...directories].map((directory) => ({ path: directory })), + ...[...files].map(([filePath, data]) => ({ data, path: `${topLevel}/${filePath}` })), + ].sort((left, right) => compareBytes(left.path, right.path)); + + const parts: Buffer[] = []; + for (const entry of entries) { + if (entry.data === undefined) { + parts.push(header(entry.path, true, 0)); + } else { + parts.push(header(entry.path, false, entry.data.length), entry.data); + const padding = (BLOCK - (entry.data.length % BLOCK)) % BLOCK; + if (padding > 0) { + parts.push(Buffer.alloc(padding)); + } + } + } + parts.push(Buffer.alloc(BLOCK * 2)); + return gzipSync(Buffer.concat(parts), { level: GZIP_LEVEL }); +} diff --git a/packages/module-pack/tests/cli.test.ts b/packages/module-pack/tests/cli.test.ts new file mode 100644 index 0000000..22094be --- /dev/null +++ b/packages/module-pack/tests/cli.test.ts @@ -0,0 +1,74 @@ +import { describe, expect, it } from "vitest"; +import { parsePackModuleArguments } from "../src/cli.js"; + +const revision = "86ae907f59458508e3277666cdd214f5bd7c1078"; + +describe("ekklesia-pack-module arguments", () => { + it("takes the module, the target, the output directory, and the source revision", () => { + expect( + parsePackModuleArguments([ + "--module", + "templates/module", + "--target", + "any", + "--out", + "dist/packages", + "--source-revision", + revision, + ]), + ).toEqual({ + moduleDirectory: "templates/module", + outDirectory: "dist/packages", + sourceRevision: revision, + target: "any", + }); + }); + + it("records a repository only when one is given", () => { + expect( + parsePackModuleArguments([ + "--module", + ".", + "--target", + "any", + "--out", + "out", + "--source-revision", + revision, + "--repository", + "https://example.org/acme/example.git", + ]).repository, + ).toBe("https://example.org/acme/example.git"); + }); + + it.each([ + ["no arguments", []], + ["a missing target", ["--module", ".", "--out", "o", "--source-revision", revision]], + ["a missing revision", ["--module", ".", "--target", "any", "--out", "o"]], + [ + "an unknown option", + ["--module", ".", "--target", "any", "--out", "o", "--source-revision", revision, "--legacy"], + ], + [ + "a repeated option", + [ + "--module", + ".", + "--module", + "..", + "--target", + "any", + "--out", + "o", + "--source-revision", + revision, + ], + ], + [ + "a stray positional argument", + ["module", "--target", "any", "--out", "o", "--source-revision", revision], + ], + ])("refuses %s", (_label, argv) => { + expect(() => parsePackModuleArguments(argv)).toThrow(); + }); +}); diff --git a/packages/module-pack/tests/pack-module.test.ts b/packages/module-pack/tests/pack-module.test.ts new file mode 100644 index 0000000..956b206 --- /dev/null +++ b/packages/module-pack/tests/pack-module.test.ts @@ -0,0 +1,434 @@ +import { createHash } from "node:crypto"; +import { readFile } from "node:fs/promises"; +import { isBuiltin } from "node:module"; +import path from "node:path"; +import { afterEach, describe, expect, it } from "vitest"; +import { type ModulePackErrorCode, ModulePackError, packModule } from "../src/index.js"; +import { + cleanupFixtures, + type FixtureModuleOptions, + fixtureModule, + SOURCE_REVISION, + scratchDirectory, +} from "./support/fixture-module.js"; +import { fileEntries, readTarGz } from "./support/tar.js"; + +afterEach(cleanupFixtures); + +const MIT = "MIT License\n\nCopyright (c) 2026 Example Author\n\nPermission is hereby granted.\n"; + +async function pack(options: FixtureModuleOptions = {}, overrides: Record = {}) { + const moduleDirectory = await fixtureModule(options); + const outDirectory = await scratchDirectory("ekklesia-pack-out-"); + const result = await packModule({ + moduleDirectory, + nodeRange: ">=24.11.0 <25", + outDirectory, + sourceRevision: SOURCE_REVISION, + target: "any", + ...overrides, + }); + return { archive: await readFile(result.archivePath), moduleDirectory, outDirectory, result }; +} + +async function packRejection( + options: FixtureModuleOptions, + code: ModulePackErrorCode, + overrides: Record = {}, +): Promise { + try { + await pack(options, overrides); + } catch (error) { + expect(error).toBeInstanceOf(ModulePackError); + expect((error as ModulePackError).code).toBe(code); + return error as ModulePackError; + } + throw new Error(`Expected ${code}.`); +} + +describe("packModule: what a package contains (ADR 0049)", () => { + it("produces one archive and its sha256sum line, under the recommended name", async () => { + const { archive, outDirectory, result } = await pack(); + const digest = createHash("sha256").update(archive).digest("hex"); + expect(path.basename(result.archivePath)).toBe("example-0.1.0-any.tar.gz"); + expect(result.sha256).toBe(digest); + expect(await readFile(path.join(outDirectory, "example-0.1.0-any.tar.gz.sha256"), "utf8")).toBe( + `${digest} example-0.1.0-any.tar.gz\n`, + ); + }); + + it("holds exactly the runtime payload, under one top-level directory named with the module id", async () => { + const { archive } = await pack({ withSchema: true }); + const files = [...fileEntries(archive).keys()].sort(); + expect(files).toEqual([ + "example/THIRD_PARTY_NOTICES", + expect.stringMatching(/^example\/dist\/chunks\/chunk-[A-Z0-9]+\.js$/u), + "example/dist/schema/index.js", + "example/dist/server/index.js", + "example/drizzle/0000_initial.sql", + "example/drizzle/meta/0000_snapshot.json", + "example/drizzle/meta/_ekklesia.json", + "example/drizzle/meta/_journal.json", + "example/ekklesia.module.json", + "example/ekklesia.package.json", + "example/package.json", + ]); + for (const file of files) { + expect(file).not.toMatch(/\/src\/|\/tests\/|\.d\.ts$|\.map$|README|node_modules/u); + } + expect(new Set(readTarGz(archive).map((entry) => entry.path.split("/")[0]))).toEqual( + new Set(["example"]), + ); + }); + + it("generates the runtime package.json instead of copying the development one", async () => { + const { archive } = await pack(); + const runtimePackage = fileEntries(archive).get("example/package.json")?.toString("utf8"); + expect(JSON.parse(runtimePackage ?? "")).toEqual({ + name: "@test/module-example", + version: "0.1.0", + private: true, + type: "module", + ekklesia: { layer: "module" }, + }); + for (const [file, data] of fileEntries(archive)) { + if (file.endsWith(".json")) { + expect(data.toString("utf8"), file).not.toContain("workspace:"); + } + } + }); + + it("copies the manifest byte for byte", async () => { + const { archive, moduleDirectory } = await pack(); + expect(fileEntries(archive).get("example/ekklesia.module.json")).toEqual( + await readFile(path.join(moduleDirectory, "ekklesia.module.json")), + ); + }); + + it("describes the package completely in ekklesia.package.json, never hashing itself", async () => { + const { archive } = await pack({ withSchema: true }); + const files = fileEntries(archive); + const metadata = JSON.parse(files.get("example/ekklesia.package.json")?.toString("utf8") ?? ""); + expect(metadata).toMatchObject({ + packageFormat: 1, + module: { id: "example", version: "0.1.0" }, + target: "any", + runtime: { node: ">=24.11.0 <25" }, + provenance: { + revision: SOURCE_REVISION, + packager: { name: "@ekklesia/module-pack", version: expect.any(String) }, + }, + incorporates: { + "@ekklesia/contracts": expect.any(String), + "@ekklesia/module-sdk": expect.any(String), + }, + transitional: { databaseRuntime: { "drizzle-orm": "0.45.2" } }, + }); + const listed = metadata.files.map((file: { path: string }) => `example/${file.path}`); + expect(listed).not.toContain("example/ekklesia.package.json"); + expect(new Set(listed)).toEqual( + new Set([...files.keys()].filter((file) => file !== "example/ekklesia.package.json")), + ); + for (const file of metadata.files as { path: string; size: number; sha256: string }[]) { + const data = files.get(`example/${file.path}`); + expect(data?.length).toBe(file.size); + expect( + createHash("sha256") + .update(data ?? "") + .digest("hex"), + ).toBe(file.sha256); + } + }); + + it("records only the provenance it was given, and a repository only when asked", async () => { + const { archive } = await pack({}, { repository: "https://example.org/acme/example.git" }); + const metadata = JSON.parse( + fileEntries(archive).get("example/ekklesia.package.json")?.toString("utf8") ?? "", + ); + expect(metadata.provenance.repository).toBe("https://example.org/acme/example.git"); + }); + + it("omits transitional database facts from a module that bundles no Drizzle", async () => { + const { archive } = await pack({ + serverSource: 'export default { id: "example", version: "0.1.0", register() {} };\n', + }); + const metadata = JSON.parse( + fileEntries(archive).get("example/ekklesia.package.json")?.toString("utf8") ?? "", + ); + expect(metadata.transitional).toBeUndefined(); + expect(metadata.incorporates).toEqual({}); + }); +}); + +describe("packModule: a self-contained backend", () => { + it("bundles the SDK, contracts, and Drizzle, leaving only Node built-ins to import", async () => { + const { archive } = await pack({ withSchema: true }); + for (const [file, data] of fileEntries(archive)) { + if (!file.endsWith(".js")) { + continue; + } + const code = data.toString("utf8"); + const specifiers = [ + ...code.matchAll(/(?:^|\n)\s*(?:import|export)\s[^;]*?from\s*"([^"]+)"/gu), + ...code.matchAll(/(?:^|\n)\s*import\s*"([^"]+)"/gu), + ...code.matchAll(/\bimport\(\s*"([^"]+)"\s*\)/gu), + ].map((match) => match[1] ?? ""); + for (const specifier of specifiers) { + expect(specifier.startsWith(".") || isBuiltin(specifier), `${file}: ${specifier}`).toBe( + true, + ); + } + } + }); + + it("refuses a dependency the bundle cannot resolve rather than leaving it external", async () => { + await packRejection( + { + serverSource: + 'import pad from "ekklesia-not-installed";\nexport default { id: "example", version: "0.1.0", register() { pad(); } };\n', + }, + "MODULE_PACK_BUNDLE_FAILED", + ); + }); + + it.each([ + [ + "a package that declares an operating system", + { + name: "fake-native", + packageJson: { os: ["linux"], cpu: ["x64"] }, + files: { "index.js": "export default 1;\n" }, + }, + ], + [ + "a package built with node-gyp", + { + name: "fake-gyp", + packageJson: { gypfile: true }, + files: { "index.js": "export default 1;\n", "binding.gyp": "{}" }, + }, + ], + [ + "a package that ships a compiled addon", + { + name: "fake-addon", + files: { + "index.js": "export default 1;\n", + "build/Release/addon.node": new Uint8Array([0x7f, 0x45, 0x4c, 0x46]), + }, + }, + ], + ])("refuses %s in an any-target package", async (_label, fake) => { + await packRejection( + { + fakePackages: [fake], + serverSource: `import value from "${fake.name}";\nexport default { id: "example", version: "0.1.0", register() { return value; } };\n`, + }, + "MODULE_PACK_NATIVE_DEPENDENCY", + ); + }); + + it("refuses a package whose installed dependency is platform-specific", async () => { + await packRejection( + { + fakePackages: [ + { + name: "fake-wrapper", + packageJson: { optionalDependencies: { "fake-wrapper-linux-x64": "1.0.0" } }, + files: { "index.js": "export default 1;\n", LICENSE: MIT }, + }, + { + name: "fake-wrapper-linux-x64", + packageJson: { os: ["linux"], cpu: ["x64"] }, + files: { "index.js": "export default 2;\n" }, + }, + ], + serverSource: + 'import value from "fake-wrapper";\nexport default { id: "example", version: "0.1.0", register() { return value; } };\n', + }, + "MODULE_PACK_NATIVE_DEPENDENCY", + ); + }); + + it("refuses any target but any in this release", async () => { + await packRejection({}, "MODULE_PACK_TARGET_UNSUPPORTED", { target: "linux-x64-glibc" }); + }); +}); + +describe("packModule: identity and inputs", () => { + it("refuses a development package.json whose version differs from the manifest", async () => { + await packRejection( + { manifest: (manifest) => ({ ...manifest, version: "0.2.0" }) }, + "MODULE_PACK_IDENTITY_MISMATCH", + ); + }); + + it("refuses a module whose backend was never built", async () => { + await packRejection( + { + manifest: (manifest) => ({ + ...manifest, + entrypoints: { server: "./dist/missing/index.js" }, + }), + }, + "MODULE_PACK_ENTRYPOINT_MISSING", + ); + }); + + it("refuses an abbreviated or missing source revision", async () => { + await packRejection({}, "MODULE_PACK_REVISION_INVALID", { sourceRevision: "86ae907" }); + }); + + it("refuses an incorporated SDK outside the manifest's own range", async () => { + await packRejection( + { + manifest: (manifest) => ({ + ...manifest, + compatibility: { platform: "^0.1.0", moduleSdk: "^9.0.0" }, + }), + }, + "MODULE_PACK_SDK_INCOMPATIBLE", + ); + }); +}); + +describe("packModule: THIRD_PARTY_NOTICES", () => { + const usesFake = (name: string) => + `import value from "${name}";\nexport default { id: "example", version: "0.1.0", register() { return value; } };\n`; + + it("credits every bundled third-party package once, with its licence and notice text", async () => { + const { archive } = await pack({ + fakePackages: [ + { + name: "fake-mit", + version: "2.3.4", + files: { + "index.js": 'import other from "fake-apache-notice";\nexport default other;\n', + LICENSE: MIT, + }, + }, + { + name: "fake-apache-notice", + version: "1.0.0", + packageJson: { license: "Apache-2.0" }, + files: { + "index.js": "export default 1;\n", + "LICENSE.txt": "Apache License text shipped by the package.\n", + NOTICE: "Fake Apache Notice\nCopyright 2026 The Fake Foundation\n", + }, + }, + ], + serverSource: usesFake("fake-mit"), + }); + const notices = fileEntries(archive).get("example/THIRD_PARTY_NOTICES")?.toString("utf8") ?? ""; + expect(notices).toContain("fake-mit 2.3.4"); + expect(notices).toContain("Copyright (c) 2026 Example Author"); + expect(notices).toContain("fake-apache-notice 1.0.0"); + expect(notices).toContain("Copyright 2026 The Fake Foundation"); + expect(notices.match(/^fake-mit 2\.3\.4$/gmu)).toHaveLength(1); + // Only what the bundle actually contains is credited: this backend uses no Drizzle code. + expect(notices).not.toContain("drizzle-orm"); + // The Ekklesia platform's own packages are credited in ekklesia.package.json, not here. + expect(notices).not.toMatch(/^@ekklesia\//mu); + }); + + it("supplies the standard Apache-2.0 text only for a package that ships none, and says so", async () => { + const { archive } = await pack({ withSchema: true }); + const notices = fileEntries(archive).get("example/THIRD_PARTY_NOTICES")?.toString("utf8") ?? ""; + const drizzle = notices.slice(notices.indexOf("drizzle-orm 0.45.2")); + expect(drizzle).toContain("License: Apache-2.0"); + expect(drizzle).toContain("ships no licence file"); + expect(drizzle).toContain("TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION"); + }); + + it("is byte-identical across packages of the same inputs", async () => { + const options = { withSchema: true }; + const first = await pack(options); + const second = await pack(options); + expect(fileEntries(first.archive).get("example/THIRD_PARTY_NOTICES")).toEqual( + fileEntries(second.archive).get("example/THIRD_PARTY_NOTICES"), + ); + }); + + it("refuses a bundled package that declares no licence", async () => { + await packRejection( + { + fakePackages: [ + { + name: "fake-unlicensed", + packageJson: { license: undefined }, + files: { "index.js": "export default 1;\n" }, + }, + ], + serverSource: usesFake("fake-unlicensed"), + }, + "MODULE_PACK_LICENCE_UNKNOWN", + ); + }); + + it("refuses an MIT package that ships no licence text, because only it holds the copyright line", async () => { + await packRejection( + { + fakePackages: [{ name: "fake-textless", files: { "index.js": "export default 1;\n" } }], + serverSource: usesFake("fake-textless"), + }, + "MODULE_PACK_LICENCE_TEXT_MISSING", + ); + }); +}); + +describe("packModule: deterministic archives", () => { + it("produces byte-identical archives and digests for identical inputs", async () => { + const moduleDirectory = await fixtureModule({ withSchema: true }); + const runs = []; + for (let run = 0; run < 2; run += 1) { + const outDirectory = await scratchDirectory("ekklesia-pack-out-"); + const result = await packModule({ + moduleDirectory, + nodeRange: ">=24.11.0 <25", + outDirectory, + sourceRevision: SOURCE_REVISION, + target: "any", + }); + runs.push({ bytes: await readFile(result.archivePath), sha256: result.sha256 }); + } + expect(runs[0]?.sha256).toBe(runs[1]?.sha256); + expect(runs[0]?.bytes.equals(runs[1]?.bytes ?? Buffer.alloc(0))).toBe(true); + }); + + it("normalizes every entry and the gzip header, and contains no links", async () => { + const { archive } = await pack({ withSchema: true }); + // gzip: no file name flag, no modification time. + expect(archive[3]).toBe(0); + expect(archive.readUInt32LE(4)).toBe(0); + + const entries = readTarGz(archive); + const paths = entries.map((entry) => entry.path); + expect(paths).toEqual( + [...paths].sort((left, right) => (left < right ? -1 : left > right ? 1 : 0)), + ); + for (const entry of entries) { + expect(["0", "5"], entry.path).toContain(entry.typeflag); + expect(entry.mode, entry.path).toBe(entry.typeflag === "5" ? 0o755 : 0o644); + expect([entry.uid, entry.gid, entry.mtime]).toEqual([0, 0, 0]); + expect([entry.uname, entry.gname]).toEqual(["", ""]); + expect(entry.magic).toBe("ustar/00"); + } + // Every directory precedes what it contains. + for (const entry of entries) { + const parent = entry.path.split("/").slice(0, -1).join("/"); + if (parent !== "") { + expect(paths.indexOf(parent), entry.path).toBeLessThan(paths.indexOf(entry.path)); + } + } + }); + + it("refers to no absolute path of the build machine", async () => { + const { archive, moduleDirectory } = await pack({ withSchema: true }); + const root = path.dirname(path.dirname(moduleDirectory)); + for (const [file, data] of fileEntries(archive)) { + expect(data.toString("utf8"), file).not.toContain(moduleDirectory); + expect(data.toString("utf8"), file).not.toContain(root); + } + }); +}); diff --git a/packages/module-pack/tests/support/fixture-module.ts b/packages/module-pack/tests/support/fixture-module.ts new file mode 100644 index 0000000..75a4465 --- /dev/null +++ b/packages/module-pack/tests/support/fixture-module.ts @@ -0,0 +1,193 @@ +import { mkdir, mkdtemp, realpath, rm, symlink, writeFile } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import path from "node:path"; +import { fileURLToPath } from "node:url"; + +/** + * Generated module fixtures for the packager's tests: a built module directory in a temporary + * location, whose `node_modules` links to the real Module SDK and Drizzle this package depends on. + * Everything a development module directory holds is there, including what must never ship + * (`src/`, `tests/`, declarations, source maps, `workspace:` ranges), so a test can prove the + * packager leaves it out. + */ + +const packageRoot = path.resolve(path.dirname(fileURLToPath(import.meta.url)), "..", ".."); + +export interface FakePackage { + readonly name: string; + readonly version?: string; + readonly packageJson?: Record; + /** Files relative to the fake package's root. */ + readonly files?: Record; +} + +export interface FixtureModuleOptions { + readonly id?: string; + readonly version?: string; + readonly serverSource?: string; + readonly withSchema?: boolean; + readonly fakePackages?: readonly FakePackage[]; + readonly manifest?: (manifest: Record) => Record; +} + +const temporaryDirectories: string[] = []; + +export async function cleanupFixtures(): Promise { + await Promise.all( + temporaryDirectories + .splice(0) + .map((directory) => rm(directory, { force: true, recursive: true })), + ); +} + +export async function scratchDirectory(prefix = "ekklesia-pack-"): Promise { + const directory = await realpath(await mkdtemp(path.join(tmpdir(), prefix))); + temporaryDirectories.push(directory); + return directory; +} + +async function write(file: string, contents: string | Uint8Array): Promise { + await mkdir(path.dirname(file), { recursive: true }); + await writeFile(file, contents); +} + +const defaultServer = ( + id: string, + version: string, +) => `import { defineModuleRuntime } from "@ekklesia/module-sdk"; +export default defineModuleRuntime({ id: ${JSON.stringify(id)}, version: ${JSON.stringify(version)}, register() {} }); +`; + +const schemaSource = ( + id: string, +) => `import { createModuleTable, defineModuleSchema } from "@ekklesia/module-sdk"; +import { text, uuid } from "drizzle-orm/pg-core"; +export const records = createModuleTable(${JSON.stringify(id)})("records", { + id: uuid("id").primaryKey(), + value: text("value").notNull(), +}); +export default defineModuleSchema({ moduleId: ${JSON.stringify(id)}, tables: { records } }); +`; + +export async function fixtureModule(options: FixtureModuleOptions = {}): Promise { + const id = options.id ?? "example"; + const version = options.version ?? "0.1.0"; + const root = await scratchDirectory("ekklesia-pack-fixture-"); + const moduleDirectory = path.join(root, id); + const tableName = `${id.replaceAll("-", "_")}_records`; + + const baseManifest: Record = { + manifestVersion: 7, + id, + name: `${id} module`, + version, + compatibility: { platform: "^0.1.0", moduleSdk: "^0.1.0" }, + capabilities: [], + permissions: [], + entrypoints: { + server: "./dist/server/index.js", + ...(options.withSchema ? { schema: "./dist/schema/index.js" } : {}), + migrations: "./drizzle", + }, + }; + const manifest = options.manifest ? options.manifest(baseManifest) : baseManifest; + + await write( + path.join(moduleDirectory, "ekklesia.module.json"), + `${JSON.stringify(manifest, null, 2)}\n`, + ); + await write( + path.join(moduleDirectory, "package.json"), + `${JSON.stringify( + { + name: `@test/module-${id}`, + version, + private: true, + type: "module", + scripts: { build: "tsc" }, + dependencies: { "@ekklesia/module-sdk": "workspace:*", "drizzle-orm": "0.45.2" }, + devDependencies: { typescript: "7.0.2" }, + exports: { ".": "./dist/server/index.js" }, + ekklesia: { layer: "module" }, + }, + null, + 2, + )}\n`, + ); + await write( + path.join(moduleDirectory, "dist/server/index.js"), + options.serverSource ?? defaultServer(id, version), + ); + await write(path.join(moduleDirectory, "dist/server/index.js.map"), '{"version":3}\n'); + await write(path.join(moduleDirectory, "dist/server/index.d.ts"), "export {};\n"); + if (options.withSchema) { + await write(path.join(moduleDirectory, "dist/schema/index.js"), schemaSource(id)); + } + await write(path.join(moduleDirectory, "src/server/index.ts"), "export {};\n"); + await write(path.join(moduleDirectory, "tests/server.test.ts"), "export {};\n"); + await write(path.join(moduleDirectory, "README.md"), "# Development notes\n"); + + await write( + path.join(moduleDirectory, "drizzle/0000_initial.sql"), + `CREATE TABLE "${tableName}" ("id" uuid PRIMARY KEY NOT NULL, "value" text NOT NULL);\n`, + ); + await write( + path.join(moduleDirectory, "drizzle/meta/_journal.json"), + JSON.stringify({ + version: "7", + dialect: "postgresql", + entries: [ + { idx: 0, version: "7", when: 1_700_000_000_000, tag: "0000_initial", breakpoints: true }, + ], + }), + ); + await write( + path.join(moduleDirectory, "drizzle/meta/_ekklesia.json"), + JSON.stringify({ + version: 1, + moduleId: id, + entries: [{ tag: "0000_initial", destructive: { mode: "none" } }], + }), + ); + await write( + path.join(moduleDirectory, "drizzle/meta/0000_snapshot.json"), + JSON.stringify({ + version: "7", + dialect: "postgresql", + id: "10000000-0000-4000-8000-000000000001", + prevId: "00000000-0000-0000-0000-000000000000", + }), + ); + + // The module's dependencies, as a workspace links them: the real packages this package depends on. + const modules = path.join(moduleDirectory, "node_modules"); + await mkdir(path.join(modules, "@ekklesia"), { recursive: true }); + await symlink( + await realpath(path.join(packageRoot, "node_modules", "@ekklesia", "module-sdk")), + path.join(modules, "@ekklesia", "module-sdk"), + ); + await symlink( + await realpath(path.join(packageRoot, "node_modules", "drizzle-orm")), + path.join(modules, "drizzle-orm"), + ); + for (const fake of options.fakePackages ?? []) { + const fakeRoot = path.join(modules, ...fake.name.split("/")); + await write( + path.join(fakeRoot, "package.json"), + JSON.stringify({ + name: fake.name, + version: fake.version ?? "1.0.0", + type: "module", + main: "index.js", + license: "MIT", + ...fake.packageJson, + }), + ); + for (const [relative, contents] of Object.entries(fake.files ?? {})) { + await write(path.join(fakeRoot, relative), contents); + } + } + return moduleDirectory; +} + +export const SOURCE_REVISION = "86ae907f59458508e3277666cdd214f5bd7c1078"; diff --git a/packages/module-pack/tests/support/tar.ts b/packages/module-pack/tests/support/tar.ts new file mode 100644 index 0000000..ba246dd --- /dev/null +++ b/packages/module-pack/tests/support/tar.ts @@ -0,0 +1,62 @@ +import { gunzipSync } from "node:zlib"; + +/** A test-side view of every ustar header field the determinism rules constrain (ADR 0049). */ +export interface TarEntry { + readonly path: string; + readonly typeflag: string; + readonly mode: number; + readonly uid: number; + readonly gid: number; + readonly uname: string; + readonly gname: string; + readonly mtime: number; + readonly magic: string; + readonly data: Buffer; +} + +function text(block: Buffer, offset: number, length: number): string { + const raw = block.subarray(offset, offset + length); + const end = raw.indexOf(0); + return raw.subarray(0, end === -1 ? length : end).toString("latin1"); +} + +function octal(block: Buffer, offset: number, length: number): number { + return Number.parseInt(text(block, offset, length).trim() || "0", 8); +} + +export function readTarGz(archive: Buffer): readonly TarEntry[] { + const raw = gunzipSync(archive); + const entries: TarEntry[] = []; + let offset = 0; + while (offset + 512 <= raw.length) { + const block = raw.subarray(offset, offset + 512); + if (block.every((byte) => byte === 0)) { + break; + } + const name = text(block, 0, 100); + const prefix = text(block, 345, 155); + const size = octal(block, 124, 12); + entries.push({ + data: Buffer.from(raw.subarray(offset + 512, offset + 512 + size)), + gid: octal(block, 116, 8), + gname: text(block, 297, 32), + magic: `${text(block, 257, 6)}/${text(block, 263, 2)}`, + mode: octal(block, 100, 8), + mtime: octal(block, 136, 12), + path: prefix === "" ? name : `${prefix}/${name}`, + typeflag: text(block, 156, 1), + uid: octal(block, 108, 8), + uname: text(block, 265, 32), + }); + offset += 512 + Math.ceil(size / 512) * 512; + } + return entries; +} + +export function fileEntries(archive: Buffer): Map { + return new Map( + readTarGz(archive) + .filter((entry) => entry.typeflag === "0") + .map((entry) => [entry.path, entry.data]), + ); +} diff --git a/packages/module-pack/tests/template-package.test.ts b/packages/module-pack/tests/template-package.test.ts new file mode 100644 index 0000000..1d9ab1a --- /dev/null +++ b/packages/module-pack/tests/template-package.test.ts @@ -0,0 +1,195 @@ +import { execFile } from "node:child_process"; +import { createHash } from "node:crypto"; +import { lstat, readdir, readFile, realpath } from "node:fs/promises"; +import path from "node:path"; +import { fileURLToPath } from "node:url"; +import { promisify } from "node:util"; +import { verifyModulePackage } from "@ekklesia/core/modules"; +import { afterEach, describe, expect, it } from "vitest"; +import { packModule } from "../src/index.js"; +import { cleanupFixtures, SOURCE_REVISION, scratchDirectory } from "./support/fixture-module.js"; +import { fileEntries } from "./support/tar.js"; + +/** + * The canonical template module, packaged end to end (ADR 0049): the real built template, the real + * packager, and Core's real verifier. Turbo builds the template and the verifier before this suite + * runs (both are development dependencies of this package), and runs the template's own tests, which + * rebuild its UI in place, before this one (turbo.json). + */ + +const execFileAsync = promisify(execFile); +const packageRoot = path.resolve(path.dirname(fileURLToPath(import.meta.url)), ".."); +const repositoryRoot = path.resolve(packageRoot, "..", ".."); + +afterEach(cleanupFixtures); + +async function templateDirectory(): Promise { + return realpath(path.join(packageRoot, "node_modules", "@ekklesia", "module-template")); +} + +async function readJson(file: string): Promise> { + return JSON.parse(await readFile(file, "utf8")) as Record; +} + +async function platform() { + const core = await readJson(path.join(repositoryRoot, "core", "package.json")); + return { + databaseRuntime: { + "drizzle-orm": (core.dependencies as Record)["drizzle-orm"] ?? "", + }, + moduleSdkVersion: String( + (await readJson(path.join(repositoryRoot, "packages", "module-sdk", "package.json"))).version, + ), + platformVersion: String((await readJson(path.join(repositoryRoot, "package.json"))).version), + }; +} + +async function fingerprint(file: string): Promise { + try { + return createHash("sha256") + .update(await readFile(file)) + .digest("hex"); + } catch { + return "absent"; + } +} + +async function treeListing(directory: string): Promise { + const entries = await readdir(directory, { recursive: true, withFileTypes: true }); + return entries + .filter((entry) => !path.join(entry.parentPath, entry.name).includes(`${path.sep}node_modules`)) + .map((entry) => path.relative(directory, path.join(entry.parentPath, entry.name))) + .sort(); +} + +async function packTemplate() { + const outDirectory = await scratchDirectory("ekklesia-template-package-"); + const result = await packModule({ + moduleDirectory: await templateDirectory(), + outDirectory, + sourceRevision: SOURCE_REVISION, + target: "any", + }); + return { archive: await readFile(result.archivePath), result }; +} + +describe("the module template as a Module Package v1", () => { + it("packs deterministically, without touching the workspace it was built in", async () => { + const template = await templateDirectory(); + const watched = [ + path.join(repositoryRoot, "node_modules", ".pnpm-workspace-state-v1.json"), + path.join(repositoryRoot, "node_modules", ".modules.yaml"), + path.join(repositoryRoot, "pnpm-lock.yaml"), + ]; + const before = await Promise.all(watched.map(fingerprint)); + const templateBefore = await treeListing(template); + + const first = await packTemplate(); + const second = await packTemplate(); + + expect(first.result.sha256).toBe(second.result.sha256); + expect(first.archive.equals(second.archive)).toBe(true); + expect(path.basename(first.result.archivePath)).toMatch( + /^example-\d+\.\d+\.\d+-any\.tar\.gz$/u, + ); + expect(await Promise.all(watched.map(fingerprint))).toEqual(before); + expect(await treeListing(template)).toEqual(templateBefore); + }); + + it("verifies against this platform and extracts to an unrelated directory that holds no link", async () => { + const { result } = await packTemplate(); + const target = path.join(await scratchDirectory("ekklesia-installed-elsewhere-"), "example"); + const verified = await verifyModulePackage({ + archivePath: result.archivePath, + expectedSha256: result.sha256, + extractTo: target, + platform: await platform(), + }); + expect(verified).toMatchObject({ moduleId: "example", target: "any" }); + + for (const entry of await readdir(target, { recursive: true, withFileTypes: true })) { + const stats = await lstat(path.join(entry.parentPath, entry.name)); + expect(stats.isSymbolicLink(), entry.name).toBe(false); + expect(stats.isFile() ? stats.nlink : 1, entry.name).toBe(1); + } + const listing = await treeListing(target); + expect(listing).not.toContain("node_modules"); + expect(listing.some((file) => /(^|\/)(src|tests)(\/|$)|\.d\.ts$|\.map$/u.test(file))).toBe( + false, + ); + }); + + it("loads from its new location without resolving anything outside itself", async () => { + const { result } = await packTemplate(); + const target = path.join(await scratchDirectory("ekklesia-portable-"), "example"); + await verifyModulePackage({ + archivePath: result.archivePath, + extractTo: target, + platform: await platform(), + }); + + // Every specifier the backend resolves while loading, recorded by Node itself. + const script = ` + import { registerHooks } from "node:module"; + import { pathToFileURL } from "node:url"; + const resolved = []; + registerHooks({ resolve(specifier, context, next) { const result = next(specifier, context); resolved.push(result.url); return result; } }); + for (const file of ["dist/server/index.js", "dist/schema/index.js"]) { + await import(pathToFileURL(${JSON.stringify(target)} + "/" + file).href); + } + process.stdout.write(JSON.stringify(resolved)); + `; + const { stdout } = await execFileAsync( + process.execPath, + ["--input-type=module", "-e", script], + { + cwd: path.dirname(target), + env: {}, + }, + ); + const resolved = JSON.parse(stdout) as string[]; + expect(resolved.length).toBeGreaterThan(0); + for (const url of resolved) { + const inside = url.startsWith(`file://${target}/`); + expect(inside || url.startsWith("node:"), url).toBe(true); + } + }); + + it("credits every third-party package its bundles say they contain", async () => { + const { archive } = await packTemplate(); + const files = fileEntries(archive); + const notices = files.get("example/THIRD_PARTY_NOTICES")?.toString("utf8") ?? ""; + const bundled = new Set(); + for (const [file, data] of files) { + if (file.endsWith(".js")) { + // esbuild names every bundled source file in a path comment. + for (const match of data + .toString("utf8") + .matchAll(/node_modules\/((?:@[^/]+\/)?[^/.][^/]*)\//gu)) { + if (match[1] && !match[1].startsWith("@ekklesia/")) { + bundled.add(match[1]); + } + } + } + } + expect([...bundled]).toEqual( + expect.arrayContaining(["drizzle-orm", "react", "react-dom", "zod"]), + ); + for (const name of bundled) { + expect(notices, name).toMatch( + new RegExp(`^${name.replace(/[.*+?^${}()|[\]\\/]/gu, "\\$&")} \\S+$`, "mu"), + ); + } + const licence = await readFile(path.join(packageRoot, "licenses", "Apache-2.0.txt")); + expect(createHash("sha256").update(licence).digest("hex")).toBe( + "cfc7749b96f63bd31c3c42b5c471bf756814053e847c10f3eb003417bc523d30", + ); + }); + + it("refers to no absolute path of the checkout it was built in", async () => { + const { archive } = await packTemplate(); + for (const [file, data] of fileEntries(archive)) { + expect(data.toString("utf8"), file).not.toContain(repositoryRoot); + } + }); +}); diff --git a/packages/module-pack/tsconfig.build.json b/packages/module-pack/tsconfig.build.json new file mode 100644 index 0000000..4f5dbdb --- /dev/null +++ b/packages/module-pack/tsconfig.build.json @@ -0,0 +1,14 @@ +{ + "extends": "./tsconfig.json", + "compilerOptions": { + "declaration": true, + "declarationMap": true, + "noEmit": false, + "outDir": "dist", + "rootDir": "src", + "sourceMap": true, + "types": ["node"] + }, + "include": ["src/**/*.ts"], + "exclude": ["src/**/*.test.ts", "tests"] +} diff --git a/packages/module-pack/tsconfig.json b/packages/module-pack/tsconfig.json new file mode 100644 index 0000000..0d93162 --- /dev/null +++ b/packages/module-pack/tsconfig.json @@ -0,0 +1,8 @@ +{ + "extends": "../typescript-config/node.json", + "compilerOptions": { + "noEmit": true, + "types": ["node", "vitest/globals"] + }, + "include": ["src/**/*.ts", "tests/**/*.ts"] +} diff --git a/packages/module-ui-build/src/build-module-ui.ts b/packages/module-ui-build/src/build-module-ui.ts index d85298b..c9b68c9 100644 --- a/packages/module-ui-build/src/build-module-ui.ts +++ b/packages/module-ui-build/src/build-module-ui.ts @@ -6,7 +6,7 @@ import { moduleDisplayEntrypointSchema, packageRelativePathSchema, } from "@ekklesia/contracts"; -import { build, type Plugin } from "esbuild"; +import { type BuildOptions, build, type Plugin } from "esbuild"; /** * The one supported way to turn a module's React/TSX UI sources into the compiled browser artifacts @@ -108,7 +108,7 @@ const SURVIVING_IMPORT = /@import\s/iu; */ const STYLESHEET_PLUGIN_NAME = "ekklesia-compile-stylesheet-imports"; -function compileStylesheetImports(): Plugin { +function compileStylesheetImports(recordInputs?: (paths: readonly string[]) => void): Plugin { const decoder = new TextDecoder(); return { name: STYLESHEET_PLUGIN_NAME, @@ -127,15 +127,30 @@ function compileStylesheetImports(): Plugin { ], })); + // The nested build works from the module directory, exactly like the outer one. Without it + // esbuild names each stylesheet relative to whatever directory the caller ran from, and the + // artifact's bytes would depend on the caller (ADR 0049 requires a deterministic package). + const workingDirectory = pluginBuild.initialOptions.absWorkingDir ?? process.cwd(); pluginBuild.onLoad({ filter: /\.css$/i }, async (args) => { const compiled = await build({ + absWorkingDir: workingDirectory, bundle: true, entryPoints: [args.path], logLevel: "silent", + metafile: recordInputs !== undefined, minify: false, target: MODULE_UI_BUILD_TARGET, write: false, }); + // A stylesheet's own @imports are bundled by this nested build, so the outer build never + // sees them. An analysis must, because they ship inside the artifact. + if (recordInputs && compiled.metafile) { + recordInputs( + Object.keys(compiled.metafile.inputs).map((input) => + path.resolve(workingDirectory, input), + ), + ); + } const stylesheet = compiled.outputFiles .map((outputFile) => decoder.decode(outputFile.contents)) .join("\n"); @@ -323,6 +338,44 @@ async function resolveDeclaredArtifacts( * Never runs during discovery, startup, or install: this is an authoring-time and packaging-time * step only. Core validates the artifact a module ships; it never produces one (ADR 0021). */ +/** The one esbuild configuration for a declared artifact, shared by building and analysing so the + * two can never describe different builds. */ +function moduleUiBuildOptions( + moduleDirectory: string, + artifact: DeclaredModuleUiArtifact, + recordInputs?: (paths: readonly string[]) => void, +): BuildOptions & { write: false } { + return { + absWorkingDir: moduleDirectory, + // Every dependency, React included, is bundled. The artifact must run in any host, including + // one that uses no React at all, so nothing is externalized (ADR 0023). + bundle: true, + define: { + // React reads this at module scope. Without it the bundle would reference `process`, a + // Node global that does not exist in a browser, and crash on load. + "process.env.NODE_ENV": '"production"', + }, + entryPoints: [artifact.sourceFile], + // ESM only: the artifact is imported by a host with a dynamic `import()`, and the contract + // requires a real default export. + format: "esm", + jsx: "automatic", + // Unminified deliberately: an operator is expected to review the artifact they install + // (see docs/development/security.md), and nothing else in this repository minifies. + minify: false, + outfile: artifact.outputFile, + platform: "browser", + plugins: [compileStylesheetImports(recordInputs)], + sourcemap: true, + target: MODULE_UI_BUILD_TARGET, + // Fail rather than silently shipping a partial bundle. + logLevel: "silent", + // Compile every role successfully before writing any role. This makes a two-artifact build + // atomic with respect to parse/resolve/bundle failures rather than leaving a stale partial set. + write: false, + }; +} + export async function buildModuleUi(options: BuildModuleUiOptions): Promise { const moduleDirectory = path.resolve(options.moduleDirectory); const artifacts = await resolveDeclaredArtifacts(moduleDirectory); @@ -330,35 +383,7 @@ export async function buildModuleUi(options: BuildModuleUiOptions): Promise Object.freeze({ ...artifact }))), }); } + +export interface ModuleUiAnalysis { + /** Every declared artifact, in the same stable role order `buildModuleUi` produces them. */ + readonly artifacts: readonly ModuleUiArtifactAnalysis[]; +} + +export interface ModuleUiArtifactAnalysis { + readonly kind: ModuleUiArtifactKind; + readonly contributionId?: string; + /** Absolute path of the artifact the manifest declares. */ + readonly outputFile: string; + /** Absolute paths of every source file bundled into it, stylesheets and their imports included, + * sorted and unique. What a packager credits for the code it redistributes. */ + readonly inputs: readonly string[]; + /** Whether the artifact on disk is byte-for-byte what these sources build today. False when it + * was never built or is stale, so a caller never describes one artifact and ships another. */ + readonly builtArtifactCurrent: boolean; +} + +/** + * Runs the canonical module UI build in memory, exactly as `buildModuleUi` would, and reports what + * went into each declared artifact. Writes nothing. Used by the module packager (ADR 0049) to find + * the third-party code inside already-built browser artifacts, and to refuse a stale build. + */ +export async function analyzeModuleUi(options: BuildModuleUiOptions): Promise { + const moduleDirectory = path.resolve(options.moduleDirectory); + const artifacts = await resolveDeclaredArtifacts(moduleDirectory); + const analyses: ModuleUiArtifactAnalysis[] = []; + for (const artifact of artifacts) { + const inputs = new Set(); + let result: Awaited>; + try { + result = await build({ + ...moduleUiBuildOptions(moduleDirectory, artifact, (paths) => { + for (const input of paths) { + inputs.add(input); + } + }), + metafile: true, + }); + } catch { + throw new ModuleUiBuildError("MODULE_UI_BUILD_FAILED", artifact.kind); + } + for (const input of Object.keys(result.metafile?.inputs ?? {})) { + inputs.add(path.resolve(moduleDirectory, input)); + } + const built = result.outputFiles?.find((outputFile) => outputFile.path === artifact.outputFile); + let onDisk: Buffer | undefined; + try { + onDisk = await readFile(artifact.outputFile); + } catch { + onDisk = undefined; + } + analyses.push( + Object.freeze({ + ...(artifact.contributionId === undefined + ? {} + : { contributionId: artifact.contributionId }), + builtArtifactCurrent: + built !== undefined && onDisk !== undefined && Buffer.from(built.contents).equals(onDisk), + inputs: Object.freeze([...inputs].sort()), + kind: artifact.kind, + outputFile: artifact.outputFile, + }), + ); + } + return Object.freeze({ artifacts: Object.freeze(analyses) }); +} diff --git a/packages/module-ui-build/tests/build-module-ui.test.ts b/packages/module-ui-build/tests/build-module-ui.test.ts index 15fa39d..1830379 100644 --- a/packages/module-ui-build/tests/build-module-ui.test.ts +++ b/packages/module-ui-build/tests/build-module-ui.test.ts @@ -2,7 +2,7 @@ import { mkdir, mkdtemp, readFile, rm, writeFile } from "node:fs/promises"; import { tmpdir } from "node:os"; import path from "node:path"; import { afterEach, describe, expect, it } from "vitest"; -import { buildModuleUi, ModuleUiBuildError } from "../src/index.js"; +import { analyzeModuleUi, buildModuleUi, ModuleUiBuildError } from "../src/index.js"; const temporaryDirectories: string[] = []; @@ -502,3 +502,87 @@ describe("module UI build", () => { expect(artifact).not.toContain("process.env.NODE_ENV"); }); }); + +describe("module UI build analysis", () => { + const styledSource = `import styles from "./styles.css"; +${plainUiSource.replace("const definition = {", "const definition = {\n styles,")}`; + + it("reports every source file behind each artifact, stylesheets included, and writes nothing", async () => { + const directory = await createModule({ + ui: "./dist/ui/index.js", + files: { + "src/ui/index.tsx": styledSource, + "src/ui/styles.css": + '@import "./base.css";\n.example { color: var(--ekklesia-color-text); }\n', + "src/ui/base.css": ".example-base { margin: 0; }\n", + }, + }); + await buildModuleUi({ moduleDirectory: directory }); + const built = await readFile(path.join(directory, "dist/ui/index.js")); + + const analysis = await analyzeModuleUi({ moduleDirectory: directory }); + + expect(analysis.artifacts).toHaveLength(1); + const [artifact] = analysis.artifacts; + expect(artifact?.kind).toBe("ui"); + expect(artifact?.outputFile).toBe(path.join(directory, "dist/ui/index.js")); + expect(artifact?.builtArtifactCurrent).toBe(true); + expect(artifact?.inputs).toEqual( + expect.arrayContaining([ + path.join(directory, "src/ui/index.tsx"), + path.join(directory, "src/ui/styles.css"), + path.join(directory, "src/ui/base.css"), + ]), + ); + expect([...(artifact?.inputs ?? [])]).toEqual([...new Set(artifact?.inputs)].sort()); + // Nothing was written: the built artifact is byte-for-byte what it was. + expect(await readFile(path.join(directory, "dist/ui/index.js"))).toEqual(built); + }); + + it("compiles stylesheets relative to the module, whatever directory the build runs from", async () => { + const directory = await createModule({ + ui: "./dist/ui/index.js", + files: { + "src/ui/index.tsx": styledSource, + "src/ui/styles.css": '@import "./base.css";\n.example { margin: 0; }\n', + "src/ui/base.css": ".example-base { margin: 0; }\n", + }, + }); + // This test runs from the package directory, never from the module's own: the artifact must + // still name its stylesheets relative to the module, or its bytes would depend on the caller. + expect(process.cwd()).not.toBe(directory); + await buildModuleUi({ moduleDirectory: directory }); + const artifact = await readFile(path.join(directory, "dist/ui/index.js"), "utf8"); + expect(artifact).toContain("/* src/ui/styles.css */"); + expect(artifact).toContain("/* src/ui/base.css */"); + expect(artifact).not.toContain(path.basename(directory)); + }); + + it("says when a built artifact no longer matches its sources", async () => { + const directory = await createModule({ + ui: "./dist/ui/index.js", + files: { "src/ui/index.tsx": plainUiSource }, + }); + await buildModuleUi({ moduleDirectory: directory }); + await writeFile(path.join(directory, "dist/ui/index.js"), "export default {};\n"); + + const analysis = await analyzeModuleUi({ moduleDirectory: directory }); + expect(analysis.artifacts[0]?.builtArtifactCurrent).toBe(false); + }); + + it("says when a declared artifact was never built", async () => { + const directory = await createModule({ + ui: "./dist/ui/index.js", + files: { "src/ui/index.tsx": plainUiSource }, + }); + const analysis = await analyzeModuleUi({ moduleDirectory: directory }); + expect(analysis.artifacts[0]?.builtArtifactCurrent).toBe(false); + }); + + it("returns an empty analysis for a backend-only module", async () => { + const directory = await createModule(); + await expect(analyzeModuleUi({ moduleDirectory: directory })).resolves.toEqual({ + artifacts: [], + }); + }); +}); diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index ffd7375..c2c8e85 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -196,6 +196,49 @@ importers: specifier: 4.1.10 version: 4.1.10(@types/node@24.13.3)(happy-dom@20.0.11)(vite@8.1.4(@types/node@24.13.3)(esbuild@0.28.1)(jiti@2.7.0)(tsx@4.23.1)(yaml@2.9.0)) + packages/module-pack: + dependencies: + '@ekklesia/contracts': + specifier: workspace:* + version: link:../contracts + '@ekklesia/module-ui-build': + specifier: workspace:* + version: link:../module-ui-build + esbuild: + specifier: 0.28.1 + version: 0.28.1 + semver: + specifier: 7.8.5 + version: 7.8.5 + devDependencies: + '@biomejs/biome': + specifier: 2.5.3 + version: 2.5.3 + '@ekklesia/core': + specifier: workspace:* + version: link:../../core + '@ekklesia/module-sdk': + specifier: workspace:* + version: link:../module-sdk + '@ekklesia/module-template': + specifier: workspace:* + version: link:../../templates/module + '@types/node': + specifier: 24.13.3 + version: 24.13.3 + '@types/semver': + specifier: 7.7.1 + version: 7.7.1 + drizzle-orm: + specifier: 0.45.2 + version: 0.45.2(@types/pg@8.20.0)(kysely@0.29.4)(pg@8.22.0) + typescript: + specifier: 7.0.2 + version: 7.0.2 + vitest: + specifier: 4.1.10 + version: 4.1.10(@types/node@24.13.3)(happy-dom@20.0.11)(vite@8.1.4(@types/node@24.13.3)(esbuild@0.28.1)(jiti@2.7.0)(tsx@4.23.1)(yaml@2.9.0)) + packages/module-sdk: dependencies: '@ekklesia/contracts': diff --git a/scripts/lib/build-module-package.Dockerfile b/scripts/lib/build-module-package.Dockerfile new file mode 100644 index 0000000..97af113 --- /dev/null +++ b/scripts/lib/build-module-package.Dockerfile @@ -0,0 +1,31 @@ +# Builds the repository's module template and packages it with the canonical Ekklesia packager +# (`@ekklesia/module-pack`, ADR 0049), in a clean container from a clean copy of the checkout. The +# deploy smoke test uses it because a CI job that drives Docker has no Node.js of its own. The +# packaging itself is exactly what a module's CI runs; this file only provides the environment. +# +# DOCKER_BUILDKIT=1 docker build --file scripts/lib/build-module-package.Dockerfile \ +# --build-arg SOURCE_REVISION= --output type=local,dest= . +# +# writes /example--any.tar.gz and its .sha256 line. No image is produced. +# +# The same pinned Node.js base as deploy/Dockerfile, so the package is built by the runtime it is +# installed into. +FROM node:24.18.0-trixie-slim@sha256:ae91dcc111a68c9d2d81ff2a17bda61be126426176fde6fe7d08ab13b7f50573 AS build +ENV COREPACK_ENABLE_DOWNLOAD_PROMPT=0 \ + PNPM_HOME=/pnpm \ + TURBO_TELEMETRY_DISABLED=1 \ + DO_NOT_TRACK=1 +WORKDIR /opt/source +COPY package.json ./ +RUN corepack enable && corepack install +COPY . . +RUN --mount=type=cache,id=ekklesia-pnpm-store,target=/pnpm/store \ + pnpm install --frozen-lockfile --store-dir /pnpm/store +RUN pnpm exec turbo run build --filter=@ekklesia/module-template --filter=@ekklesia/module-pack +ARG SOURCE_REVISION +RUN test -n "$SOURCE_REVISION" && \ + node packages/module-pack/bin/ekklesia-pack-module.mjs --module templates/module --target any \ + --out /package --source-revision "$SOURCE_REVISION" + +FROM scratch AS package +COPY --from=build /package/ / diff --git a/scripts/lib/module-package-release.sh b/scripts/lib/module-package-release.sh new file mode 100644 index 0000000..ae94bc1 --- /dev/null +++ b/scripts/lib/module-package-release.sh @@ -0,0 +1,53 @@ +# shellcheck shell=bash +# The deploy smoke test's module package release pair (ADR 0049). +# +# A package archive and its `.sha256` are one logical release: they travel together, and the +# verifier reads both. This turns one build's output into exactly that pair, in a directory any user +# can traverse, because the verifier runs as the image's unprivileged user. +# +# BuildKit's local exporter creates its output directory with mode 0700, owned by whoever ran the +# build. On a workstation that is usually uid 1000, which is also the image's `node` user, so the +# verifier happened to read it; in CI the build runs as root, and the verifier could read neither file. +# Nothing here depends on who built the package. + +# module_package_release +# +# Checks the build output holds exactly one archive and its digest and nothing else, and that the +# digest line names that archive. Then creates the release directory, which must not exist yet (so an +# earlier run's files can never be mixed in), mode 0755, and copies both files into it, mode 0644. +# Prints the archive's file name. Fails, creating nothing, on any deviation. +module_package_release() { + local output="$1" release="$2" archives=() archive digest_line digest named entries + + shopt -s nullglob + archives=("$output"/*.tar.gz) + shopt -u nullglob + if ((${#archives[@]} != 1)); then + echo "module_package_release: the build output must hold exactly one package archive." >&2 + return 1 + fi + archive="$(basename -- "${archives[0]}")" + if [[ ! -f "$output/$archive.sha256" ]]; then + echo "module_package_release: the package's .sha256 is missing." >&2 + return 1 + fi + entries="$(find "$output" -mindepth 1 -maxdepth 1 | wc -l)" + if ((entries != 2)); then + echo "module_package_release: the build output holds more than the archive and its digest." >&2 + return 1 + fi + digest_line="$(<"$output/$archive.sha256")" + digest="${digest_line%% *}" + named="${digest_line#* }" + if [[ ! "$digest" =~ ^[0-9a-f]{64}$ || "$named" != "$archive" ]]; then + echo "module_package_release: the .sha256 line does not name this archive." >&2 + return 1 + fi + + mkdir -m 755 -- "$release" || return 1 + if ! install -m 0644 -- "$output/$archive" "$output/$archive.sha256" "$release/"; then + rm -rf -- "$release" + return 1 + fi + printf '%s\n' "$archive" +} diff --git a/scripts/lib/test-module-package.Dockerfile b/scripts/lib/test-module-package.Dockerfile deleted file mode 100644 index b05eae5..0000000 --- a/scripts/lib/test-module-package.Dockerfile +++ /dev/null @@ -1,50 +0,0 @@ -# TEST-ONLY. Builds the repository's module template as one self-contained installed-module directory -# for scripts/smoke-deploy.sh, which installs it into a deployment's module storage (ADR 0048). It is -# not a module package format and defines none: the archive format, signing, and transport of real -# module packages are deferred. No image is produced; the build's only output is the directory: -# -# DOCKER_BUILDKIT=1 docker build --file scripts/lib/test-module-package.Dockerfile \ -# --output type=local,dest= . -# -# writes /example: the module's package.json, manifest, built artifacts, migrations, and a -# node_modules holding exactly its production dependencies, including its own copies of the Module -# SDK, the contracts, and Drizzle. Nothing in it resolves to anything outside it, because the -# platform lends an installed module nothing. -# -# The same pinned Node.js base as deploy/Dockerfile, so the package targets the runtime it is -# installed into. Drizzle comes from the platform's own lockfile, which is what keeps it the exact -# version the platform uses: a constraint of the current implementation (ADR 0048), not a contract. -FROM node:24.18.0-trixie-slim@sha256:ae91dcc111a68c9d2d81ff2a17bda61be126426176fde6fe7d08ab13b7f50573 AS build -ENV COREPACK_ENABLE_DOWNLOAD_PROMPT=0 \ - PNPM_HOME=/pnpm \ - TURBO_TELEMETRY_DISABLED=1 \ - DO_NOT_TRACK=1 -WORKDIR /opt/source -COPY package.json ./ -RUN corepack enable && corepack install -COPY . . -RUN --mount=type=cache,id=ekklesia-pnpm-store,target=/pnpm/store \ - pnpm install --frozen-lockfile --store-dir /pnpm/store -RUN pnpm exec turbo run build --filter=@ekklesia/module-template -# `--legacy` deploys from this workspace without switching it to injected workspace packages; the -# Module SDK and every other workspace dependency are still copied into the package, built. -RUN --mount=type=cache,id=ekklesia-pnpm-store,target=/pnpm/store \ - set -eu; \ - pnpm --filter @ekklesia/module-template deploy --prod --legacy --store-dir /pnpm/store \ - /package/example; \ - rm -f /package/example/README.md; \ - # pnpm's hoisted self-reference points back into this workspace; an installed module has none. - rm -f /package/example/node_modules/.pnpm/node_modules/@ekklesia/module-template; \ - escaping=""; \ - for link in $(find /package/example -type l); do \ - case "$(realpath -m "$link")" in \ - /package/example/*) ;; \ - *) escaping="$escaping $link" ;; \ - esac; \ - done; \ - if [ -n "$escaping" ]; then echo "Links escape the package:$escaping" >&2; exit 1; fi; \ - chmod -R a+rX,go-w /package/example; \ - cd /package/example && node -e 'import("./dist/server/index.js").then((m) => { if (m.default.id !== "example") process.exit(1); })' - -FROM scratch AS package -COPY --from=build /package/ / diff --git a/scripts/smoke-deploy.sh b/scripts/smoke-deploy.sh index 84277b9..73b3eb0 100755 --- a/scripts/smoke-deploy.sh +++ b/scripts/smoke-deploy.sh @@ -15,13 +15,15 @@ # ADR 0048 end to end with that same image: # # - it boots with no modules; -# - the repository's module template, packaged as one self-contained directory -# (scripts/lib/test-module-package.Dockerfile, TEST-ONLY), is installed into module storage, its -# migrations are planned and explicitly applied, and the API is restarted; +# - the repository's module template is packaged by the canonical packager as an Ekklesia Module +# Package v1 (ADR 0049; scripts/lib/build-module-package.Dockerfile only provides the build +# environment), verified by the same image's own verifier, extracted to staging, installed into +# module storage, its migrations planned and explicitly applied, and the API restarted; # - a real headless Chromium drives the Dashboard, the module, its display, and sign-out; # - the module is removed and the API restarted: Core still runs, migration planning reports the # module's retained history, and its data survives; -# - the module is restored and the API restarted: it returns with its previous data. +# - the exact same package is verified and installed again, and the API restarted: the module +# returns with its previous data. # # EKKLESIA_SMOKE_IMAGE uses the given image instead of building one. It must be a generic image # (ADR 0048): modules come from module storage, never from the image. @@ -52,6 +54,8 @@ cd "$repository_root" source "$repository_root/scripts/lib/compose-diagnostics.sh" # shellcheck source=scripts/lib/ci-project.sh source "$repository_root/scripts/lib/ci-project.sh" +# shellcheck source=scripts/lib/module-package-release.sh +source "$repository_root/scripts/lib/module-package-release.sh" for command in docker openssl awk; do command -v "$command" >/dev/null || { @@ -179,7 +183,7 @@ cleanup() { # job itself runs as root), so only the staged copy on the Docker host needs this. if [[ "$daemon_work" == "$work" ]]; then docker run --rm --entrypoint rm -v "$work:/work" "$postgres_image" -rf /work/tls /work/env \ - /work/modules >/dev/null 2>&1 || true + /work/modules /work/staging >/dev/null 2>&1 || true else docker run --rm --entrypoint rm -v /tmp:/host-tmp "$postgres_image" -rf "/host-tmp/${project}" \ >/dev/null 2>&1 || true @@ -213,15 +217,23 @@ if [[ -z "$image" ]]; then fi if [[ "$topology" == split ]]; then - # TEST-ONLY: the module template as one self-contained installed-module directory, built outside - # the platform image and installed into module storage below. Never a second platform image. - step "Packaging the module template as a self-contained test module" - DOCKER_BUILDKIT=1 docker build --quiet --file scripts/lib/test-module-package.Dockerfile \ - --label "${EKKLESIA_CI_LABEL}=${project}" \ + # The module template as a Module Package v1, from the canonical packager. Never a second platform + # image: the package is installed into module storage below, after the image verifies it. + step "Packaging the module template with the canonical Ekklesia packager" + source_revision="${GITHUB_SHA:-$(git rev-parse HEAD 2>/dev/null || true)}" + [[ "$source_revision" =~ ^[0-9a-f]{40}$ ]] || fail "no source revision to record in the package." + DOCKER_BUILDKIT=1 docker build --quiet --file scripts/lib/build-module-package.Dockerfile \ + --label "${EKKLESIA_CI_LABEL}=${project}" --build-arg SOURCE_REVISION="$source_revision" \ --output "type=local,dest=${work}/module-package" . >/dev/null || - fail "the test module package did not build." - [[ -f "$work/module-package/example/ekklesia.module.json" ]] || - fail "the test module package has no manifest." + fail "the module package did not build." + # The archive and its digest are one release: copied together, from this run's own build output, + # into a fresh directory the verifier's unprivileged user can read. BuildKit's output directory is + # mode 0700 and owned by whoever built it, so it is never handed to the verifier itself. + package_archive="$(module_package_release "$work/module-package" "$work/package")" || + fail "the packager did not produce exactly one archive and its digest." + [[ "$package_archive" =~ ^example-[0-9]+\.[0-9]+\.[0-9]+-any\.tar\.gz$ ]] || + fail "the packager produced an unexpected archive name." + echo "ok packaged ${package_archive} ($(wc -c <"$work/package/$package_archive") bytes, sha256 $(cut -d' ' -f1 "$work/package/${package_archive}.sha256"))" fi # The compose project directory: its relative bind mount (postgres/pg_hba.conf) resolves here. @@ -232,6 +244,13 @@ stage deploy # `up`: Compose refuses to start `api` or `migrate` without it, and never creates it. mkdir -m 755 "$work/modules" stage modules +if [[ "$topology" == split ]]; then + stage package + # TEST-ONLY staging the verifier extracts into, as the image's unprivileged user. Nothing here is + # module storage: the install step below moves a verified tree into module storage as root. + mkdir -m 777 "$work/staging" + stage staging +fi step "Creating the installation-private CA and server certificates" init_status=0 @@ -420,15 +439,27 @@ module_storage() { "$postgres_image" -ec "$1" } -# install_test_module: stages the package under a hidden name, which discovery ignores, then renames -# it into place in one step, owned by root and readable by the API's unprivileged user. +# install_test_module: verifies the package with the generic image's own verifier, which writes the +# verified module directory to staging, then moves it into module storage under a hidden name, which +# discovery ignores, and renames it into place in one step, owned by root and readable by the API's +# unprivileged user. Nothing unverified ever reaches module storage. install_test_module() { - tar -C "$work/module-package" -cf - example | - module_storage 'rm -rf /modules/.incoming && mkdir /modules/.incoming && - tar -xf - -C /modules/.incoming && chown -R 0:0 /modules/.incoming/example && - chmod -R a+rX,go-w /modules/.incoming/example && - mv /modules/.incoming/example /modules/example && rmdir /modules/.incoming' || - fail "the test module could not be installed into module storage." + local verified + verified="$( + docker run --rm --network none --read-only --tmpfs /tmp --cap-drop ALL \ + --security-opt no-new-privileges \ + -v "$daemon_work/package:/package:ro" -v "$daemon_work/staging:/staging" \ + --entrypoint node "$image" core/dist/cli/verify-module-package.js "/package/${package_archive}" \ + --sha256-file "/package/${package_archive}.sha256" --extract-to /staging/example 2>&1 + )" || fail "the module package did not verify: ${verified}" + echo "ok ${verified%%$'\n'*}" + docker run --rm --network none --entrypoint sh -v "$daemon_work/modules:/modules" \ + -v "$daemon_work/staging:/staging" "$postgres_image" -ec ' + rm -rf /modules/.incoming-example && cp -R /staging/example /modules/.incoming-example && + rm -rf /staging/example && chown -R 0:0 /modules/.incoming-example && + chmod -R a+rX,go-w /modules/.incoming-example && + mv /modules/.incoming-example /modules/example' || + fail "the verified module could not be installed into module storage." } # restart_api : stops the API, runs the command, and starts the API again. A controlled @@ -590,7 +621,7 @@ if [[ "$topology" == split ]]; then fail "the removed module's data did not survive." echo "ok the module is gone, Core runs, its history is retained, and its data survives" - step "Restoring the module and restarting: it returns with its previous data" + step "Reinstalling the exact same package and restarting: the module returns with its previous data" restart_api install_test_module plan="$(plan_output)" grep -q '^- \[applied\] example:0000_initial ' <<<"$plan" || diff --git a/templates/module/README.md b/templates/module/README.md index da7d421..7d1cc67 100644 --- a/templates/module/README.md +++ b/templates/module/README.md @@ -55,10 +55,16 @@ To create a module inside this repository: That `modules/` directory is where a module is **developed**. An installation never runs it from there: Ekklesia's image contains no modules, and an installation installs a self-contained -copy of the module, with its own production dependencies, into its module storage +copy of the module into its module storage ([An installed module is self-contained](../../docs/module-development/README.md#an-installed-module-is-self-contained), ADR 0048). Copying `dist/` alone is not enough, because the compiled server code imports the Module -SDK and Drizzle by name. +SDK and Drizzle by name. After building, package the module with the canonical packager, which +bundles the backend into one deterministic Ekklesia Module Package (ADR 0049): + +```bash +pnpm modules:pack --module templates/module --target any --out \ + --source-revision "$(git rev-parse HEAD)" +``` The module owns its feature screens and display renderer. Do not add platform navigation, the containing shell, global theme, authentication screens, or top-level routing; those belong to diff --git a/tests/ci-scripts.test.ts b/tests/ci-scripts.test.ts index 5d986b3..e11c287 100644 --- a/tests/ci-scripts.test.ts +++ b/tests/ci-scripts.test.ts @@ -151,6 +151,41 @@ describe("CI scripts on a shared Docker daemon", () => { } }); + it("install the smoke test's module only as a canonical package the image itself verified (ADR 0049)", async () => { + const deploy = (await source("scripts/smoke-deploy.sh")).replace(/\\\n\s*/gu, " "); + const packaging = await source("scripts/lib/build-module-package.Dockerfile"); + expect(packaging).toContain("packages/module-pack/bin/ekklesia-pack-module.mjs"); + expect(packaging).not.toMatch(/pnpm[^\n]*\bdeploy\b/u); + expect(deploy).toContain("scripts/lib/build-module-package.Dockerfile"); + expect(deploy).toMatch( + /--entrypoint node "\$image" core\/dist\/cli\/verify-module-package\.js[^\n]*--sha256-file[^\n]*--extract-to \/staging\/example/u, + ); + // Module storage only ever receives the verified tree from staging. + expect(deploy).toContain("cp -R /staging/example /modules/.incoming-example"); + }); + + it("hand the verifier the package's archive and digest together, from a fresh release pair, never from build output", async () => { + const deploy = (await source("scripts/smoke-deploy.sh")).replace(/\\\n\s*/gu, " "); + expect(deploy).toContain('source "$repository_root/scripts/lib/module-package-release.sh"'); + // The pair is made from this run's own build output, into a directory that must not exist yet. + expect(deploy).toContain('--output "type=local,dest=${work}/module-package"'); + expect(deploy).toContain( + 'package_archive="$(module_package_release "$work/module-package" "$work/package")"', + ); + expect(deploy).toMatch(/^\s*stage package$/mu); + // BuildKit's output directory is 0700 and owned by whoever built it: it is never mounted. + expect(deploy).not.toContain('"$daemon_work/module-package:'); + expect(deploy).not.toMatch(/^\s*stage module-package$/mu); + // The verifier reads both files from the one mount, and always checks the digest file. + const verifier = deploy + .split("\n") + .find((line) => line.includes("core/dist/cli/verify-module-package.js")); + expect(verifier).toContain('-v "$daemon_work/package:/package:ro"'); + expect(verifier).toContain('"/package/${package_archive}"'); + expect(verifier).toContain('--sha256-file "/package/${package_archive}.sha256"'); + expect(verifier).not.toMatch(/--sha256\s/u); + }); + it("run every deploy smoke against the one generic image, never a second image with a module in it", async () => { // ADR 0048: the split topology installs its test module into module storage at run time. const deploy = (await source("scripts/smoke-deploy.sh")).replace(/\\\n\s*/gu, " "); diff --git a/tests/module-package-release.test.ts b/tests/module-package-release.test.ts new file mode 100644 index 0000000..232613d --- /dev/null +++ b/tests/module-package-release.test.ts @@ -0,0 +1,127 @@ +import { execFile } from "node:child_process"; +import { chmod, mkdir, mkdtemp, readdir, readFile, rm, stat, writeFile } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import path from "node:path"; +import { fileURLToPath } from "node:url"; +import { promisify } from "node:util"; +import { afterEach, describe, expect, it } from "vitest"; + +/** + * The deploy smoke test's release pair (ADR 0049): a package archive and its `.sha256` are one + * logical release and reach the verifier together, from a directory every user can traverse. + * + * The regression this exists for: BuildKit's local exporter creates its output directory with mode + * 0700, owned by whoever ran the build. On a workstation that is uid 1000, which is also the image's + * unprivileged `node` user, so the verifier could read it; in CI the build runs as root, and the + * verifier could read neither file ("The SHA-256 file could not be read"). + */ + +const execFileAsync = promisify(execFile); +const helper = path.resolve( + path.dirname(fileURLToPath(import.meta.url)), + "..", + "scripts", + "lib", + "module-package-release.sh", +); +const archive = "example-0.1.0-any.tar.gz"; +const digest = "91e5dfee80880027bd1c2e1796513f07023c03847e9ca3870b51f8272c3accf7"; +const temporaryDirectories: string[] = []; + +afterEach(async () => { + await Promise.all( + temporaryDirectories + .splice(0) + .map((directory) => rm(directory, { force: true, recursive: true })), + ); +}); + +/** A build output exactly as BuildKit leaves it: mode 0700, holding the archive and its digest. */ +async function buildOutput( + files: Readonly> = { + [archive]: "archive bytes", + [`${archive}.sha256`]: `${digest} ${archive}\n`, + }, +): Promise<{ readonly root: string; readonly output: string }> { + const root = await mkdtemp(path.join(tmpdir(), "ekklesia-release-pair-")); + temporaryDirectories.push(root); + const output = path.join(root, "module-package"); + await mkdir(output); + for (const [name, contents] of Object.entries(files)) { + await writeFile(path.join(output, name), contents, { mode: 0o600 }); + } + await chmod(output, 0o700); + return { output, root }; +} + +async function release(output: string, target: string) { + return execFileAsync("bash", [ + "-c", + 'source "$1"; module_package_release "$2" "$3"', + "release", + helper, + output, + target, + ]); +} + +describe("the smoke test's package release pair", () => { + it("copies the archive and its digest, together, into a fresh world-traversable directory", async () => { + const { output, root } = await buildOutput(); + const target = path.join(root, "package"); + + const { stdout } = await release(output, target); + + expect(stdout.trim()).toBe(archive); + expect((await readdir(target)).sort()).toEqual([archive, `${archive}.sha256`]); + expect((await stat(target)).mode & 0o777).toBe(0o755); + for (const name of [archive, `${archive}.sha256`]) { + expect((await stat(path.join(target, name))).mode & 0o777, name).toBe(0o644); + expect(await readFile(path.join(target, name))).toEqual( + await readFile(path.join(output, name)), + ); + } + }); + + it.each([ + ["the digest is missing", { [archive]: "archive bytes" }], + [ + "the digest names another file", + { [archive]: "a", [`${archive}.sha256`]: `${digest} other.tar.gz\n` }, + ], + ["the digest is malformed", { [archive]: "a", [`${archive}.sha256`]: "not-a-digest x\n" }], + [ + "the build left something else beside the pair", + { + [archive]: "a", + [`${archive}.sha256`]: `${digest} ${archive}\n`, + "stale.tar.gz.sha256": "x", + }, + ], + [ + "there is more than one archive", + { + [archive]: "a", + [`${archive}.sha256`]: `${digest} ${archive}\n`, + "example-0.0.9-any.tar.gz": "old", + }, + ], + ["there is no archive at all", {}], + ])("refuses when %s", async (_label, files) => { + const { output, root } = await buildOutput(files); + const target = path.join(root, "package"); + await expect(release(output, target)).rejects.toThrow(); + await expect(stat(target)).rejects.toMatchObject({ code: "ENOENT" }); + }); + + it("never reuses a release directory that already exists", async () => { + const { output, root } = await buildOutput(); + const target = path.join(root, "package"); + await mkdir(target); + await writeFile(path.join(target, `${archive}.sha256`), `${"0".repeat(64)} ${archive}\n`); + + await expect(release(output, target)).rejects.toThrow(); + // The stale digest is left exactly as it was; nothing was mixed into it. + expect(await readdir(target)).toEqual([`${archive}.sha256`]); + }); +}); diff --git a/turbo.json b/turbo.json index 31b68ad..696c603 100644 --- a/turbo.json +++ b/turbo.json @@ -14,6 +14,10 @@ "dependsOn": ["^build"], "outputs": [] }, + "@ekklesia/module-pack#test": { + "dependsOn": ["^build", "@ekklesia/module-template#test"], + "outputs": [] + }, "typecheck": { "dependsOn": ["^build", "^typecheck"], "outputs": []