Zayin/Ekklesia
3
0
Fork 0
Code Issues 1 Pull Requests 3 Actions Packages Projects Releases 8 Wiki Activity

feat: Ekklesia Module Package v1, the canonical packager and verifier (ADR 0049) #66

Merged
Zayin_Admin merged 2 commits from feat/module-package-v1 into main 2026-10-07 18:33:28 +00:00
Conversation 1 Commits 2 Files Changed 47
+5083 -151
47 changed files with 5083 additions and 151 deletions
Download Patch File Download Diff File Expand all files Collapse all files
README.md
+6 -2
View File
@@ -44,8 +44,12 @@ selected by `EKKLESIA_MODULES_DIR` (`/var/lib/ekklesia/modules` in the image, th
self-contained directory carrying its own production dependencies. Installing, updating, or removing
a module needs only a controlled restart and, when it has migrations, an explicit plan and apply.
Nothing is hot loaded. Removing a module's code keeps its data and its stored state; its applied
migration history is retained and reported, and blocks no other migration. Purge, a package format,
signing, and any installer or marketplace do not exist yet.
migration history is retained and reported, and blocks no other migration. A module is distributed
as one **Ekklesia Module Package v1**
([ADR 0049](docs/architecture/decisions/0049-module-package-v1.md)): a deterministic `.tar.gz` and
its SHA-256, produced by the canonical packager (`@ekklesia/module-pack`) with the module's backend
bundled, and checked by Core's verifier before it is installed. Only the portable `any` target exists
so far. Purge, native-target packages, signing, and any installer or marketplace do not exist yet.
Phase 4A added Core-owned identity foundations: a Core-owned Better Auth database schema
(`ekklesia_core_user`, `ekklesia_core_session`, `ekklesia_core_account`,
core/src/cli/verify-module-package.ts
+175
View File
@@ -0,0 +1,175 @@
import { readFile } from "node:fs/promises";
import path from "node:path";
import { pathToFileURL } from "node:url";
import {
type ModulePackagePlatform,
ModulePackageVerificationError,
verifyModulePackage,
} from "../modules/index.js";
/**
* `verify-module-package` — verifies an Ekklesia Module Package v1 archive against this platform
* (ADR 0049), and never installs it.
*
* node core/dist/cli/verify-module-package.js <archive>
* [--sha256 <hex> | --sha256-file <file>] the archive's expected SHA-256
* [--extract-to <new directory>] write the verified module directory there
*
* Run from the platform's root (`/opt/ekklesia` in the image), like every operator command: the
* platform's own versions are read from there. `--extract-to` writes only after every check passed,
* into a directory that must not exist yet; it is the staging step of an installation, not the
* installation itself.
*/
class VerifyArgumentsError extends Error {}
const USAGE =
"Usage: verify-module-package <archive> [--sha256 <hex> | --sha256-file <file>] [--extract-to <new directory>]";
export interface VerifyModulePackageArguments {
readonly archivePath: string;
readonly expectedSha256?: string;
readonly sha256File?: string;
readonly extractTo?: string;
}
function normalizedDigest(value: string): string {
const digest = value.trim().toLowerCase();
if (!/^[0-9a-f]{64}$/u.test(digest)) {
throw new VerifyArgumentsError("The SHA-256 must be 64 hexadecimal characters.");
}
return digest;
}
export function parseVerifyModulePackageArguments(
argv: readonly string[],
): VerifyModulePackageArguments {
const positional: string[] = [];
const options = new Map<string, string>();
for (let index = 0; index < argv.length; index += 1) {
const argument = argv[index] ?? "";
if (argument.startsWith("--")) {
if (!["--sha256", "--sha256-file", "--extract-to"].includes(argument)) {
throw new VerifyArgumentsError(USAGE);
}
const value = argv[index + 1];
if (value === undefined || value.startsWith("--") || options.has(argument)) {
throw new VerifyArgumentsError(USAGE);
}
options.set(argument, value);
index += 1;
} else {
positional.push(argument);
}
}
const archivePath = positional[0];
if (positional.length !== 1 || archivePath === undefined || archivePath === "") {
throw new VerifyArgumentsError(USAGE);
}
if (options.has("--sha256") && options.has("--sha256-file")) {
throw new VerifyArgumentsError(USAGE);
}
const sha256 = options.get("--sha256");
const sha256File = options.get("--sha256-file");
const extractTo = options.get("--extract-to");
return {
archivePath,
...(sha256 === undefined ? {} : { expectedSha256: normalizedDigest(sha256) }),
...(sha256File === undefined ? {} : { sha256File }),
...(extractTo === undefined ? {} : { extractTo }),
};
}
/** The `sha256sum` format: one line, the digest, whitespace, and a file name that is never trusted. */
export function parseSha256File(contents: string): string {
const lines = contents.split("\n").filter((line) => line.trim() !== "");
const match = lines.length === 1 ? /^([0-9a-fA-F]{64})\s+\*?\S/u.exec(lines[0] ?? "") : null;
if (!match?.[1]) {
throw new VerifyArgumentsError("The SHA-256 file must hold one sha256sum line.");
}
return match[1].toLowerCase();
}
async function readJson(file: string): Promise<Record<string, unknown>> {
return JSON.parse(await readFile(file, "utf8")) as Record<string, unknown>;
}
function requiredString(value: unknown): string {
if (typeof value !== "string" || value === "") {
throw new VerifyArgumentsError("The platform's own package metadata could not be read.");
}
return value;
}
/** What this platform is, read from where it is installed. The transitional database runtime is
* the Drizzle version Core itself depends on (ADR 0049). */
export async function readModulePackagePlatform(
repositoryRoot: string,
): Promise<ModulePackagePlatform> {
const [root, sdk, core] = await Promise.all([
readJson(path.join(repositoryRoot, "package.json")),
readJson(path.join(repositoryRoot, "packages", "module-sdk", "package.json")),
readJson(path.join(repositoryRoot, "core", "package.json")),
]);
const dependencies = core.dependencies as Record<string, unknown> | undefined;
return {
databaseRuntime: { "drizzle-orm": requiredString(dependencies?.["drizzle-orm"]) },
moduleSdkVersion: requiredString(sdk.version),
platformVersion: requiredString(root.version),
};
}
async function main(): Promise<void> {
const parsed = parseVerifyModulePackageArguments(process.argv.slice(2));
let expectedSha256 = parsed.expectedSha256;
if (parsed.sha256File !== undefined) {
let contents: string;
try {
contents = await readFile(parsed.sha256File, "utf8");
} catch {
throw new VerifyArgumentsError("The SHA-256 file could not be read.");
}
expectedSha256 = parseSha256File(contents);
}
const platform = await readModulePackagePlatform(process.cwd()).catch(() => {
throw new VerifyArgumentsError("The platform's own package metadata could not be read.");
});
const result = await verifyModulePackage({
archivePath: parsed.archivePath,
platform,
...(expectedSha256 === undefined ? {} : { expectedSha256 }),
...(parsed.extractTo === undefined ? {} : { extractTo: parsed.extractTo }),
});
console.log(
`Module package verified: ${result.moduleId}@${result.version} (target ${result.target}), ${result.fileCount} files, sha256 ${result.archiveSha256}.`,
);
if (expectedSha256 === undefined) {
console.log("The archive's SHA-256 was not checked: no expected digest was given.");
}
if (parsed.extractTo !== undefined) {
console.log("The verified module directory was extracted. Nothing was installed.");
}
}
function isDirectExecution(): boolean {
const entrypoint = process.argv[1];
return (
entrypoint !== undefined && import.meta.url === pathToFileURL(path.resolve(entrypoint)).href
);
}
if (isDirectExecution()) {
main().catch((error: unknown) => {
if (error instanceof ModulePackageVerificationError) {
console.error(`[${error.code}] ${error.message}`);
if (error.issueCodes.length > 0) {
console.error(`Module validation issues: ${error.issueCodes.join(", ")}`);
}
} else if (error instanceof VerifyArgumentsError) {
console.error(error.message);
} else {
console.error("Module package verification failed unexpectedly.");
}
process.exitCode = 1;
});
}
core/src/modules/index.ts
+1
View File
@@ -8,6 +8,7 @@ export * from "./lifecycle-schema.js";
export * from "./lifecycle-store.js";
export * from "./migration-discovery.js";
export * from "./modules-directory.js";
export * from "./module-package-verification.js";
export * from "./permissions.js";
export * from "./realtime.js";
export * from "./runtime.js";
core/src/modules/module-package-archive.ts
+203
View File
@@ -0,0 +1,203 @@
import { gunzipSync } from "node:zlib";
/**
* Reads an Ekklesia Module Package v1 archive (ADR 0049) entirely in memory and enforces the
* archive-safety rules before anything is written anywhere.
*
* Deliberately a strict reader rather than a general tar implementation: v1 packages hold only
* POSIX ustar directories and regular files, so every other entry type is refused instead of
* interpreted. Nothing here touches the filesystem; extraction happens only after the caller has
* verified the whole package.
*/
export type ModulePackageArchiveErrorCode =
| "MODULE_PACKAGE_LAYOUT_INVALID"
| "MODULE_PACKAGE_LINK"
| "MODULE_PACKAGE_SPECIAL_ENTRY"
| "MODULE_PACKAGE_TOO_LARGE"
| "MODULE_PACKAGE_UNREADABLE"
| "MODULE_PACKAGE_UNSAFE_MODE"
| "MODULE_PACKAGE_UNSAFE_PATH";
export class ModulePackageArchiveError extends Error {
readonly code: ModulePackageArchiveErrorCode;
constructor(code: ModulePackageArchiveErrorCode) {
super(code);
this.name = "ModulePackageArchiveError";
this.code = code;
}
}
/** Generous for a module, small enough that a hostile archive cannot exhaust the host. */
export const MODULE_PACKAGE_MAX_COMPRESSED_BYTES = 256 * 1024 * 1024;
export const MODULE_PACKAGE_MAX_UNCOMPRESSED_BYTES = 1024 * 1024 * 1024;
export const MODULE_PACKAGE_MAX_ENTRIES = 100_000;
export interface ModulePackageArchiveEntry {
/** Relative to the top-level module directory; empty for that directory itself. */
readonly path: string;
readonly type: "directory" | "file";
readonly data: Buffer;
}
export interface ModulePackageArchive {
/** The single top-level directory's name. Validated as a module id by the caller. */
readonly topLevel: string;
/** Every entry except the top-level directory itself, in archive order. */
readonly entries: readonly ModulePackageArchiveEntry[];
}
const BLOCK = 512;
function field(block: Buffer, offset: number, length: number): string {
const raw = block.subarray(offset, offset + length);
const end = raw.indexOf(0);
return raw.subarray(0, end === -1 ? length : end).toString("latin1");
}
function octalField(block: Buffer, offset: number, length: number): number {
const text = field(block, offset, length).trim();
if (!/^[0-7]+$/u.test(text)) {
throw new ModulePackageArchiveError("MODULE_PACKAGE_UNREADABLE");
}
const value = Number.parseInt(text, 8);
if (!Number.isSafeInteger(value)) {
throw new ModulePackageArchiveError("MODULE_PACKAGE_UNREADABLE");
}
return value;
}
function headerChecksumValid(block: Buffer): boolean {
const declared = octalField(block, 148, 8);
let sum = 0;
for (let index = 0; index < BLOCK; index += 1) {
sum += index >= 148 && index < 156 ? 0x20 : (block[index] ?? 0);
}
return sum === declared;
}
/** Printable ASCII, no backslash, relative, and no empty, `.`, or `..` segment. */
function safeSegments(value: string): readonly string[] {
if (
value.length === 0 ||
!/^[\x20-\x7e]+$/u.test(value) ||
value.includes("\\") ||
value.startsWith("/")
) {
throw new ModulePackageArchiveError("MODULE_PACKAGE_UNSAFE_PATH");
}
const segments = value.split("/");
if (segments.some((segment) => segment === "" || segment === "." || segment === "..")) {
throw new ModulePackageArchiveError("MODULE_PACKAGE_UNSAFE_PATH");
}
return segments;
}
export function readModulePackageArchive(compressed: Buffer): ModulePackageArchive {
if (compressed.length > MODULE_PACKAGE_MAX_COMPRESSED_BYTES) {
throw new ModulePackageArchiveError("MODULE_PACKAGE_TOO_LARGE");
}
let raw: Buffer;
try {
raw = gunzipSync(compressed, { maxOutputLength: MODULE_PACKAGE_MAX_UNCOMPRESSED_BYTES });
} catch (error) {
throw new ModulePackageArchiveError(
error instanceof RangeError ? "MODULE_PACKAGE_TOO_LARGE" : "MODULE_PACKAGE_UNREADABLE",
);
}
const entries: ModulePackageArchiveEntry[] = [];
const kinds = new Map<string, "directory" | "file">();
/** Every path that is an ancestor of some entry, so a file can never claim one. */
const ancestors = new Set<string>();
let topLevel: string | undefined;
let offset = 0;
let ended = false;
while (offset + BLOCK <= raw.length) {
const block = raw.subarray(offset, offset + BLOCK);
if (block.every((byte) => byte === 0)) {
ended = true;
break;
}
if (field(block, 257, 6) !== "ustar" || field(block, 263, 2) !== "00") {
throw new ModulePackageArchiveError("MODULE_PACKAGE_UNREADABLE");
}
if (!headerChecksumValid(block)) {
throw new ModulePackageArchiveError("MODULE_PACKAGE_UNREADABLE");
}
if (entries.length + 1 > MODULE_PACKAGE_MAX_ENTRIES) {
throw new ModulePackageArchiveError("MODULE_PACKAGE_TOO_LARGE");
}
const typeflag = field(block, 156, 1);
if (typeflag === "1" || typeflag === "2") {
throw new ModulePackageArchiveError("MODULE_PACKAGE_LINK");
}
const type =
typeflag === "5" ? "directory" : typeflag === "0" || typeflag === "" ? "file" : undefined;
if (type === undefined) {
// Devices, FIFOs, contiguous files, and every extended or long-name header: v1 packages hold
// directories and regular files, so nothing else is interpreted.
throw new ModulePackageArchiveError("MODULE_PACKAGE_SPECIAL_ENTRY");
}
const mode = octalField(block, 100, 8);
if ((mode & 0o7000) !== 0) {
throw new ModulePackageArchiveError("MODULE_PACKAGE_UNSAFE_MODE");
}
const name = field(block, 0, 100);
const prefix = field(block, 345, 155);
let fullPath = prefix === "" ? name : `${prefix}/${name}`;
if (type === "directory" && fullPath.endsWith("/")) {
fullPath = fullPath.slice(0, -1);
}
const segments = safeSegments(fullPath);
const top = segments[0] ?? "";
if (topLevel === undefined) {
topLevel = top;
} else if (top !== topLevel) {
throw new ModulePackageArchiveError("MODULE_PACKAGE_LAYOUT_INVALID");
}
const size = type === "file" ? octalField(block, 124, 12) : 0;
const dataStart = offset + BLOCK;
const dataEnd = dataStart + size;
if (dataEnd > raw.length) {
throw new ModulePackageArchiveError("MODULE_PACKAGE_UNREADABLE");
}
const relative = segments.slice(1).join("/");
if (relative === "") {
if (type !== "directory") {
throw new ModulePackageArchiveError("MODULE_PACKAGE_LAYOUT_INVALID");
}
} else {
if (kinds.has(relative)) {
throw new ModulePackageArchiveError("MODULE_PACKAGE_LAYOUT_INVALID");
}
// No entry may sit below a file, and no file may claim a path something already sits below.
// Linear in the total number of path segments, whatever order the entries arrive in.
if (type === "file" && ancestors.has(relative)) {
throw new ModulePackageArchiveError("MODULE_PACKAGE_LAYOUT_INVALID");
}
for (let index = 1; index < segments.length - 1; index += 1) {
const ancestor = segments.slice(1, index + 1).join("/");
if (kinds.get(ancestor) === "file") {
throw new ModulePackageArchiveError("MODULE_PACKAGE_LAYOUT_INVALID");
}
ancestors.add(ancestor);
}
kinds.set(relative, type);
entries.push({ data: Buffer.from(raw.subarray(dataStart, dataEnd)), path: relative, type });
}
offset = dataStart + Math.ceil(size / BLOCK) * BLOCK;
}
if (!ended || topLevel === undefined) {
throw new ModulePackageArchiveError("MODULE_PACKAGE_UNREADABLE");
}
return Object.freeze({ entries: Object.freeze(entries), topLevel });
}
core/src/modules/module-package-probe.ts
+89
View File
@@ -0,0 +1,89 @@
// The isolated, short-lived validation process of Module Package v1 verification (ADR 0049).
//
// The verifier forks this file with an empty environment and one JSON argument naming a candidate
// module's backend entrypoints. It imports each one, so a package whose code cannot load, or whose
// bundle still reaches for a dependency it does not contain, is refused before it is ever installed.
// Whatever the module's top-level code does happens here, in a process with no database URL, no
// secret, and no platform state, and that is discarded as soon as it reports.
//
// The result travels over the IPC channel, never stdout, so nothing a module prints can forge it.
// It carries fixed role and reason names only: never an error message, a path, or module output.
//
// Written as erasable TypeScript with no relative imports, so Node can run the source file directly
// in tests as well as the compiled file in the image.
import { pathToFileURL } from "node:url";
import { parseModuleRuntimeDefinition, parseModuleSchemaDefinition } from "@ekklesia/contracts";
interface ProbeRequest {
readonly moduleId: string;
readonly version: string;
readonly entrypoints: Readonly<Record<string, string>>;
}
interface ProbeFailure {
readonly role: string;
readonly reason: "export" | "identity" | "load";
}
function defaultExport(namespace: unknown): unknown {
return typeof namespace === "object" && namespace !== null && "default" in namespace
? (namespace as { default: unknown }).default
: undefined;
}
async function probe(request: ProbeRequest): Promise<readonly ProbeFailure[]> {
const failures: ProbeFailure[] = [];
for (const role of Object.keys(request.entrypoints).sort()) {
const entrypoint = request.entrypoints[role];
if (entrypoint === undefined) {
continue;
}
let namespace: unknown;
try {
namespace = await import(pathToFileURL(entrypoint).href);
} catch {
failures.push({ reason: "load", role });
continue;
}
try {
if (role === "server") {
const definition = parseModuleRuntimeDefinition(defaultExport(namespace));
if (definition.id !== request.moduleId || definition.version !== request.version) {
failures.push({ reason: "identity", role });
}
} else if (role === "schema") {
const definition = parseModuleSchemaDefinition(defaultExport(namespace));
if (definition.moduleId !== request.moduleId) {
failures.push({ reason: "identity", role });
}
}
} catch {
failures.push({ reason: "export", role });
}
}
return failures;
}
function report(message: unknown): void {
if (typeof process.send !== "function") {
process.exitCode = 2;
return;
}
process.send(message, () => {
// A module may leave timers or handles behind; the probe's work is done either way.
process.exit(0);
});
}
let request: ProbeRequest | undefined;
try {
request = JSON.parse(process.argv[2] ?? "") as ProbeRequest;
} catch {
report({ failures: [{ reason: "load", role: "request" }] });
}
if (request !== undefined) {
probe(request).then(
(failures) => report({ failures }),
() => report({ failures: [{ reason: "load", role: "probe" }] }),
);
}
core/src/modules/module-package-verification.ts
+479
View File
@@ -0,0 +1,479 @@
import { fork } from "node:child_process";
import { createHash } from "node:crypto";
import { chmod, mkdir, mkdtemp, readFile, rm, writeFile } from "node:fs/promises";
import { tmpdir } from "node:os";
import path from "node:path";
import { fileURLToPath } from "node:url";
import {
isVersionCompatible,
MODULE_PACKAGE_ANY_TARGET,
MODULE_PACKAGE_FORMAT,
MODULE_PACKAGE_METADATA_FILENAME,
type ModuleManifest,
type ModulePackageMetadata,
safeParseModuleManifest,
safeParseModulePackageMetadata,
safeParseModulePackageRuntimePackage,
} from "@ekklesia/contracts";
import { discoverModules, MODULE_MANIFEST_FILENAME, ModuleDiscoveryError } from "./discovery.js";
import { discoverModuleMigrations, ModuleMigrationDiscoveryError } from "./migration-discovery.js";
import {
type ModulePackageArchive,
ModulePackageArchiveError,
type ModulePackageArchiveErrorCode,
readModulePackageArchive,
} from "./module-package-archive.js";
import { discoverModuleSchemas, ModuleSchemaDiscoveryError } from "./schema-discovery.js";
/**
* Verifies an Ekklesia Module Package v1 archive (ADR 0049) against the platform it runs on, without
* ever installing it.
*
* Order matters: everything that can be decided from bytes is decided before anything is written,
* and nothing is executed until the package has proven itself well-formed. The archive is read and
* checked for safety in memory; its metadata, every payload digest, and every identity statement
* are checked next; only then is it written to a fresh temporary directory, validated by the same
* discovery the platform uses, and its backend loaded once in a separate, short-lived process with
* an empty environment. A caller that asks for extraction gets the verified tree written to a new
* directory only after all of that succeeded.
*
* Errors carry a stable code and a fixed message. Paths, archive entry names, file contents, module
* output, and exception messages never appear in them: an archive is untrusted input, and none of
* it is ever echoed.
*/
export type ModulePackageVerificationErrorCode =
| ModulePackageArchiveErrorCode
| "MODULE_PACKAGE_DATABASE_RUNTIME_INCOMPATIBLE"
| "MODULE_PACKAGE_DIGEST_MISMATCH"
| "MODULE_PACKAGE_ENTRYPOINT_LOAD_FAILED"
| "MODULE_PACKAGE_EXTRACTION_FAILED"
| "MODULE_PACKAGE_FILE_MISMATCH"
| "MODULE_PACKAGE_FILE_MISSING"
| "MODULE_PACKAGE_FILE_UNLISTED"
| "MODULE_PACKAGE_FORMAT_UNSUPPORTED"
| "MODULE_PACKAGE_IDENTITY_MISMATCH"
| "MODULE_PACKAGE_MANIFEST_INVALID"
| "MODULE_PACKAGE_METADATA_INVALID"
| "MODULE_PACKAGE_METADATA_MISSING"
| "MODULE_PACKAGE_MODULE_INVALID"
| "MODULE_PACKAGE_NATIVE_FILE"
| "MODULE_PACKAGE_RUNTIME_INCOMPATIBLE"
| "MODULE_PACKAGE_RUNTIME_PACKAGE_INVALID"
| "MODULE_PACKAGE_SDK_INCOMPATIBLE"
| "MODULE_PACKAGE_TARGET_INCOMPATIBLE"
| "MODULE_PACKAGE_TRANSITIONAL_UNSUPPORTED";
const messages: Readonly<Record<ModulePackageVerificationErrorCode, string>> = Object.freeze({
MODULE_PACKAGE_DATABASE_RUNTIME_INCOMPATIBLE:
"The package's bundled database runtime does not match this platform's (a transitional constraint).",
MODULE_PACKAGE_DIGEST_MISMATCH: "The archive does not match the SHA-256 it was given.",
MODULE_PACKAGE_ENTRYPOINT_LOAD_FAILED:
"A backend entrypoint could not be loaded from the package alone.",
MODULE_PACKAGE_EXTRACTION_FAILED:
"The verified package could not be extracted to a new directory.",
MODULE_PACKAGE_FILE_MISMATCH: "A payload file does not match its listed size or SHA-256.",
MODULE_PACKAGE_FILE_MISSING: "A file listed in the package metadata is missing from the archive.",
MODULE_PACKAGE_FILE_UNLISTED: "The archive contains a file the package metadata does not list.",
MODULE_PACKAGE_FORMAT_UNSUPPORTED: "The package format is not one this platform implements.",
MODULE_PACKAGE_IDENTITY_MISMATCH:
"The package metadata, runtime package.json, and manifest disagree about the module's identity.",
MODULE_PACKAGE_LAYOUT_INVALID:
"The archive must contain exactly one top-level module directory, with each path once.",
MODULE_PACKAGE_LINK: "The archive contains a link; module packages may contain none.",
MODULE_PACKAGE_MANIFEST_INVALID: "The package's module manifest is missing or invalid.",
MODULE_PACKAGE_METADATA_INVALID: "The package metadata is invalid.",
MODULE_PACKAGE_METADATA_MISSING: "The archive contains no package metadata.",
MODULE_PACKAGE_MODULE_INVALID: "The packaged module failed module validation.",
MODULE_PACKAGE_NATIVE_FILE: "An any-target package contains a native file.",
MODULE_PACKAGE_RUNTIME_INCOMPATIBLE:
"The package needs a Node.js version this host does not run.",
MODULE_PACKAGE_RUNTIME_PACKAGE_INVALID:
"The package's runtime package.json is not the generated runtime shape.",
MODULE_PACKAGE_SDK_INCOMPATIBLE:
"The Module SDK bundled into the package is outside the manifest's own moduleSdk range.",
MODULE_PACKAGE_SPECIAL_ENTRY:
"The archive contains an entry that is neither a directory nor a regular file.",
MODULE_PACKAGE_TARGET_INCOMPATIBLE: "The package's target is not one this platform can run.",
MODULE_PACKAGE_TOO_LARGE: "The archive exceeds the module package size limits.",
MODULE_PACKAGE_TRANSITIONAL_UNSUPPORTED:
"The package metadata contains transitional facts this platform does not understand.",
MODULE_PACKAGE_UNREADABLE: "The file is not a readable gzip-compressed ustar archive.",
MODULE_PACKAGE_UNSAFE_MODE: "The archive contains a setuid, setgid, or sticky entry.",
MODULE_PACKAGE_UNSAFE_PATH: "The archive contains an absolute, escaping, or non-portable path.",
});
export class ModulePackageVerificationError extends Error {
readonly code: ModulePackageVerificationErrorCode;
/** Stable discovery issue codes, when module validation failed; never their messages. */
readonly issueCodes: readonly string[];
constructor(code: ModulePackageVerificationErrorCode, issueCodes: readonly string[] = []) {
super(messages[code]);
this.name = "ModulePackageVerificationError";
this.code = code;
this.issueCodes = Object.freeze([...new Set(issueCodes)].sort());
this.stack = `${this.name}: ${this.message}`;
Object.freeze(this);
}
}
/** What this platform is, for the checks that compare a package against it. */
export interface ModulePackagePlatform {
readonly platformVersion: string;
readonly moduleSdkVersion: string;
/** The transitional database runtime facts this platform requires a package to agree with. */
readonly databaseRuntime: Readonly<Record<string, string>>;
/** Defaults to the running process's Node.js version. */
readonly nodeVersion?: string;
}
export interface VerifyModulePackageOptions {
readonly archivePath: string;
/** When given, the archive's SHA-256 must equal it (lowercase hexadecimal). */
readonly expectedSha256?: string;
readonly platform: ModulePackagePlatform;
/** A directory that must not exist yet; the verified module directory is written there. */
readonly extractTo?: string;
readonly loadTimeoutMs?: number;
}
export interface ModulePackageVerificationResult {
readonly archiveSha256: string;
/** Payload files, the metadata file not included. */
readonly fileCount: number;
readonly moduleId: string;
readonly target: string;
readonly version: string;
}
function fail(code: ModulePackageVerificationErrorCode, issueCodes?: readonly string[]): never {
throw new ModulePackageVerificationError(code, issueCodes);
}
function sha256(data: Uint8Array): string {
return createHash("sha256").update(data).digest("hex");
}
function isRecord(value: unknown): value is Record<string, unknown> {
return typeof value === "object" && value !== null && !Array.isArray(value);
}
function parseJson(data: Buffer | undefined): unknown {
if (data === undefined) {
return undefined;
}
try {
return JSON.parse(data.toString("utf8")) as unknown;
} catch {
return undefined;
}
}
/** The transitional facts this verifier understands, and nothing else (ADR 0049). */
const UNDERSTOOD_TRANSITIONAL: Readonly<Record<string, readonly string[]>> = Object.freeze({
databaseRuntime: Object.freeze(["drizzle-orm"]),
});
function checkTransitional(raw: Record<string, unknown>): void {
const transitional = raw.transitional;
if (transitional === undefined) {
return;
}
if (!isRecord(transitional)) {
fail("MODULE_PACKAGE_TRANSITIONAL_UNSUPPORTED");
}
for (const [key, value] of Object.entries(transitional)) {
const understood = UNDERSTOOD_TRANSITIONAL[key];
if (!understood || !isRecord(value)) {
fail("MODULE_PACKAGE_TRANSITIONAL_UNSUPPORTED");
}
if (Object.keys(value).some((name) => !understood.includes(name))) {
fail("MODULE_PACKAGE_TRANSITIONAL_UNSUPPORTED");
}
}
}
function readMetadata(files: ReadonlyMap<string, Buffer>): ModulePackageMetadata {
const raw = parseJson(files.get(MODULE_PACKAGE_METADATA_FILENAME));
if (!files.has(MODULE_PACKAGE_METADATA_FILENAME)) {
fail("MODULE_PACKAGE_METADATA_MISSING");
}
if (!isRecord(raw)) {
fail("MODULE_PACKAGE_METADATA_INVALID");
}
if (raw.packageFormat !== MODULE_PACKAGE_FORMAT) {
fail("MODULE_PACKAGE_FORMAT_UNSUPPORTED");
}
checkTransitional(raw);
const parsed = safeParseModulePackageMetadata(raw);
if (!parsed.success) {
fail("MODULE_PACKAGE_METADATA_INVALID");
}
return parsed.data;
}
function checkPayload(metadata: ModulePackageMetadata, files: ReadonlyMap<string, Buffer>): void {
const listed = new Set(metadata.files.map((file) => file.path));
for (const filePath of files.keys()) {
if (filePath !== MODULE_PACKAGE_METADATA_FILENAME && !listed.has(filePath)) {
fail("MODULE_PACKAGE_FILE_UNLISTED");
}
}
for (const file of metadata.files) {
const data = files.get(file.path);
if (data === undefined) {
fail("MODULE_PACKAGE_FILE_MISSING");
}
if (data.length !== file.size || sha256(data) !== file.sha256) {
fail("MODULE_PACKAGE_FILE_MISMATCH");
}
}
}
function checkIdentity(
archive: ModulePackageArchive,
metadata: ModulePackageMetadata,
files: ReadonlyMap<string, Buffer>,
): ModuleManifest {
const runtimePackage = safeParseModulePackageRuntimePackage(parseJson(files.get("package.json")));
if (!runtimePackage.success) {
fail("MODULE_PACKAGE_RUNTIME_PACKAGE_INVALID");
}
const manifest = safeParseModuleManifest(parseJson(files.get(MODULE_MANIFEST_FILENAME)));
if (!manifest.success) {
fail("MODULE_PACKAGE_MANIFEST_INVALID");
}
if (
archive.topLevel !== metadata.module.id ||
manifest.data.id !== metadata.module.id ||
manifest.data.version !== metadata.module.version ||
runtimePackage.data.version !== metadata.module.version
) {
fail("MODULE_PACKAGE_IDENTITY_MISMATCH");
}
return manifest.data;
}
function checkCompatibility(
metadata: ModulePackageMetadata,
manifest: ModuleManifest,
files: ReadonlyMap<string, Buffer>,
platform: ModulePackagePlatform,
): void {
// This verifier implements the portable target only. Native targets are a later slice.
if (metadata.target !== MODULE_PACKAGE_ANY_TARGET) {
fail("MODULE_PACKAGE_TARGET_INCOMPATIBLE");
}
if ([...files.keys()].some((filePath) => filePath.toLowerCase().endsWith(".node"))) {
fail("MODULE_PACKAGE_NATIVE_FILE");
}
if (!isVersionCompatible(platform.nodeVersion ?? process.versions.node, metadata.runtime.node)) {
fail("MODULE_PACKAGE_RUNTIME_INCOMPATIBLE");
}
const sdk = metadata.incorporates["@ekklesia/module-sdk"];
if (sdk !== undefined && !isVersionCompatible(sdk, manifest.compatibility.moduleSdk)) {
fail("MODULE_PACKAGE_SDK_INCOMPATIBLE");
}
// Transitional (ADR 0049): today a module's Drizzle copy must be interchangeable with the
// platform's, because Core queries the module's tables with its own Drizzle instance. This is an
// implementation constraint pending a Module Database ABI, not that ABI.
for (const [runtime, version] of Object.entries(metadata.transitional?.databaseRuntime ?? {})) {
if (platform.databaseRuntime[runtime] !== version) {
fail("MODULE_PACKAGE_DATABASE_RUNTIME_INCOMPATIBLE");
}
}
}
/** Writes the archive's directories (`0755`) and files (`0644`) under `root`, which must not exist.
* Safe by construction: every path was validated as relative and non-escaping, and only directories
* and regular files exist. Ownership is the writing process's, never the archive's. */
async function writeTree(root: string, archive: ModulePackageArchive): Promise<void> {
await mkdir(root, { mode: 0o755 });
await chmod(root, 0o755);
for (const entry of archive.entries) {
const target = path.join(root, ...entry.path.split("/"));
if (entry.type === "directory") {
await mkdir(target, { mode: 0o755, recursive: true });
await chmod(target, 0o755);
} else {
await mkdir(path.dirname(target), { mode: 0o755, recursive: true });
await writeFile(target, entry.data, { flag: "wx", mode: 0o644 });
await chmod(target, 0o644);
}
}
}
function probeEntrypointPath(): string {
// The probe sits beside this file: compiled `.js` in an installation, source `.ts` under a test
// runner, which Node runs directly. Either way it is a separate process.
const self = fileURLToPath(import.meta.url);
return path.join(path.dirname(self), `module-package-probe${path.extname(self)}`);
}
/** Loads every backend entrypoint in a short-lived child with an empty environment, and reports
* only whether each loaded and exported a definition naming this module. */
async function loadEntrypoints(
moduleId: string,
version: string,
entrypoints: Readonly<Record<string, string>>,
timeoutMs: number,
): Promise<void> {
const outcome = await new Promise<"ok" | "failed">((resolve) => {
let settled = false;
let child: ReturnType<typeof fork> | undefined;
let timer: ReturnType<typeof setTimeout> | undefined;
const finish = (value: "ok" | "failed"): void => {
if (!settled) {
settled = true;
clearTimeout(timer);
child?.kill("SIGKILL");
resolve(value);
}
};
try {
child = fork(probeEntrypointPath(), [JSON.stringify({ entrypoints, moduleId, version })], {
cwd: tmpdir(),
env: {},
execArgv: [],
serialization: "json",
stdio: ["ignore", "ignore", "ignore", "ipc"],
});
} catch {
finish("failed");
return;
}
timer = setTimeout(() => finish("failed"), timeoutMs);
child.on("message", (message: unknown) => {
const failures = isRecord(message) ? message.failures : undefined;
finish(Array.isArray(failures) && failures.length === 0 ? "ok" : "failed");
});
child.on("error", () => finish("failed"));
child.on("exit", () => finish("failed"));
});
if (outcome !== "ok") {
fail("MODULE_PACKAGE_ENTRYPOINT_LOAD_FAILED");
}
}
function discoveryIssueCodes(error: unknown): readonly string[] {
if (
error instanceof ModuleDiscoveryError ||
error instanceof ModuleMigrationDiscoveryError ||
error instanceof ModuleSchemaDiscoveryError
) {
return error.issues.map((issue) => issue.code);
}
return [];
}
export async function verifyModulePackage(
options: VerifyModulePackageOptions,
): Promise<ModulePackageVerificationResult> {
let compressed: Buffer;
try {
compressed = await readFile(options.archivePath);
} catch {
fail("MODULE_PACKAGE_UNREADABLE");
}
const archiveSha256 = sha256(compressed);
if (options.expectedSha256 !== undefined && options.expectedSha256 !== archiveSha256) {
fail("MODULE_PACKAGE_DIGEST_MISMATCH");
}
let archive: ModulePackageArchive;
try {
archive = readModulePackageArchive(compressed);
} catch (error) {
if (error instanceof ModulePackageArchiveError) {
fail(error.code);
}
fail("MODULE_PACKAGE_UNREADABLE");
}
const files = new Map(
archive.entries
.filter((entry) => entry.type === "file")
.map((entry) => [entry.path, entry.data] as const),
);
const metadata = readMetadata(files);
checkPayload(metadata, files);
const manifest = checkIdentity(archive, metadata, files);
checkCompatibility(metadata, manifest, files, options.platform);
const scratch = await mkdtemp(path.join(tmpdir(), "ekklesia-package-verify-"));
try {
// The candidate alone, in a scratch inventory of its own: never module storage.
const modulesDirectory = path.join(scratch, "candidates");
await mkdir(modulesDirectory, { mode: 0o755 });
await writeTree(path.join(modulesDirectory, metadata.module.id), archive);
const discoveryOptions = {
moduleSdkVersion: options.platform.moduleSdkVersion,
modulesDirectory,
modulesDirectoryRequired: true,
platformVersion: options.platform.platformVersion,
} as const;
let entrypoints: Record<string, string>;
try {
// The same static validation the platform runs, against this package alone. A module that
// requires another module is validated with its dependencies when it is installed.
const [discovered] = await discoverModules(discoveryOptions);
if (discovered === undefined) {
fail("MODULE_PACKAGE_MODULE_INVALID");
}
await discoverModuleMigrations(discoveryOptions);
entrypoints = {
server: discovered.entrypoints.server,
...(discovered.entrypoints.schema ? { schema: discovered.entrypoints.schema } : {}),
...(discovered.entrypoints.jobs ? { jobs: discovered.entrypoints.jobs } : {}),
};
} catch (error) {
if (error instanceof ModulePackageVerificationError) {
throw error;
}
fail("MODULE_PACKAGE_MODULE_INVALID", discoveryIssueCodes(error));
}
// Executed only now, and first in isolation: a schema is imported in this process below only
// after the probe proved it loads on its own.
await loadEntrypoints(
metadata.module.id,
metadata.module.version,
entrypoints,
options.loadTimeoutMs ?? 30_000,
);
try {
await discoverModuleSchemas(discoveryOptions);
} catch (error) {
fail("MODULE_PACKAGE_MODULE_INVALID", discoveryIssueCodes(error));
}
if (options.extractTo !== undefined) {
const extractTo = options.extractTo;
let created = false;
try {
await mkdir(extractTo, { mode: 0o755 });
created = true;
await rm(extractTo, { recursive: true });
await writeTree(extractTo, archive);
} catch {
// Never leave a partial module directory behind; never remove one this call did not create.
if (created) {
await rm(extractTo, { force: true, recursive: true }).catch(() => undefined);
}
fail("MODULE_PACKAGE_EXTRACTION_FAILED");
}
}
} finally {
await rm(scratch, { force: true, recursive: true });
}
return Object.freeze({
archiveSha256,
fileCount: metadata.files.length,
moduleId: metadata.module.id,
target: metadata.target,
version: metadata.module.version,
});
}
core/tests/module-package-verification.test.ts
+387
View File
@@ -0,0 +1,387 @@
import { mkdtemp, readdir, rm, stat, writeFile } from "node:fs/promises";
import { tmpdir } from "node:os";
import path from "node:path";
import { afterEach, describe, expect, it } from "vitest";
import {
type ModulePackageVerificationErrorCode,
ModulePackageVerificationError,
verifyModulePackage,
} from "../src/modules/index.js";
import {
type FixtureEntry,
modulePackageFixture,
type PackageFixtureOptions,
sha256,
} from "./support/module-package-fixture.js";
const platform = {
databaseRuntime: { "drizzle-orm": "0.45.2" },
moduleSdkVersion: "0.1.2",
platformVersion: "0.1.2",
} as const;
const temporaryDirectories: string[] = [];
afterEach(async () => {
await Promise.all(
temporaryDirectories
.splice(0)
.map((directory) => rm(directory, { force: true, recursive: true })),
);
});
async function scratch(): Promise<string> {
const directory = await mkdtemp(path.join(tmpdir(), "ekklesia-verify-"));
temporaryDirectories.push(directory);
return directory;
}
async function archiveFile(options: PackageFixtureOptions = {}): Promise<{
readonly archivePath: string;
readonly sha256: string;
}> {
const fixture = await modulePackageFixture(options);
const archivePath = path.join(await scratch(), "private-host-name-5e1d.tar.gz");
await writeFile(archivePath, fixture.archive);
return { archivePath, sha256: fixture.sha256 };
}
async function rejection(
options: PackageFixtureOptions,
code: ModulePackageVerificationErrorCode,
extra: { readonly expectedSha256?: string } = {},
): Promise<ModulePackageVerificationError> {
const { archivePath } = await archiveFile(options);
try {
await verifyModulePackage({ archivePath, platform, ...extra });
} catch (error) {
expect(error).toBeInstanceOf(ModulePackageVerificationError);
const verificationError = error as ModulePackageVerificationError;
expect(verificationError.code).toBe(code);
// Sanitized: never the archive's location, a temporary directory, or text from inside it.
const rendered = JSON.stringify({ ...verificationError, message: verificationError.message });
expect(rendered).not.toContain("private-host-name-5e1d");
expect(rendered).not.toContain(tmpdir());
expect(rendered).not.toContain("attacker-chosen");
return verificationError;
}
throw new Error(`Expected ${code}.`);
}
function withEntry(entry: FixtureEntry): PackageFixtureOptions {
return { extraEntries: [entry] };
}
describe("Module Package v1 verification (ADR 0049)", () => {
it("verifies a valid any-target package, its digest, and its module", async () => {
const { archivePath, sha256: digest } = await archiveFile();
const result = await verifyModulePackage({ archivePath, expectedSha256: digest, platform });
expect(result).toEqual({
archiveSha256: digest,
fileCount: 8,
moduleId: "example",
target: "any",
version: "0.1.0",
});
});
it("extracts only a verified package, with normalized modes, into a new directory", async () => {
const { archivePath } = await archiveFile();
const target = path.join(await scratch(), "example");
await verifyModulePackage({ archivePath, extractTo: target, platform });
const names = (await readdir(target, { recursive: true })).sort();
expect(names).toContain("ekklesia.package.json");
expect(names).toContain("dist/server/index.js");
expect((await stat(target)).mode & 0o7777).toBe(0o755);
expect((await stat(path.join(target, "dist", "server", "index.js"))).mode & 0o7777).toBe(0o644);
const { archivePath: broken } = await archiveFile({ runtimePackage: { name: "x" } });
const refused = path.join(await scratch(), "refused");
await expect(
verifyModulePackage({ archivePath: broken, extractTo: refused, platform }),
).rejects.toBeInstanceOf(ModulePackageVerificationError);
await expect(stat(refused)).rejects.toMatchObject({ code: "ENOENT" });
});
it("refuses to extract over an existing directory", async () => {
const { archivePath } = await archiveFile();
const existing = await scratch();
await expect(
verifyModulePackage({ archivePath, extractTo: existing, platform }),
).rejects.toMatchObject({ code: "MODULE_PACKAGE_EXTRACTION_FAILED" });
});
it("fails a wrong archive digest", async () => {
await rejection({}, "MODULE_PACKAGE_DIGEST_MISMATCH", { expectedSha256: sha256("other") });
});
it("fails a payload whose contents no longer match their digest", async () => {
await rejection(
{
tamper: (files) => {
files.set("dist/server/index.js", Buffer.from("export default {}; // attacker-chosen\n"));
},
},
"MODULE_PACKAGE_FILE_MISMATCH",
);
});
it("fails a listed file missing from the archive", async () => {
await rejection(
{ tamper: (files) => void files.delete("THIRD_PARTY_NOTICES") },
"MODULE_PACKAGE_FILE_MISSING",
);
});
it("fails an archive file that is not listed", async () => {
await rejection(
withEntry({ path: "example/attacker-chosen.js", data: "extra" }),
"MODULE_PACKAGE_FILE_UNLISTED",
);
});
it("fails when the manifest and the package disagree about the version", async () => {
await rejection(
{ manifest: (manifest) => ({ ...manifest, version: "0.2.0" }) },
"MODULE_PACKAGE_IDENTITY_MISMATCH",
);
});
it("fails when the metadata names another module", async () => {
await rejection(
{ metadata: (metadata) => ({ ...metadata, module: { id: "other", version: "0.1.0" } }) },
"MODULE_PACKAGE_IDENTITY_MISMATCH",
);
});
it("fails a runtime package.json that is not the generated shape", async () => {
await rejection(
{
runtimePackage: {
name: "@test/module-example",
version: "0.1.0",
private: true,
type: "module",
ekklesia: { layer: "module" },
dependencies: { "@ekklesia/module-sdk": "workspace:*" },
},
},
"MODULE_PACKAGE_RUNTIME_PACKAGE_INVALID",
);
});
it("fails an unsupported package format", async () => {
await rejection(
{ metadata: (metadata) => ({ ...metadata, packageFormat: 2 }) },
"MODULE_PACKAGE_FORMAT_UNSUPPORTED",
);
});
it("fails transitional metadata it does not understand", async () => {
await rejection(
{ metadata: (metadata) => ({ ...metadata, transitional: { orm: { prisma: "6.0.0" } } }) },
"MODULE_PACKAGE_TRANSITIONAL_UNSUPPORTED",
);
await rejection(
{
metadata: (metadata) => ({
...metadata,
transitional: { databaseRuntime: { "drizzle-orm": "0.45.2", prisma: "6.0.0" } },
}),
},
"MODULE_PACKAGE_TRANSITIONAL_UNSUPPORTED",
);
});
it("enforces today's database runtime agreement, and accepts it when it holds", async () => {
await rejection(
{
metadata: (metadata) => ({
...metadata,
transitional: { databaseRuntime: { "drizzle-orm": "0.44.0" } },
}),
},
"MODULE_PACKAGE_DATABASE_RUNTIME_INCOMPATIBLE",
);
const { archivePath } = await archiveFile({
metadata: (metadata) => ({
...metadata,
transitional: { databaseRuntime: { "drizzle-orm": "0.45.2" } },
}),
});
await expect(verifyModulePackage({ archivePath, platform })).resolves.toMatchObject({
moduleId: "example",
});
});
it("fails a target this platform cannot run", async () => {
await rejection(
{
metadata: (metadata) => ({
...metadata,
target: "linux-arm64-musl",
runtime: { node: ">=20", nodeApi: 9 },
}),
},
"MODULE_PACKAGE_TARGET_INCOMPATIBLE",
);
});
it("fails a native file inside an any-target package", async () => {
const native = await modulePackageFixture({
tamper: (files) => {
files.set("dist/addon.node", Buffer.from([0x7f, 0x45, 0x4c, 0x46]));
},
metadata: (metadata) => {
const files = [
...(metadata.files as { path: string; size: number; sha256: string }[]),
{
path: "dist/addon.node",
size: 4,
sha256: sha256(Buffer.from([0x7f, 0x45, 0x4c, 0x46])),
},
].sort((left, right) => (left.path < right.path ? -1 : 1));
return { ...metadata, files };
},
});
const archivePath = path.join(await scratch(), "native.tar.gz");
await writeFile(archivePath, native.archive);
await expect(verifyModulePackage({ archivePath, platform })).rejects.toMatchObject({
code: "MODULE_PACKAGE_NATIVE_FILE",
});
});
it("fails a package that needs a newer Node.js", async () => {
await rejection(
{ metadata: (metadata) => ({ ...metadata, runtime: { node: ">=99" } }) },
"MODULE_PACKAGE_RUNTIME_INCOMPATIBLE",
);
});
it("fails an incorporated SDK outside the manifest's own range", async () => {
await rejection(
{
metadata: (metadata) => ({
...metadata,
incorporates: { "@ekklesia/module-sdk": "0.9.0" },
}),
},
"MODULE_PACKAGE_SDK_INCOMPATIBLE",
);
});
it("fails a module whose declared entrypoint is missing", async () => {
await rejection(
{
manifest: (manifest) => ({
...manifest,
version: "0.1.0",
entrypoints: {
...(manifest.entrypoints as Record<string, unknown>),
schema: "./dist/schema/index.js",
},
}),
},
"MODULE_PACKAGE_MODULE_INVALID",
);
});
it("fails a backend that imports a dependency the package does not contain", async () => {
const error = await rejection(
{
serverSource:
'import pad from "ekklesia-dependency-not-in-this-package";\nexport default { id: "example", version: "0.1.0", register() { pad(); } };\n',
},
"MODULE_PACKAGE_ENTRYPOINT_LOAD_FAILED",
);
expect(error.message).not.toContain("ekklesia-dependency-not-in-this-package");
});
it("fails a backend whose default export names another module", async () => {
await rejection(
{ serverSource: 'export default { id: "other", version: "0.1.0" };\n' },
"MODULE_PACKAGE_ENTRYPOINT_LOAD_FAILED",
);
});
it.each([
["an absolute path", { path: "/etc/attacker-chosen", data: "x" }, "MODULE_PACKAGE_UNSAFE_PATH"],
[
"a .. segment",
{ path: "example/../attacker-chosen", data: "x" },
"MODULE_PACKAGE_UNSAFE_PATH",
],
[
"a symbolic link",
{ path: "example/link", type: "symlink", linkname: "/etc/passwd" },
"MODULE_PACKAGE_LINK",
],
[
"a hard link",
{ path: "example/hard", type: "hardlink", linkname: "example/package.json" },
"MODULE_PACKAGE_LINK",
],
[
"a device node",
{ path: "example/device", type: "character" },
"MODULE_PACKAGE_SPECIAL_ENTRY",
],
["a FIFO", { path: "example/fifo", type: "fifo" }, "MODULE_PACKAGE_SPECIAL_ENTRY"],
[
"a setuid file",
{ path: "example/setuid", data: "x", mode: 0o4755 },
"MODULE_PACKAGE_UNSAFE_MODE",
],
[
"a setgid directory",
{ path: "example/setgid", type: "directory", mode: 0o2755 },
"MODULE_PACKAGE_UNSAFE_MODE",
],
[
"a second top-level directory",
{ path: "attacker-chosen/file", data: "x" },
"MODULE_PACKAGE_LAYOUT_INVALID",
],
[
"a duplicate entry",
{ path: "example/package.json", data: "{}" },
"MODULE_PACKAGE_LAYOUT_INVALID",
],
] as const)("fails %s", async (_label, entry, code) => {
await rejection(withEntry(entry as FixtureEntry), code);
});
it("fails a file that is not a gzip-compressed tar archive", async () => {
const archivePath = path.join(await scratch(), "plain.tar.gz");
await writeFile(archivePath, "not an archive");
await expect(verifyModulePackage({ archivePath, platform })).rejects.toMatchObject({
code: "MODULE_PACKAGE_UNREADABLE",
});
});
it("fails an archive without package metadata", async () => {
const fixture = await modulePackageFixture();
const { gunzipSync, gzipSync } = await import("node:zlib");
// Rebuild without the metadata entry by filtering the fixture through a fresh archive.
const archivePath = path.join(await scratch(), "no-metadata.tar.gz");
const raw = gunzipSync(fixture.archive);
const blocks: Buffer[] = [];
for (let offset = 0; offset + 512 <= raw.length; ) {
const headerBlock = raw.subarray(offset, offset + 512);
if (headerBlock.every((byte) => byte === 0)) {
break;
}
const name = headerBlock.subarray(0, 100).toString("latin1").replace(/\0.*$/su, "");
const size = Number.parseInt(headerBlock.subarray(124, 136).toString("latin1"), 8);
const length = 512 + Math.ceil(size / 512) * 512;
if (!name.endsWith("ekklesia.package.json")) {
blocks.push(raw.subarray(offset, offset + length));
}
offset += length;
}
await writeFile(archivePath, gzipSync(Buffer.concat([...blocks, Buffer.alloc(1024)])));
await expect(verifyModulePackage({ archivePath, platform })).rejects.toMatchObject({
code: "MODULE_PACKAGE_METADATA_MISSING",
});
});
});
core/tests/support/module-package-fixture.ts
+205
View File
@@ -0,0 +1,205 @@
import { createHash } from "node:crypto";
import { mkdtemp, readdir, readFile, rm } from "node:fs/promises";
import { tmpdir } from "node:os";
import path from "node:path";
import { gzipSync } from "node:zlib";
import { createMigrationModule } from "./migration-module-fixture.js";
/**
* Test-only builders for Ekklesia Module Package v1 archives (ADR 0049). They write ustar archives
* by hand so a test can produce exactly what the canonical packager never would: links, device
* nodes, traversal, a second top-level directory, a corrupted payload.
*/
export type FixtureEntryType = "file" | "directory" | "symlink" | "hardlink" | "character" | "fifo";
export interface FixtureEntry {
readonly path: string;
readonly type?: FixtureEntryType;
readonly mode?: number;
readonly data?: Uint8Array | string;
readonly linkname?: string;
}
const typeflags: Readonly<Record<FixtureEntryType, string>> = {
character: "3",
directory: "5",
fifo: "6",
file: "0",
hardlink: "1",
symlink: "2",
};
function octal(value: number, width: number): string {
return `${value.toString(8).padStart(width - 1, "0")}\u0000`;
}
function header(entry: FixtureEntry, size: number): Buffer {
const block = Buffer.alloc(512);
const name =
entry.type === "directory" && !entry.path.endsWith("/") ? `${entry.path}/` : entry.path;
let prefix = "";
let base = name;
if (Buffer.byteLength(name) > 100) {
const split = name.lastIndexOf("/", name.length - 2);
prefix = name.slice(0, split);
base = name.slice(split + 1);
}
block.write(base, 0, 100, "latin1");
block.write(octal(entry.mode ?? (entry.type === "directory" ? 0o755 : 0o644), 8), 100, "latin1");
block.write(octal(0, 8), 108, "latin1");
block.write(octal(0, 8), 116, "latin1");
block.write(octal(size, 12), 124, "latin1");
block.write(octal(0, 12), 136, "latin1");
block.write(" ", 148, "latin1");
block.write(typeflags[entry.type ?? "file"], 156, "latin1");
block.write(entry.linkname ?? "", 157, 100, "latin1");
block.write("ustar\u000000", 257, "latin1");
block.write(prefix, 345, 155, "latin1");
let checksum = 0;
for (const byte of block) {
checksum += byte;
}
block.write(`${checksum.toString(8).padStart(6, "0")}\u0000 `, 148, "latin1");
return block;
}
/** A gzip-compressed ustar archive of exactly these entries, in this order. */
export function tarGz(entries: readonly FixtureEntry[]): Buffer {
const parts: Buffer[] = [];
for (const entry of entries) {
const data =
entry.type === undefined || entry.type === "file"
? Buffer.from(entry.data ?? "")
: Buffer.alloc(0);
parts.push(header(entry, data.length), data, Buffer.alloc((512 - (data.length % 512)) % 512));
}
parts.push(Buffer.alloc(1024));
return gzipSync(Buffer.concat(parts), { level: 9 });
}
export function sha256(data: Uint8Array | string): string {
return createHash("sha256").update(data).digest("hex");
}
export interface PackageFixtureOptions {
readonly id?: string;
readonly version?: string;
readonly serverSource?: string;
readonly manifest?: (manifest: Record<string, unknown>) => Record<string, unknown>;
readonly runtimePackage?: Record<string, unknown>;
readonly metadata?: (metadata: Record<string, unknown>) => Record<string, unknown>;
/** Changes the payload after its digests were computed. */
readonly tamper?: (files: Map<string, Buffer>) => void;
/** Entries appended after the package's own, under any path. */
readonly extraEntries?: readonly FixtureEntry[];
}
export interface PackageFixture {
readonly archive: Buffer;
readonly sha256: string;
}
async function readTree(root: string): Promise<Map<string, Buffer>> {
const files = new Map<string, Buffer>();
for (const entry of await readdir(root, { recursive: true, withFileTypes: true })) {
if (entry.isFile()) {
const absolute = path.join(entry.parentPath, entry.name);
files.set(path.relative(root, absolute).split(path.sep).join("/"), await readFile(absolute));
}
}
return files;
}
/**
* A valid any-target package of a minimal module: a server stub with no imports, one migration, a
* notices file, the generated runtime package.json, and complete metadata. Every option changes one
* thing, so a test can break exactly the rule it is about.
*/
export async function modulePackageFixture(
options: PackageFixtureOptions = {},
): Promise<PackageFixture> {
const id = options.id ?? "example";
const version = options.version ?? "0.1.0";
const scratch = await mkdtemp(path.join(tmpdir(), "ekklesia-package-fixture-"));
let files: Map<string, Buffer>;
try {
const created = await createMigrationModule(scratch, id, {
serverSource:
options.serverSource ??
`export default { id: ${JSON.stringify(id)}, version: ${JSON.stringify(version)}, register() {} };\n`,
});
files = await readTree(created.moduleDirectory);
} finally {
await rm(scratch, { force: true, recursive: true });
}
const manifest = JSON.parse(
files.get("ekklesia.module.json")?.toString("utf8") ?? "{}",
) as Record<string, unknown>;
files.set(
"ekklesia.module.json",
Buffer.from(
`${JSON.stringify(options.manifest ? options.manifest(manifest) : { ...manifest, version }, null, 2)}\n`,
),
);
files.set(
"package.json",
Buffer.from(
`${JSON.stringify(
options.runtimePackage ?? {
name: `@test/module-${id}`,
version,
private: true,
type: "module",
ekklesia: { layer: "module" },
},
null,
2,
)}\n`,
),
);
files.set("THIRD_PARTY_NOTICES", Buffer.from("No third-party software is included.\n"));
const listed = [...files.entries()]
.sort(([left], [right]) => (left < right ? -1 : left > right ? 1 : 0))
.map(([filePath, data]) => ({ path: filePath, size: data.length, sha256: sha256(data) }));
const baseMetadata: Record<string, unknown> = {
packageFormat: 1,
module: { id, version },
target: "any",
runtime: { node: ">=20" },
provenance: {
revision: "86ae907f59458508e3277666cdd214f5bd7c1078",
packager: { name: "@ekklesia/module-pack", version: "0.1.2" },
},
incorporates: { "@ekklesia/module-sdk": "0.1.2" },
files: listed,
};
const metadata = options.metadata ? options.metadata(baseMetadata) : baseMetadata;
options.tamper?.(files);
const directories = new Set<string>([id]);
for (const filePath of files.keys()) {
const segments = filePath.split("/");
for (let index = 1; index < segments.length; index += 1) {
directories.add(`${id}/${segments.slice(0, index).join("/")}`);
}
}
const all = new Map<string, FixtureEntry>();
for (const directory of directories) {
all.set(directory, { path: directory, type: "directory" });
}
for (const [filePath, data] of files) {
all.set(`${id}/${filePath}`, { path: `${id}/${filePath}`, data });
}
all.set(`${id}/ekklesia.package.json`, {
path: `${id}/ekklesia.package.json`,
data: `${JSON.stringify(metadata, null, 2)}\n`,
});
const ordered = [...all.values()].sort((left, right) =>
left.path < right.path ? -1 : left.path > right.path ? 1 : 0,
);
const archive = tarGz([...ordered, ...(options.extraEntries ?? [])]);
return { archive, sha256: sha256(archive) };
}
core/tests/verify-module-package-cli.test.ts
+75
View File
@@ -0,0 +1,75 @@
import { readFile } from "node:fs/promises";
import path from "node:path";
import { fileURLToPath } from "node:url";
import { describe, expect, it } from "vitest";
import {
parseSha256File,
parseVerifyModulePackageArguments,
readModulePackagePlatform,
} from "../src/cli/verify-module-package.js";
const repositoryRoot = path.resolve(path.dirname(fileURLToPath(import.meta.url)), "..", "..");
const digest = "ab".repeat(32);
describe("verify-module-package arguments", () => {
it("takes an archive and, optionally, a digest and an extraction directory", () => {
expect(parseVerifyModulePackageArguments(["example-0.1.0-any.tar.gz"])).toEqual({
archivePath: "example-0.1.0-any.tar.gz",
});
expect(
parseVerifyModulePackageArguments([
"x.tar.gz",
"--sha256",
digest.toUpperCase(),
"--extract-to",
"/staging/example",
]),
).toEqual({ archivePath: "x.tar.gz", expectedSha256: digest, extractTo: "/staging/example" });
expect(
parseVerifyModulePackageArguments(["x.tar.gz", "--sha256-file", "x.tar.gz.sha256"]),
).toEqual({ archivePath: "x.tar.gz", sha256File: "x.tar.gz.sha256" });
});
it.each([
["no archive", []],
["two archives", ["a.tar.gz", "b.tar.gz"]],
["an unknown option", ["a.tar.gz", "--force"]],
["both digest options", ["a.tar.gz", "--sha256", digest, "--sha256-file", "a.sha256"]],
["a malformed digest", ["a.tar.gz", "--sha256", "abc"]],
["an option without its value", ["a.tar.gz", "--extract-to"]],
])("refuses %s", (_label, argv) => {
expect(() => parseVerifyModulePackageArguments(argv)).toThrow();
});
});
describe("verify-module-package digest files", () => {
it("reads the sha256sum format, ignoring the recorded file name", () => {
expect(parseSha256File(`${digest} example-0.1.0-any.tar.gz\n`)).toBe(digest);
expect(parseSha256File(`${digest.toUpperCase()} *renamed.tar.gz\n`)).toBe(digest);
});
it.each([
"",
"not-a-digest x.tar.gz\n",
`${digest}\n${digest} second\n`,
])("refuses %j", (contents) => {
expect(() => parseSha256File(contents)).toThrow();
});
});
describe("verify-module-package platform facts", () => {
it("reads this platform's versions and its transitional database runtime from the repository", async () => {
const root = JSON.parse(await readFile(path.join(repositoryRoot, "package.json"), "utf8"));
const core = JSON.parse(
await readFile(path.join(repositoryRoot, "core", "package.json"), "utf8"),
);
const sdk = JSON.parse(
await readFile(path.join(repositoryRoot, "packages", "module-sdk", "package.json"), "utf8"),
);
await expect(readModulePackagePlatform(repositoryRoot)).resolves.toEqual({
databaseRuntime: { "drizzle-orm": core.dependencies["drizzle-orm"] },
moduleSdkVersion: sdk.version,
platformVersion: root.version,
});
});
});
deploy/README.md
+33 -12
View File
@@ -142,12 +142,16 @@ Installed modules live on the host, in module storage: `/var/lib/ekklesia/module
and `migrate`, which is where `EKKLESIA_MODULES_DIR` points. Ekklesia never writes there.
Installing, updating, or removing a module never needs a new image, only a restart.
**What belongs there.** One directory per module, each a self-contained installed module: its
`package.json`, `ekklesia.module.json`, built artifacts, migrations, and the production
dependencies its server code imports, in its own `node_modules`
**What belongs there.** One directory per module, each extracted from that module's **Ekklesia
Module Package**
([ADR 0049](../docs/architecture/decisions/0049-module-package-v1.md)): one archive,
`<id>-<version>-<target>.tar.gz`, and the SHA-256 published beside it. A package holds the module's
generated `package.json`, its manifest, `ekklesia.package.json`, bundled backend code, built browser
artifacts, migrations, and `THIRD_PARTY_NOTICES`
([Module development](../docs/module-development/README.md#an-installed-module-is-self-contained)).
Copying a module's `dist/` alone is not enough. Files are owned by root and readable by everyone
(directories 755, files 644), so the API's unprivileged user can read them and never change them.
The digest proves the archive is the one you were given; it does not prove who published it.
**Nothing else belongs there.** Every non-hidden directory must be a valid module, or the API does
not start. That includes `lost+found`, so if module storage is a file system of its own, mount it
@@ -159,12 +163,28 @@ never loads or unloads a module while it runs. Change module storage only while
it reads browser artifacts from disk as it serves them, so files changed under a running API would
serve one version's browser code beside another's server code.
To **install or update** a module, with `<id>` its module id:
To **install or update** a module, with `<id>` its module id and the package and its `.sha256` in
the current directory, first **verify** it with the image's own verifier. It checks the archive's
digest, its safety (no links, special files, or escaping paths), every file's digest, the package's
identity, target, and Node.js range against this platform, runs the platform's module and migration
validation, and loads the backend once in a short-lived process with no environment. Only then does
it write the verified module directory, to a new directory outside module storage:
```sh
mkdir -m 777 staging # writable by the image's unprivileged user; nothing else is
docker run --rm --network none --read-only --tmpfs /tmp \
-v "$PWD:/package:ro" -v "$PWD/staging:/staging" \
--entrypoint node "$EKKLESIA_IMAGE" core/dist/cli/verify-module-package.js \
/package/<id>-<version>-any.tar.gz --sha256-file /package/<id>-<version>-any.tar.gz.sha256 \
--extract-to /staging/<id>
```
Then install the verified directory:
```sh
docker compose stop api
# Stage under a hidden name, which discovery ignores, then rename it into place in one step.
sudo cp -R ./<id> /var/lib/ekklesia/modules/.incoming-<id>
sudo cp -R staging/<id> /var/lib/ekklesia/modules/.incoming-<id>
sudo chown -R root:root /var/lib/ekklesia/modules/.incoming-<id>
sudo chmod -R a+rX,go-w /var/lib/ekklesia/modules/.incoming-<id>
sudo mv /var/lib/ekklesia/modules/<id> /var/lib/ekklesia/modules/.previous-<id> # update only
@@ -188,7 +208,8 @@ is information, not a problem, and it blocks no other migration. To stop a modul
it, disable it instead (`set-module-enabled.js --disable`, [Operator commands](#operator-commands)).
There is no purge: deleting a module's data is a separate operation that does not exist yet.
To **restore** a removed module, install it again as above, at the same version or a newer one. It
To **restore** a removed module, verify and install its package again as above, at the same version
or a newer one. It
finds its data, its settings, and its history where it left them. Its history is strict again: a
package that no longer contains a migration that was applied, or contains one with different
contents, is refused by the migration plan, so an older version cannot be installed over newer data.
@@ -409,10 +430,9 @@ deployment:
instead, on the **same generic image**, and also proves [Installed modules](#installed-modules)
end to end:
- the image boots with no module;
- the repository's module template, packaged as a self-contained test module
(`scripts/lib/test-module-package.Dockerfile`, test-only and not a package format), is installed
into module storage, its migration is planned and explicitly applied, the API restarts, and the
module reads data it owns;
- the repository's module template, packaged as an Ekklesia Module Package v1 by the canonical
packager, is verified by the image itself and installed into module storage, its migration is
planned and explicitly applied, the API restarts, and the module reads data it owns;
- every hostname's served and refused paths, exact CORS for the Dashboard's origin and none for
others;
- a real headless Chromium (pinned by digest, driven over the DevTools protocol) signs in on the
@@ -421,8 +441,8 @@ deployment:
API's hostname with its service worker, and signs out;
- the module is removed and the API restarted: Core still runs, the module's routes are gone,
migration planning and applying succeed and report its retained history, and its data is intact;
- the module is restored and the API restarted: its history validates strictly and it returns its
previous data.
- the exact same package is verified and installed again and the API restarted: its history
validates strictly and it returns its previous data.
CI runs both topologies on every change.
- **Test-only:** the smoke test runs Caddy with its internal CA (`local_certs`), because a test run
@@ -586,6 +606,7 @@ reports direct mode for an API that trusts its proxy. The running API's own valu
| Doctor (read-only report) | `docker compose run --rm migrate node core/dist/cli/doctor.js` |
| Create the initial owner (interactive) | `docker compose run --rm -it migrate node core/dist/cli/create-initial-owner.js --email owner@example.org` |
| Validate the installed modules (read-only) | `docker compose run --rm migrate node core/dist/cli/validate-modules.js` |
| Verify a module package (read-only unless extracting) | `node core/dist/cli/verify-module-package.js <archive> --sha256-file <archive>.sha256 [--extract-to <new directory>]`, in a `docker run` of the image ([Installed modules](#installed-modules)) |
| Enable or disable a module | `docker compose run --rm migrate node core/dist/cli/set-module-enabled.js --enable --module <id>` (or `--disable`) |
Every command reads installed modules from the same `EKKLESIA_MODULES_DIR` the API does. The plan,
docs/architecture/decisions/0049-module-package-v1.md
+1 -1
View File
@@ -1,6 +1,6 @@
# 0049: A module release is one immutable, self-contained package: Ekklesia Module Package v1
- Status: Proposed
- Status: Accepted
- Date: 2026-10-07
- Supersedes: none
- Extends: [0048](0048-generic-image-and-installed-module-storage.md)
docs/architecture/decisions/README.md
+1 -1
View File
@@ -57,4 +57,4 @@ ADRs capture durable choices that affect more than one package or phase.
- [0046: Ekklesia's branding is never forced on an installation](0046-installation-owned-branding.md) — Proposed
- [0047: Every module proves itself inside Ekklesia, with one harness the platform publishes](0047-module-ci-harness.md) — Proposed
- [0048: Ekklesia ships one generic image, and installed modules live outside it](0048-generic-image-and-installed-module-storage.md) — Accepted
- [0049: A module release is one immutable, self-contained package: Ekklesia Module Package v1](0049-module-package-v1.md) — Proposed
- [0049: A module release is one immutable, self-contained package: Ekklesia Module Package v1](0049-module-package-v1.md) — Accepted
docs/architecture/platform-architecture.md
+15
View File
@@ -672,3 +672,18 @@ artifact from what an installation installs:
module that lost or changed an applied migration is still refused.
- **Trust is unchanged.** Installed modules are still trusted code; module storage is mounted
read-only, which keeps Ekklesia from modifying installed code but does not sandbox it.
[ADR 0049](decisions/0049-module-package-v1.md) defines how a module is distributed: one Ekklesia
Module Package v1, a deterministic `.tar.gz` with its SHA-256.
- **The packager.** `@ekklesia/module-pack` bundles the module's backend with every pure-JavaScript
dependency, the SDK, contracts, and Drizzle included. It copies the built browser artifacts and
migrations, generates the runtime `package.json`, `ekklesia.package.json` with every file's digest,
and `THIRD_PARTY_NOTICES`, and writes the archive.
- **The verifier.** Core's `verifyModulePackage` reads the archive in memory and refuses links,
special files, and escaping paths. It checks every digest and identity, the target and runtime,
and the transitional Drizzle agreement, runs the platform's own module and migration validation,
and loads the backend once in a short-lived process with an empty environment. Only then does it
extract anything.
- **What exists today.** Only the `any` target. Drizzle agreement is recorded as transitional, not a
database ABI.
docs/development/security.md
+6 -1
View File
@@ -479,7 +479,12 @@ Installed modules come from installation-owned module storage, which the referen
is an integrity property, not a sandbox: an installed module is still trusted code with the API's
database role, exactly as above. The repository's boundary checks prove only source modules in a
workspace, and an installed package usually ships none, so installing one means trusting that
module's own CI and whoever placed it in module storage.
module's own CI and whoever placed it in module storage. A Module Package v1 (ADR 0049) is verified
before it is installed: its digest, its archive safety (no links, special files, setuid bits, or
escaping paths), every file's digest, and its identity and compatibility. The digest proves
integrity, not who published the package; signing is not implemented. The verifier loads the backend
in a short-lived process with an empty environment, so module code never sees a database URL or a
secret while it is being checked.
Two production defaults are relevant but are **not** by themselves a deployment guarantee:
docs/development/testing.md
+26 -3
View File
@@ -564,6 +564,29 @@ the following pull-request status context to `main` branch protection:
CI / migration-smoke (pull_request)
```
## Module packages
Ekklesia Module Package v1 (ADR 0049) is proven at three levels, all in `pnpm verify`:
- `packages/contracts/tests/module-package.test.ts`: the `ekklesia.package.json` schema, target
names, and the generated runtime `package.json`.
- `core/tests/module-package-verification.test.ts`: the verifier against hand-built archives,
including what the packager never produces. A valid package verifies and extracts with normalized
modes. A wrong digest, a corrupted or missing or unlisted file, disagreeing identities, an unknown
format, unknown transitional metadata, an incompatible target, Node.js range, SDK, or database
runtime, a missing entrypoint, a backend that imports something the package lacks, absolute and
`..` paths, symbolic and hard links, device nodes and FIFOs, setuid and setgid entries, a second
top-level directory, and a duplicate entry each fail with their own sanitized code. The backend
load check runs in a real child process.
- `packages/module-pack/tests/`: the packager against generated fixture modules (payload contents,
generated `package.json`, metadata, bundled backend with only built-in imports, refusal of native
and platform-specific dependencies and of unknown or textless licences, notices, byte-identical
archives, normalized tar and gzip headers), and end to end against the built module template:
packed twice to identical bytes without changing the workspace's install state, verified by Core's
verifier, extracted elsewhere, and loaded with Node's own resolution hooks proving nothing outside
the package is resolved. Turbo builds the template and Core first, and runs the template's own
tests (which rebuild its UI in place) before these.
## The generic platform image and the reference deployment
These need Docker and build images locally; nothing is pushed or published.
@@ -581,13 +604,13 @@ These need Docker and build images locally; nothing is pushed or published.
- `pnpm deploy:smoke` (`scripts/smoke-deploy.sh`) runs the reference deployment end to end on the
generic image with empty module storage. With `EKKLESIA_SMOKE_TOPOLOGY=split`, the same image also
runs the installed-module lifecycle:
- the module template, packaged as a self-contained test module
(`scripts/lib/test-module-package.Dockerfile`, test-only), is installed into module storage;
- the module template is packaged by the canonical packager (Module Package v1, ADR 0049),
verified by the image's own verifier, and installed into module storage;
- its migration is planned and explicitly applied, and the API restarted;
- a real browser mounts it;
- it is removed and the API restarted, and Core still runs, migrations still plan and apply, and
its data survives;
- it is restored, with its data.
- the exact same package is verified and installed again, and the module returns with its data.
See [deploy/README.md](../../deploy/README.md#testing-this-deployment).
docs/module-development/README.md
+62 -25
View File
@@ -78,42 +78,79 @@ names (`/var/lib/ekklesia/modules` in the production image, the checkout's `modu
unset). Discovery reads it only when a process starts. Nothing is hot loaded, and an installed module
is read-only to the platform.
An installed module directory is **self-contained for runtime use**. Everything the platform reads
or loads from it, and everything those files import at run time, resolves inside it:
- `package.json`, with no `workspace:` range needed at run time, and `ekklesia.module.json`;
- every built artifact the manifest declares: `dist/server`, `dist/schema`, jobs, and the UI,
display, and contribution bundles;
- the `drizzle/` migrations directory, with its journal and `_ekklesia.json`;
- a `node_modules` holding the production dependencies of your server, schema, and jobs code,
including your own copies of `@ekklesia/module-sdk`, `@ekklesia/contracts`, and `drizzle-orm`.
An installed module directory is **self-contained for runtime use**: everything the platform reads
or loads from it, and everything those files import at run time, resolves inside it. You never
assemble that directory by hand. You build your module, and the canonical packager turns it into an
**Ekklesia Module Package v1**
([ADR 0049](../architecture/decisions/0049-module-package-v1.md)): one deterministic archive and its
SHA-256, which an installation verifies and extracts into module storage.
**Copying `dist/` is not an installed module.** Your compiled server code imports
`@ekklesia/module-sdk` and `drizzle-orm` by name, and Node resolves those only from your module's
own directory upwards. The platform's `node_modules` is never on that path, and Ekklesia adds it
through no link, `NODE_PATH`, loader hook, or alias. A module whose imports do not resolve stops the
API from starting with a sanitized `MODULE_RUNTIME_IMPORT_FAILED`, so test the directory you ship,
not your workspace.
through no link, `NODE_PATH`, loader hook, or alias. The packager therefore bundles your backend.
**Native dependencies** must match the platform image's runtime: Node.js 24.18.0 on Debian 13
(glibc), on the image's CPU architecture.
#### Packaging your module
Build your module, then run the packager from the workspace your module is built in. It is a
development dependency like `@ekklesia/module-ui-build`:
```bash
pnpm exec ekklesia-pack-module --module . --target any --out ../packages \
--source-revision "$(git rev-parse HEAD)"
```
It writes `<id>-<version>-any.tar.gz` and its `.sha256` line, and nothing else anywhere: it never
installs, links, or deploys, so your workspace is left exactly as it was. Never run `pnpm deploy` in
a workspace instead: it rewrites that workspace's install state to production-only.
The archive holds one directory named with your module id:
```text
<id>/
├── package.json generated: name, version, private, "type": "module", the layer marker
├── ekklesia.module.json your manifest, byte for byte
├── ekklesia.package.json package metadata: target, runtime, provenance, file digests
├── THIRD_PARTY_NOTICES licence and notice text of every third-party package bundled
├── dist/server/index.js your backend entrypoints, bundled
├── dist/schema/index.js
├── dist/chunks/ code your backend entrypoints share
├── dist/ui/ … dist/display/ … dist/contributions/ your browser artifacts, exactly as built
└── drizzle/ your migrations
```
- **Bundled backend.** Your server, schema, and jobs entrypoints are bundled with every
pure-JavaScript dependency they use, the Module SDK, the contracts, and Drizzle included. The only
imports left are Node.js built-ins. A dependency the bundle cannot resolve fails packaging; nothing
is left to be found at run time.
- **Never shipped:** `src/`, tests, TypeScript declarations, source maps, development dependencies,
your development `package.json`, any `workspace:` range, and links of any kind.
- **Browser artifacts must be current.** The packager rebuilds them in memory and refuses an artifact
on disk that differs, so run your build first.
- **`any` is the only target today.** A dependency that is native or platform-specific (a compiled
addon, a package that declares an operating system or CPU, or one that loads a per-platform binary
such as `sharp`) cannot be in an `any` package, and packaging fails. Native targets come later.
- **Licences.** `THIRD_PARTY_NOTICES` credits every third-party package your bundles contain, with
its declared licence and the licence and notice files it ships. A package that declares no licence
fails packaging, and so does one that ships no licence text, unless its licence's standard text is
complete without a copyright line (Apache-2.0, which the packager then supplies and says so). The
Ekklesia platform's own packages are listed in `ekklesia.package.json` under `incorporates`.
- **Deterministic.** The same packager version and the same built inputs produce the same archive,
byte for byte, wherever the checkout is.
**Drizzle, for now.** You define tables with your own copy of Drizzle, and Core runs your queries
through the platform's Drizzle instance (see [Database access](#database-access)). Objects from two
copies interoperate only when the copies are interchangeable, so the current implementation requires
**exactly the Drizzle version the platform uses**: the one `@ekklesia/module-sdk` depends on. This is
a constraint of today's implementation, not a permanent contract.
**exactly the Drizzle version the platform uses**: the one `@ekklesia/module-sdk` depends on. The
package records it under `transitional`, and an installation refuses a package whose Drizzle differs.
This is a constraint of today's implementation, not a permanent contract.
Browser artifacts are already self-contained bundles, so nothing changes for them.
#### Verifying a package
The format modules are delivered in is proposed in
[ADR 0049](../architecture/decisions/0049-module-package-v1.md): one deterministic archive per
target, with bundled backend code, produced by an Ekklesia-owned packager that does not exist yet.
Until it does, the repository's smoke test packages the module template with `pnpm deploy` in a
disposable Docker build (`scripts/lib/test-module-package.Dockerfile`), a test fixture and not a
package format. Never run `pnpm deploy` in your own workspace: it rewrites that workspace's install
state to production-only, and the next build or `pnpm verify` then fails until a normal
`pnpm install` repairs it.
`node core/dist/cli/verify-module-package.js <archive> --sha256-file <archive>.sha256`, run from an
Ekklesia installation (`/opt/ekklesia` in the image), checks a package against that platform without
installing it. Pass `--extract-to <new directory>` to receive the verified module directory.
[deploy/README.md](../../deploy/README.md#installed-modules) shows the whole installation.
The repository's boundary checks (`pnpm boundaries`) prove only source modules in a workspace. An
installed package normally has no source, so the platform cannot check it the same way. Your
package.json
+2
View File
@@ -34,7 +34,9 @@
"migrations:validate": "turbo run build && node core/dist/cli/validate-migrations.js",
"modules:disable": "turbo run build && node --env-file-if-exists=.env core/dist/cli/set-module-enabled.js --disable",
"modules:enable": "turbo run build && node --env-file-if-exists=.env core/dist/cli/set-module-enabled.js --enable",
"modules:pack": "turbo run build && node packages/module-pack/bin/ekklesia-pack-module.mjs",
"modules:validate": "turbo run build && node core/dist/cli/validate-modules.js",
"modules:verify-package": "turbo run build && node core/dist/cli/verify-module-package.js",
"test": "turbo run test test:root",
"test:migrations:integration": "pnpm build && bash scripts/test-migrations-integration.sh",
"test:root": "vitest run --config vitest.config.ts",
packages/contracts/src/index.ts
+1
View File
@@ -8,6 +8,7 @@ export * from "./module-http.js";
export * from "./module-lifecycle.js";
export * from "./module-manifest.js";
export * from "./module-migration.js";
export * from "./module-package.js";
export * from "./module-realtime.js";
export * from "./module-runtime.js";
export * from "./module-schema.js";
packages/contracts/src/module-package.ts
+203
View File
@@ -0,0 +1,203 @@
import { z } from "zod";
import { moduleIdSchema } from "./identifiers.js";
import { semanticVersionRangeSchema, semanticVersionSchema } from "./versions.js";
/**
* Ekklesia Module Package v1 (ADR 0049): the format a separately versioned module is distributed
* in. One deterministic `.tar.gz` holding exactly one top-level directory named with the module id,
* whose contents are the installed module directory ADR 0048 requires. This file defines the
* package's metadata (`ekklesia.package.json`), its target names, and the runtime `package.json`
* the packager generates. The archive encoding itself belongs to the packager and the verifier.
*/
export const MODULE_PACKAGE_FORMAT = 1;
export const MODULE_PACKAGE_METADATA_FILENAME = "ekklesia.package.json";
export const MODULE_PACKAGE_NOTICES_FILENAME = "THIRD_PARTY_NOTICES";
/** The portable target: a package with no native or platform-specific file. */
export const MODULE_PACKAGE_ANY_TARGET = "any";
/**
* `any`, or `<platform>-<arch>[-<libc>]` in Node.js's own `process.platform` and `process.arch`
* vocabulary. On Linux the C library is required, because a native addon built against one does
* not load against the other; elsewhere it is meaningless and refused.
*/
export const modulePackageTargetSchema = z.string().superRefine((value, context) => {
if (value === MODULE_PACKAGE_ANY_TARGET) {
return;
}
const match = /^([a-z][a-z0-9]*)-([a-z][a-z0-9]*)(?:-([a-z]+))?$/u.exec(value);
const platform = match?.[1];
const libc = match?.[3];
const valid =
match !== null &&
platform !== "any" &&
(platform === "linux" ? libc === "glibc" || libc === "musl" : libc === undefined);
if (!valid) {
context.addIssue({
code: "custom",
message: "Must be any, or <platform>-<arch>, with -glibc or -musl on Linux.",
});
}
});
/** A path inside the module directory: relative, POSIX, printable ASCII, and never escaping. ASCII
* keeps JavaScript's string order identical to the archive's byte-wise order. */
const payloadPathSchema = z.string().superRefine((value, context) => {
const segments = value.split("/");
const valid =
/^[\x20-\x7e]+$/u.test(value) &&
!value.includes("\\") &&
!value.startsWith("/") &&
segments.every((segment) => segment !== "" && segment !== "." && segment !== "..") &&
value !== MODULE_PACKAGE_METADATA_FILENAME;
if (!valid) {
context.addIssue({
code: "custom",
message: "Must be a portable path inside the module directory, other than the metadata file.",
});
}
});
const modulePackageFileSchema = z.strictObject({
path: payloadPathSchema,
size: z.number().int().nonnegative(),
sha256: z.string().regex(/^[0-9a-f]{64}$/u, "Must be a lowercase hexadecimal SHA-256."),
});
const modulePackageRuntimeSchema = z.strictObject({
/** The Node.js versions the package was built for. Always present. */
node: semanticVersionRangeSchema,
/** Native packages only: the minimum Node-API version their addons need. */
nodeApi: z.number().int().positive().optional(),
/** Native packages only: the exact `process.versions.modules` an ABI-specific addon needs. */
nodeAbi: z
.string()
.regex(/^[0-9]+$/u)
.optional(),
/** Native Linux packages only: the minimum C library version. */
libcVersion: semanticVersionRangeSchema.optional(),
});
const ekklesiaPackageNameSchema = z
.string()
.regex(/^@ekklesia\/[a-z0-9][a-z0-9-]*$/u, "Must name an @ekklesia package.");
/**
* Compatibility facts today's implementation needs and that are not part of the long-term contract
* (ADR 0049). Strict at every level: a reader that finds a fact it does not understand must refuse
* the package rather than ignore it.
*
* `databaseRuntime` records the Drizzle version bundled with the module's backend, because Core runs
* a module's queries through its own Drizzle instance over tables the module defined with its own
* copy. That is an implementation constraint pending a Module Database ABI decision, **not** a
* database contract a module may rely on.
*/
const modulePackageTransitionalSchema = z.strictObject({
databaseRuntime: z
.strictObject({
"drizzle-orm": semanticVersionSchema.optional(),
})
.optional(),
});
export const modulePackageMetadataSchema = z
.strictObject({
packageFormat: z.literal(MODULE_PACKAGE_FORMAT),
module: z.strictObject({ id: moduleIdSchema, version: semanticVersionSchema }),
target: modulePackageTargetSchema,
runtime: modulePackageRuntimeSchema,
provenance: z.strictObject({
/** The exact source revision the package was built from: a full SHA-1 or SHA-256 commit id. */
revision: z.string().regex(/^(?:[0-9a-f]{40}|[0-9a-f]{64})$/u),
/** Informational only: nothing trusts it, and it ties the package to no host. */
repository: z.string().min(1).max(2048).optional(),
packager: z.strictObject({
name: z.string().min(1).max(214),
version: semanticVersionSchema,
}),
}),
/** The exact versions of the Ekklesia SDK and runtime libraries bundled into the package. */
incorporates: z.record(ekklesiaPackageNameSchema, semanticVersionSchema),
transitional: modulePackageTransitionalSchema.optional(),
/** Every regular file in the module directory except this metadata file, in byte-wise path
* order. The set is exact: nothing missing, nothing extra. */
files: z.array(modulePackageFileSchema),
})
.superRefine((metadata, context) => {
if (
metadata.target === MODULE_PACKAGE_ANY_TARGET &&
(metadata.runtime.nodeApi !== undefined ||
metadata.runtime.nodeAbi !== undefined ||
metadata.runtime.libcVersion !== undefined)
) {
context.addIssue({
code: "custom",
message: "An any-target package carries no native runtime requirement.",
path: ["runtime"],
});
}
for (let index = 1; index < metadata.files.length; index += 1) {
const previous = metadata.files[index - 1]?.path ?? "";
const current = metadata.files[index]?.path ?? "";
if (!(previous < current)) {
context.addIssue({
code: "custom",
message: "Files must be listed once each, in byte-wise path order.",
path: ["files", index, "path"],
});
}
}
});
export type ModulePackageMetadata = z.output<typeof modulePackageMetadataSchema>;
export type ModulePackageFile = z.output<typeof modulePackageFileSchema>;
export type ModulePackageTarget = z.output<typeof modulePackageTargetSchema>;
export function safeParseModulePackageMetadata(input: unknown) {
return modulePackageMetadataSchema.safeParse(input);
}
export function safeParseModulePackageTarget(input: unknown) {
return modulePackageTargetSchema.safeParse(input);
}
/**
* The runtime `package.json` of an installed module: exactly what module discovery reads, generated
* by the packager and never copied from a development workspace. No dependencies, scripts, exports,
* or imports, because nothing at run time resolves through them.
*/
export const modulePackageRuntimePackageSchema = z.strictObject({
name: z.string().min(1).max(214),
version: semanticVersionSchema,
private: z.literal(true),
type: z.literal("module"),
ekklesia: z.strictObject({ layer: z.literal("module") }),
});
export type ModulePackageRuntimePackage = z.output<typeof modulePackageRuntimePackageSchema>;
export function modulePackageRuntimePackage(identity: {
readonly name: string;
readonly version: string;
}): ModulePackageRuntimePackage {
return {
name: identity.name,
version: identity.version,
private: true,
type: "module",
ekklesia: { layer: "module" },
};
}
export function safeParseModulePackageRuntimePackage(input: unknown) {
return modulePackageRuntimePackageSchema.safeParse(input);
}
/** The recommended archive file name. A convenience only: a file name is never trusted. */
export function modulePackageArchiveFileName(
moduleId: string,
version: string,
target: string,
): string {
return `${moduleId}-${version}-${target}.tar.gz`;
}
packages/contracts/tests/module-package.test.ts
+167
View File
@@ -0,0 +1,167 @@
import { describe, expect, it } from "vitest";
import {
MODULE_PACKAGE_FORMAT,
MODULE_PACKAGE_METADATA_FILENAME,
modulePackageArchiveFileName,
modulePackageRuntimePackage,
safeParseModulePackageMetadata,
safeParseModulePackageRuntimePackage,
safeParseModulePackageTarget,
} from "../src/index.js";
const digest = "a".repeat(64);
function metadata(overrides: Record<string, unknown> = {}): Record<string, unknown> {
return {
packageFormat: 1,
module: { id: "example", version: "0.1.0" },
target: "any",
runtime: { node: ">=24.11.0 <25" },
provenance: {
revision: "86ae907f59458508e3277666cdd214f5bd7c1078",
packager: { name: "@ekklesia/module-pack", version: "0.1.2" },
},
incorporates: { "@ekklesia/contracts": "0.1.2", "@ekklesia/module-sdk": "0.1.2" },
transitional: { databaseRuntime: { "drizzle-orm": "0.45.2" } },
files: [
{ path: "THIRD_PARTY_NOTICES", size: 10, sha256: digest },
{ path: "dist/server/index.js", size: 20, sha256: digest },
{ path: "ekklesia.module.json", size: 30, sha256: digest },
{ path: "package.json", size: 40, sha256: digest },
],
...overrides,
};
}
describe("Ekklesia Module Package v1 metadata (ADR 0049)", () => {
it("names the format, its metadata file, and the recommended archive file name", () => {
expect(MODULE_PACKAGE_FORMAT).toBe(1);
expect(MODULE_PACKAGE_METADATA_FILENAME).toBe("ekklesia.package.json");
expect(modulePackageArchiveFileName("example", "0.1.0", "any")).toBe(
"example-0.1.0-any.tar.gz",
);
});
it("accepts a complete any-target package description", () => {
expect(safeParseModulePackageMetadata(metadata()).success).toBe(true);
});
it("accepts a package that incorporates nothing and needs no transitional facts", () => {
const { transitional: _transitional, ...rest } = metadata();
expect(safeParseModulePackageMetadata({ ...rest, incorporates: {} }).success).toBe(true);
});
it.each([
["an unknown top-level field", { signature: "x" }],
["format 2", { packageFormat: 2 }],
["a missing source revision", { provenance: { packager: { name: "p", version: "1.0.0" } } }],
[
"an abbreviated revision",
{ provenance: { revision: "86ae907", packager: { name: "p", version: "1.0.0" } } },
],
["a non-semver module version", { module: { id: "example", version: "1.0" } }],
["an invalid module id", { module: { id: "Example", version: "0.1.0" } }],
["an invalid Node range", { runtime: { node: "soon" } }],
["native runtime facts on an any target", { runtime: { node: ">=24", nodeApi: 9 } }],
["a non-Ekklesia incorporated package", { incorporates: { zod: "4.4.3" } }],
[
"an incorporated range instead of a version",
{ incorporates: { "@ekklesia/module-sdk": "^0.1.0" } },
],
["an unknown transitional key", { transitional: { orm: {} } }],
["an unknown database runtime", { transitional: { databaseRuntime: { prisma: "6.0.0" } } }],
])("rejects %s", (_label, overrides) => {
expect(safeParseModulePackageMetadata(metadata(overrides)).success).toBe(false);
});
it.each([
["an absolute path", "/etc/passwd"],
["a parent segment", "dist/../../secret"],
["a dot segment", "./package.json"],
["an empty segment", "dist//index.js"],
["a backslash", "dist\\index.js"],
["a NUL byte", "dist/\u0000"],
["the metadata file itself", "ekklesia.package.json"],
])("rejects a file entry with %s", (_label, path) => {
const files = [{ path, size: 1, sha256: digest }];
expect(safeParseModulePackageMetadata(metadata({ files })).success).toBe(false);
});
it("requires file entries in byte-wise path order, without duplicates", () => {
const entry = (path: string) => ({ path, size: 1, sha256: digest });
expect(
safeParseModulePackageMetadata(metadata({ files: [entry("b"), entry("a")] })).success,
).toBe(false);
expect(
safeParseModulePackageMetadata(metadata({ files: [entry("a"), entry("a")] })).success,
).toBe(false);
// Byte-wise, not locale order: upper case sorts before lower case.
expect(
safeParseModulePackageMetadata(metadata({ files: [entry("THIRD"), entry("dist")] })).success,
).toBe(true);
});
it.each([
["an uppercase digest", { sha256: "A".repeat(64) }],
["a short digest", { sha256: "a".repeat(63) }],
["a negative size", { size: -1 }],
["a fractional size", { size: 1.5 }],
])("rejects a file entry with %s", (_label, override) => {
const files = [{ path: "package.json", size: 1, sha256: digest, ...override }];
expect(safeParseModulePackageMetadata(metadata({ files })).success).toBe(false);
});
});
describe("module package targets", () => {
it.each([
"any",
"linux-x64-glibc",
"linux-arm64-glibc",
"linux-x64-musl",
"darwin-arm64",
])("accepts %s", (target) => {
expect(safeParseModulePackageTarget(target).success).toBe(true);
});
it.each([
"",
"ANY",
"linux-x64",
"linux-x64-gnu",
"darwin-arm64-glibc",
"linux",
"any-x64",
])("rejects %j", (target) => {
expect(safeParseModulePackageTarget(target).success).toBe(false);
});
});
describe("the generated runtime package.json", () => {
it("holds exactly what module discovery reads", () => {
expect(
modulePackageRuntimePackage({ name: "@ekklesia/module-template", version: "0.1.0" }),
).toEqual({
name: "@ekklesia/module-template",
version: "0.1.0",
private: true,
type: "module",
ekklesia: { layer: "module" },
});
});
it("accepts the generated shape and nothing more", () => {
const generated = modulePackageRuntimePackage({ name: "@scope/example", version: "1.2.3" });
expect(safeParseModulePackageRuntimePackage(generated).success).toBe(true);
for (const extra of [
{ dependencies: { "@ekklesia/module-sdk": "workspace:*" } },
{ scripts: { build: "tsc" } },
{ exports: { ".": "./dist/server/index.js" } },
{ imports: { "#x": "./x.js" } },
]) {
expect(safeParseModulePackageRuntimePackage({ ...generated, ...extra }).success).toBe(false);
}
expect(safeParseModulePackageRuntimePackage({ ...generated, type: "commonjs" }).success).toBe(
false,
);
});
});
packages/module-pack/bin/ekklesia-pack-module.mjs
Executable
+25
View File
@@ -0,0 +1,25 @@
#!/usr/bin/env node
// Committed launcher, deliberately not generated.
//
// pnpm creates `.bin` links during `pnpm install`, which runs before Turborepo builds anything. A
// `bin` pointing straight at `dist/cli.js` therefore fails to link on a fresh clone — the compiled
// file does not exist yet — and the packager would not link in CI. This file always exists,
// so the link always succeeds; it defers to the compiled CLI at run time, by which point
// `dependsOn: ["^build"]` has produced it.
import { fileURLToPath } from "node:url";
const compiledCli = new URL("../dist/cli.js", import.meta.url);
try {
await import(compiledCli.href);
} catch (error) {
if (error?.code === "ERR_MODULE_NOT_FOUND") {
process.stderr.write(
`@ekklesia/module-pack is not built yet (${fileURLToPath(compiledCli)}).\n` +
"Run `pnpm build` from the repository root first.\n",
);
process.exitCode = 1;
} else {
throw error;
}
}
packages/module-pack/licenses/Apache-2.0.txt
+202
View File
@@ -0,0 +1,202 @@
Apache License
Version 2.0, January 2004
http://www.apache.org/licenses/
TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
1. Definitions.
"License" shall mean the terms and conditions for use, reproduction,
and distribution as defined by Sections 1 through 9 of this document.
"Licensor" shall mean the copyright owner or entity authorized by
the copyright owner that is granting the License.
"Legal Entity" shall mean the union of the acting entity and all
other entities that control, are controlled by, or are under common
control with that entity. For the purposes of this definition,
"control" means (i) the power, direct or indirect, to cause the
direction or management of such entity, whether by contract or
otherwise, or (ii) ownership of fifty percent (50%) or more of the
outstanding shares, or (iii) beneficial ownership of such entity.
"You" (or "Your") shall mean an individual or Legal Entity
exercising permissions granted by this License.
"Source" form shall mean the preferred form for making modifications,
including but not limited to software source code, documentation
source, and configuration files.
"Object" form shall mean any form resulting from mechanical
transformation or translation of a Source form, including but
not limited to compiled object code, generated documentation,
and conversions to other media types.
"Work" shall mean the work of authorship, whether in Source or
Object form, made available under the License, as indicated by a
copyright notice that is included in or attached to the work
(an example is provided in the Appendix below).
"Derivative Works" shall mean any work, whether in Source or Object
form, that is based on (or derived from) the Work and for which the
editorial revisions, annotations, elaborations, or other modifications
represent, as a whole, an original work of authorship. For the purposes
of this License, Derivative Works shall not include works that remain
separable from, or merely link (or bind by name) to the interfaces of,
the Work and Derivative Works thereof.
"Contribution" shall mean any work of authorship, including
the original version of the Work and any modifications or additions
to that Work or Derivative Works thereof, that is intentionally
submitted to Licensor for inclusion in the Work by the copyright owner
or by an individual or Legal Entity authorized to submit on behalf of
the copyright owner. For the purposes of this definition, "submitted"
means any form of electronic, verbal, or written communication sent
to the Licensor or its representatives, including but not limited to
communication on electronic mailing lists, source code control systems,
and issue tracking systems that are managed by, or on behalf of, the
Licensor for the purpose of discussing and improving the Work, but
excluding communication that is conspicuously marked or otherwise
designated in writing by the copyright owner as "Not a Contribution."
"Contributor" shall mean Licensor and any individual or Legal Entity
on behalf of whom a Contribution has been received by Licensor and
subsequently incorporated within the Work.
2. Grant of Copyright License. Subject to the terms and conditions of
this License, each Contributor hereby grants to You a perpetual,
worldwide, non-exclusive, no-charge, royalty-free, irrevocable
copyright license to reproduce, prepare Derivative Works of,
publicly display, publicly perform, sublicense, and distribute the
Work and such Derivative Works in Source or Object form.
3. Grant of Patent License. Subject to the terms and conditions of
this License, each Contributor hereby grants to You a perpetual,
worldwide, non-exclusive, no-charge, royalty-free, irrevocable
(except as stated in this section) patent license to make, have made,
use, offer to sell, sell, import, and otherwise transfer the Work,
where such license applies only to those patent claims licensable
by such Contributor that are necessarily infringed by their
Contribution(s) alone or by combination of their Contribution(s)
with the Work to which such Contribution(s) was submitted. If You
institute patent litigation against any entity (including a
cross-claim or counterclaim in a lawsuit) alleging that the Work
or a Contribution incorporated within the Work constitutes direct
or contributory patent infringement, then any patent licenses
granted to You under this License for that Work shall terminate
as of the date such litigation is filed.
4. Redistribution. You may reproduce and distribute copies of the
Work or Derivative Works thereof in any medium, with or without
modifications, and in Source or Object form, provided that You
meet the following conditions:
(a) You must give any other recipients of the Work or
Derivative Works a copy of this License; and
(b) You must cause any modified files to carry prominent notices
stating that You changed the files; and
(c) You must retain, in the Source form of any Derivative Works
that You distribute, all copyright, patent, trademark, and
attribution notices from the Source form of the Work,
excluding those notices that do not pertain to any part of
the Derivative Works; and
(d) If the Work includes a "NOTICE" text file as part of its
distribution, then any Derivative Works that You distribute must
include a readable copy of the attribution notices contained
within such NOTICE file, excluding those notices that do not
pertain to any part of the Derivative Works, in at least one
of the following places: within a NOTICE text file distributed
as part of the Derivative Works; within the Source form or
documentation, if provided along with the Derivative Works; or,
within a display generated by the Derivative Works, if and
wherever such third-party notices normally appear. The contents
of the NOTICE file are for informational purposes only and
do not modify the License. You may add Your own attribution
notices within Derivative Works that You distribute, alongside
or as an addendum to the NOTICE text from the Work, provided
that such additional attribution notices cannot be construed
as modifying the License.
You may add Your own copyright statement to Your modifications and
may provide additional or different license terms and conditions
for use, reproduction, or distribution of Your modifications, or
for any such Derivative Works as a whole, provided Your use,
reproduction, and distribution of the Work otherwise complies with
the conditions stated in this License.
5. Submission of Contributions. Unless You explicitly state otherwise,
any Contribution intentionally submitted for inclusion in the Work
by You to the Licensor shall be under the terms and conditions of
this License, without any additional terms or conditions.
Notwithstanding the above, nothing herein shall supersede or modify
the terms of any separate license agreement you may have executed
with Licensor regarding such Contributions.
6. Trademarks. This License does not grant permission to use the trade
names, trademarks, service marks, or product names of the Licensor,
except as required for reasonable and customary use in describing the
origin of the Work and reproducing the content of the NOTICE file.
7. Disclaimer of Warranty. Unless required by applicable law or
agreed to in writing, Licensor provides the Work (and each
Contributor provides its Contributions) on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
implied, including, without limitation, any warranties or conditions
of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
PARTICULAR PURPOSE. You are solely responsible for determining the
appropriateness of using or redistributing the Work and assume any
risks associated with Your exercise of permissions under this License.
8. Limitation of Liability. In no event and under no legal theory,
whether in tort (including negligence), contract, or otherwise,
unless required by applicable law (such as deliberate and grossly
negligent acts) or agreed to in writing, shall any Contributor be
liable to You for damages, including any direct, indirect, special,
incidental, or consequential damages of any character arising as a
result of this License or out of the use or inability to use the
Work (including but not limited to damages for loss of goodwill,
work stoppage, computer failure or malfunction, or any and all
other commercial damages or losses), even if such Contributor
has been advised of the possibility of such damages.
9. Accepting Warranty or Additional Liability. While redistributing
the Work or Derivative Works thereof, You may choose to offer,
and charge a fee for, acceptance of support, warranty, indemnity,
or other liability obligations and/or rights consistent with this
License. However, in accepting such obligations, You may act only
on Your own behalf and on Your sole responsibility, not on behalf
of any other Contributor, and only if You agree to indemnify,
defend, and hold each Contributor harmless for any liability
incurred by, or claims asserted against, such Contributor by reason
of your accepting any such warranty or additional liability.
END OF TERMS AND CONDITIONS
APPENDIX: How to apply the Apache License to your work.
To apply the Apache License to your work, attach the following
boilerplate notice, with the fields enclosed by brackets "[]"
replaced with your own identifying information. (Don't include
the brackets!) The text should be enclosed in the appropriate
comment syntax for the file format. We also recommend that a
file or class name and description of purpose be included on the
same "printed page" as the copyright notice for easier
identification within third-party archives.
Copyright [yyyy] [name of copyright owner]
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
packages/module-pack/package.json
+46
View File
@@ -0,0 +1,46 @@
{
"name": "@ekklesia/module-pack",
"version": "0.1.2",
"type": "module",
"files": [
"bin",
"dist",
"licenses"
],
"bin": {
"ekklesia-pack-module": "./bin/ekklesia-pack-module.mjs"
},
"exports": {
".": {
"types": "./dist/index.d.ts",
"import": "./dist/index.js"
}
},
"scripts": {
"build": "tsc --project tsconfig.build.json",
"lint": "biome lint .",
"prepack": "pnpm --workspace-root exec turbo run build --filter=@ekklesia/module-pack",
"test": "vitest run",
"typecheck": "tsc --noEmit --project tsconfig.json"
},
"dependencies": {
"@ekklesia/contracts": "workspace:*",
"@ekklesia/module-ui-build": "workspace:*",
"esbuild": "0.28.1",
"semver": "7.8.5"
},
"devDependencies": {
"@biomejs/biome": "2.5.3",
"@ekklesia/core": "workspace:*",
"@ekklesia/module-sdk": "workspace:*",
"@ekklesia/module-template": "workspace:*",
"@types/node": "24.13.3",
"@types/semver": "7.7.1",
"drizzle-orm": "0.45.2",
"typescript": "7.0.2",
"vitest": "4.1.10"
},
"ekklesia": {
"layer": "tooling"
}
}
packages/module-pack/src/cli.ts
+102
View File
@@ -0,0 +1,102 @@
#!/usr/bin/env node
import path from "node:path";
import process from "node:process";
import { pathToFileURL } from "node:url";
import { formatMessages } from "esbuild";
import { ModulePackError, type PackModuleOptions, packModule } from "./pack-module.js";
/**
* `ekklesia-pack-module` — the canonical Ekklesia module packager (ADR 0049).
*
* ekklesia-pack-module --module <built module directory> --target any --out <directory>
* --source-revision <full commit id> [--repository <url>]
*
* Writes `<id>-<version>-any.tar.gz` and its `.sha256` line to the output directory, and nothing
* else anywhere. Run it in the module's CI after building the module.
*/
class PackArgumentsError extends Error {}
const USAGE =
"Usage: ekklesia-pack-module --module <directory> --target any --out <directory> --source-revision <commit> [--repository <url>]";
const OPTIONS: Readonly<Record<string, keyof PackModuleOptions>> = Object.freeze({
"--module": "moduleDirectory",
"--out": "outDirectory",
"--repository": "repository",
"--source-revision": "sourceRevision",
"--target": "target",
});
export function parsePackModuleArguments(argv: readonly string[]): PackModuleOptions {
const values = new Map<keyof PackModuleOptions, string>();
for (let index = 0; index < argv.length; index += 2) {
const option = OPTIONS[argv[index] ?? ""];
const value = argv[index + 1];
if (
option === undefined ||
value === undefined ||
value.startsWith("--") ||
values.has(option)
) {
throw new PackArgumentsError(USAGE);
}
values.set(option, value);
}
const required = ["moduleDirectory", "outDirectory", "sourceRevision", "target"] as const;
if (required.some((option) => !values.has(option))) {
throw new PackArgumentsError(USAGE);
}
const repository = values.get("repository");
return {
moduleDirectory: values.get("moduleDirectory") ?? "",
outDirectory: values.get("outDirectory") ?? "",
...(repository === undefined ? {} : { repository }),
sourceRevision: values.get("sourceRevision") ?? "",
target: values.get("target") ?? "",
};
}
async function main(): Promise<number> {
try {
const result = await packModule(parsePackModuleArguments(process.argv.slice(2)));
process.stdout.write(
`Packed ${result.moduleId}@${result.version} (target ${result.target}): ${path.basename(result.archivePath)}, ${result.archiveBytes} bytes, ${result.fileCount} files\n`,
);
process.stdout.write(`sha256 ${result.sha256}\n`);
return 0;
} catch (error) {
if (error instanceof PackArgumentsError) {
process.stderr.write(`${error.message}\n`);
return 64;
}
if (error instanceof ModulePackError) {
process.stderr.write(`[${error.code}] ${error.message}\n`);
if (error.diagnostics.length > 0) {
const formatted = await formatMessages([...error.diagnostics], {
color: false,
kind: "error",
});
process.stderr.write(formatted.join(""));
}
return 1;
}
process.stderr.write("Module packaging failed unexpectedly.\n");
return 1;
}
}
function isDirectExecution(): boolean {
const entrypoint = process.argv[1];
// Run through the committed launcher, which imports this file, or directly.
return (
entrypoint !== undefined &&
(import.meta.url === pathToFileURL(path.resolve(entrypoint)).href ||
path.basename(entrypoint) === "ekklesia-pack-module.mjs" ||
path.basename(entrypoint) === "ekklesia-pack-module")
);
}
if (isDirectExecution()) {
process.exitCode = await main();
}
packages/module-pack/src/index.ts
+1
View File
@@ -0,0 +1 @@
export * from "./pack-module.js";
packages/module-pack/src/notices.ts
+176
View File
@@ -0,0 +1,176 @@
import { readdir, readFile } from "node:fs/promises";
import path from "node:path";
import { fileURLToPath } from "node:url";
import type { BundledPackage } from "./packages.js";
/**
* `THIRD_PARTY_NOTICES` (ADR 0049): the licence and notice text that redistributing a module
* package requires, generated from what was actually bundled.
*
* - **Discovery.** Every source file in every bundle (the backend, and each browser artifact) is
* traced to the package it belongs to. Packages installed under `node_modules` are third-party
* software and are credited here. The Ekklesia platform's own packages, and the module itself,
* are not third-party software: the platform packages appear in `ekklesia.package.json` under
* `incorporates`.
* - **Content.** For each third-party package: its name, version, declared licence, where in the
* package it is used, and the complete text of every licence, copying, copyright, and notice file
* it ships at its root, plus the file a `SEE LICENSE IN` declaration names.
* - **Duplicates.** A package bundled into several artifacts is credited once, listing every
* artifact kind it is used in. Different packages are never merged, even when their texts match,
* so each component keeps its own notice.
* - **Gaps fail.** A package that declares no licence, or whose licence cannot be established,
* stops packaging. When a package ships no licence file, the standard text of its declared licence
* is supplied only if that licence's standard text is complete without project-specific lines
* (Apache-2.0). An MIT, ISC, or BSD package without its file is refused, because its copyright line
* exists only in that file.
*
* Everything is sorted, and line endings are normalized, so the file is byte-identical for identical
* inputs.
*/
export type NoticeErrorCode = "MODULE_PACK_LICENCE_TEXT_MISSING" | "MODULE_PACK_LICENCE_UNKNOWN";
export class NoticeError extends Error {
readonly code: NoticeErrorCode;
readonly component: string;
constructor(code: NoticeErrorCode, component: string) {
super(code);
this.name = "NoticeError";
this.code = code;
this.component = component;
}
}
export type NoticeUse = "backend" | "browser";
export interface NoticeComponent {
readonly bundled: BundledPackage;
readonly uses: ReadonlySet<NoticeUse>;
}
/** Licences whose standard text is complete without any project-specific line, so supplying it
* for a package that ships none is faithful. Checked into this package (`licenses/`). */
const STANDARD_TEXTS: Readonly<Record<string, string>> = Object.freeze({
"Apache-2.0": "Apache-2.0.txt",
});
const LICENCE_FILE =
/^(?:licen[cs]e(?:[-._][a-z0-9.-]+)?|copying(?:[-._][a-z0-9.-]+)?|notice(?:[-._][a-z0-9.-]+)?|copyright(?:[-._][a-z0-9.-]+)?)$/iu;
const RULE = "-".repeat(80);
function licenceOf(bundled: BundledPackage): string | undefined {
const declared = bundled.manifest.license;
if (typeof declared === "string" && declared.trim() !== "") {
return declared.trim();
}
if (typeof declared === "object" && declared !== null && "type" in declared) {
const type = (declared as { type?: unknown }).type;
return typeof type === "string" && type.trim() !== "" ? type.trim() : undefined;
}
return undefined;
}
function normalized(text: string): string {
return `${text.replaceAll("\r\n", "\n").replaceAll("\r", "\n").replace(/\s+$/u, "")}\n`;
}
function compare(left: string, right: string): number {
return left < right ? -1 : left > right ? 1 : 0;
}
async function licenceTexts(
bundled: BundledPackage,
licence: string,
): Promise<readonly { readonly title: string; readonly text: string }[]> {
const names = (await readdir(bundled.root, { withFileTypes: true }))
.filter((entry) => entry.isFile() && LICENCE_FILE.test(entry.name))
.map((entry) => entry.name)
.sort(compare);
const seeFile = /^SEE LICEN[CS]E IN (.+)$/iu.exec(licence)?.[1]?.trim();
if (seeFile && !names.includes(seeFile)) {
const resolved = path.resolve(bundled.root, seeFile);
if (resolved.startsWith(`${bundled.root}${path.sep}`)) {
names.push(seeFile);
}
}
const texts: { title: string; text: string }[] = [];
for (const name of names) {
texts.push({
text: normalized(await readFile(path.join(bundled.root, name), "utf8")),
title: name,
});
}
return texts;
}
const packageRoot = path.resolve(path.dirname(fileURLToPath(import.meta.url)), "..");
async function standardText(licence: string): Promise<string | undefined> {
const file = STANDARD_TEXTS[licence];
return file === undefined
? undefined
: normalized(await readFile(path.join(packageRoot, "licenses", file), "utf8"));
}
export async function thirdPartyNotices(
components: ReadonlyMap<string, NoticeComponent>,
): Promise<string> {
const ordered = [...components.values()].sort(
(left, right) =>
compare(left.bundled.name, right.bundled.name) ||
compare(left.bundled.version, right.bundled.version),
);
const sections: string[] = [];
for (const { bundled, uses } of ordered) {
const component = `${bundled.name}@${bundled.version}`;
const licence = licenceOf(bundled);
if (licence === undefined || /^UNLICEN[CS]ED$/iu.test(licence)) {
throw new NoticeError("MODULE_PACK_LICENCE_UNKNOWN", component);
}
let texts = await licenceTexts(bundled, licence);
// A NOTICE file alone is not the licence: the licence text must still come from somewhere.
if (!texts.some((text) => !/^notice/iu.test(text.title))) {
const standard = await standardText(licence);
if (standard === undefined) {
throw new NoticeError("MODULE_PACK_LICENCE_TEXT_MISSING", component);
}
texts = [
...texts,
{
text: `The package ships no licence file. This is the standard text of the licence it declares (${licence}).\n\n${standard}`,
title: `Standard ${licence} text`,
},
];
}
sections.push(
[
RULE,
`${bundled.name} ${bundled.version}`,
`License: ${licence}`,
`Used in: ${[...uses].sort(compare).join(", ")}`,
RULE,
"",
...texts.flatMap((text) => [`[${text.title}]`, "", text.text]),
].join("\n"),
);
}
const header = [
"THIRD-PARTY SOFTWARE NOTICES",
"",
"This module package includes the third-party software listed below. Each entry names the",
"component, its version, its declared licence, where in the package it is used, and the licence",
"and notice text that came with it.",
"",
"The Ekklesia platform's own packages are not third-party software. They are listed in",
'ekklesia.package.json under "incorporates".',
"",
ordered.length === 0
? "This module package includes no third-party software."
: `${ordered.length} ${ordered.length === 1 ? "component" : "components"}.`,
"",
].join("\n");
return `${[header, ...sections].join("\n").replace(/\n+$/u, "")}\n`;
}
packages/module-pack/src/pack-module.ts
+588
View File
@@ -0,0 +1,588 @@
import { createHash } from "node:crypto";
import { lstat, mkdir, readdir, readFile, realpath, rename, writeFile } from "node:fs/promises";
import { isBuiltin } from "node:module";
import path from "node:path";
import { fileURLToPath } from "node:url";
import {
MODULE_PACKAGE_ANY_TARGET,
MODULE_PACKAGE_FORMAT,
MODULE_PACKAGE_METADATA_FILENAME,
MODULE_PACKAGE_NOTICES_FILENAME,
type ModuleManifest,
modulePackageArchiveFileName,
modulePackageRuntimePackage,
safeParseModuleManifest,
safeParseModulePackageMetadata,
} from "@ekklesia/contracts";
import { analyzeModuleUi, ModuleUiBuildError } from "@ekklesia/module-ui-build";
import { build, type Message, type Plugin } from "esbuild";
import { minVersion, satisfies, validRange } from "semver";
import { NoticeError, type NoticeComponent, type NoticeUse, thirdPartyNotices } from "./notices.js";
import { BundledPackageIndex, isNativeOrPlatformSpecific } from "./packages.js";
import { deterministicTarGz, TarPathError } from "./tar.js";
/**
* The canonical Ekklesia module packager (ADR 0049): turns a built module directory into one
* deterministic Module Package v1 archive and its `.sha256` line.
*
* It reads the module directory and the dependencies its workspace installed, and writes only the
* two output files. It never installs, links, or deploys anything, so the workspace it runs in is
* left exactly as it was. In this release it produces the portable `any` target only.
*/
export type ModulePackErrorCode =
| "MODULE_PACK_BUNDLE_FAILED"
| "MODULE_PACK_DATABASE_RUNTIME_CONFLICT"
| "MODULE_PACK_ENTRYPOINT_MISSING"
| "MODULE_PACK_ENTRYPOINT_UNSUPPORTED"
| "MODULE_PACK_EXTERNAL_DEPENDENCY"
| "MODULE_PACK_FORBIDDEN_FILE"
| "MODULE_PACK_IDENTITY_MISMATCH"
| "MODULE_PACK_INCORPORATED_CONFLICT"
| "MODULE_PACK_LAYOUT_CONFLICT"
| "MODULE_PACK_LICENCE_TEXT_MISSING"
| "MODULE_PACK_LICENCE_UNKNOWN"
| "MODULE_PACK_MANIFEST_INVALID"
| "MODULE_PACK_METADATA_INVALID"
| "MODULE_PACK_MIGRATIONS_INVALID"
| "MODULE_PACK_NATIVE_DEPENDENCY"
| "MODULE_PACK_OUTPUT_FAILED"
| "MODULE_PACK_PATH_UNSUPPORTED"
| "MODULE_PACK_REVISION_INVALID"
| "MODULE_PACK_RUNTIME_UNKNOWN"
| "MODULE_PACK_SDK_INCOMPATIBLE"
| "MODULE_PACK_TARGET_UNSUPPORTED"
| "MODULE_PACK_UI_STALE";
const messages: Readonly<Record<ModulePackErrorCode, string>> = Object.freeze({
MODULE_PACK_BUNDLE_FAILED:
"The backend could not be bundled into self-contained files. Every import must resolve and bundle cleanly.",
MODULE_PACK_DATABASE_RUNTIME_CONFLICT: "The backend bundles more than one Drizzle version.",
MODULE_PACK_ENTRYPOINT_MISSING:
"A declared backend entrypoint does not exist. Build the module before packaging it.",
MODULE_PACK_ENTRYPOINT_UNSUPPORTED:
"Backend entrypoints must be compiled .js files to be bundled into a Module Package v1.",
MODULE_PACK_EXTERNAL_DEPENDENCY:
"The backend bundle would import a dependency at run time; an any-target package bundles everything.",
MODULE_PACK_FORBIDDEN_FILE:
"The package would contain a source, test, declaration, source-map, or dependency file.",
MODULE_PACK_IDENTITY_MISMATCH:
"package.json must declare a name and the same version as ekklesia.module.json.",
MODULE_PACK_INCORPORATED_CONFLICT:
"The package bundles more than one version of an Ekklesia package.",
MODULE_PACK_LAYOUT_CONFLICT: "Two package files would have the same path.",
MODULE_PACK_LICENCE_TEXT_MISSING:
"A bundled package ships no licence text, and its licence has no complete standard text to supply.",
MODULE_PACK_LICENCE_UNKNOWN: "A bundled package declares no licence that permits redistribution.",
MODULE_PACK_MANIFEST_INVALID: "ekklesia.module.json is missing or invalid.",
MODULE_PACK_METADATA_INVALID: "The generated package metadata failed its own contract.",
MODULE_PACK_MIGRATIONS_INVALID:
"The migrations directory must hold only regular files and directories.",
MODULE_PACK_NATIVE_DEPENDENCY:
"The backend bundles a native or platform-specific dependency, which an any-target package cannot contain.",
MODULE_PACK_OUTPUT_FAILED: "The package could not be written to the output directory.",
MODULE_PACK_PATH_UNSUPPORTED:
"A package path is not portable printable ASCII that fits the archive format.",
MODULE_PACK_REVISION_INVALID: "The source revision must be a full 40- or 64-character commit id.",
MODULE_PACK_RUNTIME_UNKNOWN:
"The Node.js range could not be determined. Run inside an Ekklesia workspace, or pass one.",
MODULE_PACK_SDK_INCOMPATIBLE:
"The bundled Module SDK is outside the manifest's own compatibility.moduleSdk range.",
MODULE_PACK_TARGET_UNSUPPORTED: "This packager produces the any target only.",
MODULE_PACK_UI_STALE:
"A declared browser artifact is missing or out of date. Build the module before packaging it.",
});
export class ModulePackError extends Error {
readonly code: ModulePackErrorCode;
/** The package or path the failure concerns, when there is one. A module author's own build
* input, never a secret. */
readonly detail?: string;
/** esbuild's structured diagnostics, for a command line to print. */
readonly diagnostics: readonly Message[];
constructor(code: ModulePackErrorCode, detail?: string, diagnostics: readonly Message[] = []) {
super(detail === undefined ? messages[code] : `${messages[code]} (${detail})`);
this.name = "ModulePackError";
this.code = code;
if (detail !== undefined) {
this.detail = detail;
}
this.diagnostics = diagnostics;
}
}
export interface PackModuleOptions {
/** A built module directory: its manifest, package.json, and compiled entrypoints. */
readonly moduleDirectory: string;
/** `any` only, in this release. */
readonly target: string;
readonly outDirectory: string;
/** The full commit id the module was built from. */
readonly sourceRevision: string;
/** Informational only; recorded as given. */
readonly repository?: string;
/** The Node.js range the package is built for. Defaults to the enclosing workspace's
* `engines.node`, the toolchain the module was built and tested with. */
readonly nodeRange?: string;
}
export interface ModulePackResult {
readonly archivePath: string;
readonly sha256Path: string;
readonly sha256: string;
readonly archiveBytes: number;
readonly moduleId: string;
readonly version: string;
readonly target: string;
/** Payload files, the metadata file not included. */
readonly fileCount: number;
}
const NATIVE_GUARD = "ekklesia-native-addon-guard";
/** Refuses a compiled addon the moment the bundle reaches one. */
const nativeAddonGuard: Plugin = {
name: NATIVE_GUARD,
setup(pluginBuild) {
pluginBuild.onLoad({ filter: /\.node$/i }, () => ({
errors: [{ text: "A compiled Node.js addon cannot be bundled into an any-target package." }],
}));
},
};
/** Bundled CommonJS code that requires a Node built-in needs a `require` in module scope. Nothing
* else can be required at run time: every other dependency is inside the bundle, and the packager
* refuses a bundle that would import anything but a built-in. */
const REQUIRE_BANNER =
'import { createRequire as __ekklesiaCreateRequire } from "node:module";\nconst require = __ekklesiaCreateRequire(import.meta.url);';
function compare(left: string, right: string): number {
return left < right ? -1 : left > right ? 1 : 0;
}
function sha256(data: Uint8Array): string {
return createHash("sha256").update(data).digest("hex");
}
function isRecord(value: unknown): value is Record<string, unknown> {
return typeof value === "object" && value !== null && !Array.isArray(value);
}
async function readJson(file: string): Promise<unknown> {
return JSON.parse(await readFile(file, "utf8")) as unknown;
}
const ownManifestPath = path.resolve(
path.dirname(fileURLToPath(import.meta.url)),
"..",
"package.json",
);
async function packagerIdentity(): Promise<{ readonly name: string; readonly version: string }> {
const manifest = (await readJson(ownManifestPath)) as Record<string, unknown>;
return { name: String(manifest.name), version: String(manifest.version) };
}
async function workspaceNodeRange(moduleDirectory: string): Promise<string> {
let directory = moduleDirectory;
for (;;) {
try {
await lstat(path.join(directory, "pnpm-workspace.yaml"));
const manifest = await readJson(path.join(directory, "package.json"));
const engines = isRecord(manifest) ? manifest.engines : undefined;
const range = isRecord(engines) ? engines.node : undefined;
if (typeof range === "string") {
return range;
}
break;
} catch {
const parent = path.dirname(directory);
if (parent === directory) {
break;
}
directory = parent;
}
}
throw new ModulePackError("MODULE_PACK_RUNTIME_UNKNOWN");
}
/** `./dist/server/index.js` → `dist/server/index.js`, inside the module directory. */
function moduleRelative(declared: string): string {
return path.posix.normalize(declared.replace(/^\.\//u, ""));
}
function portable(relative: string): boolean {
const segments = relative.split("/");
return (
/^[\x20-\x7e]+$/u.test(relative) &&
!relative.includes("\\") &&
segments.every((segment) => segment !== "" && segment !== "." && segment !== "..")
);
}
function forbidden(relative: string): boolean {
return (
/(^|\/)(src|tests?|node_modules)\//u.test(relative) ||
/\.(map|d\.ts|d\.mts|d\.cts)$/u.test(relative)
);
}
async function readTree(root: string, relative: string, files: Map<string, Buffer>): Promise<void> {
const directory = path.join(root, relative);
for (const entry of (await readdir(directory, { withFileTypes: true })).sort((left, right) =>
compare(left.name, right.name),
)) {
const childRelative = `${relative}/${entry.name}`;
const stats = await lstat(path.join(directory, entry.name));
if (stats.isDirectory()) {
await readTree(root, childRelative, files);
} else if (stats.isFile()) {
files.set(childRelative, await readFile(path.join(directory, entry.name)));
} else {
throw new ModulePackError("MODULE_PACK_MIGRATIONS_INVALID", childRelative);
}
}
}
interface BackendBundle {
readonly outputs: ReadonlyMap<string, Buffer>;
readonly inputs: readonly string[];
}
async function bundleBackend(
moduleDirectory: string,
manifest: ModuleManifest,
nodeRange: string,
): Promise<BackendBundle> {
const declared = [
manifest.entrypoints.server,
manifest.entrypoints.schema,
manifest.entrypoints.jobs,
].filter((entrypoint): entrypoint is string => entrypoint !== undefined);
const entryPoints: { in: string; out: string }[] = [];
for (const entrypoint of declared) {
const relative = moduleRelative(entrypoint);
if (!relative.endsWith(".js")) {
throw new ModulePackError("MODULE_PACK_ENTRYPOINT_UNSUPPORTED", relative);
}
try {
if (!(await lstat(path.join(moduleDirectory, relative))).isFile()) {
throw new Error("not a file");
}
} catch {
throw new ModulePackError("MODULE_PACK_ENTRYPOINT_MISSING", relative);
}
entryPoints.push({ in: `./${relative}`, out: relative.slice(0, -".js".length) });
}
const minimum = minVersion(nodeRange);
let result: Awaited<ReturnType<typeof build>>;
try {
result = await build({
absWorkingDir: moduleDirectory,
banner: { js: REQUIRE_BANNER },
bundle: true,
chunkNames: "dist/chunks/[name]-[hash]",
entryPoints,
format: "esm",
// Licence comments stay in the code as well; THIRD_PARTY_NOTICES is what satisfies the
// licences, never these comments alone.
legalComments: "eof",
logLevel: "silent",
metafile: true,
minify: false,
outdir: moduleDirectory,
platform: "node",
plugins: [nativeAddonGuard],
sourcemap: false,
splitting: true,
target: `node${minimum?.version ?? "24"}`,
write: false,
});
} catch (error) {
const errors =
isRecord(error) && Array.isArray(error.errors) ? (error.errors as Message[]) : [];
const native = errors.some((message) => message.pluginName === NATIVE_GUARD);
throw new ModulePackError(
native ? "MODULE_PACK_NATIVE_DEPENDENCY" : "MODULE_PACK_BUNDLE_FAILED",
undefined,
errors,
);
}
// A warning is a bundle that might not run: a require it could not follow, an import it could
// not see. An installed module has nothing to fall back on, so a warning fails packaging.
if (result.warnings.length > 0) {
throw new ModulePackError("MODULE_PACK_BUNDLE_FAILED", undefined, result.warnings);
}
const { metafile, outputFiles } = result;
if (metafile === undefined || outputFiles === undefined) {
throw new ModulePackError("MODULE_PACK_BUNDLE_FAILED");
}
for (const output of Object.values(metafile.outputs)) {
for (const imported of output.imports) {
if (imported.external && !isBuiltin(imported.path)) {
throw new ModulePackError("MODULE_PACK_EXTERNAL_DEPENDENCY", imported.path);
}
}
}
const outputs = new Map<string, Buffer>();
for (const outputFile of outputFiles) {
outputs.set(
path.relative(moduleDirectory, outputFile.path).split(path.sep).join("/"),
Buffer.from(outputFile.contents),
);
}
return {
inputs: Object.keys(metafile.inputs).map((input) => path.resolve(moduleDirectory, input)),
outputs,
};
}
export async function packModule(options: PackModuleOptions): Promise<ModulePackResult> {
if (options.target !== MODULE_PACKAGE_ANY_TARGET) {
throw new ModulePackError("MODULE_PACK_TARGET_UNSUPPORTED", options.target);
}
if (!/^(?:[0-9a-f]{40}|[0-9a-f]{64})$/u.test(options.sourceRevision)) {
throw new ModulePackError("MODULE_PACK_REVISION_INVALID");
}
const moduleDirectory = await realpath(path.resolve(options.moduleDirectory)).catch(() => {
throw new ModulePackError("MODULE_PACK_MANIFEST_INVALID");
});
const manifestBytes = await readFile(path.join(moduleDirectory, "ekklesia.module.json")).catch(
() => {
throw new ModulePackError("MODULE_PACK_MANIFEST_INVALID");
},
);
let manifestInput: unknown;
try {
manifestInput = JSON.parse(manifestBytes.toString("utf8"));
} catch {
throw new ModulePackError("MODULE_PACK_MANIFEST_INVALID");
}
const parsedManifest = safeParseModuleManifest(manifestInput);
if (!parsedManifest.success) {
throw new ModulePackError("MODULE_PACK_MANIFEST_INVALID");
}
const manifest = parsedManifest.data;
const developmentPackage = await readJson(path.join(moduleDirectory, "package.json")).catch(
() => undefined,
);
if (
!isRecord(developmentPackage) ||
typeof developmentPackage.name !== "string" ||
developmentPackage.name === "" ||
developmentPackage.version !== manifest.version
) {
throw new ModulePackError("MODULE_PACK_IDENTITY_MISMATCH");
}
const nodeRange = options.nodeRange ?? (await workspaceNodeRange(moduleDirectory));
if (validRange(nodeRange) === null) {
throw new ModulePackError("MODULE_PACK_RUNTIME_UNKNOWN");
}
// 1. The backend, bundled with every pure-JavaScript dependency it uses.
const backend = await bundleBackend(moduleDirectory, manifest, nodeRange);
// 2. The browser artifacts, exactly as built, with what went into each.
let ui: Awaited<ReturnType<typeof analyzeModuleUi>>;
try {
ui = await analyzeModuleUi({ moduleDirectory });
} catch (error) {
throw new ModulePackError(
"MODULE_PACK_UI_STALE",
error instanceof ModuleUiBuildError ? error.code : undefined,
);
}
// 3. Every bundled source file traced to its package.
const index = new BundledPackageIndex();
const components = new Map<
string,
{ bundled: Awaited<ReturnType<typeof index.packageOf>>; uses: Set<NoticeUse> }
>();
const incorporates = new Map<string, string>();
const drizzleVersions = new Set<string>();
const sources: { readonly file: string; readonly use: NoticeUse }[] = [
...backend.inputs.map((file) => ({ file, use: "backend" as const })),
...ui.artifacts.flatMap((artifact) =>
artifact.inputs.map((file) => ({ file, use: "browser" as const })),
),
];
for (const { file, use } of sources) {
const bundled = await index.packageOf(file);
if (bundled === undefined || bundled.root === moduleDirectory) {
continue;
}
if (!bundled.thirdParty) {
if (bundled.name.startsWith("@ekklesia/")) {
const known = incorporates.get(bundled.name);
if (known !== undefined && known !== bundled.version) {
throw new ModulePackError("MODULE_PACK_INCORPORATED_CONFLICT", bundled.name);
}
incorporates.set(bundled.name, bundled.version);
}
continue;
}
if (use === "backend" && bundled.name === "drizzle-orm") {
drizzleVersions.add(bundled.version);
}
const key = `${bundled.name}@${bundled.version}`;
const existing = components.get(key);
if (existing) {
existing.uses.add(use);
} else {
if (use === "backend" && (await isNativeOrPlatformSpecific(bundled))) {
throw new ModulePackError("MODULE_PACK_NATIVE_DEPENDENCY", key);
}
components.set(key, { bundled, uses: new Set([use]) });
}
}
if (drizzleVersions.size > 1) {
throw new ModulePackError("MODULE_PACK_DATABASE_RUNTIME_CONFLICT");
}
const sdk = incorporates.get("@ekklesia/module-sdk");
if (sdk !== undefined && !satisfies(sdk, manifest.compatibility.moduleSdk)) {
throw new ModulePackError("MODULE_PACK_SDK_INCOMPATIBLE", sdk);
}
// 4. The payload.
const files = new Map<string, Buffer>();
const add = (relative: string, data: Buffer): void => {
if (!portable(relative)) {
throw new ModulePackError("MODULE_PACK_PATH_UNSUPPORTED", relative);
}
if (forbidden(relative)) {
throw new ModulePackError("MODULE_PACK_FORBIDDEN_FILE", relative);
}
if (files.has(relative) || relative === MODULE_PACKAGE_METADATA_FILENAME) {
throw new ModulePackError("MODULE_PACK_LAYOUT_CONFLICT", relative);
}
files.set(relative, data);
};
for (const [relative, data] of backend.outputs) {
add(relative, data);
}
for (const artifact of ui.artifacts) {
if (!artifact.builtArtifactCurrent) {
throw new ModulePackError(
"MODULE_PACK_UI_STALE",
path.relative(moduleDirectory, artifact.outputFile),
);
}
add(
path.relative(moduleDirectory, artifact.outputFile).split(path.sep).join("/"),
await readFile(artifact.outputFile),
);
}
if (manifest.entrypoints.migrations !== undefined) {
const migrations = new Map<string, Buffer>();
try {
await readTree(moduleDirectory, moduleRelative(manifest.entrypoints.migrations), migrations);
} catch (error) {
if (error instanceof ModulePackError) {
throw error;
}
throw new ModulePackError("MODULE_PACK_MIGRATIONS_INVALID");
}
for (const [relative, data] of migrations) {
add(relative, data);
}
}
add("ekklesia.module.json", manifestBytes);
add(
"package.json",
Buffer.from(
`${JSON.stringify(
modulePackageRuntimePackage({ name: developmentPackage.name, version: manifest.version }),
null,
2,
)}\n`,
),
);
try {
add(
MODULE_PACKAGE_NOTICES_FILENAME,
Buffer.from(await thirdPartyNotices(components as ReadonlyMap<string, NoticeComponent>)),
);
} catch (error) {
if (error instanceof NoticeError) {
throw new ModulePackError(error.code, error.component);
}
throw error;
}
// 5. The metadata, checked against its own contract before anything is written.
const listed = [...files.entries()]
.sort(([left], [right]) => compare(left, right))
.map(([relative, data]) => ({ path: relative, sha256: sha256(data), size: data.length }));
const metadata = {
packageFormat: MODULE_PACKAGE_FORMAT,
module: { id: manifest.id, version: manifest.version },
target: MODULE_PACKAGE_ANY_TARGET,
runtime: { node: nodeRange },
provenance: {
revision: options.sourceRevision,
...(options.repository === undefined ? {} : { repository: options.repository }),
packager: await packagerIdentity(),
},
incorporates: Object.fromEntries(
[...incorporates].sort(([left], [right]) => compare(left, right)),
),
...(drizzleVersions.size === 1
? { transitional: { databaseRuntime: { "drizzle-orm": [...drizzleVersions][0] } } }
: {}),
files: listed.map((file) => ({ path: file.path, size: file.size, sha256: file.sha256 })),
};
if (!safeParseModulePackageMetadata(metadata).success) {
throw new ModulePackError("MODULE_PACK_METADATA_INVALID");
}
const payload = new Map(files);
payload.set(
MODULE_PACKAGE_METADATA_FILENAME,
Buffer.from(`${JSON.stringify(metadata, null, 2)}\n`),
);
// 6. The archive and its digest.
let archive: Buffer;
try {
archive = deterministicTarGz(manifest.id, payload);
} catch (error) {
if (error instanceof TarPathError) {
throw new ModulePackError("MODULE_PACK_PATH_UNSUPPORTED", error.path);
}
throw error;
}
const digest = sha256(archive);
const fileName = modulePackageArchiveFileName(
manifest.id,
manifest.version,
MODULE_PACKAGE_ANY_TARGET,
);
const outDirectory = path.resolve(options.outDirectory);
const archivePath = path.join(outDirectory, fileName);
const sha256Path = `${archivePath}.sha256`;
try {
await mkdir(outDirectory, { recursive: true });
const staging = `${archivePath}.partial-${process.pid}`;
await writeFile(staging, archive);
await rename(staging, archivePath);
await writeFile(`${sha256Path}.partial-${process.pid}`, `${digest} ${fileName}\n`);
await rename(`${sha256Path}.partial-${process.pid}`, sha256Path);
} catch {
throw new ModulePackError("MODULE_PACK_OUTPUT_FAILED");
}
return Object.freeze({
archiveBytes: archive.length,
archivePath,
fileCount: listed.length,
moduleId: manifest.id,
sha256: digest,
sha256Path,
target: MODULE_PACKAGE_ANY_TARGET,
version: manifest.version,
});
}
packages/module-pack/src/packages.ts
+154
View File
@@ -0,0 +1,154 @@
import { access, readdir, readFile, realpath } from "node:fs/promises";
import path from "node:path";
/**
* What the packager knows about a package whose files were bundled: where it is, what it is called,
* and whether it is third-party software. A package installed under a `node_modules` directory is
* third-party; one that is not (a workspace package, or the module itself) is the platform's or the
* module's own code.
*/
export interface BundledPackage {
readonly root: string;
readonly name: string;
readonly version: string;
readonly thirdParty: boolean;
readonly manifest: Readonly<Record<string, unknown>>;
}
function isRecord(value: unknown): value is Record<string, unknown> {
return typeof value === "object" && value !== null && !Array.isArray(value);
}
async function readManifest(directory: string): Promise<Record<string, unknown> | undefined> {
try {
const parsed = JSON.parse(
await readFile(path.join(directory, "package.json"), "utf8"),
) as unknown;
return isRecord(parsed) ? parsed : undefined;
} catch {
return undefined;
}
}
/** Finds the package a bundled file belongs to: the nearest enclosing directory whose package.json
* names a package and its version. Nested `package.json` files that only set a module type, as
* some libraries ship in subdirectories, are skipped. */
export class BundledPackageIndex {
readonly #byDirectory = new Map<string, BundledPackage | undefined>();
async packageOf(file: string): Promise<BundledPackage | undefined> {
let directory = path.dirname(await realpath(file).catch(() => file));
const visited: string[] = [];
let found: BundledPackage | undefined;
for (;;) {
if (this.#byDirectory.has(directory)) {
found = this.#byDirectory.get(directory);
break;
}
visited.push(directory);
const manifest = await readManifest(directory);
if (manifest && typeof manifest.name === "string" && typeof manifest.version === "string") {
found = Object.freeze({
manifest: Object.freeze(manifest),
name: manifest.name,
root: directory,
thirdParty: directory.split(path.sep).includes("node_modules"),
version: manifest.version,
});
break;
}
const parent = path.dirname(directory);
if (parent === directory) {
break;
}
directory = parent;
}
for (const seen of visited) {
this.#byDirectory.set(seen, found);
}
return found;
}
}
async function exists(file: string): Promise<boolean> {
try {
await access(file);
return true;
} catch {
return false;
}
}
function declaresPlatform(manifest: Readonly<Record<string, unknown>>): boolean {
return ["os", "cpu", "libc"].some((field) => manifest[field] !== undefined);
}
async function containsCompiledAddon(directory: string): Promise<boolean> {
const entries = await readdir(directory, { withFileTypes: true }).catch(() => []);
for (const entry of entries) {
if (entry.isDirectory()) {
if (
entry.name !== "node_modules" &&
(await containsCompiledAddon(path.join(directory, entry.name)))
) {
return true;
}
} else if (entry.name.toLowerCase().endsWith(".node")) {
return true;
}
}
return false;
}
/** The installed package a dependency name resolves to from `root`, by Node's own directory walk. */
async function installedDependency(
root: string,
name: string,
): Promise<Record<string, unknown> | undefined> {
let directory = root;
for (;;) {
if (path.basename(directory) !== "node_modules") {
const manifest = await readManifest(path.join(directory, "node_modules", ...name.split("/")));
if (manifest) {
return manifest;
}
}
const parent = path.dirname(directory);
if (parent === directory) {
return undefined;
}
directory = parent;
}
}
/**
* Whether a bundled package is native or platform-specific, which an `any` package may never
* contain (ADR 0049): it declares an operating system, CPU, or C library; it is built with node-gyp;
* it ships a compiled `.node` addon; or one of its installed dependencies declares a platform, as
* packages that load a per-platform binary do.
*/
export async function isNativeOrPlatformSpecific(bundled: BundledPackage): Promise<boolean> {
const manifest = bundled.manifest;
if (declaresPlatform(manifest) || manifest.gypfile === true) {
return true;
}
if (await exists(path.join(bundled.root, "binding.gyp"))) {
return true;
}
if (await containsCompiledAddon(bundled.root)) {
return true;
}
for (const field of ["dependencies", "optionalDependencies"]) {
const dependencies = manifest[field];
if (!isRecord(dependencies)) {
continue;
}
for (const name of Object.keys(dependencies)) {
const dependency = await installedDependency(bundled.root, name);
if (dependency && declaresPlatform(dependency)) {
return true;
}
}
}
return false;
}
packages/module-pack/src/tar.ts
+109
View File
@@ -0,0 +1,109 @@
import { gzipSync } from "node:zlib";
/**
* The deterministic archive encoding of Ekklesia Module Package v1 (ADR 0049).
*
* POSIX ustar, written by hand rather than by a general tar library, so every byte is decided here:
* - one top-level directory, then every directory and regular file in byte-wise path order, which
* puts every directory before what it contains;
* - uid and gid 0, empty user and group names, modification time 0, device fields 0;
* - directories 0755 and files 0644; no links, no special files, no extended or long-name headers;
* - exactly two zero blocks at the end;
* - gzip at a fixed level, with Node's header: no file name and a modification time of 0.
*
* The same packager version and the same input bytes therefore always produce the same archive.
*/
export class TarPathError extends Error {
readonly path: string;
constructor(entryPath: string) {
super("A path does not fit a portable ustar header.");
this.name = "TarPathError";
this.path = entryPath;
}
}
const BLOCK = 512;
const GZIP_LEVEL = 9;
function octal(value: number, width: number): string {
return `${value.toString(8).padStart(width - 1, "0")}\u0000`;
}
/** Splits a path into ustar's 155-byte prefix and 100-byte name, at a `/`. */
function splitPath(entryPath: string): { readonly name: string; readonly prefix: string } {
if (entryPath.length <= 100) {
return { name: entryPath, prefix: "" };
}
for (let index = entryPath.length - 1; index > 0; index -= 1) {
if (entryPath[index] === "/") {
const prefix = entryPath.slice(0, index);
const name = entryPath.slice(index + 1);
if (prefix.length <= 155 && name.length > 0 && name.length <= 100) {
return { name, prefix };
}
}
}
throw new TarPathError(entryPath);
}
function header(entryPath: string, directory: boolean, size: number): Buffer {
const block = Buffer.alloc(BLOCK);
const { name, prefix } = splitPath(directory ? `${entryPath}/` : entryPath);
block.write(name, 0, 100, "latin1");
block.write(octal(directory ? 0o755 : 0o644, 8), 100, "latin1");
block.write(octal(0, 8), 108, "latin1");
block.write(octal(0, 8), 116, "latin1");
block.write(octal(size, 12), 124, "latin1");
block.write(octal(0, 12), 136, "latin1");
block.write(" ", 148, "latin1");
block.write(directory ? "5" : "0", 156, "latin1");
block.write("ustar\u000000", 257, "latin1");
block.write(octal(0, 8), 329, "latin1");
block.write(octal(0, 8), 337, "latin1");
block.write(prefix, 345, 155, "latin1");
let checksum = 0;
for (const byte of block) {
checksum += byte;
}
block.write(`${checksum.toString(8).padStart(6, "0")}\u0000 `, 148, "latin1");
return block;
}
function compareBytes(left: string, right: string): number {
return left < right ? -1 : left > right ? 1 : 0;
}
/**
* Archives `files` (paths relative to the module directory, printable ASCII) under `topLevel`.
* Directories are implied by the file paths.
*/
export function deterministicTarGz(topLevel: string, files: ReadonlyMap<string, Buffer>): Buffer {
const directories = new Set<string>([topLevel]);
for (const filePath of files.keys()) {
const segments = filePath.split("/");
for (let index = 1; index < segments.length; index += 1) {
directories.add(`${topLevel}/${segments.slice(0, index).join("/")}`);
}
}
const entries: { readonly path: string; readonly data?: Buffer }[] = [
...[...directories].map((directory) => ({ path: directory })),
...[...files].map(([filePath, data]) => ({ data, path: `${topLevel}/${filePath}` })),
].sort((left, right) => compareBytes(left.path, right.path));
const parts: Buffer[] = [];
for (const entry of entries) {
if (entry.data === undefined) {
parts.push(header(entry.path, true, 0));
} else {
parts.push(header(entry.path, false, entry.data.length), entry.data);
const padding = (BLOCK - (entry.data.length % BLOCK)) % BLOCK;
if (padding > 0) {
parts.push(Buffer.alloc(padding));
}
}
}
parts.push(Buffer.alloc(BLOCK * 2));
return gzipSync(Buffer.concat(parts), { level: GZIP_LEVEL });
}
packages/module-pack/tests/cli.test.ts
+74
View File
@@ -0,0 +1,74 @@
import { describe, expect, it } from "vitest";
import { parsePackModuleArguments } from "../src/cli.js";
const revision = "86ae907f59458508e3277666cdd214f5bd7c1078";
describe("ekklesia-pack-module arguments", () => {
it("takes the module, the target, the output directory, and the source revision", () => {
expect(
parsePackModuleArguments([
"--module",
"templates/module",
"--target",
"any",
"--out",
"dist/packages",
"--source-revision",
revision,
]),
).toEqual({
moduleDirectory: "templates/module",
outDirectory: "dist/packages",
sourceRevision: revision,
target: "any",
});
});
it("records a repository only when one is given", () => {
expect(
parsePackModuleArguments([
"--module",
".",
"--target",
"any",
"--out",
"out",
"--source-revision",
revision,
"--repository",
"https://example.org/acme/example.git",
]).repository,
).toBe("https://example.org/acme/example.git");
});
it.each([
["no arguments", []],
["a missing target", ["--module", ".", "--out", "o", "--source-revision", revision]],
["a missing revision", ["--module", ".", "--target", "any", "--out", "o"]],
[
"an unknown option",
["--module", ".", "--target", "any", "--out", "o", "--source-revision", revision, "--legacy"],
],
[
"a repeated option",
[
"--module",
".",
"--module",
"..",
"--target",
"any",
"--out",
"o",
"--source-revision",
revision,
],
],
[
"a stray positional argument",
["module", "--target", "any", "--out", "o", "--source-revision", revision],
],
])("refuses %s", (_label, argv) => {
expect(() => parsePackModuleArguments(argv)).toThrow();
});
});
packages/module-pack/tests/pack-module.test.ts
+434
View File
@@ -0,0 +1,434 @@
import { createHash } from "node:crypto";
import { readFile } from "node:fs/promises";
import { isBuiltin } from "node:module";
import path from "node:path";
import { afterEach, describe, expect, it } from "vitest";
import { type ModulePackErrorCode, ModulePackError, packModule } from "../src/index.js";
import {
cleanupFixtures,
type FixtureModuleOptions,
fixtureModule,
SOURCE_REVISION,
scratchDirectory,
} from "./support/fixture-module.js";
import { fileEntries, readTarGz } from "./support/tar.js";
afterEach(cleanupFixtures);
const MIT = "MIT License\n\nCopyright (c) 2026 Example Author\n\nPermission is hereby granted.\n";
async function pack(options: FixtureModuleOptions = {}, overrides: Record<string, unknown> = {}) {
const moduleDirectory = await fixtureModule(options);
const outDirectory = await scratchDirectory("ekklesia-pack-out-");
const result = await packModule({
moduleDirectory,
nodeRange: ">=24.11.0 <25",
outDirectory,
sourceRevision: SOURCE_REVISION,
target: "any",
...overrides,
});
return { archive: await readFile(result.archivePath), moduleDirectory, outDirectory, result };
}
async function packRejection(
options: FixtureModuleOptions,
code: ModulePackErrorCode,
overrides: Record<string, unknown> = {},
): Promise<ModulePackError> {
try {
await pack(options, overrides);
} catch (error) {
expect(error).toBeInstanceOf(ModulePackError);
expect((error as ModulePackError).code).toBe(code);
return error as ModulePackError;
}
throw new Error(`Expected ${code}.`);
}
describe("packModule: what a package contains (ADR 0049)", () => {
it("produces one archive and its sha256sum line, under the recommended name", async () => {
const { archive, outDirectory, result } = await pack();
const digest = createHash("sha256").update(archive).digest("hex");
expect(path.basename(result.archivePath)).toBe("example-0.1.0-any.tar.gz");
expect(result.sha256).toBe(digest);
expect(await readFile(path.join(outDirectory, "example-0.1.0-any.tar.gz.sha256"), "utf8")).toBe(
`${digest} example-0.1.0-any.tar.gz\n`,
);
});
it("holds exactly the runtime payload, under one top-level directory named with the module id", async () => {
const { archive } = await pack({ withSchema: true });
const files = [...fileEntries(archive).keys()].sort();
expect(files).toEqual([
"example/THIRD_PARTY_NOTICES",
expect.stringMatching(/^example\/dist\/chunks\/chunk-[A-Z0-9]+\.js$/u),
"example/dist/schema/index.js",
"example/dist/server/index.js",
"example/drizzle/0000_initial.sql",
"example/drizzle/meta/0000_snapshot.json",
"example/drizzle/meta/_ekklesia.json",
"example/drizzle/meta/_journal.json",
"example/ekklesia.module.json",
"example/ekklesia.package.json",
"example/package.json",
]);
for (const file of files) {
expect(file).not.toMatch(/\/src\/|\/tests\/|\.d\.ts$|\.map$|README|node_modules/u);
}
expect(new Set(readTarGz(archive).map((entry) => entry.path.split("/")[0]))).toEqual(
new Set(["example"]),
);
});
it("generates the runtime package.json instead of copying the development one", async () => {
const { archive } = await pack();
const runtimePackage = fileEntries(archive).get("example/package.json")?.toString("utf8");
expect(JSON.parse(runtimePackage ?? "")).toEqual({
name: "@test/module-example",
version: "0.1.0",
private: true,
type: "module",
ekklesia: { layer: "module" },
});
for (const [file, data] of fileEntries(archive)) {
if (file.endsWith(".json")) {
expect(data.toString("utf8"), file).not.toContain("workspace:");
}
}
});
it("copies the manifest byte for byte", async () => {
const { archive, moduleDirectory } = await pack();
expect(fileEntries(archive).get("example/ekklesia.module.json")).toEqual(
await readFile(path.join(moduleDirectory, "ekklesia.module.json")),
);
});
it("describes the package completely in ekklesia.package.json, never hashing itself", async () => {
const { archive } = await pack({ withSchema: true });
const files = fileEntries(archive);
const metadata = JSON.parse(files.get("example/ekklesia.package.json")?.toString("utf8") ?? "");
expect(metadata).toMatchObject({
packageFormat: 1,
module: { id: "example", version: "0.1.0" },
target: "any",
runtime: { node: ">=24.11.0 <25" },
provenance: {
revision: SOURCE_REVISION,
packager: { name: "@ekklesia/module-pack", version: expect.any(String) },
},
incorporates: {
"@ekklesia/contracts": expect.any(String),
"@ekklesia/module-sdk": expect.any(String),
},
transitional: { databaseRuntime: { "drizzle-orm": "0.45.2" } },
});
const listed = metadata.files.map((file: { path: string }) => `example/${file.path}`);
expect(listed).not.toContain("example/ekklesia.package.json");
expect(new Set(listed)).toEqual(
new Set([...files.keys()].filter((file) => file !== "example/ekklesia.package.json")),
);
for (const file of metadata.files as { path: string; size: number; sha256: string }[]) {
const data = files.get(`example/${file.path}`);
expect(data?.length).toBe(file.size);
expect(
createHash("sha256")
.update(data ?? "")
.digest("hex"),
).toBe(file.sha256);
}
});
it("records only the provenance it was given, and a repository only when asked", async () => {
const { archive } = await pack({}, { repository: "https://example.org/acme/example.git" });
const metadata = JSON.parse(
fileEntries(archive).get("example/ekklesia.package.json")?.toString("utf8") ?? "",
);
expect(metadata.provenance.repository).toBe("https://example.org/acme/example.git");
});
it("omits transitional database facts from a module that bundles no Drizzle", async () => {
const { archive } = await pack({
serverSource: 'export default { id: "example", version: "0.1.0", register() {} };\n',
});
const metadata = JSON.parse(
fileEntries(archive).get("example/ekklesia.package.json")?.toString("utf8") ?? "",
);
expect(metadata.transitional).toBeUndefined();
expect(metadata.incorporates).toEqual({});
});
});
describe("packModule: a self-contained backend", () => {
it("bundles the SDK, contracts, and Drizzle, leaving only Node built-ins to import", async () => {
const { archive } = await pack({ withSchema: true });
for (const [file, data] of fileEntries(archive)) {
if (!file.endsWith(".js")) {
continue;
}
const code = data.toString("utf8");
const specifiers = [
...code.matchAll(/(?:^|\n)\s*(?:import|export)\s[^;]*?from\s*"([^"]+)"/gu),
...code.matchAll(/(?:^|\n)\s*import\s*"([^"]+)"/gu),
...code.matchAll(/\bimport\(\s*"([^"]+)"\s*\)/gu),
].map((match) => match[1] ?? "");
for (const specifier of specifiers) {
expect(specifier.startsWith(".") || isBuiltin(specifier), `${file}: ${specifier}`).toBe(
true,
);
}
}
});
it("refuses a dependency the bundle cannot resolve rather than leaving it external", async () => {
await packRejection(
{
serverSource:
'import pad from "ekklesia-not-installed";\nexport default { id: "example", version: "0.1.0", register() { pad(); } };\n',
},
"MODULE_PACK_BUNDLE_FAILED",
);
});
it.each([
[
"a package that declares an operating system",
{
name: "fake-native",
packageJson: { os: ["linux"], cpu: ["x64"] },
files: { "index.js": "export default 1;\n" },
},
],
[
"a package built with node-gyp",
{
name: "fake-gyp",
packageJson: { gypfile: true },
files: { "index.js": "export default 1;\n", "binding.gyp": "{}" },
},
],
[
"a package that ships a compiled addon",
{
name: "fake-addon",
files: {
"index.js": "export default 1;\n",
"build/Release/addon.node": new Uint8Array([0x7f, 0x45, 0x4c, 0x46]),
},
},
],
])("refuses %s in an any-target package", async (_label, fake) => {
await packRejection(
{
fakePackages: [fake],
serverSource: `import value from "${fake.name}";\nexport default { id: "example", version: "0.1.0", register() { return value; } };\n`,
},
"MODULE_PACK_NATIVE_DEPENDENCY",
);
});
it("refuses a package whose installed dependency is platform-specific", async () => {
await packRejection(
{
fakePackages: [
{
name: "fake-wrapper",
packageJson: { optionalDependencies: { "fake-wrapper-linux-x64": "1.0.0" } },
files: { "index.js": "export default 1;\n", LICENSE: MIT },
},
{
name: "fake-wrapper-linux-x64",
packageJson: { os: ["linux"], cpu: ["x64"] },
files: { "index.js": "export default 2;\n" },
},
],
serverSource:
'import value from "fake-wrapper";\nexport default { id: "example", version: "0.1.0", register() { return value; } };\n',
},
"MODULE_PACK_NATIVE_DEPENDENCY",
);
});
it("refuses any target but any in this release", async () => {
await packRejection({}, "MODULE_PACK_TARGET_UNSUPPORTED", { target: "linux-x64-glibc" });
});
});
describe("packModule: identity and inputs", () => {
it("refuses a development package.json whose version differs from the manifest", async () => {
await packRejection(
{ manifest: (manifest) => ({ ...manifest, version: "0.2.0" }) },
"MODULE_PACK_IDENTITY_MISMATCH",
);
});
it("refuses a module whose backend was never built", async () => {
await packRejection(
{
manifest: (manifest) => ({
...manifest,
entrypoints: { server: "./dist/missing/index.js" },
}),
},
"MODULE_PACK_ENTRYPOINT_MISSING",
);
});
it("refuses an abbreviated or missing source revision", async () => {
await packRejection({}, "MODULE_PACK_REVISION_INVALID", { sourceRevision: "86ae907" });
});
it("refuses an incorporated SDK outside the manifest's own range", async () => {
await packRejection(
{
manifest: (manifest) => ({
...manifest,
compatibility: { platform: "^0.1.0", moduleSdk: "^9.0.0" },
}),
},
"MODULE_PACK_SDK_INCOMPATIBLE",
);
});
});
describe("packModule: THIRD_PARTY_NOTICES", () => {
const usesFake = (name: string) =>
`import value from "${name}";\nexport default { id: "example", version: "0.1.0", register() { return value; } };\n`;
it("credits every bundled third-party package once, with its licence and notice text", async () => {
const { archive } = await pack({
fakePackages: [
{
name: "fake-mit",
version: "2.3.4",
files: {
"index.js": 'import other from "fake-apache-notice";\nexport default other;\n',
LICENSE: MIT,
},
},
{
name: "fake-apache-notice",
version: "1.0.0",
packageJson: { license: "Apache-2.0" },
files: {
"index.js": "export default 1;\n",
"LICENSE.txt": "Apache License text shipped by the package.\n",
NOTICE: "Fake Apache Notice\nCopyright 2026 The Fake Foundation\n",
},
},
],
serverSource: usesFake("fake-mit"),
});
const notices = fileEntries(archive).get("example/THIRD_PARTY_NOTICES")?.toString("utf8") ?? "";
expect(notices).toContain("fake-mit 2.3.4");
expect(notices).toContain("Copyright (c) 2026 Example Author");
expect(notices).toContain("fake-apache-notice 1.0.0");
expect(notices).toContain("Copyright 2026 The Fake Foundation");
expect(notices.match(/^fake-mit 2\.3\.4$/gmu)).toHaveLength(1);
// Only what the bundle actually contains is credited: this backend uses no Drizzle code.
expect(notices).not.toContain("drizzle-orm");
// The Ekklesia platform's own packages are credited in ekklesia.package.json, not here.
expect(notices).not.toMatch(/^@ekklesia\//mu);
});
it("supplies the standard Apache-2.0 text only for a package that ships none, and says so", async () => {
const { archive } = await pack({ withSchema: true });
const notices = fileEntries(archive).get("example/THIRD_PARTY_NOTICES")?.toString("utf8") ?? "";
const drizzle = notices.slice(notices.indexOf("drizzle-orm 0.45.2"));
expect(drizzle).toContain("License: Apache-2.0");
expect(drizzle).toContain("ships no licence file");
expect(drizzle).toContain("TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION");
});
it("is byte-identical across packages of the same inputs", async () => {
const options = { withSchema: true };
const first = await pack(options);
const second = await pack(options);
expect(fileEntries(first.archive).get("example/THIRD_PARTY_NOTICES")).toEqual(
fileEntries(second.archive).get("example/THIRD_PARTY_NOTICES"),
);
});
it("refuses a bundled package that declares no licence", async () => {
await packRejection(
{
fakePackages: [
{
name: "fake-unlicensed",
packageJson: { license: undefined },
files: { "index.js": "export default 1;\n" },
},
],
serverSource: usesFake("fake-unlicensed"),
},
"MODULE_PACK_LICENCE_UNKNOWN",
);
});
it("refuses an MIT package that ships no licence text, because only it holds the copyright line", async () => {
await packRejection(
{
fakePackages: [{ name: "fake-textless", files: { "index.js": "export default 1;\n" } }],
serverSource: usesFake("fake-textless"),
},
"MODULE_PACK_LICENCE_TEXT_MISSING",
);
});
});
describe("packModule: deterministic archives", () => {
it("produces byte-identical archives and digests for identical inputs", async () => {
const moduleDirectory = await fixtureModule({ withSchema: true });
const runs = [];
for (let run = 0; run < 2; run += 1) {
const outDirectory = await scratchDirectory("ekklesia-pack-out-");
const result = await packModule({
moduleDirectory,
nodeRange: ">=24.11.0 <25",
outDirectory,
sourceRevision: SOURCE_REVISION,
target: "any",
});
runs.push({ bytes: await readFile(result.archivePath), sha256: result.sha256 });
}
expect(runs[0]?.sha256).toBe(runs[1]?.sha256);
expect(runs[0]?.bytes.equals(runs[1]?.bytes ?? Buffer.alloc(0))).toBe(true);
});
it("normalizes every entry and the gzip header, and contains no links", async () => {
const { archive } = await pack({ withSchema: true });
// gzip: no file name flag, no modification time.
expect(archive[3]).toBe(0);
expect(archive.readUInt32LE(4)).toBe(0);
const entries = readTarGz(archive);
const paths = entries.map((entry) => entry.path);
expect(paths).toEqual(
[...paths].sort((left, right) => (left < right ? -1 : left > right ? 1 : 0)),
);
for (const entry of entries) {
expect(["0", "5"], entry.path).toContain(entry.typeflag);
expect(entry.mode, entry.path).toBe(entry.typeflag === "5" ? 0o755 : 0o644);
expect([entry.uid, entry.gid, entry.mtime]).toEqual([0, 0, 0]);
expect([entry.uname, entry.gname]).toEqual(["", ""]);
expect(entry.magic).toBe("ustar/00");
}
// Every directory precedes what it contains.
for (const entry of entries) {
const parent = entry.path.split("/").slice(0, -1).join("/");
if (parent !== "") {
expect(paths.indexOf(parent), entry.path).toBeLessThan(paths.indexOf(entry.path));
}
}
});
it("refers to no absolute path of the build machine", async () => {
const { archive, moduleDirectory } = await pack({ withSchema: true });
const root = path.dirname(path.dirname(moduleDirectory));
for (const [file, data] of fileEntries(archive)) {
expect(data.toString("utf8"), file).not.toContain(moduleDirectory);
expect(data.toString("utf8"), file).not.toContain(root);
}
});
});
packages/module-pack/tests/support/fixture-module.ts
+193
View File
@@ -0,0 +1,193 @@
import { mkdir, mkdtemp, realpath, rm, symlink, writeFile } from "node:fs/promises";
import { tmpdir } from "node:os";
import path from "node:path";
import { fileURLToPath } from "node:url";
/**
* Generated module fixtures for the packager's tests: a built module directory in a temporary
* location, whose `node_modules` links to the real Module SDK and Drizzle this package depends on.
* Everything a development module directory holds is there, including what must never ship
* (`src/`, `tests/`, declarations, source maps, `workspace:` ranges), so a test can prove the
* packager leaves it out.
*/
const packageRoot = path.resolve(path.dirname(fileURLToPath(import.meta.url)), "..", "..");
export interface FakePackage {
readonly name: string;
readonly version?: string;
readonly packageJson?: Record<string, unknown>;
/** Files relative to the fake package's root. */
readonly files?: Record<string, string | Uint8Array>;
}
export interface FixtureModuleOptions {
readonly id?: string;
readonly version?: string;
readonly serverSource?: string;
readonly withSchema?: boolean;
readonly fakePackages?: readonly FakePackage[];
readonly manifest?: (manifest: Record<string, unknown>) => Record<string, unknown>;
}
const temporaryDirectories: string[] = [];
export async function cleanupFixtures(): Promise<void> {
await Promise.all(
temporaryDirectories
.splice(0)
.map((directory) => rm(directory, { force: true, recursive: true })),
);
}
export async function scratchDirectory(prefix = "ekklesia-pack-"): Promise<string> {
const directory = await realpath(await mkdtemp(path.join(tmpdir(), prefix)));
temporaryDirectories.push(directory);
return directory;
}
async function write(file: string, contents: string | Uint8Array): Promise<void> {
await mkdir(path.dirname(file), { recursive: true });
await writeFile(file, contents);
}
const defaultServer = (
id: string,
version: string,
) => `import { defineModuleRuntime } from "@ekklesia/module-sdk";
export default defineModuleRuntime({ id: ${JSON.stringify(id)}, version: ${JSON.stringify(version)}, register() {} });
`;
const schemaSource = (
id: string,
) => `import { createModuleTable, defineModuleSchema } from "@ekklesia/module-sdk";
import { text, uuid } from "drizzle-orm/pg-core";
export const records = createModuleTable(${JSON.stringify(id)})("records", {
id: uuid("id").primaryKey(),
value: text("value").notNull(),
});
export default defineModuleSchema({ moduleId: ${JSON.stringify(id)}, tables: { records } });
`;
export async function fixtureModule(options: FixtureModuleOptions = {}): Promise<string> {
const id = options.id ?? "example";
const version = options.version ?? "0.1.0";
const root = await scratchDirectory("ekklesia-pack-fixture-");
const moduleDirectory = path.join(root, id);
const tableName = `${id.replaceAll("-", "_")}_records`;
const baseManifest: Record<string, unknown> = {
manifestVersion: 7,
id,
name: `${id} module`,
version,
compatibility: { platform: "^0.1.0", moduleSdk: "^0.1.0" },
capabilities: [],
permissions: [],
entrypoints: {
server: "./dist/server/index.js",
...(options.withSchema ? { schema: "./dist/schema/index.js" } : {}),
migrations: "./drizzle",
},
};
const manifest = options.manifest ? options.manifest(baseManifest) : baseManifest;
await write(
path.join(moduleDirectory, "ekklesia.module.json"),
`${JSON.stringify(manifest, null, 2)}\n`,
);
await write(
path.join(moduleDirectory, "package.json"),
`${JSON.stringify(
{
name: `@test/module-${id}`,
version,
private: true,
type: "module",
scripts: { build: "tsc" },
dependencies: { "@ekklesia/module-sdk": "workspace:*", "drizzle-orm": "0.45.2" },
devDependencies: { typescript: "7.0.2" },
exports: { ".": "./dist/server/index.js" },
ekklesia: { layer: "module" },
},
null,
2,
)}\n`,
);
await write(
path.join(moduleDirectory, "dist/server/index.js"),
options.serverSource ?? defaultServer(id, version),
);
await write(path.join(moduleDirectory, "dist/server/index.js.map"), '{"version":3}\n');
await write(path.join(moduleDirectory, "dist/server/index.d.ts"), "export {};\n");
if (options.withSchema) {
await write(path.join(moduleDirectory, "dist/schema/index.js"), schemaSource(id));
}
await write(path.join(moduleDirectory, "src/server/index.ts"), "export {};\n");
await write(path.join(moduleDirectory, "tests/server.test.ts"), "export {};\n");
await write(path.join(moduleDirectory, "README.md"), "# Development notes\n");
await write(
path.join(moduleDirectory, "drizzle/0000_initial.sql"),
`CREATE TABLE "${tableName}" ("id" uuid PRIMARY KEY NOT NULL, "value" text NOT NULL);\n`,
);
await write(
path.join(moduleDirectory, "drizzle/meta/_journal.json"),
JSON.stringify({
version: "7",
dialect: "postgresql",
entries: [
{ idx: 0, version: "7", when: 1_700_000_000_000, tag: "0000_initial", breakpoints: true },
],
}),
);
await write(
path.join(moduleDirectory, "drizzle/meta/_ekklesia.json"),
JSON.stringify({
version: 1,
moduleId: id,
entries: [{ tag: "0000_initial", destructive: { mode: "none" } }],
}),
);
await write(
path.join(moduleDirectory, "drizzle/meta/0000_snapshot.json"),
JSON.stringify({
version: "7",
dialect: "postgresql",
id: "10000000-0000-4000-8000-000000000001",
prevId: "00000000-0000-0000-0000-000000000000",
}),
);
// The module's dependencies, as a workspace links them: the real packages this package depends on.
const modules = path.join(moduleDirectory, "node_modules");
await mkdir(path.join(modules, "@ekklesia"), { recursive: true });
await symlink(
await realpath(path.join(packageRoot, "node_modules", "@ekklesia", "module-sdk")),
path.join(modules, "@ekklesia", "module-sdk"),
);
await symlink(
await realpath(path.join(packageRoot, "node_modules", "drizzle-orm")),
path.join(modules, "drizzle-orm"),
);
for (const fake of options.fakePackages ?? []) {
const fakeRoot = path.join(modules, ...fake.name.split("/"));
await write(
path.join(fakeRoot, "package.json"),
JSON.stringify({
name: fake.name,
version: fake.version ?? "1.0.0",
type: "module",
main: "index.js",
license: "MIT",
...fake.packageJson,
}),
);
for (const [relative, contents] of Object.entries(fake.files ?? {})) {
await write(path.join(fakeRoot, relative), contents);
}
}
return moduleDirectory;
}
export const SOURCE_REVISION = "86ae907f59458508e3277666cdd214f5bd7c1078";
packages/module-pack/tests/support/tar.ts
+62
View File
@@ -0,0 +1,62 @@
import { gunzipSync } from "node:zlib";
/** A test-side view of every ustar header field the determinism rules constrain (ADR 0049). */
export interface TarEntry {
readonly path: string;
readonly typeflag: string;
readonly mode: number;
readonly uid: number;
readonly gid: number;
readonly uname: string;
readonly gname: string;
readonly mtime: number;
readonly magic: string;
readonly data: Buffer;
}
function text(block: Buffer, offset: number, length: number): string {
const raw = block.subarray(offset, offset + length);
const end = raw.indexOf(0);
return raw.subarray(0, end === -1 ? length : end).toString("latin1");
}
function octal(block: Buffer, offset: number, length: number): number {
return Number.parseInt(text(block, offset, length).trim() || "0", 8);
}
export function readTarGz(archive: Buffer): readonly TarEntry[] {
const raw = gunzipSync(archive);
const entries: TarEntry[] = [];
let offset = 0;
while (offset + 512 <= raw.length) {
const block = raw.subarray(offset, offset + 512);
if (block.every((byte) => byte === 0)) {
break;
}
const name = text(block, 0, 100);
const prefix = text(block, 345, 155);
const size = octal(block, 124, 12);
entries.push({
data: Buffer.from(raw.subarray(offset + 512, offset + 512 + size)),
gid: octal(block, 116, 8),
gname: text(block, 297, 32),
magic: `${text(block, 257, 6)}/${text(block, 263, 2)}`,
mode: octal(block, 100, 8),
mtime: octal(block, 136, 12),
path: prefix === "" ? name : `${prefix}/${name}`,
typeflag: text(block, 156, 1),
uid: octal(block, 108, 8),
uname: text(block, 265, 32),
});
offset += 512 + Math.ceil(size / 512) * 512;
}
return entries;
}
export function fileEntries(archive: Buffer): Map<string, Buffer> {
return new Map(
readTarGz(archive)
.filter((entry) => entry.typeflag === "0")
.map((entry) => [entry.path, entry.data]),
);
}
packages/module-pack/tests/template-package.test.ts
+195
View File
@@ -0,0 +1,195 @@
import { execFile } from "node:child_process";
import { createHash } from "node:crypto";
import { lstat, readdir, readFile, realpath } from "node:fs/promises";
import path from "node:path";
import { fileURLToPath } from "node:url";
import { promisify } from "node:util";
import { verifyModulePackage } from "@ekklesia/core/modules";
import { afterEach, describe, expect, it } from "vitest";
import { packModule } from "../src/index.js";
import { cleanupFixtures, SOURCE_REVISION, scratchDirectory } from "./support/fixture-module.js";
import { fileEntries } from "./support/tar.js";
/**
* The canonical template module, packaged end to end (ADR 0049): the real built template, the real
* packager, and Core's real verifier. Turbo builds the template and the verifier before this suite
* runs (both are development dependencies of this package), and runs the template's own tests, which
* rebuild its UI in place, before this one (turbo.json).
*/
const execFileAsync = promisify(execFile);
const packageRoot = path.resolve(path.dirname(fileURLToPath(import.meta.url)), "..");
const repositoryRoot = path.resolve(packageRoot, "..", "..");
afterEach(cleanupFixtures);
async function templateDirectory(): Promise<string> {
return realpath(path.join(packageRoot, "node_modules", "@ekklesia", "module-template"));
}
async function readJson(file: string): Promise<Record<string, unknown>> {
return JSON.parse(await readFile(file, "utf8")) as Record<string, unknown>;
}
async function platform() {
const core = await readJson(path.join(repositoryRoot, "core", "package.json"));
return {
databaseRuntime: {
"drizzle-orm": (core.dependencies as Record<string, string>)["drizzle-orm"] ?? "",
},
moduleSdkVersion: String(
(await readJson(path.join(repositoryRoot, "packages", "module-sdk", "package.json"))).version,
),
platformVersion: String((await readJson(path.join(repositoryRoot, "package.json"))).version),
};
}
async function fingerprint(file: string): Promise<string> {
try {
return createHash("sha256")
.update(await readFile(file))
.digest("hex");
} catch {
return "absent";
}
}
async function treeListing(directory: string): Promise<readonly string[]> {
const entries = await readdir(directory, { recursive: true, withFileTypes: true });
return entries
.filter((entry) => !path.join(entry.parentPath, entry.name).includes(`${path.sep}node_modules`))
.map((entry) => path.relative(directory, path.join(entry.parentPath, entry.name)))
.sort();
}
async function packTemplate() {
const outDirectory = await scratchDirectory("ekklesia-template-package-");
const result = await packModule({
moduleDirectory: await templateDirectory(),
outDirectory,
sourceRevision: SOURCE_REVISION,
target: "any",
});
return { archive: await readFile(result.archivePath), result };
}
describe("the module template as a Module Package v1", () => {
it("packs deterministically, without touching the workspace it was built in", async () => {
const template = await templateDirectory();
const watched = [
path.join(repositoryRoot, "node_modules", ".pnpm-workspace-state-v1.json"),
path.join(repositoryRoot, "node_modules", ".modules.yaml"),
path.join(repositoryRoot, "pnpm-lock.yaml"),
];
const before = await Promise.all(watched.map(fingerprint));
const templateBefore = await treeListing(template);
const first = await packTemplate();
const second = await packTemplate();
expect(first.result.sha256).toBe(second.result.sha256);
expect(first.archive.equals(second.archive)).toBe(true);
expect(path.basename(first.result.archivePath)).toMatch(
/^example-\d+\.\d+\.\d+-any\.tar\.gz$/u,
);
expect(await Promise.all(watched.map(fingerprint))).toEqual(before);
expect(await treeListing(template)).toEqual(templateBefore);
});
it("verifies against this platform and extracts to an unrelated directory that holds no link", async () => {
const { result } = await packTemplate();
const target = path.join(await scratchDirectory("ekklesia-installed-elsewhere-"), "example");
const verified = await verifyModulePackage({
archivePath: result.archivePath,
expectedSha256: result.sha256,
extractTo: target,
platform: await platform(),
});
expect(verified).toMatchObject({ moduleId: "example", target: "any" });
for (const entry of await readdir(target, { recursive: true, withFileTypes: true })) {
const stats = await lstat(path.join(entry.parentPath, entry.name));
expect(stats.isSymbolicLink(), entry.name).toBe(false);
expect(stats.isFile() ? stats.nlink : 1, entry.name).toBe(1);
}
const listing = await treeListing(target);
expect(listing).not.toContain("node_modules");
expect(listing.some((file) => /(^|\/)(src|tests)(\/|$)|\.d\.ts$|\.map$/u.test(file))).toBe(
false,
);
});
it("loads from its new location without resolving anything outside itself", async () => {
const { result } = await packTemplate();
const target = path.join(await scratchDirectory("ekklesia-portable-"), "example");
await verifyModulePackage({
archivePath: result.archivePath,
extractTo: target,
platform: await platform(),
});
// Every specifier the backend resolves while loading, recorded by Node itself.
const script = `
import { registerHooks } from "node:module";
import { pathToFileURL } from "node:url";
const resolved = [];
registerHooks({ resolve(specifier, context, next) { const result = next(specifier, context); resolved.push(result.url); return result; } });
for (const file of ["dist/server/index.js", "dist/schema/index.js"]) {
await import(pathToFileURL(${JSON.stringify(target)} + "/" + file).href);
}
process.stdout.write(JSON.stringify(resolved));
`;
const { stdout } = await execFileAsync(
process.execPath,
["--input-type=module", "-e", script],
{
cwd: path.dirname(target),
env: {},
},
);
const resolved = JSON.parse(stdout) as string[];
expect(resolved.length).toBeGreaterThan(0);
for (const url of resolved) {
const inside = url.startsWith(`file://${target}/`);
expect(inside || url.startsWith("node:"), url).toBe(true);
}
});
it("credits every third-party package its bundles say they contain", async () => {
const { archive } = await packTemplate();
const files = fileEntries(archive);
const notices = files.get("example/THIRD_PARTY_NOTICES")?.toString("utf8") ?? "";
const bundled = new Set<string>();
for (const [file, data] of files) {
if (file.endsWith(".js")) {
// esbuild names every bundled source file in a path comment.
for (const match of data
.toString("utf8")
.matchAll(/node_modules\/((?:@[^/]+\/)?[^/.][^/]*)\//gu)) {
if (match[1] && !match[1].startsWith("@ekklesia/")) {
bundled.add(match[1]);
}
}
}
}
expect([...bundled]).toEqual(
expect.arrayContaining(["drizzle-orm", "react", "react-dom", "zod"]),
);
for (const name of bundled) {
expect(notices, name).toMatch(
new RegExp(`^${name.replace(/[.*+?^${}()|[\]\\/]/gu, "\\$&")} \\S+$`, "mu"),
);
}
const licence = await readFile(path.join(packageRoot, "licenses", "Apache-2.0.txt"));
expect(createHash("sha256").update(licence).digest("hex")).toBe(
"cfc7749b96f63bd31c3c42b5c471bf756814053e847c10f3eb003417bc523d30",
);
});
it("refers to no absolute path of the checkout it was built in", async () => {
const { archive } = await packTemplate();
for (const [file, data] of fileEntries(archive)) {
expect(data.toString("utf8"), file).not.toContain(repositoryRoot);
}
});
});
packages/module-pack/tsconfig.build.json
+14
View File
@@ -0,0 +1,14 @@
{
"extends": "./tsconfig.json",
"compilerOptions": {
"declaration": true,
"declarationMap": true,
"noEmit": false,
"outDir": "dist",
"rootDir": "src",
"sourceMap": true,
"types": ["node"]
},
"include": ["src/**/*.ts"],
"exclude": ["src/**/*.test.ts", "tests"]
}
packages/module-pack/tsconfig.json
+8
View File
@@ -0,0 +1,8 @@
{
"extends": "../typescript-config/node.json",
"compilerOptions": {
"noEmit": true,
"types": ["node", "vitest/globals"]
},
"include": ["src/**/*.ts", "tests/**/*.ts"]
}
packages/module-ui-build/src/build-module-ui.ts
+105 -12
View File
@@ -6,7 +6,7 @@ import {
moduleDisplayEntrypointSchema,
packageRelativePathSchema,
} from "@ekklesia/contracts";
import { build, type Plugin } from "esbuild";
import { type BuildOptions, build, type Plugin } from "esbuild";
/**
* The one supported way to turn a module's React/TSX UI sources into the compiled browser artifacts
@@ -108,7 +108,7 @@ const SURVIVING_IMPORT = /@import\s/iu;
*/
const STYLESHEET_PLUGIN_NAME = "ekklesia-compile-stylesheet-imports";
function compileStylesheetImports(): Plugin {
function compileStylesheetImports(recordInputs?: (paths: readonly string[]) => void): Plugin {
const decoder = new TextDecoder();
return {
name: STYLESHEET_PLUGIN_NAME,
@@ -127,15 +127,30 @@ function compileStylesheetImports(): Plugin {
],
}));
// The nested build works from the module directory, exactly like the outer one. Without it
// esbuild names each stylesheet relative to whatever directory the caller ran from, and the
// artifact's bytes would depend on the caller (ADR 0049 requires a deterministic package).
const workingDirectory = pluginBuild.initialOptions.absWorkingDir ?? process.cwd();
pluginBuild.onLoad({ filter: /\.css$/i }, async (args) => {
const compiled = await build({
absWorkingDir: workingDirectory,
bundle: true,
entryPoints: [args.path],
logLevel: "silent",
metafile: recordInputs !== undefined,
minify: false,
target: MODULE_UI_BUILD_TARGET,
write: false,
});
// A stylesheet's own @imports are bundled by this nested build, so the outer build never
// sees them. An analysis must, because they ship inside the artifact.
if (recordInputs && compiled.metafile) {
recordInputs(
Object.keys(compiled.metafile.inputs).map((input) =>
path.resolve(workingDirectory, input),
),
);
}
const stylesheet = compiled.outputFiles
.map((outputFile) => decoder.decode(outputFile.contents))
.join("\n");
@@ -323,14 +338,14 @@ async function resolveDeclaredArtifacts(
* Never runs during discovery, startup, or install: this is an authoring-time and packaging-time
* step only. Core validates the artifact a module ships; it never produces one (ADR 0021).
*/
export async function buildModuleUi(options: BuildModuleUiOptions): Promise<ModuleUiBuildResult> {
const moduleDirectory = path.resolve(options.moduleDirectory);
const artifacts = await resolveDeclaredArtifacts(moduleDirectory);
const pendingWrites: Array<readonly { path: string; contents: Uint8Array }[]> = [];
for (const artifact of artifacts) {
try {
const result = await build({
/** The one esbuild configuration for a declared artifact, shared by building and analysing so the
* two can never describe different builds. */
function moduleUiBuildOptions(
moduleDirectory: string,
artifact: DeclaredModuleUiArtifact,
recordInputs?: (paths: readonly string[]) => void,
): BuildOptions & { write: false } {
return {
absWorkingDir: moduleDirectory,
// Every dependency, React included, is bundled. The artifact must run in any host, including
// one that uses no React at all, so nothing is externalized (ADR 0023).
@@ -350,7 +365,7 @@ export async function buildModuleUi(options: BuildModuleUiOptions): Promise<Modu
minify: false,
outfile: artifact.outputFile,
platform: "browser",
plugins: [compileStylesheetImports()],
plugins: [compileStylesheetImports(recordInputs)],
sourcemap: true,
target: MODULE_UI_BUILD_TARGET,
// Fail rather than silently shipping a partial bundle.
@@ -358,7 +373,17 @@ export async function buildModuleUi(options: BuildModuleUiOptions): Promise<Modu
// Compile every role successfully before writing any role. This makes a two-artifact build
// atomic with respect to parse/resolve/bundle failures rather than leaving a stale partial set.
write: false,
});
};
}
export async function buildModuleUi(options: BuildModuleUiOptions): Promise<ModuleUiBuildResult> {
const moduleDirectory = path.resolve(options.moduleDirectory);
const artifacts = await resolveDeclaredArtifacts(moduleDirectory);
const pendingWrites: Array<readonly { path: string; contents: Uint8Array }[]> = [];
for (const artifact of artifacts) {
try {
const result = await build(moduleUiBuildOptions(moduleDirectory, artifact));
// Belt and braces on the stylesheet contract. The plugin above turns every `.css` import into
// JavaScript, so esbuild has nothing left to emit a stylesheet from — but a future option or
// loader change could reintroduce a sidecar, and a sidecar is precisely the failure this
@@ -414,3 +439,71 @@ export async function buildModuleUi(options: BuildModuleUiOptions): Promise<Modu
artifacts: Object.freeze(artifacts.map((artifact) => Object.freeze({ ...artifact }))),
});
}
export interface ModuleUiAnalysis {
/** Every declared artifact, in the same stable role order `buildModuleUi` produces them. */
readonly artifacts: readonly ModuleUiArtifactAnalysis[];
}
export interface ModuleUiArtifactAnalysis {
readonly kind: ModuleUiArtifactKind;
readonly contributionId?: string;
/** Absolute path of the artifact the manifest declares. */
readonly outputFile: string;
/** Absolute paths of every source file bundled into it, stylesheets and their imports included,
* sorted and unique. What a packager credits for the code it redistributes. */
readonly inputs: readonly string[];
/** Whether the artifact on disk is byte-for-byte what these sources build today. False when it
* was never built or is stale, so a caller never describes one artifact and ships another. */
readonly builtArtifactCurrent: boolean;
}
/**
* Runs the canonical module UI build in memory, exactly as `buildModuleUi` would, and reports what
* went into each declared artifact. Writes nothing. Used by the module packager (ADR 0049) to find
* the third-party code inside already-built browser artifacts, and to refuse a stale build.
*/
export async function analyzeModuleUi(options: BuildModuleUiOptions): Promise<ModuleUiAnalysis> {
const moduleDirectory = path.resolve(options.moduleDirectory);
const artifacts = await resolveDeclaredArtifacts(moduleDirectory);
const analyses: ModuleUiArtifactAnalysis[] = [];
for (const artifact of artifacts) {
const inputs = new Set<string>();
let result: Awaited<ReturnType<typeof build>>;
try {
result = await build({
...moduleUiBuildOptions(moduleDirectory, artifact, (paths) => {
for (const input of paths) {
inputs.add(input);
}
}),
metafile: true,
});
} catch {
throw new ModuleUiBuildError("MODULE_UI_BUILD_FAILED", artifact.kind);
}
for (const input of Object.keys(result.metafile?.inputs ?? {})) {
inputs.add(path.resolve(moduleDirectory, input));
}
const built = result.outputFiles?.find((outputFile) => outputFile.path === artifact.outputFile);
let onDisk: Buffer | undefined;
try {
onDisk = await readFile(artifact.outputFile);
} catch {
onDisk = undefined;
}
analyses.push(
Object.freeze({
...(artifact.contributionId === undefined
? {}
: { contributionId: artifact.contributionId }),
builtArtifactCurrent:
built !== undefined && onDisk !== undefined && Buffer.from(built.contents).equals(onDisk),
inputs: Object.freeze([...inputs].sort()),
kind: artifact.kind,
outputFile: artifact.outputFile,
}),
);
}
return Object.freeze({ artifacts: Object.freeze(analyses) });
}
packages/module-ui-build/tests/build-module-ui.test.ts
+85 -1
View File
@@ -2,7 +2,7 @@ import { mkdir, mkdtemp, readFile, rm, writeFile } from "node:fs/promises";
import { tmpdir } from "node:os";
import path from "node:path";
import { afterEach, describe, expect, it } from "vitest";
import { buildModuleUi, ModuleUiBuildError } from "../src/index.js";
import { analyzeModuleUi, buildModuleUi, ModuleUiBuildError } from "../src/index.js";
const temporaryDirectories: string[] = [];
@@ -502,3 +502,87 @@ describe("module UI build", () => {
expect(artifact).not.toContain("process.env.NODE_ENV");
});
});
describe("module UI build analysis", () => {
const styledSource = `import styles from "./styles.css";
${plainUiSource.replace("const definition = {", "const definition = {\n styles,")}`;
it("reports every source file behind each artifact, stylesheets included, and writes nothing", async () => {
const directory = await createModule({
ui: "./dist/ui/index.js",
files: {
"src/ui/index.tsx": styledSource,
"src/ui/styles.css":
'@import "./base.css";\n.example { color: var(--ekklesia-color-text); }\n',
"src/ui/base.css": ".example-base { margin: 0; }\n",
},
});
await buildModuleUi({ moduleDirectory: directory });
const built = await readFile(path.join(directory, "dist/ui/index.js"));
const analysis = await analyzeModuleUi({ moduleDirectory: directory });
expect(analysis.artifacts).toHaveLength(1);
const [artifact] = analysis.artifacts;
expect(artifact?.kind).toBe("ui");
expect(artifact?.outputFile).toBe(path.join(directory, "dist/ui/index.js"));
expect(artifact?.builtArtifactCurrent).toBe(true);
expect(artifact?.inputs).toEqual(
expect.arrayContaining([
path.join(directory, "src/ui/index.tsx"),
path.join(directory, "src/ui/styles.css"),
path.join(directory, "src/ui/base.css"),
]),
);
expect([...(artifact?.inputs ?? [])]).toEqual([...new Set(artifact?.inputs)].sort());
// Nothing was written: the built artifact is byte-for-byte what it was.
expect(await readFile(path.join(directory, "dist/ui/index.js"))).toEqual(built);
});
it("compiles stylesheets relative to the module, whatever directory the build runs from", async () => {
const directory = await createModule({
ui: "./dist/ui/index.js",
files: {
"src/ui/index.tsx": styledSource,
"src/ui/styles.css": '@import "./base.css";\n.example { margin: 0; }\n',
"src/ui/base.css": ".example-base { margin: 0; }\n",
},
});
// This test runs from the package directory, never from the module's own: the artifact must
// still name its stylesheets relative to the module, or its bytes would depend on the caller.
expect(process.cwd()).not.toBe(directory);
await buildModuleUi({ moduleDirectory: directory });
const artifact = await readFile(path.join(directory, "dist/ui/index.js"), "utf8");
expect(artifact).toContain("/* src/ui/styles.css */");
expect(artifact).toContain("/* src/ui/base.css */");
expect(artifact).not.toContain(path.basename(directory));
});
it("says when a built artifact no longer matches its sources", async () => {
const directory = await createModule({
ui: "./dist/ui/index.js",
files: { "src/ui/index.tsx": plainUiSource },
});
await buildModuleUi({ moduleDirectory: directory });
await writeFile(path.join(directory, "dist/ui/index.js"), "export default {};\n");
const analysis = await analyzeModuleUi({ moduleDirectory: directory });
expect(analysis.artifacts[0]?.builtArtifactCurrent).toBe(false);
});
it("says when a declared artifact was never built", async () => {
const directory = await createModule({
ui: "./dist/ui/index.js",
files: { "src/ui/index.tsx": plainUiSource },
});
const analysis = await analyzeModuleUi({ moduleDirectory: directory });
expect(analysis.artifacts[0]?.builtArtifactCurrent).toBe(false);
});
it("returns an empty analysis for a backend-only module", async () => {
const directory = await createModule();
await expect(analyzeModuleUi({ moduleDirectory: directory })).resolves.toEqual({
artifacts: [],
});
});
});
pnpm-lock.yaml
Generated
+43
View File
@@ -196,6 +196,49 @@ importers:
specifier: 4.1.10
version: 4.1.10(@types/[email protected])([email protected])([email protected](@types/[email protected])([email protected])([email protected])([email protected])([email protected]))
packages/module-pack:
dependencies:
'@ekklesia/contracts':
specifier: workspace:*
version: link:../contracts
'@ekklesia/module-ui-build':
specifier: workspace:*
version: link:../module-ui-build
esbuild:
specifier: 0.28.1
version: 0.28.1
semver:
specifier: 7.8.5
version: 7.8.5
devDependencies:
'@biomejs/biome':
specifier: 2.5.3
version: 2.5.3
'@ekklesia/core':
specifier: workspace:*
version: link:../../core
'@ekklesia/module-sdk':
specifier: workspace:*
version: link:../module-sdk
'@ekklesia/module-template':
specifier: workspace:*
version: link:../../templates/module
'@types/node':
specifier: 24.13.3
version: 24.13.3
'@types/semver':
specifier: 7.7.1
version: 7.7.1
drizzle-orm:
specifier: 0.45.2
version: 0.45.2(@types/[email protected])([email protected])([email protected])
typescript:
specifier: 7.0.2
version: 7.0.2
vitest:
specifier: 4.1.10
version: 4.1.10(@types/[email protected])([email protected])([email protected](@types/[email protected])([email protected])([email protected])([email protected])([email protected]))
packages/module-sdk:
dependencies:
'@ekklesia/contracts':
scripts/lib/build-module-package.Dockerfile
+31
View File
@@ -0,0 +1,31 @@
# Builds the repository's module template and packages it with the canonical Ekklesia packager
# (`@ekklesia/module-pack`, ADR 0049), in a clean container from a clean copy of the checkout. The
# deploy smoke test uses it because a CI job that drives Docker has no Node.js of its own. The
# packaging itself is exactly what a module's CI runs; this file only provides the environment.
#
# DOCKER_BUILDKIT=1 docker build --file scripts/lib/build-module-package.Dockerfile \
# --build-arg SOURCE_REVISION=<full commit id> --output type=local,dest=<directory> .
#
# writes <directory>/example-<version>-any.tar.gz and its .sha256 line. No image is produced.
#
# The same pinned Node.js base as deploy/Dockerfile, so the package is built by the runtime it is
# installed into.
FROM node:24.18.0-trixie-slim@sha256:ae91dcc111a68c9d2d81ff2a17bda61be126426176fde6fe7d08ab13b7f50573 AS build
ENV COREPACK_ENABLE_DOWNLOAD_PROMPT=0 \
PNPM_HOME=/pnpm \
TURBO_TELEMETRY_DISABLED=1 \
DO_NOT_TRACK=1
WORKDIR /opt/source
COPY package.json ./
RUN corepack enable && corepack install
COPY . .
RUN --mount=type=cache,id=ekklesia-pnpm-store,target=/pnpm/store \
pnpm install --frozen-lockfile --store-dir /pnpm/store
RUN pnpm exec turbo run build --filter=@ekklesia/module-template --filter=@ekklesia/module-pack
ARG SOURCE_REVISION
RUN test -n "$SOURCE_REVISION" && \
node packages/module-pack/bin/ekklesia-pack-module.mjs --module templates/module --target any \
--out /package --source-revision "$SOURCE_REVISION"
FROM scratch AS package
COPY --from=build /package/ /
scripts/lib/module-package-release.sh
+53
View File
@@ -0,0 +1,53 @@
# shellcheck shell=bash
# The deploy smoke test's module package release pair (ADR 0049).
#
# A package archive and its `.sha256` are one logical release: they travel together, and the
# verifier reads both. This turns one build's output into exactly that pair, in a directory any user
# can traverse, because the verifier runs as the image's unprivileged user.
#
# BuildKit's local exporter creates its output directory with mode 0700, owned by whoever ran the
# build. On a workstation that is usually uid 1000, which is also the image's `node` user, so the
# verifier happened to read it; in CI the build runs as root, and the verifier could read neither file.
# Nothing here depends on who built the package.
# module_package_release <build output directory> <new release directory>
#
# Checks the build output holds exactly one archive and its digest and nothing else, and that the
# digest line names that archive. Then creates the release directory, which must not exist yet (so an
# earlier run's files can never be mixed in), mode 0755, and copies both files into it, mode 0644.
# Prints the archive's file name. Fails, creating nothing, on any deviation.
module_package_release() {
local output="$1" release="$2" archives=() archive digest_line digest named entries
shopt -s nullglob
archives=("$output"/*.tar.gz)
shopt -u nullglob
if ((${#archives[@]} != 1)); then
echo "module_package_release: the build output must hold exactly one package archive." >&2
return 1
fi
archive="$(basename -- "${archives[0]}")"
if [[ ! -f "$output/$archive.sha256" ]]; then
echo "module_package_release: the package's .sha256 is missing." >&2
return 1
fi
entries="$(find "$output" -mindepth 1 -maxdepth 1 | wc -l)"
if ((entries != 2)); then
echo "module_package_release: the build output holds more than the archive and its digest." >&2
return 1
fi
digest_line="$(<"$output/$archive.sha256")"
digest="${digest_line%% *}"
named="${digest_line#* }"
if [[ ! "$digest" =~ ^[0-9a-f]{64}$ || "$named" != "$archive" ]]; then
echo "module_package_release: the .sha256 line does not name this archive." >&2
return 1
fi
mkdir -m 755 -- "$release" || return 1
if ! install -m 0644 -- "$output/$archive" "$output/$archive.sha256" "$release/"; then
rm -rf -- "$release"
return 1
fi
printf '%s\n' "$archive"
}
scripts/lib/test-module-package.Dockerfile
-50
View File
@@ -1,50 +0,0 @@
# TEST-ONLY. Builds the repository's module template as one self-contained installed-module directory
# for scripts/smoke-deploy.sh, which installs it into a deployment's module storage (ADR 0048). It is
# not a module package format and defines none: the archive format, signing, and transport of real
# module packages are deferred. No image is produced; the build's only output is the directory:
#
# DOCKER_BUILDKIT=1 docker build --file scripts/lib/test-module-package.Dockerfile \
# --output type=local,dest=<directory> .
#
# writes <directory>/example: the module's package.json, manifest, built artifacts, migrations, and a
# node_modules holding exactly its production dependencies, including its own copies of the Module
# SDK, the contracts, and Drizzle. Nothing in it resolves to anything outside it, because the
# platform lends an installed module nothing.
#
# The same pinned Node.js base as deploy/Dockerfile, so the package targets the runtime it is
# installed into. Drizzle comes from the platform's own lockfile, which is what keeps it the exact
# version the platform uses: a constraint of the current implementation (ADR 0048), not a contract.
FROM node:24.18.0-trixie-slim@sha256:ae91dcc111a68c9d2d81ff2a17bda61be126426176fde6fe7d08ab13b7f50573 AS build
ENV COREPACK_ENABLE_DOWNLOAD_PROMPT=0 \
PNPM_HOME=/pnpm \
TURBO_TELEMETRY_DISABLED=1 \
DO_NOT_TRACK=1
WORKDIR /opt/source
COPY package.json ./
RUN corepack enable && corepack install
COPY . .
RUN --mount=type=cache,id=ekklesia-pnpm-store,target=/pnpm/store \
pnpm install --frozen-lockfile --store-dir /pnpm/store
RUN pnpm exec turbo run build --filter=@ekklesia/module-template
# `--legacy` deploys from this workspace without switching it to injected workspace packages; the
# Module SDK and every other workspace dependency are still copied into the package, built.
RUN --mount=type=cache,id=ekklesia-pnpm-store,target=/pnpm/store \
set -eu; \
pnpm --filter @ekklesia/module-template deploy --prod --legacy --store-dir /pnpm/store \
/package/example; \
rm -f /package/example/README.md; \
# pnpm's hoisted self-reference points back into this workspace; an installed module has none.
rm -f /package/example/node_modules/.pnpm/node_modules/@ekklesia/module-template; \
escaping=""; \
for link in $(find /package/example -type l); do \
case "$(realpath -m "$link")" in \
/package/example/*) ;; \
*) escaping="$escaping $link" ;; \
esac; \
done; \
if [ -n "$escaping" ]; then echo "Links escape the package:$escaping" >&2; exit 1; fi; \
chmod -R a+rX,go-w /package/example; \
cd /package/example && node -e 'import("./dist/server/index.js").then((m) => { if (m.default.id !== "example") process.exit(1); })'
FROM scratch AS package
COPY --from=build /package/ /
scripts/smoke-deploy.sh
+53 -22
View File
@@ -15,13 +15,15 @@
# ADR 0048 end to end with that same image:
#
# - it boots with no modules;
# - the repository's module template, packaged as one self-contained directory
# (scripts/lib/test-module-package.Dockerfile, TEST-ONLY), is installed into module storage, its
# migrations are planned and explicitly applied, and the API is restarted;
# - the repository's module template is packaged by the canonical packager as an Ekklesia Module
# Package v1 (ADR 0049; scripts/lib/build-module-package.Dockerfile only provides the build
# environment), verified by the same image's own verifier, extracted to staging, installed into
# module storage, its migrations planned and explicitly applied, and the API restarted;
# - a real headless Chromium drives the Dashboard, the module, its display, and sign-out;
# - the module is removed and the API restarted: Core still runs, migration planning reports the
# module's retained history, and its data survives;
# - the module is restored and the API restarted: it returns with its previous data.
# - the exact same package is verified and installed again, and the API restarted: the module
# returns with its previous data.
#
# EKKLESIA_SMOKE_IMAGE uses the given image instead of building one. It must be a generic image
# (ADR 0048): modules come from module storage, never from the image.
@@ -52,6 +54,8 @@ cd "$repository_root"
source "$repository_root/scripts/lib/compose-diagnostics.sh"
# shellcheck source=scripts/lib/ci-project.sh
source "$repository_root/scripts/lib/ci-project.sh"
# shellcheck source=scripts/lib/module-package-release.sh
source "$repository_root/scripts/lib/module-package-release.sh"
for command in docker openssl awk; do
command -v "$command" >/dev/null || {
@@ -179,7 +183,7 @@ cleanup() {
# job itself runs as root), so only the staged copy on the Docker host needs this.
if [[ "$daemon_work" == "$work" ]]; then
docker run --rm --entrypoint rm -v "$work:/work" "$postgres_image" -rf /work/tls /work/env \
/work/modules >/dev/null 2>&1 || true
/work/modules /work/staging >/dev/null 2>&1 || true
else
docker run --rm --entrypoint rm -v /tmp:/host-tmp "$postgres_image" -rf "/host-tmp/${project}" \
>/dev/null 2>&1 || true
@@ -213,15 +217,23 @@ if [[ -z "$image" ]]; then
fi
if [[ "$topology" == split ]]; then
# TEST-ONLY: the module template as one self-contained installed-module directory, built outside
# the platform image and installed into module storage below. Never a second platform image.
step "Packaging the module template as a self-contained test module"
DOCKER_BUILDKIT=1 docker build --quiet --file scripts/lib/test-module-package.Dockerfile \
--label "${EKKLESIA_CI_LABEL}=${project}" \
# The module template as a Module Package v1, from the canonical packager. Never a second platform
# image: the package is installed into module storage below, after the image verifies it.
step "Packaging the module template with the canonical Ekklesia packager"
source_revision="${GITHUB_SHA:-$(git rev-parse HEAD 2>/dev/null || true)}"
[[ "$source_revision" =~ ^[0-9a-f]{40}$ ]] || fail "no source revision to record in the package."
DOCKER_BUILDKIT=1 docker build --quiet --file scripts/lib/build-module-package.Dockerfile \
--label "${EKKLESIA_CI_LABEL}=${project}" --build-arg SOURCE_REVISION="$source_revision" \
--output "type=local,dest=${work}/module-package" . >/dev/null ||
fail "the test module package did not build."
[[ -f "$work/module-package/example/ekklesia.module.json" ]] ||
fail "the test module package has no manifest."
fail "the module package did not build."
# The archive and its digest are one release: copied together, from this run's own build output,
# into a fresh directory the verifier's unprivileged user can read. BuildKit's output directory is
# mode 0700 and owned by whoever built it, so it is never handed to the verifier itself.
package_archive="$(module_package_release "$work/module-package" "$work/package")" ||
fail "the packager did not produce exactly one archive and its digest."
[[ "$package_archive" =~ ^example-[0-9]+\.[0-9]+\.[0-9]+-any\.tar\.gz$ ]] ||
fail "the packager produced an unexpected archive name."
echo "ok packaged ${package_archive} ($(wc -c <"$work/package/$package_archive") bytes, sha256 $(cut -d' ' -f1 "$work/package/${package_archive}.sha256"))"
fi
# The compose project directory: its relative bind mount (postgres/pg_hba.conf) resolves here.
@@ -232,6 +244,13 @@ stage deploy
# `up`: Compose refuses to start `api` or `migrate` without it, and never creates it.
mkdir -m 755 "$work/modules"
stage modules
if [[ "$topology" == split ]]; then
stage package
# TEST-ONLY staging the verifier extracts into, as the image's unprivileged user. Nothing here is
# module storage: the install step below moves a verified tree into module storage as root.
mkdir -m 777 "$work/staging"
stage staging
fi
step "Creating the installation-private CA and server certificates"
init_status=0
@@ -420,15 +439,27 @@ module_storage() {
"$postgres_image" -ec "$1"
}
# install_test_module: stages the package under a hidden name, which discovery ignores, then renames
# it into place in one step, owned by root and readable by the API's unprivileged user.
# install_test_module: verifies the package with the generic image's own verifier, which writes the
# verified module directory to staging, then moves it into module storage under a hidden name, which
# discovery ignores, and renames it into place in one step, owned by root and readable by the API's
# unprivileged user. Nothing unverified ever reaches module storage.
install_test_module() {
tar -C "$work/module-package" -cf - example |
module_storage 'rm -rf /modules/.incoming && mkdir /modules/.incoming &&
tar -xf - -C /modules/.incoming && chown -R 0:0 /modules/.incoming/example &&
chmod -R a+rX,go-w /modules/.incoming/example &&
mv /modules/.incoming/example /modules/example && rmdir /modules/.incoming' ||
fail "the test module could not be installed into module storage."
local verified
verified="$(
docker run --rm --network none --read-only --tmpfs /tmp --cap-drop ALL \
--security-opt no-new-privileges \
-v "$daemon_work/package:/package:ro" -v "$daemon_work/staging:/staging" \
--entrypoint node "$image" core/dist/cli/verify-module-package.js "/package/${package_archive}" \
--sha256-file "/package/${package_archive}.sha256" --extract-to /staging/example 2>&1
)" || fail "the module package did not verify: ${verified}"
echo "ok ${verified%%$'\n'*}"
docker run --rm --network none --entrypoint sh -v "$daemon_work/modules:/modules" \
-v "$daemon_work/staging:/staging" "$postgres_image" -ec '
rm -rf /modules/.incoming-example && cp -R /staging/example /modules/.incoming-example &&
rm -rf /staging/example && chown -R 0:0 /modules/.incoming-example &&
chmod -R a+rX,go-w /modules/.incoming-example &&
mv /modules/.incoming-example /modules/example' ||
fail "the verified module could not be installed into module storage."
}
# restart_api <command...>: stops the API, runs the command, and starts the API again. A controlled
@@ -590,7 +621,7 @@ if [[ "$topology" == split ]]; then
fail "the removed module's data did not survive."
echo "ok the module is gone, Core runs, its history is retained, and its data survives"
step "Restoring the module and restarting: it returns with its previous data"
step "Reinstalling the exact same package and restarting: the module returns with its previous data"
restart_api install_test_module
plan="$(plan_output)"
grep -q '^- \[applied\] example:0000_initial ' <<<"$plan" ||
templates/module/README.md
+8 -2
View File
@@ -55,10 +55,16 @@ To create a module inside this repository:
That `modules/<module-id>` directory is where a module is **developed**. An installation never runs
it from there: Ekklesia's image contains no modules, and an installation installs a self-contained
copy of the module, with its own production dependencies, into its module storage
copy of the module into its module storage
([An installed module is self-contained](../../docs/module-development/README.md#an-installed-module-is-self-contained),
ADR 0048). Copying `dist/` alone is not enough, because the compiled server code imports the Module
SDK and Drizzle by name.
SDK and Drizzle by name. After building, package the module with the canonical packager, which
bundles the backend into one deterministic Ekklesia Module Package (ADR 0049):
```bash
pnpm modules:pack --module templates/module --target any --out <directory> \
--source-revision "$(git rev-parse HEAD)"
```
The module owns its feature screens and display renderer. Do not add platform navigation, the
containing shell, global theme, authentication screens, or top-level routing; those belong to
tests/ci-scripts.test.ts
+35
View File
@@ -151,6 +151,41 @@ describe("CI scripts on a shared Docker daemon", () => {
}
});
it("install the smoke test's module only as a canonical package the image itself verified (ADR 0049)", async () => {
const deploy = (await source("scripts/smoke-deploy.sh")).replace(/\\\n\s*/gu, " ");
const packaging = await source("scripts/lib/build-module-package.Dockerfile");
expect(packaging).toContain("packages/module-pack/bin/ekklesia-pack-module.mjs");
expect(packaging).not.toMatch(/pnpm[^\n]*\bdeploy\b/u);
expect(deploy).toContain("scripts/lib/build-module-package.Dockerfile");
expect(deploy).toMatch(
/--entrypoint node "\$image" core\/dist\/cli\/verify-module-package\.js[^\n]*--sha256-file[^\n]*--extract-to \/staging\/example/u,
);
// Module storage only ever receives the verified tree from staging.
expect(deploy).toContain("cp -R /staging/example /modules/.incoming-example");
});
it("hand the verifier the package's archive and digest together, from a fresh release pair, never from build output", async () => {
const deploy = (await source("scripts/smoke-deploy.sh")).replace(/\\\n\s*/gu, " ");
expect(deploy).toContain('source "$repository_root/scripts/lib/module-package-release.sh"');
// The pair is made from this run's own build output, into a directory that must not exist yet.
expect(deploy).toContain('--output "type=local,dest=${work}/module-package"');
expect(deploy).toContain(
'package_archive="$(module_package_release "$work/module-package" "$work/package")"',
);
expect(deploy).toMatch(/^\s*stage package$/mu);
// BuildKit's output directory is 0700 and owned by whoever built it: it is never mounted.
expect(deploy).not.toContain('"$daemon_work/module-package:');
expect(deploy).not.toMatch(/^\s*stage module-package$/mu);
// The verifier reads both files from the one mount, and always checks the digest file.
const verifier = deploy
.split("\n")
.find((line) => line.includes("core/dist/cli/verify-module-package.js"));
expect(verifier).toContain('-v "$daemon_work/package:/package:ro"');
expect(verifier).toContain('"/package/${package_archive}"');
expect(verifier).toContain('--sha256-file "/package/${package_archive}.sha256"');
expect(verifier).not.toMatch(/--sha256\s/u);
});
it("run every deploy smoke against the one generic image, never a second image with a module in it", async () => {
// ADR 0048: the split topology installs its test module into module storage at run time.
const deploy = (await source("scripts/smoke-deploy.sh")).replace(/\\\n\s*/gu, " ");
tests/module-package-release.test.ts
+127
View File
@@ -0,0 +1,127 @@
import { execFile } from "node:child_process";
import { chmod, mkdir, mkdtemp, readdir, readFile, rm, stat, writeFile } from "node:fs/promises";
import { tmpdir } from "node:os";
import path from "node:path";
import { fileURLToPath } from "node:url";
import { promisify } from "node:util";
import { afterEach, describe, expect, it } from "vitest";
/**
* The deploy smoke test's release pair (ADR 0049): a package archive and its `.sha256` are one
* logical release and reach the verifier together, from a directory every user can traverse.
*
* The regression this exists for: BuildKit's local exporter creates its output directory with mode
* 0700, owned by whoever ran the build. On a workstation that is uid 1000, which is also the image's
* unprivileged `node` user, so the verifier could read it; in CI the build runs as root, and the
* verifier could read neither file ("The SHA-256 file could not be read").
*/
const execFileAsync = promisify(execFile);
const helper = path.resolve(
path.dirname(fileURLToPath(import.meta.url)),
"..",
"scripts",
"lib",
"module-package-release.sh",
);
const archive = "example-0.1.0-any.tar.gz";
const digest = "91e5dfee80880027bd1c2e1796513f07023c03847e9ca3870b51f8272c3accf7";
const temporaryDirectories: string[] = [];
afterEach(async () => {
await Promise.all(
temporaryDirectories
.splice(0)
.map((directory) => rm(directory, { force: true, recursive: true })),
);
});
/** A build output exactly as BuildKit leaves it: mode 0700, holding the archive and its digest. */
async function buildOutput(
files: Readonly<Record<string, string>> = {
[archive]: "archive bytes",
[`${archive}.sha256`]: `${digest} ${archive}\n`,
},
): Promise<{ readonly root: string; readonly output: string }> {
const root = await mkdtemp(path.join(tmpdir(), "ekklesia-release-pair-"));
temporaryDirectories.push(root);
const output = path.join(root, "module-package");
await mkdir(output);
for (const [name, contents] of Object.entries(files)) {
await writeFile(path.join(output, name), contents, { mode: 0o600 });
}
await chmod(output, 0o700);
return { output, root };
}
async function release(output: string, target: string) {
return execFileAsync("bash", [
"-c",
'source "$1"; module_package_release "$2" "$3"',
"release",
helper,
output,
target,
]);
}
describe("the smoke test's package release pair", () => {
it("copies the archive and its digest, together, into a fresh world-traversable directory", async () => {
const { output, root } = await buildOutput();
const target = path.join(root, "package");
const { stdout } = await release(output, target);
expect(stdout.trim()).toBe(archive);
expect((await readdir(target)).sort()).toEqual([archive, `${archive}.sha256`]);
expect((await stat(target)).mode & 0o777).toBe(0o755);
for (const name of [archive, `${archive}.sha256`]) {
expect((await stat(path.join(target, name))).mode & 0o777, name).toBe(0o644);
expect(await readFile(path.join(target, name))).toEqual(
await readFile(path.join(output, name)),
);
}
});
it.each([
["the digest is missing", { [archive]: "archive bytes" }],
[
"the digest names another file",
{ [archive]: "a", [`${archive}.sha256`]: `${digest} other.tar.gz\n` },
],
["the digest is malformed", { [archive]: "a", [`${archive}.sha256`]: "not-a-digest x\n" }],
[
"the build left something else beside the pair",
{
[archive]: "a",
[`${archive}.sha256`]: `${digest} ${archive}\n`,
"stale.tar.gz.sha256": "x",
},
],
[
"there is more than one archive",
{
[archive]: "a",
[`${archive}.sha256`]: `${digest} ${archive}\n`,
"example-0.0.9-any.tar.gz": "old",
},
],
["there is no archive at all", {}],
])("refuses when %s", async (_label, files) => {
const { output, root } = await buildOutput(files);
const target = path.join(root, "package");
await expect(release(output, target)).rejects.toThrow();
await expect(stat(target)).rejects.toMatchObject({ code: "ENOENT" });
});
it("never reuses a release directory that already exists", async () => {
const { output, root } = await buildOutput();
const target = path.join(root, "package");
await mkdir(target);
await writeFile(path.join(target, `${archive}.sha256`), `${"0".repeat(64)} ${archive}\n`);
await expect(release(output, target)).rejects.toThrow();
// The stale digest is left exactly as it was; nothing was mixed into it.
expect(await readdir(target)).toEqual([`${archive}.sha256`]);
});
});
turbo.json
+4
View File
@@ -14,6 +14,10 @@
"dependsOn": ["^build"],
"outputs": []
},
"@ekklesia/module-pack#test": {
"dependsOn": ["^build", "@ekklesia/module-template#test"],
"outputs": []
},
"typecheck": {
"dependsOn": ["^build", "^typecheck"],
"outputs": []