feat: native linux-x64-glibc module packages and reviewed license supplements (ADR 0049, ADR 0052)
CI / verify (pull_request) Successful in 1m6s
CI / infrastructure-smoke (pull_request) Successful in 27s
CI / migration-smoke (pull_request) Successful in 1m54s
CI / image-build (pull_request) Successful in 29s
CI / deploy-smoke (same-origin) (pull_request) Successful in 43s
CI / deploy-smoke (split) (pull_request) Successful in 1m30s

Implements ADR 0049's deferred native target and ADR 0052's reviewed
license supplements. The proof case is a real module using sharp.
packageFormat stays 1, and `any` is unchanged.

Packager (@ekklesia/module-pack):
- --target linux-x64-glibc. Native or platform-specific packages, found by
  the same generic rule that refuses them in `any`, stay outside the bundle.
- Each native package's production closure is walked through its own
  dependencies, installed optionalDependencies that match the target, and
  required peers. It is copied as real files flat under node_modules/<name>,
  with no links, dev dependencies, declarations, source maps, or other
  platforms' packages. Missing or foreign-platform required dependencies,
  version conflicts, links, foreign machines, and Node.js range conflicts
  fail closed.
- runtime.nodeApi, nodeAbi, and libcVersion are read from the binaries by a
  small ELF reader: the addon's node_api_module_get_api_version_v1 and the
  newest GLIBC_ symbol version needed. No shelling out. A native target with
  no addon is refused.
- ADR 0052 supplements: package-provided material stays primary. An exact
  match on name, version, and ekklesia-content-v1 digest (sorted normalized
  paths with each file's size and SHA-256; no metadata) adds labelled
  material after the package's own. Any mismatch, a declared-licence
  conflict, or a malformed or unreviewed supplement file fails. Missing
  material with no supplement still fails.

Supplement for @img/[email protected]:
- 184 components and 143 unique texts, assembled from the exact upstream
  sources sharp-libvips v1.3.3 builds linux-x64 from: 27 C/C++ libraries,
  glib's gvdb, librsvg's Rust crates (reproduced with cargo tree), the Rust
  standard library at the rustc commit recorded in the binary, plus the
  LGPL-3.0, GPL-3.0, and MPL-2.0 texts. Each source is recorded with its
  SHA-256 in PROVENANCE.md.
- It pins digest ekklesia-content-v1:sha256:50647b0b…, which is identical
  from the installed copy and from the registry tarball.
- LGPL/MPL source-availability determinations stay open, as ADR 0052
  states. Nothing claims they are met.

Verifier (Core):
- linux-x64-glibc is accepted only on a host whose platform, arch, glibc
  family and version, and Node-API version (from Node itself) match.
- Every ELF file must be x86-64; every .node must be an ELF object.
- `any` packages now also refuse node_modules/ and ELF files.
- Contract: native metadata must state nodeApi or nodeAbi, and Linux targets
  libcVersion (ADR 0049).

Proof:
- A checked-in AGPL native fixture packs byte-identically from unrelated
  copies (8.86 MB) and verifies with Core's verifier.
- Extracted elsewhere, it runs a sharp resize and PNG round trip in an
  empty-environment process. Every resolution stays in the package, and
  only the package's addon and libvips plus host system libraries load.

Signage is untouched.

Co-Authored-By: Claude Opus 5.5 <[email protected]>
This commit is contained in:
ekklesia
2026-10-07 20:16:53 -05:00
co-authored by Claude Opus 5.5
parent f53506b3f0
commit 44d385cd7f
173 changed files with 20638 additions and 53 deletions
+239 -25
View File
@@ -17,9 +17,23 @@ import {
} from "@ekklesia/contracts/module-package";
import { analyzeModuleUi, ModuleUiBuildError } from "@ekklesia/module-ui-build";
import { build, type Message, type Plugin } from "esbuild";
import { minVersion, satisfies, validRange } from "semver";
import { minVersion, satisfies, subset, validRange } from "semver";
import {
NATIVE_TARGETS,
type NativePackage,
type NativeTarget,
NativeError,
type NativeErrorCode,
nativeClosure,
nativeRuntime,
} from "./native.js";
import { type NoticeComponent, NoticeError, type NoticeUse, thirdPartyNotices } from "./notices.js";
import { BundledPackageIndex, isNativeOrPlatformSpecific } from "./packages.js";
import {
type BundledPackage,
BundledPackageIndex,
isNativeOrPlatformSpecific,
} from "./packages.js";
import { loadLicenseSupplements, SupplementError } from "./supplements.js";
import { deterministicTarGz, TarPathError } from "./tar.js";
/**
@@ -28,10 +42,12 @@ import { deterministicTarGz, TarPathError } from "./tar.js";
*
* It reads the module directory and the dependencies its workspace installed, and writes only the
* two output files. It never installs, links, or deploys anything, so the workspace it runs in is
* left exactly as it was. In this release it produces the portable `any` target only.
* left exactly as it was. It produces the portable `any` target, and the native `linux-x64-glibc`
* target, whose native runtime dependencies are copied rather than bundled (see `./native.ts`).
*/
export type ModulePackErrorCode =
| NativeErrorCode
| "MODULE_PACK_BUNDLE_FAILED"
| "MODULE_PACK_DATABASE_RUNTIME_CONFLICT"
| "MODULE_PACK_ENTRYPOINT_MISSING"
@@ -43,12 +59,16 @@ export type ModulePackErrorCode =
| "MODULE_PACK_LAYOUT_CONFLICT"
| "MODULE_PACK_MODULE_LICENSE_FILE_INVALID"
| "MODULE_PACK_MODULE_LICENSE_INVALID"
| "MODULE_PACK_LICENCE_SUPPLEMENT_INVALID"
| "MODULE_PACK_LICENCE_SUPPLEMENT_MISMATCH"
| "MODULE_PACK_LICENCE_TEXT_MISSING"
| "MODULE_PACK_LICENCE_UNKNOWN"
| "MODULE_PACK_MANIFEST_INVALID"
| "MODULE_PACK_METADATA_INVALID"
| "MODULE_PACK_MIGRATIONS_INVALID"
| "MODULE_PACK_NATIVE_DEPENDENCY"
| "MODULE_PACK_NATIVE_NODE_INCOMPATIBLE"
| "MODULE_PACK_NATIVE_TARGET_UNNEEDED"
| "MODULE_PACK_OUTPUT_FAILED"
| "MODULE_PACK_PATH_UNSUPPORTED"
| "MODULE_PACK_REVISION_INVALID"
@@ -78,6 +98,10 @@ const messages: Readonly<Record<ModulePackErrorCode, string>> = Object.freeze({
"The module license text must be supplied as an explicit, non-empty, readable regular file.",
MODULE_PACK_MODULE_LICENSE_INVALID:
"The source package.json must declare a valid SPDX license expression as a string.",
MODULE_PACK_LICENCE_SUPPLEMENT_INVALID:
"A reviewed license supplement is malformed or does not match its reviewed files.",
MODULE_PACK_LICENCE_SUPPLEMENT_MISMATCH:
"A license supplement names this package, but not its exact version, content digest, and declared licence.",
MODULE_PACK_LICENCE_TEXT_MISSING:
"A bundled package ships no licence text, and its licence has no complete standard text to supply.",
MODULE_PACK_LICENCE_UNKNOWN: "A bundled package declares no licence that permits redistribution.",
@@ -87,6 +111,20 @@ const messages: Readonly<Record<ModulePackErrorCode, string>> = Object.freeze({
"The migrations directory must hold only regular files and directories.",
MODULE_PACK_NATIVE_DEPENDENCY:
"The backend bundles a native or platform-specific dependency, which an any-target package cannot contain.",
MODULE_PACK_NATIVE_DEPENDENCY_MISSING:
"A native runtime dependency requires a package that is not installed.",
MODULE_PACK_NATIVE_LAYOUT_CONFLICT:
"Two versions of one package would share the native node_modules layout.",
MODULE_PACK_NATIVE_NODE_INCOMPATIBLE:
"A native runtime dependency does not support every Node.js version the package targets.",
MODULE_PACK_NATIVE_PACKAGE_INVALID:
"A native runtime dependency could not be read, or contains a link or special file.",
MODULE_PACK_NATIVE_RUNTIME_UNKNOWN:
"A native addon's Node.js interface could not be established exactly.",
MODULE_PACK_NATIVE_TARGET_MISMATCH:
"A native runtime dependency, or a file in one, is not built for the package's target.",
MODULE_PACK_NATIVE_TARGET_UNNEEDED:
"A native target needs a native addon; a package without one uses the any target.",
MODULE_PACK_OUTPUT_FAILED: "The package could not be written to the output directory.",
MODULE_PACK_PATH_UNSUPPORTED:
"A package path is not portable printable ASCII that fits the archive format.",
@@ -95,7 +133,8 @@ const messages: Readonly<Record<ModulePackErrorCode, string>> = Object.freeze({
"The Node.js range could not be determined. Run inside an Ekklesia workspace, or pass one.",
MODULE_PACK_SDK_INCOMPATIBLE:
"The bundled Module SDK is outside the manifest's own compatibility.moduleSdk range.",
MODULE_PACK_TARGET_UNSUPPORTED: "This packager produces the any target only.",
MODULE_PACK_TARGET_UNSUPPORTED:
"This packager produces the any and linux-x64-glibc targets only.",
MODULE_PACK_UI_STALE:
"A declared browser artifact is missing or out of date. Build the module before packaging it.",
});
@@ -124,7 +163,7 @@ export interface PackModuleOptions {
readonly moduleDirectory: string;
/** Explicit license-text input. Its bytes are copied unchanged to the package root as LICENSE. */
readonly licenseFile: string;
/** `any` only, in this release. */
/** `any`, or a native target this packager supports (`linux-x64-glibc`). */
readonly target: string;
readonly outDirectory: string;
/** The full commit id the module was built from. */
@@ -134,6 +173,8 @@ export interface PackModuleOptions {
/** The Node.js range the package is built for. Defaults to the enclosing workspace's
* `engines.node`, the toolchain the module was built and tested with. */
readonly nodeRange?: string;
/** Reviewed license supplements (ADR 0052). Defaults to the ones this packager ships. */
readonly supplementsDirectory?: string;
}
export interface ModulePackResult {
@@ -254,15 +295,93 @@ async function readTree(root: string, relative: string, files: Map<string, Buffe
}
}
/** The files redistributed for a component, relative to its root: what it copies, or the source
* files of it that were bundled. The content digest a license supplement names is over these. */
async function redistributedFiles(component: {
readonly bundled: BundledPackage;
readonly files: ReadonlySet<string>;
readonly copied?: ReadonlyMap<string, Buffer>;
}): Promise<ReadonlyMap<string, Uint8Array>> {
if (component.copied !== undefined) {
return component.copied;
}
const files = new Map<string, Uint8Array>();
for (const file of component.files) {
const real = await realpath(file).catch(() => file);
files.set(
path.relative(component.bundled.root, real).split(path.sep).join("/"),
await readFile(real),
);
}
return files;
}
interface BackendBundle {
readonly outputs: ReadonlyMap<string, Buffer>;
readonly inputs: readonly string[];
}
const NATIVE_EXTERNALS = "ekklesia-native-externals";
/** The bare specifier's package name: `sharp/x` → `sharp`, `@scope/name/x` → `@scope/name`. */
function specifierPackage(specifier: string): string {
const segments = specifier.split("/");
return specifier.startsWith("@") ? segments.slice(0, 2).join("/") : (segments[0] ?? "");
}
/**
* For a native target: keeps every native or platform-specific third-party package out of the
* bundle, as an import of the package itself, and records it as a root of the copied runtime
* closure. Decided per resolved package, by the same rule that refuses one in an `any` package.
*/
function nativeExternals(
index: BundledPackageIndex,
externals: Map<string, BundledPackage>,
): Plugin {
return {
name: NATIVE_EXTERNALS,
setup(pluginBuild) {
pluginBuild.onResolve({ filter: /^[^./]/ }, async (args) => {
if (
(args.pluginData as { ekklesiaNative?: boolean } | undefined)?.ekklesiaNative ||
isBuiltin(args.path)
) {
return undefined;
}
const resolved = await pluginBuild.resolve(args.path, {
importer: args.importer,
kind: args.kind,
pluginData: { ekklesiaNative: true },
resolveDir: args.resolveDir,
});
if (resolved.errors.length > 0 || resolved.external || resolved.path === "") {
return undefined;
}
const owner = await index.packageOf(resolved.path);
if (
owner === undefined ||
!owner.thirdParty ||
!(await isNativeOrPlatformSpecific(owner))
) {
return undefined;
}
if (specifierPackage(args.path) !== owner.name) {
return {
errors: [{ text: "A native package must be imported by its own package name." }],
};
}
externals.set(owner.name, owner);
return { external: true, path: args.path };
});
},
};
}
async function bundleBackend(
moduleDirectory: string,
manifest: ModuleManifest,
nodeRange: string,
native?: { readonly index: BundledPackageIndex; readonly externals: Map<string, BundledPackage> },
): Promise<BackendBundle> {
const declared = [
manifest.entrypoints.server,
@@ -303,7 +422,10 @@ async function bundleBackend(
minify: false,
outdir: moduleDirectory,
platform: "node",
plugins: [nativeAddonGuard],
plugins:
native === undefined
? [nativeAddonGuard]
: [nativeExternals(native.index, native.externals), nativeAddonGuard],
sourcemap: false,
splitting: true,
target: `node${minimum?.version ?? "24"}`,
@@ -330,7 +452,11 @@ async function bundleBackend(
}
for (const output of Object.values(metafile.outputs)) {
for (const imported of output.imports) {
if (imported.external && !isBuiltin(imported.path)) {
if (
imported.external &&
!isBuiltin(imported.path) &&
!native?.externals.has(specifierPackage(imported.path))
) {
throw new ModulePackError("MODULE_PACK_EXTERNAL_DEPENDENCY", imported.path);
}
}
@@ -349,9 +475,11 @@ async function bundleBackend(
}
export async function packModule(options: PackModuleOptions): Promise<ModulePackResult> {
if (options.target !== MODULE_PACKAGE_ANY_TARGET) {
const nativeTarget: NativeTarget | undefined = NATIVE_TARGETS.get(options.target);
if (options.target !== MODULE_PACKAGE_ANY_TARGET && nativeTarget === undefined) {
throw new ModulePackError("MODULE_PACK_TARGET_UNSUPPORTED", options.target);
}
const target = options.target;
if (!/^(?:[0-9a-f]{40}|[0-9a-f]{64})$/u.test(options.sourceRevision)) {
throw new ModulePackError("MODULE_PACK_REVISION_INVALID");
}
@@ -414,8 +542,53 @@ export async function packModule(options: PackModuleOptions): Promise<ModulePack
throw new ModulePackError("MODULE_PACK_RUNTIME_UNKNOWN");
}
// 1. The backend, bundled with every pure-JavaScript dependency it uses.
const backend = await bundleBackend(moduleDirectory, manifest, nodeRange);
let supplements: Awaited<ReturnType<typeof loadLicenseSupplements>>;
try {
supplements = await loadLicenseSupplements(options.supplementsDirectory);
} catch (error) {
throw new ModulePackError(
"MODULE_PACK_LICENCE_SUPPLEMENT_INVALID",
error instanceof SupplementError ? error.supplement : undefined,
);
}
// 1. The backend, bundled with every pure-JavaScript dependency it uses. For a native target,
// native packages stay outside the bundle and are copied with their runtime closure instead.
const index = new BundledPackageIndex();
const externals = new Map<string, BundledPackage>();
const backend = await bundleBackend(
moduleDirectory,
manifest,
nodeRange,
nativeTarget === undefined ? undefined : { externals, index },
);
let nativePackages: readonly NativePackage[] = [];
let runtimeRequirements: Record<string, unknown> = {};
if (nativeTarget !== undefined) {
try {
nativePackages = await nativeClosure([...externals.values()], nativeTarget);
const { addons, ...requirements } = nativeRuntime(nativePackages, nativeTarget);
if (addons === 0) {
throw new ModulePackError("MODULE_PACK_NATIVE_TARGET_UNNEEDED", options.target);
}
runtimeRequirements = requirements;
} catch (error) {
if (error instanceof NativeError) {
throw new ModulePackError(error.code, error.detail);
}
throw error;
}
for (const nativePackage of nativePackages) {
const engines = nativePackage.manifest.engines;
const engine = isRecord(engines) ? engines.node : undefined;
if (typeof engine === "string" && !subset(nodeRange, engine)) {
throw new ModulePackError(
"MODULE_PACK_NATIVE_NODE_INCOMPATIBLE",
`${nativePackage.name}@${nativePackage.version}`,
);
}
}
}
// 2. The browser artifacts, exactly as built, with what went into each.
let ui: Awaited<ReturnType<typeof analyzeModuleUi>>;
@@ -428,11 +601,15 @@ export async function packModule(options: PackModuleOptions): Promise<ModulePack
);
}
// 3. Every bundled source file traced to its package.
const index = new BundledPackageIndex();
// 3. Every bundled source file traced to its package, and every copied package.
const components = new Map<
string,
{ bundled: Awaited<ReturnType<typeof index.packageOf>>; uses: Set<NoticeUse> }
{
bundled: BundledPackage;
uses: Set<NoticeUse>;
files: Set<string>;
copied?: ReadonlyMap<string, Buffer>;
}
>();
const incorporates = new Map<string, string>();
const drizzleVersions = new Set<string>();
@@ -464,13 +641,32 @@ export async function packModule(options: PackModuleOptions): Promise<ModulePack
const existing = components.get(key);
if (existing) {
existing.uses.add(use);
existing.files.add(file);
} else {
if (use === "backend" && (await isNativeOrPlatformSpecific(bundled))) {
throw new ModulePackError("MODULE_PACK_NATIVE_DEPENDENCY", key);
}
components.set(key, { bundled, uses: new Set([use]) });
components.set(key, { bundled, files: new Set([file]), uses: new Set([use]) });
}
}
for (const nativePackage of nativePackages) {
const key = `${nativePackage.name}@${nativePackage.version}`;
const existing = components.get(key);
const entry = existing ?? {
bundled: {
manifest: nativePackage.manifest,
name: nativePackage.name,
root: nativePackage.root,
thirdParty: true,
version: nativePackage.version,
},
files: new Set<string>(),
uses: new Set<NoticeUse>(),
};
entry.uses.add("node_modules");
entry.copied = nativePackage.files;
components.set(key, entry);
}
const usesDatabaseAbi = manifest.database?.abi === 1;
if (!usesDatabaseAbi && drizzleVersions.size > 1) {
throw new ModulePackError("MODULE_PACK_DATABASE_RUNTIME_CONFLICT");
@@ -482,11 +678,11 @@ export async function packModule(options: PackModuleOptions): Promise<ModulePack
// 4. The payload.
const files = new Map<string, Buffer>();
const add = (relative: string, data: Buffer): void => {
const add = (relative: string, data: Buffer, copied = false): void => {
if (!portable(relative)) {
throw new ModulePackError("MODULE_PACK_PATH_UNSUPPORTED", relative);
}
if (forbidden(relative)) {
if (!copied && forbidden(relative)) {
throw new ModulePackError("MODULE_PACK_FORBIDDEN_FILE", relative);
}
if (files.has(relative) || relative === MODULE_PACKAGE_METADATA_FILENAME) {
@@ -523,6 +719,14 @@ export async function packModule(options: PackModuleOptions): Promise<ModulePack
add(relative, data);
}
}
// A native target's runtime dependencies, as real files, flat under node_modules/<name>. They are
// third-party packages carried as published, so the module's own source and test path rules do
// not apply to their contents; declarations and source maps were already left out.
for (const nativePackage of nativePackages) {
for (const [relative, data] of nativePackage.files) {
add(`node_modules/${nativePackage.name}/${relative}`, data, true);
}
}
add("ekklesia.module.json", manifestBytes);
add(MODULE_PACKAGE_LICENSE_FILENAME, licenseBytes);
add(
@@ -538,7 +742,21 @@ export async function packModule(options: PackModuleOptions): Promise<ModulePack
try {
add(
MODULE_PACKAGE_NOTICES_FILENAME,
Buffer.from(await thirdPartyNotices(components as ReadonlyMap<string, NoticeComponent>)),
Buffer.from(
await thirdPartyNotices(
new Map(
[...components].map(([key, component]): [string, NoticeComponent] => [
key,
{
bundled: component.bundled,
redistributedFiles: () => redistributedFiles(component),
uses: component.uses,
},
]),
),
supplements,
),
),
);
} catch (error) {
if (error instanceof NoticeError) {
@@ -555,8 +773,8 @@ export async function packModule(options: PackModuleOptions): Promise<ModulePack
packageFormat: MODULE_PACKAGE_FORMAT,
module: { id: manifest.id, version: manifest.version },
license,
target: MODULE_PACKAGE_ANY_TARGET,
runtime: { node: nodeRange },
target,
runtime: { node: nodeRange, ...runtimeRequirements },
provenance: {
revision: options.sourceRevision,
...(options.repository === undefined ? {} : { repository: options.repository }),
@@ -590,11 +808,7 @@ export async function packModule(options: PackModuleOptions): Promise<ModulePack
throw error;
}
const digest = sha256(archive);
const fileName = modulePackageArchiveFileName(
manifest.id,
manifest.version,
MODULE_PACKAGE_ANY_TARGET,
);
const fileName = modulePackageArchiveFileName(manifest.id, manifest.version, target);
const outDirectory = path.resolve(options.outDirectory);
const archivePath = path.join(outDirectory, fileName);
const sha256Path = `${archivePath}.sha256`;
@@ -616,7 +830,7 @@ export async function packModule(options: PackModuleOptions): Promise<ModulePack
moduleId: manifest.id,
sha256: digest,
sha256Path,
target: MODULE_PACKAGE_ANY_TARGET,
target,
version: manifest.version,
});
}