feat: native linux-x64-glibc module packages and reviewed license supplements (ADR 0049, ADR 0052)
CI / verify (pull_request) Successful in 1m6s
CI / infrastructure-smoke (pull_request) Successful in 27s
CI / migration-smoke (pull_request) Successful in 1m54s
CI / image-build (pull_request) Successful in 29s
CI / deploy-smoke (same-origin) (pull_request) Successful in 43s
CI / deploy-smoke (split) (pull_request) Successful in 1m30s
CI / verify (pull_request) Successful in 1m6s
CI / infrastructure-smoke (pull_request) Successful in 27s
CI / migration-smoke (pull_request) Successful in 1m54s
CI / image-build (pull_request) Successful in 29s
CI / deploy-smoke (same-origin) (pull_request) Successful in 43s
CI / deploy-smoke (split) (pull_request) Successful in 1m30s
Implements ADR 0049's deferred native target and ADR 0052's reviewed license supplements. The proof case is a real module using sharp. packageFormat stays 1, and `any` is unchanged. Packager (@ekklesia/module-pack): - --target linux-x64-glibc. Native or platform-specific packages, found by the same generic rule that refuses them in `any`, stay outside the bundle. - Each native package's production closure is walked through its own dependencies, installed optionalDependencies that match the target, and required peers. It is copied as real files flat under node_modules/<name>, with no links, dev dependencies, declarations, source maps, or other platforms' packages. Missing or foreign-platform required dependencies, version conflicts, links, foreign machines, and Node.js range conflicts fail closed. - runtime.nodeApi, nodeAbi, and libcVersion are read from the binaries by a small ELF reader: the addon's node_api_module_get_api_version_v1 and the newest GLIBC_ symbol version needed. No shelling out. A native target with no addon is refused. - ADR 0052 supplements: package-provided material stays primary. An exact match on name, version, and ekklesia-content-v1 digest (sorted normalized paths with each file's size and SHA-256; no metadata) adds labelled material after the package's own. Any mismatch, a declared-licence conflict, or a malformed or unreviewed supplement file fails. Missing material with no supplement still fails. Supplement for @img/[email protected]: - 184 components and 143 unique texts, assembled from the exact upstream sources sharp-libvips v1.3.3 builds linux-x64 from: 27 C/C++ libraries, glib's gvdb, librsvg's Rust crates (reproduced with cargo tree), the Rust standard library at the rustc commit recorded in the binary, plus the LGPL-3.0, GPL-3.0, and MPL-2.0 texts. Each source is recorded with its SHA-256 in PROVENANCE.md. - It pins digest ekklesia-content-v1:sha256:50647b0b…, which is identical from the installed copy and from the registry tarball. - LGPL/MPL source-availability determinations stay open, as ADR 0052 states. Nothing claims they are met. Verifier (Core): - linux-x64-glibc is accepted only on a host whose platform, arch, glibc family and version, and Node-API version (from Node itself) match. - Every ELF file must be x86-64; every .node must be an ELF object. - `any` packages now also refuse node_modules/ and ELF files. - Contract: native metadata must state nodeApi or nodeAbi, and Linux targets libcVersion (ADR 0049). Proof: - A checked-in AGPL native fixture packs byte-identically from unrelated copies (8.86 MB) and verifies with Core's verifier. - Extracted elsewhere, it runs a sharp resize and PNG round trip in an empty-environment process. Every resolution stays in the package, and only the package's addon and libvips plus host system libraries load. Signage is untouched. Co-Authored-By: Claude Opus 5.5 <[email protected]>
This commit is contained in:
co-authored by
Claude Opus 5.5
parent
f53506b3f0
commit
44d385cd7f
@@ -0,0 +1,357 @@
|
||||
import { lstat, readdir, readFile, realpath } from "node:fs/promises";
|
||||
import path from "node:path";
|
||||
import { describeElf, ELF_MACHINE_X86_64, ElfFormatError, isElf } from "./elf.js";
|
||||
|
||||
/**
|
||||
* Native targets of Ekklesia Module Package v1 (ADR 0049): what a `<platform>-<arch>[-<libc>]`
|
||||
* package may contain beyond bundled JavaScript, and how the packager establishes it.
|
||||
*
|
||||
* A native or platform-specific dependency cannot be bundled. The packager keeps it outside the
|
||||
* bundle and copies it, with exactly its own production runtime dependencies, as real files flat
|
||||
* under `node_modules/<name>`. Nothing is installed, linked, or downloaded. Which packages qualify is
|
||||
* decided generically, from each package's own metadata and files, never from a list of names.
|
||||
*/
|
||||
|
||||
export interface NativeTarget {
|
||||
readonly id: string;
|
||||
readonly os: string;
|
||||
readonly cpu: string;
|
||||
readonly libc: string;
|
||||
readonly elfMachine: number;
|
||||
/** Shared libraries whose presence means a binary was linked against another C library. */
|
||||
readonly foreignLibc: RegExp;
|
||||
readonly libcVersionPrefix: string;
|
||||
}
|
||||
|
||||
/** The native targets this packager produces. Every other target is refused. */
|
||||
export const NATIVE_TARGETS: ReadonlyMap<string, NativeTarget> = new Map([
|
||||
[
|
||||
"linux-x64-glibc",
|
||||
Object.freeze({
|
||||
cpu: "x64",
|
||||
elfMachine: ELF_MACHINE_X86_64,
|
||||
foreignLibc: /^(?:libc\.musl-|ld-musl-)/u,
|
||||
id: "linux-x64-glibc",
|
||||
libc: "glibc",
|
||||
libcVersionPrefix: "GLIBC_",
|
||||
os: "linux",
|
||||
}),
|
||||
],
|
||||
]);
|
||||
|
||||
export type NativeErrorCode =
|
||||
| "MODULE_PACK_NATIVE_DEPENDENCY_MISSING"
|
||||
| "MODULE_PACK_NATIVE_LAYOUT_CONFLICT"
|
||||
| "MODULE_PACK_NATIVE_PACKAGE_INVALID"
|
||||
| "MODULE_PACK_NATIVE_RUNTIME_UNKNOWN"
|
||||
| "MODULE_PACK_NATIVE_TARGET_MISMATCH";
|
||||
|
||||
export class NativeError extends Error {
|
||||
readonly code: NativeErrorCode;
|
||||
readonly detail: string;
|
||||
|
||||
constructor(code: NativeErrorCode, detail: string) {
|
||||
super(code);
|
||||
this.name = "NativeError";
|
||||
this.code = code;
|
||||
this.detail = detail;
|
||||
}
|
||||
}
|
||||
|
||||
function isRecord(value: unknown): value is Record<string, unknown> {
|
||||
return typeof value === "object" && value !== null && !Array.isArray(value);
|
||||
}
|
||||
|
||||
function compare(left: string, right: string): number {
|
||||
return left < right ? -1 : left > right ? 1 : 0;
|
||||
}
|
||||
|
||||
/** npm's `os`, `cpu`, and `libc` fields: absent means any; entries are names or `!`-negations. */
|
||||
function fieldAllows(field: unknown, value: string): boolean {
|
||||
if (!Array.isArray(field) || field.length === 0) {
|
||||
return true;
|
||||
}
|
||||
const entries = field.filter((entry): entry is string => typeof entry === "string");
|
||||
if (entries.includes(`!${value}`)) {
|
||||
return false;
|
||||
}
|
||||
const positive = entries.filter((entry) => !entry.startsWith("!"));
|
||||
return positive.length === 0 || positive.includes(value);
|
||||
}
|
||||
|
||||
export function packageSupportsTarget(
|
||||
manifest: Readonly<Record<string, unknown>>,
|
||||
target: NativeTarget,
|
||||
): boolean {
|
||||
return (
|
||||
fieldAllows(manifest.os, target.os) &&
|
||||
fieldAllows(manifest.cpu, target.cpu) &&
|
||||
fieldAllows(manifest.libc, target.libc)
|
||||
);
|
||||
}
|
||||
|
||||
async function readManifest(directory: string): Promise<Record<string, unknown> | undefined> {
|
||||
try {
|
||||
const parsed = JSON.parse(
|
||||
await readFile(path.join(directory, "package.json"), "utf8"),
|
||||
) as unknown;
|
||||
return isRecord(parsed) ? parsed : undefined;
|
||||
} catch {
|
||||
return undefined;
|
||||
}
|
||||
}
|
||||
|
||||
/** The installed package `name` resolves to from `fromDirectory`, by Node's own directory walk. */
|
||||
async function resolveInstalled(fromDirectory: string, name: string): Promise<string | undefined> {
|
||||
let directory = fromDirectory;
|
||||
for (;;) {
|
||||
if (path.basename(directory) !== "node_modules") {
|
||||
const candidate = path.join(directory, "node_modules", ...name.split("/"));
|
||||
if (await readManifest(candidate)) {
|
||||
return realpath(candidate);
|
||||
}
|
||||
}
|
||||
const parent = path.dirname(directory);
|
||||
if (parent === directory) {
|
||||
return undefined;
|
||||
}
|
||||
directory = parent;
|
||||
}
|
||||
}
|
||||
|
||||
/** Declarations and source maps never ship (ADR 0049); everything else a package publishes does,
|
||||
* because the packager cannot know which of a third-party package's files it reads at run time. */
|
||||
function excludedFromCopy(relative: string): boolean {
|
||||
return /\.(?:d\.ts|d\.mts|d\.cts|map)$/u.test(relative);
|
||||
}
|
||||
|
||||
/** Every regular file a package publishes, by path relative to its root, nested `node_modules`
|
||||
* excluded (those are separate packages). A link or special file fails: none may enter a package. */
|
||||
async function readPackageFiles(root: string, packageId: string): Promise<Map<string, Buffer>> {
|
||||
const files = new Map<string, Buffer>();
|
||||
const walk = async (relative: string): Promise<void> => {
|
||||
const directory = relative === "" ? root : path.join(root, relative);
|
||||
const entries = (await readdir(directory, { withFileTypes: true })).sort((left, right) =>
|
||||
compare(left.name, right.name),
|
||||
);
|
||||
for (const entry of entries) {
|
||||
const childRelative = relative === "" ? entry.name : `${relative}/${entry.name}`;
|
||||
const stats = await lstat(path.join(directory, entry.name));
|
||||
if (stats.isDirectory()) {
|
||||
if (entry.name !== "node_modules") {
|
||||
await walk(childRelative);
|
||||
}
|
||||
} else if (stats.isFile()) {
|
||||
if (!excludedFromCopy(childRelative)) {
|
||||
files.set(childRelative, await readFile(path.join(directory, entry.name)));
|
||||
}
|
||||
} else {
|
||||
throw new NativeError("MODULE_PACK_NATIVE_PACKAGE_INVALID", packageId);
|
||||
}
|
||||
}
|
||||
};
|
||||
await walk("");
|
||||
return files;
|
||||
}
|
||||
|
||||
/** A package copied into the module package's `node_modules`. */
|
||||
export interface NativePackage {
|
||||
readonly name: string;
|
||||
readonly version: string;
|
||||
readonly root: string;
|
||||
readonly manifest: Readonly<Record<string, unknown>>;
|
||||
/** The files copied, by path relative to the package root: exactly what is redistributed. */
|
||||
readonly files: ReadonlyMap<string, Buffer>;
|
||||
}
|
||||
|
||||
export interface NativeRuntime {
|
||||
readonly nodeApi?: number;
|
||||
readonly nodeAbi?: string;
|
||||
readonly libcVersion?: string;
|
||||
}
|
||||
|
||||
function dependencyNames(manifest: Readonly<Record<string, unknown>>): {
|
||||
readonly required: readonly string[];
|
||||
readonly optional: readonly string[];
|
||||
} {
|
||||
const names = (field: unknown): string[] => (isRecord(field) ? Object.keys(field) : []);
|
||||
const optional = new Set(names(manifest.optionalDependencies));
|
||||
const peerMeta = isRecord(manifest.peerDependenciesMeta) ? manifest.peerDependenciesMeta : {};
|
||||
for (const peer of names(manifest.peerDependencies)) {
|
||||
const meta = peerMeta[peer];
|
||||
if (isRecord(meta) && meta.optional === true) {
|
||||
optional.add(peer);
|
||||
}
|
||||
}
|
||||
const required = new Set(
|
||||
[...names(manifest.dependencies), ...names(manifest.peerDependencies)].filter(
|
||||
(name) => !optional.has(name),
|
||||
),
|
||||
);
|
||||
return {
|
||||
optional: [...optional].sort(compare),
|
||||
required: [...required].sort(compare),
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* The runtime closure of the native roots for `target`: each root, its production `dependencies`,
|
||||
* its installed `optionalDependencies` whose `os`/`cpu`/`libc` admit the target, and its required
|
||||
* peers, recursively. Development dependencies are never followed. A required dependency that is
|
||||
* missing or built for another platform fails; an optional one for another platform is left out.
|
||||
* Two different versions of one name cannot share the flat layout and fail.
|
||||
*/
|
||||
export async function nativeClosure(
|
||||
roots: readonly { readonly name: string; readonly root: string }[],
|
||||
target: NativeTarget,
|
||||
): Promise<readonly NativePackage[]> {
|
||||
const byName = new Map<string, NativePackage>();
|
||||
const queue = [...roots].sort((left, right) => compare(left.name, right.name));
|
||||
for (let next = queue.shift(); next !== undefined; next = queue.shift()) {
|
||||
const root = await realpath(next.root);
|
||||
const manifest = await readManifest(root);
|
||||
if (
|
||||
manifest === undefined ||
|
||||
typeof manifest.name !== "string" ||
|
||||
typeof manifest.version !== "string" ||
|
||||
manifest.name !== next.name
|
||||
) {
|
||||
throw new NativeError("MODULE_PACK_NATIVE_PACKAGE_INVALID", next.name);
|
||||
}
|
||||
const packageId = `${manifest.name}@${manifest.version}`;
|
||||
const existing = byName.get(manifest.name);
|
||||
if (existing !== undefined) {
|
||||
if (existing.version !== manifest.version) {
|
||||
throw new NativeError("MODULE_PACK_NATIVE_LAYOUT_CONFLICT", manifest.name);
|
||||
}
|
||||
continue;
|
||||
}
|
||||
if (!packageSupportsTarget(manifest, target)) {
|
||||
throw new NativeError("MODULE_PACK_NATIVE_TARGET_MISMATCH", packageId);
|
||||
}
|
||||
byName.set(
|
||||
manifest.name,
|
||||
Object.freeze({
|
||||
files: await readPackageFiles(root, packageId),
|
||||
manifest: Object.freeze(manifest),
|
||||
name: manifest.name,
|
||||
root,
|
||||
version: manifest.version,
|
||||
}),
|
||||
);
|
||||
const { optional, required } = dependencyNames(manifest);
|
||||
for (const name of required) {
|
||||
const resolved = await resolveInstalled(root, name);
|
||||
if (resolved === undefined) {
|
||||
throw new NativeError("MODULE_PACK_NATIVE_DEPENDENCY_MISSING", `${packageId} -> ${name}`);
|
||||
}
|
||||
queue.push({ name, root: resolved });
|
||||
}
|
||||
for (const name of optional) {
|
||||
const resolved = await resolveInstalled(root, name);
|
||||
const dependency = resolved === undefined ? undefined : await readManifest(resolved);
|
||||
if (resolved !== undefined && dependency && packageSupportsTarget(dependency, target)) {
|
||||
queue.push({ name, root: resolved });
|
||||
}
|
||||
}
|
||||
}
|
||||
return [...byName.values()].sort((left, right) => compare(left.name, right.name));
|
||||
}
|
||||
|
||||
/** Compares `GLIBC_2.28`-style versions numerically. */
|
||||
function compareVersions(left: readonly number[], right: readonly number[]): number {
|
||||
for (let index = 0; index < Math.max(left.length, right.length); index += 1) {
|
||||
const difference = (left[index] ?? 0) - (right[index] ?? 0);
|
||||
if (difference !== 0) {
|
||||
return difference;
|
||||
}
|
||||
}
|
||||
return 0;
|
||||
}
|
||||
|
||||
/**
|
||||
* The runtime requirements of the copied native files, read from the binaries themselves (ADR 0049):
|
||||
*
|
||||
* - every ELF file must be built for the target's machine and must not link another C library;
|
||||
* - every `.node` file must be an ELF Node.js addon;
|
||||
* - `nodeApi` is the highest Node-API version a Node-API addon declares through its exported
|
||||
* `node_api_module_get_api_version_v1` (an addon without that export is version 8, as Node.js
|
||||
* itself treats it); an ABI-specific addon records its `node_register_module_v<ABI>` as `nodeAbi`;
|
||||
* - `libcVersion` is the highest C library symbol version any file requires.
|
||||
*
|
||||
* Anything that cannot be established exactly fails closed.
|
||||
*/
|
||||
export function nativeRuntime(
|
||||
packages: readonly NativePackage[],
|
||||
target: NativeTarget,
|
||||
): NativeRuntime & { readonly addons: number } {
|
||||
let nodeApi: number | undefined;
|
||||
let nodeAbi: string | undefined;
|
||||
let libc: number[] | undefined;
|
||||
let addons = 0;
|
||||
for (const nativePackage of packages) {
|
||||
for (const [relative, data] of nativePackage.files) {
|
||||
const fileId = `${nativePackage.name}@${nativePackage.version}/${relative}`;
|
||||
const addon = relative.toLowerCase().endsWith(".node");
|
||||
if (!isElf(data)) {
|
||||
if (addon) {
|
||||
throw new NativeError("MODULE_PACK_NATIVE_TARGET_MISMATCH", fileId);
|
||||
}
|
||||
continue;
|
||||
}
|
||||
let elf: ReturnType<typeof describeElf>;
|
||||
try {
|
||||
elf = describeElf(data);
|
||||
} catch (error) {
|
||||
if (error instanceof ElfFormatError) {
|
||||
throw new NativeError("MODULE_PACK_NATIVE_TARGET_MISMATCH", fileId);
|
||||
}
|
||||
throw error;
|
||||
}
|
||||
if (
|
||||
elf.machine !== target.elfMachine ||
|
||||
elf.needed.some((name) => target.foreignLibc.test(name))
|
||||
) {
|
||||
throw new NativeError("MODULE_PACK_NATIVE_TARGET_MISMATCH", fileId);
|
||||
}
|
||||
for (const version of elf.requiredVersions) {
|
||||
if (!version.startsWith(target.libcVersionPrefix)) {
|
||||
continue;
|
||||
}
|
||||
const parts = version.slice(target.libcVersionPrefix.length).split(".").map(Number);
|
||||
if (parts.some((part) => !Number.isInteger(part))) {
|
||||
continue;
|
||||
}
|
||||
if (libc === undefined || compareVersions(parts, libc) > 0) {
|
||||
libc = parts;
|
||||
}
|
||||
}
|
||||
if (!addon) {
|
||||
continue;
|
||||
}
|
||||
addons += 1;
|
||||
if (elf.exported.has("napi_register_module_v1")) {
|
||||
const declared = elf.nodeApiModuleVersion;
|
||||
if (declared === "unrecognized" || (typeof declared === "number" && declared < 1)) {
|
||||
throw new NativeError("MODULE_PACK_NATIVE_RUNTIME_UNKNOWN", fileId);
|
||||
}
|
||||
const version = declared ?? 8;
|
||||
nodeApi = nodeApi === undefined ? version : Math.max(nodeApi, version);
|
||||
} else {
|
||||
const abi = [...elf.exported]
|
||||
.map((name) => /^node_register_module_v(\d+)$/u.exec(name)?.[1])
|
||||
.find((value) => value !== undefined);
|
||||
if (abi === undefined || (nodeAbi !== undefined && nodeAbi !== abi)) {
|
||||
throw new NativeError("MODULE_PACK_NATIVE_RUNTIME_UNKNOWN", fileId);
|
||||
}
|
||||
nodeAbi = abi;
|
||||
}
|
||||
}
|
||||
}
|
||||
return {
|
||||
addons,
|
||||
...(nodeApi === undefined ? {} : { nodeApi }),
|
||||
...(nodeAbi === undefined ? {} : { nodeAbi }),
|
||||
...(libc === undefined ? {} : { libcVersion: `>=${[...libc, 0, 0].slice(0, 3).join(".")}` }),
|
||||
};
|
||||
}
|
||||
Reference in New Issue
Block a user