import { lstat, readdir, readFile, realpath } from "node:fs/promises"; import path from "node:path"; import { describeElf, ELF_MACHINE_X86_64, ElfFormatError, isElf } from "./elf.js"; /** * Native targets of Ekklesia Module Package v1 (ADR 0049): what a `-[-]` * package may contain beyond bundled JavaScript, and how the packager establishes it. * * A native or platform-specific dependency cannot be bundled. The packager keeps it outside the * bundle and copies it, with exactly its own production runtime dependencies, as real files flat * under `node_modules/`. Nothing is installed, linked, or downloaded. Which packages qualify is * decided generically, from each package's own metadata and files, never from a list of names. */ export interface NativeTarget { readonly id: string; readonly os: string; readonly cpu: string; readonly libc: string; readonly elfMachine: number; /** Shared libraries whose presence means a binary was linked against another C library. */ readonly foreignLibc: RegExp; readonly libcVersionPrefix: string; } /** The native targets this packager produces. Every other target is refused. */ export const NATIVE_TARGETS: ReadonlyMap = new Map([ [ "linux-x64-glibc", Object.freeze({ cpu: "x64", elfMachine: ELF_MACHINE_X86_64, foreignLibc: /^(?:libc\.musl-|ld-musl-)/u, id: "linux-x64-glibc", libc: "glibc", libcVersionPrefix: "GLIBC_", os: "linux", }), ], ]); export type NativeErrorCode = | "MODULE_PACK_NATIVE_DEPENDENCY_MISSING" | "MODULE_PACK_NATIVE_LAYOUT_CONFLICT" | "MODULE_PACK_NATIVE_PACKAGE_INVALID" | "MODULE_PACK_NATIVE_RUNTIME_UNKNOWN" | "MODULE_PACK_NATIVE_TARGET_MISMATCH"; export class NativeError extends Error { readonly code: NativeErrorCode; readonly detail: string; constructor(code: NativeErrorCode, detail: string) { super(code); this.name = "NativeError"; this.code = code; this.detail = detail; } } function isRecord(value: unknown): value is Record { return typeof value === "object" && value !== null && !Array.isArray(value); } function compare(left: string, right: string): number { return left < right ? -1 : left > right ? 1 : 0; } /** npm's `os`, `cpu`, and `libc` fields: absent means any; entries are names or `!`-negations. */ function fieldAllows(field: unknown, value: string): boolean { if (!Array.isArray(field) || field.length === 0) { return true; } const entries = field.filter((entry): entry is string => typeof entry === "string"); if (entries.includes(`!${value}`)) { return false; } const positive = entries.filter((entry) => !entry.startsWith("!")); return positive.length === 0 || positive.includes(value); } export function packageSupportsTarget( manifest: Readonly>, target: NativeTarget, ): boolean { return ( fieldAllows(manifest.os, target.os) && fieldAllows(manifest.cpu, target.cpu) && fieldAllows(manifest.libc, target.libc) ); } async function readManifest(directory: string): Promise | undefined> { try { const parsed = JSON.parse( await readFile(path.join(directory, "package.json"), "utf8"), ) as unknown; return isRecord(parsed) ? parsed : undefined; } catch { return undefined; } } /** The installed package `name` resolves to from `fromDirectory`, by Node's own directory walk. */ async function resolveInstalled(fromDirectory: string, name: string): Promise { let directory = fromDirectory; for (;;) { if (path.basename(directory) !== "node_modules") { const candidate = path.join(directory, "node_modules", ...name.split("/")); if (await readManifest(candidate)) { return realpath(candidate); } } const parent = path.dirname(directory); if (parent === directory) { return undefined; } directory = parent; } } /** Declarations and source maps never ship (ADR 0049); everything else a package publishes does, * because the packager cannot know which of a third-party package's files it reads at run time. */ function excludedFromCopy(relative: string): boolean { return /\.(?:d\.ts|d\.mts|d\.cts|map)$/u.test(relative); } /** Every regular file a package publishes, by path relative to its root, nested `node_modules` * excluded (those are separate packages). A link or special file fails: none may enter a package. */ async function readPackageFiles(root: string, packageId: string): Promise> { const files = new Map(); const walk = async (relative: string): Promise => { const directory = relative === "" ? root : path.join(root, relative); const entries = (await readdir(directory, { withFileTypes: true })).sort((left, right) => compare(left.name, right.name), ); for (const entry of entries) { const childRelative = relative === "" ? entry.name : `${relative}/${entry.name}`; const stats = await lstat(path.join(directory, entry.name)); if (stats.isDirectory()) { if (entry.name !== "node_modules") { await walk(childRelative); } } else if (stats.isFile()) { if (!excludedFromCopy(childRelative)) { files.set(childRelative, await readFile(path.join(directory, entry.name))); } } else { throw new NativeError("MODULE_PACK_NATIVE_PACKAGE_INVALID", packageId); } } }; await walk(""); return files; } /** A package copied into the module package's `node_modules`. */ export interface NativePackage { readonly name: string; readonly version: string; readonly root: string; readonly manifest: Readonly>; /** The files copied, by path relative to the package root: exactly what is redistributed. */ readonly files: ReadonlyMap; } export interface NativeRuntime { readonly nodeApi?: number; readonly nodeAbi?: string; readonly libcVersion?: string; } function dependencyNames(manifest: Readonly>): { readonly required: readonly string[]; readonly optional: readonly string[]; } { const names = (field: unknown): string[] => (isRecord(field) ? Object.keys(field) : []); const optional = new Set(names(manifest.optionalDependencies)); const peerMeta = isRecord(manifest.peerDependenciesMeta) ? manifest.peerDependenciesMeta : {}; for (const peer of names(manifest.peerDependencies)) { const meta = peerMeta[peer]; if (isRecord(meta) && meta.optional === true) { optional.add(peer); } } const required = new Set( [...names(manifest.dependencies), ...names(manifest.peerDependencies)].filter( (name) => !optional.has(name), ), ); return { optional: [...optional].sort(compare), required: [...required].sort(compare), }; } /** * The runtime closure of the native roots for `target`: each root, its production `dependencies`, * its installed `optionalDependencies` whose `os`/`cpu`/`libc` admit the target, and its required * peers, recursively. Development dependencies are never followed. A required dependency that is * missing or built for another platform fails; an optional one for another platform is left out. * Two different versions of one name cannot share the flat layout and fail. */ export async function nativeClosure( roots: readonly { readonly name: string; readonly root: string }[], target: NativeTarget, ): Promise { const byName = new Map(); const queue = [...roots].sort((left, right) => compare(left.name, right.name)); for (let next = queue.shift(); next !== undefined; next = queue.shift()) { const root = await realpath(next.root); const manifest = await readManifest(root); if ( manifest === undefined || typeof manifest.name !== "string" || typeof manifest.version !== "string" || manifest.name !== next.name ) { throw new NativeError("MODULE_PACK_NATIVE_PACKAGE_INVALID", next.name); } const packageId = `${manifest.name}@${manifest.version}`; const existing = byName.get(manifest.name); if (existing !== undefined) { if (existing.version !== manifest.version) { throw new NativeError("MODULE_PACK_NATIVE_LAYOUT_CONFLICT", manifest.name); } continue; } if (!packageSupportsTarget(manifest, target)) { throw new NativeError("MODULE_PACK_NATIVE_TARGET_MISMATCH", packageId); } byName.set( manifest.name, Object.freeze({ files: await readPackageFiles(root, packageId), manifest: Object.freeze(manifest), name: manifest.name, root, version: manifest.version, }), ); const { optional, required } = dependencyNames(manifest); for (const name of required) { const resolved = await resolveInstalled(root, name); if (resolved === undefined) { throw new NativeError("MODULE_PACK_NATIVE_DEPENDENCY_MISSING", `${packageId} -> ${name}`); } queue.push({ name, root: resolved }); } for (const name of optional) { const resolved = await resolveInstalled(root, name); const dependency = resolved === undefined ? undefined : await readManifest(resolved); if (resolved !== undefined && dependency && packageSupportsTarget(dependency, target)) { queue.push({ name, root: resolved }); } } } return [...byName.values()].sort((left, right) => compare(left.name, right.name)); } /** Compares `GLIBC_2.28`-style versions numerically. */ function compareVersions(left: readonly number[], right: readonly number[]): number { for (let index = 0; index < Math.max(left.length, right.length); index += 1) { const difference = (left[index] ?? 0) - (right[index] ?? 0); if (difference !== 0) { return difference; } } return 0; } /** * The runtime requirements of the copied native files, read from the binaries themselves (ADR 0049): * * - every ELF file must be built for the target's machine and must not link another C library; * - every `.node` file must be an ELF Node.js addon; * - `nodeApi` is the highest Node-API version a Node-API addon declares through its exported * `node_api_module_get_api_version_v1` (an addon without that export is version 8, as Node.js * itself treats it); an ABI-specific addon records its `node_register_module_v` as `nodeAbi`; * - `libcVersion` is the highest C library symbol version any file requires. * * Anything that cannot be established exactly fails closed. */ export function nativeRuntime( packages: readonly NativePackage[], target: NativeTarget, ): NativeRuntime & { readonly addons: number } { let nodeApi: number | undefined; let nodeAbi: string | undefined; let libc: number[] | undefined; let addons = 0; for (const nativePackage of packages) { for (const [relative, data] of nativePackage.files) { const fileId = `${nativePackage.name}@${nativePackage.version}/${relative}`; const addon = relative.toLowerCase().endsWith(".node"); if (!isElf(data)) { if (addon) { throw new NativeError("MODULE_PACK_NATIVE_TARGET_MISMATCH", fileId); } continue; } let elf: ReturnType; try { elf = describeElf(data); } catch (error) { if (error instanceof ElfFormatError) { throw new NativeError("MODULE_PACK_NATIVE_TARGET_MISMATCH", fileId); } throw error; } if ( elf.machine !== target.elfMachine || elf.needed.some((name) => target.foreignLibc.test(name)) ) { throw new NativeError("MODULE_PACK_NATIVE_TARGET_MISMATCH", fileId); } for (const version of elf.requiredVersions) { if (!version.startsWith(target.libcVersionPrefix)) { continue; } const parts = version.slice(target.libcVersionPrefix.length).split(".").map(Number); if (parts.some((part) => !Number.isInteger(part))) { continue; } if (libc === undefined || compareVersions(parts, libc) > 0) { libc = parts; } } if (!addon) { continue; } addons += 1; if (elf.exported.has("napi_register_module_v1")) { const declared = elf.nodeApiModuleVersion; if (declared === "unrecognized" || (typeof declared === "number" && declared < 1)) { throw new NativeError("MODULE_PACK_NATIVE_RUNTIME_UNKNOWN", fileId); } const version = declared ?? 8; nodeApi = nodeApi === undefined ? version : Math.max(nodeApi, version); } else { const abi = [...elf.exported] .map((name) => /^node_register_module_v(\d+)$/u.exec(name)?.[1]) .find((value) => value !== undefined); if (abi === undefined || (nodeAbi !== undefined && nodeAbi !== abi)) { throw new NativeError("MODULE_PACK_NATIVE_RUNTIME_UNKNOWN", fileId); } nodeAbi = abi; } } } return { addons, ...(nodeApi === undefined ? {} : { nodeApi }), ...(nodeAbi === undefined ? {} : { nodeAbi }), ...(libc === undefined ? {} : { libcVersion: `>=${[...libc, 0, 0].slice(0, 3).join(".")}` }), }; }