Ekklesia/packages/module-pack/src/native.ts

import { lstat, readdir, readFile, realpath } from "node:fs/promises";
import path from "node:path";
import { describeElf, ELF_MACHINE_X86_64, ElfFormatError, isElf } from "./elf.js";

/**
 * Native targets of Ekklesia Module Package v1 (ADR 0049): what a `<platform>-<arch>[-<libc>]`
 * package may contain beyond bundled JavaScript, and how the packager establishes it.
 *
 * A native or platform-specific dependency cannot be bundled. The packager keeps it outside the
 * bundle and copies it, with exactly its own production runtime dependencies, as real files flat
 * under `node_modules/<name>`. Nothing is installed, linked, or downloaded. Which packages qualify is
 * decided generically, from each package's own metadata and files, never from a list of names.
 */

export interface NativeTarget {
  readonly id: string;
  readonly os: string;
  readonly cpu: string;
  readonly libc: string;
  readonly elfMachine: number;
  /** Shared libraries whose presence means a binary was linked against another C library. */
  readonly foreignLibc: RegExp;
  readonly libcVersionPrefix: string;
}

/** The native targets this packager produces. Every other target is refused. */
export const NATIVE_TARGETS: ReadonlyMap<string, NativeTarget> = new Map([
  [
    "linux-x64-glibc",
    Object.freeze({
      cpu: "x64",
      elfMachine: ELF_MACHINE_X86_64,
      foreignLibc: /^(?:libc\.musl-|ld-musl-)/u,
      id: "linux-x64-glibc",
      libc: "glibc",
      libcVersionPrefix: "GLIBC_",
      os: "linux",
    }),
  ],
]);

export type NativeErrorCode =
  | "MODULE_PACK_NATIVE_DEPENDENCY_MISSING"
  | "MODULE_PACK_NATIVE_LAYOUT_CONFLICT"
  | "MODULE_PACK_NATIVE_PACKAGE_INVALID"
  | "MODULE_PACK_NATIVE_RUNTIME_UNKNOWN"
  | "MODULE_PACK_NATIVE_TARGET_MISMATCH";

export class NativeError extends Error {
  readonly code: NativeErrorCode;
  readonly detail: string;

  constructor(code: NativeErrorCode, detail: string) {
    super(code);
    this.name = "NativeError";
    this.code = code;
    this.detail = detail;
  }
}

function isRecord(value: unknown): value is Record<string, unknown> {
  return typeof value === "object" && value !== null && !Array.isArray(value);
}

function compare(left: string, right: string): number {
  return left < right ? -1 : left > right ? 1 : 0;
}

/** npm's `os`, `cpu`, and `libc` fields: absent means any; entries are names or `!`-negations. */
function fieldAllows(field: unknown, value: string): boolean {
  if (!Array.isArray(field) || field.length === 0) {
    return true;
  }
  const entries = field.filter((entry): entry is string => typeof entry === "string");
  if (entries.includes(`!${value}`)) {
    return false;
  }
  const positive = entries.filter((entry) => !entry.startsWith("!"));
  return positive.length === 0 || positive.includes(value);
}

export function packageSupportsTarget(
  manifest: Readonly<Record<string, unknown>>,
  target: NativeTarget,
): boolean {
  return (
    fieldAllows(manifest.os, target.os) &&
    fieldAllows(manifest.cpu, target.cpu) &&
    fieldAllows(manifest.libc, target.libc)
  );
}

async function readManifest(directory: string): Promise<Record<string, unknown> | undefined> {
  try {
    const parsed = JSON.parse(
      await readFile(path.join(directory, "package.json"), "utf8"),
    ) as unknown;
    return isRecord(parsed) ? parsed : undefined;
  } catch {
    return undefined;
  }
}

/** The installed package `name` resolves to from `fromDirectory`, by Node's own directory walk. */
async function resolveInstalled(fromDirectory: string, name: string): Promise<string | undefined> {
  let directory = fromDirectory;
  for (;;) {
    if (path.basename(directory) !== "node_modules") {
      const candidate = path.join(directory, "node_modules", ...name.split("/"));
      if (await readManifest(candidate)) {
        return realpath(candidate);
      }
    }
    const parent = path.dirname(directory);
    if (parent === directory) {
      return undefined;
    }
    directory = parent;
  }
}

/** Declarations and source maps never ship (ADR 0049); everything else a package publishes does,
 * because the packager cannot know which of a third-party package's files it reads at run time. */
function excludedFromCopy(relative: string): boolean {
  return /\.(?:d\.ts|d\.mts|d\.cts|map)$/u.test(relative);
}

/** Every regular file a package publishes, by path relative to its root, nested `node_modules`
 * excluded (those are separate packages). A link or special file fails: none may enter a package. */
async function readPackageFiles(root: string, packageId: string): Promise<Map<string, Buffer>> {
  const files = new Map<string, Buffer>();
  const walk = async (relative: string): Promise<void> => {
    const directory = relative === "" ? root : path.join(root, relative);
    const entries = (await readdir(directory, { withFileTypes: true })).sort((left, right) =>
      compare(left.name, right.name),
    );
    for (const entry of entries) {
      const childRelative = relative === "" ? entry.name : `${relative}/${entry.name}`;
      const stats = await lstat(path.join(directory, entry.name));
      if (stats.isDirectory()) {
        if (entry.name !== "node_modules") {
          await walk(childRelative);
        }
      } else if (stats.isFile()) {
        if (!excludedFromCopy(childRelative)) {
          files.set(childRelative, await readFile(path.join(directory, entry.name)));
        }
      } else {
        throw new NativeError("MODULE_PACK_NATIVE_PACKAGE_INVALID", packageId);
      }
    }
  };
  await walk("");
  return files;
}

/** A package copied into the module package's `node_modules`. */
export interface NativePackage {
  readonly name: string;
  readonly version: string;
  readonly root: string;
  readonly manifest: Readonly<Record<string, unknown>>;
  /** The files copied, by path relative to the package root: exactly what is redistributed. */
  readonly files: ReadonlyMap<string, Buffer>;
}

export interface NativeRuntime {
  readonly nodeApi?: number;
  readonly nodeAbi?: string;
  readonly libcVersion?: string;
}

function dependencyNames(manifest: Readonly<Record<string, unknown>>): {
  readonly required: readonly string[];
  readonly optional: readonly string[];
} {
  const names = (field: unknown): string[] => (isRecord(field) ? Object.keys(field) : []);
  const optional = new Set(names(manifest.optionalDependencies));
  const peerMeta = isRecord(manifest.peerDependenciesMeta) ? manifest.peerDependenciesMeta : {};
  for (const peer of names(manifest.peerDependencies)) {
    const meta = peerMeta[peer];
    if (isRecord(meta) && meta.optional === true) {
      optional.add(peer);
    }
  }
  const required = new Set(
    [...names(manifest.dependencies), ...names(manifest.peerDependencies)].filter(
      (name) => !optional.has(name),
    ),
  );
  return {
    optional: [...optional].sort(compare),
    required: [...required].sort(compare),
  };
}

/**
 * The runtime closure of the native roots for `target`: each root, its production `dependencies`,
 * its installed `optionalDependencies` whose `os`/`cpu`/`libc` admit the target, and its required
 * peers, recursively. Development dependencies are never followed. A required dependency that is
 * missing or built for another platform fails; an optional one for another platform is left out.
 * Two different versions of one name cannot share the flat layout and fail.
 */
export async function nativeClosure(
  roots: readonly { readonly name: string; readonly root: string }[],
  target: NativeTarget,
): Promise<readonly NativePackage[]> {
  const byName = new Map<string, NativePackage>();
  const queue = [...roots].sort((left, right) => compare(left.name, right.name));
  for (let next = queue.shift(); next !== undefined; next = queue.shift()) {
    const root = await realpath(next.root);
    const manifest = await readManifest(root);
    if (
      manifest === undefined ||
      typeof manifest.name !== "string" ||
      typeof manifest.version !== "string" ||
      manifest.name !== next.name
    ) {
      throw new NativeError("MODULE_PACK_NATIVE_PACKAGE_INVALID", next.name);
    }
    const packageId = `${manifest.name}@${manifest.version}`;
    const existing = byName.get(manifest.name);
    if (existing !== undefined) {
      if (existing.version !== manifest.version) {
        throw new NativeError("MODULE_PACK_NATIVE_LAYOUT_CONFLICT", manifest.name);
      }
      continue;
    }
    if (!packageSupportsTarget(manifest, target)) {
      throw new NativeError("MODULE_PACK_NATIVE_TARGET_MISMATCH", packageId);
    }
    byName.set(
      manifest.name,
      Object.freeze({
        files: await readPackageFiles(root, packageId),
        manifest: Object.freeze(manifest),
        name: manifest.name,
        root,
        version: manifest.version,
      }),
    );
    const { optional, required } = dependencyNames(manifest);
    for (const name of required) {
      const resolved = await resolveInstalled(root, name);
      if (resolved === undefined) {
        throw new NativeError("MODULE_PACK_NATIVE_DEPENDENCY_MISSING", `${packageId} -> ${name}`);
      }
      queue.push({ name, root: resolved });
    }
    for (const name of optional) {
      const resolved = await resolveInstalled(root, name);
      const dependency = resolved === undefined ? undefined : await readManifest(resolved);
      if (resolved !== undefined && dependency && packageSupportsTarget(dependency, target)) {
        queue.push({ name, root: resolved });
      }
    }
  }
  return [...byName.values()].sort((left, right) => compare(left.name, right.name));
}

/** Compares `GLIBC_2.28`-style versions numerically. */
function compareVersions(left: readonly number[], right: readonly number[]): number {
  for (let index = 0; index < Math.max(left.length, right.length); index += 1) {
    const difference = (left[index] ?? 0) - (right[index] ?? 0);
    if (difference !== 0) {
      return difference;
    }
  }
  return 0;
}

/**
 * The runtime requirements of the copied native files, read from the binaries themselves (ADR 0049):
 *
 * - every ELF file must be built for the target's machine and must not link another C library;
 * - every `.node` file must be an ELF Node.js addon;
 * - `nodeApi` is the highest Node-API version a Node-API addon declares through its exported
 *   `node_api_module_get_api_version_v1` (an addon without that export is version 8, as Node.js
 *   itself treats it); an ABI-specific addon records its `node_register_module_v<ABI>` as `nodeAbi`;
 * - `libcVersion` is the highest C library symbol version any file requires.
 *
 * Anything that cannot be established exactly fails closed.
 */
export function nativeRuntime(
  packages: readonly NativePackage[],
  target: NativeTarget,
): NativeRuntime & { readonly addons: number } {
  let nodeApi: number | undefined;
  let nodeAbi: string | undefined;
  let libc: number[] | undefined;
  let addons = 0;
  for (const nativePackage of packages) {
    for (const [relative, data] of nativePackage.files) {
      const fileId = `${nativePackage.name}@${nativePackage.version}/${relative}`;
      const addon = relative.toLowerCase().endsWith(".node");
      if (!isElf(data)) {
        if (addon) {
          throw new NativeError("MODULE_PACK_NATIVE_TARGET_MISMATCH", fileId);
        }
        continue;
      }
      let elf: ReturnType<typeof describeElf>;
      try {
        elf = describeElf(data);
      } catch (error) {
        if (error instanceof ElfFormatError) {
          throw new NativeError("MODULE_PACK_NATIVE_TARGET_MISMATCH", fileId);
        }
        throw error;
      }
      if (
        elf.machine !== target.elfMachine ||
        elf.needed.some((name) => target.foreignLibc.test(name))
      ) {
        throw new NativeError("MODULE_PACK_NATIVE_TARGET_MISMATCH", fileId);
      }
      for (const version of elf.requiredVersions) {
        if (!version.startsWith(target.libcVersionPrefix)) {
          continue;
        }
        const parts = version.slice(target.libcVersionPrefix.length).split(".").map(Number);
        if (parts.some((part) => !Number.isInteger(part))) {
          continue;
        }
        if (libc === undefined || compareVersions(parts, libc) > 0) {
          libc = parts;
        }
      }
      if (!addon) {
        continue;
      }
      addons += 1;
      if (elf.exported.has("napi_register_module_v1")) {
        const declared = elf.nodeApiModuleVersion;
        if (declared === "unrecognized" || (typeof declared === "number" && declared < 1)) {
          throw new NativeError("MODULE_PACK_NATIVE_RUNTIME_UNKNOWN", fileId);
        }
        const version = declared ?? 8;
        nodeApi = nodeApi === undefined ? version : Math.max(nodeApi, version);
      } else {
        const abi = [...elf.exported]
          .map((name) => /^node_register_module_v(\d+)$/u.exec(name)?.[1])
          .find((value) => value !== undefined);
        if (abi === undefined || (nodeAbi !== undefined && nodeAbi !== abi)) {
          throw new NativeError("MODULE_PACK_NATIVE_RUNTIME_UNKNOWN", fileId);
        }
        nodeAbi = abi;
      }
    }
  }
  return {
    addons,
    ...(nodeApi === undefined ? {} : { nodeApi }),
    ...(nodeAbi === undefined ? {} : { nodeAbi }),
    ...(libc === undefined ? {} : { libcVersion: `>=${[...libc, 0, 0].slice(0, 3).join(".")}` }),
  };
}