Ekklesia/packages/module-pack/src/supplements.ts

import { createHash } from "node:crypto";
import { lstat, readdir, readFile } from "node:fs/promises";
import path from "node:path";
import { fileURLToPath } from "node:url";

/**
 * Reviewed license supplements (ADR 0052).
 *
 * When a redistributed dependency's own package omits licence or notice material that
 * redistribution requires, the canonical packager may add material from a checked-in, reviewed
 * supplement. A supplement is data, not code: nothing here knows about any particular package.
 *
 * A supplement lives at `license-supplements/<package name>/<version>/` beside this package. It
 * holds `supplement.json` and the text files that file lists:
 *
 * ```json
 * {
 *   "supplementFormat": 1,
 *   "package": {
 *     "name": "…", "version": "…", "license": "<SPDX the package declares>",
 *     "contentDigest": "ekklesia-content-v1:sha256:<hex>"
 *   },
 *   "statement": "<what the supplement covers, printed with it>",
 *   "components": [
 *     {
 *       "name": "…", "version": "…", "license": "<as stated by its source>",
 *       "source": "<where the material came from>",
 *       "material": [{ "title": "LICENSE", "text": "texts/<sha256 of the text>.txt" }]
 *     }
 *   ]
 * }
 * ```
 *
 * Every text file is named by its own SHA-256. The loader checks every name against its bytes, and
 * refuses a file the supplement does not list, so a supplement can only change through review.
 *
 * Repository review of a supplement is project and code review of a build input. It is not legal
 * review, and carrying a supplement does not establish that any redistribution obligation is met.
 */

export const CONTENT_DIGEST_ALGORITHM = "ekklesia-content-v1";
export const SUPPLEMENT_FORMAT = 1;

/**
 * The content digest a supplement is matched against (ADR 0052), over exactly the files the
 * packager redistributes for a package, by path relative to the package root.
 *
 * 1. Each path is normalized: `/` separators, no leading `./`, no empty, `.`, or `..` segment.
 * 2. Paths are sorted by their UTF-8 bytes.
 * 3. For each file, in that order, this line is appended to a manifest:
 *    `<path> NUL <decimal byte length> NUL <lowercase hex SHA-256 of the bytes> LF`.
 * 4. The digest is `ekklesia-content-v1:sha256:` followed by the lowercase hex SHA-256 of the
 *    manifest's UTF-8 bytes.
 *
 * Nothing else is an input: not timestamps, ownership, permissions, the directory the package was
 * installed in, the package manager, or the order files were read or extracted in.
 */
export function contentDigest(files: ReadonlyMap<string, Uint8Array>): string {
  const entries = [...files].map(([relative, data]) => {
    const normalized = relative.split(path.sep).join("/").replace(/^\.\//u, "");
    if (
      normalized.split("/").some((segment) => segment === "" || segment === "." || segment === "..")
    ) {
      throw new Error("A redistributed path must be relative and normalized.");
    }
    return [Buffer.from(normalized, "utf8"), data] as const;
  });
  entries.sort(([left], [right]) => Buffer.compare(left, right));
  const manifest = createHash("sha256");
  for (const [name, data] of entries) {
    const digest = createHash("sha256").update(data).digest("hex");
    manifest.update(name);
    manifest.update(`\0${data.length}\0${digest}\n`);
  }
  return `${CONTENT_DIGEST_ALGORITHM}:sha256:${manifest.digest("hex")}`;
}

export interface SupplementMaterial {
  readonly title: string;
  /** The text, exactly as reviewed. */
  readonly text: string;
  /** SHA-256 of the text's bytes, so identical texts are recognized. */
  readonly sha256: string;
}

export interface SupplementComponent {
  readonly name: string;
  readonly version: string;
  readonly license: string;
  readonly source: string;
  readonly material: readonly SupplementMaterial[];
}

export interface LicenseSupplement {
  readonly name: string;
  readonly version: string;
  readonly license: string;
  readonly contentDigest: string;
  readonly statement: string;
  readonly components: readonly SupplementComponent[];
}

export class SupplementError extends Error {
  readonly supplement: string;

  constructor(supplement: string) {
    super("A license supplement is malformed or does not match its reviewed files.");
    this.name = "SupplementError";
    this.supplement = supplement;
  }
}

function isRecord(value: unknown): value is Record<string, unknown> {
  return typeof value === "object" && value !== null && !Array.isArray(value);
}

function nonEmptyString(value: unknown): value is string {
  return typeof value === "string" && value.trim() !== "";
}

function strictKeys(value: Record<string, unknown>, keys: readonly string[]): boolean {
  return Object.keys(value).every((key) => keys.includes(key)) && keys.every((key) => key in value);
}

export const DEFAULT_SUPPLEMENTS_DIRECTORY = path.resolve(
  path.dirname(fileURLToPath(import.meta.url)),
  "..",
  "license-supplements",
);

async function listFiles(root: string, relative = ""): Promise<string[]> {
  const directory = relative === "" ? root : path.join(root, relative);
  const found: string[] = [];
  for (const entry of await readdir(directory, { withFileTypes: true })) {
    const childRelative = relative === "" ? entry.name : `${relative}/${entry.name}`;
    const stats = await lstat(path.join(directory, entry.name));
    if (stats.isDirectory()) {
      found.push(...(await listFiles(root, childRelative)));
    } else if (stats.isFile()) {
      found.push(childRelative);
    } else {
      throw new SupplementError(childRelative);
    }
  }
  return found;
}

async function loadOne(directory: string, label: string): Promise<LicenseSupplement> {
  let raw: unknown;
  try {
    raw = JSON.parse(await readFile(path.join(directory, "supplement.json"), "utf8")) as unknown;
  } catch {
    throw new SupplementError(label);
  }
  if (
    !isRecord(raw) ||
    !strictKeys(raw, ["supplementFormat", "package", "statement", "components"]) ||
    raw.supplementFormat !== SUPPLEMENT_FORMAT ||
    !isRecord(raw.package) ||
    !strictKeys(raw.package, ["name", "version", "license", "contentDigest"]) ||
    !nonEmptyString(raw.statement) ||
    !Array.isArray(raw.components) ||
    raw.components.length === 0
  ) {
    throw new SupplementError(label);
  }
  const { contentDigest: digest, license, name, version } = raw.package;
  if (
    !nonEmptyString(name) ||
    !nonEmptyString(version) ||
    !nonEmptyString(license) ||
    typeof digest !== "string" ||
    !new RegExp(`^${CONTENT_DIGEST_ALGORITHM}:sha256:[0-9a-f]{64}$`, "u").test(digest) ||
    `${name}/${version}` !== label
  ) {
    throw new SupplementError(label);
  }

  const listed = new Set<string>();
  const components: SupplementComponent[] = [];
  for (const component of raw.components) {
    if (
      !isRecord(component) ||
      !strictKeys(component, ["name", "version", "license", "source", "material"]) ||
      !nonEmptyString(component.name) ||
      !nonEmptyString(component.version) ||
      !nonEmptyString(component.license) ||
      !nonEmptyString(component.source) ||
      !Array.isArray(component.material)
    ) {
      throw new SupplementError(label);
    }
    const material: SupplementMaterial[] = [];
    for (const item of component.material) {
      if (
        !isRecord(item) ||
        !strictKeys(item, ["title", "text"]) ||
        !nonEmptyString(item.title) ||
        typeof item.text !== "string" ||
        !/^texts\/[0-9a-f]{64}\.txt$/u.test(item.text)
      ) {
        throw new SupplementError(label);
      }
      const expected = item.text.slice("texts/".length, -".txt".length);
      let bytes: Buffer;
      try {
        bytes = await readFile(path.join(directory, ...item.text.split("/")));
      } catch {
        throw new SupplementError(label);
      }
      if (bytes.length === 0 || createHash("sha256").update(bytes).digest("hex") !== expected) {
        throw new SupplementError(label);
      }
      listed.add(item.text);
      material.push(
        Object.freeze({ sha256: expected, text: bytes.toString("utf8"), title: item.title }),
      );
    }
    components.push(
      Object.freeze({
        license: component.license,
        material: Object.freeze(material),
        name: component.name,
        source: component.source,
        version: component.version,
      }),
    );
  }
  // Only reviewed files: supplement.json, the texts it lists, and a human-readable provenance note.
  for (const file of await listFiles(directory)) {
    if (file !== "supplement.json" && file !== "PROVENANCE.md" && !listed.has(file)) {
      throw new SupplementError(label);
    }
  }
  return Object.freeze({
    components: Object.freeze(components),
    contentDigest: digest,
    license,
    name,
    statement: raw.statement,
    version,
  });
}

/**
 * Loads every supplement under `directory`, indexed by package name. A directory that does not exist
 * holds none. Any malformed supplement fails the whole load: a broken reviewed input is never
 * skipped.
 */
export async function loadLicenseSupplements(
  directory: string = DEFAULT_SUPPLEMENTS_DIRECTORY,
): Promise<ReadonlyMap<string, readonly LicenseSupplement[]>> {
  const index = new Map<string, LicenseSupplement[]>();
  let files: string[];
  try {
    files = await listFiles(directory);
  } catch (error) {
    if (isRecord(error) && error.code === "ENOENT") {
      return index;
    }
    throw error;
  }
  const roots = files
    .filter((file) => file.endsWith("/supplement.json"))
    .map((file) => file.slice(0, -"/supplement.json".length))
    .sort();
  for (const label of roots) {
    const supplement = await loadOne(path.join(directory, ...label.split("/")), label);
    const list = index.get(supplement.name) ?? [];
    list.push(supplement);
    index.set(supplement.name, list);
  }
  // Every file must belong to some supplement directory.
  for (const file of files) {
    if (!roots.some((root) => file.startsWith(`${root}/`))) {
      throw new SupplementError(file);
    }
  }
  return index;
}