Ekklesia/scripts/package-release.ts

import { execFile } from "node:child_process";
import { createHash } from "node:crypto";
import { mkdir, mkdtemp, readdir, readFile, rename, rm, stat, writeFile } from "node:fs/promises";
import { homedir, tmpdir } from "node:os";
import path from "node:path";
import process from "node:process";
import { fileURLToPath, pathToFileURL } from "node:url";
import { promisify } from "node:util";

/**
 * The public package release train (ADR 0053): the canonical way to produce the exact
 * `@ekklesia/*` package tarballs of one platform release. It publishes nothing.
 *
 *   node scripts/package-release.ts --out <directory> [--allow-modified]
 *
 * From a clean source tree, it removes the public packages' build output, rebuilds them without the
 * build cache, packs each with `pnpm pack`, inspects every tarball, and only when everything passed
 * writes the release directory `<directory>/ekklesia-packages-<version>/`:
 *
 *   ekklesia-<name>-<version>.tgz        one per public package, exactly the expected set
 *   SHA256SUMS                          `sha256sum -c` compatible
 *   ekklesia-package-train.json         the release manifest (`packageTrainManifest`)
 *
 * Those files are what a future registry publication step publishes, unchanged, after checking them
 * against the manifest. `--allow-modified` builds from a source tree with uncommitted changes, for
 * local development only; the manifest then says so, and such a release must never be published.
 */

const execFileAsync = promisify(execFile);
const repositoryRoot = path.resolve(path.dirname(fileURLToPath(import.meta.url)), "..");
/** pnpm hooks applied to `pnpm pack` only (see the file itself). */
const PACK_HOOKS = path.join(repositoryRoot, "scripts", "lib", "pack-hooks.cjs");

export type PublicPackageRole = "author-api" | "tooling" | "pre-1.0" | "implementation";

export interface PublicPackage {
  readonly name: string;
  /** Relative to the repository root. */
  readonly directory: string;
  readonly role: PublicPackageRole;
}

/** ADR 0053's public set. Every other workspace package is private and never published. */
export const PUBLIC_PACKAGES: readonly PublicPackage[] = Object.freeze([
  { directory: "packages/module-sdk", name: "@ekklesia/module-sdk", role: "author-api" },
  { directory: "packages/module-database", name: "@ekklesia/module-database", role: "author-api" },
  { directory: "packages/module-ui-sdk", name: "@ekklesia/module-ui-sdk", role: "author-api" },
  { directory: "packages/design-tokens", name: "@ekklesia/design-tokens", role: "author-api" },
  { directory: "packages/module-ui-build", name: "@ekklesia/module-ui-build", role: "tooling" },
  { directory: "packages/module-pack", name: "@ekklesia/module-pack", role: "tooling" },
  {
    directory: "packages/typescript-config",
    name: "@ekklesia/typescript-config",
    role: "tooling",
  },
  { directory: "packages/ui", name: "@ekklesia/ui", role: "pre-1.0" },
  // A dependency of the packages above, never a supported direct module-author dependency.
  { directory: "packages/contracts", name: "@ekklesia/contracts", role: "implementation" },
]);

/** `ekklesia.distribution` in a public package's manifest: the identity the canonical packager
 * recognizes an installed Ekklesia package by (`packages/module-pack/src/packages.ts`). */
export const PUBLIC_PACKAGE_DISTRIBUTION = "module-development";

/** The release manifest's file name, and its format version. */
export const PACKAGE_TRAIN_MANIFEST = "ekklesia-package-train.json";
export const PACKAGE_TRAIN_FORMAT = 1;

/** `@ekklesia/module-sdk` → `ekklesia-module-sdk-<version>.tgz`, the name `pnpm pack` writes. */
export function tarballName(name: string, version: string): string {
  return `${name.replace(/^@/u, "").replace("/", "-")}-${version}.tgz`;
}

/** The directory one release's artifacts are written to, inside `--out`. */
export function releaseDirectoryName(version: string): string {
  return `ekklesia-packages-${version}`;
}

/** Scripts a package manager runs on a consumer's machine when it installs, packs, or publishes. */
const LIFECYCLE_SCRIPTS = Object.freeze([
  "preinstall",
  "install",
  "postinstall",
  "prepare",
  "prepack",
  "postpack",
  "prepublish",
  "prepublishOnly",
  "publish",
  "postpublish",
]);

const DEPENDENCY_FIELDS = Object.freeze([
  "dependencies",
  "peerDependencies",
  "optionalDependencies",
  "devDependencies",
]);

function isRecord(value: unknown): value is Readonly<Record<string, unknown>> {
  return typeof value === "object" && value !== null && !Array.isArray(value);
}

/**
 * What is wrong with a packed `package.json`, or nothing. `source` is the package's own source
 * manifest, when there is one to compare with: a packed dependency map must keep its key order.
 */
export function packedManifestProblems(
  manifest: Readonly<Record<string, unknown>>,
  expected: { readonly name: string; readonly version: string },
  source?: Readonly<Record<string, unknown>>,
): string[] {
  const problems: string[] = [];
  if (manifest.name !== expected.name) problems.push(`is named ${String(manifest.name)}`);
  if (manifest.version !== expected.version) {
    problems.push(
      `has version ${String(manifest.version)}, not the coordinated ${expected.version}`,
    );
  }
  if (manifest.private !== undefined) problems.push("declares private");
  if (manifest.license !== "AGPL-3.0-or-later") problems.push("does not declare its licence");
  if (JSON.stringify(manifest).includes("workspace:")) problems.push("keeps a workspace: range");
  // A registry belongs to the publication step's configuration, never to an artifact.
  if (manifest.publishConfig !== undefined) problems.push("names a publishConfig");
  if (manifest.devDependencies !== undefined) problems.push("keeps devDependencies");
  const ekklesia = manifest.ekklesia;
  if (!isRecord(ekklesia) || ekklesia.distribution !== PUBLIC_PACKAGE_DISTRIBUTION) {
    problems.push("lacks the module-development distribution marker");
  }
  const scripts = isRecord(manifest.scripts) ? manifest.scripts : {};
  for (const script of LIFECYCLE_SCRIPTS) {
    if (Object.hasOwn(scripts, script)) problems.push(`runs a ${script} script`);
  }
  const publicNames = new Set(PUBLIC_PACKAGES.map((entry) => entry.name));
  for (const field of DEPENDENCY_FIELDS) {
    const dependencies = manifest[field];
    if (!isRecord(dependencies)) continue;
    for (const [name, range] of Object.entries(dependencies)) {
      if (!name.startsWith("@ekklesia/")) continue;
      if (!publicNames.has(name)) problems.push(`${field} names the private ${name}`);
      else if (range !== expected.version) {
        problems.push(`${field} names ${name}@${String(range)}, not ${expected.version}`);
      }
    }
    const sourceOrder = source?.[field];
    if (isRecord(sourceOrder)) {
      const order = Object.keys(sourceOrder).filter((key) => Object.hasOwn(dependencies, key));
      if (JSON.stringify(order) !== JSON.stringify(Object.keys(dependencies))) {
        problems.push(`orders ${field} differently from its source`);
      }
    }
  }
  return problems;
}

const BUILT_FILE = /^dist\/(.+?)(?:\.d\.ts|\.js)(?:\.map)?$/u;

/**
 * What is wrong with a tarball's file list, or nothing. Every file is `package.json`, `LICENSE`, or
 * under one of the manifest's `files` entries; nothing is a source, test, or configuration file; and
 * every built file has the source it was compiled from, so stale output from a deleted source fails.
 */
export function tarballFileProblems(
  files: readonly string[],
  manifest: Readonly<Record<string, unknown>>,
  hasSource: (relativeWithoutExtension: string) => boolean,
): string[] {
  const problems: string[] = [];
  const declared = Array.isArray(manifest.files) ? manifest.files.map(String) : [];
  if (!files.includes("LICENSE")) problems.push("has no LICENSE");
  if (!files.includes("package.json")) problems.push("has no package.json");
  for (const file of files) {
    if (file === "package.json" || file === "LICENSE") continue;
    if (!declared.some((entry) => file === entry || file.startsWith(`${entry}/`))) {
      problems.push(`ships ${file}, which its files field does not name`);
      continue;
    }
    if (
      /(?:^|\/)(?:src|tests?|node_modules)\//u.test(file) ||
      /\.test\.[cm]?[jt]sx?$/u.test(file)
    ) {
      problems.push(`ships ${file}`);
      continue;
    }
    const built = BUILT_FILE.exec(file);
    if (file.startsWith("dist/") && (built === null || !hasSource(built[1] ?? ""))) {
      problems.push(`ships ${file}, which no source produces`);
    }
  }
  return problems;
}

export interface WorkspacePackage {
  readonly name: string;
  /** Relative to the repository root. */
  readonly directory: string;
  readonly manifest: Readonly<Record<string, unknown>>;
}

/** Every package `pnpm-workspace.yaml` names, read without a YAML dependency: the file lists
 * literal directories and `<directory>/*` patterns only. */
export async function workspacePackages(root = repositoryRoot): Promise<WorkspacePackage[]> {
  const workspace = await readFile(path.join(root, "pnpm-workspace.yaml"), "utf8");
  const block = /^packages:\n((?:[ \t]+-[^\n]*\n)+)/mu.exec(workspace)?.[1] ?? "";
  const patterns = [...block.matchAll(/-\s*["']?([^"'\n]+?)["']?\s*$/gmu)].map(
    (match) => match[1] ?? "",
  );
  const packages: WorkspacePackage[] = [];
  for (const pattern of patterns) {
    const directories = pattern.endsWith("/*")
      ? (
          await readdir(path.join(root, pattern.slice(0, -2)), { withFileTypes: true }).catch(
            () => [],
          )
        )
          .filter((entry) => entry.isDirectory())
          .map((entry) => `${pattern.slice(0, -2)}/${entry.name}`)
      : [pattern];
    for (const directory of directories) {
      const manifest = await readJson(path.join(root, directory, "package.json")).catch(
        () => undefined,
      );
      if (manifest && typeof manifest.name === "string") {
        packages.push({ directory, manifest, name: manifest.name });
      }
    }
  }
  return packages.sort((left, right) => (left.name < right.name ? -1 : 1));
}

/**
 * What is wrong with the workspace's release set, or nothing: every public package exists where the
 * list says, carries the marker and the platform version, and is not private; every other package is
 * private and unmarked, so nothing can enter the train unnoticed.
 */
export function releaseSetProblems(
  packages: readonly WorkspacePackage[],
  platformVersion: string,
): string[] {
  const problems: string[] = [];
  const byName = new Map(packages.map((entry) => [entry.name, entry]));
  const publicNames = new Set(PUBLIC_PACKAGES.map((entry) => entry.name));
  for (const expected of PUBLIC_PACKAGES) {
    const found = byName.get(expected.name);
    if (found === undefined || found.directory !== expected.directory) {
      problems.push(`${expected.name} is missing from ${expected.directory}.`);
      continue;
    }
    if (found.manifest.version !== platformVersion) {
      problems.push(
        `${expected.name} is version ${String(found.manifest.version)}, not the platform's ${platformVersion}.`,
      );
    }
    if (found.manifest.private !== undefined) problems.push(`${expected.name} is private.`);
  }
  for (const entry of packages) {
    if (publicNames.has(entry.name)) continue;
    const ekklesia = entry.manifest.ekklesia;
    if (isRecord(ekklesia) && ekklesia.distribution === PUBLIC_PACKAGE_DISTRIBUTION) {
      problems.push(`${entry.name} carries the distribution marker but is not in the release set.`);
    }
    if (entry.manifest.private !== true) {
      problems.push(`${entry.name} is publishable but not in the release set.`);
    }
  }
  return problems;
}

export interface TrainPackage {
  readonly name: string;
  readonly version: string;
  readonly role: PublicPackageRole;
  readonly file: string;
  readonly size: number;
  /** Hexadecimal SHA-256 of the tarball. */
  readonly sha256: string;
  /** The npm `dist.integrity` of the tarball: `sha512-<base64>`. */
  readonly integrity: string;
}

/** The release manifest: what one train is, by content. Provider-neutral, and no registry catalog. */
export interface PackageTrainManifest {
  readonly packageTrain: typeof PACKAGE_TRAIN_FORMAT;
  /** The platform release version, and every package's version. */
  readonly version: string;
  readonly source: {
    readonly revision: string;
    /** True only for a local `--allow-modified` build, which must never be published. */
    readonly modified: boolean;
  };
  readonly packages: readonly TrainPackage[];
}

/** What is wrong with a release directory's contents against its manifest, or nothing. */
export function releaseDirectoryProblems(
  manifest: PackageTrainManifest,
  files: ReadonlyMap<string, Uint8Array>,
): string[] {
  const problems: string[] = [];
  if (manifest.packageTrain !== PACKAGE_TRAIN_FORMAT) problems.push("unknown manifest format.");
  const expected = new Map(
    PUBLIC_PACKAGES.map((entry) => [tarballName(entry.name, manifest.version), entry.name]),
  );
  const listed = new Map(manifest.packages.map((entry) => [entry.file, entry]));
  for (const [file, name] of expected) {
    if (!listed.has(file)) problems.push(`the manifest does not list ${name}.`);
    if (!files.has(file)) problems.push(`${file} is missing.`);
  }
  for (const file of files.keys()) {
    if (!expected.has(file) && file !== PACKAGE_TRAIN_MANIFEST && file !== "SHA256SUMS") {
      problems.push(`${file} is not part of the release.`);
    }
  }
  for (const entry of manifest.packages) {
    if (!expected.has(entry.file) || expected.get(entry.file) !== entry.name) {
      problems.push(`the manifest lists ${entry.name} as ${entry.file}, which is not expected.`);
      continue;
    }
    if (entry.version !== manifest.version) {
      problems.push(`the manifest lists ${entry.name} at ${entry.version}.`);
    }
    const data = files.get(entry.file);
    if (data === undefined) continue;
    const digests = artifactDigests(data);
    if (
      data.length !== entry.size ||
      digests.sha256 !== entry.sha256 ||
      digests.integrity !== entry.integrity
    ) {
      problems.push(`${entry.file} does not match its size, SHA-256, or integrity.`);
    }
  }
  return problems;
}

export function artifactDigests(data: Uint8Array): { sha256: string; integrity: string } {
  return {
    integrity: `sha512-${createHash("sha512").update(data).digest("base64")}`,
    sha256: createHash("sha256").update(data).digest("hex"),
  };
}

/** The manifest's exact bytes: sorted packages, two-space JSON, one trailing newline. */
export function serializeManifest(manifest: PackageTrainManifest): string {
  const packages = [...manifest.packages]
    .sort((left, right) => (left.name < right.name ? -1 : 1))
    .map((entry) => ({
      file: entry.file,
      integrity: entry.integrity,
      name: entry.name,
      role: entry.role,
      sha256: entry.sha256,
      size: entry.size,
      version: entry.version,
    }));
  return `${JSON.stringify(
    {
      packageTrain: manifest.packageTrain,
      version: manifest.version,
      source: { modified: manifest.source.modified, revision: manifest.source.revision },
      packages,
    },
    null,
    2,
  )}\n`;
}

async function readJson(file: string): Promise<Record<string, unknown>> {
  return JSON.parse(await readFile(file, "utf8")) as Record<string, unknown>;
}

async function filesUnder(directory: string): Promise<string[]> {
  const entries = await readdir(directory, { recursive: true, withFileTypes: true });
  return entries
    .filter((entry) => entry.isFile())
    .map((entry) =>
      path.relative(directory, path.join(entry.parentPath, entry.name)).split(path.sep).join("/"),
    )
    .sort();
}

async function sourceExists(packageDirectory: string, relative: string): Promise<boolean> {
  for (const extension of [".ts", ".tsx"]) {
    try {
      await stat(path.join(packageDirectory, "src", `${relative}${extension}`));
      return true;
    } catch {}
  }
  return false;
}

async function run(command: string, arguments_: readonly string[], cwd: string): Promise<string> {
  const { stdout } = await execFileAsync(command, [...arguments_], {
    cwd,
    maxBuffer: 64 * 1024 * 1024,
  });
  return stdout;
}

async function inspectTarball(
  entry: PublicPackage,
  version: string,
  tarball: string,
  scratch: string,
): Promise<string[]> {
  const directory = path.join(repositoryRoot, entry.directory);
  const extracted = path.join(scratch, entry.name.replace("/", "+"));
  await mkdir(extracted, { recursive: true });
  await run("tar", ["-xzf", tarball, "-C", extracted], repositoryRoot);
  const root = path.join(extracted, "package");
  const files = await filesUnder(root);
  const manifest = await readJson(path.join(root, "package.json"));
  const source = await readJson(path.join(directory, "package.json"));

  const sources = new Map<string, boolean>();
  for (const file of files) {
    const built = BUILT_FILE.exec(file)?.[1];
    if (built !== undefined && !sources.has(built)) {
      sources.set(built, await sourceExists(directory, built));
    }
  }
  const problems = [
    ...packedManifestProblems(manifest, { name: entry.name, version }, source),
    ...tarballFileProblems(files, manifest, (relative) => sources.get(relative) === true),
  ];
  // Nothing the package says may name the machine or checkout it was packed on.
  for (const file of files) {
    const text = (await readFile(path.join(root, file))).toString("latin1");
    for (const local of [repositoryRoot, homedir(), tmpdir()]) {
      if (text.includes(local)) problems.push(`${file} names a local path`);
    }
  }
  return [...new Set(problems)].map((problem) => `${entry.name} ${problem}.`);
}

export class PackageReleaseError extends Error {
  readonly problems: readonly string[];

  constructor(problems: readonly string[]) {
    super("The package release failed its checks.");
    this.name = "PackageReleaseError";
    this.problems = problems;
  }
}

export interface PackageReleaseOptions {
  /** The parent of the release directory, which must not exist yet. */
  readonly outDirectory: string;
  readonly allowModified?: boolean;
}

/**
 * Builds, packs, and inspects the public package train, and writes the release directory only when
 * every check passed. Returns its path and manifest; throws `PackageReleaseError` with every problem
 * found otherwise.
 */
export async function buildPackageRelease(
  options: PackageReleaseOptions,
): Promise<{ readonly directory: string; readonly manifest: PackageTrainManifest }> {
  const version = String((await readJson(path.join(repositoryRoot, "package.json"))).version);
  const setProblems = releaseSetProblems(await workspacePackages(), version);
  if (setProblems.length > 0) throw new PackageReleaseError(setProblems);

  const revision = (await run("git", ["rev-parse", "HEAD"], repositoryRoot)).trim();
  const modified =
    (
      await run("git", ["status", "--porcelain", "--untracked-files=all"], repositoryRoot)
    ).trim() !== "";
  if (modified && options.allowModified !== true) {
    throw new PackageReleaseError([
      "The source tree has uncommitted changes. A release is built from a clean commit.",
    ]);
  }

  const release = path.join(path.resolve(options.outDirectory), releaseDirectoryName(version));
  try {
    await stat(release);
    throw new PackageReleaseError([`${release} already exists. A release is written once.`]);
  } catch (error) {
    if (error instanceof PackageReleaseError) throw error;
  }

  // Stale output from a deleted source would otherwise ship: the build only ever adds files, and
  // Turbo's cache would restore whatever an earlier build left behind.
  for (const entry of PUBLIC_PACKAGES) {
    await rm(path.join(repositoryRoot, entry.directory, "dist"), { force: true, recursive: true });
  }
  await run(
    "pnpm",
    [
      "exec",
      "turbo",
      "run",
      "build",
      "--force",
      "--output-logs=errors-only",
      ...PUBLIC_PACKAGES.map((entry) => `--filter=${entry.name}`),
    ],
    repositoryRoot,
  );

  const scratch = await mkdtemp(path.join(tmpdir(), "ekklesia-package-release-"));
  try {
    const staging = path.join(scratch, "release");
    await mkdir(staging);
    const problems: string[] = [];
    const packages: TrainPackage[] = [];
    for (const entry of PUBLIC_PACKAGES) {
      const report = JSON.parse(
        await run(
          "pnpm",
          [`--config.pnpmfile=${PACK_HOOKS}`, "pack", "--json", "--pack-destination", staging],
          path.join(repositoryRoot, entry.directory),
        ),
      ) as { filename: string };
      const file = path.basename(report.filename);
      if (file !== tarballName(entry.name, version)) {
        problems.push(`${entry.name} packed as ${file}.`);
        continue;
      }
      problems.push(...(await inspectTarball(entry, version, report.filename, scratch)));
      const data = await readFile(report.filename);
      packages.push({
        file,
        name: entry.name,
        role: entry.role,
        size: data.length,
        version,
        ...artifactDigests(data),
      });
    }
    const manifest: PackageTrainManifest = {
      packageTrain: PACKAGE_TRAIN_FORMAT,
      packages,
      source: { modified, revision },
      version,
    };
    const contents = new Map<string, Uint8Array>();
    for (const file of await readdir(staging)) {
      contents.set(file, await readFile(path.join(staging, file)));
    }
    problems.push(...releaseDirectoryProblems(manifest, contents));
    if (problems.length > 0) throw new PackageReleaseError(problems);

    const sorted = [...manifest.packages].sort((left, right) => (left.file < right.file ? -1 : 1));
    await writeFile(
      path.join(staging, "SHA256SUMS"),
      sorted.map((entry) => `${entry.sha256}  ${entry.file}\n`).join(""),
    );
    await writeFile(path.join(staging, PACKAGE_TRAIN_MANIFEST), serializeManifest(manifest));
    await mkdir(path.dirname(release), { recursive: true });
    // Into place in one step: the release directory either holds a whole, checked release or does
    // not exist. A rename across file systems is not atomic, so copy then, never half a release.
    try {
      await rename(staging, release);
    } catch {
      await run("cp", ["-R", staging, release], repositoryRoot);
    }
    return { directory: release, manifest };
  } finally {
    await rm(scratch, { force: true, recursive: true });
  }
}

async function main(argv: readonly string[]): Promise<number> {
  const allowModified = argv.includes("--allow-modified");
  const rest = argv.filter((argument) => argument !== "--allow-modified");
  if (rest.length !== 2 || rest[0] !== "--out" || !rest[1]) {
    process.stderr.write(
      "Usage: node scripts/package-release.ts --out <directory> [--allow-modified]\n",
    );
    return 64;
  }
  try {
    const { directory, manifest } = await buildPackageRelease({
      allowModified,
      outDirectory: rest[1],
    });
    for (const entry of manifest.packages) {
      process.stdout.write(`${entry.file}: ${entry.size} bytes, sha256 ${entry.sha256}\n`);
    }
    process.stdout.write(
      `Built and inspected the ${manifest.version} package train (${manifest.packages.length} packages${
        manifest.source.modified ? ", from a modified source tree: never publish it" : ""
      }) in ${directory}\n`,
    );
    return 0;
  } catch (error) {
    if (error instanceof PackageReleaseError) {
      for (const problem of error.problems) process.stderr.write(`${problem}\n`);
      return 1;
    }
    process.stderr.write("Building the package release failed while building or packing.\n");
    return 1;
  }
}

if (process.argv[1] && import.meta.url === pathToFileURL(process.argv[1]).href) {
  process.exitCode = await main(process.argv.slice(2));
}