import { execFile } from "node:child_process"; import { createHash } from "node:crypto"; import { mkdir, mkdtemp, readdir, readFile, rename, rm, stat, writeFile } from "node:fs/promises"; import { homedir, tmpdir } from "node:os"; import path from "node:path"; import process from "node:process"; import { fileURLToPath, pathToFileURL } from "node:url"; import { promisify } from "node:util"; /** * The public package release train (ADR 0053): the canonical way to produce the exact * `@ekklesia/*` package tarballs of one platform release. It publishes nothing. * * node scripts/package-release.ts --out [--allow-modified] * * From a clean source tree, it removes the public packages' build output, rebuilds them without the * build cache, packs each with `pnpm pack`, inspects every tarball, and only when everything passed * writes the release directory `/ekklesia-packages-/`: * * ekklesia--.tgz one per public package, exactly the expected set * SHA256SUMS `sha256sum -c` compatible * ekklesia-package-train.json the release manifest (`packageTrainManifest`) * * Those files are what a future registry publication step publishes, unchanged, after checking them * against the manifest. `--allow-modified` builds from a source tree with uncommitted changes, for * local development only; the manifest then says so, and such a release must never be published. */ const execFileAsync = promisify(execFile); const repositoryRoot = path.resolve(path.dirname(fileURLToPath(import.meta.url)), ".."); /** pnpm hooks applied to `pnpm pack` only (see the file itself). */ const PACK_HOOKS = path.join(repositoryRoot, "scripts", "lib", "pack-hooks.cjs"); export type PublicPackageRole = "author-api" | "tooling" | "pre-1.0" | "implementation"; export interface PublicPackage { readonly name: string; /** Relative to the repository root. */ readonly directory: string; readonly role: PublicPackageRole; } /** ADR 0053's public set. Every other workspace package is private and never published. */ export const PUBLIC_PACKAGES: readonly PublicPackage[] = Object.freeze([ { directory: "packages/module-sdk", name: "@ekklesia/module-sdk", role: "author-api" }, { directory: "packages/module-database", name: "@ekklesia/module-database", role: "author-api" }, { directory: "packages/module-ui-sdk", name: "@ekklesia/module-ui-sdk", role: "author-api" }, { directory: "packages/design-tokens", name: "@ekklesia/design-tokens", role: "author-api" }, { directory: "packages/module-ui-build", name: "@ekklesia/module-ui-build", role: "tooling" }, { directory: "packages/module-pack", name: "@ekklesia/module-pack", role: "tooling" }, { directory: "packages/typescript-config", name: "@ekklesia/typescript-config", role: "tooling", }, { directory: "packages/ui", name: "@ekklesia/ui", role: "pre-1.0" }, // A dependency of the packages above, never a supported direct module-author dependency. { directory: "packages/contracts", name: "@ekklesia/contracts", role: "implementation" }, ]); /** `ekklesia.distribution` in a public package's manifest: the identity the canonical packager * recognizes an installed Ekklesia package by (`packages/module-pack/src/packages.ts`). */ export const PUBLIC_PACKAGE_DISTRIBUTION = "module-development"; /** The release manifest's file name, and its format version. */ export const PACKAGE_TRAIN_MANIFEST = "ekklesia-package-train.json"; export const PACKAGE_TRAIN_FORMAT = 1; /** `@ekklesia/module-sdk` → `ekklesia-module-sdk-.tgz`, the name `pnpm pack` writes. */ export function tarballName(name: string, version: string): string { return `${name.replace(/^@/u, "").replace("/", "-")}-${version}.tgz`; } /** The directory one release's artifacts are written to, inside `--out`. */ export function releaseDirectoryName(version: string): string { return `ekklesia-packages-${version}`; } /** Scripts a package manager runs on a consumer's machine when it installs, packs, or publishes. */ const LIFECYCLE_SCRIPTS = Object.freeze([ "preinstall", "install", "postinstall", "prepare", "prepack", "postpack", "prepublish", "prepublishOnly", "publish", "postpublish", ]); const DEPENDENCY_FIELDS = Object.freeze([ "dependencies", "peerDependencies", "optionalDependencies", "devDependencies", ]); function isRecord(value: unknown): value is Readonly> { return typeof value === "object" && value !== null && !Array.isArray(value); } /** * What is wrong with a packed `package.json`, or nothing. `source` is the package's own source * manifest, when there is one to compare with: a packed dependency map must keep its key order. */ export function packedManifestProblems( manifest: Readonly>, expected: { readonly name: string; readonly version: string }, source?: Readonly>, ): string[] { const problems: string[] = []; if (manifest.name !== expected.name) problems.push(`is named ${String(manifest.name)}`); if (manifest.version !== expected.version) { problems.push( `has version ${String(manifest.version)}, not the coordinated ${expected.version}`, ); } if (manifest.private !== undefined) problems.push("declares private"); if (manifest.license !== "AGPL-3.0-or-later") problems.push("does not declare its licence"); if (JSON.stringify(manifest).includes("workspace:")) problems.push("keeps a workspace: range"); // A registry belongs to the publication step's configuration, never to an artifact. if (manifest.publishConfig !== undefined) problems.push("names a publishConfig"); if (manifest.devDependencies !== undefined) problems.push("keeps devDependencies"); const ekklesia = manifest.ekklesia; if (!isRecord(ekklesia) || ekklesia.distribution !== PUBLIC_PACKAGE_DISTRIBUTION) { problems.push("lacks the module-development distribution marker"); } const scripts = isRecord(manifest.scripts) ? manifest.scripts : {}; for (const script of LIFECYCLE_SCRIPTS) { if (Object.hasOwn(scripts, script)) problems.push(`runs a ${script} script`); } const publicNames = new Set(PUBLIC_PACKAGES.map((entry) => entry.name)); for (const field of DEPENDENCY_FIELDS) { const dependencies = manifest[field]; if (!isRecord(dependencies)) continue; for (const [name, range] of Object.entries(dependencies)) { if (!name.startsWith("@ekklesia/")) continue; if (!publicNames.has(name)) problems.push(`${field} names the private ${name}`); else if (range !== expected.version) { problems.push(`${field} names ${name}@${String(range)}, not ${expected.version}`); } } const sourceOrder = source?.[field]; if (isRecord(sourceOrder)) { const order = Object.keys(sourceOrder).filter((key) => Object.hasOwn(dependencies, key)); if (JSON.stringify(order) !== JSON.stringify(Object.keys(dependencies))) { problems.push(`orders ${field} differently from its source`); } } } return problems; } const BUILT_FILE = /^dist\/(.+?)(?:\.d\.ts|\.js)(?:\.map)?$/u; /** * What is wrong with a tarball's file list, or nothing. Every file is `package.json`, `LICENSE`, or * under one of the manifest's `files` entries; nothing is a source, test, or configuration file; and * every built file has the source it was compiled from, so stale output from a deleted source fails. */ export function tarballFileProblems( files: readonly string[], manifest: Readonly>, hasSource: (relativeWithoutExtension: string) => boolean, ): string[] { const problems: string[] = []; const declared = Array.isArray(manifest.files) ? manifest.files.map(String) : []; if (!files.includes("LICENSE")) problems.push("has no LICENSE"); if (!files.includes("package.json")) problems.push("has no package.json"); for (const file of files) { if (file === "package.json" || file === "LICENSE") continue; if (!declared.some((entry) => file === entry || file.startsWith(`${entry}/`))) { problems.push(`ships ${file}, which its files field does not name`); continue; } if ( /(?:^|\/)(?:src|tests?|node_modules)\//u.test(file) || /\.test\.[cm]?[jt]sx?$/u.test(file) ) { problems.push(`ships ${file}`); continue; } const built = BUILT_FILE.exec(file); if (file.startsWith("dist/") && (built === null || !hasSource(built[1] ?? ""))) { problems.push(`ships ${file}, which no source produces`); } } return problems; } export interface WorkspacePackage { readonly name: string; /** Relative to the repository root. */ readonly directory: string; readonly manifest: Readonly>; } /** Every package `pnpm-workspace.yaml` names, read without a YAML dependency: the file lists * literal directories and `/*` patterns only. */ export async function workspacePackages(root = repositoryRoot): Promise { const workspace = await readFile(path.join(root, "pnpm-workspace.yaml"), "utf8"); const block = /^packages:\n((?:[ \t]+-[^\n]*\n)+)/mu.exec(workspace)?.[1] ?? ""; const patterns = [...block.matchAll(/-\s*["']?([^"'\n]+?)["']?\s*$/gmu)].map( (match) => match[1] ?? "", ); const packages: WorkspacePackage[] = []; for (const pattern of patterns) { const directories = pattern.endsWith("/*") ? ( await readdir(path.join(root, pattern.slice(0, -2)), { withFileTypes: true }).catch( () => [], ) ) .filter((entry) => entry.isDirectory()) .map((entry) => `${pattern.slice(0, -2)}/${entry.name}`) : [pattern]; for (const directory of directories) { const manifest = await readJson(path.join(root, directory, "package.json")).catch( () => undefined, ); if (manifest && typeof manifest.name === "string") { packages.push({ directory, manifest, name: manifest.name }); } } } return packages.sort((left, right) => (left.name < right.name ? -1 : 1)); } /** * What is wrong with the workspace's release set, or nothing: every public package exists where the * list says, carries the marker and the platform version, and is not private; every other package is * private and unmarked, so nothing can enter the train unnoticed. */ export function releaseSetProblems( packages: readonly WorkspacePackage[], platformVersion: string, ): string[] { const problems: string[] = []; const byName = new Map(packages.map((entry) => [entry.name, entry])); const publicNames = new Set(PUBLIC_PACKAGES.map((entry) => entry.name)); for (const expected of PUBLIC_PACKAGES) { const found = byName.get(expected.name); if (found === undefined || found.directory !== expected.directory) { problems.push(`${expected.name} is missing from ${expected.directory}.`); continue; } if (found.manifest.version !== platformVersion) { problems.push( `${expected.name} is version ${String(found.manifest.version)}, not the platform's ${platformVersion}.`, ); } if (found.manifest.private !== undefined) problems.push(`${expected.name} is private.`); } for (const entry of packages) { if (publicNames.has(entry.name)) continue; const ekklesia = entry.manifest.ekklesia; if (isRecord(ekklesia) && ekklesia.distribution === PUBLIC_PACKAGE_DISTRIBUTION) { problems.push(`${entry.name} carries the distribution marker but is not in the release set.`); } if (entry.manifest.private !== true) { problems.push(`${entry.name} is publishable but not in the release set.`); } } return problems; } export interface TrainPackage { readonly name: string; readonly version: string; readonly role: PublicPackageRole; readonly file: string; readonly size: number; /** Hexadecimal SHA-256 of the tarball. */ readonly sha256: string; /** The npm `dist.integrity` of the tarball: `sha512-`. */ readonly integrity: string; } /** The release manifest: what one train is, by content. Provider-neutral, and no registry catalog. */ export interface PackageTrainManifest { readonly packageTrain: typeof PACKAGE_TRAIN_FORMAT; /** The platform release version, and every package's version. */ readonly version: string; readonly source: { readonly revision: string; /** True only for a local `--allow-modified` build, which must never be published. */ readonly modified: boolean; }; readonly packages: readonly TrainPackage[]; } /** What is wrong with a release directory's contents against its manifest, or nothing. */ export function releaseDirectoryProblems( manifest: PackageTrainManifest, files: ReadonlyMap, ): string[] { const problems: string[] = []; if (manifest.packageTrain !== PACKAGE_TRAIN_FORMAT) problems.push("unknown manifest format."); const expected = new Map( PUBLIC_PACKAGES.map((entry) => [tarballName(entry.name, manifest.version), entry.name]), ); const listed = new Map(manifest.packages.map((entry) => [entry.file, entry])); for (const [file, name] of expected) { if (!listed.has(file)) problems.push(`the manifest does not list ${name}.`); if (!files.has(file)) problems.push(`${file} is missing.`); } for (const file of files.keys()) { if (!expected.has(file) && file !== PACKAGE_TRAIN_MANIFEST && file !== "SHA256SUMS") { problems.push(`${file} is not part of the release.`); } } for (const entry of manifest.packages) { if (!expected.has(entry.file) || expected.get(entry.file) !== entry.name) { problems.push(`the manifest lists ${entry.name} as ${entry.file}, which is not expected.`); continue; } if (entry.version !== manifest.version) { problems.push(`the manifest lists ${entry.name} at ${entry.version}.`); } const data = files.get(entry.file); if (data === undefined) continue; const digests = artifactDigests(data); if ( data.length !== entry.size || digests.sha256 !== entry.sha256 || digests.integrity !== entry.integrity ) { problems.push(`${entry.file} does not match its size, SHA-256, or integrity.`); } } return problems; } export function artifactDigests(data: Uint8Array): { sha256: string; integrity: string } { return { integrity: `sha512-${createHash("sha512").update(data).digest("base64")}`, sha256: createHash("sha256").update(data).digest("hex"), }; } /** The manifest's exact bytes: sorted packages, two-space JSON, one trailing newline. */ export function serializeManifest(manifest: PackageTrainManifest): string { const packages = [...manifest.packages] .sort((left, right) => (left.name < right.name ? -1 : 1)) .map((entry) => ({ file: entry.file, integrity: entry.integrity, name: entry.name, role: entry.role, sha256: entry.sha256, size: entry.size, version: entry.version, })); return `${JSON.stringify( { packageTrain: manifest.packageTrain, version: manifest.version, source: { modified: manifest.source.modified, revision: manifest.source.revision }, packages, }, null, 2, )}\n`; } async function readJson(file: string): Promise> { return JSON.parse(await readFile(file, "utf8")) as Record; } async function filesUnder(directory: string): Promise { const entries = await readdir(directory, { recursive: true, withFileTypes: true }); return entries .filter((entry) => entry.isFile()) .map((entry) => path.relative(directory, path.join(entry.parentPath, entry.name)).split(path.sep).join("/"), ) .sort(); } async function sourceExists(packageDirectory: string, relative: string): Promise { for (const extension of [".ts", ".tsx"]) { try { await stat(path.join(packageDirectory, "src", `${relative}${extension}`)); return true; } catch {} } return false; } async function run(command: string, arguments_: readonly string[], cwd: string): Promise { const { stdout } = await execFileAsync(command, [...arguments_], { cwd, maxBuffer: 64 * 1024 * 1024, }); return stdout; } async function inspectTarball( entry: PublicPackage, version: string, tarball: string, scratch: string, ): Promise { const directory = path.join(repositoryRoot, entry.directory); const extracted = path.join(scratch, entry.name.replace("/", "+")); await mkdir(extracted, { recursive: true }); await run("tar", ["-xzf", tarball, "-C", extracted], repositoryRoot); const root = path.join(extracted, "package"); const files = await filesUnder(root); const manifest = await readJson(path.join(root, "package.json")); const source = await readJson(path.join(directory, "package.json")); const sources = new Map(); for (const file of files) { const built = BUILT_FILE.exec(file)?.[1]; if (built !== undefined && !sources.has(built)) { sources.set(built, await sourceExists(directory, built)); } } const problems = [ ...packedManifestProblems(manifest, { name: entry.name, version }, source), ...tarballFileProblems(files, manifest, (relative) => sources.get(relative) === true), ]; // Nothing the package says may name the machine or checkout it was packed on. for (const file of files) { const text = (await readFile(path.join(root, file))).toString("latin1"); for (const local of [repositoryRoot, homedir(), tmpdir()]) { if (text.includes(local)) problems.push(`${file} names a local path`); } } return [...new Set(problems)].map((problem) => `${entry.name} ${problem}.`); } /** * Every uncommitted change in the source tree, as `git status --short` lines followed by * `git diff --stat`, all relative to the repository root; empty for a clean tree. Ignored files * (build output, `node_modules`) are not source and never count. */ async function sourceTreeChanges(): Promise { const status = ( await run("git", ["status", "--short", "--untracked-files=all"], repositoryRoot) ).trimEnd(); if (status === "") return []; const stat = (await run("git", ["diff", "--stat"], repositoryRoot)).trimEnd(); return [...status.split("\n"), ...(stat === "" ? [] : stat.split("\n"))]; } export class PackageReleaseError extends Error { readonly problems: readonly string[]; constructor(problems: readonly string[]) { super("The package release failed its checks."); this.name = "PackageReleaseError"; this.problems = problems; } } export interface PackageReleaseOptions { /** The parent of the release directory, which must not exist yet. */ readonly outDirectory: string; readonly allowModified?: boolean; } /** * Builds, packs, and inspects the public package train, and writes the release directory only when * every check passed. Returns its path and manifest; throws `PackageReleaseError` with every problem * found otherwise. */ export async function buildPackageRelease( options: PackageReleaseOptions, ): Promise<{ readonly directory: string; readonly manifest: PackageTrainManifest }> { const version = String((await readJson(path.join(repositoryRoot, "package.json"))).version); const setProblems = releaseSetProblems(await workspacePackages(), version); if (setProblems.length > 0) throw new PackageReleaseError(setProblems); const revision = (await run("git", ["rev-parse", "HEAD"], repositoryRoot)).trim(); const dirty = await sourceTreeChanges(); const modified = dirty.length > 0; if (modified && options.allowModified !== true) { throw new PackageReleaseError([ "The source tree has uncommitted changes. A release is built from a clean commit.", ...dirty, ]); } const release = path.join(path.resolve(options.outDirectory), releaseDirectoryName(version)); try { await stat(release); throw new PackageReleaseError([`${release} already exists. A release is written once.`]); } catch (error) { if (error instanceof PackageReleaseError) throw error; } // Stale output from a deleted source would otherwise ship: the build only ever adds files, and // Turbo's cache would restore whatever an earlier build left behind. for (const entry of PUBLIC_PACKAGES) { await rm(path.join(repositoryRoot, entry.directory, "dist"), { force: true, recursive: true }); } await run( "pnpm", [ "exec", "turbo", "run", "build", "--force", "--output-logs=errors-only", ...PUBLIC_PACKAGES.map((entry) => `--filter=${entry.name}`), ], repositoryRoot, ); // The source was clean before the build; it must still be, so what is packed is the commit. A build // that wrote a tracked file, or an untracked file Git does not ignore, fails the release here. const changedByBuild = (await sourceTreeChanges()).filter((line) => !dirty.includes(line)); if (changedByBuild.length > 0) { throw new PackageReleaseError([ "The build changed the source tree. A release packs exactly its commit.", ...changedByBuild, ]); } const scratch = await mkdtemp(path.join(tmpdir(), "ekklesia-package-release-")); try { const staging = path.join(scratch, "release"); await mkdir(staging); const problems: string[] = []; const packages: TrainPackage[] = []; for (const entry of PUBLIC_PACKAGES) { const report = JSON.parse( await run( "pnpm", [`--config.pnpmfile=${PACK_HOOKS}`, "pack", "--json", "--pack-destination", staging], path.join(repositoryRoot, entry.directory), ), ) as { filename: string }; const file = path.basename(report.filename); if (file !== tarballName(entry.name, version)) { problems.push(`${entry.name} packed as ${file}.`); continue; } problems.push(...(await inspectTarball(entry, version, report.filename, scratch))); const data = await readFile(report.filename); packages.push({ file, name: entry.name, role: entry.role, size: data.length, version, ...artifactDigests(data), }); } const manifest: PackageTrainManifest = { packageTrain: PACKAGE_TRAIN_FORMAT, packages, source: { modified, revision }, version, }; const contents = new Map(); for (const file of await readdir(staging)) { contents.set(file, await readFile(path.join(staging, file))); } problems.push(...releaseDirectoryProblems(manifest, contents)); if (problems.length > 0) throw new PackageReleaseError(problems); const sorted = [...manifest.packages].sort((left, right) => (left.file < right.file ? -1 : 1)); await writeFile( path.join(staging, "SHA256SUMS"), sorted.map((entry) => `${entry.sha256} ${entry.file}\n`).join(""), ); await writeFile(path.join(staging, PACKAGE_TRAIN_MANIFEST), serializeManifest(manifest)); await mkdir(path.dirname(release), { recursive: true }); // Into place in one step: the release directory either holds a whole, checked release or does // not exist. A rename across file systems is not atomic, so copy then, never half a release. try { await rename(staging, release); } catch { await run("cp", ["-R", staging, release], repositoryRoot); } return { directory: release, manifest }; } finally { await rm(scratch, { force: true, recursive: true }); } } async function main(argv: readonly string[]): Promise { const allowModified = argv.includes("--allow-modified"); const rest = argv.filter((argument) => argument !== "--allow-modified"); if (rest.length !== 2 || rest[0] !== "--out" || !rest[1]) { process.stderr.write( "Usage: node scripts/package-release.ts --out [--allow-modified]\n", ); return 64; } try { const { directory, manifest } = await buildPackageRelease({ allowModified, outDirectory: rest[1], }); for (const entry of manifest.packages) { process.stdout.write(`${entry.file}: ${entry.size} bytes, sha256 ${entry.sha256}\n`); } process.stdout.write( `Built and inspected the ${manifest.version} package train (${manifest.packages.length} packages${ manifest.source.modified ? ", from a modified source tree: never publish it" : "" }) in ${directory}\n`, ); return 0; } catch (error) { if (error instanceof PackageReleaseError) { for (const problem of error.problems) process.stderr.write(`${problem}\n`); return 1; } process.stderr.write("Building the package release failed while building or packing.\n"); return 1; } } if (process.argv[1] && import.meta.url === pathToFileURL(process.argv[1]).href) { process.exitCode = await main(process.argv.slice(2)); }