feat: publish the exact @ekklesia package train to the npm registry (ADR 0053)
CI / verify (pull_request) Successful in 1m1s
CI / infrastructure-smoke (pull_request) Successful in 35s
CI / migration-smoke (pull_request) Successful in 1m53s
CI / package-release (pull_request) Failing after 43s
CI / package-publication (pull_request) Skipped
CI / image-build (pull_request) Successful in 29s
CI / deploy-smoke (split) (pull_request) Canceled after 0s
CI / deploy-smoke (same-origin) (pull_request) Canceled after 38s
CI / verify (pull_request) Successful in 1m1s
CI / infrastructure-smoke (pull_request) Successful in 35s
CI / migration-smoke (pull_request) Successful in 1m53s
CI / package-release (pull_request) Failing after 43s
CI / package-publication (pull_request) Skipped
CI / image-build (pull_request) Successful in 29s
CI / deploy-smoke (split) (pull_request) Canceled after 0s
CI / deploy-smoke (same-origin) (pull_request) Canceled after 38s
Publishes one package release train to the Gitea npm registry under River_of_Glory_Church, whose packages are publicly readable on purpose: exactly the .tgz files the package-release gate proved, and nothing else. Nothing is published by this change. - scripts/publish-package-train.ts validates the whole local train before any request (gate manifest hash, version, revision, unmodified source, the nine packages, every artifact's size, SHA-256, integrity, SHA256SUMS line, and inner package.json), then publishes in dependency order with `npm publish <file.tgz>` under a staging tag. Each package is reconciled first (absent: publish; same bytes: accept; other bytes: hard stop, version burned), then read back: metadata (name, version, integrity, dependency maps, normalized bin, tarball on the registry) and the tarball itself, byte for byte, also anonymously. `latest` moves only after all nine verify; reruns converge and never delete anything. It writes a publication record. The token comes from EKKLESIA_PACKAGES_PUBLISH_TOKEN only, through a temporary npm config that refers to the variable. - scripts/test-package-publication.sh proves it against a disposable Gitea 1.27.2 with public owners, a publisher team and an outsider: anonymous reads, refused unauthorized publishing, byte-identical round trips, the registry's own 409, partial-train recovery, the burned version hard stop, and dist-tag retry convergence. - CI: package-release keeps its proven release and hands it on as an artifact with its manifest hash; the new package-publication job publishes exactly that artifact to the disposable Gitea. Artifact actions v3 (node20): v4 refuses any server other than github.com. - .github/workflows/publish-packages.yml publishes to the real registry only by manual dispatch on main, after its own gate run, with the secret in the publish step alone. - docs: the public-read model, the workflow, the record, credentials, immutability policy, provider moves, and the first-publication steps. Co-Authored-By: Claude Opus 5.5 <[email protected]>
This commit is contained in:
co-authored by
Claude Opus 5.5
parent
57df2e15f2
commit
d3c62e63bf
@@ -0,0 +1,889 @@
|
||||
import { execFile } from "node:child_process";
|
||||
import { createHash } from "node:crypto";
|
||||
import { mkdtemp, readdir, readFile, rm, writeFile } from "node:fs/promises";
|
||||
import { tmpdir } from "node:os";
|
||||
import path from "node:path";
|
||||
import process from "node:process";
|
||||
import { pathToFileURL } from "node:url";
|
||||
import { promisify } from "node:util";
|
||||
import { gunzipSync } from "node:zlib";
|
||||
import {
|
||||
artifactDigests,
|
||||
PACKAGE_TRAIN_FORMAT,
|
||||
PACKAGE_TRAIN_MANIFEST,
|
||||
type PackageTrainManifest,
|
||||
releaseDirectoryProblems,
|
||||
type TrainPackage,
|
||||
} from "./package-release.ts";
|
||||
|
||||
/**
|
||||
* Publishes one package release train (ADR 0053) to an npm-protocol registry: exactly the `.tgz`
|
||||
* files `pnpm packages:release` produced, as `ekklesia-package-train.json` records them, and nothing
|
||||
* else. It never builds, packs, or edits a package, and never deletes or unpublishes anything.
|
||||
*
|
||||
* EKKLESIA_PACKAGES_PUBLISH_TOKEN=… node scripts/publish-package-train.ts \
|
||||
* --release <release directory> --registry <npm registry URL> --version <train version> \
|
||||
* [--revision <full commit id>] [--manifest-sha256 <hex>] [--record <file>] \
|
||||
* [--staging-tag <tag>] [--final-tag <tag>] [--require-public-read]
|
||||
*
|
||||
* 1. The whole local train is validated before anything is sent: the manifest, every artifact's
|
||||
* name, size, SHA-256, integrity, and the package.json inside it.
|
||||
* 2. Packages go out in dependency order. For each: absent from the registry, it is published with
|
||||
* `npm publish <file.tgz>` under a staging dist-tag; present with the same bytes, it is accepted
|
||||
* as already published; present with other bytes, publication stops for good (the version is
|
||||
* burned). Every package is then read back: its metadata, and its tarball, which must be the
|
||||
* release artifact byte for byte.
|
||||
* 3. Only when all of them are verified does the final dist-tag (`latest`) move, and the train is
|
||||
* published once every tag agrees. A rerun converges: it publishes only what is missing and moves
|
||||
* only tags that are not yet right.
|
||||
*
|
||||
* The publish token is read from the environment only, and reaches npm through a temporary npm
|
||||
* configuration that refers to the variable, never the value; the configuration is removed after.
|
||||
*/
|
||||
|
||||
const execFileAsync = promisify(execFile);
|
||||
|
||||
export const PUBLISH_TOKEN_VARIABLE = "EKKLESIA_PACKAGES_PUBLISH_TOKEN";
|
||||
export const DEFAULT_STAGING_TAG = "ekklesia-staging";
|
||||
export const DEFAULT_FINAL_TAG = "latest";
|
||||
export const PUBLICATION_RECORD_FORMAT = 1;
|
||||
const DEPENDENCY_FIELDS = Object.freeze([
|
||||
"dependencies",
|
||||
"peerDependencies",
|
||||
"optionalDependencies",
|
||||
] as const);
|
||||
|
||||
export type PublicationErrorCode =
|
||||
| "INVALID_OPTIONS"
|
||||
| "INVALID_RELEASE"
|
||||
| "BURNED_VERSION"
|
||||
| "REGISTRY_UNAVAILABLE"
|
||||
| "PUBLISH_FAILED"
|
||||
| "VERIFICATION_FAILED"
|
||||
| "TAGS_NOT_CONVERGED";
|
||||
|
||||
export class PublicationError extends Error {
|
||||
readonly code: PublicationErrorCode;
|
||||
readonly problems: readonly string[];
|
||||
|
||||
constructor(code: PublicationErrorCode, message: string, problems: readonly string[] = []) {
|
||||
super(message);
|
||||
this.name = "PublicationError";
|
||||
this.code = code;
|
||||
this.problems = problems;
|
||||
}
|
||||
}
|
||||
|
||||
function isRecord(value: unknown): value is Readonly<Record<string, unknown>> {
|
||||
return typeof value === "object" && value !== null && !Array.isArray(value);
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------------------------
|
||||
// The local train
|
||||
// ---------------------------------------------------------------------------------------------
|
||||
|
||||
/** The `package/package.json` inside an npm tarball, read without unpacking anything to disk. */
|
||||
export function tarballPackageJson(tarball: Uint8Array): Record<string, unknown> {
|
||||
const data = gunzipSync(tarball);
|
||||
for (let offset = 0; offset + 512 <= data.length; ) {
|
||||
const header = data.subarray(offset, offset + 512);
|
||||
if (header.every((byte) => byte === 0)) break;
|
||||
const field = (start: number, length: number) =>
|
||||
header
|
||||
.subarray(start, start + length)
|
||||
.toString("utf8")
|
||||
.replace(/\0.*$/su, "");
|
||||
const name = field(0, 100);
|
||||
const prefix = header.subarray(257, 262).toString("latin1") === "ustar" ? field(345, 155) : "";
|
||||
const size = Number.parseInt(field(124, 12).trim() || "0", 8);
|
||||
const type = field(156, 1);
|
||||
const fullName = prefix ? `${prefix}/${name}` : name;
|
||||
if ((type === "0" || type === "") && fullName === "package/package.json") {
|
||||
const body = data.subarray(offset + 512, offset + 512 + size);
|
||||
return JSON.parse(body.toString("utf8")) as Record<string, unknown>;
|
||||
}
|
||||
offset += 512 + Math.ceil(size / 512) * 512;
|
||||
}
|
||||
throw new Error("The tarball holds no package/package.json.");
|
||||
}
|
||||
|
||||
export interface LocalPackage {
|
||||
readonly entry: TrainPackage;
|
||||
readonly path: string;
|
||||
readonly data: Buffer;
|
||||
readonly packageJson: Readonly<Record<string, unknown>>;
|
||||
}
|
||||
|
||||
export interface LocalTrain {
|
||||
readonly manifest: PackageTrainManifest;
|
||||
/** SHA-256 of `ekklesia-package-train.json`'s exact bytes. */
|
||||
readonly manifestSha256: string;
|
||||
readonly packages: readonly LocalPackage[];
|
||||
}
|
||||
|
||||
/**
|
||||
* Reads and validates a release directory completely, or throws `INVALID_RELEASE` with every
|
||||
* problem: nothing may be published from a train that is not whole and unchanged.
|
||||
*/
|
||||
export async function loadReleaseTrain(
|
||||
releaseDirectory: string,
|
||||
expected: { readonly version: string; readonly revision?: string },
|
||||
): Promise<LocalTrain> {
|
||||
const files = new Map<string, Buffer>();
|
||||
for (const file of await readdir(releaseDirectory)) {
|
||||
files.set(file, await readFile(path.join(releaseDirectory, file)));
|
||||
}
|
||||
const manifestBytes = files.get(PACKAGE_TRAIN_MANIFEST);
|
||||
if (manifestBytes === undefined) {
|
||||
throw new PublicationError("INVALID_RELEASE", `The release has no ${PACKAGE_TRAIN_MANIFEST}.`);
|
||||
}
|
||||
let manifest: PackageTrainManifest;
|
||||
try {
|
||||
manifest = JSON.parse(manifestBytes.toString("utf8")) as PackageTrainManifest;
|
||||
} catch {
|
||||
throw new PublicationError("INVALID_RELEASE", `${PACKAGE_TRAIN_MANIFEST} is not JSON.`);
|
||||
}
|
||||
const problems = trainProblems(manifest, files, expected);
|
||||
if (problems.length > 0) {
|
||||
throw new PublicationError("INVALID_RELEASE", "The release train failed validation.", problems);
|
||||
}
|
||||
return {
|
||||
manifest,
|
||||
manifestSha256: createHash("sha256").update(manifestBytes).digest("hex"),
|
||||
packages: manifest.packages.map((entry) => {
|
||||
const data = files.get(entry.file) as Buffer;
|
||||
return {
|
||||
data,
|
||||
entry,
|
||||
packageJson: tarballPackageJson(data),
|
||||
path: path.join(releaseDirectory, entry.file),
|
||||
};
|
||||
}),
|
||||
};
|
||||
}
|
||||
|
||||
/** Every reason a release directory's contents may not be published, or nothing. */
|
||||
export function trainProblems(
|
||||
manifest: PackageTrainManifest,
|
||||
files: ReadonlyMap<string, Uint8Array>,
|
||||
expected: { readonly version: string; readonly revision?: string },
|
||||
): string[] {
|
||||
if (
|
||||
!isRecord(manifest) ||
|
||||
manifest.packageTrain !== PACKAGE_TRAIN_FORMAT ||
|
||||
!Array.isArray(manifest.packages) ||
|
||||
!isRecord(manifest.source)
|
||||
) {
|
||||
return ["The release manifest is not a package train manifest."];
|
||||
}
|
||||
const problems = releaseDirectoryProblems(manifest, files);
|
||||
if (manifest.source.modified !== false) {
|
||||
problems.push("The release was built from a modified source tree; it is never published.");
|
||||
}
|
||||
if (manifest.version !== expected.version) {
|
||||
problems.push(
|
||||
`The release is version ${String(manifest.version)}, not the requested ${expected.version}.`,
|
||||
);
|
||||
}
|
||||
if (!/^[0-9a-f]{40}([0-9a-f]{24})?$/u.test(String(manifest.source.revision))) {
|
||||
problems.push("The release names no full source revision.");
|
||||
}
|
||||
if (expected.revision !== undefined && manifest.source.revision !== expected.revision) {
|
||||
problems.push(
|
||||
`The release was built from ${String(manifest.source.revision)}, not the expected ${expected.revision}.`,
|
||||
);
|
||||
}
|
||||
const sums = files.get("SHA256SUMS");
|
||||
const expectedSums = [...manifest.packages]
|
||||
.sort((left, right) => (left.file < right.file ? -1 : 1))
|
||||
.map((entry) => `${entry.sha256} ${entry.file}\n`)
|
||||
.join("");
|
||||
if (sums === undefined || Buffer.from(sums).toString("utf8") !== expectedSums) {
|
||||
problems.push("SHA256SUMS does not match the release manifest.");
|
||||
}
|
||||
for (const entry of manifest.packages) {
|
||||
const data = files.get(entry.file);
|
||||
if (data === undefined) continue;
|
||||
try {
|
||||
const packageJson = tarballPackageJson(data);
|
||||
if (packageJson.name !== entry.name || packageJson.version !== entry.version) {
|
||||
problems.push(
|
||||
`${entry.file} holds ${String(packageJson.name)}@${String(packageJson.version)}, not ${entry.name}@${entry.version}.`,
|
||||
);
|
||||
}
|
||||
} catch {
|
||||
problems.push(`${entry.file} is not a readable npm package tarball.`);
|
||||
}
|
||||
}
|
||||
return problems;
|
||||
}
|
||||
|
||||
/**
|
||||
* The order packages are published in: every package after the train packages it depends on (in
|
||||
* `dependencies`, `peerDependencies`, or `optionalDependencies`), and otherwise by name, so the order
|
||||
* never depends on how a directory happens to list its files.
|
||||
*/
|
||||
export function publicationOrder<
|
||||
Package extends {
|
||||
readonly entry: { readonly name: string };
|
||||
readonly packageJson: Readonly<Record<string, unknown>>;
|
||||
},
|
||||
>(packages: readonly Package[]): Package[] {
|
||||
const byName = new Map(packages.map((entry) => [entry.entry.name, entry]));
|
||||
const dependsOn = new Map<string, Set<string>>();
|
||||
for (const entry of packages) {
|
||||
const names = new Set<string>();
|
||||
for (const field of DEPENDENCY_FIELDS) {
|
||||
const dependencies = entry.packageJson[field];
|
||||
if (!isRecord(dependencies)) continue;
|
||||
for (const name of Object.keys(dependencies)) {
|
||||
if (byName.has(name) && name !== entry.entry.name) names.add(name);
|
||||
}
|
||||
}
|
||||
dependsOn.set(entry.entry.name, names);
|
||||
}
|
||||
const order: Package[] = [];
|
||||
const done = new Set<string>();
|
||||
while (order.length < packages.length) {
|
||||
const ready = [...byName.keys()]
|
||||
.filter((name) => !done.has(name))
|
||||
.filter((name) =>
|
||||
[...(dependsOn.get(name) ?? [])].every((dependency) => done.has(dependency)),
|
||||
)
|
||||
.sort();
|
||||
const next = ready[0];
|
||||
if (next === undefined) {
|
||||
throw new PublicationError(
|
||||
"INVALID_RELEASE",
|
||||
"The release's packages depend on each other in a cycle.",
|
||||
);
|
||||
}
|
||||
done.add(next);
|
||||
order.push(byName.get(next) as Package);
|
||||
}
|
||||
return order;
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------------------------
|
||||
// Comparing a registry copy with the release
|
||||
// ---------------------------------------------------------------------------------------------
|
||||
|
||||
/** `bin` as npm publishes it: a string becomes `{ <package name without scope>: path }`, and paths
|
||||
* lose a leading `./`. The tarball keeps the author's form; the registry metadata gets this one. */
|
||||
export function normalizeBin(bin: unknown, packageName: string): Record<string, string> {
|
||||
const entries: [string, string][] =
|
||||
typeof bin === "string"
|
||||
? [[packageName.replace(/^@[^/]+\//u, ""), bin]]
|
||||
: isRecord(bin)
|
||||
? Object.entries(bin).map(([name, target]) => [name, String(target)])
|
||||
: [];
|
||||
return Object.fromEntries(
|
||||
entries
|
||||
.map(([name, target]): [string, string] => [name, target.replace(/^(?:\.\/)+/u, "")])
|
||||
.sort(([left], [right]) => (left < right ? -1 : 1)),
|
||||
);
|
||||
}
|
||||
|
||||
function sortedRecord(value: unknown): Record<string, unknown> {
|
||||
if (!isRecord(value)) return {};
|
||||
return Object.fromEntries(
|
||||
Object.keys(value)
|
||||
.sort()
|
||||
.map((key) => [key, value[key]]),
|
||||
);
|
||||
}
|
||||
|
||||
/** Whether `url` is the registry itself or below it: same origin, same path prefix. */
|
||||
export function isUnderRegistry(url: string, registry: string): boolean {
|
||||
try {
|
||||
const target = new URL(url);
|
||||
const base = new URL(registry);
|
||||
return target.origin === base.origin && target.pathname.startsWith(base.pathname);
|
||||
} catch {
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Every way a registry's copy of a package differs from the release artifact, or nothing:
|
||||
* the version metadata the registry serves, and the bytes it serves for it.
|
||||
*/
|
||||
export function registryCopyProblems(
|
||||
local: Pick<LocalPackage, "entry" | "packageJson" | "data">,
|
||||
version: Readonly<Record<string, unknown>>,
|
||||
downloaded: Uint8Array,
|
||||
registry: string,
|
||||
): string[] {
|
||||
const problems: string[] = [];
|
||||
const { entry, packageJson } = local;
|
||||
if (version.name !== entry.name || version.version !== entry.version) {
|
||||
problems.push(`the registry names ${String(version.name)}@${String(version.version)}`);
|
||||
}
|
||||
const dist = isRecord(version.dist) ? version.dist : {};
|
||||
if (dist.integrity !== entry.integrity) {
|
||||
problems.push("the registry reports a different integrity");
|
||||
}
|
||||
const sha1 = createHash("sha1").update(local.data).digest("hex");
|
||||
if (dist.shasum !== undefined && dist.shasum !== sha1) {
|
||||
problems.push("the registry reports a different shasum");
|
||||
}
|
||||
if (typeof dist.tarball !== "string" || !isUnderRegistry(dist.tarball, registry)) {
|
||||
problems.push("the registry's tarball URL is not on the configured registry");
|
||||
}
|
||||
for (const field of DEPENDENCY_FIELDS) {
|
||||
if (
|
||||
JSON.stringify(sortedRecord(version[field])) !==
|
||||
JSON.stringify(sortedRecord(packageJson[field]))
|
||||
) {
|
||||
problems.push(`the registry's ${field} differ from the package's`);
|
||||
}
|
||||
}
|
||||
if (
|
||||
JSON.stringify(normalizeBin(version.bin, entry.name)) !==
|
||||
JSON.stringify(normalizeBin(packageJson.bin, entry.name))
|
||||
) {
|
||||
problems.push("the registry's bin differs from the package's");
|
||||
}
|
||||
const digests = artifactDigests(downloaded);
|
||||
if (
|
||||
!Buffer.from(downloaded).equals(local.data) ||
|
||||
digests.sha256 !== entry.sha256 ||
|
||||
digests.integrity !== entry.integrity
|
||||
) {
|
||||
problems.push("the registry serves different bytes from the release artifact");
|
||||
}
|
||||
return problems;
|
||||
}
|
||||
|
||||
/** SemVer 2.0.0 precedence: negative, zero, or positive. Build metadata is ignored. */
|
||||
export function compareSemver(left: string, right: string): number {
|
||||
const parse = (version: string) => {
|
||||
const match = /^(\d+)\.(\d+)\.(\d+)(?:-([0-9A-Za-z.-]+))?(?:\+[0-9A-Za-z.-]+)?$/u.exec(version);
|
||||
if (match === null) throw new Error(`Not a semantic version: ${version}`);
|
||||
return {
|
||||
core: [Number(match[1]), Number(match[2]), Number(match[3])],
|
||||
pre: match[4] === undefined ? [] : match[4].split("."),
|
||||
};
|
||||
};
|
||||
const a = parse(left);
|
||||
const b = parse(right);
|
||||
for (let index = 0; index < 3; index += 1) {
|
||||
const difference = (a.core[index] ?? 0) - (b.core[index] ?? 0);
|
||||
if (difference !== 0) return Math.sign(difference);
|
||||
}
|
||||
if (a.pre.length === 0 || b.pre.length === 0) return Math.sign(b.pre.length - a.pre.length);
|
||||
for (let index = 0; index < Math.max(a.pre.length, b.pre.length); index += 1) {
|
||||
const x = a.pre[index];
|
||||
const y = b.pre[index];
|
||||
if (x === undefined || y === undefined) return x === undefined ? -1 : 1;
|
||||
if (x === y) continue;
|
||||
const xNumber = /^\d+$/u.test(x);
|
||||
const yNumber = /^\d+$/u.test(y);
|
||||
if (xNumber && yNumber) return Math.sign(Number(x) - Number(y));
|
||||
if (xNumber !== yNumber) return xNumber ? -1 : 1;
|
||||
return x < y ? -1 : 1;
|
||||
}
|
||||
return 0;
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------------------------
|
||||
// The registry: read over HTTP, written with npm
|
||||
// ---------------------------------------------------------------------------------------------
|
||||
|
||||
/** The registry URL, validated, with exactly one trailing slash. */
|
||||
export function normalizeRegistry(registry: string, allowHttp = false): string {
|
||||
let url: URL;
|
||||
try {
|
||||
url = new URL(registry);
|
||||
} catch {
|
||||
throw new PublicationError("INVALID_OPTIONS", "The registry is not a URL.");
|
||||
}
|
||||
if (url.username !== "" || url.password !== "" || url.search !== "" || url.hash !== "") {
|
||||
throw new PublicationError(
|
||||
"INVALID_OPTIONS",
|
||||
"The registry URL may hold no credentials, query, or fragment.",
|
||||
);
|
||||
}
|
||||
if (url.protocol !== "https:" && !(allowHttp && url.protocol === "http:")) {
|
||||
throw new PublicationError("INVALID_OPTIONS", "The registry must be reached over HTTPS.");
|
||||
}
|
||||
url.pathname = url.pathname.endsWith("/") ? url.pathname : `${url.pathname}/`;
|
||||
return url.toString();
|
||||
}
|
||||
|
||||
/** The npm configuration a publish uses: the scope and registry, and the token by reference. */
|
||||
export function npmConfiguration(registry: string): string {
|
||||
const url = new URL(registry);
|
||||
return [
|
||||
`registry=${registry}`,
|
||||
`@ekklesia:registry=${registry}`,
|
||||
`//${url.host}${url.pathname}:_authToken=\${${PUBLISH_TOKEN_VARIABLE}}`,
|
||||
"",
|
||||
].join("\n");
|
||||
}
|
||||
|
||||
/** `@ekklesia/module-sdk` → `<registry>@ekklesia%2fmodule-sdk`, the npm packument path. */
|
||||
export function packumentUrl(registry: string, name: string): string {
|
||||
return `${registry}${name.replace("/", "%2f")}`;
|
||||
}
|
||||
|
||||
export interface RegistryReader {
|
||||
/** The package's packument, or `undefined` when the registry has no such package. */
|
||||
packument(name: string): Promise<Record<string, unknown> | undefined>;
|
||||
/** A tarball's bytes; authenticated only when the URL is on the registry itself. */
|
||||
download(url: string, anonymous?: boolean): Promise<Buffer>;
|
||||
}
|
||||
|
||||
export function registryReader(
|
||||
registry: string,
|
||||
token: string | undefined,
|
||||
fetchImplementation: typeof fetch = fetch,
|
||||
): RegistryReader {
|
||||
const headers = (url: string, anonymous: boolean): Record<string, string> =>
|
||||
!anonymous && token && isUnderRegistry(url, registry)
|
||||
? { authorization: `Bearer ${token}` }
|
||||
: {};
|
||||
const get = async (url: string, accept: string, anonymous: boolean) => {
|
||||
try {
|
||||
return await fetchImplementation(url, {
|
||||
headers: { accept, ...headers(url, anonymous) },
|
||||
redirect: "error",
|
||||
});
|
||||
} catch {
|
||||
throw new PublicationError("REGISTRY_UNAVAILABLE", "The registry could not be reached.");
|
||||
}
|
||||
};
|
||||
return {
|
||||
async download(url, anonymous = false) {
|
||||
const response = await get(url, "application/octet-stream", anonymous);
|
||||
if (!response.ok) {
|
||||
throw new PublicationError(
|
||||
"REGISTRY_UNAVAILABLE",
|
||||
`The registry answered ${response.status} for a package tarball.`,
|
||||
);
|
||||
}
|
||||
return Buffer.from(await response.arrayBuffer());
|
||||
},
|
||||
async packument(name) {
|
||||
const response = await get(packumentUrl(registry, name), "application/json", false);
|
||||
if (response.status === 404) return undefined;
|
||||
if (!response.ok) {
|
||||
throw new PublicationError(
|
||||
"REGISTRY_UNAVAILABLE",
|
||||
`The registry answered ${response.status} for ${name}.`,
|
||||
);
|
||||
}
|
||||
return (await response.json()) as Record<string, unknown>;
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
function versionOf(
|
||||
packument: Record<string, unknown> | undefined,
|
||||
version: string,
|
||||
): Record<string, unknown> | undefined {
|
||||
const versions = packument?.versions;
|
||||
const found = isRecord(versions) ? versions[version] : undefined;
|
||||
return isRecord(found) ? { ...found } : undefined;
|
||||
}
|
||||
|
||||
function distTagsOf(packument: Record<string, unknown> | undefined): Record<string, string> {
|
||||
const tags = packument?.["dist-tags"];
|
||||
return isRecord(tags)
|
||||
? Object.fromEntries(Object.entries(tags).map(([tag, version]) => [tag, String(version)]))
|
||||
: {};
|
||||
}
|
||||
|
||||
interface NpmSession {
|
||||
publish(file: string, tag: string): Promise<void>;
|
||||
addTag(name: string, version: string, tag: string): Promise<void>;
|
||||
dispose(): Promise<void>;
|
||||
}
|
||||
|
||||
/** npm with only a temporary configuration: no user or global npmrc, no project directory. */
|
||||
async function npmSession(registry: string, token: string, npm: string): Promise<NpmSession> {
|
||||
const home = await mkdtemp(path.join(tmpdir(), "ekklesia-publish-"));
|
||||
const userconfig = path.join(home, "npmrc");
|
||||
const globalconfig = path.join(home, "global-npmrc");
|
||||
await writeFile(userconfig, npmConfiguration(registry), { mode: 0o600 });
|
||||
await writeFile(globalconfig, "", { mode: 0o600 });
|
||||
const environment = {
|
||||
[PUBLISH_TOKEN_VARIABLE]: token,
|
||||
HOME: home,
|
||||
PATH: process.env.PATH ?? "",
|
||||
npm_config_audit: "false",
|
||||
npm_config_cache: path.join(home, "cache"),
|
||||
npm_config_fund: "false",
|
||||
npm_config_globalconfig: globalconfig,
|
||||
npm_config_update_notifier: "false",
|
||||
npm_config_userconfig: userconfig,
|
||||
};
|
||||
const run = async (args: readonly string[], what: string) => {
|
||||
try {
|
||||
await execFileAsync(npm, [...args], { cwd: home, env: environment, maxBuffer: 16 << 20 });
|
||||
} catch (error) {
|
||||
const stderr = String((error as { stderr?: unknown }).stderr ?? "");
|
||||
const summary = stderr
|
||||
.split("\n")
|
||||
.filter((line) => /^npm error (?:code|\d{3}|[A-Z])/u.test(line))
|
||||
.filter((line) => !line.includes("complete log"))
|
||||
.slice(0, 3)
|
||||
.join(" / ")
|
||||
.replaceAll(token, "***")
|
||||
.replaceAll(home, "<npm home>");
|
||||
throw new PublicationError(
|
||||
"PUBLISH_FAILED",
|
||||
`${what} failed${summary ? `: ${summary}` : "."}`,
|
||||
);
|
||||
}
|
||||
};
|
||||
return {
|
||||
addTag: (name, version, tag) =>
|
||||
run(["dist-tag", "add", `${name}@${version}`, tag], `Moving ${tag} to ${name}@${version}`),
|
||||
dispose: () => rm(home, { force: true, recursive: true }),
|
||||
publish: (file, tag) =>
|
||||
run(
|
||||
// A file spec runs no lifecycle script and sends the file's own bytes; --ignore-scripts
|
||||
// says so explicitly. --access public is what a scoped package needs on a public registry.
|
||||
["publish", file, "--tag", tag, "--access", "public", "--ignore-scripts"],
|
||||
`Publishing ${path.basename(file)}`,
|
||||
),
|
||||
};
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------------------------
|
||||
// Publishing a train
|
||||
// ---------------------------------------------------------------------------------------------
|
||||
|
||||
export type PackageOutcome = "published" | "already-present";
|
||||
export type TagOutcome = "moved" | "already-current" | "kept-newer";
|
||||
|
||||
export interface PublicationHooks {
|
||||
/** Called before each package is published (not for one already present). Tests use it to
|
||||
* interrupt a train part way. */
|
||||
beforePublish?(name: string, index: number): void | Promise<void>;
|
||||
/** Called before each final dist-tag is moved. */
|
||||
beforeTag?(name: string, index: number): void | Promise<void>;
|
||||
}
|
||||
|
||||
export interface PublishOptions {
|
||||
readonly releaseDirectory: string;
|
||||
readonly registry: string;
|
||||
readonly version: string;
|
||||
readonly revision?: string;
|
||||
/** SHA-256 of the release manifest the package-release gate proved, which binds the artifact a
|
||||
* publish job received to that gate. */
|
||||
readonly manifestSha256?: string;
|
||||
readonly token: string;
|
||||
readonly stagingTag?: string;
|
||||
readonly finalTag?: string;
|
||||
readonly requirePublicRead?: boolean;
|
||||
/** Only for a disposable test registry. */
|
||||
readonly allowHttp?: boolean;
|
||||
readonly npm?: string;
|
||||
readonly fetch?: typeof fetch;
|
||||
readonly hooks?: PublicationHooks;
|
||||
readonly log?: (line: string) => void;
|
||||
readonly now?: () => Date;
|
||||
}
|
||||
|
||||
export interface PublishedPackage {
|
||||
readonly name: string;
|
||||
readonly version: string;
|
||||
readonly file: string;
|
||||
readonly outcome: PackageOutcome;
|
||||
readonly artifact: { readonly sha256: string; readonly integrity: string; readonly size: number };
|
||||
readonly registry: {
|
||||
readonly tarball: string;
|
||||
readonly sha256: string;
|
||||
readonly integrity: string;
|
||||
readonly publicRead: boolean;
|
||||
};
|
||||
readonly finalTag: TagOutcome;
|
||||
readonly distTags: Readonly<Record<string, string>>;
|
||||
}
|
||||
|
||||
/** Evidence that one train was published: not a package catalog, and never a secret. */
|
||||
export interface PublicationRecord {
|
||||
readonly publicationRecord: typeof PUBLICATION_RECORD_FORMAT;
|
||||
readonly packageTrain: {
|
||||
readonly version: string;
|
||||
readonly revision: string;
|
||||
readonly manifestSha256: string;
|
||||
};
|
||||
readonly registry: string;
|
||||
readonly finalTag: string;
|
||||
readonly completedAt: string;
|
||||
readonly packages: readonly PublishedPackage[];
|
||||
}
|
||||
|
||||
export function serializeRecord(record: PublicationRecord): string {
|
||||
return `${JSON.stringify(record, null, 2)}\n`;
|
||||
}
|
||||
|
||||
const TOKEN_FORMAT = /^[\x21-\x7e]{8,}$/u;
|
||||
|
||||
export async function publishPackageTrain(options: PublishOptions): Promise<PublicationRecord> {
|
||||
const log = options.log ?? (() => {});
|
||||
const stagingTag = options.stagingTag ?? DEFAULT_STAGING_TAG;
|
||||
const finalTag = options.finalTag ?? DEFAULT_FINAL_TAG;
|
||||
if (!TOKEN_FORMAT.test(options.token)) {
|
||||
throw new PublicationError(
|
||||
"INVALID_OPTIONS",
|
||||
`${PUBLISH_TOKEN_VARIABLE} must hold the publish token.`,
|
||||
);
|
||||
}
|
||||
for (const tag of [stagingTag, finalTag]) {
|
||||
if (!/^[a-z][a-z0-9-]*$/u.test(tag)) {
|
||||
throw new PublicationError("INVALID_OPTIONS", `${tag} is not a usable dist-tag.`);
|
||||
}
|
||||
}
|
||||
if (stagingTag === finalTag) {
|
||||
throw new PublicationError("INVALID_OPTIONS", "The staging and final tags must differ.");
|
||||
}
|
||||
const registry = normalizeRegistry(options.registry, options.allowHttp);
|
||||
|
||||
// Nothing is sent until the whole local train is valid.
|
||||
const train = await loadReleaseTrain(options.releaseDirectory, {
|
||||
version: options.version,
|
||||
...(options.revision === undefined ? {} : { revision: options.revision }),
|
||||
});
|
||||
if (options.manifestSha256 !== undefined && options.manifestSha256 !== train.manifestSha256) {
|
||||
throw new PublicationError(
|
||||
"INVALID_RELEASE",
|
||||
"The release manifest is not the one the package-release gate proved.",
|
||||
);
|
||||
}
|
||||
const order = publicationOrder(train.packages);
|
||||
log(
|
||||
`Package train ${train.manifest.version} (${train.manifest.source.revision}) validated: ${order
|
||||
.map((entry) => entry.entry.name)
|
||||
.join(", ")}`,
|
||||
);
|
||||
|
||||
const reader = registryReader(registry, options.token, options.fetch);
|
||||
const npm = await npmSession(registry, options.token, options.npm ?? "npm");
|
||||
try {
|
||||
const verified: {
|
||||
local: LocalPackage;
|
||||
outcome: PackageOutcome;
|
||||
tarball: string;
|
||||
publicRead: boolean;
|
||||
downloaded: { sha256: string; integrity: string };
|
||||
}[] = [];
|
||||
for (const [index, local] of order.entries()) {
|
||||
const { name, version } = local.entry;
|
||||
let outcome: PackageOutcome = "already-present";
|
||||
let existing = versionOf(await reader.packument(name), version);
|
||||
if (existing === undefined) {
|
||||
await options.hooks?.beforePublish?.(name, index);
|
||||
try {
|
||||
await npm.publish(local.path, stagingTag);
|
||||
outcome = "published";
|
||||
} catch (error) {
|
||||
// Someone may have published it meanwhile; what is there now decides.
|
||||
if (versionOf(await reader.packument(name), version) === undefined) throw error;
|
||||
}
|
||||
existing = versionOf(await reader.packument(name), version);
|
||||
if (existing === undefined) {
|
||||
throw new PublicationError(
|
||||
"VERIFICATION_FAILED",
|
||||
`${name}@${version} is not on the registry after publishing it.`,
|
||||
);
|
||||
}
|
||||
}
|
||||
const tarball = isRecord(existing.dist) ? String(existing.dist.tarball) : "";
|
||||
if (!isUnderRegistry(tarball, registry)) {
|
||||
throw new PublicationError(
|
||||
"VERIFICATION_FAILED",
|
||||
`${name}@${version}'s tarball is not on the configured registry.`,
|
||||
);
|
||||
}
|
||||
const downloaded = await reader.download(tarball);
|
||||
if (!downloaded.equals(local.data)) {
|
||||
throw new PublicationError(
|
||||
"BURNED_VERSION",
|
||||
`${name}@${version} is already on the registry with different bytes. It is never deleted or replaced: the version is burned, publication stops here, and the train needs a new version after an investigation.`,
|
||||
);
|
||||
}
|
||||
const problems = registryCopyProblems(local, existing, downloaded, registry);
|
||||
if (problems.length > 0) {
|
||||
throw new PublicationError(
|
||||
"VERIFICATION_FAILED",
|
||||
`${name}@${version} does not match the release artifact.`,
|
||||
problems,
|
||||
);
|
||||
}
|
||||
const publicRead = await reader
|
||||
.download(tarball, true)
|
||||
.then((anonymous) => anonymous.equals(local.data))
|
||||
.catch(() => false);
|
||||
if (options.requirePublicRead === true && !publicRead) {
|
||||
throw new PublicationError(
|
||||
"VERIFICATION_FAILED",
|
||||
`${name}@${version} cannot be downloaded anonymously.`,
|
||||
);
|
||||
}
|
||||
log(`${name}@${version}: ${outcome}, verified byte for byte`);
|
||||
verified.push({
|
||||
downloaded: artifactDigests(downloaded),
|
||||
local,
|
||||
outcome,
|
||||
publicRead,
|
||||
tarball,
|
||||
});
|
||||
}
|
||||
|
||||
// Every package is there and verified: only now does the final tag move.
|
||||
const tagOutcomes = new Map<string, TagOutcome>();
|
||||
for (const [index, { local }] of verified.entries()) {
|
||||
const { name, version } = local.entry;
|
||||
const current = distTagsOf(await reader.packument(name))[finalTag];
|
||||
if (current === version) {
|
||||
tagOutcomes.set(name, "already-current");
|
||||
} else if (current !== undefined && compareSemver(current, version) > 0) {
|
||||
tagOutcomes.set(name, "kept-newer");
|
||||
} else {
|
||||
await options.hooks?.beforeTag?.(name, index);
|
||||
await npm.addTag(name, version, finalTag);
|
||||
tagOutcomes.set(name, "moved");
|
||||
}
|
||||
log(`${name}: ${finalTag} ${tagOutcomes.get(name)}`);
|
||||
}
|
||||
|
||||
const packages: PublishedPackage[] = [];
|
||||
const unconverged: string[] = [];
|
||||
for (const { downloaded, local, outcome, publicRead, tarball } of verified) {
|
||||
const { name, version } = local.entry;
|
||||
const distTags = distTagsOf(await reader.packument(name));
|
||||
const finalVersion = distTags[finalTag];
|
||||
const converged =
|
||||
finalVersion === version ||
|
||||
(tagOutcomes.get(name) === "kept-newer" &&
|
||||
finalVersion !== undefined &&
|
||||
compareSemver(finalVersion, version) > 0);
|
||||
if (!converged) unconverged.push(`${name}: ${finalTag} is ${finalVersion ?? "unset"}`);
|
||||
packages.push({
|
||||
artifact: {
|
||||
integrity: local.entry.integrity,
|
||||
sha256: local.entry.sha256,
|
||||
size: local.entry.size,
|
||||
},
|
||||
distTags: Object.fromEntries(Object.entries(distTags).sort()),
|
||||
file: local.entry.file,
|
||||
finalTag: tagOutcomes.get(name) ?? "moved",
|
||||
name,
|
||||
outcome,
|
||||
registry: { ...downloaded, publicRead, tarball },
|
||||
version,
|
||||
});
|
||||
}
|
||||
if (unconverged.length > 0) {
|
||||
throw new PublicationError(
|
||||
"TAGS_NOT_CONVERGED",
|
||||
"The final dist-tags did not converge.",
|
||||
unconverged,
|
||||
);
|
||||
}
|
||||
return {
|
||||
completedAt: (options.now ?? (() => new Date()))().toISOString(),
|
||||
finalTag,
|
||||
packageTrain: {
|
||||
manifestSha256: train.manifestSha256,
|
||||
revision: train.manifest.source.revision,
|
||||
version: train.manifest.version,
|
||||
},
|
||||
packages,
|
||||
publicationRecord: PUBLICATION_RECORD_FORMAT,
|
||||
registry,
|
||||
};
|
||||
} finally {
|
||||
await npm.dispose();
|
||||
}
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------------------------
|
||||
// Command line
|
||||
// ---------------------------------------------------------------------------------------------
|
||||
|
||||
const VALUE_OPTIONS = Object.freeze({
|
||||
"--final-tag": "finalTag",
|
||||
"--manifest-sha256": "manifestSha256",
|
||||
"--record": "record",
|
||||
"--registry": "registry",
|
||||
"--release": "release",
|
||||
"--revision": "revision",
|
||||
"--staging-tag": "stagingTag",
|
||||
"--version": "version",
|
||||
} as const);
|
||||
const FLAG_OPTIONS = Object.freeze({
|
||||
"--allow-http": "allowHttp",
|
||||
"--require-public-read": "requirePublicRead",
|
||||
} as const);
|
||||
|
||||
const USAGE = `Usage: ${PUBLISH_TOKEN_VARIABLE}=… node scripts/publish-package-train.ts --release <directory> --registry <url> --version <version> [--revision <commit>] [--manifest-sha256 <hex>] [--record <file>] [--staging-tag <tag>] [--final-tag <tag>] [--require-public-read]`;
|
||||
|
||||
export function parsePublishArguments(argv: readonly string[]): {
|
||||
values: Partial<Record<(typeof VALUE_OPTIONS)[keyof typeof VALUE_OPTIONS], string>>;
|
||||
flags: Partial<Record<(typeof FLAG_OPTIONS)[keyof typeof FLAG_OPTIONS], true>>;
|
||||
} {
|
||||
const values: Partial<Record<(typeof VALUE_OPTIONS)[keyof typeof VALUE_OPTIONS], string>> = {};
|
||||
const flags: Partial<Record<(typeof FLAG_OPTIONS)[keyof typeof FLAG_OPTIONS], true>> = {};
|
||||
for (let index = 0; index < argv.length; index += 1) {
|
||||
const argument = argv[index] ?? "";
|
||||
if (Object.hasOwn(FLAG_OPTIONS, argument)) {
|
||||
flags[FLAG_OPTIONS[argument as keyof typeof FLAG_OPTIONS]] = true;
|
||||
continue;
|
||||
}
|
||||
const key = Object.hasOwn(VALUE_OPTIONS, argument)
|
||||
? VALUE_OPTIONS[argument as keyof typeof VALUE_OPTIONS]
|
||||
: undefined;
|
||||
const value = argv[index + 1];
|
||||
if (key === undefined || value === undefined || value.startsWith("--") || key in values) {
|
||||
throw new PublicationError("INVALID_OPTIONS", USAGE);
|
||||
}
|
||||
values[key] = value;
|
||||
index += 1;
|
||||
}
|
||||
if (!values.release || !values.registry || !values.version) {
|
||||
throw new PublicationError("INVALID_OPTIONS", USAGE);
|
||||
}
|
||||
return { flags, values };
|
||||
}
|
||||
|
||||
async function main(argv: readonly string[]): Promise<number> {
|
||||
try {
|
||||
const { flags, values } = parsePublishArguments(argv);
|
||||
const record = await publishPackageTrain({
|
||||
allowHttp: flags.allowHttp === true,
|
||||
log: (line) => process.stdout.write(`${line}\n`),
|
||||
registry: values.registry ?? "",
|
||||
releaseDirectory: values.release ?? "",
|
||||
requirePublicRead: flags.requirePublicRead === true,
|
||||
token: process.env[PUBLISH_TOKEN_VARIABLE] ?? "",
|
||||
version: values.version ?? "",
|
||||
...(values.revision === undefined ? {} : { revision: values.revision }),
|
||||
...(values.manifestSha256 === undefined ? {} : { manifestSha256: values.manifestSha256 }),
|
||||
...(values.stagingTag === undefined ? {} : { stagingTag: values.stagingTag }),
|
||||
...(values.finalTag === undefined ? {} : { finalTag: values.finalTag }),
|
||||
});
|
||||
if (values.record !== undefined) await writeFile(values.record, serializeRecord(record));
|
||||
process.stdout.write(
|
||||
`Published package train ${record.packageTrain.version}: ${record.packages.length} packages verified, ${record.finalTag} converged.\n`,
|
||||
);
|
||||
return 0;
|
||||
} catch (error) {
|
||||
if (error instanceof PublicationError) {
|
||||
process.stderr.write(`${error.message}\n`);
|
||||
for (const problem of error.problems) process.stderr.write(` ${problem}\n`);
|
||||
if (error.code === "INVALID_OPTIONS") return 64;
|
||||
return error.code === "BURNED_VERSION" ? 3 : 1;
|
||||
}
|
||||
process.stderr.write("Publishing the package train failed unexpectedly.\n");
|
||||
return 1;
|
||||
}
|
||||
}
|
||||
|
||||
if (process.argv[1] && import.meta.url === pathToFileURL(process.argv[1]).href) {
|
||||
process.exitCode = await main(process.argv.slice(2));
|
||||
}
|
||||
Reference in New Issue
Block a user