diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 91ff490..7469df8 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -132,6 +132,11 @@ jobs: needs: verify runs-on: ubuntu-latest timeout-minutes: 15 + # What the gate proved, for the job that consumes its artifact to check it received exactly that. + outputs: + version: ${{ steps.release.outputs.version }} + revision: ${{ steps.release.outputs.revision }} + manifest_sha256: ${{ steps.release.outputs.manifest_sha256 }} steps: - name: Check out repository uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 @@ -160,7 +165,91 @@ jobs: # packages, and verifies. Nothing is published and no credential exists here. A future publish # job must depend on this job and publish only the artifacts it proved. - name: Build, reproduce, and consume the public package train - run: pnpm test:package-release + run: pnpm test:package-release --keep package-train + + - name: Describe the proven release train + id: release + run: | + release="$(find package-train -mindepth 1 -maxdepth 1 -type d)" + manifest="$release/ekklesia-package-train.json" + { + echo "version=$(node -p 'JSON.parse(require("fs").readFileSync(process.argv[1], "utf8")).version' "$manifest")" + echo "revision=$(node -p 'JSON.parse(require("fs").readFileSync(process.argv[1], "utf8")).source.revision' "$manifest")" + echo "manifest_sha256=$(sha256sum "$manifest" | cut -d' ' -f1)" + } >>"$GITHUB_OUTPUT" + cat "$manifest" + + # The exact release artifacts, for the jobs that consume them, from a directory whose name does + # not start with a dot: the artifact actions leave hidden files out. Version 3 of the artifact + # actions: version 4 refuses any server that is not github.com, which this Gitea is not, while + # Gitea implements the protocol version 3 uses. A consumer checks the files, not the wrapper. + - name: Hand the release train on + uses: actions/upload-artifact@c6a3b2bd78b3985e4b2f15397fec357f0fd808de # v3.2.2-node20 + with: + name: package-train + path: package-train/ + if-no-files-found: error + retention-days: 7 + + # The publication tool, given exactly the artifact the package-release job proved, against a + # disposable Gitea 1.27.2 (scripts/test-package-publication.sh). Never the real registry, and no + # credential: the disposable registry's accounts are created and discarded inside the job. + package-publication: + needs: package-release + runs-on: ubuntu-latest + timeout-minutes: 15 + env: + # Names this job's stacks, so its final step removes exactly these (scripts/lib/ci-project.sh). + EKKLESIA_CI_JOB_KEY: package-publication + steps: + - name: Check out repository + uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + with: + persist-credentials: false + + # A job the runner killed (cancelled, or past the runner's own time limit) never ran its + # cleanup. Anything a CI job left on this shared daemon more than three hours ago goes first. + - name: Sweep stale CI stacks from the runner + run: bash scripts/ci-sweep.sh --stale + + - name: Set up Node.js + uses: actions/setup-node@48b55a011bda9f5d6aeb4c2d9c7362e8dae4041e # v6.4.0 + with: + node-version-file: .node-version + + - name: Activate pinned pnpm + run: | + corepack enable + corepack prepare pnpm@11.12.0 --activate + + - name: Install dependencies + run: pnpm install --frozen-lockfile + + - name: Receive the release train the gate proved + uses: actions/download-artifact@ad191675b41f6a5b46da9a048cb6893812da158b # v3.1.0-node20 + with: + name: package-train + path: package-train/ + + - name: Check it is exactly that train + env: + EXPECTED_MANIFEST_SHA256: ${{ needs.package-release.outputs.manifest_sha256 }} + run: | + release="$(find package-train -mindepth 1 -maxdepth 1 -type d)" + actual="$(sha256sum "$release/ekklesia-package-train.json" | cut -d' ' -f1)" + if [[ -z "$EXPECTED_MANIFEST_SHA256" || "$actual" != "$EXPECTED_MANIFEST_SHA256" ]]; then + echo "The received release manifest is not the one the package-release job proved." >&2 + exit 1 + fi + (cd "$release" && sha256sum --check --strict SHA256SUMS) + + - name: Publish it to a disposable Gitea, and prove the round trip and recovery + run: bash scripts/test-package-publication.sh --release "$(find package-train -mindepth 1 -maxdepth 1 -type d)" + + # Whatever happened above, this job's own stacks do not outlive it. + - name: Remove this job's CI stacks + if: always() + run: bash scripts/ci-sweep.sh --job image-build: needs: verify diff --git a/.github/workflows/publish-packages.yml b/.github/workflows/publish-packages.yml new file mode 100644 index 0000000..9b2563d --- /dev/null +++ b/.github/workflows/publish-packages.yml @@ -0,0 +1,133 @@ +name: Publish packages + +# Publishes one public @ekklesia/* package train (ADR 0053, docs/development/releases.md) to the +# npm registry: exactly the artifacts this run's package-release gate proves, and nothing else. +# +# Only ever started by hand, on `main`, by someone with write access to this repository: no pull +# request or push can start it. It needs the version-set commit of a release (`version` must equal +# the platform version there), and the repository secret EKKLESIA_PACKAGES_PUBLISH_TOKEN, a token of +# the dedicated publisher account with only the `write:package` scope. Packages are publicly +# readable (ADR 0054); publishing is what this protects. + +on: + workflow_dispatch: + inputs: + version: + description: The package train version to publish, equal to the platform version on main. + required: true + type: string + +permissions: + contents: read + +# Never cancel a publication part way, and never run two at once. A rerun resumes safely. +concurrency: + group: publish-packages + cancel-in-progress: false + +env: + # The only registry-specific setting: where the train is published. + EKKLESIA_PACKAGES_REGISTRY: https://git.riverofglorymn.org/api/packages/River_of_Glory_Church/npm/ + +jobs: + package-release: + if: github.ref == 'refs/heads/main' + runs-on: ubuntu-latest + timeout-minutes: 20 + outputs: + version: ${{ steps.release.outputs.version }} + revision: ${{ steps.release.outputs.revision }} + manifest_sha256: ${{ steps.release.outputs.manifest_sha256 }} + steps: + - name: Check out repository + uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + with: + persist-credentials: false + + - name: Set up Node.js + uses: actions/setup-node@48b55a011bda9f5d6aeb4c2d9c7362e8dae4041e # v6.4.0 + with: + node-version-file: .node-version + + - name: Activate pinned pnpm + run: | + corepack enable + corepack prepare pnpm@11.12.0 --activate + + - name: Install dependencies + run: pnpm install --frozen-lockfile + + # The same credential-free gate as CI: build, reproduce, and consume the train. + - name: Build, reproduce, and consume the public package train + run: pnpm test:package-release --keep package-train + + - name: Describe the proven release train + id: release + run: | + release="$(find package-train -mindepth 1 -maxdepth 1 -type d)" + manifest="$release/ekklesia-package-train.json" + { + echo "version=$(node -p 'JSON.parse(require("fs").readFileSync(process.argv[1], "utf8")).version' "$manifest")" + echo "revision=$(node -p 'JSON.parse(require("fs").readFileSync(process.argv[1], "utf8")).source.revision' "$manifest")" + echo "manifest_sha256=$(sha256sum "$manifest" | cut -d' ' -f1)" + } >>"$GITHUB_OUTPUT" + + # Artifact actions version 3: version 4 refuses any server that is not github.com. The directory + # name must not start with a dot: the artifact actions leave hidden files out. + - name: Hand the release train on + uses: actions/upload-artifact@c6a3b2bd78b3985e4b2f15397fec357f0fd808de # v3.2.2-node20 + with: + name: package-train + path: package-train/ + if-no-files-found: error + retention-days: 90 + + publish: + needs: package-release + if: github.ref == 'refs/heads/main' + runs-on: ubuntu-latest + timeout-minutes: 20 + steps: + # Only for the publication tool itself: nothing is built or installed in this job. + - name: Check out repository + uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + with: + persist-credentials: false + + - name: Set up Node.js + uses: actions/setup-node@48b55a011bda9f5d6aeb4c2d9c7362e8dae4041e # v6.4.0 + with: + node-version-file: .node-version + + - name: Receive the release train the gate proved + uses: actions/download-artifact@ad191675b41f6a5b46da9a048cb6893812da158b # v3.1.0-node20 + with: + name: package-train + path: package-train/ + + # The tool validates the whole train against its manifest, that manifest against the gate's, + # its version against the request, and its revision against this commit, before it sends + # anything; then publishes, verifies every package byte for byte, and moves `latest` last. + - name: Publish the exact release train + env: + EKKLESIA_PACKAGES_PUBLISH_TOKEN: ${{ secrets.EKKLESIA_PACKAGES_PUBLISH_TOKEN }} + TRAIN_VERSION: ${{ inputs.version }} + TRAIN_REVISION: ${{ github.sha }} + MANIFEST_SHA256: ${{ needs.package-release.outputs.manifest_sha256 }} + run: | + node scripts/publish-package-train.ts \ + --release "$(find package-train -mindepth 1 -maxdepth 1 -type d)" \ + --registry "$EKKLESIA_PACKAGES_REGISTRY" \ + --version "$TRAIN_VERSION" \ + --revision "$TRAIN_REVISION" \ + --manifest-sha256 "$MANIFEST_SHA256" \ + --record publication-record.json \ + --require-public-read + + - name: Keep the publication record + uses: actions/upload-artifact@c6a3b2bd78b3985e4b2f15397fec357f0fd808de # v3.2.2-node20 + with: + name: package-train-publication-record + path: publication-record.json + if-no-files-found: error + retention-days: 90 diff --git a/.gitignore b/.gitignore index be58ca3..cf6133f 100644 --- a/.gitignore +++ b/.gitignore @@ -26,3 +26,5 @@ packages/module-pack/.fixtures/ # `pnpm packages:release` writes package release trains here (scripts/package-release.ts). .package-release/ +# CI keeps the proven release here to hand it on as a workflow artifact (scripts/test-package-release.sh --keep). +/package-train/ diff --git a/CLAUDE.md b/CLAUDE.md index 0fb3cf8..43e2af8 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -81,6 +81,10 @@ ever disagree: - `pnpm test:package-release` — builds the public `@ekklesia/*` package train, proves it reproducible, and proves a standalone module consumes exactly those tarballs. Publishes nothing. Run it for any change to the public packages or their release tooling. +- `pnpm test:package-publication` — publishes a release train to a disposable Gitea (Docker) and + proves the publication tool. Never the real registry. The real publication is the manual + `Publish packages` workflow only; never run `scripts/publish-package-train.ts` against the real + registry without the user's explicit intent. - `pnpm doctor` — read-only local readiness report. Never treat it as a substitute for `pnpm verify` or the integration suite. - `pnpm migrations:plan` / `pnpm migrations:apply` — read-only planning vs. mutating execution diff --git a/README.md b/README.md index 4c19ac4..34a77cd 100644 --- a/README.md +++ b/README.md @@ -67,8 +67,11 @@ A module can be built in its own repository from Ekklesia's public module-develo a curated `@ekklesia/module-sdk`, `@ekklesia/module-database`, the UI packages, and the build and packaging tools. They are released as one coordinated train of tarballs with a release manifest, and CI proves on every run that two builds of a commit are byte-identical and that a module outside -the repository installs, builds, packages, and verifies from those artifacts alone. They are not -published to any registry yet +the repository installs, builds, packages, and verifies from those artifacts alone. A manual +workflow, the only way to publish, publishes exactly those artifacts to the npm registry and +verifies every package byte for byte. Anyone may read and install the packages before launch +([ADR 0054](docs/architecture/decisions/0054-public-package-reads-before-launch.md)). The first train +published will be the next platform release ([Releases and the public package train](docs/development/releases.md)). Phase 4A added Core-owned identity foundations: a Core-owned Better Auth database schema diff --git a/docs/architecture/decisions/0053-standalone-module-development-packages.md b/docs/architecture/decisions/0053-standalone-module-development-packages.md index bcc1b14..9122d3d 100644 --- a/docs/architecture/decisions/0053-standalone-module-development-packages.md +++ b/docs/architecture/decisions/0053-standalone-module-development-packages.md @@ -2,6 +2,8 @@ - Status: Accepted - Date: 2026-10-07 +- Superseded in part by: [0054](0054-public-package-reads-before-launch.md) — only where this ADR + describes the package phase before launch as private, or requires authenticated package reads. - Extends: [0049](0049-module-package-v1.md). This is the step ADR 0049 deferred: "publishing the Ekklesia SDK packages for module builds outside a platform workspace". A registry is used only to build a module. Installing one stays registry-free: an installed package is self-contained, as ADR diff --git a/docs/architecture/decisions/0054-public-package-reads-before-launch.md b/docs/architecture/decisions/0054-public-package-reads-before-launch.md new file mode 100644 index 0000000..fc23a58 --- /dev/null +++ b/docs/architecture/decisions/0054-public-package-reads-before-launch.md @@ -0,0 +1,85 @@ +# 0054: Ekklesia's module-development packages are publicly readable before launch + +- Status: Accepted +- Date: 2026-10-08 +- Supersedes (partial): [0053](0053-standalone-module-development-packages.md) — only where it + describes the package phase before launch as private: "Gitea may host the private phase", the + read credential for module CI in "Registry access and the private phase", and its requirement + that packages not be served anonymously while Ekklesia is not public. Every other part of that ADR + remains accepted and unchanged: the npm registry protocol, the public package set and its + boundaries, the curated SDK, the dependency rules, the coordinated package train, exact release + artifacts, standalone module builds, provider portability, supply chain and immutability, and the + credential handling for publishing. +- Related: [ADR 0049](0049-module-package-v1.md) + +## Context + +ADR 0053 decided how Ekklesia's module-development packages (`@ekklesia/*`) are built, released, and +distributed over the npm registry protocol. It assumed that the phase before Ekklesia's public +launch would be private: module CI would carry a read credential where reads needed one, and the +first publication had to make sure the registry would not serve the packages anonymously. + +Checking the chosen host before the first publication showed what keeping the packages private would +take: + +- Gitea 1.27.2 decides package read access by the package **owner's** visibility, never a + repository's. The `River_of_Glory_Church` owner is a public organization, so its packages are + readable anonymously. +- That organization's repositories, Ekklesia's source among them, are already publicly readable. +- The instance allows self-registration, so "limited" visibility, readable by any signed-in user, + is readable by anyone who creates an account. +- Private packages would therefore need a separate private package owner, plus reader accounts and + read tokens for every module repository, every module's CI, and every developer. + +The project owner has decided that Ekklesia is not secret. Its source is public; the project is +simply not being launched or promoted yet. Hiding the packages built from that public source would +protect nothing, and would still cost every module author an account and a token. + +## Decision + +1. **Ekklesia's module-development packages may be anonymously readable before the project's public + launch.** This covers the public package set of ADR 0053: search, metadata, and download. +2. **Readability is not a launch, and promises nothing more.** Public readability does not mean: + - that Ekklesia has been publicly launched; + - that the APIs are stable beyond the project's pre-1.0 SemVer policy; + - that third-party production use is supported; + - that a marketplace or public module ecosystem exists. +3. **Reading needs no authentication in this phase.** Installing `@ekklesia/*` needs no account, + reader account, read token, or authenticated install workflow. A module repository configures only + the scope's registry. +4. **Publication stays authenticated and tightly controlled.** Only Ekklesia's release process + publishes, with one dedicated publisher credential that holds nothing beyond package write + access, used only by that process, as ADR 0053's credential handling for publishing already + requires. +5. **Published versions stay immutable by Ekklesia policy.** A published version is never deleted or + republished with other bytes, whatever the registry technically allows. +6. **The current Gitea owner is an acceptable host.** The `River_of_Glory_Church` package namespace + on the current Gitea may host this phase, as long as publishing permission stays restricted to + the publisher. No private package organization is needed. +7. **A public launch is a separate decision.** Launching or promoting Ekklesia publicly, and anything + that changes for its packages then, is a product and release decision this ADR does not make. + +## Consequences + +- **Module authors and module CI need no Ekklesia credential.** The committed `.npmrc` scope mapping + is the whole registry configuration. ADR 0053's example of a read credential in user or CI + configuration is no longer needed in this phase. +- **ADR 0053's pre-publication access check changes meaning.** It now confirms that anonymous reads + work as intended and that publishing is refused without the publisher's permission, rather than + that reads are private. The disposable-registry suite proves both, and the publication tool can + require an anonymous download to match each published artifact. +- **Restricting publishing is now the whole access boundary**, so the publisher credential's scope, + where it is held, and who can reach it matter all the more. +- **Making the packages private again would be a new decision.** Package names, versions, and + contents already published stay public, because a published version is never deleted. + +## Alternatives considered + +- **A private package organization on the same Gitea.** Rejected: it hides nothing that is not + already public in the source, and it requires reader accounts and tokens for every module + repository, its CI, and every developer. +- **"Limited" package visibility.** Rejected: with self-registration open, anyone can read anyway, + while the configuration suggests otherwise. +- **Requiring sign-in for the whole instance.** Rejected: it would also hide the public + repositories. +- **Another registry for a private phase.** Unnecessary once reads are public. diff --git a/docs/architecture/decisions/README.md b/docs/architecture/decisions/README.md index a0d8c8e..0efa5b9 100644 --- a/docs/architecture/decisions/README.md +++ b/docs/architecture/decisions/README.md @@ -62,3 +62,4 @@ ADRs capture durable choices that affect more than one package or phase. - [0051: Module packages declare their own license and carry its text](0051-module-package-license-material.md) — Accepted - [0052: Reviewed license supplements cover redistributed dependencies whose packages omit required material](0052-reviewed-license-supplements.md) — Proposed - [0053: A module builds outside the monorepo, from Ekklesia packages published over the npm registry protocol](0053-standalone-module-development-packages.md) — Accepted +- [0054: Ekklesia's module-development packages are publicly readable before launch](0054-public-package-reads-before-launch.md) — Accepted diff --git a/docs/development/releases.md b/docs/development/releases.md index 5b1e6b1..968dc31 100644 --- a/docs/development/releases.md +++ b/docs/development/releases.md @@ -4,19 +4,22 @@ An Ekklesia platform release is one version. The platform image, the private wor and the public `@ekklesia/*` module-development packages all carry it. This page covers the order a release happens in and the public package train ([ADR 0053](../architecture/decisions/0053-standalone-module-development-packages.md)): how its -artifacts are produced, what proves them, and what a future publication step will do with them. +artifacts are produced, what proves them, and how exactly those artifacts are published. -> **Registry publication is not implemented.** Nothing in this repository publishes a package, and -> no CI job holds a registry credential. The release artifacts below are produced and proven on -> every CI run. A future publication step publishes **exactly these artifacts**, unchanged, and -> nothing else. +> **The packages are publicly readable, on purpose** +> ([ADR 0054](../architecture/decisions/0054-public-package-reads-before-launch.md)). Anyone can +> search, read, and install `@ekklesia/*` from the registry with no account and no token, like any +> public npm package. That is not a public launch: Ekklesia's source is already publicly readable, +> and the project is simply not promoted yet. Only **publishing** is restricted, to one manual, +> authenticated workflow. Before 1.0, the APIs may change between versions under the project's +> SemVer policy: a 0.x minor version may break them. ## Where each step sits ```text version set ─▶ build ─▶ test ─▶ pack the public package train ─▶ standalone consumer proof │ - tag and release (maintainer) ◀─ [future: publish] ◀──┘ + tag and release (maintainer) ◀─ publish the exact train (manual) ◀───┘ ``` 1. **Version set.** A `chore(release): prepare X` pull request sets the release version: the root @@ -28,14 +31,15 @@ version set ─▶ build ─▶ test ─▶ pack the public package train ─▶ 3. **Pack the public package train.** `pnpm packages:release` (below). 4. **Standalone consumer proof.** A module outside the repository consumes exactly those artifacts (below). -5. **[Future] publish.** Not implemented. Once it is, it must run only after steps 3 and 4 passed - for the same commit. It must check every artifact against the release manifest before - publishing it, and publish nothing else. +5. **Publish the exact train.** By hand, with the `Publish packages` workflow on the version-set + commit on `main` ([Publishing the train](#publishing-the-train)). It publishes the artifacts its + own run's gate (steps 3 and 4) proved, and nothing else. 6. **Tag and release.** Unchanged: the maintainer tags the merge commit of the version-set pull request. Nothing here creates, moves, or requires a tag. -Steps 2 to 4 run in CI on every pull request and on `main`. The `package-release` job is the gate a -future publication job must depend on. +Steps 2 to 4 run in CI on every pull request and on `main`, and CI also publishes each proven train +to a disposable registry (`package-publication`), never the real one. Step 5 runs only when a +maintainer starts it. ## The public package train @@ -192,3 +196,166 @@ Two builds of the same commit produce byte-identical tarballs, and CI proves it `packageManager`. Another Node.js or zlib build may compress differently. Each artifact's integrity is recorded either way, and that, not byte identity, is what a publication step must verify. + +## Publishing the train + +The registry is the Gitea npm registry under the `River_of_Glory_Church` owner: + +```text +https://git.riverofglorymn.org/api/packages/River_of_Glory_Church/npm/ +``` + +| Who | May | +| --- | --- | +| Anyone, signed in or not | search, read metadata, download, and install `@ekklesia/*` | +| The publisher account, through the publish workflow | publish new versions and move dist-tags | +| Anyone else | nothing more: publishing needs the publisher team's package write permission | + +### The workflow + +`.github/workflows/publish-packages.yml` (Actions → Publish packages → Run workflow, on `main`, +with the release version). Nothing else can start it: no pull request or push does, and CI holds no +registry credential at all. + +1. **The gate.** `package-release` runs the same credential-free gate as CI (build, reproduce, + consume) and hands the proven release on as a workflow artifact, with its manifest's SHA-256. +2. **The publish job.** It builds and installs nothing. It downloads the artifact and runs + `scripts/publish-package-train.ts` with the secret `EKKLESIA_PACKAGES_PUBLISH_TOKEN`, which only + this step receives. +3. **The publication record** is kept as an artifact of the run. + +### What the publication tool does + +`scripts/publish-package-train.ts` publishes exactly the `.tgz` files in the release directory. It +never builds, packs, or edits a package, and it never deletes, unpublishes, or overwrites anything. + +1. **Validates the whole train before sending anything.** The manifest must be the one the gate + proved (`--manifest-sha256`), for the requested version and this commit (`--revision`), from a + clean tree (`"modified": false`), with exactly the nine expected packages. Every artifact's file + name, size, SHA-256, integrity, `SHA256SUMS` line, and the `package.json` inside it must match. +2. **Publishes in dependency order.** Each package comes after the train packages it depends on, + then alphabetically. For each package: + - **absent:** `npm publish ` under the staging dist-tag `ekklesia-staging`. npm sends the + file's own bytes and runs no script for a file; the registry checks them against their + integrity and stores them as they are; + - **present with the same bytes:** accepted as already published, and verified; + - **present with other bytes:** a hard stop. Nothing is deleted or overwritten, nothing after it + is published, and the version is burned ([Immutability](#immutability)). +3. **Reads every package back.** The registry's metadata must name the package and version, report + the artifact's integrity, and list the same `dependencies`, `peerDependencies`, + `optionalDependencies`, and `bin` as the tarball's `package.json` (npm writes `bin` paths without + a leading `./`). Its tarball URL must be on the configured registry, and the tarball it serves must + be the release artifact, byte for byte. With `--require-public-read`, an anonymous download must + give the same bytes too. A package is published only once all of that holds. +4. **Moves `latest` last.** Only when all nine are verified does `latest` move to the train, never + backwards past a newer version. The train is published when every package's `latest` agrees. +5. **Writes the publication record** (`--record`). + +A rerun is always safe. It publishes only what is missing, accepts what is already there and +identical, and moves only the tags that are not yet right. A train interrupted after five packages, +or after four tags, finishes on the next run. No retry ever deletes anything. + +### The publication record + +Evidence that one train was published, not a second package catalog, and never a secret: + +```json +{ + "publicationRecord": 1, + "packageTrain": { "version": "0.2.0", "revision": "", "manifestSha256": "" }, + "registry": "https://git.riverofglorymn.org/api/packages/River_of_Glory_Church/npm/", + "finalTag": "latest", + "completedAt": "", + "packages": [ + { + "name": "@ekklesia/contracts", + "version": "0.2.0", + "file": "ekklesia-contracts-0.2.0.tgz", + "outcome": "published", + "artifact": { "sha256": "", "integrity": "sha512-…", "size": 87379 }, + "registry": { "tarball": "", "sha256": "", "integrity": "sha512-…", "publicRead": true }, + "finalTag": "moved", + "distTags": { "ekklesia-staging": "0.2.0", "latest": "0.2.0" } + } + ] +} +``` + +`outcome` is `published` or `already-present`. `finalTag` is `moved`, `already-current`, or +`kept-newer` (a newer version already holds `latest`). + +### Credentials + +- **One credential:** `EKKLESIA_PACKAGES_PUBLISH_TOKEN`, a repository Actions secret, used only by + the publish step. Nobody needs a token to read or install. +- **The publisher** is a dedicated account, member only of a `River_of_Glory_Church` team whose one + permission is Packages: Write. Its token has only the `write:package` scope. Gitea's + `write:package` also allows deleting packages; the tool never does, and nothing else uses the + token. +- **No token in git or in an `.npmrc`.** The tool writes a temporary npm configuration that refers to + the variable (`_authToken=${EKKLESIA_PACKAGES_PUBLISH_TOKEN}`), ignores every user, global, and + project npm configuration, sends the token only to the registry itself, and removes the + configuration afterwards. +- **What remains trusted:** Gitea gives repository secrets to workflows run for pull requests from + branches of this same repository, so anyone who can push a branch here could read the secret + through a changed workflow. Keep write access to this repository to the maintainers. + +### Immutability + +Gitea refuses to upload a version that exists (409). But it lets anyone with package write +permission delete a version, deletion leaves nothing behind, and an owner's package cleanup rules +can delete versions on a schedule. So immutability is Ekklesia's policy, not the registry's +guarantee: + +- **A published version is never deleted,** by anyone, for any reason short of a security incident. +- **A version number is never reused with other bytes.** A version found on the registry with other + bytes is burned: investigate how it got there, and publish the train under a new version. +- **No package cleanup rule** may be configured for the `River_of_Glory_Church` owner. +- **The publisher never deletes.** The tool has no delete, unpublish, or overwrite, and its tests + check that. + +Module lockfiles record each package's integrity, so a replaced package would fail every install of +it, rather than install silently. + +### Moving to another registry + +Only configuration changes: the registry URL (the workflow's `EKKLESIA_PACKAGES_REGISTRY`), the +publisher credential, and modules' `@ekklesia:registry`. Modules then regenerate their lockfiles, +because Gitea's tarball URLs are its own. Package sources, the train, its manifests, and every +artifact already built stay as they are; the same bytes keep the same integrity. + +### Proving the publication machinery + +`pnpm test:package-publication` (`scripts/test-package-publication.sh`, the CI `package-publication` +job) publishes a real release train to a disposable Gitea 1.27.2 configured like the real one: +public owners, a publisher team, and an outsider. In CI it publishes the very artifact the +`package-release` job handed on, after checking its manifest hash. It proves: + +- anonymous search, metadata, and download work; +- publishing without credentials, or by a user outside the publisher team, fails; +- the round trip is byte-identical, also anonymously; +- the registry refuses a second upload of a published version; +- a train interrupted before its sixth package resumes, verifying the five and publishing the rest, + with `latest` untouched until all nine are verified; +- a version already published with other bytes stops the train, deleting and overwriting nothing, + and publishing nothing after it, on every retry; +- a train interrupted while moving `latest` converges on a rerun, moving only the missing tags. + +### The first publication + +The first train published is the **next** platform release version, not 0.1.2. The `v0.1.2` tag +predates the curated SDK and this release tooling, so packages built now and called 0.1.2 would +claim to be source that 0.1.2 never contained. + +Before it, an administrator of the Gitea instance: + +1. creates the dedicated publisher account; +2. creates a `River_of_Glory_Church` team, for example `package-publishers`, with only the Packages + unit at Write, and adds the account to it; +3. creates a token for the account with only `write:package`, and stores it as the repository + Actions secret `EKKLESIA_PACKAGES_PUBLISH_TOKEN`; +4. checks that the owner has no package cleanup rule. + +Then the release proceeds as usual: the version-set pull request merges, a maintainer starts +Publish packages on `main` with that version, and tags the commit once the train is published. + diff --git a/docs/development/testing.md b/docs/development/testing.md index f20f4f6..ffcb303 100644 --- a/docs/development/testing.md +++ b/docs/development/testing.md @@ -660,6 +660,13 @@ versions, manifest inspection, release directory, manifest bytes, the pack hook) `tests/public-packages.test.ts`, part of `pnpm verify`. See [Releases and the public package train](releases.md). +`pnpm test:package-publication` (the CI `package-publication` job, given the `package-release` job's +artifact) publishes a release train to a disposable Gitea 1.27.2 with Docker, never the real +registry, and proves the publication tool: byte-identical round trips, anonymous reads, refused +unauthorized publishing, partial-train recovery, the hard stop on a burned version, and dist-tag +convergence (`tests/package-publication.integration.ts`). The tool's own rules are unit-tested in +`tests/package-publication.test.ts`, part of `pnpm verify`. + ## The generic platform image and the reference deployment These need Docker and build images locally; nothing is pushed or published. diff --git a/docs/module-development/README.md b/docs/module-development/README.md index 823ba5b..740b94d 100644 --- a/docs/module-development/README.md +++ b/docs/module-development/README.md @@ -73,16 +73,37 @@ entrypoints. A module in its own repository depends on the published packages in ### Developing a module in its own repository -> **Status:** the public packages are ready for a standalone module, and a release check proves it -> from locally packed tarballs. They are **not published yet**, and there is no registry URL to use -> yet: registry publication is the next slice -> ([ADR 0053](../architecture/decisions/0053-standalone-module-development-packages.md)). This -> section describes the workflow that publication enables. +> **Status:** the packages are published by release +> ([ADR 0053](../architecture/decisions/0053-standalone-module-development-packages.md), +> [Releases and the public package train](../development/releases.md)). The first published train +> is the next platform release after 0.1.2. A module does not need an Ekklesia checkout. Do not clone Ekklesia and put your module in its `modules/` directory: your module is its own repository, and it depends on Ekklesia's module-development packages like any other npm packages. +The packages are **publicly readable** +([ADR 0054](../architecture/decisions/0054-public-package-reads-before-launch.md)): installing them +needs no account, no token, and no access to Ekklesia's Gitea. That is not a public launch, and it +promises no API stability beyond the pre-1.0 SemVer policy. Only Ekklesia's release workflow can publish them. Point the `@ekklesia` scope at +the registry in your repository's committed `.npmrc`; that is all the configuration there is: + +```ini +@ekklesia:registry=https://git.riverofglorymn.org/api/packages/River_of_Glory_Church/npm/ +``` + +Then the module builds and packages like any other: + +```bash +pnpm install +pnpm build +pnpm exec ekklesia-pack-module --module . --target any --out ../packages \ + --license-file ./LICENSE --source-revision "$(git rev-parse HEAD)" +``` + +Before 1.0, a minor version may change these APIs (the project's SemVer policy). Pin the train you +build against with ranges in `package.json` and exact versions in your lockfile. + | Package | Where it goes | What it is | | --- | --- | --- | | `@ekklesia/module-sdk` | `dependencies` | `defineModuleRuntime`, routes, configuration, storage, realtime, principals | @@ -148,6 +169,9 @@ import it yourself, because what it exports is the platform's, not a supported a allowBuilds: esbuild: true ``` +- **Lockfiles name the registry.** pnpm records Gitea's tarball URLs in your lockfile. If the + packages ever move to another registry, change `@ekklesia:registry` and regenerate the lockfile; + nothing else changes, and the integrity of every package stays the same. - **TypeScript.** Extend the shared configurations by package name, for example `"extends": "@ekklesia/typescript-config/node.json"` for the backend and `@ekklesia/typescript-config/browser-react.json` for the UI. @@ -160,8 +184,8 @@ credited as third-party software, wherever they came from. Ekklesia proves this on every CI run: `pnpm test:package-release` builds the public packages of a release as one train of tarballs, proves two builds identical, and installs exactly those tarballs into a module outside the repository (`tests/fixtures/standalone-module`), with no workspace link -and no registry, then typechecks, builds, packages, and verifies it. Those artifacts are what a -future publication step publishes ([Releases and the public package train](../development/releases.md)). +and no registry, then typechecks, builds, packages, and verifies it. Those exact artifacts are what +the release publishes ([Releases and the public package train](../development/releases.md)). ### An installed module is self-contained diff --git a/package.json b/package.json index 7ea987a..bb70c0b 100644 --- a/package.json +++ b/package.json @@ -43,6 +43,7 @@ "test:migrations:integration": "pnpm build && bash scripts/test-migrations-integration.sh", "test:root": "vitest run --config vitest.config.ts", "test:package-release": "bash scripts/test-package-release.sh", + "test:package-publication": "bash scripts/test-package-publication.sh", "test:watch": "vitest", "typecheck": "turbo run typecheck typecheck:root", "typecheck:root": "tsc --noEmit --project tsconfig.json", diff --git a/scripts/lib/package-registry.compose.yaml b/scripts/lib/package-registry.compose.yaml new file mode 100644 index 0000000..8cfb409 --- /dev/null +++ b/scripts/lib/package-registry.compose.yaml @@ -0,0 +1,25 @@ +# A disposable Gitea, the same version as the real one, whose npm registry the package publication +# tests run against (scripts/test-package-publication.sh). Never the real registry: it starts empty, +# keeps everything in the container, and is removed with its volumes when the test ends. +services: + gitea: + image: docker.gitea.com/gitea:1.27.2@sha256:d20286ca2b2e170fdf628e7231b8a31a3220ade39ff462b55041d43d1fc757dd + environment: + GITEA__security__INSTALL_LOCK: "true" + GITEA__database__DB_TYPE: sqlite3 + # The URL clients reach it by: package metadata names tarballs under it. + GITEA__server__ROOT_URL: ${GITEA_ROOT_URL:?GITEA_ROOT_URL is required} + GITEA__server__HTTP_PORT: "3000" + GITEA__server__OFFLINE_MODE: "true" + GITEA__service__DISABLE_REGISTRATION: "true" + GITEA__packages__ENABLED: "true" + GITEA__actions__ENABLED: "false" + GITEA__mailer__ENABLED: "false" + GITEA__log__LEVEL: warn + ports: + - "127.0.0.1:${GITEA_PORT:?GITEA_PORT is required}:3000" + healthcheck: + test: ["CMD", "curl", "-fsS", "http://127.0.0.1:3000/api/healthz"] + interval: 2s + timeout: 3s + retries: 60 diff --git a/scripts/publish-package-train.ts b/scripts/publish-package-train.ts new file mode 100644 index 0000000..4711ff4 --- /dev/null +++ b/scripts/publish-package-train.ts @@ -0,0 +1,889 @@ +import { execFile } from "node:child_process"; +import { createHash } from "node:crypto"; +import { mkdtemp, readdir, readFile, rm, writeFile } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import path from "node:path"; +import process from "node:process"; +import { pathToFileURL } from "node:url"; +import { promisify } from "node:util"; +import { gunzipSync } from "node:zlib"; +import { + artifactDigests, + PACKAGE_TRAIN_FORMAT, + PACKAGE_TRAIN_MANIFEST, + type PackageTrainManifest, + releaseDirectoryProblems, + type TrainPackage, +} from "./package-release.ts"; + +/** + * Publishes one package release train (ADR 0053) to an npm-protocol registry: exactly the `.tgz` + * files `pnpm packages:release` produced, as `ekklesia-package-train.json` records them, and nothing + * else. It never builds, packs, or edits a package, and never deletes or unpublishes anything. + * + * EKKLESIA_PACKAGES_PUBLISH_TOKEN=… node scripts/publish-package-train.ts \ + * --release --registry --version \ + * [--revision ] [--manifest-sha256 ] [--record ] \ + * [--staging-tag ] [--final-tag ] [--require-public-read] + * + * 1. The whole local train is validated before anything is sent: the manifest, every artifact's + * name, size, SHA-256, integrity, and the package.json inside it. + * 2. Packages go out in dependency order. For each: absent from the registry, it is published with + * `npm publish ` under a staging dist-tag; present with the same bytes, it is accepted + * as already published; present with other bytes, publication stops for good (the version is + * burned). Every package is then read back: its metadata, and its tarball, which must be the + * release artifact byte for byte. + * 3. Only when all of them are verified does the final dist-tag (`latest`) move, and the train is + * published once every tag agrees. A rerun converges: it publishes only what is missing and moves + * only tags that are not yet right. + * + * The publish token is read from the environment only, and reaches npm through a temporary npm + * configuration that refers to the variable, never the value; the configuration is removed after. + */ + +const execFileAsync = promisify(execFile); + +export const PUBLISH_TOKEN_VARIABLE = "EKKLESIA_PACKAGES_PUBLISH_TOKEN"; +export const DEFAULT_STAGING_TAG = "ekklesia-staging"; +export const DEFAULT_FINAL_TAG = "latest"; +export const PUBLICATION_RECORD_FORMAT = 1; +const DEPENDENCY_FIELDS = Object.freeze([ + "dependencies", + "peerDependencies", + "optionalDependencies", +] as const); + +export type PublicationErrorCode = + | "INVALID_OPTIONS" + | "INVALID_RELEASE" + | "BURNED_VERSION" + | "REGISTRY_UNAVAILABLE" + | "PUBLISH_FAILED" + | "VERIFICATION_FAILED" + | "TAGS_NOT_CONVERGED"; + +export class PublicationError extends Error { + readonly code: PublicationErrorCode; + readonly problems: readonly string[]; + + constructor(code: PublicationErrorCode, message: string, problems: readonly string[] = []) { + super(message); + this.name = "PublicationError"; + this.code = code; + this.problems = problems; + } +} + +function isRecord(value: unknown): value is Readonly> { + return typeof value === "object" && value !== null && !Array.isArray(value); +} + +// --------------------------------------------------------------------------------------------- +// The local train +// --------------------------------------------------------------------------------------------- + +/** The `package/package.json` inside an npm tarball, read without unpacking anything to disk. */ +export function tarballPackageJson(tarball: Uint8Array): Record { + const data = gunzipSync(tarball); + for (let offset = 0; offset + 512 <= data.length; ) { + const header = data.subarray(offset, offset + 512); + if (header.every((byte) => byte === 0)) break; + const field = (start: number, length: number) => + header + .subarray(start, start + length) + .toString("utf8") + .replace(/\0.*$/su, ""); + const name = field(0, 100); + const prefix = header.subarray(257, 262).toString("latin1") === "ustar" ? field(345, 155) : ""; + const size = Number.parseInt(field(124, 12).trim() || "0", 8); + const type = field(156, 1); + const fullName = prefix ? `${prefix}/${name}` : name; + if ((type === "0" || type === "") && fullName === "package/package.json") { + const body = data.subarray(offset + 512, offset + 512 + size); + return JSON.parse(body.toString("utf8")) as Record; + } + offset += 512 + Math.ceil(size / 512) * 512; + } + throw new Error("The tarball holds no package/package.json."); +} + +export interface LocalPackage { + readonly entry: TrainPackage; + readonly path: string; + readonly data: Buffer; + readonly packageJson: Readonly>; +} + +export interface LocalTrain { + readonly manifest: PackageTrainManifest; + /** SHA-256 of `ekklesia-package-train.json`'s exact bytes. */ + readonly manifestSha256: string; + readonly packages: readonly LocalPackage[]; +} + +/** + * Reads and validates a release directory completely, or throws `INVALID_RELEASE` with every + * problem: nothing may be published from a train that is not whole and unchanged. + */ +export async function loadReleaseTrain( + releaseDirectory: string, + expected: { readonly version: string; readonly revision?: string }, +): Promise { + const files = new Map(); + for (const file of await readdir(releaseDirectory)) { + files.set(file, await readFile(path.join(releaseDirectory, file))); + } + const manifestBytes = files.get(PACKAGE_TRAIN_MANIFEST); + if (manifestBytes === undefined) { + throw new PublicationError("INVALID_RELEASE", `The release has no ${PACKAGE_TRAIN_MANIFEST}.`); + } + let manifest: PackageTrainManifest; + try { + manifest = JSON.parse(manifestBytes.toString("utf8")) as PackageTrainManifest; + } catch { + throw new PublicationError("INVALID_RELEASE", `${PACKAGE_TRAIN_MANIFEST} is not JSON.`); + } + const problems = trainProblems(manifest, files, expected); + if (problems.length > 0) { + throw new PublicationError("INVALID_RELEASE", "The release train failed validation.", problems); + } + return { + manifest, + manifestSha256: createHash("sha256").update(manifestBytes).digest("hex"), + packages: manifest.packages.map((entry) => { + const data = files.get(entry.file) as Buffer; + return { + data, + entry, + packageJson: tarballPackageJson(data), + path: path.join(releaseDirectory, entry.file), + }; + }), + }; +} + +/** Every reason a release directory's contents may not be published, or nothing. */ +export function trainProblems( + manifest: PackageTrainManifest, + files: ReadonlyMap, + expected: { readonly version: string; readonly revision?: string }, +): string[] { + if ( + !isRecord(manifest) || + manifest.packageTrain !== PACKAGE_TRAIN_FORMAT || + !Array.isArray(manifest.packages) || + !isRecord(manifest.source) + ) { + return ["The release manifest is not a package train manifest."]; + } + const problems = releaseDirectoryProblems(manifest, files); + if (manifest.source.modified !== false) { + problems.push("The release was built from a modified source tree; it is never published."); + } + if (manifest.version !== expected.version) { + problems.push( + `The release is version ${String(manifest.version)}, not the requested ${expected.version}.`, + ); + } + if (!/^[0-9a-f]{40}([0-9a-f]{24})?$/u.test(String(manifest.source.revision))) { + problems.push("The release names no full source revision."); + } + if (expected.revision !== undefined && manifest.source.revision !== expected.revision) { + problems.push( + `The release was built from ${String(manifest.source.revision)}, not the expected ${expected.revision}.`, + ); + } + const sums = files.get("SHA256SUMS"); + const expectedSums = [...manifest.packages] + .sort((left, right) => (left.file < right.file ? -1 : 1)) + .map((entry) => `${entry.sha256} ${entry.file}\n`) + .join(""); + if (sums === undefined || Buffer.from(sums).toString("utf8") !== expectedSums) { + problems.push("SHA256SUMS does not match the release manifest."); + } + for (const entry of manifest.packages) { + const data = files.get(entry.file); + if (data === undefined) continue; + try { + const packageJson = tarballPackageJson(data); + if (packageJson.name !== entry.name || packageJson.version !== entry.version) { + problems.push( + `${entry.file} holds ${String(packageJson.name)}@${String(packageJson.version)}, not ${entry.name}@${entry.version}.`, + ); + } + } catch { + problems.push(`${entry.file} is not a readable npm package tarball.`); + } + } + return problems; +} + +/** + * The order packages are published in: every package after the train packages it depends on (in + * `dependencies`, `peerDependencies`, or `optionalDependencies`), and otherwise by name, so the order + * never depends on how a directory happens to list its files. + */ +export function publicationOrder< + Package extends { + readonly entry: { readonly name: string }; + readonly packageJson: Readonly>; + }, +>(packages: readonly Package[]): Package[] { + const byName = new Map(packages.map((entry) => [entry.entry.name, entry])); + const dependsOn = new Map>(); + for (const entry of packages) { + const names = new Set(); + for (const field of DEPENDENCY_FIELDS) { + const dependencies = entry.packageJson[field]; + if (!isRecord(dependencies)) continue; + for (const name of Object.keys(dependencies)) { + if (byName.has(name) && name !== entry.entry.name) names.add(name); + } + } + dependsOn.set(entry.entry.name, names); + } + const order: Package[] = []; + const done = new Set(); + while (order.length < packages.length) { + const ready = [...byName.keys()] + .filter((name) => !done.has(name)) + .filter((name) => + [...(dependsOn.get(name) ?? [])].every((dependency) => done.has(dependency)), + ) + .sort(); + const next = ready[0]; + if (next === undefined) { + throw new PublicationError( + "INVALID_RELEASE", + "The release's packages depend on each other in a cycle.", + ); + } + done.add(next); + order.push(byName.get(next) as Package); + } + return order; +} + +// --------------------------------------------------------------------------------------------- +// Comparing a registry copy with the release +// --------------------------------------------------------------------------------------------- + +/** `bin` as npm publishes it: a string becomes `{ : path }`, and paths + * lose a leading `./`. The tarball keeps the author's form; the registry metadata gets this one. */ +export function normalizeBin(bin: unknown, packageName: string): Record { + const entries: [string, string][] = + typeof bin === "string" + ? [[packageName.replace(/^@[^/]+\//u, ""), bin]] + : isRecord(bin) + ? Object.entries(bin).map(([name, target]) => [name, String(target)]) + : []; + return Object.fromEntries( + entries + .map(([name, target]): [string, string] => [name, target.replace(/^(?:\.\/)+/u, "")]) + .sort(([left], [right]) => (left < right ? -1 : 1)), + ); +} + +function sortedRecord(value: unknown): Record { + if (!isRecord(value)) return {}; + return Object.fromEntries( + Object.keys(value) + .sort() + .map((key) => [key, value[key]]), + ); +} + +/** Whether `url` is the registry itself or below it: same origin, same path prefix. */ +export function isUnderRegistry(url: string, registry: string): boolean { + try { + const target = new URL(url); + const base = new URL(registry); + return target.origin === base.origin && target.pathname.startsWith(base.pathname); + } catch { + return false; + } +} + +/** + * Every way a registry's copy of a package differs from the release artifact, or nothing: + * the version metadata the registry serves, and the bytes it serves for it. + */ +export function registryCopyProblems( + local: Pick, + version: Readonly>, + downloaded: Uint8Array, + registry: string, +): string[] { + const problems: string[] = []; + const { entry, packageJson } = local; + if (version.name !== entry.name || version.version !== entry.version) { + problems.push(`the registry names ${String(version.name)}@${String(version.version)}`); + } + const dist = isRecord(version.dist) ? version.dist : {}; + if (dist.integrity !== entry.integrity) { + problems.push("the registry reports a different integrity"); + } + const sha1 = createHash("sha1").update(local.data).digest("hex"); + if (dist.shasum !== undefined && dist.shasum !== sha1) { + problems.push("the registry reports a different shasum"); + } + if (typeof dist.tarball !== "string" || !isUnderRegistry(dist.tarball, registry)) { + problems.push("the registry's tarball URL is not on the configured registry"); + } + for (const field of DEPENDENCY_FIELDS) { + if ( + JSON.stringify(sortedRecord(version[field])) !== + JSON.stringify(sortedRecord(packageJson[field])) + ) { + problems.push(`the registry's ${field} differ from the package's`); + } + } + if ( + JSON.stringify(normalizeBin(version.bin, entry.name)) !== + JSON.stringify(normalizeBin(packageJson.bin, entry.name)) + ) { + problems.push("the registry's bin differs from the package's"); + } + const digests = artifactDigests(downloaded); + if ( + !Buffer.from(downloaded).equals(local.data) || + digests.sha256 !== entry.sha256 || + digests.integrity !== entry.integrity + ) { + problems.push("the registry serves different bytes from the release artifact"); + } + return problems; +} + +/** SemVer 2.0.0 precedence: negative, zero, or positive. Build metadata is ignored. */ +export function compareSemver(left: string, right: string): number { + const parse = (version: string) => { + const match = /^(\d+)\.(\d+)\.(\d+)(?:-([0-9A-Za-z.-]+))?(?:\+[0-9A-Za-z.-]+)?$/u.exec(version); + if (match === null) throw new Error(`Not a semantic version: ${version}`); + return { + core: [Number(match[1]), Number(match[2]), Number(match[3])], + pre: match[4] === undefined ? [] : match[4].split("."), + }; + }; + const a = parse(left); + const b = parse(right); + for (let index = 0; index < 3; index += 1) { + const difference = (a.core[index] ?? 0) - (b.core[index] ?? 0); + if (difference !== 0) return Math.sign(difference); + } + if (a.pre.length === 0 || b.pre.length === 0) return Math.sign(b.pre.length - a.pre.length); + for (let index = 0; index < Math.max(a.pre.length, b.pre.length); index += 1) { + const x = a.pre[index]; + const y = b.pre[index]; + if (x === undefined || y === undefined) return x === undefined ? -1 : 1; + if (x === y) continue; + const xNumber = /^\d+$/u.test(x); + const yNumber = /^\d+$/u.test(y); + if (xNumber && yNumber) return Math.sign(Number(x) - Number(y)); + if (xNumber !== yNumber) return xNumber ? -1 : 1; + return x < y ? -1 : 1; + } + return 0; +} + +// --------------------------------------------------------------------------------------------- +// The registry: read over HTTP, written with npm +// --------------------------------------------------------------------------------------------- + +/** The registry URL, validated, with exactly one trailing slash. */ +export function normalizeRegistry(registry: string, allowHttp = false): string { + let url: URL; + try { + url = new URL(registry); + } catch { + throw new PublicationError("INVALID_OPTIONS", "The registry is not a URL."); + } + if (url.username !== "" || url.password !== "" || url.search !== "" || url.hash !== "") { + throw new PublicationError( + "INVALID_OPTIONS", + "The registry URL may hold no credentials, query, or fragment.", + ); + } + if (url.protocol !== "https:" && !(allowHttp && url.protocol === "http:")) { + throw new PublicationError("INVALID_OPTIONS", "The registry must be reached over HTTPS."); + } + url.pathname = url.pathname.endsWith("/") ? url.pathname : `${url.pathname}/`; + return url.toString(); +} + +/** The npm configuration a publish uses: the scope and registry, and the token by reference. */ +export function npmConfiguration(registry: string): string { + const url = new URL(registry); + return [ + `registry=${registry}`, + `@ekklesia:registry=${registry}`, + `//${url.host}${url.pathname}:_authToken=\${${PUBLISH_TOKEN_VARIABLE}}`, + "", + ].join("\n"); +} + +/** `@ekklesia/module-sdk` → `@ekklesia%2fmodule-sdk`, the npm packument path. */ +export function packumentUrl(registry: string, name: string): string { + return `${registry}${name.replace("/", "%2f")}`; +} + +export interface RegistryReader { + /** The package's packument, or `undefined` when the registry has no such package. */ + packument(name: string): Promise | undefined>; + /** A tarball's bytes; authenticated only when the URL is on the registry itself. */ + download(url: string, anonymous?: boolean): Promise; +} + +export function registryReader( + registry: string, + token: string | undefined, + fetchImplementation: typeof fetch = fetch, +): RegistryReader { + const headers = (url: string, anonymous: boolean): Record => + !anonymous && token && isUnderRegistry(url, registry) + ? { authorization: `Bearer ${token}` } + : {}; + const get = async (url: string, accept: string, anonymous: boolean) => { + try { + return await fetchImplementation(url, { + headers: { accept, ...headers(url, anonymous) }, + redirect: "error", + }); + } catch { + throw new PublicationError("REGISTRY_UNAVAILABLE", "The registry could not be reached."); + } + }; + return { + async download(url, anonymous = false) { + const response = await get(url, "application/octet-stream", anonymous); + if (!response.ok) { + throw new PublicationError( + "REGISTRY_UNAVAILABLE", + `The registry answered ${response.status} for a package tarball.`, + ); + } + return Buffer.from(await response.arrayBuffer()); + }, + async packument(name) { + const response = await get(packumentUrl(registry, name), "application/json", false); + if (response.status === 404) return undefined; + if (!response.ok) { + throw new PublicationError( + "REGISTRY_UNAVAILABLE", + `The registry answered ${response.status} for ${name}.`, + ); + } + return (await response.json()) as Record; + }, + }; +} + +function versionOf( + packument: Record | undefined, + version: string, +): Record | undefined { + const versions = packument?.versions; + const found = isRecord(versions) ? versions[version] : undefined; + return isRecord(found) ? { ...found } : undefined; +} + +function distTagsOf(packument: Record | undefined): Record { + const tags = packument?.["dist-tags"]; + return isRecord(tags) + ? Object.fromEntries(Object.entries(tags).map(([tag, version]) => [tag, String(version)])) + : {}; +} + +interface NpmSession { + publish(file: string, tag: string): Promise; + addTag(name: string, version: string, tag: string): Promise; + dispose(): Promise; +} + +/** npm with only a temporary configuration: no user or global npmrc, no project directory. */ +async function npmSession(registry: string, token: string, npm: string): Promise { + const home = await mkdtemp(path.join(tmpdir(), "ekklesia-publish-")); + const userconfig = path.join(home, "npmrc"); + const globalconfig = path.join(home, "global-npmrc"); + await writeFile(userconfig, npmConfiguration(registry), { mode: 0o600 }); + await writeFile(globalconfig, "", { mode: 0o600 }); + const environment = { + [PUBLISH_TOKEN_VARIABLE]: token, + HOME: home, + PATH: process.env.PATH ?? "", + npm_config_audit: "false", + npm_config_cache: path.join(home, "cache"), + npm_config_fund: "false", + npm_config_globalconfig: globalconfig, + npm_config_update_notifier: "false", + npm_config_userconfig: userconfig, + }; + const run = async (args: readonly string[], what: string) => { + try { + await execFileAsync(npm, [...args], { cwd: home, env: environment, maxBuffer: 16 << 20 }); + } catch (error) { + const stderr = String((error as { stderr?: unknown }).stderr ?? ""); + const summary = stderr + .split("\n") + .filter((line) => /^npm error (?:code|\d{3}|[A-Z])/u.test(line)) + .filter((line) => !line.includes("complete log")) + .slice(0, 3) + .join(" / ") + .replaceAll(token, "***") + .replaceAll(home, ""); + throw new PublicationError( + "PUBLISH_FAILED", + `${what} failed${summary ? `: ${summary}` : "."}`, + ); + } + }; + return { + addTag: (name, version, tag) => + run(["dist-tag", "add", `${name}@${version}`, tag], `Moving ${tag} to ${name}@${version}`), + dispose: () => rm(home, { force: true, recursive: true }), + publish: (file, tag) => + run( + // A file spec runs no lifecycle script and sends the file's own bytes; --ignore-scripts + // says so explicitly. --access public is what a scoped package needs on a public registry. + ["publish", file, "--tag", tag, "--access", "public", "--ignore-scripts"], + `Publishing ${path.basename(file)}`, + ), + }; +} + +// --------------------------------------------------------------------------------------------- +// Publishing a train +// --------------------------------------------------------------------------------------------- + +export type PackageOutcome = "published" | "already-present"; +export type TagOutcome = "moved" | "already-current" | "kept-newer"; + +export interface PublicationHooks { + /** Called before each package is published (not for one already present). Tests use it to + * interrupt a train part way. */ + beforePublish?(name: string, index: number): void | Promise; + /** Called before each final dist-tag is moved. */ + beforeTag?(name: string, index: number): void | Promise; +} + +export interface PublishOptions { + readonly releaseDirectory: string; + readonly registry: string; + readonly version: string; + readonly revision?: string; + /** SHA-256 of the release manifest the package-release gate proved, which binds the artifact a + * publish job received to that gate. */ + readonly manifestSha256?: string; + readonly token: string; + readonly stagingTag?: string; + readonly finalTag?: string; + readonly requirePublicRead?: boolean; + /** Only for a disposable test registry. */ + readonly allowHttp?: boolean; + readonly npm?: string; + readonly fetch?: typeof fetch; + readonly hooks?: PublicationHooks; + readonly log?: (line: string) => void; + readonly now?: () => Date; +} + +export interface PublishedPackage { + readonly name: string; + readonly version: string; + readonly file: string; + readonly outcome: PackageOutcome; + readonly artifact: { readonly sha256: string; readonly integrity: string; readonly size: number }; + readonly registry: { + readonly tarball: string; + readonly sha256: string; + readonly integrity: string; + readonly publicRead: boolean; + }; + readonly finalTag: TagOutcome; + readonly distTags: Readonly>; +} + +/** Evidence that one train was published: not a package catalog, and never a secret. */ +export interface PublicationRecord { + readonly publicationRecord: typeof PUBLICATION_RECORD_FORMAT; + readonly packageTrain: { + readonly version: string; + readonly revision: string; + readonly manifestSha256: string; + }; + readonly registry: string; + readonly finalTag: string; + readonly completedAt: string; + readonly packages: readonly PublishedPackage[]; +} + +export function serializeRecord(record: PublicationRecord): string { + return `${JSON.stringify(record, null, 2)}\n`; +} + +const TOKEN_FORMAT = /^[\x21-\x7e]{8,}$/u; + +export async function publishPackageTrain(options: PublishOptions): Promise { + const log = options.log ?? (() => {}); + const stagingTag = options.stagingTag ?? DEFAULT_STAGING_TAG; + const finalTag = options.finalTag ?? DEFAULT_FINAL_TAG; + if (!TOKEN_FORMAT.test(options.token)) { + throw new PublicationError( + "INVALID_OPTIONS", + `${PUBLISH_TOKEN_VARIABLE} must hold the publish token.`, + ); + } + for (const tag of [stagingTag, finalTag]) { + if (!/^[a-z][a-z0-9-]*$/u.test(tag)) { + throw new PublicationError("INVALID_OPTIONS", `${tag} is not a usable dist-tag.`); + } + } + if (stagingTag === finalTag) { + throw new PublicationError("INVALID_OPTIONS", "The staging and final tags must differ."); + } + const registry = normalizeRegistry(options.registry, options.allowHttp); + + // Nothing is sent until the whole local train is valid. + const train = await loadReleaseTrain(options.releaseDirectory, { + version: options.version, + ...(options.revision === undefined ? {} : { revision: options.revision }), + }); + if (options.manifestSha256 !== undefined && options.manifestSha256 !== train.manifestSha256) { + throw new PublicationError( + "INVALID_RELEASE", + "The release manifest is not the one the package-release gate proved.", + ); + } + const order = publicationOrder(train.packages); + log( + `Package train ${train.manifest.version} (${train.manifest.source.revision}) validated: ${order + .map((entry) => entry.entry.name) + .join(", ")}`, + ); + + const reader = registryReader(registry, options.token, options.fetch); + const npm = await npmSession(registry, options.token, options.npm ?? "npm"); + try { + const verified: { + local: LocalPackage; + outcome: PackageOutcome; + tarball: string; + publicRead: boolean; + downloaded: { sha256: string; integrity: string }; + }[] = []; + for (const [index, local] of order.entries()) { + const { name, version } = local.entry; + let outcome: PackageOutcome = "already-present"; + let existing = versionOf(await reader.packument(name), version); + if (existing === undefined) { + await options.hooks?.beforePublish?.(name, index); + try { + await npm.publish(local.path, stagingTag); + outcome = "published"; + } catch (error) { + // Someone may have published it meanwhile; what is there now decides. + if (versionOf(await reader.packument(name), version) === undefined) throw error; + } + existing = versionOf(await reader.packument(name), version); + if (existing === undefined) { + throw new PublicationError( + "VERIFICATION_FAILED", + `${name}@${version} is not on the registry after publishing it.`, + ); + } + } + const tarball = isRecord(existing.dist) ? String(existing.dist.tarball) : ""; + if (!isUnderRegistry(tarball, registry)) { + throw new PublicationError( + "VERIFICATION_FAILED", + `${name}@${version}'s tarball is not on the configured registry.`, + ); + } + const downloaded = await reader.download(tarball); + if (!downloaded.equals(local.data)) { + throw new PublicationError( + "BURNED_VERSION", + `${name}@${version} is already on the registry with different bytes. It is never deleted or replaced: the version is burned, publication stops here, and the train needs a new version after an investigation.`, + ); + } + const problems = registryCopyProblems(local, existing, downloaded, registry); + if (problems.length > 0) { + throw new PublicationError( + "VERIFICATION_FAILED", + `${name}@${version} does not match the release artifact.`, + problems, + ); + } + const publicRead = await reader + .download(tarball, true) + .then((anonymous) => anonymous.equals(local.data)) + .catch(() => false); + if (options.requirePublicRead === true && !publicRead) { + throw new PublicationError( + "VERIFICATION_FAILED", + `${name}@${version} cannot be downloaded anonymously.`, + ); + } + log(`${name}@${version}: ${outcome}, verified byte for byte`); + verified.push({ + downloaded: artifactDigests(downloaded), + local, + outcome, + publicRead, + tarball, + }); + } + + // Every package is there and verified: only now does the final tag move. + const tagOutcomes = new Map(); + for (const [index, { local }] of verified.entries()) { + const { name, version } = local.entry; + const current = distTagsOf(await reader.packument(name))[finalTag]; + if (current === version) { + tagOutcomes.set(name, "already-current"); + } else if (current !== undefined && compareSemver(current, version) > 0) { + tagOutcomes.set(name, "kept-newer"); + } else { + await options.hooks?.beforeTag?.(name, index); + await npm.addTag(name, version, finalTag); + tagOutcomes.set(name, "moved"); + } + log(`${name}: ${finalTag} ${tagOutcomes.get(name)}`); + } + + const packages: PublishedPackage[] = []; + const unconverged: string[] = []; + for (const { downloaded, local, outcome, publicRead, tarball } of verified) { + const { name, version } = local.entry; + const distTags = distTagsOf(await reader.packument(name)); + const finalVersion = distTags[finalTag]; + const converged = + finalVersion === version || + (tagOutcomes.get(name) === "kept-newer" && + finalVersion !== undefined && + compareSemver(finalVersion, version) > 0); + if (!converged) unconverged.push(`${name}: ${finalTag} is ${finalVersion ?? "unset"}`); + packages.push({ + artifact: { + integrity: local.entry.integrity, + sha256: local.entry.sha256, + size: local.entry.size, + }, + distTags: Object.fromEntries(Object.entries(distTags).sort()), + file: local.entry.file, + finalTag: tagOutcomes.get(name) ?? "moved", + name, + outcome, + registry: { ...downloaded, publicRead, tarball }, + version, + }); + } + if (unconverged.length > 0) { + throw new PublicationError( + "TAGS_NOT_CONVERGED", + "The final dist-tags did not converge.", + unconverged, + ); + } + return { + completedAt: (options.now ?? (() => new Date()))().toISOString(), + finalTag, + packageTrain: { + manifestSha256: train.manifestSha256, + revision: train.manifest.source.revision, + version: train.manifest.version, + }, + packages, + publicationRecord: PUBLICATION_RECORD_FORMAT, + registry, + }; + } finally { + await npm.dispose(); + } +} + +// --------------------------------------------------------------------------------------------- +// Command line +// --------------------------------------------------------------------------------------------- + +const VALUE_OPTIONS = Object.freeze({ + "--final-tag": "finalTag", + "--manifest-sha256": "manifestSha256", + "--record": "record", + "--registry": "registry", + "--release": "release", + "--revision": "revision", + "--staging-tag": "stagingTag", + "--version": "version", +} as const); +const FLAG_OPTIONS = Object.freeze({ + "--allow-http": "allowHttp", + "--require-public-read": "requirePublicRead", +} as const); + +const USAGE = `Usage: ${PUBLISH_TOKEN_VARIABLE}=… node scripts/publish-package-train.ts --release --registry --version [--revision ] [--manifest-sha256 ] [--record ] [--staging-tag ] [--final-tag ] [--require-public-read]`; + +export function parsePublishArguments(argv: readonly string[]): { + values: Partial>; + flags: Partial>; +} { + const values: Partial> = {}; + const flags: Partial> = {}; + for (let index = 0; index < argv.length; index += 1) { + const argument = argv[index] ?? ""; + if (Object.hasOwn(FLAG_OPTIONS, argument)) { + flags[FLAG_OPTIONS[argument as keyof typeof FLAG_OPTIONS]] = true; + continue; + } + const key = Object.hasOwn(VALUE_OPTIONS, argument) + ? VALUE_OPTIONS[argument as keyof typeof VALUE_OPTIONS] + : undefined; + const value = argv[index + 1]; + if (key === undefined || value === undefined || value.startsWith("--") || key in values) { + throw new PublicationError("INVALID_OPTIONS", USAGE); + } + values[key] = value; + index += 1; + } + if (!values.release || !values.registry || !values.version) { + throw new PublicationError("INVALID_OPTIONS", USAGE); + } + return { flags, values }; +} + +async function main(argv: readonly string[]): Promise { + try { + const { flags, values } = parsePublishArguments(argv); + const record = await publishPackageTrain({ + allowHttp: flags.allowHttp === true, + log: (line) => process.stdout.write(`${line}\n`), + registry: values.registry ?? "", + releaseDirectory: values.release ?? "", + requirePublicRead: flags.requirePublicRead === true, + token: process.env[PUBLISH_TOKEN_VARIABLE] ?? "", + version: values.version ?? "", + ...(values.revision === undefined ? {} : { revision: values.revision }), + ...(values.manifestSha256 === undefined ? {} : { manifestSha256: values.manifestSha256 }), + ...(values.stagingTag === undefined ? {} : { stagingTag: values.stagingTag }), + ...(values.finalTag === undefined ? {} : { finalTag: values.finalTag }), + }); + if (values.record !== undefined) await writeFile(values.record, serializeRecord(record)); + process.stdout.write( + `Published package train ${record.packageTrain.version}: ${record.packages.length} packages verified, ${record.finalTag} converged.\n`, + ); + return 0; + } catch (error) { + if (error instanceof PublicationError) { + process.stderr.write(`${error.message}\n`); + for (const problem of error.problems) process.stderr.write(` ${problem}\n`); + if (error.code === "INVALID_OPTIONS") return 64; + return error.code === "BURNED_VERSION" ? 3 : 1; + } + process.stderr.write("Publishing the package train failed unexpectedly.\n"); + return 1; + } +} + +if (process.argv[1] && import.meta.url === pathToFileURL(process.argv[1]).href) { + process.exitCode = await main(process.argv.slice(2)); +} diff --git a/scripts/test-package-publication.sh b/scripts/test-package-publication.sh new file mode 100755 index 0000000..b39db52 --- /dev/null +++ b/scripts/test-package-publication.sh @@ -0,0 +1,136 @@ +#!/usr/bin/env bash +# The package train publication proof (ADR 0053), against a disposable Gitea of the same version as +# the real registry, never the real one: the publication tool publishes one exact release train, +# and the suite proves the round trip, partial-train recovery, the hard stop on a burned version, +# dist-tag convergence, and that the registry's public owner serves packages anonymously while only +# the publisher team may publish. +# +# bash scripts/test-package-publication.sh [--release ] +# +# With --release, it publishes that release (CI hands it the package-release job's artifact). +# Without, it builds one from a worktree of HEAD, so local changes to the tool are tested against a +# release of the committed packages. +set -Eeuo pipefail + +repository_root="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")/.." && pwd)" +cd "$repository_root" + +# shellcheck source=scripts/lib/compose-job-network.sh +source "$repository_root/scripts/lib/compose-job-network.sh" +# shellcheck source=scripts/lib/compose-diagnostics.sh +source "$repository_root/scripts/lib/compose-diagnostics.sh" +# shellcheck source=scripts/lib/ci-project.sh +source "$repository_root/scripts/lib/ci-project.sh" + +fail() { + echo "package publication proof failed: $*" >&2 + exit 1 +} + +release="" +while (($#)); do + case "$1" in + --release) + release="${2:-}" + shift 2 || fail "--release needs a directory" + ;; + *) fail "usage: test-package-publication.sh [--release ]" ;; + esac +done + +command -v docker >/dev/null || fail "Docker is required" +docker compose version >/dev/null 2>&1 || fail "Docker Compose is required" +compose_job_network_initialize || fail "the current Docker job container could not be identified" + +project="$(ci_project_name package-registry)" +work="$(mktemp -d "${TMPDIR:-/tmp}/ekklesia-package-publication-XXXXXX")" +worktree="$work/release-checkout" + +if [[ "$COMPOSE_JOB_NETWORK_RUNNING_IN_CONTAINER" == "true" ]]; then + export GITEA_PORT="0" GITEA_ROOT_URL="http://gitea:3000/" +else + GITEA_PORT="$(node -e 'const s = require("net").createServer().listen(0, "127.0.0.1", () => { console.log(s.address().port); s.close(); })')" + export GITEA_PORT GITEA_ROOT_URL="http://127.0.0.1:$GITEA_PORT/" +fi +compose=(docker compose --project-name "$project" --file scripts/lib/package-registry.compose.yaml) + +cleanup() { + local exit_status=$? + trap - EXIT INT TERM + compose_job_network_disconnect || exit_status=1 + "${compose[@]}" down --volumes --remove-orphans --timeout 10 >/dev/null 2>&1 || exit_status=1 + if [[ -d "$worktree" ]]; then + git -C "$repository_root" worktree remove --force "$worktree" >/dev/null 2>&1 || true + fi + rm -rf "$work" + exit "$exit_status" +} +trap cleanup EXIT +trap 'exit 130' INT +trap 'exit 143' TERM + +if [[ -z "$release" ]]; then + echo "==> Building a release of HEAD to publish" + store="$(dirname "$(pnpm store path)")" + git worktree add --detach --quiet "$worktree" HEAD + (cd "$worktree" && pnpm install --offline --frozen-lockfile --store-dir "$store" --reporter=silent) + node "$worktree/scripts/package-release.ts" --out "$work/release" >/dev/null + release="$(find "$work/release" -mindepth 1 -maxdepth 1 -type d)" +fi +release="$(cd "$release" && pwd)" + +# Masked in any Compose diagnostics below. +COMPOSE_DIAGNOSTICS_SECRETS=() + +echo "==> Starting a disposable Gitea 1.27.2" +"${compose[@]}" config --quiet || fail "the disposable registry's configuration is invalid" +if ! up_output="$("${compose[@]}" up --detach --wait --wait-timeout 180 gitea 2>&1)"; then + compose_report_not_ready "$up_output" "${compose[@]}" + fail "the disposable Gitea did not become ready" +fi +if [[ "$COMPOSE_JOB_NETWORK_RUNNING_IN_CONTAINER" == "true" ]]; then + compose_job_network_connect gitea "${compose[@]}" || fail "the job could not join the registry's network" +fi +origin="${GITEA_ROOT_URL%/}" + +admin_password="admin-$RANDOM$RANDOM$RANDOM-$RANDOM" +publisher_password="publisher-$RANDOM$RANDOM$RANDOM-$RANDOM" +outsider_password="outsider-$RANDOM$RANDOM$RANDOM-$RANDOM" +"${compose[@]}" exec -T -u git gitea gitea admin user create --admin --username ekklesia-admin \ + --password "$admin_password" --email admin@example.invalid --must-change-password=false >/dev/null + +api() { + local method="$1" path="$2" body="${3:-}" auth="${4:-ekklesia-admin:$admin_password}" + curl -fsS -X "$method" -u "$auth" -H "Content-Type: application/json" \ + ${body:+--data "$body"} "$origin/api/v1$path" +} + +echo "==> Creating public package owners, the publisher team, and an outsider" +for user in publisher outsider; do + password_variable="${user}_password" + api POST /admin/users "{\"username\":\"$user\",\"password\":\"${!password_variable}\",\"email\":\"$user@example.invalid\",\"must_change_password\":false}" >/dev/null +done +owners=(River_of_Glory_Church ekklesia-burned-version ekklesia-tag-retry) +for owner in "${owners[@]}"; do + api POST /orgs "{\"username\":\"$owner\",\"visibility\":\"public\"}" >/dev/null + team="$(api POST "/orgs/$owner/teams" '{"name":"package-publishers","includes_all_repositories":false,"can_create_org_repo":false,"units":["repo.packages"],"units_map":{"repo.packages":"write"}}' | + node -e 'process.stdout.write(String(JSON.parse(require("fs").readFileSync(0, "utf8")).id))')" + api PUT "/teams/$team/members/publisher" >/dev/null +done +token_for() { + local user="$1" password="$2" + api POST "/users/$user/tokens" "{\"name\":\"publish-$RANDOM\",\"scopes\":[\"write:package\"]}" "$user:$password" | + node -e 'process.stdout.write(JSON.parse(require("fs").readFileSync(0, "utf8")).sha1)' +} +publish_token="$(token_for publisher "$publisher_password")" +outsider_token="$(token_for outsider "$outsider_password")" + +echo "==> Publishing the release train to it" +EKKLESIA_TEST_REGISTRY_ORIGIN="$origin" \ +EKKLESIA_TEST_RELEASE="$release" \ +EKKLESIA_TEST_PUBLISH_TOKEN="$publish_token" \ +EKKLESIA_TEST_OUTSIDER_TOKEN="$outsider_token" \ +EKKLESIA_TEST_OWNERS="${owners[*]}" \ + pnpm exec vitest run --config vitest.packages.integration.config.ts + +echo "Package publication proof passed." diff --git a/scripts/test-package-release.sh b/scripts/test-package-release.sh index 8fcf390..864c594 100755 --- a/scripts/test-package-release.sh +++ b/scripts/test-package-release.sh @@ -2,7 +2,7 @@ # The package release gate (ADR 0053): what must pass before a future step may publish a package # train. It publishes nothing. # -# bash scripts/test-package-release.sh [--allow-modified] +# bash scripts/test-package-release.sh [--allow-modified] [--keep ] # # 1. Builds the public package train from this checkout (`scripts/package-release.ts`): clean build # output, pack, inspect, release manifest. @@ -20,13 +20,24 @@ set -euo pipefail repository_root="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" -allow_modified=() -if [[ "${1:-}" == "--allow-modified" ]]; then - allow_modified=(--allow-modified) -elif (($#)); then - echo "usage: test-package-release.sh [--allow-modified]" >&2 - exit 64 -fi +allow_modified=() keep="" +while (($#)); do + case "$1" in + --allow-modified) + allow_modified=(--allow-modified) + shift + ;; + --keep) + keep="${2:-}" + [[ -n "$keep" ]] || { echo "usage: --keep " >&2; exit 64; } + shift 2 + ;; + *) + echo "usage: test-package-release.sh [--allow-modified] [--keep ]" >&2 + exit 64 + ;; + esac +done # The workspace install's own pnpm store. pnpm chooses a store per file system, so an install in a # temporary directory would otherwise look in another, empty one; every install here is offline. @@ -69,4 +80,13 @@ echo "==> Building Core's verifier" echo "==> Consuming the release from a standalone module" bash "$repository_root/scripts/test-standalone-module.sh" --release "$release" "${allow_modified[@]}" +# The release this gate proved, for a later job to publish exactly (`--keep`): copied only now, after +# every check passed, into a directory that must not hold an earlier one. +if [[ -n "$keep" ]]; then + mkdir -p "$keep" + [[ ! -e "$keep/$(basename "$release")" ]] || { echo "$keep already holds this release" >&2; exit 1; } + cp -R "$release" "$keep/" + echo " kept in $keep/$(basename "$release")" +fi + echo "Package release gate passed." diff --git a/tests/ci-scripts.test.ts b/tests/ci-scripts.test.ts index e11c287..684df82 100644 --- a/tests/ci-scripts.test.ts +++ b/tests/ci-scripts.test.ts @@ -97,12 +97,18 @@ describe("CI scripts on a shared Docker daemon", () => { >; }; const startsStacks = - /smoke-infrastructure\.sh|test:migrations:integration|smoke-deploy\.sh|check-image\.sh|docker build/u; + /smoke-infrastructure\.sh|test:migrations:integration|smoke-deploy\.sh|check-image\.sh|test-package-publication\.sh|docker build/u; const jobs = Object.entries(workflow.jobs).filter(([, job]) => job.steps.some((step) => startsStacks.test(step.run ?? "")), ); expect(jobs.map(([id]) => id).sort()).toEqual( - ["deploy-smoke", "image-build", "infrastructure-smoke", "migration-smoke"].sort(), + [ + "deploy-smoke", + "image-build", + "infrastructure-smoke", + "migration-smoke", + "package-publication", + ].sort(), ); for (const [id, job] of jobs) { expect(job.env?.EKKLESIA_CI_JOB_KEY, id).toBeTruthy(); diff --git a/tests/package-publication.integration.ts b/tests/package-publication.integration.ts new file mode 100644 index 0000000..d44c71a --- /dev/null +++ b/tests/package-publication.integration.ts @@ -0,0 +1,305 @@ +import { execFile } from "node:child_process"; +import { mkdtemp, readFile, rm, writeFile } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import path from "node:path"; +import { promisify } from "node:util"; +import { afterAll, beforeAll, describe, expect, it } from "vitest"; +import { + type LocalTrain, + loadReleaseTrain, + normalizeRegistry, + type PublicationRecord, + PUBLISH_TOKEN_VARIABLE, + packumentUrl, + publicationOrder, + publishPackageTrain, +} from "../scripts/publish-package-train.js"; + +/** + * The publication tool against a disposable Gitea 1.27.2 (scripts/test-package-publication.sh), + * configured as the real registry is meant to be: public package owners anyone may read from, and a + * publisher team that alone may publish. Each scenario uses its own owner, so they never interfere. + */ + +const execFileAsync = promisify(execFile); +/** Set by scripts/test-package-publication.sh for the disposable registry it started. */ +const setting = (name: string): string => process.env[name] ?? ""; +const origin = setting("EKKLESIA_TEST_REGISTRY_ORIGIN"); +const releaseDirectory = setting("EKKLESIA_TEST_RELEASE"); +const publishToken = setting("EKKLESIA_TEST_PUBLISH_TOKEN"); +const outsiderToken = setting("EKKLESIA_TEST_OUTSIDER_TOKEN"); +const [mainOwner, burnedOwner, tagOwner] = setting("EKKLESIA_TEST_OWNERS").split(" "); +if (!origin || !releaseDirectory || !publishToken || !outsiderToken || !tagOwner) { + throw new Error("Run this suite with scripts/test-package-publication.sh."); +} +const registryOf = (owner: string | undefined) => + normalizeRegistry(`${origin}/api/packages/${owner}/npm/`, true); + +let train: LocalTrain; +let order: string[]; +const scratch: string[] = []; + +beforeAll(async () => { + const manifest = JSON.parse( + await readFile(path.join(releaseDirectory, "ekklesia-package-train.json"), "utf8"), + ) as { version: string; source: { revision: string } }; + train = await loadReleaseTrain(releaseDirectory, { + revision: manifest.source.revision, + version: manifest.version, + }); + order = publicationOrder(train.packages).map((entry) => entry.entry.name); +}); + +afterAll(async () => { + await Promise.all(scratch.splice(0).map((dir) => rm(dir, { force: true, recursive: true }))); +}); + +function publish( + owner: string | undefined, + hooks?: Parameters[0]["hooks"], +) { + return publishPackageTrain({ + allowHttp: true, + registry: registryOf(owner), + releaseDirectory, + requirePublicRead: true, + revision: train.manifest.source.revision, + token: publishToken, + version: train.manifest.version, + ...(hooks === undefined ? {} : { hooks }), + }); +} + +/** What anyone, signed in or not, gets for a package: its packument, or undefined on 404. */ +async function anonymousPackument(owner: string | undefined, name: string) { + const response = await fetch(packumentUrl(registryOf(owner), name)); + if (response.status === 404) return undefined; + expect(response.status).toBe(200); + return (await response.json()) as { + versions: Record; + "dist-tags": Record; + }; +} + +async function anonymousBytes(url: string): Promise { + const response = await fetch(url); + expect(response.status).toBe(200); + return Buffer.from(await response.arrayBuffer()); +} + +function local(name: string) { + const found = train.packages.find((entry) => entry.entry.name === name); + if (!found) throw new Error(name); + return found; +} + +/** `npm publish ` outside the tool, as an operator or an attacker might run it. */ +async function npmPublish(owner: string | undefined, file: string, token: string): Promise { + const home = await mkdtemp(path.join(tmpdir(), "ekklesia-npm-")); + scratch.push(home); + const registry = registryOf(owner); + const url = new URL(registry); + await writeFile( + path.join(home, "npmrc"), + `registry=${registry}\n//${url.host}${url.pathname}:_authToken=\${${PUBLISH_TOKEN_VARIABLE}}\n`, + ); + try { + await execFileAsync( + "npm", + ["publish", file, "--tag", "manual", "--access", "public", "--ignore-scripts"], + { + cwd: home, + env: { + [PUBLISH_TOKEN_VARIABLE]: token, + HOME: home, + PATH: process.env.PATH ?? "", + npm_config_globalconfig: path.join(home, "none"), + npm_config_userconfig: path.join(home, "npmrc"), + }, + }, + ); + return "published"; + } catch (error) { + return String((error as { stderr?: unknown }).stderr ?? "failed"); + } +} + +describe("the registry's access model", () => { + it("lets anyone search and read package metadata, with no account at all", async () => { + const search = await fetch(`${registryOf(mainOwner)}-/v1/search?text=ekklesia`); + expect(search.status).toBe(200); + // An absent package is 404 to an anonymous reader, not 401: reading is open. + expect(await anonymousPackument(mainOwner, "@ekklesia/contracts")).toBeUndefined(); + }); + + it("refuses publishing without credentials, and from a user outside the publisher team", async () => { + const contracts = local("@ekklesia/contracts").path; + expect(await npmPublish(mainOwner, contracts, "")).toMatch(/ENEEDAUTH|E401|401/u); + expect(await npmPublish(mainOwner, contracts, outsiderToken)).toMatch(/E401|E403|40[13]/u); + expect(await anonymousPackument(mainOwner, "@ekklesia/contracts")).toBeUndefined(); + }); +}); + +describe("publishing a train, interrupted and resumed", () => { + let record: PublicationRecord; + + it("run 1: stops before the sixth package, with five published and no final tag", async () => { + await expect( + publish(mainOwner, { + beforePublish: (_name, index) => { + if (index === 5) throw new Error("injected failure before the sixth package"); + }, + }), + ).rejects.toThrow("injected failure"); + for (const [index, name] of order.entries()) { + const packument = await anonymousPackument(mainOwner, name); + if (index < 5) { + const version = packument?.versions[train.manifest.version]; + expect(version, name).toBeDefined(); + expect(packument?.["dist-tags"].latest, name).toBeUndefined(); + expect(packument?.["dist-tags"]["ekklesia-staging"], name).toBe(train.manifest.version); + expect((await anonymousBytes(version?.dist.tarball ?? "")).equals(local(name).data)).toBe( + true, + ); + } else { + expect(packument, name).toBeUndefined(); + } + } + }); + + it("run 2: verifies the five as they are, publishes the other four, and only then moves latest", async () => { + record = await publish(mainOwner); + expect(record.packages.map((entry) => entry.name)).toEqual(order); + expect(record.packages.map((entry) => entry.outcome)).toEqual([ + ...Array(5).fill("already-present"), + ...Array(4).fill("published"), + ]); + expect(record.packages.every((entry) => entry.finalTag === "moved")).toBe(true); + expect(record.packageTrain).toEqual({ + manifestSha256: train.manifestSha256, + revision: train.manifest.source.revision, + version: train.manifest.version, + }); + expect(JSON.stringify(record)).not.toContain(publishToken); + }); + + it("serves every package anonymously, byte for byte the release artifact", async () => { + for (const name of order) { + const packument = await anonymousPackument(mainOwner, name); + const version = packument?.versions[train.manifest.version]; + expect(packument?.["dist-tags"].latest, name).toBe(train.manifest.version); + expect(version?.dist.integrity, name).toBe(local(name).entry.integrity); + expect( + (await anonymousBytes(version?.dist.tarball ?? "")).equals(local(name).data), + name, + ).toBe(true); + const entry = record.packages.find((candidate) => candidate.name === name); + expect(entry?.registry.sha256).toBe(local(name).entry.sha256); + expect(entry?.registry.publicRead).toBe(true); + } + }); + + it("run 3: a rerun of a published train changes nothing", async () => { + const again = await publish(mainOwner); + expect(again.packages.every((entry) => entry.outcome === "already-present")).toBe(true); + expect(again.packages.every((entry) => entry.finalTag === "already-current")).toBe(true); + }); + + it("the registry itself refuses a second upload of a published version", async () => { + const name = "@ekklesia/module-sdk"; + const packument = await anonymousPackument(mainOwner, name); + const tarballUrl = packument?.versions[train.manifest.version]?.dist.tarball ?? ""; + // A raw upload, past npm's own client-side check, with the publisher's own token. + const { data, entry, packageJson } = local(name); + const response = await fetch(packumentUrl(registryOf(mainOwner), name), { + body: JSON.stringify({ + _attachments: { + [path.basename(new URL(tarballUrl).pathname)]: { + content_type: "application/octet-stream", + data: data.toString("base64"), + length: data.length, + }, + }, + _id: name, + "dist-tags": { manual: entry.version }, + name, + versions: { [entry.version]: { ...packageJson, dist: { integrity: entry.integrity } } }, + }), + headers: { authorization: `Bearer ${publishToken}`, "content-type": "application/json" }, + method: "PUT", + }); + expect(response.status).toBe(409); + expect((await anonymousBytes(tarballUrl)).equals(data)).toBe(true); + }); +}); + +describe("a version already published with other bytes", () => { + it("stops the train for good at that package, deleting and overwriting nothing", async () => { + const poisonedName = "@ekklesia/module-sdk"; + const index = order.indexOf(poisonedName); + expect(index).toBeGreaterThan(0); + expect(index).toBeLessThan(order.length - 1); + + // Same name and version, other bytes: the release tarball with one extra file. + const work = await mkdtemp(path.join(tmpdir(), "ekklesia-burned-")); + scratch.push(work); + await execFileAsync("tar", ["-xzf", local(poisonedName).path, "-C", work]); + await writeFile(path.join(work, "package", "UNEXPECTED"), "not the release\n"); + const poisoned = path.join(work, "poisoned.tgz"); + await execFileAsync("tar", ["-czf", poisoned, "-C", work, "package"]); + expect(await npmPublish(burnedOwner, poisoned, publishToken)).toBe("published"); + const poisonedBytes = await readFile(poisoned); + + await expect(publish(burnedOwner)).rejects.toMatchObject({ code: "BURNED_VERSION" }); + + for (const [position, name] of order.entries()) { + const packument = await anonymousPackument(burnedOwner, name); + if (position < index) { + expect(packument?.versions[train.manifest.version], name).toBeDefined(); + } else if (position === index) { + const url = packument?.versions[train.manifest.version]?.dist.tarball ?? ""; + expect((await anonymousBytes(url)).equals(poisonedBytes)).toBe(true); + } else { + expect(packument, `${name} must not be published after the stop`).toBeUndefined(); + } + expect(packument?.["dist-tags"].latest, name).toBeUndefined(); + } + + // A retry stops in exactly the same place, and still changes nothing. + await expect(publish(burnedOwner)).rejects.toMatchObject({ code: "BURNED_VERSION" }); + expect(await anonymousPackument(burnedOwner, order[index + 1] ?? "")).toBeUndefined(); + }); +}); + +describe("moving the final tag, interrupted and resumed", () => { + it("leaves the train unfinished when moving latest fails part way", async () => { + await expect( + publish(tagOwner, { + beforeTag: (_name, index) => { + if (index === 4) throw new Error("injected failure before the fifth tag"); + }, + }), + ).rejects.toThrow("injected failure"); + for (const [index, name] of order.entries()) { + const packument = await anonymousPackument(tagOwner, name); + expect(packument?.versions[train.manifest.version], name).toBeDefined(); + expect(packument?.["dist-tags"].latest, name).toBe( + index < 4 ? train.manifest.version : undefined, + ); + } + }); + + it("a retry republishes nothing and moves only the tags still missing", async () => { + const record = await publish(tagOwner); + expect(record.packages.every((entry) => entry.outcome === "already-present")).toBe(true); + expect(record.packages.map((entry) => entry.finalTag)).toEqual([ + ...Array(4).fill("already-current"), + ...Array(5).fill("moved"), + ]); + for (const name of order) { + expect((await anonymousPackument(tagOwner, name))?.["dist-tags"].latest).toBe( + train.manifest.version, + ); + } + }); +}); diff --git a/tests/package-publication.test.ts b/tests/package-publication.test.ts new file mode 100644 index 0000000..988bead --- /dev/null +++ b/tests/package-publication.test.ts @@ -0,0 +1,546 @@ +import { mkdtemp, readFile, rm, writeFile } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import path from "node:path"; +import { fileURLToPath } from "node:url"; +import { gzipSync } from "node:zlib"; +import { afterEach, describe, expect, it } from "vitest"; +import { parse } from "yaml"; +import { + artifactDigests, + PACKAGE_TRAIN_MANIFEST, + type PackageTrainManifest, + PUBLIC_PACKAGES, + serializeManifest, + tarballName, +} from "../scripts/package-release.js"; +import { + compareSemver, + isUnderRegistry, + loadReleaseTrain, + normalizeBin, + normalizeRegistry, + npmConfiguration, + PUBLISH_TOKEN_VARIABLE, + PublicationError, + packumentUrl, + parsePublishArguments, + publicationOrder, + publishPackageTrain, + registryCopyProblems, + registryReader, + tarballPackageJson, + trainProblems, +} from "../scripts/publish-package-train.js"; + +/** + * The package train publication tool's own rules (ADR 0053), without a registry: what it accepts + * as a publishable train, the order it publishes in, how it compares a registry copy with the + * release, and what it refuses before it ever reaches the network. The registry behaviour itself is + * proven against a disposable Gitea by `scripts/test-package-publication.sh`. + */ + +const repositoryRoot = path.resolve(path.dirname(fileURLToPath(import.meta.url)), ".."); +const VERSION = "0.1.2"; +const REVISION = "86ae907f59458508e3277666cdd214f5bd7c1078"; +const REGISTRY = "https://registry.example/api/packages/owner/npm/"; +const scratch: string[] = []; + +afterEach(async () => { + await Promise.all(scratch.splice(0).map((dir) => rm(dir, { force: true, recursive: true }))); +}); + +/** A minimal npm tarball: gzip of a ustar archive holding the given files under `package/`. */ +function tarball(files: Record): Buffer { + const blocks: Buffer[] = []; + for (const [name, contents] of Object.entries(files)) { + const body = Buffer.from(contents); + const header = Buffer.alloc(512); + header.write(`package/${name}`, 0, "utf8"); + header.write("0000644\0", 100); + header.write("0000000\0", 108); + header.write("0000000\0", 116); + header.write(`${body.length.toString(8).padStart(11, "0")}\0`, 124); + header.write("00000000000\0", 136); + header.write(" ", 148); + header.write("0", 156); + header.write("ustar\0", 257); + header.write("00", 263); + let checksum = 0; + for (const byte of header) checksum += byte; + header.write(`${checksum.toString(8).padStart(6, "0")}\0 `, 148); + blocks.push(header, body, Buffer.alloc((512 - (body.length % 512)) % 512)); + } + blocks.push(Buffer.alloc(1024)); + return gzipSync(Buffer.concat(blocks)); +} + +const TRAIN_DEPENDENCIES: Record = { + "@ekklesia/contracts": [], + "@ekklesia/design-tokens": [], + "@ekklesia/module-database": ["@ekklesia/contracts"], + "@ekklesia/module-pack": ["@ekklesia/contracts", "@ekklesia/module-ui-build"], + "@ekklesia/module-sdk": ["@ekklesia/contracts"], + "@ekklesia/module-ui-build": ["@ekklesia/contracts"], + "@ekklesia/module-ui-sdk": [], + "@ekklesia/typescript-config": [], + "@ekklesia/ui": ["@ekklesia/design-tokens"], +}; + +function packageJsonFor(name: string): Record { + return { + dependencies: Object.fromEntries((TRAIN_DEPENDENCIES[name] ?? []).map((dep) => [dep, VERSION])), + name, + version: VERSION, + ...(name === "@ekklesia/module-pack" + ? { bin: { "ekklesia-pack-module": "./bin/ekklesia-pack-module.mjs" } } + : {}), + }; +} + +async function releaseDirectory( + mutate?: (files: Map, manifest: PackageTrainManifest) => PackageTrainManifest, +): Promise { + const directory = await mkdtemp(path.join(tmpdir(), "ekklesia-publication-test-")); + scratch.push(directory); + const files = new Map(); + for (const entry of PUBLIC_PACKAGES) { + files.set( + tarballName(entry.name, VERSION), + tarball({ "package.json": JSON.stringify(packageJsonFor(entry.name)), LICENSE: "AGPL\n" }), + ); + } + let manifest: PackageTrainManifest = { + packageTrain: 1, + packages: PUBLIC_PACKAGES.map((entry) => { + const file = tarballName(entry.name, VERSION); + const data = files.get(file) as Buffer; + return { + file, + name: entry.name, + role: entry.role, + size: data.length, + version: VERSION, + ...artifactDigests(data), + }; + }), + source: { modified: false, revision: REVISION }, + version: VERSION, + }; + files.set( + "SHA256SUMS", + Buffer.from( + [...manifest.packages] + .sort((left, right) => (left.file < right.file ? -1 : 1)) + .map((entry) => `${entry.sha256} ${entry.file}\n`) + .join(""), + ), + ); + manifest = mutate ? mutate(files, manifest) : manifest; + files.set(PACKAGE_TRAIN_MANIFEST, Buffer.from(serializeManifest(manifest))); + for (const [file, data] of files) await writeFile(path.join(directory, file), data); + return directory; +} + +describe("the local train", () => { + it("reads package.json from inside an npm tarball", () => { + const data = tarball({ + LICENSE: "x", + "package.json": '{"name":"@ekklesia/x","version":"1.0.0"}', + }); + expect(tarballPackageJson(data)).toEqual({ name: "@ekklesia/x", version: "1.0.0" }); + expect(() => tarballPackageJson(tarball({ LICENSE: "x" }))).toThrow(); + }); + + it("accepts a whole, unchanged train and records its manifest's own hash", async () => { + const directory = await releaseDirectory(); + const train = await loadReleaseTrain(directory, { revision: REVISION, version: VERSION }); + expect(train.packages).toHaveLength(9); + expect(train.manifestSha256).toBe( + artifactDigests(await readFile(path.join(directory, PACKAGE_TRAIN_MANIFEST))).sha256, + ); + }); + + it("refuses a modified build, another version or revision, and any changed, missing, or extra file", async () => { + const cases: Record[0]> = { + altered: (files, manifest) => { + files.set(tarballName("@ekklesia/ui", VERSION), tarball({ "package.json": "{}" })); + return manifest; + }, + extra: (files, manifest) => { + files.set("ekklesia-core-0.1.2.tgz", Buffer.from("x")); + return manifest; + }, + missing: (files, manifest) => { + files.delete(tarballName("@ekklesia/module-sdk", VERSION)); + return manifest; + }, + modified: (_files, manifest) => ({ + ...manifest, + source: { ...manifest.source, modified: true }, + }), + sums: (files, manifest) => { + files.set("SHA256SUMS", Buffer.from("0 x\n")); + return manifest; + }, + wrongInside: (files, manifest) => { + // Bytes that match the manifest, but a package.json that names another package. + const file = tarballName("@ekklesia/ui", VERSION); + const data = tarball({ + "package.json": JSON.stringify({ name: "@ekklesia/other", version: VERSION }), + }); + files.set(file, data); + return { + ...manifest, + packages: manifest.packages.map((entry) => + entry.file === file ? { ...entry, size: data.length, ...artifactDigests(data) } : entry, + ), + }; + }, + }; + for (const [name, mutate] of Object.entries(cases)) { + const directory = await releaseDirectory(mutate); + await expect( + loadReleaseTrain(directory, { revision: REVISION, version: VERSION }), + name, + ).rejects.toMatchObject({ code: "INVALID_RELEASE" }); + } + const valid = await releaseDirectory(); + await expect(loadReleaseTrain(valid, { version: "0.1.3" })).rejects.toMatchObject({ + code: "INVALID_RELEASE", + }); + await expect( + loadReleaseTrain(valid, { revision: "f".repeat(40), version: VERSION }), + ).rejects.toMatchObject({ code: "INVALID_RELEASE" }); + }); + + it("explains every problem at once", () => { + const problems = trainProblems( + { + packageTrain: 1, + packages: [], + source: { modified: true, revision: "abc" }, + version: "9.9.9", + }, + new Map(), + { version: VERSION }, + ); + expect(problems.join(" ")).toContain("modified source tree"); + expect(problems.join(" ")).toContain("not the requested 0.1.2"); + expect(problems.join(" ")).toContain("no full source revision"); + expect(problems.join(" ")).toContain("is missing"); + }); +}); + +describe("the publication order", () => { + const entries = (names: readonly string[]) => + names.map((name) => ({ entry: { name }, packageJson: packageJsonFor(name) })); + + it("puts every package after the train packages it depends on, and is otherwise alphabetical", () => { + const order = publicationOrder(entries(Object.keys(TRAIN_DEPENDENCIES))).map( + (e) => e.entry.name, + ); + expect(order).toEqual([ + "@ekklesia/contracts", + "@ekklesia/design-tokens", + "@ekklesia/module-database", + "@ekklesia/module-sdk", + "@ekklesia/module-ui-build", + "@ekklesia/module-pack", + "@ekklesia/module-ui-sdk", + "@ekklesia/typescript-config", + "@ekklesia/ui", + ]); + }); + + it("does not depend on the order the files are listed in", () => { + const names = Object.keys(TRAIN_DEPENDENCIES); + const forward = publicationOrder(entries(names)).map((e) => e.entry.name); + const backward = publicationOrder(entries([...names].reverse())).map((e) => e.entry.name); + expect(backward).toEqual(forward); + }); + + it("follows the real packages' own dependencies", async () => { + const real = await Promise.all( + PUBLIC_PACKAGES.map(async (entry) => ({ + entry: { name: entry.name }, + packageJson: JSON.parse( + await readFile(path.join(repositoryRoot, entry.directory, "package.json"), "utf8"), + ) as Record, + })), + ); + const order = publicationOrder(real).map((e) => e.entry.name); + for (const { entry, packageJson } of real) { + for (const dependency of Object.keys((packageJson.dependencies as object) ?? {})) { + if (!dependency.startsWith("@ekklesia/")) continue; + expect(order.indexOf(dependency), `${dependency} before ${entry.name}`).toBeLessThan( + order.indexOf(entry.name), + ); + } + } + }); + + it("refuses a dependency cycle", () => { + expect(() => + publicationOrder([ + { entry: { name: "a" }, packageJson: { dependencies: { b: "1" } } }, + { entry: { name: "b" }, packageJson: { dependencies: { a: "1" } } }, + ]), + ).toThrow(PublicationError); + }); +}); + +describe("comparing a registry copy with the release", () => { + const data = tarball({ "package.json": JSON.stringify(packageJsonFor("@ekklesia/module-pack")) }); + const digests = artifactDigests(data); + const local = { + data, + entry: { + file: "ekklesia-module-pack-0.1.2.tgz", + name: "@ekklesia/module-pack", + role: "tooling" as const, + size: data.length, + version: VERSION, + ...digests, + }, + packageJson: packageJsonFor("@ekklesia/module-pack"), + }; + const registryVersion = { + // npm normalizes bin paths in the metadata it uploads; the tarball keeps the author's form. + bin: { "ekklesia-pack-module": "bin/ekklesia-pack-module.mjs" }, + dependencies: { "@ekklesia/module-ui-build": VERSION, "@ekklesia/contracts": VERSION }, + dist: { + integrity: digests.integrity, + tarball: `${REGISTRY}%40ekklesia%2Fmodule-pack/-/0.1.2/module-pack-0.1.2.tgz`, + }, + name: "@ekklesia/module-pack", + version: VERSION, + }; + + it("accepts the same bytes and metadata, however npm normalized bin", () => { + expect(registryCopyProblems(local, registryVersion, data, REGISTRY)).toEqual([]); + expect(normalizeBin("./cli.js", "@ekklesia/tool")).toEqual({ tool: "cli.js" }); + }); + + it("refuses other bytes, integrity, dependencies, bin, or a tarball elsewhere", () => { + const other = tarball({ "package.json": "{}" }); + expect(registryCopyProblems(local, registryVersion, other, REGISTRY).join(" ")).toContain( + "different bytes", + ); + const cases = [ + { ...registryVersion, dist: { ...registryVersion.dist, integrity: "sha512-AAAA" } }, + { ...registryVersion, dependencies: { "@ekklesia/contracts": "0.1.1" } }, + { ...registryVersion, bin: { other: "bin/other.mjs" } }, + { + ...registryVersion, + dist: { ...registryVersion.dist, tarball: "https://elsewhere.example/x.tgz" }, + }, + { ...registryVersion, version: "0.1.3" }, + ]; + for (const version of cases) { + expect( + registryCopyProblems(local, version, data, REGISTRY), + JSON.stringify(version), + ).not.toEqual([]); + } + }); + + it("orders versions by SemVer precedence", () => { + expect(compareSemver("0.1.10", "0.1.9")).toBe(1); + expect(compareSemver("0.2.0", "0.10.0")).toBe(-1); + expect(compareSemver("1.0.0-rc.1", "1.0.0")).toBe(-1); + expect(compareSemver("1.0.0-rc.2", "1.0.0-rc.10")).toBe(-1); + expect(compareSemver("1.0.0-alpha", "1.0.0-1")).toBe(1); + expect(compareSemver("0.1.2", "0.1.2+build")).toBe(0); + }); +}); + +describe("the registry and its credentials", () => { + it("accepts an HTTPS registry, adds the trailing slash, and refuses anything else", () => { + expect(normalizeRegistry("https://registry.example/api/packages/o/npm")).toBe( + "https://registry.example/api/packages/o/npm/", + ); + expect(() => normalizeRegistry("http://registry.example/npm/")).toThrow(PublicationError); + expect(normalizeRegistry("http://gitea:3000/npm/", true)).toBe("http://gitea:3000/npm/"); + expect(() => normalizeRegistry("https://user:secret@registry.example/npm/")).toThrow( + PublicationError, + ); + }); + + it("refers to the token by its variable in npm's configuration, never by value", () => { + const configuration = npmConfiguration(REGISTRY); + expect(configuration).toContain( + `//registry.example/api/packages/owner/npm/:_authToken=\${${PUBLISH_TOKEN_VARIABLE}}`, + ); + expect(configuration).toContain(`@ekklesia:registry=${REGISTRY}`); + expect(packumentUrl(REGISTRY, "@ekklesia/module-sdk")).toBe( + `${REGISTRY}@ekklesia%2fmodule-sdk`, + ); + }); + + it("sends the token only to the registry itself", async () => { + const seen: Record = {}; + const fake = (async (url: string, init?: RequestInit) => { + seen[url] = ((init?.headers ?? {}) as Record).authorization; + return new Response(new Uint8Array([1]), { status: 200 }); + }) as typeof fetch; + const reader = registryReader(REGISTRY, "token-value", fake); + await reader.download(`${REGISTRY}x.tgz`); + await reader.download("https://elsewhere.example/x.tgz"); + await reader.download(`${REGISTRY}y.tgz`, true); + expect(seen[`${REGISTRY}x.tgz`]).toBe("Bearer token-value"); + expect(seen["https://elsewhere.example/x.tgz"]).toBeUndefined(); + expect(seen[`${REGISTRY}y.tgz`]).toBeUndefined(); + expect(isUnderRegistry("https://registry.example/api/packages/other/npm/x", REGISTRY)).toBe( + false, + ); + }); + + it("refuses to start without a token, or with an invalid train, before any request", async () => { + let requests = 0; + const counting = (async () => { + requests += 1; + return new Response("{}"); + }) as typeof fetch; + const directory = await releaseDirectory(); + await expect( + publishPackageTrain({ + fetch: counting, + registry: REGISTRY, + releaseDirectory: directory, + token: "", + version: VERSION, + }), + ).rejects.toMatchObject({ code: "INVALID_OPTIONS" }); + const invalid = await releaseDirectory((_files, manifest) => ({ + ...manifest, + source: { ...manifest.source, modified: true }, + })); + await expect( + publishPackageTrain({ + fetch: counting, + npm: "/nonexistent/npm", + registry: REGISTRY, + releaseDirectory: invalid, + token: "publish-token-value", + version: VERSION, + }), + ).rejects.toMatchObject({ code: "INVALID_RELEASE" }); + const valid = await releaseDirectory(); + await expect( + publishPackageTrain({ + fetch: counting, + manifestSha256: "0".repeat(64), + npm: "/nonexistent/npm", + registry: REGISTRY, + releaseDirectory: valid, + token: "publish-token-value", + version: VERSION, + }), + ).rejects.toMatchObject({ code: "INVALID_RELEASE" }); + expect(requests).toBe(0); + }); + + it("parses its command line, and takes no token from it", () => { + expect( + parsePublishArguments([ + "--release", + "r", + "--registry", + REGISTRY, + "--version", + VERSION, + "--require-public-read", + ]), + ).toEqual({ + flags: { requirePublicRead: true }, + values: { registry: REGISTRY, release: "r", version: VERSION }, + }); + expect(() => parsePublishArguments(["--release", "r", "--token", "x"])).toThrow( + PublicationError, + ); + expect(() => parsePublishArguments(["--release", "r"])).toThrow(PublicationError); + }); +}); + +describe("what the publisher may do to a registry", () => { + it("never deletes, unpublishes, or overwrites: it only publishes new versions and moves a tag", async () => { + // Code only: the tool's comments may say what it never does. + const source = ( + await readFile(path.join(repositoryRoot, "scripts/publish-package-train.ts"), "utf8") + ) + .replace(/\/\*[\s\S]*?\*\//gu, "") + .replace(/^\s*\/\/.*$/gmu, ""); + expect(source).not.toMatch( + /unpublish|deprecate|"DELETE"|method:\s*"(?:PUT|DELETE|POST)"|-rev\/|--force/u, + ); + const npmCommands = [...source.matchAll(/\["(publish|dist-tag)"/gu)].map((match) => match[1]); + expect([...new Set(npmCommands)].sort()).toEqual(["dist-tag", "publish"]); + expect(source).toMatch(/\["dist-tag", "add",/u); + }); +}); + +describe("when the real registry can be written to", () => { + type Workflow = { + on: Record; + jobs: Record< + string, + { + if?: string; + needs?: string | string[]; + steps: { run?: string; env?: Record; uses?: string }[]; + } + >; + }; + const workflow = async (file: string) => + parse(await readFile(path.join(repositoryRoot, ".github/workflows", file), "utf8")) as Workflow; + + it("never from CI: no pull request or push job holds the publish secret or publishes", async () => { + const ci = await readFile(path.join(repositoryRoot, ".github/workflows/ci.yml"), "utf8"); + expect(ci).not.toContain("EKKLESIA_PACKAGES_PUBLISH_TOKEN"); + expect(ci).not.toContain("secrets."); + expect(ci).not.toContain("publish-package-train.ts"); + // CI publishes only to the disposable registry its own job starts. + expect(ci).toContain("bash scripts/test-package-publication.sh --release"); + }); + + it("only by hand, on main, after this run's own gate, with the secret in the publish step alone", async () => { + const publish = await workflow("publish-packages.yml"); + expect(Object.keys(publish.on)).toEqual(["workflow_dispatch"]); + for (const [name, job] of Object.entries(publish.jobs)) { + expect(job.if, name).toBe("github.ref == 'refs/heads/main'"); + } + expect(publish.jobs.publish?.needs).toBe("package-release"); + const steps = Object.entries(publish.jobs).flatMap(([job, { steps }]) => + steps.map((step) => ({ job, step })), + ); + const withSecret = steps.filter(({ step }) => JSON.stringify(step).includes("secrets.")); + expect(withSecret).toHaveLength(1); + expect(withSecret[0]?.job).toBe("publish"); + expect(withSecret[0]?.step.env?.EKKLESIA_PACKAGES_PUBLISH_TOKEN).toBe( + "${{ secrets.EKKLESIA_PACKAGES_PUBLISH_TOKEN }}", + ); + const command = withSecret[0]?.step.run ?? ""; + for (const option of ["--revision", "--manifest-sha256", "--require-public-read", "--record"]) { + expect(command).toContain(option); + } + expect(command).not.toContain("--allow-http"); + // The publish job never builds or packs: it installs nothing and runs no package script. + const publishRuns = (publish.jobs.publish?.steps ?? []) + .map((step) => step.run ?? "") + .join("\n"); + expect(publishRuns).not.toMatch(/pnpm|npm (?:install|ci|run|pack)|package-release\.ts/u); + }); + + it("uses the artifact actions that work on this Gitea, pinned by commit, on visible paths", async () => { + for (const file of ["ci.yml", "publish-packages.yml"]) { + const text = await readFile(path.join(repositoryRoot, ".github/workflows", file), "utf8"); + for (const use of text.matchAll(/uses: (actions\/(?:up|down)load-artifact)@(\S+)/gu)) { + expect(use[2], `${file} ${use[1]}`).toMatch(/^[0-9a-f]{40}$/u); + } + // The artifact actions leave hidden files out: a path starting with a dot uploads nothing. + for (const artifactPath of text.matchAll(/^\s+path: (\S+)$/gmu)) { + expect(artifactPath[1], file).not.toMatch(/^\.|\/\./u); + } + } + }); +}); diff --git a/tsconfig.json b/tsconfig.json index a33d681..456366b 100644 --- a/tsconfig.json +++ b/tsconfig.json @@ -1,13 +1,15 @@ { "extends": "./packages/typescript-config/node.json", "compilerOptions": { - "noEmit": true + "noEmit": true, + "allowImportingTsExtensions": true }, "include": [ "scripts/**/*.ts", "tests/**/*.ts", "vitest.config.ts", - "vitest.migrations.integration.config.ts" + "vitest.migrations.integration.config.ts", + "vitest.packages.integration.config.ts" ], "exclude": ["node_modules", "tests/fixtures"] } diff --git a/vitest.packages.integration.config.ts b/vitest.packages.integration.config.ts new file mode 100644 index 0000000..cd1a7bd --- /dev/null +++ b/vitest.packages.integration.config.ts @@ -0,0 +1,14 @@ +import { defineConfig } from "vitest/config"; + +/** The package train publication suite, run only by `scripts/test-package-publication.sh` against + * the disposable Gitea it starts. */ +export default defineConfig({ + test: { + fileParallelism: false, + hookTimeout: 60_000, + include: ["tests/package-publication.integration.ts"], + passWithNoTests: false, + sequence: { concurrent: false }, + testTimeout: 180_000, + }, +});