Merge pull request 'feat: publish the exact @ekklesia package train to the npm registry (ADR 0053)' (#81) from feat/package-train-publication into main
CI / verify (push) Successful in 1m1s
CI / infrastructure-smoke (push) Successful in 36s
CI / migration-smoke (push) Successful in 1m53s
CI / package-release (push) Successful in 32s
CI / image-build (push) Successful in 11s
CI / deploy-smoke (same-origin) (push) Successful in 42s
CI / deploy-smoke (split) (push) Successful in 1m22s
CI / package-publication (push) Successful in 42s

Reviewed-on: https://git.riverofglorymn.org/River_of_Glory_Church/Ekklesia/pulls/81
Reviewed-by: The_River <[email protected]>
This commit was merged in pull request #81.
This commit is contained in:
The_River
2026-10-08 16:31:57 +00:00
21 changed files with 2494 additions and 33 deletions
+90 -1
View File
@@ -132,6 +132,11 @@ jobs:
needs: verify
runs-on: ubuntu-latest
timeout-minutes: 15
# What the gate proved, for the job that consumes its artifact to check it received exactly that.
outputs:
version: ${{ steps.release.outputs.version }}
revision: ${{ steps.release.outputs.revision }}
manifest_sha256: ${{ steps.release.outputs.manifest_sha256 }}
steps:
- name: Check out repository
uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
@@ -160,7 +165,91 @@ jobs:
# packages, and verifies. Nothing is published and no credential exists here. A future publish
# job must depend on this job and publish only the artifacts it proved.
- name: Build, reproduce, and consume the public package train
run: pnpm test:package-release
run: pnpm test:package-release --keep package-train
- name: Describe the proven release train
id: release
run: |
release="$(find package-train -mindepth 1 -maxdepth 1 -type d)"
manifest="$release/ekklesia-package-train.json"
{
echo "version=$(node -p 'JSON.parse(require("fs").readFileSync(process.argv[1], "utf8")).version' "$manifest")"
echo "revision=$(node -p 'JSON.parse(require("fs").readFileSync(process.argv[1], "utf8")).source.revision' "$manifest")"
echo "manifest_sha256=$(sha256sum "$manifest" | cut -d' ' -f1)"
} >>"$GITHUB_OUTPUT"
cat "$manifest"
# The exact release artifacts, for the jobs that consume them, from a directory whose name does
# not start with a dot: the artifact actions leave hidden files out. Version 3 of the artifact
# actions: version 4 refuses any server that is not github.com, which this Gitea is not, while
# Gitea implements the protocol version 3 uses. A consumer checks the files, not the wrapper.
- name: Hand the release train on
uses: actions/upload-artifact@c6a3b2bd78b3985e4b2f15397fec357f0fd808de # v3.2.2-node20
with:
name: package-train
path: package-train/
if-no-files-found: error
retention-days: 7
# The publication tool, given exactly the artifact the package-release job proved, against a
# disposable Gitea 1.27.2 (scripts/test-package-publication.sh). Never the real registry, and no
# credential: the disposable registry's accounts are created and discarded inside the job.
package-publication:
needs: package-release
runs-on: ubuntu-latest
timeout-minutes: 15
env:
# Names this job's stacks, so its final step removes exactly these (scripts/lib/ci-project.sh).
EKKLESIA_CI_JOB_KEY: package-publication
steps:
- name: Check out repository
uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
with:
persist-credentials: false
# A job the runner killed (cancelled, or past the runner's own time limit) never ran its
# cleanup. Anything a CI job left on this shared daemon more than three hours ago goes first.
- name: Sweep stale CI stacks from the runner
run: bash scripts/ci-sweep.sh --stale
- name: Set up Node.js
uses: actions/setup-node@48b55a011bda9f5d6aeb4c2d9c7362e8dae4041e # v6.4.0
with:
node-version-file: .node-version
- name: Activate pinned pnpm
run: |
corepack enable
corepack prepare [email protected] --activate
- name: Install dependencies
run: pnpm install --frozen-lockfile
- name: Receive the release train the gate proved
uses: actions/download-artifact@ad191675b41f6a5b46da9a048cb6893812da158b # v3.1.0-node20
with:
name: package-train
path: package-train/
- name: Check it is exactly that train
env:
EXPECTED_MANIFEST_SHA256: ${{ needs.package-release.outputs.manifest_sha256 }}
run: |
release="$(find package-train -mindepth 1 -maxdepth 1 -type d)"
actual="$(sha256sum "$release/ekklesia-package-train.json" | cut -d' ' -f1)"
if [[ -z "$EXPECTED_MANIFEST_SHA256" || "$actual" != "$EXPECTED_MANIFEST_SHA256" ]]; then
echo "The received release manifest is not the one the package-release job proved." >&2
exit 1
fi
(cd "$release" && sha256sum --check --strict SHA256SUMS)
- name: Publish it to a disposable Gitea, and prove the round trip and recovery
run: bash scripts/test-package-publication.sh --release "$(find package-train -mindepth 1 -maxdepth 1 -type d)"
# Whatever happened above, this job's own stacks do not outlive it.
- name: Remove this job's CI stacks
if: always()
run: bash scripts/ci-sweep.sh --job
image-build:
needs: verify
+133
View File
@@ -0,0 +1,133 @@
name: Publish packages
# Publishes one public @ekklesia/* package train (ADR 0053, docs/development/releases.md) to the
# npm registry: exactly the artifacts this run's package-release gate proves, and nothing else.
#
# Only ever started by hand, on `main`, by someone with write access to this repository: no pull
# request or push can start it. It needs the version-set commit of a release (`version` must equal
# the platform version there), and the repository secret EKKLESIA_PACKAGES_PUBLISH_TOKEN, a token of
# the dedicated publisher account with only the `write:package` scope. Packages are publicly
# readable (ADR 0054); publishing is what this protects.
on:
workflow_dispatch:
inputs:
version:
description: The package train version to publish, equal to the platform version on main.
required: true
type: string
permissions:
contents: read
# Never cancel a publication part way, and never run two at once. A rerun resumes safely.
concurrency:
group: publish-packages
cancel-in-progress: false
env:
# The only registry-specific setting: where the train is published.
EKKLESIA_PACKAGES_REGISTRY: https://git.riverofglorymn.org/api/packages/River_of_Glory_Church/npm/
jobs:
package-release:
if: github.ref == 'refs/heads/main'
runs-on: ubuntu-latest
timeout-minutes: 20
outputs:
version: ${{ steps.release.outputs.version }}
revision: ${{ steps.release.outputs.revision }}
manifest_sha256: ${{ steps.release.outputs.manifest_sha256 }}
steps:
- name: Check out repository
uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
with:
persist-credentials: false
- name: Set up Node.js
uses: actions/setup-node@48b55a011bda9f5d6aeb4c2d9c7362e8dae4041e # v6.4.0
with:
node-version-file: .node-version
- name: Activate pinned pnpm
run: |
corepack enable
corepack prepare [email protected] --activate
- name: Install dependencies
run: pnpm install --frozen-lockfile
# The same credential-free gate as CI: build, reproduce, and consume the train.
- name: Build, reproduce, and consume the public package train
run: pnpm test:package-release --keep package-train
- name: Describe the proven release train
id: release
run: |
release="$(find package-train -mindepth 1 -maxdepth 1 -type d)"
manifest="$release/ekklesia-package-train.json"
{
echo "version=$(node -p 'JSON.parse(require("fs").readFileSync(process.argv[1], "utf8")).version' "$manifest")"
echo "revision=$(node -p 'JSON.parse(require("fs").readFileSync(process.argv[1], "utf8")).source.revision' "$manifest")"
echo "manifest_sha256=$(sha256sum "$manifest" | cut -d' ' -f1)"
} >>"$GITHUB_OUTPUT"
# Artifact actions version 3: version 4 refuses any server that is not github.com. The directory
# name must not start with a dot: the artifact actions leave hidden files out.
- name: Hand the release train on
uses: actions/upload-artifact@c6a3b2bd78b3985e4b2f15397fec357f0fd808de # v3.2.2-node20
with:
name: package-train
path: package-train/
if-no-files-found: error
retention-days: 90
publish:
needs: package-release
if: github.ref == 'refs/heads/main'
runs-on: ubuntu-latest
timeout-minutes: 20
steps:
# Only for the publication tool itself: nothing is built or installed in this job.
- name: Check out repository
uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
with:
persist-credentials: false
- name: Set up Node.js
uses: actions/setup-node@48b55a011bda9f5d6aeb4c2d9c7362e8dae4041e # v6.4.0
with:
node-version-file: .node-version
- name: Receive the release train the gate proved
uses: actions/download-artifact@ad191675b41f6a5b46da9a048cb6893812da158b # v3.1.0-node20
with:
name: package-train
path: package-train/
# The tool validates the whole train against its manifest, that manifest against the gate's,
# its version against the request, and its revision against this commit, before it sends
# anything; then publishes, verifies every package byte for byte, and moves `latest` last.
- name: Publish the exact release train
env:
EKKLESIA_PACKAGES_PUBLISH_TOKEN: ${{ secrets.EKKLESIA_PACKAGES_PUBLISH_TOKEN }}
TRAIN_VERSION: ${{ inputs.version }}
TRAIN_REVISION: ${{ github.sha }}
MANIFEST_SHA256: ${{ needs.package-release.outputs.manifest_sha256 }}
run: |
node scripts/publish-package-train.ts \
--release "$(find package-train -mindepth 1 -maxdepth 1 -type d)" \
--registry "$EKKLESIA_PACKAGES_REGISTRY" \
--version "$TRAIN_VERSION" \
--revision "$TRAIN_REVISION" \
--manifest-sha256 "$MANIFEST_SHA256" \
--record publication-record.json \
--require-public-read
- name: Keep the publication record
uses: actions/upload-artifact@c6a3b2bd78b3985e4b2f15397fec357f0fd808de # v3.2.2-node20
with:
name: package-train-publication-record
path: publication-record.json
if-no-files-found: error
retention-days: 90
+2
View File
@@ -26,3 +26,5 @@ packages/module-pack/.fixtures/
# `pnpm packages:release` writes package release trains here (scripts/package-release.ts).
.package-release/
# CI keeps the proven release here to hand it on as a workflow artifact (scripts/test-package-release.sh --keep).
/package-train/
+4
View File
@@ -81,6 +81,10 @@ ever disagree:
- `pnpm test:package-release` — builds the public `@ekklesia/*` package train, proves it
reproducible, and proves a standalone module consumes exactly those tarballs. Publishes nothing.
Run it for any change to the public packages or their release tooling.
- `pnpm test:package-publication` — publishes a release train to a disposable Gitea (Docker) and
proves the publication tool. Never the real registry. The real publication is the manual
`Publish packages` workflow only; never run `scripts/publish-package-train.ts` against the real
registry without the user's explicit intent.
- `pnpm doctor` — read-only local readiness report. Never treat it as a substitute for `pnpm
verify` or the integration suite.
- `pnpm migrations:plan` / `pnpm migrations:apply` — read-only planning vs. mutating execution
+5 -2
View File
@@ -67,8 +67,11 @@ A module can be built in its own repository from Ekklesia's public module-develo
a curated `@ekklesia/module-sdk`, `@ekklesia/module-database`, the UI packages, and the build and
packaging tools. They are released as one coordinated train of tarballs with a release manifest,
and CI proves on every run that two builds of a commit are byte-identical and that a module outside
the repository installs, builds, packages, and verifies from those artifacts alone. They are not
published to any registry yet
the repository installs, builds, packages, and verifies from those artifacts alone. A manual
workflow, the only way to publish, publishes exactly those artifacts to the npm registry and
verifies every package byte for byte. Anyone may read and install the packages before launch
([ADR 0054](docs/architecture/decisions/0054-public-package-reads-before-launch.md)). The first train
published will be the next platform release
([Releases and the public package train](docs/development/releases.md)).
Phase 4A added Core-owned identity foundations: a Core-owned Better Auth database schema
@@ -2,6 +2,8 @@
- Status: Accepted
- Date: 2026-10-07
- Superseded in part by: [0054](0054-public-package-reads-before-launch.md) — only where this ADR
describes the package phase before launch as private, or requires authenticated package reads.
- Extends: [0049](0049-module-package-v1.md). This is the step ADR 0049 deferred: "publishing the
Ekklesia SDK packages for module builds outside a platform workspace". A registry is used only to
build a module. Installing one stays registry-free: an installed package is self-contained, as ADR
@@ -0,0 +1,85 @@
# 0054: Ekklesia's module-development packages are publicly readable before launch
- Status: Accepted
- Date: 2026-10-08
- Supersedes (partial): [0053](0053-standalone-module-development-packages.md) — only where it
describes the package phase before launch as private: "Gitea may host the private phase", the
read credential for module CI in "Registry access and the private phase", and its requirement
that packages not be served anonymously while Ekklesia is not public. Every other part of that ADR
remains accepted and unchanged: the npm registry protocol, the public package set and its
boundaries, the curated SDK, the dependency rules, the coordinated package train, exact release
artifacts, standalone module builds, provider portability, supply chain and immutability, and the
credential handling for publishing.
- Related: [ADR 0049](0049-module-package-v1.md)
## Context
ADR 0053 decided how Ekklesia's module-development packages (`@ekklesia/*`) are built, released, and
distributed over the npm registry protocol. It assumed that the phase before Ekklesia's public
launch would be private: module CI would carry a read credential where reads needed one, and the
first publication had to make sure the registry would not serve the packages anonymously.
Checking the chosen host before the first publication showed what keeping the packages private would
take:
- Gitea 1.27.2 decides package read access by the package **owner's** visibility, never a
repository's. The `River_of_Glory_Church` owner is a public organization, so its packages are
readable anonymously.
- That organization's repositories, Ekklesia's source among them, are already publicly readable.
- The instance allows self-registration, so "limited" visibility, readable by any signed-in user,
is readable by anyone who creates an account.
- Private packages would therefore need a separate private package owner, plus reader accounts and
read tokens for every module repository, every module's CI, and every developer.
The project owner has decided that Ekklesia is not secret. Its source is public; the project is
simply not being launched or promoted yet. Hiding the packages built from that public source would
protect nothing, and would still cost every module author an account and a token.
## Decision
1. **Ekklesia's module-development packages may be anonymously readable before the project's public
launch.** This covers the public package set of ADR 0053: search, metadata, and download.
2. **Readability is not a launch, and promises nothing more.** Public readability does not mean:
- that Ekklesia has been publicly launched;
- that the APIs are stable beyond the project's pre-1.0 SemVer policy;
- that third-party production use is supported;
- that a marketplace or public module ecosystem exists.
3. **Reading needs no authentication in this phase.** Installing `@ekklesia/*` needs no account,
reader account, read token, or authenticated install workflow. A module repository configures only
the scope's registry.
4. **Publication stays authenticated and tightly controlled.** Only Ekklesia's release process
publishes, with one dedicated publisher credential that holds nothing beyond package write
access, used only by that process, as ADR 0053's credential handling for publishing already
requires.
5. **Published versions stay immutable by Ekklesia policy.** A published version is never deleted or
republished with other bytes, whatever the registry technically allows.
6. **The current Gitea owner is an acceptable host.** The `River_of_Glory_Church` package namespace
on the current Gitea may host this phase, as long as publishing permission stays restricted to
the publisher. No private package organization is needed.
7. **A public launch is a separate decision.** Launching or promoting Ekklesia publicly, and anything
that changes for its packages then, is a product and release decision this ADR does not make.
## Consequences
- **Module authors and module CI need no Ekklesia credential.** The committed `.npmrc` scope mapping
is the whole registry configuration. ADR 0053's example of a read credential in user or CI
configuration is no longer needed in this phase.
- **ADR 0053's pre-publication access check changes meaning.** It now confirms that anonymous reads
work as intended and that publishing is refused without the publisher's permission, rather than
that reads are private. The disposable-registry suite proves both, and the publication tool can
require an anonymous download to match each published artifact.
- **Restricting publishing is now the whole access boundary**, so the publisher credential's scope,
where it is held, and who can reach it matter all the more.
- **Making the packages private again would be a new decision.** Package names, versions, and
contents already published stay public, because a published version is never deleted.
## Alternatives considered
- **A private package organization on the same Gitea.** Rejected: it hides nothing that is not
already public in the source, and it requires reader accounts and tokens for every module
repository, its CI, and every developer.
- **"Limited" package visibility.** Rejected: with self-registration open, anyone can read anyway,
while the configuration suggests otherwise.
- **Requiring sign-in for the whole instance.** Rejected: it would also hide the public
repositories.
- **Another registry for a private phase.** Unnecessary once reads are public.
+1
View File
@@ -62,3 +62,4 @@ ADRs capture durable choices that affect more than one package or phase.
- [0051: Module packages declare their own license and carry its text](0051-module-package-license-material.md) — Accepted
- [0052: Reviewed license supplements cover redistributed dependencies whose packages omit required material](0052-reviewed-license-supplements.md) — Proposed
- [0053: A module builds outside the monorepo, from Ekklesia packages published over the npm registry protocol](0053-standalone-module-development-packages.md) — Accepted
- [0054: Ekklesia's module-development packages are publicly readable before launch](0054-public-package-reads-before-launch.md) — Accepted
+178 -11
View File
@@ -4,19 +4,22 @@ An Ekklesia platform release is one version. The platform image, the private wor
and the public `@ekklesia/*` module-development packages all carry it. This page covers the order a
release happens in and the public package train
([ADR 0053](../architecture/decisions/0053-standalone-module-development-packages.md)): how its
artifacts are produced, what proves them, and what a future publication step will do with them.
artifacts are produced, what proves them, and how exactly those artifacts are published.
> **Registry publication is not implemented.** Nothing in this repository publishes a package, and
> no CI job holds a registry credential. The release artifacts below are produced and proven on
> every CI run. A future publication step publishes **exactly these artifacts**, unchanged, and
> nothing else.
> **The packages are publicly readable, on purpose**
> ([ADR 0054](../architecture/decisions/0054-public-package-reads-before-launch.md)). Anyone can
> search, read, and install `@ekklesia/*` from the registry with no account and no token, like any
> public npm package. That is not a public launch: Ekklesia's source is already publicly readable,
> and the project is simply not promoted yet. Only **publishing** is restricted, to one manual,
> authenticated workflow. Before 1.0, the APIs may change between versions under the project's
> SemVer policy: a 0.x minor version may break them.
## Where each step sits
```text
version set ─▶ build ─▶ test ─▶ pack the public package train ─▶ standalone consumer proof
│
tag and release (maintainer) ◀─ [future: publish] ◀──┘
tag and release (maintainer) ◀─ publish the exact train (manual) ◀───┘
```
1. **Version set.** A `chore(release): prepare X` pull request sets the release version: the root
@@ -28,14 +31,15 @@ version set ─▶ build ─▶ test ─▶ pack the public package train ─▶
3. **Pack the public package train.** `pnpm packages:release` (below).
4. **Standalone consumer proof.** A module outside the repository consumes exactly those
artifacts (below).
5. **[Future] publish.** Not implemented. Once it is, it must run only after steps 3 and 4 passed
for the same commit. It must check every artifact against the release manifest before
publishing it, and publish nothing else.
5. **Publish the exact train.** By hand, with the `Publish packages` workflow on the version-set
commit on `main` ([Publishing the train](#publishing-the-train)). It publishes the artifacts its
own run's gate (steps 3 and 4) proved, and nothing else.
6. **Tag and release.** Unchanged: the maintainer tags the merge commit of the version-set pull
request. Nothing here creates, moves, or requires a tag.
Steps 2 to 4 run in CI on every pull request and on `main`. The `package-release` job is the gate a
future publication job must depend on.
Steps 2 to 4 run in CI on every pull request and on `main`, and CI also publishes each proven train
to a disposable registry (`package-publication`), never the real one. Step 5 runs only when a
maintainer starts it.
## The public package train
@@ -192,3 +196,166 @@ Two builds of the same commit produce byte-identical tarballs, and CI proves it
`packageManager`. Another Node.js or zlib build may compress differently. Each artifact's
integrity is recorded either way, and that, not byte identity, is what a publication step must
verify.
## Publishing the train
The registry is the Gitea npm registry under the `River_of_Glory_Church` owner:
```text
https://git.riverofglorymn.org/api/packages/River_of_Glory_Church/npm/
```
| Who | May |
| --- | --- |
| Anyone, signed in or not | search, read metadata, download, and install `@ekklesia/*` |
| The publisher account, through the publish workflow | publish new versions and move dist-tags |
| Anyone else | nothing more: publishing needs the publisher team's package write permission |
### The workflow
`.github/workflows/publish-packages.yml` (Actions → Publish packages → Run workflow, on `main`,
with the release version). Nothing else can start it: no pull request or push does, and CI holds no
registry credential at all.
1. **The gate.** `package-release` runs the same credential-free gate as CI (build, reproduce,
consume) and hands the proven release on as a workflow artifact, with its manifest's SHA-256.
2. **The publish job.** It builds and installs nothing. It downloads the artifact and runs
`scripts/publish-package-train.ts` with the secret `EKKLESIA_PACKAGES_PUBLISH_TOKEN`, which only
this step receives.
3. **The publication record** is kept as an artifact of the run.
### What the publication tool does
`scripts/publish-package-train.ts` publishes exactly the `.tgz` files in the release directory. It
never builds, packs, or edits a package, and it never deletes, unpublishes, or overwrites anything.
1. **Validates the whole train before sending anything.** The manifest must be the one the gate
proved (`--manifest-sha256`), for the requested version and this commit (`--revision`), from a
clean tree (`"modified": false`), with exactly the nine expected packages. Every artifact's file
name, size, SHA-256, integrity, `SHA256SUMS` line, and the `package.json` inside it must match.
2. **Publishes in dependency order.** Each package comes after the train packages it depends on,
then alphabetically. For each package:
- **absent:** `npm publish <file.tgz>` under the staging dist-tag `ekklesia-staging`. npm sends the
file's own bytes and runs no script for a file; the registry checks them against their
integrity and stores them as they are;
- **present with the same bytes:** accepted as already published, and verified;
- **present with other bytes:** a hard stop. Nothing is deleted or overwritten, nothing after it
is published, and the version is burned ([Immutability](#immutability)).
3. **Reads every package back.** The registry's metadata must name the package and version, report
the artifact's integrity, and list the same `dependencies`, `peerDependencies`,
`optionalDependencies`, and `bin` as the tarball's `package.json` (npm writes `bin` paths without
a leading `./`). Its tarball URL must be on the configured registry, and the tarball it serves must
be the release artifact, byte for byte. With `--require-public-read`, an anonymous download must
give the same bytes too. A package is published only once all of that holds.
4. **Moves `latest` last.** Only when all nine are verified does `latest` move to the train, never
backwards past a newer version. The train is published when every package's `latest` agrees.
5. **Writes the publication record** (`--record`).
A rerun is always safe. It publishes only what is missing, accepts what is already there and
identical, and moves only the tags that are not yet right. A train interrupted after five packages,
or after four tags, finishes on the next run. No retry ever deletes anything.
### The publication record
Evidence that one train was published, not a second package catalog, and never a secret:
```json
{
"publicationRecord": 1,
"packageTrain": { "version": "0.2.0", "revision": "<commit>", "manifestSha256": "<hex>" },
"registry": "https://git.riverofglorymn.org/api/packages/River_of_Glory_Church/npm/",
"finalTag": "latest",
"completedAt": "<ISO time>",
"packages": [
{
"name": "@ekklesia/contracts",
"version": "0.2.0",
"file": "ekklesia-contracts-0.2.0.tgz",
"outcome": "published",
"artifact": { "sha256": "<hex>", "integrity": "sha512-…", "size": 87379 },
"registry": { "tarball": "<url>", "sha256": "<hex>", "integrity": "sha512-…", "publicRead": true },
"finalTag": "moved",
"distTags": { "ekklesia-staging": "0.2.0", "latest": "0.2.0" }
}
]
}
```
`outcome` is `published` or `already-present`. `finalTag` is `moved`, `already-current`, or
`kept-newer` (a newer version already holds `latest`).
### Credentials
- **One credential:** `EKKLESIA_PACKAGES_PUBLISH_TOKEN`, a repository Actions secret, used only by
the publish step. Nobody needs a token to read or install.
- **The publisher** is a dedicated account, member only of a `River_of_Glory_Church` team whose one
permission is Packages: Write. Its token has only the `write:package` scope. Gitea's
`write:package` also allows deleting packages; the tool never does, and nothing else uses the
token.
- **No token in git or in an `.npmrc`.** The tool writes a temporary npm configuration that refers to
the variable (`_authToken=${EKKLESIA_PACKAGES_PUBLISH_TOKEN}`), ignores every user, global, and
project npm configuration, sends the token only to the registry itself, and removes the
configuration afterwards.
- **What remains trusted:** Gitea gives repository secrets to workflows run for pull requests from
branches of this same repository, so anyone who can push a branch here could read the secret
through a changed workflow. Keep write access to this repository to the maintainers.
### Immutability
Gitea refuses to upload a version that exists (409). But it lets anyone with package write
permission delete a version, deletion leaves nothing behind, and an owner's package cleanup rules
can delete versions on a schedule. So immutability is Ekklesia's policy, not the registry's
guarantee:
- **A published version is never deleted,** by anyone, for any reason short of a security incident.
- **A version number is never reused with other bytes.** A version found on the registry with other
bytes is burned: investigate how it got there, and publish the train under a new version.
- **No package cleanup rule** may be configured for the `River_of_Glory_Church` owner.
- **The publisher never deletes.** The tool has no delete, unpublish, or overwrite, and its tests
check that.
Module lockfiles record each package's integrity, so a replaced package would fail every install of
it, rather than install silently.
### Moving to another registry
Only configuration changes: the registry URL (the workflow's `EKKLESIA_PACKAGES_REGISTRY`), the
publisher credential, and modules' `@ekklesia:registry`. Modules then regenerate their lockfiles,
because Gitea's tarball URLs are its own. Package sources, the train, its manifests, and every
artifact already built stay as they are; the same bytes keep the same integrity.
### Proving the publication machinery
`pnpm test:package-publication` (`scripts/test-package-publication.sh`, the CI `package-publication`
job) publishes a real release train to a disposable Gitea 1.27.2 configured like the real one:
public owners, a publisher team, and an outsider. In CI it publishes the very artifact the
`package-release` job handed on, after checking its manifest hash. It proves:
- anonymous search, metadata, and download work;
- publishing without credentials, or by a user outside the publisher team, fails;
- the round trip is byte-identical, also anonymously;
- the registry refuses a second upload of a published version;
- a train interrupted before its sixth package resumes, verifying the five and publishing the rest,
with `latest` untouched until all nine are verified;
- a version already published with other bytes stops the train, deleting and overwriting nothing,
and publishing nothing after it, on every retry;
- a train interrupted while moving `latest` converges on a rerun, moving only the missing tags.
### The first publication
The first train published is the **next** platform release version, not 0.1.2. The `v0.1.2` tag
predates the curated SDK and this release tooling, so packages built now and called 0.1.2 would
claim to be source that 0.1.2 never contained.
Before it, an administrator of the Gitea instance:
1. creates the dedicated publisher account;
2. creates a `River_of_Glory_Church` team, for example `package-publishers`, with only the Packages
unit at Write, and adds the account to it;
3. creates a token for the account with only `write:package`, and stores it as the repository
Actions secret `EKKLESIA_PACKAGES_PUBLISH_TOKEN`;
4. checks that the owner has no package cleanup rule.
Then the release proceeds as usual: the version-set pull request merges, a maintainer starts
Publish packages on `main` with that version, and tags the commit once the train is published.
+7
View File
@@ -660,6 +660,13 @@ versions, manifest inspection, release directory, manifest bytes, the pack hook)
`tests/public-packages.test.ts`, part of `pnpm verify`. See
[Releases and the public package train](releases.md).
`pnpm test:package-publication` (the CI `package-publication` job, given the `package-release` job's
artifact) publishes a release train to a disposable Gitea 1.27.2 with Docker, never the real
registry, and proves the publication tool: byte-identical round trips, anonymous reads, refused
unauthorized publishing, partial-train recovery, the hard stop on a burned version, and dist-tag
convergence (`tests/package-publication.integration.ts`). The tool's own rules are unit-tested in
`tests/package-publication.test.ts`, part of `pnpm verify`.
## The generic platform image and the reference deployment
These need Docker and build images locally; nothing is pushed or published.
+31 -7
View File
@@ -73,16 +73,37 @@ entrypoints. A module in its own repository depends on the published packages in
### Developing a module in its own repository
> **Status:** the public packages are ready for a standalone module, and a release check proves it
> from locally packed tarballs. They are **not published yet**, and there is no registry URL to use
> yet: registry publication is the next slice
> ([ADR 0053](../architecture/decisions/0053-standalone-module-development-packages.md)). This
> section describes the workflow that publication enables.
> **Status:** the packages are published by release
> ([ADR 0053](../architecture/decisions/0053-standalone-module-development-packages.md),
> [Releases and the public package train](../development/releases.md)). The first published train
> is the next platform release after 0.1.2.
A module does not need an Ekklesia checkout. Do not clone Ekklesia and put your module in its
`modules/` directory: your module is its own repository, and it depends on Ekklesia's
module-development packages like any other npm packages.
The packages are **publicly readable**
([ADR 0054](../architecture/decisions/0054-public-package-reads-before-launch.md)): installing them
needs no account, no token, and no access to Ekklesia's Gitea. That is not a public launch, and it
promises no API stability beyond the pre-1.0 SemVer policy. Only Ekklesia's release workflow can publish them. Point the `@ekklesia` scope at
the registry in your repository's committed `.npmrc`; that is all the configuration there is:
```ini
@ekklesia:registry=https://git.riverofglorymn.org/api/packages/River_of_Glory_Church/npm/
```
Then the module builds and packages like any other:
```bash
pnpm install
pnpm build
pnpm exec ekklesia-pack-module --module . --target any --out ../packages \
--license-file ./LICENSE --source-revision "$(git rev-parse HEAD)"
```
Before 1.0, a minor version may change these APIs (the project's SemVer policy). Pin the train you
build against with ranges in `package.json` and exact versions in your lockfile.
| Package | Where it goes | What it is |
| --- | --- | --- |
| `@ekklesia/module-sdk` | `dependencies` | `defineModuleRuntime`, routes, configuration, storage, realtime, principals |
@@ -148,6 +169,9 @@ import it yourself, because what it exports is the platform's, not a supported a
allowBuilds:
esbuild: true
```
- **Lockfiles name the registry.** pnpm records Gitea's tarball URLs in your lockfile. If the
packages ever move to another registry, change `@ekklesia:registry` and regenerate the lockfile;
nothing else changes, and the integrity of every package stays the same.
- **TypeScript.** Extend the shared configurations by package name, for example
`"extends": "@ekklesia/typescript-config/node.json"` for the backend and
`@ekklesia/typescript-config/browser-react.json` for the UI.
@@ -160,8 +184,8 @@ credited as third-party software, wherever they came from.
Ekklesia proves this on every CI run: `pnpm test:package-release` builds the public packages of a
release as one train of tarballs, proves two builds identical, and installs exactly those tarballs
into a module outside the repository (`tests/fixtures/standalone-module`), with no workspace link
and no registry, then typechecks, builds, packages, and verifies it. Those artifacts are what a
future publication step publishes ([Releases and the public package train](../development/releases.md)).
and no registry, then typechecks, builds, packages, and verifies it. Those exact artifacts are what
the release publishes ([Releases and the public package train](../development/releases.md)).
### An installed module is self-contained
+1
View File
@@ -43,6 +43,7 @@
"test:migrations:integration": "pnpm build && bash scripts/test-migrations-integration.sh",
"test:root": "vitest run --config vitest.config.ts",
"test:package-release": "bash scripts/test-package-release.sh",
"test:package-publication": "bash scripts/test-package-publication.sh",
"test:watch": "vitest",
"typecheck": "turbo run typecheck typecheck:root",
"typecheck:root": "tsc --noEmit --project tsconfig.json",
+25
View File
@@ -0,0 +1,25 @@
# A disposable Gitea, the same version as the real one, whose npm registry the package publication
# tests run against (scripts/test-package-publication.sh). Never the real registry: it starts empty,
# keeps everything in the container, and is removed with its volumes when the test ends.
services:
gitea:
image: docker.gitea.com/gitea:1.27.2@sha256:d20286ca2b2e170fdf628e7231b8a31a3220ade39ff462b55041d43d1fc757dd
environment:
GITEA__security__INSTALL_LOCK: "true"
GITEA__database__DB_TYPE: sqlite3
# The URL clients reach it by: package metadata names tarballs under it.
GITEA__server__ROOT_URL: ${GITEA_ROOT_URL:?GITEA_ROOT_URL is required}
GITEA__server__HTTP_PORT: "3000"
GITEA__server__OFFLINE_MODE: "true"
GITEA__service__DISABLE_REGISTRATION: "true"
GITEA__packages__ENABLED: "true"
GITEA__actions__ENABLED: "false"
GITEA__mailer__ENABLED: "false"
GITEA__log__LEVEL: warn
ports:
- "127.0.0.1:${GITEA_PORT:?GITEA_PORT is required}:3000"
healthcheck:
test: ["CMD", "curl", "-fsS", "http://127.0.0.1:3000/api/healthz"]
interval: 2s
timeout: 3s
retries: 60
+889
View File
@@ -0,0 +1,889 @@
import { execFile } from "node:child_process";
import { createHash } from "node:crypto";
import { mkdtemp, readdir, readFile, rm, writeFile } from "node:fs/promises";
import { tmpdir } from "node:os";
import path from "node:path";
import process from "node:process";
import { pathToFileURL } from "node:url";
import { promisify } from "node:util";
import { gunzipSync } from "node:zlib";
import {
artifactDigests,
PACKAGE_TRAIN_FORMAT,
PACKAGE_TRAIN_MANIFEST,
type PackageTrainManifest,
releaseDirectoryProblems,
type TrainPackage,
} from "./package-release.ts";
/**
* Publishes one package release train (ADR 0053) to an npm-protocol registry: exactly the `.tgz`
* files `pnpm packages:release` produced, as `ekklesia-package-train.json` records them, and nothing
* else. It never builds, packs, or edits a package, and never deletes or unpublishes anything.
*
* EKKLESIA_PACKAGES_PUBLISH_TOKEN=… node scripts/publish-package-train.ts \
* --release <release directory> --registry <npm registry URL> --version <train version> \
* [--revision <full commit id>] [--manifest-sha256 <hex>] [--record <file>] \
* [--staging-tag <tag>] [--final-tag <tag>] [--require-public-read]
*
* 1. The whole local train is validated before anything is sent: the manifest, every artifact's
* name, size, SHA-256, integrity, and the package.json inside it.
* 2. Packages go out in dependency order. For each: absent from the registry, it is published with
* `npm publish <file.tgz>` under a staging dist-tag; present with the same bytes, it is accepted
* as already published; present with other bytes, publication stops for good (the version is
* burned). Every package is then read back: its metadata, and its tarball, which must be the
* release artifact byte for byte.
* 3. Only when all of them are verified does the final dist-tag (`latest`) move, and the train is
* published once every tag agrees. A rerun converges: it publishes only what is missing and moves
* only tags that are not yet right.
*
* The publish token is read from the environment only, and reaches npm through a temporary npm
* configuration that refers to the variable, never the value; the configuration is removed after.
*/
const execFileAsync = promisify(execFile);
export const PUBLISH_TOKEN_VARIABLE = "EKKLESIA_PACKAGES_PUBLISH_TOKEN";
export const DEFAULT_STAGING_TAG = "ekklesia-staging";
export const DEFAULT_FINAL_TAG = "latest";
export const PUBLICATION_RECORD_FORMAT = 1;
const DEPENDENCY_FIELDS = Object.freeze([
"dependencies",
"peerDependencies",
"optionalDependencies",
] as const);
export type PublicationErrorCode =
| "INVALID_OPTIONS"
| "INVALID_RELEASE"
| "BURNED_VERSION"
| "REGISTRY_UNAVAILABLE"
| "PUBLISH_FAILED"
| "VERIFICATION_FAILED"
| "TAGS_NOT_CONVERGED";
export class PublicationError extends Error {
readonly code: PublicationErrorCode;
readonly problems: readonly string[];
constructor(code: PublicationErrorCode, message: string, problems: readonly string[] = []) {
super(message);
this.name = "PublicationError";
this.code = code;
this.problems = problems;
}
}
function isRecord(value: unknown): value is Readonly<Record<string, unknown>> {
return typeof value === "object" && value !== null && !Array.isArray(value);
}
// ---------------------------------------------------------------------------------------------
// The local train
// ---------------------------------------------------------------------------------------------
/** The `package/package.json` inside an npm tarball, read without unpacking anything to disk. */
export function tarballPackageJson(tarball: Uint8Array): Record<string, unknown> {
const data = gunzipSync(tarball);
for (let offset = 0; offset + 512 <= data.length; ) {
const header = data.subarray(offset, offset + 512);
if (header.every((byte) => byte === 0)) break;
const field = (start: number, length: number) =>
header
.subarray(start, start + length)
.toString("utf8")
.replace(/\0.*$/su, "");
const name = field(0, 100);
const prefix = header.subarray(257, 262).toString("latin1") === "ustar" ? field(345, 155) : "";
const size = Number.parseInt(field(124, 12).trim() || "0", 8);
const type = field(156, 1);
const fullName = prefix ? `${prefix}/${name}` : name;
if ((type === "0" || type === "") && fullName === "package/package.json") {
const body = data.subarray(offset + 512, offset + 512 + size);
return JSON.parse(body.toString("utf8")) as Record<string, unknown>;
}
offset += 512 + Math.ceil(size / 512) * 512;
}
throw new Error("The tarball holds no package/package.json.");
}
export interface LocalPackage {
readonly entry: TrainPackage;
readonly path: string;
readonly data: Buffer;
readonly packageJson: Readonly<Record<string, unknown>>;
}
export interface LocalTrain {
readonly manifest: PackageTrainManifest;
/** SHA-256 of `ekklesia-package-train.json`'s exact bytes. */
readonly manifestSha256: string;
readonly packages: readonly LocalPackage[];
}
/**
* Reads and validates a release directory completely, or throws `INVALID_RELEASE` with every
* problem: nothing may be published from a train that is not whole and unchanged.
*/
export async function loadReleaseTrain(
releaseDirectory: string,
expected: { readonly version: string; readonly revision?: string },
): Promise<LocalTrain> {
const files = new Map<string, Buffer>();
for (const file of await readdir(releaseDirectory)) {
files.set(file, await readFile(path.join(releaseDirectory, file)));
}
const manifestBytes = files.get(PACKAGE_TRAIN_MANIFEST);
if (manifestBytes === undefined) {
throw new PublicationError("INVALID_RELEASE", `The release has no ${PACKAGE_TRAIN_MANIFEST}.`);
}
let manifest: PackageTrainManifest;
try {
manifest = JSON.parse(manifestBytes.toString("utf8")) as PackageTrainManifest;
} catch {
throw new PublicationError("INVALID_RELEASE", `${PACKAGE_TRAIN_MANIFEST} is not JSON.`);
}
const problems = trainProblems(manifest, files, expected);
if (problems.length > 0) {
throw new PublicationError("INVALID_RELEASE", "The release train failed validation.", problems);
}
return {
manifest,
manifestSha256: createHash("sha256").update(manifestBytes).digest("hex"),
packages: manifest.packages.map((entry) => {
const data = files.get(entry.file) as Buffer;
return {
data,
entry,
packageJson: tarballPackageJson(data),
path: path.join(releaseDirectory, entry.file),
};
}),
};
}
/** Every reason a release directory's contents may not be published, or nothing. */
export function trainProblems(
manifest: PackageTrainManifest,
files: ReadonlyMap<string, Uint8Array>,
expected: { readonly version: string; readonly revision?: string },
): string[] {
if (
!isRecord(manifest) ||
manifest.packageTrain !== PACKAGE_TRAIN_FORMAT ||
!Array.isArray(manifest.packages) ||
!isRecord(manifest.source)
) {
return ["The release manifest is not a package train manifest."];
}
const problems = releaseDirectoryProblems(manifest, files);
if (manifest.source.modified !== false) {
problems.push("The release was built from a modified source tree; it is never published.");
}
if (manifest.version !== expected.version) {
problems.push(
`The release is version ${String(manifest.version)}, not the requested ${expected.version}.`,
);
}
if (!/^[0-9a-f]{40}([0-9a-f]{24})?$/u.test(String(manifest.source.revision))) {
problems.push("The release names no full source revision.");
}
if (expected.revision !== undefined && manifest.source.revision !== expected.revision) {
problems.push(
`The release was built from ${String(manifest.source.revision)}, not the expected ${expected.revision}.`,
);
}
const sums = files.get("SHA256SUMS");
const expectedSums = [...manifest.packages]
.sort((left, right) => (left.file < right.file ? -1 : 1))
.map((entry) => `${entry.sha256} ${entry.file}\n`)
.join("");
if (sums === undefined || Buffer.from(sums).toString("utf8") !== expectedSums) {
problems.push("SHA256SUMS does not match the release manifest.");
}
for (const entry of manifest.packages) {
const data = files.get(entry.file);
if (data === undefined) continue;
try {
const packageJson = tarballPackageJson(data);
if (packageJson.name !== entry.name || packageJson.version !== entry.version) {
problems.push(
`${entry.file} holds ${String(packageJson.name)}@${String(packageJson.version)}, not ${entry.name}@${entry.version}.`,
);
}
} catch {
problems.push(`${entry.file} is not a readable npm package tarball.`);
}
}
return problems;
}
/**
* The order packages are published in: every package after the train packages it depends on (in
* `dependencies`, `peerDependencies`, or `optionalDependencies`), and otherwise by name, so the order
* never depends on how a directory happens to list its files.
*/
export function publicationOrder<
Package extends {
readonly entry: { readonly name: string };
readonly packageJson: Readonly<Record<string, unknown>>;
},
>(packages: readonly Package[]): Package[] {
const byName = new Map(packages.map((entry) => [entry.entry.name, entry]));
const dependsOn = new Map<string, Set<string>>();
for (const entry of packages) {
const names = new Set<string>();
for (const field of DEPENDENCY_FIELDS) {
const dependencies = entry.packageJson[field];
if (!isRecord(dependencies)) continue;
for (const name of Object.keys(dependencies)) {
if (byName.has(name) && name !== entry.entry.name) names.add(name);
}
}
dependsOn.set(entry.entry.name, names);
}
const order: Package[] = [];
const done = new Set<string>();
while (order.length < packages.length) {
const ready = [...byName.keys()]
.filter((name) => !done.has(name))
.filter((name) =>
[...(dependsOn.get(name) ?? [])].every((dependency) => done.has(dependency)),
)
.sort();
const next = ready[0];
if (next === undefined) {
throw new PublicationError(
"INVALID_RELEASE",
"The release's packages depend on each other in a cycle.",
);
}
done.add(next);
order.push(byName.get(next) as Package);
}
return order;
}
// ---------------------------------------------------------------------------------------------
// Comparing a registry copy with the release
// ---------------------------------------------------------------------------------------------
/** `bin` as npm publishes it: a string becomes `{ <package name without scope>: path }`, and paths
* lose a leading `./`. The tarball keeps the author's form; the registry metadata gets this one. */
export function normalizeBin(bin: unknown, packageName: string): Record<string, string> {
const entries: [string, string][] =
typeof bin === "string"
? [[packageName.replace(/^@[^/]+\//u, ""), bin]]
: isRecord(bin)
? Object.entries(bin).map(([name, target]) => [name, String(target)])
: [];
return Object.fromEntries(
entries
.map(([name, target]): [string, string] => [name, target.replace(/^(?:\.\/)+/u, "")])
.sort(([left], [right]) => (left < right ? -1 : 1)),
);
}
function sortedRecord(value: unknown): Record<string, unknown> {
if (!isRecord(value)) return {};
return Object.fromEntries(
Object.keys(value)
.sort()
.map((key) => [key, value[key]]),
);
}
/** Whether `url` is the registry itself or below it: same origin, same path prefix. */
export function isUnderRegistry(url: string, registry: string): boolean {
try {
const target = new URL(url);
const base = new URL(registry);
return target.origin === base.origin && target.pathname.startsWith(base.pathname);
} catch {
return false;
}
}
/**
* Every way a registry's copy of a package differs from the release artifact, or nothing:
* the version metadata the registry serves, and the bytes it serves for it.
*/
export function registryCopyProblems(
local: Pick<LocalPackage, "entry" | "packageJson" | "data">,
version: Readonly<Record<string, unknown>>,
downloaded: Uint8Array,
registry: string,
): string[] {
const problems: string[] = [];
const { entry, packageJson } = local;
if (version.name !== entry.name || version.version !== entry.version) {
problems.push(`the registry names ${String(version.name)}@${String(version.version)}`);
}
const dist = isRecord(version.dist) ? version.dist : {};
if (dist.integrity !== entry.integrity) {
problems.push("the registry reports a different integrity");
}
const sha1 = createHash("sha1").update(local.data).digest("hex");
if (dist.shasum !== undefined && dist.shasum !== sha1) {
problems.push("the registry reports a different shasum");
}
if (typeof dist.tarball !== "string" || !isUnderRegistry(dist.tarball, registry)) {
problems.push("the registry's tarball URL is not on the configured registry");
}
for (const field of DEPENDENCY_FIELDS) {
if (
JSON.stringify(sortedRecord(version[field])) !==
JSON.stringify(sortedRecord(packageJson[field]))
) {
problems.push(`the registry's ${field} differ from the package's`);
}
}
if (
JSON.stringify(normalizeBin(version.bin, entry.name)) !==
JSON.stringify(normalizeBin(packageJson.bin, entry.name))
) {
problems.push("the registry's bin differs from the package's");
}
const digests = artifactDigests(downloaded);
if (
!Buffer.from(downloaded).equals(local.data) ||
digests.sha256 !== entry.sha256 ||
digests.integrity !== entry.integrity
) {
problems.push("the registry serves different bytes from the release artifact");
}
return problems;
}
/** SemVer 2.0.0 precedence: negative, zero, or positive. Build metadata is ignored. */
export function compareSemver(left: string, right: string): number {
const parse = (version: string) => {
const match = /^(\d+)\.(\d+)\.(\d+)(?:-([0-9A-Za-z.-]+))?(?:\+[0-9A-Za-z.-]+)?$/u.exec(version);
if (match === null) throw new Error(`Not a semantic version: ${version}`);
return {
core: [Number(match[1]), Number(match[2]), Number(match[3])],
pre: match[4] === undefined ? [] : match[4].split("."),
};
};
const a = parse(left);
const b = parse(right);
for (let index = 0; index < 3; index += 1) {
const difference = (a.core[index] ?? 0) - (b.core[index] ?? 0);
if (difference !== 0) return Math.sign(difference);
}
if (a.pre.length === 0 || b.pre.length === 0) return Math.sign(b.pre.length - a.pre.length);
for (let index = 0; index < Math.max(a.pre.length, b.pre.length); index += 1) {
const x = a.pre[index];
const y = b.pre[index];
if (x === undefined || y === undefined) return x === undefined ? -1 : 1;
if (x === y) continue;
const xNumber = /^\d+$/u.test(x);
const yNumber = /^\d+$/u.test(y);
if (xNumber && yNumber) return Math.sign(Number(x) - Number(y));
if (xNumber !== yNumber) return xNumber ? -1 : 1;
return x < y ? -1 : 1;
}
return 0;
}
// ---------------------------------------------------------------------------------------------
// The registry: read over HTTP, written with npm
// ---------------------------------------------------------------------------------------------
/** The registry URL, validated, with exactly one trailing slash. */
export function normalizeRegistry(registry: string, allowHttp = false): string {
let url: URL;
try {
url = new URL(registry);
} catch {
throw new PublicationError("INVALID_OPTIONS", "The registry is not a URL.");
}
if (url.username !== "" || url.password !== "" || url.search !== "" || url.hash !== "") {
throw new PublicationError(
"INVALID_OPTIONS",
"The registry URL may hold no credentials, query, or fragment.",
);
}
if (url.protocol !== "https:" && !(allowHttp && url.protocol === "http:")) {
throw new PublicationError("INVALID_OPTIONS", "The registry must be reached over HTTPS.");
}
url.pathname = url.pathname.endsWith("/") ? url.pathname : `${url.pathname}/`;
return url.toString();
}
/** The npm configuration a publish uses: the scope and registry, and the token by reference. */
export function npmConfiguration(registry: string): string {
const url = new URL(registry);
return [
`registry=${registry}`,
`@ekklesia:registry=${registry}`,
`//${url.host}${url.pathname}:_authToken=\${${PUBLISH_TOKEN_VARIABLE}}`,
"",
].join("\n");
}
/** `@ekklesia/module-sdk` → `<registry>@ekklesia%2fmodule-sdk`, the npm packument path. */
export function packumentUrl(registry: string, name: string): string {
return `${registry}${name.replace("/", "%2f")}`;
}
export interface RegistryReader {
/** The package's packument, or `undefined` when the registry has no such package. */
packument(name: string): Promise<Record<string, unknown> | undefined>;
/** A tarball's bytes; authenticated only when the URL is on the registry itself. */
download(url: string, anonymous?: boolean): Promise<Buffer>;
}
export function registryReader(
registry: string,
token: string | undefined,
fetchImplementation: typeof fetch = fetch,
): RegistryReader {
const headers = (url: string, anonymous: boolean): Record<string, string> =>
!anonymous && token && isUnderRegistry(url, registry)
? { authorization: `Bearer ${token}` }
: {};
const get = async (url: string, accept: string, anonymous: boolean) => {
try {
return await fetchImplementation(url, {
headers: { accept, ...headers(url, anonymous) },
redirect: "error",
});
} catch {
throw new PublicationError("REGISTRY_UNAVAILABLE", "The registry could not be reached.");
}
};
return {
async download(url, anonymous = false) {
const response = await get(url, "application/octet-stream", anonymous);
if (!response.ok) {
throw new PublicationError(
"REGISTRY_UNAVAILABLE",
`The registry answered ${response.status} for a package tarball.`,
);
}
return Buffer.from(await response.arrayBuffer());
},
async packument(name) {
const response = await get(packumentUrl(registry, name), "application/json", false);
if (response.status === 404) return undefined;
if (!response.ok) {
throw new PublicationError(
"REGISTRY_UNAVAILABLE",
`The registry answered ${response.status} for ${name}.`,
);
}
return (await response.json()) as Record<string, unknown>;
},
};
}
function versionOf(
packument: Record<string, unknown> | undefined,
version: string,
): Record<string, unknown> | undefined {
const versions = packument?.versions;
const found = isRecord(versions) ? versions[version] : undefined;
return isRecord(found) ? { ...found } : undefined;
}
function distTagsOf(packument: Record<string, unknown> | undefined): Record<string, string> {
const tags = packument?.["dist-tags"];
return isRecord(tags)
? Object.fromEntries(Object.entries(tags).map(([tag, version]) => [tag, String(version)]))
: {};
}
interface NpmSession {
publish(file: string, tag: string): Promise<void>;
addTag(name: string, version: string, tag: string): Promise<void>;
dispose(): Promise<void>;
}
/** npm with only a temporary configuration: no user or global npmrc, no project directory. */
async function npmSession(registry: string, token: string, npm: string): Promise<NpmSession> {
const home = await mkdtemp(path.join(tmpdir(), "ekklesia-publish-"));
const userconfig = path.join(home, "npmrc");
const globalconfig = path.join(home, "global-npmrc");
await writeFile(userconfig, npmConfiguration(registry), { mode: 0o600 });
await writeFile(globalconfig, "", { mode: 0o600 });
const environment = {
[PUBLISH_TOKEN_VARIABLE]: token,
HOME: home,
PATH: process.env.PATH ?? "",
npm_config_audit: "false",
npm_config_cache: path.join(home, "cache"),
npm_config_fund: "false",
npm_config_globalconfig: globalconfig,
npm_config_update_notifier: "false",
npm_config_userconfig: userconfig,
};
const run = async (args: readonly string[], what: string) => {
try {
await execFileAsync(npm, [...args], { cwd: home, env: environment, maxBuffer: 16 << 20 });
} catch (error) {
const stderr = String((error as { stderr?: unknown }).stderr ?? "");
const summary = stderr
.split("\n")
.filter((line) => /^npm error (?:code|\d{3}|[A-Z])/u.test(line))
.filter((line) => !line.includes("complete log"))
.slice(0, 3)
.join(" / ")
.replaceAll(token, "***")
.replaceAll(home, "<npm home>");
throw new PublicationError(
"PUBLISH_FAILED",
`${what} failed${summary ? `: ${summary}` : "."}`,
);
}
};
return {
addTag: (name, version, tag) =>
run(["dist-tag", "add", `${name}@${version}`, tag], `Moving ${tag} to ${name}@${version}`),
dispose: () => rm(home, { force: true, recursive: true }),
publish: (file, tag) =>
run(
// A file spec runs no lifecycle script and sends the file's own bytes; --ignore-scripts
// says so explicitly. --access public is what a scoped package needs on a public registry.
["publish", file, "--tag", tag, "--access", "public", "--ignore-scripts"],
`Publishing ${path.basename(file)}`,
),
};
}
// ---------------------------------------------------------------------------------------------
// Publishing a train
// ---------------------------------------------------------------------------------------------
export type PackageOutcome = "published" | "already-present";
export type TagOutcome = "moved" | "already-current" | "kept-newer";
export interface PublicationHooks {
/** Called before each package is published (not for one already present). Tests use it to
* interrupt a train part way. */
beforePublish?(name: string, index: number): void | Promise<void>;
/** Called before each final dist-tag is moved. */
beforeTag?(name: string, index: number): void | Promise<void>;
}
export interface PublishOptions {
readonly releaseDirectory: string;
readonly registry: string;
readonly version: string;
readonly revision?: string;
/** SHA-256 of the release manifest the package-release gate proved, which binds the artifact a
* publish job received to that gate. */
readonly manifestSha256?: string;
readonly token: string;
readonly stagingTag?: string;
readonly finalTag?: string;
readonly requirePublicRead?: boolean;
/** Only for a disposable test registry. */
readonly allowHttp?: boolean;
readonly npm?: string;
readonly fetch?: typeof fetch;
readonly hooks?: PublicationHooks;
readonly log?: (line: string) => void;
readonly now?: () => Date;
}
export interface PublishedPackage {
readonly name: string;
readonly version: string;
readonly file: string;
readonly outcome: PackageOutcome;
readonly artifact: { readonly sha256: string; readonly integrity: string; readonly size: number };
readonly registry: {
readonly tarball: string;
readonly sha256: string;
readonly integrity: string;
readonly publicRead: boolean;
};
readonly finalTag: TagOutcome;
readonly distTags: Readonly<Record<string, string>>;
}
/** Evidence that one train was published: not a package catalog, and never a secret. */
export interface PublicationRecord {
readonly publicationRecord: typeof PUBLICATION_RECORD_FORMAT;
readonly packageTrain: {
readonly version: string;
readonly revision: string;
readonly manifestSha256: string;
};
readonly registry: string;
readonly finalTag: string;
readonly completedAt: string;
readonly packages: readonly PublishedPackage[];
}
export function serializeRecord(record: PublicationRecord): string {
return `${JSON.stringify(record, null, 2)}\n`;
}
const TOKEN_FORMAT = /^[\x21-\x7e]{8,}$/u;
export async function publishPackageTrain(options: PublishOptions): Promise<PublicationRecord> {
const log = options.log ?? (() => {});
const stagingTag = options.stagingTag ?? DEFAULT_STAGING_TAG;
const finalTag = options.finalTag ?? DEFAULT_FINAL_TAG;
if (!TOKEN_FORMAT.test(options.token)) {
throw new PublicationError(
"INVALID_OPTIONS",
`${PUBLISH_TOKEN_VARIABLE} must hold the publish token.`,
);
}
for (const tag of [stagingTag, finalTag]) {
if (!/^[a-z][a-z0-9-]*$/u.test(tag)) {
throw new PublicationError("INVALID_OPTIONS", `${tag} is not a usable dist-tag.`);
}
}
if (stagingTag === finalTag) {
throw new PublicationError("INVALID_OPTIONS", "The staging and final tags must differ.");
}
const registry = normalizeRegistry(options.registry, options.allowHttp);
// Nothing is sent until the whole local train is valid.
const train = await loadReleaseTrain(options.releaseDirectory, {
version: options.version,
...(options.revision === undefined ? {} : { revision: options.revision }),
});
if (options.manifestSha256 !== undefined && options.manifestSha256 !== train.manifestSha256) {
throw new PublicationError(
"INVALID_RELEASE",
"The release manifest is not the one the package-release gate proved.",
);
}
const order = publicationOrder(train.packages);
log(
`Package train ${train.manifest.version} (${train.manifest.source.revision}) validated: ${order
.map((entry) => entry.entry.name)
.join(", ")}`,
);
const reader = registryReader(registry, options.token, options.fetch);
const npm = await npmSession(registry, options.token, options.npm ?? "npm");
try {
const verified: {
local: LocalPackage;
outcome: PackageOutcome;
tarball: string;
publicRead: boolean;
downloaded: { sha256: string; integrity: string };
}[] = [];
for (const [index, local] of order.entries()) {
const { name, version } = local.entry;
let outcome: PackageOutcome = "already-present";
let existing = versionOf(await reader.packument(name), version);
if (existing === undefined) {
await options.hooks?.beforePublish?.(name, index);
try {
await npm.publish(local.path, stagingTag);
outcome = "published";
} catch (error) {
// Someone may have published it meanwhile; what is there now decides.
if (versionOf(await reader.packument(name), version) === undefined) throw error;
}
existing = versionOf(await reader.packument(name), version);
if (existing === undefined) {
throw new PublicationError(
"VERIFICATION_FAILED",
`${name}@${version} is not on the registry after publishing it.`,
);
}
}
const tarball = isRecord(existing.dist) ? String(existing.dist.tarball) : "";
if (!isUnderRegistry(tarball, registry)) {
throw new PublicationError(
"VERIFICATION_FAILED",
`${name}@${version}'s tarball is not on the configured registry.`,
);
}
const downloaded = await reader.download(tarball);
if (!downloaded.equals(local.data)) {
throw new PublicationError(
"BURNED_VERSION",
`${name}@${version} is already on the registry with different bytes. It is never deleted or replaced: the version is burned, publication stops here, and the train needs a new version after an investigation.`,
);
}
const problems = registryCopyProblems(local, existing, downloaded, registry);
if (problems.length > 0) {
throw new PublicationError(
"VERIFICATION_FAILED",
`${name}@${version} does not match the release artifact.`,
problems,
);
}
const publicRead = await reader
.download(tarball, true)
.then((anonymous) => anonymous.equals(local.data))
.catch(() => false);
if (options.requirePublicRead === true && !publicRead) {
throw new PublicationError(
"VERIFICATION_FAILED",
`${name}@${version} cannot be downloaded anonymously.`,
);
}
log(`${name}@${version}: ${outcome}, verified byte for byte`);
verified.push({
downloaded: artifactDigests(downloaded),
local,
outcome,
publicRead,
tarball,
});
}
// Every package is there and verified: only now does the final tag move.
const tagOutcomes = new Map<string, TagOutcome>();
for (const [index, { local }] of verified.entries()) {
const { name, version } = local.entry;
const current = distTagsOf(await reader.packument(name))[finalTag];
if (current === version) {
tagOutcomes.set(name, "already-current");
} else if (current !== undefined && compareSemver(current, version) > 0) {
tagOutcomes.set(name, "kept-newer");
} else {
await options.hooks?.beforeTag?.(name, index);
await npm.addTag(name, version, finalTag);
tagOutcomes.set(name, "moved");
}
log(`${name}: ${finalTag} ${tagOutcomes.get(name)}`);
}
const packages: PublishedPackage[] = [];
const unconverged: string[] = [];
for (const { downloaded, local, outcome, publicRead, tarball } of verified) {
const { name, version } = local.entry;
const distTags = distTagsOf(await reader.packument(name));
const finalVersion = distTags[finalTag];
const converged =
finalVersion === version ||
(tagOutcomes.get(name) === "kept-newer" &&
finalVersion !== undefined &&
compareSemver(finalVersion, version) > 0);
if (!converged) unconverged.push(`${name}: ${finalTag} is ${finalVersion ?? "unset"}`);
packages.push({
artifact: {
integrity: local.entry.integrity,
sha256: local.entry.sha256,
size: local.entry.size,
},
distTags: Object.fromEntries(Object.entries(distTags).sort()),
file: local.entry.file,
finalTag: tagOutcomes.get(name) ?? "moved",
name,
outcome,
registry: { ...downloaded, publicRead, tarball },
version,
});
}
if (unconverged.length > 0) {
throw new PublicationError(
"TAGS_NOT_CONVERGED",
"The final dist-tags did not converge.",
unconverged,
);
}
return {
completedAt: (options.now ?? (() => new Date()))().toISOString(),
finalTag,
packageTrain: {
manifestSha256: train.manifestSha256,
revision: train.manifest.source.revision,
version: train.manifest.version,
},
packages,
publicationRecord: PUBLICATION_RECORD_FORMAT,
registry,
};
} finally {
await npm.dispose();
}
}
// ---------------------------------------------------------------------------------------------
// Command line
// ---------------------------------------------------------------------------------------------
const VALUE_OPTIONS = Object.freeze({
"--final-tag": "finalTag",
"--manifest-sha256": "manifestSha256",
"--record": "record",
"--registry": "registry",
"--release": "release",
"--revision": "revision",
"--staging-tag": "stagingTag",
"--version": "version",
} as const);
const FLAG_OPTIONS = Object.freeze({
"--allow-http": "allowHttp",
"--require-public-read": "requirePublicRead",
} as const);
const USAGE = `Usage: ${PUBLISH_TOKEN_VARIABLE}=… node scripts/publish-package-train.ts --release <directory> --registry <url> --version <version> [--revision <commit>] [--manifest-sha256 <hex>] [--record <file>] [--staging-tag <tag>] [--final-tag <tag>] [--require-public-read]`;
export function parsePublishArguments(argv: readonly string[]): {
values: Partial<Record<(typeof VALUE_OPTIONS)[keyof typeof VALUE_OPTIONS], string>>;
flags: Partial<Record<(typeof FLAG_OPTIONS)[keyof typeof FLAG_OPTIONS], true>>;
} {
const values: Partial<Record<(typeof VALUE_OPTIONS)[keyof typeof VALUE_OPTIONS], string>> = {};
const flags: Partial<Record<(typeof FLAG_OPTIONS)[keyof typeof FLAG_OPTIONS], true>> = {};
for (let index = 0; index < argv.length; index += 1) {
const argument = argv[index] ?? "";
if (Object.hasOwn(FLAG_OPTIONS, argument)) {
flags[FLAG_OPTIONS[argument as keyof typeof FLAG_OPTIONS]] = true;
continue;
}
const key = Object.hasOwn(VALUE_OPTIONS, argument)
? VALUE_OPTIONS[argument as keyof typeof VALUE_OPTIONS]
: undefined;
const value = argv[index + 1];
if (key === undefined || value === undefined || value.startsWith("--") || key in values) {
throw new PublicationError("INVALID_OPTIONS", USAGE);
}
values[key] = value;
index += 1;
}
if (!values.release || !values.registry || !values.version) {
throw new PublicationError("INVALID_OPTIONS", USAGE);
}
return { flags, values };
}
async function main(argv: readonly string[]): Promise<number> {
try {
const { flags, values } = parsePublishArguments(argv);
const record = await publishPackageTrain({
allowHttp: flags.allowHttp === true,
log: (line) => process.stdout.write(`${line}\n`),
registry: values.registry ?? "",
releaseDirectory: values.release ?? "",
requirePublicRead: flags.requirePublicRead === true,
token: process.env[PUBLISH_TOKEN_VARIABLE] ?? "",
version: values.version ?? "",
...(values.revision === undefined ? {} : { revision: values.revision }),
...(values.manifestSha256 === undefined ? {} : { manifestSha256: values.manifestSha256 }),
...(values.stagingTag === undefined ? {} : { stagingTag: values.stagingTag }),
...(values.finalTag === undefined ? {} : { finalTag: values.finalTag }),
});
if (values.record !== undefined) await writeFile(values.record, serializeRecord(record));
process.stdout.write(
`Published package train ${record.packageTrain.version}: ${record.packages.length} packages verified, ${record.finalTag} converged.\n`,
);
return 0;
} catch (error) {
if (error instanceof PublicationError) {
process.stderr.write(`${error.message}\n`);
for (const problem of error.problems) process.stderr.write(` ${problem}\n`);
if (error.code === "INVALID_OPTIONS") return 64;
return error.code === "BURNED_VERSION" ? 3 : 1;
}
process.stderr.write("Publishing the package train failed unexpectedly.\n");
return 1;
}
}
if (process.argv[1] && import.meta.url === pathToFileURL(process.argv[1]).href) {
process.exitCode = await main(process.argv.slice(2));
}
+136
View File
@@ -0,0 +1,136 @@
#!/usr/bin/env bash
# The package train publication proof (ADR 0053), against a disposable Gitea of the same version as
# the real registry, never the real one: the publication tool publishes one exact release train,
# and the suite proves the round trip, partial-train recovery, the hard stop on a burned version,
# dist-tag convergence, and that the registry's public owner serves packages anonymously while only
# the publisher team may publish.
#
# bash scripts/test-package-publication.sh [--release <release directory>]
#
# With --release, it publishes that release (CI hands it the package-release job's artifact).
# Without, it builds one from a worktree of HEAD, so local changes to the tool are tested against a
# release of the committed packages.
set -Eeuo pipefail
repository_root="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")/.." && pwd)"
cd "$repository_root"
# shellcheck source=scripts/lib/compose-job-network.sh
source "$repository_root/scripts/lib/compose-job-network.sh"
# shellcheck source=scripts/lib/compose-diagnostics.sh
source "$repository_root/scripts/lib/compose-diagnostics.sh"
# shellcheck source=scripts/lib/ci-project.sh
source "$repository_root/scripts/lib/ci-project.sh"
fail() {
echo "package publication proof failed: $*" >&2
exit 1
}
release=""
while (($#)); do
case "$1" in
--release)
release="${2:-}"
shift 2 || fail "--release needs a directory"
;;
*) fail "usage: test-package-publication.sh [--release <release directory>]" ;;
esac
done
command -v docker >/dev/null || fail "Docker is required"
docker compose version >/dev/null 2>&1 || fail "Docker Compose is required"
compose_job_network_initialize || fail "the current Docker job container could not be identified"
project="$(ci_project_name package-registry)"
work="$(mktemp -d "${TMPDIR:-/tmp}/ekklesia-package-publication-XXXXXX")"
worktree="$work/release-checkout"
if [[ "$COMPOSE_JOB_NETWORK_RUNNING_IN_CONTAINER" == "true" ]]; then
export GITEA_PORT="0" GITEA_ROOT_URL="http://gitea:3000/"
else
GITEA_PORT="$(node -e 'const s = require("net").createServer().listen(0, "127.0.0.1", () => { console.log(s.address().port); s.close(); })')"
export GITEA_PORT GITEA_ROOT_URL="http://127.0.0.1:$GITEA_PORT/"
fi
compose=(docker compose --project-name "$project" --file scripts/lib/package-registry.compose.yaml)
cleanup() {
local exit_status=$?
trap - EXIT INT TERM
compose_job_network_disconnect || exit_status=1
"${compose[@]}" down --volumes --remove-orphans --timeout 10 >/dev/null 2>&1 || exit_status=1
if [[ -d "$worktree" ]]; then
git -C "$repository_root" worktree remove --force "$worktree" >/dev/null 2>&1 || true
fi
rm -rf "$work"
exit "$exit_status"
}
trap cleanup EXIT
trap 'exit 130' INT
trap 'exit 143' TERM
if [[ -z "$release" ]]; then
echo "==> Building a release of HEAD to publish"
store="$(dirname "$(pnpm store path)")"
git worktree add --detach --quiet "$worktree" HEAD
(cd "$worktree" && pnpm install --offline --frozen-lockfile --store-dir "$store" --reporter=silent)
node "$worktree/scripts/package-release.ts" --out "$work/release" >/dev/null
release="$(find "$work/release" -mindepth 1 -maxdepth 1 -type d)"
fi
release="$(cd "$release" && pwd)"
# Masked in any Compose diagnostics below.
COMPOSE_DIAGNOSTICS_SECRETS=()
echo "==> Starting a disposable Gitea 1.27.2"
"${compose[@]}" config --quiet || fail "the disposable registry's configuration is invalid"
if ! up_output="$("${compose[@]}" up --detach --wait --wait-timeout 180 gitea 2>&1)"; then
compose_report_not_ready "$up_output" "${compose[@]}"
fail "the disposable Gitea did not become ready"
fi
if [[ "$COMPOSE_JOB_NETWORK_RUNNING_IN_CONTAINER" == "true" ]]; then
compose_job_network_connect gitea "${compose[@]}" || fail "the job could not join the registry's network"
fi
origin="${GITEA_ROOT_URL%/}"
admin_password="admin-$RANDOM$RANDOM$RANDOM-$RANDOM"
publisher_password="publisher-$RANDOM$RANDOM$RANDOM-$RANDOM"
outsider_password="outsider-$RANDOM$RANDOM$RANDOM-$RANDOM"
"${compose[@]}" exec -T -u git gitea gitea admin user create --admin --username ekklesia-admin \
--password "$admin_password" --email [email protected] --must-change-password=false >/dev/null
api() {
local method="$1" path="$2" body="${3:-}" auth="${4:-ekklesia-admin:$admin_password}"
curl -fsS -X "$method" -u "$auth" -H "Content-Type: application/json" \
${body:+--data "$body"} "$origin/api/v1$path"
}
echo "==> Creating public package owners, the publisher team, and an outsider"
for user in publisher outsider; do
password_variable="${user}_password"
api POST /admin/users "{\"username\":\"$user\",\"password\":\"${!password_variable}\",\"email\":\"$user@example.invalid\",\"must_change_password\":false}" >/dev/null
done
owners=(River_of_Glory_Church ekklesia-burned-version ekklesia-tag-retry)
for owner in "${owners[@]}"; do
api POST /orgs "{\"username\":\"$owner\",\"visibility\":\"public\"}" >/dev/null
team="$(api POST "/orgs/$owner/teams" '{"name":"package-publishers","includes_all_repositories":false,"can_create_org_repo":false,"units":["repo.packages"],"units_map":{"repo.packages":"write"}}' |
node -e 'process.stdout.write(String(JSON.parse(require("fs").readFileSync(0, "utf8")).id))')"
api PUT "/teams/$team/members/publisher" >/dev/null
done
token_for() {
local user="$1" password="$2"
api POST "/users/$user/tokens" "{\"name\":\"publish-$RANDOM\",\"scopes\":[\"write:package\"]}" "$user:$password" |
node -e 'process.stdout.write(JSON.parse(require("fs").readFileSync(0, "utf8")).sha1)'
}
publish_token="$(token_for publisher "$publisher_password")"
outsider_token="$(token_for outsider "$outsider_password")"
echo "==> Publishing the release train to it"
EKKLESIA_TEST_REGISTRY_ORIGIN="$origin" \
EKKLESIA_TEST_RELEASE="$release" \
EKKLESIA_TEST_PUBLISH_TOKEN="$publish_token" \
EKKLESIA_TEST_OUTSIDER_TOKEN="$outsider_token" \
EKKLESIA_TEST_OWNERS="${owners[*]}" \
pnpm exec vitest run --config vitest.packages.integration.config.ts
echo "Package publication proof passed."
+28 -8
View File
@@ -2,7 +2,7 @@
# The package release gate (ADR 0053): what must pass before a future step may publish a package
# train. It publishes nothing.
#
# bash scripts/test-package-release.sh [--allow-modified]
# bash scripts/test-package-release.sh [--allow-modified] [--keep <directory>]
#
# 1. Builds the public package train from this checkout (`scripts/package-release.ts`): clean build
# output, pack, inspect, release manifest.
@@ -20,13 +20,24 @@
set -euo pipefail
repository_root="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)"
allow_modified=()
if [[ "${1:-}" == "--allow-modified" ]]; then
allow_modified=(--allow-modified)
elif (($#)); then
echo "usage: test-package-release.sh [--allow-modified]" >&2
exit 64
fi
allow_modified=() keep=""
while (($#)); do
case "$1" in
--allow-modified)
allow_modified=(--allow-modified)
shift
;;
--keep)
keep="${2:-}"
[[ -n "$keep" ]] || { echo "usage: --keep <directory>" >&2; exit 64; }
shift 2
;;
*)
echo "usage: test-package-release.sh [--allow-modified] [--keep <directory>]" >&2
exit 64
;;
esac
done
# The workspace install's own pnpm store. pnpm chooses a store per file system, so an install in a
# temporary directory would otherwise look in another, empty one; every install here is offline.
@@ -69,4 +80,13 @@ echo "==> Building Core's verifier"
echo "==> Consuming the release from a standalone module"
bash "$repository_root/scripts/test-standalone-module.sh" --release "$release" "${allow_modified[@]}"
# The release this gate proved, for a later job to publish exactly (`--keep`): copied only now, after
# every check passed, into a directory that must not hold an earlier one.
if [[ -n "$keep" ]]; then
mkdir -p "$keep"
[[ ! -e "$keep/$(basename "$release")" ]] || { echo "$keep already holds this release" >&2; exit 1; }
cp -R "$release" "$keep/"
echo " kept in $keep/$(basename "$release")"
fi
echo "Package release gate passed."
+8 -2
View File
@@ -97,12 +97,18 @@ describe("CI scripts on a shared Docker daemon", () => {
>;
};
const startsStacks =
/smoke-infrastructure\.sh|test:migrations:integration|smoke-deploy\.sh|check-image\.sh|docker build/u;
/smoke-infrastructure\.sh|test:migrations:integration|smoke-deploy\.sh|check-image\.sh|test-package-publication\.sh|docker build/u;
const jobs = Object.entries(workflow.jobs).filter(([, job]) =>
job.steps.some((step) => startsStacks.test(step.run ?? "")),
);
expect(jobs.map(([id]) => id).sort()).toEqual(
["deploy-smoke", "image-build", "infrastructure-smoke", "migration-smoke"].sort(),
[
"deploy-smoke",
"image-build",
"infrastructure-smoke",
"migration-smoke",
"package-publication",
].sort(),
);
for (const [id, job] of jobs) {
expect(job.env?.EKKLESIA_CI_JOB_KEY, id).toBeTruthy();
+305
View File
@@ -0,0 +1,305 @@
import { execFile } from "node:child_process";
import { mkdtemp, readFile, rm, writeFile } from "node:fs/promises";
import { tmpdir } from "node:os";
import path from "node:path";
import { promisify } from "node:util";
import { afterAll, beforeAll, describe, expect, it } from "vitest";
import {
type LocalTrain,
loadReleaseTrain,
normalizeRegistry,
type PublicationRecord,
PUBLISH_TOKEN_VARIABLE,
packumentUrl,
publicationOrder,
publishPackageTrain,
} from "../scripts/publish-package-train.js";
/**
* The publication tool against a disposable Gitea 1.27.2 (scripts/test-package-publication.sh),
* configured as the real registry is meant to be: public package owners anyone may read from, and a
* publisher team that alone may publish. Each scenario uses its own owner, so they never interfere.
*/
const execFileAsync = promisify(execFile);
/** Set by scripts/test-package-publication.sh for the disposable registry it started. */
const setting = (name: string): string => process.env[name] ?? "";
const origin = setting("EKKLESIA_TEST_REGISTRY_ORIGIN");
const releaseDirectory = setting("EKKLESIA_TEST_RELEASE");
const publishToken = setting("EKKLESIA_TEST_PUBLISH_TOKEN");
const outsiderToken = setting("EKKLESIA_TEST_OUTSIDER_TOKEN");
const [mainOwner, burnedOwner, tagOwner] = setting("EKKLESIA_TEST_OWNERS").split(" ");
if (!origin || !releaseDirectory || !publishToken || !outsiderToken || !tagOwner) {
throw new Error("Run this suite with scripts/test-package-publication.sh.");
}
const registryOf = (owner: string | undefined) =>
normalizeRegistry(`${origin}/api/packages/${owner}/npm/`, true);
let train: LocalTrain;
let order: string[];
const scratch: string[] = [];
beforeAll(async () => {
const manifest = JSON.parse(
await readFile(path.join(releaseDirectory, "ekklesia-package-train.json"), "utf8"),
) as { version: string; source: { revision: string } };
train = await loadReleaseTrain(releaseDirectory, {
revision: manifest.source.revision,
version: manifest.version,
});
order = publicationOrder(train.packages).map((entry) => entry.entry.name);
});
afterAll(async () => {
await Promise.all(scratch.splice(0).map((dir) => rm(dir, { force: true, recursive: true })));
});
function publish(
owner: string | undefined,
hooks?: Parameters<typeof publishPackageTrain>[0]["hooks"],
) {
return publishPackageTrain({
allowHttp: true,
registry: registryOf(owner),
releaseDirectory,
requirePublicRead: true,
revision: train.manifest.source.revision,
token: publishToken,
version: train.manifest.version,
...(hooks === undefined ? {} : { hooks }),
});
}
/** What anyone, signed in or not, gets for a package: its packument, or undefined on 404. */
async function anonymousPackument(owner: string | undefined, name: string) {
const response = await fetch(packumentUrl(registryOf(owner), name));
if (response.status === 404) return undefined;
expect(response.status).toBe(200);
return (await response.json()) as {
versions: Record<string, { dist: { tarball: string; integrity: string } }>;
"dist-tags": Record<string, string>;
};
}
async function anonymousBytes(url: string): Promise<Buffer> {
const response = await fetch(url);
expect(response.status).toBe(200);
return Buffer.from(await response.arrayBuffer());
}
function local(name: string) {
const found = train.packages.find((entry) => entry.entry.name === name);
if (!found) throw new Error(name);
return found;
}
/** `npm publish <file>` outside the tool, as an operator or an attacker might run it. */
async function npmPublish(owner: string | undefined, file: string, token: string): Promise<string> {
const home = await mkdtemp(path.join(tmpdir(), "ekklesia-npm-"));
scratch.push(home);
const registry = registryOf(owner);
const url = new URL(registry);
await writeFile(
path.join(home, "npmrc"),
`registry=${registry}\n//${url.host}${url.pathname}:_authToken=\${${PUBLISH_TOKEN_VARIABLE}}\n`,
);
try {
await execFileAsync(
"npm",
["publish", file, "--tag", "manual", "--access", "public", "--ignore-scripts"],
{
cwd: home,
env: {
[PUBLISH_TOKEN_VARIABLE]: token,
HOME: home,
PATH: process.env.PATH ?? "",
npm_config_globalconfig: path.join(home, "none"),
npm_config_userconfig: path.join(home, "npmrc"),
},
},
);
return "published";
} catch (error) {
return String((error as { stderr?: unknown }).stderr ?? "failed");
}
}
describe("the registry's access model", () => {
it("lets anyone search and read package metadata, with no account at all", async () => {
const search = await fetch(`${registryOf(mainOwner)}-/v1/search?text=ekklesia`);
expect(search.status).toBe(200);
// An absent package is 404 to an anonymous reader, not 401: reading is open.
expect(await anonymousPackument(mainOwner, "@ekklesia/contracts")).toBeUndefined();
});
it("refuses publishing without credentials, and from a user outside the publisher team", async () => {
const contracts = local("@ekklesia/contracts").path;
expect(await npmPublish(mainOwner, contracts, "")).toMatch(/ENEEDAUTH|E401|401/u);
expect(await npmPublish(mainOwner, contracts, outsiderToken)).toMatch(/E401|E403|40[13]/u);
expect(await anonymousPackument(mainOwner, "@ekklesia/contracts")).toBeUndefined();
});
});
describe("publishing a train, interrupted and resumed", () => {
let record: PublicationRecord;
it("run 1: stops before the sixth package, with five published and no final tag", async () => {
await expect(
publish(mainOwner, {
beforePublish: (_name, index) => {
if (index === 5) throw new Error("injected failure before the sixth package");
},
}),
).rejects.toThrow("injected failure");
for (const [index, name] of order.entries()) {
const packument = await anonymousPackument(mainOwner, name);
if (index < 5) {
const version = packument?.versions[train.manifest.version];
expect(version, name).toBeDefined();
expect(packument?.["dist-tags"].latest, name).toBeUndefined();
expect(packument?.["dist-tags"]["ekklesia-staging"], name).toBe(train.manifest.version);
expect((await anonymousBytes(version?.dist.tarball ?? "")).equals(local(name).data)).toBe(
true,
);
} else {
expect(packument, name).toBeUndefined();
}
}
});
it("run 2: verifies the five as they are, publishes the other four, and only then moves latest", async () => {
record = await publish(mainOwner);
expect(record.packages.map((entry) => entry.name)).toEqual(order);
expect(record.packages.map((entry) => entry.outcome)).toEqual([
...Array(5).fill("already-present"),
...Array(4).fill("published"),
]);
expect(record.packages.every((entry) => entry.finalTag === "moved")).toBe(true);
expect(record.packageTrain).toEqual({
manifestSha256: train.manifestSha256,
revision: train.manifest.source.revision,
version: train.manifest.version,
});
expect(JSON.stringify(record)).not.toContain(publishToken);
});
it("serves every package anonymously, byte for byte the release artifact", async () => {
for (const name of order) {
const packument = await anonymousPackument(mainOwner, name);
const version = packument?.versions[train.manifest.version];
expect(packument?.["dist-tags"].latest, name).toBe(train.manifest.version);
expect(version?.dist.integrity, name).toBe(local(name).entry.integrity);
expect(
(await anonymousBytes(version?.dist.tarball ?? "")).equals(local(name).data),
name,
).toBe(true);
const entry = record.packages.find((candidate) => candidate.name === name);
expect(entry?.registry.sha256).toBe(local(name).entry.sha256);
expect(entry?.registry.publicRead).toBe(true);
}
});
it("run 3: a rerun of a published train changes nothing", async () => {
const again = await publish(mainOwner);
expect(again.packages.every((entry) => entry.outcome === "already-present")).toBe(true);
expect(again.packages.every((entry) => entry.finalTag === "already-current")).toBe(true);
});
it("the registry itself refuses a second upload of a published version", async () => {
const name = "@ekklesia/module-sdk";
const packument = await anonymousPackument(mainOwner, name);
const tarballUrl = packument?.versions[train.manifest.version]?.dist.tarball ?? "";
// A raw upload, past npm's own client-side check, with the publisher's own token.
const { data, entry, packageJson } = local(name);
const response = await fetch(packumentUrl(registryOf(mainOwner), name), {
body: JSON.stringify({
_attachments: {
[path.basename(new URL(tarballUrl).pathname)]: {
content_type: "application/octet-stream",
data: data.toString("base64"),
length: data.length,
},
},
_id: name,
"dist-tags": { manual: entry.version },
name,
versions: { [entry.version]: { ...packageJson, dist: { integrity: entry.integrity } } },
}),
headers: { authorization: `Bearer ${publishToken}`, "content-type": "application/json" },
method: "PUT",
});
expect(response.status).toBe(409);
expect((await anonymousBytes(tarballUrl)).equals(data)).toBe(true);
});
});
describe("a version already published with other bytes", () => {
it("stops the train for good at that package, deleting and overwriting nothing", async () => {
const poisonedName = "@ekklesia/module-sdk";
const index = order.indexOf(poisonedName);
expect(index).toBeGreaterThan(0);
expect(index).toBeLessThan(order.length - 1);
// Same name and version, other bytes: the release tarball with one extra file.
const work = await mkdtemp(path.join(tmpdir(), "ekklesia-burned-"));
scratch.push(work);
await execFileAsync("tar", ["-xzf", local(poisonedName).path, "-C", work]);
await writeFile(path.join(work, "package", "UNEXPECTED"), "not the release\n");
const poisoned = path.join(work, "poisoned.tgz");
await execFileAsync("tar", ["-czf", poisoned, "-C", work, "package"]);
expect(await npmPublish(burnedOwner, poisoned, publishToken)).toBe("published");
const poisonedBytes = await readFile(poisoned);
await expect(publish(burnedOwner)).rejects.toMatchObject({ code: "BURNED_VERSION" });
for (const [position, name] of order.entries()) {
const packument = await anonymousPackument(burnedOwner, name);
if (position < index) {
expect(packument?.versions[train.manifest.version], name).toBeDefined();
} else if (position === index) {
const url = packument?.versions[train.manifest.version]?.dist.tarball ?? "";
expect((await anonymousBytes(url)).equals(poisonedBytes)).toBe(true);
} else {
expect(packument, `${name} must not be published after the stop`).toBeUndefined();
}
expect(packument?.["dist-tags"].latest, name).toBeUndefined();
}
// A retry stops in exactly the same place, and still changes nothing.
await expect(publish(burnedOwner)).rejects.toMatchObject({ code: "BURNED_VERSION" });
expect(await anonymousPackument(burnedOwner, order[index + 1] ?? "")).toBeUndefined();
});
});
describe("moving the final tag, interrupted and resumed", () => {
it("leaves the train unfinished when moving latest fails part way", async () => {
await expect(
publish(tagOwner, {
beforeTag: (_name, index) => {
if (index === 4) throw new Error("injected failure before the fifth tag");
},
}),
).rejects.toThrow("injected failure");
for (const [index, name] of order.entries()) {
const packument = await anonymousPackument(tagOwner, name);
expect(packument?.versions[train.manifest.version], name).toBeDefined();
expect(packument?.["dist-tags"].latest, name).toBe(
index < 4 ? train.manifest.version : undefined,
);
}
});
it("a retry republishes nothing and moves only the tags still missing", async () => {
const record = await publish(tagOwner);
expect(record.packages.every((entry) => entry.outcome === "already-present")).toBe(true);
expect(record.packages.map((entry) => entry.finalTag)).toEqual([
...Array(4).fill("already-current"),
...Array(5).fill("moved"),
]);
for (const name of order) {
expect((await anonymousPackument(tagOwner, name))?.["dist-tags"].latest).toBe(
train.manifest.version,
);
}
});
});
+546
View File
@@ -0,0 +1,546 @@
import { mkdtemp, readFile, rm, writeFile } from "node:fs/promises";
import { tmpdir } from "node:os";
import path from "node:path";
import { fileURLToPath } from "node:url";
import { gzipSync } from "node:zlib";
import { afterEach, describe, expect, it } from "vitest";
import { parse } from "yaml";
import {
artifactDigests,
PACKAGE_TRAIN_MANIFEST,
type PackageTrainManifest,
PUBLIC_PACKAGES,
serializeManifest,
tarballName,
} from "../scripts/package-release.js";
import {
compareSemver,
isUnderRegistry,
loadReleaseTrain,
normalizeBin,
normalizeRegistry,
npmConfiguration,
PUBLISH_TOKEN_VARIABLE,
PublicationError,
packumentUrl,
parsePublishArguments,
publicationOrder,
publishPackageTrain,
registryCopyProblems,
registryReader,
tarballPackageJson,
trainProblems,
} from "../scripts/publish-package-train.js";
/**
* The package train publication tool's own rules (ADR 0053), without a registry: what it accepts
* as a publishable train, the order it publishes in, how it compares a registry copy with the
* release, and what it refuses before it ever reaches the network. The registry behaviour itself is
* proven against a disposable Gitea by `scripts/test-package-publication.sh`.
*/
const repositoryRoot = path.resolve(path.dirname(fileURLToPath(import.meta.url)), "..");
const VERSION = "0.1.2";
const REVISION = "86ae907f59458508e3277666cdd214f5bd7c1078";
const REGISTRY = "https://registry.example/api/packages/owner/npm/";
const scratch: string[] = [];
afterEach(async () => {
await Promise.all(scratch.splice(0).map((dir) => rm(dir, { force: true, recursive: true })));
});
/** A minimal npm tarball: gzip of a ustar archive holding the given files under `package/`. */
function tarball(files: Record<string, string>): Buffer {
const blocks: Buffer[] = [];
for (const [name, contents] of Object.entries(files)) {
const body = Buffer.from(contents);
const header = Buffer.alloc(512);
header.write(`package/${name}`, 0, "utf8");
header.write("0000644\0", 100);
header.write("0000000\0", 108);
header.write("0000000\0", 116);
header.write(`${body.length.toString(8).padStart(11, "0")}\0`, 124);
header.write("00000000000\0", 136);
header.write(" ", 148);
header.write("0", 156);
header.write("ustar\0", 257);
header.write("00", 263);
let checksum = 0;
for (const byte of header) checksum += byte;
header.write(`${checksum.toString(8).padStart(6, "0")}\0 `, 148);
blocks.push(header, body, Buffer.alloc((512 - (body.length % 512)) % 512));
}
blocks.push(Buffer.alloc(1024));
return gzipSync(Buffer.concat(blocks));
}
const TRAIN_DEPENDENCIES: Record<string, string[]> = {
"@ekklesia/contracts": [],
"@ekklesia/design-tokens": [],
"@ekklesia/module-database": ["@ekklesia/contracts"],
"@ekklesia/module-pack": ["@ekklesia/contracts", "@ekklesia/module-ui-build"],
"@ekklesia/module-sdk": ["@ekklesia/contracts"],
"@ekklesia/module-ui-build": ["@ekklesia/contracts"],
"@ekklesia/module-ui-sdk": [],
"@ekklesia/typescript-config": [],
"@ekklesia/ui": ["@ekklesia/design-tokens"],
};
function packageJsonFor(name: string): Record<string, unknown> {
return {
dependencies: Object.fromEntries((TRAIN_DEPENDENCIES[name] ?? []).map((dep) => [dep, VERSION])),
name,
version: VERSION,
...(name === "@ekklesia/module-pack"
? { bin: { "ekklesia-pack-module": "./bin/ekklesia-pack-module.mjs" } }
: {}),
};
}
async function releaseDirectory(
mutate?: (files: Map<string, Buffer>, manifest: PackageTrainManifest) => PackageTrainManifest,
): Promise<string> {
const directory = await mkdtemp(path.join(tmpdir(), "ekklesia-publication-test-"));
scratch.push(directory);
const files = new Map<string, Buffer>();
for (const entry of PUBLIC_PACKAGES) {
files.set(
tarballName(entry.name, VERSION),
tarball({ "package.json": JSON.stringify(packageJsonFor(entry.name)), LICENSE: "AGPL\n" }),
);
}
let manifest: PackageTrainManifest = {
packageTrain: 1,
packages: PUBLIC_PACKAGES.map((entry) => {
const file = tarballName(entry.name, VERSION);
const data = files.get(file) as Buffer;
return {
file,
name: entry.name,
role: entry.role,
size: data.length,
version: VERSION,
...artifactDigests(data),
};
}),
source: { modified: false, revision: REVISION },
version: VERSION,
};
files.set(
"SHA256SUMS",
Buffer.from(
[...manifest.packages]
.sort((left, right) => (left.file < right.file ? -1 : 1))
.map((entry) => `${entry.sha256} ${entry.file}\n`)
.join(""),
),
);
manifest = mutate ? mutate(files, manifest) : manifest;
files.set(PACKAGE_TRAIN_MANIFEST, Buffer.from(serializeManifest(manifest)));
for (const [file, data] of files) await writeFile(path.join(directory, file), data);
return directory;
}
describe("the local train", () => {
it("reads package.json from inside an npm tarball", () => {
const data = tarball({
LICENSE: "x",
"package.json": '{"name":"@ekklesia/x","version":"1.0.0"}',
});
expect(tarballPackageJson(data)).toEqual({ name: "@ekklesia/x", version: "1.0.0" });
expect(() => tarballPackageJson(tarball({ LICENSE: "x" }))).toThrow();
});
it("accepts a whole, unchanged train and records its manifest's own hash", async () => {
const directory = await releaseDirectory();
const train = await loadReleaseTrain(directory, { revision: REVISION, version: VERSION });
expect(train.packages).toHaveLength(9);
expect(train.manifestSha256).toBe(
artifactDigests(await readFile(path.join(directory, PACKAGE_TRAIN_MANIFEST))).sha256,
);
});
it("refuses a modified build, another version or revision, and any changed, missing, or extra file", async () => {
const cases: Record<string, Parameters<typeof releaseDirectory>[0]> = {
altered: (files, manifest) => {
files.set(tarballName("@ekklesia/ui", VERSION), tarball({ "package.json": "{}" }));
return manifest;
},
extra: (files, manifest) => {
files.set("ekklesia-core-0.1.2.tgz", Buffer.from("x"));
return manifest;
},
missing: (files, manifest) => {
files.delete(tarballName("@ekklesia/module-sdk", VERSION));
return manifest;
},
modified: (_files, manifest) => ({
...manifest,
source: { ...manifest.source, modified: true },
}),
sums: (files, manifest) => {
files.set("SHA256SUMS", Buffer.from("0 x\n"));
return manifest;
},
wrongInside: (files, manifest) => {
// Bytes that match the manifest, but a package.json that names another package.
const file = tarballName("@ekklesia/ui", VERSION);
const data = tarball({
"package.json": JSON.stringify({ name: "@ekklesia/other", version: VERSION }),
});
files.set(file, data);
return {
...manifest,
packages: manifest.packages.map((entry) =>
entry.file === file ? { ...entry, size: data.length, ...artifactDigests(data) } : entry,
),
};
},
};
for (const [name, mutate] of Object.entries(cases)) {
const directory = await releaseDirectory(mutate);
await expect(
loadReleaseTrain(directory, { revision: REVISION, version: VERSION }),
name,
).rejects.toMatchObject({ code: "INVALID_RELEASE" });
}
const valid = await releaseDirectory();
await expect(loadReleaseTrain(valid, { version: "0.1.3" })).rejects.toMatchObject({
code: "INVALID_RELEASE",
});
await expect(
loadReleaseTrain(valid, { revision: "f".repeat(40), version: VERSION }),
).rejects.toMatchObject({ code: "INVALID_RELEASE" });
});
it("explains every problem at once", () => {
const problems = trainProblems(
{
packageTrain: 1,
packages: [],
source: { modified: true, revision: "abc" },
version: "9.9.9",
},
new Map(),
{ version: VERSION },
);
expect(problems.join(" ")).toContain("modified source tree");
expect(problems.join(" ")).toContain("not the requested 0.1.2");
expect(problems.join(" ")).toContain("no full source revision");
expect(problems.join(" ")).toContain("is missing");
});
});
describe("the publication order", () => {
const entries = (names: readonly string[]) =>
names.map((name) => ({ entry: { name }, packageJson: packageJsonFor(name) }));
it("puts every package after the train packages it depends on, and is otherwise alphabetical", () => {
const order = publicationOrder(entries(Object.keys(TRAIN_DEPENDENCIES))).map(
(e) => e.entry.name,
);
expect(order).toEqual([
"@ekklesia/contracts",
"@ekklesia/design-tokens",
"@ekklesia/module-database",
"@ekklesia/module-sdk",
"@ekklesia/module-ui-build",
"@ekklesia/module-pack",
"@ekklesia/module-ui-sdk",
"@ekklesia/typescript-config",
"@ekklesia/ui",
]);
});
it("does not depend on the order the files are listed in", () => {
const names = Object.keys(TRAIN_DEPENDENCIES);
const forward = publicationOrder(entries(names)).map((e) => e.entry.name);
const backward = publicationOrder(entries([...names].reverse())).map((e) => e.entry.name);
expect(backward).toEqual(forward);
});
it("follows the real packages' own dependencies", async () => {
const real = await Promise.all(
PUBLIC_PACKAGES.map(async (entry) => ({
entry: { name: entry.name },
packageJson: JSON.parse(
await readFile(path.join(repositoryRoot, entry.directory, "package.json"), "utf8"),
) as Record<string, unknown>,
})),
);
const order = publicationOrder(real).map((e) => e.entry.name);
for (const { entry, packageJson } of real) {
for (const dependency of Object.keys((packageJson.dependencies as object) ?? {})) {
if (!dependency.startsWith("@ekklesia/")) continue;
expect(order.indexOf(dependency), `${dependency} before ${entry.name}`).toBeLessThan(
order.indexOf(entry.name),
);
}
}
});
it("refuses a dependency cycle", () => {
expect(() =>
publicationOrder([
{ entry: { name: "a" }, packageJson: { dependencies: { b: "1" } } },
{ entry: { name: "b" }, packageJson: { dependencies: { a: "1" } } },
]),
).toThrow(PublicationError);
});
});
describe("comparing a registry copy with the release", () => {
const data = tarball({ "package.json": JSON.stringify(packageJsonFor("@ekklesia/module-pack")) });
const digests = artifactDigests(data);
const local = {
data,
entry: {
file: "ekklesia-module-pack-0.1.2.tgz",
name: "@ekklesia/module-pack",
role: "tooling" as const,
size: data.length,
version: VERSION,
...digests,
},
packageJson: packageJsonFor("@ekklesia/module-pack"),
};
const registryVersion = {
// npm normalizes bin paths in the metadata it uploads; the tarball keeps the author's form.
bin: { "ekklesia-pack-module": "bin/ekklesia-pack-module.mjs" },
dependencies: { "@ekklesia/module-ui-build": VERSION, "@ekklesia/contracts": VERSION },
dist: {
integrity: digests.integrity,
tarball: `${REGISTRY}%40ekklesia%2Fmodule-pack/-/0.1.2/module-pack-0.1.2.tgz`,
},
name: "@ekklesia/module-pack",
version: VERSION,
};
it("accepts the same bytes and metadata, however npm normalized bin", () => {
expect(registryCopyProblems(local, registryVersion, data, REGISTRY)).toEqual([]);
expect(normalizeBin("./cli.js", "@ekklesia/tool")).toEqual({ tool: "cli.js" });
});
it("refuses other bytes, integrity, dependencies, bin, or a tarball elsewhere", () => {
const other = tarball({ "package.json": "{}" });
expect(registryCopyProblems(local, registryVersion, other, REGISTRY).join(" ")).toContain(
"different bytes",
);
const cases = [
{ ...registryVersion, dist: { ...registryVersion.dist, integrity: "sha512-AAAA" } },
{ ...registryVersion, dependencies: { "@ekklesia/contracts": "0.1.1" } },
{ ...registryVersion, bin: { other: "bin/other.mjs" } },
{
...registryVersion,
dist: { ...registryVersion.dist, tarball: "https://elsewhere.example/x.tgz" },
},
{ ...registryVersion, version: "0.1.3" },
];
for (const version of cases) {
expect(
registryCopyProblems(local, version, data, REGISTRY),
JSON.stringify(version),
).not.toEqual([]);
}
});
it("orders versions by SemVer precedence", () => {
expect(compareSemver("0.1.10", "0.1.9")).toBe(1);
expect(compareSemver("0.2.0", "0.10.0")).toBe(-1);
expect(compareSemver("1.0.0-rc.1", "1.0.0")).toBe(-1);
expect(compareSemver("1.0.0-rc.2", "1.0.0-rc.10")).toBe(-1);
expect(compareSemver("1.0.0-alpha", "1.0.0-1")).toBe(1);
expect(compareSemver("0.1.2", "0.1.2+build")).toBe(0);
});
});
describe("the registry and its credentials", () => {
it("accepts an HTTPS registry, adds the trailing slash, and refuses anything else", () => {
expect(normalizeRegistry("https://registry.example/api/packages/o/npm")).toBe(
"https://registry.example/api/packages/o/npm/",
);
expect(() => normalizeRegistry("http://registry.example/npm/")).toThrow(PublicationError);
expect(normalizeRegistry("http://gitea:3000/npm/", true)).toBe("http://gitea:3000/npm/");
expect(() => normalizeRegistry("https://user:[email protected]/npm/")).toThrow(
PublicationError,
);
});
it("refers to the token by its variable in npm's configuration, never by value", () => {
const configuration = npmConfiguration(REGISTRY);
expect(configuration).toContain(
`//registry.example/api/packages/owner/npm/:_authToken=\${${PUBLISH_TOKEN_VARIABLE}}`,
);
expect(configuration).toContain(`@ekklesia:registry=${REGISTRY}`);
expect(packumentUrl(REGISTRY, "@ekklesia/module-sdk")).toBe(
`${REGISTRY}@ekklesia%2fmodule-sdk`,
);
});
it("sends the token only to the registry itself", async () => {
const seen: Record<string, string | undefined> = {};
const fake = (async (url: string, init?: RequestInit) => {
seen[url] = ((init?.headers ?? {}) as Record<string, string>).authorization;
return new Response(new Uint8Array([1]), { status: 200 });
}) as typeof fetch;
const reader = registryReader(REGISTRY, "token-value", fake);
await reader.download(`${REGISTRY}x.tgz`);
await reader.download("https://elsewhere.example/x.tgz");
await reader.download(`${REGISTRY}y.tgz`, true);
expect(seen[`${REGISTRY}x.tgz`]).toBe("Bearer token-value");
expect(seen["https://elsewhere.example/x.tgz"]).toBeUndefined();
expect(seen[`${REGISTRY}y.tgz`]).toBeUndefined();
expect(isUnderRegistry("https://registry.example/api/packages/other/npm/x", REGISTRY)).toBe(
false,
);
});
it("refuses to start without a token, or with an invalid train, before any request", async () => {
let requests = 0;
const counting = (async () => {
requests += 1;
return new Response("{}");
}) as typeof fetch;
const directory = await releaseDirectory();
await expect(
publishPackageTrain({
fetch: counting,
registry: REGISTRY,
releaseDirectory: directory,
token: "",
version: VERSION,
}),
).rejects.toMatchObject({ code: "INVALID_OPTIONS" });
const invalid = await releaseDirectory((_files, manifest) => ({
...manifest,
source: { ...manifest.source, modified: true },
}));
await expect(
publishPackageTrain({
fetch: counting,
npm: "/nonexistent/npm",
registry: REGISTRY,
releaseDirectory: invalid,
token: "publish-token-value",
version: VERSION,
}),
).rejects.toMatchObject({ code: "INVALID_RELEASE" });
const valid = await releaseDirectory();
await expect(
publishPackageTrain({
fetch: counting,
manifestSha256: "0".repeat(64),
npm: "/nonexistent/npm",
registry: REGISTRY,
releaseDirectory: valid,
token: "publish-token-value",
version: VERSION,
}),
).rejects.toMatchObject({ code: "INVALID_RELEASE" });
expect(requests).toBe(0);
});
it("parses its command line, and takes no token from it", () => {
expect(
parsePublishArguments([
"--release",
"r",
"--registry",
REGISTRY,
"--version",
VERSION,
"--require-public-read",
]),
).toEqual({
flags: { requirePublicRead: true },
values: { registry: REGISTRY, release: "r", version: VERSION },
});
expect(() => parsePublishArguments(["--release", "r", "--token", "x"])).toThrow(
PublicationError,
);
expect(() => parsePublishArguments(["--release", "r"])).toThrow(PublicationError);
});
});
describe("what the publisher may do to a registry", () => {
it("never deletes, unpublishes, or overwrites: it only publishes new versions and moves a tag", async () => {
// Code only: the tool's comments may say what it never does.
const source = (
await readFile(path.join(repositoryRoot, "scripts/publish-package-train.ts"), "utf8")
)
.replace(/\/\*[\s\S]*?\*\//gu, "")
.replace(/^\s*\/\/.*$/gmu, "");
expect(source).not.toMatch(
/unpublish|deprecate|"DELETE"|method:\s*"(?:PUT|DELETE|POST)"|-rev\/|--force/u,
);
const npmCommands = [...source.matchAll(/\["(publish|dist-tag)"/gu)].map((match) => match[1]);
expect([...new Set(npmCommands)].sort()).toEqual(["dist-tag", "publish"]);
expect(source).toMatch(/\["dist-tag", "add",/u);
});
});
describe("when the real registry can be written to", () => {
type Workflow = {
on: Record<string, unknown>;
jobs: Record<
string,
{
if?: string;
needs?: string | string[];
steps: { run?: string; env?: Record<string, string>; uses?: string }[];
}
>;
};
const workflow = async (file: string) =>
parse(await readFile(path.join(repositoryRoot, ".github/workflows", file), "utf8")) as Workflow;
it("never from CI: no pull request or push job holds the publish secret or publishes", async () => {
const ci = await readFile(path.join(repositoryRoot, ".github/workflows/ci.yml"), "utf8");
expect(ci).not.toContain("EKKLESIA_PACKAGES_PUBLISH_TOKEN");
expect(ci).not.toContain("secrets.");
expect(ci).not.toContain("publish-package-train.ts");
// CI publishes only to the disposable registry its own job starts.
expect(ci).toContain("bash scripts/test-package-publication.sh --release");
});
it("only by hand, on main, after this run's own gate, with the secret in the publish step alone", async () => {
const publish = await workflow("publish-packages.yml");
expect(Object.keys(publish.on)).toEqual(["workflow_dispatch"]);
for (const [name, job] of Object.entries(publish.jobs)) {
expect(job.if, name).toBe("github.ref == 'refs/heads/main'");
}
expect(publish.jobs.publish?.needs).toBe("package-release");
const steps = Object.entries(publish.jobs).flatMap(([job, { steps }]) =>
steps.map((step) => ({ job, step })),
);
const withSecret = steps.filter(({ step }) => JSON.stringify(step).includes("secrets."));
expect(withSecret).toHaveLength(1);
expect(withSecret[0]?.job).toBe("publish");
expect(withSecret[0]?.step.env?.EKKLESIA_PACKAGES_PUBLISH_TOKEN).toBe(
"${{ secrets.EKKLESIA_PACKAGES_PUBLISH_TOKEN }}",
);
const command = withSecret[0]?.step.run ?? "";
for (const option of ["--revision", "--manifest-sha256", "--require-public-read", "--record"]) {
expect(command).toContain(option);
}
expect(command).not.toContain("--allow-http");
// The publish job never builds or packs: it installs nothing and runs no package script.
const publishRuns = (publish.jobs.publish?.steps ?? [])
.map((step) => step.run ?? "")
.join("\n");
expect(publishRuns).not.toMatch(/pnpm|npm (?:install|ci|run|pack)|package-release\.ts/u);
});
it("uses the artifact actions that work on this Gitea, pinned by commit, on visible paths", async () => {
for (const file of ["ci.yml", "publish-packages.yml"]) {
const text = await readFile(path.join(repositoryRoot, ".github/workflows", file), "utf8");
for (const use of text.matchAll(/uses: (actions\/(?:up|down)load-artifact)@(\S+)/gu)) {
expect(use[2], `${file} ${use[1]}`).toMatch(/^[0-9a-f]{40}$/u);
}
// The artifact actions leave hidden files out: a path starting with a dot uploads nothing.
for (const artifactPath of text.matchAll(/^\s+path: (\S+)$/gmu)) {
expect(artifactPath[1], file).not.toMatch(/^\.|\/\./u);
}
}
});
});
+4 -2
View File
@@ -1,13 +1,15 @@
{
"extends": "./packages/typescript-config/node.json",
"compilerOptions": {
"noEmit": true
"noEmit": true,
"allowImportingTsExtensions": true
},
"include": [
"scripts/**/*.ts",
"tests/**/*.ts",
"vitest.config.ts",
"vitest.migrations.integration.config.ts"
"vitest.migrations.integration.config.ts",
"vitest.packages.integration.config.ts"
],
"exclude": ["node_modules", "tests/fixtures"]
}
+14
View File
@@ -0,0 +1,14 @@
import { defineConfig } from "vitest/config";
/** The package train publication suite, run only by `scripts/test-package-publication.sh` against
* the disposable Gitea it starts. */
export default defineConfig({
test: {
fileParallelism: false,
hookTimeout: 60_000,
include: ["tests/package-publication.integration.ts"],
passWithNoTests: false,
sequence: { concurrent: false },
testTimeout: 180_000,
},
});