From 844bbcf2f17103ffed90f6886e6e2bfb9ff88612 Mon Sep 17 00:00:00 2001 From: ekklesia Date: Sat, 10 Oct 2026 21:16:05 -0500 Subject: [PATCH] ci: publish tested module release assets --- .gitea/workflows/ci.yml | 122 +++++++++++++++++++- .gitignore | 1 + ci/check-cleanup.sh | 25 +++++ ci/integration.sh | 33 +++--- ci/package.sh | 35 ++++++ ci/release-assets.py | 228 ++++++++++++++++++++++++++++++++++++++ ci/test_release_assets.py | 205 ++++++++++++++++++++++++++++++++++ 7 files changed, 634 insertions(+), 15 deletions(-) create mode 100755 ci/package.sh create mode 100755 ci/release-assets.py create mode 100755 ci/test_release_assets.py diff --git a/.gitea/workflows/ci.yml b/.gitea/workflows/ci.yml index 21a525d..5316505 100644 --- a/.gitea/workflows/ci.yml +++ b/.gitea/workflows/ci.yml @@ -15,12 +15,16 @@ name: CI # # The server is Gitea 1.27, which honours `concurrency` cancellation; `permissions` and # `timeout-minutes` are declared so the workflow is correct wherever they are enforced. +# Artifact actions v3 are deliberate: Gitea implements that protocol, while v4 refuses non-GitHub +# servers. A tag job uses Gitea's automatic per-job token only after every required job succeeds. on: pull_request: push: branches: - main + tags: + - "v*" permissions: contents: read @@ -59,7 +63,9 @@ jobs: run: pnpm lint - name: Check that CI cleans up every stack it starts - run: bash ci/check-cleanup.sh + run: | + bash ci/check-cleanup.sh + python3 -m unittest ci/test_release_assets.py typecheck: name: typecheck @@ -150,6 +156,43 @@ jobs: - name: Validate the built module and its migrations run: pnpm validate + # Build the canonical pair once. Integration downloads these exact bytes, and a successful tag + # run's release job later downloads the same artifact; neither consumer is allowed to repackage. + package: + name: package + runs-on: ubuntu-latest + timeout-minutes: 20 + outputs: + sha256: ${{ steps.canonical.outputs.sha256 }} + steps: + - name: Check out repository + uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + with: + persist-credentials: false + + - name: Set up Node.js at the repository's version + uses: actions/setup-node@48b55a011bda9f5d6aeb4c2d9c7362e8dae4041e # v6.4.0 + with: + node-version-file: .node-version + + - name: Install (frozen lockfile) + run: | + corepack enable + corepack install + pnpm install --frozen-lockfile + + - name: Build the canonical Module Package pair + id: canonical + run: bash ci/package.sh + + - name: Hand the canonical pair to its consumers + uses: actions/upload-artifact@c6a3b2bd78b3985e4b2f15397fec357f0fd808de # v3.2.2-node20 + with: + name: signage-module-package + path: build/module-package/ + if-no-files-found: error + retention-days: 30 + # The real linux-x64-glibc Signage Module Package inside Ekklesia, end to end. The pinned platform # builds one generic image with no bundled module source, verifies and installs Signage into # persistent module storage, applies its migrations, and drives its split-origin Dashboard and @@ -160,10 +203,12 @@ jobs: needs: - test - build + - package runs-on: ubuntu-latest timeout-minutes: 45 env: DOCKER_BUILDKIT: "1" + EXPECTED_PACKAGE_SHA256: ${{ needs.package.outputs.sha256 }} # Names this job's stacks, so its final step removes exactly these (the platform's # scripts/lib/ci-project.sh). EKKLESIA_CI_JOB_KEY: signage-integration @@ -184,6 +229,12 @@ jobs: corepack install pnpm install --frozen-lockfile + - name: Receive the canonical Module Package pair + uses: actions/download-artifact@ad191675b41f6a5b46da9a048cb6893812da158b # v3.1.0-node20 + with: + name: signage-module-package + path: build/module-package/ + - name: Fetch the pinned Ekklesia integration harness run: bash ci/fetch-platform.sh @@ -192,7 +243,7 @@ jobs: - name: Sweep stale CI stacks from the runner run: bash build/platform/scripts/ci-sweep.sh --stale - - name: Build, install, and run the external Signage package + - name: Install and run the prebuilt external Signage package run: bash ci/integration.sh # Whatever happened above, this job's own stacks do not outlive it. If the first step failed, @@ -203,3 +254,70 @@ jobs: if [ -f build/platform/scripts/ci-sweep.sh ]; then bash build/platform/scripts/ci-sweep.sh --job fi + + # Existing Signage releases and tags are created by a person. On a v* tag, attach only the + # package pair this run's integration consumed, plus ADR 0055's required corresponding-source + # material. No release or tag is created here, and an existing asset is never replaced. + release: + name: release + needs: + - lint + - typecheck + - test + - build + - package + - integration + if: startsWith(github.ref, 'refs/tags/v') + permissions: + contents: write + runs-on: ubuntu-latest + timeout-minutes: 30 + steps: + - name: Check out repository + uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + with: + persist-credentials: false + + - name: Set up Node.js at the repository's version + uses: actions/setup-node@48b55a011bda9f5d6aeb4c2d9c7362e8dae4041e # v6.4.0 + with: + node-version-file: .node-version + + - name: Install the reviewed release tooling + run: | + corepack enable + corepack install + pnpm install --frozen-lockfile + + - name: Receive the exact pair integration tested + uses: actions/download-artifact@ad191675b41f6a5b46da9a048cb6893812da158b # v3.1.0-node20 + with: + name: signage-module-package + path: build/release-assets/ + + - name: Verify tag, revision, package bytes, and public-distribution material + env: + EXPECTED_PACKAGE_SHA256: ${{ needs.package.outputs.sha256 }} + RELEASE_TAG: ${{ github.ref_name }} + RELEASE_REVISION: ${{ github.sha }} + run: | + python3 ci/release-assets.py verify-package build/release-assets "$RELEASE_TAG" "$RELEASE_REVISION" + package="$(find build/release-assets -maxdepth 1 -type f -name 'signage-*-linux-x64-glibc.tar.gz')" + pnpm exec ekklesia-corresponding-source acquire \ + --package "$package" --cache "$RUNNER_TEMP/signage-corresponding-sources" + pnpm exec ekklesia-corresponding-source build \ + --package "$package" --cache "$RUNNER_TEMP/signage-corresponding-sources" \ + --out build/release-assets + source="${package%.tar.gz}-corresponding-source.tar.gz" + metadata="${package%.tar.gz}.redistribution.json" + pnpm exec ekklesia-corresponding-source verify \ + --package "$package" --source "$source" --metadata "$metadata" + python3 ci/release-assets.py verify build/release-assets "$RELEASE_TAG" "$RELEASE_REVISION" + + - name: Attach the exact tested package and its required source material + env: + EXPECTED_PACKAGE_SHA256: ${{ needs.package.outputs.sha256 }} + GITHUB_TOKEN: ${{ github.token }} + RELEASE_TAG: ${{ github.ref_name }} + RELEASE_REVISION: ${{ github.sha }} + run: python3 ci/release-assets.py upload build/release-assets "$RELEASE_TAG" "$RELEASE_REVISION" diff --git a/.gitignore b/.gitignore index e9aa3bc..b0155e8 100644 --- a/.gitignore +++ b/.gitignore @@ -4,5 +4,6 @@ dist/ .turbo/ coverage/ build/ +__pycache__/ *.log .DS_Store diff --git a/ci/check-cleanup.sh b/ci/check-cleanup.sh index f45fd69..afa776c 100755 --- a/ci/check-cleanup.sh +++ b/ci/check-cleanup.sh @@ -44,6 +44,31 @@ check 'grep -Eq "^EKKLESIA_PLATFORM_REF=refs/(pull/[0-9]+/head|heads/main)$" "$r check 'grep -q "rev-parse refs/remotes/pinned" "$root/ci/fetch-platform.sh" && grep -q "does not resolve to EKKLESIA_PLATFORM_COMMIT" "$root/ci/fetch-platform.sh"' \ "the fetch verifies that the pinned ref resolves to the pinned commit" +# The package job is the only place that builds the canonical archive. Both integration and release +# receive the same Gitea Actions v3 artifact, and release cannot run until every required job passed. +package_job="$(awk '/^ package:$/ { inside = 1; print; next } inside && /^ [a-z][a-z-]*:$/ { exit } inside { print }' "$workflow")" +release_job="$(awk '/^ release:$/ { inside = 1; print; next } inside && /^ [a-z][a-z-]*:$/ { exit } inside { print }' "$workflow")" +check 'grep -q "tags:" "$workflow" && grep -q '\"v\*\"' "$workflow"' \ + "v tags start the gated workflow" +check 'grep -q "bash ci/package.sh" <<<"$package_job" && grep -q "actions/upload-artifact@c6a3b2bd78b3985e4b2f15397fec357f0fd808de" <<<"$package_job"' \ + "the package job builds and transfers one canonical artifact" +check 'grep -q "actions/download-artifact@ad191675b41f6a5b46da9a048cb6893812da158b" <<<"$job" && ! grep -Eq "ci/package.sh|ekklesia-pack-module" <<<"$job"' \ + "integration downloads that artifact and never rebuilds it" +check 'grep -q "needs.package.outputs.sha256" <<<"$job" && grep -q "GITHUB_OUTPUT" "$root/ci/package.sh"' \ + "integration binds the downloaded bytes to the package job's digest" +check 'for required in lint typecheck test build package integration; do grep -q " - $required" <<<"$release_job" || exit 1; done' \ + "release waits for every required job" +check 'grep -q "if: startsWith(github.ref, '\''refs/tags/v'\'')" <<<"$release_job" && grep -q "contents: write" <<<"$release_job"' \ + "only a v tag gives the release job write permission" +check 'grep -q "actions/download-artifact@ad191675b41f6a5b46da9a048cb6893812da158b" <<<"$release_job" && ! grep -Eq "ci/package.sh|ekklesia-pack-module" <<<"$release_job"' \ + "release downloads the tested artifact and never rebuilds it" +check 'grep -q "EXPECTED_PACKAGE_SHA256:.*needs.package.outputs.sha256" <<<"$release_job"' \ + "release binds its selected and uploaded bytes to the package job's digest" +check 'grep -q "ekklesia-corresponding-source verify" <<<"$release_job" && grep -q "release-assets.py upload" <<<"$release_job"' \ + "the redistribution gate runs before the exact assets are uploaded" +check 'grep -q "GITHUB_TOKEN:.*github.token" <<<"$release_job"' \ + "upload uses Gitea's automatic per-job token" + ((failures == 0)) || { echo "check-cleanup: ${failures} check(s) failed." >&2 exit 1 diff --git a/ci/integration.sh b/ci/integration.sh index c6bfd7a..0e0f843 100755 --- a/ci/integration.sh +++ b/ci/integration.sh @@ -1,40 +1,47 @@ #!/usr/bin/env bash -# Builds Signage as the canonical linux-x64-glibc Module Package, then gives that exact release pair -# to the pinned platform's generic deployment smoke. The platform image contains no Signage source: +# Gives the canonical linux-x64-glibc Module Package pair from the package job to the pinned +# platform's generic deployment smoke. The platform image contains no Signage source: # its verifier extracts the package into installation-owned module storage before migrations, # discovery, browser UI/display checks, restart, container recreation, disable/re-enable, removal, # and reinstall are exercised against disposable PostgreSQL and object-storage services. # +# bash ci/package.sh # bash ci/fetch-platform.sh # bash ci/integration.sh set -Eeuo pipefail root="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")/.." && pwd)" platform="$root/build/platform" -release="$root/build/module-package" +release="${1:-$root/build/module-package}" [[ -f "$platform/scripts/smoke-deploy.sh" ]] || { echo "integration: run bash ci/fetch-platform.sh first." >&2 exit 1 } -source_revision="${GITHUB_SHA:-$(git -C "$root" rev-parse HEAD 2>/dev/null || true)}" -[[ "$source_revision" =~ ^[0-9a-f]{40}$ ]] || { - echo "integration: no full source revision is available for the package." >&2 +mapfile -t archives < <(find "$release" -maxdepth 1 -type f -name 'signage-*-linux-x64-glibc.tar.gz' -print) +[[ "${#archives[@]}" -eq 1 ]] || { + echo "integration: expected exactly one prebuilt Module Package archive." >&2 exit 1 } - -rm -rf "$release" -mkdir -p "$release" -pnpm --dir "$root" build -pnpm --dir "$root" exec ekklesia-pack-module --module "$root" --target linux-x64-glibc \ - --out "$release" --license-file "$root/LICENSE" --source-revision "$source_revision" +archive="${archives[0]}" +checksum="$archive.sha256" +[[ -f "$checksum" ]] || { + echo "integration: missing $(basename "$checksum")." >&2 + exit 1 +} +(cd "$release" && sha256sum --check --strict "$(basename "$checksum")") +digest="$(sha256sum "$archive" | cut -d' ' -f1)" +if [[ -n "${EXPECTED_PACKAGE_SHA256:-}" && "$digest" != "$EXPECTED_PACKAGE_SHA256" ]]; then + echo "integration: prebuilt package sha256 $digest differs from the package job's $EXPECTED_PACKAGE_SHA256." >&2 + exit 1 +fi scenarios="$(sed -e 's/#.*//' -e '/^[[:space:]]*$/d' "$root/ci/display-scenarios" | tr '\n' ' ')" [[ -n "${scenarios// /}" ]] || { echo "integration: ci/display-scenarios lists no scenario." >&2 exit 1 } -echo "integration: external Signage package built; scenarios: ${scenarios}" +echo "integration: consuming prebuilt $(basename "$archive"); scenarios: ${scenarios}" cd "$platform" EKKLESIA_SMOKE_TOPOLOGY=split \ diff --git a/ci/package.sh b/ci/package.sh new file mode 100755 index 0000000..1176e14 --- /dev/null +++ b/ci/package.sh @@ -0,0 +1,35 @@ +#!/usr/bin/env bash +# Builds the one canonical linux-x64-glibc Module Package pair that later jobs must consume. +set -Eeuo pipefail + +root="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")/.." && pwd)" +release="${1:-$root/build/module-package}" +source_revision="${GITHUB_SHA:-$(git -C "$root" rev-parse HEAD 2>/dev/null || true)}" +[[ "$source_revision" =~ ^[0-9a-f]{40}$ ]] || { + echo "package: no full source revision is available." >&2 + exit 1 +} + +rm -rf -- "$release" +mkdir -p -- "$release" +pnpm --dir "$root" build +pnpm --dir "$root" exec ekklesia-pack-module --module "$root" --target linux-x64-glibc \ + --out "$release" --license-file "$root/LICENSE" --source-revision "$source_revision" + +mapfile -t archives < <(find "$release" -maxdepth 1 -type f -name 'signage-*-linux-x64-glibc.tar.gz' -print) +[[ "${#archives[@]}" -eq 1 ]] || { + echo "package: expected exactly one Module Package archive." >&2 + exit 1 +} +archive="${archives[0]}" +checksum="$archive.sha256" +[[ -f "$checksum" ]] || { + echo "package: missing $(basename "$checksum")." >&2 + exit 1 +} +(cd "$release" && sha256sum --check --strict "$(basename "$checksum")") +digest="$(sha256sum "$archive" | cut -d' ' -f1)" +if [[ -n "${GITHUB_OUTPUT:-}" ]]; then + echo "sha256=$digest" >>"$GITHUB_OUTPUT" +fi +echo "package: canonical pair is $(basename "$archive") and $(basename "$checksum")." diff --git a/ci/release-assets.py b/ci/release-assets.py new file mode 100755 index 0000000..46daf1d --- /dev/null +++ b/ci/release-assets.py @@ -0,0 +1,228 @@ +#!/usr/bin/env python3 +"""Validate and attach one tested Signage release bundle to an existing Gitea release.""" + +from __future__ import annotations + +import hashlib +import json +import os +import re +import sys +import tarfile +import urllib.error +import urllib.parse +import urllib.request +import uuid +from pathlib import Path + + +ARCHIVE = re.compile(r"^signage-(\d+\.\d+\.\d+)-linux-x64-glibc\.tar\.gz$") + + +class Refused(Exception): + """The release does not meet a condition that must fail closed.""" + + +def sha256(path: Path) -> str: + digest = hashlib.sha256() + with path.open("rb") as handle: + for chunk in iter(lambda: handle.read(1024 * 1024), b""): + digest.update(chunk) + return digest.hexdigest() + + +def package_metadata(archive: Path) -> dict: + try: + with tarfile.open(archive, "r:gz") as bundle: + member = next( + entry for entry in bundle.getmembers() + if entry.name == "signage/ekklesia.package.json" and entry.isfile() + ) + extracted = bundle.extractfile(member) + if extracted is None: + raise KeyError(member.name) + return json.loads(extracted.read()) + except (KeyError, StopIteration, tarfile.TarError, json.JSONDecodeError) as error: + raise Refused(f"cannot read signage/ekklesia.package.json from {archive.name}") from error + + +def package_pair(directory: Path, tag: str, revision: str) -> list[Path]: + archives = [path for path in directory.iterdir() if path.is_file() and ARCHIVE.fullmatch(path.name)] + if len(archives) != 1: + raise Refused(f"expected one Signage Module Package, found {len(archives)}") + + package = archives[0] + match = ARCHIVE.fullmatch(package.name) + assert match is not None + version = match.group(1) + if tag != f"v{version}": + raise Refused(f"tag {tag} does not match package version {version}") + if not re.fullmatch(r"[0-9a-f]{40}", revision): + raise Refused("release revision is not a full Git object ID") + + checksum = directory / f"{package.name}.sha256" + expected = [package, checksum] + missing = [path.name for path in expected if not path.is_file()] + if missing: + raise Refused(f"Module Package pair is incomplete; missing={missing}") + package_digest = sha256(package) + checksum_line = checksum.read_text(encoding="utf-8") + wanted_line = f"{package_digest} {package.name}\n" + if checksum_line != wanted_line: + raise Refused(f"{checksum.name} does not match {package.name}") + built_digest = os.environ.get("EXPECTED_PACKAGE_SHA256") + if built_digest is not None and built_digest != package_digest: + raise Refused("selected Module Package differs from the package job's bytes") + metadata = package_metadata(package) + if metadata.get("module") != {"id": "signage", "version": version}: + raise Refused("package metadata does not identify the expected Signage version") + if metadata.get("provenance", {}).get("revision") != revision: + raise Refused("package provenance revision does not match the tag commit") + + return expected + + +def expected_paths(directory: Path, tag: str, revision: str) -> list[Path]: + package, checksum = package_pair(directory, tag, revision) + package_digest = sha256(package) + match = ARCHIVE.fullmatch(package.name) + assert match is not None + version = match.group(1) + source = directory / f"signage-{version}-linux-x64-glibc-corresponding-source.tar.gz" + source_checksum = directory / f"{source.name}.sha256" + redistribution = directory / f"signage-{version}-linux-x64-glibc.redistribution.json" + notice = directory / f"signage-{version}-linux-x64-glibc.source-notice.txt" + expected = [package, checksum, source, source_checksum, redistribution, notice] + + missing = [path.name for path in expected if not path.is_file()] + extras = sorted(path.name for path in directory.iterdir() if path.is_file() and path not in expected) + if missing or extras: + raise Refused(f"release bundle differs from the six required files; missing={missing}, extra={extras}") + + source_line = source_checksum.read_text(encoding="utf-8") + wanted_source_line = f"{sha256(source)} {source.name}\n" + if source_line != wanted_source_line: + raise Refused(f"{source_checksum.name} does not match {source.name}") + + binding = json.loads(redistribution.read_text(encoding="utf-8")) + package_binding = binding.get("package", {}) + if package_binding.get("file") != package.name or package_binding.get("sha256") != package_digest: + raise Refused("redistribution metadata does not bind the selected Module Package") + corresponding = binding.get("correspondingSource", {}) + if corresponding.get("file") != source.name or corresponding.get("sha256") != sha256(source): + raise Refused("redistribution metadata does not bind the selected corresponding source") + + return expected + + +class Gitea: + def __init__(self) -> None: + self.server = os.environ.get("GITHUB_SERVER_URL", "").rstrip("/") + self.repository = os.environ.get("GITHUB_REPOSITORY", "") + self.token = os.environ.get("GITHUB_TOKEN", "") + if not self.server or not self.repository or not self.token: + raise Refused("GITHUB_SERVER_URL, GITHUB_REPOSITORY, and GITHUB_TOKEN are required") + + def request(self, method: str, path: str, body: bytes | None = None, + headers: dict[str, str] | None = None, *, authenticated: bool = True) -> bytes: + target = path if path.startswith("http") else f"{self.server}/api/v1/repos/{self.repository}{path}" + request_headers = dict(headers or {}) + if authenticated: + request_headers["Authorization"] = f"token {self.token}" + request = urllib.request.Request(target, data=body, method=method, headers=request_headers) + try: + with urllib.request.urlopen(request, timeout=120) as response: + return response.read() + except urllib.error.HTTPError as error: + detail = error.read()[:300].decode("utf-8", "replace") + raise Refused(f"{method} {path} answered {error.code}: {detail}") from error + + def json(self, method: str, path: str) -> object: + data = self.request(method, path) + return json.loads(data) if data else None + + def public_download(self, url: str) -> bytes: + return self.request("GET", url, authenticated=False) + + +def upload(directory: Path, tag: str, revision: str) -> None: + files = expected_paths(directory, tag, revision) + gitea = Gitea() + quoted_tag = urllib.parse.quote(tag, safe="") + release = gitea.json("GET", f"/releases/tags/{quoted_tag}") + tagged = gitea.json("GET", f"/tags/{quoted_tag}") + if not isinstance(release, dict): + raise Refused(f"release {tag} does not exist") + if not isinstance(tagged, dict) or tagged.get("commit", {}).get("sha") != revision: + raise Refused(f"tag {tag} does not name {revision}") + + release_id = release.get("id") + listed = gitea.json("GET", f"/releases/{release_id}/assets") + present = {asset["name"]: asset for asset in listed or []} + unexpected = sorted(set(present) - {path.name for path in files}) + if unexpected: + raise Refused(f"release already has unexpected assets: {unexpected}") + + # Preflight every published name before changing the release. This prevents a later conflict + # from leaving newly uploaded earlier files beside immutable, different bytes. + verified: dict[str, str] = {} + for path in files: + asset = present.get(path.name) + if asset is None: + continue + public_digest = hashlib.sha256(gitea.public_download(asset["browser_download_url"])).hexdigest() + expected_digest = sha256(path) + if public_digest != expected_digest: + raise Refused(f"public {path.name} has sha256 {public_digest}, expected {expected_digest}") + verified[path.name] = public_digest + + for path in files: + digest = sha256(path) + asset = present.get(path.name) + if asset is None: + boundary = uuid.uuid4().hex + body = ( + f"--{boundary}\r\n" + f'Content-Disposition: form-data; name="attachment"; filename="{path.name}"\r\n' + "Content-Type: application/octet-stream\r\n\r\n" + ).encode() + path.read_bytes() + f"\r\n--{boundary}--\r\n".encode() + gitea.request( + "POST", + f"/releases/{release_id}/assets?name={urllib.parse.quote(path.name)}", + body, + {"Content-Type": f"multipart/form-data; boundary={boundary}"}, + ) + listed = gitea.json("GET", f"/releases/{release_id}/assets") + asset = next((item for item in listed or [] if item.get("name") == path.name), None) + if asset is None: + raise Refused(f"Gitea did not list {path.name} after upload") + + public_digest = verified.get(path.name) + if public_digest is None: + public_digest = hashlib.sha256(gitea.public_download(asset["browser_download_url"])).hexdigest() + if public_digest != digest: + raise Refused(f"public {path.name} has sha256 {public_digest}, expected {digest}") + print(f"release-assets: public {path.name}, sha256 {digest}", flush=True) + + +def main(argv: list[str]) -> int: + try: + if len(argv) != 4 or argv[0] not in {"verify-package", "verify", "upload"}: + raise Refused("usage: release-assets.py (verify-package|verify|upload) DIRECTORY TAG REVISION") + command, directory, tag, revision = argv + selected = (package_pair(Path(directory), tag, revision) + if command == "verify-package" + else expected_paths(Path(directory), tag, revision)) + if command in {"verify-package", "verify"}: + for path in selected: + print(path) + else: + upload(Path(directory), tag, revision) + return 0 + except (OSError, json.JSONDecodeError, Refused) as error: + print(f"release-assets: REFUSED: {error}", file=sys.stderr) + return 1 + + +if __name__ == "__main__": + raise SystemExit(main(sys.argv[1:])) diff --git a/ci/test_release_assets.py b/ci/test_release_assets.py new file mode 100755 index 0000000..7f6348f --- /dev/null +++ b/ci/test_release_assets.py @@ -0,0 +1,205 @@ +#!/usr/bin/env python3 +"""Release selection and Gitea upload tests; no real server or credential is used.""" + +from __future__ import annotations + +import contextlib +import hashlib +import http.server +import importlib.util +import io +import json +import os +import socketserver +import tarfile +import tempfile +import threading +import unittest +import urllib.parse +from pathlib import Path +from unittest import mock + + +HERE = Path(__file__).resolve().parent +SPEC = importlib.util.spec_from_file_location("release_assets", HERE / "release-assets.py") +assert SPEC is not None and SPEC.loader is not None +release_assets = importlib.util.module_from_spec(SPEC) +SPEC.loader.exec_module(release_assets) + +VERSION = "1.2.3" +REVISION = "a" * 40 + + +def digest(data: bytes) -> str: + return hashlib.sha256(data).hexdigest() + + +def release_bundle(directory: Path) -> list[Path]: + package = directory / f"signage-{VERSION}-linux-x64-glibc.tar.gz" + metadata = json.dumps({ + "module": {"id": "signage", "version": VERSION}, + "provenance": {"revision": REVISION}, + }).encode() + with tarfile.open(package, "w:gz") as archive: + member = tarfile.TarInfo("signage/ekklesia.package.json") + member.size = len(metadata) + archive.addfile(member, io.BytesIO(metadata)) + + checksum = directory / f"{package.name}.sha256" + checksum.write_text(f"{release_assets.sha256(package)} {package.name}\n", encoding="utf-8") + source = directory / f"signage-{VERSION}-linux-x64-glibc-corresponding-source.tar.gz" + source.write_bytes(b"corresponding source") + source_checksum = directory / f"{source.name}.sha256" + source_checksum.write_text(f"{release_assets.sha256(source)} {source.name}\n", encoding="utf-8") + redistribution = directory / f"signage-{VERSION}-linux-x64-glibc.redistribution.json" + redistribution.write_text(json.dumps({ + "package": {"file": package.name, "sha256": release_assets.sha256(package)}, + "correspondingSource": {"file": source.name, "sha256": release_assets.sha256(source)}, + }), encoding="utf-8") + notice = directory / f"signage-{VERSION}-linux-x64-glibc.source-notice.txt" + notice.write_text("source notice\n", encoding="utf-8") + return [package, checksum, source, source_checksum, redistribution, notice] + + +class FakeGitea(http.server.BaseHTTPRequestHandler): + assets: dict[str, bytes] = {} + api_authorizations: list[str | None] = [] + public_authorizations: list[str | None] = [] + posts = 0 + + def log_message(self, _format: str, *_args: object) -> None: + pass + + def answer(self, status: int, body: object = None) -> None: + data = b"" if body is None else json.dumps(body).encode() + self.send_response(status) + self.send_header("Content-Type", "application/json") + self.send_header("Content-Length", str(len(data))) + self.end_headers() + self.wfile.write(data) + + def asset_list(self) -> list[dict[str, object]]: + server = self.server + assert isinstance(server, socketserver.TCPServer) + host, port = server.server_address + return [ + {"id": index, "name": name, "browser_download_url": f"http://{host}:{port}/public/{name}"} + for index, name in enumerate(self.assets, 1) + ] + + def do_GET(self) -> None: # noqa: N802 - BaseHTTPRequestHandler's API + if self.path.startswith("/public/"): + self.public_authorizations.append(self.headers.get("Authorization")) + name = urllib.parse.unquote(self.path.removeprefix("/public/")) + data = self.assets[name] + self.send_response(200) + self.send_header("Content-Length", str(len(data))) + self.end_headers() + self.wfile.write(data) + return + + self.api_authorizations.append(self.headers.get("Authorization")) + if "/releases/tags/" in self.path: + self.answer(200, {"id": 7}) + elif "/tags/" in self.path: + self.answer(200, {"commit": {"sha": REVISION}}) + elif self.path.endswith("/releases/7/assets"): + self.answer(200, self.asset_list()) + else: + self.answer(404, {"message": "not found"}) + + def do_POST(self) -> None: # noqa: N802 - BaseHTTPRequestHandler's API + self.api_authorizations.append(self.headers.get("Authorization")) + self.__class__.posts += 1 + name = urllib.parse.parse_qs(urllib.parse.urlsplit(self.path).query)["name"][0] + length = int(self.headers["Content-Length"]) + body = self.rfile.read(length) + boundary = self.headers["Content-Type"].split("boundary=", 1)[1].encode() + payload = body.split(b"\r\n\r\n", 1)[1].rsplit(b"\r\n--" + boundary + b"--", 1)[0] + self.assets[name] = payload + self.answer(201, {"name": name}) + + +@contextlib.contextmanager +def fake_gitea(initial: dict[str, bytes] | None = None): + FakeGitea.assets = dict(initial or {}) + FakeGitea.api_authorizations = [] + FakeGitea.public_authorizations = [] + FakeGitea.posts = 0 + with socketserver.TCPServer(("127.0.0.1", 0), FakeGitea) as server: + thread = threading.Thread(target=server.serve_forever, daemon=True) + thread.start() + host, port = server.server_address + old = {name: os.environ.get(name) for name in ("GITHUB_SERVER_URL", "GITHUB_REPOSITORY", "GITHUB_TOKEN")} + os.environ.update({ + "GITHUB_SERVER_URL": f"http://{host}:{port}", + "GITHUB_REPOSITORY": "owner/repository", + "GITHUB_TOKEN": "test-token", + }) + try: + yield + finally: + server.shutdown() + thread.join() + for name, value in old.items(): + if value is None: + os.environ.pop(name, None) + else: + os.environ[name] = value + + +class ReleaseAssetsTest(unittest.TestCase): + def test_selects_only_the_complete_bound_bundle(self) -> None: + with tempfile.TemporaryDirectory() as temporary: + directory = Path(temporary) + expected = release_bundle(directory) + self.assertEqual(release_assets.expected_paths(directory, f"v{VERSION}", REVISION), expected) + + (directory / "unexpected.txt").write_text("no", encoding="utf-8") + with self.assertRaisesRegex(release_assets.Refused, "six required files"): + release_assets.expected_paths(directory, f"v{VERSION}", REVISION) + + def test_refuses_wrong_tag_revision_and_checksum(self) -> None: + with tempfile.TemporaryDirectory() as temporary: + directory = Path(temporary) + paths = release_bundle(directory) + with self.assertRaisesRegex(release_assets.Refused, "tag"): + release_assets.expected_paths(directory, "v9.9.9", REVISION) + with self.assertRaisesRegex(release_assets.Refused, "provenance"): + release_assets.expected_paths(directory, f"v{VERSION}", "b" * 40) + with mock.patch.dict(os.environ, {"EXPECTED_PACKAGE_SHA256": "0" * 64}): + with self.assertRaisesRegex(release_assets.Refused, "package job's bytes"): + release_assets.expected_paths(directory, f"v{VERSION}", REVISION) + paths[1].write_text(f"{'0' * 64} {paths[0].name}\n", encoding="utf-8") + with self.assertRaisesRegex(release_assets.Refused, "does not match"): + release_assets.expected_paths(directory, f"v{VERSION}", REVISION) + + def test_uploads_each_selected_byte_and_reads_it_back_anonymously(self) -> None: + with tempfile.TemporaryDirectory() as temporary: + directory = Path(temporary) + expected = release_bundle(directory) + with fake_gitea(): + release_assets.upload(directory, f"v{VERSION}", REVISION) + self.assertEqual(FakeGitea.posts, 6) + self.assertEqual(FakeGitea.assets, {path.name: path.read_bytes() for path in expected}) + self.assertTrue(all(value == "token test-token" for value in FakeGitea.api_authorizations)) + self.assertEqual(FakeGitea.public_authorizations, [None] * 6) + + def test_existing_identical_assets_are_idempotent_but_conflicts_fail(self) -> None: + with tempfile.TemporaryDirectory() as temporary: + directory = Path(temporary) + expected = release_bundle(directory) + initial = {path.name: path.read_bytes() for path in expected} + with fake_gitea(initial): + release_assets.upload(directory, f"v{VERSION}", REVISION) + self.assertEqual(FakeGitea.posts, 0) + + conflicting = {expected[1].name: b"different published bytes"} + with fake_gitea(conflicting): + with self.assertRaisesRegex(release_assets.Refused, "public .* expected"): + release_assets.upload(directory, f"v{VERSION}", REVISION) + self.assertEqual(FakeGitea.posts, 0) + + +if __name__ == "__main__": + unittest.main() -- 2.54.0