Compare commits
2
Commits
88a536464a
...
8a7e96dd61
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
8a7e96dd61 | ||
|
|
844bbcf2f1 |
+120
-2
@@ -15,12 +15,16 @@ name: CI
|
|||||||
#
|
#
|
||||||
# The server is Gitea 1.27, which honours `concurrency` cancellation; `permissions` and
|
# The server is Gitea 1.27, which honours `concurrency` cancellation; `permissions` and
|
||||||
# `timeout-minutes` are declared so the workflow is correct wherever they are enforced.
|
# `timeout-minutes` are declared so the workflow is correct wherever they are enforced.
|
||||||
|
# Artifact actions v3 are deliberate: Gitea implements that protocol, while v4 refuses non-GitHub
|
||||||
|
# servers. A tag job uses Gitea's automatic per-job token only after every required job succeeds.
|
||||||
|
|
||||||
on:
|
on:
|
||||||
pull_request:
|
pull_request:
|
||||||
push:
|
push:
|
||||||
branches:
|
branches:
|
||||||
- main
|
- main
|
||||||
|
tags:
|
||||||
|
- "v*"
|
||||||
|
|
||||||
permissions:
|
permissions:
|
||||||
contents: read
|
contents: read
|
||||||
@@ -59,7 +63,9 @@ jobs:
|
|||||||
run: pnpm lint
|
run: pnpm lint
|
||||||
|
|
||||||
- name: Check that CI cleans up every stack it starts
|
- name: Check that CI cleans up every stack it starts
|
||||||
run: bash ci/check-cleanup.sh
|
run: |
|
||||||
|
bash ci/check-cleanup.sh
|
||||||
|
python3 -m unittest ci/test_release_assets.py
|
||||||
|
|
||||||
typecheck:
|
typecheck:
|
||||||
name: typecheck
|
name: typecheck
|
||||||
@@ -150,6 +156,43 @@ jobs:
|
|||||||
- name: Validate the built module and its migrations
|
- name: Validate the built module and its migrations
|
||||||
run: pnpm validate
|
run: pnpm validate
|
||||||
|
|
||||||
|
# Build the canonical pair once. Integration downloads these exact bytes, and a successful tag
|
||||||
|
# run's release job later downloads the same artifact; neither consumer is allowed to repackage.
|
||||||
|
package:
|
||||||
|
name: package
|
||||||
|
runs-on: ubuntu-latest
|
||||||
|
timeout-minutes: 20
|
||||||
|
outputs:
|
||||||
|
sha256: ${{ steps.canonical.outputs.sha256 }}
|
||||||
|
steps:
|
||||||
|
- name: Check out repository
|
||||||
|
uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
|
||||||
|
with:
|
||||||
|
persist-credentials: false
|
||||||
|
|
||||||
|
- name: Set up Node.js at the repository's version
|
||||||
|
uses: actions/setup-node@48b55a011bda9f5d6aeb4c2d9c7362e8dae4041e # v6.4.0
|
||||||
|
with:
|
||||||
|
node-version-file: .node-version
|
||||||
|
|
||||||
|
- name: Install (frozen lockfile)
|
||||||
|
run: |
|
||||||
|
corepack enable
|
||||||
|
corepack install
|
||||||
|
pnpm install --frozen-lockfile
|
||||||
|
|
||||||
|
- name: Build the canonical Module Package pair
|
||||||
|
id: canonical
|
||||||
|
run: bash ci/package.sh
|
||||||
|
|
||||||
|
- name: Hand the canonical pair to its consumers
|
||||||
|
uses: actions/upload-artifact@c6a3b2bd78b3985e4b2f15397fec357f0fd808de # v3.2.2-node20
|
||||||
|
with:
|
||||||
|
name: signage-module-package
|
||||||
|
path: build/module-package/
|
||||||
|
if-no-files-found: error
|
||||||
|
retention-days: 30
|
||||||
|
|
||||||
# The real linux-x64-glibc Signage Module Package inside Ekklesia, end to end. The pinned platform
|
# The real linux-x64-glibc Signage Module Package inside Ekklesia, end to end. The pinned platform
|
||||||
# builds one generic image with no bundled module source, verifies and installs Signage into
|
# builds one generic image with no bundled module source, verifies and installs Signage into
|
||||||
# persistent module storage, applies its migrations, and drives its split-origin Dashboard and
|
# persistent module storage, applies its migrations, and drives its split-origin Dashboard and
|
||||||
@@ -160,10 +203,12 @@ jobs:
|
|||||||
needs:
|
needs:
|
||||||
- test
|
- test
|
||||||
- build
|
- build
|
||||||
|
- package
|
||||||
runs-on: ubuntu-latest
|
runs-on: ubuntu-latest
|
||||||
timeout-minutes: 45
|
timeout-minutes: 45
|
||||||
env:
|
env:
|
||||||
DOCKER_BUILDKIT: "1"
|
DOCKER_BUILDKIT: "1"
|
||||||
|
EXPECTED_PACKAGE_SHA256: ${{ needs.package.outputs.sha256 }}
|
||||||
# Names this job's stacks, so its final step removes exactly these (the platform's
|
# Names this job's stacks, so its final step removes exactly these (the platform's
|
||||||
# scripts/lib/ci-project.sh).
|
# scripts/lib/ci-project.sh).
|
||||||
EKKLESIA_CI_JOB_KEY: signage-integration
|
EKKLESIA_CI_JOB_KEY: signage-integration
|
||||||
@@ -184,6 +229,12 @@ jobs:
|
|||||||
corepack install
|
corepack install
|
||||||
pnpm install --frozen-lockfile
|
pnpm install --frozen-lockfile
|
||||||
|
|
||||||
|
- name: Receive the canonical Module Package pair
|
||||||
|
uses: actions/download-artifact@ad191675b41f6a5b46da9a048cb6893812da158b # v3.1.0-node20
|
||||||
|
with:
|
||||||
|
name: signage-module-package
|
||||||
|
path: build/module-package/
|
||||||
|
|
||||||
- name: Fetch the pinned Ekklesia integration harness
|
- name: Fetch the pinned Ekklesia integration harness
|
||||||
run: bash ci/fetch-platform.sh
|
run: bash ci/fetch-platform.sh
|
||||||
|
|
||||||
@@ -192,7 +243,7 @@ jobs:
|
|||||||
- name: Sweep stale CI stacks from the runner
|
- name: Sweep stale CI stacks from the runner
|
||||||
run: bash build/platform/scripts/ci-sweep.sh --stale
|
run: bash build/platform/scripts/ci-sweep.sh --stale
|
||||||
|
|
||||||
- name: Build, install, and run the external Signage package
|
- name: Install and run the prebuilt external Signage package
|
||||||
run: bash ci/integration.sh
|
run: bash ci/integration.sh
|
||||||
|
|
||||||
# Whatever happened above, this job's own stacks do not outlive it. If the first step failed,
|
# Whatever happened above, this job's own stacks do not outlive it. If the first step failed,
|
||||||
@@ -203,3 +254,70 @@ jobs:
|
|||||||
if [ -f build/platform/scripts/ci-sweep.sh ]; then
|
if [ -f build/platform/scripts/ci-sweep.sh ]; then
|
||||||
bash build/platform/scripts/ci-sweep.sh --job
|
bash build/platform/scripts/ci-sweep.sh --job
|
||||||
fi
|
fi
|
||||||
|
|
||||||
|
# Existing Signage releases and tags are created by a person. On a v* tag, attach only the
|
||||||
|
# package pair this run's integration consumed, plus ADR 0055's required corresponding-source
|
||||||
|
# material. No release or tag is created here, and an existing asset is never replaced.
|
||||||
|
release:
|
||||||
|
name: release
|
||||||
|
needs:
|
||||||
|
- lint
|
||||||
|
- typecheck
|
||||||
|
- test
|
||||||
|
- build
|
||||||
|
- package
|
||||||
|
- integration
|
||||||
|
if: startsWith(github.ref, 'refs/tags/v')
|
||||||
|
permissions:
|
||||||
|
contents: write
|
||||||
|
runs-on: ubuntu-latest
|
||||||
|
timeout-minutes: 30
|
||||||
|
steps:
|
||||||
|
- name: Check out repository
|
||||||
|
uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
|
||||||
|
with:
|
||||||
|
persist-credentials: false
|
||||||
|
|
||||||
|
- name: Set up Node.js at the repository's version
|
||||||
|
uses: actions/setup-node@48b55a011bda9f5d6aeb4c2d9c7362e8dae4041e # v6.4.0
|
||||||
|
with:
|
||||||
|
node-version-file: .node-version
|
||||||
|
|
||||||
|
- name: Install the reviewed release tooling
|
||||||
|
run: |
|
||||||
|
corepack enable
|
||||||
|
corepack install
|
||||||
|
pnpm install --frozen-lockfile
|
||||||
|
|
||||||
|
- name: Receive the exact pair integration tested
|
||||||
|
uses: actions/download-artifact@ad191675b41f6a5b46da9a048cb6893812da158b # v3.1.0-node20
|
||||||
|
with:
|
||||||
|
name: signage-module-package
|
||||||
|
path: build/release-assets/
|
||||||
|
|
||||||
|
- name: Verify tag, revision, package bytes, and public-distribution material
|
||||||
|
env:
|
||||||
|
EXPECTED_PACKAGE_SHA256: ${{ needs.package.outputs.sha256 }}
|
||||||
|
RELEASE_TAG: ${{ github.ref_name }}
|
||||||
|
RELEASE_REVISION: ${{ github.sha }}
|
||||||
|
run: |
|
||||||
|
python3 ci/release-assets.py verify-package build/release-assets "$RELEASE_TAG" "$RELEASE_REVISION"
|
||||||
|
package="$(find build/release-assets -maxdepth 1 -type f -name 'signage-*-linux-x64-glibc.tar.gz')"
|
||||||
|
pnpm exec ekklesia-corresponding-source acquire \
|
||||||
|
--package "$package" --cache "$RUNNER_TEMP/signage-corresponding-sources"
|
||||||
|
pnpm exec ekklesia-corresponding-source build \
|
||||||
|
--package "$package" --cache "$RUNNER_TEMP/signage-corresponding-sources" \
|
||||||
|
--out build/release-assets
|
||||||
|
source="${package%.tar.gz}-corresponding-source.tar.gz"
|
||||||
|
metadata="${package%.tar.gz}.redistribution.json"
|
||||||
|
pnpm exec ekklesia-corresponding-source verify \
|
||||||
|
--package "$package" --source "$source" --metadata "$metadata"
|
||||||
|
python3 ci/release-assets.py verify build/release-assets "$RELEASE_TAG" "$RELEASE_REVISION"
|
||||||
|
|
||||||
|
- name: Attach the exact tested package and its required source material
|
||||||
|
env:
|
||||||
|
EXPECTED_PACKAGE_SHA256: ${{ needs.package.outputs.sha256 }}
|
||||||
|
GITHUB_TOKEN: ${{ github.token }}
|
||||||
|
RELEASE_TAG: ${{ github.ref_name }}
|
||||||
|
RELEASE_REVISION: ${{ github.sha }}
|
||||||
|
run: python3 ci/release-assets.py upload build/release-assets "$RELEASE_TAG" "$RELEASE_REVISION"
|
||||||
|
|||||||
@@ -4,5 +4,6 @@ dist/
|
|||||||
.turbo/
|
.turbo/
|
||||||
coverage/
|
coverage/
|
||||||
build/
|
build/
|
||||||
|
__pycache__/
|
||||||
*.log
|
*.log
|
||||||
.DS_Store
|
.DS_Store
|
||||||
|
|||||||
@@ -44,6 +44,31 @@ check 'grep -Eq "^EKKLESIA_PLATFORM_REF=refs/(pull/[0-9]+/head|heads/main)$" "$r
|
|||||||
check 'grep -q "rev-parse refs/remotes/pinned" "$root/ci/fetch-platform.sh" && grep -q "does not resolve to EKKLESIA_PLATFORM_COMMIT" "$root/ci/fetch-platform.sh"' \
|
check 'grep -q "rev-parse refs/remotes/pinned" "$root/ci/fetch-platform.sh" && grep -q "does not resolve to EKKLESIA_PLATFORM_COMMIT" "$root/ci/fetch-platform.sh"' \
|
||||||
"the fetch verifies that the pinned ref resolves to the pinned commit"
|
"the fetch verifies that the pinned ref resolves to the pinned commit"
|
||||||
|
|
||||||
|
# The package job is the only place that builds the canonical archive. Both integration and release
|
||||||
|
# receive the same Gitea Actions v3 artifact, and release cannot run until every required job passed.
|
||||||
|
package_job="$(awk '/^ package:$/ { inside = 1; print; next } inside && /^ [a-z][a-z-]*:$/ { exit } inside { print }' "$workflow")"
|
||||||
|
release_job="$(awk '/^ release:$/ { inside = 1; print; next } inside && /^ [a-z][a-z-]*:$/ { exit } inside { print }' "$workflow")"
|
||||||
|
check 'grep -q "tags:" "$workflow" && grep -q '\"v\*\"' "$workflow"' \
|
||||||
|
"v tags start the gated workflow"
|
||||||
|
check 'grep -q "bash ci/package.sh" <<<"$package_job" && grep -q "actions/upload-artifact@c6a3b2bd78b3985e4b2f15397fec357f0fd808de" <<<"$package_job"' \
|
||||||
|
"the package job builds and transfers one canonical artifact"
|
||||||
|
check 'grep -q "actions/download-artifact@ad191675b41f6a5b46da9a048cb6893812da158b" <<<"$job" && ! grep -Eq "ci/package.sh|ekklesia-pack-module" <<<"$job"' \
|
||||||
|
"integration downloads that artifact and never rebuilds it"
|
||||||
|
check 'grep -q "needs.package.outputs.sha256" <<<"$job" && grep -q "GITHUB_OUTPUT" "$root/ci/package.sh"' \
|
||||||
|
"integration binds the downloaded bytes to the package job's digest"
|
||||||
|
check 'for required in lint typecheck test build package integration; do grep -q " - $required" <<<"$release_job" || exit 1; done' \
|
||||||
|
"release waits for every required job"
|
||||||
|
check 'grep -q "if: startsWith(github.ref, '\''refs/tags/v'\'')" <<<"$release_job" && grep -q "contents: write" <<<"$release_job"' \
|
||||||
|
"only a v tag gives the release job write permission"
|
||||||
|
check 'grep -q "actions/download-artifact@ad191675b41f6a5b46da9a048cb6893812da158b" <<<"$release_job" && ! grep -Eq "ci/package.sh|ekklesia-pack-module" <<<"$release_job"' \
|
||||||
|
"release downloads the tested artifact and never rebuilds it"
|
||||||
|
check 'grep -q "EXPECTED_PACKAGE_SHA256:.*needs.package.outputs.sha256" <<<"$release_job"' \
|
||||||
|
"release binds its selected and uploaded bytes to the package job's digest"
|
||||||
|
check 'grep -q "ekklesia-corresponding-source verify" <<<"$release_job" && grep -q "release-assets.py upload" <<<"$release_job"' \
|
||||||
|
"the redistribution gate runs before the exact assets are uploaded"
|
||||||
|
check 'grep -q "GITHUB_TOKEN:.*github.token" <<<"$release_job"' \
|
||||||
|
"upload uses Gitea's automatic per-job token"
|
||||||
|
|
||||||
((failures == 0)) || {
|
((failures == 0)) || {
|
||||||
echo "check-cleanup: ${failures} check(s) failed." >&2
|
echo "check-cleanup: ${failures} check(s) failed." >&2
|
||||||
exit 1
|
exit 1
|
||||||
|
|||||||
+20
-13
@@ -1,40 +1,47 @@
|
|||||||
#!/usr/bin/env bash
|
#!/usr/bin/env bash
|
||||||
# Builds Signage as the canonical linux-x64-glibc Module Package, then gives that exact release pair
|
# Gives the canonical linux-x64-glibc Module Package pair from the package job to the pinned
|
||||||
# to the pinned platform's generic deployment smoke. The platform image contains no Signage source:
|
# platform's generic deployment smoke. The platform image contains no Signage source:
|
||||||
# its verifier extracts the package into installation-owned module storage before migrations,
|
# its verifier extracts the package into installation-owned module storage before migrations,
|
||||||
# discovery, browser UI/display checks, restart, container recreation, disable/re-enable, removal,
|
# discovery, browser UI/display checks, restart, container recreation, disable/re-enable, removal,
|
||||||
# and reinstall are exercised against disposable PostgreSQL and object-storage services.
|
# and reinstall are exercised against disposable PostgreSQL and object-storage services.
|
||||||
#
|
#
|
||||||
|
# bash ci/package.sh
|
||||||
# bash ci/fetch-platform.sh
|
# bash ci/fetch-platform.sh
|
||||||
# bash ci/integration.sh
|
# bash ci/integration.sh
|
||||||
set -Eeuo pipefail
|
set -Eeuo pipefail
|
||||||
|
|
||||||
root="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")/.." && pwd)"
|
root="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")/.." && pwd)"
|
||||||
platform="$root/build/platform"
|
platform="$root/build/platform"
|
||||||
release="$root/build/module-package"
|
release="${1:-$root/build/module-package}"
|
||||||
[[ -f "$platform/scripts/smoke-deploy.sh" ]] || {
|
[[ -f "$platform/scripts/smoke-deploy.sh" ]] || {
|
||||||
echo "integration: run bash ci/fetch-platform.sh first." >&2
|
echo "integration: run bash ci/fetch-platform.sh first." >&2
|
||||||
exit 1
|
exit 1
|
||||||
}
|
}
|
||||||
|
|
||||||
source_revision="${GITHUB_SHA:-$(git -C "$root" rev-parse HEAD 2>/dev/null || true)}"
|
mapfile -t archives < <(find "$release" -maxdepth 1 -type f -name 'signage-*-linux-x64-glibc.tar.gz' -print)
|
||||||
[[ "$source_revision" =~ ^[0-9a-f]{40}$ ]] || {
|
[[ "${#archives[@]}" -eq 1 ]] || {
|
||||||
echo "integration: no full source revision is available for the package." >&2
|
echo "integration: expected exactly one prebuilt Module Package archive." >&2
|
||||||
exit 1
|
exit 1
|
||||||
}
|
}
|
||||||
|
archive="${archives[0]}"
|
||||||
rm -rf "$release"
|
checksum="$archive.sha256"
|
||||||
mkdir -p "$release"
|
[[ -f "$checksum" ]] || {
|
||||||
pnpm --dir "$root" build
|
echo "integration: missing $(basename "$checksum")." >&2
|
||||||
pnpm --dir "$root" exec ekklesia-pack-module --module "$root" --target linux-x64-glibc \
|
exit 1
|
||||||
--out "$release" --license-file "$root/LICENSE" --source-revision "$source_revision"
|
}
|
||||||
|
(cd "$release" && sha256sum --check --strict "$(basename "$checksum")")
|
||||||
|
digest="$(sha256sum "$archive" | cut -d' ' -f1)"
|
||||||
|
if [[ -n "${EXPECTED_PACKAGE_SHA256:-}" && "$digest" != "$EXPECTED_PACKAGE_SHA256" ]]; then
|
||||||
|
echo "integration: prebuilt package sha256 $digest differs from the package job's $EXPECTED_PACKAGE_SHA256." >&2
|
||||||
|
exit 1
|
||||||
|
fi
|
||||||
|
|
||||||
scenarios="$(sed -e 's/#.*//' -e '/^[[:space:]]*$/d' "$root/ci/display-scenarios" | tr '\n' ' ')"
|
scenarios="$(sed -e 's/#.*//' -e '/^[[:space:]]*$/d' "$root/ci/display-scenarios" | tr '\n' ' ')"
|
||||||
[[ -n "${scenarios// /}" ]] || {
|
[[ -n "${scenarios// /}" ]] || {
|
||||||
echo "integration: ci/display-scenarios lists no scenario." >&2
|
echo "integration: ci/display-scenarios lists no scenario." >&2
|
||||||
exit 1
|
exit 1
|
||||||
}
|
}
|
||||||
echo "integration: external Signage package built; scenarios: ${scenarios}"
|
echo "integration: consuming prebuilt $(basename "$archive"); scenarios: ${scenarios}"
|
||||||
|
|
||||||
cd "$platform"
|
cd "$platform"
|
||||||
EKKLESIA_SMOKE_TOPOLOGY=split \
|
EKKLESIA_SMOKE_TOPOLOGY=split \
|
||||||
|
|||||||
Executable
+35
@@ -0,0 +1,35 @@
|
|||||||
|
#!/usr/bin/env bash
|
||||||
|
# Builds the one canonical linux-x64-glibc Module Package pair that later jobs must consume.
|
||||||
|
set -Eeuo pipefail
|
||||||
|
|
||||||
|
root="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")/.." && pwd)"
|
||||||
|
release="${1:-$root/build/module-package}"
|
||||||
|
source_revision="${GITHUB_SHA:-$(git -C "$root" rev-parse HEAD 2>/dev/null || true)}"
|
||||||
|
[[ "$source_revision" =~ ^[0-9a-f]{40}$ ]] || {
|
||||||
|
echo "package: no full source revision is available." >&2
|
||||||
|
exit 1
|
||||||
|
}
|
||||||
|
|
||||||
|
rm -rf -- "$release"
|
||||||
|
mkdir -p -- "$release"
|
||||||
|
pnpm --dir "$root" build
|
||||||
|
pnpm --dir "$root" exec ekklesia-pack-module --module "$root" --target linux-x64-glibc \
|
||||||
|
--out "$release" --license-file "$root/LICENSE" --source-revision "$source_revision"
|
||||||
|
|
||||||
|
mapfile -t archives < <(find "$release" -maxdepth 1 -type f -name 'signage-*-linux-x64-glibc.tar.gz' -print)
|
||||||
|
[[ "${#archives[@]}" -eq 1 ]] || {
|
||||||
|
echo "package: expected exactly one Module Package archive." >&2
|
||||||
|
exit 1
|
||||||
|
}
|
||||||
|
archive="${archives[0]}"
|
||||||
|
checksum="$archive.sha256"
|
||||||
|
[[ -f "$checksum" ]] || {
|
||||||
|
echo "package: missing $(basename "$checksum")." >&2
|
||||||
|
exit 1
|
||||||
|
}
|
||||||
|
(cd "$release" && sha256sum --check --strict "$(basename "$checksum")")
|
||||||
|
digest="$(sha256sum "$archive" | cut -d' ' -f1)"
|
||||||
|
if [[ -n "${GITHUB_OUTPUT:-}" ]]; then
|
||||||
|
echo "sha256=$digest" >>"$GITHUB_OUTPUT"
|
||||||
|
fi
|
||||||
|
echo "package: canonical pair is $(basename "$archive") and $(basename "$checksum")."
|
||||||
Executable
+228
@@ -0,0 +1,228 @@
|
|||||||
|
#!/usr/bin/env python3
|
||||||
|
"""Validate and attach one tested Signage release bundle to an existing Gitea release."""
|
||||||
|
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import hashlib
|
||||||
|
import json
|
||||||
|
import os
|
||||||
|
import re
|
||||||
|
import sys
|
||||||
|
import tarfile
|
||||||
|
import urllib.error
|
||||||
|
import urllib.parse
|
||||||
|
import urllib.request
|
||||||
|
import uuid
|
||||||
|
from pathlib import Path
|
||||||
|
|
||||||
|
|
||||||
|
ARCHIVE = re.compile(r"^signage-(\d+\.\d+\.\d+)-linux-x64-glibc\.tar\.gz$")
|
||||||
|
|
||||||
|
|
||||||
|
class Refused(Exception):
|
||||||
|
"""The release does not meet a condition that must fail closed."""
|
||||||
|
|
||||||
|
|
||||||
|
def sha256(path: Path) -> str:
|
||||||
|
digest = hashlib.sha256()
|
||||||
|
with path.open("rb") as handle:
|
||||||
|
for chunk in iter(lambda: handle.read(1024 * 1024), b""):
|
||||||
|
digest.update(chunk)
|
||||||
|
return digest.hexdigest()
|
||||||
|
|
||||||
|
|
||||||
|
def package_metadata(archive: Path) -> dict:
|
||||||
|
try:
|
||||||
|
with tarfile.open(archive, "r:gz") as bundle:
|
||||||
|
member = next(
|
||||||
|
entry for entry in bundle.getmembers()
|
||||||
|
if entry.name == "signage/ekklesia.package.json" and entry.isfile()
|
||||||
|
)
|
||||||
|
extracted = bundle.extractfile(member)
|
||||||
|
if extracted is None:
|
||||||
|
raise KeyError(member.name)
|
||||||
|
return json.loads(extracted.read())
|
||||||
|
except (KeyError, StopIteration, tarfile.TarError, json.JSONDecodeError) as error:
|
||||||
|
raise Refused(f"cannot read signage/ekklesia.package.json from {archive.name}") from error
|
||||||
|
|
||||||
|
|
||||||
|
def package_pair(directory: Path, tag: str, revision: str) -> list[Path]:
|
||||||
|
archives = [path for path in directory.iterdir() if path.is_file() and ARCHIVE.fullmatch(path.name)]
|
||||||
|
if len(archives) != 1:
|
||||||
|
raise Refused(f"expected one Signage Module Package, found {len(archives)}")
|
||||||
|
|
||||||
|
package = archives[0]
|
||||||
|
match = ARCHIVE.fullmatch(package.name)
|
||||||
|
assert match is not None
|
||||||
|
version = match.group(1)
|
||||||
|
if tag != f"v{version}":
|
||||||
|
raise Refused(f"tag {tag} does not match package version {version}")
|
||||||
|
if not re.fullmatch(r"[0-9a-f]{40}", revision):
|
||||||
|
raise Refused("release revision is not a full Git object ID")
|
||||||
|
|
||||||
|
checksum = directory / f"{package.name}.sha256"
|
||||||
|
expected = [package, checksum]
|
||||||
|
missing = [path.name for path in expected if not path.is_file()]
|
||||||
|
if missing:
|
||||||
|
raise Refused(f"Module Package pair is incomplete; missing={missing}")
|
||||||
|
package_digest = sha256(package)
|
||||||
|
checksum_line = checksum.read_text(encoding="utf-8")
|
||||||
|
wanted_line = f"{package_digest} {package.name}\n"
|
||||||
|
if checksum_line != wanted_line:
|
||||||
|
raise Refused(f"{checksum.name} does not match {package.name}")
|
||||||
|
built_digest = os.environ.get("EXPECTED_PACKAGE_SHA256")
|
||||||
|
if built_digest is not None and built_digest != package_digest:
|
||||||
|
raise Refused("selected Module Package differs from the package job's bytes")
|
||||||
|
metadata = package_metadata(package)
|
||||||
|
if metadata.get("module") != {"id": "signage", "version": version}:
|
||||||
|
raise Refused("package metadata does not identify the expected Signage version")
|
||||||
|
if metadata.get("provenance", {}).get("revision") != revision:
|
||||||
|
raise Refused("package provenance revision does not match the tag commit")
|
||||||
|
|
||||||
|
return expected
|
||||||
|
|
||||||
|
|
||||||
|
def expected_paths(directory: Path, tag: str, revision: str) -> list[Path]:
|
||||||
|
package, checksum = package_pair(directory, tag, revision)
|
||||||
|
package_digest = sha256(package)
|
||||||
|
match = ARCHIVE.fullmatch(package.name)
|
||||||
|
assert match is not None
|
||||||
|
version = match.group(1)
|
||||||
|
source = directory / f"signage-{version}-linux-x64-glibc-corresponding-source.tar.gz"
|
||||||
|
source_checksum = directory / f"{source.name}.sha256"
|
||||||
|
redistribution = directory / f"signage-{version}-linux-x64-glibc.redistribution.json"
|
||||||
|
notice = directory / f"signage-{version}-linux-x64-glibc.source-notice.txt"
|
||||||
|
expected = [package, checksum, source, source_checksum, redistribution, notice]
|
||||||
|
|
||||||
|
missing = [path.name for path in expected if not path.is_file()]
|
||||||
|
extras = sorted(path.name for path in directory.iterdir() if path.is_file() and path not in expected)
|
||||||
|
if missing or extras:
|
||||||
|
raise Refused(f"release bundle differs from the six required files; missing={missing}, extra={extras}")
|
||||||
|
|
||||||
|
source_line = source_checksum.read_text(encoding="utf-8")
|
||||||
|
wanted_source_line = f"{sha256(source)} {source.name}\n"
|
||||||
|
if source_line != wanted_source_line:
|
||||||
|
raise Refused(f"{source_checksum.name} does not match {source.name}")
|
||||||
|
|
||||||
|
binding = json.loads(redistribution.read_text(encoding="utf-8"))
|
||||||
|
package_binding = binding.get("package", {})
|
||||||
|
if package_binding.get("file") != package.name or package_binding.get("sha256") != package_digest:
|
||||||
|
raise Refused("redistribution metadata does not bind the selected Module Package")
|
||||||
|
corresponding = binding.get("correspondingSource", {})
|
||||||
|
if corresponding.get("file") != source.name or corresponding.get("sha256") != sha256(source):
|
||||||
|
raise Refused("redistribution metadata does not bind the selected corresponding source")
|
||||||
|
|
||||||
|
return expected
|
||||||
|
|
||||||
|
|
||||||
|
class Gitea:
|
||||||
|
def __init__(self) -> None:
|
||||||
|
self.server = os.environ.get("GITHUB_SERVER_URL", "").rstrip("/")
|
||||||
|
self.repository = os.environ.get("GITHUB_REPOSITORY", "")
|
||||||
|
self.token = os.environ.get("GITHUB_TOKEN", "")
|
||||||
|
if not self.server or not self.repository or not self.token:
|
||||||
|
raise Refused("GITHUB_SERVER_URL, GITHUB_REPOSITORY, and GITHUB_TOKEN are required")
|
||||||
|
|
||||||
|
def request(self, method: str, path: str, body: bytes | None = None,
|
||||||
|
headers: dict[str, str] | None = None, *, authenticated: bool = True) -> bytes:
|
||||||
|
target = path if path.startswith("http") else f"{self.server}/api/v1/repos/{self.repository}{path}"
|
||||||
|
request_headers = dict(headers or {})
|
||||||
|
if authenticated:
|
||||||
|
request_headers["Authorization"] = f"token {self.token}"
|
||||||
|
request = urllib.request.Request(target, data=body, method=method, headers=request_headers)
|
||||||
|
try:
|
||||||
|
with urllib.request.urlopen(request, timeout=120) as response:
|
||||||
|
return response.read()
|
||||||
|
except urllib.error.HTTPError as error:
|
||||||
|
detail = error.read()[:300].decode("utf-8", "replace")
|
||||||
|
raise Refused(f"{method} {path} answered {error.code}: {detail}") from error
|
||||||
|
|
||||||
|
def json(self, method: str, path: str) -> object:
|
||||||
|
data = self.request(method, path)
|
||||||
|
return json.loads(data) if data else None
|
||||||
|
|
||||||
|
def public_download(self, url: str) -> bytes:
|
||||||
|
return self.request("GET", url, authenticated=False)
|
||||||
|
|
||||||
|
|
||||||
|
def upload(directory: Path, tag: str, revision: str) -> None:
|
||||||
|
files = expected_paths(directory, tag, revision)
|
||||||
|
gitea = Gitea()
|
||||||
|
quoted_tag = urllib.parse.quote(tag, safe="")
|
||||||
|
release = gitea.json("GET", f"/releases/tags/{quoted_tag}")
|
||||||
|
tagged = gitea.json("GET", f"/tags/{quoted_tag}")
|
||||||
|
if not isinstance(release, dict):
|
||||||
|
raise Refused(f"release {tag} does not exist")
|
||||||
|
if not isinstance(tagged, dict) or tagged.get("commit", {}).get("sha") != revision:
|
||||||
|
raise Refused(f"tag {tag} does not name {revision}")
|
||||||
|
|
||||||
|
release_id = release.get("id")
|
||||||
|
listed = gitea.json("GET", f"/releases/{release_id}/assets")
|
||||||
|
present = {asset["name"]: asset for asset in listed or []}
|
||||||
|
unexpected = sorted(set(present) - {path.name for path in files})
|
||||||
|
if unexpected:
|
||||||
|
raise Refused(f"release already has unexpected assets: {unexpected}")
|
||||||
|
|
||||||
|
# Preflight every published name before changing the release. This prevents a later conflict
|
||||||
|
# from leaving newly uploaded earlier files beside immutable, different bytes.
|
||||||
|
verified: dict[str, str] = {}
|
||||||
|
for path in files:
|
||||||
|
asset = present.get(path.name)
|
||||||
|
if asset is None:
|
||||||
|
continue
|
||||||
|
public_digest = hashlib.sha256(gitea.public_download(asset["browser_download_url"])).hexdigest()
|
||||||
|
expected_digest = sha256(path)
|
||||||
|
if public_digest != expected_digest:
|
||||||
|
raise Refused(f"public {path.name} has sha256 {public_digest}, expected {expected_digest}")
|
||||||
|
verified[path.name] = public_digest
|
||||||
|
|
||||||
|
for path in files:
|
||||||
|
digest = sha256(path)
|
||||||
|
asset = present.get(path.name)
|
||||||
|
if asset is None:
|
||||||
|
boundary = uuid.uuid4().hex
|
||||||
|
body = (
|
||||||
|
f"--{boundary}\r\n"
|
||||||
|
f'Content-Disposition: form-data; name="attachment"; filename="{path.name}"\r\n'
|
||||||
|
"Content-Type: application/octet-stream\r\n\r\n"
|
||||||
|
).encode() + path.read_bytes() + f"\r\n--{boundary}--\r\n".encode()
|
||||||
|
gitea.request(
|
||||||
|
"POST",
|
||||||
|
f"/releases/{release_id}/assets?name={urllib.parse.quote(path.name)}",
|
||||||
|
body,
|
||||||
|
{"Content-Type": f"multipart/form-data; boundary={boundary}"},
|
||||||
|
)
|
||||||
|
listed = gitea.json("GET", f"/releases/{release_id}/assets")
|
||||||
|
asset = next((item for item in listed or [] if item.get("name") == path.name), None)
|
||||||
|
if asset is None:
|
||||||
|
raise Refused(f"Gitea did not list {path.name} after upload")
|
||||||
|
|
||||||
|
public_digest = verified.get(path.name)
|
||||||
|
if public_digest is None:
|
||||||
|
public_digest = hashlib.sha256(gitea.public_download(asset["browser_download_url"])).hexdigest()
|
||||||
|
if public_digest != digest:
|
||||||
|
raise Refused(f"public {path.name} has sha256 {public_digest}, expected {digest}")
|
||||||
|
print(f"release-assets: public {path.name}, sha256 {digest}", flush=True)
|
||||||
|
|
||||||
|
|
||||||
|
def main(argv: list[str]) -> int:
|
||||||
|
try:
|
||||||
|
if len(argv) != 4 or argv[0] not in {"verify-package", "verify", "upload"}:
|
||||||
|
raise Refused("usage: release-assets.py (verify-package|verify|upload) DIRECTORY TAG REVISION")
|
||||||
|
command, directory, tag, revision = argv
|
||||||
|
selected = (package_pair(Path(directory), tag, revision)
|
||||||
|
if command == "verify-package"
|
||||||
|
else expected_paths(Path(directory), tag, revision))
|
||||||
|
if command in {"verify-package", "verify"}:
|
||||||
|
for path in selected:
|
||||||
|
print(path)
|
||||||
|
else:
|
||||||
|
upload(Path(directory), tag, revision)
|
||||||
|
return 0
|
||||||
|
except (OSError, json.JSONDecodeError, Refused) as error:
|
||||||
|
print(f"release-assets: REFUSED: {error}", file=sys.stderr)
|
||||||
|
return 1
|
||||||
|
|
||||||
|
|
||||||
|
if __name__ == "__main__":
|
||||||
|
raise SystemExit(main(sys.argv[1:]))
|
||||||
Executable
+205
@@ -0,0 +1,205 @@
|
|||||||
|
#!/usr/bin/env python3
|
||||||
|
"""Release selection and Gitea upload tests; no real server or credential is used."""
|
||||||
|
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import contextlib
|
||||||
|
import hashlib
|
||||||
|
import http.server
|
||||||
|
import importlib.util
|
||||||
|
import io
|
||||||
|
import json
|
||||||
|
import os
|
||||||
|
import socketserver
|
||||||
|
import tarfile
|
||||||
|
import tempfile
|
||||||
|
import threading
|
||||||
|
import unittest
|
||||||
|
import urllib.parse
|
||||||
|
from pathlib import Path
|
||||||
|
from unittest import mock
|
||||||
|
|
||||||
|
|
||||||
|
HERE = Path(__file__).resolve().parent
|
||||||
|
SPEC = importlib.util.spec_from_file_location("release_assets", HERE / "release-assets.py")
|
||||||
|
assert SPEC is not None and SPEC.loader is not None
|
||||||
|
release_assets = importlib.util.module_from_spec(SPEC)
|
||||||
|
SPEC.loader.exec_module(release_assets)
|
||||||
|
|
||||||
|
VERSION = "1.2.3"
|
||||||
|
REVISION = "a" * 40
|
||||||
|
|
||||||
|
|
||||||
|
def digest(data: bytes) -> str:
|
||||||
|
return hashlib.sha256(data).hexdigest()
|
||||||
|
|
||||||
|
|
||||||
|
def release_bundle(directory: Path) -> list[Path]:
|
||||||
|
package = directory / f"signage-{VERSION}-linux-x64-glibc.tar.gz"
|
||||||
|
metadata = json.dumps({
|
||||||
|
"module": {"id": "signage", "version": VERSION},
|
||||||
|
"provenance": {"revision": REVISION},
|
||||||
|
}).encode()
|
||||||
|
with tarfile.open(package, "w:gz") as archive:
|
||||||
|
member = tarfile.TarInfo("signage/ekklesia.package.json")
|
||||||
|
member.size = len(metadata)
|
||||||
|
archive.addfile(member, io.BytesIO(metadata))
|
||||||
|
|
||||||
|
checksum = directory / f"{package.name}.sha256"
|
||||||
|
checksum.write_text(f"{release_assets.sha256(package)} {package.name}\n", encoding="utf-8")
|
||||||
|
source = directory / f"signage-{VERSION}-linux-x64-glibc-corresponding-source.tar.gz"
|
||||||
|
source.write_bytes(b"corresponding source")
|
||||||
|
source_checksum = directory / f"{source.name}.sha256"
|
||||||
|
source_checksum.write_text(f"{release_assets.sha256(source)} {source.name}\n", encoding="utf-8")
|
||||||
|
redistribution = directory / f"signage-{VERSION}-linux-x64-glibc.redistribution.json"
|
||||||
|
redistribution.write_text(json.dumps({
|
||||||
|
"package": {"file": package.name, "sha256": release_assets.sha256(package)},
|
||||||
|
"correspondingSource": {"file": source.name, "sha256": release_assets.sha256(source)},
|
||||||
|
}), encoding="utf-8")
|
||||||
|
notice = directory / f"signage-{VERSION}-linux-x64-glibc.source-notice.txt"
|
||||||
|
notice.write_text("source notice\n", encoding="utf-8")
|
||||||
|
return [package, checksum, source, source_checksum, redistribution, notice]
|
||||||
|
|
||||||
|
|
||||||
|
class FakeGitea(http.server.BaseHTTPRequestHandler):
|
||||||
|
assets: dict[str, bytes] = {}
|
||||||
|
api_authorizations: list[str | None] = []
|
||||||
|
public_authorizations: list[str | None] = []
|
||||||
|
posts = 0
|
||||||
|
|
||||||
|
def log_message(self, _format: str, *_args: object) -> None:
|
||||||
|
pass
|
||||||
|
|
||||||
|
def answer(self, status: int, body: object = None) -> None:
|
||||||
|
data = b"" if body is None else json.dumps(body).encode()
|
||||||
|
self.send_response(status)
|
||||||
|
self.send_header("Content-Type", "application/json")
|
||||||
|
self.send_header("Content-Length", str(len(data)))
|
||||||
|
self.end_headers()
|
||||||
|
self.wfile.write(data)
|
||||||
|
|
||||||
|
def asset_list(self) -> list[dict[str, object]]:
|
||||||
|
server = self.server
|
||||||
|
assert isinstance(server, socketserver.TCPServer)
|
||||||
|
host, port = server.server_address
|
||||||
|
return [
|
||||||
|
{"id": index, "name": name, "browser_download_url": f"http://{host}:{port}/public/{name}"}
|
||||||
|
for index, name in enumerate(self.assets, 1)
|
||||||
|
]
|
||||||
|
|
||||||
|
def do_GET(self) -> None: # noqa: N802 - BaseHTTPRequestHandler's API
|
||||||
|
if self.path.startswith("/public/"):
|
||||||
|
self.public_authorizations.append(self.headers.get("Authorization"))
|
||||||
|
name = urllib.parse.unquote(self.path.removeprefix("/public/"))
|
||||||
|
data = self.assets[name]
|
||||||
|
self.send_response(200)
|
||||||
|
self.send_header("Content-Length", str(len(data)))
|
||||||
|
self.end_headers()
|
||||||
|
self.wfile.write(data)
|
||||||
|
return
|
||||||
|
|
||||||
|
self.api_authorizations.append(self.headers.get("Authorization"))
|
||||||
|
if "/releases/tags/" in self.path:
|
||||||
|
self.answer(200, {"id": 7})
|
||||||
|
elif "/tags/" in self.path:
|
||||||
|
self.answer(200, {"commit": {"sha": REVISION}})
|
||||||
|
elif self.path.endswith("/releases/7/assets"):
|
||||||
|
self.answer(200, self.asset_list())
|
||||||
|
else:
|
||||||
|
self.answer(404, {"message": "not found"})
|
||||||
|
|
||||||
|
def do_POST(self) -> None: # noqa: N802 - BaseHTTPRequestHandler's API
|
||||||
|
self.api_authorizations.append(self.headers.get("Authorization"))
|
||||||
|
self.__class__.posts += 1
|
||||||
|
name = urllib.parse.parse_qs(urllib.parse.urlsplit(self.path).query)["name"][0]
|
||||||
|
length = int(self.headers["Content-Length"])
|
||||||
|
body = self.rfile.read(length)
|
||||||
|
boundary = self.headers["Content-Type"].split("boundary=", 1)[1].encode()
|
||||||
|
payload = body.split(b"\r\n\r\n", 1)[1].rsplit(b"\r\n--" + boundary + b"--", 1)[0]
|
||||||
|
self.assets[name] = payload
|
||||||
|
self.answer(201, {"name": name})
|
||||||
|
|
||||||
|
|
||||||
|
@contextlib.contextmanager
|
||||||
|
def fake_gitea(initial: dict[str, bytes] | None = None):
|
||||||
|
FakeGitea.assets = dict(initial or {})
|
||||||
|
FakeGitea.api_authorizations = []
|
||||||
|
FakeGitea.public_authorizations = []
|
||||||
|
FakeGitea.posts = 0
|
||||||
|
with socketserver.TCPServer(("127.0.0.1", 0), FakeGitea) as server:
|
||||||
|
thread = threading.Thread(target=server.serve_forever, daemon=True)
|
||||||
|
thread.start()
|
||||||
|
host, port = server.server_address
|
||||||
|
old = {name: os.environ.get(name) for name in ("GITHUB_SERVER_URL", "GITHUB_REPOSITORY", "GITHUB_TOKEN")}
|
||||||
|
os.environ.update({
|
||||||
|
"GITHUB_SERVER_URL": f"http://{host}:{port}",
|
||||||
|
"GITHUB_REPOSITORY": "owner/repository",
|
||||||
|
"GITHUB_TOKEN": "test-token",
|
||||||
|
})
|
||||||
|
try:
|
||||||
|
yield
|
||||||
|
finally:
|
||||||
|
server.shutdown()
|
||||||
|
thread.join()
|
||||||
|
for name, value in old.items():
|
||||||
|
if value is None:
|
||||||
|
os.environ.pop(name, None)
|
||||||
|
else:
|
||||||
|
os.environ[name] = value
|
||||||
|
|
||||||
|
|
||||||
|
class ReleaseAssetsTest(unittest.TestCase):
|
||||||
|
def test_selects_only_the_complete_bound_bundle(self) -> None:
|
||||||
|
with tempfile.TemporaryDirectory() as temporary:
|
||||||
|
directory = Path(temporary)
|
||||||
|
expected = release_bundle(directory)
|
||||||
|
self.assertEqual(release_assets.expected_paths(directory, f"v{VERSION}", REVISION), expected)
|
||||||
|
|
||||||
|
(directory / "unexpected.txt").write_text("no", encoding="utf-8")
|
||||||
|
with self.assertRaisesRegex(release_assets.Refused, "six required files"):
|
||||||
|
release_assets.expected_paths(directory, f"v{VERSION}", REVISION)
|
||||||
|
|
||||||
|
def test_refuses_wrong_tag_revision_and_checksum(self) -> None:
|
||||||
|
with tempfile.TemporaryDirectory() as temporary:
|
||||||
|
directory = Path(temporary)
|
||||||
|
paths = release_bundle(directory)
|
||||||
|
with self.assertRaisesRegex(release_assets.Refused, "tag"):
|
||||||
|
release_assets.expected_paths(directory, "v9.9.9", REVISION)
|
||||||
|
with self.assertRaisesRegex(release_assets.Refused, "provenance"):
|
||||||
|
release_assets.expected_paths(directory, f"v{VERSION}", "b" * 40)
|
||||||
|
with mock.patch.dict(os.environ, {"EXPECTED_PACKAGE_SHA256": "0" * 64}):
|
||||||
|
with self.assertRaisesRegex(release_assets.Refused, "package job's bytes"):
|
||||||
|
release_assets.expected_paths(directory, f"v{VERSION}", REVISION)
|
||||||
|
paths[1].write_text(f"{'0' * 64} {paths[0].name}\n", encoding="utf-8")
|
||||||
|
with self.assertRaisesRegex(release_assets.Refused, "does not match"):
|
||||||
|
release_assets.expected_paths(directory, f"v{VERSION}", REVISION)
|
||||||
|
|
||||||
|
def test_uploads_each_selected_byte_and_reads_it_back_anonymously(self) -> None:
|
||||||
|
with tempfile.TemporaryDirectory() as temporary:
|
||||||
|
directory = Path(temporary)
|
||||||
|
expected = release_bundle(directory)
|
||||||
|
with fake_gitea():
|
||||||
|
release_assets.upload(directory, f"v{VERSION}", REVISION)
|
||||||
|
self.assertEqual(FakeGitea.posts, 6)
|
||||||
|
self.assertEqual(FakeGitea.assets, {path.name: path.read_bytes() for path in expected})
|
||||||
|
self.assertTrue(all(value == "token test-token" for value in FakeGitea.api_authorizations))
|
||||||
|
self.assertEqual(FakeGitea.public_authorizations, [None] * 6)
|
||||||
|
|
||||||
|
def test_existing_identical_assets_are_idempotent_but_conflicts_fail(self) -> None:
|
||||||
|
with tempfile.TemporaryDirectory() as temporary:
|
||||||
|
directory = Path(temporary)
|
||||||
|
expected = release_bundle(directory)
|
||||||
|
initial = {path.name: path.read_bytes() for path in expected}
|
||||||
|
with fake_gitea(initial):
|
||||||
|
release_assets.upload(directory, f"v{VERSION}", REVISION)
|
||||||
|
self.assertEqual(FakeGitea.posts, 0)
|
||||||
|
|
||||||
|
conflicting = {expected[1].name: b"different published bytes"}
|
||||||
|
with fake_gitea(conflicting):
|
||||||
|
with self.assertRaisesRegex(release_assets.Refused, "public .* expected"):
|
||||||
|
release_assets.upload(directory, f"v{VERSION}", REVISION)
|
||||||
|
self.assertEqual(FakeGitea.posts, 0)
|
||||||
|
|
||||||
|
|
||||||
|
if __name__ == "__main__":
|
||||||
|
unittest.main()
|
||||||
Reference in New Issue
Block a user