diff --git a/tests/fake-gitea.py b/tests/fake-gitea.py new file mode 100755 index 0000000..812ea42 --- /dev/null +++ b/tests/fake-gitea.py @@ -0,0 +1,115 @@ +#!/usr/bin/env python3 + +"""A stand-in for the part of Gitea 1.27's API that tools/gitea-release uses. + +Tags, releases and assets in memory. Writes need the token +"not-a-real-token", as Gitea needs a real one. Like Gitea, it takes a +second asset under a name a release already has: refusing that is +tools/gitea-release's job, and the tests check that it does. GET +/test/tamper/TAG/NAME changes an asset's bytes, the way somebody could in +the web UI. Prints the port it listens on, then serves until killed. +""" + +import json, re +from http.server import BaseHTTPRequestHandler, HTTPServer +from urllib.parse import urlparse, parse_qs, unquote + +TOKEN = "token not-a-real-token" +PREFIX = "/api/v1/repos/church/classroom" +state = {"tags": {"v0.1.5": "1" * 40}, "releases": [], "next": 1} + + +def find(tag): + return next((r for r in state["releases"] if r["tag_name"] == tag), None) + + +class Gitea(BaseHTTPRequestHandler): + def log_message(self, *args): + pass + + def reply(self, status, body=None, raw=None): + data = raw if raw is not None else json.dumps(body).encode() if body is not None else b"" + self.send_response(status) + self.send_header("Content-Length", str(len(data))) + self.end_headers() + self.wfile.write(data) + + def authorised(self): + if self.headers.get("Authorization") != TOKEN: + self.reply(401, {"message": "token is required"}) + return False + return True + + def release_json(self, r): + port = self.server.server_port + return {**{k: v for k, v in r.items() if k != "assets"}, + "html_url": f"http://127.0.0.1:{port}/church/classroom/releases/tag/{r['tag_name']}"} + + def do_GET(self): + path = urlparse(self.path).path + if m := re.fullmatch(PREFIX + r"/releases/tags/(.+)", path): + r = find(unquote(m.group(1))) + return self.reply(200, self.release_json(r)) if r else self.reply(404, {}) + if m := re.fullmatch(PREFIX + r"/tags/(.+)", path): + sha = state["tags"].get(unquote(m.group(1))) + return self.reply(200, {"name": m.group(1), "commit": {"sha": sha}}) if sha else self.reply(404, {}) + if m := re.fullmatch(PREFIX + r"/releases/(\d+)/assets", path): + r = next((r for r in state["releases"] if r["id"] == int(m.group(1))), None) + if not r: + return self.reply(404, {}) + port = self.server.server_port + return self.reply(200, [{"id": i, "name": n, "size": len(d), + "browser_download_url": f"http://127.0.0.1:{port}/download/{r['id']}/{n}"} + for i, (n, d) in enumerate(r["assets"].items())]) + if m := re.fullmatch(r"/download/(\d+)/(.+)", path): + r = next((r for r in state["releases"] if r["id"] == int(m.group(1))), None) + data = r and r["assets"].get(unquote(m.group(2))) + return self.reply(200, raw=data) if data is not None else self.reply(404, {}) + # For the test: somebody replaces an asset by hand in the web UI. + if m := re.fullmatch(r"/test/tamper/(.+)/(.+)", path): + find(unquote(m.group(1)))["assets"][unquote(m.group(2))] = b"tampered" + return self.reply(200, {}) + return self.reply(404, {}) + + def do_POST(self): + if not self.authorised(): + return + url = urlparse(self.path) + length = int(self.headers.get("Content-Length", 0)) + body = self.rfile.read(length) + if url.path == PREFIX + "/releases": + data = json.loads(body) + if find(data["tag_name"]): + return self.reply(409, {"message": "release exists"}) + state["tags"].setdefault(data["tag_name"], data["target_commitish"]) + r = {"id": state["next"], "tag_name": data["tag_name"], "name": data["name"], + "body": data["body"], "prerelease": data["prerelease"], "draft": False, "assets": {}} + state["next"] += 1 + state["releases"].append(r) + return self.reply(201, self.release_json(r)) + if m := re.fullmatch(PREFIX + r"/releases/(\d+)/assets", url.path): + r = next(r for r in state["releases"] if r["id"] == int(m.group(1))) + name = parse_qs(url.query)["name"][0] + boundary = self.headers["Content-Type"].split("boundary=")[1].encode() + part = body.split(b"--" + boundary)[1] + content = part.split(b"\r\n\r\n", 1)[1].rsplit(b"\r\n", 1)[0] + # Gitea itself happily takes a second asset of the same name. + r["assets"][name] = content + return self.reply(201, {"name": name}) + return self.reply(404, {}) + + def do_PATCH(self): + if not self.authorised(): + return + path = urlparse(self.path).path + if m := re.fullmatch(PREFIX + r"/releases/(\d+)", path): + r = next(r for r in state["releases"] if r["id"] == int(m.group(1))) + data = json.loads(self.rfile.read(int(self.headers["Content-Length"]))) + r.update({k: v for k, v in data.items() if k in ("body", "prerelease")}) + return self.reply(200, self.release_json(r)) + return self.reply(404, {}) + + +server = HTTPServer(("127.0.0.1", 0), Gitea) +print(server.server_port, flush=True) +server.serve_forever() diff --git a/tests/test-gitea-release.sh b/tests/test-gitea-release.sh new file mode 100755 index 0000000..c3c714c --- /dev/null +++ b/tests/test-gitea-release.sh @@ -0,0 +1,128 @@ +#!/bin/bash +# The Gitea release record, against a stand-in for Gitea's API. +# +# Nothing here reaches the real Gitea. The stand-in keeps tags, releases +# and assets in memory, answers the handful of calls tools/gitea-release +# makes the way Gitea 1.27 answers them, and refuses any write that does +# not carry the token. The questions are the ones that matter for a +# permanent record: a tag is never moved, an asset's bytes are never +# replaced, and the record only ever grows. +set -uo pipefail + +ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" +WORK="$(mktemp -d)" +SERVER_PID="" + +cleanup() { + [[ -z "$SERVER_PID" ]] || kill "$SERVER_PID" 2>/dev/null + rm -rf "$WORK" +} +trap cleanup EXIT + +passed=0 +failed=0 + +check() { + local label="$1" + shift + + if "$@" >/dev/null 2>&1; then + echo " ok $label" + passed=$((passed + 1)) + else + echo " FAIL $label" + failed=$((failed + 1)) + fi +} + +refuses() { + ! "$@" +} + + +python3 "$ROOT/tests/fake-gitea.py" > "$WORK/port" 2>"$WORK/gitea.log" & +SERVER_PID=$! + +for _ in $(seq 1 50); do + [[ -s "$WORK/port" ]] && break + sleep 0.1 +done + +export GITEA_URL="http://127.0.0.1:$(cat "$WORK/port")" +export GITEA_REPO=church/classroom +export GITEA_TOKEN=not-a-real-token +RELEASE="$ROOT/tools/gitea-release" +A=$(printf 'a%.0s' {1..40}) +B=$(printf 'b%.0s' {1..40}) + + +printf 'the package\n' > "$WORK/ekklesia-classroom_0.9.1_amd64.deb" +printf '# 0.9.1\n\nNotes.\n' > "$WORK/notes.md" + +echo "1. a release, created once, at one commit" +check "no release yet is its own answer" \ + bash -c "'$RELEASE' show v0.9.1; [[ \$? -eq 3 ]]" +check "it is created at the commit named" \ + "$RELEASE" create v0.9.1 "$A" "0.9.1" "$WORK/notes.md" --prerelease +check "and its tag names that commit" \ + bash -c "[[ \$(curl -fsS '$GITEA_URL/api/v1/repos/church/classroom/tags/v0.9.1' | python3 -c 'import json,sys; print(json.load(sys.stdin)[\"commit\"][\"sha\"])') == $A ]]" +check "it is a pre-release until it is promoted" \ + bash -c "api() { curl -fsS '$GITEA_URL/api/v1/repos/church/classroom'\$1; }; api /releases/tags/v0.9.1 | grep -q '\"prerelease\": true'" +check "creating it again at the same commit changes nothing" \ + "$RELEASE" create v0.9.1 "$A" "0.9.1" "$WORK/notes.md" --prerelease +check "a tag is never moved to another commit" \ + refuses "$RELEASE" create v0.9.1 "$B" "0.9.1" "$WORK/notes.md" +check "nor is a tag that already exists taken over" \ + refuses "$RELEASE" create v0.1.5 "$B" "0.1.5" "$WORK/notes.md" + +echo "2. assets: attached once, never replaced" +check "the package is attached" \ + "$RELEASE" upload v0.9.1 "$WORK/ekklesia-classroom_0.9.1_amd64.deb" +check "and reads back as the same bytes" \ + bash -c "'$RELEASE' download v0.9.1 ekklesia-classroom_0.9.1_amd64.deb '$WORK/back.deb' \ + && cmp -s '$WORK/back.deb' '$WORK/ekklesia-classroom_0.9.1_amd64.deb'" +check "the same file again is left alone, and is not an error" \ + bash -c "'$RELEASE' upload v0.9.1 '$WORK/ekklesia-classroom_0.9.1_amd64.deb' | grep -q 'left alone'" +mkdir -p "$WORK/rebuilt" +printf 'a different build\n' > "$WORK/rebuilt/ekklesia-classroom_0.9.1_amd64.deb" +check "different bytes under the same name are refused" \ + refuses "$RELEASE" upload v0.9.1 "$WORK/rebuilt/ekklesia-classroom_0.9.1_amd64.deb" +check "and the published bytes are still the first ones" \ + bash -c "'$RELEASE' download v0.9.1 ekklesia-classroom_0.9.1_amd64.deb '$WORK/back2.deb' \ + && cmp -s '$WORK/back2.deb' '$WORK/ekklesia-classroom_0.9.1_amd64.deb'" +check "an asset the release does not have is refused, not invented" \ + refuses "$RELEASE" download v0.9.1 nothing.deb "$WORK/nothing" +curl -fsS "$GITEA_URL/test/tamper/v0.9.1/ekklesia-classroom_0.9.1_amd64.deb" >/dev/null +check "an asset changed by hand is caught at the next upload" \ + refuses "$RELEASE" upload v0.9.1 "$WORK/ekklesia-classroom_0.9.1_amd64.deb" + +echo "3. the record only grows" +check "a line is recorded" \ + "$RELEASE" record v0.9.1 "published to Preview (canary)" +check "and another" \ + "$RELEASE" record v0.9.1 "promoted to Stable" +check "the notes, the creation line and both records are all there, in order" \ + bash -c "curl -fsS '$GITEA_URL/api/v1/repos/church/classroom/releases/tags/v0.9.1' \ + | python3 -c 'import json,sys +b = json.load(sys.stdin)[\"body\"] +order = [b.index(t) for t in (\"Notes.\", \"## Record\", \"release created at\", \"Preview (canary)\", \"promoted to Stable\")] +assert order == sorted(order), order +assert b.count(\"## Record\") == 1'" +check "the notes and record read back as text" \ + bash -c "'$RELEASE' notes v0.9.1 | grep -q 'Preview (canary)'" +check "and asking for a release there is none of is refused" \ + refuses "$RELEASE" notes v0.0.0 +check "promoted, it is no longer a pre-release" \ + bash -c "'$RELEASE' stable v0.9.1 && curl -fsS '$GITEA_URL/api/v1/repos/church/classroom/releases/tags/v0.9.1' | grep -q '\"prerelease\": false'" + +echo "4. credentials" +check "a write without the token is refused by Gitea, and reported" \ + refuses env GITEA_TOKEN= "$RELEASE" record v0.9.1 "nobody" +check "the token never appears in anything it prints" \ + bash -c "! { '$RELEASE' show v0.9.1; '$RELEASE' record v0.9.1 x; '$RELEASE' upload v0.9.1 '$WORK/rebuilt/ekklesia-classroom_0.9.1_amd64.deb'; } 2>&1 | grep -q not-a-real-token" +check "nothing works without knowing which Gitea" \ + refuses env -u GITEA_URL -u GITHUB_SERVER_URL "$RELEASE" show v0.9.1 + +echo +echo "$passed passed, $failed failed" +[[ $failed -eq 0 ]] diff --git a/tools/gitea-release b/tools/gitea-release new file mode 100755 index 0000000..5176a4e --- /dev/null +++ b/tools/gitea-release @@ -0,0 +1,304 @@ +#!/usr/bin/env python3 + +"""The Gitea release that is the permanent record of one version. + +A Gitea release is where a version's package, its checksum, how it was +built and what happened to it afterwards are kept for good: which commit, +which gate, when it went to Preview (canary), when it was promoted to +Stable and on whose approval. The update server is where classrooms fetch +it; this is where a person months from now finds out why. + +PUBLISHED BYTES ARE NEVER REPLACED. Gitea lets anyone with write access +delete an asset and upload another under the same name, and nothing in +Gitea stops a workflow doing it either, so this does: an asset is only +ever uploaded under a name the release does not have yet. An asset that +is already there with exactly the same bytes is left alone and reported; +one with different bytes fails, loudly, and nothing is changed. The same +rule the update server has had since 0.2.6 was republished over itself +(tools/release-lib.sh). + +A tag, likewise, is never moved: a release is only created for a tag that +does not exist, or that already names the commit being released. + +The record is append-only. `record` adds a dated line under "## Record"; +nothing here rewrites what an earlier line says. + + gitea-release show TAG the release, or exit 3 if none + gitea-release notes TAG its notes and record, as text + gitea-release create TAG COMMIT TITLE NOTES_FILE [--prerelease] + gitea-release upload TAG FILE... each only if not there yet + gitea-release download TAG NAME DEST an asset, by name + gitea-release record TAG LINE append to the record + gitea-release stable TAG no longer a pre-release + +Environment: + + GITEA_URL https://git.riverofglorymn.org (GITHUB_SERVER_URL in CI) + GITEA_REPO River_of_Glory_Church/Ekklesia_Classroom (GITHUB_REPOSITORY) + GITEA_TOKEN a token that may write releases; never printed +""" + +import hashlib +import json +import os +import sys +import urllib.error +import urllib.parse +import urllib.request +import uuid +from datetime import datetime, timezone + +RECORD = "## Record" + + +class Refused(Exception): + """Something that must not happen, said in words.""" + + +def say(message): + print(f"gitea-release: {message}", flush=True) + + +def config(): + url = os.environ.get("GITEA_URL") or os.environ.get("GITHUB_SERVER_URL") + repo = os.environ.get("GITEA_REPO") or os.environ.get("GITHUB_REPOSITORY") + token = os.environ.get("GITEA_TOKEN", "") + + if not url or not repo: + raise Refused("GITEA_URL and GITEA_REPO must be set") + + return url.rstrip("/"), repo, token + + +def request(method, path, body=None, headers=None, raw=False): + """(status, parsed body). 404 is an answer, not an error.""" + url, repo, token = config() + target = path if path.startswith("http") else f"{url}/api/v1/repos/{repo}{path}" + headers = dict(headers or {}) + + if token: + headers["Authorization"] = f"token {token}" + + if isinstance(body, (dict, list)): + body = json.dumps(body).encode() + headers["Content-Type"] = "application/json" + + call = urllib.request.Request(target, data=body, method=method, headers=headers) + + try: + with urllib.request.urlopen(call, timeout=120) as response: + data = response.read() + status = response.status + except urllib.error.HTTPError as error: + data = error.read() + status = error.code + + if status == 404: + return 404, None + + if status >= 400: + detail = data[:300].decode("utf-8", "replace") + raise Refused(f"{method} {path} answered {status}: {detail}") + + if raw: + return status, data + + return status, json.loads(data) if data else None + + +def release(tag): + status, found = request("GET", f"/releases/tags/{urllib.parse.quote(tag)}") + return None if status == 404 else found + + +def need_release(tag): + found = release(tag) + + if found is None: + raise Refused(f"there is no release {tag}") + + return found + + +def tag_commit(tag): + status, found = request("GET", f"/tags/{urllib.parse.quote(tag)}") + + if status == 404: + return None + + return found["commit"]["sha"] + + +def sha256(data): + return hashlib.sha256(data).hexdigest() + + +def now(): + return datetime.now(timezone.utc).strftime("%Y-%m-%d %H:%M UTC") + + +def assets(found): + status, listed = request("GET", f"/releases/{found['id']}/assets") + return {} if status == 404 else {asset["name"]: asset for asset in listed or []} + + +def fetch(asset): + _status, data = request("GET", asset["browser_download_url"], raw=True) + return data + + +# --------------------------------------------------------------------------- + +def show(tag): + found = release(tag) + + if found is None: + say(f"no release {tag}") + return 3 + + print(json.dumps({ + "tag": found["tag_name"], + "commit": tag_commit(tag), + "prerelease": found.get("prerelease"), + "assets": sorted(assets(found)), + "url": found.get("html_url"), + }, indent=2)) + return 0 + + +def notes(tag): + print(need_release(tag).get("body") or "") + return 0 + + +def create(tag, commit, title, notes_file, prerelease): + notes = open(notes_file, encoding="utf-8").read() + at = tag_commit(tag) + + if at is not None and at != commit: + raise Refused(f"tag {tag} already names {at}, not {commit}. " + "A tag is never moved; release a new version.") + + found = release(tag) + + if found is not None: + say(f"release {tag} already exists at {commit}; nothing created") + return 0 + + request("POST", "/releases", { + "tag_name": tag, + "target_commitish": commit, + "name": title, + "body": notes.rstrip() + f"\n\n{RECORD}\n\n- {now()}: release created at `{commit}`\n", + "draft": False, + "prerelease": prerelease, + }) + + if tag_commit(tag) != commit: + raise Refused(f"release {tag} was created, but its tag does not name {commit}") + + say(f"created release {tag} at {commit}") + return 0 + + +def upload(tag, files): + found = need_release(tag) + present = assets(found) + + for path in files: + name = os.path.basename(path) + data = open(path, "rb").read() + digest = sha256(data) + + if name in present: + there = sha256(fetch(present[name])) + + if there == digest: + say(f"{name} is already attached, byte for byte; left alone") + continue + + raise Refused(f"{name} is already attached to {tag} with sha256 {there}; " + f"this file is {digest}. A published asset is never replaced.") + + boundary = uuid.uuid4().hex + body = (f"--{boundary}\r\n" + f'Content-Disposition: form-data; name="attachment"; filename="{name}"\r\n' + f"Content-Type: application/octet-stream\r\n\r\n").encode() \ + + data + f"\r\n--{boundary}--\r\n".encode() + + request("POST", f"/releases/{found['id']}/assets?name={urllib.parse.quote(name)}", + body, {"Content-Type": f"multipart/form-data; boundary={boundary}"}) + + # Read back: what Gitea serves is what was sent. + there = assets(need_release(tag)).get(name) + + if there is None or sha256(fetch(there)) != digest: + raise Refused(f"{name} was uploaded but does not read back as sha256 {digest}") + + say(f"attached {name}, sha256 {digest}") + + return 0 + + +def download(tag, name, dest): + present = assets(need_release(tag)) + + if name not in present: + raise Refused(f"{tag} has no asset {name}") + + data = fetch(present[name]) + + with open(dest, "wb") as handle: + handle.write(data) + + say(f"{name} -> {dest}, sha256 {sha256(data)}") + return 0 + + +def record(tag, line): + found = need_release(tag) + body = found.get("body") or "" + + if RECORD not in body: + body = body.rstrip() + f"\n\n{RECORD}\n" + + body = body.rstrip() + f"\n- {now()}: {line}\n" + request("PATCH", f"/releases/{found['id']}", {"body": body}) + say(f"recorded on {tag}: {line}") + return 0 + + +def stable(tag): + found = need_release(tag) + request("PATCH", f"/releases/{found['id']}", {"prerelease": False}) + say(f"{tag} is no longer marked as a pre-release") + return 0 + + +def main(argv): + try: + match argv: + case ["show", tag]: + return show(tag) + case ["notes", tag]: + return notes(tag) + case ["create", tag, commit, title, notes_file, *rest] if rest in ([], ["--prerelease"]): + return create(tag, commit, title, notes_file, rest == ["--prerelease"]) + case ["upload", tag, *files] if files: + return upload(tag, files) + case ["download", tag, name, dest]: + return download(tag, name, dest) + case ["record", tag, line]: + return record(tag, line) + case ["stable", tag]: + return stable(tag) + case _: + print(__doc__, file=sys.stderr) + return 2 + except Refused as refusal: + say(f"REFUSED: {refusal}") + return 1 + + +if __name__ == "__main__": + sys.exit(main(sys.argv[1:]))