Classroom_Desktop/tests/test-workflows.sh

#!/bin/bash
# The Gitea workflows keep the rules docs/releasing.md makes of them.
#
# Read as text, job by job, because the runner that runs this has no YAML
# library and must not need one. The rules are the ones that matter if a
# workflow is edited carelessly later: what a pull request can reach,
# what publishes and when, and that the work stays in the scripts.
# Whether Gitea itself parses and plans them was measured with act_runner
# (docs/releasing.md, "What was measured").
set -uo pipefail

ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)"

python3 - "$ROOT" <<'PY'
import re
import sys
from pathlib import Path

root = Path(sys.argv[1])
flows = root / ".gitea/workflows"
passed = failed = 0


def check(label, condition, detail=""):
    global passed, failed
    if condition:
        print(f"  ok   {label}")
        passed += 1
    else:
        print(f"  FAIL {label}")
        if detail:
            print(f"       {detail}")
        failed += 1


def read(name):
    return (flows / name).read_text()


def triggers(text):
    block = re.search(r"^on:\n((?:  .*\n|\n)+)", text, re.M)
    return set(re.findall(r"^  ([a-z_]+):", block.group(1), re.M)) if block else set()


def jobs(text):
    """{job id: its text}, from the jobs: block."""
    body = text[text.index("\njobs:\n") + 7:]
    found = {}
    for match in re.finditer(r"^  ([a-z_]+):\n((?:    .*\n|\n)*)", body, re.M):
        found[match.group(1)] = match.group(2)
    return found


def runs_on(job):
    match = re.search(r"^    runs-on: (\S+)$", job, re.M)
    return match.group(1) if match else None


names = sorted(p.name for p in flows.glob("*.yml"))
check("the workflows are the ones documented", names == [
    "ci.yml", "release-build.yml", "release-import.yml",
    "release-preview.yml", "release-stable.yml"], str(names))

releasing = (root / "docs/releasing.md").read_text()

print("1. CI: every change, no secrets, nothing kept that could be installed")
ci = read("ci.yml")
check("it runs on push and on pull requests", triggers(ci) == {"push", "pull_request"}, str(triggers(ci)))
check("it reads no secret and no token", "secrets." not in ci and "github.token" not in ci)
check("every job runs on the CI runner",
      {runs_on(job) for job in jobs(ci).values()} == {"ekklesia-ci"})
check("the checks are tools/check.sh, the same list a release build runs",
      "tools/check.sh --base" in ci
      and '"$ROOT/tools/check.sh"' in (root / "tools/build-release.sh").read_text())
check("the container gate is the release gate, in its CI mode", "tools/release-gate.sh --ci" in ci)
check("the package is inspected, not kept",
      "dist/" not in re.sub(r"(?s)^.*?Keep the package's contents", "", ci)
      .split("path:", 1)[-1])
check("building proves the version and changes nothing else in the tree",
      'dpkg-deb --field "$deb" Version)" = "$version"' in ci
      and "git status --porcelain --untracked-files=no" in ci)

print("2. releases: only by hand, and only through the scripts")
release_flows = [n for n in names if n.startswith("release-")]
for name in release_flows:
    text = read(name)
    check(f"{name} only runs when a person runs it", triggers(text) == {"workflow_dispatch"},
          str(triggers(text)))
    # An input pasted into a script is a script anyone who can run the
    # workflow can write. Inputs reach the shell as environment only.
    runs = "\n".join(re.findall(r"run: \|?\n?((?:          .*\n)+|.*\n)", text))
    check(f"{name} never pastes an input into a command",
          "${{" not in runs, re.findall(r"\$\{\{[^}]*\}\}", runs)[:3])
    check(f"{name}'s work is tools/release-ci.sh", "tools/release-ci.sh " in text)

print("3. who holds what")
where = {}
for name in names:
    for job_id, job in jobs(read(name)).items():
        where[f"{name}:{job_id}"] = (runs_on(job), job)

key_jobs = {k for k, (_, job) in where.items() if "CLASSROOM_UPDATES_SSH_KEY" in job}
check("the update server's key is given to exactly Preview's publish and Stable's promote",
      key_jobs == {"release-preview.yml:publish", "release-stable.yml:promote"}, str(key_jobs))
check("and those run only on the release runner",
      all(where[k][0] == "ekklesia-release" for k in key_jobs))
check("nothing on the release runner builds or gates (it has no Docker)",
      all("release-ci.sh build" not in job and "release-ci.sh gate" not in job
          and "release-gate.sh" not in job
          for runner, job in where.values() if runner == "ekklesia-release"))
check("nothing on the CI runner is given the key",
      all("CLASSROOM_UPDATES_SSH_KEY" not in job
          for runner, job in where.values() if runner == "ekklesia-ci"))
check("the key is a secret; where it goes is a variable",
      all("secrets.CLASSROOM_UPDATES_SSH_KEY" in where[k][1]
          and "vars.CLASSROOM_UPDATES_KNOWN_HOSTS" in where[k][1] for k in key_jobs))

print("4. Preview and Stable")
preview = jobs(read("release-preview.yml"))
check("Preview gates on the CI runner, then publishes on the release runner",
      runs_on(preview["gate"]) == "ekklesia-ci" and runs_on(preview["publish"]) == "ekklesia-release"
      and "needs: gate" in preview["publish"])
check("the publish job relies on that gate's report, by name",
      "needs.gate.outputs.report" in preview["publish"])
check("Preview asks for the sha256 of the file", "sha256:" in read("release-preview.yml"))
stable = read("release-stable.yml")
check("Stable asks for the sha256, an approval and a typed confirmation",
      all(f"      {field}:" in stable for field in ("sha256", "approval", "confirm"))
      and 'PROMOTE $VERSION TO STABLE' in stable)
check("Stable never builds or gates", "build" not in re.sub(r"#.*", "", stable).replace("rebuild", "")
      and "release-gate" not in stable)

print("5. what the measurements found")
check("artifacts are uploaded with v3, which Gitea 1.27 takes (v4 is refused)",
      all("upload-artifact@v4" not in read(n) for n in names)
      and "upload-artifact@v3" in ci)
check("the release token falls back to Gitea's own, which was measured to work",
      all("secrets.RELEASE_API_TOKEN || github.token" in read(n) for n in release_flows))
check("the docs say labels in a runner's config file win over --labels",
      "OVERRIDE --labels" in releasing)

print()
print(f"{passed} passed, {failed} failed")
sys.exit(1 if failed else 0)
PY