feat: corresponding-source artifacts for native packages that redistribute copyleft code (ADR 0055)

Ekklesia may publicly distribute a native Module Package containing LGPL or MPL components only beside a version-specific corresponding-source artifact. The artifact carries the exact source, patches, build definitions, and rebuild and relink instructions for those components. ADR 0055 (Accepted) records the policy: project compliance policy, not legal certification. It decides the item ADR 0052 deferred, and extends ADR 0049's distributable-release gate. Module Package v1, packageFormat 1, and both libvips supplements are unchanged. The mechanism is generic and lives in @ekklesia/module-pack: - Source recipes: reviewed data under corresponding-sources/<package>/<version>/. recipe.json names the exact package (name, version, content digest) and the exact supplement it was prepared with, and lists every component with its role, licence, licence material, and relationship to the binary. Every source file and patch is recorded by origin, size, and SHA-256; documents are hashed too, and every exclusion has a reason. Loading is strict and refuses anything unlisted. - Acquisition (the only network step) fetches each recorded file once into a cache keyed by SHA-256. It uses node:https with exact headers and no decoding, retries transient failures, and stores only bytes that match. - Construction is offline. From a built package it finds the covered native dependencies (by declared licence, or by a supplement component's licence), verifies every byte, and writes four sibling artifacts: the deterministic <package>-corresponding-source.tar.gz and its .sha256; <package>.redistribution.json, linking package, dependencies, supplement, recipe, and source by digest; and <package>.source-notice.txt, the release notice. - The public-distribution gate (verifyRedistribution, `ekklesia-corresponding-source verify`) approves only when the recipe, supplement, canonical archive, manifest, and metadata identify exactly these files. It refuses bundled copyleft code, and approves packages with no covered native code without an artifact. packModule is untouched. First complete proof: @img/[email protected] (sharp 0.35.5), from the reviewed supplement and provenance. - 383 components and 387 source files, about 208 MB: the sharp-libvips v1.3.4 build definitions, the 27 library archives, the five build-time patches, the 353 crates librsvg's resolved Cargo.lock records, and rust-src for nightly-2026-09-27 (rustc 75a75c3e0), which vendors std's dependencies. - BUILDING.md and two helpers point sharp-libvips's unchanged build logic at those files. - Followed in a Rocky Linux 8 container with networking disabled, they rebuilt libvips-cpp.so.8.18.7, byte-identically across two builds. Swapped into a verified, extracted native package, the rebuilt library was the one loaded, and sharp ran. - The native proof's source artifact, native-proof-0.1.0-linux-x64-glibc-corresponding-source.tar.gz, is 207242785 bytes, sha256 cc405492032b04a5391ab84e64f0e5e50f78e571ef7818204782c25a03eb50b7. It is byte-identical from unrelated directories and from a cache acquired fresh from upstream. Tests: - corresponding-source.test.ts runs in CI: synthetic inputs, every negative case, acquisition with retries and refusal, and determinism. - corresponding-source-native.test.ts checks the real recipe everywhere. With a verified cache (`pnpm test:corresponding-source`, which acquires once) it builds the real artifact twice, gates it, fails it closed, and runs sharp from the package. Docs: the module guide's distribution section, the README and architecture notes, and an ADR 0052 cross-reference. No Signage change, version bump, publication, or tag. Co-Authored-By: Claude Opus 5.5 <[email protected]>
2026-10-08 23:17:35 -05:00
parent 4919db6168
commit e083c44e05
22 changed files with 14580 additions and 6 deletions
@@ -0,0 +1,342 @@
+import { execFile } from "node:child_process";
+import { createHash } from "node:crypto";
+import { existsSync } from "node:fs";
+import { readFile, realpath, rm, writeFile } from "node:fs/promises";
+import { tmpdir } from "node:os";
+import path from "node:path";
+import { fileURLToPath } from "node:url";
+import { promisify } from "node:util";
+import { currentModulePackageHost, verifyModulePackage } from "@ekklesia/core/modules";
+import { afterEach, describe, expect, it } from "vitest";
+import {
+  buildCorrespondingSource,
+  CorrespondingSourceError,
+  loadSourceRecipes,
+  verifyRedistribution,
+} from "../src/corresponding-source.js";
+import { packModule } from "../src/index.js";
+import { NATIVE_TARGETS, nativeClosure } from "../src/native.js";
+import {
+  contentDigest,
+  DEFAULT_SUPPLEMENTS_DIRECTORY,
+  loadLicenseSupplements,
+} from "../src/supplements.js";
+import { readTarGz } from "../src/tar.js";
+import {
+  cleanupFixtures,
+  installedPackageFile,
+  nativeFixtureModule,
+  SOURCE_REVISION,
+  scratchDirectory,
+} from "./support/fixture-module.js";
+
+/**
+ * ADR 0055's first complete proof: the native proof module with sharp 0.35.5, whose libvips 1.3.4
+ * binary contains LGPL and MPL components, and its real corresponding-source artifact.
+ *
+ * The recipe checks run everywhere. Building the artifact needs the recorded sources in a verified
+ * cache, which `pnpm test:corresponding-source` acquires once and passes in as
+ * EKKLESIA_CORRESPONDING_SOURCE_CACHE; without it, that part is skipped rather than downloading
+ * about 200 MB on every run.
+ */
+
+const execFileAsync = promisify(execFile);
+const packageRoot = path.resolve(path.dirname(fileURLToPath(import.meta.url)), "..");
+const repositoryRoot = path.resolve(packageRoot, "..", "..");
+const LIBVIPS = "@img/sharp-libvips-linux-x64";
+const cache = process.env.EKKLESIA_CORRESPONDING_SOURCE_CACHE;
+const host = currentModulePackageHost();
+const onTargetHost =
+  host.platform === "linux" && host.arch === "x64" && host.libc?.family === "glibc";
+
+afterEach(cleanupFixtures);
+
+// `pnpm test:corresponding-source` asks for the full proof: then a missing cache is a failure, not a
+// skip.
+it.runIf(process.env.EKKLESIA_CORRESPONDING_SOURCE_REQUIRED === "1")(
+  "has the verified source cache it was asked to use, on a linux-x64-glibc host",
+  () => {
+    expect(onTargetHost).toBe(true);
+    expect(cache !== undefined && existsSync(cache)).toBe(true);
+  },
+);
+
+describe("the libvips 1.3.4 source recipe", () => {
+  it("loads intact, matches its reviewed supplement exactly, and pins the installed package", async () => {
+    const recipes = await loadSourceRecipes();
+    const [loaded] = recipes.get(LIBVIPS) ?? [];
+    expect(loaded?.recipe.package).toMatchObject({
+      license: "LGPL-3.0-or-later",
+      target: "linux-x64-glibc",
+      version: "1.3.4",
+    });
+
+    // Prepared against exactly the reviewed supplement for these package bytes.
+    const supplementJson = await readFile(
+      path.join(DEFAULT_SUPPLEMENTS_DIRECTORY, LIBVIPS, "1.3.4", "supplement.json"),
+    );
+    expect(loaded?.recipe.supplement.sha256).toBe(
+      createHash("sha256").update(supplementJson).digest("hex"),
+    );
+    const supplement = (await loadLicenseSupplements())
+      .get(LIBVIPS)
+      ?.find((candidate) => candidate.version === "1.3.4");
+    expect(loaded?.recipe.package.contentDigest).toBe(supplement?.contentDigest);
+
+    // And those are the bytes sharp 0.35.5 installs.
+    const root = await realpath(await installedPackageFile("sharp"));
+    const target = NATIVE_TARGETS.get("linux-x64-glibc");
+    if (target === undefined) throw new Error("target");
+    const installed = (await nativeClosure([{ name: "sharp", root }], target)).find(
+      (entry) => entry.name === LIBVIPS,
+    );
+    expect(installed && contentDigest(installed.files)).toBe(loaded?.recipe.package.contentDigest);
+
+    // Every component the supplement records inside the binary maps to a source component. The one
+    // exception is the package's own entry, which carries the standard texts of its declared licence.
+    const referenced = new Set(
+      (loaded?.recipe.components ?? []).flatMap((component) =>
+        component.licenseMaterial.map((material) => `${material.name} ${material.version}`),
+      ),
+    );
+    const unmapped = (supplement?.components ?? [])
+      .map((component) => `${component.name} ${component.version}`)
+      .filter((component) => !referenced.has(component));
+    expect(unmapped).toEqual([`${LIBVIPS} 1.3.4`]);
+
+    const components = loaded?.recipe.components ?? [];
+    const byRole = (role: string) =>
+      components.filter((component) => component.role === role).length;
+    expect({
+      buildDefinition: byRole("build-definition"),
+      buildInput: byRole("build-input"),
+      compiled: byRole("compiled"),
+    }).toEqual({
+      buildDefinition: 1,
+      buildInput: 206,
+      compiled: 176,
+    });
+    // The five patches build/posix.sh applies, and the Rust standard library of the recorded nightly.
+    expect(
+      components.flatMap((component) => component.patches.map((patch) => patch.file)).sort(),
+    ).toEqual([
+      "patches/glib-without-gregex.patch",
+      "patches/librsvg-9106011db93d701728ac2c9da50c9ab2c1bb5dc6.patch",
+      "patches/libultrahdr-pull-383.patch",
+      "patches/libvips-cpp-soversion.patch",
+      "patches/mozjpeg-f90668e0e4fb79c81e1f24a0ccc0e2090af761bf.patch",
+    ]);
+    expect(
+      components.find((component) => component.name === "Rust standard library")?.sources[0]?.file,
+    ).toBe("sources/rust/rust-src-nightly-2026-09-27.tar.xz");
+    // gvdb ships inside glib's archive; proxy-libintl is never built for this target.
+    expect(components.find((component) => component.name === "gvdb")?.includedIn).toEqual({
+      component: "glib",
+      path: "subprojects/gvdb",
+    });
+    expect(components.some((component) => component.name === "proxy-libintl")).toBe(false);
+    expect(loaded?.recipe.exclusions.map((exclusion) => exclusion.name)).toContain("proxy-libintl");
+  });
+});
+
+async function packNative(): Promise<string> {
+  const moduleDirectory = await nativeFixtureModule();
+  const outDirectory = await scratchDirectory("ekklesia-native-out-");
+  const result = await packModule({
+    licenseFile: path.join(repositoryRoot, "LICENSE"),
+    moduleDirectory,
+    nodeRange: ">=24.11.0 <25",
+    outDirectory,
+    sourceRevision: SOURCE_REVISION,
+    target: "linux-x64-glibc",
+  });
+  return result.archivePath;
+}
+
+describe.runIf(onTargetHost && cache !== undefined && existsSync(cache ?? ""))(
+  "the native proof's corresponding source, from the verified cache",
+  () => {
+    it("builds deterministically, is approved by the gate, and fails closed without it", {
+      timeout: 600_000,
+    }, async () => {
+      const first = await packNative();
+      const second = await packNative();
+      expect((await readFile(first)).equals(await readFile(second))).toBe(true);
+
+      const outA = await scratchDirectory("ekklesia-source-a-");
+      const outB = path.join(await scratchDirectory("ekklesia-unrelated-"), "nested", "out");
+      const a = await buildCorrespondingSource({
+        cacheDirectory: cache ?? "",
+        outDirectory: outA,
+        packageArchivePath: first,
+      });
+      const b = await buildCorrespondingSource({
+        cacheDirectory: cache ?? "",
+        outDirectory: outB,
+        packageArchivePath: second,
+      });
+      expect(a.required).toBe(true);
+      expect(a.sourceSha256).toBe(b.sourceSha256);
+      expect(
+        (await readFile(a.sourceArchivePath ?? "")).equals(
+          await readFile(b.sourceArchivePath ?? ""),
+        ),
+      ).toBe(true);
+      expect(path.basename(a.sourceArchivePath ?? "")).toBe(
+        "native-proof-0.1.0-linux-x64-glibc-corresponding-source.tar.gz",
+      );
+
+      // What it holds: every recorded source, as bytes, and nothing unrecorded.
+      const archive = await readFile(a.sourceArchivePath ?? "");
+      const entries = readTarGz(archive);
+      const top = "native-proof-0.1.0-linux-x64-glibc-corresponding-source";
+      const files = entries
+        .filter((entry) => !entry.directory)
+        .map((entry) => entry.path.slice(top.length + 1));
+      const manifest = JSON.parse(
+        entries
+          .find((entry) => entry.path === `${top}/corresponding-source.json`)
+          ?.data.toString("utf8") ?? "{}",
+      );
+      const dependency = manifest.dependencies[0];
+      expect(dependency).toMatchObject({
+        name: LIBVIPS,
+        version: "1.3.4",
+        binaries: ["lib/libvips-cpp.so.8.18.7"],
+      });
+      const base = `dependencies/${LIBVIPS}/1.3.4`;
+      const sources = files.filter((file) => file.startsWith(`${base}/sources/`));
+      expect(sources.filter((file) => file.startsWith(`${base}/sources/crates/`))).toHaveLength(
+        353,
+      );
+      expect(
+        sources.filter((file) => /\/sources\/[^/]+\.tar\.(?:gz|xz)$/u.test(file)),
+      ).toHaveLength(28);
+      expect(files.filter((file) => file.startsWith(`${base}/patches/`))).toHaveLength(5);
+      for (const required of [
+        "BUILDING.md",
+        "PROVENANCE.md",
+        "recipe.json",
+        "licenses/supplement.json",
+        "build/prepare-offline.sh",
+        "build/Cargo.lock",
+        "sources/rust/rust-src-nightly-2026-09-27.tar.xz",
+      ]) {
+        expect(files, required).toContain(`${base}/${required}`);
+      }
+      expect(new Set(manifest.files.map((file: { path: string }) => file.path))).toEqual(
+        new Set(files.filter((file) => file !== "corresponding-source.json")),
+      );
+
+      // The binary-source relationship, by digest.
+      const metadata = JSON.parse(await readFile(a.metadataPath ?? "", "utf8"));
+      const packageSha256 = createHash("sha256")
+        .update(await readFile(first))
+        .digest("hex");
+      expect(metadata).toMatchObject({
+        correspondingSource: {
+          file: path.basename(a.sourceArchivePath ?? ""),
+          sha256: a.sourceSha256,
+          size: archive.length,
+        },
+        covered: [
+          {
+            contentDigest:
+              "ekklesia-content-v1:sha256:2927fe11493461fa34847191d0b9b1c6441007c3e99f306e5708cb07fb0a5e59",
+            name: LIBVIPS,
+            version: "1.3.4",
+          },
+        ],
+        package: {
+          file: path.basename(first),
+          module: { id: "native-proof", version: "0.1.0" },
+          sha256: packageSha256,
+          target: "linux-x64-glibc",
+        },
+      });
+      expect(metadata.notice).toContain("GNU LGPL and MPL");
+      expect(metadata.notice).toContain(`SHA-256 ${a.sourceSha256}`);
+
+      // The gate approves exactly this, and fails closed when it is removed or corrupted.
+      await expect(
+        verifyRedistribution({
+          metadataPath: a.metadataPath ?? "",
+          packageArchivePath: first,
+          sourceArchivePath: a.sourceArchivePath ?? "",
+        }),
+      ).resolves.toMatchObject({ approved: true, correspondingSource: { sha256: a.sourceSha256 } });
+      const missing = verifyRedistribution({
+        metadataPath: a.metadataPath ?? "",
+        packageArchivePath: first,
+      });
+      await expect(missing).rejects.toBeInstanceOf(CorrespondingSourceError);
+      await expect(missing).rejects.toMatchObject({
+        code: "CORRESPONDING_SOURCE_ARTIFACT_MISSING",
+      });
+      const corrupted = Buffer.from(archive);
+      const flip = Math.floor(corrupted.length / 3);
+      corrupted[flip] = (corrupted[flip] ?? 0) ^ 0x01;
+      await writeFile(b.sourceArchivePath ?? "", corrupted);
+      await expect(
+        verifyRedistribution({
+          metadataPath: b.metadataPath ?? "",
+          packageArchivePath: second,
+          sourceArchivePath: b.sourceArchivePath ?? "",
+        }),
+      ).rejects.toMatchObject({
+        code: expect.stringMatching(
+          /^CORRESPONDING_SOURCE_(?:ARTIFACT_INVALID|MANIFEST_MISMATCH|METADATA_MISMATCH)$/u,
+        ),
+      });
+      await rm(b.sourceArchivePath ?? "");
+      await expect(
+        verifyRedistribution({
+          metadataPath: b.metadataPath ?? "",
+          packageArchivePath: second,
+          sourceArchivePath: b.sourceArchivePath ?? "",
+        }),
+      ).rejects.toMatchObject({ code: "CORRESPONDING_SOURCE_ARTIFACT_MISSING" });
+    });
+
+    it("leaves the binary package itself unchanged: it verifies, and sharp runs from it", {
+      timeout: 300_000,
+    }, async () => {
+      const archivePath = await packNative();
+      const archive = await readFile(archivePath);
+      const extractTo = path.join(
+        await scratchDirectory("ekklesia-native-installed-"),
+        "native-proof",
+      );
+      await expect(
+        verifyModulePackage({
+          archivePath,
+          expectedSha256: createHash("sha256").update(archive).digest("hex"),
+          extractTo,
+          platform: {
+            databaseRuntime: { "drizzle-orm": "0.45.2" },
+            moduleSdkVersion: "0.2.1",
+            platformVersion: "0.2.1",
+          },
+        }),
+      ).resolves.toMatchObject({ moduleId: "native-proof", target: "linux-x64-glibc" });
+      // The source archive never travels inside the package.
+      expect(readTarGz(archive).some((entry) => /corresponding-source/u.test(entry.path))).toBe(
+        false,
+      );
+      const { stdout } = await execFileAsync(
+        process.execPath,
+        [
+          "--input-type=module",
+          "-e",
+          `const m = await import(${JSON.stringify(`file://${extractTo}/server/index.js`)}); process.stdout.write(JSON.stringify(await m.proof()));`,
+        ],
+        { cwd: tmpdir(), env: {} },
+      );
+      expect(JSON.parse(stdout)).toEqual({
+        pixels: [255, 0, 0, 255, 0, 0, 255, 0, 0, 255, 0, 0],
+        png: { format: "png", height: 3, width: 5 },
+        resized: { channels: 3, height: 2, width: 2 },
+      });
+    });
+  },
+);