Merge pull request 'ci: restore external package integration' (#13) from feat/external-signage-integration into main

Reviewed-on: https://git.riverofglorymn.org/River_of_Glory_Church/Ekklesia_Signage/pulls/13
Reviewed-by: The_River <[email protected]>

.gitea/workflows/ci.yml

@@ -1,9 +1,10 @@
 name: CI
 
-# Signage is a standalone repository: every job checks out only this repository, installs it with
+# Signage is a standalone repository: every job installs it with --frozen-lockfile from its own
-# --frozen-lockfile from its own pnpm-lock.yaml, and takes Ekklesia's module-development packages
+# pnpm-lock.yaml and takes Ekklesia's module-development packages (@ekklesia/*) from Ekklesia's
-# (@ekklesia/*) from Ekklesia's public package registry (.npmrc; platform ADRs 0053 and 0054). No job
+# public package registry (.npmrc; platform ADRs 0053 and 0054). Only the integration job fetches
-# clones Ekklesia.
+# Core source: one immutable commit through the pinned Gitea pull ref in ci/platform.env, used as the
+# generic deployment harness and image source, never as a workspace dependency.
 #
 # The same trust rules as the platform's own CI (docs/development/security.md there):
 #   * every action is pinned by a full commit SHA;
@@ -149,26 +150,13 @@ jobs:
       - name: Validate the built module and its migrations
         run: pnpm validate
 
-  # Signage inside Ekklesia, end to end, in a real browser: the platform's own Dockerfile builds the
+  # The real linux-x64-glibc Signage Module Package inside Ekklesia, end to end. The pinned platform
-  # deployment image with Signage in it, and the platform's split-topology deployment smoke runs
+  # builds one generic image with no bundled module source, verifies and installs Signage into
-  # against it, pairing a kiosk display and running the scenarios in ci/display-scenarios. A
+  # persistent module storage, applies its migrations, and drives its split-origin Dashboard and
-  # disposable database and object store with generated secrets; nothing is pushed or logged in to.
+  # kiosk display in real browsers. It then checks process restart, container recreation, lifecycle
-  #
+  # disable/re-enable, removal, and reinstall against the same database and package.
-  # DISABLED. From Ekklesia 0.2.0 on, the platform's image is the one generic image (platform ADR
-  # 0048): deploy/Dockerfile keeps `modules/` out of its build context and refuses one that has it,
-  # and the deployment smoke installs modules only as canonical Module Packages (ADR 0049). Signage is
-  # not one yet: its static sharp dependency needs the native linux-x64-glibc package flow. Run as it
-  # was, this job would build an image without Signage and drive the module template's display
-  # instead, so it is switched off rather than left reporting on something else.
-  #
-  # It returns when Signage is built as a canonical linux-x64-glibc Module Package and installed into
-  # Ekklesia's generic image; that change removes `if: false` and rewrites these steps. They still
-  # expect the platform tree ci/assemble.sh used to build at build/workspace, and that assembly is
-  # gone now that Signage builds on its own. The platform's deployment smoke and stack sweep will
-  # still come from Ekklesia, at the release Signage targets, because they are the platform's to run.
   integration:
     name: integration
-    if: false
     needs:
       - test
       - build
@@ -185,17 +173,26 @@ jobs:
         with:
           persist-credentials: false
 
-      - name: Fetch Ekklesia's deployment smoke and build Signage's Module Package
+      - name: Set up Node.js at the repository's version
+        uses: actions/setup-node@48b55a011bda9f5d6aeb4c2d9c7362e8dae4041e # v6.4.0
+        with:
+          node-version-file: .node-version
+
+      - name: Install (frozen lockfile)
         run: |
-          echo "integration: rewritten when Signage is a canonical linux-x64-glibc Module Package." >&2
+          corepack enable
-          exit 1
+          corepack install
+          pnpm install --frozen-lockfile
+
+      - name: Fetch the pinned Ekklesia integration harness
+        run: bash ci/fetch-platform.sh
 
       # A job the runner killed (cancelled, or past the runner's own time limit) never ran its
       # cleanup. Anything a CI job left on this shared daemon more than three hours ago goes first.
       - name: Sweep stale CI stacks from the runner
-        run: bash build/workspace/scripts/ci-sweep.sh --stale
+        run: bash build/platform/scripts/ci-sweep.sh --stale
 
-      - name: Build the image and run the display scenarios
+      - name: Build, install, and run the external Signage package
         run: bash ci/integration.sh
 
       # Whatever happened above, this job's own stacks do not outlive it. If the first step failed,
@@ -203,6 +200,6 @@ jobs:
       - name: Remove this job's CI stacks
         if: always()
         run: |
-          if [ -f build/workspace/scripts/ci-sweep.sh ]; then
+          if [ -f build/platform/scripts/ci-sweep.sh ]; then
-            bash build/workspace/scripts/ci-sweep.sh --job
+            bash build/platform/scripts/ci-sweep.sh --job
           fi

.gitignore

@@ -3,5 +3,6 @@ node_modules/
 dist/
 .turbo/
 coverage/
+build/
 *.log
 .DS_Store

biome.json

@@ -5,6 +5,7 @@
       "**",
       "!**/.pnpm-store",
       "!**/.turbo",
+      "!**/build",
       "!**/coverage",
       "!**/dist",
       "!**/node_modules",

ci/check-cleanup.sh

@@ -6,7 +6,7 @@
 #
 # The `integration` job is the one that starts Docker stacks. It must have its own
 # EKKLESIA_CI_JOB_KEY, sweep stale CI stacks before it starts anything, and remove its own in a final
-# `if: always()` step; and the image ci/integration.sh builds must carry the CI labels the sweep finds.
+# `if: always()` step. The pinned platform smoke owns and labels every image and container.
 set -Eeuo pipefail
 
 root="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")/.." && pwd)"
@@ -31,13 +31,18 @@ own="$(line_of 'ci-sweep.sh --job')"
 last_step="$(grep -n -- "      - name:" <<<"$job" | tail -1 | cut -d: -f1 || true)"
 
 check '[[ -n "$job" ]]' "the workflow has an integration job"
+check '! grep -q "if: false" <<<"$job"' "the integration job is enabled"
 check 'grep -q "EKKLESIA_CI_JOB_KEY:" <<<"$job"' "the integration job names its stacks with its own key"
 check '[[ -n "$stale" && -n "$run" ]] && ((stale < run))' "it sweeps stale CI stacks before it starts anything"
 check '[[ -n "$own" && -n "$run" ]] && ((own > run)) && ((own > last_step))' \
   "its last step removes its own stacks"
 check 'sed -n "${last_step},\$p" <<<"$job" | grep -q "if: always()"' "that last step always runs"
-check 'grep -q "EKKLESIA_CI_LABEL" "$root/ci/integration.sh" && grep -q "EKKLESIA_CI_CREATED_LABEL" "$root/ci/integration.sh"' \
+check 'grep -q "EKKLESIA_SMOKE_MODULE_RELEASE_DIR" "$root/ci/integration.sh" && grep -q "bash scripts/smoke-deploy.sh" "$root/ci/integration.sh"' \
-  "the image ci/integration.sh builds carries the CI labels"
+  "the integration delegates its external package to the platform smoke"
+check 'grep -Eq "^EKKLESIA_PLATFORM_REF=refs/(pull/[0-9]+/head|heads/main)$" "$root/ci/platform.env" && grep -Eq "^EKKLESIA_PLATFORM_COMMIT=[0-9a-f]{40}$" "$root/ci/platform.env"' \
+  "the platform harness is pinned by full ref and commit"
+check 'grep -q "rev-parse refs/remotes/pinned" "$root/ci/fetch-platform.sh" && grep -q "does not resolve to EKKLESIA_PLATFORM_COMMIT" "$root/ci/fetch-platform.sh"' \
+  "the fetch verifies that the pinned ref resolves to the pinned commit"
 
 ((failures == 0)) || {
   echo "check-cleanup: ${failures} check(s) failed." >&2

ci/fetch-platform.sh

@@ -0,0 +1,39 @@
+#!/usr/bin/env bash
+# Fetches the exact Ekklesia source revision Signage's external-package integration runs against.
+# The ref is fetched only to make the pinned object reachable; the checkout is always by full SHA.
+set -Eeuo pipefail
+
+root="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")/.." && pwd)"
+out="$root/build/platform"
+
+fail() {
+  echo "fetch-platform: $*" >&2
+  exit 1
+}
+
+(($# == 0)) || fail "this command takes no arguments."
+
+pin() {
+  local value
+  value="$(sed -n "s/^$1=//p" "$root/ci/platform.env")"
+  [[ -n "$value" ]] || fail "ci/platform.env has no $1."
+  printf '%s' "$value"
+}
+
+repository="$(pin EKKLESIA_PLATFORM_REPOSITORY)"
+ref="$(pin EKKLESIA_PLATFORM_REF)"
+commit="$(pin EKKLESIA_PLATFORM_COMMIT)"
+[[ "$commit" =~ ^[0-9a-f]{40}$ ]] || fail "EKKLESIA_PLATFORM_COMMIT must be a full commit SHA."
+[[ "$ref" == refs/* ]] || fail "EKKLESIA_PLATFORM_REF must be a full Git ref."
+
+rm -rf "$out"
+mkdir -p "$out"
+git -C "$out" init --quiet
+git -C "$out" remote add origin "$repository"
+git -C "$out" fetch --quiet --no-tags --depth=1 origin "+${ref}:refs/remotes/pinned" ||
+  fail "cannot fetch the pinned platform ref."
+[[ "$(git -C "$out" rev-parse refs/remotes/pinned)" == "$commit" ]] ||
+  fail "the pinned ref does not resolve to EKKLESIA_PLATFORM_COMMIT."
+git -C "$out" -c advice.detachedHead=false checkout --quiet --detach "$commit"
+rm -rf "$out/.git"
+echo "fetch-platform: Ekklesia ${commit}"

ci/integration.sh

@@ -1,46 +1,46 @@
 #!/usr/bin/env bash
-# Signage inside Ekklesia, end to end: builds the deployment image from the assembled workspace with
+# Builds Signage as the canonical linux-x64-glibc Module Package, then gives that exact release pair
-# the platform's own Dockerfile, then runs the platform's split-topology deployment smoke against it
+# to the pinned platform's generic deployment smoke. The platform image contains no Signage source:
-# with the display-device scenarios listed in ci/display-scenarios. A disposable PostgreSQL and object
+# its verifier extracts the package into installation-owned module storage before migrations,
-# store, generated secrets, and an explicit migration apply against that fresh database only; nothing
+# discovery, browser UI/display checks, restart, container recreation, disable/re-enable, removal,
-# is pushed, published, or logged in to. The image is removed afterwards.
+# and reinstall are exercised against disposable PostgreSQL and object-storage services.
 #
+#   bash ci/fetch-platform.sh
 #   bash ci/integration.sh
-#
-# NOT RUN: the integration job is disabled (.gitea/workflows/ci.yml says why). This script still
-# expects Ekklesia's deployment tree at build/workspace, which the removed ci/assemble.sh used to
-# build; it is rewritten when Signage is installed into the generic image as a Module Package.
 set -Eeuo pipefail
 
 root="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")/.." && pwd)"
-workspace="$root/build/workspace"
+platform="$root/build/platform"
-[[ -f "$workspace/deploy/Dockerfile" ]] || {
+release="$root/build/module-package"
-  echo "integration: Ekklesia's deployment tree is not at build/workspace." >&2
+[[ -f "$platform/scripts/smoke-deploy.sh" ]] || {
+  echo "integration: run bash ci/fetch-platform.sh first." >&2
   exit 1
 }
 
-# Named and labeled as a CI stack of this run and job (the platform's scripts/lib/ci-project.sh), so
+source_revision="${GITHUB_SHA:-$(git -C "$root" rev-parse HEAD 2>/dev/null || true)}"
-# the workflow's final `ci-sweep.sh --job` removes it even if this script is killed before its trap.
+[[ "$source_revision" =~ ^[0-9a-f]{40}$ ]] || {
-# shellcheck source=/dev/null
+  echo "integration: no full source revision is available for the package." >&2
-source "$workspace/scripts/lib/ci-project.sh"
+  exit 1
-project="$(ci_project_name signage-image)"
+}
-image="${project}:latest"
+
-trap 'docker image rm --force "$image" >/dev/null 2>&1 || true' EXIT
+rm -rf "$release"
+mkdir -p "$release"
+pnpm --dir "$root" build
+pnpm --dir "$root" exec ekklesia-pack-module --module "$root" --target linux-x64-glibc \
+  --out "$release" --license-file "$root/LICENSE" --source-revision "$source_revision"
 
 scenarios="$(sed -e 's/#.*//' -e '/^[[:space:]]*$/d' "$root/ci/display-scenarios" | tr '\n' ' ')"
 [[ -n "${scenarios// /}" ]] || {
   echo "integration: ci/display-scenarios lists no scenario." >&2
   exit 1
 }
+echo "integration: external Signage package built; scenarios: ${scenarios}"
 
-DOCKER_BUILDKIT=1 docker build --quiet --file "$workspace/deploy/Dockerfile" --tag "$image" \
+cd "$platform"
-  --label "${EKKLESIA_CI_LABEL}=${project}" --label "${EKKLESIA_CI_CREATED_LABEL}=$(date +%s)" \
-  "$workspace" >/dev/null
-echo "integration: built the deployment image with Signage; scenarios: ${scenarios}"
-
-cd "$workspace"
 EKKLESIA_SMOKE_TOPOLOGY=split \
-  EKKLESIA_SMOKE_IMAGE="$image" \
+  EKKLESIA_SMOKE_MODULE_RELEASE_DIR="$release" \
+  EKKLESIA_SMOKE_MODULE_ID=signage \
+  EKKLESIA_SMOKE_MODULE_PROBE_PATH=/api/v1/modules/signage/channel-groups \
   EKKLESIA_SMOKE_DISPLAY_DEVICE=1 \
   EKKLESIA_SMOKE_DISPLAY_SCENARIOS="$scenarios" \
   EKKLESIA_SMOKE_WATCH_MS=120000 \

ci/platform.env

@@ -0,0 +1,5 @@
+# The companion Core harness Signage uses to install and run its real external package. The pull ref
+# remains reachable after its branch is deleted; the full commit pin makes the checkout immutable.
+EKKLESIA_PLATFORM_REPOSITORY=https://git.riverofglorymn.org/River_of_Glory_Church/Ekklesia.git
+EKKLESIA_PLATFORM_REF=refs/pull/90/head
+EKKLESIA_PLATFORM_COMMIT=40ef0ff802c124ae5a3a0b9aa51a1502a7b72644

src/server/images/processing.ts

@@ -177,10 +177,11 @@ export async function processSignageImage(bytes: Uint8Array): Promise<SignagePro
     height: metadata.height as number,
     width: metadata.width as number,
   };
-  const [display, thumbnail] = await Promise.all([
+  // Keep peak native CPU and memory bounded inside the shared API process. Upload concurrency is
-    renderRendition(bytes, SIGNAGE_IMAGE_DISPLAY_RENDITION),
+  // already bounded at the module UI; running both renditions for every upload in parallel would
-    renderRendition(bytes, SIGNAGE_IMAGE_THUMBNAIL_RENDITION),
+  // multiply that concurrency again and can starve the database and storage clients on a small host.
-  ]);
+  const display = await renderRendition(bytes, SIGNAGE_IMAGE_DISPLAY_RENDITION);
+  const thumbnail = await renderRendition(bytes, SIGNAGE_IMAGE_THUMBNAIL_RENDITION);
 
   return {
     byteSize: bytes.byteLength,

src/server/images/service.ts

@@ -1,8 +1,9 @@
 import { randomUUID } from "node:crypto";
 import {
+  MODULE_STORAGE_ERROR_CODES,
   type ModuleStorage,
   type ModuleStorageBucket,
-  ModuleStorageError,
+  type ModuleStorageErrorCode,
 } from "@ekklesia/module-sdk";
 import { SignageError } from "../errors.js";
 import { type SignageStore, SignageStoreConflict, type StoredImage } from "../store/types.js";
@@ -17,6 +18,20 @@ export function nativeImageBucket(storage: ModuleStorage): ModuleStorageBucket {
   return storage.bucket(SIGNAGE_IMAGE_BUCKET);
 }
 
+/** Host errors cross the installed-package boundary from a different copy of the SDK, so class
+ * identity is not part of the contract (ADR 0049). Recognize only the SDK's closed, sanitized
+ * storage-error shape; provider messages and causes remain inaccessible. */
+function storageErrorCode(error: unknown): ModuleStorageErrorCode | undefined {
+  if (!(error instanceof Error) || error.name !== "ModuleStorageError") {
+    return undefined;
+  }
+  const code = (error as { readonly code?: unknown }).code;
+  return typeof code === "string" &&
+    (MODULE_STORAGE_ERROR_CODES as readonly string[]).includes(code)
+    ? (code as ModuleStorageErrorCode)
+    : undefined;
+}
+
 /**
  * Object keys are scoped to the asset they belong to.
  *
@@ -41,7 +56,7 @@ export function renditionObjectKey(assetId: string, kind: string): string {
  * not a caller's business.
  */
 function storageFailure(error: unknown): SignageError {
-  if (error instanceof ModuleStorageError && error.code === "STORAGE_OBJECT_NOT_FOUND") {
+  if (storageErrorCode(error) === "STORAGE_OBJECT_NOT_FOUND") {
     return new SignageError("SIGNAGE_NOT_FOUND");
   }
   return new SignageError("SIGNAGE_STORAGE_UNAVAILABLE");
@@ -150,7 +165,7 @@ export async function ingestNativeImage(
     if (error instanceof SignageError) {
       throw error;
     }
-    if (error instanceof ModuleStorageError) {
+    if (storageErrorCode(error) !== undefined) {
       throw storageFailure(error);
     }
     throw new SignageError("SIGNAGE_OPERATION_FAILED");

tests/images.test.ts

@@ -1,3 +1,4 @@
+import type { ModuleStorage } from "@ekklesia/module-sdk";
 import sharp from "sharp";
 import { beforeEach, describe, expect, it } from "vitest";
 import { SignageError } from "../src/server/errors.js";
@@ -372,6 +373,30 @@ describe("native image ingest", () => {
     expect(store.images.size).toBe(0);
   });
 
+  it("recognizes a sanitized storage failure from the host's separate SDK copy", async () => {
+    const bucket = storage.bucket("native-images");
+    const externalError = Object.assign(new Error("The storage service could not be reached."), {
+      code: "STORAGE_PROVIDER_UNAVAILABLE",
+      name: "ModuleStorageError",
+    });
+    const externalStorage: ModuleStorage = {
+      bucket: () => ({ ...bucket, put: () => Promise.reject(externalError) }),
+      buckets: () => ["native-images"],
+    };
+
+    await expectSignageCode(
+      ingestNativeImage({
+        bytes: await makeImage({ format: "png", height: 360, width: 640 }),
+        displayName: "welcome.png",
+        expiresAt: null,
+        filename: "welcome.png",
+        storage: externalStorage,
+        store,
+      }),
+      "SIGNAGE_STORAGE_UNAVAILABLE",
+    );
+  });
+
   it("treats a race with an identical upload as a success rather than a failure", async () => {
     const bytes = await makeImage({ format: "png", height: 1080, width: 1920 });
     const first = await ingest(bytes);