feat: one generic image, with installed modules in installation-owned storage (ADR 0048)

The production image now contains zero feature modules and is the same for every installation. Installed modules live in module storage that one Core resolver selects from EKKLESIA_MODULES_DIR: the checkout's modules/ when unset, an absolute path that must exist when set (/var/lib/ekklesia/modules in the image). A relative, blank, missing, or unreadable configured directory fails closed with a sanitized error that never contains the value. - Host configuration reads EKKLESIA_MODULES_DIR. The API, migration plan, apply and validate, module validation, module enablement, and Doctor all resolve it the same way; the test-only API override still takes precedence. - Doctor receives Core's migrations directory explicitly instead of deriving it from the modules directory's parent. - The migration runner retains and reports the applied history of a module that is not installed, and plans and applies Core's and present modules' migrations around it. Core's history and every installed module's history stay strict: a missing or changed applied migration, or a downgrade, still fails. No down migrations, no data deletion. - deploy/Dockerfile copies no module and installs nothing for the module template; .dockerignore excludes modules/ and the build refuses a context that contains it; the runtime has an empty root-owned /var/lib/ekklesia/modules and sets EKKLESIA_MODULES_DIR. - deploy/compose.yaml bind-mounts EKKLESIA_MODULES_HOST_DIR read-only into api and migrate, with create_host_path disabled. - scripts/check-image.sh proves the image holds no modules and that a populated modules/ directory leaves its runtime file system identical; CI runs it. - The split deploy smoke now uses the same generic image: it boots with no modules, installs a self-contained test module packaged from the template (test-only), migrates and restarts, removes it (Core runs, planning reports the retained history, data survives), and restores it with its data. Docs: deploy README (module storage, install/update/remove/restore, the runtime modules target), .env.example, API configuration, module development guide, testing, workflow, security, and the architecture overview. Co-Authored-By: Claude Opus 5.5 <[email protected]>
2026-10-07 08:38:14 -05:00
parent 4d28a82937
commit e36b70d8d4
50 changed files with 2275 additions and 224 deletions
@@ -0,0 +1,146 @@
+#!/usr/bin/env bash
+# Proves that Ekklesia's image is the one generic platform artifact ADR 0048 requires:
+#
+#   - it contains zero feature modules: no /opt/ekklesia/modules, no module template, and no
+#     ekklesia.module.json anywhere outside installed dependencies;
+#   - its module storage mount point, /var/lib/ekklesia/modules, exists, is root-owned, and is empty;
+#   - EKKLESIA_MODULES_DIR names that directory, in the image configuration and in a running process;
+#   - a checkout with a populated modules/ directory builds the same image: the runtime file system
+#     of both builds is identical, and no module from that directory enters it.
+#
+#   bash scripts/check-image.sh                        builds both images from this checkout
+#   EKKLESIA_CHECK_IMAGE=<image> bash scripts/check-image.sh   checks one existing image's contents
+#
+# Both builds use a copy of the files Git would track (committed, staged, or untracked but not
+# ignored), so they differ in exactly one thing: the second context also has modules/example. Images
+# are built and inspected locally only; nothing is pushed, published, or logged in to.
+
+set -Eeuo pipefail
+
+repository_root="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")/.." && pwd)"
+cd "$repository_root"
+
+# shellcheck source=scripts/lib/ci-project.sh
+source "$repository_root/scripts/lib/ci-project.sh"
+
+command -v docker >/dev/null || {
+  echo "check-image: docker is required." >&2
+  exit 1
+}
+
+project="$(ci_project_name image)"
+modules_directory="/var/lib/ekklesia/modules"
+work="$(mktemp -d)"
+built_images=()
+
+cleanup() {
+  local status=$?
+  trap - EXIT INT TERM
+  for built in "${built_images[@]}"; do
+    docker image rm --force "$built" >/dev/null 2>&1 || true
+  done
+  rm -rf "$work" || true
+  exit "$status"
+}
+trap cleanup EXIT INT TERM
+
+step() {
+  echo "==> $*"
+}
+
+fail() {
+  echo "check-image: $*" >&2
+  exit 1
+}
+
+# check_contents <image>: every content guarantee, inside the image and in its configuration.
+check_contents() {
+  local image="$1" configured
+  configured="$(docker image inspect --format '{{range .Config.Env}}{{println .}}{{end}}' "$image")" ||
+    fail "could not inspect ${image}."
+  grep -qx "EKKLESIA_MODULES_DIR=${modules_directory}" <<<"$configured" ||
+    fail "the image does not set EKKLESIA_MODULES_DIR=${modules_directory}."
+
+  # As root, so a directory the runtime user cannot read cannot hide a manifest. No network.
+  docker run --rm --network none --user 0:0 --entrypoint sh "$image" -ec '
+    fail() { echo "check-image: $*" >&2; exit 1; }
+    [ ! -e /opt/ekklesia/modules ] || fail "/opt/ekklesia/modules exists in the image."
+    [ ! -e /opt/ekklesia/templates ] || fail "the module template is in the image."
+    [ -d /var/lib/ekklesia/modules ] || fail "/var/lib/ekklesia/modules is missing."
+    [ "$(stat -c %U:%G:%a /var/lib/ekklesia/modules)" = "root:root:755" ] ||
+      fail "/var/lib/ekklesia/modules is not root-owned with mode 755."
+    [ -z "$(ls -A /var/lib/ekklesia/modules)" ] || fail "/var/lib/ekklesia/modules is not empty."
+    manifests="$(find / -xdev -name ekklesia.module.json -not -path "*/node_modules/*" 2>/dev/null)"
+    [ -z "$manifests" ] || fail "a module manifest is in the image: ${manifests}"
+    [ "$EKKLESIA_MODULES_DIR" = /var/lib/ekklesia/modules ] ||
+      fail "a process in the image does not see EKKLESIA_MODULES_DIR."
+  ' || fail "${image} is not the generic platform image."
+  echo "ok   ${image}: no modules, no template, empty root-owned module storage, EKKLESIA_MODULES_DIR set"
+}
+
+# runtime_listing <image>: every path the platform and its module storage consist of, with its
+# type, mode, and size, in a stable order. Two builds that differ here differ in content.
+runtime_listing() {
+  docker run --rm --network none --user 0:0 --entrypoint sh "$1" -ec \
+    'find /opt/ekklesia /var/lib/ekklesia -xdev -printf "%y %m %s %p\n" | LC_ALL=C sort'
+}
+
+if [[ -n "${EKKLESIA_CHECK_IMAGE:-}" ]]; then
+  check_contents "$EKKLESIA_CHECK_IMAGE"
+  exit 0
+fi
+
+command -v git >/dev/null || fail "git is required to copy the build context."
+
+# context_copy <directory>: the files Git would track, and nothing it ignores.
+context_copy() {
+  mkdir -p "$1"
+  git ls-files -z --cached --others --exclude-standard |
+    tar --null --files-from=- --create --file=- |
+    tar -C "$1" -xf -
+}
+
+tag_base="ekklesia-ci:${GITHUB_SHA:-local}-${GITHUB_RUN_ID:-0}-${GITHUB_RUN_ATTEMPT:-1}-"
+generic_image="${tag_base}generic-$$"
+populated_image="${tag_base}populated-$$"
+
+# build_image <context> <tag>
+build_image() {
+  built_images+=("$2")
+  DOCKER_BUILDKIT=1 docker build --quiet --file "$1/deploy/Dockerfile" \
+    --build-arg REVISION="${GITHUB_SHA:-}" \
+    --label "${EKKLESIA_CI_LABEL}=${project}" --label "${EKKLESIA_CI_CREATED_LABEL}=$(date +%s)" \
+    --tag "$2" "$1" >/dev/null || fail "the image did not build from ${1##*/}."
+}
+
+step "Building the platform image from this checkout"
+context_copy "$work/generic"
+[[ ! -e "$work/generic/modules" ]] || fail "this checkout itself has a modules/ directory."
+build_image "$work/generic" "$generic_image"
+check_contents "$generic_image"
+
+step "Building again from the same checkout with a populated modules/ directory"
+context_copy "$work/populated"
+mkdir -p "$work/populated/modules"
+cp -R "$work/populated/templates/module" "$work/populated/modules/example"
+printf 'must never reach an image\n' >"$work/populated/modules/example/canary.txt"
+build_image "$work/populated" "$populated_image"
+check_contents "$populated_image"
+
+step "Comparing the two images' runtime file systems"
+runtime_listing "$generic_image" >"$work/generic.listing"
+runtime_listing "$populated_image" >"$work/populated.listing"
+[[ -s "$work/generic.listing" ]] || fail "could not list the image's file system."
+if ! diff -u "$work/generic.listing" "$work/populated.listing" >"$work/listing.diff"; then
+  head -40 "$work/listing.diff" >&2
+  fail "a populated modules/ directory changed the image."
+fi
+echo "ok   both builds have the identical runtime file system ($(wc -l <"$work/generic.listing") paths)"
+if [[ "$(docker image inspect --format '{{json .RootFS.Layers}}' "$generic_image")" == \
+  "$(docker image inspect --format '{{json .RootFS.Layers}}' "$populated_image")" ]]; then
+  echo "ok   both builds share every layer"
+else
+  echo "note the two builds' layers differ (a cold build cache); their file systems do not"
+fi
+
+echo "Generic platform image check passed."
@@ -6,6 +6,9 @@
 //   node deploy-smoke-client.mjs edge     an ordinary, untrusted peer of the proxy
 //   node deploy-smoke-client.mjs tunnel   the Cloudflare Tunnel connector's fixed address
 //   node deploy-smoke-client.mjs session  the initial owner signing in and out through the proxy
+//   node deploy-smoke-client.mjs module   the test module `example` as the initial owner sees it:
+//                                          SMOKE_MODULE_EXPECT is "absent", or a JSON array of the
+//                                          exact values its records route must return (ADR 0048)
 //
 // Either topology (ADR 0045): SMOKE_HOSTNAME is the API's hostname, and SMOKE_DASHBOARD_HOSTNAME,
 // when set, the Dashboard's. Unset means the same-origin topology, where both are one hostname. Every
@@ -443,14 +446,72 @@ async function sessionChecks() {
  );
 }

+/** Signs the initial owner in, through the proxy, and returns the session cookie. */
+async function ownerCookie() {
+  const signIn = await fetch(`${apiOrigin}/api/auth/sign-in/email`, {
+    body: JSON.stringify({
+      email: required("SMOKE_OWNER_EMAIL"),
+      password: required("SMOKE_OWNER_PASSWORD"),
+    }),
+    headers: { "content-type": "application/json", origin: dashboardOrigin },
+    method: "POST",
+  });
+  await signIn.arrayBuffer();
+  check(signIn.status === 200, "the initial owner signs in");
+  return (signIn.headers.get("set-cookie") ?? "").split(";")[0] ?? "";
+}
+
+/**
+ * The installed-module checks (ADR 0048): whether `example` is in the module inventory, and what
+ * its own route returns. Absent, it is in no inventory and its routes are not mounted at all;
+ * present, its records route returns exactly the expected values, which proves its tables, its
+ * dependencies, and the platform's database facade all work from module storage.
+ */
+async function moduleChecks() {
+  const expectation = required("SMOKE_MODULE_EXPECT");
+  const cookie = await ownerCookie();
+  const headers = { cookie, origin: dashboardOrigin };
+
+  const inventory = await fetch(`${apiOrigin}/api/v1/module-ui`, { headers });
+  const inventoryBody = await inventory.json();
+  const installed =
+    inventory.status === 200 && inventoryBody.modules.some((module) => module.id === "example");
+  const records = await fetch(`${apiOrigin}/api/v1/modules/example/records`, { headers });
+
+  if (expectation === "absent") {
+    check(inventory.status === 200 && !installed, "the module inventory has no `example` module");
+    check(
+      records.status === 404 && (await records.text()) === '{"code":"NOT_FOUND"}',
+      "the absent module's routes are not mounted",
+    );
+  } else {
+    const expected = JSON.parse(expectation);
+    check(installed, "the module inventory lists the `example` module");
+    const values = records.status === 200 ? (await records.json()).map((row) => row.value) : null;
+    check(
+      JSON.stringify(values) === JSON.stringify(expected),
+      `the module's records route returns ${JSON.stringify(expected)}`,
+    );
+  }
+
+  const signOut = await fetch(`${apiOrigin}/api/auth/sign-out`, {
+    body: "{}",
+    headers: { ...headers, "content-type": "application/json" },
+    method: "POST",
+  });
+  await signOut.arrayBuffer();
+}
+
 if (mode === "edge") {
  await edgeChecks();
 } else if (mode === "tunnel") {
  await tunnelChecks();
 } else if (mode === "session") {
  await sessionChecks();
+} else if (mode === "module") {
+  await moduleChecks();
 } else {
-  throw new Error('Mode must be "edge", "tunnel", or "session".');
+  throw new Error('Mode must be "edge", "tunnel", "session", or "module".');
 }

 if (failures > 0) {
@@ -0,0 +1,50 @@
+# TEST-ONLY. Builds the repository's module template as one self-contained installed-module directory
+# for scripts/smoke-deploy.sh, which installs it into a deployment's module storage (ADR 0048). It is
+# not a module package format and defines none: the archive format, signing, and transport of real
+# module packages are deferred. No image is produced; the build's only output is the directory:
+#
+#   DOCKER_BUILDKIT=1 docker build --file scripts/lib/test-module-package.Dockerfile \
+#     --output type=local,dest=<directory> .
+#
+# writes <directory>/example: the module's package.json, manifest, built artifacts, migrations, and a
+# node_modules holding exactly its production dependencies, including its own copies of the Module
+# SDK, the contracts, and Drizzle. Nothing in it resolves to anything outside it, because the
+# platform lends an installed module nothing.
+#
+# The same pinned Node.js base as deploy/Dockerfile, so the package targets the runtime it is
+# installed into. Drizzle comes from the platform's own lockfile, which is what keeps it the exact
+# version the platform uses: a constraint of the current implementation (ADR 0048), not a contract.
+FROM node:24.18.0-trixie-slim@sha256:ae91dcc111a68c9d2d81ff2a17bda61be126426176fde6fe7d08ab13b7f50573 AS build
+ENV COREPACK_ENABLE_DOWNLOAD_PROMPT=0 \
+    PNPM_HOME=/pnpm \
+    TURBO_TELEMETRY_DISABLED=1 \
+    DO_NOT_TRACK=1
+WORKDIR /opt/source
+COPY package.json ./
+RUN corepack enable && corepack install
+COPY . .
+RUN --mount=type=cache,id=ekklesia-pnpm-store,target=/pnpm/store \
+    pnpm install --frozen-lockfile --store-dir /pnpm/store
+RUN pnpm exec turbo run build --filter=@ekklesia/module-template
+# `--legacy` deploys from this workspace without switching it to injected workspace packages; the
+# Module SDK and every other workspace dependency are still copied into the package, built.
+RUN --mount=type=cache,id=ekklesia-pnpm-store,target=/pnpm/store \
+    set -eu; \
+    pnpm --filter @ekklesia/module-template deploy --prod --legacy --store-dir /pnpm/store \
+      /package/example; \
+    rm -f /package/example/README.md; \
+    # pnpm's hoisted self-reference points back into this workspace; an installed module has none.
+    rm -f /package/example/node_modules/.pnpm/node_modules/@ekklesia/module-template; \
+    escaping=""; \
+    for link in $(find /package/example -type l); do \
+      case "$(realpath -m "$link")" in \
+        /package/example/*) ;; \
+        *) escaping="$escaping $link" ;; \
+      esac; \
+    done; \
+    if [ -n "$escaping" ]; then echo "Links escape the package:$escaping" >&2; exit 1; fi; \
+    chmod -R a+rX,go-w /package/example; \
+    cd /package/example && node -e 'import("./dist/server/index.js").then((m) => { if (m.default.id !== "example") process.exit(1); })'
+
+FROM scratch AS package
+COPY --from=build /package/ /
@@ -1,19 +1,30 @@
 #!/usr/bin/env bash
-# End-to-end smoke test of the reference deployment (ADR 0044): deploy/compose.yaml with
-# deploy/compose.proxy.yaml, a platform-only image, and a disposable PostgreSQL and SeaweedFS that
-# exist only for this run. CI runs it; it is safe to run on a workstation.
+# End-to-end smoke test of the reference deployment (ADR 0044, ADR 0048): deploy/compose.yaml with
+# deploy/compose.proxy.yaml, the one generic platform image, installation-owned module storage, and
+# a disposable PostgreSQL and SeaweedFS that exist only for this run. CI runs it; it is safe to run
+# on a workstation.
 #
 #   bash scripts/smoke-deploy.sh                 builds the image first
 #   EKKLESIA_SMOKE_IMAGE=<image> bash scripts/smoke-deploy.sh
 #   EKKLESIA_SMOKE_TOPOLOGY=split bash scripts/smoke-deploy.sh
 #
-# Either topology (ADR 0045). `same-origin` (the default) serves the Dashboard, the API, and displays
-# from one hostname with Caddyfile.example and a platform-only image. `split` serves the Dashboard on
-# its own hostname with Caddyfile.split.example, builds an image with the repository's module
-# template installed as a test-only module (so a module UI, its API routes, and a display exist), and
-# then drives a real headless Chromium through the Dashboard, the module, the display, and sign-out.
-# With EKKLESIA_SMOKE_IMAGE, the given image is used as is in either topology; a split run then
-# exercises whichever of its modules publishes UI and a display.
+# Either topology (ADR 0045), and in both the same generic image, built once from deploy/Dockerfile.
+# It contains no module; module storage starts empty, as on a fresh installation. `same-origin` (the
+# default) serves the Dashboard, the API, and displays from one hostname with Caddyfile.example.
+# `split` serves the Dashboard on its own hostname with Caddyfile.split.example and then exercises
+# ADR 0048 end to end with that same image:
+#
+#   - it boots with no modules;
+#   - the repository's module template, packaged as one self-contained directory
+#     (scripts/lib/test-module-package.Dockerfile, TEST-ONLY), is installed into module storage, its
+#     migrations are planned and explicitly applied, and the API is restarted;
+#   - a real headless Chromium drives the Dashboard, the module, its display, and sign-out;
+#   - the module is removed and the API restarted: Core still runs, migration planning reports the
+#     module's retained history, and its data survives;
+#   - the module is restored and the API restarted: it returns with its previous data.
+#
+# EKKLESIA_SMOKE_IMAGE uses the given image instead of building one. It must be a generic image
+# (ADR 0048): modules come from module storage, never from the image.
 #
 # Opt-in, split topology only: EKKLESIA_SMOKE_DISPLAY_DEVICE=1 adds the display-device scenarios in
 # scripts/lib/deploy-smoke-display.mjs (pairing a kiosk browser with a persistent profile, a clean
@@ -113,6 +124,8 @@ choose_edge_network() {
      session_client_address="10.249.${octet}.252"
      # The headless browser's fixed address, likewise (split topology only).
      browser_address="10.249.${octet}.251"
+      # The installed-module checks' own client address, so their sign-ins have their own bucket.
+      module_client_address="10.249.${octet}.249"
      return 0
    fi
  done
@@ -166,7 +179,7 @@ cleanup() {
  # job itself runs as root), so only the staged copy on the Docker host needs this.
  if [[ "$daemon_work" == "$work" ]]; then
    docker run --rm --entrypoint rm -v "$work:/work" "$postgres_image" -rf /work/tls /work/env \
-      >/dev/null 2>&1 || true
+      /work/modules >/dev/null 2>&1 || true
  else
    docker run --rm --entrypoint rm -v /tmp:/host-tmp "$postgres_image" -rf "/host-tmp/${project}" \
      >/dev/null 2>&1 || true
@@ -188,41 +201,38 @@ fail() {
  exit 1
 }

-if [[ -z "$image" && "$topology" == same-origin ]]; then
-  step "Building the platform-only image"
+if [[ -z "$image" ]]; then
+  # The one generic platform image, for both topologies. Nothing about either topology, and no
+  # module, is built into it.
+  step "Building the generic platform image"
  built_image="ekklesia-deploy-smoke:${run_id}"
  DOCKER_BUILDKIT=1 docker build --quiet --file deploy/Dockerfile --tag "$built_image" \
    --label "${EKKLESIA_CI_LABEL}=${project}" --label "${EKKLESIA_CI_CREATED_LABEL}=$(date +%s)" \
    . >/dev/null
  image="$built_image"
-elif [[ -z "$image" ]]; then
-  # TEST-ONLY: the repository's module template, installed as the module `example`, in a copy of the
-  # working tree. Its lockfile importer moves with it; the depth is the same, so every relative link
-  # in that importer stays correct and the install stays frozen.
-  step "Building an image with the module template installed as a test-only module"
-  command -v git >/dev/null || fail "git is required to build the split topology's test image."
-  context="$work/build-context"
-  mkdir -p "$context"
-  git ls-files -z --cached --others --exclude-standard |
-    tar --null --files-from=- --create --file=- |
-    tar -C "$context" -xf -
-  mkdir -p "$context/modules"
-  mv "$context/templates/module" "$context/modules/example"
-  [[ "$(grep -c '^  templates/module:$' "$context/pnpm-lock.yaml")" == 1 ]] ||
-    fail "the lockfile has no single templates/module importer to move."
-  sed -i 's|^  templates/module:$|  modules/example:|' "$context/pnpm-lock.yaml"
-  built_image="ekklesia-deploy-smoke:${run_id}"
-  DOCKER_BUILDKIT=1 docker build --quiet --file "$context/deploy/Dockerfile" --tag "$built_image" \
-    --label "${EKKLESIA_CI_LABEL}=${project}" --label "${EKKLESIA_CI_CREATED_LABEL}=$(date +%s)" \
-    "$context" >/dev/null || fail "the test-module image did not build."
-  rm -rf "$context"
-  image="$built_image"
+fi
+
+if [[ "$topology" == split ]]; then
+  # TEST-ONLY: the module template as one self-contained installed-module directory, built outside
+  # the platform image and installed into module storage below. Never a second platform image.
+  step "Packaging the module template as a self-contained test module"
+  DOCKER_BUILDKIT=1 docker build --quiet --file scripts/lib/test-module-package.Dockerfile \
+    --label "${EKKLESIA_CI_LABEL}=${project}" \
+    --output "type=local,dest=${work}/module-package" . >/dev/null ||
+    fail "the test module package did not build."
+  [[ -f "$work/module-package/example/ekklesia.module.json" ]] ||
+    fail "the test module package has no manifest."
 fi

 # The compose project directory: its relative bind mount (postgres/pg_hba.conf) resolves here.
 cp -R deploy "$work/deploy"
 stage deploy

+# Installation-owned module storage (ADR 0048), empty, as an operator creates it before the first
+# `up`: Compose refuses to start `api` or `migrate` without it, and never creates it.
+mkdir -m 755 "$work/modules"
+stage modules
+
 step "Creating the installation-private CA and server certificates"
 init_status=0
 sh deploy/scripts/init-tls.sh "$work/tls" >/dev/null 2>&1 || init_status=$?
@@ -309,6 +319,7 @@ EKKLESIA_EDGE_SUBNET=${edge_subnet}
 EKKLESIA_EDGE_DYNAMIC_RANGE=${edge_range}
 EKKLESIA_PROXY_ADDRESS=${proxy_address}
 EKKLESIA_TUNNEL_ADDRESS=${tunnel_address}
+EKKLESIA_MODULES_HOST_DIR=${daemon_work}/modules
 EOF

 step "Starting PostgreSQL and object storage"
@@ -398,7 +409,70 @@ client session --ip "$session_client_address" \
  -e SMOKE_OWNER_EMAIL="$owner_email" -e SMOKE_OWNER_PASSWORD="$owner_password" ||
  fail "session checks failed."

+psql_as_superuser() {
+  "${compose[@]}" exec -T postgres psql -U ekklesia -d ekklesia -Atc "$1"
+}
+
+# module_storage <script>: runs a root shell over this run's module storage, as an operator changing
+# installed modules on the host would. The API never writes it; it is mounted read-only.
+module_storage() {
+  docker run --rm -i --network none --entrypoint sh -v "$daemon_work/modules:/modules" \
+    "$postgres_image" -ec "$1"
+}
+
+# install_test_module: stages the package under a hidden name, which discovery ignores, then renames
+# it into place in one step, owned by root and readable by the API's unprivileged user.
+install_test_module() {
+  tar -C "$work/module-package" -cf - example |
+    module_storage 'rm -rf /modules/.incoming && mkdir /modules/.incoming &&
+      tar -xf - -C /modules/.incoming && chown -R 0:0 /modules/.incoming/example &&
+      chmod -R a+rX,go-w /modules/.incoming/example &&
+      mv /modules/.incoming/example /modules/example && rmdir /modules/.incoming' ||
+    fail "the test module could not be installed into module storage."
+}
+
+# restart_api <command...>: stops the API, runs the command, and starts the API again. A controlled
+# restart is the only way a module change is picked up; nothing is ever hot loaded.
+restart_api() {
+  "${compose[@]}" stop api >/dev/null 2>&1 || fail "the API did not stop."
+  "$@"
+  if ! up_output="$("${compose[@]}" up --detach --wait --wait-timeout 180 api 2>&1)"; then
+    compose_report_not_ready "$up_output" "${compose[@]}"
+    fail "the API did not become healthy after a restart."
+  fi
+}
+
+# module_check <expectation>: "absent", or the exact JSON array of values `example` must return.
+module_check() {
+  client module --ip "$module_client_address" -e SMOKE_MODULE_EXPECT="$1" \
+    -e SMOKE_OWNER_EMAIL="$owner_email" -e SMOKE_OWNER_PASSWORD="$owner_password"
+}
+
+# plan_output: the read-only migration plan, as `docker compose run --rm migrate` prints it.
+plan_output() {
+  "${compose[@]}" run --rm --no-deps migrate 2>&1 || fail "migration planning failed."
+}
+
 if [[ "$topology" == split ]]; then
+  step "Booting the generic image with empty module storage"
+  module_check absent || fail "the image without modules did not run module-free."
+
+  step "Installing the test module into module storage, migrating explicitly, and restarting the API"
+  install_module_and_migrate() {
+    install_test_module
+    local plan
+    plan="$(plan_output)"
+    grep -q '^- \[pending\] example:0000_initial ' <<<"$plan" ||
+      fail "the plan does not list the installed module's migration as pending."
+    "${compose[@]}" run --rm --no-deps migrate node core/dist/cli/apply-migrations.js >/dev/null ||
+      fail "applying the installed module's migration failed."
+  }
+  restart_api install_module_and_migrate
+  module_check '[]' || fail "the installed module did not run from module storage."
+  psql_as_superuser "INSERT INTO example_records (id, value) VALUES (gen_random_uuid(), 'kept')" \
+    >/dev/null || fail "could not write the module's own data."
+  module_check '["Hello, kept"]' || fail "the installed module did not read its own data."
+
  step "Driving a real browser through the split Dashboard, a module, its display, and sign-out"
  # Chromium trusts this run's proxy through the public-key hash of Caddy's internal intermediate CA,
  # which signs every certificate the test proxy serves: no CA is installed anywhere, and nothing but
@@ -501,6 +575,31 @@ if [[ "$topology" == split ]]; then
      while read -r ts uri; do echo "     $(date -u -d "@${ts}" +%T) ${uri}"; done || true
    ((display_status == 0)) || fail "display-device checks failed."
  fi
+
+  step "Removing the module from module storage and restarting: Core runs, its data stays"
+  restart_api module_storage 'rm -rf /modules/example'
+  module_check absent || fail "the removed module is still running."
+  plan="$(plan_output)"
+  grep -q '^- \[retained\] example: 1 applied migration kept; the module is not installed$' \
+    <<<"$plan" || fail "migration planning did not report the removed module's retained history."
+  grep -q '^Migration plan (.*0 pending, 0 approval-blocked, 1 retained):$' <<<"$plan" ||
+    fail "migration planning did not succeed cleanly with the module absent."
+  "${compose[@]}" run --rm --no-deps migrate node core/dist/cli/apply-migrations.js >/dev/null ||
+    fail "applying migrations failed with the module absent."
+  [[ "$(psql_as_superuser "SELECT value FROM example_records")" == "kept" ]] ||
+    fail "the removed module's data did not survive."
+  echo "ok   the module is gone, Core runs, its history is retained, and its data survives"
+
+  step "Restoring the module and restarting: it returns with its previous data"
+  restart_api install_test_module
+  plan="$(plan_output)"
+  grep -q '^- \[applied\] example:0000_initial ' <<<"$plan" ||
+    fail "the restored module's migration history is not valid."
+  if grep -q '\[retained\]\|\[pending\]' <<<"$plan"; then
+    fail "the restored module left retained or pending migrations."
+  fi
+  module_check '["Hello, kept"]' || fail "the restored module did not return with its data."
+  echo "ok   the restored module runs again, with its history strict and its data intact"
 fi

 step "Proving the API's database connection is certificate-verified TLS"
@@ -510,9 +609,6 @@ api_data_address="$(
    "$api_container"
 )"
 [[ -n "$api_data_address" ]] || fail "could not find the API's database-network address."
-psql_as_superuser() {
-  "${compose[@]}" exec -T postgres psql -U ekklesia -d ekklesia -Atc "$1"
-}
 tls_connections="$(psql_as_superuser "SELECT count(*) FROM pg_stat_ssl s JOIN pg_stat_activity a USING (pid) WHERE a.client_addr = '${api_data_address}' AND s.ssl AND s.version = 'TLSv1.3'")"
 plain_connections="$(psql_as_superuser "SELECT count(*) FROM pg_stat_ssl s JOIN pg_stat_activity a USING (pid) WHERE a.client_addr = '${api_data_address}' AND NOT s.ssl")"
 ((tls_connections > 0)) || fail "the API holds no TLS 1.3 database connection."