Complete Phase 5B offline display resilience

Make the /display kiosk survive losing its infrastructure, then surrender authority cleanly when it returns. The Phase 5A renderer is unchanged: this builds on its two permanent scene layers, explicit Cache Storage, IndexedDB, Service Worker, timing anchors, and structured errors. While connected the server is authoritative. While disconnected the display becomes a deterministic local continuation of the last authoritative state it persisted, and the moment the server speaks again the local cursor is abandoned rather than reconciled or reported back. A small client evaluator resolves offline playback from the persisted manifest and clock anchor. Catch-up is arithmetic rather than iterative: whole rotations are skipped between eligibility boundaries, so a month offline resolves in milliseconds. Mixed durations, wraparound, [start, end) schedules, and expirations are evaluated locally, and a null expiration stays permanent. Mode off keeps the hidden cursor advancing, pause holds its slide and remainder, static holds until the cached metadata says the asset genuinely expired, and an activation queued for an unknown playlist holds the last good frame instead of continuing into a playlist that is being replaced. Contract tests run this evaluator and the real Phase 3B engine over identical scenarios and assert the same slide, start, and deadline, so the client tracks the server without refactoring it. Cold start restores signage from the Service Worker shell, IndexedDB, and Cache Storage with the entire stack down, keeping the same persistent display identity. Validated display error codes now become Home Assistant incidents through the Phase 4 machinery, keyed per display and condition so a persisting error reports once and recovers once. Arbitrary browser text stays telemetry. Corrects the deployment assumption: the kiosk runs on a separate machine, so production requires a trusted HTTPS origin and localhost is development only. Verifying on a real trusted certificate exposed two production bugs that localhost had hidden — behind a reverse proxy an outage arrives as a 502 rather than a refused connection, and an unbounded shell fetch left a restarting kiosk blank for the proxy's whole connect timeout. The worker now treats a gateway error as an outage and bounds that wait. Verified with 200/200 backend, 20 client unit, and 30 Playwright tests on a trusted HTTPS origin with no browser security flags weakened. Co-Authored-By: Claude Opus 5 <[email protected]>
2026-08-17 20:36:13 -05:00
parent 147e375702
commit ba678d08d5
17 changed files with 2300 additions and 21 deletions
@@ -0,0 +1,363 @@
+import { expect, test } from "@playwright/test";
+import { ensureFixtures } from "./fixtures.js";
+import { activatePlaylist, api, resetPlayback } from "./helpers.js";
+import {
+  BASE_URL,
+  cachedMedia,
+  diagnostics,
+  launchKiosk,
+  opaqueLayerCount,
+  waitForCachedAssets,
+  waitForSlide,
+} from "./offline-helpers.js";
+import { CONTAINERS, start, stop, waitUntilHealthy, waitUntilUnreachable } from "./stack.js";
+
+// These scenarios remove real infrastructure, so they must not interleave.
+test.describe.configure({ mode: "serial" });
+test.use({ baseURL: BASE_URL });
+
+async function restoreStack() {
+  await start(CONTAINERS.storage, CONTAINERS.api, CONTAINERS.web);
+  await waitUntilHealthy(BASE_URL);
+}
+
+test.afterAll(async () => {
+  await restoreStack();
+});
+
+test("cached rotation continues through a complete server outage", async ({ request }) => {
+  const library = await ensureFixtures(request);
+  await resetPlayback(request);
+  await activatePlaylist(request, "Outage", [
+    { assetId: library["wide-a"].id, durationOverrideMs: 3_000 },
+    { assetId: library["wide-b"].id, durationOverrideMs: 9_000 },
+    { assetId: library["wide-c"].id, durationOverrideMs: 4_000 },
+  ]);
+
+  const { context, page } = await launchKiosk();
+  try {
+    await page.goto(`${BASE_URL}/display`);
+    await waitForSlide(page);
+    // Every slide of this playlist must be cached before the outage begins.
+    await waitForCachedAssets(page, [
+      library["wide-a"].id,
+      library["wide-b"].id,
+      library["wide-c"].id,
+    ]);
+
+    await stop(CONTAINERS.api, CONTAINERS.storage);
+    await waitUntilUnreachable(BASE_URL);
+
+    // Sample continuously across multiple full cycles with nothing serving.
+    const seen = new Set();
+    const starts = new Map();
+    const deadline = Date.now() + 40_000;
+    while (Date.now() < deadline) {
+      const state = await page.evaluate(() => ({
+        rendered: window.__display?.diagnostics()?.renderedAssetId ?? null,
+        offline: window.__display?.diagnostics()?.offlineActive ?? false,
+        opaque: [...document.querySelectorAll(".scene")].filter(
+          (layer) => Number(getComputedStyle(layer).opacity) > 0.99,
+        ).length,
+        gate: document.querySelector(".output-gate").dataset.active,
+      }));
+      // The original black-frame bug must remain impossible offline too.
+      expect(state.opaque, "a good frame is always on screen offline").toBeGreaterThanOrEqual(1);
+      expect(state.gate).toBe("false");
+      if (state.rendered) {
+        seen.add(state.rendered);
+        if (!starts.has(state.rendered)) starts.set(state.rendered, Date.now());
+      }
+      await page.waitForTimeout(150);
+    }
+
+    expect(await diagnostics(page).then((d) => d.offlineActive)).toBe(true);
+    expect(seen.size, "mixed-duration rotation kept cycling with no server").toBe(3);
+    expect(await page.evaluate(() => window.__loadCount ?? 0)).toBe(0);
+
+    await restoreStack();
+    // The server is authoritative again the moment it speaks.
+    await expect
+      .poll(async () => (await diagnostics(page))?.offlineActive, { timeout: 60_000 })
+      .toBe(false);
+    const playback = await api(request, "/api/playback");
+    await expect
+      .poll(async () => (await diagnostics(page))?.acceptedRevision, { timeout: 30_000 })
+      .toBeGreaterThanOrEqual(playback.playbackRevision);
+  } finally {
+    await context.close();
+    await restoreStack();
+  }
+});
+
+test("a cold Chromium restart with the whole stack down restores signage", async ({
+  request,
+}) => {
+  const library = await ensureFixtures(request);
+  await resetPlayback(request);
+  await activatePlaylist(request, "Cold start", [
+    { assetId: library["wide-a"].id, durationOverrideMs: 4_000 },
+    { assetId: library["wide-d"].id, durationOverrideMs: 6_000 },
+  ]);
+
+  // A. Warm the persistent profile while everything is healthy.
+  let clientId;
+  let cachedBefore;
+  {
+    const { context, page } = await launchKiosk();
+    await page.goto(`${BASE_URL}/display`);
+    await waitForSlide(page);
+    await expect
+      .poll(async () =>
+        page.evaluate(async () =>
+          Boolean((await navigator.serviceWorker.getRegistration("/"))?.active),
+        ),
+      )
+      .toBe(true);
+    await expect.poll(async () => (await cachedMedia(page)).length).toBeGreaterThanOrEqual(2);
+    clientId = await page.evaluate(() => window.__display.clientId);
+    cachedBefore = await cachedMedia(page);
+    await context.close();
+  }
+
+  // B. Remove the entire origin: no web server, no API, no object storage.
+  await stop(CONTAINERS.web, CONTAINERS.api, CONTAINERS.storage);
+  await waitUntilUnreachable(BASE_URL);
+
+  try {
+    // C. Relaunch the kiosk against the same URL with the same profile.
+    const { context, page } = await launchKiosk();
+    try {
+      const response = await page.goto(`${BASE_URL}/display`, { waitUntil: "domcontentloaded" });
+      expect(
+        response?.status(),
+        "the Service Worker served the app shell with the origin down",
+      ).toBe(200);
+
+      await waitForSlide(page);
+      const offline = await diagnostics(page);
+      expect(offline.offlineActive, "the display took local authority").toBe(true);
+      expect(
+        await page.evaluate(() => window.__display.clientId),
+        "the persistent display identity survived the restart",
+      ).toBe(clientId);
+      expect(cachedBefore).toContain(
+        (await cachedMedia(page)).find((path) => path.includes(library["wide-a"].id)),
+      );
+      expect(await opaqueLayerCount(page)).toBeGreaterThanOrEqual(1);
+
+      // D. It must keep advancing, not merely show one restored frame.
+      const first = offline.renderedAssetId;
+      await expect
+        .poll(async () => (await diagnostics(page))?.renderedAssetId, { timeout: 30_000 })
+        .not.toBe(first);
+      expect(await opaqueLayerCount(page)).toBeGreaterThanOrEqual(1);
+
+      // E. Restore the stack and confirm it surrenders authority cleanly.
+      await restoreStack();
+      await expect
+        .poll(async () => (await diagnostics(page))?.offlineActive, { timeout: 90_000 })
+        .toBe(false);
+      const playback = await api(request, "/api/playback");
+      await expect
+        .poll(async () => (await diagnostics(page))?.renderedEntryId, { timeout: 45_000 })
+        .toBe(playback.currentEntryId);
+
+      const displays = await api(request, "/api/displays");
+      expect(
+        displays.displays.filter((display) => display.clientId === clientId).length,
+        "one physical display stays one record across a cold restart",
+      ).toBe(1);
+    } finally {
+      await context.close();
+    }
+  } finally {
+    await restoreStack();
+  }
+});
+
+test("an object storage outage never disturbs already cached slides", async ({ request }) => {
+  const library = await ensureFixtures(request);
+  await resetPlayback(request);
+  await activatePlaylist(request, "Storage outage", [
+    { assetId: library["wide-a"].id, durationOverrideMs: 3_000 },
+    { assetId: library["wide-b"].id, durationOverrideMs: 3_000 },
+  ]);
+
+  const { context, page } = await launchKiosk();
+  try {
+    await page.goto(`${BASE_URL}/display`);
+    await waitForSlide(page);
+    await waitForCachedAssets(page, [library["wide-a"].id, library["wide-b"].id]);
+
+    // The API stays up; only the bytes become unreachable.
+    await stop(CONTAINERS.storage);
+
+    const seen = new Set();
+    const deadline = Date.now() + 14_000;
+    while (Date.now() < deadline) {
+      const state = await page.evaluate(() => ({
+        rendered: window.__display?.diagnostics()?.renderedAssetId ?? null,
+        opaque: [...document.querySelectorAll(".scene")].filter(
+          (layer) => Number(getComputedStyle(layer).opacity) > 0.99,
+        ).length,
+      }));
+      expect(state.opaque).toBeGreaterThanOrEqual(1);
+      if (state.rendered) seen.add(state.rendered);
+      await page.waitForTimeout(200);
+    }
+    expect(seen.size, "cached slides keep rotating without object storage").toBe(2);
+
+    // A brand new asset cannot be fetched; the last good frame must survive.
+    const held = await diagnostics(page);
+    await activatePlaylist(request, "Uncached", [
+      { assetId: library.standalone.id, durationOverrideMs: 60_000 },
+    ]);
+    await page.waitForTimeout(6_000);
+    expect(await opaqueLayerCount(page)).toBeGreaterThanOrEqual(1);
+    const during = await diagnostics(page);
+    expect(during.renderedAssetId, "the last good frame is retained").toBe(held.renderedAssetId);
+    // Nothing in the newly active playlist can be rendered locally, which is a
+    // more severe condition than one failed asset and is reported as such.
+    expect(during.mostRecentError?.kind).toBe("manifest_invalid");
+
+    // That condition reaches the server as a critical incident, exactly once.
+    const clientId = await page.evaluate(() => window.__display.clientId);
+    await page.evaluate(() => window.__display.heartbeat());
+    await expect
+      .poll(
+        async () => {
+          const displays = await api(request, "/api/displays");
+          return (
+            displays.displays.find((display) => display.clientId === clientId)?.mostRecentError
+              ?.kind ?? null
+          );
+        },
+        { timeout: 30_000 },
+      )
+      .toBe("manifest_invalid");
+
+    // Restoring storage recovers without a page reload.
+    await start(CONTAINERS.storage);
+    await waitUntilHealthy(BASE_URL);
+    await expect
+      .poll(async () => (await diagnostics(page))?.renderedAssetId, { timeout: 90_000 })
+      .toBe(library.standalone.id);
+    expect(await page.evaluate(() => window.__loadCount ?? 0)).toBe(0);
+  } finally {
+    await context.close();
+    await restoreStack();
+  }
+});
+
+test("an API outage keeps local playback running and resynchronizes on return", async ({
+  request,
+}) => {
+  const library = await ensureFixtures(request);
+  await resetPlayback(request);
+  await activatePlaylist(request, "API outage", [
+    { assetId: library["wide-a"].id, durationOverrideMs: 3_000 },
+    { assetId: library["wide-b"].id, durationOverrideMs: 3_000 },
+    { assetId: library["wide-c"].id, durationOverrideMs: 3_000 },
+  ]);
+
+  const { context, page } = await launchKiosk();
+  try {
+    await page.goto(`${BASE_URL}/display`);
+    await waitForSlide(page);
+    await waitForCachedAssets(page, [
+      library["wide-a"].id,
+      library["wide-b"].id,
+      library["wide-c"].id,
+    ]);
+
+    await stop(CONTAINERS.api);
+    await expect
+      .poll(async () => (await diagnostics(page))?.offlineActive, { timeout: 45_000 })
+      .toBe(true);
+
+    const seen = new Set();
+    const deadline = Date.now() + 14_000;
+    while (Date.now() < deadline) {
+      const state = await diagnostics(page);
+      if (state?.renderedAssetId) seen.add(state.renderedAssetId);
+      expect(await opaqueLayerCount(page)).toBeGreaterThanOrEqual(1);
+      await page.waitForTimeout(250);
+    }
+    expect(seen.size).toBeGreaterThanOrEqual(2);
+
+    // While the display was blind, the operator changed authoritative state.
+    await start(CONTAINERS.api);
+    await waitUntilHealthy(BASE_URL);
+    await api(request, "/api/playback/mode", { method: "PUT", data: { mode: "off" } });
+
+    // Server wins immediately on reconnect, including the visual gate.
+    await expect(page.getByTestId("output-gate")).toHaveAttribute("data-active", "true", {
+      timeout: 60_000,
+    });
+    expect((await diagnostics(page)).offlineActive).toBe(false);
+
+    await api(request, "/api/playback/mode", { method: "PUT", data: { mode: "on" } });
+    await expect(page.getByTestId("output-gate")).toHaveAttribute("data-active", "false", {
+      timeout: 30_000,
+    });
+    const playback = await api(request, "/api/playback");
+    await expect
+      .poll(async () => (await diagnostics(page))?.renderedEntryId, { timeout: 30_000 })
+      .toBe(playback.currentEntryId);
+    expect(await page.evaluate(() => window.__loadCount ?? 0)).toBe(0);
+  } finally {
+    await context.close();
+    await restoreStack();
+  }
+});
+
+test("a playlist changed during the outage is adopted cleanly on reconnect", async ({
+  request,
+}) => {
+  const library = await ensureFixtures(request);
+  await resetPlayback(request);
+  await activatePlaylist(request, "Before outage", [
+    { assetId: library["wide-a"].id, durationOverrideMs: 5_000 },
+    { assetId: library["wide-b"].id, durationOverrideMs: 5_000 },
+  ]);
+
+  const { context, page } = await launchKiosk();
+  try {
+    await page.goto(`${BASE_URL}/display`);
+    await waitForSlide(page);
+    const before = await diagnostics(page);
+
+    await stop(CONTAINERS.api);
+    await expect
+      .poll(async () => (await diagnostics(page))?.offlineActive, { timeout: 45_000 })
+      .toBe(true);
+    await page.waitForTimeout(6_000);
+
+    await start(CONTAINERS.api);
+    await waitUntilHealthy(BASE_URL);
+    // A different playlist entirely, activated while the display was offline.
+    await activatePlaylist(request, "After outage", [
+      { assetId: library.square.id, durationOverrideMs: 60_000 },
+    ]);
+
+    await expect
+      .poll(async () => (await diagnostics(page))?.renderedAssetId, { timeout: 90_000 })
+      .toBe(library.square.id);
+    const after = await diagnostics(page);
+    expect(after.offlineActive).toBe(false);
+    expect(after.renderedAssetId).not.toBe(before.renderedAssetId);
+    expect(await opaqueLayerCount(page)).toBeGreaterThanOrEqual(1);
+
+    // The superseded playlist's bytes are pruned once the new one is promoted.
+    await expect
+      .poll(async () => (await cachedMedia(page)).some((path) => path.includes(library.square.id)))
+      .toBe(true);
+    const cached = await cachedMedia(page);
+    expect(cached.length, "repeated playlist changes must not grow the cache forever")
+      .toBeLessThanOrEqual(6);
+  } finally {
+    await context.close();
+    await restoreStack();
+  }
+});