Complete Phase 5B offline display resilience

Make the /display kiosk survive losing its infrastructure, then surrender
authority cleanly when it returns. The Phase 5A renderer is unchanged: this
builds on its two permanent scene layers, explicit Cache Storage, IndexedDB,
Service Worker, timing anchors, and structured errors.

While connected the server is authoritative. While disconnected the display
becomes a deterministic local continuation of the last authoritative state
it persisted, and the moment the server speaks again the local cursor is
abandoned rather than reconciled or reported back.

A small client evaluator resolves offline playback from the persisted
manifest and clock anchor. Catch-up is arithmetic rather than iterative:
whole rotations are skipped between eligibility boundaries, so a month
offline resolves in milliseconds. Mixed durations, wraparound, [start, end)
schedules, and expirations are evaluated locally, and a null expiration
stays permanent. Mode off keeps the hidden cursor advancing, pause holds
its slide and remainder, static holds until the cached metadata says the
asset genuinely expired, and an activation queued for an unknown playlist
holds the last good frame instead of continuing into a playlist that is
being replaced. Contract tests run this evaluator and the real Phase 3B
engine over identical scenarios and assert the same slide, start, and
deadline, so the client tracks the server without refactoring it.

Cold start restores signage from the Service Worker shell, IndexedDB, and
Cache Storage with the entire stack down, keeping the same persistent
display identity.

Validated display error codes now become Home Assistant incidents through
the Phase 4 machinery, keyed per display and condition so a persisting
error reports once and recovers once. Arbitrary browser text stays
telemetry.

Corrects the deployment assumption: the kiosk runs on a separate machine,
so production requires a trusted HTTPS origin and localhost is development
only. Verifying on a real trusted certificate exposed two production bugs
that localhost had hidden — behind a reverse proxy an outage arrives as a
502 rather than a refused connection, and an unbounded shell fetch left a
restarting kiosk blank for the proxy's whole connect timeout. The worker
now treats a gateway error as an outage and bounds that wait.

Verified with 200/200 backend, 20 client unit, and 30 Playwright tests on
a trusted HTTPS origin with no browser security flags weakened.

Co-Authored-By: Claude Opus 5 <[email protected]>

server/src/displays/service.js

@@ -66,6 +66,32 @@ function parseDisplayError(value) {
   return value;
 }
 
+// Only validated error codes become notifications. Arbitrary browser text
+// stays telemetry and never reaches Home Assistant.
+const DISPLAY_ERROR_SEVERITY = Object.freeze({
+  slide_decode_failed: "warning",
+  asset_fetch_failed: "warning",
+  // A display that cannot write its cache still shows slides now, but has lost
+  // its ability to survive the next outage.
+  cache_write_failed: "warning",
+  // Nothing renderable for an active playlist is an on-air problem.
+  manifest_invalid: "critical",
+  offline_capability_unavailable: "warning",
+});
+
+const DISPLAY_ERROR_MESSAGES = Object.freeze({
+  slide_decode_failed: "could not decode a slide",
+  asset_fetch_failed: "could not fetch a slide",
+  cache_write_failed: "could not store slides for offline use",
+  manifest_invalid: "cannot render any slide of its active playlist",
+  offline_capability_unavailable: "cannot cache slides for offline use",
+});
+
+function errorConditionKey(clientId, error) {
+  const scope = error.assetId ? `:${error.assetId}` : "";
+  return `display:${clientId}:${error.kind}${scope}`;
+}
+
 function validateHeartbeat(payload) {
   if (payload === null || typeof payload !== "object" || Array.isArray(payload)) {
     throw new TypeError("heartbeat must be an object");
@@ -221,6 +247,47 @@ function createDisplayService({
     return result.recovered;
   }
 
+  // Reconciles the one display-error condition this client currently reports
+  // against whatever it had open before. Phase 4 deduplication stays
+  // authoritative: a persisting error is never re-notified, and a resolved one
+  // recovers exactly once.
+  function reconcileErrorIncidents(clientId, row, at) {
+    if (!notifications) return;
+    const parsed = parseDisplayError(row.last_error);
+    const structured =
+      parsed && typeof parsed === "object" && DISPLAY_ERROR_SEVERITY[parsed.kind]
+        ? parsed
+        : null;
+    const desired = structured ? errorConditionKey(clientId, structured) : null;
+    const offlineKey = conditionKey(clientId);
+
+    for (const key of notifications.openConditionKeys(`display:${clientId}:`)) {
+      // The display-offline condition is a separate concern and is never
+      // touched here.
+      if (key === offlineKey || key === desired) continue;
+      notifications.observeRecovery(key, {
+        event: "display.error_cleared",
+        message: `Display ${clientId} no longer reports that error.`,
+        context: { clientId, recoveredAt: new Date(at).toISOString() },
+      });
+    }
+
+    if (!structured) return;
+    notifications.observeFailure(desired, {
+      event: `display.${structured.kind}`,
+      severity: DISPLAY_ERROR_SEVERITY[structured.kind],
+      message: `Display ${clientId} ${DISPLAY_ERROR_MESSAGES[structured.kind]}.`,
+      context: {
+        clientId,
+        outputId: row.output_id,
+        kind: structured.kind,
+        assetId: structured.assetId ?? null,
+        entryId: structured.entryId ?? null,
+        detail: structured.message ?? null,
+      },
+    });
+  }
+
   function heartbeat(payload) {
     const telemetry = validateHeartbeat(payload);
     const at = now();
@@ -237,6 +304,7 @@ function createDisplayService({
     const row = findClient.get(telemetry.clientId);
     const wasOffline = existing ? isOffline(existing, at) : false;
     const recovered = recoverOffline(row, at);
+    reconcileErrorIncidents(telemetry.clientId, row, at);
     const errorChanged = (existing?.last_error ?? null) !== telemetry.lastError;
 
     if (!existing) {
@@ -339,7 +407,9 @@ module.exports = {
   DEFAULT_OFFLINE_THRESHOLD_MS,
   HEARTBEAT_PROTOCOL_VERSION,
   MAX_ERROR_LENGTH,
+  DISPLAY_ERROR_SEVERITY,
   createDisplayService,
+  errorConditionKey,
   parseDisplayError,
   validateHeartbeat,
 };

server/src/notifications/index.js

@@ -232,6 +232,15 @@ function createNotificationService({
     return Boolean(findOpen.get(conditionKey));
   }
 
+  // Lets a caller find the conditions it currently has open, so a changed
+  // condition can be recovered without inventing a second bookkeeping store.
+  function openConditionKeys(prefix) {
+    return listOpenIncidents
+      .all()
+      .map((incident) => incident.condition_key)
+      .filter((key) => (prefix ? key.startsWith(prefix) : true));
+  }
+
   async function deliverOne(row) {
     const at = now();
     const headers = { "content-type": "application/json" };
@@ -341,6 +350,7 @@ function createNotificationService({
     deliverDue,
     isOpen,
     observeFailure,
+    openConditionKeys,
     observeRecovery,
     start,
     status,

server/test/displays.test.js

@@ -3,7 +3,9 @@ const assert = require("node:assert/strict");
 const { openDatabase } = require("../src/database");
 const { runMigrations } = require("../src/database/migrate");
 const {
+  DISPLAY_ERROR_SEVERITY,
   createDisplayService,
+  errorConditionKey,
   parseDisplayError,
 } = require("../src/displays/service");
 const { createNotificationService } = require("../src/notifications");
@@ -422,3 +424,163 @@ test("a structured display error round-trips without loosening validation", (t)
   assert.equal(parseDisplayError("{not json"), "{not json");
   assert.equal(parseDisplayError('{"no":"kind"}'), '{"no":"kind"}');
 });
+
+function structuredError(kind, overrides = {}) {
+  return JSON.stringify({
+    kind,
+    assetId: null,
+    entryId: null,
+    message: `${kind} detail`,
+    at: BASE_TIME,
+    ...overrides,
+  });
+}
+
+test("a structured display error opens exactly one incident and never repeats", (t) => {
+  const harness = createHarness();
+  t.after(() => harness.database.close());
+
+  for (let beat = 0; beat < 6; beat += 1) {
+    harness.clock.set(BASE_TIME + beat * 12_000);
+    harness.service.heartbeat(
+      heartbeatPayload({
+        mostRecentError: structuredError("slide_decode_failed", { assetId: "asset-a" }),
+      }),
+    );
+  }
+
+  const incidents = harness.incidents();
+  assert.equal(incidents.length, 1, "a persisting error is reported once, not per heartbeat");
+  assert.equal(incidents[0].event, "display.slide_decode_failed");
+  assert.equal(incidents[0].severity, "warning");
+  assert.equal(
+    incidents[0].condition_key,
+    errorConditionKey("lobby-display", { kind: "slide_decode_failed", assetId: "asset-a" }),
+  );
+  assert.equal(JSON.parse(incidents[0].context_json).assetId, "asset-a");
+  assert.equal(harness.outbox().length, 1);
+});
+
+test("a resolved display error recovers exactly once", (t) => {
+  const harness = createHarness();
+  t.after(() => harness.database.close());
+
+  harness.service.heartbeat(
+    heartbeatPayload({
+      mostRecentError: structuredError("asset_fetch_failed", { assetId: "asset-a" }),
+    }),
+  );
+  assert.equal(harness.outbox().length, 1);
+
+  for (let beat = 1; beat <= 3; beat += 1) {
+    harness.clock.set(BASE_TIME + beat * 12_000);
+    harness.service.heartbeat(heartbeatPayload({ mostRecentError: null }));
+  }
+
+  const incidents = harness.incidents();
+  assert.equal(incidents.length, 1);
+  assert.equal(incidents[0].state, "recovered");
+  assert.equal(harness.outbox().length, 2, "one failure and one recovery");
+  assert.equal(harness.outbox()[1].transition, "recovery");
+});
+
+test("an error moving to another asset recovers the old condition and opens the new one", (t) => {
+  const harness = createHarness();
+  t.after(() => harness.database.close());
+
+  harness.service.heartbeat(
+    heartbeatPayload({
+      mostRecentError: structuredError("asset_fetch_failed", { assetId: "asset-a" }),
+    }),
+  );
+  harness.clock.set(BASE_TIME + 12_000);
+  harness.service.heartbeat(
+    heartbeatPayload({
+      mostRecentError: structuredError("asset_fetch_failed", { assetId: "asset-b" }),
+    }),
+  );
+
+  const byKey = Object.fromEntries(
+    harness.incidents().map((incident) => [incident.condition_key, incident]),
+  );
+  const keyA = errorConditionKey("lobby-display", {
+    kind: "asset_fetch_failed",
+    assetId: "asset-a",
+  });
+  const keyB = errorConditionKey("lobby-display", {
+    kind: "asset_fetch_failed",
+    assetId: "asset-b",
+  });
+  assert.equal(byKey[keyA].state, "recovered", "the resolved asset condition closes");
+  assert.equal(byKey[keyB].state, "open", "the new asset condition opens");
+  assert.equal(harness.outbox().length, 3);
+});
+
+test("only validated error codes become incidents", (t) => {
+  const harness = createHarness();
+  t.after(() => harness.database.close());
+
+  // Arbitrary browser text is telemetry, never a Home Assistant notification.
+  harness.service.heartbeat(
+    heartbeatPayload({ mostRecentError: "TypeError: undefined is not a function" }),
+  );
+  assert.equal(harness.incidents().length, 0);
+  assert.equal(
+    harness.service.list()[0].mostRecentError,
+    "TypeError: undefined is not a function",
+    "the raw telemetry is still visible to operators",
+  );
+
+  // A structured payload with an unrecognised code is equally inert.
+  harness.clock.set(BASE_TIME + 12_000);
+  harness.service.heartbeat(
+    heartbeatPayload({ mostRecentError: structuredError("something_invented") }),
+  );
+  assert.equal(harness.incidents().length, 0);
+  assert.equal(harness.outbox().length, 0);
+  assert.equal(harness.service.list()[0].mostRecentError.kind, "something_invented");
+});
+
+test("a manifest that cannot render locally is critical", (t) => {
+  const harness = createHarness();
+  t.after(() => harness.database.close());
+
+  harness.service.heartbeat(
+    heartbeatPayload({ mostRecentError: structuredError("manifest_invalid") }),
+  );
+
+  assert.equal(harness.incidents()[0].severity, "critical");
+  assert.equal(DISPLAY_ERROR_SEVERITY.manifest_invalid, "critical");
+  assert.equal(DISPLAY_ERROR_SEVERITY.offline_capability_unavailable, "warning");
+});
+
+test("display error incidents never disturb the display offline condition", (t) => {
+  const harness = createHarness();
+  t.after(() => harness.database.close());
+
+  harness.service.heartbeat(heartbeatPayload());
+  // The display goes offline, then returns while reporting a slide problem.
+  harness.clock.runTo(BASE_TIME + THRESHOLD);
+  assert.equal(harness.incidents().length, 1);
+  assert.equal(harness.incidents()[0].event, "display.offline");
+
+  harness.clock.set(BASE_TIME + 60_000);
+  harness.service.heartbeat(
+    heartbeatPayload({
+      mostRecentError: structuredError("slide_decode_failed", { assetId: "asset-a" }),
+    }),
+  );
+
+  const offline = harness
+    .incidents()
+    .find((incident) => incident.condition_key === "display:lobby-display");
+  assert.equal(offline.state, "recovered", "returning still recovers the offline condition once");
+  const decode = harness
+    .incidents()
+    .find((incident) => incident.event === "display.slide_decode_failed");
+  assert.equal(decode.state, "open");
+  assert.equal(
+    harness.incidents().filter((incident) => incident.event === "display.offline").length,
+    1,
+  );
+});

server/test/offline-contract.test.js

@@ -0,0 +1,288 @@
+const test = require("node:test");
+const assert = require("node:assert/strict");
+const path = require("node:path");
+const { pathToFileURL } = require("node:url");
+const { openDatabase } = require("../src/database");
+const { runMigrations } = require("../src/database/migrate");
+const { createPlaybackService } = require("../src/playback/service");
+
+// Client/server contract.
+//
+// The disconnected display necessarily reimplements part of the playback
+// timeline. Rather than trusting that the two agree, every scenario below runs
+// the real Phase 3B engine and the browser evaluator over identical data and
+// asserts they land on the same slide, start, and deadline. Phase 3B is not
+// refactored to share code; it stays the verified authority.
+const CLIENT_MODULE = pathToFileURL(
+  path.join(__dirname, "..", "..", "client", "src", "display", "offlineTimeline.js"),
+).href;
+
+const BASE_TIME = 2_000_000_000_000;
+
+function createHarness({ defaultDurationMs = 1_000 } = {}) {
+  const database = openDatabase({ path: ":memory:" });
+  runMigrations(database);
+  database
+    .prepare("UPDATE app_settings SET default_duration_ms = ?, updated_at = ? WHERE id = 1")
+    .run(defaultDurationMs, BASE_TIME);
+
+  let current = BASE_TIME;
+  const clock = {
+    now: () => current,
+    set(value) {
+      current = value;
+    },
+  };
+  let assetSequence = 0;
+  let service = null;
+
+  function seedAsset({ expirationAt = null } = {}) {
+    const ordinal = ++assetSequence;
+    const id = `asset-${ordinal}`;
+    const sha = ordinal.toString(16).padStart(64, "0");
+    database
+      .prepare(
+        `INSERT INTO assets (
+          id, source_sha256, display_name, original_filename, status, expiration_at,
+          display_object_key, display_sha256, display_content_type, display_byte_size,
+          display_width, display_height, created_at, updated_at
+        ) VALUES (?, ?, ?, ?, 'ready', ?, ?, ?, 'image/webp', 10, 1920, 1080, ?, ?)`,
+      )
+      .run(id, sha, id, `${id}.png`, expirationAt, `private/${id}.webp`, sha, BASE_TIME, BASE_TIME);
+    return id;
+  }
+
+  function seedPlaylist(entries) {
+    database
+      .prepare(
+        "INSERT INTO playlists (id, name, revision, created_at, updated_at) VALUES ('p', 'P', 1, ?, ?)",
+      )
+      .run(BASE_TIME, BASE_TIME);
+    entries.forEach((options, position) => {
+      const assetId = seedAsset({ expirationAt: options.assetExpirationAt ?? null });
+      database
+        .prepare(
+          `INSERT INTO playlist_entries (
+            id, playlist_id, asset_id, enabled, duration_override_ms,
+            schedule_start_at, schedule_end_at, position, created_at, updated_at
+          ) VALUES (?, 'p', ?, 1, ?, ?, ?, ?, ?, ?)`,
+        )
+        .run(
+          options.id,
+          assetId,
+          options.durationMs ?? null,
+          options.scheduleStartAt ?? null,
+          options.scheduleEndAt ?? null,
+          position,
+          BASE_TIME,
+          BASE_TIME,
+        );
+    });
+    return "p";
+  }
+
+  function startService() {
+    service = createPlaybackService({
+      database,
+      now: clock.now,
+      setTimeoutFn: () => null,
+      clearTimeoutFn: () => {},
+      logger: { warn() {} },
+    });
+    service.initialize();
+    return service;
+  }
+
+  function playback() {
+    return database.prepare("SELECT * FROM playback_state WHERE output_id = 'global'").get();
+  }
+
+  return {
+    clock,
+    close: () => database.close(),
+    database,
+    playback,
+    seedPlaylist,
+    startService,
+    get service() {
+      return service;
+    },
+  };
+}
+
+// Runs one disconnected scenario through both implementations.
+async function compare(t, { entries, defaultDurationMs, offlineAt, label }) {
+  const { resolveOfflineState } = await import(CLIENT_MODULE);
+  const harness = createHarness({ defaultDurationMs });
+  t.after(() => harness.close());
+  harness.seedPlaylist(entries);
+
+  const service = harness.startService();
+  service.activatePlaylist("p");
+
+  // What the display would have persisted as its last authoritative state.
+  const snapshot = service.getSnapshot();
+  const anchor = {
+    entryId: snapshot.currentEntryId,
+    startedAt: snapshot.timeline.slideStartedAt,
+    deadlineAt: snapshot.timeline.nextSlideAt,
+    transportState: snapshot.transportState,
+    pausedRemainingMs: snapshot.timeline.pausedRemainingMs,
+    staticAssetId: snapshot.staticTarget?.assetId ?? null,
+    staticTarget: snapshot.staticTarget,
+    pendingPlaylistId: snapshot.pendingPlaylistId ?? null,
+  };
+  const manifest = snapshot.manifest;
+
+  // The server catches up the same outage on restart.
+  harness.clock.set(offlineAt);
+  harness.startService();
+  const server = harness.playback();
+
+  const client = resolveOfflineState({ manifest, anchor, at: offlineAt });
+
+  if (server.current_entry_id === null) {
+    assert.equal(client.kind, "black", `${label}: both must report no playable content`);
+    return { client, server };
+  }
+
+  assert.equal(client.kind, "normal", `${label}: client must resolve a normal slide`);
+  assert.equal(client.entryId, server.current_entry_id, `${label}: same slide`);
+  assert.equal(client.startedAt, server.slide_started_at, `${label}: same start instant`);
+  assert.equal(client.deadlineAt, server.next_slide_at, `${label}: same deadline`);
+  return { client, server };
+}
+
+test("offline catch-up matches the server across mixed durations and wraparound", async (t) => {
+  const entries = [
+    { id: "a", durationMs: 3_000 },
+    { id: "b", durationMs: 9_000 },
+    { id: "c", durationMs: 4_000 },
+  ];
+  // One cycle is 16s; each instant lands in a different part of the rotation.
+  for (const offset of [1_000, 3_000, 5_000, 12_000, 16_000, 17_000, 31_999, 32_000, 100_000]) {
+    await compare(t, {
+      entries,
+      offlineAt: BASE_TIME + offset,
+      label: `mixed durations +${offset}ms`,
+    });
+  }
+});
+
+test("offline catch-up matches the server over hours and days offline", async (t) => {
+  const entries = [
+    { id: "a", durationMs: 3_000 },
+    { id: "b", durationMs: 9_000 },
+    { id: "c", durationMs: 4_000 },
+  ];
+  const started = process.hrtime.bigint();
+  for (const offset of [
+    3_600_000, // one hour
+    86_400_000, // one day
+    604_800_000, // one week
+    2_592_000_000, // thirty days
+  ]) {
+    await compare(t, {
+      entries,
+      offlineAt: BASE_TIME + offset + 1_234,
+      label: `long outage +${offset}ms`,
+    });
+  }
+  const elapsedMs = Number(process.hrtime.bigint() - started) / 1e6;
+  // Thirty days of 16-second cycles is ~160k slides; arithmetic catch-up must
+  // not step through them one at a time.
+  assert.ok(
+    elapsedMs < 4_000,
+    `catch-up must be arithmetic, not iterative (took ${elapsedMs.toFixed(0)}ms)`,
+  );
+});
+
+test("offline catch-up matches the server across a schedule start", async (t) => {
+  const entries = [
+    { id: "always", durationMs: 1_000 },
+    { id: "joins", durationMs: 1_000, scheduleStartAt: BASE_TIME + 20_000 },
+  ];
+  for (const offset of [19_999, 20_000, 20_001, 25_000, 60_000]) {
+    await compare(t, {
+      entries,
+      offlineAt: BASE_TIME + offset,
+      label: `schedule start +${offset}ms`,
+    });
+  }
+});
+
+test("offline catch-up matches the server across a schedule end", async (t) => {
+  const entries = [
+    { id: "ends", durationMs: 5_000, scheduleEndAt: BASE_TIME + 12_500 },
+    { id: "stays", durationMs: 5_000 },
+  ];
+  for (const offset of [12_499, 12_500, 12_501, 30_000, 120_000]) {
+    await compare(t, {
+      entries,
+      offlineAt: BASE_TIME + offset,
+      label: `schedule end +${offset}ms`,
+    });
+  }
+});
+
+test("offline catch-up matches the server across an asset expiration", async (t) => {
+  const entries = [
+    { id: "expires", durationMs: 6_000, assetExpirationAt: BASE_TIME + 15_000 },
+    { id: "permanent", durationMs: 4_000 },
+  ];
+  for (const offset of [14_999, 15_000, 15_001, 40_000, 300_000]) {
+    await compare(t, {
+      entries,
+      offlineAt: BASE_TIME + offset,
+      label: `expiration +${offset}ms`,
+    });
+  }
+});
+
+test("offline evaluation agrees with the server when nothing is playable", async (t) => {
+  const entries = [{ id: "only", durationMs: 5_000, scheduleEndAt: BASE_TIME + 8_000 }];
+  const { client } = await compare(t, {
+    entries,
+    offlineAt: BASE_TIME + 30_000,
+    label: "no playable content",
+  });
+  assert.equal(client.kind, "black");
+  assert.equal(client.reason, "no_playable_content");
+});
+
+test("a permanent asset never expires locally no matter how long the display is offline", async (t) => {
+  const { resolveOfflineState } = await import(CLIENT_MODULE);
+  const manifest = {
+    entries: [
+      {
+        id: "permanent",
+        position: 0,
+        assetId: "asset-permanent",
+        effectiveDurationMs: 5_000,
+        scheduleStartAt: null,
+        scheduleEndAt: null,
+        // A null expiration is permanent and must stay permanent.
+        expirationAt: null,
+      },
+    ],
+  };
+  const anchor = {
+    entryId: "permanent",
+    startedAt: BASE_TIME,
+    deadlineAt: BASE_TIME + 5_000,
+    transportState: "playing",
+  };
+
+  const afterAYear = resolveOfflineState({
+    manifest,
+    anchor,
+    at: BASE_TIME + 365 * 24 * 3_600_000,
+  });
+  assert.equal(afterAYear.kind, "normal");
+  assert.equal(afterAYear.entryId, "permanent");
+  assert.equal(
+    (afterAYear.deadlineAt - afterAYear.startedAt) % 5_000,
+    0,
+    "the slide keeps its configured duration on the original phase",
+  );
+});