Complete Phase 5B offline display resilience

Make the /display kiosk survive losing its infrastructure, then surrender authority cleanly when it returns. The Phase 5A renderer is unchanged: this builds on its two permanent scene layers, explicit Cache Storage, IndexedDB, Service Worker, timing anchors, and structured errors. While connected the server is authoritative. While disconnected the display becomes a deterministic local continuation of the last authoritative state it persisted, and the moment the server speaks again the local cursor is abandoned rather than reconciled or reported back. A small client evaluator resolves offline playback from the persisted manifest and clock anchor. Catch-up is arithmetic rather than iterative: whole rotations are skipped between eligibility boundaries, so a month offline resolves in milliseconds. Mixed durations, wraparound, [start, end) schedules, and expirations are evaluated locally, and a null expiration stays permanent. Mode off keeps the hidden cursor advancing, pause holds its slide and remainder, static holds until the cached metadata says the asset genuinely expired, and an activation queued for an unknown playlist holds the last good frame instead of continuing into a playlist that is being replaced. Contract tests run this evaluator and the real Phase 3B engine over identical scenarios and assert the same slide, start, and deadline, so the client tracks the server without refactoring it. Cold start restores signage from the Service Worker shell, IndexedDB, and Cache Storage with the entire stack down, keeping the same persistent display identity. Validated display error codes now become Home Assistant incidents through the Phase 4 machinery, keyed per display and condition so a persisting error reports once and recovers once. Arbitrary browser text stays telemetry. Corrects the deployment assumption: the kiosk runs on a separate machine, so production requires a trusted HTTPS origin and localhost is development only. Verifying on a real trusted certificate exposed two production bugs that localhost had hidden — behind a reverse proxy an outage arrives as a 502 rather than a refused connection, and an unbounded shell fetch left a restarting kiosk blank for the proxy's whole connect timeout. The worker now treats a gateway error as an outage and bounds that wait. Verified with 200/200 backend, 20 client unit, and 30 Playwright tests on a trusted HTTPS origin with no browser security flags weakened. Co-Authored-By: Claude Opus 5 <[email protected]>
2026-08-17 20:36:13 -05:00
parent 147e375702
commit ba678d08d5
17 changed files with 2300 additions and 21 deletions
@@ -3,7 +3,9 @@ const assert = require("node:assert/strict");
 const { openDatabase } = require("../src/database");
 const { runMigrations } = require("../src/database/migrate");
 const {
+  DISPLAY_ERROR_SEVERITY,
  createDisplayService,
+  errorConditionKey,
  parseDisplayError,
 } = require("../src/displays/service");
 const { createNotificationService } = require("../src/notifications");
@@ -422,3 +424,163 @@ test("a structured display error round-trips without loosening validation", (t)
  assert.equal(parseDisplayError("{not json"), "{not json");
  assert.equal(parseDisplayError('{"no":"kind"}'), '{"no":"kind"}');
 });
+
+function structuredError(kind, overrides = {}) {
+  return JSON.stringify({
+    kind,
+    assetId: null,
+    entryId: null,
+    message: `${kind} detail`,
+    at: BASE_TIME,
+    ...overrides,
+  });
+}
+
+test("a structured display error opens exactly one incident and never repeats", (t) => {
+  const harness = createHarness();
+  t.after(() => harness.database.close());
+
+  for (let beat = 0; beat < 6; beat += 1) {
+    harness.clock.set(BASE_TIME + beat * 12_000);
+    harness.service.heartbeat(
+      heartbeatPayload({
+        mostRecentError: structuredError("slide_decode_failed", { assetId: "asset-a" }),
+      }),
+    );
+  }
+
+  const incidents = harness.incidents();
+  assert.equal(incidents.length, 1, "a persisting error is reported once, not per heartbeat");
+  assert.equal(incidents[0].event, "display.slide_decode_failed");
+  assert.equal(incidents[0].severity, "warning");
+  assert.equal(
+    incidents[0].condition_key,
+    errorConditionKey("lobby-display", { kind: "slide_decode_failed", assetId: "asset-a" }),
+  );
+  assert.equal(JSON.parse(incidents[0].context_json).assetId, "asset-a");
+  assert.equal(harness.outbox().length, 1);
+});
+
+test("a resolved display error recovers exactly once", (t) => {
+  const harness = createHarness();
+  t.after(() => harness.database.close());
+
+  harness.service.heartbeat(
+    heartbeatPayload({
+      mostRecentError: structuredError("asset_fetch_failed", { assetId: "asset-a" }),
+    }),
+  );
+  assert.equal(harness.outbox().length, 1);
+
+  for (let beat = 1; beat <= 3; beat += 1) {
+    harness.clock.set(BASE_TIME + beat * 12_000);
+    harness.service.heartbeat(heartbeatPayload({ mostRecentError: null }));
+  }
+
+  const incidents = harness.incidents();
+  assert.equal(incidents.length, 1);
+  assert.equal(incidents[0].state, "recovered");
+  assert.equal(harness.outbox().length, 2, "one failure and one recovery");
+  assert.equal(harness.outbox()[1].transition, "recovery");
+});
+
+test("an error moving to another asset recovers the old condition and opens the new one", (t) => {
+  const harness = createHarness();
+  t.after(() => harness.database.close());
+
+  harness.service.heartbeat(
+    heartbeatPayload({
+      mostRecentError: structuredError("asset_fetch_failed", { assetId: "asset-a" }),
+    }),
+  );
+  harness.clock.set(BASE_TIME + 12_000);
+  harness.service.heartbeat(
+    heartbeatPayload({
+      mostRecentError: structuredError("asset_fetch_failed", { assetId: "asset-b" }),
+    }),
+  );
+
+  const byKey = Object.fromEntries(
+    harness.incidents().map((incident) => [incident.condition_key, incident]),
+  );
+  const keyA = errorConditionKey("lobby-display", {
+    kind: "asset_fetch_failed",
+    assetId: "asset-a",
+  });
+  const keyB = errorConditionKey("lobby-display", {
+    kind: "asset_fetch_failed",
+    assetId: "asset-b",
+  });
+  assert.equal(byKey[keyA].state, "recovered", "the resolved asset condition closes");
+  assert.equal(byKey[keyB].state, "open", "the new asset condition opens");
+  assert.equal(harness.outbox().length, 3);
+});
+
+test("only validated error codes become incidents", (t) => {
+  const harness = createHarness();
+  t.after(() => harness.database.close());
+
+  // Arbitrary browser text is telemetry, never a Home Assistant notification.
+  harness.service.heartbeat(
+    heartbeatPayload({ mostRecentError: "TypeError: undefined is not a function" }),
+  );
+  assert.equal(harness.incidents().length, 0);
+  assert.equal(
+    harness.service.list()[0].mostRecentError,
+    "TypeError: undefined is not a function",
+    "the raw telemetry is still visible to operators",
+  );
+
+  // A structured payload with an unrecognised code is equally inert.
+  harness.clock.set(BASE_TIME + 12_000);
+  harness.service.heartbeat(
+    heartbeatPayload({ mostRecentError: structuredError("something_invented") }),
+  );
+  assert.equal(harness.incidents().length, 0);
+  assert.equal(harness.outbox().length, 0);
+  assert.equal(harness.service.list()[0].mostRecentError.kind, "something_invented");
+});
+
+test("a manifest that cannot render locally is critical", (t) => {
+  const harness = createHarness();
+  t.after(() => harness.database.close());
+
+  harness.service.heartbeat(
+    heartbeatPayload({ mostRecentError: structuredError("manifest_invalid") }),
+  );
+
+  assert.equal(harness.incidents()[0].severity, "critical");
+  assert.equal(DISPLAY_ERROR_SEVERITY.manifest_invalid, "critical");
+  assert.equal(DISPLAY_ERROR_SEVERITY.offline_capability_unavailable, "warning");
+});
+
+test("display error incidents never disturb the display offline condition", (t) => {
+  const harness = createHarness();
+  t.after(() => harness.database.close());
+
+  harness.service.heartbeat(heartbeatPayload());
+  // The display goes offline, then returns while reporting a slide problem.
+  harness.clock.runTo(BASE_TIME + THRESHOLD);
+  assert.equal(harness.incidents().length, 1);
+  assert.equal(harness.incidents()[0].event, "display.offline");
+
+  harness.clock.set(BASE_TIME + 60_000);
+  harness.service.heartbeat(
+    heartbeatPayload({
+      mostRecentError: structuredError("slide_decode_failed", { assetId: "asset-a" }),
+    }),
+  );
+
+  const offline = harness
+    .incidents()
+    .find((incident) => incident.condition_key === "display:lobby-display");
+  assert.equal(offline.state, "recovered", "returning still recovers the offline condition once");
+  const decode = harness
+    .incidents()
+    .find((incident) => incident.event === "display.slide_decode_failed");
+  assert.equal(decode.state, "open");
+  assert.equal(
+    harness.incidents().filter((incident) => incident.event === "display.offline").length,
+    1,
+  );
+});
@@ -0,0 +1,288 @@
+const test = require("node:test");
+const assert = require("node:assert/strict");
+const path = require("node:path");
+const { pathToFileURL } = require("node:url");
+const { openDatabase } = require("../src/database");
+const { runMigrations } = require("../src/database/migrate");
+const { createPlaybackService } = require("../src/playback/service");
+
+// Client/server contract.
+//
+// The disconnected display necessarily reimplements part of the playback
+// timeline. Rather than trusting that the two agree, every scenario below runs
+// the real Phase 3B engine and the browser evaluator over identical data and
+// asserts they land on the same slide, start, and deadline. Phase 3B is not
+// refactored to share code; it stays the verified authority.
+const CLIENT_MODULE = pathToFileURL(
+  path.join(__dirname, "..", "..", "client", "src", "display", "offlineTimeline.js"),
+).href;
+
+const BASE_TIME = 2_000_000_000_000;
+
+function createHarness({ defaultDurationMs = 1_000 } = {}) {
+  const database = openDatabase({ path: ":memory:" });
+  runMigrations(database);
+  database
+    .prepare("UPDATE app_settings SET default_duration_ms = ?, updated_at = ? WHERE id = 1")
+    .run(defaultDurationMs, BASE_TIME);
+
+  let current = BASE_TIME;
+  const clock = {
+    now: () => current,
+    set(value) {
+      current = value;
+    },
+  };
+  let assetSequence = 0;
+  let service = null;
+
+  function seedAsset({ expirationAt = null } = {}) {
+    const ordinal = ++assetSequence;
+    const id = `asset-${ordinal}`;
+    const sha = ordinal.toString(16).padStart(64, "0");
+    database
+      .prepare(
+        `INSERT INTO assets (
+          id, source_sha256, display_name, original_filename, status, expiration_at,
+          display_object_key, display_sha256, display_content_type, display_byte_size,
+          display_width, display_height, created_at, updated_at
+        ) VALUES (?, ?, ?, ?, 'ready', ?, ?, ?, 'image/webp', 10, 1920, 1080, ?, ?)`,
+      )
+      .run(id, sha, id, `${id}.png`, expirationAt, `private/${id}.webp`, sha, BASE_TIME, BASE_TIME);
+    return id;
+  }
+
+  function seedPlaylist(entries) {
+    database
+      .prepare(
+        "INSERT INTO playlists (id, name, revision, created_at, updated_at) VALUES ('p', 'P', 1, ?, ?)",
+      )
+      .run(BASE_TIME, BASE_TIME);
+    entries.forEach((options, position) => {
+      const assetId = seedAsset({ expirationAt: options.assetExpirationAt ?? null });
+      database
+        .prepare(
+          `INSERT INTO playlist_entries (
+            id, playlist_id, asset_id, enabled, duration_override_ms,
+            schedule_start_at, schedule_end_at, position, created_at, updated_at
+          ) VALUES (?, 'p', ?, 1, ?, ?, ?, ?, ?, ?)`,
+        )
+        .run(
+          options.id,
+          assetId,
+          options.durationMs ?? null,
+          options.scheduleStartAt ?? null,
+          options.scheduleEndAt ?? null,
+          position,
+          BASE_TIME,
+          BASE_TIME,
+        );
+    });
+    return "p";
+  }
+
+  function startService() {
+    service = createPlaybackService({
+      database,
+      now: clock.now,
+      setTimeoutFn: () => null,
+      clearTimeoutFn: () => {},
+      logger: { warn() {} },
+    });
+    service.initialize();
+    return service;
+  }
+
+  function playback() {
+    return database.prepare("SELECT * FROM playback_state WHERE output_id = 'global'").get();
+  }
+
+  return {
+    clock,
+    close: () => database.close(),
+    database,
+    playback,
+    seedPlaylist,
+    startService,
+    get service() {
+      return service;
+    },
+  };
+}
+
+// Runs one disconnected scenario through both implementations.
+async function compare(t, { entries, defaultDurationMs, offlineAt, label }) {
+  const { resolveOfflineState } = await import(CLIENT_MODULE);
+  const harness = createHarness({ defaultDurationMs });
+  t.after(() => harness.close());
+  harness.seedPlaylist(entries);
+
+  const service = harness.startService();
+  service.activatePlaylist("p");
+
+  // What the display would have persisted as its last authoritative state.
+  const snapshot = service.getSnapshot();
+  const anchor = {
+    entryId: snapshot.currentEntryId,
+    startedAt: snapshot.timeline.slideStartedAt,
+    deadlineAt: snapshot.timeline.nextSlideAt,
+    transportState: snapshot.transportState,
+    pausedRemainingMs: snapshot.timeline.pausedRemainingMs,
+    staticAssetId: snapshot.staticTarget?.assetId ?? null,
+    staticTarget: snapshot.staticTarget,
+    pendingPlaylistId: snapshot.pendingPlaylistId ?? null,
+  };
+  const manifest = snapshot.manifest;
+
+  // The server catches up the same outage on restart.
+  harness.clock.set(offlineAt);
+  harness.startService();
+  const server = harness.playback();
+
+  const client = resolveOfflineState({ manifest, anchor, at: offlineAt });
+
+  if (server.current_entry_id === null) {
+    assert.equal(client.kind, "black", `${label}: both must report no playable content`);
+    return { client, server };
+  }
+
+  assert.equal(client.kind, "normal", `${label}: client must resolve a normal slide`);
+  assert.equal(client.entryId, server.current_entry_id, `${label}: same slide`);
+  assert.equal(client.startedAt, server.slide_started_at, `${label}: same start instant`);
+  assert.equal(client.deadlineAt, server.next_slide_at, `${label}: same deadline`);
+  return { client, server };
+}
+
+test("offline catch-up matches the server across mixed durations and wraparound", async (t) => {
+  const entries = [
+    { id: "a", durationMs: 3_000 },
+    { id: "b", durationMs: 9_000 },
+    { id: "c", durationMs: 4_000 },
+  ];
+  // One cycle is 16s; each instant lands in a different part of the rotation.
+  for (const offset of [1_000, 3_000, 5_000, 12_000, 16_000, 17_000, 31_999, 32_000, 100_000]) {
+    await compare(t, {
+      entries,
+      offlineAt: BASE_TIME + offset,
+      label: `mixed durations +${offset}ms`,
+    });
+  }
+});
+
+test("offline catch-up matches the server over hours and days offline", async (t) => {
+  const entries = [
+    { id: "a", durationMs: 3_000 },
+    { id: "b", durationMs: 9_000 },
+    { id: "c", durationMs: 4_000 },
+  ];
+  const started = process.hrtime.bigint();
+  for (const offset of [
+    3_600_000, // one hour
+    86_400_000, // one day
+    604_800_000, // one week
+    2_592_000_000, // thirty days
+  ]) {
+    await compare(t, {
+      entries,
+      offlineAt: BASE_TIME + offset + 1_234,
+      label: `long outage +${offset}ms`,
+    });
+  }
+  const elapsedMs = Number(process.hrtime.bigint() - started) / 1e6;
+  // Thirty days of 16-second cycles is ~160k slides; arithmetic catch-up must
+  // not step through them one at a time.
+  assert.ok(
+    elapsedMs < 4_000,
+    `catch-up must be arithmetic, not iterative (took ${elapsedMs.toFixed(0)}ms)`,
+  );
+});
+
+test("offline catch-up matches the server across a schedule start", async (t) => {
+  const entries = [
+    { id: "always", durationMs: 1_000 },
+    { id: "joins", durationMs: 1_000, scheduleStartAt: BASE_TIME + 20_000 },
+  ];
+  for (const offset of [19_999, 20_000, 20_001, 25_000, 60_000]) {
+    await compare(t, {
+      entries,
+      offlineAt: BASE_TIME + offset,
+      label: `schedule start +${offset}ms`,
+    });
+  }
+});
+
+test("offline catch-up matches the server across a schedule end", async (t) => {
+  const entries = [
+    { id: "ends", durationMs: 5_000, scheduleEndAt: BASE_TIME + 12_500 },
+    { id: "stays", durationMs: 5_000 },
+  ];
+  for (const offset of [12_499, 12_500, 12_501, 30_000, 120_000]) {
+    await compare(t, {
+      entries,
+      offlineAt: BASE_TIME + offset,
+      label: `schedule end +${offset}ms`,
+    });
+  }
+});
+
+test("offline catch-up matches the server across an asset expiration", async (t) => {
+  const entries = [
+    { id: "expires", durationMs: 6_000, assetExpirationAt: BASE_TIME + 15_000 },
+    { id: "permanent", durationMs: 4_000 },
+  ];
+  for (const offset of [14_999, 15_000, 15_001, 40_000, 300_000]) {
+    await compare(t, {
+      entries,
+      offlineAt: BASE_TIME + offset,
+      label: `expiration +${offset}ms`,
+    });
+  }
+});
+
+test("offline evaluation agrees with the server when nothing is playable", async (t) => {
+  const entries = [{ id: "only", durationMs: 5_000, scheduleEndAt: BASE_TIME + 8_000 }];
+  const { client } = await compare(t, {
+    entries,
+    offlineAt: BASE_TIME + 30_000,
+    label: "no playable content",
+  });
+  assert.equal(client.kind, "black");
+  assert.equal(client.reason, "no_playable_content");
+});
+
+test("a permanent asset never expires locally no matter how long the display is offline", async (t) => {
+  const { resolveOfflineState } = await import(CLIENT_MODULE);
+  const manifest = {
+    entries: [
+      {
+        id: "permanent",
+        position: 0,
+        assetId: "asset-permanent",
+        effectiveDurationMs: 5_000,
+        scheduleStartAt: null,
+        scheduleEndAt: null,
+        // A null expiration is permanent and must stay permanent.
+        expirationAt: null,
+      },
+    ],
+  };
+  const anchor = {
+    entryId: "permanent",
+    startedAt: BASE_TIME,
+    deadlineAt: BASE_TIME + 5_000,
+    transportState: "playing",
+  };
+
+  const afterAYear = resolveOfflineState({
+    manifest,
+    anchor,
+    at: BASE_TIME + 365 * 24 * 3_600_000,
+  });
+  assert.equal(afterAYear.kind, "normal");
+  assert.equal(afterAYear.entryId, "permanent");
+  assert.equal(
+    (afterAYear.deadlineAt - afterAYear.startedAt) % 5_000,
+    0,
+    "the slide keeps its configured duration on the original phase",
+  );
+});