Implement Phase 4 integrations and display telemetry

Add MQTT playback control, durable Home Assistant incident notifications, and display heartbeat telemetry on top of the verified Phase 3B playback authority. No playback logic is duplicated: every MQTT command and every observed condition goes through the existing playback service, and all integration work happens after the authoritative SQLite commit. MQTT (server/src/mqtt) subscribes the exact required command topics under a configurable prefix that defaults to announcements. Durable commands use QoS 1; next and previous stay at QoS 0 because they are not idempotent and carry no command ID. Retained authoritative state is published only on the separate availability, mode/state, and playback/state topics, never onto a command topic, and only after a mutation commits. A registered last will reports an ungraceful disappearance, a clean shutdown retracts it, and a broker outage degrades status while playback, HTTP, and the display protocol continue untouched. Notifications (server/src/notifications) use the Phase 1 incident and outbox tables. A failing condition opens one incident and queues exactly one notification; continued failure only updates its observation time; recovery queues exactly one recovery; a later failure is a new episode. Delivery retries on a documented capped schedule with a stable notification ID, survives restart, and never blocks playback. With no webhook URL configured, incidents are still recorded but no delivery work is queued. Conditions come from storage health transitions, committed playback state, display heartbeat staleness, and newly failed uploads. Displays (server/src/displays) ingest validated, bounded display:heartbeat telemetry keyed by persistent client ID rather than socket ID. Telemetry is read-only and never touches playback or playlist revisions. Offline is decided by heartbeat staleness, not by a closed socket, and a startup grace period keeps an API restart from manufacturing offline alerts. Adds GET /api/displays and meaningful-change display:status events. Health and status now report redacted storage, MQTT, notification, playback, and display state. External integrations degrade status and never affect readiness, so no integration outage can cause a restart loop. No MQTT password, S3 credential, webhook URL, or webhook secret is exposed. Uses the existing Phase 1 schema unchanged; no migration was required. Co-Authored-By: Claude Opus 5 <[email protected]>
2026-08-16 22:52:59 -05:00
parent 993986c25e
commit ae70bf1dd8
22 changed files with 3691 additions and 24 deletions
@@ -0,0 +1,376 @@
+const test = require("node:test");
+const assert = require("node:assert/strict");
+const { openDatabase } = require("../src/database");
+const { runMigrations } = require("../src/database/migrate");
+const { createDisplayService } = require("../src/displays/service");
+const { createNotificationService } = require("../src/notifications");
+
+const BASE_TIME = 2_000_000_000_000;
+const THRESHOLD = 45_000;
+
+function createClock(initial = BASE_TIME) {
+  let current = initial;
+  let sequence = 0;
+  const timers = new Set();
+  return {
+    now: () => current,
+    set(value) {
+      current = value;
+    },
+    setTimeout(callback, delay) {
+      const timer = { at: current + delay, callback, cleared: false, id: ++sequence };
+      timers.add(timer);
+      return timer;
+    },
+    clearTimeout(timer) {
+      if (timer) timer.cleared = true;
+    },
+    runTo(value) {
+      current = value;
+      for (let guard = 0; guard < 200; guard += 1) {
+        const timer = [...timers]
+          .filter((candidate) => !candidate.cleared && !candidate.fired && candidate.at <= current)
+          .sort((left, right) => left.at - right.at || left.id - right.id)[0];
+        if (!timer) return;
+        timer.fired = true;
+        timer.callback();
+      }
+      throw new Error("timer loop did not settle");
+    },
+  };
+}
+
+function heartbeatPayload(overrides = {}) {
+  return {
+    protocolVersion: 1,
+    clientId: "lobby-display",
+    outputId: "global",
+    activePlaylistId: "playlist-a",
+    playlistRevision: 12,
+    playbackRevision: 42,
+    currentEntryId: "entry-a",
+    currentAssetId: "asset-a",
+    cachedAssetCount: 10,
+    playableAssetCount: 10,
+    mostRecentError: null,
+    ...overrides,
+  };
+}
+
+function createHarness({ start = true } = {}) {
+  const database = openDatabase({ path: ":memory:" });
+  runMigrations(database);
+  const clock = createClock();
+  const events = [];
+  let idCounter = 0;
+  const notifications = createNotificationService({
+    database,
+    webhookUrl: "http://127.0.0.1:9/hook",
+    now: clock.now,
+    setTimeoutFn: clock.setTimeout,
+    clearTimeoutFn: clock.clearTimeout,
+    createId: () => `note-${++idCounter}`,
+    fetchFn: () => Promise.resolve({ status: 200 }),
+    logger: { warn() {} },
+  });
+
+  function build() {
+    const service = createDisplayService({
+      database,
+      notifications,
+      now: clock.now,
+      setTimeoutFn: clock.setTimeout,
+      clearTimeoutFn: clock.clearTimeout,
+      offlineThresholdMs: THRESHOLD,
+      onStatusChanged: (event) => events.push(event),
+      logger: { warn() {} },
+    });
+    if (start) service.start();
+    return service;
+  }
+
+  return {
+    build,
+    clock,
+    database,
+    events,
+    notifications,
+    service: build(),
+    outbox: () =>
+      database.prepare("SELECT * FROM notification_outbox ORDER BY created_at, id").all(),
+    incidents: () =>
+      database.prepare("SELECT * FROM notification_incidents ORDER BY opened_at").all(),
+    rows: () => database.prepare("SELECT * FROM display_clients").all(),
+  };
+}
+
+test("a first heartbeat creates the display and later heartbeats update the same record", (t) => {
+  const harness = createHarness();
+  t.after(() => harness.database.close());
+
+  const first = harness.service.heartbeat(heartbeatPayload());
+  assert.equal(first.status, "online");
+  assert.equal(first.clientId, "lobby-display");
+  assert.equal(harness.rows().length, 1);
+  assert.equal(harness.events.at(-1).reason, "first_seen");
+
+  harness.clock.set(BASE_TIME + 12_000);
+  const second = harness.service.heartbeat(
+    heartbeatPayload({ playbackRevision: 43, cachedAssetCount: 11 }),
+  );
+  assert.equal(harness.rows().length, 1, "the persistent client ID is the display identity");
+  assert.equal(second.playbackRevision, 43);
+  assert.equal(second.cachedAssetCount, 11);
+  assert.equal(
+    new Date(second.firstSeenAt).getTime(),
+    BASE_TIME,
+    "first_seen_at is preserved across heartbeats",
+  );
+  assert.equal(new Date(second.lastSeenAt).getTime(), BASE_TIME + 12_000);
+  assert.equal(
+    harness.events.filter((event) => event.reason === "first_seen").length,
+    1,
+    "a steady heartbeat creates no realtime churn",
+  );
+});
+
+test("the same display keeps its identity across a new Socket.IO connection", (t) => {
+  const harness = createHarness();
+  t.after(() => harness.database.close());
+
+  harness.service.heartbeat(heartbeatPayload());
+  // A browser reconnect produces a new socket but the same persisted client ID.
+  harness.clock.set(BASE_TIME + 20_000);
+  harness.service.heartbeat(heartbeatPayload());
+
+  assert.equal(harness.rows().length, 1);
+  assert.equal(harness.service.list().length, 1);
+  assert.equal(harness.service.list()[0].status, "online");
+});
+
+test("malformed telemetry is rejected safely and never creates a display", (t) => {
+  const harness = createHarness();
+  t.after(() => harness.database.close());
+
+  const invalid = [
+    null,
+    "string",
+    [],
+    {},
+    heartbeatPayload({ protocolVersion: 2 }),
+    heartbeatPayload({ protocolVersion: undefined }),
+    heartbeatPayload({ clientId: "" }),
+    heartbeatPayload({ clientId: "   " }),
+    heartbeatPayload({ clientId: 42 }),
+    heartbeatPayload({ clientId: "x".repeat(300) }),
+    heartbeatPayload({ cachedAssetCount: -1 }),
+    heartbeatPayload({ cachedAssetCount: 1.5 }),
+    heartbeatPayload({ playbackRevision: -5 }),
+    heartbeatPayload({ mostRecentError: { nested: true } }),
+  ];
+  for (const payload of invalid) {
+    assert.throws(() => harness.service.heartbeat(payload), TypeError);
+  }
+  assert.equal(harness.rows().length, 0);
+});
+
+test("an oversized error string is bounded before it is stored", (t) => {
+  const harness = createHarness();
+  t.after(() => harness.database.close());
+
+  const result = harness.service.heartbeat(
+    heartbeatPayload({ mostRecentError: "E".repeat(5_000_000) }),
+  );
+
+  assert.ok(result.mostRecentError.length <= 500, "a client cannot store megabytes");
+  assert.ok(harness.rows()[0].last_error.length <= 500);
+});
+
+test("heartbeats never change playback or playlist revisions", (t) => {
+  const harness = createHarness();
+  t.after(() => harness.database.close());
+  const playbackBefore = harness.database
+    .prepare("SELECT revision, updated_at FROM playback_state WHERE output_id = 'global'")
+    .get();
+
+  for (let beat = 0; beat < 5; beat += 1) {
+    harness.clock.set(BASE_TIME + beat * 10_000);
+    harness.service.heartbeat(heartbeatPayload({ playbackRevision: 99 }));
+  }
+
+  assert.deepEqual(
+    harness.database
+      .prepare("SELECT revision, updated_at FROM playback_state WHERE output_id = 'global'")
+      .get(),
+    playbackBefore,
+    "telemetry is read-only reporting",
+  );
+});
+
+test("one missed heartbeat does not mark a display offline before the threshold", (t) => {
+  const harness = createHarness();
+  t.after(() => harness.database.close());
+  harness.service.heartbeat(heartbeatPayload());
+
+  // Two heartbeat intervals have passed but the 45s threshold has not.
+  harness.clock.runTo(BASE_TIME + 30_000);
+  assert.equal(harness.service.list()[0].status, "online");
+  assert.equal(harness.incidents().length, 0);
+
+  harness.clock.runTo(BASE_TIME + THRESHOLD - 1);
+  assert.equal(harness.service.list()[0].status, "online");
+  assert.equal(harness.incidents().length, 0);
+});
+
+test("crossing the threshold opens exactly one offline incident that never repeats", (t) => {
+  const harness = createHarness();
+  t.after(() => harness.database.close());
+  harness.service.heartbeat(heartbeatPayload());
+
+  harness.clock.runTo(BASE_TIME + THRESHOLD);
+  assert.equal(harness.service.list()[0].status, "offline");
+  const incidents = harness.incidents();
+  assert.equal(incidents.length, 1);
+  assert.equal(incidents[0].event, "display.offline");
+  assert.equal(incidents[0].condition_key, "display:lobby-display");
+  assert.equal(JSON.parse(incidents[0].context_json).clientId, "lobby-display");
+  assert.equal(harness.outbox().length, 1);
+  assert.equal(harness.events.at(-1).reason, "offline");
+
+  // Staying offline for a long time must not produce more notifications.
+  harness.clock.runTo(BASE_TIME + 30 * 60_000);
+  assert.equal(harness.incidents().length, 1);
+  assert.equal(harness.outbox().length, 1);
+  assert.equal(
+    harness.events.filter((event) => event.reason === "offline").length,
+    1,
+  );
+});
+
+test("a returning heartbeat recovers the display exactly once", (t) => {
+  const harness = createHarness();
+  t.after(() => harness.database.close());
+  harness.service.heartbeat(heartbeatPayload());
+  harness.clock.runTo(BASE_TIME + THRESHOLD);
+  assert.equal(harness.outbox().length, 1);
+
+  harness.clock.set(BASE_TIME + 60_000);
+  const recovered = harness.service.heartbeat(heartbeatPayload());
+  assert.equal(recovered.status, "online");
+  assert.equal(harness.incidents()[0].state, "recovered");
+  assert.equal(harness.outbox().length, 2);
+  assert.equal(harness.outbox()[1].transition, "recovery");
+  assert.equal(harness.events.at(-1).reason, "recovered");
+
+  // Continued healthy heartbeats add nothing further.
+  harness.clock.set(BASE_TIME + 70_000);
+  harness.service.heartbeat(heartbeatPayload());
+  assert.equal(harness.outbox().length, 2);
+  assert.equal(
+    harness.events.filter((event) => event.reason === "recovered").length,
+    1,
+  );
+
+  // A later disappearance is a new episode with its own single notification.
+  harness.clock.runTo(BASE_TIME + 70_000 + THRESHOLD);
+  assert.equal(harness.incidents().length, 2);
+  assert.equal(harness.outbox().length, 3);
+});
+
+test("a restart preserves known displays and cannot manufacture offline alerts", (t) => {
+  const harness = createHarness();
+  t.after(() => harness.database.close());
+  harness.service.heartbeat(heartbeatPayload());
+  harness.service.stop();
+
+  // The API was down for well over the threshold. A healthy display simply
+  // has not had the chance to report yet, so startup stays silent until a
+  // full threshold has elapsed since the restart.
+  harness.clock.set(BASE_TIME + 10 * 60_000);
+  const restarted = harness.build();
+  assert.equal(harness.rows().length, 1, "display history survives a restart");
+  assert.equal(restarted.list()[0].status, "offline", "status still reflects staleness");
+  assert.equal(harness.incidents().length, 0, "no notification storm after a restart");
+
+  harness.clock.runTo(BASE_TIME + 10 * 60_000 + 20_000);
+  assert.equal(harness.incidents().length, 0, "still inside the startup grace period");
+
+  restarted.heartbeat(heartbeatPayload());
+  assert.equal(restarted.list()[0].status, "online");
+  assert.equal(harness.incidents().length, 0, "a display that comes back was never offline");
+});
+
+test("a display still missing after the startup grace period is reported once", (t) => {
+  const harness = createHarness();
+  t.after(() => harness.database.close());
+  harness.service.heartbeat(heartbeatPayload());
+  harness.service.stop();
+
+  harness.clock.set(BASE_TIME + 10 * 60_000);
+  harness.build();
+  harness.clock.runTo(BASE_TIME + 10 * 60_000 + THRESHOLD);
+
+  assert.equal(harness.incidents().length, 1);
+  assert.equal(harness.incidents()[0].event, "display.offline");
+  assert.equal(harness.outbox().length, 1);
+});
+
+test("an already-open offline incident is not duplicated by a restart", (t) => {
+  const harness = createHarness();
+  t.after(() => harness.database.close());
+  harness.service.heartbeat(heartbeatPayload());
+  harness.clock.runTo(BASE_TIME + THRESHOLD);
+  assert.equal(harness.incidents().length, 1);
+  harness.service.stop();
+
+  harness.clock.set(BASE_TIME + 20 * 60_000);
+  harness.build();
+  harness.clock.runTo(BASE_TIME + 20 * 60_000 + THRESHOLD + 1_000);
+
+  assert.equal(harness.incidents().length, 1, "the same episode stays open");
+  assert.equal(harness.outbox().length, 1);
+});
+
+test("a changed error is surfaced as a meaningful status update", (t) => {
+  const harness = createHarness();
+  t.after(() => harness.database.close());
+  harness.service.heartbeat(heartbeatPayload());
+  harness.events.length = 0;
+
+  harness.clock.set(BASE_TIME + 10_000);
+  harness.service.heartbeat(heartbeatPayload({ mostRecentError: "decode failed for asset-a" }));
+  assert.equal(harness.events.length, 1);
+  assert.equal(harness.events[0].reason, "error_changed");
+  assert.equal(harness.events[0].mostRecentError, "decode failed for asset-a");
+
+  // The same error repeating is not a new event.
+  harness.clock.set(BASE_TIME + 20_000);
+  harness.service.heartbeat(heartbeatPayload({ mostRecentError: "decode failed for asset-a" }));
+  assert.equal(harness.events.length, 1);
+
+  // An arbitrary browser string is telemetry only, never a notification.
+  assert.equal(harness.incidents().length, 0);
+  assert.equal(harness.outbox().length, 0);
+});
+
+test("display status derives from heartbeat freshness for every known client", (t) => {
+  const harness = createHarness();
+  t.after(() => harness.database.close());
+
+  harness.service.heartbeat(heartbeatPayload({ clientId: "lobby" }));
+  harness.clock.set(BASE_TIME + 40_000);
+  harness.service.heartbeat(heartbeatPayload({ clientId: "narthex" }));
+  harness.clock.set(BASE_TIME + 50_000);
+
+  const displays = Object.fromEntries(
+    harness.service.list().map((display) => [display.clientId, display]),
+  );
+  assert.equal(displays.lobby.status, "offline");
+  assert.equal(displays.narthex.status, "online");
+  assert.equal(harness.service.status().online, 1);
+  assert.equal(harness.service.status().offline, 1);
+  assert.equal(harness.service.status().known, 2);
+
+  const serialized = JSON.stringify(harness.service.list());
+  assert.doesNotMatch(serialized, /socket|sid|transport/i);
+});