Implement Phase 4 integrations and display telemetry

Add MQTT playback control, durable Home Assistant incident notifications, and display heartbeat telemetry on top of the verified Phase 3B playback authority. No playback logic is duplicated: every MQTT command and every observed condition goes through the existing playback service, and all integration work happens after the authoritative SQLite commit. MQTT (server/src/mqtt) subscribes the exact required command topics under a configurable prefix that defaults to announcements. Durable commands use QoS 1; next and previous stay at QoS 0 because they are not idempotent and carry no command ID. Retained authoritative state is published only on the separate availability, mode/state, and playback/state topics, never onto a command topic, and only after a mutation commits. A registered last will reports an ungraceful disappearance, a clean shutdown retracts it, and a broker outage degrades status while playback, HTTP, and the display protocol continue untouched. Notifications (server/src/notifications) use the Phase 1 incident and outbox tables. A failing condition opens one incident and queues exactly one notification; continued failure only updates its observation time; recovery queues exactly one recovery; a later failure is a new episode. Delivery retries on a documented capped schedule with a stable notification ID, survives restart, and never blocks playback. With no webhook URL configured, incidents are still recorded but no delivery work is queued. Conditions come from storage health transitions, committed playback state, display heartbeat staleness, and newly failed uploads. Displays (server/src/displays) ingest validated, bounded display:heartbeat telemetry keyed by persistent client ID rather than socket ID. Telemetry is read-only and never touches playback or playlist revisions. Offline is decided by heartbeat staleness, not by a closed socket, and a startup grace period keeps an API restart from manufacturing offline alerts. Adds GET /api/displays and meaningful-change display:status events. Health and status now report redacted storage, MQTT, notification, playback, and display state. External integrations degrade status and never affect readiness, so no integration outage can cause a restart loop. No MQTT password, S3 credential, webhook URL, or webhook secret is exposed. Uses the existing Phase 1 schema unchanged; no migration was required. Co-Authored-By: Claude Opus 5 <[email protected]>
2026-08-16 22:52:59 -05:00
parent 993986c25e
commit ae70bf1dd8
22 changed files with 3691 additions and 24 deletions
@@ -0,0 +1,383 @@
+const test = require("node:test");
+const assert = require("node:assert/strict");
+const { openDatabase } = require("../src/database");
+const { runMigrations } = require("../src/database/migrate");
+const {
+  RETRY_BACKOFF_MS,
+  backoffMs,
+  createNotificationService,
+} = require("../src/notifications");
+const {
+  observeAssetProcessingFailure,
+  observePlayback,
+  observeStorageHealth,
+} = require("../src/notifications/conditions");
+
+const BASE_TIME = 2_000_000_000_000;
+const WEBHOOK = "http://127.0.0.1:9/announcements";
+
+function createClock(initial = BASE_TIME) {
+  let current = initial;
+  let sequence = 0;
+  const timers = new Set();
+  return {
+    now: () => current,
+    set(value) {
+      current = value;
+    },
+    setTimeout(callback, delay) {
+      const timer = { at: current + delay, callback, cleared: false, id: ++sequence };
+      timers.add(timer);
+      return timer;
+    },
+    clearTimeout(timer) {
+      if (timer) timer.cleared = true;
+    },
+    pending() {
+      return [...timers].filter((timer) => !timer.cleared && !timer.fired);
+    },
+    async runTo(value) {
+      current = value;
+      for (let guard = 0; guard < 100; guard += 1) {
+        const timer = this.pending()
+          .filter((candidate) => candidate.at <= current)
+          .sort((left, right) => left.at - right.at || left.id - right.id)[0];
+        if (!timer) return;
+        timer.fired = true;
+        await timer.callback();
+      }
+      throw new Error("timer loop did not settle");
+    },
+  };
+}
+
+function createHarness({ webhookUrl = WEBHOOK, responses = [] } = {}) {
+  const database = openDatabase({ path: ":memory:" });
+  runMigrations(database);
+  const clock = createClock();
+  const calls = [];
+  let idCounter = 0;
+
+  function fetchFn(url, init) {
+    calls.push({ url, body: JSON.parse(init.body), headers: init.headers });
+    const next = responses.shift();
+    if (typeof next === "function") return next();
+    if (next instanceof Error) return Promise.reject(next);
+    return Promise.resolve({ status: next ?? 200 });
+  }
+
+  function build() {
+    return createNotificationService({
+      database,
+      webhookUrl,
+      webhookSecret: "super-secret-webhook-id",
+      now: clock.now,
+      setTimeoutFn: clock.setTimeout,
+      clearTimeoutFn: clock.clearTimeout,
+      createId: () => `note-${++idCounter}`,
+      fetchFn,
+      logger: { warn() {} },
+    });
+  }
+
+  return {
+    build,
+    calls,
+    clock,
+    database,
+    responses,
+    outbox: () =>
+      database.prepare("SELECT * FROM notification_outbox ORDER BY created_at, id").all(),
+    incidents: () =>
+      database
+        .prepare("SELECT * FROM notification_incidents ORDER BY opened_at, episode")
+        .all(),
+  };
+}
+
+const STORAGE_FAILURE = {
+  event: "storage.unavailable",
+  severity: "critical",
+  message: "Announcement storage is unavailable.",
+  context: { checkedAt: "now" },
+};
+
+test("a persistent failure opens one incident and queues exactly one notification", async (t) => {
+  const harness = createHarness();
+  const service = harness.build();
+  t.after(() => harness.database.close());
+  service.start();
+
+  const first = service.observeFailure("storage", STORAGE_FAILURE);
+  assert.equal(first.opened, true);
+
+  // A long outage is observed many times; none of it may become new work.
+  for (let observation = 0; observation < 25; observation += 1) {
+    harness.clock.set(BASE_TIME + observation * 60_000);
+    assert.equal(service.observeFailure("storage", STORAGE_FAILURE).opened, false);
+  }
+
+  assert.equal(harness.incidents().length, 1);
+  assert.equal(harness.outbox().length, 1);
+  assert.equal(harness.incidents()[0].state, "open");
+  assert.equal(
+    harness.incidents()[0].last_observed_at,
+    BASE_TIME + 24 * 60_000,
+    "a continuing failure only updates its observation time",
+  );
+});
+
+test("recovery queues exactly one recovery and a later failure is a new episode", async (t) => {
+  const harness = createHarness();
+  const service = harness.build();
+  t.after(() => harness.database.close());
+  service.start();
+
+  service.observeFailure("storage", STORAGE_FAILURE);
+  harness.clock.set(BASE_TIME + 30 * 60_000);
+  const recovery = service.observeRecovery("storage", {
+    event: "storage.recovered",
+    message: "Announcement storage is reachable again.",
+  });
+  assert.equal(recovery.recovered, true);
+  assert.equal(service.observeRecovery("storage", {}).recovered, false);
+  assert.equal(harness.outbox().length, 2);
+
+  harness.clock.set(BASE_TIME + 60 * 60_000);
+  const second = service.observeFailure("storage", STORAGE_FAILURE);
+  assert.equal(second.opened, true);
+  const incidents = harness.incidents();
+  assert.equal(incidents.length, 2);
+  assert.deepEqual(
+    incidents.map((incident) => incident.episode),
+    [1, 2],
+    "a repeat condition is a new episode, not a reopened incident",
+  );
+  assert.equal(incidents[0].state, "recovered");
+  assert.equal(incidents[1].state, "open");
+  assert.equal(harness.outbox().length, 3);
+});
+
+test("a 2xx response marks delivery complete and carries the versioned payload", async (t) => {
+  const harness = createHarness({ responses: [204] });
+  const service = harness.build();
+  t.after(() => harness.database.close());
+  service.start();
+
+  service.observeFailure("storage", STORAGE_FAILURE);
+  await harness.clock.runTo(BASE_TIME);
+
+  assert.equal(harness.calls.length, 1);
+  const payload = harness.calls[0].body;
+  assert.equal(payload.version, 1);
+  assert.equal(payload.event, "storage.unavailable");
+  assert.equal(payload.severity, "critical");
+  assert.equal(payload.state, "failure");
+  assert.equal(payload.message, "Announcement storage is unavailable.");
+  assert.match(payload.timestamp, /^\d{4}-\d{2}-\d{2}T/);
+  assert.ok(payload.notificationId && payload.incidentId);
+  assert.equal(payload.context.conditionKey, "storage");
+  assert.equal(payload.context.episode, 1);
+  assert.equal(harness.outbox()[0].delivered_at, BASE_TIME);
+  assert.equal(service.status().pendingCount, 0);
+  assert.equal(service.status().deliveredCount, 1);
+});
+
+test("failed deliveries retry on a capped schedule with a stable notification id", async (t) => {
+  const harness = createHarness({
+    // One attempt per documented backoff step: transport errors and both 4xx
+    // and 5xx responses follow the same retry policy.
+    responses: [
+      new Error("ECONNREFUSED"),
+      500,
+      404,
+      new Error("ECONNREFUSED"),
+      new Error("ECONNREFUSED"),
+      200,
+    ],
+  });
+  const service = harness.build();
+  t.after(() => harness.database.close());
+  service.start();
+
+  service.observeFailure("storage", STORAGE_FAILURE);
+  let elapsed = 0;
+  await harness.clock.runTo(BASE_TIME);
+  assert.equal(harness.calls.length, 1);
+
+  // Documented schedule: immediate, then 30s, 1m, 5m, 15m, 30m, capped at 30m.
+  for (const expected of RETRY_BACKOFF_MS) {
+    assert.equal(
+      harness.outbox()[0].next_attempt_at,
+      BASE_TIME + elapsed + expected,
+      `retry must be scheduled ${expected}ms after the failed attempt`,
+    );
+    elapsed += expected;
+    await harness.clock.runTo(BASE_TIME + elapsed);
+  }
+  assert.equal(backoffMs(50), 1_800_000, "backoff is capped, not unbounded");
+
+  const ids = new Set(harness.calls.map((call) => call.body.notificationId));
+  assert.equal(ids.size, 1, "retries must reuse one stable delivery identity");
+  assert.equal(harness.outbox()[0].delivered_at, BASE_TIME + elapsed);
+  assert.equal(harness.outbox().length, 1, "retrying never creates new rows");
+});
+
+test("open incidents and pending deliveries survive an API restart", async (t) => {
+  const harness = createHarness({ responses: [new Error("ECONNREFUSED")] });
+  const first = harness.build();
+  t.after(() => harness.database.close());
+  first.start();
+  first.observeFailure("storage", STORAGE_FAILURE);
+  await harness.clock.runTo(BASE_TIME);
+  assert.equal(harness.outbox()[0].delivered_at, null);
+  first.stop();
+
+  // A restart re-reads durable state; it must not re-notify or lose work.
+  harness.clock.set(BASE_TIME + 5 * 60_000);
+  const restarted = harness.build();
+  restarted.start();
+  assert.equal(restarted.isOpen("storage"), true);
+  assert.equal(
+    restarted.observeFailure("storage", STORAGE_FAILURE).opened,
+    false,
+    "a restart must not reopen an already-open incident",
+  );
+  assert.equal(harness.outbox().length, 1);
+  assert.equal(restarted.status().pendingCount, 1);
+
+  harness.responses.push(200);
+  await harness.clock.runTo(BASE_TIME + 10 * 60_000);
+  assert.equal(harness.outbox()[0].delivered_at, BASE_TIME + 10 * 60_000);
+  assert.equal(harness.calls.at(-1).body.notificationId, harness.calls[0].body.notificationId);
+});
+
+test("an unconfigured webhook records incidents without queuing endless delivery work", async (t) => {
+  const harness = createHarness({ webhookUrl: null });
+  const service = harness.build();
+  t.after(() => harness.database.close());
+  service.start();
+
+  service.observeFailure("storage", STORAGE_FAILURE);
+  service.observeRecovery("storage", { event: "storage.recovered" });
+  await harness.clock.runTo(BASE_TIME + 60 * 60_000);
+
+  assert.equal(harness.incidents().length, 1, "incident history stays useful");
+  assert.equal(harness.outbox().length, 0, "nothing retries against a receiver that does not exist");
+  assert.equal(harness.calls.length, 0);
+  const status = service.status();
+  assert.equal(status.configured, false);
+  assert.equal(status.status, "disabled");
+});
+
+test("notification status never reveals the webhook URL or secret", async (t) => {
+  const harness = createHarness({ responses: [new Error("ECONNREFUSED")] });
+  const service = harness.build();
+  t.after(() => harness.database.close());
+  service.start();
+  service.observeFailure("storage", STORAGE_FAILURE);
+  await harness.clock.runTo(BASE_TIME);
+
+  const serialized = JSON.stringify(service.status());
+  assert.doesNotMatch(serialized, /super-secret-webhook-id/);
+  assert.doesNotMatch(serialized, /127\.0\.0\.1:9/);
+  assert.doesNotMatch(serialized, /announcements\/?"?$/);
+  assert.equal(service.status().configured, true);
+  assert.equal(service.status().openIncidents.length, 1);
+  assert.equal(service.status().openIncidents[0].conditionKey, "storage");
+});
+
+test("storage health transitions drive exactly one incident pair", async (t) => {
+  const harness = createHarness();
+  const service = harness.build();
+  t.after(() => harness.database.close());
+  service.start();
+
+  observeStorageHealth(service, {
+    status: "degraded",
+    code: "STORAGE_NOT_CHECKED",
+  });
+  assert.equal(harness.incidents().length, 0, "startup state is not an outage");
+
+  for (let observation = 0; observation < 5; observation += 1) {
+    observeStorageHealth(service, {
+      status: "degraded",
+      code: "STORAGE_UNAVAILABLE",
+      checkedAt: "2026-08-16T12:00:00.000Z",
+    });
+  }
+  assert.equal(harness.outbox().length, 1);
+
+  observeStorageHealth(service, { status: "healthy", checkedAt: "2026-08-16T12:30:00.000Z" });
+  assert.equal(harness.outbox().length, 2);
+  assert.equal(harness.outbox()[1].transition, "recovery");
+});
+
+test("no playable slides is reported even while a static override fills the screen", async (t) => {
+  const harness = createHarness();
+  const service = harness.build();
+  t.after(() => harness.database.close());
+  service.start();
+
+  observePlayback(service, {
+    activePlaylistId: "playlist-a",
+    normalTarget: { id: "entry" },
+    mode: "on",
+  });
+  assert.equal(harness.incidents().length, 0);
+
+  observePlayback(service, {
+    activePlaylistId: "playlist-a",
+    normalTarget: null,
+    staticTarget: { assetId: "static-asset" },
+    mode: "on",
+  });
+  const open = harness.incidents();
+  assert.equal(open.length, 1);
+  assert.equal(open[0].event, "playback.no_playable_slides");
+  assert.equal(JSON.parse(open[0].context_json).staticActive, true);
+
+  observePlayback(service, {
+    activePlaylistId: "playlist-a",
+    normalTarget: null,
+    staticTarget: { assetId: "static-asset" },
+    mode: "on",
+  });
+  assert.equal(harness.outbox().length, 1, "a continuing condition never re-notifies");
+
+  observePlayback(service, {
+    activePlaylistId: "playlist-a",
+    normalTarget: { id: "entry" },
+    mode: "on",
+  });
+  assert.equal(harness.incidents()[0].state, "recovered");
+  assert.equal(harness.outbox().length, 2);
+});
+
+test("each failed upload opens one bounded warning incident", async (t) => {
+  const harness = createHarness();
+  const service = harness.build();
+  t.after(() => harness.database.close());
+  service.start();
+
+  observeAssetProcessingFailure(service, {
+    assetId: "asset-1",
+    displayName: "Bulletin",
+    message: "x".repeat(5_000),
+  });
+  observeAssetProcessingFailure(service, {
+    assetId: "asset-1",
+    displayName: "Bulletin",
+    message: "again",
+  });
+  observeAssetProcessingFailure(service, { assetId: "asset-2", displayName: "Notice" });
+
+  assert.equal(harness.incidents().length, 2, "one incident per failed asset");
+  assert.equal(harness.outbox().length, 2);
+  const context = JSON.parse(harness.incidents()[0].context_json);
+  assert.ok(
+    context.processingError.length <= 500,
+    "a processing error is bounded before it is stored",
+  );
+  assert.equal(harness.incidents()[0].severity, "warning");
+});