const test = require("node:test"); const assert = require("node:assert/strict"); const { openDatabase } = require("../src/database"); const { runMigrations } = require("../src/database/migrate"); const { RETRY_BACKOFF_MS, backoffMs, createNotificationService, } = require("../src/notifications"); const { observeAssetProcessingFailure, observePlayback, observeStorageHealth, } = require("../src/notifications/conditions"); const BASE_TIME = 2_000_000_000_000; const WEBHOOK = "http://127.0.0.1:9/announcements"; function createClock(initial = BASE_TIME) { let current = initial; let sequence = 0; const timers = new Set(); return { now: () => current, set(value) { current = value; }, setTimeout(callback, delay) { const timer = { at: current + delay, callback, cleared: false, id: ++sequence }; timers.add(timer); return timer; }, clearTimeout(timer) { if (timer) timer.cleared = true; }, pending() { return [...timers].filter((timer) => !timer.cleared && !timer.fired); }, async runTo(value) { current = value; for (let guard = 0; guard < 100; guard += 1) { const timer = this.pending() .filter((candidate) => candidate.at <= current) .sort((left, right) => left.at - right.at || left.id - right.id)[0]; if (!timer) return; timer.fired = true; await timer.callback(); } throw new Error("timer loop did not settle"); }, }; } function createHarness({ webhookUrl = WEBHOOK, responses = [] } = {}) { const database = openDatabase({ path: ":memory:" }); runMigrations(database); const clock = createClock(); const calls = []; let idCounter = 0; function fetchFn(url, init) { calls.push({ url, body: JSON.parse(init.body), headers: init.headers }); const next = responses.shift(); if (typeof next === "function") return next(); if (next instanceof Error) return Promise.reject(next); return Promise.resolve({ status: next ?? 200 }); } function build() { return createNotificationService({ database, webhookUrl, webhookSecret: "super-secret-webhook-id", now: clock.now, setTimeoutFn: clock.setTimeout, clearTimeoutFn: clock.clearTimeout, createId: () => `note-${++idCounter}`, fetchFn, logger: { warn() {} }, }); } return { build, calls, clock, database, responses, outbox: () => database.prepare("SELECT * FROM notification_outbox ORDER BY created_at, id").all(), incidents: () => database .prepare("SELECT * FROM notification_incidents ORDER BY opened_at, episode") .all(), }; } const STORAGE_FAILURE = { event: "storage.unavailable", severity: "critical", message: "Announcement storage is unavailable.", context: { checkedAt: "now" }, }; test("a persistent failure opens one incident and queues exactly one notification", async (t) => { const harness = createHarness(); const service = harness.build(); t.after(() => harness.database.close()); service.start(); const first = service.observeFailure("storage", STORAGE_FAILURE); assert.equal(first.opened, true); // A long outage is observed many times; none of it may become new work. for (let observation = 0; observation < 25; observation += 1) { harness.clock.set(BASE_TIME + observation * 60_000); assert.equal(service.observeFailure("storage", STORAGE_FAILURE).opened, false); } assert.equal(harness.incidents().length, 1); assert.equal(harness.outbox().length, 1); assert.equal(harness.incidents()[0].state, "open"); assert.equal( harness.incidents()[0].last_observed_at, BASE_TIME + 24 * 60_000, "a continuing failure only updates its observation time", ); }); test("recovery queues exactly one recovery and a later failure is a new episode", async (t) => { const harness = createHarness(); const service = harness.build(); t.after(() => harness.database.close()); service.start(); service.observeFailure("storage", STORAGE_FAILURE); harness.clock.set(BASE_TIME + 30 * 60_000); const recovery = service.observeRecovery("storage", { event: "storage.recovered", message: "Announcement storage is reachable again.", }); assert.equal(recovery.recovered, true); assert.equal(service.observeRecovery("storage", {}).recovered, false); assert.equal(harness.outbox().length, 2); harness.clock.set(BASE_TIME + 60 * 60_000); const second = service.observeFailure("storage", STORAGE_FAILURE); assert.equal(second.opened, true); const incidents = harness.incidents(); assert.equal(incidents.length, 2); assert.deepEqual( incidents.map((incident) => incident.episode), [1, 2], "a repeat condition is a new episode, not a reopened incident", ); assert.equal(incidents[0].state, "recovered"); assert.equal(incidents[1].state, "open"); assert.equal(harness.outbox().length, 3); }); test("a 2xx response marks delivery complete and carries the versioned payload", async (t) => { const harness = createHarness({ responses: [204] }); const service = harness.build(); t.after(() => harness.database.close()); service.start(); service.observeFailure("storage", STORAGE_FAILURE); await harness.clock.runTo(BASE_TIME); assert.equal(harness.calls.length, 1); const payload = harness.calls[0].body; assert.equal(payload.version, 1); assert.equal(payload.event, "storage.unavailable"); assert.equal(payload.severity, "critical"); assert.equal(payload.state, "failure"); assert.equal(payload.message, "Announcement storage is unavailable."); assert.match(payload.timestamp, /^\d{4}-\d{2}-\d{2}T/); assert.ok(payload.notificationId && payload.incidentId); assert.equal(payload.context.conditionKey, "storage"); assert.equal(payload.context.episode, 1); assert.equal(harness.outbox()[0].delivered_at, BASE_TIME); assert.equal(service.status().pendingCount, 0); assert.equal(service.status().deliveredCount, 1); }); test("failed deliveries retry on a capped schedule with a stable notification id", async (t) => { const harness = createHarness({ // One attempt per documented backoff step: transport errors and both 4xx // and 5xx responses follow the same retry policy. responses: [ new Error("ECONNREFUSED"), 500, 404, new Error("ECONNREFUSED"), new Error("ECONNREFUSED"), 200, ], }); const service = harness.build(); t.after(() => harness.database.close()); service.start(); service.observeFailure("storage", STORAGE_FAILURE); let elapsed = 0; await harness.clock.runTo(BASE_TIME); assert.equal(harness.calls.length, 1); // Documented schedule: immediate, then 30s, 1m, 5m, 15m, 30m, capped at 30m. for (const expected of RETRY_BACKOFF_MS) { assert.equal( harness.outbox()[0].next_attempt_at, BASE_TIME + elapsed + expected, `retry must be scheduled ${expected}ms after the failed attempt`, ); elapsed += expected; await harness.clock.runTo(BASE_TIME + elapsed); } assert.equal(backoffMs(50), 1_800_000, "backoff is capped, not unbounded"); const ids = new Set(harness.calls.map((call) => call.body.notificationId)); assert.equal(ids.size, 1, "retries must reuse one stable delivery identity"); assert.equal(harness.outbox()[0].delivered_at, BASE_TIME + elapsed); assert.equal(harness.outbox().length, 1, "retrying never creates new rows"); }); test("open incidents and pending deliveries survive an API restart", async (t) => { const harness = createHarness({ responses: [new Error("ECONNREFUSED")] }); const first = harness.build(); t.after(() => harness.database.close()); first.start(); first.observeFailure("storage", STORAGE_FAILURE); await harness.clock.runTo(BASE_TIME); assert.equal(harness.outbox()[0].delivered_at, null); first.stop(); // A restart re-reads durable state; it must not re-notify or lose work. harness.clock.set(BASE_TIME + 5 * 60_000); const restarted = harness.build(); restarted.start(); assert.equal(restarted.isOpen("storage"), true); assert.equal( restarted.observeFailure("storage", STORAGE_FAILURE).opened, false, "a restart must not reopen an already-open incident", ); assert.equal(harness.outbox().length, 1); assert.equal(restarted.status().pendingCount, 1); harness.responses.push(200); await harness.clock.runTo(BASE_TIME + 10 * 60_000); assert.equal(harness.outbox()[0].delivered_at, BASE_TIME + 10 * 60_000); assert.equal(harness.calls.at(-1).body.notificationId, harness.calls[0].body.notificationId); }); test("an unconfigured webhook records incidents without queuing endless delivery work", async (t) => { const harness = createHarness({ webhookUrl: null }); const service = harness.build(); t.after(() => harness.database.close()); service.start(); service.observeFailure("storage", STORAGE_FAILURE); service.observeRecovery("storage", { event: "storage.recovered" }); await harness.clock.runTo(BASE_TIME + 60 * 60_000); assert.equal(harness.incidents().length, 1, "incident history stays useful"); assert.equal(harness.outbox().length, 0, "nothing retries against a receiver that does not exist"); assert.equal(harness.calls.length, 0); const status = service.status(); assert.equal(status.configured, false); assert.equal(status.status, "disabled"); }); test("notification status never reveals the webhook URL or secret", async (t) => { const harness = createHarness({ responses: [new Error("ECONNREFUSED")] }); const service = harness.build(); t.after(() => harness.database.close()); service.start(); service.observeFailure("storage", STORAGE_FAILURE); await harness.clock.runTo(BASE_TIME); const serialized = JSON.stringify(service.status()); assert.doesNotMatch(serialized, /super-secret-webhook-id/); assert.doesNotMatch(serialized, /127\.0\.0\.1:9/); assert.doesNotMatch(serialized, /announcements\/?"?$/); assert.equal(service.status().configured, true); assert.equal(service.status().openIncidents.length, 1); assert.equal(service.status().openIncidents[0].conditionKey, "storage"); }); test("storage health transitions drive exactly one incident pair", async (t) => { const harness = createHarness(); const service = harness.build(); t.after(() => harness.database.close()); service.start(); observeStorageHealth(service, { status: "degraded", code: "STORAGE_NOT_CHECKED", }); assert.equal(harness.incidents().length, 0, "startup state is not an outage"); for (let observation = 0; observation < 5; observation += 1) { observeStorageHealth(service, { status: "degraded", code: "STORAGE_UNAVAILABLE", checkedAt: "2026-08-16T12:00:00.000Z", }); } assert.equal(harness.outbox().length, 1); observeStorageHealth(service, { status: "healthy", checkedAt: "2026-08-16T12:30:00.000Z" }); assert.equal(harness.outbox().length, 2); assert.equal(harness.outbox()[1].transition, "recovery"); }); test("no playable slides is reported even while a static override fills the screen", async (t) => { const harness = createHarness(); const service = harness.build(); t.after(() => harness.database.close()); service.start(); observePlayback(service, { activePlaylistId: "playlist-a", normalTarget: { id: "entry" }, mode: "on", }); assert.equal(harness.incidents().length, 0); observePlayback(service, { activePlaylistId: "playlist-a", normalTarget: null, staticTarget: { assetId: "static-asset" }, mode: "on", }); const open = harness.incidents(); assert.equal(open.length, 1); assert.equal(open[0].event, "playback.no_playable_slides"); assert.equal(JSON.parse(open[0].context_json).staticActive, true); observePlayback(service, { activePlaylistId: "playlist-a", normalTarget: null, staticTarget: { assetId: "static-asset" }, mode: "on", }); assert.equal(harness.outbox().length, 1, "a continuing condition never re-notifies"); observePlayback(service, { activePlaylistId: "playlist-a", normalTarget: { id: "entry" }, mode: "on", }); assert.equal(harness.incidents()[0].state, "recovered"); assert.equal(harness.outbox().length, 2); }); test("each failed upload opens one bounded warning incident", async (t) => { const harness = createHarness(); const service = harness.build(); t.after(() => harness.database.close()); service.start(); observeAssetProcessingFailure(service, { assetId: "asset-1", displayName: "Bulletin", message: "x".repeat(5_000), }); observeAssetProcessingFailure(service, { assetId: "asset-1", displayName: "Bulletin", message: "again", }); observeAssetProcessingFailure(service, { assetId: "asset-2", displayName: "Notice" }); assert.equal(harness.incidents().length, 2, "one incident per failed asset"); assert.equal(harness.outbox().length, 2); const context = JSON.parse(harness.incidents()[0].context_json); assert.ok( context.processingError.length <= 500, "a processing error is bounded before it is stored", ); assert.equal(harness.incidents()[0].severity, "warning"); });