Implement Phase 4 integrations and display telemetry
Add MQTT playback control, durable Home Assistant incident notifications, and display heartbeat telemetry on top of the verified Phase 3B playback authority. No playback logic is duplicated: every MQTT command and every observed condition goes through the existing playback service, and all integration work happens after the authoritative SQLite commit. MQTT (server/src/mqtt) subscribes the exact required command topics under a configurable prefix that defaults to announcements. Durable commands use QoS 1; next and previous stay at QoS 0 because they are not idempotent and carry no command ID. Retained authoritative state is published only on the separate availability, mode/state, and playback/state topics, never onto a command topic, and only after a mutation commits. A registered last will reports an ungraceful disappearance, a clean shutdown retracts it, and a broker outage degrades status while playback, HTTP, and the display protocol continue untouched. Notifications (server/src/notifications) use the Phase 1 incident and outbox tables. A failing condition opens one incident and queues exactly one notification; continued failure only updates its observation time; recovery queues exactly one recovery; a later failure is a new episode. Delivery retries on a documented capped schedule with a stable notification ID, survives restart, and never blocks playback. With no webhook URL configured, incidents are still recorded but no delivery work is queued. Conditions come from storage health transitions, committed playback state, display heartbeat staleness, and newly failed uploads. Displays (server/src/displays) ingest validated, bounded display:heartbeat telemetry keyed by persistent client ID rather than socket ID. Telemetry is read-only and never touches playback or playlist revisions. Offline is decided by heartbeat staleness, not by a closed socket, and a startup grace period keeps an API restart from manufacturing offline alerts. Adds GET /api/displays and meaningful-change display:status events. Health and status now report redacted storage, MQTT, notification, playback, and display state. External integrations degrade status and never affect readiness, so no integration outage can cause a restart loop. No MQTT password, S3 credential, webhook URL, or webhook secret is exposed. Uses the existing Phase 1 schema unchanged; no migration was required. Co-Authored-By: Claude Opus 5 <[email protected]>
This commit is contained in:
Media
@@ -0,0 +1,376 @@
|
||||
const test = require("node:test");
|
||||
const assert = require("node:assert/strict");
|
||||
const { openDatabase } = require("../src/database");
|
||||
const { runMigrations } = require("../src/database/migrate");
|
||||
const { createDisplayService } = require("../src/displays/service");
|
||||
const { createNotificationService } = require("../src/notifications");
|
||||
|
||||
const BASE_TIME = 2_000_000_000_000;
|
||||
const THRESHOLD = 45_000;
|
||||
|
||||
function createClock(initial = BASE_TIME) {
|
||||
let current = initial;
|
||||
let sequence = 0;
|
||||
const timers = new Set();
|
||||
return {
|
||||
now: () => current,
|
||||
set(value) {
|
||||
current = value;
|
||||
},
|
||||
setTimeout(callback, delay) {
|
||||
const timer = { at: current + delay, callback, cleared: false, id: ++sequence };
|
||||
timers.add(timer);
|
||||
return timer;
|
||||
},
|
||||
clearTimeout(timer) {
|
||||
if (timer) timer.cleared = true;
|
||||
},
|
||||
runTo(value) {
|
||||
current = value;
|
||||
for (let guard = 0; guard < 200; guard += 1) {
|
||||
const timer = [...timers]
|
||||
.filter((candidate) => !candidate.cleared && !candidate.fired && candidate.at <= current)
|
||||
.sort((left, right) => left.at - right.at || left.id - right.id)[0];
|
||||
if (!timer) return;
|
||||
timer.fired = true;
|
||||
timer.callback();
|
||||
}
|
||||
throw new Error("timer loop did not settle");
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
function heartbeatPayload(overrides = {}) {
|
||||
return {
|
||||
protocolVersion: 1,
|
||||
clientId: "lobby-display",
|
||||
outputId: "global",
|
||||
activePlaylistId: "playlist-a",
|
||||
playlistRevision: 12,
|
||||
playbackRevision: 42,
|
||||
currentEntryId: "entry-a",
|
||||
currentAssetId: "asset-a",
|
||||
cachedAssetCount: 10,
|
||||
playableAssetCount: 10,
|
||||
mostRecentError: null,
|
||||
...overrides,
|
||||
};
|
||||
}
|
||||
|
||||
function createHarness({ start = true } = {}) {
|
||||
const database = openDatabase({ path: ":memory:" });
|
||||
runMigrations(database);
|
||||
const clock = createClock();
|
||||
const events = [];
|
||||
let idCounter = 0;
|
||||
const notifications = createNotificationService({
|
||||
database,
|
||||
webhookUrl: "http://127.0.0.1:9/hook",
|
||||
now: clock.now,
|
||||
setTimeoutFn: clock.setTimeout,
|
||||
clearTimeoutFn: clock.clearTimeout,
|
||||
createId: () => `note-${++idCounter}`,
|
||||
fetchFn: () => Promise.resolve({ status: 200 }),
|
||||
logger: { warn() {} },
|
||||
});
|
||||
|
||||
function build() {
|
||||
const service = createDisplayService({
|
||||
database,
|
||||
notifications,
|
||||
now: clock.now,
|
||||
setTimeoutFn: clock.setTimeout,
|
||||
clearTimeoutFn: clock.clearTimeout,
|
||||
offlineThresholdMs: THRESHOLD,
|
||||
onStatusChanged: (event) => events.push(event),
|
||||
logger: { warn() {} },
|
||||
});
|
||||
if (start) service.start();
|
||||
return service;
|
||||
}
|
||||
|
||||
return {
|
||||
build,
|
||||
clock,
|
||||
database,
|
||||
events,
|
||||
notifications,
|
||||
service: build(),
|
||||
outbox: () =>
|
||||
database.prepare("SELECT * FROM notification_outbox ORDER BY created_at, id").all(),
|
||||
incidents: () =>
|
||||
database.prepare("SELECT * FROM notification_incidents ORDER BY opened_at").all(),
|
||||
rows: () => database.prepare("SELECT * FROM display_clients").all(),
|
||||
};
|
||||
}
|
||||
|
||||
test("a first heartbeat creates the display and later heartbeats update the same record", (t) => {
|
||||
const harness = createHarness();
|
||||
t.after(() => harness.database.close());
|
||||
|
||||
const first = harness.service.heartbeat(heartbeatPayload());
|
||||
assert.equal(first.status, "online");
|
||||
assert.equal(first.clientId, "lobby-display");
|
||||
assert.equal(harness.rows().length, 1);
|
||||
assert.equal(harness.events.at(-1).reason, "first_seen");
|
||||
|
||||
harness.clock.set(BASE_TIME + 12_000);
|
||||
const second = harness.service.heartbeat(
|
||||
heartbeatPayload({ playbackRevision: 43, cachedAssetCount: 11 }),
|
||||
);
|
||||
assert.equal(harness.rows().length, 1, "the persistent client ID is the display identity");
|
||||
assert.equal(second.playbackRevision, 43);
|
||||
assert.equal(second.cachedAssetCount, 11);
|
||||
assert.equal(
|
||||
new Date(second.firstSeenAt).getTime(),
|
||||
BASE_TIME,
|
||||
"first_seen_at is preserved across heartbeats",
|
||||
);
|
||||
assert.equal(new Date(second.lastSeenAt).getTime(), BASE_TIME + 12_000);
|
||||
assert.equal(
|
||||
harness.events.filter((event) => event.reason === "first_seen").length,
|
||||
1,
|
||||
"a steady heartbeat creates no realtime churn",
|
||||
);
|
||||
});
|
||||
|
||||
test("the same display keeps its identity across a new Socket.IO connection", (t) => {
|
||||
const harness = createHarness();
|
||||
t.after(() => harness.database.close());
|
||||
|
||||
harness.service.heartbeat(heartbeatPayload());
|
||||
// A browser reconnect produces a new socket but the same persisted client ID.
|
||||
harness.clock.set(BASE_TIME + 20_000);
|
||||
harness.service.heartbeat(heartbeatPayload());
|
||||
|
||||
assert.equal(harness.rows().length, 1);
|
||||
assert.equal(harness.service.list().length, 1);
|
||||
assert.equal(harness.service.list()[0].status, "online");
|
||||
});
|
||||
|
||||
test("malformed telemetry is rejected safely and never creates a display", (t) => {
|
||||
const harness = createHarness();
|
||||
t.after(() => harness.database.close());
|
||||
|
||||
const invalid = [
|
||||
null,
|
||||
"string",
|
||||
[],
|
||||
{},
|
||||
heartbeatPayload({ protocolVersion: 2 }),
|
||||
heartbeatPayload({ protocolVersion: undefined }),
|
||||
heartbeatPayload({ clientId: "" }),
|
||||
heartbeatPayload({ clientId: " " }),
|
||||
heartbeatPayload({ clientId: 42 }),
|
||||
heartbeatPayload({ clientId: "x".repeat(300) }),
|
||||
heartbeatPayload({ cachedAssetCount: -1 }),
|
||||
heartbeatPayload({ cachedAssetCount: 1.5 }),
|
||||
heartbeatPayload({ playbackRevision: -5 }),
|
||||
heartbeatPayload({ mostRecentError: { nested: true } }),
|
||||
];
|
||||
for (const payload of invalid) {
|
||||
assert.throws(() => harness.service.heartbeat(payload), TypeError);
|
||||
}
|
||||
assert.equal(harness.rows().length, 0);
|
||||
});
|
||||
|
||||
test("an oversized error string is bounded before it is stored", (t) => {
|
||||
const harness = createHarness();
|
||||
t.after(() => harness.database.close());
|
||||
|
||||
const result = harness.service.heartbeat(
|
||||
heartbeatPayload({ mostRecentError: "E".repeat(5_000_000) }),
|
||||
);
|
||||
|
||||
assert.ok(result.mostRecentError.length <= 500, "a client cannot store megabytes");
|
||||
assert.ok(harness.rows()[0].last_error.length <= 500);
|
||||
});
|
||||
|
||||
test("heartbeats never change playback or playlist revisions", (t) => {
|
||||
const harness = createHarness();
|
||||
t.after(() => harness.database.close());
|
||||
const playbackBefore = harness.database
|
||||
.prepare("SELECT revision, updated_at FROM playback_state WHERE output_id = 'global'")
|
||||
.get();
|
||||
|
||||
for (let beat = 0; beat < 5; beat += 1) {
|
||||
harness.clock.set(BASE_TIME + beat * 10_000);
|
||||
harness.service.heartbeat(heartbeatPayload({ playbackRevision: 99 }));
|
||||
}
|
||||
|
||||
assert.deepEqual(
|
||||
harness.database
|
||||
.prepare("SELECT revision, updated_at FROM playback_state WHERE output_id = 'global'")
|
||||
.get(),
|
||||
playbackBefore,
|
||||
"telemetry is read-only reporting",
|
||||
);
|
||||
});
|
||||
|
||||
test("one missed heartbeat does not mark a display offline before the threshold", (t) => {
|
||||
const harness = createHarness();
|
||||
t.after(() => harness.database.close());
|
||||
harness.service.heartbeat(heartbeatPayload());
|
||||
|
||||
// Two heartbeat intervals have passed but the 45s threshold has not.
|
||||
harness.clock.runTo(BASE_TIME + 30_000);
|
||||
assert.equal(harness.service.list()[0].status, "online");
|
||||
assert.equal(harness.incidents().length, 0);
|
||||
|
||||
harness.clock.runTo(BASE_TIME + THRESHOLD - 1);
|
||||
assert.equal(harness.service.list()[0].status, "online");
|
||||
assert.equal(harness.incidents().length, 0);
|
||||
});
|
||||
|
||||
test("crossing the threshold opens exactly one offline incident that never repeats", (t) => {
|
||||
const harness = createHarness();
|
||||
t.after(() => harness.database.close());
|
||||
harness.service.heartbeat(heartbeatPayload());
|
||||
|
||||
harness.clock.runTo(BASE_TIME + THRESHOLD);
|
||||
assert.equal(harness.service.list()[0].status, "offline");
|
||||
const incidents = harness.incidents();
|
||||
assert.equal(incidents.length, 1);
|
||||
assert.equal(incidents[0].event, "display.offline");
|
||||
assert.equal(incidents[0].condition_key, "display:lobby-display");
|
||||
assert.equal(JSON.parse(incidents[0].context_json).clientId, "lobby-display");
|
||||
assert.equal(harness.outbox().length, 1);
|
||||
assert.equal(harness.events.at(-1).reason, "offline");
|
||||
|
||||
// Staying offline for a long time must not produce more notifications.
|
||||
harness.clock.runTo(BASE_TIME + 30 * 60_000);
|
||||
assert.equal(harness.incidents().length, 1);
|
||||
assert.equal(harness.outbox().length, 1);
|
||||
assert.equal(
|
||||
harness.events.filter((event) => event.reason === "offline").length,
|
||||
1,
|
||||
);
|
||||
});
|
||||
|
||||
test("a returning heartbeat recovers the display exactly once", (t) => {
|
||||
const harness = createHarness();
|
||||
t.after(() => harness.database.close());
|
||||
harness.service.heartbeat(heartbeatPayload());
|
||||
harness.clock.runTo(BASE_TIME + THRESHOLD);
|
||||
assert.equal(harness.outbox().length, 1);
|
||||
|
||||
harness.clock.set(BASE_TIME + 60_000);
|
||||
const recovered = harness.service.heartbeat(heartbeatPayload());
|
||||
assert.equal(recovered.status, "online");
|
||||
assert.equal(harness.incidents()[0].state, "recovered");
|
||||
assert.equal(harness.outbox().length, 2);
|
||||
assert.equal(harness.outbox()[1].transition, "recovery");
|
||||
assert.equal(harness.events.at(-1).reason, "recovered");
|
||||
|
||||
// Continued healthy heartbeats add nothing further.
|
||||
harness.clock.set(BASE_TIME + 70_000);
|
||||
harness.service.heartbeat(heartbeatPayload());
|
||||
assert.equal(harness.outbox().length, 2);
|
||||
assert.equal(
|
||||
harness.events.filter((event) => event.reason === "recovered").length,
|
||||
1,
|
||||
);
|
||||
|
||||
// A later disappearance is a new episode with its own single notification.
|
||||
harness.clock.runTo(BASE_TIME + 70_000 + THRESHOLD);
|
||||
assert.equal(harness.incidents().length, 2);
|
||||
assert.equal(harness.outbox().length, 3);
|
||||
});
|
||||
|
||||
test("a restart preserves known displays and cannot manufacture offline alerts", (t) => {
|
||||
const harness = createHarness();
|
||||
t.after(() => harness.database.close());
|
||||
harness.service.heartbeat(heartbeatPayload());
|
||||
harness.service.stop();
|
||||
|
||||
// The API was down for well over the threshold. A healthy display simply
|
||||
// has not had the chance to report yet, so startup stays silent until a
|
||||
// full threshold has elapsed since the restart.
|
||||
harness.clock.set(BASE_TIME + 10 * 60_000);
|
||||
const restarted = harness.build();
|
||||
assert.equal(harness.rows().length, 1, "display history survives a restart");
|
||||
assert.equal(restarted.list()[0].status, "offline", "status still reflects staleness");
|
||||
assert.equal(harness.incidents().length, 0, "no notification storm after a restart");
|
||||
|
||||
harness.clock.runTo(BASE_TIME + 10 * 60_000 + 20_000);
|
||||
assert.equal(harness.incidents().length, 0, "still inside the startup grace period");
|
||||
|
||||
restarted.heartbeat(heartbeatPayload());
|
||||
assert.equal(restarted.list()[0].status, "online");
|
||||
assert.equal(harness.incidents().length, 0, "a display that comes back was never offline");
|
||||
});
|
||||
|
||||
test("a display still missing after the startup grace period is reported once", (t) => {
|
||||
const harness = createHarness();
|
||||
t.after(() => harness.database.close());
|
||||
harness.service.heartbeat(heartbeatPayload());
|
||||
harness.service.stop();
|
||||
|
||||
harness.clock.set(BASE_TIME + 10 * 60_000);
|
||||
harness.build();
|
||||
harness.clock.runTo(BASE_TIME + 10 * 60_000 + THRESHOLD);
|
||||
|
||||
assert.equal(harness.incidents().length, 1);
|
||||
assert.equal(harness.incidents()[0].event, "display.offline");
|
||||
assert.equal(harness.outbox().length, 1);
|
||||
});
|
||||
|
||||
test("an already-open offline incident is not duplicated by a restart", (t) => {
|
||||
const harness = createHarness();
|
||||
t.after(() => harness.database.close());
|
||||
harness.service.heartbeat(heartbeatPayload());
|
||||
harness.clock.runTo(BASE_TIME + THRESHOLD);
|
||||
assert.equal(harness.incidents().length, 1);
|
||||
harness.service.stop();
|
||||
|
||||
harness.clock.set(BASE_TIME + 20 * 60_000);
|
||||
harness.build();
|
||||
harness.clock.runTo(BASE_TIME + 20 * 60_000 + THRESHOLD + 1_000);
|
||||
|
||||
assert.equal(harness.incidents().length, 1, "the same episode stays open");
|
||||
assert.equal(harness.outbox().length, 1);
|
||||
});
|
||||
|
||||
test("a changed error is surfaced as a meaningful status update", (t) => {
|
||||
const harness = createHarness();
|
||||
t.after(() => harness.database.close());
|
||||
harness.service.heartbeat(heartbeatPayload());
|
||||
harness.events.length = 0;
|
||||
|
||||
harness.clock.set(BASE_TIME + 10_000);
|
||||
harness.service.heartbeat(heartbeatPayload({ mostRecentError: "decode failed for asset-a" }));
|
||||
assert.equal(harness.events.length, 1);
|
||||
assert.equal(harness.events[0].reason, "error_changed");
|
||||
assert.equal(harness.events[0].mostRecentError, "decode failed for asset-a");
|
||||
|
||||
// The same error repeating is not a new event.
|
||||
harness.clock.set(BASE_TIME + 20_000);
|
||||
harness.service.heartbeat(heartbeatPayload({ mostRecentError: "decode failed for asset-a" }));
|
||||
assert.equal(harness.events.length, 1);
|
||||
|
||||
// An arbitrary browser string is telemetry only, never a notification.
|
||||
assert.equal(harness.incidents().length, 0);
|
||||
assert.equal(harness.outbox().length, 0);
|
||||
});
|
||||
|
||||
test("display status derives from heartbeat freshness for every known client", (t) => {
|
||||
const harness = createHarness();
|
||||
t.after(() => harness.database.close());
|
||||
|
||||
harness.service.heartbeat(heartbeatPayload({ clientId: "lobby" }));
|
||||
harness.clock.set(BASE_TIME + 40_000);
|
||||
harness.service.heartbeat(heartbeatPayload({ clientId: "narthex" }));
|
||||
harness.clock.set(BASE_TIME + 50_000);
|
||||
|
||||
const displays = Object.fromEntries(
|
||||
harness.service.list().map((display) => [display.clientId, display]),
|
||||
);
|
||||
assert.equal(displays.lobby.status, "offline");
|
||||
assert.equal(displays.narthex.status, "online");
|
||||
assert.equal(harness.service.status().online, 1);
|
||||
assert.equal(harness.service.status().offline, 1);
|
||||
assert.equal(harness.service.status().known, 2);
|
||||
|
||||
const serialized = JSON.stringify(harness.service.list());
|
||||
assert.doesNotMatch(serialized, /socket|sid|transport/i);
|
||||
});
|
||||
@@ -0,0 +1,430 @@
|
||||
const test = require("node:test");
|
||||
const assert = require("node:assert/strict");
|
||||
const { loadConfig } = require("../src/config");
|
||||
const { createRuntime } = require("../src/runtime");
|
||||
|
||||
const silentLogger = { info() {}, warn() {}, error() {} };
|
||||
|
||||
function testConfig(overrides = {}) {
|
||||
return loadConfig({
|
||||
NODE_ENV: "test",
|
||||
HOST: "127.0.0.1",
|
||||
PORT: "0",
|
||||
DATABASE_PATH: ":memory:",
|
||||
APP_TIMEZONE: "America/Chicago",
|
||||
SHUTDOWN_TIMEOUT_MS: "1000",
|
||||
S3_ENDPOINT: "http://seaweedfs:8333",
|
||||
S3_BUCKET: "announcements-test",
|
||||
MQTT_URL: "mqtt://broker:1883",
|
||||
MQTT_PASSWORD: "mqtt-password-that-must-not-leak",
|
||||
HA_WEBHOOK_URL: "http://home-assistant:8123/api/webhook/private-id",
|
||||
...overrides,
|
||||
});
|
||||
}
|
||||
|
||||
function testStorage(health = { status: "healthy", checkedAt: null, lastSuccessAt: null }) {
|
||||
return {
|
||||
async start() {},
|
||||
stop() {},
|
||||
health: () => health,
|
||||
};
|
||||
}
|
||||
|
||||
function createFakeBroker({ failPublish = false } = {}) {
|
||||
const listeners = new Map();
|
||||
const client = {
|
||||
published: [],
|
||||
subscriptions: [],
|
||||
on(event, handler) {
|
||||
listeners.set(event, [...(listeners.get(event) || []), handler]);
|
||||
return client;
|
||||
},
|
||||
emit(event, ...args) {
|
||||
for (const handler of listeners.get(event) || []) handler(...args);
|
||||
},
|
||||
publish(topic, payload, options, callback) {
|
||||
if (failPublish) throw new Error("broker is gone");
|
||||
client.published.push({ topic, payload: String(payload) });
|
||||
callback?.(null);
|
||||
},
|
||||
subscribe(topic, options, callback) {
|
||||
client.subscriptions.push(topic);
|
||||
callback?.(null);
|
||||
},
|
||||
unsubscribe() {},
|
||||
end(force, options, callback) {
|
||||
callback?.();
|
||||
},
|
||||
deliver(topic, payload) {
|
||||
client.emit("message", topic, Buffer.from(payload));
|
||||
},
|
||||
};
|
||||
return client;
|
||||
}
|
||||
|
||||
async function jsonRequest(baseUrl, route, { body, ...init } = {}) {
|
||||
const response = await fetch(`${baseUrl}${route}`, {
|
||||
...init,
|
||||
...(body === undefined
|
||||
? {}
|
||||
: {
|
||||
headers: { "content-type": "application/json", ...init.headers },
|
||||
body: JSON.stringify(body),
|
||||
}),
|
||||
});
|
||||
return { response, body: await response.json() };
|
||||
}
|
||||
|
||||
function splitEnginePackets(payload) {
|
||||
return payload.split("\x1e").filter(Boolean);
|
||||
}
|
||||
|
||||
async function connectSocket(baseUrl) {
|
||||
let requestNumber = 0;
|
||||
const endpoint = (sid) => {
|
||||
const parameters = new URLSearchParams({
|
||||
EIO: "4",
|
||||
transport: "polling",
|
||||
t: String(requestNumber++),
|
||||
});
|
||||
if (sid) parameters.set("sid", sid);
|
||||
return `${baseUrl}/socket.io/?${parameters}`;
|
||||
};
|
||||
const opening = splitEnginePackets(await (await fetch(endpoint())).text());
|
||||
const { sid } = JSON.parse(opening.find((packet) => packet.startsWith("0")).slice(1));
|
||||
await (
|
||||
await fetch(endpoint(sid), {
|
||||
method: "POST",
|
||||
headers: { "content-type": "text/plain;charset=UTF-8" },
|
||||
body: "40",
|
||||
})
|
||||
).text();
|
||||
await (await fetch(endpoint(sid))).text();
|
||||
return {
|
||||
async send(event, payload) {
|
||||
const response = await fetch(endpoint(sid), {
|
||||
method: "POST",
|
||||
headers: { "content-type": "text/plain;charset=UTF-8" },
|
||||
body: `42${JSON.stringify([event, payload])}`,
|
||||
});
|
||||
await response.text();
|
||||
},
|
||||
async poll() {
|
||||
return (await fetch(endpoint(sid), { signal: AbortSignal.timeout(3_000) })).text();
|
||||
},
|
||||
async close() {
|
||||
await (
|
||||
await fetch(endpoint(sid), {
|
||||
method: "POST",
|
||||
headers: { "content-type": "text/plain;charset=UTF-8" },
|
||||
body: "41",
|
||||
})
|
||||
).text();
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
function socketEvents(payload) {
|
||||
return splitEnginePackets(payload)
|
||||
.filter((packet) => packet.startsWith("42"))
|
||||
.map((packet) => JSON.parse(packet.slice(2)));
|
||||
}
|
||||
|
||||
async function startRuntime(t, { broker, config, storage, notificationFetch } = {}) {
|
||||
const runtime = createRuntime({
|
||||
config: config || testConfig(),
|
||||
logger: silentLogger,
|
||||
storage: storage || testStorage(),
|
||||
...(broker === null ? {} : { mqttConnect: () => broker }),
|
||||
...(notificationFetch ? { notificationFetch } : {}),
|
||||
});
|
||||
await runtime.start();
|
||||
t.after(() => runtime.stop("test cleanup"));
|
||||
broker?.emit("connect");
|
||||
return { runtime, baseUrl: `http://127.0.0.1:${runtime.address.port}` };
|
||||
}
|
||||
|
||||
async function seedPlaylist(runtime, baseUrl) {
|
||||
const timestamp = Date.now();
|
||||
runtime.database
|
||||
.prepare(
|
||||
`INSERT INTO assets (
|
||||
id, source_sha256, display_name, original_filename, status,
|
||||
display_object_key, display_sha256, display_content_type,
|
||||
display_byte_size, display_width, display_height, created_at, updated_at
|
||||
) VALUES (?, ?, ?, ?, 'ready', 'private/display/a.webp', ?, 'image/webp', 10, 1920, 1080, ?, ?)`,
|
||||
)
|
||||
.run("asset-one", "a".repeat(64), "One", "one.png", "b".repeat(64), timestamp, timestamp);
|
||||
runtime.database
|
||||
.prepare(
|
||||
`INSERT INTO assets (
|
||||
id, source_sha256, display_name, original_filename, status,
|
||||
display_object_key, display_sha256, display_content_type,
|
||||
display_byte_size, display_width, display_height, created_at, updated_at
|
||||
) VALUES (?, ?, ?, ?, 'ready', 'private/display/b.webp', ?, 'image/webp', 10, 1920, 1080, ?, ?)`,
|
||||
)
|
||||
.run("asset-two", "c".repeat(64), "Two", "two.png", "d".repeat(64), timestamp, timestamp);
|
||||
const playlist = await jsonRequest(baseUrl, "/api/playlists", {
|
||||
method: "POST",
|
||||
body: { name: "Integration" },
|
||||
});
|
||||
for (const assetId of ["asset-one", "asset-two"]) {
|
||||
await jsonRequest(baseUrl, `/api/playlists/${playlist.body.id}/entries`, {
|
||||
method: "POST",
|
||||
body: { assetId, durationOverrideMs: 60_000 },
|
||||
});
|
||||
}
|
||||
await jsonRequest(baseUrl, `/api/playlists/${playlist.body.id}/activate`, {
|
||||
method: "POST",
|
||||
});
|
||||
return playlist.body;
|
||||
}
|
||||
|
||||
test("an MQTT command and its REST equivalent reach the same playback authority", async (t) => {
|
||||
const broker = createFakeBroker();
|
||||
const { runtime, baseUrl } = await startRuntime(t, { broker });
|
||||
await seedPlaylist(runtime, baseUrl);
|
||||
|
||||
const start = (await jsonRequest(baseUrl, "/api/playback")).body;
|
||||
assert.equal(start.mode, "on");
|
||||
|
||||
broker.deliver("announcements/mode", "off");
|
||||
const afterMqtt = (await jsonRequest(baseUrl, "/api/playback")).body;
|
||||
assert.equal(afterMqtt.mode, "off", "MQTT committed through the playback service");
|
||||
assert.equal(afterMqtt.playbackRevision, start.playbackRevision + 1);
|
||||
|
||||
await jsonRequest(baseUrl, "/api/playback/mode", { method: "PUT", body: { mode: "on" } });
|
||||
const afterRest = (await jsonRequest(baseUrl, "/api/playback")).body;
|
||||
assert.equal(afterRest.mode, "on");
|
||||
|
||||
// Both transports move the same cursor with identical semantics.
|
||||
const beforeNext = (await jsonRequest(baseUrl, "/api/playback")).body;
|
||||
broker.deliver("announcements/next", "");
|
||||
const mqttNext = (await jsonRequest(baseUrl, "/api/playback")).body;
|
||||
assert.notEqual(mqttNext.currentEntryId, beforeNext.currentEntryId);
|
||||
const restNext = (await jsonRequest(baseUrl, "/api/playback/next", { method: "POST" })).body;
|
||||
assert.equal(restNext.currentEntryId, beforeNext.currentEntryId, "next wrapped back");
|
||||
|
||||
broker.deliver("announcements/pause", "");
|
||||
assert.equal((await jsonRequest(baseUrl, "/api/playback")).body.transportState, "paused");
|
||||
broker.deliver("announcements/resume", "");
|
||||
assert.equal((await jsonRequest(baseUrl, "/api/playback")).body.transportState, "playing");
|
||||
broker.deliver("announcements/static", "asset-one");
|
||||
assert.equal(
|
||||
(await jsonRequest(baseUrl, "/api/playback")).body.transportState,
|
||||
"playing",
|
||||
"a non-UUID static payload is rejected before it reaches playback",
|
||||
);
|
||||
});
|
||||
|
||||
test("MQTT publishes retained authoritative state only after a committed change", async (t) => {
|
||||
const broker = createFakeBroker();
|
||||
const { runtime, baseUrl } = await startRuntime(t, { broker });
|
||||
await seedPlaylist(runtime, baseUrl);
|
||||
|
||||
assert.equal(
|
||||
broker.published.find((message) => message.topic === "announcements/availability").payload,
|
||||
"online",
|
||||
);
|
||||
broker.published.length = 0;
|
||||
|
||||
await jsonRequest(baseUrl, "/api/playback/mode", { method: "PUT", body: { mode: "off" } });
|
||||
assert.equal(
|
||||
broker.published.filter((message) => message.topic === "announcements/mode/state").at(-1)
|
||||
.payload,
|
||||
"off",
|
||||
);
|
||||
|
||||
// A read and a no-op command commit nothing, so nothing is republished.
|
||||
broker.published.length = 0;
|
||||
await jsonRequest(baseUrl, "/api/playback");
|
||||
await jsonRequest(baseUrl, "/api/playback/mode", { method: "PUT", body: { mode: "off" } });
|
||||
assert.equal(broker.published.length, 0);
|
||||
|
||||
const commandTopics = new Set(runtime.mqttService.commandTopics);
|
||||
assert.ok(
|
||||
broker.published.every((message) => !commandTopics.has(message.topic)),
|
||||
"state is never echoed onto a command topic",
|
||||
);
|
||||
});
|
||||
|
||||
test("a broker outage degrades status while playback and HTTP continue", async (t) => {
|
||||
const broker = createFakeBroker();
|
||||
const { runtime, baseUrl } = await startRuntime(t, { broker });
|
||||
await seedPlaylist(runtime, baseUrl);
|
||||
|
||||
broker.emit("error", new Error("connect ECONNREFUSED"));
|
||||
broker.emit("close");
|
||||
|
||||
const health = await jsonRequest(baseUrl, "/api/health");
|
||||
assert.equal(health.response.status, 200, "MQTT is never readiness-critical");
|
||||
assert.equal(health.body.services.mqtt.status, "disconnected");
|
||||
assert.equal(health.body.status, "degraded");
|
||||
assert.equal((await jsonRequest(baseUrl, "/health/ready")).response.status, 200);
|
||||
|
||||
// Playback keeps working through REST while the broker is unreachable.
|
||||
const advanced = await jsonRequest(baseUrl, "/api/playback/next", { method: "POST" });
|
||||
assert.equal(advanced.response.status, 200);
|
||||
assert.ok(advanced.body.currentEntryId);
|
||||
assert.equal(runtime.playbackService.getSnapshot().transportState, "playing");
|
||||
});
|
||||
|
||||
test("an MQTT publish failure after commit never rolls back playback", async (t) => {
|
||||
const broker = createFakeBroker({ failPublish: true });
|
||||
const { runtime, baseUrl } = await startRuntime(t, { broker });
|
||||
await seedPlaylist(runtime, baseUrl);
|
||||
|
||||
const before = (await jsonRequest(baseUrl, "/api/playback")).body;
|
||||
const changed = await jsonRequest(baseUrl, "/api/playback/mode", {
|
||||
method: "PUT",
|
||||
body: { mode: "off" },
|
||||
});
|
||||
|
||||
assert.equal(changed.response.status, 200);
|
||||
assert.equal(changed.body.mode, "off");
|
||||
assert.equal(changed.body.playbackRevision, before.playbackRevision + 1);
|
||||
assert.equal(
|
||||
runtime.database
|
||||
.prepare("SELECT mode FROM playback_state WHERE output_id = 'global'")
|
||||
.get().mode,
|
||||
"off",
|
||||
"the SQLite commit is authoritative regardless of MQTT",
|
||||
);
|
||||
});
|
||||
|
||||
test("an unreachable Home Assistant never blocks playback or the API", async (t) => {
|
||||
const broker = createFakeBroker();
|
||||
const { runtime, baseUrl } = await startRuntime(t, {
|
||||
broker,
|
||||
notificationFetch: () => Promise.reject(new Error("ECONNREFUSED")),
|
||||
storage: testStorage({
|
||||
status: "degraded",
|
||||
code: "STORAGE_UNAVAILABLE",
|
||||
checkedAt: null,
|
||||
lastSuccessAt: null,
|
||||
}),
|
||||
});
|
||||
await seedPlaylist(runtime, baseUrl);
|
||||
|
||||
runtime.notifications.observeFailure("storage", {
|
||||
event: "storage.unavailable",
|
||||
severity: "critical",
|
||||
message: "Announcement storage is unavailable.",
|
||||
context: {},
|
||||
});
|
||||
await runtime.notifications.deliverDue();
|
||||
|
||||
assert.equal(
|
||||
runtime.database
|
||||
.prepare("SELECT COUNT(*) AS count FROM notification_outbox WHERE delivered_at IS NULL")
|
||||
.get().count,
|
||||
1,
|
||||
"the failed delivery stays pending for retry",
|
||||
);
|
||||
const playback = await jsonRequest(baseUrl, "/api/playback/next", { method: "POST" });
|
||||
assert.equal(playback.response.status, 200);
|
||||
assert.equal((await jsonRequest(baseUrl, "/health/ready")).response.status, 200);
|
||||
});
|
||||
|
||||
test("display heartbeats flow over Socket.IO into the status API without touching playback", async (t) => {
|
||||
const broker = createFakeBroker();
|
||||
const { runtime, baseUrl } = await startRuntime(t, { broker });
|
||||
await seedPlaylist(runtime, baseUrl);
|
||||
const socket = await connectSocket(baseUrl);
|
||||
t.after(() => socket.close().catch(() => {}));
|
||||
|
||||
const before = (await jsonRequest(baseUrl, "/api/playback")).body;
|
||||
const playlistBefore = (await jsonRequest(baseUrl, "/api/playlists")).body;
|
||||
|
||||
await socket.send("display:heartbeat", {
|
||||
protocolVersion: 1,
|
||||
clientId: "lobby-display",
|
||||
outputId: "global",
|
||||
activePlaylistId: before.activePlaylistId,
|
||||
playlistRevision: 1,
|
||||
playbackRevision: before.playbackRevision,
|
||||
currentEntryId: before.currentEntryId,
|
||||
currentAssetId: "asset-one",
|
||||
cachedAssetCount: 2,
|
||||
playableAssetCount: 2,
|
||||
mostRecentError: null,
|
||||
});
|
||||
await socket.send("display:heartbeat", { protocolVersion: 99, clientId: "bogus" });
|
||||
await socket.send("display:heartbeat", "not-an-object");
|
||||
await new Promise((resolve) => setTimeout(resolve, 100));
|
||||
|
||||
const displays = await jsonRequest(baseUrl, "/api/displays");
|
||||
assert.equal(displays.response.status, 200);
|
||||
assert.equal(displays.body.displays.length, 1, "malformed telemetry created no display");
|
||||
const display = displays.body.displays[0];
|
||||
assert.equal(display.clientId, "lobby-display");
|
||||
assert.equal(display.status, "online");
|
||||
assert.equal(display.cachedAssetCount, 2);
|
||||
assert.equal(display.mostRecentError, null);
|
||||
|
||||
assert.equal(
|
||||
(await jsonRequest(baseUrl, "/api/playback")).body.playbackRevision,
|
||||
before.playbackRevision,
|
||||
"telemetry never increments the playback revision",
|
||||
);
|
||||
assert.deepEqual(
|
||||
(await jsonRequest(baseUrl, "/api/playlists")).body,
|
||||
playlistBefore,
|
||||
"telemetry never increments a playlist revision",
|
||||
);
|
||||
|
||||
const health = await jsonRequest(baseUrl, "/api/health");
|
||||
assert.equal(health.body.services.displays.known, 1);
|
||||
assert.equal(health.body.services.displays.online, 1);
|
||||
});
|
||||
|
||||
test("a first display heartbeat is announced once as a display:status event", async (t) => {
|
||||
const broker = createFakeBroker();
|
||||
const { runtime, baseUrl } = await startRuntime(t, { broker });
|
||||
await seedPlaylist(runtime, baseUrl);
|
||||
const socket = await connectSocket(baseUrl);
|
||||
t.after(() => socket.close().catch(() => {}));
|
||||
|
||||
const beat = {
|
||||
protocolVersion: 1,
|
||||
clientId: "narthex-display",
|
||||
outputId: "global",
|
||||
cachedAssetCount: 1,
|
||||
playableAssetCount: 1,
|
||||
};
|
||||
await socket.send("display:heartbeat", beat);
|
||||
await socket.send("display:heartbeat", beat);
|
||||
await socket.send("display:heartbeat", beat);
|
||||
await new Promise((resolve) => setTimeout(resolve, 100));
|
||||
|
||||
const events = socketEvents(await socket.poll()).filter(
|
||||
([name]) => name === "display:status",
|
||||
);
|
||||
assert.equal(events.length, 1, "steady telemetry does not create realtime churn");
|
||||
assert.equal(events[0][1].reason, "first_seen");
|
||||
assert.equal(events[0][1].clientId, "narthex-display");
|
||||
assert.doesNotMatch(JSON.stringify(events[0][1]), /socket|sid/i);
|
||||
});
|
||||
|
||||
test("detailed status reports every integration without leaking any secret", async (t) => {
|
||||
const broker = createFakeBroker();
|
||||
const { baseUrl } = await startRuntime(t, { broker });
|
||||
|
||||
const health = await jsonRequest(baseUrl, "/api/health");
|
||||
const status = await jsonRequest(baseUrl, "/api/status");
|
||||
assert.equal(health.body.services.mqtt.status, "connected");
|
||||
assert.equal(health.body.services.mqtt.configured, true);
|
||||
assert.equal(health.body.services.mqtt.topicPrefix, "announcements");
|
||||
assert.equal(health.body.services.notifications.configured, true);
|
||||
assert.equal(health.body.services.notifications.status, "ok");
|
||||
assert.equal(health.body.services.displays.status, "none_seen");
|
||||
assert.equal(health.body.services.playback.status, "ok");
|
||||
assert.equal(status.response.status, 200);
|
||||
|
||||
const serialized = JSON.stringify(health.body) + JSON.stringify(status.body);
|
||||
assert.doesNotMatch(serialized, /mqtt-password-that-must-not-leak/);
|
||||
assert.doesNotMatch(serialized, /home-assistant:8123/);
|
||||
assert.doesNotMatch(serialized, /private-id/);
|
||||
assert.doesNotMatch(serialized, /webhook.?url/i);
|
||||
assert.doesNotMatch(serialized, /broker:1883/);
|
||||
});
|
||||
@@ -0,0 +1,250 @@
|
||||
const test = require("node:test");
|
||||
const assert = require("node:assert/strict");
|
||||
const mqtt = require("mqtt");
|
||||
const { loadConfig } = require("../src/config");
|
||||
const { createRuntime } = require("../src/runtime");
|
||||
|
||||
// Protocol behaviour that a hand-written fake cannot prove — retained
|
||||
// messages, subscription delivery, and a broker-published last will — is
|
||||
// verified against a real broker. The isolated verification stack sets
|
||||
// MQTT_TEST_URL; without it these checks are skipped rather than faked.
|
||||
const BROKER_URL = process.env.MQTT_TEST_URL;
|
||||
const silentLogger = { info() {}, warn() {}, error() {} };
|
||||
|
||||
function testConfig(overrides = {}) {
|
||||
return loadConfig({
|
||||
NODE_ENV: "test",
|
||||
HOST: "127.0.0.1",
|
||||
PORT: "0",
|
||||
DATABASE_PATH: ":memory:",
|
||||
APP_TIMEZONE: "America/Chicago",
|
||||
SHUTDOWN_TIMEOUT_MS: "1000",
|
||||
S3_ENDPOINT: "http://seaweedfs:8333",
|
||||
S3_BUCKET: "announcements-test",
|
||||
MQTT_URL: BROKER_URL,
|
||||
...overrides,
|
||||
});
|
||||
}
|
||||
|
||||
function testStorage() {
|
||||
return {
|
||||
async start() {},
|
||||
stop() {},
|
||||
health: () => ({ status: "healthy", checkedAt: null, lastSuccessAt: null }),
|
||||
};
|
||||
}
|
||||
|
||||
function connectObserver() {
|
||||
return new Promise((resolve, reject) => {
|
||||
const client = mqtt.connect(BROKER_URL, { reconnectPeriod: 0 });
|
||||
client.once("connect", () => resolve(client));
|
||||
client.once("error", reject);
|
||||
});
|
||||
}
|
||||
|
||||
function collect(client, topic) {
|
||||
const received = [];
|
||||
client.on("message", (messageTopic, payload) => {
|
||||
if (messageTopic === topic) received.push(payload.toString("utf8"));
|
||||
});
|
||||
return received;
|
||||
}
|
||||
|
||||
function waitFor(predicate, { timeoutMs = 5_000, label = "condition" } = {}) {
|
||||
return new Promise((resolve, reject) => {
|
||||
const deadline = Date.now() + timeoutMs;
|
||||
const tick = () => {
|
||||
let value;
|
||||
try {
|
||||
value = predicate();
|
||||
} catch (error) {
|
||||
reject(error);
|
||||
return;
|
||||
}
|
||||
if (value) {
|
||||
resolve(value);
|
||||
return;
|
||||
}
|
||||
if (Date.now() > deadline) {
|
||||
reject(new Error(`Timed out waiting for ${label}`));
|
||||
return;
|
||||
}
|
||||
setTimeout(tick, 25);
|
||||
};
|
||||
tick();
|
||||
});
|
||||
}
|
||||
|
||||
async function subscribe(client, topic, options = {}) {
|
||||
await new Promise((resolve, reject) => {
|
||||
client.subscribe(topic, { qos: 1, ...options }, (error) =>
|
||||
error ? reject(error) : resolve(),
|
||||
);
|
||||
});
|
||||
}
|
||||
|
||||
async function publish(client, topic, payload, options = {}) {
|
||||
await new Promise((resolve, reject) => {
|
||||
client.publish(topic, payload, { qos: 1, ...options }, (error) =>
|
||||
error ? reject(error) : resolve(),
|
||||
);
|
||||
});
|
||||
}
|
||||
|
||||
async function startRuntime(t) {
|
||||
const runtime = createRuntime({
|
||||
config: testConfig(),
|
||||
logger: silentLogger,
|
||||
storage: testStorage(),
|
||||
});
|
||||
await runtime.start();
|
||||
t.after(async () => {
|
||||
try {
|
||||
await runtime.stop("test cleanup");
|
||||
} catch {
|
||||
// A test that already stopped the runtime is fine.
|
||||
}
|
||||
});
|
||||
await waitFor(() => runtime.mqttService.status().connected, {
|
||||
label: "the API to connect to the broker",
|
||||
});
|
||||
return { runtime, baseUrl: `http://127.0.0.1:${runtime.address.port}` };
|
||||
}
|
||||
|
||||
async function seedPlayback(runtime, baseUrl) {
|
||||
const timestamp = Date.now();
|
||||
for (const [id, hash] of [
|
||||
["asset-one", "a"],
|
||||
["asset-two", "c"],
|
||||
]) {
|
||||
runtime.database
|
||||
.prepare(
|
||||
`INSERT INTO assets (
|
||||
id, source_sha256, display_name, original_filename, status,
|
||||
display_object_key, display_sha256, display_content_type,
|
||||
display_byte_size, display_width, display_height, created_at, updated_at
|
||||
) VALUES (?, ?, ?, ?, 'ready', 'private/display/x.webp', ?, 'image/webp', 10, 1920, 1080, ?, ?)`,
|
||||
)
|
||||
.run(id, hash.repeat(64), id, `${id}.png`, hash.repeat(64), timestamp, timestamp);
|
||||
}
|
||||
const created = await fetch(`${baseUrl}/api/playlists`, {
|
||||
method: "POST",
|
||||
headers: { "content-type": "application/json" },
|
||||
body: JSON.stringify({ name: "Broker" }),
|
||||
});
|
||||
const playlist = await created.json();
|
||||
for (const assetId of ["asset-one", "asset-two"]) {
|
||||
await fetch(`${baseUrl}/api/playlists/${playlist.id}/entries`, {
|
||||
method: "POST",
|
||||
headers: { "content-type": "application/json" },
|
||||
body: JSON.stringify({ assetId, durationOverrideMs: 60_000 }),
|
||||
});
|
||||
}
|
||||
await fetch(`${baseUrl}/api/playlists/${playlist.id}/activate`, { method: "POST" });
|
||||
return playlist;
|
||||
}
|
||||
|
||||
const options = BROKER_URL ? {} : { skip: "MQTT_TEST_URL is not configured" };
|
||||
|
||||
test("a real broker delivers commands and retains authoritative state", options, async (t) => {
|
||||
const { runtime, baseUrl } = await startRuntime(t);
|
||||
await seedPlayback(runtime, baseUrl);
|
||||
|
||||
const observer = await connectObserver();
|
||||
t.after(() => observer.endAsync?.() ?? observer.end(true));
|
||||
const availability = collect(observer, "announcements/availability");
|
||||
const modeState = collect(observer, "announcements/mode/state");
|
||||
await subscribe(observer, "announcements/availability");
|
||||
await subscribe(observer, "announcements/mode/state");
|
||||
|
||||
// A late subscriber receives retained state immediately, which is what a
|
||||
// Home Assistant restart depends on.
|
||||
await waitFor(() => availability.includes("online"), {
|
||||
label: "retained availability",
|
||||
});
|
||||
await waitFor(() => modeState.length > 0, { label: "retained mode state" });
|
||||
assert.equal(modeState.at(-1), "on");
|
||||
|
||||
const commander = await connectObserver();
|
||||
t.after(() => commander.endAsync?.() ?? commander.end(true));
|
||||
await publish(commander, "announcements/mode", "off");
|
||||
await waitFor(
|
||||
() =>
|
||||
runtime.database
|
||||
.prepare("SELECT mode FROM playback_state WHERE output_id = 'global'")
|
||||
.get().mode === "off",
|
||||
{ label: "the mode command to commit" },
|
||||
);
|
||||
await waitFor(() => modeState.at(-1) === "off", { label: "retained mode state to follow" });
|
||||
|
||||
const before = runtime.playbackService.getSnapshot();
|
||||
await publish(commander, "announcements/next", "", { qos: 0 });
|
||||
await waitFor(
|
||||
() => runtime.playbackService.getSnapshot().currentEntryId !== before.currentEntryId,
|
||||
{ label: "the next command to advance playback" },
|
||||
);
|
||||
|
||||
await publish(commander, "announcements/pause", "");
|
||||
await waitFor(() => runtime.playbackService.getSnapshot().transportState === "paused", {
|
||||
label: "pause",
|
||||
});
|
||||
await publish(commander, "announcements/resume", "");
|
||||
await waitFor(() => runtime.playbackService.getSnapshot().transportState === "playing", {
|
||||
label: "resume",
|
||||
});
|
||||
|
||||
// An invalid payload is ignored over the real broker exactly as it is
|
||||
// locally: the last committed mode survives untouched.
|
||||
await publish(commander, "announcements/mode", " ON ");
|
||||
await waitFor(() => runtime.playbackService.getSnapshot().mode === "on", {
|
||||
label: "a whitespace-padded mode command to commit",
|
||||
});
|
||||
await publish(commander, "announcements/mode", "dim");
|
||||
await new Promise((resolve) => setTimeout(resolve, 250));
|
||||
assert.equal(runtime.playbackService.getSnapshot().mode, "on");
|
||||
assert.equal(modeState.at(-1), "on", "no fake state was published for a rejected command");
|
||||
});
|
||||
|
||||
test("a clean API shutdown retracts its retained availability", options, async (t) => {
|
||||
const { runtime } = await startRuntime(t);
|
||||
|
||||
const observer = await connectObserver();
|
||||
t.after(() => observer.endAsync?.() ?? observer.end(true));
|
||||
const availability = collect(observer, "announcements/availability");
|
||||
await subscribe(observer, "announcements/availability");
|
||||
await waitFor(() => availability.includes("online"), { label: "online availability" });
|
||||
|
||||
await runtime.stop("clean shutdown test");
|
||||
|
||||
await waitFor(() => availability.at(-1) === "offline", {
|
||||
label: "availability to be retracted on a planned shutdown",
|
||||
});
|
||||
});
|
||||
|
||||
test("an ungraceful disappearance makes the broker publish the last will", options, async (t) => {
|
||||
const observer = await connectObserver();
|
||||
t.after(() => observer.endAsync?.() ?? observer.end(true));
|
||||
const availability = collect(observer, "test/availability");
|
||||
await subscribe(observer, "test/availability");
|
||||
|
||||
// A crashed or power-cut API never sends DISCONNECT. Only a will can report
|
||||
// that, so the will configuration is exercised against a real broker here;
|
||||
// the unit suite proves the service registers exactly these options.
|
||||
const willed = mqtt.connect(BROKER_URL, {
|
||||
reconnectPeriod: 0,
|
||||
will: { topic: "test/availability", payload: "offline", qos: 1, retain: true },
|
||||
});
|
||||
await new Promise((resolve, reject) => {
|
||||
willed.once("connect", resolve);
|
||||
willed.once("error", reject);
|
||||
});
|
||||
await publish(willed, "test/availability", "online", { retain: true });
|
||||
await waitFor(() => availability.includes("online"), { label: "online availability" });
|
||||
|
||||
willed.stream.destroy();
|
||||
|
||||
await waitFor(() => availability.at(-1) === "offline", {
|
||||
label: "the broker to publish the last will",
|
||||
timeoutMs: 10_000,
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,403 @@
|
||||
const test = require("node:test");
|
||||
const assert = require("node:assert/strict");
|
||||
const { COMMANDS, createMqttService } = require("../src/mqtt");
|
||||
|
||||
const PREFIX = "announcements";
|
||||
|
||||
function createFakeClient() {
|
||||
const listeners = new Map();
|
||||
const client = {
|
||||
published: [],
|
||||
subscriptions: [],
|
||||
unsubscribed: [],
|
||||
ended: false,
|
||||
options: null,
|
||||
on(event, handler) {
|
||||
listeners.set(event, [...(listeners.get(event) || []), handler]);
|
||||
return client;
|
||||
},
|
||||
emit(event, ...args) {
|
||||
for (const handler of listeners.get(event) || []) handler(...args);
|
||||
},
|
||||
publish(topic, payload, options, callback) {
|
||||
client.published.push({ topic, payload: String(payload), options });
|
||||
callback?.(null);
|
||||
},
|
||||
subscribe(topic, options, callback) {
|
||||
client.subscriptions.push({ topic, qos: options.qos });
|
||||
callback?.(null);
|
||||
},
|
||||
unsubscribe(topic) {
|
||||
client.unsubscribed.push(topic);
|
||||
},
|
||||
end(force, options, callback) {
|
||||
client.ended = true;
|
||||
callback?.();
|
||||
},
|
||||
};
|
||||
return client;
|
||||
}
|
||||
|
||||
function createPlaybackStub(snapshot = {}) {
|
||||
const calls = [];
|
||||
const base = {
|
||||
mode: "on",
|
||||
transportState: "playing",
|
||||
transport: { rotationPaused: false },
|
||||
activePlaylistId: "playlist-a",
|
||||
currentEntryId: "entry-a",
|
||||
normalTarget: { assetId: "asset-a" },
|
||||
staticTarget: null,
|
||||
playbackRevision: 7,
|
||||
...snapshot,
|
||||
};
|
||||
return {
|
||||
calls,
|
||||
getSnapshot: () => base,
|
||||
setMode(mode) {
|
||||
calls.push(["setMode", mode]);
|
||||
},
|
||||
next() {
|
||||
calls.push(["next"]);
|
||||
},
|
||||
previous() {
|
||||
calls.push(["previous"]);
|
||||
},
|
||||
pause() {
|
||||
calls.push(["pause"]);
|
||||
},
|
||||
resume() {
|
||||
calls.push(["resume"]);
|
||||
},
|
||||
reload() {
|
||||
calls.push(["reload"]);
|
||||
},
|
||||
enterStatic(assetId) {
|
||||
calls.push(["enterStatic", assetId]);
|
||||
if (assetId === "11111111-1111-4111-8111-111111111111") {
|
||||
const error = new Error("Library asset not found");
|
||||
error.code = "ASSET_NOT_FOUND";
|
||||
throw error;
|
||||
}
|
||||
if (assetId === "22222222-2222-4222-8222-222222222222") {
|
||||
const error = new Error("Static playback requires a ready library asset");
|
||||
error.code = "ASSET_NOT_READY";
|
||||
throw error;
|
||||
}
|
||||
if (assetId === "33333333-3333-4333-8333-333333333333") {
|
||||
const error = new Error("Expired library assets cannot be held statically");
|
||||
error.code = "ASSET_EXPIRED";
|
||||
throw error;
|
||||
}
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
function createHarness({ snapshot, url = "mqtt://broker:1883" } = {}) {
|
||||
const client = createFakeClient();
|
||||
const playbackService = createPlaybackStub(snapshot);
|
||||
const service = createMqttService({
|
||||
config: {
|
||||
mqtt: {
|
||||
url,
|
||||
username: "user",
|
||||
password: "must-not-leak",
|
||||
topicPrefix: PREFIX,
|
||||
reconnectPeriodMs: 5_000,
|
||||
connectTimeoutMs: 10_000,
|
||||
},
|
||||
},
|
||||
playbackService,
|
||||
logger: { info() {}, warn() {} },
|
||||
connect: (connectUrl, options) => {
|
||||
client.options = options;
|
||||
client.url = connectUrl;
|
||||
return client;
|
||||
},
|
||||
});
|
||||
return { client, playbackService, service };
|
||||
}
|
||||
|
||||
function connectHarness(options) {
|
||||
const harness = createHarness(options);
|
||||
harness.service.start();
|
||||
harness.client.emit("connect");
|
||||
return harness;
|
||||
}
|
||||
|
||||
test("the default configuration resolves the exact required command topics", () => {
|
||||
const harness = createHarness();
|
||||
assert.deepEqual(harness.service.commandTopics.sort(), [
|
||||
"announcements/mode",
|
||||
"announcements/next",
|
||||
"announcements/pause",
|
||||
"announcements/previous",
|
||||
"announcements/reload",
|
||||
"announcements/resume",
|
||||
"announcements/static",
|
||||
]);
|
||||
assert.equal(harness.service.availabilityTopic, "announcements/availability");
|
||||
assert.equal(harness.service.modeStateTopic, "announcements/mode/state");
|
||||
assert.equal(harness.service.playbackStateTopic, "announcements/playback/state");
|
||||
});
|
||||
|
||||
test("connecting configures a retained last will and publishes availability", () => {
|
||||
const harness = connectHarness();
|
||||
|
||||
assert.deepEqual(harness.client.options.will, {
|
||||
topic: "announcements/availability",
|
||||
payload: "offline",
|
||||
qos: 1,
|
||||
retain: true,
|
||||
});
|
||||
assert.equal(harness.client.options.reconnectPeriod, 5_000);
|
||||
const availability = harness.client.published.find(
|
||||
(message) => message.topic === "announcements/availability",
|
||||
);
|
||||
assert.equal(availability.payload, "online");
|
||||
assert.equal(availability.options.retain, true);
|
||||
assert.equal(availability.options.qos, 1);
|
||||
});
|
||||
|
||||
test("every required command topic is subscribed with its delivery guarantee", () => {
|
||||
const harness = connectHarness();
|
||||
const byTopic = Object.fromEntries(
|
||||
harness.client.subscriptions.map((entry) => [entry.topic, entry.qos]),
|
||||
);
|
||||
|
||||
assert.equal(byTopic["announcements/mode"], 1);
|
||||
assert.equal(byTopic["announcements/pause"], 1);
|
||||
assert.equal(byTopic["announcements/resume"], 1);
|
||||
assert.equal(byTopic["announcements/reload"], 1);
|
||||
assert.equal(byTopic["announcements/static"], 1);
|
||||
// Redelivering a non-idempotent edge command would silently skip a slide.
|
||||
assert.equal(byTopic["announcements/next"], 0);
|
||||
assert.equal(byTopic["announcements/previous"], 0);
|
||||
assert.equal(harness.client.subscriptions.length, COMMANDS.length);
|
||||
});
|
||||
|
||||
test("mode commands delegate to the playback service and tolerate whitespace and case", () => {
|
||||
const harness = connectHarness();
|
||||
|
||||
harness.client.emit("message", "announcements/mode", Buffer.from("off"));
|
||||
harness.client.emit("message", "announcements/mode", Buffer.from(" ON \n"));
|
||||
harness.client.emit("message", "announcements/mode", Buffer.from("OFF"));
|
||||
|
||||
assert.deepEqual(harness.playbackService.calls, [
|
||||
["setMode", "off"],
|
||||
["setMode", "on"],
|
||||
["setMode", "off"],
|
||||
]);
|
||||
});
|
||||
|
||||
test("an invalid mode payload changes nothing and publishes no fake state", () => {
|
||||
const harness = connectHarness();
|
||||
const publishedBefore = harness.client.published.length;
|
||||
|
||||
for (const payload of ["dim", "", " ", "1", "{\"mode\":\"on\"}"]) {
|
||||
harness.client.emit("message", "announcements/mode", Buffer.from(payload));
|
||||
}
|
||||
|
||||
assert.deepEqual(harness.playbackService.calls, []);
|
||||
assert.equal(harness.client.published.length, publishedBefore);
|
||||
assert.equal(harness.service.status().commandCount, 0);
|
||||
});
|
||||
|
||||
test("edge commands invoke the exact playback operation regardless of payload", () => {
|
||||
const harness = connectHarness();
|
||||
|
||||
harness.client.emit("message", "announcements/next", Buffer.from(""));
|
||||
harness.client.emit("message", "announcements/previous", Buffer.from(" "));
|
||||
harness.client.emit("message", "announcements/pause", Buffer.from("press"));
|
||||
harness.client.emit("message", "announcements/resume", Buffer.from("\n"));
|
||||
harness.client.emit("message", "announcements/reload", null);
|
||||
|
||||
assert.deepEqual(harness.playbackService.calls, [
|
||||
["next"],
|
||||
["previous"],
|
||||
["pause"],
|
||||
["resume"],
|
||||
["reload"],
|
||||
]);
|
||||
});
|
||||
|
||||
test("static accepts a LibraryAsset UUID and rejects anything that is not one", () => {
|
||||
const harness = connectHarness();
|
||||
const valid = "44444444-4444-4444-8444-444444444444";
|
||||
|
||||
harness.client.emit("message", "announcements/static", Buffer.from(` ${valid} `));
|
||||
for (const payload of [
|
||||
"not-a-uuid",
|
||||
"https://example.com/image.png",
|
||||
"private/display/asset.webp",
|
||||
"bulletin.png",
|
||||
"../../etc/passwd",
|
||||
"",
|
||||
]) {
|
||||
harness.client.emit("message", "announcements/static", Buffer.from(payload));
|
||||
}
|
||||
|
||||
assert.deepEqual(
|
||||
harness.playbackService.calls,
|
||||
[["enterStatic", valid]],
|
||||
"only a stable asset UUID may reach the playback service",
|
||||
);
|
||||
});
|
||||
|
||||
test("a static command the playback service rejects leaves playback untouched", () => {
|
||||
const harness = connectHarness();
|
||||
const publishedBefore = harness.client.published.length;
|
||||
|
||||
for (const assetId of [
|
||||
"11111111-1111-4111-8111-111111111111",
|
||||
"22222222-2222-4222-8222-222222222222",
|
||||
"33333333-3333-4333-8333-333333333333",
|
||||
]) {
|
||||
harness.client.emit("message", "announcements/static", Buffer.from(assetId));
|
||||
}
|
||||
|
||||
// The rejection is logged; nothing is published and the output never goes
|
||||
// black merely because an operator sent a bad asset ID.
|
||||
assert.equal(harness.client.published.length, publishedBefore);
|
||||
assert.equal(harness.service.status().connected, true);
|
||||
});
|
||||
|
||||
test("authoritative state is retained on state topics and never echoed to commands", () => {
|
||||
const harness = connectHarness();
|
||||
|
||||
harness.service.publishState({
|
||||
mode: "off",
|
||||
transport: { rotationPaused: true },
|
||||
activePlaylistId: "playlist-b",
|
||||
currentEntryId: "entry-b",
|
||||
normalTarget: { assetId: "asset-b" },
|
||||
staticTarget: null,
|
||||
playbackRevision: 12,
|
||||
});
|
||||
|
||||
const modeState = harness.client.published.filter(
|
||||
(message) => message.topic === "announcements/mode/state",
|
||||
);
|
||||
assert.equal(modeState.at(-1).payload, "off");
|
||||
assert.equal(modeState.at(-1).options.retain, true);
|
||||
assert.equal(modeState.at(-1).options.qos, 1);
|
||||
|
||||
const compact = JSON.parse(
|
||||
harness.client.published
|
||||
.filter((message) => message.topic === "announcements/playback/state")
|
||||
.at(-1).payload,
|
||||
);
|
||||
assert.deepEqual(compact, {
|
||||
version: 1,
|
||||
mode: "off",
|
||||
transport: "paused",
|
||||
activePlaylistId: "playlist-b",
|
||||
currentEntryId: "entry-b",
|
||||
currentAssetId: "asset-b",
|
||||
staticAssetId: null,
|
||||
revision: 12,
|
||||
});
|
||||
|
||||
const commandTopics = new Set(harness.service.commandTopics);
|
||||
assert.ok(
|
||||
harness.client.published.every((message) => !commandTopics.has(message.topic)),
|
||||
"the server must never publish onto a command topic",
|
||||
);
|
||||
});
|
||||
|
||||
test("the compact state topic never carries the manifest or private storage details", () => {
|
||||
const harness = connectHarness();
|
||||
harness.service.publishState({
|
||||
mode: "on",
|
||||
transport: { rotationPaused: false },
|
||||
activePlaylistId: "playlist-a",
|
||||
currentEntryId: "entry-a",
|
||||
normalTarget: {
|
||||
assetId: "asset-a",
|
||||
asset: { media: { url: "/media/assets/a/display/abc.webp" } },
|
||||
},
|
||||
staticTarget: null,
|
||||
playbackRevision: 3,
|
||||
manifest: { entries: [{ id: "entry-a" }, { id: "entry-b" }] },
|
||||
});
|
||||
|
||||
const payload = harness.client.published
|
||||
.filter((message) => message.topic === "announcements/playback/state")
|
||||
.at(-1).payload;
|
||||
assert.ok(payload.length < 400, "the compact topic stays small");
|
||||
assert.doesNotMatch(payload, /manifest|media|object_key|bucket|private/i);
|
||||
});
|
||||
|
||||
test("reconnecting resubscribes and republishes availability and authoritative state", () => {
|
||||
const harness = connectHarness();
|
||||
harness.client.published.length = 0;
|
||||
harness.client.subscriptions.length = 0;
|
||||
|
||||
harness.client.emit("close");
|
||||
assert.equal(harness.service.status().connected, false);
|
||||
assert.equal(harness.service.status().status, "disconnected");
|
||||
|
||||
harness.client.emit("connect");
|
||||
|
||||
assert.equal(harness.client.subscriptions.length, COMMANDS.length);
|
||||
assert.equal(
|
||||
harness.client.published.find(
|
||||
(message) => message.topic === "announcements/availability",
|
||||
).payload,
|
||||
"online",
|
||||
);
|
||||
assert.equal(
|
||||
harness.client.published.find(
|
||||
(message) => message.topic === "announcements/mode/state",
|
||||
).payload,
|
||||
"on",
|
||||
);
|
||||
assert.equal(harness.service.status().connectionCount, 2);
|
||||
});
|
||||
|
||||
test("a broker error degrades the integration without throwing or leaking the password", () => {
|
||||
const harness = connectHarness();
|
||||
|
||||
assert.doesNotThrow(() =>
|
||||
harness.client.emit("error", new Error("connect ECONNREFUSED 10.0.0.5:1883")),
|
||||
);
|
||||
|
||||
const status = harness.service.status();
|
||||
assert.equal(status.connected, false);
|
||||
assert.equal(status.status, "disconnected");
|
||||
assert.match(status.lastErrorMessage, /ECONNREFUSED/);
|
||||
assert.doesNotMatch(JSON.stringify(status), /must-not-leak/);
|
||||
});
|
||||
|
||||
test("an unconfigured broker leaves the integration disabled and inert", async () => {
|
||||
const harness = createHarness({ url: null });
|
||||
|
||||
assert.equal(harness.service.start(), null);
|
||||
assert.equal(harness.service.status().status, "disabled");
|
||||
assert.equal(harness.service.status().configured, false);
|
||||
assert.doesNotThrow(() => harness.service.publishState({ mode: "on" }));
|
||||
await harness.service.stop();
|
||||
assert.equal(harness.client.published.length, 0);
|
||||
});
|
||||
|
||||
test("messages on unknown topics are ignored without crashing", () => {
|
||||
const harness = connectHarness();
|
||||
assert.doesNotThrow(() => {
|
||||
harness.client.emit("message", "announcements/unknown", Buffer.from("x"));
|
||||
harness.client.emit("message", "other/mode", Buffer.from("on"));
|
||||
});
|
||||
assert.deepEqual(harness.playbackService.calls, []);
|
||||
});
|
||||
|
||||
test("a clean shutdown retracts availability and unsubscribes every command", async () => {
|
||||
const harness = connectHarness();
|
||||
await harness.service.stop();
|
||||
|
||||
const availability = harness.client.published
|
||||
.filter((message) => message.topic === "announcements/availability")
|
||||
.at(-1);
|
||||
assert.equal(availability.payload, "offline");
|
||||
assert.equal(availability.options.retain, true);
|
||||
assert.equal(harness.client.unsubscribed.length, COMMANDS.length);
|
||||
assert.equal(harness.client.ended, true);
|
||||
});
|
||||
@@ -0,0 +1,383 @@
|
||||
const test = require("node:test");
|
||||
const assert = require("node:assert/strict");
|
||||
const { openDatabase } = require("../src/database");
|
||||
const { runMigrations } = require("../src/database/migrate");
|
||||
const {
|
||||
RETRY_BACKOFF_MS,
|
||||
backoffMs,
|
||||
createNotificationService,
|
||||
} = require("../src/notifications");
|
||||
const {
|
||||
observeAssetProcessingFailure,
|
||||
observePlayback,
|
||||
observeStorageHealth,
|
||||
} = require("../src/notifications/conditions");
|
||||
|
||||
const BASE_TIME = 2_000_000_000_000;
|
||||
const WEBHOOK = "http://127.0.0.1:9/announcements";
|
||||
|
||||
function createClock(initial = BASE_TIME) {
|
||||
let current = initial;
|
||||
let sequence = 0;
|
||||
const timers = new Set();
|
||||
return {
|
||||
now: () => current,
|
||||
set(value) {
|
||||
current = value;
|
||||
},
|
||||
setTimeout(callback, delay) {
|
||||
const timer = { at: current + delay, callback, cleared: false, id: ++sequence };
|
||||
timers.add(timer);
|
||||
return timer;
|
||||
},
|
||||
clearTimeout(timer) {
|
||||
if (timer) timer.cleared = true;
|
||||
},
|
||||
pending() {
|
||||
return [...timers].filter((timer) => !timer.cleared && !timer.fired);
|
||||
},
|
||||
async runTo(value) {
|
||||
current = value;
|
||||
for (let guard = 0; guard < 100; guard += 1) {
|
||||
const timer = this.pending()
|
||||
.filter((candidate) => candidate.at <= current)
|
||||
.sort((left, right) => left.at - right.at || left.id - right.id)[0];
|
||||
if (!timer) return;
|
||||
timer.fired = true;
|
||||
await timer.callback();
|
||||
}
|
||||
throw new Error("timer loop did not settle");
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
function createHarness({ webhookUrl = WEBHOOK, responses = [] } = {}) {
|
||||
const database = openDatabase({ path: ":memory:" });
|
||||
runMigrations(database);
|
||||
const clock = createClock();
|
||||
const calls = [];
|
||||
let idCounter = 0;
|
||||
|
||||
function fetchFn(url, init) {
|
||||
calls.push({ url, body: JSON.parse(init.body), headers: init.headers });
|
||||
const next = responses.shift();
|
||||
if (typeof next === "function") return next();
|
||||
if (next instanceof Error) return Promise.reject(next);
|
||||
return Promise.resolve({ status: next ?? 200 });
|
||||
}
|
||||
|
||||
function build() {
|
||||
return createNotificationService({
|
||||
database,
|
||||
webhookUrl,
|
||||
webhookSecret: "super-secret-webhook-id",
|
||||
now: clock.now,
|
||||
setTimeoutFn: clock.setTimeout,
|
||||
clearTimeoutFn: clock.clearTimeout,
|
||||
createId: () => `note-${++idCounter}`,
|
||||
fetchFn,
|
||||
logger: { warn() {} },
|
||||
});
|
||||
}
|
||||
|
||||
return {
|
||||
build,
|
||||
calls,
|
||||
clock,
|
||||
database,
|
||||
responses,
|
||||
outbox: () =>
|
||||
database.prepare("SELECT * FROM notification_outbox ORDER BY created_at, id").all(),
|
||||
incidents: () =>
|
||||
database
|
||||
.prepare("SELECT * FROM notification_incidents ORDER BY opened_at, episode")
|
||||
.all(),
|
||||
};
|
||||
}
|
||||
|
||||
const STORAGE_FAILURE = {
|
||||
event: "storage.unavailable",
|
||||
severity: "critical",
|
||||
message: "Announcement storage is unavailable.",
|
||||
context: { checkedAt: "now" },
|
||||
};
|
||||
|
||||
test("a persistent failure opens one incident and queues exactly one notification", async (t) => {
|
||||
const harness = createHarness();
|
||||
const service = harness.build();
|
||||
t.after(() => harness.database.close());
|
||||
service.start();
|
||||
|
||||
const first = service.observeFailure("storage", STORAGE_FAILURE);
|
||||
assert.equal(first.opened, true);
|
||||
|
||||
// A long outage is observed many times; none of it may become new work.
|
||||
for (let observation = 0; observation < 25; observation += 1) {
|
||||
harness.clock.set(BASE_TIME + observation * 60_000);
|
||||
assert.equal(service.observeFailure("storage", STORAGE_FAILURE).opened, false);
|
||||
}
|
||||
|
||||
assert.equal(harness.incidents().length, 1);
|
||||
assert.equal(harness.outbox().length, 1);
|
||||
assert.equal(harness.incidents()[0].state, "open");
|
||||
assert.equal(
|
||||
harness.incidents()[0].last_observed_at,
|
||||
BASE_TIME + 24 * 60_000,
|
||||
"a continuing failure only updates its observation time",
|
||||
);
|
||||
});
|
||||
|
||||
test("recovery queues exactly one recovery and a later failure is a new episode", async (t) => {
|
||||
const harness = createHarness();
|
||||
const service = harness.build();
|
||||
t.after(() => harness.database.close());
|
||||
service.start();
|
||||
|
||||
service.observeFailure("storage", STORAGE_FAILURE);
|
||||
harness.clock.set(BASE_TIME + 30 * 60_000);
|
||||
const recovery = service.observeRecovery("storage", {
|
||||
event: "storage.recovered",
|
||||
message: "Announcement storage is reachable again.",
|
||||
});
|
||||
assert.equal(recovery.recovered, true);
|
||||
assert.equal(service.observeRecovery("storage", {}).recovered, false);
|
||||
assert.equal(harness.outbox().length, 2);
|
||||
|
||||
harness.clock.set(BASE_TIME + 60 * 60_000);
|
||||
const second = service.observeFailure("storage", STORAGE_FAILURE);
|
||||
assert.equal(second.opened, true);
|
||||
const incidents = harness.incidents();
|
||||
assert.equal(incidents.length, 2);
|
||||
assert.deepEqual(
|
||||
incidents.map((incident) => incident.episode),
|
||||
[1, 2],
|
||||
"a repeat condition is a new episode, not a reopened incident",
|
||||
);
|
||||
assert.equal(incidents[0].state, "recovered");
|
||||
assert.equal(incidents[1].state, "open");
|
||||
assert.equal(harness.outbox().length, 3);
|
||||
});
|
||||
|
||||
test("a 2xx response marks delivery complete and carries the versioned payload", async (t) => {
|
||||
const harness = createHarness({ responses: [204] });
|
||||
const service = harness.build();
|
||||
t.after(() => harness.database.close());
|
||||
service.start();
|
||||
|
||||
service.observeFailure("storage", STORAGE_FAILURE);
|
||||
await harness.clock.runTo(BASE_TIME);
|
||||
|
||||
assert.equal(harness.calls.length, 1);
|
||||
const payload = harness.calls[0].body;
|
||||
assert.equal(payload.version, 1);
|
||||
assert.equal(payload.event, "storage.unavailable");
|
||||
assert.equal(payload.severity, "critical");
|
||||
assert.equal(payload.state, "failure");
|
||||
assert.equal(payload.message, "Announcement storage is unavailable.");
|
||||
assert.match(payload.timestamp, /^\d{4}-\d{2}-\d{2}T/);
|
||||
assert.ok(payload.notificationId && payload.incidentId);
|
||||
assert.equal(payload.context.conditionKey, "storage");
|
||||
assert.equal(payload.context.episode, 1);
|
||||
assert.equal(harness.outbox()[0].delivered_at, BASE_TIME);
|
||||
assert.equal(service.status().pendingCount, 0);
|
||||
assert.equal(service.status().deliveredCount, 1);
|
||||
});
|
||||
|
||||
test("failed deliveries retry on a capped schedule with a stable notification id", async (t) => {
|
||||
const harness = createHarness({
|
||||
// One attempt per documented backoff step: transport errors and both 4xx
|
||||
// and 5xx responses follow the same retry policy.
|
||||
responses: [
|
||||
new Error("ECONNREFUSED"),
|
||||
500,
|
||||
404,
|
||||
new Error("ECONNREFUSED"),
|
||||
new Error("ECONNREFUSED"),
|
||||
200,
|
||||
],
|
||||
});
|
||||
const service = harness.build();
|
||||
t.after(() => harness.database.close());
|
||||
service.start();
|
||||
|
||||
service.observeFailure("storage", STORAGE_FAILURE);
|
||||
let elapsed = 0;
|
||||
await harness.clock.runTo(BASE_TIME);
|
||||
assert.equal(harness.calls.length, 1);
|
||||
|
||||
// Documented schedule: immediate, then 30s, 1m, 5m, 15m, 30m, capped at 30m.
|
||||
for (const expected of RETRY_BACKOFF_MS) {
|
||||
assert.equal(
|
||||
harness.outbox()[0].next_attempt_at,
|
||||
BASE_TIME + elapsed + expected,
|
||||
`retry must be scheduled ${expected}ms after the failed attempt`,
|
||||
);
|
||||
elapsed += expected;
|
||||
await harness.clock.runTo(BASE_TIME + elapsed);
|
||||
}
|
||||
assert.equal(backoffMs(50), 1_800_000, "backoff is capped, not unbounded");
|
||||
|
||||
const ids = new Set(harness.calls.map((call) => call.body.notificationId));
|
||||
assert.equal(ids.size, 1, "retries must reuse one stable delivery identity");
|
||||
assert.equal(harness.outbox()[0].delivered_at, BASE_TIME + elapsed);
|
||||
assert.equal(harness.outbox().length, 1, "retrying never creates new rows");
|
||||
});
|
||||
|
||||
test("open incidents and pending deliveries survive an API restart", async (t) => {
|
||||
const harness = createHarness({ responses: [new Error("ECONNREFUSED")] });
|
||||
const first = harness.build();
|
||||
t.after(() => harness.database.close());
|
||||
first.start();
|
||||
first.observeFailure("storage", STORAGE_FAILURE);
|
||||
await harness.clock.runTo(BASE_TIME);
|
||||
assert.equal(harness.outbox()[0].delivered_at, null);
|
||||
first.stop();
|
||||
|
||||
// A restart re-reads durable state; it must not re-notify or lose work.
|
||||
harness.clock.set(BASE_TIME + 5 * 60_000);
|
||||
const restarted = harness.build();
|
||||
restarted.start();
|
||||
assert.equal(restarted.isOpen("storage"), true);
|
||||
assert.equal(
|
||||
restarted.observeFailure("storage", STORAGE_FAILURE).opened,
|
||||
false,
|
||||
"a restart must not reopen an already-open incident",
|
||||
);
|
||||
assert.equal(harness.outbox().length, 1);
|
||||
assert.equal(restarted.status().pendingCount, 1);
|
||||
|
||||
harness.responses.push(200);
|
||||
await harness.clock.runTo(BASE_TIME + 10 * 60_000);
|
||||
assert.equal(harness.outbox()[0].delivered_at, BASE_TIME + 10 * 60_000);
|
||||
assert.equal(harness.calls.at(-1).body.notificationId, harness.calls[0].body.notificationId);
|
||||
});
|
||||
|
||||
test("an unconfigured webhook records incidents without queuing endless delivery work", async (t) => {
|
||||
const harness = createHarness({ webhookUrl: null });
|
||||
const service = harness.build();
|
||||
t.after(() => harness.database.close());
|
||||
service.start();
|
||||
|
||||
service.observeFailure("storage", STORAGE_FAILURE);
|
||||
service.observeRecovery("storage", { event: "storage.recovered" });
|
||||
await harness.clock.runTo(BASE_TIME + 60 * 60_000);
|
||||
|
||||
assert.equal(harness.incidents().length, 1, "incident history stays useful");
|
||||
assert.equal(harness.outbox().length, 0, "nothing retries against a receiver that does not exist");
|
||||
assert.equal(harness.calls.length, 0);
|
||||
const status = service.status();
|
||||
assert.equal(status.configured, false);
|
||||
assert.equal(status.status, "disabled");
|
||||
});
|
||||
|
||||
test("notification status never reveals the webhook URL or secret", async (t) => {
|
||||
const harness = createHarness({ responses: [new Error("ECONNREFUSED")] });
|
||||
const service = harness.build();
|
||||
t.after(() => harness.database.close());
|
||||
service.start();
|
||||
service.observeFailure("storage", STORAGE_FAILURE);
|
||||
await harness.clock.runTo(BASE_TIME);
|
||||
|
||||
const serialized = JSON.stringify(service.status());
|
||||
assert.doesNotMatch(serialized, /super-secret-webhook-id/);
|
||||
assert.doesNotMatch(serialized, /127\.0\.0\.1:9/);
|
||||
assert.doesNotMatch(serialized, /announcements\/?"?$/);
|
||||
assert.equal(service.status().configured, true);
|
||||
assert.equal(service.status().openIncidents.length, 1);
|
||||
assert.equal(service.status().openIncidents[0].conditionKey, "storage");
|
||||
});
|
||||
|
||||
test("storage health transitions drive exactly one incident pair", async (t) => {
|
||||
const harness = createHarness();
|
||||
const service = harness.build();
|
||||
t.after(() => harness.database.close());
|
||||
service.start();
|
||||
|
||||
observeStorageHealth(service, {
|
||||
status: "degraded",
|
||||
code: "STORAGE_NOT_CHECKED",
|
||||
});
|
||||
assert.equal(harness.incidents().length, 0, "startup state is not an outage");
|
||||
|
||||
for (let observation = 0; observation < 5; observation += 1) {
|
||||
observeStorageHealth(service, {
|
||||
status: "degraded",
|
||||
code: "STORAGE_UNAVAILABLE",
|
||||
checkedAt: "2026-08-16T12:00:00.000Z",
|
||||
});
|
||||
}
|
||||
assert.equal(harness.outbox().length, 1);
|
||||
|
||||
observeStorageHealth(service, { status: "healthy", checkedAt: "2026-08-16T12:30:00.000Z" });
|
||||
assert.equal(harness.outbox().length, 2);
|
||||
assert.equal(harness.outbox()[1].transition, "recovery");
|
||||
});
|
||||
|
||||
test("no playable slides is reported even while a static override fills the screen", async (t) => {
|
||||
const harness = createHarness();
|
||||
const service = harness.build();
|
||||
t.after(() => harness.database.close());
|
||||
service.start();
|
||||
|
||||
observePlayback(service, {
|
||||
activePlaylistId: "playlist-a",
|
||||
normalTarget: { id: "entry" },
|
||||
mode: "on",
|
||||
});
|
||||
assert.equal(harness.incidents().length, 0);
|
||||
|
||||
observePlayback(service, {
|
||||
activePlaylistId: "playlist-a",
|
||||
normalTarget: null,
|
||||
staticTarget: { assetId: "static-asset" },
|
||||
mode: "on",
|
||||
});
|
||||
const open = harness.incidents();
|
||||
assert.equal(open.length, 1);
|
||||
assert.equal(open[0].event, "playback.no_playable_slides");
|
||||
assert.equal(JSON.parse(open[0].context_json).staticActive, true);
|
||||
|
||||
observePlayback(service, {
|
||||
activePlaylistId: "playlist-a",
|
||||
normalTarget: null,
|
||||
staticTarget: { assetId: "static-asset" },
|
||||
mode: "on",
|
||||
});
|
||||
assert.equal(harness.outbox().length, 1, "a continuing condition never re-notifies");
|
||||
|
||||
observePlayback(service, {
|
||||
activePlaylistId: "playlist-a",
|
||||
normalTarget: { id: "entry" },
|
||||
mode: "on",
|
||||
});
|
||||
assert.equal(harness.incidents()[0].state, "recovered");
|
||||
assert.equal(harness.outbox().length, 2);
|
||||
});
|
||||
|
||||
test("each failed upload opens one bounded warning incident", async (t) => {
|
||||
const harness = createHarness();
|
||||
const service = harness.build();
|
||||
t.after(() => harness.database.close());
|
||||
service.start();
|
||||
|
||||
observeAssetProcessingFailure(service, {
|
||||
assetId: "asset-1",
|
||||
displayName: "Bulletin",
|
||||
message: "x".repeat(5_000),
|
||||
});
|
||||
observeAssetProcessingFailure(service, {
|
||||
assetId: "asset-1",
|
||||
displayName: "Bulletin",
|
||||
message: "again",
|
||||
});
|
||||
observeAssetProcessingFailure(service, { assetId: "asset-2", displayName: "Notice" });
|
||||
|
||||
assert.equal(harness.incidents().length, 2, "one incident per failed asset");
|
||||
assert.equal(harness.outbox().length, 2);
|
||||
const context = JSON.parse(harness.incidents()[0].context_json);
|
||||
assert.ok(
|
||||
context.processingError.length <= 500,
|
||||
"a processing error is bounded before it is stored",
|
||||
);
|
||||
assert.equal(harness.incidents()[0].severity, "warning");
|
||||
});
|
||||
@@ -30,8 +30,17 @@ function config() {
|
||||
maxFiles: 2,
|
||||
maxInputPixels: 1000,
|
||||
},
|
||||
mqtt: { url: null },
|
||||
notifications: { homeAssistantWebhookUrl: null },
|
||||
mqtt: {
|
||||
url: null,
|
||||
topicPrefix: "announcements",
|
||||
reconnectPeriodMs: 5000,
|
||||
connectTimeoutMs: 10000,
|
||||
},
|
||||
notifications: {
|
||||
homeAssistantWebhookUrl: null,
|
||||
webhookTimeoutMs: 10000,
|
||||
},
|
||||
displays: { offlineThresholdMs: 45000 },
|
||||
};
|
||||
}
|
||||
|
||||
|
||||
Reference in New Issue
Block a user