Files
Signage/tests/browser/display-recovery.test.ts
T
ekklesiaandClaude Opus 5.5 fbc8af7510 fix(display): retry every failed request with backoff, and never give up
A display tried each of its network paths once:
- A registration that failed left it continuing from its persisted
  state, behind the offline banner, with no realtime, heartbeat or
  retry, until someone reloaded it. This is what the production kiosk
  showed after a restart.
- A failed state refresh scheduled no further refresh, so a display
  whose realtime was also down never fetched state again.
- A realtime stream the host reported closed was never subscribed
  again.

Each path now retries on its own backoff: about 1 s, doubling to a
one-minute ceiling, with jitter so a building's screens do not return
in lockstep. There is no attempt limit, because a television has
nobody to press Retry. Each resets when its request works again.
- Registration retries until it works, then brings the display live
  exactly as a first-time success does.
- A failed refresh schedules its own retry, with no realtime needed.
- A closed realtime subscription is replaced.
- Stopping clears every retry.

The recovery test's virtual window shrinks from an hour to five
minutes: eight failures back off for at most about 183 s, and an hour
of offline-continuation timers made the test slow without proving more.

Co-Authored-By: Claude Opus 5.5 <[email protected]>
2026-10-05 21:31:55 -05:00

308 lines
9.0 KiB
TypeScript

// @vitest-environment happy-dom
import type { ModuleUiHostContext } from "@ekklesia/module-ui-sdk";
import { beforeEach, describe, expect, it } from "vitest";
import {
createMemoryMediaCache,
createMemoryStore,
type SignageLocalStore,
} from "../../src/browser/storage.js";
import {
createDisplayController,
type DisplayChannelStateView,
} from "../../src/display/controller.js";
import { createFakeHost, type FakeHost, jsonResponse } from "./support/fake-host.js";
/**
* A display never reaches a dead end.
*
* Every network path a display depends on — registering, fetching state, realtime — can fail once,
* as a kiosk that boots before its network settles or an API that restarts will make it. Each must
* be tried again, with backoff, until it works, and the display must then be exactly as live as if
* nothing had happened. None of it may need a person to reload a television.
*
* Time is virtual: every timer the controller schedules goes through the queue below.
*/
const SESSION = {
channelId: "chan-1",
channelKey: "default",
groupId: "group-1",
heartbeatIntervalMs: 12_000,
viewerId: "11111111-1111-4111-8111-111111111111",
viewerName: "Foyer",
};
function selection(id: string) {
return {
availableUntilMs: null,
deadlineAtMs: 5000,
durationMs: 5000,
id,
itemId: `image-${id}`,
itemRevision: "rev-1",
media: {
display: {
expiresAtMs: 900_000,
height: 1080,
url: `https://cdn.invalid/${id}`,
width: 1920,
},
thumbnail: {
expiresAtMs: 900_000,
height: 270,
url: `https://cdn.invalid/${id}-t`,
width: 480,
},
},
payload: { kind: "native-image" },
presentation: null,
providerId: "signage.native-images",
renderer: { kind: "native-image" as const },
revision: 1,
scene: {
backgroundColor: "#000000",
backgroundImage: false,
backgroundMode: "none" as const,
blurPixels: 0,
darkenOpacity: 0,
fit: "fill" as const,
shadow: false,
},
slotId: "slot-1",
stale: false,
startedAtMs: 0,
withdrawn: false,
};
}
function channelState(revision = 3): DisplayChannelStateView {
const current = selection("sel-1");
return {
channel: { groupId: "group-1", id: "chan-1", key: "default" },
crossfadeMs: 0,
current,
playback: {
activeTimelineId: "timeline-1",
blackout: false,
currentSelectionId: current.id,
currentSlotId: current.slotId,
deadlineAtMs: 5000,
pendingTimelineId: null,
remainingMs: 5000,
revision,
selectionRevision: 1,
startedAtMs: 0,
transport: "playing",
},
selections: [current],
serverNowMs: 0,
staticHold: null,
timeline: null,
};
}
/** Server behaviour that a test can change mid-run. */
interface Server {
sessionFailures: number;
stateFailing: boolean;
sessionRequests: number;
stateRequests: number;
}
describe("a display recovers from every failed request without a reload", () => {
let clock: number;
let timers: Map<number, { at: number; run: () => void }>;
let nextTimer: number;
let root: HTMLElement;
let local: SignageLocalStore;
let server: Server;
let host: FakeHost;
let subscriptions: { onStatus?: (status: string) => void }[];
beforeEach(() => {
clock = 0;
timers = new Map();
nextTimer = 1;
root = document.createElement("div");
document.body.append(root);
local = createMemoryStore();
server = { sessionFailures: 0, sessionRequests: 0, stateFailing: false, stateRequests: 0 };
subscriptions = [];
host = createFakeHost({
permissions: ["signage.display.view"],
routes: [
{
method: "POST",
path: "viewers/session",
respond: () => {
server.sessionRequests += 1;
if (server.sessionFailures > 0) {
server.sessionFailures -= 1;
throw new TypeError("Failed to fetch");
}
return jsonResponse(SESSION);
},
},
{
method: "GET",
path: "channels/chan-1/display-state",
respond: () => {
server.stateRequests += 1;
if (server.stateFailing) {
throw new TypeError("Failed to fetch");
}
return jsonResponse(channelState());
},
},
{ method: "POST", path: /^viewers\/.*\/heartbeat$/u, respond: () => jsonResponse({}) },
],
});
Object.assign(host.context, {
realtime: {
subscribe: (_channel: string, handlers: { onStatus?: (status: string) => void }) => {
subscriptions.push(handlers);
return { close: () => undefined };
},
} satisfies Pick<ModuleUiHostContext["realtime"], "subscribe"> as never,
});
});
async function settle(): Promise<void> {
for (let index = 0; index < 20; index += 1) {
await new Promise((resolve) => setTimeout(resolve, 0));
}
}
/** Moves virtual time forward, firing every timer that falls due, in order. */
async function advance(ms: number): Promise<void> {
const end = clock + ms;
for (;;) {
const due = [...timers]
.filter(([, timer]) => timer.at <= end)
.sort((left, right) => left[1].at - right[1].at)[0];
if (!due) {
break;
}
timers.delete(due[0]);
clock = due[1].at;
due[1].run();
await settle();
}
clock = end;
}
function build() {
return createDisplayController({
clearTimer: (handle) => timers.delete(handle as number),
context: host.context,
decodeImage: () => Promise.resolve(),
media: createMemoryMediaCache(),
newViewerKey: () => "viewer-fixed-key-0001",
now: () => clock,
root,
setTimer: (run, delayMs) => {
const id = nextTimer;
nextTimer += 1;
timers.set(id, { at: clock + delayMs, run });
return id;
},
store: local,
});
}
/** One ordinary live run, so the display has a persisted snapshot to continue from, exactly as a
* screen that has been on the wall for a while does. */
async function runOnceLive(): Promise<void> {
const display = build();
await display.start();
await settle();
expect(root.dataset.offline).toBe("false");
await display.stop();
timers.clear();
}
it("registers once the server answers, after its first registration failed", async () => {
await runOnceLive();
server.sessionFailures = 1;
const sessionsBefore = server.sessionRequests;
const display = build();
await display.start();
await settle();
// Continuing from its own state while it cannot register: the honest banner.
expect(root.dataset.offline).toBe("true");
await advance(60_000);
expect(server.sessionRequests - sessionsBefore).toBeGreaterThanOrEqual(2);
expect(root.dataset.offline).toBe("false");
expect(subscriptions.length).toBeGreaterThanOrEqual(2);
await display.stop();
});
it("keeps retrying registration with backoff, and never gives up", async () => {
await runOnceLive();
server.sessionFailures = 8;
const sessionsBefore = server.sessionRequests;
const display = build();
await display.start();
await settle();
// Eight failures back off for at most 1 + 2 + 4 + 8 + 16 + 32 + 60 + 60 seconds, the last two at
// the one-minute ceiling. Five virtual minutes is enough for all of them and the success after.
await advance(5 * 60_000);
expect(server.sessionRequests - sessionsBefore).toBe(9);
expect(root.dataset.offline).toBe("false");
await display.stop();
});
it("retries a failed state refresh until the server answers, with no realtime nudge", async () => {
const display = build();
await display.start();
await settle();
expect(root.dataset.offline).toBe("false");
// The slot's deadline refresh fails, as it would mid-restart. Realtime says nothing.
server.stateFailing = true;
await advance(6_000);
expect(root.dataset.offline).toBe("true");
const failedRequests = server.stateRequests;
server.stateFailing = false;
await advance(120_000);
expect(server.stateRequests).toBeGreaterThan(failedRequests);
expect(root.dataset.offline).toBe("false");
await display.stop();
});
it("subscribes again when the host reports its realtime stream closed for good", async () => {
const display = build();
await display.start();
await settle();
expect(subscriptions).toHaveLength(1);
subscriptions[0]?.onStatus?.("closed");
await settle();
await advance(60_000);
expect(subscriptions.length).toBeGreaterThanOrEqual(2);
await display.stop();
});
it("schedules nothing after it is stopped", async () => {
await runOnceLive();
server.sessionFailures = 3;
const display = build();
await display.start();
await settle();
await display.stop();
const sessionsAtStop = server.sessionRequests;
await advance(5 * 60_000);
expect(server.sessionRequests).toBe(sessionsAtStop);
});
});