From 518fd101a2bbff426864dc282cfdc64f6935002b Mon Sep 17 00:00:00 2001 From: ekklesia Date: Mon, 5 Oct 2026 18:34:25 -0500 Subject: [PATCH 1/3] test(display): a display recovers from every failed request without a reload MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Pins what a kiosk needs from each network path, on a virtual clock: a registration that fails once is retried with backoff until it works, a state refresh that fails is retried without a realtime nudge, and a realtime stream the host reports closed is subscribed again. Nothing is scheduled after stop. Fails today on all three paths: each is tried once. The first matches the production screen exactly: mounted, showing "Offline — continuing from the last update", with no heartbeat and no retry until someone reloads it. Co-Authored-By: Claude Opus 5.5 --- tests/browser/display-recovery.test.ts | 306 +++++++++++++++++++++++++ 1 file changed, 306 insertions(+) create mode 100644 tests/browser/display-recovery.test.ts diff --git a/tests/browser/display-recovery.test.ts b/tests/browser/display-recovery.test.ts new file mode 100644 index 0000000..cdef776 --- /dev/null +++ b/tests/browser/display-recovery.test.ts @@ -0,0 +1,306 @@ +// @vitest-environment happy-dom +import type { ModuleUiHostContext } from "@ekklesia/module-ui-sdk"; +import { beforeEach, describe, expect, it } from "vitest"; +import { + createMemoryMediaCache, + createMemoryStore, + type SignageLocalStore, +} from "../../src/browser/storage.js"; +import { + createDisplayController, + type DisplayChannelStateView, +} from "../../src/display/controller.js"; +import { createFakeHost, type FakeHost, jsonResponse } from "./support/fake-host.js"; + +/** + * A display never reaches a dead end. + * + * Every network path a display depends on — registering, fetching state, realtime — can fail once, + * as a kiosk that boots before its network settles or an API that restarts will make it. Each must + * be tried again, with backoff, until it works, and the display must then be exactly as live as if + * nothing had happened. None of it may need a person to reload a television. + * + * Time is virtual: every timer the controller schedules goes through the queue below. + */ + +const SESSION = { + channelId: "chan-1", + channelKey: "default", + groupId: "group-1", + heartbeatIntervalMs: 12_000, + viewerId: "11111111-1111-4111-8111-111111111111", + viewerName: "Foyer", +}; + +function selection(id: string) { + return { + availableUntilMs: null, + deadlineAtMs: 5000, + durationMs: 5000, + id, + itemId: `image-${id}`, + itemRevision: "rev-1", + media: { + display: { + expiresAtMs: 900_000, + height: 1080, + url: `https://cdn.invalid/${id}`, + width: 1920, + }, + thumbnail: { + expiresAtMs: 900_000, + height: 270, + url: `https://cdn.invalid/${id}-t`, + width: 480, + }, + }, + payload: { kind: "native-image" }, + presentation: null, + providerId: "signage.native-images", + renderer: { kind: "native-image" as const }, + revision: 1, + scene: { + backgroundColor: "#000000", + backgroundImage: false, + backgroundMode: "none" as const, + blurPixels: 0, + darkenOpacity: 0, + fit: "fill" as const, + shadow: false, + }, + slotId: "slot-1", + stale: false, + startedAtMs: 0, + withdrawn: false, + }; +} + +function channelState(revision = 3): DisplayChannelStateView { + const current = selection("sel-1"); + return { + channel: { groupId: "group-1", id: "chan-1", key: "default" }, + crossfadeMs: 0, + current, + playback: { + activeTimelineId: "timeline-1", + blackout: false, + currentSelectionId: current.id, + currentSlotId: current.slotId, + deadlineAtMs: 5000, + pendingTimelineId: null, + remainingMs: 5000, + revision, + selectionRevision: 1, + startedAtMs: 0, + transport: "playing", + }, + selections: [current], + serverNowMs: 0, + staticHold: null, + timeline: null, + }; +} + +/** Server behaviour that a test can change mid-run. */ +interface Server { + sessionFailures: number; + stateFailing: boolean; + sessionRequests: number; + stateRequests: number; +} + +describe("a display recovers from every failed request without a reload", () => { + let clock: number; + let timers: Map void }>; + let nextTimer: number; + let root: HTMLElement; + let local: SignageLocalStore; + let server: Server; + let host: FakeHost; + let subscriptions: { onStatus?: (status: string) => void }[]; + + beforeEach(() => { + clock = 0; + timers = new Map(); + nextTimer = 1; + root = document.createElement("div"); + document.body.append(root); + local = createMemoryStore(); + server = { sessionFailures: 0, sessionRequests: 0, stateFailing: false, stateRequests: 0 }; + subscriptions = []; + host = createFakeHost({ + permissions: ["signage.display.view"], + routes: [ + { + method: "POST", + path: "viewers/session", + respond: () => { + server.sessionRequests += 1; + if (server.sessionFailures > 0) { + server.sessionFailures -= 1; + throw new TypeError("Failed to fetch"); + } + return jsonResponse(SESSION); + }, + }, + { + method: "GET", + path: "channels/chan-1/display-state", + respond: () => { + server.stateRequests += 1; + if (server.stateFailing) { + throw new TypeError("Failed to fetch"); + } + return jsonResponse(channelState()); + }, + }, + { method: "POST", path: /^viewers\/.*\/heartbeat$/u, respond: () => jsonResponse({}) }, + ], + }); + Object.assign(host.context, { + realtime: { + subscribe: (_channel: string, handlers: { onStatus?: (status: string) => void }) => { + subscriptions.push(handlers); + return { close: () => undefined }; + }, + } satisfies Pick as never, + }); + }); + + async function settle(): Promise { + for (let index = 0; index < 20; index += 1) { + await new Promise((resolve) => setTimeout(resolve, 0)); + } + } + + /** Moves virtual time forward, firing every timer that falls due, in order. */ + async function advance(ms: number): Promise { + const end = clock + ms; + for (;;) { + const due = [...timers] + .filter(([, timer]) => timer.at <= end) + .sort((left, right) => left[1].at - right[1].at)[0]; + if (!due) { + break; + } + timers.delete(due[0]); + clock = due[1].at; + due[1].run(); + await settle(); + } + clock = end; + } + + function build() { + return createDisplayController({ + clearTimer: (handle) => timers.delete(handle as number), + context: host.context, + decodeImage: () => Promise.resolve(), + media: createMemoryMediaCache(), + newViewerKey: () => "viewer-fixed-key-0001", + now: () => clock, + root, + setTimer: (run, delayMs) => { + const id = nextTimer; + nextTimer += 1; + timers.set(id, { at: clock + delayMs, run }); + return id; + }, + store: local, + }); + } + + /** One ordinary live run, so the display has a persisted snapshot to continue from, exactly as a + * screen that has been on the wall for a while does. */ + async function runOnceLive(): Promise { + const display = build(); + await display.start(); + await settle(); + expect(root.dataset.offline).toBe("false"); + await display.stop(); + timers.clear(); + } + + it("registers once the server answers, after its first registration failed", async () => { + await runOnceLive(); + server.sessionFailures = 1; + const sessionsBefore = server.sessionRequests; + + const display = build(); + await display.start(); + await settle(); + // Continuing from its own state while it cannot register: the honest banner. + expect(root.dataset.offline).toBe("true"); + + await advance(60_000); + + expect(server.sessionRequests - sessionsBefore).toBeGreaterThanOrEqual(2); + expect(root.dataset.offline).toBe("false"); + expect(subscriptions.length).toBeGreaterThanOrEqual(2); + await display.stop(); + }); + + it("keeps retrying registration with backoff, and never gives up", async () => { + await runOnceLive(); + server.sessionFailures = 8; + const sessionsBefore = server.sessionRequests; + + const display = build(); + await display.start(); + await settle(); + // An hour of virtual time: enough for every backoff step, and the cap, to have been reached. + await advance(60 * 60_000); + + expect(server.sessionRequests - sessionsBefore).toBe(9); + expect(root.dataset.offline).toBe("false"); + await display.stop(); + }); + + it("retries a failed state refresh until the server answers, with no realtime nudge", async () => { + const display = build(); + await display.start(); + await settle(); + expect(root.dataset.offline).toBe("false"); + + // The slot's deadline refresh fails, as it would mid-restart. Realtime says nothing. + server.stateFailing = true; + await advance(6_000); + expect(root.dataset.offline).toBe("true"); + const failedRequests = server.stateRequests; + + server.stateFailing = false; + await advance(120_000); + + expect(server.stateRequests).toBeGreaterThan(failedRequests); + expect(root.dataset.offline).toBe("false"); + await display.stop(); + }); + + it("subscribes again when the host reports its realtime stream closed for good", async () => { + const display = build(); + await display.start(); + await settle(); + expect(subscriptions).toHaveLength(1); + + subscriptions[0]?.onStatus?.("closed"); + await settle(); + await advance(60_000); + + expect(subscriptions.length).toBeGreaterThanOrEqual(2); + await display.stop(); + }); + + it("schedules nothing after it is stopped", async () => { + await runOnceLive(); + server.sessionFailures = 3; + const display = build(); + await display.start(); + await settle(); + await display.stop(); + const sessionsAtStop = server.sessionRequests; + + await advance(60 * 60_000); + + expect(server.sessionRequests).toBe(sessionsAtStop); + }); +}); From fbc8af7510ae94a3786747dc91d7cf76625b691e Mon Sep 17 00:00:00 2001 From: ekklesia Date: Mon, 5 Oct 2026 18:35:46 -0500 Subject: [PATCH 2/3] fix(display): retry every failed request with backoff, and never give up A display tried each of its network paths once: - A registration that failed left it continuing from its persisted state, behind the offline banner, with no realtime, heartbeat or retry, until someone reloaded it. This is what the production kiosk showed after a restart. - A failed state refresh scheduled no further refresh, so a display whose realtime was also down never fetched state again. - A realtime stream the host reported closed was never subscribed again. Each path now retries on its own backoff: about 1 s, doubling to a one-minute ceiling, with jitter so a building's screens do not return in lockstep. There is no attempt limit, because a television has nobody to press Retry. Each resets when its request works again. - Registration retries until it works, then brings the display live exactly as a first-time success does. - A failed refresh schedules its own retry, with no realtime needed. - A closed realtime subscription is replaced. - Stopping clears every retry. The recovery test's virtual window shrinks from an hour to five minutes: eight failures back off for at most about 183 s, and an hour of offline-continuation timers made the test slow without proving more. Co-Authored-By: Claude Opus 5.5 --- src/browser/backoff.ts | 21 ++++ src/display/controller.ts | 166 +++++++++++++++++++------ tests/browser/backoff.test.ts | 34 +++++ tests/browser/display-recovery.test.ts | 7 +- 4 files changed, 184 insertions(+), 44 deletions(-) create mode 100644 src/browser/backoff.ts create mode 100644 tests/browser/backoff.test.ts diff --git a/src/browser/backoff.ts b/src/browser/backoff.ts new file mode 100644 index 0000000..721e439 --- /dev/null +++ b/src/browser/backoff.ts @@ -0,0 +1,21 @@ +/** + * How long a display waits before trying a failed request again. + * + * Exponential from one second to a one-minute ceiling, so a screen whose server is restarting is + * back within seconds, and one whose server is down for an evening asks once a minute rather than + * hammering it. The wait is jittered between half and all of that step, so a building's worth of + * screens that lost the server at the same moment do not all come back at the same moment. + * + * There is no attempt limit. A television on a wall has nobody to press Retry, so giving up is the + * one thing it must never do. + */ +export const DISPLAY_RETRY_BASE_MS = 1_000; +export const DISPLAY_RETRY_MAX_MS = 60_000; + +export function displayRetryDelayMs(attempt: number, random: () => number = Math.random): number { + const step = Math.min( + DISPLAY_RETRY_MAX_MS, + DISPLAY_RETRY_BASE_MS * 2 ** Math.min(Math.max(0, attempt), 16), + ); + return Math.round(step / 2 + (step / 2) * Math.min(1, Math.max(0, random()))); +} diff --git a/src/display/controller.ts b/src/display/controller.ts index 98db5cc..d531c11 100644 --- a/src/display/controller.ts +++ b/src/display/controller.ts @@ -1,4 +1,5 @@ import type { ModuleUiHostContext } from "@ekklesia/module-ui-sdk"; +import { displayRetryDelayMs } from "../browser/backoff.js"; import { type OfflineSnapshot, resolveOffline, @@ -122,6 +123,13 @@ export function createDisplayController(options: DisplayControllerOptions): Disp let snapshot: OfflineSnapshot | undefined; let offlineActive = false; let offlineHandle: unknown; + /** Retries, each on its own backoff. A display has nobody to press Retry, so none of them give up; + * each resets once its request works again. */ + let sessionHandle: unknown; + let sessionAttempt = 0; + let refreshAttempt = 0; + let realtimeHandle: unknown; + let realtimeAttempt = 0; /** Selections this display has tried and failed to render. A broken item is skipped rather than * retried in a tight loop, which is what keeps one bad slide from becoming a flicker. */ const unusable = new Set(); @@ -497,19 +505,121 @@ export function createDisplayController(options: DisplayControllerOptions): Disp `channels/${session.channelId}/display-state`, ); if (!response.ok) { - online = false; - await runOffline(); + await refreshFailed(); return; } - await apply((await response.json()) as DisplayChannelStateView); + const state = (await response.json()) as DisplayChannelStateView; + refreshAttempt = 0; + await apply(state); } catch { - // The server is unreachable. The frame stays and local continuation takes over; nothing about - // a network failure belongs on a lobby wall. - online = false; - await runOffline(); + await refreshFailed(); } } + /** + * The server is unreachable, or answered with an error. The frame stays and local continuation + * takes over — nothing about a network failure belongs on a lobby wall — and the refresh is tried + * again on its own, because a realtime message that would otherwise prompt it may never come. + */ + async function refreshFailed(): Promise { + online = false; + await runOffline(); + if (stopped) { + return; + } + if (refreshHandle !== undefined) { + clearTimer(refreshHandle); + } + refreshHandle = setTimer(() => { + refreshHandle = undefined; + void refresh(); + }, displayRetryDelayMs(refreshAttempt)); + refreshAttempt += 1; + } + + /** + * Registers this display, and keeps trying until it can. + * + * Until it has registered, a display continues from its own persisted state, honestly marked as + * offline. The first registration that works brings it live — realtime, heartbeats, and fresh + * state — exactly as if it had worked the first time. + */ + async function register(): Promise { + if (stopped || session) { + return; + } + let registered: DisplaySessionView | undefined; + try { + const response = await options.context.api.request("viewers/session", { + body: JSON.stringify({ viewerKey: await viewerKey() }), + headers: { "content-type": "application/json" }, + method: "POST", + }); + if (response.ok) { + // Read, not cast: a registration the display cannot understand is no registration, and is + // tried again like any other failure. + const parsed = parseViewerSessionResponse(await response.json()); + registered = parsed.ok ? parsed.value : undefined; + } + } catch { + registered = undefined; + } + if (stopped) { + return; + } + if (!registered) { + online = false; + report(); + sessionHandle = setTimer(() => { + sessionHandle = undefined; + void register(); + }, displayRetryDelayMs(sessionAttempt)); + sessionAttempt += 1; + return; + } + sessionAttempt = 0; + session = registered; + online = true; + subscribe(); + heartbeatHandle = setInterval( + () => void heartbeat(), + registered.heartbeatIntervalMs, + ) as unknown; + await refresh(); + void heartbeat(); + report(); + } + + /** + * Subscribes to the display's realtime channel, and again whenever the host reports that a + * subscription has ended for good. The host retries ordinary disconnects itself; a stream that + * is refused outright ends, and until this resubscribes the display would only learn of changes + * at its next slot deadline. + */ + function subscribe(): void { + if (stopped || !session) { + return; + } + subscription?.close(); + subscription = options.context.realtime.subscribe("display-state", { + onSnapshot: (state) => onRealtime(state), + onStatus: (status) => { + online = status === "connected"; + if (status === "connected") { + realtimeAttempt = 0; + } else if (status === "closed" && !stopped && realtimeHandle === undefined) { + realtimeHandle = setTimer(() => { + realtimeHandle = undefined; + subscribe(); + }, displayRetryDelayMs(realtimeAttempt)); + realtimeAttempt += 1; + } + report(); + }, + onUpdate: (state) => onRealtime(state), + }); + } + async function heartbeat(): Promise { if (stopped || !session || !telemetry) { return; @@ -578,40 +688,7 @@ export function createDisplayController(options: DisplayControllerOptions): Disp } if (telemetry) { - try { - const response = await options.context.api.request("viewers/session", { - body: JSON.stringify({ viewerKey: await viewerKey() }), - headers: { "content-type": "application/json" }, - method: "POST", - }); - if (response.ok) { - // Read, not cast: a registration the display cannot understand is no registration. - const parsed = parseViewerSessionResponse(await response.json()); - if (parsed.ok) { - session = parsed.value; - online = true; - } - } - } catch { - online = false; - } - } - - if (session) { - subscription = options.context.realtime.subscribe("display-state", { - onSnapshot: (state) => onRealtime(state), - onStatus: (status) => { - online = status === "connected"; - report(); - }, - onUpdate: (state) => onRealtime(state), - }); - heartbeatHandle = setInterval( - () => void heartbeat(), - session.heartbeatIntervalMs, - ) as unknown; - await refresh(); - void heartbeat(); + await register(); } report(); }, @@ -628,6 +705,13 @@ export function createDisplayController(options: DisplayControllerOptions): Disp clearTimer(refreshHandle); refreshHandle = undefined; } + for (const handle of [sessionHandle, realtimeHandle]) { + if (handle !== undefined) { + clearTimer(handle); + } + } + sessionHandle = undefined; + realtimeHandle = undefined; exitOffline(); await stage?.destroy(); stage = undefined; diff --git a/tests/browser/backoff.test.ts b/tests/browser/backoff.test.ts new file mode 100644 index 0000000..550ab7a --- /dev/null +++ b/tests/browser/backoff.test.ts @@ -0,0 +1,34 @@ +import { describe, expect, it } from "vitest"; +import { + DISPLAY_RETRY_BASE_MS, + DISPLAY_RETRY_MAX_MS, + displayRetryDelayMs, +} from "../../src/browser/backoff.js"; + +describe("display retry backoff", () => { + it("doubles from one second and stops at one minute", () => { + const longest = (attempt: number) => displayRetryDelayMs(attempt, () => 1); + expect([0, 1, 2, 3, 4, 5, 6, 7, 50].map(longest)).toEqual([ + 1_000, 2_000, 4_000, 8_000, 16_000, 32_000, 60_000, 60_000, 60_000, + ]); + }); + + it("jitters between half and all of each step, and never waits zero", () => { + for (const attempt of [0, 3, 6, 40]) { + const step = Math.min(DISPLAY_RETRY_MAX_MS, DISPLAY_RETRY_BASE_MS * 2 ** attempt); + expect(displayRetryDelayMs(attempt, () => 0)).toBe(step / 2); + expect(displayRetryDelayMs(attempt, () => 1)).toBe(step); + for (let index = 0; index < 50; index += 1) { + const delay = displayRetryDelayMs(attempt); + expect(delay).toBeGreaterThanOrEqual(step / 2); + expect(delay).toBeLessThanOrEqual(step); + } + } + }); + + it("treats a nonsensical attempt or random value as the nearest sensible one", () => { + expect(displayRetryDelayMs(-3, () => 1)).toBe(1_000); + expect(displayRetryDelayMs(2, () => 7)).toBe(4_000); + expect(displayRetryDelayMs(2, () => -1)).toBe(2_000); + }); +}); diff --git a/tests/browser/display-recovery.test.ts b/tests/browser/display-recovery.test.ts index cdef776..c138491 100644 --- a/tests/browser/display-recovery.test.ts +++ b/tests/browser/display-recovery.test.ts @@ -248,8 +248,9 @@ describe("a display recovers from every failed request without a reload", () => const display = build(); await display.start(); await settle(); - // An hour of virtual time: enough for every backoff step, and the cap, to have been reached. - await advance(60 * 60_000); + // Eight failures back off for at most 1 + 2 + 4 + 8 + 16 + 32 + 60 + 60 seconds, the last two at + // the one-minute ceiling. Five virtual minutes is enough for all of them and the success after. + await advance(5 * 60_000); expect(server.sessionRequests - sessionsBefore).toBe(9); expect(root.dataset.offline).toBe("false"); @@ -299,7 +300,7 @@ describe("a display recovers from every failed request without a reload", () => await display.stop(); const sessionsAtStop = server.sessionRequests; - await advance(60 * 60_000); + await advance(5 * 60_000); expect(server.sessionRequests).toBe(sessionsAtStop); }); From bf4e98fa308152fde5c7b9fbd22014286f018edb Mon Sep 17 00:00:00 2001 From: ekklesia Date: Mon, 5 Oct 2026 21:32:03 -0500 Subject: [PATCH 3/3] ci: run session-blip and stream-failure on every change Both failed on main before this PR: a display that failed its first viewer-session request, or its first realtime connection, never recovered without a reload. With the retries in place they pass, and CI now holds them there. Co-Authored-By: Claude Opus 5.5 --- ci/display-scenarios | 2 ++ 1 file changed, 2 insertions(+) diff --git a/ci/display-scenarios b/ci/display-scenarios index 363e38a..1d06710 100644 --- a/ci/display-scenarios +++ b/ci/display-scenarios @@ -2,3 +2,5 @@ # (scripts/lib/deploy-smoke-display.mjs in the pinned platform). Pairing a kiosk and restarting it on # the same profile always run first. A change that makes another scenario pass adds it here. two-pages +session-blip +stream-failure