diff --git a/ci/display-scenarios b/ci/display-scenarios index 363e38a..1d06710 100644 --- a/ci/display-scenarios +++ b/ci/display-scenarios @@ -2,3 +2,5 @@ # (scripts/lib/deploy-smoke-display.mjs in the pinned platform). Pairing a kiosk and restarting it on # the same profile always run first. A change that makes another scenario pass adds it here. two-pages +session-blip +stream-failure diff --git a/src/browser/backoff.ts b/src/browser/backoff.ts new file mode 100644 index 0000000..721e439 --- /dev/null +++ b/src/browser/backoff.ts @@ -0,0 +1,21 @@ +/** + * How long a display waits before trying a failed request again. + * + * Exponential from one second to a one-minute ceiling, so a screen whose server is restarting is + * back within seconds, and one whose server is down for an evening asks once a minute rather than + * hammering it. The wait is jittered between half and all of that step, so a building's worth of + * screens that lost the server at the same moment do not all come back at the same moment. + * + * There is no attempt limit. A television on a wall has nobody to press Retry, so giving up is the + * one thing it must never do. + */ +export const DISPLAY_RETRY_BASE_MS = 1_000; +export const DISPLAY_RETRY_MAX_MS = 60_000; + +export function displayRetryDelayMs(attempt: number, random: () => number = Math.random): number { + const step = Math.min( + DISPLAY_RETRY_MAX_MS, + DISPLAY_RETRY_BASE_MS * 2 ** Math.min(Math.max(0, attempt), 16), + ); + return Math.round(step / 2 + (step / 2) * Math.min(1, Math.max(0, random()))); +} diff --git a/src/display/controller.ts b/src/display/controller.ts index 98db5cc..d531c11 100644 --- a/src/display/controller.ts +++ b/src/display/controller.ts @@ -1,4 +1,5 @@ import type { ModuleUiHostContext } from "@ekklesia/module-ui-sdk"; +import { displayRetryDelayMs } from "../browser/backoff.js"; import { type OfflineSnapshot, resolveOffline, @@ -122,6 +123,13 @@ export function createDisplayController(options: DisplayControllerOptions): Disp let snapshot: OfflineSnapshot | undefined; let offlineActive = false; let offlineHandle: unknown; + /** Retries, each on its own backoff. A display has nobody to press Retry, so none of them give up; + * each resets once its request works again. */ + let sessionHandle: unknown; + let sessionAttempt = 0; + let refreshAttempt = 0; + let realtimeHandle: unknown; + let realtimeAttempt = 0; /** Selections this display has tried and failed to render. A broken item is skipped rather than * retried in a tight loop, which is what keeps one bad slide from becoming a flicker. */ const unusable = new Set(); @@ -497,19 +505,121 @@ export function createDisplayController(options: DisplayControllerOptions): Disp `channels/${session.channelId}/display-state`, ); if (!response.ok) { - online = false; - await runOffline(); + await refreshFailed(); return; } - await apply((await response.json()) as DisplayChannelStateView); + const state = (await response.json()) as DisplayChannelStateView; + refreshAttempt = 0; + await apply(state); } catch { - // The server is unreachable. The frame stays and local continuation takes over; nothing about - // a network failure belongs on a lobby wall. - online = false; - await runOffline(); + await refreshFailed(); } } + /** + * The server is unreachable, or answered with an error. The frame stays and local continuation + * takes over — nothing about a network failure belongs on a lobby wall — and the refresh is tried + * again on its own, because a realtime message that would otherwise prompt it may never come. + */ + async function refreshFailed(): Promise { + online = false; + await runOffline(); + if (stopped) { + return; + } + if (refreshHandle !== undefined) { + clearTimer(refreshHandle); + } + refreshHandle = setTimer(() => { + refreshHandle = undefined; + void refresh(); + }, displayRetryDelayMs(refreshAttempt)); + refreshAttempt += 1; + } + + /** + * Registers this display, and keeps trying until it can. + * + * Until it has registered, a display continues from its own persisted state, honestly marked as + * offline. The first registration that works brings it live — realtime, heartbeats, and fresh + * state — exactly as if it had worked the first time. + */ + async function register(): Promise { + if (stopped || session) { + return; + } + let registered: DisplaySessionView | undefined; + try { + const response = await options.context.api.request("viewers/session", { + body: JSON.stringify({ viewerKey: await viewerKey() }), + headers: { "content-type": "application/json" }, + method: "POST", + }); + if (response.ok) { + // Read, not cast: a registration the display cannot understand is no registration, and is + // tried again like any other failure. + const parsed = parseViewerSessionResponse(await response.json()); + registered = parsed.ok ? parsed.value : undefined; + } + } catch { + registered = undefined; + } + if (stopped) { + return; + } + if (!registered) { + online = false; + report(); + sessionHandle = setTimer(() => { + sessionHandle = undefined; + void register(); + }, displayRetryDelayMs(sessionAttempt)); + sessionAttempt += 1; + return; + } + sessionAttempt = 0; + session = registered; + online = true; + subscribe(); + heartbeatHandle = setInterval( + () => void heartbeat(), + registered.heartbeatIntervalMs, + ) as unknown; + await refresh(); + void heartbeat(); + report(); + } + + /** + * Subscribes to the display's realtime channel, and again whenever the host reports that a + * subscription has ended for good. The host retries ordinary disconnects itself; a stream that + * is refused outright ends, and until this resubscribes the display would only learn of changes + * at its next slot deadline. + */ + function subscribe(): void { + if (stopped || !session) { + return; + } + subscription?.close(); + subscription = options.context.realtime.subscribe("display-state", { + onSnapshot: (state) => onRealtime(state), + onStatus: (status) => { + online = status === "connected"; + if (status === "connected") { + realtimeAttempt = 0; + } else if (status === "closed" && !stopped && realtimeHandle === undefined) { + realtimeHandle = setTimer(() => { + realtimeHandle = undefined; + subscribe(); + }, displayRetryDelayMs(realtimeAttempt)); + realtimeAttempt += 1; + } + report(); + }, + onUpdate: (state) => onRealtime(state), + }); + } + async function heartbeat(): Promise { if (stopped || !session || !telemetry) { return; @@ -578,40 +688,7 @@ export function createDisplayController(options: DisplayControllerOptions): Disp } if (telemetry) { - try { - const response = await options.context.api.request("viewers/session", { - body: JSON.stringify({ viewerKey: await viewerKey() }), - headers: { "content-type": "application/json" }, - method: "POST", - }); - if (response.ok) { - // Read, not cast: a registration the display cannot understand is no registration. - const parsed = parseViewerSessionResponse(await response.json()); - if (parsed.ok) { - session = parsed.value; - online = true; - } - } - } catch { - online = false; - } - } - - if (session) { - subscription = options.context.realtime.subscribe("display-state", { - onSnapshot: (state) => onRealtime(state), - onStatus: (status) => { - online = status === "connected"; - report(); - }, - onUpdate: (state) => onRealtime(state), - }); - heartbeatHandle = setInterval( - () => void heartbeat(), - session.heartbeatIntervalMs, - ) as unknown; - await refresh(); - void heartbeat(); + await register(); } report(); }, @@ -628,6 +705,13 @@ export function createDisplayController(options: DisplayControllerOptions): Disp clearTimer(refreshHandle); refreshHandle = undefined; } + for (const handle of [sessionHandle, realtimeHandle]) { + if (handle !== undefined) { + clearTimer(handle); + } + } + sessionHandle = undefined; + realtimeHandle = undefined; exitOffline(); await stage?.destroy(); stage = undefined; diff --git a/tests/browser/backoff.test.ts b/tests/browser/backoff.test.ts new file mode 100644 index 0000000..550ab7a --- /dev/null +++ b/tests/browser/backoff.test.ts @@ -0,0 +1,34 @@ +import { describe, expect, it } from "vitest"; +import { + DISPLAY_RETRY_BASE_MS, + DISPLAY_RETRY_MAX_MS, + displayRetryDelayMs, +} from "../../src/browser/backoff.js"; + +describe("display retry backoff", () => { + it("doubles from one second and stops at one minute", () => { + const longest = (attempt: number) => displayRetryDelayMs(attempt, () => 1); + expect([0, 1, 2, 3, 4, 5, 6, 7, 50].map(longest)).toEqual([ + 1_000, 2_000, 4_000, 8_000, 16_000, 32_000, 60_000, 60_000, 60_000, + ]); + }); + + it("jitters between half and all of each step, and never waits zero", () => { + for (const attempt of [0, 3, 6, 40]) { + const step = Math.min(DISPLAY_RETRY_MAX_MS, DISPLAY_RETRY_BASE_MS * 2 ** attempt); + expect(displayRetryDelayMs(attempt, () => 0)).toBe(step / 2); + expect(displayRetryDelayMs(attempt, () => 1)).toBe(step); + for (let index = 0; index < 50; index += 1) { + const delay = displayRetryDelayMs(attempt); + expect(delay).toBeGreaterThanOrEqual(step / 2); + expect(delay).toBeLessThanOrEqual(step); + } + } + }); + + it("treats a nonsensical attempt or random value as the nearest sensible one", () => { + expect(displayRetryDelayMs(-3, () => 1)).toBe(1_000); + expect(displayRetryDelayMs(2, () => 7)).toBe(4_000); + expect(displayRetryDelayMs(2, () => -1)).toBe(2_000); + }); +}); diff --git a/tests/browser/display-recovery.test.ts b/tests/browser/display-recovery.test.ts new file mode 100644 index 0000000..c138491 --- /dev/null +++ b/tests/browser/display-recovery.test.ts @@ -0,0 +1,307 @@ +// @vitest-environment happy-dom +import type { ModuleUiHostContext } from "@ekklesia/module-ui-sdk"; +import { beforeEach, describe, expect, it } from "vitest"; +import { + createMemoryMediaCache, + createMemoryStore, + type SignageLocalStore, +} from "../../src/browser/storage.js"; +import { + createDisplayController, + type DisplayChannelStateView, +} from "../../src/display/controller.js"; +import { createFakeHost, type FakeHost, jsonResponse } from "./support/fake-host.js"; + +/** + * A display never reaches a dead end. + * + * Every network path a display depends on — registering, fetching state, realtime — can fail once, + * as a kiosk that boots before its network settles or an API that restarts will make it. Each must + * be tried again, with backoff, until it works, and the display must then be exactly as live as if + * nothing had happened. None of it may need a person to reload a television. + * + * Time is virtual: every timer the controller schedules goes through the queue below. + */ + +const SESSION = { + channelId: "chan-1", + channelKey: "default", + groupId: "group-1", + heartbeatIntervalMs: 12_000, + viewerId: "11111111-1111-4111-8111-111111111111", + viewerName: "Foyer", +}; + +function selection(id: string) { + return { + availableUntilMs: null, + deadlineAtMs: 5000, + durationMs: 5000, + id, + itemId: `image-${id}`, + itemRevision: "rev-1", + media: { + display: { + expiresAtMs: 900_000, + height: 1080, + url: `https://cdn.invalid/${id}`, + width: 1920, + }, + thumbnail: { + expiresAtMs: 900_000, + height: 270, + url: `https://cdn.invalid/${id}-t`, + width: 480, + }, + }, + payload: { kind: "native-image" }, + presentation: null, + providerId: "signage.native-images", + renderer: { kind: "native-image" as const }, + revision: 1, + scene: { + backgroundColor: "#000000", + backgroundImage: false, + backgroundMode: "none" as const, + blurPixels: 0, + darkenOpacity: 0, + fit: "fill" as const, + shadow: false, + }, + slotId: "slot-1", + stale: false, + startedAtMs: 0, + withdrawn: false, + }; +} + +function channelState(revision = 3): DisplayChannelStateView { + const current = selection("sel-1"); + return { + channel: { groupId: "group-1", id: "chan-1", key: "default" }, + crossfadeMs: 0, + current, + playback: { + activeTimelineId: "timeline-1", + blackout: false, + currentSelectionId: current.id, + currentSlotId: current.slotId, + deadlineAtMs: 5000, + pendingTimelineId: null, + remainingMs: 5000, + revision, + selectionRevision: 1, + startedAtMs: 0, + transport: "playing", + }, + selections: [current], + serverNowMs: 0, + staticHold: null, + timeline: null, + }; +} + +/** Server behaviour that a test can change mid-run. */ +interface Server { + sessionFailures: number; + stateFailing: boolean; + sessionRequests: number; + stateRequests: number; +} + +describe("a display recovers from every failed request without a reload", () => { + let clock: number; + let timers: Map void }>; + let nextTimer: number; + let root: HTMLElement; + let local: SignageLocalStore; + let server: Server; + let host: FakeHost; + let subscriptions: { onStatus?: (status: string) => void }[]; + + beforeEach(() => { + clock = 0; + timers = new Map(); + nextTimer = 1; + root = document.createElement("div"); + document.body.append(root); + local = createMemoryStore(); + server = { sessionFailures: 0, sessionRequests: 0, stateFailing: false, stateRequests: 0 }; + subscriptions = []; + host = createFakeHost({ + permissions: ["signage.display.view"], + routes: [ + { + method: "POST", + path: "viewers/session", + respond: () => { + server.sessionRequests += 1; + if (server.sessionFailures > 0) { + server.sessionFailures -= 1; + throw new TypeError("Failed to fetch"); + } + return jsonResponse(SESSION); + }, + }, + { + method: "GET", + path: "channels/chan-1/display-state", + respond: () => { + server.stateRequests += 1; + if (server.stateFailing) { + throw new TypeError("Failed to fetch"); + } + return jsonResponse(channelState()); + }, + }, + { method: "POST", path: /^viewers\/.*\/heartbeat$/u, respond: () => jsonResponse({}) }, + ], + }); + Object.assign(host.context, { + realtime: { + subscribe: (_channel: string, handlers: { onStatus?: (status: string) => void }) => { + subscriptions.push(handlers); + return { close: () => undefined }; + }, + } satisfies Pick as never, + }); + }); + + async function settle(): Promise { + for (let index = 0; index < 20; index += 1) { + await new Promise((resolve) => setTimeout(resolve, 0)); + } + } + + /** Moves virtual time forward, firing every timer that falls due, in order. */ + async function advance(ms: number): Promise { + const end = clock + ms; + for (;;) { + const due = [...timers] + .filter(([, timer]) => timer.at <= end) + .sort((left, right) => left[1].at - right[1].at)[0]; + if (!due) { + break; + } + timers.delete(due[0]); + clock = due[1].at; + due[1].run(); + await settle(); + } + clock = end; + } + + function build() { + return createDisplayController({ + clearTimer: (handle) => timers.delete(handle as number), + context: host.context, + decodeImage: () => Promise.resolve(), + media: createMemoryMediaCache(), + newViewerKey: () => "viewer-fixed-key-0001", + now: () => clock, + root, + setTimer: (run, delayMs) => { + const id = nextTimer; + nextTimer += 1; + timers.set(id, { at: clock + delayMs, run }); + return id; + }, + store: local, + }); + } + + /** One ordinary live run, so the display has a persisted snapshot to continue from, exactly as a + * screen that has been on the wall for a while does. */ + async function runOnceLive(): Promise { + const display = build(); + await display.start(); + await settle(); + expect(root.dataset.offline).toBe("false"); + await display.stop(); + timers.clear(); + } + + it("registers once the server answers, after its first registration failed", async () => { + await runOnceLive(); + server.sessionFailures = 1; + const sessionsBefore = server.sessionRequests; + + const display = build(); + await display.start(); + await settle(); + // Continuing from its own state while it cannot register: the honest banner. + expect(root.dataset.offline).toBe("true"); + + await advance(60_000); + + expect(server.sessionRequests - sessionsBefore).toBeGreaterThanOrEqual(2); + expect(root.dataset.offline).toBe("false"); + expect(subscriptions.length).toBeGreaterThanOrEqual(2); + await display.stop(); + }); + + it("keeps retrying registration with backoff, and never gives up", async () => { + await runOnceLive(); + server.sessionFailures = 8; + const sessionsBefore = server.sessionRequests; + + const display = build(); + await display.start(); + await settle(); + // Eight failures back off for at most 1 + 2 + 4 + 8 + 16 + 32 + 60 + 60 seconds, the last two at + // the one-minute ceiling. Five virtual minutes is enough for all of them and the success after. + await advance(5 * 60_000); + + expect(server.sessionRequests - sessionsBefore).toBe(9); + expect(root.dataset.offline).toBe("false"); + await display.stop(); + }); + + it("retries a failed state refresh until the server answers, with no realtime nudge", async () => { + const display = build(); + await display.start(); + await settle(); + expect(root.dataset.offline).toBe("false"); + + // The slot's deadline refresh fails, as it would mid-restart. Realtime says nothing. + server.stateFailing = true; + await advance(6_000); + expect(root.dataset.offline).toBe("true"); + const failedRequests = server.stateRequests; + + server.stateFailing = false; + await advance(120_000); + + expect(server.stateRequests).toBeGreaterThan(failedRequests); + expect(root.dataset.offline).toBe("false"); + await display.stop(); + }); + + it("subscribes again when the host reports its realtime stream closed for good", async () => { + const display = build(); + await display.start(); + await settle(); + expect(subscriptions).toHaveLength(1); + + subscriptions[0]?.onStatus?.("closed"); + await settle(); + await advance(60_000); + + expect(subscriptions.length).toBeGreaterThanOrEqual(2); + await display.stop(); + }); + + it("schedules nothing after it is stopped", async () => { + await runOnceLive(); + server.sessionFailures = 3; + const display = build(); + await display.start(); + await settle(); + await display.stop(); + const sessionsAtStop = server.sessionRequests; + + await advance(5 * 60_000); + + expect(server.sessionRequests).toBe(sessionsAtStop); + }); +});