diff --git a/src/browser/backoff.ts b/src/browser/backoff.ts new file mode 100644 index 0000000..721e439 --- /dev/null +++ b/src/browser/backoff.ts @@ -0,0 +1,21 @@ +/** + * How long a display waits before trying a failed request again. + * + * Exponential from one second to a one-minute ceiling, so a screen whose server is restarting is + * back within seconds, and one whose server is down for an evening asks once a minute rather than + * hammering it. The wait is jittered between half and all of that step, so a building's worth of + * screens that lost the server at the same moment do not all come back at the same moment. + * + * There is no attempt limit. A television on a wall has nobody to press Retry, so giving up is the + * one thing it must never do. + */ +export const DISPLAY_RETRY_BASE_MS = 1_000; +export const DISPLAY_RETRY_MAX_MS = 60_000; + +export function displayRetryDelayMs(attempt: number, random: () => number = Math.random): number { + const step = Math.min( + DISPLAY_RETRY_MAX_MS, + DISPLAY_RETRY_BASE_MS * 2 ** Math.min(Math.max(0, attempt), 16), + ); + return Math.round(step / 2 + (step / 2) * Math.min(1, Math.max(0, random()))); +} diff --git a/src/display/controller.ts b/src/display/controller.ts index 35bd1aa..7a5be54 100644 --- a/src/display/controller.ts +++ b/src/display/controller.ts @@ -1,4 +1,5 @@ import type { ModuleUiHostContext } from "@ekklesia/module-ui-sdk"; +import { displayRetryDelayMs } from "../browser/backoff.js"; import { type OfflineSnapshot, resolveOffline, @@ -140,6 +141,13 @@ export function createDisplayController(options: DisplayControllerOptions): Disp let snapshot: OfflineSnapshot | undefined; let offlineActive = false; let offlineHandle: unknown; + /** Retries, each on its own backoff. A display has nobody to press Retry, so none of them give up; + * each resets once its request works again. */ + let sessionHandle: unknown; + let sessionAttempt = 0; + let refreshAttempt = 0; + let realtimeHandle: unknown; + let realtimeAttempt = 0; /** Selections this display has tried and failed to render. A broken item is skipped rather than * retried in a tight loop, which is what keeps one bad slide from becoming a flicker. */ const unusable = new Set(); @@ -515,19 +523,118 @@ export function createDisplayController(options: DisplayControllerOptions): Disp `channels/${session.channelId}/display-state`, ); if (!response.ok) { - online = false; - await runOffline(); + await refreshFailed(); return; } - await apply((await response.json()) as DisplayChannelStateView); + const state = (await response.json()) as DisplayChannelStateView; + refreshAttempt = 0; + await apply(state); } catch { - // The server is unreachable. The frame stays and local continuation takes over; nothing about - // a network failure belongs on a lobby wall. - online = false; - await runOffline(); + await refreshFailed(); } } + /** + * The server is unreachable, or answered with an error. The frame stays and local continuation + * takes over — nothing about a network failure belongs on a lobby wall — and the refresh is tried + * again on its own, because a realtime message that would otherwise prompt it may never come. + */ + async function refreshFailed(): Promise { + online = false; + await runOffline(); + if (stopped) { + return; + } + if (refreshHandle !== undefined) { + clearTimer(refreshHandle); + } + refreshHandle = setTimer(() => { + refreshHandle = undefined; + void refresh(); + }, displayRetryDelayMs(refreshAttempt)); + refreshAttempt += 1; + } + + /** + * Registers this display, and keeps trying until it can. + * + * Until it has registered, a display continues from its own persisted state, honestly marked as + * offline. The first registration that works brings it live — realtime, heartbeats, and fresh + * state — exactly as if it had worked the first time. + */ + async function register(): Promise { + if (stopped || session) { + return; + } + let registered: DisplaySessionView | undefined; + try { + const response = await options.context.api.request("viewers/session", { + body: JSON.stringify({ viewerKey: await viewerKey() }), + headers: { "content-type": "application/json" }, + method: "POST", + }); + if (response.ok) { + registered = (await response.json()) as DisplaySessionView; + } + } catch { + registered = undefined; + } + if (stopped) { + return; + } + if (!registered) { + online = false; + report(); + sessionHandle = setTimer(() => { + sessionHandle = undefined; + void register(); + }, displayRetryDelayMs(sessionAttempt)); + sessionAttempt += 1; + return; + } + sessionAttempt = 0; + session = registered; + online = true; + subscribe(); + heartbeatHandle = setInterval( + () => void heartbeat(), + registered.heartbeatIntervalMs, + ) as unknown; + await refresh(); + void heartbeat(); + report(); + } + + /** + * Subscribes to the display's realtime channel, and again whenever the host reports that a + * subscription has ended for good. The host retries ordinary disconnects itself; a stream that + * is refused outright ends, and until this resubscribes the display would only learn of changes + * at its next slot deadline. + */ + function subscribe(): void { + if (stopped || !session) { + return; + } + subscription?.close(); + subscription = options.context.realtime.subscribe("display-state", { + onSnapshot: (state) => onRealtime(state), + onStatus: (status) => { + online = status === "connected"; + if (status === "connected") { + realtimeAttempt = 0; + } else if (status === "closed" && !stopped && realtimeHandle === undefined) { + realtimeHandle = setTimer(() => { + realtimeHandle = undefined; + subscribe(); + }, displayRetryDelayMs(realtimeAttempt)); + realtimeAttempt += 1; + } + report(); + }, + onUpdate: (state) => onRealtime(state), + }); + } + async function heartbeat(): Promise { if (stopped || !session || !telemetry) { return; @@ -595,36 +702,7 @@ export function createDisplayController(options: DisplayControllerOptions): Disp } if (telemetry) { - try { - const response = await options.context.api.request("viewers/session", { - body: JSON.stringify({ viewerKey: await viewerKey() }), - headers: { "content-type": "application/json" }, - method: "POST", - }); - if (response.ok) { - session = (await response.json()) as DisplaySessionView; - online = true; - } - } catch { - online = false; - } - } - - if (session) { - subscription = options.context.realtime.subscribe("display-state", { - onSnapshot: (state) => onRealtime(state), - onStatus: (status) => { - online = status === "connected"; - report(); - }, - onUpdate: (state) => onRealtime(state), - }); - heartbeatHandle = setInterval( - () => void heartbeat(), - session.heartbeatIntervalMs, - ) as unknown; - await refresh(); - void heartbeat(); + await register(); } report(); }, @@ -641,6 +719,13 @@ export function createDisplayController(options: DisplayControllerOptions): Disp clearTimer(refreshHandle); refreshHandle = undefined; } + for (const handle of [sessionHandle, realtimeHandle]) { + if (handle !== undefined) { + clearTimer(handle); + } + } + sessionHandle = undefined; + realtimeHandle = undefined; exitOffline(); await stage?.destroy(); stage = undefined; diff --git a/tests/browser/backoff.test.ts b/tests/browser/backoff.test.ts new file mode 100644 index 0000000..550ab7a --- /dev/null +++ b/tests/browser/backoff.test.ts @@ -0,0 +1,34 @@ +import { describe, expect, it } from "vitest"; +import { + DISPLAY_RETRY_BASE_MS, + DISPLAY_RETRY_MAX_MS, + displayRetryDelayMs, +} from "../../src/browser/backoff.js"; + +describe("display retry backoff", () => { + it("doubles from one second and stops at one minute", () => { + const longest = (attempt: number) => displayRetryDelayMs(attempt, () => 1); + expect([0, 1, 2, 3, 4, 5, 6, 7, 50].map(longest)).toEqual([ + 1_000, 2_000, 4_000, 8_000, 16_000, 32_000, 60_000, 60_000, 60_000, + ]); + }); + + it("jitters between half and all of each step, and never waits zero", () => { + for (const attempt of [0, 3, 6, 40]) { + const step = Math.min(DISPLAY_RETRY_MAX_MS, DISPLAY_RETRY_BASE_MS * 2 ** attempt); + expect(displayRetryDelayMs(attempt, () => 0)).toBe(step / 2); + expect(displayRetryDelayMs(attempt, () => 1)).toBe(step); + for (let index = 0; index < 50; index += 1) { + const delay = displayRetryDelayMs(attempt); + expect(delay).toBeGreaterThanOrEqual(step / 2); + expect(delay).toBeLessThanOrEqual(step); + } + } + }); + + it("treats a nonsensical attempt or random value as the nearest sensible one", () => { + expect(displayRetryDelayMs(-3, () => 1)).toBe(1_000); + expect(displayRetryDelayMs(2, () => 7)).toBe(4_000); + expect(displayRetryDelayMs(2, () => -1)).toBe(2_000); + }); +}); diff --git a/tests/browser/display-recovery.test.ts b/tests/browser/display-recovery.test.ts index cdef776..c138491 100644 --- a/tests/browser/display-recovery.test.ts +++ b/tests/browser/display-recovery.test.ts @@ -248,8 +248,9 @@ describe("a display recovers from every failed request without a reload", () => const display = build(); await display.start(); await settle(); - // An hour of virtual time: enough for every backoff step, and the cap, to have been reached. - await advance(60 * 60_000); + // Eight failures back off for at most 1 + 2 + 4 + 8 + 16 + 32 + 60 + 60 seconds, the last two at + // the one-minute ceiling. Five virtual minutes is enough for all of them and the success after. + await advance(5 * 60_000); expect(server.sessionRequests - sessionsBefore).toBe(9); expect(root.dataset.offline).toBe("false"); @@ -299,7 +300,7 @@ describe("a display recovers from every failed request without a reload", () => await display.stop(); const sessionsAtStop = server.sessionRequests; - await advance(60 * 60_000); + await advance(5 * 60_000); expect(server.sessionRequests).toBe(sessionsAtStop); });