fix(display): retry every failed request with backoff, and never give up

A display tried each of its network paths once:
- A registration that failed left it continuing from its persisted
  state, behind the offline banner, with no realtime, heartbeat or
  retry, until someone reloaded it. This is what the production kiosk
  showed after a restart.
- A failed state refresh scheduled no further refresh, so a display
  whose realtime was also down never fetched state again.
- A realtime stream the host reported closed was never subscribed
  again.

Each path now retries on its own backoff: about 1 s, doubling to a
one-minute ceiling, with jitter so a building's screens do not return
in lockstep. There is no attempt limit, because a television has
nobody to press Retry. Each resets when its request works again.
- Registration retries until it works, then brings the display live
  exactly as a first-time success does.
- A failed refresh schedules its own retry, with no realtime needed.
- A closed realtime subscription is replaced.
- Stopping clears every retry.

The recovery test's virtual window shrinks from an hour to five
minutes: eight failures back off for at most about 183 s, and an hour
of offline-continuation timers made the test slow without proving more.

Co-Authored-By: Claude Opus 5.5 <[email protected]>

src/browser/backoff.ts

@@ -0,0 +1,21 @@
+/**
+ * How long a display waits before trying a failed request again.
+ *
+ * Exponential from one second to a one-minute ceiling, so a screen whose server is restarting is
+ * back within seconds, and one whose server is down for an evening asks once a minute rather than
+ * hammering it. The wait is jittered between half and all of that step, so a building's worth of
+ * screens that lost the server at the same moment do not all come back at the same moment.
+ *
+ * There is no attempt limit. A television on a wall has nobody to press Retry, so giving up is the
+ * one thing it must never do.
+ */
+export const DISPLAY_RETRY_BASE_MS = 1_000;
+export const DISPLAY_RETRY_MAX_MS = 60_000;
+
+export function displayRetryDelayMs(attempt: number, random: () => number = Math.random): number {
+  const step = Math.min(
+    DISPLAY_RETRY_MAX_MS,
+    DISPLAY_RETRY_BASE_MS * 2 ** Math.min(Math.max(0, attempt), 16),
+  );
+  return Math.round(step / 2 + (step / 2) * Math.min(1, Math.max(0, random())));
+}

src/display/controller.ts

@@ -1,4 +1,5 @@
 import type { ModuleUiHostContext } from "@ekklesia/module-ui-sdk";
+import { displayRetryDelayMs } from "../browser/backoff.js";
 import {
   type OfflineSnapshot,
   resolveOffline,
@@ -122,6 +123,13 @@ export function createDisplayController(options: DisplayControllerOptions): Disp
   let snapshot: OfflineSnapshot | undefined;
   let offlineActive = false;
   let offlineHandle: unknown;
+  /** Retries, each on its own backoff. A display has nobody to press Retry, so none of them give up;
+   * each resets once its request works again. */
+  let sessionHandle: unknown;
+  let sessionAttempt = 0;
+  let refreshAttempt = 0;
+  let realtimeHandle: unknown;
+  let realtimeAttempt = 0;
   /** Selections this display has tried and failed to render. A broken item is skipped rather than
    * retried in a tight loop, which is what keeps one bad slide from becoming a flicker. */
   const unusable = new Set<string>();
@@ -497,17 +505,119 @@ export function createDisplayController(options: DisplayControllerOptions): Disp
         `channels/${session.channelId}/display-state`,
       );
       if (!response.ok) {
-        online = false;
+        await refreshFailed();
-        await runOffline();
         return;
       }
-      await apply((await response.json()) as DisplayChannelStateView);
+      const state = (await response.json()) as DisplayChannelStateView;
+      refreshAttempt = 0;
+      await apply(state);
     } catch {
-      // The server is unreachable. The frame stays and local continuation takes over; nothing about
+      await refreshFailed();
-      // a network failure belongs on a lobby wall.
+    }
+  }
+
+  /**
+   * The server is unreachable, or answered with an error. The frame stays and local continuation
+   * takes over — nothing about a network failure belongs on a lobby wall — and the refresh is tried
+   * again on its own, because a realtime message that would otherwise prompt it may never come.
+   */
+  async function refreshFailed(): Promise<void> {
     online = false;
     await runOffline();
+    if (stopped) {
+      return;
     }
+    if (refreshHandle !== undefined) {
+      clearTimer(refreshHandle);
+    }
+    refreshHandle = setTimer(() => {
+      refreshHandle = undefined;
+      void refresh();
+    }, displayRetryDelayMs(refreshAttempt));
+    refreshAttempt += 1;
+  }
+
+  /**
+   * Registers this display, and keeps trying until it can.
+   *
+   * Until it has registered, a display continues from its own persisted state, honestly marked as
+   * offline. The first registration that works brings it live — realtime, heartbeats, and fresh
+   * state — exactly as if it had worked the first time.
+   */
+  async function register(): Promise<void> {
+    if (stopped || session) {
+      return;
+    }
+    let registered: DisplaySessionView | undefined;
+    try {
+      const response = await options.context.api.request("viewers/session", {
+        body: JSON.stringify({ viewerKey: await viewerKey() }),
+        headers: { "content-type": "application/json" },
+        method: "POST",
+      });
+      if (response.ok) {
+        // Read, not cast: a registration the display cannot understand is no registration, and is
+        // tried again like any other failure.
+        const parsed = parseViewerSessionResponse(await response.json());
+        registered = parsed.ok ? parsed.value : undefined;
+      }
+    } catch {
+      registered = undefined;
+    }
+    if (stopped) {
+      return;
+    }
+    if (!registered) {
+      online = false;
+      report();
+      sessionHandle = setTimer(() => {
+        sessionHandle = undefined;
+        void register();
+      }, displayRetryDelayMs(sessionAttempt));
+      sessionAttempt += 1;
+      return;
+    }
+    sessionAttempt = 0;
+    session = registered;
+    online = true;
+    subscribe();
+    heartbeatHandle = setInterval(
+      () => void heartbeat(),
+      registered.heartbeatIntervalMs,
+    ) as unknown;
+    await refresh();
+    void heartbeat();
+    report();
+  }
+
+  /**
+   * Subscribes to the display's realtime channel, and again whenever the host reports that a
+   * subscription has ended for good. The host retries ordinary disconnects itself; a stream that
+   * is refused outright ends, and until this resubscribes the display would only learn of changes
+   * at its next slot deadline.
+   */
+  function subscribe(): void {
+    if (stopped || !session) {
+      return;
+    }
+    subscription?.close();
+    subscription = options.context.realtime.subscribe("display-state", {
+      onSnapshot: (state) => onRealtime(state),
+      onStatus: (status) => {
+        online = status === "connected";
+        if (status === "connected") {
+          realtimeAttempt = 0;
+        } else if (status === "closed" && !stopped && realtimeHandle === undefined) {
+          realtimeHandle = setTimer(() => {
+            realtimeHandle = undefined;
+            subscribe();
+          }, displayRetryDelayMs(realtimeAttempt));
+          realtimeAttempt += 1;
+        }
+        report();
+      },
+      onUpdate: (state) => onRealtime(state),
+    });
   }
 
   async function heartbeat(): Promise<void> {
@@ -578,40 +688,7 @@ export function createDisplayController(options: DisplayControllerOptions): Disp
       }
 
       if (telemetry) {
-        try {
+        await register();
-          const response = await options.context.api.request("viewers/session", {
-            body: JSON.stringify({ viewerKey: await viewerKey() }),
-            headers: { "content-type": "application/json" },
-            method: "POST",
-          });
-          if (response.ok) {
-            // Read, not cast: a registration the display cannot understand is no registration.
-            const parsed = parseViewerSessionResponse(await response.json());
-            if (parsed.ok) {
-              session = parsed.value;
-              online = true;
-            }
-          }
-        } catch {
-          online = false;
-        }
-      }
-
-      if (session) {
-        subscription = options.context.realtime.subscribe("display-state", {
-          onSnapshot: (state) => onRealtime(state),
-          onStatus: (status) => {
-            online = status === "connected";
-            report();
-          },
-          onUpdate: (state) => onRealtime(state),
-        });
-        heartbeatHandle = setInterval(
-          () => void heartbeat(),
-          session.heartbeatIntervalMs,
-        ) as unknown;
-        await refresh();
-        void heartbeat();
       }
       report();
     },
@@ -628,6 +705,13 @@ export function createDisplayController(options: DisplayControllerOptions): Disp
         clearTimer(refreshHandle);
         refreshHandle = undefined;
       }
+      for (const handle of [sessionHandle, realtimeHandle]) {
+        if (handle !== undefined) {
+          clearTimer(handle);
+        }
+      }
+      sessionHandle = undefined;
+      realtimeHandle = undefined;
       exitOffline();
       await stage?.destroy();
       stage = undefined;

tests/browser/backoff.test.ts

@@ -0,0 +1,34 @@
+import { describe, expect, it } from "vitest";
+import {
+  DISPLAY_RETRY_BASE_MS,
+  DISPLAY_RETRY_MAX_MS,
+  displayRetryDelayMs,
+} from "../../src/browser/backoff.js";
+
+describe("display retry backoff", () => {
+  it("doubles from one second and stops at one minute", () => {
+    const longest = (attempt: number) => displayRetryDelayMs(attempt, () => 1);
+    expect([0, 1, 2, 3, 4, 5, 6, 7, 50].map(longest)).toEqual([
+      1_000, 2_000, 4_000, 8_000, 16_000, 32_000, 60_000, 60_000, 60_000,
+    ]);
+  });
+
+  it("jitters between half and all of each step, and never waits zero", () => {
+    for (const attempt of [0, 3, 6, 40]) {
+      const step = Math.min(DISPLAY_RETRY_MAX_MS, DISPLAY_RETRY_BASE_MS * 2 ** attempt);
+      expect(displayRetryDelayMs(attempt, () => 0)).toBe(step / 2);
+      expect(displayRetryDelayMs(attempt, () => 1)).toBe(step);
+      for (let index = 0; index < 50; index += 1) {
+        const delay = displayRetryDelayMs(attempt);
+        expect(delay).toBeGreaterThanOrEqual(step / 2);
+        expect(delay).toBeLessThanOrEqual(step);
+      }
+    }
+  });
+
+  it("treats a nonsensical attempt or random value as the nearest sensible one", () => {
+    expect(displayRetryDelayMs(-3, () => 1)).toBe(1_000);
+    expect(displayRetryDelayMs(2, () => 7)).toBe(4_000);
+    expect(displayRetryDelayMs(2, () => -1)).toBe(2_000);
+  });
+});

tests/browser/display-recovery.test.ts

@@ -248,8 +248,9 @@ describe("a display recovers from every failed request without a reload", () =>
     const display = build();
     await display.start();
     await settle();
-    // An hour of virtual time: enough for every backoff step, and the cap, to have been reached.
+    // Eight failures back off for at most 1 + 2 + 4 + 8 + 16 + 32 + 60 + 60 seconds, the last two at
-    await advance(60 * 60_000);
+    // the one-minute ceiling. Five virtual minutes is enough for all of them and the success after.
+    await advance(5 * 60_000);
 
     expect(server.sessionRequests - sessionsBefore).toBe(9);
     expect(root.dataset.offline).toBe("false");
@@ -299,7 +300,7 @@ describe("a display recovers from every failed request without a reload", () =>
     await display.stop();
     const sessionsAtStop = server.sessionRequests;
 
-    await advance(60 * 60_000);
+    await advance(5 * 60_000);
 
     expect(server.sessionRequests).toBe(sessionsAtStop);
   });