src/browser/backoff.ts

@@ -0,0 +1,21 @@
+/**
+ * How long a display waits before trying a failed request again.
+ *
+ * Exponential from one second to a one-minute ceiling, so a screen whose server is restarting is
+ * back within seconds, and one whose server is down for an evening asks once a minute rather than
+ * hammering it. The wait is jittered between half and all of that step, so a building's worth of
+ * screens that lost the server at the same moment do not all come back at the same moment.
+ *
+ * There is no attempt limit. A television on a wall has nobody to press Retry, so giving up is the
+ * one thing it must never do.
+ */
+export const DISPLAY_RETRY_BASE_MS = 1_000;
+export const DISPLAY_RETRY_MAX_MS = 60_000;
+
+export function displayRetryDelayMs(attempt: number, random: () => number = Math.random): number {
+  const step = Math.min(
+    DISPLAY_RETRY_MAX_MS,
+    DISPLAY_RETRY_BASE_MS * 2 ** Math.min(Math.max(0, attempt), 16),
+  );
+  return Math.round(step / 2 + (step / 2) * Math.min(1, Math.max(0, random())));
+}

src/display/controller.ts

@@ -1,4 +1,5 @@
 import type { ModuleUiHostContext } from "@ekklesia/module-ui-sdk";
+import { displayRetryDelayMs } from "../browser/backoff.js";
 import {
   type OfflineSnapshot,
   resolveOffline,
@@ -122,6 +123,13 @@ export function createDisplayController(options: DisplayControllerOptions): Disp
   let snapshot: OfflineSnapshot | undefined;
   let offlineActive = false;
   let offlineHandle: unknown;
+  /** Retries, each on its own backoff. A display has nobody to press Retry, so none of them give up;
+   * each resets once its request works again. */
+  let sessionHandle: unknown;
+  let sessionAttempt = 0;
+  let refreshAttempt = 0;
+  let realtimeHandle: unknown;
+  let realtimeAttempt = 0;
   /** Selections this display has tried and failed to render. A broken item is skipped rather than
    * retried in a tight loop, which is what keeps one bad slide from becoming a flicker. */
   const unusable = new Set<string>();
@@ -497,17 +505,119 @@ export function createDisplayController(options: DisplayControllerOptions): Disp
         `channels/${session.channelId}/display-state`,
       );
       if (!response.ok) {
-        online = false;
+        await refreshFailed();
-        await runOffline();
         return;
       }
-      await apply((await response.json()) as DisplayChannelStateView);
+      const state = (await response.json()) as DisplayChannelStateView;
+      refreshAttempt = 0;
+      await apply(state);
     } catch {
-      // The server is unreachable. The frame stays and local continuation takes over; nothing about
+      await refreshFailed();
-      // a network failure belongs on a lobby wall.
+    }
+  }
+
+  /**
+   * The server is unreachable, or answered with an error. The frame stays and local continuation
+   * takes over — nothing about a network failure belongs on a lobby wall — and the refresh is tried
+   * again on its own, because a realtime message that would otherwise prompt it may never come.
+   */
+  async function refreshFailed(): Promise<void> {
     online = false;
     await runOffline();
+    if (stopped) {
+      return;
     }
+    if (refreshHandle !== undefined) {
+      clearTimer(refreshHandle);
+    }
+    refreshHandle = setTimer(() => {
+      refreshHandle = undefined;
+      void refresh();
+    }, displayRetryDelayMs(refreshAttempt));
+    refreshAttempt += 1;
+  }
+
+  /**
+   * Registers this display, and keeps trying until it can.
+   *
+   * Until it has registered, a display continues from its own persisted state, honestly marked as
+   * offline. The first registration that works brings it live — realtime, heartbeats, and fresh
+   * state — exactly as if it had worked the first time.
+   */
+  async function register(): Promise<void> {
+    if (stopped || session) {
+      return;
+    }
+    let registered: DisplaySessionView | undefined;
+    try {
+      const response = await options.context.api.request("viewers/session", {
+        body: JSON.stringify({ viewerKey: await viewerKey() }),
+        headers: { "content-type": "application/json" },
+        method: "POST",
+      });
+      if (response.ok) {
+        // Read, not cast: a registration the display cannot understand is no registration, and is
+        // tried again like any other failure.
+        const parsed = parseViewerSessionResponse(await response.json());
+        registered = parsed.ok ? parsed.value : undefined;
+      }
+    } catch {
+      registered = undefined;
+    }
+    if (stopped) {
+      return;
+    }
+    if (!registered) {
+      online = false;
+      report();
+      sessionHandle = setTimer(() => {
+        sessionHandle = undefined;
+        void register();
+      }, displayRetryDelayMs(sessionAttempt));
+      sessionAttempt += 1;
+      return;
+    }
+    sessionAttempt = 0;
+    session = registered;
+    online = true;
+    subscribe();
+    heartbeatHandle = setInterval(
+      () => void heartbeat(),
+      registered.heartbeatIntervalMs,
+    ) as unknown;
+    await refresh();
+    void heartbeat();
+    report();
+  }
+
+  /**
+   * Subscribes to the display's realtime channel, and again whenever the host reports that a
+   * subscription has ended for good. The host retries ordinary disconnects itself; a stream that
+   * is refused outright ends, and until this resubscribes the display would only learn of changes
+   * at its next slot deadline.
+   */
+  function subscribe(): void {
+    if (stopped || !session) {
+      return;
+    }
+    subscription?.close();
+    subscription = options.context.realtime.subscribe("display-state", {
+      onSnapshot: (state) => onRealtime(state),
+      onStatus: (status) => {
+        online = status === "connected";
+        if (status === "connected") {
+          realtimeAttempt = 0;
+        } else if (status === "closed" && !stopped && realtimeHandle === undefined) {
+          realtimeHandle = setTimer(() => {
+            realtimeHandle = undefined;
+            subscribe();
+          }, displayRetryDelayMs(realtimeAttempt));
+          realtimeAttempt += 1;
+        }
+        report();
+      },
+      onUpdate: (state) => onRealtime(state),
+    });
   }
 
   async function heartbeat(): Promise<void> {
@@ -578,40 +688,7 @@ export function createDisplayController(options: DisplayControllerOptions): Disp
       }
 
       if (telemetry) {
-        try {
+        await register();
-          const response = await options.context.api.request("viewers/session", {
-            body: JSON.stringify({ viewerKey: await viewerKey() }),
-            headers: { "content-type": "application/json" },
-            method: "POST",
-          });
-          if (response.ok) {
-            // Read, not cast: a registration the display cannot understand is no registration.
-            const parsed = parseViewerSessionResponse(await response.json());
-            if (parsed.ok) {
-              session = parsed.value;
-              online = true;
-            }
-          }
-        } catch {
-          online = false;
-        }
-      }
-
-      if (session) {
-        subscription = options.context.realtime.subscribe("display-state", {
-          onSnapshot: (state) => onRealtime(state),
-          onStatus: (status) => {
-            online = status === "connected";
-            report();
-          },
-          onUpdate: (state) => onRealtime(state),
-        });
-        heartbeatHandle = setInterval(
-          () => void heartbeat(),
-          session.heartbeatIntervalMs,
-        ) as unknown;
-        await refresh();
-        void heartbeat();
       }
       report();
     },
@@ -628,6 +705,13 @@ export function createDisplayController(options: DisplayControllerOptions): Disp
         clearTimer(refreshHandle);
         refreshHandle = undefined;
       }
+      for (const handle of [sessionHandle, realtimeHandle]) {
+        if (handle !== undefined) {
+          clearTimer(handle);
+        }
+      }
+      sessionHandle = undefined;
+      realtimeHandle = undefined;
       exitOffline();
       await stage?.destroy();
       stage = undefined;

tests/browser/backoff.test.ts

@@ -0,0 +1,34 @@
+import { describe, expect, it } from "vitest";
+import {
+  DISPLAY_RETRY_BASE_MS,
+  DISPLAY_RETRY_MAX_MS,
+  displayRetryDelayMs,
+} from "../../src/browser/backoff.js";
+
+describe("display retry backoff", () => {
+  it("doubles from one second and stops at one minute", () => {
+    const longest = (attempt: number) => displayRetryDelayMs(attempt, () => 1);
+    expect([0, 1, 2, 3, 4, 5, 6, 7, 50].map(longest)).toEqual([
+      1_000, 2_000, 4_000, 8_000, 16_000, 32_000, 60_000, 60_000, 60_000,
+    ]);
+  });
+
+  it("jitters between half and all of each step, and never waits zero", () => {
+    for (const attempt of [0, 3, 6, 40]) {
+      const step = Math.min(DISPLAY_RETRY_MAX_MS, DISPLAY_RETRY_BASE_MS * 2 ** attempt);
+      expect(displayRetryDelayMs(attempt, () => 0)).toBe(step / 2);
+      expect(displayRetryDelayMs(attempt, () => 1)).toBe(step);
+      for (let index = 0; index < 50; index += 1) {
+        const delay = displayRetryDelayMs(attempt);
+        expect(delay).toBeGreaterThanOrEqual(step / 2);
+        expect(delay).toBeLessThanOrEqual(step);
+      }
+    }
+  });
+
+  it("treats a nonsensical attempt or random value as the nearest sensible one", () => {
+    expect(displayRetryDelayMs(-3, () => 1)).toBe(1_000);
+    expect(displayRetryDelayMs(2, () => 7)).toBe(4_000);
+    expect(displayRetryDelayMs(2, () => -1)).toBe(2_000);
+  });
+});

tests/browser/display-recovery.test.ts

@@ -248,8 +248,9 @@ describe("a display recovers from every failed request without a reload", () =>
     const display = build();
     await display.start();
     await settle();
-    // An hour of virtual time: enough for every backoff step, and the cap, to have been reached.
+    // Eight failures back off for at most 1 + 2 + 4 + 8 + 16 + 32 + 60 + 60 seconds, the last two at
-    await advance(60 * 60_000);
+    // the one-minute ceiling. Five virtual minutes is enough for all of them and the success after.
+    await advance(5 * 60_000);
 
     expect(server.sessionRequests - sessionsBefore).toBe(9);
     expect(root.dataset.offline).toBe("false");
@@ -299,7 +300,7 @@ describe("a display recovers from every failed request without a reload", () =>
     await display.stop();
     const sessionsAtStop = server.sessionRequests;
 
-    await advance(60 * 60_000);
+    await advance(5 * 60_000);
 
     expect(server.sessionRequests).toBe(sessionsAtStop);
   });