fix(display): retry every failed request with backoff, and never give up

A display tried each of its network paths once:
- A registration that failed left it continuing from its persisted
  state, behind the offline banner, with no realtime, heartbeat or
  retry, until someone reloaded it. This is what the production kiosk
  showed after a restart.
- A failed state refresh scheduled no further refresh, so a display
  whose realtime was also down never fetched state again.
- A realtime stream the host reported closed was never subscribed
  again.

Each path now retries on its own backoff: about 1 s, doubling to a
one-minute ceiling, with jitter so a building's screens do not return
in lockstep. There is no attempt limit, because a television has
nobody to press Retry. Each resets when its request works again.
- Registration retries until it works, then brings the display live
  exactly as a first-time success does.
- A failed refresh schedules its own retry, with no realtime needed.
- A closed realtime subscription is replaced.
- Stopping clears every retry.

The recovery test's virtual window shrinks from an hour to five
minutes: eight failures back off for at most about 183 s, and an hour
of offline-continuation timers made the test slow without proving more.

Co-Authored-By: Claude Opus 5.5 <[email protected]>

src/browser/backoff.ts

@@ -0,0 +1,21 @@
+/**
+ * How long a display waits before trying a failed request again.
+ *
+ * Exponential from one second to a one-minute ceiling, so a screen whose server is restarting is
+ * back within seconds, and one whose server is down for an evening asks once a minute rather than
+ * hammering it. The wait is jittered between half and all of that step, so a building's worth of
+ * screens that lost the server at the same moment do not all come back at the same moment.
+ *
+ * There is no attempt limit. A television on a wall has nobody to press Retry, so giving up is the
+ * one thing it must never do.
+ */
+export const DISPLAY_RETRY_BASE_MS = 1_000;
+export const DISPLAY_RETRY_MAX_MS = 60_000;
+
+export function displayRetryDelayMs(attempt: number, random: () => number = Math.random): number {
+  const step = Math.min(
+    DISPLAY_RETRY_MAX_MS,
+    DISPLAY_RETRY_BASE_MS * 2 ** Math.min(Math.max(0, attempt), 16),
+  );
+  return Math.round(step / 2 + (step / 2) * Math.min(1, Math.max(0, random())));
+}

src/display/controller.ts

@@ -1,4 +1,5 @@
 import type { ModuleUiHostContext } from "@ekklesia/module-ui-sdk";
+import { displayRetryDelayMs } from "../browser/backoff.js";
 import {
   type OfflineSnapshot,
   resolveOffline,
@@ -140,6 +141,13 @@ export function createDisplayController(options: DisplayControllerOptions): Disp
   let snapshot: OfflineSnapshot | undefined;
   let offlineActive = false;
   let offlineHandle: unknown;
+  /** Retries, each on its own backoff. A display has nobody to press Retry, so none of them give up;
+   * each resets once its request works again. */
+  let sessionHandle: unknown;
+  let sessionAttempt = 0;
+  let refreshAttempt = 0;
+  let realtimeHandle: unknown;
+  let realtimeAttempt = 0;
   /** Selections this display has tried and failed to render. A broken item is skipped rather than
    * retried in a tight loop, which is what keeps one bad slide from becoming a flicker. */
   const unusable = new Set<string>();
@@ -515,19 +523,118 @@ export function createDisplayController(options: DisplayControllerOptions): Disp
         `channels/${session.channelId}/display-state`,
       );
       if (!response.ok) {
-        online = false;
-        await runOffline();
+        await refreshFailed();
         return;
       }
-      await apply((await response.json()) as DisplayChannelStateView);
+      const state = (await response.json()) as DisplayChannelStateView;
+      refreshAttempt = 0;
+      await apply(state);
     } catch {
-      // The server is unreachable. The frame stays and local continuation takes over; nothing about
-      // a network failure belongs on a lobby wall.
-      online = false;
-      await runOffline();
+      await refreshFailed();
     }
   }
 
+  /**
+   * The server is unreachable, or answered with an error. The frame stays and local continuation
+   * takes over — nothing about a network failure belongs on a lobby wall — and the refresh is tried
+   * again on its own, because a realtime message that would otherwise prompt it may never come.
+   */
+  async function refreshFailed(): Promise<void> {
+    online = false;
+    await runOffline();
+    if (stopped) {
+      return;
+    }
+    if (refreshHandle !== undefined) {
+      clearTimer(refreshHandle);
+    }
+    refreshHandle = setTimer(() => {
+      refreshHandle = undefined;
+      void refresh();
+    }, displayRetryDelayMs(refreshAttempt));
+    refreshAttempt += 1;
+  }
+
+  /**
+   * Registers this display, and keeps trying until it can.
+   *
+   * Until it has registered, a display continues from its own persisted state, honestly marked as
+   * offline. The first registration that works brings it live — realtime, heartbeats, and fresh
+   * state — exactly as if it had worked the first time.
+   */
+  async function register(): Promise<void> {
+    if (stopped || session) {
+      return;
+    }
+    let registered: DisplaySessionView | undefined;
+    try {
+      const response = await options.context.api.request("viewers/session", {
+        body: JSON.stringify({ viewerKey: await viewerKey() }),
+        headers: { "content-type": "application/json" },
+        method: "POST",
+      });
+      if (response.ok) {
+        registered = (await response.json()) as DisplaySessionView;
+      }
+    } catch {
+      registered = undefined;
+    }
+    if (stopped) {
+      return;
+    }
+    if (!registered) {
+      online = false;
+      report();
+      sessionHandle = setTimer(() => {
+        sessionHandle = undefined;
+        void register();
+      }, displayRetryDelayMs(sessionAttempt));
+      sessionAttempt += 1;
+      return;
+    }
+    sessionAttempt = 0;
+    session = registered;
+    online = true;
+    subscribe();
+    heartbeatHandle = setInterval(
+      () => void heartbeat(),
+      registered.heartbeatIntervalMs,
+    ) as unknown;
+    await refresh();
+    void heartbeat();
+    report();
+  }
+
+  /**
+   * Subscribes to the display's realtime channel, and again whenever the host reports that a
+   * subscription has ended for good. The host retries ordinary disconnects itself; a stream that
+   * is refused outright ends, and until this resubscribes the display would only learn of changes
+   * at its next slot deadline.
+   */
+  function subscribe(): void {
+    if (stopped || !session) {
+      return;
+    }
+    subscription?.close();
+    subscription = options.context.realtime.subscribe("display-state", {
+      onSnapshot: (state) => onRealtime(state),
+      onStatus: (status) => {
+        online = status === "connected";
+        if (status === "connected") {
+          realtimeAttempt = 0;
+        } else if (status === "closed" && !stopped && realtimeHandle === undefined) {
+          realtimeHandle = setTimer(() => {
+            realtimeHandle = undefined;
+            subscribe();
+          }, displayRetryDelayMs(realtimeAttempt));
+          realtimeAttempt += 1;
+        }
+        report();
+      },
+      onUpdate: (state) => onRealtime(state),
+    });
+  }
+
   async function heartbeat(): Promise<void> {
     if (stopped || !session || !telemetry) {
       return;
@@ -595,36 +702,7 @@ export function createDisplayController(options: DisplayControllerOptions): Disp
       }
 
       if (telemetry) {
-        try {
-          const response = await options.context.api.request("viewers/session", {
-            body: JSON.stringify({ viewerKey: await viewerKey() }),
-            headers: { "content-type": "application/json" },
-            method: "POST",
-          });
-          if (response.ok) {
-            session = (await response.json()) as DisplaySessionView;
-            online = true;
-          }
-        } catch {
-          online = false;
-        }
-      }
-
-      if (session) {
-        subscription = options.context.realtime.subscribe("display-state", {
-          onSnapshot: (state) => onRealtime(state),
-          onStatus: (status) => {
-            online = status === "connected";
-            report();
-          },
-          onUpdate: (state) => onRealtime(state),
-        });
-        heartbeatHandle = setInterval(
-          () => void heartbeat(),
-          session.heartbeatIntervalMs,
-        ) as unknown;
-        await refresh();
-        void heartbeat();
+        await register();
       }
       report();
     },
@@ -641,6 +719,13 @@ export function createDisplayController(options: DisplayControllerOptions): Disp
         clearTimer(refreshHandle);
         refreshHandle = undefined;
       }
+      for (const handle of [sessionHandle, realtimeHandle]) {
+        if (handle !== undefined) {
+          clearTimer(handle);
+        }
+      }
+      sessionHandle = undefined;
+      realtimeHandle = undefined;
       exitOffline();
       await stage?.destroy();
       stage = undefined;

tests/browser/backoff.test.ts

@@ -0,0 +1,34 @@
+import { describe, expect, it } from "vitest";
+import {
+  DISPLAY_RETRY_BASE_MS,
+  DISPLAY_RETRY_MAX_MS,
+  displayRetryDelayMs,
+} from "../../src/browser/backoff.js";
+
+describe("display retry backoff", () => {
+  it("doubles from one second and stops at one minute", () => {
+    const longest = (attempt: number) => displayRetryDelayMs(attempt, () => 1);
+    expect([0, 1, 2, 3, 4, 5, 6, 7, 50].map(longest)).toEqual([
+      1_000, 2_000, 4_000, 8_000, 16_000, 32_000, 60_000, 60_000, 60_000,
+    ]);
+  });
+
+  it("jitters between half and all of each step, and never waits zero", () => {
+    for (const attempt of [0, 3, 6, 40]) {
+      const step = Math.min(DISPLAY_RETRY_MAX_MS, DISPLAY_RETRY_BASE_MS * 2 ** attempt);
+      expect(displayRetryDelayMs(attempt, () => 0)).toBe(step / 2);
+      expect(displayRetryDelayMs(attempt, () => 1)).toBe(step);
+      for (let index = 0; index < 50; index += 1) {
+        const delay = displayRetryDelayMs(attempt);
+        expect(delay).toBeGreaterThanOrEqual(step / 2);
+        expect(delay).toBeLessThanOrEqual(step);
+      }
+    }
+  });
+
+  it("treats a nonsensical attempt or random value as the nearest sensible one", () => {
+    expect(displayRetryDelayMs(-3, () => 1)).toBe(1_000);
+    expect(displayRetryDelayMs(2, () => 7)).toBe(4_000);
+    expect(displayRetryDelayMs(2, () => -1)).toBe(2_000);
+  });
+});

tests/browser/display-recovery.test.ts

@@ -0,0 +1,307 @@
+// @vitest-environment happy-dom
+import type { ModuleUiHostContext } from "@ekklesia/module-ui-sdk";
+import { beforeEach, describe, expect, it } from "vitest";
+import {
+  createMemoryMediaCache,
+  createMemoryStore,
+  type SignageLocalStore,
+} from "../../src/browser/storage.js";
+import {
+  createDisplayController,
+  type DisplayChannelStateView,
+} from "../../src/display/controller.js";
+import { createFakeHost, type FakeHost, jsonResponse } from "./support/fake-host.js";
+
+/**
+ * A display never reaches a dead end.
+ *
+ * Every network path a display depends on — registering, fetching state, realtime — can fail once,
+ * as a kiosk that boots before its network settles or an API that restarts will make it. Each must
+ * be tried again, with backoff, until it works, and the display must then be exactly as live as if
+ * nothing had happened. None of it may need a person to reload a television.
+ *
+ * Time is virtual: every timer the controller schedules goes through the queue below.
+ */
+
+const SESSION = {
+  channelId: "chan-1",
+  channelKey: "default",
+  groupId: "group-1",
+  heartbeatIntervalMs: 12_000,
+  viewerId: "11111111-1111-4111-8111-111111111111",
+  viewerName: "Foyer",
+};
+
+function selection(id: string) {
+  return {
+    availableUntilMs: null,
+    deadlineAtMs: 5000,
+    durationMs: 5000,
+    id,
+    itemId: `image-${id}`,
+    itemRevision: "rev-1",
+    media: {
+      display: {
+        expiresAtMs: 900_000,
+        height: 1080,
+        url: `https://cdn.invalid/${id}`,
+        width: 1920,
+      },
+      thumbnail: {
+        expiresAtMs: 900_000,
+        height: 270,
+        url: `https://cdn.invalid/${id}-t`,
+        width: 480,
+      },
+    },
+    payload: { kind: "native-image" },
+    presentation: null,
+    providerId: "signage.native-images",
+    renderer: { kind: "native-image" as const },
+    revision: 1,
+    scene: {
+      backgroundColor: "#000000",
+      backgroundImage: false,
+      backgroundMode: "none" as const,
+      blurPixels: 0,
+      darkenOpacity: 0,
+      fit: "fill" as const,
+      shadow: false,
+    },
+    slotId: "slot-1",
+    stale: false,
+    startedAtMs: 0,
+    withdrawn: false,
+  };
+}
+
+function channelState(revision = 3): DisplayChannelStateView {
+  const current = selection("sel-1");
+  return {
+    channel: { groupId: "group-1", id: "chan-1", key: "default" },
+    crossfadeMs: 0,
+    current,
+    playback: {
+      activeTimelineId: "timeline-1",
+      blackout: false,
+      currentSelectionId: current.id,
+      currentSlotId: current.slotId,
+      deadlineAtMs: 5000,
+      pendingTimelineId: null,
+      remainingMs: 5000,
+      revision,
+      selectionRevision: 1,
+      startedAtMs: 0,
+      transport: "playing",
+    },
+    selections: [current],
+    serverNowMs: 0,
+    staticHold: null,
+    timeline: null,
+  };
+}
+
+/** Server behaviour that a test can change mid-run. */
+interface Server {
+  sessionFailures: number;
+  stateFailing: boolean;
+  sessionRequests: number;
+  stateRequests: number;
+}
+
+describe("a display recovers from every failed request without a reload", () => {
+  let clock: number;
+  let timers: Map<number, { at: number; run: () => void }>;
+  let nextTimer: number;
+  let root: HTMLElement;
+  let local: SignageLocalStore;
+  let server: Server;
+  let host: FakeHost;
+  let subscriptions: { onStatus?: (status: string) => void }[];
+
+  beforeEach(() => {
+    clock = 0;
+    timers = new Map();
+    nextTimer = 1;
+    root = document.createElement("div");
+    document.body.append(root);
+    local = createMemoryStore();
+    server = { sessionFailures: 0, sessionRequests: 0, stateFailing: false, stateRequests: 0 };
+    subscriptions = [];
+    host = createFakeHost({
+      permissions: ["signage.display.view"],
+      routes: [
+        {
+          method: "POST",
+          path: "viewers/session",
+          respond: () => {
+            server.sessionRequests += 1;
+            if (server.sessionFailures > 0) {
+              server.sessionFailures -= 1;
+              throw new TypeError("Failed to fetch");
+            }
+            return jsonResponse(SESSION);
+          },
+        },
+        {
+          method: "GET",
+          path: "channels/chan-1/display-state",
+          respond: () => {
+            server.stateRequests += 1;
+            if (server.stateFailing) {
+              throw new TypeError("Failed to fetch");
+            }
+            return jsonResponse(channelState());
+          },
+        },
+        { method: "POST", path: /^viewers\/.*\/heartbeat$/u, respond: () => jsonResponse({}) },
+      ],
+    });
+    Object.assign(host.context, {
+      realtime: {
+        subscribe: (_channel: string, handlers: { onStatus?: (status: string) => void }) => {
+          subscriptions.push(handlers);
+          return { close: () => undefined };
+        },
+      } satisfies Pick<ModuleUiHostContext["realtime"], "subscribe"> as never,
+    });
+  });
+
+  async function settle(): Promise<void> {
+    for (let index = 0; index < 20; index += 1) {
+      await new Promise((resolve) => setTimeout(resolve, 0));
+    }
+  }
+
+  /** Moves virtual time forward, firing every timer that falls due, in order. */
+  async function advance(ms: number): Promise<void> {
+    const end = clock + ms;
+    for (;;) {
+      const due = [...timers]
+        .filter(([, timer]) => timer.at <= end)
+        .sort((left, right) => left[1].at - right[1].at)[0];
+      if (!due) {
+        break;
+      }
+      timers.delete(due[0]);
+      clock = due[1].at;
+      due[1].run();
+      await settle();
+    }
+    clock = end;
+  }
+
+  function build() {
+    return createDisplayController({
+      clearTimer: (handle) => timers.delete(handle as number),
+      context: host.context,
+      decodeImage: () => Promise.resolve(),
+      media: createMemoryMediaCache(),
+      newViewerKey: () => "viewer-fixed-key-0001",
+      now: () => clock,
+      root,
+      setTimer: (run, delayMs) => {
+        const id = nextTimer;
+        nextTimer += 1;
+        timers.set(id, { at: clock + delayMs, run });
+        return id;
+      },
+      store: local,
+    });
+  }
+
+  /** One ordinary live run, so the display has a persisted snapshot to continue from, exactly as a
+   * screen that has been on the wall for a while does. */
+  async function runOnceLive(): Promise<void> {
+    const display = build();
+    await display.start();
+    await settle();
+    expect(root.dataset.offline).toBe("false");
+    await display.stop();
+    timers.clear();
+  }
+
+  it("registers once the server answers, after its first registration failed", async () => {
+    await runOnceLive();
+    server.sessionFailures = 1;
+    const sessionsBefore = server.sessionRequests;
+
+    const display = build();
+    await display.start();
+    await settle();
+    // Continuing from its own state while it cannot register: the honest banner.
+    expect(root.dataset.offline).toBe("true");
+
+    await advance(60_000);
+
+    expect(server.sessionRequests - sessionsBefore).toBeGreaterThanOrEqual(2);
+    expect(root.dataset.offline).toBe("false");
+    expect(subscriptions.length).toBeGreaterThanOrEqual(2);
+    await display.stop();
+  });
+
+  it("keeps retrying registration with backoff, and never gives up", async () => {
+    await runOnceLive();
+    server.sessionFailures = 8;
+    const sessionsBefore = server.sessionRequests;
+
+    const display = build();
+    await display.start();
+    await settle();
+    // Eight failures back off for at most 1 + 2 + 4 + 8 + 16 + 32 + 60 + 60 seconds, the last two at
+    // the one-minute ceiling. Five virtual minutes is enough for all of them and the success after.
+    await advance(5 * 60_000);
+
+    expect(server.sessionRequests - sessionsBefore).toBe(9);
+    expect(root.dataset.offline).toBe("false");
+    await display.stop();
+  });
+
+  it("retries a failed state refresh until the server answers, with no realtime nudge", async () => {
+    const display = build();
+    await display.start();
+    await settle();
+    expect(root.dataset.offline).toBe("false");
+
+    // The slot's deadline refresh fails, as it would mid-restart. Realtime says nothing.
+    server.stateFailing = true;
+    await advance(6_000);
+    expect(root.dataset.offline).toBe("true");
+    const failedRequests = server.stateRequests;
+
+    server.stateFailing = false;
+    await advance(120_000);
+
+    expect(server.stateRequests).toBeGreaterThan(failedRequests);
+    expect(root.dataset.offline).toBe("false");
+    await display.stop();
+  });
+
+  it("subscribes again when the host reports its realtime stream closed for good", async () => {
+    const display = build();
+    await display.start();
+    await settle();
+    expect(subscriptions).toHaveLength(1);
+
+    subscriptions[0]?.onStatus?.("closed");
+    await settle();
+    await advance(60_000);
+
+    expect(subscriptions.length).toBeGreaterThanOrEqual(2);
+    await display.stop();
+  });
+
+  it("schedules nothing after it is stopped", async () => {
+    await runOnceLive();
+    server.sessionFailures = 3;
+    const display = build();
+    await display.start();
+    await settle();
+    await display.stop();
+    const sessionsAtStop = server.sessionRequests;
+
+    await advance(5 * 60_000);
+
+    expect(server.sessionRequests).toBe(sessionsAtStop);
+  });
+});