Merge pull request 'fix(display): retry every failed request with backoff, and never give up' (#5) from fix/display-recovery into main

Reviewed-on: https://git.riverofglorymn.org/River_of_Glory_Church/Ekklesia_Signage/pulls/5
Reviewed-by: The_River <[email protected]>

ci/display-scenarios

@@ -2,3 +2,5 @@
 # (scripts/lib/deploy-smoke-display.mjs in the pinned platform). Pairing a kiosk and restarting it on
 # the same profile always run first. A change that makes another scenario pass adds it here.
 two-pages
+session-blip
+stream-failure

src/browser/backoff.ts

@@ -0,0 +1,21 @@
+/**
+ * How long a display waits before trying a failed request again.
+ *
+ * Exponential from one second to a one-minute ceiling, so a screen whose server is restarting is
+ * back within seconds, and one whose server is down for an evening asks once a minute rather than
+ * hammering it. The wait is jittered between half and all of that step, so a building's worth of
+ * screens that lost the server at the same moment do not all come back at the same moment.
+ *
+ * There is no attempt limit. A television on a wall has nobody to press Retry, so giving up is the
+ * one thing it must never do.
+ */
+export const DISPLAY_RETRY_BASE_MS = 1_000;
+export const DISPLAY_RETRY_MAX_MS = 60_000;
+
+export function displayRetryDelayMs(attempt: number, random: () => number = Math.random): number {
+  const step = Math.min(
+    DISPLAY_RETRY_MAX_MS,
+    DISPLAY_RETRY_BASE_MS * 2 ** Math.min(Math.max(0, attempt), 16),
+  );
+  return Math.round(step / 2 + (step / 2) * Math.min(1, Math.max(0, random())));
+}

src/display/controller.ts

@@ -1,4 +1,5 @@
 import type { ModuleUiHostContext } from "@ekklesia/module-ui-sdk";
+import { displayRetryDelayMs } from "../browser/backoff.js";
 import {
   type OfflineSnapshot,
   resolveOffline,
@@ -122,6 +123,13 @@ export function createDisplayController(options: DisplayControllerOptions): Disp
   let snapshot: OfflineSnapshot | undefined;
   let offlineActive = false;
   let offlineHandle: unknown;
+  /** Retries, each on its own backoff. A display has nobody to press Retry, so none of them give up;
+   * each resets once its request works again. */
+  let sessionHandle: unknown;
+  let sessionAttempt = 0;
+  let refreshAttempt = 0;
+  let realtimeHandle: unknown;
+  let realtimeAttempt = 0;
   /** Selections this display has tried and failed to render. A broken item is skipped rather than
    * retried in a tight loop, which is what keeps one bad slide from becoming a flicker. */
   const unusable = new Set<string>();
@@ -497,19 +505,121 @@ export function createDisplayController(options: DisplayControllerOptions): Disp
         `channels/${session.channelId}/display-state`,
       );
       if (!response.ok) {
-        online = false;
-        await runOffline();
+        await refreshFailed();
         return;
       }
-      await apply((await response.json()) as DisplayChannelStateView);
+      const state = (await response.json()) as DisplayChannelStateView;
+      refreshAttempt = 0;
+      await apply(state);
     } catch {
-      // The server is unreachable. The frame stays and local continuation takes over; nothing about
-      // a network failure belongs on a lobby wall.
-      online = false;
-      await runOffline();
+      await refreshFailed();
     }
   }
 
+  /**
+   * The server is unreachable, or answered with an error. The frame stays and local continuation
+   * takes over — nothing about a network failure belongs on a lobby wall — and the refresh is tried
+   * again on its own, because a realtime message that would otherwise prompt it may never come.
+   */
+  async function refreshFailed(): Promise<void> {
+    online = false;
+    await runOffline();
+    if (stopped) {
+      return;
+    }
+    if (refreshHandle !== undefined) {
+      clearTimer(refreshHandle);
+    }
+    refreshHandle = setTimer(() => {
+      refreshHandle = undefined;
+      void refresh();
+    }, displayRetryDelayMs(refreshAttempt));
+    refreshAttempt += 1;
+  }
+
+  /**
+   * Registers this display, and keeps trying until it can.
+   *
+   * Until it has registered, a display continues from its own persisted state, honestly marked as
+   * offline. The first registration that works brings it live — realtime, heartbeats, and fresh
+   * state — exactly as if it had worked the first time.
+   */
+  async function register(): Promise<void> {
+    if (stopped || session) {
+      return;
+    }
+    let registered: DisplaySessionView | undefined;
+    try {
+      const response = await options.context.api.request("viewers/session", {
+        body: JSON.stringify({ viewerKey: await viewerKey() }),
+        headers: { "content-type": "application/json" },
+        method: "POST",
+      });
+      if (response.ok) {
+        // Read, not cast: a registration the display cannot understand is no registration, and is
+        // tried again like any other failure.
+        const parsed = parseViewerSessionResponse(await response.json());
+        registered = parsed.ok ? parsed.value : undefined;
+      }
+    } catch {
+      registered = undefined;
+    }
+    if (stopped) {
+      return;
+    }
+    if (!registered) {
+      online = false;
+      report();
+      sessionHandle = setTimer(() => {
+        sessionHandle = undefined;
+        void register();
+      }, displayRetryDelayMs(sessionAttempt));
+      sessionAttempt += 1;
+      return;
+    }
+    sessionAttempt = 0;
+    session = registered;
+    online = true;
+    subscribe();
+    heartbeatHandle = setInterval(
+      () => void heartbeat(),
+      registered.heartbeatIntervalMs,
+    ) as unknown;
+    await refresh();
+    void heartbeat();
+    report();
+  }
+
+  /**
+   * Subscribes to the display's realtime channel, and again whenever the host reports that a
+   * subscription has ended for good. The host retries ordinary disconnects itself; a stream that
+   * is refused outright ends, and until this resubscribes the display would only learn of changes
+   * at its next slot deadline.
+   */
+  function subscribe(): void {
+    if (stopped || !session) {
+      return;
+    }
+    subscription?.close();
+    subscription = options.context.realtime.subscribe("display-state", {
+      onSnapshot: (state) => onRealtime(state),
+      onStatus: (status) => {
+        online = status === "connected";
+        if (status === "connected") {
+          realtimeAttempt = 0;
+        } else if (status === "closed" && !stopped && realtimeHandle === undefined) {
+          realtimeHandle = setTimer(() => {
+            realtimeHandle = undefined;
+            subscribe();
+          }, displayRetryDelayMs(realtimeAttempt));
+          realtimeAttempt += 1;
+        }
+        report();
+      },
+      onUpdate: (state) => onRealtime(state),
+    });
+  }
+
   async function heartbeat(): Promise<void> {
     if (stopped || !session || !telemetry) {
       return;
@@ -578,40 +688,7 @@ export function createDisplayController(options: DisplayControllerOptions): Disp
       }
 
       if (telemetry) {
-        try {
-          const response = await options.context.api.request("viewers/session", {
-            body: JSON.stringify({ viewerKey: await viewerKey() }),
-            headers: { "content-type": "application/json" },
-            method: "POST",
-          });
-          if (response.ok) {
-            // Read, not cast: a registration the display cannot understand is no registration.
-            const parsed = parseViewerSessionResponse(await response.json());
-            if (parsed.ok) {
-              session = parsed.value;
-              online = true;
-            }
-          }
-        } catch {
-          online = false;
-        }
-      }
-
-      if (session) {
-        subscription = options.context.realtime.subscribe("display-state", {
-          onSnapshot: (state) => onRealtime(state),
-          onStatus: (status) => {
-            online = status === "connected";
-            report();
-          },
-          onUpdate: (state) => onRealtime(state),
-        });
-        heartbeatHandle = setInterval(
-          () => void heartbeat(),
-          session.heartbeatIntervalMs,
-        ) as unknown;
-        await refresh();
-        void heartbeat();
+        await register();
       }
       report();
     },
@@ -628,6 +705,13 @@ export function createDisplayController(options: DisplayControllerOptions): Disp
         clearTimer(refreshHandle);
         refreshHandle = undefined;
       }
+      for (const handle of [sessionHandle, realtimeHandle]) {
+        if (handle !== undefined) {
+          clearTimer(handle);
+        }
+      }
+      sessionHandle = undefined;
+      realtimeHandle = undefined;
       exitOffline();
       await stage?.destroy();
       stage = undefined;

tests/browser/backoff.test.ts

@@ -0,0 +1,34 @@
+import { describe, expect, it } from "vitest";
+import {
+  DISPLAY_RETRY_BASE_MS,
+  DISPLAY_RETRY_MAX_MS,
+  displayRetryDelayMs,
+} from "../../src/browser/backoff.js";
+
+describe("display retry backoff", () => {
+  it("doubles from one second and stops at one minute", () => {
+    const longest = (attempt: number) => displayRetryDelayMs(attempt, () => 1);
+    expect([0, 1, 2, 3, 4, 5, 6, 7, 50].map(longest)).toEqual([
+      1_000, 2_000, 4_000, 8_000, 16_000, 32_000, 60_000, 60_000, 60_000,
+    ]);
+  });
+
+  it("jitters between half and all of each step, and never waits zero", () => {
+    for (const attempt of [0, 3, 6, 40]) {
+      const step = Math.min(DISPLAY_RETRY_MAX_MS, DISPLAY_RETRY_BASE_MS * 2 ** attempt);
+      expect(displayRetryDelayMs(attempt, () => 0)).toBe(step / 2);
+      expect(displayRetryDelayMs(attempt, () => 1)).toBe(step);
+      for (let index = 0; index < 50; index += 1) {
+        const delay = displayRetryDelayMs(attempt);
+        expect(delay).toBeGreaterThanOrEqual(step / 2);
+        expect(delay).toBeLessThanOrEqual(step);
+      }
+    }
+  });
+
+  it("treats a nonsensical attempt or random value as the nearest sensible one", () => {
+    expect(displayRetryDelayMs(-3, () => 1)).toBe(1_000);
+    expect(displayRetryDelayMs(2, () => 7)).toBe(4_000);
+    expect(displayRetryDelayMs(2, () => -1)).toBe(2_000);
+  });
+});

tests/browser/display-recovery.test.ts

@@ -0,0 +1,307 @@
+// @vitest-environment happy-dom
+import type { ModuleUiHostContext } from "@ekklesia/module-ui-sdk";
+import { beforeEach, describe, expect, it } from "vitest";
+import {
+  createMemoryMediaCache,
+  createMemoryStore,
+  type SignageLocalStore,
+} from "../../src/browser/storage.js";
+import {
+  createDisplayController,
+  type DisplayChannelStateView,
+} from "../../src/display/controller.js";
+import { createFakeHost, type FakeHost, jsonResponse } from "./support/fake-host.js";
+
+/**
+ * A display never reaches a dead end.
+ *
+ * Every network path a display depends on — registering, fetching state, realtime — can fail once,
+ * as a kiosk that boots before its network settles or an API that restarts will make it. Each must
+ * be tried again, with backoff, until it works, and the display must then be exactly as live as if
+ * nothing had happened. None of it may need a person to reload a television.
+ *
+ * Time is virtual: every timer the controller schedules goes through the queue below.
+ */
+
+const SESSION = {
+  channelId: "chan-1",
+  channelKey: "default",
+  groupId: "group-1",
+  heartbeatIntervalMs: 12_000,
+  viewerId: "11111111-1111-4111-8111-111111111111",
+  viewerName: "Foyer",
+};
+
+function selection(id: string) {
+  return {
+    availableUntilMs: null,
+    deadlineAtMs: 5000,
+    durationMs: 5000,
+    id,
+    itemId: `image-${id}`,
+    itemRevision: "rev-1",
+    media: {
+      display: {
+        expiresAtMs: 900_000,
+        height: 1080,
+        url: `https://cdn.invalid/${id}`,
+        width: 1920,
+      },
+      thumbnail: {
+        expiresAtMs: 900_000,
+        height: 270,
+        url: `https://cdn.invalid/${id}-t`,
+        width: 480,
+      },
+    },
+    payload: { kind: "native-image" },
+    presentation: null,
+    providerId: "signage.native-images",
+    renderer: { kind: "native-image" as const },
+    revision: 1,
+    scene: {
+      backgroundColor: "#000000",
+      backgroundImage: false,
+      backgroundMode: "none" as const,
+      blurPixels: 0,
+      darkenOpacity: 0,
+      fit: "fill" as const,
+      shadow: false,
+    },
+    slotId: "slot-1",
+    stale: false,
+    startedAtMs: 0,
+    withdrawn: false,
+  };
+}
+
+function channelState(revision = 3): DisplayChannelStateView {
+  const current = selection("sel-1");
+  return {
+    channel: { groupId: "group-1", id: "chan-1", key: "default" },
+    crossfadeMs: 0,
+    current,
+    playback: {
+      activeTimelineId: "timeline-1",
+      blackout: false,
+      currentSelectionId: current.id,
+      currentSlotId: current.slotId,
+      deadlineAtMs: 5000,
+      pendingTimelineId: null,
+      remainingMs: 5000,
+      revision,
+      selectionRevision: 1,
+      startedAtMs: 0,
+      transport: "playing",
+    },
+    selections: [current],
+    serverNowMs: 0,
+    staticHold: null,
+    timeline: null,
+  };
+}
+
+/** Server behaviour that a test can change mid-run. */
+interface Server {
+  sessionFailures: number;
+  stateFailing: boolean;
+  sessionRequests: number;
+  stateRequests: number;
+}
+
+describe("a display recovers from every failed request without a reload", () => {
+  let clock: number;
+  let timers: Map<number, { at: number; run: () => void }>;
+  let nextTimer: number;
+  let root: HTMLElement;
+  let local: SignageLocalStore;
+  let server: Server;
+  let host: FakeHost;
+  let subscriptions: { onStatus?: (status: string) => void }[];
+
+  beforeEach(() => {
+    clock = 0;
+    timers = new Map();
+    nextTimer = 1;
+    root = document.createElement("div");
+    document.body.append(root);
+    local = createMemoryStore();
+    server = { sessionFailures: 0, sessionRequests: 0, stateFailing: false, stateRequests: 0 };
+    subscriptions = [];
+    host = createFakeHost({
+      permissions: ["signage.display.view"],
+      routes: [
+        {
+          method: "POST",
+          path: "viewers/session",
+          respond: () => {
+            server.sessionRequests += 1;
+            if (server.sessionFailures > 0) {
+              server.sessionFailures -= 1;
+              throw new TypeError("Failed to fetch");
+            }
+            return jsonResponse(SESSION);
+          },
+        },
+        {
+          method: "GET",
+          path: "channels/chan-1/display-state",
+          respond: () => {
+            server.stateRequests += 1;
+            if (server.stateFailing) {
+              throw new TypeError("Failed to fetch");
+            }
+            return jsonResponse(channelState());
+          },
+        },
+        { method: "POST", path: /^viewers\/.*\/heartbeat$/u, respond: () => jsonResponse({}) },
+      ],
+    });
+    Object.assign(host.context, {
+      realtime: {
+        subscribe: (_channel: string, handlers: { onStatus?: (status: string) => void }) => {
+          subscriptions.push(handlers);
+          return { close: () => undefined };
+        },
+      } satisfies Pick<ModuleUiHostContext["realtime"], "subscribe"> as never,
+    });
+  });
+
+  async function settle(): Promise<void> {
+    for (let index = 0; index < 20; index += 1) {
+      await new Promise((resolve) => setTimeout(resolve, 0));
+    }
+  }
+
+  /** Moves virtual time forward, firing every timer that falls due, in order. */
+  async function advance(ms: number): Promise<void> {
+    const end = clock + ms;
+    for (;;) {
+      const due = [...timers]
+        .filter(([, timer]) => timer.at <= end)
+        .sort((left, right) => left[1].at - right[1].at)[0];
+      if (!due) {
+        break;
+      }
+      timers.delete(due[0]);
+      clock = due[1].at;
+      due[1].run();
+      await settle();
+    }
+    clock = end;
+  }
+
+  function build() {
+    return createDisplayController({
+      clearTimer: (handle) => timers.delete(handle as number),
+      context: host.context,
+      decodeImage: () => Promise.resolve(),
+      media: createMemoryMediaCache(),
+      newViewerKey: () => "viewer-fixed-key-0001",
+      now: () => clock,
+      root,
+      setTimer: (run, delayMs) => {
+        const id = nextTimer;
+        nextTimer += 1;
+        timers.set(id, { at: clock + delayMs, run });
+        return id;
+      },
+      store: local,
+    });
+  }
+
+  /** One ordinary live run, so the display has a persisted snapshot to continue from, exactly as a
+   * screen that has been on the wall for a while does. */
+  async function runOnceLive(): Promise<void> {
+    const display = build();
+    await display.start();
+    await settle();
+    expect(root.dataset.offline).toBe("false");
+    await display.stop();
+    timers.clear();
+  }
+
+  it("registers once the server answers, after its first registration failed", async () => {
+    await runOnceLive();
+    server.sessionFailures = 1;
+    const sessionsBefore = server.sessionRequests;
+
+    const display = build();
+    await display.start();
+    await settle();
+    // Continuing from its own state while it cannot register: the honest banner.
+    expect(root.dataset.offline).toBe("true");
+
+    await advance(60_000);
+
+    expect(server.sessionRequests - sessionsBefore).toBeGreaterThanOrEqual(2);
+    expect(root.dataset.offline).toBe("false");
+    expect(subscriptions.length).toBeGreaterThanOrEqual(2);
+    await display.stop();
+  });
+
+  it("keeps retrying registration with backoff, and never gives up", async () => {
+    await runOnceLive();
+    server.sessionFailures = 8;
+    const sessionsBefore = server.sessionRequests;
+
+    const display = build();
+    await display.start();
+    await settle();
+    // Eight failures back off for at most 1 + 2 + 4 + 8 + 16 + 32 + 60 + 60 seconds, the last two at
+    // the one-minute ceiling. Five virtual minutes is enough for all of them and the success after.
+    await advance(5 * 60_000);
+
+    expect(server.sessionRequests - sessionsBefore).toBe(9);
+    expect(root.dataset.offline).toBe("false");
+    await display.stop();
+  });
+
+  it("retries a failed state refresh until the server answers, with no realtime nudge", async () => {
+    const display = build();
+    await display.start();
+    await settle();
+    expect(root.dataset.offline).toBe("false");
+
+    // The slot's deadline refresh fails, as it would mid-restart. Realtime says nothing.
+    server.stateFailing = true;
+    await advance(6_000);
+    expect(root.dataset.offline).toBe("true");
+    const failedRequests = server.stateRequests;
+
+    server.stateFailing = false;
+    await advance(120_000);
+
+    expect(server.stateRequests).toBeGreaterThan(failedRequests);
+    expect(root.dataset.offline).toBe("false");
+    await display.stop();
+  });
+
+  it("subscribes again when the host reports its realtime stream closed for good", async () => {
+    const display = build();
+    await display.start();
+    await settle();
+    expect(subscriptions).toHaveLength(1);
+
+    subscriptions[0]?.onStatus?.("closed");
+    await settle();
+    await advance(60_000);
+
+    expect(subscriptions.length).toBeGreaterThanOrEqual(2);
+    await display.stop();
+  });
+
+  it("schedules nothing after it is stopped", async () => {
+    await runOnceLive();
+    server.sessionFailures = 3;
+    const display = build();
+    await display.start();
+    await settle();
+    await display.stop();
+    const sessionsAtStop = server.sessionRequests;
+
+    await advance(5 * 60_000);
+
+    expect(server.sessionRequests).toBe(sessionsAtStop);
+  });
+});