import { expect, test } from "@playwright/test"; import { ensureFixtures } from "./fixtures.js"; import { activatePlaylist, api, resetPlayback } from "./helpers.js"; import { BASE_URL, cachedMedia, diagnostics, launchKiosk, opaqueLayerCount, waitForCachedAssets, waitForSlide, } from "./offline-helpers.js"; import { CONTAINERS, start, stop, waitUntilHealthy, waitUntilUnreachable } from "./stack.js"; // These scenarios remove real infrastructure, so they must not interleave. test.describe.configure({ mode: "serial" }); test.use({ baseURL: BASE_URL }); async function restoreStack() { await start(CONTAINERS.storage, CONTAINERS.api, CONTAINERS.web); await waitUntilHealthy(BASE_URL); } test.afterAll(async () => { await restoreStack(); }); test("cached rotation continues through a complete server outage", async ({ request }) => { const library = await ensureFixtures(request); await resetPlayback(request); await activatePlaylist(request, "Outage", [ { assetId: library["wide-a"].id, durationOverrideMs: 3_000 }, { assetId: library["wide-b"].id, durationOverrideMs: 9_000 }, { assetId: library["wide-c"].id, durationOverrideMs: 4_000 }, ]); const { context, page } = await launchKiosk(); try { await page.goto(`${BASE_URL}/display`); await waitForSlide(page); // Every slide of this playlist must be cached before the outage begins. await waitForCachedAssets(page, [ library["wide-a"].id, library["wide-b"].id, library["wide-c"].id, ]); await stop(CONTAINERS.api, CONTAINERS.storage); await waitUntilUnreachable(BASE_URL); // Sample continuously across multiple full cycles with nothing serving. const seen = new Set(); const starts = new Map(); const deadline = Date.now() + 40_000; while (Date.now() < deadline) { const state = await page.evaluate(() => ({ rendered: window.__display?.diagnostics()?.renderedAssetId ?? null, offline: window.__display?.diagnostics()?.offlineActive ?? false, opaque: [...document.querySelectorAll(".scene")].filter( (layer) => Number(getComputedStyle(layer).opacity) > 0.99, ).length, gate: document.querySelector(".output-gate").dataset.active, })); // The original black-frame bug must remain impossible offline too. expect(state.opaque, "a good frame is always on screen offline").toBeGreaterThanOrEqual(1); expect(state.gate).toBe("false"); if (state.rendered) { seen.add(state.rendered); if (!starts.has(state.rendered)) starts.set(state.rendered, Date.now()); } await page.waitForTimeout(150); } expect(await diagnostics(page).then((d) => d.offlineActive)).toBe(true); expect(seen.size, "mixed-duration rotation kept cycling with no server").toBe(3); expect(await page.evaluate(() => window.__loadCount ?? 0)).toBe(0); await restoreStack(); // The server is authoritative again the moment it speaks. await expect .poll(async () => (await diagnostics(page))?.offlineActive, { timeout: 60_000 }) .toBe(false); const playback = await api(request, "/api/playback"); await expect .poll(async () => (await diagnostics(page))?.acceptedRevision, { timeout: 30_000 }) .toBeGreaterThanOrEqual(playback.playbackRevision); } finally { await context.close(); await restoreStack(); } }); test("a cold Chromium restart with the whole stack down restores signage", async ({ request, }) => { const library = await ensureFixtures(request); await resetPlayback(request); await activatePlaylist(request, "Cold start", [ { assetId: library["wide-a"].id, durationOverrideMs: 4_000 }, { assetId: library["wide-d"].id, durationOverrideMs: 6_000 }, ]); // A. Warm the persistent profile while everything is healthy. let clientId; let cachedBefore; { const { context, page } = await launchKiosk(); await page.goto(`${BASE_URL}/display`); await waitForSlide(page); await expect .poll(async () => page.evaluate(async () => Boolean((await navigator.serviceWorker.getRegistration("/"))?.active), ), ) .toBe(true); await expect.poll(async () => (await cachedMedia(page)).length).toBeGreaterThanOrEqual(2); clientId = await page.evaluate(() => window.__display.clientId); cachedBefore = await cachedMedia(page); await context.close(); } // B. Remove the entire origin: no web server, no API, no object storage. await stop(CONTAINERS.web, CONTAINERS.api, CONTAINERS.storage); await waitUntilUnreachable(BASE_URL); try { // C. Relaunch the kiosk against the same URL with the same profile. const { context, page } = await launchKiosk(); try { const response = await page.goto(`${BASE_URL}/display`, { waitUntil: "domcontentloaded" }); expect( response?.status(), "the Service Worker served the app shell with the origin down", ).toBe(200); await waitForSlide(page); const offline = await diagnostics(page); expect(offline.offlineActive, "the display took local authority").toBe(true); expect( await page.evaluate(() => window.__display.clientId), "the persistent display identity survived the restart", ).toBe(clientId); expect(cachedBefore).toContain( (await cachedMedia(page)).find((path) => path.includes(library["wide-a"].id)), ); expect(await opaqueLayerCount(page)).toBeGreaterThanOrEqual(1); // D. It must keep advancing, not merely show one restored frame. const first = offline.renderedAssetId; await expect .poll(async () => (await diagnostics(page))?.renderedAssetId, { timeout: 30_000 }) .not.toBe(first); expect(await opaqueLayerCount(page)).toBeGreaterThanOrEqual(1); // E. Restore the stack and confirm it surrenders authority cleanly. await restoreStack(); await expect .poll(async () => (await diagnostics(page))?.offlineActive, { timeout: 90_000 }) .toBe(false); const playback = await api(request, "/api/playback"); await expect .poll(async () => (await diagnostics(page))?.renderedEntryId, { timeout: 45_000 }) .toBe(playback.currentEntryId); const displays = await api(request, "/api/displays"); expect( displays.displays.filter((display) => display.clientId === clientId).length, "one physical display stays one record across a cold restart", ).toBe(1); } finally { await context.close(); } } finally { await restoreStack(); } }); test("an object storage outage never disturbs already cached slides", async ({ request }) => { const library = await ensureFixtures(request); await resetPlayback(request); await activatePlaylist(request, "Storage outage", [ { assetId: library["wide-a"].id, durationOverrideMs: 3_000 }, { assetId: library["wide-b"].id, durationOverrideMs: 3_000 }, ]); const { context, page } = await launchKiosk(); try { await page.goto(`${BASE_URL}/display`); await waitForSlide(page); await waitForCachedAssets(page, [library["wide-a"].id, library["wide-b"].id]); // The API stays up; only the bytes become unreachable. await stop(CONTAINERS.storage); const seen = new Set(); const deadline = Date.now() + 14_000; while (Date.now() < deadline) { const state = await page.evaluate(() => ({ rendered: window.__display?.diagnostics()?.renderedAssetId ?? null, opaque: [...document.querySelectorAll(".scene")].filter( (layer) => Number(getComputedStyle(layer).opacity) > 0.99, ).length, })); expect(state.opaque).toBeGreaterThanOrEqual(1); if (state.rendered) seen.add(state.rendered); await page.waitForTimeout(200); } expect(seen.size, "cached slides keep rotating without object storage").toBe(2); // A brand new asset cannot be fetched; the last good frame must survive. const held = await diagnostics(page); await activatePlaylist(request, "Uncached", [ { assetId: library.standalone.id, durationOverrideMs: 60_000 }, ]); await page.waitForTimeout(6_000); expect(await opaqueLayerCount(page)).toBeGreaterThanOrEqual(1); const during = await diagnostics(page); expect(during.renderedAssetId, "the last good frame is retained").toBe(held.renderedAssetId); // Nothing in the newly active playlist can be rendered locally, which is a // more severe condition than one failed asset and is reported as such. expect(during.mostRecentError?.kind).toBe("manifest_invalid"); // That condition reaches the server as a critical incident, exactly once. const clientId = await page.evaluate(() => window.__display.clientId); await page.evaluate(() => window.__display.heartbeat()); await expect .poll( async () => { const displays = await api(request, "/api/displays"); return ( displays.displays.find((display) => display.clientId === clientId)?.mostRecentError ?.kind ?? null ); }, { timeout: 30_000 }, ) .toBe("manifest_invalid"); // Restoring storage recovers without a page reload. await start(CONTAINERS.storage); await waitUntilHealthy(BASE_URL); await expect .poll(async () => (await diagnostics(page))?.renderedAssetId, { timeout: 90_000 }) .toBe(library.standalone.id); expect(await page.evaluate(() => window.__loadCount ?? 0)).toBe(0); } finally { await context.close(); await restoreStack(); } }); test("an API outage keeps local playback running and resynchronizes on return", async ({ request, }) => { const library = await ensureFixtures(request); await resetPlayback(request); await activatePlaylist(request, "API outage", [ { assetId: library["wide-a"].id, durationOverrideMs: 3_000 }, { assetId: library["wide-b"].id, durationOverrideMs: 3_000 }, { assetId: library["wide-c"].id, durationOverrideMs: 3_000 }, ]); const { context, page } = await launchKiosk(); try { await page.goto(`${BASE_URL}/display`); await waitForSlide(page); await waitForCachedAssets(page, [ library["wide-a"].id, library["wide-b"].id, library["wide-c"].id, ]); await stop(CONTAINERS.api); await expect .poll(async () => (await diagnostics(page))?.offlineActive, { timeout: 45_000 }) .toBe(true); const seen = new Set(); const deadline = Date.now() + 14_000; while (Date.now() < deadline) { const state = await diagnostics(page); if (state?.renderedAssetId) seen.add(state.renderedAssetId); expect(await opaqueLayerCount(page)).toBeGreaterThanOrEqual(1); await page.waitForTimeout(250); } expect(seen.size).toBeGreaterThanOrEqual(2); // While the display was blind, the operator changed authoritative state. await start(CONTAINERS.api); await waitUntilHealthy(BASE_URL); await api(request, "/api/playback/mode", { method: "PUT", data: { mode: "off" } }); // Server wins immediately on reconnect, including the visual gate. await expect(page.getByTestId("output-gate")).toHaveAttribute("data-active", "true", { timeout: 60_000, }); expect((await diagnostics(page)).offlineActive).toBe(false); await api(request, "/api/playback/mode", { method: "PUT", data: { mode: "on" } }); await expect(page.getByTestId("output-gate")).toHaveAttribute("data-active", "false", { timeout: 30_000, }); const playback = await api(request, "/api/playback"); await expect .poll(async () => (await diagnostics(page))?.renderedEntryId, { timeout: 30_000 }) .toBe(playback.currentEntryId); expect(await page.evaluate(() => window.__loadCount ?? 0)).toBe(0); } finally { await context.close(); await restoreStack(); } }); test("a playlist changed during the outage is adopted cleanly on reconnect", async ({ request, }) => { const library = await ensureFixtures(request); await resetPlayback(request); await activatePlaylist(request, "Before outage", [ { assetId: library["wide-a"].id, durationOverrideMs: 5_000 }, { assetId: library["wide-b"].id, durationOverrideMs: 5_000 }, ]); const { context, page } = await launchKiosk(); try { await page.goto(`${BASE_URL}/display`); await waitForSlide(page); const before = await diagnostics(page); await stop(CONTAINERS.api); await expect .poll(async () => (await diagnostics(page))?.offlineActive, { timeout: 45_000 }) .toBe(true); await page.waitForTimeout(6_000); await start(CONTAINERS.api); await waitUntilHealthy(BASE_URL); // A different playlist entirely, activated while the display was offline. await activatePlaylist(request, "After outage", [ { assetId: library.square.id, durationOverrideMs: 60_000 }, ]); await expect .poll(async () => (await diagnostics(page))?.renderedAssetId, { timeout: 90_000 }) .toBe(library.square.id); const after = await diagnostics(page); expect(after.offlineActive).toBe(false); expect(after.renderedAssetId).not.toBe(before.renderedAssetId); expect(await opaqueLayerCount(page)).toBeGreaterThanOrEqual(1); // The superseded playlist's bytes are pruned once the new one is promoted. await expect .poll(async () => (await cachedMedia(page)).some((path) => path.includes(library.square.id))) .toBe(true); const cached = await cachedMedia(page); expect(cached.length, "repeated playlist changes must not grow the cache forever") .toBeLessThanOrEqual(6); } finally { await context.close(); await restoreStack(); } });