From ff8a3e8097cb4b7241f0201977f5f2e5b9199219 Mon Sep 17 00:00:00 2001 From: ekklesia Date: Mon, 5 Oct 2026 18:25:12 -0500 Subject: [PATCH 1/2] test(deploy): add opt-in display-device scenarios to the split smoke Reproduction harness for what the first production Signage deployment found. With EKKLESIA_SMOKE_DISPLAY_DEVICE=1, the split smoke starts a second headless Chromium (the "kiosk") whose profile directory is a Docker volume. It pairs that kiosk through the real flow, closes it cleanly, restarts it on the same profile, and then runs injected-failure scenarios: two pages on one credential, a connectivity blip, one refused viewer-session request, one refused realtime connection, a cross-origin image-caching probe, a full realtime recycle, and revocation. Every DevTools command and page-side request has its own deadline, and each step has a budget. A step that overruns prints the exact call it was waiting for. The proxy's "aborting with incomplete response" lines are listed afterwards by UTC time and path only. Off by default: several scenarios expect Signage and fail on today's v0.1.1, by design. Co-Authored-By: Claude Opus 5.5 --- scripts/lib/deploy-smoke-display.mjs | 979 +++++++++++++++++++++++++++ scripts/smoke-deploy.sh | 87 +++ 2 files changed, 1066 insertions(+) create mode 100644 scripts/lib/deploy-smoke-display.mjs diff --git a/scripts/lib/deploy-smoke-display.mjs b/scripts/lib/deploy-smoke-display.mjs new file mode 100644 index 0000000..ec5fd60 --- /dev/null +++ b/scripts/lib/deploy-smoke-display.mjs @@ -0,0 +1,979 @@ +// Real-browser display-device scenarios for scripts/smoke-deploy.sh in the split topology (ADR 0036, +// ADR 0045). Runs inside a disposable container on the deployment's `edge` network and drives two +// headless Chromiums over the Chrome DevTools Protocol, with Node.js's own fetch and WebSocket and no +// other dependency: the smoke run's browser, where the owner signs in, and a "kiosk" browser whose +// profile directory is a Docker volume that outlives the kiosk container. +// +// node deploy-smoke-display.mjs +// +// pair seed content, pair the kiosk, then close its browser cleanly +// restart same profile, new browser process: still paired and live? +// two-pages two pages, one device credential: does either flap? +// connectivity-blip the browser reports offline then online: is a live display remounted? +// session-blip Signage's first viewer-session request fails once: does the display recover? +// stream-failure the first realtime connection is refused: is realtime re-established? +// caching-probe can a display on the API origin store storage-origin images in Cache Storage? +// stream-recycle one page past the five-minute realtime recycle, with a timestamped timeline +// revoke revoke the paired kiosk while its display is open +// +// Every DevTools command and every page-side request has its own deadline, and a step has an overall +// budget. A call that never answers fails the step with a line naming that exact call, because a call +// that can hang the driver may be a request that can hang a real display. +// +// Output is one line per check or observation; nothing printed carries a credential, cookie value, +// signed URL, query string, or response body. + +function required(name) { + const value = process.env[name]; + if (!value) { + throw new Error(`${name} is required.`); + } + return value; +} + +/** A positive whole number of milliseconds from the environment, or the default. */ +function milliseconds(name, fallback) { + const raw = process.env[name]; + const value = raw === undefined || raw === "" ? fallback : Number(raw); + if (!Number.isSafeInteger(value) || value <= 0) { + throw new Error(`${name} must be a positive whole number of milliseconds.`); + } + return value; +} + +const mode = process.argv[2]; +const apiOrigin = `https://${required("SMOKE_HOSTNAME")}`; +const dashboardOrigin = `https://${required("SMOKE_DASHBOARD_HOSTNAME")}`; +const ownerAddress = required("SMOKE_CDP_ADDRESS"); +const kioskAddress = required("SMOKE_KIOSK_CDP_ADDRESS"); +const watchMs = milliseconds("SMOKE_WATCH_MS", 120_000); +const recycleWatchMs = milliseconds("SMOKE_RECYCLE_WATCH_MS", 330_000); +const stepBudgetMs = milliseconds("SMOKE_STEP_BUDGET_MS", 180_000); +/** How long one DevTools command may go unanswered. */ +const callTimeoutMs = milliseconds("SMOKE_CALL_TIMEOUT_MS", 10_000); +/** How long one page-side HTTP request may go unanswered; shorter than the call deadline, so a hung + * request is reported as the request rather than as the evaluation that awaited it. */ +const requestTimeoutMs = Math.max(1_000, callTimeoutMs - 2_000); +const waitTimeoutMs = 30_000; +const deviceCookie = "ekklesia_device_session"; +/** Far above what three cached slides need, which is one object URL each. */ +const MAX_LIVE_OBJECT_URLS = 50; +const stepStartedAt = Date.now(); + +let failures = 0; +function check(condition, description) { + console.log(`${condition ? "ok " : "FAIL"} ${description}`); + if (!condition) { + failures += 1; + } +} +function note(description) { + console.log(` ${description}`); +} +const clock = (ms = Date.now()) => new Date(ms).toISOString().slice(11, 19); +const since = (from) => `+${Math.round((Date.now() - from) / 1000)}s`; +function progress(description) { + console.log(`···· ${clock()} ${since(stepStartedAt)} ${description}`); +} +const sleep = (ms) => new Promise((resolve) => setTimeout(resolve, ms)); +/** A URL's path only: a signed delivery URL's query is a credential. */ +const pathOf = (url) => { + try { + return new URL(url).pathname; + } catch { + return "(unparseable)"; + } +}; + +// --- Hang guards --------------------------------------------------------------------------------- + +class HangError extends Error { + constructor(label, ms) { + super(`HANG ${label}: no answer within ${ms / 1000}s`); + this.name = "HangError"; + } +} + +/** Everything awaited right now, by name, so a step that overruns its budget says what it was + * waiting for. */ +const inFlight = new Map(); +let nextOperation = 1; + +function guard(label, promise, ms = callTimeoutMs) { + const id = nextOperation; + nextOperation += 1; + inFlight.set(id, { at: Date.now(), label }); + let timer; + return Promise.race([ + promise, + new Promise((_, reject) => { + timer = setTimeout(() => reject(new HangError(label, ms)), ms); + }), + ]).finally(() => { + clearTimeout(timer); + inFlight.delete(id); + }); +} + +function describeInFlight() { + const pending = [...inFlight.values()].map( + (operation) => `${operation.label} (${Math.round((Date.now() - operation.at) / 1000)}s)`, + ); + return pending.length === 0 ? "nothing — the step itself overran" : pending.join("; "); +} + +setTimeout(() => { + console.log( + `FAIL ${mode}: step budget of ${stepBudgetMs / 1000}s exceeded; in flight: ${describeInFlight()}`, + ); + process.exit(2); +}, stepBudgetMs).unref(); + +// --- DevTools ------------------------------------------------------------------------------------ + +/** The same minimal DevTools client as deploy-smoke-browser.mjs, with a deadline on every command. */ +async function connect(devtools, name) { + const deadline = Date.now() + 30_000; + let version; + while (version === undefined) { + try { + const response = await fetch(`http://${devtools}/json/version`, { + signal: AbortSignal.timeout(2_000), + }); + version = await response.json(); + } catch { + if (Date.now() > deadline) { + throw new HangError(`the ${name} browser's DevTools endpoint`, 30_000); + } + await sleep(500); + } + } + const socketUrl = version.webSocketDebuggerUrl.replace(/^ws:\/\/[^/]+/u, `ws://${devtools}`); + const socket = new WebSocket(socketUrl); + await guard( + `the ${name} browser's DevTools socket`, + new Promise((resolve, reject) => { + socket.addEventListener("open", resolve, { once: true }); + socket.addEventListener("error", () => reject(new Error("DevTools connection failed.")), { + once: true, + }); + }), + ); + let nextId = 1; + const pending = new Map(); + const listeners = []; + socket.addEventListener("message", (event) => { + const message = JSON.parse(String(event.data)); + if (message.id !== undefined) { + const waiter = pending.get(message.id); + pending.delete(message.id); + if (message.error) { + waiter?.reject(new Error(`DevTools ${waiter.method} failed.`)); + } else { + waiter?.resolve(message.result); + } + return; + } + for (const listener of listeners) { + listener(message); + } + }); + return { + close: () => socket.close(), + on: (listener) => listeners.push(listener), + send(method, params = {}, sessionId = undefined, where = name) { + const id = nextId; + nextId += 1; + socket.send(JSON.stringify({ id, method, params, ...(sessionId ? { sessionId } : {}) })); + return guard( + `DevTools ${method} on ${where}`, + new Promise((resolve, reject) => pending.set(id, { method, reject, resolve })), + ); + }, + }; +} + +/** Finds a test id anywhere in the document, including inside module artifacts' open shadow roots + * (ADR 0040). */ +const deepFind = `((testId) => { + const walk = (root) => { + const found = root.querySelector('[data-testid="' + testId + '"]'); + if (found) return found; + for (const element of root.querySelectorAll("*")) { + if (element.shadowRoot) { + const inner = walk(element.shadowRoot); + if (inner) return inner; + } + } + return null; + }; + return walk(document); +})`; +const find = (testId) => `${deepFind}(${JSON.stringify(testId)})`; + +/** Installed before any page script: the browser's own connectivity events and the display service + * worker's fallback reports, which are what the display host remounts on (ADR 0043). */ +const instrumentation = `(() => { + const events = []; + // Object URLs created and revoked in this document. Each live one pins its blob in the browser's + // blob storage until it is revoked or the document goes away. + const blobs = { created: 0, revoked: 0 }; + Object.defineProperty(window, "__displayRepro", { value: { blobs, events } }); + const create = URL.createObjectURL.bind(URL); + const revoke = URL.revokeObjectURL.bind(URL); + URL.createObjectURL = (object) => { blobs.created += 1; return create(object); }; + URL.revokeObjectURL = (url) => { blobs.revoked += 1; return revoke(url); }; + const record = (kind) => events.push({ at: Date.now(), kind }); + addEventListener("online", () => record("online")); + addEventListener("offline", () => record("offline")); + navigator.serviceWorker?.addEventListener("message", (event) => { + const status = event.data && event.data.ekklesiaDisplayOffline; + if (status) record(status.usingFallback ? "sw-fallback:" + (status.kind || "") : "sw-network"); + }); +})()`; + +/** One page target with its own flat session, and what it does on the network. */ +async function openPage(cdp, url, label) { + const { targetId } = await cdp.send("Target.createTarget", { url: "about:blank" }); + const { sessionId } = await cdp.send("Target.attachToTarget", { flatten: true, targetId }); + const send = (method, params) => cdp.send(method, params, sessionId, `page "${label}"`); + const requests = new Map(); + const page = { + label, + /** Every realtime connection: when it opened, its status, and when and how it ended. */ + streams: [], + /** Signage viewer-session requests: one per module mount. */ + sessionRequests: [], + /** Heartbeat statuses, by status code. */ + heartbeats: new Map(), + /** Channel-state fetches and how they went. */ + stateFetches: [], + /** Non-2xx responses and failed requests, by path only. */ + problems: [], + }; + cdp.on((message) => { + if (message.sessionId !== sessionId) { + return; + } + const params = message.params ?? {}; + if (message.method === "Network.requestWillBeSent") { + const path = pathOf(params.request.url); + const entry = { at: Date.now(), path, type: params.type }; + requests.set(params.requestId, entry); + if (params.type === "EventSource") { + entry.stream = { + openedAt: Date.now(), + path, + status: undefined, + endedAt: undefined, + how: undefined, + }; + page.streams.push(entry.stream); + } else if (path.endsWith("/viewers/session")) { + page.sessionRequests.push(Date.now()); + } else if (path.endsWith("/display-state") && !path.includes("module-realtime")) { + entry.state = { at: Date.now(), outcome: "pending" }; + page.stateFetches.push(entry.state); + } + } else if (message.method === "Network.responseReceived") { + const entry = requests.get(params.requestId); + const { status } = params.response; + if (entry?.stream) { + entry.stream.status = status; + } + if (entry?.state) { + entry.state.outcome = String(status); + } + if (entry?.path.endsWith("/heartbeat")) { + page.heartbeats.set(status, (page.heartbeats.get(status) ?? 0) + 1); + } + if (status >= 400 || status === 0) { + page.problems.push(`${clock()} ${status} ${pathOf(params.response.url)}`); + } + } else if (message.method === "Network.loadingFinished") { + const entry = requests.get(params.requestId); + if (entry?.stream) { + entry.stream.endedAt = Date.now(); + entry.stream.how = "finished"; + } + } else if (message.method === "Network.loadingFailed") { + const entry = requests.get(params.requestId); + if (entry?.stream) { + entry.stream.endedAt = Date.now(); + entry.stream.how = params.canceled ? "canceled" : `failed ${params.errorText}`; + } + if (entry?.state) { + entry.state.outcome = params.canceled ? "canceled" : `failed ${params.errorText}`; + } + if (entry && !params.canceled) { + page.problems.push(`${clock()} failed ${params.errorText} ${entry.path}`); + } + } + }); + await send("Network.enable"); + await send("Page.enable"); + await send("Runtime.enable"); + await send("Page.addScriptToEvaluateOnNewDocument", { source: instrumentation }); + await send("Emulation.setDeviceMetricsOverride", { + deviceScaleFactor: 1, + height: 1080, + mobile: false, + width: 1920, + }); + page.send = send; + /** Runs an expression in the page. `what` names it if it never comes back. */ + page.evaluate = async (expression, what = "an evaluation") => { + const result = await guard( + `page "${label}": ${what}`, + send("Runtime.evaluate", { awaitPromise: true, expression, returnByValue: true }), + ); + if (result.exceptionDetails) { + throw new Error(`A page expression threw (${what}).`); + } + return result.result.value; + }; + page.waitFor = async (expression, description, ms = waitTimeoutMs) => { + const deadline = Date.now() + ms; + while (Date.now() < deadline) { + if ( + await page + .evaluate(`Boolean(${expression})`, `waiting for ${description}`) + .catch((error) => { + if (error instanceof HangError) throw error; + return false; + }) + ) { + return true; + } + await sleep(250); + } + note(`timed out waiting for ${description}`); + return false; + }; + /** A credentialed request from this page, bounded in the page itself, so a request that never + * answers is reported as that request. */ + page.request = async (method, url, body) => { + const result = await page.evaluate( + `fetch(${JSON.stringify(url)}, { + credentials: "include", + method: ${JSON.stringify(method)}, + signal: AbortSignal.timeout(${requestTimeoutMs}), + ${body === undefined ? "" : `headers: { "content-type": "application/json" }, body: ${JSON.stringify(JSON.stringify(body))},`} + }).then( + async (response) => ({ status: response.status, body: await response.json().catch(() => null) }), + (error) => ({ status: error && error.name === "TimeoutError" ? "timeout" : "network-error" }), + )`, + `${method} ${pathOf(url)}`, + ); + if (result.status === "timeout") { + throw new HangError(`HTTP ${method} ${pathOf(url)} from page "${label}"`, requestTimeoutMs); + } + return result; + }; + page.close = () => cdp.send("Target.closeTarget", { targetId }); + page.navigate = (to) => send("Page.navigate", { url: to }); + if (url) { + await page.navigate(url); + } + return page; +} + +/** The owner, signed in on the Dashboard hostname of the smoke run's own browser. */ +async function ownerPage() { + const cdp = await connect(ownerAddress, "owner"); + const page = await openPage(cdp, `${dashboardOrigin}/`, "owner"); + await page.waitFor(`${find("login-email")} || ${find("host-shell")}`, "the Dashboard"); + if (!(await page.evaluate(`Boolean(${find("host-shell")})`, "checking the session"))) { + await page.evaluate( + `(() => { + const setValue = (element, value) => { + Object.getOwnPropertyDescriptor(HTMLInputElement.prototype, "value").set.call(element, value); + element.dispatchEvent(new Event("input", { bubbles: true })); + }; + setValue(${find("login-email")}, ${JSON.stringify(required("SMOKE_OWNER_EMAIL"))}); + setValue(${find("login-password")}, ${JSON.stringify(required("SMOKE_OWNER_PASSWORD"))}); + ${find("login-email")}.form.requestSubmit(); + })()`, + "submitting the sign-in form", + ); + check(await page.waitFor(find("host-shell"), "the signed-in Dashboard"), "the owner signs in"); + } + page.api = (method, path, body) => page.request(method, `${apiOrigin}${path}`, body); + return { cdp, page }; +} + +/** Closes the owner's tab as well as the connection: the owner browser is shared by every step, and a + * tab left behind by each one accumulates until the Dashboard no longer renders in time. */ +async function finishOwner(owner) { + await owner.page.close().catch(() => undefined); + owner.cdp.close(); +} + +async function displayModuleId(owner) { + const navigation = await owner.api("GET", "/api/v1/dashboard/navigation"); + const entries = [ + ...navigation.body.ungrouped, + ...navigation.body.groups.flatMap((group) => group.modules), + ]; + const withDisplay = entries.filter((entry) => entry.module.display !== null); + const signage = withDisplay.find((entry) => entry.module.id === "signage"); + return (signage ?? withDisplay[0])?.module.id; +} + +/** Three slides of plain colour, on a five-second rotation, so a display has real content to persist + * and continue from, as the production screen did. Signage only. */ +async function seedSignage(owner) { + const base = "/api/v1/modules/signage"; + const images = []; + for (const [index, colour] of ["#b03030", "#3060b0", "#30a050"].entries()) { + const uploaded = await owner.evaluate( + `(async () => { + const canvas = document.createElement("canvas"); + canvas.width = 640; + canvas.height = 360; + const context = canvas.getContext("2d"); + context.fillStyle = ${JSON.stringify(colour)}; + context.fillRect(0, 0, 640, 360); + const blob = await new Promise((resolve) => canvas.toBlob(resolve, "image/png")); + const response = await fetch(${JSON.stringify(`${apiOrigin}${base}/images?filename=slide-${index}.png&displayName=Slide%20${index}`)}, { + body: blob, credentials: "include", headers: { "content-type": "image/png" }, method: "POST", + signal: AbortSignal.timeout(${requestTimeoutMs}), + }); + return { status: response.status, body: await response.json().catch(() => null) }; + })()`, + `uploading slide ${index}`, + ); + images.push(uploaded.body?.image?.id); + } + check( + images.every((id) => typeof id === "string"), + "three slides are uploaded to Signage", + ); + const timeline = await owner.api("POST", `${base}/timelines`, { name: "Display repro" }); + const timelineId = timeline.body?.timeline?.id ?? timeline.body?.id; + for (const itemId of images) { + await owner.api("POST", `${base}/timelines/${timelineId}/slots`, { + durationOverrideMs: 5_000, + itemId, + providerId: "signage.native-images", + }); + } + const groups = await owner.api("GET", `${base}/channel-groups`); + const groupId = groups.body?.groups?.[0]?.id; + const activated = await owner.api("POST", `${base}/playback/${groupId}/activate`, { + timelineId, + }); + check(activated.status === 200, "a three-slide timeline is playing on the default channel group"); +} + +async function signageViewers(owner) { + const listed = await owner.api("GET", "/api/v1/modules/signage/viewers"); + return listed.body?.viewers ?? []; +} + +async function coreDevices(owner) { + const listed = await owner.api("GET", "/api/v1/devices"); + return listed.body?.devices ?? []; +} + +/** The display's own view of itself: which host state is showing, and Signage's offline flag. */ +function displayState(page) { + return page.evaluate( + `(() => { + const stage = ${find("signage-stage")}; + return { + pairing: Boolean(${find("display-pairing")}), + hostLoading: Boolean(${find("display-host-loading")}) || Boolean(${find("display-loading")}), + hostError: Boolean(${find("display-host-error")}) || Boolean(${find("display-error")}), + hostOffline: Boolean(${find("display-offline")}), + mounted: (${find("display-module-container")}?.childElementCount ?? 0) > 0, + signageOffline: stage ? stage.dataset.offline === "true" : null, + }; + })()`, + "reading the display state", + ); +} + +async function blobCounts(page) { + return page.evaluate( + "window.__displayRepro ? window.__displayRepro.blobs : { created: 0, revoked: 0 }", + "reading object-URL counts", + ); +} + +async function connectivityEvents(page) { + return page.evaluate( + "window.__displayRepro ? window.__displayRepro.events : []", + "reading connectivity events", + ); +} + +async function describeDeviceCookie(cdp) { + const { cookies } = await cdp.send("Storage.getCookies"); + const cookie = cookies.find((candidate) => candidate.name === deviceCookie); + if (!cookie) { + note(`${deviceCookie}: absent from the profile`); + return undefined; + } + const days = cookie.session + ? "session" + : `${((cookie.expires * 1000 - Date.now()) / 86_400_000).toFixed(1)} days`; + note( + `${deviceCookie}: domain=${cookie.domain} session=${cookie.session} lifetime=${days} httpOnly=${cookie.httpOnly} secure=${cookie.secure} sameSite=${cookie.sameSite}`, + ); + return cookie; +} + +const liveExpression = `(() => { + const stage = ${find("signage-stage")}; + const mounted = (${find("display-module-container")}?.childElementCount ?? 0) > 0; + return mounted && !${find("display-pairing")} && (stage === null || stage.dataset.offline === "false"); +})()`; + +function waitLive(page, description, ms = 45_000) { + return page.waitFor(liveExpression, description, ms); +} + +/** A page's network story, one line each, by path only. */ +function summarize(page, from) { + const at = (ms) => (ms === undefined ? "-" : `${clock(ms)} (${Math.round((ms - from) / 1000)}s)`); + note(`${page.label}: viewer sessions opened ${page.sessionRequests.length} (= module mounts)`); + note( + `${page.label}: heartbeats ${[...page.heartbeats].map(([status, count]) => `${status}×${count}`).join(" ") || "none"}`, + ); + const states = page.stateFetches.reduce((tally, fetch) => { + tally[fetch.outcome] = (tally[fetch.outcome] ?? 0) + 1; + return tally; + }, {}); + note(`${page.label}: channel-state fetches ${JSON.stringify(states)}`); + for (const stream of page.streams) { + const lived = + stream.endedAt === undefined + ? "open" + : `${Math.round((stream.endedAt - stream.openedAt) / 1000)}s`; + note( + `${page.label}: stream opened ${at(stream.openedAt)} status ${stream.status ?? "-"} ended ${at(stream.endedAt)} ${stream.how ?? ""} lived ${lived}`, + ); + } + note(`${page.label}: problems ${page.problems.slice(-12).join(" | ") || "none"}`); +} + +/** Samples display state on an interval for a fixed time, printing progress, and returns a tally. */ +async function watch(pages, ms, { onSample, viewerOwner } = {}) { + const from = Date.now(); + const tally = new Map( + pages.map((page) => [ + page.label, + { loading: 0, offline: 0, pairing: 0, samples: 0, transitions: [] }, + ]), + ); + const last = new Map(); + let nextProgress = from + 15_000; + const roster = []; + while (Date.now() - from < ms) { + for (const page of pages) { + const state = await displayState(page); + const t = tally.get(page.label); + t.samples += 1; + if (state.signageOffline || state.hostOffline) t.offline += 1; + if (state.pairing) t.pairing += 1; + if (state.hostLoading) t.loading += 1; + const key = JSON.stringify(state); + if (last.get(page.label) !== key) { + last.set(page.label, key); + t.transitions.push(`${clock()} ${since(from)} ${key}`); + } + if (onSample && (await onSample(page, state))) { + return { from, roster, stopped: true, tally }; + } + } + if (Date.now() >= nextProgress) { + nextProgress = Date.now() + 15_000; + if (viewerOwner) { + const viewers = await signageViewers(viewerOwner); + roster.push( + `${since(from)} ${viewers.map((v) => `${v.name}:${v.online ? "Online" : "Offline"}`).join(",")}`, + ); + } + const lines = []; + for (const page of pages) { + const t = tally.get(page.label); + const blobs = await blobCounts(page); + lines.push( + `${page.label}: ${t.samples} samples, offline ${t.offline}, mounts ${page.sessionRequests.length}, streams ${page.streams.length}, live object URLs ${blobs.created - blobs.revoked}`, + ); + } + progress(lines.join(" · ")); + } + await sleep(2_000); + } + return { from, roster, stopped: false, tally }; +} + +function printTally(result) { + for (const [label, t] of result.tally) { + note( + `${label}: ${t.samples} samples, offline in ${t.offline}, pairing in ${t.pairing}, loading in ${t.loading}`, + ); + for (const transition of t.transitions.slice(0, 30)) { + note(` ${label} ${transition}`); + } + } + if (result.roster.length > 0) { + note(`roster over time: ${result.roster.join(" ; ")}`); + } +} + +// --- Steps --------------------------------------------------------------------------------------- + +async function run() { + if (mode === "pair") { + const owner = await ownerPage(); + const moduleId = await displayModuleId(owner.page); + check(moduleId !== undefined, `a module publishes a display (${moduleId})`); + if (moduleId === "signage") { + await seedSignage(owner.page); + } + const kiosk = await connect(kioskAddress, "kiosk"); + const display = await openPage(kiosk, `${apiOrigin}/display/${moduleId}`, "kiosk"); + check( + await display.waitFor(find("display-pairing-code"), "a pairing code"), + "an unpaired kiosk shows a pairing code", + ); + const code = ( + await display.evaluate(`${find("display-pairing-code")}.textContent`, "reading the code") + ).replace(/\D/gu, ""); + const approved = await owner.page.api("POST", "/api/v1/device-pairing/approvals", { + code, + deviceName: "Repro kiosk", + }); + check(approved.status === 200, "the owner approves the code"); + check( + await waitLive(display, "the paired display to go live"), + "the paired kiosk mounts the display and is live", + ); + await describeDeviceCookie(kiosk); + progress("letting heartbeats, a persisted snapshot, and media prefetch happen (20s)"); + await sleep(20_000); + for (const viewer of await signageViewers(owner.page)) { + note( + `signage viewer "${viewer.name}" online=${viewer.online} offlineReady=${viewer.offlineReady} cachedAssets=${viewer.cachedAssetCount} error=${viewer.lastError?.code ?? "none"}`, + ); + } + summarize(display, stepStartedAt); + // A clean exit: the browser shuts down normally, exactly as a systemd stop of a well-behaved kiosk. + await kiosk.send("Browser.close").catch(() => undefined); + kiosk.close(); + await finishOwner(owner); + } else if (mode === "restart") { + const owner = await ownerPage(); + const moduleId = await displayModuleId(owner.page); + const kiosk = await connect(kioskAddress, "kiosk"); + const cookie = await describeDeviceCookie(kiosk); + check(cookie !== undefined, "the device credential survived a clean browser exit"); + const display = await openPage(kiosk, `${apiOrigin}/display/${moduleId}`, "kiosk"); + const live = await waitLive(display, "the restarted display to go live"); + const state = await displayState(display); + check( + live && !state.pairing, + `after a clean restart the kiosk is still paired and live (${JSON.stringify(state)})`, + ); + summarize(display, stepStartedAt); + await display.close(); + kiosk.close(); + await finishOwner(owner); + } else if (mode === "two-pages") { + const owner = await ownerPage(); + const moduleId = await displayModuleId(owner.page); + const kiosk = await connect(kioskAddress, "kiosk"); + const url = `${apiOrigin}/display/${moduleId}`; + const first = await openPage(kiosk, url, "first"); + check(await waitLive(first, "the first page"), "the first page is live"); + const second = await openPage(kiosk, url, "second"); + check(await waitLive(second, "the second page"), "the second page is live"); + const result = await watch([first, second], watchMs, { viewerOwner: owner.page }); + printTally(result); + for (const page of [first, second]) { + summarize(page, result.from); + note(`${page.label}: connectivity events ${JSON.stringify(await connectivityEvents(page))}`); + } + check( + [...result.tally.values()].every((t) => t.offline === 0), + `neither page showed an offline state while both were open (${watchMs / 1000}s)`, + ); + check( + [first, second].every((page) => page.sessionRequests.length === 1), + "neither page was remounted while both were open", + ); + await first.close(); + await second.close(); + kiosk.close(); + await finishOwner(owner); + } else if (mode === "connectivity-blip") { + // The browser's own offline/online events, as a Wi-Fi reassociation or a link flap produces. The + // display host re-resolves its descriptor on "online" (ADR 0043); this measures what that costs. + const owner = await ownerPage(); + const moduleId = await displayModuleId(owner.page); + const kiosk = await connect(kioskAddress, "kiosk"); + const page = await openPage(kiosk, `${apiOrigin}/display/${moduleId}`, "blip"); + check(await waitLive(page, "the display"), "the display is live"); + const mountsBefore = page.sessionRequests.length; + await page.evaluate( + `dispatchEvent(new Event("offline")); setTimeout(() => dispatchEvent(new Event("online")), 1000)`, + "dispatching offline then online", + ); + let firstOfflineAt; + let lastOfflineAt; + const result = await watch([page], 30_000, { + onSample: (_page, state) => { + if (state.signageOffline || state.hostOffline || state.hostLoading || !state.mounted) { + firstOfflineAt ??= Date.now(); + lastOfflineAt = Date.now(); + } + return false; + }, + }); + printTally(result); + const remounts = page.sessionRequests.length - mountsBefore; + note(`remounts caused by one offline/online pair: ${remounts}`); + note( + firstOfflineAt === undefined + ? "nothing visible changed" + : `visible disruption from ${clock(firstOfflineAt)} to ${clock(lastOfflineAt)} (sampled every 2s)`, + ); + summarize(page, result.from); + check(remounts === 0, "a connectivity blip does not remount a live display"); + check( + firstOfflineAt === undefined, + "a connectivity blip does not show an offline state on a live display", + ); + await page.close(); + kiosk.close(); + await finishOwner(owner); + } else if (mode === "session-blip") { + // One transient failure of Signage's viewer-session request at start-up, as a kiosk booting + // before its network settles, or an API restart, would produce. Everything else reaches the + // server. + const owner = await ownerPage(); + const moduleId = await displayModuleId(owner.page); + const kiosk = await connect(kioskAddress, "kiosk"); + const page = await openPage(kiosk, undefined, "session-blip"); + let blocked = 0; + kiosk.on((message) => { + if (message.method === "Fetch.requestPaused") { + const { requestId, request } = message.params; + if (blocked === 0 && pathOf(request.url).endsWith("/viewers/session")) { + blocked += 1; + page + .send("Fetch.failRequest", { errorReason: "ConnectionRefused", requestId }) + .catch((error) => note(`interception reply failed: ${error.message}`)); + } else { + page + .send("Fetch.continueRequest", { requestId }) + .catch((error) => note(`interception reply failed: ${error.message}`)); + } + } + }); + await page.send("Fetch.enable", { patterns: [{ urlPattern: "*/api/v1/modules/*" }] }); + await page.navigate(`${apiOrigin}/display/${moduleId}`); + await page.waitFor( + `(${find("display-module-container")}?.childElementCount ?? 0) > 0`, + "the module to mount", + ); + note(`viewer-session requests failed on purpose: ${blocked}`); + const recovered = await page.waitFor( + `${find("signage-stage")}?.dataset.offline === "false"`, + "recovery from one failed viewer-session request", + 90_000, + ); + summarize(page, stepStartedAt); + check( + recovered, + `one failed viewer-session request does not leave the display offline (${JSON.stringify(await displayState(page))})`, + ); + await page.close(); + kiosk.close(); + await finishOwner(owner); + } else if (mode === "stream-failure") { + // The first realtime connection is refused with a 503, as an API mid-restart or a proxy with no + // upstream answers. A non-200 ends an EventSource for good; the question is whether anything + // re-establishes realtime without a reload. + const owner = await ownerPage(); + const moduleId = await displayModuleId(owner.page); + const kiosk = await connect(kioskAddress, "kiosk"); + const page = await openPage(kiosk, undefined, "stream-failure"); + let refused = 0; + kiosk.on((message) => { + if (message.method === "Fetch.requestPaused") { + const { requestId } = message.params; + if (refused === 0) { + refused += 1; + page + .send("Fetch.fulfillRequest", { + body: Buffer.from('{"code":"REALTIME_UNAVAILABLE"}').toString("base64"), + requestId, + responseCode: 503, + responseHeaders: [ + { name: "content-type", value: "application/json" }, + { name: "cache-control", value: "no-store" }, + ], + }) + .catch((error) => note(`interception reply failed: ${error.message}`)); + } else { + page + .send("Fetch.continueRequest", { requestId }) + .catch((error) => note(`interception reply failed: ${error.message}`)); + } + } + }); + await page.send("Fetch.enable", { patterns: [{ urlPattern: "*/api/v1/module-realtime/*" }] }); + await page.navigate(`${apiOrigin}/display/${moduleId}`); + await page.waitFor( + `(${find("display-module-container")}?.childElementCount ?? 0) > 0`, + "the module to mount", + ); + note(`realtime connections refused on purpose: ${refused}`); + const result = await watch([page], 90_000, { + onSample: () => page.streams.some((stream) => stream.status === 200), + }); + printTally(result); + summarize(page, result.from); + check( + page.streams.some((stream) => stream.status === 200), + "after one refused realtime connection, realtime is re-established without a reload", + ); + await page.close(); + kiosk.close(); + await finishOwner(owner); + } else if (mode === "caching-probe") { + // Whether a display on the API origin can store an image from the storage origin in Cache + // Storage, which is what Signage's offline media relies on. Reports only statuses, response + // types, and whether CORS headers were present — never a URL. + const owner = await ownerPage(); + const moduleId = await displayModuleId(owner.page); + const groups = await owner.page.api("GET", "/api/v1/modules/signage/channel-groups"); + const channelId = groups.body?.groups?.[0]?.channels?.[0]?.id; + const kiosk = await connect(kioskAddress, "kiosk"); + const page = await openPage(kiosk, `${apiOrigin}/display/${moduleId}`, "probe"); + check(await waitLive(page, "the display"), "the display is live"); + progress("letting the display prefetch its slides (15s)"); + await sleep(15_000); + const probe = await page.evaluate( + `(async () => { + const result = {}; + const state = await fetch(${JSON.stringify(`${apiOrigin}/api/v1/modules/signage/channels/${channelId}/display-state`)}, { credentials: "include", signal: AbortSignal.timeout(${requestTimeoutMs}) }).then((r) => r.json()); + const url = (state.selections || []).map((s) => s.media && s.media.display && s.media.display.url).find(Boolean); + result.selections = (state.selections || []).length; + result.haveUrl = Boolean(url); + if (!url) return result; + result.sameOrigin = new URL(url).origin === location.origin; + try { + const response = await fetch(url, { signal: AbortSignal.timeout(${requestTimeoutMs}) }); + result.cors = { status: response.status, type: response.type, allowOrigin: response.headers.get("access-control-allow-origin") === null ? "absent" : (response.headers.get("access-control-allow-origin") === location.origin || response.headers.get("access-control-allow-origin") === "*" ? "matches" : "other") }; + try { + const cache = await caches.open("display-repro-probe"); + await cache.put("/display-repro-probe/cors", response.clone()); + result.cors.cachePut = "stored"; + } catch (error) { result.cors.cachePut = "refused: " + error.name; } + } catch (error) { result.cors = { error: error.name }; } + try { + const response = await fetch(url, { mode: "no-cors", signal: AbortSignal.timeout(${requestTimeoutMs}) }); + result.noCors = { status: response.status, type: response.type, ok: response.ok }; + try { + const cache = await caches.open("display-repro-probe"); + await cache.put("/display-repro-probe/opaque", response.clone()); + result.noCors.cachePut = "stored"; + } catch (error) { result.noCors.cachePut = "refused: " + error.name; } + } catch (error) { result.noCors = { error: error.name }; } + result.imgPlain = await new Promise((resolve) => { const i = new Image(); i.onload = () => resolve("loaded"); i.onerror = () => resolve("error"); i.src = url; }); + result.imgCors = await new Promise((resolve) => { const i = new Image(); i.crossOrigin = "anonymous"; i.onload = () => resolve("loaded"); i.onerror = () => resolve("error"); i.src = url + (url.includes("?") ? "&" : "?") + "cors-probe=1"; }); + await caches.delete("display-repro-probe"); + result.signageCachedEntries = await caches.open("ekklesia-signage-media-v1").then((c) => c.keys()).then((k) => k.length).catch(() => "unavailable"); + return result; + })()`, + "probing cross-origin image caching", + ); + note(`caching probe: ${JSON.stringify(probe)}`); + summarize(page, stepStartedAt); + check(probe.signageCachedEntries > 0, "Signage has stored slide images in Cache Storage"); + await page.close(); + kiosk.close(); + await finishOwner(owner); + } else if (mode === "stream-recycle") { + // One page, past the five-minute realtime connection age, with every stream's open and end + // time in UTC so they can be laid against the proxy's own log. + const owner = await ownerPage(); + const moduleId = await displayModuleId(owner.page); + const kiosk = await connect(kioskAddress, "kiosk"); + const page = await openPage(kiosk, `${apiOrigin}/display/${moduleId}`, "recycle"); + check(await waitLive(page, "the display"), "the display is live"); + const result = await watch([page], recycleWatchMs, { viewerOwner: owner.page }); + printTally(result); + summarize(page, result.from); + note(`connectivity events ${JSON.stringify(await connectivityEvents(page))}`); + const blobs = await blobCounts(page); + const exhausted = page.stateFetches.filter((fetch) => + fetch.outcome.includes("INSUFFICIENT_RESOURCES"), + ); + note( + `object URLs: created ${blobs.created}, revoked ${blobs.revoked}, live ${blobs.created - blobs.revoked}; channel-state fetches refused for insufficient resources: ${exhausted.length}`, + ); + check( + blobs.created - blobs.revoked <= MAX_LIVE_OBJECT_URLS, + `live object URLs stay bounded (at most ${MAX_LIVE_OBJECT_URLS}; ${blobs.created - blobs.revoked} live)`, + ); + check( + result.tally.get("recycle").offline === 0, + "the display stayed live across the realtime recycle", + ); + check(page.sessionRequests.length === 1, "the realtime recycle did not remount the display"); + await page.close(); + kiosk.close(); + await finishOwner(owner); + } else if (mode === "revoke") { + const owner = await ownerPage(); + const moduleId = await displayModuleId(owner.page); + const kiosk = await connect(kioskAddress, "kiosk"); + const page = await openPage(kiosk, `${apiOrigin}/display/${moduleId}`, "revoked"); + check( + await waitLive(page, "the display before revocation"), + "the display is live before revocation", + ); + const devices = (await coreDevices(owner.page)).filter((device) => !device.revoked); + for (const device of devices) { + await owner.page.api("POST", `/api/v1/devices/${device.id}/revoke`); + } + note(`revoked ${devices.length} device(s) at ${clock()}; watching the open page`); + const result = await watch([page], watchMs, { onSample: (_page, state) => state.pairing }); + printTally(result); + summarize(page, result.from); + check( + result.stopped, + `a revoked display returns to the pairing screen without a reload (within ${watchMs / 1000}s)`, + ); + await page.close(); + kiosk.close(); + await finishOwner(owner); + } else { + throw new Error( + "usage: deploy-smoke-display.mjs pair|restart|two-pages|connectivity-blip|session-blip|stream-failure|caching-probe|stream-recycle|revoke", + ); + } +} + +try { + await run(); +} catch (error) { + if (error instanceof HangError) { + console.log(`FAIL ${mode}: ${error.message}; also in flight: ${describeInFlight()}`); + } else { + console.log(`FAIL ${mode}: ${error instanceof Error ? error.message : "an unexpected error"}`); + } + process.exit(2); +} + +if (failures > 0) { + console.error(`${failures} check(s) failed`); + process.exit(1); +} +process.exit(0); diff --git a/scripts/smoke-deploy.sh b/scripts/smoke-deploy.sh index a133a65..170370b 100755 --- a/scripts/smoke-deploy.sh +++ b/scripts/smoke-deploy.sh @@ -15,6 +15,12 @@ # With EKKLESIA_SMOKE_IMAGE, the given image is used as is in either topology; a split run then # exercises whichever of its modules publishes UI and a display. # +# Opt-in, split topology only: EKKLESIA_SMOKE_DISPLAY_DEVICE=1 adds the display-device scenarios in +# scripts/lib/deploy-smoke-display.mjs (pairing a kiosk browser with a persistent profile, a clean +# restart, two pages, injected network failures, revocation), each with its own time budget. +# EKKLESIA_SMOKE_DISPLAY_SCENARIOS selects a subset; EKKLESIA_SMOKE_WATCH_MS and +# EKKLESIA_SMOKE_RECYCLE_WATCH_MS change how long they watch. Several scenarios expect Signage. +# # TEST-ONLY: the proxy runs with Caddy's internal CA (`local_certs`) instead of ACME, because a test # run has no public hostname. Never use that option in an installation. # @@ -141,6 +147,12 @@ cleanup() { if [[ -n "$browser_container" ]]; then docker rm --force "$browser_container" >/dev/null 2>&1 || true fi + if [[ -n "${kiosk_container:-}" ]]; then + docker ps --all --quiet --filter "name=${project}-display-" | xargs -r docker rm --force \ + >/dev/null 2>&1 || true + docker rm --force "$kiosk_container" >/dev/null 2>&1 || true + docker volume rm --force "$kiosk_volume" >/dev/null 2>&1 || true + fi if [[ -f "$work/compose.env" ]]; then "${compose[@]}" --profile ops down --volumes --remove-orphans --timeout 10 >/dev/null 2>&1 || status=1 fi @@ -404,6 +416,81 @@ if [[ "$topology" == split ]]; then -e SMOKE_OWNER_EMAIL="$owner_email" -e SMOKE_OWNER_PASSWORD="$owner_password" \ -v "$daemon_work/browser.mjs:/smoke/browser.mjs:ro" \ --entrypoint node "$image" /smoke/browser.mjs || fail "browser checks failed." + + if [[ -n "${EKKLESIA_SMOKE_DISPLAY_DEVICE:-}" ]]; then + # A kiosk: a second browser whose profile directory is a volume, so it outlives the container + # exactly as a kiosk's --user-data-dir outlives a browser restart. + kiosk_container="${project}-kiosk" + kiosk_volume="${project}-kiosk-profile" + kiosk_address="10.249.${octet}.250" + docker volume create "$kiosk_volume" >/dev/null + start_kiosk() { + docker run --detach --rm --name "$kiosk_container" \ + --cap-drop ALL --security-opt no-new-privileges --tmpfs /tmp \ + --network "$edge_network" --ip "$kiosk_address" "${host_mappings[@]}" \ + -v "${kiosk_volume}:/profile" \ + "$browser_image" \ + "--ignore-certificate-errors-spki-list=${spki}" \ + --user-data-dir=/profile >/dev/null || fail "the kiosk browser did not start." + } + wait_kiosk_exit() { + for _ in $(seq 1 60); do + docker inspect "$kiosk_container" >/dev/null 2>&1 || return 0 + sleep 0.5 + done + fail "the kiosk browser did not exit after a clean close." + } + cp scripts/lib/deploy-smoke-display.mjs "$work/display.mjs" + stage display.mjs + # display_driver : one scenario with a hard budget. The driver enforces it + # itself and names whatever call was still waiting; `timeout` is the backstop if it cannot. + display_driver() { + local budget="$1" step_name="$2" driver="${project}-display-${2}" + timeout --signal=TERM "$((budget + 30))" \ + docker run --rm --name "$driver" --read-only --cap-drop ALL --security-opt no-new-privileges \ + --network "$edge_network" \ + -e SMOKE_CDP_ADDRESS="${browser_address}:9222" \ + -e SMOKE_KIOSK_CDP_ADDRESS="${kiosk_address}:9222" \ + -e SMOKE_STEP_BUDGET_MS="$((budget * 1000))" \ + -e SMOKE_WATCH_MS="${EKKLESIA_SMOKE_WATCH_MS:-120000}" \ + -e SMOKE_RECYCLE_WATCH_MS="${EKKLESIA_SMOKE_RECYCLE_WATCH_MS:-330000}" \ + -e SMOKE_HOSTNAME="$hostname" -e SMOKE_DASHBOARD_HOSTNAME="$dashboard_hostname" \ + -e SMOKE_OWNER_EMAIL="$owner_email" -e SMOKE_OWNER_PASSWORD="$owner_password" \ + -v "$daemon_work/display.mjs:/smoke/display.mjs:ro" \ + --entrypoint node "$image" /smoke/display.mjs "$step_name" + local status=$? + if ((status == 124)); then + echo "FAIL ${step_name}: the driver did not stop within its budget plus 30s; removed it" + docker rm --force "$driver" >/dev/null 2>&1 || true + fi + return "$status" + } + display_status=0 + step "Pairing a kiosk display, then closing its browser cleanly" + start_kiosk + display_driver 180 pair || display_status=1 + wait_kiosk_exit + step "Restarting the kiosk browser on the same profile" + start_kiosk + display_driver 120 restart || display_status=1 + for scenario in ${EKKLESIA_SMOKE_DISPLAY_SCENARIOS:-two-pages connectivity-blip session-blip stream-failure caching-probe stream-recycle revoke}; do + case "$scenario" in + two-pages) budget=240 ;; + stream-recycle) budget=420 ;; + revoke) budget=210 ;; + *) budget=150 ;; + esac + step "Display scenario: ${scenario} (budget ${budget}s)" + display_driver "$budget" "$scenario" || display_status=1 + done + step "Proxy: streams aborted with an incomplete response (UTC time, path only)" + # Captured, never printed: only each line's time and path (no query, so no signature) leave it. + proxy_logs="$("${compose[@]}" logs --no-color proxy 2>/dev/null)" || proxy_logs="" + grep -o '{.*"aborting with incomplete response".*}' <<<"$proxy_logs" | + sed -E 's/.*"ts":([0-9]+)[^,]*,.*"uri":"([^"?]*)[^"]*".*/\1 \2/' | + while read -r ts uri; do echo " $(date -u -d "@${ts}" +%T) ${uri}"; done || true + ((display_status == 0)) || fail "display-device checks failed." + fi fi step "Proving the API's database connection is certificate-verified TLS" -- 2.54.0 From f50e8f42018c15698709739e2ba54771e97df08a Mon Sep 17 00:00:00 2001 From: ekklesia Date: Mon, 5 Oct 2026 19:09:51 -0500 Subject: [PATCH 2/2] test(deploy): word the long-watch checks for any watch length The recycle step's checks spoke of "the realtime recycle" even when run with a watch shorter than the five-minute connection age, as module CI does. They now say what they check: live for the whole watch (plus "across a realtime recycle" when the watch is long enough), and never remounted. Co-Authored-By: Claude Opus 5.5 --- scripts/lib/deploy-smoke-display.mjs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/scripts/lib/deploy-smoke-display.mjs b/scripts/lib/deploy-smoke-display.mjs index ec5fd60..1291faa 100644 --- a/scripts/lib/deploy-smoke-display.mjs +++ b/scripts/lib/deploy-smoke-display.mjs @@ -924,9 +924,9 @@ async function run() { ); check( result.tally.get("recycle").offline === 0, - "the display stayed live across the realtime recycle", + `the display stayed live for the whole ${recycleWatchMs / 1000}s watch${recycleWatchMs > 300_000 ? ", across a realtime recycle" : ""}`, ); - check(page.sessionRequests.length === 1, "the realtime recycle did not remount the display"); + check(page.sessionRequests.length === 1, "the display was never remounted during the watch"); await page.close(); kiosk.close(); await finishOwner(owner); -- 2.54.0