From dcbd7613eedcbd04b167e32b519fa539d29e1439 Mon Sep 17 00:00:00 2001 From: ekklesia Date: Mon, 5 Oct 2026 19:02:36 -0500 Subject: [PATCH 1/2] test(deploy): load the same image twice over HTTP/2, and pin storage CORS Two checks on storage delivery through the proxy, in both topologies. The same signed image, twice, over a real HTTP/2 connection (node:http2; Node's fetch speaks HTTP/1.1, where the failure does not exist). The second load is conditional, as a browser's revalidation is, and must be a clean 304. Today Caddy reaches SeaweedFS over HTTP/2, SeaweedFS's 304 declares Content-Length: 327 and sends no body, and Caddy resets the stream. A browser reports that as ERR_HTTP2_PROTOCOL_ERROR on every reload of a cached files. image. CORS on delivery is the deployment's own: - exactly the API origin (and the Dashboard origin when split) is allowed, with Vary: Origin and nothing else; - any other origin gets no CORS header; - a preflight is answered by the proxy (204, GET and HEAD only) and never reaches the object store. Today SeaweedFS reflects every origin with PUT, POST and DELETE, and answers the preflight itself. Co-Authored-By: Claude Opus 5.5 --- scripts/lib/deploy-smoke-client.mjs | 115 ++++++++++++++++++++++++++++ 1 file changed, 115 insertions(+) diff --git a/scripts/lib/deploy-smoke-client.mjs b/scripts/lib/deploy-smoke-client.mjs index 4ec349f..4157d6e 100644 --- a/scripts/lib/deploy-smoke-client.mjs +++ b/scripts/lib/deploy-smoke-client.mjs @@ -14,6 +14,8 @@ // Output is one line per check; any failed check exits non-zero. Nothing printed carries a // credential, cookie, signed URL, or response body beyond the fixed shapes asserted here. +import { connect as connectHttp2 } from "node:http2"; + const mode = process.argv[2]; const apiOrigin = `https://${required("SMOKE_HOSTNAME")}`; const dashboardOrigin = process.env.SMOKE_DASHBOARD_HOSTNAME @@ -192,6 +194,46 @@ async function splitCorsChecks() { ); } +/** + * One GET over a real HTTP/2 connection. Node's fetch speaks HTTP/1.1, and the failure this guards + * against exists only on HTTP/2: a reset stream, which a browser reports as ERR_HTTP2_PROTOCOL_ERROR. + */ +function http2Get(url, headers = {}) { + return new Promise((resolve) => { + const session = connectHttp2(url.origin); + const finish = (result) => { + clearTimeout(timer); + session.close(); + resolve(result); + }; + const timer = setTimeout(() => finish({ error: "timeout" }), 10_000); + session.on("error", () => finish({ error: "session" })); + const stream = session.request({ + ":method": "GET", + ":path": `${url.pathname}${url.search}`, + ...headers, + }); + let status; + let responseHeaders = {}; + let bytes = 0; + stream.on("response", (received) => { + status = received[":status"]; + responseHeaders = received; + }); + stream.on("data", (chunk) => { + bytes += chunk.length; + }); + stream.on("error", (error) => finish({ error: error.code ?? "stream" })); + stream.on("end", () => finish({ bytes, headers: responseHeaders, status })); + stream.end(); + }); +} + +/** The CORS headers of a response, by name: what a browser would act on. */ +function corsOf(response) { + return [...response.headers.keys()].filter((name) => name.startsWith("access-control-")); +} + async function storageDeliveryCheck() { // Core's own provider, from the image, exactly as the API composes it: it reaches object-storage // over HTTPS on the private network and signs read URLs for the storage hostname. @@ -238,6 +280,79 @@ async function storageDeliveryCheck() { "a delivered object is Cache-Control: private", ); await provider.delete(bucket, "smoke/object.txt"); + + // The same image, twice, over HTTP/2: a browser's second load of a signed URL it has cached is a + // conditional request, and must come back as a clean 304, not a reset stream. + const png = Buffer.from( + "iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAADUlEQVR42mP8z8BQDwAEhQGAhKmMIQAAAABJRU5ErkJggg==", + "base64", + ); + await provider.put(bucket, "smoke/image.png", png, { + contentLength: png.byteLength, + contentType: "image/png", + }); + const image = new URL( + (await provider.createReadUrl(bucket, "smoke/image.png", { expiresInSeconds: 300 })).url, + ); + const first = await http2Get(image); + check( + first.status === 200 && first.bytes === png.byteLength && Boolean(first.headers?.etag), + "an image loads over HTTP/2 through the proxy, with a validator (200, ETag)", + ); + const revalidated = await http2Get(image, { "if-none-match": first.headers?.etag ?? "" }); + check( + revalidated.status === 304 && revalidated.error === undefined, + `the same image loaded again over HTTP/2 revalidates cleanly (${revalidated.status ?? revalidated.error})`, + ); + const sinceModified = await http2Get(image, { + "if-modified-since": first.headers?.["last-modified"] ?? "", + }); + check( + sinceModified.status === 304 && sinceModified.error === undefined, + `a revalidation by date over HTTP/2 is a clean 304 too (${sinceModified.status ?? sinceModified.error})`, + ); + + // Delivery-layer CORS is the deployment's own, set by the proxy: exactly the origins whose pages + // read objects with fetch(), GET and HEAD only, and nothing of the object store's own answer. + const fromApi = await fetch(image, { headers: { origin: apiOrigin } }); + check( + fromApi.headers.get("access-control-allow-origin") === apiOrigin && + (fromApi.headers.get("vary") ?? "").toLowerCase().includes("origin") && + corsOf(fromApi).length === 1, + "a read from the API origin is allowed for exactly that origin, with Vary: Origin and nothing else", + ); + if (split) { + const fromDashboard = await fetch(image, { headers: { origin: dashboardOrigin } }); + check( + fromDashboard.headers.get("access-control-allow-origin") === dashboardOrigin, + "a read from the Dashboard origin is allowed for exactly that origin", + ); + } + const fromElsewhere = await fetch(image, { headers: { origin: untrustedOrigin } }); + check( + fromElsewhere.status === 200 && corsOf(fromElsewhere).length === 0, + "a read from any other origin gets no CORS header at all", + ); + const preflight = await fetch(image, { + headers: { "access-control-request-method": "GET", origin: apiOrigin }, + method: "OPTIONS", + }); + check( + preflight.status === 204 && + preflight.headers.get("access-control-allow-origin") === apiOrigin && + preflight.headers.get("access-control-allow-methods") === "GET, HEAD" && + preflight.headers.get("x-amz-request-id") === null, + "the proxy answers a preflight itself (204, GET and HEAD only), never the object store", + ); + const foreignPreflight = await fetch(image, { + headers: { "access-control-request-method": "GET", origin: untrustedOrigin }, + method: "OPTIONS", + }); + check( + foreignPreflight.status === 204 && corsOf(foreignPreflight).length === 0, + "a preflight from any other origin is answered without permission", + ); + await provider.delete(bucket, "smoke/image.png"); } finally { await provider.close(); } From e1ac1da4166cdfe78b4b286cdbb97bb4bd2c09db Mon Sep 17 00:00:00 2001 From: ekklesia Date: Mon, 5 Oct 2026 19:04:06 -0500 Subject: [PATCH 2/2] fix(deploy): reach storage over HTTP/1.1, and own storage CORS in the proxy Every reload of a cached files. image failed in the browser with ERR_HTTP2_PROTOCOL_ERROR. A browser revalidates a cached signed URL with If-None-Match. Caddy reached SeaweedFS over HTTP/2, and there SeaweedFS answers with a 304 declaring content-type: application/xml and content-length: 327 while sending no body. Caddy then logged "aborting with incomplete response ... unexpected EOF" and reset the stream. Over HTTP/1.1 SeaweedFS's 304 is clean. Measured layer by layer: DevTools, curl through Caddy, and curl direct over HTTP/1.1 and HTTP/2. The storage upstream is now HTTP/1.1 only (`versions 1.1`). Browsers still reach the proxy over HTTP/2 and HTTP/3. Delivery-layer CORS is the deployment's, so it no longer depends on which S3 backend is behind it. SeaweedFS's default reflected any origin and advertised PUT, POST and DELETE. Now the proxy: - grants CORS only to the origins that read objects with fetch(): the Ekklesia origin, or, when split, the API origin (displays) and the Dashboard origin (a module's preview caches images through fetch()); - allows GET and HEAD only, with Vary: Origin; - answers preflights itself, so they never reach storage; - strips the object store's own CORS headers. The deploy README documents the policy, its reasons, and what a self-run proxy must do. Co-Authored-By: Claude Opus 5.5 --- deploy/Caddyfile.example | 63 ++++++++++++++++++++++++++------ deploy/Caddyfile.split.example | 67 ++++++++++++++++++++++++++++------ deploy/README.md | 30 ++++++++++++++- 3 files changed, 135 insertions(+), 25 deletions(-) diff --git a/deploy/Caddyfile.example b/deploy/Caddyfile.example index 0bb9357..f25a23a 100644 --- a/deploy/Caddyfile.example +++ b/deploy/Caddyfile.example @@ -70,18 +70,57 @@ # Browsers fetch signed object URLs here (S3_DELIVERY_ENDPOINT, ADR 0031). {$EKKLESIA_STORAGE_HOSTNAME} { - reverse_proxy https://object-storage:8443 { - # An S3 signature covers the Host header: pass the original through unchanged, or every signed - # URL fails verification. - header_up Host {host} - header_up -CF-Connecting-IP - header_up -X-Real-IP - header_up -Forwarded - header_up -True-Client-IP - transport http { - # object-storage presents a certificate from the installation-private CA. - tls_trust_pool file /etc/caddy/ekklesia-ca.crt - tls_server_name object-storage + # Delivery-layer CORS belongs to the deployment, not to the object store: this block decides it, + # so it stays the same with any S3 backend behind it. Only the origins whose pages read objects + # with fetch() get CORS: the Ekklesia origin, which serves the Dashboard and the displays. An needs none. Every object + # URL is still a signed, expiring capability; CORS only decides which pages may read the bytes. + @cors_origin header Origin https://{$EKKLESIA_HOSTNAME} + + # A preflight is answered here and never reaches storage. Only GET and HEAD are ever allowed, and + # no request header beyond the CORS-safelisted ones. + @preflight method OPTIONS + handle @preflight { + header @cors_origin { + Access-Control-Allow-Origin {http.request.header.Origin} + Access-Control-Allow-Methods "GET, HEAD" + Access-Control-Max-Age 600 + } + header Vary Origin + respond 204 + } + + handle { + # Set when the response is written, after the upstream's own CORS headers were removed below. + header @cors_origin >Access-Control-Allow-Origin {http.request.header.Origin} + header >Vary Origin + + reverse_proxy https://object-storage:8443 { + # An S3 signature covers the Host header: pass the original through unchanged, or every + # signed URL fails verification. + header_up Host {host} + header_up -CF-Connecting-IP + header_up -X-Real-IP + header_up -Forwarded + header_up -True-Client-IP + # The object store's own CORS answer (SeaweedFS reflects any origin, with every method) + # is not the deployment's policy. + header_down -Access-Control-Allow-Origin + header_down -Access-Control-Allow-Methods + header_down -Access-Control-Allow-Headers + header_down -Access-Control-Expose-Headers + header_down -Access-Control-Allow-Credentials + header_down -Access-Control-Max-Age + header_down -Vary + transport http { + # object-storage presents a certificate from the installation-private CA. + tls_trust_pool file /etc/caddy/ekklesia-ca.crt + tls_server_name object-storage + # HTTP/1.1 only. Over HTTP/2, SeaweedFS answers a conditional request with a 304 that + # declares a Content-Length it never sends; Caddy then aborts the browser's stream, + # which a browser reports as ERR_HTTP2_PROTOCOL_ERROR on every revalidation. Its + # HTTP/1.1 304 is clean. Browsers still reach this proxy over HTTP/2 and HTTP/3. + versions 1.1 + } } } } diff --git a/deploy/Caddyfile.split.example b/deploy/Caddyfile.split.example index ae0ffe4..19196a3 100644 --- a/deploy/Caddyfile.split.example +++ b/deploy/Caddyfile.split.example @@ -92,18 +92,61 @@ # Browsers fetch signed object URLs here (S3_DELIVERY_ENDPOINT, ADR 0031). {$EKKLESIA_STORAGE_HOSTNAME} { - reverse_proxy https://object-storage:8443 { - # An S3 signature covers the Host header: pass the original through unchanged, or every signed - # URL fails verification. - header_up Host {host} - header_up -CF-Connecting-IP - header_up -X-Real-IP - header_up -Forwarded - header_up -True-Client-IP - transport http { - # object-storage presents a certificate from the installation-private CA. - tls_trust_pool file /etc/caddy/ekklesia-ca.crt - tls_server_name object-storage + # Delivery-layer CORS belongs to the deployment, not to the object store: this block decides it, + # so it stays the same with any S3 backend behind it. Only the origins whose pages read objects + # with fetch() get CORS: the API origin, where displays run, and the Dashboard origin, + # where a module's preview may cache images too. An needs none. Every object + # URL is still a signed, expiring capability; CORS only decides which pages may read the bytes. + @cors_origin { + header Origin https://{$EKKLESIA_HOSTNAME} + header Origin https://{$EKKLESIA_DASHBOARD_HOSTNAME} + } + + # A preflight is answered here and never reaches storage. Only GET and HEAD are ever allowed, and + # no request header beyond the CORS-safelisted ones. + @preflight method OPTIONS + handle @preflight { + header @cors_origin { + Access-Control-Allow-Origin {http.request.header.Origin} + Access-Control-Allow-Methods "GET, HEAD" + Access-Control-Max-Age 600 + } + header Vary Origin + respond 204 + } + + handle { + # Set when the response is written, after the upstream's own CORS headers were removed below. + header @cors_origin >Access-Control-Allow-Origin {http.request.header.Origin} + header >Vary Origin + + reverse_proxy https://object-storage:8443 { + # An S3 signature covers the Host header: pass the original through unchanged, or every + # signed URL fails verification. + header_up Host {host} + header_up -CF-Connecting-IP + header_up -X-Real-IP + header_up -Forwarded + header_up -True-Client-IP + # The object store's own CORS answer (SeaweedFS reflects any origin, with every method) + # is not the deployment's policy. + header_down -Access-Control-Allow-Origin + header_down -Access-Control-Allow-Methods + header_down -Access-Control-Allow-Headers + header_down -Access-Control-Expose-Headers + header_down -Access-Control-Allow-Credentials + header_down -Access-Control-Max-Age + header_down -Vary + transport http { + # object-storage presents a certificate from the installation-private CA. + tls_trust_pool file /etc/caddy/ekklesia-ca.crt + tls_server_name object-storage + # HTTP/1.1 only. Over HTTP/2, SeaweedFS answers a conditional request with a 304 that + # declares a Content-Length it never sends; Caddy then aborts the browser's stream, + # which a browser reports as ERR_HTTP2_PROTOCOL_ERROR on every revalidation. Its + # HTTP/1.1 304 is clean. Browsers still reach this proxy over HTTP/2 and HTTP/3. + versions 1.1 + } } } } diff --git a/deploy/README.md b/deploy/README.md index 42c4618..62d1452 100644 --- a/deploy/README.md +++ b/deploy/README.md @@ -178,6 +178,29 @@ the topology and warns when the two hostnames are not siblings. The Dashboard learns the API's origin at run time: the API writes its identity base URL into the page it serves (``). Nothing about the topology is built into the image. +### Storage delivery: CORS is the deployment's + +Browsers fetch objects directly from the storage hostname through signed, expiring URLs (ADR 0031). +**Which pages may read those bytes with `fetch()` is decided by the delivery layer, the proxy, and +never by the object store.** That policy therefore stays the same with SeaweedFS, R2, or any other +S3 backend behind it, and an installation that changes backends changes nothing about CORS. + +The reference proxy's storage block: + +- grants CORS only to the origins whose pages read objects with `fetch()`: the Ekklesia origin in the + same-origin topology; the API origin (displays) and the Dashboard origin (a module's preview) in + the split topology. An `` needs no CORS and is unaffected; +- allows `GET` and `HEAD` only, and no request header beyond the CORS-safelisted ones, always with + `Vary: Origin`; +- answers preflight `OPTIONS` requests itself, so they never reach storage; +- removes the object store's own CORS headers. SeaweedFS's default reflects any origin and + advertises `PUT`, `POST`, and `DELETE`. + +The proxy reaches storage over **HTTP/1.1 only**. Over HTTP/2, SeaweedFS answers a conditional +request with a `304` that declares a `Content-Length` it never sends. The proxy then resets the +browser's stream, and every reload of a cached image fails with `ERR_HTTP2_PROTOCOL_ERROR`. +Browsers still reach the proxy itself over HTTP/2 and HTTP/3. + ### Behind an existing reverse proxy Run the base alone and attach `api` and `object-storage` to your proxy's network with an overlay of @@ -213,6 +236,8 @@ Then make your proxy do what `Caddyfile.example` does: - forward the storage hostname to `https://object-storage:8443`, verifying it against `/etc/ekklesia/tls/ca.crt` with server name `object-storage`, and **pass the original `Host` through unchanged**, or every signed URL fails (ADR 0031). +- reach `object-storage` over **HTTP/1.1** only, and own the storage hostname's CORS as described in + [Storage delivery](#storage-delivery-cors-is-the-deployments). The proxy's address must be fixed and must never be handed to anything else. With Docker, give its network an explicit subnet, limit `ip_range` to part of it, and fix the proxy's address outside that @@ -288,7 +313,10 @@ deployment: platform-only image and a disposable PostgreSQL and SeaweedFS that exist only for that run. It follows the first boot above and checks the following through the proxy: - readiness and the Dashboard; - - a signed object URL fetched through the storage hostname; + - a signed object URL fetched through the storage hostname, and the same image loaded twice over + HTTP/2, the second time as a conditional request that must be a clean `304`; + - storage CORS: exactly the allowed origins, `GET` and `HEAD` only, preflights answered by the + proxy, and none of the object store's own CORS headers; - the client-address chain, both from an ordinary peer and from the tunnel connector's address; - initial-owner sign-in and sign-out; - that the API's database connection is TLS 1.3, and that it is refused by a foreign CA and by a