diff --git a/deploy/Caddyfile.example b/deploy/Caddyfile.example index 0bb9357..f25a23a 100644 --- a/deploy/Caddyfile.example +++ b/deploy/Caddyfile.example @@ -70,18 +70,57 @@ # Browsers fetch signed object URLs here (S3_DELIVERY_ENDPOINT, ADR 0031). {$EKKLESIA_STORAGE_HOSTNAME} { - reverse_proxy https://object-storage:8443 { - # An S3 signature covers the Host header: pass the original through unchanged, or every signed - # URL fails verification. - header_up Host {host} - header_up -CF-Connecting-IP - header_up -X-Real-IP - header_up -Forwarded - header_up -True-Client-IP - transport http { - # object-storage presents a certificate from the installation-private CA. - tls_trust_pool file /etc/caddy/ekklesia-ca.crt - tls_server_name object-storage + # Delivery-layer CORS belongs to the deployment, not to the object store: this block decides it, + # so it stays the same with any S3 backend behind it. Only the origins whose pages read objects + # with fetch() get CORS: the Ekklesia origin, which serves the Dashboard and the displays. An needs none. Every object + # URL is still a signed, expiring capability; CORS only decides which pages may read the bytes. + @cors_origin header Origin https://{$EKKLESIA_HOSTNAME} + + # A preflight is answered here and never reaches storage. Only GET and HEAD are ever allowed, and + # no request header beyond the CORS-safelisted ones. + @preflight method OPTIONS + handle @preflight { + header @cors_origin { + Access-Control-Allow-Origin {http.request.header.Origin} + Access-Control-Allow-Methods "GET, HEAD" + Access-Control-Max-Age 600 + } + header Vary Origin + respond 204 + } + + handle { + # Set when the response is written, after the upstream's own CORS headers were removed below. + header @cors_origin >Access-Control-Allow-Origin {http.request.header.Origin} + header >Vary Origin + + reverse_proxy https://object-storage:8443 { + # An S3 signature covers the Host header: pass the original through unchanged, or every + # signed URL fails verification. + header_up Host {host} + header_up -CF-Connecting-IP + header_up -X-Real-IP + header_up -Forwarded + header_up -True-Client-IP + # The object store's own CORS answer (SeaweedFS reflects any origin, with every method) + # is not the deployment's policy. + header_down -Access-Control-Allow-Origin + header_down -Access-Control-Allow-Methods + header_down -Access-Control-Allow-Headers + header_down -Access-Control-Expose-Headers + header_down -Access-Control-Allow-Credentials + header_down -Access-Control-Max-Age + header_down -Vary + transport http { + # object-storage presents a certificate from the installation-private CA. + tls_trust_pool file /etc/caddy/ekklesia-ca.crt + tls_server_name object-storage + # HTTP/1.1 only. Over HTTP/2, SeaweedFS answers a conditional request with a 304 that + # declares a Content-Length it never sends; Caddy then aborts the browser's stream, + # which a browser reports as ERR_HTTP2_PROTOCOL_ERROR on every revalidation. Its + # HTTP/1.1 304 is clean. Browsers still reach this proxy over HTTP/2 and HTTP/3. + versions 1.1 + } } } } diff --git a/deploy/Caddyfile.split.example b/deploy/Caddyfile.split.example index ae0ffe4..19196a3 100644 --- a/deploy/Caddyfile.split.example +++ b/deploy/Caddyfile.split.example @@ -92,18 +92,61 @@ # Browsers fetch signed object URLs here (S3_DELIVERY_ENDPOINT, ADR 0031). {$EKKLESIA_STORAGE_HOSTNAME} { - reverse_proxy https://object-storage:8443 { - # An S3 signature covers the Host header: pass the original through unchanged, or every signed - # URL fails verification. - header_up Host {host} - header_up -CF-Connecting-IP - header_up -X-Real-IP - header_up -Forwarded - header_up -True-Client-IP - transport http { - # object-storage presents a certificate from the installation-private CA. - tls_trust_pool file /etc/caddy/ekklesia-ca.crt - tls_server_name object-storage + # Delivery-layer CORS belongs to the deployment, not to the object store: this block decides it, + # so it stays the same with any S3 backend behind it. Only the origins whose pages read objects + # with fetch() get CORS: the API origin, where displays run, and the Dashboard origin, + # where a module's preview may cache images too. An needs none. Every object + # URL is still a signed, expiring capability; CORS only decides which pages may read the bytes. + @cors_origin { + header Origin https://{$EKKLESIA_HOSTNAME} + header Origin https://{$EKKLESIA_DASHBOARD_HOSTNAME} + } + + # A preflight is answered here and never reaches storage. Only GET and HEAD are ever allowed, and + # no request header beyond the CORS-safelisted ones. + @preflight method OPTIONS + handle @preflight { + header @cors_origin { + Access-Control-Allow-Origin {http.request.header.Origin} + Access-Control-Allow-Methods "GET, HEAD" + Access-Control-Max-Age 600 + } + header Vary Origin + respond 204 + } + + handle { + # Set when the response is written, after the upstream's own CORS headers were removed below. + header @cors_origin >Access-Control-Allow-Origin {http.request.header.Origin} + header >Vary Origin + + reverse_proxy https://object-storage:8443 { + # An S3 signature covers the Host header: pass the original through unchanged, or every + # signed URL fails verification. + header_up Host {host} + header_up -CF-Connecting-IP + header_up -X-Real-IP + header_up -Forwarded + header_up -True-Client-IP + # The object store's own CORS answer (SeaweedFS reflects any origin, with every method) + # is not the deployment's policy. + header_down -Access-Control-Allow-Origin + header_down -Access-Control-Allow-Methods + header_down -Access-Control-Allow-Headers + header_down -Access-Control-Expose-Headers + header_down -Access-Control-Allow-Credentials + header_down -Access-Control-Max-Age + header_down -Vary + transport http { + # object-storage presents a certificate from the installation-private CA. + tls_trust_pool file /etc/caddy/ekklesia-ca.crt + tls_server_name object-storage + # HTTP/1.1 only. Over HTTP/2, SeaweedFS answers a conditional request with a 304 that + # declares a Content-Length it never sends; Caddy then aborts the browser's stream, + # which a browser reports as ERR_HTTP2_PROTOCOL_ERROR on every revalidation. Its + # HTTP/1.1 304 is clean. Browsers still reach this proxy over HTTP/2 and HTTP/3. + versions 1.1 + } } } } diff --git a/deploy/README.md b/deploy/README.md index 42c4618..62d1452 100644 --- a/deploy/README.md +++ b/deploy/README.md @@ -178,6 +178,29 @@ the topology and warns when the two hostnames are not siblings. The Dashboard learns the API's origin at run time: the API writes its identity base URL into the page it serves (``). Nothing about the topology is built into the image. +### Storage delivery: CORS is the deployment's + +Browsers fetch objects directly from the storage hostname through signed, expiring URLs (ADR 0031). +**Which pages may read those bytes with `fetch()` is decided by the delivery layer, the proxy, and +never by the object store.** That policy therefore stays the same with SeaweedFS, R2, or any other +S3 backend behind it, and an installation that changes backends changes nothing about CORS. + +The reference proxy's storage block: + +- grants CORS only to the origins whose pages read objects with `fetch()`: the Ekklesia origin in the + same-origin topology; the API origin (displays) and the Dashboard origin (a module's preview) in + the split topology. An `` needs no CORS and is unaffected; +- allows `GET` and `HEAD` only, and no request header beyond the CORS-safelisted ones, always with + `Vary: Origin`; +- answers preflight `OPTIONS` requests itself, so they never reach storage; +- removes the object store's own CORS headers. SeaweedFS's default reflects any origin and + advertises `PUT`, `POST`, and `DELETE`. + +The proxy reaches storage over **HTTP/1.1 only**. Over HTTP/2, SeaweedFS answers a conditional +request with a `304` that declares a `Content-Length` it never sends. The proxy then resets the +browser's stream, and every reload of a cached image fails with `ERR_HTTP2_PROTOCOL_ERROR`. +Browsers still reach the proxy itself over HTTP/2 and HTTP/3. + ### Behind an existing reverse proxy Run the base alone and attach `api` and `object-storage` to your proxy's network with an overlay of @@ -213,6 +236,8 @@ Then make your proxy do what `Caddyfile.example` does: - forward the storage hostname to `https://object-storage:8443`, verifying it against `/etc/ekklesia/tls/ca.crt` with server name `object-storage`, and **pass the original `Host` through unchanged**, or every signed URL fails (ADR 0031). +- reach `object-storage` over **HTTP/1.1** only, and own the storage hostname's CORS as described in + [Storage delivery](#storage-delivery-cors-is-the-deployments). The proxy's address must be fixed and must never be handed to anything else. With Docker, give its network an explicit subnet, limit `ip_range` to part of it, and fix the proxy's address outside that @@ -288,7 +313,10 @@ deployment: platform-only image and a disposable PostgreSQL and SeaweedFS that exist only for that run. It follows the first boot above and checks the following through the proxy: - readiness and the Dashboard; - - a signed object URL fetched through the storage hostname; + - a signed object URL fetched through the storage hostname, and the same image loaded twice over + HTTP/2, the second time as a conditional request that must be a clean `304`; + - storage CORS: exactly the allowed origins, `GET` and `HEAD` only, preflights answered by the + proxy, and none of the object store's own CORS headers; - the client-address chain, both from an ordinary peer and from the tunnel connector's address; - initial-owner sign-in and sign-out; - that the API's database connection is TLS 1.3, and that it is refused by a foreign CA and by a diff --git a/scripts/lib/deploy-smoke-client.mjs b/scripts/lib/deploy-smoke-client.mjs index 4ec349f..4157d6e 100644 --- a/scripts/lib/deploy-smoke-client.mjs +++ b/scripts/lib/deploy-smoke-client.mjs @@ -14,6 +14,8 @@ // Output is one line per check; any failed check exits non-zero. Nothing printed carries a // credential, cookie, signed URL, or response body beyond the fixed shapes asserted here. +import { connect as connectHttp2 } from "node:http2"; + const mode = process.argv[2]; const apiOrigin = `https://${required("SMOKE_HOSTNAME")}`; const dashboardOrigin = process.env.SMOKE_DASHBOARD_HOSTNAME @@ -192,6 +194,46 @@ async function splitCorsChecks() { ); } +/** + * One GET over a real HTTP/2 connection. Node's fetch speaks HTTP/1.1, and the failure this guards + * against exists only on HTTP/2: a reset stream, which a browser reports as ERR_HTTP2_PROTOCOL_ERROR. + */ +function http2Get(url, headers = {}) { + return new Promise((resolve) => { + const session = connectHttp2(url.origin); + const finish = (result) => { + clearTimeout(timer); + session.close(); + resolve(result); + }; + const timer = setTimeout(() => finish({ error: "timeout" }), 10_000); + session.on("error", () => finish({ error: "session" })); + const stream = session.request({ + ":method": "GET", + ":path": `${url.pathname}${url.search}`, + ...headers, + }); + let status; + let responseHeaders = {}; + let bytes = 0; + stream.on("response", (received) => { + status = received[":status"]; + responseHeaders = received; + }); + stream.on("data", (chunk) => { + bytes += chunk.length; + }); + stream.on("error", (error) => finish({ error: error.code ?? "stream" })); + stream.on("end", () => finish({ bytes, headers: responseHeaders, status })); + stream.end(); + }); +} + +/** The CORS headers of a response, by name: what a browser would act on. */ +function corsOf(response) { + return [...response.headers.keys()].filter((name) => name.startsWith("access-control-")); +} + async function storageDeliveryCheck() { // Core's own provider, from the image, exactly as the API composes it: it reaches object-storage // over HTTPS on the private network and signs read URLs for the storage hostname. @@ -238,6 +280,79 @@ async function storageDeliveryCheck() { "a delivered object is Cache-Control: private", ); await provider.delete(bucket, "smoke/object.txt"); + + // The same image, twice, over HTTP/2: a browser's second load of a signed URL it has cached is a + // conditional request, and must come back as a clean 304, not a reset stream. + const png = Buffer.from( + "iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAADUlEQVR42mP8z8BQDwAEhQGAhKmMIQAAAABJRU5ErkJggg==", + "base64", + ); + await provider.put(bucket, "smoke/image.png", png, { + contentLength: png.byteLength, + contentType: "image/png", + }); + const image = new URL( + (await provider.createReadUrl(bucket, "smoke/image.png", { expiresInSeconds: 300 })).url, + ); + const first = await http2Get(image); + check( + first.status === 200 && first.bytes === png.byteLength && Boolean(first.headers?.etag), + "an image loads over HTTP/2 through the proxy, with a validator (200, ETag)", + ); + const revalidated = await http2Get(image, { "if-none-match": first.headers?.etag ?? "" }); + check( + revalidated.status === 304 && revalidated.error === undefined, + `the same image loaded again over HTTP/2 revalidates cleanly (${revalidated.status ?? revalidated.error})`, + ); + const sinceModified = await http2Get(image, { + "if-modified-since": first.headers?.["last-modified"] ?? "", + }); + check( + sinceModified.status === 304 && sinceModified.error === undefined, + `a revalidation by date over HTTP/2 is a clean 304 too (${sinceModified.status ?? sinceModified.error})`, + ); + + // Delivery-layer CORS is the deployment's own, set by the proxy: exactly the origins whose pages + // read objects with fetch(), GET and HEAD only, and nothing of the object store's own answer. + const fromApi = await fetch(image, { headers: { origin: apiOrigin } }); + check( + fromApi.headers.get("access-control-allow-origin") === apiOrigin && + (fromApi.headers.get("vary") ?? "").toLowerCase().includes("origin") && + corsOf(fromApi).length === 1, + "a read from the API origin is allowed for exactly that origin, with Vary: Origin and nothing else", + ); + if (split) { + const fromDashboard = await fetch(image, { headers: { origin: dashboardOrigin } }); + check( + fromDashboard.headers.get("access-control-allow-origin") === dashboardOrigin, + "a read from the Dashboard origin is allowed for exactly that origin", + ); + } + const fromElsewhere = await fetch(image, { headers: { origin: untrustedOrigin } }); + check( + fromElsewhere.status === 200 && corsOf(fromElsewhere).length === 0, + "a read from any other origin gets no CORS header at all", + ); + const preflight = await fetch(image, { + headers: { "access-control-request-method": "GET", origin: apiOrigin }, + method: "OPTIONS", + }); + check( + preflight.status === 204 && + preflight.headers.get("access-control-allow-origin") === apiOrigin && + preflight.headers.get("access-control-allow-methods") === "GET, HEAD" && + preflight.headers.get("x-amz-request-id") === null, + "the proxy answers a preflight itself (204, GET and HEAD only), never the object store", + ); + const foreignPreflight = await fetch(image, { + headers: { "access-control-request-method": "GET", origin: untrustedOrigin }, + method: "OPTIONS", + }); + check( + foreignPreflight.status === 204 && corsOf(foreignPreflight).length === 0, + "a preflight from any other origin is answered without permission", + ); + await provider.delete(bucket, "smoke/image.png"); } finally { await provider.close(); }