fix(deploy): reach storage over HTTP/1.1, and own storage CORS in the proxy

Every reload of a cached files. image failed in the browser with
ERR_HTTP2_PROTOCOL_ERROR. A browser revalidates a cached signed URL
with If-None-Match. Caddy reached SeaweedFS over HTTP/2, and there
SeaweedFS answers with a 304 declaring content-type: application/xml
and content-length: 327 while sending no body. Caddy then logged
"aborting with incomplete response ... unexpected EOF" and reset the
stream. Over HTTP/1.1 SeaweedFS's 304 is clean. Measured layer by layer:
DevTools, curl through Caddy, and curl direct over HTTP/1.1 and HTTP/2.

The storage upstream is now HTTP/1.1 only (`versions 1.1`). Browsers
still reach the proxy over HTTP/2 and HTTP/3.

Delivery-layer CORS is the deployment's, so it no longer depends on
which S3 backend is behind it. SeaweedFS's default reflected any origin
and advertised PUT, POST and DELETE. Now the proxy:
- grants CORS only to the origins that read objects with fetch(): the
  Ekklesia origin, or, when split, the API origin (displays) and the
  Dashboard origin (a module's preview caches images through fetch());
- allows GET and HEAD only, with Vary: Origin;
- answers preflights itself, so they never reach storage;
- strips the object store's own CORS headers.

The deploy README documents the policy, its reasons, and what a
self-run proxy must do.

Co-Authored-By: Claude Opus 5.5 <[email protected]>

deploy/Caddyfile.example

@@ -70,18 +70,57 @@
 
 # Browsers fetch signed object URLs here (S3_DELIVERY_ENDPOINT, ADR 0031).
 {$EKKLESIA_STORAGE_HOSTNAME} {
-	reverse_proxy https://object-storage:8443 {
+	# Delivery-layer CORS belongs to the deployment, not to the object store: this block decides it,
-		# An S3 signature covers the Host header: pass the original through unchanged, or every signed
+	# so it stays the same with any S3 backend behind it. Only the origins whose pages read objects
-		# URL fails verification.
+	# with fetch() get CORS: the Ekklesia origin, which serves the Dashboard and the displays. An <img> needs none. Every object
-		header_up Host {host}
+	# URL is still a signed, expiring capability; CORS only decides which pages may read the bytes.
-		header_up -CF-Connecting-IP
+	@cors_origin header Origin https://{$EKKLESIA_HOSTNAME}
-		header_up -X-Real-IP
+
-		header_up -Forwarded
+	# A preflight is answered here and never reaches storage. Only GET and HEAD are ever allowed, and
-		header_up -True-Client-IP
+	# no request header beyond the CORS-safelisted ones.
-		transport http {
+	@preflight method OPTIONS
-			# object-storage presents a certificate from the installation-private CA.
+	handle @preflight {
-			tls_trust_pool file /etc/caddy/ekklesia-ca.crt
+		header @cors_origin {
-			tls_server_name object-storage
+			Access-Control-Allow-Origin {http.request.header.Origin}
+			Access-Control-Allow-Methods "GET, HEAD"
+			Access-Control-Max-Age 600
+		}
+		header Vary Origin
+		respond 204
+	}
+
+	handle {
+		# Set when the response is written, after the upstream's own CORS headers were removed below.
+		header @cors_origin >Access-Control-Allow-Origin {http.request.header.Origin}
+		header >Vary Origin
+
+		reverse_proxy https://object-storage:8443 {
+			# An S3 signature covers the Host header: pass the original through unchanged, or every
+			# signed URL fails verification.
+			header_up Host {host}
+			header_up -CF-Connecting-IP
+			header_up -X-Real-IP
+			header_up -Forwarded
+			header_up -True-Client-IP
+			# The object store's own CORS answer (SeaweedFS reflects any origin, with every method)
+			# is not the deployment's policy.
+			header_down -Access-Control-Allow-Origin
+			header_down -Access-Control-Allow-Methods
+			header_down -Access-Control-Allow-Headers
+			header_down -Access-Control-Expose-Headers
+			header_down -Access-Control-Allow-Credentials
+			header_down -Access-Control-Max-Age
+			header_down -Vary
+			transport http {
+				# object-storage presents a certificate from the installation-private CA.
+				tls_trust_pool file /etc/caddy/ekklesia-ca.crt
+				tls_server_name object-storage
+				# HTTP/1.1 only. Over HTTP/2, SeaweedFS answers a conditional request with a 304 that
+				# declares a Content-Length it never sends; Caddy then aborts the browser's stream,
+				# which a browser reports as ERR_HTTP2_PROTOCOL_ERROR on every revalidation. Its
+				# HTTP/1.1 304 is clean. Browsers still reach this proxy over HTTP/2 and HTTP/3.
+				versions 1.1
+			}
 		}
 	}
 }

deploy/Caddyfile.split.example

@@ -92,18 +92,61 @@
 
 # Browsers fetch signed object URLs here (S3_DELIVERY_ENDPOINT, ADR 0031).
 {$EKKLESIA_STORAGE_HOSTNAME} {
-	reverse_proxy https://object-storage:8443 {
+	# Delivery-layer CORS belongs to the deployment, not to the object store: this block decides it,
-		# An S3 signature covers the Host header: pass the original through unchanged, or every signed
+	# so it stays the same with any S3 backend behind it. Only the origins whose pages read objects
-		# URL fails verification.
+	# with fetch() get CORS: the API origin, where displays run, and the Dashboard origin,
-		header_up Host {host}
+	# where a module's preview may cache images too. An <img> needs none. Every object
-		header_up -CF-Connecting-IP
+	# URL is still a signed, expiring capability; CORS only decides which pages may read the bytes.
-		header_up -X-Real-IP
+	@cors_origin {
-		header_up -Forwarded
+		header Origin https://{$EKKLESIA_HOSTNAME}
-		header_up -True-Client-IP
+		header Origin https://{$EKKLESIA_DASHBOARD_HOSTNAME}
-		transport http {
+	}
-			# object-storage presents a certificate from the installation-private CA.
+
-			tls_trust_pool file /etc/caddy/ekklesia-ca.crt
+	# A preflight is answered here and never reaches storage. Only GET and HEAD are ever allowed, and
-			tls_server_name object-storage
+	# no request header beyond the CORS-safelisted ones.
+	@preflight method OPTIONS
+	handle @preflight {
+		header @cors_origin {
+			Access-Control-Allow-Origin {http.request.header.Origin}
+			Access-Control-Allow-Methods "GET, HEAD"
+			Access-Control-Max-Age 600
+		}
+		header Vary Origin
+		respond 204
+	}
+
+	handle {
+		# Set when the response is written, after the upstream's own CORS headers were removed below.
+		header @cors_origin >Access-Control-Allow-Origin {http.request.header.Origin}
+		header >Vary Origin
+
+		reverse_proxy https://object-storage:8443 {
+			# An S3 signature covers the Host header: pass the original through unchanged, or every
+			# signed URL fails verification.
+			header_up Host {host}
+			header_up -CF-Connecting-IP
+			header_up -X-Real-IP
+			header_up -Forwarded
+			header_up -True-Client-IP
+			# The object store's own CORS answer (SeaweedFS reflects any origin, with every method)
+			# is not the deployment's policy.
+			header_down -Access-Control-Allow-Origin
+			header_down -Access-Control-Allow-Methods
+			header_down -Access-Control-Allow-Headers
+			header_down -Access-Control-Expose-Headers
+			header_down -Access-Control-Allow-Credentials
+			header_down -Access-Control-Max-Age
+			header_down -Vary
+			transport http {
+				# object-storage presents a certificate from the installation-private CA.
+				tls_trust_pool file /etc/caddy/ekklesia-ca.crt
+				tls_server_name object-storage
+				# HTTP/1.1 only. Over HTTP/2, SeaweedFS answers a conditional request with a 304 that
+				# declares a Content-Length it never sends; Caddy then aborts the browser's stream,
+				# which a browser reports as ERR_HTTP2_PROTOCOL_ERROR on every revalidation. Its
+				# HTTP/1.1 304 is clean. Browsers still reach this proxy over HTTP/2 and HTTP/3.
+				versions 1.1
+			}
 		}
 	}
 }

deploy/README.md

@@ -178,6 +178,29 @@ the topology and warns when the two hostnames are not siblings.
 The Dashboard learns the API's origin at run time: the API writes its identity base URL into the page
 it serves (`<meta name="ekklesia-api-origin">`). Nothing about the topology is built into the image.
 
+### Storage delivery: CORS is the deployment's
+
+Browsers fetch objects directly from the storage hostname through signed, expiring URLs (ADR 0031).
+**Which pages may read those bytes with `fetch()` is decided by the delivery layer, the proxy, and
+never by the object store.** That policy therefore stays the same with SeaweedFS, R2, or any other
+S3 backend behind it, and an installation that changes backends changes nothing about CORS.
+
+The reference proxy's storage block:
+
+- grants CORS only to the origins whose pages read objects with `fetch()`: the Ekklesia origin in the
+  same-origin topology; the API origin (displays) and the Dashboard origin (a module's preview) in
+  the split topology. An `<img>` needs no CORS and is unaffected;
+- allows `GET` and `HEAD` only, and no request header beyond the CORS-safelisted ones, always with
+  `Vary: Origin`;
+- answers preflight `OPTIONS` requests itself, so they never reach storage;
+- removes the object store's own CORS headers. SeaweedFS's default reflects any origin and
+  advertises `PUT`, `POST`, and `DELETE`.
+
+The proxy reaches storage over **HTTP/1.1 only**. Over HTTP/2, SeaweedFS answers a conditional
+request with a `304` that declares a `Content-Length` it never sends. The proxy then resets the
+browser's stream, and every reload of a cached image fails with `ERR_HTTP2_PROTOCOL_ERROR`.
+Browsers still reach the proxy itself over HTTP/2 and HTTP/3.
+
 ### Behind an existing reverse proxy
 
 Run the base alone and attach `api` and `object-storage` to your proxy's network with an overlay of
@@ -213,6 +236,8 @@ Then make your proxy do what `Caddyfile.example` does:
 - forward the storage hostname to `https://object-storage:8443`, verifying it against
   `/etc/ekklesia/tls/ca.crt` with server name `object-storage`, and **pass the original `Host`
   through unchanged**, or every signed URL fails (ADR 0031).
+- reach `object-storage` over **HTTP/1.1** only, and own the storage hostname's CORS as described in
+  [Storage delivery](#storage-delivery-cors-is-the-deployments).
 
 The proxy's address must be fixed and must never be handed to anything else. With Docker, give its
 network an explicit subnet, limit `ip_range` to part of it, and fix the proxy's address outside that
@@ -288,7 +313,10 @@ deployment:
   platform-only image and a disposable PostgreSQL and SeaweedFS that exist only for that run. It
   follows the first boot above and checks the following through the proxy:
   - readiness and the Dashboard;
-  - a signed object URL fetched through the storage hostname;
+  - a signed object URL fetched through the storage hostname, and the same image loaded twice over
+    HTTP/2, the second time as a conditional request that must be a clean `304`;
+  - storage CORS: exactly the allowed origins, `GET` and `HEAD` only, preflights answered by the
+    proxy, and none of the object store's own CORS headers;
   - the client-address chain, both from an ordinary peer and from the tunnel connector's address;
   - initial-owner sign-in and sign-out;
   - that the API's database connection is TLS 1.3, and that it is refused by a foreign CA and by a

scripts/lib/deploy-smoke-client.mjs

@@ -14,6 +14,8 @@
 // Output is one line per check; any failed check exits non-zero. Nothing printed carries a
 // credential, cookie, signed URL, or response body beyond the fixed shapes asserted here.
 
+import { connect as connectHttp2 } from "node:http2";
+
 const mode = process.argv[2];
 const apiOrigin = `https://${required("SMOKE_HOSTNAME")}`;
 const dashboardOrigin = process.env.SMOKE_DASHBOARD_HOSTNAME
@@ -192,6 +194,46 @@ async function splitCorsChecks() {
   );
 }
 
+/**
+ * One GET over a real HTTP/2 connection. Node's fetch speaks HTTP/1.1, and the failure this guards
+ * against exists only on HTTP/2: a reset stream, which a browser reports as ERR_HTTP2_PROTOCOL_ERROR.
+ */
+function http2Get(url, headers = {}) {
+  return new Promise((resolve) => {
+    const session = connectHttp2(url.origin);
+    const finish = (result) => {
+      clearTimeout(timer);
+      session.close();
+      resolve(result);
+    };
+    const timer = setTimeout(() => finish({ error: "timeout" }), 10_000);
+    session.on("error", () => finish({ error: "session" }));
+    const stream = session.request({
+      ":method": "GET",
+      ":path": `${url.pathname}${url.search}`,
+      ...headers,
+    });
+    let status;
+    let responseHeaders = {};
+    let bytes = 0;
+    stream.on("response", (received) => {
+      status = received[":status"];
+      responseHeaders = received;
+    });
+    stream.on("data", (chunk) => {
+      bytes += chunk.length;
+    });
+    stream.on("error", (error) => finish({ error: error.code ?? "stream" }));
+    stream.on("end", () => finish({ bytes, headers: responseHeaders, status }));
+    stream.end();
+  });
+}
+
+/** The CORS headers of a response, by name: what a browser would act on. */
+function corsOf(response) {
+  return [...response.headers.keys()].filter((name) => name.startsWith("access-control-"));
+}
+
 async function storageDeliveryCheck() {
   // Core's own provider, from the image, exactly as the API composes it: it reaches object-storage
   // over HTTPS on the private network and signs read URLs for the storage hostname.
@@ -238,6 +280,79 @@ async function storageDeliveryCheck() {
       "a delivered object is Cache-Control: private",
     );
     await provider.delete(bucket, "smoke/object.txt");
+
+    // The same image, twice, over HTTP/2: a browser's second load of a signed URL it has cached is a
+    // conditional request, and must come back as a clean 304, not a reset stream.
+    const png = Buffer.from(
+      "iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAADUlEQVR42mP8z8BQDwAEhQGAhKmMIQAAAABJRU5ErkJggg==",
+      "base64",
+    );
+    await provider.put(bucket, "smoke/image.png", png, {
+      contentLength: png.byteLength,
+      contentType: "image/png",
+    });
+    const image = new URL(
+      (await provider.createReadUrl(bucket, "smoke/image.png", { expiresInSeconds: 300 })).url,
+    );
+    const first = await http2Get(image);
+    check(
+      first.status === 200 && first.bytes === png.byteLength && Boolean(first.headers?.etag),
+      "an image loads over HTTP/2 through the proxy, with a validator (200, ETag)",
+    );
+    const revalidated = await http2Get(image, { "if-none-match": first.headers?.etag ?? "" });
+    check(
+      revalidated.status === 304 && revalidated.error === undefined,
+      `the same image loaded again over HTTP/2 revalidates cleanly (${revalidated.status ?? revalidated.error})`,
+    );
+    const sinceModified = await http2Get(image, {
+      "if-modified-since": first.headers?.["last-modified"] ?? "",
+    });
+    check(
+      sinceModified.status === 304 && sinceModified.error === undefined,
+      `a revalidation by date over HTTP/2 is a clean 304 too (${sinceModified.status ?? sinceModified.error})`,
+    );
+
+    // Delivery-layer CORS is the deployment's own, set by the proxy: exactly the origins whose pages
+    // read objects with fetch(), GET and HEAD only, and nothing of the object store's own answer.
+    const fromApi = await fetch(image, { headers: { origin: apiOrigin } });
+    check(
+      fromApi.headers.get("access-control-allow-origin") === apiOrigin &&
+        (fromApi.headers.get("vary") ?? "").toLowerCase().includes("origin") &&
+        corsOf(fromApi).length === 1,
+      "a read from the API origin is allowed for exactly that origin, with Vary: Origin and nothing else",
+    );
+    if (split) {
+      const fromDashboard = await fetch(image, { headers: { origin: dashboardOrigin } });
+      check(
+        fromDashboard.headers.get("access-control-allow-origin") === dashboardOrigin,
+        "a read from the Dashboard origin is allowed for exactly that origin",
+      );
+    }
+    const fromElsewhere = await fetch(image, { headers: { origin: untrustedOrigin } });
+    check(
+      fromElsewhere.status === 200 && corsOf(fromElsewhere).length === 0,
+      "a read from any other origin gets no CORS header at all",
+    );
+    const preflight = await fetch(image, {
+      headers: { "access-control-request-method": "GET", origin: apiOrigin },
+      method: "OPTIONS",
+    });
+    check(
+      preflight.status === 204 &&
+        preflight.headers.get("access-control-allow-origin") === apiOrigin &&
+        preflight.headers.get("access-control-allow-methods") === "GET, HEAD" &&
+        preflight.headers.get("x-amz-request-id") === null,
+      "the proxy answers a preflight itself (204, GET and HEAD only), never the object store",
+    );
+    const foreignPreflight = await fetch(image, {
+      headers: { "access-control-request-method": "GET", origin: untrustedOrigin },
+      method: "OPTIONS",
+    });
+    check(
+      foreignPreflight.status === 204 && corsOf(foreignPreflight).length === 0,
+      "a preflight from any other origin is answered without permission",
+    );
+    await provider.delete(bucket, "smoke/image.png");
   } finally {
     await provider.close();
   }