test: prove bounded API cleanup

Pass 4 bounded module hooks and the HTTP drain, and Core proves an individual
hook deadline in isolation — but nothing under apps/api ever supplied
moduleHookTimeoutMsOverride, so the composition's actual promise ("the pool
always closes, whatever a hook does") was untested at the level where it either
holds or breaks.

Four deterministic proofs, no production change:

A never-settling startup rollback now proves startApi rejects with
MODULE_START_FAILED followed by MODULE_ROLLBACK_TIMEOUT, that the failing
module's own stop hook was attempted first and the earlier module still received
its rollback afterwards, that the one pool closed exactly once, and that no
listener was ever bound.

A never-settling normal stop proves shutdown rejects with MODULE_STOP_TIMEOUT
metadata, the earlier module still got its stop attempt, the pool closed, and
repeated and concurrent callers share the one memoized result without
re-entering a single hook.

A combined failure proves both deadlines report together: a server whose close
callback never fires plus a never-settling stop yields drainTimedOut and
moduleStopFailed on one ApiShutdownError, the platform force-closed through
Node's public closeIdleConnections/closeAllConnections, module stop still ran,
the pool still closed, and nothing about the request, socket, connection,
credentials, SQL, or module exception appears in the error.

The graceful ordering test was strengthened rather than left racing an
unresolved fetch: a raw socket sends a complete request with Connection: close,
shutdown begins only once the server provably holds that connection — observed
by polling the server's own connection count, never by sleeping — and the
already-entered request is then proven to complete with a full 200 response and
a clean transport close instead of being force-closed.

The fixture gains one test-only option for a stop hook that never settles.

Co-Authored-By: Claude Opus 5 <[email protected]>

apps/api/tests/api-http.integration.ts

@@ -2779,6 +2779,73 @@ describe.sequential("live API — bounded HTTP drain and shutdown", () => {
     }
   });
 
+  it("lets an already-entered request finish and close cleanly rather than force-closing it", async () => {
+    // Deterministic rather than racing an unresolved `fetch`: a raw socket sends a complete request
+    // with `Connection: close`, and shutdown only begins once the server provably holds that
+    // connection — observed by polling the real server's own connection count, never by sleeping.
+    const database = await createDisposableDatabase("drain_order");
+    await applyCoreMigrations(database.url);
+    const running = await startConfiguredApi(database.url, { drainTimeoutMsOverride: 20_000 });
+    const address = running.server.address();
+    if (!address || typeof address !== "object") {
+      throw harnessError("The API integration server has no bound address.");
+    }
+    const socket = net.connect({ host: "127.0.0.1", port: address.port });
+    try {
+      const responseBytes: Buffer[] = [];
+      const responseClosed = new Promise<void>((resolve) => {
+        socket.once("close", () => resolve());
+      });
+      socket.on("data", (chunk: Buffer) => {
+        responseBytes.push(chunk);
+      });
+      await new Promise<void>((resolve, reject) => {
+        socket.once("error", reject);
+        socket.write(
+          "GET /api/v1/principal HTTP/1.1\r\nHost: 127.0.0.1\r\nConnection: close\r\n\r\n",
+          () => resolve(),
+        );
+      });
+
+      // Bounded, condition-based: wait until the server actually holds the connection, so the
+      // request has demonstrably entered before shutdown starts.
+      const connectionCounter = running.server as unknown as {
+        getConnections?: (callback: (error: Error | null, count: number) => void) => void;
+      };
+      const deadline = Date.now() + 15_000;
+      for (;;) {
+        const connections = await new Promise<number>((resolve) => {
+          if (typeof connectionCounter.getConnections !== "function") {
+            resolve(1);
+            return;
+          }
+          connectionCounter.getConnections((_error, count) => resolve(count ?? 0));
+        });
+        if (connections > 0) {
+          break;
+        }
+        if (Date.now() > deadline) {
+          throw harnessError("The request never reached the server.");
+        }
+        await new Promise((resolve) => setTimeout(resolve, 10));
+      }
+
+      // The graceful path: nothing is stalled, so shutdown completes without reaching the drain
+      // deadline and without force-closing anything.
+      await expect(withDeadline(running.shutdown(), 30_000)).resolves.toBeUndefined();
+
+      // The already-entered request completed normally — full response, clean transport close.
+      await withDeadline(responseClosed, 10_000);
+      const raw = Buffer.concat(responseBytes).toString("utf8");
+      expect(raw).toContain("HTTP/1.1 200");
+      expect(raw).toContain('{"kind":"anonymous"}');
+    } finally {
+      socket.destroy();
+      await running.shutdown().catch(() => undefined);
+      await dropDatabase(database.name);
+    }
+  });
+
   it("force-closes a stalled connection at the drain deadline and still reaches pool closure", async () => {
     const database = await createDisposableDatabase("drain_forced");
     await applyCoreMigrations(database.url);

apps/api/tests/lifecycle.test.ts

@@ -130,6 +130,9 @@ interface WorkingFixtureModuleOptions {
   readonly registerThrows?: boolean;
   readonly startThrows?: boolean;
   readonly stopThrows?: boolean;
+  /** Test-only: the stop hook records its marker and then never settles, so an API-level deadline
+   * can be exercised deterministically without any sleep. */
+  readonly stopNeverSettles?: boolean;
   readonly dependencies?: Record<string, unknown>;
 }
 
@@ -174,6 +177,7 @@ async function createWorkingFixtureModule(
     stop() {
       ${recorder.expression}.push(${JSON.stringify(`${id}:stop`)});
       ${options.stopThrows ? 'throw new Error("stop failed");' : ""}
+      ${options.stopNeverSettles ? "return new Promise(() => {});" : ""}
     },
   };`;
   await writeFile(path.join(modulesDirectory, id, "dist/server/index.js"), serverSource);
@@ -1230,4 +1234,154 @@ describe("Ekklesia API module runtime startup and shutdown", () => {
     expect(betterAuthMock).not.toHaveBeenCalled();
     expect(serveMock).not.toHaveBeenCalled();
   });
+
+  /**
+   * Pass 4 correction: the API composition's own bounded-cleanup guarantees.
+   *
+   * Core proves an individual hook deadline in isolation; these prove that `startApi`/`shutdown`
+   * actually reach pool closure through those deadlines, which is where the platform's promise ("the
+   * pool always closes") is either kept or broken. Every deadline here is a short, test-only
+   * override; production keeps the 10-second hook and 30-second drain defaults.
+   */
+  describe("Ekklesia API bounded module cleanup", () => {
+    it("bounds a never-settling startup rollback, still rolls back earlier modules, closes the pool, and binds no listener", async () => {
+      const modulesDirectory = await createTemporaryModulesDirectory();
+      const recorder = createEventRecorder();
+      await createWorkingFixtureModule(modulesDirectory, "one", recorder);
+      await createWorkingFixtureModule(modulesDirectory, "two", recorder, {
+        dependencies: { modules: { required: { one: "^0.1.0" } } },
+        startThrows: true,
+        stopNeverSettles: true,
+      });
+      const { poolEnd } = mockPoolAndAuth();
+
+      await expect(
+        startApi(validConfig(), {
+          moduleHookTimeoutMsOverride: 25,
+          modulesDirectoryOverride: modulesDirectory,
+        }),
+      ).rejects.toMatchObject({
+        issues: [
+          expect.objectContaining({ code: "MODULE_START_FAILED", moduleId: "two" }),
+          expect.objectContaining({ code: "MODULE_ROLLBACK_TIMEOUT", moduleId: "two" }),
+        ],
+      });
+
+      // The failing module's own stop hook was attempted first and timed out; the earlier module
+      // still received its rollback afterwards.
+      expect(recorder.events).toEqual([
+        "one:register",
+        "two:register",
+        "one:start",
+        "two:start",
+        "two:stop",
+        "one:stop",
+      ]);
+      expect(poolConstructor).toHaveBeenCalledTimes(1);
+      expect(poolEnd).toHaveBeenCalledTimes(1);
+      expect(serveMock).not.toHaveBeenCalled();
+    });
+
+    it("bounds a never-settling normal stop, attempts the earlier module, and still closes the pool", async () => {
+      const modulesDirectory = await createTemporaryModulesDirectory();
+      const recorder = createEventRecorder();
+      await createWorkingFixtureModule(modulesDirectory, "one", recorder);
+      await createWorkingFixtureModule(modulesDirectory, "two", recorder, {
+        dependencies: { modules: { required: { one: "^0.1.0" } } },
+        stopNeverSettles: true,
+      });
+      const { poolEnd } = mockPoolAndAuth();
+      mockSuccessfulServe();
+
+      const running = await startApi(validConfig(), {
+        moduleHookTimeoutMsOverride: 25,
+        modulesDirectoryOverride: modulesDirectory,
+      });
+
+      await expect(running.shutdown()).rejects.toMatchObject({
+        drainTimedOut: false,
+        moduleStopError: {
+          issues: [expect.objectContaining({ code: "MODULE_STOP_TIMEOUT", moduleId: "two" })],
+        },
+        moduleStopFailed: true,
+        name: "ApiShutdownError",
+      });
+      expect(recorder.events.slice(-2)).toEqual(["two:stop", "one:stop"]);
+      expect(poolEnd).toHaveBeenCalledTimes(1);
+
+      // Repeated and concurrent callers share the one memoized result and restart nothing.
+      const [first, second] = await Promise.allSettled([running.shutdown(), running.shutdown()]);
+      expect(first.status).toBe("rejected");
+      expect(second.status).toBe("rejected");
+      expect(recorder.events.filter((event) => event === "two:stop")).toHaveLength(1);
+      expect(recorder.events.filter((event) => event === "one:stop")).toHaveLength(1);
+      expect(poolEnd).toHaveBeenCalledTimes(1);
+    });
+
+    it("reports a drain timeout and a module-stop timeout together, after the pool has closed", async () => {
+      const modulesDirectory = await createTemporaryModulesDirectory();
+      const recorder = createEventRecorder();
+      await createWorkingFixtureModule(modulesDirectory, "stalls", recorder, {
+        stopNeverSettles: true,
+      });
+      const { poolEnd } = mockPoolAndAuth();
+      // A server whose `close` callback never fires stands in for a connection that never drains,
+      // deterministically and without a real socket. `closeIdleConnections`/`closeAllConnections`
+      // are the supported public APIs the platform calls at the deadline.
+      const closeAllConnections = vi.fn();
+      const closeIdleConnections = vi.fn();
+      const { server } = fakeServer();
+      // `close` deliberately never invokes its callback: nothing drains, which is exactly what the
+      // drain deadline exists for — deterministic, with no real socket and no sleep.
+      const stalled = Object.assign(server, {
+        close: vi.fn(() => server),
+        closeAllConnections,
+        closeIdleConnections,
+      });
+      serveMock.mockImplementation((_options: unknown, listeningListener?: () => void) => {
+        // Deferred exactly like `mockSuccessfulServe`: `bindServer` resolves with `server`, which
+        // is still in its temporal dead zone during the synchronous `serve()` call.
+        queueMicrotask(() => listeningListener?.());
+        return stalled;
+      });
+
+      const running = await startApi(validConfig(), {
+        drainTimeoutMsOverride: 25,
+        moduleHookTimeoutMsOverride: 25,
+        modulesDirectoryOverride: modulesDirectory,
+      });
+
+      let captured: unknown;
+      try {
+        await running.shutdown();
+      } catch (error) {
+        captured = error;
+      }
+
+      expect(captured).toMatchObject({
+        drainTimedOut: true,
+        moduleStopFailed: true,
+        name: "ApiShutdownError",
+      });
+      const issues = (captured as { moduleStopError: { issues: Array<{ code: string }> } })
+        .moduleStopError.issues;
+      expect(issues).toEqual([
+        expect.objectContaining({ code: "MODULE_STOP_TIMEOUT", moduleId: "stalls" }),
+      ]);
+      // The platform force-closed through Node's public APIs, then still stopped modules and closed
+      // the pool — neither deadline skipped a later stage.
+      expect(closeIdleConnections).toHaveBeenCalledTimes(1);
+      expect(closeAllConnections).toHaveBeenCalledTimes(1);
+      expect(recorder.events).toContain("stalls:stop");
+      expect(poolEnd).toHaveBeenCalledTimes(1);
+
+      // Nothing about the request, socket, connection, or module exception leaks.
+      const publicText = `${String(captured)}\n${(captured as Error).stack ?? ""}\n${JSON.stringify(
+        captured,
+      )}`;
+      for (const secret of ["127.0.0.1", "postgres", "password", "select ", "Error: stop"]) {
+        expect(publicText.toLowerCase()).not.toContain(secret.toLowerCase());
+      }
+    });
+  });
 });