From f2fc069e66ee62366ba4ba822bbbf2e6be4a4bb6 Mon Sep 17 00:00:00 2001 From: ekklesia Date: Tue, 13 Oct 2026 23:13:21 -0500 Subject: [PATCH] test(api): let a shutdown's closed connections leave pg_stat_activity Four integration tests shut the API down and immediately count the database's rows in pg_stat_activity, expecting zero. PostgreSQL ends a backend asynchronously: pool.end() resolves once the client side has closed, and the server can still list that backend for a moment. On a loaded runner an immediate count saw one, failing migration-smoke on a docs-only PR (Ekklesia CI run #238, module-database.integration.ts:764, "expected 1 to be +0"). Reproduced against postgres:18.4-alpine3.24 limited to 0.05 CPU: after pool.end() resolved, the backend was still listed in 7 of 300 trials, and gone about a millisecond later. The checks now wait up to 3 s for the count to reach zero. A connection that is genuinely left open is still caught: an idle pooled client stays listed for pg-pool's 10 s idle timeout, far past the deadline. Co-Authored-By: Claude Opus 5.5 --- apps/api/tests/api-http.integration.ts | 29 +++++++++++++++++-- apps/api/tests/module-database.integration.ts | 25 +++++++++++++++- 2 files changed, 50 insertions(+), 4 deletions(-) diff --git a/apps/api/tests/api-http.integration.ts b/apps/api/tests/api-http.integration.ts index c673f1f..8b86cc7 100644 --- a/apps/api/tests/api-http.integration.ts +++ b/apps/api/tests/api-http.integration.ts @@ -208,6 +208,29 @@ async function activeConnectionCount(databaseName: string): Promise { } } +/** How long a shutdown's closed connections may take to leave `pg_stat_activity`. Far below pg's + * 10 s idle timeout, so a connection that is genuinely still open is still caught. */ +const SHUTDOWN_SETTLE_MS = 3_000; + +/** + * The connections left on `databaseName` once a shutdown has had a moment to land. + * + * PostgreSQL ends a backend asynchronously: `pool.end()` resolves once the client side has closed, + * and the server can still list that backend for a moment afterwards. On a loaded CI runner that + * moment was long enough for an immediate count to see one. So this waits, briefly, for the count to + * reach zero; a leaked pooled client would stay idle and listed far past the deadline. + */ +async function connectionsLeftAfterShutdown(databaseName: string): Promise { + const deadline = Date.now() + SHUTDOWN_SETTLE_MS; + for (;;) { + const count = await activeConnectionCount(databaseName); + if (count === 0 || Date.now() >= deadline) { + return count; + } + await new Promise((resolve) => setTimeout(resolve, 50)); + } +} + async function createDisposableDatabase(label: string): Promise<{ name: string; url: string }> { const name = makeDatabaseName(label); await createDatabase(name); @@ -1349,7 +1372,7 @@ describe.sequential("live API — signup, sign-in, principal, sign-out", () => { await running.shutdown(); await expect(fetch(origin, { signal: AbortSignal.timeout(1_000) })).rejects.toBeTruthy(); - const remainingConnections = await activeConnectionCount(database.name); + const remainingConnections = await connectionsLeftAfterShutdown(database.name); expect(remainingConnections).toBe(0); await dropDatabase(database.name); @@ -2230,7 +2253,7 @@ describe.sequential("live API — platform owner and delegated grant management" it("shuts down cleanly, closing the one shared pool", async () => { await running.shutdown(); - const remainingConnections = await activeConnectionCount(database.name); + const remainingConnections = await connectionsLeftAfterShutdown(database.name); expect(remainingConnections).toBe(0); await dropDatabase(database.name); }); @@ -2837,7 +2860,7 @@ describe.sequential("live API — installed module permission end-to-end", () => it("shuts down cleanly, then restarting with the module absent rejects new grants for its former permission, returns NOT_FOUND for its former route, while the historical grant remains listable", async () => { await running.shutdown(); - const remainingConnections = await activeConnectionCount(database.name); + const remainingConnections = await connectionsLeftAfterShutdown(database.name); expect(remainingConnections).toBe(0); // Exactly one stop, matching the exactly-one register/start already asserted above. expect(recorder.events.filter((event) => event === "calendar:stop")).toHaveLength(1); diff --git a/apps/api/tests/module-database.integration.ts b/apps/api/tests/module-database.integration.ts index bbc6a93..fcde53f 100644 --- a/apps/api/tests/module-database.integration.ts +++ b/apps/api/tests/module-database.integration.ts @@ -195,6 +195,29 @@ async function activeConnectionCount(databaseName: string): Promise { } } +/** How long a shutdown's closed connections may take to leave `pg_stat_activity`. Far below pg's + * 10 s idle timeout, so a connection that is genuinely still open is still caught. */ +const SHUTDOWN_SETTLE_MS = 3_000; + +/** + * The connections left on `databaseName` once a shutdown has had a moment to land. + * + * PostgreSQL ends a backend asynchronously: `pool.end()` resolves once the client side has closed, + * and the server can still list that backend for a moment afterwards. On a loaded CI runner that + * moment was long enough for an immediate count to see one. So this waits, briefly, for the count to + * reach zero; a leaked pooled client would stay idle and listed far past the deadline. + */ +async function connectionsLeftAfterShutdown(databaseName: string): Promise { + const deadline = Date.now() + SHUTDOWN_SETTLE_MS; + for (;;) { + const count = await activeConnectionCount(databaseName); + if (count === 0 || Date.now() >= deadline) { + return count; + } + await new Promise((resolve) => setTimeout(resolve, 50)); + } +} + async function createDisposableDatabase(label: string): Promise<{ name: string; url: string }> { const name = makeDatabaseName(label); await createDatabase(name); @@ -760,7 +783,7 @@ describe.sequential("live API — module database access through ModuleRuntimeSe await running.shutdown(); - const remainingConnections = await activeConnectionCount(database.name); + const remainingConnections = await connectionsLeftAfterShutdown(database.name); expect(remainingConnections).toBe(0); const eventsAfterShutdown = readRecorder();