diff --git a/.env.example b/.env.example index d86af41..a0d4639 100644 --- a/.env.example +++ b/.env.example @@ -12,10 +12,22 @@ POSTGRES_PASSWORD=ekklesia-development-only POSTGRES_PORT=5432 DATABASE_URL=postgresql://ekklesia:ekklesia-development-only@127.0.0.1:5432/ekklesia +# Ekklesia's object-storage provider (ADR 0030). Absent S3_ENDPOINT means this installation has no +# object storage, which is a complete deployment: only a module that declares a logical bucket in its +# manifest is affected, and it is reported blocked rather than breaking anything. Production requires +# https, exactly as the database transport requires verified TLS. +# +# These are host infrastructure. No module ever sees them: a module names a logical bucket and the +# host resolves the binding. S3_ACCESS_KEY=ekklesia-development S3_SECRET_KEY=ekklesia-development-secret -S3_BUCKET=ekklesia-development S3_ENDPOINT=http://127.0.0.1:8333 S3_PORT=8333 S3_REGION=us-east-1 S3_FORCE_PATH_STYLE=true +# Namespaces every physical bucket this installation owns, so several Ekklesia-compatible deployments +# can share one S3-compatible service. Optional; defaults to "ekklesia". +S3_BUCKET_PREFIX=ekklesia +# Consumed only by the development Compose service's own bootstrap, never by Ekklesia's provider — +# physical buckets are derived from a module's manifest declaration, not configured here. +S3_BUCKET=ekklesia-development diff --git a/README.md b/README.md index dd115fb..c1bbc3f 100644 --- a/README.md +++ b/README.md @@ -12,8 +12,8 @@ transport), and Phase 5C (module database access) have landed. **Phase 5D (secur against an independent security audit) is complete: all five passes have landed and the final independent whole-repository re-audit has closed, under the current trusted-contributor, not-yet-deployed threat model — and Phase 6 is unblocked and underway.** Phase 6 has since -completed through 6D, and **Phase 7A pass 1 (the minimal operational module lifecycle) and -pass 2 (the module configuration and secrets contract) have landed.** The operational +completed through 6D, and **Phase 7A passes 1 (the minimal operational module lifecycle), 2 (the module configuration and +secrets contract), and 3 (host-provided module object storage) have landed.** The operational requirements that remain are prerequisites for two specific *future* activities — executing untrusted CI contributions, and a public production launch — and neither gates Phase 6; see the pass 5 closure below and @@ -506,10 +506,71 @@ and credentials for shared infrastructure stay platform concerns: a module decla needs, not an S3 endpoint, leaving room for the object storage service Core still lacks. See [ADR 0029](docs/architecture/decisions/0029-module-configuration-and-secrets.md). +Phase 7A pass 3 turns `manifest.storage.buckets` — declarable since manifest version 1 and inert ever +since — into a real **host-provided object-storage capability**. This is what stands between +Announcements and the module system: it currently fetches `http://10.40.0.2:9000/announcements`, parses +S3 XML by hand, and hands the browser direct S3 URLs. + +A module declares a **logical** bucket and gets a scoped facade: + +```json +"storage": { "buckets": [{ "id": "announcement-assets" }] } +``` + +```ts +const assets = context.storage.bucket("announcement-assets"); +const object = await assets.get("slides/welcome.png"); +return new Response(object.body, { headers: { "content-type": object.contentType } }); +``` + +**The contract names no provider.** No endpoint, region, credential, or physical bucket is reachable +from it — not as a field, not as a type, not as a string. That is what lets the same module run under +Ekklesia and under a future standalone host backed by its own SeaweedFS, a shared one, MinIO, or AWS +S3. A module that configures its own S3 endpoint is a module welded to one deployment, which is the +mistake this pass exists to prevent. + +Core's first provider speaks the S3-compatible protocol through the AWS SDK. SeaweedFS is the intended +target ([ADR 0004](docs/architecture/decisions/0004-replace-unmaintained-minio.md)) and already runs in +the development Compose stack — but it is a deployment choice, never an API name. Probing it directly +found a real divergence worth having caught: `CreateBucket` on an existing bucket reports +`BucketAlreadyExists`, **not** AWS's `BucketAlreadyOwnedByYou`, so treating only the AWS spelling as +success would have failed on every second boot. A live test against the real server pins it. + +**Logical → physical mapping is deterministic and collision-safe.** +`ekklesia-announcements--announcement-assets-3f0c1d9a6b21`: an installation prefix (so several +deployments can share one storage service), the module and bucket ids joined by `--` — which cannot +occur inside either identifier, so `a-b`+`c` and `a`+`b-c` can never collide — and a digest over the +full untruncated identity. Two modules both declaring `assets` always get separate namespaces, and a +module can never see or choose the physical name. + +**Declaring a bucket is now a real requirement.** With no provider configured, such a module is +reported `blocked · storage_unavailable` naming the logical bucket, and its required dependents block +behind it through the existing resolver — no second eligibility system. An installation with *no* +object storage remains a complete deployment; modules declaring no bucket are unaffected, which is why +the template now declares none. + +**A storage outage is not a lifecycle event.** Eligibility asks whether storage is *configured*, never +whether it answered. Bucket creation is lazy on first use rather than at startup, so a storage service +being down can never stop the platform from booting — and a running module whose object store blinks +stays `running`, catches a structured error, and retries, exactly as Announcements does today. Nothing +ever deletes a bucket, on disable or otherwise. + +Credentials are deployment-supplied (`S3_ENDPOINT`, `S3_ACCESS_KEY`, `S3_SECRET_KEY`, +`S3_BUCKET_PREFIX`), never persisted, never routed through module configuration, and never visible to +modules, module UI, diagnostics, or any API response — errors are translated to a closed code set that +carries no endpoint, bucket, request id, or XML. This also retires the transitional `bucket` +*configuration* key ADR 0029 flagged: the manifest declaration is the requirement. + +Browsers reach objects through a module's own routes (`browser → module API → scoped storage`), so no +S3 knowledge or credential goes near the browser. **Signed URLs, public object delivery, range +requests, and multipart upload are deliberately deferred**, and the provider is not yet +"production-ready" by ADR 0004's own bar. See +[ADR 0030](docs/architecture/decisions/0030-host-provided-module-object-storage.md). + **Still not the finished Dashboard.** There is no navigation editor or drag and drop, no nested folders, no icons or colors, no Dashboard Home or widgets, no design system, no client-side routing, -and no module-management or settings UI — module enablement and configuration are an API and a CLI, -not a screen. Sign-in UI, a production CSP and deployment for the built shell, asset publication beyond +and no module-management, settings, or storage UI — module enablement and configuration are an API and +a CLI, not a screen, and object storage is deployment configuration. Sign-in UI, a production CSP and deployment for the built shell, asset publication beyond the UI entrypoint, React runtime sharing, the standalone host, and every real feature module also do not exist. @@ -601,8 +662,10 @@ Identity Trusts only the actual immediate socket peer for rate limiting; no forwarded header is trusted. Object storage - ○ Check unavailable - Core storage integration has not been implemented yet. + ○ Object storage not configured + No S3_ENDPOINT is set. This is a complete deployment unless a module requires storage. + ✓ No module requires object storage + 0 modules declare a logical storage bucket. Result: READY WITH WARNINGS ``` diff --git a/apps/api/src/app.ts b/apps/api/src/app.ts index f37d504..0d66c96 100644 --- a/apps/api/src/app.ts +++ b/apps/api/src/app.ts @@ -18,6 +18,7 @@ import type { ModuleLifecycleSnapshot, ModuleConfigurationLookup, ModuleLifecycleStore, + ModuleStorageLookup, ModuleUiPublicationRegistry, } from "@ekklesia/core/modules"; import { Hono } from "hono"; @@ -93,6 +94,19 @@ export interface CreateApiAppOptions { * read the same object rather than two independently resolved values. */ readonly snapshots: ModuleConfigurationLookup; }; + /** + * The platform's module object storage (ADR 0030): the per-module scoped facades the runtime + * started with, and whether a host provider is configured at all. + * + * Required rather than defaulted, like every other module option here — a caller exercising no + * storage says so with an empty map and `providerConfigured: false`, never falls through to an + * implicit one. Nothing here carries an endpoint, a region, or a credential: those never leave + * Core's provider. + */ + readonly moduleStorage: { + readonly snapshots: ModuleStorageLookup; + readonly providerConfigured: boolean; + }; /** Core's persistence for how this Dashboard presents installed modules (ADR 0027). Required * rather than defaulted, like every other module option here: a caller that exercises no * Dashboard says so with an explicit store, never falls through to an implicit one. Grouping is @@ -273,6 +287,7 @@ export function createApiApp(options: CreateApiAppOptions): Hono { secrets: options.moduleConfiguration.secrets, store: options.moduleConfiguration.store, }, + storage: { providerConfigured: options.moduleStorage.providerConfigured }, snapshot: options.moduleLifecycle.snapshot, store: options.moduleLifecycle.store, trustedOrigins: allowedOrigins, @@ -297,6 +312,7 @@ export function createApiApp(options: CreateApiAppOptions): Hono { mountModuleHttpRoutes(app, options.moduleRoutes, { configurations: options.moduleConfiguration.snapshots, + storages: options.moduleStorage.snapshots, evaluator: options.authorization.evaluator, registry: options.authorization.registry, runtime, diff --git a/apps/api/src/config.ts b/apps/api/src/config.ts index 48cca77..49a99e2 100644 --- a/apps/api/src/config.ts +++ b/apps/api/src/config.ts @@ -10,6 +10,11 @@ import { type IdentityConfig, type IdentityConfigInput, } from "@ekklesia/core/identity"; +import { + createStorageProviderConfig, + type StorageProviderConfig, + type StorageProviderConfigInput, +} from "@ekklesia/core/storage"; import { type ClientAddressConfig, type ClientAddressConfigInput, @@ -55,6 +60,16 @@ export interface ApiRuntimeConfig { readonly host: string; readonly identity: IdentityConfig; readonly port: number; + /** + * The validated host object-storage provider, or `undefined` when this installation has none + * (ADR 0030). + * + * `undefined` is a complete, valid deployment: modules that declare no logical buckets are + * unaffected, and a module that does declare one is reported blocked rather than crashing + * anything. These values reach Core's provider and nowhere else — no module, module UI, browser + * bundle, diagnostic, or API response ever sees the endpoint or the credentials. + */ + readonly storage: StorageProviderConfig | undefined; } export interface ApiRuntimeConfigInput { @@ -68,6 +83,8 @@ export interface ApiRuntimeConfigInput { readonly host?: unknown; readonly identity: IdentityConfigInput; readonly port?: unknown; + /** Optional. Omitted, or with no endpoint, means this installation has no object storage. */ + readonly storage?: StorageProviderConfigInput; } export interface CreateApiRuntimeConfigOptions { @@ -142,6 +159,11 @@ export function createApiRuntimeConfig( // the URL carries TLS parameters that could override the policy. assertProductionDatabaseTransport(databaseUrl, databaseTransport, environment); + // Validated in the same pass as every other runtime value, before `startApi` constructs a pool, + // a provider client, or a listener — so a malformed endpoint or a half-supplied credential pair + // fails startup with a sanitized code rather than surfacing later as a confusing storage error. + const storage = createStorageProviderConfig(input.storage, { environment }); + return Object.freeze({ clientAddress, databaseTransport, @@ -149,5 +171,6 @@ export function createApiRuntimeConfig( host, identity, port, + storage, }); } diff --git a/apps/api/src/module-http-routes.ts b/apps/api/src/module-http-routes.ts index 6d0ccfc..060a23a 100644 --- a/apps/api/src/module-http-routes.ts +++ b/apps/api/src/module-http-routes.ts @@ -1,11 +1,12 @@ import { EMPTY_MODULE_CONFIGURATION, + EMPTY_MODULE_STORAGE, type ModuleId, type ModuleRouteHandlerContext, type ModuleRuntimeServices, type RegisteredModuleHttpRoute, } from "@ekklesia/contracts"; -import type { ModuleConfigurationLookup } from "@ekklesia/core/modules"; +import type { ModuleConfigurationLookup, ModuleStorageLookup } from "@ekklesia/core/modules"; import type { Context, Hono } from "hono"; import { type AuthorizationMiddlewareOptions, @@ -43,6 +44,15 @@ export interface ModuleHttpRoutesOptions extends AuthorizationMiddlewareOptions * ever reach its own module's configuration, because that is the only entry looked up for it. */ readonly configurations: ModuleConfigurationLookup; + /** + * Per-module scoped object storage — the exact map the module runtime started from (ADR 0030), so + * a handler and that module's lifecycle hooks share one facade rather than two independently + * resolved ones. + * + * Keyed by module id, like `configurations` and unlike `services`: a handler can only ever reach + * its own module's storage, because that is the only entry looked up for it. + */ + readonly storages: ModuleStorageLookup; } /** Never carries the offending permission id, route path, module id, or any other configuration @@ -226,6 +236,7 @@ export function mountModuleHttpRoutes( const context: ModuleRouteHandlerContext = Object.freeze({ config: options.configurations.get(route.moduleId) ?? EMPTY_MODULE_CONFIGURATION, moduleId: route.moduleId, + storage: options.storages.get(route.moduleId) ?? EMPTY_MODULE_STORAGE, params: Object.freeze({ ...c.req.param() }), principal: getAuthorizedPrincipal(c), services: options.services, diff --git a/apps/api/src/module-lifecycle-routes.ts b/apps/api/src/module-lifecycle-routes.ts index d363f81..2fed086 100644 --- a/apps/api/src/module-lifecycle-routes.ts +++ b/apps/api/src/module-lifecycle-routes.ts @@ -8,6 +8,7 @@ import { CORE_MODULES_LIFECYCLE_MANAGE } from "@ekklesia/core/authorization"; import { type ModuleConfigurationStore, type ModuleLifecycleSnapshot, + type ModuleStorageAvailability, type ModuleLifecycleStore, ModuleLifecycleStoreError, type ModuleLifecycleView, @@ -65,6 +66,9 @@ export interface ModuleLifecycleModuleView { /** Present for a configuration reason (ADR 0029): the offending key, never its value. A key is * declared in a public manifest; a value may be a credential or arbitrary operator text. */ readonly configurationKey?: string; + /** Present for a storage reason (ADR 0030): the logical bucket the module declared, never an + * endpoint, a credential, or the physical bucket the host maps it to. */ + readonly storageBucketId?: string; }; readonly restartRequired: boolean; } @@ -90,6 +94,9 @@ export interface CreateModuleLifecycleRoutesOptions extends AuthorizationMiddlew readonly store: ModuleConfigurationStore; readonly secrets: ModuleSecretSource; }; + /** The other axis that decides eligibility (ADR 0030). A boolean, never a probe: whether storage + * is *configured* decides eligibility, while whether it is *reachable right now* never does. */ + readonly storage: ModuleStorageAvailability; readonly trustedOrigins: ReadonlySet; } @@ -156,6 +163,9 @@ function present(view: ModuleLifecycleView): ModuleLifecycleModuleView { ...(view.runtime.configurationKey === undefined ? {} : { configurationKey: view.runtime.configurationKey }), + ...(view.runtime.storageBucketId === undefined + ? {} + : { storageBucketId: view.runtime.storageBucketId }), }, version: view.version, }; @@ -220,10 +230,12 @@ export function createModuleLifecycleRoutes(options: CreateModuleLifecycleRoutes options.store.readConfiguration(), options.configuration.store.readSettings(), ]); - const modules = resolveModuleLifecycleView(options.snapshot, configuration, { - secrets: options.configuration.secrets, - settings, - }).map(present); + const modules = resolveModuleLifecycleView( + options.snapshot, + configuration, + { secrets: options.configuration.secrets, settings }, + options.storage, + ).map(present); return c.json({ modules } satisfies ModuleLifecycleResponse, 200); } catch (error) { return storeFailure(c, error, "read"); @@ -254,10 +266,12 @@ export function createModuleLifecycleRoutes(options: CreateModuleLifecycleRoutes options.store.readConfiguration(), options.configuration.store.readSettings(), ]); - const view = resolveModuleLifecycleView(options.snapshot, configuration, { - secrets: options.configuration.secrets, - settings, - }).find((entry) => entry.moduleId === moduleId); + const view = resolveModuleLifecycleView( + options.snapshot, + configuration, + { secrets: options.configuration.secrets, settings }, + options.storage, + ).find((entry) => entry.moduleId === moduleId); if (!view) { return sanitizedFailure(c, 503, "MODULE_LIFECYCLE_READ_FAILED"); } diff --git a/apps/api/src/server.ts b/apps/api/src/server.ts index 4788036..5229e9c 100644 --- a/apps/api/src/server.ts +++ b/apps/api/src/server.ts @@ -12,6 +12,11 @@ import { } from "@ekklesia/core/authorization"; import { createDashboardStore } from "@ekklesia/core/dashboard"; import { createIdentityRuntime, type IdentityRateLimitOverride } from "@ekklesia/core/identity"; +import { + createModuleStorageLookup, + createS3StorageProvider, + type StorageProvider, +} from "@ekklesia/core/storage"; import { createEnvironmentModuleSecretSource, createModuleConfigurationStore, @@ -171,6 +176,19 @@ async function safeCloseDatabase(handle: ApiDatabaseHandle): Promise { } } +/** Same swallow-on-cleanup-path rule as `safeCloseDatabase`, for the storage provider's own + * connection pool. */ +async function safeCloseStorage(provider: StorageProvider | undefined): Promise { + if (!provider) { + return; + } + try { + await provider.close(); + } catch { + // See safeCloseDatabase: never mask a failure that is already propagating. + } +} + /** Same swallow-on-cleanup-path rule as `safeCloseDatabase`, used only while unwinding a startup * failure that already occurred after the module runtime started — never during normal shutdown, * where a module stop failure remains an observable result instead (see `startApi`'s `shutdown`). */ @@ -422,6 +440,13 @@ export async function startApi( // start, before a single module entrypoint is imported. A failure here fails startup rather than // falling back to "start everything": the platform cannot know what it was told, and starting a // module an administrator disabled is worse than not starting at all. + // The host's object-storage provider, constructed from validated configuration before any module + // runs — and **without touching the network**. Building the client opens no connection, so a + // storage outage can never stop the platform from booting; a bucket is ensured lazily on first + // use instead (ADR 0030). An installation with no storage configured gets `undefined` here, which + // is a complete, valid deployment. + const storageProvider = config.storage ? createS3StorageProvider(config.storage) : undefined; + const moduleLifecycleStore = createModuleLifecycleStore(databaseHandle.moduleLifecycle); const moduleConfigurationStore = createModuleConfigurationStore( databaseHandle.moduleConfiguration, @@ -441,15 +466,29 @@ export async function startApi( const lifecycleConfiguration = await moduleLifecycleStore.readConfiguration(); const settings = await moduleConfigurationStore.readSettings(); moduleConfigurationSettings = settings; - eligibility = resolveModuleEligibility(inventory.modules, lifecycleConfiguration, { - secrets: moduleSecrets, - settings, - }); + eligibility = resolveModuleEligibility( + inventory.modules, + lifecycleConfiguration, + { secrets: moduleSecrets, settings }, + { providerConfigured: storageProvider !== undefined }, + ); } catch (error) { + await safeCloseStorage(storageProvider); await safeCloseDatabase(databaseHandle); throw error; } + // One scoped facade per eligible module, over the exact buckets that module's own manifest + // declares. Built from the eligible set so a blocked module never receives storage at all. + const moduleStorages = createModuleStorageLookup( + storageProvider, + config.storage?.bucketPrefix ?? "", + eligibility.eligibleModules.map((module) => ({ + logicalBucketIds: module.manifest.storage.buckets.map((bucket) => bucket.id), + moduleId: module.manifest.id, + })), + ); + // Module load/register/start failures use Core's own existing rollback (already-started // modules are stopped, in reverse order, before this rejects) — there is no separate runtime // handle to stop here, only the pool this failure leaves open. @@ -462,11 +501,13 @@ export async function startApi( try { moduleRuntime = await startDiscoveredModuleRuntime(eligibility.eligibleModules, services, { configurations: eligibility.configurations, + storages: moduleStorages, ...(options.moduleHookTimeoutMsOverride === undefined ? {} : { hookTimeoutMs: options.moduleHookTimeoutMsOverride }), }); } catch (error) { + await safeCloseStorage(storageProvider); await safeCloseDatabase(databaseHandle); throw error; } @@ -507,6 +548,10 @@ export async function startApi( snapshots: eligibility.configurations, store: moduleConfigurationStore, }, + moduleStorage: { + providerConfigured: storageProvider !== undefined, + snapshots: moduleStorages, + }, // A typed view over the same single pool as identity, authorization, and the module facade. dashboard: createDashboardStore(databaseHandle.dashboard), runtime, @@ -514,6 +559,7 @@ export async function startApi( }); } catch (error) { await safeStopModuleRuntime(moduleRuntime); + await safeCloseStorage(storageProvider); await safeCloseDatabase(databaseHandle); throw error; } @@ -529,6 +575,7 @@ export async function startApi( server = await bindServer(app, config.host, config.port, clientAddressResolver, timeouts); } catch (error) { await safeStopModuleRuntime(moduleRuntime); + await safeCloseStorage(storageProvider); await safeCloseDatabase(databaseHandle); throw error; } @@ -558,6 +605,9 @@ export async function startApi( } catch (error) { moduleStopError = error; } + // After the module runtime has stopped, so a `stop` hook may still use its storage + // facade, and before the pool closes, matching the existing ordering discipline exactly. + await safeCloseStorage(storageProvider); await safeCloseDatabase(databaseHandle); // Every stage above ran regardless of what the earlier ones did, so a drain timeout or a // module-stop failure can never leave the pool open. Both failures are reported together, @@ -652,6 +702,17 @@ function readConfigFromEnvironment(): { trustedOrigins, }, port: process.env.EKKLESIA_API_PORT, + // The one place this process turns its environment into storage configuration. The names are + // the `S3_*` ones ADR 0004 already established for this repository's Compose stack, and Core's + // validator never reads `process.env` itself. + storage: { + accessKey: process.env.S3_ACCESS_KEY, + bucketPrefix: process.env.S3_BUCKET_PREFIX, + endpoint: process.env.S3_ENDPOINT, + forcePathStyle: process.env.S3_FORCE_PATH_STYLE, + region: process.env.S3_REGION, + secretKey: process.env.S3_SECRET_KEY, + }, }, }; } diff --git a/apps/api/tests/app.test.ts b/apps/api/tests/app.test.ts index 9fe7cb7..350f913 100644 --- a/apps/api/tests/app.test.ts +++ b/apps/api/tests/app.test.ts @@ -22,6 +22,7 @@ import { createApiApp, type CreateApiAppAuthorizationOptions } from "../src/app. import { AUTH_REQUEST_MAX_BODY_BYTES } from "../src/body-limits.js"; import { emptyModuleLifecycle } from "./support/in-memory-module-lifecycle.js"; import { emptyModuleConfiguration } from "./support/in-memory-module-configuration.js"; +import { emptyModuleStorage } from "./support/in-memory-module-storage.js"; const trustedOrigin = "https://dashboard.example.test"; const untrustedOrigin = "https://evil.example.test"; @@ -92,6 +93,7 @@ describe("Ekklesia API Hono application", () => { moduleUi: emptyModuleUiPublication(), moduleLifecycle: emptyModuleLifecycle(), moduleConfiguration: emptyModuleConfiguration(), + moduleStorage: emptyModuleStorage(), dashboard: createInMemoryDashboardStore(), moduleServices: fakeModuleServices(), authorization: fakeAuthorization({}), @@ -113,6 +115,7 @@ describe("Ekklesia API Hono application", () => { moduleUi: emptyModuleUiPublication(), moduleLifecycle: emptyModuleLifecycle(), moduleConfiguration: emptyModuleConfiguration(), + moduleStorage: emptyModuleStorage(), dashboard: createInMemoryDashboardStore(), moduleServices: fakeModuleServices(), authorization: fakeAuthorization({}), @@ -135,6 +138,7 @@ describe("Ekklesia API Hono application", () => { moduleUi: emptyModuleUiPublication(), moduleLifecycle: emptyModuleLifecycle(), moduleConfiguration: emptyModuleConfiguration(), + moduleStorage: emptyModuleStorage(), dashboard: createInMemoryDashboardStore(), moduleServices: fakeModuleServices(), authorization: fakeAuthorization({}), @@ -159,6 +163,7 @@ describe("Ekklesia API Hono application", () => { moduleUi: emptyModuleUiPublication(), moduleLifecycle: emptyModuleLifecycle(), moduleConfiguration: emptyModuleConfiguration(), + moduleStorage: emptyModuleStorage(), dashboard: createInMemoryDashboardStore(), moduleServices: fakeModuleServices(), authorization: fakeAuthorization({}), @@ -185,6 +190,7 @@ describe("Ekklesia API Hono application", () => { moduleUi: emptyModuleUiPublication(), moduleLifecycle: emptyModuleLifecycle(), moduleConfiguration: emptyModuleConfiguration(), + moduleStorage: emptyModuleStorage(), dashboard: createInMemoryDashboardStore(), moduleServices: fakeModuleServices(), authorization: fakeAuthorization({}), @@ -211,6 +217,7 @@ describe("Ekklesia API Hono application", () => { moduleUi: emptyModuleUiPublication(), moduleLifecycle: emptyModuleLifecycle(), moduleConfiguration: emptyModuleConfiguration(), + moduleStorage: emptyModuleStorage(), dashboard: createInMemoryDashboardStore(), moduleServices: fakeModuleServices(), authorization: fakeAuthorization({}), @@ -234,6 +241,7 @@ describe("Ekklesia API Hono application", () => { moduleUi: emptyModuleUiPublication(), moduleLifecycle: emptyModuleLifecycle(), moduleConfiguration: emptyModuleConfiguration(), + moduleStorage: emptyModuleStorage(), dashboard: createInMemoryDashboardStore(), moduleServices: fakeModuleServices(), authorization: fakeAuthorization({}), @@ -259,6 +267,7 @@ describe("Ekklesia API Hono application", () => { moduleUi: emptyModuleUiPublication(), moduleLifecycle: emptyModuleLifecycle(), moduleConfiguration: emptyModuleConfiguration(), + moduleStorage: emptyModuleStorage(), dashboard: createInMemoryDashboardStore(), moduleServices: fakeModuleServices(), authorization: fakeAuthorization({}), @@ -281,6 +290,7 @@ describe("Ekklesia API Hono application", () => { moduleUi: emptyModuleUiPublication(), moduleLifecycle: emptyModuleLifecycle(), moduleConfiguration: emptyModuleConfiguration(), + moduleStorage: emptyModuleStorage(), dashboard: createInMemoryDashboardStore(), moduleServices: fakeModuleServices(), authorization: fakeAuthorization({}), @@ -305,6 +315,7 @@ describe("Ekklesia API Hono application", () => { moduleUi: emptyModuleUiPublication(), moduleLifecycle: emptyModuleLifecycle(), moduleConfiguration: emptyModuleConfiguration(), + moduleStorage: emptyModuleStorage(), dashboard: createInMemoryDashboardStore(), moduleServices: fakeModuleServices(), authorization: fakeAuthorization({}), @@ -336,6 +347,7 @@ describe("Ekklesia API Hono application", () => { moduleUi: emptyModuleUiPublication(), moduleLifecycle: emptyModuleLifecycle(), moduleConfiguration: emptyModuleConfiguration(), + moduleStorage: emptyModuleStorage(), dashboard: createInMemoryDashboardStore(), moduleServices: fakeModuleServices(), authorization: fakeAuthorization({}), @@ -368,6 +380,7 @@ describe("Ekklesia API Hono application", () => { moduleUi: emptyModuleUiPublication(), moduleLifecycle: emptyModuleLifecycle(), moduleConfiguration: emptyModuleConfiguration(), + moduleStorage: emptyModuleStorage(), dashboard: createInMemoryDashboardStore(), moduleServices: fakeModuleServices(), authorization: fakeAuthorization({}), @@ -405,6 +418,7 @@ describe("Ekklesia API Hono application", () => { moduleUi: emptyModuleUiPublication(), moduleLifecycle: emptyModuleLifecycle(), moduleConfiguration: emptyModuleConfiguration(), + moduleStorage: emptyModuleStorage(), dashboard: createInMemoryDashboardStore(), moduleServices: fakeModuleServices(), authorization: fakeAuthorization({}), @@ -424,6 +438,7 @@ describe("Ekklesia API Hono application", () => { moduleUi: emptyModuleUiPublication(), moduleLifecycle: emptyModuleLifecycle(), moduleConfiguration: emptyModuleConfiguration(), + moduleStorage: emptyModuleStorage(), dashboard: createInMemoryDashboardStore(), moduleServices: fakeModuleServices(), authorization: fakeAuthorization({}), @@ -445,6 +460,7 @@ describe("Ekklesia API Hono application", () => { moduleUi: emptyModuleUiPublication(), moduleLifecycle: emptyModuleLifecycle(), moduleConfiguration: emptyModuleConfiguration(), + moduleStorage: emptyModuleStorage(), dashboard: createInMemoryDashboardStore(), moduleServices: fakeModuleServices(), authorization: fakeAuthorization({}), @@ -465,6 +481,7 @@ describe("Ekklesia API Hono application", () => { moduleUi: emptyModuleUiPublication(), moduleLifecycle: emptyModuleLifecycle(), moduleConfiguration: emptyModuleConfiguration(), + moduleStorage: emptyModuleStorage(), dashboard: createInMemoryDashboardStore(), moduleServices: fakeModuleServices(), authorization: fakeAuthorization({}), @@ -483,6 +500,7 @@ describe("Ekklesia API Hono application", () => { moduleUi: emptyModuleUiPublication(), moduleLifecycle: emptyModuleLifecycle(), moduleConfiguration: emptyModuleConfiguration(), + moduleStorage: emptyModuleStorage(), dashboard: createInMemoryDashboardStore(), moduleServices: fakeModuleServices(), authorization: fakeAuthorization({}), @@ -508,6 +526,7 @@ describe("Ekklesia API Hono application", () => { moduleUi: emptyModuleUiPublication(), moduleLifecycle: emptyModuleLifecycle(), moduleConfiguration: emptyModuleConfiguration(), + moduleStorage: emptyModuleStorage(), dashboard: createInMemoryDashboardStore(), moduleServices: fakeModuleServices(), authorization: fakeAuthorization({}), @@ -536,6 +555,7 @@ describe("Ekklesia API Hono application", () => { moduleUi: emptyModuleUiPublication(), moduleLifecycle: emptyModuleLifecycle(), moduleConfiguration: emptyModuleConfiguration(), + moduleStorage: emptyModuleStorage(), dashboard: createInMemoryDashboardStore(), moduleServices: fakeModuleServices(), authorization: fakeAuthorization({ evaluator: { authorize: evaluatorAuthorize } }), @@ -567,6 +587,7 @@ describe("Ekklesia API Hono application", () => { moduleUi: emptyModuleUiPublication(), moduleLifecycle: emptyModuleLifecycle(), moduleConfiguration: emptyModuleConfiguration(), + moduleStorage: emptyModuleStorage(), dashboard: createInMemoryDashboardStore(), moduleServices: fakeModuleServices(), authorization: fakeAuthorization({ @@ -589,6 +610,7 @@ describe("Ekklesia API Hono application", () => { moduleUi: emptyModuleUiPublication(), moduleLifecycle: emptyModuleLifecycle(), moduleConfiguration: emptyModuleConfiguration(), + moduleStorage: emptyModuleStorage(), dashboard: createInMemoryDashboardStore(), moduleServices: fakeModuleServices(), authorization: fakeAuthorization({}), diff --git a/apps/api/tests/dashboard-routes.test.ts b/apps/api/tests/dashboard-routes.test.ts index 5cf2680..4dd00b1 100644 --- a/apps/api/tests/dashboard-routes.test.ts +++ b/apps/api/tests/dashboard-routes.test.ts @@ -31,6 +31,7 @@ import { moduleLifecycleSnapshotFor, } from "./support/in-memory-module-lifecycle.js"; import { moduleConfigurationOptionFor } from "./support/in-memory-module-configuration.js"; +import { moduleStorageOptionFor } from "./support/in-memory-module-storage.js"; const trustedOrigin = "https://dashboard.example.test"; const untrustedOrigin = "https://evil.example.test"; @@ -147,6 +148,7 @@ async function install(options: InstallOptions = {}): Promise<{ app: Hono }> { store: createInMemoryModuleLifecycleStore(), }, moduleConfiguration: moduleConfigurationOptionFor(modules), + moduleStorage: moduleStorageOptionFor(modules), runtime, trustedOrigins: [trustedOrigin], }); diff --git a/apps/api/tests/module-configuration-routes.test.ts b/apps/api/tests/module-configuration-routes.test.ts index 317396e..7b8226f 100644 --- a/apps/api/tests/module-configuration-routes.test.ts +++ b/apps/api/tests/module-configuration-routes.test.ts @@ -39,6 +39,7 @@ import { createInMemoryModuleConfigurationStore, } from "./support/in-memory-module-configuration.js"; import { createInMemoryModuleLifecycleStore } from "./support/in-memory-module-lifecycle.js"; +import { moduleStorageOptionFor } from "./support/in-memory-module-storage.js"; const trustedOrigin = "https://dashboard.example.test"; const untrustedOrigin = "https://evil.example.test"; @@ -194,6 +195,7 @@ async function install(options: InstallOptions = {}): Promise { snapshots: eligibility.configurations, store, }, + moduleStorage: moduleStorageOptionFor(modules), moduleLifecycle: { snapshot: { discoveredModules: modules, diff --git a/apps/api/tests/module-database.integration.ts b/apps/api/tests/module-database.integration.ts index 44b9ee7..653e3a4 100644 --- a/apps/api/tests/module-database.integration.ts +++ b/apps/api/tests/module-database.integration.ts @@ -519,12 +519,18 @@ describe.sequential("live API — module database access through ModuleRuntimeSe const readHandlerEvent = [...events] .reverse() .find((event) => event.type === "handler" && event.route === "GET /records/:id"); + // The complete handler-context surface, asserted exactly so a new field is a deliberate change + // here rather than an unnoticed one: `config` (ADR 0029) and `storage` (ADR 0030) are the two + // scoped, per-module additions; everything else a module might want — a Hono Context, the + // authorization store, raw PostgreSQL, the storage provider, an endpoint or credential — remains + // unreachable. expect(readHandlerEvent?.contextKeys).toEqual([ "config", "moduleId", "params", "principal", "services", + "storage", ]); expect(readHandlerEvent?.services).toBeDefined(); expect(readHandlerEvent?.services?.database).toBe(startEvent?.services?.database); diff --git a/apps/api/tests/module-http-routes.test.ts b/apps/api/tests/module-http-routes.test.ts index 11d4b94..8bb40c1 100644 --- a/apps/api/tests/module-http-routes.test.ts +++ b/apps/api/tests/module-http-routes.test.ts @@ -28,6 +28,7 @@ import { } from "../src/module-http-routes.js"; import { emptyModuleLifecycle } from "./support/in-memory-module-lifecycle.js"; import { emptyModuleConfiguration } from "./support/in-memory-module-configuration.js"; +import { emptyModuleStorage } from "./support/in-memory-module-storage.js"; const trustedOrigin = "https://dashboard.example.test"; const untrustedOrigin = "https://evil.example.test"; @@ -113,6 +114,7 @@ function buildModuleApp(options: { moduleUi: createModuleUiPublicationRegistry([]), moduleLifecycle: emptyModuleLifecycle(), moduleConfiguration: emptyModuleConfiguration(), + moduleStorage: emptyModuleStorage(), dashboard: createInMemoryDashboardStore(), runtime, trustedOrigins: options.trustedOrigins ?? [trustedOrigin], @@ -339,17 +341,19 @@ describe("module route handler adaptation", () => { expect(body.url).toContain("/api/v1/modules/calendar/events/evt-1/rsvp?notify=true"); expect(capturedRequest).toBeInstanceOf(Request); expect(capturedRequest?.headers.get("x-custom")).toBe("abc"); - // Exactly config, moduleId, params, principal, services — and nothing else. `config` is this - // module's own frozen configuration snapshot, added deliberately by ADR 0029; a Hono Context, - // the authorization store or evaluator, raw PostgreSQL, platform configuration, and another - // module's configuration all remain unreachable, which is what this exact-shape assertion - // exists to keep true. + // Exactly config, moduleId, params, principal, services, storage — and nothing else. `config` + // is this module's own frozen configuration snapshot (ADR 0029) and `storage` its own scoped + // object storage (ADR 0030); a Hono Context, the authorization store or evaluator, raw + // PostgreSQL, the storage provider, an S3 endpoint or credential, platform configuration, and + // another module's configuration or storage all remain unreachable, which is what this + // exact-shape assertion exists to keep true. expect(Object.keys(capturedContext ?? {}).sort()).toEqual([ "config", "moduleId", "params", "principal", "services", + "storage", ]); expect(capturedContext).toMatchObject({ moduleId: "calendar", @@ -366,6 +370,10 @@ describe("module route handler adaptation", () => { // This module declared no configuration, so its snapshot is empty rather than absent — module // code never has to branch on whether the platform supplied one. expect(capturedContext?.config.keys()).toEqual([]); + // Likewise storage: this module declared no buckets, so it gets a frozen facade that refuses + // every one rather than `undefined`. + expect(Object.isFrozen(capturedContext?.storage)).toBe(true); + expect(capturedContext?.storage.buckets()).toEqual([]); }); it("gives static and root routes a frozen, empty params object", async () => { diff --git a/apps/api/tests/module-lifecycle-routes.test.ts b/apps/api/tests/module-lifecycle-routes.test.ts index 7899e85..7ea66e1 100644 --- a/apps/api/tests/module-lifecycle-routes.test.ts +++ b/apps/api/tests/module-lifecycle-routes.test.ts @@ -34,6 +34,7 @@ import { MODULE_UI_INVENTORY_PATH } from "../src/module-ui-inventory.js"; import { createInMemoryDashboardStore } from "./support/in-memory-dashboard-store.js"; import { createInMemoryModuleLifecycleStore } from "./support/in-memory-module-lifecycle.js"; import { moduleConfigurationOptionFor } from "./support/in-memory-module-configuration.js"; +import { moduleStorageOptionFor } from "./support/in-memory-module-storage.js"; const trustedOrigin = "https://dashboard.example.test"; const untrustedOrigin = "https://evil.example.test"; @@ -170,6 +171,7 @@ async function install(options: InstallOptions = {}): Promise { }, dashboard: createInMemoryDashboardStore(), moduleConfiguration: moduleConfigurationOptionFor(modules), + moduleStorage: moduleStorageOptionFor(modules), moduleLifecycle: { snapshot: { discoveredModules: modules, diff --git a/apps/api/tests/module-ui-host-e2e.test.ts b/apps/api/tests/module-ui-host-e2e.test.ts index c95668e..57eed0b 100644 --- a/apps/api/tests/module-ui-host-e2e.test.ts +++ b/apps/api/tests/module-ui-host-e2e.test.ts @@ -35,6 +35,7 @@ import { moduleLifecycleSnapshotFor, } from "./support/in-memory-module-lifecycle.js"; import { moduleConfigurationOptionFor } from "./support/in-memory-module-configuration.js"; +import { moduleStorageOptionFor } from "./support/in-memory-module-storage.js"; /** * The whole chain, with nothing between the halves faked: @@ -223,6 +224,7 @@ async function startHarness(options: HarnessOptions = {}): Promise { store: createInMemoryModuleLifecycleStore(), }, moduleConfiguration: moduleConfigurationOptionFor(modules), + moduleStorage: moduleStorageOptionFor(modules), dashboard: createInMemoryDashboardStore(), runtime, trustedOrigins: ["https://dashboard.example.test"], diff --git a/apps/api/tests/module-ui-inventory.test.ts b/apps/api/tests/module-ui-inventory.test.ts index 34660d9..e263b72 100644 --- a/apps/api/tests/module-ui-inventory.test.ts +++ b/apps/api/tests/module-ui-inventory.test.ts @@ -33,6 +33,7 @@ import { moduleLifecycleSnapshotFor, } from "./support/in-memory-module-lifecycle.js"; import { moduleConfigurationOptionFor } from "./support/in-memory-module-configuration.js"; +import { moduleStorageOptionFor } from "./support/in-memory-module-storage.js"; const trustedOrigin = "https://dashboard.example.test"; const viewerId = parseEkklesiaUserId("11111111-1111-4111-8111-111111111111"); @@ -179,6 +180,7 @@ async function install(options: InstallOptions): Promise { store: createInMemoryModuleLifecycleStore(), }, moduleConfiguration: moduleConfigurationOptionFor(modules), + moduleStorage: moduleStorageOptionFor(modules), dashboard: createInMemoryDashboardStore(), runtime, trustedOrigins: [trustedOrigin], diff --git a/apps/api/tests/module-ui-routes.test.ts b/apps/api/tests/module-ui-routes.test.ts index 242ebda..5eb4118 100644 --- a/apps/api/tests/module-ui-routes.test.ts +++ b/apps/api/tests/module-ui-routes.test.ts @@ -33,6 +33,7 @@ import { moduleLifecycleSnapshotFor, } from "./support/in-memory-module-lifecycle.js"; import { moduleConfigurationOptionFor } from "./support/in-memory-module-configuration.js"; +import { moduleStorageOptionFor } from "./support/in-memory-module-storage.js"; const trustedOrigin = "https://dashboard.example.test"; @@ -179,6 +180,7 @@ async function install(fixtures: readonly ModuleFixture[]): Promise { + return new ReadableStream({ + start(controller) { + controller.enqueue(bytes); + controller.close(); + }, + }); +} + +async function collect(stream: ReadableStream): Promise { + const reader = stream.getReader(); + const chunks: Uint8Array[] = []; + let total = 0; + for (;;) { + const { done, value } = await reader.read(); + if (done) { + break; + } + chunks.push(value); + total += value.byteLength; + } + const merged = new Uint8Array(total); + let offset = 0; + for (const chunk of chunks) { + merged.set(chunk, offset); + offset += chunk.byteLength; + } + return merged; +} + +interface StoredObject { + readonly bytes: Uint8Array; + readonly contentType?: string; + readonly lastModified: Date; +} + +/** + * An in-memory `StorageProvider` for tests that exercise **wiring, scoping, and routing** rather + * than the S3 protocol. + * + * It implements the same interface the real S3 provider does, so everything above the provider seam + * — binding resolution, the scoped facade, lifecycle eligibility, route wiring — is exercised + * unchanged. What it deliberately does *not* reproduce is the protocol itself: real SeaweedFS + * semantics (non-idempotent `CreateBucket`, `NoSuchKey` vs `NoSuchBucket`, ETag quoting, streaming + * bodies) are proven against a live server in `core/tests/module-storage.integration.ts`, so this + * double can never quietly stand in for a provider guarantee nobody checked. + * + * It keys everything by **physical** bucket, exactly as the real provider does, which is what lets a + * test assert that two modules declaring the same logical name really did land in different + * namespaces. + */ +export function createInMemoryStorageProvider(): StorageProvider & { + readonly buckets: ReadonlyMap>; + ensureBucketCalls: number; +} { + const buckets = new Map>(); + const state = { ensureBucketCalls: 0 }; + + const objectsIn = (physicalBucket: string): Map => { + const bucket = buckets.get(physicalBucket); + if (!bucket) { + throw new ModuleStorageError("STORAGE_BUCKET_UNAVAILABLE"); + } + return bucket; + }; + + const metadataFor = (key: string, stored: StoredObject): ModuleStorageObjectMetadata => + Object.freeze({ + contentLength: stored.bytes.byteLength, + key, + lastModified: stored.lastModified, + ...(stored.contentType === undefined ? {} : { contentType: stored.contentType }), + }); + + const provider: StorageProvider = { + async close(): Promise { + // Nothing to release. + }, + async delete(physicalBucket: string, key: string): Promise { + objectsIn(physicalBucket).delete(key); + }, + async ensureBucket(physicalBucket: string): Promise { + state.ensureBucketCalls += 1; + if (!buckets.has(physicalBucket)) { + buckets.set(physicalBucket, new Map()); + } + }, + async get(physicalBucket: string, key: string): Promise { + const stored = objectsIn(physicalBucket).get(key); + if (!stored) { + throw new ModuleStorageError("STORAGE_OBJECT_NOT_FOUND", key); + } + return Object.freeze({ + ...metadataFor(key, stored), + body: streamOf(stored.bytes), + }); + }, + async head(physicalBucket: string, key: string): Promise { + const stored = objectsIn(physicalBucket).get(key); + if (!stored) { + throw new ModuleStorageError("STORAGE_OBJECT_NOT_FOUND", key); + } + return metadataFor(key, stored); + }, + async list( + physicalBucket: string, + options: ModuleStorageListOptions, + ): Promise { + const entries = [...objectsIn(physicalBucket).entries()] + .filter(([key]) => options.prefix === undefined || key.startsWith(options.prefix)) + .sort(([left], [right]) => (left < right ? -1 : left > right ? 1 : 0)); + return Object.freeze({ + objects: Object.freeze(entries.map(([key, stored]) => metadataFor(key, stored))), + }); + }, + async probe(): Promise { + // Always reachable. + }, + async put( + physicalBucket: string, + key: string, + body: ModuleStorageBody, + options: ModuleStoragePutOptions, + ): Promise { + const bytes = + typeof body === "string" + ? new TextEncoder().encode(body) + : body instanceof Uint8Array + ? body + : await collect(body as ReadableStream); + const stored: StoredObject = { + bytes, + lastModified: new Date(0), + ...(options.contentType === undefined ? {} : { contentType: options.contentType }), + }; + objectsIn(physicalBucket).set(key, stored); + return metadataFor(key, stored); + }, + }; + + return Object.assign(provider, { + buckets: buckets as ReadonlyMap>, + get ensureBucketCalls(): number { + return state.ensureBucketCalls; + }, + }); +} + +export interface ModuleStorageOption { + readonly snapshots: ModuleStorageLookup; + readonly providerConfigured: boolean; +} + +/** + * Builds the `createApiApp` storage option the same way `startApi` does — resolve every module's + * declared buckets against one provider and one prefix — rather than hand-writing a map, so a test + * can never assert against a shape the real composition would not produce. + */ +export function moduleStorageOptionFor( + discoveredModules: readonly DiscoveredModule[], + provider?: StorageProvider, +): ModuleStorageOption { + return { + providerConfigured: provider !== undefined, + snapshots: createModuleStorageLookup( + provider, + "ekklesia", + discoveredModules.map((module) => ({ + logicalBucketIds: module.manifest.storage.buckets.map((bucket) => bucket.id), + moduleId: module.manifest.id, + })), + ), + }; +} + +/** The storage option for a `createApiApp` caller that exercises no storage at all — the explicit + * "this installation has no object storage" statement the required option asks for. */ +export function emptyModuleStorage(): ModuleStorageOption { + return { providerConfigured: false, snapshots: new Map() }; +} diff --git a/core/package.json b/core/package.json index 0b16a98..49e4675 100644 --- a/core/package.json +++ b/core/package.json @@ -15,6 +15,10 @@ "types": "./dist/modules/index.d.ts", "import": "./dist/modules/index.js" }, + "./storage": { + "types": "./dist/storage/index.d.ts", + "import": "./dist/storage/index.js" + }, "./dashboard": { "types": "./dist/dashboard/index.d.ts", "import": "./dist/dashboard/index.js" @@ -49,6 +53,7 @@ "typecheck": "tsc --noEmit --project tsconfig.json" }, "dependencies": { + "@aws-sdk/client-s3": "3.1116.0", "@ekklesia/contracts": "workspace:*", "better-auth": "1.6.23", "drizzle-orm": "0.45.2", diff --git a/core/src/cli/set-module-enabled.ts b/core/src/cli/set-module-enabled.ts index 6bfeeab..892db59 100644 --- a/core/src/cli/set-module-enabled.ts +++ b/core/src/cli/set-module-enabled.ts @@ -5,6 +5,8 @@ import { createModuleLifecycleDatabase, createModuleLifecycleStore, discoverModules, + isDependencyBlockedDetail, + isStorageBlockedDetail, ModuleDiscoveryError, ModuleLifecycleStoreError, resolveModuleEligibility, @@ -132,9 +134,13 @@ async function main(): Promise { if (!blocked) { continue; } - // Names the dependency or the configuration key, never a configuration value. - const cause = - "dependency" in blocked ? blocked.dependency.moduleId : blocked.configurationKey; + // Names the dependency, the configuration key, or the logical storage bucket — never a + // configuration value, an endpoint, or a credential. + const cause = isDependencyBlockedDetail(blocked) + ? blocked.dependency.moduleId + : isStorageBlockedDetail(blocked) + ? blocked.storageBucketId + : blocked.configurationKey; console.log(`- "${entry.moduleId}" will not start: ${blocked.reason} (${cause}).`); } } finally { diff --git a/core/src/diagnostics/doctor-report.ts b/core/src/diagnostics/doctor-report.ts index 199df52..1e7d3fe 100644 --- a/core/src/diagnostics/doctor-report.ts +++ b/core/src/diagnostics/doctor-report.ts @@ -21,7 +21,7 @@ const sections: readonly DoctorSection[] = Object.freeze([ section("Data definitions", ["schemas.discovery", "migrations.discovery"]), section("PostgreSQL", ["database.transport", "postgresql.connection", "postgresql.plan"]), section("Identity", ["identity.configuration", "identity.client-address"]), - section("Object storage", ["object-storage.integration"]), + section("Object storage", ["object-storage.integration", "object-storage.bindings"]), ]); const statusSymbols: Readonly> = Object.freeze({ diff --git a/core/src/diagnostics/doctor.ts b/core/src/diagnostics/doctor.ts index 47b13ac..0176279 100644 --- a/core/src/diagnostics/doctor.ts +++ b/core/src/diagnostics/doctor.ts @@ -3,6 +3,7 @@ import { capabilityIdSchema, EMPTY_MODULE_SECRET_SOURCE, isVersionCompatible, + logicalBucketIdSchema, moduleConfigurationKeySchema, moduleIdSchema, type ModuleSecretSource, @@ -42,8 +43,16 @@ import { type PostgresMigrationDatabaseOptions, planModuleMigrations, } from "../migrations/index.js"; +import { + createS3StorageProvider, + describeStorageProvider, + physicalBucketName, + type StorageProviderConfig, +} from "../storage/index.js"; import { composePlatformPermissionRegistry, + isConfigurationBlockedDetail, + isStorageBlockedDetail, createModuleConfigurationDatabase, createModuleConfigurationStore, createModuleLifecycleDatabase, @@ -75,7 +84,8 @@ export type DoctorCheckId = | "postgresql.plan" | "identity.configuration" | "identity.client-address" - | "object-storage.integration"; + | "object-storage.integration" + | "object-storage.bindings"; export type DoctorCheckStatus = "failure" | "pass" | "skipped" | "warning"; export type DoctorResult = "not-ready" | "ready" | "ready-with-warnings"; @@ -113,6 +123,9 @@ export interface DoctorOptions { * matches what `startApi` would resolve. Omitted supplies nothing — Doctor then reports a required * secret as missing, which is the honest answer for a deployment that has not set it. */ readonly moduleSecrets?: ModuleSecretSource; + /** The validated host storage provider, or `undefined` when this installation has none. Doctor + * never receives raw environment values and never prints a credential. */ + readonly storage?: StorageProviderConfig | undefined; readonly runtimeRange: string; readonly runtimeVersion: string; } @@ -150,6 +163,9 @@ export interface DoctorDependencies { readonly database: MigrationDatabase; }) => Promise; readonly runtimeSatisfies: (version: string, range: string) => boolean; + /** Probes the configured storage provider for reachability. Behind the dependency boundary like + * every other I/O Doctor performs, so the whole report stays exercisable without a network. */ + readonly probeStorage: (config: StorageProviderConfig) => Promise; } const doctorPostgresTimeoutMs = 10_000; @@ -194,6 +210,16 @@ async function readModuleConfigurationForRuntime( } } +/** Opens a provider client, probes, and always closes it — Doctor never leaves a connection behind. */ +async function probeStorageForRuntime(config: StorageProviderConfig): Promise { + const provider = createS3StorageProvider(config); + try { + await provider.probe(); + } finally { + await provider.close().catch(() => undefined); + } +} + const productionDependencies: DoctorDependencies = Object.freeze({ composePermissions: composePlatformPermissionRegistry, createMigrationDatabase: createPostgresMigrationDatabase, @@ -201,6 +227,7 @@ const productionDependencies: DoctorDependencies = Object.freeze({ discoverModules, discoverSchemas: discoverModuleSchemas, planMigrations: planModuleMigrations, + probeStorage: probeStorageForRuntime, readModuleConfiguration: readModuleConfigurationForRuntime, readModuleLifecycle: readModuleLifecycleForRuntime, runtimeSatisfies: isVersionCompatible, @@ -333,7 +360,17 @@ function safeLifecycleDetail(entry: ModuleEligibility): string { if (!entry.blocked) { return `${identity} · enabled`; } - if (!("dependency" in entry.blocked)) { + if (isStorageBlockedDetail(entry.blocked)) { + // A storage block names the **logical** bucket the module declared — never an endpoint, a + // credential, or the physical bucket the host maps it to. Re-validated against the same grammar + // the manifest accepted it under, for the same defense-in-depth reason as every other + // interpolation here. + if (!logicalBucketIdSchema.safeParse(entry.blocked.storageBucketId).success) { + throw new TypeError("Unsafe module lifecycle diagnostic value."); + } + return `${identity} · blocked · ${entry.blocked.reason} · ${entry.blocked.storageBucketId}`; + } + if (isConfigurationBlockedDetail(entry.blocked)) { // A configuration block names the offending **key** and never its value: a key is declared in a // public manifest, while a value may be a credential or arbitrary operator-entered text. The key // is re-validated against the same grammar the manifest accepted it under, for the same @@ -407,7 +444,7 @@ function safeConfigurationDetail(entry: ModuleEligibility, declaredCount: number return `${identity} · no configuration declared`; } const declared = `${declaredCount} ${plural(declaredCount, "key")} declared`; - if (entry.blocked && !("dependency" in entry.blocked)) { + if (entry.blocked && isConfigurationBlockedDetail(entry.blocked)) { if (!moduleConfigurationKeySchema.safeParse(entry.blocked.configurationKey).success) { throw new TypeError("Unsafe module configuration diagnostic value."); } @@ -458,6 +495,65 @@ function moduleConfigurationCheck( return freezeCheck("modules.configuration", "pass", "Module configuration valid", details); } +/** + * Reports which installed modules require object storage, and where the host keeps each binding. + * + * Names the module, the **logical** bucket it declared, and the physical bucket the host maps it to + * — the last of which is safe and useful here precisely because Doctor is an operator tool: someone + * debugging a permissions problem needs to know which bucket to look at. It is still never reachable + * from module-facing code (ADR 0030). No endpoint, access key, or secret key appears in any form. + */ +function storageBindingsCheck( + modules: readonly DiscoveredModule[], + entries: readonly ModuleEligibility[], + config: StorageProviderConfig | undefined, +): DoctorCheck { + const requiring = modules.filter((module) => module.manifest.storage.buckets.length > 0); + if (requiring.length === 0) { + // An installation whose modules need no object storage is not unhealthy for lacking a provider. + return freezeCheck("object-storage.bindings", "pass", "No module requires object storage", [ + "0 modules declare a logical storage bucket.", + ]); + } + + const blocked = entries.filter( + (entry) => entry.blocked !== undefined && isStorageBlockedDetail(entry.blocked), + ); + const details = requiring.flatMap((module) => { + const moduleId = module.manifest.id; + if (!moduleIdSchema.safeParse(moduleId).success) { + throw new TypeError("Unsafe module storage diagnostic value."); + } + return module.manifest.storage.buckets.map((bucket) => { + if (!logicalBucketIdSchema.safeParse(bucket.id).success) { + throw new TypeError("Unsafe module storage diagnostic value."); + } + return config === undefined + ? `${moduleId} · ${bucket.id} · unbound (no storage provider configured)` + : `${moduleId} · ${bucket.id} → ${physicalBucketName(config.bucketPrefix, moduleId, bucket.id)}`; + }); + }); + + if (config === undefined) { + return freezeCheck( + "object-storage.bindings", + "failure", + blocked.length === 1 + ? "1 module requires object storage that is not configured" + : `${blocked.length} modules require object storage that is not configured`, + [ + "Configure an S3-compatible Ekklesia storage provider (S3_ENDPOINT, S3_ACCESS_KEY, S3_SECRET_KEY).", + ...details, + ], + ); + } + return freezeCheck( + "object-storage.bindings", + "pass", + `${details.length} ${plural(details.length, "storage binding")} resolved`, + details, + ); +} function moduleDiscoveryFailure(error: unknown): DoctorCheck { const details = error instanceof Error && error.name === "ModuleDiscoveryError" @@ -788,6 +884,9 @@ export async function diagnoseEkklesia( // failed. It needs only a successful discovery and a configured database. let lifecycleCheck: DoctorCheck; let configurationCheck: DoctorCheck; + // Hoisted so the storage-bindings check below reports the same eligibility this pass resolved, + // rather than resolving a second time and risking a different answer. + let eligibilityEntries: readonly ModuleEligibility[] = []; const skippedConfiguration = (detail: string): DoctorCheck => freezeCheck("modules.configuration", "skipped", "Module configuration skipped", [detail]); if (!discoveredModules) { @@ -819,10 +918,13 @@ export async function diagnoseEkklesia( poolOptions, ); const settings = await dependencies.readModuleConfiguration(options.databaseUrl, poolOptions); - const resolution = resolveModuleEligibility(discoveredModules, configuration, { - secrets: options.moduleSecrets ?? EMPTY_MODULE_SECRET_SOURCE, - settings, - }); + const resolution = resolveModuleEligibility( + discoveredModules, + configuration, + { secrets: options.moduleSecrets ?? EMPTY_MODULE_SECRET_SOURCE, settings }, + { providerConfigured: options.storage !== undefined }, + ); + eligibilityEntries = resolution.entries; lifecycleCheck = moduleLifecycleCheck(resolution.entries); configurationCheck = moduleConfigurationCheck( resolution.entries, @@ -930,9 +1032,45 @@ export async function diagnoseEkklesia( options.databaseEnvironment, ); - const storageCheck = freezeCheck("object-storage.integration", "skipped", "Check unavailable", [ - "Core storage integration has not been implemented yet.", - ]); + // Two separate questions, deliberately. "Is a provider configured and reachable?" is + // infrastructure diagnostics; "can every module's declared bucket be bound?" is what decides + // whether modules run. Conflating them would let a momentary network blip read as a module + // problem (ADR 0030). + let storageCheck: DoctorCheck; + if (!options.storage) { + storageCheck = freezeCheck( + "object-storage.integration", + "skipped", + "Object storage not configured", + ["No S3_ENDPOINT is set. This is a complete deployment unless a module requires storage."], + ); + } else { + try { + await dependencies.probeStorage(options.storage); + storageCheck = freezeCheck("object-storage.integration", "pass", "Object storage reachable", [ + describeStorageProvider(options.storage), + ]); + } catch { + // A warning, never a failure: the provider is configured, and a module that is already + // running keeps running through a transient outage. Reporting this as `not-ready` would make + // the platform look broken because a network hiccup happened while Doctor ran. + storageCheck = freezeCheck( + "object-storage.integration", + "warning", + "Object storage unreachable", + [ + describeStorageProvider(options.storage), + "The configured storage service did not respond. Modules already running are unaffected.", + ], + ); + } + } + + const storageBindings = discoveredModules + ? storageBindingsCheck(discoveredModules, eligibilityEntries, options.storage) + : freezeCheck("object-storage.bindings", "skipped", "Storage bindings skipped", [ + "Complete module discovery is required.", + ]); return publishReport([ runtimeCheck, @@ -949,5 +1087,6 @@ export async function diagnoseEkklesia( identityCheck, clientAddressDoctorCheck, storageCheck, + storageBindings, ]); } diff --git a/core/src/modules/lifecycle.ts b/core/src/modules/lifecycle.ts index 3b532b0..fd895d7 100644 --- a/core/src/modules/lifecycle.ts +++ b/core/src/modules/lifecycle.ts @@ -62,7 +62,59 @@ export interface ModuleConfigurationBlockedDetail { readonly configurationKey: string; } -export type ModuleBlockedDetail = ModuleDependencyBlockedDetail | ModuleConfigurationBlockedDetail; +/** A block caused by unsatisfiable object-storage requirements (ADR 0030), naming the **logical** + * bucket the module declared — never an endpoint, a credential, or a physical bucket name. */ +export interface ModuleStorageBlockedDetail { + readonly reason: "storage_unavailable"; + readonly storageBucketId: string; +} + +export type ModuleBlockedDetail = + | ModuleDependencyBlockedDetail + | ModuleConfigurationBlockedDetail + | ModuleStorageBlockedDetail; + +/** Narrows a block to the dependency variant. Exported so every consumer discriminates the same + * way — a bare `"dependency" in detail` repeated at each call site is how one of them eventually + * gets it wrong as variants are added. */ +export function isDependencyBlockedDetail( + detail: ModuleBlockedDetail, +): detail is ModuleDependencyBlockedDetail { + return ( + detail.reason === "required_dependency_disabled" || + detail.reason === "required_dependency_blocked" + ); +} + +export function isStorageBlockedDetail( + detail: ModuleBlockedDetail, +): detail is ModuleStorageBlockedDetail { + return detail.reason === "storage_unavailable"; +} + +export function isConfigurationBlockedDetail( + detail: ModuleBlockedDetail, +): detail is ModuleConfigurationBlockedDetail { + return ( + detail.reason === "configuration_missing" || + detail.reason === "configuration_invalid" || + detail.reason === "secret_missing" + ); +} + +/** + * Whether this host can bind the logical object-storage buckets modules declare (ADR 0030). + * + * A boolean, deliberately, and resolved without touching the network: eligibility must stay a pure, + * synchronous function, and it must never depend on whether the storage service happened to answer a + * probe during boot. A configured provider that is temporarily unreachable is an operational + * condition a module retries through, not an administrative state change — which is the distinction + * the real Announcements product already relies on, since its slideshow survives its object store + * going away. + */ +export interface ModuleStorageAvailability { + readonly providerConfigured: boolean; +} export interface ModuleEligibilityResolution { /** @@ -108,10 +160,15 @@ function frozenEligibility(entry: MutableEligibility): ModuleEligibility { dependency: Object.freeze({ ...entry.blocked.dependency }), reason: entry.blocked.reason, }) - : Object.freeze({ - configurationKey: entry.blocked.configurationKey, - reason: entry.blocked.reason, - }), + : "storageBucketId" in entry.blocked + ? Object.freeze({ + reason: entry.blocked.reason, + storageBucketId: entry.blocked.storageBucketId, + }) + : Object.freeze({ + configurationKey: entry.blocked.configurationKey, + reason: entry.blocked.reason, + }), } : {}), }); @@ -162,6 +219,7 @@ export function resolveModuleEligibility( discoveredModules: readonly DiscoveredModule[], configuration: ModuleLifecycleConfiguration, moduleConfiguration: ModuleConfigurationInput = { settings: { records: [] } }, + storage: ModuleStorageAvailability = { providerConfigured: false }, ): ModuleEligibilityResolution { const configuredByModuleId = new Map( configuration.modules.map((record) => [record.moduleId, record.enabled] as const), @@ -180,6 +238,26 @@ export function resolveModuleEligibility( }); } + // A module that declares logical object-storage buckets needs a host provider to bind them to. The + // check is deliberately about *binding availability*, not provider health: it asks whether this + // deployment has storage configured at all, never whether the service answered just now + // (ADR 0030). Runs before the dependency pass so a storage-blocked module propagates to its + // required dependents through exactly the existing mechanism rather than a second one. + for (const discovered of discoveredModules) { + const entry = entries.get(discovered.manifest.id); + if (!entry?.eligible) { + continue; + } + const declaredBucket = discovered.manifest.storage.buckets[0]; + if (declaredBucket !== undefined && !storage.providerConfigured) { + entry.eligible = false; + entry.blocked = { + reason: "storage_unavailable", + storageBucketId: declaredBucket.id, + }; + } + } + // Validate each enabled module's own configuration before the dependency pass, so a module // blocked by its own configuration propagates to its required dependents through exactly the // existing mechanism rather than a second one. Runs only for modules that are still eligible: a @@ -306,6 +384,9 @@ export interface ModuleLifecycleView { readonly dependency?: ModuleBlockingDependency; /** Present for a configuration reason (ADR 0029) — the offending key, never its value. */ readonly configurationKey?: string; + /** Present for a storage reason (ADR 0030) — the logical bucket the module declared, never an + * endpoint, a credential, or a physical bucket name. */ + readonly storageBucketId?: string; }; /** * `true` when applying the *current* configuration would produce a different running set than @@ -347,6 +428,7 @@ export function resolveModuleLifecycleView( snapshot: ModuleLifecycleSnapshot, configuration: ModuleLifecycleConfiguration, moduleConfiguration: ModuleConfigurationInput = { settings: { records: [] } }, + storage: ModuleStorageAvailability = { providerConfigured: false }, ): readonly ModuleLifecycleView[] { const started = new Set(snapshot.startedModuleIds); const bootEligibility = new Map( @@ -361,6 +443,7 @@ export function resolveModuleLifecycleView( snapshot.discoveredModules, configuration, moduleConfiguration, + storage, ); const wouldRun = new Set(pending.eligibleModules.map((module) => module.manifest.id)); @@ -379,11 +462,17 @@ export function resolveModuleLifecycleView( reason: boot.blocked.reason, state: "blocked" as const, } - : { - configurationKey: boot.blocked.configurationKey, - reason: boot.blocked.reason, - state: "blocked" as const, - } + : "storageBucketId" in boot.blocked + ? { + reason: boot.blocked.reason, + state: "blocked" as const, + storageBucketId: boot.blocked.storageBucketId, + } + : { + configurationKey: boot.blocked.configurationKey, + reason: boot.blocked.reason, + state: "blocked" as const, + } : // Enabled and unblocked at boot, yet not started: unreachable while startup stays // all-or-nothing, and reported as blocked-by-nothing rather than as `running`, so a // future change to startup atomicity can never make this silently claim success. diff --git a/core/src/modules/runtime.ts b/core/src/modules/runtime.ts index 6307a19..8e0faea 100644 --- a/core/src/modules/runtime.ts +++ b/core/src/modules/runtime.ts @@ -2,7 +2,9 @@ import { pathToFileURL } from "node:url"; import { type CapabilityId, EMPTY_MODULE_CONFIGURATION, + EMPTY_MODULE_STORAGE, type ModuleConfiguration, + type ModuleStorage, type CapabilityRegistrar, type CapabilityResolver, capabilityIdSchema, @@ -96,6 +98,20 @@ function configurationFor( return configurations.get(moduleId) ?? EMPTY_MODULE_CONFIGURATION; } +/** + * Per-module scoped object storage, keyed by module id (ADR 0030). + * + * A lookup for the same reason configuration is one, and emphatically **not** a field on + * `ModuleRuntimeServices`: `services` is one shared object identity every module receives, while a + * storage facade is scoped to exactly the buckets one module declared. Sharing it would hand every + * module every other module's storage. + */ +export type ModuleStorageLookup = ReadonlyMap; + +function storageFor(storages: ModuleStorageLookup, moduleId: string): ModuleStorage { + return storages.get(moduleId) ?? EMPTY_MODULE_STORAGE; +} + export interface ModuleRuntimeTimeoutOptions { /** Test-only narrowing of `MODULE_STOP_HOOK_TIMEOUT_MS`, applied to both normal stop and startup * rollback, so a deadline can be proven in milliseconds instead of ten seconds. Production @@ -115,6 +131,12 @@ export interface ModuleRuntimeCompositionOptions extends ModuleRuntimeTimeoutOpt * `EMPTY_MODULE_CONFIGURATION`. */ readonly configurations?: ModuleConfigurationLookup; + /** + * The per-module scoped storage facades this runtime hands to each module's `start`, `stop`, and + * HTTP handler contexts (ADR 0030). Omitted by a caller exercising no storage; every module then + * receives `EMPTY_MODULE_STORAGE`, which refuses every bucket. + */ + readonly storages?: ModuleStorageLookup; } export interface StartModuleRuntimeOptions @@ -257,6 +279,7 @@ class StagedCapabilityRegistry { phase: "rollback" | "start" | "stop", services: ModuleRuntimeServices, config: ModuleConfiguration, + storage: ModuleStorage, ): ModuleRuntimeContext { const capabilities: CapabilityResolver = Object.freeze({ resolve: (capabilityId: CapabilityId): RegisteredCapability | undefined => { @@ -264,7 +287,13 @@ class StagedCapabilityRegistry { return this.#registered.get(capabilityId); }, }); - return Object.freeze({ capabilities, config, moduleId: module.manifest.id, services }); + return Object.freeze({ + capabilities, + config, + moduleId: module.manifest.id, + services, + storage, + }); } publish(): CapabilityRegistry { @@ -599,6 +628,7 @@ async function stopModules( phase: "rollback" | "stop", services: ModuleRuntimeServices, configurations: ModuleConfigurationLookup, + storages: ModuleStorageLookup, options: StopModulesOptions = {}, ): Promise { const timeoutMs = options.hookTimeoutMs ?? MODULE_STOP_HOOK_TIMEOUT_MS; @@ -619,6 +649,7 @@ async function stopModules( phase, services, configurationFor(configurations, moduleId), + storageFor(storages, moduleId), ), ), ), @@ -657,6 +688,7 @@ async function startModules( registry: StagedCapabilityRegistry, services: ModuleRuntimeServices, configurations: ModuleConfigurationLookup, + storages: ModuleStorageLookup, options: StopModulesOptions = {}, ): Promise { // A module joins the rollback set the moment its `start` hook is *invoked*, not when it @@ -675,6 +707,7 @@ async function startModules( "start", services, configurationFor(configurations, module.discovered.manifest.id), + storageFor(storages, module.discovered.manifest.id), ), ); } catch (error) { @@ -693,6 +726,7 @@ async function startModules( "rollback", services, configurations, + storages, options, ); throw new ModuleRuntimeError([startIssue, ...rollbackIssues]); @@ -717,8 +751,9 @@ export async function startDiscoveredModuleRuntime( const capabilityRegistry = new StagedCapabilityRegistry(); const httpRoutes = new StagedModuleHttpRoutes(); const configurations = timeouts.configurations ?? new Map(); + const storages = timeouts.storages ?? new Map(); await registerModules(modules, capabilityRegistry, httpRoutes, configurations); - await startModules(modules, capabilityRegistry, services, configurations, timeouts); + await startModules(modules, capabilityRegistry, services, configurations, storages, timeouts); const capabilities = capabilityRegistry.publish(); const routes = httpRoutes.publish(); @@ -744,6 +779,7 @@ export async function startDiscoveredModuleRuntime( "stop", services, configurations, + storages, timeouts, ); if (issues.length > 0) { @@ -761,6 +797,7 @@ export async function startModuleRuntime( const discovered = await discoverModules(options); return startDiscoveredModuleRuntime(discovered, options.services, { ...(options.configurations === undefined ? {} : { configurations: options.configurations }), + ...(options.storages === undefined ? {} : { storages: options.storages }), ...(options.hookTimeoutMs === undefined ? {} : { hookTimeoutMs: options.hookTimeoutMs }), }); } diff --git a/core/src/storage/binding.ts b/core/src/storage/binding.ts new file mode 100644 index 0000000..6bd1b1c --- /dev/null +++ b/core/src/storage/binding.ts @@ -0,0 +1,116 @@ +import { createHash } from "node:crypto"; + +/** + * How a module's logical bucket becomes a physical bucket on the host's storage service. + * + * This is **host implementation detail**. Nothing module-facing exposes the result: a module names + * its logical bucket and the binding is resolved for it, so a module can neither observe nor choose + * a physical bucket (ADR 0030). + */ + +/** + * The prefix every physical bucket this installation owns begins with. + * + * It exists so several Ekklesia-compatible deployments — or an Ekklesia installation and an + * unrelated application — can share one S3-compatible service without colliding, which is exactly + * the deployment shape the standalone story needs. Bounded and grammar-checked because it becomes + * the leading characters of an S3 bucket name. + */ +export const DEFAULT_STORAGE_BUCKET_PREFIX = "ekklesia"; +export const STORAGE_BUCKET_PREFIX_MAX_LENGTH = 16; +const BUCKET_PREFIX_PATTERN = /^[a-z][a-z0-9]*(?:-[a-z0-9]+)*$/u; + +export function isValidStorageBucketPrefix(value: unknown): value is string { + return ( + typeof value === "string" && + value.length > 0 && + value.length <= STORAGE_BUCKET_PREFIX_MAX_LENGTH && + BUCKET_PREFIX_PATTERN.test(value) + ); +} + +/** + * S3's own bucket-name ceiling, which every S3-compatible provider shares. Verified against the + * pinned SeaweedFS 4.28 development image: a 63-character name is accepted. + */ +const PHYSICAL_BUCKET_MAX_LENGTH = 63; +/** Long enough that a collision is not a practical concern, short enough to leave the readable part + * of the name legible. */ +const DIGEST_LENGTH = 12; + +/** + * The unambiguous logical identity of one binding: a module and one of its declared buckets. + * + * `--` is the separator because it **cannot occur inside either identifier**. Both `moduleIdSchema` + * and `logicalBucketIdSchema` require at least one `[a-z0-9]` after every hyphen, so a doubled + * hyphen is unrepresentable in each half. A single hyphen would have been ambiguous in exactly the + * way that matters: module `a-b` + bucket `c` and module `a` + bucket `b-c` would both render as + * `a-b-c`, silently handing two unrelated modules the same physical namespace. + */ +function bindingIdentity(moduleId: string, logicalBucketId: string): string { + return `${moduleId}--${logicalBucketId}`; +} + +/** + * Maps a module's logical bucket to the physical bucket name the host uses for it. + * + * ```text + * announcements + announcement-assets + * → ekklesia-announcements--announcement-assets-3f0c1d9a6b21 + * ``` + * + * Deterministic, so the same declaration always resolves to the same bucket across restarts, + * deployments, and hosts — which is what makes the binding stable enough to hold data. + * + * Collision-safe on two independent grounds. The `--` separator makes the logical identity + * unambiguous, and the appended digest is taken over that **full untruncated identity**, so even two + * modules whose names truncate to the same readable prefix still receive different buckets. Two + * different modules both declaring `assets` therefore always land in different physical namespaces — + * which is the isolation property this whole scheme exists to provide. + * + * The readable middle is truncated only as far as S3's 63-character ceiling requires, and never at a + * trailing hyphen, so the result always satisfies S3's naming rules: it begins with a letter, ends + * with an alphanumeric, contains only lowercase letters, digits, and single hyphens, and can never + * resemble an IP address. + */ +export function physicalBucketName( + prefix: string, + moduleId: string, + logicalBucketId: string, +): string { + const identity = bindingIdentity(moduleId, logicalBucketId); + const digest = createHash("sha256") + .update(identity, "utf8") + .digest("hex") + .slice(0, DIGEST_LENGTH); + // prefix + "-" + readable + "-" + digest + const budget = PHYSICAL_BUCKET_MAX_LENGTH - prefix.length - 1 - 1 - DIGEST_LENGTH; + const readable = identity.slice(0, Math.max(1, budget)).replace(/-+$/u, ""); + return `${prefix}-${readable}-${digest}`; +} + +/** One resolved binding: what the module calls it, and where the host actually keeps it. */ +export interface ModuleStorageBinding { + readonly moduleId: string; + readonly logicalBucketId: string; + /** Host detail. Never reachable from `ModuleStorage`; present here because Core's provider needs + * it, and because diagnostics may name it for an operator. */ + readonly physicalBucket: string; +} + +/** Resolves every binding one module's manifest declares, in declaration order. */ +export function resolveModuleStorageBindings( + prefix: string, + moduleId: string, + logicalBucketIds: readonly string[], +): readonly ModuleStorageBinding[] { + return Object.freeze( + logicalBucketIds.map((logicalBucketId) => + Object.freeze({ + logicalBucketId, + moduleId, + physicalBucket: physicalBucketName(prefix, moduleId, logicalBucketId), + }), + ), + ); +} diff --git a/core/src/storage/config.ts b/core/src/storage/config.ts new file mode 100644 index 0000000..ae5b4d7 --- /dev/null +++ b/core/src/storage/config.ts @@ -0,0 +1,185 @@ +import { + DEFAULT_STORAGE_BUCKET_PREFIX, + isValidStorageBucketPrefix, + STORAGE_BUCKET_PREFIX_MAX_LENGTH, +} from "./binding.js"; + +export type StorageConfigErrorCode = + | "STORAGE_ACCESS_KEY_MISSING" + | "STORAGE_BUCKET_PREFIX_INVALID" + | "STORAGE_ENDPOINT_INSECURE_IN_PRODUCTION" + | "STORAGE_ENDPOINT_INVALID" + | "STORAGE_ENDPOINT_WITH_CREDENTIALS" + | "STORAGE_REGION_INVALID" + | "STORAGE_SECRET_KEY_MISSING"; + +const storageConfigMessages: Readonly> = Object.freeze({ + STORAGE_ACCESS_KEY_MISSING: "A storage access key is required when a storage endpoint is set.", + STORAGE_BUCKET_PREFIX_INVALID: "The storage bucket prefix must be lowercase kebab-case.", + STORAGE_ENDPOINT_INSECURE_IN_PRODUCTION: + "A production storage endpoint must use https, exactly as the database transport must use TLS.", + STORAGE_ENDPOINT_INVALID: "The storage endpoint must be an absolute http or https URL.", + STORAGE_ENDPOINT_WITH_CREDENTIALS: + "The storage endpoint must not embed credentials; supply them separately.", + STORAGE_REGION_INVALID: "The storage region must be a short alphanumeric identifier.", + STORAGE_SECRET_KEY_MISSING: "A storage secret key is required when a storage endpoint is set.", +}); + +/** + * Never carries the endpoint, region, access key, secret key, or any other configured value — only a + * fixed code and message. Matching `IdentityConfigError` and `DatabaseTransportError` exactly, and + * for a sharper reason here: two of the values this validates are credentials. + */ +export class StorageConfigError extends Error { + readonly code: StorageConfigErrorCode; + + constructor(code: StorageConfigErrorCode) { + super(storageConfigMessages[code]); + this.name = "StorageConfigError"; + this.code = code; + this.stack = `${this.name}: ${this.message}`; + Object.freeze(this); + } +} + +export type StorageEnvironment = "development" | "production"; + +export interface StorageProviderConfigInput { + readonly endpoint?: string | undefined; + readonly region?: string | undefined; + readonly accessKey?: string | undefined; + readonly secretKey?: string | undefined; + readonly forcePathStyle?: string | boolean | undefined; + readonly bucketPrefix?: string | undefined; +} + +/** + * The validated host storage configuration. + * + * **Modules never see this.** It reaches Core's provider and nothing else; the module-facing + * `ModuleStorage` contract has no accessor that could return an endpoint, a region, or a credential + * (ADR 0030). + */ +export interface StorageProviderConfig { + readonly endpoint: string; + readonly region: string; + readonly accessKey: string; + readonly secretKey: string; + /** True for every S3-compatible service that is not AWS itself — SeaweedFS and MinIO both require + * it — so it defaults on. Virtual-host addressing needs wildcard DNS the typical self-hosted + * deployment does not have. */ + readonly forcePathStyle: boolean; + readonly bucketPrefix: string; +} + +const DEFAULT_REGION = "us-east-1"; +const REGION_PATTERN = /^[a-z0-9][a-z0-9-]{0,31}$/u; + +function parseBoolean(value: string | boolean | undefined, fallback: boolean): boolean { + if (value === undefined) { + return fallback; + } + if (typeof value === "boolean") { + return value; + } + return value !== "false"; +} + +/** + * Validates the host's storage configuration, or reports that storage is not configured at all. + * + * **Storage is optional.** An installation with no `endpoint` returns `undefined`, and that is a + * complete, valid deployment: modules that declare no buckets run exactly as before. Only a module + * that actually requires a bucket is affected, and it is reported blocked rather than crashing + * anything (ADR 0030). + * + * Once an endpoint *is* set, credentials become required — a half-configured provider that silently + * attempted anonymous access would be a worse failure than a loud one. + * + * Production requires `https`, on the same reasoning the database transport requires verified TLS: + * object bytes for a church may include member photographs and pastoral material, and shipping them + * over plaintext because it was convenient is not a tradeoff this platform makes. Development may + * use `http`, which is what the repository's own SeaweedFS Compose service speaks. + * + * An endpoint carrying userinfo is rejected outright rather than stripped: credentials belong in + * their own fields, where they are never logged, and a URL that embeds them tends to end up in + * places a URL is printed. + */ +export function createStorageProviderConfig( + input: StorageProviderConfigInput | undefined, + options: { readonly environment?: StorageEnvironment } = {}, +): StorageProviderConfig | undefined { + const endpoint = input?.endpoint?.trim(); + if (endpoint === undefined || endpoint === "") { + return undefined; + } + + let parsed: URL; + try { + parsed = new URL(endpoint); + } catch { + throw new StorageConfigError("STORAGE_ENDPOINT_INVALID"); + } + if (parsed.protocol !== "http:" && parsed.protocol !== "https:") { + throw new StorageConfigError("STORAGE_ENDPOINT_INVALID"); + } + if (parsed.username !== "" || parsed.password !== "") { + throw new StorageConfigError("STORAGE_ENDPOINT_WITH_CREDENTIALS"); + } + const environment = options.environment ?? "production"; + if (environment === "production" && parsed.protocol !== "https:") { + throw new StorageConfigError("STORAGE_ENDPOINT_INSECURE_IN_PRODUCTION"); + } + + const region = input?.region?.trim() === "" ? undefined : input?.region?.trim(); + if (region !== undefined && !REGION_PATTERN.test(region)) { + throw new StorageConfigError("STORAGE_REGION_INVALID"); + } + + const accessKey = input?.accessKey; + if (typeof accessKey !== "string" || accessKey.trim() === "") { + throw new StorageConfigError("STORAGE_ACCESS_KEY_MISSING"); + } + const secretKey = input?.secretKey; + if (typeof secretKey !== "string" || secretKey.trim() === "") { + throw new StorageConfigError("STORAGE_SECRET_KEY_MISSING"); + } + + const bucketPrefix = input?.bucketPrefix?.trim(); + if ( + bucketPrefix !== undefined && + bucketPrefix !== "" && + !isValidStorageBucketPrefix(bucketPrefix) + ) { + throw new StorageConfigError("STORAGE_BUCKET_PREFIX_INVALID"); + } + + return Object.freeze({ + accessKey, + bucketPrefix: + bucketPrefix === undefined || bucketPrefix === "" + ? DEFAULT_STORAGE_BUCKET_PREFIX + : bucketPrefix, + endpoint: parsed.origin + (parsed.pathname === "/" ? "" : parsed.pathname.replace(/\/+$/u, "")), + forcePathStyle: parseBoolean(input?.forcePathStyle, true), + region: region ?? DEFAULT_REGION, + secretKey, + }); +} + +/** + * A one-line operator-facing description of the configured provider that is safe to print anywhere — + * Doctor output, a log line, a support transcript. + * + * It names the endpoint's **origin only** (never a path, query, or userinfo), the region, the + * addressing style, and the bucket prefix. It never includes the access key, the secret key, or any + * value derived from them. `describeDatabaseTransport` sets the same precedent. + */ +export function describeStorageProvider(config: StorageProviderConfig): string { + const origin = new URL(config.endpoint).origin; + return `S3-compatible endpoint ${origin} · region ${config.region} · ${ + config.forcePathStyle ? "path-style" : "virtual-host-style" + } · bucket prefix "${config.bucketPrefix}"`; +} + +export { STORAGE_BUCKET_PREFIX_MAX_LENGTH }; diff --git a/core/src/storage/index.ts b/core/src/storage/index.ts new file mode 100644 index 0000000..570ba0f --- /dev/null +++ b/core/src/storage/index.ts @@ -0,0 +1,5 @@ +export * from "./binding.js"; +export * from "./config.js"; +export * from "./module-storage.js"; +export * from "./provider.js"; +export * from "./s3-provider.js"; diff --git a/core/src/storage/module-storage.ts b/core/src/storage/module-storage.ts new file mode 100644 index 0000000..bd83ad4 --- /dev/null +++ b/core/src/storage/module-storage.ts @@ -0,0 +1,185 @@ +import { + EMPTY_MODULE_STORAGE, + isValidModuleStorageKey, + type ModuleStorage, + type ModuleStorageBody, + type ModuleStorageBucket, + ModuleStorageError, + type ModuleStorageListOptions, + type ModuleStorageListing, + type ModuleStorageObject, + type ModuleStorageObjectMetadata, + type ModuleStoragePutOptions, +} from "@ekklesia/contracts"; +import { type ModuleStorageBinding, resolveModuleStorageBindings } from "./binding.js"; +import type { StorageProvider } from "./provider.js"; + +/** + * Builds the scoped storage facade one module receives. + * + * **This is where isolation is enforced, and it is enforced by construction.** The facade closes over + * exactly the bindings this module's own manifest declared; there is no lookup table spanning + * modules, no name that resolves outside those bindings, and no accessor taking a physical bucket. + * Module A cannot reach module B's storage by knowing its bucket name, its module id, or its + * physical bucket name, because none of those is an input to anything here (ADR 0030). + * + * Two modules that both declare `assets` receive two facades over two different physical buckets — + * see `physicalBucketName` for why that separation is collision-safe rather than merely likely. + */ + +/** Ensures a binding's physical bucket exists, at most once per process per binding. + * + * Lazy rather than eager at startup, deliberately: a storage outage must never stop the platform + * from booting. The promise is memoized so concurrent first uses share one round trip, and a + * *failed* attempt is discarded so a transient outage does not poison the binding for the life of + * the process — the next call retries, which is exactly the resilience a module like a slideshow + * depends on (ADR 0030). */ +function createBucketEnsurer( + provider: StorageProvider, + physicalBucket: string, +): () => Promise { + let pending: Promise | undefined; + return () => { + pending ??= provider.ensureBucket(physicalBucket).catch((error: unknown) => { + pending = undefined; + throw error; + }); + return pending; + }; +} + +/** Echoes the offending key only when it is a short string — a module's own input is safe to hand + * back, but a non-string or an enormous value is not something to put into an error message. */ +function assertValidKey(key: unknown): asserts key is string { + if (isValidModuleStorageKey(key)) { + return; + } + const echo = typeof key === "string" && key.length <= 256 ? key : undefined; + throw new ModuleStorageError("STORAGE_KEY_INVALID", echo); +} + +function createBucketHandle( + provider: StorageProvider, + binding: ModuleStorageBinding, +): ModuleStorageBucket { + const ensure = createBucketEnsurer(provider, binding.physicalBucket); + + return Object.freeze({ + // The **logical** id, never the physical bucket. A module that logs this leaks nothing about + // where the host keeps its data. + id: binding.logicalBucketId, + + async delete(key: string): Promise { + assertValidKey(key); + await ensure(); + await provider.delete(binding.physicalBucket, key); + }, + + async get(key: string): Promise { + assertValidKey(key); + await ensure(); + return provider.get(binding.physicalBucket, key); + }, + + async head(key: string): Promise { + assertValidKey(key); + await ensure(); + return provider.head(binding.physicalBucket, key); + }, + + async list(options: ModuleStorageListOptions = {}): Promise { + await ensure(); + return provider.list(binding.physicalBucket, options); + }, + + async put( + key: string, + body: ModuleStorageBody, + options: ModuleStoragePutOptions = {}, + ): Promise { + assertValidKey(key); + if (body instanceof ReadableStream && options.contentLength === undefined) { + throw new ModuleStorageError("STORAGE_CONTENT_LENGTH_REQUIRED", key); + } + await ensure(); + return provider.put(binding.physicalBucket, key, body, options); + }, + }); +} + +export interface CreateModuleStorageOptions { + readonly provider: StorageProvider; + readonly bucketPrefix: string; + readonly moduleId: string; + /** The logical bucket ids from this module's own manifest, in declaration order. */ + readonly logicalBucketIds: readonly string[]; +} + +/** + * The scoped `ModuleStorage` for one module. + * + * A module declaring no buckets gets `EMPTY_MODULE_STORAGE` — present, frozen, and refusing every + * bucket — so `context.storage` is never `undefined` and module code never branches on whether the + * host supplied storage. + */ +export function createModuleStorage(options: CreateModuleStorageOptions): ModuleStorage { + if (options.logicalBucketIds.length === 0) { + return EMPTY_MODULE_STORAGE; + } + const bindings = resolveModuleStorageBindings( + options.bucketPrefix, + options.moduleId, + options.logicalBucketIds, + ); + // A `Map` rather than a plain object: every lookup key here comes from module code, and indexing a + // plain object with such a key is the shape of a prototype-pollution bug. With a `Map` there is no + // prototype to reach, so `constructor` or `__proto__` are ordinary absent keys. + const handles = new Map( + bindings.map((binding) => [ + binding.logicalBucketId, + createBucketHandle(options.provider, binding), + ]), + ); + const declared = Object.freeze([...handles.keys()].sort()); + + return Object.freeze({ + bucket: (logicalBucketId: string): ModuleStorageBucket => { + const handle = typeof logicalBucketId === "string" ? handles.get(logicalBucketId) : undefined; + if (!handle) { + // Including another module's bucket id, which is simply not a key in this map. + throw new ModuleStorageError("STORAGE_BUCKET_UNDECLARED", logicalBucketId); + } + return handle; + }, + buckets: (): readonly string[] => declared, + }); +} + +/** + * Every module's storage, resolved once at startup. + * + * Returns an empty map when the host has no storage provider. That is not an error: an installation + * with no object storage is a complete, valid deployment, and only modules that *declared* a bucket + * are affected — those are reported blocked by the lifecycle resolver before any of this is reached + * (ADR 0030). + */ +export function createModuleStorageLookup( + provider: StorageProvider | undefined, + bucketPrefix: string, + modules: readonly { readonly moduleId: string; readonly logicalBucketIds: readonly string[] }[], +): ReadonlyMap { + if (!provider) { + return new Map(); + } + return new Map( + modules.map((module) => [ + module.moduleId, + createModuleStorage({ + bucketPrefix, + logicalBucketIds: module.logicalBucketIds, + moduleId: module.moduleId, + provider, + }), + ]), + ); +} diff --git a/core/src/storage/provider.ts b/core/src/storage/provider.ts new file mode 100644 index 0000000..0d381e1 --- /dev/null +++ b/core/src/storage/provider.ts @@ -0,0 +1,51 @@ +import type { + ModuleStorageBody, + ModuleStorageListOptions, + ModuleStorageListing, + ModuleStorageObject, + ModuleStorageObjectMetadata, + ModuleStoragePutOptions, +} from "@ekklesia/contracts"; + +/** + * The seam between Core's module-facing storage facade and whatever actually stores bytes. + * + * This is the **portability boundary made executable**. Everything above it — the scoped facade, + * binding resolution, lifecycle eligibility, diagnostics — is provider-agnostic and would be + * identical in a standalone host. Everything below it is one implementation + * (`createS3StorageProvider`) that could be replaced without any module changing a line. + * + * It deliberately speaks in **physical buckets**: resolving a module's logical bucket to a physical + * one has already happened by the time anything reaches here (`./binding.js`), so a provider never + * has to know what a module is. That split is what keeps scoping enforcement in exactly one place + * rather than duplicated into every provider. + * + * Also deliberately absent: any operation that *destroys* a bucket. Core never deletes a bucket, so + * there is no method to call by mistake, and disabling or removing a module cannot take data with it + * (ADR 0030). + */ +export interface StorageProvider { + /** + * Makes sure a physical bucket exists, idempotently. + * + * Called lazily on first use of a binding rather than at startup, so a storage outage can never + * prevent the platform from booting. + */ + ensureBucket(physicalBucket: string): Promise; + list(physicalBucket: string, options: ModuleStorageListOptions): Promise; + head(physicalBucket: string, key: string): Promise; + get(physicalBucket: string, key: string): Promise; + put( + physicalBucket: string, + key: string, + body: ModuleStorageBody, + options: ModuleStoragePutOptions, + ): Promise; + delete(physicalBucket: string, key: string): Promise; + /** A cheap reachability probe for diagnostics only. Its result is **never** persisted and never + * influences a module's lifecycle state — a provider being briefly unreachable is an operational + * condition, not an administrative decision (ADR 0030). */ + probe(): Promise; + /** Releases provider connections. Called once, during API shutdown. */ + close(): Promise; +} diff --git a/core/src/storage/s3-provider.ts b/core/src/storage/s3-provider.ts new file mode 100644 index 0000000..ef01b84 --- /dev/null +++ b/core/src/storage/s3-provider.ts @@ -0,0 +1,289 @@ +import { Readable } from "node:stream"; +import { + CreateBucketCommand, + DeleteObjectCommand, + GetObjectCommand, + HeadObjectCommand, + ListBucketsCommand, + ListObjectsV2Command, + PutObjectCommand, + S3Client, +} from "@aws-sdk/client-s3"; +import { + type ModuleStorageBody, + ModuleStorageError, + type ModuleStorageListOptions, + type ModuleStorageListing, + type ModuleStorageObject, + type ModuleStorageObjectMetadata, + type ModuleStoragePutOptions, +} from "@ekklesia/contracts"; +import type { StorageProviderConfig } from "./config.js"; +import type { StorageProvider } from "./provider.js"; + +/** + * The platform's first real `StorageProvider`, speaking the S3-compatible protocol. + * + * **It is not named after a product, and it must not become one.** SeaweedFS is what the + * repository's development Compose stack runs and the likely deployment target (ADR 0004), but + * nothing here depends on SeaweedFS: the same code addresses MinIO, AWS S3, Garage, or anything else + * speaking the protocol, selected purely by host configuration (ADR 0030). + * + * Server-only, and structurally so: this module imports `node:stream` and the AWS SDK, so a browser + * bundle importing it would fail to build. `@ekklesia/module-ui-sdk` and `surfaces/*` never reach it, + * because a module never imports Core at all. + */ + +/** Bounds a single listing page. S3-compatible providers cap this at 1000 regardless. */ +const MAX_LIST_PAGE_SIZE = 1000; + +interface S3LikeError { + readonly name?: unknown; + readonly $metadata?: { readonly httpStatusCode?: unknown }; +} + +function errorName(error: unknown): string { + const candidate = error as S3LikeError | null; + return typeof candidate?.name === "string" ? candidate.name : ""; +} + +function statusCode(error: unknown): number | undefined { + const candidate = error as S3LikeError | null; + const status = candidate?.$metadata?.httpStatusCode; + return typeof status === "number" ? status : undefined; +} + +/** + * Reduces any provider failure to one of the contract's closed codes. + * + * **Nothing from the caught error survives this function** — not its message, not its request id, + * not the XML body, not the endpoint it was talking to. Provider errors routinely carry the bucket + * name, the host, and sometimes the full signed request, and a module is entitled to log whatever it + * catches, so translation here is the only place that boundary can be held. + * + * The mapping is verified against the pinned SeaweedFS 4.28 image rather than assumed from AWS + * documentation: a missing object reports `NoSuchKey`, a missing bucket `NoSuchBucket`, and — the + * one that actually differs from AWS — creating an existing bucket reports `BucketAlreadyExists` + * where AWS reports `BucketAlreadyOwnedByYou`. + */ +function translateStorageError(error: unknown, key?: string): ModuleStorageError { + const name = errorName(error); + const status = statusCode(error); + + if (name === "NoSuchKey" || name === "NotFound" || (status === 404 && key !== undefined)) { + return new ModuleStorageError("STORAGE_OBJECT_NOT_FOUND", key); + } + if (name === "NoSuchBucket" || status === 404) { + return new ModuleStorageError("STORAGE_BUCKET_UNAVAILABLE"); + } + if ( + name === "AccessDenied" || + name === "InvalidAccessKeyId" || + name === "SignatureDoesNotMatch" + ) { + return new ModuleStorageError("STORAGE_OPERATION_DENIED"); + } + if (status === 403) { + return new ModuleStorageError("STORAGE_OPERATION_DENIED"); + } + // Connection-level failures surface as Node system errors or the SDK's own timeout types rather + // than as an HTTP status, which is exactly the transient case a module is expected to retry. + if ( + status === undefined && + (name.startsWith("Timeout") || + name === "TimeoutError" || + name === "AbortError" || + /ECONN|ENOTFOUND|EAI_AGAIN|EHOSTUNREACH|ENETUNREACH|EPIPE|ETIMEDOUT/u.test( + `${name} ${String((error as { code?: unknown } | null)?.code ?? "")}`, + )) + ) { + return new ModuleStorageError("STORAGE_PROVIDER_UNAVAILABLE"); + } + if (status !== undefined && status >= 500) { + return new ModuleStorageError("STORAGE_PROVIDER_UNAVAILABLE"); + } + return new ModuleStorageError("STORAGE_OPERATION_FAILED", key); +} + +function metadataFrom( + key: string, + source: { + readonly ContentType?: string | undefined; + readonly ContentLength?: number | undefined; + readonly ETag?: string | undefined; + readonly LastModified?: Date | undefined; + }, +): ModuleStorageObjectMetadata { + return Object.freeze({ + key, + // Providers quote the ETag; unquoting keeps the value usable as an HTTP validator without every + // module learning to strip quotes. + ...(source.ETag === undefined ? {} : { etag: source.ETag.replaceAll('"', "") }), + ...(source.ContentLength === undefined ? {} : { contentLength: source.ContentLength }), + ...(source.ContentType === undefined ? {} : { contentType: source.ContentType }), + ...(source.LastModified === undefined ? {} : { lastModified: source.LastModified }), + }); +} + +/** Converts the contract's accepted body shapes into something the SDK can send without this + * process buffering a large object. A Web stream becomes a Node stream rather than an array of + * bytes, which is the whole reason the contract accepts one. */ +function toRequestBody(body: ModuleStorageBody): Uint8Array | string | Readable | Blob { + if (body instanceof ReadableStream) { + return Readable.fromWeb(body as Parameters[0]); + } + return body; +} + +export function createS3StorageProvider(config: StorageProviderConfig): StorageProvider { + const client = new S3Client({ + credentials: { accessKeyId: config.accessKey, secretAccessKey: config.secretKey }, + endpoint: config.endpoint, + forcePathStyle: config.forcePathStyle, + region: config.region, + }); + + return Object.freeze({ + async ensureBucket(physicalBucket: string): Promise { + try { + await client.send(new CreateBucketCommand({ Bucket: physicalBucket })); + } catch (error) { + const name = errorName(error); + // Idempotency, spelled out for both dialects. AWS reports `BucketAlreadyOwnedByYou`; the + // pinned SeaweedFS image reports `BucketAlreadyExists` with a 409 — verified directly, not + // assumed. Treating only the AWS spelling as success would make every second boot fail + // against the repository's own development storage service. + if (name === "BucketAlreadyOwnedByYou" || name === "BucketAlreadyExists") { + return; + } + if (name === "AccessDenied" || statusCode(error) === 403) { + // The host's credentials cannot create it. Surfaced as an unavailable bucket rather than + // silently falling back to "maybe it exists": an operator needs to know their credentials + // are insufficient, not discover it through a later confusing read failure. + throw new ModuleStorageError("STORAGE_BUCKET_UNAVAILABLE"); + } + throw translateStorageError(error); + } + }, + + async list( + physicalBucket: string, + options: ModuleStorageListOptions, + ): Promise { + try { + const response = await client.send( + new ListObjectsV2Command({ + Bucket: physicalBucket, + ...(options.cursor === undefined ? {} : { ContinuationToken: options.cursor }), + ...(options.limit === undefined + ? {} + : { MaxKeys: Math.max(1, Math.min(options.limit, MAX_LIST_PAGE_SIZE)) }), + ...(options.prefix === undefined ? {} : { Prefix: options.prefix }), + }), + ); + const objects = (response.Contents ?? []).flatMap((entry) => + entry.Key === undefined + ? [] + : [ + metadataFrom(entry.Key, { + ETag: entry.ETag, + ContentLength: entry.Size, + LastModified: entry.LastModified, + }), + ], + ); + return Object.freeze({ + objects: Object.freeze(objects), + // Only forwarded when the provider says the listing is truncated, so an absent cursor + // always means "complete" rather than "the provider happened to omit a token". + ...(response.IsTruncated === true && response.NextContinuationToken !== undefined + ? { cursor: response.NextContinuationToken } + : {}), + }); + } catch (error) { + throw translateStorageError(error); + } + }, + + async head(physicalBucket: string, key: string): Promise { + try { + const response = await client.send( + new HeadObjectCommand({ Bucket: physicalBucket, Key: key }), + ); + return metadataFrom(key, response); + } catch (error) { + throw translateStorageError(error, key); + } + }, + + async get(physicalBucket: string, key: string): Promise { + try { + const response = await client.send( + new GetObjectCommand({ Bucket: physicalBucket, Key: key }), + ); + if (!response.Body) { + throw new ModuleStorageError("STORAGE_OPERATION_FAILED", key); + } + // A Web stream, never a buffered body: a module hands this straight to a `Response`, so a + // large object never passes through this process's memory in one piece. + const body = response.Body.transformToWebStream() as ReadableStream; + return Object.freeze({ ...metadataFrom(key, response), body }); + } catch (error) { + if (error instanceof ModuleStorageError) { + throw error; + } + throw translateStorageError(error, key); + } + }, + + async put( + physicalBucket: string, + key: string, + body: ModuleStorageBody, + options: ModuleStoragePutOptions, + ): Promise { + try { + const response = await client.send( + new PutObjectCommand({ + Body: toRequestBody(body), + Bucket: physicalBucket, + Key: key, + ...(options.contentLength === undefined + ? {} + : { ContentLength: options.contentLength }), + ...(options.contentType === undefined ? {} : { ContentType: options.contentType }), + }), + ); + return metadataFrom(key, { + ETag: response.ETag, + ...(options.contentLength === undefined ? {} : { ContentLength: options.contentLength }), + ...(options.contentType === undefined ? {} : { ContentType: options.contentType }), + }); + } catch (error) { + throw translateStorageError(error, key); + } + }, + + async delete(physicalBucket: string, key: string): Promise { + try { + await client.send(new DeleteObjectCommand({ Bucket: physicalBucket, Key: key })); + } catch (error) { + throw translateStorageError(error, key); + } + }, + + async probe(): Promise { + try { + // The cheapest call that proves the endpoint is reachable and the credentials sign + // correctly, without creating, reading, or writing anything. + await client.send(new ListBucketsCommand({})); + } catch (error) { + throw translateStorageError(error); + } + }, + + async close(): Promise { + client.destroy(); + }, + }); +} diff --git a/core/tests/doctor-report.test.ts b/core/tests/doctor-report.test.ts index 15c5b1e..5911974 100644 --- a/core/tests/doctor-report.test.ts +++ b/core/tests/doctor-report.test.ts @@ -47,6 +47,9 @@ function noConfigurationDependencies(): DoctorDependencies { planMigrations: async () => { throw new Error("database must not be planned"); }, + probeStorage: async () => { + throw new Error("storage must not be probed"); + }, readModuleConfiguration: async () => { throw new Error("module configuration must not be read"); }, @@ -112,8 +115,10 @@ Identity Trusts only the actual immediate socket peer for rate limiting; no forwarded header is trusted. Object storage - ○ Check unavailable - Core storage integration has not been implemented yet. + ○ Object storage not configured + No S3_ENDPOINT is set. This is a complete deployment unless a module requires storage. + ✓ No module requires object storage + 0 modules declare a logical storage bucket. Result: READY WITH WARNINGS`); }); diff --git a/core/tests/doctor.test.ts b/core/tests/doctor.test.ts index 4c6cf6f..68375e3 100644 --- a/core/tests/doctor.test.ts +++ b/core/tests/doctor.test.ts @@ -138,6 +138,7 @@ function dependencies(overrides: Partial = {}): DoctorDepend discoverModules: vi.fn(async () => Object.freeze([])), discoverSchemas: vi.fn(async () => schemaAggregate()), planMigrations: vi.fn(async () => plan()), + probeStorage: vi.fn(async () => undefined), readModuleConfiguration: vi.fn(async () => Object.freeze({ records: Object.freeze([]) })), readModuleLifecycle: vi.fn(async () => Object.freeze({ modules: Object.freeze([]) })), runtimeSatisfies: vi.fn(() => true), @@ -217,6 +218,7 @@ describe("Ekklesia Doctor diagnostics", () => { "identity.configuration", "identity.client-address", "object-storage.integration", + "object-storage.bindings", ]); expect(check(report, "modules.discovery").details).toEqual(["people 0.1.0", "calendar 0.2.0"]); expect(check(report, "modules.dependencies").details).toEqual(["Order: people → calendar"]); @@ -553,15 +555,19 @@ describe("Ekklesia Doctor diagnostics", () => { expect(serialized).not.toContain("also-secret.invalid"); }); - it("always reports object storage as unavailable without probing configuration", async () => { - const report = await diagnoseEkklesia(options(), dependencies()); + it("skips the storage check, without probing, when no provider is configured", async () => { + const services = dependencies(); + const report = await diagnoseEkklesia(options(), services); expect(check(report, "object-storage.integration")).toEqual({ - details: ["Core storage integration has not been implemented yet."], + details: [ + "No S3_ENDPOINT is set. This is a complete deployment unless a module requires storage.", + ], id: "object-storage.integration", status: "skipped", - summary: "Check unavailable", + summary: "Object storage not configured", }); + expect(services.probeStorage).not.toHaveBeenCalled(); }); it("continues static discovery when runtime validation fails", async () => { @@ -1208,3 +1214,164 @@ describe("Ekklesia Doctor — module configuration", () => { ); }); }); + +describe("Ekklesia Doctor — object storage", () => { + const storageConfig = { + accessKey: "AKIAEXAMPLEACCESSID", + bucketPrefix: "ekklesia", + endpoint: "https://storage.example.test", + forcePathStyle: true, + region: "us-east-1", + secretKey: "sk-example-secret-material", + } as const; + + function storingModule(id: string, buckets: readonly string[]): DiscoveredModule { + const base = discoveredModule(id); + return { + ...base, + manifest: { + ...base.manifest, + storage: { buckets: buckets.map((bucketId) => ({ id: bucketId })) }, + } as unknown as ModuleManifest, + }; + } + + it("does not probe, and stays healthy, when no provider is configured and nothing needs one", async () => { + const services = dependencies({ + discoverModules: vi.fn(async () => Object.freeze([discoveredModule("calendar")])), + }); + + const report = await diagnoseEkklesia(options(databaseUrl), services); + + expect(services.probeStorage).not.toHaveBeenCalled(); + expect(check(report, "object-storage.integration").status).toBe("skipped"); + // The load-bearing assertion: an installation whose modules need no object storage is not + // unhealthy for lacking a provider. + expect(check(report, "object-storage.bindings").status).toBe("pass"); + expect(check(report, "object-storage.bindings").details).toEqual([ + "0 modules declare a logical storage bucket.", + ]); + }); + + it("fails the bindings check when a module requires storage and none is configured", async () => { + const services = dependencies({ + discoverModules: vi.fn(async () => + Object.freeze([storingModule("announcements", ["announcement-assets"])]), + ), + }); + + const report = await diagnoseEkklesia(options(databaseUrl), services); + const bindings = check(report, "object-storage.bindings"); + + expect(bindings.status).toBe("failure"); + expect(bindings.details[0]).toContain("Configure an S3-compatible Ekklesia storage provider"); + expect(bindings.details).toContain( + "announcements · announcement-assets · unbound (no storage provider configured)", + ); + // And the lifecycle check agrees, because both come from one resolution. + expect(check(report, "modules.lifecycle").details).toContain( + "announcements 0.1.0 · blocked · storage_unavailable · announcement-assets", + ); + }); + + it("resolves and lists every binding when a provider is configured", async () => { + const services = dependencies({ + discoverModules: vi.fn(async () => + Object.freeze([storingModule("announcements", ["announcement-assets"])]), + ), + }); + + const report = await diagnoseEkklesia( + { ...options(databaseUrl), storage: storageConfig }, + services, + ); + const bindings = check(report, "object-storage.bindings"); + + expect(bindings.status).toBe("pass"); + // The physical bucket is named here deliberately: Doctor is an operator tool, and someone + // debugging a permissions problem needs to know which bucket to look at. It is still never + // reachable from module-facing code. + expect(bindings.details[0]).toMatch( + /^announcements · announcement-assets → ekklesia-announcements--announcement-assets-[0-9a-f]{12}$/u, + ); + }); + + it("probes a configured provider and reports it reachable, without printing a credential", async () => { + const services = dependencies({ + discoverModules: vi.fn(async () => Object.freeze([discoveredModule("calendar")])), + }); + + const report = await diagnoseEkklesia( + { ...options(databaseUrl), storage: storageConfig }, + services, + ); + const integration = check(report, "object-storage.integration"); + + expect(services.probeStorage).toHaveBeenCalledWith(storageConfig); + expect(integration.status).toBe("pass"); + expect(integration.details[0]).toContain("https://storage.example.test"); + const serialized = JSON.stringify(integration); + expect(serialized).not.toContain("AKIAEXAMPLEACCESSID"); + expect(serialized).not.toContain("sk-example-secret-material"); + }); + + it("warns rather than fails when a configured provider is unreachable", async () => { + const services = dependencies({ + discoverModules: vi.fn(async () => Object.freeze([discoveredModule("calendar")])), + probeStorage: vi.fn(async () => { + throw new Error("connect ECONNREFUSED 10.40.0.2:8333 accessKey=AKIAEXAMPLEACCESSID"); + }), + }); + + const report = await diagnoseEkklesia( + { ...options(databaseUrl), storage: storageConfig }, + services, + ); + const integration = check(report, "object-storage.integration"); + + // A warning, never a failure. A momentary network blip must not make the whole platform read as + // NOT READY, and it must never look like a module problem (ADR 0030). + expect(integration.status).toBe("warning"); + expect(report.result).not.toBe("not-ready"); + expect(integration.details).toContain( + "The configured storage service did not respond. Modules already running are unaffected.", + ); + const serialized = JSON.stringify(integration); + expect(serialized).not.toContain("AKIAEXAMPLEACCESSID"); + expect(serialized).not.toContain("10.40.0.2"); + }); + + it("still resolves bindings when the provider is unreachable, because binding is not health", async () => { + const services = dependencies({ + discoverModules: vi.fn(async () => + Object.freeze([storingModule("announcements", ["announcement-assets"])]), + ), + probeStorage: vi.fn(async () => { + throw new Error("unreachable"); + }), + }); + + const report = await diagnoseEkklesia( + { ...options(databaseUrl), storage: storageConfig }, + services, + ); + + expect(check(report, "object-storage.integration").status).toBe("warning"); + // The module is still eligible: a configured-but-briefly-unreachable provider is an operational + // condition, not a lifecycle state. + expect(check(report, "object-storage.bindings").status).toBe("pass"); + expect(check(report, "modules.lifecycle").details).toContain("announcements 0.1.0 · enabled"); + }); + + it("skips bindings when discovery itself failed", async () => { + const services = dependencies({ + discoverModules: vi.fn(async () => { + throw new Error("broken inventory"); + }), + }); + + const report = await diagnoseEkklesia(options(databaseUrl), services); + + expect(check(report, "object-storage.bindings").status).toBe("skipped"); + }); +}); diff --git a/core/tests/module-storage-lifecycle.test.ts b/core/tests/module-storage-lifecycle.test.ts new file mode 100644 index 0000000..818c5d5 --- /dev/null +++ b/core/tests/module-storage-lifecycle.test.ts @@ -0,0 +1,309 @@ +import { parseModuleManifest } from "@ekklesia/contracts"; +import { describe, expect, it } from "vitest"; +import type { DiscoveredModule } from "../src/modules/discovery.js"; +import { + isStorageBlockedDetail, + resolveModuleEligibility, + resolveModuleLifecycleView, +} from "../src/modules/lifecycle.js"; + +/** A minimal in-memory `DiscoveredModule`, built through `parseModuleManifest` so a fixture real + * discovery would reject can never stand in for one it would accept. */ +function fakeModule( + id: string, + options: { + readonly buckets?: readonly string[]; + readonly dependencies?: Record; + readonly configuration?: readonly Record[]; + } = {}, +): DiscoveredModule { + const manifest = parseModuleManifest({ + manifestVersion: 5, + id, + name: `${id} module`, + version: "1.0.0", + compatibility: { platform: "^0.1.0", moduleSdk: "^0.1.0" }, + ...(options.configuration ? { configuration: options.configuration } : {}), + ...(options.dependencies ? { dependencies: options.dependencies } : {}), + ...(options.buckets ? { storage: { buckets: options.buckets.map((id) => ({ id })) } } : {}), + entrypoints: { server: "./dist/does-not-exist/index.js" }, + }); + return { + directory: `/modules/${id}`, + entrypoints: { server: `/modules/${id}/dist/does-not-exist/index.js` }, + manifest, + manifestPath: `/modules/${id}/ekklesia.module.json`, + packageName: `@test/${id}`, + }; +} + +function enablement(...records: readonly (readonly [string, boolean])[]) { + return { modules: records.map(([moduleId, enabled]) => ({ moduleId, enabled })) }; +} + +const noConfiguration = { settings: { records: [] } }; +const withStorage = { providerConfigured: true }; +const withoutStorage = { providerConfigured: false }; + +function ids(modules: readonly DiscoveredModule[]): readonly string[] { + return modules.map((module) => module.manifest.id); +} + +function entry( + resolution: ReturnType, + moduleId: string, +): ReturnType["entries"][number] { + const found = resolution.entries.find((candidate) => candidate.moduleId === moduleId); + if (!found) { + throw new Error(`No eligibility entry for ${moduleId}.`); + } + return found; +} + +describe("declared storage decides eligibility", () => { + it("keeps a module requiring storage eligible when a provider is configured", () => { + const resolution = resolveModuleEligibility( + [fakeModule("announcements", { buckets: ["announcement-assets"] })], + enablement(), + noConfiguration, + withStorage, + ); + + expect(ids(resolution.eligibleModules)).toEqual(["announcements"]); + expect(entry(resolution, "announcements").blocked).toBeUndefined(); + }); + + it("blocks a module requiring storage when the host has no provider, naming the logical bucket", () => { + const resolution = resolveModuleEligibility( + [fakeModule("announcements", { buckets: ["announcement-assets"] })], + enablement(), + noConfiguration, + withoutStorage, + ); + + expect(resolution.eligibleModules).toEqual([]); + expect(entry(resolution, "announcements").blocked).toEqual({ + reason: "storage_unavailable", + storageBucketId: "announcement-assets", + }); + }); + + it("leaves a module declaring no buckets entirely unaffected by storage", () => { + const resolution = resolveModuleEligibility( + [fakeModule("calendar")], + enablement(), + noConfiguration, + withoutStorage, + ); + + // An installation with no object storage is a complete, valid deployment. + expect(ids(resolution.eligibleModules)).toEqual(["calendar"]); + expect(entry(resolution, "calendar").blocked).toBeUndefined(); + }); + + it("does not evaluate a disabled module's storage requirement", () => { + const resolution = resolveModuleEligibility( + [fakeModule("announcements", { buckets: ["announcement-assets"] })], + enablement(["announcements", false]), + noConfiguration, + withoutStorage, + ); + + // Disabled is not blocked: reporting a storage problem for something nobody asked to run would + // be noise. + expect(entry(resolution, "announcements").blocked).toBeUndefined(); + expect(entry(resolution, "announcements").configured.enabled).toBe(false); + }); + + it("defaults to no provider when the caller supplies no storage input at all", () => { + // Every pre-existing caller keeps its exact behaviour: the parameter is optional, and a module + // that declares no bucket is unaffected either way. + const withoutArgument = resolveModuleEligibility([fakeModule("calendar")], enablement()); + expect(ids(withoutArgument.eligibleModules)).toEqual(["calendar"]); + + const requiring = resolveModuleEligibility( + [fakeModule("announcements", { buckets: ["assets"] })], + enablement(), + ); + expect(requiring.eligibleModules).toEqual([]); + }); + + it("names only the logical bucket — never an endpoint, credential, or physical bucket", () => { + const resolution = resolveModuleEligibility( + [fakeModule("announcements", { buckets: ["announcement-assets"] })], + enablement(), + noConfiguration, + withoutStorage, + ); + const serialized = JSON.stringify(resolution.entries); + + expect(serialized).toContain("announcement-assets"); + expect(serialized).not.toContain("ekklesia-announcements--"); + expect(serialized).not.toContain("http"); + expect(serialized).not.toMatch(/accessKey|secretKey|endpoint/iu); + }); +}); + +describe("storage blocking propagates through the dependency graph", () => { + const graph = [ + fakeModule("media-storage", { buckets: ["objects"] }), + fakeModule("media", { dependencies: { modules: { required: { "media-storage": "^1.0.0" } } } }), + fakeModule("announcements", { dependencies: { modules: { required: { media: "^1.0.0" } } } }), + ]; + + it("blocks required dependents of a storage-blocked module, transitively", () => { + const resolution = resolveModuleEligibility( + graph, + enablement(), + noConfiguration, + withoutStorage, + ); + + expect(resolution.eligibleModules).toEqual([]); + // One mechanism, not two: the chain terminates at the module whose own requirement is + // unsatisfiable, exactly as it terminates at a disabled or misconfigured module. + expect(entry(resolution, "media-storage").blocked).toEqual({ + reason: "storage_unavailable", + storageBucketId: "objects", + }); + expect(entry(resolution, "media").blocked).toEqual({ + dependency: { kind: "module", moduleId: "media-storage" }, + reason: "required_dependency_blocked", + }); + expect(entry(resolution, "announcements").blocked).toEqual({ + dependency: { kind: "module", moduleId: "media" }, + reason: "required_dependency_blocked", + }); + }); + + it("unblocks the whole chain once a provider is configured", () => { + const resolution = resolveModuleEligibility(graph, enablement(), noConfiguration, withStorage); + + expect(ids(resolution.eligibleModules)).toEqual(["media-storage", "media", "announcements"]); + }); + + it("never blocks an unrelated module because another module needs storage", () => { + const resolution = resolveModuleEligibility( + [...graph, fakeModule("calendar")], + enablement(), + noConfiguration, + withoutStorage, + ); + + expect(ids(resolution.eligibleModules)).toEqual(["calendar"]); + }); +}); + +describe("storage and configuration blocking coexist without a second mechanism", () => { + it("reports the storage requirement first when both are unsatisfiable", () => { + const resolution = resolveModuleEligibility( + [ + fakeModule("announcements", { + buckets: ["assets"], + configuration: [{ key: "bucketLabel", type: "string", required: true }], + }), + ], + enablement(), + noConfiguration, + withoutStorage, + ); + + // Deterministic ordering — storage is evaluated before configuration — so an operator always + // sees the same reason for the same state rather than one that depends on evaluation order. + const blocked = entry(resolution, "announcements").blocked; + expect(blocked && isStorageBlockedDetail(blocked)).toBe(true); + }); + + it("falls through to the configuration reason once storage is available", () => { + const resolution = resolveModuleEligibility( + [ + fakeModule("announcements", { + buckets: ["assets"], + configuration: [{ key: "bucketLabel", type: "string", required: true }], + }), + ], + enablement(), + noConfiguration, + withStorage, + ); + + expect(entry(resolution, "announcements").blocked).toEqual({ + configurationKey: "bucketLabel", + reason: "configuration_missing", + }); + }); +}); + +describe("the lifecycle view reports storage state honestly", () => { + const discoveredModules = [fakeModule("announcements", { buckets: ["announcement-assets"] })]; + + function bootSnapshot(storage: { providerConfigured: boolean }) { + const resolution = resolveModuleEligibility( + discoveredModules, + enablement(), + noConfiguration, + storage, + ); + return { + discoveredModules, + eligibility: resolution.entries, + startedModuleIds: resolution.eligibleModules.map((module) => module.manifest.id), + }; + } + + it("reports a storage-blocked module with the logical bucket and no pending restart", () => { + const view = resolveModuleLifecycleView( + bootSnapshot(withoutStorage), + enablement(), + noConfiguration, + withoutStorage, + ); + + expect(view[0]?.runtime).toEqual({ + reason: "storage_unavailable", + state: "blocked", + storageBucketId: "announcement-assets", + }); + expect(view[0]?.restartRequired).toBe(false); + }); + + it("reports a pending restart once a provider is configured for a blocked module", () => { + const view = resolveModuleLifecycleView( + bootSnapshot(withoutStorage), + enablement(), + noConfiguration, + withStorage, + ); + + // The next boot really will start it, so the honest answer is that a restart is pending — while + // the runtime line still describes what *this* process did. + expect(view[0]?.runtime.state).toBe("blocked"); + expect(view[0]?.restartRequired).toBe(true); + }); + + it("reports a running module with storage as ordinary and unremarkable", () => { + const view = resolveModuleLifecycleView( + bootSnapshot(withStorage), + enablement(), + noConfiguration, + withStorage, + ); + + expect(view[0]?.runtime).toEqual({ state: "running" }); + expect(view[0]?.restartRequired).toBe(false); + }); + + it("puts no infrastructure detail into the view", () => { + const view = resolveModuleLifecycleView( + bootSnapshot(withoutStorage), + enablement(), + noConfiguration, + withoutStorage, + ); + const serialized = JSON.stringify(view); + + expect(serialized).toContain("announcement-assets"); + expect(serialized).not.toContain("ekklesia-announcements--"); + expect(serialized).not.toMatch(/http|accessKey|secretKey|seaweed/iu); + }); +}); diff --git a/core/tests/module-storage.integration.ts b/core/tests/module-storage.integration.ts new file mode 100644 index 0000000..4ac6f3a --- /dev/null +++ b/core/tests/module-storage.integration.ts @@ -0,0 +1,357 @@ +import { randomBytes } from "node:crypto"; +import { ModuleStorageError } from "@ekklesia/contracts"; +import { afterAll, describe, expect, it } from "vitest"; +import { physicalBucketName } from "../src/storage/binding.js"; +import { createStorageProviderConfig } from "../src/storage/config.js"; +import { createModuleStorage } from "../src/storage/module-storage.js"; +import type { StorageProvider } from "../src/storage/provider.js"; +import { createS3StorageProvider } from "../src/storage/s3-provider.js"; + +/** + * The storage contract against a **real S3-compatible server**. + * + * The unit suite proves scoping, binding, and validation with an in-memory provider; this proves the + * things only a real server can — and the reason it must exist is concrete. Probing the pinned + * SeaweedFS 4.28 image directly revealed that `CreateBucket` on an existing bucket reports + * `BucketAlreadyExists`, **not** AWS's `BucketAlreadyOwnedByYou`. A provider that only treated the + * AWS spelling as success would have failed on every second boot against this repository's own + * development storage service, and no amount of mocking would have caught it. + * + * SeaweedFS is used rather than a generic mock because it is the intended deployment target + * (ADR 0004) and because the repository's Compose file already pins and provisions it, so this adds + * no new CI infrastructure. Nothing here is SeaweedFS-specific in the *contract* — the same suite + * would pass against MinIO or AWS S3, which is the portability claim being tested. + */ +const endpointEnvironment = "EKKLESIA_TEST_STORAGE_ENDPOINT"; +const accessKeyEnvironment = "EKKLESIA_TEST_STORAGE_ACCESS_KEY"; +const secretKeyEnvironment = "EKKLESIA_TEST_STORAGE_SECRET_KEY"; + +class IntegrationHarnessError extends Error { + constructor(message: string) { + super(message); + this.name = "IntegrationHarnessError"; + this.stack = `${this.name}: ${this.message}`; + } +} + +function providerConfig() { + const endpoint = process.env[endpointEnvironment]; + if (typeof endpoint !== "string" || endpoint.trim() === "") { + throw new IntegrationHarnessError("The storage integration endpoint is not configured."); + } + const config = createStorageProviderConfig( + { + accessKey: process.env[accessKeyEnvironment], + endpoint, + secretKey: process.env[secretKeyEnvironment], + }, + // The disposable service speaks plaintext on loopback, exactly like the disposable PostgreSQL + // this suite already uses; production configuration would refuse it. + { environment: "development" }, + ); + if (!config) { + throw new IntegrationHarnessError("The storage integration configuration is invalid."); + } + return config; +} + +/** A unique prefix per run, so repeated runs and parallel jobs never collide and no test depends on + * another's leftovers. */ +function disposablePrefix(): string { + return `ekktest${randomBytes(3).toString("hex")}`; +} + +const openProviders: StorageProvider[] = []; + +function provider(): StorageProvider { + const created = createS3StorageProvider(providerConfig()); + openProviders.push(created); + return created; +} + +afterAll(async () => { + for (const open of openProviders.splice(0)) { + await open.close().catch(() => undefined); + } +}); + +describe.sequential("live S3-compatible module storage", () => { + it("creates a module's bucket idempotently, across both provider dialects", async () => { + const shared = provider(); + const prefix = disposablePrefix(); + const bucket = physicalBucketName(prefix, "announcements", "announcement-assets"); + + await shared.ensureBucket(bucket); + // The assertion this whole suite exists for: the pinned SeaweedFS image answers a repeat + // `CreateBucket` with `BucketAlreadyExists` rather than AWS's `BucketAlreadyOwnedByYou`, and the + // provider must treat both as success. Every restart depends on this. + await expect(shared.ensureBucket(bucket)).resolves.toBeUndefined(); + await expect(shared.ensureBucket(bucket)).resolves.toBeUndefined(); + }); + + it("round-trips an object through the scoped facade with its content type preserved", async () => { + const prefix = disposablePrefix(); + const storage = createModuleStorage({ + bucketPrefix: prefix, + logicalBucketIds: ["announcement-assets"], + moduleId: "announcements", + provider: provider(), + }); + const bucket = storage.bucket("announcement-assets"); + const png = new Uint8Array([0x89, 0x50, 0x4e, 0x47, 0x0d, 0x0a, 0x1a, 0x0a]); + + const written = await bucket.put("slides/welcome.png", png, { contentType: "image/png" }); + expect(written.key).toBe("slides/welcome.png"); + + const object = await bucket.get("slides/welcome.png"); + expect(object.contentType).toBe("image/png"); + expect(object.contentLength).toBe(png.byteLength); + // Unquoted for the caller — providers return a quoted ETag, and every module learning to strip + // quotes would be a contract failure rather than a module failure. + expect(object.etag).toMatch(/^[0-9a-f]{32}$/u); + expect(new Uint8Array(await new Response(object.body).arrayBuffer())).toEqual(png); + }); + + it("streams a body without buffering, when the length is supplied", async () => { + const prefix = disposablePrefix(); + const bucket = createModuleStorage({ + bucketPrefix: prefix, + logicalBucketIds: ["assets"], + moduleId: "media", + provider: provider(), + }).bucket("assets"); + const payload = "x".repeat(64 * 1024); + const stream = new Blob([payload]).stream() as ReadableStream; + + await bucket.put("large.bin", stream, { + contentLength: payload.length, + contentType: "application/octet-stream", + }); + + const object = await bucket.get("large.bin"); + expect(object.contentLength).toBe(payload.length); + expect(await new Response(object.body).text()).toBe(payload); + }); + + it("returns a Web stream a module can hand straight to a Response", async () => { + const prefix = disposablePrefix(); + const bucket = createModuleStorage({ + bucketPrefix: prefix, + logicalBucketIds: ["assets"], + moduleId: "announcements", + provider: provider(), + }).bucket("assets"); + await bucket.put("welcome.png", new Uint8Array([1, 2, 3]), { contentType: "image/png" }); + + const object = await bucket.get("welcome.png"); + expect(object.body).toBeInstanceOf(ReadableStream); + // The browser-delivery shape this pass settles on: browser → module route → scoped storage, + // with no S3 knowledge or credential anywhere near the browser. + const response = new Response(object.body, { + headers: { "content-type": object.contentType ?? "application/octet-stream" }, + }); + expect(response.headers.get("content-type")).toBe("image/png"); + expect(new Uint8Array(await response.arrayBuffer())).toEqual(new Uint8Array([1, 2, 3])); + }); + + it("heads, lists with a prefix, and deletes", async () => { + const prefix = disposablePrefix(); + const bucket = createModuleStorage({ + bucketPrefix: prefix, + logicalBucketIds: ["assets"], + moduleId: "announcements", + provider: provider(), + }).bucket("assets"); + + await bucket.put("slides/a.png", "a", { contentType: "image/png" }); + await bucket.put("slides/b.png", "b", { contentType: "image/png" }); + await bucket.put("other/c.txt", "c"); + + const metadata = await bucket.head("slides/a.png"); + expect(metadata.contentType).toBe("image/png"); + expect(metadata.contentLength).toBe(1); + expect(metadata.lastModified).toBeInstanceOf(Date); + + const listing = await bucket.list({ prefix: "slides/" }); + expect(listing.objects.map((object) => object.key)).toEqual(["slides/a.png", "slides/b.png"]); + expect(listing.objects[0]?.contentLength).toBe(1); + // A complete listing carries no cursor, so an absent cursor always means "done". + expect(listing.cursor).toBeUndefined(); + + await bucket.delete("slides/a.png"); + expect((await bucket.list({ prefix: "slides/" })).objects.map((o) => o.key)).toEqual([ + "slides/b.png", + ]); + }); + + it("paginates a listing rather than silently truncating it", async () => { + const prefix = disposablePrefix(); + const bucket = createModuleStorage({ + bucketPrefix: prefix, + logicalBucketIds: ["assets"], + moduleId: "paged", + provider: provider(), + }).bucket("assets"); + for (const index of [0, 1, 2, 3, 4]) { + await bucket.put(`object-${index}.txt`, String(index)); + } + + const first = await bucket.list({ limit: 2 }); + expect(first.objects).toHaveLength(2); + expect(first.cursor).toBeDefined(); + + const keys = [...first.objects.map((object) => object.key)]; + let cursor = first.cursor; + while (cursor !== undefined) { + const page = await bucket.list({ cursor, limit: 2 }); + keys.push(...page.objects.map((object) => object.key)); + cursor = page.cursor; + } + + expect(keys.sort()).toEqual([ + "object-0.txt", + "object-1.txt", + "object-2.txt", + "object-3.txt", + "object-4.txt", + ]); + }); + + it("maps a missing object to a structured not-found error, never a provider exception", async () => { + const prefix = disposablePrefix(); + const bucket = createModuleStorage({ + bucketPrefix: prefix, + logicalBucketIds: ["assets"], + moduleId: "announcements", + provider: provider(), + }).bucket("assets"); + await bucket.put("present.txt", "x"); + + let caught: unknown; + try { + await bucket.get("absent.txt"); + } catch (error) { + caught = error; + } + + expect(caught).toBeInstanceOf(ModuleStorageError); + expect((caught as ModuleStorageError).code).toBe("STORAGE_OBJECT_NOT_FOUND"); + expect((caught as ModuleStorageError).key).toBe("absent.txt"); + // Nothing from the provider survives translation: no endpoint, no bucket, no request id, no XML. + const rendered = `${String(caught)} ${JSON.stringify(caught)}`; + expect(rendered).not.toContain("http"); + expect(rendered).not.toContain(prefix); + expect(rendered).not.toContain(" { + const shared = provider(); + const prefix = disposablePrefix(); + const announcements = createModuleStorage({ + bucketPrefix: prefix, + logicalBucketIds: ["assets"], + moduleId: "announcements", + provider: shared, + }); + const video = createModuleStorage({ + bucketPrefix: prefix, + logicalBucketIds: ["assets"], + moduleId: "video", + provider: shared, + }); + + await announcements.bucket("assets").put("same-key.txt", "from-announcements"); + await video.bucket("assets").put("same-key.txt", "from-video"); + + // Same logical name, same key, one real storage service — and two different objects. + expect( + await new Response((await announcements.bucket("assets").get("same-key.txt")).body).text(), + ).toBe("from-announcements"); + expect(await new Response((await video.bucket("assets").get("same-key.txt")).body).text()).toBe( + "from-video", + ); + + // Neither module can see the other's object, even by listing. + expect((await announcements.bucket("assets").list()).objects.map((o) => o.key)).toEqual([ + "same-key.txt", + ]); + // And there is no way to ask for the other module's bucket at all. + expect(() => announcements.bucket("video-assets")).toThrow(ModuleStorageError); + }); + + it("accepts the longest bucket name the mapping can produce", async () => { + const shared = provider(); + const prefix = disposablePrefix(); + const moduleId = `m${"o".repeat(62)}`; + const bucket = physicalBucketName(prefix, moduleId, `b${"u".repeat(46)}`); + + expect(bucket.length).toBeLessThanOrEqual(63); + // Proven against the real server rather than against the documented rule. + await expect(shared.ensureBucket(bucket)).resolves.toBeUndefined(); + }); + + it("supports nested object keys, which modules genuinely use", async () => { + const prefix = disposablePrefix(); + const bucket = createModuleStorage({ + bucketPrefix: prefix, + logicalBucketIds: ["assets"], + moduleId: "announcements", + provider: provider(), + }).bucket("assets"); + + await bucket.put("2026/easter/slides/welcome.png", "x", { contentType: "image/png" }); + expect((await bucket.list({ prefix: "2026/easter/" })).objects.map((o) => o.key)).toEqual([ + "2026/easter/slides/welcome.png", + ]); + }); + + it("probes a reachable provider, and reports an unreachable one as provider-unavailable", async () => { + await expect(provider().probe()).resolves.toBeUndefined(); + + const unreachable = createS3StorageProvider( + createStorageProviderConfig( + { + accessKey: "unused", + // A port nothing listens on: a real connection failure rather than a simulated one. + endpoint: "http://127.0.0.1:1", + secretKey: "unused", + }, + { environment: "development" }, + )!, + ); + try { + let caught: unknown; + try { + await unreachable.probe(); + } catch (error) { + caught = error; + } + expect(caught).toBeInstanceOf(ModuleStorageError); + // Transient by nature: a module retries through this, and no lifecycle state changes + // (ADR 0030). + expect((caught as ModuleStorageError).code).toBe("STORAGE_PROVIDER_UNAVAILABLE"); + } finally { + await unreachable.close().catch(() => undefined); + } + }); + + it("never deletes a bucket, even when every object in it is removed", async () => { + const shared = provider(); + const prefix = disposablePrefix(); + const bucket = createModuleStorage({ + bucketPrefix: prefix, + logicalBucketIds: ["assets"], + moduleId: "announcements", + provider: shared, + }).bucket("assets"); + + await bucket.put("only.txt", "x"); + await bucket.delete("only.txt"); + + // The bucket still exists and still accepts writes: emptying is not removal, and nothing in this + // platform destroys a bucket (ADR 0030). + expect((await bucket.list()).objects).toEqual([]); + await expect(bucket.put("again.txt", "y")).resolves.toMatchObject({ key: "again.txt" }); + }); +}); diff --git a/core/tests/module-storage.test.ts b/core/tests/module-storage.test.ts new file mode 100644 index 0000000..3dee37d --- /dev/null +++ b/core/tests/module-storage.test.ts @@ -0,0 +1,775 @@ +import { + EMPTY_MODULE_STORAGE, + isValidModuleStorageKey, + MODULE_STORAGE_KEY_MAX_BYTES, + type ModuleStorageBody, + ModuleStorageError, + type ModuleStorageListOptions, + type ModuleStorageListing, + type ModuleStorageObject, + type ModuleStorageObjectMetadata, + type ModuleStoragePutOptions, +} from "@ekklesia/contracts"; +import { describe, expect, it } from "vitest"; +import { + DEFAULT_STORAGE_BUCKET_PREFIX, + isValidStorageBucketPrefix, + physicalBucketName, + resolveModuleStorageBindings, +} from "../src/storage/binding.js"; +import { createStorageProviderConfig, describeStorageProvider } from "../src/storage/config.js"; +import { createModuleStorage, createModuleStorageLookup } from "../src/storage/module-storage.js"; +import type { StorageProvider } from "../src/storage/provider.js"; + +/** Reads a Web stream to bytes without depending on DOM-only helper types. */ +async function collect(stream: ReadableStream): Promise { + const reader = stream.getReader(); + const chunks: Uint8Array[] = []; + let total = 0; + for (;;) { + const { done, value } = await reader.read(); + if (done) { + break; + } + chunks.push(value); + total += value.byteLength; + } + const merged = new Uint8Array(total); + let offset = 0; + for (const chunk of chunks) { + merged.set(chunk, offset); + offset += chunk.byteLength; + } + return merged; +} + +function streamOf(bytes: Uint8Array): ReadableStream { + return new ReadableStream({ + start(controller) { + controller.enqueue(bytes); + controller.close(); + }, + }); +} + +interface RecordedCall { + readonly operation: string; + readonly physicalBucket: string; + readonly key?: string; +} + +/** + * A recording in-memory `StorageProvider`. + * + * Deliberately keyed by **physical** bucket, exactly as the real provider is, so a test can assert + * which physical namespace an operation actually reached — which is the only way to prove isolation + * rather than assume it. + */ +function fakeProvider(options: { failEnsure?: () => Error } = {}): StorageProvider & { + readonly calls: readonly RecordedCall[]; + readonly buckets: ReadonlyMap>; +} { + const calls: RecordedCall[] = []; + const buckets = new Map>(); + const objectsIn = (physicalBucket: string) => { + const bucket = buckets.get(physicalBucket); + if (!bucket) { + throw new ModuleStorageError("STORAGE_BUCKET_UNAVAILABLE"); + } + return bucket; + }; + + const provider: StorageProvider = { + async close() {}, + async delete(physicalBucket, key) { + calls.push({ key, operation: "delete", physicalBucket }); + objectsIn(physicalBucket).delete(key); + }, + async ensureBucket(physicalBucket) { + calls.push({ operation: "ensureBucket", physicalBucket }); + const failure = options.failEnsure?.(); + if (failure) { + throw failure; + } + if (!buckets.has(physicalBucket)) { + buckets.set(physicalBucket, new Map()); + } + }, + async get(physicalBucket, key): Promise { + calls.push({ key, operation: "get", physicalBucket }); + const stored = objectsIn(physicalBucket).get(key); + if (!stored) { + throw new ModuleStorageError("STORAGE_OBJECT_NOT_FOUND", key); + } + return Object.freeze({ + body: streamOf(stored.bytes), + contentLength: stored.bytes.byteLength, + key, + ...(stored.contentType === undefined ? {} : { contentType: stored.contentType }), + }); + }, + async head(physicalBucket, key): Promise { + calls.push({ key, operation: "head", physicalBucket }); + const stored = objectsIn(physicalBucket).get(key); + if (!stored) { + throw new ModuleStorageError("STORAGE_OBJECT_NOT_FOUND", key); + } + return Object.freeze({ contentLength: stored.bytes.byteLength, key }); + }, + async list( + physicalBucket, + listOptions: ModuleStorageListOptions, + ): Promise { + calls.push({ operation: "list", physicalBucket }); + const entries = [...objectsIn(physicalBucket).keys()] + .filter((key) => listOptions.prefix === undefined || key.startsWith(listOptions.prefix)) + .sort(); + return Object.freeze({ + objects: Object.freeze(entries.map((key) => Object.freeze({ key }))), + }); + }, + async probe() {}, + async put( + physicalBucket, + key, + body: ModuleStorageBody, + putOptions: ModuleStoragePutOptions, + ): Promise { + calls.push({ key, operation: "put", physicalBucket }); + const bytes = + typeof body === "string" + ? new TextEncoder().encode(body) + : body instanceof Uint8Array + ? body + : await collect(body as ReadableStream); + objectsIn(physicalBucket).set(key, { + bytes, + ...(putOptions.contentType === undefined ? {} : { contentType: putOptions.contentType }), + }); + return Object.freeze({ contentLength: bytes.byteLength, key }); + }, + }; + return Object.assign(provider, { buckets: buckets as never, calls: calls as never }); +} + +describe("physical bucket naming", () => { + it("is deterministic, so a binding survives restarts and redeployments", () => { + const first = physicalBucketName("ekklesia", "announcements", "announcement-assets"); + const second = physicalBucketName("ekklesia", "announcements", "announcement-assets"); + + expect(first).toBe(second); + expect(first).toContain("announcements--announcement-assets"); + }); + + it("keeps two modules declaring the same logical name in different namespaces", () => { + const announcements = physicalBucketName("ekklesia", "announcements", "assets"); + const video = physicalBucketName("ekklesia", "video", "assets"); + + // The property the whole scheme exists for: `assets` is an obvious name two unrelated modules + // will both pick. + expect(announcements).not.toBe(video); + }); + + it("cannot be confused by a hyphen in either identifier", () => { + // The ambiguity a single-hyphen separator would have produced: module `a-b` + bucket `c` and + // module `a` + bucket `b-c` would both render as `a-b-c`, silently sharing a namespace. + const left = physicalBucketName("ekklesia", "a-b", "c"); + const right = physicalBucketName("ekklesia", "a", "b-c"); + + expect(left).not.toBe(right); + }); + + it("stays within S3's 63-character bucket-name limit even for maximal identifiers", () => { + const moduleId = `m${"o".repeat(62)}`.slice(0, 64); + const bucketId = `b${"u".repeat(46)}`.slice(0, 48); + const name = physicalBucketName("ekklesia", moduleId, bucketId); + + expect(name.length).toBeLessThanOrEqual(63); + // Still distinct from a neighbour that truncates to the same readable prefix, because the digest + // is taken over the full untruncated identity. + expect(name).not.toBe(physicalBucketName("ekklesia", moduleId, `${bucketId.slice(0, 47)}x`)); + }); + + it("always produces a name S3 itself would accept", () => { + const samples = [ + physicalBucketName("ekklesia", "a", "b"), + physicalBucketName("ekklesia", `m${"o".repeat(62)}`, `b${"u".repeat(46)}`), + physicalBucketName("church", "announcements", "announcement-assets"), + physicalBucketName("ekklesia", "a-b", "c-d"), + ]; + for (const name of samples) { + expect(name.length).toBeGreaterThanOrEqual(3); + expect(name.length).toBeLessThanOrEqual(63); + expect(name).toMatch(/^[a-z0-9][a-z0-9-]*[a-z0-9]$/u); + // No dots at all, so S3's adjacent-period and period-next-to-hyphen rules are unreachable and + // virtual-host-style addressing stays TLS-safe should a provider ever use it. Consecutive + // hyphens are explicitly fine — `--` is the separator, and the pinned SeaweedFS image accepts + // such names (verified directly against it in the integration suite). + expect(name).not.toContain("."); + // Never IP-shaped, and never a reserved prefix, because it always begins with the validated + // alphabetic prefix. + expect(name).not.toMatch(/^\d+-\d+-\d+-\d+$/u); + expect(name.startsWith("xn--")).toBe(false); + expect(name.startsWith("sthree-")).toBe(false); + expect(name.endsWith("-s3alias")).toBe(false); + expect(name.endsWith("--ol-s3")).toBe(false); + } + }); + + it("isolates two installations sharing one storage service through the prefix", () => { + const first = physicalBucketName("ekklesia", "announcements", "assets"); + const second = physicalBucketName("stanne", "announcements", "assets"); + + expect(first).not.toBe(second); + }); + + it("accepts only a safe bucket prefix", () => { + expect(isValidStorageBucketPrefix(DEFAULT_STORAGE_BUCKET_PREFIX)).toBe(true); + for (const bad of ["", "Ekklesia", "ekklesia_", "-ekklesia", "ekklesia--x", "a".repeat(17)]) { + expect(isValidStorageBucketPrefix(bad)).toBe(false); + } + }); + + it("resolves every declared bucket in declaration order", () => { + const bindings = resolveModuleStorageBindings("ekklesia", "announcements", [ + "announcement-assets", + "thumbnails", + ]); + + expect(bindings.map((binding) => binding.logicalBucketId)).toEqual([ + "announcement-assets", + "thumbnails", + ]); + expect(new Set(bindings.map((binding) => binding.physicalBucket)).size).toBe(2); + }); +}); + +describe("object key validation", () => { + it("accepts ordinary and nested keys", () => { + for (const key of ["welcome.png", "slides/welcome.png", "a/b/c/deep.webp", "ünïcode.png"]) { + expect(isValidModuleStorageKey(key)).toBe(true); + } + }); + + it("rejects keys that would be ambiguous or unrepresentable", () => { + for (const key of [ + "", + "/leading-slash.png", + "back\\slash.png", + "controlcharacter.png", + "controlcharacter.png", + "deletecharacter.png", + "\ud800lone-surrogate.png", + "x".repeat(MODULE_STORAGE_KEY_MAX_BYTES + 1), + ]) { + expect(isValidModuleStorageKey(key)).toBe(false); + } + }); + + it("measures the length limit in UTF-8 bytes, not characters", () => { + // 512 three-byte characters is 1536 bytes — over the limit despite being 512 characters. + expect(isValidModuleStorageKey("あ".repeat(512))).toBe(false); + expect(isValidModuleStorageKey("あ".repeat(341))).toBe(true); + }); + + it("does not treat a key as a filesystem path", () => { + // `..` has no traversal meaning to an object store, and the physical bucket never comes from the + // key, so this is an ordinary (if odd) object name rather than an escape. + expect(isValidModuleStorageKey("../still-just-a-name.png")).toBe(true); + }); +}); + +describe("module storage scoping", () => { + const provider = () => fakeProvider(); + + it("gives a module exactly the buckets its manifest declared", () => { + const storage = createModuleStorage({ + bucketPrefix: "ekklesia", + logicalBucketIds: ["announcement-assets", "thumbnails"], + moduleId: "announcements", + provider: provider(), + }); + + expect(storage.buckets()).toEqual(["announcement-assets", "thumbnails"]); + expect(storage.bucket("announcement-assets").id).toBe("announcement-assets"); + }); + + it("refuses a logical bucket the module did not declare", () => { + const storage = createModuleStorage({ + bucketPrefix: "ekklesia", + logicalBucketIds: ["announcement-assets"], + moduleId: "announcements", + provider: provider(), + }); + + expect(() => storage.bucket("video-assets")).toThrow(ModuleStorageError); + try { + storage.bucket("video-assets"); + } catch (error) { + expect((error as ModuleStorageError).code).toBe("STORAGE_BUCKET_UNDECLARED"); + } + }); + + it("gives a module no way to reach another module's storage by knowing its name", async () => { + const shared = fakeProvider(); + const announcements = createModuleStorage({ + bucketPrefix: "ekklesia", + logicalBucketIds: ["assets"], + moduleId: "announcements", + provider: shared, + }); + const video = createModuleStorage({ + bucketPrefix: "ekklesia", + logicalBucketIds: ["assets"], + moduleId: "video", + provider: shared, + }); + + await announcements.bucket("assets").put("shared-name.txt", "from-announcements"); + await video.bucket("assets").put("shared-name.txt", "from-video"); + + // Same logical name, same key, same provider — and two different objects, because the physical + // namespaces differ. + const decoder = new TextDecoder(); + const fromAnnouncements = decoder.decode( + await collect((await announcements.bucket("assets").get("shared-name.txt")).body), + ); + const fromVideo = decoder.decode( + await collect((await video.bucket("assets").get("shared-name.txt")).body), + ); + + expect(fromAnnouncements).toBe("from-announcements"); + expect(fromVideo).toBe("from-video"); + expect(shared.buckets.size).toBe(2); + }); + + it("exposes no physical bucket, endpoint, or credential through the module-facing surface", () => { + const storage = createModuleStorage({ + bucketPrefix: "ekklesia", + logicalBucketIds: ["announcement-assets"], + moduleId: "announcements", + provider: provider(), + }); + const bucket = storage.bucket("announcement-assets"); + + // The handle names the *logical* bucket, and has no accessor for anything else. + expect(bucket.id).toBe("announcement-assets"); + expect(Object.keys(bucket).sort()).toEqual(["delete", "get", "head", "id", "list", "put"]); + expect(Object.keys(storage).sort()).toEqual(["bucket", "buckets"]); + const serialized = JSON.stringify({ bucket, storage }); + expect(serialized).not.toContain( + physicalBucketName("ekklesia", "announcements", "announcement-assets"), + ); + expect(serialized).not.toContain("http"); + }); + + it("gives a module declaring no buckets an empty, frozen storage that refuses everything", () => { + const storage = createModuleStorage({ + bucketPrefix: "ekklesia", + logicalBucketIds: [], + moduleId: "calendar", + provider: provider(), + }); + + expect(storage).toBe(EMPTY_MODULE_STORAGE); + expect(storage.buckets()).toEqual([]); + expect(() => storage.bucket("anything")).toThrow(ModuleStorageError); + expect(Object.isFrozen(storage)).toBe(true); + }); + + it("resolves nothing at all when the host has no provider", () => { + const lookup = createModuleStorageLookup(undefined, "ekklesia", [ + { logicalBucketIds: ["assets"], moduleId: "announcements" }, + ]); + + expect(lookup.size).toBe(0); + }); + + it.each([ + "__proto__", + "constructor", + "prototype", + "toString", + ])("treats the reserved property name %s as an ordinary undeclared bucket", (name) => { + const storage = createModuleStorage({ + bucketPrefix: "ekklesia", + logicalBucketIds: ["assets"], + moduleId: "announcements", + provider: provider(), + }); + + expect(() => storage.bucket(name)).toThrow(ModuleStorageError); + }); +}); + +describe("bucket provisioning", () => { + it("ensures the bucket lazily on first use, not at construction", async () => { + const shared = fakeProvider(); + const storage = createModuleStorage({ + bucketPrefix: "ekklesia", + logicalBucketIds: ["assets"], + moduleId: "announcements", + provider: shared, + }); + + // Constructing the facade — which is what startup does — touches the provider zero times, so a + // storage outage can never stop the platform from booting. + expect(shared.calls).toHaveLength(0); + + await storage.bucket("assets").list(); + expect(shared.calls.filter((call) => call.operation === "ensureBucket")).toHaveLength(1); + }); + + it("ensures at most once per binding across many operations", async () => { + const shared = fakeProvider(); + const bucket = createModuleStorage({ + bucketPrefix: "ekklesia", + logicalBucketIds: ["assets"], + moduleId: "announcements", + provider: shared, + }).bucket("assets"); + + await Promise.all([bucket.list(), bucket.list(), bucket.put("a.txt", "a")]); + await bucket.list(); + + expect(shared.calls.filter((call) => call.operation === "ensureBucket")).toHaveLength(1); + }); + + it("retries provisioning after a transient failure rather than poisoning the binding", async () => { + let failures = 1; + const shared = fakeProvider({ + failEnsure: () => + failures-- > 0 ? new ModuleStorageError("STORAGE_PROVIDER_UNAVAILABLE") : undefined!, + }); + const bucket = createModuleStorage({ + bucketPrefix: "ekklesia", + logicalBucketIds: ["assets"], + moduleId: "announcements", + provider: shared, + }).bucket("assets"); + + await expect(bucket.list()).rejects.toMatchObject({ code: "STORAGE_PROVIDER_UNAVAILABLE" }); + // The next call tries again. A memoized *failure* would leave a module permanently broken by one + // network blip, which is exactly the resilience the real Announcements product depends on. + await expect(bucket.list()).resolves.toEqual({ objects: [] }); + }); + + it("never offers a way to delete a bucket", () => { + const storage = createModuleStorage({ + bucketPrefix: "ekklesia", + logicalBucketIds: ["assets"], + moduleId: "announcements", + provider: fakeProvider(), + }); + + const surface = [...Object.keys(storage), ...Object.keys(storage.bucket("assets"))]; + expect(surface.join(" ")).not.toMatch(/deleteBucket|destroy|drop|removeBucket/iu); + }); +}); + +describe("storage operations through the scoped facade", () => { + async function announcementsStorage() { + const provider = fakeProvider(); + const storage = createModuleStorage({ + bucketPrefix: "ekklesia", + logicalBucketIds: ["announcement-assets"], + moduleId: "announcements", + provider, + }); + return { bucket: storage.bucket("announcement-assets"), provider }; + } + + it("round-trips an object and preserves its content type", async () => { + const { bucket } = await announcementsStorage(); + const bytes = new Uint8Array([0x89, 0x50, 0x4e, 0x47]); + + await bucket.put("slides/welcome.png", bytes, { contentType: "image/png" }); + const object = await bucket.get("slides/welcome.png"); + + expect(object.contentType).toBe("image/png"); + expect(await collect(object.body)).toEqual(bytes); + }); + + it("lists with a prefix and deletes", async () => { + const { bucket } = await announcementsStorage(); + await bucket.put("slides/a.png", "a"); + await bucket.put("slides/b.png", "b"); + await bucket.put("other/c.png", "c"); + + expect((await bucket.list({ prefix: "slides/" })).objects.map((o) => o.key)).toEqual([ + "slides/a.png", + "slides/b.png", + ]); + + await bucket.delete("slides/a.png"); + expect((await bucket.list({ prefix: "slides/" })).objects.map((o) => o.key)).toEqual([ + "slides/b.png", + ]); + }); + + it("reports a missing object as a structured not-found error", async () => { + const { bucket } = await announcementsStorage(); + await bucket.put("present.png", "x"); + + await expect(bucket.get("absent.png")).rejects.toMatchObject({ + code: "STORAGE_OBJECT_NOT_FOUND", + key: "absent.png", + }); + await expect(bucket.head("absent.png")).rejects.toMatchObject({ + code: "STORAGE_OBJECT_NOT_FOUND", + }); + }); + + it("rejects an invalid key before reaching the provider", async () => { + const { bucket, provider } = await announcementsStorage(); + + await expect(bucket.get("/leading-slash.png")).rejects.toMatchObject({ + code: "STORAGE_KEY_INVALID", + }); + // Not even the bucket was ensured: validation happens first. + expect(provider.calls).toHaveLength(0); + }); + + it("refuses a streamed body with no content length rather than buffering it", async () => { + const { bucket } = await announcementsStorage(); + const stream = streamOf(new TextEncoder().encode("x")); + + await expect(bucket.put("streamed.bin", stream)).rejects.toMatchObject({ + code: "STORAGE_CONTENT_LENGTH_REQUIRED", + }); + }); + + it("accepts a streamed body when the length is supplied", async () => { + const { bucket } = await announcementsStorage(); + const stream = streamOf(new TextEncoder().encode("hello")); + + const metadata = await bucket.put("streamed.bin", stream, { contentLength: 5 }); + expect(metadata.key).toBe("streamed.bin"); + }); +}); + +describe("storage provider configuration", () => { + // Distinctive values, deliberately: a generic `"key"` would make a "no credential leaked" + // assertion pass or fail on whether a fixed error message happens to use the English word. + const base = { + accessKey: "AKIAEXAMPLEACCESSID", + endpoint: "https://storage.example.test", + secretKey: "sk-example-secret-material", + }; + + it("treats an absent endpoint as an installation with no object storage", () => { + expect(createStorageProviderConfig(undefined)).toBeUndefined(); + expect(createStorageProviderConfig({ endpoint: "" })).toBeUndefined(); + expect(createStorageProviderConfig({})).toBeUndefined(); + }); + + it("accepts a valid S3-compatible configuration with safe defaults", () => { + const config = createStorageProviderConfig(base); + + expect(config).toEqual({ + accessKey: "AKIAEXAMPLEACCESSID", + bucketPrefix: "ekklesia", + endpoint: "https://storage.example.test", + // Path-style by default: every self-hosted S3-compatible service needs it, and virtual-host + // addressing would require wildcard DNS a typical deployment does not have. + forcePathStyle: true, + region: "us-east-1", + secretKey: "sk-example-secret-material", + }); + }); + + it("requires both credentials once an endpoint is set", () => { + expect(() => createStorageProviderConfig({ ...base, accessKey: undefined })).toThrow( + expect.objectContaining({ code: "STORAGE_ACCESS_KEY_MISSING" }), + ); + expect(() => createStorageProviderConfig({ ...base, secretKey: " " })).toThrow( + expect.objectContaining({ code: "STORAGE_SECRET_KEY_MISSING" }), + ); + }); + + it("rejects an endpoint that embeds credentials", () => { + expect(() => + createStorageProviderConfig({ ...base, endpoint: "https://key:secret@storage.example.test" }), + ).toThrow(expect.objectContaining({ code: "STORAGE_ENDPOINT_WITH_CREDENTIALS" })); + }); + + it.each([ + "not-a-url", + "ftp://storage.example.test", + "//storage.example.test", + ])("rejects the malformed endpoint %s", (endpoint) => { + expect(() => createStorageProviderConfig({ ...base, endpoint })).toThrow( + expect.objectContaining({ code: "STORAGE_ENDPOINT_INVALID" }), + ); + }); + + it("requires https in production but allows http in development", () => { + const http = { ...base, endpoint: "http://127.0.0.1:8333" }; + + expect(() => createStorageProviderConfig(http, { environment: "production" })).toThrow( + expect.objectContaining({ code: "STORAGE_ENDPOINT_INSECURE_IN_PRODUCTION" }), + ); + expect(createStorageProviderConfig(http, { environment: "development" })?.endpoint).toBe( + "http://127.0.0.1:8333", + ); + }); + + it("rejects an unsafe bucket prefix", () => { + expect(() => createStorageProviderConfig({ ...base, bucketPrefix: "Not Valid" })).toThrow( + expect.objectContaining({ code: "STORAGE_BUCKET_PREFIX_INVALID" }), + ); + }); + + it("never carries a configured value in a configuration error", () => { + let captured: unknown; + try { + createStorageProviderConfig({ + ...base, + endpoint: "https://super-secret-host.invalid", + secretKey: undefined, + }); + } catch (error) { + captured = error; + } + + const rendered = `${String(captured)} ${JSON.stringify(captured)}`; + // No configured *value* survives: not the endpoint, not either credential. (The fixed message + // legitimately contains the word "key" — what must never appear is a key's value.) + expect(rendered).not.toContain("super-secret-host.invalid"); + expect(rendered).not.toContain(base.accessKey); + expect(rendered).not.toContain(base.secretKey); + }); + + it("describes the provider for an operator without printing a credential", () => { + const config = createStorageProviderConfig({ + ...base, + accessKey: "AKIAVERYSECRETKEY", + endpoint: "https://storage.example.test/ignored-path", + secretKey: "a-very-secret-value", + }); + if (!config) { + throw new Error("expected a configuration"); + } + + const description = describeStorageProvider(config); + expect(description).toContain("https://storage.example.test"); + expect(description).toContain("path-style"); + expect(description).not.toContain("AKIAVERYSECRETKEY"); + expect(description).not.toContain("a-very-secret-value"); + }); +}); + +/** + * The Announcements-shaped proof. + * + * Modelled on the real product's requirements — a bucket of announcement images, listed and read — + * and never importing it. The point is what the fixture *cannot* see: it names a logical bucket and + * object keys, and has no access to SeaweedFS, an endpoint, an access key, or the physical bucket. + */ +describe("an Announcements-shaped module reads its slides without knowing any infrastructure", () => { + /** Everything this fixture knows. Compare with the real product, which today hardcodes + * `http://10.40.0.2:9000/announcements?list-type=2` and parses S3 XML by hand. */ + const LOGICAL_BUCKET = "announcement-assets"; + + async function loadSlideKeys(storage: { + bucket: (id: string) => { + list: (options?: ModuleStorageListOptions) => Promise; + }; + }): Promise { + const listing = await storage.bucket(LOGICAL_BUCKET).list(); + // Extension filtering is the module's own domain logic, exactly as it is today — Core has no + // opinion about which objects are slides. + return listing.objects + .map((object) => object.key) + .filter((key) => /\.(png|jpe?g|webp)$/iu.test(key)); + } + + it("lists and reads its own images through the logical bucket alone", async () => { + const provider = fakeProvider(); + const storage = createModuleStorage({ + bucketPrefix: "ekklesia", + logicalBucketIds: [LOGICAL_BUCKET], + moduleId: "announcements", + provider, + }); + const bucket = storage.bucket(LOGICAL_BUCKET); + await bucket.put("welcome.png", new Uint8Array([1, 2, 3]), { contentType: "image/png" }); + await bucket.put("notes.txt", "not a slide"); + + expect(await loadSlideKeys(storage)).toEqual(["welcome.png"]); + + const slide = await bucket.get("welcome.png"); + expect(slide.contentType).toBe("image/png"); + // Straight into a `Response` — the shape a module route uses to serve an image to a browser + // without the browser ever learning that object storage exists. + // The shape a module route uses: the stream and its content type go straight into a `Response`, + // so a browser receives the image without ever learning that object storage exists. + const headers = new Headers(); + if (slide.contentType) { + headers.set("content-type", slide.contentType); + } + expect(headers.get("content-type")).toBe("image/png"); + expect(await collect(slide.body)).toEqual(new Uint8Array([1, 2, 3])); + }); + + it("knows the logical bucket and object keys, and nothing about the host", async () => { + const provider = fakeProvider(); + const storage = createModuleStorage({ + bucketPrefix: "ekklesia", + logicalBucketIds: [LOGICAL_BUCKET], + moduleId: "announcements", + provider, + }); + await storage.bucket(LOGICAL_BUCKET).list(); + + const reachable = JSON.stringify({ + bucketHandle: storage.bucket(LOGICAL_BUCKET), + buckets: storage.buckets(), + storage, + }); + for (const infrastructure of [ + "seaweed", + "SeaweedFS", + "10.40.0.2", + "9000", + "8333", + "s3", + "S3", + "accessKey", + "secretKey", + physicalBucketName("ekklesia", "announcements", LOGICAL_BUCKET), + ]) { + expect(reachable).not.toContain(infrastructure); + } + // The physical bucket exists — the host created it — but only the provider ever saw its name. + expect([...provider.buckets.keys()]).toEqual([ + physicalBucketName("ekklesia", "announcements", LOGICAL_BUCKET), + ]); + }); + + it("surfaces a storage outage to the module without any lifecycle state changing", async () => { + const provider = fakeProvider({ + failEnsure: () => new ModuleStorageError("STORAGE_PROVIDER_UNAVAILABLE"), + }); + const bucket = createModuleStorage({ + bucketPrefix: "ekklesia", + logicalBucketIds: [LOGICAL_BUCKET], + moduleId: "announcements", + provider, + }).bucket(LOGICAL_BUCKET); + + // Exactly what the real product does today: catch, keep the previous slides, and retry on the + // next poll. Nothing here writes anything, anywhere — a transient outage is not an + // administrative decision (ADR 0030). + let caught: unknown; + try { + await bucket.list(); + } catch (error) { + caught = error; + } + + expect(caught).toBeInstanceOf(ModuleStorageError); + expect((caught as ModuleStorageError).code).toBe("STORAGE_PROVIDER_UNAVAILABLE"); + expect(JSON.stringify(caught)).not.toContain("http"); + }); +}); diff --git a/docs/architecture/decisions/0028-module-operational-lifecycle.md b/docs/architecture/decisions/0028-module-operational-lifecycle.md index 24df2f4..4fafb70 100644 --- a/docs/architecture/decisions/0028-module-operational-lifecycle.md +++ b/docs/architecture/decisions/0028-module-operational-lifecycle.md @@ -175,6 +175,12 @@ shape as `pnpm authorization:bootstrap-owner`: explicit, never HTTP-exposed, nev > discriminated union so a configuration reason names a **key** where a dependency reason names a > dependency. Everything else here is unchanged: startup atomicity, the absence of a `failed` state, > the default-enabled policy, and the dependency propagation those new reasons flow through. +> +> **See also [ADR 0030](0030-host-provided-module-object-storage.md)**, which adds one further +> reason, `storage_unavailable`, for a module whose declared logical object-storage bucket this host +> cannot bind. It deliberately reports a *binding* problem and never a *health* one: a configured +> provider that is briefly unreachable remains an operational condition a module retries through, +> exactly as this ADR established for external resources generally. There is consequently **no `failed` runtime state**, because no live process could ever report one. Naming a state nothing can produce is exactly the mistake `MODULE_LIFECYCLE_STAGES` made. It is the diff --git a/docs/architecture/decisions/0029-module-configuration-and-secrets.md b/docs/architecture/decisions/0029-module-configuration-and-secrets.md index 3b4b2f1..2bb100e 100644 --- a/docs/architecture/decisions/0029-module-configuration-and-secrets.md +++ b/docs/architecture/decisions/0029-module-configuration-and-secrets.md @@ -303,6 +303,13 @@ already declares logical buckets: when the storage service exists, the bucket bi there and the configuration key should be retired. That overlap is recorded here deliberately so it is resolved by decision rather than by accident. +> **Cross-reference — see [ADR 0030](0030-host-provided-module-object-storage.md).** +> That overlap is now resolved, in the direction this section anticipated. `manifest.storage.buckets` +> is the logical storage requirement, Core provides a host-managed binding for it, and the +> transitional `bucket` *configuration* key is retired: a logical bucket is a static declaration the +> platform validates before any module code runs, not a value an operator types. Everything else here +> is unchanged. + ## Validated against the real product Every configuration-shaped literal in Announcements at commit `9aab486`, classified: diff --git a/docs/architecture/decisions/0030-host-provided-module-object-storage.md b/docs/architecture/decisions/0030-host-provided-module-object-storage.md new file mode 100644 index 0000000..46a4963 --- /dev/null +++ b/docs/architecture/decisions/0030-host-provided-module-object-storage.md @@ -0,0 +1,307 @@ +# 0030: Modules declare logical buckets; hosts provide object storage + +- Status: Accepted +- Date: 2026-08-23 +- Extends: [0004](0004-replace-unmaintained-minio.md), + [0016](0016-module-database-access.md), + [0021](0021-modules-as-hostable-products.md), + [0028](0028-module-operational-lifecycle.md), + [0029](0029-module-configuration-and-secrets.md) + +## Context + +`manifest.storage.buckets` has existed since manifest version 1 and meant nothing. A module could +declare `{ "id": "records" }`, discovery would validate the identifier and reject duplicates, and +then nothing anywhere read the field. Doctor said so plainly: *"Core storage integration has not +been implemented yet."* + +Meanwhile the product that will become Ekklesia's first storage consumer does this +(`The_River/Announcements`, commit `9aab486`): + +```js +const response = await fetch("http://10.40.0.2:9000/announcements?list-type=2"); +const data = new XMLParser().parse(await response.text()); +slides = (data.ListBucketResult?.Contents ?? []) + .map((file) => file.Key) + .filter((name) => /\.(png|jpg|jpeg|webp)$/i.test(name)) + .map((name) => `http://10.40.0.2:9000/announcements/${encode(name)}`); +``` + +A hardcoded LAN address, a hand-rolled `ListObjectsV2` XML parse, an anonymously readable bucket, +and direct S3 URLs handed to the browser. Every one of those has to disappear, and none of them can +be replaced with "let the module configure its own S3 endpoint" — that would make every module a +separate S3 deployment decision and reintroduce the coupling in a tidier costume. + +Two facts from the repository shaped what was possible. + +**The provider question was already decided.** ADR 0004 replaced the archived MinIO baseline with +SeaweedFS 4.28 in the development Compose stack, deliberately using `S3_*` configuration names +rather than product-specific ones, and recorded that *"provider-specific Core APIs were rejected +because they would couple every module to this choice."* The service is already pinned, already +provisioned, and already has credentials in `.env.example`. + +**ADR 0029 left a loose end and named it.** It accepted a `bucket` *configuration key* as a +transitional answer while observing that `manifest.storage.buckets` already existed for exactly this +purpose, and asked for the overlap to be resolved rather than left to drift. This ADR resolves it. + +## Decision + +### The logical/physical split, and which side owns what + +```text +a module owns I require logical bucket "announcement-assets" +a host owns endpoint · region · credentials · path style · TLS · physical bucket naming +``` + +That line is the whole design. A module names a bucket; the host decides what a bucket *is*. The +module-facing contract (`packages/contracts/src/module-storage.ts`) contains no endpoint, no region, +no credential, no signing, no `S3`, and no `SeaweedFS` — not as a field, not as a type, not as a +string. A future standalone host constructs the same interface over its own SeaweedFS, a shared one, +MinIO, or AWS S3, and the module cannot tell. + +### `manifest.storage.buckets` becomes a real requirement, and the manifest does not change + +Declaring a bucket now means "I require this". No new field, no new version — **manifest version 5 +is unchanged**. The existing declaration already expressed the requirement precisely; it simply had +no consumer. + +Every declared bucket is required. A separate `required: false` would have meant a manifest version +bump (`moduleBucketDeclarationSchema` is `.strict()`) for a case no module has, and "declare it if +you need it" is a simpler rule than one with an escape hatch. + +**This retires ADR 0029's transitional `bucket` configuration key.** A logical bucket is a static +declaration the platform can validate before any module code runs, not a value an operator types. +Module configuration is for things an operator decides; storage requirements are for things a module +knows about itself. + +The template consequently declares **no** bucket any more. A starting point that could not run +without S3 configured would be a poor starting point, so storage is something a module author opts +into deliberately. + +### Physical naming: deterministic, collision-safe, and never module-visible + +```text +ekklesia-announcements--announcement-assets-3f0c1d9a6b21 +└─prefix─┘└──── module--bucket (truncated) ────┘└─digest─┘ +``` + +Three properties, each load-bearing: + +**`--` as the separator, because it cannot occur inside either identifier.** Both `moduleIdSchema` +and `logicalBucketIdSchema` require `[a-z0-9]` after every hyphen, so a doubled hyphen is +unrepresentable in each half. A single hyphen would have been ambiguous in precisely the way that +matters: module `a-b` + bucket `c` and module `a` + bucket `b-c` would both render `a-b-c`, silently +handing two unrelated modules one namespace. + +**A digest over the full, untruncated identity.** S3 caps bucket names at 63 characters while +Ekklesia identifiers allow 64 + 48, so the readable middle is truncated. The digest is taken *before* +truncation, so two modules whose names truncate identically still get different buckets. + +**A configurable prefix** (`S3_BUCKET_PREFIX`, default `ekklesia`), so several Ekklesia-compatible +deployments can share one S3-compatible service — which is exactly the standalone story: one +SeaweedFS holding Announcements, Video, Sermons, and Signage, isolated by binding rather than by +running four storage servers. + +The result is never exposed to module code. `ModuleStorageBucket.id` is the **logical** id, and there +is no accessor for anything else. + +### The module-facing API, kept small + +```ts +const assets = context.storage.bucket("announcement-assets"); +await assets.list({ prefix: "slides/" }); +await assets.get("slides/welcome.png"); // → { body: ReadableStream, contentType, ... } +await assets.put("slides/welcome.png", bytes, { contentType: "image/png" }); +await assets.head("slides/welcome.png"); +await assets.delete("slides/welcome.png"); +``` + +Five operations, chosen against what Announcements actually does — list a bucket, read images — plus +the writes any real product needs. There is deliberately no multipart upload, no lifecycle rule, no +replication, no ACL or bucket-policy management, and **no bucket creation or deletion**: which +buckets exist follows from an installed manifest, not from running module code. + +`get` returns a Web `ReadableStream`, so an object goes straight into a `Response` without this +process holding it in memory. `put` accepts a stream too, but requires `contentLength` alongside one +— every S3-compatible provider needs the length up front for a single-part upload, and inferring it +would mean buffering the very thing the stream exists to avoid. Refusing loudly beats discovering it +through production memory pressure. + +Metadata is `contentType`, `contentLength`, `etag`, `lastModified` — the four fields every +S3-compatible provider actually returns, not a mirror of S3's metadata surface. Content type matters +because Announcements serves images and must set a correct header without sniffing bytes. + +Object keys allow `/` (modules genuinely use `slides/welcome.png`) and reject what would be +*ambiguous* rather than dangerous: empty, over 1024 UTF-8 bytes, control characters, lone surrogates, +backslashes, and a leading `/`. A key cannot escape its bucket in any case — the physical bucket +comes from the binding, never from the key. + +### Errors are a closed set, and nothing from the provider survives + +`ModuleStorageError` with eight codes (`STORAGE_OBJECT_NOT_FOUND`, `STORAGE_PROVIDER_UNAVAILABLE`, +`STORAGE_OPERATION_DENIED`, …). No message, request id, XML body, bucket name, endpoint, or caught +cause crosses the translation boundary. Provider errors routinely carry the host and sometimes the +signed request, and a module is entitled to log whatever it catches, so this is the only place that +boundary can be held. The one echoed value is the module's own object key. + +### `context.storage`, beside `services` rather than inside it + +Storage sits on `ModuleRuntimeContext` and `ModuleRouteHandlerContext` — **not** on +`ModuleRegistrationContext`, exactly like `services`, because it performs network I/O and +registration stays declarative. (`config` is on registration precisely because it is the opposite: a +resolved in-memory snapshot.) + +It is emphatically not a field on `ModuleRuntimeServices`. That object is deliberately **one shared +identity every module receives**, a property the runtime has a test for; a storage facade is scoped +to one module's buckets. Putting it there would have forced one of those two properties to give, and +would have handed every module every other module's storage. + +### Startup performs no storage I/O, and bucket creation is lazy + +Constructing the provider opens no connection. Buckets are ensured on **first use** of a binding, +memoized per binding so it happens once per process — and a *failed* attempt is discarded, so a +transient outage does not poison the binding for the life of the process. + +Eager provisioning at startup was rejected, and this was the sharpest decision in the pass. If +Ekklesia ensured buckets during boot, a storage service that was briefly down would stop the whole +platform from starting — strictly worse than the product being replaced, whose slideshow survives its +object store vanishing. The platform must not become less resilient because storage got abstracted. + +Creation is idempotent and destruction does not exist. Nothing deletes a bucket: not disabling a +module, not removing one, not emptying it. + +### Lifecycle: binding availability, never provider health + +One new blocked reason, `storage_unavailable`, meaning *this module declares a logical bucket and +this host has no storage provider configured*. It names the logical bucket and nothing else. + +```text +enabled → storage bindable? → configuration valid? → dependencies eligible? → runs +``` + +Resolution stays **pure and synchronous**. It asks whether storage is *configured*, never whether the +service answered — so eligibility can never depend on a probe that happened to time out during boot. +A storage-blocked module propagates to its required dependents through ADR 0028's existing mechanism +rather than a parallel one. + +The distinction this preserves is the one the real product already proved matters: + +| situation | classification | +| --- | --- | +| no provider configured | lifecycle problem — module blocked | +| required bucket cannot be bound | lifecycle problem — module blocked | +| configured provider times out at 10:15 | operational error the module catches and retries | + +`running` still means the `start` hook completed, never that every external resource stays healthy. +A transient outage writes nothing, anywhere. + +### Host configuration, using the names ADR 0004 already chose + +`S3_ENDPOINT`, `S3_ACCESS_KEY`, `S3_SECRET_KEY`, `S3_REGION`, `S3_FORCE_PATH_STYLE`, and the new +`S3_BUCKET_PREFIX` — read only by the executable entry point, validated into a typed config, and +never seen by a module, a module UI, a browser bundle, a diagnostic, or an API response. + +Storage is **optional**. No `S3_ENDPOINT` means no provider, which is a complete deployment: modules +that declare no bucket are unaffected. Once an endpoint *is* set, credentials become required — a +half-configured provider silently attempting anonymous access would be a worse failure than a loud +one. Production requires `https` on the same reasoning the database transport requires verified TLS; +an endpoint embedding userinfo is rejected outright rather than stripped. + +Credentials are never persisted in PostgreSQL and never routed through module configuration +(ADR 0029), which has no encryption at rest to offer them. + +### The provider: S3-compatible, via the AWS SDK, not hand-rolled + +`@aws-sdk/client-s3` addressing an arbitrary endpoint with path-style URLs. Hand-rolling SigV4 for a +security-sensitive path was rejected; the SDK is the mature, maintained implementation and works +against any S3-compatible service. + +Probing the pinned SeaweedFS image directly, rather than trusting AWS documentation, found a real +divergence: **`CreateBucket` on an existing bucket reports `BucketAlreadyExists`, not AWS's +`BucketAlreadyOwnedByYou`.** A provider treating only the AWS spelling as success would have failed +on every second boot against this repository's own development storage. Both are accepted, and a live +test pins it. This is exactly the provider-quirk risk ADR 0004 anticipated. + +### Browser delivery goes through the module's own routes + +```text +browser → module API route → context.storage → object bytes +``` + +No S3 credentials reach the browser, no provider client is bundled for it, and `ModuleUiHostContext` +is unchanged. A module streams an object into a `Response` from its own authorized route. + +This is a genuine behavioural change for Announcements, which today hands the browser direct S3 URLs. +Proxying costs a hop and makes the module API the media path; signed URLs and public-object routing +are the follow-up design, deliberately not built here without a real requirement to shape them. + +### Standalone → Ekklesia migration: the seam, not the engine + +Core provides list, read, and write. **A module owns semantic migration**, because only the module +knows which objects are canonical, which are generated, whether database references need updating, +and how to verify success. Core will not inspect Announcements filenames and guess. + +```text +copy → verify → switch binding/state → verify → optionally remove old data later +``` + +Nothing in this pass deletes source data, and nothing ever should implicitly. + +### Storage unavailable never means "start your own" + +A module running inside Ekklesia that finds storage unconfigured is **blocked with a clear +diagnostic**, never permitted to launch its own storage server. A future standalone deployment +wrapper may bundle SeaweedFS as a convenience — that is deployment composition, not module runtime +behaviour, and the distinction must not blur. + +## Alternatives + +- **Per-module `storageEndpoint` / `storageAccessKey` configuration.** Rejected — the mistake this + ADR exists to prevent. Every module would become a separate S3 deployment decision. +- **A new manifest field or version.** Rejected: `manifest.storage.buckets` already expressed the + requirement exactly; it only lacked a consumer. +- **Keeping ADR 0029's `bucket` configuration key.** Retired: two mechanisms for one fact. +- **Handing modules a raw bucket name or provider client.** Rejected: it would let any module open + any bucket and would weld every module to S3. +- **A single hyphen between module and bucket.** Rejected: demonstrably ambiguous, and the ambiguity + silently merges two modules' namespaces. +- **Hashing without a readable portion.** Rejected: an operator debugging a permissions problem needs + to recognize the bucket. +- **Eager bucket provisioning at startup.** Rejected: it would make a storage outage a platform + outage, less resilient than the product being replaced. +- **Treating provider unreachability as a lifecycle state.** Rejected explicitly: it would mark + running modules blocked over a network blip and destroy the distinction ADR 0028 established. +- **Hand-rolled SigV4.** Rejected for a security-sensitive path with a mature alternative. +- **Storage on `ModuleRuntimeServices`.** Rejected: `services` is one shared object; storage is + per-module. +- **Signed URLs / public object routing now.** Deferred: no requirement yet exists to shape them, and + guessing would produce a contract we would have to break. +- **Range requests in `get`.** Deferred: an additive option, cheap to add when video needs seeking, + and outside "keep the first contract small". +- **Naming the contract after SeaweedFS.** Rejected by ADR 0004 before this pass began. + +## Consequences + +`manifest.storage.buckets` means something for the first time since it was introduced. A module asks +for a logical bucket and gets isolated, host-managed storage; the host decides what backs it; and +Announcements has a path away from `http://10.40.0.2:9000` that does not replace one hardcoded +address with a configurable one. + +Core gains one dependency (`@aws-sdk/client-s3`, server-only), one new export path +(`@ekklesia/core/storage`), one blocked reason, one Doctor check, and a required `moduleStorage` +option on `createApiApp`. The template no longer declares a bucket. The live integration harness now +brings up the Compose file's already-pinned SeaweedFS alongside PostgreSQL. + +Costs worth naming. Object delivery to browsers now goes through module routes, which is a real +change in shape for Announcements and will want signed URLs or caching before large media. The +provider is **not** "production-ready" by ADR 0004's own bar — that list includes ranges, multipart, +and signed URLs, none of which exist here. And bucket bindings are platform-global, needing a scope +column whenever an organization model arrives, like every other Core table. + +## Deferred + +Not implemented or promised here: signed URLs, public object routing, or a CDN; range requests; +multipart upload; object lifecycle rules, replication, or bucket policies; a storage settings +Dashboard; per-organization storage; provider administration or SeaweedFS process management; the +standalone host and its Compose stack; automatic standalone → Ekklesia migration and any migration +UI; and the Announcements migration itself. diff --git a/docs/architecture/decisions/README.md b/docs/architecture/decisions/README.md index dcaa709..bfeaafd 100644 --- a/docs/architecture/decisions/README.md +++ b/docs/architecture/decisions/README.md @@ -38,3 +38,4 @@ ADRs capture durable choices that affect more than one package or phase. - [0027: Dashboard navigation is presentation configuration, and modules stay flat](0027-dashboard-navigation-and-flat-modules.md) — Accepted - [0028: Module operational lifecycle — enablement, eligibility, and what "running" means](0028-module-operational-lifecycle.md) — Accepted - [0029: Module configuration is manifest-declared, host-resolved, and secret-free at rest](0029-module-configuration-and-secrets.md) — Accepted +- [0030: Modules declare logical buckets; hosts provide object storage](0030-host-provided-module-object-storage.md) — Accepted diff --git a/docs/architecture/platform-architecture.md b/docs/architecture/platform-architecture.md index 28e19a3..0a21a81 100644 --- a/docs/architecture/platform-architecture.md +++ b/docs/architecture/platform-architecture.md @@ -356,3 +356,29 @@ is hot-reloaded. Migration discovery remains independent of configuration validi independent of enabled state. Endpoints and credentials for shared infrastructure stay platform concerns — a module declares which bucket it needs, not an S3 endpoint — leaving room for the object storage service Core still does not have. There is no settings UI. + +Phase 7A pass 3 turns `manifest.storage.buckets` — inert since manifest version 1 — into a real +host-provided capability ([ADR 0030](decisions/0030-host-provided-module-object-storage.md)). A module +declares a **logical** bucket and receives a scoped facade at `context.storage` offering list, head, +get, put, and delete, with `get` returning a Web stream so an object never has to be buffered. The +contract names no provider: there is no endpoint, region, credential, or physical bucket reachable +from it, which is what lets the same module run under a future standalone host backed by its own or a +shared S3-compatible service. + +The host owns everything else. Core's first provider speaks the S3-compatible protocol through the +AWS SDK against a configured endpoint, with SeaweedFS as the intended deployment target (ADR 0004) but +never as an API name. A logical bucket maps to a physical one deterministically — a configurable +installation prefix, the module and bucket ids joined by a separator that cannot occur inside either, +and a digest over the full identity — so two modules declaring `assets` are always isolated and one +storage service can host several installations. Bucket creation is idempotent and lazy on first use; +nothing ever deletes a bucket. + +Storage is optional: an installation without a provider is complete, and only a module that *declares* +a bucket is affected — it is reported `blocked · storage_unavailable` through ADR 0028's existing +resolver, propagating to required dependents. Crucially, eligibility asks whether storage is +**configured**, never whether it answered: a configured provider that is briefly unreachable stays an +operational error a module retries through, never a lifecycle state, so the platform is no less +resilient than the products it will absorb. Startup performs no storage I/O at all. Credentials are +deployment-supplied, never persisted, never routed through module configuration, and never visible to +modules, module UI, diagnostics, or API responses. Browsers reach objects through a module's own +routes; signed URLs and public object delivery are deferred. diff --git a/docs/development/api.md b/docs/development/api.md index b548a74..099cbb1 100644 --- a/docs/development/api.md +++ b/docs/development/api.md @@ -389,9 +389,11 @@ account. 3. Construct the module database facade and the frozen `ModuleRuntimeServices` object from that same pool (see [Module database access](#module-database-access)). 4. Construct the Core identity runtime, authorization store, and evaluator. -5. Read the stored module enablement configuration and then the stored module configuration, and - resolve, over that exact same discovered inventory, which modules are **eligible** to run and - what each eligible module's frozen configuration snapshot is (`resolveModuleEligibility`, +5. Construct the host object-storage provider from validated configuration, if this installation has + one — building the client opens no connection, so a storage outage can never stop the platform + from booting (ADR 0030). Then read the stored module enablement configuration and the stored module + configuration, and resolve, over that exact same discovered inventory, which modules are + **eligible** to run and what each eligible module's frozen configuration snapshot is (`resolveModuleEligibility`, `@ekklesia/core/modules` — see [Module lifecycle](#module-lifecycle) and [Module configuration](#module-configuration)). The two reads are sequential rather than raced so a broken deployment reports the same error code every time. @@ -664,6 +666,66 @@ different authorities. Mutations additionally require an exact trusted `Origin` No response carries a filesystem path, package name, manifest path, caught error, stack, or secret value. +## Module object storage + +Ekklesia can provide modules with logical object storage backed by any S3-compatible service +([ADR 0030](../architecture/decisions/0030-host-provided-module-object-storage.md)). There is **no +HTTP API for it**: storage is a runtime capability modules use, not a resource the platform exposes, +and no route returns a provider endpoint or credential. + +### Host configuration + +| variable | meaning | +| --- | --- | +| `S3_ENDPOINT` | Absolute `http`/`https` URL. **Absent means this installation has no object storage.** | +| `S3_ACCESS_KEY`, `S3_SECRET_KEY` | Required once an endpoint is set | +| `S3_REGION` | Defaults to `us-east-1` | +| `S3_FORCE_PATH_STYLE` | Defaults to `true`; self-hosted services need it | +| `S3_BUCKET_PREFIX` | Defaults to `ekklesia`; namespaces this installation on a shared service | + +Production requires `https`, exactly as the database transport requires verified TLS. An endpoint +embedding credentials is rejected rather than stripped. These values reach Core's provider and +nothing else — never a module, module UI, browser bundle, diagnostic, or API response. + +The repository's development Compose stack already runs SeaweedFS for this, and `.env.example` +already carries matching values. + +### Logical buckets, and what modules get + +A module declares logical buckets in its manifest (`storage.buckets`) and receives a scoped facade at +`context.storage` on its runtime and HTTP handler contexts. It names a logical bucket; the host maps +it to a physical one deterministically and never exposes that name. + +An installation with **no** provider is a complete deployment: modules declaring no bucket run +normally. A module that *does* declare one is reported `blocked · storage_unavailable` on +`GET /api/v1/module-lifecycle`, naming the logical bucket: + +```json +{ "runtime": { "state": "blocked", "reason": "storage_unavailable", + "storageBucketId": "announcement-assets" } } +``` + +Its required dependents block behind it through the ordinary dependency mechanism. + +### Binding availability is not provider health + +Eligibility asks whether storage is **configured**, never whether the service answered. A configured +provider that is temporarily unreachable is an operational error the module catches and retries — +`running` still means the `start` hook completed, and no lifecycle state changes because a storage +call timed out. Buckets are created lazily on first use rather than at startup, so a storage outage +never prevents the platform from booting, and nothing ever deletes a bucket. + +### Serving objects to browsers + +There are no signed URLs and no public object routing. A module streams objects through its own +authorized route: + +```text +browser → GET /api/v1/modules//assets/ → context.storage → object bytes +``` + +which keeps every S3 concern server-side and gives module UI no credentials. + ## Module HTTP routes An installed module's routes (declared via `http.route(...)` in its `register` hook — see diff --git a/docs/development/security.md b/docs/development/security.md index 7fad83e..3e1e052 100644 --- a/docs/development/security.md +++ b/docs/development/security.md @@ -245,6 +245,44 @@ Reading or changing configuration requires `core.modules.configuration.manage` ( `Origin` and a bounded body, like every other Core mutation. Module UI receives no configuration at all in this phase. +### Object storage is scoped by binding, and modules never hold credentials + +A module declares a logical bucket; the host resolves it to a physical bucket on an S3-compatible +service and hands the module a facade scoped to exactly that +([ADR 0030](../architecture/decisions/0030-host-provided-module-object-storage.md)). + +What that gives, structurally rather than by discipline: + +- **No credential reaches a module.** The endpoint, region, access key, and secret key live in the + host's validated configuration and reach Core's provider only. `ModuleStorage` has no accessor that + could return any of them, and none appears in any error, diagnostic, or API response. +- **No module can open another module's bucket.** The facade closes over exactly the bindings that + module's manifest declared. Knowing another module's bucket name, module id, or physical bucket + name buys nothing, because none of those is an input to anything — an undeclared name is simply not + a key in the map. +- **No module chooses a physical bucket or an endpoint.** Both come from host configuration and the + deterministic binding, never from module input, so a module cannot aim the platform's S3 client at + a server of its choosing. +- **Two modules declaring the same logical name are isolated.** Physical names are derived from + module id *and* bucket id with a separator that cannot occur inside either, plus a digest over the + full identity — so `assets` in two modules is two namespaces, not one. +- **Object keys cannot escape a bucket**, because the bucket never comes from the key. +- **Nothing deletes a bucket.** There is no such operation to call, so disabling or removing a module + cannot take data with it. +- **Provider errors are translated, never forwarded.** No endpoint, request id, bucket name, or XML + body survives into a `ModuleStorageError` a module might log or return. + +Endpoints must be absolute `http`/`https`, must not embed credentials, and must use `https` in +production — the same standard the database transport holds. Credentials are never persisted in +PostgreSQL and never routed through module configuration, which has no encryption at rest to offer +them (ADR 0029). + +**What this does not claim.** A module is still trusted code in the API process, so a malicious module +could read `process.env` directly and obtain the provider credentials that way. Scoped storage is a +correctness and portability boundary like every other module-facing mechanism here — not a sandbox. +What it does guarantee is that Ekklesia never hands a module another module's data, never gives module +UI or a browser a credential, and never prints one in diagnostics. + ### Client-side permission checks are not a security boundary A host may tell a module UI what permissions it believes the viewer holds diff --git a/docs/development/testing.md b/docs/development/testing.md index 1d4e616..b0f38a0 100644 --- a/docs/development/testing.md +++ b/docs/development/testing.md @@ -193,6 +193,29 @@ rejected without touching `Object.prototype`, and a module blocked by its own co from the mountable UI inventory exactly like a disabled one. See [ADR 0029](../architecture/decisions/0029-module-configuration-and-secrets.md). +A seventh dedicated suite (`core/tests/module-storage.integration.ts`, included in the same +`pnpm test:migrations:integration` run) proves the object-storage contract against a **real +S3-compatible server** — the SeaweedFS service the repository's Compose file already pins, brought up +alongside PostgreSQL by the same harness, so this adds no new CI infrastructure. + +It exists because mocking would have hidden a real divergence: probing the pinned SeaweedFS 4.28 image +directly showed that `CreateBucket` on an existing bucket reports `BucketAlreadyExists`, **not** AWS's +`BucketAlreadyOwnedByYou`. A provider treating only the AWS spelling as success would have failed on +every second boot against this repository's own development storage, and no unit test would have +caught it. The suite pins that idempotency, plus a real round trip with content type and unquoted +ETag preserved, streaming a body without buffering, a Web stream going straight into a `Response`, +prefix listing and real pagination, a missing object mapping to a structured not-found error carrying +no endpoint or XML, two modules declaring the same logical bucket staying genuinely isolated on one +server, the longest bucket name the mapping can produce being accepted, a real connection failure +mapping to `STORAGE_PROVIDER_UNAVAILABLE`, and a bucket surviving the deletion of every object in it. + +Unit suites (`core/tests/module-storage.test.ts`, `core/tests/module-storage-lifecycle.test.ts`) use an +injected in-memory provider and never touch the network. They prove the parts the protocol cannot: +binding determinism and collision-safety, scoping and cross-module isolation, key validation, lazy +provisioning and retry-after-failure, configuration validation and credential redaction, and the +lifecycle integration including dependency propagation. See +[ADR 0030](../architecture/decisions/0030-host-provided-module-object-storage.md). + ### Which kind of timing proof lives where Pass 4 introduced several deadlines, and their tests deliberately differ in kind: diff --git a/docs/module-development/README.md b/docs/module-development/README.md index d136402..d82bc31 100644 --- a/docs/module-development/README.md +++ b/docs/module-development/README.md @@ -969,6 +969,12 @@ write one row, and start nothing. The change takes effect at the next `pnpm api: so a module that breaks startup can be turned off without the API — the HTTP lifecycle API is served by the process such a module takes down. +`pnpm doctor`'s `object-storage.integration` check reports whether a storage provider is configured +and reachable, and `object-storage.bindings` lists which modules require storage and where each +logical bucket is bound. Neither prints a credential. A module requiring storage on an installation +with none configured makes the bindings check fail with a remediation line; a provider that is merely +unreachable is a warning, because modules already running are unaffected. + `pnpm doctor`'s `modules.configuration` check reports whether each installed module's configuration would let it run, naming the offending key when it would not. It prints **no configuration values** — secret or otherwise — only counts, keys, and reasons. @@ -1096,6 +1102,108 @@ leaves room for.) Configuration tells you *how to behave*, never *whether to run*. Whether your module runs is its host's decision, described next. +## Object storage + +Declare the logical buckets you need in `ekklesia.module.json`, and read and write them through +`context.storage` ([ADR 0030](../architecture/decisions/0030-host-provided-module-object-storage.md)). + +```json +"storage": { + "buckets": [ + { "id": "announcement-assets", "description": "Slide images an operator uploads." } + ] +} +``` + +```ts +export default defineModuleRuntime({ + id: "announcements", + version: "1.0.0", + register({ http }) { + http.route({ + method: "GET", + path: "/assets/:key", + permission: "announcements.assets.read", + maxBodyBytes: 0, + async handler(_request, context) { + const object = await context.storage.bucket("announcement-assets").get(context.params.key); + return new Response(object.body, { + headers: { "content-type": object.contentType ?? "application/octet-stream" }, + }); + }, + }); + }, +}); +``` + +**You name a logical bucket. You never name a service.** There is no endpoint, region, credential, or +physical bucket anywhere in this contract, and no accessor that could return one. That is what lets +the same module run under Ekklesia and under a future standalone host backed by its own SeaweedFS, a +shared one, MinIO, or AWS S3 — without your code changing. + +Never add `storageEndpoint`, `storageAccessKey`, or similar to your module configuration. Those are +the host's, and a module that configures its own object store is a module welded to one deployment. + +### Declaring a bucket is a real requirement + +If you declare a bucket and the installation has no storage provider configured, your module is +**blocked** — reported as `storage_unavailable` naming that bucket, with its required dependents +blocked behind it. Your `start` hook never runs, so it never has to defend against missing storage. + +Declare a bucket only if you need one. A module that declares none is completely unaffected by +whether the installation has object storage, which is why the template declares none. + +### The operations + +```ts +bucket.list({ prefix, limit, cursor }); // → { objects, cursor? } +bucket.head(key); // → metadata only +bucket.get(key); // → metadata + body: ReadableStream +bucket.put(key, body, { contentType, contentLength }); +bucket.delete(key); +``` + +`get` gives you a Web stream, so hand it straight to a `Response` rather than buffering. `put` takes +a `Uint8Array`, string, `Blob`, or a `ReadableStream` — a stream **must** come with `contentLength`, +because every S3-compatible provider needs the length up front and inferring it would mean buffering +the object you were trying to stream. + +A listing is paginated. When `cursor` is present, more objects remain; when it is absent, the listing +is complete. Ignoring it will silently truncate a large bucket. + +There is deliberately **no** way to create or delete a bucket. Which buckets exist follows from your +manifest and an operator's installation, and nothing your module does can destroy one. + +### Object keys + +Keys may nest (`slides/2026/welcome.png`). They are not filesystem paths — `..` means nothing to an +object store, and your bucket comes from the host's binding rather than from the key, so a key cannot +escape it. Empty keys, keys over 1024 UTF-8 bytes, control characters, backslashes, and a leading `/` +are rejected as `STORAGE_KEY_INVALID`. + +### Storage failures are yours to handle, and never change your lifecycle + +```ts +try { + const listing = await context.storage.bucket("announcement-assets").list(); +} catch (error) { + // STORAGE_PROVIDER_UNAVAILABLE, STORAGE_OBJECT_NOT_FOUND, STORAGE_OPERATION_DENIED, ... +} +``` + +`ModuleStorageError` carries a code and nothing else — no endpoint, credential, bucket name, or +provider body — so it is safe to log or return. + +**A storage outage is not a lifecycle event.** If the object store is briefly unreachable, your module +stays `running`; retry on your own schedule and report health in your own UI. The platform will never +disable or block a running module because a storage call timed out. + +### Serving objects to a browser + +Module UI gets no storage access and no credentials. Serve bytes through your own authorized route, +as in the example above: `browser → your module API → your scoped storage`. Signed URLs and public +object delivery do not exist yet. + ### Whether your module runs at all is your host's decision Between discovery and startup, the platform consults its own **enablement configuration** and diff --git a/packages/contracts/src/index.ts b/packages/contracts/src/index.ts index a808ad3..cc522b4 100644 --- a/packages/contracts/src/index.ts +++ b/packages/contracts/src/index.ts @@ -9,4 +9,5 @@ export * from "./module-manifest.js"; export * from "./module-migration.js"; export * from "./module-runtime.js"; export * from "./module-schema.js"; +export * from "./module-storage.js"; export * from "./versions.js"; diff --git a/packages/contracts/src/module-http.ts b/packages/contracts/src/module-http.ts index c8981a6..ed82633 100644 --- a/packages/contracts/src/module-http.ts +++ b/packages/contracts/src/module-http.ts @@ -2,6 +2,7 @@ import { z } from "zod"; import type { EkklesiaPrincipal } from "./identity.js"; import { type ModuleId, type PermissionId, permissionIdSchema } from "./identifiers.js"; import type { ModuleConfiguration } from "./module-configuration.js"; +import type { ModuleStorage } from "./module-storage.js"; import type { ModuleRuntimeServices } from "./module-database.js"; // Not imported from `./module-runtime.js`'s own `Awaitable` (structurally identical) — importing @@ -220,6 +221,10 @@ export interface ModuleRouteHandlerContext { readonly moduleId: ModuleId; readonly principal: EkklesiaPrincipal; readonly params: Readonly>; + /** The identical scoped storage facade this module's `start`/`stop` hooks received — the same + * object, not a re-resolved one (ADR 0030). This is what lets a module stream an object straight + * into a `Response` without the browser ever learning that object storage exists. */ + readonly storage: ModuleStorage; readonly services: ModuleRuntimeServices; /** The identical frozen configuration snapshot this module's `start`/`stop` hooks received — the * same object, not a re-read — so a handler and a lifecycle hook can never disagree about a value diff --git a/packages/contracts/src/module-lifecycle.ts b/packages/contracts/src/module-lifecycle.ts index 0d65328..6cf53e1 100644 --- a/packages/contracts/src/module-lifecycle.ts +++ b/packages/contracts/src/module-lifecycle.ts @@ -61,6 +61,7 @@ export const MODULE_BLOCKED_REASONS = [ "configuration_missing", "configuration_invalid", "secret_missing", + "storage_unavailable", ] as const; export const moduleBlockedReasonSchema = z.enum(MODULE_BLOCKED_REASONS); @@ -93,6 +94,21 @@ export const MODULE_CONFIGURATION_BLOCKED_REASONS = [ export type ModuleConfigurationBlockedReason = (typeof MODULE_CONFIGURATION_BLOCKED_REASONS)[number]; +/** + * The module declares a logical object-storage bucket, and this host has no storage provider + * configured to bind it to (ADR 0030). Fix: configure an S3-compatible storage provider for the + * deployment. + * + * It names the **logical bucket** the module declared, never an endpoint, a credential, or a + * physical bucket name. + * + * Deliberately narrow: this is a *binding* problem, not a health problem. A configured provider that + * is briefly unreachable is an operational condition a module handles and retries — it never changes + * a module's lifecycle state, because a slideshow whose object store blinked is still a running + * slideshow. + */ +export const MODULE_STORAGE_BLOCKED_REASON = "storage_unavailable" as const; + export function isModuleConfigurationBlockedReason( reason: ModuleBlockedReason, ): reason is ModuleConfigurationBlockedReason { diff --git a/packages/contracts/src/module-runtime.ts b/packages/contracts/src/module-runtime.ts index 3150c3f..df011b0 100644 --- a/packages/contracts/src/module-runtime.ts +++ b/packages/contracts/src/module-runtime.ts @@ -1,4 +1,5 @@ import type { ModuleConfiguration } from "./module-configuration.js"; +import type { ModuleStorage } from "./module-storage.js"; import type { CapabilityId, ModuleId } from "./identifiers.js"; import { moduleIdSchema } from "./identifiers.js"; import type { ModuleRuntimeServices } from "./module-database.js"; @@ -48,6 +49,18 @@ export interface ModuleRuntimeContext { readonly capabilities: CapabilityResolver; readonly moduleId: ModuleId; readonly services: ModuleRuntimeServices; + /** + * This module's own scoped object storage (ADR 0030). + * + * Present here and on HTTP handler contexts but **not** on `ModuleRegistrationContext`, exactly + * like `services`: this is a live facade that performs network I/O, and registration stays + * declarative and I/O-free. `config` is on registration precisely because it is the opposite — an + * already-resolved in-memory snapshot. + * + * Scoped per module and never shared: it resolves only the logical buckets this module's own + * manifest declared, and carries no endpoint, credential, or physical bucket name. + */ + readonly storage: ModuleStorage; /** * This module's own validated configuration, frozen for the lifetime of the process. * diff --git a/packages/contracts/src/module-storage.ts b/packages/contracts/src/module-storage.ts new file mode 100644 index 0000000..55e2b0f --- /dev/null +++ b/packages/contracts/src/module-storage.ts @@ -0,0 +1,223 @@ +/** + * The one module-facing object-storage contract. + * + * **Nothing in this file names a provider.** There is no endpoint, no region, no credential, no + * bucket policy, no signing, no `S3`, and no `SeaweedFS` — a module asks for the *logical* bucket it + * declared in its manifest and reads and writes objects in it. Which physical service satisfies that + * requirement is entirely the host's business (ADR 0030). + * + * That is what makes the same module code runnable under Ekklesia and under a future standalone + * host: both construct this interface, one over a shared S3-compatible service and the other over + * whatever it has, and the module cannot tell the difference. + */ + +/** Every failure a module can observe from storage, as a closed set of codes rather than a provider + * library's exception hierarchy. */ +export const MODULE_STORAGE_ERROR_CODES = [ + /** The module asked for a logical bucket its own manifest does not declare. A programming error, + * not a runtime condition — the platform cannot invent a binding a module never asked for. */ + "STORAGE_BUCKET_UNDECLARED", + /** The key does not exist in this bucket. */ + "STORAGE_OBJECT_NOT_FOUND", + /** The bucket this binding names is not usable — absent, or not creatable with the host's + * credentials. Distinct from a provider-wide outage. */ + "STORAGE_BUCKET_UNAVAILABLE", + /** The storage service could not be reached at all. Transient by nature: a module may retry, and + * this never changes the module's lifecycle state (ADR 0030). */ + "STORAGE_PROVIDER_UNAVAILABLE", + /** The host's credentials do not permit this operation. */ + "STORAGE_OPERATION_DENIED", + /** The object key is not one this contract accepts. See `isValidModuleStorageKey`. */ + "STORAGE_KEY_INVALID", + /** A `ReadableStream` body was supplied without `contentLength`. Refused rather than buffered: + * buffering would defeat the only reason to pass a stream, and discovering that through memory + * pressure in production is worse than discovering it at the call site. */ + "STORAGE_CONTENT_LENGTH_REQUIRED", + /** Anything else the provider reported, reduced to a code. */ + "STORAGE_OPERATION_FAILED", +] as const; + +export type ModuleStorageErrorCode = (typeof MODULE_STORAGE_ERROR_CODES)[number]; + +const storageErrorMessages: Readonly> = Object.freeze({ + STORAGE_BUCKET_UNAVAILABLE: "The requested storage bucket is not available.", + STORAGE_BUCKET_UNDECLARED: "This module did not declare the requested storage bucket.", + STORAGE_CONTENT_LENGTH_REQUIRED: "A streamed storage body requires an explicit content length.", + STORAGE_KEY_INVALID: "The storage object key is invalid.", + STORAGE_OBJECT_NOT_FOUND: "The storage object was not found.", + STORAGE_OPERATION_DENIED: "The storage operation was denied.", + STORAGE_OPERATION_FAILED: "The storage operation failed.", + STORAGE_PROVIDER_UNAVAILABLE: "The storage service could not be reached.", +}); + +/** + * A sanitized storage failure. + * + * Carries a code and a fixed message, and deliberately **nothing else** — no endpoint, no region, no + * access key, no bucket name (physical or logical), no request id, no provider XML body, and no + * caught cause. A module may catch this, log it, and return it to a browser; every one of those + * paths would otherwise be a way for host infrastructure detail to escape. It follows the same + * discipline as `ModuleLifecycleStoreError` and `ModuleConfigurationStoreError`. + * + * `key` echoes a value the **module itself supplied** — an object key for the object errors, or the + * logical bucket id for `STORAGE_BUCKET_UNDECLARED`. It is safe precisely because it never came from + * the host: it is the module's own input handed back so a caught error is actionable. + */ +export class ModuleStorageError extends Error { + readonly code: ModuleStorageErrorCode; + readonly key?: string; + + constructor(code: ModuleStorageErrorCode, key?: string) { + super(storageErrorMessages[code]); + this.name = "ModuleStorageError"; + this.code = code; + if (key !== undefined) { + this.key = key; + } + this.stack = `${this.name}: ${this.message}`; + Object.freeze(this); + } +} + +/** The S3 object-key ceiling, which every S3-compatible provider shares. Measured in UTF-8 bytes, + * not characters, because that is what the wire format bounds. */ +export const MODULE_STORAGE_KEY_MAX_BYTES = 1024; + +/** + * Whether a value is an object key this contract accepts. + * + * An object key is **not a filesystem path** — `..` has no traversal meaning to an object store, and + * the physical bucket comes from the host's binding rather than from anything in the key, so a key + * structurally cannot escape its bucket. What is rejected here is what would be *ambiguous* or + * unrepresentable rather than what would be dangerous: an empty key, one that exceeds the wire + * limit, control characters (which break header and XML encodings), lone surrogates (unencodable as + * UTF-8), a backslash (whose meaning differs between providers), and a leading `/` (which some + * providers normalize away and others do not, so one key would name two objects). + * + * `/` inside a key is allowed and ordinary: object stores use it as a display and prefix convention, + * and modules genuinely need `slides/welcome.png`. + */ +export function isValidModuleStorageKey(value: unknown): value is string { + if (typeof value !== "string" || value.length === 0) { + return false; + } + if (value.startsWith("/") || value.includes("\\")) { + return false; + } + for (const character of value) { + const codePoint = character.codePointAt(0); + if (codePoint === undefined) { + return false; + } + // C0 controls, DEL, and C1 controls. + if (codePoint <= 0x1f || (codePoint >= 0x7f && codePoint <= 0x9f)) { + return false; + } + // A lone surrogate survives in a JavaScript string but cannot be encoded as UTF-8. + if (codePoint >= 0xd800 && codePoint <= 0xdfff) { + return false; + } + } + return new TextEncoder().encode(value).length <= MODULE_STORAGE_KEY_MAX_BYTES; +} + +/** What a module may hand to `put`. A `ReadableStream` is accepted so a large object never has to be + * buffered in memory, but it must be accompanied by `contentLength` — every S3-compatible provider + * needs the length up front for a single-part upload, and guessing it would mean buffering the very + * thing the stream exists to avoid. */ +export type ModuleStorageBody = Uint8Array | string | Blob | ReadableStream; + +/** What the store knows about an object besides its bytes. Deliberately the four fields every + * S3-compatible provider actually returns, rather than a mirror of S3's full metadata surface. */ +export interface ModuleStorageObjectMetadata { + readonly key: string; + /** Preserved exactly as it was written. A module serving images through its own route needs this + * to set a correct `Content-Type` without sniffing bytes. */ + readonly contentType?: string; + readonly contentLength?: number; + readonly etag?: string; + readonly lastModified?: Date; +} + +/** An object's metadata plus its bytes as a Web stream, so a module can pass it straight into a + * `Response` without ever holding the whole object in memory. */ +export interface ModuleStorageObject extends ModuleStorageObjectMetadata { + readonly body: ReadableStream; +} + +export interface ModuleStoragePutOptions { + readonly contentType?: string; + /** Required when `body` is a `ReadableStream`, ignored otherwise. */ + readonly contentLength?: number; +} + +export interface ModuleStorageListOptions { + readonly prefix?: string; + /** Provider-bounded; a provider may return fewer. */ + readonly limit?: number; + /** The `cursor` from a previous listing. Opaque — a module passes it back and never parses it. */ + readonly cursor?: string; +} + +export interface ModuleStorageListing { + readonly objects: readonly ModuleStorageObjectMetadata[]; + /** Present when more objects remain. Absent means the listing is complete — which is why this + * exists at all: every S3-compatible provider caps a single listing, and a contract without a + * cursor would silently truncate a bucket with more objects than that cap. */ + readonly cursor?: string; +} + +/** + * One logical bucket a module declared, already bound to wherever the host keeps it. + * + * `id` is the **logical** id from the module's own manifest. The physical bucket, its naming scheme, + * and the service holding it are not reachable from here by design — there is no accessor for them, + * so a module cannot come to depend on the host's implementation and cannot be ported away from it + * later. + */ +export interface ModuleStorageBucket { + readonly id: string; + list(options?: ModuleStorageListOptions): Promise; + head(key: string): Promise; + get(key: string): Promise; + put( + key: string, + body: ModuleStorageBody, + options?: ModuleStoragePutOptions, + ): Promise; + delete(key: string): Promise; +} + +/** + * A module's own scoped storage. + * + * Scoped by construction: `bucket()` resolves only against the logical buckets **this** module + * declared, so knowing another module's bucket name buys nothing — there is no name that maps + * outside this module's own bindings, and no accessor that takes a physical bucket, an endpoint, or + * a credential. + * + * There is deliberately no `createBucket`, `deleteBucket`, or bucket-listing operation. Which + * buckets exist is decided by the manifest an operator installed, not by running module code, and + * nothing a module does can destroy a bucket (ADR 0030). + */ +export interface ModuleStorage { + /** The logical bucket ids this module declared, sorted. */ + buckets(): readonly string[]; + /** Throws `ModuleStorageError("STORAGE_BUCKET_UNDECLARED")` for anything this module did not + * declare — including another module's bucket. It throws rather than returning `undefined` because + * asking for an undeclared bucket is a manifest/code mismatch, and a silent no-op would hide it + * until data went missing. */ + bucket(logicalBucketId: string): ModuleStorageBucket; +} + +/** + * The storage a module with no declared buckets receives. `context.storage` is therefore always + * present and never `undefined`, so module code never branches on whether the host supplied one — + * the same reasoning `EMPTY_MODULE_CONFIGURATION` follows. + */ +export const EMPTY_MODULE_STORAGE: ModuleStorage = Object.freeze({ + bucket: (logicalBucketId: string): ModuleStorageBucket => { + throw new ModuleStorageError("STORAGE_BUCKET_UNDECLARED", logicalBucketId); + }, + buckets: (): readonly string[] => Object.freeze([]), +}); diff --git a/packages/contracts/tests/module-manifest.test.ts b/packages/contracts/tests/module-manifest.test.ts index 8a0c75f..def6d1c 100644 --- a/packages/contracts/tests/module-manifest.test.ts +++ b/packages/contracts/tests/module-manifest.test.ts @@ -5,6 +5,7 @@ import { MODULE_BLOCKED_REASONS, MODULE_CONFIGURATION_BLOCKED_REASONS, MODULE_DEPENDENCY_KINDS, + MODULE_STORAGE_BLOCKED_REASON, MODULE_MANIFEST_VERSION, MODULE_RUNTIME_STATES, packageRelativePathSchema, @@ -364,7 +365,13 @@ describe("module manifest contract", () => { "configuration_missing", "configuration_invalid", "secret_missing", + "storage_unavailable", ]); + // Storage adds exactly one reason, and only one: a *binding* that cannot be supplied. A + // configured provider that is temporarily unreachable is deliberately not a lifecycle state + // (ADR 0030), so there is no `storage_unreachable` here and there must never be. + expect(MODULE_BLOCKED_REASONS).not.toContain("storage_unreachable"); + expect(MODULE_STORAGE_BLOCKED_REASON).toBe("storage_unavailable"); // The three configuration reasons are a subset, and each corresponds to a real implemented // check with a distinct operator fix (ADR 0029) — the only justification for a separate code. expect(MODULE_CONFIGURATION_BLOCKED_REASONS).toEqual([ diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 009ac7d..dba6722 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -82,6 +82,9 @@ importers: core: dependencies: + '@aws-sdk/client-s3': + specifier: 3.1116.0 + version: 3.1116.0 '@ekklesia/contracts': specifier: workspace:* version: link:../packages/contracts @@ -325,6 +328,78 @@ importers: packages: + '@aws-sdk/checksums@3.1000.29': + resolution: {integrity: sha512-Dtu0gr4dnATZAPwEYbpCsG+MpLM7OAliy2gTepEFQwl1vZ6DL3QMH2FveMa3HLvPsOdhJsPRB3KtxVhph9T75A==} + engines: {node: '>=20.0.0'} + + '@aws-sdk/client-s3@3.1116.0': + resolution: {integrity: sha512-UKRl9qSVW0rZpvSOauQNpYAy8+ONBAVYnpfKVtCyOF+FZVT1tl6MunYuHvuarCrroD2/YJs+tHTALYNgAlec3Q==} + engines: {node: '>=20.0.0'} + + '@aws-sdk/core@3.977.9': + resolution: {integrity: sha512-reqPFEQrZxDZpeGj4PFMepBeR5LGYHRqq/L0motTzgFkCRBA4rFdaVXDSLYyGHhxVz7sT2PDnPN9CluGSfgyJA==} + engines: {node: '>=20.0.0'} + + '@aws-sdk/credential-provider-env@3.972.70': + resolution: {integrity: sha512-H404B7dJl2mCrBqahDEYsanB0xhdDp6tXnXcTUnXmmpy2Q3J0Ho0bUajZ2jr/RdwzCyS59Gi8xXIFwPLGBl6Uw==} + engines: {node: '>=20.0.0'} + + '@aws-sdk/credential-provider-http@3.972.72': + resolution: {integrity: sha512-X98zYOrVOeuosCX+6ktf29FC2N2GHPLia7qv6mzPzTc+RPAuHWCDS++Z6JK7eGYqb/v6uaW7bAXaOvDBfol+0w==} + engines: {node: '>=20.0.0'} + + '@aws-sdk/credential-provider-ini@3.973.15': + resolution: {integrity: sha512-Rykg6s5ceBuynMOGWgoowO4N+27JfnqXAnVaSunZl0hOO1XodSrxGNz6sCEbnmS0lAfQZDKyb3fbr46gSuv6Sg==} + engines: {node: '>=20.0.0'} + + '@aws-sdk/credential-provider-login@3.972.77': + resolution: {integrity: sha512-Jb59xfEISoN5mmbnA+HYqdtrSX3CgCtJoof+V5D8/TgUI56W63GEEd5Y58WijU3Ou6+WEgaLD1feVzaRXV5IDQ==} + engines: {node: '>=20.0.0'} + + '@aws-sdk/credential-provider-node@3.972.81': + resolution: {integrity: sha512-Rml+WitoFvXmv6JZ18U/xGdGDGGvB/mOin0ya0lTnTrdC0Z1lrVxTYh7iNklZBcvcRMrs4DoEf6xy1KWyrLQQw==} + engines: {node: '>=20.0.0'} + + '@aws-sdk/credential-provider-process@3.972.70': + resolution: {integrity: sha512-2ry03fGRJr4sV3jI+ocjj5JqALnFD6ymM5KiNCDZMvq8bX2GSbE0vji4aM43TVCl2nXqqLRZaUxdq/KeWRAY4Q==} + engines: {node: '>=20.0.0'} + + '@aws-sdk/credential-provider-sso@3.973.14': + resolution: {integrity: sha512-jkhg/8ocAAoc0RFyLMhCw+/zZh7gystQgd4F4hznNa8P4Cc501PQmxd+jGLiMHodPJ+7Zv/3znM62gZojyasmA==} + engines: {node: '>=20.0.0'} + + '@aws-sdk/credential-provider-web-identity@3.972.76': + resolution: {integrity: sha512-d3AGyVu759PGr35mEB2s22xxlNEA5rpdxtSPJthfPFJvoQ8dt357iVPECqWfUxXp1toJAvKmbtcIYVGigaGsCA==} + engines: {node: '>=20.0.0'} + + '@aws-sdk/middleware-sdk-s3@3.972.75': + resolution: {integrity: sha512-wMIsNumRVKaNMKhvU/s9VrdEwE8S6gSzXp4RygFG5BEMnGkkXf8cjh8zf7cKJBpUDpqTWqwbz5isEgp9rH6Lng==} + engines: {node: '>=20.0.0'} + + '@aws-sdk/nested-clients@3.997.44': + resolution: {integrity: sha512-NhEgryjlBF9w38ZXqGymQV28IhkYa1mKhlbYnqIis57AYwWGVYfUPgg/qC2rLRqOUfblxx++irvju10kVTa8Vw==} + engines: {node: '>=20.0.0'} + + '@aws-sdk/signature-v4-multi-region@3.996.46': + resolution: {integrity: sha512-L+2xZTye/2T96f3lwCws0Zw6GG2JHZW9e8FpVgGBeeExSKyeoZ6CWRpBml/7DNiK/O26jrgPM9F+Ay8VkgzUWQ==} + engines: {node: '>=20.0.0'} + + '@aws-sdk/token-providers@3.1116.0': + resolution: {integrity: sha512-ygIivKqh8aHzNkucOCXHyIBgBpLPfrSI0mCqXF+vLBsPTUKqj0VSqAY0GFPe7lQl4HntjOcQ+KSyS7oUV2C54Q==} + engines: {node: '>=20.0.0'} + + '@aws-sdk/types@3.974.5': + resolution: {integrity: sha512-LkwLL2BLbC6wNNm4JaH9mbEqBMdOZCct6VAYqhdN4U1xrWM+fUJQEfbHwQgDypapOWTRtlk25akb5afM0P8CIQ==} + engines: {node: '>=20.0.0'} + + '@aws-sdk/xml-builder@3.972.40': + resolution: {integrity: sha512-wlFmCIGUlwF4zx/kncw+bmxTQh1HeSJq4mYV/V5cZUSJadDP3kXvGW8Rn21cimj/7y9ju+47oYWXi97vF7czaA==} + engines: {node: '>=20.0.0'} + + '@aws/lambda-invoke-store@0.3.0': + resolution: {integrity: sha512-sl4Bm6yiMNYrZKkqqDFWN0UfnWhlS8ivKxrYl+6t0gCLrqr8y3B2IqZZbFRkfaVVp7C/baApyh71P+LeE1A2sQ==} + engines: {node: '>=18.0.0'} + '@babel/code-frame@7.29.7': resolution: {integrity: sha512-Aup7aUOfpbAUg2ROOJN6Iw5f9DMBlzu0mIkm/malLQFN/YQgO48wCj0Kxa3sEHJvPVFg7siR+qRInwXd2qhQKw==} engines: {node: '>=6.9.0'} @@ -1115,6 +1190,30 @@ packages: '@rolldown/pluginutils@1.0.1': resolution: {integrity: sha512-2j9bGt5Jh8hj+vPtgzPtl72j0yRxHAyumoo6TNfAjsLB04UtpSvPbPcDcBMxz7n+9CYB0c1GxQFxYRg2jimqGw==} + '@smithy/core@3.33.3': + resolution: {integrity: sha512-CsOeKq/9kA3y6VJHt+/+VTCtBaxJ4OTFpgrjIUhPpDIKxBci1k2bJaQASF2h/ELWrulGp+t97DZ0mevfAD8idg==} + engines: {node: '>=18.0.0'} + + '@smithy/credential-provider-imds@4.5.2': + resolution: {integrity: sha512-A9uSdn72ozbRUSit0eib0TW7nXuNPlaeM0zcGkJ+nE6tFcSDbnmtwoxbTCFBukVQcszDAyvsd7+rTduPTXpygg==} + engines: {node: '>=18.0.0'} + + '@smithy/fetch-http-handler@5.7.2': + resolution: {integrity: sha512-nZyWTmSpJEXl6VtWVMBJve/7x12DZu6sIX1z1a+ZMaHlQQRs9Zpu6NbTe/gmxYXVRpkjxyDYpZ5gx2IM6f/Wkw==} + engines: {node: '>=18.0.0'} + + '@smithy/node-http-handler@4.11.3': + resolution: {integrity: sha512-2jY1tSpERfPfWqyBV2pH+iGFaghVsIJszJNsT7hxtQYhVJpWDyc0LqOWI+nXOxOAHaEfZ4PXXtp1wW1TGpHhkA==} + engines: {node: '>=18.0.0'} + + '@smithy/signature-v4@5.7.3': + resolution: {integrity: sha512-7ImGm+FkHRLcBaRttIAMZ6bzJZWb2cJGoYjq46F2UjycujWzrL9GEN9h4w7eQyXJYnltrUhxbbieBAIRrdqpow==} + engines: {node: '>=18.0.0'} + + '@smithy/types@4.17.2': + resolution: {integrity: sha512-FOKpVZob9MPTn2znRzGrnsMHv7BOsKVw3XiP/cOyYLDVZ9qKp4nifIiSCuUU/fIj5Vu0UOAxCFr+qRAtG0NUkA==} + engines: {node: '>=18.0.0'} + '@standard-schema/spec@1.1.0': resolution: {integrity: sha512-l2aFy5jALhniG5HgqrD6jXLi/rUWrKvqN/qJx6yoJsgKhblVd+iqqU4RCXavm/jPityDo5TCvKMnpjKnOriy0w==} @@ -1415,6 +1514,9 @@ packages: zod: optional: true + bowser@2.14.1: + resolution: {integrity: sha512-tzPjzCxygAKWFOJP011oxFHs57HzIhOEracIgAePE4pqB3LikALKnSzUyU4MGs9/iCEUuHlAJTjTc5M+u7YEGg==} + browserslist@4.28.6: resolution: {integrity: sha512-FQBYNK15VMslhLHpA7+n+n1GOlF1kId2xcCg7/j95f24AOF6VDYMNH4mFxF7KuaTdv627faazpOAjFzMrfJOUw==} engines: {node: ^6 || ^7 || ^8 || ^9 || ^10 || ^11 || ^12 || >=13.7} @@ -2137,6 +2239,171 @@ packages: snapshots: + '@aws-sdk/checksums@3.1000.29': + dependencies: + '@aws-sdk/core': 3.977.9 + '@aws-sdk/types': 3.974.5 + '@smithy/core': 3.33.3 + '@smithy/types': 4.17.2 + tslib: 2.8.1 + + '@aws-sdk/client-s3@3.1116.0': + dependencies: + '@aws-sdk/checksums': 3.1000.29 + '@aws-sdk/core': 3.977.9 + '@aws-sdk/credential-provider-node': 3.972.81 + '@aws-sdk/middleware-sdk-s3': 3.972.75 + '@aws-sdk/signature-v4-multi-region': 3.996.46 + '@aws-sdk/types': 3.974.5 + '@smithy/core': 3.33.3 + '@smithy/fetch-http-handler': 5.7.2 + '@smithy/node-http-handler': 4.11.3 + '@smithy/types': 4.17.2 + tslib: 2.8.1 + + '@aws-sdk/core@3.977.9': + dependencies: + '@aws-sdk/types': 3.974.5 + '@aws-sdk/xml-builder': 3.972.40 + '@aws/lambda-invoke-store': 0.3.0 + '@smithy/core': 3.33.3 + '@smithy/signature-v4': 5.7.3 + '@smithy/types': 4.17.2 + bowser: 2.14.1 + tslib: 2.8.1 + + '@aws-sdk/credential-provider-env@3.972.70': + dependencies: + '@aws-sdk/core': 3.977.9 + '@aws-sdk/types': 3.974.5 + '@smithy/core': 3.33.3 + '@smithy/types': 4.17.2 + tslib: 2.8.1 + + '@aws-sdk/credential-provider-http@3.972.72': + dependencies: + '@aws-sdk/core': 3.977.9 + '@aws-sdk/types': 3.974.5 + '@smithy/core': 3.33.3 + '@smithy/fetch-http-handler': 5.7.2 + '@smithy/node-http-handler': 4.11.3 + '@smithy/types': 4.17.2 + tslib: 2.8.1 + + '@aws-sdk/credential-provider-ini@3.973.15': + dependencies: + '@aws-sdk/core': 3.977.9 + '@aws-sdk/credential-provider-env': 3.972.70 + '@aws-sdk/credential-provider-http': 3.972.72 + '@aws-sdk/credential-provider-login': 3.972.77 + '@aws-sdk/credential-provider-process': 3.972.70 + '@aws-sdk/credential-provider-sso': 3.973.14 + '@aws-sdk/credential-provider-web-identity': 3.972.76 + '@aws-sdk/nested-clients': 3.997.44 + '@aws-sdk/types': 3.974.5 + '@smithy/core': 3.33.3 + '@smithy/credential-provider-imds': 4.5.2 + '@smithy/types': 4.17.2 + tslib: 2.8.1 + + '@aws-sdk/credential-provider-login@3.972.77': + dependencies: + '@aws-sdk/core': 3.977.9 + '@aws-sdk/nested-clients': 3.997.44 + '@aws-sdk/types': 3.974.5 + '@smithy/core': 3.33.3 + '@smithy/types': 4.17.2 + tslib: 2.8.1 + + '@aws-sdk/credential-provider-node@3.972.81': + dependencies: + '@aws-sdk/credential-provider-env': 3.972.70 + '@aws-sdk/credential-provider-http': 3.972.72 + '@aws-sdk/credential-provider-ini': 3.973.15 + '@aws-sdk/credential-provider-process': 3.972.70 + '@aws-sdk/credential-provider-sso': 3.973.14 + '@aws-sdk/credential-provider-web-identity': 3.972.76 + '@aws-sdk/types': 3.974.5 + '@smithy/core': 3.33.3 + '@smithy/credential-provider-imds': 4.5.2 + '@smithy/types': 4.17.2 + tslib: 2.8.1 + + '@aws-sdk/credential-provider-process@3.972.70': + dependencies: + '@aws-sdk/core': 3.977.9 + '@aws-sdk/types': 3.974.5 + '@smithy/core': 3.33.3 + '@smithy/types': 4.17.2 + tslib: 2.8.1 + + '@aws-sdk/credential-provider-sso@3.973.14': + dependencies: + '@aws-sdk/core': 3.977.9 + '@aws-sdk/nested-clients': 3.997.44 + '@aws-sdk/token-providers': 3.1116.0 + '@aws-sdk/types': 3.974.5 + '@smithy/core': 3.33.3 + '@smithy/types': 4.17.2 + tslib: 2.8.1 + + '@aws-sdk/credential-provider-web-identity@3.972.76': + dependencies: + '@aws-sdk/core': 3.977.9 + '@aws-sdk/nested-clients': 3.997.44 + '@aws-sdk/types': 3.974.5 + '@smithy/core': 3.33.3 + '@smithy/types': 4.17.2 + tslib: 2.8.1 + + '@aws-sdk/middleware-sdk-s3@3.972.75': + dependencies: + '@aws-sdk/core': 3.977.9 + '@aws-sdk/signature-v4-multi-region': 3.996.46 + '@aws-sdk/types': 3.974.5 + '@smithy/core': 3.33.3 + '@smithy/types': 4.17.2 + tslib: 2.8.1 + + '@aws-sdk/nested-clients@3.997.44': + dependencies: + '@aws-sdk/core': 3.977.9 + '@aws-sdk/signature-v4-multi-region': 3.996.46 + '@aws-sdk/types': 3.974.5 + '@smithy/core': 3.33.3 + '@smithy/fetch-http-handler': 5.7.2 + '@smithy/node-http-handler': 4.11.3 + '@smithy/types': 4.17.2 + tslib: 2.8.1 + + '@aws-sdk/signature-v4-multi-region@3.996.46': + dependencies: + '@aws-sdk/types': 3.974.5 + '@smithy/signature-v4': 5.7.3 + '@smithy/types': 4.17.2 + tslib: 2.8.1 + + '@aws-sdk/token-providers@3.1116.0': + dependencies: + '@aws-sdk/core': 3.977.9 + '@aws-sdk/nested-clients': 3.997.44 + '@aws-sdk/types': 3.974.5 + '@smithy/core': 3.33.3 + '@smithy/types': 4.17.2 + tslib: 2.8.1 + + '@aws-sdk/types@3.974.5': + dependencies: + '@smithy/types': 4.17.2 + tslib: 2.8.1 + + '@aws-sdk/xml-builder@3.972.40': + dependencies: + '@smithy/types': 4.17.2 + tslib: 2.8.1 + + '@aws/lambda-invoke-store@0.3.0': {} + '@babel/code-frame@7.29.7': dependencies: '@babel/helper-validator-identifier': 7.29.7 @@ -2756,6 +3023,39 @@ snapshots: '@rolldown/pluginutils@1.0.1': {} + '@smithy/core@3.33.3': + dependencies: + '@smithy/types': 4.17.2 + tslib: 2.8.1 + + '@smithy/credential-provider-imds@4.5.2': + dependencies: + '@smithy/core': 3.33.3 + '@smithy/types': 4.17.2 + tslib: 2.8.1 + + '@smithy/fetch-http-handler@5.7.2': + dependencies: + '@smithy/core': 3.33.3 + '@smithy/types': 4.17.2 + tslib: 2.8.1 + + '@smithy/node-http-handler@4.11.3': + dependencies: + '@smithy/core': 3.33.3 + '@smithy/types': 4.17.2 + tslib: 2.8.1 + + '@smithy/signature-v4@5.7.3': + dependencies: + '@smithy/core': 3.33.3 + '@smithy/types': 4.17.2 + tslib: 2.8.1 + + '@smithy/types@4.17.2': + dependencies: + tslib: 2.8.1 + '@standard-schema/spec@1.1.0': {} '@turbo/darwin-64@2.10.4': @@ -3012,6 +3312,8 @@ snapshots: optionalDependencies: zod: 4.4.3 + bowser@2.14.1: {} + browserslist@4.28.6: dependencies: baseline-browser-mapping: 2.10.44 @@ -3454,8 +3756,7 @@ snapshots: tinyrainbow@3.1.0: {} - tslib@2.8.1: - optional: true + tslib@2.8.1: {} tsx@4.23.1: dependencies: diff --git a/scripts/test-migrations-integration.sh b/scripts/test-migrations-integration.sh index 40b609c..a476323 100755 --- a/scripts/test-migrations-integration.sh +++ b/scripts/test-migrations-integration.sh @@ -67,8 +67,13 @@ if ! "${compose[@]}" config --quiet >/dev/null 2>&1; then echo "PostgreSQL migration test infrastructure configuration is invalid." >&2 exit 1 fi -if ! "${compose[@]}" up --detach --wait --wait-timeout 120 postgres >/dev/null 2>&1; then - echo "Disposable PostgreSQL did not become ready." >&2 +# PostgreSQL and the S3-compatible object-storage service are both brought up from the repository's +# own pinned Compose file, so the live suites exercise exactly the services a developer runs locally +# and CI already knows how to provision. SeaweedFS is the intended deployment target (ADR 0004), and +# proving the storage contract against a mock instead would have hidden the provider quirks this +# suite exists to catch. +if ! "${compose[@]}" up --detach --wait --wait-timeout 180 postgres object-storage >/dev/null 2>&1; then + echo "Disposable PostgreSQL and object storage did not become ready." >&2 exit 1 fi @@ -77,8 +82,13 @@ if [[ "$COMPOSE_JOB_NETWORK_RUNNING_IN_CONTAINER" == "true" ]]; then echo "The migration test job could not join the isolated PostgreSQL network." >&2 exit 1 fi + if ! compose_job_network_connect object-storage "${compose[@]}"; then + echo "The migration test job could not join the isolated object-storage network." >&2 + exit 1 + fi database_host="postgres" database_port="5432" + storage_endpoint="http://object-storage:8333" else if ! postgres_binding="$("${compose[@]}" port postgres 5432 2>/dev/null)"; then echo "The disposable PostgreSQL loopback port could not be determined." >&2 @@ -94,11 +104,31 @@ else echo "The disposable PostgreSQL loopback port is invalid." >&2 exit 1 fi + + if ! storage_binding="$("${compose[@]}" port object-storage 8333 2>/dev/null)"; then + echo "The disposable object-storage loopback port could not be determined." >&2 + exit 1 + fi + if [[ ! "$storage_binding" =~ ^127\.0\.0\.1:([0-9]+)$ ]]; then + echo "The disposable object-storage port is not bound safely to loopback." >&2 + exit 1 + fi + storage_port="${BASH_REMATCH[1]}" + if ((storage_port < 1 || storage_port > 65535)); then + echo "The disposable object-storage loopback port is invalid." >&2 + exit 1 + fi + storage_endpoint="http://127.0.0.1:$storage_port" fi # The TLS suite provisions its own disposable PostgreSQL rather than a Compose service, so it # cannot call `compose_job_network_connect`. It receives the same topology decision this script # already made, instead of detecting the topology a second time. +# The storage credentials come from `.env.example`, the same file `--env-file` already hands the +# Compose services, so the suite signs requests with exactly what the service was started with. EKKLESIA_TEST_JOB_CONTAINER="$(compose_job_network_job_container)" \ EKKLESIA_MIGRATION_TEST_DATABASE_URL="postgresql://$POSTGRES_USER:$POSTGRES_PASSWORD@$database_host:$database_port/$POSTGRES_DB" \ +EKKLESIA_TEST_STORAGE_ENDPOINT="$storage_endpoint" \ +EKKLESIA_TEST_STORAGE_ACCESS_KEY="$(grep -E '^S3_ACCESS_KEY=' .env.example | cut -d= -f2-)" \ +EKKLESIA_TEST_STORAGE_SECRET_KEY="$(grep -E '^S3_SECRET_KEY=' .env.example | cut -d= -f2-)" \ pnpm exec vitest run --config vitest.migrations.integration.config.ts diff --git a/templates/module/ekklesia.module.json b/templates/module/ekklesia.module.json index cf12b89..966350e 100644 --- a/templates/module/ekklesia.module.json +++ b/templates/module/ekklesia.module.json @@ -30,14 +30,6 @@ "description": "Prefix this module puts in front of a record value." } ], - "storage": { - "buckets": [ - { - "id": "records", - "description": "Private objects owned by this example module." - } - ] - }, "entrypoints": { "server": "./dist/server/index.js", "schema": "./dist/schema/index.js", diff --git a/templates/module/tests/manifest.test.ts b/templates/module/tests/manifest.test.ts index 757a3d8..26e6f30 100644 --- a/templates/module/tests/manifest.test.ts +++ b/templates/module/tests/manifest.test.ts @@ -4,6 +4,7 @@ import { ANONYMOUS_PRINCIPAL, type CapabilityId, EMPTY_MODULE_CONFIGURATION, + EMPTY_MODULE_STORAGE, type ModuleConfiguration, type ModuleDatabase, type ModuleRouteDefinitionInput, @@ -69,7 +70,11 @@ describe("module template", () => { description: "Read example records.", }, ]); - expect(manifest.storage.buckets.map((bucket) => bucket.id)).toEqual(["records"]); + // The template declares **no** logical storage bucket, deliberately. Since ADR 0030 a declared + // bucket is a real requirement: a module that declares one is blocked on an installation with no + // object-storage provider. A starting point that could not run without S3 configured would be a + // poor starting point, so storage is something a module author opts into. + expect(manifest.storage.buckets).toEqual([]); expect(manifest.entrypoints.schema).toBe("./dist/schema/index.js"); expect(manifest.entrypoints.migrations).toBe("./drizzle"); }); @@ -134,6 +139,9 @@ describe("module template", () => { params: Object.freeze({}), principal: ANONYMOUS_PRINCIPAL, services, + // The template declares no logical bucket, so it receives storage that refuses every one — + // present rather than `undefined`, so module code never branches on whether the host has it. + storage: EMPTY_MODULE_STORAGE, }); const response = await route.handler(new Request("http://localhost/records"), context); @@ -175,6 +183,7 @@ describe("module template", () => { params: Object.freeze({}), principal: ANONYMOUS_PRINCIPAL, services: Object.freeze({ database: fakeModuleDatabase(fakeRows) }), + storage: EMPTY_MODULE_STORAGE, }); const response = await route.handler(new Request("http://localhost/records"), context); diff --git a/tests/infrastructure.test.ts b/tests/infrastructure.test.ts index 2f98820..c2cd094 100644 --- a/tests/infrastructure.test.ts +++ b/tests/infrastructure.test.ts @@ -138,8 +138,15 @@ describe("development infrastructure", () => { ); expect(source).toContain('source "$repository_root/scripts/lib/compose-job-network.sh"'); expect(source).toContain('export POSTGRES_PORT="0"'); - expect(source).toContain("up --detach --wait --wait-timeout 120 postgres"); + // Both live services come from the repository's own pinned Compose file, so the integration + // suites exercise exactly what a developer runs locally (ADR 0030 added the object-storage half). + expect(source).toContain("up --detach --wait --wait-timeout 180 postgres object-storage"); expect(source).toContain(`compose_job_network_connect postgres "\${compose[@]}"`); + expect(source).toContain(`compose_job_network_connect object-storage "\${compose[@]}"`); + // The storage suite receives an endpoint and the same credentials the service was started with, + // and never a hardcoded host. + expect(source).toContain("EKKLESIA_TEST_STORAGE_ENDPOINT="); + expect(source).toContain("EKKLESIA_TEST_STORAGE_ACCESS_KEY="); expect(source).toContain('database_host="postgres"'); expect(source).toContain('database_port="5432"'); expect(source).toContain('database_host="127.0.0.1"'); diff --git a/vitest.migrations.integration.config.ts b/vitest.migrations.integration.config.ts index a6d7ac5..6d7fe0f 100644 --- a/vitest.migrations.integration.config.ts +++ b/vitest.migrations.integration.config.ts @@ -12,6 +12,7 @@ export default defineConfig({ "core/tests/dashboard-store.integration.ts", "core/tests/module-lifecycle-store.integration.ts", "core/tests/module-configuration-store.integration.ts", + "core/tests/module-storage.integration.ts", "apps/api/tests/api-http.integration.ts", "apps/api/tests/module-database.integration.ts", ],