feat: add host-provided module object storage
`manifest.storage.buckets` has existed since manifest version 1 and meant nothing — discovery validated it and no code read it. This turns it into a real capability, which is what stands between Announcements and the module system: that product currently fetches a hardcoded LAN S3 endpoint, parses ListObjectsV2 XML by hand, and hands the browser direct object URLs. - A module declares a logical bucket and receives a scoped facade at `context.storage` — list, head, get, put, delete. The contract names no provider: no endpoint, region, credential, or physical bucket is reachable from it, which is what lets the same module run under a future standalone host over its own or a shared S3-compatible service. - Core's first provider speaks S3 through the AWS SDK. SeaweedFS is the deployment target (ADR 0004), never an API name. Probing the pinned image found `CreateBucket` reporting `BucketAlreadyExists` rather than AWS's `BucketAlreadyOwnedByYou`; treating only the AWS spelling as success would have failed every second boot. A live test pins it. - Logical → physical mapping is deterministic and collision-safe: an installation prefix, the module and bucket ids joined by `--` (which cannot occur inside either identifier, so `a-b`+`c` and `a`+`b-c` cannot collide), and a digest over the full untruncated identity. Two modules declaring `assets` always get separate namespaces; neither can see the physical name. - Declaring a bucket is now a requirement: with no provider configured the module is `blocked · storage_unavailable` through ADR 0028's existing resolver, propagating to required dependents. An installation with no object storage stays a complete deployment. - Binding availability is not provider health. Eligibility asks whether storage is configured, never whether it answered; bucket creation is lazy on first use so a storage outage cannot stop the platform booting; a running module whose store blinks stays running and retries. Nothing deletes a bucket. - Credentials are deployment-supplied, never persisted, never routed through module configuration, and never visible to modules, module UI, diagnostics, or API responses. Provider errors translate to a closed code set carrying no endpoint, bucket, request id, or XML. - Retires ADR 0029's transitional `bucket` configuration key. No manifest version change. The template declares no bucket, so it still runs without S3. See ADR 0030. Co-Authored-By: Claude Opus 5 <[email protected]>
This commit is contained in:
co-authored by
Claude Opus 5
parent
527239f6de
commit
b9ca0d560d
@@ -5,6 +5,8 @@ import {
|
||||
createModuleLifecycleDatabase,
|
||||
createModuleLifecycleStore,
|
||||
discoverModules,
|
||||
isDependencyBlockedDetail,
|
||||
isStorageBlockedDetail,
|
||||
ModuleDiscoveryError,
|
||||
ModuleLifecycleStoreError,
|
||||
resolveModuleEligibility,
|
||||
@@ -132,9 +134,13 @@ async function main(): Promise<void> {
|
||||
if (!blocked) {
|
||||
continue;
|
||||
}
|
||||
// Names the dependency or the configuration key, never a configuration value.
|
||||
const cause =
|
||||
"dependency" in blocked ? blocked.dependency.moduleId : blocked.configurationKey;
|
||||
// Names the dependency, the configuration key, or the logical storage bucket — never a
|
||||
// configuration value, an endpoint, or a credential.
|
||||
const cause = isDependencyBlockedDetail(blocked)
|
||||
? blocked.dependency.moduleId
|
||||
: isStorageBlockedDetail(blocked)
|
||||
? blocked.storageBucketId
|
||||
: blocked.configurationKey;
|
||||
console.log(`- "${entry.moduleId}" will not start: ${blocked.reason} (${cause}).`);
|
||||
}
|
||||
} finally {
|
||||
|
||||
@@ -21,7 +21,7 @@ const sections: readonly DoctorSection[] = Object.freeze([
|
||||
section("Data definitions", ["schemas.discovery", "migrations.discovery"]),
|
||||
section("PostgreSQL", ["database.transport", "postgresql.connection", "postgresql.plan"]),
|
||||
section("Identity", ["identity.configuration", "identity.client-address"]),
|
||||
section("Object storage", ["object-storage.integration"]),
|
||||
section("Object storage", ["object-storage.integration", "object-storage.bindings"]),
|
||||
]);
|
||||
|
||||
const statusSymbols: Readonly<Record<DoctorCheck["status"], string>> = Object.freeze({
|
||||
|
||||
+149
-10
@@ -3,6 +3,7 @@ import {
|
||||
capabilityIdSchema,
|
||||
EMPTY_MODULE_SECRET_SOURCE,
|
||||
isVersionCompatible,
|
||||
logicalBucketIdSchema,
|
||||
moduleConfigurationKeySchema,
|
||||
moduleIdSchema,
|
||||
type ModuleSecretSource,
|
||||
@@ -42,8 +43,16 @@ import {
|
||||
type PostgresMigrationDatabaseOptions,
|
||||
planModuleMigrations,
|
||||
} from "../migrations/index.js";
|
||||
import {
|
||||
createS3StorageProvider,
|
||||
describeStorageProvider,
|
||||
physicalBucketName,
|
||||
type StorageProviderConfig,
|
||||
} from "../storage/index.js";
|
||||
import {
|
||||
composePlatformPermissionRegistry,
|
||||
isConfigurationBlockedDetail,
|
||||
isStorageBlockedDetail,
|
||||
createModuleConfigurationDatabase,
|
||||
createModuleConfigurationStore,
|
||||
createModuleLifecycleDatabase,
|
||||
@@ -75,7 +84,8 @@ export type DoctorCheckId =
|
||||
| "postgresql.plan"
|
||||
| "identity.configuration"
|
||||
| "identity.client-address"
|
||||
| "object-storage.integration";
|
||||
| "object-storage.integration"
|
||||
| "object-storage.bindings";
|
||||
|
||||
export type DoctorCheckStatus = "failure" | "pass" | "skipped" | "warning";
|
||||
export type DoctorResult = "not-ready" | "ready" | "ready-with-warnings";
|
||||
@@ -113,6 +123,9 @@ export interface DoctorOptions {
|
||||
* matches what `startApi` would resolve. Omitted supplies nothing — Doctor then reports a required
|
||||
* secret as missing, which is the honest answer for a deployment that has not set it. */
|
||||
readonly moduleSecrets?: ModuleSecretSource;
|
||||
/** The validated host storage provider, or `undefined` when this installation has none. Doctor
|
||||
* never receives raw environment values and never prints a credential. */
|
||||
readonly storage?: StorageProviderConfig | undefined;
|
||||
readonly runtimeRange: string;
|
||||
readonly runtimeVersion: string;
|
||||
}
|
||||
@@ -150,6 +163,9 @@ export interface DoctorDependencies {
|
||||
readonly database: MigrationDatabase;
|
||||
}) => Promise<MigrationPlanResult>;
|
||||
readonly runtimeSatisfies: (version: string, range: string) => boolean;
|
||||
/** Probes the configured storage provider for reachability. Behind the dependency boundary like
|
||||
* every other I/O Doctor performs, so the whole report stays exercisable without a network. */
|
||||
readonly probeStorage: (config: StorageProviderConfig) => Promise<void>;
|
||||
}
|
||||
|
||||
const doctorPostgresTimeoutMs = 10_000;
|
||||
@@ -194,6 +210,16 @@ async function readModuleConfigurationForRuntime(
|
||||
}
|
||||
}
|
||||
|
||||
/** Opens a provider client, probes, and always closes it — Doctor never leaves a connection behind. */
|
||||
async function probeStorageForRuntime(config: StorageProviderConfig): Promise<void> {
|
||||
const provider = createS3StorageProvider(config);
|
||||
try {
|
||||
await provider.probe();
|
||||
} finally {
|
||||
await provider.close().catch(() => undefined);
|
||||
}
|
||||
}
|
||||
|
||||
const productionDependencies: DoctorDependencies = Object.freeze({
|
||||
composePermissions: composePlatformPermissionRegistry,
|
||||
createMigrationDatabase: createPostgresMigrationDatabase,
|
||||
@@ -201,6 +227,7 @@ const productionDependencies: DoctorDependencies = Object.freeze({
|
||||
discoverModules,
|
||||
discoverSchemas: discoverModuleSchemas,
|
||||
planMigrations: planModuleMigrations,
|
||||
probeStorage: probeStorageForRuntime,
|
||||
readModuleConfiguration: readModuleConfigurationForRuntime,
|
||||
readModuleLifecycle: readModuleLifecycleForRuntime,
|
||||
runtimeSatisfies: isVersionCompatible,
|
||||
@@ -333,7 +360,17 @@ function safeLifecycleDetail(entry: ModuleEligibility): string {
|
||||
if (!entry.blocked) {
|
||||
return `${identity} · enabled`;
|
||||
}
|
||||
if (!("dependency" in entry.blocked)) {
|
||||
if (isStorageBlockedDetail(entry.blocked)) {
|
||||
// A storage block names the **logical** bucket the module declared — never an endpoint, a
|
||||
// credential, or the physical bucket the host maps it to. Re-validated against the same grammar
|
||||
// the manifest accepted it under, for the same defense-in-depth reason as every other
|
||||
// interpolation here.
|
||||
if (!logicalBucketIdSchema.safeParse(entry.blocked.storageBucketId).success) {
|
||||
throw new TypeError("Unsafe module lifecycle diagnostic value.");
|
||||
}
|
||||
return `${identity} · blocked · ${entry.blocked.reason} · ${entry.blocked.storageBucketId}`;
|
||||
}
|
||||
if (isConfigurationBlockedDetail(entry.blocked)) {
|
||||
// A configuration block names the offending **key** and never its value: a key is declared in a
|
||||
// public manifest, while a value may be a credential or arbitrary operator-entered text. The key
|
||||
// is re-validated against the same grammar the manifest accepted it under, for the same
|
||||
@@ -407,7 +444,7 @@ function safeConfigurationDetail(entry: ModuleEligibility, declaredCount: number
|
||||
return `${identity} · no configuration declared`;
|
||||
}
|
||||
const declared = `${declaredCount} ${plural(declaredCount, "key")} declared`;
|
||||
if (entry.blocked && !("dependency" in entry.blocked)) {
|
||||
if (entry.blocked && isConfigurationBlockedDetail(entry.blocked)) {
|
||||
if (!moduleConfigurationKeySchema.safeParse(entry.blocked.configurationKey).success) {
|
||||
throw new TypeError("Unsafe module configuration diagnostic value.");
|
||||
}
|
||||
@@ -458,6 +495,65 @@ function moduleConfigurationCheck(
|
||||
return freezeCheck("modules.configuration", "pass", "Module configuration valid", details);
|
||||
}
|
||||
|
||||
/**
|
||||
* Reports which installed modules require object storage, and where the host keeps each binding.
|
||||
*
|
||||
* Names the module, the **logical** bucket it declared, and the physical bucket the host maps it to
|
||||
* — the last of which is safe and useful here precisely because Doctor is an operator tool: someone
|
||||
* debugging a permissions problem needs to know which bucket to look at. It is still never reachable
|
||||
* from module-facing code (ADR 0030). No endpoint, access key, or secret key appears in any form.
|
||||
*/
|
||||
function storageBindingsCheck(
|
||||
modules: readonly DiscoveredModule[],
|
||||
entries: readonly ModuleEligibility[],
|
||||
config: StorageProviderConfig | undefined,
|
||||
): DoctorCheck {
|
||||
const requiring = modules.filter((module) => module.manifest.storage.buckets.length > 0);
|
||||
if (requiring.length === 0) {
|
||||
// An installation whose modules need no object storage is not unhealthy for lacking a provider.
|
||||
return freezeCheck("object-storage.bindings", "pass", "No module requires object storage", [
|
||||
"0 modules declare a logical storage bucket.",
|
||||
]);
|
||||
}
|
||||
|
||||
const blocked = entries.filter(
|
||||
(entry) => entry.blocked !== undefined && isStorageBlockedDetail(entry.blocked),
|
||||
);
|
||||
const details = requiring.flatMap((module) => {
|
||||
const moduleId = module.manifest.id;
|
||||
if (!moduleIdSchema.safeParse(moduleId).success) {
|
||||
throw new TypeError("Unsafe module storage diagnostic value.");
|
||||
}
|
||||
return module.manifest.storage.buckets.map((bucket) => {
|
||||
if (!logicalBucketIdSchema.safeParse(bucket.id).success) {
|
||||
throw new TypeError("Unsafe module storage diagnostic value.");
|
||||
}
|
||||
return config === undefined
|
||||
? `${moduleId} · ${bucket.id} · unbound (no storage provider configured)`
|
||||
: `${moduleId} · ${bucket.id} → ${physicalBucketName(config.bucketPrefix, moduleId, bucket.id)}`;
|
||||
});
|
||||
});
|
||||
|
||||
if (config === undefined) {
|
||||
return freezeCheck(
|
||||
"object-storage.bindings",
|
||||
"failure",
|
||||
blocked.length === 1
|
||||
? "1 module requires object storage that is not configured"
|
||||
: `${blocked.length} modules require object storage that is not configured`,
|
||||
[
|
||||
"Configure an S3-compatible Ekklesia storage provider (S3_ENDPOINT, S3_ACCESS_KEY, S3_SECRET_KEY).",
|
||||
...details,
|
||||
],
|
||||
);
|
||||
}
|
||||
return freezeCheck(
|
||||
"object-storage.bindings",
|
||||
"pass",
|
||||
`${details.length} ${plural(details.length, "storage binding")} resolved`,
|
||||
details,
|
||||
);
|
||||
}
|
||||
function moduleDiscoveryFailure(error: unknown): DoctorCheck {
|
||||
const details =
|
||||
error instanceof Error && error.name === "ModuleDiscoveryError"
|
||||
@@ -788,6 +884,9 @@ export async function diagnoseEkklesia(
|
||||
// failed. It needs only a successful discovery and a configured database.
|
||||
let lifecycleCheck: DoctorCheck;
|
||||
let configurationCheck: DoctorCheck;
|
||||
// Hoisted so the storage-bindings check below reports the same eligibility this pass resolved,
|
||||
// rather than resolving a second time and risking a different answer.
|
||||
let eligibilityEntries: readonly ModuleEligibility[] = [];
|
||||
const skippedConfiguration = (detail: string): DoctorCheck =>
|
||||
freezeCheck("modules.configuration", "skipped", "Module configuration skipped", [detail]);
|
||||
if (!discoveredModules) {
|
||||
@@ -819,10 +918,13 @@ export async function diagnoseEkklesia(
|
||||
poolOptions,
|
||||
);
|
||||
const settings = await dependencies.readModuleConfiguration(options.databaseUrl, poolOptions);
|
||||
const resolution = resolveModuleEligibility(discoveredModules, configuration, {
|
||||
secrets: options.moduleSecrets ?? EMPTY_MODULE_SECRET_SOURCE,
|
||||
settings,
|
||||
});
|
||||
const resolution = resolveModuleEligibility(
|
||||
discoveredModules,
|
||||
configuration,
|
||||
{ secrets: options.moduleSecrets ?? EMPTY_MODULE_SECRET_SOURCE, settings },
|
||||
{ providerConfigured: options.storage !== undefined },
|
||||
);
|
||||
eligibilityEntries = resolution.entries;
|
||||
lifecycleCheck = moduleLifecycleCheck(resolution.entries);
|
||||
configurationCheck = moduleConfigurationCheck(
|
||||
resolution.entries,
|
||||
@@ -930,9 +1032,45 @@ export async function diagnoseEkklesia(
|
||||
options.databaseEnvironment,
|
||||
);
|
||||
|
||||
const storageCheck = freezeCheck("object-storage.integration", "skipped", "Check unavailable", [
|
||||
"Core storage integration has not been implemented yet.",
|
||||
]);
|
||||
// Two separate questions, deliberately. "Is a provider configured and reachable?" is
|
||||
// infrastructure diagnostics; "can every module's declared bucket be bound?" is what decides
|
||||
// whether modules run. Conflating them would let a momentary network blip read as a module
|
||||
// problem (ADR 0030).
|
||||
let storageCheck: DoctorCheck;
|
||||
if (!options.storage) {
|
||||
storageCheck = freezeCheck(
|
||||
"object-storage.integration",
|
||||
"skipped",
|
||||
"Object storage not configured",
|
||||
["No S3_ENDPOINT is set. This is a complete deployment unless a module requires storage."],
|
||||
);
|
||||
} else {
|
||||
try {
|
||||
await dependencies.probeStorage(options.storage);
|
||||
storageCheck = freezeCheck("object-storage.integration", "pass", "Object storage reachable", [
|
||||
describeStorageProvider(options.storage),
|
||||
]);
|
||||
} catch {
|
||||
// A warning, never a failure: the provider is configured, and a module that is already
|
||||
// running keeps running through a transient outage. Reporting this as `not-ready` would make
|
||||
// the platform look broken because a network hiccup happened while Doctor ran.
|
||||
storageCheck = freezeCheck(
|
||||
"object-storage.integration",
|
||||
"warning",
|
||||
"Object storage unreachable",
|
||||
[
|
||||
describeStorageProvider(options.storage),
|
||||
"The configured storage service did not respond. Modules already running are unaffected.",
|
||||
],
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
const storageBindings = discoveredModules
|
||||
? storageBindingsCheck(discoveredModules, eligibilityEntries, options.storage)
|
||||
: freezeCheck("object-storage.bindings", "skipped", "Storage bindings skipped", [
|
||||
"Complete module discovery is required.",
|
||||
]);
|
||||
|
||||
return publishReport([
|
||||
runtimeCheck,
|
||||
@@ -949,5 +1087,6 @@ export async function diagnoseEkklesia(
|
||||
identityCheck,
|
||||
clientAddressDoctorCheck,
|
||||
storageCheck,
|
||||
storageBindings,
|
||||
]);
|
||||
}
|
||||
|
||||
@@ -62,7 +62,59 @@ export interface ModuleConfigurationBlockedDetail {
|
||||
readonly configurationKey: string;
|
||||
}
|
||||
|
||||
export type ModuleBlockedDetail = ModuleDependencyBlockedDetail | ModuleConfigurationBlockedDetail;
|
||||
/** A block caused by unsatisfiable object-storage requirements (ADR 0030), naming the **logical**
|
||||
* bucket the module declared — never an endpoint, a credential, or a physical bucket name. */
|
||||
export interface ModuleStorageBlockedDetail {
|
||||
readonly reason: "storage_unavailable";
|
||||
readonly storageBucketId: string;
|
||||
}
|
||||
|
||||
export type ModuleBlockedDetail =
|
||||
| ModuleDependencyBlockedDetail
|
||||
| ModuleConfigurationBlockedDetail
|
||||
| ModuleStorageBlockedDetail;
|
||||
|
||||
/** Narrows a block to the dependency variant. Exported so every consumer discriminates the same
|
||||
* way — a bare `"dependency" in detail` repeated at each call site is how one of them eventually
|
||||
* gets it wrong as variants are added. */
|
||||
export function isDependencyBlockedDetail(
|
||||
detail: ModuleBlockedDetail,
|
||||
): detail is ModuleDependencyBlockedDetail {
|
||||
return (
|
||||
detail.reason === "required_dependency_disabled" ||
|
||||
detail.reason === "required_dependency_blocked"
|
||||
);
|
||||
}
|
||||
|
||||
export function isStorageBlockedDetail(
|
||||
detail: ModuleBlockedDetail,
|
||||
): detail is ModuleStorageBlockedDetail {
|
||||
return detail.reason === "storage_unavailable";
|
||||
}
|
||||
|
||||
export function isConfigurationBlockedDetail(
|
||||
detail: ModuleBlockedDetail,
|
||||
): detail is ModuleConfigurationBlockedDetail {
|
||||
return (
|
||||
detail.reason === "configuration_missing" ||
|
||||
detail.reason === "configuration_invalid" ||
|
||||
detail.reason === "secret_missing"
|
||||
);
|
||||
}
|
||||
|
||||
/**
|
||||
* Whether this host can bind the logical object-storage buckets modules declare (ADR 0030).
|
||||
*
|
||||
* A boolean, deliberately, and resolved without touching the network: eligibility must stay a pure,
|
||||
* synchronous function, and it must never depend on whether the storage service happened to answer a
|
||||
* probe during boot. A configured provider that is temporarily unreachable is an operational
|
||||
* condition a module retries through, not an administrative state change — which is the distinction
|
||||
* the real Announcements product already relies on, since its slideshow survives its object store
|
||||
* going away.
|
||||
*/
|
||||
export interface ModuleStorageAvailability {
|
||||
readonly providerConfigured: boolean;
|
||||
}
|
||||
|
||||
export interface ModuleEligibilityResolution {
|
||||
/**
|
||||
@@ -108,10 +160,15 @@ function frozenEligibility(entry: MutableEligibility): ModuleEligibility {
|
||||
dependency: Object.freeze({ ...entry.blocked.dependency }),
|
||||
reason: entry.blocked.reason,
|
||||
})
|
||||
: Object.freeze({
|
||||
configurationKey: entry.blocked.configurationKey,
|
||||
reason: entry.blocked.reason,
|
||||
}),
|
||||
: "storageBucketId" in entry.blocked
|
||||
? Object.freeze({
|
||||
reason: entry.blocked.reason,
|
||||
storageBucketId: entry.blocked.storageBucketId,
|
||||
})
|
||||
: Object.freeze({
|
||||
configurationKey: entry.blocked.configurationKey,
|
||||
reason: entry.blocked.reason,
|
||||
}),
|
||||
}
|
||||
: {}),
|
||||
});
|
||||
@@ -162,6 +219,7 @@ export function resolveModuleEligibility(
|
||||
discoveredModules: readonly DiscoveredModule[],
|
||||
configuration: ModuleLifecycleConfiguration,
|
||||
moduleConfiguration: ModuleConfigurationInput = { settings: { records: [] } },
|
||||
storage: ModuleStorageAvailability = { providerConfigured: false },
|
||||
): ModuleEligibilityResolution {
|
||||
const configuredByModuleId = new Map(
|
||||
configuration.modules.map((record) => [record.moduleId, record.enabled] as const),
|
||||
@@ -180,6 +238,26 @@ export function resolveModuleEligibility(
|
||||
});
|
||||
}
|
||||
|
||||
// A module that declares logical object-storage buckets needs a host provider to bind them to. The
|
||||
// check is deliberately about *binding availability*, not provider health: it asks whether this
|
||||
// deployment has storage configured at all, never whether the service answered just now
|
||||
// (ADR 0030). Runs before the dependency pass so a storage-blocked module propagates to its
|
||||
// required dependents through exactly the existing mechanism rather than a second one.
|
||||
for (const discovered of discoveredModules) {
|
||||
const entry = entries.get(discovered.manifest.id);
|
||||
if (!entry?.eligible) {
|
||||
continue;
|
||||
}
|
||||
const declaredBucket = discovered.manifest.storage.buckets[0];
|
||||
if (declaredBucket !== undefined && !storage.providerConfigured) {
|
||||
entry.eligible = false;
|
||||
entry.blocked = {
|
||||
reason: "storage_unavailable",
|
||||
storageBucketId: declaredBucket.id,
|
||||
};
|
||||
}
|
||||
}
|
||||
|
||||
// Validate each enabled module's own configuration before the dependency pass, so a module
|
||||
// blocked by its own configuration propagates to its required dependents through exactly the
|
||||
// existing mechanism rather than a second one. Runs only for modules that are still eligible: a
|
||||
@@ -306,6 +384,9 @@ export interface ModuleLifecycleView {
|
||||
readonly dependency?: ModuleBlockingDependency;
|
||||
/** Present for a configuration reason (ADR 0029) — the offending key, never its value. */
|
||||
readonly configurationKey?: string;
|
||||
/** Present for a storage reason (ADR 0030) — the logical bucket the module declared, never an
|
||||
* endpoint, a credential, or a physical bucket name. */
|
||||
readonly storageBucketId?: string;
|
||||
};
|
||||
/**
|
||||
* `true` when applying the *current* configuration would produce a different running set than
|
||||
@@ -347,6 +428,7 @@ export function resolveModuleLifecycleView(
|
||||
snapshot: ModuleLifecycleSnapshot,
|
||||
configuration: ModuleLifecycleConfiguration,
|
||||
moduleConfiguration: ModuleConfigurationInput = { settings: { records: [] } },
|
||||
storage: ModuleStorageAvailability = { providerConfigured: false },
|
||||
): readonly ModuleLifecycleView[] {
|
||||
const started = new Set(snapshot.startedModuleIds);
|
||||
const bootEligibility = new Map(
|
||||
@@ -361,6 +443,7 @@ export function resolveModuleLifecycleView(
|
||||
snapshot.discoveredModules,
|
||||
configuration,
|
||||
moduleConfiguration,
|
||||
storage,
|
||||
);
|
||||
const wouldRun = new Set(pending.eligibleModules.map((module) => module.manifest.id));
|
||||
|
||||
@@ -379,11 +462,17 @@ export function resolveModuleLifecycleView(
|
||||
reason: boot.blocked.reason,
|
||||
state: "blocked" as const,
|
||||
}
|
||||
: {
|
||||
configurationKey: boot.blocked.configurationKey,
|
||||
reason: boot.blocked.reason,
|
||||
state: "blocked" as const,
|
||||
}
|
||||
: "storageBucketId" in boot.blocked
|
||||
? {
|
||||
reason: boot.blocked.reason,
|
||||
state: "blocked" as const,
|
||||
storageBucketId: boot.blocked.storageBucketId,
|
||||
}
|
||||
: {
|
||||
configurationKey: boot.blocked.configurationKey,
|
||||
reason: boot.blocked.reason,
|
||||
state: "blocked" as const,
|
||||
}
|
||||
: // Enabled and unblocked at boot, yet not started: unreachable while startup stays
|
||||
// all-or-nothing, and reported as blocked-by-nothing rather than as `running`, so a
|
||||
// future change to startup atomicity can never make this silently claim success.
|
||||
|
||||
@@ -2,7 +2,9 @@ import { pathToFileURL } from "node:url";
|
||||
import {
|
||||
type CapabilityId,
|
||||
EMPTY_MODULE_CONFIGURATION,
|
||||
EMPTY_MODULE_STORAGE,
|
||||
type ModuleConfiguration,
|
||||
type ModuleStorage,
|
||||
type CapabilityRegistrar,
|
||||
type CapabilityResolver,
|
||||
capabilityIdSchema,
|
||||
@@ -96,6 +98,20 @@ function configurationFor(
|
||||
return configurations.get(moduleId) ?? EMPTY_MODULE_CONFIGURATION;
|
||||
}
|
||||
|
||||
/**
|
||||
* Per-module scoped object storage, keyed by module id (ADR 0030).
|
||||
*
|
||||
* A lookup for the same reason configuration is one, and emphatically **not** a field on
|
||||
* `ModuleRuntimeServices`: `services` is one shared object identity every module receives, while a
|
||||
* storage facade is scoped to exactly the buckets one module declared. Sharing it would hand every
|
||||
* module every other module's storage.
|
||||
*/
|
||||
export type ModuleStorageLookup = ReadonlyMap<string, ModuleStorage>;
|
||||
|
||||
function storageFor(storages: ModuleStorageLookup, moduleId: string): ModuleStorage {
|
||||
return storages.get(moduleId) ?? EMPTY_MODULE_STORAGE;
|
||||
}
|
||||
|
||||
export interface ModuleRuntimeTimeoutOptions {
|
||||
/** Test-only narrowing of `MODULE_STOP_HOOK_TIMEOUT_MS`, applied to both normal stop and startup
|
||||
* rollback, so a deadline can be proven in milliseconds instead of ten seconds. Production
|
||||
@@ -115,6 +131,12 @@ export interface ModuleRuntimeCompositionOptions extends ModuleRuntimeTimeoutOpt
|
||||
* `EMPTY_MODULE_CONFIGURATION`.
|
||||
*/
|
||||
readonly configurations?: ModuleConfigurationLookup;
|
||||
/**
|
||||
* The per-module scoped storage facades this runtime hands to each module's `start`, `stop`, and
|
||||
* HTTP handler contexts (ADR 0030). Omitted by a caller exercising no storage; every module then
|
||||
* receives `EMPTY_MODULE_STORAGE`, which refuses every bucket.
|
||||
*/
|
||||
readonly storages?: ModuleStorageLookup;
|
||||
}
|
||||
|
||||
export interface StartModuleRuntimeOptions
|
||||
@@ -257,6 +279,7 @@ class StagedCapabilityRegistry {
|
||||
phase: "rollback" | "start" | "stop",
|
||||
services: ModuleRuntimeServices,
|
||||
config: ModuleConfiguration,
|
||||
storage: ModuleStorage,
|
||||
): ModuleRuntimeContext {
|
||||
const capabilities: CapabilityResolver = Object.freeze({
|
||||
resolve: (capabilityId: CapabilityId): RegisteredCapability | undefined => {
|
||||
@@ -264,7 +287,13 @@ class StagedCapabilityRegistry {
|
||||
return this.#registered.get(capabilityId);
|
||||
},
|
||||
});
|
||||
return Object.freeze({ capabilities, config, moduleId: module.manifest.id, services });
|
||||
return Object.freeze({
|
||||
capabilities,
|
||||
config,
|
||||
moduleId: module.manifest.id,
|
||||
services,
|
||||
storage,
|
||||
});
|
||||
}
|
||||
|
||||
publish(): CapabilityRegistry {
|
||||
@@ -599,6 +628,7 @@ async function stopModules(
|
||||
phase: "rollback" | "stop",
|
||||
services: ModuleRuntimeServices,
|
||||
configurations: ModuleConfigurationLookup,
|
||||
storages: ModuleStorageLookup,
|
||||
options: StopModulesOptions = {},
|
||||
): Promise<readonly ModuleRuntimeIssue[]> {
|
||||
const timeoutMs = options.hookTimeoutMs ?? MODULE_STOP_HOOK_TIMEOUT_MS;
|
||||
@@ -619,6 +649,7 @@ async function stopModules(
|
||||
phase,
|
||||
services,
|
||||
configurationFor(configurations, moduleId),
|
||||
storageFor(storages, moduleId),
|
||||
),
|
||||
),
|
||||
),
|
||||
@@ -657,6 +688,7 @@ async function startModules(
|
||||
registry: StagedCapabilityRegistry,
|
||||
services: ModuleRuntimeServices,
|
||||
configurations: ModuleConfigurationLookup,
|
||||
storages: ModuleStorageLookup,
|
||||
options: StopModulesOptions = {},
|
||||
): Promise<void> {
|
||||
// A module joins the rollback set the moment its `start` hook is *invoked*, not when it
|
||||
@@ -675,6 +707,7 @@ async function startModules(
|
||||
"start",
|
||||
services,
|
||||
configurationFor(configurations, module.discovered.manifest.id),
|
||||
storageFor(storages, module.discovered.manifest.id),
|
||||
),
|
||||
);
|
||||
} catch (error) {
|
||||
@@ -693,6 +726,7 @@ async function startModules(
|
||||
"rollback",
|
||||
services,
|
||||
configurations,
|
||||
storages,
|
||||
options,
|
||||
);
|
||||
throw new ModuleRuntimeError([startIssue, ...rollbackIssues]);
|
||||
@@ -717,8 +751,9 @@ export async function startDiscoveredModuleRuntime(
|
||||
const capabilityRegistry = new StagedCapabilityRegistry();
|
||||
const httpRoutes = new StagedModuleHttpRoutes();
|
||||
const configurations = timeouts.configurations ?? new Map<string, ModuleConfiguration>();
|
||||
const storages = timeouts.storages ?? new Map<string, ModuleStorage>();
|
||||
await registerModules(modules, capabilityRegistry, httpRoutes, configurations);
|
||||
await startModules(modules, capabilityRegistry, services, configurations, timeouts);
|
||||
await startModules(modules, capabilityRegistry, services, configurations, storages, timeouts);
|
||||
|
||||
const capabilities = capabilityRegistry.publish();
|
||||
const routes = httpRoutes.publish();
|
||||
@@ -744,6 +779,7 @@ export async function startDiscoveredModuleRuntime(
|
||||
"stop",
|
||||
services,
|
||||
configurations,
|
||||
storages,
|
||||
timeouts,
|
||||
);
|
||||
if (issues.length > 0) {
|
||||
@@ -761,6 +797,7 @@ export async function startModuleRuntime(
|
||||
const discovered = await discoverModules(options);
|
||||
return startDiscoveredModuleRuntime(discovered, options.services, {
|
||||
...(options.configurations === undefined ? {} : { configurations: options.configurations }),
|
||||
...(options.storages === undefined ? {} : { storages: options.storages }),
|
||||
...(options.hookTimeoutMs === undefined ? {} : { hookTimeoutMs: options.hookTimeoutMs }),
|
||||
});
|
||||
}
|
||||
|
||||
@@ -0,0 +1,116 @@
|
||||
import { createHash } from "node:crypto";
|
||||
|
||||
/**
|
||||
* How a module's logical bucket becomes a physical bucket on the host's storage service.
|
||||
*
|
||||
* This is **host implementation detail**. Nothing module-facing exposes the result: a module names
|
||||
* its logical bucket and the binding is resolved for it, so a module can neither observe nor choose
|
||||
* a physical bucket (ADR 0030).
|
||||
*/
|
||||
|
||||
/**
|
||||
* The prefix every physical bucket this installation owns begins with.
|
||||
*
|
||||
* It exists so several Ekklesia-compatible deployments — or an Ekklesia installation and an
|
||||
* unrelated application — can share one S3-compatible service without colliding, which is exactly
|
||||
* the deployment shape the standalone story needs. Bounded and grammar-checked because it becomes
|
||||
* the leading characters of an S3 bucket name.
|
||||
*/
|
||||
export const DEFAULT_STORAGE_BUCKET_PREFIX = "ekklesia";
|
||||
export const STORAGE_BUCKET_PREFIX_MAX_LENGTH = 16;
|
||||
const BUCKET_PREFIX_PATTERN = /^[a-z][a-z0-9]*(?:-[a-z0-9]+)*$/u;
|
||||
|
||||
export function isValidStorageBucketPrefix(value: unknown): value is string {
|
||||
return (
|
||||
typeof value === "string" &&
|
||||
value.length > 0 &&
|
||||
value.length <= STORAGE_BUCKET_PREFIX_MAX_LENGTH &&
|
||||
BUCKET_PREFIX_PATTERN.test(value)
|
||||
);
|
||||
}
|
||||
|
||||
/**
|
||||
* S3's own bucket-name ceiling, which every S3-compatible provider shares. Verified against the
|
||||
* pinned SeaweedFS 4.28 development image: a 63-character name is accepted.
|
||||
*/
|
||||
const PHYSICAL_BUCKET_MAX_LENGTH = 63;
|
||||
/** Long enough that a collision is not a practical concern, short enough to leave the readable part
|
||||
* of the name legible. */
|
||||
const DIGEST_LENGTH = 12;
|
||||
|
||||
/**
|
||||
* The unambiguous logical identity of one binding: a module and one of its declared buckets.
|
||||
*
|
||||
* `--` is the separator because it **cannot occur inside either identifier**. Both `moduleIdSchema`
|
||||
* and `logicalBucketIdSchema` require at least one `[a-z0-9]` after every hyphen, so a doubled
|
||||
* hyphen is unrepresentable in each half. A single hyphen would have been ambiguous in exactly the
|
||||
* way that matters: module `a-b` + bucket `c` and module `a` + bucket `b-c` would both render as
|
||||
* `a-b-c`, silently handing two unrelated modules the same physical namespace.
|
||||
*/
|
||||
function bindingIdentity(moduleId: string, logicalBucketId: string): string {
|
||||
return `${moduleId}--${logicalBucketId}`;
|
||||
}
|
||||
|
||||
/**
|
||||
* Maps a module's logical bucket to the physical bucket name the host uses for it.
|
||||
*
|
||||
* ```text
|
||||
* announcements + announcement-assets
|
||||
* → ekklesia-announcements--announcement-assets-3f0c1d9a6b21
|
||||
* ```
|
||||
*
|
||||
* Deterministic, so the same declaration always resolves to the same bucket across restarts,
|
||||
* deployments, and hosts — which is what makes the binding stable enough to hold data.
|
||||
*
|
||||
* Collision-safe on two independent grounds. The `--` separator makes the logical identity
|
||||
* unambiguous, and the appended digest is taken over that **full untruncated identity**, so even two
|
||||
* modules whose names truncate to the same readable prefix still receive different buckets. Two
|
||||
* different modules both declaring `assets` therefore always land in different physical namespaces —
|
||||
* which is the isolation property this whole scheme exists to provide.
|
||||
*
|
||||
* The readable middle is truncated only as far as S3's 63-character ceiling requires, and never at a
|
||||
* trailing hyphen, so the result always satisfies S3's naming rules: it begins with a letter, ends
|
||||
* with an alphanumeric, contains only lowercase letters, digits, and single hyphens, and can never
|
||||
* resemble an IP address.
|
||||
*/
|
||||
export function physicalBucketName(
|
||||
prefix: string,
|
||||
moduleId: string,
|
||||
logicalBucketId: string,
|
||||
): string {
|
||||
const identity = bindingIdentity(moduleId, logicalBucketId);
|
||||
const digest = createHash("sha256")
|
||||
.update(identity, "utf8")
|
||||
.digest("hex")
|
||||
.slice(0, DIGEST_LENGTH);
|
||||
// prefix + "-" + readable + "-" + digest
|
||||
const budget = PHYSICAL_BUCKET_MAX_LENGTH - prefix.length - 1 - 1 - DIGEST_LENGTH;
|
||||
const readable = identity.slice(0, Math.max(1, budget)).replace(/-+$/u, "");
|
||||
return `${prefix}-${readable}-${digest}`;
|
||||
}
|
||||
|
||||
/** One resolved binding: what the module calls it, and where the host actually keeps it. */
|
||||
export interface ModuleStorageBinding {
|
||||
readonly moduleId: string;
|
||||
readonly logicalBucketId: string;
|
||||
/** Host detail. Never reachable from `ModuleStorage`; present here because Core's provider needs
|
||||
* it, and because diagnostics may name it for an operator. */
|
||||
readonly physicalBucket: string;
|
||||
}
|
||||
|
||||
/** Resolves every binding one module's manifest declares, in declaration order. */
|
||||
export function resolveModuleStorageBindings(
|
||||
prefix: string,
|
||||
moduleId: string,
|
||||
logicalBucketIds: readonly string[],
|
||||
): readonly ModuleStorageBinding[] {
|
||||
return Object.freeze(
|
||||
logicalBucketIds.map((logicalBucketId) =>
|
||||
Object.freeze({
|
||||
logicalBucketId,
|
||||
moduleId,
|
||||
physicalBucket: physicalBucketName(prefix, moduleId, logicalBucketId),
|
||||
}),
|
||||
),
|
||||
);
|
||||
}
|
||||
@@ -0,0 +1,185 @@
|
||||
import {
|
||||
DEFAULT_STORAGE_BUCKET_PREFIX,
|
||||
isValidStorageBucketPrefix,
|
||||
STORAGE_BUCKET_PREFIX_MAX_LENGTH,
|
||||
} from "./binding.js";
|
||||
|
||||
export type StorageConfigErrorCode =
|
||||
| "STORAGE_ACCESS_KEY_MISSING"
|
||||
| "STORAGE_BUCKET_PREFIX_INVALID"
|
||||
| "STORAGE_ENDPOINT_INSECURE_IN_PRODUCTION"
|
||||
| "STORAGE_ENDPOINT_INVALID"
|
||||
| "STORAGE_ENDPOINT_WITH_CREDENTIALS"
|
||||
| "STORAGE_REGION_INVALID"
|
||||
| "STORAGE_SECRET_KEY_MISSING";
|
||||
|
||||
const storageConfigMessages: Readonly<Record<StorageConfigErrorCode, string>> = Object.freeze({
|
||||
STORAGE_ACCESS_KEY_MISSING: "A storage access key is required when a storage endpoint is set.",
|
||||
STORAGE_BUCKET_PREFIX_INVALID: "The storage bucket prefix must be lowercase kebab-case.",
|
||||
STORAGE_ENDPOINT_INSECURE_IN_PRODUCTION:
|
||||
"A production storage endpoint must use https, exactly as the database transport must use TLS.",
|
||||
STORAGE_ENDPOINT_INVALID: "The storage endpoint must be an absolute http or https URL.",
|
||||
STORAGE_ENDPOINT_WITH_CREDENTIALS:
|
||||
"The storage endpoint must not embed credentials; supply them separately.",
|
||||
STORAGE_REGION_INVALID: "The storage region must be a short alphanumeric identifier.",
|
||||
STORAGE_SECRET_KEY_MISSING: "A storage secret key is required when a storage endpoint is set.",
|
||||
});
|
||||
|
||||
/**
|
||||
* Never carries the endpoint, region, access key, secret key, or any other configured value — only a
|
||||
* fixed code and message. Matching `IdentityConfigError` and `DatabaseTransportError` exactly, and
|
||||
* for a sharper reason here: two of the values this validates are credentials.
|
||||
*/
|
||||
export class StorageConfigError extends Error {
|
||||
readonly code: StorageConfigErrorCode;
|
||||
|
||||
constructor(code: StorageConfigErrorCode) {
|
||||
super(storageConfigMessages[code]);
|
||||
this.name = "StorageConfigError";
|
||||
this.code = code;
|
||||
this.stack = `${this.name}: ${this.message}`;
|
||||
Object.freeze(this);
|
||||
}
|
||||
}
|
||||
|
||||
export type StorageEnvironment = "development" | "production";
|
||||
|
||||
export interface StorageProviderConfigInput {
|
||||
readonly endpoint?: string | undefined;
|
||||
readonly region?: string | undefined;
|
||||
readonly accessKey?: string | undefined;
|
||||
readonly secretKey?: string | undefined;
|
||||
readonly forcePathStyle?: string | boolean | undefined;
|
||||
readonly bucketPrefix?: string | undefined;
|
||||
}
|
||||
|
||||
/**
|
||||
* The validated host storage configuration.
|
||||
*
|
||||
* **Modules never see this.** It reaches Core's provider and nothing else; the module-facing
|
||||
* `ModuleStorage` contract has no accessor that could return an endpoint, a region, or a credential
|
||||
* (ADR 0030).
|
||||
*/
|
||||
export interface StorageProviderConfig {
|
||||
readonly endpoint: string;
|
||||
readonly region: string;
|
||||
readonly accessKey: string;
|
||||
readonly secretKey: string;
|
||||
/** True for every S3-compatible service that is not AWS itself — SeaweedFS and MinIO both require
|
||||
* it — so it defaults on. Virtual-host addressing needs wildcard DNS the typical self-hosted
|
||||
* deployment does not have. */
|
||||
readonly forcePathStyle: boolean;
|
||||
readonly bucketPrefix: string;
|
||||
}
|
||||
|
||||
const DEFAULT_REGION = "us-east-1";
|
||||
const REGION_PATTERN = /^[a-z0-9][a-z0-9-]{0,31}$/u;
|
||||
|
||||
function parseBoolean(value: string | boolean | undefined, fallback: boolean): boolean {
|
||||
if (value === undefined) {
|
||||
return fallback;
|
||||
}
|
||||
if (typeof value === "boolean") {
|
||||
return value;
|
||||
}
|
||||
return value !== "false";
|
||||
}
|
||||
|
||||
/**
|
||||
* Validates the host's storage configuration, or reports that storage is not configured at all.
|
||||
*
|
||||
* **Storage is optional.** An installation with no `endpoint` returns `undefined`, and that is a
|
||||
* complete, valid deployment: modules that declare no buckets run exactly as before. Only a module
|
||||
* that actually requires a bucket is affected, and it is reported blocked rather than crashing
|
||||
* anything (ADR 0030).
|
||||
*
|
||||
* Once an endpoint *is* set, credentials become required — a half-configured provider that silently
|
||||
* attempted anonymous access would be a worse failure than a loud one.
|
||||
*
|
||||
* Production requires `https`, on the same reasoning the database transport requires verified TLS:
|
||||
* object bytes for a church may include member photographs and pastoral material, and shipping them
|
||||
* over plaintext because it was convenient is not a tradeoff this platform makes. Development may
|
||||
* use `http`, which is what the repository's own SeaweedFS Compose service speaks.
|
||||
*
|
||||
* An endpoint carrying userinfo is rejected outright rather than stripped: credentials belong in
|
||||
* their own fields, where they are never logged, and a URL that embeds them tends to end up in
|
||||
* places a URL is printed.
|
||||
*/
|
||||
export function createStorageProviderConfig(
|
||||
input: StorageProviderConfigInput | undefined,
|
||||
options: { readonly environment?: StorageEnvironment } = {},
|
||||
): StorageProviderConfig | undefined {
|
||||
const endpoint = input?.endpoint?.trim();
|
||||
if (endpoint === undefined || endpoint === "") {
|
||||
return undefined;
|
||||
}
|
||||
|
||||
let parsed: URL;
|
||||
try {
|
||||
parsed = new URL(endpoint);
|
||||
} catch {
|
||||
throw new StorageConfigError("STORAGE_ENDPOINT_INVALID");
|
||||
}
|
||||
if (parsed.protocol !== "http:" && parsed.protocol !== "https:") {
|
||||
throw new StorageConfigError("STORAGE_ENDPOINT_INVALID");
|
||||
}
|
||||
if (parsed.username !== "" || parsed.password !== "") {
|
||||
throw new StorageConfigError("STORAGE_ENDPOINT_WITH_CREDENTIALS");
|
||||
}
|
||||
const environment = options.environment ?? "production";
|
||||
if (environment === "production" && parsed.protocol !== "https:") {
|
||||
throw new StorageConfigError("STORAGE_ENDPOINT_INSECURE_IN_PRODUCTION");
|
||||
}
|
||||
|
||||
const region = input?.region?.trim() === "" ? undefined : input?.region?.trim();
|
||||
if (region !== undefined && !REGION_PATTERN.test(region)) {
|
||||
throw new StorageConfigError("STORAGE_REGION_INVALID");
|
||||
}
|
||||
|
||||
const accessKey = input?.accessKey;
|
||||
if (typeof accessKey !== "string" || accessKey.trim() === "") {
|
||||
throw new StorageConfigError("STORAGE_ACCESS_KEY_MISSING");
|
||||
}
|
||||
const secretKey = input?.secretKey;
|
||||
if (typeof secretKey !== "string" || secretKey.trim() === "") {
|
||||
throw new StorageConfigError("STORAGE_SECRET_KEY_MISSING");
|
||||
}
|
||||
|
||||
const bucketPrefix = input?.bucketPrefix?.trim();
|
||||
if (
|
||||
bucketPrefix !== undefined &&
|
||||
bucketPrefix !== "" &&
|
||||
!isValidStorageBucketPrefix(bucketPrefix)
|
||||
) {
|
||||
throw new StorageConfigError("STORAGE_BUCKET_PREFIX_INVALID");
|
||||
}
|
||||
|
||||
return Object.freeze({
|
||||
accessKey,
|
||||
bucketPrefix:
|
||||
bucketPrefix === undefined || bucketPrefix === ""
|
||||
? DEFAULT_STORAGE_BUCKET_PREFIX
|
||||
: bucketPrefix,
|
||||
endpoint: parsed.origin + (parsed.pathname === "/" ? "" : parsed.pathname.replace(/\/+$/u, "")),
|
||||
forcePathStyle: parseBoolean(input?.forcePathStyle, true),
|
||||
region: region ?? DEFAULT_REGION,
|
||||
secretKey,
|
||||
});
|
||||
}
|
||||
|
||||
/**
|
||||
* A one-line operator-facing description of the configured provider that is safe to print anywhere —
|
||||
* Doctor output, a log line, a support transcript.
|
||||
*
|
||||
* It names the endpoint's **origin only** (never a path, query, or userinfo), the region, the
|
||||
* addressing style, and the bucket prefix. It never includes the access key, the secret key, or any
|
||||
* value derived from them. `describeDatabaseTransport` sets the same precedent.
|
||||
*/
|
||||
export function describeStorageProvider(config: StorageProviderConfig): string {
|
||||
const origin = new URL(config.endpoint).origin;
|
||||
return `S3-compatible endpoint ${origin} · region ${config.region} · ${
|
||||
config.forcePathStyle ? "path-style" : "virtual-host-style"
|
||||
} · bucket prefix "${config.bucketPrefix}"`;
|
||||
}
|
||||
|
||||
export { STORAGE_BUCKET_PREFIX_MAX_LENGTH };
|
||||
@@ -0,0 +1,5 @@
|
||||
export * from "./binding.js";
|
||||
export * from "./config.js";
|
||||
export * from "./module-storage.js";
|
||||
export * from "./provider.js";
|
||||
export * from "./s3-provider.js";
|
||||
@@ -0,0 +1,185 @@
|
||||
import {
|
||||
EMPTY_MODULE_STORAGE,
|
||||
isValidModuleStorageKey,
|
||||
type ModuleStorage,
|
||||
type ModuleStorageBody,
|
||||
type ModuleStorageBucket,
|
||||
ModuleStorageError,
|
||||
type ModuleStorageListOptions,
|
||||
type ModuleStorageListing,
|
||||
type ModuleStorageObject,
|
||||
type ModuleStorageObjectMetadata,
|
||||
type ModuleStoragePutOptions,
|
||||
} from "@ekklesia/contracts";
|
||||
import { type ModuleStorageBinding, resolveModuleStorageBindings } from "./binding.js";
|
||||
import type { StorageProvider } from "./provider.js";
|
||||
|
||||
/**
|
||||
* Builds the scoped storage facade one module receives.
|
||||
*
|
||||
* **This is where isolation is enforced, and it is enforced by construction.** The facade closes over
|
||||
* exactly the bindings this module's own manifest declared; there is no lookup table spanning
|
||||
* modules, no name that resolves outside those bindings, and no accessor taking a physical bucket.
|
||||
* Module A cannot reach module B's storage by knowing its bucket name, its module id, or its
|
||||
* physical bucket name, because none of those is an input to anything here (ADR 0030).
|
||||
*
|
||||
* Two modules that both declare `assets` receive two facades over two different physical buckets —
|
||||
* see `physicalBucketName` for why that separation is collision-safe rather than merely likely.
|
||||
*/
|
||||
|
||||
/** Ensures a binding's physical bucket exists, at most once per process per binding.
|
||||
*
|
||||
* Lazy rather than eager at startup, deliberately: a storage outage must never stop the platform
|
||||
* from booting. The promise is memoized so concurrent first uses share one round trip, and a
|
||||
* *failed* attempt is discarded so a transient outage does not poison the binding for the life of
|
||||
* the process — the next call retries, which is exactly the resilience a module like a slideshow
|
||||
* depends on (ADR 0030). */
|
||||
function createBucketEnsurer(
|
||||
provider: StorageProvider,
|
||||
physicalBucket: string,
|
||||
): () => Promise<void> {
|
||||
let pending: Promise<void> | undefined;
|
||||
return () => {
|
||||
pending ??= provider.ensureBucket(physicalBucket).catch((error: unknown) => {
|
||||
pending = undefined;
|
||||
throw error;
|
||||
});
|
||||
return pending;
|
||||
};
|
||||
}
|
||||
|
||||
/** Echoes the offending key only when it is a short string — a module's own input is safe to hand
|
||||
* back, but a non-string or an enormous value is not something to put into an error message. */
|
||||
function assertValidKey(key: unknown): asserts key is string {
|
||||
if (isValidModuleStorageKey(key)) {
|
||||
return;
|
||||
}
|
||||
const echo = typeof key === "string" && key.length <= 256 ? key : undefined;
|
||||
throw new ModuleStorageError("STORAGE_KEY_INVALID", echo);
|
||||
}
|
||||
|
||||
function createBucketHandle(
|
||||
provider: StorageProvider,
|
||||
binding: ModuleStorageBinding,
|
||||
): ModuleStorageBucket {
|
||||
const ensure = createBucketEnsurer(provider, binding.physicalBucket);
|
||||
|
||||
return Object.freeze({
|
||||
// The **logical** id, never the physical bucket. A module that logs this leaks nothing about
|
||||
// where the host keeps its data.
|
||||
id: binding.logicalBucketId,
|
||||
|
||||
async delete(key: string): Promise<void> {
|
||||
assertValidKey(key);
|
||||
await ensure();
|
||||
await provider.delete(binding.physicalBucket, key);
|
||||
},
|
||||
|
||||
async get(key: string): Promise<ModuleStorageObject> {
|
||||
assertValidKey(key);
|
||||
await ensure();
|
||||
return provider.get(binding.physicalBucket, key);
|
||||
},
|
||||
|
||||
async head(key: string): Promise<ModuleStorageObjectMetadata> {
|
||||
assertValidKey(key);
|
||||
await ensure();
|
||||
return provider.head(binding.physicalBucket, key);
|
||||
},
|
||||
|
||||
async list(options: ModuleStorageListOptions = {}): Promise<ModuleStorageListing> {
|
||||
await ensure();
|
||||
return provider.list(binding.physicalBucket, options);
|
||||
},
|
||||
|
||||
async put(
|
||||
key: string,
|
||||
body: ModuleStorageBody,
|
||||
options: ModuleStoragePutOptions = {},
|
||||
): Promise<ModuleStorageObjectMetadata> {
|
||||
assertValidKey(key);
|
||||
if (body instanceof ReadableStream && options.contentLength === undefined) {
|
||||
throw new ModuleStorageError("STORAGE_CONTENT_LENGTH_REQUIRED", key);
|
||||
}
|
||||
await ensure();
|
||||
return provider.put(binding.physicalBucket, key, body, options);
|
||||
},
|
||||
});
|
||||
}
|
||||
|
||||
export interface CreateModuleStorageOptions {
|
||||
readonly provider: StorageProvider;
|
||||
readonly bucketPrefix: string;
|
||||
readonly moduleId: string;
|
||||
/** The logical bucket ids from this module's own manifest, in declaration order. */
|
||||
readonly logicalBucketIds: readonly string[];
|
||||
}
|
||||
|
||||
/**
|
||||
* The scoped `ModuleStorage` for one module.
|
||||
*
|
||||
* A module declaring no buckets gets `EMPTY_MODULE_STORAGE` — present, frozen, and refusing every
|
||||
* bucket — so `context.storage` is never `undefined` and module code never branches on whether the
|
||||
* host supplied storage.
|
||||
*/
|
||||
export function createModuleStorage(options: CreateModuleStorageOptions): ModuleStorage {
|
||||
if (options.logicalBucketIds.length === 0) {
|
||||
return EMPTY_MODULE_STORAGE;
|
||||
}
|
||||
const bindings = resolveModuleStorageBindings(
|
||||
options.bucketPrefix,
|
||||
options.moduleId,
|
||||
options.logicalBucketIds,
|
||||
);
|
||||
// A `Map` rather than a plain object: every lookup key here comes from module code, and indexing a
|
||||
// plain object with such a key is the shape of a prototype-pollution bug. With a `Map` there is no
|
||||
// prototype to reach, so `constructor` or `__proto__` are ordinary absent keys.
|
||||
const handles = new Map<string, ModuleStorageBucket>(
|
||||
bindings.map((binding) => [
|
||||
binding.logicalBucketId,
|
||||
createBucketHandle(options.provider, binding),
|
||||
]),
|
||||
);
|
||||
const declared = Object.freeze([...handles.keys()].sort());
|
||||
|
||||
return Object.freeze({
|
||||
bucket: (logicalBucketId: string): ModuleStorageBucket => {
|
||||
const handle = typeof logicalBucketId === "string" ? handles.get(logicalBucketId) : undefined;
|
||||
if (!handle) {
|
||||
// Including another module's bucket id, which is simply not a key in this map.
|
||||
throw new ModuleStorageError("STORAGE_BUCKET_UNDECLARED", logicalBucketId);
|
||||
}
|
||||
return handle;
|
||||
},
|
||||
buckets: (): readonly string[] => declared,
|
||||
});
|
||||
}
|
||||
|
||||
/**
|
||||
* Every module's storage, resolved once at startup.
|
||||
*
|
||||
* Returns an empty map when the host has no storage provider. That is not an error: an installation
|
||||
* with no object storage is a complete, valid deployment, and only modules that *declared* a bucket
|
||||
* are affected — those are reported blocked by the lifecycle resolver before any of this is reached
|
||||
* (ADR 0030).
|
||||
*/
|
||||
export function createModuleStorageLookup(
|
||||
provider: StorageProvider | undefined,
|
||||
bucketPrefix: string,
|
||||
modules: readonly { readonly moduleId: string; readonly logicalBucketIds: readonly string[] }[],
|
||||
): ReadonlyMap<string, ModuleStorage> {
|
||||
if (!provider) {
|
||||
return new Map();
|
||||
}
|
||||
return new Map(
|
||||
modules.map((module) => [
|
||||
module.moduleId,
|
||||
createModuleStorage({
|
||||
bucketPrefix,
|
||||
logicalBucketIds: module.logicalBucketIds,
|
||||
moduleId: module.moduleId,
|
||||
provider,
|
||||
}),
|
||||
]),
|
||||
);
|
||||
}
|
||||
@@ -0,0 +1,51 @@
|
||||
import type {
|
||||
ModuleStorageBody,
|
||||
ModuleStorageListOptions,
|
||||
ModuleStorageListing,
|
||||
ModuleStorageObject,
|
||||
ModuleStorageObjectMetadata,
|
||||
ModuleStoragePutOptions,
|
||||
} from "@ekklesia/contracts";
|
||||
|
||||
/**
|
||||
* The seam between Core's module-facing storage facade and whatever actually stores bytes.
|
||||
*
|
||||
* This is the **portability boundary made executable**. Everything above it — the scoped facade,
|
||||
* binding resolution, lifecycle eligibility, diagnostics — is provider-agnostic and would be
|
||||
* identical in a standalone host. Everything below it is one implementation
|
||||
* (`createS3StorageProvider`) that could be replaced without any module changing a line.
|
||||
*
|
||||
* It deliberately speaks in **physical buckets**: resolving a module's logical bucket to a physical
|
||||
* one has already happened by the time anything reaches here (`./binding.js`), so a provider never
|
||||
* has to know what a module is. That split is what keeps scoping enforcement in exactly one place
|
||||
* rather than duplicated into every provider.
|
||||
*
|
||||
* Also deliberately absent: any operation that *destroys* a bucket. Core never deletes a bucket, so
|
||||
* there is no method to call by mistake, and disabling or removing a module cannot take data with it
|
||||
* (ADR 0030).
|
||||
*/
|
||||
export interface StorageProvider {
|
||||
/**
|
||||
* Makes sure a physical bucket exists, idempotently.
|
||||
*
|
||||
* Called lazily on first use of a binding rather than at startup, so a storage outage can never
|
||||
* prevent the platform from booting.
|
||||
*/
|
||||
ensureBucket(physicalBucket: string): Promise<void>;
|
||||
list(physicalBucket: string, options: ModuleStorageListOptions): Promise<ModuleStorageListing>;
|
||||
head(physicalBucket: string, key: string): Promise<ModuleStorageObjectMetadata>;
|
||||
get(physicalBucket: string, key: string): Promise<ModuleStorageObject>;
|
||||
put(
|
||||
physicalBucket: string,
|
||||
key: string,
|
||||
body: ModuleStorageBody,
|
||||
options: ModuleStoragePutOptions,
|
||||
): Promise<ModuleStorageObjectMetadata>;
|
||||
delete(physicalBucket: string, key: string): Promise<void>;
|
||||
/** A cheap reachability probe for diagnostics only. Its result is **never** persisted and never
|
||||
* influences a module's lifecycle state — a provider being briefly unreachable is an operational
|
||||
* condition, not an administrative decision (ADR 0030). */
|
||||
probe(): Promise<void>;
|
||||
/** Releases provider connections. Called once, during API shutdown. */
|
||||
close(): Promise<void>;
|
||||
}
|
||||
@@ -0,0 +1,289 @@
|
||||
import { Readable } from "node:stream";
|
||||
import {
|
||||
CreateBucketCommand,
|
||||
DeleteObjectCommand,
|
||||
GetObjectCommand,
|
||||
HeadObjectCommand,
|
||||
ListBucketsCommand,
|
||||
ListObjectsV2Command,
|
||||
PutObjectCommand,
|
||||
S3Client,
|
||||
} from "@aws-sdk/client-s3";
|
||||
import {
|
||||
type ModuleStorageBody,
|
||||
ModuleStorageError,
|
||||
type ModuleStorageListOptions,
|
||||
type ModuleStorageListing,
|
||||
type ModuleStorageObject,
|
||||
type ModuleStorageObjectMetadata,
|
||||
type ModuleStoragePutOptions,
|
||||
} from "@ekklesia/contracts";
|
||||
import type { StorageProviderConfig } from "./config.js";
|
||||
import type { StorageProvider } from "./provider.js";
|
||||
|
||||
/**
|
||||
* The platform's first real `StorageProvider`, speaking the S3-compatible protocol.
|
||||
*
|
||||
* **It is not named after a product, and it must not become one.** SeaweedFS is what the
|
||||
* repository's development Compose stack runs and the likely deployment target (ADR 0004), but
|
||||
* nothing here depends on SeaweedFS: the same code addresses MinIO, AWS S3, Garage, or anything else
|
||||
* speaking the protocol, selected purely by host configuration (ADR 0030).
|
||||
*
|
||||
* Server-only, and structurally so: this module imports `node:stream` and the AWS SDK, so a browser
|
||||
* bundle importing it would fail to build. `@ekklesia/module-ui-sdk` and `surfaces/*` never reach it,
|
||||
* because a module never imports Core at all.
|
||||
*/
|
||||
|
||||
/** Bounds a single listing page. S3-compatible providers cap this at 1000 regardless. */
|
||||
const MAX_LIST_PAGE_SIZE = 1000;
|
||||
|
||||
interface S3LikeError {
|
||||
readonly name?: unknown;
|
||||
readonly $metadata?: { readonly httpStatusCode?: unknown };
|
||||
}
|
||||
|
||||
function errorName(error: unknown): string {
|
||||
const candidate = error as S3LikeError | null;
|
||||
return typeof candidate?.name === "string" ? candidate.name : "";
|
||||
}
|
||||
|
||||
function statusCode(error: unknown): number | undefined {
|
||||
const candidate = error as S3LikeError | null;
|
||||
const status = candidate?.$metadata?.httpStatusCode;
|
||||
return typeof status === "number" ? status : undefined;
|
||||
}
|
||||
|
||||
/**
|
||||
* Reduces any provider failure to one of the contract's closed codes.
|
||||
*
|
||||
* **Nothing from the caught error survives this function** — not its message, not its request id,
|
||||
* not the XML body, not the endpoint it was talking to. Provider errors routinely carry the bucket
|
||||
* name, the host, and sometimes the full signed request, and a module is entitled to log whatever it
|
||||
* catches, so translation here is the only place that boundary can be held.
|
||||
*
|
||||
* The mapping is verified against the pinned SeaweedFS 4.28 image rather than assumed from AWS
|
||||
* documentation: a missing object reports `NoSuchKey`, a missing bucket `NoSuchBucket`, and — the
|
||||
* one that actually differs from AWS — creating an existing bucket reports `BucketAlreadyExists`
|
||||
* where AWS reports `BucketAlreadyOwnedByYou`.
|
||||
*/
|
||||
function translateStorageError(error: unknown, key?: string): ModuleStorageError {
|
||||
const name = errorName(error);
|
||||
const status = statusCode(error);
|
||||
|
||||
if (name === "NoSuchKey" || name === "NotFound" || (status === 404 && key !== undefined)) {
|
||||
return new ModuleStorageError("STORAGE_OBJECT_NOT_FOUND", key);
|
||||
}
|
||||
if (name === "NoSuchBucket" || status === 404) {
|
||||
return new ModuleStorageError("STORAGE_BUCKET_UNAVAILABLE");
|
||||
}
|
||||
if (
|
||||
name === "AccessDenied" ||
|
||||
name === "InvalidAccessKeyId" ||
|
||||
name === "SignatureDoesNotMatch"
|
||||
) {
|
||||
return new ModuleStorageError("STORAGE_OPERATION_DENIED");
|
||||
}
|
||||
if (status === 403) {
|
||||
return new ModuleStorageError("STORAGE_OPERATION_DENIED");
|
||||
}
|
||||
// Connection-level failures surface as Node system errors or the SDK's own timeout types rather
|
||||
// than as an HTTP status, which is exactly the transient case a module is expected to retry.
|
||||
if (
|
||||
status === undefined &&
|
||||
(name.startsWith("Timeout") ||
|
||||
name === "TimeoutError" ||
|
||||
name === "AbortError" ||
|
||||
/ECONN|ENOTFOUND|EAI_AGAIN|EHOSTUNREACH|ENETUNREACH|EPIPE|ETIMEDOUT/u.test(
|
||||
`${name} ${String((error as { code?: unknown } | null)?.code ?? "")}`,
|
||||
))
|
||||
) {
|
||||
return new ModuleStorageError("STORAGE_PROVIDER_UNAVAILABLE");
|
||||
}
|
||||
if (status !== undefined && status >= 500) {
|
||||
return new ModuleStorageError("STORAGE_PROVIDER_UNAVAILABLE");
|
||||
}
|
||||
return new ModuleStorageError("STORAGE_OPERATION_FAILED", key);
|
||||
}
|
||||
|
||||
function metadataFrom(
|
||||
key: string,
|
||||
source: {
|
||||
readonly ContentType?: string | undefined;
|
||||
readonly ContentLength?: number | undefined;
|
||||
readonly ETag?: string | undefined;
|
||||
readonly LastModified?: Date | undefined;
|
||||
},
|
||||
): ModuleStorageObjectMetadata {
|
||||
return Object.freeze({
|
||||
key,
|
||||
// Providers quote the ETag; unquoting keeps the value usable as an HTTP validator without every
|
||||
// module learning to strip quotes.
|
||||
...(source.ETag === undefined ? {} : { etag: source.ETag.replaceAll('"', "") }),
|
||||
...(source.ContentLength === undefined ? {} : { contentLength: source.ContentLength }),
|
||||
...(source.ContentType === undefined ? {} : { contentType: source.ContentType }),
|
||||
...(source.LastModified === undefined ? {} : { lastModified: source.LastModified }),
|
||||
});
|
||||
}
|
||||
|
||||
/** Converts the contract's accepted body shapes into something the SDK can send without this
|
||||
* process buffering a large object. A Web stream becomes a Node stream rather than an array of
|
||||
* bytes, which is the whole reason the contract accepts one. */
|
||||
function toRequestBody(body: ModuleStorageBody): Uint8Array | string | Readable | Blob {
|
||||
if (body instanceof ReadableStream) {
|
||||
return Readable.fromWeb(body as Parameters<typeof Readable.fromWeb>[0]);
|
||||
}
|
||||
return body;
|
||||
}
|
||||
|
||||
export function createS3StorageProvider(config: StorageProviderConfig): StorageProvider {
|
||||
const client = new S3Client({
|
||||
credentials: { accessKeyId: config.accessKey, secretAccessKey: config.secretKey },
|
||||
endpoint: config.endpoint,
|
||||
forcePathStyle: config.forcePathStyle,
|
||||
region: config.region,
|
||||
});
|
||||
|
||||
return Object.freeze({
|
||||
async ensureBucket(physicalBucket: string): Promise<void> {
|
||||
try {
|
||||
await client.send(new CreateBucketCommand({ Bucket: physicalBucket }));
|
||||
} catch (error) {
|
||||
const name = errorName(error);
|
||||
// Idempotency, spelled out for both dialects. AWS reports `BucketAlreadyOwnedByYou`; the
|
||||
// pinned SeaweedFS image reports `BucketAlreadyExists` with a 409 — verified directly, not
|
||||
// assumed. Treating only the AWS spelling as success would make every second boot fail
|
||||
// against the repository's own development storage service.
|
||||
if (name === "BucketAlreadyOwnedByYou" || name === "BucketAlreadyExists") {
|
||||
return;
|
||||
}
|
||||
if (name === "AccessDenied" || statusCode(error) === 403) {
|
||||
// The host's credentials cannot create it. Surfaced as an unavailable bucket rather than
|
||||
// silently falling back to "maybe it exists": an operator needs to know their credentials
|
||||
// are insufficient, not discover it through a later confusing read failure.
|
||||
throw new ModuleStorageError("STORAGE_BUCKET_UNAVAILABLE");
|
||||
}
|
||||
throw translateStorageError(error);
|
||||
}
|
||||
},
|
||||
|
||||
async list(
|
||||
physicalBucket: string,
|
||||
options: ModuleStorageListOptions,
|
||||
): Promise<ModuleStorageListing> {
|
||||
try {
|
||||
const response = await client.send(
|
||||
new ListObjectsV2Command({
|
||||
Bucket: physicalBucket,
|
||||
...(options.cursor === undefined ? {} : { ContinuationToken: options.cursor }),
|
||||
...(options.limit === undefined
|
||||
? {}
|
||||
: { MaxKeys: Math.max(1, Math.min(options.limit, MAX_LIST_PAGE_SIZE)) }),
|
||||
...(options.prefix === undefined ? {} : { Prefix: options.prefix }),
|
||||
}),
|
||||
);
|
||||
const objects = (response.Contents ?? []).flatMap((entry) =>
|
||||
entry.Key === undefined
|
||||
? []
|
||||
: [
|
||||
metadataFrom(entry.Key, {
|
||||
ETag: entry.ETag,
|
||||
ContentLength: entry.Size,
|
||||
LastModified: entry.LastModified,
|
||||
}),
|
||||
],
|
||||
);
|
||||
return Object.freeze({
|
||||
objects: Object.freeze(objects),
|
||||
// Only forwarded when the provider says the listing is truncated, so an absent cursor
|
||||
// always means "complete" rather than "the provider happened to omit a token".
|
||||
...(response.IsTruncated === true && response.NextContinuationToken !== undefined
|
||||
? { cursor: response.NextContinuationToken }
|
||||
: {}),
|
||||
});
|
||||
} catch (error) {
|
||||
throw translateStorageError(error);
|
||||
}
|
||||
},
|
||||
|
||||
async head(physicalBucket: string, key: string): Promise<ModuleStorageObjectMetadata> {
|
||||
try {
|
||||
const response = await client.send(
|
||||
new HeadObjectCommand({ Bucket: physicalBucket, Key: key }),
|
||||
);
|
||||
return metadataFrom(key, response);
|
||||
} catch (error) {
|
||||
throw translateStorageError(error, key);
|
||||
}
|
||||
},
|
||||
|
||||
async get(physicalBucket: string, key: string): Promise<ModuleStorageObject> {
|
||||
try {
|
||||
const response = await client.send(
|
||||
new GetObjectCommand({ Bucket: physicalBucket, Key: key }),
|
||||
);
|
||||
if (!response.Body) {
|
||||
throw new ModuleStorageError("STORAGE_OPERATION_FAILED", key);
|
||||
}
|
||||
// A Web stream, never a buffered body: a module hands this straight to a `Response`, so a
|
||||
// large object never passes through this process's memory in one piece.
|
||||
const body = response.Body.transformToWebStream() as ReadableStream<Uint8Array>;
|
||||
return Object.freeze({ ...metadataFrom(key, response), body });
|
||||
} catch (error) {
|
||||
if (error instanceof ModuleStorageError) {
|
||||
throw error;
|
||||
}
|
||||
throw translateStorageError(error, key);
|
||||
}
|
||||
},
|
||||
|
||||
async put(
|
||||
physicalBucket: string,
|
||||
key: string,
|
||||
body: ModuleStorageBody,
|
||||
options: ModuleStoragePutOptions,
|
||||
): Promise<ModuleStorageObjectMetadata> {
|
||||
try {
|
||||
const response = await client.send(
|
||||
new PutObjectCommand({
|
||||
Body: toRequestBody(body),
|
||||
Bucket: physicalBucket,
|
||||
Key: key,
|
||||
...(options.contentLength === undefined
|
||||
? {}
|
||||
: { ContentLength: options.contentLength }),
|
||||
...(options.contentType === undefined ? {} : { ContentType: options.contentType }),
|
||||
}),
|
||||
);
|
||||
return metadataFrom(key, {
|
||||
ETag: response.ETag,
|
||||
...(options.contentLength === undefined ? {} : { ContentLength: options.contentLength }),
|
||||
...(options.contentType === undefined ? {} : { ContentType: options.contentType }),
|
||||
});
|
||||
} catch (error) {
|
||||
throw translateStorageError(error, key);
|
||||
}
|
||||
},
|
||||
|
||||
async delete(physicalBucket: string, key: string): Promise<void> {
|
||||
try {
|
||||
await client.send(new DeleteObjectCommand({ Bucket: physicalBucket, Key: key }));
|
||||
} catch (error) {
|
||||
throw translateStorageError(error, key);
|
||||
}
|
||||
},
|
||||
|
||||
async probe(): Promise<void> {
|
||||
try {
|
||||
// The cheapest call that proves the endpoint is reachable and the credentials sign
|
||||
// correctly, without creating, reading, or writing anything.
|
||||
await client.send(new ListBucketsCommand({}));
|
||||
} catch (error) {
|
||||
throw translateStorageError(error);
|
||||
}
|
||||
},
|
||||
|
||||
async close(): Promise<void> {
|
||||
client.destroy();
|
||||
},
|
||||
});
|
||||
}
|
||||
Reference in New Issue
Block a user