feat: add host-provided module object storage
`manifest.storage.buckets` has existed since manifest version 1 and meant nothing — discovery validated it and no code read it. This turns it into a real capability, which is what stands between Announcements and the module system: that product currently fetches a hardcoded LAN S3 endpoint, parses ListObjectsV2 XML by hand, and hands the browser direct object URLs. - A module declares a logical bucket and receives a scoped facade at `context.storage` — list, head, get, put, delete. The contract names no provider: no endpoint, region, credential, or physical bucket is reachable from it, which is what lets the same module run under a future standalone host over its own or a shared S3-compatible service. - Core's first provider speaks S3 through the AWS SDK. SeaweedFS is the deployment target (ADR 0004), never an API name. Probing the pinned image found `CreateBucket` reporting `BucketAlreadyExists` rather than AWS's `BucketAlreadyOwnedByYou`; treating only the AWS spelling as success would have failed every second boot. A live test pins it. - Logical → physical mapping is deterministic and collision-safe: an installation prefix, the module and bucket ids joined by `--` (which cannot occur inside either identifier, so `a-b`+`c` and `a`+`b-c` cannot collide), and a digest over the full untruncated identity. Two modules declaring `assets` always get separate namespaces; neither can see the physical name. - Declaring a bucket is now a requirement: with no provider configured the module is `blocked · storage_unavailable` through ADR 0028's existing resolver, propagating to required dependents. An installation with no object storage stays a complete deployment. - Binding availability is not provider health. Eligibility asks whether storage is configured, never whether it answered; bucket creation is lazy on first use so a storage outage cannot stop the platform booting; a running module whose store blinks stays running and retries. Nothing deletes a bucket. - Credentials are deployment-supplied, never persisted, never routed through module configuration, and never visible to modules, module UI, diagnostics, or API responses. Provider errors translate to a closed code set carrying no endpoint, bucket, request id, or XML. - Retires ADR 0029's transitional `bucket` configuration key. No manifest version change. The template declares no bucket, so it still runs without S3. See ADR 0030. Co-Authored-By: Claude Opus 5 <[email protected]>
This commit is contained in:
co-authored by
Claude Opus 5
parent
527239f6de
commit
b9ca0d560d
@@ -21,7 +21,7 @@ const sections: readonly DoctorSection[] = Object.freeze([
|
||||
section("Data definitions", ["schemas.discovery", "migrations.discovery"]),
|
||||
section("PostgreSQL", ["database.transport", "postgresql.connection", "postgresql.plan"]),
|
||||
section("Identity", ["identity.configuration", "identity.client-address"]),
|
||||
section("Object storage", ["object-storage.integration"]),
|
||||
section("Object storage", ["object-storage.integration", "object-storage.bindings"]),
|
||||
]);
|
||||
|
||||
const statusSymbols: Readonly<Record<DoctorCheck["status"], string>> = Object.freeze({
|
||||
|
||||
+149
-10
@@ -3,6 +3,7 @@ import {
|
||||
capabilityIdSchema,
|
||||
EMPTY_MODULE_SECRET_SOURCE,
|
||||
isVersionCompatible,
|
||||
logicalBucketIdSchema,
|
||||
moduleConfigurationKeySchema,
|
||||
moduleIdSchema,
|
||||
type ModuleSecretSource,
|
||||
@@ -42,8 +43,16 @@ import {
|
||||
type PostgresMigrationDatabaseOptions,
|
||||
planModuleMigrations,
|
||||
} from "../migrations/index.js";
|
||||
import {
|
||||
createS3StorageProvider,
|
||||
describeStorageProvider,
|
||||
physicalBucketName,
|
||||
type StorageProviderConfig,
|
||||
} from "../storage/index.js";
|
||||
import {
|
||||
composePlatformPermissionRegistry,
|
||||
isConfigurationBlockedDetail,
|
||||
isStorageBlockedDetail,
|
||||
createModuleConfigurationDatabase,
|
||||
createModuleConfigurationStore,
|
||||
createModuleLifecycleDatabase,
|
||||
@@ -75,7 +84,8 @@ export type DoctorCheckId =
|
||||
| "postgresql.plan"
|
||||
| "identity.configuration"
|
||||
| "identity.client-address"
|
||||
| "object-storage.integration";
|
||||
| "object-storage.integration"
|
||||
| "object-storage.bindings";
|
||||
|
||||
export type DoctorCheckStatus = "failure" | "pass" | "skipped" | "warning";
|
||||
export type DoctorResult = "not-ready" | "ready" | "ready-with-warnings";
|
||||
@@ -113,6 +123,9 @@ export interface DoctorOptions {
|
||||
* matches what `startApi` would resolve. Omitted supplies nothing — Doctor then reports a required
|
||||
* secret as missing, which is the honest answer for a deployment that has not set it. */
|
||||
readonly moduleSecrets?: ModuleSecretSource;
|
||||
/** The validated host storage provider, or `undefined` when this installation has none. Doctor
|
||||
* never receives raw environment values and never prints a credential. */
|
||||
readonly storage?: StorageProviderConfig | undefined;
|
||||
readonly runtimeRange: string;
|
||||
readonly runtimeVersion: string;
|
||||
}
|
||||
@@ -150,6 +163,9 @@ export interface DoctorDependencies {
|
||||
readonly database: MigrationDatabase;
|
||||
}) => Promise<MigrationPlanResult>;
|
||||
readonly runtimeSatisfies: (version: string, range: string) => boolean;
|
||||
/** Probes the configured storage provider for reachability. Behind the dependency boundary like
|
||||
* every other I/O Doctor performs, so the whole report stays exercisable without a network. */
|
||||
readonly probeStorage: (config: StorageProviderConfig) => Promise<void>;
|
||||
}
|
||||
|
||||
const doctorPostgresTimeoutMs = 10_000;
|
||||
@@ -194,6 +210,16 @@ async function readModuleConfigurationForRuntime(
|
||||
}
|
||||
}
|
||||
|
||||
/** Opens a provider client, probes, and always closes it — Doctor never leaves a connection behind. */
|
||||
async function probeStorageForRuntime(config: StorageProviderConfig): Promise<void> {
|
||||
const provider = createS3StorageProvider(config);
|
||||
try {
|
||||
await provider.probe();
|
||||
} finally {
|
||||
await provider.close().catch(() => undefined);
|
||||
}
|
||||
}
|
||||
|
||||
const productionDependencies: DoctorDependencies = Object.freeze({
|
||||
composePermissions: composePlatformPermissionRegistry,
|
||||
createMigrationDatabase: createPostgresMigrationDatabase,
|
||||
@@ -201,6 +227,7 @@ const productionDependencies: DoctorDependencies = Object.freeze({
|
||||
discoverModules,
|
||||
discoverSchemas: discoverModuleSchemas,
|
||||
planMigrations: planModuleMigrations,
|
||||
probeStorage: probeStorageForRuntime,
|
||||
readModuleConfiguration: readModuleConfigurationForRuntime,
|
||||
readModuleLifecycle: readModuleLifecycleForRuntime,
|
||||
runtimeSatisfies: isVersionCompatible,
|
||||
@@ -333,7 +360,17 @@ function safeLifecycleDetail(entry: ModuleEligibility): string {
|
||||
if (!entry.blocked) {
|
||||
return `${identity} · enabled`;
|
||||
}
|
||||
if (!("dependency" in entry.blocked)) {
|
||||
if (isStorageBlockedDetail(entry.blocked)) {
|
||||
// A storage block names the **logical** bucket the module declared — never an endpoint, a
|
||||
// credential, or the physical bucket the host maps it to. Re-validated against the same grammar
|
||||
// the manifest accepted it under, for the same defense-in-depth reason as every other
|
||||
// interpolation here.
|
||||
if (!logicalBucketIdSchema.safeParse(entry.blocked.storageBucketId).success) {
|
||||
throw new TypeError("Unsafe module lifecycle diagnostic value.");
|
||||
}
|
||||
return `${identity} · blocked · ${entry.blocked.reason} · ${entry.blocked.storageBucketId}`;
|
||||
}
|
||||
if (isConfigurationBlockedDetail(entry.blocked)) {
|
||||
// A configuration block names the offending **key** and never its value: a key is declared in a
|
||||
// public manifest, while a value may be a credential or arbitrary operator-entered text. The key
|
||||
// is re-validated against the same grammar the manifest accepted it under, for the same
|
||||
@@ -407,7 +444,7 @@ function safeConfigurationDetail(entry: ModuleEligibility, declaredCount: number
|
||||
return `${identity} · no configuration declared`;
|
||||
}
|
||||
const declared = `${declaredCount} ${plural(declaredCount, "key")} declared`;
|
||||
if (entry.blocked && !("dependency" in entry.blocked)) {
|
||||
if (entry.blocked && isConfigurationBlockedDetail(entry.blocked)) {
|
||||
if (!moduleConfigurationKeySchema.safeParse(entry.blocked.configurationKey).success) {
|
||||
throw new TypeError("Unsafe module configuration diagnostic value.");
|
||||
}
|
||||
@@ -458,6 +495,65 @@ function moduleConfigurationCheck(
|
||||
return freezeCheck("modules.configuration", "pass", "Module configuration valid", details);
|
||||
}
|
||||
|
||||
/**
|
||||
* Reports which installed modules require object storage, and where the host keeps each binding.
|
||||
*
|
||||
* Names the module, the **logical** bucket it declared, and the physical bucket the host maps it to
|
||||
* — the last of which is safe and useful here precisely because Doctor is an operator tool: someone
|
||||
* debugging a permissions problem needs to know which bucket to look at. It is still never reachable
|
||||
* from module-facing code (ADR 0030). No endpoint, access key, or secret key appears in any form.
|
||||
*/
|
||||
function storageBindingsCheck(
|
||||
modules: readonly DiscoveredModule[],
|
||||
entries: readonly ModuleEligibility[],
|
||||
config: StorageProviderConfig | undefined,
|
||||
): DoctorCheck {
|
||||
const requiring = modules.filter((module) => module.manifest.storage.buckets.length > 0);
|
||||
if (requiring.length === 0) {
|
||||
// An installation whose modules need no object storage is not unhealthy for lacking a provider.
|
||||
return freezeCheck("object-storage.bindings", "pass", "No module requires object storage", [
|
||||
"0 modules declare a logical storage bucket.",
|
||||
]);
|
||||
}
|
||||
|
||||
const blocked = entries.filter(
|
||||
(entry) => entry.blocked !== undefined && isStorageBlockedDetail(entry.blocked),
|
||||
);
|
||||
const details = requiring.flatMap((module) => {
|
||||
const moduleId = module.manifest.id;
|
||||
if (!moduleIdSchema.safeParse(moduleId).success) {
|
||||
throw new TypeError("Unsafe module storage diagnostic value.");
|
||||
}
|
||||
return module.manifest.storage.buckets.map((bucket) => {
|
||||
if (!logicalBucketIdSchema.safeParse(bucket.id).success) {
|
||||
throw new TypeError("Unsafe module storage diagnostic value.");
|
||||
}
|
||||
return config === undefined
|
||||
? `${moduleId} · ${bucket.id} · unbound (no storage provider configured)`
|
||||
: `${moduleId} · ${bucket.id} → ${physicalBucketName(config.bucketPrefix, moduleId, bucket.id)}`;
|
||||
});
|
||||
});
|
||||
|
||||
if (config === undefined) {
|
||||
return freezeCheck(
|
||||
"object-storage.bindings",
|
||||
"failure",
|
||||
blocked.length === 1
|
||||
? "1 module requires object storage that is not configured"
|
||||
: `${blocked.length} modules require object storage that is not configured`,
|
||||
[
|
||||
"Configure an S3-compatible Ekklesia storage provider (S3_ENDPOINT, S3_ACCESS_KEY, S3_SECRET_KEY).",
|
||||
...details,
|
||||
],
|
||||
);
|
||||
}
|
||||
return freezeCheck(
|
||||
"object-storage.bindings",
|
||||
"pass",
|
||||
`${details.length} ${plural(details.length, "storage binding")} resolved`,
|
||||
details,
|
||||
);
|
||||
}
|
||||
function moduleDiscoveryFailure(error: unknown): DoctorCheck {
|
||||
const details =
|
||||
error instanceof Error && error.name === "ModuleDiscoveryError"
|
||||
@@ -788,6 +884,9 @@ export async function diagnoseEkklesia(
|
||||
// failed. It needs only a successful discovery and a configured database.
|
||||
let lifecycleCheck: DoctorCheck;
|
||||
let configurationCheck: DoctorCheck;
|
||||
// Hoisted so the storage-bindings check below reports the same eligibility this pass resolved,
|
||||
// rather than resolving a second time and risking a different answer.
|
||||
let eligibilityEntries: readonly ModuleEligibility[] = [];
|
||||
const skippedConfiguration = (detail: string): DoctorCheck =>
|
||||
freezeCheck("modules.configuration", "skipped", "Module configuration skipped", [detail]);
|
||||
if (!discoveredModules) {
|
||||
@@ -819,10 +918,13 @@ export async function diagnoseEkklesia(
|
||||
poolOptions,
|
||||
);
|
||||
const settings = await dependencies.readModuleConfiguration(options.databaseUrl, poolOptions);
|
||||
const resolution = resolveModuleEligibility(discoveredModules, configuration, {
|
||||
secrets: options.moduleSecrets ?? EMPTY_MODULE_SECRET_SOURCE,
|
||||
settings,
|
||||
});
|
||||
const resolution = resolveModuleEligibility(
|
||||
discoveredModules,
|
||||
configuration,
|
||||
{ secrets: options.moduleSecrets ?? EMPTY_MODULE_SECRET_SOURCE, settings },
|
||||
{ providerConfigured: options.storage !== undefined },
|
||||
);
|
||||
eligibilityEntries = resolution.entries;
|
||||
lifecycleCheck = moduleLifecycleCheck(resolution.entries);
|
||||
configurationCheck = moduleConfigurationCheck(
|
||||
resolution.entries,
|
||||
@@ -930,9 +1032,45 @@ export async function diagnoseEkklesia(
|
||||
options.databaseEnvironment,
|
||||
);
|
||||
|
||||
const storageCheck = freezeCheck("object-storage.integration", "skipped", "Check unavailable", [
|
||||
"Core storage integration has not been implemented yet.",
|
||||
]);
|
||||
// Two separate questions, deliberately. "Is a provider configured and reachable?" is
|
||||
// infrastructure diagnostics; "can every module's declared bucket be bound?" is what decides
|
||||
// whether modules run. Conflating them would let a momentary network blip read as a module
|
||||
// problem (ADR 0030).
|
||||
let storageCheck: DoctorCheck;
|
||||
if (!options.storage) {
|
||||
storageCheck = freezeCheck(
|
||||
"object-storage.integration",
|
||||
"skipped",
|
||||
"Object storage not configured",
|
||||
["No S3_ENDPOINT is set. This is a complete deployment unless a module requires storage."],
|
||||
);
|
||||
} else {
|
||||
try {
|
||||
await dependencies.probeStorage(options.storage);
|
||||
storageCheck = freezeCheck("object-storage.integration", "pass", "Object storage reachable", [
|
||||
describeStorageProvider(options.storage),
|
||||
]);
|
||||
} catch {
|
||||
// A warning, never a failure: the provider is configured, and a module that is already
|
||||
// running keeps running through a transient outage. Reporting this as `not-ready` would make
|
||||
// the platform look broken because a network hiccup happened while Doctor ran.
|
||||
storageCheck = freezeCheck(
|
||||
"object-storage.integration",
|
||||
"warning",
|
||||
"Object storage unreachable",
|
||||
[
|
||||
describeStorageProvider(options.storage),
|
||||
"The configured storage service did not respond. Modules already running are unaffected.",
|
||||
],
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
const storageBindings = discoveredModules
|
||||
? storageBindingsCheck(discoveredModules, eligibilityEntries, options.storage)
|
||||
: freezeCheck("object-storage.bindings", "skipped", "Storage bindings skipped", [
|
||||
"Complete module discovery is required.",
|
||||
]);
|
||||
|
||||
return publishReport([
|
||||
runtimeCheck,
|
||||
@@ -949,5 +1087,6 @@ export async function diagnoseEkklesia(
|
||||
identityCheck,
|
||||
clientAddressDoctorCheck,
|
||||
storageCheck,
|
||||
storageBindings,
|
||||
]);
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user