`manifest.storage.buckets` has existed since manifest version 1 and meant nothing — discovery validated it and no code read it. This turns it into a real capability, which is what stands between Announcements and the module system: that product currently fetches a hardcoded LAN S3 endpoint, parses ListObjectsV2 XML by hand, and hands the browser direct object URLs. - A module declares a logical bucket and receives a scoped facade at `context.storage` — list, head, get, put, delete. The contract names no provider: no endpoint, region, credential, or physical bucket is reachable from it, which is what lets the same module run under a future standalone host over its own or a shared S3-compatible service. - Core's first provider speaks S3 through the AWS SDK. SeaweedFS is the deployment target (ADR 0004), never an API name. Probing the pinned image found `CreateBucket` reporting `BucketAlreadyExists` rather than AWS's `BucketAlreadyOwnedByYou`; treating only the AWS spelling as success would have failed every second boot. A live test pins it. - Logical → physical mapping is deterministic and collision-safe: an installation prefix, the module and bucket ids joined by `--` (which cannot occur inside either identifier, so `a-b`+`c` and `a`+`b-c` cannot collide), and a digest over the full untruncated identity. Two modules declaring `assets` always get separate namespaces; neither can see the physical name. - Declaring a bucket is now a requirement: with no provider configured the module is `blocked · storage_unavailable` through ADR 0028's existing resolver, propagating to required dependents. An installation with no object storage stays a complete deployment. - Binding availability is not provider health. Eligibility asks whether storage is configured, never whether it answered; bucket creation is lazy on first use so a storage outage cannot stop the platform booting; a running module whose store blinks stays running and retries. Nothing deletes a bucket. - Credentials are deployment-supplied, never persisted, never routed through module configuration, and never visible to modules, module UI, diagnostics, or API responses. Provider errors translate to a closed code set carrying no endpoint, bucket, request id, or XML. - Retires ADR 0029's transitional `bucket` configuration key. No manifest version change. The template declares no bucket, so it still runs without S3. See ADR 0030. Co-Authored-By: Claude Opus 5 <[email protected]>
776 lines
29 KiB
TypeScript
776 lines
29 KiB
TypeScript
import {
|
|
EMPTY_MODULE_STORAGE,
|
|
isValidModuleStorageKey,
|
|
MODULE_STORAGE_KEY_MAX_BYTES,
|
|
type ModuleStorageBody,
|
|
ModuleStorageError,
|
|
type ModuleStorageListOptions,
|
|
type ModuleStorageListing,
|
|
type ModuleStorageObject,
|
|
type ModuleStorageObjectMetadata,
|
|
type ModuleStoragePutOptions,
|
|
} from "@ekklesia/contracts";
|
|
import { describe, expect, it } from "vitest";
|
|
import {
|
|
DEFAULT_STORAGE_BUCKET_PREFIX,
|
|
isValidStorageBucketPrefix,
|
|
physicalBucketName,
|
|
resolveModuleStorageBindings,
|
|
} from "../src/storage/binding.js";
|
|
import { createStorageProviderConfig, describeStorageProvider } from "../src/storage/config.js";
|
|
import { createModuleStorage, createModuleStorageLookup } from "../src/storage/module-storage.js";
|
|
import type { StorageProvider } from "../src/storage/provider.js";
|
|
|
|
/** Reads a Web stream to bytes without depending on DOM-only helper types. */
|
|
async function collect(stream: ReadableStream<Uint8Array>): Promise<Uint8Array> {
|
|
const reader = stream.getReader();
|
|
const chunks: Uint8Array[] = [];
|
|
let total = 0;
|
|
for (;;) {
|
|
const { done, value } = await reader.read();
|
|
if (done) {
|
|
break;
|
|
}
|
|
chunks.push(value);
|
|
total += value.byteLength;
|
|
}
|
|
const merged = new Uint8Array(total);
|
|
let offset = 0;
|
|
for (const chunk of chunks) {
|
|
merged.set(chunk, offset);
|
|
offset += chunk.byteLength;
|
|
}
|
|
return merged;
|
|
}
|
|
|
|
function streamOf(bytes: Uint8Array): ReadableStream<Uint8Array> {
|
|
return new ReadableStream<Uint8Array>({
|
|
start(controller) {
|
|
controller.enqueue(bytes);
|
|
controller.close();
|
|
},
|
|
});
|
|
}
|
|
|
|
interface RecordedCall {
|
|
readonly operation: string;
|
|
readonly physicalBucket: string;
|
|
readonly key?: string;
|
|
}
|
|
|
|
/**
|
|
* A recording in-memory `StorageProvider`.
|
|
*
|
|
* Deliberately keyed by **physical** bucket, exactly as the real provider is, so a test can assert
|
|
* which physical namespace an operation actually reached — which is the only way to prove isolation
|
|
* rather than assume it.
|
|
*/
|
|
function fakeProvider(options: { failEnsure?: () => Error } = {}): StorageProvider & {
|
|
readonly calls: readonly RecordedCall[];
|
|
readonly buckets: ReadonlyMap<string, Map<string, { bytes: Uint8Array; contentType?: string }>>;
|
|
} {
|
|
const calls: RecordedCall[] = [];
|
|
const buckets = new Map<string, Map<string, { bytes: Uint8Array; contentType?: string }>>();
|
|
const objectsIn = (physicalBucket: string) => {
|
|
const bucket = buckets.get(physicalBucket);
|
|
if (!bucket) {
|
|
throw new ModuleStorageError("STORAGE_BUCKET_UNAVAILABLE");
|
|
}
|
|
return bucket;
|
|
};
|
|
|
|
const provider: StorageProvider = {
|
|
async close() {},
|
|
async delete(physicalBucket, key) {
|
|
calls.push({ key, operation: "delete", physicalBucket });
|
|
objectsIn(physicalBucket).delete(key);
|
|
},
|
|
async ensureBucket(physicalBucket) {
|
|
calls.push({ operation: "ensureBucket", physicalBucket });
|
|
const failure = options.failEnsure?.();
|
|
if (failure) {
|
|
throw failure;
|
|
}
|
|
if (!buckets.has(physicalBucket)) {
|
|
buckets.set(physicalBucket, new Map());
|
|
}
|
|
},
|
|
async get(physicalBucket, key): Promise<ModuleStorageObject> {
|
|
calls.push({ key, operation: "get", physicalBucket });
|
|
const stored = objectsIn(physicalBucket).get(key);
|
|
if (!stored) {
|
|
throw new ModuleStorageError("STORAGE_OBJECT_NOT_FOUND", key);
|
|
}
|
|
return Object.freeze({
|
|
body: streamOf(stored.bytes),
|
|
contentLength: stored.bytes.byteLength,
|
|
key,
|
|
...(stored.contentType === undefined ? {} : { contentType: stored.contentType }),
|
|
});
|
|
},
|
|
async head(physicalBucket, key): Promise<ModuleStorageObjectMetadata> {
|
|
calls.push({ key, operation: "head", physicalBucket });
|
|
const stored = objectsIn(physicalBucket).get(key);
|
|
if (!stored) {
|
|
throw new ModuleStorageError("STORAGE_OBJECT_NOT_FOUND", key);
|
|
}
|
|
return Object.freeze({ contentLength: stored.bytes.byteLength, key });
|
|
},
|
|
async list(
|
|
physicalBucket,
|
|
listOptions: ModuleStorageListOptions,
|
|
): Promise<ModuleStorageListing> {
|
|
calls.push({ operation: "list", physicalBucket });
|
|
const entries = [...objectsIn(physicalBucket).keys()]
|
|
.filter((key) => listOptions.prefix === undefined || key.startsWith(listOptions.prefix))
|
|
.sort();
|
|
return Object.freeze({
|
|
objects: Object.freeze(entries.map((key) => Object.freeze({ key }))),
|
|
});
|
|
},
|
|
async probe() {},
|
|
async put(
|
|
physicalBucket,
|
|
key,
|
|
body: ModuleStorageBody,
|
|
putOptions: ModuleStoragePutOptions,
|
|
): Promise<ModuleStorageObjectMetadata> {
|
|
calls.push({ key, operation: "put", physicalBucket });
|
|
const bytes =
|
|
typeof body === "string"
|
|
? new TextEncoder().encode(body)
|
|
: body instanceof Uint8Array
|
|
? body
|
|
: await collect(body as ReadableStream<Uint8Array>);
|
|
objectsIn(physicalBucket).set(key, {
|
|
bytes,
|
|
...(putOptions.contentType === undefined ? {} : { contentType: putOptions.contentType }),
|
|
});
|
|
return Object.freeze({ contentLength: bytes.byteLength, key });
|
|
},
|
|
};
|
|
return Object.assign(provider, { buckets: buckets as never, calls: calls as never });
|
|
}
|
|
|
|
describe("physical bucket naming", () => {
|
|
it("is deterministic, so a binding survives restarts and redeployments", () => {
|
|
const first = physicalBucketName("ekklesia", "announcements", "announcement-assets");
|
|
const second = physicalBucketName("ekklesia", "announcements", "announcement-assets");
|
|
|
|
expect(first).toBe(second);
|
|
expect(first).toContain("announcements--announcement-assets");
|
|
});
|
|
|
|
it("keeps two modules declaring the same logical name in different namespaces", () => {
|
|
const announcements = physicalBucketName("ekklesia", "announcements", "assets");
|
|
const video = physicalBucketName("ekklesia", "video", "assets");
|
|
|
|
// The property the whole scheme exists for: `assets` is an obvious name two unrelated modules
|
|
// will both pick.
|
|
expect(announcements).not.toBe(video);
|
|
});
|
|
|
|
it("cannot be confused by a hyphen in either identifier", () => {
|
|
// The ambiguity a single-hyphen separator would have produced: module `a-b` + bucket `c` and
|
|
// module `a` + bucket `b-c` would both render as `a-b-c`, silently sharing a namespace.
|
|
const left = physicalBucketName("ekklesia", "a-b", "c");
|
|
const right = physicalBucketName("ekklesia", "a", "b-c");
|
|
|
|
expect(left).not.toBe(right);
|
|
});
|
|
|
|
it("stays within S3's 63-character bucket-name limit even for maximal identifiers", () => {
|
|
const moduleId = `m${"o".repeat(62)}`.slice(0, 64);
|
|
const bucketId = `b${"u".repeat(46)}`.slice(0, 48);
|
|
const name = physicalBucketName("ekklesia", moduleId, bucketId);
|
|
|
|
expect(name.length).toBeLessThanOrEqual(63);
|
|
// Still distinct from a neighbour that truncates to the same readable prefix, because the digest
|
|
// is taken over the full untruncated identity.
|
|
expect(name).not.toBe(physicalBucketName("ekklesia", moduleId, `${bucketId.slice(0, 47)}x`));
|
|
});
|
|
|
|
it("always produces a name S3 itself would accept", () => {
|
|
const samples = [
|
|
physicalBucketName("ekklesia", "a", "b"),
|
|
physicalBucketName("ekklesia", `m${"o".repeat(62)}`, `b${"u".repeat(46)}`),
|
|
physicalBucketName("church", "announcements", "announcement-assets"),
|
|
physicalBucketName("ekklesia", "a-b", "c-d"),
|
|
];
|
|
for (const name of samples) {
|
|
expect(name.length).toBeGreaterThanOrEqual(3);
|
|
expect(name.length).toBeLessThanOrEqual(63);
|
|
expect(name).toMatch(/^[a-z0-9][a-z0-9-]*[a-z0-9]$/u);
|
|
// No dots at all, so S3's adjacent-period and period-next-to-hyphen rules are unreachable and
|
|
// virtual-host-style addressing stays TLS-safe should a provider ever use it. Consecutive
|
|
// hyphens are explicitly fine — `--` is the separator, and the pinned SeaweedFS image accepts
|
|
// such names (verified directly against it in the integration suite).
|
|
expect(name).not.toContain(".");
|
|
// Never IP-shaped, and never a reserved prefix, because it always begins with the validated
|
|
// alphabetic prefix.
|
|
expect(name).not.toMatch(/^\d+-\d+-\d+-\d+$/u);
|
|
expect(name.startsWith("xn--")).toBe(false);
|
|
expect(name.startsWith("sthree-")).toBe(false);
|
|
expect(name.endsWith("-s3alias")).toBe(false);
|
|
expect(name.endsWith("--ol-s3")).toBe(false);
|
|
}
|
|
});
|
|
|
|
it("isolates two installations sharing one storage service through the prefix", () => {
|
|
const first = physicalBucketName("ekklesia", "announcements", "assets");
|
|
const second = physicalBucketName("stanne", "announcements", "assets");
|
|
|
|
expect(first).not.toBe(second);
|
|
});
|
|
|
|
it("accepts only a safe bucket prefix", () => {
|
|
expect(isValidStorageBucketPrefix(DEFAULT_STORAGE_BUCKET_PREFIX)).toBe(true);
|
|
for (const bad of ["", "Ekklesia", "ekklesia_", "-ekklesia", "ekklesia--x", "a".repeat(17)]) {
|
|
expect(isValidStorageBucketPrefix(bad)).toBe(false);
|
|
}
|
|
});
|
|
|
|
it("resolves every declared bucket in declaration order", () => {
|
|
const bindings = resolveModuleStorageBindings("ekklesia", "announcements", [
|
|
"announcement-assets",
|
|
"thumbnails",
|
|
]);
|
|
|
|
expect(bindings.map((binding) => binding.logicalBucketId)).toEqual([
|
|
"announcement-assets",
|
|
"thumbnails",
|
|
]);
|
|
expect(new Set(bindings.map((binding) => binding.physicalBucket)).size).toBe(2);
|
|
});
|
|
});
|
|
|
|
describe("object key validation", () => {
|
|
it("accepts ordinary and nested keys", () => {
|
|
for (const key of ["welcome.png", "slides/welcome.png", "a/b/c/deep.webp", "ünïcode.png"]) {
|
|
expect(isValidModuleStorageKey(key)).toBe(true);
|
|
}
|
|
});
|
|
|
|
it("rejects keys that would be ambiguous or unrepresentable", () => {
|
|
for (const key of [
|
|
"",
|
|
"/leading-slash.png",
|
|
"back\\slash.png",
|
|
"control |