feat: add host-provided module object storage
`manifest.storage.buckets` has existed since manifest version 1 and meant nothing — discovery validated it and no code read it. This turns it into a real capability, which is what stands between Announcements and the module system: that product currently fetches a hardcoded LAN S3 endpoint, parses ListObjectsV2 XML by hand, and hands the browser direct object URLs. - A module declares a logical bucket and receives a scoped facade at `context.storage` — list, head, get, put, delete. The contract names no provider: no endpoint, region, credential, or physical bucket is reachable from it, which is what lets the same module run under a future standalone host over its own or a shared S3-compatible service. - Core's first provider speaks S3 through the AWS SDK. SeaweedFS is the deployment target (ADR 0004), never an API name. Probing the pinned image found `CreateBucket` reporting `BucketAlreadyExists` rather than AWS's `BucketAlreadyOwnedByYou`; treating only the AWS spelling as success would have failed every second boot. A live test pins it. - Logical → physical mapping is deterministic and collision-safe: an installation prefix, the module and bucket ids joined by `--` (which cannot occur inside either identifier, so `a-b`+`c` and `a`+`b-c` cannot collide), and a digest over the full untruncated identity. Two modules declaring `assets` always get separate namespaces; neither can see the physical name. - Declaring a bucket is now a requirement: with no provider configured the module is `blocked · storage_unavailable` through ADR 0028's existing resolver, propagating to required dependents. An installation with no object storage stays a complete deployment. - Binding availability is not provider health. Eligibility asks whether storage is configured, never whether it answered; bucket creation is lazy on first use so a storage outage cannot stop the platform booting; a running module whose store blinks stays running and retries. Nothing deletes a bucket. - Credentials are deployment-supplied, never persisted, never routed through module configuration, and never visible to modules, module UI, diagnostics, or API responses. Provider errors translate to a closed code set carrying no endpoint, bucket, request id, or XML. - Retires ADR 0029's transitional `bucket` configuration key. No manifest version change. The template declares no bucket, so it still runs without S3. See ADR 0030. Co-Authored-By: Claude Opus 5 <[email protected]>
This commit is contained in:
ekklesia
co-authored by
Claude Opus 5
Claude Opus 5
parent
527239f6de
commit
b9ca0d560d
@@ -0,0 +1,775 @@
|
||||
import {
|
||||
EMPTY_MODULE_STORAGE,
|
||||
isValidModuleStorageKey,
|
||||
MODULE_STORAGE_KEY_MAX_BYTES,
|
||||
type ModuleStorageBody,
|
||||
ModuleStorageError,
|
||||
type ModuleStorageListOptions,
|
||||
type ModuleStorageListing,
|
||||
type ModuleStorageObject,
|
||||
type ModuleStorageObjectMetadata,
|
||||
type ModuleStoragePutOptions,
|
||||
} from "@ekklesia/contracts";
|
||||
import { describe, expect, it } from "vitest";
|
||||
import {
|
||||
DEFAULT_STORAGE_BUCKET_PREFIX,
|
||||
isValidStorageBucketPrefix,
|
||||
physicalBucketName,
|
||||
resolveModuleStorageBindings,
|
||||
} from "../src/storage/binding.js";
|
||||
import { createStorageProviderConfig, describeStorageProvider } from "../src/storage/config.js";
|
||||
import { createModuleStorage, createModuleStorageLookup } from "../src/storage/module-storage.js";
|
||||
import type { StorageProvider } from "../src/storage/provider.js";
|
||||
|
||||
/** Reads a Web stream to bytes without depending on DOM-only helper types. */
|
||||
async function collect(stream: ReadableStream<Uint8Array>): Promise<Uint8Array> {
|
||||
const reader = stream.getReader();
|
||||
const chunks: Uint8Array[] = [];
|
||||
let total = 0;
|
||||
for (;;) {
|
||||
const { done, value } = await reader.read();
|
||||
if (done) {
|
||||
break;
|
||||
}
|
||||
chunks.push(value);
|
||||
total += value.byteLength;
|
||||
}
|
||||
const merged = new Uint8Array(total);
|
||||
let offset = 0;
|
||||
for (const chunk of chunks) {
|
||||
merged.set(chunk, offset);
|
||||
offset += chunk.byteLength;
|
||||
}
|
||||
return merged;
|
||||
}
|
||||
|
||||
function streamOf(bytes: Uint8Array): ReadableStream<Uint8Array> {
|
||||
return new ReadableStream<Uint8Array>({
|
||||
start(controller) {
|
||||
controller.enqueue(bytes);
|
||||
controller.close();
|
||||
},
|
||||
});
|
||||
}
|
||||
|
||||
interface RecordedCall {
|
||||
readonly operation: string;
|
||||
readonly physicalBucket: string;
|
||||
readonly key?: string;
|
||||
}
|
||||
|
||||
/**
|
||||
* A recording in-memory `StorageProvider`.
|
||||
*
|
||||
* Deliberately keyed by **physical** bucket, exactly as the real provider is, so a test can assert
|
||||
* which physical namespace an operation actually reached — which is the only way to prove isolation
|
||||
* rather than assume it.
|
||||
*/
|
||||
function fakeProvider(options: { failEnsure?: () => Error } = {}): StorageProvider & {
|
||||
readonly calls: readonly RecordedCall[];
|
||||
readonly buckets: ReadonlyMap<string, Map<string, { bytes: Uint8Array; contentType?: string }>>;
|
||||
} {
|
||||
const calls: RecordedCall[] = [];
|
||||
const buckets = new Map<string, Map<string, { bytes: Uint8Array; contentType?: string }>>();
|
||||
const objectsIn = (physicalBucket: string) => {
|
||||
const bucket = buckets.get(physicalBucket);
|
||||
if (!bucket) {
|
||||
throw new ModuleStorageError("STORAGE_BUCKET_UNAVAILABLE");
|
||||
}
|
||||
return bucket;
|
||||
};
|
||||
|
||||
const provider: StorageProvider = {
|
||||
async close() {},
|
||||
async delete(physicalBucket, key) {
|
||||
calls.push({ key, operation: "delete", physicalBucket });
|
||||
objectsIn(physicalBucket).delete(key);
|
||||
},
|
||||
async ensureBucket(physicalBucket) {
|
||||
calls.push({ operation: "ensureBucket", physicalBucket });
|
||||
const failure = options.failEnsure?.();
|
||||
if (failure) {
|
||||
throw failure;
|
||||
}
|
||||
if (!buckets.has(physicalBucket)) {
|
||||
buckets.set(physicalBucket, new Map());
|
||||
}
|
||||
},
|
||||
async get(physicalBucket, key): Promise<ModuleStorageObject> {
|
||||
calls.push({ key, operation: "get", physicalBucket });
|
||||
const stored = objectsIn(physicalBucket).get(key);
|
||||
if (!stored) {
|
||||
throw new ModuleStorageError("STORAGE_OBJECT_NOT_FOUND", key);
|
||||
}
|
||||
return Object.freeze({
|
||||
body: streamOf(stored.bytes),
|
||||
contentLength: stored.bytes.byteLength,
|
||||
key,
|
||||
...(stored.contentType === undefined ? {} : { contentType: stored.contentType }),
|
||||
});
|
||||
},
|
||||
async head(physicalBucket, key): Promise<ModuleStorageObjectMetadata> {
|
||||
calls.push({ key, operation: "head", physicalBucket });
|
||||
const stored = objectsIn(physicalBucket).get(key);
|
||||
if (!stored) {
|
||||
throw new ModuleStorageError("STORAGE_OBJECT_NOT_FOUND", key);
|
||||
}
|
||||
return Object.freeze({ contentLength: stored.bytes.byteLength, key });
|
||||
},
|
||||
async list(
|
||||
physicalBucket,
|
||||
listOptions: ModuleStorageListOptions,
|
||||
): Promise<ModuleStorageListing> {
|
||||
calls.push({ operation: "list", physicalBucket });
|
||||
const entries = [...objectsIn(physicalBucket).keys()]
|
||||
.filter((key) => listOptions.prefix === undefined || key.startsWith(listOptions.prefix))
|
||||
.sort();
|
||||
return Object.freeze({
|
||||
objects: Object.freeze(entries.map((key) => Object.freeze({ key }))),
|
||||
});
|
||||
},
|
||||
async probe() {},
|
||||
async put(
|
||||
physicalBucket,
|
||||
key,
|
||||
body: ModuleStorageBody,
|
||||
putOptions: ModuleStoragePutOptions,
|
||||
): Promise<ModuleStorageObjectMetadata> {
|
||||
calls.push({ key, operation: "put", physicalBucket });
|
||||
const bytes =
|
||||
typeof body === "string"
|
||||
? new TextEncoder().encode(body)
|
||||
: body instanceof Uint8Array
|
||||
? body
|
||||
: await collect(body as ReadableStream<Uint8Array>);
|
||||
objectsIn(physicalBucket).set(key, {
|
||||
bytes,
|
||||
...(putOptions.contentType === undefined ? {} : { contentType: putOptions.contentType }),
|
||||
});
|
||||
return Object.freeze({ contentLength: bytes.byteLength, key });
|
||||
},
|
||||
};
|
||||
return Object.assign(provider, { buckets: buckets as never, calls: calls as never });
|
||||
}
|
||||
|
||||
describe("physical bucket naming", () => {
|
||||
it("is deterministic, so a binding survives restarts and redeployments", () => {
|
||||
const first = physicalBucketName("ekklesia", "announcements", "announcement-assets");
|
||||
const second = physicalBucketName("ekklesia", "announcements", "announcement-assets");
|
||||
|
||||
expect(first).toBe(second);
|
||||
expect(first).toContain("announcements--announcement-assets");
|
||||
});
|
||||
|
||||
it("keeps two modules declaring the same logical name in different namespaces", () => {
|
||||
const announcements = physicalBucketName("ekklesia", "announcements", "assets");
|
||||
const video = physicalBucketName("ekklesia", "video", "assets");
|
||||
|
||||
// The property the whole scheme exists for: `assets` is an obvious name two unrelated modules
|
||||
// will both pick.
|
||||
expect(announcements).not.toBe(video);
|
||||
});
|
||||
|
||||
it("cannot be confused by a hyphen in either identifier", () => {
|
||||
// The ambiguity a single-hyphen separator would have produced: module `a-b` + bucket `c` and
|
||||
// module `a` + bucket `b-c` would both render as `a-b-c`, silently sharing a namespace.
|
||||
const left = physicalBucketName("ekklesia", "a-b", "c");
|
||||
const right = physicalBucketName("ekklesia", "a", "b-c");
|
||||
|
||||
expect(left).not.toBe(right);
|
||||
});
|
||||
|
||||
it("stays within S3's 63-character bucket-name limit even for maximal identifiers", () => {
|
||||
const moduleId = `m${"o".repeat(62)}`.slice(0, 64);
|
||||
const bucketId = `b${"u".repeat(46)}`.slice(0, 48);
|
||||
const name = physicalBucketName("ekklesia", moduleId, bucketId);
|
||||
|
||||
expect(name.length).toBeLessThanOrEqual(63);
|
||||
// Still distinct from a neighbour that truncates to the same readable prefix, because the digest
|
||||
// is taken over the full untruncated identity.
|
||||
expect(name).not.toBe(physicalBucketName("ekklesia", moduleId, `${bucketId.slice(0, 47)}x`));
|
||||
});
|
||||
|
||||
it("always produces a name S3 itself would accept", () => {
|
||||
const samples = [
|
||||
physicalBucketName("ekklesia", "a", "b"),
|
||||
physicalBucketName("ekklesia", `m${"o".repeat(62)}`, `b${"u".repeat(46)}`),
|
||||
physicalBucketName("church", "announcements", "announcement-assets"),
|
||||
physicalBucketName("ekklesia", "a-b", "c-d"),
|
||||
];
|
||||
for (const name of samples) {
|
||||
expect(name.length).toBeGreaterThanOrEqual(3);
|
||||
expect(name.length).toBeLessThanOrEqual(63);
|
||||
expect(name).toMatch(/^[a-z0-9][a-z0-9-]*[a-z0-9]$/u);
|
||||
// No dots at all, so S3's adjacent-period and period-next-to-hyphen rules are unreachable and
|
||||
// virtual-host-style addressing stays TLS-safe should a provider ever use it. Consecutive
|
||||
// hyphens are explicitly fine — `--` is the separator, and the pinned SeaweedFS image accepts
|
||||
// such names (verified directly against it in the integration suite).
|
||||
expect(name).not.toContain(".");
|
||||
// Never IP-shaped, and never a reserved prefix, because it always begins with the validated
|
||||
// alphabetic prefix.
|
||||
expect(name).not.toMatch(/^\d+-\d+-\d+-\d+$/u);
|
||||
expect(name.startsWith("xn--")).toBe(false);
|
||||
expect(name.startsWith("sthree-")).toBe(false);
|
||||
expect(name.endsWith("-s3alias")).toBe(false);
|
||||
expect(name.endsWith("--ol-s3")).toBe(false);
|
||||
}
|
||||
});
|
||||
|
||||
it("isolates two installations sharing one storage service through the prefix", () => {
|
||||
const first = physicalBucketName("ekklesia", "announcements", "assets");
|
||||
const second = physicalBucketName("stanne", "announcements", "assets");
|
||||
|
||||
expect(first).not.toBe(second);
|
||||
});
|
||||
|
||||
it("accepts only a safe bucket prefix", () => {
|
||||
expect(isValidStorageBucketPrefix(DEFAULT_STORAGE_BUCKET_PREFIX)).toBe(true);
|
||||
for (const bad of ["", "Ekklesia", "ekklesia_", "-ekklesia", "ekklesia--x", "a".repeat(17)]) {
|
||||
expect(isValidStorageBucketPrefix(bad)).toBe(false);
|
||||
}
|
||||
});
|
||||
|
||||
it("resolves every declared bucket in declaration order", () => {
|
||||
const bindings = resolveModuleStorageBindings("ekklesia", "announcements", [
|
||||
"announcement-assets",
|
||||
"thumbnails",
|
||||
]);
|
||||
|
||||
expect(bindings.map((binding) => binding.logicalBucketId)).toEqual([
|
||||
"announcement-assets",
|
||||
"thumbnails",
|
||||
]);
|
||||
expect(new Set(bindings.map((binding) => binding.physicalBucket)).size).toBe(2);
|
||||
});
|
||||
});
|
||||
|
||||
describe("object key validation", () => {
|
||||
it("accepts ordinary and nested keys", () => {
|
||||
for (const key of ["welcome.png", "slides/welcome.png", "a/b/c/deep.webp", "ünïcode.png"]) {
|
||||
expect(isValidModuleStorageKey(key)).toBe(true);
|
||||
}
|
||||
});
|
||||
|
||||
it("rejects keys that would be ambiguous or unrepresentable", () => {
|
||||
for (const key of [
|
||||
"",
|
||||
"/leading-slash.png",
|
||||
"back\\slash.png",
|
||||
"control character.png",
|
||||
"control character.png",
|
||||
"deletecharacter.png",
|
||||
"\ud800lone-surrogate.png",
|
||||
"x".repeat(MODULE_STORAGE_KEY_MAX_BYTES + 1),
|
||||
]) {
|
||||
expect(isValidModuleStorageKey(key)).toBe(false);
|
||||
}
|
||||
});
|
||||
|
||||
it("measures the length limit in UTF-8 bytes, not characters", () => {
|
||||
// 512 three-byte characters is 1536 bytes — over the limit despite being 512 characters.
|
||||
expect(isValidModuleStorageKey("あ".repeat(512))).toBe(false);
|
||||
expect(isValidModuleStorageKey("あ".repeat(341))).toBe(true);
|
||||
});
|
||||
|
||||
it("does not treat a key as a filesystem path", () => {
|
||||
// `..` has no traversal meaning to an object store, and the physical bucket never comes from the
|
||||
// key, so this is an ordinary (if odd) object name rather than an escape.
|
||||
expect(isValidModuleStorageKey("../still-just-a-name.png")).toBe(true);
|
||||
});
|
||||
});
|
||||
|
||||
describe("module storage scoping", () => {
|
||||
const provider = () => fakeProvider();
|
||||
|
||||
it("gives a module exactly the buckets its manifest declared", () => {
|
||||
const storage = createModuleStorage({
|
||||
bucketPrefix: "ekklesia",
|
||||
logicalBucketIds: ["announcement-assets", "thumbnails"],
|
||||
moduleId: "announcements",
|
||||
provider: provider(),
|
||||
});
|
||||
|
||||
expect(storage.buckets()).toEqual(["announcement-assets", "thumbnails"]);
|
||||
expect(storage.bucket("announcement-assets").id).toBe("announcement-assets");
|
||||
});
|
||||
|
||||
it("refuses a logical bucket the module did not declare", () => {
|
||||
const storage = createModuleStorage({
|
||||
bucketPrefix: "ekklesia",
|
||||
logicalBucketIds: ["announcement-assets"],
|
||||
moduleId: "announcements",
|
||||
provider: provider(),
|
||||
});
|
||||
|
||||
expect(() => storage.bucket("video-assets")).toThrow(ModuleStorageError);
|
||||
try {
|
||||
storage.bucket("video-assets");
|
||||
} catch (error) {
|
||||
expect((error as ModuleStorageError).code).toBe("STORAGE_BUCKET_UNDECLARED");
|
||||
}
|
||||
});
|
||||
|
||||
it("gives a module no way to reach another module's storage by knowing its name", async () => {
|
||||
const shared = fakeProvider();
|
||||
const announcements = createModuleStorage({
|
||||
bucketPrefix: "ekklesia",
|
||||
logicalBucketIds: ["assets"],
|
||||
moduleId: "announcements",
|
||||
provider: shared,
|
||||
});
|
||||
const video = createModuleStorage({
|
||||
bucketPrefix: "ekklesia",
|
||||
logicalBucketIds: ["assets"],
|
||||
moduleId: "video",
|
||||
provider: shared,
|
||||
});
|
||||
|
||||
await announcements.bucket("assets").put("shared-name.txt", "from-announcements");
|
||||
await video.bucket("assets").put("shared-name.txt", "from-video");
|
||||
|
||||
// Same logical name, same key, same provider — and two different objects, because the physical
|
||||
// namespaces differ.
|
||||
const decoder = new TextDecoder();
|
||||
const fromAnnouncements = decoder.decode(
|
||||
await collect((await announcements.bucket("assets").get("shared-name.txt")).body),
|
||||
);
|
||||
const fromVideo = decoder.decode(
|
||||
await collect((await video.bucket("assets").get("shared-name.txt")).body),
|
||||
);
|
||||
|
||||
expect(fromAnnouncements).toBe("from-announcements");
|
||||
expect(fromVideo).toBe("from-video");
|
||||
expect(shared.buckets.size).toBe(2);
|
||||
});
|
||||
|
||||
it("exposes no physical bucket, endpoint, or credential through the module-facing surface", () => {
|
||||
const storage = createModuleStorage({
|
||||
bucketPrefix: "ekklesia",
|
||||
logicalBucketIds: ["announcement-assets"],
|
||||
moduleId: "announcements",
|
||||
provider: provider(),
|
||||
});
|
||||
const bucket = storage.bucket("announcement-assets");
|
||||
|
||||
// The handle names the *logical* bucket, and has no accessor for anything else.
|
||||
expect(bucket.id).toBe("announcement-assets");
|
||||
expect(Object.keys(bucket).sort()).toEqual(["delete", "get", "head", "id", "list", "put"]);
|
||||
expect(Object.keys(storage).sort()).toEqual(["bucket", "buckets"]);
|
||||
const serialized = JSON.stringify({ bucket, storage });
|
||||
expect(serialized).not.toContain(
|
||||
physicalBucketName("ekklesia", "announcements", "announcement-assets"),
|
||||
);
|
||||
expect(serialized).not.toContain("http");
|
||||
});
|
||||
|
||||
it("gives a module declaring no buckets an empty, frozen storage that refuses everything", () => {
|
||||
const storage = createModuleStorage({
|
||||
bucketPrefix: "ekklesia",
|
||||
logicalBucketIds: [],
|
||||
moduleId: "calendar",
|
||||
provider: provider(),
|
||||
});
|
||||
|
||||
expect(storage).toBe(EMPTY_MODULE_STORAGE);
|
||||
expect(storage.buckets()).toEqual([]);
|
||||
expect(() => storage.bucket("anything")).toThrow(ModuleStorageError);
|
||||
expect(Object.isFrozen(storage)).toBe(true);
|
||||
});
|
||||
|
||||
it("resolves nothing at all when the host has no provider", () => {
|
||||
const lookup = createModuleStorageLookup(undefined, "ekklesia", [
|
||||
{ logicalBucketIds: ["assets"], moduleId: "announcements" },
|
||||
]);
|
||||
|
||||
expect(lookup.size).toBe(0);
|
||||
});
|
||||
|
||||
it.each([
|
||||
"__proto__",
|
||||
"constructor",
|
||||
"prototype",
|
||||
"toString",
|
||||
])("treats the reserved property name %s as an ordinary undeclared bucket", (name) => {
|
||||
const storage = createModuleStorage({
|
||||
bucketPrefix: "ekklesia",
|
||||
logicalBucketIds: ["assets"],
|
||||
moduleId: "announcements",
|
||||
provider: provider(),
|
||||
});
|
||||
|
||||
expect(() => storage.bucket(name)).toThrow(ModuleStorageError);
|
||||
});
|
||||
});
|
||||
|
||||
describe("bucket provisioning", () => {
|
||||
it("ensures the bucket lazily on first use, not at construction", async () => {
|
||||
const shared = fakeProvider();
|
||||
const storage = createModuleStorage({
|
||||
bucketPrefix: "ekklesia",
|
||||
logicalBucketIds: ["assets"],
|
||||
moduleId: "announcements",
|
||||
provider: shared,
|
||||
});
|
||||
|
||||
// Constructing the facade — which is what startup does — touches the provider zero times, so a
|
||||
// storage outage can never stop the platform from booting.
|
||||
expect(shared.calls).toHaveLength(0);
|
||||
|
||||
await storage.bucket("assets").list();
|
||||
expect(shared.calls.filter((call) => call.operation === "ensureBucket")).toHaveLength(1);
|
||||
});
|
||||
|
||||
it("ensures at most once per binding across many operations", async () => {
|
||||
const shared = fakeProvider();
|
||||
const bucket = createModuleStorage({
|
||||
bucketPrefix: "ekklesia",
|
||||
logicalBucketIds: ["assets"],
|
||||
moduleId: "announcements",
|
||||
provider: shared,
|
||||
}).bucket("assets");
|
||||
|
||||
await Promise.all([bucket.list(), bucket.list(), bucket.put("a.txt", "a")]);
|
||||
await bucket.list();
|
||||
|
||||
expect(shared.calls.filter((call) => call.operation === "ensureBucket")).toHaveLength(1);
|
||||
});
|
||||
|
||||
it("retries provisioning after a transient failure rather than poisoning the binding", async () => {
|
||||
let failures = 1;
|
||||
const shared = fakeProvider({
|
||||
failEnsure: () =>
|
||||
failures-- > 0 ? new ModuleStorageError("STORAGE_PROVIDER_UNAVAILABLE") : undefined!,
|
||||
});
|
||||
const bucket = createModuleStorage({
|
||||
bucketPrefix: "ekklesia",
|
||||
logicalBucketIds: ["assets"],
|
||||
moduleId: "announcements",
|
||||
provider: shared,
|
||||
}).bucket("assets");
|
||||
|
||||
await expect(bucket.list()).rejects.toMatchObject({ code: "STORAGE_PROVIDER_UNAVAILABLE" });
|
||||
// The next call tries again. A memoized *failure* would leave a module permanently broken by one
|
||||
// network blip, which is exactly the resilience the real Announcements product depends on.
|
||||
await expect(bucket.list()).resolves.toEqual({ objects: [] });
|
||||
});
|
||||
|
||||
it("never offers a way to delete a bucket", () => {
|
||||
const storage = createModuleStorage({
|
||||
bucketPrefix: "ekklesia",
|
||||
logicalBucketIds: ["assets"],
|
||||
moduleId: "announcements",
|
||||
provider: fakeProvider(),
|
||||
});
|
||||
|
||||
const surface = [...Object.keys(storage), ...Object.keys(storage.bucket("assets"))];
|
||||
expect(surface.join(" ")).not.toMatch(/deleteBucket|destroy|drop|removeBucket/iu);
|
||||
});
|
||||
});
|
||||
|
||||
describe("storage operations through the scoped facade", () => {
|
||||
async function announcementsStorage() {
|
||||
const provider = fakeProvider();
|
||||
const storage = createModuleStorage({
|
||||
bucketPrefix: "ekklesia",
|
||||
logicalBucketIds: ["announcement-assets"],
|
||||
moduleId: "announcements",
|
||||
provider,
|
||||
});
|
||||
return { bucket: storage.bucket("announcement-assets"), provider };
|
||||
}
|
||||
|
||||
it("round-trips an object and preserves its content type", async () => {
|
||||
const { bucket } = await announcementsStorage();
|
||||
const bytes = new Uint8Array([0x89, 0x50, 0x4e, 0x47]);
|
||||
|
||||
await bucket.put("slides/welcome.png", bytes, { contentType: "image/png" });
|
||||
const object = await bucket.get("slides/welcome.png");
|
||||
|
||||
expect(object.contentType).toBe("image/png");
|
||||
expect(await collect(object.body)).toEqual(bytes);
|
||||
});
|
||||
|
||||
it("lists with a prefix and deletes", async () => {
|
||||
const { bucket } = await announcementsStorage();
|
||||
await bucket.put("slides/a.png", "a");
|
||||
await bucket.put("slides/b.png", "b");
|
||||
await bucket.put("other/c.png", "c");
|
||||
|
||||
expect((await bucket.list({ prefix: "slides/" })).objects.map((o) => o.key)).toEqual([
|
||||
"slides/a.png",
|
||||
"slides/b.png",
|
||||
]);
|
||||
|
||||
await bucket.delete("slides/a.png");
|
||||
expect((await bucket.list({ prefix: "slides/" })).objects.map((o) => o.key)).toEqual([
|
||||
"slides/b.png",
|
||||
]);
|
||||
});
|
||||
|
||||
it("reports a missing object as a structured not-found error", async () => {
|
||||
const { bucket } = await announcementsStorage();
|
||||
await bucket.put("present.png", "x");
|
||||
|
||||
await expect(bucket.get("absent.png")).rejects.toMatchObject({
|
||||
code: "STORAGE_OBJECT_NOT_FOUND",
|
||||
key: "absent.png",
|
||||
});
|
||||
await expect(bucket.head("absent.png")).rejects.toMatchObject({
|
||||
code: "STORAGE_OBJECT_NOT_FOUND",
|
||||
});
|
||||
});
|
||||
|
||||
it("rejects an invalid key before reaching the provider", async () => {
|
||||
const { bucket, provider } = await announcementsStorage();
|
||||
|
||||
await expect(bucket.get("/leading-slash.png")).rejects.toMatchObject({
|
||||
code: "STORAGE_KEY_INVALID",
|
||||
});
|
||||
// Not even the bucket was ensured: validation happens first.
|
||||
expect(provider.calls).toHaveLength(0);
|
||||
});
|
||||
|
||||
it("refuses a streamed body with no content length rather than buffering it", async () => {
|
||||
const { bucket } = await announcementsStorage();
|
||||
const stream = streamOf(new TextEncoder().encode("x"));
|
||||
|
||||
await expect(bucket.put("streamed.bin", stream)).rejects.toMatchObject({
|
||||
code: "STORAGE_CONTENT_LENGTH_REQUIRED",
|
||||
});
|
||||
});
|
||||
|
||||
it("accepts a streamed body when the length is supplied", async () => {
|
||||
const { bucket } = await announcementsStorage();
|
||||
const stream = streamOf(new TextEncoder().encode("hello"));
|
||||
|
||||
const metadata = await bucket.put("streamed.bin", stream, { contentLength: 5 });
|
||||
expect(metadata.key).toBe("streamed.bin");
|
||||
});
|
||||
});
|
||||
|
||||
describe("storage provider configuration", () => {
|
||||
// Distinctive values, deliberately: a generic `"key"` would make a "no credential leaked"
|
||||
// assertion pass or fail on whether a fixed error message happens to use the English word.
|
||||
const base = {
|
||||
accessKey: "AKIAEXAMPLEACCESSID",
|
||||
endpoint: "https://storage.example.test",
|
||||
secretKey: "sk-example-secret-material",
|
||||
};
|
||||
|
||||
it("treats an absent endpoint as an installation with no object storage", () => {
|
||||
expect(createStorageProviderConfig(undefined)).toBeUndefined();
|
||||
expect(createStorageProviderConfig({ endpoint: "" })).toBeUndefined();
|
||||
expect(createStorageProviderConfig({})).toBeUndefined();
|
||||
});
|
||||
|
||||
it("accepts a valid S3-compatible configuration with safe defaults", () => {
|
||||
const config = createStorageProviderConfig(base);
|
||||
|
||||
expect(config).toEqual({
|
||||
accessKey: "AKIAEXAMPLEACCESSID",
|
||||
bucketPrefix: "ekklesia",
|
||||
endpoint: "https://storage.example.test",
|
||||
// Path-style by default: every self-hosted S3-compatible service needs it, and virtual-host
|
||||
// addressing would require wildcard DNS a typical deployment does not have.
|
||||
forcePathStyle: true,
|
||||
region: "us-east-1",
|
||||
secretKey: "sk-example-secret-material",
|
||||
});
|
||||
});
|
||||
|
||||
it("requires both credentials once an endpoint is set", () => {
|
||||
expect(() => createStorageProviderConfig({ ...base, accessKey: undefined })).toThrow(
|
||||
expect.objectContaining({ code: "STORAGE_ACCESS_KEY_MISSING" }),
|
||||
);
|
||||
expect(() => createStorageProviderConfig({ ...base, secretKey: " " })).toThrow(
|
||||
expect.objectContaining({ code: "STORAGE_SECRET_KEY_MISSING" }),
|
||||
);
|
||||
});
|
||||
|
||||
it("rejects an endpoint that embeds credentials", () => {
|
||||
expect(() =>
|
||||
createStorageProviderConfig({ ...base, endpoint: "https://key:[email protected]" }),
|
||||
).toThrow(expect.objectContaining({ code: "STORAGE_ENDPOINT_WITH_CREDENTIALS" }));
|
||||
});
|
||||
|
||||
it.each([
|
||||
"not-a-url",
|
||||
"ftp://storage.example.test",
|
||||
"//storage.example.test",
|
||||
])("rejects the malformed endpoint %s", (endpoint) => {
|
||||
expect(() => createStorageProviderConfig({ ...base, endpoint })).toThrow(
|
||||
expect.objectContaining({ code: "STORAGE_ENDPOINT_INVALID" }),
|
||||
);
|
||||
});
|
||||
|
||||
it("requires https in production but allows http in development", () => {
|
||||
const http = { ...base, endpoint: "http://127.0.0.1:8333" };
|
||||
|
||||
expect(() => createStorageProviderConfig(http, { environment: "production" })).toThrow(
|
||||
expect.objectContaining({ code: "STORAGE_ENDPOINT_INSECURE_IN_PRODUCTION" }),
|
||||
);
|
||||
expect(createStorageProviderConfig(http, { environment: "development" })?.endpoint).toBe(
|
||||
"http://127.0.0.1:8333",
|
||||
);
|
||||
});
|
||||
|
||||
it("rejects an unsafe bucket prefix", () => {
|
||||
expect(() => createStorageProviderConfig({ ...base, bucketPrefix: "Not Valid" })).toThrow(
|
||||
expect.objectContaining({ code: "STORAGE_BUCKET_PREFIX_INVALID" }),
|
||||
);
|
||||
});
|
||||
|
||||
it("never carries a configured value in a configuration error", () => {
|
||||
let captured: unknown;
|
||||
try {
|
||||
createStorageProviderConfig({
|
||||
...base,
|
||||
endpoint: "https://super-secret-host.invalid",
|
||||
secretKey: undefined,
|
||||
});
|
||||
} catch (error) {
|
||||
captured = error;
|
||||
}
|
||||
|
||||
const rendered = `${String(captured)} ${JSON.stringify(captured)}`;
|
||||
// No configured *value* survives: not the endpoint, not either credential. (The fixed message
|
||||
// legitimately contains the word "key" — what must never appear is a key's value.)
|
||||
expect(rendered).not.toContain("super-secret-host.invalid");
|
||||
expect(rendered).not.toContain(base.accessKey);
|
||||
expect(rendered).not.toContain(base.secretKey);
|
||||
});
|
||||
|
||||
it("describes the provider for an operator without printing a credential", () => {
|
||||
const config = createStorageProviderConfig({
|
||||
...base,
|
||||
accessKey: "AKIAVERYSECRETKEY",
|
||||
endpoint: "https://storage.example.test/ignored-path",
|
||||
secretKey: "a-very-secret-value",
|
||||
});
|
||||
if (!config) {
|
||||
throw new Error("expected a configuration");
|
||||
}
|
||||
|
||||
const description = describeStorageProvider(config);
|
||||
expect(description).toContain("https://storage.example.test");
|
||||
expect(description).toContain("path-style");
|
||||
expect(description).not.toContain("AKIAVERYSECRETKEY");
|
||||
expect(description).not.toContain("a-very-secret-value");
|
||||
});
|
||||
});
|
||||
|
||||
/**
|
||||
* The Announcements-shaped proof.
|
||||
*
|
||||
* Modelled on the real product's requirements — a bucket of announcement images, listed and read —
|
||||
* and never importing it. The point is what the fixture *cannot* see: it names a logical bucket and
|
||||
* object keys, and has no access to SeaweedFS, an endpoint, an access key, or the physical bucket.
|
||||
*/
|
||||
describe("an Announcements-shaped module reads its slides without knowing any infrastructure", () => {
|
||||
/** Everything this fixture knows. Compare with the real product, which today hardcodes
|
||||
* `http://10.40.0.2:9000/announcements?list-type=2` and parses S3 XML by hand. */
|
||||
const LOGICAL_BUCKET = "announcement-assets";
|
||||
|
||||
async function loadSlideKeys(storage: {
|
||||
bucket: (id: string) => {
|
||||
list: (options?: ModuleStorageListOptions) => Promise<ModuleStorageListing>;
|
||||
};
|
||||
}): Promise<readonly string[]> {
|
||||
const listing = await storage.bucket(LOGICAL_BUCKET).list();
|
||||
// Extension filtering is the module's own domain logic, exactly as it is today — Core has no
|
||||
// opinion about which objects are slides.
|
||||
return listing.objects
|
||||
.map((object) => object.key)
|
||||
.filter((key) => /\.(png|jpe?g|webp)$/iu.test(key));
|
||||
}
|
||||
|
||||
it("lists and reads its own images through the logical bucket alone", async () => {
|
||||
const provider = fakeProvider();
|
||||
const storage = createModuleStorage({
|
||||
bucketPrefix: "ekklesia",
|
||||
logicalBucketIds: [LOGICAL_BUCKET],
|
||||
moduleId: "announcements",
|
||||
provider,
|
||||
});
|
||||
const bucket = storage.bucket(LOGICAL_BUCKET);
|
||||
await bucket.put("welcome.png", new Uint8Array([1, 2, 3]), { contentType: "image/png" });
|
||||
await bucket.put("notes.txt", "not a slide");
|
||||
|
||||
expect(await loadSlideKeys(storage)).toEqual(["welcome.png"]);
|
||||
|
||||
const slide = await bucket.get("welcome.png");
|
||||
expect(slide.contentType).toBe("image/png");
|
||||
// Straight into a `Response` — the shape a module route uses to serve an image to a browser
|
||||
// without the browser ever learning that object storage exists.
|
||||
// The shape a module route uses: the stream and its content type go straight into a `Response`,
|
||||
// so a browser receives the image without ever learning that object storage exists.
|
||||
const headers = new Headers();
|
||||
if (slide.contentType) {
|
||||
headers.set("content-type", slide.contentType);
|
||||
}
|
||||
expect(headers.get("content-type")).toBe("image/png");
|
||||
expect(await collect(slide.body)).toEqual(new Uint8Array([1, 2, 3]));
|
||||
});
|
||||
|
||||
it("knows the logical bucket and object keys, and nothing about the host", async () => {
|
||||
const provider = fakeProvider();
|
||||
const storage = createModuleStorage({
|
||||
bucketPrefix: "ekklesia",
|
||||
logicalBucketIds: [LOGICAL_BUCKET],
|
||||
moduleId: "announcements",
|
||||
provider,
|
||||
});
|
||||
await storage.bucket(LOGICAL_BUCKET).list();
|
||||
|
||||
const reachable = JSON.stringify({
|
||||
bucketHandle: storage.bucket(LOGICAL_BUCKET),
|
||||
buckets: storage.buckets(),
|
||||
storage,
|
||||
});
|
||||
for (const infrastructure of [
|
||||
"seaweed",
|
||||
"SeaweedFS",
|
||||
"10.40.0.2",
|
||||
"9000",
|
||||
"8333",
|
||||
"s3",
|
||||
"S3",
|
||||
"accessKey",
|
||||
"secretKey",
|
||||
physicalBucketName("ekklesia", "announcements", LOGICAL_BUCKET),
|
||||
]) {
|
||||
expect(reachable).not.toContain(infrastructure);
|
||||
}
|
||||
// The physical bucket exists — the host created it — but only the provider ever saw its name.
|
||||
expect([...provider.buckets.keys()]).toEqual([
|
||||
physicalBucketName("ekklesia", "announcements", LOGICAL_BUCKET),
|
||||
]);
|
||||
});
|
||||
|
||||
it("surfaces a storage outage to the module without any lifecycle state changing", async () => {
|
||||
const provider = fakeProvider({
|
||||
failEnsure: () => new ModuleStorageError("STORAGE_PROVIDER_UNAVAILABLE"),
|
||||
});
|
||||
const bucket = createModuleStorage({
|
||||
bucketPrefix: "ekklesia",
|
||||
logicalBucketIds: [LOGICAL_BUCKET],
|
||||
moduleId: "announcements",
|
||||
provider,
|
||||
}).bucket(LOGICAL_BUCKET);
|
||||
|
||||
// Exactly what the real product does today: catch, keep the previous slides, and retry on the
|
||||
// next poll. Nothing here writes anything, anywhere — a transient outage is not an
|
||||
// administrative decision (ADR 0030).
|
||||
let caught: unknown;
|
||||
try {
|
||||
await bucket.list();
|
||||
} catch (error) {
|
||||
caught = error;
|
||||
}
|
||||
|
||||
expect(caught).toBeInstanceOf(ModuleStorageError);
|
||||
expect((caught as ModuleStorageError).code).toBe("STORAGE_PROVIDER_UNAVAILABLE");
|
||||
expect(JSON.stringify(caught)).not.toContain("http");
|
||||
});
|
||||
});
|
||||
Reference in New Issue
Block a user