Implement Phase 2 asset storage pipeline
This commit is contained in:
Media
@@ -0,0 +1,442 @@
|
||||
const test = require("node:test");
|
||||
const assert = require("node:assert/strict");
|
||||
const crypto = require("node:crypto");
|
||||
const fs = require("node:fs");
|
||||
const http = require("node:http");
|
||||
const os = require("node:os");
|
||||
const path = require("node:path");
|
||||
const { Readable } = require("node:stream");
|
||||
const sharp = require("sharp");
|
||||
const { createApp } = require("../src/app");
|
||||
const { loadConfig } = require("../src/config");
|
||||
const { openDatabase } = require("../src/database");
|
||||
const { runMigrations } = require("../src/database/migrate");
|
||||
const { createImageProcessor } = require("../src/image-processing");
|
||||
const { StorageObjectNotFoundError } = require("../src/storage");
|
||||
|
||||
async function readBody(body) {
|
||||
if (Buffer.isBuffer(body)) return body;
|
||||
const chunks = [];
|
||||
for await (const chunk of body) chunks.push(Buffer.from(chunk));
|
||||
return Buffer.concat(chunks);
|
||||
}
|
||||
|
||||
function memoryStorage() {
|
||||
const objects = new Map();
|
||||
return {
|
||||
objects,
|
||||
health() {
|
||||
return {
|
||||
status: "healthy",
|
||||
checkedAt: "2033-05-18T03:33:20.000Z",
|
||||
lastSuccessAt: "2033-05-18T03:33:20.000Z",
|
||||
};
|
||||
},
|
||||
async putObject(input) {
|
||||
objects.set(input.key, {
|
||||
body: await readBody(input.body),
|
||||
contentType: input.contentType,
|
||||
metadata: input.metadata,
|
||||
});
|
||||
return { key: input.key };
|
||||
},
|
||||
async getObject(key) {
|
||||
const object = objects.get(key);
|
||||
if (!object) throw new StorageObjectNotFoundError(key);
|
||||
return {
|
||||
body: Readable.from([object.body]),
|
||||
contentLength: object.body.length,
|
||||
contentType: object.contentType,
|
||||
};
|
||||
},
|
||||
async deleteObject(key) {
|
||||
objects.delete(key);
|
||||
return { key, deleted: true };
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
async function createHarness(t, { maxFileBytes = 1024 * 1024 } = {}) {
|
||||
const tempDirectory = fs.mkdtempSync(
|
||||
path.join(os.tmpdir(), "announcements-api-uploads-"),
|
||||
);
|
||||
const config = loadConfig({
|
||||
NODE_ENV: "test",
|
||||
HOST: "127.0.0.1",
|
||||
PORT: "0",
|
||||
DATABASE_PATH: ":memory:",
|
||||
APP_TIMEZONE: "America/Chicago",
|
||||
S3_ENDPOINT: "http://private-seaweedfs:8333",
|
||||
S3_BUCKET: "announcements-test",
|
||||
UPLOAD_TEMP_DIRECTORY: tempDirectory,
|
||||
UPLOAD_MAX_FILE_BYTES: String(maxFileBytes),
|
||||
UPLOAD_MAX_FILES: "5",
|
||||
IMAGE_MAX_INPUT_PIXELS: "1000000",
|
||||
});
|
||||
const database = openDatabase({ path: ":memory:" });
|
||||
const migrations = runMigrations(database);
|
||||
const storage = memoryStorage();
|
||||
const app = createApp({
|
||||
config,
|
||||
database,
|
||||
imageProcessor: createImageProcessor({
|
||||
maxInputPixels: config.processing.maxInputPixels,
|
||||
}),
|
||||
lifecycle: { ready: true },
|
||||
migrations,
|
||||
storage,
|
||||
});
|
||||
const server = app.listen(0, "127.0.0.1");
|
||||
await new Promise((resolve, reject) => {
|
||||
server.once("listening", resolve);
|
||||
server.once("error", reject);
|
||||
});
|
||||
|
||||
t.after(async () => {
|
||||
await new Promise((resolve) => server.close(resolve));
|
||||
database.close();
|
||||
fs.rmSync(tempDirectory, { recursive: true, force: true });
|
||||
});
|
||||
|
||||
return {
|
||||
baseUrl: `http://127.0.0.1:${server.address().port}`,
|
||||
database,
|
||||
storage,
|
||||
tempDirectory,
|
||||
};
|
||||
}
|
||||
|
||||
function appendFile(form, field, buffer, filename, contentType) {
|
||||
form.append(field, new Blob([buffer], { type: contentType }), filename);
|
||||
}
|
||||
|
||||
async function jsonResponse(baseUrl, route, init) {
|
||||
const response = await fetch(`${baseUrl}${route}`, init);
|
||||
return { response, body: await response.json() };
|
||||
}
|
||||
|
||||
async function eventuallyEmpty(directory) {
|
||||
for (let attempt = 0; attempt < 50; attempt += 1) {
|
||||
if (fs.readdirSync(directory).length === 0) return;
|
||||
await new Promise((resolve) => setTimeout(resolve, 10));
|
||||
}
|
||||
assert.deepEqual(fs.readdirSync(directory), []);
|
||||
}
|
||||
|
||||
async function eventuallyContainsFile(directory) {
|
||||
for (let attempt = 0; attempt < 50; attempt += 1) {
|
||||
if (fs.readdirSync(directory).length > 0) return;
|
||||
await new Promise((resolve) => setTimeout(resolve, 10));
|
||||
}
|
||||
assert.fail("The upload did not create its temporary file");
|
||||
}
|
||||
|
||||
async function sampleImages() {
|
||||
const source = sharp({
|
||||
create: {
|
||||
width: 80,
|
||||
height: 60,
|
||||
channels: 4,
|
||||
background: { r: 31, g: 72, b: 120, alpha: 0.5 },
|
||||
},
|
||||
});
|
||||
return {
|
||||
jpeg: await source.clone().jpeg().toBuffer(),
|
||||
png: await source.clone().png().toBuffer(),
|
||||
webp: await source.clone().webp().toBuffer(),
|
||||
};
|
||||
}
|
||||
|
||||
test("asset HTTP API accepts JPEG, PNG, and WebP and serves private immutable media", async (t) => {
|
||||
const harness = await createHarness(t);
|
||||
const images = await sampleImages();
|
||||
const form = new FormData();
|
||||
appendFile(form, "files", images.jpeg, "welcome.jpg", "image/jpeg");
|
||||
appendFile(form, "files", images.png, "../../bulletin.png", "image/png");
|
||||
appendFile(form, "files", images.webp, "events.webp", "image/webp");
|
||||
|
||||
const uploaded = await jsonResponse(harness.baseUrl, "/api/assets", {
|
||||
method: "POST",
|
||||
body: form,
|
||||
});
|
||||
assert.equal(uploaded.response.status, 201);
|
||||
assert.equal(uploaded.body.createdCount, 3);
|
||||
assert.equal(uploaded.body.duplicateCount, 0);
|
||||
assert.equal(uploaded.body.results.length, 3);
|
||||
assert.deepEqual(
|
||||
uploaded.body.results.map((result) => result.asset.original.contentType),
|
||||
["image/jpeg", "image/png", "image/webp"],
|
||||
);
|
||||
assert.equal(uploaded.body.results[1].asset.originalFilename, "bulletin.png");
|
||||
assert.doesNotMatch(JSON.stringify(uploaded.body), /objectKey|object_key|private-seaweedfs/);
|
||||
assert.equal(harness.storage.objects.size, 9);
|
||||
await eventuallyEmpty(harness.tempDirectory);
|
||||
|
||||
const asset = uploaded.body.results[0].asset;
|
||||
const listed = await jsonResponse(harness.baseUrl, "/api/assets");
|
||||
assert.equal(listed.response.status, 200);
|
||||
assert.equal(listed.body.assets.length, 3);
|
||||
assert.deepEqual(
|
||||
(await jsonResponse(harness.baseUrl, `/api/assets/${asset.id}`)).body,
|
||||
asset,
|
||||
);
|
||||
|
||||
for (const media of [asset.display, asset.thumbnail]) {
|
||||
const response = await fetch(`${harness.baseUrl}${media.url}`);
|
||||
const body = Buffer.from(await response.arrayBuffer());
|
||||
assert.equal(response.status, 200);
|
||||
assert.equal(response.headers.get("content-type"), "image/webp");
|
||||
assert.equal(
|
||||
response.headers.get("cache-control"),
|
||||
"public, max-age=31536000, immutable",
|
||||
);
|
||||
assert.equal(response.headers.get("content-length"), String(body.length));
|
||||
assert.equal(response.headers.get("etag"), `"${media.sha256}"`);
|
||||
assert.equal(crypto.createHash("sha256").update(body).digest("hex"), media.sha256);
|
||||
}
|
||||
|
||||
const original = await fetch(`${harness.baseUrl}${asset.original.url}`);
|
||||
assert.equal(original.status, 200);
|
||||
assert.equal(original.headers.get("content-type"), "image/jpeg");
|
||||
assert.equal(original.headers.get("cache-control"), "private, no-store");
|
||||
assert.match(original.headers.get("content-disposition"), /^attachment;/);
|
||||
assert.deepEqual(Buffer.from(await original.arrayBuffer()), images.jpeg);
|
||||
|
||||
const wrongVersion = await jsonResponse(
|
||||
harness.baseUrl,
|
||||
`/media/assets/${asset.id}/display/${"a".repeat(64)}.webp`,
|
||||
);
|
||||
assert.equal(wrongVersion.response.status, 404);
|
||||
assert.equal(wrongVersion.body.error.code, "MEDIA_NOT_FOUND");
|
||||
const arbitraryKey = await jsonResponse(
|
||||
harness.baseUrl,
|
||||
`/media/assets/${asset.id}/display/${encodeURIComponent("../../original/key")}.webp`,
|
||||
);
|
||||
assert.equal(arbitraryKey.response.status, 404);
|
||||
assert.equal(arbitraryKey.body.error.code, "MEDIA_NOT_FOUND");
|
||||
|
||||
const displayKey = [...harness.storage.objects.keys()].find(
|
||||
(key) => key.includes(`assets/${asset.id}/display/`),
|
||||
);
|
||||
harness.storage.objects.delete(displayKey);
|
||||
const missingStoredBytes = await jsonResponse(
|
||||
harness.baseUrl,
|
||||
asset.display.url,
|
||||
);
|
||||
assert.equal(missingStoredBytes.response.status, 503);
|
||||
assert.equal(missingStoredBytes.body.error.code, "STORAGE_INCONSISTENCY");
|
||||
assert.equal(
|
||||
(await jsonResponse(harness.baseUrl, "/health/live")).response.status,
|
||||
200,
|
||||
);
|
||||
});
|
||||
|
||||
test("asset HTTP API reports duplicates, name conflicts, and persists editable metadata", async (t) => {
|
||||
const harness = await createHarness(t);
|
||||
const { png, jpeg } = await sampleImages();
|
||||
const firstForm = new FormData();
|
||||
appendFile(firstForm, "file", png, "weekly.png", "application/octet-stream");
|
||||
firstForm.append("display_name", "Weekly Announcement");
|
||||
const first = await jsonResponse(harness.baseUrl, "/api/assets", {
|
||||
method: "POST",
|
||||
body: firstForm,
|
||||
});
|
||||
assert.equal(first.response.status, 201);
|
||||
assert.equal(first.body.created, true);
|
||||
const assetId = first.body.asset.id;
|
||||
|
||||
const duplicateForm = new FormData();
|
||||
appendFile(duplicateForm, "file", png, "unrelated-name.png", "image/png");
|
||||
duplicateForm.append("display_name", "Ignored duplicate proposal");
|
||||
const duplicate = await jsonResponse(harness.baseUrl, "/api/assets", {
|
||||
method: "POST",
|
||||
body: duplicateForm,
|
||||
});
|
||||
assert.equal(duplicate.response.status, 200);
|
||||
assert.equal(duplicate.body.created, false);
|
||||
assert.equal(duplicate.body.duplicate, true);
|
||||
assert.equal(duplicate.body.asset.id, assetId);
|
||||
assert.equal(harness.storage.objects.size, 3);
|
||||
|
||||
const conflictForm = new FormData();
|
||||
appendFile(conflictForm, "file", jpeg, "different.jpg", "image/jpeg");
|
||||
conflictForm.append("displayName", "weekly announcement");
|
||||
const conflict = await jsonResponse(harness.baseUrl, "/api/assets", {
|
||||
method: "POST",
|
||||
body: conflictForm,
|
||||
});
|
||||
assert.equal(conflict.response.status, 409);
|
||||
assert.equal(conflict.body.error.code, "ASSET_NAME_CONFLICT");
|
||||
|
||||
const expiresAt = Date.parse("2034-01-02T03:04:05.000Z");
|
||||
const edited = await jsonResponse(harness.baseUrl, `/api/assets/${assetId}`, {
|
||||
method: "PATCH",
|
||||
headers: { "content-type": "application/json" },
|
||||
body: JSON.stringify({
|
||||
display_name: "Renamed announcement",
|
||||
expiration_at: expiresAt,
|
||||
}),
|
||||
});
|
||||
assert.equal(edited.response.status, 200);
|
||||
assert.equal(edited.body.displayName, "Renamed announcement");
|
||||
assert.equal(edited.body.expirationAt, expiresAt);
|
||||
const permanent = await jsonResponse(harness.baseUrl, `/api/assets/${assetId}`, {
|
||||
method: "PATCH",
|
||||
headers: { "content-type": "application/json" },
|
||||
body: JSON.stringify({ expiration_at: null }),
|
||||
});
|
||||
assert.equal(permanent.body.expirationAt, null);
|
||||
assert.equal(harness.storage.objects.size, 3);
|
||||
await eventuallyEmpty(harness.tempDirectory);
|
||||
|
||||
const timestamp = Date.now();
|
||||
harness.database
|
||||
.prepare(
|
||||
"INSERT INTO playlists (id, name, created_at, updated_at) VALUES (?, ?, ?, ?)",
|
||||
)
|
||||
.run("api-playlist", "API playlist", timestamp, timestamp);
|
||||
harness.database
|
||||
.prepare(
|
||||
`INSERT INTO playlist_entries
|
||||
(id, playlist_id, asset_id, position, created_at, updated_at)
|
||||
VALUES (?, ?, ?, 0, ?, ?)`,
|
||||
)
|
||||
.run("api-entry", "api-playlist", assetId, timestamp, timestamp);
|
||||
const references = await jsonResponse(
|
||||
harness.baseUrl,
|
||||
`/api/assets/${assetId}/references`,
|
||||
);
|
||||
assert.equal(references.response.status, 200);
|
||||
assert.equal(references.body.canDelete, false);
|
||||
assert.equal(references.body.playlists[0].playlistEntryId, "api-entry");
|
||||
const blockedDelete = await jsonResponse(
|
||||
harness.baseUrl,
|
||||
`/api/assets/${assetId}`,
|
||||
{ method: "DELETE" },
|
||||
);
|
||||
assert.equal(blockedDelete.response.status, 409);
|
||||
assert.equal(blockedDelete.body.error.code, "ASSET_REFERENCED");
|
||||
assert.equal(harness.storage.objects.size, 3);
|
||||
harness.database
|
||||
.prepare("DELETE FROM playlist_entries WHERE id = ?")
|
||||
.run("api-entry");
|
||||
|
||||
const removed = await jsonResponse(harness.baseUrl, `/api/assets/${assetId}`, {
|
||||
method: "DELETE",
|
||||
});
|
||||
assert.equal(removed.response.status, 200);
|
||||
assert.deepEqual(removed.body, { id: assetId, deleted: true });
|
||||
assert.equal(harness.storage.objects.size, 0);
|
||||
assert.equal(
|
||||
(await jsonResponse(harness.baseUrl, `/api/assets/${assetId}`)).response.status,
|
||||
404,
|
||||
);
|
||||
});
|
||||
|
||||
test("upload validation returns structured errors and always cleans temporary files", async (t) => {
|
||||
const harness = await createHarness(t, { maxFileBytes: 1024 });
|
||||
|
||||
const malformed = new FormData();
|
||||
appendFile(malformed, "file", Buffer.from("not an image"), "broken.png", "image/png");
|
||||
const malformedResult = await jsonResponse(harness.baseUrl, "/api/assets", {
|
||||
method: "POST",
|
||||
body: malformed,
|
||||
});
|
||||
assert.equal(malformedResult.response.status, 422);
|
||||
assert.equal(malformedResult.body.error.code, "MALFORMED_IMAGE");
|
||||
await eventuallyEmpty(harness.tempDirectory);
|
||||
|
||||
const unsupportedBuffer = await sharp({
|
||||
create: {
|
||||
width: 20,
|
||||
height: 20,
|
||||
channels: 3,
|
||||
background: "red",
|
||||
},
|
||||
})
|
||||
.gif()
|
||||
.toBuffer();
|
||||
const unsupported = new FormData();
|
||||
appendFile(unsupported, "file", unsupportedBuffer, "still.gif", "image/png");
|
||||
const unsupportedResult = await jsonResponse(harness.baseUrl, "/api/assets", {
|
||||
method: "POST",
|
||||
body: unsupported,
|
||||
});
|
||||
assert.equal(unsupportedResult.response.status, 415);
|
||||
assert.equal(unsupportedResult.body.error.code, "UNSUPPORTED_IMAGE_FORMAT");
|
||||
await eventuallyEmpty(harness.tempDirectory);
|
||||
|
||||
const oversized = new FormData();
|
||||
appendFile(
|
||||
oversized,
|
||||
"file",
|
||||
crypto.randomBytes(1025),
|
||||
"too-large.png",
|
||||
"image/png",
|
||||
);
|
||||
const oversizedResult = await jsonResponse(harness.baseUrl, "/api/assets", {
|
||||
method: "POST",
|
||||
body: oversized,
|
||||
});
|
||||
assert.equal(oversizedResult.response.status, 413);
|
||||
assert.equal(oversizedResult.body.error.code, "UPLOAD_FILE_TOO_LARGE");
|
||||
assert.deepEqual(oversizedResult.body.error.details, { maxFileBytes: 1024 });
|
||||
await eventuallyEmpty(harness.tempDirectory);
|
||||
|
||||
const tooMany = new FormData();
|
||||
for (let index = 0; index < 6; index += 1) {
|
||||
appendFile(tooMany, "files", Buffer.from("x"), `${index}.png`, "image/png");
|
||||
}
|
||||
const tooManyResult = await jsonResponse(harness.baseUrl, "/api/assets", {
|
||||
method: "POST",
|
||||
body: tooMany,
|
||||
});
|
||||
assert.equal(tooManyResult.response.status, 413);
|
||||
assert.equal(tooManyResult.body.error.code, "UPLOAD_FILE_COUNT_EXCEEDED");
|
||||
assert.deepEqual(tooManyResult.body.error.details, { maxFiles: 5 });
|
||||
await eventuallyEmpty(harness.tempDirectory);
|
||||
|
||||
const absent = await jsonResponse(harness.baseUrl, "/api/assets", {
|
||||
method: "POST",
|
||||
body: new FormData(),
|
||||
});
|
||||
assert.equal(absent.response.status, 400);
|
||||
assert.equal(absent.body.error.code, "UPLOAD_FILE_REQUIRED");
|
||||
assert.equal(harness.storage.objects.size, 0);
|
||||
assert.equal(
|
||||
harness.database.prepare("SELECT COUNT(*) AS count FROM assets").get().count,
|
||||
0,
|
||||
);
|
||||
});
|
||||
|
||||
test("an aborted multipart request removes its in-flight temporary file", async (t) => {
|
||||
const harness = await createHarness(t);
|
||||
const boundary = `announcements-${crypto.randomUUID()}`;
|
||||
const preamble = Buffer.from(
|
||||
`--${boundary}\r\n` +
|
||||
'Content-Disposition: form-data; name="file"; filename="aborted.png"\r\n' +
|
||||
"Content-Type: image/png\r\n\r\n",
|
||||
);
|
||||
const declaredLength = preamble.length + 512 * 1024;
|
||||
const request = http.request(`${harness.baseUrl}/api/assets`, {
|
||||
method: "POST",
|
||||
headers: {
|
||||
"content-type": `multipart/form-data; boundary=${boundary}`,
|
||||
"content-length": String(declaredLength),
|
||||
},
|
||||
});
|
||||
request.on("error", () => {});
|
||||
request.write(preamble);
|
||||
request.write(Buffer.alloc(64 * 1024, 1));
|
||||
|
||||
await eventuallyContainsFile(harness.tempDirectory);
|
||||
const closed = new Promise((resolve) => request.once("close", resolve));
|
||||
request.destroy();
|
||||
await closed;
|
||||
await eventuallyEmpty(harness.tempDirectory);
|
||||
assert.equal(harness.storage.objects.size, 0);
|
||||
assert.equal(
|
||||
harness.database.prepare("SELECT COUNT(*) AS count FROM assets").get().count,
|
||||
0,
|
||||
);
|
||||
});
|
||||
@@ -0,0 +1,35 @@
|
||||
const test = require("node:test");
|
||||
const assert = require("node:assert/strict");
|
||||
const {
|
||||
mapMulterError,
|
||||
safeOriginalFilename,
|
||||
} = require("../src/assets/multipart");
|
||||
|
||||
test("original filename metadata keeps a safe leaf without path traversal", () => {
|
||||
assert.equal(safeOriginalFilename("../../Sunday/welcome.png"), "welcome.png");
|
||||
assert.equal(safeOriginalFilename("..\\..\\Sunday\\welcome.png"), "welcome.png");
|
||||
assert.equal(safeOriginalFilename(" welcome sign.png "), " welcome sign.png ");
|
||||
});
|
||||
|
||||
test("original filename metadata removes controls and remains bounded", () => {
|
||||
assert.equal(safeOriginalFilename("wel\u0000come\n.png"), "welcome.png");
|
||||
assert.equal([...safeOriginalFilename(`${"a".repeat(300)}.png`)].length, 255);
|
||||
assert.throws(
|
||||
() => safeOriginalFilename(" \u0000\n "),
|
||||
(error) => error.code === "INVALID_ORIGINAL_FILENAME",
|
||||
);
|
||||
});
|
||||
|
||||
test("temporary-storage failures are server errors, not invalid client uploads", () => {
|
||||
const storageFailure = mapMulterError(
|
||||
Object.assign(new Error("disk full at a private path"), { code: "ENOSPC" }),
|
||||
{},
|
||||
);
|
||||
assert.equal(storageFailure.code, "UPLOAD_TEMP_STORAGE_UNAVAILABLE");
|
||||
assert.equal(storageFailure.statusCode, 503);
|
||||
assert.doesNotMatch(storageFailure.message, /private path/);
|
||||
|
||||
const malformed = mapMulterError(new Error("Unexpected end of form"), {});
|
||||
assert.equal(malformed.code, "INVALID_MULTIPART_UPLOAD");
|
||||
assert.equal(malformed.statusCode, 400);
|
||||
});
|
||||
@@ -0,0 +1,608 @@
|
||||
const test = require("node:test");
|
||||
const assert = require("node:assert/strict");
|
||||
const crypto = require("node:crypto");
|
||||
const fs = require("node:fs");
|
||||
const os = require("node:os");
|
||||
const path = require("node:path");
|
||||
const { Readable } = require("node:stream");
|
||||
const { openDatabase } = require("../src/database");
|
||||
const { runMigrations } = require("../src/database/migrate");
|
||||
const { createAssetService } = require("../src/assets/service");
|
||||
const {
|
||||
StorageObjectNotFoundError,
|
||||
StorageUnavailableError,
|
||||
} = require("../src/storage");
|
||||
|
||||
async function bufferBody(body) {
|
||||
if (Buffer.isBuffer(body)) return body;
|
||||
if (body instanceof Uint8Array) return Buffer.from(body);
|
||||
const chunks = [];
|
||||
for await (const chunk of body) chunks.push(Buffer.from(chunk));
|
||||
return Buffer.concat(chunks);
|
||||
}
|
||||
|
||||
function createMemoryStorage() {
|
||||
const objects = new Map();
|
||||
const calls = [];
|
||||
let failNextPut = null;
|
||||
let successfulPutsBeforeFailure = 0;
|
||||
let failNextDelete = null;
|
||||
let putDelayMs = 0;
|
||||
|
||||
return {
|
||||
objects,
|
||||
calls,
|
||||
setPutFailure(error, putsBeforeFailure = 0) {
|
||||
failNextPut = error;
|
||||
successfulPutsBeforeFailure = putsBeforeFailure;
|
||||
},
|
||||
setDeleteFailure(error) {
|
||||
failNextDelete = error;
|
||||
},
|
||||
setPutDelay(milliseconds) {
|
||||
putDelayMs = milliseconds;
|
||||
},
|
||||
async putObject(input) {
|
||||
calls.push({ operation: "put", ...input, body: undefined });
|
||||
if (putDelayMs) {
|
||||
await new Promise((resolve) => setTimeout(resolve, putDelayMs));
|
||||
}
|
||||
if (failNextPut) {
|
||||
if (successfulPutsBeforeFailure === 0) {
|
||||
const error = failNextPut;
|
||||
failNextPut = null;
|
||||
throw error;
|
||||
}
|
||||
successfulPutsBeforeFailure -= 1;
|
||||
}
|
||||
objects.set(input.key, {
|
||||
body: await bufferBody(input.body),
|
||||
contentType: input.contentType,
|
||||
metadata: input.metadata,
|
||||
});
|
||||
return { key: input.key };
|
||||
},
|
||||
async getObject(key) {
|
||||
calls.push({ operation: "get", key });
|
||||
const object = objects.get(key);
|
||||
if (!object) throw new StorageObjectNotFoundError(key);
|
||||
return {
|
||||
body: Readable.from(object.body),
|
||||
contentLength: object.body.length,
|
||||
contentType: object.contentType,
|
||||
};
|
||||
},
|
||||
async deleteObject(key) {
|
||||
calls.push({ operation: "delete", key });
|
||||
if (failNextDelete) {
|
||||
const error = failNextDelete;
|
||||
failNextDelete = null;
|
||||
throw error;
|
||||
}
|
||||
objects.delete(key);
|
||||
return { key, deleted: true };
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
function createFakeImageProcessor({ reject } = {}) {
|
||||
const calls = [];
|
||||
return {
|
||||
calls,
|
||||
async process(filename) {
|
||||
calls.push(filename);
|
||||
if (reject) throw reject;
|
||||
const original = await fs.promises.readFile(filename);
|
||||
const sourceSha256 = crypto.createHash("sha256").update(original).digest("hex");
|
||||
const displayBuffer = Buffer.from(`display:${sourceSha256}`);
|
||||
const thumbnailBuffer = Buffer.from(`thumbnail:${sourceSha256}`);
|
||||
return {
|
||||
original: {
|
||||
sha256: sourceSha256,
|
||||
contentType: "image/png",
|
||||
byteSize: original.length,
|
||||
width: 1200,
|
||||
height: 800,
|
||||
},
|
||||
display: {
|
||||
buffer: displayBuffer,
|
||||
sha256: crypto.createHash("sha256").update(displayBuffer).digest("hex"),
|
||||
contentType: "image/webp",
|
||||
byteSize: displayBuffer.length,
|
||||
width: 1200,
|
||||
height: 800,
|
||||
},
|
||||
thumbnail: {
|
||||
buffer: thumbnailBuffer,
|
||||
sha256: crypto.createHash("sha256").update(thumbnailBuffer).digest("hex"),
|
||||
contentType: "image/webp",
|
||||
byteSize: thumbnailBuffer.length,
|
||||
width: 405,
|
||||
height: 270,
|
||||
},
|
||||
};
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
function createHarness(options = {}) {
|
||||
const database = openDatabase({ path: ":memory:" });
|
||||
runMigrations(database);
|
||||
const directory = fs.mkdtempSync(path.join(os.tmpdir(), "announcements-assets-"));
|
||||
const storage = options.storage || createMemoryStorage();
|
||||
const imageProcessor =
|
||||
options.imageProcessor || createFakeImageProcessor(options.processorOptions);
|
||||
let idCounter = 0;
|
||||
let clock = 2_000_000_000_000;
|
||||
const service = createAssetService({
|
||||
database,
|
||||
storage,
|
||||
imageProcessor,
|
||||
now: () => ++clock,
|
||||
createId:
|
||||
options.createId ||
|
||||
(() => `00000000-0000-4000-8000-${String(++idCounter).padStart(12, "0")}`),
|
||||
concurrentUploadWaitMs: options.concurrentUploadWaitMs ?? 2_000,
|
||||
});
|
||||
|
||||
function uploadFile(originalFilename, contents) {
|
||||
const buffer = Buffer.isBuffer(contents) ? contents : Buffer.from(contents);
|
||||
const filename = path.join(directory, crypto.randomUUID());
|
||||
fs.writeFileSync(filename, buffer);
|
||||
return { path: filename, byteSize: buffer.length, originalFilename };
|
||||
}
|
||||
|
||||
return {
|
||||
database,
|
||||
directory,
|
||||
imageProcessor,
|
||||
service,
|
||||
storage,
|
||||
uploadFile,
|
||||
close() {
|
||||
database.close();
|
||||
fs.rmSync(directory, { recursive: true, force: true });
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
function rawAsset(database, assetId) {
|
||||
return database.prepare("SELECT * FROM assets WHERE id = ?").get(assetId);
|
||||
}
|
||||
|
||||
test("upload stores exact bytes under content-addressed keys without leaking keys", async (t) => {
|
||||
const harness = createHarness();
|
||||
t.after(() => harness.close());
|
||||
const original = Buffer.from("exact uploaded bytes");
|
||||
|
||||
const [result] = await harness.service.upload([
|
||||
harness.uploadFile(" Sunday Welcome.PNG ", original),
|
||||
]);
|
||||
|
||||
assert.equal(result.created, true);
|
||||
assert.equal(result.duplicate, false);
|
||||
assert.match(result.asset.id, /^[0-9a-f-]{36}$/);
|
||||
assert.equal(result.asset.displayName, "Sunday Welcome");
|
||||
assert.equal(result.asset.originalFilename, " Sunday Welcome.PNG ");
|
||||
assert.equal(result.asset.expirationAt, null);
|
||||
assert.equal(result.asset.status, "ready");
|
||||
assert.equal(result.asset.original.byteSize, original.length);
|
||||
assert.equal(result.asset.display.contentType, "image/webp");
|
||||
assert.equal(result.asset.thumbnail.width, 405);
|
||||
assert.match(result.asset.display.url, /^\/media\/assets\//);
|
||||
assert.doesNotMatch(
|
||||
JSON.stringify(result.asset),
|
||||
/object_key|originalObjectKey|displayObjectKey|thumbnailObjectKey|seaweed/i,
|
||||
);
|
||||
|
||||
const row = rawAsset(harness.database, result.asset.id);
|
||||
const expectedSourceHash = crypto.createHash("sha256").update(original).digest("hex");
|
||||
assert.equal(row.source_sha256, expectedSourceHash);
|
||||
assert.equal(row.original_object_key, `assets/${row.id}/original/${expectedSourceHash}`);
|
||||
assert.equal(
|
||||
row.display_object_key,
|
||||
`assets/${row.id}/display/${row.display_sha256}.webp`,
|
||||
);
|
||||
assert.equal(
|
||||
row.thumbnail_object_key,
|
||||
`assets/${row.id}/thumbnail/${row.thumbnail_sha256}.webp`,
|
||||
);
|
||||
assert.deepEqual(harness.storage.objects.get(row.original_object_key).body, original);
|
||||
assert.equal(harness.storage.objects.size, 3);
|
||||
|
||||
assert.deepEqual(harness.service.list(), [result.asset]);
|
||||
assert.deepEqual(harness.service.get(result.asset.id), result.asset);
|
||||
});
|
||||
|
||||
test("exact duplicate detection takes precedence over filename and display name", async (t) => {
|
||||
const harness = createHarness();
|
||||
t.after(() => harness.close());
|
||||
const contents = Buffer.from("same image bytes");
|
||||
|
||||
const [created] = await harness.service.upload([
|
||||
harness.uploadFile("first.png", contents),
|
||||
]);
|
||||
const [sameFilename] = await harness.service.upload([
|
||||
harness.uploadFile("first.png", contents),
|
||||
]);
|
||||
const [differentFilename] = await harness.service.upload(
|
||||
[harness.uploadFile("completely different name.webp", contents)],
|
||||
{ displayName: "A conflicting requested name is ignored for an exact duplicate" },
|
||||
);
|
||||
|
||||
for (const duplicate of [sameFilename, differentFilename]) {
|
||||
assert.equal(duplicate.created, false);
|
||||
assert.equal(duplicate.duplicate, true);
|
||||
assert.equal(duplicate.asset.id, created.asset.id);
|
||||
}
|
||||
assert.equal(harness.service.list().length, 1);
|
||||
assert.equal(harness.storage.calls.filter((call) => call.operation === "put").length, 3);
|
||||
assert.equal(harness.imageProcessor.calls.length, 1);
|
||||
});
|
||||
|
||||
test("different content with the same proposed name is a conflict, not a duplicate", async (t) => {
|
||||
const harness = createHarness();
|
||||
t.after(() => harness.close());
|
||||
await harness.service.upload([harness.uploadFile("bulletin.png", "image one")]);
|
||||
|
||||
await assert.rejects(
|
||||
harness.service.upload([harness.uploadFile("bulletin.jpg", "image two")]),
|
||||
(error) => {
|
||||
assert.equal(error.statusCode, 409);
|
||||
assert.equal(error.code, "ASSET_NAME_CONFLICT");
|
||||
assert.equal(error.details.asset.displayName, "bulletin");
|
||||
return true;
|
||||
},
|
||||
);
|
||||
assert.equal(harness.service.list().length, 1);
|
||||
assert.equal(harness.imageProcessor.calls.length, 1);
|
||||
|
||||
const [created] = await harness.service.upload(
|
||||
[harness.uploadFile("bulletin.jpg", "image two")],
|
||||
{ displayName: "Second bulletin" },
|
||||
);
|
||||
assert.equal(created.created, true);
|
||||
assert.notEqual(created.asset.sourceSha256, harness.service.list()[1].sourceSha256);
|
||||
assert.equal(harness.service.list().length, 2);
|
||||
});
|
||||
|
||||
test("multiple-file upload preserves order and deduplicates repeated bytes in the batch", async (t) => {
|
||||
const harness = createHarness();
|
||||
t.after(() => harness.close());
|
||||
|
||||
const results = await harness.service.upload([
|
||||
harness.uploadFile("alpha.png", "alpha bytes"),
|
||||
harness.uploadFile("beta.png", "beta bytes"),
|
||||
harness.uploadFile("alpha-copy.png", "alpha bytes"),
|
||||
]);
|
||||
|
||||
assert.deepEqual(
|
||||
results.map(({ created, duplicate }) => ({ created, duplicate })),
|
||||
[
|
||||
{ created: true, duplicate: false },
|
||||
{ created: true, duplicate: false },
|
||||
{ created: false, duplicate: true },
|
||||
],
|
||||
);
|
||||
assert.equal(results[0].asset.id, results[2].asset.id);
|
||||
assert.equal(harness.service.list().length, 2);
|
||||
assert.equal(harness.imageProcessor.calls.length, 2);
|
||||
assert.equal(harness.storage.objects.size, 6);
|
||||
});
|
||||
|
||||
test("all new images are decoded before the batch mutates SQLite or storage", async (t) => {
|
||||
const expected = Object.assign(new Error("bad second image"), {
|
||||
code: "MALFORMED_IMAGE",
|
||||
statusCode: 422,
|
||||
});
|
||||
let processCount = 0;
|
||||
const regularProcessor = createFakeImageProcessor();
|
||||
const harness = createHarness({
|
||||
imageProcessor: {
|
||||
async process(filename) {
|
||||
processCount += 1;
|
||||
if (processCount === 2) throw expected;
|
||||
return regularProcessor.process(filename);
|
||||
},
|
||||
},
|
||||
});
|
||||
t.after(() => harness.close());
|
||||
|
||||
await assert.rejects(
|
||||
harness.service.upload([
|
||||
harness.uploadFile("valid.png", "valid"),
|
||||
harness.uploadFile("broken.png", "broken"),
|
||||
]),
|
||||
expected,
|
||||
);
|
||||
assert.equal(harness.service.list().length, 0);
|
||||
assert.equal(harness.storage.objects.size, 0);
|
||||
assert.equal(harness.storage.calls.length, 0);
|
||||
});
|
||||
|
||||
test("concurrent identical uploads converge on one ready asset", async (t) => {
|
||||
const storage = createMemoryStorage();
|
||||
storage.setPutDelay(30);
|
||||
const harness = createHarness({ storage });
|
||||
t.after(() => harness.close());
|
||||
const bytes = Buffer.from("racing identical upload");
|
||||
|
||||
const [[left], [right]] = await Promise.all([
|
||||
harness.service.upload([harness.uploadFile("race.png", bytes)]),
|
||||
harness.service.upload([harness.uploadFile("other-name.png", bytes)]),
|
||||
]);
|
||||
|
||||
assert.equal(harness.service.list().length, 1);
|
||||
assert.equal(left.asset.id, right.asset.id);
|
||||
assert.deepEqual([left.created, right.created].sort(), [false, true]);
|
||||
assert.deepEqual([left.duplicate, right.duplicate].sort(), [false, true]);
|
||||
assert.equal(storage.calls.filter((call) => call.operation === "put").length, 3);
|
||||
});
|
||||
|
||||
test("a partial storage failure leaves a diagnosable failed row and no ready URLs", async (t) => {
|
||||
const storage = createMemoryStorage();
|
||||
storage.setPutFailure(new StorageUnavailableError(new Error("offline")), 1);
|
||||
const harness = createHarness({ storage });
|
||||
t.after(() => harness.close());
|
||||
|
||||
await assert.rejects(
|
||||
harness.service.upload([harness.uploadFile("failure.png", "failure bytes")]),
|
||||
(error) => {
|
||||
assert.equal(error.statusCode, 503);
|
||||
assert.equal(error.details.retryable, true);
|
||||
assert.match(error.details.assetId, /^[0-9a-f-]{36}$/);
|
||||
return true;
|
||||
},
|
||||
);
|
||||
|
||||
const [failed] = harness.service.list();
|
||||
assert.equal(failed.status, "failed");
|
||||
assert.match(failed.processingError, /^STORAGE_UNAVAILABLE:/);
|
||||
assert.equal(failed.display.url, null);
|
||||
assert.equal(failed.thumbnail.url, null);
|
||||
assert.equal(failed.original.url, null);
|
||||
assert.equal(storage.objects.size, 0);
|
||||
const row = rawAsset(harness.database, failed.id);
|
||||
assert.match(
|
||||
row.original_object_key,
|
||||
new RegExp(`^assets/${failed.id}/original/[0-9a-f]{64}$`),
|
||||
);
|
||||
assert.equal(
|
||||
row.display_object_key,
|
||||
`assets/${failed.id}/display/${row.display_sha256}.webp`,
|
||||
);
|
||||
assert.equal(
|
||||
row.thumbnail_object_key,
|
||||
`assets/${failed.id}/thumbnail/${row.thumbnail_sha256}.webp`,
|
||||
);
|
||||
assert.equal(storage.calls.filter((call) => call.operation === "put").length, 2);
|
||||
assert.deepEqual(
|
||||
new Set(
|
||||
storage.calls
|
||||
.filter((call) => call.operation === "delete")
|
||||
.map((call) => call.key),
|
||||
),
|
||||
new Set([
|
||||
row.original_object_key,
|
||||
row.display_object_key,
|
||||
row.thumbnail_object_key,
|
||||
]),
|
||||
);
|
||||
|
||||
await assert.rejects(
|
||||
harness.service.upload([harness.uploadFile("retry.png", "failure bytes")]),
|
||||
(error) => error.code === "ASSET_PROCESSING_FAILED" && error.statusCode === 409,
|
||||
);
|
||||
|
||||
assert.deepEqual(await harness.service.remove(failed.id), {
|
||||
id: failed.id,
|
||||
deleted: true,
|
||||
});
|
||||
assert.equal(harness.service.list().length, 0);
|
||||
});
|
||||
|
||||
test("rename and expiration edits change only approved SQLite metadata", async (t) => {
|
||||
const harness = createHarness();
|
||||
t.after(() => harness.close());
|
||||
const [upload] = await harness.service.upload([
|
||||
harness.uploadFile("editable.png", "editable bytes"),
|
||||
]);
|
||||
const before = rawAsset(harness.database, upload.asset.id);
|
||||
|
||||
const expiring = harness.service.update(upload.asset.id, {
|
||||
display_name: " New display name ",
|
||||
expiration_at: "2035-05-06T07:08:09.000Z",
|
||||
});
|
||||
assert.equal(expiring.displayName, "New display name");
|
||||
assert.equal(expiring.expirationAt, Date.parse("2035-05-06T07:08:09.000Z"));
|
||||
|
||||
const permanent = harness.service.update(upload.asset.id, { expirationAt: null });
|
||||
assert.equal(permanent.expirationAt, null);
|
||||
const after = rawAsset(harness.database, upload.asset.id);
|
||||
assert.equal(after.original_object_key, before.original_object_key);
|
||||
assert.equal(after.display_object_key, before.display_object_key);
|
||||
assert.equal(after.thumbnail_object_key, before.thumbnail_object_key);
|
||||
assert.equal(harness.storage.calls.filter((call) => call.operation === "put").length, 3);
|
||||
|
||||
await assert.rejects(
|
||||
async () => harness.service.update(upload.asset.id, { status: "failed" }),
|
||||
(error) => error.code === "INVALID_ASSET_UPDATE",
|
||||
);
|
||||
assert.throws(
|
||||
() => harness.service.update(upload.asset.id, { expirationAt: -1 }),
|
||||
(error) => error.code === "INVALID_ASSET_EXPIRATION",
|
||||
);
|
||||
});
|
||||
|
||||
test("references describe playlist, normal, and static use and block deletion", async (t) => {
|
||||
const harness = createHarness();
|
||||
t.after(() => harness.close());
|
||||
const [upload] = await harness.service.upload([
|
||||
harness.uploadFile("referenced.png", "referenced bytes"),
|
||||
]);
|
||||
const id = upload.asset.id;
|
||||
const now = 2_100_000_000_000;
|
||||
harness.database
|
||||
.prepare(
|
||||
"INSERT INTO playlists (id, name, created_at, updated_at) VALUES (?, ?, ?, ?)",
|
||||
)
|
||||
.run("playlist-1", "Sunday", now, now);
|
||||
harness.database
|
||||
.prepare(
|
||||
`INSERT INTO playlist_entries
|
||||
(id, playlist_id, asset_id, enabled, position, created_at, updated_at)
|
||||
VALUES (?, ?, ?, 1, 0, ?, ?)`,
|
||||
)
|
||||
.run("entry-1", "playlist-1", id, now, now);
|
||||
harness.database
|
||||
.prepare(
|
||||
`UPDATE playback_state
|
||||
SET active_playlist_id = ?, current_entry_id = ?, static_asset_id = ?, updated_at = ?
|
||||
WHERE output_id = 'global'`,
|
||||
)
|
||||
.run("playlist-1", "entry-1", id, now);
|
||||
|
||||
const references = harness.service.references(id);
|
||||
assert.equal(references.canDelete, false);
|
||||
assert.deepEqual(references.blockerCodes, [
|
||||
"PLAYLIST_REFERENCES",
|
||||
"CURRENT_PLAYBACK_TARGET",
|
||||
"CURRENT_STATIC_TARGET",
|
||||
]);
|
||||
assert.deepEqual(references.playlists[0], {
|
||||
playlistId: "playlist-1",
|
||||
playlistName: "Sunday",
|
||||
playlistEntryId: "entry-1",
|
||||
enabled: true,
|
||||
position: 0,
|
||||
isActivePlaylist: true,
|
||||
isCurrentNormalTarget: true,
|
||||
});
|
||||
assert.equal(references.playback[0].isCurrentNormalTarget, true);
|
||||
assert.equal(references.playback[0].isCurrentStaticTarget, true);
|
||||
|
||||
const callsBeforeDelete = harness.storage.calls.length;
|
||||
await assert.rejects(
|
||||
harness.service.remove(id),
|
||||
(error) => {
|
||||
assert.equal(error.code, "ASSET_REFERENCED");
|
||||
assert.equal(error.statusCode, 409);
|
||||
assert.deepEqual(error.details.blockerCodes, [
|
||||
"PLAYLIST_REFERENCES",
|
||||
"CURRENT_PLAYBACK_TARGET",
|
||||
"CURRENT_STATIC_TARGET",
|
||||
]);
|
||||
return true;
|
||||
},
|
||||
);
|
||||
assert.equal(harness.storage.calls.length, callsBeforeDelete);
|
||||
assert.equal(rawAsset(harness.database, id).status, "ready");
|
||||
|
||||
harness.database
|
||||
.prepare(
|
||||
`UPDATE playback_state
|
||||
SET active_playlist_id = NULL, current_entry_id = NULL, static_asset_id = NULL
|
||||
WHERE output_id = 'global'`,
|
||||
)
|
||||
.run();
|
||||
harness.database.prepare("DELETE FROM playlist_entries WHERE id = ?").run("entry-1");
|
||||
assert.equal(harness.service.references(id).canDelete, true);
|
||||
await harness.service.remove(id);
|
||||
assert.equal(harness.storage.objects.size, 0);
|
||||
assert.equal(rawAsset(harness.database, id), undefined);
|
||||
});
|
||||
|
||||
test("failed storage deletion retains a retryable deleting row", async (t) => {
|
||||
const storage = createMemoryStorage();
|
||||
const harness = createHarness({ storage });
|
||||
t.after(() => harness.close());
|
||||
const [upload] = await harness.service.upload([
|
||||
harness.uploadFile("delete-me.png", "delete me"),
|
||||
]);
|
||||
storage.setDeleteFailure(new StorageUnavailableError(new Error("offline")));
|
||||
|
||||
await assert.rejects(harness.service.remove(upload.asset.id), (error) => {
|
||||
assert.equal(error.statusCode, 503);
|
||||
assert.equal(error.details.assetId, upload.asset.id);
|
||||
assert.equal(error.details.retryable, true);
|
||||
return true;
|
||||
});
|
||||
assert.equal(rawAsset(harness.database, upload.asset.id).status, "deleting");
|
||||
assert.throws(
|
||||
() => harness.service.update(upload.asset.id, { displayName: "Cannot edit" }),
|
||||
(error) => error.code === "ASSET_DELETING",
|
||||
);
|
||||
|
||||
await harness.service.remove(upload.asset.id);
|
||||
assert.equal(rawAsset(harness.database, upload.asset.id), undefined);
|
||||
assert.equal(storage.objects.size, 0);
|
||||
});
|
||||
|
||||
test("processing assets cannot be deleted and storage is not touched", async (t) => {
|
||||
const harness = createHarness();
|
||||
t.after(() => harness.close());
|
||||
const timestamp = Date.now();
|
||||
harness.database
|
||||
.prepare(
|
||||
`INSERT INTO assets
|
||||
(id, source_sha256, display_name, original_filename, status, created_at, updated_at)
|
||||
VALUES (?, ?, ?, ?, 'processing', ?, ?)`,
|
||||
)
|
||||
.run(
|
||||
"00000000-0000-4000-8000-999999999999",
|
||||
"f".repeat(64),
|
||||
"Still processing",
|
||||
"processing.png",
|
||||
timestamp,
|
||||
timestamp,
|
||||
);
|
||||
|
||||
assert.equal(
|
||||
harness.service.references("00000000-0000-4000-8000-999999999999")
|
||||
.canDelete,
|
||||
false,
|
||||
);
|
||||
await assert.rejects(
|
||||
harness.service.remove("00000000-0000-4000-8000-999999999999"),
|
||||
(error) => error.code === "ASSET_PROCESSING" && error.statusCode === 409,
|
||||
);
|
||||
assert.equal(harness.storage.calls.length, 0);
|
||||
assert.equal(
|
||||
rawAsset(harness.database, "00000000-0000-4000-8000-999999999999").status,
|
||||
"processing",
|
||||
);
|
||||
});
|
||||
|
||||
test("media descriptors only allow the recorded version and map missing bytes to 503", async (t) => {
|
||||
const harness = createHarness();
|
||||
t.after(() => harness.close());
|
||||
const [upload] = await harness.service.upload([
|
||||
harness.uploadFile("media.png", "media bytes"),
|
||||
]);
|
||||
const { asset } = upload;
|
||||
|
||||
const display = harness.service.mediaDescriptor(
|
||||
asset.id,
|
||||
"display",
|
||||
asset.display.sha256,
|
||||
);
|
||||
assert.equal(display.contentType, "image/webp");
|
||||
assert.equal(display.contentLength, asset.display.byteSize);
|
||||
const stored = await harness.service.readStoredObject(display);
|
||||
assert.deepEqual(await bufferBody(stored.body), harness.storage.objects.get(display.key).body);
|
||||
|
||||
for (const requestedHash of ["a".repeat(64), "../original/key", "not-a-hash"]) {
|
||||
assert.throws(
|
||||
() => harness.service.mediaDescriptor(asset.id, "display", requestedHash),
|
||||
(error) => error.code === "MEDIA_NOT_FOUND" && error.statusCode === 404,
|
||||
);
|
||||
}
|
||||
|
||||
harness.storage.objects.delete(display.key);
|
||||
await assert.rejects(harness.service.readStoredObject(display), (error) => {
|
||||
assert.equal(error.code, "STORAGE_INCONSISTENCY");
|
||||
assert.equal(error.statusCode, 503);
|
||||
assert.deepEqual(error.details, { assetId: asset.id, retryable: true });
|
||||
return true;
|
||||
});
|
||||
});
|
||||
@@ -32,6 +32,14 @@ test("loadConfig validates and normalizes the supported environment", () => {
|
||||
assert.equal(config.timeZone, "America/Chicago");
|
||||
assert.equal(config.storage.endpoint, "http://seaweedfs:8333");
|
||||
assert.equal(config.storage.forcePathStyle, true);
|
||||
assert.equal(config.storage.healthProbeIntervalMs, 30_000);
|
||||
assert.equal(config.storage.connectionTimeoutMs, 2_000);
|
||||
assert.equal(config.storage.requestTimeoutMs, 5_000);
|
||||
assert.equal(config.storage.maxAttempts, 2);
|
||||
assert.equal(config.processing.tempDirectory, "/tmp/announcements-uploads");
|
||||
assert.equal(config.processing.maxFileBytes, 25 * 1024 * 1024);
|
||||
assert.equal(config.processing.maxFiles, 20);
|
||||
assert.equal(config.processing.maxInputPixels, 40_000_000);
|
||||
});
|
||||
|
||||
test("production requires private S3 credentials", () => {
|
||||
@@ -65,10 +73,42 @@ test("loadConfig rejects invalid typed values", () => {
|
||||
() => loadConfig(validEnvironment({ S3_FORCE_PATH_STYLE: "sometimes" })),
|
||||
/must be a boolean/,
|
||||
);
|
||||
assert.throws(
|
||||
() => loadConfig(validEnvironment({ S3_HEALTH_PROBE_INTERVAL_MS: "999" })),
|
||||
/S3_HEALTH_PROBE_INTERVAL_MS must be at least 1000/,
|
||||
);
|
||||
assert.throws(
|
||||
() => loadConfig(validEnvironment({ S3_REQUEST_TIMEOUT_MS: "999" })),
|
||||
/S3_REQUEST_TIMEOUT_MS must be at least 1000/,
|
||||
);
|
||||
assert.throws(
|
||||
() => loadConfig(validEnvironment({ S3_CONNECT_TIMEOUT_MS: "249" })),
|
||||
/S3_CONNECT_TIMEOUT_MS must be at least 250/,
|
||||
);
|
||||
assert.throws(
|
||||
() => loadConfig(validEnvironment({ S3_MAX_ATTEMPTS: "6" })),
|
||||
/S3_MAX_ATTEMPTS must be at most 5/,
|
||||
);
|
||||
assert.throws(
|
||||
() => loadConfig(validEnvironment({ DATABASE_PATH: "relative.sqlite" })),
|
||||
/must be an absolute path/,
|
||||
);
|
||||
assert.throws(
|
||||
() => loadConfig(validEnvironment({ UPLOAD_TEMP_DIRECTORY: "relative" })),
|
||||
/UPLOAD_TEMP_DIRECTORY must be an absolute path/,
|
||||
);
|
||||
assert.throws(
|
||||
() => loadConfig(validEnvironment({ UPLOAD_MAX_FILE_BYTES: "0" })),
|
||||
/UPLOAD_MAX_FILE_BYTES must be at least 1/,
|
||||
);
|
||||
assert.throws(
|
||||
() => loadConfig(validEnvironment({ UPLOAD_MAX_FILES: "101" })),
|
||||
/UPLOAD_MAX_FILES must be at most 100/,
|
||||
);
|
||||
assert.throws(
|
||||
() => loadConfig(validEnvironment({ IMAGE_MAX_INPUT_PIXELS: "no-limit" })),
|
||||
/IMAGE_MAX_INPUT_PIXELS must be an integer/,
|
||||
);
|
||||
});
|
||||
|
||||
test("redactedConfig never returns credentials, passwords, paths, or webhook URLs", () => {
|
||||
@@ -83,5 +123,6 @@ test("redactedConfig never returns credentials, passwords, paths, or webhook URL
|
||||
assert.doesNotMatch(serialized, /mqtt-password/);
|
||||
assert.doesNotMatch(serialized, /webhook-secret/);
|
||||
assert.doesNotMatch(serialized, /announcements\.sqlite/);
|
||||
assert.doesNotMatch(serialized, /seaweedfs/);
|
||||
assert.doesNotMatch(serialized, /home-assistant/);
|
||||
});
|
||||
|
||||
@@ -0,0 +1,304 @@
|
||||
const test = require("node:test");
|
||||
const assert = require("node:assert/strict");
|
||||
const crypto = require("node:crypto");
|
||||
const fs = require("node:fs");
|
||||
const os = require("node:os");
|
||||
const path = require("node:path");
|
||||
const sharp = require("sharp");
|
||||
const {
|
||||
DEFAULT_IMAGE_DERIVATIVE_OPTIONS,
|
||||
ImageProcessingError,
|
||||
createImageProcessor,
|
||||
} = require("../src/image-processing");
|
||||
|
||||
const MAX_INPUT_PIXELS = 12_000_000;
|
||||
const CONTENT_TYPES = Object.freeze({
|
||||
jpeg: "image/jpeg",
|
||||
png: "image/png",
|
||||
webp: "image/webp",
|
||||
});
|
||||
|
||||
function createProcessor(overrides = {}) {
|
||||
return createImageProcessor({
|
||||
maxInputPixels: MAX_INPUT_PIXELS,
|
||||
...overrides,
|
||||
});
|
||||
}
|
||||
|
||||
function hash(buffer) {
|
||||
return crypto.createHash("sha256").update(buffer).digest("hex");
|
||||
}
|
||||
|
||||
async function solidImage(format, width, height, background = "#4a76b8") {
|
||||
return sharp({
|
||||
create: {
|
||||
width,
|
||||
height,
|
||||
channels: 4,
|
||||
background,
|
||||
},
|
||||
})
|
||||
.toFormat(format)
|
||||
.toBuffer();
|
||||
}
|
||||
|
||||
async function twoFrameImage(format) {
|
||||
const width = 20;
|
||||
const frameHeight = 10;
|
||||
const channels = 4;
|
||||
const firstFrame = Buffer.alloc(width * frameHeight * channels);
|
||||
const secondFrame = Buffer.alloc(width * frameHeight * channels);
|
||||
|
||||
for (let offset = 0; offset < firstFrame.length; offset += channels) {
|
||||
firstFrame.set([255, 0, 0, 255], offset);
|
||||
secondFrame.set([0, 0, 255, 255], offset);
|
||||
}
|
||||
|
||||
const image = sharp(Buffer.concat([firstFrame, secondFrame]), {
|
||||
raw: {
|
||||
width,
|
||||
height: frameHeight * 2,
|
||||
channels,
|
||||
pageHeight: frameHeight,
|
||||
},
|
||||
});
|
||||
|
||||
if (format === "tiff") {
|
||||
return image.tiff().toBuffer();
|
||||
}
|
||||
return image[format]({ delay: [100, 100], loop: 0 }).toBuffer();
|
||||
}
|
||||
|
||||
test("accepts decoded still JPEG, PNG, and WebP content", async (t) => {
|
||||
const processor = createProcessor();
|
||||
|
||||
for (const format of ["jpeg", "png", "webp"]) {
|
||||
await t.test(format, async () => {
|
||||
const source = await solidImage(format, 640, 360);
|
||||
const result = await processor.process(source);
|
||||
|
||||
assert.equal(result.original.format, format);
|
||||
assert.equal(result.original.contentType, CONTENT_TYPES[format]);
|
||||
assert.equal(result.original.byteSize, source.length);
|
||||
assert.equal(result.original.sha256, hash(source));
|
||||
assert.equal(result.original.width, 640);
|
||||
assert.equal(result.original.height, 360);
|
||||
assert.equal(result.original.pages, 1);
|
||||
|
||||
for (const derivative of [result.display, result.thumbnail]) {
|
||||
assert.ok(Buffer.isBuffer(derivative.buffer));
|
||||
assert.equal(derivative.format, "webp");
|
||||
assert.equal(derivative.contentType, "image/webp");
|
||||
assert.equal(derivative.byteSize, derivative.buffer.length);
|
||||
assert.equal(derivative.sha256, hash(derivative.buffer));
|
||||
assert.match(derivative.sha256, /^[0-9a-f]{64}$/);
|
||||
}
|
||||
});
|
||||
}
|
||||
});
|
||||
|
||||
test("uses bounding boxes without cropping or changing source aspect ratio", async () => {
|
||||
const source = await solidImage("png", 3000, 1000);
|
||||
const result = await createProcessor().process(source);
|
||||
|
||||
assert.deepEqual(
|
||||
{ width: result.display.width, height: result.display.height },
|
||||
{ width: 1920, height: 640 },
|
||||
);
|
||||
assert.deepEqual(
|
||||
{ width: result.thumbnail.width, height: result.thumbnail.height },
|
||||
{ width: 480, height: 160 },
|
||||
);
|
||||
assert.equal(result.display.width / result.display.height, 3);
|
||||
assert.equal(result.thumbnail.width / result.thumbnail.height, 3);
|
||||
});
|
||||
|
||||
test("does not enlarge a source that fits within both derivative bounds", async () => {
|
||||
const source = await solidImage("jpeg", 320, 180);
|
||||
const result = await createProcessor().process(source);
|
||||
|
||||
assert.deepEqual(
|
||||
{ width: result.display.width, height: result.display.height },
|
||||
{ width: 320, height: 180 },
|
||||
);
|
||||
assert.deepEqual(
|
||||
{ width: result.thumbnail.width, height: result.thumbnail.height },
|
||||
{ width: 320, height: 180 },
|
||||
);
|
||||
});
|
||||
|
||||
test("applies EXIF orientation and strips it from derivative metadata", async () => {
|
||||
const source = await sharp({
|
||||
create: {
|
||||
width: 120,
|
||||
height: 80,
|
||||
channels: 3,
|
||||
background: "#cb7235",
|
||||
},
|
||||
})
|
||||
.jpeg()
|
||||
.withMetadata({ orientation: 6 })
|
||||
.toBuffer();
|
||||
|
||||
const result = await createProcessor().process(source);
|
||||
const outputMetadata = await sharp(result.display.buffer).metadata();
|
||||
|
||||
assert.equal(result.original.orientation, 6);
|
||||
assert.equal(result.original.encodedWidth, 120);
|
||||
assert.equal(result.original.encodedHeight, 80);
|
||||
assert.equal(result.original.width, 80);
|
||||
assert.equal(result.original.height, 120);
|
||||
assert.equal(result.display.width, 80);
|
||||
assert.equal(result.display.height, 120);
|
||||
assert.equal(outputMetadata.orientation, undefined);
|
||||
assert.equal(outputMetadata.exif, undefined);
|
||||
});
|
||||
|
||||
test("flattens transparent pixels against black in both derivatives", async () => {
|
||||
const source = await solidImage("png", 40, 30, {
|
||||
r: 255,
|
||||
g: 0,
|
||||
b: 0,
|
||||
alpha: 0,
|
||||
});
|
||||
const result = await createProcessor().process(source);
|
||||
|
||||
assert.equal(result.original.hasAlpha, true);
|
||||
for (const derivative of [result.display, result.thumbnail]) {
|
||||
const metadata = await sharp(derivative.buffer).metadata();
|
||||
const pixels = await sharp(derivative.buffer).raw().toBuffer();
|
||||
|
||||
assert.equal(metadata.hasAlpha, false);
|
||||
assert.ok(
|
||||
pixels.every((channel) => channel <= 2),
|
||||
"fully transparent source pixels should become black",
|
||||
);
|
||||
}
|
||||
});
|
||||
|
||||
test("supports a temporary file path without changing source identity", async (t) => {
|
||||
const temporaryDirectory = fs.mkdtempSync(
|
||||
path.join(os.tmpdir(), "announcements-image-processing-"),
|
||||
);
|
||||
t.after(() => fs.rmSync(temporaryDirectory, { recursive: true, force: true }));
|
||||
|
||||
const source = await solidImage("webp", 800, 600);
|
||||
const filename = path.join(temporaryDirectory, "upload.tmp");
|
||||
fs.writeFileSync(filename, source);
|
||||
|
||||
const result = await createProcessor().process(filename);
|
||||
assert.equal(result.original.sha256, hash(source));
|
||||
assert.equal(result.original.byteSize, source.length);
|
||||
assert.equal(result.original.format, "webp");
|
||||
});
|
||||
|
||||
test("rejects malformed and unsupported decoded content with typed errors", async () => {
|
||||
const processor = createProcessor();
|
||||
const gif = await solidImage("gif", 64, 64);
|
||||
|
||||
await assert.rejects(
|
||||
processor.process(Buffer.from("not an image")),
|
||||
(error) =>
|
||||
error instanceof ImageProcessingError &&
|
||||
error.code === "MALFORMED_IMAGE" &&
|
||||
error.statusCode === 422,
|
||||
);
|
||||
await assert.rejects(
|
||||
processor.process(gif),
|
||||
(error) =>
|
||||
error instanceof ImageProcessingError &&
|
||||
error.code === "UNSUPPORTED_IMAGE_FORMAT" &&
|
||||
error.statusCode === 415 &&
|
||||
error.details.detectedFormat === "gif",
|
||||
);
|
||||
});
|
||||
|
||||
test("rejects animated WebP and GIF instead of selecting the first frame", async (t) => {
|
||||
for (const format of ["webp", "gif"]) {
|
||||
await t.test(format, async () => {
|
||||
const animated = await twoFrameImage(format);
|
||||
const metadata = await sharp(animated, { animated: true }).metadata();
|
||||
assert.equal(metadata.pages, 2, "test fixture must contain two frames");
|
||||
await assert.rejects(
|
||||
createProcessor().process(animated),
|
||||
(error) =>
|
||||
error instanceof ImageProcessingError &&
|
||||
error.code === "ANIMATED_IMAGE_NOT_SUPPORTED" &&
|
||||
error.details.pages === 2,
|
||||
);
|
||||
});
|
||||
}
|
||||
});
|
||||
|
||||
test("rejects unexpected multipage inputs with a distinct typed error", async () => {
|
||||
const multipageTiff = await twoFrameImage("tiff");
|
||||
const metadata = await sharp(multipageTiff, { animated: true }).metadata();
|
||||
assert.equal(metadata.pages, 2, "test fixture must contain two pages");
|
||||
|
||||
await assert.rejects(
|
||||
createProcessor().process(multipageTiff),
|
||||
(error) =>
|
||||
error instanceof ImageProcessingError &&
|
||||
error.code === "MULTIPAGE_IMAGE_NOT_SUPPORTED" &&
|
||||
error.details.pages === 2,
|
||||
);
|
||||
});
|
||||
|
||||
test("enforces the injected decoded-pixel limit", async () => {
|
||||
const source = await solidImage("png", 100, 100);
|
||||
const processor = createImageProcessor({ maxInputPixels: 9_999 });
|
||||
|
||||
await assert.rejects(
|
||||
processor.process(source),
|
||||
(error) =>
|
||||
error instanceof ImageProcessingError &&
|
||||
error.code === "IMAGE_PIXEL_LIMIT_EXCEEDED" &&
|
||||
error.statusCode === 413 &&
|
||||
error.details.maxInputPixels === 9_999,
|
||||
);
|
||||
});
|
||||
|
||||
test("uses documented defaults while allowing validated target overrides", async () => {
|
||||
assert.deepEqual(DEFAULT_IMAGE_DERIVATIVE_OPTIONS, {
|
||||
display: { maxWidth: 1920, maxHeight: 1080, quality: 82 },
|
||||
thumbnail: { maxWidth: 480, maxHeight: 270, quality: 72 },
|
||||
});
|
||||
|
||||
const source = await solidImage("png", 900, 600);
|
||||
const processor = createProcessor({
|
||||
display: { maxWidth: 300, maxHeight: 300, quality: 90 },
|
||||
thumbnail: { maxWidth: 100, maxHeight: 50, quality: 65 },
|
||||
});
|
||||
const result = await processor.process(source);
|
||||
|
||||
assert.deepEqual(
|
||||
{ width: result.display.width, height: result.display.height },
|
||||
{ width: 300, height: 200 },
|
||||
);
|
||||
assert.deepEqual(
|
||||
{ width: result.thumbnail.width, height: result.thumbnail.height },
|
||||
{ width: 75, height: 50 },
|
||||
);
|
||||
assert.throws(
|
||||
() => createImageProcessor({ maxInputPixels: 0 }),
|
||||
/maxInputPixels must be at least 1/,
|
||||
);
|
||||
assert.throws(
|
||||
() =>
|
||||
createImageProcessor({
|
||||
maxInputPixels: 10_000,
|
||||
display: { quality: 101 },
|
||||
}),
|
||||
/display\.quality must be at most 100/,
|
||||
);
|
||||
});
|
||||
|
||||
test("rejects invalid input interfaces with structured errors", async () => {
|
||||
await assert.rejects(
|
||||
createProcessor().process({ path: "/tmp/not-an-interface" }),
|
||||
(error) =>
|
||||
error instanceof ImageProcessingError &&
|
||||
error.code === "INVALID_IMAGE_INPUT" &&
|
||||
error.statusCode === 400,
|
||||
);
|
||||
});
|
||||
@@ -29,13 +29,33 @@ const silentLogger = {
|
||||
error() {},
|
||||
};
|
||||
|
||||
function testStorage() {
|
||||
return {
|
||||
async start() {},
|
||||
stop() {},
|
||||
health() {
|
||||
return {
|
||||
status: "degraded",
|
||||
checkedAt: "2026-08-16T00:00:00.000Z",
|
||||
lastSuccessAt: null,
|
||||
code: "STORAGE_UNAVAILABLE",
|
||||
message: "Object storage is unavailable",
|
||||
};
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
async function responseJson(baseUrl, path, init) {
|
||||
const response = await fetch(`${baseUrl}${path}`, init);
|
||||
return { response, body: await response.json() };
|
||||
}
|
||||
|
||||
test("HTTP foundation exposes live, ready, detailed health, and persisted settings", async (t) => {
|
||||
const runtime = createRuntime({ config: testConfig(), logger: silentLogger });
|
||||
const runtime = createRuntime({
|
||||
config: testConfig(),
|
||||
logger: silentLogger,
|
||||
storage: testStorage(),
|
||||
});
|
||||
await runtime.start();
|
||||
t.after(() => runtime.stop("test cleanup"));
|
||||
const baseUrl = `http://127.0.0.1:${runtime.address.port}`;
|
||||
@@ -52,7 +72,7 @@ test("HTTP foundation exposes live, ready, detailed health, and persisted settin
|
||||
assert.equal(status.response.status, 200);
|
||||
assert.equal(status.body.status, "degraded");
|
||||
assert.equal(status.body.services.database.currentSchemaVersion, 1);
|
||||
assert.equal(status.body.services.storage.status, "not_checked");
|
||||
assert.equal(status.body.services.storage.status, "degraded");
|
||||
const statusText = JSON.stringify(status.body);
|
||||
assert.doesNotMatch(statusText, /access-key-that-must-not-leak/);
|
||||
assert.doesNotMatch(statusText, /secret-key-that-must-not-leak/);
|
||||
@@ -83,7 +103,11 @@ test("HTTP foundation exposes live, ready, detailed health, and persisted settin
|
||||
});
|
||||
|
||||
test("invalid settings receive a structured client error", async (t) => {
|
||||
const runtime = createRuntime({ config: testConfig(), logger: silentLogger });
|
||||
const runtime = createRuntime({
|
||||
config: testConfig(),
|
||||
logger: silentLogger,
|
||||
storage: testStorage(),
|
||||
});
|
||||
await runtime.start();
|
||||
t.after(() => runtime.stop("test cleanup"));
|
||||
const baseUrl = `http://127.0.0.1:${runtime.address.port}`;
|
||||
@@ -99,7 +123,11 @@ test("invalid settings receive a structured client error", async (t) => {
|
||||
});
|
||||
|
||||
test("liveness stays up while critical readiness failures return 503", async (t) => {
|
||||
const runtime = createRuntime({ config: testConfig(), logger: silentLogger });
|
||||
const runtime = createRuntime({
|
||||
config: testConfig(),
|
||||
logger: silentLogger,
|
||||
storage: testStorage(),
|
||||
});
|
||||
await runtime.start();
|
||||
t.after(() => runtime.stop("test cleanup"));
|
||||
const baseUrl = `http://127.0.0.1:${runtime.address.port}`;
|
||||
@@ -127,7 +155,11 @@ test("liveness stays up while critical readiness failures return 503", async (t)
|
||||
});
|
||||
|
||||
test("runtime startup and shutdown are idempotent", async () => {
|
||||
const runtime = createRuntime({ config: testConfig(), logger: silentLogger });
|
||||
const runtime = createRuntime({
|
||||
config: testConfig(),
|
||||
logger: silentLogger,
|
||||
storage: testStorage(),
|
||||
});
|
||||
const [firstAddress, secondAddress] = await Promise.all([
|
||||
runtime.start(),
|
||||
runtime.start(),
|
||||
|
||||
@@ -0,0 +1,105 @@
|
||||
const test = require("node:test");
|
||||
const assert = require("node:assert/strict");
|
||||
const express = require("express");
|
||||
const { openDatabase } = require("../src/database");
|
||||
const { runMigrations } = require("../src/database/migrate");
|
||||
const {
|
||||
createDetailedHealthRouter,
|
||||
createHealthRouter,
|
||||
} = require("../src/health/routes");
|
||||
|
||||
function config() {
|
||||
return {
|
||||
environment: "test",
|
||||
host: "127.0.0.1",
|
||||
port: 0,
|
||||
timeZone: "America/Chicago",
|
||||
trustProxy: true,
|
||||
database: { path: ":memory:" },
|
||||
storage: {
|
||||
endpoint: "http://private-storage:8333",
|
||||
region: "us-east-1",
|
||||
bucket: "announcements-v2",
|
||||
accessKeyId: "must-not-leak",
|
||||
secretAccessKey: "must-not-leak-either",
|
||||
forcePathStyle: true,
|
||||
},
|
||||
processing: {
|
||||
tempDirectory: "/tmp/announcements-uploads",
|
||||
maxFileBytes: 1024,
|
||||
maxFiles: 2,
|
||||
maxInputPixels: 1000,
|
||||
},
|
||||
mqtt: { url: null },
|
||||
notifications: { homeAssistantWebhookUrl: null },
|
||||
};
|
||||
}
|
||||
|
||||
async function withServer(storage, callback) {
|
||||
const database = openDatabase({ path: ":memory:", busyTimeoutMs: 1000 });
|
||||
const migrations = runMigrations(database);
|
||||
const lifecycle = { ready: true };
|
||||
const app = express();
|
||||
const options = { config: config(), database, lifecycle, migrations, storage };
|
||||
app.use("/health", createHealthRouter(options));
|
||||
app.use("/api/health", createDetailedHealthRouter(options));
|
||||
const server = app.listen(0, "127.0.0.1");
|
||||
await new Promise((resolve) => server.once("listening", resolve));
|
||||
|
||||
try {
|
||||
await callback(`http://127.0.0.1:${server.address().port}`);
|
||||
} finally {
|
||||
await new Promise((resolve) => server.close(resolve));
|
||||
database.close();
|
||||
}
|
||||
}
|
||||
|
||||
test("storage degradation is reported but does not fail API readiness", async () => {
|
||||
const storage = {
|
||||
health() {
|
||||
return {
|
||||
status: "degraded",
|
||||
checkedAt: "2026-08-16T12:00:00.000Z",
|
||||
lastSuccessAt: "2026-08-16T11:59:00.000Z",
|
||||
code: "STORAGE_UNAVAILABLE",
|
||||
};
|
||||
},
|
||||
};
|
||||
|
||||
await withServer(storage, async (baseUrl) => {
|
||||
const ready = await fetch(`${baseUrl}/health/ready`);
|
||||
const readyBody = await ready.json();
|
||||
assert.equal(ready.status, 200);
|
||||
assert.equal(readyBody.status, "ready");
|
||||
assert.equal(readyBody.checks.storage, "degraded");
|
||||
|
||||
const detailed = await fetch(`${baseUrl}/api/health`);
|
||||
const detailedBody = await detailed.json();
|
||||
assert.equal(detailed.status, 200);
|
||||
assert.equal(detailedBody.status, "degraded");
|
||||
assert.equal(detailedBody.services.storage.status, "degraded");
|
||||
assert.equal(
|
||||
detailedBody.services.storage.lastSuccessAt,
|
||||
"2026-08-16T11:59:00.000Z",
|
||||
);
|
||||
assert.equal(detailedBody.services.storage.code, "STORAGE_UNAVAILABLE");
|
||||
assert.doesNotMatch(JSON.stringify(detailedBody), /must-not-leak/);
|
||||
assert.doesNotMatch(JSON.stringify(detailedBody), /private-storage/);
|
||||
});
|
||||
});
|
||||
|
||||
test("healthy storage produces healthy detailed status", async () => {
|
||||
const storage = {
|
||||
health() {
|
||||
return { status: "healthy", checkedAt: "2026-08-16T12:00:00.000Z" };
|
||||
},
|
||||
};
|
||||
|
||||
await withServer(storage, async (baseUrl) => {
|
||||
const detailed = await fetch(`${baseUrl}/api/health`);
|
||||
const body = await detailed.json();
|
||||
assert.equal(detailed.status, 200);
|
||||
assert.equal(body.status, "ok");
|
||||
assert.equal(body.services.storage.status, "healthy");
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,397 @@
|
||||
const test = require("node:test");
|
||||
const assert = require("node:assert/strict");
|
||||
const http = require("node:http");
|
||||
const { PassThrough, Readable } = require("node:stream");
|
||||
const {
|
||||
StorageObjectNotFoundError,
|
||||
StorageUnavailableError,
|
||||
createStorageService,
|
||||
} = require("../src/storage");
|
||||
|
||||
const storageConfig = {
|
||||
endpoint: "http://seaweedfs:8333",
|
||||
region: "us-east-1",
|
||||
bucket: "announcements-v2",
|
||||
accessKeyId: "test-key",
|
||||
secretAccessKey: "test-secret",
|
||||
forcePathStyle: true,
|
||||
};
|
||||
|
||||
function fakeClient(handler) {
|
||||
const commands = [];
|
||||
return {
|
||||
commands,
|
||||
async send(command) {
|
||||
commands.push(command);
|
||||
return handler(command, commands.length - 1);
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
function commandName(command) {
|
||||
return command.constructor.name;
|
||||
}
|
||||
|
||||
test("storage initialization probes an existing bucket without creating it", async () => {
|
||||
const client = fakeClient(async () => ({}));
|
||||
const service = createStorageService(storageConfig, { client });
|
||||
|
||||
const health = await service.initialize();
|
||||
assert.equal(health.status, "healthy");
|
||||
assert.match(health.checkedAt, /T/);
|
||||
assert.equal(health.lastSuccessAt, health.checkedAt);
|
||||
assert.deepEqual(client.commands.map(commandName), ["HeadBucketCommand"]);
|
||||
});
|
||||
|
||||
test("storage initialization creates only a missing bucket", async () => {
|
||||
let exists = false;
|
||||
const missing = Object.assign(new Error("missing"), {
|
||||
name: "NoSuchBucket",
|
||||
$metadata: { httpStatusCode: 404 },
|
||||
});
|
||||
const client = fakeClient(async (command) => {
|
||||
if (commandName(command) === "HeadBucketCommand" && !exists) throw missing;
|
||||
if (commandName(command) === "CreateBucketCommand") exists = true;
|
||||
return {};
|
||||
});
|
||||
const service = createStorageService(storageConfig, { client });
|
||||
|
||||
assert.deepEqual(await service.ensureBucket(), { created: true });
|
||||
assert.deepEqual(client.commands.map(commandName), [
|
||||
"HeadBucketCommand",
|
||||
"CreateBucketCommand",
|
||||
]);
|
||||
assert.equal(client.commands[1].input.Bucket, storageConfig.bucket);
|
||||
|
||||
assert.deepEqual(await service.ensureBucket(), { created: false });
|
||||
assert.deepEqual(client.commands.map(commandName), [
|
||||
"HeadBucketCommand",
|
||||
"CreateBucketCommand",
|
||||
"HeadBucketCommand",
|
||||
]);
|
||||
});
|
||||
|
||||
test("storage start performs a nonfatal immediate check and stop releases its client", async () => {
|
||||
let destroyed = false;
|
||||
const client = fakeClient(async () => ({}));
|
||||
client.destroy = () => {
|
||||
destroyed = true;
|
||||
};
|
||||
const service = createStorageService(
|
||||
{ ...storageConfig, healthProbeIntervalMs: 1_000 },
|
||||
{ client },
|
||||
);
|
||||
|
||||
const health = await service.start();
|
||||
assert.equal(health.status, "healthy");
|
||||
assert.deepEqual(client.commands.map(commandName), ["HeadBucketCommand"]);
|
||||
service.stop();
|
||||
assert.equal(destroyed, true);
|
||||
});
|
||||
|
||||
test("a create-bucket race is healthy only after the bucket can be verified", async () => {
|
||||
const missing = Object.assign(new Error("missing"), {
|
||||
name: "NoSuchBucket",
|
||||
$metadata: { httpStatusCode: 404 },
|
||||
});
|
||||
const conflict = Object.assign(new Error("race"), {
|
||||
name: "BucketAlreadyOwnedByYou",
|
||||
$metadata: { httpStatusCode: 409 },
|
||||
});
|
||||
let headCalls = 0;
|
||||
const client = fakeClient(async (command) => {
|
||||
if (commandName(command) === "HeadBucketCommand") {
|
||||
headCalls += 1;
|
||||
if (headCalls === 1) throw missing;
|
||||
return {};
|
||||
}
|
||||
if (commandName(command) === "CreateBucketCommand") throw conflict;
|
||||
throw new Error(`Unexpected ${commandName(command)}`);
|
||||
});
|
||||
const service = createStorageService(storageConfig, { client });
|
||||
|
||||
assert.deepEqual(await service.ensureBucket(), { created: false });
|
||||
assert.equal(service.health().status, "healthy");
|
||||
assert.deepEqual(client.commands.map(commandName), [
|
||||
"HeadBucketCommand",
|
||||
"CreateBucketCommand",
|
||||
"HeadBucketCommand",
|
||||
]);
|
||||
});
|
||||
|
||||
test("an older failed probe cannot overwrite a newer successful operation", async () => {
|
||||
let rejectProbe;
|
||||
let notifyProbeStarted;
|
||||
const probeStarted = new Promise((resolve) => {
|
||||
notifyProbeStarted = resolve;
|
||||
});
|
||||
let headCalls = 0;
|
||||
const client = fakeClient(async (command) => {
|
||||
if (commandName(command) === "HeadBucketCommand") {
|
||||
headCalls += 1;
|
||||
if (headCalls === 1) return {};
|
||||
notifyProbeStarted();
|
||||
return new Promise((resolve, reject) => {
|
||||
rejectProbe = reject;
|
||||
});
|
||||
}
|
||||
if (commandName(command) === "PutObjectCommand") return {};
|
||||
throw new Error(`Unexpected ${commandName(command)}`);
|
||||
});
|
||||
const service = createStorageService(storageConfig, { client });
|
||||
await service.initialize();
|
||||
|
||||
const staleProbe = service.probe();
|
||||
await probeStarted;
|
||||
await service.putObject({
|
||||
key: "assets/id/original/hash",
|
||||
body: Buffer.from("bytes"),
|
||||
contentLength: 5,
|
||||
});
|
||||
const successfulState = service.health();
|
||||
rejectProbe(new Error("older request failed"));
|
||||
await staleProbe;
|
||||
|
||||
assert.equal(successfulState.status, "healthy");
|
||||
assert.deepEqual(service.health(), successfulState);
|
||||
});
|
||||
|
||||
test("temporary storage failure degrades without throwing during initialization and recovers", async () => {
|
||||
let available = false;
|
||||
const warnings = [];
|
||||
const client = fakeClient(async () => {
|
||||
if (!available) throw new Error("connection refused at a private address");
|
||||
return {};
|
||||
});
|
||||
const service = createStorageService(storageConfig, {
|
||||
client,
|
||||
logger: { warn(message) { warnings.push(message); } },
|
||||
});
|
||||
|
||||
const initial = await service.initialize();
|
||||
assert.equal(initial.status, "degraded");
|
||||
assert.equal(initial.code, "STORAGE_UNAVAILABLE");
|
||||
assert.equal(initial.lastSuccessAt, null);
|
||||
assert.deepEqual(warnings, ["Object storage is unavailable during initialization"]);
|
||||
|
||||
available = true;
|
||||
const recovered = await service.probe();
|
||||
assert.equal(recovered.status, "healthy");
|
||||
assert.equal(recovered.lastSuccessAt, recovered.checkedAt);
|
||||
assert.equal(service.health().status, "healthy");
|
||||
|
||||
available = false;
|
||||
const degradedAgain = await service.probe();
|
||||
assert.equal(degradedAgain.status, "degraded");
|
||||
assert.equal(degradedAgain.lastSuccessAt, recovered.lastSuccessAt);
|
||||
});
|
||||
|
||||
test("put, get, head, existence, and delete use the configured private bucket", async () => {
|
||||
const body = Readable.from([Buffer.from("image bytes")]);
|
||||
const client = fakeClient(async (command) => {
|
||||
switch (commandName(command)) {
|
||||
case "PutObjectCommand":
|
||||
return { ETag: '"etag"' };
|
||||
case "GetObjectCommand":
|
||||
return {
|
||||
Body: body,
|
||||
ContentType: "image/webp",
|
||||
ContentLength: 11,
|
||||
ETag: '"etag"',
|
||||
};
|
||||
case "HeadObjectCommand":
|
||||
return {
|
||||
ContentType: "image/webp",
|
||||
ContentLength: 11,
|
||||
ETag: '"etag"',
|
||||
};
|
||||
case "DeleteObjectCommand":
|
||||
return {};
|
||||
case "HeadBucketCommand":
|
||||
return {};
|
||||
default:
|
||||
throw new Error(`Unexpected ${commandName(command)}`);
|
||||
}
|
||||
});
|
||||
const service = createStorageService(storageConfig, { client });
|
||||
const key = "assets/id/display/hash.webp";
|
||||
await service.initialize();
|
||||
client.commands.length = 0;
|
||||
|
||||
const put = await service.putObject({
|
||||
key,
|
||||
body: Buffer.from("image bytes"),
|
||||
contentType: "image/webp",
|
||||
contentLength: 11,
|
||||
});
|
||||
assert.deepEqual(put, { key, etag: '"etag"', versionId: null });
|
||||
|
||||
const fetched = await service.getObject(key);
|
||||
assert.equal(fetched.body, body);
|
||||
assert.equal(fetched.contentType, "image/webp");
|
||||
assert.equal(fetched.contentLength, 11);
|
||||
|
||||
assert.equal((await service.headObject(key)).etag, '"etag"');
|
||||
assert.equal(await service.objectExists(key), true);
|
||||
await service.deleteObject(key);
|
||||
|
||||
for (const command of client.commands) {
|
||||
assert.equal(command.input.Bucket, storageConfig.bucket);
|
||||
assert.equal(command.input.Key, key);
|
||||
}
|
||||
});
|
||||
|
||||
test("missing objects are distinct from unavailable storage", async () => {
|
||||
let missing = true;
|
||||
const client = fakeClient(async (command) => {
|
||||
if (commandName(command) === "HeadBucketCommand") return {};
|
||||
if (missing) {
|
||||
throw Object.assign(new Error("missing"), {
|
||||
name: "NoSuchKey",
|
||||
$metadata: { httpStatusCode: 404 },
|
||||
});
|
||||
}
|
||||
throw new Error("network offline");
|
||||
});
|
||||
const service = createStorageService(storageConfig, { client });
|
||||
|
||||
await service.initialize();
|
||||
client.commands.length = 0;
|
||||
|
||||
assert.equal(await service.headObject("assets/id/original/hash"), null);
|
||||
assert.equal(await service.objectExists("assets/id/original/hash"), false);
|
||||
await assert.rejects(
|
||||
service.getObject("assets/id/original/hash"),
|
||||
StorageObjectNotFoundError,
|
||||
);
|
||||
assert.equal(service.health().status, "healthy");
|
||||
|
||||
missing = false;
|
||||
await assert.rejects(
|
||||
service.putObject({ key: "assets/id/original/hash", body: Buffer.alloc(0) }),
|
||||
(error) =>
|
||||
error instanceof StorageUnavailableError &&
|
||||
error.code === "STORAGE_UNAVAILABLE" &&
|
||||
error.statusCode === 503,
|
||||
);
|
||||
assert.equal(service.health().status, "degraded");
|
||||
});
|
||||
|
||||
test("a missing bucket during object access is a storage outage, not a missing object", async () => {
|
||||
let initialized = false;
|
||||
const client = fakeClient(async (command) => {
|
||||
if (commandName(command) === "HeadBucketCommand" && !initialized) {
|
||||
initialized = true;
|
||||
return {};
|
||||
}
|
||||
throw Object.assign(new Error("bucket missing"), {
|
||||
name: "NoSuchBucket",
|
||||
$metadata: { httpStatusCode: 404 },
|
||||
});
|
||||
});
|
||||
const service = createStorageService(storageConfig, { client });
|
||||
await service.initialize();
|
||||
|
||||
await assert.rejects(
|
||||
service.getObject("assets/id/original/hash"),
|
||||
StorageUnavailableError,
|
||||
);
|
||||
assert.equal(service.health().status, "degraded");
|
||||
});
|
||||
|
||||
test("storage rejects unsafe internal object keys before contacting S3", async () => {
|
||||
const client = fakeClient(async () => ({}));
|
||||
const service = createStorageService(storageConfig, { client });
|
||||
|
||||
await assert.rejects(
|
||||
service.getObject("../private"),
|
||||
/Storage object key is invalid/,
|
||||
);
|
||||
await assert.rejects(
|
||||
service.deleteObject("/absolute"),
|
||||
/Storage object key is invalid/,
|
||||
);
|
||||
assert.equal(client.commands.length, 0);
|
||||
});
|
||||
|
||||
test("real HTTP requests time out instead of hanging indefinitely", async (t) => {
|
||||
const server = http.createServer(() => {
|
||||
// Intentionally accept the connection without sending response headers.
|
||||
});
|
||||
server.listen(0, "127.0.0.1");
|
||||
await new Promise((resolve) => server.once("listening", resolve));
|
||||
t.after(async () => {
|
||||
server.closeAllConnections();
|
||||
await new Promise((resolve) => server.close(resolve));
|
||||
});
|
||||
|
||||
const service = createStorageService({
|
||||
...storageConfig,
|
||||
endpoint: `http://127.0.0.1:${server.address().port}`,
|
||||
connectionTimeoutMs: 100,
|
||||
requestTimeoutMs: 100,
|
||||
maxAttempts: 1,
|
||||
});
|
||||
t.after(() => service.stop());
|
||||
|
||||
const startedAt = Date.now();
|
||||
const health = await service.probe();
|
||||
assert.equal(health.status, "degraded");
|
||||
assert.equal(health.code, "STORAGE_UNAVAILABLE");
|
||||
assert.ok(Date.now() - startedAt < 2_000);
|
||||
});
|
||||
|
||||
test("consumer stream cancellation does not falsely degrade storage", async () => {
|
||||
let body = new PassThrough();
|
||||
const client = fakeClient(async (command) => {
|
||||
if (commandName(command) === "HeadBucketCommand") return {};
|
||||
if (commandName(command) === "GetObjectCommand") {
|
||||
return { Body: body, ContentLength: 10, ContentType: "image/webp" };
|
||||
}
|
||||
throw new Error(`Unexpected ${commandName(command)}`);
|
||||
});
|
||||
const service = createStorageService(storageConfig, { client });
|
||||
await service.initialize();
|
||||
|
||||
const cancelled = await service.getObject("assets/id/display/hash.webp");
|
||||
cancelled.body.destroy(
|
||||
Object.assign(new Error("consumer closed"), {
|
||||
code: "ERR_STREAM_PREMATURE_CLOSE",
|
||||
}),
|
||||
);
|
||||
await new Promise((resolve) => setImmediate(resolve));
|
||||
assert.equal(service.health().status, "healthy");
|
||||
|
||||
body = new PassThrough();
|
||||
const failed = await service.getObject("assets/id/display/hash.webp");
|
||||
failed.body.destroy(Object.assign(new Error("upstream reset"), { code: "ECONNRESET" }));
|
||||
await new Promise((resolve) => setImmediate(resolve));
|
||||
assert.equal(service.health().status, "degraded");
|
||||
});
|
||||
|
||||
test("delete treats an already-missing object as an idempotent success", async () => {
|
||||
let missingBucket = false;
|
||||
const client = fakeClient(async (command) => {
|
||||
if (commandName(command) === "HeadBucketCommand") return {};
|
||||
if (commandName(command) === "DeleteObjectCommand") {
|
||||
throw Object.assign(new Error("missing"), {
|
||||
name: missingBucket ? "NoSuchBucket" : "NoSuchKey",
|
||||
$metadata: { httpStatusCode: 404 },
|
||||
});
|
||||
}
|
||||
throw new Error(`Unexpected ${commandName(command)}`);
|
||||
});
|
||||
const service = createStorageService(storageConfig, { client });
|
||||
await service.initialize();
|
||||
|
||||
await assert.doesNotReject(
|
||||
service.deleteObject("assets/id/original/hash"),
|
||||
);
|
||||
assert.equal(service.health().status, "healthy");
|
||||
|
||||
missingBucket = true;
|
||||
await assert.rejects(
|
||||
service.deleteObject("assets/id/original/hash"),
|
||||
StorageUnavailableError,
|
||||
);
|
||||
});
|
||||
Reference in New Issue
Block a user