Files
Announcements/server/test/assets-service.test.js

609 lines
21 KiB
JavaScript

const test = require("node:test");
const assert = require("node:assert/strict");
const crypto = require("node:crypto");
const fs = require("node:fs");
const os = require("node:os");
const path = require("node:path");
const { Readable } = require("node:stream");
const { openDatabase } = require("../src/database");
const { runMigrations } = require("../src/database/migrate");
const { createAssetService } = require("../src/assets/service");
const {
StorageObjectNotFoundError,
StorageUnavailableError,
} = require("../src/storage");
async function bufferBody(body) {
if (Buffer.isBuffer(body)) return body;
if (body instanceof Uint8Array) return Buffer.from(body);
const chunks = [];
for await (const chunk of body) chunks.push(Buffer.from(chunk));
return Buffer.concat(chunks);
}
function createMemoryStorage() {
const objects = new Map();
const calls = [];
let failNextPut = null;
let successfulPutsBeforeFailure = 0;
let failNextDelete = null;
let putDelayMs = 0;
return {
objects,
calls,
setPutFailure(error, putsBeforeFailure = 0) {
failNextPut = error;
successfulPutsBeforeFailure = putsBeforeFailure;
},
setDeleteFailure(error) {
failNextDelete = error;
},
setPutDelay(milliseconds) {
putDelayMs = milliseconds;
},
async putObject(input) {
calls.push({ operation: "put", ...input, body: undefined });
if (putDelayMs) {
await new Promise((resolve) => setTimeout(resolve, putDelayMs));
}
if (failNextPut) {
if (successfulPutsBeforeFailure === 0) {
const error = failNextPut;
failNextPut = null;
throw error;
}
successfulPutsBeforeFailure -= 1;
}
objects.set(input.key, {
body: await bufferBody(input.body),
contentType: input.contentType,
metadata: input.metadata,
});
return { key: input.key };
},
async getObject(key) {
calls.push({ operation: "get", key });
const object = objects.get(key);
if (!object) throw new StorageObjectNotFoundError(key);
return {
body: Readable.from(object.body),
contentLength: object.body.length,
contentType: object.contentType,
};
},
async deleteObject(key) {
calls.push({ operation: "delete", key });
if (failNextDelete) {
const error = failNextDelete;
failNextDelete = null;
throw error;
}
objects.delete(key);
return { key, deleted: true };
},
};
}
function createFakeImageProcessor({ reject } = {}) {
const calls = [];
return {
calls,
async process(filename) {
calls.push(filename);
if (reject) throw reject;
const original = await fs.promises.readFile(filename);
const sourceSha256 = crypto.createHash("sha256").update(original).digest("hex");
const displayBuffer = Buffer.from(`display:${sourceSha256}`);
const thumbnailBuffer = Buffer.from(`thumbnail:${sourceSha256}`);
return {
original: {
sha256: sourceSha256,
contentType: "image/png",
byteSize: original.length,
width: 1200,
height: 800,
},
display: {
buffer: displayBuffer,
sha256: crypto.createHash("sha256").update(displayBuffer).digest("hex"),
contentType: "image/webp",
byteSize: displayBuffer.length,
width: 1200,
height: 800,
},
thumbnail: {
buffer: thumbnailBuffer,
sha256: crypto.createHash("sha256").update(thumbnailBuffer).digest("hex"),
contentType: "image/webp",
byteSize: thumbnailBuffer.length,
width: 405,
height: 270,
},
};
},
};
}
function createHarness(options = {}) {
const database = openDatabase({ path: ":memory:" });
runMigrations(database);
const directory = fs.mkdtempSync(path.join(os.tmpdir(), "announcements-assets-"));
const storage = options.storage || createMemoryStorage();
const imageProcessor =
options.imageProcessor || createFakeImageProcessor(options.processorOptions);
let idCounter = 0;
let clock = 2_000_000_000_000;
const service = createAssetService({
database,
storage,
imageProcessor,
now: () => ++clock,
createId:
options.createId ||
(() => `00000000-0000-4000-8000-${String(++idCounter).padStart(12, "0")}`),
concurrentUploadWaitMs: options.concurrentUploadWaitMs ?? 2_000,
});
function uploadFile(originalFilename, contents) {
const buffer = Buffer.isBuffer(contents) ? contents : Buffer.from(contents);
const filename = path.join(directory, crypto.randomUUID());
fs.writeFileSync(filename, buffer);
return { path: filename, byteSize: buffer.length, originalFilename };
}
return {
database,
directory,
imageProcessor,
service,
storage,
uploadFile,
close() {
database.close();
fs.rmSync(directory, { recursive: true, force: true });
},
};
}
function rawAsset(database, assetId) {
return database.prepare("SELECT * FROM assets WHERE id = ?").get(assetId);
}
test("upload stores exact bytes under content-addressed keys without leaking keys", async (t) => {
const harness = createHarness();
t.after(() => harness.close());
const original = Buffer.from("exact uploaded bytes");
const [result] = await harness.service.upload([
harness.uploadFile(" Sunday Welcome.PNG ", original),
]);
assert.equal(result.created, true);
assert.equal(result.duplicate, false);
assert.match(result.asset.id, /^[0-9a-f-]{36}$/);
assert.equal(result.asset.displayName, "Sunday Welcome");
assert.equal(result.asset.originalFilename, " Sunday Welcome.PNG ");
assert.equal(result.asset.expirationAt, null);
assert.equal(result.asset.status, "ready");
assert.equal(result.asset.original.byteSize, original.length);
assert.equal(result.asset.display.contentType, "image/webp");
assert.equal(result.asset.thumbnail.width, 405);
assert.match(result.asset.display.url, /^\/media\/assets\//);
assert.doesNotMatch(
JSON.stringify(result.asset),
/object_key|originalObjectKey|displayObjectKey|thumbnailObjectKey|seaweed/i,
);
const row = rawAsset(harness.database, result.asset.id);
const expectedSourceHash = crypto.createHash("sha256").update(original).digest("hex");
assert.equal(row.source_sha256, expectedSourceHash);
assert.equal(row.original_object_key, `assets/${row.id}/original/${expectedSourceHash}`);
assert.equal(
row.display_object_key,
`assets/${row.id}/display/${row.display_sha256}.webp`,
);
assert.equal(
row.thumbnail_object_key,
`assets/${row.id}/thumbnail/${row.thumbnail_sha256}.webp`,
);
assert.deepEqual(harness.storage.objects.get(row.original_object_key).body, original);
assert.equal(harness.storage.objects.size, 3);
assert.deepEqual(harness.service.list(), [result.asset]);
assert.deepEqual(harness.service.get(result.asset.id), result.asset);
});
test("exact duplicate detection takes precedence over filename and display name", async (t) => {
const harness = createHarness();
t.after(() => harness.close());
const contents = Buffer.from("same image bytes");
const [created] = await harness.service.upload([
harness.uploadFile("first.png", contents),
]);
const [sameFilename] = await harness.service.upload([
harness.uploadFile("first.png", contents),
]);
const [differentFilename] = await harness.service.upload(
[harness.uploadFile("completely different name.webp", contents)],
{ displayName: "A conflicting requested name is ignored for an exact duplicate" },
);
for (const duplicate of [sameFilename, differentFilename]) {
assert.equal(duplicate.created, false);
assert.equal(duplicate.duplicate, true);
assert.equal(duplicate.asset.id, created.asset.id);
}
assert.equal(harness.service.list().length, 1);
assert.equal(harness.storage.calls.filter((call) => call.operation === "put").length, 3);
assert.equal(harness.imageProcessor.calls.length, 1);
});
test("different content with the same proposed name is a conflict, not a duplicate", async (t) => {
const harness = createHarness();
t.after(() => harness.close());
await harness.service.upload([harness.uploadFile("bulletin.png", "image one")]);
await assert.rejects(
harness.service.upload([harness.uploadFile("bulletin.jpg", "image two")]),
(error) => {
assert.equal(error.statusCode, 409);
assert.equal(error.code, "ASSET_NAME_CONFLICT");
assert.equal(error.details.asset.displayName, "bulletin");
return true;
},
);
assert.equal(harness.service.list().length, 1);
assert.equal(harness.imageProcessor.calls.length, 1);
const [created] = await harness.service.upload(
[harness.uploadFile("bulletin.jpg", "image two")],
{ displayName: "Second bulletin" },
);
assert.equal(created.created, true);
assert.notEqual(created.asset.sourceSha256, harness.service.list()[1].sourceSha256);
assert.equal(harness.service.list().length, 2);
});
test("multiple-file upload preserves order and deduplicates repeated bytes in the batch", async (t) => {
const harness = createHarness();
t.after(() => harness.close());
const results = await harness.service.upload([
harness.uploadFile("alpha.png", "alpha bytes"),
harness.uploadFile("beta.png", "beta bytes"),
harness.uploadFile("alpha-copy.png", "alpha bytes"),
]);
assert.deepEqual(
results.map(({ created, duplicate }) => ({ created, duplicate })),
[
{ created: true, duplicate: false },
{ created: true, duplicate: false },
{ created: false, duplicate: true },
],
);
assert.equal(results[0].asset.id, results[2].asset.id);
assert.equal(harness.service.list().length, 2);
assert.equal(harness.imageProcessor.calls.length, 2);
assert.equal(harness.storage.objects.size, 6);
});
test("all new images are decoded before the batch mutates SQLite or storage", async (t) => {
const expected = Object.assign(new Error("bad second image"), {
code: "MALFORMED_IMAGE",
statusCode: 422,
});
let processCount = 0;
const regularProcessor = createFakeImageProcessor();
const harness = createHarness({
imageProcessor: {
async process(filename) {
processCount += 1;
if (processCount === 2) throw expected;
return regularProcessor.process(filename);
},
},
});
t.after(() => harness.close());
await assert.rejects(
harness.service.upload([
harness.uploadFile("valid.png", "valid"),
harness.uploadFile("broken.png", "broken"),
]),
expected,
);
assert.equal(harness.service.list().length, 0);
assert.equal(harness.storage.objects.size, 0);
assert.equal(harness.storage.calls.length, 0);
});
test("concurrent identical uploads converge on one ready asset", async (t) => {
const storage = createMemoryStorage();
storage.setPutDelay(30);
const harness = createHarness({ storage });
t.after(() => harness.close());
const bytes = Buffer.from("racing identical upload");
const [[left], [right]] = await Promise.all([
harness.service.upload([harness.uploadFile("race.png", bytes)]),
harness.service.upload([harness.uploadFile("other-name.png", bytes)]),
]);
assert.equal(harness.service.list().length, 1);
assert.equal(left.asset.id, right.asset.id);
assert.deepEqual([left.created, right.created].sort(), [false, true]);
assert.deepEqual([left.duplicate, right.duplicate].sort(), [false, true]);
assert.equal(storage.calls.filter((call) => call.operation === "put").length, 3);
});
test("a partial storage failure leaves a diagnosable failed row and no ready URLs", async (t) => {
const storage = createMemoryStorage();
storage.setPutFailure(new StorageUnavailableError(new Error("offline")), 1);
const harness = createHarness({ storage });
t.after(() => harness.close());
await assert.rejects(
harness.service.upload([harness.uploadFile("failure.png", "failure bytes")]),
(error) => {
assert.equal(error.statusCode, 503);
assert.equal(error.details.retryable, true);
assert.match(error.details.assetId, /^[0-9a-f-]{36}$/);
return true;
},
);
const [failed] = harness.service.list();
assert.equal(failed.status, "failed");
assert.match(failed.processingError, /^STORAGE_UNAVAILABLE:/);
assert.equal(failed.display.url, null);
assert.equal(failed.thumbnail.url, null);
assert.equal(failed.original.url, null);
assert.equal(storage.objects.size, 0);
const row = rawAsset(harness.database, failed.id);
assert.match(
row.original_object_key,
new RegExp(`^assets/${failed.id}/original/[0-9a-f]{64}$`),
);
assert.equal(
row.display_object_key,
`assets/${failed.id}/display/${row.display_sha256}.webp`,
);
assert.equal(
row.thumbnail_object_key,
`assets/${failed.id}/thumbnail/${row.thumbnail_sha256}.webp`,
);
assert.equal(storage.calls.filter((call) => call.operation === "put").length, 2);
assert.deepEqual(
new Set(
storage.calls
.filter((call) => call.operation === "delete")
.map((call) => call.key),
),
new Set([
row.original_object_key,
row.display_object_key,
row.thumbnail_object_key,
]),
);
await assert.rejects(
harness.service.upload([harness.uploadFile("retry.png", "failure bytes")]),
(error) => error.code === "ASSET_PROCESSING_FAILED" && error.statusCode === 409,
);
assert.deepEqual(await harness.service.remove(failed.id), {
id: failed.id,
deleted: true,
});
assert.equal(harness.service.list().length, 0);
});
test("rename and expiration edits change only approved SQLite metadata", async (t) => {
const harness = createHarness();
t.after(() => harness.close());
const [upload] = await harness.service.upload([
harness.uploadFile("editable.png", "editable bytes"),
]);
const before = rawAsset(harness.database, upload.asset.id);
const expiring = harness.service.update(upload.asset.id, {
display_name: " New display name ",
expiration_at: "2035-05-06T07:08:09.000Z",
});
assert.equal(expiring.displayName, "New display name");
assert.equal(expiring.expirationAt, Date.parse("2035-05-06T07:08:09.000Z"));
const permanent = harness.service.update(upload.asset.id, { expirationAt: null });
assert.equal(permanent.expirationAt, null);
const after = rawAsset(harness.database, upload.asset.id);
assert.equal(after.original_object_key, before.original_object_key);
assert.equal(after.display_object_key, before.display_object_key);
assert.equal(after.thumbnail_object_key, before.thumbnail_object_key);
assert.equal(harness.storage.calls.filter((call) => call.operation === "put").length, 3);
await assert.rejects(
async () => harness.service.update(upload.asset.id, { status: "failed" }),
(error) => error.code === "INVALID_ASSET_UPDATE",
);
assert.throws(
() => harness.service.update(upload.asset.id, { expirationAt: -1 }),
(error) => error.code === "INVALID_ASSET_EXPIRATION",
);
});
test("references describe playlist, normal, and static use and block deletion", async (t) => {
const harness = createHarness();
t.after(() => harness.close());
const [upload] = await harness.service.upload([
harness.uploadFile("referenced.png", "referenced bytes"),
]);
const id = upload.asset.id;
const now = 2_100_000_000_000;
harness.database
.prepare(
"INSERT INTO playlists (id, name, created_at, updated_at) VALUES (?, ?, ?, ?)",
)
.run("playlist-1", "Sunday", now, now);
harness.database
.prepare(
`INSERT INTO playlist_entries
(id, playlist_id, asset_id, enabled, position, created_at, updated_at)
VALUES (?, ?, ?, 1, 0, ?, ?)`,
)
.run("entry-1", "playlist-1", id, now, now);
harness.database
.prepare(
`UPDATE playback_state
SET active_playlist_id = ?, current_entry_id = ?, static_asset_id = ?, updated_at = ?
WHERE output_id = 'global'`,
)
.run("playlist-1", "entry-1", id, now);
const references = harness.service.references(id);
assert.equal(references.canDelete, false);
assert.deepEqual(references.blockerCodes, [
"PLAYLIST_REFERENCES",
"CURRENT_PLAYBACK_TARGET",
"CURRENT_STATIC_TARGET",
]);
assert.deepEqual(references.playlists[0], {
playlistId: "playlist-1",
playlistName: "Sunday",
playlistEntryId: "entry-1",
enabled: true,
position: 0,
isActivePlaylist: true,
isCurrentNormalTarget: true,
});
assert.equal(references.playback[0].isCurrentNormalTarget, true);
assert.equal(references.playback[0].isCurrentStaticTarget, true);
const callsBeforeDelete = harness.storage.calls.length;
await assert.rejects(
harness.service.remove(id),
(error) => {
assert.equal(error.code, "ASSET_REFERENCED");
assert.equal(error.statusCode, 409);
assert.deepEqual(error.details.blockerCodes, [
"PLAYLIST_REFERENCES",
"CURRENT_PLAYBACK_TARGET",
"CURRENT_STATIC_TARGET",
]);
return true;
},
);
assert.equal(harness.storage.calls.length, callsBeforeDelete);
assert.equal(rawAsset(harness.database, id).status, "ready");
harness.database
.prepare(
`UPDATE playback_state
SET active_playlist_id = NULL, current_entry_id = NULL, static_asset_id = NULL
WHERE output_id = 'global'`,
)
.run();
harness.database.prepare("DELETE FROM playlist_entries WHERE id = ?").run("entry-1");
assert.equal(harness.service.references(id).canDelete, true);
await harness.service.remove(id);
assert.equal(harness.storage.objects.size, 0);
assert.equal(rawAsset(harness.database, id), undefined);
});
test("failed storage deletion retains a retryable deleting row", async (t) => {
const storage = createMemoryStorage();
const harness = createHarness({ storage });
t.after(() => harness.close());
const [upload] = await harness.service.upload([
harness.uploadFile("delete-me.png", "delete me"),
]);
storage.setDeleteFailure(new StorageUnavailableError(new Error("offline")));
await assert.rejects(harness.service.remove(upload.asset.id), (error) => {
assert.equal(error.statusCode, 503);
assert.equal(error.details.assetId, upload.asset.id);
assert.equal(error.details.retryable, true);
return true;
});
assert.equal(rawAsset(harness.database, upload.asset.id).status, "deleting");
assert.throws(
() => harness.service.update(upload.asset.id, { displayName: "Cannot edit" }),
(error) => error.code === "ASSET_DELETING",
);
await harness.service.remove(upload.asset.id);
assert.equal(rawAsset(harness.database, upload.asset.id), undefined);
assert.equal(storage.objects.size, 0);
});
test("processing assets cannot be deleted and storage is not touched", async (t) => {
const harness = createHarness();
t.after(() => harness.close());
const timestamp = Date.now();
harness.database
.prepare(
`INSERT INTO assets
(id, source_sha256, display_name, original_filename, status, created_at, updated_at)
VALUES (?, ?, ?, ?, 'processing', ?, ?)`,
)
.run(
"00000000-0000-4000-8000-999999999999",
"f".repeat(64),
"Still processing",
"processing.png",
timestamp,
timestamp,
);
assert.equal(
harness.service.references("00000000-0000-4000-8000-999999999999")
.canDelete,
false,
);
await assert.rejects(
harness.service.remove("00000000-0000-4000-8000-999999999999"),
(error) => error.code === "ASSET_PROCESSING" && error.statusCode === 409,
);
assert.equal(harness.storage.calls.length, 0);
assert.equal(
rawAsset(harness.database, "00000000-0000-4000-8000-999999999999").status,
"processing",
);
});
test("media descriptors only allow the recorded version and map missing bytes to 503", async (t) => {
const harness = createHarness();
t.after(() => harness.close());
const [upload] = await harness.service.upload([
harness.uploadFile("media.png", "media bytes"),
]);
const { asset } = upload;
const display = harness.service.mediaDescriptor(
asset.id,
"display",
asset.display.sha256,
);
assert.equal(display.contentType, "image/webp");
assert.equal(display.contentLength, asset.display.byteSize);
const stored = await harness.service.readStoredObject(display);
assert.deepEqual(await bufferBody(stored.body), harness.storage.objects.get(display.key).body);
for (const requestedHash of ["a".repeat(64), "../original/key", "not-a-hash"]) {
assert.throws(
() => harness.service.mediaDescriptor(asset.id, "display", requestedHash),
(error) => error.code === "MEDIA_NOT_FOUND" && error.statusCode === 404,
);
}
harness.storage.objects.delete(display.key);
await assert.rejects(harness.service.readStoredObject(display), (error) => {
assert.equal(error.code, "STORAGE_INCONSISTENCY");
assert.equal(error.statusCode, 503);
assert.deepEqual(error.details, { assetId: asset.id, retryable: true });
return true;
});
});