Implement Phase 2 asset storage pipeline

2026-08-16 19:59:10 -05:00
parent eb19675a5c
commit 722c6cf77d
29 changed files with 5059 additions and 16 deletions
@@ -0,0 +1,442 @@
+const test = require("node:test");
+const assert = require("node:assert/strict");
+const crypto = require("node:crypto");
+const fs = require("node:fs");
+const http = require("node:http");
+const os = require("node:os");
+const path = require("node:path");
+const { Readable } = require("node:stream");
+const sharp = require("sharp");
+const { createApp } = require("../src/app");
+const { loadConfig } = require("../src/config");
+const { openDatabase } = require("../src/database");
+const { runMigrations } = require("../src/database/migrate");
+const { createImageProcessor } = require("../src/image-processing");
+const { StorageObjectNotFoundError } = require("../src/storage");
+
+async function readBody(body) {
+  if (Buffer.isBuffer(body)) return body;
+  const chunks = [];
+  for await (const chunk of body) chunks.push(Buffer.from(chunk));
+  return Buffer.concat(chunks);
+}
+
+function memoryStorage() {
+  const objects = new Map();
+  return {
+    objects,
+    health() {
+      return {
+        status: "healthy",
+        checkedAt: "2033-05-18T03:33:20.000Z",
+        lastSuccessAt: "2033-05-18T03:33:20.000Z",
+      };
+    },
+    async putObject(input) {
+      objects.set(input.key, {
+        body: await readBody(input.body),
+        contentType: input.contentType,
+        metadata: input.metadata,
+      });
+      return { key: input.key };
+    },
+    async getObject(key) {
+      const object = objects.get(key);
+      if (!object) throw new StorageObjectNotFoundError(key);
+      return {
+        body: Readable.from([object.body]),
+        contentLength: object.body.length,
+        contentType: object.contentType,
+      };
+    },
+    async deleteObject(key) {
+      objects.delete(key);
+      return { key, deleted: true };
+    },
+  };
+}
+
+async function createHarness(t, { maxFileBytes = 1024 * 1024 } = {}) {
+  const tempDirectory = fs.mkdtempSync(
+    path.join(os.tmpdir(), "announcements-api-uploads-"),
+  );
+  const config = loadConfig({
+    NODE_ENV: "test",
+    HOST: "127.0.0.1",
+    PORT: "0",
+    DATABASE_PATH: ":memory:",
+    APP_TIMEZONE: "America/Chicago",
+    S3_ENDPOINT: "http://private-seaweedfs:8333",
+    S3_BUCKET: "announcements-test",
+    UPLOAD_TEMP_DIRECTORY: tempDirectory,
+    UPLOAD_MAX_FILE_BYTES: String(maxFileBytes),
+    UPLOAD_MAX_FILES: "5",
+    IMAGE_MAX_INPUT_PIXELS: "1000000",
+  });
+  const database = openDatabase({ path: ":memory:" });
+  const migrations = runMigrations(database);
+  const storage = memoryStorage();
+  const app = createApp({
+    config,
+    database,
+    imageProcessor: createImageProcessor({
+      maxInputPixels: config.processing.maxInputPixels,
+    }),
+    lifecycle: { ready: true },
+    migrations,
+    storage,
+  });
+  const server = app.listen(0, "127.0.0.1");
+  await new Promise((resolve, reject) => {
+    server.once("listening", resolve);
+    server.once("error", reject);
+  });
+
+  t.after(async () => {
+    await new Promise((resolve) => server.close(resolve));
+    database.close();
+    fs.rmSync(tempDirectory, { recursive: true, force: true });
+  });
+
+  return {
+    baseUrl: `http://127.0.0.1:${server.address().port}`,
+    database,
+    storage,
+    tempDirectory,
+  };
+}
+
+function appendFile(form, field, buffer, filename, contentType) {
+  form.append(field, new Blob([buffer], { type: contentType }), filename);
+}
+
+async function jsonResponse(baseUrl, route, init) {
+  const response = await fetch(`${baseUrl}${route}`, init);
+  return { response, body: await response.json() };
+}
+
+async function eventuallyEmpty(directory) {
+  for (let attempt = 0; attempt < 50; attempt += 1) {
+    if (fs.readdirSync(directory).length === 0) return;
+    await new Promise((resolve) => setTimeout(resolve, 10));
+  }
+  assert.deepEqual(fs.readdirSync(directory), []);
+}
+
+async function eventuallyContainsFile(directory) {
+  for (let attempt = 0; attempt < 50; attempt += 1) {
+    if (fs.readdirSync(directory).length > 0) return;
+    await new Promise((resolve) => setTimeout(resolve, 10));
+  }
+  assert.fail("The upload did not create its temporary file");
+}
+
+async function sampleImages() {
+  const source = sharp({
+    create: {
+      width: 80,
+      height: 60,
+      channels: 4,
+      background: { r: 31, g: 72, b: 120, alpha: 0.5 },
+    },
+  });
+  return {
+    jpeg: await source.clone().jpeg().toBuffer(),
+    png: await source.clone().png().toBuffer(),
+    webp: await source.clone().webp().toBuffer(),
+  };
+}
+
+test("asset HTTP API accepts JPEG, PNG, and WebP and serves private immutable media", async (t) => {
+  const harness = await createHarness(t);
+  const images = await sampleImages();
+  const form = new FormData();
+  appendFile(form, "files", images.jpeg, "welcome.jpg", "image/jpeg");
+  appendFile(form, "files", images.png, "../../bulletin.png", "image/png");
+  appendFile(form, "files", images.webp, "events.webp", "image/webp");
+
+  const uploaded = await jsonResponse(harness.baseUrl, "/api/assets", {
+    method: "POST",
+    body: form,
+  });
+  assert.equal(uploaded.response.status, 201);
+  assert.equal(uploaded.body.createdCount, 3);
+  assert.equal(uploaded.body.duplicateCount, 0);
+  assert.equal(uploaded.body.results.length, 3);
+  assert.deepEqual(
+    uploaded.body.results.map((result) => result.asset.original.contentType),
+    ["image/jpeg", "image/png", "image/webp"],
+  );
+  assert.equal(uploaded.body.results[1].asset.originalFilename, "bulletin.png");
+  assert.doesNotMatch(JSON.stringify(uploaded.body), /objectKey|object_key|private-seaweedfs/);
+  assert.equal(harness.storage.objects.size, 9);
+  await eventuallyEmpty(harness.tempDirectory);
+
+  const asset = uploaded.body.results[0].asset;
+  const listed = await jsonResponse(harness.baseUrl, "/api/assets");
+  assert.equal(listed.response.status, 200);
+  assert.equal(listed.body.assets.length, 3);
+  assert.deepEqual(
+    (await jsonResponse(harness.baseUrl, `/api/assets/${asset.id}`)).body,
+    asset,
+  );
+
+  for (const media of [asset.display, asset.thumbnail]) {
+    const response = await fetch(`${harness.baseUrl}${media.url}`);
+    const body = Buffer.from(await response.arrayBuffer());
+    assert.equal(response.status, 200);
+    assert.equal(response.headers.get("content-type"), "image/webp");
+    assert.equal(
+      response.headers.get("cache-control"),
+      "public, max-age=31536000, immutable",
+    );
+    assert.equal(response.headers.get("content-length"), String(body.length));
+    assert.equal(response.headers.get("etag"), `"${media.sha256}"`);
+    assert.equal(crypto.createHash("sha256").update(body).digest("hex"), media.sha256);
+  }
+
+  const original = await fetch(`${harness.baseUrl}${asset.original.url}`);
+  assert.equal(original.status, 200);
+  assert.equal(original.headers.get("content-type"), "image/jpeg");
+  assert.equal(original.headers.get("cache-control"), "private, no-store");
+  assert.match(original.headers.get("content-disposition"), /^attachment;/);
+  assert.deepEqual(Buffer.from(await original.arrayBuffer()), images.jpeg);
+
+  const wrongVersion = await jsonResponse(
+    harness.baseUrl,
+    `/media/assets/${asset.id}/display/${"a".repeat(64)}.webp`,
+  );
+  assert.equal(wrongVersion.response.status, 404);
+  assert.equal(wrongVersion.body.error.code, "MEDIA_NOT_FOUND");
+  const arbitraryKey = await jsonResponse(
+    harness.baseUrl,
+    `/media/assets/${asset.id}/display/${encodeURIComponent("../../original/key")}.webp`,
+  );
+  assert.equal(arbitraryKey.response.status, 404);
+  assert.equal(arbitraryKey.body.error.code, "MEDIA_NOT_FOUND");
+
+  const displayKey = [...harness.storage.objects.keys()].find(
+    (key) => key.includes(`assets/${asset.id}/display/`),
+  );
+  harness.storage.objects.delete(displayKey);
+  const missingStoredBytes = await jsonResponse(
+    harness.baseUrl,
+    asset.display.url,
+  );
+  assert.equal(missingStoredBytes.response.status, 503);
+  assert.equal(missingStoredBytes.body.error.code, "STORAGE_INCONSISTENCY");
+  assert.equal(
+    (await jsonResponse(harness.baseUrl, "/health/live")).response.status,
+    200,
+  );
+});
+
+test("asset HTTP API reports duplicates, name conflicts, and persists editable metadata", async (t) => {
+  const harness = await createHarness(t);
+  const { png, jpeg } = await sampleImages();
+  const firstForm = new FormData();
+  appendFile(firstForm, "file", png, "weekly.png", "application/octet-stream");
+  firstForm.append("display_name", "Weekly Announcement");
+  const first = await jsonResponse(harness.baseUrl, "/api/assets", {
+    method: "POST",
+    body: firstForm,
+  });
+  assert.equal(first.response.status, 201);
+  assert.equal(first.body.created, true);
+  const assetId = first.body.asset.id;
+
+  const duplicateForm = new FormData();
+  appendFile(duplicateForm, "file", png, "unrelated-name.png", "image/png");
+  duplicateForm.append("display_name", "Ignored duplicate proposal");
+  const duplicate = await jsonResponse(harness.baseUrl, "/api/assets", {
+    method: "POST",
+    body: duplicateForm,
+  });
+  assert.equal(duplicate.response.status, 200);
+  assert.equal(duplicate.body.created, false);
+  assert.equal(duplicate.body.duplicate, true);
+  assert.equal(duplicate.body.asset.id, assetId);
+  assert.equal(harness.storage.objects.size, 3);
+
+  const conflictForm = new FormData();
+  appendFile(conflictForm, "file", jpeg, "different.jpg", "image/jpeg");
+  conflictForm.append("displayName", "weekly announcement");
+  const conflict = await jsonResponse(harness.baseUrl, "/api/assets", {
+    method: "POST",
+    body: conflictForm,
+  });
+  assert.equal(conflict.response.status, 409);
+  assert.equal(conflict.body.error.code, "ASSET_NAME_CONFLICT");
+
+  const expiresAt = Date.parse("2034-01-02T03:04:05.000Z");
+  const edited = await jsonResponse(harness.baseUrl, `/api/assets/${assetId}`, {
+    method: "PATCH",
+    headers: { "content-type": "application/json" },
+    body: JSON.stringify({
+      display_name: "Renamed announcement",
+      expiration_at: expiresAt,
+    }),
+  });
+  assert.equal(edited.response.status, 200);
+  assert.equal(edited.body.displayName, "Renamed announcement");
+  assert.equal(edited.body.expirationAt, expiresAt);
+  const permanent = await jsonResponse(harness.baseUrl, `/api/assets/${assetId}`, {
+    method: "PATCH",
+    headers: { "content-type": "application/json" },
+    body: JSON.stringify({ expiration_at: null }),
+  });
+  assert.equal(permanent.body.expirationAt, null);
+  assert.equal(harness.storage.objects.size, 3);
+  await eventuallyEmpty(harness.tempDirectory);
+
+  const timestamp = Date.now();
+  harness.database
+    .prepare(
+      "INSERT INTO playlists (id, name, created_at, updated_at) VALUES (?, ?, ?, ?)",
+    )
+    .run("api-playlist", "API playlist", timestamp, timestamp);
+  harness.database
+    .prepare(
+      `INSERT INTO playlist_entries
+        (id, playlist_id, asset_id, position, created_at, updated_at)
+       VALUES (?, ?, ?, 0, ?, ?)`,
+    )
+    .run("api-entry", "api-playlist", assetId, timestamp, timestamp);
+  const references = await jsonResponse(
+    harness.baseUrl,
+    `/api/assets/${assetId}/references`,
+  );
+  assert.equal(references.response.status, 200);
+  assert.equal(references.body.canDelete, false);
+  assert.equal(references.body.playlists[0].playlistEntryId, "api-entry");
+  const blockedDelete = await jsonResponse(
+    harness.baseUrl,
+    `/api/assets/${assetId}`,
+    { method: "DELETE" },
+  );
+  assert.equal(blockedDelete.response.status, 409);
+  assert.equal(blockedDelete.body.error.code, "ASSET_REFERENCED");
+  assert.equal(harness.storage.objects.size, 3);
+  harness.database
+    .prepare("DELETE FROM playlist_entries WHERE id = ?")
+    .run("api-entry");
+
+  const removed = await jsonResponse(harness.baseUrl, `/api/assets/${assetId}`, {
+    method: "DELETE",
+  });
+  assert.equal(removed.response.status, 200);
+  assert.deepEqual(removed.body, { id: assetId, deleted: true });
+  assert.equal(harness.storage.objects.size, 0);
+  assert.equal(
+    (await jsonResponse(harness.baseUrl, `/api/assets/${assetId}`)).response.status,
+    404,
+  );
+});
+
+test("upload validation returns structured errors and always cleans temporary files", async (t) => {
+  const harness = await createHarness(t, { maxFileBytes: 1024 });
+
+  const malformed = new FormData();
+  appendFile(malformed, "file", Buffer.from("not an image"), "broken.png", "image/png");
+  const malformedResult = await jsonResponse(harness.baseUrl, "/api/assets", {
+    method: "POST",
+    body: malformed,
+  });
+  assert.equal(malformedResult.response.status, 422);
+  assert.equal(malformedResult.body.error.code, "MALFORMED_IMAGE");
+  await eventuallyEmpty(harness.tempDirectory);
+
+  const unsupportedBuffer = await sharp({
+    create: {
+      width: 20,
+      height: 20,
+      channels: 3,
+      background: "red",
+    },
+  })
+    .gif()
+    .toBuffer();
+  const unsupported = new FormData();
+  appendFile(unsupported, "file", unsupportedBuffer, "still.gif", "image/png");
+  const unsupportedResult = await jsonResponse(harness.baseUrl, "/api/assets", {
+    method: "POST",
+    body: unsupported,
+  });
+  assert.equal(unsupportedResult.response.status, 415);
+  assert.equal(unsupportedResult.body.error.code, "UNSUPPORTED_IMAGE_FORMAT");
+  await eventuallyEmpty(harness.tempDirectory);
+
+  const oversized = new FormData();
+  appendFile(
+    oversized,
+    "file",
+    crypto.randomBytes(1025),
+    "too-large.png",
+    "image/png",
+  );
+  const oversizedResult = await jsonResponse(harness.baseUrl, "/api/assets", {
+    method: "POST",
+    body: oversized,
+  });
+  assert.equal(oversizedResult.response.status, 413);
+  assert.equal(oversizedResult.body.error.code, "UPLOAD_FILE_TOO_LARGE");
+  assert.deepEqual(oversizedResult.body.error.details, { maxFileBytes: 1024 });
+  await eventuallyEmpty(harness.tempDirectory);
+
+  const tooMany = new FormData();
+  for (let index = 0; index < 6; index += 1) {
+    appendFile(tooMany, "files", Buffer.from("x"), `${index}.png`, "image/png");
+  }
+  const tooManyResult = await jsonResponse(harness.baseUrl, "/api/assets", {
+    method: "POST",
+    body: tooMany,
+  });
+  assert.equal(tooManyResult.response.status, 413);
+  assert.equal(tooManyResult.body.error.code, "UPLOAD_FILE_COUNT_EXCEEDED");
+  assert.deepEqual(tooManyResult.body.error.details, { maxFiles: 5 });
+  await eventuallyEmpty(harness.tempDirectory);
+
+  const absent = await jsonResponse(harness.baseUrl, "/api/assets", {
+    method: "POST",
+    body: new FormData(),
+  });
+  assert.equal(absent.response.status, 400);
+  assert.equal(absent.body.error.code, "UPLOAD_FILE_REQUIRED");
+  assert.equal(harness.storage.objects.size, 0);
+  assert.equal(
+    harness.database.prepare("SELECT COUNT(*) AS count FROM assets").get().count,
+    0,
+  );
+});
+
+test("an aborted multipart request removes its in-flight temporary file", async (t) => {
+  const harness = await createHarness(t);
+  const boundary = `announcements-${crypto.randomUUID()}`;
+  const preamble = Buffer.from(
+    `--${boundary}\r\n` +
+      'Content-Disposition: form-data; name="file"; filename="aborted.png"\r\n' +
+      "Content-Type: image/png\r\n\r\n",
+  );
+  const declaredLength = preamble.length + 512 * 1024;
+  const request = http.request(`${harness.baseUrl}/api/assets`, {
+    method: "POST",
+    headers: {
+      "content-type": `multipart/form-data; boundary=${boundary}`,
+      "content-length": String(declaredLength),
+    },
+  });
+  request.on("error", () => {});
+  request.write(preamble);
+  request.write(Buffer.alloc(64 * 1024, 1));
+
+  await eventuallyContainsFile(harness.tempDirectory);
+  const closed = new Promise((resolve) => request.once("close", resolve));
+  request.destroy();
+  await closed;
+  await eventuallyEmpty(harness.tempDirectory);
+  assert.equal(harness.storage.objects.size, 0);
+  assert.equal(
+    harness.database.prepare("SELECT COUNT(*) AS count FROM assets").get().count,
+    0,
+  );
+});