Implement Phase 2 asset storage pipeline

This commit is contained in:
Media
2026-08-16 19:59:10 -05:00
parent eb19675a5c
commit 722c6cf77d
29 changed files with 5059 additions and 16 deletions
+1
View File
@@ -6,6 +6,7 @@
.env.* .env.*
!.env.example !.env.example
**/node_modules **/node_modules
.pnpm-store
**/dist **/dist
**/*.log **/*.log
**/*.sqlite **/*.sqlite
+13 -1
View File
@@ -25,8 +25,20 @@ S3_REGION=us-east-1
S3_BUCKET=announcements-v2 S3_BUCKET=announcements-v2
S3_ACCESS_KEY_ID=replace-with-a-random-access-key S3_ACCESS_KEY_ID=replace-with-a-random-access-key
S3_SECRET_ACCESS_KEY=replace-with-a-long-random-secret S3_SECRET_ACCESS_KEY=replace-with-a-long-random-secret
S3_CONNECT_TIMEOUT_MS=2000
# Maximum socket inactivity during one S3 operation (not a total upload deadline).
S3_REQUEST_TIMEOUT_MS=5000
S3_MAX_ATTEMPTS=2
S3_HEALTH_PROBE_INTERVAL_MS=30000
# Optional integrations. Leave blank to disable them during Phase 1. # Image upload and decode limits. The default proxy body limit is 512 MiB,
# which accommodates the default twenty 25 MiB files plus multipart overhead.
UPLOAD_MAX_FILE_BYTES=26214400
UPLOAD_MAX_FILES=20
IMAGE_MAX_INPUT_PIXELS=40000000
# UPLOAD_TEMP_DIRECTORY=/tmp/announcements-uploads
# Optional integrations. Leave blank to disable them during Phase 2.
MQTT_URL= MQTT_URL=
MQTT_USERNAME= MQTT_USERNAME=
MQTT_PASSWORD= MQTT_PASSWORD=
+1
View File
@@ -1,4 +1,5 @@
node_modules/ node_modules/
.pnpm-store/
# Local configuration and secrets # Local configuration and secrets
.env .env
+93
View File
@@ -0,0 +1,93 @@
# Announcements v2
Announcements is a small Express, React, Socket.IO, MQTT, SQLite, and
S3-compatible signage application. Version 2 is being built in phases and is
intended to run beside the existing production version until cutover.
## Development
The repository is a pnpm workspace with one lockfile. It requires Node.js
22.13 or newer and pnpm 10.29.3.
```sh
cp .env.example .env
pnpm install --frozen-lockfile
pnpm check
docker compose up --build
```
Only the Nginx web service publishes a host port (5175 by default). The API and
SeaweedFS S3 endpoint remain private on Compose networks. Replace the example
S3 credentials before starting the stack.
## Library assets
`POST /api/assets` accepts multipart uploads in the repeated `file` or `files`
field. An optional `display_name` (or `displayName`) is accepted for a
single-file upload. Supported decoded formats are still JPEG, PNG, and WebP;
extensions and multipart content types are not trusted. Animated, multipage,
malformed, unsupported, oversized, and excessive-pixel inputs are rejected.
Default limits are:
- 25 MiB per file (`UPLOAD_MAX_FILE_BYTES`)
- 20 files per request (`UPLOAD_MAX_FILES`)
- 40 million decoded pixels per image (`IMAGE_MAX_INPUT_PIXELS`)
Nginx accepts up to 512 MiB per request, enough for the default aggregate plus
multipart overhead. If the backend limits are raised, update
`client_max_body_size` in `client/nginx.conf` as well. Proxied API requests use
a ten-minute inactivity timeout so a valid large batch can finish processing.
Original bytes are archived unchanged. Content identity is the SHA-256 of
those bytes, so uploading the same content returns the existing asset even
when the filename differs. Display-name collisions with different content
return `409 ASSET_NAME_CONFLICT`. Multi-file uploads validate every new image
locally before persisting any of them. If a later storage operation fails,
already completed items remain valid and retrying the request is idempotent.
Sharp produces two metadata-free, auto-oriented WebP renditions:
- display: inside-fit at most 1920×1080, quality 82
- thumbnail: inside-fit at most 480×270, quality 72
Neither rendition crops or upscales. Transparency is flattened against black.
Presentation effects such as blur, shadows, darkening, and framing are not
baked into stored media.
## Asset and media API
- `GET /api/assets`
- `POST /api/assets`
- `GET /api/assets/:assetId`
- `PATCH /api/assets/:assetId`
- `GET /api/assets/:assetId/references`
- `DELETE /api/assets/:assetId`
- `GET /api/assets/:assetId/original`
- `GET /media/assets/:assetId/display/:sha256.webp`
- `GET /media/assets/:assetId/thumbnail/:sha256.webp`
PATCH accepts only display name and expiration. A null expiration is permanent.
Manual deletion refuses playlist, current playback, and static references. It
marks an asset as deleting before idempotently removing its three objects and
removes SQLite metadata only after storage succeeds. A failed processing row
can be inspected, safely deleted, and uploaded again; no automatic retry loop
is run.
Media URLs are derived from SQLite metadata and never accept object keys. The
display and thumbnail routes stream private S3 objects through the API with a
one-year immutable cache policy. Original downloads use private, no-store
caching.
## Storage and health
The API uses a small path-style S3 adapter configured by `S3_ENDPOINT`,
`S3_REGION`, `S3_BUCKET`, `S3_ACCESS_KEY_ID`, and `S3_SECRET_ACCESS_KEY`.
Connection timeout, socket-inactivity timeout (`S3_REQUEST_TIMEOUT_MS`),
attempts, and background probe interval are configurable in `.env.example`.
Startup verifies or safely creates only the configured bucket.
Storage outages do not stop the API or fail process readiness. `/health/ready`
reports storage as degraded while remaining HTTP 200 when SQLite and the API
runtime are healthy. `/api/health` and `/api/status` expose cached, redacted
storage state and recover after a successful probe or storage operation.
+8
View File
@@ -10,6 +10,9 @@ server {
root /usr/share/nginx/html; root /usr/share/nginx/html;
index index.html; index index.html;
# Matches the default API allowance of twenty 25 MiB files plus multipart
# overhead. Keep this in sync if the backend upload limits are raised.
client_max_body_size 512m;
resolver 127.0.0.11 valid=10s ipv6=off; resolver 127.0.0.11 valid=10s ipv6=off;
set $api_origin http://api:7532; set $api_origin http://api:7532;
@@ -17,6 +20,11 @@ server {
location ~ ^/(api|health|media)(/|$) { location ~ ^/(api|health|media)(/|$) {
proxy_pass $api_origin; proxy_pass $api_origin;
proxy_http_version 1.1; proxy_http_version 1.1;
proxy_connect_timeout 5s;
# Multipart requests return only after image processing and private-S3
# writes complete. This is an inactivity timeout, not a size limit.
proxy_read_timeout 10m;
proxy_send_timeout 10m;
proxy_set_header Host $host; proxy_set_header Host $host;
proxy_set_header X-Real-IP $remote_addr; proxy_set_header X-Real-IP $remote_addr;
proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
+8
View File
@@ -42,6 +42,14 @@ services:
S3_ACCESS_KEY_ID: "${S3_ACCESS_KEY_ID:?Set S3_ACCESS_KEY_ID in .env}" S3_ACCESS_KEY_ID: "${S3_ACCESS_KEY_ID:?Set S3_ACCESS_KEY_ID in .env}"
S3_SECRET_ACCESS_KEY: "${S3_SECRET_ACCESS_KEY:?Set S3_SECRET_ACCESS_KEY in .env}" S3_SECRET_ACCESS_KEY: "${S3_SECRET_ACCESS_KEY:?Set S3_SECRET_ACCESS_KEY in .env}"
S3_FORCE_PATH_STYLE: "true" S3_FORCE_PATH_STYLE: "true"
S3_CONNECT_TIMEOUT_MS: "${S3_CONNECT_TIMEOUT_MS:-2000}"
S3_REQUEST_TIMEOUT_MS: "${S3_REQUEST_TIMEOUT_MS:-5000}"
S3_MAX_ATTEMPTS: "${S3_MAX_ATTEMPTS:-2}"
S3_HEALTH_PROBE_INTERVAL_MS: "${S3_HEALTH_PROBE_INTERVAL_MS:-30000}"
UPLOAD_TEMP_DIRECTORY: /tmp/announcements-uploads
UPLOAD_MAX_FILE_BYTES: "${UPLOAD_MAX_FILE_BYTES:-26214400}"
UPLOAD_MAX_FILES: "${UPLOAD_MAX_FILES:-20}"
IMAGE_MAX_INPUT_PIXELS: "${IMAGE_MAX_INPUT_PIXELS:-40000000}"
MQTT_URL: "${MQTT_URL:-}" MQTT_URL: "${MQTT_URL:-}"
MQTT_USERNAME: "${MQTT_USERNAME:-}" MQTT_USERNAME: "${MQTT_USERNAME:-}"
MQTT_PASSWORD: "${MQTT_PASSWORD:-}" MQTT_PASSWORD: "${MQTT_PASSWORD:-}"
+752
View File
File diff suppressed because it is too large Load Diff
+1
View File
@@ -4,3 +4,4 @@ packages:
onlyBuiltDependencies: onlyBuiltDependencies:
- better-sqlite3 - better-sqlite3
- sharp
+3
View File
@@ -13,9 +13,12 @@
"node": ">=22.13.0" "node": ">=22.13.0"
}, },
"dependencies": { "dependencies": {
"@aws-sdk/client-s3": "3.1101.0",
"better-sqlite3": "13.0.2", "better-sqlite3": "13.0.2",
"dotenv": "^16.5.0", "dotenv": "^16.5.0",
"express": "^5.2.1", "express": "^5.2.1",
"multer": "2.0.2",
"sharp": "0.35.3",
"socket.io": "^4.8.3" "socket.io": "^4.8.3"
} }
} }
+23 -2
View File
@@ -3,12 +3,27 @@ const {
createDetailedHealthRouter, createDetailedHealthRouter,
createHealthRouter, createHealthRouter,
} = require("./health/routes"); } = require("./health/routes");
const { createAssetRouter } = require("./assets/routes");
const { createMediaRouter } = require("./assets/media-routes");
const { createAssetService } = require("./assets/service");
const { createSettingsRouter } = require("./settings/routes"); const { createSettingsRouter } = require("./settings/routes");
const { createSettingsService } = require("./settings/service"); const { createSettingsService } = require("./settings/service");
function createApp({ config, database, lifecycle, migrations }) { function createApp({
config,
database,
imageProcessor,
lifecycle,
migrations,
storage,
}) {
const app = express(); const app = express();
const settingsService = createSettingsService(database); const settingsService = createSettingsService(database);
const assetService = createAssetService({
database,
imageProcessor,
storage,
});
app.disable("x-powered-by"); app.disable("x-powered-by");
app.set("trust proxy", config.trustProxy); app.set("trust proxy", config.trustProxy);
@@ -19,13 +34,14 @@ function createApp({ config, database, lifecycle, migrations }) {
}); });
app.use( app.use(
"/health", "/health",
createHealthRouter({ database, lifecycle, migrations }), createHealthRouter({ database, lifecycle, migrations, storage }),
); );
const detailedHealthRouterOptions = { const detailedHealthRouterOptions = {
config, config,
database, database,
lifecycle, lifecycle,
migrations, migrations,
storage,
}; };
app.use( app.use(
"/api/health", "/api/health",
@@ -36,6 +52,11 @@ function createApp({ config, database, lifecycle, migrations }) {
createDetailedHealthRouter(detailedHealthRouterOptions), createDetailedHealthRouter(detailedHealthRouterOptions),
); );
app.use("/api/settings", createSettingsRouter(settingsService)); app.use("/api/settings", createSettingsRouter(settingsService));
app.use(
"/api/assets",
createAssetRouter({ assetService, processingConfig: config.processing }),
);
app.use("/media", createMediaRouter(assetService));
app.use((request, response) => { app.use((request, response) => {
response.status(404).json({ response.status(404).json({
+23
View File
@@ -0,0 +1,23 @@
class AssetError extends Error {
constructor(code, message, { statusCode = 400, details, cause } = {}) {
super(message, cause ? { cause } : undefined);
this.name = "AssetError";
this.code = code;
this.statusCode = statusCode;
if (details !== undefined) {
this.details = details;
}
}
}
function assetNotFound(assetId) {
return new AssetError("ASSET_NOT_FOUND", "Library asset not found", {
statusCode: 404,
details: { assetId },
});
}
module.exports = {
AssetError,
assetNotFound,
};
+49
View File
@@ -0,0 +1,49 @@
const express = require("express");
const { sendImmutableMedia } = require("./streaming");
function createMediaRouter(assetService) {
const router = express.Router();
async function sendMedia(request, response, next, kind) {
const version = request.params.version;
if (!version.endsWith(".webp")) {
response.status(404).json({
error: {
code: "MEDIA_NOT_FOUND",
message: "Asset media version not found",
},
});
return;
}
try {
const descriptor = assetService.mediaDescriptor(
request.params.assetId,
kind,
version.slice(0, -".webp".length),
);
await sendImmutableMedia(assetService, response, descriptor);
} catch (error) {
if (response.headersSent) {
response.destroy(error);
return;
}
next(error);
}
}
router.get("/assets/:assetId/display/:version", (request, response, next) =>
sendMedia(request, response, next, "display"),
);
router.get(
"/assets/:assetId/thumbnail/:version",
(request, response, next) =>
sendMedia(request, response, next, "thumbnail"),
);
return router;
}
module.exports = {
createMediaRouter,
};
+235
View File
@@ -0,0 +1,235 @@
const crypto = require("node:crypto");
const fs = require("node:fs");
const path = require("node:path");
const multer = require("multer");
const { AssetError } = require("./errors");
const ALLOWED_FILE_FIELDS = new Set(["file", "files"]);
const ALLOWED_TEXT_FIELDS = new Set(["display_name", "displayName"]);
const MAX_FIELD_BYTES = 4 * 1024;
const MAX_ORIGINAL_FILENAME_LENGTH = 255;
const TEMPORARY_UPLOAD_PATHS = Symbol("temporaryUploadPaths");
const TEMP_STORAGE_ERROR_CODES = new Set([
"EACCES",
"EEXIST",
"EIO",
"EMFILE",
"ENFILE",
"ENOSPC",
"ENOTDIR",
"EPERM",
"EROFS",
]);
function safeOriginalFilename(value) {
const normalized = String(value || "")
.replaceAll("\\", "/")
.replace(/[\u0000-\u001f\u007f]/g, "");
const leafName = path.posix.basename(normalized);
const bounded = [...leafName].slice(0, MAX_ORIGINAL_FILENAME_LENGTH).join("");
if (!bounded.trim()) {
throw new AssetError(
"INVALID_ORIGINAL_FILENAME",
"Uploaded files must include a usable filename",
);
}
return bounded;
}
function mapMulterError(error, config) {
if (!(error instanceof multer.MulterError)) {
if (TEMP_STORAGE_ERROR_CODES.has(error?.code)) {
return new AssetError(
"UPLOAD_TEMP_STORAGE_UNAVAILABLE",
"Temporary upload storage is unavailable",
{ statusCode: 503, cause: error },
);
}
return new AssetError(
"INVALID_MULTIPART_UPLOAD",
"The multipart upload could not be read",
{ cause: error },
);
}
if (error.code === "LIMIT_FILE_SIZE") {
return new AssetError(
"UPLOAD_FILE_TOO_LARGE",
"An uploaded file exceeds the configured size limit",
{
statusCode: 413,
details: { maxFileBytes: config.maxFileBytes },
},
);
}
if (error.code === "LIMIT_FILE_COUNT") {
return new AssetError(
"UPLOAD_FILE_COUNT_EXCEEDED",
"The upload contains too many files",
{
statusCode: 413,
details: { maxFiles: config.maxFiles },
},
);
}
if (
[
"LIMIT_FIELD_COUNT",
"LIMIT_FIELD_KEY",
"LIMIT_FIELD_VALUE",
"LIMIT_PART_COUNT",
"LIMIT_UNEXPECTED_FILE",
].includes(error.code)
) {
return new AssetError(
"INVALID_MULTIPART_UPLOAD",
"The multipart upload contains too many or invalid parts",
{ details: { reason: error.code } },
);
}
return new AssetError(
"INVALID_MULTIPART_UPLOAD",
"The multipart upload could not be read",
{ details: { reason: error.code } },
);
}
function createTemporaryStorage(tempDirectory) {
return multer.diskStorage({
destination(request, file, callback) {
fs.mkdir(tempDirectory, { recursive: true, mode: 0o700 }, (error) => {
callback(error, tempDirectory);
});
},
filename(request, file, callback) {
const temporaryName = crypto.randomUUID();
if (!request[TEMPORARY_UPLOAD_PATHS]) {
request[TEMPORARY_UPLOAD_PATHS] = [];
}
request[TEMPORARY_UPLOAD_PATHS].push(
path.join(tempDirectory, temporaryName),
);
callback(null, temporaryName);
},
});
}
function temporaryUploadFiles(request) {
const paths = new Set(request[TEMPORARY_UPLOAD_PATHS] || []);
for (const file of Array.isArray(request.files) ? request.files : []) {
if (file?.path) paths.add(file.path);
}
return [...paths].map((temporaryPath) => ({ path: temporaryPath }));
}
function createMultipartUpload(config) {
const middleware = multer({
storage: createTemporaryStorage(config.tempDirectory),
limits: {
fileSize: config.maxFileBytes,
files: config.maxFiles,
fields: 2,
parts: config.maxFiles + 2,
fieldNameSize: 64,
fieldSize: MAX_FIELD_BYTES,
headerPairs: 100,
},
}).any();
return function receiveAssetUpload(request, response) {
return new Promise((resolve, reject) => {
middleware(request, response, (error) => {
if (error) {
reject(mapMulterError(error, config));
return;
}
const files = Array.isArray(request.files) ? request.files : [];
try {
if (files.length === 0) {
throw new AssetError(
"UPLOAD_FILE_REQUIRED",
"At least one image file is required",
);
}
const unexpectedFile = files.find(
(file) => !ALLOWED_FILE_FIELDS.has(file.fieldname),
);
if (unexpectedFile) {
throw new AssetError(
"UNEXPECTED_UPLOAD_FIELD",
"Image files must use the file or files multipart field",
{ details: { field: unexpectedFile.fieldname } },
);
}
const body = request.body || {};
const unknownFields = Object.keys(body).filter(
(name) => !ALLOWED_TEXT_FIELDS.has(name),
);
if (unknownFields.length > 0) {
throw new AssetError(
"UNEXPECTED_UPLOAD_FIELD",
"The multipart upload contains unsupported fields",
{ details: { fields: unknownFields } },
);
}
if (
Object.hasOwn(body, "display_name") &&
Object.hasOwn(body, "displayName")
) {
throw new AssetError(
"DUPLICATE_DISPLAY_NAME_FIELD",
"Provide only one display name field",
);
}
const displayName = body.display_name ?? body.displayName ?? null;
if (displayName !== null && files.length !== 1) {
throw new AssetError(
"DISPLAY_NAME_REQUIRES_SINGLE_FILE",
"An explicit display name can be used only with one uploaded file",
);
}
resolve({
displayName,
files: files.map((file) => ({
path: file.path,
byteSize: file.size,
originalFilename: safeOriginalFilename(file.originalname),
suppliedContentType: file.mimetype,
})),
});
} catch (validationError) {
reject(validationError);
}
});
});
};
}
async function cleanupTemporaryFiles(files = []) {
await Promise.all(
files.map(async (file) => {
if (!file?.path) return;
try {
await fs.promises.rm(file.path, { force: true });
} catch {
// Temporary cleanup is best-effort and must not mask the API result.
}
}),
);
}
module.exports = {
cleanupTemporaryFiles,
createMultipartUpload,
mapMulterError,
safeOriginalFilename,
temporaryUploadFiles,
};
+102
View File
@@ -0,0 +1,102 @@
const express = require("express");
const {
cleanupTemporaryFiles,
createMultipartUpload,
temporaryUploadFiles,
} = require("./multipart");
const { sendOriginal } = require("./streaming");
function createAssetRouter({ assetService, processingConfig }) {
const router = express.Router();
const receiveUpload = createMultipartUpload(processingConfig);
router.get("/", (request, response, next) => {
try {
response.json({ assets: assetService.list() });
} catch (error) {
next(error);
}
});
router.post("/", async (request, response, next) => {
const cleanupAfterAbort = () => {
void cleanupTemporaryFiles(temporaryUploadFiles(request));
};
request.once("aborted", cleanupAfterAbort);
try {
const upload = await receiveUpload(request, response);
const results = await assetService.upload(upload.files, {
displayName: upload.displayName,
});
const createdCount = results.filter((result) => result.created).length;
if (results.length === 1) {
response.status(createdCount > 0 ? 201 : 200).json(results[0]);
return;
}
response.status(createdCount > 0 ? 201 : 200).json({
results,
createdCount,
duplicateCount: results.length - createdCount,
});
} catch (error) {
next(error);
} finally {
request.off("aborted", cleanupAfterAbort);
await cleanupTemporaryFiles(temporaryUploadFiles(request));
}
});
router.get("/:assetId/references", (request, response, next) => {
try {
response.json(assetService.references(request.params.assetId));
} catch (error) {
next(error);
}
});
router.get("/:assetId/original", async (request, response, next) => {
try {
const descriptor = assetService.originalDescriptor(request.params.assetId);
await sendOriginal(assetService, response, descriptor);
} catch (error) {
if (response.headersSent) {
response.destroy(error);
return;
}
next(error);
}
});
router.get("/:assetId", (request, response, next) => {
try {
response.json(assetService.get(request.params.assetId));
} catch (error) {
next(error);
}
});
router.patch("/:assetId", (request, response, next) => {
try {
response.json(assetService.update(request.params.assetId, request.body));
} catch (error) {
next(error);
}
});
router.delete("/:assetId", async (request, response, next) => {
try {
response.json(await assetService.remove(request.params.assetId));
} catch (error) {
next(error);
}
});
return router;
}
module.exports = {
createAssetRouter,
};
+835
View File
@@ -0,0 +1,835 @@
const crypto = require("node:crypto");
const fs = require("node:fs");
const path = require("node:path");
const { AssetError, assetNotFound } = require("./errors");
const { StorageObjectNotFoundError } = require("../storage");
const MAX_DISPLAY_NAME_LENGTH = 200;
const MAX_PROCESSING_ERROR_LENGTH = 1_000;
const CONCURRENT_UPLOAD_WAIT_MS = 15_000;
const CONCURRENT_UPLOAD_POLL_MS = 50;
const SHA256_PATTERN = /^[0-9a-f]{64}$/;
function delay(milliseconds) {
return new Promise((resolve) => setTimeout(resolve, milliseconds));
}
function normalizeDisplayName(value) {
if (typeof value !== "string") {
throw new AssetError(
"INVALID_ASSET_DISPLAY_NAME",
"Asset display name must be a string",
);
}
const normalized = value
.replace(/[\u0000-\u001f\u007f]/g, " ")
.replace(/\s+/g, " ")
.trim();
if (!normalized) {
throw new AssetError(
"INVALID_ASSET_DISPLAY_NAME",
"Asset display name cannot be empty",
);
}
if ([...normalized].length > MAX_DISPLAY_NAME_LENGTH) {
throw new AssetError(
"INVALID_ASSET_DISPLAY_NAME",
`Asset display name cannot exceed ${MAX_DISPLAY_NAME_LENGTH} characters`,
{ details: { maxLength: MAX_DISPLAY_NAME_LENGTH } },
);
}
return normalized;
}
function defaultDisplayName(originalFilename) {
const extension = path.extname(originalFilename);
const withoutExtension = extension
? originalFilename.slice(0, -extension.length)
: originalFilename;
return normalizeDisplayName(withoutExtension);
}
function normalizeExpiration(value) {
if (value === null) return null;
if (Number.isSafeInteger(value) && value >= 0) return value;
if (typeof value === "string" && value.trim()) {
const timestamp = Date.parse(value);
if (Number.isSafeInteger(timestamp) && timestamp >= 0) {
return timestamp;
}
}
throw new AssetError(
"INVALID_ASSET_EXPIRATION",
"Asset expiration must be null, an epoch-millisecond integer, or an ISO date",
);
}
function mediaUrls(row) {
if (row.status !== "ready") {
return {
originalUrl: null,
displayUrl: null,
thumbnailUrl: null,
};
}
return {
originalUrl: `/api/assets/${row.id}/original`,
displayUrl: `/media/assets/${row.id}/display/${row.display_sha256}.webp`,
thumbnailUrl: `/media/assets/${row.id}/thumbnail/${row.thumbnail_sha256}.webp`,
};
}
function mapAsset(row) {
if (!row) return null;
const urls = mediaUrls(row);
return {
id: row.id,
displayName: row.display_name,
originalFilename: row.original_filename,
status: row.status,
processingError: row.processing_error || null,
sourceSha256: row.source_sha256,
expirationAt: row.expiration_at ?? null,
createdAt: row.created_at,
updatedAt: row.updated_at,
referenceCount: Number(row.reference_count || 0),
original: row.original_content_type
? {
contentType: row.original_content_type,
byteSize: row.original_byte_size,
width: row.original_width,
height: row.original_height,
url: urls.originalUrl,
}
: null,
display: row.display_content_type
? {
sha256: row.display_sha256,
contentType: row.display_content_type,
byteSize: row.display_byte_size,
width: row.display_width,
height: row.display_height,
url: urls.displayUrl,
}
: null,
thumbnail: row.thumbnail_content_type
? {
sha256: row.thumbnail_sha256,
contentType: row.thumbnail_content_type,
byteSize: row.thumbnail_byte_size,
width: row.thumbnail_width,
height: row.thumbnail_height,
url: urls.thumbnailUrl,
}
: null,
};
}
async function hashFile(filename) {
const hash = crypto.createHash("sha256");
let byteSize = 0;
for await (const chunk of fs.createReadStream(filename)) {
byteSize += chunk.length;
hash.update(chunk);
}
return { sourceSha256: hash.digest("hex"), byteSize };
}
function objectKeys(assetId, processed) {
return {
original: `assets/${assetId}/original/${processed.original.sha256}`,
display: `assets/${assetId}/display/${processed.display.sha256}.webp`,
thumbnail: `assets/${assetId}/thumbnail/${processed.thumbnail.sha256}.webp`,
};
}
function boundedProcessingError(error) {
const code = typeof error?.code === "string" ? error.code : "PROCESSING_FAILED";
const message =
typeof error?.message === "string" && error.message
? error.message
: "Asset processing failed";
return `${code}: ${message}`.slice(0, MAX_PROCESSING_ERROR_LENGTH);
}
function createAssetService({
database,
storage,
imageProcessor,
now = Date.now,
createId = crypto.randomUUID,
wait = delay,
concurrentUploadWaitMs = CONCURRENT_UPLOAD_WAIT_MS,
}) {
const assetSelect = `
SELECT a.*,
(SELECT COUNT(*) FROM playlist_entries pe WHERE pe.asset_id = a.id)
AS reference_count
FROM assets a
`;
const listAssets = database.prepare(`${assetSelect} ORDER BY a.created_at DESC, a.id`);
const findById = database.prepare(`${assetSelect} WHERE a.id = ?`);
const findByHash = database.prepare(`${assetSelect} WHERE a.source_sha256 = ?`);
const findByName = database.prepare(`${assetSelect} WHERE a.display_name = ? COLLATE NOCASE`);
const insertProcessing = database.prepare(`
INSERT INTO assets (
id, source_sha256, display_name, original_filename, status,
created_at, updated_at
) VALUES (
@id, @sourceSha256, @displayName, @originalFilename, 'processing',
@createdAt, @updatedAt
)
`);
const markReady = database.prepare(`
UPDATE assets
SET status = 'ready',
processing_error = NULL,
original_object_key = @originalObjectKey,
original_content_type = @originalContentType,
original_byte_size = @originalByteSize,
original_width = @originalWidth,
original_height = @originalHeight,
display_object_key = @displayObjectKey,
display_sha256 = @displaySha256,
display_content_type = @displayContentType,
display_byte_size = @displayByteSize,
display_width = @displayWidth,
display_height = @displayHeight,
thumbnail_object_key = @thumbnailObjectKey,
thumbnail_sha256 = @thumbnailSha256,
thumbnail_content_type = @thumbnailContentType,
thumbnail_byte_size = @thumbnailByteSize,
thumbnail_width = @thumbnailWidth,
thumbnail_height = @thumbnailHeight,
updated_at = @updatedAt
WHERE id = @id AND status = 'processing'
`);
const markFailed = database.prepare(`
UPDATE assets
SET status = 'failed',
processing_error = @processingError,
original_object_key = @originalObjectKey,
original_content_type = @originalContentType,
original_byte_size = @originalByteSize,
original_width = @originalWidth,
original_height = @originalHeight,
display_object_key = @displayObjectKey,
display_sha256 = @displaySha256,
display_content_type = @displayContentType,
display_byte_size = @displayByteSize,
display_width = @displayWidth,
display_height = @displayHeight,
thumbnail_object_key = @thumbnailObjectKey,
thumbnail_sha256 = @thumbnailSha256,
thumbnail_content_type = @thumbnailContentType,
thumbnail_byte_size = @thumbnailByteSize,
thumbnail_width = @thumbnailWidth,
thumbnail_height = @thumbnailHeight,
updated_at = @updatedAt
WHERE id = @id AND status = 'processing'
`);
const updateEditable = database.prepare(`
UPDATE assets
SET display_name = @displayName,
expiration_at = @expirationAt,
updated_at = @updatedAt
WHERE id = @id AND status != 'deleting'
`);
const playlistReferences = database.prepare(`
SELECT
pe.id AS entry_id,
pe.playlist_id,
p.name AS playlist_name,
pe.enabled,
pe.position,
EXISTS (
SELECT 1 FROM playback_state active
WHERE active.active_playlist_id = pe.playlist_id
) AS is_active_playlist,
EXISTS (
SELECT 1 FROM playback_state current
WHERE current.current_entry_id = pe.id
) AS is_current_normal_target
FROM playlist_entries pe
JOIN playlists p ON p.id = pe.playlist_id
WHERE pe.asset_id = ?
ORDER BY p.name COLLATE NOCASE, pe.position, pe.id
`);
const playbackReferences = database.prepare(`
SELECT
output_id,
mode,
active_playlist_id,
current_entry_id,
static_asset_id
FROM playback_state
WHERE static_asset_id = ?
OR current_entry_id IN (
SELECT id FROM playlist_entries WHERE asset_id = ?
)
ORDER BY output_id
`);
const markDeleting = database.prepare(`
UPDATE assets
SET status = 'deleting', updated_at = ?
WHERE id = ? AND status != 'processing'
`);
const deleteAssetRow = database.prepare(
"DELETE FROM assets WHERE id = ? AND status = 'deleting'",
);
const completeAsset = database.transaction((metadata) => {
const result = markReady.run(metadata);
if (result.changes !== 1) {
throw new Error("Asset processing state changed before completion");
}
return findById.get(metadata.id);
});
function getRaw(assetId) {
const row = findById.get(assetId);
if (!row) throw assetNotFound(assetId);
return row;
}
function duplicateStateError(row) {
if (row.status === "failed") {
return new AssetError(
"ASSET_PROCESSING_FAILED",
"These bytes match an asset whose processing previously failed",
{ statusCode: 409, details: { asset: mapAsset(row) } },
);
}
if (row.status === "deleting") {
return new AssetError(
"ASSET_DELETING",
"These bytes match an asset that is being deleted",
{ statusCode: 409, details: { assetId: row.id } },
);
}
return new AssetError(
"ASSET_PROCESSING",
"These bytes are already being processed",
{ statusCode: 409, details: { assetId: row.id } },
);
}
async function resolveExistingHash(row) {
if (row.status === "ready") {
return { created: false, duplicate: true, asset: mapAsset(row) };
}
if (row.status !== "processing") {
throw duplicateStateError(row);
}
const deadline = Date.now() + concurrentUploadWaitMs;
while (Date.now() < deadline) {
await wait(CONCURRENT_UPLOAD_POLL_MS);
const latest = findByHash.get(row.source_sha256);
if (!latest) break;
if (latest.status === "ready") {
return { created: false, duplicate: true, asset: mapAsset(latest) };
}
if (latest.status !== "processing") {
throw duplicateStateError(latest);
}
}
throw duplicateStateError(findByHash.get(row.source_sha256) || row);
}
function nameConflict(displayName, row = findByName.get(displayName)) {
return new AssetError(
"ASSET_NAME_CONFLICT",
"Another library asset already uses that display name",
{
statusCode: 409,
details: row ? { asset: mapAsset(row) } : { displayName },
},
);
}
function processingMetadata(id, processed, keys, error) {
return {
id,
processingError: error ? boundedProcessingError(error) : null,
originalObjectKey: keys.original,
originalContentType: processed.original.contentType,
originalByteSize: processed.original.byteSize,
originalWidth: processed.original.width,
originalHeight: processed.original.height,
displayObjectKey: keys.display,
displaySha256: processed.display.sha256,
displayContentType: processed.display.contentType,
displayByteSize: processed.display.byteSize,
displayWidth: processed.display.width,
displayHeight: processed.display.height,
thumbnailObjectKey: keys.thumbnail,
thumbnailSha256: processed.thumbnail.sha256,
thumbnailContentType: processed.thumbnail.contentType,
thumbnailByteSize: processed.thumbnail.byteSize,
thumbnailWidth: processed.thumbnail.width,
thumbnailHeight: processed.thumbnail.height,
updatedAt: now(),
};
}
async function cleanObjects(keys) {
const results = await Promise.allSettled(
[...new Set(Object.values(keys).filter(Boolean))].map((key) =>
storage.deleteObject(key),
),
);
const rejected = results.find((result) => result.status === "rejected");
if (rejected) throw rejected.reason;
}
async function createProcessedAsset(plan) {
const id = createId();
const timestamp = now();
try {
insertProcessing.run({
id,
sourceSha256: plan.sourceSha256,
displayName: plan.displayName,
originalFilename: plan.file.originalFilename,
createdAt: timestamp,
updatedAt: timestamp,
});
} catch (error) {
const hashMatch = findByHash.get(plan.sourceSha256);
if (hashMatch) return resolveExistingHash(hashMatch);
const nameMatch = findByName.get(plan.displayName);
if (nameMatch) throw nameConflict(plan.displayName, nameMatch);
throw error;
}
const keys = objectKeys(id, plan.processed);
try {
await storage.putObject({
key: keys.original,
body: fs.createReadStream(plan.file.path),
contentType: plan.processed.original.contentType,
contentLength: plan.processed.original.byteSize,
metadata: { sha256: plan.processed.original.sha256 },
});
await storage.putObject({
key: keys.display,
body: plan.processed.display.buffer,
contentType: plan.processed.display.contentType,
contentLength: plan.processed.display.byteSize,
metadata: { sha256: plan.processed.display.sha256 },
});
await storage.putObject({
key: keys.thumbnail,
body: plan.processed.thumbnail.buffer,
contentType: plan.processed.thumbnail.contentType,
contentLength: plan.processed.thumbnail.byteSize,
metadata: { sha256: plan.processed.thumbnail.sha256 },
});
const row = completeAsset(processingMetadata(id, plan.processed, keys));
return { created: true, duplicate: false, asset: mapAsset(row) };
} catch (error) {
const failure = processingMetadata(id, plan.processed, keys, error);
try {
markFailed.run(failure);
} catch {
// Preserve the original failure; the processing row remains diagnosable.
}
try {
await cleanObjects(keys);
} catch {
// The failed row retains deterministic keys so DELETE can retry cleanup.
}
if (error?.statusCode) {
error.details = { ...(error.details || {}), assetId: id, retryable: true };
}
throw error;
}
}
async function upload(files, { displayName = null } = {}) {
if (!Array.isArray(files) || files.length === 0) {
throw new AssetError(
"UPLOAD_FILE_REQUIRED",
"At least one image file is required",
);
}
const plans = [];
const firstPlanByHash = new Map();
const plannedNames = new Map();
for (const [index, file] of files.entries()) {
const identity = await hashFile(file.path);
if (identity.byteSize !== file.byteSize) {
throw new AssetError(
"UPLOAD_CHANGED_DURING_PROCESSING",
"An uploaded temporary file changed before processing",
{ statusCode: 409 },
);
}
const existing = findByHash.get(identity.sourceSha256);
if (existing) {
plans.push({ index, existing });
continue;
}
const priorPlan = firstPlanByHash.get(identity.sourceSha256);
if (priorPlan) {
plans.push({ index, duplicateOf: priorPlan });
continue;
}
const proposedName = normalizeDisplayName(
displayName ?? defaultDisplayName(file.originalFilename),
);
const existingName = findByName.get(proposedName);
if (existingName) throw nameConflict(proposedName, existingName);
const normalizedNameKey = proposedName.toLowerCase();
const batchNameMatch = plannedNames.get(normalizedNameKey);
if (batchNameMatch) throw nameConflict(proposedName);
const plan = {
index,
file,
displayName: proposedName,
sourceSha256: identity.sourceSha256,
};
firstPlanByHash.set(identity.sourceSha256, plan);
plannedNames.set(normalizedNameKey, plan);
plans.push(plan);
}
// Decode every new image before any database or object-storage mutation.
for (const plan of plans) {
if (plan.existing || plan.duplicateOf) continue;
plan.processed = await imageProcessor.process(plan.file.path);
if (plan.processed.original.sha256 !== plan.sourceSha256) {
throw new AssetError(
"UPLOAD_CHANGED_DURING_PROCESSING",
"An uploaded temporary file changed before processing",
{ statusCode: 409 },
);
}
}
const resultByPlan = new Map();
const results = new Array(files.length);
for (const plan of plans) {
let result;
if (plan.existing) {
result = await resolveExistingHash(plan.existing);
} else if (plan.duplicateOf) {
const originalResult = resultByPlan.get(plan.duplicateOf);
result = {
created: false,
duplicate: true,
asset: originalResult.asset,
};
} else {
result = await createProcessedAsset(plan);
resultByPlan.set(plan, result);
}
results[plan.index] = result;
}
return results;
}
function list() {
return listAssets.all().map(mapAsset);
}
function get(assetId) {
return mapAsset(getRaw(assetId));
}
function update(assetId, input) {
if (input === null || typeof input !== "object" || Array.isArray(input)) {
throw new AssetError("INVALID_ASSET_UPDATE", "Asset update must be an object");
}
const allowed = new Set([
"displayName",
"display_name",
"expirationAt",
"expiration_at",
]);
const unknown = Object.keys(input).filter((key) => !allowed.has(key));
if (unknown.length > 0) {
throw new AssetError(
"INVALID_ASSET_UPDATE",
"Asset update contains unsupported fields",
{ details: { fields: unknown } },
);
}
if (Object.keys(input).length === 0) {
throw new AssetError("INVALID_ASSET_UPDATE", "At least one field is required");
}
if (
Object.hasOwn(input, "displayName") &&
Object.hasOwn(input, "display_name")
) {
throw new AssetError("INVALID_ASSET_UPDATE", "Display name was provided twice");
}
if (
Object.hasOwn(input, "expirationAt") &&
Object.hasOwn(input, "expiration_at")
) {
throw new AssetError("INVALID_ASSET_UPDATE", "Expiration was provided twice");
}
const current = getRaw(assetId);
if (current.status === "deleting") {
throw new AssetError("ASSET_DELETING", "Deleting assets cannot be edited", {
statusCode: 409,
details: { assetId },
});
}
const hasDisplayName =
Object.hasOwn(input, "displayName") || Object.hasOwn(input, "display_name");
const hasExpiration =
Object.hasOwn(input, "expirationAt") || Object.hasOwn(input, "expiration_at");
const nextDisplayName = hasDisplayName
? normalizeDisplayName(
Object.hasOwn(input, "displayName")
? input.displayName
: input.display_name,
)
: current.display_name;
const nextExpiration = hasExpiration
? normalizeExpiration(
Object.hasOwn(input, "expirationAt")
? input.expirationAt
: input.expiration_at,
)
: current.expiration_at;
if (
nextDisplayName === current.display_name &&
nextExpiration === current.expiration_at
) {
return mapAsset(current);
}
try {
const result = updateEditable.run({
id: assetId,
displayName: nextDisplayName,
expirationAt: nextExpiration,
updatedAt: Math.max(now(), current.updated_at + 1),
});
if (result.changes !== 1) {
throw new AssetError("ASSET_DELETING", "Deleting assets cannot be edited", {
statusCode: 409,
details: { assetId },
});
}
} catch (error) {
const nameMatch = findByName.get(nextDisplayName);
if (nameMatch && nameMatch.id !== assetId) {
throw nameConflict(nextDisplayName, nameMatch);
}
throw error;
}
return mapAsset(getRaw(assetId));
}
function references(assetId) {
const asset = getRaw(assetId);
const playlists = playlistReferences.all(assetId).map((row) => ({
playlistId: row.playlist_id,
playlistName: row.playlist_name,
playlistEntryId: row.entry_id,
enabled: Boolean(row.enabled),
position: row.position,
isActivePlaylist: Boolean(row.is_active_playlist),
isCurrentNormalTarget: Boolean(row.is_current_normal_target),
}));
const playback = playbackReferences.all(assetId, assetId).map((row) => ({
outputId: row.output_id,
mode: row.mode,
activePlaylistId: row.active_playlist_id,
currentPlaylistEntryId: row.current_entry_id,
isCurrentNormalTarget: playlists.some(
(entry) => entry.playlistEntryId === row.current_entry_id,
),
isCurrentStaticTarget: row.static_asset_id === assetId,
}));
const currentNormalTarget = playback.some((state) => state.isCurrentNormalTarget);
const currentStaticTarget = playback.some((state) => state.isCurrentStaticTarget);
const blockerCodes = [];
if (playlists.length > 0) blockerCodes.push("PLAYLIST_REFERENCES");
if (currentNormalTarget) blockerCodes.push("CURRENT_PLAYBACK_TARGET");
if (currentStaticTarget) blockerCodes.push("CURRENT_STATIC_TARGET");
if (asset.status === "processing") blockerCodes.push("ASSET_PROCESSING");
if (asset.status === "deleting") blockerCodes.push("ASSET_DELETING");
return {
assetId,
playlists,
playback,
currentNormalTarget,
currentStaticTarget,
canDelete: blockerCodes.length === 0,
blockerCodes,
};
}
const beginDelete = database.transaction((assetId) => {
const asset = getRaw(assetId);
const referenceInfo = references(assetId);
const relationshipBlockers = referenceInfo.blockerCodes.filter((code) =>
[
"PLAYLIST_REFERENCES",
"CURRENT_PLAYBACK_TARGET",
"CURRENT_STATIC_TARGET",
].includes(code),
);
if (relationshipBlockers.length > 0) {
throw new AssetError(
referenceInfo.playlists.length > 0 ? "ASSET_REFERENCED" : "ASSET_IN_USE",
"The library asset is still referenced and cannot be deleted",
{
statusCode: 409,
details: { ...referenceInfo, blockerCodes: relationshipBlockers },
},
);
}
if (asset.status === "processing") {
throw new AssetError(
"ASSET_PROCESSING",
"An asset cannot be deleted while it is processing",
{ statusCode: 409, details: { assetId } },
);
}
if (asset.status !== "deleting") {
markDeleting.run(Math.max(now(), asset.updated_at + 1), assetId);
}
return findById.get(assetId);
});
async function remove(assetId) {
const deleting = beginDelete(assetId);
const keys = {
original: deleting.original_object_key,
display: deleting.display_object_key,
thumbnail: deleting.thumbnail_object_key,
};
try {
await cleanObjects(keys);
} catch (error) {
if (error?.statusCode) {
error.details = { ...(error.details || {}), assetId, retryable: true };
}
throw error;
}
try {
const result = deleteAssetRow.run(assetId);
if (result.changes !== 1) throw assetNotFound(assetId);
} catch (error) {
if (error?.code?.startsWith?.("SQLITE_CONSTRAINT")) {
throw new AssetError(
"ASSET_REFERENCED",
"The library asset became referenced before deletion completed",
{ statusCode: 409, details: { assetId, retryable: true } },
);
}
throw error;
}
return { id: assetId, deleted: true };
}
function mediaDescriptor(assetId, kind, requestedHash) {
if (!SHA256_PATTERN.test(requestedHash)) {
throw new AssetError("MEDIA_NOT_FOUND", "Asset media version not found", {
statusCode: 404,
});
}
const asset = getRaw(assetId);
const fields =
kind === "display"
? {
hash: asset.display_sha256,
key: asset.display_object_key,
contentType: asset.display_content_type,
contentLength: asset.display_byte_size,
}
: {
hash: asset.thumbnail_sha256,
key: asset.thumbnail_object_key,
contentType: asset.thumbnail_content_type,
contentLength: asset.thumbnail_byte_size,
};
if (asset.status !== "ready" || fields.hash !== requestedHash || !fields.key) {
throw new AssetError("MEDIA_NOT_FOUND", "Asset media version not found", {
statusCode: 404,
});
}
return { asset, kind, ...fields };
}
function originalDescriptor(assetId) {
const asset = getRaw(assetId);
if (asset.status !== "ready" || !asset.original_object_key) {
throw new AssetError("ASSET_ORIGINAL_NOT_READY", "Asset original is not ready", {
statusCode: 409,
details: { assetId, status: asset.status },
});
}
return {
asset,
key: asset.original_object_key,
hash: asset.source_sha256,
contentType: asset.original_content_type,
contentLength: asset.original_byte_size,
};
}
async function readStoredObject(descriptor) {
try {
return await storage.getObject(descriptor.key);
} catch (error) {
if (error instanceof StorageObjectNotFoundError) {
throw new AssetError(
"STORAGE_INCONSISTENCY",
"Asset metadata exists but its stored bytes are unavailable",
{
statusCode: 503,
details: { assetId: descriptor.asset.id, retryable: true },
cause: error,
},
);
}
throw error;
}
}
return Object.freeze({
list,
get,
upload,
update,
references,
remove,
mediaDescriptor,
originalDescriptor,
readStoredObject,
});
}
module.exports = {
createAssetService,
defaultDisplayName,
mapAsset,
normalizeDisplayName,
normalizeExpiration,
};
+66
View File
@@ -0,0 +1,66 @@
const { Readable } = require("node:stream");
const { pipeline } = require("node:stream/promises");
const { AssetError } = require("./errors");
function readableBody(body) {
if (body && typeof body.pipe === "function") return body;
if (Buffer.isBuffer(body) || body instanceof Uint8Array) {
return Readable.from([body]);
}
if (body && typeof body.getReader === "function") {
return Readable.fromWeb(body);
}
throw new AssetError(
"STORAGE_INCONSISTENCY",
"Stored asset bytes could not be streamed",
{ statusCode: 503 },
);
}
function validateDescriptor(descriptor) {
if (
typeof descriptor.contentType !== "string" ||
!Number.isSafeInteger(descriptor.contentLength) ||
descriptor.contentLength < 0
) {
throw new AssetError(
"STORAGE_INCONSISTENCY",
"Stored asset metadata is incomplete",
{ statusCode: 503 },
);
}
}
function setCommonHeaders(response, descriptor) {
response.set("Content-Type", descriptor.contentType);
response.set("Content-Length", String(descriptor.contentLength));
response.set("X-Content-Type-Options", "nosniff");
}
async function storedBody(assetService, descriptor) {
const stored = await assetService.readStoredObject(descriptor);
return readableBody(stored.body);
}
async function sendImmutableMedia(assetService, response, descriptor) {
validateDescriptor(descriptor);
const body = await storedBody(assetService, descriptor);
setCommonHeaders(response, descriptor);
response.set("Cache-Control", "public, max-age=31536000, immutable");
response.set("ETag", `"${descriptor.hash}"`);
await pipeline(body, response);
}
async function sendOriginal(assetService, response, descriptor) {
validateDescriptor(descriptor);
const body = await storedBody(assetService, descriptor);
response.attachment(descriptor.asset.original_filename);
setCommonHeaders(response, descriptor);
response.set("Cache-Control", "private, no-store");
await pipeline(body, response);
}
module.exports = {
sendImmutableMedia,
sendOriginal,
};
+46
View File
@@ -156,6 +156,42 @@ function loadConfig(env = process.env) {
accessKeyId: storageCredential(env, "S3_ACCESS_KEY_ID", environment), accessKeyId: storageCredential(env, "S3_ACCESS_KEY_ID", environment),
secretAccessKey: storageCredential(env, "S3_SECRET_ACCESS_KEY", environment), secretAccessKey: storageCredential(env, "S3_SECRET_ACCESS_KEY", environment),
forcePathStyle: boolean(env, "S3_FORCE_PATH_STYLE", true), forcePathStyle: boolean(env, "S3_FORCE_PATH_STYLE", true),
healthProbeIntervalMs: integer(
env,
"S3_HEALTH_PROBE_INTERVAL_MS",
30_000,
{ min: 1_000, max: 300_000 },
),
connectionTimeoutMs: integer(env, "S3_CONNECT_TIMEOUT_MS", 2_000, {
min: 250,
max: 60_000,
}),
requestTimeoutMs: integer(env, "S3_REQUEST_TIMEOUT_MS", 5_000, {
min: 1_000,
max: 120_000,
}),
maxAttempts: integer(env, "S3_MAX_ATTEMPTS", 2, { min: 1, max: 5 }),
}),
processing: Object.freeze({
tempDirectory: (() => {
const value = env.UPLOAD_TEMP_DIRECTORY?.trim() || "/tmp/announcements-uploads";
if (!path.isAbsolute(value)) {
throw new ConfigError(
"UPLOAD_TEMP_DIRECTORY must be an absolute path",
"UPLOAD_TEMP_DIRECTORY",
);
}
return value;
})(),
maxFileBytes: integer(env, "UPLOAD_MAX_FILE_BYTES", 25 * 1024 * 1024, {
min: 1,
max: 1024 * 1024 * 1024,
}),
maxFiles: integer(env, "UPLOAD_MAX_FILES", 20, { min: 1, max: 100 }),
maxInputPixels: integer(env, "IMAGE_MAX_INPUT_PIXELS", 40_000_000, {
min: 1,
max: 500_000_000,
}),
}), }),
mqtt: Object.freeze({ mqtt: Object.freeze({
url: url(env, "MQTT_URL", { url: url(env, "MQTT_URL", {
@@ -193,6 +229,16 @@ function redactedConfig(config) {
region: config.storage.region, region: config.storage.region,
bucket: config.storage.bucket, bucket: config.storage.bucket,
forcePathStyle: config.storage.forcePathStyle, forcePathStyle: config.storage.forcePathStyle,
healthProbeIntervalMs: config.storage.healthProbeIntervalMs,
connectionTimeoutMs: config.storage.connectionTimeoutMs,
requestTimeoutMs: config.storage.requestTimeoutMs,
maxAttempts: config.storage.maxAttempts,
},
processing: {
tempDirectoryConfigured: Boolean(config.processing.tempDirectory),
maxFileBytes: config.processing.maxFileBytes,
maxFiles: config.processing.maxFiles,
maxInputPixels: config.processing.maxInputPixels,
}, },
mqtt: { mqtt: {
configured: Boolean(config.mqtt.url), configured: Boolean(config.mqtt.url),
+60 -6
View File
@@ -14,15 +14,62 @@ function databaseHealth(database) {
} }
} }
function createHealthRouter({ database, lifecycle, migrations }) { function storageHealth(storage) {
if (!storage) {
return {
status: "degraded",
checkedAt: null,
lastSuccessAt: null,
code: "STORAGE_NOT_INITIALIZED",
message: "Object storage has not been initialized",
};
}
try {
const result = storage.health();
if (result?.status === "healthy") {
return {
status: "healthy",
checkedAt: result.checkedAt || null,
lastSuccessAt: result.lastSuccessAt || result.checkedAt || null,
};
}
return {
status: "degraded",
checkedAt: result?.checkedAt || null,
lastSuccessAt: result?.lastSuccessAt || null,
code:
result?.code === "STORAGE_NOT_CHECKED"
? "STORAGE_NOT_CHECKED"
: "STORAGE_UNAVAILABLE",
message:
result?.code === "STORAGE_NOT_CHECKED"
? "Object storage has not been checked"
: "Object storage is unavailable",
};
} catch {
// A health endpoint must remain available when its dependency is not.
}
return {
status: "degraded",
checkedAt: null,
lastSuccessAt: null,
code: "STORAGE_UNAVAILABLE",
message: "Object storage is unavailable",
};
}
function createHealthRouter({ database, lifecycle, migrations, storage }) {
const router = express.Router(); const router = express.Router();
router.get("/live", (request, response) => { router.get("/live", (request, response) => {
response.json({ status: "ok" }); response.json({ status: "ok" });
}); });
router.get("/ready", (request, response) => { router.get("/ready", async (request, response) => {
const databaseCheck = databaseHealth(database); const databaseCheck = databaseHealth(database);
const storageCheck = storageHealth(storage);
const migrationsCurrent = migrations.pendingVersions.length === 0; const migrationsCurrent = migrations.pendingVersions.length === 0;
const ready = const ready =
lifecycle.ready && databaseCheck.status === "ok" && migrationsCurrent; lifecycle.ready && databaseCheck.status === "ok" && migrationsCurrent;
@@ -32,6 +79,7 @@ function createHealthRouter({ database, lifecycle, migrations }) {
checks: { checks: {
database: databaseCheck.status, database: databaseCheck.status,
migrations: migrationsCurrent ? "current" : "pending", migrations: migrationsCurrent ? "current" : "pending",
storage: storageCheck.status,
}, },
}); });
}); });
@@ -39,18 +87,23 @@ function createHealthRouter({ database, lifecycle, migrations }) {
return router; return router;
} }
function createDetailedHealthRouter({ config, database, lifecycle, migrations }) { function createDetailedHealthRouter({ config, database, lifecycle, migrations, storage }) {
const router = express.Router(); const router = express.Router();
router.get("/", (request, response) => { router.get("/", async (request, response) => {
const databaseCheck = databaseHealth(database); const databaseCheck = databaseHealth(database);
const storageCheck = storageHealth(storage);
const ready = const ready =
lifecycle.ready && lifecycle.ready &&
databaseCheck.status === "ok" && databaseCheck.status === "ok" &&
migrations.pendingVersions.length === 0; migrations.pendingVersions.length === 0;
const configuration = redactedConfig(config); const configuration = redactedConfig(config);
const status = ready ? "degraded" : "not_ready"; const status = !ready
? "not_ready"
: storageCheck.status === "healthy"
? "ok"
: "degraded";
response.status(ready ? 200 : 503).json({ response.status(ready ? 200 : 503).json({
status, status,
@@ -62,7 +115,7 @@ function createDetailedHealthRouter({ config, database, lifecycle, migrations })
latestSchemaVersion: migrations.latestAvailableVersion, latestSchemaVersion: migrations.latestAvailableVersion,
}, },
storage: { storage: {
status: "not_checked", ...storageCheck,
configured: configuration.storage.configured, configured: configuration.storage.configured,
}, },
mqtt: { mqtt: {
@@ -86,4 +139,5 @@ module.exports = {
createDetailedHealthRouter, createDetailedHealthRouter,
createHealthRouter, createHealthRouter,
databaseHealth, databaseHealth,
storageHealth,
}; };
+330
View File
@@ -0,0 +1,330 @@
const crypto = require("node:crypto");
const fs = require("node:fs");
const sharp = require("sharp");
const SUPPORTED_FORMATS = Object.freeze(new Set(["jpeg", "png", "webp"]));
const CONTENT_TYPES = Object.freeze({
jpeg: "image/jpeg",
png: "image/png",
webp: "image/webp",
});
const BLACK = Object.freeze({ r: 0, g: 0, b: 0, alpha: 1 });
const DEFAULT_IMAGE_DERIVATIVE_OPTIONS = Object.freeze({
display: Object.freeze({ maxWidth: 1920, maxHeight: 1080, quality: 82 }),
thumbnail: Object.freeze({ maxWidth: 480, maxHeight: 270, quality: 72 }),
});
class ImageProcessingError extends Error {
constructor(code, message, { statusCode = 422, details, cause } = {}) {
super(message, cause ? { cause } : undefined);
this.name = "ImageProcessingError";
this.code = code;
this.statusCode = statusCode;
if (details !== undefined) {
this.details = details;
}
}
}
function requireInteger(value, name, { min, max } = {}) {
if (!Number.isSafeInteger(value)) {
throw new TypeError(`${name} must be an integer`);
}
if (min !== undefined && value < min) {
throw new RangeError(`${name} must be at least ${min}`);
}
if (max !== undefined && value > max) {
throw new RangeError(`${name} must be at most ${max}`);
}
return value;
}
function normalizeTargetOptions(target, name, defaults) {
if (
target !== undefined &&
(target === null || typeof target !== "object" || Array.isArray(target))
) {
throw new TypeError(`${name} image options must be an object`);
}
const configured = { ...defaults, ...target };
return Object.freeze({
maxWidth: requireInteger(configured.maxWidth, `${name}.maxWidth`, { min: 1 }),
maxHeight: requireInteger(configured.maxHeight, `${name}.maxHeight`, { min: 1 }),
quality: requireInteger(configured.quality, `${name}.quality`, {
min: 1,
max: 100,
}),
});
}
function normalizeOptions(options) {
if (options === null || typeof options !== "object" || Array.isArray(options)) {
throw new TypeError("Image processing options are required");
}
return Object.freeze({
maxInputPixels: requireInteger(
options.maxInputPixels,
"maxInputPixels",
{ min: 1 },
),
display: normalizeTargetOptions(
options.display,
"display",
DEFAULT_IMAGE_DERIVATIVE_OPTIONS.display,
),
thumbnail: normalizeTargetOptions(
options.thumbnail,
"thumbnail",
DEFAULT_IMAGE_DERIVATIVE_OPTIONS.thumbnail,
),
});
}
function sha256(buffer) {
return crypto.createHash("sha256").update(buffer).digest("hex");
}
async function hashFile(filename) {
const hash = crypto.createHash("sha256");
let byteSize = 0;
try {
for await (const chunk of fs.createReadStream(filename)) {
byteSize += chunk.length;
hash.update(chunk);
}
} catch (error) {
throw new ImageProcessingError(
"IMAGE_INPUT_UNREADABLE",
"The uploaded image could not be read",
{ statusCode: 422, cause: error },
);
}
return { byteSize, sha256: hash.digest("hex") };
}
async function describeInput(input) {
if (Buffer.isBuffer(input)) {
return {
sharpInput: input,
byteSize: input.length,
sha256: sha256(input),
};
}
if (typeof input !== "string" || input.length === 0) {
throw new ImageProcessingError(
"INVALID_IMAGE_INPUT",
"Image input must be a Buffer or temporary file path",
{ statusCode: 400 },
);
}
let stats;
try {
stats = await fs.promises.stat(input);
} catch (error) {
throw new ImageProcessingError(
"IMAGE_INPUT_UNREADABLE",
"The uploaded image could not be read",
{ statusCode: 422, cause: error },
);
}
if (!stats.isFile()) {
throw new ImageProcessingError(
"INVALID_IMAGE_INPUT",
"Image input path must identify a regular file",
{ statusCode: 400 },
);
}
return {
sharpInput: input,
...(await hashFile(input)),
};
}
function decodeError(error, maxInputPixels) {
if (error instanceof ImageProcessingError) {
return error;
}
if (/pixel limit/i.test(error?.message || "")) {
return new ImageProcessingError(
"IMAGE_PIXEL_LIMIT_EXCEEDED",
"The uploaded image exceeds the configured decoded-pixel limit",
{
statusCode: 413,
details: { maxInputPixels },
cause: error,
},
);
}
return new ImageProcessingError(
"MALFORMED_IMAGE",
"The uploaded file is not a valid decodable image",
{ statusCode: 422, cause: error },
);
}
async function readMetadata(input, maxInputPixels) {
try {
return await sharp(input, {
animated: true,
failOn: "error",
limitInputPixels: maxInputPixels,
}).metadata();
} catch (error) {
throw decodeError(error, maxInputPixels);
}
}
function validateMetadata(metadata) {
const pages = metadata.pages ?? 1;
if (pages > 1) {
const hasAnimationTiming =
metadata.loop !== undefined ||
(Array.isArray(metadata.delay) && metadata.delay.length > 0);
throw new ImageProcessingError(
hasAnimationTiming
? "ANIMATED_IMAGE_NOT_SUPPORTED"
: "MULTIPAGE_IMAGE_NOT_SUPPORTED",
hasAnimationTiming
? "Animated images are not supported"
: "Multipage images are not supported",
{ statusCode: 422, details: { pages } },
);
}
if (!SUPPORTED_FORMATS.has(metadata.format)) {
throw new ImageProcessingError(
"UNSUPPORTED_IMAGE_FORMAT",
"Only still JPEG, PNG, and WebP images are supported",
{
statusCode: 415,
details: {
detectedFormat: metadata.format || null,
supportedFormats: [...SUPPORTED_FORMATS],
},
},
);
}
if (
!Number.isSafeInteger(metadata.width) ||
metadata.width < 1 ||
!Number.isSafeInteger(metadata.height) ||
metadata.height < 1
) {
throw new ImageProcessingError(
"MALFORMED_IMAGE",
"The uploaded image does not contain valid dimensions",
{ statusCode: 422 },
);
}
}
async function generateDerivative(input, target, maxInputPixels) {
let output;
try {
output = await sharp(input, {
failOn: "error",
limitInputPixels: maxInputPixels,
})
.autoOrient()
.flatten({ background: BLACK })
.resize({
width: target.maxWidth,
height: target.maxHeight,
fit: "inside",
withoutEnlargement: true,
})
.toColourspace("srgb")
.webp({ quality: target.quality, effort: 4 })
.toBuffer({ resolveWithObject: true });
} catch (error) {
throw decodeError(error, maxInputPixels);
}
return {
buffer: output.data,
sha256: sha256(output.data),
format: "webp",
contentType: CONTENT_TYPES.webp,
byteSize: output.data.length,
width: output.info.width,
height: output.info.height,
};
}
function originalMetadata(metadata, inputDescription) {
const orientedDimensions = metadata.autoOrient || {
width: metadata.width,
height: metadata.height,
};
return {
sha256: inputDescription.sha256,
format: metadata.format,
contentType: CONTENT_TYPES[metadata.format],
byteSize: inputDescription.byteSize,
// Width and height describe how the original should appear after applying
// its source orientation. The encoded dimensions remain available for
// diagnostics without implying that the archived bytes were rewritten.
width: orientedDimensions.width,
height: orientedDimensions.height,
encodedWidth: metadata.width,
encodedHeight: metadata.height,
orientation: metadata.orientation ?? null,
pages: metadata.pages ?? 1,
channels: metadata.channels,
hasAlpha: Boolean(metadata.hasAlpha),
};
}
function createImageProcessor(options) {
const config = normalizeOptions(options);
async function process(input) {
const inputDescription = await describeInput(input);
const metadata = await readMetadata(
inputDescription.sharpInput,
config.maxInputPixels,
);
validateMetadata(metadata);
const [display, thumbnail] = await Promise.all([
generateDerivative(
inputDescription.sharpInput,
config.display,
config.maxInputPixels,
),
generateDerivative(
inputDescription.sharpInput,
config.thumbnail,
config.maxInputPixels,
),
]);
return {
original: originalMetadata(metadata, inputDescription),
display,
thumbnail,
};
}
return Object.freeze({ process });
}
module.exports = {
DEFAULT_IMAGE_DERIVATIVE_OPTIONS,
ImageProcessingError,
createImageProcessor,
};
+43 -2
View File
@@ -3,6 +3,8 @@ const { Server: SocketServer } = require("socket.io");
const { createApp } = require("./app"); const { createApp } = require("./app");
const { openDatabase } = require("./database"); const { openDatabase } = require("./database");
const { runMigrations } = require("./database/migrate"); const { runMigrations } = require("./database/migrate");
const { createImageProcessor } = require("./image-processing");
const { createStorageService } = require("./storage");
function listen(httpServer, port, host) { function listen(httpServer, port, host) {
return new Promise((resolve, reject) => { return new Promise((resolve, reject) => {
@@ -20,7 +22,12 @@ function listen(httpServer, port, host) {
}); });
} }
function createRuntime({ config, logger = console }) { function createRuntime({
config,
imageProcessor: suppliedImageProcessor,
logger = console,
storage: suppliedStorage,
}) {
const database = openDatabase(config.database); const database = openDatabase(config.database);
let migrations; let migrations;
try { try {
@@ -34,7 +41,32 @@ function createRuntime({ config, logger = console }) {
ready: false, ready: false,
stopping: false, stopping: false,
}; };
const app = createApp({ config, database, lifecycle, migrations }); let storage;
let imageProcessor;
try {
storage = suppliedStorage || createStorageService(config.storage, { logger });
imageProcessor =
suppliedImageProcessor ||
createImageProcessor({ maxInputPixels: config.processing.maxInputPixels });
} catch (error) {
database.close();
throw error;
}
let app;
try {
app = createApp({
config,
database,
imageProcessor,
lifecycle,
migrations,
storage,
});
} catch (error) {
storage.stop();
database.close();
throw error;
}
const httpServer = http.createServer(app); const httpServer = http.createServer(app);
const io = new SocketServer(httpServer, { const io = new SocketServer(httpServer, {
serveClient: false, serveClient: false,
@@ -63,9 +95,15 @@ function createRuntime({ config, logger = console }) {
.then((listeningAddress) => { .then((listeningAddress) => {
address = listeningAddress; address = listeningAddress;
lifecycle.ready = true; lifecycle.ready = true;
Promise.resolve()
.then(() => storage.start())
.catch(() => {
logger.warn?.("Object storage health checks could not be started");
});
return listeningAddress; return listeningAddress;
}) })
.catch((error) => { .catch((error) => {
storage.stop();
database.close(); database.close();
throw error; throw error;
}); });
@@ -85,6 +123,7 @@ function createRuntime({ config, logger = console }) {
if (settled) return; if (settled) return;
settled = true; settled = true;
clearTimeout(timeout); clearTimeout(timeout);
storage.stop();
if (database.open) { if (database.open) {
database.close(); database.close();
} }
@@ -119,6 +158,8 @@ function createRuntime({ config, logger = console }) {
io, io,
lifecycle, lifecycle,
migrations, migrations,
storage,
imageProcessor,
get address() { get address() {
return address; return address;
}, },
+398
View File
@@ -0,0 +1,398 @@
const {
CreateBucketCommand,
DeleteObjectCommand,
GetObjectCommand,
HeadBucketCommand,
HeadObjectCommand,
PutObjectCommand,
S3Client,
} = require("@aws-sdk/client-s3");
class StorageError extends Error {
constructor(message, { code, statusCode, cause } = {}) {
super(message, { cause });
this.name = "StorageError";
this.code = code || "STORAGE_ERROR";
this.statusCode = statusCode || 500;
}
}
class StorageUnavailableError extends StorageError {
constructor(cause) {
super("Object storage is unavailable", {
code: "STORAGE_UNAVAILABLE",
statusCode: 503,
cause,
});
this.name = "StorageUnavailableError";
}
}
class StorageObjectNotFoundError extends StorageError {
constructor(key, cause) {
super("Stored object was not found", {
code: "STORAGE_OBJECT_NOT_FOUND",
statusCode: 404,
cause,
});
this.name = "StorageObjectNotFoundError";
this.key = key;
}
}
function errorStatus(error) {
return error?.$metadata?.httpStatusCode || error?.statusCode;
}
function isMissingBucket(error) {
return (
error?.name === "NoSuchBucket" ||
error?.Code === "NoSuchBucket" ||
error?.code === "NoSuchBucket" ||
errorStatus(error) === 404
);
}
function isMissingObject(error) {
if (
error?.name === "NoSuchBucket" ||
error?.Code === "NoSuchBucket" ||
error?.code === "NoSuchBucket"
) {
return false;
}
return (
error?.name === "NoSuchKey" ||
error?.name === "NotFound" ||
error?.Code === "NoSuchKey" ||
error?.code === "NoSuchKey" ||
errorStatus(error) === 404
);
}
function isBucketCreationConflict(error) {
return (
error?.name === "BucketAlreadyOwnedByYou" ||
error?.name === "BucketAlreadyExists" ||
error?.Code === "BucketAlreadyOwnedByYou" ||
error?.Code === "BucketAlreadyExists" ||
error?.code === "BucketAlreadyOwnedByYou" ||
error?.code === "BucketAlreadyExists"
);
}
function isConsumerCancellation(error) {
return (
error?.name === "AbortError" ||
error?.code === "ABORT_ERR" ||
error?.code === "ERR_STREAM_PREMATURE_CLOSE"
);
}
function assertObjectKey(key) {
if (typeof key !== "string" || key.length === 0) {
throw new TypeError("Storage object key must be a non-empty string");
}
const segments = key.split("/");
if (
key.startsWith("/") ||
key.includes("\\") ||
key.includes("\0") ||
segments.some((segment) => segment === "." || segment === "..")
) {
throw new TypeError("Storage object key is invalid");
}
}
function publicHealth(health) {
return Object.freeze({ ...health });
}
function createS3Client(config) {
const options = {
endpoint: config.endpoint,
region: config.region,
forcePathStyle: config.forcePathStyle,
maxAttempts: config.maxAttempts,
requestHandler: {
connectionTimeout: config.connectionTimeoutMs,
// Use an inactivity timeout rather than an absolute request deadline so
// a large but continuously progressing upload is not aborted.
socketTimeout: config.requestTimeoutMs,
},
// Optional AWS checksums are not uniformly implemented by S3-compatible
// providers. Source and derivative SHA-256 hashes are verified by the app.
requestChecksumCalculation: "WHEN_REQUIRED",
responseChecksumValidation: "WHEN_REQUIRED",
};
if (config.accessKeyId && config.secretAccessKey) {
options.credentials = {
accessKeyId: config.accessKeyId,
secretAccessKey: config.secretAccessKey,
};
}
return new S3Client(options);
}
function createStorageService(config, { client, logger = console, now = () => new Date() } = {}) {
const s3 = client || createS3Client(config);
let ensurePromise = null;
let healthTimer = null;
let bucketReady = false;
let issuedHealthSequence = 0;
let appliedHealthSequence = 0;
let currentHealth = {
status: "degraded",
checkedAt: null,
lastSuccessAt: null,
code: "STORAGE_NOT_CHECKED",
message: "Object storage has not been checked",
};
function nextHealthSequence() {
issuedHealthSequence += 1;
return issuedHealthSequence;
}
function markHealthy(sequence) {
if (sequence < appliedHealthSequence) return publicHealth(currentHealth);
appliedHealthSequence = sequence;
bucketReady = true;
const checkedAt = now().toISOString();
currentHealth = {
status: "healthy",
checkedAt,
lastSuccessAt: checkedAt,
};
return publicHealth(currentHealth);
}
function markDegraded(sequence) {
if (sequence < appliedHealthSequence) return publicHealth(currentHealth);
appliedHealthSequence = sequence;
bucketReady = false;
currentHealth = {
status: "degraded",
checkedAt: now().toISOString(),
lastSuccessAt: currentHealth.lastSuccessAt,
code: "STORAGE_UNAVAILABLE",
message: "Object storage is unavailable",
};
return publicHealth(currentHealth);
}
function unavailable(error, sequence) {
markDegraded(sequence);
return error instanceof StorageUnavailableError
? error
: new StorageUnavailableError(error);
}
async function ensureBucketOnce() {
const sequence = nextHealthSequence();
try {
await s3.send(new HeadBucketCommand({ Bucket: config.bucket }));
markHealthy(sequence);
return { created: false };
} catch (error) {
if (!isMissingBucket(error)) {
throw unavailable(error, sequence);
}
}
try {
await s3.send(new CreateBucketCommand({ Bucket: config.bucket }));
markHealthy(sequence);
return { created: true };
} catch (error) {
if (isBucketCreationConflict(error)) {
try {
await s3.send(new HeadBucketCommand({ Bucket: config.bucket }));
markHealthy(sequence);
return { created: false };
} catch (verificationError) {
throw unavailable(verificationError, sequence);
}
}
throw unavailable(error, sequence);
}
}
async function ensureBucket() {
if (!ensurePromise) {
ensurePromise = ensureBucketOnce().finally(() => {
ensurePromise = null;
});
}
return ensurePromise;
}
async function initialize() {
try {
await ensureBucket();
} catch (error) {
logger.warn?.("Object storage is unavailable during initialization");
}
return publicHealth(currentHealth);
}
async function start() {
if (!healthTimer) {
healthTimer = setInterval(() => {
ensureBucket().catch(() => {});
}, config.healthProbeIntervalMs || 30_000);
healthTimer.unref?.();
}
return initialize();
}
async function probe() {
const sequence = nextHealthSequence();
try {
await s3.send(new HeadBucketCommand({ Bucket: config.bucket }));
return markHealthy(sequence);
} catch (error) {
return markDegraded(sequence);
}
}
async function putObject({ key, body, contentType, contentLength, metadata }) {
assertObjectKey(key);
if (!bucketReady) await ensureBucket();
const sequence = nextHealthSequence();
const input = {
Bucket: config.bucket,
Key: key,
Body: body,
...(contentType ? { ContentType: contentType } : {}),
...(contentLength !== undefined ? { ContentLength: contentLength } : {}),
...(metadata ? { Metadata: metadata } : {}),
};
try {
const result = await s3.send(new PutObjectCommand(input));
markHealthy(sequence);
return {
key,
etag: result.ETag || null,
versionId: result.VersionId || null,
};
} catch (error) {
throw unavailable(error, sequence);
}
}
function objectMetadata(key, result) {
return {
key,
contentType: result.ContentType || null,
contentLength:
result.ContentLength === undefined ? null : Number(result.ContentLength),
etag: result.ETag || null,
lastModified: result.LastModified || null,
metadata: result.Metadata || {},
};
}
async function getObject(key) {
assertObjectKey(key);
if (!bucketReady) await ensureBucket();
const sequence = nextHealthSequence();
try {
const result = await s3.send(
new GetObjectCommand({ Bucket: config.bucket, Key: key }),
);
markHealthy(sequence);
result.Body?.once?.("error", (error) => {
if (!isConsumerCancellation(error)) {
markDegraded(nextHealthSequence());
}
});
return {
...objectMetadata(key, result),
body: result.Body,
};
} catch (error) {
if (isMissingObject(error)) {
markHealthy(sequence);
throw new StorageObjectNotFoundError(key, error);
}
throw unavailable(error, sequence);
}
}
async function headObject(key) {
assertObjectKey(key);
if (!bucketReady) await ensureBucket();
const sequence = nextHealthSequence();
try {
const result = await s3.send(
new HeadObjectCommand({ Bucket: config.bucket, Key: key }),
);
markHealthy(sequence);
return objectMetadata(key, result);
} catch (error) {
if (isMissingObject(error)) {
markHealthy(sequence);
return null;
}
throw unavailable(error, sequence);
}
}
async function objectExists(key) {
return (await headObject(key)) !== null;
}
async function deleteObject(key) {
assertObjectKey(key);
if (!bucketReady) await ensureBucket();
const sequence = nextHealthSequence();
try {
await s3.send(
new DeleteObjectCommand({ Bucket: config.bucket, Key: key }),
);
markHealthy(sequence);
} catch (error) {
if (isMissingObject(error)) {
markHealthy(sequence);
return;
}
throw unavailable(error, sequence);
}
}
function stop() {
if (healthTimer) {
clearInterval(healthTimer);
healthTimer = null;
}
s3.destroy?.();
}
return Object.freeze({
initialize,
start,
stop,
ensureBucket,
probe,
health: () => publicHealth(currentHealth),
putObject,
getObject,
headObject,
objectExists,
deleteObject,
destroy: stop,
});
}
module.exports = {
StorageError,
StorageObjectNotFoundError,
StorageUnavailableError,
createS3Client,
createStorageService,
};
+442
View File
@@ -0,0 +1,442 @@
const test = require("node:test");
const assert = require("node:assert/strict");
const crypto = require("node:crypto");
const fs = require("node:fs");
const http = require("node:http");
const os = require("node:os");
const path = require("node:path");
const { Readable } = require("node:stream");
const sharp = require("sharp");
const { createApp } = require("../src/app");
const { loadConfig } = require("../src/config");
const { openDatabase } = require("../src/database");
const { runMigrations } = require("../src/database/migrate");
const { createImageProcessor } = require("../src/image-processing");
const { StorageObjectNotFoundError } = require("../src/storage");
async function readBody(body) {
if (Buffer.isBuffer(body)) return body;
const chunks = [];
for await (const chunk of body) chunks.push(Buffer.from(chunk));
return Buffer.concat(chunks);
}
function memoryStorage() {
const objects = new Map();
return {
objects,
health() {
return {
status: "healthy",
checkedAt: "2033-05-18T03:33:20.000Z",
lastSuccessAt: "2033-05-18T03:33:20.000Z",
};
},
async putObject(input) {
objects.set(input.key, {
body: await readBody(input.body),
contentType: input.contentType,
metadata: input.metadata,
});
return { key: input.key };
},
async getObject(key) {
const object = objects.get(key);
if (!object) throw new StorageObjectNotFoundError(key);
return {
body: Readable.from([object.body]),
contentLength: object.body.length,
contentType: object.contentType,
};
},
async deleteObject(key) {
objects.delete(key);
return { key, deleted: true };
},
};
}
async function createHarness(t, { maxFileBytes = 1024 * 1024 } = {}) {
const tempDirectory = fs.mkdtempSync(
path.join(os.tmpdir(), "announcements-api-uploads-"),
);
const config = loadConfig({
NODE_ENV: "test",
HOST: "127.0.0.1",
PORT: "0",
DATABASE_PATH: ":memory:",
APP_TIMEZONE: "America/Chicago",
S3_ENDPOINT: "http://private-seaweedfs:8333",
S3_BUCKET: "announcements-test",
UPLOAD_TEMP_DIRECTORY: tempDirectory,
UPLOAD_MAX_FILE_BYTES: String(maxFileBytes),
UPLOAD_MAX_FILES: "5",
IMAGE_MAX_INPUT_PIXELS: "1000000",
});
const database = openDatabase({ path: ":memory:" });
const migrations = runMigrations(database);
const storage = memoryStorage();
const app = createApp({
config,
database,
imageProcessor: createImageProcessor({
maxInputPixels: config.processing.maxInputPixels,
}),
lifecycle: { ready: true },
migrations,
storage,
});
const server = app.listen(0, "127.0.0.1");
await new Promise((resolve, reject) => {
server.once("listening", resolve);
server.once("error", reject);
});
t.after(async () => {
await new Promise((resolve) => server.close(resolve));
database.close();
fs.rmSync(tempDirectory, { recursive: true, force: true });
});
return {
baseUrl: `http://127.0.0.1:${server.address().port}`,
database,
storage,
tempDirectory,
};
}
function appendFile(form, field, buffer, filename, contentType) {
form.append(field, new Blob([buffer], { type: contentType }), filename);
}
async function jsonResponse(baseUrl, route, init) {
const response = await fetch(`${baseUrl}${route}`, init);
return { response, body: await response.json() };
}
async function eventuallyEmpty(directory) {
for (let attempt = 0; attempt < 50; attempt += 1) {
if (fs.readdirSync(directory).length === 0) return;
await new Promise((resolve) => setTimeout(resolve, 10));
}
assert.deepEqual(fs.readdirSync(directory), []);
}
async function eventuallyContainsFile(directory) {
for (let attempt = 0; attempt < 50; attempt += 1) {
if (fs.readdirSync(directory).length > 0) return;
await new Promise((resolve) => setTimeout(resolve, 10));
}
assert.fail("The upload did not create its temporary file");
}
async function sampleImages() {
const source = sharp({
create: {
width: 80,
height: 60,
channels: 4,
background: { r: 31, g: 72, b: 120, alpha: 0.5 },
},
});
return {
jpeg: await source.clone().jpeg().toBuffer(),
png: await source.clone().png().toBuffer(),
webp: await source.clone().webp().toBuffer(),
};
}
test("asset HTTP API accepts JPEG, PNG, and WebP and serves private immutable media", async (t) => {
const harness = await createHarness(t);
const images = await sampleImages();
const form = new FormData();
appendFile(form, "files", images.jpeg, "welcome.jpg", "image/jpeg");
appendFile(form, "files", images.png, "../../bulletin.png", "image/png");
appendFile(form, "files", images.webp, "events.webp", "image/webp");
const uploaded = await jsonResponse(harness.baseUrl, "/api/assets", {
method: "POST",
body: form,
});
assert.equal(uploaded.response.status, 201);
assert.equal(uploaded.body.createdCount, 3);
assert.equal(uploaded.body.duplicateCount, 0);
assert.equal(uploaded.body.results.length, 3);
assert.deepEqual(
uploaded.body.results.map((result) => result.asset.original.contentType),
["image/jpeg", "image/png", "image/webp"],
);
assert.equal(uploaded.body.results[1].asset.originalFilename, "bulletin.png");
assert.doesNotMatch(JSON.stringify(uploaded.body), /objectKey|object_key|private-seaweedfs/);
assert.equal(harness.storage.objects.size, 9);
await eventuallyEmpty(harness.tempDirectory);
const asset = uploaded.body.results[0].asset;
const listed = await jsonResponse(harness.baseUrl, "/api/assets");
assert.equal(listed.response.status, 200);
assert.equal(listed.body.assets.length, 3);
assert.deepEqual(
(await jsonResponse(harness.baseUrl, `/api/assets/${asset.id}`)).body,
asset,
);
for (const media of [asset.display, asset.thumbnail]) {
const response = await fetch(`${harness.baseUrl}${media.url}`);
const body = Buffer.from(await response.arrayBuffer());
assert.equal(response.status, 200);
assert.equal(response.headers.get("content-type"), "image/webp");
assert.equal(
response.headers.get("cache-control"),
"public, max-age=31536000, immutable",
);
assert.equal(response.headers.get("content-length"), String(body.length));
assert.equal(response.headers.get("etag"), `"${media.sha256}"`);
assert.equal(crypto.createHash("sha256").update(body).digest("hex"), media.sha256);
}
const original = await fetch(`${harness.baseUrl}${asset.original.url}`);
assert.equal(original.status, 200);
assert.equal(original.headers.get("content-type"), "image/jpeg");
assert.equal(original.headers.get("cache-control"), "private, no-store");
assert.match(original.headers.get("content-disposition"), /^attachment;/);
assert.deepEqual(Buffer.from(await original.arrayBuffer()), images.jpeg);
const wrongVersion = await jsonResponse(
harness.baseUrl,
`/media/assets/${asset.id}/display/${"a".repeat(64)}.webp`,
);
assert.equal(wrongVersion.response.status, 404);
assert.equal(wrongVersion.body.error.code, "MEDIA_NOT_FOUND");
const arbitraryKey = await jsonResponse(
harness.baseUrl,
`/media/assets/${asset.id}/display/${encodeURIComponent("../../original/key")}.webp`,
);
assert.equal(arbitraryKey.response.status, 404);
assert.equal(arbitraryKey.body.error.code, "MEDIA_NOT_FOUND");
const displayKey = [...harness.storage.objects.keys()].find(
(key) => key.includes(`assets/${asset.id}/display/`),
);
harness.storage.objects.delete(displayKey);
const missingStoredBytes = await jsonResponse(
harness.baseUrl,
asset.display.url,
);
assert.equal(missingStoredBytes.response.status, 503);
assert.equal(missingStoredBytes.body.error.code, "STORAGE_INCONSISTENCY");
assert.equal(
(await jsonResponse(harness.baseUrl, "/health/live")).response.status,
200,
);
});
test("asset HTTP API reports duplicates, name conflicts, and persists editable metadata", async (t) => {
const harness = await createHarness(t);
const { png, jpeg } = await sampleImages();
const firstForm = new FormData();
appendFile(firstForm, "file", png, "weekly.png", "application/octet-stream");
firstForm.append("display_name", "Weekly Announcement");
const first = await jsonResponse(harness.baseUrl, "/api/assets", {
method: "POST",
body: firstForm,
});
assert.equal(first.response.status, 201);
assert.equal(first.body.created, true);
const assetId = first.body.asset.id;
const duplicateForm = new FormData();
appendFile(duplicateForm, "file", png, "unrelated-name.png", "image/png");
duplicateForm.append("display_name", "Ignored duplicate proposal");
const duplicate = await jsonResponse(harness.baseUrl, "/api/assets", {
method: "POST",
body: duplicateForm,
});
assert.equal(duplicate.response.status, 200);
assert.equal(duplicate.body.created, false);
assert.equal(duplicate.body.duplicate, true);
assert.equal(duplicate.body.asset.id, assetId);
assert.equal(harness.storage.objects.size, 3);
const conflictForm = new FormData();
appendFile(conflictForm, "file", jpeg, "different.jpg", "image/jpeg");
conflictForm.append("displayName", "weekly announcement");
const conflict = await jsonResponse(harness.baseUrl, "/api/assets", {
method: "POST",
body: conflictForm,
});
assert.equal(conflict.response.status, 409);
assert.equal(conflict.body.error.code, "ASSET_NAME_CONFLICT");
const expiresAt = Date.parse("2034-01-02T03:04:05.000Z");
const edited = await jsonResponse(harness.baseUrl, `/api/assets/${assetId}`, {
method: "PATCH",
headers: { "content-type": "application/json" },
body: JSON.stringify({
display_name: "Renamed announcement",
expiration_at: expiresAt,
}),
});
assert.equal(edited.response.status, 200);
assert.equal(edited.body.displayName, "Renamed announcement");
assert.equal(edited.body.expirationAt, expiresAt);
const permanent = await jsonResponse(harness.baseUrl, `/api/assets/${assetId}`, {
method: "PATCH",
headers: { "content-type": "application/json" },
body: JSON.stringify({ expiration_at: null }),
});
assert.equal(permanent.body.expirationAt, null);
assert.equal(harness.storage.objects.size, 3);
await eventuallyEmpty(harness.tempDirectory);
const timestamp = Date.now();
harness.database
.prepare(
"INSERT INTO playlists (id, name, created_at, updated_at) VALUES (?, ?, ?, ?)",
)
.run("api-playlist", "API playlist", timestamp, timestamp);
harness.database
.prepare(
`INSERT INTO playlist_entries
(id, playlist_id, asset_id, position, created_at, updated_at)
VALUES (?, ?, ?, 0, ?, ?)`,
)
.run("api-entry", "api-playlist", assetId, timestamp, timestamp);
const references = await jsonResponse(
harness.baseUrl,
`/api/assets/${assetId}/references`,
);
assert.equal(references.response.status, 200);
assert.equal(references.body.canDelete, false);
assert.equal(references.body.playlists[0].playlistEntryId, "api-entry");
const blockedDelete = await jsonResponse(
harness.baseUrl,
`/api/assets/${assetId}`,
{ method: "DELETE" },
);
assert.equal(blockedDelete.response.status, 409);
assert.equal(blockedDelete.body.error.code, "ASSET_REFERENCED");
assert.equal(harness.storage.objects.size, 3);
harness.database
.prepare("DELETE FROM playlist_entries WHERE id = ?")
.run("api-entry");
const removed = await jsonResponse(harness.baseUrl, `/api/assets/${assetId}`, {
method: "DELETE",
});
assert.equal(removed.response.status, 200);
assert.deepEqual(removed.body, { id: assetId, deleted: true });
assert.equal(harness.storage.objects.size, 0);
assert.equal(
(await jsonResponse(harness.baseUrl, `/api/assets/${assetId}`)).response.status,
404,
);
});
test("upload validation returns structured errors and always cleans temporary files", async (t) => {
const harness = await createHarness(t, { maxFileBytes: 1024 });
const malformed = new FormData();
appendFile(malformed, "file", Buffer.from("not an image"), "broken.png", "image/png");
const malformedResult = await jsonResponse(harness.baseUrl, "/api/assets", {
method: "POST",
body: malformed,
});
assert.equal(malformedResult.response.status, 422);
assert.equal(malformedResult.body.error.code, "MALFORMED_IMAGE");
await eventuallyEmpty(harness.tempDirectory);
const unsupportedBuffer = await sharp({
create: {
width: 20,
height: 20,
channels: 3,
background: "red",
},
})
.gif()
.toBuffer();
const unsupported = new FormData();
appendFile(unsupported, "file", unsupportedBuffer, "still.gif", "image/png");
const unsupportedResult = await jsonResponse(harness.baseUrl, "/api/assets", {
method: "POST",
body: unsupported,
});
assert.equal(unsupportedResult.response.status, 415);
assert.equal(unsupportedResult.body.error.code, "UNSUPPORTED_IMAGE_FORMAT");
await eventuallyEmpty(harness.tempDirectory);
const oversized = new FormData();
appendFile(
oversized,
"file",
crypto.randomBytes(1025),
"too-large.png",
"image/png",
);
const oversizedResult = await jsonResponse(harness.baseUrl, "/api/assets", {
method: "POST",
body: oversized,
});
assert.equal(oversizedResult.response.status, 413);
assert.equal(oversizedResult.body.error.code, "UPLOAD_FILE_TOO_LARGE");
assert.deepEqual(oversizedResult.body.error.details, { maxFileBytes: 1024 });
await eventuallyEmpty(harness.tempDirectory);
const tooMany = new FormData();
for (let index = 0; index < 6; index += 1) {
appendFile(tooMany, "files", Buffer.from("x"), `${index}.png`, "image/png");
}
const tooManyResult = await jsonResponse(harness.baseUrl, "/api/assets", {
method: "POST",
body: tooMany,
});
assert.equal(tooManyResult.response.status, 413);
assert.equal(tooManyResult.body.error.code, "UPLOAD_FILE_COUNT_EXCEEDED");
assert.deepEqual(tooManyResult.body.error.details, { maxFiles: 5 });
await eventuallyEmpty(harness.tempDirectory);
const absent = await jsonResponse(harness.baseUrl, "/api/assets", {
method: "POST",
body: new FormData(),
});
assert.equal(absent.response.status, 400);
assert.equal(absent.body.error.code, "UPLOAD_FILE_REQUIRED");
assert.equal(harness.storage.objects.size, 0);
assert.equal(
harness.database.prepare("SELECT COUNT(*) AS count FROM assets").get().count,
0,
);
});
test("an aborted multipart request removes its in-flight temporary file", async (t) => {
const harness = await createHarness(t);
const boundary = `announcements-${crypto.randomUUID()}`;
const preamble = Buffer.from(
`--${boundary}\r\n` +
'Content-Disposition: form-data; name="file"; filename="aborted.png"\r\n' +
"Content-Type: image/png\r\n\r\n",
);
const declaredLength = preamble.length + 512 * 1024;
const request = http.request(`${harness.baseUrl}/api/assets`, {
method: "POST",
headers: {
"content-type": `multipart/form-data; boundary=${boundary}`,
"content-length": String(declaredLength),
},
});
request.on("error", () => {});
request.write(preamble);
request.write(Buffer.alloc(64 * 1024, 1));
await eventuallyContainsFile(harness.tempDirectory);
const closed = new Promise((resolve) => request.once("close", resolve));
request.destroy();
await closed;
await eventuallyEmpty(harness.tempDirectory);
assert.equal(harness.storage.objects.size, 0);
assert.equal(
harness.database.prepare("SELECT COUNT(*) AS count FROM assets").get().count,
0,
);
});
+35
View File
@@ -0,0 +1,35 @@
const test = require("node:test");
const assert = require("node:assert/strict");
const {
mapMulterError,
safeOriginalFilename,
} = require("../src/assets/multipart");
test("original filename metadata keeps a safe leaf without path traversal", () => {
assert.equal(safeOriginalFilename("../../Sunday/welcome.png"), "welcome.png");
assert.equal(safeOriginalFilename("..\\..\\Sunday\\welcome.png"), "welcome.png");
assert.equal(safeOriginalFilename(" welcome sign.png "), " welcome sign.png ");
});
test("original filename metadata removes controls and remains bounded", () => {
assert.equal(safeOriginalFilename("wel\u0000come\n.png"), "welcome.png");
assert.equal([...safeOriginalFilename(`${"a".repeat(300)}.png`)].length, 255);
assert.throws(
() => safeOriginalFilename(" \u0000\n "),
(error) => error.code === "INVALID_ORIGINAL_FILENAME",
);
});
test("temporary-storage failures are server errors, not invalid client uploads", () => {
const storageFailure = mapMulterError(
Object.assign(new Error("disk full at a private path"), { code: "ENOSPC" }),
{},
);
assert.equal(storageFailure.code, "UPLOAD_TEMP_STORAGE_UNAVAILABLE");
assert.equal(storageFailure.statusCode, 503);
assert.doesNotMatch(storageFailure.message, /private path/);
const malformed = mapMulterError(new Error("Unexpected end of form"), {});
assert.equal(malformed.code, "INVALID_MULTIPART_UPLOAD");
assert.equal(malformed.statusCode, 400);
});
+608
View File
@@ -0,0 +1,608 @@
const test = require("node:test");
const assert = require("node:assert/strict");
const crypto = require("node:crypto");
const fs = require("node:fs");
const os = require("node:os");
const path = require("node:path");
const { Readable } = require("node:stream");
const { openDatabase } = require("../src/database");
const { runMigrations } = require("../src/database/migrate");
const { createAssetService } = require("../src/assets/service");
const {
StorageObjectNotFoundError,
StorageUnavailableError,
} = require("../src/storage");
async function bufferBody(body) {
if (Buffer.isBuffer(body)) return body;
if (body instanceof Uint8Array) return Buffer.from(body);
const chunks = [];
for await (const chunk of body) chunks.push(Buffer.from(chunk));
return Buffer.concat(chunks);
}
function createMemoryStorage() {
const objects = new Map();
const calls = [];
let failNextPut = null;
let successfulPutsBeforeFailure = 0;
let failNextDelete = null;
let putDelayMs = 0;
return {
objects,
calls,
setPutFailure(error, putsBeforeFailure = 0) {
failNextPut = error;
successfulPutsBeforeFailure = putsBeforeFailure;
},
setDeleteFailure(error) {
failNextDelete = error;
},
setPutDelay(milliseconds) {
putDelayMs = milliseconds;
},
async putObject(input) {
calls.push({ operation: "put", ...input, body: undefined });
if (putDelayMs) {
await new Promise((resolve) => setTimeout(resolve, putDelayMs));
}
if (failNextPut) {
if (successfulPutsBeforeFailure === 0) {
const error = failNextPut;
failNextPut = null;
throw error;
}
successfulPutsBeforeFailure -= 1;
}
objects.set(input.key, {
body: await bufferBody(input.body),
contentType: input.contentType,
metadata: input.metadata,
});
return { key: input.key };
},
async getObject(key) {
calls.push({ operation: "get", key });
const object = objects.get(key);
if (!object) throw new StorageObjectNotFoundError(key);
return {
body: Readable.from(object.body),
contentLength: object.body.length,
contentType: object.contentType,
};
},
async deleteObject(key) {
calls.push({ operation: "delete", key });
if (failNextDelete) {
const error = failNextDelete;
failNextDelete = null;
throw error;
}
objects.delete(key);
return { key, deleted: true };
},
};
}
function createFakeImageProcessor({ reject } = {}) {
const calls = [];
return {
calls,
async process(filename) {
calls.push(filename);
if (reject) throw reject;
const original = await fs.promises.readFile(filename);
const sourceSha256 = crypto.createHash("sha256").update(original).digest("hex");
const displayBuffer = Buffer.from(`display:${sourceSha256}`);
const thumbnailBuffer = Buffer.from(`thumbnail:${sourceSha256}`);
return {
original: {
sha256: sourceSha256,
contentType: "image/png",
byteSize: original.length,
width: 1200,
height: 800,
},
display: {
buffer: displayBuffer,
sha256: crypto.createHash("sha256").update(displayBuffer).digest("hex"),
contentType: "image/webp",
byteSize: displayBuffer.length,
width: 1200,
height: 800,
},
thumbnail: {
buffer: thumbnailBuffer,
sha256: crypto.createHash("sha256").update(thumbnailBuffer).digest("hex"),
contentType: "image/webp",
byteSize: thumbnailBuffer.length,
width: 405,
height: 270,
},
};
},
};
}
function createHarness(options = {}) {
const database = openDatabase({ path: ":memory:" });
runMigrations(database);
const directory = fs.mkdtempSync(path.join(os.tmpdir(), "announcements-assets-"));
const storage = options.storage || createMemoryStorage();
const imageProcessor =
options.imageProcessor || createFakeImageProcessor(options.processorOptions);
let idCounter = 0;
let clock = 2_000_000_000_000;
const service = createAssetService({
database,
storage,
imageProcessor,
now: () => ++clock,
createId:
options.createId ||
(() => `00000000-0000-4000-8000-${String(++idCounter).padStart(12, "0")}`),
concurrentUploadWaitMs: options.concurrentUploadWaitMs ?? 2_000,
});
function uploadFile(originalFilename, contents) {
const buffer = Buffer.isBuffer(contents) ? contents : Buffer.from(contents);
const filename = path.join(directory, crypto.randomUUID());
fs.writeFileSync(filename, buffer);
return { path: filename, byteSize: buffer.length, originalFilename };
}
return {
database,
directory,
imageProcessor,
service,
storage,
uploadFile,
close() {
database.close();
fs.rmSync(directory, { recursive: true, force: true });
},
};
}
function rawAsset(database, assetId) {
return database.prepare("SELECT * FROM assets WHERE id = ?").get(assetId);
}
test("upload stores exact bytes under content-addressed keys without leaking keys", async (t) => {
const harness = createHarness();
t.after(() => harness.close());
const original = Buffer.from("exact uploaded bytes");
const [result] = await harness.service.upload([
harness.uploadFile(" Sunday Welcome.PNG ", original),
]);
assert.equal(result.created, true);
assert.equal(result.duplicate, false);
assert.match(result.asset.id, /^[0-9a-f-]{36}$/);
assert.equal(result.asset.displayName, "Sunday Welcome");
assert.equal(result.asset.originalFilename, " Sunday Welcome.PNG ");
assert.equal(result.asset.expirationAt, null);
assert.equal(result.asset.status, "ready");
assert.equal(result.asset.original.byteSize, original.length);
assert.equal(result.asset.display.contentType, "image/webp");
assert.equal(result.asset.thumbnail.width, 405);
assert.match(result.asset.display.url, /^\/media\/assets\//);
assert.doesNotMatch(
JSON.stringify(result.asset),
/object_key|originalObjectKey|displayObjectKey|thumbnailObjectKey|seaweed/i,
);
const row = rawAsset(harness.database, result.asset.id);
const expectedSourceHash = crypto.createHash("sha256").update(original).digest("hex");
assert.equal(row.source_sha256, expectedSourceHash);
assert.equal(row.original_object_key, `assets/${row.id}/original/${expectedSourceHash}`);
assert.equal(
row.display_object_key,
`assets/${row.id}/display/${row.display_sha256}.webp`,
);
assert.equal(
row.thumbnail_object_key,
`assets/${row.id}/thumbnail/${row.thumbnail_sha256}.webp`,
);
assert.deepEqual(harness.storage.objects.get(row.original_object_key).body, original);
assert.equal(harness.storage.objects.size, 3);
assert.deepEqual(harness.service.list(), [result.asset]);
assert.deepEqual(harness.service.get(result.asset.id), result.asset);
});
test("exact duplicate detection takes precedence over filename and display name", async (t) => {
const harness = createHarness();
t.after(() => harness.close());
const contents = Buffer.from("same image bytes");
const [created] = await harness.service.upload([
harness.uploadFile("first.png", contents),
]);
const [sameFilename] = await harness.service.upload([
harness.uploadFile("first.png", contents),
]);
const [differentFilename] = await harness.service.upload(
[harness.uploadFile("completely different name.webp", contents)],
{ displayName: "A conflicting requested name is ignored for an exact duplicate" },
);
for (const duplicate of [sameFilename, differentFilename]) {
assert.equal(duplicate.created, false);
assert.equal(duplicate.duplicate, true);
assert.equal(duplicate.asset.id, created.asset.id);
}
assert.equal(harness.service.list().length, 1);
assert.equal(harness.storage.calls.filter((call) => call.operation === "put").length, 3);
assert.equal(harness.imageProcessor.calls.length, 1);
});
test("different content with the same proposed name is a conflict, not a duplicate", async (t) => {
const harness = createHarness();
t.after(() => harness.close());
await harness.service.upload([harness.uploadFile("bulletin.png", "image one")]);
await assert.rejects(
harness.service.upload([harness.uploadFile("bulletin.jpg", "image two")]),
(error) => {
assert.equal(error.statusCode, 409);
assert.equal(error.code, "ASSET_NAME_CONFLICT");
assert.equal(error.details.asset.displayName, "bulletin");
return true;
},
);
assert.equal(harness.service.list().length, 1);
assert.equal(harness.imageProcessor.calls.length, 1);
const [created] = await harness.service.upload(
[harness.uploadFile("bulletin.jpg", "image two")],
{ displayName: "Second bulletin" },
);
assert.equal(created.created, true);
assert.notEqual(created.asset.sourceSha256, harness.service.list()[1].sourceSha256);
assert.equal(harness.service.list().length, 2);
});
test("multiple-file upload preserves order and deduplicates repeated bytes in the batch", async (t) => {
const harness = createHarness();
t.after(() => harness.close());
const results = await harness.service.upload([
harness.uploadFile("alpha.png", "alpha bytes"),
harness.uploadFile("beta.png", "beta bytes"),
harness.uploadFile("alpha-copy.png", "alpha bytes"),
]);
assert.deepEqual(
results.map(({ created, duplicate }) => ({ created, duplicate })),
[
{ created: true, duplicate: false },
{ created: true, duplicate: false },
{ created: false, duplicate: true },
],
);
assert.equal(results[0].asset.id, results[2].asset.id);
assert.equal(harness.service.list().length, 2);
assert.equal(harness.imageProcessor.calls.length, 2);
assert.equal(harness.storage.objects.size, 6);
});
test("all new images are decoded before the batch mutates SQLite or storage", async (t) => {
const expected = Object.assign(new Error("bad second image"), {
code: "MALFORMED_IMAGE",
statusCode: 422,
});
let processCount = 0;
const regularProcessor = createFakeImageProcessor();
const harness = createHarness({
imageProcessor: {
async process(filename) {
processCount += 1;
if (processCount === 2) throw expected;
return regularProcessor.process(filename);
},
},
});
t.after(() => harness.close());
await assert.rejects(
harness.service.upload([
harness.uploadFile("valid.png", "valid"),
harness.uploadFile("broken.png", "broken"),
]),
expected,
);
assert.equal(harness.service.list().length, 0);
assert.equal(harness.storage.objects.size, 0);
assert.equal(harness.storage.calls.length, 0);
});
test("concurrent identical uploads converge on one ready asset", async (t) => {
const storage = createMemoryStorage();
storage.setPutDelay(30);
const harness = createHarness({ storage });
t.after(() => harness.close());
const bytes = Buffer.from("racing identical upload");
const [[left], [right]] = await Promise.all([
harness.service.upload([harness.uploadFile("race.png", bytes)]),
harness.service.upload([harness.uploadFile("other-name.png", bytes)]),
]);
assert.equal(harness.service.list().length, 1);
assert.equal(left.asset.id, right.asset.id);
assert.deepEqual([left.created, right.created].sort(), [false, true]);
assert.deepEqual([left.duplicate, right.duplicate].sort(), [false, true]);
assert.equal(storage.calls.filter((call) => call.operation === "put").length, 3);
});
test("a partial storage failure leaves a diagnosable failed row and no ready URLs", async (t) => {
const storage = createMemoryStorage();
storage.setPutFailure(new StorageUnavailableError(new Error("offline")), 1);
const harness = createHarness({ storage });
t.after(() => harness.close());
await assert.rejects(
harness.service.upload([harness.uploadFile("failure.png", "failure bytes")]),
(error) => {
assert.equal(error.statusCode, 503);
assert.equal(error.details.retryable, true);
assert.match(error.details.assetId, /^[0-9a-f-]{36}$/);
return true;
},
);
const [failed] = harness.service.list();
assert.equal(failed.status, "failed");
assert.match(failed.processingError, /^STORAGE_UNAVAILABLE:/);
assert.equal(failed.display.url, null);
assert.equal(failed.thumbnail.url, null);
assert.equal(failed.original.url, null);
assert.equal(storage.objects.size, 0);
const row = rawAsset(harness.database, failed.id);
assert.match(
row.original_object_key,
new RegExp(`^assets/${failed.id}/original/[0-9a-f]{64}$`),
);
assert.equal(
row.display_object_key,
`assets/${failed.id}/display/${row.display_sha256}.webp`,
);
assert.equal(
row.thumbnail_object_key,
`assets/${failed.id}/thumbnail/${row.thumbnail_sha256}.webp`,
);
assert.equal(storage.calls.filter((call) => call.operation === "put").length, 2);
assert.deepEqual(
new Set(
storage.calls
.filter((call) => call.operation === "delete")
.map((call) => call.key),
),
new Set([
row.original_object_key,
row.display_object_key,
row.thumbnail_object_key,
]),
);
await assert.rejects(
harness.service.upload([harness.uploadFile("retry.png", "failure bytes")]),
(error) => error.code === "ASSET_PROCESSING_FAILED" && error.statusCode === 409,
);
assert.deepEqual(await harness.service.remove(failed.id), {
id: failed.id,
deleted: true,
});
assert.equal(harness.service.list().length, 0);
});
test("rename and expiration edits change only approved SQLite metadata", async (t) => {
const harness = createHarness();
t.after(() => harness.close());
const [upload] = await harness.service.upload([
harness.uploadFile("editable.png", "editable bytes"),
]);
const before = rawAsset(harness.database, upload.asset.id);
const expiring = harness.service.update(upload.asset.id, {
display_name: " New display name ",
expiration_at: "2035-05-06T07:08:09.000Z",
});
assert.equal(expiring.displayName, "New display name");
assert.equal(expiring.expirationAt, Date.parse("2035-05-06T07:08:09.000Z"));
const permanent = harness.service.update(upload.asset.id, { expirationAt: null });
assert.equal(permanent.expirationAt, null);
const after = rawAsset(harness.database, upload.asset.id);
assert.equal(after.original_object_key, before.original_object_key);
assert.equal(after.display_object_key, before.display_object_key);
assert.equal(after.thumbnail_object_key, before.thumbnail_object_key);
assert.equal(harness.storage.calls.filter((call) => call.operation === "put").length, 3);
await assert.rejects(
async () => harness.service.update(upload.asset.id, { status: "failed" }),
(error) => error.code === "INVALID_ASSET_UPDATE",
);
assert.throws(
() => harness.service.update(upload.asset.id, { expirationAt: -1 }),
(error) => error.code === "INVALID_ASSET_EXPIRATION",
);
});
test("references describe playlist, normal, and static use and block deletion", async (t) => {
const harness = createHarness();
t.after(() => harness.close());
const [upload] = await harness.service.upload([
harness.uploadFile("referenced.png", "referenced bytes"),
]);
const id = upload.asset.id;
const now = 2_100_000_000_000;
harness.database
.prepare(
"INSERT INTO playlists (id, name, created_at, updated_at) VALUES (?, ?, ?, ?)",
)
.run("playlist-1", "Sunday", now, now);
harness.database
.prepare(
`INSERT INTO playlist_entries
(id, playlist_id, asset_id, enabled, position, created_at, updated_at)
VALUES (?, ?, ?, 1, 0, ?, ?)`,
)
.run("entry-1", "playlist-1", id, now, now);
harness.database
.prepare(
`UPDATE playback_state
SET active_playlist_id = ?, current_entry_id = ?, static_asset_id = ?, updated_at = ?
WHERE output_id = 'global'`,
)
.run("playlist-1", "entry-1", id, now);
const references = harness.service.references(id);
assert.equal(references.canDelete, false);
assert.deepEqual(references.blockerCodes, [
"PLAYLIST_REFERENCES",
"CURRENT_PLAYBACK_TARGET",
"CURRENT_STATIC_TARGET",
]);
assert.deepEqual(references.playlists[0], {
playlistId: "playlist-1",
playlistName: "Sunday",
playlistEntryId: "entry-1",
enabled: true,
position: 0,
isActivePlaylist: true,
isCurrentNormalTarget: true,
});
assert.equal(references.playback[0].isCurrentNormalTarget, true);
assert.equal(references.playback[0].isCurrentStaticTarget, true);
const callsBeforeDelete = harness.storage.calls.length;
await assert.rejects(
harness.service.remove(id),
(error) => {
assert.equal(error.code, "ASSET_REFERENCED");
assert.equal(error.statusCode, 409);
assert.deepEqual(error.details.blockerCodes, [
"PLAYLIST_REFERENCES",
"CURRENT_PLAYBACK_TARGET",
"CURRENT_STATIC_TARGET",
]);
return true;
},
);
assert.equal(harness.storage.calls.length, callsBeforeDelete);
assert.equal(rawAsset(harness.database, id).status, "ready");
harness.database
.prepare(
`UPDATE playback_state
SET active_playlist_id = NULL, current_entry_id = NULL, static_asset_id = NULL
WHERE output_id = 'global'`,
)
.run();
harness.database.prepare("DELETE FROM playlist_entries WHERE id = ?").run("entry-1");
assert.equal(harness.service.references(id).canDelete, true);
await harness.service.remove(id);
assert.equal(harness.storage.objects.size, 0);
assert.equal(rawAsset(harness.database, id), undefined);
});
test("failed storage deletion retains a retryable deleting row", async (t) => {
const storage = createMemoryStorage();
const harness = createHarness({ storage });
t.after(() => harness.close());
const [upload] = await harness.service.upload([
harness.uploadFile("delete-me.png", "delete me"),
]);
storage.setDeleteFailure(new StorageUnavailableError(new Error("offline")));
await assert.rejects(harness.service.remove(upload.asset.id), (error) => {
assert.equal(error.statusCode, 503);
assert.equal(error.details.assetId, upload.asset.id);
assert.equal(error.details.retryable, true);
return true;
});
assert.equal(rawAsset(harness.database, upload.asset.id).status, "deleting");
assert.throws(
() => harness.service.update(upload.asset.id, { displayName: "Cannot edit" }),
(error) => error.code === "ASSET_DELETING",
);
await harness.service.remove(upload.asset.id);
assert.equal(rawAsset(harness.database, upload.asset.id), undefined);
assert.equal(storage.objects.size, 0);
});
test("processing assets cannot be deleted and storage is not touched", async (t) => {
const harness = createHarness();
t.after(() => harness.close());
const timestamp = Date.now();
harness.database
.prepare(
`INSERT INTO assets
(id, source_sha256, display_name, original_filename, status, created_at, updated_at)
VALUES (?, ?, ?, ?, 'processing', ?, ?)`,
)
.run(
"00000000-0000-4000-8000-999999999999",
"f".repeat(64),
"Still processing",
"processing.png",
timestamp,
timestamp,
);
assert.equal(
harness.service.references("00000000-0000-4000-8000-999999999999")
.canDelete,
false,
);
await assert.rejects(
harness.service.remove("00000000-0000-4000-8000-999999999999"),
(error) => error.code === "ASSET_PROCESSING" && error.statusCode === 409,
);
assert.equal(harness.storage.calls.length, 0);
assert.equal(
rawAsset(harness.database, "00000000-0000-4000-8000-999999999999").status,
"processing",
);
});
test("media descriptors only allow the recorded version and map missing bytes to 503", async (t) => {
const harness = createHarness();
t.after(() => harness.close());
const [upload] = await harness.service.upload([
harness.uploadFile("media.png", "media bytes"),
]);
const { asset } = upload;
const display = harness.service.mediaDescriptor(
asset.id,
"display",
asset.display.sha256,
);
assert.equal(display.contentType, "image/webp");
assert.equal(display.contentLength, asset.display.byteSize);
const stored = await harness.service.readStoredObject(display);
assert.deepEqual(await bufferBody(stored.body), harness.storage.objects.get(display.key).body);
for (const requestedHash of ["a".repeat(64), "../original/key", "not-a-hash"]) {
assert.throws(
() => harness.service.mediaDescriptor(asset.id, "display", requestedHash),
(error) => error.code === "MEDIA_NOT_FOUND" && error.statusCode === 404,
);
}
harness.storage.objects.delete(display.key);
await assert.rejects(harness.service.readStoredObject(display), (error) => {
assert.equal(error.code, "STORAGE_INCONSISTENCY");
assert.equal(error.statusCode, 503);
assert.deepEqual(error.details, { assetId: asset.id, retryable: true });
return true;
});
});
+41
View File
@@ -32,6 +32,14 @@ test("loadConfig validates and normalizes the supported environment", () => {
assert.equal(config.timeZone, "America/Chicago"); assert.equal(config.timeZone, "America/Chicago");
assert.equal(config.storage.endpoint, "http://seaweedfs:8333"); assert.equal(config.storage.endpoint, "http://seaweedfs:8333");
assert.equal(config.storage.forcePathStyle, true); assert.equal(config.storage.forcePathStyle, true);
assert.equal(config.storage.healthProbeIntervalMs, 30_000);
assert.equal(config.storage.connectionTimeoutMs, 2_000);
assert.equal(config.storage.requestTimeoutMs, 5_000);
assert.equal(config.storage.maxAttempts, 2);
assert.equal(config.processing.tempDirectory, "/tmp/announcements-uploads");
assert.equal(config.processing.maxFileBytes, 25 * 1024 * 1024);
assert.equal(config.processing.maxFiles, 20);
assert.equal(config.processing.maxInputPixels, 40_000_000);
}); });
test("production requires private S3 credentials", () => { test("production requires private S3 credentials", () => {
@@ -65,10 +73,42 @@ test("loadConfig rejects invalid typed values", () => {
() => loadConfig(validEnvironment({ S3_FORCE_PATH_STYLE: "sometimes" })), () => loadConfig(validEnvironment({ S3_FORCE_PATH_STYLE: "sometimes" })),
/must be a boolean/, /must be a boolean/,
); );
assert.throws(
() => loadConfig(validEnvironment({ S3_HEALTH_PROBE_INTERVAL_MS: "999" })),
/S3_HEALTH_PROBE_INTERVAL_MS must be at least 1000/,
);
assert.throws(
() => loadConfig(validEnvironment({ S3_REQUEST_TIMEOUT_MS: "999" })),
/S3_REQUEST_TIMEOUT_MS must be at least 1000/,
);
assert.throws(
() => loadConfig(validEnvironment({ S3_CONNECT_TIMEOUT_MS: "249" })),
/S3_CONNECT_TIMEOUT_MS must be at least 250/,
);
assert.throws(
() => loadConfig(validEnvironment({ S3_MAX_ATTEMPTS: "6" })),
/S3_MAX_ATTEMPTS must be at most 5/,
);
assert.throws( assert.throws(
() => loadConfig(validEnvironment({ DATABASE_PATH: "relative.sqlite" })), () => loadConfig(validEnvironment({ DATABASE_PATH: "relative.sqlite" })),
/must be an absolute path/, /must be an absolute path/,
); );
assert.throws(
() => loadConfig(validEnvironment({ UPLOAD_TEMP_DIRECTORY: "relative" })),
/UPLOAD_TEMP_DIRECTORY must be an absolute path/,
);
assert.throws(
() => loadConfig(validEnvironment({ UPLOAD_MAX_FILE_BYTES: "0" })),
/UPLOAD_MAX_FILE_BYTES must be at least 1/,
);
assert.throws(
() => loadConfig(validEnvironment({ UPLOAD_MAX_FILES: "101" })),
/UPLOAD_MAX_FILES must be at most 100/,
);
assert.throws(
() => loadConfig(validEnvironment({ IMAGE_MAX_INPUT_PIXELS: "no-limit" })),
/IMAGE_MAX_INPUT_PIXELS must be an integer/,
);
}); });
test("redactedConfig never returns credentials, passwords, paths, or webhook URLs", () => { test("redactedConfig never returns credentials, passwords, paths, or webhook URLs", () => {
@@ -83,5 +123,6 @@ test("redactedConfig never returns credentials, passwords, paths, or webhook URL
assert.doesNotMatch(serialized, /mqtt-password/); assert.doesNotMatch(serialized, /mqtt-password/);
assert.doesNotMatch(serialized, /webhook-secret/); assert.doesNotMatch(serialized, /webhook-secret/);
assert.doesNotMatch(serialized, /announcements\.sqlite/); assert.doesNotMatch(serialized, /announcements\.sqlite/);
assert.doesNotMatch(serialized, /seaweedfs/);
assert.doesNotMatch(serialized, /home-assistant/); assert.doesNotMatch(serialized, /home-assistant/);
}); });
+304
View File
@@ -0,0 +1,304 @@
const test = require("node:test");
const assert = require("node:assert/strict");
const crypto = require("node:crypto");
const fs = require("node:fs");
const os = require("node:os");
const path = require("node:path");
const sharp = require("sharp");
const {
DEFAULT_IMAGE_DERIVATIVE_OPTIONS,
ImageProcessingError,
createImageProcessor,
} = require("../src/image-processing");
const MAX_INPUT_PIXELS = 12_000_000;
const CONTENT_TYPES = Object.freeze({
jpeg: "image/jpeg",
png: "image/png",
webp: "image/webp",
});
function createProcessor(overrides = {}) {
return createImageProcessor({
maxInputPixels: MAX_INPUT_PIXELS,
...overrides,
});
}
function hash(buffer) {
return crypto.createHash("sha256").update(buffer).digest("hex");
}
async function solidImage(format, width, height, background = "#4a76b8") {
return sharp({
create: {
width,
height,
channels: 4,
background,
},
})
.toFormat(format)
.toBuffer();
}
async function twoFrameImage(format) {
const width = 20;
const frameHeight = 10;
const channels = 4;
const firstFrame = Buffer.alloc(width * frameHeight * channels);
const secondFrame = Buffer.alloc(width * frameHeight * channels);
for (let offset = 0; offset < firstFrame.length; offset += channels) {
firstFrame.set([255, 0, 0, 255], offset);
secondFrame.set([0, 0, 255, 255], offset);
}
const image = sharp(Buffer.concat([firstFrame, secondFrame]), {
raw: {
width,
height: frameHeight * 2,
channels,
pageHeight: frameHeight,
},
});
if (format === "tiff") {
return image.tiff().toBuffer();
}
return image[format]({ delay: [100, 100], loop: 0 }).toBuffer();
}
test("accepts decoded still JPEG, PNG, and WebP content", async (t) => {
const processor = createProcessor();
for (const format of ["jpeg", "png", "webp"]) {
await t.test(format, async () => {
const source = await solidImage(format, 640, 360);
const result = await processor.process(source);
assert.equal(result.original.format, format);
assert.equal(result.original.contentType, CONTENT_TYPES[format]);
assert.equal(result.original.byteSize, source.length);
assert.equal(result.original.sha256, hash(source));
assert.equal(result.original.width, 640);
assert.equal(result.original.height, 360);
assert.equal(result.original.pages, 1);
for (const derivative of [result.display, result.thumbnail]) {
assert.ok(Buffer.isBuffer(derivative.buffer));
assert.equal(derivative.format, "webp");
assert.equal(derivative.contentType, "image/webp");
assert.equal(derivative.byteSize, derivative.buffer.length);
assert.equal(derivative.sha256, hash(derivative.buffer));
assert.match(derivative.sha256, /^[0-9a-f]{64}$/);
}
});
}
});
test("uses bounding boxes without cropping or changing source aspect ratio", async () => {
const source = await solidImage("png", 3000, 1000);
const result = await createProcessor().process(source);
assert.deepEqual(
{ width: result.display.width, height: result.display.height },
{ width: 1920, height: 640 },
);
assert.deepEqual(
{ width: result.thumbnail.width, height: result.thumbnail.height },
{ width: 480, height: 160 },
);
assert.equal(result.display.width / result.display.height, 3);
assert.equal(result.thumbnail.width / result.thumbnail.height, 3);
});
test("does not enlarge a source that fits within both derivative bounds", async () => {
const source = await solidImage("jpeg", 320, 180);
const result = await createProcessor().process(source);
assert.deepEqual(
{ width: result.display.width, height: result.display.height },
{ width: 320, height: 180 },
);
assert.deepEqual(
{ width: result.thumbnail.width, height: result.thumbnail.height },
{ width: 320, height: 180 },
);
});
test("applies EXIF orientation and strips it from derivative metadata", async () => {
const source = await sharp({
create: {
width: 120,
height: 80,
channels: 3,
background: "#cb7235",
},
})
.jpeg()
.withMetadata({ orientation: 6 })
.toBuffer();
const result = await createProcessor().process(source);
const outputMetadata = await sharp(result.display.buffer).metadata();
assert.equal(result.original.orientation, 6);
assert.equal(result.original.encodedWidth, 120);
assert.equal(result.original.encodedHeight, 80);
assert.equal(result.original.width, 80);
assert.equal(result.original.height, 120);
assert.equal(result.display.width, 80);
assert.equal(result.display.height, 120);
assert.equal(outputMetadata.orientation, undefined);
assert.equal(outputMetadata.exif, undefined);
});
test("flattens transparent pixels against black in both derivatives", async () => {
const source = await solidImage("png", 40, 30, {
r: 255,
g: 0,
b: 0,
alpha: 0,
});
const result = await createProcessor().process(source);
assert.equal(result.original.hasAlpha, true);
for (const derivative of [result.display, result.thumbnail]) {
const metadata = await sharp(derivative.buffer).metadata();
const pixels = await sharp(derivative.buffer).raw().toBuffer();
assert.equal(metadata.hasAlpha, false);
assert.ok(
pixels.every((channel) => channel <= 2),
"fully transparent source pixels should become black",
);
}
});
test("supports a temporary file path without changing source identity", async (t) => {
const temporaryDirectory = fs.mkdtempSync(
path.join(os.tmpdir(), "announcements-image-processing-"),
);
t.after(() => fs.rmSync(temporaryDirectory, { recursive: true, force: true }));
const source = await solidImage("webp", 800, 600);
const filename = path.join(temporaryDirectory, "upload.tmp");
fs.writeFileSync(filename, source);
const result = await createProcessor().process(filename);
assert.equal(result.original.sha256, hash(source));
assert.equal(result.original.byteSize, source.length);
assert.equal(result.original.format, "webp");
});
test("rejects malformed and unsupported decoded content with typed errors", async () => {
const processor = createProcessor();
const gif = await solidImage("gif", 64, 64);
await assert.rejects(
processor.process(Buffer.from("not an image")),
(error) =>
error instanceof ImageProcessingError &&
error.code === "MALFORMED_IMAGE" &&
error.statusCode === 422,
);
await assert.rejects(
processor.process(gif),
(error) =>
error instanceof ImageProcessingError &&
error.code === "UNSUPPORTED_IMAGE_FORMAT" &&
error.statusCode === 415 &&
error.details.detectedFormat === "gif",
);
});
test("rejects animated WebP and GIF instead of selecting the first frame", async (t) => {
for (const format of ["webp", "gif"]) {
await t.test(format, async () => {
const animated = await twoFrameImage(format);
const metadata = await sharp(animated, { animated: true }).metadata();
assert.equal(metadata.pages, 2, "test fixture must contain two frames");
await assert.rejects(
createProcessor().process(animated),
(error) =>
error instanceof ImageProcessingError &&
error.code === "ANIMATED_IMAGE_NOT_SUPPORTED" &&
error.details.pages === 2,
);
});
}
});
test("rejects unexpected multipage inputs with a distinct typed error", async () => {
const multipageTiff = await twoFrameImage("tiff");
const metadata = await sharp(multipageTiff, { animated: true }).metadata();
assert.equal(metadata.pages, 2, "test fixture must contain two pages");
await assert.rejects(
createProcessor().process(multipageTiff),
(error) =>
error instanceof ImageProcessingError &&
error.code === "MULTIPAGE_IMAGE_NOT_SUPPORTED" &&
error.details.pages === 2,
);
});
test("enforces the injected decoded-pixel limit", async () => {
const source = await solidImage("png", 100, 100);
const processor = createImageProcessor({ maxInputPixels: 9_999 });
await assert.rejects(
processor.process(source),
(error) =>
error instanceof ImageProcessingError &&
error.code === "IMAGE_PIXEL_LIMIT_EXCEEDED" &&
error.statusCode === 413 &&
error.details.maxInputPixels === 9_999,
);
});
test("uses documented defaults while allowing validated target overrides", async () => {
assert.deepEqual(DEFAULT_IMAGE_DERIVATIVE_OPTIONS, {
display: { maxWidth: 1920, maxHeight: 1080, quality: 82 },
thumbnail: { maxWidth: 480, maxHeight: 270, quality: 72 },
});
const source = await solidImage("png", 900, 600);
const processor = createProcessor({
display: { maxWidth: 300, maxHeight: 300, quality: 90 },
thumbnail: { maxWidth: 100, maxHeight: 50, quality: 65 },
});
const result = await processor.process(source);
assert.deepEqual(
{ width: result.display.width, height: result.display.height },
{ width: 300, height: 200 },
);
assert.deepEqual(
{ width: result.thumbnail.width, height: result.thumbnail.height },
{ width: 75, height: 50 },
);
assert.throws(
() => createImageProcessor({ maxInputPixels: 0 }),
/maxInputPixels must be at least 1/,
);
assert.throws(
() =>
createImageProcessor({
maxInputPixels: 10_000,
display: { quality: 101 },
}),
/display\.quality must be at most 100/,
);
});
test("rejects invalid input interfaces with structured errors", async () => {
await assert.rejects(
createProcessor().process({ path: "/tmp/not-an-interface" }),
(error) =>
error instanceof ImageProcessingError &&
error.code === "INVALID_IMAGE_INPUT" &&
error.statusCode === 400,
);
});
+37 -5
View File
@@ -29,13 +29,33 @@ const silentLogger = {
error() {}, error() {},
}; };
function testStorage() {
return {
async start() {},
stop() {},
health() {
return {
status: "degraded",
checkedAt: "2026-08-16T00:00:00.000Z",
lastSuccessAt: null,
code: "STORAGE_UNAVAILABLE",
message: "Object storage is unavailable",
};
},
};
}
async function responseJson(baseUrl, path, init) { async function responseJson(baseUrl, path, init) {
const response = await fetch(`${baseUrl}${path}`, init); const response = await fetch(`${baseUrl}${path}`, init);
return { response, body: await response.json() }; return { response, body: await response.json() };
} }
test("HTTP foundation exposes live, ready, detailed health, and persisted settings", async (t) => { test("HTTP foundation exposes live, ready, detailed health, and persisted settings", async (t) => {
const runtime = createRuntime({ config: testConfig(), logger: silentLogger }); const runtime = createRuntime({
config: testConfig(),
logger: silentLogger,
storage: testStorage(),
});
await runtime.start(); await runtime.start();
t.after(() => runtime.stop("test cleanup")); t.after(() => runtime.stop("test cleanup"));
const baseUrl = `http://127.0.0.1:${runtime.address.port}`; const baseUrl = `http://127.0.0.1:${runtime.address.port}`;
@@ -52,7 +72,7 @@ test("HTTP foundation exposes live, ready, detailed health, and persisted settin
assert.equal(status.response.status, 200); assert.equal(status.response.status, 200);
assert.equal(status.body.status, "degraded"); assert.equal(status.body.status, "degraded");
assert.equal(status.body.services.database.currentSchemaVersion, 1); assert.equal(status.body.services.database.currentSchemaVersion, 1);
assert.equal(status.body.services.storage.status, "not_checked"); assert.equal(status.body.services.storage.status, "degraded");
const statusText = JSON.stringify(status.body); const statusText = JSON.stringify(status.body);
assert.doesNotMatch(statusText, /access-key-that-must-not-leak/); assert.doesNotMatch(statusText, /access-key-that-must-not-leak/);
assert.doesNotMatch(statusText, /secret-key-that-must-not-leak/); assert.doesNotMatch(statusText, /secret-key-that-must-not-leak/);
@@ -83,7 +103,11 @@ test("HTTP foundation exposes live, ready, detailed health, and persisted settin
}); });
test("invalid settings receive a structured client error", async (t) => { test("invalid settings receive a structured client error", async (t) => {
const runtime = createRuntime({ config: testConfig(), logger: silentLogger }); const runtime = createRuntime({
config: testConfig(),
logger: silentLogger,
storage: testStorage(),
});
await runtime.start(); await runtime.start();
t.after(() => runtime.stop("test cleanup")); t.after(() => runtime.stop("test cleanup"));
const baseUrl = `http://127.0.0.1:${runtime.address.port}`; const baseUrl = `http://127.0.0.1:${runtime.address.port}`;
@@ -99,7 +123,11 @@ test("invalid settings receive a structured client error", async (t) => {
}); });
test("liveness stays up while critical readiness failures return 503", async (t) => { test("liveness stays up while critical readiness failures return 503", async (t) => {
const runtime = createRuntime({ config: testConfig(), logger: silentLogger }); const runtime = createRuntime({
config: testConfig(),
logger: silentLogger,
storage: testStorage(),
});
await runtime.start(); await runtime.start();
t.after(() => runtime.stop("test cleanup")); t.after(() => runtime.stop("test cleanup"));
const baseUrl = `http://127.0.0.1:${runtime.address.port}`; const baseUrl = `http://127.0.0.1:${runtime.address.port}`;
@@ -127,7 +155,11 @@ test("liveness stays up while critical readiness failures return 503", async (t)
}); });
test("runtime startup and shutdown are idempotent", async () => { test("runtime startup and shutdown are idempotent", async () => {
const runtime = createRuntime({ config: testConfig(), logger: silentLogger }); const runtime = createRuntime({
config: testConfig(),
logger: silentLogger,
storage: testStorage(),
});
const [firstAddress, secondAddress] = await Promise.all([ const [firstAddress, secondAddress] = await Promise.all([
runtime.start(), runtime.start(),
runtime.start(), runtime.start(),
+105
View File
@@ -0,0 +1,105 @@
const test = require("node:test");
const assert = require("node:assert/strict");
const express = require("express");
const { openDatabase } = require("../src/database");
const { runMigrations } = require("../src/database/migrate");
const {
createDetailedHealthRouter,
createHealthRouter,
} = require("../src/health/routes");
function config() {
return {
environment: "test",
host: "127.0.0.1",
port: 0,
timeZone: "America/Chicago",
trustProxy: true,
database: { path: ":memory:" },
storage: {
endpoint: "http://private-storage:8333",
region: "us-east-1",
bucket: "announcements-v2",
accessKeyId: "must-not-leak",
secretAccessKey: "must-not-leak-either",
forcePathStyle: true,
},
processing: {
tempDirectory: "/tmp/announcements-uploads",
maxFileBytes: 1024,
maxFiles: 2,
maxInputPixels: 1000,
},
mqtt: { url: null },
notifications: { homeAssistantWebhookUrl: null },
};
}
async function withServer(storage, callback) {
const database = openDatabase({ path: ":memory:", busyTimeoutMs: 1000 });
const migrations = runMigrations(database);
const lifecycle = { ready: true };
const app = express();
const options = { config: config(), database, lifecycle, migrations, storage };
app.use("/health", createHealthRouter(options));
app.use("/api/health", createDetailedHealthRouter(options));
const server = app.listen(0, "127.0.0.1");
await new Promise((resolve) => server.once("listening", resolve));
try {
await callback(`http://127.0.0.1:${server.address().port}`);
} finally {
await new Promise((resolve) => server.close(resolve));
database.close();
}
}
test("storage degradation is reported but does not fail API readiness", async () => {
const storage = {
health() {
return {
status: "degraded",
checkedAt: "2026-08-16T12:00:00.000Z",
lastSuccessAt: "2026-08-16T11:59:00.000Z",
code: "STORAGE_UNAVAILABLE",
};
},
};
await withServer(storage, async (baseUrl) => {
const ready = await fetch(`${baseUrl}/health/ready`);
const readyBody = await ready.json();
assert.equal(ready.status, 200);
assert.equal(readyBody.status, "ready");
assert.equal(readyBody.checks.storage, "degraded");
const detailed = await fetch(`${baseUrl}/api/health`);
const detailedBody = await detailed.json();
assert.equal(detailed.status, 200);
assert.equal(detailedBody.status, "degraded");
assert.equal(detailedBody.services.storage.status, "degraded");
assert.equal(
detailedBody.services.storage.lastSuccessAt,
"2026-08-16T11:59:00.000Z",
);
assert.equal(detailedBody.services.storage.code, "STORAGE_UNAVAILABLE");
assert.doesNotMatch(JSON.stringify(detailedBody), /must-not-leak/);
assert.doesNotMatch(JSON.stringify(detailedBody), /private-storage/);
});
});
test("healthy storage produces healthy detailed status", async () => {
const storage = {
health() {
return { status: "healthy", checkedAt: "2026-08-16T12:00:00.000Z" };
},
};
await withServer(storage, async (baseUrl) => {
const detailed = await fetch(`${baseUrl}/api/health`);
const body = await detailed.json();
assert.equal(detailed.status, 200);
assert.equal(body.status, "ok");
assert.equal(body.services.storage.status, "healthy");
});
});
+397
View File
@@ -0,0 +1,397 @@
const test = require("node:test");
const assert = require("node:assert/strict");
const http = require("node:http");
const { PassThrough, Readable } = require("node:stream");
const {
StorageObjectNotFoundError,
StorageUnavailableError,
createStorageService,
} = require("../src/storage");
const storageConfig = {
endpoint: "http://seaweedfs:8333",
region: "us-east-1",
bucket: "announcements-v2",
accessKeyId: "test-key",
secretAccessKey: "test-secret",
forcePathStyle: true,
};
function fakeClient(handler) {
const commands = [];
return {
commands,
async send(command) {
commands.push(command);
return handler(command, commands.length - 1);
},
};
}
function commandName(command) {
return command.constructor.name;
}
test("storage initialization probes an existing bucket without creating it", async () => {
const client = fakeClient(async () => ({}));
const service = createStorageService(storageConfig, { client });
const health = await service.initialize();
assert.equal(health.status, "healthy");
assert.match(health.checkedAt, /T/);
assert.equal(health.lastSuccessAt, health.checkedAt);
assert.deepEqual(client.commands.map(commandName), ["HeadBucketCommand"]);
});
test("storage initialization creates only a missing bucket", async () => {
let exists = false;
const missing = Object.assign(new Error("missing"), {
name: "NoSuchBucket",
$metadata: { httpStatusCode: 404 },
});
const client = fakeClient(async (command) => {
if (commandName(command) === "HeadBucketCommand" && !exists) throw missing;
if (commandName(command) === "CreateBucketCommand") exists = true;
return {};
});
const service = createStorageService(storageConfig, { client });
assert.deepEqual(await service.ensureBucket(), { created: true });
assert.deepEqual(client.commands.map(commandName), [
"HeadBucketCommand",
"CreateBucketCommand",
]);
assert.equal(client.commands[1].input.Bucket, storageConfig.bucket);
assert.deepEqual(await service.ensureBucket(), { created: false });
assert.deepEqual(client.commands.map(commandName), [
"HeadBucketCommand",
"CreateBucketCommand",
"HeadBucketCommand",
]);
});
test("storage start performs a nonfatal immediate check and stop releases its client", async () => {
let destroyed = false;
const client = fakeClient(async () => ({}));
client.destroy = () => {
destroyed = true;
};
const service = createStorageService(
{ ...storageConfig, healthProbeIntervalMs: 1_000 },
{ client },
);
const health = await service.start();
assert.equal(health.status, "healthy");
assert.deepEqual(client.commands.map(commandName), ["HeadBucketCommand"]);
service.stop();
assert.equal(destroyed, true);
});
test("a create-bucket race is healthy only after the bucket can be verified", async () => {
const missing = Object.assign(new Error("missing"), {
name: "NoSuchBucket",
$metadata: { httpStatusCode: 404 },
});
const conflict = Object.assign(new Error("race"), {
name: "BucketAlreadyOwnedByYou",
$metadata: { httpStatusCode: 409 },
});
let headCalls = 0;
const client = fakeClient(async (command) => {
if (commandName(command) === "HeadBucketCommand") {
headCalls += 1;
if (headCalls === 1) throw missing;
return {};
}
if (commandName(command) === "CreateBucketCommand") throw conflict;
throw new Error(`Unexpected ${commandName(command)}`);
});
const service = createStorageService(storageConfig, { client });
assert.deepEqual(await service.ensureBucket(), { created: false });
assert.equal(service.health().status, "healthy");
assert.deepEqual(client.commands.map(commandName), [
"HeadBucketCommand",
"CreateBucketCommand",
"HeadBucketCommand",
]);
});
test("an older failed probe cannot overwrite a newer successful operation", async () => {
let rejectProbe;
let notifyProbeStarted;
const probeStarted = new Promise((resolve) => {
notifyProbeStarted = resolve;
});
let headCalls = 0;
const client = fakeClient(async (command) => {
if (commandName(command) === "HeadBucketCommand") {
headCalls += 1;
if (headCalls === 1) return {};
notifyProbeStarted();
return new Promise((resolve, reject) => {
rejectProbe = reject;
});
}
if (commandName(command) === "PutObjectCommand") return {};
throw new Error(`Unexpected ${commandName(command)}`);
});
const service = createStorageService(storageConfig, { client });
await service.initialize();
const staleProbe = service.probe();
await probeStarted;
await service.putObject({
key: "assets/id/original/hash",
body: Buffer.from("bytes"),
contentLength: 5,
});
const successfulState = service.health();
rejectProbe(new Error("older request failed"));
await staleProbe;
assert.equal(successfulState.status, "healthy");
assert.deepEqual(service.health(), successfulState);
});
test("temporary storage failure degrades without throwing during initialization and recovers", async () => {
let available = false;
const warnings = [];
const client = fakeClient(async () => {
if (!available) throw new Error("connection refused at a private address");
return {};
});
const service = createStorageService(storageConfig, {
client,
logger: { warn(message) { warnings.push(message); } },
});
const initial = await service.initialize();
assert.equal(initial.status, "degraded");
assert.equal(initial.code, "STORAGE_UNAVAILABLE");
assert.equal(initial.lastSuccessAt, null);
assert.deepEqual(warnings, ["Object storage is unavailable during initialization"]);
available = true;
const recovered = await service.probe();
assert.equal(recovered.status, "healthy");
assert.equal(recovered.lastSuccessAt, recovered.checkedAt);
assert.equal(service.health().status, "healthy");
available = false;
const degradedAgain = await service.probe();
assert.equal(degradedAgain.status, "degraded");
assert.equal(degradedAgain.lastSuccessAt, recovered.lastSuccessAt);
});
test("put, get, head, existence, and delete use the configured private bucket", async () => {
const body = Readable.from([Buffer.from("image bytes")]);
const client = fakeClient(async (command) => {
switch (commandName(command)) {
case "PutObjectCommand":
return { ETag: '"etag"' };
case "GetObjectCommand":
return {
Body: body,
ContentType: "image/webp",
ContentLength: 11,
ETag: '"etag"',
};
case "HeadObjectCommand":
return {
ContentType: "image/webp",
ContentLength: 11,
ETag: '"etag"',
};
case "DeleteObjectCommand":
return {};
case "HeadBucketCommand":
return {};
default:
throw new Error(`Unexpected ${commandName(command)}`);
}
});
const service = createStorageService(storageConfig, { client });
const key = "assets/id/display/hash.webp";
await service.initialize();
client.commands.length = 0;
const put = await service.putObject({
key,
body: Buffer.from("image bytes"),
contentType: "image/webp",
contentLength: 11,
});
assert.deepEqual(put, { key, etag: '"etag"', versionId: null });
const fetched = await service.getObject(key);
assert.equal(fetched.body, body);
assert.equal(fetched.contentType, "image/webp");
assert.equal(fetched.contentLength, 11);
assert.equal((await service.headObject(key)).etag, '"etag"');
assert.equal(await service.objectExists(key), true);
await service.deleteObject(key);
for (const command of client.commands) {
assert.equal(command.input.Bucket, storageConfig.bucket);
assert.equal(command.input.Key, key);
}
});
test("missing objects are distinct from unavailable storage", async () => {
let missing = true;
const client = fakeClient(async (command) => {
if (commandName(command) === "HeadBucketCommand") return {};
if (missing) {
throw Object.assign(new Error("missing"), {
name: "NoSuchKey",
$metadata: { httpStatusCode: 404 },
});
}
throw new Error("network offline");
});
const service = createStorageService(storageConfig, { client });
await service.initialize();
client.commands.length = 0;
assert.equal(await service.headObject("assets/id/original/hash"), null);
assert.equal(await service.objectExists("assets/id/original/hash"), false);
await assert.rejects(
service.getObject("assets/id/original/hash"),
StorageObjectNotFoundError,
);
assert.equal(service.health().status, "healthy");
missing = false;
await assert.rejects(
service.putObject({ key: "assets/id/original/hash", body: Buffer.alloc(0) }),
(error) =>
error instanceof StorageUnavailableError &&
error.code === "STORAGE_UNAVAILABLE" &&
error.statusCode === 503,
);
assert.equal(service.health().status, "degraded");
});
test("a missing bucket during object access is a storage outage, not a missing object", async () => {
let initialized = false;
const client = fakeClient(async (command) => {
if (commandName(command) === "HeadBucketCommand" && !initialized) {
initialized = true;
return {};
}
throw Object.assign(new Error("bucket missing"), {
name: "NoSuchBucket",
$metadata: { httpStatusCode: 404 },
});
});
const service = createStorageService(storageConfig, { client });
await service.initialize();
await assert.rejects(
service.getObject("assets/id/original/hash"),
StorageUnavailableError,
);
assert.equal(service.health().status, "degraded");
});
test("storage rejects unsafe internal object keys before contacting S3", async () => {
const client = fakeClient(async () => ({}));
const service = createStorageService(storageConfig, { client });
await assert.rejects(
service.getObject("../private"),
/Storage object key is invalid/,
);
await assert.rejects(
service.deleteObject("/absolute"),
/Storage object key is invalid/,
);
assert.equal(client.commands.length, 0);
});
test("real HTTP requests time out instead of hanging indefinitely", async (t) => {
const server = http.createServer(() => {
// Intentionally accept the connection without sending response headers.
});
server.listen(0, "127.0.0.1");
await new Promise((resolve) => server.once("listening", resolve));
t.after(async () => {
server.closeAllConnections();
await new Promise((resolve) => server.close(resolve));
});
const service = createStorageService({
...storageConfig,
endpoint: `http://127.0.0.1:${server.address().port}`,
connectionTimeoutMs: 100,
requestTimeoutMs: 100,
maxAttempts: 1,
});
t.after(() => service.stop());
const startedAt = Date.now();
const health = await service.probe();
assert.equal(health.status, "degraded");
assert.equal(health.code, "STORAGE_UNAVAILABLE");
assert.ok(Date.now() - startedAt < 2_000);
});
test("consumer stream cancellation does not falsely degrade storage", async () => {
let body = new PassThrough();
const client = fakeClient(async (command) => {
if (commandName(command) === "HeadBucketCommand") return {};
if (commandName(command) === "GetObjectCommand") {
return { Body: body, ContentLength: 10, ContentType: "image/webp" };
}
throw new Error(`Unexpected ${commandName(command)}`);
});
const service = createStorageService(storageConfig, { client });
await service.initialize();
const cancelled = await service.getObject("assets/id/display/hash.webp");
cancelled.body.destroy(
Object.assign(new Error("consumer closed"), {
code: "ERR_STREAM_PREMATURE_CLOSE",
}),
);
await new Promise((resolve) => setImmediate(resolve));
assert.equal(service.health().status, "healthy");
body = new PassThrough();
const failed = await service.getObject("assets/id/display/hash.webp");
failed.body.destroy(Object.assign(new Error("upstream reset"), { code: "ECONNRESET" }));
await new Promise((resolve) => setImmediate(resolve));
assert.equal(service.health().status, "degraded");
});
test("delete treats an already-missing object as an idempotent success", async () => {
let missingBucket = false;
const client = fakeClient(async (command) => {
if (commandName(command) === "HeadBucketCommand") return {};
if (commandName(command) === "DeleteObjectCommand") {
throw Object.assign(new Error("missing"), {
name: missingBucket ? "NoSuchBucket" : "NoSuchKey",
$metadata: { httpStatusCode: 404 },
});
}
throw new Error(`Unexpected ${commandName(command)}`);
});
const service = createStorageService(storageConfig, { client });
await service.initialize();
await assert.doesNotReject(
service.deleteObject("assets/id/original/hash"),
);
assert.equal(service.health().status, "healthy");
missingBucket = true;
await assert.rejects(
service.deleteObject("assets/id/original/hash"),
StorageUnavailableError,
);
});