Implement Phase 2 asset storage pipeline
This commit is contained in:
@@ -6,6 +6,7 @@
|
|||||||
.env.*
|
.env.*
|
||||||
!.env.example
|
!.env.example
|
||||||
**/node_modules
|
**/node_modules
|
||||||
|
.pnpm-store
|
||||||
**/dist
|
**/dist
|
||||||
**/*.log
|
**/*.log
|
||||||
**/*.sqlite
|
**/*.sqlite
|
||||||
|
|||||||
+13
-1
@@ -25,8 +25,20 @@ S3_REGION=us-east-1
|
|||||||
S3_BUCKET=announcements-v2
|
S3_BUCKET=announcements-v2
|
||||||
S3_ACCESS_KEY_ID=replace-with-a-random-access-key
|
S3_ACCESS_KEY_ID=replace-with-a-random-access-key
|
||||||
S3_SECRET_ACCESS_KEY=replace-with-a-long-random-secret
|
S3_SECRET_ACCESS_KEY=replace-with-a-long-random-secret
|
||||||
|
S3_CONNECT_TIMEOUT_MS=2000
|
||||||
|
# Maximum socket inactivity during one S3 operation (not a total upload deadline).
|
||||||
|
S3_REQUEST_TIMEOUT_MS=5000
|
||||||
|
S3_MAX_ATTEMPTS=2
|
||||||
|
S3_HEALTH_PROBE_INTERVAL_MS=30000
|
||||||
|
|
||||||
# Optional integrations. Leave blank to disable them during Phase 1.
|
# Image upload and decode limits. The default proxy body limit is 512 MiB,
|
||||||
|
# which accommodates the default twenty 25 MiB files plus multipart overhead.
|
||||||
|
UPLOAD_MAX_FILE_BYTES=26214400
|
||||||
|
UPLOAD_MAX_FILES=20
|
||||||
|
IMAGE_MAX_INPUT_PIXELS=40000000
|
||||||
|
# UPLOAD_TEMP_DIRECTORY=/tmp/announcements-uploads
|
||||||
|
|
||||||
|
# Optional integrations. Leave blank to disable them during Phase 2.
|
||||||
MQTT_URL=
|
MQTT_URL=
|
||||||
MQTT_USERNAME=
|
MQTT_USERNAME=
|
||||||
MQTT_PASSWORD=
|
MQTT_PASSWORD=
|
||||||
|
|||||||
@@ -1,4 +1,5 @@
|
|||||||
node_modules/
|
node_modules/
|
||||||
|
.pnpm-store/
|
||||||
|
|
||||||
# Local configuration and secrets
|
# Local configuration and secrets
|
||||||
.env
|
.env
|
||||||
|
|||||||
@@ -0,0 +1,93 @@
|
|||||||
|
# Announcements v2
|
||||||
|
|
||||||
|
Announcements is a small Express, React, Socket.IO, MQTT, SQLite, and
|
||||||
|
S3-compatible signage application. Version 2 is being built in phases and is
|
||||||
|
intended to run beside the existing production version until cutover.
|
||||||
|
|
||||||
|
## Development
|
||||||
|
|
||||||
|
The repository is a pnpm workspace with one lockfile. It requires Node.js
|
||||||
|
22.13 or newer and pnpm 10.29.3.
|
||||||
|
|
||||||
|
```sh
|
||||||
|
cp .env.example .env
|
||||||
|
pnpm install --frozen-lockfile
|
||||||
|
pnpm check
|
||||||
|
docker compose up --build
|
||||||
|
```
|
||||||
|
|
||||||
|
Only the Nginx web service publishes a host port (5175 by default). The API and
|
||||||
|
SeaweedFS S3 endpoint remain private on Compose networks. Replace the example
|
||||||
|
S3 credentials before starting the stack.
|
||||||
|
|
||||||
|
## Library assets
|
||||||
|
|
||||||
|
`POST /api/assets` accepts multipart uploads in the repeated `file` or `files`
|
||||||
|
field. An optional `display_name` (or `displayName`) is accepted for a
|
||||||
|
single-file upload. Supported decoded formats are still JPEG, PNG, and WebP;
|
||||||
|
extensions and multipart content types are not trusted. Animated, multipage,
|
||||||
|
malformed, unsupported, oversized, and excessive-pixel inputs are rejected.
|
||||||
|
|
||||||
|
Default limits are:
|
||||||
|
|
||||||
|
- 25 MiB per file (`UPLOAD_MAX_FILE_BYTES`)
|
||||||
|
- 20 files per request (`UPLOAD_MAX_FILES`)
|
||||||
|
- 40 million decoded pixels per image (`IMAGE_MAX_INPUT_PIXELS`)
|
||||||
|
|
||||||
|
Nginx accepts up to 512 MiB per request, enough for the default aggregate plus
|
||||||
|
multipart overhead. If the backend limits are raised, update
|
||||||
|
`client_max_body_size` in `client/nginx.conf` as well. Proxied API requests use
|
||||||
|
a ten-minute inactivity timeout so a valid large batch can finish processing.
|
||||||
|
|
||||||
|
Original bytes are archived unchanged. Content identity is the SHA-256 of
|
||||||
|
those bytes, so uploading the same content returns the existing asset even
|
||||||
|
when the filename differs. Display-name collisions with different content
|
||||||
|
return `409 ASSET_NAME_CONFLICT`. Multi-file uploads validate every new image
|
||||||
|
locally before persisting any of them. If a later storage operation fails,
|
||||||
|
already completed items remain valid and retrying the request is idempotent.
|
||||||
|
|
||||||
|
Sharp produces two metadata-free, auto-oriented WebP renditions:
|
||||||
|
|
||||||
|
- display: inside-fit at most 1920×1080, quality 82
|
||||||
|
- thumbnail: inside-fit at most 480×270, quality 72
|
||||||
|
|
||||||
|
Neither rendition crops or upscales. Transparency is flattened against black.
|
||||||
|
Presentation effects such as blur, shadows, darkening, and framing are not
|
||||||
|
baked into stored media.
|
||||||
|
|
||||||
|
## Asset and media API
|
||||||
|
|
||||||
|
- `GET /api/assets`
|
||||||
|
- `POST /api/assets`
|
||||||
|
- `GET /api/assets/:assetId`
|
||||||
|
- `PATCH /api/assets/:assetId`
|
||||||
|
- `GET /api/assets/:assetId/references`
|
||||||
|
- `DELETE /api/assets/:assetId`
|
||||||
|
- `GET /api/assets/:assetId/original`
|
||||||
|
- `GET /media/assets/:assetId/display/:sha256.webp`
|
||||||
|
- `GET /media/assets/:assetId/thumbnail/:sha256.webp`
|
||||||
|
|
||||||
|
PATCH accepts only display name and expiration. A null expiration is permanent.
|
||||||
|
Manual deletion refuses playlist, current playback, and static references. It
|
||||||
|
marks an asset as deleting before idempotently removing its three objects and
|
||||||
|
removes SQLite metadata only after storage succeeds. A failed processing row
|
||||||
|
can be inspected, safely deleted, and uploaded again; no automatic retry loop
|
||||||
|
is run.
|
||||||
|
|
||||||
|
Media URLs are derived from SQLite metadata and never accept object keys. The
|
||||||
|
display and thumbnail routes stream private S3 objects through the API with a
|
||||||
|
one-year immutable cache policy. Original downloads use private, no-store
|
||||||
|
caching.
|
||||||
|
|
||||||
|
## Storage and health
|
||||||
|
|
||||||
|
The API uses a small path-style S3 adapter configured by `S3_ENDPOINT`,
|
||||||
|
`S3_REGION`, `S3_BUCKET`, `S3_ACCESS_KEY_ID`, and `S3_SECRET_ACCESS_KEY`.
|
||||||
|
Connection timeout, socket-inactivity timeout (`S3_REQUEST_TIMEOUT_MS`),
|
||||||
|
attempts, and background probe interval are configurable in `.env.example`.
|
||||||
|
Startup verifies or safely creates only the configured bucket.
|
||||||
|
|
||||||
|
Storage outages do not stop the API or fail process readiness. `/health/ready`
|
||||||
|
reports storage as degraded while remaining HTTP 200 when SQLite and the API
|
||||||
|
runtime are healthy. `/api/health` and `/api/status` expose cached, redacted
|
||||||
|
storage state and recover after a successful probe or storage operation.
|
||||||
@@ -10,6 +10,9 @@ server {
|
|||||||
|
|
||||||
root /usr/share/nginx/html;
|
root /usr/share/nginx/html;
|
||||||
index index.html;
|
index index.html;
|
||||||
|
# Matches the default API allowance of twenty 25 MiB files plus multipart
|
||||||
|
# overhead. Keep this in sync if the backend upload limits are raised.
|
||||||
|
client_max_body_size 512m;
|
||||||
|
|
||||||
resolver 127.0.0.11 valid=10s ipv6=off;
|
resolver 127.0.0.11 valid=10s ipv6=off;
|
||||||
set $api_origin http://api:7532;
|
set $api_origin http://api:7532;
|
||||||
@@ -17,6 +20,11 @@ server {
|
|||||||
location ~ ^/(api|health|media)(/|$) {
|
location ~ ^/(api|health|media)(/|$) {
|
||||||
proxy_pass $api_origin;
|
proxy_pass $api_origin;
|
||||||
proxy_http_version 1.1;
|
proxy_http_version 1.1;
|
||||||
|
proxy_connect_timeout 5s;
|
||||||
|
# Multipart requests return only after image processing and private-S3
|
||||||
|
# writes complete. This is an inactivity timeout, not a size limit.
|
||||||
|
proxy_read_timeout 10m;
|
||||||
|
proxy_send_timeout 10m;
|
||||||
proxy_set_header Host $host;
|
proxy_set_header Host $host;
|
||||||
proxy_set_header X-Real-IP $remote_addr;
|
proxy_set_header X-Real-IP $remote_addr;
|
||||||
proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
|
proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
|
||||||
|
|||||||
@@ -42,6 +42,14 @@ services:
|
|||||||
S3_ACCESS_KEY_ID: "${S3_ACCESS_KEY_ID:?Set S3_ACCESS_KEY_ID in .env}"
|
S3_ACCESS_KEY_ID: "${S3_ACCESS_KEY_ID:?Set S3_ACCESS_KEY_ID in .env}"
|
||||||
S3_SECRET_ACCESS_KEY: "${S3_SECRET_ACCESS_KEY:?Set S3_SECRET_ACCESS_KEY in .env}"
|
S3_SECRET_ACCESS_KEY: "${S3_SECRET_ACCESS_KEY:?Set S3_SECRET_ACCESS_KEY in .env}"
|
||||||
S3_FORCE_PATH_STYLE: "true"
|
S3_FORCE_PATH_STYLE: "true"
|
||||||
|
S3_CONNECT_TIMEOUT_MS: "${S3_CONNECT_TIMEOUT_MS:-2000}"
|
||||||
|
S3_REQUEST_TIMEOUT_MS: "${S3_REQUEST_TIMEOUT_MS:-5000}"
|
||||||
|
S3_MAX_ATTEMPTS: "${S3_MAX_ATTEMPTS:-2}"
|
||||||
|
S3_HEALTH_PROBE_INTERVAL_MS: "${S3_HEALTH_PROBE_INTERVAL_MS:-30000}"
|
||||||
|
UPLOAD_TEMP_DIRECTORY: /tmp/announcements-uploads
|
||||||
|
UPLOAD_MAX_FILE_BYTES: "${UPLOAD_MAX_FILE_BYTES:-26214400}"
|
||||||
|
UPLOAD_MAX_FILES: "${UPLOAD_MAX_FILES:-20}"
|
||||||
|
IMAGE_MAX_INPUT_PIXELS: "${IMAGE_MAX_INPUT_PIXELS:-40000000}"
|
||||||
MQTT_URL: "${MQTT_URL:-}"
|
MQTT_URL: "${MQTT_URL:-}"
|
||||||
MQTT_USERNAME: "${MQTT_USERNAME:-}"
|
MQTT_USERNAME: "${MQTT_USERNAME:-}"
|
||||||
MQTT_PASSWORD: "${MQTT_PASSWORD:-}"
|
MQTT_PASSWORD: "${MQTT_PASSWORD:-}"
|
||||||
|
|||||||
Generated
+752
File diff suppressed because it is too large
Load Diff
@@ -4,3 +4,4 @@ packages:
|
|||||||
|
|
||||||
onlyBuiltDependencies:
|
onlyBuiltDependencies:
|
||||||
- better-sqlite3
|
- better-sqlite3
|
||||||
|
- sharp
|
||||||
|
|||||||
@@ -13,9 +13,12 @@
|
|||||||
"node": ">=22.13.0"
|
"node": ">=22.13.0"
|
||||||
},
|
},
|
||||||
"dependencies": {
|
"dependencies": {
|
||||||
|
"@aws-sdk/client-s3": "3.1101.0",
|
||||||
"better-sqlite3": "13.0.2",
|
"better-sqlite3": "13.0.2",
|
||||||
"dotenv": "^16.5.0",
|
"dotenv": "^16.5.0",
|
||||||
"express": "^5.2.1",
|
"express": "^5.2.1",
|
||||||
|
"multer": "2.0.2",
|
||||||
|
"sharp": "0.35.3",
|
||||||
"socket.io": "^4.8.3"
|
"socket.io": "^4.8.3"
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
+23
-2
@@ -3,12 +3,27 @@ const {
|
|||||||
createDetailedHealthRouter,
|
createDetailedHealthRouter,
|
||||||
createHealthRouter,
|
createHealthRouter,
|
||||||
} = require("./health/routes");
|
} = require("./health/routes");
|
||||||
|
const { createAssetRouter } = require("./assets/routes");
|
||||||
|
const { createMediaRouter } = require("./assets/media-routes");
|
||||||
|
const { createAssetService } = require("./assets/service");
|
||||||
const { createSettingsRouter } = require("./settings/routes");
|
const { createSettingsRouter } = require("./settings/routes");
|
||||||
const { createSettingsService } = require("./settings/service");
|
const { createSettingsService } = require("./settings/service");
|
||||||
|
|
||||||
function createApp({ config, database, lifecycle, migrations }) {
|
function createApp({
|
||||||
|
config,
|
||||||
|
database,
|
||||||
|
imageProcessor,
|
||||||
|
lifecycle,
|
||||||
|
migrations,
|
||||||
|
storage,
|
||||||
|
}) {
|
||||||
const app = express();
|
const app = express();
|
||||||
const settingsService = createSettingsService(database);
|
const settingsService = createSettingsService(database);
|
||||||
|
const assetService = createAssetService({
|
||||||
|
database,
|
||||||
|
imageProcessor,
|
||||||
|
storage,
|
||||||
|
});
|
||||||
|
|
||||||
app.disable("x-powered-by");
|
app.disable("x-powered-by");
|
||||||
app.set("trust proxy", config.trustProxy);
|
app.set("trust proxy", config.trustProxy);
|
||||||
@@ -19,13 +34,14 @@ function createApp({ config, database, lifecycle, migrations }) {
|
|||||||
});
|
});
|
||||||
app.use(
|
app.use(
|
||||||
"/health",
|
"/health",
|
||||||
createHealthRouter({ database, lifecycle, migrations }),
|
createHealthRouter({ database, lifecycle, migrations, storage }),
|
||||||
);
|
);
|
||||||
const detailedHealthRouterOptions = {
|
const detailedHealthRouterOptions = {
|
||||||
config,
|
config,
|
||||||
database,
|
database,
|
||||||
lifecycle,
|
lifecycle,
|
||||||
migrations,
|
migrations,
|
||||||
|
storage,
|
||||||
};
|
};
|
||||||
app.use(
|
app.use(
|
||||||
"/api/health",
|
"/api/health",
|
||||||
@@ -36,6 +52,11 @@ function createApp({ config, database, lifecycle, migrations }) {
|
|||||||
createDetailedHealthRouter(detailedHealthRouterOptions),
|
createDetailedHealthRouter(detailedHealthRouterOptions),
|
||||||
);
|
);
|
||||||
app.use("/api/settings", createSettingsRouter(settingsService));
|
app.use("/api/settings", createSettingsRouter(settingsService));
|
||||||
|
app.use(
|
||||||
|
"/api/assets",
|
||||||
|
createAssetRouter({ assetService, processingConfig: config.processing }),
|
||||||
|
);
|
||||||
|
app.use("/media", createMediaRouter(assetService));
|
||||||
|
|
||||||
app.use((request, response) => {
|
app.use((request, response) => {
|
||||||
response.status(404).json({
|
response.status(404).json({
|
||||||
|
|||||||
@@ -0,0 +1,23 @@
|
|||||||
|
class AssetError extends Error {
|
||||||
|
constructor(code, message, { statusCode = 400, details, cause } = {}) {
|
||||||
|
super(message, cause ? { cause } : undefined);
|
||||||
|
this.name = "AssetError";
|
||||||
|
this.code = code;
|
||||||
|
this.statusCode = statusCode;
|
||||||
|
if (details !== undefined) {
|
||||||
|
this.details = details;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
function assetNotFound(assetId) {
|
||||||
|
return new AssetError("ASSET_NOT_FOUND", "Library asset not found", {
|
||||||
|
statusCode: 404,
|
||||||
|
details: { assetId },
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
module.exports = {
|
||||||
|
AssetError,
|
||||||
|
assetNotFound,
|
||||||
|
};
|
||||||
@@ -0,0 +1,49 @@
|
|||||||
|
const express = require("express");
|
||||||
|
const { sendImmutableMedia } = require("./streaming");
|
||||||
|
|
||||||
|
function createMediaRouter(assetService) {
|
||||||
|
const router = express.Router();
|
||||||
|
|
||||||
|
async function sendMedia(request, response, next, kind) {
|
||||||
|
const version = request.params.version;
|
||||||
|
if (!version.endsWith(".webp")) {
|
||||||
|
response.status(404).json({
|
||||||
|
error: {
|
||||||
|
code: "MEDIA_NOT_FOUND",
|
||||||
|
message: "Asset media version not found",
|
||||||
|
},
|
||||||
|
});
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
try {
|
||||||
|
const descriptor = assetService.mediaDescriptor(
|
||||||
|
request.params.assetId,
|
||||||
|
kind,
|
||||||
|
version.slice(0, -".webp".length),
|
||||||
|
);
|
||||||
|
await sendImmutableMedia(assetService, response, descriptor);
|
||||||
|
} catch (error) {
|
||||||
|
if (response.headersSent) {
|
||||||
|
response.destroy(error);
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
next(error);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
router.get("/assets/:assetId/display/:version", (request, response, next) =>
|
||||||
|
sendMedia(request, response, next, "display"),
|
||||||
|
);
|
||||||
|
router.get(
|
||||||
|
"/assets/:assetId/thumbnail/:version",
|
||||||
|
(request, response, next) =>
|
||||||
|
sendMedia(request, response, next, "thumbnail"),
|
||||||
|
);
|
||||||
|
|
||||||
|
return router;
|
||||||
|
}
|
||||||
|
|
||||||
|
module.exports = {
|
||||||
|
createMediaRouter,
|
||||||
|
};
|
||||||
@@ -0,0 +1,235 @@
|
|||||||
|
const crypto = require("node:crypto");
|
||||||
|
const fs = require("node:fs");
|
||||||
|
const path = require("node:path");
|
||||||
|
const multer = require("multer");
|
||||||
|
const { AssetError } = require("./errors");
|
||||||
|
|
||||||
|
const ALLOWED_FILE_FIELDS = new Set(["file", "files"]);
|
||||||
|
const ALLOWED_TEXT_FIELDS = new Set(["display_name", "displayName"]);
|
||||||
|
const MAX_FIELD_BYTES = 4 * 1024;
|
||||||
|
const MAX_ORIGINAL_FILENAME_LENGTH = 255;
|
||||||
|
const TEMPORARY_UPLOAD_PATHS = Symbol("temporaryUploadPaths");
|
||||||
|
const TEMP_STORAGE_ERROR_CODES = new Set([
|
||||||
|
"EACCES",
|
||||||
|
"EEXIST",
|
||||||
|
"EIO",
|
||||||
|
"EMFILE",
|
||||||
|
"ENFILE",
|
||||||
|
"ENOSPC",
|
||||||
|
"ENOTDIR",
|
||||||
|
"EPERM",
|
||||||
|
"EROFS",
|
||||||
|
]);
|
||||||
|
|
||||||
|
function safeOriginalFilename(value) {
|
||||||
|
const normalized = String(value || "")
|
||||||
|
.replaceAll("\\", "/")
|
||||||
|
.replace(/[\u0000-\u001f\u007f]/g, "");
|
||||||
|
const leafName = path.posix.basename(normalized);
|
||||||
|
const bounded = [...leafName].slice(0, MAX_ORIGINAL_FILENAME_LENGTH).join("");
|
||||||
|
|
||||||
|
if (!bounded.trim()) {
|
||||||
|
throw new AssetError(
|
||||||
|
"INVALID_ORIGINAL_FILENAME",
|
||||||
|
"Uploaded files must include a usable filename",
|
||||||
|
);
|
||||||
|
}
|
||||||
|
return bounded;
|
||||||
|
}
|
||||||
|
|
||||||
|
function mapMulterError(error, config) {
|
||||||
|
if (!(error instanceof multer.MulterError)) {
|
||||||
|
if (TEMP_STORAGE_ERROR_CODES.has(error?.code)) {
|
||||||
|
return new AssetError(
|
||||||
|
"UPLOAD_TEMP_STORAGE_UNAVAILABLE",
|
||||||
|
"Temporary upload storage is unavailable",
|
||||||
|
{ statusCode: 503, cause: error },
|
||||||
|
);
|
||||||
|
}
|
||||||
|
return new AssetError(
|
||||||
|
"INVALID_MULTIPART_UPLOAD",
|
||||||
|
"The multipart upload could not be read",
|
||||||
|
{ cause: error },
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
if (error.code === "LIMIT_FILE_SIZE") {
|
||||||
|
return new AssetError(
|
||||||
|
"UPLOAD_FILE_TOO_LARGE",
|
||||||
|
"An uploaded file exceeds the configured size limit",
|
||||||
|
{
|
||||||
|
statusCode: 413,
|
||||||
|
details: { maxFileBytes: config.maxFileBytes },
|
||||||
|
},
|
||||||
|
);
|
||||||
|
}
|
||||||
|
if (error.code === "LIMIT_FILE_COUNT") {
|
||||||
|
return new AssetError(
|
||||||
|
"UPLOAD_FILE_COUNT_EXCEEDED",
|
||||||
|
"The upload contains too many files",
|
||||||
|
{
|
||||||
|
statusCode: 413,
|
||||||
|
details: { maxFiles: config.maxFiles },
|
||||||
|
},
|
||||||
|
);
|
||||||
|
}
|
||||||
|
if (
|
||||||
|
[
|
||||||
|
"LIMIT_FIELD_COUNT",
|
||||||
|
"LIMIT_FIELD_KEY",
|
||||||
|
"LIMIT_FIELD_VALUE",
|
||||||
|
"LIMIT_PART_COUNT",
|
||||||
|
"LIMIT_UNEXPECTED_FILE",
|
||||||
|
].includes(error.code)
|
||||||
|
) {
|
||||||
|
return new AssetError(
|
||||||
|
"INVALID_MULTIPART_UPLOAD",
|
||||||
|
"The multipart upload contains too many or invalid parts",
|
||||||
|
{ details: { reason: error.code } },
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
return new AssetError(
|
||||||
|
"INVALID_MULTIPART_UPLOAD",
|
||||||
|
"The multipart upload could not be read",
|
||||||
|
{ details: { reason: error.code } },
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
function createTemporaryStorage(tempDirectory) {
|
||||||
|
return multer.diskStorage({
|
||||||
|
destination(request, file, callback) {
|
||||||
|
fs.mkdir(tempDirectory, { recursive: true, mode: 0o700 }, (error) => {
|
||||||
|
callback(error, tempDirectory);
|
||||||
|
});
|
||||||
|
},
|
||||||
|
filename(request, file, callback) {
|
||||||
|
const temporaryName = crypto.randomUUID();
|
||||||
|
if (!request[TEMPORARY_UPLOAD_PATHS]) {
|
||||||
|
request[TEMPORARY_UPLOAD_PATHS] = [];
|
||||||
|
}
|
||||||
|
request[TEMPORARY_UPLOAD_PATHS].push(
|
||||||
|
path.join(tempDirectory, temporaryName),
|
||||||
|
);
|
||||||
|
callback(null, temporaryName);
|
||||||
|
},
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
function temporaryUploadFiles(request) {
|
||||||
|
const paths = new Set(request[TEMPORARY_UPLOAD_PATHS] || []);
|
||||||
|
for (const file of Array.isArray(request.files) ? request.files : []) {
|
||||||
|
if (file?.path) paths.add(file.path);
|
||||||
|
}
|
||||||
|
return [...paths].map((temporaryPath) => ({ path: temporaryPath }));
|
||||||
|
}
|
||||||
|
|
||||||
|
function createMultipartUpload(config) {
|
||||||
|
const middleware = multer({
|
||||||
|
storage: createTemporaryStorage(config.tempDirectory),
|
||||||
|
limits: {
|
||||||
|
fileSize: config.maxFileBytes,
|
||||||
|
files: config.maxFiles,
|
||||||
|
fields: 2,
|
||||||
|
parts: config.maxFiles + 2,
|
||||||
|
fieldNameSize: 64,
|
||||||
|
fieldSize: MAX_FIELD_BYTES,
|
||||||
|
headerPairs: 100,
|
||||||
|
},
|
||||||
|
}).any();
|
||||||
|
|
||||||
|
return function receiveAssetUpload(request, response) {
|
||||||
|
return new Promise((resolve, reject) => {
|
||||||
|
middleware(request, response, (error) => {
|
||||||
|
if (error) {
|
||||||
|
reject(mapMulterError(error, config));
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
const files = Array.isArray(request.files) ? request.files : [];
|
||||||
|
try {
|
||||||
|
if (files.length === 0) {
|
||||||
|
throw new AssetError(
|
||||||
|
"UPLOAD_FILE_REQUIRED",
|
||||||
|
"At least one image file is required",
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
const unexpectedFile = files.find(
|
||||||
|
(file) => !ALLOWED_FILE_FIELDS.has(file.fieldname),
|
||||||
|
);
|
||||||
|
if (unexpectedFile) {
|
||||||
|
throw new AssetError(
|
||||||
|
"UNEXPECTED_UPLOAD_FIELD",
|
||||||
|
"Image files must use the file or files multipart field",
|
||||||
|
{ details: { field: unexpectedFile.fieldname } },
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
const body = request.body || {};
|
||||||
|
const unknownFields = Object.keys(body).filter(
|
||||||
|
(name) => !ALLOWED_TEXT_FIELDS.has(name),
|
||||||
|
);
|
||||||
|
if (unknownFields.length > 0) {
|
||||||
|
throw new AssetError(
|
||||||
|
"UNEXPECTED_UPLOAD_FIELD",
|
||||||
|
"The multipart upload contains unsupported fields",
|
||||||
|
{ details: { fields: unknownFields } },
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
if (
|
||||||
|
Object.hasOwn(body, "display_name") &&
|
||||||
|
Object.hasOwn(body, "displayName")
|
||||||
|
) {
|
||||||
|
throw new AssetError(
|
||||||
|
"DUPLICATE_DISPLAY_NAME_FIELD",
|
||||||
|
"Provide only one display name field",
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
const displayName = body.display_name ?? body.displayName ?? null;
|
||||||
|
if (displayName !== null && files.length !== 1) {
|
||||||
|
throw new AssetError(
|
||||||
|
"DISPLAY_NAME_REQUIRES_SINGLE_FILE",
|
||||||
|
"An explicit display name can be used only with one uploaded file",
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
resolve({
|
||||||
|
displayName,
|
||||||
|
files: files.map((file) => ({
|
||||||
|
path: file.path,
|
||||||
|
byteSize: file.size,
|
||||||
|
originalFilename: safeOriginalFilename(file.originalname),
|
||||||
|
suppliedContentType: file.mimetype,
|
||||||
|
})),
|
||||||
|
});
|
||||||
|
} catch (validationError) {
|
||||||
|
reject(validationError);
|
||||||
|
}
|
||||||
|
});
|
||||||
|
});
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
async function cleanupTemporaryFiles(files = []) {
|
||||||
|
await Promise.all(
|
||||||
|
files.map(async (file) => {
|
||||||
|
if (!file?.path) return;
|
||||||
|
try {
|
||||||
|
await fs.promises.rm(file.path, { force: true });
|
||||||
|
} catch {
|
||||||
|
// Temporary cleanup is best-effort and must not mask the API result.
|
||||||
|
}
|
||||||
|
}),
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
module.exports = {
|
||||||
|
cleanupTemporaryFiles,
|
||||||
|
createMultipartUpload,
|
||||||
|
mapMulterError,
|
||||||
|
safeOriginalFilename,
|
||||||
|
temporaryUploadFiles,
|
||||||
|
};
|
||||||
@@ -0,0 +1,102 @@
|
|||||||
|
const express = require("express");
|
||||||
|
const {
|
||||||
|
cleanupTemporaryFiles,
|
||||||
|
createMultipartUpload,
|
||||||
|
temporaryUploadFiles,
|
||||||
|
} = require("./multipart");
|
||||||
|
const { sendOriginal } = require("./streaming");
|
||||||
|
|
||||||
|
function createAssetRouter({ assetService, processingConfig }) {
|
||||||
|
const router = express.Router();
|
||||||
|
const receiveUpload = createMultipartUpload(processingConfig);
|
||||||
|
|
||||||
|
router.get("/", (request, response, next) => {
|
||||||
|
try {
|
||||||
|
response.json({ assets: assetService.list() });
|
||||||
|
} catch (error) {
|
||||||
|
next(error);
|
||||||
|
}
|
||||||
|
});
|
||||||
|
|
||||||
|
router.post("/", async (request, response, next) => {
|
||||||
|
const cleanupAfterAbort = () => {
|
||||||
|
void cleanupTemporaryFiles(temporaryUploadFiles(request));
|
||||||
|
};
|
||||||
|
request.once("aborted", cleanupAfterAbort);
|
||||||
|
|
||||||
|
try {
|
||||||
|
const upload = await receiveUpload(request, response);
|
||||||
|
const results = await assetService.upload(upload.files, {
|
||||||
|
displayName: upload.displayName,
|
||||||
|
});
|
||||||
|
const createdCount = results.filter((result) => result.created).length;
|
||||||
|
|
||||||
|
if (results.length === 1) {
|
||||||
|
response.status(createdCount > 0 ? 201 : 200).json(results[0]);
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
response.status(createdCount > 0 ? 201 : 200).json({
|
||||||
|
results,
|
||||||
|
createdCount,
|
||||||
|
duplicateCount: results.length - createdCount,
|
||||||
|
});
|
||||||
|
} catch (error) {
|
||||||
|
next(error);
|
||||||
|
} finally {
|
||||||
|
request.off("aborted", cleanupAfterAbort);
|
||||||
|
await cleanupTemporaryFiles(temporaryUploadFiles(request));
|
||||||
|
}
|
||||||
|
});
|
||||||
|
|
||||||
|
router.get("/:assetId/references", (request, response, next) => {
|
||||||
|
try {
|
||||||
|
response.json(assetService.references(request.params.assetId));
|
||||||
|
} catch (error) {
|
||||||
|
next(error);
|
||||||
|
}
|
||||||
|
});
|
||||||
|
|
||||||
|
router.get("/:assetId/original", async (request, response, next) => {
|
||||||
|
try {
|
||||||
|
const descriptor = assetService.originalDescriptor(request.params.assetId);
|
||||||
|
await sendOriginal(assetService, response, descriptor);
|
||||||
|
} catch (error) {
|
||||||
|
if (response.headersSent) {
|
||||||
|
response.destroy(error);
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
next(error);
|
||||||
|
}
|
||||||
|
});
|
||||||
|
|
||||||
|
router.get("/:assetId", (request, response, next) => {
|
||||||
|
try {
|
||||||
|
response.json(assetService.get(request.params.assetId));
|
||||||
|
} catch (error) {
|
||||||
|
next(error);
|
||||||
|
}
|
||||||
|
});
|
||||||
|
|
||||||
|
router.patch("/:assetId", (request, response, next) => {
|
||||||
|
try {
|
||||||
|
response.json(assetService.update(request.params.assetId, request.body));
|
||||||
|
} catch (error) {
|
||||||
|
next(error);
|
||||||
|
}
|
||||||
|
});
|
||||||
|
|
||||||
|
router.delete("/:assetId", async (request, response, next) => {
|
||||||
|
try {
|
||||||
|
response.json(await assetService.remove(request.params.assetId));
|
||||||
|
} catch (error) {
|
||||||
|
next(error);
|
||||||
|
}
|
||||||
|
});
|
||||||
|
|
||||||
|
return router;
|
||||||
|
}
|
||||||
|
|
||||||
|
module.exports = {
|
||||||
|
createAssetRouter,
|
||||||
|
};
|
||||||
@@ -0,0 +1,835 @@
|
|||||||
|
const crypto = require("node:crypto");
|
||||||
|
const fs = require("node:fs");
|
||||||
|
const path = require("node:path");
|
||||||
|
const { AssetError, assetNotFound } = require("./errors");
|
||||||
|
const { StorageObjectNotFoundError } = require("../storage");
|
||||||
|
|
||||||
|
const MAX_DISPLAY_NAME_LENGTH = 200;
|
||||||
|
const MAX_PROCESSING_ERROR_LENGTH = 1_000;
|
||||||
|
const CONCURRENT_UPLOAD_WAIT_MS = 15_000;
|
||||||
|
const CONCURRENT_UPLOAD_POLL_MS = 50;
|
||||||
|
const SHA256_PATTERN = /^[0-9a-f]{64}$/;
|
||||||
|
|
||||||
|
function delay(milliseconds) {
|
||||||
|
return new Promise((resolve) => setTimeout(resolve, milliseconds));
|
||||||
|
}
|
||||||
|
|
||||||
|
function normalizeDisplayName(value) {
|
||||||
|
if (typeof value !== "string") {
|
||||||
|
throw new AssetError(
|
||||||
|
"INVALID_ASSET_DISPLAY_NAME",
|
||||||
|
"Asset display name must be a string",
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
const normalized = value
|
||||||
|
.replace(/[\u0000-\u001f\u007f]/g, " ")
|
||||||
|
.replace(/\s+/g, " ")
|
||||||
|
.trim();
|
||||||
|
if (!normalized) {
|
||||||
|
throw new AssetError(
|
||||||
|
"INVALID_ASSET_DISPLAY_NAME",
|
||||||
|
"Asset display name cannot be empty",
|
||||||
|
);
|
||||||
|
}
|
||||||
|
if ([...normalized].length > MAX_DISPLAY_NAME_LENGTH) {
|
||||||
|
throw new AssetError(
|
||||||
|
"INVALID_ASSET_DISPLAY_NAME",
|
||||||
|
`Asset display name cannot exceed ${MAX_DISPLAY_NAME_LENGTH} characters`,
|
||||||
|
{ details: { maxLength: MAX_DISPLAY_NAME_LENGTH } },
|
||||||
|
);
|
||||||
|
}
|
||||||
|
return normalized;
|
||||||
|
}
|
||||||
|
|
||||||
|
function defaultDisplayName(originalFilename) {
|
||||||
|
const extension = path.extname(originalFilename);
|
||||||
|
const withoutExtension = extension
|
||||||
|
? originalFilename.slice(0, -extension.length)
|
||||||
|
: originalFilename;
|
||||||
|
return normalizeDisplayName(withoutExtension);
|
||||||
|
}
|
||||||
|
|
||||||
|
function normalizeExpiration(value) {
|
||||||
|
if (value === null) return null;
|
||||||
|
if (Number.isSafeInteger(value) && value >= 0) return value;
|
||||||
|
|
||||||
|
if (typeof value === "string" && value.trim()) {
|
||||||
|
const timestamp = Date.parse(value);
|
||||||
|
if (Number.isSafeInteger(timestamp) && timestamp >= 0) {
|
||||||
|
return timestamp;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
throw new AssetError(
|
||||||
|
"INVALID_ASSET_EXPIRATION",
|
||||||
|
"Asset expiration must be null, an epoch-millisecond integer, or an ISO date",
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
function mediaUrls(row) {
|
||||||
|
if (row.status !== "ready") {
|
||||||
|
return {
|
||||||
|
originalUrl: null,
|
||||||
|
displayUrl: null,
|
||||||
|
thumbnailUrl: null,
|
||||||
|
};
|
||||||
|
}
|
||||||
|
return {
|
||||||
|
originalUrl: `/api/assets/${row.id}/original`,
|
||||||
|
displayUrl: `/media/assets/${row.id}/display/${row.display_sha256}.webp`,
|
||||||
|
thumbnailUrl: `/media/assets/${row.id}/thumbnail/${row.thumbnail_sha256}.webp`,
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
function mapAsset(row) {
|
||||||
|
if (!row) return null;
|
||||||
|
const urls = mediaUrls(row);
|
||||||
|
return {
|
||||||
|
id: row.id,
|
||||||
|
displayName: row.display_name,
|
||||||
|
originalFilename: row.original_filename,
|
||||||
|
status: row.status,
|
||||||
|
processingError: row.processing_error || null,
|
||||||
|
sourceSha256: row.source_sha256,
|
||||||
|
expirationAt: row.expiration_at ?? null,
|
||||||
|
createdAt: row.created_at,
|
||||||
|
updatedAt: row.updated_at,
|
||||||
|
referenceCount: Number(row.reference_count || 0),
|
||||||
|
original: row.original_content_type
|
||||||
|
? {
|
||||||
|
contentType: row.original_content_type,
|
||||||
|
byteSize: row.original_byte_size,
|
||||||
|
width: row.original_width,
|
||||||
|
height: row.original_height,
|
||||||
|
url: urls.originalUrl,
|
||||||
|
}
|
||||||
|
: null,
|
||||||
|
display: row.display_content_type
|
||||||
|
? {
|
||||||
|
sha256: row.display_sha256,
|
||||||
|
contentType: row.display_content_type,
|
||||||
|
byteSize: row.display_byte_size,
|
||||||
|
width: row.display_width,
|
||||||
|
height: row.display_height,
|
||||||
|
url: urls.displayUrl,
|
||||||
|
}
|
||||||
|
: null,
|
||||||
|
thumbnail: row.thumbnail_content_type
|
||||||
|
? {
|
||||||
|
sha256: row.thumbnail_sha256,
|
||||||
|
contentType: row.thumbnail_content_type,
|
||||||
|
byteSize: row.thumbnail_byte_size,
|
||||||
|
width: row.thumbnail_width,
|
||||||
|
height: row.thumbnail_height,
|
||||||
|
url: urls.thumbnailUrl,
|
||||||
|
}
|
||||||
|
: null,
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
async function hashFile(filename) {
|
||||||
|
const hash = crypto.createHash("sha256");
|
||||||
|
let byteSize = 0;
|
||||||
|
for await (const chunk of fs.createReadStream(filename)) {
|
||||||
|
byteSize += chunk.length;
|
||||||
|
hash.update(chunk);
|
||||||
|
}
|
||||||
|
return { sourceSha256: hash.digest("hex"), byteSize };
|
||||||
|
}
|
||||||
|
|
||||||
|
function objectKeys(assetId, processed) {
|
||||||
|
return {
|
||||||
|
original: `assets/${assetId}/original/${processed.original.sha256}`,
|
||||||
|
display: `assets/${assetId}/display/${processed.display.sha256}.webp`,
|
||||||
|
thumbnail: `assets/${assetId}/thumbnail/${processed.thumbnail.sha256}.webp`,
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
function boundedProcessingError(error) {
|
||||||
|
const code = typeof error?.code === "string" ? error.code : "PROCESSING_FAILED";
|
||||||
|
const message =
|
||||||
|
typeof error?.message === "string" && error.message
|
||||||
|
? error.message
|
||||||
|
: "Asset processing failed";
|
||||||
|
return `${code}: ${message}`.slice(0, MAX_PROCESSING_ERROR_LENGTH);
|
||||||
|
}
|
||||||
|
|
||||||
|
function createAssetService({
|
||||||
|
database,
|
||||||
|
storage,
|
||||||
|
imageProcessor,
|
||||||
|
now = Date.now,
|
||||||
|
createId = crypto.randomUUID,
|
||||||
|
wait = delay,
|
||||||
|
concurrentUploadWaitMs = CONCURRENT_UPLOAD_WAIT_MS,
|
||||||
|
}) {
|
||||||
|
const assetSelect = `
|
||||||
|
SELECT a.*,
|
||||||
|
(SELECT COUNT(*) FROM playlist_entries pe WHERE pe.asset_id = a.id)
|
||||||
|
AS reference_count
|
||||||
|
FROM assets a
|
||||||
|
`;
|
||||||
|
const listAssets = database.prepare(`${assetSelect} ORDER BY a.created_at DESC, a.id`);
|
||||||
|
const findById = database.prepare(`${assetSelect} WHERE a.id = ?`);
|
||||||
|
const findByHash = database.prepare(`${assetSelect} WHERE a.source_sha256 = ?`);
|
||||||
|
const findByName = database.prepare(`${assetSelect} WHERE a.display_name = ? COLLATE NOCASE`);
|
||||||
|
const insertProcessing = database.prepare(`
|
||||||
|
INSERT INTO assets (
|
||||||
|
id, source_sha256, display_name, original_filename, status,
|
||||||
|
created_at, updated_at
|
||||||
|
) VALUES (
|
||||||
|
@id, @sourceSha256, @displayName, @originalFilename, 'processing',
|
||||||
|
@createdAt, @updatedAt
|
||||||
|
)
|
||||||
|
`);
|
||||||
|
const markReady = database.prepare(`
|
||||||
|
UPDATE assets
|
||||||
|
SET status = 'ready',
|
||||||
|
processing_error = NULL,
|
||||||
|
original_object_key = @originalObjectKey,
|
||||||
|
original_content_type = @originalContentType,
|
||||||
|
original_byte_size = @originalByteSize,
|
||||||
|
original_width = @originalWidth,
|
||||||
|
original_height = @originalHeight,
|
||||||
|
display_object_key = @displayObjectKey,
|
||||||
|
display_sha256 = @displaySha256,
|
||||||
|
display_content_type = @displayContentType,
|
||||||
|
display_byte_size = @displayByteSize,
|
||||||
|
display_width = @displayWidth,
|
||||||
|
display_height = @displayHeight,
|
||||||
|
thumbnail_object_key = @thumbnailObjectKey,
|
||||||
|
thumbnail_sha256 = @thumbnailSha256,
|
||||||
|
thumbnail_content_type = @thumbnailContentType,
|
||||||
|
thumbnail_byte_size = @thumbnailByteSize,
|
||||||
|
thumbnail_width = @thumbnailWidth,
|
||||||
|
thumbnail_height = @thumbnailHeight,
|
||||||
|
updated_at = @updatedAt
|
||||||
|
WHERE id = @id AND status = 'processing'
|
||||||
|
`);
|
||||||
|
const markFailed = database.prepare(`
|
||||||
|
UPDATE assets
|
||||||
|
SET status = 'failed',
|
||||||
|
processing_error = @processingError,
|
||||||
|
original_object_key = @originalObjectKey,
|
||||||
|
original_content_type = @originalContentType,
|
||||||
|
original_byte_size = @originalByteSize,
|
||||||
|
original_width = @originalWidth,
|
||||||
|
original_height = @originalHeight,
|
||||||
|
display_object_key = @displayObjectKey,
|
||||||
|
display_sha256 = @displaySha256,
|
||||||
|
display_content_type = @displayContentType,
|
||||||
|
display_byte_size = @displayByteSize,
|
||||||
|
display_width = @displayWidth,
|
||||||
|
display_height = @displayHeight,
|
||||||
|
thumbnail_object_key = @thumbnailObjectKey,
|
||||||
|
thumbnail_sha256 = @thumbnailSha256,
|
||||||
|
thumbnail_content_type = @thumbnailContentType,
|
||||||
|
thumbnail_byte_size = @thumbnailByteSize,
|
||||||
|
thumbnail_width = @thumbnailWidth,
|
||||||
|
thumbnail_height = @thumbnailHeight,
|
||||||
|
updated_at = @updatedAt
|
||||||
|
WHERE id = @id AND status = 'processing'
|
||||||
|
`);
|
||||||
|
const updateEditable = database.prepare(`
|
||||||
|
UPDATE assets
|
||||||
|
SET display_name = @displayName,
|
||||||
|
expiration_at = @expirationAt,
|
||||||
|
updated_at = @updatedAt
|
||||||
|
WHERE id = @id AND status != 'deleting'
|
||||||
|
`);
|
||||||
|
const playlistReferences = database.prepare(`
|
||||||
|
SELECT
|
||||||
|
pe.id AS entry_id,
|
||||||
|
pe.playlist_id,
|
||||||
|
p.name AS playlist_name,
|
||||||
|
pe.enabled,
|
||||||
|
pe.position,
|
||||||
|
EXISTS (
|
||||||
|
SELECT 1 FROM playback_state active
|
||||||
|
WHERE active.active_playlist_id = pe.playlist_id
|
||||||
|
) AS is_active_playlist,
|
||||||
|
EXISTS (
|
||||||
|
SELECT 1 FROM playback_state current
|
||||||
|
WHERE current.current_entry_id = pe.id
|
||||||
|
) AS is_current_normal_target
|
||||||
|
FROM playlist_entries pe
|
||||||
|
JOIN playlists p ON p.id = pe.playlist_id
|
||||||
|
WHERE pe.asset_id = ?
|
||||||
|
ORDER BY p.name COLLATE NOCASE, pe.position, pe.id
|
||||||
|
`);
|
||||||
|
const playbackReferences = database.prepare(`
|
||||||
|
SELECT
|
||||||
|
output_id,
|
||||||
|
mode,
|
||||||
|
active_playlist_id,
|
||||||
|
current_entry_id,
|
||||||
|
static_asset_id
|
||||||
|
FROM playback_state
|
||||||
|
WHERE static_asset_id = ?
|
||||||
|
OR current_entry_id IN (
|
||||||
|
SELECT id FROM playlist_entries WHERE asset_id = ?
|
||||||
|
)
|
||||||
|
ORDER BY output_id
|
||||||
|
`);
|
||||||
|
const markDeleting = database.prepare(`
|
||||||
|
UPDATE assets
|
||||||
|
SET status = 'deleting', updated_at = ?
|
||||||
|
WHERE id = ? AND status != 'processing'
|
||||||
|
`);
|
||||||
|
const deleteAssetRow = database.prepare(
|
||||||
|
"DELETE FROM assets WHERE id = ? AND status = 'deleting'",
|
||||||
|
);
|
||||||
|
|
||||||
|
const completeAsset = database.transaction((metadata) => {
|
||||||
|
const result = markReady.run(metadata);
|
||||||
|
if (result.changes !== 1) {
|
||||||
|
throw new Error("Asset processing state changed before completion");
|
||||||
|
}
|
||||||
|
return findById.get(metadata.id);
|
||||||
|
});
|
||||||
|
|
||||||
|
function getRaw(assetId) {
|
||||||
|
const row = findById.get(assetId);
|
||||||
|
if (!row) throw assetNotFound(assetId);
|
||||||
|
return row;
|
||||||
|
}
|
||||||
|
|
||||||
|
function duplicateStateError(row) {
|
||||||
|
if (row.status === "failed") {
|
||||||
|
return new AssetError(
|
||||||
|
"ASSET_PROCESSING_FAILED",
|
||||||
|
"These bytes match an asset whose processing previously failed",
|
||||||
|
{ statusCode: 409, details: { asset: mapAsset(row) } },
|
||||||
|
);
|
||||||
|
}
|
||||||
|
if (row.status === "deleting") {
|
||||||
|
return new AssetError(
|
||||||
|
"ASSET_DELETING",
|
||||||
|
"These bytes match an asset that is being deleted",
|
||||||
|
{ statusCode: 409, details: { assetId: row.id } },
|
||||||
|
);
|
||||||
|
}
|
||||||
|
return new AssetError(
|
||||||
|
"ASSET_PROCESSING",
|
||||||
|
"These bytes are already being processed",
|
||||||
|
{ statusCode: 409, details: { assetId: row.id } },
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
async function resolveExistingHash(row) {
|
||||||
|
if (row.status === "ready") {
|
||||||
|
return { created: false, duplicate: true, asset: mapAsset(row) };
|
||||||
|
}
|
||||||
|
if (row.status !== "processing") {
|
||||||
|
throw duplicateStateError(row);
|
||||||
|
}
|
||||||
|
|
||||||
|
const deadline = Date.now() + concurrentUploadWaitMs;
|
||||||
|
while (Date.now() < deadline) {
|
||||||
|
await wait(CONCURRENT_UPLOAD_POLL_MS);
|
||||||
|
const latest = findByHash.get(row.source_sha256);
|
||||||
|
if (!latest) break;
|
||||||
|
if (latest.status === "ready") {
|
||||||
|
return { created: false, duplicate: true, asset: mapAsset(latest) };
|
||||||
|
}
|
||||||
|
if (latest.status !== "processing") {
|
||||||
|
throw duplicateStateError(latest);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
throw duplicateStateError(findByHash.get(row.source_sha256) || row);
|
||||||
|
}
|
||||||
|
|
||||||
|
function nameConflict(displayName, row = findByName.get(displayName)) {
|
||||||
|
return new AssetError(
|
||||||
|
"ASSET_NAME_CONFLICT",
|
||||||
|
"Another library asset already uses that display name",
|
||||||
|
{
|
||||||
|
statusCode: 409,
|
||||||
|
details: row ? { asset: mapAsset(row) } : { displayName },
|
||||||
|
},
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
function processingMetadata(id, processed, keys, error) {
|
||||||
|
return {
|
||||||
|
id,
|
||||||
|
processingError: error ? boundedProcessingError(error) : null,
|
||||||
|
originalObjectKey: keys.original,
|
||||||
|
originalContentType: processed.original.contentType,
|
||||||
|
originalByteSize: processed.original.byteSize,
|
||||||
|
originalWidth: processed.original.width,
|
||||||
|
originalHeight: processed.original.height,
|
||||||
|
displayObjectKey: keys.display,
|
||||||
|
displaySha256: processed.display.sha256,
|
||||||
|
displayContentType: processed.display.contentType,
|
||||||
|
displayByteSize: processed.display.byteSize,
|
||||||
|
displayWidth: processed.display.width,
|
||||||
|
displayHeight: processed.display.height,
|
||||||
|
thumbnailObjectKey: keys.thumbnail,
|
||||||
|
thumbnailSha256: processed.thumbnail.sha256,
|
||||||
|
thumbnailContentType: processed.thumbnail.contentType,
|
||||||
|
thumbnailByteSize: processed.thumbnail.byteSize,
|
||||||
|
thumbnailWidth: processed.thumbnail.width,
|
||||||
|
thumbnailHeight: processed.thumbnail.height,
|
||||||
|
updatedAt: now(),
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
async function cleanObjects(keys) {
|
||||||
|
const results = await Promise.allSettled(
|
||||||
|
[...new Set(Object.values(keys).filter(Boolean))].map((key) =>
|
||||||
|
storage.deleteObject(key),
|
||||||
|
),
|
||||||
|
);
|
||||||
|
const rejected = results.find((result) => result.status === "rejected");
|
||||||
|
if (rejected) throw rejected.reason;
|
||||||
|
}
|
||||||
|
|
||||||
|
async function createProcessedAsset(plan) {
|
||||||
|
const id = createId();
|
||||||
|
const timestamp = now();
|
||||||
|
try {
|
||||||
|
insertProcessing.run({
|
||||||
|
id,
|
||||||
|
sourceSha256: plan.sourceSha256,
|
||||||
|
displayName: plan.displayName,
|
||||||
|
originalFilename: plan.file.originalFilename,
|
||||||
|
createdAt: timestamp,
|
||||||
|
updatedAt: timestamp,
|
||||||
|
});
|
||||||
|
} catch (error) {
|
||||||
|
const hashMatch = findByHash.get(plan.sourceSha256);
|
||||||
|
if (hashMatch) return resolveExistingHash(hashMatch);
|
||||||
|
const nameMatch = findByName.get(plan.displayName);
|
||||||
|
if (nameMatch) throw nameConflict(plan.displayName, nameMatch);
|
||||||
|
throw error;
|
||||||
|
}
|
||||||
|
|
||||||
|
const keys = objectKeys(id, plan.processed);
|
||||||
|
try {
|
||||||
|
await storage.putObject({
|
||||||
|
key: keys.original,
|
||||||
|
body: fs.createReadStream(plan.file.path),
|
||||||
|
contentType: plan.processed.original.contentType,
|
||||||
|
contentLength: plan.processed.original.byteSize,
|
||||||
|
metadata: { sha256: plan.processed.original.sha256 },
|
||||||
|
});
|
||||||
|
await storage.putObject({
|
||||||
|
key: keys.display,
|
||||||
|
body: plan.processed.display.buffer,
|
||||||
|
contentType: plan.processed.display.contentType,
|
||||||
|
contentLength: plan.processed.display.byteSize,
|
||||||
|
metadata: { sha256: plan.processed.display.sha256 },
|
||||||
|
});
|
||||||
|
await storage.putObject({
|
||||||
|
key: keys.thumbnail,
|
||||||
|
body: plan.processed.thumbnail.buffer,
|
||||||
|
contentType: plan.processed.thumbnail.contentType,
|
||||||
|
contentLength: plan.processed.thumbnail.byteSize,
|
||||||
|
metadata: { sha256: plan.processed.thumbnail.sha256 },
|
||||||
|
});
|
||||||
|
|
||||||
|
const row = completeAsset(processingMetadata(id, plan.processed, keys));
|
||||||
|
return { created: true, duplicate: false, asset: mapAsset(row) };
|
||||||
|
} catch (error) {
|
||||||
|
const failure = processingMetadata(id, plan.processed, keys, error);
|
||||||
|
try {
|
||||||
|
markFailed.run(failure);
|
||||||
|
} catch {
|
||||||
|
// Preserve the original failure; the processing row remains diagnosable.
|
||||||
|
}
|
||||||
|
try {
|
||||||
|
await cleanObjects(keys);
|
||||||
|
} catch {
|
||||||
|
// The failed row retains deterministic keys so DELETE can retry cleanup.
|
||||||
|
}
|
||||||
|
|
||||||
|
if (error?.statusCode) {
|
||||||
|
error.details = { ...(error.details || {}), assetId: id, retryable: true };
|
||||||
|
}
|
||||||
|
throw error;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
async function upload(files, { displayName = null } = {}) {
|
||||||
|
if (!Array.isArray(files) || files.length === 0) {
|
||||||
|
throw new AssetError(
|
||||||
|
"UPLOAD_FILE_REQUIRED",
|
||||||
|
"At least one image file is required",
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
const plans = [];
|
||||||
|
const firstPlanByHash = new Map();
|
||||||
|
const plannedNames = new Map();
|
||||||
|
|
||||||
|
for (const [index, file] of files.entries()) {
|
||||||
|
const identity = await hashFile(file.path);
|
||||||
|
if (identity.byteSize !== file.byteSize) {
|
||||||
|
throw new AssetError(
|
||||||
|
"UPLOAD_CHANGED_DURING_PROCESSING",
|
||||||
|
"An uploaded temporary file changed before processing",
|
||||||
|
{ statusCode: 409 },
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
const existing = findByHash.get(identity.sourceSha256);
|
||||||
|
if (existing) {
|
||||||
|
plans.push({ index, existing });
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
|
||||||
|
const priorPlan = firstPlanByHash.get(identity.sourceSha256);
|
||||||
|
if (priorPlan) {
|
||||||
|
plans.push({ index, duplicateOf: priorPlan });
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
|
||||||
|
const proposedName = normalizeDisplayName(
|
||||||
|
displayName ?? defaultDisplayName(file.originalFilename),
|
||||||
|
);
|
||||||
|
const existingName = findByName.get(proposedName);
|
||||||
|
if (existingName) throw nameConflict(proposedName, existingName);
|
||||||
|
|
||||||
|
const normalizedNameKey = proposedName.toLowerCase();
|
||||||
|
const batchNameMatch = plannedNames.get(normalizedNameKey);
|
||||||
|
if (batchNameMatch) throw nameConflict(proposedName);
|
||||||
|
|
||||||
|
const plan = {
|
||||||
|
index,
|
||||||
|
file,
|
||||||
|
displayName: proposedName,
|
||||||
|
sourceSha256: identity.sourceSha256,
|
||||||
|
};
|
||||||
|
firstPlanByHash.set(identity.sourceSha256, plan);
|
||||||
|
plannedNames.set(normalizedNameKey, plan);
|
||||||
|
plans.push(plan);
|
||||||
|
}
|
||||||
|
|
||||||
|
// Decode every new image before any database or object-storage mutation.
|
||||||
|
for (const plan of plans) {
|
||||||
|
if (plan.existing || plan.duplicateOf) continue;
|
||||||
|
plan.processed = await imageProcessor.process(plan.file.path);
|
||||||
|
if (plan.processed.original.sha256 !== plan.sourceSha256) {
|
||||||
|
throw new AssetError(
|
||||||
|
"UPLOAD_CHANGED_DURING_PROCESSING",
|
||||||
|
"An uploaded temporary file changed before processing",
|
||||||
|
{ statusCode: 409 },
|
||||||
|
);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
const resultByPlan = new Map();
|
||||||
|
const results = new Array(files.length);
|
||||||
|
for (const plan of plans) {
|
||||||
|
let result;
|
||||||
|
if (plan.existing) {
|
||||||
|
result = await resolveExistingHash(plan.existing);
|
||||||
|
} else if (plan.duplicateOf) {
|
||||||
|
const originalResult = resultByPlan.get(plan.duplicateOf);
|
||||||
|
result = {
|
||||||
|
created: false,
|
||||||
|
duplicate: true,
|
||||||
|
asset: originalResult.asset,
|
||||||
|
};
|
||||||
|
} else {
|
||||||
|
result = await createProcessedAsset(plan);
|
||||||
|
resultByPlan.set(plan, result);
|
||||||
|
}
|
||||||
|
results[plan.index] = result;
|
||||||
|
}
|
||||||
|
return results;
|
||||||
|
}
|
||||||
|
|
||||||
|
function list() {
|
||||||
|
return listAssets.all().map(mapAsset);
|
||||||
|
}
|
||||||
|
|
||||||
|
function get(assetId) {
|
||||||
|
return mapAsset(getRaw(assetId));
|
||||||
|
}
|
||||||
|
|
||||||
|
function update(assetId, input) {
|
||||||
|
if (input === null || typeof input !== "object" || Array.isArray(input)) {
|
||||||
|
throw new AssetError("INVALID_ASSET_UPDATE", "Asset update must be an object");
|
||||||
|
}
|
||||||
|
|
||||||
|
const allowed = new Set([
|
||||||
|
"displayName",
|
||||||
|
"display_name",
|
||||||
|
"expirationAt",
|
||||||
|
"expiration_at",
|
||||||
|
]);
|
||||||
|
const unknown = Object.keys(input).filter((key) => !allowed.has(key));
|
||||||
|
if (unknown.length > 0) {
|
||||||
|
throw new AssetError(
|
||||||
|
"INVALID_ASSET_UPDATE",
|
||||||
|
"Asset update contains unsupported fields",
|
||||||
|
{ details: { fields: unknown } },
|
||||||
|
);
|
||||||
|
}
|
||||||
|
if (Object.keys(input).length === 0) {
|
||||||
|
throw new AssetError("INVALID_ASSET_UPDATE", "At least one field is required");
|
||||||
|
}
|
||||||
|
if (
|
||||||
|
Object.hasOwn(input, "displayName") &&
|
||||||
|
Object.hasOwn(input, "display_name")
|
||||||
|
) {
|
||||||
|
throw new AssetError("INVALID_ASSET_UPDATE", "Display name was provided twice");
|
||||||
|
}
|
||||||
|
if (
|
||||||
|
Object.hasOwn(input, "expirationAt") &&
|
||||||
|
Object.hasOwn(input, "expiration_at")
|
||||||
|
) {
|
||||||
|
throw new AssetError("INVALID_ASSET_UPDATE", "Expiration was provided twice");
|
||||||
|
}
|
||||||
|
|
||||||
|
const current = getRaw(assetId);
|
||||||
|
if (current.status === "deleting") {
|
||||||
|
throw new AssetError("ASSET_DELETING", "Deleting assets cannot be edited", {
|
||||||
|
statusCode: 409,
|
||||||
|
details: { assetId },
|
||||||
|
});
|
||||||
|
}
|
||||||
|
const hasDisplayName =
|
||||||
|
Object.hasOwn(input, "displayName") || Object.hasOwn(input, "display_name");
|
||||||
|
const hasExpiration =
|
||||||
|
Object.hasOwn(input, "expirationAt") || Object.hasOwn(input, "expiration_at");
|
||||||
|
const nextDisplayName = hasDisplayName
|
||||||
|
? normalizeDisplayName(
|
||||||
|
Object.hasOwn(input, "displayName")
|
||||||
|
? input.displayName
|
||||||
|
: input.display_name,
|
||||||
|
)
|
||||||
|
: current.display_name;
|
||||||
|
const nextExpiration = hasExpiration
|
||||||
|
? normalizeExpiration(
|
||||||
|
Object.hasOwn(input, "expirationAt")
|
||||||
|
? input.expirationAt
|
||||||
|
: input.expiration_at,
|
||||||
|
)
|
||||||
|
: current.expiration_at;
|
||||||
|
|
||||||
|
if (
|
||||||
|
nextDisplayName === current.display_name &&
|
||||||
|
nextExpiration === current.expiration_at
|
||||||
|
) {
|
||||||
|
return mapAsset(current);
|
||||||
|
}
|
||||||
|
|
||||||
|
try {
|
||||||
|
const result = updateEditable.run({
|
||||||
|
id: assetId,
|
||||||
|
displayName: nextDisplayName,
|
||||||
|
expirationAt: nextExpiration,
|
||||||
|
updatedAt: Math.max(now(), current.updated_at + 1),
|
||||||
|
});
|
||||||
|
if (result.changes !== 1) {
|
||||||
|
throw new AssetError("ASSET_DELETING", "Deleting assets cannot be edited", {
|
||||||
|
statusCode: 409,
|
||||||
|
details: { assetId },
|
||||||
|
});
|
||||||
|
}
|
||||||
|
} catch (error) {
|
||||||
|
const nameMatch = findByName.get(nextDisplayName);
|
||||||
|
if (nameMatch && nameMatch.id !== assetId) {
|
||||||
|
throw nameConflict(nextDisplayName, nameMatch);
|
||||||
|
}
|
||||||
|
throw error;
|
||||||
|
}
|
||||||
|
return mapAsset(getRaw(assetId));
|
||||||
|
}
|
||||||
|
|
||||||
|
function references(assetId) {
|
||||||
|
const asset = getRaw(assetId);
|
||||||
|
const playlists = playlistReferences.all(assetId).map((row) => ({
|
||||||
|
playlistId: row.playlist_id,
|
||||||
|
playlistName: row.playlist_name,
|
||||||
|
playlistEntryId: row.entry_id,
|
||||||
|
enabled: Boolean(row.enabled),
|
||||||
|
position: row.position,
|
||||||
|
isActivePlaylist: Boolean(row.is_active_playlist),
|
||||||
|
isCurrentNormalTarget: Boolean(row.is_current_normal_target),
|
||||||
|
}));
|
||||||
|
const playback = playbackReferences.all(assetId, assetId).map((row) => ({
|
||||||
|
outputId: row.output_id,
|
||||||
|
mode: row.mode,
|
||||||
|
activePlaylistId: row.active_playlist_id,
|
||||||
|
currentPlaylistEntryId: row.current_entry_id,
|
||||||
|
isCurrentNormalTarget: playlists.some(
|
||||||
|
(entry) => entry.playlistEntryId === row.current_entry_id,
|
||||||
|
),
|
||||||
|
isCurrentStaticTarget: row.static_asset_id === assetId,
|
||||||
|
}));
|
||||||
|
const currentNormalTarget = playback.some((state) => state.isCurrentNormalTarget);
|
||||||
|
const currentStaticTarget = playback.some((state) => state.isCurrentStaticTarget);
|
||||||
|
const blockerCodes = [];
|
||||||
|
if (playlists.length > 0) blockerCodes.push("PLAYLIST_REFERENCES");
|
||||||
|
if (currentNormalTarget) blockerCodes.push("CURRENT_PLAYBACK_TARGET");
|
||||||
|
if (currentStaticTarget) blockerCodes.push("CURRENT_STATIC_TARGET");
|
||||||
|
if (asset.status === "processing") blockerCodes.push("ASSET_PROCESSING");
|
||||||
|
if (asset.status === "deleting") blockerCodes.push("ASSET_DELETING");
|
||||||
|
|
||||||
|
return {
|
||||||
|
assetId,
|
||||||
|
playlists,
|
||||||
|
playback,
|
||||||
|
currentNormalTarget,
|
||||||
|
currentStaticTarget,
|
||||||
|
canDelete: blockerCodes.length === 0,
|
||||||
|
blockerCodes,
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
const beginDelete = database.transaction((assetId) => {
|
||||||
|
const asset = getRaw(assetId);
|
||||||
|
const referenceInfo = references(assetId);
|
||||||
|
const relationshipBlockers = referenceInfo.blockerCodes.filter((code) =>
|
||||||
|
[
|
||||||
|
"PLAYLIST_REFERENCES",
|
||||||
|
"CURRENT_PLAYBACK_TARGET",
|
||||||
|
"CURRENT_STATIC_TARGET",
|
||||||
|
].includes(code),
|
||||||
|
);
|
||||||
|
if (relationshipBlockers.length > 0) {
|
||||||
|
throw new AssetError(
|
||||||
|
referenceInfo.playlists.length > 0 ? "ASSET_REFERENCED" : "ASSET_IN_USE",
|
||||||
|
"The library asset is still referenced and cannot be deleted",
|
||||||
|
{
|
||||||
|
statusCode: 409,
|
||||||
|
details: { ...referenceInfo, blockerCodes: relationshipBlockers },
|
||||||
|
},
|
||||||
|
);
|
||||||
|
}
|
||||||
|
if (asset.status === "processing") {
|
||||||
|
throw new AssetError(
|
||||||
|
"ASSET_PROCESSING",
|
||||||
|
"An asset cannot be deleted while it is processing",
|
||||||
|
{ statusCode: 409, details: { assetId } },
|
||||||
|
);
|
||||||
|
}
|
||||||
|
if (asset.status !== "deleting") {
|
||||||
|
markDeleting.run(Math.max(now(), asset.updated_at + 1), assetId);
|
||||||
|
}
|
||||||
|
return findById.get(assetId);
|
||||||
|
});
|
||||||
|
|
||||||
|
async function remove(assetId) {
|
||||||
|
const deleting = beginDelete(assetId);
|
||||||
|
const keys = {
|
||||||
|
original: deleting.original_object_key,
|
||||||
|
display: deleting.display_object_key,
|
||||||
|
thumbnail: deleting.thumbnail_object_key,
|
||||||
|
};
|
||||||
|
|
||||||
|
try {
|
||||||
|
await cleanObjects(keys);
|
||||||
|
} catch (error) {
|
||||||
|
if (error?.statusCode) {
|
||||||
|
error.details = { ...(error.details || {}), assetId, retryable: true };
|
||||||
|
}
|
||||||
|
throw error;
|
||||||
|
}
|
||||||
|
|
||||||
|
try {
|
||||||
|
const result = deleteAssetRow.run(assetId);
|
||||||
|
if (result.changes !== 1) throw assetNotFound(assetId);
|
||||||
|
} catch (error) {
|
||||||
|
if (error?.code?.startsWith?.("SQLITE_CONSTRAINT")) {
|
||||||
|
throw new AssetError(
|
||||||
|
"ASSET_REFERENCED",
|
||||||
|
"The library asset became referenced before deletion completed",
|
||||||
|
{ statusCode: 409, details: { assetId, retryable: true } },
|
||||||
|
);
|
||||||
|
}
|
||||||
|
throw error;
|
||||||
|
}
|
||||||
|
return { id: assetId, deleted: true };
|
||||||
|
}
|
||||||
|
|
||||||
|
function mediaDescriptor(assetId, kind, requestedHash) {
|
||||||
|
if (!SHA256_PATTERN.test(requestedHash)) {
|
||||||
|
throw new AssetError("MEDIA_NOT_FOUND", "Asset media version not found", {
|
||||||
|
statusCode: 404,
|
||||||
|
});
|
||||||
|
}
|
||||||
|
const asset = getRaw(assetId);
|
||||||
|
const fields =
|
||||||
|
kind === "display"
|
||||||
|
? {
|
||||||
|
hash: asset.display_sha256,
|
||||||
|
key: asset.display_object_key,
|
||||||
|
contentType: asset.display_content_type,
|
||||||
|
contentLength: asset.display_byte_size,
|
||||||
|
}
|
||||||
|
: {
|
||||||
|
hash: asset.thumbnail_sha256,
|
||||||
|
key: asset.thumbnail_object_key,
|
||||||
|
contentType: asset.thumbnail_content_type,
|
||||||
|
contentLength: asset.thumbnail_byte_size,
|
||||||
|
};
|
||||||
|
if (asset.status !== "ready" || fields.hash !== requestedHash || !fields.key) {
|
||||||
|
throw new AssetError("MEDIA_NOT_FOUND", "Asset media version not found", {
|
||||||
|
statusCode: 404,
|
||||||
|
});
|
||||||
|
}
|
||||||
|
return { asset, kind, ...fields };
|
||||||
|
}
|
||||||
|
|
||||||
|
function originalDescriptor(assetId) {
|
||||||
|
const asset = getRaw(assetId);
|
||||||
|
if (asset.status !== "ready" || !asset.original_object_key) {
|
||||||
|
throw new AssetError("ASSET_ORIGINAL_NOT_READY", "Asset original is not ready", {
|
||||||
|
statusCode: 409,
|
||||||
|
details: { assetId, status: asset.status },
|
||||||
|
});
|
||||||
|
}
|
||||||
|
return {
|
||||||
|
asset,
|
||||||
|
key: asset.original_object_key,
|
||||||
|
hash: asset.source_sha256,
|
||||||
|
contentType: asset.original_content_type,
|
||||||
|
contentLength: asset.original_byte_size,
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
async function readStoredObject(descriptor) {
|
||||||
|
try {
|
||||||
|
return await storage.getObject(descriptor.key);
|
||||||
|
} catch (error) {
|
||||||
|
if (error instanceof StorageObjectNotFoundError) {
|
||||||
|
throw new AssetError(
|
||||||
|
"STORAGE_INCONSISTENCY",
|
||||||
|
"Asset metadata exists but its stored bytes are unavailable",
|
||||||
|
{
|
||||||
|
statusCode: 503,
|
||||||
|
details: { assetId: descriptor.asset.id, retryable: true },
|
||||||
|
cause: error,
|
||||||
|
},
|
||||||
|
);
|
||||||
|
}
|
||||||
|
throw error;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
return Object.freeze({
|
||||||
|
list,
|
||||||
|
get,
|
||||||
|
upload,
|
||||||
|
update,
|
||||||
|
references,
|
||||||
|
remove,
|
||||||
|
mediaDescriptor,
|
||||||
|
originalDescriptor,
|
||||||
|
readStoredObject,
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
module.exports = {
|
||||||
|
createAssetService,
|
||||||
|
defaultDisplayName,
|
||||||
|
mapAsset,
|
||||||
|
normalizeDisplayName,
|
||||||
|
normalizeExpiration,
|
||||||
|
};
|
||||||
@@ -0,0 +1,66 @@
|
|||||||
|
const { Readable } = require("node:stream");
|
||||||
|
const { pipeline } = require("node:stream/promises");
|
||||||
|
const { AssetError } = require("./errors");
|
||||||
|
|
||||||
|
function readableBody(body) {
|
||||||
|
if (body && typeof body.pipe === "function") return body;
|
||||||
|
if (Buffer.isBuffer(body) || body instanceof Uint8Array) {
|
||||||
|
return Readable.from([body]);
|
||||||
|
}
|
||||||
|
if (body && typeof body.getReader === "function") {
|
||||||
|
return Readable.fromWeb(body);
|
||||||
|
}
|
||||||
|
throw new AssetError(
|
||||||
|
"STORAGE_INCONSISTENCY",
|
||||||
|
"Stored asset bytes could not be streamed",
|
||||||
|
{ statusCode: 503 },
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
function validateDescriptor(descriptor) {
|
||||||
|
if (
|
||||||
|
typeof descriptor.contentType !== "string" ||
|
||||||
|
!Number.isSafeInteger(descriptor.contentLength) ||
|
||||||
|
descriptor.contentLength < 0
|
||||||
|
) {
|
||||||
|
throw new AssetError(
|
||||||
|
"STORAGE_INCONSISTENCY",
|
||||||
|
"Stored asset metadata is incomplete",
|
||||||
|
{ statusCode: 503 },
|
||||||
|
);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
function setCommonHeaders(response, descriptor) {
|
||||||
|
response.set("Content-Type", descriptor.contentType);
|
||||||
|
response.set("Content-Length", String(descriptor.contentLength));
|
||||||
|
response.set("X-Content-Type-Options", "nosniff");
|
||||||
|
}
|
||||||
|
|
||||||
|
async function storedBody(assetService, descriptor) {
|
||||||
|
const stored = await assetService.readStoredObject(descriptor);
|
||||||
|
return readableBody(stored.body);
|
||||||
|
}
|
||||||
|
|
||||||
|
async function sendImmutableMedia(assetService, response, descriptor) {
|
||||||
|
validateDescriptor(descriptor);
|
||||||
|
const body = await storedBody(assetService, descriptor);
|
||||||
|
setCommonHeaders(response, descriptor);
|
||||||
|
response.set("Cache-Control", "public, max-age=31536000, immutable");
|
||||||
|
response.set("ETag", `"${descriptor.hash}"`);
|
||||||
|
await pipeline(body, response);
|
||||||
|
}
|
||||||
|
|
||||||
|
async function sendOriginal(assetService, response, descriptor) {
|
||||||
|
validateDescriptor(descriptor);
|
||||||
|
const body = await storedBody(assetService, descriptor);
|
||||||
|
response.attachment(descriptor.asset.original_filename);
|
||||||
|
setCommonHeaders(response, descriptor);
|
||||||
|
response.set("Cache-Control", "private, no-store");
|
||||||
|
await pipeline(body, response);
|
||||||
|
}
|
||||||
|
|
||||||
|
module.exports = {
|
||||||
|
sendImmutableMedia,
|
||||||
|
sendOriginal,
|
||||||
|
};
|
||||||
@@ -156,6 +156,42 @@ function loadConfig(env = process.env) {
|
|||||||
accessKeyId: storageCredential(env, "S3_ACCESS_KEY_ID", environment),
|
accessKeyId: storageCredential(env, "S3_ACCESS_KEY_ID", environment),
|
||||||
secretAccessKey: storageCredential(env, "S3_SECRET_ACCESS_KEY", environment),
|
secretAccessKey: storageCredential(env, "S3_SECRET_ACCESS_KEY", environment),
|
||||||
forcePathStyle: boolean(env, "S3_FORCE_PATH_STYLE", true),
|
forcePathStyle: boolean(env, "S3_FORCE_PATH_STYLE", true),
|
||||||
|
healthProbeIntervalMs: integer(
|
||||||
|
env,
|
||||||
|
"S3_HEALTH_PROBE_INTERVAL_MS",
|
||||||
|
30_000,
|
||||||
|
{ min: 1_000, max: 300_000 },
|
||||||
|
),
|
||||||
|
connectionTimeoutMs: integer(env, "S3_CONNECT_TIMEOUT_MS", 2_000, {
|
||||||
|
min: 250,
|
||||||
|
max: 60_000,
|
||||||
|
}),
|
||||||
|
requestTimeoutMs: integer(env, "S3_REQUEST_TIMEOUT_MS", 5_000, {
|
||||||
|
min: 1_000,
|
||||||
|
max: 120_000,
|
||||||
|
}),
|
||||||
|
maxAttempts: integer(env, "S3_MAX_ATTEMPTS", 2, { min: 1, max: 5 }),
|
||||||
|
}),
|
||||||
|
processing: Object.freeze({
|
||||||
|
tempDirectory: (() => {
|
||||||
|
const value = env.UPLOAD_TEMP_DIRECTORY?.trim() || "/tmp/announcements-uploads";
|
||||||
|
if (!path.isAbsolute(value)) {
|
||||||
|
throw new ConfigError(
|
||||||
|
"UPLOAD_TEMP_DIRECTORY must be an absolute path",
|
||||||
|
"UPLOAD_TEMP_DIRECTORY",
|
||||||
|
);
|
||||||
|
}
|
||||||
|
return value;
|
||||||
|
})(),
|
||||||
|
maxFileBytes: integer(env, "UPLOAD_MAX_FILE_BYTES", 25 * 1024 * 1024, {
|
||||||
|
min: 1,
|
||||||
|
max: 1024 * 1024 * 1024,
|
||||||
|
}),
|
||||||
|
maxFiles: integer(env, "UPLOAD_MAX_FILES", 20, { min: 1, max: 100 }),
|
||||||
|
maxInputPixels: integer(env, "IMAGE_MAX_INPUT_PIXELS", 40_000_000, {
|
||||||
|
min: 1,
|
||||||
|
max: 500_000_000,
|
||||||
|
}),
|
||||||
}),
|
}),
|
||||||
mqtt: Object.freeze({
|
mqtt: Object.freeze({
|
||||||
url: url(env, "MQTT_URL", {
|
url: url(env, "MQTT_URL", {
|
||||||
@@ -193,6 +229,16 @@ function redactedConfig(config) {
|
|||||||
region: config.storage.region,
|
region: config.storage.region,
|
||||||
bucket: config.storage.bucket,
|
bucket: config.storage.bucket,
|
||||||
forcePathStyle: config.storage.forcePathStyle,
|
forcePathStyle: config.storage.forcePathStyle,
|
||||||
|
healthProbeIntervalMs: config.storage.healthProbeIntervalMs,
|
||||||
|
connectionTimeoutMs: config.storage.connectionTimeoutMs,
|
||||||
|
requestTimeoutMs: config.storage.requestTimeoutMs,
|
||||||
|
maxAttempts: config.storage.maxAttempts,
|
||||||
|
},
|
||||||
|
processing: {
|
||||||
|
tempDirectoryConfigured: Boolean(config.processing.tempDirectory),
|
||||||
|
maxFileBytes: config.processing.maxFileBytes,
|
||||||
|
maxFiles: config.processing.maxFiles,
|
||||||
|
maxInputPixels: config.processing.maxInputPixels,
|
||||||
},
|
},
|
||||||
mqtt: {
|
mqtt: {
|
||||||
configured: Boolean(config.mqtt.url),
|
configured: Boolean(config.mqtt.url),
|
||||||
|
|||||||
@@ -14,15 +14,62 @@ function databaseHealth(database) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
function createHealthRouter({ database, lifecycle, migrations }) {
|
function storageHealth(storage) {
|
||||||
|
if (!storage) {
|
||||||
|
return {
|
||||||
|
status: "degraded",
|
||||||
|
checkedAt: null,
|
||||||
|
lastSuccessAt: null,
|
||||||
|
code: "STORAGE_NOT_INITIALIZED",
|
||||||
|
message: "Object storage has not been initialized",
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
try {
|
||||||
|
const result = storage.health();
|
||||||
|
if (result?.status === "healthy") {
|
||||||
|
return {
|
||||||
|
status: "healthy",
|
||||||
|
checkedAt: result.checkedAt || null,
|
||||||
|
lastSuccessAt: result.lastSuccessAt || result.checkedAt || null,
|
||||||
|
};
|
||||||
|
}
|
||||||
|
return {
|
||||||
|
status: "degraded",
|
||||||
|
checkedAt: result?.checkedAt || null,
|
||||||
|
lastSuccessAt: result?.lastSuccessAt || null,
|
||||||
|
code:
|
||||||
|
result?.code === "STORAGE_NOT_CHECKED"
|
||||||
|
? "STORAGE_NOT_CHECKED"
|
||||||
|
: "STORAGE_UNAVAILABLE",
|
||||||
|
message:
|
||||||
|
result?.code === "STORAGE_NOT_CHECKED"
|
||||||
|
? "Object storage has not been checked"
|
||||||
|
: "Object storage is unavailable",
|
||||||
|
};
|
||||||
|
} catch {
|
||||||
|
// A health endpoint must remain available when its dependency is not.
|
||||||
|
}
|
||||||
|
|
||||||
|
return {
|
||||||
|
status: "degraded",
|
||||||
|
checkedAt: null,
|
||||||
|
lastSuccessAt: null,
|
||||||
|
code: "STORAGE_UNAVAILABLE",
|
||||||
|
message: "Object storage is unavailable",
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
function createHealthRouter({ database, lifecycle, migrations, storage }) {
|
||||||
const router = express.Router();
|
const router = express.Router();
|
||||||
|
|
||||||
router.get("/live", (request, response) => {
|
router.get("/live", (request, response) => {
|
||||||
response.json({ status: "ok" });
|
response.json({ status: "ok" });
|
||||||
});
|
});
|
||||||
|
|
||||||
router.get("/ready", (request, response) => {
|
router.get("/ready", async (request, response) => {
|
||||||
const databaseCheck = databaseHealth(database);
|
const databaseCheck = databaseHealth(database);
|
||||||
|
const storageCheck = storageHealth(storage);
|
||||||
const migrationsCurrent = migrations.pendingVersions.length === 0;
|
const migrationsCurrent = migrations.pendingVersions.length === 0;
|
||||||
const ready =
|
const ready =
|
||||||
lifecycle.ready && databaseCheck.status === "ok" && migrationsCurrent;
|
lifecycle.ready && databaseCheck.status === "ok" && migrationsCurrent;
|
||||||
@@ -32,6 +79,7 @@ function createHealthRouter({ database, lifecycle, migrations }) {
|
|||||||
checks: {
|
checks: {
|
||||||
database: databaseCheck.status,
|
database: databaseCheck.status,
|
||||||
migrations: migrationsCurrent ? "current" : "pending",
|
migrations: migrationsCurrent ? "current" : "pending",
|
||||||
|
storage: storageCheck.status,
|
||||||
},
|
},
|
||||||
});
|
});
|
||||||
});
|
});
|
||||||
@@ -39,18 +87,23 @@ function createHealthRouter({ database, lifecycle, migrations }) {
|
|||||||
return router;
|
return router;
|
||||||
}
|
}
|
||||||
|
|
||||||
function createDetailedHealthRouter({ config, database, lifecycle, migrations }) {
|
function createDetailedHealthRouter({ config, database, lifecycle, migrations, storage }) {
|
||||||
const router = express.Router();
|
const router = express.Router();
|
||||||
|
|
||||||
router.get("/", (request, response) => {
|
router.get("/", async (request, response) => {
|
||||||
const databaseCheck = databaseHealth(database);
|
const databaseCheck = databaseHealth(database);
|
||||||
|
const storageCheck = storageHealth(storage);
|
||||||
const ready =
|
const ready =
|
||||||
lifecycle.ready &&
|
lifecycle.ready &&
|
||||||
databaseCheck.status === "ok" &&
|
databaseCheck.status === "ok" &&
|
||||||
migrations.pendingVersions.length === 0;
|
migrations.pendingVersions.length === 0;
|
||||||
|
|
||||||
const configuration = redactedConfig(config);
|
const configuration = redactedConfig(config);
|
||||||
const status = ready ? "degraded" : "not_ready";
|
const status = !ready
|
||||||
|
? "not_ready"
|
||||||
|
: storageCheck.status === "healthy"
|
||||||
|
? "ok"
|
||||||
|
: "degraded";
|
||||||
|
|
||||||
response.status(ready ? 200 : 503).json({
|
response.status(ready ? 200 : 503).json({
|
||||||
status,
|
status,
|
||||||
@@ -62,7 +115,7 @@ function createDetailedHealthRouter({ config, database, lifecycle, migrations })
|
|||||||
latestSchemaVersion: migrations.latestAvailableVersion,
|
latestSchemaVersion: migrations.latestAvailableVersion,
|
||||||
},
|
},
|
||||||
storage: {
|
storage: {
|
||||||
status: "not_checked",
|
...storageCheck,
|
||||||
configured: configuration.storage.configured,
|
configured: configuration.storage.configured,
|
||||||
},
|
},
|
||||||
mqtt: {
|
mqtt: {
|
||||||
@@ -86,4 +139,5 @@ module.exports = {
|
|||||||
createDetailedHealthRouter,
|
createDetailedHealthRouter,
|
||||||
createHealthRouter,
|
createHealthRouter,
|
||||||
databaseHealth,
|
databaseHealth,
|
||||||
|
storageHealth,
|
||||||
};
|
};
|
||||||
|
|||||||
@@ -0,0 +1,330 @@
|
|||||||
|
const crypto = require("node:crypto");
|
||||||
|
const fs = require("node:fs");
|
||||||
|
const sharp = require("sharp");
|
||||||
|
|
||||||
|
const SUPPORTED_FORMATS = Object.freeze(new Set(["jpeg", "png", "webp"]));
|
||||||
|
const CONTENT_TYPES = Object.freeze({
|
||||||
|
jpeg: "image/jpeg",
|
||||||
|
png: "image/png",
|
||||||
|
webp: "image/webp",
|
||||||
|
});
|
||||||
|
const BLACK = Object.freeze({ r: 0, g: 0, b: 0, alpha: 1 });
|
||||||
|
const DEFAULT_IMAGE_DERIVATIVE_OPTIONS = Object.freeze({
|
||||||
|
display: Object.freeze({ maxWidth: 1920, maxHeight: 1080, quality: 82 }),
|
||||||
|
thumbnail: Object.freeze({ maxWidth: 480, maxHeight: 270, quality: 72 }),
|
||||||
|
});
|
||||||
|
|
||||||
|
class ImageProcessingError extends Error {
|
||||||
|
constructor(code, message, { statusCode = 422, details, cause } = {}) {
|
||||||
|
super(message, cause ? { cause } : undefined);
|
||||||
|
this.name = "ImageProcessingError";
|
||||||
|
this.code = code;
|
||||||
|
this.statusCode = statusCode;
|
||||||
|
if (details !== undefined) {
|
||||||
|
this.details = details;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
function requireInteger(value, name, { min, max } = {}) {
|
||||||
|
if (!Number.isSafeInteger(value)) {
|
||||||
|
throw new TypeError(`${name} must be an integer`);
|
||||||
|
}
|
||||||
|
if (min !== undefined && value < min) {
|
||||||
|
throw new RangeError(`${name} must be at least ${min}`);
|
||||||
|
}
|
||||||
|
if (max !== undefined && value > max) {
|
||||||
|
throw new RangeError(`${name} must be at most ${max}`);
|
||||||
|
}
|
||||||
|
return value;
|
||||||
|
}
|
||||||
|
|
||||||
|
function normalizeTargetOptions(target, name, defaults) {
|
||||||
|
if (
|
||||||
|
target !== undefined &&
|
||||||
|
(target === null || typeof target !== "object" || Array.isArray(target))
|
||||||
|
) {
|
||||||
|
throw new TypeError(`${name} image options must be an object`);
|
||||||
|
}
|
||||||
|
|
||||||
|
const configured = { ...defaults, ...target };
|
||||||
|
|
||||||
|
return Object.freeze({
|
||||||
|
maxWidth: requireInteger(configured.maxWidth, `${name}.maxWidth`, { min: 1 }),
|
||||||
|
maxHeight: requireInteger(configured.maxHeight, `${name}.maxHeight`, { min: 1 }),
|
||||||
|
quality: requireInteger(configured.quality, `${name}.quality`, {
|
||||||
|
min: 1,
|
||||||
|
max: 100,
|
||||||
|
}),
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
function normalizeOptions(options) {
|
||||||
|
if (options === null || typeof options !== "object" || Array.isArray(options)) {
|
||||||
|
throw new TypeError("Image processing options are required");
|
||||||
|
}
|
||||||
|
|
||||||
|
return Object.freeze({
|
||||||
|
maxInputPixels: requireInteger(
|
||||||
|
options.maxInputPixels,
|
||||||
|
"maxInputPixels",
|
||||||
|
{ min: 1 },
|
||||||
|
),
|
||||||
|
display: normalizeTargetOptions(
|
||||||
|
options.display,
|
||||||
|
"display",
|
||||||
|
DEFAULT_IMAGE_DERIVATIVE_OPTIONS.display,
|
||||||
|
),
|
||||||
|
thumbnail: normalizeTargetOptions(
|
||||||
|
options.thumbnail,
|
||||||
|
"thumbnail",
|
||||||
|
DEFAULT_IMAGE_DERIVATIVE_OPTIONS.thumbnail,
|
||||||
|
),
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
function sha256(buffer) {
|
||||||
|
return crypto.createHash("sha256").update(buffer).digest("hex");
|
||||||
|
}
|
||||||
|
|
||||||
|
async function hashFile(filename) {
|
||||||
|
const hash = crypto.createHash("sha256");
|
||||||
|
let byteSize = 0;
|
||||||
|
|
||||||
|
try {
|
||||||
|
for await (const chunk of fs.createReadStream(filename)) {
|
||||||
|
byteSize += chunk.length;
|
||||||
|
hash.update(chunk);
|
||||||
|
}
|
||||||
|
} catch (error) {
|
||||||
|
throw new ImageProcessingError(
|
||||||
|
"IMAGE_INPUT_UNREADABLE",
|
||||||
|
"The uploaded image could not be read",
|
||||||
|
{ statusCode: 422, cause: error },
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
return { byteSize, sha256: hash.digest("hex") };
|
||||||
|
}
|
||||||
|
|
||||||
|
async function describeInput(input) {
|
||||||
|
if (Buffer.isBuffer(input)) {
|
||||||
|
return {
|
||||||
|
sharpInput: input,
|
||||||
|
byteSize: input.length,
|
||||||
|
sha256: sha256(input),
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
if (typeof input !== "string" || input.length === 0) {
|
||||||
|
throw new ImageProcessingError(
|
||||||
|
"INVALID_IMAGE_INPUT",
|
||||||
|
"Image input must be a Buffer or temporary file path",
|
||||||
|
{ statusCode: 400 },
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
let stats;
|
||||||
|
try {
|
||||||
|
stats = await fs.promises.stat(input);
|
||||||
|
} catch (error) {
|
||||||
|
throw new ImageProcessingError(
|
||||||
|
"IMAGE_INPUT_UNREADABLE",
|
||||||
|
"The uploaded image could not be read",
|
||||||
|
{ statusCode: 422, cause: error },
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
if (!stats.isFile()) {
|
||||||
|
throw new ImageProcessingError(
|
||||||
|
"INVALID_IMAGE_INPUT",
|
||||||
|
"Image input path must identify a regular file",
|
||||||
|
{ statusCode: 400 },
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
return {
|
||||||
|
sharpInput: input,
|
||||||
|
...(await hashFile(input)),
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
function decodeError(error, maxInputPixels) {
|
||||||
|
if (error instanceof ImageProcessingError) {
|
||||||
|
return error;
|
||||||
|
}
|
||||||
|
|
||||||
|
if (/pixel limit/i.test(error?.message || "")) {
|
||||||
|
return new ImageProcessingError(
|
||||||
|
"IMAGE_PIXEL_LIMIT_EXCEEDED",
|
||||||
|
"The uploaded image exceeds the configured decoded-pixel limit",
|
||||||
|
{
|
||||||
|
statusCode: 413,
|
||||||
|
details: { maxInputPixels },
|
||||||
|
cause: error,
|
||||||
|
},
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
return new ImageProcessingError(
|
||||||
|
"MALFORMED_IMAGE",
|
||||||
|
"The uploaded file is not a valid decodable image",
|
||||||
|
{ statusCode: 422, cause: error },
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
async function readMetadata(input, maxInputPixels) {
|
||||||
|
try {
|
||||||
|
return await sharp(input, {
|
||||||
|
animated: true,
|
||||||
|
failOn: "error",
|
||||||
|
limitInputPixels: maxInputPixels,
|
||||||
|
}).metadata();
|
||||||
|
} catch (error) {
|
||||||
|
throw decodeError(error, maxInputPixels);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
function validateMetadata(metadata) {
|
||||||
|
const pages = metadata.pages ?? 1;
|
||||||
|
|
||||||
|
if (pages > 1) {
|
||||||
|
const hasAnimationTiming =
|
||||||
|
metadata.loop !== undefined ||
|
||||||
|
(Array.isArray(metadata.delay) && metadata.delay.length > 0);
|
||||||
|
throw new ImageProcessingError(
|
||||||
|
hasAnimationTiming
|
||||||
|
? "ANIMATED_IMAGE_NOT_SUPPORTED"
|
||||||
|
: "MULTIPAGE_IMAGE_NOT_SUPPORTED",
|
||||||
|
hasAnimationTiming
|
||||||
|
? "Animated images are not supported"
|
||||||
|
: "Multipage images are not supported",
|
||||||
|
{ statusCode: 422, details: { pages } },
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
if (!SUPPORTED_FORMATS.has(metadata.format)) {
|
||||||
|
throw new ImageProcessingError(
|
||||||
|
"UNSUPPORTED_IMAGE_FORMAT",
|
||||||
|
"Only still JPEG, PNG, and WebP images are supported",
|
||||||
|
{
|
||||||
|
statusCode: 415,
|
||||||
|
details: {
|
||||||
|
detectedFormat: metadata.format || null,
|
||||||
|
supportedFormats: [...SUPPORTED_FORMATS],
|
||||||
|
},
|
||||||
|
},
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
if (
|
||||||
|
!Number.isSafeInteger(metadata.width) ||
|
||||||
|
metadata.width < 1 ||
|
||||||
|
!Number.isSafeInteger(metadata.height) ||
|
||||||
|
metadata.height < 1
|
||||||
|
) {
|
||||||
|
throw new ImageProcessingError(
|
||||||
|
"MALFORMED_IMAGE",
|
||||||
|
"The uploaded image does not contain valid dimensions",
|
||||||
|
{ statusCode: 422 },
|
||||||
|
);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
async function generateDerivative(input, target, maxInputPixels) {
|
||||||
|
let output;
|
||||||
|
try {
|
||||||
|
output = await sharp(input, {
|
||||||
|
failOn: "error",
|
||||||
|
limitInputPixels: maxInputPixels,
|
||||||
|
})
|
||||||
|
.autoOrient()
|
||||||
|
.flatten({ background: BLACK })
|
||||||
|
.resize({
|
||||||
|
width: target.maxWidth,
|
||||||
|
height: target.maxHeight,
|
||||||
|
fit: "inside",
|
||||||
|
withoutEnlargement: true,
|
||||||
|
})
|
||||||
|
.toColourspace("srgb")
|
||||||
|
.webp({ quality: target.quality, effort: 4 })
|
||||||
|
.toBuffer({ resolveWithObject: true });
|
||||||
|
} catch (error) {
|
||||||
|
throw decodeError(error, maxInputPixels);
|
||||||
|
}
|
||||||
|
|
||||||
|
return {
|
||||||
|
buffer: output.data,
|
||||||
|
sha256: sha256(output.data),
|
||||||
|
format: "webp",
|
||||||
|
contentType: CONTENT_TYPES.webp,
|
||||||
|
byteSize: output.data.length,
|
||||||
|
width: output.info.width,
|
||||||
|
height: output.info.height,
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
function originalMetadata(metadata, inputDescription) {
|
||||||
|
const orientedDimensions = metadata.autoOrient || {
|
||||||
|
width: metadata.width,
|
||||||
|
height: metadata.height,
|
||||||
|
};
|
||||||
|
|
||||||
|
return {
|
||||||
|
sha256: inputDescription.sha256,
|
||||||
|
format: metadata.format,
|
||||||
|
contentType: CONTENT_TYPES[metadata.format],
|
||||||
|
byteSize: inputDescription.byteSize,
|
||||||
|
// Width and height describe how the original should appear after applying
|
||||||
|
// its source orientation. The encoded dimensions remain available for
|
||||||
|
// diagnostics without implying that the archived bytes were rewritten.
|
||||||
|
width: orientedDimensions.width,
|
||||||
|
height: orientedDimensions.height,
|
||||||
|
encodedWidth: metadata.width,
|
||||||
|
encodedHeight: metadata.height,
|
||||||
|
orientation: metadata.orientation ?? null,
|
||||||
|
pages: metadata.pages ?? 1,
|
||||||
|
channels: metadata.channels,
|
||||||
|
hasAlpha: Boolean(metadata.hasAlpha),
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
function createImageProcessor(options) {
|
||||||
|
const config = normalizeOptions(options);
|
||||||
|
|
||||||
|
async function process(input) {
|
||||||
|
const inputDescription = await describeInput(input);
|
||||||
|
const metadata = await readMetadata(
|
||||||
|
inputDescription.sharpInput,
|
||||||
|
config.maxInputPixels,
|
||||||
|
);
|
||||||
|
validateMetadata(metadata);
|
||||||
|
|
||||||
|
const [display, thumbnail] = await Promise.all([
|
||||||
|
generateDerivative(
|
||||||
|
inputDescription.sharpInput,
|
||||||
|
config.display,
|
||||||
|
config.maxInputPixels,
|
||||||
|
),
|
||||||
|
generateDerivative(
|
||||||
|
inputDescription.sharpInput,
|
||||||
|
config.thumbnail,
|
||||||
|
config.maxInputPixels,
|
||||||
|
),
|
||||||
|
]);
|
||||||
|
|
||||||
|
return {
|
||||||
|
original: originalMetadata(metadata, inputDescription),
|
||||||
|
display,
|
||||||
|
thumbnail,
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
return Object.freeze({ process });
|
||||||
|
}
|
||||||
|
|
||||||
|
module.exports = {
|
||||||
|
DEFAULT_IMAGE_DERIVATIVE_OPTIONS,
|
||||||
|
ImageProcessingError,
|
||||||
|
createImageProcessor,
|
||||||
|
};
|
||||||
+43
-2
@@ -3,6 +3,8 @@ const { Server: SocketServer } = require("socket.io");
|
|||||||
const { createApp } = require("./app");
|
const { createApp } = require("./app");
|
||||||
const { openDatabase } = require("./database");
|
const { openDatabase } = require("./database");
|
||||||
const { runMigrations } = require("./database/migrate");
|
const { runMigrations } = require("./database/migrate");
|
||||||
|
const { createImageProcessor } = require("./image-processing");
|
||||||
|
const { createStorageService } = require("./storage");
|
||||||
|
|
||||||
function listen(httpServer, port, host) {
|
function listen(httpServer, port, host) {
|
||||||
return new Promise((resolve, reject) => {
|
return new Promise((resolve, reject) => {
|
||||||
@@ -20,7 +22,12 @@ function listen(httpServer, port, host) {
|
|||||||
});
|
});
|
||||||
}
|
}
|
||||||
|
|
||||||
function createRuntime({ config, logger = console }) {
|
function createRuntime({
|
||||||
|
config,
|
||||||
|
imageProcessor: suppliedImageProcessor,
|
||||||
|
logger = console,
|
||||||
|
storage: suppliedStorage,
|
||||||
|
}) {
|
||||||
const database = openDatabase(config.database);
|
const database = openDatabase(config.database);
|
||||||
let migrations;
|
let migrations;
|
||||||
try {
|
try {
|
||||||
@@ -34,7 +41,32 @@ function createRuntime({ config, logger = console }) {
|
|||||||
ready: false,
|
ready: false,
|
||||||
stopping: false,
|
stopping: false,
|
||||||
};
|
};
|
||||||
const app = createApp({ config, database, lifecycle, migrations });
|
let storage;
|
||||||
|
let imageProcessor;
|
||||||
|
try {
|
||||||
|
storage = suppliedStorage || createStorageService(config.storage, { logger });
|
||||||
|
imageProcessor =
|
||||||
|
suppliedImageProcessor ||
|
||||||
|
createImageProcessor({ maxInputPixels: config.processing.maxInputPixels });
|
||||||
|
} catch (error) {
|
||||||
|
database.close();
|
||||||
|
throw error;
|
||||||
|
}
|
||||||
|
let app;
|
||||||
|
try {
|
||||||
|
app = createApp({
|
||||||
|
config,
|
||||||
|
database,
|
||||||
|
imageProcessor,
|
||||||
|
lifecycle,
|
||||||
|
migrations,
|
||||||
|
storage,
|
||||||
|
});
|
||||||
|
} catch (error) {
|
||||||
|
storage.stop();
|
||||||
|
database.close();
|
||||||
|
throw error;
|
||||||
|
}
|
||||||
const httpServer = http.createServer(app);
|
const httpServer = http.createServer(app);
|
||||||
const io = new SocketServer(httpServer, {
|
const io = new SocketServer(httpServer, {
|
||||||
serveClient: false,
|
serveClient: false,
|
||||||
@@ -63,9 +95,15 @@ function createRuntime({ config, logger = console }) {
|
|||||||
.then((listeningAddress) => {
|
.then((listeningAddress) => {
|
||||||
address = listeningAddress;
|
address = listeningAddress;
|
||||||
lifecycle.ready = true;
|
lifecycle.ready = true;
|
||||||
|
Promise.resolve()
|
||||||
|
.then(() => storage.start())
|
||||||
|
.catch(() => {
|
||||||
|
logger.warn?.("Object storage health checks could not be started");
|
||||||
|
});
|
||||||
return listeningAddress;
|
return listeningAddress;
|
||||||
})
|
})
|
||||||
.catch((error) => {
|
.catch((error) => {
|
||||||
|
storage.stop();
|
||||||
database.close();
|
database.close();
|
||||||
throw error;
|
throw error;
|
||||||
});
|
});
|
||||||
@@ -85,6 +123,7 @@ function createRuntime({ config, logger = console }) {
|
|||||||
if (settled) return;
|
if (settled) return;
|
||||||
settled = true;
|
settled = true;
|
||||||
clearTimeout(timeout);
|
clearTimeout(timeout);
|
||||||
|
storage.stop();
|
||||||
if (database.open) {
|
if (database.open) {
|
||||||
database.close();
|
database.close();
|
||||||
}
|
}
|
||||||
@@ -119,6 +158,8 @@ function createRuntime({ config, logger = console }) {
|
|||||||
io,
|
io,
|
||||||
lifecycle,
|
lifecycle,
|
||||||
migrations,
|
migrations,
|
||||||
|
storage,
|
||||||
|
imageProcessor,
|
||||||
get address() {
|
get address() {
|
||||||
return address;
|
return address;
|
||||||
},
|
},
|
||||||
|
|||||||
@@ -0,0 +1,398 @@
|
|||||||
|
const {
|
||||||
|
CreateBucketCommand,
|
||||||
|
DeleteObjectCommand,
|
||||||
|
GetObjectCommand,
|
||||||
|
HeadBucketCommand,
|
||||||
|
HeadObjectCommand,
|
||||||
|
PutObjectCommand,
|
||||||
|
S3Client,
|
||||||
|
} = require("@aws-sdk/client-s3");
|
||||||
|
|
||||||
|
class StorageError extends Error {
|
||||||
|
constructor(message, { code, statusCode, cause } = {}) {
|
||||||
|
super(message, { cause });
|
||||||
|
this.name = "StorageError";
|
||||||
|
this.code = code || "STORAGE_ERROR";
|
||||||
|
this.statusCode = statusCode || 500;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
class StorageUnavailableError extends StorageError {
|
||||||
|
constructor(cause) {
|
||||||
|
super("Object storage is unavailable", {
|
||||||
|
code: "STORAGE_UNAVAILABLE",
|
||||||
|
statusCode: 503,
|
||||||
|
cause,
|
||||||
|
});
|
||||||
|
this.name = "StorageUnavailableError";
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
class StorageObjectNotFoundError extends StorageError {
|
||||||
|
constructor(key, cause) {
|
||||||
|
super("Stored object was not found", {
|
||||||
|
code: "STORAGE_OBJECT_NOT_FOUND",
|
||||||
|
statusCode: 404,
|
||||||
|
cause,
|
||||||
|
});
|
||||||
|
this.name = "StorageObjectNotFoundError";
|
||||||
|
this.key = key;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
function errorStatus(error) {
|
||||||
|
return error?.$metadata?.httpStatusCode || error?.statusCode;
|
||||||
|
}
|
||||||
|
|
||||||
|
function isMissingBucket(error) {
|
||||||
|
return (
|
||||||
|
error?.name === "NoSuchBucket" ||
|
||||||
|
error?.Code === "NoSuchBucket" ||
|
||||||
|
error?.code === "NoSuchBucket" ||
|
||||||
|
errorStatus(error) === 404
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
function isMissingObject(error) {
|
||||||
|
if (
|
||||||
|
error?.name === "NoSuchBucket" ||
|
||||||
|
error?.Code === "NoSuchBucket" ||
|
||||||
|
error?.code === "NoSuchBucket"
|
||||||
|
) {
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
return (
|
||||||
|
error?.name === "NoSuchKey" ||
|
||||||
|
error?.name === "NotFound" ||
|
||||||
|
error?.Code === "NoSuchKey" ||
|
||||||
|
error?.code === "NoSuchKey" ||
|
||||||
|
errorStatus(error) === 404
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
function isBucketCreationConflict(error) {
|
||||||
|
return (
|
||||||
|
error?.name === "BucketAlreadyOwnedByYou" ||
|
||||||
|
error?.name === "BucketAlreadyExists" ||
|
||||||
|
error?.Code === "BucketAlreadyOwnedByYou" ||
|
||||||
|
error?.Code === "BucketAlreadyExists" ||
|
||||||
|
error?.code === "BucketAlreadyOwnedByYou" ||
|
||||||
|
error?.code === "BucketAlreadyExists"
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
function isConsumerCancellation(error) {
|
||||||
|
return (
|
||||||
|
error?.name === "AbortError" ||
|
||||||
|
error?.code === "ABORT_ERR" ||
|
||||||
|
error?.code === "ERR_STREAM_PREMATURE_CLOSE"
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
function assertObjectKey(key) {
|
||||||
|
if (typeof key !== "string" || key.length === 0) {
|
||||||
|
throw new TypeError("Storage object key must be a non-empty string");
|
||||||
|
}
|
||||||
|
|
||||||
|
const segments = key.split("/");
|
||||||
|
if (
|
||||||
|
key.startsWith("/") ||
|
||||||
|
key.includes("\\") ||
|
||||||
|
key.includes("\0") ||
|
||||||
|
segments.some((segment) => segment === "." || segment === "..")
|
||||||
|
) {
|
||||||
|
throw new TypeError("Storage object key is invalid");
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
function publicHealth(health) {
|
||||||
|
return Object.freeze({ ...health });
|
||||||
|
}
|
||||||
|
|
||||||
|
function createS3Client(config) {
|
||||||
|
const options = {
|
||||||
|
endpoint: config.endpoint,
|
||||||
|
region: config.region,
|
||||||
|
forcePathStyle: config.forcePathStyle,
|
||||||
|
maxAttempts: config.maxAttempts,
|
||||||
|
requestHandler: {
|
||||||
|
connectionTimeout: config.connectionTimeoutMs,
|
||||||
|
// Use an inactivity timeout rather than an absolute request deadline so
|
||||||
|
// a large but continuously progressing upload is not aborted.
|
||||||
|
socketTimeout: config.requestTimeoutMs,
|
||||||
|
},
|
||||||
|
// Optional AWS checksums are not uniformly implemented by S3-compatible
|
||||||
|
// providers. Source and derivative SHA-256 hashes are verified by the app.
|
||||||
|
requestChecksumCalculation: "WHEN_REQUIRED",
|
||||||
|
responseChecksumValidation: "WHEN_REQUIRED",
|
||||||
|
};
|
||||||
|
|
||||||
|
if (config.accessKeyId && config.secretAccessKey) {
|
||||||
|
options.credentials = {
|
||||||
|
accessKeyId: config.accessKeyId,
|
||||||
|
secretAccessKey: config.secretAccessKey,
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
return new S3Client(options);
|
||||||
|
}
|
||||||
|
|
||||||
|
function createStorageService(config, { client, logger = console, now = () => new Date() } = {}) {
|
||||||
|
const s3 = client || createS3Client(config);
|
||||||
|
let ensurePromise = null;
|
||||||
|
let healthTimer = null;
|
||||||
|
let bucketReady = false;
|
||||||
|
let issuedHealthSequence = 0;
|
||||||
|
let appliedHealthSequence = 0;
|
||||||
|
let currentHealth = {
|
||||||
|
status: "degraded",
|
||||||
|
checkedAt: null,
|
||||||
|
lastSuccessAt: null,
|
||||||
|
code: "STORAGE_NOT_CHECKED",
|
||||||
|
message: "Object storage has not been checked",
|
||||||
|
};
|
||||||
|
|
||||||
|
function nextHealthSequence() {
|
||||||
|
issuedHealthSequence += 1;
|
||||||
|
return issuedHealthSequence;
|
||||||
|
}
|
||||||
|
|
||||||
|
function markHealthy(sequence) {
|
||||||
|
if (sequence < appliedHealthSequence) return publicHealth(currentHealth);
|
||||||
|
appliedHealthSequence = sequence;
|
||||||
|
bucketReady = true;
|
||||||
|
const checkedAt = now().toISOString();
|
||||||
|
currentHealth = {
|
||||||
|
status: "healthy",
|
||||||
|
checkedAt,
|
||||||
|
lastSuccessAt: checkedAt,
|
||||||
|
};
|
||||||
|
return publicHealth(currentHealth);
|
||||||
|
}
|
||||||
|
|
||||||
|
function markDegraded(sequence) {
|
||||||
|
if (sequence < appliedHealthSequence) return publicHealth(currentHealth);
|
||||||
|
appliedHealthSequence = sequence;
|
||||||
|
bucketReady = false;
|
||||||
|
currentHealth = {
|
||||||
|
status: "degraded",
|
||||||
|
checkedAt: now().toISOString(),
|
||||||
|
lastSuccessAt: currentHealth.lastSuccessAt,
|
||||||
|
code: "STORAGE_UNAVAILABLE",
|
||||||
|
message: "Object storage is unavailable",
|
||||||
|
};
|
||||||
|
return publicHealth(currentHealth);
|
||||||
|
}
|
||||||
|
|
||||||
|
function unavailable(error, sequence) {
|
||||||
|
markDegraded(sequence);
|
||||||
|
return error instanceof StorageUnavailableError
|
||||||
|
? error
|
||||||
|
: new StorageUnavailableError(error);
|
||||||
|
}
|
||||||
|
|
||||||
|
async function ensureBucketOnce() {
|
||||||
|
const sequence = nextHealthSequence();
|
||||||
|
try {
|
||||||
|
await s3.send(new HeadBucketCommand({ Bucket: config.bucket }));
|
||||||
|
markHealthy(sequence);
|
||||||
|
return { created: false };
|
||||||
|
} catch (error) {
|
||||||
|
if (!isMissingBucket(error)) {
|
||||||
|
throw unavailable(error, sequence);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
try {
|
||||||
|
await s3.send(new CreateBucketCommand({ Bucket: config.bucket }));
|
||||||
|
markHealthy(sequence);
|
||||||
|
return { created: true };
|
||||||
|
} catch (error) {
|
||||||
|
if (isBucketCreationConflict(error)) {
|
||||||
|
try {
|
||||||
|
await s3.send(new HeadBucketCommand({ Bucket: config.bucket }));
|
||||||
|
markHealthy(sequence);
|
||||||
|
return { created: false };
|
||||||
|
} catch (verificationError) {
|
||||||
|
throw unavailable(verificationError, sequence);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
throw unavailable(error, sequence);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
async function ensureBucket() {
|
||||||
|
if (!ensurePromise) {
|
||||||
|
ensurePromise = ensureBucketOnce().finally(() => {
|
||||||
|
ensurePromise = null;
|
||||||
|
});
|
||||||
|
}
|
||||||
|
return ensurePromise;
|
||||||
|
}
|
||||||
|
|
||||||
|
async function initialize() {
|
||||||
|
try {
|
||||||
|
await ensureBucket();
|
||||||
|
} catch (error) {
|
||||||
|
logger.warn?.("Object storage is unavailable during initialization");
|
||||||
|
}
|
||||||
|
return publicHealth(currentHealth);
|
||||||
|
}
|
||||||
|
|
||||||
|
async function start() {
|
||||||
|
if (!healthTimer) {
|
||||||
|
healthTimer = setInterval(() => {
|
||||||
|
ensureBucket().catch(() => {});
|
||||||
|
}, config.healthProbeIntervalMs || 30_000);
|
||||||
|
healthTimer.unref?.();
|
||||||
|
}
|
||||||
|
return initialize();
|
||||||
|
}
|
||||||
|
|
||||||
|
async function probe() {
|
||||||
|
const sequence = nextHealthSequence();
|
||||||
|
try {
|
||||||
|
await s3.send(new HeadBucketCommand({ Bucket: config.bucket }));
|
||||||
|
return markHealthy(sequence);
|
||||||
|
} catch (error) {
|
||||||
|
return markDegraded(sequence);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
async function putObject({ key, body, contentType, contentLength, metadata }) {
|
||||||
|
assertObjectKey(key);
|
||||||
|
if (!bucketReady) await ensureBucket();
|
||||||
|
const sequence = nextHealthSequence();
|
||||||
|
const input = {
|
||||||
|
Bucket: config.bucket,
|
||||||
|
Key: key,
|
||||||
|
Body: body,
|
||||||
|
...(contentType ? { ContentType: contentType } : {}),
|
||||||
|
...(contentLength !== undefined ? { ContentLength: contentLength } : {}),
|
||||||
|
...(metadata ? { Metadata: metadata } : {}),
|
||||||
|
};
|
||||||
|
|
||||||
|
try {
|
||||||
|
const result = await s3.send(new PutObjectCommand(input));
|
||||||
|
markHealthy(sequence);
|
||||||
|
return {
|
||||||
|
key,
|
||||||
|
etag: result.ETag || null,
|
||||||
|
versionId: result.VersionId || null,
|
||||||
|
};
|
||||||
|
} catch (error) {
|
||||||
|
throw unavailable(error, sequence);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
function objectMetadata(key, result) {
|
||||||
|
return {
|
||||||
|
key,
|
||||||
|
contentType: result.ContentType || null,
|
||||||
|
contentLength:
|
||||||
|
result.ContentLength === undefined ? null : Number(result.ContentLength),
|
||||||
|
etag: result.ETag || null,
|
||||||
|
lastModified: result.LastModified || null,
|
||||||
|
metadata: result.Metadata || {},
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
async function getObject(key) {
|
||||||
|
assertObjectKey(key);
|
||||||
|
if (!bucketReady) await ensureBucket();
|
||||||
|
const sequence = nextHealthSequence();
|
||||||
|
try {
|
||||||
|
const result = await s3.send(
|
||||||
|
new GetObjectCommand({ Bucket: config.bucket, Key: key }),
|
||||||
|
);
|
||||||
|
markHealthy(sequence);
|
||||||
|
result.Body?.once?.("error", (error) => {
|
||||||
|
if (!isConsumerCancellation(error)) {
|
||||||
|
markDegraded(nextHealthSequence());
|
||||||
|
}
|
||||||
|
});
|
||||||
|
return {
|
||||||
|
...objectMetadata(key, result),
|
||||||
|
body: result.Body,
|
||||||
|
};
|
||||||
|
} catch (error) {
|
||||||
|
if (isMissingObject(error)) {
|
||||||
|
markHealthy(sequence);
|
||||||
|
throw new StorageObjectNotFoundError(key, error);
|
||||||
|
}
|
||||||
|
throw unavailable(error, sequence);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
async function headObject(key) {
|
||||||
|
assertObjectKey(key);
|
||||||
|
if (!bucketReady) await ensureBucket();
|
||||||
|
const sequence = nextHealthSequence();
|
||||||
|
try {
|
||||||
|
const result = await s3.send(
|
||||||
|
new HeadObjectCommand({ Bucket: config.bucket, Key: key }),
|
||||||
|
);
|
||||||
|
markHealthy(sequence);
|
||||||
|
return objectMetadata(key, result);
|
||||||
|
} catch (error) {
|
||||||
|
if (isMissingObject(error)) {
|
||||||
|
markHealthy(sequence);
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
throw unavailable(error, sequence);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
async function objectExists(key) {
|
||||||
|
return (await headObject(key)) !== null;
|
||||||
|
}
|
||||||
|
|
||||||
|
async function deleteObject(key) {
|
||||||
|
assertObjectKey(key);
|
||||||
|
if (!bucketReady) await ensureBucket();
|
||||||
|
const sequence = nextHealthSequence();
|
||||||
|
try {
|
||||||
|
await s3.send(
|
||||||
|
new DeleteObjectCommand({ Bucket: config.bucket, Key: key }),
|
||||||
|
);
|
||||||
|
markHealthy(sequence);
|
||||||
|
} catch (error) {
|
||||||
|
if (isMissingObject(error)) {
|
||||||
|
markHealthy(sequence);
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
throw unavailable(error, sequence);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
function stop() {
|
||||||
|
if (healthTimer) {
|
||||||
|
clearInterval(healthTimer);
|
||||||
|
healthTimer = null;
|
||||||
|
}
|
||||||
|
s3.destroy?.();
|
||||||
|
}
|
||||||
|
|
||||||
|
return Object.freeze({
|
||||||
|
initialize,
|
||||||
|
start,
|
||||||
|
stop,
|
||||||
|
ensureBucket,
|
||||||
|
probe,
|
||||||
|
health: () => publicHealth(currentHealth),
|
||||||
|
putObject,
|
||||||
|
getObject,
|
||||||
|
headObject,
|
||||||
|
objectExists,
|
||||||
|
deleteObject,
|
||||||
|
destroy: stop,
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
module.exports = {
|
||||||
|
StorageError,
|
||||||
|
StorageObjectNotFoundError,
|
||||||
|
StorageUnavailableError,
|
||||||
|
createS3Client,
|
||||||
|
createStorageService,
|
||||||
|
};
|
||||||
@@ -0,0 +1,442 @@
|
|||||||
|
const test = require("node:test");
|
||||||
|
const assert = require("node:assert/strict");
|
||||||
|
const crypto = require("node:crypto");
|
||||||
|
const fs = require("node:fs");
|
||||||
|
const http = require("node:http");
|
||||||
|
const os = require("node:os");
|
||||||
|
const path = require("node:path");
|
||||||
|
const { Readable } = require("node:stream");
|
||||||
|
const sharp = require("sharp");
|
||||||
|
const { createApp } = require("../src/app");
|
||||||
|
const { loadConfig } = require("../src/config");
|
||||||
|
const { openDatabase } = require("../src/database");
|
||||||
|
const { runMigrations } = require("../src/database/migrate");
|
||||||
|
const { createImageProcessor } = require("../src/image-processing");
|
||||||
|
const { StorageObjectNotFoundError } = require("../src/storage");
|
||||||
|
|
||||||
|
async function readBody(body) {
|
||||||
|
if (Buffer.isBuffer(body)) return body;
|
||||||
|
const chunks = [];
|
||||||
|
for await (const chunk of body) chunks.push(Buffer.from(chunk));
|
||||||
|
return Buffer.concat(chunks);
|
||||||
|
}
|
||||||
|
|
||||||
|
function memoryStorage() {
|
||||||
|
const objects = new Map();
|
||||||
|
return {
|
||||||
|
objects,
|
||||||
|
health() {
|
||||||
|
return {
|
||||||
|
status: "healthy",
|
||||||
|
checkedAt: "2033-05-18T03:33:20.000Z",
|
||||||
|
lastSuccessAt: "2033-05-18T03:33:20.000Z",
|
||||||
|
};
|
||||||
|
},
|
||||||
|
async putObject(input) {
|
||||||
|
objects.set(input.key, {
|
||||||
|
body: await readBody(input.body),
|
||||||
|
contentType: input.contentType,
|
||||||
|
metadata: input.metadata,
|
||||||
|
});
|
||||||
|
return { key: input.key };
|
||||||
|
},
|
||||||
|
async getObject(key) {
|
||||||
|
const object = objects.get(key);
|
||||||
|
if (!object) throw new StorageObjectNotFoundError(key);
|
||||||
|
return {
|
||||||
|
body: Readable.from([object.body]),
|
||||||
|
contentLength: object.body.length,
|
||||||
|
contentType: object.contentType,
|
||||||
|
};
|
||||||
|
},
|
||||||
|
async deleteObject(key) {
|
||||||
|
objects.delete(key);
|
||||||
|
return { key, deleted: true };
|
||||||
|
},
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
async function createHarness(t, { maxFileBytes = 1024 * 1024 } = {}) {
|
||||||
|
const tempDirectory = fs.mkdtempSync(
|
||||||
|
path.join(os.tmpdir(), "announcements-api-uploads-"),
|
||||||
|
);
|
||||||
|
const config = loadConfig({
|
||||||
|
NODE_ENV: "test",
|
||||||
|
HOST: "127.0.0.1",
|
||||||
|
PORT: "0",
|
||||||
|
DATABASE_PATH: ":memory:",
|
||||||
|
APP_TIMEZONE: "America/Chicago",
|
||||||
|
S3_ENDPOINT: "http://private-seaweedfs:8333",
|
||||||
|
S3_BUCKET: "announcements-test",
|
||||||
|
UPLOAD_TEMP_DIRECTORY: tempDirectory,
|
||||||
|
UPLOAD_MAX_FILE_BYTES: String(maxFileBytes),
|
||||||
|
UPLOAD_MAX_FILES: "5",
|
||||||
|
IMAGE_MAX_INPUT_PIXELS: "1000000",
|
||||||
|
});
|
||||||
|
const database = openDatabase({ path: ":memory:" });
|
||||||
|
const migrations = runMigrations(database);
|
||||||
|
const storage = memoryStorage();
|
||||||
|
const app = createApp({
|
||||||
|
config,
|
||||||
|
database,
|
||||||
|
imageProcessor: createImageProcessor({
|
||||||
|
maxInputPixels: config.processing.maxInputPixels,
|
||||||
|
}),
|
||||||
|
lifecycle: { ready: true },
|
||||||
|
migrations,
|
||||||
|
storage,
|
||||||
|
});
|
||||||
|
const server = app.listen(0, "127.0.0.1");
|
||||||
|
await new Promise((resolve, reject) => {
|
||||||
|
server.once("listening", resolve);
|
||||||
|
server.once("error", reject);
|
||||||
|
});
|
||||||
|
|
||||||
|
t.after(async () => {
|
||||||
|
await new Promise((resolve) => server.close(resolve));
|
||||||
|
database.close();
|
||||||
|
fs.rmSync(tempDirectory, { recursive: true, force: true });
|
||||||
|
});
|
||||||
|
|
||||||
|
return {
|
||||||
|
baseUrl: `http://127.0.0.1:${server.address().port}`,
|
||||||
|
database,
|
||||||
|
storage,
|
||||||
|
tempDirectory,
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
function appendFile(form, field, buffer, filename, contentType) {
|
||||||
|
form.append(field, new Blob([buffer], { type: contentType }), filename);
|
||||||
|
}
|
||||||
|
|
||||||
|
async function jsonResponse(baseUrl, route, init) {
|
||||||
|
const response = await fetch(`${baseUrl}${route}`, init);
|
||||||
|
return { response, body: await response.json() };
|
||||||
|
}
|
||||||
|
|
||||||
|
async function eventuallyEmpty(directory) {
|
||||||
|
for (let attempt = 0; attempt < 50; attempt += 1) {
|
||||||
|
if (fs.readdirSync(directory).length === 0) return;
|
||||||
|
await new Promise((resolve) => setTimeout(resolve, 10));
|
||||||
|
}
|
||||||
|
assert.deepEqual(fs.readdirSync(directory), []);
|
||||||
|
}
|
||||||
|
|
||||||
|
async function eventuallyContainsFile(directory) {
|
||||||
|
for (let attempt = 0; attempt < 50; attempt += 1) {
|
||||||
|
if (fs.readdirSync(directory).length > 0) return;
|
||||||
|
await new Promise((resolve) => setTimeout(resolve, 10));
|
||||||
|
}
|
||||||
|
assert.fail("The upload did not create its temporary file");
|
||||||
|
}
|
||||||
|
|
||||||
|
async function sampleImages() {
|
||||||
|
const source = sharp({
|
||||||
|
create: {
|
||||||
|
width: 80,
|
||||||
|
height: 60,
|
||||||
|
channels: 4,
|
||||||
|
background: { r: 31, g: 72, b: 120, alpha: 0.5 },
|
||||||
|
},
|
||||||
|
});
|
||||||
|
return {
|
||||||
|
jpeg: await source.clone().jpeg().toBuffer(),
|
||||||
|
png: await source.clone().png().toBuffer(),
|
||||||
|
webp: await source.clone().webp().toBuffer(),
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
test("asset HTTP API accepts JPEG, PNG, and WebP and serves private immutable media", async (t) => {
|
||||||
|
const harness = await createHarness(t);
|
||||||
|
const images = await sampleImages();
|
||||||
|
const form = new FormData();
|
||||||
|
appendFile(form, "files", images.jpeg, "welcome.jpg", "image/jpeg");
|
||||||
|
appendFile(form, "files", images.png, "../../bulletin.png", "image/png");
|
||||||
|
appendFile(form, "files", images.webp, "events.webp", "image/webp");
|
||||||
|
|
||||||
|
const uploaded = await jsonResponse(harness.baseUrl, "/api/assets", {
|
||||||
|
method: "POST",
|
||||||
|
body: form,
|
||||||
|
});
|
||||||
|
assert.equal(uploaded.response.status, 201);
|
||||||
|
assert.equal(uploaded.body.createdCount, 3);
|
||||||
|
assert.equal(uploaded.body.duplicateCount, 0);
|
||||||
|
assert.equal(uploaded.body.results.length, 3);
|
||||||
|
assert.deepEqual(
|
||||||
|
uploaded.body.results.map((result) => result.asset.original.contentType),
|
||||||
|
["image/jpeg", "image/png", "image/webp"],
|
||||||
|
);
|
||||||
|
assert.equal(uploaded.body.results[1].asset.originalFilename, "bulletin.png");
|
||||||
|
assert.doesNotMatch(JSON.stringify(uploaded.body), /objectKey|object_key|private-seaweedfs/);
|
||||||
|
assert.equal(harness.storage.objects.size, 9);
|
||||||
|
await eventuallyEmpty(harness.tempDirectory);
|
||||||
|
|
||||||
|
const asset = uploaded.body.results[0].asset;
|
||||||
|
const listed = await jsonResponse(harness.baseUrl, "/api/assets");
|
||||||
|
assert.equal(listed.response.status, 200);
|
||||||
|
assert.equal(listed.body.assets.length, 3);
|
||||||
|
assert.deepEqual(
|
||||||
|
(await jsonResponse(harness.baseUrl, `/api/assets/${asset.id}`)).body,
|
||||||
|
asset,
|
||||||
|
);
|
||||||
|
|
||||||
|
for (const media of [asset.display, asset.thumbnail]) {
|
||||||
|
const response = await fetch(`${harness.baseUrl}${media.url}`);
|
||||||
|
const body = Buffer.from(await response.arrayBuffer());
|
||||||
|
assert.equal(response.status, 200);
|
||||||
|
assert.equal(response.headers.get("content-type"), "image/webp");
|
||||||
|
assert.equal(
|
||||||
|
response.headers.get("cache-control"),
|
||||||
|
"public, max-age=31536000, immutable",
|
||||||
|
);
|
||||||
|
assert.equal(response.headers.get("content-length"), String(body.length));
|
||||||
|
assert.equal(response.headers.get("etag"), `"${media.sha256}"`);
|
||||||
|
assert.equal(crypto.createHash("sha256").update(body).digest("hex"), media.sha256);
|
||||||
|
}
|
||||||
|
|
||||||
|
const original = await fetch(`${harness.baseUrl}${asset.original.url}`);
|
||||||
|
assert.equal(original.status, 200);
|
||||||
|
assert.equal(original.headers.get("content-type"), "image/jpeg");
|
||||||
|
assert.equal(original.headers.get("cache-control"), "private, no-store");
|
||||||
|
assert.match(original.headers.get("content-disposition"), /^attachment;/);
|
||||||
|
assert.deepEqual(Buffer.from(await original.arrayBuffer()), images.jpeg);
|
||||||
|
|
||||||
|
const wrongVersion = await jsonResponse(
|
||||||
|
harness.baseUrl,
|
||||||
|
`/media/assets/${asset.id}/display/${"a".repeat(64)}.webp`,
|
||||||
|
);
|
||||||
|
assert.equal(wrongVersion.response.status, 404);
|
||||||
|
assert.equal(wrongVersion.body.error.code, "MEDIA_NOT_FOUND");
|
||||||
|
const arbitraryKey = await jsonResponse(
|
||||||
|
harness.baseUrl,
|
||||||
|
`/media/assets/${asset.id}/display/${encodeURIComponent("../../original/key")}.webp`,
|
||||||
|
);
|
||||||
|
assert.equal(arbitraryKey.response.status, 404);
|
||||||
|
assert.equal(arbitraryKey.body.error.code, "MEDIA_NOT_FOUND");
|
||||||
|
|
||||||
|
const displayKey = [...harness.storage.objects.keys()].find(
|
||||||
|
(key) => key.includes(`assets/${asset.id}/display/`),
|
||||||
|
);
|
||||||
|
harness.storage.objects.delete(displayKey);
|
||||||
|
const missingStoredBytes = await jsonResponse(
|
||||||
|
harness.baseUrl,
|
||||||
|
asset.display.url,
|
||||||
|
);
|
||||||
|
assert.equal(missingStoredBytes.response.status, 503);
|
||||||
|
assert.equal(missingStoredBytes.body.error.code, "STORAGE_INCONSISTENCY");
|
||||||
|
assert.equal(
|
||||||
|
(await jsonResponse(harness.baseUrl, "/health/live")).response.status,
|
||||||
|
200,
|
||||||
|
);
|
||||||
|
});
|
||||||
|
|
||||||
|
test("asset HTTP API reports duplicates, name conflicts, and persists editable metadata", async (t) => {
|
||||||
|
const harness = await createHarness(t);
|
||||||
|
const { png, jpeg } = await sampleImages();
|
||||||
|
const firstForm = new FormData();
|
||||||
|
appendFile(firstForm, "file", png, "weekly.png", "application/octet-stream");
|
||||||
|
firstForm.append("display_name", "Weekly Announcement");
|
||||||
|
const first = await jsonResponse(harness.baseUrl, "/api/assets", {
|
||||||
|
method: "POST",
|
||||||
|
body: firstForm,
|
||||||
|
});
|
||||||
|
assert.equal(first.response.status, 201);
|
||||||
|
assert.equal(first.body.created, true);
|
||||||
|
const assetId = first.body.asset.id;
|
||||||
|
|
||||||
|
const duplicateForm = new FormData();
|
||||||
|
appendFile(duplicateForm, "file", png, "unrelated-name.png", "image/png");
|
||||||
|
duplicateForm.append("display_name", "Ignored duplicate proposal");
|
||||||
|
const duplicate = await jsonResponse(harness.baseUrl, "/api/assets", {
|
||||||
|
method: "POST",
|
||||||
|
body: duplicateForm,
|
||||||
|
});
|
||||||
|
assert.equal(duplicate.response.status, 200);
|
||||||
|
assert.equal(duplicate.body.created, false);
|
||||||
|
assert.equal(duplicate.body.duplicate, true);
|
||||||
|
assert.equal(duplicate.body.asset.id, assetId);
|
||||||
|
assert.equal(harness.storage.objects.size, 3);
|
||||||
|
|
||||||
|
const conflictForm = new FormData();
|
||||||
|
appendFile(conflictForm, "file", jpeg, "different.jpg", "image/jpeg");
|
||||||
|
conflictForm.append("displayName", "weekly announcement");
|
||||||
|
const conflict = await jsonResponse(harness.baseUrl, "/api/assets", {
|
||||||
|
method: "POST",
|
||||||
|
body: conflictForm,
|
||||||
|
});
|
||||||
|
assert.equal(conflict.response.status, 409);
|
||||||
|
assert.equal(conflict.body.error.code, "ASSET_NAME_CONFLICT");
|
||||||
|
|
||||||
|
const expiresAt = Date.parse("2034-01-02T03:04:05.000Z");
|
||||||
|
const edited = await jsonResponse(harness.baseUrl, `/api/assets/${assetId}`, {
|
||||||
|
method: "PATCH",
|
||||||
|
headers: { "content-type": "application/json" },
|
||||||
|
body: JSON.stringify({
|
||||||
|
display_name: "Renamed announcement",
|
||||||
|
expiration_at: expiresAt,
|
||||||
|
}),
|
||||||
|
});
|
||||||
|
assert.equal(edited.response.status, 200);
|
||||||
|
assert.equal(edited.body.displayName, "Renamed announcement");
|
||||||
|
assert.equal(edited.body.expirationAt, expiresAt);
|
||||||
|
const permanent = await jsonResponse(harness.baseUrl, `/api/assets/${assetId}`, {
|
||||||
|
method: "PATCH",
|
||||||
|
headers: { "content-type": "application/json" },
|
||||||
|
body: JSON.stringify({ expiration_at: null }),
|
||||||
|
});
|
||||||
|
assert.equal(permanent.body.expirationAt, null);
|
||||||
|
assert.equal(harness.storage.objects.size, 3);
|
||||||
|
await eventuallyEmpty(harness.tempDirectory);
|
||||||
|
|
||||||
|
const timestamp = Date.now();
|
||||||
|
harness.database
|
||||||
|
.prepare(
|
||||||
|
"INSERT INTO playlists (id, name, created_at, updated_at) VALUES (?, ?, ?, ?)",
|
||||||
|
)
|
||||||
|
.run("api-playlist", "API playlist", timestamp, timestamp);
|
||||||
|
harness.database
|
||||||
|
.prepare(
|
||||||
|
`INSERT INTO playlist_entries
|
||||||
|
(id, playlist_id, asset_id, position, created_at, updated_at)
|
||||||
|
VALUES (?, ?, ?, 0, ?, ?)`,
|
||||||
|
)
|
||||||
|
.run("api-entry", "api-playlist", assetId, timestamp, timestamp);
|
||||||
|
const references = await jsonResponse(
|
||||||
|
harness.baseUrl,
|
||||||
|
`/api/assets/${assetId}/references`,
|
||||||
|
);
|
||||||
|
assert.equal(references.response.status, 200);
|
||||||
|
assert.equal(references.body.canDelete, false);
|
||||||
|
assert.equal(references.body.playlists[0].playlistEntryId, "api-entry");
|
||||||
|
const blockedDelete = await jsonResponse(
|
||||||
|
harness.baseUrl,
|
||||||
|
`/api/assets/${assetId}`,
|
||||||
|
{ method: "DELETE" },
|
||||||
|
);
|
||||||
|
assert.equal(blockedDelete.response.status, 409);
|
||||||
|
assert.equal(blockedDelete.body.error.code, "ASSET_REFERENCED");
|
||||||
|
assert.equal(harness.storage.objects.size, 3);
|
||||||
|
harness.database
|
||||||
|
.prepare("DELETE FROM playlist_entries WHERE id = ?")
|
||||||
|
.run("api-entry");
|
||||||
|
|
||||||
|
const removed = await jsonResponse(harness.baseUrl, `/api/assets/${assetId}`, {
|
||||||
|
method: "DELETE",
|
||||||
|
});
|
||||||
|
assert.equal(removed.response.status, 200);
|
||||||
|
assert.deepEqual(removed.body, { id: assetId, deleted: true });
|
||||||
|
assert.equal(harness.storage.objects.size, 0);
|
||||||
|
assert.equal(
|
||||||
|
(await jsonResponse(harness.baseUrl, `/api/assets/${assetId}`)).response.status,
|
||||||
|
404,
|
||||||
|
);
|
||||||
|
});
|
||||||
|
|
||||||
|
test("upload validation returns structured errors and always cleans temporary files", async (t) => {
|
||||||
|
const harness = await createHarness(t, { maxFileBytes: 1024 });
|
||||||
|
|
||||||
|
const malformed = new FormData();
|
||||||
|
appendFile(malformed, "file", Buffer.from("not an image"), "broken.png", "image/png");
|
||||||
|
const malformedResult = await jsonResponse(harness.baseUrl, "/api/assets", {
|
||||||
|
method: "POST",
|
||||||
|
body: malformed,
|
||||||
|
});
|
||||||
|
assert.equal(malformedResult.response.status, 422);
|
||||||
|
assert.equal(malformedResult.body.error.code, "MALFORMED_IMAGE");
|
||||||
|
await eventuallyEmpty(harness.tempDirectory);
|
||||||
|
|
||||||
|
const unsupportedBuffer = await sharp({
|
||||||
|
create: {
|
||||||
|
width: 20,
|
||||||
|
height: 20,
|
||||||
|
channels: 3,
|
||||||
|
background: "red",
|
||||||
|
},
|
||||||
|
})
|
||||||
|
.gif()
|
||||||
|
.toBuffer();
|
||||||
|
const unsupported = new FormData();
|
||||||
|
appendFile(unsupported, "file", unsupportedBuffer, "still.gif", "image/png");
|
||||||
|
const unsupportedResult = await jsonResponse(harness.baseUrl, "/api/assets", {
|
||||||
|
method: "POST",
|
||||||
|
body: unsupported,
|
||||||
|
});
|
||||||
|
assert.equal(unsupportedResult.response.status, 415);
|
||||||
|
assert.equal(unsupportedResult.body.error.code, "UNSUPPORTED_IMAGE_FORMAT");
|
||||||
|
await eventuallyEmpty(harness.tempDirectory);
|
||||||
|
|
||||||
|
const oversized = new FormData();
|
||||||
|
appendFile(
|
||||||
|
oversized,
|
||||||
|
"file",
|
||||||
|
crypto.randomBytes(1025),
|
||||||
|
"too-large.png",
|
||||||
|
"image/png",
|
||||||
|
);
|
||||||
|
const oversizedResult = await jsonResponse(harness.baseUrl, "/api/assets", {
|
||||||
|
method: "POST",
|
||||||
|
body: oversized,
|
||||||
|
});
|
||||||
|
assert.equal(oversizedResult.response.status, 413);
|
||||||
|
assert.equal(oversizedResult.body.error.code, "UPLOAD_FILE_TOO_LARGE");
|
||||||
|
assert.deepEqual(oversizedResult.body.error.details, { maxFileBytes: 1024 });
|
||||||
|
await eventuallyEmpty(harness.tempDirectory);
|
||||||
|
|
||||||
|
const tooMany = new FormData();
|
||||||
|
for (let index = 0; index < 6; index += 1) {
|
||||||
|
appendFile(tooMany, "files", Buffer.from("x"), `${index}.png`, "image/png");
|
||||||
|
}
|
||||||
|
const tooManyResult = await jsonResponse(harness.baseUrl, "/api/assets", {
|
||||||
|
method: "POST",
|
||||||
|
body: tooMany,
|
||||||
|
});
|
||||||
|
assert.equal(tooManyResult.response.status, 413);
|
||||||
|
assert.equal(tooManyResult.body.error.code, "UPLOAD_FILE_COUNT_EXCEEDED");
|
||||||
|
assert.deepEqual(tooManyResult.body.error.details, { maxFiles: 5 });
|
||||||
|
await eventuallyEmpty(harness.tempDirectory);
|
||||||
|
|
||||||
|
const absent = await jsonResponse(harness.baseUrl, "/api/assets", {
|
||||||
|
method: "POST",
|
||||||
|
body: new FormData(),
|
||||||
|
});
|
||||||
|
assert.equal(absent.response.status, 400);
|
||||||
|
assert.equal(absent.body.error.code, "UPLOAD_FILE_REQUIRED");
|
||||||
|
assert.equal(harness.storage.objects.size, 0);
|
||||||
|
assert.equal(
|
||||||
|
harness.database.prepare("SELECT COUNT(*) AS count FROM assets").get().count,
|
||||||
|
0,
|
||||||
|
);
|
||||||
|
});
|
||||||
|
|
||||||
|
test("an aborted multipart request removes its in-flight temporary file", async (t) => {
|
||||||
|
const harness = await createHarness(t);
|
||||||
|
const boundary = `announcements-${crypto.randomUUID()}`;
|
||||||
|
const preamble = Buffer.from(
|
||||||
|
`--${boundary}\r\n` +
|
||||||
|
'Content-Disposition: form-data; name="file"; filename="aborted.png"\r\n' +
|
||||||
|
"Content-Type: image/png\r\n\r\n",
|
||||||
|
);
|
||||||
|
const declaredLength = preamble.length + 512 * 1024;
|
||||||
|
const request = http.request(`${harness.baseUrl}/api/assets`, {
|
||||||
|
method: "POST",
|
||||||
|
headers: {
|
||||||
|
"content-type": `multipart/form-data; boundary=${boundary}`,
|
||||||
|
"content-length": String(declaredLength),
|
||||||
|
},
|
||||||
|
});
|
||||||
|
request.on("error", () => {});
|
||||||
|
request.write(preamble);
|
||||||
|
request.write(Buffer.alloc(64 * 1024, 1));
|
||||||
|
|
||||||
|
await eventuallyContainsFile(harness.tempDirectory);
|
||||||
|
const closed = new Promise((resolve) => request.once("close", resolve));
|
||||||
|
request.destroy();
|
||||||
|
await closed;
|
||||||
|
await eventuallyEmpty(harness.tempDirectory);
|
||||||
|
assert.equal(harness.storage.objects.size, 0);
|
||||||
|
assert.equal(
|
||||||
|
harness.database.prepare("SELECT COUNT(*) AS count FROM assets").get().count,
|
||||||
|
0,
|
||||||
|
);
|
||||||
|
});
|
||||||
@@ -0,0 +1,35 @@
|
|||||||
|
const test = require("node:test");
|
||||||
|
const assert = require("node:assert/strict");
|
||||||
|
const {
|
||||||
|
mapMulterError,
|
||||||
|
safeOriginalFilename,
|
||||||
|
} = require("../src/assets/multipart");
|
||||||
|
|
||||||
|
test("original filename metadata keeps a safe leaf without path traversal", () => {
|
||||||
|
assert.equal(safeOriginalFilename("../../Sunday/welcome.png"), "welcome.png");
|
||||||
|
assert.equal(safeOriginalFilename("..\\..\\Sunday\\welcome.png"), "welcome.png");
|
||||||
|
assert.equal(safeOriginalFilename(" welcome sign.png "), " welcome sign.png ");
|
||||||
|
});
|
||||||
|
|
||||||
|
test("original filename metadata removes controls and remains bounded", () => {
|
||||||
|
assert.equal(safeOriginalFilename("wel\u0000come\n.png"), "welcome.png");
|
||||||
|
assert.equal([...safeOriginalFilename(`${"a".repeat(300)}.png`)].length, 255);
|
||||||
|
assert.throws(
|
||||||
|
() => safeOriginalFilename(" \u0000\n "),
|
||||||
|
(error) => error.code === "INVALID_ORIGINAL_FILENAME",
|
||||||
|
);
|
||||||
|
});
|
||||||
|
|
||||||
|
test("temporary-storage failures are server errors, not invalid client uploads", () => {
|
||||||
|
const storageFailure = mapMulterError(
|
||||||
|
Object.assign(new Error("disk full at a private path"), { code: "ENOSPC" }),
|
||||||
|
{},
|
||||||
|
);
|
||||||
|
assert.equal(storageFailure.code, "UPLOAD_TEMP_STORAGE_UNAVAILABLE");
|
||||||
|
assert.equal(storageFailure.statusCode, 503);
|
||||||
|
assert.doesNotMatch(storageFailure.message, /private path/);
|
||||||
|
|
||||||
|
const malformed = mapMulterError(new Error("Unexpected end of form"), {});
|
||||||
|
assert.equal(malformed.code, "INVALID_MULTIPART_UPLOAD");
|
||||||
|
assert.equal(malformed.statusCode, 400);
|
||||||
|
});
|
||||||
@@ -0,0 +1,608 @@
|
|||||||
|
const test = require("node:test");
|
||||||
|
const assert = require("node:assert/strict");
|
||||||
|
const crypto = require("node:crypto");
|
||||||
|
const fs = require("node:fs");
|
||||||
|
const os = require("node:os");
|
||||||
|
const path = require("node:path");
|
||||||
|
const { Readable } = require("node:stream");
|
||||||
|
const { openDatabase } = require("../src/database");
|
||||||
|
const { runMigrations } = require("../src/database/migrate");
|
||||||
|
const { createAssetService } = require("../src/assets/service");
|
||||||
|
const {
|
||||||
|
StorageObjectNotFoundError,
|
||||||
|
StorageUnavailableError,
|
||||||
|
} = require("../src/storage");
|
||||||
|
|
||||||
|
async function bufferBody(body) {
|
||||||
|
if (Buffer.isBuffer(body)) return body;
|
||||||
|
if (body instanceof Uint8Array) return Buffer.from(body);
|
||||||
|
const chunks = [];
|
||||||
|
for await (const chunk of body) chunks.push(Buffer.from(chunk));
|
||||||
|
return Buffer.concat(chunks);
|
||||||
|
}
|
||||||
|
|
||||||
|
function createMemoryStorage() {
|
||||||
|
const objects = new Map();
|
||||||
|
const calls = [];
|
||||||
|
let failNextPut = null;
|
||||||
|
let successfulPutsBeforeFailure = 0;
|
||||||
|
let failNextDelete = null;
|
||||||
|
let putDelayMs = 0;
|
||||||
|
|
||||||
|
return {
|
||||||
|
objects,
|
||||||
|
calls,
|
||||||
|
setPutFailure(error, putsBeforeFailure = 0) {
|
||||||
|
failNextPut = error;
|
||||||
|
successfulPutsBeforeFailure = putsBeforeFailure;
|
||||||
|
},
|
||||||
|
setDeleteFailure(error) {
|
||||||
|
failNextDelete = error;
|
||||||
|
},
|
||||||
|
setPutDelay(milliseconds) {
|
||||||
|
putDelayMs = milliseconds;
|
||||||
|
},
|
||||||
|
async putObject(input) {
|
||||||
|
calls.push({ operation: "put", ...input, body: undefined });
|
||||||
|
if (putDelayMs) {
|
||||||
|
await new Promise((resolve) => setTimeout(resolve, putDelayMs));
|
||||||
|
}
|
||||||
|
if (failNextPut) {
|
||||||
|
if (successfulPutsBeforeFailure === 0) {
|
||||||
|
const error = failNextPut;
|
||||||
|
failNextPut = null;
|
||||||
|
throw error;
|
||||||
|
}
|
||||||
|
successfulPutsBeforeFailure -= 1;
|
||||||
|
}
|
||||||
|
objects.set(input.key, {
|
||||||
|
body: await bufferBody(input.body),
|
||||||
|
contentType: input.contentType,
|
||||||
|
metadata: input.metadata,
|
||||||
|
});
|
||||||
|
return { key: input.key };
|
||||||
|
},
|
||||||
|
async getObject(key) {
|
||||||
|
calls.push({ operation: "get", key });
|
||||||
|
const object = objects.get(key);
|
||||||
|
if (!object) throw new StorageObjectNotFoundError(key);
|
||||||
|
return {
|
||||||
|
body: Readable.from(object.body),
|
||||||
|
contentLength: object.body.length,
|
||||||
|
contentType: object.contentType,
|
||||||
|
};
|
||||||
|
},
|
||||||
|
async deleteObject(key) {
|
||||||
|
calls.push({ operation: "delete", key });
|
||||||
|
if (failNextDelete) {
|
||||||
|
const error = failNextDelete;
|
||||||
|
failNextDelete = null;
|
||||||
|
throw error;
|
||||||
|
}
|
||||||
|
objects.delete(key);
|
||||||
|
return { key, deleted: true };
|
||||||
|
},
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
function createFakeImageProcessor({ reject } = {}) {
|
||||||
|
const calls = [];
|
||||||
|
return {
|
||||||
|
calls,
|
||||||
|
async process(filename) {
|
||||||
|
calls.push(filename);
|
||||||
|
if (reject) throw reject;
|
||||||
|
const original = await fs.promises.readFile(filename);
|
||||||
|
const sourceSha256 = crypto.createHash("sha256").update(original).digest("hex");
|
||||||
|
const displayBuffer = Buffer.from(`display:${sourceSha256}`);
|
||||||
|
const thumbnailBuffer = Buffer.from(`thumbnail:${sourceSha256}`);
|
||||||
|
return {
|
||||||
|
original: {
|
||||||
|
sha256: sourceSha256,
|
||||||
|
contentType: "image/png",
|
||||||
|
byteSize: original.length,
|
||||||
|
width: 1200,
|
||||||
|
height: 800,
|
||||||
|
},
|
||||||
|
display: {
|
||||||
|
buffer: displayBuffer,
|
||||||
|
sha256: crypto.createHash("sha256").update(displayBuffer).digest("hex"),
|
||||||
|
contentType: "image/webp",
|
||||||
|
byteSize: displayBuffer.length,
|
||||||
|
width: 1200,
|
||||||
|
height: 800,
|
||||||
|
},
|
||||||
|
thumbnail: {
|
||||||
|
buffer: thumbnailBuffer,
|
||||||
|
sha256: crypto.createHash("sha256").update(thumbnailBuffer).digest("hex"),
|
||||||
|
contentType: "image/webp",
|
||||||
|
byteSize: thumbnailBuffer.length,
|
||||||
|
width: 405,
|
||||||
|
height: 270,
|
||||||
|
},
|
||||||
|
};
|
||||||
|
},
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
function createHarness(options = {}) {
|
||||||
|
const database = openDatabase({ path: ":memory:" });
|
||||||
|
runMigrations(database);
|
||||||
|
const directory = fs.mkdtempSync(path.join(os.tmpdir(), "announcements-assets-"));
|
||||||
|
const storage = options.storage || createMemoryStorage();
|
||||||
|
const imageProcessor =
|
||||||
|
options.imageProcessor || createFakeImageProcessor(options.processorOptions);
|
||||||
|
let idCounter = 0;
|
||||||
|
let clock = 2_000_000_000_000;
|
||||||
|
const service = createAssetService({
|
||||||
|
database,
|
||||||
|
storage,
|
||||||
|
imageProcessor,
|
||||||
|
now: () => ++clock,
|
||||||
|
createId:
|
||||||
|
options.createId ||
|
||||||
|
(() => `00000000-0000-4000-8000-${String(++idCounter).padStart(12, "0")}`),
|
||||||
|
concurrentUploadWaitMs: options.concurrentUploadWaitMs ?? 2_000,
|
||||||
|
});
|
||||||
|
|
||||||
|
function uploadFile(originalFilename, contents) {
|
||||||
|
const buffer = Buffer.isBuffer(contents) ? contents : Buffer.from(contents);
|
||||||
|
const filename = path.join(directory, crypto.randomUUID());
|
||||||
|
fs.writeFileSync(filename, buffer);
|
||||||
|
return { path: filename, byteSize: buffer.length, originalFilename };
|
||||||
|
}
|
||||||
|
|
||||||
|
return {
|
||||||
|
database,
|
||||||
|
directory,
|
||||||
|
imageProcessor,
|
||||||
|
service,
|
||||||
|
storage,
|
||||||
|
uploadFile,
|
||||||
|
close() {
|
||||||
|
database.close();
|
||||||
|
fs.rmSync(directory, { recursive: true, force: true });
|
||||||
|
},
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
function rawAsset(database, assetId) {
|
||||||
|
return database.prepare("SELECT * FROM assets WHERE id = ?").get(assetId);
|
||||||
|
}
|
||||||
|
|
||||||
|
test("upload stores exact bytes under content-addressed keys without leaking keys", async (t) => {
|
||||||
|
const harness = createHarness();
|
||||||
|
t.after(() => harness.close());
|
||||||
|
const original = Buffer.from("exact uploaded bytes");
|
||||||
|
|
||||||
|
const [result] = await harness.service.upload([
|
||||||
|
harness.uploadFile(" Sunday Welcome.PNG ", original),
|
||||||
|
]);
|
||||||
|
|
||||||
|
assert.equal(result.created, true);
|
||||||
|
assert.equal(result.duplicate, false);
|
||||||
|
assert.match(result.asset.id, /^[0-9a-f-]{36}$/);
|
||||||
|
assert.equal(result.asset.displayName, "Sunday Welcome");
|
||||||
|
assert.equal(result.asset.originalFilename, " Sunday Welcome.PNG ");
|
||||||
|
assert.equal(result.asset.expirationAt, null);
|
||||||
|
assert.equal(result.asset.status, "ready");
|
||||||
|
assert.equal(result.asset.original.byteSize, original.length);
|
||||||
|
assert.equal(result.asset.display.contentType, "image/webp");
|
||||||
|
assert.equal(result.asset.thumbnail.width, 405);
|
||||||
|
assert.match(result.asset.display.url, /^\/media\/assets\//);
|
||||||
|
assert.doesNotMatch(
|
||||||
|
JSON.stringify(result.asset),
|
||||||
|
/object_key|originalObjectKey|displayObjectKey|thumbnailObjectKey|seaweed/i,
|
||||||
|
);
|
||||||
|
|
||||||
|
const row = rawAsset(harness.database, result.asset.id);
|
||||||
|
const expectedSourceHash = crypto.createHash("sha256").update(original).digest("hex");
|
||||||
|
assert.equal(row.source_sha256, expectedSourceHash);
|
||||||
|
assert.equal(row.original_object_key, `assets/${row.id}/original/${expectedSourceHash}`);
|
||||||
|
assert.equal(
|
||||||
|
row.display_object_key,
|
||||||
|
`assets/${row.id}/display/${row.display_sha256}.webp`,
|
||||||
|
);
|
||||||
|
assert.equal(
|
||||||
|
row.thumbnail_object_key,
|
||||||
|
`assets/${row.id}/thumbnail/${row.thumbnail_sha256}.webp`,
|
||||||
|
);
|
||||||
|
assert.deepEqual(harness.storage.objects.get(row.original_object_key).body, original);
|
||||||
|
assert.equal(harness.storage.objects.size, 3);
|
||||||
|
|
||||||
|
assert.deepEqual(harness.service.list(), [result.asset]);
|
||||||
|
assert.deepEqual(harness.service.get(result.asset.id), result.asset);
|
||||||
|
});
|
||||||
|
|
||||||
|
test("exact duplicate detection takes precedence over filename and display name", async (t) => {
|
||||||
|
const harness = createHarness();
|
||||||
|
t.after(() => harness.close());
|
||||||
|
const contents = Buffer.from("same image bytes");
|
||||||
|
|
||||||
|
const [created] = await harness.service.upload([
|
||||||
|
harness.uploadFile("first.png", contents),
|
||||||
|
]);
|
||||||
|
const [sameFilename] = await harness.service.upload([
|
||||||
|
harness.uploadFile("first.png", contents),
|
||||||
|
]);
|
||||||
|
const [differentFilename] = await harness.service.upload(
|
||||||
|
[harness.uploadFile("completely different name.webp", contents)],
|
||||||
|
{ displayName: "A conflicting requested name is ignored for an exact duplicate" },
|
||||||
|
);
|
||||||
|
|
||||||
|
for (const duplicate of [sameFilename, differentFilename]) {
|
||||||
|
assert.equal(duplicate.created, false);
|
||||||
|
assert.equal(duplicate.duplicate, true);
|
||||||
|
assert.equal(duplicate.asset.id, created.asset.id);
|
||||||
|
}
|
||||||
|
assert.equal(harness.service.list().length, 1);
|
||||||
|
assert.equal(harness.storage.calls.filter((call) => call.operation === "put").length, 3);
|
||||||
|
assert.equal(harness.imageProcessor.calls.length, 1);
|
||||||
|
});
|
||||||
|
|
||||||
|
test("different content with the same proposed name is a conflict, not a duplicate", async (t) => {
|
||||||
|
const harness = createHarness();
|
||||||
|
t.after(() => harness.close());
|
||||||
|
await harness.service.upload([harness.uploadFile("bulletin.png", "image one")]);
|
||||||
|
|
||||||
|
await assert.rejects(
|
||||||
|
harness.service.upload([harness.uploadFile("bulletin.jpg", "image two")]),
|
||||||
|
(error) => {
|
||||||
|
assert.equal(error.statusCode, 409);
|
||||||
|
assert.equal(error.code, "ASSET_NAME_CONFLICT");
|
||||||
|
assert.equal(error.details.asset.displayName, "bulletin");
|
||||||
|
return true;
|
||||||
|
},
|
||||||
|
);
|
||||||
|
assert.equal(harness.service.list().length, 1);
|
||||||
|
assert.equal(harness.imageProcessor.calls.length, 1);
|
||||||
|
|
||||||
|
const [created] = await harness.service.upload(
|
||||||
|
[harness.uploadFile("bulletin.jpg", "image two")],
|
||||||
|
{ displayName: "Second bulletin" },
|
||||||
|
);
|
||||||
|
assert.equal(created.created, true);
|
||||||
|
assert.notEqual(created.asset.sourceSha256, harness.service.list()[1].sourceSha256);
|
||||||
|
assert.equal(harness.service.list().length, 2);
|
||||||
|
});
|
||||||
|
|
||||||
|
test("multiple-file upload preserves order and deduplicates repeated bytes in the batch", async (t) => {
|
||||||
|
const harness = createHarness();
|
||||||
|
t.after(() => harness.close());
|
||||||
|
|
||||||
|
const results = await harness.service.upload([
|
||||||
|
harness.uploadFile("alpha.png", "alpha bytes"),
|
||||||
|
harness.uploadFile("beta.png", "beta bytes"),
|
||||||
|
harness.uploadFile("alpha-copy.png", "alpha bytes"),
|
||||||
|
]);
|
||||||
|
|
||||||
|
assert.deepEqual(
|
||||||
|
results.map(({ created, duplicate }) => ({ created, duplicate })),
|
||||||
|
[
|
||||||
|
{ created: true, duplicate: false },
|
||||||
|
{ created: true, duplicate: false },
|
||||||
|
{ created: false, duplicate: true },
|
||||||
|
],
|
||||||
|
);
|
||||||
|
assert.equal(results[0].asset.id, results[2].asset.id);
|
||||||
|
assert.equal(harness.service.list().length, 2);
|
||||||
|
assert.equal(harness.imageProcessor.calls.length, 2);
|
||||||
|
assert.equal(harness.storage.objects.size, 6);
|
||||||
|
});
|
||||||
|
|
||||||
|
test("all new images are decoded before the batch mutates SQLite or storage", async (t) => {
|
||||||
|
const expected = Object.assign(new Error("bad second image"), {
|
||||||
|
code: "MALFORMED_IMAGE",
|
||||||
|
statusCode: 422,
|
||||||
|
});
|
||||||
|
let processCount = 0;
|
||||||
|
const regularProcessor = createFakeImageProcessor();
|
||||||
|
const harness = createHarness({
|
||||||
|
imageProcessor: {
|
||||||
|
async process(filename) {
|
||||||
|
processCount += 1;
|
||||||
|
if (processCount === 2) throw expected;
|
||||||
|
return regularProcessor.process(filename);
|
||||||
|
},
|
||||||
|
},
|
||||||
|
});
|
||||||
|
t.after(() => harness.close());
|
||||||
|
|
||||||
|
await assert.rejects(
|
||||||
|
harness.service.upload([
|
||||||
|
harness.uploadFile("valid.png", "valid"),
|
||||||
|
harness.uploadFile("broken.png", "broken"),
|
||||||
|
]),
|
||||||
|
expected,
|
||||||
|
);
|
||||||
|
assert.equal(harness.service.list().length, 0);
|
||||||
|
assert.equal(harness.storage.objects.size, 0);
|
||||||
|
assert.equal(harness.storage.calls.length, 0);
|
||||||
|
});
|
||||||
|
|
||||||
|
test("concurrent identical uploads converge on one ready asset", async (t) => {
|
||||||
|
const storage = createMemoryStorage();
|
||||||
|
storage.setPutDelay(30);
|
||||||
|
const harness = createHarness({ storage });
|
||||||
|
t.after(() => harness.close());
|
||||||
|
const bytes = Buffer.from("racing identical upload");
|
||||||
|
|
||||||
|
const [[left], [right]] = await Promise.all([
|
||||||
|
harness.service.upload([harness.uploadFile("race.png", bytes)]),
|
||||||
|
harness.service.upload([harness.uploadFile("other-name.png", bytes)]),
|
||||||
|
]);
|
||||||
|
|
||||||
|
assert.equal(harness.service.list().length, 1);
|
||||||
|
assert.equal(left.asset.id, right.asset.id);
|
||||||
|
assert.deepEqual([left.created, right.created].sort(), [false, true]);
|
||||||
|
assert.deepEqual([left.duplicate, right.duplicate].sort(), [false, true]);
|
||||||
|
assert.equal(storage.calls.filter((call) => call.operation === "put").length, 3);
|
||||||
|
});
|
||||||
|
|
||||||
|
test("a partial storage failure leaves a diagnosable failed row and no ready URLs", async (t) => {
|
||||||
|
const storage = createMemoryStorage();
|
||||||
|
storage.setPutFailure(new StorageUnavailableError(new Error("offline")), 1);
|
||||||
|
const harness = createHarness({ storage });
|
||||||
|
t.after(() => harness.close());
|
||||||
|
|
||||||
|
await assert.rejects(
|
||||||
|
harness.service.upload([harness.uploadFile("failure.png", "failure bytes")]),
|
||||||
|
(error) => {
|
||||||
|
assert.equal(error.statusCode, 503);
|
||||||
|
assert.equal(error.details.retryable, true);
|
||||||
|
assert.match(error.details.assetId, /^[0-9a-f-]{36}$/);
|
||||||
|
return true;
|
||||||
|
},
|
||||||
|
);
|
||||||
|
|
||||||
|
const [failed] = harness.service.list();
|
||||||
|
assert.equal(failed.status, "failed");
|
||||||
|
assert.match(failed.processingError, /^STORAGE_UNAVAILABLE:/);
|
||||||
|
assert.equal(failed.display.url, null);
|
||||||
|
assert.equal(failed.thumbnail.url, null);
|
||||||
|
assert.equal(failed.original.url, null);
|
||||||
|
assert.equal(storage.objects.size, 0);
|
||||||
|
const row = rawAsset(harness.database, failed.id);
|
||||||
|
assert.match(
|
||||||
|
row.original_object_key,
|
||||||
|
new RegExp(`^assets/${failed.id}/original/[0-9a-f]{64}$`),
|
||||||
|
);
|
||||||
|
assert.equal(
|
||||||
|
row.display_object_key,
|
||||||
|
`assets/${failed.id}/display/${row.display_sha256}.webp`,
|
||||||
|
);
|
||||||
|
assert.equal(
|
||||||
|
row.thumbnail_object_key,
|
||||||
|
`assets/${failed.id}/thumbnail/${row.thumbnail_sha256}.webp`,
|
||||||
|
);
|
||||||
|
assert.equal(storage.calls.filter((call) => call.operation === "put").length, 2);
|
||||||
|
assert.deepEqual(
|
||||||
|
new Set(
|
||||||
|
storage.calls
|
||||||
|
.filter((call) => call.operation === "delete")
|
||||||
|
.map((call) => call.key),
|
||||||
|
),
|
||||||
|
new Set([
|
||||||
|
row.original_object_key,
|
||||||
|
row.display_object_key,
|
||||||
|
row.thumbnail_object_key,
|
||||||
|
]),
|
||||||
|
);
|
||||||
|
|
||||||
|
await assert.rejects(
|
||||||
|
harness.service.upload([harness.uploadFile("retry.png", "failure bytes")]),
|
||||||
|
(error) => error.code === "ASSET_PROCESSING_FAILED" && error.statusCode === 409,
|
||||||
|
);
|
||||||
|
|
||||||
|
assert.deepEqual(await harness.service.remove(failed.id), {
|
||||||
|
id: failed.id,
|
||||||
|
deleted: true,
|
||||||
|
});
|
||||||
|
assert.equal(harness.service.list().length, 0);
|
||||||
|
});
|
||||||
|
|
||||||
|
test("rename and expiration edits change only approved SQLite metadata", async (t) => {
|
||||||
|
const harness = createHarness();
|
||||||
|
t.after(() => harness.close());
|
||||||
|
const [upload] = await harness.service.upload([
|
||||||
|
harness.uploadFile("editable.png", "editable bytes"),
|
||||||
|
]);
|
||||||
|
const before = rawAsset(harness.database, upload.asset.id);
|
||||||
|
|
||||||
|
const expiring = harness.service.update(upload.asset.id, {
|
||||||
|
display_name: " New display name ",
|
||||||
|
expiration_at: "2035-05-06T07:08:09.000Z",
|
||||||
|
});
|
||||||
|
assert.equal(expiring.displayName, "New display name");
|
||||||
|
assert.equal(expiring.expirationAt, Date.parse("2035-05-06T07:08:09.000Z"));
|
||||||
|
|
||||||
|
const permanent = harness.service.update(upload.asset.id, { expirationAt: null });
|
||||||
|
assert.equal(permanent.expirationAt, null);
|
||||||
|
const after = rawAsset(harness.database, upload.asset.id);
|
||||||
|
assert.equal(after.original_object_key, before.original_object_key);
|
||||||
|
assert.equal(after.display_object_key, before.display_object_key);
|
||||||
|
assert.equal(after.thumbnail_object_key, before.thumbnail_object_key);
|
||||||
|
assert.equal(harness.storage.calls.filter((call) => call.operation === "put").length, 3);
|
||||||
|
|
||||||
|
await assert.rejects(
|
||||||
|
async () => harness.service.update(upload.asset.id, { status: "failed" }),
|
||||||
|
(error) => error.code === "INVALID_ASSET_UPDATE",
|
||||||
|
);
|
||||||
|
assert.throws(
|
||||||
|
() => harness.service.update(upload.asset.id, { expirationAt: -1 }),
|
||||||
|
(error) => error.code === "INVALID_ASSET_EXPIRATION",
|
||||||
|
);
|
||||||
|
});
|
||||||
|
|
||||||
|
test("references describe playlist, normal, and static use and block deletion", async (t) => {
|
||||||
|
const harness = createHarness();
|
||||||
|
t.after(() => harness.close());
|
||||||
|
const [upload] = await harness.service.upload([
|
||||||
|
harness.uploadFile("referenced.png", "referenced bytes"),
|
||||||
|
]);
|
||||||
|
const id = upload.asset.id;
|
||||||
|
const now = 2_100_000_000_000;
|
||||||
|
harness.database
|
||||||
|
.prepare(
|
||||||
|
"INSERT INTO playlists (id, name, created_at, updated_at) VALUES (?, ?, ?, ?)",
|
||||||
|
)
|
||||||
|
.run("playlist-1", "Sunday", now, now);
|
||||||
|
harness.database
|
||||||
|
.prepare(
|
||||||
|
`INSERT INTO playlist_entries
|
||||||
|
(id, playlist_id, asset_id, enabled, position, created_at, updated_at)
|
||||||
|
VALUES (?, ?, ?, 1, 0, ?, ?)`,
|
||||||
|
)
|
||||||
|
.run("entry-1", "playlist-1", id, now, now);
|
||||||
|
harness.database
|
||||||
|
.prepare(
|
||||||
|
`UPDATE playback_state
|
||||||
|
SET active_playlist_id = ?, current_entry_id = ?, static_asset_id = ?, updated_at = ?
|
||||||
|
WHERE output_id = 'global'`,
|
||||||
|
)
|
||||||
|
.run("playlist-1", "entry-1", id, now);
|
||||||
|
|
||||||
|
const references = harness.service.references(id);
|
||||||
|
assert.equal(references.canDelete, false);
|
||||||
|
assert.deepEqual(references.blockerCodes, [
|
||||||
|
"PLAYLIST_REFERENCES",
|
||||||
|
"CURRENT_PLAYBACK_TARGET",
|
||||||
|
"CURRENT_STATIC_TARGET",
|
||||||
|
]);
|
||||||
|
assert.deepEqual(references.playlists[0], {
|
||||||
|
playlistId: "playlist-1",
|
||||||
|
playlistName: "Sunday",
|
||||||
|
playlistEntryId: "entry-1",
|
||||||
|
enabled: true,
|
||||||
|
position: 0,
|
||||||
|
isActivePlaylist: true,
|
||||||
|
isCurrentNormalTarget: true,
|
||||||
|
});
|
||||||
|
assert.equal(references.playback[0].isCurrentNormalTarget, true);
|
||||||
|
assert.equal(references.playback[0].isCurrentStaticTarget, true);
|
||||||
|
|
||||||
|
const callsBeforeDelete = harness.storage.calls.length;
|
||||||
|
await assert.rejects(
|
||||||
|
harness.service.remove(id),
|
||||||
|
(error) => {
|
||||||
|
assert.equal(error.code, "ASSET_REFERENCED");
|
||||||
|
assert.equal(error.statusCode, 409);
|
||||||
|
assert.deepEqual(error.details.blockerCodes, [
|
||||||
|
"PLAYLIST_REFERENCES",
|
||||||
|
"CURRENT_PLAYBACK_TARGET",
|
||||||
|
"CURRENT_STATIC_TARGET",
|
||||||
|
]);
|
||||||
|
return true;
|
||||||
|
},
|
||||||
|
);
|
||||||
|
assert.equal(harness.storage.calls.length, callsBeforeDelete);
|
||||||
|
assert.equal(rawAsset(harness.database, id).status, "ready");
|
||||||
|
|
||||||
|
harness.database
|
||||||
|
.prepare(
|
||||||
|
`UPDATE playback_state
|
||||||
|
SET active_playlist_id = NULL, current_entry_id = NULL, static_asset_id = NULL
|
||||||
|
WHERE output_id = 'global'`,
|
||||||
|
)
|
||||||
|
.run();
|
||||||
|
harness.database.prepare("DELETE FROM playlist_entries WHERE id = ?").run("entry-1");
|
||||||
|
assert.equal(harness.service.references(id).canDelete, true);
|
||||||
|
await harness.service.remove(id);
|
||||||
|
assert.equal(harness.storage.objects.size, 0);
|
||||||
|
assert.equal(rawAsset(harness.database, id), undefined);
|
||||||
|
});
|
||||||
|
|
||||||
|
test("failed storage deletion retains a retryable deleting row", async (t) => {
|
||||||
|
const storage = createMemoryStorage();
|
||||||
|
const harness = createHarness({ storage });
|
||||||
|
t.after(() => harness.close());
|
||||||
|
const [upload] = await harness.service.upload([
|
||||||
|
harness.uploadFile("delete-me.png", "delete me"),
|
||||||
|
]);
|
||||||
|
storage.setDeleteFailure(new StorageUnavailableError(new Error("offline")));
|
||||||
|
|
||||||
|
await assert.rejects(harness.service.remove(upload.asset.id), (error) => {
|
||||||
|
assert.equal(error.statusCode, 503);
|
||||||
|
assert.equal(error.details.assetId, upload.asset.id);
|
||||||
|
assert.equal(error.details.retryable, true);
|
||||||
|
return true;
|
||||||
|
});
|
||||||
|
assert.equal(rawAsset(harness.database, upload.asset.id).status, "deleting");
|
||||||
|
assert.throws(
|
||||||
|
() => harness.service.update(upload.asset.id, { displayName: "Cannot edit" }),
|
||||||
|
(error) => error.code === "ASSET_DELETING",
|
||||||
|
);
|
||||||
|
|
||||||
|
await harness.service.remove(upload.asset.id);
|
||||||
|
assert.equal(rawAsset(harness.database, upload.asset.id), undefined);
|
||||||
|
assert.equal(storage.objects.size, 0);
|
||||||
|
});
|
||||||
|
|
||||||
|
test("processing assets cannot be deleted and storage is not touched", async (t) => {
|
||||||
|
const harness = createHarness();
|
||||||
|
t.after(() => harness.close());
|
||||||
|
const timestamp = Date.now();
|
||||||
|
harness.database
|
||||||
|
.prepare(
|
||||||
|
`INSERT INTO assets
|
||||||
|
(id, source_sha256, display_name, original_filename, status, created_at, updated_at)
|
||||||
|
VALUES (?, ?, ?, ?, 'processing', ?, ?)`,
|
||||||
|
)
|
||||||
|
.run(
|
||||||
|
"00000000-0000-4000-8000-999999999999",
|
||||||
|
"f".repeat(64),
|
||||||
|
"Still processing",
|
||||||
|
"processing.png",
|
||||||
|
timestamp,
|
||||||
|
timestamp,
|
||||||
|
);
|
||||||
|
|
||||||
|
assert.equal(
|
||||||
|
harness.service.references("00000000-0000-4000-8000-999999999999")
|
||||||
|
.canDelete,
|
||||||
|
false,
|
||||||
|
);
|
||||||
|
await assert.rejects(
|
||||||
|
harness.service.remove("00000000-0000-4000-8000-999999999999"),
|
||||||
|
(error) => error.code === "ASSET_PROCESSING" && error.statusCode === 409,
|
||||||
|
);
|
||||||
|
assert.equal(harness.storage.calls.length, 0);
|
||||||
|
assert.equal(
|
||||||
|
rawAsset(harness.database, "00000000-0000-4000-8000-999999999999").status,
|
||||||
|
"processing",
|
||||||
|
);
|
||||||
|
});
|
||||||
|
|
||||||
|
test("media descriptors only allow the recorded version and map missing bytes to 503", async (t) => {
|
||||||
|
const harness = createHarness();
|
||||||
|
t.after(() => harness.close());
|
||||||
|
const [upload] = await harness.service.upload([
|
||||||
|
harness.uploadFile("media.png", "media bytes"),
|
||||||
|
]);
|
||||||
|
const { asset } = upload;
|
||||||
|
|
||||||
|
const display = harness.service.mediaDescriptor(
|
||||||
|
asset.id,
|
||||||
|
"display",
|
||||||
|
asset.display.sha256,
|
||||||
|
);
|
||||||
|
assert.equal(display.contentType, "image/webp");
|
||||||
|
assert.equal(display.contentLength, asset.display.byteSize);
|
||||||
|
const stored = await harness.service.readStoredObject(display);
|
||||||
|
assert.deepEqual(await bufferBody(stored.body), harness.storage.objects.get(display.key).body);
|
||||||
|
|
||||||
|
for (const requestedHash of ["a".repeat(64), "../original/key", "not-a-hash"]) {
|
||||||
|
assert.throws(
|
||||||
|
() => harness.service.mediaDescriptor(asset.id, "display", requestedHash),
|
||||||
|
(error) => error.code === "MEDIA_NOT_FOUND" && error.statusCode === 404,
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
harness.storage.objects.delete(display.key);
|
||||||
|
await assert.rejects(harness.service.readStoredObject(display), (error) => {
|
||||||
|
assert.equal(error.code, "STORAGE_INCONSISTENCY");
|
||||||
|
assert.equal(error.statusCode, 503);
|
||||||
|
assert.deepEqual(error.details, { assetId: asset.id, retryable: true });
|
||||||
|
return true;
|
||||||
|
});
|
||||||
|
});
|
||||||
@@ -32,6 +32,14 @@ test("loadConfig validates and normalizes the supported environment", () => {
|
|||||||
assert.equal(config.timeZone, "America/Chicago");
|
assert.equal(config.timeZone, "America/Chicago");
|
||||||
assert.equal(config.storage.endpoint, "http://seaweedfs:8333");
|
assert.equal(config.storage.endpoint, "http://seaweedfs:8333");
|
||||||
assert.equal(config.storage.forcePathStyle, true);
|
assert.equal(config.storage.forcePathStyle, true);
|
||||||
|
assert.equal(config.storage.healthProbeIntervalMs, 30_000);
|
||||||
|
assert.equal(config.storage.connectionTimeoutMs, 2_000);
|
||||||
|
assert.equal(config.storage.requestTimeoutMs, 5_000);
|
||||||
|
assert.equal(config.storage.maxAttempts, 2);
|
||||||
|
assert.equal(config.processing.tempDirectory, "/tmp/announcements-uploads");
|
||||||
|
assert.equal(config.processing.maxFileBytes, 25 * 1024 * 1024);
|
||||||
|
assert.equal(config.processing.maxFiles, 20);
|
||||||
|
assert.equal(config.processing.maxInputPixels, 40_000_000);
|
||||||
});
|
});
|
||||||
|
|
||||||
test("production requires private S3 credentials", () => {
|
test("production requires private S3 credentials", () => {
|
||||||
@@ -65,10 +73,42 @@ test("loadConfig rejects invalid typed values", () => {
|
|||||||
() => loadConfig(validEnvironment({ S3_FORCE_PATH_STYLE: "sometimes" })),
|
() => loadConfig(validEnvironment({ S3_FORCE_PATH_STYLE: "sometimes" })),
|
||||||
/must be a boolean/,
|
/must be a boolean/,
|
||||||
);
|
);
|
||||||
|
assert.throws(
|
||||||
|
() => loadConfig(validEnvironment({ S3_HEALTH_PROBE_INTERVAL_MS: "999" })),
|
||||||
|
/S3_HEALTH_PROBE_INTERVAL_MS must be at least 1000/,
|
||||||
|
);
|
||||||
|
assert.throws(
|
||||||
|
() => loadConfig(validEnvironment({ S3_REQUEST_TIMEOUT_MS: "999" })),
|
||||||
|
/S3_REQUEST_TIMEOUT_MS must be at least 1000/,
|
||||||
|
);
|
||||||
|
assert.throws(
|
||||||
|
() => loadConfig(validEnvironment({ S3_CONNECT_TIMEOUT_MS: "249" })),
|
||||||
|
/S3_CONNECT_TIMEOUT_MS must be at least 250/,
|
||||||
|
);
|
||||||
|
assert.throws(
|
||||||
|
() => loadConfig(validEnvironment({ S3_MAX_ATTEMPTS: "6" })),
|
||||||
|
/S3_MAX_ATTEMPTS must be at most 5/,
|
||||||
|
);
|
||||||
assert.throws(
|
assert.throws(
|
||||||
() => loadConfig(validEnvironment({ DATABASE_PATH: "relative.sqlite" })),
|
() => loadConfig(validEnvironment({ DATABASE_PATH: "relative.sqlite" })),
|
||||||
/must be an absolute path/,
|
/must be an absolute path/,
|
||||||
);
|
);
|
||||||
|
assert.throws(
|
||||||
|
() => loadConfig(validEnvironment({ UPLOAD_TEMP_DIRECTORY: "relative" })),
|
||||||
|
/UPLOAD_TEMP_DIRECTORY must be an absolute path/,
|
||||||
|
);
|
||||||
|
assert.throws(
|
||||||
|
() => loadConfig(validEnvironment({ UPLOAD_MAX_FILE_BYTES: "0" })),
|
||||||
|
/UPLOAD_MAX_FILE_BYTES must be at least 1/,
|
||||||
|
);
|
||||||
|
assert.throws(
|
||||||
|
() => loadConfig(validEnvironment({ UPLOAD_MAX_FILES: "101" })),
|
||||||
|
/UPLOAD_MAX_FILES must be at most 100/,
|
||||||
|
);
|
||||||
|
assert.throws(
|
||||||
|
() => loadConfig(validEnvironment({ IMAGE_MAX_INPUT_PIXELS: "no-limit" })),
|
||||||
|
/IMAGE_MAX_INPUT_PIXELS must be an integer/,
|
||||||
|
);
|
||||||
});
|
});
|
||||||
|
|
||||||
test("redactedConfig never returns credentials, passwords, paths, or webhook URLs", () => {
|
test("redactedConfig never returns credentials, passwords, paths, or webhook URLs", () => {
|
||||||
@@ -83,5 +123,6 @@ test("redactedConfig never returns credentials, passwords, paths, or webhook URL
|
|||||||
assert.doesNotMatch(serialized, /mqtt-password/);
|
assert.doesNotMatch(serialized, /mqtt-password/);
|
||||||
assert.doesNotMatch(serialized, /webhook-secret/);
|
assert.doesNotMatch(serialized, /webhook-secret/);
|
||||||
assert.doesNotMatch(serialized, /announcements\.sqlite/);
|
assert.doesNotMatch(serialized, /announcements\.sqlite/);
|
||||||
|
assert.doesNotMatch(serialized, /seaweedfs/);
|
||||||
assert.doesNotMatch(serialized, /home-assistant/);
|
assert.doesNotMatch(serialized, /home-assistant/);
|
||||||
});
|
});
|
||||||
|
|||||||
@@ -0,0 +1,304 @@
|
|||||||
|
const test = require("node:test");
|
||||||
|
const assert = require("node:assert/strict");
|
||||||
|
const crypto = require("node:crypto");
|
||||||
|
const fs = require("node:fs");
|
||||||
|
const os = require("node:os");
|
||||||
|
const path = require("node:path");
|
||||||
|
const sharp = require("sharp");
|
||||||
|
const {
|
||||||
|
DEFAULT_IMAGE_DERIVATIVE_OPTIONS,
|
||||||
|
ImageProcessingError,
|
||||||
|
createImageProcessor,
|
||||||
|
} = require("../src/image-processing");
|
||||||
|
|
||||||
|
const MAX_INPUT_PIXELS = 12_000_000;
|
||||||
|
const CONTENT_TYPES = Object.freeze({
|
||||||
|
jpeg: "image/jpeg",
|
||||||
|
png: "image/png",
|
||||||
|
webp: "image/webp",
|
||||||
|
});
|
||||||
|
|
||||||
|
function createProcessor(overrides = {}) {
|
||||||
|
return createImageProcessor({
|
||||||
|
maxInputPixels: MAX_INPUT_PIXELS,
|
||||||
|
...overrides,
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
function hash(buffer) {
|
||||||
|
return crypto.createHash("sha256").update(buffer).digest("hex");
|
||||||
|
}
|
||||||
|
|
||||||
|
async function solidImage(format, width, height, background = "#4a76b8") {
|
||||||
|
return sharp({
|
||||||
|
create: {
|
||||||
|
width,
|
||||||
|
height,
|
||||||
|
channels: 4,
|
||||||
|
background,
|
||||||
|
},
|
||||||
|
})
|
||||||
|
.toFormat(format)
|
||||||
|
.toBuffer();
|
||||||
|
}
|
||||||
|
|
||||||
|
async function twoFrameImage(format) {
|
||||||
|
const width = 20;
|
||||||
|
const frameHeight = 10;
|
||||||
|
const channels = 4;
|
||||||
|
const firstFrame = Buffer.alloc(width * frameHeight * channels);
|
||||||
|
const secondFrame = Buffer.alloc(width * frameHeight * channels);
|
||||||
|
|
||||||
|
for (let offset = 0; offset < firstFrame.length; offset += channels) {
|
||||||
|
firstFrame.set([255, 0, 0, 255], offset);
|
||||||
|
secondFrame.set([0, 0, 255, 255], offset);
|
||||||
|
}
|
||||||
|
|
||||||
|
const image = sharp(Buffer.concat([firstFrame, secondFrame]), {
|
||||||
|
raw: {
|
||||||
|
width,
|
||||||
|
height: frameHeight * 2,
|
||||||
|
channels,
|
||||||
|
pageHeight: frameHeight,
|
||||||
|
},
|
||||||
|
});
|
||||||
|
|
||||||
|
if (format === "tiff") {
|
||||||
|
return image.tiff().toBuffer();
|
||||||
|
}
|
||||||
|
return image[format]({ delay: [100, 100], loop: 0 }).toBuffer();
|
||||||
|
}
|
||||||
|
|
||||||
|
test("accepts decoded still JPEG, PNG, and WebP content", async (t) => {
|
||||||
|
const processor = createProcessor();
|
||||||
|
|
||||||
|
for (const format of ["jpeg", "png", "webp"]) {
|
||||||
|
await t.test(format, async () => {
|
||||||
|
const source = await solidImage(format, 640, 360);
|
||||||
|
const result = await processor.process(source);
|
||||||
|
|
||||||
|
assert.equal(result.original.format, format);
|
||||||
|
assert.equal(result.original.contentType, CONTENT_TYPES[format]);
|
||||||
|
assert.equal(result.original.byteSize, source.length);
|
||||||
|
assert.equal(result.original.sha256, hash(source));
|
||||||
|
assert.equal(result.original.width, 640);
|
||||||
|
assert.equal(result.original.height, 360);
|
||||||
|
assert.equal(result.original.pages, 1);
|
||||||
|
|
||||||
|
for (const derivative of [result.display, result.thumbnail]) {
|
||||||
|
assert.ok(Buffer.isBuffer(derivative.buffer));
|
||||||
|
assert.equal(derivative.format, "webp");
|
||||||
|
assert.equal(derivative.contentType, "image/webp");
|
||||||
|
assert.equal(derivative.byteSize, derivative.buffer.length);
|
||||||
|
assert.equal(derivative.sha256, hash(derivative.buffer));
|
||||||
|
assert.match(derivative.sha256, /^[0-9a-f]{64}$/);
|
||||||
|
}
|
||||||
|
});
|
||||||
|
}
|
||||||
|
});
|
||||||
|
|
||||||
|
test("uses bounding boxes without cropping or changing source aspect ratio", async () => {
|
||||||
|
const source = await solidImage("png", 3000, 1000);
|
||||||
|
const result = await createProcessor().process(source);
|
||||||
|
|
||||||
|
assert.deepEqual(
|
||||||
|
{ width: result.display.width, height: result.display.height },
|
||||||
|
{ width: 1920, height: 640 },
|
||||||
|
);
|
||||||
|
assert.deepEqual(
|
||||||
|
{ width: result.thumbnail.width, height: result.thumbnail.height },
|
||||||
|
{ width: 480, height: 160 },
|
||||||
|
);
|
||||||
|
assert.equal(result.display.width / result.display.height, 3);
|
||||||
|
assert.equal(result.thumbnail.width / result.thumbnail.height, 3);
|
||||||
|
});
|
||||||
|
|
||||||
|
test("does not enlarge a source that fits within both derivative bounds", async () => {
|
||||||
|
const source = await solidImage("jpeg", 320, 180);
|
||||||
|
const result = await createProcessor().process(source);
|
||||||
|
|
||||||
|
assert.deepEqual(
|
||||||
|
{ width: result.display.width, height: result.display.height },
|
||||||
|
{ width: 320, height: 180 },
|
||||||
|
);
|
||||||
|
assert.deepEqual(
|
||||||
|
{ width: result.thumbnail.width, height: result.thumbnail.height },
|
||||||
|
{ width: 320, height: 180 },
|
||||||
|
);
|
||||||
|
});
|
||||||
|
|
||||||
|
test("applies EXIF orientation and strips it from derivative metadata", async () => {
|
||||||
|
const source = await sharp({
|
||||||
|
create: {
|
||||||
|
width: 120,
|
||||||
|
height: 80,
|
||||||
|
channels: 3,
|
||||||
|
background: "#cb7235",
|
||||||
|
},
|
||||||
|
})
|
||||||
|
.jpeg()
|
||||||
|
.withMetadata({ orientation: 6 })
|
||||||
|
.toBuffer();
|
||||||
|
|
||||||
|
const result = await createProcessor().process(source);
|
||||||
|
const outputMetadata = await sharp(result.display.buffer).metadata();
|
||||||
|
|
||||||
|
assert.equal(result.original.orientation, 6);
|
||||||
|
assert.equal(result.original.encodedWidth, 120);
|
||||||
|
assert.equal(result.original.encodedHeight, 80);
|
||||||
|
assert.equal(result.original.width, 80);
|
||||||
|
assert.equal(result.original.height, 120);
|
||||||
|
assert.equal(result.display.width, 80);
|
||||||
|
assert.equal(result.display.height, 120);
|
||||||
|
assert.equal(outputMetadata.orientation, undefined);
|
||||||
|
assert.equal(outputMetadata.exif, undefined);
|
||||||
|
});
|
||||||
|
|
||||||
|
test("flattens transparent pixels against black in both derivatives", async () => {
|
||||||
|
const source = await solidImage("png", 40, 30, {
|
||||||
|
r: 255,
|
||||||
|
g: 0,
|
||||||
|
b: 0,
|
||||||
|
alpha: 0,
|
||||||
|
});
|
||||||
|
const result = await createProcessor().process(source);
|
||||||
|
|
||||||
|
assert.equal(result.original.hasAlpha, true);
|
||||||
|
for (const derivative of [result.display, result.thumbnail]) {
|
||||||
|
const metadata = await sharp(derivative.buffer).metadata();
|
||||||
|
const pixels = await sharp(derivative.buffer).raw().toBuffer();
|
||||||
|
|
||||||
|
assert.equal(metadata.hasAlpha, false);
|
||||||
|
assert.ok(
|
||||||
|
pixels.every((channel) => channel <= 2),
|
||||||
|
"fully transparent source pixels should become black",
|
||||||
|
);
|
||||||
|
}
|
||||||
|
});
|
||||||
|
|
||||||
|
test("supports a temporary file path without changing source identity", async (t) => {
|
||||||
|
const temporaryDirectory = fs.mkdtempSync(
|
||||||
|
path.join(os.tmpdir(), "announcements-image-processing-"),
|
||||||
|
);
|
||||||
|
t.after(() => fs.rmSync(temporaryDirectory, { recursive: true, force: true }));
|
||||||
|
|
||||||
|
const source = await solidImage("webp", 800, 600);
|
||||||
|
const filename = path.join(temporaryDirectory, "upload.tmp");
|
||||||
|
fs.writeFileSync(filename, source);
|
||||||
|
|
||||||
|
const result = await createProcessor().process(filename);
|
||||||
|
assert.equal(result.original.sha256, hash(source));
|
||||||
|
assert.equal(result.original.byteSize, source.length);
|
||||||
|
assert.equal(result.original.format, "webp");
|
||||||
|
});
|
||||||
|
|
||||||
|
test("rejects malformed and unsupported decoded content with typed errors", async () => {
|
||||||
|
const processor = createProcessor();
|
||||||
|
const gif = await solidImage("gif", 64, 64);
|
||||||
|
|
||||||
|
await assert.rejects(
|
||||||
|
processor.process(Buffer.from("not an image")),
|
||||||
|
(error) =>
|
||||||
|
error instanceof ImageProcessingError &&
|
||||||
|
error.code === "MALFORMED_IMAGE" &&
|
||||||
|
error.statusCode === 422,
|
||||||
|
);
|
||||||
|
await assert.rejects(
|
||||||
|
processor.process(gif),
|
||||||
|
(error) =>
|
||||||
|
error instanceof ImageProcessingError &&
|
||||||
|
error.code === "UNSUPPORTED_IMAGE_FORMAT" &&
|
||||||
|
error.statusCode === 415 &&
|
||||||
|
error.details.detectedFormat === "gif",
|
||||||
|
);
|
||||||
|
});
|
||||||
|
|
||||||
|
test("rejects animated WebP and GIF instead of selecting the first frame", async (t) => {
|
||||||
|
for (const format of ["webp", "gif"]) {
|
||||||
|
await t.test(format, async () => {
|
||||||
|
const animated = await twoFrameImage(format);
|
||||||
|
const metadata = await sharp(animated, { animated: true }).metadata();
|
||||||
|
assert.equal(metadata.pages, 2, "test fixture must contain two frames");
|
||||||
|
await assert.rejects(
|
||||||
|
createProcessor().process(animated),
|
||||||
|
(error) =>
|
||||||
|
error instanceof ImageProcessingError &&
|
||||||
|
error.code === "ANIMATED_IMAGE_NOT_SUPPORTED" &&
|
||||||
|
error.details.pages === 2,
|
||||||
|
);
|
||||||
|
});
|
||||||
|
}
|
||||||
|
});
|
||||||
|
|
||||||
|
test("rejects unexpected multipage inputs with a distinct typed error", async () => {
|
||||||
|
const multipageTiff = await twoFrameImage("tiff");
|
||||||
|
const metadata = await sharp(multipageTiff, { animated: true }).metadata();
|
||||||
|
assert.equal(metadata.pages, 2, "test fixture must contain two pages");
|
||||||
|
|
||||||
|
await assert.rejects(
|
||||||
|
createProcessor().process(multipageTiff),
|
||||||
|
(error) =>
|
||||||
|
error instanceof ImageProcessingError &&
|
||||||
|
error.code === "MULTIPAGE_IMAGE_NOT_SUPPORTED" &&
|
||||||
|
error.details.pages === 2,
|
||||||
|
);
|
||||||
|
});
|
||||||
|
|
||||||
|
test("enforces the injected decoded-pixel limit", async () => {
|
||||||
|
const source = await solidImage("png", 100, 100);
|
||||||
|
const processor = createImageProcessor({ maxInputPixels: 9_999 });
|
||||||
|
|
||||||
|
await assert.rejects(
|
||||||
|
processor.process(source),
|
||||||
|
(error) =>
|
||||||
|
error instanceof ImageProcessingError &&
|
||||||
|
error.code === "IMAGE_PIXEL_LIMIT_EXCEEDED" &&
|
||||||
|
error.statusCode === 413 &&
|
||||||
|
error.details.maxInputPixels === 9_999,
|
||||||
|
);
|
||||||
|
});
|
||||||
|
|
||||||
|
test("uses documented defaults while allowing validated target overrides", async () => {
|
||||||
|
assert.deepEqual(DEFAULT_IMAGE_DERIVATIVE_OPTIONS, {
|
||||||
|
display: { maxWidth: 1920, maxHeight: 1080, quality: 82 },
|
||||||
|
thumbnail: { maxWidth: 480, maxHeight: 270, quality: 72 },
|
||||||
|
});
|
||||||
|
|
||||||
|
const source = await solidImage("png", 900, 600);
|
||||||
|
const processor = createProcessor({
|
||||||
|
display: { maxWidth: 300, maxHeight: 300, quality: 90 },
|
||||||
|
thumbnail: { maxWidth: 100, maxHeight: 50, quality: 65 },
|
||||||
|
});
|
||||||
|
const result = await processor.process(source);
|
||||||
|
|
||||||
|
assert.deepEqual(
|
||||||
|
{ width: result.display.width, height: result.display.height },
|
||||||
|
{ width: 300, height: 200 },
|
||||||
|
);
|
||||||
|
assert.deepEqual(
|
||||||
|
{ width: result.thumbnail.width, height: result.thumbnail.height },
|
||||||
|
{ width: 75, height: 50 },
|
||||||
|
);
|
||||||
|
assert.throws(
|
||||||
|
() => createImageProcessor({ maxInputPixels: 0 }),
|
||||||
|
/maxInputPixels must be at least 1/,
|
||||||
|
);
|
||||||
|
assert.throws(
|
||||||
|
() =>
|
||||||
|
createImageProcessor({
|
||||||
|
maxInputPixels: 10_000,
|
||||||
|
display: { quality: 101 },
|
||||||
|
}),
|
||||||
|
/display\.quality must be at most 100/,
|
||||||
|
);
|
||||||
|
});
|
||||||
|
|
||||||
|
test("rejects invalid input interfaces with structured errors", async () => {
|
||||||
|
await assert.rejects(
|
||||||
|
createProcessor().process({ path: "/tmp/not-an-interface" }),
|
||||||
|
(error) =>
|
||||||
|
error instanceof ImageProcessingError &&
|
||||||
|
error.code === "INVALID_IMAGE_INPUT" &&
|
||||||
|
error.statusCode === 400,
|
||||||
|
);
|
||||||
|
});
|
||||||
@@ -29,13 +29,33 @@ const silentLogger = {
|
|||||||
error() {},
|
error() {},
|
||||||
};
|
};
|
||||||
|
|
||||||
|
function testStorage() {
|
||||||
|
return {
|
||||||
|
async start() {},
|
||||||
|
stop() {},
|
||||||
|
health() {
|
||||||
|
return {
|
||||||
|
status: "degraded",
|
||||||
|
checkedAt: "2026-08-16T00:00:00.000Z",
|
||||||
|
lastSuccessAt: null,
|
||||||
|
code: "STORAGE_UNAVAILABLE",
|
||||||
|
message: "Object storage is unavailable",
|
||||||
|
};
|
||||||
|
},
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
async function responseJson(baseUrl, path, init) {
|
async function responseJson(baseUrl, path, init) {
|
||||||
const response = await fetch(`${baseUrl}${path}`, init);
|
const response = await fetch(`${baseUrl}${path}`, init);
|
||||||
return { response, body: await response.json() };
|
return { response, body: await response.json() };
|
||||||
}
|
}
|
||||||
|
|
||||||
test("HTTP foundation exposes live, ready, detailed health, and persisted settings", async (t) => {
|
test("HTTP foundation exposes live, ready, detailed health, and persisted settings", async (t) => {
|
||||||
const runtime = createRuntime({ config: testConfig(), logger: silentLogger });
|
const runtime = createRuntime({
|
||||||
|
config: testConfig(),
|
||||||
|
logger: silentLogger,
|
||||||
|
storage: testStorage(),
|
||||||
|
});
|
||||||
await runtime.start();
|
await runtime.start();
|
||||||
t.after(() => runtime.stop("test cleanup"));
|
t.after(() => runtime.stop("test cleanup"));
|
||||||
const baseUrl = `http://127.0.0.1:${runtime.address.port}`;
|
const baseUrl = `http://127.0.0.1:${runtime.address.port}`;
|
||||||
@@ -52,7 +72,7 @@ test("HTTP foundation exposes live, ready, detailed health, and persisted settin
|
|||||||
assert.equal(status.response.status, 200);
|
assert.equal(status.response.status, 200);
|
||||||
assert.equal(status.body.status, "degraded");
|
assert.equal(status.body.status, "degraded");
|
||||||
assert.equal(status.body.services.database.currentSchemaVersion, 1);
|
assert.equal(status.body.services.database.currentSchemaVersion, 1);
|
||||||
assert.equal(status.body.services.storage.status, "not_checked");
|
assert.equal(status.body.services.storage.status, "degraded");
|
||||||
const statusText = JSON.stringify(status.body);
|
const statusText = JSON.stringify(status.body);
|
||||||
assert.doesNotMatch(statusText, /access-key-that-must-not-leak/);
|
assert.doesNotMatch(statusText, /access-key-that-must-not-leak/);
|
||||||
assert.doesNotMatch(statusText, /secret-key-that-must-not-leak/);
|
assert.doesNotMatch(statusText, /secret-key-that-must-not-leak/);
|
||||||
@@ -83,7 +103,11 @@ test("HTTP foundation exposes live, ready, detailed health, and persisted settin
|
|||||||
});
|
});
|
||||||
|
|
||||||
test("invalid settings receive a structured client error", async (t) => {
|
test("invalid settings receive a structured client error", async (t) => {
|
||||||
const runtime = createRuntime({ config: testConfig(), logger: silentLogger });
|
const runtime = createRuntime({
|
||||||
|
config: testConfig(),
|
||||||
|
logger: silentLogger,
|
||||||
|
storage: testStorage(),
|
||||||
|
});
|
||||||
await runtime.start();
|
await runtime.start();
|
||||||
t.after(() => runtime.stop("test cleanup"));
|
t.after(() => runtime.stop("test cleanup"));
|
||||||
const baseUrl = `http://127.0.0.1:${runtime.address.port}`;
|
const baseUrl = `http://127.0.0.1:${runtime.address.port}`;
|
||||||
@@ -99,7 +123,11 @@ test("invalid settings receive a structured client error", async (t) => {
|
|||||||
});
|
});
|
||||||
|
|
||||||
test("liveness stays up while critical readiness failures return 503", async (t) => {
|
test("liveness stays up while critical readiness failures return 503", async (t) => {
|
||||||
const runtime = createRuntime({ config: testConfig(), logger: silentLogger });
|
const runtime = createRuntime({
|
||||||
|
config: testConfig(),
|
||||||
|
logger: silentLogger,
|
||||||
|
storage: testStorage(),
|
||||||
|
});
|
||||||
await runtime.start();
|
await runtime.start();
|
||||||
t.after(() => runtime.stop("test cleanup"));
|
t.after(() => runtime.stop("test cleanup"));
|
||||||
const baseUrl = `http://127.0.0.1:${runtime.address.port}`;
|
const baseUrl = `http://127.0.0.1:${runtime.address.port}`;
|
||||||
@@ -127,7 +155,11 @@ test("liveness stays up while critical readiness failures return 503", async (t)
|
|||||||
});
|
});
|
||||||
|
|
||||||
test("runtime startup and shutdown are idempotent", async () => {
|
test("runtime startup and shutdown are idempotent", async () => {
|
||||||
const runtime = createRuntime({ config: testConfig(), logger: silentLogger });
|
const runtime = createRuntime({
|
||||||
|
config: testConfig(),
|
||||||
|
logger: silentLogger,
|
||||||
|
storage: testStorage(),
|
||||||
|
});
|
||||||
const [firstAddress, secondAddress] = await Promise.all([
|
const [firstAddress, secondAddress] = await Promise.all([
|
||||||
runtime.start(),
|
runtime.start(),
|
||||||
runtime.start(),
|
runtime.start(),
|
||||||
|
|||||||
@@ -0,0 +1,105 @@
|
|||||||
|
const test = require("node:test");
|
||||||
|
const assert = require("node:assert/strict");
|
||||||
|
const express = require("express");
|
||||||
|
const { openDatabase } = require("../src/database");
|
||||||
|
const { runMigrations } = require("../src/database/migrate");
|
||||||
|
const {
|
||||||
|
createDetailedHealthRouter,
|
||||||
|
createHealthRouter,
|
||||||
|
} = require("../src/health/routes");
|
||||||
|
|
||||||
|
function config() {
|
||||||
|
return {
|
||||||
|
environment: "test",
|
||||||
|
host: "127.0.0.1",
|
||||||
|
port: 0,
|
||||||
|
timeZone: "America/Chicago",
|
||||||
|
trustProxy: true,
|
||||||
|
database: { path: ":memory:" },
|
||||||
|
storage: {
|
||||||
|
endpoint: "http://private-storage:8333",
|
||||||
|
region: "us-east-1",
|
||||||
|
bucket: "announcements-v2",
|
||||||
|
accessKeyId: "must-not-leak",
|
||||||
|
secretAccessKey: "must-not-leak-either",
|
||||||
|
forcePathStyle: true,
|
||||||
|
},
|
||||||
|
processing: {
|
||||||
|
tempDirectory: "/tmp/announcements-uploads",
|
||||||
|
maxFileBytes: 1024,
|
||||||
|
maxFiles: 2,
|
||||||
|
maxInputPixels: 1000,
|
||||||
|
},
|
||||||
|
mqtt: { url: null },
|
||||||
|
notifications: { homeAssistantWebhookUrl: null },
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
async function withServer(storage, callback) {
|
||||||
|
const database = openDatabase({ path: ":memory:", busyTimeoutMs: 1000 });
|
||||||
|
const migrations = runMigrations(database);
|
||||||
|
const lifecycle = { ready: true };
|
||||||
|
const app = express();
|
||||||
|
const options = { config: config(), database, lifecycle, migrations, storage };
|
||||||
|
app.use("/health", createHealthRouter(options));
|
||||||
|
app.use("/api/health", createDetailedHealthRouter(options));
|
||||||
|
const server = app.listen(0, "127.0.0.1");
|
||||||
|
await new Promise((resolve) => server.once("listening", resolve));
|
||||||
|
|
||||||
|
try {
|
||||||
|
await callback(`http://127.0.0.1:${server.address().port}`);
|
||||||
|
} finally {
|
||||||
|
await new Promise((resolve) => server.close(resolve));
|
||||||
|
database.close();
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
test("storage degradation is reported but does not fail API readiness", async () => {
|
||||||
|
const storage = {
|
||||||
|
health() {
|
||||||
|
return {
|
||||||
|
status: "degraded",
|
||||||
|
checkedAt: "2026-08-16T12:00:00.000Z",
|
||||||
|
lastSuccessAt: "2026-08-16T11:59:00.000Z",
|
||||||
|
code: "STORAGE_UNAVAILABLE",
|
||||||
|
};
|
||||||
|
},
|
||||||
|
};
|
||||||
|
|
||||||
|
await withServer(storage, async (baseUrl) => {
|
||||||
|
const ready = await fetch(`${baseUrl}/health/ready`);
|
||||||
|
const readyBody = await ready.json();
|
||||||
|
assert.equal(ready.status, 200);
|
||||||
|
assert.equal(readyBody.status, "ready");
|
||||||
|
assert.equal(readyBody.checks.storage, "degraded");
|
||||||
|
|
||||||
|
const detailed = await fetch(`${baseUrl}/api/health`);
|
||||||
|
const detailedBody = await detailed.json();
|
||||||
|
assert.equal(detailed.status, 200);
|
||||||
|
assert.equal(detailedBody.status, "degraded");
|
||||||
|
assert.equal(detailedBody.services.storage.status, "degraded");
|
||||||
|
assert.equal(
|
||||||
|
detailedBody.services.storage.lastSuccessAt,
|
||||||
|
"2026-08-16T11:59:00.000Z",
|
||||||
|
);
|
||||||
|
assert.equal(detailedBody.services.storage.code, "STORAGE_UNAVAILABLE");
|
||||||
|
assert.doesNotMatch(JSON.stringify(detailedBody), /must-not-leak/);
|
||||||
|
assert.doesNotMatch(JSON.stringify(detailedBody), /private-storage/);
|
||||||
|
});
|
||||||
|
});
|
||||||
|
|
||||||
|
test("healthy storage produces healthy detailed status", async () => {
|
||||||
|
const storage = {
|
||||||
|
health() {
|
||||||
|
return { status: "healthy", checkedAt: "2026-08-16T12:00:00.000Z" };
|
||||||
|
},
|
||||||
|
};
|
||||||
|
|
||||||
|
await withServer(storage, async (baseUrl) => {
|
||||||
|
const detailed = await fetch(`${baseUrl}/api/health`);
|
||||||
|
const body = await detailed.json();
|
||||||
|
assert.equal(detailed.status, 200);
|
||||||
|
assert.equal(body.status, "ok");
|
||||||
|
assert.equal(body.services.storage.status, "healthy");
|
||||||
|
});
|
||||||
|
});
|
||||||
@@ -0,0 +1,397 @@
|
|||||||
|
const test = require("node:test");
|
||||||
|
const assert = require("node:assert/strict");
|
||||||
|
const http = require("node:http");
|
||||||
|
const { PassThrough, Readable } = require("node:stream");
|
||||||
|
const {
|
||||||
|
StorageObjectNotFoundError,
|
||||||
|
StorageUnavailableError,
|
||||||
|
createStorageService,
|
||||||
|
} = require("../src/storage");
|
||||||
|
|
||||||
|
const storageConfig = {
|
||||||
|
endpoint: "http://seaweedfs:8333",
|
||||||
|
region: "us-east-1",
|
||||||
|
bucket: "announcements-v2",
|
||||||
|
accessKeyId: "test-key",
|
||||||
|
secretAccessKey: "test-secret",
|
||||||
|
forcePathStyle: true,
|
||||||
|
};
|
||||||
|
|
||||||
|
function fakeClient(handler) {
|
||||||
|
const commands = [];
|
||||||
|
return {
|
||||||
|
commands,
|
||||||
|
async send(command) {
|
||||||
|
commands.push(command);
|
||||||
|
return handler(command, commands.length - 1);
|
||||||
|
},
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
function commandName(command) {
|
||||||
|
return command.constructor.name;
|
||||||
|
}
|
||||||
|
|
||||||
|
test("storage initialization probes an existing bucket without creating it", async () => {
|
||||||
|
const client = fakeClient(async () => ({}));
|
||||||
|
const service = createStorageService(storageConfig, { client });
|
||||||
|
|
||||||
|
const health = await service.initialize();
|
||||||
|
assert.equal(health.status, "healthy");
|
||||||
|
assert.match(health.checkedAt, /T/);
|
||||||
|
assert.equal(health.lastSuccessAt, health.checkedAt);
|
||||||
|
assert.deepEqual(client.commands.map(commandName), ["HeadBucketCommand"]);
|
||||||
|
});
|
||||||
|
|
||||||
|
test("storage initialization creates only a missing bucket", async () => {
|
||||||
|
let exists = false;
|
||||||
|
const missing = Object.assign(new Error("missing"), {
|
||||||
|
name: "NoSuchBucket",
|
||||||
|
$metadata: { httpStatusCode: 404 },
|
||||||
|
});
|
||||||
|
const client = fakeClient(async (command) => {
|
||||||
|
if (commandName(command) === "HeadBucketCommand" && !exists) throw missing;
|
||||||
|
if (commandName(command) === "CreateBucketCommand") exists = true;
|
||||||
|
return {};
|
||||||
|
});
|
||||||
|
const service = createStorageService(storageConfig, { client });
|
||||||
|
|
||||||
|
assert.deepEqual(await service.ensureBucket(), { created: true });
|
||||||
|
assert.deepEqual(client.commands.map(commandName), [
|
||||||
|
"HeadBucketCommand",
|
||||||
|
"CreateBucketCommand",
|
||||||
|
]);
|
||||||
|
assert.equal(client.commands[1].input.Bucket, storageConfig.bucket);
|
||||||
|
|
||||||
|
assert.deepEqual(await service.ensureBucket(), { created: false });
|
||||||
|
assert.deepEqual(client.commands.map(commandName), [
|
||||||
|
"HeadBucketCommand",
|
||||||
|
"CreateBucketCommand",
|
||||||
|
"HeadBucketCommand",
|
||||||
|
]);
|
||||||
|
});
|
||||||
|
|
||||||
|
test("storage start performs a nonfatal immediate check and stop releases its client", async () => {
|
||||||
|
let destroyed = false;
|
||||||
|
const client = fakeClient(async () => ({}));
|
||||||
|
client.destroy = () => {
|
||||||
|
destroyed = true;
|
||||||
|
};
|
||||||
|
const service = createStorageService(
|
||||||
|
{ ...storageConfig, healthProbeIntervalMs: 1_000 },
|
||||||
|
{ client },
|
||||||
|
);
|
||||||
|
|
||||||
|
const health = await service.start();
|
||||||
|
assert.equal(health.status, "healthy");
|
||||||
|
assert.deepEqual(client.commands.map(commandName), ["HeadBucketCommand"]);
|
||||||
|
service.stop();
|
||||||
|
assert.equal(destroyed, true);
|
||||||
|
});
|
||||||
|
|
||||||
|
test("a create-bucket race is healthy only after the bucket can be verified", async () => {
|
||||||
|
const missing = Object.assign(new Error("missing"), {
|
||||||
|
name: "NoSuchBucket",
|
||||||
|
$metadata: { httpStatusCode: 404 },
|
||||||
|
});
|
||||||
|
const conflict = Object.assign(new Error("race"), {
|
||||||
|
name: "BucketAlreadyOwnedByYou",
|
||||||
|
$metadata: { httpStatusCode: 409 },
|
||||||
|
});
|
||||||
|
let headCalls = 0;
|
||||||
|
const client = fakeClient(async (command) => {
|
||||||
|
if (commandName(command) === "HeadBucketCommand") {
|
||||||
|
headCalls += 1;
|
||||||
|
if (headCalls === 1) throw missing;
|
||||||
|
return {};
|
||||||
|
}
|
||||||
|
if (commandName(command) === "CreateBucketCommand") throw conflict;
|
||||||
|
throw new Error(`Unexpected ${commandName(command)}`);
|
||||||
|
});
|
||||||
|
const service = createStorageService(storageConfig, { client });
|
||||||
|
|
||||||
|
assert.deepEqual(await service.ensureBucket(), { created: false });
|
||||||
|
assert.equal(service.health().status, "healthy");
|
||||||
|
assert.deepEqual(client.commands.map(commandName), [
|
||||||
|
"HeadBucketCommand",
|
||||||
|
"CreateBucketCommand",
|
||||||
|
"HeadBucketCommand",
|
||||||
|
]);
|
||||||
|
});
|
||||||
|
|
||||||
|
test("an older failed probe cannot overwrite a newer successful operation", async () => {
|
||||||
|
let rejectProbe;
|
||||||
|
let notifyProbeStarted;
|
||||||
|
const probeStarted = new Promise((resolve) => {
|
||||||
|
notifyProbeStarted = resolve;
|
||||||
|
});
|
||||||
|
let headCalls = 0;
|
||||||
|
const client = fakeClient(async (command) => {
|
||||||
|
if (commandName(command) === "HeadBucketCommand") {
|
||||||
|
headCalls += 1;
|
||||||
|
if (headCalls === 1) return {};
|
||||||
|
notifyProbeStarted();
|
||||||
|
return new Promise((resolve, reject) => {
|
||||||
|
rejectProbe = reject;
|
||||||
|
});
|
||||||
|
}
|
||||||
|
if (commandName(command) === "PutObjectCommand") return {};
|
||||||
|
throw new Error(`Unexpected ${commandName(command)}`);
|
||||||
|
});
|
||||||
|
const service = createStorageService(storageConfig, { client });
|
||||||
|
await service.initialize();
|
||||||
|
|
||||||
|
const staleProbe = service.probe();
|
||||||
|
await probeStarted;
|
||||||
|
await service.putObject({
|
||||||
|
key: "assets/id/original/hash",
|
||||||
|
body: Buffer.from("bytes"),
|
||||||
|
contentLength: 5,
|
||||||
|
});
|
||||||
|
const successfulState = service.health();
|
||||||
|
rejectProbe(new Error("older request failed"));
|
||||||
|
await staleProbe;
|
||||||
|
|
||||||
|
assert.equal(successfulState.status, "healthy");
|
||||||
|
assert.deepEqual(service.health(), successfulState);
|
||||||
|
});
|
||||||
|
|
||||||
|
test("temporary storage failure degrades without throwing during initialization and recovers", async () => {
|
||||||
|
let available = false;
|
||||||
|
const warnings = [];
|
||||||
|
const client = fakeClient(async () => {
|
||||||
|
if (!available) throw new Error("connection refused at a private address");
|
||||||
|
return {};
|
||||||
|
});
|
||||||
|
const service = createStorageService(storageConfig, {
|
||||||
|
client,
|
||||||
|
logger: { warn(message) { warnings.push(message); } },
|
||||||
|
});
|
||||||
|
|
||||||
|
const initial = await service.initialize();
|
||||||
|
assert.equal(initial.status, "degraded");
|
||||||
|
assert.equal(initial.code, "STORAGE_UNAVAILABLE");
|
||||||
|
assert.equal(initial.lastSuccessAt, null);
|
||||||
|
assert.deepEqual(warnings, ["Object storage is unavailable during initialization"]);
|
||||||
|
|
||||||
|
available = true;
|
||||||
|
const recovered = await service.probe();
|
||||||
|
assert.equal(recovered.status, "healthy");
|
||||||
|
assert.equal(recovered.lastSuccessAt, recovered.checkedAt);
|
||||||
|
assert.equal(service.health().status, "healthy");
|
||||||
|
|
||||||
|
available = false;
|
||||||
|
const degradedAgain = await service.probe();
|
||||||
|
assert.equal(degradedAgain.status, "degraded");
|
||||||
|
assert.equal(degradedAgain.lastSuccessAt, recovered.lastSuccessAt);
|
||||||
|
});
|
||||||
|
|
||||||
|
test("put, get, head, existence, and delete use the configured private bucket", async () => {
|
||||||
|
const body = Readable.from([Buffer.from("image bytes")]);
|
||||||
|
const client = fakeClient(async (command) => {
|
||||||
|
switch (commandName(command)) {
|
||||||
|
case "PutObjectCommand":
|
||||||
|
return { ETag: '"etag"' };
|
||||||
|
case "GetObjectCommand":
|
||||||
|
return {
|
||||||
|
Body: body,
|
||||||
|
ContentType: "image/webp",
|
||||||
|
ContentLength: 11,
|
||||||
|
ETag: '"etag"',
|
||||||
|
};
|
||||||
|
case "HeadObjectCommand":
|
||||||
|
return {
|
||||||
|
ContentType: "image/webp",
|
||||||
|
ContentLength: 11,
|
||||||
|
ETag: '"etag"',
|
||||||
|
};
|
||||||
|
case "DeleteObjectCommand":
|
||||||
|
return {};
|
||||||
|
case "HeadBucketCommand":
|
||||||
|
return {};
|
||||||
|
default:
|
||||||
|
throw new Error(`Unexpected ${commandName(command)}`);
|
||||||
|
}
|
||||||
|
});
|
||||||
|
const service = createStorageService(storageConfig, { client });
|
||||||
|
const key = "assets/id/display/hash.webp";
|
||||||
|
await service.initialize();
|
||||||
|
client.commands.length = 0;
|
||||||
|
|
||||||
|
const put = await service.putObject({
|
||||||
|
key,
|
||||||
|
body: Buffer.from("image bytes"),
|
||||||
|
contentType: "image/webp",
|
||||||
|
contentLength: 11,
|
||||||
|
});
|
||||||
|
assert.deepEqual(put, { key, etag: '"etag"', versionId: null });
|
||||||
|
|
||||||
|
const fetched = await service.getObject(key);
|
||||||
|
assert.equal(fetched.body, body);
|
||||||
|
assert.equal(fetched.contentType, "image/webp");
|
||||||
|
assert.equal(fetched.contentLength, 11);
|
||||||
|
|
||||||
|
assert.equal((await service.headObject(key)).etag, '"etag"');
|
||||||
|
assert.equal(await service.objectExists(key), true);
|
||||||
|
await service.deleteObject(key);
|
||||||
|
|
||||||
|
for (const command of client.commands) {
|
||||||
|
assert.equal(command.input.Bucket, storageConfig.bucket);
|
||||||
|
assert.equal(command.input.Key, key);
|
||||||
|
}
|
||||||
|
});
|
||||||
|
|
||||||
|
test("missing objects are distinct from unavailable storage", async () => {
|
||||||
|
let missing = true;
|
||||||
|
const client = fakeClient(async (command) => {
|
||||||
|
if (commandName(command) === "HeadBucketCommand") return {};
|
||||||
|
if (missing) {
|
||||||
|
throw Object.assign(new Error("missing"), {
|
||||||
|
name: "NoSuchKey",
|
||||||
|
$metadata: { httpStatusCode: 404 },
|
||||||
|
});
|
||||||
|
}
|
||||||
|
throw new Error("network offline");
|
||||||
|
});
|
||||||
|
const service = createStorageService(storageConfig, { client });
|
||||||
|
|
||||||
|
await service.initialize();
|
||||||
|
client.commands.length = 0;
|
||||||
|
|
||||||
|
assert.equal(await service.headObject("assets/id/original/hash"), null);
|
||||||
|
assert.equal(await service.objectExists("assets/id/original/hash"), false);
|
||||||
|
await assert.rejects(
|
||||||
|
service.getObject("assets/id/original/hash"),
|
||||||
|
StorageObjectNotFoundError,
|
||||||
|
);
|
||||||
|
assert.equal(service.health().status, "healthy");
|
||||||
|
|
||||||
|
missing = false;
|
||||||
|
await assert.rejects(
|
||||||
|
service.putObject({ key: "assets/id/original/hash", body: Buffer.alloc(0) }),
|
||||||
|
(error) =>
|
||||||
|
error instanceof StorageUnavailableError &&
|
||||||
|
error.code === "STORAGE_UNAVAILABLE" &&
|
||||||
|
error.statusCode === 503,
|
||||||
|
);
|
||||||
|
assert.equal(service.health().status, "degraded");
|
||||||
|
});
|
||||||
|
|
||||||
|
test("a missing bucket during object access is a storage outage, not a missing object", async () => {
|
||||||
|
let initialized = false;
|
||||||
|
const client = fakeClient(async (command) => {
|
||||||
|
if (commandName(command) === "HeadBucketCommand" && !initialized) {
|
||||||
|
initialized = true;
|
||||||
|
return {};
|
||||||
|
}
|
||||||
|
throw Object.assign(new Error("bucket missing"), {
|
||||||
|
name: "NoSuchBucket",
|
||||||
|
$metadata: { httpStatusCode: 404 },
|
||||||
|
});
|
||||||
|
});
|
||||||
|
const service = createStorageService(storageConfig, { client });
|
||||||
|
await service.initialize();
|
||||||
|
|
||||||
|
await assert.rejects(
|
||||||
|
service.getObject("assets/id/original/hash"),
|
||||||
|
StorageUnavailableError,
|
||||||
|
);
|
||||||
|
assert.equal(service.health().status, "degraded");
|
||||||
|
});
|
||||||
|
|
||||||
|
test("storage rejects unsafe internal object keys before contacting S3", async () => {
|
||||||
|
const client = fakeClient(async () => ({}));
|
||||||
|
const service = createStorageService(storageConfig, { client });
|
||||||
|
|
||||||
|
await assert.rejects(
|
||||||
|
service.getObject("../private"),
|
||||||
|
/Storage object key is invalid/,
|
||||||
|
);
|
||||||
|
await assert.rejects(
|
||||||
|
service.deleteObject("/absolute"),
|
||||||
|
/Storage object key is invalid/,
|
||||||
|
);
|
||||||
|
assert.equal(client.commands.length, 0);
|
||||||
|
});
|
||||||
|
|
||||||
|
test("real HTTP requests time out instead of hanging indefinitely", async (t) => {
|
||||||
|
const server = http.createServer(() => {
|
||||||
|
// Intentionally accept the connection without sending response headers.
|
||||||
|
});
|
||||||
|
server.listen(0, "127.0.0.1");
|
||||||
|
await new Promise((resolve) => server.once("listening", resolve));
|
||||||
|
t.after(async () => {
|
||||||
|
server.closeAllConnections();
|
||||||
|
await new Promise((resolve) => server.close(resolve));
|
||||||
|
});
|
||||||
|
|
||||||
|
const service = createStorageService({
|
||||||
|
...storageConfig,
|
||||||
|
endpoint: `http://127.0.0.1:${server.address().port}`,
|
||||||
|
connectionTimeoutMs: 100,
|
||||||
|
requestTimeoutMs: 100,
|
||||||
|
maxAttempts: 1,
|
||||||
|
});
|
||||||
|
t.after(() => service.stop());
|
||||||
|
|
||||||
|
const startedAt = Date.now();
|
||||||
|
const health = await service.probe();
|
||||||
|
assert.equal(health.status, "degraded");
|
||||||
|
assert.equal(health.code, "STORAGE_UNAVAILABLE");
|
||||||
|
assert.ok(Date.now() - startedAt < 2_000);
|
||||||
|
});
|
||||||
|
|
||||||
|
test("consumer stream cancellation does not falsely degrade storage", async () => {
|
||||||
|
let body = new PassThrough();
|
||||||
|
const client = fakeClient(async (command) => {
|
||||||
|
if (commandName(command) === "HeadBucketCommand") return {};
|
||||||
|
if (commandName(command) === "GetObjectCommand") {
|
||||||
|
return { Body: body, ContentLength: 10, ContentType: "image/webp" };
|
||||||
|
}
|
||||||
|
throw new Error(`Unexpected ${commandName(command)}`);
|
||||||
|
});
|
||||||
|
const service = createStorageService(storageConfig, { client });
|
||||||
|
await service.initialize();
|
||||||
|
|
||||||
|
const cancelled = await service.getObject("assets/id/display/hash.webp");
|
||||||
|
cancelled.body.destroy(
|
||||||
|
Object.assign(new Error("consumer closed"), {
|
||||||
|
code: "ERR_STREAM_PREMATURE_CLOSE",
|
||||||
|
}),
|
||||||
|
);
|
||||||
|
await new Promise((resolve) => setImmediate(resolve));
|
||||||
|
assert.equal(service.health().status, "healthy");
|
||||||
|
|
||||||
|
body = new PassThrough();
|
||||||
|
const failed = await service.getObject("assets/id/display/hash.webp");
|
||||||
|
failed.body.destroy(Object.assign(new Error("upstream reset"), { code: "ECONNRESET" }));
|
||||||
|
await new Promise((resolve) => setImmediate(resolve));
|
||||||
|
assert.equal(service.health().status, "degraded");
|
||||||
|
});
|
||||||
|
|
||||||
|
test("delete treats an already-missing object as an idempotent success", async () => {
|
||||||
|
let missingBucket = false;
|
||||||
|
const client = fakeClient(async (command) => {
|
||||||
|
if (commandName(command) === "HeadBucketCommand") return {};
|
||||||
|
if (commandName(command) === "DeleteObjectCommand") {
|
||||||
|
throw Object.assign(new Error("missing"), {
|
||||||
|
name: missingBucket ? "NoSuchBucket" : "NoSuchKey",
|
||||||
|
$metadata: { httpStatusCode: 404 },
|
||||||
|
});
|
||||||
|
}
|
||||||
|
throw new Error(`Unexpected ${commandName(command)}`);
|
||||||
|
});
|
||||||
|
const service = createStorageService(storageConfig, { client });
|
||||||
|
await service.initialize();
|
||||||
|
|
||||||
|
await assert.doesNotReject(
|
||||||
|
service.deleteObject("assets/id/original/hash"),
|
||||||
|
);
|
||||||
|
assert.equal(service.health().status, "healthy");
|
||||||
|
|
||||||
|
missingBucket = true;
|
||||||
|
await assert.rejects(
|
||||||
|
service.deleteObject("assets/id/original/hash"),
|
||||||
|
StorageUnavailableError,
|
||||||
|
);
|
||||||
|
});
|
||||||
Reference in New Issue
Block a user