From 722c6cf77de5a8fb125ef73663d6d8147527d57c Mon Sep 17 00:00:00 2001 From: Media Date: Sun, 16 Aug 2026 19:59:10 -0500 Subject: [PATCH] Implement Phase 2 asset storage pipeline --- .dockerignore | 1 + .env.example | 14 +- .gitignore | 1 + README.md | 93 +++ client/nginx.conf | 8 + docker-compose.yaml | 8 + pnpm-lock.yaml | 752 ++++++++++++++++++++++++ pnpm-workspace.yaml | 1 + server/package.json | 3 + server/src/app.js | 25 +- server/src/assets/errors.js | 23 + server/src/assets/media-routes.js | 49 ++ server/src/assets/multipart.js | 235 ++++++++ server/src/assets/routes.js | 102 ++++ server/src/assets/service.js | 835 +++++++++++++++++++++++++++ server/src/assets/streaming.js | 66 +++ server/src/config/index.js | 46 ++ server/src/health/routes.js | 66 ++- server/src/image-processing/index.js | 330 +++++++++++ server/src/runtime.js | 45 +- server/src/storage/index.js | 398 +++++++++++++ server/test/assets-api.test.js | 442 ++++++++++++++ server/test/assets-multipart.test.js | 35 ++ server/test/assets-service.test.js | 608 +++++++++++++++++++ server/test/config.test.js | 41 ++ server/test/image-processing.test.js | 304 ++++++++++ server/test/runtime.test.js | 42 +- server/test/storage-health.test.js | 105 ++++ server/test/storage.test.js | 397 +++++++++++++ 29 files changed, 5059 insertions(+), 16 deletions(-) create mode 100644 README.md create mode 100644 server/src/assets/errors.js create mode 100644 server/src/assets/media-routes.js create mode 100644 server/src/assets/multipart.js create mode 100644 server/src/assets/routes.js create mode 100644 server/src/assets/service.js create mode 100644 server/src/assets/streaming.js create mode 100644 server/src/image-processing/index.js create mode 100644 server/src/storage/index.js create mode 100644 server/test/assets-api.test.js create mode 100644 server/test/assets-multipart.test.js create mode 100644 server/test/assets-service.test.js create mode 100644 server/test/image-processing.test.js create mode 100644 server/test/storage-health.test.js create mode 100644 server/test/storage.test.js diff --git a/.dockerignore b/.dockerignore index c6ec275..fa81f62 100644 --- a/.dockerignore +++ b/.dockerignore @@ -6,6 +6,7 @@ .env.* !.env.example **/node_modules +.pnpm-store **/dist **/*.log **/*.sqlite diff --git a/.env.example b/.env.example index 85ed4f0..cb0ebaf 100644 --- a/.env.example +++ b/.env.example @@ -25,8 +25,20 @@ S3_REGION=us-east-1 S3_BUCKET=announcements-v2 S3_ACCESS_KEY_ID=replace-with-a-random-access-key S3_SECRET_ACCESS_KEY=replace-with-a-long-random-secret +S3_CONNECT_TIMEOUT_MS=2000 +# Maximum socket inactivity during one S3 operation (not a total upload deadline). +S3_REQUEST_TIMEOUT_MS=5000 +S3_MAX_ATTEMPTS=2 +S3_HEALTH_PROBE_INTERVAL_MS=30000 -# Optional integrations. Leave blank to disable them during Phase 1. +# Image upload and decode limits. The default proxy body limit is 512 MiB, +# which accommodates the default twenty 25 MiB files plus multipart overhead. +UPLOAD_MAX_FILE_BYTES=26214400 +UPLOAD_MAX_FILES=20 +IMAGE_MAX_INPUT_PIXELS=40000000 +# UPLOAD_TEMP_DIRECTORY=/tmp/announcements-uploads + +# Optional integrations. Leave blank to disable them during Phase 2. MQTT_URL= MQTT_USERNAME= MQTT_PASSWORD= diff --git a/.gitignore b/.gitignore index 1ce3877..edfc8d9 100644 --- a/.gitignore +++ b/.gitignore @@ -1,4 +1,5 @@ node_modules/ +.pnpm-store/ # Local configuration and secrets .env diff --git a/README.md b/README.md new file mode 100644 index 0000000..4c9e3bf --- /dev/null +++ b/README.md @@ -0,0 +1,93 @@ +# Announcements v2 + +Announcements is a small Express, React, Socket.IO, MQTT, SQLite, and +S3-compatible signage application. Version 2 is being built in phases and is +intended to run beside the existing production version until cutover. + +## Development + +The repository is a pnpm workspace with one lockfile. It requires Node.js +22.13 or newer and pnpm 10.29.3. + +```sh +cp .env.example .env +pnpm install --frozen-lockfile +pnpm check +docker compose up --build +``` + +Only the Nginx web service publishes a host port (5175 by default). The API and +SeaweedFS S3 endpoint remain private on Compose networks. Replace the example +S3 credentials before starting the stack. + +## Library assets + +`POST /api/assets` accepts multipart uploads in the repeated `file` or `files` +field. An optional `display_name` (or `displayName`) is accepted for a +single-file upload. Supported decoded formats are still JPEG, PNG, and WebP; +extensions and multipart content types are not trusted. Animated, multipage, +malformed, unsupported, oversized, and excessive-pixel inputs are rejected. + +Default limits are: + +- 25 MiB per file (`UPLOAD_MAX_FILE_BYTES`) +- 20 files per request (`UPLOAD_MAX_FILES`) +- 40 million decoded pixels per image (`IMAGE_MAX_INPUT_PIXELS`) + +Nginx accepts up to 512 MiB per request, enough for the default aggregate plus +multipart overhead. If the backend limits are raised, update +`client_max_body_size` in `client/nginx.conf` as well. Proxied API requests use +a ten-minute inactivity timeout so a valid large batch can finish processing. + +Original bytes are archived unchanged. Content identity is the SHA-256 of +those bytes, so uploading the same content returns the existing asset even +when the filename differs. Display-name collisions with different content +return `409 ASSET_NAME_CONFLICT`. Multi-file uploads validate every new image +locally before persisting any of them. If a later storage operation fails, +already completed items remain valid and retrying the request is idempotent. + +Sharp produces two metadata-free, auto-oriented WebP renditions: + +- display: inside-fit at most 1920×1080, quality 82 +- thumbnail: inside-fit at most 480×270, quality 72 + +Neither rendition crops or upscales. Transparency is flattened against black. +Presentation effects such as blur, shadows, darkening, and framing are not +baked into stored media. + +## Asset and media API + +- `GET /api/assets` +- `POST /api/assets` +- `GET /api/assets/:assetId` +- `PATCH /api/assets/:assetId` +- `GET /api/assets/:assetId/references` +- `DELETE /api/assets/:assetId` +- `GET /api/assets/:assetId/original` +- `GET /media/assets/:assetId/display/:sha256.webp` +- `GET /media/assets/:assetId/thumbnail/:sha256.webp` + +PATCH accepts only display name and expiration. A null expiration is permanent. +Manual deletion refuses playlist, current playback, and static references. It +marks an asset as deleting before idempotently removing its three objects and +removes SQLite metadata only after storage succeeds. A failed processing row +can be inspected, safely deleted, and uploaded again; no automatic retry loop +is run. + +Media URLs are derived from SQLite metadata and never accept object keys. The +display and thumbnail routes stream private S3 objects through the API with a +one-year immutable cache policy. Original downloads use private, no-store +caching. + +## Storage and health + +The API uses a small path-style S3 adapter configured by `S3_ENDPOINT`, +`S3_REGION`, `S3_BUCKET`, `S3_ACCESS_KEY_ID`, and `S3_SECRET_ACCESS_KEY`. +Connection timeout, socket-inactivity timeout (`S3_REQUEST_TIMEOUT_MS`), +attempts, and background probe interval are configurable in `.env.example`. +Startup verifies or safely creates only the configured bucket. + +Storage outages do not stop the API or fail process readiness. `/health/ready` +reports storage as degraded while remaining HTTP 200 when SQLite and the API +runtime are healthy. `/api/health` and `/api/status` expose cached, redacted +storage state and recover after a successful probe or storage operation. diff --git a/client/nginx.conf b/client/nginx.conf index f1c77db..47812b2 100644 --- a/client/nginx.conf +++ b/client/nginx.conf @@ -10,6 +10,9 @@ server { root /usr/share/nginx/html; index index.html; + # Matches the default API allowance of twenty 25 MiB files plus multipart + # overhead. Keep this in sync if the backend upload limits are raised. + client_max_body_size 512m; resolver 127.0.0.11 valid=10s ipv6=off; set $api_origin http://api:7532; @@ -17,6 +20,11 @@ server { location ~ ^/(api|health|media)(/|$) { proxy_pass $api_origin; proxy_http_version 1.1; + proxy_connect_timeout 5s; + # Multipart requests return only after image processing and private-S3 + # writes complete. This is an inactivity timeout, not a size limit. + proxy_read_timeout 10m; + proxy_send_timeout 10m; proxy_set_header Host $host; proxy_set_header X-Real-IP $remote_addr; proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; diff --git a/docker-compose.yaml b/docker-compose.yaml index 766610a..d7f449a 100644 --- a/docker-compose.yaml +++ b/docker-compose.yaml @@ -42,6 +42,14 @@ services: S3_ACCESS_KEY_ID: "${S3_ACCESS_KEY_ID:?Set S3_ACCESS_KEY_ID in .env}" S3_SECRET_ACCESS_KEY: "${S3_SECRET_ACCESS_KEY:?Set S3_SECRET_ACCESS_KEY in .env}" S3_FORCE_PATH_STYLE: "true" + S3_CONNECT_TIMEOUT_MS: "${S3_CONNECT_TIMEOUT_MS:-2000}" + S3_REQUEST_TIMEOUT_MS: "${S3_REQUEST_TIMEOUT_MS:-5000}" + S3_MAX_ATTEMPTS: "${S3_MAX_ATTEMPTS:-2}" + S3_HEALTH_PROBE_INTERVAL_MS: "${S3_HEALTH_PROBE_INTERVAL_MS:-30000}" + UPLOAD_TEMP_DIRECTORY: /tmp/announcements-uploads + UPLOAD_MAX_FILE_BYTES: "${UPLOAD_MAX_FILE_BYTES:-26214400}" + UPLOAD_MAX_FILES: "${UPLOAD_MAX_FILES:-20}" + IMAGE_MAX_INPUT_PIXELS: "${IMAGE_MAX_INPUT_PIXELS:-40000000}" MQTT_URL: "${MQTT_URL:-}" MQTT_USERNAME: "${MQTT_USERNAME:-}" MQTT_PASSWORD: "${MQTT_PASSWORD:-}" diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index a6044e2..73526fa 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -50,6 +50,9 @@ importers: server: dependencies: + '@aws-sdk/client-s3': + specifier: 3.1101.0 + version: 3.1101.0 better-sqlite3: specifier: 13.0.2 version: 13.0.2 @@ -59,12 +62,90 @@ importers: express: specifier: ^5.2.1 version: 5.2.1 + multer: + specifier: 2.0.2 + version: 2.0.2 + sharp: + specifier: 0.35.3 + version: 0.35.3 socket.io: specifier: ^4.8.3 version: 4.8.3 packages: + '@aws-sdk/checksums@3.1000.28': + resolution: {integrity: sha512-VCpnmyHQ1IH49ni3LXnQj7DPr7rmcJmzYeiCkYdCcfgNtkvOj38cdcL9lapBWoItZWFACJPFJlymqC7/gem3Gw==} + engines: {node: '>=20.0.0'} + + '@aws-sdk/client-s3@3.1101.0': + resolution: {integrity: sha512-16EFb1aTEBgPcfUAWAjjlB57IZCyn7B3rlfT+xqE7M6WoH8AMMU3vFZO0UOitwh/xvvzVx73YED1/n0PU4qBMw==} + engines: {node: '>=20.0.0'} + + '@aws-sdk/core@3.977.8': + resolution: {integrity: sha512-7+Kcrkvrk9lM/m7jRhHpT4jCdvzGHsuaSRbF8TdzzkY1mRzp/Ogwf9c7H29k4gGhey0BBWhCWr16+t0J61gwmg==} + engines: {node: '>=20.0.0'} + + '@aws-sdk/credential-provider-env@3.972.69': + resolution: {integrity: sha512-AreCFzcB4kH2HF9031Ot0jSJr3KXvRg6e8uDeub20JEVdZU3Bv0sTq1plc7VsT3KiqutlzH7l0j50UcCWHUioA==} + engines: {node: '>=20.0.0'} + + '@aws-sdk/credential-provider-http@3.972.71': + resolution: {integrity: sha512-A8ObcqVmDMnk4F9NozZ7JwmUu9Q4xyBJkmyq1C5U+wNM9ht9J7+EuuyabsLWXZnOoTqFaJuYBYTKf5CTipkEjA==} + engines: {node: '>=20.0.0'} + + '@aws-sdk/credential-provider-ini@3.973.14': + resolution: {integrity: sha512-7c+Wti2LsERNWMfm7ySz3/6RPopFW3Nmn7s63Xpcq6R/tRuY5hpvkHA2xVgi5ukJbvok9l0IDtVEvqTtg+X7dw==} + engines: {node: '>=20.0.0'} + + '@aws-sdk/credential-provider-login@3.972.76': + resolution: {integrity: sha512-LVixwOnEJfrrfKHeZjBA8pIMTZjNDq8ak8VpcoWUuCJDrSnBNU8POJksULMgvN089P0MXtQYH2Zs627/MK1K0g==} + engines: {node: '>=20.0.0'} + + '@aws-sdk/credential-provider-node@3.972.80': + resolution: {integrity: sha512-bE2qh8ww4iClO1jHsBXdOE8FUgzDbdxbyorNjSCoPSkQd51k3jODItuPZfuwcLHZqDXsH+bI4AMHhqtuyR7mSg==} + engines: {node: '>=20.0.0'} + + '@aws-sdk/credential-provider-process@3.972.69': + resolution: {integrity: sha512-9kpTNdZTrcqXTfhxM7fgl9Z68ek3Fu5oe3Yf+A/pJGibEqpgZxz2tSY7SinmyCIU2PJ+ygY4FPoBBnLpocMtrQ==} + engines: {node: '>=20.0.0'} + + '@aws-sdk/credential-provider-sso@3.973.13': + resolution: {integrity: sha512-Oc81qauMPzUoTnAS2YKpNwY6sY/LUyQTEeaf6yP197WMxkEBQfcKLR1MFpD7+pNTubXnfkH6gwpji+Gc7iyD2Q==} + engines: {node: '>=20.0.0'} + + '@aws-sdk/credential-provider-web-identity@3.972.75': + resolution: {integrity: sha512-YPN6uoGDgjjjeVFZrcOeCJqmB6zpXoeeNgIjqe+DexJaWqdjVfCCe+VAZwli9Z2h8KhFW8oxkO39emQ1tyz/Mw==} + engines: {node: '>=20.0.0'} + + '@aws-sdk/middleware-sdk-s3@3.972.74': + resolution: {integrity: sha512-2lzoV2z2QO5KJZYGOCnIZ1WVQgzMECvwuzr1xb034a++8QW4U4eGrmC2u4yg1xvNv4TLL/Uv5DLyuAiw0b9z7Q==} + engines: {node: '>=20.0.0'} + + '@aws-sdk/nested-clients@3.997.43': + resolution: {integrity: sha512-bit+VpqWNyi3wHxFoTsTliNXimCSL2r2OeDTm7ZrG+YsTZ2D7ofDJ6r/t9PVBn80i6/v0X2h9Tgw6QP2MAKfPw==} + engines: {node: '>=20.0.0'} + + '@aws-sdk/signature-v4-multi-region@3.996.45': + resolution: {integrity: sha512-bBuyztukzXq6plzFGHAWiQt0QXo+HL8b8lX5cFTzkez/74PtS1c0qPFCIVuHkyoT+miH2qOjAcm1/yoro2ESPA==} + engines: {node: '>=20.0.0'} + + '@aws-sdk/token-providers@3.1111.0': + resolution: {integrity: sha512-JfljgoVtl+s3Qy21n9a7Z48uCQaOXcN74KJ3TEQfPoB293GrXFSt6HSQJF1sTZ8c/5QedEvd3NjJQMO4u9qa5A==} + engines: {node: '>=20.0.0'} + + '@aws-sdk/types@3.974.4': + resolution: {integrity: sha512-dSFDNG00MEz0/xl5gxL62giLd1iYyJsTxZ1I1DOj6lC+bbgLB4TRsYClJg3b62dhXT1uATzsTNXPnC+33EJV3A==} + engines: {node: '>=20.0.0'} + + '@aws-sdk/xml-builder@3.972.39': + resolution: {integrity: sha512-FTti8DS5MMWXNUWiRwXAJeYS+0GHHiMy0+7XOhcwk63ILHmfS2UFy2z/HNpZCSOJJ3P3dnWY6hfYNW3DF0nXUA==} + engines: {node: '>=20.0.0'} + + '@aws/lambda-invoke-store@0.3.0': + resolution: {integrity: sha512-sl4Bm6yiMNYrZKkqqDFWN0UfnWhlS8ivKxrYl+6t0gCLrqr8y3B2IqZZbFRkfaVVp7C/baApyh71P+LeE1A2sQ==} + engines: {node: '>=18.0.0'} + '@babel/code-frame@7.29.7': resolution: {integrity: sha512-Aup7aUOfpbAUg2ROOJN6Iw5f9DMBlzu0mIkm/malLQFN/YQgO48wCj0Kxa3sEHJvPVFg7siR+qRInwXd2qhQKw==} engines: {node: '>=6.9.0'} @@ -132,6 +213,9 @@ packages: resolution: {integrity: sha512-Vj1jF3cPfxg7OAfoI7QnVKLoILlm2JF9pnVHrX8qx7AHMiYWT+NDAA7jChlNgRS4WTLc/fD1lXLmPixluj+3Gg==} engines: {node: '>=6.9.0'} + '@emnapi/runtime@1.11.3': + resolution: {integrity: sha512-Xz4Tpyki7XyrpbUK1jR1AhdAdaXyhhY4lZ3neLodmhpuWfy2PAQN5B46sAiU4liOXGLkHypn/qU+jvfWSCYYLA==} + '@eslint-community/eslint-utils@4.10.1': resolution: {integrity: sha512-cuadcxVFE8sDK6iWJbs8Sn0av2Nrh2QSGQhVlBW9AaAHqHwjWsZHT8LJ4hFGPh7ASBV2deFdM7H/DPjulmh8rg==} engines: {node: ^12.22.0 || ^14.17.0 || >=16.0.0} @@ -191,6 +275,168 @@ packages: resolution: {integrity: sha512-bV0Tgo9K4hfPCek+aMAn81RppFKv2ySDQeMoSZuvTASywNTnVJCArCZE2FWqpvIatKu7VMRLWlR1EazvVhDyhQ==} engines: {node: '>=18.18'} + '@img/colour@1.1.0': + resolution: {integrity: sha512-Td76q7j57o/tLVdgS746cYARfSyxk8iEfRxewL9h4OMzYhbW4TAcppl0mT4eyqXddh6L/jwoM75mo7ixa/pCeQ==} + engines: {node: '>=18'} + + '@img/sharp-darwin-arm64@0.35.3': + resolution: {integrity: sha512-RMnFX7YQsMoh7lWfcM4NEHHymBX/rLuKNPVM84XE9ONPcaSCDgE7CHIHpSgPcO2xcRthgBy1HfNO319mwhIAkg==} + engines: {node: '>=20.9.0'} + cpu: [arm64] + os: [darwin] + + '@img/sharp-darwin-x64@0.35.3': + resolution: {integrity: sha512-Xo+5uFBtLN0BKqieTxiFzFPQAUlBbbH5iBKyRX/z1JrbnYsHTfKJnUfL8+p2TPXr1pXqao4eeL4Rl144uDpK9w==} + engines: {node: '>=20.9.0'} + cpu: [x64] + os: [darwin] + + '@img/sharp-freebsd-wasm32@0.35.3': + resolution: {integrity: sha512-lUxcqWIj2wMQ9BrwNjngcr1gWUr5xgaGThBRqPPalIC2n67Cqj1uPh8NnA/ZhAg8hUbKl+kVHKwgUIwe6ZYPrg==} + engines: {node: '>=20.9.0'} + os: [freebsd] + + '@img/sharp-libvips-darwin-arm64@1.3.2': + resolution: {integrity: sha512-9J6ypZFpQBj4YnePGoq/S38w6nz+vqg5WZLrLGY4YuSemdMq47GMLBPO42MzwdGwpg/agZ7xzZcFHa48xlywfg==} + cpu: [arm64] + os: [darwin] + + '@img/sharp-libvips-darwin-x64@1.3.2': + resolution: {integrity: sha512-m2pW1n6cns9VaubNwsZ+c3CRYjxNQWgJ5gPlnL1nbBcpkBvFm6SCFN5o0psFHI8w9n11NKhFkeEDns98tiqbEw==} + cpu: [x64] + os: [darwin] + + '@img/sharp-libvips-linux-arm64@1.3.2': + resolution: {integrity: sha512-dqVSFynCox4C/J8kT16V7SIFAns0IjgLwkvYT7p8LQVmJ5OS5b6tI9IGflxTeuBS//zXeFIUbwt5dwxyZ17cnA==} + cpu: [arm64] + os: [linux] + libc: [glibc] + + '@img/sharp-libvips-linux-arm@1.3.2': + resolution: {integrity: sha512-1eMLzy92I4J6rmi4mAT8yC3HxOtniyGELlzGbNMLLeqe052ahFQ0h6LFq+lh5DsDIdYViIDst08abvSbcEdLXQ==} + cpu: [arm] + os: [linux] + libc: [glibc] + + '@img/sharp-libvips-linux-ppc64@1.3.2': + resolution: {integrity: sha512-3z0NHDxD6n5I9gc05U1eW1AyRm+Gznzq3naMrthPNqE6oYykcogW0l/jfpJdjYnuNl8R7yI9pNbE1XiUeyq0Aw==} + cpu: [ppc64] + os: [linux] + libc: [glibc] + + '@img/sharp-libvips-linux-riscv64@1.3.2': + resolution: {integrity: sha512-bsb4rI+NldGOsXuej2r8OdSS8+zXDVaCWxyWrcv6kneTOlgAHtZABRzBBCwdsPiD90J4myNJuHpg6kA20ImW/w==} + cpu: [riscv64] + os: [linux] + libc: [glibc] + + '@img/sharp-libvips-linux-s390x@1.3.2': + resolution: {integrity: sha512-/ABshyj8gCpyIrNXnHn4LorDJ0HHm1VhXPBlxZ8zAtfVPAaSafXPGn+sUSIRiwaSBy0mmFjSjiXI5mkcwdChKQ==} + cpu: [s390x] + os: [linux] + libc: [glibc] + + '@img/sharp-libvips-linux-x64@1.3.2': + resolution: {integrity: sha512-ITPEtgffGJ0S6G9dRyw/366tJQqFRcHWPHhC+Stpg3Z8AEMrDrTr2lhdz4f/Y/HMbRh//7Z5mBzEpVdi62Oc3w==} + cpu: [x64] + os: [linux] + libc: [glibc] + + '@img/sharp-libvips-linuxmusl-arm64@1.3.2': + resolution: {integrity: sha512-zE9EdiUzUmg5mDT5a1rk5fYJ6GWPloTwWBYDS14naqHsL+EaMpDj1AWnpLgh3u0YCORv2Tt50wrcrpYqkP97Kw==} + cpu: [arm64] + os: [linux] + libc: [musl] + + '@img/sharp-libvips-linuxmusl-x64@1.3.2': + resolution: {integrity: sha512-m0lrLiUt+lBYnCFr8qV/65yMR4E/c7/wf78I5eKTdkEakFAlZ9QlzEM3QIhhAwVeUhLAHLcCq7a7Vszq/oFNZQ==} + cpu: [x64] + os: [linux] + libc: [musl] + + '@img/sharp-linux-arm64@0.35.3': + resolution: {integrity: sha512-QgKDspHPnrU+GQ55XPhGwyhC8acLVOOSyAvo1oVfFmrIXLkDNmGWzAfDZ4xK8oSA1qBQrALcHX0G5UZni/SuFQ==} + engines: {node: '>=20.9.0'} + cpu: [arm64] + os: [linux] + libc: [glibc] + + '@img/sharp-linux-arm@0.35.3': + resolution: {integrity: sha512-affVWCTLooy8TSxbDx2qkzuDeaWLNVBA+P//FNBirHsXpP2fuBhk5AuboYUnrDnzoXes8GFjpTx0SBFOCRg+FA==} + engines: {node: '>=20.9.0'} + cpu: [arm] + os: [linux] + libc: [glibc] + + '@img/sharp-linux-ppc64@0.35.3': + resolution: {integrity: sha512-sMd8rDxmpLOwv/7N44klFjOD5DUO7FLdjiXDI0hoxYaf7Ar262dQIEkosE98bps+5HPLtp/EvNqeqQtOycP/IA==} + engines: {node: '>=20.9.0'} + cpu: [ppc64] + os: [linux] + libc: [glibc] + + '@img/sharp-linux-riscv64@0.35.3': + resolution: {integrity: sha512-0Eob78yjlYPfL5vMNWAW55l3R9Y6BQS/gOfe0ZcP9mEz9ohhKSt4im1hayiknXgf8AWrFqMvJcKIdmLmEe7yeQ==} + engines: {node: '>=20.9.0'} + cpu: [riscv64] + os: [linux] + libc: [glibc] + + '@img/sharp-linux-s390x@0.35.3': + resolution: {integrity: sha512-KgAxQ0DxpNOq1rG2t5cgTgShJFGSuU7XO45cqC+1NVOuZnP6tlgZRuSYOfNupGkHID0o3cJOsw4DVeJpMovcGw==} + engines: {node: '>=20.9.0'} + cpu: [s390x] + os: [linux] + libc: [glibc] + + '@img/sharp-linux-x64@0.35.3': + resolution: {integrity: sha512-8pqvxubL2PGdhlPy6GLqzDYMUjyRmKAwKHYKixpdJYBUK7PJ0C029XdsnpFIdgRZG68fZiGdHVWcKPvtiPB4cA==} + engines: {node: '>=20.9.0'} + cpu: [x64] + os: [linux] + libc: [glibc] + + '@img/sharp-linuxmusl-arm64@0.35.3': + resolution: {integrity: sha512-Vz0iQjzzcSX3HCbfwFfCSG/9SCIqyO0mH2sXyiHaAYfBk0cRsCWXRyQYX0ovCK/PAQBbTzQ0dsPQHh5MAFL59w==} + engines: {node: '>=20.9.0'} + cpu: [arm64] + os: [linux] + libc: [musl] + + '@img/sharp-linuxmusl-x64@0.35.3': + resolution: {integrity: sha512-6O1NPKcDVj9QEdg7Hx549EX8U0rp6yXQERqru6yRN7fGBn32UvIRJUlWnk+8xDCiG76hXVBbX82NZ/ZKr0euIg==} + engines: {node: '>=20.9.0'} + cpu: [x64] + os: [linux] + libc: [musl] + + '@img/sharp-wasm32@0.35.3': + resolution: {integrity: sha512-cZ0XkcYGpHZkqW6iCkqTcmUC0CD9DhD5d/qeZlZkfRBn6GnHniZXLUo5+9xw8Iv76YE6LQFN9YNBlKREcCG76w==} + engines: {node: '>=20.9.0'} + + '@img/sharp-webcontainers-wasm32@0.35.3': + resolution: {integrity: sha512-2rnq7bX3NzeR2T4YWgz8qiG4h3TSdMe+vN1iQXpJleSJ3SM5zQ8Fy2SyyXAWlbxpEZ2Y+Z4u1BePgJEYbSy80Q==} + engines: {node: '>=20.9.0'} + cpu: [wasm32] + + '@img/sharp-win32-arm64@0.35.3': + resolution: {integrity: sha512-4bPwFdMbeC4JQ8L8LOyWp6nsHcboP5fxkp6iPOXz2Vg49R42TuMs2whkJ5OAP4/Ul035qOzy0AecOF9VOscn4w==} + engines: {node: '>=20.9.0'} + cpu: [arm64] + os: [win32] + + '@img/sharp-win32-ia32@0.35.3': + resolution: {integrity: sha512-r53mXsBN6lFUDiST764SvgwUdHAqM4rPAiDzAmf4fLoB6X/rkfyTrLCg6+g17wJJiCmB3JYgHuUldCWUIRFSXw==} + engines: {node: ^20.9.0} + cpu: [ia32] + os: [win32] + + '@img/sharp-win32-x64@0.35.3': + resolution: {integrity: sha512-D4y1vNeZrIIJCN+uHaWVtH86B+aCrdMYYjicy9pXHvbGZeGYLLSd3wdVuC37FxVXlU1ARsk84eKWfWMXGYEqvA==} + engines: {node: '>=20.9.0'} + cpu: [x64] + os: [win32] + '@jridgewell/gen-mapping@0.3.13': resolution: {integrity: sha512-2kkt/7niJ6MgEPxF0bYdQ6etZaA+fQvDcLKckhy1yIQOzaoKjBBjSj63/aLVjYE3qhRt5dvM+uUyfCg6UKCBbA==} @@ -303,6 +549,30 @@ packages: '@rolldown/pluginutils@1.0.1': resolution: {integrity: sha512-2j9bGt5Jh8hj+vPtgzPtl72j0yRxHAyumoo6TNfAjsLB04UtpSvPbPcDcBMxz7n+9CYB0c1GxQFxYRg2jimqGw==} + '@smithy/core@3.33.2': + resolution: {integrity: sha512-CUGXpnPkVdjUCbix+83sWLW9VFgQOm44MDOx/ihITJMAnOZKvL8YYIc7DR9pP/tZ8CIRvMiON/TucvygqbHO3w==} + engines: {node: '>=18.0.0'} + + '@smithy/credential-provider-imds@4.5.2': + resolution: {integrity: sha512-A9uSdn72ozbRUSit0eib0TW7nXuNPlaeM0zcGkJ+nE6tFcSDbnmtwoxbTCFBukVQcszDAyvsd7+rTduPTXpygg==} + engines: {node: '>=18.0.0'} + + '@smithy/fetch-http-handler@5.7.2': + resolution: {integrity: sha512-nZyWTmSpJEXl6VtWVMBJve/7x12DZu6sIX1z1a+ZMaHlQQRs9Zpu6NbTe/gmxYXVRpkjxyDYpZ5gx2IM6f/Wkw==} + engines: {node: '>=18.0.0'} + + '@smithy/node-http-handler@4.11.2': + resolution: {integrity: sha512-avwAh9HM3h2lcfjvP3zYIZGf+XVgLQ91wOJ2qoFbNpW1UZeZb33aGlhTZvtkANHfcGhJroRY64525OjfgOg30g==} + engines: {node: '>=18.0.0'} + + '@smithy/signature-v4@5.7.2': + resolution: {integrity: sha512-P7Ki6px6OOrxVtx8K7nLmyx4SlXUW/uTKDdMG44UHefmPGSRMBKe2v+TM59WdLcpUIrBrnuCsIqiM2MbsZjmhw==} + engines: {node: '>=18.0.0'} + + '@smithy/types@4.17.2': + resolution: {integrity: sha512-FOKpVZob9MPTn2znRzGrnsMHv7BOsKVw3XiP/cOyYLDVZ9qKp4nifIiSCuUU/fIj5Vu0UOAxCFr+qRAtG0NUkA==} + engines: {node: '>=18.0.0'} + '@socket.io/component-emitter@3.1.2': resolution: {integrity: sha512-9BCxFwvbGg/RsZK9tjXd8s4UcwR0MWeFQ1XEKIQVVvAGJyINdrqKMcTRyLoK8Rse1GjzLV9cwjWV1olXRWEXVA==} @@ -366,6 +636,9 @@ packages: ajv@6.15.0: resolution: {integrity: sha512-fgFx7Hfoq60ytK2c7DhnF8jIvzYgOMxfugjLOSMHjLIPgenqa7S7oaagATUq99mV6IYvN2tRmC0wnTYX6iPbMw==} + append-field@1.0.0: + resolution: {integrity: sha512-klpgFSWLW1ZEs8svjfb7g4qWY0YS5imI82dTg+QahUvJ8YqAY0P10Uk8tTyh9ZGuYEZEMaeJYCF5BFuX552hsw==} + balanced-match@4.0.4: resolution: {integrity: sha512-BLrgEcRTwX2o6gGxGOCNyMvGSp35YofuYzw9h1IMTRmKqttAZZVU67bdb9Pr2vUHA8+j3i2tJfjO6C6+4myGTA==} engines: {node: 18 || 20 || >=22} @@ -387,6 +660,9 @@ packages: resolution: {integrity: sha512-oP5VkATKlNwcgvxi0vM0p/D3n2C3EReYVX+DNYs5TjZFn/oQt2j+4sVJtSMr18pdRr8wjTcBl6LoV+FUwzPmNA==} engines: {node: '>=18'} + bowser@2.14.1: + resolution: {integrity: sha512-tzPjzCxygAKWFOJP011oxFHs57HzIhOEracIgAePE4pqB3LikALKnSzUyU4MGs9/iCEUuHlAJTjTc5M+u7YEGg==} + brace-expansion@5.0.6: resolution: {integrity: sha512-kLpxurY4Z4r9sgMsyG0Z9uzsBlgiU/EFKhj/h91/8yHu0edo7XuixOIH3VcJ8kkxs6/jPzoI6U9Vj3WqbMQ94g==} engines: {node: 18 || 20 || >=22} @@ -396,6 +672,13 @@ packages: engines: {node: ^6 || ^7 || ^8 || ^9 || ^10 || ^11 || ^12 || >=13.7} hasBin: true + buffer-from@1.1.2: + resolution: {integrity: sha512-E+XQCRwSbaaiChtv6k6Dwgc+bx+Bs6vuKJHHl5kox/BaKbhiXzqQOwK4cO22yElGp2OCmjwVhT3HmxgyPGnJfQ==} + + busboy@1.6.0: + resolution: {integrity: sha512-8SFQbg/0hQ9xy3UNTB0YEnsNBbWfhf7RtnzpL7TkBiTBRfrQ9Fxcnz7VJsleJpyp6rVLvXiuORqjlHi5q+PYuA==} + engines: {node: '>=10.16.0'} + bytes@3.1.2: resolution: {integrity: sha512-/Nf7TyzTx6S3yRJObOAV7956r8cr2+Oj8AC5dt8wSP3BQAoeX58NoHyCU8P8zGkNXStjTSi6fzO6F0pBdcYbEg==} engines: {node: '>= 0.8'} @@ -411,6 +694,10 @@ packages: caniuse-lite@1.0.30001809: resolution: {integrity: sha512-xxWVywk6a6Arlk+hymeycyn/VgqEfLDxupvhH/xiY5SJ/18kmi9o6MiO320DCUzypORHLtvh0I4i04tUhCNHNQ==} + concat-stream@2.0.0: + resolution: {integrity: sha512-MWufYdFw53ccGjCA+Ol7XJYpAlW6/prSMzuPOTRnJGcGzuhLn4Scrz7qf6o8bROZ514ltazcIFJZevcfbo0x7A==} + engines: {'0': node >= 6.0} + content-disposition@1.1.0: resolution: {integrity: sha512-5jRCH9Z/+DRP7rkvY83B+yGIGX96OYdJmzngqnw2SBSxqCFPd0w2km3s5iawpGX8krnwSGmF0FW5Nhr0Hfai3g==} engines: {node: '>=18'} @@ -820,6 +1107,10 @@ packages: resolution: {integrity: sha512-/IXtbwEk5HTPyEwyKX6hGkYXxM9nbj64B+ilVJnC/R6B0pH5G4V3b0pVbL7DBj4tkhBAppbQUlf6F6Xl9LHu1g==} engines: {node: '>= 0.4'} + media-typer@0.3.0: + resolution: {integrity: sha512-dq+qelQ9akHpcOl/gUVRTxVIOkAJ1wR3QAvb4RsVjS8oVoFjDGTc679wJYmUmknUF5HwMLOgb5O+a3KxfWapPQ==} + engines: {node: '>= 0.6'} + media-typer@1.1.0: resolution: {integrity: sha512-aisnrDP4GNe06UcKFnV5bfMNPBUw4jsLGaWwWfnH3v02GnBuXX2MCVn5RbrWo0j3pczUilYblq7fQ7Nw2t5XKw==} engines: {node: '>= 0.8'} @@ -848,9 +1139,20 @@ packages: resolution: {integrity: sha512-MULkVLfKGYDFYejP07QOurDLLQpcjk7Fw+7jXS2R2czRQzR56yHRveU5NDJEOviH+hETZKSkIk5c+T23GjFUMg==} engines: {node: 18 || 20 || >=22} + minimist@1.2.8: + resolution: {integrity: sha512-2yyAR8qBkN3YuheJanUpWC5U3bb5osDywNB8RzDVlDwDHbocAJveqqj1u8+SVD7jkWT4yvsHCpWqqWqAxb0zCA==} + + mkdirp@0.5.6: + resolution: {integrity: sha512-FP+p8RB8OWpF3YZBCrP5gtADmtXApB5AMLn+vdyA+PyxCjrCs00mjyUozssO33cwDeT3wNGdLxJ5M//YqtHAJw==} + hasBin: true + ms@2.1.3: resolution: {integrity: sha512-6FlzubTLZG3J2a/NVCAleEhjzq5oxgHyaCU9yYXvcLsvoVaHJq/s5xXI6/XXP6tz7R9xAOtHnSO/tXtF3WRTlA==} + multer@2.0.2: + resolution: {integrity: sha512-u7f2xaZ/UG8oLXHvtF/oWTRvT44p9ecwBBqTwgJVq0+4BW1g8OW01TyMEGWBHbyMOYVHXslaut7qEQ1meATXgw==} + engines: {node: '>= 10.16.0'} + nanoid@3.3.18: resolution: {integrity: sha512-DTg4MJbGMWkfi6VZFdNt2/caMbQy4Ou+Op/hJQvGEWcnVfoA1QA+xzRKAzw9jD6+GVOOeYr/mIcuDSdug6F6+w==} engines: {node: ^10 || ^12 || ^13.7 || ^14 || >=15.0.1} @@ -961,6 +1263,10 @@ packages: resolution: {integrity: sha512-PWaYA1L/q9u2u7xYQi+Y3L3Yfnie7XyLeaJICV1MGD6LprsBxcAqGjYyr0eY3p+QdsA+x/Irkt4Qif8D63+Sbw==} engines: {node: '>=0.10.0'} + readable-stream@3.6.2: + resolution: {integrity: sha512-9u/sniCrY3D5WdsERHzHE4G2YCXqoG5FTHUiCC4SIbr6XcLZBY05ya9EKjYek9O5xOAwjGq+1JdGBAS7Q9ScoA==} + engines: {node: '>= 6'} + rolldown@1.2.4: resolution: {integrity: sha512-rSr7irW0K7QRWzjdJXqZowkcRdDtjRduh43rBltnVKd0VFq839l1lJoDvGJb6gl7+4rTTCrPWu+YfujUL8Ug7w==} engines: {node: ^20.19.0 || >=22.12.0} @@ -970,6 +1276,9 @@ packages: resolution: {integrity: sha512-nLTrUKm2UyiL7rlhapu/Zl45FwNgkZGaCpZbIHajDYgwlJCOzLSk+cIPAnsEqV955GjILJnKbdQC1nVPz+gAYQ==} engines: {node: '>= 18'} + safe-buffer@5.2.1: + resolution: {integrity: sha512-rp3So07KcdmmKbGvgaNxQSJr7bGVSVk5S9Eq1F+ppbRo70+YeaDxkw5Dd8NPN+GD6bjnYm2VuPuCXmpuYvmCXQ==} + safer-buffer@2.1.2: resolution: {integrity: sha512-YZo3K82SD7Riyi0E1EQPojLz7kpepnSQI9IyPbHHg1XXXevb5dJI7tpyN2ADxGcQbHG7vcyRHk0cbwqcQriUtg==} @@ -980,6 +1289,11 @@ packages: resolution: {integrity: sha512-BR7VvDCVHO+q2xBEWskxS6DJE1qRnb7DxzUrogb71CWoSficBxYsiAGd+Kl0mmq/MprG9yArRkyrQxTO6XjMzA==} hasBin: true + semver@7.8.5: + resolution: {integrity: sha512-Y7/KDsb8LjooZpwaqGyulO6DQlksgCncchHGk+sZIY4SBvUocMBEFH5Ur1fI4dV+Jvl0w6cjvucaIi40puRioA==} + engines: {node: '>=10'} + hasBin: true + send@1.2.1: resolution: {integrity: sha512-1gnZf7DFcoIcajTjTwjwuDjzuz4PPcY2StKPlsGAQ1+YH20IRVrBaXSWmdjowTJ6u8Rc01PoYOGHXfP1mYcZNQ==} engines: {node: '>= 18'} @@ -991,6 +1305,15 @@ packages: setprototypeof@1.2.0: resolution: {integrity: sha512-E5LDX7Wrp85Kil5bhZv46j8jOeboKq5JMmYM3gVGdGH8xFpPWXUMsNrlODCrkoxMEeNi/XZIwuRvY4XNwYMJpw==} + sharp@0.35.3: + resolution: {integrity: sha512-ej0zVHuZGHCiABXcNxeYhpRnPNPAcvbG8RMdBAhDAxLKkCRVSpK3Iyu7qbqw3JMzoj0REeM6f3tJLtVwl0023Q==} + engines: {node: '>=20.9.0'} + peerDependencies: + '@types/node': '*' + peerDependenciesMeta: + '@types/node': + optional: true + shebang-command@2.0.0: resolution: {integrity: sha512-kHxr2zZpYtdmrN1qDjrrX/Z1rR1kG8Dx+gkpK1G4eXmvXswmcE1hTWBWYUzlraYw1/yZp6YuDY77YtvbN0dmDA==} engines: {node: '>=8'} @@ -1038,6 +1361,13 @@ packages: resolution: {integrity: sha512-DvEy55V3DB7uknRo+4iOGT5fP1slR8wQohVdknigZPMpMstaKJQWhwiYBACJE3Ul2pTnATihhBYnRhZQHGBiRw==} engines: {node: '>= 0.8'} + streamsearch@1.1.0: + resolution: {integrity: sha512-Mcc5wHehp9aXz1ax6bZUyY5afg9u2rv5cqQI3mRrYkGC8rW2hM02jWuwjtL++LS5qinSyhj2QfLyNsuc+VsExg==} + engines: {node: '>=10.0.0'} + + string_decoder@1.3.0: + resolution: {integrity: sha512-hkRX8U1WjJFd8LsDJ2yQ/wWWxaopEsABU1XfkM8A+j0+85JAGppt16cr1Whg6KIbb4okU6Mql6BOj+uup/wKeA==} + tinyglobby@0.2.17: resolution: {integrity: sha512-wXR/dYpcqKmfWpEdZjiKJOwCNFndD0DMnrW/cYjVGttEkBfVgcLFHoNrlj47mjOVic9yyNu65alsgF4NQyTa2g==} engines: {node: '>=12.0.0'} @@ -1046,14 +1376,24 @@ packages: resolution: {integrity: sha512-o5sSPKEkg/DIQNmH43V0/uerLrpzVedkUh8tGNvaeXpfpuwjKenlSox/2O/BTlZUtEe+JG7s5YhEz608PlAHRA==} engines: {node: '>=0.6'} + tslib@2.8.1: + resolution: {integrity: sha512-oJFu94HQb+KVduSUQL7wnpmqnfmLsOA/nAh6b6EH0wCEoK0/mPeXU6c3wKDV83MkOuHPRHtSXKKU99IBazS/2w==} + type-check@0.4.0: resolution: {integrity: sha512-XleUoc9uwGXqjWwXaUTZAmzMcFZ5858QA2vvx1Ur5xIcixXIP+8LnFDgRplU30us6teqdlskFfu+ae4K79Ooew==} engines: {node: '>= 0.8.0'} + type-is@1.6.18: + resolution: {integrity: sha512-TkRKr9sUTxEH8MdfuCSP7VizJyzRNMjj2J2do2Jr3Kym598JVdEksuzPQCnlFPW4ky9Q+iA+ma9BGm06XQBy8g==} + engines: {node: '>= 0.6'} + type-is@2.1.0: resolution: {integrity: sha512-faYHw0anBbc/kWF3zFTEnxSFOAGUX9GFbOBthvDdLsIlEoWOFOtS0zgCiQYwIskL9iGXZL3kAXD8OoZ4GmMATA==} engines: {node: '>= 18'} + typedarray@0.0.6: + resolution: {integrity: sha512-/aCDEGatGvZ2BIk+HmLf4ifCJFwvKFNb9/JeZPMulfgFracn9QFcAf5GO8B/mweUjSoblS5In0cWhqpfs/5PQA==} + undici-types@7.24.6: resolution: {integrity: sha512-WRNW+sJgj5OBN4/0JpHFqtqzhpbnV0GuB+OozA9gCL7a993SmU+1JBZCzLNxYsbMfIeDL+lTsphD5jN5N+n0zg==} @@ -1070,6 +1410,9 @@ packages: uri-js@4.4.1: resolution: {integrity: sha512-7rKUyy33Q1yc98pQ1DAmLtwX109F7TIfWlW1Ydo8Wl1ii1SeHieeh0HHfPeL2fMXK6z0s8ecKs9frCuLJvndBg==} + util-deprecate@1.0.2: + resolution: {integrity: sha512-EPD5q1uXyFxJpCrLnCc1nHnq3gOa6DZBocAIiI2TaSCA7VCJ1UJDMagCzIkXNsUYfD1daK//LTEQ8xiIbrHtcw==} + vary@1.1.2: resolution: {integrity: sha512-BNGbWLfd0eUPabhkXUVm0j8uuvREyTh5ovRa/dyow/BqAbZJyC+5fU+IzQOzmAKzYqYRAISoRhdQr3eIZ/PXqg==} engines: {node: '>= 0.8'} @@ -1157,6 +1500,10 @@ packages: resolution: {integrity: sha512-TEU+nJVUUnA4CYJFLvK5X9AOeH4KvDvhIfm0vV1GaQRtchnG0hgK5p8hw/xjv8cunWYCsiPCSDzObPyhEwq3KQ==} engines: {node: '>=0.4.0'} + xtend@4.0.2: + resolution: {integrity: sha512-LKYU1iAXJXUgAXn9URjiu+MWhyUXHsvfp7mcuYm9dSUKK0/CjtrUwFAxD82/mCWbtLsGjFIad0wIsod4zrTAEQ==} + engines: {node: '>=0.4'} + yallist@3.1.1: resolution: {integrity: sha512-a4UGQaWPH59mOXUYnAG2ewncQS4i4F43Tv3JoAM+s2VDAmS9NsK8GpDMLrCHPksFT7h3K6TOoUNn2pb7RoXx4g==} @@ -1175,6 +1522,171 @@ packages: snapshots: + '@aws-sdk/checksums@3.1000.28': + dependencies: + '@aws-sdk/core': 3.977.8 + '@aws-sdk/types': 3.974.4 + '@smithy/core': 3.33.2 + '@smithy/types': 4.17.2 + tslib: 2.8.1 + + '@aws-sdk/client-s3@3.1101.0': + dependencies: + '@aws-sdk/checksums': 3.1000.28 + '@aws-sdk/core': 3.977.8 + '@aws-sdk/credential-provider-node': 3.972.80 + '@aws-sdk/middleware-sdk-s3': 3.972.74 + '@aws-sdk/signature-v4-multi-region': 3.996.45 + '@aws-sdk/types': 3.974.4 + '@smithy/core': 3.33.2 + '@smithy/fetch-http-handler': 5.7.2 + '@smithy/node-http-handler': 4.11.2 + '@smithy/types': 4.17.2 + tslib: 2.8.1 + + '@aws-sdk/core@3.977.8': + dependencies: + '@aws-sdk/types': 3.974.4 + '@aws-sdk/xml-builder': 3.972.39 + '@aws/lambda-invoke-store': 0.3.0 + '@smithy/core': 3.33.2 + '@smithy/signature-v4': 5.7.2 + '@smithy/types': 4.17.2 + bowser: 2.14.1 + tslib: 2.8.1 + + '@aws-sdk/credential-provider-env@3.972.69': + dependencies: + '@aws-sdk/core': 3.977.8 + '@aws-sdk/types': 3.974.4 + '@smithy/core': 3.33.2 + '@smithy/types': 4.17.2 + tslib: 2.8.1 + + '@aws-sdk/credential-provider-http@3.972.71': + dependencies: + '@aws-sdk/core': 3.977.8 + '@aws-sdk/types': 3.974.4 + '@smithy/core': 3.33.2 + '@smithy/fetch-http-handler': 5.7.2 + '@smithy/node-http-handler': 4.11.2 + '@smithy/types': 4.17.2 + tslib: 2.8.1 + + '@aws-sdk/credential-provider-ini@3.973.14': + dependencies: + '@aws-sdk/core': 3.977.8 + '@aws-sdk/credential-provider-env': 3.972.69 + '@aws-sdk/credential-provider-http': 3.972.71 + '@aws-sdk/credential-provider-login': 3.972.76 + '@aws-sdk/credential-provider-process': 3.972.69 + '@aws-sdk/credential-provider-sso': 3.973.13 + '@aws-sdk/credential-provider-web-identity': 3.972.75 + '@aws-sdk/nested-clients': 3.997.43 + '@aws-sdk/types': 3.974.4 + '@smithy/core': 3.33.2 + '@smithy/credential-provider-imds': 4.5.2 + '@smithy/types': 4.17.2 + tslib: 2.8.1 + + '@aws-sdk/credential-provider-login@3.972.76': + dependencies: + '@aws-sdk/core': 3.977.8 + '@aws-sdk/nested-clients': 3.997.43 + '@aws-sdk/types': 3.974.4 + '@smithy/core': 3.33.2 + '@smithy/types': 4.17.2 + tslib: 2.8.1 + + '@aws-sdk/credential-provider-node@3.972.80': + dependencies: + '@aws-sdk/credential-provider-env': 3.972.69 + '@aws-sdk/credential-provider-http': 3.972.71 + '@aws-sdk/credential-provider-ini': 3.973.14 + '@aws-sdk/credential-provider-process': 3.972.69 + '@aws-sdk/credential-provider-sso': 3.973.13 + '@aws-sdk/credential-provider-web-identity': 3.972.75 + '@aws-sdk/types': 3.974.4 + '@smithy/core': 3.33.2 + '@smithy/credential-provider-imds': 4.5.2 + '@smithy/types': 4.17.2 + tslib: 2.8.1 + + '@aws-sdk/credential-provider-process@3.972.69': + dependencies: + '@aws-sdk/core': 3.977.8 + '@aws-sdk/types': 3.974.4 + '@smithy/core': 3.33.2 + '@smithy/types': 4.17.2 + tslib: 2.8.1 + + '@aws-sdk/credential-provider-sso@3.973.13': + dependencies: + '@aws-sdk/core': 3.977.8 + '@aws-sdk/nested-clients': 3.997.43 + '@aws-sdk/token-providers': 3.1111.0 + '@aws-sdk/types': 3.974.4 + '@smithy/core': 3.33.2 + '@smithy/types': 4.17.2 + tslib: 2.8.1 + + '@aws-sdk/credential-provider-web-identity@3.972.75': + dependencies: + '@aws-sdk/core': 3.977.8 + '@aws-sdk/nested-clients': 3.997.43 + '@aws-sdk/types': 3.974.4 + '@smithy/core': 3.33.2 + '@smithy/types': 4.17.2 + tslib: 2.8.1 + + '@aws-sdk/middleware-sdk-s3@3.972.74': + dependencies: + '@aws-sdk/core': 3.977.8 + '@aws-sdk/signature-v4-multi-region': 3.996.45 + '@aws-sdk/types': 3.974.4 + '@smithy/core': 3.33.2 + '@smithy/types': 4.17.2 + tslib: 2.8.1 + + '@aws-sdk/nested-clients@3.997.43': + dependencies: + '@aws-sdk/core': 3.977.8 + '@aws-sdk/signature-v4-multi-region': 3.996.45 + '@aws-sdk/types': 3.974.4 + '@smithy/core': 3.33.2 + '@smithy/fetch-http-handler': 5.7.2 + '@smithy/node-http-handler': 4.11.2 + '@smithy/types': 4.17.2 + tslib: 2.8.1 + + '@aws-sdk/signature-v4-multi-region@3.996.45': + dependencies: + '@aws-sdk/types': 3.974.4 + '@smithy/signature-v4': 5.7.2 + '@smithy/types': 4.17.2 + tslib: 2.8.1 + + '@aws-sdk/token-providers@3.1111.0': + dependencies: + '@aws-sdk/core': 3.977.8 + '@aws-sdk/nested-clients': 3.997.43 + '@aws-sdk/types': 3.974.4 + '@smithy/core': 3.33.2 + '@smithy/types': 4.17.2 + tslib: 2.8.1 + + '@aws-sdk/types@3.974.4': + dependencies: + '@smithy/types': 4.17.2 + tslib: 2.8.1 + + '@aws-sdk/xml-builder@3.972.39': + dependencies: + '@smithy/types': 4.17.2 + tslib: 2.8.1 + + '@aws/lambda-invoke-store@0.3.0': {} + '@babel/code-frame@7.29.7': dependencies: '@babel/helper-validator-identifier': 7.29.7 @@ -1275,6 +1787,11 @@ snapshots: '@babel/helper-string-parser': 7.29.7 '@babel/helper-validator-identifier': 7.29.7 + '@emnapi/runtime@1.11.3': + dependencies: + tslib: 2.8.1 + optional: true + '@eslint-community/eslint-utils@4.10.1(eslint@10.8.1)': dependencies: eslint: 10.8.1 @@ -1325,6 +1842,112 @@ snapshots: '@humanwhocodes/retry@0.4.3': {} + '@img/colour@1.1.0': {} + + '@img/sharp-darwin-arm64@0.35.3': + optionalDependencies: + '@img/sharp-libvips-darwin-arm64': 1.3.2 + optional: true + + '@img/sharp-darwin-x64@0.35.3': + optionalDependencies: + '@img/sharp-libvips-darwin-x64': 1.3.2 + optional: true + + '@img/sharp-freebsd-wasm32@0.35.3': + dependencies: + '@img/sharp-wasm32': 0.35.3 + optional: true + + '@img/sharp-libvips-darwin-arm64@1.3.2': + optional: true + + '@img/sharp-libvips-darwin-x64@1.3.2': + optional: true + + '@img/sharp-libvips-linux-arm64@1.3.2': + optional: true + + '@img/sharp-libvips-linux-arm@1.3.2': + optional: true + + '@img/sharp-libvips-linux-ppc64@1.3.2': + optional: true + + '@img/sharp-libvips-linux-riscv64@1.3.2': + optional: true + + '@img/sharp-libvips-linux-s390x@1.3.2': + optional: true + + '@img/sharp-libvips-linux-x64@1.3.2': + optional: true + + '@img/sharp-libvips-linuxmusl-arm64@1.3.2': + optional: true + + '@img/sharp-libvips-linuxmusl-x64@1.3.2': + optional: true + + '@img/sharp-linux-arm64@0.35.3': + optionalDependencies: + '@img/sharp-libvips-linux-arm64': 1.3.2 + optional: true + + '@img/sharp-linux-arm@0.35.3': + optionalDependencies: + '@img/sharp-libvips-linux-arm': 1.3.2 + optional: true + + '@img/sharp-linux-ppc64@0.35.3': + optionalDependencies: + '@img/sharp-libvips-linux-ppc64': 1.3.2 + optional: true + + '@img/sharp-linux-riscv64@0.35.3': + optionalDependencies: + '@img/sharp-libvips-linux-riscv64': 1.3.2 + optional: true + + '@img/sharp-linux-s390x@0.35.3': + optionalDependencies: + '@img/sharp-libvips-linux-s390x': 1.3.2 + optional: true + + '@img/sharp-linux-x64@0.35.3': + optionalDependencies: + '@img/sharp-libvips-linux-x64': 1.3.2 + optional: true + + '@img/sharp-linuxmusl-arm64@0.35.3': + optionalDependencies: + '@img/sharp-libvips-linuxmusl-arm64': 1.3.2 + optional: true + + '@img/sharp-linuxmusl-x64@0.35.3': + optionalDependencies: + '@img/sharp-libvips-linuxmusl-x64': 1.3.2 + optional: true + + '@img/sharp-wasm32@0.35.3': + dependencies: + '@emnapi/runtime': 1.11.3 + optional: true + + '@img/sharp-webcontainers-wasm32@0.35.3': + dependencies: + '@img/sharp-wasm32': 0.35.3 + optional: true + + '@img/sharp-win32-arm64@0.35.3': + optional: true + + '@img/sharp-win32-ia32@0.35.3': + optional: true + + '@img/sharp-win32-x64@0.35.3': + optional: true + '@jridgewell/gen-mapping@0.3.13': dependencies: '@jridgewell/sourcemap-codec': 1.5.5 @@ -1390,6 +2013,39 @@ snapshots: '@rolldown/pluginutils@1.0.1': {} + '@smithy/core@3.33.2': + dependencies: + '@smithy/types': 4.17.2 + tslib: 2.8.1 + + '@smithy/credential-provider-imds@4.5.2': + dependencies: + '@smithy/core': 3.33.2 + '@smithy/types': 4.17.2 + tslib: 2.8.1 + + '@smithy/fetch-http-handler@5.7.2': + dependencies: + '@smithy/core': 3.33.2 + '@smithy/types': 4.17.2 + tslib: 2.8.1 + + '@smithy/node-http-handler@4.11.2': + dependencies: + '@smithy/core': 3.33.2 + '@smithy/types': 4.17.2 + tslib: 2.8.1 + + '@smithy/signature-v4@5.7.2': + dependencies: + '@smithy/core': 3.33.2 + '@smithy/types': 4.17.2 + tslib: 2.8.1 + + '@smithy/types@4.17.2': + dependencies: + tslib: 2.8.1 + '@socket.io/component-emitter@3.1.2': {} '@types/cors@2.8.19': @@ -1446,6 +2102,8 @@ snapshots: json-schema-traverse: 0.4.1 uri-js: 4.4.1 + append-field@1.0.0: {} + balanced-match@4.0.4: {} base64id@2.0.0: {} @@ -1470,6 +2128,8 @@ snapshots: transitivePeerDependencies: - supports-color + bowser@2.14.1: {} + brace-expansion@5.0.6: dependencies: balanced-match: 4.0.4 @@ -1482,6 +2142,12 @@ snapshots: node-releases: 2.0.53 update-browserslist-db: 1.3.1(browserslist@4.28.8) + buffer-from@1.1.2: {} + + busboy@1.6.0: + dependencies: + streamsearch: 1.1.0 + bytes@3.1.2: {} call-bind-apply-helpers@1.0.2: @@ -1496,6 +2162,13 @@ snapshots: caniuse-lite@1.0.30001809: {} + concat-stream@2.0.0: + dependencies: + buffer-from: 1.1.2 + inherits: 2.0.4 + readable-stream: 3.6.2 + typedarray: 0.0.6 + content-disposition@1.1.0: {} content-type@1.0.5: {} @@ -1900,6 +2573,8 @@ snapshots: math-intrinsics@1.1.0: {} + media-typer@0.3.0: {} + media-typer@1.1.0: {} merge-descriptors@2.0.0: {} @@ -1920,8 +2595,24 @@ snapshots: dependencies: brace-expansion: 5.0.6 + minimist@1.2.8: {} + + mkdirp@0.5.6: + dependencies: + minimist: 1.2.8 + ms@2.1.3: {} + multer@2.0.2: + dependencies: + append-field: 1.0.0 + busboy: 1.6.0 + concat-stream: 2.0.0 + mkdirp: 0.5.6 + object-assign: 4.1.1 + type-is: 1.6.18 + xtend: 4.0.2 + nanoid@3.3.18: {} natural-compare@1.4.0: {} @@ -2010,6 +2701,12 @@ snapshots: react@19.2.8: {} + readable-stream@3.6.2: + dependencies: + inherits: 2.0.4 + string_decoder: 1.3.0 + util-deprecate: 1.0.2 + rolldown@1.2.4: dependencies: '@oxc-project/types': 0.144.0 @@ -2040,12 +2737,16 @@ snapshots: transitivePeerDependencies: - supports-color + safe-buffer@5.2.1: {} + safer-buffer@2.1.2: {} scheduler@0.27.0: {} semver@6.3.1: {} + semver@7.8.5: {} + send@1.2.1: dependencies: debug: 4.4.3 @@ -2073,6 +2774,38 @@ snapshots: setprototypeof@1.2.0: {} + sharp@0.35.3: + dependencies: + '@img/colour': 1.1.0 + detect-libc: 2.1.2 + semver: 7.8.5 + optionalDependencies: + '@img/sharp-darwin-arm64': 0.35.3 + '@img/sharp-darwin-x64': 0.35.3 + '@img/sharp-freebsd-wasm32': 0.35.3 + '@img/sharp-libvips-darwin-arm64': 1.3.2 + '@img/sharp-libvips-darwin-x64': 1.3.2 + '@img/sharp-libvips-linux-arm': 1.3.2 + '@img/sharp-libvips-linux-arm64': 1.3.2 + '@img/sharp-libvips-linux-ppc64': 1.3.2 + '@img/sharp-libvips-linux-riscv64': 1.3.2 + '@img/sharp-libvips-linux-s390x': 1.3.2 + '@img/sharp-libvips-linux-x64': 1.3.2 + '@img/sharp-libvips-linuxmusl-arm64': 1.3.2 + '@img/sharp-libvips-linuxmusl-x64': 1.3.2 + '@img/sharp-linux-arm': 0.35.3 + '@img/sharp-linux-arm64': 0.35.3 + '@img/sharp-linux-ppc64': 0.35.3 + '@img/sharp-linux-riscv64': 0.35.3 + '@img/sharp-linux-s390x': 0.35.3 + '@img/sharp-linux-x64': 0.35.3 + '@img/sharp-linuxmusl-arm64': 0.35.3 + '@img/sharp-linuxmusl-x64': 0.35.3 + '@img/sharp-webcontainers-wasm32': 0.35.3 + '@img/sharp-win32-arm64': 0.35.3 + '@img/sharp-win32-ia32': 0.35.3 + '@img/sharp-win32-x64': 0.35.3 + shebang-command@2.0.0: dependencies: shebang-regex: 3.0.0 @@ -2152,6 +2885,12 @@ snapshots: statuses@2.0.2: {} + streamsearch@1.1.0: {} + + string_decoder@1.3.0: + dependencies: + safe-buffer: 5.2.1 + tinyglobby@0.2.17: dependencies: fdir: 6.5.0(picomatch@4.0.5) @@ -2159,16 +2898,25 @@ snapshots: toidentifier@1.0.1: {} + tslib@2.8.1: {} + type-check@0.4.0: dependencies: prelude-ls: 1.2.1 + type-is@1.6.18: + dependencies: + media-typer: 0.3.0 + mime-types: 2.1.35 + type-is@2.1.0: dependencies: content-type: 2.0.0 media-typer: 1.1.0 mime-types: 3.0.2 + typedarray@0.0.6: {} + undici-types@7.24.6: {} unpipe@1.0.0: {} @@ -2183,6 +2931,8 @@ snapshots: dependencies: punycode: 2.3.1 + util-deprecate@1.0.2: {} + vary@1.1.2: {} vite@8.2.1(@types/node@25.8.0): @@ -2210,6 +2960,8 @@ snapshots: xmlhttprequest-ssl@2.1.2: {} + xtend@4.0.2: {} + yallist@3.1.1: {} yocto-queue@0.1.0: {} diff --git a/pnpm-workspace.yaml b/pnpm-workspace.yaml index 06f8a7e..ecf4019 100644 --- a/pnpm-workspace.yaml +++ b/pnpm-workspace.yaml @@ -4,3 +4,4 @@ packages: onlyBuiltDependencies: - better-sqlite3 + - sharp diff --git a/server/package.json b/server/package.json index 3d00562..fa72769 100644 --- a/server/package.json +++ b/server/package.json @@ -13,9 +13,12 @@ "node": ">=22.13.0" }, "dependencies": { + "@aws-sdk/client-s3": "3.1101.0", "better-sqlite3": "13.0.2", "dotenv": "^16.5.0", "express": "^5.2.1", + "multer": "2.0.2", + "sharp": "0.35.3", "socket.io": "^4.8.3" } } diff --git a/server/src/app.js b/server/src/app.js index 6bca135..a40e5c4 100644 --- a/server/src/app.js +++ b/server/src/app.js @@ -3,12 +3,27 @@ const { createDetailedHealthRouter, createHealthRouter, } = require("./health/routes"); +const { createAssetRouter } = require("./assets/routes"); +const { createMediaRouter } = require("./assets/media-routes"); +const { createAssetService } = require("./assets/service"); const { createSettingsRouter } = require("./settings/routes"); const { createSettingsService } = require("./settings/service"); -function createApp({ config, database, lifecycle, migrations }) { +function createApp({ + config, + database, + imageProcessor, + lifecycle, + migrations, + storage, +}) { const app = express(); const settingsService = createSettingsService(database); + const assetService = createAssetService({ + database, + imageProcessor, + storage, + }); app.disable("x-powered-by"); app.set("trust proxy", config.trustProxy); @@ -19,13 +34,14 @@ function createApp({ config, database, lifecycle, migrations }) { }); app.use( "/health", - createHealthRouter({ database, lifecycle, migrations }), + createHealthRouter({ database, lifecycle, migrations, storage }), ); const detailedHealthRouterOptions = { config, database, lifecycle, migrations, + storage, }; app.use( "/api/health", @@ -36,6 +52,11 @@ function createApp({ config, database, lifecycle, migrations }) { createDetailedHealthRouter(detailedHealthRouterOptions), ); app.use("/api/settings", createSettingsRouter(settingsService)); + app.use( + "/api/assets", + createAssetRouter({ assetService, processingConfig: config.processing }), + ); + app.use("/media", createMediaRouter(assetService)); app.use((request, response) => { response.status(404).json({ diff --git a/server/src/assets/errors.js b/server/src/assets/errors.js new file mode 100644 index 0000000..df0b88f --- /dev/null +++ b/server/src/assets/errors.js @@ -0,0 +1,23 @@ +class AssetError extends Error { + constructor(code, message, { statusCode = 400, details, cause } = {}) { + super(message, cause ? { cause } : undefined); + this.name = "AssetError"; + this.code = code; + this.statusCode = statusCode; + if (details !== undefined) { + this.details = details; + } + } +} + +function assetNotFound(assetId) { + return new AssetError("ASSET_NOT_FOUND", "Library asset not found", { + statusCode: 404, + details: { assetId }, + }); +} + +module.exports = { + AssetError, + assetNotFound, +}; diff --git a/server/src/assets/media-routes.js b/server/src/assets/media-routes.js new file mode 100644 index 0000000..83dfa2b --- /dev/null +++ b/server/src/assets/media-routes.js @@ -0,0 +1,49 @@ +const express = require("express"); +const { sendImmutableMedia } = require("./streaming"); + +function createMediaRouter(assetService) { + const router = express.Router(); + + async function sendMedia(request, response, next, kind) { + const version = request.params.version; + if (!version.endsWith(".webp")) { + response.status(404).json({ + error: { + code: "MEDIA_NOT_FOUND", + message: "Asset media version not found", + }, + }); + return; + } + + try { + const descriptor = assetService.mediaDescriptor( + request.params.assetId, + kind, + version.slice(0, -".webp".length), + ); + await sendImmutableMedia(assetService, response, descriptor); + } catch (error) { + if (response.headersSent) { + response.destroy(error); + return; + } + next(error); + } + } + + router.get("/assets/:assetId/display/:version", (request, response, next) => + sendMedia(request, response, next, "display"), + ); + router.get( + "/assets/:assetId/thumbnail/:version", + (request, response, next) => + sendMedia(request, response, next, "thumbnail"), + ); + + return router; +} + +module.exports = { + createMediaRouter, +}; diff --git a/server/src/assets/multipart.js b/server/src/assets/multipart.js new file mode 100644 index 0000000..0cd4f35 --- /dev/null +++ b/server/src/assets/multipart.js @@ -0,0 +1,235 @@ +const crypto = require("node:crypto"); +const fs = require("node:fs"); +const path = require("node:path"); +const multer = require("multer"); +const { AssetError } = require("./errors"); + +const ALLOWED_FILE_FIELDS = new Set(["file", "files"]); +const ALLOWED_TEXT_FIELDS = new Set(["display_name", "displayName"]); +const MAX_FIELD_BYTES = 4 * 1024; +const MAX_ORIGINAL_FILENAME_LENGTH = 255; +const TEMPORARY_UPLOAD_PATHS = Symbol("temporaryUploadPaths"); +const TEMP_STORAGE_ERROR_CODES = new Set([ + "EACCES", + "EEXIST", + "EIO", + "EMFILE", + "ENFILE", + "ENOSPC", + "ENOTDIR", + "EPERM", + "EROFS", +]); + +function safeOriginalFilename(value) { + const normalized = String(value || "") + .replaceAll("\\", "/") + .replace(/[\u0000-\u001f\u007f]/g, ""); + const leafName = path.posix.basename(normalized); + const bounded = [...leafName].slice(0, MAX_ORIGINAL_FILENAME_LENGTH).join(""); + + if (!bounded.trim()) { + throw new AssetError( + "INVALID_ORIGINAL_FILENAME", + "Uploaded files must include a usable filename", + ); + } + return bounded; +} + +function mapMulterError(error, config) { + if (!(error instanceof multer.MulterError)) { + if (TEMP_STORAGE_ERROR_CODES.has(error?.code)) { + return new AssetError( + "UPLOAD_TEMP_STORAGE_UNAVAILABLE", + "Temporary upload storage is unavailable", + { statusCode: 503, cause: error }, + ); + } + return new AssetError( + "INVALID_MULTIPART_UPLOAD", + "The multipart upload could not be read", + { cause: error }, + ); + } + + if (error.code === "LIMIT_FILE_SIZE") { + return new AssetError( + "UPLOAD_FILE_TOO_LARGE", + "An uploaded file exceeds the configured size limit", + { + statusCode: 413, + details: { maxFileBytes: config.maxFileBytes }, + }, + ); + } + if (error.code === "LIMIT_FILE_COUNT") { + return new AssetError( + "UPLOAD_FILE_COUNT_EXCEEDED", + "The upload contains too many files", + { + statusCode: 413, + details: { maxFiles: config.maxFiles }, + }, + ); + } + if ( + [ + "LIMIT_FIELD_COUNT", + "LIMIT_FIELD_KEY", + "LIMIT_FIELD_VALUE", + "LIMIT_PART_COUNT", + "LIMIT_UNEXPECTED_FILE", + ].includes(error.code) + ) { + return new AssetError( + "INVALID_MULTIPART_UPLOAD", + "The multipart upload contains too many or invalid parts", + { details: { reason: error.code } }, + ); + } + + return new AssetError( + "INVALID_MULTIPART_UPLOAD", + "The multipart upload could not be read", + { details: { reason: error.code } }, + ); +} + +function createTemporaryStorage(tempDirectory) { + return multer.diskStorage({ + destination(request, file, callback) { + fs.mkdir(tempDirectory, { recursive: true, mode: 0o700 }, (error) => { + callback(error, tempDirectory); + }); + }, + filename(request, file, callback) { + const temporaryName = crypto.randomUUID(); + if (!request[TEMPORARY_UPLOAD_PATHS]) { + request[TEMPORARY_UPLOAD_PATHS] = []; + } + request[TEMPORARY_UPLOAD_PATHS].push( + path.join(tempDirectory, temporaryName), + ); + callback(null, temporaryName); + }, + }); +} + +function temporaryUploadFiles(request) { + const paths = new Set(request[TEMPORARY_UPLOAD_PATHS] || []); + for (const file of Array.isArray(request.files) ? request.files : []) { + if (file?.path) paths.add(file.path); + } + return [...paths].map((temporaryPath) => ({ path: temporaryPath })); +} + +function createMultipartUpload(config) { + const middleware = multer({ + storage: createTemporaryStorage(config.tempDirectory), + limits: { + fileSize: config.maxFileBytes, + files: config.maxFiles, + fields: 2, + parts: config.maxFiles + 2, + fieldNameSize: 64, + fieldSize: MAX_FIELD_BYTES, + headerPairs: 100, + }, + }).any(); + + return function receiveAssetUpload(request, response) { + return new Promise((resolve, reject) => { + middleware(request, response, (error) => { + if (error) { + reject(mapMulterError(error, config)); + return; + } + + const files = Array.isArray(request.files) ? request.files : []; + try { + if (files.length === 0) { + throw new AssetError( + "UPLOAD_FILE_REQUIRED", + "At least one image file is required", + ); + } + + const unexpectedFile = files.find( + (file) => !ALLOWED_FILE_FIELDS.has(file.fieldname), + ); + if (unexpectedFile) { + throw new AssetError( + "UNEXPECTED_UPLOAD_FIELD", + "Image files must use the file or files multipart field", + { details: { field: unexpectedFile.fieldname } }, + ); + } + + const body = request.body || {}; + const unknownFields = Object.keys(body).filter( + (name) => !ALLOWED_TEXT_FIELDS.has(name), + ); + if (unknownFields.length > 0) { + throw new AssetError( + "UNEXPECTED_UPLOAD_FIELD", + "The multipart upload contains unsupported fields", + { details: { fields: unknownFields } }, + ); + } + + if ( + Object.hasOwn(body, "display_name") && + Object.hasOwn(body, "displayName") + ) { + throw new AssetError( + "DUPLICATE_DISPLAY_NAME_FIELD", + "Provide only one display name field", + ); + } + + const displayName = body.display_name ?? body.displayName ?? null; + if (displayName !== null && files.length !== 1) { + throw new AssetError( + "DISPLAY_NAME_REQUIRES_SINGLE_FILE", + "An explicit display name can be used only with one uploaded file", + ); + } + + resolve({ + displayName, + files: files.map((file) => ({ + path: file.path, + byteSize: file.size, + originalFilename: safeOriginalFilename(file.originalname), + suppliedContentType: file.mimetype, + })), + }); + } catch (validationError) { + reject(validationError); + } + }); + }); + }; +} + +async function cleanupTemporaryFiles(files = []) { + await Promise.all( + files.map(async (file) => { + if (!file?.path) return; + try { + await fs.promises.rm(file.path, { force: true }); + } catch { + // Temporary cleanup is best-effort and must not mask the API result. + } + }), + ); +} + +module.exports = { + cleanupTemporaryFiles, + createMultipartUpload, + mapMulterError, + safeOriginalFilename, + temporaryUploadFiles, +}; diff --git a/server/src/assets/routes.js b/server/src/assets/routes.js new file mode 100644 index 0000000..17712ec --- /dev/null +++ b/server/src/assets/routes.js @@ -0,0 +1,102 @@ +const express = require("express"); +const { + cleanupTemporaryFiles, + createMultipartUpload, + temporaryUploadFiles, +} = require("./multipart"); +const { sendOriginal } = require("./streaming"); + +function createAssetRouter({ assetService, processingConfig }) { + const router = express.Router(); + const receiveUpload = createMultipartUpload(processingConfig); + + router.get("/", (request, response, next) => { + try { + response.json({ assets: assetService.list() }); + } catch (error) { + next(error); + } + }); + + router.post("/", async (request, response, next) => { + const cleanupAfterAbort = () => { + void cleanupTemporaryFiles(temporaryUploadFiles(request)); + }; + request.once("aborted", cleanupAfterAbort); + + try { + const upload = await receiveUpload(request, response); + const results = await assetService.upload(upload.files, { + displayName: upload.displayName, + }); + const createdCount = results.filter((result) => result.created).length; + + if (results.length === 1) { + response.status(createdCount > 0 ? 201 : 200).json(results[0]); + return; + } + + response.status(createdCount > 0 ? 201 : 200).json({ + results, + createdCount, + duplicateCount: results.length - createdCount, + }); + } catch (error) { + next(error); + } finally { + request.off("aborted", cleanupAfterAbort); + await cleanupTemporaryFiles(temporaryUploadFiles(request)); + } + }); + + router.get("/:assetId/references", (request, response, next) => { + try { + response.json(assetService.references(request.params.assetId)); + } catch (error) { + next(error); + } + }); + + router.get("/:assetId/original", async (request, response, next) => { + try { + const descriptor = assetService.originalDescriptor(request.params.assetId); + await sendOriginal(assetService, response, descriptor); + } catch (error) { + if (response.headersSent) { + response.destroy(error); + return; + } + next(error); + } + }); + + router.get("/:assetId", (request, response, next) => { + try { + response.json(assetService.get(request.params.assetId)); + } catch (error) { + next(error); + } + }); + + router.patch("/:assetId", (request, response, next) => { + try { + response.json(assetService.update(request.params.assetId, request.body)); + } catch (error) { + next(error); + } + }); + + router.delete("/:assetId", async (request, response, next) => { + try { + response.json(await assetService.remove(request.params.assetId)); + } catch (error) { + next(error); + } + }); + + return router; +} + +module.exports = { + createAssetRouter, +}; diff --git a/server/src/assets/service.js b/server/src/assets/service.js new file mode 100644 index 0000000..4a828c3 --- /dev/null +++ b/server/src/assets/service.js @@ -0,0 +1,835 @@ +const crypto = require("node:crypto"); +const fs = require("node:fs"); +const path = require("node:path"); +const { AssetError, assetNotFound } = require("./errors"); +const { StorageObjectNotFoundError } = require("../storage"); + +const MAX_DISPLAY_NAME_LENGTH = 200; +const MAX_PROCESSING_ERROR_LENGTH = 1_000; +const CONCURRENT_UPLOAD_WAIT_MS = 15_000; +const CONCURRENT_UPLOAD_POLL_MS = 50; +const SHA256_PATTERN = /^[0-9a-f]{64}$/; + +function delay(milliseconds) { + return new Promise((resolve) => setTimeout(resolve, milliseconds)); +} + +function normalizeDisplayName(value) { + if (typeof value !== "string") { + throw new AssetError( + "INVALID_ASSET_DISPLAY_NAME", + "Asset display name must be a string", + ); + } + + const normalized = value + .replace(/[\u0000-\u001f\u007f]/g, " ") + .replace(/\s+/g, " ") + .trim(); + if (!normalized) { + throw new AssetError( + "INVALID_ASSET_DISPLAY_NAME", + "Asset display name cannot be empty", + ); + } + if ([...normalized].length > MAX_DISPLAY_NAME_LENGTH) { + throw new AssetError( + "INVALID_ASSET_DISPLAY_NAME", + `Asset display name cannot exceed ${MAX_DISPLAY_NAME_LENGTH} characters`, + { details: { maxLength: MAX_DISPLAY_NAME_LENGTH } }, + ); + } + return normalized; +} + +function defaultDisplayName(originalFilename) { + const extension = path.extname(originalFilename); + const withoutExtension = extension + ? originalFilename.slice(0, -extension.length) + : originalFilename; + return normalizeDisplayName(withoutExtension); +} + +function normalizeExpiration(value) { + if (value === null) return null; + if (Number.isSafeInteger(value) && value >= 0) return value; + + if (typeof value === "string" && value.trim()) { + const timestamp = Date.parse(value); + if (Number.isSafeInteger(timestamp) && timestamp >= 0) { + return timestamp; + } + } + + throw new AssetError( + "INVALID_ASSET_EXPIRATION", + "Asset expiration must be null, an epoch-millisecond integer, or an ISO date", + ); +} + +function mediaUrls(row) { + if (row.status !== "ready") { + return { + originalUrl: null, + displayUrl: null, + thumbnailUrl: null, + }; + } + return { + originalUrl: `/api/assets/${row.id}/original`, + displayUrl: `/media/assets/${row.id}/display/${row.display_sha256}.webp`, + thumbnailUrl: `/media/assets/${row.id}/thumbnail/${row.thumbnail_sha256}.webp`, + }; +} + +function mapAsset(row) { + if (!row) return null; + const urls = mediaUrls(row); + return { + id: row.id, + displayName: row.display_name, + originalFilename: row.original_filename, + status: row.status, + processingError: row.processing_error || null, + sourceSha256: row.source_sha256, + expirationAt: row.expiration_at ?? null, + createdAt: row.created_at, + updatedAt: row.updated_at, + referenceCount: Number(row.reference_count || 0), + original: row.original_content_type + ? { + contentType: row.original_content_type, + byteSize: row.original_byte_size, + width: row.original_width, + height: row.original_height, + url: urls.originalUrl, + } + : null, + display: row.display_content_type + ? { + sha256: row.display_sha256, + contentType: row.display_content_type, + byteSize: row.display_byte_size, + width: row.display_width, + height: row.display_height, + url: urls.displayUrl, + } + : null, + thumbnail: row.thumbnail_content_type + ? { + sha256: row.thumbnail_sha256, + contentType: row.thumbnail_content_type, + byteSize: row.thumbnail_byte_size, + width: row.thumbnail_width, + height: row.thumbnail_height, + url: urls.thumbnailUrl, + } + : null, + }; +} + +async function hashFile(filename) { + const hash = crypto.createHash("sha256"); + let byteSize = 0; + for await (const chunk of fs.createReadStream(filename)) { + byteSize += chunk.length; + hash.update(chunk); + } + return { sourceSha256: hash.digest("hex"), byteSize }; +} + +function objectKeys(assetId, processed) { + return { + original: `assets/${assetId}/original/${processed.original.sha256}`, + display: `assets/${assetId}/display/${processed.display.sha256}.webp`, + thumbnail: `assets/${assetId}/thumbnail/${processed.thumbnail.sha256}.webp`, + }; +} + +function boundedProcessingError(error) { + const code = typeof error?.code === "string" ? error.code : "PROCESSING_FAILED"; + const message = + typeof error?.message === "string" && error.message + ? error.message + : "Asset processing failed"; + return `${code}: ${message}`.slice(0, MAX_PROCESSING_ERROR_LENGTH); +} + +function createAssetService({ + database, + storage, + imageProcessor, + now = Date.now, + createId = crypto.randomUUID, + wait = delay, + concurrentUploadWaitMs = CONCURRENT_UPLOAD_WAIT_MS, +}) { + const assetSelect = ` + SELECT a.*, + (SELECT COUNT(*) FROM playlist_entries pe WHERE pe.asset_id = a.id) + AS reference_count + FROM assets a + `; + const listAssets = database.prepare(`${assetSelect} ORDER BY a.created_at DESC, a.id`); + const findById = database.prepare(`${assetSelect} WHERE a.id = ?`); + const findByHash = database.prepare(`${assetSelect} WHERE a.source_sha256 = ?`); + const findByName = database.prepare(`${assetSelect} WHERE a.display_name = ? COLLATE NOCASE`); + const insertProcessing = database.prepare(` + INSERT INTO assets ( + id, source_sha256, display_name, original_filename, status, + created_at, updated_at + ) VALUES ( + @id, @sourceSha256, @displayName, @originalFilename, 'processing', + @createdAt, @updatedAt + ) + `); + const markReady = database.prepare(` + UPDATE assets + SET status = 'ready', + processing_error = NULL, + original_object_key = @originalObjectKey, + original_content_type = @originalContentType, + original_byte_size = @originalByteSize, + original_width = @originalWidth, + original_height = @originalHeight, + display_object_key = @displayObjectKey, + display_sha256 = @displaySha256, + display_content_type = @displayContentType, + display_byte_size = @displayByteSize, + display_width = @displayWidth, + display_height = @displayHeight, + thumbnail_object_key = @thumbnailObjectKey, + thumbnail_sha256 = @thumbnailSha256, + thumbnail_content_type = @thumbnailContentType, + thumbnail_byte_size = @thumbnailByteSize, + thumbnail_width = @thumbnailWidth, + thumbnail_height = @thumbnailHeight, + updated_at = @updatedAt + WHERE id = @id AND status = 'processing' + `); + const markFailed = database.prepare(` + UPDATE assets + SET status = 'failed', + processing_error = @processingError, + original_object_key = @originalObjectKey, + original_content_type = @originalContentType, + original_byte_size = @originalByteSize, + original_width = @originalWidth, + original_height = @originalHeight, + display_object_key = @displayObjectKey, + display_sha256 = @displaySha256, + display_content_type = @displayContentType, + display_byte_size = @displayByteSize, + display_width = @displayWidth, + display_height = @displayHeight, + thumbnail_object_key = @thumbnailObjectKey, + thumbnail_sha256 = @thumbnailSha256, + thumbnail_content_type = @thumbnailContentType, + thumbnail_byte_size = @thumbnailByteSize, + thumbnail_width = @thumbnailWidth, + thumbnail_height = @thumbnailHeight, + updated_at = @updatedAt + WHERE id = @id AND status = 'processing' + `); + const updateEditable = database.prepare(` + UPDATE assets + SET display_name = @displayName, + expiration_at = @expirationAt, + updated_at = @updatedAt + WHERE id = @id AND status != 'deleting' + `); + const playlistReferences = database.prepare(` + SELECT + pe.id AS entry_id, + pe.playlist_id, + p.name AS playlist_name, + pe.enabled, + pe.position, + EXISTS ( + SELECT 1 FROM playback_state active + WHERE active.active_playlist_id = pe.playlist_id + ) AS is_active_playlist, + EXISTS ( + SELECT 1 FROM playback_state current + WHERE current.current_entry_id = pe.id + ) AS is_current_normal_target + FROM playlist_entries pe + JOIN playlists p ON p.id = pe.playlist_id + WHERE pe.asset_id = ? + ORDER BY p.name COLLATE NOCASE, pe.position, pe.id + `); + const playbackReferences = database.prepare(` + SELECT + output_id, + mode, + active_playlist_id, + current_entry_id, + static_asset_id + FROM playback_state + WHERE static_asset_id = ? + OR current_entry_id IN ( + SELECT id FROM playlist_entries WHERE asset_id = ? + ) + ORDER BY output_id + `); + const markDeleting = database.prepare(` + UPDATE assets + SET status = 'deleting', updated_at = ? + WHERE id = ? AND status != 'processing' + `); + const deleteAssetRow = database.prepare( + "DELETE FROM assets WHERE id = ? AND status = 'deleting'", + ); + + const completeAsset = database.transaction((metadata) => { + const result = markReady.run(metadata); + if (result.changes !== 1) { + throw new Error("Asset processing state changed before completion"); + } + return findById.get(metadata.id); + }); + + function getRaw(assetId) { + const row = findById.get(assetId); + if (!row) throw assetNotFound(assetId); + return row; + } + + function duplicateStateError(row) { + if (row.status === "failed") { + return new AssetError( + "ASSET_PROCESSING_FAILED", + "These bytes match an asset whose processing previously failed", + { statusCode: 409, details: { asset: mapAsset(row) } }, + ); + } + if (row.status === "deleting") { + return new AssetError( + "ASSET_DELETING", + "These bytes match an asset that is being deleted", + { statusCode: 409, details: { assetId: row.id } }, + ); + } + return new AssetError( + "ASSET_PROCESSING", + "These bytes are already being processed", + { statusCode: 409, details: { assetId: row.id } }, + ); + } + + async function resolveExistingHash(row) { + if (row.status === "ready") { + return { created: false, duplicate: true, asset: mapAsset(row) }; + } + if (row.status !== "processing") { + throw duplicateStateError(row); + } + + const deadline = Date.now() + concurrentUploadWaitMs; + while (Date.now() < deadline) { + await wait(CONCURRENT_UPLOAD_POLL_MS); + const latest = findByHash.get(row.source_sha256); + if (!latest) break; + if (latest.status === "ready") { + return { created: false, duplicate: true, asset: mapAsset(latest) }; + } + if (latest.status !== "processing") { + throw duplicateStateError(latest); + } + } + throw duplicateStateError(findByHash.get(row.source_sha256) || row); + } + + function nameConflict(displayName, row = findByName.get(displayName)) { + return new AssetError( + "ASSET_NAME_CONFLICT", + "Another library asset already uses that display name", + { + statusCode: 409, + details: row ? { asset: mapAsset(row) } : { displayName }, + }, + ); + } + + function processingMetadata(id, processed, keys, error) { + return { + id, + processingError: error ? boundedProcessingError(error) : null, + originalObjectKey: keys.original, + originalContentType: processed.original.contentType, + originalByteSize: processed.original.byteSize, + originalWidth: processed.original.width, + originalHeight: processed.original.height, + displayObjectKey: keys.display, + displaySha256: processed.display.sha256, + displayContentType: processed.display.contentType, + displayByteSize: processed.display.byteSize, + displayWidth: processed.display.width, + displayHeight: processed.display.height, + thumbnailObjectKey: keys.thumbnail, + thumbnailSha256: processed.thumbnail.sha256, + thumbnailContentType: processed.thumbnail.contentType, + thumbnailByteSize: processed.thumbnail.byteSize, + thumbnailWidth: processed.thumbnail.width, + thumbnailHeight: processed.thumbnail.height, + updatedAt: now(), + }; + } + + async function cleanObjects(keys) { + const results = await Promise.allSettled( + [...new Set(Object.values(keys).filter(Boolean))].map((key) => + storage.deleteObject(key), + ), + ); + const rejected = results.find((result) => result.status === "rejected"); + if (rejected) throw rejected.reason; + } + + async function createProcessedAsset(plan) { + const id = createId(); + const timestamp = now(); + try { + insertProcessing.run({ + id, + sourceSha256: plan.sourceSha256, + displayName: plan.displayName, + originalFilename: plan.file.originalFilename, + createdAt: timestamp, + updatedAt: timestamp, + }); + } catch (error) { + const hashMatch = findByHash.get(plan.sourceSha256); + if (hashMatch) return resolveExistingHash(hashMatch); + const nameMatch = findByName.get(plan.displayName); + if (nameMatch) throw nameConflict(plan.displayName, nameMatch); + throw error; + } + + const keys = objectKeys(id, plan.processed); + try { + await storage.putObject({ + key: keys.original, + body: fs.createReadStream(plan.file.path), + contentType: plan.processed.original.contentType, + contentLength: plan.processed.original.byteSize, + metadata: { sha256: plan.processed.original.sha256 }, + }); + await storage.putObject({ + key: keys.display, + body: plan.processed.display.buffer, + contentType: plan.processed.display.contentType, + contentLength: plan.processed.display.byteSize, + metadata: { sha256: plan.processed.display.sha256 }, + }); + await storage.putObject({ + key: keys.thumbnail, + body: plan.processed.thumbnail.buffer, + contentType: plan.processed.thumbnail.contentType, + contentLength: plan.processed.thumbnail.byteSize, + metadata: { sha256: plan.processed.thumbnail.sha256 }, + }); + + const row = completeAsset(processingMetadata(id, plan.processed, keys)); + return { created: true, duplicate: false, asset: mapAsset(row) }; + } catch (error) { + const failure = processingMetadata(id, plan.processed, keys, error); + try { + markFailed.run(failure); + } catch { + // Preserve the original failure; the processing row remains diagnosable. + } + try { + await cleanObjects(keys); + } catch { + // The failed row retains deterministic keys so DELETE can retry cleanup. + } + + if (error?.statusCode) { + error.details = { ...(error.details || {}), assetId: id, retryable: true }; + } + throw error; + } + } + + async function upload(files, { displayName = null } = {}) { + if (!Array.isArray(files) || files.length === 0) { + throw new AssetError( + "UPLOAD_FILE_REQUIRED", + "At least one image file is required", + ); + } + + const plans = []; + const firstPlanByHash = new Map(); + const plannedNames = new Map(); + + for (const [index, file] of files.entries()) { + const identity = await hashFile(file.path); + if (identity.byteSize !== file.byteSize) { + throw new AssetError( + "UPLOAD_CHANGED_DURING_PROCESSING", + "An uploaded temporary file changed before processing", + { statusCode: 409 }, + ); + } + + const existing = findByHash.get(identity.sourceSha256); + if (existing) { + plans.push({ index, existing }); + continue; + } + + const priorPlan = firstPlanByHash.get(identity.sourceSha256); + if (priorPlan) { + plans.push({ index, duplicateOf: priorPlan }); + continue; + } + + const proposedName = normalizeDisplayName( + displayName ?? defaultDisplayName(file.originalFilename), + ); + const existingName = findByName.get(proposedName); + if (existingName) throw nameConflict(proposedName, existingName); + + const normalizedNameKey = proposedName.toLowerCase(); + const batchNameMatch = plannedNames.get(normalizedNameKey); + if (batchNameMatch) throw nameConflict(proposedName); + + const plan = { + index, + file, + displayName: proposedName, + sourceSha256: identity.sourceSha256, + }; + firstPlanByHash.set(identity.sourceSha256, plan); + plannedNames.set(normalizedNameKey, plan); + plans.push(plan); + } + + // Decode every new image before any database or object-storage mutation. + for (const plan of plans) { + if (plan.existing || plan.duplicateOf) continue; + plan.processed = await imageProcessor.process(plan.file.path); + if (plan.processed.original.sha256 !== plan.sourceSha256) { + throw new AssetError( + "UPLOAD_CHANGED_DURING_PROCESSING", + "An uploaded temporary file changed before processing", + { statusCode: 409 }, + ); + } + } + + const resultByPlan = new Map(); + const results = new Array(files.length); + for (const plan of plans) { + let result; + if (plan.existing) { + result = await resolveExistingHash(plan.existing); + } else if (plan.duplicateOf) { + const originalResult = resultByPlan.get(plan.duplicateOf); + result = { + created: false, + duplicate: true, + asset: originalResult.asset, + }; + } else { + result = await createProcessedAsset(plan); + resultByPlan.set(plan, result); + } + results[plan.index] = result; + } + return results; + } + + function list() { + return listAssets.all().map(mapAsset); + } + + function get(assetId) { + return mapAsset(getRaw(assetId)); + } + + function update(assetId, input) { + if (input === null || typeof input !== "object" || Array.isArray(input)) { + throw new AssetError("INVALID_ASSET_UPDATE", "Asset update must be an object"); + } + + const allowed = new Set([ + "displayName", + "display_name", + "expirationAt", + "expiration_at", + ]); + const unknown = Object.keys(input).filter((key) => !allowed.has(key)); + if (unknown.length > 0) { + throw new AssetError( + "INVALID_ASSET_UPDATE", + "Asset update contains unsupported fields", + { details: { fields: unknown } }, + ); + } + if (Object.keys(input).length === 0) { + throw new AssetError("INVALID_ASSET_UPDATE", "At least one field is required"); + } + if ( + Object.hasOwn(input, "displayName") && + Object.hasOwn(input, "display_name") + ) { + throw new AssetError("INVALID_ASSET_UPDATE", "Display name was provided twice"); + } + if ( + Object.hasOwn(input, "expirationAt") && + Object.hasOwn(input, "expiration_at") + ) { + throw new AssetError("INVALID_ASSET_UPDATE", "Expiration was provided twice"); + } + + const current = getRaw(assetId); + if (current.status === "deleting") { + throw new AssetError("ASSET_DELETING", "Deleting assets cannot be edited", { + statusCode: 409, + details: { assetId }, + }); + } + const hasDisplayName = + Object.hasOwn(input, "displayName") || Object.hasOwn(input, "display_name"); + const hasExpiration = + Object.hasOwn(input, "expirationAt") || Object.hasOwn(input, "expiration_at"); + const nextDisplayName = hasDisplayName + ? normalizeDisplayName( + Object.hasOwn(input, "displayName") + ? input.displayName + : input.display_name, + ) + : current.display_name; + const nextExpiration = hasExpiration + ? normalizeExpiration( + Object.hasOwn(input, "expirationAt") + ? input.expirationAt + : input.expiration_at, + ) + : current.expiration_at; + + if ( + nextDisplayName === current.display_name && + nextExpiration === current.expiration_at + ) { + return mapAsset(current); + } + + try { + const result = updateEditable.run({ + id: assetId, + displayName: nextDisplayName, + expirationAt: nextExpiration, + updatedAt: Math.max(now(), current.updated_at + 1), + }); + if (result.changes !== 1) { + throw new AssetError("ASSET_DELETING", "Deleting assets cannot be edited", { + statusCode: 409, + details: { assetId }, + }); + } + } catch (error) { + const nameMatch = findByName.get(nextDisplayName); + if (nameMatch && nameMatch.id !== assetId) { + throw nameConflict(nextDisplayName, nameMatch); + } + throw error; + } + return mapAsset(getRaw(assetId)); + } + + function references(assetId) { + const asset = getRaw(assetId); + const playlists = playlistReferences.all(assetId).map((row) => ({ + playlistId: row.playlist_id, + playlistName: row.playlist_name, + playlistEntryId: row.entry_id, + enabled: Boolean(row.enabled), + position: row.position, + isActivePlaylist: Boolean(row.is_active_playlist), + isCurrentNormalTarget: Boolean(row.is_current_normal_target), + })); + const playback = playbackReferences.all(assetId, assetId).map((row) => ({ + outputId: row.output_id, + mode: row.mode, + activePlaylistId: row.active_playlist_id, + currentPlaylistEntryId: row.current_entry_id, + isCurrentNormalTarget: playlists.some( + (entry) => entry.playlistEntryId === row.current_entry_id, + ), + isCurrentStaticTarget: row.static_asset_id === assetId, + })); + const currentNormalTarget = playback.some((state) => state.isCurrentNormalTarget); + const currentStaticTarget = playback.some((state) => state.isCurrentStaticTarget); + const blockerCodes = []; + if (playlists.length > 0) blockerCodes.push("PLAYLIST_REFERENCES"); + if (currentNormalTarget) blockerCodes.push("CURRENT_PLAYBACK_TARGET"); + if (currentStaticTarget) blockerCodes.push("CURRENT_STATIC_TARGET"); + if (asset.status === "processing") blockerCodes.push("ASSET_PROCESSING"); + if (asset.status === "deleting") blockerCodes.push("ASSET_DELETING"); + + return { + assetId, + playlists, + playback, + currentNormalTarget, + currentStaticTarget, + canDelete: blockerCodes.length === 0, + blockerCodes, + }; + } + + const beginDelete = database.transaction((assetId) => { + const asset = getRaw(assetId); + const referenceInfo = references(assetId); + const relationshipBlockers = referenceInfo.blockerCodes.filter((code) => + [ + "PLAYLIST_REFERENCES", + "CURRENT_PLAYBACK_TARGET", + "CURRENT_STATIC_TARGET", + ].includes(code), + ); + if (relationshipBlockers.length > 0) { + throw new AssetError( + referenceInfo.playlists.length > 0 ? "ASSET_REFERENCED" : "ASSET_IN_USE", + "The library asset is still referenced and cannot be deleted", + { + statusCode: 409, + details: { ...referenceInfo, blockerCodes: relationshipBlockers }, + }, + ); + } + if (asset.status === "processing") { + throw new AssetError( + "ASSET_PROCESSING", + "An asset cannot be deleted while it is processing", + { statusCode: 409, details: { assetId } }, + ); + } + if (asset.status !== "deleting") { + markDeleting.run(Math.max(now(), asset.updated_at + 1), assetId); + } + return findById.get(assetId); + }); + + async function remove(assetId) { + const deleting = beginDelete(assetId); + const keys = { + original: deleting.original_object_key, + display: deleting.display_object_key, + thumbnail: deleting.thumbnail_object_key, + }; + + try { + await cleanObjects(keys); + } catch (error) { + if (error?.statusCode) { + error.details = { ...(error.details || {}), assetId, retryable: true }; + } + throw error; + } + + try { + const result = deleteAssetRow.run(assetId); + if (result.changes !== 1) throw assetNotFound(assetId); + } catch (error) { + if (error?.code?.startsWith?.("SQLITE_CONSTRAINT")) { + throw new AssetError( + "ASSET_REFERENCED", + "The library asset became referenced before deletion completed", + { statusCode: 409, details: { assetId, retryable: true } }, + ); + } + throw error; + } + return { id: assetId, deleted: true }; + } + + function mediaDescriptor(assetId, kind, requestedHash) { + if (!SHA256_PATTERN.test(requestedHash)) { + throw new AssetError("MEDIA_NOT_FOUND", "Asset media version not found", { + statusCode: 404, + }); + } + const asset = getRaw(assetId); + const fields = + kind === "display" + ? { + hash: asset.display_sha256, + key: asset.display_object_key, + contentType: asset.display_content_type, + contentLength: asset.display_byte_size, + } + : { + hash: asset.thumbnail_sha256, + key: asset.thumbnail_object_key, + contentType: asset.thumbnail_content_type, + contentLength: asset.thumbnail_byte_size, + }; + if (asset.status !== "ready" || fields.hash !== requestedHash || !fields.key) { + throw new AssetError("MEDIA_NOT_FOUND", "Asset media version not found", { + statusCode: 404, + }); + } + return { asset, kind, ...fields }; + } + + function originalDescriptor(assetId) { + const asset = getRaw(assetId); + if (asset.status !== "ready" || !asset.original_object_key) { + throw new AssetError("ASSET_ORIGINAL_NOT_READY", "Asset original is not ready", { + statusCode: 409, + details: { assetId, status: asset.status }, + }); + } + return { + asset, + key: asset.original_object_key, + hash: asset.source_sha256, + contentType: asset.original_content_type, + contentLength: asset.original_byte_size, + }; + } + + async function readStoredObject(descriptor) { + try { + return await storage.getObject(descriptor.key); + } catch (error) { + if (error instanceof StorageObjectNotFoundError) { + throw new AssetError( + "STORAGE_INCONSISTENCY", + "Asset metadata exists but its stored bytes are unavailable", + { + statusCode: 503, + details: { assetId: descriptor.asset.id, retryable: true }, + cause: error, + }, + ); + } + throw error; + } + } + + return Object.freeze({ + list, + get, + upload, + update, + references, + remove, + mediaDescriptor, + originalDescriptor, + readStoredObject, + }); +} + +module.exports = { + createAssetService, + defaultDisplayName, + mapAsset, + normalizeDisplayName, + normalizeExpiration, +}; diff --git a/server/src/assets/streaming.js b/server/src/assets/streaming.js new file mode 100644 index 0000000..da43edc --- /dev/null +++ b/server/src/assets/streaming.js @@ -0,0 +1,66 @@ +const { Readable } = require("node:stream"); +const { pipeline } = require("node:stream/promises"); +const { AssetError } = require("./errors"); + +function readableBody(body) { + if (body && typeof body.pipe === "function") return body; + if (Buffer.isBuffer(body) || body instanceof Uint8Array) { + return Readable.from([body]); + } + if (body && typeof body.getReader === "function") { + return Readable.fromWeb(body); + } + throw new AssetError( + "STORAGE_INCONSISTENCY", + "Stored asset bytes could not be streamed", + { statusCode: 503 }, + ); +} + +function validateDescriptor(descriptor) { + if ( + typeof descriptor.contentType !== "string" || + !Number.isSafeInteger(descriptor.contentLength) || + descriptor.contentLength < 0 + ) { + throw new AssetError( + "STORAGE_INCONSISTENCY", + "Stored asset metadata is incomplete", + { statusCode: 503 }, + ); + } +} + +function setCommonHeaders(response, descriptor) { + response.set("Content-Type", descriptor.contentType); + response.set("Content-Length", String(descriptor.contentLength)); + response.set("X-Content-Type-Options", "nosniff"); +} + +async function storedBody(assetService, descriptor) { + const stored = await assetService.readStoredObject(descriptor); + return readableBody(stored.body); +} + +async function sendImmutableMedia(assetService, response, descriptor) { + validateDescriptor(descriptor); + const body = await storedBody(assetService, descriptor); + setCommonHeaders(response, descriptor); + response.set("Cache-Control", "public, max-age=31536000, immutable"); + response.set("ETag", `"${descriptor.hash}"`); + await pipeline(body, response); +} + +async function sendOriginal(assetService, response, descriptor) { + validateDescriptor(descriptor); + const body = await storedBody(assetService, descriptor); + response.attachment(descriptor.asset.original_filename); + setCommonHeaders(response, descriptor); + response.set("Cache-Control", "private, no-store"); + await pipeline(body, response); +} + +module.exports = { + sendImmutableMedia, + sendOriginal, +}; diff --git a/server/src/config/index.js b/server/src/config/index.js index 7f2a988..c75357b 100644 --- a/server/src/config/index.js +++ b/server/src/config/index.js @@ -156,6 +156,42 @@ function loadConfig(env = process.env) { accessKeyId: storageCredential(env, "S3_ACCESS_KEY_ID", environment), secretAccessKey: storageCredential(env, "S3_SECRET_ACCESS_KEY", environment), forcePathStyle: boolean(env, "S3_FORCE_PATH_STYLE", true), + healthProbeIntervalMs: integer( + env, + "S3_HEALTH_PROBE_INTERVAL_MS", + 30_000, + { min: 1_000, max: 300_000 }, + ), + connectionTimeoutMs: integer(env, "S3_CONNECT_TIMEOUT_MS", 2_000, { + min: 250, + max: 60_000, + }), + requestTimeoutMs: integer(env, "S3_REQUEST_TIMEOUT_MS", 5_000, { + min: 1_000, + max: 120_000, + }), + maxAttempts: integer(env, "S3_MAX_ATTEMPTS", 2, { min: 1, max: 5 }), + }), + processing: Object.freeze({ + tempDirectory: (() => { + const value = env.UPLOAD_TEMP_DIRECTORY?.trim() || "/tmp/announcements-uploads"; + if (!path.isAbsolute(value)) { + throw new ConfigError( + "UPLOAD_TEMP_DIRECTORY must be an absolute path", + "UPLOAD_TEMP_DIRECTORY", + ); + } + return value; + })(), + maxFileBytes: integer(env, "UPLOAD_MAX_FILE_BYTES", 25 * 1024 * 1024, { + min: 1, + max: 1024 * 1024 * 1024, + }), + maxFiles: integer(env, "UPLOAD_MAX_FILES", 20, { min: 1, max: 100 }), + maxInputPixels: integer(env, "IMAGE_MAX_INPUT_PIXELS", 40_000_000, { + min: 1, + max: 500_000_000, + }), }), mqtt: Object.freeze({ url: url(env, "MQTT_URL", { @@ -193,6 +229,16 @@ function redactedConfig(config) { region: config.storage.region, bucket: config.storage.bucket, forcePathStyle: config.storage.forcePathStyle, + healthProbeIntervalMs: config.storage.healthProbeIntervalMs, + connectionTimeoutMs: config.storage.connectionTimeoutMs, + requestTimeoutMs: config.storage.requestTimeoutMs, + maxAttempts: config.storage.maxAttempts, + }, + processing: { + tempDirectoryConfigured: Boolean(config.processing.tempDirectory), + maxFileBytes: config.processing.maxFileBytes, + maxFiles: config.processing.maxFiles, + maxInputPixels: config.processing.maxInputPixels, }, mqtt: { configured: Boolean(config.mqtt.url), diff --git a/server/src/health/routes.js b/server/src/health/routes.js index bef8e5b..1663fdc 100644 --- a/server/src/health/routes.js +++ b/server/src/health/routes.js @@ -14,15 +14,62 @@ function databaseHealth(database) { } } -function createHealthRouter({ database, lifecycle, migrations }) { +function storageHealth(storage) { + if (!storage) { + return { + status: "degraded", + checkedAt: null, + lastSuccessAt: null, + code: "STORAGE_NOT_INITIALIZED", + message: "Object storage has not been initialized", + }; + } + + try { + const result = storage.health(); + if (result?.status === "healthy") { + return { + status: "healthy", + checkedAt: result.checkedAt || null, + lastSuccessAt: result.lastSuccessAt || result.checkedAt || null, + }; + } + return { + status: "degraded", + checkedAt: result?.checkedAt || null, + lastSuccessAt: result?.lastSuccessAt || null, + code: + result?.code === "STORAGE_NOT_CHECKED" + ? "STORAGE_NOT_CHECKED" + : "STORAGE_UNAVAILABLE", + message: + result?.code === "STORAGE_NOT_CHECKED" + ? "Object storage has not been checked" + : "Object storage is unavailable", + }; + } catch { + // A health endpoint must remain available when its dependency is not. + } + + return { + status: "degraded", + checkedAt: null, + lastSuccessAt: null, + code: "STORAGE_UNAVAILABLE", + message: "Object storage is unavailable", + }; +} + +function createHealthRouter({ database, lifecycle, migrations, storage }) { const router = express.Router(); router.get("/live", (request, response) => { response.json({ status: "ok" }); }); - router.get("/ready", (request, response) => { + router.get("/ready", async (request, response) => { const databaseCheck = databaseHealth(database); + const storageCheck = storageHealth(storage); const migrationsCurrent = migrations.pendingVersions.length === 0; const ready = lifecycle.ready && databaseCheck.status === "ok" && migrationsCurrent; @@ -32,6 +79,7 @@ function createHealthRouter({ database, lifecycle, migrations }) { checks: { database: databaseCheck.status, migrations: migrationsCurrent ? "current" : "pending", + storage: storageCheck.status, }, }); }); @@ -39,18 +87,23 @@ function createHealthRouter({ database, lifecycle, migrations }) { return router; } -function createDetailedHealthRouter({ config, database, lifecycle, migrations }) { +function createDetailedHealthRouter({ config, database, lifecycle, migrations, storage }) { const router = express.Router(); - router.get("/", (request, response) => { + router.get("/", async (request, response) => { const databaseCheck = databaseHealth(database); + const storageCheck = storageHealth(storage); const ready = lifecycle.ready && databaseCheck.status === "ok" && migrations.pendingVersions.length === 0; const configuration = redactedConfig(config); - const status = ready ? "degraded" : "not_ready"; + const status = !ready + ? "not_ready" + : storageCheck.status === "healthy" + ? "ok" + : "degraded"; response.status(ready ? 200 : 503).json({ status, @@ -62,7 +115,7 @@ function createDetailedHealthRouter({ config, database, lifecycle, migrations }) latestSchemaVersion: migrations.latestAvailableVersion, }, storage: { - status: "not_checked", + ...storageCheck, configured: configuration.storage.configured, }, mqtt: { @@ -86,4 +139,5 @@ module.exports = { createDetailedHealthRouter, createHealthRouter, databaseHealth, + storageHealth, }; diff --git a/server/src/image-processing/index.js b/server/src/image-processing/index.js new file mode 100644 index 0000000..93d2a06 --- /dev/null +++ b/server/src/image-processing/index.js @@ -0,0 +1,330 @@ +const crypto = require("node:crypto"); +const fs = require("node:fs"); +const sharp = require("sharp"); + +const SUPPORTED_FORMATS = Object.freeze(new Set(["jpeg", "png", "webp"])); +const CONTENT_TYPES = Object.freeze({ + jpeg: "image/jpeg", + png: "image/png", + webp: "image/webp", +}); +const BLACK = Object.freeze({ r: 0, g: 0, b: 0, alpha: 1 }); +const DEFAULT_IMAGE_DERIVATIVE_OPTIONS = Object.freeze({ + display: Object.freeze({ maxWidth: 1920, maxHeight: 1080, quality: 82 }), + thumbnail: Object.freeze({ maxWidth: 480, maxHeight: 270, quality: 72 }), +}); + +class ImageProcessingError extends Error { + constructor(code, message, { statusCode = 422, details, cause } = {}) { + super(message, cause ? { cause } : undefined); + this.name = "ImageProcessingError"; + this.code = code; + this.statusCode = statusCode; + if (details !== undefined) { + this.details = details; + } + } +} + +function requireInteger(value, name, { min, max } = {}) { + if (!Number.isSafeInteger(value)) { + throw new TypeError(`${name} must be an integer`); + } + if (min !== undefined && value < min) { + throw new RangeError(`${name} must be at least ${min}`); + } + if (max !== undefined && value > max) { + throw new RangeError(`${name} must be at most ${max}`); + } + return value; +} + +function normalizeTargetOptions(target, name, defaults) { + if ( + target !== undefined && + (target === null || typeof target !== "object" || Array.isArray(target)) + ) { + throw new TypeError(`${name} image options must be an object`); + } + + const configured = { ...defaults, ...target }; + + return Object.freeze({ + maxWidth: requireInteger(configured.maxWidth, `${name}.maxWidth`, { min: 1 }), + maxHeight: requireInteger(configured.maxHeight, `${name}.maxHeight`, { min: 1 }), + quality: requireInteger(configured.quality, `${name}.quality`, { + min: 1, + max: 100, + }), + }); +} + +function normalizeOptions(options) { + if (options === null || typeof options !== "object" || Array.isArray(options)) { + throw new TypeError("Image processing options are required"); + } + + return Object.freeze({ + maxInputPixels: requireInteger( + options.maxInputPixels, + "maxInputPixels", + { min: 1 }, + ), + display: normalizeTargetOptions( + options.display, + "display", + DEFAULT_IMAGE_DERIVATIVE_OPTIONS.display, + ), + thumbnail: normalizeTargetOptions( + options.thumbnail, + "thumbnail", + DEFAULT_IMAGE_DERIVATIVE_OPTIONS.thumbnail, + ), + }); +} + +function sha256(buffer) { + return crypto.createHash("sha256").update(buffer).digest("hex"); +} + +async function hashFile(filename) { + const hash = crypto.createHash("sha256"); + let byteSize = 0; + + try { + for await (const chunk of fs.createReadStream(filename)) { + byteSize += chunk.length; + hash.update(chunk); + } + } catch (error) { + throw new ImageProcessingError( + "IMAGE_INPUT_UNREADABLE", + "The uploaded image could not be read", + { statusCode: 422, cause: error }, + ); + } + + return { byteSize, sha256: hash.digest("hex") }; +} + +async function describeInput(input) { + if (Buffer.isBuffer(input)) { + return { + sharpInput: input, + byteSize: input.length, + sha256: sha256(input), + }; + } + + if (typeof input !== "string" || input.length === 0) { + throw new ImageProcessingError( + "INVALID_IMAGE_INPUT", + "Image input must be a Buffer or temporary file path", + { statusCode: 400 }, + ); + } + + let stats; + try { + stats = await fs.promises.stat(input); + } catch (error) { + throw new ImageProcessingError( + "IMAGE_INPUT_UNREADABLE", + "The uploaded image could not be read", + { statusCode: 422, cause: error }, + ); + } + + if (!stats.isFile()) { + throw new ImageProcessingError( + "INVALID_IMAGE_INPUT", + "Image input path must identify a regular file", + { statusCode: 400 }, + ); + } + + return { + sharpInput: input, + ...(await hashFile(input)), + }; +} + +function decodeError(error, maxInputPixels) { + if (error instanceof ImageProcessingError) { + return error; + } + + if (/pixel limit/i.test(error?.message || "")) { + return new ImageProcessingError( + "IMAGE_PIXEL_LIMIT_EXCEEDED", + "The uploaded image exceeds the configured decoded-pixel limit", + { + statusCode: 413, + details: { maxInputPixels }, + cause: error, + }, + ); + } + + return new ImageProcessingError( + "MALFORMED_IMAGE", + "The uploaded file is not a valid decodable image", + { statusCode: 422, cause: error }, + ); +} + +async function readMetadata(input, maxInputPixels) { + try { + return await sharp(input, { + animated: true, + failOn: "error", + limitInputPixels: maxInputPixels, + }).metadata(); + } catch (error) { + throw decodeError(error, maxInputPixels); + } +} + +function validateMetadata(metadata) { + const pages = metadata.pages ?? 1; + + if (pages > 1) { + const hasAnimationTiming = + metadata.loop !== undefined || + (Array.isArray(metadata.delay) && metadata.delay.length > 0); + throw new ImageProcessingError( + hasAnimationTiming + ? "ANIMATED_IMAGE_NOT_SUPPORTED" + : "MULTIPAGE_IMAGE_NOT_SUPPORTED", + hasAnimationTiming + ? "Animated images are not supported" + : "Multipage images are not supported", + { statusCode: 422, details: { pages } }, + ); + } + + if (!SUPPORTED_FORMATS.has(metadata.format)) { + throw new ImageProcessingError( + "UNSUPPORTED_IMAGE_FORMAT", + "Only still JPEG, PNG, and WebP images are supported", + { + statusCode: 415, + details: { + detectedFormat: metadata.format || null, + supportedFormats: [...SUPPORTED_FORMATS], + }, + }, + ); + } + + if ( + !Number.isSafeInteger(metadata.width) || + metadata.width < 1 || + !Number.isSafeInteger(metadata.height) || + metadata.height < 1 + ) { + throw new ImageProcessingError( + "MALFORMED_IMAGE", + "The uploaded image does not contain valid dimensions", + { statusCode: 422 }, + ); + } +} + +async function generateDerivative(input, target, maxInputPixels) { + let output; + try { + output = await sharp(input, { + failOn: "error", + limitInputPixels: maxInputPixels, + }) + .autoOrient() + .flatten({ background: BLACK }) + .resize({ + width: target.maxWidth, + height: target.maxHeight, + fit: "inside", + withoutEnlargement: true, + }) + .toColourspace("srgb") + .webp({ quality: target.quality, effort: 4 }) + .toBuffer({ resolveWithObject: true }); + } catch (error) { + throw decodeError(error, maxInputPixels); + } + + return { + buffer: output.data, + sha256: sha256(output.data), + format: "webp", + contentType: CONTENT_TYPES.webp, + byteSize: output.data.length, + width: output.info.width, + height: output.info.height, + }; +} + +function originalMetadata(metadata, inputDescription) { + const orientedDimensions = metadata.autoOrient || { + width: metadata.width, + height: metadata.height, + }; + + return { + sha256: inputDescription.sha256, + format: metadata.format, + contentType: CONTENT_TYPES[metadata.format], + byteSize: inputDescription.byteSize, + // Width and height describe how the original should appear after applying + // its source orientation. The encoded dimensions remain available for + // diagnostics without implying that the archived bytes were rewritten. + width: orientedDimensions.width, + height: orientedDimensions.height, + encodedWidth: metadata.width, + encodedHeight: metadata.height, + orientation: metadata.orientation ?? null, + pages: metadata.pages ?? 1, + channels: metadata.channels, + hasAlpha: Boolean(metadata.hasAlpha), + }; +} + +function createImageProcessor(options) { + const config = normalizeOptions(options); + + async function process(input) { + const inputDescription = await describeInput(input); + const metadata = await readMetadata( + inputDescription.sharpInput, + config.maxInputPixels, + ); + validateMetadata(metadata); + + const [display, thumbnail] = await Promise.all([ + generateDerivative( + inputDescription.sharpInput, + config.display, + config.maxInputPixels, + ), + generateDerivative( + inputDescription.sharpInput, + config.thumbnail, + config.maxInputPixels, + ), + ]); + + return { + original: originalMetadata(metadata, inputDescription), + display, + thumbnail, + }; + } + + return Object.freeze({ process }); +} + +module.exports = { + DEFAULT_IMAGE_DERIVATIVE_OPTIONS, + ImageProcessingError, + createImageProcessor, +}; diff --git a/server/src/runtime.js b/server/src/runtime.js index aa55768..37b78cd 100644 --- a/server/src/runtime.js +++ b/server/src/runtime.js @@ -3,6 +3,8 @@ const { Server: SocketServer } = require("socket.io"); const { createApp } = require("./app"); const { openDatabase } = require("./database"); const { runMigrations } = require("./database/migrate"); +const { createImageProcessor } = require("./image-processing"); +const { createStorageService } = require("./storage"); function listen(httpServer, port, host) { return new Promise((resolve, reject) => { @@ -20,7 +22,12 @@ function listen(httpServer, port, host) { }); } -function createRuntime({ config, logger = console }) { +function createRuntime({ + config, + imageProcessor: suppliedImageProcessor, + logger = console, + storage: suppliedStorage, +}) { const database = openDatabase(config.database); let migrations; try { @@ -34,7 +41,32 @@ function createRuntime({ config, logger = console }) { ready: false, stopping: false, }; - const app = createApp({ config, database, lifecycle, migrations }); + let storage; + let imageProcessor; + try { + storage = suppliedStorage || createStorageService(config.storage, { logger }); + imageProcessor = + suppliedImageProcessor || + createImageProcessor({ maxInputPixels: config.processing.maxInputPixels }); + } catch (error) { + database.close(); + throw error; + } + let app; + try { + app = createApp({ + config, + database, + imageProcessor, + lifecycle, + migrations, + storage, + }); + } catch (error) { + storage.stop(); + database.close(); + throw error; + } const httpServer = http.createServer(app); const io = new SocketServer(httpServer, { serveClient: false, @@ -63,9 +95,15 @@ function createRuntime({ config, logger = console }) { .then((listeningAddress) => { address = listeningAddress; lifecycle.ready = true; + Promise.resolve() + .then(() => storage.start()) + .catch(() => { + logger.warn?.("Object storage health checks could not be started"); + }); return listeningAddress; }) .catch((error) => { + storage.stop(); database.close(); throw error; }); @@ -85,6 +123,7 @@ function createRuntime({ config, logger = console }) { if (settled) return; settled = true; clearTimeout(timeout); + storage.stop(); if (database.open) { database.close(); } @@ -119,6 +158,8 @@ function createRuntime({ config, logger = console }) { io, lifecycle, migrations, + storage, + imageProcessor, get address() { return address; }, diff --git a/server/src/storage/index.js b/server/src/storage/index.js new file mode 100644 index 0000000..36d7da3 --- /dev/null +++ b/server/src/storage/index.js @@ -0,0 +1,398 @@ +const { + CreateBucketCommand, + DeleteObjectCommand, + GetObjectCommand, + HeadBucketCommand, + HeadObjectCommand, + PutObjectCommand, + S3Client, +} = require("@aws-sdk/client-s3"); + +class StorageError extends Error { + constructor(message, { code, statusCode, cause } = {}) { + super(message, { cause }); + this.name = "StorageError"; + this.code = code || "STORAGE_ERROR"; + this.statusCode = statusCode || 500; + } +} + +class StorageUnavailableError extends StorageError { + constructor(cause) { + super("Object storage is unavailable", { + code: "STORAGE_UNAVAILABLE", + statusCode: 503, + cause, + }); + this.name = "StorageUnavailableError"; + } +} + +class StorageObjectNotFoundError extends StorageError { + constructor(key, cause) { + super("Stored object was not found", { + code: "STORAGE_OBJECT_NOT_FOUND", + statusCode: 404, + cause, + }); + this.name = "StorageObjectNotFoundError"; + this.key = key; + } +} + +function errorStatus(error) { + return error?.$metadata?.httpStatusCode || error?.statusCode; +} + +function isMissingBucket(error) { + return ( + error?.name === "NoSuchBucket" || + error?.Code === "NoSuchBucket" || + error?.code === "NoSuchBucket" || + errorStatus(error) === 404 + ); +} + +function isMissingObject(error) { + if ( + error?.name === "NoSuchBucket" || + error?.Code === "NoSuchBucket" || + error?.code === "NoSuchBucket" + ) { + return false; + } + return ( + error?.name === "NoSuchKey" || + error?.name === "NotFound" || + error?.Code === "NoSuchKey" || + error?.code === "NoSuchKey" || + errorStatus(error) === 404 + ); +} + +function isBucketCreationConflict(error) { + return ( + error?.name === "BucketAlreadyOwnedByYou" || + error?.name === "BucketAlreadyExists" || + error?.Code === "BucketAlreadyOwnedByYou" || + error?.Code === "BucketAlreadyExists" || + error?.code === "BucketAlreadyOwnedByYou" || + error?.code === "BucketAlreadyExists" + ); +} + +function isConsumerCancellation(error) { + return ( + error?.name === "AbortError" || + error?.code === "ABORT_ERR" || + error?.code === "ERR_STREAM_PREMATURE_CLOSE" + ); +} + +function assertObjectKey(key) { + if (typeof key !== "string" || key.length === 0) { + throw new TypeError("Storage object key must be a non-empty string"); + } + + const segments = key.split("/"); + if ( + key.startsWith("/") || + key.includes("\\") || + key.includes("\0") || + segments.some((segment) => segment === "." || segment === "..") + ) { + throw new TypeError("Storage object key is invalid"); + } +} + +function publicHealth(health) { + return Object.freeze({ ...health }); +} + +function createS3Client(config) { + const options = { + endpoint: config.endpoint, + region: config.region, + forcePathStyle: config.forcePathStyle, + maxAttempts: config.maxAttempts, + requestHandler: { + connectionTimeout: config.connectionTimeoutMs, + // Use an inactivity timeout rather than an absolute request deadline so + // a large but continuously progressing upload is not aborted. + socketTimeout: config.requestTimeoutMs, + }, + // Optional AWS checksums are not uniformly implemented by S3-compatible + // providers. Source and derivative SHA-256 hashes are verified by the app. + requestChecksumCalculation: "WHEN_REQUIRED", + responseChecksumValidation: "WHEN_REQUIRED", + }; + + if (config.accessKeyId && config.secretAccessKey) { + options.credentials = { + accessKeyId: config.accessKeyId, + secretAccessKey: config.secretAccessKey, + }; + } + + return new S3Client(options); +} + +function createStorageService(config, { client, logger = console, now = () => new Date() } = {}) { + const s3 = client || createS3Client(config); + let ensurePromise = null; + let healthTimer = null; + let bucketReady = false; + let issuedHealthSequence = 0; + let appliedHealthSequence = 0; + let currentHealth = { + status: "degraded", + checkedAt: null, + lastSuccessAt: null, + code: "STORAGE_NOT_CHECKED", + message: "Object storage has not been checked", + }; + + function nextHealthSequence() { + issuedHealthSequence += 1; + return issuedHealthSequence; + } + + function markHealthy(sequence) { + if (sequence < appliedHealthSequence) return publicHealth(currentHealth); + appliedHealthSequence = sequence; + bucketReady = true; + const checkedAt = now().toISOString(); + currentHealth = { + status: "healthy", + checkedAt, + lastSuccessAt: checkedAt, + }; + return publicHealth(currentHealth); + } + + function markDegraded(sequence) { + if (sequence < appliedHealthSequence) return publicHealth(currentHealth); + appliedHealthSequence = sequence; + bucketReady = false; + currentHealth = { + status: "degraded", + checkedAt: now().toISOString(), + lastSuccessAt: currentHealth.lastSuccessAt, + code: "STORAGE_UNAVAILABLE", + message: "Object storage is unavailable", + }; + return publicHealth(currentHealth); + } + + function unavailable(error, sequence) { + markDegraded(sequence); + return error instanceof StorageUnavailableError + ? error + : new StorageUnavailableError(error); + } + + async function ensureBucketOnce() { + const sequence = nextHealthSequence(); + try { + await s3.send(new HeadBucketCommand({ Bucket: config.bucket })); + markHealthy(sequence); + return { created: false }; + } catch (error) { + if (!isMissingBucket(error)) { + throw unavailable(error, sequence); + } + } + + try { + await s3.send(new CreateBucketCommand({ Bucket: config.bucket })); + markHealthy(sequence); + return { created: true }; + } catch (error) { + if (isBucketCreationConflict(error)) { + try { + await s3.send(new HeadBucketCommand({ Bucket: config.bucket })); + markHealthy(sequence); + return { created: false }; + } catch (verificationError) { + throw unavailable(verificationError, sequence); + } + } + throw unavailable(error, sequence); + } + } + + async function ensureBucket() { + if (!ensurePromise) { + ensurePromise = ensureBucketOnce().finally(() => { + ensurePromise = null; + }); + } + return ensurePromise; + } + + async function initialize() { + try { + await ensureBucket(); + } catch (error) { + logger.warn?.("Object storage is unavailable during initialization"); + } + return publicHealth(currentHealth); + } + + async function start() { + if (!healthTimer) { + healthTimer = setInterval(() => { + ensureBucket().catch(() => {}); + }, config.healthProbeIntervalMs || 30_000); + healthTimer.unref?.(); + } + return initialize(); + } + + async function probe() { + const sequence = nextHealthSequence(); + try { + await s3.send(new HeadBucketCommand({ Bucket: config.bucket })); + return markHealthy(sequence); + } catch (error) { + return markDegraded(sequence); + } + } + + async function putObject({ key, body, contentType, contentLength, metadata }) { + assertObjectKey(key); + if (!bucketReady) await ensureBucket(); + const sequence = nextHealthSequence(); + const input = { + Bucket: config.bucket, + Key: key, + Body: body, + ...(contentType ? { ContentType: contentType } : {}), + ...(contentLength !== undefined ? { ContentLength: contentLength } : {}), + ...(metadata ? { Metadata: metadata } : {}), + }; + + try { + const result = await s3.send(new PutObjectCommand(input)); + markHealthy(sequence); + return { + key, + etag: result.ETag || null, + versionId: result.VersionId || null, + }; + } catch (error) { + throw unavailable(error, sequence); + } + } + + function objectMetadata(key, result) { + return { + key, + contentType: result.ContentType || null, + contentLength: + result.ContentLength === undefined ? null : Number(result.ContentLength), + etag: result.ETag || null, + lastModified: result.LastModified || null, + metadata: result.Metadata || {}, + }; + } + + async function getObject(key) { + assertObjectKey(key); + if (!bucketReady) await ensureBucket(); + const sequence = nextHealthSequence(); + try { + const result = await s3.send( + new GetObjectCommand({ Bucket: config.bucket, Key: key }), + ); + markHealthy(sequence); + result.Body?.once?.("error", (error) => { + if (!isConsumerCancellation(error)) { + markDegraded(nextHealthSequence()); + } + }); + return { + ...objectMetadata(key, result), + body: result.Body, + }; + } catch (error) { + if (isMissingObject(error)) { + markHealthy(sequence); + throw new StorageObjectNotFoundError(key, error); + } + throw unavailable(error, sequence); + } + } + + async function headObject(key) { + assertObjectKey(key); + if (!bucketReady) await ensureBucket(); + const sequence = nextHealthSequence(); + try { + const result = await s3.send( + new HeadObjectCommand({ Bucket: config.bucket, Key: key }), + ); + markHealthy(sequence); + return objectMetadata(key, result); + } catch (error) { + if (isMissingObject(error)) { + markHealthy(sequence); + return null; + } + throw unavailable(error, sequence); + } + } + + async function objectExists(key) { + return (await headObject(key)) !== null; + } + + async function deleteObject(key) { + assertObjectKey(key); + if (!bucketReady) await ensureBucket(); + const sequence = nextHealthSequence(); + try { + await s3.send( + new DeleteObjectCommand({ Bucket: config.bucket, Key: key }), + ); + markHealthy(sequence); + } catch (error) { + if (isMissingObject(error)) { + markHealthy(sequence); + return; + } + throw unavailable(error, sequence); + } + } + + function stop() { + if (healthTimer) { + clearInterval(healthTimer); + healthTimer = null; + } + s3.destroy?.(); + } + + return Object.freeze({ + initialize, + start, + stop, + ensureBucket, + probe, + health: () => publicHealth(currentHealth), + putObject, + getObject, + headObject, + objectExists, + deleteObject, + destroy: stop, + }); +} + +module.exports = { + StorageError, + StorageObjectNotFoundError, + StorageUnavailableError, + createS3Client, + createStorageService, +}; diff --git a/server/test/assets-api.test.js b/server/test/assets-api.test.js new file mode 100644 index 0000000..335e1d2 --- /dev/null +++ b/server/test/assets-api.test.js @@ -0,0 +1,442 @@ +const test = require("node:test"); +const assert = require("node:assert/strict"); +const crypto = require("node:crypto"); +const fs = require("node:fs"); +const http = require("node:http"); +const os = require("node:os"); +const path = require("node:path"); +const { Readable } = require("node:stream"); +const sharp = require("sharp"); +const { createApp } = require("../src/app"); +const { loadConfig } = require("../src/config"); +const { openDatabase } = require("../src/database"); +const { runMigrations } = require("../src/database/migrate"); +const { createImageProcessor } = require("../src/image-processing"); +const { StorageObjectNotFoundError } = require("../src/storage"); + +async function readBody(body) { + if (Buffer.isBuffer(body)) return body; + const chunks = []; + for await (const chunk of body) chunks.push(Buffer.from(chunk)); + return Buffer.concat(chunks); +} + +function memoryStorage() { + const objects = new Map(); + return { + objects, + health() { + return { + status: "healthy", + checkedAt: "2033-05-18T03:33:20.000Z", + lastSuccessAt: "2033-05-18T03:33:20.000Z", + }; + }, + async putObject(input) { + objects.set(input.key, { + body: await readBody(input.body), + contentType: input.contentType, + metadata: input.metadata, + }); + return { key: input.key }; + }, + async getObject(key) { + const object = objects.get(key); + if (!object) throw new StorageObjectNotFoundError(key); + return { + body: Readable.from([object.body]), + contentLength: object.body.length, + contentType: object.contentType, + }; + }, + async deleteObject(key) { + objects.delete(key); + return { key, deleted: true }; + }, + }; +} + +async function createHarness(t, { maxFileBytes = 1024 * 1024 } = {}) { + const tempDirectory = fs.mkdtempSync( + path.join(os.tmpdir(), "announcements-api-uploads-"), + ); + const config = loadConfig({ + NODE_ENV: "test", + HOST: "127.0.0.1", + PORT: "0", + DATABASE_PATH: ":memory:", + APP_TIMEZONE: "America/Chicago", + S3_ENDPOINT: "http://private-seaweedfs:8333", + S3_BUCKET: "announcements-test", + UPLOAD_TEMP_DIRECTORY: tempDirectory, + UPLOAD_MAX_FILE_BYTES: String(maxFileBytes), + UPLOAD_MAX_FILES: "5", + IMAGE_MAX_INPUT_PIXELS: "1000000", + }); + const database = openDatabase({ path: ":memory:" }); + const migrations = runMigrations(database); + const storage = memoryStorage(); + const app = createApp({ + config, + database, + imageProcessor: createImageProcessor({ + maxInputPixels: config.processing.maxInputPixels, + }), + lifecycle: { ready: true }, + migrations, + storage, + }); + const server = app.listen(0, "127.0.0.1"); + await new Promise((resolve, reject) => { + server.once("listening", resolve); + server.once("error", reject); + }); + + t.after(async () => { + await new Promise((resolve) => server.close(resolve)); + database.close(); + fs.rmSync(tempDirectory, { recursive: true, force: true }); + }); + + return { + baseUrl: `http://127.0.0.1:${server.address().port}`, + database, + storage, + tempDirectory, + }; +} + +function appendFile(form, field, buffer, filename, contentType) { + form.append(field, new Blob([buffer], { type: contentType }), filename); +} + +async function jsonResponse(baseUrl, route, init) { + const response = await fetch(`${baseUrl}${route}`, init); + return { response, body: await response.json() }; +} + +async function eventuallyEmpty(directory) { + for (let attempt = 0; attempt < 50; attempt += 1) { + if (fs.readdirSync(directory).length === 0) return; + await new Promise((resolve) => setTimeout(resolve, 10)); + } + assert.deepEqual(fs.readdirSync(directory), []); +} + +async function eventuallyContainsFile(directory) { + for (let attempt = 0; attempt < 50; attempt += 1) { + if (fs.readdirSync(directory).length > 0) return; + await new Promise((resolve) => setTimeout(resolve, 10)); + } + assert.fail("The upload did not create its temporary file"); +} + +async function sampleImages() { + const source = sharp({ + create: { + width: 80, + height: 60, + channels: 4, + background: { r: 31, g: 72, b: 120, alpha: 0.5 }, + }, + }); + return { + jpeg: await source.clone().jpeg().toBuffer(), + png: await source.clone().png().toBuffer(), + webp: await source.clone().webp().toBuffer(), + }; +} + +test("asset HTTP API accepts JPEG, PNG, and WebP and serves private immutable media", async (t) => { + const harness = await createHarness(t); + const images = await sampleImages(); + const form = new FormData(); + appendFile(form, "files", images.jpeg, "welcome.jpg", "image/jpeg"); + appendFile(form, "files", images.png, "../../bulletin.png", "image/png"); + appendFile(form, "files", images.webp, "events.webp", "image/webp"); + + const uploaded = await jsonResponse(harness.baseUrl, "/api/assets", { + method: "POST", + body: form, + }); + assert.equal(uploaded.response.status, 201); + assert.equal(uploaded.body.createdCount, 3); + assert.equal(uploaded.body.duplicateCount, 0); + assert.equal(uploaded.body.results.length, 3); + assert.deepEqual( + uploaded.body.results.map((result) => result.asset.original.contentType), + ["image/jpeg", "image/png", "image/webp"], + ); + assert.equal(uploaded.body.results[1].asset.originalFilename, "bulletin.png"); + assert.doesNotMatch(JSON.stringify(uploaded.body), /objectKey|object_key|private-seaweedfs/); + assert.equal(harness.storage.objects.size, 9); + await eventuallyEmpty(harness.tempDirectory); + + const asset = uploaded.body.results[0].asset; + const listed = await jsonResponse(harness.baseUrl, "/api/assets"); + assert.equal(listed.response.status, 200); + assert.equal(listed.body.assets.length, 3); + assert.deepEqual( + (await jsonResponse(harness.baseUrl, `/api/assets/${asset.id}`)).body, + asset, + ); + + for (const media of [asset.display, asset.thumbnail]) { + const response = await fetch(`${harness.baseUrl}${media.url}`); + const body = Buffer.from(await response.arrayBuffer()); + assert.equal(response.status, 200); + assert.equal(response.headers.get("content-type"), "image/webp"); + assert.equal( + response.headers.get("cache-control"), + "public, max-age=31536000, immutable", + ); + assert.equal(response.headers.get("content-length"), String(body.length)); + assert.equal(response.headers.get("etag"), `"${media.sha256}"`); + assert.equal(crypto.createHash("sha256").update(body).digest("hex"), media.sha256); + } + + const original = await fetch(`${harness.baseUrl}${asset.original.url}`); + assert.equal(original.status, 200); + assert.equal(original.headers.get("content-type"), "image/jpeg"); + assert.equal(original.headers.get("cache-control"), "private, no-store"); + assert.match(original.headers.get("content-disposition"), /^attachment;/); + assert.deepEqual(Buffer.from(await original.arrayBuffer()), images.jpeg); + + const wrongVersion = await jsonResponse( + harness.baseUrl, + `/media/assets/${asset.id}/display/${"a".repeat(64)}.webp`, + ); + assert.equal(wrongVersion.response.status, 404); + assert.equal(wrongVersion.body.error.code, "MEDIA_NOT_FOUND"); + const arbitraryKey = await jsonResponse( + harness.baseUrl, + `/media/assets/${asset.id}/display/${encodeURIComponent("../../original/key")}.webp`, + ); + assert.equal(arbitraryKey.response.status, 404); + assert.equal(arbitraryKey.body.error.code, "MEDIA_NOT_FOUND"); + + const displayKey = [...harness.storage.objects.keys()].find( + (key) => key.includes(`assets/${asset.id}/display/`), + ); + harness.storage.objects.delete(displayKey); + const missingStoredBytes = await jsonResponse( + harness.baseUrl, + asset.display.url, + ); + assert.equal(missingStoredBytes.response.status, 503); + assert.equal(missingStoredBytes.body.error.code, "STORAGE_INCONSISTENCY"); + assert.equal( + (await jsonResponse(harness.baseUrl, "/health/live")).response.status, + 200, + ); +}); + +test("asset HTTP API reports duplicates, name conflicts, and persists editable metadata", async (t) => { + const harness = await createHarness(t); + const { png, jpeg } = await sampleImages(); + const firstForm = new FormData(); + appendFile(firstForm, "file", png, "weekly.png", "application/octet-stream"); + firstForm.append("display_name", "Weekly Announcement"); + const first = await jsonResponse(harness.baseUrl, "/api/assets", { + method: "POST", + body: firstForm, + }); + assert.equal(first.response.status, 201); + assert.equal(first.body.created, true); + const assetId = first.body.asset.id; + + const duplicateForm = new FormData(); + appendFile(duplicateForm, "file", png, "unrelated-name.png", "image/png"); + duplicateForm.append("display_name", "Ignored duplicate proposal"); + const duplicate = await jsonResponse(harness.baseUrl, "/api/assets", { + method: "POST", + body: duplicateForm, + }); + assert.equal(duplicate.response.status, 200); + assert.equal(duplicate.body.created, false); + assert.equal(duplicate.body.duplicate, true); + assert.equal(duplicate.body.asset.id, assetId); + assert.equal(harness.storage.objects.size, 3); + + const conflictForm = new FormData(); + appendFile(conflictForm, "file", jpeg, "different.jpg", "image/jpeg"); + conflictForm.append("displayName", "weekly announcement"); + const conflict = await jsonResponse(harness.baseUrl, "/api/assets", { + method: "POST", + body: conflictForm, + }); + assert.equal(conflict.response.status, 409); + assert.equal(conflict.body.error.code, "ASSET_NAME_CONFLICT"); + + const expiresAt = Date.parse("2034-01-02T03:04:05.000Z"); + const edited = await jsonResponse(harness.baseUrl, `/api/assets/${assetId}`, { + method: "PATCH", + headers: { "content-type": "application/json" }, + body: JSON.stringify({ + display_name: "Renamed announcement", + expiration_at: expiresAt, + }), + }); + assert.equal(edited.response.status, 200); + assert.equal(edited.body.displayName, "Renamed announcement"); + assert.equal(edited.body.expirationAt, expiresAt); + const permanent = await jsonResponse(harness.baseUrl, `/api/assets/${assetId}`, { + method: "PATCH", + headers: { "content-type": "application/json" }, + body: JSON.stringify({ expiration_at: null }), + }); + assert.equal(permanent.body.expirationAt, null); + assert.equal(harness.storage.objects.size, 3); + await eventuallyEmpty(harness.tempDirectory); + + const timestamp = Date.now(); + harness.database + .prepare( + "INSERT INTO playlists (id, name, created_at, updated_at) VALUES (?, ?, ?, ?)", + ) + .run("api-playlist", "API playlist", timestamp, timestamp); + harness.database + .prepare( + `INSERT INTO playlist_entries + (id, playlist_id, asset_id, position, created_at, updated_at) + VALUES (?, ?, ?, 0, ?, ?)`, + ) + .run("api-entry", "api-playlist", assetId, timestamp, timestamp); + const references = await jsonResponse( + harness.baseUrl, + `/api/assets/${assetId}/references`, + ); + assert.equal(references.response.status, 200); + assert.equal(references.body.canDelete, false); + assert.equal(references.body.playlists[0].playlistEntryId, "api-entry"); + const blockedDelete = await jsonResponse( + harness.baseUrl, + `/api/assets/${assetId}`, + { method: "DELETE" }, + ); + assert.equal(blockedDelete.response.status, 409); + assert.equal(blockedDelete.body.error.code, "ASSET_REFERENCED"); + assert.equal(harness.storage.objects.size, 3); + harness.database + .prepare("DELETE FROM playlist_entries WHERE id = ?") + .run("api-entry"); + + const removed = await jsonResponse(harness.baseUrl, `/api/assets/${assetId}`, { + method: "DELETE", + }); + assert.equal(removed.response.status, 200); + assert.deepEqual(removed.body, { id: assetId, deleted: true }); + assert.equal(harness.storage.objects.size, 0); + assert.equal( + (await jsonResponse(harness.baseUrl, `/api/assets/${assetId}`)).response.status, + 404, + ); +}); + +test("upload validation returns structured errors and always cleans temporary files", async (t) => { + const harness = await createHarness(t, { maxFileBytes: 1024 }); + + const malformed = new FormData(); + appendFile(malformed, "file", Buffer.from("not an image"), "broken.png", "image/png"); + const malformedResult = await jsonResponse(harness.baseUrl, "/api/assets", { + method: "POST", + body: malformed, + }); + assert.equal(malformedResult.response.status, 422); + assert.equal(malformedResult.body.error.code, "MALFORMED_IMAGE"); + await eventuallyEmpty(harness.tempDirectory); + + const unsupportedBuffer = await sharp({ + create: { + width: 20, + height: 20, + channels: 3, + background: "red", + }, + }) + .gif() + .toBuffer(); + const unsupported = new FormData(); + appendFile(unsupported, "file", unsupportedBuffer, "still.gif", "image/png"); + const unsupportedResult = await jsonResponse(harness.baseUrl, "/api/assets", { + method: "POST", + body: unsupported, + }); + assert.equal(unsupportedResult.response.status, 415); + assert.equal(unsupportedResult.body.error.code, "UNSUPPORTED_IMAGE_FORMAT"); + await eventuallyEmpty(harness.tempDirectory); + + const oversized = new FormData(); + appendFile( + oversized, + "file", + crypto.randomBytes(1025), + "too-large.png", + "image/png", + ); + const oversizedResult = await jsonResponse(harness.baseUrl, "/api/assets", { + method: "POST", + body: oversized, + }); + assert.equal(oversizedResult.response.status, 413); + assert.equal(oversizedResult.body.error.code, "UPLOAD_FILE_TOO_LARGE"); + assert.deepEqual(oversizedResult.body.error.details, { maxFileBytes: 1024 }); + await eventuallyEmpty(harness.tempDirectory); + + const tooMany = new FormData(); + for (let index = 0; index < 6; index += 1) { + appendFile(tooMany, "files", Buffer.from("x"), `${index}.png`, "image/png"); + } + const tooManyResult = await jsonResponse(harness.baseUrl, "/api/assets", { + method: "POST", + body: tooMany, + }); + assert.equal(tooManyResult.response.status, 413); + assert.equal(tooManyResult.body.error.code, "UPLOAD_FILE_COUNT_EXCEEDED"); + assert.deepEqual(tooManyResult.body.error.details, { maxFiles: 5 }); + await eventuallyEmpty(harness.tempDirectory); + + const absent = await jsonResponse(harness.baseUrl, "/api/assets", { + method: "POST", + body: new FormData(), + }); + assert.equal(absent.response.status, 400); + assert.equal(absent.body.error.code, "UPLOAD_FILE_REQUIRED"); + assert.equal(harness.storage.objects.size, 0); + assert.equal( + harness.database.prepare("SELECT COUNT(*) AS count FROM assets").get().count, + 0, + ); +}); + +test("an aborted multipart request removes its in-flight temporary file", async (t) => { + const harness = await createHarness(t); + const boundary = `announcements-${crypto.randomUUID()}`; + const preamble = Buffer.from( + `--${boundary}\r\n` + + 'Content-Disposition: form-data; name="file"; filename="aborted.png"\r\n' + + "Content-Type: image/png\r\n\r\n", + ); + const declaredLength = preamble.length + 512 * 1024; + const request = http.request(`${harness.baseUrl}/api/assets`, { + method: "POST", + headers: { + "content-type": `multipart/form-data; boundary=${boundary}`, + "content-length": String(declaredLength), + }, + }); + request.on("error", () => {}); + request.write(preamble); + request.write(Buffer.alloc(64 * 1024, 1)); + + await eventuallyContainsFile(harness.tempDirectory); + const closed = new Promise((resolve) => request.once("close", resolve)); + request.destroy(); + await closed; + await eventuallyEmpty(harness.tempDirectory); + assert.equal(harness.storage.objects.size, 0); + assert.equal( + harness.database.prepare("SELECT COUNT(*) AS count FROM assets").get().count, + 0, + ); +}); diff --git a/server/test/assets-multipart.test.js b/server/test/assets-multipart.test.js new file mode 100644 index 0000000..d818ae6 --- /dev/null +++ b/server/test/assets-multipart.test.js @@ -0,0 +1,35 @@ +const test = require("node:test"); +const assert = require("node:assert/strict"); +const { + mapMulterError, + safeOriginalFilename, +} = require("../src/assets/multipart"); + +test("original filename metadata keeps a safe leaf without path traversal", () => { + assert.equal(safeOriginalFilename("../../Sunday/welcome.png"), "welcome.png"); + assert.equal(safeOriginalFilename("..\\..\\Sunday\\welcome.png"), "welcome.png"); + assert.equal(safeOriginalFilename(" welcome sign.png "), " welcome sign.png "); +}); + +test("original filename metadata removes controls and remains bounded", () => { + assert.equal(safeOriginalFilename("wel\u0000come\n.png"), "welcome.png"); + assert.equal([...safeOriginalFilename(`${"a".repeat(300)}.png`)].length, 255); + assert.throws( + () => safeOriginalFilename(" \u0000\n "), + (error) => error.code === "INVALID_ORIGINAL_FILENAME", + ); +}); + +test("temporary-storage failures are server errors, not invalid client uploads", () => { + const storageFailure = mapMulterError( + Object.assign(new Error("disk full at a private path"), { code: "ENOSPC" }), + {}, + ); + assert.equal(storageFailure.code, "UPLOAD_TEMP_STORAGE_UNAVAILABLE"); + assert.equal(storageFailure.statusCode, 503); + assert.doesNotMatch(storageFailure.message, /private path/); + + const malformed = mapMulterError(new Error("Unexpected end of form"), {}); + assert.equal(malformed.code, "INVALID_MULTIPART_UPLOAD"); + assert.equal(malformed.statusCode, 400); +}); diff --git a/server/test/assets-service.test.js b/server/test/assets-service.test.js new file mode 100644 index 0000000..3ae6e3b --- /dev/null +++ b/server/test/assets-service.test.js @@ -0,0 +1,608 @@ +const test = require("node:test"); +const assert = require("node:assert/strict"); +const crypto = require("node:crypto"); +const fs = require("node:fs"); +const os = require("node:os"); +const path = require("node:path"); +const { Readable } = require("node:stream"); +const { openDatabase } = require("../src/database"); +const { runMigrations } = require("../src/database/migrate"); +const { createAssetService } = require("../src/assets/service"); +const { + StorageObjectNotFoundError, + StorageUnavailableError, +} = require("../src/storage"); + +async function bufferBody(body) { + if (Buffer.isBuffer(body)) return body; + if (body instanceof Uint8Array) return Buffer.from(body); + const chunks = []; + for await (const chunk of body) chunks.push(Buffer.from(chunk)); + return Buffer.concat(chunks); +} + +function createMemoryStorage() { + const objects = new Map(); + const calls = []; + let failNextPut = null; + let successfulPutsBeforeFailure = 0; + let failNextDelete = null; + let putDelayMs = 0; + + return { + objects, + calls, + setPutFailure(error, putsBeforeFailure = 0) { + failNextPut = error; + successfulPutsBeforeFailure = putsBeforeFailure; + }, + setDeleteFailure(error) { + failNextDelete = error; + }, + setPutDelay(milliseconds) { + putDelayMs = milliseconds; + }, + async putObject(input) { + calls.push({ operation: "put", ...input, body: undefined }); + if (putDelayMs) { + await new Promise((resolve) => setTimeout(resolve, putDelayMs)); + } + if (failNextPut) { + if (successfulPutsBeforeFailure === 0) { + const error = failNextPut; + failNextPut = null; + throw error; + } + successfulPutsBeforeFailure -= 1; + } + objects.set(input.key, { + body: await bufferBody(input.body), + contentType: input.contentType, + metadata: input.metadata, + }); + return { key: input.key }; + }, + async getObject(key) { + calls.push({ operation: "get", key }); + const object = objects.get(key); + if (!object) throw new StorageObjectNotFoundError(key); + return { + body: Readable.from(object.body), + contentLength: object.body.length, + contentType: object.contentType, + }; + }, + async deleteObject(key) { + calls.push({ operation: "delete", key }); + if (failNextDelete) { + const error = failNextDelete; + failNextDelete = null; + throw error; + } + objects.delete(key); + return { key, deleted: true }; + }, + }; +} + +function createFakeImageProcessor({ reject } = {}) { + const calls = []; + return { + calls, + async process(filename) { + calls.push(filename); + if (reject) throw reject; + const original = await fs.promises.readFile(filename); + const sourceSha256 = crypto.createHash("sha256").update(original).digest("hex"); + const displayBuffer = Buffer.from(`display:${sourceSha256}`); + const thumbnailBuffer = Buffer.from(`thumbnail:${sourceSha256}`); + return { + original: { + sha256: sourceSha256, + contentType: "image/png", + byteSize: original.length, + width: 1200, + height: 800, + }, + display: { + buffer: displayBuffer, + sha256: crypto.createHash("sha256").update(displayBuffer).digest("hex"), + contentType: "image/webp", + byteSize: displayBuffer.length, + width: 1200, + height: 800, + }, + thumbnail: { + buffer: thumbnailBuffer, + sha256: crypto.createHash("sha256").update(thumbnailBuffer).digest("hex"), + contentType: "image/webp", + byteSize: thumbnailBuffer.length, + width: 405, + height: 270, + }, + }; + }, + }; +} + +function createHarness(options = {}) { + const database = openDatabase({ path: ":memory:" }); + runMigrations(database); + const directory = fs.mkdtempSync(path.join(os.tmpdir(), "announcements-assets-")); + const storage = options.storage || createMemoryStorage(); + const imageProcessor = + options.imageProcessor || createFakeImageProcessor(options.processorOptions); + let idCounter = 0; + let clock = 2_000_000_000_000; + const service = createAssetService({ + database, + storage, + imageProcessor, + now: () => ++clock, + createId: + options.createId || + (() => `00000000-0000-4000-8000-${String(++idCounter).padStart(12, "0")}`), + concurrentUploadWaitMs: options.concurrentUploadWaitMs ?? 2_000, + }); + + function uploadFile(originalFilename, contents) { + const buffer = Buffer.isBuffer(contents) ? contents : Buffer.from(contents); + const filename = path.join(directory, crypto.randomUUID()); + fs.writeFileSync(filename, buffer); + return { path: filename, byteSize: buffer.length, originalFilename }; + } + + return { + database, + directory, + imageProcessor, + service, + storage, + uploadFile, + close() { + database.close(); + fs.rmSync(directory, { recursive: true, force: true }); + }, + }; +} + +function rawAsset(database, assetId) { + return database.prepare("SELECT * FROM assets WHERE id = ?").get(assetId); +} + +test("upload stores exact bytes under content-addressed keys without leaking keys", async (t) => { + const harness = createHarness(); + t.after(() => harness.close()); + const original = Buffer.from("exact uploaded bytes"); + + const [result] = await harness.service.upload([ + harness.uploadFile(" Sunday Welcome.PNG ", original), + ]); + + assert.equal(result.created, true); + assert.equal(result.duplicate, false); + assert.match(result.asset.id, /^[0-9a-f-]{36}$/); + assert.equal(result.asset.displayName, "Sunday Welcome"); + assert.equal(result.asset.originalFilename, " Sunday Welcome.PNG "); + assert.equal(result.asset.expirationAt, null); + assert.equal(result.asset.status, "ready"); + assert.equal(result.asset.original.byteSize, original.length); + assert.equal(result.asset.display.contentType, "image/webp"); + assert.equal(result.asset.thumbnail.width, 405); + assert.match(result.asset.display.url, /^\/media\/assets\//); + assert.doesNotMatch( + JSON.stringify(result.asset), + /object_key|originalObjectKey|displayObjectKey|thumbnailObjectKey|seaweed/i, + ); + + const row = rawAsset(harness.database, result.asset.id); + const expectedSourceHash = crypto.createHash("sha256").update(original).digest("hex"); + assert.equal(row.source_sha256, expectedSourceHash); + assert.equal(row.original_object_key, `assets/${row.id}/original/${expectedSourceHash}`); + assert.equal( + row.display_object_key, + `assets/${row.id}/display/${row.display_sha256}.webp`, + ); + assert.equal( + row.thumbnail_object_key, + `assets/${row.id}/thumbnail/${row.thumbnail_sha256}.webp`, + ); + assert.deepEqual(harness.storage.objects.get(row.original_object_key).body, original); + assert.equal(harness.storage.objects.size, 3); + + assert.deepEqual(harness.service.list(), [result.asset]); + assert.deepEqual(harness.service.get(result.asset.id), result.asset); +}); + +test("exact duplicate detection takes precedence over filename and display name", async (t) => { + const harness = createHarness(); + t.after(() => harness.close()); + const contents = Buffer.from("same image bytes"); + + const [created] = await harness.service.upload([ + harness.uploadFile("first.png", contents), + ]); + const [sameFilename] = await harness.service.upload([ + harness.uploadFile("first.png", contents), + ]); + const [differentFilename] = await harness.service.upload( + [harness.uploadFile("completely different name.webp", contents)], + { displayName: "A conflicting requested name is ignored for an exact duplicate" }, + ); + + for (const duplicate of [sameFilename, differentFilename]) { + assert.equal(duplicate.created, false); + assert.equal(duplicate.duplicate, true); + assert.equal(duplicate.asset.id, created.asset.id); + } + assert.equal(harness.service.list().length, 1); + assert.equal(harness.storage.calls.filter((call) => call.operation === "put").length, 3); + assert.equal(harness.imageProcessor.calls.length, 1); +}); + +test("different content with the same proposed name is a conflict, not a duplicate", async (t) => { + const harness = createHarness(); + t.after(() => harness.close()); + await harness.service.upload([harness.uploadFile("bulletin.png", "image one")]); + + await assert.rejects( + harness.service.upload([harness.uploadFile("bulletin.jpg", "image two")]), + (error) => { + assert.equal(error.statusCode, 409); + assert.equal(error.code, "ASSET_NAME_CONFLICT"); + assert.equal(error.details.asset.displayName, "bulletin"); + return true; + }, + ); + assert.equal(harness.service.list().length, 1); + assert.equal(harness.imageProcessor.calls.length, 1); + + const [created] = await harness.service.upload( + [harness.uploadFile("bulletin.jpg", "image two")], + { displayName: "Second bulletin" }, + ); + assert.equal(created.created, true); + assert.notEqual(created.asset.sourceSha256, harness.service.list()[1].sourceSha256); + assert.equal(harness.service.list().length, 2); +}); + +test("multiple-file upload preserves order and deduplicates repeated bytes in the batch", async (t) => { + const harness = createHarness(); + t.after(() => harness.close()); + + const results = await harness.service.upload([ + harness.uploadFile("alpha.png", "alpha bytes"), + harness.uploadFile("beta.png", "beta bytes"), + harness.uploadFile("alpha-copy.png", "alpha bytes"), + ]); + + assert.deepEqual( + results.map(({ created, duplicate }) => ({ created, duplicate })), + [ + { created: true, duplicate: false }, + { created: true, duplicate: false }, + { created: false, duplicate: true }, + ], + ); + assert.equal(results[0].asset.id, results[2].asset.id); + assert.equal(harness.service.list().length, 2); + assert.equal(harness.imageProcessor.calls.length, 2); + assert.equal(harness.storage.objects.size, 6); +}); + +test("all new images are decoded before the batch mutates SQLite or storage", async (t) => { + const expected = Object.assign(new Error("bad second image"), { + code: "MALFORMED_IMAGE", + statusCode: 422, + }); + let processCount = 0; + const regularProcessor = createFakeImageProcessor(); + const harness = createHarness({ + imageProcessor: { + async process(filename) { + processCount += 1; + if (processCount === 2) throw expected; + return regularProcessor.process(filename); + }, + }, + }); + t.after(() => harness.close()); + + await assert.rejects( + harness.service.upload([ + harness.uploadFile("valid.png", "valid"), + harness.uploadFile("broken.png", "broken"), + ]), + expected, + ); + assert.equal(harness.service.list().length, 0); + assert.equal(harness.storage.objects.size, 0); + assert.equal(harness.storage.calls.length, 0); +}); + +test("concurrent identical uploads converge on one ready asset", async (t) => { + const storage = createMemoryStorage(); + storage.setPutDelay(30); + const harness = createHarness({ storage }); + t.after(() => harness.close()); + const bytes = Buffer.from("racing identical upload"); + + const [[left], [right]] = await Promise.all([ + harness.service.upload([harness.uploadFile("race.png", bytes)]), + harness.service.upload([harness.uploadFile("other-name.png", bytes)]), + ]); + + assert.equal(harness.service.list().length, 1); + assert.equal(left.asset.id, right.asset.id); + assert.deepEqual([left.created, right.created].sort(), [false, true]); + assert.deepEqual([left.duplicate, right.duplicate].sort(), [false, true]); + assert.equal(storage.calls.filter((call) => call.operation === "put").length, 3); +}); + +test("a partial storage failure leaves a diagnosable failed row and no ready URLs", async (t) => { + const storage = createMemoryStorage(); + storage.setPutFailure(new StorageUnavailableError(new Error("offline")), 1); + const harness = createHarness({ storage }); + t.after(() => harness.close()); + + await assert.rejects( + harness.service.upload([harness.uploadFile("failure.png", "failure bytes")]), + (error) => { + assert.equal(error.statusCode, 503); + assert.equal(error.details.retryable, true); + assert.match(error.details.assetId, /^[0-9a-f-]{36}$/); + return true; + }, + ); + + const [failed] = harness.service.list(); + assert.equal(failed.status, "failed"); + assert.match(failed.processingError, /^STORAGE_UNAVAILABLE:/); + assert.equal(failed.display.url, null); + assert.equal(failed.thumbnail.url, null); + assert.equal(failed.original.url, null); + assert.equal(storage.objects.size, 0); + const row = rawAsset(harness.database, failed.id); + assert.match( + row.original_object_key, + new RegExp(`^assets/${failed.id}/original/[0-9a-f]{64}$`), + ); + assert.equal( + row.display_object_key, + `assets/${failed.id}/display/${row.display_sha256}.webp`, + ); + assert.equal( + row.thumbnail_object_key, + `assets/${failed.id}/thumbnail/${row.thumbnail_sha256}.webp`, + ); + assert.equal(storage.calls.filter((call) => call.operation === "put").length, 2); + assert.deepEqual( + new Set( + storage.calls + .filter((call) => call.operation === "delete") + .map((call) => call.key), + ), + new Set([ + row.original_object_key, + row.display_object_key, + row.thumbnail_object_key, + ]), + ); + + await assert.rejects( + harness.service.upload([harness.uploadFile("retry.png", "failure bytes")]), + (error) => error.code === "ASSET_PROCESSING_FAILED" && error.statusCode === 409, + ); + + assert.deepEqual(await harness.service.remove(failed.id), { + id: failed.id, + deleted: true, + }); + assert.equal(harness.service.list().length, 0); +}); + +test("rename and expiration edits change only approved SQLite metadata", async (t) => { + const harness = createHarness(); + t.after(() => harness.close()); + const [upload] = await harness.service.upload([ + harness.uploadFile("editable.png", "editable bytes"), + ]); + const before = rawAsset(harness.database, upload.asset.id); + + const expiring = harness.service.update(upload.asset.id, { + display_name: " New display name ", + expiration_at: "2035-05-06T07:08:09.000Z", + }); + assert.equal(expiring.displayName, "New display name"); + assert.equal(expiring.expirationAt, Date.parse("2035-05-06T07:08:09.000Z")); + + const permanent = harness.service.update(upload.asset.id, { expirationAt: null }); + assert.equal(permanent.expirationAt, null); + const after = rawAsset(harness.database, upload.asset.id); + assert.equal(after.original_object_key, before.original_object_key); + assert.equal(after.display_object_key, before.display_object_key); + assert.equal(after.thumbnail_object_key, before.thumbnail_object_key); + assert.equal(harness.storage.calls.filter((call) => call.operation === "put").length, 3); + + await assert.rejects( + async () => harness.service.update(upload.asset.id, { status: "failed" }), + (error) => error.code === "INVALID_ASSET_UPDATE", + ); + assert.throws( + () => harness.service.update(upload.asset.id, { expirationAt: -1 }), + (error) => error.code === "INVALID_ASSET_EXPIRATION", + ); +}); + +test("references describe playlist, normal, and static use and block deletion", async (t) => { + const harness = createHarness(); + t.after(() => harness.close()); + const [upload] = await harness.service.upload([ + harness.uploadFile("referenced.png", "referenced bytes"), + ]); + const id = upload.asset.id; + const now = 2_100_000_000_000; + harness.database + .prepare( + "INSERT INTO playlists (id, name, created_at, updated_at) VALUES (?, ?, ?, ?)", + ) + .run("playlist-1", "Sunday", now, now); + harness.database + .prepare( + `INSERT INTO playlist_entries + (id, playlist_id, asset_id, enabled, position, created_at, updated_at) + VALUES (?, ?, ?, 1, 0, ?, ?)`, + ) + .run("entry-1", "playlist-1", id, now, now); + harness.database + .prepare( + `UPDATE playback_state + SET active_playlist_id = ?, current_entry_id = ?, static_asset_id = ?, updated_at = ? + WHERE output_id = 'global'`, + ) + .run("playlist-1", "entry-1", id, now); + + const references = harness.service.references(id); + assert.equal(references.canDelete, false); + assert.deepEqual(references.blockerCodes, [ + "PLAYLIST_REFERENCES", + "CURRENT_PLAYBACK_TARGET", + "CURRENT_STATIC_TARGET", + ]); + assert.deepEqual(references.playlists[0], { + playlistId: "playlist-1", + playlistName: "Sunday", + playlistEntryId: "entry-1", + enabled: true, + position: 0, + isActivePlaylist: true, + isCurrentNormalTarget: true, + }); + assert.equal(references.playback[0].isCurrentNormalTarget, true); + assert.equal(references.playback[0].isCurrentStaticTarget, true); + + const callsBeforeDelete = harness.storage.calls.length; + await assert.rejects( + harness.service.remove(id), + (error) => { + assert.equal(error.code, "ASSET_REFERENCED"); + assert.equal(error.statusCode, 409); + assert.deepEqual(error.details.blockerCodes, [ + "PLAYLIST_REFERENCES", + "CURRENT_PLAYBACK_TARGET", + "CURRENT_STATIC_TARGET", + ]); + return true; + }, + ); + assert.equal(harness.storage.calls.length, callsBeforeDelete); + assert.equal(rawAsset(harness.database, id).status, "ready"); + + harness.database + .prepare( + `UPDATE playback_state + SET active_playlist_id = NULL, current_entry_id = NULL, static_asset_id = NULL + WHERE output_id = 'global'`, + ) + .run(); + harness.database.prepare("DELETE FROM playlist_entries WHERE id = ?").run("entry-1"); + assert.equal(harness.service.references(id).canDelete, true); + await harness.service.remove(id); + assert.equal(harness.storage.objects.size, 0); + assert.equal(rawAsset(harness.database, id), undefined); +}); + +test("failed storage deletion retains a retryable deleting row", async (t) => { + const storage = createMemoryStorage(); + const harness = createHarness({ storage }); + t.after(() => harness.close()); + const [upload] = await harness.service.upload([ + harness.uploadFile("delete-me.png", "delete me"), + ]); + storage.setDeleteFailure(new StorageUnavailableError(new Error("offline"))); + + await assert.rejects(harness.service.remove(upload.asset.id), (error) => { + assert.equal(error.statusCode, 503); + assert.equal(error.details.assetId, upload.asset.id); + assert.equal(error.details.retryable, true); + return true; + }); + assert.equal(rawAsset(harness.database, upload.asset.id).status, "deleting"); + assert.throws( + () => harness.service.update(upload.asset.id, { displayName: "Cannot edit" }), + (error) => error.code === "ASSET_DELETING", + ); + + await harness.service.remove(upload.asset.id); + assert.equal(rawAsset(harness.database, upload.asset.id), undefined); + assert.equal(storage.objects.size, 0); +}); + +test("processing assets cannot be deleted and storage is not touched", async (t) => { + const harness = createHarness(); + t.after(() => harness.close()); + const timestamp = Date.now(); + harness.database + .prepare( + `INSERT INTO assets + (id, source_sha256, display_name, original_filename, status, created_at, updated_at) + VALUES (?, ?, ?, ?, 'processing', ?, ?)`, + ) + .run( + "00000000-0000-4000-8000-999999999999", + "f".repeat(64), + "Still processing", + "processing.png", + timestamp, + timestamp, + ); + + assert.equal( + harness.service.references("00000000-0000-4000-8000-999999999999") + .canDelete, + false, + ); + await assert.rejects( + harness.service.remove("00000000-0000-4000-8000-999999999999"), + (error) => error.code === "ASSET_PROCESSING" && error.statusCode === 409, + ); + assert.equal(harness.storage.calls.length, 0); + assert.equal( + rawAsset(harness.database, "00000000-0000-4000-8000-999999999999").status, + "processing", + ); +}); + +test("media descriptors only allow the recorded version and map missing bytes to 503", async (t) => { + const harness = createHarness(); + t.after(() => harness.close()); + const [upload] = await harness.service.upload([ + harness.uploadFile("media.png", "media bytes"), + ]); + const { asset } = upload; + + const display = harness.service.mediaDescriptor( + asset.id, + "display", + asset.display.sha256, + ); + assert.equal(display.contentType, "image/webp"); + assert.equal(display.contentLength, asset.display.byteSize); + const stored = await harness.service.readStoredObject(display); + assert.deepEqual(await bufferBody(stored.body), harness.storage.objects.get(display.key).body); + + for (const requestedHash of ["a".repeat(64), "../original/key", "not-a-hash"]) { + assert.throws( + () => harness.service.mediaDescriptor(asset.id, "display", requestedHash), + (error) => error.code === "MEDIA_NOT_FOUND" && error.statusCode === 404, + ); + } + + harness.storage.objects.delete(display.key); + await assert.rejects(harness.service.readStoredObject(display), (error) => { + assert.equal(error.code, "STORAGE_INCONSISTENCY"); + assert.equal(error.statusCode, 503); + assert.deepEqual(error.details, { assetId: asset.id, retryable: true }); + return true; + }); +}); diff --git a/server/test/config.test.js b/server/test/config.test.js index e060d43..512ea57 100644 --- a/server/test/config.test.js +++ b/server/test/config.test.js @@ -32,6 +32,14 @@ test("loadConfig validates and normalizes the supported environment", () => { assert.equal(config.timeZone, "America/Chicago"); assert.equal(config.storage.endpoint, "http://seaweedfs:8333"); assert.equal(config.storage.forcePathStyle, true); + assert.equal(config.storage.healthProbeIntervalMs, 30_000); + assert.equal(config.storage.connectionTimeoutMs, 2_000); + assert.equal(config.storage.requestTimeoutMs, 5_000); + assert.equal(config.storage.maxAttempts, 2); + assert.equal(config.processing.tempDirectory, "/tmp/announcements-uploads"); + assert.equal(config.processing.maxFileBytes, 25 * 1024 * 1024); + assert.equal(config.processing.maxFiles, 20); + assert.equal(config.processing.maxInputPixels, 40_000_000); }); test("production requires private S3 credentials", () => { @@ -65,10 +73,42 @@ test("loadConfig rejects invalid typed values", () => { () => loadConfig(validEnvironment({ S3_FORCE_PATH_STYLE: "sometimes" })), /must be a boolean/, ); + assert.throws( + () => loadConfig(validEnvironment({ S3_HEALTH_PROBE_INTERVAL_MS: "999" })), + /S3_HEALTH_PROBE_INTERVAL_MS must be at least 1000/, + ); + assert.throws( + () => loadConfig(validEnvironment({ S3_REQUEST_TIMEOUT_MS: "999" })), + /S3_REQUEST_TIMEOUT_MS must be at least 1000/, + ); + assert.throws( + () => loadConfig(validEnvironment({ S3_CONNECT_TIMEOUT_MS: "249" })), + /S3_CONNECT_TIMEOUT_MS must be at least 250/, + ); + assert.throws( + () => loadConfig(validEnvironment({ S3_MAX_ATTEMPTS: "6" })), + /S3_MAX_ATTEMPTS must be at most 5/, + ); assert.throws( () => loadConfig(validEnvironment({ DATABASE_PATH: "relative.sqlite" })), /must be an absolute path/, ); + assert.throws( + () => loadConfig(validEnvironment({ UPLOAD_TEMP_DIRECTORY: "relative" })), + /UPLOAD_TEMP_DIRECTORY must be an absolute path/, + ); + assert.throws( + () => loadConfig(validEnvironment({ UPLOAD_MAX_FILE_BYTES: "0" })), + /UPLOAD_MAX_FILE_BYTES must be at least 1/, + ); + assert.throws( + () => loadConfig(validEnvironment({ UPLOAD_MAX_FILES: "101" })), + /UPLOAD_MAX_FILES must be at most 100/, + ); + assert.throws( + () => loadConfig(validEnvironment({ IMAGE_MAX_INPUT_PIXELS: "no-limit" })), + /IMAGE_MAX_INPUT_PIXELS must be an integer/, + ); }); test("redactedConfig never returns credentials, passwords, paths, or webhook URLs", () => { @@ -83,5 +123,6 @@ test("redactedConfig never returns credentials, passwords, paths, or webhook URL assert.doesNotMatch(serialized, /mqtt-password/); assert.doesNotMatch(serialized, /webhook-secret/); assert.doesNotMatch(serialized, /announcements\.sqlite/); + assert.doesNotMatch(serialized, /seaweedfs/); assert.doesNotMatch(serialized, /home-assistant/); }); diff --git a/server/test/image-processing.test.js b/server/test/image-processing.test.js new file mode 100644 index 0000000..4b88090 --- /dev/null +++ b/server/test/image-processing.test.js @@ -0,0 +1,304 @@ +const test = require("node:test"); +const assert = require("node:assert/strict"); +const crypto = require("node:crypto"); +const fs = require("node:fs"); +const os = require("node:os"); +const path = require("node:path"); +const sharp = require("sharp"); +const { + DEFAULT_IMAGE_DERIVATIVE_OPTIONS, + ImageProcessingError, + createImageProcessor, +} = require("../src/image-processing"); + +const MAX_INPUT_PIXELS = 12_000_000; +const CONTENT_TYPES = Object.freeze({ + jpeg: "image/jpeg", + png: "image/png", + webp: "image/webp", +}); + +function createProcessor(overrides = {}) { + return createImageProcessor({ + maxInputPixels: MAX_INPUT_PIXELS, + ...overrides, + }); +} + +function hash(buffer) { + return crypto.createHash("sha256").update(buffer).digest("hex"); +} + +async function solidImage(format, width, height, background = "#4a76b8") { + return sharp({ + create: { + width, + height, + channels: 4, + background, + }, + }) + .toFormat(format) + .toBuffer(); +} + +async function twoFrameImage(format) { + const width = 20; + const frameHeight = 10; + const channels = 4; + const firstFrame = Buffer.alloc(width * frameHeight * channels); + const secondFrame = Buffer.alloc(width * frameHeight * channels); + + for (let offset = 0; offset < firstFrame.length; offset += channels) { + firstFrame.set([255, 0, 0, 255], offset); + secondFrame.set([0, 0, 255, 255], offset); + } + + const image = sharp(Buffer.concat([firstFrame, secondFrame]), { + raw: { + width, + height: frameHeight * 2, + channels, + pageHeight: frameHeight, + }, + }); + + if (format === "tiff") { + return image.tiff().toBuffer(); + } + return image[format]({ delay: [100, 100], loop: 0 }).toBuffer(); +} + +test("accepts decoded still JPEG, PNG, and WebP content", async (t) => { + const processor = createProcessor(); + + for (const format of ["jpeg", "png", "webp"]) { + await t.test(format, async () => { + const source = await solidImage(format, 640, 360); + const result = await processor.process(source); + + assert.equal(result.original.format, format); + assert.equal(result.original.contentType, CONTENT_TYPES[format]); + assert.equal(result.original.byteSize, source.length); + assert.equal(result.original.sha256, hash(source)); + assert.equal(result.original.width, 640); + assert.equal(result.original.height, 360); + assert.equal(result.original.pages, 1); + + for (const derivative of [result.display, result.thumbnail]) { + assert.ok(Buffer.isBuffer(derivative.buffer)); + assert.equal(derivative.format, "webp"); + assert.equal(derivative.contentType, "image/webp"); + assert.equal(derivative.byteSize, derivative.buffer.length); + assert.equal(derivative.sha256, hash(derivative.buffer)); + assert.match(derivative.sha256, /^[0-9a-f]{64}$/); + } + }); + } +}); + +test("uses bounding boxes without cropping or changing source aspect ratio", async () => { + const source = await solidImage("png", 3000, 1000); + const result = await createProcessor().process(source); + + assert.deepEqual( + { width: result.display.width, height: result.display.height }, + { width: 1920, height: 640 }, + ); + assert.deepEqual( + { width: result.thumbnail.width, height: result.thumbnail.height }, + { width: 480, height: 160 }, + ); + assert.equal(result.display.width / result.display.height, 3); + assert.equal(result.thumbnail.width / result.thumbnail.height, 3); +}); + +test("does not enlarge a source that fits within both derivative bounds", async () => { + const source = await solidImage("jpeg", 320, 180); + const result = await createProcessor().process(source); + + assert.deepEqual( + { width: result.display.width, height: result.display.height }, + { width: 320, height: 180 }, + ); + assert.deepEqual( + { width: result.thumbnail.width, height: result.thumbnail.height }, + { width: 320, height: 180 }, + ); +}); + +test("applies EXIF orientation and strips it from derivative metadata", async () => { + const source = await sharp({ + create: { + width: 120, + height: 80, + channels: 3, + background: "#cb7235", + }, + }) + .jpeg() + .withMetadata({ orientation: 6 }) + .toBuffer(); + + const result = await createProcessor().process(source); + const outputMetadata = await sharp(result.display.buffer).metadata(); + + assert.equal(result.original.orientation, 6); + assert.equal(result.original.encodedWidth, 120); + assert.equal(result.original.encodedHeight, 80); + assert.equal(result.original.width, 80); + assert.equal(result.original.height, 120); + assert.equal(result.display.width, 80); + assert.equal(result.display.height, 120); + assert.equal(outputMetadata.orientation, undefined); + assert.equal(outputMetadata.exif, undefined); +}); + +test("flattens transparent pixels against black in both derivatives", async () => { + const source = await solidImage("png", 40, 30, { + r: 255, + g: 0, + b: 0, + alpha: 0, + }); + const result = await createProcessor().process(source); + + assert.equal(result.original.hasAlpha, true); + for (const derivative of [result.display, result.thumbnail]) { + const metadata = await sharp(derivative.buffer).metadata(); + const pixels = await sharp(derivative.buffer).raw().toBuffer(); + + assert.equal(metadata.hasAlpha, false); + assert.ok( + pixels.every((channel) => channel <= 2), + "fully transparent source pixels should become black", + ); + } +}); + +test("supports a temporary file path without changing source identity", async (t) => { + const temporaryDirectory = fs.mkdtempSync( + path.join(os.tmpdir(), "announcements-image-processing-"), + ); + t.after(() => fs.rmSync(temporaryDirectory, { recursive: true, force: true })); + + const source = await solidImage("webp", 800, 600); + const filename = path.join(temporaryDirectory, "upload.tmp"); + fs.writeFileSync(filename, source); + + const result = await createProcessor().process(filename); + assert.equal(result.original.sha256, hash(source)); + assert.equal(result.original.byteSize, source.length); + assert.equal(result.original.format, "webp"); +}); + +test("rejects malformed and unsupported decoded content with typed errors", async () => { + const processor = createProcessor(); + const gif = await solidImage("gif", 64, 64); + + await assert.rejects( + processor.process(Buffer.from("not an image")), + (error) => + error instanceof ImageProcessingError && + error.code === "MALFORMED_IMAGE" && + error.statusCode === 422, + ); + await assert.rejects( + processor.process(gif), + (error) => + error instanceof ImageProcessingError && + error.code === "UNSUPPORTED_IMAGE_FORMAT" && + error.statusCode === 415 && + error.details.detectedFormat === "gif", + ); +}); + +test("rejects animated WebP and GIF instead of selecting the first frame", async (t) => { + for (const format of ["webp", "gif"]) { + await t.test(format, async () => { + const animated = await twoFrameImage(format); + const metadata = await sharp(animated, { animated: true }).metadata(); + assert.equal(metadata.pages, 2, "test fixture must contain two frames"); + await assert.rejects( + createProcessor().process(animated), + (error) => + error instanceof ImageProcessingError && + error.code === "ANIMATED_IMAGE_NOT_SUPPORTED" && + error.details.pages === 2, + ); + }); + } +}); + +test("rejects unexpected multipage inputs with a distinct typed error", async () => { + const multipageTiff = await twoFrameImage("tiff"); + const metadata = await sharp(multipageTiff, { animated: true }).metadata(); + assert.equal(metadata.pages, 2, "test fixture must contain two pages"); + + await assert.rejects( + createProcessor().process(multipageTiff), + (error) => + error instanceof ImageProcessingError && + error.code === "MULTIPAGE_IMAGE_NOT_SUPPORTED" && + error.details.pages === 2, + ); +}); + +test("enforces the injected decoded-pixel limit", async () => { + const source = await solidImage("png", 100, 100); + const processor = createImageProcessor({ maxInputPixels: 9_999 }); + + await assert.rejects( + processor.process(source), + (error) => + error instanceof ImageProcessingError && + error.code === "IMAGE_PIXEL_LIMIT_EXCEEDED" && + error.statusCode === 413 && + error.details.maxInputPixels === 9_999, + ); +}); + +test("uses documented defaults while allowing validated target overrides", async () => { + assert.deepEqual(DEFAULT_IMAGE_DERIVATIVE_OPTIONS, { + display: { maxWidth: 1920, maxHeight: 1080, quality: 82 }, + thumbnail: { maxWidth: 480, maxHeight: 270, quality: 72 }, + }); + + const source = await solidImage("png", 900, 600); + const processor = createProcessor({ + display: { maxWidth: 300, maxHeight: 300, quality: 90 }, + thumbnail: { maxWidth: 100, maxHeight: 50, quality: 65 }, + }); + const result = await processor.process(source); + + assert.deepEqual( + { width: result.display.width, height: result.display.height }, + { width: 300, height: 200 }, + ); + assert.deepEqual( + { width: result.thumbnail.width, height: result.thumbnail.height }, + { width: 75, height: 50 }, + ); + assert.throws( + () => createImageProcessor({ maxInputPixels: 0 }), + /maxInputPixels must be at least 1/, + ); + assert.throws( + () => + createImageProcessor({ + maxInputPixels: 10_000, + display: { quality: 101 }, + }), + /display\.quality must be at most 100/, + ); +}); + +test("rejects invalid input interfaces with structured errors", async () => { + await assert.rejects( + createProcessor().process({ path: "/tmp/not-an-interface" }), + (error) => + error instanceof ImageProcessingError && + error.code === "INVALID_IMAGE_INPUT" && + error.statusCode === 400, + ); +}); diff --git a/server/test/runtime.test.js b/server/test/runtime.test.js index a9c212d..934d34a 100644 --- a/server/test/runtime.test.js +++ b/server/test/runtime.test.js @@ -29,13 +29,33 @@ const silentLogger = { error() {}, }; +function testStorage() { + return { + async start() {}, + stop() {}, + health() { + return { + status: "degraded", + checkedAt: "2026-08-16T00:00:00.000Z", + lastSuccessAt: null, + code: "STORAGE_UNAVAILABLE", + message: "Object storage is unavailable", + }; + }, + }; +} + async function responseJson(baseUrl, path, init) { const response = await fetch(`${baseUrl}${path}`, init); return { response, body: await response.json() }; } test("HTTP foundation exposes live, ready, detailed health, and persisted settings", async (t) => { - const runtime = createRuntime({ config: testConfig(), logger: silentLogger }); + const runtime = createRuntime({ + config: testConfig(), + logger: silentLogger, + storage: testStorage(), + }); await runtime.start(); t.after(() => runtime.stop("test cleanup")); const baseUrl = `http://127.0.0.1:${runtime.address.port}`; @@ -52,7 +72,7 @@ test("HTTP foundation exposes live, ready, detailed health, and persisted settin assert.equal(status.response.status, 200); assert.equal(status.body.status, "degraded"); assert.equal(status.body.services.database.currentSchemaVersion, 1); - assert.equal(status.body.services.storage.status, "not_checked"); + assert.equal(status.body.services.storage.status, "degraded"); const statusText = JSON.stringify(status.body); assert.doesNotMatch(statusText, /access-key-that-must-not-leak/); assert.doesNotMatch(statusText, /secret-key-that-must-not-leak/); @@ -83,7 +103,11 @@ test("HTTP foundation exposes live, ready, detailed health, and persisted settin }); test("invalid settings receive a structured client error", async (t) => { - const runtime = createRuntime({ config: testConfig(), logger: silentLogger }); + const runtime = createRuntime({ + config: testConfig(), + logger: silentLogger, + storage: testStorage(), + }); await runtime.start(); t.after(() => runtime.stop("test cleanup")); const baseUrl = `http://127.0.0.1:${runtime.address.port}`; @@ -99,7 +123,11 @@ test("invalid settings receive a structured client error", async (t) => { }); test("liveness stays up while critical readiness failures return 503", async (t) => { - const runtime = createRuntime({ config: testConfig(), logger: silentLogger }); + const runtime = createRuntime({ + config: testConfig(), + logger: silentLogger, + storage: testStorage(), + }); await runtime.start(); t.after(() => runtime.stop("test cleanup")); const baseUrl = `http://127.0.0.1:${runtime.address.port}`; @@ -127,7 +155,11 @@ test("liveness stays up while critical readiness failures return 503", async (t) }); test("runtime startup and shutdown are idempotent", async () => { - const runtime = createRuntime({ config: testConfig(), logger: silentLogger }); + const runtime = createRuntime({ + config: testConfig(), + logger: silentLogger, + storage: testStorage(), + }); const [firstAddress, secondAddress] = await Promise.all([ runtime.start(), runtime.start(), diff --git a/server/test/storage-health.test.js b/server/test/storage-health.test.js new file mode 100644 index 0000000..14e146e --- /dev/null +++ b/server/test/storage-health.test.js @@ -0,0 +1,105 @@ +const test = require("node:test"); +const assert = require("node:assert/strict"); +const express = require("express"); +const { openDatabase } = require("../src/database"); +const { runMigrations } = require("../src/database/migrate"); +const { + createDetailedHealthRouter, + createHealthRouter, +} = require("../src/health/routes"); + +function config() { + return { + environment: "test", + host: "127.0.0.1", + port: 0, + timeZone: "America/Chicago", + trustProxy: true, + database: { path: ":memory:" }, + storage: { + endpoint: "http://private-storage:8333", + region: "us-east-1", + bucket: "announcements-v2", + accessKeyId: "must-not-leak", + secretAccessKey: "must-not-leak-either", + forcePathStyle: true, + }, + processing: { + tempDirectory: "/tmp/announcements-uploads", + maxFileBytes: 1024, + maxFiles: 2, + maxInputPixels: 1000, + }, + mqtt: { url: null }, + notifications: { homeAssistantWebhookUrl: null }, + }; +} + +async function withServer(storage, callback) { + const database = openDatabase({ path: ":memory:", busyTimeoutMs: 1000 }); + const migrations = runMigrations(database); + const lifecycle = { ready: true }; + const app = express(); + const options = { config: config(), database, lifecycle, migrations, storage }; + app.use("/health", createHealthRouter(options)); + app.use("/api/health", createDetailedHealthRouter(options)); + const server = app.listen(0, "127.0.0.1"); + await new Promise((resolve) => server.once("listening", resolve)); + + try { + await callback(`http://127.0.0.1:${server.address().port}`); + } finally { + await new Promise((resolve) => server.close(resolve)); + database.close(); + } +} + +test("storage degradation is reported but does not fail API readiness", async () => { + const storage = { + health() { + return { + status: "degraded", + checkedAt: "2026-08-16T12:00:00.000Z", + lastSuccessAt: "2026-08-16T11:59:00.000Z", + code: "STORAGE_UNAVAILABLE", + }; + }, + }; + + await withServer(storage, async (baseUrl) => { + const ready = await fetch(`${baseUrl}/health/ready`); + const readyBody = await ready.json(); + assert.equal(ready.status, 200); + assert.equal(readyBody.status, "ready"); + assert.equal(readyBody.checks.storage, "degraded"); + + const detailed = await fetch(`${baseUrl}/api/health`); + const detailedBody = await detailed.json(); + assert.equal(detailed.status, 200); + assert.equal(detailedBody.status, "degraded"); + assert.equal(detailedBody.services.storage.status, "degraded"); + assert.equal( + detailedBody.services.storage.lastSuccessAt, + "2026-08-16T11:59:00.000Z", + ); + assert.equal(detailedBody.services.storage.code, "STORAGE_UNAVAILABLE"); + assert.doesNotMatch(JSON.stringify(detailedBody), /must-not-leak/); + assert.doesNotMatch(JSON.stringify(detailedBody), /private-storage/); + }); +}); + +test("healthy storage produces healthy detailed status", async () => { + const storage = { + health() { + return { status: "healthy", checkedAt: "2026-08-16T12:00:00.000Z" }; + }, + }; + + await withServer(storage, async (baseUrl) => { + const detailed = await fetch(`${baseUrl}/api/health`); + const body = await detailed.json(); + assert.equal(detailed.status, 200); + assert.equal(body.status, "ok"); + assert.equal(body.services.storage.status, "healthy"); + }); +}); diff --git a/server/test/storage.test.js b/server/test/storage.test.js new file mode 100644 index 0000000..215e515 --- /dev/null +++ b/server/test/storage.test.js @@ -0,0 +1,397 @@ +const test = require("node:test"); +const assert = require("node:assert/strict"); +const http = require("node:http"); +const { PassThrough, Readable } = require("node:stream"); +const { + StorageObjectNotFoundError, + StorageUnavailableError, + createStorageService, +} = require("../src/storage"); + +const storageConfig = { + endpoint: "http://seaweedfs:8333", + region: "us-east-1", + bucket: "announcements-v2", + accessKeyId: "test-key", + secretAccessKey: "test-secret", + forcePathStyle: true, +}; + +function fakeClient(handler) { + const commands = []; + return { + commands, + async send(command) { + commands.push(command); + return handler(command, commands.length - 1); + }, + }; +} + +function commandName(command) { + return command.constructor.name; +} + +test("storage initialization probes an existing bucket without creating it", async () => { + const client = fakeClient(async () => ({})); + const service = createStorageService(storageConfig, { client }); + + const health = await service.initialize(); + assert.equal(health.status, "healthy"); + assert.match(health.checkedAt, /T/); + assert.equal(health.lastSuccessAt, health.checkedAt); + assert.deepEqual(client.commands.map(commandName), ["HeadBucketCommand"]); +}); + +test("storage initialization creates only a missing bucket", async () => { + let exists = false; + const missing = Object.assign(new Error("missing"), { + name: "NoSuchBucket", + $metadata: { httpStatusCode: 404 }, + }); + const client = fakeClient(async (command) => { + if (commandName(command) === "HeadBucketCommand" && !exists) throw missing; + if (commandName(command) === "CreateBucketCommand") exists = true; + return {}; + }); + const service = createStorageService(storageConfig, { client }); + + assert.deepEqual(await service.ensureBucket(), { created: true }); + assert.deepEqual(client.commands.map(commandName), [ + "HeadBucketCommand", + "CreateBucketCommand", + ]); + assert.equal(client.commands[1].input.Bucket, storageConfig.bucket); + + assert.deepEqual(await service.ensureBucket(), { created: false }); + assert.deepEqual(client.commands.map(commandName), [ + "HeadBucketCommand", + "CreateBucketCommand", + "HeadBucketCommand", + ]); +}); + +test("storage start performs a nonfatal immediate check and stop releases its client", async () => { + let destroyed = false; + const client = fakeClient(async () => ({})); + client.destroy = () => { + destroyed = true; + }; + const service = createStorageService( + { ...storageConfig, healthProbeIntervalMs: 1_000 }, + { client }, + ); + + const health = await service.start(); + assert.equal(health.status, "healthy"); + assert.deepEqual(client.commands.map(commandName), ["HeadBucketCommand"]); + service.stop(); + assert.equal(destroyed, true); +}); + +test("a create-bucket race is healthy only after the bucket can be verified", async () => { + const missing = Object.assign(new Error("missing"), { + name: "NoSuchBucket", + $metadata: { httpStatusCode: 404 }, + }); + const conflict = Object.assign(new Error("race"), { + name: "BucketAlreadyOwnedByYou", + $metadata: { httpStatusCode: 409 }, + }); + let headCalls = 0; + const client = fakeClient(async (command) => { + if (commandName(command) === "HeadBucketCommand") { + headCalls += 1; + if (headCalls === 1) throw missing; + return {}; + } + if (commandName(command) === "CreateBucketCommand") throw conflict; + throw new Error(`Unexpected ${commandName(command)}`); + }); + const service = createStorageService(storageConfig, { client }); + + assert.deepEqual(await service.ensureBucket(), { created: false }); + assert.equal(service.health().status, "healthy"); + assert.deepEqual(client.commands.map(commandName), [ + "HeadBucketCommand", + "CreateBucketCommand", + "HeadBucketCommand", + ]); +}); + +test("an older failed probe cannot overwrite a newer successful operation", async () => { + let rejectProbe; + let notifyProbeStarted; + const probeStarted = new Promise((resolve) => { + notifyProbeStarted = resolve; + }); + let headCalls = 0; + const client = fakeClient(async (command) => { + if (commandName(command) === "HeadBucketCommand") { + headCalls += 1; + if (headCalls === 1) return {}; + notifyProbeStarted(); + return new Promise((resolve, reject) => { + rejectProbe = reject; + }); + } + if (commandName(command) === "PutObjectCommand") return {}; + throw new Error(`Unexpected ${commandName(command)}`); + }); + const service = createStorageService(storageConfig, { client }); + await service.initialize(); + + const staleProbe = service.probe(); + await probeStarted; + await service.putObject({ + key: "assets/id/original/hash", + body: Buffer.from("bytes"), + contentLength: 5, + }); + const successfulState = service.health(); + rejectProbe(new Error("older request failed")); + await staleProbe; + + assert.equal(successfulState.status, "healthy"); + assert.deepEqual(service.health(), successfulState); +}); + +test("temporary storage failure degrades without throwing during initialization and recovers", async () => { + let available = false; + const warnings = []; + const client = fakeClient(async () => { + if (!available) throw new Error("connection refused at a private address"); + return {}; + }); + const service = createStorageService(storageConfig, { + client, + logger: { warn(message) { warnings.push(message); } }, + }); + + const initial = await service.initialize(); + assert.equal(initial.status, "degraded"); + assert.equal(initial.code, "STORAGE_UNAVAILABLE"); + assert.equal(initial.lastSuccessAt, null); + assert.deepEqual(warnings, ["Object storage is unavailable during initialization"]); + + available = true; + const recovered = await service.probe(); + assert.equal(recovered.status, "healthy"); + assert.equal(recovered.lastSuccessAt, recovered.checkedAt); + assert.equal(service.health().status, "healthy"); + + available = false; + const degradedAgain = await service.probe(); + assert.equal(degradedAgain.status, "degraded"); + assert.equal(degradedAgain.lastSuccessAt, recovered.lastSuccessAt); +}); + +test("put, get, head, existence, and delete use the configured private bucket", async () => { + const body = Readable.from([Buffer.from("image bytes")]); + const client = fakeClient(async (command) => { + switch (commandName(command)) { + case "PutObjectCommand": + return { ETag: '"etag"' }; + case "GetObjectCommand": + return { + Body: body, + ContentType: "image/webp", + ContentLength: 11, + ETag: '"etag"', + }; + case "HeadObjectCommand": + return { + ContentType: "image/webp", + ContentLength: 11, + ETag: '"etag"', + }; + case "DeleteObjectCommand": + return {}; + case "HeadBucketCommand": + return {}; + default: + throw new Error(`Unexpected ${commandName(command)}`); + } + }); + const service = createStorageService(storageConfig, { client }); + const key = "assets/id/display/hash.webp"; + await service.initialize(); + client.commands.length = 0; + + const put = await service.putObject({ + key, + body: Buffer.from("image bytes"), + contentType: "image/webp", + contentLength: 11, + }); + assert.deepEqual(put, { key, etag: '"etag"', versionId: null }); + + const fetched = await service.getObject(key); + assert.equal(fetched.body, body); + assert.equal(fetched.contentType, "image/webp"); + assert.equal(fetched.contentLength, 11); + + assert.equal((await service.headObject(key)).etag, '"etag"'); + assert.equal(await service.objectExists(key), true); + await service.deleteObject(key); + + for (const command of client.commands) { + assert.equal(command.input.Bucket, storageConfig.bucket); + assert.equal(command.input.Key, key); + } +}); + +test("missing objects are distinct from unavailable storage", async () => { + let missing = true; + const client = fakeClient(async (command) => { + if (commandName(command) === "HeadBucketCommand") return {}; + if (missing) { + throw Object.assign(new Error("missing"), { + name: "NoSuchKey", + $metadata: { httpStatusCode: 404 }, + }); + } + throw new Error("network offline"); + }); + const service = createStorageService(storageConfig, { client }); + + await service.initialize(); + client.commands.length = 0; + + assert.equal(await service.headObject("assets/id/original/hash"), null); + assert.equal(await service.objectExists("assets/id/original/hash"), false); + await assert.rejects( + service.getObject("assets/id/original/hash"), + StorageObjectNotFoundError, + ); + assert.equal(service.health().status, "healthy"); + + missing = false; + await assert.rejects( + service.putObject({ key: "assets/id/original/hash", body: Buffer.alloc(0) }), + (error) => + error instanceof StorageUnavailableError && + error.code === "STORAGE_UNAVAILABLE" && + error.statusCode === 503, + ); + assert.equal(service.health().status, "degraded"); +}); + +test("a missing bucket during object access is a storage outage, not a missing object", async () => { + let initialized = false; + const client = fakeClient(async (command) => { + if (commandName(command) === "HeadBucketCommand" && !initialized) { + initialized = true; + return {}; + } + throw Object.assign(new Error("bucket missing"), { + name: "NoSuchBucket", + $metadata: { httpStatusCode: 404 }, + }); + }); + const service = createStorageService(storageConfig, { client }); + await service.initialize(); + + await assert.rejects( + service.getObject("assets/id/original/hash"), + StorageUnavailableError, + ); + assert.equal(service.health().status, "degraded"); +}); + +test("storage rejects unsafe internal object keys before contacting S3", async () => { + const client = fakeClient(async () => ({})); + const service = createStorageService(storageConfig, { client }); + + await assert.rejects( + service.getObject("../private"), + /Storage object key is invalid/, + ); + await assert.rejects( + service.deleteObject("/absolute"), + /Storage object key is invalid/, + ); + assert.equal(client.commands.length, 0); +}); + +test("real HTTP requests time out instead of hanging indefinitely", async (t) => { + const server = http.createServer(() => { + // Intentionally accept the connection without sending response headers. + }); + server.listen(0, "127.0.0.1"); + await new Promise((resolve) => server.once("listening", resolve)); + t.after(async () => { + server.closeAllConnections(); + await new Promise((resolve) => server.close(resolve)); + }); + + const service = createStorageService({ + ...storageConfig, + endpoint: `http://127.0.0.1:${server.address().port}`, + connectionTimeoutMs: 100, + requestTimeoutMs: 100, + maxAttempts: 1, + }); + t.after(() => service.stop()); + + const startedAt = Date.now(); + const health = await service.probe(); + assert.equal(health.status, "degraded"); + assert.equal(health.code, "STORAGE_UNAVAILABLE"); + assert.ok(Date.now() - startedAt < 2_000); +}); + +test("consumer stream cancellation does not falsely degrade storage", async () => { + let body = new PassThrough(); + const client = fakeClient(async (command) => { + if (commandName(command) === "HeadBucketCommand") return {}; + if (commandName(command) === "GetObjectCommand") { + return { Body: body, ContentLength: 10, ContentType: "image/webp" }; + } + throw new Error(`Unexpected ${commandName(command)}`); + }); + const service = createStorageService(storageConfig, { client }); + await service.initialize(); + + const cancelled = await service.getObject("assets/id/display/hash.webp"); + cancelled.body.destroy( + Object.assign(new Error("consumer closed"), { + code: "ERR_STREAM_PREMATURE_CLOSE", + }), + ); + await new Promise((resolve) => setImmediate(resolve)); + assert.equal(service.health().status, "healthy"); + + body = new PassThrough(); + const failed = await service.getObject("assets/id/display/hash.webp"); + failed.body.destroy(Object.assign(new Error("upstream reset"), { code: "ECONNRESET" })); + await new Promise((resolve) => setImmediate(resolve)); + assert.equal(service.health().status, "degraded"); +}); + +test("delete treats an already-missing object as an idempotent success", async () => { + let missingBucket = false; + const client = fakeClient(async (command) => { + if (commandName(command) === "HeadBucketCommand") return {}; + if (commandName(command) === "DeleteObjectCommand") { + throw Object.assign(new Error("missing"), { + name: missingBucket ? "NoSuchBucket" : "NoSuchKey", + $metadata: { httpStatusCode: 404 }, + }); + } + throw new Error(`Unexpected ${commandName(command)}`); + }); + const service = createStorageService(storageConfig, { client }); + await service.initialize(); + + await assert.doesNotReject( + service.deleteObject("assets/id/original/hash"), + ); + assert.equal(service.health().status, "healthy"); + + missingBucket = true; + await assert.rejects( + service.deleteObject("assets/id/original/hash"), + StorageUnavailableError, + ); +});