Implement Phase 2 asset storage pipeline
This commit is contained in:
Media
@@ -13,9 +13,12 @@
|
||||
"node": ">=22.13.0"
|
||||
},
|
||||
"dependencies": {
|
||||
"@aws-sdk/client-s3": "3.1101.0",
|
||||
"better-sqlite3": "13.0.2",
|
||||
"dotenv": "^16.5.0",
|
||||
"express": "^5.2.1",
|
||||
"multer": "2.0.2",
|
||||
"sharp": "0.35.3",
|
||||
"socket.io": "^4.8.3"
|
||||
}
|
||||
}
|
||||
|
||||
+23
-2
@@ -3,12 +3,27 @@ const {
|
||||
createDetailedHealthRouter,
|
||||
createHealthRouter,
|
||||
} = require("./health/routes");
|
||||
const { createAssetRouter } = require("./assets/routes");
|
||||
const { createMediaRouter } = require("./assets/media-routes");
|
||||
const { createAssetService } = require("./assets/service");
|
||||
const { createSettingsRouter } = require("./settings/routes");
|
||||
const { createSettingsService } = require("./settings/service");
|
||||
|
||||
function createApp({ config, database, lifecycle, migrations }) {
|
||||
function createApp({
|
||||
config,
|
||||
database,
|
||||
imageProcessor,
|
||||
lifecycle,
|
||||
migrations,
|
||||
storage,
|
||||
}) {
|
||||
const app = express();
|
||||
const settingsService = createSettingsService(database);
|
||||
const assetService = createAssetService({
|
||||
database,
|
||||
imageProcessor,
|
||||
storage,
|
||||
});
|
||||
|
||||
app.disable("x-powered-by");
|
||||
app.set("trust proxy", config.trustProxy);
|
||||
@@ -19,13 +34,14 @@ function createApp({ config, database, lifecycle, migrations }) {
|
||||
});
|
||||
app.use(
|
||||
"/health",
|
||||
createHealthRouter({ database, lifecycle, migrations }),
|
||||
createHealthRouter({ database, lifecycle, migrations, storage }),
|
||||
);
|
||||
const detailedHealthRouterOptions = {
|
||||
config,
|
||||
database,
|
||||
lifecycle,
|
||||
migrations,
|
||||
storage,
|
||||
};
|
||||
app.use(
|
||||
"/api/health",
|
||||
@@ -36,6 +52,11 @@ function createApp({ config, database, lifecycle, migrations }) {
|
||||
createDetailedHealthRouter(detailedHealthRouterOptions),
|
||||
);
|
||||
app.use("/api/settings", createSettingsRouter(settingsService));
|
||||
app.use(
|
||||
"/api/assets",
|
||||
createAssetRouter({ assetService, processingConfig: config.processing }),
|
||||
);
|
||||
app.use("/media", createMediaRouter(assetService));
|
||||
|
||||
app.use((request, response) => {
|
||||
response.status(404).json({
|
||||
|
||||
@@ -0,0 +1,23 @@
|
||||
class AssetError extends Error {
|
||||
constructor(code, message, { statusCode = 400, details, cause } = {}) {
|
||||
super(message, cause ? { cause } : undefined);
|
||||
this.name = "AssetError";
|
||||
this.code = code;
|
||||
this.statusCode = statusCode;
|
||||
if (details !== undefined) {
|
||||
this.details = details;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
function assetNotFound(assetId) {
|
||||
return new AssetError("ASSET_NOT_FOUND", "Library asset not found", {
|
||||
statusCode: 404,
|
||||
details: { assetId },
|
||||
});
|
||||
}
|
||||
|
||||
module.exports = {
|
||||
AssetError,
|
||||
assetNotFound,
|
||||
};
|
||||
@@ -0,0 +1,49 @@
|
||||
const express = require("express");
|
||||
const { sendImmutableMedia } = require("./streaming");
|
||||
|
||||
function createMediaRouter(assetService) {
|
||||
const router = express.Router();
|
||||
|
||||
async function sendMedia(request, response, next, kind) {
|
||||
const version = request.params.version;
|
||||
if (!version.endsWith(".webp")) {
|
||||
response.status(404).json({
|
||||
error: {
|
||||
code: "MEDIA_NOT_FOUND",
|
||||
message: "Asset media version not found",
|
||||
},
|
||||
});
|
||||
return;
|
||||
}
|
||||
|
||||
try {
|
||||
const descriptor = assetService.mediaDescriptor(
|
||||
request.params.assetId,
|
||||
kind,
|
||||
version.slice(0, -".webp".length),
|
||||
);
|
||||
await sendImmutableMedia(assetService, response, descriptor);
|
||||
} catch (error) {
|
||||
if (response.headersSent) {
|
||||
response.destroy(error);
|
||||
return;
|
||||
}
|
||||
next(error);
|
||||
}
|
||||
}
|
||||
|
||||
router.get("/assets/:assetId/display/:version", (request, response, next) =>
|
||||
sendMedia(request, response, next, "display"),
|
||||
);
|
||||
router.get(
|
||||
"/assets/:assetId/thumbnail/:version",
|
||||
(request, response, next) =>
|
||||
sendMedia(request, response, next, "thumbnail"),
|
||||
);
|
||||
|
||||
return router;
|
||||
}
|
||||
|
||||
module.exports = {
|
||||
createMediaRouter,
|
||||
};
|
||||
@@ -0,0 +1,235 @@
|
||||
const crypto = require("node:crypto");
|
||||
const fs = require("node:fs");
|
||||
const path = require("node:path");
|
||||
const multer = require("multer");
|
||||
const { AssetError } = require("./errors");
|
||||
|
||||
const ALLOWED_FILE_FIELDS = new Set(["file", "files"]);
|
||||
const ALLOWED_TEXT_FIELDS = new Set(["display_name", "displayName"]);
|
||||
const MAX_FIELD_BYTES = 4 * 1024;
|
||||
const MAX_ORIGINAL_FILENAME_LENGTH = 255;
|
||||
const TEMPORARY_UPLOAD_PATHS = Symbol("temporaryUploadPaths");
|
||||
const TEMP_STORAGE_ERROR_CODES = new Set([
|
||||
"EACCES",
|
||||
"EEXIST",
|
||||
"EIO",
|
||||
"EMFILE",
|
||||
"ENFILE",
|
||||
"ENOSPC",
|
||||
"ENOTDIR",
|
||||
"EPERM",
|
||||
"EROFS",
|
||||
]);
|
||||
|
||||
function safeOriginalFilename(value) {
|
||||
const normalized = String(value || "")
|
||||
.replaceAll("\\", "/")
|
||||
.replace(/[\u0000-\u001f\u007f]/g, "");
|
||||
const leafName = path.posix.basename(normalized);
|
||||
const bounded = [...leafName].slice(0, MAX_ORIGINAL_FILENAME_LENGTH).join("");
|
||||
|
||||
if (!bounded.trim()) {
|
||||
throw new AssetError(
|
||||
"INVALID_ORIGINAL_FILENAME",
|
||||
"Uploaded files must include a usable filename",
|
||||
);
|
||||
}
|
||||
return bounded;
|
||||
}
|
||||
|
||||
function mapMulterError(error, config) {
|
||||
if (!(error instanceof multer.MulterError)) {
|
||||
if (TEMP_STORAGE_ERROR_CODES.has(error?.code)) {
|
||||
return new AssetError(
|
||||
"UPLOAD_TEMP_STORAGE_UNAVAILABLE",
|
||||
"Temporary upload storage is unavailable",
|
||||
{ statusCode: 503, cause: error },
|
||||
);
|
||||
}
|
||||
return new AssetError(
|
||||
"INVALID_MULTIPART_UPLOAD",
|
||||
"The multipart upload could not be read",
|
||||
{ cause: error },
|
||||
);
|
||||
}
|
||||
|
||||
if (error.code === "LIMIT_FILE_SIZE") {
|
||||
return new AssetError(
|
||||
"UPLOAD_FILE_TOO_LARGE",
|
||||
"An uploaded file exceeds the configured size limit",
|
||||
{
|
||||
statusCode: 413,
|
||||
details: { maxFileBytes: config.maxFileBytes },
|
||||
},
|
||||
);
|
||||
}
|
||||
if (error.code === "LIMIT_FILE_COUNT") {
|
||||
return new AssetError(
|
||||
"UPLOAD_FILE_COUNT_EXCEEDED",
|
||||
"The upload contains too many files",
|
||||
{
|
||||
statusCode: 413,
|
||||
details: { maxFiles: config.maxFiles },
|
||||
},
|
||||
);
|
||||
}
|
||||
if (
|
||||
[
|
||||
"LIMIT_FIELD_COUNT",
|
||||
"LIMIT_FIELD_KEY",
|
||||
"LIMIT_FIELD_VALUE",
|
||||
"LIMIT_PART_COUNT",
|
||||
"LIMIT_UNEXPECTED_FILE",
|
||||
].includes(error.code)
|
||||
) {
|
||||
return new AssetError(
|
||||
"INVALID_MULTIPART_UPLOAD",
|
||||
"The multipart upload contains too many or invalid parts",
|
||||
{ details: { reason: error.code } },
|
||||
);
|
||||
}
|
||||
|
||||
return new AssetError(
|
||||
"INVALID_MULTIPART_UPLOAD",
|
||||
"The multipart upload could not be read",
|
||||
{ details: { reason: error.code } },
|
||||
);
|
||||
}
|
||||
|
||||
function createTemporaryStorage(tempDirectory) {
|
||||
return multer.diskStorage({
|
||||
destination(request, file, callback) {
|
||||
fs.mkdir(tempDirectory, { recursive: true, mode: 0o700 }, (error) => {
|
||||
callback(error, tempDirectory);
|
||||
});
|
||||
},
|
||||
filename(request, file, callback) {
|
||||
const temporaryName = crypto.randomUUID();
|
||||
if (!request[TEMPORARY_UPLOAD_PATHS]) {
|
||||
request[TEMPORARY_UPLOAD_PATHS] = [];
|
||||
}
|
||||
request[TEMPORARY_UPLOAD_PATHS].push(
|
||||
path.join(tempDirectory, temporaryName),
|
||||
);
|
||||
callback(null, temporaryName);
|
||||
},
|
||||
});
|
||||
}
|
||||
|
||||
function temporaryUploadFiles(request) {
|
||||
const paths = new Set(request[TEMPORARY_UPLOAD_PATHS] || []);
|
||||
for (const file of Array.isArray(request.files) ? request.files : []) {
|
||||
if (file?.path) paths.add(file.path);
|
||||
}
|
||||
return [...paths].map((temporaryPath) => ({ path: temporaryPath }));
|
||||
}
|
||||
|
||||
function createMultipartUpload(config) {
|
||||
const middleware = multer({
|
||||
storage: createTemporaryStorage(config.tempDirectory),
|
||||
limits: {
|
||||
fileSize: config.maxFileBytes,
|
||||
files: config.maxFiles,
|
||||
fields: 2,
|
||||
parts: config.maxFiles + 2,
|
||||
fieldNameSize: 64,
|
||||
fieldSize: MAX_FIELD_BYTES,
|
||||
headerPairs: 100,
|
||||
},
|
||||
}).any();
|
||||
|
||||
return function receiveAssetUpload(request, response) {
|
||||
return new Promise((resolve, reject) => {
|
||||
middleware(request, response, (error) => {
|
||||
if (error) {
|
||||
reject(mapMulterError(error, config));
|
||||
return;
|
||||
}
|
||||
|
||||
const files = Array.isArray(request.files) ? request.files : [];
|
||||
try {
|
||||
if (files.length === 0) {
|
||||
throw new AssetError(
|
||||
"UPLOAD_FILE_REQUIRED",
|
||||
"At least one image file is required",
|
||||
);
|
||||
}
|
||||
|
||||
const unexpectedFile = files.find(
|
||||
(file) => !ALLOWED_FILE_FIELDS.has(file.fieldname),
|
||||
);
|
||||
if (unexpectedFile) {
|
||||
throw new AssetError(
|
||||
"UNEXPECTED_UPLOAD_FIELD",
|
||||
"Image files must use the file or files multipart field",
|
||||
{ details: { field: unexpectedFile.fieldname } },
|
||||
);
|
||||
}
|
||||
|
||||
const body = request.body || {};
|
||||
const unknownFields = Object.keys(body).filter(
|
||||
(name) => !ALLOWED_TEXT_FIELDS.has(name),
|
||||
);
|
||||
if (unknownFields.length > 0) {
|
||||
throw new AssetError(
|
||||
"UNEXPECTED_UPLOAD_FIELD",
|
||||
"The multipart upload contains unsupported fields",
|
||||
{ details: { fields: unknownFields } },
|
||||
);
|
||||
}
|
||||
|
||||
if (
|
||||
Object.hasOwn(body, "display_name") &&
|
||||
Object.hasOwn(body, "displayName")
|
||||
) {
|
||||
throw new AssetError(
|
||||
"DUPLICATE_DISPLAY_NAME_FIELD",
|
||||
"Provide only one display name field",
|
||||
);
|
||||
}
|
||||
|
||||
const displayName = body.display_name ?? body.displayName ?? null;
|
||||
if (displayName !== null && files.length !== 1) {
|
||||
throw new AssetError(
|
||||
"DISPLAY_NAME_REQUIRES_SINGLE_FILE",
|
||||
"An explicit display name can be used only with one uploaded file",
|
||||
);
|
||||
}
|
||||
|
||||
resolve({
|
||||
displayName,
|
||||
files: files.map((file) => ({
|
||||
path: file.path,
|
||||
byteSize: file.size,
|
||||
originalFilename: safeOriginalFilename(file.originalname),
|
||||
suppliedContentType: file.mimetype,
|
||||
})),
|
||||
});
|
||||
} catch (validationError) {
|
||||
reject(validationError);
|
||||
}
|
||||
});
|
||||
});
|
||||
};
|
||||
}
|
||||
|
||||
async function cleanupTemporaryFiles(files = []) {
|
||||
await Promise.all(
|
||||
files.map(async (file) => {
|
||||
if (!file?.path) return;
|
||||
try {
|
||||
await fs.promises.rm(file.path, { force: true });
|
||||
} catch {
|
||||
// Temporary cleanup is best-effort and must not mask the API result.
|
||||
}
|
||||
}),
|
||||
);
|
||||
}
|
||||
|
||||
module.exports = {
|
||||
cleanupTemporaryFiles,
|
||||
createMultipartUpload,
|
||||
mapMulterError,
|
||||
safeOriginalFilename,
|
||||
temporaryUploadFiles,
|
||||
};
|
||||
@@ -0,0 +1,102 @@
|
||||
const express = require("express");
|
||||
const {
|
||||
cleanupTemporaryFiles,
|
||||
createMultipartUpload,
|
||||
temporaryUploadFiles,
|
||||
} = require("./multipart");
|
||||
const { sendOriginal } = require("./streaming");
|
||||
|
||||
function createAssetRouter({ assetService, processingConfig }) {
|
||||
const router = express.Router();
|
||||
const receiveUpload = createMultipartUpload(processingConfig);
|
||||
|
||||
router.get("/", (request, response, next) => {
|
||||
try {
|
||||
response.json({ assets: assetService.list() });
|
||||
} catch (error) {
|
||||
next(error);
|
||||
}
|
||||
});
|
||||
|
||||
router.post("/", async (request, response, next) => {
|
||||
const cleanupAfterAbort = () => {
|
||||
void cleanupTemporaryFiles(temporaryUploadFiles(request));
|
||||
};
|
||||
request.once("aborted", cleanupAfterAbort);
|
||||
|
||||
try {
|
||||
const upload = await receiveUpload(request, response);
|
||||
const results = await assetService.upload(upload.files, {
|
||||
displayName: upload.displayName,
|
||||
});
|
||||
const createdCount = results.filter((result) => result.created).length;
|
||||
|
||||
if (results.length === 1) {
|
||||
response.status(createdCount > 0 ? 201 : 200).json(results[0]);
|
||||
return;
|
||||
}
|
||||
|
||||
response.status(createdCount > 0 ? 201 : 200).json({
|
||||
results,
|
||||
createdCount,
|
||||
duplicateCount: results.length - createdCount,
|
||||
});
|
||||
} catch (error) {
|
||||
next(error);
|
||||
} finally {
|
||||
request.off("aborted", cleanupAfterAbort);
|
||||
await cleanupTemporaryFiles(temporaryUploadFiles(request));
|
||||
}
|
||||
});
|
||||
|
||||
router.get("/:assetId/references", (request, response, next) => {
|
||||
try {
|
||||
response.json(assetService.references(request.params.assetId));
|
||||
} catch (error) {
|
||||
next(error);
|
||||
}
|
||||
});
|
||||
|
||||
router.get("/:assetId/original", async (request, response, next) => {
|
||||
try {
|
||||
const descriptor = assetService.originalDescriptor(request.params.assetId);
|
||||
await sendOriginal(assetService, response, descriptor);
|
||||
} catch (error) {
|
||||
if (response.headersSent) {
|
||||
response.destroy(error);
|
||||
return;
|
||||
}
|
||||
next(error);
|
||||
}
|
||||
});
|
||||
|
||||
router.get("/:assetId", (request, response, next) => {
|
||||
try {
|
||||
response.json(assetService.get(request.params.assetId));
|
||||
} catch (error) {
|
||||
next(error);
|
||||
}
|
||||
});
|
||||
|
||||
router.patch("/:assetId", (request, response, next) => {
|
||||
try {
|
||||
response.json(assetService.update(request.params.assetId, request.body));
|
||||
} catch (error) {
|
||||
next(error);
|
||||
}
|
||||
});
|
||||
|
||||
router.delete("/:assetId", async (request, response, next) => {
|
||||
try {
|
||||
response.json(await assetService.remove(request.params.assetId));
|
||||
} catch (error) {
|
||||
next(error);
|
||||
}
|
||||
});
|
||||
|
||||
return router;
|
||||
}
|
||||
|
||||
module.exports = {
|
||||
createAssetRouter,
|
||||
};
|
||||
@@ -0,0 +1,835 @@
|
||||
const crypto = require("node:crypto");
|
||||
const fs = require("node:fs");
|
||||
const path = require("node:path");
|
||||
const { AssetError, assetNotFound } = require("./errors");
|
||||
const { StorageObjectNotFoundError } = require("../storage");
|
||||
|
||||
const MAX_DISPLAY_NAME_LENGTH = 200;
|
||||
const MAX_PROCESSING_ERROR_LENGTH = 1_000;
|
||||
const CONCURRENT_UPLOAD_WAIT_MS = 15_000;
|
||||
const CONCURRENT_UPLOAD_POLL_MS = 50;
|
||||
const SHA256_PATTERN = /^[0-9a-f]{64}$/;
|
||||
|
||||
function delay(milliseconds) {
|
||||
return new Promise((resolve) => setTimeout(resolve, milliseconds));
|
||||
}
|
||||
|
||||
function normalizeDisplayName(value) {
|
||||
if (typeof value !== "string") {
|
||||
throw new AssetError(
|
||||
"INVALID_ASSET_DISPLAY_NAME",
|
||||
"Asset display name must be a string",
|
||||
);
|
||||
}
|
||||
|
||||
const normalized = value
|
||||
.replace(/[\u0000-\u001f\u007f]/g, " ")
|
||||
.replace(/\s+/g, " ")
|
||||
.trim();
|
||||
if (!normalized) {
|
||||
throw new AssetError(
|
||||
"INVALID_ASSET_DISPLAY_NAME",
|
||||
"Asset display name cannot be empty",
|
||||
);
|
||||
}
|
||||
if ([...normalized].length > MAX_DISPLAY_NAME_LENGTH) {
|
||||
throw new AssetError(
|
||||
"INVALID_ASSET_DISPLAY_NAME",
|
||||
`Asset display name cannot exceed ${MAX_DISPLAY_NAME_LENGTH} characters`,
|
||||
{ details: { maxLength: MAX_DISPLAY_NAME_LENGTH } },
|
||||
);
|
||||
}
|
||||
return normalized;
|
||||
}
|
||||
|
||||
function defaultDisplayName(originalFilename) {
|
||||
const extension = path.extname(originalFilename);
|
||||
const withoutExtension = extension
|
||||
? originalFilename.slice(0, -extension.length)
|
||||
: originalFilename;
|
||||
return normalizeDisplayName(withoutExtension);
|
||||
}
|
||||
|
||||
function normalizeExpiration(value) {
|
||||
if (value === null) return null;
|
||||
if (Number.isSafeInteger(value) && value >= 0) return value;
|
||||
|
||||
if (typeof value === "string" && value.trim()) {
|
||||
const timestamp = Date.parse(value);
|
||||
if (Number.isSafeInteger(timestamp) && timestamp >= 0) {
|
||||
return timestamp;
|
||||
}
|
||||
}
|
||||
|
||||
throw new AssetError(
|
||||
"INVALID_ASSET_EXPIRATION",
|
||||
"Asset expiration must be null, an epoch-millisecond integer, or an ISO date",
|
||||
);
|
||||
}
|
||||
|
||||
function mediaUrls(row) {
|
||||
if (row.status !== "ready") {
|
||||
return {
|
||||
originalUrl: null,
|
||||
displayUrl: null,
|
||||
thumbnailUrl: null,
|
||||
};
|
||||
}
|
||||
return {
|
||||
originalUrl: `/api/assets/${row.id}/original`,
|
||||
displayUrl: `/media/assets/${row.id}/display/${row.display_sha256}.webp`,
|
||||
thumbnailUrl: `/media/assets/${row.id}/thumbnail/${row.thumbnail_sha256}.webp`,
|
||||
};
|
||||
}
|
||||
|
||||
function mapAsset(row) {
|
||||
if (!row) return null;
|
||||
const urls = mediaUrls(row);
|
||||
return {
|
||||
id: row.id,
|
||||
displayName: row.display_name,
|
||||
originalFilename: row.original_filename,
|
||||
status: row.status,
|
||||
processingError: row.processing_error || null,
|
||||
sourceSha256: row.source_sha256,
|
||||
expirationAt: row.expiration_at ?? null,
|
||||
createdAt: row.created_at,
|
||||
updatedAt: row.updated_at,
|
||||
referenceCount: Number(row.reference_count || 0),
|
||||
original: row.original_content_type
|
||||
? {
|
||||
contentType: row.original_content_type,
|
||||
byteSize: row.original_byte_size,
|
||||
width: row.original_width,
|
||||
height: row.original_height,
|
||||
url: urls.originalUrl,
|
||||
}
|
||||
: null,
|
||||
display: row.display_content_type
|
||||
? {
|
||||
sha256: row.display_sha256,
|
||||
contentType: row.display_content_type,
|
||||
byteSize: row.display_byte_size,
|
||||
width: row.display_width,
|
||||
height: row.display_height,
|
||||
url: urls.displayUrl,
|
||||
}
|
||||
: null,
|
||||
thumbnail: row.thumbnail_content_type
|
||||
? {
|
||||
sha256: row.thumbnail_sha256,
|
||||
contentType: row.thumbnail_content_type,
|
||||
byteSize: row.thumbnail_byte_size,
|
||||
width: row.thumbnail_width,
|
||||
height: row.thumbnail_height,
|
||||
url: urls.thumbnailUrl,
|
||||
}
|
||||
: null,
|
||||
};
|
||||
}
|
||||
|
||||
async function hashFile(filename) {
|
||||
const hash = crypto.createHash("sha256");
|
||||
let byteSize = 0;
|
||||
for await (const chunk of fs.createReadStream(filename)) {
|
||||
byteSize += chunk.length;
|
||||
hash.update(chunk);
|
||||
}
|
||||
return { sourceSha256: hash.digest("hex"), byteSize };
|
||||
}
|
||||
|
||||
function objectKeys(assetId, processed) {
|
||||
return {
|
||||
original: `assets/${assetId}/original/${processed.original.sha256}`,
|
||||
display: `assets/${assetId}/display/${processed.display.sha256}.webp`,
|
||||
thumbnail: `assets/${assetId}/thumbnail/${processed.thumbnail.sha256}.webp`,
|
||||
};
|
||||
}
|
||||
|
||||
function boundedProcessingError(error) {
|
||||
const code = typeof error?.code === "string" ? error.code : "PROCESSING_FAILED";
|
||||
const message =
|
||||
typeof error?.message === "string" && error.message
|
||||
? error.message
|
||||
: "Asset processing failed";
|
||||
return `${code}: ${message}`.slice(0, MAX_PROCESSING_ERROR_LENGTH);
|
||||
}
|
||||
|
||||
function createAssetService({
|
||||
database,
|
||||
storage,
|
||||
imageProcessor,
|
||||
now = Date.now,
|
||||
createId = crypto.randomUUID,
|
||||
wait = delay,
|
||||
concurrentUploadWaitMs = CONCURRENT_UPLOAD_WAIT_MS,
|
||||
}) {
|
||||
const assetSelect = `
|
||||
SELECT a.*,
|
||||
(SELECT COUNT(*) FROM playlist_entries pe WHERE pe.asset_id = a.id)
|
||||
AS reference_count
|
||||
FROM assets a
|
||||
`;
|
||||
const listAssets = database.prepare(`${assetSelect} ORDER BY a.created_at DESC, a.id`);
|
||||
const findById = database.prepare(`${assetSelect} WHERE a.id = ?`);
|
||||
const findByHash = database.prepare(`${assetSelect} WHERE a.source_sha256 = ?`);
|
||||
const findByName = database.prepare(`${assetSelect} WHERE a.display_name = ? COLLATE NOCASE`);
|
||||
const insertProcessing = database.prepare(`
|
||||
INSERT INTO assets (
|
||||
id, source_sha256, display_name, original_filename, status,
|
||||
created_at, updated_at
|
||||
) VALUES (
|
||||
@id, @sourceSha256, @displayName, @originalFilename, 'processing',
|
||||
@createdAt, @updatedAt
|
||||
)
|
||||
`);
|
||||
const markReady = database.prepare(`
|
||||
UPDATE assets
|
||||
SET status = 'ready',
|
||||
processing_error = NULL,
|
||||
original_object_key = @originalObjectKey,
|
||||
original_content_type = @originalContentType,
|
||||
original_byte_size = @originalByteSize,
|
||||
original_width = @originalWidth,
|
||||
original_height = @originalHeight,
|
||||
display_object_key = @displayObjectKey,
|
||||
display_sha256 = @displaySha256,
|
||||
display_content_type = @displayContentType,
|
||||
display_byte_size = @displayByteSize,
|
||||
display_width = @displayWidth,
|
||||
display_height = @displayHeight,
|
||||
thumbnail_object_key = @thumbnailObjectKey,
|
||||
thumbnail_sha256 = @thumbnailSha256,
|
||||
thumbnail_content_type = @thumbnailContentType,
|
||||
thumbnail_byte_size = @thumbnailByteSize,
|
||||
thumbnail_width = @thumbnailWidth,
|
||||
thumbnail_height = @thumbnailHeight,
|
||||
updated_at = @updatedAt
|
||||
WHERE id = @id AND status = 'processing'
|
||||
`);
|
||||
const markFailed = database.prepare(`
|
||||
UPDATE assets
|
||||
SET status = 'failed',
|
||||
processing_error = @processingError,
|
||||
original_object_key = @originalObjectKey,
|
||||
original_content_type = @originalContentType,
|
||||
original_byte_size = @originalByteSize,
|
||||
original_width = @originalWidth,
|
||||
original_height = @originalHeight,
|
||||
display_object_key = @displayObjectKey,
|
||||
display_sha256 = @displaySha256,
|
||||
display_content_type = @displayContentType,
|
||||
display_byte_size = @displayByteSize,
|
||||
display_width = @displayWidth,
|
||||
display_height = @displayHeight,
|
||||
thumbnail_object_key = @thumbnailObjectKey,
|
||||
thumbnail_sha256 = @thumbnailSha256,
|
||||
thumbnail_content_type = @thumbnailContentType,
|
||||
thumbnail_byte_size = @thumbnailByteSize,
|
||||
thumbnail_width = @thumbnailWidth,
|
||||
thumbnail_height = @thumbnailHeight,
|
||||
updated_at = @updatedAt
|
||||
WHERE id = @id AND status = 'processing'
|
||||
`);
|
||||
const updateEditable = database.prepare(`
|
||||
UPDATE assets
|
||||
SET display_name = @displayName,
|
||||
expiration_at = @expirationAt,
|
||||
updated_at = @updatedAt
|
||||
WHERE id = @id AND status != 'deleting'
|
||||
`);
|
||||
const playlistReferences = database.prepare(`
|
||||
SELECT
|
||||
pe.id AS entry_id,
|
||||
pe.playlist_id,
|
||||
p.name AS playlist_name,
|
||||
pe.enabled,
|
||||
pe.position,
|
||||
EXISTS (
|
||||
SELECT 1 FROM playback_state active
|
||||
WHERE active.active_playlist_id = pe.playlist_id
|
||||
) AS is_active_playlist,
|
||||
EXISTS (
|
||||
SELECT 1 FROM playback_state current
|
||||
WHERE current.current_entry_id = pe.id
|
||||
) AS is_current_normal_target
|
||||
FROM playlist_entries pe
|
||||
JOIN playlists p ON p.id = pe.playlist_id
|
||||
WHERE pe.asset_id = ?
|
||||
ORDER BY p.name COLLATE NOCASE, pe.position, pe.id
|
||||
`);
|
||||
const playbackReferences = database.prepare(`
|
||||
SELECT
|
||||
output_id,
|
||||
mode,
|
||||
active_playlist_id,
|
||||
current_entry_id,
|
||||
static_asset_id
|
||||
FROM playback_state
|
||||
WHERE static_asset_id = ?
|
||||
OR current_entry_id IN (
|
||||
SELECT id FROM playlist_entries WHERE asset_id = ?
|
||||
)
|
||||
ORDER BY output_id
|
||||
`);
|
||||
const markDeleting = database.prepare(`
|
||||
UPDATE assets
|
||||
SET status = 'deleting', updated_at = ?
|
||||
WHERE id = ? AND status != 'processing'
|
||||
`);
|
||||
const deleteAssetRow = database.prepare(
|
||||
"DELETE FROM assets WHERE id = ? AND status = 'deleting'",
|
||||
);
|
||||
|
||||
const completeAsset = database.transaction((metadata) => {
|
||||
const result = markReady.run(metadata);
|
||||
if (result.changes !== 1) {
|
||||
throw new Error("Asset processing state changed before completion");
|
||||
}
|
||||
return findById.get(metadata.id);
|
||||
});
|
||||
|
||||
function getRaw(assetId) {
|
||||
const row = findById.get(assetId);
|
||||
if (!row) throw assetNotFound(assetId);
|
||||
return row;
|
||||
}
|
||||
|
||||
function duplicateStateError(row) {
|
||||
if (row.status === "failed") {
|
||||
return new AssetError(
|
||||
"ASSET_PROCESSING_FAILED",
|
||||
"These bytes match an asset whose processing previously failed",
|
||||
{ statusCode: 409, details: { asset: mapAsset(row) } },
|
||||
);
|
||||
}
|
||||
if (row.status === "deleting") {
|
||||
return new AssetError(
|
||||
"ASSET_DELETING",
|
||||
"These bytes match an asset that is being deleted",
|
||||
{ statusCode: 409, details: { assetId: row.id } },
|
||||
);
|
||||
}
|
||||
return new AssetError(
|
||||
"ASSET_PROCESSING",
|
||||
"These bytes are already being processed",
|
||||
{ statusCode: 409, details: { assetId: row.id } },
|
||||
);
|
||||
}
|
||||
|
||||
async function resolveExistingHash(row) {
|
||||
if (row.status === "ready") {
|
||||
return { created: false, duplicate: true, asset: mapAsset(row) };
|
||||
}
|
||||
if (row.status !== "processing") {
|
||||
throw duplicateStateError(row);
|
||||
}
|
||||
|
||||
const deadline = Date.now() + concurrentUploadWaitMs;
|
||||
while (Date.now() < deadline) {
|
||||
await wait(CONCURRENT_UPLOAD_POLL_MS);
|
||||
const latest = findByHash.get(row.source_sha256);
|
||||
if (!latest) break;
|
||||
if (latest.status === "ready") {
|
||||
return { created: false, duplicate: true, asset: mapAsset(latest) };
|
||||
}
|
||||
if (latest.status !== "processing") {
|
||||
throw duplicateStateError(latest);
|
||||
}
|
||||
}
|
||||
throw duplicateStateError(findByHash.get(row.source_sha256) || row);
|
||||
}
|
||||
|
||||
function nameConflict(displayName, row = findByName.get(displayName)) {
|
||||
return new AssetError(
|
||||
"ASSET_NAME_CONFLICT",
|
||||
"Another library asset already uses that display name",
|
||||
{
|
||||
statusCode: 409,
|
||||
details: row ? { asset: mapAsset(row) } : { displayName },
|
||||
},
|
||||
);
|
||||
}
|
||||
|
||||
function processingMetadata(id, processed, keys, error) {
|
||||
return {
|
||||
id,
|
||||
processingError: error ? boundedProcessingError(error) : null,
|
||||
originalObjectKey: keys.original,
|
||||
originalContentType: processed.original.contentType,
|
||||
originalByteSize: processed.original.byteSize,
|
||||
originalWidth: processed.original.width,
|
||||
originalHeight: processed.original.height,
|
||||
displayObjectKey: keys.display,
|
||||
displaySha256: processed.display.sha256,
|
||||
displayContentType: processed.display.contentType,
|
||||
displayByteSize: processed.display.byteSize,
|
||||
displayWidth: processed.display.width,
|
||||
displayHeight: processed.display.height,
|
||||
thumbnailObjectKey: keys.thumbnail,
|
||||
thumbnailSha256: processed.thumbnail.sha256,
|
||||
thumbnailContentType: processed.thumbnail.contentType,
|
||||
thumbnailByteSize: processed.thumbnail.byteSize,
|
||||
thumbnailWidth: processed.thumbnail.width,
|
||||
thumbnailHeight: processed.thumbnail.height,
|
||||
updatedAt: now(),
|
||||
};
|
||||
}
|
||||
|
||||
async function cleanObjects(keys) {
|
||||
const results = await Promise.allSettled(
|
||||
[...new Set(Object.values(keys).filter(Boolean))].map((key) =>
|
||||
storage.deleteObject(key),
|
||||
),
|
||||
);
|
||||
const rejected = results.find((result) => result.status === "rejected");
|
||||
if (rejected) throw rejected.reason;
|
||||
}
|
||||
|
||||
async function createProcessedAsset(plan) {
|
||||
const id = createId();
|
||||
const timestamp = now();
|
||||
try {
|
||||
insertProcessing.run({
|
||||
id,
|
||||
sourceSha256: plan.sourceSha256,
|
||||
displayName: plan.displayName,
|
||||
originalFilename: plan.file.originalFilename,
|
||||
createdAt: timestamp,
|
||||
updatedAt: timestamp,
|
||||
});
|
||||
} catch (error) {
|
||||
const hashMatch = findByHash.get(plan.sourceSha256);
|
||||
if (hashMatch) return resolveExistingHash(hashMatch);
|
||||
const nameMatch = findByName.get(plan.displayName);
|
||||
if (nameMatch) throw nameConflict(plan.displayName, nameMatch);
|
||||
throw error;
|
||||
}
|
||||
|
||||
const keys = objectKeys(id, plan.processed);
|
||||
try {
|
||||
await storage.putObject({
|
||||
key: keys.original,
|
||||
body: fs.createReadStream(plan.file.path),
|
||||
contentType: plan.processed.original.contentType,
|
||||
contentLength: plan.processed.original.byteSize,
|
||||
metadata: { sha256: plan.processed.original.sha256 },
|
||||
});
|
||||
await storage.putObject({
|
||||
key: keys.display,
|
||||
body: plan.processed.display.buffer,
|
||||
contentType: plan.processed.display.contentType,
|
||||
contentLength: plan.processed.display.byteSize,
|
||||
metadata: { sha256: plan.processed.display.sha256 },
|
||||
});
|
||||
await storage.putObject({
|
||||
key: keys.thumbnail,
|
||||
body: plan.processed.thumbnail.buffer,
|
||||
contentType: plan.processed.thumbnail.contentType,
|
||||
contentLength: plan.processed.thumbnail.byteSize,
|
||||
metadata: { sha256: plan.processed.thumbnail.sha256 },
|
||||
});
|
||||
|
||||
const row = completeAsset(processingMetadata(id, plan.processed, keys));
|
||||
return { created: true, duplicate: false, asset: mapAsset(row) };
|
||||
} catch (error) {
|
||||
const failure = processingMetadata(id, plan.processed, keys, error);
|
||||
try {
|
||||
markFailed.run(failure);
|
||||
} catch {
|
||||
// Preserve the original failure; the processing row remains diagnosable.
|
||||
}
|
||||
try {
|
||||
await cleanObjects(keys);
|
||||
} catch {
|
||||
// The failed row retains deterministic keys so DELETE can retry cleanup.
|
||||
}
|
||||
|
||||
if (error?.statusCode) {
|
||||
error.details = { ...(error.details || {}), assetId: id, retryable: true };
|
||||
}
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
|
||||
async function upload(files, { displayName = null } = {}) {
|
||||
if (!Array.isArray(files) || files.length === 0) {
|
||||
throw new AssetError(
|
||||
"UPLOAD_FILE_REQUIRED",
|
||||
"At least one image file is required",
|
||||
);
|
||||
}
|
||||
|
||||
const plans = [];
|
||||
const firstPlanByHash = new Map();
|
||||
const plannedNames = new Map();
|
||||
|
||||
for (const [index, file] of files.entries()) {
|
||||
const identity = await hashFile(file.path);
|
||||
if (identity.byteSize !== file.byteSize) {
|
||||
throw new AssetError(
|
||||
"UPLOAD_CHANGED_DURING_PROCESSING",
|
||||
"An uploaded temporary file changed before processing",
|
||||
{ statusCode: 409 },
|
||||
);
|
||||
}
|
||||
|
||||
const existing = findByHash.get(identity.sourceSha256);
|
||||
if (existing) {
|
||||
plans.push({ index, existing });
|
||||
continue;
|
||||
}
|
||||
|
||||
const priorPlan = firstPlanByHash.get(identity.sourceSha256);
|
||||
if (priorPlan) {
|
||||
plans.push({ index, duplicateOf: priorPlan });
|
||||
continue;
|
||||
}
|
||||
|
||||
const proposedName = normalizeDisplayName(
|
||||
displayName ?? defaultDisplayName(file.originalFilename),
|
||||
);
|
||||
const existingName = findByName.get(proposedName);
|
||||
if (existingName) throw nameConflict(proposedName, existingName);
|
||||
|
||||
const normalizedNameKey = proposedName.toLowerCase();
|
||||
const batchNameMatch = plannedNames.get(normalizedNameKey);
|
||||
if (batchNameMatch) throw nameConflict(proposedName);
|
||||
|
||||
const plan = {
|
||||
index,
|
||||
file,
|
||||
displayName: proposedName,
|
||||
sourceSha256: identity.sourceSha256,
|
||||
};
|
||||
firstPlanByHash.set(identity.sourceSha256, plan);
|
||||
plannedNames.set(normalizedNameKey, plan);
|
||||
plans.push(plan);
|
||||
}
|
||||
|
||||
// Decode every new image before any database or object-storage mutation.
|
||||
for (const plan of plans) {
|
||||
if (plan.existing || plan.duplicateOf) continue;
|
||||
plan.processed = await imageProcessor.process(plan.file.path);
|
||||
if (plan.processed.original.sha256 !== plan.sourceSha256) {
|
||||
throw new AssetError(
|
||||
"UPLOAD_CHANGED_DURING_PROCESSING",
|
||||
"An uploaded temporary file changed before processing",
|
||||
{ statusCode: 409 },
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
const resultByPlan = new Map();
|
||||
const results = new Array(files.length);
|
||||
for (const plan of plans) {
|
||||
let result;
|
||||
if (plan.existing) {
|
||||
result = await resolveExistingHash(plan.existing);
|
||||
} else if (plan.duplicateOf) {
|
||||
const originalResult = resultByPlan.get(plan.duplicateOf);
|
||||
result = {
|
||||
created: false,
|
||||
duplicate: true,
|
||||
asset: originalResult.asset,
|
||||
};
|
||||
} else {
|
||||
result = await createProcessedAsset(plan);
|
||||
resultByPlan.set(plan, result);
|
||||
}
|
||||
results[plan.index] = result;
|
||||
}
|
||||
return results;
|
||||
}
|
||||
|
||||
function list() {
|
||||
return listAssets.all().map(mapAsset);
|
||||
}
|
||||
|
||||
function get(assetId) {
|
||||
return mapAsset(getRaw(assetId));
|
||||
}
|
||||
|
||||
function update(assetId, input) {
|
||||
if (input === null || typeof input !== "object" || Array.isArray(input)) {
|
||||
throw new AssetError("INVALID_ASSET_UPDATE", "Asset update must be an object");
|
||||
}
|
||||
|
||||
const allowed = new Set([
|
||||
"displayName",
|
||||
"display_name",
|
||||
"expirationAt",
|
||||
"expiration_at",
|
||||
]);
|
||||
const unknown = Object.keys(input).filter((key) => !allowed.has(key));
|
||||
if (unknown.length > 0) {
|
||||
throw new AssetError(
|
||||
"INVALID_ASSET_UPDATE",
|
||||
"Asset update contains unsupported fields",
|
||||
{ details: { fields: unknown } },
|
||||
);
|
||||
}
|
||||
if (Object.keys(input).length === 0) {
|
||||
throw new AssetError("INVALID_ASSET_UPDATE", "At least one field is required");
|
||||
}
|
||||
if (
|
||||
Object.hasOwn(input, "displayName") &&
|
||||
Object.hasOwn(input, "display_name")
|
||||
) {
|
||||
throw new AssetError("INVALID_ASSET_UPDATE", "Display name was provided twice");
|
||||
}
|
||||
if (
|
||||
Object.hasOwn(input, "expirationAt") &&
|
||||
Object.hasOwn(input, "expiration_at")
|
||||
) {
|
||||
throw new AssetError("INVALID_ASSET_UPDATE", "Expiration was provided twice");
|
||||
}
|
||||
|
||||
const current = getRaw(assetId);
|
||||
if (current.status === "deleting") {
|
||||
throw new AssetError("ASSET_DELETING", "Deleting assets cannot be edited", {
|
||||
statusCode: 409,
|
||||
details: { assetId },
|
||||
});
|
||||
}
|
||||
const hasDisplayName =
|
||||
Object.hasOwn(input, "displayName") || Object.hasOwn(input, "display_name");
|
||||
const hasExpiration =
|
||||
Object.hasOwn(input, "expirationAt") || Object.hasOwn(input, "expiration_at");
|
||||
const nextDisplayName = hasDisplayName
|
||||
? normalizeDisplayName(
|
||||
Object.hasOwn(input, "displayName")
|
||||
? input.displayName
|
||||
: input.display_name,
|
||||
)
|
||||
: current.display_name;
|
||||
const nextExpiration = hasExpiration
|
||||
? normalizeExpiration(
|
||||
Object.hasOwn(input, "expirationAt")
|
||||
? input.expirationAt
|
||||
: input.expiration_at,
|
||||
)
|
||||
: current.expiration_at;
|
||||
|
||||
if (
|
||||
nextDisplayName === current.display_name &&
|
||||
nextExpiration === current.expiration_at
|
||||
) {
|
||||
return mapAsset(current);
|
||||
}
|
||||
|
||||
try {
|
||||
const result = updateEditable.run({
|
||||
id: assetId,
|
||||
displayName: nextDisplayName,
|
||||
expirationAt: nextExpiration,
|
||||
updatedAt: Math.max(now(), current.updated_at + 1),
|
||||
});
|
||||
if (result.changes !== 1) {
|
||||
throw new AssetError("ASSET_DELETING", "Deleting assets cannot be edited", {
|
||||
statusCode: 409,
|
||||
details: { assetId },
|
||||
});
|
||||
}
|
||||
} catch (error) {
|
||||
const nameMatch = findByName.get(nextDisplayName);
|
||||
if (nameMatch && nameMatch.id !== assetId) {
|
||||
throw nameConflict(nextDisplayName, nameMatch);
|
||||
}
|
||||
throw error;
|
||||
}
|
||||
return mapAsset(getRaw(assetId));
|
||||
}
|
||||
|
||||
function references(assetId) {
|
||||
const asset = getRaw(assetId);
|
||||
const playlists = playlistReferences.all(assetId).map((row) => ({
|
||||
playlistId: row.playlist_id,
|
||||
playlistName: row.playlist_name,
|
||||
playlistEntryId: row.entry_id,
|
||||
enabled: Boolean(row.enabled),
|
||||
position: row.position,
|
||||
isActivePlaylist: Boolean(row.is_active_playlist),
|
||||
isCurrentNormalTarget: Boolean(row.is_current_normal_target),
|
||||
}));
|
||||
const playback = playbackReferences.all(assetId, assetId).map((row) => ({
|
||||
outputId: row.output_id,
|
||||
mode: row.mode,
|
||||
activePlaylistId: row.active_playlist_id,
|
||||
currentPlaylistEntryId: row.current_entry_id,
|
||||
isCurrentNormalTarget: playlists.some(
|
||||
(entry) => entry.playlistEntryId === row.current_entry_id,
|
||||
),
|
||||
isCurrentStaticTarget: row.static_asset_id === assetId,
|
||||
}));
|
||||
const currentNormalTarget = playback.some((state) => state.isCurrentNormalTarget);
|
||||
const currentStaticTarget = playback.some((state) => state.isCurrentStaticTarget);
|
||||
const blockerCodes = [];
|
||||
if (playlists.length > 0) blockerCodes.push("PLAYLIST_REFERENCES");
|
||||
if (currentNormalTarget) blockerCodes.push("CURRENT_PLAYBACK_TARGET");
|
||||
if (currentStaticTarget) blockerCodes.push("CURRENT_STATIC_TARGET");
|
||||
if (asset.status === "processing") blockerCodes.push("ASSET_PROCESSING");
|
||||
if (asset.status === "deleting") blockerCodes.push("ASSET_DELETING");
|
||||
|
||||
return {
|
||||
assetId,
|
||||
playlists,
|
||||
playback,
|
||||
currentNormalTarget,
|
||||
currentStaticTarget,
|
||||
canDelete: blockerCodes.length === 0,
|
||||
blockerCodes,
|
||||
};
|
||||
}
|
||||
|
||||
const beginDelete = database.transaction((assetId) => {
|
||||
const asset = getRaw(assetId);
|
||||
const referenceInfo = references(assetId);
|
||||
const relationshipBlockers = referenceInfo.blockerCodes.filter((code) =>
|
||||
[
|
||||
"PLAYLIST_REFERENCES",
|
||||
"CURRENT_PLAYBACK_TARGET",
|
||||
"CURRENT_STATIC_TARGET",
|
||||
].includes(code),
|
||||
);
|
||||
if (relationshipBlockers.length > 0) {
|
||||
throw new AssetError(
|
||||
referenceInfo.playlists.length > 0 ? "ASSET_REFERENCED" : "ASSET_IN_USE",
|
||||
"The library asset is still referenced and cannot be deleted",
|
||||
{
|
||||
statusCode: 409,
|
||||
details: { ...referenceInfo, blockerCodes: relationshipBlockers },
|
||||
},
|
||||
);
|
||||
}
|
||||
if (asset.status === "processing") {
|
||||
throw new AssetError(
|
||||
"ASSET_PROCESSING",
|
||||
"An asset cannot be deleted while it is processing",
|
||||
{ statusCode: 409, details: { assetId } },
|
||||
);
|
||||
}
|
||||
if (asset.status !== "deleting") {
|
||||
markDeleting.run(Math.max(now(), asset.updated_at + 1), assetId);
|
||||
}
|
||||
return findById.get(assetId);
|
||||
});
|
||||
|
||||
async function remove(assetId) {
|
||||
const deleting = beginDelete(assetId);
|
||||
const keys = {
|
||||
original: deleting.original_object_key,
|
||||
display: deleting.display_object_key,
|
||||
thumbnail: deleting.thumbnail_object_key,
|
||||
};
|
||||
|
||||
try {
|
||||
await cleanObjects(keys);
|
||||
} catch (error) {
|
||||
if (error?.statusCode) {
|
||||
error.details = { ...(error.details || {}), assetId, retryable: true };
|
||||
}
|
||||
throw error;
|
||||
}
|
||||
|
||||
try {
|
||||
const result = deleteAssetRow.run(assetId);
|
||||
if (result.changes !== 1) throw assetNotFound(assetId);
|
||||
} catch (error) {
|
||||
if (error?.code?.startsWith?.("SQLITE_CONSTRAINT")) {
|
||||
throw new AssetError(
|
||||
"ASSET_REFERENCED",
|
||||
"The library asset became referenced before deletion completed",
|
||||
{ statusCode: 409, details: { assetId, retryable: true } },
|
||||
);
|
||||
}
|
||||
throw error;
|
||||
}
|
||||
return { id: assetId, deleted: true };
|
||||
}
|
||||
|
||||
function mediaDescriptor(assetId, kind, requestedHash) {
|
||||
if (!SHA256_PATTERN.test(requestedHash)) {
|
||||
throw new AssetError("MEDIA_NOT_FOUND", "Asset media version not found", {
|
||||
statusCode: 404,
|
||||
});
|
||||
}
|
||||
const asset = getRaw(assetId);
|
||||
const fields =
|
||||
kind === "display"
|
||||
? {
|
||||
hash: asset.display_sha256,
|
||||
key: asset.display_object_key,
|
||||
contentType: asset.display_content_type,
|
||||
contentLength: asset.display_byte_size,
|
||||
}
|
||||
: {
|
||||
hash: asset.thumbnail_sha256,
|
||||
key: asset.thumbnail_object_key,
|
||||
contentType: asset.thumbnail_content_type,
|
||||
contentLength: asset.thumbnail_byte_size,
|
||||
};
|
||||
if (asset.status !== "ready" || fields.hash !== requestedHash || !fields.key) {
|
||||
throw new AssetError("MEDIA_NOT_FOUND", "Asset media version not found", {
|
||||
statusCode: 404,
|
||||
});
|
||||
}
|
||||
return { asset, kind, ...fields };
|
||||
}
|
||||
|
||||
function originalDescriptor(assetId) {
|
||||
const asset = getRaw(assetId);
|
||||
if (asset.status !== "ready" || !asset.original_object_key) {
|
||||
throw new AssetError("ASSET_ORIGINAL_NOT_READY", "Asset original is not ready", {
|
||||
statusCode: 409,
|
||||
details: { assetId, status: asset.status },
|
||||
});
|
||||
}
|
||||
return {
|
||||
asset,
|
||||
key: asset.original_object_key,
|
||||
hash: asset.source_sha256,
|
||||
contentType: asset.original_content_type,
|
||||
contentLength: asset.original_byte_size,
|
||||
};
|
||||
}
|
||||
|
||||
async function readStoredObject(descriptor) {
|
||||
try {
|
||||
return await storage.getObject(descriptor.key);
|
||||
} catch (error) {
|
||||
if (error instanceof StorageObjectNotFoundError) {
|
||||
throw new AssetError(
|
||||
"STORAGE_INCONSISTENCY",
|
||||
"Asset metadata exists but its stored bytes are unavailable",
|
||||
{
|
||||
statusCode: 503,
|
||||
details: { assetId: descriptor.asset.id, retryable: true },
|
||||
cause: error,
|
||||
},
|
||||
);
|
||||
}
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
|
||||
return Object.freeze({
|
||||
list,
|
||||
get,
|
||||
upload,
|
||||
update,
|
||||
references,
|
||||
remove,
|
||||
mediaDescriptor,
|
||||
originalDescriptor,
|
||||
readStoredObject,
|
||||
});
|
||||
}
|
||||
|
||||
module.exports = {
|
||||
createAssetService,
|
||||
defaultDisplayName,
|
||||
mapAsset,
|
||||
normalizeDisplayName,
|
||||
normalizeExpiration,
|
||||
};
|
||||
@@ -0,0 +1,66 @@
|
||||
const { Readable } = require("node:stream");
|
||||
const { pipeline } = require("node:stream/promises");
|
||||
const { AssetError } = require("./errors");
|
||||
|
||||
function readableBody(body) {
|
||||
if (body && typeof body.pipe === "function") return body;
|
||||
if (Buffer.isBuffer(body) || body instanceof Uint8Array) {
|
||||
return Readable.from([body]);
|
||||
}
|
||||
if (body && typeof body.getReader === "function") {
|
||||
return Readable.fromWeb(body);
|
||||
}
|
||||
throw new AssetError(
|
||||
"STORAGE_INCONSISTENCY",
|
||||
"Stored asset bytes could not be streamed",
|
||||
{ statusCode: 503 },
|
||||
);
|
||||
}
|
||||
|
||||
function validateDescriptor(descriptor) {
|
||||
if (
|
||||
typeof descriptor.contentType !== "string" ||
|
||||
!Number.isSafeInteger(descriptor.contentLength) ||
|
||||
descriptor.contentLength < 0
|
||||
) {
|
||||
throw new AssetError(
|
||||
"STORAGE_INCONSISTENCY",
|
||||
"Stored asset metadata is incomplete",
|
||||
{ statusCode: 503 },
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
function setCommonHeaders(response, descriptor) {
|
||||
response.set("Content-Type", descriptor.contentType);
|
||||
response.set("Content-Length", String(descriptor.contentLength));
|
||||
response.set("X-Content-Type-Options", "nosniff");
|
||||
}
|
||||
|
||||
async function storedBody(assetService, descriptor) {
|
||||
const stored = await assetService.readStoredObject(descriptor);
|
||||
return readableBody(stored.body);
|
||||
}
|
||||
|
||||
async function sendImmutableMedia(assetService, response, descriptor) {
|
||||
validateDescriptor(descriptor);
|
||||
const body = await storedBody(assetService, descriptor);
|
||||
setCommonHeaders(response, descriptor);
|
||||
response.set("Cache-Control", "public, max-age=31536000, immutable");
|
||||
response.set("ETag", `"${descriptor.hash}"`);
|
||||
await pipeline(body, response);
|
||||
}
|
||||
|
||||
async function sendOriginal(assetService, response, descriptor) {
|
||||
validateDescriptor(descriptor);
|
||||
const body = await storedBody(assetService, descriptor);
|
||||
response.attachment(descriptor.asset.original_filename);
|
||||
setCommonHeaders(response, descriptor);
|
||||
response.set("Cache-Control", "private, no-store");
|
||||
await pipeline(body, response);
|
||||
}
|
||||
|
||||
module.exports = {
|
||||
sendImmutableMedia,
|
||||
sendOriginal,
|
||||
};
|
||||
@@ -156,6 +156,42 @@ function loadConfig(env = process.env) {
|
||||
accessKeyId: storageCredential(env, "S3_ACCESS_KEY_ID", environment),
|
||||
secretAccessKey: storageCredential(env, "S3_SECRET_ACCESS_KEY", environment),
|
||||
forcePathStyle: boolean(env, "S3_FORCE_PATH_STYLE", true),
|
||||
healthProbeIntervalMs: integer(
|
||||
env,
|
||||
"S3_HEALTH_PROBE_INTERVAL_MS",
|
||||
30_000,
|
||||
{ min: 1_000, max: 300_000 },
|
||||
),
|
||||
connectionTimeoutMs: integer(env, "S3_CONNECT_TIMEOUT_MS", 2_000, {
|
||||
min: 250,
|
||||
max: 60_000,
|
||||
}),
|
||||
requestTimeoutMs: integer(env, "S3_REQUEST_TIMEOUT_MS", 5_000, {
|
||||
min: 1_000,
|
||||
max: 120_000,
|
||||
}),
|
||||
maxAttempts: integer(env, "S3_MAX_ATTEMPTS", 2, { min: 1, max: 5 }),
|
||||
}),
|
||||
processing: Object.freeze({
|
||||
tempDirectory: (() => {
|
||||
const value = env.UPLOAD_TEMP_DIRECTORY?.trim() || "/tmp/announcements-uploads";
|
||||
if (!path.isAbsolute(value)) {
|
||||
throw new ConfigError(
|
||||
"UPLOAD_TEMP_DIRECTORY must be an absolute path",
|
||||
"UPLOAD_TEMP_DIRECTORY",
|
||||
);
|
||||
}
|
||||
return value;
|
||||
})(),
|
||||
maxFileBytes: integer(env, "UPLOAD_MAX_FILE_BYTES", 25 * 1024 * 1024, {
|
||||
min: 1,
|
||||
max: 1024 * 1024 * 1024,
|
||||
}),
|
||||
maxFiles: integer(env, "UPLOAD_MAX_FILES", 20, { min: 1, max: 100 }),
|
||||
maxInputPixels: integer(env, "IMAGE_MAX_INPUT_PIXELS", 40_000_000, {
|
||||
min: 1,
|
||||
max: 500_000_000,
|
||||
}),
|
||||
}),
|
||||
mqtt: Object.freeze({
|
||||
url: url(env, "MQTT_URL", {
|
||||
@@ -193,6 +229,16 @@ function redactedConfig(config) {
|
||||
region: config.storage.region,
|
||||
bucket: config.storage.bucket,
|
||||
forcePathStyle: config.storage.forcePathStyle,
|
||||
healthProbeIntervalMs: config.storage.healthProbeIntervalMs,
|
||||
connectionTimeoutMs: config.storage.connectionTimeoutMs,
|
||||
requestTimeoutMs: config.storage.requestTimeoutMs,
|
||||
maxAttempts: config.storage.maxAttempts,
|
||||
},
|
||||
processing: {
|
||||
tempDirectoryConfigured: Boolean(config.processing.tempDirectory),
|
||||
maxFileBytes: config.processing.maxFileBytes,
|
||||
maxFiles: config.processing.maxFiles,
|
||||
maxInputPixels: config.processing.maxInputPixels,
|
||||
},
|
||||
mqtt: {
|
||||
configured: Boolean(config.mqtt.url),
|
||||
|
||||
@@ -14,15 +14,62 @@ function databaseHealth(database) {
|
||||
}
|
||||
}
|
||||
|
||||
function createHealthRouter({ database, lifecycle, migrations }) {
|
||||
function storageHealth(storage) {
|
||||
if (!storage) {
|
||||
return {
|
||||
status: "degraded",
|
||||
checkedAt: null,
|
||||
lastSuccessAt: null,
|
||||
code: "STORAGE_NOT_INITIALIZED",
|
||||
message: "Object storage has not been initialized",
|
||||
};
|
||||
}
|
||||
|
||||
try {
|
||||
const result = storage.health();
|
||||
if (result?.status === "healthy") {
|
||||
return {
|
||||
status: "healthy",
|
||||
checkedAt: result.checkedAt || null,
|
||||
lastSuccessAt: result.lastSuccessAt || result.checkedAt || null,
|
||||
};
|
||||
}
|
||||
return {
|
||||
status: "degraded",
|
||||
checkedAt: result?.checkedAt || null,
|
||||
lastSuccessAt: result?.lastSuccessAt || null,
|
||||
code:
|
||||
result?.code === "STORAGE_NOT_CHECKED"
|
||||
? "STORAGE_NOT_CHECKED"
|
||||
: "STORAGE_UNAVAILABLE",
|
||||
message:
|
||||
result?.code === "STORAGE_NOT_CHECKED"
|
||||
? "Object storage has not been checked"
|
||||
: "Object storage is unavailable",
|
||||
};
|
||||
} catch {
|
||||
// A health endpoint must remain available when its dependency is not.
|
||||
}
|
||||
|
||||
return {
|
||||
status: "degraded",
|
||||
checkedAt: null,
|
||||
lastSuccessAt: null,
|
||||
code: "STORAGE_UNAVAILABLE",
|
||||
message: "Object storage is unavailable",
|
||||
};
|
||||
}
|
||||
|
||||
function createHealthRouter({ database, lifecycle, migrations, storage }) {
|
||||
const router = express.Router();
|
||||
|
||||
router.get("/live", (request, response) => {
|
||||
response.json({ status: "ok" });
|
||||
});
|
||||
|
||||
router.get("/ready", (request, response) => {
|
||||
router.get("/ready", async (request, response) => {
|
||||
const databaseCheck = databaseHealth(database);
|
||||
const storageCheck = storageHealth(storage);
|
||||
const migrationsCurrent = migrations.pendingVersions.length === 0;
|
||||
const ready =
|
||||
lifecycle.ready && databaseCheck.status === "ok" && migrationsCurrent;
|
||||
@@ -32,6 +79,7 @@ function createHealthRouter({ database, lifecycle, migrations }) {
|
||||
checks: {
|
||||
database: databaseCheck.status,
|
||||
migrations: migrationsCurrent ? "current" : "pending",
|
||||
storage: storageCheck.status,
|
||||
},
|
||||
});
|
||||
});
|
||||
@@ -39,18 +87,23 @@ function createHealthRouter({ database, lifecycle, migrations }) {
|
||||
return router;
|
||||
}
|
||||
|
||||
function createDetailedHealthRouter({ config, database, lifecycle, migrations }) {
|
||||
function createDetailedHealthRouter({ config, database, lifecycle, migrations, storage }) {
|
||||
const router = express.Router();
|
||||
|
||||
router.get("/", (request, response) => {
|
||||
router.get("/", async (request, response) => {
|
||||
const databaseCheck = databaseHealth(database);
|
||||
const storageCheck = storageHealth(storage);
|
||||
const ready =
|
||||
lifecycle.ready &&
|
||||
databaseCheck.status === "ok" &&
|
||||
migrations.pendingVersions.length === 0;
|
||||
|
||||
const configuration = redactedConfig(config);
|
||||
const status = ready ? "degraded" : "not_ready";
|
||||
const status = !ready
|
||||
? "not_ready"
|
||||
: storageCheck.status === "healthy"
|
||||
? "ok"
|
||||
: "degraded";
|
||||
|
||||
response.status(ready ? 200 : 503).json({
|
||||
status,
|
||||
@@ -62,7 +115,7 @@ function createDetailedHealthRouter({ config, database, lifecycle, migrations })
|
||||
latestSchemaVersion: migrations.latestAvailableVersion,
|
||||
},
|
||||
storage: {
|
||||
status: "not_checked",
|
||||
...storageCheck,
|
||||
configured: configuration.storage.configured,
|
||||
},
|
||||
mqtt: {
|
||||
@@ -86,4 +139,5 @@ module.exports = {
|
||||
createDetailedHealthRouter,
|
||||
createHealthRouter,
|
||||
databaseHealth,
|
||||
storageHealth,
|
||||
};
|
||||
|
||||
@@ -0,0 +1,330 @@
|
||||
const crypto = require("node:crypto");
|
||||
const fs = require("node:fs");
|
||||
const sharp = require("sharp");
|
||||
|
||||
const SUPPORTED_FORMATS = Object.freeze(new Set(["jpeg", "png", "webp"]));
|
||||
const CONTENT_TYPES = Object.freeze({
|
||||
jpeg: "image/jpeg",
|
||||
png: "image/png",
|
||||
webp: "image/webp",
|
||||
});
|
||||
const BLACK = Object.freeze({ r: 0, g: 0, b: 0, alpha: 1 });
|
||||
const DEFAULT_IMAGE_DERIVATIVE_OPTIONS = Object.freeze({
|
||||
display: Object.freeze({ maxWidth: 1920, maxHeight: 1080, quality: 82 }),
|
||||
thumbnail: Object.freeze({ maxWidth: 480, maxHeight: 270, quality: 72 }),
|
||||
});
|
||||
|
||||
class ImageProcessingError extends Error {
|
||||
constructor(code, message, { statusCode = 422, details, cause } = {}) {
|
||||
super(message, cause ? { cause } : undefined);
|
||||
this.name = "ImageProcessingError";
|
||||
this.code = code;
|
||||
this.statusCode = statusCode;
|
||||
if (details !== undefined) {
|
||||
this.details = details;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
function requireInteger(value, name, { min, max } = {}) {
|
||||
if (!Number.isSafeInteger(value)) {
|
||||
throw new TypeError(`${name} must be an integer`);
|
||||
}
|
||||
if (min !== undefined && value < min) {
|
||||
throw new RangeError(`${name} must be at least ${min}`);
|
||||
}
|
||||
if (max !== undefined && value > max) {
|
||||
throw new RangeError(`${name} must be at most ${max}`);
|
||||
}
|
||||
return value;
|
||||
}
|
||||
|
||||
function normalizeTargetOptions(target, name, defaults) {
|
||||
if (
|
||||
target !== undefined &&
|
||||
(target === null || typeof target !== "object" || Array.isArray(target))
|
||||
) {
|
||||
throw new TypeError(`${name} image options must be an object`);
|
||||
}
|
||||
|
||||
const configured = { ...defaults, ...target };
|
||||
|
||||
return Object.freeze({
|
||||
maxWidth: requireInteger(configured.maxWidth, `${name}.maxWidth`, { min: 1 }),
|
||||
maxHeight: requireInteger(configured.maxHeight, `${name}.maxHeight`, { min: 1 }),
|
||||
quality: requireInteger(configured.quality, `${name}.quality`, {
|
||||
min: 1,
|
||||
max: 100,
|
||||
}),
|
||||
});
|
||||
}
|
||||
|
||||
function normalizeOptions(options) {
|
||||
if (options === null || typeof options !== "object" || Array.isArray(options)) {
|
||||
throw new TypeError("Image processing options are required");
|
||||
}
|
||||
|
||||
return Object.freeze({
|
||||
maxInputPixels: requireInteger(
|
||||
options.maxInputPixels,
|
||||
"maxInputPixels",
|
||||
{ min: 1 },
|
||||
),
|
||||
display: normalizeTargetOptions(
|
||||
options.display,
|
||||
"display",
|
||||
DEFAULT_IMAGE_DERIVATIVE_OPTIONS.display,
|
||||
),
|
||||
thumbnail: normalizeTargetOptions(
|
||||
options.thumbnail,
|
||||
"thumbnail",
|
||||
DEFAULT_IMAGE_DERIVATIVE_OPTIONS.thumbnail,
|
||||
),
|
||||
});
|
||||
}
|
||||
|
||||
function sha256(buffer) {
|
||||
return crypto.createHash("sha256").update(buffer).digest("hex");
|
||||
}
|
||||
|
||||
async function hashFile(filename) {
|
||||
const hash = crypto.createHash("sha256");
|
||||
let byteSize = 0;
|
||||
|
||||
try {
|
||||
for await (const chunk of fs.createReadStream(filename)) {
|
||||
byteSize += chunk.length;
|
||||
hash.update(chunk);
|
||||
}
|
||||
} catch (error) {
|
||||
throw new ImageProcessingError(
|
||||
"IMAGE_INPUT_UNREADABLE",
|
||||
"The uploaded image could not be read",
|
||||
{ statusCode: 422, cause: error },
|
||||
);
|
||||
}
|
||||
|
||||
return { byteSize, sha256: hash.digest("hex") };
|
||||
}
|
||||
|
||||
async function describeInput(input) {
|
||||
if (Buffer.isBuffer(input)) {
|
||||
return {
|
||||
sharpInput: input,
|
||||
byteSize: input.length,
|
||||
sha256: sha256(input),
|
||||
};
|
||||
}
|
||||
|
||||
if (typeof input !== "string" || input.length === 0) {
|
||||
throw new ImageProcessingError(
|
||||
"INVALID_IMAGE_INPUT",
|
||||
"Image input must be a Buffer or temporary file path",
|
||||
{ statusCode: 400 },
|
||||
);
|
||||
}
|
||||
|
||||
let stats;
|
||||
try {
|
||||
stats = await fs.promises.stat(input);
|
||||
} catch (error) {
|
||||
throw new ImageProcessingError(
|
||||
"IMAGE_INPUT_UNREADABLE",
|
||||
"The uploaded image could not be read",
|
||||
{ statusCode: 422, cause: error },
|
||||
);
|
||||
}
|
||||
|
||||
if (!stats.isFile()) {
|
||||
throw new ImageProcessingError(
|
||||
"INVALID_IMAGE_INPUT",
|
||||
"Image input path must identify a regular file",
|
||||
{ statusCode: 400 },
|
||||
);
|
||||
}
|
||||
|
||||
return {
|
||||
sharpInput: input,
|
||||
...(await hashFile(input)),
|
||||
};
|
||||
}
|
||||
|
||||
function decodeError(error, maxInputPixels) {
|
||||
if (error instanceof ImageProcessingError) {
|
||||
return error;
|
||||
}
|
||||
|
||||
if (/pixel limit/i.test(error?.message || "")) {
|
||||
return new ImageProcessingError(
|
||||
"IMAGE_PIXEL_LIMIT_EXCEEDED",
|
||||
"The uploaded image exceeds the configured decoded-pixel limit",
|
||||
{
|
||||
statusCode: 413,
|
||||
details: { maxInputPixels },
|
||||
cause: error,
|
||||
},
|
||||
);
|
||||
}
|
||||
|
||||
return new ImageProcessingError(
|
||||
"MALFORMED_IMAGE",
|
||||
"The uploaded file is not a valid decodable image",
|
||||
{ statusCode: 422, cause: error },
|
||||
);
|
||||
}
|
||||
|
||||
async function readMetadata(input, maxInputPixels) {
|
||||
try {
|
||||
return await sharp(input, {
|
||||
animated: true,
|
||||
failOn: "error",
|
||||
limitInputPixels: maxInputPixels,
|
||||
}).metadata();
|
||||
} catch (error) {
|
||||
throw decodeError(error, maxInputPixels);
|
||||
}
|
||||
}
|
||||
|
||||
function validateMetadata(metadata) {
|
||||
const pages = metadata.pages ?? 1;
|
||||
|
||||
if (pages > 1) {
|
||||
const hasAnimationTiming =
|
||||
metadata.loop !== undefined ||
|
||||
(Array.isArray(metadata.delay) && metadata.delay.length > 0);
|
||||
throw new ImageProcessingError(
|
||||
hasAnimationTiming
|
||||
? "ANIMATED_IMAGE_NOT_SUPPORTED"
|
||||
: "MULTIPAGE_IMAGE_NOT_SUPPORTED",
|
||||
hasAnimationTiming
|
||||
? "Animated images are not supported"
|
||||
: "Multipage images are not supported",
|
||||
{ statusCode: 422, details: { pages } },
|
||||
);
|
||||
}
|
||||
|
||||
if (!SUPPORTED_FORMATS.has(metadata.format)) {
|
||||
throw new ImageProcessingError(
|
||||
"UNSUPPORTED_IMAGE_FORMAT",
|
||||
"Only still JPEG, PNG, and WebP images are supported",
|
||||
{
|
||||
statusCode: 415,
|
||||
details: {
|
||||
detectedFormat: metadata.format || null,
|
||||
supportedFormats: [...SUPPORTED_FORMATS],
|
||||
},
|
||||
},
|
||||
);
|
||||
}
|
||||
|
||||
if (
|
||||
!Number.isSafeInteger(metadata.width) ||
|
||||
metadata.width < 1 ||
|
||||
!Number.isSafeInteger(metadata.height) ||
|
||||
metadata.height < 1
|
||||
) {
|
||||
throw new ImageProcessingError(
|
||||
"MALFORMED_IMAGE",
|
||||
"The uploaded image does not contain valid dimensions",
|
||||
{ statusCode: 422 },
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
async function generateDerivative(input, target, maxInputPixels) {
|
||||
let output;
|
||||
try {
|
||||
output = await sharp(input, {
|
||||
failOn: "error",
|
||||
limitInputPixels: maxInputPixels,
|
||||
})
|
||||
.autoOrient()
|
||||
.flatten({ background: BLACK })
|
||||
.resize({
|
||||
width: target.maxWidth,
|
||||
height: target.maxHeight,
|
||||
fit: "inside",
|
||||
withoutEnlargement: true,
|
||||
})
|
||||
.toColourspace("srgb")
|
||||
.webp({ quality: target.quality, effort: 4 })
|
||||
.toBuffer({ resolveWithObject: true });
|
||||
} catch (error) {
|
||||
throw decodeError(error, maxInputPixels);
|
||||
}
|
||||
|
||||
return {
|
||||
buffer: output.data,
|
||||
sha256: sha256(output.data),
|
||||
format: "webp",
|
||||
contentType: CONTENT_TYPES.webp,
|
||||
byteSize: output.data.length,
|
||||
width: output.info.width,
|
||||
height: output.info.height,
|
||||
};
|
||||
}
|
||||
|
||||
function originalMetadata(metadata, inputDescription) {
|
||||
const orientedDimensions = metadata.autoOrient || {
|
||||
width: metadata.width,
|
||||
height: metadata.height,
|
||||
};
|
||||
|
||||
return {
|
||||
sha256: inputDescription.sha256,
|
||||
format: metadata.format,
|
||||
contentType: CONTENT_TYPES[metadata.format],
|
||||
byteSize: inputDescription.byteSize,
|
||||
// Width and height describe how the original should appear after applying
|
||||
// its source orientation. The encoded dimensions remain available for
|
||||
// diagnostics without implying that the archived bytes were rewritten.
|
||||
width: orientedDimensions.width,
|
||||
height: orientedDimensions.height,
|
||||
encodedWidth: metadata.width,
|
||||
encodedHeight: metadata.height,
|
||||
orientation: metadata.orientation ?? null,
|
||||
pages: metadata.pages ?? 1,
|
||||
channels: metadata.channels,
|
||||
hasAlpha: Boolean(metadata.hasAlpha),
|
||||
};
|
||||
}
|
||||
|
||||
function createImageProcessor(options) {
|
||||
const config = normalizeOptions(options);
|
||||
|
||||
async function process(input) {
|
||||
const inputDescription = await describeInput(input);
|
||||
const metadata = await readMetadata(
|
||||
inputDescription.sharpInput,
|
||||
config.maxInputPixels,
|
||||
);
|
||||
validateMetadata(metadata);
|
||||
|
||||
const [display, thumbnail] = await Promise.all([
|
||||
generateDerivative(
|
||||
inputDescription.sharpInput,
|
||||
config.display,
|
||||
config.maxInputPixels,
|
||||
),
|
||||
generateDerivative(
|
||||
inputDescription.sharpInput,
|
||||
config.thumbnail,
|
||||
config.maxInputPixels,
|
||||
),
|
||||
]);
|
||||
|
||||
return {
|
||||
original: originalMetadata(metadata, inputDescription),
|
||||
display,
|
||||
thumbnail,
|
||||
};
|
||||
}
|
||||
|
||||
return Object.freeze({ process });
|
||||
}
|
||||
|
||||
module.exports = {
|
||||
DEFAULT_IMAGE_DERIVATIVE_OPTIONS,
|
||||
ImageProcessingError,
|
||||
createImageProcessor,
|
||||
};
|
||||
+43
-2
@@ -3,6 +3,8 @@ const { Server: SocketServer } = require("socket.io");
|
||||
const { createApp } = require("./app");
|
||||
const { openDatabase } = require("./database");
|
||||
const { runMigrations } = require("./database/migrate");
|
||||
const { createImageProcessor } = require("./image-processing");
|
||||
const { createStorageService } = require("./storage");
|
||||
|
||||
function listen(httpServer, port, host) {
|
||||
return new Promise((resolve, reject) => {
|
||||
@@ -20,7 +22,12 @@ function listen(httpServer, port, host) {
|
||||
});
|
||||
}
|
||||
|
||||
function createRuntime({ config, logger = console }) {
|
||||
function createRuntime({
|
||||
config,
|
||||
imageProcessor: suppliedImageProcessor,
|
||||
logger = console,
|
||||
storage: suppliedStorage,
|
||||
}) {
|
||||
const database = openDatabase(config.database);
|
||||
let migrations;
|
||||
try {
|
||||
@@ -34,7 +41,32 @@ function createRuntime({ config, logger = console }) {
|
||||
ready: false,
|
||||
stopping: false,
|
||||
};
|
||||
const app = createApp({ config, database, lifecycle, migrations });
|
||||
let storage;
|
||||
let imageProcessor;
|
||||
try {
|
||||
storage = suppliedStorage || createStorageService(config.storage, { logger });
|
||||
imageProcessor =
|
||||
suppliedImageProcessor ||
|
||||
createImageProcessor({ maxInputPixels: config.processing.maxInputPixels });
|
||||
} catch (error) {
|
||||
database.close();
|
||||
throw error;
|
||||
}
|
||||
let app;
|
||||
try {
|
||||
app = createApp({
|
||||
config,
|
||||
database,
|
||||
imageProcessor,
|
||||
lifecycle,
|
||||
migrations,
|
||||
storage,
|
||||
});
|
||||
} catch (error) {
|
||||
storage.stop();
|
||||
database.close();
|
||||
throw error;
|
||||
}
|
||||
const httpServer = http.createServer(app);
|
||||
const io = new SocketServer(httpServer, {
|
||||
serveClient: false,
|
||||
@@ -63,9 +95,15 @@ function createRuntime({ config, logger = console }) {
|
||||
.then((listeningAddress) => {
|
||||
address = listeningAddress;
|
||||
lifecycle.ready = true;
|
||||
Promise.resolve()
|
||||
.then(() => storage.start())
|
||||
.catch(() => {
|
||||
logger.warn?.("Object storage health checks could not be started");
|
||||
});
|
||||
return listeningAddress;
|
||||
})
|
||||
.catch((error) => {
|
||||
storage.stop();
|
||||
database.close();
|
||||
throw error;
|
||||
});
|
||||
@@ -85,6 +123,7 @@ function createRuntime({ config, logger = console }) {
|
||||
if (settled) return;
|
||||
settled = true;
|
||||
clearTimeout(timeout);
|
||||
storage.stop();
|
||||
if (database.open) {
|
||||
database.close();
|
||||
}
|
||||
@@ -119,6 +158,8 @@ function createRuntime({ config, logger = console }) {
|
||||
io,
|
||||
lifecycle,
|
||||
migrations,
|
||||
storage,
|
||||
imageProcessor,
|
||||
get address() {
|
||||
return address;
|
||||
},
|
||||
|
||||
@@ -0,0 +1,398 @@
|
||||
const {
|
||||
CreateBucketCommand,
|
||||
DeleteObjectCommand,
|
||||
GetObjectCommand,
|
||||
HeadBucketCommand,
|
||||
HeadObjectCommand,
|
||||
PutObjectCommand,
|
||||
S3Client,
|
||||
} = require("@aws-sdk/client-s3");
|
||||
|
||||
class StorageError extends Error {
|
||||
constructor(message, { code, statusCode, cause } = {}) {
|
||||
super(message, { cause });
|
||||
this.name = "StorageError";
|
||||
this.code = code || "STORAGE_ERROR";
|
||||
this.statusCode = statusCode || 500;
|
||||
}
|
||||
}
|
||||
|
||||
class StorageUnavailableError extends StorageError {
|
||||
constructor(cause) {
|
||||
super("Object storage is unavailable", {
|
||||
code: "STORAGE_UNAVAILABLE",
|
||||
statusCode: 503,
|
||||
cause,
|
||||
});
|
||||
this.name = "StorageUnavailableError";
|
||||
}
|
||||
}
|
||||
|
||||
class StorageObjectNotFoundError extends StorageError {
|
||||
constructor(key, cause) {
|
||||
super("Stored object was not found", {
|
||||
code: "STORAGE_OBJECT_NOT_FOUND",
|
||||
statusCode: 404,
|
||||
cause,
|
||||
});
|
||||
this.name = "StorageObjectNotFoundError";
|
||||
this.key = key;
|
||||
}
|
||||
}
|
||||
|
||||
function errorStatus(error) {
|
||||
return error?.$metadata?.httpStatusCode || error?.statusCode;
|
||||
}
|
||||
|
||||
function isMissingBucket(error) {
|
||||
return (
|
||||
error?.name === "NoSuchBucket" ||
|
||||
error?.Code === "NoSuchBucket" ||
|
||||
error?.code === "NoSuchBucket" ||
|
||||
errorStatus(error) === 404
|
||||
);
|
||||
}
|
||||
|
||||
function isMissingObject(error) {
|
||||
if (
|
||||
error?.name === "NoSuchBucket" ||
|
||||
error?.Code === "NoSuchBucket" ||
|
||||
error?.code === "NoSuchBucket"
|
||||
) {
|
||||
return false;
|
||||
}
|
||||
return (
|
||||
error?.name === "NoSuchKey" ||
|
||||
error?.name === "NotFound" ||
|
||||
error?.Code === "NoSuchKey" ||
|
||||
error?.code === "NoSuchKey" ||
|
||||
errorStatus(error) === 404
|
||||
);
|
||||
}
|
||||
|
||||
function isBucketCreationConflict(error) {
|
||||
return (
|
||||
error?.name === "BucketAlreadyOwnedByYou" ||
|
||||
error?.name === "BucketAlreadyExists" ||
|
||||
error?.Code === "BucketAlreadyOwnedByYou" ||
|
||||
error?.Code === "BucketAlreadyExists" ||
|
||||
error?.code === "BucketAlreadyOwnedByYou" ||
|
||||
error?.code === "BucketAlreadyExists"
|
||||
);
|
||||
}
|
||||
|
||||
function isConsumerCancellation(error) {
|
||||
return (
|
||||
error?.name === "AbortError" ||
|
||||
error?.code === "ABORT_ERR" ||
|
||||
error?.code === "ERR_STREAM_PREMATURE_CLOSE"
|
||||
);
|
||||
}
|
||||
|
||||
function assertObjectKey(key) {
|
||||
if (typeof key !== "string" || key.length === 0) {
|
||||
throw new TypeError("Storage object key must be a non-empty string");
|
||||
}
|
||||
|
||||
const segments = key.split("/");
|
||||
if (
|
||||
key.startsWith("/") ||
|
||||
key.includes("\\") ||
|
||||
key.includes("\0") ||
|
||||
segments.some((segment) => segment === "." || segment === "..")
|
||||
) {
|
||||
throw new TypeError("Storage object key is invalid");
|
||||
}
|
||||
}
|
||||
|
||||
function publicHealth(health) {
|
||||
return Object.freeze({ ...health });
|
||||
}
|
||||
|
||||
function createS3Client(config) {
|
||||
const options = {
|
||||
endpoint: config.endpoint,
|
||||
region: config.region,
|
||||
forcePathStyle: config.forcePathStyle,
|
||||
maxAttempts: config.maxAttempts,
|
||||
requestHandler: {
|
||||
connectionTimeout: config.connectionTimeoutMs,
|
||||
// Use an inactivity timeout rather than an absolute request deadline so
|
||||
// a large but continuously progressing upload is not aborted.
|
||||
socketTimeout: config.requestTimeoutMs,
|
||||
},
|
||||
// Optional AWS checksums are not uniformly implemented by S3-compatible
|
||||
// providers. Source and derivative SHA-256 hashes are verified by the app.
|
||||
requestChecksumCalculation: "WHEN_REQUIRED",
|
||||
responseChecksumValidation: "WHEN_REQUIRED",
|
||||
};
|
||||
|
||||
if (config.accessKeyId && config.secretAccessKey) {
|
||||
options.credentials = {
|
||||
accessKeyId: config.accessKeyId,
|
||||
secretAccessKey: config.secretAccessKey,
|
||||
};
|
||||
}
|
||||
|
||||
return new S3Client(options);
|
||||
}
|
||||
|
||||
function createStorageService(config, { client, logger = console, now = () => new Date() } = {}) {
|
||||
const s3 = client || createS3Client(config);
|
||||
let ensurePromise = null;
|
||||
let healthTimer = null;
|
||||
let bucketReady = false;
|
||||
let issuedHealthSequence = 0;
|
||||
let appliedHealthSequence = 0;
|
||||
let currentHealth = {
|
||||
status: "degraded",
|
||||
checkedAt: null,
|
||||
lastSuccessAt: null,
|
||||
code: "STORAGE_NOT_CHECKED",
|
||||
message: "Object storage has not been checked",
|
||||
};
|
||||
|
||||
function nextHealthSequence() {
|
||||
issuedHealthSequence += 1;
|
||||
return issuedHealthSequence;
|
||||
}
|
||||
|
||||
function markHealthy(sequence) {
|
||||
if (sequence < appliedHealthSequence) return publicHealth(currentHealth);
|
||||
appliedHealthSequence = sequence;
|
||||
bucketReady = true;
|
||||
const checkedAt = now().toISOString();
|
||||
currentHealth = {
|
||||
status: "healthy",
|
||||
checkedAt,
|
||||
lastSuccessAt: checkedAt,
|
||||
};
|
||||
return publicHealth(currentHealth);
|
||||
}
|
||||
|
||||
function markDegraded(sequence) {
|
||||
if (sequence < appliedHealthSequence) return publicHealth(currentHealth);
|
||||
appliedHealthSequence = sequence;
|
||||
bucketReady = false;
|
||||
currentHealth = {
|
||||
status: "degraded",
|
||||
checkedAt: now().toISOString(),
|
||||
lastSuccessAt: currentHealth.lastSuccessAt,
|
||||
code: "STORAGE_UNAVAILABLE",
|
||||
message: "Object storage is unavailable",
|
||||
};
|
||||
return publicHealth(currentHealth);
|
||||
}
|
||||
|
||||
function unavailable(error, sequence) {
|
||||
markDegraded(sequence);
|
||||
return error instanceof StorageUnavailableError
|
||||
? error
|
||||
: new StorageUnavailableError(error);
|
||||
}
|
||||
|
||||
async function ensureBucketOnce() {
|
||||
const sequence = nextHealthSequence();
|
||||
try {
|
||||
await s3.send(new HeadBucketCommand({ Bucket: config.bucket }));
|
||||
markHealthy(sequence);
|
||||
return { created: false };
|
||||
} catch (error) {
|
||||
if (!isMissingBucket(error)) {
|
||||
throw unavailable(error, sequence);
|
||||
}
|
||||
}
|
||||
|
||||
try {
|
||||
await s3.send(new CreateBucketCommand({ Bucket: config.bucket }));
|
||||
markHealthy(sequence);
|
||||
return { created: true };
|
||||
} catch (error) {
|
||||
if (isBucketCreationConflict(error)) {
|
||||
try {
|
||||
await s3.send(new HeadBucketCommand({ Bucket: config.bucket }));
|
||||
markHealthy(sequence);
|
||||
return { created: false };
|
||||
} catch (verificationError) {
|
||||
throw unavailable(verificationError, sequence);
|
||||
}
|
||||
}
|
||||
throw unavailable(error, sequence);
|
||||
}
|
||||
}
|
||||
|
||||
async function ensureBucket() {
|
||||
if (!ensurePromise) {
|
||||
ensurePromise = ensureBucketOnce().finally(() => {
|
||||
ensurePromise = null;
|
||||
});
|
||||
}
|
||||
return ensurePromise;
|
||||
}
|
||||
|
||||
async function initialize() {
|
||||
try {
|
||||
await ensureBucket();
|
||||
} catch (error) {
|
||||
logger.warn?.("Object storage is unavailable during initialization");
|
||||
}
|
||||
return publicHealth(currentHealth);
|
||||
}
|
||||
|
||||
async function start() {
|
||||
if (!healthTimer) {
|
||||
healthTimer = setInterval(() => {
|
||||
ensureBucket().catch(() => {});
|
||||
}, config.healthProbeIntervalMs || 30_000);
|
||||
healthTimer.unref?.();
|
||||
}
|
||||
return initialize();
|
||||
}
|
||||
|
||||
async function probe() {
|
||||
const sequence = nextHealthSequence();
|
||||
try {
|
||||
await s3.send(new HeadBucketCommand({ Bucket: config.bucket }));
|
||||
return markHealthy(sequence);
|
||||
} catch (error) {
|
||||
return markDegraded(sequence);
|
||||
}
|
||||
}
|
||||
|
||||
async function putObject({ key, body, contentType, contentLength, metadata }) {
|
||||
assertObjectKey(key);
|
||||
if (!bucketReady) await ensureBucket();
|
||||
const sequence = nextHealthSequence();
|
||||
const input = {
|
||||
Bucket: config.bucket,
|
||||
Key: key,
|
||||
Body: body,
|
||||
...(contentType ? { ContentType: contentType } : {}),
|
||||
...(contentLength !== undefined ? { ContentLength: contentLength } : {}),
|
||||
...(metadata ? { Metadata: metadata } : {}),
|
||||
};
|
||||
|
||||
try {
|
||||
const result = await s3.send(new PutObjectCommand(input));
|
||||
markHealthy(sequence);
|
||||
return {
|
||||
key,
|
||||
etag: result.ETag || null,
|
||||
versionId: result.VersionId || null,
|
||||
};
|
||||
} catch (error) {
|
||||
throw unavailable(error, sequence);
|
||||
}
|
||||
}
|
||||
|
||||
function objectMetadata(key, result) {
|
||||
return {
|
||||
key,
|
||||
contentType: result.ContentType || null,
|
||||
contentLength:
|
||||
result.ContentLength === undefined ? null : Number(result.ContentLength),
|
||||
etag: result.ETag || null,
|
||||
lastModified: result.LastModified || null,
|
||||
metadata: result.Metadata || {},
|
||||
};
|
||||
}
|
||||
|
||||
async function getObject(key) {
|
||||
assertObjectKey(key);
|
||||
if (!bucketReady) await ensureBucket();
|
||||
const sequence = nextHealthSequence();
|
||||
try {
|
||||
const result = await s3.send(
|
||||
new GetObjectCommand({ Bucket: config.bucket, Key: key }),
|
||||
);
|
||||
markHealthy(sequence);
|
||||
result.Body?.once?.("error", (error) => {
|
||||
if (!isConsumerCancellation(error)) {
|
||||
markDegraded(nextHealthSequence());
|
||||
}
|
||||
});
|
||||
return {
|
||||
...objectMetadata(key, result),
|
||||
body: result.Body,
|
||||
};
|
||||
} catch (error) {
|
||||
if (isMissingObject(error)) {
|
||||
markHealthy(sequence);
|
||||
throw new StorageObjectNotFoundError(key, error);
|
||||
}
|
||||
throw unavailable(error, sequence);
|
||||
}
|
||||
}
|
||||
|
||||
async function headObject(key) {
|
||||
assertObjectKey(key);
|
||||
if (!bucketReady) await ensureBucket();
|
||||
const sequence = nextHealthSequence();
|
||||
try {
|
||||
const result = await s3.send(
|
||||
new HeadObjectCommand({ Bucket: config.bucket, Key: key }),
|
||||
);
|
||||
markHealthy(sequence);
|
||||
return objectMetadata(key, result);
|
||||
} catch (error) {
|
||||
if (isMissingObject(error)) {
|
||||
markHealthy(sequence);
|
||||
return null;
|
||||
}
|
||||
throw unavailable(error, sequence);
|
||||
}
|
||||
}
|
||||
|
||||
async function objectExists(key) {
|
||||
return (await headObject(key)) !== null;
|
||||
}
|
||||
|
||||
async function deleteObject(key) {
|
||||
assertObjectKey(key);
|
||||
if (!bucketReady) await ensureBucket();
|
||||
const sequence = nextHealthSequence();
|
||||
try {
|
||||
await s3.send(
|
||||
new DeleteObjectCommand({ Bucket: config.bucket, Key: key }),
|
||||
);
|
||||
markHealthy(sequence);
|
||||
} catch (error) {
|
||||
if (isMissingObject(error)) {
|
||||
markHealthy(sequence);
|
||||
return;
|
||||
}
|
||||
throw unavailable(error, sequence);
|
||||
}
|
||||
}
|
||||
|
||||
function stop() {
|
||||
if (healthTimer) {
|
||||
clearInterval(healthTimer);
|
||||
healthTimer = null;
|
||||
}
|
||||
s3.destroy?.();
|
||||
}
|
||||
|
||||
return Object.freeze({
|
||||
initialize,
|
||||
start,
|
||||
stop,
|
||||
ensureBucket,
|
||||
probe,
|
||||
health: () => publicHealth(currentHealth),
|
||||
putObject,
|
||||
getObject,
|
||||
headObject,
|
||||
objectExists,
|
||||
deleteObject,
|
||||
destroy: stop,
|
||||
});
|
||||
}
|
||||
|
||||
module.exports = {
|
||||
StorageError,
|
||||
StorageObjectNotFoundError,
|
||||
StorageUnavailableError,
|
||||
createS3Client,
|
||||
createStorageService,
|
||||
};
|
||||
@@ -0,0 +1,442 @@
|
||||
const test = require("node:test");
|
||||
const assert = require("node:assert/strict");
|
||||
const crypto = require("node:crypto");
|
||||
const fs = require("node:fs");
|
||||
const http = require("node:http");
|
||||
const os = require("node:os");
|
||||
const path = require("node:path");
|
||||
const { Readable } = require("node:stream");
|
||||
const sharp = require("sharp");
|
||||
const { createApp } = require("../src/app");
|
||||
const { loadConfig } = require("../src/config");
|
||||
const { openDatabase } = require("../src/database");
|
||||
const { runMigrations } = require("../src/database/migrate");
|
||||
const { createImageProcessor } = require("../src/image-processing");
|
||||
const { StorageObjectNotFoundError } = require("../src/storage");
|
||||
|
||||
async function readBody(body) {
|
||||
if (Buffer.isBuffer(body)) return body;
|
||||
const chunks = [];
|
||||
for await (const chunk of body) chunks.push(Buffer.from(chunk));
|
||||
return Buffer.concat(chunks);
|
||||
}
|
||||
|
||||
function memoryStorage() {
|
||||
const objects = new Map();
|
||||
return {
|
||||
objects,
|
||||
health() {
|
||||
return {
|
||||
status: "healthy",
|
||||
checkedAt: "2033-05-18T03:33:20.000Z",
|
||||
lastSuccessAt: "2033-05-18T03:33:20.000Z",
|
||||
};
|
||||
},
|
||||
async putObject(input) {
|
||||
objects.set(input.key, {
|
||||
body: await readBody(input.body),
|
||||
contentType: input.contentType,
|
||||
metadata: input.metadata,
|
||||
});
|
||||
return { key: input.key };
|
||||
},
|
||||
async getObject(key) {
|
||||
const object = objects.get(key);
|
||||
if (!object) throw new StorageObjectNotFoundError(key);
|
||||
return {
|
||||
body: Readable.from([object.body]),
|
||||
contentLength: object.body.length,
|
||||
contentType: object.contentType,
|
||||
};
|
||||
},
|
||||
async deleteObject(key) {
|
||||
objects.delete(key);
|
||||
return { key, deleted: true };
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
async function createHarness(t, { maxFileBytes = 1024 * 1024 } = {}) {
|
||||
const tempDirectory = fs.mkdtempSync(
|
||||
path.join(os.tmpdir(), "announcements-api-uploads-"),
|
||||
);
|
||||
const config = loadConfig({
|
||||
NODE_ENV: "test",
|
||||
HOST: "127.0.0.1",
|
||||
PORT: "0",
|
||||
DATABASE_PATH: ":memory:",
|
||||
APP_TIMEZONE: "America/Chicago",
|
||||
S3_ENDPOINT: "http://private-seaweedfs:8333",
|
||||
S3_BUCKET: "announcements-test",
|
||||
UPLOAD_TEMP_DIRECTORY: tempDirectory,
|
||||
UPLOAD_MAX_FILE_BYTES: String(maxFileBytes),
|
||||
UPLOAD_MAX_FILES: "5",
|
||||
IMAGE_MAX_INPUT_PIXELS: "1000000",
|
||||
});
|
||||
const database = openDatabase({ path: ":memory:" });
|
||||
const migrations = runMigrations(database);
|
||||
const storage = memoryStorage();
|
||||
const app = createApp({
|
||||
config,
|
||||
database,
|
||||
imageProcessor: createImageProcessor({
|
||||
maxInputPixels: config.processing.maxInputPixels,
|
||||
}),
|
||||
lifecycle: { ready: true },
|
||||
migrations,
|
||||
storage,
|
||||
});
|
||||
const server = app.listen(0, "127.0.0.1");
|
||||
await new Promise((resolve, reject) => {
|
||||
server.once("listening", resolve);
|
||||
server.once("error", reject);
|
||||
});
|
||||
|
||||
t.after(async () => {
|
||||
await new Promise((resolve) => server.close(resolve));
|
||||
database.close();
|
||||
fs.rmSync(tempDirectory, { recursive: true, force: true });
|
||||
});
|
||||
|
||||
return {
|
||||
baseUrl: `http://127.0.0.1:${server.address().port}`,
|
||||
database,
|
||||
storage,
|
||||
tempDirectory,
|
||||
};
|
||||
}
|
||||
|
||||
function appendFile(form, field, buffer, filename, contentType) {
|
||||
form.append(field, new Blob([buffer], { type: contentType }), filename);
|
||||
}
|
||||
|
||||
async function jsonResponse(baseUrl, route, init) {
|
||||
const response = await fetch(`${baseUrl}${route}`, init);
|
||||
return { response, body: await response.json() };
|
||||
}
|
||||
|
||||
async function eventuallyEmpty(directory) {
|
||||
for (let attempt = 0; attempt < 50; attempt += 1) {
|
||||
if (fs.readdirSync(directory).length === 0) return;
|
||||
await new Promise((resolve) => setTimeout(resolve, 10));
|
||||
}
|
||||
assert.deepEqual(fs.readdirSync(directory), []);
|
||||
}
|
||||
|
||||
async function eventuallyContainsFile(directory) {
|
||||
for (let attempt = 0; attempt < 50; attempt += 1) {
|
||||
if (fs.readdirSync(directory).length > 0) return;
|
||||
await new Promise((resolve) => setTimeout(resolve, 10));
|
||||
}
|
||||
assert.fail("The upload did not create its temporary file");
|
||||
}
|
||||
|
||||
async function sampleImages() {
|
||||
const source = sharp({
|
||||
create: {
|
||||
width: 80,
|
||||
height: 60,
|
||||
channels: 4,
|
||||
background: { r: 31, g: 72, b: 120, alpha: 0.5 },
|
||||
},
|
||||
});
|
||||
return {
|
||||
jpeg: await source.clone().jpeg().toBuffer(),
|
||||
png: await source.clone().png().toBuffer(),
|
||||
webp: await source.clone().webp().toBuffer(),
|
||||
};
|
||||
}
|
||||
|
||||
test("asset HTTP API accepts JPEG, PNG, and WebP and serves private immutable media", async (t) => {
|
||||
const harness = await createHarness(t);
|
||||
const images = await sampleImages();
|
||||
const form = new FormData();
|
||||
appendFile(form, "files", images.jpeg, "welcome.jpg", "image/jpeg");
|
||||
appendFile(form, "files", images.png, "../../bulletin.png", "image/png");
|
||||
appendFile(form, "files", images.webp, "events.webp", "image/webp");
|
||||
|
||||
const uploaded = await jsonResponse(harness.baseUrl, "/api/assets", {
|
||||
method: "POST",
|
||||
body: form,
|
||||
});
|
||||
assert.equal(uploaded.response.status, 201);
|
||||
assert.equal(uploaded.body.createdCount, 3);
|
||||
assert.equal(uploaded.body.duplicateCount, 0);
|
||||
assert.equal(uploaded.body.results.length, 3);
|
||||
assert.deepEqual(
|
||||
uploaded.body.results.map((result) => result.asset.original.contentType),
|
||||
["image/jpeg", "image/png", "image/webp"],
|
||||
);
|
||||
assert.equal(uploaded.body.results[1].asset.originalFilename, "bulletin.png");
|
||||
assert.doesNotMatch(JSON.stringify(uploaded.body), /objectKey|object_key|private-seaweedfs/);
|
||||
assert.equal(harness.storage.objects.size, 9);
|
||||
await eventuallyEmpty(harness.tempDirectory);
|
||||
|
||||
const asset = uploaded.body.results[0].asset;
|
||||
const listed = await jsonResponse(harness.baseUrl, "/api/assets");
|
||||
assert.equal(listed.response.status, 200);
|
||||
assert.equal(listed.body.assets.length, 3);
|
||||
assert.deepEqual(
|
||||
(await jsonResponse(harness.baseUrl, `/api/assets/${asset.id}`)).body,
|
||||
asset,
|
||||
);
|
||||
|
||||
for (const media of [asset.display, asset.thumbnail]) {
|
||||
const response = await fetch(`${harness.baseUrl}${media.url}`);
|
||||
const body = Buffer.from(await response.arrayBuffer());
|
||||
assert.equal(response.status, 200);
|
||||
assert.equal(response.headers.get("content-type"), "image/webp");
|
||||
assert.equal(
|
||||
response.headers.get("cache-control"),
|
||||
"public, max-age=31536000, immutable",
|
||||
);
|
||||
assert.equal(response.headers.get("content-length"), String(body.length));
|
||||
assert.equal(response.headers.get("etag"), `"${media.sha256}"`);
|
||||
assert.equal(crypto.createHash("sha256").update(body).digest("hex"), media.sha256);
|
||||
}
|
||||
|
||||
const original = await fetch(`${harness.baseUrl}${asset.original.url}`);
|
||||
assert.equal(original.status, 200);
|
||||
assert.equal(original.headers.get("content-type"), "image/jpeg");
|
||||
assert.equal(original.headers.get("cache-control"), "private, no-store");
|
||||
assert.match(original.headers.get("content-disposition"), /^attachment;/);
|
||||
assert.deepEqual(Buffer.from(await original.arrayBuffer()), images.jpeg);
|
||||
|
||||
const wrongVersion = await jsonResponse(
|
||||
harness.baseUrl,
|
||||
`/media/assets/${asset.id}/display/${"a".repeat(64)}.webp`,
|
||||
);
|
||||
assert.equal(wrongVersion.response.status, 404);
|
||||
assert.equal(wrongVersion.body.error.code, "MEDIA_NOT_FOUND");
|
||||
const arbitraryKey = await jsonResponse(
|
||||
harness.baseUrl,
|
||||
`/media/assets/${asset.id}/display/${encodeURIComponent("../../original/key")}.webp`,
|
||||
);
|
||||
assert.equal(arbitraryKey.response.status, 404);
|
||||
assert.equal(arbitraryKey.body.error.code, "MEDIA_NOT_FOUND");
|
||||
|
||||
const displayKey = [...harness.storage.objects.keys()].find(
|
||||
(key) => key.includes(`assets/${asset.id}/display/`),
|
||||
);
|
||||
harness.storage.objects.delete(displayKey);
|
||||
const missingStoredBytes = await jsonResponse(
|
||||
harness.baseUrl,
|
||||
asset.display.url,
|
||||
);
|
||||
assert.equal(missingStoredBytes.response.status, 503);
|
||||
assert.equal(missingStoredBytes.body.error.code, "STORAGE_INCONSISTENCY");
|
||||
assert.equal(
|
||||
(await jsonResponse(harness.baseUrl, "/health/live")).response.status,
|
||||
200,
|
||||
);
|
||||
});
|
||||
|
||||
test("asset HTTP API reports duplicates, name conflicts, and persists editable metadata", async (t) => {
|
||||
const harness = await createHarness(t);
|
||||
const { png, jpeg } = await sampleImages();
|
||||
const firstForm = new FormData();
|
||||
appendFile(firstForm, "file", png, "weekly.png", "application/octet-stream");
|
||||
firstForm.append("display_name", "Weekly Announcement");
|
||||
const first = await jsonResponse(harness.baseUrl, "/api/assets", {
|
||||
method: "POST",
|
||||
body: firstForm,
|
||||
});
|
||||
assert.equal(first.response.status, 201);
|
||||
assert.equal(first.body.created, true);
|
||||
const assetId = first.body.asset.id;
|
||||
|
||||
const duplicateForm = new FormData();
|
||||
appendFile(duplicateForm, "file", png, "unrelated-name.png", "image/png");
|
||||
duplicateForm.append("display_name", "Ignored duplicate proposal");
|
||||
const duplicate = await jsonResponse(harness.baseUrl, "/api/assets", {
|
||||
method: "POST",
|
||||
body: duplicateForm,
|
||||
});
|
||||
assert.equal(duplicate.response.status, 200);
|
||||
assert.equal(duplicate.body.created, false);
|
||||
assert.equal(duplicate.body.duplicate, true);
|
||||
assert.equal(duplicate.body.asset.id, assetId);
|
||||
assert.equal(harness.storage.objects.size, 3);
|
||||
|
||||
const conflictForm = new FormData();
|
||||
appendFile(conflictForm, "file", jpeg, "different.jpg", "image/jpeg");
|
||||
conflictForm.append("displayName", "weekly announcement");
|
||||
const conflict = await jsonResponse(harness.baseUrl, "/api/assets", {
|
||||
method: "POST",
|
||||
body: conflictForm,
|
||||
});
|
||||
assert.equal(conflict.response.status, 409);
|
||||
assert.equal(conflict.body.error.code, "ASSET_NAME_CONFLICT");
|
||||
|
||||
const expiresAt = Date.parse("2034-01-02T03:04:05.000Z");
|
||||
const edited = await jsonResponse(harness.baseUrl, `/api/assets/${assetId}`, {
|
||||
method: "PATCH",
|
||||
headers: { "content-type": "application/json" },
|
||||
body: JSON.stringify({
|
||||
display_name: "Renamed announcement",
|
||||
expiration_at: expiresAt,
|
||||
}),
|
||||
});
|
||||
assert.equal(edited.response.status, 200);
|
||||
assert.equal(edited.body.displayName, "Renamed announcement");
|
||||
assert.equal(edited.body.expirationAt, expiresAt);
|
||||
const permanent = await jsonResponse(harness.baseUrl, `/api/assets/${assetId}`, {
|
||||
method: "PATCH",
|
||||
headers: { "content-type": "application/json" },
|
||||
body: JSON.stringify({ expiration_at: null }),
|
||||
});
|
||||
assert.equal(permanent.body.expirationAt, null);
|
||||
assert.equal(harness.storage.objects.size, 3);
|
||||
await eventuallyEmpty(harness.tempDirectory);
|
||||
|
||||
const timestamp = Date.now();
|
||||
harness.database
|
||||
.prepare(
|
||||
"INSERT INTO playlists (id, name, created_at, updated_at) VALUES (?, ?, ?, ?)",
|
||||
)
|
||||
.run("api-playlist", "API playlist", timestamp, timestamp);
|
||||
harness.database
|
||||
.prepare(
|
||||
`INSERT INTO playlist_entries
|
||||
(id, playlist_id, asset_id, position, created_at, updated_at)
|
||||
VALUES (?, ?, ?, 0, ?, ?)`,
|
||||
)
|
||||
.run("api-entry", "api-playlist", assetId, timestamp, timestamp);
|
||||
const references = await jsonResponse(
|
||||
harness.baseUrl,
|
||||
`/api/assets/${assetId}/references`,
|
||||
);
|
||||
assert.equal(references.response.status, 200);
|
||||
assert.equal(references.body.canDelete, false);
|
||||
assert.equal(references.body.playlists[0].playlistEntryId, "api-entry");
|
||||
const blockedDelete = await jsonResponse(
|
||||
harness.baseUrl,
|
||||
`/api/assets/${assetId}`,
|
||||
{ method: "DELETE" },
|
||||
);
|
||||
assert.equal(blockedDelete.response.status, 409);
|
||||
assert.equal(blockedDelete.body.error.code, "ASSET_REFERENCED");
|
||||
assert.equal(harness.storage.objects.size, 3);
|
||||
harness.database
|
||||
.prepare("DELETE FROM playlist_entries WHERE id = ?")
|
||||
.run("api-entry");
|
||||
|
||||
const removed = await jsonResponse(harness.baseUrl, `/api/assets/${assetId}`, {
|
||||
method: "DELETE",
|
||||
});
|
||||
assert.equal(removed.response.status, 200);
|
||||
assert.deepEqual(removed.body, { id: assetId, deleted: true });
|
||||
assert.equal(harness.storage.objects.size, 0);
|
||||
assert.equal(
|
||||
(await jsonResponse(harness.baseUrl, `/api/assets/${assetId}`)).response.status,
|
||||
404,
|
||||
);
|
||||
});
|
||||
|
||||
test("upload validation returns structured errors and always cleans temporary files", async (t) => {
|
||||
const harness = await createHarness(t, { maxFileBytes: 1024 });
|
||||
|
||||
const malformed = new FormData();
|
||||
appendFile(malformed, "file", Buffer.from("not an image"), "broken.png", "image/png");
|
||||
const malformedResult = await jsonResponse(harness.baseUrl, "/api/assets", {
|
||||
method: "POST",
|
||||
body: malformed,
|
||||
});
|
||||
assert.equal(malformedResult.response.status, 422);
|
||||
assert.equal(malformedResult.body.error.code, "MALFORMED_IMAGE");
|
||||
await eventuallyEmpty(harness.tempDirectory);
|
||||
|
||||
const unsupportedBuffer = await sharp({
|
||||
create: {
|
||||
width: 20,
|
||||
height: 20,
|
||||
channels: 3,
|
||||
background: "red",
|
||||
},
|
||||
})
|
||||
.gif()
|
||||
.toBuffer();
|
||||
const unsupported = new FormData();
|
||||
appendFile(unsupported, "file", unsupportedBuffer, "still.gif", "image/png");
|
||||
const unsupportedResult = await jsonResponse(harness.baseUrl, "/api/assets", {
|
||||
method: "POST",
|
||||
body: unsupported,
|
||||
});
|
||||
assert.equal(unsupportedResult.response.status, 415);
|
||||
assert.equal(unsupportedResult.body.error.code, "UNSUPPORTED_IMAGE_FORMAT");
|
||||
await eventuallyEmpty(harness.tempDirectory);
|
||||
|
||||
const oversized = new FormData();
|
||||
appendFile(
|
||||
oversized,
|
||||
"file",
|
||||
crypto.randomBytes(1025),
|
||||
"too-large.png",
|
||||
"image/png",
|
||||
);
|
||||
const oversizedResult = await jsonResponse(harness.baseUrl, "/api/assets", {
|
||||
method: "POST",
|
||||
body: oversized,
|
||||
});
|
||||
assert.equal(oversizedResult.response.status, 413);
|
||||
assert.equal(oversizedResult.body.error.code, "UPLOAD_FILE_TOO_LARGE");
|
||||
assert.deepEqual(oversizedResult.body.error.details, { maxFileBytes: 1024 });
|
||||
await eventuallyEmpty(harness.tempDirectory);
|
||||
|
||||
const tooMany = new FormData();
|
||||
for (let index = 0; index < 6; index += 1) {
|
||||
appendFile(tooMany, "files", Buffer.from("x"), `${index}.png`, "image/png");
|
||||
}
|
||||
const tooManyResult = await jsonResponse(harness.baseUrl, "/api/assets", {
|
||||
method: "POST",
|
||||
body: tooMany,
|
||||
});
|
||||
assert.equal(tooManyResult.response.status, 413);
|
||||
assert.equal(tooManyResult.body.error.code, "UPLOAD_FILE_COUNT_EXCEEDED");
|
||||
assert.deepEqual(tooManyResult.body.error.details, { maxFiles: 5 });
|
||||
await eventuallyEmpty(harness.tempDirectory);
|
||||
|
||||
const absent = await jsonResponse(harness.baseUrl, "/api/assets", {
|
||||
method: "POST",
|
||||
body: new FormData(),
|
||||
});
|
||||
assert.equal(absent.response.status, 400);
|
||||
assert.equal(absent.body.error.code, "UPLOAD_FILE_REQUIRED");
|
||||
assert.equal(harness.storage.objects.size, 0);
|
||||
assert.equal(
|
||||
harness.database.prepare("SELECT COUNT(*) AS count FROM assets").get().count,
|
||||
0,
|
||||
);
|
||||
});
|
||||
|
||||
test("an aborted multipart request removes its in-flight temporary file", async (t) => {
|
||||
const harness = await createHarness(t);
|
||||
const boundary = `announcements-${crypto.randomUUID()}`;
|
||||
const preamble = Buffer.from(
|
||||
`--${boundary}\r\n` +
|
||||
'Content-Disposition: form-data; name="file"; filename="aborted.png"\r\n' +
|
||||
"Content-Type: image/png\r\n\r\n",
|
||||
);
|
||||
const declaredLength = preamble.length + 512 * 1024;
|
||||
const request = http.request(`${harness.baseUrl}/api/assets`, {
|
||||
method: "POST",
|
||||
headers: {
|
||||
"content-type": `multipart/form-data; boundary=${boundary}`,
|
||||
"content-length": String(declaredLength),
|
||||
},
|
||||
});
|
||||
request.on("error", () => {});
|
||||
request.write(preamble);
|
||||
request.write(Buffer.alloc(64 * 1024, 1));
|
||||
|
||||
await eventuallyContainsFile(harness.tempDirectory);
|
||||
const closed = new Promise((resolve) => request.once("close", resolve));
|
||||
request.destroy();
|
||||
await closed;
|
||||
await eventuallyEmpty(harness.tempDirectory);
|
||||
assert.equal(harness.storage.objects.size, 0);
|
||||
assert.equal(
|
||||
harness.database.prepare("SELECT COUNT(*) AS count FROM assets").get().count,
|
||||
0,
|
||||
);
|
||||
});
|
||||
@@ -0,0 +1,35 @@
|
||||
const test = require("node:test");
|
||||
const assert = require("node:assert/strict");
|
||||
const {
|
||||
mapMulterError,
|
||||
safeOriginalFilename,
|
||||
} = require("../src/assets/multipart");
|
||||
|
||||
test("original filename metadata keeps a safe leaf without path traversal", () => {
|
||||
assert.equal(safeOriginalFilename("../../Sunday/welcome.png"), "welcome.png");
|
||||
assert.equal(safeOriginalFilename("..\\..\\Sunday\\welcome.png"), "welcome.png");
|
||||
assert.equal(safeOriginalFilename(" welcome sign.png "), " welcome sign.png ");
|
||||
});
|
||||
|
||||
test("original filename metadata removes controls and remains bounded", () => {
|
||||
assert.equal(safeOriginalFilename("wel\u0000come\n.png"), "welcome.png");
|
||||
assert.equal([...safeOriginalFilename(`${"a".repeat(300)}.png`)].length, 255);
|
||||
assert.throws(
|
||||
() => safeOriginalFilename(" \u0000\n "),
|
||||
(error) => error.code === "INVALID_ORIGINAL_FILENAME",
|
||||
);
|
||||
});
|
||||
|
||||
test("temporary-storage failures are server errors, not invalid client uploads", () => {
|
||||
const storageFailure = mapMulterError(
|
||||
Object.assign(new Error("disk full at a private path"), { code: "ENOSPC" }),
|
||||
{},
|
||||
);
|
||||
assert.equal(storageFailure.code, "UPLOAD_TEMP_STORAGE_UNAVAILABLE");
|
||||
assert.equal(storageFailure.statusCode, 503);
|
||||
assert.doesNotMatch(storageFailure.message, /private path/);
|
||||
|
||||
const malformed = mapMulterError(new Error("Unexpected end of form"), {});
|
||||
assert.equal(malformed.code, "INVALID_MULTIPART_UPLOAD");
|
||||
assert.equal(malformed.statusCode, 400);
|
||||
});
|
||||
@@ -0,0 +1,608 @@
|
||||
const test = require("node:test");
|
||||
const assert = require("node:assert/strict");
|
||||
const crypto = require("node:crypto");
|
||||
const fs = require("node:fs");
|
||||
const os = require("node:os");
|
||||
const path = require("node:path");
|
||||
const { Readable } = require("node:stream");
|
||||
const { openDatabase } = require("../src/database");
|
||||
const { runMigrations } = require("../src/database/migrate");
|
||||
const { createAssetService } = require("../src/assets/service");
|
||||
const {
|
||||
StorageObjectNotFoundError,
|
||||
StorageUnavailableError,
|
||||
} = require("../src/storage");
|
||||
|
||||
async function bufferBody(body) {
|
||||
if (Buffer.isBuffer(body)) return body;
|
||||
if (body instanceof Uint8Array) return Buffer.from(body);
|
||||
const chunks = [];
|
||||
for await (const chunk of body) chunks.push(Buffer.from(chunk));
|
||||
return Buffer.concat(chunks);
|
||||
}
|
||||
|
||||
function createMemoryStorage() {
|
||||
const objects = new Map();
|
||||
const calls = [];
|
||||
let failNextPut = null;
|
||||
let successfulPutsBeforeFailure = 0;
|
||||
let failNextDelete = null;
|
||||
let putDelayMs = 0;
|
||||
|
||||
return {
|
||||
objects,
|
||||
calls,
|
||||
setPutFailure(error, putsBeforeFailure = 0) {
|
||||
failNextPut = error;
|
||||
successfulPutsBeforeFailure = putsBeforeFailure;
|
||||
},
|
||||
setDeleteFailure(error) {
|
||||
failNextDelete = error;
|
||||
},
|
||||
setPutDelay(milliseconds) {
|
||||
putDelayMs = milliseconds;
|
||||
},
|
||||
async putObject(input) {
|
||||
calls.push({ operation: "put", ...input, body: undefined });
|
||||
if (putDelayMs) {
|
||||
await new Promise((resolve) => setTimeout(resolve, putDelayMs));
|
||||
}
|
||||
if (failNextPut) {
|
||||
if (successfulPutsBeforeFailure === 0) {
|
||||
const error = failNextPut;
|
||||
failNextPut = null;
|
||||
throw error;
|
||||
}
|
||||
successfulPutsBeforeFailure -= 1;
|
||||
}
|
||||
objects.set(input.key, {
|
||||
body: await bufferBody(input.body),
|
||||
contentType: input.contentType,
|
||||
metadata: input.metadata,
|
||||
});
|
||||
return { key: input.key };
|
||||
},
|
||||
async getObject(key) {
|
||||
calls.push({ operation: "get", key });
|
||||
const object = objects.get(key);
|
||||
if (!object) throw new StorageObjectNotFoundError(key);
|
||||
return {
|
||||
body: Readable.from(object.body),
|
||||
contentLength: object.body.length,
|
||||
contentType: object.contentType,
|
||||
};
|
||||
},
|
||||
async deleteObject(key) {
|
||||
calls.push({ operation: "delete", key });
|
||||
if (failNextDelete) {
|
||||
const error = failNextDelete;
|
||||
failNextDelete = null;
|
||||
throw error;
|
||||
}
|
||||
objects.delete(key);
|
||||
return { key, deleted: true };
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
function createFakeImageProcessor({ reject } = {}) {
|
||||
const calls = [];
|
||||
return {
|
||||
calls,
|
||||
async process(filename) {
|
||||
calls.push(filename);
|
||||
if (reject) throw reject;
|
||||
const original = await fs.promises.readFile(filename);
|
||||
const sourceSha256 = crypto.createHash("sha256").update(original).digest("hex");
|
||||
const displayBuffer = Buffer.from(`display:${sourceSha256}`);
|
||||
const thumbnailBuffer = Buffer.from(`thumbnail:${sourceSha256}`);
|
||||
return {
|
||||
original: {
|
||||
sha256: sourceSha256,
|
||||
contentType: "image/png",
|
||||
byteSize: original.length,
|
||||
width: 1200,
|
||||
height: 800,
|
||||
},
|
||||
display: {
|
||||
buffer: displayBuffer,
|
||||
sha256: crypto.createHash("sha256").update(displayBuffer).digest("hex"),
|
||||
contentType: "image/webp",
|
||||
byteSize: displayBuffer.length,
|
||||
width: 1200,
|
||||
height: 800,
|
||||
},
|
||||
thumbnail: {
|
||||
buffer: thumbnailBuffer,
|
||||
sha256: crypto.createHash("sha256").update(thumbnailBuffer).digest("hex"),
|
||||
contentType: "image/webp",
|
||||
byteSize: thumbnailBuffer.length,
|
||||
width: 405,
|
||||
height: 270,
|
||||
},
|
||||
};
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
function createHarness(options = {}) {
|
||||
const database = openDatabase({ path: ":memory:" });
|
||||
runMigrations(database);
|
||||
const directory = fs.mkdtempSync(path.join(os.tmpdir(), "announcements-assets-"));
|
||||
const storage = options.storage || createMemoryStorage();
|
||||
const imageProcessor =
|
||||
options.imageProcessor || createFakeImageProcessor(options.processorOptions);
|
||||
let idCounter = 0;
|
||||
let clock = 2_000_000_000_000;
|
||||
const service = createAssetService({
|
||||
database,
|
||||
storage,
|
||||
imageProcessor,
|
||||
now: () => ++clock,
|
||||
createId:
|
||||
options.createId ||
|
||||
(() => `00000000-0000-4000-8000-${String(++idCounter).padStart(12, "0")}`),
|
||||
concurrentUploadWaitMs: options.concurrentUploadWaitMs ?? 2_000,
|
||||
});
|
||||
|
||||
function uploadFile(originalFilename, contents) {
|
||||
const buffer = Buffer.isBuffer(contents) ? contents : Buffer.from(contents);
|
||||
const filename = path.join(directory, crypto.randomUUID());
|
||||
fs.writeFileSync(filename, buffer);
|
||||
return { path: filename, byteSize: buffer.length, originalFilename };
|
||||
}
|
||||
|
||||
return {
|
||||
database,
|
||||
directory,
|
||||
imageProcessor,
|
||||
service,
|
||||
storage,
|
||||
uploadFile,
|
||||
close() {
|
||||
database.close();
|
||||
fs.rmSync(directory, { recursive: true, force: true });
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
function rawAsset(database, assetId) {
|
||||
return database.prepare("SELECT * FROM assets WHERE id = ?").get(assetId);
|
||||
}
|
||||
|
||||
test("upload stores exact bytes under content-addressed keys without leaking keys", async (t) => {
|
||||
const harness = createHarness();
|
||||
t.after(() => harness.close());
|
||||
const original = Buffer.from("exact uploaded bytes");
|
||||
|
||||
const [result] = await harness.service.upload([
|
||||
harness.uploadFile(" Sunday Welcome.PNG ", original),
|
||||
]);
|
||||
|
||||
assert.equal(result.created, true);
|
||||
assert.equal(result.duplicate, false);
|
||||
assert.match(result.asset.id, /^[0-9a-f-]{36}$/);
|
||||
assert.equal(result.asset.displayName, "Sunday Welcome");
|
||||
assert.equal(result.asset.originalFilename, " Sunday Welcome.PNG ");
|
||||
assert.equal(result.asset.expirationAt, null);
|
||||
assert.equal(result.asset.status, "ready");
|
||||
assert.equal(result.asset.original.byteSize, original.length);
|
||||
assert.equal(result.asset.display.contentType, "image/webp");
|
||||
assert.equal(result.asset.thumbnail.width, 405);
|
||||
assert.match(result.asset.display.url, /^\/media\/assets\//);
|
||||
assert.doesNotMatch(
|
||||
JSON.stringify(result.asset),
|
||||
/object_key|originalObjectKey|displayObjectKey|thumbnailObjectKey|seaweed/i,
|
||||
);
|
||||
|
||||
const row = rawAsset(harness.database, result.asset.id);
|
||||
const expectedSourceHash = crypto.createHash("sha256").update(original).digest("hex");
|
||||
assert.equal(row.source_sha256, expectedSourceHash);
|
||||
assert.equal(row.original_object_key, `assets/${row.id}/original/${expectedSourceHash}`);
|
||||
assert.equal(
|
||||
row.display_object_key,
|
||||
`assets/${row.id}/display/${row.display_sha256}.webp`,
|
||||
);
|
||||
assert.equal(
|
||||
row.thumbnail_object_key,
|
||||
`assets/${row.id}/thumbnail/${row.thumbnail_sha256}.webp`,
|
||||
);
|
||||
assert.deepEqual(harness.storage.objects.get(row.original_object_key).body, original);
|
||||
assert.equal(harness.storage.objects.size, 3);
|
||||
|
||||
assert.deepEqual(harness.service.list(), [result.asset]);
|
||||
assert.deepEqual(harness.service.get(result.asset.id), result.asset);
|
||||
});
|
||||
|
||||
test("exact duplicate detection takes precedence over filename and display name", async (t) => {
|
||||
const harness = createHarness();
|
||||
t.after(() => harness.close());
|
||||
const contents = Buffer.from("same image bytes");
|
||||
|
||||
const [created] = await harness.service.upload([
|
||||
harness.uploadFile("first.png", contents),
|
||||
]);
|
||||
const [sameFilename] = await harness.service.upload([
|
||||
harness.uploadFile("first.png", contents),
|
||||
]);
|
||||
const [differentFilename] = await harness.service.upload(
|
||||
[harness.uploadFile("completely different name.webp", contents)],
|
||||
{ displayName: "A conflicting requested name is ignored for an exact duplicate" },
|
||||
);
|
||||
|
||||
for (const duplicate of [sameFilename, differentFilename]) {
|
||||
assert.equal(duplicate.created, false);
|
||||
assert.equal(duplicate.duplicate, true);
|
||||
assert.equal(duplicate.asset.id, created.asset.id);
|
||||
}
|
||||
assert.equal(harness.service.list().length, 1);
|
||||
assert.equal(harness.storage.calls.filter((call) => call.operation === "put").length, 3);
|
||||
assert.equal(harness.imageProcessor.calls.length, 1);
|
||||
});
|
||||
|
||||
test("different content with the same proposed name is a conflict, not a duplicate", async (t) => {
|
||||
const harness = createHarness();
|
||||
t.after(() => harness.close());
|
||||
await harness.service.upload([harness.uploadFile("bulletin.png", "image one")]);
|
||||
|
||||
await assert.rejects(
|
||||
harness.service.upload([harness.uploadFile("bulletin.jpg", "image two")]),
|
||||
(error) => {
|
||||
assert.equal(error.statusCode, 409);
|
||||
assert.equal(error.code, "ASSET_NAME_CONFLICT");
|
||||
assert.equal(error.details.asset.displayName, "bulletin");
|
||||
return true;
|
||||
},
|
||||
);
|
||||
assert.equal(harness.service.list().length, 1);
|
||||
assert.equal(harness.imageProcessor.calls.length, 1);
|
||||
|
||||
const [created] = await harness.service.upload(
|
||||
[harness.uploadFile("bulletin.jpg", "image two")],
|
||||
{ displayName: "Second bulletin" },
|
||||
);
|
||||
assert.equal(created.created, true);
|
||||
assert.notEqual(created.asset.sourceSha256, harness.service.list()[1].sourceSha256);
|
||||
assert.equal(harness.service.list().length, 2);
|
||||
});
|
||||
|
||||
test("multiple-file upload preserves order and deduplicates repeated bytes in the batch", async (t) => {
|
||||
const harness = createHarness();
|
||||
t.after(() => harness.close());
|
||||
|
||||
const results = await harness.service.upload([
|
||||
harness.uploadFile("alpha.png", "alpha bytes"),
|
||||
harness.uploadFile("beta.png", "beta bytes"),
|
||||
harness.uploadFile("alpha-copy.png", "alpha bytes"),
|
||||
]);
|
||||
|
||||
assert.deepEqual(
|
||||
results.map(({ created, duplicate }) => ({ created, duplicate })),
|
||||
[
|
||||
{ created: true, duplicate: false },
|
||||
{ created: true, duplicate: false },
|
||||
{ created: false, duplicate: true },
|
||||
],
|
||||
);
|
||||
assert.equal(results[0].asset.id, results[2].asset.id);
|
||||
assert.equal(harness.service.list().length, 2);
|
||||
assert.equal(harness.imageProcessor.calls.length, 2);
|
||||
assert.equal(harness.storage.objects.size, 6);
|
||||
});
|
||||
|
||||
test("all new images are decoded before the batch mutates SQLite or storage", async (t) => {
|
||||
const expected = Object.assign(new Error("bad second image"), {
|
||||
code: "MALFORMED_IMAGE",
|
||||
statusCode: 422,
|
||||
});
|
||||
let processCount = 0;
|
||||
const regularProcessor = createFakeImageProcessor();
|
||||
const harness = createHarness({
|
||||
imageProcessor: {
|
||||
async process(filename) {
|
||||
processCount += 1;
|
||||
if (processCount === 2) throw expected;
|
||||
return regularProcessor.process(filename);
|
||||
},
|
||||
},
|
||||
});
|
||||
t.after(() => harness.close());
|
||||
|
||||
await assert.rejects(
|
||||
harness.service.upload([
|
||||
harness.uploadFile("valid.png", "valid"),
|
||||
harness.uploadFile("broken.png", "broken"),
|
||||
]),
|
||||
expected,
|
||||
);
|
||||
assert.equal(harness.service.list().length, 0);
|
||||
assert.equal(harness.storage.objects.size, 0);
|
||||
assert.equal(harness.storage.calls.length, 0);
|
||||
});
|
||||
|
||||
test("concurrent identical uploads converge on one ready asset", async (t) => {
|
||||
const storage = createMemoryStorage();
|
||||
storage.setPutDelay(30);
|
||||
const harness = createHarness({ storage });
|
||||
t.after(() => harness.close());
|
||||
const bytes = Buffer.from("racing identical upload");
|
||||
|
||||
const [[left], [right]] = await Promise.all([
|
||||
harness.service.upload([harness.uploadFile("race.png", bytes)]),
|
||||
harness.service.upload([harness.uploadFile("other-name.png", bytes)]),
|
||||
]);
|
||||
|
||||
assert.equal(harness.service.list().length, 1);
|
||||
assert.equal(left.asset.id, right.asset.id);
|
||||
assert.deepEqual([left.created, right.created].sort(), [false, true]);
|
||||
assert.deepEqual([left.duplicate, right.duplicate].sort(), [false, true]);
|
||||
assert.equal(storage.calls.filter((call) => call.operation === "put").length, 3);
|
||||
});
|
||||
|
||||
test("a partial storage failure leaves a diagnosable failed row and no ready URLs", async (t) => {
|
||||
const storage = createMemoryStorage();
|
||||
storage.setPutFailure(new StorageUnavailableError(new Error("offline")), 1);
|
||||
const harness = createHarness({ storage });
|
||||
t.after(() => harness.close());
|
||||
|
||||
await assert.rejects(
|
||||
harness.service.upload([harness.uploadFile("failure.png", "failure bytes")]),
|
||||
(error) => {
|
||||
assert.equal(error.statusCode, 503);
|
||||
assert.equal(error.details.retryable, true);
|
||||
assert.match(error.details.assetId, /^[0-9a-f-]{36}$/);
|
||||
return true;
|
||||
},
|
||||
);
|
||||
|
||||
const [failed] = harness.service.list();
|
||||
assert.equal(failed.status, "failed");
|
||||
assert.match(failed.processingError, /^STORAGE_UNAVAILABLE:/);
|
||||
assert.equal(failed.display.url, null);
|
||||
assert.equal(failed.thumbnail.url, null);
|
||||
assert.equal(failed.original.url, null);
|
||||
assert.equal(storage.objects.size, 0);
|
||||
const row = rawAsset(harness.database, failed.id);
|
||||
assert.match(
|
||||
row.original_object_key,
|
||||
new RegExp(`^assets/${failed.id}/original/[0-9a-f]{64}$`),
|
||||
);
|
||||
assert.equal(
|
||||
row.display_object_key,
|
||||
`assets/${failed.id}/display/${row.display_sha256}.webp`,
|
||||
);
|
||||
assert.equal(
|
||||
row.thumbnail_object_key,
|
||||
`assets/${failed.id}/thumbnail/${row.thumbnail_sha256}.webp`,
|
||||
);
|
||||
assert.equal(storage.calls.filter((call) => call.operation === "put").length, 2);
|
||||
assert.deepEqual(
|
||||
new Set(
|
||||
storage.calls
|
||||
.filter((call) => call.operation === "delete")
|
||||
.map((call) => call.key),
|
||||
),
|
||||
new Set([
|
||||
row.original_object_key,
|
||||
row.display_object_key,
|
||||
row.thumbnail_object_key,
|
||||
]),
|
||||
);
|
||||
|
||||
await assert.rejects(
|
||||
harness.service.upload([harness.uploadFile("retry.png", "failure bytes")]),
|
||||
(error) => error.code === "ASSET_PROCESSING_FAILED" && error.statusCode === 409,
|
||||
);
|
||||
|
||||
assert.deepEqual(await harness.service.remove(failed.id), {
|
||||
id: failed.id,
|
||||
deleted: true,
|
||||
});
|
||||
assert.equal(harness.service.list().length, 0);
|
||||
});
|
||||
|
||||
test("rename and expiration edits change only approved SQLite metadata", async (t) => {
|
||||
const harness = createHarness();
|
||||
t.after(() => harness.close());
|
||||
const [upload] = await harness.service.upload([
|
||||
harness.uploadFile("editable.png", "editable bytes"),
|
||||
]);
|
||||
const before = rawAsset(harness.database, upload.asset.id);
|
||||
|
||||
const expiring = harness.service.update(upload.asset.id, {
|
||||
display_name: " New display name ",
|
||||
expiration_at: "2035-05-06T07:08:09.000Z",
|
||||
});
|
||||
assert.equal(expiring.displayName, "New display name");
|
||||
assert.equal(expiring.expirationAt, Date.parse("2035-05-06T07:08:09.000Z"));
|
||||
|
||||
const permanent = harness.service.update(upload.asset.id, { expirationAt: null });
|
||||
assert.equal(permanent.expirationAt, null);
|
||||
const after = rawAsset(harness.database, upload.asset.id);
|
||||
assert.equal(after.original_object_key, before.original_object_key);
|
||||
assert.equal(after.display_object_key, before.display_object_key);
|
||||
assert.equal(after.thumbnail_object_key, before.thumbnail_object_key);
|
||||
assert.equal(harness.storage.calls.filter((call) => call.operation === "put").length, 3);
|
||||
|
||||
await assert.rejects(
|
||||
async () => harness.service.update(upload.asset.id, { status: "failed" }),
|
||||
(error) => error.code === "INVALID_ASSET_UPDATE",
|
||||
);
|
||||
assert.throws(
|
||||
() => harness.service.update(upload.asset.id, { expirationAt: -1 }),
|
||||
(error) => error.code === "INVALID_ASSET_EXPIRATION",
|
||||
);
|
||||
});
|
||||
|
||||
test("references describe playlist, normal, and static use and block deletion", async (t) => {
|
||||
const harness = createHarness();
|
||||
t.after(() => harness.close());
|
||||
const [upload] = await harness.service.upload([
|
||||
harness.uploadFile("referenced.png", "referenced bytes"),
|
||||
]);
|
||||
const id = upload.asset.id;
|
||||
const now = 2_100_000_000_000;
|
||||
harness.database
|
||||
.prepare(
|
||||
"INSERT INTO playlists (id, name, created_at, updated_at) VALUES (?, ?, ?, ?)",
|
||||
)
|
||||
.run("playlist-1", "Sunday", now, now);
|
||||
harness.database
|
||||
.prepare(
|
||||
`INSERT INTO playlist_entries
|
||||
(id, playlist_id, asset_id, enabled, position, created_at, updated_at)
|
||||
VALUES (?, ?, ?, 1, 0, ?, ?)`,
|
||||
)
|
||||
.run("entry-1", "playlist-1", id, now, now);
|
||||
harness.database
|
||||
.prepare(
|
||||
`UPDATE playback_state
|
||||
SET active_playlist_id = ?, current_entry_id = ?, static_asset_id = ?, updated_at = ?
|
||||
WHERE output_id = 'global'`,
|
||||
)
|
||||
.run("playlist-1", "entry-1", id, now);
|
||||
|
||||
const references = harness.service.references(id);
|
||||
assert.equal(references.canDelete, false);
|
||||
assert.deepEqual(references.blockerCodes, [
|
||||
"PLAYLIST_REFERENCES",
|
||||
"CURRENT_PLAYBACK_TARGET",
|
||||
"CURRENT_STATIC_TARGET",
|
||||
]);
|
||||
assert.deepEqual(references.playlists[0], {
|
||||
playlistId: "playlist-1",
|
||||
playlistName: "Sunday",
|
||||
playlistEntryId: "entry-1",
|
||||
enabled: true,
|
||||
position: 0,
|
||||
isActivePlaylist: true,
|
||||
isCurrentNormalTarget: true,
|
||||
});
|
||||
assert.equal(references.playback[0].isCurrentNormalTarget, true);
|
||||
assert.equal(references.playback[0].isCurrentStaticTarget, true);
|
||||
|
||||
const callsBeforeDelete = harness.storage.calls.length;
|
||||
await assert.rejects(
|
||||
harness.service.remove(id),
|
||||
(error) => {
|
||||
assert.equal(error.code, "ASSET_REFERENCED");
|
||||
assert.equal(error.statusCode, 409);
|
||||
assert.deepEqual(error.details.blockerCodes, [
|
||||
"PLAYLIST_REFERENCES",
|
||||
"CURRENT_PLAYBACK_TARGET",
|
||||
"CURRENT_STATIC_TARGET",
|
||||
]);
|
||||
return true;
|
||||
},
|
||||
);
|
||||
assert.equal(harness.storage.calls.length, callsBeforeDelete);
|
||||
assert.equal(rawAsset(harness.database, id).status, "ready");
|
||||
|
||||
harness.database
|
||||
.prepare(
|
||||
`UPDATE playback_state
|
||||
SET active_playlist_id = NULL, current_entry_id = NULL, static_asset_id = NULL
|
||||
WHERE output_id = 'global'`,
|
||||
)
|
||||
.run();
|
||||
harness.database.prepare("DELETE FROM playlist_entries WHERE id = ?").run("entry-1");
|
||||
assert.equal(harness.service.references(id).canDelete, true);
|
||||
await harness.service.remove(id);
|
||||
assert.equal(harness.storage.objects.size, 0);
|
||||
assert.equal(rawAsset(harness.database, id), undefined);
|
||||
});
|
||||
|
||||
test("failed storage deletion retains a retryable deleting row", async (t) => {
|
||||
const storage = createMemoryStorage();
|
||||
const harness = createHarness({ storage });
|
||||
t.after(() => harness.close());
|
||||
const [upload] = await harness.service.upload([
|
||||
harness.uploadFile("delete-me.png", "delete me"),
|
||||
]);
|
||||
storage.setDeleteFailure(new StorageUnavailableError(new Error("offline")));
|
||||
|
||||
await assert.rejects(harness.service.remove(upload.asset.id), (error) => {
|
||||
assert.equal(error.statusCode, 503);
|
||||
assert.equal(error.details.assetId, upload.asset.id);
|
||||
assert.equal(error.details.retryable, true);
|
||||
return true;
|
||||
});
|
||||
assert.equal(rawAsset(harness.database, upload.asset.id).status, "deleting");
|
||||
assert.throws(
|
||||
() => harness.service.update(upload.asset.id, { displayName: "Cannot edit" }),
|
||||
(error) => error.code === "ASSET_DELETING",
|
||||
);
|
||||
|
||||
await harness.service.remove(upload.asset.id);
|
||||
assert.equal(rawAsset(harness.database, upload.asset.id), undefined);
|
||||
assert.equal(storage.objects.size, 0);
|
||||
});
|
||||
|
||||
test("processing assets cannot be deleted and storage is not touched", async (t) => {
|
||||
const harness = createHarness();
|
||||
t.after(() => harness.close());
|
||||
const timestamp = Date.now();
|
||||
harness.database
|
||||
.prepare(
|
||||
`INSERT INTO assets
|
||||
(id, source_sha256, display_name, original_filename, status, created_at, updated_at)
|
||||
VALUES (?, ?, ?, ?, 'processing', ?, ?)`,
|
||||
)
|
||||
.run(
|
||||
"00000000-0000-4000-8000-999999999999",
|
||||
"f".repeat(64),
|
||||
"Still processing",
|
||||
"processing.png",
|
||||
timestamp,
|
||||
timestamp,
|
||||
);
|
||||
|
||||
assert.equal(
|
||||
harness.service.references("00000000-0000-4000-8000-999999999999")
|
||||
.canDelete,
|
||||
false,
|
||||
);
|
||||
await assert.rejects(
|
||||
harness.service.remove("00000000-0000-4000-8000-999999999999"),
|
||||
(error) => error.code === "ASSET_PROCESSING" && error.statusCode === 409,
|
||||
);
|
||||
assert.equal(harness.storage.calls.length, 0);
|
||||
assert.equal(
|
||||
rawAsset(harness.database, "00000000-0000-4000-8000-999999999999").status,
|
||||
"processing",
|
||||
);
|
||||
});
|
||||
|
||||
test("media descriptors only allow the recorded version and map missing bytes to 503", async (t) => {
|
||||
const harness = createHarness();
|
||||
t.after(() => harness.close());
|
||||
const [upload] = await harness.service.upload([
|
||||
harness.uploadFile("media.png", "media bytes"),
|
||||
]);
|
||||
const { asset } = upload;
|
||||
|
||||
const display = harness.service.mediaDescriptor(
|
||||
asset.id,
|
||||
"display",
|
||||
asset.display.sha256,
|
||||
);
|
||||
assert.equal(display.contentType, "image/webp");
|
||||
assert.equal(display.contentLength, asset.display.byteSize);
|
||||
const stored = await harness.service.readStoredObject(display);
|
||||
assert.deepEqual(await bufferBody(stored.body), harness.storage.objects.get(display.key).body);
|
||||
|
||||
for (const requestedHash of ["a".repeat(64), "../original/key", "not-a-hash"]) {
|
||||
assert.throws(
|
||||
() => harness.service.mediaDescriptor(asset.id, "display", requestedHash),
|
||||
(error) => error.code === "MEDIA_NOT_FOUND" && error.statusCode === 404,
|
||||
);
|
||||
}
|
||||
|
||||
harness.storage.objects.delete(display.key);
|
||||
await assert.rejects(harness.service.readStoredObject(display), (error) => {
|
||||
assert.equal(error.code, "STORAGE_INCONSISTENCY");
|
||||
assert.equal(error.statusCode, 503);
|
||||
assert.deepEqual(error.details, { assetId: asset.id, retryable: true });
|
||||
return true;
|
||||
});
|
||||
});
|
||||
@@ -32,6 +32,14 @@ test("loadConfig validates and normalizes the supported environment", () => {
|
||||
assert.equal(config.timeZone, "America/Chicago");
|
||||
assert.equal(config.storage.endpoint, "http://seaweedfs:8333");
|
||||
assert.equal(config.storage.forcePathStyle, true);
|
||||
assert.equal(config.storage.healthProbeIntervalMs, 30_000);
|
||||
assert.equal(config.storage.connectionTimeoutMs, 2_000);
|
||||
assert.equal(config.storage.requestTimeoutMs, 5_000);
|
||||
assert.equal(config.storage.maxAttempts, 2);
|
||||
assert.equal(config.processing.tempDirectory, "/tmp/announcements-uploads");
|
||||
assert.equal(config.processing.maxFileBytes, 25 * 1024 * 1024);
|
||||
assert.equal(config.processing.maxFiles, 20);
|
||||
assert.equal(config.processing.maxInputPixels, 40_000_000);
|
||||
});
|
||||
|
||||
test("production requires private S3 credentials", () => {
|
||||
@@ -65,10 +73,42 @@ test("loadConfig rejects invalid typed values", () => {
|
||||
() => loadConfig(validEnvironment({ S3_FORCE_PATH_STYLE: "sometimes" })),
|
||||
/must be a boolean/,
|
||||
);
|
||||
assert.throws(
|
||||
() => loadConfig(validEnvironment({ S3_HEALTH_PROBE_INTERVAL_MS: "999" })),
|
||||
/S3_HEALTH_PROBE_INTERVAL_MS must be at least 1000/,
|
||||
);
|
||||
assert.throws(
|
||||
() => loadConfig(validEnvironment({ S3_REQUEST_TIMEOUT_MS: "999" })),
|
||||
/S3_REQUEST_TIMEOUT_MS must be at least 1000/,
|
||||
);
|
||||
assert.throws(
|
||||
() => loadConfig(validEnvironment({ S3_CONNECT_TIMEOUT_MS: "249" })),
|
||||
/S3_CONNECT_TIMEOUT_MS must be at least 250/,
|
||||
);
|
||||
assert.throws(
|
||||
() => loadConfig(validEnvironment({ S3_MAX_ATTEMPTS: "6" })),
|
||||
/S3_MAX_ATTEMPTS must be at most 5/,
|
||||
);
|
||||
assert.throws(
|
||||
() => loadConfig(validEnvironment({ DATABASE_PATH: "relative.sqlite" })),
|
||||
/must be an absolute path/,
|
||||
);
|
||||
assert.throws(
|
||||
() => loadConfig(validEnvironment({ UPLOAD_TEMP_DIRECTORY: "relative" })),
|
||||
/UPLOAD_TEMP_DIRECTORY must be an absolute path/,
|
||||
);
|
||||
assert.throws(
|
||||
() => loadConfig(validEnvironment({ UPLOAD_MAX_FILE_BYTES: "0" })),
|
||||
/UPLOAD_MAX_FILE_BYTES must be at least 1/,
|
||||
);
|
||||
assert.throws(
|
||||
() => loadConfig(validEnvironment({ UPLOAD_MAX_FILES: "101" })),
|
||||
/UPLOAD_MAX_FILES must be at most 100/,
|
||||
);
|
||||
assert.throws(
|
||||
() => loadConfig(validEnvironment({ IMAGE_MAX_INPUT_PIXELS: "no-limit" })),
|
||||
/IMAGE_MAX_INPUT_PIXELS must be an integer/,
|
||||
);
|
||||
});
|
||||
|
||||
test("redactedConfig never returns credentials, passwords, paths, or webhook URLs", () => {
|
||||
@@ -83,5 +123,6 @@ test("redactedConfig never returns credentials, passwords, paths, or webhook URL
|
||||
assert.doesNotMatch(serialized, /mqtt-password/);
|
||||
assert.doesNotMatch(serialized, /webhook-secret/);
|
||||
assert.doesNotMatch(serialized, /announcements\.sqlite/);
|
||||
assert.doesNotMatch(serialized, /seaweedfs/);
|
||||
assert.doesNotMatch(serialized, /home-assistant/);
|
||||
});
|
||||
|
||||
@@ -0,0 +1,304 @@
|
||||
const test = require("node:test");
|
||||
const assert = require("node:assert/strict");
|
||||
const crypto = require("node:crypto");
|
||||
const fs = require("node:fs");
|
||||
const os = require("node:os");
|
||||
const path = require("node:path");
|
||||
const sharp = require("sharp");
|
||||
const {
|
||||
DEFAULT_IMAGE_DERIVATIVE_OPTIONS,
|
||||
ImageProcessingError,
|
||||
createImageProcessor,
|
||||
} = require("../src/image-processing");
|
||||
|
||||
const MAX_INPUT_PIXELS = 12_000_000;
|
||||
const CONTENT_TYPES = Object.freeze({
|
||||
jpeg: "image/jpeg",
|
||||
png: "image/png",
|
||||
webp: "image/webp",
|
||||
});
|
||||
|
||||
function createProcessor(overrides = {}) {
|
||||
return createImageProcessor({
|
||||
maxInputPixels: MAX_INPUT_PIXELS,
|
||||
...overrides,
|
||||
});
|
||||
}
|
||||
|
||||
function hash(buffer) {
|
||||
return crypto.createHash("sha256").update(buffer).digest("hex");
|
||||
}
|
||||
|
||||
async function solidImage(format, width, height, background = "#4a76b8") {
|
||||
return sharp({
|
||||
create: {
|
||||
width,
|
||||
height,
|
||||
channels: 4,
|
||||
background,
|
||||
},
|
||||
})
|
||||
.toFormat(format)
|
||||
.toBuffer();
|
||||
}
|
||||
|
||||
async function twoFrameImage(format) {
|
||||
const width = 20;
|
||||
const frameHeight = 10;
|
||||
const channels = 4;
|
||||
const firstFrame = Buffer.alloc(width * frameHeight * channels);
|
||||
const secondFrame = Buffer.alloc(width * frameHeight * channels);
|
||||
|
||||
for (let offset = 0; offset < firstFrame.length; offset += channels) {
|
||||
firstFrame.set([255, 0, 0, 255], offset);
|
||||
secondFrame.set([0, 0, 255, 255], offset);
|
||||
}
|
||||
|
||||
const image = sharp(Buffer.concat([firstFrame, secondFrame]), {
|
||||
raw: {
|
||||
width,
|
||||
height: frameHeight * 2,
|
||||
channels,
|
||||
pageHeight: frameHeight,
|
||||
},
|
||||
});
|
||||
|
||||
if (format === "tiff") {
|
||||
return image.tiff().toBuffer();
|
||||
}
|
||||
return image[format]({ delay: [100, 100], loop: 0 }).toBuffer();
|
||||
}
|
||||
|
||||
test("accepts decoded still JPEG, PNG, and WebP content", async (t) => {
|
||||
const processor = createProcessor();
|
||||
|
||||
for (const format of ["jpeg", "png", "webp"]) {
|
||||
await t.test(format, async () => {
|
||||
const source = await solidImage(format, 640, 360);
|
||||
const result = await processor.process(source);
|
||||
|
||||
assert.equal(result.original.format, format);
|
||||
assert.equal(result.original.contentType, CONTENT_TYPES[format]);
|
||||
assert.equal(result.original.byteSize, source.length);
|
||||
assert.equal(result.original.sha256, hash(source));
|
||||
assert.equal(result.original.width, 640);
|
||||
assert.equal(result.original.height, 360);
|
||||
assert.equal(result.original.pages, 1);
|
||||
|
||||
for (const derivative of [result.display, result.thumbnail]) {
|
||||
assert.ok(Buffer.isBuffer(derivative.buffer));
|
||||
assert.equal(derivative.format, "webp");
|
||||
assert.equal(derivative.contentType, "image/webp");
|
||||
assert.equal(derivative.byteSize, derivative.buffer.length);
|
||||
assert.equal(derivative.sha256, hash(derivative.buffer));
|
||||
assert.match(derivative.sha256, /^[0-9a-f]{64}$/);
|
||||
}
|
||||
});
|
||||
}
|
||||
});
|
||||
|
||||
test("uses bounding boxes without cropping or changing source aspect ratio", async () => {
|
||||
const source = await solidImage("png", 3000, 1000);
|
||||
const result = await createProcessor().process(source);
|
||||
|
||||
assert.deepEqual(
|
||||
{ width: result.display.width, height: result.display.height },
|
||||
{ width: 1920, height: 640 },
|
||||
);
|
||||
assert.deepEqual(
|
||||
{ width: result.thumbnail.width, height: result.thumbnail.height },
|
||||
{ width: 480, height: 160 },
|
||||
);
|
||||
assert.equal(result.display.width / result.display.height, 3);
|
||||
assert.equal(result.thumbnail.width / result.thumbnail.height, 3);
|
||||
});
|
||||
|
||||
test("does not enlarge a source that fits within both derivative bounds", async () => {
|
||||
const source = await solidImage("jpeg", 320, 180);
|
||||
const result = await createProcessor().process(source);
|
||||
|
||||
assert.deepEqual(
|
||||
{ width: result.display.width, height: result.display.height },
|
||||
{ width: 320, height: 180 },
|
||||
);
|
||||
assert.deepEqual(
|
||||
{ width: result.thumbnail.width, height: result.thumbnail.height },
|
||||
{ width: 320, height: 180 },
|
||||
);
|
||||
});
|
||||
|
||||
test("applies EXIF orientation and strips it from derivative metadata", async () => {
|
||||
const source = await sharp({
|
||||
create: {
|
||||
width: 120,
|
||||
height: 80,
|
||||
channels: 3,
|
||||
background: "#cb7235",
|
||||
},
|
||||
})
|
||||
.jpeg()
|
||||
.withMetadata({ orientation: 6 })
|
||||
.toBuffer();
|
||||
|
||||
const result = await createProcessor().process(source);
|
||||
const outputMetadata = await sharp(result.display.buffer).metadata();
|
||||
|
||||
assert.equal(result.original.orientation, 6);
|
||||
assert.equal(result.original.encodedWidth, 120);
|
||||
assert.equal(result.original.encodedHeight, 80);
|
||||
assert.equal(result.original.width, 80);
|
||||
assert.equal(result.original.height, 120);
|
||||
assert.equal(result.display.width, 80);
|
||||
assert.equal(result.display.height, 120);
|
||||
assert.equal(outputMetadata.orientation, undefined);
|
||||
assert.equal(outputMetadata.exif, undefined);
|
||||
});
|
||||
|
||||
test("flattens transparent pixels against black in both derivatives", async () => {
|
||||
const source = await solidImage("png", 40, 30, {
|
||||
r: 255,
|
||||
g: 0,
|
||||
b: 0,
|
||||
alpha: 0,
|
||||
});
|
||||
const result = await createProcessor().process(source);
|
||||
|
||||
assert.equal(result.original.hasAlpha, true);
|
||||
for (const derivative of [result.display, result.thumbnail]) {
|
||||
const metadata = await sharp(derivative.buffer).metadata();
|
||||
const pixels = await sharp(derivative.buffer).raw().toBuffer();
|
||||
|
||||
assert.equal(metadata.hasAlpha, false);
|
||||
assert.ok(
|
||||
pixels.every((channel) => channel <= 2),
|
||||
"fully transparent source pixels should become black",
|
||||
);
|
||||
}
|
||||
});
|
||||
|
||||
test("supports a temporary file path without changing source identity", async (t) => {
|
||||
const temporaryDirectory = fs.mkdtempSync(
|
||||
path.join(os.tmpdir(), "announcements-image-processing-"),
|
||||
);
|
||||
t.after(() => fs.rmSync(temporaryDirectory, { recursive: true, force: true }));
|
||||
|
||||
const source = await solidImage("webp", 800, 600);
|
||||
const filename = path.join(temporaryDirectory, "upload.tmp");
|
||||
fs.writeFileSync(filename, source);
|
||||
|
||||
const result = await createProcessor().process(filename);
|
||||
assert.equal(result.original.sha256, hash(source));
|
||||
assert.equal(result.original.byteSize, source.length);
|
||||
assert.equal(result.original.format, "webp");
|
||||
});
|
||||
|
||||
test("rejects malformed and unsupported decoded content with typed errors", async () => {
|
||||
const processor = createProcessor();
|
||||
const gif = await solidImage("gif", 64, 64);
|
||||
|
||||
await assert.rejects(
|
||||
processor.process(Buffer.from("not an image")),
|
||||
(error) =>
|
||||
error instanceof ImageProcessingError &&
|
||||
error.code === "MALFORMED_IMAGE" &&
|
||||
error.statusCode === 422,
|
||||
);
|
||||
await assert.rejects(
|
||||
processor.process(gif),
|
||||
(error) =>
|
||||
error instanceof ImageProcessingError &&
|
||||
error.code === "UNSUPPORTED_IMAGE_FORMAT" &&
|
||||
error.statusCode === 415 &&
|
||||
error.details.detectedFormat === "gif",
|
||||
);
|
||||
});
|
||||
|
||||
test("rejects animated WebP and GIF instead of selecting the first frame", async (t) => {
|
||||
for (const format of ["webp", "gif"]) {
|
||||
await t.test(format, async () => {
|
||||
const animated = await twoFrameImage(format);
|
||||
const metadata = await sharp(animated, { animated: true }).metadata();
|
||||
assert.equal(metadata.pages, 2, "test fixture must contain two frames");
|
||||
await assert.rejects(
|
||||
createProcessor().process(animated),
|
||||
(error) =>
|
||||
error instanceof ImageProcessingError &&
|
||||
error.code === "ANIMATED_IMAGE_NOT_SUPPORTED" &&
|
||||
error.details.pages === 2,
|
||||
);
|
||||
});
|
||||
}
|
||||
});
|
||||
|
||||
test("rejects unexpected multipage inputs with a distinct typed error", async () => {
|
||||
const multipageTiff = await twoFrameImage("tiff");
|
||||
const metadata = await sharp(multipageTiff, { animated: true }).metadata();
|
||||
assert.equal(metadata.pages, 2, "test fixture must contain two pages");
|
||||
|
||||
await assert.rejects(
|
||||
createProcessor().process(multipageTiff),
|
||||
(error) =>
|
||||
error instanceof ImageProcessingError &&
|
||||
error.code === "MULTIPAGE_IMAGE_NOT_SUPPORTED" &&
|
||||
error.details.pages === 2,
|
||||
);
|
||||
});
|
||||
|
||||
test("enforces the injected decoded-pixel limit", async () => {
|
||||
const source = await solidImage("png", 100, 100);
|
||||
const processor = createImageProcessor({ maxInputPixels: 9_999 });
|
||||
|
||||
await assert.rejects(
|
||||
processor.process(source),
|
||||
(error) =>
|
||||
error instanceof ImageProcessingError &&
|
||||
error.code === "IMAGE_PIXEL_LIMIT_EXCEEDED" &&
|
||||
error.statusCode === 413 &&
|
||||
error.details.maxInputPixels === 9_999,
|
||||
);
|
||||
});
|
||||
|
||||
test("uses documented defaults while allowing validated target overrides", async () => {
|
||||
assert.deepEqual(DEFAULT_IMAGE_DERIVATIVE_OPTIONS, {
|
||||
display: { maxWidth: 1920, maxHeight: 1080, quality: 82 },
|
||||
thumbnail: { maxWidth: 480, maxHeight: 270, quality: 72 },
|
||||
});
|
||||
|
||||
const source = await solidImage("png", 900, 600);
|
||||
const processor = createProcessor({
|
||||
display: { maxWidth: 300, maxHeight: 300, quality: 90 },
|
||||
thumbnail: { maxWidth: 100, maxHeight: 50, quality: 65 },
|
||||
});
|
||||
const result = await processor.process(source);
|
||||
|
||||
assert.deepEqual(
|
||||
{ width: result.display.width, height: result.display.height },
|
||||
{ width: 300, height: 200 },
|
||||
);
|
||||
assert.deepEqual(
|
||||
{ width: result.thumbnail.width, height: result.thumbnail.height },
|
||||
{ width: 75, height: 50 },
|
||||
);
|
||||
assert.throws(
|
||||
() => createImageProcessor({ maxInputPixels: 0 }),
|
||||
/maxInputPixels must be at least 1/,
|
||||
);
|
||||
assert.throws(
|
||||
() =>
|
||||
createImageProcessor({
|
||||
maxInputPixels: 10_000,
|
||||
display: { quality: 101 },
|
||||
}),
|
||||
/display\.quality must be at most 100/,
|
||||
);
|
||||
});
|
||||
|
||||
test("rejects invalid input interfaces with structured errors", async () => {
|
||||
await assert.rejects(
|
||||
createProcessor().process({ path: "/tmp/not-an-interface" }),
|
||||
(error) =>
|
||||
error instanceof ImageProcessingError &&
|
||||
error.code === "INVALID_IMAGE_INPUT" &&
|
||||
error.statusCode === 400,
|
||||
);
|
||||
});
|
||||
@@ -29,13 +29,33 @@ const silentLogger = {
|
||||
error() {},
|
||||
};
|
||||
|
||||
function testStorage() {
|
||||
return {
|
||||
async start() {},
|
||||
stop() {},
|
||||
health() {
|
||||
return {
|
||||
status: "degraded",
|
||||
checkedAt: "2026-08-16T00:00:00.000Z",
|
||||
lastSuccessAt: null,
|
||||
code: "STORAGE_UNAVAILABLE",
|
||||
message: "Object storage is unavailable",
|
||||
};
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
async function responseJson(baseUrl, path, init) {
|
||||
const response = await fetch(`${baseUrl}${path}`, init);
|
||||
return { response, body: await response.json() };
|
||||
}
|
||||
|
||||
test("HTTP foundation exposes live, ready, detailed health, and persisted settings", async (t) => {
|
||||
const runtime = createRuntime({ config: testConfig(), logger: silentLogger });
|
||||
const runtime = createRuntime({
|
||||
config: testConfig(),
|
||||
logger: silentLogger,
|
||||
storage: testStorage(),
|
||||
});
|
||||
await runtime.start();
|
||||
t.after(() => runtime.stop("test cleanup"));
|
||||
const baseUrl = `http://127.0.0.1:${runtime.address.port}`;
|
||||
@@ -52,7 +72,7 @@ test("HTTP foundation exposes live, ready, detailed health, and persisted settin
|
||||
assert.equal(status.response.status, 200);
|
||||
assert.equal(status.body.status, "degraded");
|
||||
assert.equal(status.body.services.database.currentSchemaVersion, 1);
|
||||
assert.equal(status.body.services.storage.status, "not_checked");
|
||||
assert.equal(status.body.services.storage.status, "degraded");
|
||||
const statusText = JSON.stringify(status.body);
|
||||
assert.doesNotMatch(statusText, /access-key-that-must-not-leak/);
|
||||
assert.doesNotMatch(statusText, /secret-key-that-must-not-leak/);
|
||||
@@ -83,7 +103,11 @@ test("HTTP foundation exposes live, ready, detailed health, and persisted settin
|
||||
});
|
||||
|
||||
test("invalid settings receive a structured client error", async (t) => {
|
||||
const runtime = createRuntime({ config: testConfig(), logger: silentLogger });
|
||||
const runtime = createRuntime({
|
||||
config: testConfig(),
|
||||
logger: silentLogger,
|
||||
storage: testStorage(),
|
||||
});
|
||||
await runtime.start();
|
||||
t.after(() => runtime.stop("test cleanup"));
|
||||
const baseUrl = `http://127.0.0.1:${runtime.address.port}`;
|
||||
@@ -99,7 +123,11 @@ test("invalid settings receive a structured client error", async (t) => {
|
||||
});
|
||||
|
||||
test("liveness stays up while critical readiness failures return 503", async (t) => {
|
||||
const runtime = createRuntime({ config: testConfig(), logger: silentLogger });
|
||||
const runtime = createRuntime({
|
||||
config: testConfig(),
|
||||
logger: silentLogger,
|
||||
storage: testStorage(),
|
||||
});
|
||||
await runtime.start();
|
||||
t.after(() => runtime.stop("test cleanup"));
|
||||
const baseUrl = `http://127.0.0.1:${runtime.address.port}`;
|
||||
@@ -127,7 +155,11 @@ test("liveness stays up while critical readiness failures return 503", async (t)
|
||||
});
|
||||
|
||||
test("runtime startup and shutdown are idempotent", async () => {
|
||||
const runtime = createRuntime({ config: testConfig(), logger: silentLogger });
|
||||
const runtime = createRuntime({
|
||||
config: testConfig(),
|
||||
logger: silentLogger,
|
||||
storage: testStorage(),
|
||||
});
|
||||
const [firstAddress, secondAddress] = await Promise.all([
|
||||
runtime.start(),
|
||||
runtime.start(),
|
||||
|
||||
@@ -0,0 +1,105 @@
|
||||
const test = require("node:test");
|
||||
const assert = require("node:assert/strict");
|
||||
const express = require("express");
|
||||
const { openDatabase } = require("../src/database");
|
||||
const { runMigrations } = require("../src/database/migrate");
|
||||
const {
|
||||
createDetailedHealthRouter,
|
||||
createHealthRouter,
|
||||
} = require("../src/health/routes");
|
||||
|
||||
function config() {
|
||||
return {
|
||||
environment: "test",
|
||||
host: "127.0.0.1",
|
||||
port: 0,
|
||||
timeZone: "America/Chicago",
|
||||
trustProxy: true,
|
||||
database: { path: ":memory:" },
|
||||
storage: {
|
||||
endpoint: "http://private-storage:8333",
|
||||
region: "us-east-1",
|
||||
bucket: "announcements-v2",
|
||||
accessKeyId: "must-not-leak",
|
||||
secretAccessKey: "must-not-leak-either",
|
||||
forcePathStyle: true,
|
||||
},
|
||||
processing: {
|
||||
tempDirectory: "/tmp/announcements-uploads",
|
||||
maxFileBytes: 1024,
|
||||
maxFiles: 2,
|
||||
maxInputPixels: 1000,
|
||||
},
|
||||
mqtt: { url: null },
|
||||
notifications: { homeAssistantWebhookUrl: null },
|
||||
};
|
||||
}
|
||||
|
||||
async function withServer(storage, callback) {
|
||||
const database = openDatabase({ path: ":memory:", busyTimeoutMs: 1000 });
|
||||
const migrations = runMigrations(database);
|
||||
const lifecycle = { ready: true };
|
||||
const app = express();
|
||||
const options = { config: config(), database, lifecycle, migrations, storage };
|
||||
app.use("/health", createHealthRouter(options));
|
||||
app.use("/api/health", createDetailedHealthRouter(options));
|
||||
const server = app.listen(0, "127.0.0.1");
|
||||
await new Promise((resolve) => server.once("listening", resolve));
|
||||
|
||||
try {
|
||||
await callback(`http://127.0.0.1:${server.address().port}`);
|
||||
} finally {
|
||||
await new Promise((resolve) => server.close(resolve));
|
||||
database.close();
|
||||
}
|
||||
}
|
||||
|
||||
test("storage degradation is reported but does not fail API readiness", async () => {
|
||||
const storage = {
|
||||
health() {
|
||||
return {
|
||||
status: "degraded",
|
||||
checkedAt: "2026-08-16T12:00:00.000Z",
|
||||
lastSuccessAt: "2026-08-16T11:59:00.000Z",
|
||||
code: "STORAGE_UNAVAILABLE",
|
||||
};
|
||||
},
|
||||
};
|
||||
|
||||
await withServer(storage, async (baseUrl) => {
|
||||
const ready = await fetch(`${baseUrl}/health/ready`);
|
||||
const readyBody = await ready.json();
|
||||
assert.equal(ready.status, 200);
|
||||
assert.equal(readyBody.status, "ready");
|
||||
assert.equal(readyBody.checks.storage, "degraded");
|
||||
|
||||
const detailed = await fetch(`${baseUrl}/api/health`);
|
||||
const detailedBody = await detailed.json();
|
||||
assert.equal(detailed.status, 200);
|
||||
assert.equal(detailedBody.status, "degraded");
|
||||
assert.equal(detailedBody.services.storage.status, "degraded");
|
||||
assert.equal(
|
||||
detailedBody.services.storage.lastSuccessAt,
|
||||
"2026-08-16T11:59:00.000Z",
|
||||
);
|
||||
assert.equal(detailedBody.services.storage.code, "STORAGE_UNAVAILABLE");
|
||||
assert.doesNotMatch(JSON.stringify(detailedBody), /must-not-leak/);
|
||||
assert.doesNotMatch(JSON.stringify(detailedBody), /private-storage/);
|
||||
});
|
||||
});
|
||||
|
||||
test("healthy storage produces healthy detailed status", async () => {
|
||||
const storage = {
|
||||
health() {
|
||||
return { status: "healthy", checkedAt: "2026-08-16T12:00:00.000Z" };
|
||||
},
|
||||
};
|
||||
|
||||
await withServer(storage, async (baseUrl) => {
|
||||
const detailed = await fetch(`${baseUrl}/api/health`);
|
||||
const body = await detailed.json();
|
||||
assert.equal(detailed.status, 200);
|
||||
assert.equal(body.status, "ok");
|
||||
assert.equal(body.services.storage.status, "healthy");
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,397 @@
|
||||
const test = require("node:test");
|
||||
const assert = require("node:assert/strict");
|
||||
const http = require("node:http");
|
||||
const { PassThrough, Readable } = require("node:stream");
|
||||
const {
|
||||
StorageObjectNotFoundError,
|
||||
StorageUnavailableError,
|
||||
createStorageService,
|
||||
} = require("../src/storage");
|
||||
|
||||
const storageConfig = {
|
||||
endpoint: "http://seaweedfs:8333",
|
||||
region: "us-east-1",
|
||||
bucket: "announcements-v2",
|
||||
accessKeyId: "test-key",
|
||||
secretAccessKey: "test-secret",
|
||||
forcePathStyle: true,
|
||||
};
|
||||
|
||||
function fakeClient(handler) {
|
||||
const commands = [];
|
||||
return {
|
||||
commands,
|
||||
async send(command) {
|
||||
commands.push(command);
|
||||
return handler(command, commands.length - 1);
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
function commandName(command) {
|
||||
return command.constructor.name;
|
||||
}
|
||||
|
||||
test("storage initialization probes an existing bucket without creating it", async () => {
|
||||
const client = fakeClient(async () => ({}));
|
||||
const service = createStorageService(storageConfig, { client });
|
||||
|
||||
const health = await service.initialize();
|
||||
assert.equal(health.status, "healthy");
|
||||
assert.match(health.checkedAt, /T/);
|
||||
assert.equal(health.lastSuccessAt, health.checkedAt);
|
||||
assert.deepEqual(client.commands.map(commandName), ["HeadBucketCommand"]);
|
||||
});
|
||||
|
||||
test("storage initialization creates only a missing bucket", async () => {
|
||||
let exists = false;
|
||||
const missing = Object.assign(new Error("missing"), {
|
||||
name: "NoSuchBucket",
|
||||
$metadata: { httpStatusCode: 404 },
|
||||
});
|
||||
const client = fakeClient(async (command) => {
|
||||
if (commandName(command) === "HeadBucketCommand" && !exists) throw missing;
|
||||
if (commandName(command) === "CreateBucketCommand") exists = true;
|
||||
return {};
|
||||
});
|
||||
const service = createStorageService(storageConfig, { client });
|
||||
|
||||
assert.deepEqual(await service.ensureBucket(), { created: true });
|
||||
assert.deepEqual(client.commands.map(commandName), [
|
||||
"HeadBucketCommand",
|
||||
"CreateBucketCommand",
|
||||
]);
|
||||
assert.equal(client.commands[1].input.Bucket, storageConfig.bucket);
|
||||
|
||||
assert.deepEqual(await service.ensureBucket(), { created: false });
|
||||
assert.deepEqual(client.commands.map(commandName), [
|
||||
"HeadBucketCommand",
|
||||
"CreateBucketCommand",
|
||||
"HeadBucketCommand",
|
||||
]);
|
||||
});
|
||||
|
||||
test("storage start performs a nonfatal immediate check and stop releases its client", async () => {
|
||||
let destroyed = false;
|
||||
const client = fakeClient(async () => ({}));
|
||||
client.destroy = () => {
|
||||
destroyed = true;
|
||||
};
|
||||
const service = createStorageService(
|
||||
{ ...storageConfig, healthProbeIntervalMs: 1_000 },
|
||||
{ client },
|
||||
);
|
||||
|
||||
const health = await service.start();
|
||||
assert.equal(health.status, "healthy");
|
||||
assert.deepEqual(client.commands.map(commandName), ["HeadBucketCommand"]);
|
||||
service.stop();
|
||||
assert.equal(destroyed, true);
|
||||
});
|
||||
|
||||
test("a create-bucket race is healthy only after the bucket can be verified", async () => {
|
||||
const missing = Object.assign(new Error("missing"), {
|
||||
name: "NoSuchBucket",
|
||||
$metadata: { httpStatusCode: 404 },
|
||||
});
|
||||
const conflict = Object.assign(new Error("race"), {
|
||||
name: "BucketAlreadyOwnedByYou",
|
||||
$metadata: { httpStatusCode: 409 },
|
||||
});
|
||||
let headCalls = 0;
|
||||
const client = fakeClient(async (command) => {
|
||||
if (commandName(command) === "HeadBucketCommand") {
|
||||
headCalls += 1;
|
||||
if (headCalls === 1) throw missing;
|
||||
return {};
|
||||
}
|
||||
if (commandName(command) === "CreateBucketCommand") throw conflict;
|
||||
throw new Error(`Unexpected ${commandName(command)}`);
|
||||
});
|
||||
const service = createStorageService(storageConfig, { client });
|
||||
|
||||
assert.deepEqual(await service.ensureBucket(), { created: false });
|
||||
assert.equal(service.health().status, "healthy");
|
||||
assert.deepEqual(client.commands.map(commandName), [
|
||||
"HeadBucketCommand",
|
||||
"CreateBucketCommand",
|
||||
"HeadBucketCommand",
|
||||
]);
|
||||
});
|
||||
|
||||
test("an older failed probe cannot overwrite a newer successful operation", async () => {
|
||||
let rejectProbe;
|
||||
let notifyProbeStarted;
|
||||
const probeStarted = new Promise((resolve) => {
|
||||
notifyProbeStarted = resolve;
|
||||
});
|
||||
let headCalls = 0;
|
||||
const client = fakeClient(async (command) => {
|
||||
if (commandName(command) === "HeadBucketCommand") {
|
||||
headCalls += 1;
|
||||
if (headCalls === 1) return {};
|
||||
notifyProbeStarted();
|
||||
return new Promise((resolve, reject) => {
|
||||
rejectProbe = reject;
|
||||
});
|
||||
}
|
||||
if (commandName(command) === "PutObjectCommand") return {};
|
||||
throw new Error(`Unexpected ${commandName(command)}`);
|
||||
});
|
||||
const service = createStorageService(storageConfig, { client });
|
||||
await service.initialize();
|
||||
|
||||
const staleProbe = service.probe();
|
||||
await probeStarted;
|
||||
await service.putObject({
|
||||
key: "assets/id/original/hash",
|
||||
body: Buffer.from("bytes"),
|
||||
contentLength: 5,
|
||||
});
|
||||
const successfulState = service.health();
|
||||
rejectProbe(new Error("older request failed"));
|
||||
await staleProbe;
|
||||
|
||||
assert.equal(successfulState.status, "healthy");
|
||||
assert.deepEqual(service.health(), successfulState);
|
||||
});
|
||||
|
||||
test("temporary storage failure degrades without throwing during initialization and recovers", async () => {
|
||||
let available = false;
|
||||
const warnings = [];
|
||||
const client = fakeClient(async () => {
|
||||
if (!available) throw new Error("connection refused at a private address");
|
||||
return {};
|
||||
});
|
||||
const service = createStorageService(storageConfig, {
|
||||
client,
|
||||
logger: { warn(message) { warnings.push(message); } },
|
||||
});
|
||||
|
||||
const initial = await service.initialize();
|
||||
assert.equal(initial.status, "degraded");
|
||||
assert.equal(initial.code, "STORAGE_UNAVAILABLE");
|
||||
assert.equal(initial.lastSuccessAt, null);
|
||||
assert.deepEqual(warnings, ["Object storage is unavailable during initialization"]);
|
||||
|
||||
available = true;
|
||||
const recovered = await service.probe();
|
||||
assert.equal(recovered.status, "healthy");
|
||||
assert.equal(recovered.lastSuccessAt, recovered.checkedAt);
|
||||
assert.equal(service.health().status, "healthy");
|
||||
|
||||
available = false;
|
||||
const degradedAgain = await service.probe();
|
||||
assert.equal(degradedAgain.status, "degraded");
|
||||
assert.equal(degradedAgain.lastSuccessAt, recovered.lastSuccessAt);
|
||||
});
|
||||
|
||||
test("put, get, head, existence, and delete use the configured private bucket", async () => {
|
||||
const body = Readable.from([Buffer.from("image bytes")]);
|
||||
const client = fakeClient(async (command) => {
|
||||
switch (commandName(command)) {
|
||||
case "PutObjectCommand":
|
||||
return { ETag: '"etag"' };
|
||||
case "GetObjectCommand":
|
||||
return {
|
||||
Body: body,
|
||||
ContentType: "image/webp",
|
||||
ContentLength: 11,
|
||||
ETag: '"etag"',
|
||||
};
|
||||
case "HeadObjectCommand":
|
||||
return {
|
||||
ContentType: "image/webp",
|
||||
ContentLength: 11,
|
||||
ETag: '"etag"',
|
||||
};
|
||||
case "DeleteObjectCommand":
|
||||
return {};
|
||||
case "HeadBucketCommand":
|
||||
return {};
|
||||
default:
|
||||
throw new Error(`Unexpected ${commandName(command)}`);
|
||||
}
|
||||
});
|
||||
const service = createStorageService(storageConfig, { client });
|
||||
const key = "assets/id/display/hash.webp";
|
||||
await service.initialize();
|
||||
client.commands.length = 0;
|
||||
|
||||
const put = await service.putObject({
|
||||
key,
|
||||
body: Buffer.from("image bytes"),
|
||||
contentType: "image/webp",
|
||||
contentLength: 11,
|
||||
});
|
||||
assert.deepEqual(put, { key, etag: '"etag"', versionId: null });
|
||||
|
||||
const fetched = await service.getObject(key);
|
||||
assert.equal(fetched.body, body);
|
||||
assert.equal(fetched.contentType, "image/webp");
|
||||
assert.equal(fetched.contentLength, 11);
|
||||
|
||||
assert.equal((await service.headObject(key)).etag, '"etag"');
|
||||
assert.equal(await service.objectExists(key), true);
|
||||
await service.deleteObject(key);
|
||||
|
||||
for (const command of client.commands) {
|
||||
assert.equal(command.input.Bucket, storageConfig.bucket);
|
||||
assert.equal(command.input.Key, key);
|
||||
}
|
||||
});
|
||||
|
||||
test("missing objects are distinct from unavailable storage", async () => {
|
||||
let missing = true;
|
||||
const client = fakeClient(async (command) => {
|
||||
if (commandName(command) === "HeadBucketCommand") return {};
|
||||
if (missing) {
|
||||
throw Object.assign(new Error("missing"), {
|
||||
name: "NoSuchKey",
|
||||
$metadata: { httpStatusCode: 404 },
|
||||
});
|
||||
}
|
||||
throw new Error("network offline");
|
||||
});
|
||||
const service = createStorageService(storageConfig, { client });
|
||||
|
||||
await service.initialize();
|
||||
client.commands.length = 0;
|
||||
|
||||
assert.equal(await service.headObject("assets/id/original/hash"), null);
|
||||
assert.equal(await service.objectExists("assets/id/original/hash"), false);
|
||||
await assert.rejects(
|
||||
service.getObject("assets/id/original/hash"),
|
||||
StorageObjectNotFoundError,
|
||||
);
|
||||
assert.equal(service.health().status, "healthy");
|
||||
|
||||
missing = false;
|
||||
await assert.rejects(
|
||||
service.putObject({ key: "assets/id/original/hash", body: Buffer.alloc(0) }),
|
||||
(error) =>
|
||||
error instanceof StorageUnavailableError &&
|
||||
error.code === "STORAGE_UNAVAILABLE" &&
|
||||
error.statusCode === 503,
|
||||
);
|
||||
assert.equal(service.health().status, "degraded");
|
||||
});
|
||||
|
||||
test("a missing bucket during object access is a storage outage, not a missing object", async () => {
|
||||
let initialized = false;
|
||||
const client = fakeClient(async (command) => {
|
||||
if (commandName(command) === "HeadBucketCommand" && !initialized) {
|
||||
initialized = true;
|
||||
return {};
|
||||
}
|
||||
throw Object.assign(new Error("bucket missing"), {
|
||||
name: "NoSuchBucket",
|
||||
$metadata: { httpStatusCode: 404 },
|
||||
});
|
||||
});
|
||||
const service = createStorageService(storageConfig, { client });
|
||||
await service.initialize();
|
||||
|
||||
await assert.rejects(
|
||||
service.getObject("assets/id/original/hash"),
|
||||
StorageUnavailableError,
|
||||
);
|
||||
assert.equal(service.health().status, "degraded");
|
||||
});
|
||||
|
||||
test("storage rejects unsafe internal object keys before contacting S3", async () => {
|
||||
const client = fakeClient(async () => ({}));
|
||||
const service = createStorageService(storageConfig, { client });
|
||||
|
||||
await assert.rejects(
|
||||
service.getObject("../private"),
|
||||
/Storage object key is invalid/,
|
||||
);
|
||||
await assert.rejects(
|
||||
service.deleteObject("/absolute"),
|
||||
/Storage object key is invalid/,
|
||||
);
|
||||
assert.equal(client.commands.length, 0);
|
||||
});
|
||||
|
||||
test("real HTTP requests time out instead of hanging indefinitely", async (t) => {
|
||||
const server = http.createServer(() => {
|
||||
// Intentionally accept the connection without sending response headers.
|
||||
});
|
||||
server.listen(0, "127.0.0.1");
|
||||
await new Promise((resolve) => server.once("listening", resolve));
|
||||
t.after(async () => {
|
||||
server.closeAllConnections();
|
||||
await new Promise((resolve) => server.close(resolve));
|
||||
});
|
||||
|
||||
const service = createStorageService({
|
||||
...storageConfig,
|
||||
endpoint: `http://127.0.0.1:${server.address().port}`,
|
||||
connectionTimeoutMs: 100,
|
||||
requestTimeoutMs: 100,
|
||||
maxAttempts: 1,
|
||||
});
|
||||
t.after(() => service.stop());
|
||||
|
||||
const startedAt = Date.now();
|
||||
const health = await service.probe();
|
||||
assert.equal(health.status, "degraded");
|
||||
assert.equal(health.code, "STORAGE_UNAVAILABLE");
|
||||
assert.ok(Date.now() - startedAt < 2_000);
|
||||
});
|
||||
|
||||
test("consumer stream cancellation does not falsely degrade storage", async () => {
|
||||
let body = new PassThrough();
|
||||
const client = fakeClient(async (command) => {
|
||||
if (commandName(command) === "HeadBucketCommand") return {};
|
||||
if (commandName(command) === "GetObjectCommand") {
|
||||
return { Body: body, ContentLength: 10, ContentType: "image/webp" };
|
||||
}
|
||||
throw new Error(`Unexpected ${commandName(command)}`);
|
||||
});
|
||||
const service = createStorageService(storageConfig, { client });
|
||||
await service.initialize();
|
||||
|
||||
const cancelled = await service.getObject("assets/id/display/hash.webp");
|
||||
cancelled.body.destroy(
|
||||
Object.assign(new Error("consumer closed"), {
|
||||
code: "ERR_STREAM_PREMATURE_CLOSE",
|
||||
}),
|
||||
);
|
||||
await new Promise((resolve) => setImmediate(resolve));
|
||||
assert.equal(service.health().status, "healthy");
|
||||
|
||||
body = new PassThrough();
|
||||
const failed = await service.getObject("assets/id/display/hash.webp");
|
||||
failed.body.destroy(Object.assign(new Error("upstream reset"), { code: "ECONNRESET" }));
|
||||
await new Promise((resolve) => setImmediate(resolve));
|
||||
assert.equal(service.health().status, "degraded");
|
||||
});
|
||||
|
||||
test("delete treats an already-missing object as an idempotent success", async () => {
|
||||
let missingBucket = false;
|
||||
const client = fakeClient(async (command) => {
|
||||
if (commandName(command) === "HeadBucketCommand") return {};
|
||||
if (commandName(command) === "DeleteObjectCommand") {
|
||||
throw Object.assign(new Error("missing"), {
|
||||
name: missingBucket ? "NoSuchBucket" : "NoSuchKey",
|
||||
$metadata: { httpStatusCode: 404 },
|
||||
});
|
||||
}
|
||||
throw new Error(`Unexpected ${commandName(command)}`);
|
||||
});
|
||||
const service = createStorageService(storageConfig, { client });
|
||||
await service.initialize();
|
||||
|
||||
await assert.doesNotReject(
|
||||
service.deleteObject("assets/id/original/hash"),
|
||||
);
|
||||
assert.equal(service.health().status, "healthy");
|
||||
|
||||
missingBucket = true;
|
||||
await assert.rejects(
|
||||
service.deleteObject("assets/id/original/hash"),
|
||||
StorageUnavailableError,
|
||||
);
|
||||
});
|
||||
Reference in New Issue
Block a user