import type { Dirent } from "node:fs"; import { readdir, readFile, realpath, stat } from "node:fs/promises"; import path from "node:path"; import { isVersionCompatible, type ModuleManifest, safeParseModuleManifest, semanticVersionSchema, } from "@ekklesia/contracts"; export const MODULE_MANIFEST_FILENAME = "ekklesia.module.json"; export type ModuleDiscoveryIssueCode = | "DISCOVERY_CONFIGURATION_INVALID" | "MODULE_CAPABILITY_MISSING" | "MODULE_DEPENDENCY_CYCLE" | "MODULE_DEPENDENCY_INCOMPATIBLE" | "MODULE_DEPENDENCY_MISSING" | "MODULE_DIRECTORY_INVALID" | "MODULE_ENTRYPOINT_INVALID" | "MODULE_ENTRYPOINT_MISSING" | "MODULE_ID_DUPLICATE" | "MODULE_MANIFEST_INVALID" | "MODULE_MANIFEST_MISSING" | "MODULE_PACKAGE_INVALID" | "MODULE_PACKAGE_MISSING" | "MODULE_PACKAGE_NAME_DUPLICATE" | "MODULE_PLATFORM_INCOMPATIBLE" | "MODULE_SDK_INCOMPATIBLE" | "MODULE_VERSION_MISMATCH" | "MODULES_DIRECTORY_UNREADABLE"; export interface ModuleDiscoveryIssue { readonly code: ModuleDiscoveryIssueCode; readonly message: string; readonly moduleDirectory: string; readonly moduleId?: string; readonly path?: string; } export interface ResolvedModuleEntrypoints { readonly server: string; readonly schema?: string; readonly migrations?: string; readonly jobs?: string; } type DeepReadonly = T extends (...arguments_: never[]) => unknown ? T : T extends readonly (infer Item)[] ? readonly DeepReadonly[] : T extends object ? { readonly [Key in keyof T]: DeepReadonly } : T; export type ReadonlyModuleManifest = DeepReadonly; export interface DiscoveredModule { readonly directory: string; readonly entrypoints: ResolvedModuleEntrypoints; readonly manifest: ReadonlyModuleManifest; readonly manifestPath: string; readonly packageName: string; } export interface ModuleDiscoveryOptions { readonly moduleSdkVersion: string; readonly modulesDirectory: string; readonly platformVersion: string; } interface ModulePackageData { name: string; version: string; } interface ModuleCandidate { directory: string; directoryName: string; entrypoints: ResolvedModuleEntrypoints; manifest: ModuleManifest; manifestPath: string; packageName: string; } export class ModuleDiscoveryError extends Error { readonly issues: readonly ModuleDiscoveryIssue[]; constructor(issues: readonly ModuleDiscoveryIssue[]) { const sortedIssues = [...issues].sort(compareIssues); super(`Module discovery failed with ${sortedIssues.length} issue(s).`); this.name = "ModuleDiscoveryError"; this.issues = freezeRecursively(sortedIssues); } } function compareText(left: string, right: string): number { if (left < right) { return -1; } if (left > right) { return 1; } return 0; } function compareIssues(left: ModuleDiscoveryIssue, right: ModuleDiscoveryIssue): number { return ( compareText(left.moduleDirectory, right.moduleDirectory) || compareText(left.code, right.code) || compareText(left.path ?? "", right.path ?? "") || compareText(left.message, right.message) ); } function isRecord(value: unknown): value is Record { return typeof value === "object" && value !== null && !Array.isArray(value); } function freezeRecursively(value: T): T { if (typeof value !== "object" || value === null || Object.isFrozen(value)) { return value; } for (const nestedValue of Object.values(value)) { freezeRecursively(nestedValue); } Object.freeze(value); return value; } function errorCode(error: unknown): string | undefined { if (isRecord(error) && typeof error.code === "string") { return error.code; } return undefined; } function issuePath(pathParts: readonly PropertyKey[]): string { return pathParts.map(String).join("."); } function pathIsInside(parent: string, candidate: string): boolean { const relativePath = path.relative(parent, candidate); return ( relativePath === "" || (relativePath !== ".." && !relativePath.startsWith(`..${path.sep}`) && !path.isAbsolute(relativePath)) ); } async function readJson(pathname: string): Promise { return JSON.parse(await readFile(pathname, "utf8")) as unknown; } function validatePackageData( input: unknown, directoryName: string, issues: ModuleDiscoveryIssue[], ): ModulePackageData | undefined { if (!isRecord(input)) { issues.push({ code: "MODULE_PACKAGE_INVALID", message: "package.json must contain a JSON object.", moduleDirectory: directoryName, }); return undefined; } const name = input.name; const version = input.version; const packageType = input.type; const layer = isRecord(input.ekklesia) ? input.ekklesia.layer : undefined; let validPackage = true; if (typeof name !== "string" || name.length === 0) { validPackage = false; issues.push({ code: "MODULE_PACKAGE_INVALID", message: "package.json must declare a non-empty package name.", moduleDirectory: directoryName, path: "name", }); } if (semanticVersionSchema.safeParse(version).success === false) { validPackage = false; issues.push({ code: "MODULE_PACKAGE_INVALID", message: "package.json must declare a strict semantic version.", moduleDirectory: directoryName, path: "version", }); } if (packageType !== "module") { validPackage = false; issues.push({ code: "MODULE_PACKAGE_INVALID", message: 'package.json must declare "type": "module".', moduleDirectory: directoryName, path: "type", }); } if (layer !== "module") { validPackage = false; issues.push({ code: "MODULE_PACKAGE_INVALID", message: 'package.json must declare "ekklesia.layer": "module".', moduleDirectory: directoryName, path: "ekklesia.layer", }); } if (!validPackage || typeof name !== "string" || typeof version !== "string") { return undefined; } return { name, version }; } async function resolveEntrypoints( moduleDirectory: string, directoryName: string, manifest: ModuleManifest, issues: ModuleDiscoveryIssue[], ): Promise { const resolved: Partial> = {}; const canonicalModuleDirectory = await realpath(moduleDirectory); for (const [entrypointName, relativeEntrypoint] of Object.entries(manifest.entrypoints)) { if (relativeEntrypoint === undefined) { continue; } const configuredPath = path.resolve(moduleDirectory, relativeEntrypoint); if (!pathIsInside(moduleDirectory, configuredPath)) { issues.push({ code: "MODULE_ENTRYPOINT_INVALID", message: `${entrypointName} resolves outside its module directory.`, moduleDirectory: directoryName, moduleId: manifest.id, path: `entrypoints.${entrypointName}`, }); continue; } try { const canonicalEntrypoint = await realpath(configuredPath); if (!pathIsInside(canonicalModuleDirectory, canonicalEntrypoint)) { issues.push({ code: "MODULE_ENTRYPOINT_INVALID", message: `${entrypointName} resolves through a link outside its module directory.`, moduleDirectory: directoryName, moduleId: manifest.id, path: `entrypoints.${entrypointName}`, }); continue; } const entrypointStats = await stat(canonicalEntrypoint); const hasExpectedKind = entrypointName === "migrations" ? entrypointStats.isDirectory() : entrypointStats.isFile(); if (!hasExpectedKind) { issues.push({ code: "MODULE_ENTRYPOINT_INVALID", message: entrypointName === "migrations" ? "migrations must resolve to a directory." : `${entrypointName} must resolve to a file.`, moduleDirectory: directoryName, moduleId: manifest.id, path: `entrypoints.${entrypointName}`, }); continue; } resolved[entrypointName as keyof ModuleManifest["entrypoints"]] = canonicalEntrypoint; } catch (error) { const code = errorCode(error); issues.push({ code: code === "ENOENT" ? "MODULE_ENTRYPOINT_MISSING" : "MODULE_ENTRYPOINT_INVALID", message: code === "ENOENT" ? `${entrypointName} does not exist at ${relativeEntrypoint}.` : `${entrypointName} could not be resolved at ${relativeEntrypoint}${ code ? ` (${code})` : "" }.`, moduleDirectory: directoryName, moduleId: manifest.id, path: `entrypoints.${entrypointName}`, }); } } if (!resolved.server) { return undefined; } return { server: resolved.server, ...(resolved.schema ? { schema: resolved.schema } : {}), ...(resolved.migrations ? { migrations: resolved.migrations } : {}), ...(resolved.jobs ? { jobs: resolved.jobs } : {}), }; } async function scanModuleDirectory( modulesDirectory: string, directoryName: string, issues: ModuleDiscoveryIssue[], ): Promise { const moduleDirectory = path.join(modulesDirectory, directoryName); const packagePath = path.join(moduleDirectory, "package.json"); const manifestPath = path.join(moduleDirectory, MODULE_MANIFEST_FILENAME); const canonicalModuleDirectory = await realpath(moduleDirectory); let canonicalManifestPath: string | undefined; let packageInput: unknown; let manifestInput: unknown; let packageLoaded = false; let manifestLoaded = false; try { const canonicalPackagePath = await realpath(packagePath); if (!pathIsInside(canonicalModuleDirectory, canonicalPackagePath)) { issues.push({ code: "MODULE_PACKAGE_INVALID", message: "package.json must not resolve outside its module directory.", moduleDirectory: directoryName, }); } else { packageInput = await readJson(canonicalPackagePath); packageLoaded = true; } } catch (error) { const code = errorCode(error); issues.push({ code: code === "ENOENT" ? "MODULE_PACKAGE_MISSING" : "MODULE_PACKAGE_INVALID", message: code === "ENOENT" ? "package.json is missing." : `package.json could not be read or parsed${code ? ` (${code})` : ""}.`, moduleDirectory: directoryName, }); } try { canonicalManifestPath = await realpath(manifestPath); if (!pathIsInside(canonicalModuleDirectory, canonicalManifestPath)) { issues.push({ code: "MODULE_MANIFEST_INVALID", message: `${MODULE_MANIFEST_FILENAME} must not resolve outside its module directory.`, moduleDirectory: directoryName, }); } else { manifestInput = await readJson(canonicalManifestPath); manifestLoaded = true; } } catch (error) { const code = errorCode(error); issues.push({ code: code === "ENOENT" ? "MODULE_MANIFEST_MISSING" : "MODULE_MANIFEST_INVALID", message: code === "ENOENT" ? `${MODULE_MANIFEST_FILENAME} is missing.` : `${MODULE_MANIFEST_FILENAME} could not be read or parsed${code ? ` (${code})` : ""}.`, moduleDirectory: directoryName, }); } if (!packageLoaded || !manifestLoaded || !canonicalManifestPath) { return undefined; } const packageData = validatePackageData(packageInput, directoryName, issues); const parsedManifest = safeParseModuleManifest(manifestInput); if (!parsedManifest.success) { for (const validationIssue of parsedManifest.error.issues) { issues.push({ code: "MODULE_MANIFEST_INVALID", message: validationIssue.message, moduleDirectory: directoryName, path: issuePath(validationIssue.path), }); } } if (!packageData || !parsedManifest.success) { return undefined; } const manifest = parsedManifest.data; if (packageData.version !== manifest.version) { issues.push({ code: "MODULE_VERSION_MISMATCH", message: `package.json version ${packageData.version} does not match manifest version ${manifest.version}.`, moduleDirectory: directoryName, moduleId: manifest.id, path: "version", }); } const entrypoints = await resolveEntrypoints(moduleDirectory, directoryName, manifest, issues); if (!entrypoints) { return undefined; } return { directory: canonicalModuleDirectory, directoryName, entrypoints, manifest, manifestPath: canonicalManifestPath, packageName: packageData.name, }; } function addGraphEdge( dependencyId: string, dependentId: string, dependents: Map>, indegrees: Map, ): void { const targets = dependents.get(dependencyId); if (!targets || targets.has(dependentId)) { return; } targets.add(dependentId); indegrees.set(dependentId, (indegrees.get(dependentId) ?? 0) + 1); } function nodeParticipatesInCycle(node: string, graph: ReadonlyMap>): boolean { const visited = new Set(); const pending = [...(graph.get(node) ?? [])]; while (pending.length > 0) { const candidate = pending.pop(); if (!candidate) { continue; } if (candidate === node) { return true; } if (visited.has(candidate)) { continue; } visited.add(candidate); pending.push(...(graph.get(candidate) ?? [])); } return false; } function topologicalModuleIds( modulesById: ReadonlyMap, dependents: Map>, indegrees: Map, issues: ModuleDiscoveryIssue[], ): string[] { const ready = [...modulesById.keys()] .filter((moduleId) => indegrees.get(moduleId) === 0) .sort(compareText); const ordered: string[] = []; while (ready.length > 0) { const moduleId = ready.shift(); if (!moduleId) { break; } ordered.push(moduleId); for (const dependentId of [...(dependents.get(moduleId) ?? [])].sort(compareText)) { const nextIndegree = (indegrees.get(dependentId) ?? 0) - 1; indegrees.set(dependentId, nextIndegree); if (nextIndegree === 0) { ready.push(dependentId); ready.sort(compareText); } } } if (ordered.length !== modulesById.size) { const cycleMembers = [...modulesById.keys()] .filter((moduleId) => nodeParticipatesInCycle(moduleId, dependents)) .sort(compareText); issues.push({ code: "MODULE_DEPENDENCY_CYCLE", message: `Module dependency cycle contains: ${cycleMembers.join(", ")}.`, moduleDirectory: cycleMembers.join(","), }); } return ordered; } export async function discoverModules( options: ModuleDiscoveryOptions, ): Promise { const issues: ModuleDiscoveryIssue[] = []; for (const [versionName, version] of [ ["platformVersion", options.platformVersion], ["moduleSdkVersion", options.moduleSdkVersion], ] as const) { if (!semanticVersionSchema.safeParse(version).success) { issues.push({ code: "DISCOVERY_CONFIGURATION_INVALID", message: `${versionName} must be a strict semantic version, received ${version}.`, moduleDirectory: ".", path: versionName, }); } } if (issues.length > 0) { throw new ModuleDiscoveryError(issues); } let directoryEntries: Dirent[]; try { directoryEntries = await readdir(options.modulesDirectory, { withFileTypes: true }); } catch (error) { if (errorCode(error) === "ENOENT") { return Object.freeze([]); } throw new ModuleDiscoveryError([ { code: "MODULES_DIRECTORY_UNREADABLE", message: `The configured modules directory could not be read${ errorCode(error) ? ` (${errorCode(error)})` : "" }.`, moduleDirectory: ".", }, ]); } const visibleEntries = directoryEntries .filter((entry) => !entry.name.startsWith(".")) .sort((left, right) => compareText(left.name, right.name)); const directoryNames: string[] = []; for (const entry of visibleEntries) { if (entry.isSymbolicLink()) { issues.push({ code: "MODULE_DIRECTORY_INVALID", message: "Module directories must not be symbolic links.", moduleDirectory: entry.name, }); } else if (entry.isDirectory()) { directoryNames.push(entry.name); } } const candidates: ModuleCandidate[] = []; for (const directoryName of directoryNames) { const candidate = await scanModuleDirectory(options.modulesDirectory, directoryName, issues); if (candidate) { candidates.push(candidate); } } const duplicateIds = new Set(); const duplicatePackageNames = new Set(); const firstById = new Map(); const firstByPackageName = new Map(); for (const candidate of candidates) { const existingId = firstById.get(candidate.manifest.id); if (existingId) { duplicateIds.add(candidate.manifest.id); issues.push({ code: "MODULE_ID_DUPLICATE", message: `Module ID ${candidate.manifest.id} is also declared by ${existingId.directoryName}.`, moduleDirectory: candidate.directoryName, moduleId: candidate.manifest.id, path: "id", }); } else { firstById.set(candidate.manifest.id, candidate); } const existingPackage = firstByPackageName.get(candidate.packageName); if (existingPackage) { duplicatePackageNames.add(candidate.packageName); issues.push({ code: "MODULE_PACKAGE_NAME_DUPLICATE", message: `Package name ${candidate.packageName} is also used by ${existingPackage.directoryName}.`, moduleDirectory: candidate.directoryName, moduleId: candidate.manifest.id, path: "name", }); } else { firstByPackageName.set(candidate.packageName, candidate); } } const modulesById = new Map( candidates .filter( (candidate) => !duplicateIds.has(candidate.manifest.id) && !duplicatePackageNames.has(candidate.packageName), ) .map((candidate) => [candidate.manifest.id, candidate] as const), ); const capabilityProviders = new Map(); for (const candidate of modulesById.values()) { if (!isVersionCompatible(options.platformVersion, candidate.manifest.compatibility.platform)) { issues.push({ code: "MODULE_PLATFORM_INCOMPATIBLE", message: `Platform ${options.platformVersion} does not satisfy ${candidate.manifest.compatibility.platform}.`, moduleDirectory: candidate.directoryName, moduleId: candidate.manifest.id, path: "compatibility.platform", }); } if ( !isVersionCompatible(options.moduleSdkVersion, candidate.manifest.compatibility.moduleSdk) ) { issues.push({ code: "MODULE_SDK_INCOMPATIBLE", message: `Module SDK ${options.moduleSdkVersion} does not satisfy ${candidate.manifest.compatibility.moduleSdk}.`, moduleDirectory: candidate.directoryName, moduleId: candidate.manifest.id, path: "compatibility.moduleSdk", }); } for (const capability of candidate.manifest.capabilities) { capabilityProviders.set(capability, candidate); } } const dependents = new Map>(); const indegrees = new Map(); for (const moduleId of modulesById.keys()) { dependents.set(moduleId, new Set()); indegrees.set(moduleId, 0); } for (const candidate of modulesById.values()) { for (const dependencyKind of ["required", "optional"] as const) { for (const [dependencyId, versionRange] of Object.entries( candidate.manifest.dependencies.modules[dependencyKind], )) { const dependency = modulesById.get(dependencyId); if (!dependency) { if (dependencyKind === "required") { issues.push({ code: "MODULE_DEPENDENCY_MISSING", message: `Required module ${dependencyId} is not installed.`, moduleDirectory: candidate.directoryName, moduleId: candidate.manifest.id, path: `dependencies.modules.${dependencyKind}.${dependencyId}`, }); } continue; } if (!isVersionCompatible(dependency.manifest.version, versionRange)) { issues.push({ code: "MODULE_DEPENDENCY_INCOMPATIBLE", message: `${dependencyId} ${dependency.manifest.version} does not satisfy ${versionRange}.`, moduleDirectory: candidate.directoryName, moduleId: candidate.manifest.id, path: `dependencies.modules.${dependencyKind}.${dependencyId}`, }); } addGraphEdge(dependencyId, candidate.manifest.id, dependents, indegrees); } } for (const dependencyKind of ["required", "optional"] as const) { for (const [index, capability] of candidate.manifest.dependencies.capabilities[ dependencyKind ].entries()) { const provider = capabilityProviders.get(capability); if (!provider) { if (dependencyKind === "required") { issues.push({ code: "MODULE_CAPABILITY_MISSING", message: `Required capability ${capability} is not installed.`, moduleDirectory: candidate.directoryName, moduleId: candidate.manifest.id, path: `dependencies.capabilities.${dependencyKind}.${index}`, }); } continue; } addGraphEdge(provider.manifest.id, candidate.manifest.id, dependents, indegrees); } } } const orderedIds = topologicalModuleIds(modulesById, dependents, indegrees, issues); if (issues.length > 0) { throw new ModuleDiscoveryError(issues); } return Object.freeze( orderedIds.map((moduleId) => { const candidate = modulesById.get(moduleId); if (!candidate) { throw new Error(`Internal discovery error: ${moduleId} was not indexed.`); } const { directoryName: _directoryName, ...discoveredModule } = candidate; return freezeRecursively(discoveredModule); }), ); }