diff --git a/schemas/common/db/full/v1.configs.json b/schemas/common/db/full/v1.configs.json index 99ef4539..5c65af4b 100644 --- a/schemas/common/db/full/v1.configs.json +++ b/schemas/common/db/full/v1.configs.json @@ -2,7 +2,7 @@ { "name": "infra-opala-db", "value": { - "$ref": { "configName": "db-connection", "version": "latest" }, + "$ref": { "configName": "db-connection", "version": "latest", "schemaId": "https://mapcolonies.com/common/db/partial/v1" }, "database": "infra-auth-manager" } } diff --git a/schemas/common/s3/full/v1.configs.json b/schemas/common/s3/full/v1.configs.json index baed418f..9bb8c095 100644 --- a/schemas/common/s3/full/v1.configs.json +++ b/schemas/common/s3/full/v1.configs.json @@ -2,7 +2,7 @@ { "name": "infra-opala-s3", "value": { - "$ref": { "configName": "s3-connection", "version": "latest" }, + "$ref": { "configName": "s3-connection", "version": "latest", "schemaId": "https://mapcolonies.com/common/s3/partial/v1" }, "bucket": "infra-opala" } } diff --git a/schemas/infra/opala/cron/v2.configs.json b/schemas/infra/opala/cron/v2.configs.json index 3d9abf84..992e67ba 100644 --- a/schemas/infra/opala/cron/v2.configs.json +++ b/schemas/infra/opala/cron/v2.configs.json @@ -8,18 +8,18 @@ "prettyPrint": false }, "tracing": { - "$ref": { "configName": "common-tracing", "version": "latest" } + "$ref": { "configName": "common-tracing", "version": "latest", "schemaId": "https://mapcolonies.com/common/telemetry/tracing/v1" } }, "shared": {} }, "db": { - "$ref": { "configName": "infra-opala-db", "version": "latest" } + "$ref": { "configName": "infra-opala-db", "version": "latest", "schemaId": "https://mapcolonies.com/common/db/full/v1" } }, "cron": { "prod": { "pattern": "*/5 * * * *", "s3": { - "$ref": { "configName": "infra-opala-s3", "version": "latest" }, + "$ref": { "configName": "infra-opala-s3", "version": "latest", "schemaId": "https://mapcolonies.com/common/s3/full/v1" }, "key": "prod-bundle" } } diff --git a/schemas/infra/opala/manager/v1.configs.json b/schemas/infra/opala/manager/v1.configs.json index 7970442c..12b87d01 100644 --- a/schemas/infra/opala/manager/v1.configs.json +++ b/schemas/infra/opala/manager/v1.configs.json @@ -8,10 +8,12 @@ "prettyPrint": false }, "shared": {}, - "tracing": { "$ref": { "configName": "common-tracing", "version": "latest" } } + "tracing": { + "$ref": { "configName": "common-tracing", "version": "latest", "schemaId": "https://mapcolonies.com/common/telemetry/tracing/v1" } + } }, "db": { - "$ref": { "configName": "infra-opala-db", "version": "latest" } + "$ref": { "configName": "infra-opala-db", "version": "latest", "schemaId": "https://mapcolonies.com/common/db/full/v1" } } } } diff --git a/scripts/validate/core.mts b/scripts/validate/core.mts index 3bb00db1..86bb35fe 100644 --- a/scripts/validate/core.mts +++ b/scripts/validate/core.mts @@ -12,12 +12,14 @@ const addFormats = addFormatsImport.default; export type ConfigReference = { configName: string; version: 1 | 'latest'; + schemaId: string; }; const configRefSchema: JSONSchemaType = { - required: ['configName', 'version'], + required: ['configName', 'version', 'schemaId'], properties: { configName: { type: 'string' }, + schemaId: { type: 'string' }, version: { oneOf: [ { type: 'number', minimum: 1, maximum: 1 }, diff --git a/scripts/validate/validate.mts b/scripts/validate/validate.mts index 38955e8a..3fef42a1 100644 --- a/scripts/validate/validate.mts +++ b/scripts/validate/validate.mts @@ -3,7 +3,7 @@ import fs from 'node:fs'; import path from 'node:path'; import { AsyncLocalStorage } from 'node:async_hooks'; import AjvModule from 'ajv/dist/2019.js'; -import * as draft7MetaSchema from 'ajv/dist/refs/json-schema-draft-07.json' assert { type: 'json' }; +import * as draft7MetaSchema from 'ajv/dist/refs/json-schema-draft-07.json' with { type: 'json' }; import addFormats from 'ajv-formats'; import { $RefParser } from '@apidevtools/json-schema-ref-parser'; import { presult, result } from '../util/index.mjs'; diff --git a/scripts/validate/validateConfigs.mts b/scripts/validate/validateConfigs.mts index 613deb63..4de0c6ba 100644 --- a/scripts/validate/validateConfigs.mts +++ b/scripts/validate/validateConfigs.mts @@ -4,7 +4,7 @@ import * as posixPath from 'node:path/posix'; import * as path from 'node:path'; import { AsyncLocalStorage } from 'node:async_hooks'; import { betterAjvErrors } from '@apideck/better-ajv-errors'; -import * as configsSchema from '../schemas/configs.schema.json' assert { type: 'json' }; +import * as configsSchema from '../schemas/configs.schema.json' with { type: 'json' }; import AjvModule from 'ajv'; import { ErrorHandler } from '../util/errorHandling.mjs'; import { listConfigRefs, replaceRefs, validateConfig } from './core.mjs'; @@ -18,6 +18,7 @@ const fileAsyncStorage = new AsyncLocalStorage<{ configFilePath: string; schemaPath: string; schemaVersion: number; + schemaId: string; handleError: (msg: string) => void; schema?: any; configs?: configInstance[]; @@ -34,7 +35,41 @@ const configsFileValidator = new AjvModule.default({ allErrors: true, }).compile(configsSchema); -const seenConfigs = new Map(); +const seenConfigs = new Map(); + +/** + * Constructs a schema ID from the config file directory path. + * Converts file system path to schema URL format. + * @param directory - The directory path of the config file + * @param schemaVersion - The schema version number + * @returns The schema ID in URL format + */ +function constructSchemaId(directory: string, schemaVersion: number): string { + // Convert directory path like 'schemas/common/boilerplate' to 'https://mapcolonies.com/common/boilerplate/v1' + const pathParts = directory.split(path.sep); + const schemasIndex = pathParts.findIndex((part) => part === 'schemas'); + if (schemasIndex === -1) { + throw new Error('Invalid schema directory structure'); + } + + const schemaParts = pathParts.slice(schemasIndex + 1); + const schemaPath = schemaParts.join('/'); + return `https://mapcolonies.com/${schemaPath}/v${schemaVersion}`; +} + +/** + * Removes the version part from a schema ID to get the base schema identifier. + * @param schemaId - The full schema ID (e.g., 'https://mapcolonies.com/common/db/v1') + * @returns The base schema ID without version (e.g., 'https://mapcolonies.com/common/db') + */ +function removeSchemaVersion(schemaId: string): string { + // Remove the last part of the schemaId, which is the version + const lastSlashIndex = schemaId.lastIndexOf('/'); + if (lastSlashIndex === -1) { + return schemaId; + } + return schemaId.substring(0, lastSlashIndex); +} async function validateConfigInstance() { const fileContext = fileAsyncStorage.getStore(); @@ -57,7 +92,7 @@ async function validateConfigInstance() { } function listAllRefs(config: any): Parameters[1] { - const refNames = new Map(); + const refNames = new Map(); function inner(value: any) { const refList = listConfigRefs(value); @@ -67,12 +102,20 @@ function listAllRefs(config: any): Parameters[1] { } for (const ref of refList) { - const config = seenConfigs.get(ref.configName); - if (!config) { + const configEntries = seenConfigs.get(ref.configName); + if (!configEntries || configEntries.length === 0) { throw new Error(`The config ${ref.configName} does not exist`); } - refNames.set(ref.configName, ref.version); - inner(config); + + // Find the config entry with the matching schemaId + const matchingConfigEntry = configEntries.find((entry) => entry.schemaId === ref.schemaId); + if (!matchingConfigEntry) { + const availableSchemaIds = configEntries.map((entry) => entry.schemaId).join(', '); + throw new Error(`The config ${ref.configName} does not exist with schemaId ${ref.schemaId}. Available schemaIds: ${availableSchemaIds}`); + } + + refNames.set(ref.configName, { version: ref.version, schemaId: ref.schemaId }); + inner(matchingConfigEntry.value); } } @@ -80,8 +123,23 @@ function listAllRefs(config: any): Parameters[1] { const result: Parameters[1] = []; - for (const [configName, version] of refNames.entries()) { - result.push({ configName, config: seenConfigs.get(configName), version }); + for (const [configName, refInfo] of refNames.entries()) { + const configEntries = seenConfigs.get(configName); + if (!configEntries || configEntries.length === 0) { + throw new Error(`Config ${configName} not found`); + } + + const matchingConfigEntry = configEntries.find((entry) => entry.schemaId === refInfo.schemaId); + if (!matchingConfigEntry) { + throw new Error(`Config ${configName} not found with schemaId ${refInfo.schemaId}`); + } + + result.push({ + configName, + config: matchingConfigEntry.value, + version: refInfo.version, + schemaId: refInfo.schemaId, + }); } return result; @@ -97,8 +155,10 @@ async function forEachConfigFileWithContext( const fileNameParts = configFile.fileName.split('.'); const schemaPath = posixPath.join(configFile.directory, `${fileNameParts[0]}.schema.json`); const configFilePath = path.join(configFile.directory, configFile.fileName); + const schemaVersion = parseInt(fileNameParts[0].substring(1)); + const schemaId = constructSchemaId(configFile.directory, schemaVersion); - await fileAsyncStorage.run({ schemaPath, schemaVersion: parseInt(fileNameParts[0].substring(1)), configFilePath, handleError }, async () => { + await fileAsyncStorage.run({ schemaPath, schemaVersion, configFilePath, schemaId, handleError }, async () => { await action(...args); }); } @@ -173,11 +233,28 @@ async function validateConfigNames(): Promise { } for (const { name, value } of fileContext.configs) { - if (seenConfigs.has(name)) { - fileContext.handleError(`Config name ${name} is not unique`); - return false; + const existingConfigs = seenConfigs.get(name); + if (existingConfigs && existingConfigs.length > 0) { + // Config name must be unique between different schemas, but can repeat under the same schema with different version + const currentBaseSchema = removeSchemaVersion(fileContext.schemaId); + + for (const existingConfig of existingConfigs) { + const existingBaseSchema = removeSchemaVersion(existingConfig.schemaId); + + if (existingBaseSchema !== currentBaseSchema) { + fileContext.handleError( + `Config name ${name} is not unique between different schemas. It exists in schema ${existingBaseSchema} and ${currentBaseSchema}` + ); + return false; + } + } + + // Add to existing array + existingConfigs.push({ value, schemaId: fileContext.schemaId }); + } else { + // Create new array for this config name + seenConfigs.set(name, [{ value, schemaId: fileContext.schemaId }]); } - seenConfigs.set(name, value); } return true;