feat: added support for schemaId in config ref

schemas/common/db/full/v1.configs.json

@@ -2,7 +2,7 @@
   {
     "name": "infra-opala-db",
     "value": {
-      "$ref": { "configName": "db-connection", "version": "latest" },
+      "$ref": { "configName": "db-connection", "version": "latest", "schemaId": "https://mapcolonies.com/common/db/partial/v1" },
       "database": "infra-auth-manager"
     }
   }

schemas/common/s3/full/v1.configs.json

@@ -2,7 +2,7 @@
   {
     "name": "infra-opala-s3",
     "value": {
-      "$ref": { "configName": "s3-connection", "version": "latest" },
+      "$ref": { "configName": "s3-connection", "version": "latest", "schemaId": "https://mapcolonies.com/common/s3/partial/v1" },
       "bucket": "infra-opala"
     }
   }

schemas/infra/opala/cron/v2.configs.json

@@ -8,18 +8,18 @@
           "prettyPrint": false
         },
         "tracing": {
-          "$ref": { "configName": "common-tracing", "version": "latest" }
+          "$ref": { "configName": "common-tracing", "version": "latest", "schemaId": "https://mapcolonies.com/common/telemetry/tracing/v1" }
         },
         "shared": {}
       },
       "db": {
-        "$ref": { "configName": "infra-opala-db", "version": "latest" }
+        "$ref": { "configName": "infra-opala-db", "version": "latest", "schemaId": "https://mapcolonies.com/common/db/full/v1" }
       },
       "cron": {
         "prod": {
           "pattern": "*/5 * * * *",
           "s3": {
-            "$ref": { "configName": "infra-opala-s3", "version": "latest" },
+            "$ref": { "configName": "infra-opala-s3", "version": "latest", "schemaId": "https://mapcolonies.com/common/s3/full/v1" },
             "key": "prod-bundle"
           }
         }

schemas/infra/opala/manager/v1.configs.json

@@ -8,10 +8,12 @@
           "prettyPrint": false
         },
         "shared": {},
-        "tracing": { "$ref": { "configName": "common-tracing", "version": "latest" } }
+        "tracing": {
+          "$ref": { "configName": "common-tracing", "version": "latest", "schemaId": "https://mapcolonies.com/common/telemetry/tracing/v1" }
+        }
       },
       "db": {
-        "$ref": { "configName": "infra-opala-db", "version": "latest" }
+        "$ref": { "configName": "infra-opala-db", "version": "latest", "schemaId": "https://mapcolonies.com/common/db/full/v1" }
       }
     }
   }

scripts/validate/core.mts

@@ -12,12 +12,14 @@ const addFormats = addFormatsImport.default;
 export type ConfigReference = {
   configName: string;
   version: 1 | 'latest';
+  schemaId: string;
 };
 
 const configRefSchema: JSONSchemaType<ConfigReference> = {
-  required: ['configName', 'version'],
+  required: ['configName', 'version', 'schemaId'],
   properties: {
     configName: { type: 'string' },
+    schemaId: { type: 'string' },
     version: {
       oneOf: [
         { type: 'number', minimum: 1, maximum: 1 },

scripts/validate/validate.mts

@@ -3,7 +3,7 @@ import fs from 'node:fs';
 import path from 'node:path';
 import { AsyncLocalStorage } from 'node:async_hooks';
 import AjvModule from 'ajv/dist/2019.js';
-import * as draft7MetaSchema from 'ajv/dist/refs/json-schema-draft-07.json' assert { type: 'json' };
+import * as draft7MetaSchema from 'ajv/dist/refs/json-schema-draft-07.json' with { type: 'json' };
 import addFormats from 'ajv-formats';
 import { $RefParser } from '@apidevtools/json-schema-ref-parser';
 import { presult, result } from '../util/index.mjs';

scripts/validate/validateConfigs.mts

@@ -4,7 +4,7 @@ import * as posixPath from 'node:path/posix';
 import * as path from 'node:path';
 import { AsyncLocalStorage } from 'node:async_hooks';
 import { betterAjvErrors } from '@apideck/better-ajv-errors';
-import * as configsSchema from '../schemas/configs.schema.json' assert { type: 'json' };
+import * as configsSchema from '../schemas/configs.schema.json' with { type: 'json' };
 import AjvModule from 'ajv';
 import { ErrorHandler } from '../util/errorHandling.mjs';
 import { listConfigRefs, replaceRefs, validateConfig } from './core.mjs';
@@ -18,6 +18,7 @@ const fileAsyncStorage = new AsyncLocalStorage<{
   configFilePath: string;
   schemaPath: string;
   schemaVersion: number;
+  schemaId: string;
   handleError: (msg: string) => void;
   schema?: any;
   configs?: configInstance[];
@@ -34,7 +35,41 @@ const configsFileValidator = new AjvModule.default({
   allErrors: true,
 }).compile(configsSchema);
 
-const seenConfigs = new Map<string, any>();
+const seenConfigs = new Map<string, { value: any; schemaId: string }[]>();
+
+/**
+ * Constructs a schema ID from the config file directory path.
+ * Converts file system path to schema URL format.
+ * @param directory - The directory path of the config file
+ * @param schemaVersion - The schema version number
+ * @returns The schema ID in URL format
+ */
+function constructSchemaId(directory: string, schemaVersion: number): string {
+  // Convert directory path like 'schemas/common/boilerplate' to 'https://mapcolonies.com/common/boilerplate/v1'
+  const pathParts = directory.split(path.sep);
+  const schemasIndex = pathParts.findIndex((part) => part === 'schemas');
+  if (schemasIndex === -1) {
+    throw new Error('Invalid schema directory structure');
+  }
+
+  const schemaParts = pathParts.slice(schemasIndex + 1);
+  const schemaPath = schemaParts.join('/');
+  return `https://mapcolonies.com/${schemaPath}/v${schemaVersion}`;
+}
+
+/**
+ * Removes the version part from a schema ID to get the base schema identifier.
+ * @param schemaId - The full schema ID (e.g., 'https://mapcolonies.com/common/db/v1')
+ * @returns The base schema ID without version (e.g., 'https://mapcolonies.com/common/db')
+ */
+function removeSchemaVersion(schemaId: string): string {
+  // Remove the last part of the schemaId, which is the version
+  const lastSlashIndex = schemaId.lastIndexOf('/');
+  if (lastSlashIndex === -1) {
+    return schemaId;
+  }
+  return schemaId.substring(0, lastSlashIndex);
+}
 
 async function validateConfigInstance() {
   const fileContext = fileAsyncStorage.getStore();
@@ -57,7 +92,7 @@ async function validateConfigInstance() {
 }
 
 function listAllRefs(config: any): Parameters<typeof replaceRefs>[1] {
-  const refNames = new Map<string, 'latest' | 1>();
+  const refNames = new Map<string, { version: 'latest' | 1; schemaId: string }>();
 
   function inner(value: any) {
     const refList = listConfigRefs(value);
@@ -67,21 +102,44 @@ function listAllRefs(config: any): Parameters<typeof replaceRefs>[1] {
     }
 
     for (const ref of refList) {
-      const config = seenConfigs.get(ref.configName);
-      if (!config) {
+      const configEntries = seenConfigs.get(ref.configName);
+      if (!configEntries || configEntries.length === 0) {
         throw new Error(`The config ${ref.configName} does not exist`);
       }
-      refNames.set(ref.configName, ref.version);
-      inner(config);
+
+      // Find the config entry with the matching schemaId
+      const matchingConfigEntry = configEntries.find((entry) => entry.schemaId === ref.schemaId);
+      if (!matchingConfigEntry) {
+        const availableSchemaIds = configEntries.map((entry) => entry.schemaId).join(', ');
+        throw new Error(`The config ${ref.configName} does not exist with schemaId ${ref.schemaId}. Available schemaIds: ${availableSchemaIds}`);
+      }
+
+      refNames.set(ref.configName, { version: ref.version, schemaId: ref.schemaId });
+      inner(matchingConfigEntry.value);
     }
   }
 
   inner(config);
 
   const result: Parameters<typeof replaceRefs>[1] = [];
 
-  for (const [configName, version] of refNames.entries()) {
-    result.push({ configName, config: seenConfigs.get(configName), version });
+  for (const [configName, refInfo] of refNames.entries()) {
+    const configEntries = seenConfigs.get(configName);
+    if (!configEntries || configEntries.length === 0) {
+      throw new Error(`Config ${configName} not found`);
+    }
+
+    const matchingConfigEntry = configEntries.find((entry) => entry.schemaId === refInfo.schemaId);
+    if (!matchingConfigEntry) {
+      throw new Error(`Config ${configName} not found with schemaId ${refInfo.schemaId}`);
+    }
+
+    result.push({
+      configName,
+      config: matchingConfigEntry.value,
+      version: refInfo.version,
+      schemaId: refInfo.schemaId,
+    });
   }
 
   return result;
@@ -97,8 +155,10 @@ async function forEachConfigFileWithContext<args extends unknown[]>(
     const fileNameParts = configFile.fileName.split('.');
     const schemaPath = posixPath.join(configFile.directory, `${fileNameParts[0]}.schema.json`);
     const configFilePath = path.join(configFile.directory, configFile.fileName);
+    const schemaVersion = parseInt(fileNameParts[0].substring(1));
+    const schemaId = constructSchemaId(configFile.directory, schemaVersion);
 
-    await fileAsyncStorage.run({ schemaPath, schemaVersion: parseInt(fileNameParts[0].substring(1)), configFilePath, handleError }, async () => {
+    await fileAsyncStorage.run({ schemaPath, schemaVersion, configFilePath, schemaId, handleError }, async () => {
       await action(...args);
     });
   }
@@ -173,11 +233,28 @@ async function validateConfigNames(): Promise<boolean> {
   }
 
   for (const { name, value } of fileContext.configs) {
-    if (seenConfigs.has(name)) {
-      fileContext.handleError(`Config name ${name} is not unique`);
-      return false;
+    const existingConfigs = seenConfigs.get(name);
+    if (existingConfigs && existingConfigs.length > 0) {
+      // Config name must be unique between different schemas, but can repeat under the same schema with different version
+      const currentBaseSchema = removeSchemaVersion(fileContext.schemaId);
+
+      for (const existingConfig of existingConfigs) {
+        const existingBaseSchema = removeSchemaVersion(existingConfig.schemaId);
+
+        if (existingBaseSchema !== currentBaseSchema) {
+          fileContext.handleError(
+            `Config name ${name} is not unique between different schemas. It exists in schema ${existingBaseSchema} and ${currentBaseSchema}`
+          );
+          return false;
+        }
+      }
+
+      // Add to existing array
+      existingConfigs.push({ value, schemaId: fileContext.schemaId });
+    } else {
+      // Create new array for this config name
+      seenConfigs.set(name, [{ value, schemaId: fileContext.schemaId }]);
     }
-    seenConfigs.set(name, value);
   }
 
   return true;