From 8aa987ed47c432f3f9ebded5010129f0a302c7aa Mon Sep 17 00:00:00 2001 From: Grahame Grieve Date: Sat, 7 Mar 2026 11:07:42 +1100 Subject: [PATCH 01/15] Add additional security rules --- packages/package-crawler.js | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/packages/package-crawler.js b/packages/package-crawler.js index 2866fe26..82b39d8f 100644 --- a/packages/package-crawler.js +++ b/packages/package-crawler.js @@ -485,6 +485,14 @@ class PackageCrawler { throw new Error(`NPM Canonical "${canonical}" is not valid from ${source}`); } + const isTemplate = npmPackage.kind === 2; // fhir.template + if (npmPackage.hasInstallScripts) { + throw new Error(`Package ${idver} rejected: contains install scripts (preinstall/install/postinstall)`); + } + if (npmPackage.hasJavaScript && !isTemplate) { + throw new Error(`Package ${idver} rejected: contains JavaScript files but is not a template package`); + } + // Extract URLs from package const urls = this.processPackageUrls(npmPackage); @@ -554,6 +562,14 @@ class PackageCrawler { throw new Error('package.json not found in extracted package'); } + const hasInstallScripts = !!( + packageJson.scripts && ( + packageJson.scripts.preinstall || + packageJson.scripts.install || + packageJson.scripts.postinstall + ) + ); + const hasJavaScript = Object.keys(files).some(f => f.endsWith('.js') || f.endsWith('.mjs') || f.endsWith('.cjs')); const packageJson = JSON.parse(files['package.json']); // Extract basic NPM fields @@ -664,6 +680,8 @@ class PackageCrawler { url: homepage, dependencies, kind, + hasInstallScripts, + hasJavaScript, notForPublication, files }; From b9322c62f2af2b71941a1826c2056e076fba80fa Mon Sep 17 00:00:00 2001 From: Grahame Grieve Date: Sat, 7 Mar 2026 11:17:13 +1100 Subject: [PATCH 02/15] update code system list every minute --- tx/cs/cs-provider-api.js | 26 +++++++++++++++++- tx/cs/cs-provider-list.js | 4 +-- tx/library.js | 16 ++++------- tx/library/canonical-resource.js | 7 ++++- tx/provider.js | 47 ++++++++++++++++++++++++++++++++ tx/tx.js | 8 ++++++ 6 files changed, 94 insertions(+), 14 deletions(-) diff --git a/tx/cs/cs-provider-api.js b/tx/cs/cs-provider-api.js index 3ae74369..cc7131e0 100644 --- a/tx/cs/cs-provider-api.js +++ b/tx/cs/cs-provider-api.js @@ -20,7 +20,13 @@ class AbstractCodeSystemProvider { } /** - * Returns the list of CodeSystems this provider provides + * Returns the list of CodeSystems this provider provides. This is called once at start up. + * The code systems should be fully loaded; lazy loading code systems is not considered good + * for engineering. + * + * + * Note that unlike value sets, which are accessed from the provider on the fly, code systems + * are all preloaded into the kernel (e.g. provider) at start up * * @param {string} fhirVersion - The FHIRVersion in scope - if relevant (there's always a stated version, though R5 is always used) * @param {string} context - The client's stated context - if provided. @@ -32,6 +38,24 @@ class AbstractCodeSystemProvider { throw new Error('listCodeSystems must be implemented by AbstractCodeSystemProvider subclass'); } + /** + * This is called once a minute to update the code system list that the provider maintains. + * + * return an object that has three Map: {added, changed, deleted} + * + * these use the same key as the + * + * code systems are identified by url and version + * + * @param fhirVersion + * @param context + * @returns {Promise} + */ + // eslint-disable-next-line no-unused-vars + async getCodeSystemChanges(fhirVersion, context){ + return null; + } + async close() { } diff --git a/tx/cs/cs-provider-list.js b/tx/cs/cs-provider-list.js index 9fb55891..019c457f 100644 --- a/tx/cs/cs-provider-list.js +++ b/tx/cs/cs-provider-list.js @@ -7,7 +7,7 @@ class ListCodeSystemProvider extends AbstractCodeSystemProvider { /** * {Map} A list of code system factories that contains all the preloaded native code systems */ - codeSystems = new Map(); + codeSystems = []; /** * ensure that the ids on the code systems are unique, if they are @@ -17,7 +17,7 @@ class ListCodeSystemProvider extends AbstractCodeSystemProvider { */ // eslint-disable-next-line no-unused-vars assignIds(ids) { - for (const cs of this.codeSystems.values()) { + for (const cs of this.codeSystems) { if (!cs.id || ids.has("CodeSystem/"+cs.id)) { cs.id = ""+ids.size; } diff --git a/tx/library.js b/tx/library.js index c1a23597..b8a86bb2 100644 --- a/tx/library.js +++ b/tx/library.js @@ -464,13 +464,7 @@ class Library { for (const resource of resources) { const cs = new CodeSystem(await contentLoader.loadFile(resource, contentLoader.fhirVersion())); cs.sourcePackage = contentLoader.pid(); - const existing = cp.codeSystems.get(cs.url); - if (!existing || cs.isMoreRecent(existing)) { - cp.codeSystems.set(cs.url, cs); - } - if (cs.version) { - cp.codeSystems.set(cs.vurl, cs); - } + cp.codeSystems.push(cs); csc++; } this.codeSystemProviders.push(cp); @@ -686,10 +680,12 @@ class Library { } + provider.codeSystemProviders = this.codeSystemProviders; + provider.context = context; for (const cp of this.codeSystemProviders) { - const csMap = await cp.listCodeSystems(fhirVersion, context); - for (const [key, value] of csMap) { - provider.codeSystems.set(key, value); + const csList = await cp.listCodeSystems(fhirVersion, context); + for (const cs of csList) { + provider.addCodeSystem(cs); } } // Don't clone valueSetProviders yet - we'll build it with correct order diff --git a/tx/library/canonical-resource.js b/tx/library/canonical-resource.js index 6098d876..112a221d 100644 --- a/tx/library/canonical-resource.js +++ b/tx/library/canonical-resource.js @@ -111,7 +111,12 @@ class CanonicalResource { const fmt = this.versionAlgorithm() || other.versionAlgorithm() || this.guessVersionAlgorithmFromVersion(this.version); switch (fmt) { case 'semver': - return VersionUtilities.isThisOrLater(other.version, this.version, VersionPrecision.PATCH); + try { + return VersionUtilities.isThisOrLater(other.version, this.version, VersionPrecision.PATCH); + } catch (error) { + // other is not semver. Not much we can do + return false; + } case 'date': return this.dateIsMoreRecent(this.version, other.version); case 'integer': diff --git a/tx/provider.js b/tx/provider.js index c31d420a..deca4981 100644 --- a/tx/provider.js +++ b/tx/provider.js @@ -23,6 +23,7 @@ const {PackageConceptMapProvider} = require("./cm/cm-package"); class Provider { i18n; fhirVersion; + context; /** * {Map} A list of code system factories that contains all the preloaded native code systems @@ -34,6 +35,11 @@ class Provider { */ codeSystems; + /** + * {List} code system providers, for maintaing the code system list + */ + codeSystemProviders + /** * {List} A list of value set providers that know how to provide value sets by request */ @@ -427,6 +433,47 @@ class Provider { return false; } + updateCodeSystemList() { + for (let csp of this.codeSystemProviders) { + let changes = csp.getCodeSystemChanges(this.fhirVersion, this.context); + if (changes) { + for (let cs of changes.added || []) { + this.addCodeSystem(cs); + } + for (let cs of changes.changed || []) { + this.addCodeSystem(cs); + } + for (let cs of changes.deleted || []) { + this.deleteCodeSystem(cs); + } + } + } + } + + addCodeSystem(cs) { + const existing = this.codeSystems.get(cs.url); + if (!existing || cs.isMoreRecent(existing)) { + this.codeSystems.set(cs.url, cs); + } + if (cs.version) { + this.codeSystems.set(cs.vurl, cs); + } + } + + deleteCodeSystem(cs) { + this.codeSystems.delete(cs.vurl); + this.codeSystems.delete(cs.url); + let existing = null; + for (let t of this.codeSystems.values()) { + if (!existing || t.isMoreRecent(existing)) { + existing = t; + } + } + if (existing) { + this.codeSystems.set(cs.url, cs); + } + } + } module.exports = { Provider }; diff --git a/tx/tx.js b/tx/tx.js index 84b4108c..241bbb1e 100644 --- a/tx/tx.js +++ b/tx/tx.js @@ -247,6 +247,14 @@ class TXModule { if (this.stats) { this.stats.addTask("Client Cache", "5 min"); } + this.timers.push(setInterval(async () => { + try { + await endpointInfo.provider.updateCodeSystemList(); + } catch (error) { + this.log.error(`Error updating CodeSystem list for ${endpointPath}: ${error.message}`); + } + }, 60 * 1000)); + this.log.info(`CodeSystem list update scheduled for ${endpointPath}`); this.timers.push(setInterval(() => { endpointInfo.resourceCache.prune(cacheTimeoutMs); }, pruneIntervalMs)); From ad28824986cc580e7d3177ec3c54bba041ffcc9c Mon Sep 17 00:00:00 2001 From: Grahame Grieve Date: Sat, 7 Mar 2026 11:17:23 +1100 Subject: [PATCH 03/15] remove debugging code --- tx/workers/search.js | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/tx/workers/search.js b/tx/workers/search.js index 832741d5..6cd8579b 100644 --- a/tx/workers/search.js +++ b/tx/workers/search.js @@ -146,9 +146,7 @@ class SearchWorker extends TerminologyWorker { for (const [key, cs] of this.provider.codeSystems) { this.deadCheck('searchCodeSystems'); - if (cs.url == 'http://www.cms.gov/Medicare/Coding/HCPCSReleaseCodeSets') { - console.log("debug"); - } + if (key == cs.vurl) { const json = cs.jsonObj; From b641368dd4858a13d82082aa85bd3f74e8eab486 Mon Sep 17 00:00:00 2001 From: Grahame Grieve Date: Sat, 7 Mar 2026 11:17:40 +1100 Subject: [PATCH 04/15] default search is without any specified source --- tx/tx-html.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tx/tx-html.js b/tx/tx-html.js index 2585d4df..6f115bb1 100644 --- a/tx/tx-html.js +++ b/tx/tx-html.js @@ -1246,7 +1246,7 @@ class TxHtmlRenderer { } buildSourceOptions(provider) { - let result = ''; + let result = ''; result += ``; for (let sp of provider.listValueSetSourceCodes()) { result += ``; From 03a02c6d87f9e1f378cdc4b13054f4a9869a1b3d Mon Sep 17 00:00:00 2001 From: Italo Macedo Date: Sat, 7 Mar 2026 02:57:35 -0300 Subject: [PATCH 05/15] First working Version of OCL Providers --- tx/library.js | 121 +++ tx/ocl/cm-ocl.js | 656 +++++++++++-- tx/ocl/cs-ocl.js | 2306 +++++++++++++++++++++++++++++++++++++++++++++- tx/ocl/vs-ocl.js | 1918 ++++++++++++++++++++++++++++++++++++-- 4 files changed, 4819 insertions(+), 182 deletions(-) diff --git a/tx/library.js b/tx/library.js index b8a86bb2..56def726 100644 --- a/tx/library.js +++ b/tx/library.js @@ -31,6 +31,9 @@ const { Provider } = require("./provider"); const {I18nSupport} = require("../library/i18nsupport"); const folders = require('../library/folder-setup'); const {VSACValueSetProvider} = require("./vs/vs-vsac"); +const { OCLCodeSystemProvider, OCLSourceCodeSystemFactory } = require('./ocl/cs-ocl'); +const { OCLValueSetProvider } = require('./ocl/vs-ocl'); +const { OCLConceptMapProvider } = require('./ocl/cm-ocl'); /** * This class holds all the loaded content ready for processing @@ -95,6 +98,8 @@ class Library { this.codeSystemProviders = []; this.valueSetProviders = []; this.conceptMapProviders = []; + this.oclProviderSets = new Map(); + this.oclConfig = {}; // Create package manager for FHIR packages const packageServers = ['https://packages2.fhir.org/packages']; @@ -154,6 +159,7 @@ class Library { const yamlContent = await fs.readFile(yamlPath, 'utf8'); const config = yaml.parse(yamlContent); this.baseUrl = config.base.url; + this.oclConfig = config.ocl && typeof config.ocl === 'object' ? config.ocl : {}; this.log.info('Fetching Data from '+this.baseUrl); @@ -277,12 +283,127 @@ class Library { case 'url/cs': await this.loadUrl(packageManager, details, isDefault, mode, true); break; + + case 'ocl': + await this.loadOcl(details, isDefault, mode); + break; default: throw new Error(`Unknown source type: ${type}`); } } + parseOclConfig(details) { + const text = String(details || '').trim(); + if (!text) { + throw new Error('OCL source requires details, e.g. ocl:https://ocl.example.org'); + } + + const parts = text.split('|').map(p => p.trim()).filter(Boolean); + const baseUrl = this.resolveOclConfigValue(parts[0]); + if (!baseUrl) { + throw new Error('OCL source requires a base URL'); + } + + const config = { baseUrl }; + for (let i = 1; i < parts.length; i++) { + const part = parts[i]; + const eq = part.indexOf('='); + if (eq === -1) { + continue; + } + const key = part.substring(0, eq).trim().toLowerCase(); + const value = this.resolveOclConfigValue(part.substring(eq + 1).trim()); + if (!value) { + continue; + } + if (key === 'org') { + config.org = value; + } else if (key === 'token') { + config.token = value; + } else if (key === 'timeout') { + const timeout = Number(value); + if (Number.isFinite(timeout) && timeout > 0) { + config.timeout = timeout; + } + } + } + + return config; + } + + resolveOclConfigValue(value) { + const text = String(value || '').trim(); + if (!text) { + return text; + } + + if (this.oclConfig && Object.hasOwn(this.oclConfig, text)) { + const resolved = this.oclConfig[text]; + if (typeof resolved === 'string') { + return resolved.trim(); + } + } + + return text; + } + + async loadOcl(details, isDefault, mode) { + const config = this.parseOclConfig(details); + const cacheKey = `${config.baseUrl}|${config.org || ''}`; + + let providerSet = this.oclProviderSets.get(cacheKey); + if (!providerSet) { + const codeSystemProvider = new OCLCodeSystemProvider(config); + const valueSetProvider = new OCLValueSetProvider(config); + const conceptMapProvider = new OCLConceptMapProvider(config); + providerSet = { + config, + codeSystemProvider, + valueSetProvider, + conceptMapProvider, + csRegistered: false, + factoriesRegistered: false, + vsRegistered: false, + cmRegistered: false + }; + this.oclProviderSets.set(cacheKey, providerSet); + } + + if (mode === 'fetch') { + return; + } + + if (mode === 'cs') { + if (!providerSet.csRegistered) { + this.codeSystemProviders.push(providerSet.codeSystemProvider); + providerSet.csRegistered = true; + } + + if (!providerSet.factoriesRegistered) { + await providerSet.codeSystemProvider.listCodeSystems('5.0', null); + const metas = providerSet.codeSystemProvider.getSourceMetas(); + for (const meta of metas) { + const factory = new OCLSourceCodeSystemFactory(this.i18n, providerSet.codeSystemProvider.httpClient, meta); + this.registerProvider(`ocl:${config.baseUrl}`, factory, isDefault); + } + providerSet.factoriesRegistered = true; + } + return; + } + + if (mode === 'npm') { + if (!providerSet.vsRegistered) { + this.valueSetProviders.push(providerSet.valueSetProvider); + providerSet.vsRegistered = true; + } + if (!providerSet.cmRegistered) { + this.conceptMapProviders.push(providerSet.conceptMapProvider); + providerSet.cmRegistered = true; + } + } + } + async loadInternal(details, isDefault, mode) { if (isDefault) { throw new Error("Default is not supported for internal code system providers"); diff --git a/tx/ocl/cm-ocl.js b/tx/ocl/cm-ocl.js index ccf0f362..23b76517 100644 --- a/tx/ocl/cm-ocl.js +++ b/tx/ocl/cm-ocl.js @@ -1,106 +1,610 @@ -/** - * Abstract base class for Concept Map providers - * Defines the interface that all Concept Map providers must implement - */ -// eslint-disable-next-line no-unused-vars -class OCLConceptMapProvider { - /** - * {int} Unique number assigned to this provider - */ - spaceId; - - /** - * ensure that the ids on the Concept Maps are unique, if they are - * in the global namespace - * - * @param {Set} ids - */ - // eslint-disable-next-line no-unused-vars +const axios = require('axios'); +const { AbstractConceptMapProvider } = require('../cm/cm-api'); +const { ConceptMap } = require('../library/conceptmap'); + +const DEFAULT_BASE_URL = 'https://oclapi2.ips.hsl.org.br'; +const PAGE_SIZE = 100; +const DEFAULT_MAX_SEARCH_PAGES = 10; + +class OCLConceptMapProvider extends AbstractConceptMapProvider { + constructor(config = {}) { + super(); + const options = typeof config === 'string' ? { baseUrl: config } : (config || {}); + + this.baseUrl = (options.baseUrl || DEFAULT_BASE_URL).replace(/\/+$/, ''); + this.org = options.org || null; + this.maxSearchPages = options.maxSearchPages || DEFAULT_MAX_SEARCH_PAGES; + + const headers = { + Accept: 'application/json', + 'User-Agent': 'FHIRSmith-OCL-Provider/1.0' + }; + + if (options.token) { + headers.Authorization = options.token.startsWith('Token ') || options.token.startsWith('Bearer ') + ? options.token + : `Token ${options.token}`; + } + + this.httpClient = axios.create({ + baseURL: this.baseUrl, + timeout: options.timeout || 30000, + headers + }); + + this.conceptMapMap = new Map(); + this._idMap = new Map(); + this._sourceCandidatesCache = new Map(); + this._sourceUrlsByCanonical = new Map(); + this._canonicalBySourceUrl = new Map(); + } + assignIds(ids) { - throw new Error('assignIds must be implemented by AbstractConceptMapProvider subclass'); + if (!this.spaceId) { + return; + } + + const unique = new Set(this.conceptMapMap.values()); + this._idMap.clear(); + + for (const cm of unique) { + if (!cm.id.startsWith(`${this.spaceId}-`)) { + const nextId = `${this.spaceId}-${cm.id}`; + cm.id = nextId; + cm.jsonObj.id = nextId; + } + this._idMap.set(cm.id, cm); + ids.add(`ConceptMap/${cm.id}`); + } } - /** - * Fetches a specific Concept Map by URL and version - * @param {string} url - The URL/identifier of the Concept Map - * @param {string} version - The version of the Concept Map - * @returns {Promise} The requested Concept Map - * @throws {Error} Must be implemented by subclasses - */ - // eslint-disable-next-line no-unused-vars async fetchConceptMap(url, version) { - throw new Error('fetchConceptMap must be implemented by subclass'); + this._validateFetchParams(url, version); + + const direct = this.conceptMapMap.get(`${url}|${version}`) || this.conceptMapMap.get(url); + if (direct) { + return direct; + } + + const mappingId = this.#extractMappingId(url); + if (mappingId) { + return await this.fetchConceptMapById(mappingId); + } + + const mappings = await this.#searchMappings({ from_source_url: url }, this.maxSearchPages); + for (const mapping of mappings) { + const cm = this.#toConceptMap(mapping); + if (cm) { + this.#indexConceptMap(cm); + if (cm.url === url && (!version || cm.version === version)) { + return cm; + } + } + } + + return null; } - /** - * Fetches a specific Concept Map by id. ConceptMap providers must enforce that Concept Map ids are unique - * either globally (as enforced by assignIds) or in their space - * - * @param {string} id - The id of the Concept Map - * @returns {Promise} The requested Concept Map - * @throws {Error} Must be implemented by subclasses - */ - // eslint-disable-next-line no-unused-vars async fetchConceptMapById(id) { - throw new Error('fetchConceptMapById must be implemented by subclass'); + if (this._idMap.has(id)) { + return this._idMap.get(id); + } + + let rawId = id; + if (this.spaceId && id.startsWith(`${this.spaceId}-`)) { + rawId = id.substring(this.spaceId.length + 1); + } + + if (this._idMap.has(rawId)) { + return this._idMap.get(rawId); + } + + const response = await this.httpClient.get(`/mappings/${encodeURIComponent(rawId)}/`); + const cm = this.#toConceptMap(response.data); + if (!cm) { + return null; + } + this.#indexConceptMap(cm); + return cm; } - /** - * Searches for Concept Maps based on provided criteria - * @param {Array<{name: string, value: string}>} searchParams - List of name/value pairs for search criteria - * @returns {Promise>} List of matching Concept Maps - * @throws {Error} Must be implemented by subclasses - */ // eslint-disable-next-line no-unused-vars - async searchConceptMaps(searchParams, elements = null) { - throw new Error('searchConceptMaps must be implemented by subclass'); + async searchConceptMaps(searchParams, _elements) { + this._validateSearchParams(searchParams); + + const params = Object.fromEntries(searchParams.map(({ name, value }) => [name, String(value).toLowerCase()])); + const oclParams = {}; + + if (params.source) { + oclParams.from_source_url = params.source; + } + if (params.target) { + oclParams.to_source_url = params.target; + } + + const mappings = await this.#searchMappings(oclParams, this.maxSearchPages); + const results = []; + for (const mapping of mappings) { + const cm = this.#toConceptMap(mapping); + if (!cm) { + continue; + } + this.#indexConceptMap(cm); + if (this.#matches(cm.jsonObj, params)) { + results.push(cm); + } + } + return results; } - /** - * Validates search parameters - * @param {Array<{name: string, value: string}>} searchParams - Search parameters to validate - * @protected - */ - _validateSearchParams(searchParams) { - if (!Array.isArray(searchParams)) { - throw new Error('Search parameters must be an array'); + async findConceptMapForTranslation(opContext, conceptMaps, sourceSystem, sourceScope, targetScope, targetSystem, sourceCode = null) { + const sourceCandidates = await this.#candidateSourceUrls(sourceSystem); + const targetCandidates = await this.#candidateSourceUrls(targetSystem); + + const mappings = []; + const sourcePaths = sourceCandidates.filter(s => String(s || '').startsWith('/orgs/')); + + if (sourceCode && sourcePaths.length > 0) { + for (const sourcePath of sourcePaths) { + const conceptPath = `${this.#normalizeSourcePath(sourcePath)}concepts/${encodeURIComponent(sourceCode)}/mappings/`; + const found = await this.#fetchAllPages(conceptPath, { limit: PAGE_SIZE }, Math.min(2, this.maxSearchPages)); + mappings.push(...found); + } } - for (const param of searchParams) { - if (!param || typeof param !== 'object') { - throw new Error('Each search parameter must be an object'); + const searchKeys = new Set(); + const searches = []; + + if (sourceCandidates.length === 0 && targetCandidates.length === 0) { + searches.push({}); + } else if (targetCandidates.length === 0) { + for (const src of sourceCandidates) { + const key = `from:${src}`; + if (!searchKeys.has(key)) { + searchKeys.add(key); + searches.push({ from_source_url: src }); + } + } + } else if (sourceCandidates.length === 0) { + for (const tgt of targetCandidates) { + const key = `to:${tgt}`; + if (!searchKeys.has(key)) { + searchKeys.add(key); + searches.push({ to_source_url: tgt }); + } + } + } else { + for (const src of sourceCandidates) { + for (const tgt of targetCandidates) { + const key = `from:${src}|to:${tgt}`; + if (!searchKeys.has(key)) { + searchKeys.add(key); + searches.push({ from_source_url: src, to_source_url: tgt }); + } + } + } + } + + if (mappings.length === 0) { + for (const search of searches) { + const found = await this.#searchMappings(search, Math.min(2, this.maxSearchPages)); + mappings.push(...found); + } + } + + const sourceUrlsToResolve = new Set(); + for (const mapping of mappings) { + const fromSource = mapping?.from_source_url || mapping?.fromSourceUrl; + const toSource = mapping?.to_source_url || mapping?.toSourceUrl; + if (fromSource) { + sourceUrlsToResolve.add(fromSource); + } + if (toSource) { + sourceUrlsToResolve.add(toSource); + } + } + await this.#ensureCanonicalForSourceUrls(sourceUrlsToResolve); + + const seen = new Set(conceptMaps.map(cm => cm.id || cm.url)); + for (const mapping of mappings) { + const cm = this.#toConceptMap(mapping); + if (!cm) { + continue; + } + this.#indexConceptMap(cm); + + const key = cm.id || cm.url; + if (seen.has(key)) { + continue; } - if (typeof param.name !== 'string' || typeof param.value !== 'string') { - throw new Error('Search parameter must have string name and value properties'); + + if (this.#matchesTranslationRequest(cm, sourceSystem, sourceScope, targetScope, targetSystem, sourceCandidates, targetCandidates)) { + conceptMaps.push(cm); + seen.add(key); } } } - /** - * Validates URL and version parameters - * @param {string} url - URL to validate - * @param {string} version - Version to validate - * @protected - */ - _validateFetchParams(url, version) { - if (typeof url !== 'string' || !url.trim()) { - throw new Error('URL must be a non-empty string'); + cmCount() { + return new Set(this.conceptMapMap.values()).size; + } + + async close() { + } + + #indexConceptMap(cm) { + this.conceptMapMap.set(cm.url, cm); + if (cm.version) { + this.conceptMapMap.set(`${cm.url}|${cm.version}`, cm); } - if (version != null && typeof version !== 'string') { - throw new Error('Version must be a string'); + this.conceptMapMap.set(cm.id, cm); + this._idMap.set(cm.id, cm); + } + + #toConceptMap(mapping) { + if (!mapping || typeof mapping !== 'object') { + return null; + } + + const id = mapping.id; + if (!id) { + return null; + } + + const url = mapping.url || `${this.baseUrl}/mappings/${id}`; + const source = mapping.from_source_url || mapping.fromSourceUrl || mapping.from_concept_url || mapping.fromConceptUrl || null; + const target = mapping.to_source_url || mapping.toSourceUrl || mapping.to_concept_url || mapping.toConceptUrl || null; + const sourceCode = mapping.from_concept_code || mapping.fromConceptCode; + const targetCode = mapping.to_concept_code || mapping.toConceptCode; + + if (!source || !target || !sourceCode || !targetCode) { + return null; + } + + const sourceDisplay = mapping.from_concept_name_resolved || mapping.fromConceptNameResolved || mapping.from_concept_name || mapping.fromConceptName || null; + const targetDisplay = mapping.to_concept_name_resolved || mapping.toConceptNameResolved || mapping.to_concept_name || mapping.toConceptName || null; + const sourceCanonical = this.#canonicalForSourceUrl(source) || source; + const targetCanonical = this.#canonicalForSourceUrl(target) || target; + + const relationship = this.#toRelationship(mapping.map_type || mapping.mapType); + const lastUpdated = this.#toIsoDate(mapping.updated_on || mapping.updatedOn || mapping.updated_at || mapping.updatedAt); + + const json = { + resourceType: 'ConceptMap', + id, + url, + version: mapping.version || null, + name: `mapping-${id}`, + title: mapping.name || `Mapping ${id}`, + status: 'active', + sourceScopeUri: mapping.from_collection_url || mapping.fromCollectionUrl || source, + targetScopeUri: mapping.to_collection_url || mapping.toCollectionUrl || target, + group: [ + { + source: sourceCanonical, + target: targetCanonical, + element: [ + { + code: sourceCode, + display: sourceDisplay, + target: [ + { + code: targetCode, + display: targetDisplay, + relationship, + comment: mapping.comment || null + } + ] + } + ] + } + ] + }; + + if (lastUpdated) { + json.meta = { lastUpdated }; } + + return new ConceptMap(json, 'R5'); } - // eslint-disable-next-line no-unused-vars - async findConceptMapForTranslation(opContext, conceptMaps, sourceSystem, sourceScope, targetScope, targetSystem) { - // nothing + #toRelationship(mapType) { + switch ((mapType || '').toUpperCase()) { + case 'SAME-AS': + return 'equivalent'; + case 'NARROWER-THAN': + return 'narrower-than'; + case 'BROADER-THAN': + return 'broader-than'; + case 'NOT-EQUIVALENT': + return 'not-related-to'; + default: + return 'related-to'; + } } - cmCount() { - return 0; + #matches(json, params) { + for (const [name, value] of Object.entries(params)) { + if (!value) { + continue; + } + + if (name === 'url') { + if ((json.url || '').toLowerCase() !== value) { + return false; + } + continue; + } + + if (name === 'source') { + const src = json.group?.[0]?.source || ''; + if (!src.toLowerCase().includes(value)) { + return false; + } + continue; + } + + if (name === 'target') { + const tgt = json.group?.[0]?.target || ''; + if (!tgt.toLowerCase().includes(value)) { + return false; + } + continue; + } + + const field = json[name]; + if (field == null || !String(field).toLowerCase().includes(value)) { + return false; + } + } + return true; + } + + async #searchMappings(params = {}, maxPages = this.maxSearchPages) { + const endpoint = this.org ? `/orgs/${encodeURIComponent(this.org)}/mappings/` : '/mappings/'; + return await this.#fetchAllPages(endpoint, params, maxPages); + } + + async #fetchAllPages(path, params = {}, maxPages = this.maxSearchPages) { + const results = []; + let page = 1; + let nextPath = path; + let pageCount = 0; + let usePageMode = true; + + while (nextPath && pageCount < maxPages) { + const response = usePageMode + ? await this.httpClient.get(path, { params: { ...params, page, limit: PAGE_SIZE } }) + : await this.httpClient.get(nextPath); + + if (Array.isArray(response.data)) { + results.push(...response.data); + pageCount += 1; + if (response.data.length < PAGE_SIZE) { + break; + } + page += 1; + nextPath = path; + continue; + } + + const { items, next } = this.#extractItemsAndNext(response.data); + results.push(...items); + pageCount += 1; + + if (next) { + usePageMode = false; + nextPath = next; + continue; + } + + if (usePageMode && items.length >= PAGE_SIZE && pageCount < maxPages) { + page += 1; + nextPath = path; + } else { + break; + } + } + + return results; + } + + #extractItemsAndNext(payload) { + if (Array.isArray(payload)) { + return { items: payload, next: null }; + } + + if (!payload || typeof payload !== 'object') { + return { items: [], next: null }; + } + + const items = Array.isArray(payload.results) + ? payload.results + : Array.isArray(payload.items) + ? payload.items + : Array.isArray(payload.data) + ? payload.data + : []; + + const next = payload.next || null; + if (!next) { + return { items, next: null }; + } + + if (next.startsWith(this.baseUrl)) { + return { items, next: next.replace(this.baseUrl, '') }; + } + + return { items, next }; + } + + #extractMappingId(url) { + if (!url) { + return null; + } + const match = url.match(/\/mappings\/([^/]+)\/?$/i); + return match ? match[1] : null; + } + + async #candidateSourceUrls(systemUrl) { + if (!systemUrl) { + return []; + } + + const cacheKey = this.#norm(systemUrl); + if (this._sourceCandidatesCache.has(cacheKey)) { + return this._sourceCandidatesCache.get(cacheKey); + } + + const result = new Set(); + result.add(systemUrl); + + const canonicalKey = cacheKey; + const byCanonical = this._sourceUrlsByCanonical.get(canonicalKey); + if (byCanonical) { + for (const item of byCanonical) { + result.add(item); + } + } + + const discovered = await this.#resolveSourceCandidatesFromOcl(systemUrl); + for (const item of discovered) { + result.add(item); + } + + const out = Array.from(result); + this._sourceCandidatesCache.set(cacheKey, out); + return out; + } + + async #resolveSourceCandidatesFromOcl(systemUrl) { + const endpoint = this.org ? `/orgs/${encodeURIComponent(this.org)}/sources/` : '/sources/'; + const query = this.#queryTokenFromSystem(systemUrl); + if (!query) { + return []; + } + + const sources = await this.#fetchAllPages(endpoint, { q: query, limit: PAGE_SIZE }, 2); + const targetNorm = this.#norm(systemUrl); + const candidates = new Set(); + + for (const source of sources) { + const canonical = source?.canonical_url || source?.canonicalUrl || null; + const sourceUrl = source?.url || source?.uri || null; + if (!sourceUrl) { + continue; + } + + if (canonical) { + const canonicalKey = this.#norm(canonical); + if (!this._sourceUrlsByCanonical.has(canonicalKey)) { + this._sourceUrlsByCanonical.set(canonicalKey, new Set()); + } + this._sourceUrlsByCanonical.get(canonicalKey).add(sourceUrl); + this._canonicalBySourceUrl.set(this.#norm(sourceUrl), canonical); + + if (canonicalKey === targetNorm) { + candidates.add(sourceUrl); + } + } + + if (this.#norm(sourceUrl) === targetNorm) { + candidates.add(sourceUrl); + } + } + + return Array.from(candidates); + } + + async #ensureCanonicalForSourceUrls(sourceUrls) { + for (const sourceUrl of sourceUrls || []) { + const sourceKey = this.#norm(sourceUrl); + if (!sourceKey || this._canonicalBySourceUrl.has(sourceKey)) { + continue; + } + + const sourcePath = String(sourceUrl || '').trim(); + if (!sourcePath.startsWith('/orgs/')) { + continue; + } + + try { + const response = await this.httpClient.get(sourcePath); + const source = response.data || {}; + const canonical = source.canonical_url || source.canonicalUrl; + const resolvedSourceUrl = source.url || source.uri || sourcePath; + if (!canonical) { + continue; + } + + const canonicalKey = this.#norm(canonical); + if (!this._sourceUrlsByCanonical.has(canonicalKey)) { + this._sourceUrlsByCanonical.set(canonicalKey, new Set()); + } + this._sourceUrlsByCanonical.get(canonicalKey).add(resolvedSourceUrl); + this._canonicalBySourceUrl.set(this.#norm(resolvedSourceUrl), canonical); + } catch (e) { + } + } + } + + #queryTokenFromSystem(systemUrl) { + const raw = String(systemUrl || '').trim().replace(/\/+$/, ''); + if (!raw) { + return null; + } + const slash = raw.lastIndexOf('/'); + if (slash >= 0 && slash < raw.length - 1) { + return raw.substring(slash + 1); + } + return raw; + } + + #normalizeSourcePath(sourcePath) { + const path = String(sourcePath || '').trim(); + if (!path) { + return '/'; + } + return path.endsWith('/') ? path : `${path}/`; + } + + #canonicalForSourceUrl(sourceUrl) { + if (!sourceUrl) { + return null; + } + return this._canonicalBySourceUrl.get(this.#norm(sourceUrl)) || null; + } + + #matchesTranslationRequest(cm, sourceSystem, sourceScope, targetScope, targetSystem, sourceCandidates, targetCandidates) { + if (cm.providesTranslation(sourceSystem, sourceScope, targetScope, targetSystem)) { + return true; + } + + const group = cm.jsonObj?.group?.[0] || {}; + const groupSource = this.#norm(group.source); + const groupTarget = this.#norm(group.target); + + const sourceOk = !sourceSystem || sourceCandidates.some(s => this.#norm(s) === groupSource); + const targetOk = !targetSystem || targetCandidates.some(s => this.#norm(s) === groupTarget); + return sourceOk && targetOk; + } + + #norm(url) { + return String(url || '').trim().replace(/\/+$/, '').toLowerCase(); + } + + #toIsoDate(value) { + if (!value) { + return null; + } + const date = new Date(value); + if (Number.isNaN(date.getTime())) { + return null; + } + return date.toISOString(); } } module.exports = { - AbstractConceptMapProvider + OCLConceptMapProvider }; \ No newline at end of file diff --git a/tx/ocl/cs-ocl.js b/tx/ocl/cs-ocl.js index 6e008525..a4979f48 100644 --- a/tx/ocl/cs-ocl.js +++ b/tx/ocl/cs-ocl.js @@ -1,39 +1,2285 @@ -/** - * Abstract base class for value set providers - * Defines the interface that all value set providers must implement - */ -// eslint-disable-next-line no-unused-vars -class OCLCodeSystemProvider { - /** - * {int} Unique number assigned to this provider - */ - spaceId; - - /** - * ensure that the ids on the code systems are unique, if they are - * in the global namespace - * - * @param {Set} ids - */ - // eslint-disable-next-line no-unused-vars +const axios = require('axios'); +const fs = require('fs/promises'); +const fsSync = require('fs'); +const crypto = require('crypto'); +const path = require('path'); +const { AbstractCodeSystemProvider } = require('../cs/cs-provider-api'); +const { CodeSystemProvider, CodeSystemFactoryProvider, CodeSystemContentMode, FilterExecutionContext } = require('../cs/cs-api'); +const { CodeSystem } = require('../library/codesystem'); +const { SearchFilterText } = require('../library/designations'); + +const DEFAULT_BASE_URL = 'https://oclapi2.ips.hsl.org.br'; +const PAGE_SIZE = 100; +const CONCEPT_PAGE_SIZE = 1000; +const COLD_CACHE_FRESHNESS_MS = 60 * 60 * 1000; +const OCL_CODESYSTEM_MARKER_EXTENSION = 'http://fhir.org/FHIRsmith/StructureDefinition/ocl-codesystem'; +const OCL_SEARCH_PATCH_FLAG = Symbol.for('fhirsmith.ocl.search.codesystem.code.patch'); + +function hasOCLCodeSystemMarker(resource) { + const extensions = Array.isArray(resource?.extension) ? resource.extension : []; + return extensions.some(ext => ext && ext.url === OCL_CODESYSTEM_MARKER_EXTENSION); +} + +function filterConceptTreeByCode(concepts, wantedCode) { + if (!Array.isArray(concepts) || concepts.length === 0) { + return []; + } + + const matches = []; + for (const concept of concepts) { + if (!concept || typeof concept !== 'object') { + continue; + } + + const childMatches = filterConceptTreeByCode(concept.concept, wantedCode); + const isSelfMatch = concept.code != null && String(concept.code) === wantedCode; + if (!isSelfMatch && childMatches.length === 0) { + continue; + } + + const clone = { ...concept }; + if (childMatches.length > 0) { + clone.concept = childMatches; + } else { + delete clone.concept; + } + matches.push(clone); + } + + return matches; +} + +function filterOCLCodeSystemResourceByCode(resource, code) { + if (!resource || typeof resource !== 'object') { + return resource; + } + + const filteredConcepts = filterConceptTreeByCode(resource.concept, code); + return { + ...resource, + concept: filteredConcepts + }; +} + +function patchSearchWorkerForOCLCodeFiltering() { + let SearchWorker; + try { + SearchWorker = require('../workers/search'); + } catch (_error) { + return; + } + + if (!SearchWorker || !SearchWorker.prototype) { + return; + } + + const proto = SearchWorker.prototype; + if (proto[OCL_SEARCH_PATCH_FLAG] === true || typeof proto.searchCodeSystems !== 'function') { + return; + } + + const originalSearchCodeSystems = proto.searchCodeSystems; + proto.searchCodeSystems = function patchedSearchCodeSystems(params) { + const matches = originalSearchCodeSystems.call(this, params); + const requestedCode = params?.code == null ? '' : String(params.code); + + if (!requestedCode) { + return matches; + } + + const filtered = []; + for (const resource of matches) { + if (!hasOCLCodeSystemMarker(resource)) { + filtered.push(resource); + continue; + } + + const projected = filterOCLCodeSystemResourceByCode(resource, requestedCode); + if (Array.isArray(projected?.concept) && projected.concept.length > 0) { + filtered.push(projected); + } + } + + return filtered; + }; + + Object.defineProperty(proto, OCL_SEARCH_PATCH_FLAG, { + value: true, + writable: false, + configurable: false, + enumerable: false + }); +} + +patchSearchWorkerForOCLCodeFiltering(); + +// Cold cache configuration +const CACHE_BASE_DIR = path.join(process.cwd(), 'data', 'terminology-cache', 'ocl'); +const CACHE_CS_DIR = path.join(CACHE_BASE_DIR, 'codesystems'); +const CACHE_VS_DIR = path.join(CACHE_BASE_DIR, 'valuesets'); + +// Cache file utilities +async function ensureCacheDirectories() { + try { + await fs.mkdir(CACHE_CS_DIR, { recursive: true }); + await fs.mkdir(CACHE_VS_DIR, { recursive: true }); + } catch (error) { + console.error('[OCL] Failed to create cache directories:', error.message); + } +} + +// CodeSystem fingerprint computation +function computeCodeSystemFingerprint(concepts) { + if (!Array.isArray(concepts) || concepts.length === 0) { + return null; + } + + // Normalize concepts to deterministic strings + const normalized = concepts + .map(concept => { + if (!concept || !concept.code) { + return null; + } + const code = String(concept.code || ''); + const display = String(concept.display || ''); + const definition = String(concept.definition || ''); + const retired = concept.retired === true ? '1' : '0'; + return `${code}|${display}|${definition}|${retired}`; + }) + .filter(Boolean) + .sort(); + + // Compute SHA256 hash + const hash = crypto.createHash('sha256'); + for (const item of normalized) { + hash.update(item); + hash.update('\n'); + } + return hash.digest('hex'); +} + +function sanitizeFilename(text) { + if (!text || typeof text !== 'string') { + return 'unknown'; + } + return text + .replace(/[^a-zA-Z0-9._-]/g, '_') + .replace(/_+/g, '_') + .substring(0, 200); +} + +function getCacheFilePath(baseDir, canonicalUrl, version = null) { + const filename = sanitizeFilename(canonicalUrl) + (version ? `_${sanitizeFilename(version)}` : '') + '.json'; + return path.join(baseDir, filename); +} + +function getColdCacheAgeMs(cacheFilePath) { + try { + const stats = fsSync.statSync(cacheFilePath); + if (!stats || !Number.isFinite(stats.mtimeMs)) { + return null; + } + return Math.max(0, Date.now() - stats.mtimeMs); + } catch (error) { + if (error && error.code !== 'ENOENT') { + console.error(`[OCL] Failed to inspect cold cache file ${cacheFilePath}: ${error.message}`); + } + return null; + } +} + +function formatCacheAgeMinutes(ageMs) { + const minutes = Math.max(1, Math.round(ageMs / 60000)); + return `${minutes} minute${minutes === 1 ? '' : 's'}`; +} + +class OCLConceptFilterContext { + constructor() { + this.concepts = []; + this.currentIndex = -1; + } + + add(concept, rating = 0) { + this.concepts.push({ concept, rating }); + } + + sort() { + this.concepts.sort((a, b) => b.rating - a.rating); + } + + size() { + return this.concepts.length; + } + + hasMore() { + return this.currentIndex + 1 < this.concepts.length; + } + + next() { + if (!this.hasMore()) { + return null; + } + this.currentIndex += 1; + return this.concepts[this.currentIndex].concept; + } + + reset() { + this.currentIndex = -1; + } + + findConceptByCode(code) { + for (const item of this.concepts) { + if (item.concept && item.concept.code === code) { + return item.concept; + } + } + return null; + } + + containsConcept(concept) { + return this.concepts.some(item => item.concept === concept); + } +} + +class OCLBackgroundJobQueue { + static MAX_CONCURRENT = 2; + static HEARTBEAT_INTERVAL_MS = 30000; + static UNKNOWN_JOB_SIZE = Number.MAX_SAFE_INTEGER; + static pendingJobs = []; + static activeCount = 0; + static queuedOrRunningKeys = new Set(); + static activeJobs = new Map(); + static heartbeatTimer = null; + static enqueueSequence = 0; + + static enqueue(jobKey, jobType, runJob, options = {}) { + if (!jobKey || typeof runJob !== 'function') { + return false; + } + + if (this.queuedOrRunningKeys.has(jobKey)) { + return false; + } + + this.queuedOrRunningKeys.add(jobKey); + const resolveAndEnqueue = async () => { + const resolvedSize = await this.#resolveJobSize(options); + const normalizedSize = this.#normalizeJobSize(resolvedSize); + this.#insertPendingJobOrdered({ + jobKey, + jobType: jobType || 'background-job', + jobId: options?.jobId || jobKey, + jobSize: normalizedSize, + getProgress: typeof options?.getProgress === 'function' ? options.getProgress : null, + runJob, + enqueueOrder: this.enqueueSequence++ + }); + this.ensureHeartbeatRunning(); + console.log(`[OCL] ${jobType || 'Background job'} enqueued: ${jobKey} (size=${normalizedSize}, queue=${this.pendingJobs.length}, active=${this.activeCount})`); + this.processNext(); + }; + + Promise.resolve() + .then(resolveAndEnqueue) + .catch((error) => { + this.queuedOrRunningKeys.delete(jobKey); + const message = error && error.message ? error.message : String(error); + console.error(`[OCL] Failed to enqueue background job: ${jobType || 'background-job'} ${jobKey}: ${message}`); + }); + + return true; + } + + static async #resolveJobSize(options = {}) { + if (typeof options?.resolveJobSize === 'function') { + try { + return await options.resolveJobSize(); + } catch (_error) { + return this.UNKNOWN_JOB_SIZE; + } + } + + if (options && Object.prototype.hasOwnProperty.call(options, 'jobSize')) { + return options.jobSize; + } + + return this.UNKNOWN_JOB_SIZE; + } + + static #normalizeJobSize(jobSize) { + const parsed = Number.parseInt(jobSize, 10); + if (!Number.isFinite(parsed) || parsed < 0) { + return this.UNKNOWN_JOB_SIZE; + } + return parsed; + } + + static #insertPendingJobOrdered(job) { + let index = this.pendingJobs.findIndex(existing => { + if (existing.jobSize === job.jobSize) { + return existing.enqueueOrder > job.enqueueOrder; + } + return existing.jobSize > job.jobSize; + }); + + if (index < 0) { + index = this.pendingJobs.length; + } + + this.pendingJobs.splice(index, 0, job); + } + + static isQueuedOrRunning(jobKey) { + return this.queuedOrRunningKeys.has(jobKey); + } + + static ensureHeartbeatRunning() { + if (this.heartbeatTimer) { + return; + } + + this.heartbeatTimer = setInterval(() => { + this.logHeartbeat(); + }, this.HEARTBEAT_INTERVAL_MS); + + // Do not keep the process alive only for telemetry logs. + if (typeof this.heartbeatTimer.unref === 'function') { + this.heartbeatTimer.unref(); + } + } + + static logHeartbeat() { + const activeJobs = Array.from(this.activeJobs.values()); + const lines = [ + '[OCL] OCL background status:', + ` active jobs: ${activeJobs.length}`, + ` queued jobs: ${this.pendingJobs.length}` + ]; + + activeJobs.forEach((job, index) => { + lines.push(''); + lines.push(` job ${index + 1}:`); + lines.push(` type: ${job.jobType || 'background-job'}`); + lines.push(` id: ${job.jobId || job.jobKey}`); + lines.push(` size: ${job.jobSize}`); + lines.push(` progress: ${this.formatProgress(job.getProgress)}`); + }); + + console.log(lines.join('\n')); + } + + static formatProgress(getProgress) { + if (typeof getProgress !== 'function') { + return 'unknown'; + } + + try { + const progress = getProgress(); + if (typeof progress === 'number' && Number.isFinite(progress)) { + const bounded = Math.max(0, Math.min(100, progress)); + return `${Math.round(bounded)}%`; + } + + if (progress && typeof progress === 'object') { + if (typeof progress.percentage === 'number' && Number.isFinite(progress.percentage)) { + const bounded = Math.max(0, Math.min(100, progress.percentage)); + return `${Math.round(bounded)}%`; + } + + if ( + typeof progress.processed === 'number' && + Number.isFinite(progress.processed) && + typeof progress.total === 'number' && + Number.isFinite(progress.total) && + progress.total > 0 + ) { + const ratio = progress.processed / progress.total; + const bounded = Math.max(0, Math.min(100, ratio * 100)); + return `${Math.round(bounded)}%`; + } + } + } catch (error) { + return 'unknown'; + } + + return 'unknown'; + } + + static processNext() { + while (this.activeCount < this.MAX_CONCURRENT && this.pendingJobs.length > 0) { + const job = this.pendingJobs.shift(); + this.activeCount += 1; + this.activeJobs.set(job.jobKey, { + jobKey: job.jobKey, + jobType: job.jobType, + jobId: job.jobId || job.jobKey, + jobSize: job.jobSize, + getProgress: job.getProgress || null, + startedAt: Date.now() + }); + console.log(`[OCL] Background job started: ${job.jobType} ${job.jobKey} (size=${job.jobSize}, queue=${this.pendingJobs.length}, active=${this.activeCount})`); + + Promise.resolve() + .then(() => job.runJob()) + .then(() => { + console.log(`[OCL] Background job completed: ${job.jobType} ${job.jobKey}`); + }) + .catch((error) => { + const message = error && error.message ? error.message : String(error); + console.error(`[OCL] Background job failed: ${job.jobType} ${job.jobKey}: ${message}`); + }) + .finally(() => { + this.activeCount -= 1; + this.queuedOrRunningKeys.delete(job.jobKey); + this.activeJobs.delete(job.jobKey); + console.log(`[OCL] Background queue status: queue=${this.pendingJobs.length}, active=${this.activeCount}`); + this.processNext(); + }); + } + } +} + +class OCLCodeSystemProvider extends AbstractCodeSystemProvider { + constructor(config = {}) { + super(); + const options = typeof config === 'string' ? { baseUrl: config } : (config || {}); + + this.baseUrl = (options.baseUrl || DEFAULT_BASE_URL).replace(/\/+$/, ''); + this.org = options.org || null; + + const headers = { + Accept: 'application/json', + 'User-Agent': 'FHIRSmith-OCL-Provider/1.0' + }; + + if (options.token) { + headers.Authorization = options.token.startsWith('Token ') || options.token.startsWith('Bearer ') + ? options.token + : `Token ${options.token}`; + } + + this.httpClient = axios.create({ + baseURL: this.baseUrl, + timeout: options.timeout || 30000, + headers + }); + + this._codeSystemsByCanonical = new Map(); + this._idToCodeSystem = new Map(); + this.sourceMetaByUrl = new Map(); + this._sourceStateByCanonical = new Map(); + this._usedIds = new Set(); + this._refreshPromise = null; + this._pendingChanges = null; + this._initialized = false; + this._initializePromise = null; + } + + async initialize() { + if (this._initialized) { + return; + } + + if (this._initializePromise) { + await this._initializePromise; + return; + } + + this._initializePromise = (async () => { + try { + const sources = await this.#fetchSourcesForDiscovery(); + console.log(`[OCL] Fetched ${sources.length} sources`); + + const snapshot = this.#buildSourceSnapshot(sources); + this.#applySnapshot(snapshot); + + console.log(`[OCL] Loaded ${this._codeSystemsByCanonical.size} code systems`); + this._initialized = true; + } catch (error) { + console.error(`[OCL] Initialization failed:`, error.message); + if (error.response) { + console.error(`[OCL] HTTP ${error.response.status}: ${error.response.statusText}`); + } + throw error; + } + })(); + + try { + await this._initializePromise; + } finally { + this._initializePromise = null; + } + } + assignIds(ids) { - throw new Error('assignIds must be implemented by subclass'); + this._usedIds.clear(); + for (const cs of this._idToCodeSystem.values()) { + if (!cs.id || ids.has(`CodeSystem/${cs.id}`)) { + cs.id = String(ids.size); + cs.jsonObj.id = cs.id; + } + ids.add(`CodeSystem/${cs.id}`); + this._usedIds.add(cs.id); + this._idToCodeSystem.set(cs.id, cs); + } + } + + // eslint-disable-next-line no-unused-vars + async listCodeSystems(_fhirVersion, _context) { + await this.initialize(); + return Array.from(this._codeSystemsByCanonical.values()); } - /** - * Returns the list of CodeSystems this provider provides - * - * @param {string} fhirVersion - The FHIRVersion in scope - if relevant (there's always a stated version, though R5 is always used) - * @param {string} context - The client's stated context - if provided. - * @returns {Map} The list of CodeSystems - * @throws {Error} Must be implemented by subclasses - */ // eslint-disable-next-line no-unused-vars - async listCodeSystems(fhirVersion, context) { - throw new Error('listCodeSystems must be implemented by AbstractCodeSystemProvider subclass'); + async loadCodeSystems(fhirVersion, context) { + return await this.listCodeSystems(fhirVersion, context); + } + + // Called once per minute by provider.updateCodeSystemList(). + // That caller is currently sync, so we stage async fetches and return the latest ready diff. + // eslint-disable-next-line no-unused-vars + getCodeSystemChanges(_fhirVersion, _context) { + if (!this._initialized) { + return this.#emptyChanges(); + } + + this.#scheduleRefresh(); + if (!this._pendingChanges) { + return this.#emptyChanges(); + } + + const out = this._pendingChanges; + this._pendingChanges = null; + return out; + } + + async close() { + } + + getSourceMetas() { + return Array.from(this.sourceMetaByUrl.values()); + } + + #scheduleRefresh() { + if (this._refreshPromise) { + return; + } + + this._refreshPromise = (async () => { + try { + const sources = await this.#fetchSourcesForDiscovery(); + const nextSnapshot = this.#buildSourceSnapshot(sources); + const changes = this.#diffSnapshots(this._sourceStateByCanonical, nextSnapshot); + this.#applySnapshot(nextSnapshot); + this._pendingChanges = changes; + } catch (error) { + console.error('[OCL] Incremental source refresh failed:', error.message); + this._pendingChanges = this.#emptyChanges(); + } finally { + this._refreshPromise = null; + } + })(); + } + + #emptyChanges() { + return { added: [], changed: [], deleted: [] }; + } + + #buildSourceSnapshot(sources) { + const snapshot = new Map(); + for (const source of sources || []) { + const cs = this.#toCodeSystem(source); + if (!cs) { + continue; + } + + const canonicalUrl = cs.url; + const meta = this.#buildSourceMeta(source, cs); + const checksum = this.#sourceChecksum(source); + snapshot.set(canonicalUrl, { cs, meta, checksum }); + } + return snapshot; + } + + async #fetchSourcesForDiscovery() { + const organizations = await this.#fetchOrganizationIds(); + if (organizations.length === 0) { + // Fallback for OCL instances that expose global listing but not org listing. + return await this.#fetchAllPages('/sources/'); + } + + const allSources = []; + const seen = new Set(); + + for (const orgId of organizations) { + const endpoint = `/orgs/${encodeURIComponent(orgId)}/sources/`; + const sources = await this.#fetchAllPages(endpoint); + for (const source of sources) { + if (!source || typeof source !== 'object') { + continue; + } + const key = this.#sourceIdentity(source); + if (seen.has(key)) { + continue; + } + seen.add(key); + allSources.push(source); + } + } + + return allSources; + } + + async #fetchOrganizationIds() { + const endpoint = '/orgs/'; + const orgs = await this.#fetchAllPages(endpoint); + + const ids = []; + const seen = new Set(); + for (const org of orgs || []) { + if (!org || typeof org !== 'object') { + continue; + } + + const id = org.id || org.mnemonic || org.short_code || org.shortCode || org.name || null; + if (!id) { + continue; + } + + const normalized = String(id).trim(); + if (!normalized || seen.has(normalized)) { + continue; + } + + seen.add(normalized); + ids.push(normalized); + } + + if (ids.length === 0 && this.org) { + ids.push(this.org); + } + + return ids; + } + + #sourceIdentity(source) { + if (!source || typeof source !== 'object') { + return '__invalid__'; + } + + const owner = source.owner || ''; + const canonical = source.canonical_url || source.canonicalUrl || ''; + const shortCode = source.short_code || source.shortCode || source.id || source.mnemonic || source.name || ''; + return `${owner}|${canonical}|${shortCode}`; + } + + #applySnapshot(snapshot) { + const previousSnapshot = this._sourceStateByCanonical; + this._codeSystemsByCanonical.clear(); + this._idToCodeSystem.clear(); + this.sourceMetaByUrl.clear(); + this._usedIds.clear(); + + for (const [canonicalUrl, entry] of snapshot.entries()) { + const cs = entry.cs; + const meta = entry.meta; + const previousEntry = previousSnapshot.get(canonicalUrl); + + // Preserve complete-content marker if checksum did not change. + if (previousEntry && previousEntry.checksum === entry.checksum && previousEntry.cs?.jsonObj?.content === CodeSystemContentMode.Complete) { + cs.jsonObj.content = CodeSystemContentMode.Complete; + + // Preserve materialized concepts across metadata refreshes. + if (Array.isArray(previousEntry.cs?.jsonObj?.concept)) { + cs.jsonObj.concept = previousEntry.cs.jsonObj.concept.map(concept => ({ ...concept })); + } + } + + // If a factory already has warm/cold concepts for this system, project them to the new snapshot resource. + OCLSourceCodeSystemFactory.syncCodeSystemResource(canonicalUrl, cs.version || null, cs); + + this.#trackCodeSystemId(cs); + this._codeSystemsByCanonical.set(canonicalUrl, cs); + this._idToCodeSystem.set(cs.id, cs); + this.sourceMetaByUrl.set(canonicalUrl, meta); + } + + this._sourceStateByCanonical = snapshot; + } + + #diffSnapshots(previousSnapshot, nextSnapshot) { + const added = []; + const changed = []; + const deleted = []; + + for (const [canonicalUrl, nextEntry] of nextSnapshot.entries()) { + const previousEntry = previousSnapshot.get(canonicalUrl); + if (!previousEntry) { + added.push(nextEntry.cs); + continue; + } + + // Keep stable ids across revisions so clients don't observe resource id churn. + nextEntry.cs.id = previousEntry.cs.id; + nextEntry.cs.jsonObj.id = previousEntry.cs.id; + + const previousChecksum = previousEntry.checksum || null; + const nextChecksum = nextEntry.checksum || null; + const checksumChanged = previousChecksum !== nextChecksum; + const versionChanged = (previousEntry.cs.version || null) !== (nextEntry.cs.version || null); + if (checksumChanged || versionChanged) { + if (checksumChanged) { + console.log(`[OCL] CodeSystem checksum changed: ${canonicalUrl} (${previousChecksum} -> ${nextChecksum})`); + } + changed.push(nextEntry.cs); + } + } + + for (const [canonicalUrl, previousEntry] of previousSnapshot.entries()) { + if (!nextSnapshot.has(canonicalUrl)) { + deleted.push(previousEntry.cs); + } + } + + return { added, changed, deleted }; + } + + #trackCodeSystemId(cs) { + if (!cs) { + return; + } + + if (!cs.id || this._usedIds.has(cs.id)) { + const raw = cs.id || cs.name || cs.url || 'ocl-cs'; + const base = this.spaceId ? `${this.spaceId}-${raw}` : String(raw); + let candidate = base; + let index = 1; + while (this._usedIds.has(candidate)) { + candidate = `${base}-${index}`; + index += 1; + } + cs.id = candidate; + cs.jsonObj.id = candidate; + } + + this._usedIds.add(cs.id); + } + + #toCodeSystem(source) { + if (!source || typeof source !== 'object') { + return null; + } + + const canonicalUrl = source.canonical_url || source.canonicalUrl || source.url; + if (!canonicalUrl) { + return null; + } + + const id = source.id || source.mnemonic; + if (!id) { + return null; + } + + const lastUpdated = this.#toIsoDate(source.updated_at || source.updatedAt || source.updated_on || source.updatedOn); + + const json = { + resourceType: 'CodeSystem', + id, + url: canonicalUrl, + version: source.version || null, + name: source.name || source.mnemonic || id, + title: source.full_name || source.fullName || source.name || source.mnemonic || id, + status: 'active', + experimental: source.experimental === true, + description: source.description || null, + publisher: source.owner || null, + caseSensitive: source.case_sensitive != null ? source.case_sensitive : (source.caseSensitive != null ? source.caseSensitive : true), + language: source.default_locale || source.defaultLocale || null, + filter: [ + { + code: 'code', + description: 'Match concept code', + operator: ['=', 'in', 'regex'], + value: 'code' + }, + { + code: 'display', + description: 'Match concept display text', + operator: ['=', 'in', 'regex'], + value: 'string' + }, + { + code: 'definition', + description: 'Match concept definition text', + operator: ['=', 'in', 'regex'], + value: 'string' + }, + { + code: 'inactive', + description: 'Match inactive (retired) status', + operator: ['=', 'in'], + value: 'boolean' + } + ], + property: [ + { + code: 'code', + uri: 'http://hl7.org/fhir/concept-properties#code', + description: 'Concept code', + type: 'code' + }, + { + code: 'display', + description: 'Concept display text', + type: 'string' + }, + { + code: 'definition', + description: 'Concept definition text', + type: 'string' + }, + { + code: 'inactive', + uri: 'http://hl7.org/fhir/concept-properties#status', + description: 'Whether concept is inactive (retired)', + type: 'boolean' + } + ], + extension: [ + { + url: OCL_CODESYSTEM_MARKER_EXTENSION, + valueBoolean: true + } + ], + content: 'not-present' + }; + + if (lastUpdated) { + json.meta = { lastUpdated }; + } + + return new CodeSystem(json, 'R5', true); + } + + #buildSourceMeta(source, cs) { + if (!source || !cs) { + return null; + } + + const owner = source.owner || null; + const shortCode = source.short_code || source.shortCode || source.mnemonic || source.id || null; + const canonicalUrl = cs.url; + if (!canonicalUrl) { + return; + } + + const conceptsUrl = this.#normalizePath(source.concepts_url || source.conceptsUrl || this.#buildConceptsPath(source)); + const meta = { + id: source.id || shortCode, + shortCode, + owner, + name: source.name || shortCode || cs.id, + description: source.description || null, + canonicalUrl, + version: source.version || null, + conceptsUrl, + checksum: this.#sourceChecksum(source), + codeSystem: cs + }; + + return meta; + } + + #sourceChecksum(source) { + // NOTE: OCL checksums are NOT reliable for cache invalidation decisions. + // They do not update when concepts are added or modified. + // This checksum is logged for debugging purposes only. + // Cache decisions are based on custom fingerprints computed from concept content. + + if (!source || typeof source !== 'object') { + return null; + } + + const checksums = source.checksums || {}; + const standard = checksums.standard || null; + const smart = checksums.smart || null; + if (standard) { + return String(standard); + } + if (smart) { + return String(smart); + } + + if (source.checksum) { + return String(source.checksum); + } + + const updated = source.updated_at || source.updatedAt || source.updated_on || source.updatedOn || null; + const version = source.version || null; + if (updated || version) { + return `${updated || ''}|${version || ''}`; + } + + return null; + } + + #buildConceptsPath(source) { + if (!source || typeof source !== 'object') { + return null; + } + const owner = source.owner || null; + const sourceId = source.short_code || source.shortCode || source.id || source.mnemonic || null; + if (!owner || !sourceId) { + const sourceUrl = source.url; + if (!sourceUrl || typeof sourceUrl !== 'string') { + return null; + } + const trimmed = sourceUrl.endsWith('/') ? sourceUrl : `${sourceUrl}/`; + return `${trimmed}concepts/`; + } + return `/orgs/${encodeURIComponent(owner)}/sources/${encodeURIComponent(sourceId)}/concepts/`; + } + + #normalizePath(pathValue) { + if (!pathValue) { + return null; + } + if (typeof pathValue !== 'string') { + return null; + } + if (pathValue.startsWith('http://') || pathValue.startsWith('https://')) { + return pathValue; + } + return `${this.baseUrl}${pathValue.startsWith('/') ? '' : '/'}${pathValue}`; + } + + async #fetchAllPages(path) { + const results = []; + let page = 1; + let nextPath = path; + let usePageMode = true; + + while (nextPath) { + try { + const response = usePageMode + ? await this.httpClient.get(path, { params: { page, limit: PAGE_SIZE } }) + : await this.httpClient.get(nextPath); + + if (Array.isArray(response.data)) { + results.push(...response.data); + if (response.data.length < PAGE_SIZE) { + break; + } + page += 1; + nextPath = path; + continue; + } + + const { items, next } = this.#extractItemsAndNext(response.data); + results.push(...items); + + if (next) { + usePageMode = false; + nextPath = next; + continue; + } + + if (usePageMode && items.length >= PAGE_SIZE) { + page += 1; + nextPath = path; + } else { + break; + } + } catch (error) { + console.error(`[OCL] Fetch error on page ${page}:`, error.message); + if (error.response) { + console.error(`[OCL] HTTP ${error.response.status}: ${error.response.statusText}`); + console.error(`[OCL] Response:`, error.response.data); + } + throw error; + } + } + + return results; + } + + #extractItemsAndNext(payload) { + if (Array.isArray(payload)) { + return { items: payload, next: null }; + } + + if (!payload || typeof payload !== 'object') { + return { items: [], next: null }; + } + + const items = Array.isArray(payload.results) + ? payload.results + : Array.isArray(payload.items) + ? payload.items + : Array.isArray(payload.data) + ? payload.data + : []; + + const next = payload.next || null; + if (!next) { + return { items, next: null }; + } + + if (next.startsWith(this.baseUrl)) { + return { items, next: next.replace(this.baseUrl, '') }; + } + + return { items, next }; + } + + #toIsoDate(value) { + if (!value) { + return null; + } + const date = new Date(value); + if (Number.isNaN(date.getTime())) { + return null; + } + return date.toISOString(); + } +} + +class OCLSourceCodeSystemProvider extends CodeSystemProvider { + constructor(opContext, supplements, client, meta, sharedCaches = null) { + super(opContext, supplements); + this.httpClient = client; + this.meta = meta; + this.conceptCache = sharedCaches?.conceptCache || new Map(); + this.pageCache = sharedCaches?.pageCache || new Map(); + this.pendingConceptRequests = sharedCaches?.pendingConceptRequests || new Map(); + this.pendingPageRequests = sharedCaches?.pendingPageRequests || new Map(); + this.scheduleBackgroundLoad = typeof sharedCaches?.scheduleBackgroundLoad === 'function' + ? sharedCaches.scheduleBackgroundLoad + : null; + this.isSystemComplete = typeof sharedCaches?.isSystemComplete === 'function' + ? sharedCaches.isSystemComplete + : (() => false); + this.getTotalConceptCount = typeof sharedCaches?.getTotalConceptCount === 'function' + ? sharedCaches.getTotalConceptCount + : (() => -1); + } + + system() { + return this.meta.canonicalUrl; + } + + version() { + return this.meta.version || null; + } + + description() { + return this.meta.description || null; + } + + name() { + return this.meta.name || this.meta.shortCode || this.meta.id || this.system(); + } + + contentMode() { + return this.isSystemComplete() ? CodeSystemContentMode.Complete : CodeSystemContentMode.NotPresent; + } + + totalCount() { + return this.getTotalConceptCount(); + } + + propertyDefinitions() { + return this.meta?.codeSystem?.jsonObj?.property || null; + } + + async code(code) { + const ctxt = await this.#ensureContext(code); + return ctxt ? ctxt.code : null; + } + + async display(code) { + const ctxt = await this.#ensureContext(code); + if (!ctxt) { + return null; + } + if (ctxt.display && this.opContext.langs.isEnglishOrNothing()) { + return ctxt.display; + } + const supp = this._displayFromSupplements(ctxt.code); + return supp || ctxt.display || null; + } + + async definition(code) { + const ctxt = await this.#ensureContext(code); + return ctxt ? ctxt.definition : null; + } + + async isAbstract(code) { + await this.#ensureContext(code); + return false; + } + + async isInactive(code) { + const ctxt = await this.#ensureContext(code); + return ctxt ? ctxt.retired === true : false; + } + + async isDeprecated(code) { + await this.#ensureContext(code); + return false; + } + + async getStatus(code) { + const ctxt = await this.#ensureContext(code); + if (!ctxt) { + return null; + } + return ctxt.retired === true ? 'inactive' : 'active'; + } + + async designations(code, displays) { + const ctxt = await this.#ensureContext(code); + if (ctxt && ctxt.display) { + const hasConceptDesignations = Array.isArray(ctxt.designations) && ctxt.designations.length > 0; + if (hasConceptDesignations) { + for (const d of ctxt.designations) { + if (!d || !d.value) { + continue; + } + displays.addDesignation(true, 'active', d.language || '', CodeSystem.makeUseForDisplay(), d.value); + } + } else { + displays.addDesignation(true, 'active', 'en', CodeSystem.makeUseForDisplay(), ctxt.display); + } + this._listSupplementDesignations(ctxt.code, displays); + } + } + + async locate(code) { + if (!code || typeof code !== 'string') { + return { context: null, message: 'Empty code' }; + } + + if (this.conceptCache.has(code)) { + return { context: this.conceptCache.get(code), message: null }; + } + + if (this.scheduleBackgroundLoad) { + this.scheduleBackgroundLoad('lookup-miss'); + } + + const concept = await this.#fetchConcept(code); + if (!concept) { + return { context: null, message: undefined }; + } + + this.conceptCache.set(code, concept); + return { context: concept, message: null }; + } + + async iterator(code) { + await this.#ensureContext(code); + if (code) { + return null; + } + return { + page: 1, + index: 0, + items: [], + total: -1, + done: false + }; + } + + async iteratorAll() { + return this.iterator(null); + } + + async getPrepContext(iterate) { + return new FilterExecutionContext(iterate); + } + + async doesFilter(prop, op, value) { + if (!prop || !op || value == null) { + return false; + } + + const normalizedProp = String(prop).trim().toLowerCase(); + const normalizedOp = String(op).trim().toLowerCase(); + const supportedOps = ['=', 'in', 'regex']; + if (!supportedOps.includes(normalizedOp)) { + return false; + } + + if (['concept', 'code', 'display', 'definition', 'inactive'].includes(normalizedProp)) { + return true; + } + + const defs = this.propertyDefinitions() || []; + return defs.some(def => def && def.code === normalizedProp); + } + + async searchFilter(filterContext, filter, sort) { + const matcher = this.#toSearchFilterText(filter); + const results = new OCLConceptFilterContext(); + const concepts = await this.#allConceptContexts(); + + for (const concept of concepts) { + const text = this.#conceptSearchText(concept); + const match = matcher.passes(text, true); + if (!match || match.passes !== true) { + continue; + } + + results.add(concept, this.#searchRating(concept, matcher, match.rating)); + } + + if (sort === true) { + results.sort(); + } + + if (!Array.isArray(filterContext.filters)) { + filterContext.filters = []; + } + filterContext.filters.push(results); + return filterContext; + } + + async filter(filterContext, prop, op, value) { + const normalizedProp = String(prop || '').trim().toLowerCase(); + const normalizedOp = String(op || '').trim().toLowerCase(); + + if (!await this.doesFilter(normalizedProp, normalizedOp, value)) { + throw new Error(`Filter ${prop} ${op} is not supported by OCL provider`); + } + + const set = new OCLConceptFilterContext(); + const concepts = await this.#allConceptContexts(); + const matcher = this.#buildPropertyMatcher(normalizedProp, normalizedOp, value); + + for (const concept of concepts) { + if (matcher(concept)) { + set.add(concept, 0); + } + } + + if (!Array.isArray(filterContext.filters)) { + filterContext.filters = []; + } + filterContext.filters.push(set); + return set; + } + + async executeFilters(filterContext) { + return Array.isArray(filterContext?.filters) ? filterContext.filters : []; + } + + // eslint-disable-next-line no-unused-vars + async filterSize(filterContext, set) { + return set ? set.size() : 0; + } + + // eslint-disable-next-line no-unused-vars + async filterMore(filterContext, set) { + return !!set && set.hasMore(); + } + + // eslint-disable-next-line no-unused-vars + async filterConcept(filterContext, set) { + if (!set) { + return null; + } + return set.next(); + } + + // eslint-disable-next-line no-unused-vars + async filterLocate(filterContext, set, code) { + if (!set) { + return `Code '${code}' not found: no filter results`; + } + const concept = set.findConceptByCode(code); + if (concept) { + return concept; + } + return null; + } + + // eslint-disable-next-line no-unused-vars + async filterCheck(filterContext, set, concept) { + if (!set || !concept) { + return false; + } + return set.containsConcept(concept); + } + + async filterFinish(filterContext) { + if (!Array.isArray(filterContext?.filters)) { + return; + } + for (const set of filterContext.filters) { + if (set && typeof set.reset === 'function') { + set.reset(); + } + } + filterContext.filters.length = 0; + } + + async nextContext(iteratorContext) { + if (!iteratorContext || iteratorContext.done) { + return null; + } + + if (iteratorContext.index >= iteratorContext.items.length) { + const pageItems = await this.#fetchConceptPage(iteratorContext.page); + iteratorContext.page += 1; + iteratorContext.index = 0; + iteratorContext.items = pageItems; + + if (!pageItems || pageItems.length === 0) { + iteratorContext.done = true; + return null; + } + } + + const concept = iteratorContext.items[iteratorContext.index]; + iteratorContext.index += 1; + return concept; + } + + async #ensureContext(code) { + if (!code) { + return null; + } + + // Some call paths pass a pending locate() Promise (or its wrapper result) + // instead of a raw code/context; normalize both shapes here. + if (code && typeof code === 'object' && typeof code.then === 'function') { + code = await code; + } + + if (code && typeof code === 'object' && Object.prototype.hasOwnProperty.call(code, 'context')) { + if (!code.context) { + throw new Error(code.message || 'Unknown code'); + } + code = code.context; + } + + if (typeof code === 'string') { + const result = await this.locate(code); + if (!result.context) { + throw new Error(result.message || `Unknown code ${code}`); + } + return result.context; + } + if (code && typeof code === 'object' && code.code) { + return code; + } + throw new Error(`Unknown Type at #ensureContext: ${typeof code}`); + } + + async #fetchConceptPage(page) { + if (!this.meta.conceptsUrl) { + return []; + } + const cacheKey = `${this.meta.conceptsUrl}|p=${page}|l=${CONCEPT_PAGE_SIZE}`; + if (this.pageCache.has(cacheKey)) { + const cached = this.pageCache.get(cacheKey); + return Array.isArray(cached) + ? cached + : Array.isArray(cached?.concepts) + ? cached.concepts + : []; + } + if (this.pendingPageRequests.has(cacheKey)) { + const pendingResult = await this.pendingPageRequests.get(cacheKey); + return Array.isArray(pendingResult) + ? pendingResult + : Array.isArray(pendingResult?.concepts) + ? pendingResult.concepts + : []; + } + + if (this.scheduleBackgroundLoad) { + this.scheduleBackgroundLoad('page-miss'); + } + + const pending = (async () => { + let response; + try { + response = await this.httpClient.get(this.meta.conceptsUrl, { params: { page, limit: CONCEPT_PAGE_SIZE } }); + } catch (error) { + // Some OCL instances return 404 for sources without concept listing endpoints. + // Treat this as an empty page so terminology operations degrade gracefully. + if (error && error.response && error.response.status === 404) { + this.pageCache.set(cacheKey, []); + return []; + } + throw error; + } + const payload = response.data; + const items = Array.isArray(payload) + ? payload + : Array.isArray(payload?.results) + ? payload.results + : Array.isArray(payload?.items) + ? payload.items + : Array.isArray(payload?.data) + ? payload.data + : []; + + const mapped = items.map(item => this.#toConceptContext(item)).filter(Boolean); + this.pageCache.set(cacheKey, mapped); + for (const concept of mapped) { + if (concept && concept.code) { + this.conceptCache.set(concept.code, concept); + } + } + return mapped; + })(); + + this.pendingPageRequests.set(cacheKey, pending); + try { + return await pending; + } finally { + this.pendingPageRequests.delete(cacheKey); + } + } + + async #fetchConcept(code) { + if (!this.meta.conceptsUrl) { + return null; + } + if (this.conceptCache.has(code)) { + return this.conceptCache.get(code); + } + const pendingKey = `${this.meta.conceptsUrl}|c=${code}`; + if (this.pendingConceptRequests.has(pendingKey)) { + return this.pendingConceptRequests.get(pendingKey); + } + + if (this.scheduleBackgroundLoad) { + this.scheduleBackgroundLoad('concept-miss'); + } + + const url = this.#buildConceptUrl(code); + const pending = (async () => { + let response; + try { + response = await this.httpClient.get(url); + } catch (error) { + // Missing concept should be treated as not-found, not as an internal server failure. + if (error && error.response && error.response.status === 404) { + return null; + } + throw error; + } + const concept = this.#toConceptContext(response.data); + if (concept && concept.code) { + this.conceptCache.set(concept.code, concept); + } + return concept; + })(); + + this.pendingConceptRequests.set(pendingKey, pending); + try { + return await pending; + } finally { + this.pendingConceptRequests.delete(pendingKey); + } + } + + async #allConceptContexts() { + const concepts = new Map(); + + for (const concept of this.conceptCache.values()) { + if (concept && concept.code) { + concepts.set(concept.code, concept); + } + } + + // Ensure search can operate even when content is not fully warm-loaded. + const iter = await this.iterator(null); + let concept = await this.nextContext(iter); + while (concept) { + if (concept.code && !concepts.has(concept.code)) { + concepts.set(concept.code, concept); + } + concept = await this.nextContext(iter); + } + + return Array.from(concepts.values()); + } + + #toSearchFilterText(filter) { + if (filter instanceof SearchFilterText) { + return filter; + } + if (typeof filter === 'string') { + return new SearchFilterText(filter); + } + if (filter && typeof filter.filter === 'string') { + return new SearchFilterText(filter.filter); + } + return new SearchFilterText(''); + } + + #conceptSearchText(concept) { + if (!concept || typeof concept !== 'object') { + return ''; + } + + const values = [concept.code, concept.display, concept.definition]; + if (Array.isArray(concept.designations)) { + for (const designation of concept.designations) { + if (designation && designation.value) { + values.push(designation.value); + } + } + } + + return values.filter(Boolean).join(' '); + } + + #searchRating(concept, matcher, baseRating) { + let rating = Number.isFinite(baseRating) ? baseRating : 0; + const term = matcher?.filter || ''; + if (!term) { + return rating; + } + + const code = String(concept?.code || '').toLowerCase(); + const display = String(concept?.display || '').toLowerCase(); + const definition = String(concept?.definition || '').toLowerCase(); + + if (code === term || display === term) { + rating += 100; + } else if (code.startsWith(term) || display.startsWith(term)) { + rating += 50; + } else if (definition.includes(term)) { + rating += 10; + } + + return rating; + } + + #buildPropertyMatcher(prop, op, value) { + if (op === 'regex') { + const regex = new RegExp(String(value), 'i'); + return concept => { + const candidate = this.#valueForFilter(concept, prop); + if (candidate == null) { + return false; + } + return regex.test(String(candidate)); + }; + } + + if (op === 'in') { + const tokens = String(value) + .split(',') + .map(token => token.trim().toLowerCase()) + .filter(Boolean); + return concept => { + const candidate = this.#valueForFilter(concept, prop); + if (candidate == null) { + return false; + } + return tokens.includes(String(candidate).toLowerCase()); + }; + } + + if (prop === 'inactive') { + const expected = this.#toBoolean(value); + return concept => { + const candidate = this.#toBoolean(this.#valueForFilter(concept, prop)); + return candidate === expected; + }; + } + + const expected = String(value).toLowerCase(); + return concept => { + const candidate = this.#valueForFilter(concept, prop); + if (candidate == null) { + return false; + } + return String(candidate).toLowerCase() === expected; + }; + } + + #valueForFilter(concept, prop) { + if (!concept || typeof concept !== 'object') { + return null; + } + + switch (prop) { + case 'concept': + case 'code': + return concept.code || null; + case 'display': + return concept.display || null; + case 'definition': + return concept.definition || null; + case 'inactive': + return concept.retired === true; + default: + return concept[prop] ?? null; + } + } + + #toBoolean(value) { + if (typeof value === 'boolean') { + return value; + } + + const text = String(value || '').trim().toLowerCase(); + return text === 'true' || text === '1' || text === 'yes'; + } + + #buildConceptUrl(code) { + const base = this.meta.conceptsUrl.endsWith('/') ? this.meta.conceptsUrl : `${this.meta.conceptsUrl}/`; + return `${base}${encodeURIComponent(code)}/`; + } + + #toConceptContext(concept) { + if (!concept || typeof concept !== 'object') { + return null; + } + + const code = concept.code || concept.id || null; + if (!code) { + return null; + } + + return { + code, + display: concept.display_name || concept.display || concept.name || null, + definition: concept.description || concept.definition || null, + retired: concept.retired === true, + designations: this.#extractDesignations(concept) + }; + } + + #extractDesignations(concept) { + const result = []; + const seen = new Set(); + + const add = (language, value) => { + const text = typeof value === 'string' ? value.trim() : ''; + if (!text) { + return; + } + const lang = typeof language === 'string' ? language.trim() : ''; + const key = `${lang}|${text}`; + if (seen.has(key)) { + return; + } + seen.add(key); + result.push({ language: lang, value: text }); + }; + + if (Array.isArray(concept.names)) { + for (const entry of concept.names) { + if (!entry || typeof entry !== 'object') { + continue; + } + add(entry.locale || entry.language || entry.lang || '', entry.name || entry.display_name || entry.display || entry.value || entry.term); + } + } + + if (concept.display_name || concept.display || concept.name) { + add(concept.locale || concept.default_locale || concept.language || '', concept.display_name || concept.display || concept.name); + } + + if (concept.locale_display_names && typeof concept.locale_display_names === 'object') { + for (const [lang, value] of Object.entries(concept.locale_display_names)) { + add(lang, value); + } + } + + return result; + } +} + +class OCLSourceCodeSystemFactory extends CodeSystemFactoryProvider { + static factoriesByKey = new Map(); + + static syncCodeSystemResource(system, version = null, codeSystem = null) { + if (!system) { + return; + } + + const key = `${system}|${version || ''}`; + const factory = OCLSourceCodeSystemFactory.factoriesByKey.get(key); + if (!factory) { + return; + } + + factory.#applyConceptsToCodeSystemResource(codeSystem || factory.meta?.codeSystem || null); + } + + constructor(i18n, client, meta) { + super(i18n); + this.httpClient = client; + this.meta = meta; + this.sharedConceptCache = new Map(); + this.sharedPageCache = new Map(); + this.sharedPendingConceptRequests = new Map(); + this.sharedPendingPageRequests = new Map(); + this.isComplete = meta?.codeSystem?.jsonObj?.content === CodeSystemContentMode.Complete; + this.loadedConceptCount = -1; + this.loadedChecksum = meta?.checksum || null; + this.customFingerprint = null; + this.backgroundLoadProgress = { processed: 0, total: null }; + this.materializedConceptList = null; + this.materializedConceptCount = -1; + OCLSourceCodeSystemFactory.factoriesByKey.set(this.#resourceKey(), this); + + // Load cold cache at construction + this.#loadColdCache(); + } + + async #loadColdCache() { + const canonicalUrl = this.system(); + const version = this.version(); + const cacheFilePath = getCacheFilePath(CACHE_CS_DIR, canonicalUrl, version); + + try { + const data = await fs.readFile(cacheFilePath, 'utf-8'); + const cached = JSON.parse(data); + + if (!cached || !cached.concepts || !Array.isArray(cached.concepts)) { + return; + } + + // Restore concepts to cache + for (const concept of cached.concepts) { + if (concept && concept.code) { + this.sharedConceptCache.set(concept.code, concept); + } + } + + this.loadedConceptCount = cached.concepts.length; + this.customFingerprint = cached.fingerprint || null; + this.isComplete = true; + + if (this.meta?.codeSystem?.jsonObj) { + this.meta.codeSystem.jsonObj.content = CodeSystemContentMode.Complete; + } + + this.#applyConceptsToCodeSystemResource(this.meta?.codeSystem || null); + + console.log(`[OCL] Loaded CodeSystem from cold cache: ${canonicalUrl} (${cached.concepts.length} concepts, fingerprint=${this.customFingerprint?.substring(0, 8)})`); + } catch (error) { + if (error.code !== 'ENOENT') { + console.error(`[OCL] Failed to load cold cache for CodeSystem ${canonicalUrl}:`, error.message); + } + } + } + + async #saveColdCache(concepts) { + const canonicalUrl = this.system(); + const version = this.version(); + const cacheFilePath = getCacheFilePath(CACHE_CS_DIR, canonicalUrl, version); + + try { + await ensureCacheDirectories(); + + const fingerprint = computeCodeSystemFingerprint(concepts); + const cacheData = { + canonicalUrl, + version, + fingerprint, + timestamp: new Date().toISOString(), + conceptCount: concepts.length, + concepts + }; + + await fs.writeFile(cacheFilePath, JSON.stringify(cacheData, null, 2), 'utf-8'); + console.log(`[OCL] Saved CodeSystem to cold cache: ${canonicalUrl} (${concepts.length} concepts, fingerprint=${fingerprint?.substring(0, 8)})`); + + return fingerprint; + } catch (error) { + console.error(`[OCL] Failed to save cold cache for CodeSystem ${canonicalUrl}:`, error.message); + return null; + } + } + + static scheduleBackgroundLoadByKey(system, version = null, reason = 'valueset-expansion') { + const key = `${system}|${version || ''}`; + const factory = OCLSourceCodeSystemFactory.factoriesByKey.get(key); + if (!factory) { + console.log(`[OCL] CodeSystem load not scheduled (factory unavailable): ${key}`); + return false; + } + factory.scheduleBackgroundLoad(reason); + return true; + } + + static checksumForResource(system, version = null) { + const key = `${system}|${version || ''}`; + const factory = OCLSourceCodeSystemFactory.factoriesByKey.get(key); + if (!factory) { + return null; + } + return factory.currentChecksum(); + } + + static loadProgress() { + let total = 0; + let loaded = 0; + + for (const factory of OCLSourceCodeSystemFactory.factoriesByKey.values()) { + total += 1; + if (factory && factory.isCompleteNow()) { + loaded += 1; + } + } + + const percentage = total > 0 ? (loaded / total) * 100 : 0; + return { loaded, total, percentage }; + } + + defaultVersion() { + return this.meta.version || null; + } + + build(opContext, supplements) { + this.#syncWarmStateWithChecksum(); + this.#applyConceptsToCodeSystemResource(this.meta?.codeSystem || null); + this.recordUse(); + return new OCLSourceCodeSystemProvider(opContext, supplements, this.httpClient, this.meta, { + conceptCache: this.sharedConceptCache, + pageCache: this.sharedPageCache, + pendingConceptRequests: this.sharedPendingConceptRequests, + pendingPageRequests: this.sharedPendingPageRequests, + scheduleBackgroundLoad: reason => this.scheduleBackgroundLoad(reason), + isSystemComplete: () => this.isComplete, + getTotalConceptCount: () => this.loadedConceptCount + }); + } + + scheduleBackgroundLoad(reason = 'request') { + this.#syncWarmStateWithChecksum(); + if (this.isComplete) { + return; + } + + const cacheFilePath = getCacheFilePath(CACHE_CS_DIR, this.system(), this.version()); + const cacheAgeMs = getColdCacheAgeMs(cacheFilePath); + if (cacheAgeMs != null && cacheAgeMs < COLD_CACHE_FRESHNESS_MS) { + console.log(`[OCL] Skipping warm-up for CodeSystem ${this.system()} (cold cache age: ${formatCacheAgeMinutes(cacheAgeMs)})`); + return; + } + + const key = this.#resourceKey(); + const jobKey = `cs:${key}`; + + if (OCLBackgroundJobQueue.isQueuedOrRunning(jobKey)) { + console.log(`[OCL] CodeSystem load already queued or running: ${key}`); + return; + } + + let queuedJobSize = null; + console.log(`[OCL] CodeSystem load enqueued: ${key} (${reason})`); + OCLBackgroundJobQueue.enqueue( + jobKey, + 'CodeSystem load', + async () => { + await this.#runBackgroundLoad(key, queuedJobSize); + }, + { + jobId: this.system(), + getProgress: () => this.#backgroundLoadProgressSnapshot(), + resolveJobSize: async () => { + queuedJobSize = await this.#fetchConceptCountFromHeaders(); + return queuedJobSize; + } + } + ); + } + + async #runBackgroundLoad(key, knownConceptCount = null) { + console.log(`[OCL] CodeSystem load started: ${key}`); + try { + this.backgroundLoadProgress = { processed: 0, total: null }; + const resolvedTotal = Number.isFinite(knownConceptCount) && knownConceptCount >= 0 + ? knownConceptCount + : await this.#fetchConceptCountFromHeaders(); + this.backgroundLoadProgress.total = resolvedTotal; + const count = await this.#loadAllConceptPages(); + this.loadedConceptCount = count; + this.isComplete = true; + this.loadedChecksum = this.meta?.checksum || null; + this.backgroundLoadProgress = { + processed: count, + total: count > 0 ? count : this.backgroundLoadProgress.total + }; + + if (this.meta?.codeSystem?.jsonObj) { + this.meta.codeSystem.jsonObj.content = CodeSystemContentMode.Complete; + } + + this.#applyConceptsToCodeSystemResource(this.meta?.codeSystem || null); + + // Compute custom fingerprint and compare with cold cache + const allConcepts = Array.from(this.sharedConceptCache.values()); + const newFingerprint = computeCodeSystemFingerprint(allConcepts); + + if (this.customFingerprint && newFingerprint === this.customFingerprint) { + console.log(`[OCL] CodeSystem fingerprint unchanged: ${key} (fingerprint=${newFingerprint?.substring(0, 8)})`); + } else { + if (this.customFingerprint) { + console.log(`[OCL] CodeSystem fingerprint changed: ${key} (${this.customFingerprint?.substring(0, 8)} -> ${newFingerprint?.substring(0, 8)})`); + console.log(`[OCL] Replacing cold cache with new hot cache: ${key}`); + } else { + console.log(`[OCL] Computed fingerprint for CodeSystem: ${key} (fingerprint=${newFingerprint?.substring(0, 8)})`); + } + + // Save to cold cache + const savedFingerprint = await this.#saveColdCache(allConcepts); + if (savedFingerprint) { + this.customFingerprint = savedFingerprint; + } + } + + console.log(`[OCL] CodeSystem load completed, marked content=complete: ${key}`); + const progress = OCLSourceCodeSystemFactory.loadProgress(); + console.log(`[OCL] CodeSystem load completed: ${this.system()}. Loaded ${progress.loaded}/${progress.total} CodeSystems (${progress.percentage.toFixed(2)}%)`); + console.log(`[OCL] CodeSystem now available in cache: ${key} (${count} concepts)`); + } catch (error) { + console.error(`[OCL] CodeSystem background load failed: ${key}: ${error.message}`); + } + } + + async #loadAllConceptPages() { + if (!this.meta?.conceptsUrl) { + this.loadedConceptCount = 0; + this.backgroundLoadProgress = { processed: 0, total: 0 }; + return 0; + } + + let page = 1; + let total = 0; + + // eslint-disable-next-line no-constant-condition + while (true) { + const pageData = await this.#fetchAndCacheConceptPage(page); + const concepts = Array.isArray(pageData?.concepts) ? pageData.concepts : []; + if (concepts.length === 0) { + break; + } + total += concepts.length; + this.backgroundLoadProgress.processed = total; + if (concepts.length < CONCEPT_PAGE_SIZE) { + break; + } + page += 1; + } + + return total; + } + + async #fetchAndCacheConceptPage(page) { + const cacheKey = `${this.meta.conceptsUrl}|p=${page}|l=${CONCEPT_PAGE_SIZE}`; + if (this.sharedPageCache.has(cacheKey)) { + const cached = this.sharedPageCache.get(cacheKey); + const concepts = Array.isArray(cached) + ? cached + : Array.isArray(cached?.concepts) + ? cached.concepts + : []; + const reportedTotal = this.#extractTotalFromPayload(cached?.payload || null); + return { concepts, reportedTotal }; + } + + if (this.sharedPendingPageRequests.has(cacheKey)) { + return await this.sharedPendingPageRequests.get(cacheKey); + } + + const pending = (async () => { + let response; + try { + response = await this.httpClient.get(this.meta.conceptsUrl, { params: { page, limit: CONCEPT_PAGE_SIZE } }); + } catch (error) { + if (error && error.response && error.response.status === 404) { + this.sharedPageCache.set(cacheKey, []); + return []; + } + throw error; + } + + const payload = response.data; + const items = Array.isArray(payload) + ? payload + : Array.isArray(payload?.results) + ? payload.results + : Array.isArray(payload?.items) + ? payload.items + : Array.isArray(payload?.data) + ? payload.data + : []; + + const mapped = items + .map(item => this.#toConceptContext(item)) + .filter(Boolean); + + this.sharedPageCache.set(cacheKey, { concepts: mapped, payload }); + for (const concept of mapped) { + if (concept && concept.code) { + this.sharedConceptCache.set(concept.code, concept); + } + } + return { + concepts: mapped, + reportedTotal: this.#extractTotalFromPayload(payload) + }; + })(); + + this.sharedPendingPageRequests.set(cacheKey, pending); + try { + return await pending; + } finally { + this.sharedPendingPageRequests.delete(cacheKey); + } + } + + #syncWarmStateWithChecksum() { + const checksum = this.meta?.checksum || null; + if (this.loadedChecksum == null) { + this.loadedChecksum = checksum; + return; + } + + if (checksum !== this.loadedChecksum) { + this.isComplete = false; + this.loadedConceptCount = -1; + this.backgroundLoadProgress = { processed: 0, total: null }; + this.sharedConceptCache.clear(); + this.sharedPageCache.clear(); + this.loadedChecksum = checksum; + this.materializedConceptList = null; + this.materializedConceptCount = -1; + if (this.meta?.codeSystem?.jsonObj) { + this.meta.codeSystem.jsonObj.content = CodeSystemContentMode.NotPresent; + delete this.meta.codeSystem.jsonObj.concept; + } + console.log(`[OCL] CodeSystem checksum changed, invalidated warm cache: ${this.#resourceKey()}`); + } + } + + #applyConceptsToCodeSystemResource(codeSystem) { + if (!codeSystem || typeof codeSystem !== 'object' || !codeSystem.jsonObj) { + return; + } + + if (this.isComplete !== true) { + delete codeSystem.jsonObj.concept; + return; + } + + const concepts = Array.from(this.sharedConceptCache.values()) + .filter(concept => concept && concept.code); + + if (!Array.isArray(this.materializedConceptList) || this.materializedConceptCount !== concepts.length) { + this.materializedConceptList = concepts + .sort((a, b) => String(a.code).localeCompare(String(b.code))) + .map(concept => { + const fhirConcept = { code: concept.code }; + + if (concept.display) { + fhirConcept.display = concept.display; + } + + if (concept.definition) { + fhirConcept.definition = concept.definition; + } + + if (Array.isArray(concept.designations) && concept.designations.length > 0) { + const designations = concept.designations + .filter(d => d && d.value) + .map(d => ({ + language: d.language || undefined, + value: d.value + })); + + if (designations.length > 0) { + fhirConcept.designation = designations; + } + } + + return fhirConcept; + }); + this.materializedConceptCount = concepts.length; + } + + codeSystem.jsonObj.concept = this.materializedConceptList; + codeSystem.jsonObj.content = CodeSystemContentMode.Complete; + } + + #backgroundLoadProgressSnapshot() { + const processed = this.backgroundLoadProgress?.processed; + const total = this.backgroundLoadProgress?.total; + if ( + typeof processed === 'number' && + Number.isFinite(processed) && + typeof total === 'number' && + Number.isFinite(total) && + total > 0 + ) { + return { processed, total }; + } + return null; + } + + #extractTotalFromPayload(payload) { + if (!payload || typeof payload !== 'object' || Array.isArray(payload)) { + return null; + } + + const candidates = [ + payload.total, + payload.total_count, + payload.totalCount, + payload.num_found, + payload.numFound, + payload.count + ]; + + for (const candidate of candidates) { + if (typeof candidate === 'number' && Number.isFinite(candidate) && candidate >= 0) { + return candidate; + } + } + + return null; + } + + async #fetchConceptCountFromHeaders() { + if (!this.meta?.conceptsUrl) { + return null; + } + + try { + const response = await this.httpClient.get(this.meta.conceptsUrl, { + params: { + limit: 1 + } + }); + return this.#extractNumFoundFromHeaders(response?.headers); + } catch (error) { + return null; + } + } + + #extractNumFoundFromHeaders(headers) { + if (!headers || typeof headers !== 'object') { + return null; + } + + const raw = headers.num_found ?? headers['num-found'] ?? headers.Num_Found ?? null; + const parsed = typeof raw === 'number' ? raw : Number.parseInt(raw, 10); + if (!Number.isFinite(parsed) || parsed < 0) { + return null; + } + return parsed; + } + + #resourceKey() { + return `${this.system()}|${this.version() || ''}`; + } + + currentChecksum() { + this.#syncWarmStateWithChecksum(); + return this.meta?.checksum || this.loadedChecksum || null; + } + + isCompleteNow() { + this.#syncWarmStateWithChecksum(); + return this.isComplete === true; + } + + #toConceptContext(concept) { + if (!concept || typeof concept !== 'object') { + return null; + } + + const code = concept.code || concept.id || null; + if (!code) { + return null; + } + + return { + code, + display: concept.display_name || concept.display || concept.name || null, + definition: concept.description || concept.definition || null, + retired: concept.retired === true, + designations: this.#extractDesignations(concept) + }; + } + + #extractDesignations(concept) { + const result = []; + const seen = new Set(); + + const add = (language, value) => { + const text = typeof value === 'string' ? value.trim() : ''; + if (!text) { + return; + } + const lang = typeof language === 'string' ? language.trim() : ''; + const key = `${lang}|${text}`; + if (seen.has(key)) { + return; + } + seen.add(key); + result.push({ language: lang, value: text }); + }; + + if (Array.isArray(concept.names)) { + for (const entry of concept.names) { + if (!entry || typeof entry !== 'object') { + continue; + } + add(entry.locale || entry.language || entry.lang || '', entry.name || entry.display_name || entry.display || entry.value || entry.term); + } + } + + if (concept.display_name || concept.display || concept.name) { + add(concept.locale || concept.default_locale || concept.language || '', concept.display_name || concept.display || concept.name); + } + + if (concept.locale_display_names && typeof concept.locale_display_names === 'object') { + for (const [lang, value] of Object.entries(concept.locale_display_names)) { + add(lang, value); + } + } + + return result; + } + + system() { + return this.meta.canonicalUrl; + } + + name() { + return this.meta.name || this.meta.shortCode || this.meta.id || this.system(); + } + + version() { + return this.meta.version || null; + } + + id() { + return this.meta.id || this.meta.shortCode || this.system(); + } + + iteratable() { + return true; } } module.exports = { - AbstractCodeSystemProvider + OCLCodeSystemProvider, + OCLSourceCodeSystemFactory, + OCLBackgroundJobQueue }; \ No newline at end of file diff --git a/tx/ocl/vs-ocl.js b/tx/ocl/vs-ocl.js index bfd6349f..8aabad71 100644 --- a/tx/ocl/vs-ocl.js +++ b/tx/ocl/vs-ocl.js @@ -1,105 +1,1871 @@ -/** - * Abstract base class for value set providers - * Defines the interface that all value set providers must implement - */ -// eslint-disable-next-line no-unused-vars -class OCLValueSetProvider { - /** - * {int} Unique number assigned to this provider - */ - spaceId; - - /** - * ensure that the ids on the value sets are unique, if they are - * in the global namespace - * - * @param {Set} ids - */ - // eslint-disable-next-line no-unused-vars +const axios = require('axios'); +const fs = require('fs/promises'); +const fsSync = require('fs'); +const crypto = require('crypto'); +const path = require('path'); +const { AbstractValueSetProvider } = require('../vs/vs-api'); +const { VersionUtilities } = require('../../library/version-utilities'); +const ValueSet = require('../library/valueset'); +const { SearchFilterText } = require('../library/designations'); +const { TxParameters } = require('../params'); +const { OCLSourceCodeSystemFactory, OCLBackgroundJobQueue } = require('./cs-ocl'); + +const DEFAULT_BASE_URL = 'https://oclapi2.ips.hsl.org.br'; +const PAGE_SIZE = 100; +const CONCEPT_PAGE_SIZE = 1000; +const FILTERED_CONCEPT_PAGE_SIZE = 200; +const COLD_CACHE_FRESHNESS_MS = 60 * 60 * 1000; + +// Cold cache configuration (import from cs-ocl) +const CACHE_BASE_DIR = path.join(process.cwd(), 'data', 'terminology-cache', 'ocl'); +const CACHE_VS_DIR = path.join(CACHE_BASE_DIR, 'valuesets'); +const TXPARAMS_HASH_PATCH_FLAG = Symbol.for('fhirsmith.ocl.txparameters.hash.filter.patch'); + +function normalizeFilterForCacheKey(filter) { + if (typeof filter !== 'string') { + return ''; + } + + // OCL filter matching is case-insensitive and trims surrounding whitespace. + return filter.trim().toLowerCase(); +} + +function ensureTxParametersHashIncludesFilter() { + const proto = TxParameters && TxParameters.prototype; + if (!proto || proto[TXPARAMS_HASH_PATCH_FLAG] === true || typeof proto.hashSource !== 'function') { + return; + } + + const originalHashSource = proto.hashSource; + proto.hashSource = function hashSourceWithFilter() { + const base = originalHashSource.call(this); + const normalizedFilter = normalizeFilterForCacheKey(this.filter); + return `${base}|filter=${normalizedFilter}`; + }; + + Object.defineProperty(proto, TXPARAMS_HASH_PATCH_FLAG, { + value: true, + writable: false, + configurable: false, + enumerable: false + }); +} + +ensureTxParametersHashIncludesFilter(); + +// Cache file utilities +async function ensureCacheDirectories() { + try { + await fs.mkdir(CACHE_VS_DIR, { recursive: true }); + } catch (error) { + console.error('[OCL-ValueSet] Failed to create cache directories:', error.message); + } +} + +// ValueSet expansion fingerprint computation +function computeValueSetExpansionFingerprint(expansion) { + if (!expansion || !Array.isArray(expansion.contains) || expansion.contains.length === 0) { + return null; + } + + // Normalize expansion entries to deterministic strings + const normalized = expansion.contains + .map(entry => { + if (!entry || !entry.code) { + return null; + } + const system = String(entry.system || ''); + const code = String(entry.code || ''); + const display = String(entry.display || ''); + const inactive = entry.inactive === true ? '1' : '0'; + return `${system}|${code}|${display}|${inactive}`; + }) + .filter(Boolean) + .sort(); + + // Compute SHA256 hash + const hash = crypto.createHash('sha256'); + for (const item of normalized) { + hash.update(item); + hash.update('\n'); + } + return hash.digest('hex'); +} + +function sanitizeFilename(text) { + if (!text || typeof text !== 'string') { + return 'unknown'; + } + return text + .replace(/[^a-zA-Z0-9._-]/g, '_') + .replace(/_+/g, '_') + .substring(0, 200); +} + +function getCacheFilePath(baseDir, canonicalUrl, version = null, paramsKey = 'default') { + const filename = sanitizeFilename(canonicalUrl) + + (version ? `_${sanitizeFilename(version)}` : '') + + (paramsKey && paramsKey !== 'default' ? `_p_${sanitizeFilename(paramsKey)}` : '') + + '.json'; + return path.join(baseDir, filename); +} + +function getColdCacheAgeMs(cacheFilePath) { + try { + const stats = fsSync.statSync(cacheFilePath); + if (!stats || !Number.isFinite(stats.mtimeMs)) { + return null; + } + return Math.max(0, Date.now() - stats.mtimeMs); + } catch (error) { + if (error && error.code !== 'ENOENT') { + console.error(`[OCL-ValueSet] Failed to inspect cold cache file ${cacheFilePath}: ${error.message}`); + } + return null; + } +} + +function formatCacheAgeMinutes(ageMs) { + const minutes = Math.max(1, Math.round(ageMs / 60000)); + return `${minutes} minute${minutes === 1 ? '' : 's'}`; +} + +class OCLValueSetProvider extends AbstractValueSetProvider { + constructor(config = {}) { + super(); + const options = typeof config === 'string' ? { baseUrl: config } : (config || {}); + + this.baseUrl = (options.baseUrl || DEFAULT_BASE_URL).replace(/\/+$/, ''); + this.org = options.org || null; + + const headers = { + Accept: 'application/json', + 'User-Agent': 'FHIRSmith-OCL-Provider/1.0' + }; + + if (options.token) { + headers.Authorization = options.token.startsWith('Token ') || options.token.startsWith('Bearer ') + ? options.token + : `Token ${options.token}`; + } + + this.httpClient = axios.create({ + baseURL: this.baseUrl, + timeout: options.timeout || 30000, + headers + }); + + this.valueSetMap = new Map(); + this._idMap = new Map(); + this.collectionMeta = new Map(); + this.sourceCanonicalCache = new Map(); + this.collectionConceptPageCache = new Map(); + this.pendingCollectionConceptPageRequests = new Map(); + this.collectionSourcesCache = new Map(); + this.pendingCollectionSourcesRequests = new Map(); + this.pendingSourceCanonicalRequests = new Map(); + this.collectionByCanonicalCache = new Map(); + this.pendingCollectionByCanonicalRequests = new Map(); + this._composePromises = new Map(); + this.backgroundExpansionCache = new Map(); + this.backgroundExpansionProgress = new Map(); + this.valueSetFingerprints = new Map(); + this._initialized = false; + this._initializePromise = null; + this.sourcePackageCode = this.org + ? `ocl:${this.baseUrl}|org=${this.org}` + : `ocl:${this.baseUrl}`; + } + + async #loadColdCacheForValueSets() { + try { + const files = await fs.readdir(CACHE_VS_DIR); + let loadedCount = 0; + + for (const file of files) { + if (!file.endsWith('.json')) { + continue; + } + + try { + const filePath = path.join(CACHE_VS_DIR, file); + const data = await fs.readFile(filePath, 'utf-8'); + const cached = JSON.parse(data); + + if (!cached || !cached.canonicalUrl || !cached.expansion) { + continue; + } + + const paramsKey = cached.paramsKey || 'default'; + const cacheKey = `${cached.canonicalUrl}|${cached.version || ''}|${paramsKey}`; + this.backgroundExpansionCache.set(cacheKey, { + expansion: cached.expansion, + metadataSignature: cached.metadataSignature || null, + dependencyChecksums: cached.dependencyChecksums || {}, + createdAt: cached.timestamp ? new Date(cached.timestamp).getTime() : Date.now() + }); + + this.valueSetFingerprints.set(cacheKey, cached.fingerprint || null); + loadedCount++; + } catch (error) { + console.error(`[OCL-ValueSet] Failed to load cold cache file ${file}:`, error.message); + } + } + + if (loadedCount > 0) { + console.log(`[OCL-ValueSet] Loaded ${loadedCount} ValueSet expansions from cold cache`); + } + } catch (error) { + if (error.code !== 'ENOENT') { + console.error('[OCL-ValueSet] Failed to load cold cache:', error.message); + } + } + } + + async #saveColdCacheForValueSet(vs, expansion, metadataSignature, dependencyChecksums, paramsKey = 'default') { + const canonicalUrl = vs?.url; + const version = vs?.version || null; + if (!canonicalUrl || !expansion) { + return null; + } + + const cacheFilePath = getCacheFilePath(CACHE_VS_DIR, canonicalUrl, version, paramsKey); + + try { + await ensureCacheDirectories(); + + const fingerprint = computeValueSetExpansionFingerprint(expansion); + const cacheData = { + canonicalUrl, + version, + paramsKey, + fingerprint, + timestamp: new Date().toISOString(), + conceptCount: expansion.contains?.length || 0, + expansion, + metadataSignature, + dependencyChecksums + }; + + await fs.writeFile(cacheFilePath, JSON.stringify(cacheData, null, 2), 'utf-8'); + console.log(`[OCL-ValueSet] Saved ValueSet expansion to cold cache: ${canonicalUrl} (${expansion.contains?.length || 0} concepts, fingerprint=${fingerprint?.substring(0, 8)})`); + + return fingerprint; + } catch (error) { + console.error(`[OCL-ValueSet] Failed to save cold cache for ValueSet ${canonicalUrl}:`, error.message); + return null; + } + } + + sourcePackage() { + return this.sourcePackageCode; + } + + async initialize() { + if (this._initialized) { + return; + } + + if (this._initializePromise) { + await this._initializePromise; + return; + } + + this._initializePromise = (async () => { + try { + // Load cold cache first + await this.#loadColdCacheForValueSets(); + + const collections = await this.#fetchCollectionsForDiscovery(); + console.log(`[OCL-ValueSet] Fetched ${collections.length} collections`); + + for (const collection of collections) { + const valueSet = this.#toValueSet(collection); + if (!valueSet) { + continue; + } + this.#indexValueSet(valueSet); + } + + console.log(`[OCL-ValueSet] Loaded ${this.valueSetMap.size} value sets`); + this._initialized = true; + } catch (error) { + console.error(`[OCL-ValueSet] Initialization failed:`, error.message); + if (error.response) { + console.error(`[OCL-ValueSet] HTTP ${error.response.status}: ${error.response.statusText}`); + } + throw error; + } + })(); + + try { + await this._initializePromise; + } finally { + this._initializePromise = null; + } + } + assignIds(ids) { - throw new Error('assignIds must be implemented by AbstractValueSetProvider subclass'); + if (!this.spaceId) { + return; + } + + const unique = new Set(this.valueSetMap.values()); + this._idMap.clear(); + + for (const vs of unique) { + if (!vs.id.startsWith(`${this.spaceId}-`)) { + const nextId = `${this.spaceId}-${vs.id}`; + vs.id = nextId; + vs.jsonObj.id = nextId; + } + this._idMap.set(vs.id, vs); + ids.add(`ValueSet/${vs.id}`); + } } - /** - * Fetches a specific value set by URL and version - * @param {string} url - The URL/identifier of the value set - * @param {string} version - The version of the value set - * @returns {Promise} The requested value set - * @throws {Error} Must be implemented by subclasses - */ - // eslint-disable-next-line no-unused-vars async fetchValueSet(url, version) { - throw new Error('fetchValueSet must be implemented by subclass'); + this._validateFetchParams(url, version); + + let key = `${url}|${version}`; + if (this.valueSetMap.has(key)) { + const vs = this.valueSetMap.get(key); + await this.#ensureComposeIncludes(vs); + this.#clearInlineExpansion(vs); + this.#scheduleBackgroundExpansion(vs, { reason: 'fetch-valueset' }); + return vs; + } + + if (version && VersionUtilities.isSemVer(version)) { + const majorMinor = VersionUtilities.getMajMin(version); + if (majorMinor) { + key = `${url}|${majorMinor}`; + if (this.valueSetMap.has(key)) { + const vs = this.valueSetMap.get(key); + await this.#ensureComposeIncludes(vs); + this.#clearInlineExpansion(vs); + this.#scheduleBackgroundExpansion(vs, { reason: 'fetch-valueset-mm' }); + return vs; + } + } + } + + if (this.valueSetMap.has(url)) { + const vs = this.valueSetMap.get(url); + await this.#ensureComposeIncludes(vs); + this.#clearInlineExpansion(vs); + this.#scheduleBackgroundExpansion(vs, { reason: 'fetch-valueset-url' }); + return vs; + } + + const resolved = await this.#resolveValueSetByCanonical(url, version); + if (resolved) { + await this.#ensureComposeIncludes(resolved); + this.#clearInlineExpansion(resolved); + this.#scheduleBackgroundExpansion(resolved, { reason: 'fetch-valueset-resolved' }); + return resolved; + } + + await this.initialize(); + + key = `${url}|${version}`; + if (this.valueSetMap.has(key)) { + const vs = this.valueSetMap.get(key); + await this.#ensureComposeIncludes(vs); + this.#clearInlineExpansion(vs); + this.#scheduleBackgroundExpansion(vs, { reason: 'fetch-valueset-init' }); + return vs; + } + + if (version && VersionUtilities.isSemVer(version)) { + const majorMinor = VersionUtilities.getMajMin(version); + if (majorMinor) { + key = `${url}|${majorMinor}`; + if (this.valueSetMap.has(key)) { + const vs = this.valueSetMap.get(key); + await this.#ensureComposeIncludes(vs); + this.#clearInlineExpansion(vs); + this.#scheduleBackgroundExpansion(vs, { reason: 'fetch-valueset-init-mm' }); + return vs; + } + } + } + + if (this.valueSetMap.has(url)) { + const vs = this.valueSetMap.get(url); + await this.#ensureComposeIncludes(vs); + this.#clearInlineExpansion(vs); + this.#scheduleBackgroundExpansion(vs, { reason: 'fetch-valueset-init-url' }); + return vs; + } + + return null; } - /** - * Fetches a specific value set by id. ValueSet providers must enforce that value set ids are unique - * either globally (as enforced by assignIds) or in their space - * - * @param {string} id - The id of the value set - * @returns {Promise} The requested value set - * @throws {Error} Must be implemented by subclasses - */ - // eslint-disable-next-line no-unused-vars async fetchValueSetById(id) { - throw new Error('fetchValueSetById must be implemented by subclass'); + const local = this.#getLocalValueSetById(id); + if (local) { + await this.#ensureComposeIncludes(local); + this.#clearInlineExpansion(local); + this.#scheduleBackgroundExpansion(local, { reason: 'fetch-valueset-by-id' }); + return local; + } + + await this.initialize(); + + const vs = this.#getLocalValueSetById(id); + await this.#ensureComposeIncludes(vs); + this.#clearInlineExpansion(vs); + this.#scheduleBackgroundExpansion(vs, { reason: 'fetch-valueset-by-id-init' }); + return vs; + } + + #clearInlineExpansion(vs) { + if (!vs || !vs.jsonObj || !vs.jsonObj.expansion) { + return; + } + delete vs.jsonObj.expansion; + } + + #getLocalValueSetById(id) { + if (this._idMap.has(id)) { + return this._idMap.get(id); + } + + if (this.spaceId && id.startsWith(`${this.spaceId}-`)) { + const unprefixed = id.substring(this.spaceId.length + 1); + return this._idMap.get(id) || this._idMap.get(unprefixed) || this.valueSetMap.get(unprefixed) || null; + } + + return this._idMap.get(id) || this.valueSetMap.get(id) || null; } - /** - * Searches for value sets based on provided criteria - * @param {Array<{name: string, value: string}>} searchParams - List of name/value pairs for search criteria - * @returns {Promise>} List of matching value sets - * @throws {Error} Must be implemented by subclasses - */ // eslint-disable-next-line no-unused-vars - async searchValueSets(searchParams, elements = null) { - throw new Error('searchValueSets must be implemented by subclass'); + async searchValueSets(searchParams, _elements) { + await this.initialize(); + this._validateSearchParams(searchParams); + + const params = Object.fromEntries(searchParams.map(({ name, value }) => [name, String(value).toLowerCase()])); + const values = Array.from(new Set(this.valueSetMap.values())); + + if (Object.keys(params).length === 0) { + return values; + } + + return values.filter(vs => this.#matches(vs.jsonObj, params)); } - /** - * - * @returns {number} total number of value sets - */ vsCount() { - return 0; + return new Set(this.valueSetMap.values()).size; + } + + async listAllValueSets() { + await this.initialize(); + const urls = new Set(); + for (const vs of this.valueSetMap.values()) { + if (vs && vs.url) { + urls.add(vs.url); + } + } + return Array.from(urls); + } + + async close() { + } + + #indexValueSet(vs) { + this.#invalidateExpansionCache(vs); + this.valueSetMap.set(vs.url, vs); + if (vs.version) { + this.valueSetMap.set(`${vs.url}|${vs.version}`, vs); + } + this.valueSetMap.set(vs.id, vs); + this._idMap.set(vs.id, vs); + } + + #toValueSet(collection) { + if (!collection || typeof collection !== 'object') { + return null; + } + + const canonicalUrl = collection.canonical_url || collection.canonicalUrl || collection.url; + const id = collection.id; + if (!canonicalUrl || !id) { + return null; + } + + const preferredSource = collection.preferred_source || collection.preferredSource || null; + const json = { + resourceType: 'ValueSet', + id, + url: canonicalUrl, + version: collection.version || null, + name: collection.name || id, + title: collection.full_name || collection.fullName || collection.name || id, + status: 'active', + experimental: collection.experimental === true, + immutable: collection.immutable === true, + description: collection.description || null, + publisher: collection.publisher || collection.owner || null, + language: collection.default_locale || collection.defaultLocale || null + }; + + const lastUpdated = this.#toIsoDate(collection.updated_on || collection.updatedOn || collection.updated_at || collection.updatedAt); + if (lastUpdated) { + json.meta = { lastUpdated }; + } + + if (preferredSource) { + json.compose = { + include: [{ system: preferredSource }] + }; + } + + const conceptsUrl = this.#normalizePath( + collection.concepts_url || collection.conceptsUrl || this.#buildCollectionConceptsPath(collection) + ); + const expansionUrl = this.#normalizePath( + collection.expansion_url || collection.expansionUrl || this.#buildCollectionExpansionPath(collection) + ); + + const meta = { + collectionId: collection.id || collection.short_code || collection.shortCode || null, + conceptsUrl, + expansionUrl, + preferredSource, + owner: collection.owner || null, + ownerType: collection.owner_type || collection.ownerType || null + }; + + this.#storeCollectionMeta(id, canonicalUrl, meta); + + const valueSet = new ValueSet(json, 'R5'); + this.#attachOclHelpers(valueSet, meta); + return valueSet; + } + + #attachOclHelpers(valueSet, meta) { + if (!valueSet || !meta) { + return; + } + + valueSet.oclMeta = meta; + valueSet.oclFetchConcepts = async ({ count, offset, activeOnly, filter, languageCodes }) => { + return this.#fetchCollectionConcepts(meta, { + count, + offset, + activeOnly, + filter: typeof filter === 'string' ? filter : null, + languageCodes: Array.isArray(languageCodes) ? languageCodes : [], + fallbackSystem: meta.preferredSource || valueSet.url + }); + }; + } + + #storeCollectionMeta(id, url, meta) { + if (!meta || (!meta.conceptsUrl && !meta.expansionUrl)) { + return; + } + if (id) { + this.collectionMeta.set(id, meta); + } + if (url) { + this.collectionMeta.set(url, meta); + } + } + + #normalizePath(pathValue) { + if (!pathValue) { + return null; + } + if (typeof pathValue !== 'string') { + return null; + } + if (pathValue.startsWith('http://') || pathValue.startsWith('https://')) { + return pathValue; + } + return `${this.baseUrl}${pathValue.startsWith('/') ? '' : '/'}${pathValue}`; + } + + #buildCollectionConceptsPath(collection) { + if (!collection || typeof collection !== 'object') { + return null; + } + const owner = collection.owner || null; + const ownerType = collection.owner_type || collection.ownerType || null; + const id = collection.id || collection.short_code || collection.shortCode || null; + if (!owner || !id || ownerType !== 'Organization') { + return null; + } + return `/orgs/${encodeURIComponent(owner)}/collections/${encodeURIComponent(id)}/concepts/`; + } + + #buildCollectionExpansionPath(collection) { + if (!collection || typeof collection !== 'object') { + return null; + } + const owner = collection.owner || null; + const ownerType = collection.owner_type || collection.ownerType || null; + const id = collection.id || collection.short_code || collection.shortCode || null; + if (!owner || !id || ownerType !== 'Organization') { + return null; + } + return `/orgs/${encodeURIComponent(owner)}/collections/${encodeURIComponent(id)}/HEAD/expansions/autoexpand-HEAD/`; + } + + #getCollectionMeta(vs) { + if (!vs) { + return null; + } + return this.collectionMeta.get(vs.id) || this.collectionMeta.get(vs.url) || null; + } + + async #ensureComposeIncludes(vs) { + if (!vs || !vs.jsonObj) { + return; + } + if (vs.jsonObj.compose && Array.isArray(vs.jsonObj.compose.include) && vs.jsonObj.compose.include.length > 0) { + return; + } + + const meta = this.#getCollectionMeta(vs); + if (!meta || !meta.conceptsUrl) { + return; + } + + const composeKey = vs.id || vs.url; + if (this._composePromises.has(composeKey)) { + await this._composePromises.get(composeKey); + return; + } + + const promise = (async () => { + const sources = await this.#fetchCollectionSources(meta); + if (!sources || sources.length === 0) { + return; + } + + vs.jsonObj.compose = { + include: sources.map(source => ({ + system: source.system, + version: source.version || undefined + })) + }; + })(); + + this._composePromises.set(composeKey, promise); + try { + await promise; + } finally { + this._composePromises.delete(composeKey); + } + } + + async #fetchCollectionSources(meta) { + const sourcesCacheKey = `${meta.owner || ''}|${meta.collectionId || ''}|${meta.conceptsUrl || ''}|${meta.expansionUrl || ''}`; + if (this.collectionSourcesCache.has(sourcesCacheKey)) { + return this.collectionSourcesCache.get(sourcesCacheKey); + } + if (this.pendingCollectionSourcesRequests.has(sourcesCacheKey)) { + return this.pendingCollectionSourcesRequests.get(sourcesCacheKey); + } + + const pending = (async () => { + const sources = []; + const seen = new Set(); + + if (meta.expansionUrl) { + try { + const response = await this.httpClient.get(meta.expansionUrl); + const resolved = Array.isArray(response.data?.resolved_source_versions) + ? response.data.resolved_source_versions + : []; + + for (const entry of resolved) { + const system = entry.canonical_url || entry.canonicalUrl || null; + const owner = entry.owner || meta.owner || null; + const shortCode = entry.short_code || entry.shortCode || entry.id || null; + const version = entry.version || null; + + const systemUrl = system || (owner && shortCode ? await this.#getSourceCanonicalUrl(owner, shortCode) : null); + if (systemUrl && !seen.has(systemUrl)) { + seen.add(systemUrl); + sources.push({ system: systemUrl, version }); + } + } + } catch (error) { + // fall through to concepts listing + } + } + + if (sources.length > 0) { + return sources; + } + + if (!meta.conceptsUrl) { + return sources; + } + + const sourceKeys = await this.#fetchCollectionSourceKeys(meta.conceptsUrl, meta.owner || null); + for (const { owner, source } of sourceKeys) { + const systemUrl = await this.#getSourceCanonicalUrl(owner, source); + if (systemUrl && !seen.has(systemUrl)) { + seen.add(systemUrl); + sources.push({ system: systemUrl }); + } + } + + if (sources.length === 0 && meta.preferredSource) { + sources.push({ system: meta.preferredSource }); + } + + this.collectionSourcesCache.set(sourcesCacheKey, sources); + return sources; + })(); + + this.pendingCollectionSourcesRequests.set(sourcesCacheKey, pending); + try { + return await pending; + } finally { + this.pendingCollectionSourcesRequests.delete(sourcesCacheKey); + } + } + + async #fetchCollectionConcepts(meta, options) { + if (!meta || !meta.conceptsUrl) { + return { contains: [], total: 0 }; + } + + const count = Number.isInteger(options?.count) ? options.count : CONCEPT_PAGE_SIZE; + const offset = Number.isInteger(options?.offset) ? options.offset : 0; + const activeOnly = options?.activeOnly === true; + const filter = this.#normalizeFilter(options?.filter); + const filterMatcher = filter ? new SearchFilterText(filter) : null; + const remoteQuery = this.#buildRemoteQuery(filter); + const fallbackSystem = options?.fallbackSystem || null; + const preferredLanguageCodes = this.#normalizeLanguageCodes(options?.languageCodes); + const effectiveLanguageCodes = preferredLanguageCodes.length > 0 ? preferredLanguageCodes : ['en']; + + if (count <= 0) { + return { contains: [], total: 0 }; + } + + const hasFilter = !!filter; + const limit = hasFilter + ? Math.min(FILTERED_CONCEPT_PAGE_SIZE, CONCEPT_PAGE_SIZE) + : CONCEPT_PAGE_SIZE; + let page = Math.floor(Math.max(0, offset) / limit) + 1; + let skip = Math.max(0, offset) % limit; + let remaining = count; + const contains = []; + let reportedTotal = null; + + while (remaining > 0) { + const pageData = await this.#fetchConceptPage(meta.conceptsUrl, page, limit, remoteQuery); + const pageItems = Array.isArray(pageData?.items) ? pageData.items : []; + if ( + reportedTotal == null && + typeof pageData?.reportedTotal === 'number' && + Number.isFinite(pageData.reportedTotal) && + pageData.reportedTotal >= 0 + ) { + reportedTotal = pageData.reportedTotal; + } + + if (!pageItems || pageItems.length === 0) { + break; + } + + const slice = pageItems.slice(skip); + skip = 0; + + for (const concept of slice) { + if (remaining <= 0) { + break; + } + if (activeOnly && concept.retired === true) { + continue; + } + + const localizedNames = this.#extractLocalizedNames(concept, effectiveLanguageCodes); + const localizedDefinitions = this.#extractLocalizedDefinitions(concept, effectiveLanguageCodes); + + const display = localizedNames.display || concept.display_name || concept.display || concept.name || null; + const definition = localizedDefinitions.definition || concept.definition || concept.description || concept.concept_class || null; + const code = concept.code || concept.id || null; + const searchableText = [ + code, + display, + definition, + ...localizedNames.designation.map(d => d.value), + ...localizedDefinitions.definitions.map(d => d.value) + ].filter(Boolean).join(' '); + if (filterMatcher && !this.#conceptMatchesFilter(searchableText, code, display, definition, filter, filterMatcher)) { + continue; + } + + if (!code) { + continue; + } + + const owner = concept.owner || meta.owner || null; + const source = concept.source || null; + const conceptCanonical = concept.source_canonical_url || concept.sourceCanonicalUrl || null; + const system = conceptCanonical || (owner && source + ? await this.#getSourceCanonicalUrl(owner, source) + : fallbackSystem); + + contains.push({ + system: system || fallbackSystem, + code, + display, + definition: definition || undefined, + designation: localizedNames.designation, + definitions: localizedDefinitions.definitions, + inactive: concept.retired === true ? true : undefined + }); + remaining -= 1; + } + + if (pageItems.length < limit) { + break; + } + + page += 1; + } + + return { contains, total: contains.length, reportedTotal }; + } + + async #resolveValueSetByCanonical(url, version) { + const canonicalUrl = typeof url === 'string' ? url.trim() : ''; + if (!canonicalUrl) { + return null; + } + + const collection = await this.#findCollectionByCanonical(canonicalUrl, version); + if (!collection) { + return null; + } + + const valueSet = this.#toValueSet(collection); + if (!valueSet) { + return null; + } + + this.#indexValueSet(valueSet); + return valueSet; + } + + #valueSetBaseKey(vs) { + if (!vs || !vs.url) { + return null; + } + return `${vs.url}|${vs.version || ''}`; + } + + #expansionParamsKey(params) { + if (!params || typeof params !== 'object') { + return 'default'; + } + + try { + const normalized = Object.keys(params) + .sort() + .reduce((acc, key) => { + if (key === 'tx-resource' || key === 'valueSet') { + return acc; + } + acc[key] = params[key]; + return acc; + }, {}); + + const json = JSON.stringify(normalized); + if (!json || json === '{}') { + return 'default'; + } + return crypto.createHash('sha256').update(json).digest('hex').substring(0, 16); + } catch (error) { + return 'default'; + } + } + + #expansionCacheKey(vs, paramsKey) { + const base = this.#valueSetBaseKey(vs); + if (!base) { + return null; + } + return `${base}|${paramsKey || 'default'}`; + } + + #invalidateExpansionCache(vs) { + const base = this.#valueSetBaseKey(vs); + if (!base) { + return; + } + + for (const key of this.backgroundExpansionCache.keys()) { + if (key.startsWith(`${base}|`)) { + this.backgroundExpansionCache.delete(key); + } + } + } + + #applyCachedExpansion(vs, paramsKey) { + if (!vs || !vs.jsonObj) { + return; + } + + const cacheKey = this.#expansionCacheKey(vs, paramsKey); + if (!cacheKey) { + return; + } + + const cached = this.backgroundExpansionCache.get(cacheKey); + if (!cached || !cached.expansion) { + return; + } + + if (!this.#isCachedExpansionValid(vs, cached)) { + this.backgroundExpansionCache.delete(cacheKey); + if (vs.jsonObj.expansion) { + delete vs.jsonObj.expansion; + } + console.log(`[OCL-ValueSet] Cached ValueSet expansion invalidated: ${cacheKey}`); + return; + } + + if (vs.jsonObj.expansion) { + return; + } + + vs.jsonObj.expansion = structuredClone(cached.expansion); + console.log(`[OCL-ValueSet] ValueSet expansion restored from cache: ${cacheKey}`); + } + + #scheduleBackgroundExpansion(vs, options = {}) { + if (!vs || !vs.jsonObj) { + return; + } + + const paramsKey = this.#expansionParamsKey(options.params || null); + const cacheKey = this.#expansionCacheKey(vs, paramsKey); + if (!cacheKey) { + return; + } + + const cached = this.backgroundExpansionCache.get(cacheKey); + if (cached && !this.#isCachedExpansionValid(vs, cached)) { + this.backgroundExpansionCache.delete(cacheKey); + if (vs.jsonObj.expansion) { + delete vs.jsonObj.expansion; + } + console.log(`[OCL-ValueSet] Cached ValueSet expansion invalidated: ${cacheKey}`); + } + + if (vs.jsonObj.expansion) { + return; + } + + const cacheFilePath = getCacheFilePath(CACHE_VS_DIR, vs.url, vs.version || null, paramsKey); + const cacheAgeFromFileMs = getColdCacheAgeMs(cacheFilePath); + const persistedCache = this.backgroundExpansionCache.get(cacheKey); + const cacheAgeFromMetadataMs = Number.isFinite(persistedCache?.createdAt) + ? Math.max(0, Date.now() - persistedCache.createdAt) + : null; + + // Treat cache as fresh when either file mtime or persisted timestamp is recent. + const freshnessCandidates = [cacheAgeFromFileMs, cacheAgeFromMetadataMs].filter(age => age != null); + const freshestCacheAgeMs = freshnessCandidates.length > 0 ? Math.min(...freshnessCandidates) : null; + if (freshestCacheAgeMs != null && freshestCacheAgeMs < COLD_CACHE_FRESHNESS_MS) { + const freshnessSource = cacheAgeFromFileMs != null && cacheAgeFromMetadataMs != null + ? 'file+metadata' + : cacheAgeFromFileMs != null + ? 'file' + : 'metadata'; + console.log(`[OCL-ValueSet] Skipping warm-up for ValueSet ${vs.url} (cold cache age: ${formatCacheAgeMinutes(freshestCacheAgeMs)})`); + console.log(`[OCL-ValueSet] ValueSet cold cache is fresh, not enqueueing warm-up job (${cacheKey}, source=${freshnessSource})`); + return; + } + + const jobKey = `vs:${cacheKey}`; + if (OCLBackgroundJobQueue.isQueuedOrRunning(jobKey)) { + console.log(`[OCL-ValueSet] ValueSet expansion already queued or running: ${cacheKey}`); + return; + } + + let queuedJobSize = null; + console.log(`[OCL-ValueSet] ValueSet expansion enqueued: ${cacheKey}`); + OCLBackgroundJobQueue.enqueue( + jobKey, + 'ValueSet expansion', + async () => { + await this.#runBackgroundExpansion(vs, cacheKey, paramsKey, queuedJobSize); + }, + { + jobId: vs.url || cacheKey, + getProgress: () => this.#backgroundExpansionProgressSnapshot(cacheKey), + resolveJobSize: async () => { + const meta = this.#getCollectionMeta(vs); + queuedJobSize = await this.#fetchConceptCountFromHeaders(meta?.conceptsUrl || null); + return queuedJobSize; + } + } + ); + } + + async #runBackgroundExpansion(vs, cacheKey, paramsKey = 'default', knownConceptCount = null) { + console.log(`[OCL-ValueSet] ValueSet expansion started: ${cacheKey}`); + const progressState = { processed: 0, total: null }; + this.backgroundExpansionProgress.set(cacheKey, progressState); + try { + await this.#ensureComposeIncludes(vs); + + const meta = this.#getCollectionMeta(vs); + const resolvedTotal = Number.isFinite(knownConceptCount) && knownConceptCount >= 0 + ? knownConceptCount + : await this.#fetchConceptCountFromHeaders(meta?.conceptsUrl || null); + progressState.total = resolvedTotal; + const sources = meta ? await this.#fetchCollectionSources(meta) : []; + for (const source of sources || []) { + OCLSourceCodeSystemFactory.scheduleBackgroundLoadByKey( + source.system, + source.version || null, + 'valueset-expansion' + ); + } + + const expansion = await this.#buildBackgroundExpansion(vs, progressState); + if (!expansion) { + return; + } + + progressState.processed = expansion.total || progressState.processed; + if (typeof progressState.total !== 'number' || !Number.isFinite(progressState.total) || progressState.total <= 0) { + progressState.total = expansion.total || 0; + } + + const metadataSignature = this.#valueSetMetadataSignature(vs); + const dependencyChecksums = this.#valueSetDependencyChecksums(vs); + + // Compute custom fingerprint and compare with cold cache + const newFingerprint = computeValueSetExpansionFingerprint(expansion); + const oldFingerprint = this.valueSetFingerprints.get(cacheKey); + + if (oldFingerprint && newFingerprint === oldFingerprint) { + console.log(`[OCL-ValueSet] ValueSet expansion fingerprint unchanged: ${cacheKey} (fingerprint=${newFingerprint?.substring(0, 8)})`); + } else { + if (oldFingerprint) { + console.log(`[OCL-ValueSet] ValueSet expansion fingerprint changed: ${cacheKey} (${oldFingerprint?.substring(0, 8)} -> ${newFingerprint?.substring(0, 8)})`); + console.log(`[OCL-ValueSet] Replacing cold cache with new hot cache: ${cacheKey}`); + } else { + console.log(`[OCL-ValueSet] Computed fingerprint for ValueSet expansion: ${cacheKey} (fingerprint=${newFingerprint?.substring(0, 8)})`); + } + + // Save to cold cache + const savedFingerprint = await this.#saveColdCacheForValueSet(vs, expansion, metadataSignature, dependencyChecksums, paramsKey); + if (savedFingerprint) { + this.valueSetFingerprints.set(cacheKey, savedFingerprint); + } + } + + this.backgroundExpansionCache.set(cacheKey, { + expansion, + metadataSignature, + dependencyChecksums, + createdAt: Date.now() + }); + // Keep expansions in provider-managed cache only. + // Inline expansion on ValueSet bypasses $expand filtering in worker pipeline. + + console.log(`[OCL-ValueSet] ValueSet expansion completed and cached: ${cacheKey}`); + console.log(`[OCL-ValueSet] ValueSet now available in cache: ${cacheKey}`); + } catch (error) { + console.error(`[OCL-ValueSet] ValueSet background expansion failed: ${cacheKey}: ${error.message}`); + } finally { + this.backgroundExpansionProgress.delete(cacheKey); + } + } + + async #buildBackgroundExpansion(vs, progressState = null) { + const meta = this.#getCollectionMeta(vs); + if (!meta || !meta.conceptsUrl) { + return null; + } + + const contains = []; + let offset = 0; + + // Pull all concepts in fixed-size pages until exhausted. + // eslint-disable-next-line no-constant-condition + while (true) { + const batch = await this.#fetchCollectionConcepts(meta, { + count: CONCEPT_PAGE_SIZE, + offset, + activeOnly: false, + filter: null, + languageCodes: [] + }); + + const entries = Array.isArray(batch?.contains) ? batch.contains : []; + if (entries.length === 0) { + break; + } + + for (const entry of entries) { + if (!entry?.system || !entry?.code) { + continue; + } + + const out = { + system: entry.system, + code: entry.code + }; + if (entry.display) { + out.display = entry.display; + } + if (entry.definition) { + out.definition = entry.definition; + } + if (entry.inactive === true) { + out.inactive = true; + } + if (Array.isArray(entry.designation) && entry.designation.length > 0) { + out.designation = entry.designation + .filter(d => d && d.value) + .map(d => ({ + language: d.language, + value: d.value + })); + } + contains.push(out); + } + + if (progressState) { + progressState.processed = contains.length; + } + + if (entries.length < CONCEPT_PAGE_SIZE) { + break; + } + offset += entries.length; + } + + return { + timestamp: new Date().toISOString(), + identifier: `urn:uuid:${crypto.randomUUID()}`, + total: contains.length, + contains + }; } - /** - * Validates search parameters - * @param {Array<{name: string, value: string}>} searchParams - Search parameters to validate - * @protected - */ - _validateSearchParams(searchParams) { - if (!Array.isArray(searchParams)) { - throw new Error('Search parameters must be an array'); + async #findCollectionByCanonical(canonicalUrl, version) { + const lookupKey = `${this.org || '*'}|${canonicalUrl}|${version || ''}`; + if (this.collectionByCanonicalCache.has(lookupKey)) { + return this.collectionByCanonicalCache.get(lookupKey); } + if (this.pendingCollectionByCanonicalRequests.has(lookupKey)) { + return this.pendingCollectionByCanonicalRequests.get(lookupKey); + } + + const token = this.#canonicalToken(canonicalUrl); + if (!token) { + return null; + } + + const pending = (async () => { + const organizations = await this.#fetchOrganizationIds(); + const endpoints = organizations.length > 0 + ? organizations.map(orgId => `/orgs/${encodeURIComponent(orgId)}/collections/`) + : ['/collections/']; + + const exactMatches = []; + for (const endpoint of endpoints) { + const response = await this.httpClient.get(endpoint, { + params: { + q: token, + page: 1, + limit: PAGE_SIZE + } + }); + + const payload = response.data; + const items = Array.isArray(payload) + ? payload + : Array.isArray(payload?.results) + ? payload.results + : Array.isArray(payload?.items) + ? payload.items + : Array.isArray(payload?.data) + ? payload.data + : []; + + for (const item of items) { + const itemCanonical = item?.canonical_url || item?.canonicalUrl || item?.url || null; + if (itemCanonical === canonicalUrl) { + exactMatches.push(item); + } + } + } + + let match = null; + if (exactMatches.length > 0) { + if (!version) { + match = exactMatches[0]; + } else { + const exactVersion = exactMatches.find(item => (item.version || null) === version); + if (exactVersion) { + match = exactVersion; + } else if (VersionUtilities.isSemVer(version)) { + const majorMinor = VersionUtilities.getMajMin(version); + if (majorMinor) { + const majorMinorMatch = exactMatches.find(item => (item.version || null) === majorMinor); + if (majorMinorMatch) { + match = majorMinorMatch; + } + } + } + } + } + + this.collectionByCanonicalCache.set(lookupKey, match); + return match; + })(); + + this.pendingCollectionByCanonicalRequests.set(lookupKey, pending); + try { + return await pending; + } finally { + this.pendingCollectionByCanonicalRequests.delete(lookupKey); + } + } + + #valueSetMetadataSignature(vs) { + const meta = this.#getCollectionMeta(vs); + const payload = { + url: vs?.url || null, + version: vs?.version || null, + lastUpdated: vs?.jsonObj?.meta?.lastUpdated || null, + collectionId: meta?.collectionId || null, + conceptsUrl: meta?.conceptsUrl || null, + expansionUrl: meta?.expansionUrl || null, + preferredSource: meta?.preferredSource || null + }; + return JSON.stringify(payload); + } - for (const param of searchParams) { - if (!param || typeof param !== 'object') { - throw new Error('Each search parameter must be an object'); + #valueSetDependencyChecksums(vs) { + const include = Array.isArray(vs?.jsonObj?.compose?.include) ? vs.jsonObj.compose.include : []; + const checksums = {}; + for (const item of include) { + const system = item?.system || null; + if (!system) { + continue; } - if (typeof param.name !== 'string' || typeof param.value !== 'string') { - throw new Error('Search parameter must have string name and value properties'); + const version = item?.version || null; + const key = `${system}|${version || ''}`; + checksums[key] = OCLSourceCodeSystemFactory.checksumForResource(system, version); + } + return checksums; + } + + #isCachedExpansionValid(vs, cached) { + if (!cached || typeof cached !== 'object') { + return false; + } + + if (cached.metadataSignature !== this.#valueSetMetadataSignature(vs)) { + return false; + } + + const currentDeps = this.#valueSetDependencyChecksums(vs); + const cachedDeps = cached.dependencyChecksums || {}; + const currentKeys = Object.keys(currentDeps).sort(); + const cachedKeys = Object.keys(cachedDeps).sort(); + + if (currentKeys.length !== cachedKeys.length) { + return false; + } + + for (let i = 0; i < currentKeys.length; i++) { + if (currentKeys[i] !== cachedKeys[i]) { + return false; + } + if ((currentDeps[currentKeys[i]] || null) !== (cachedDeps[cachedKeys[i]] || null)) { + return false; } } + + return true; } - /** - * Validates URL and version parameters - * @param {string} url - URL to validate - * @param {string} version - Version to validate - * @protected - */ - _validateFetchParams(url, version) { - if (typeof url !== 'string' || !url.trim()) { - throw new Error('URL must be a non-empty string'); + async #fetchCollectionsForDiscovery() { + const organizations = await this.#fetchOrganizationIds(); + if (organizations.length === 0) { + // Fallback for OCL instances that expose global listing but not org listing. + return await this.#fetchAllPages('/collections/'); } - if (version != null && typeof version !== 'string') { - throw new Error('Version must be a string'); + + const allCollections = []; + const seen = new Set(); + + for (const orgId of organizations) { + const endpoint = `/orgs/${encodeURIComponent(orgId)}/collections/`; + const collections = await this.#fetchAllPages(endpoint); + for (const collection of collections) { + if (!collection || typeof collection !== 'object') { + continue; + } + const key = this.#collectionIdentity(collection); + if (seen.has(key)) { + continue; + } + seen.add(key); + allCollections.push(collection); + } } + + return allCollections; + } + + async #fetchOrganizationIds() { + const endpoint = '/orgs/'; + console.log(`[OCL-ValueSet] Loading organizations from: ${this.baseUrl}${endpoint}`); + const orgs = await this.#fetchAllPages(endpoint); + + const ids = []; + const seen = new Set(); + for (const org of orgs || []) { + if (!org || typeof org !== 'object') { + continue; + } + + const id = org.id || org.mnemonic || org.short_code || org.shortCode || org.name || null; + if (!id) { + continue; + } + + const normalized = String(id).trim(); + if (!normalized || seen.has(normalized)) { + continue; + } + + seen.add(normalized); + ids.push(normalized); + } + + if (ids.length === 0 && this.org) { + ids.push(this.org); + } + + return ids; + } + + #collectionIdentity(collection) { + if (!collection || typeof collection !== 'object') { + return '__invalid__'; + } + + const owner = collection.owner || ''; + const canonical = collection.canonical_url || collection.canonicalUrl || ''; + const id = collection.id || collection.short_code || collection.shortCode || collection.name || ''; + return `${owner}|${canonical}|${id}`; + } + + #canonicalToken(canonicalUrl) { + if (!canonicalUrl || typeof canonicalUrl !== 'string') { + return null; + } + + const trimmed = canonicalUrl.trim(); + if (!trimmed) { + return null; + } + + const parts = trimmed.replace(/\/+$/, '').split('/').filter(Boolean); + if (parts.length === 0) { + return null; + } + + return parts[parts.length - 1]; + } + + async #fetchConceptPage(conceptsUrl, page, limit, filter = null) { + try { + const cacheKey = `${conceptsUrl}|p=${page}|l=${limit}|q=${filter || ''}|verbose=1`; + if (this.collectionConceptPageCache.has(cacheKey)) { + const cached = this.collectionConceptPageCache.get(cacheKey); + const items = Array.isArray(cached) + ? cached + : Array.isArray(cached?.items) + ? cached.items + : []; + return { + items, + reportedTotal: this.#extractTotalFromPayload(cached?.payload || null) + }; + } + if (this.pendingCollectionConceptPageRequests.has(cacheKey)) { + return this.pendingCollectionConceptPageRequests.get(cacheKey); + } + + const pending = (async () => { + const params = { page, limit, verbose: true }; + if (filter) { + params.q = filter; + } + const response = await this.httpClient.get(conceptsUrl, { params }); + const payload = response.data; + const items = Array.isArray(payload) + ? payload + : Array.isArray(payload?.results) + ? payload.results + : Array.isArray(payload?.items) + ? payload.items + : Array.isArray(payload?.data) + ? payload.data + : []; + + this.collectionConceptPageCache.set(cacheKey, { items, payload }); + return { + items, + reportedTotal: this.#extractTotalFromPayload(payload) + }; + })(); + + this.pendingCollectionConceptPageRequests.set(cacheKey, pending); + try { + return await pending; + } finally { + this.pendingCollectionConceptPageRequests.delete(cacheKey); + } + } catch (error) { + console.error(`[OCL-ValueSet] #fetchConceptPage ERROR: ${error.message}`); + throw error; + } + } + + #buildRemoteQuery(filter) { + if (!filter || typeof filter !== 'string') { + return null; + } + + const tokens = filter + .split(/\s+or\s+|\||&|\s+/i) + .map(t => t.trim()) + .filter(Boolean) + .map(t => (t.startsWith('-') || t.startsWith('!')) ? t.substring(1) : t) + .map(t => t.replace(/[%*?]/g, '')) + .map(t => t.replace(/[^\p{L}\p{N}]+/gu, '')) + .filter(t => t.length >= 3); + + if (tokens.length === 0) { + return null; + } + + tokens.sort((a, b) => b.length - a.length); + return tokens[0]; + } + + #normalizeLanguageCodes(languageCodes) { + if (!Array.isArray(languageCodes)) { + return []; + } + + const normalized = []; + for (const code of languageCodes) { + if (!code || typeof code !== 'string') { + continue; + } + normalized.push(code.toLowerCase()); + } + return normalized; + } + + #backgroundExpansionProgressSnapshot(cacheKey) { + const progress = this.backgroundExpansionProgress.get(cacheKey); + if (!progress) { + return null; + } + + const processed = progress.processed; + const total = progress.total; + if ( + typeof processed === 'number' && + Number.isFinite(processed) && + typeof total === 'number' && + Number.isFinite(total) && + total > 0 + ) { + return { processed, total }; + } + + return null; + } + + #extractTotalFromPayload(payload) { + if (!payload || typeof payload !== 'object' || Array.isArray(payload)) { + return null; + } + + const candidates = [ + payload.total, + payload.total_count, + payload.totalCount, + payload.num_found, + payload.numFound, + payload.count + ]; + + for (const candidate of candidates) { + if (typeof candidate === 'number' && Number.isFinite(candidate) && candidate >= 0) { + return candidate; + } + } + + return null; + } + + async #fetchConceptCountFromHeaders(conceptsUrl) { + if (!conceptsUrl) { + return null; + } + + try { + const response = await this.httpClient.get(conceptsUrl, { + params: { + limit: 1 + } + }); + return this.#extractNumFoundFromHeaders(response?.headers); + } catch (error) { + return null; + } + } + + #extractNumFoundFromHeaders(headers) { + if (!headers || typeof headers !== 'object') { + return null; + } + + const raw = headers.num_found ?? headers['num-found'] ?? headers.Num_Found ?? null; + const parsed = typeof raw === 'number' ? raw : Number.parseInt(raw, 10); + if (!Number.isFinite(parsed) || parsed < 0) { + return null; + } + return parsed; + } + + #languageRank(languageCode, preferredLanguageCodes) { + if (!languageCode) { + return 1000; + } + + const normalized = String(languageCode).toLowerCase(); + for (let i = 0; i < preferredLanguageCodes.length; i++) { + const preferred = preferredLanguageCodes[i]; + if (normalized === preferred || normalized.startsWith(`${preferred}-`) || preferred.startsWith(`${normalized}-`)) { + return i; + } + } + + if (normalized === 'en' || normalized.startsWith('en-')) { + return preferredLanguageCodes.length; + } + + return preferredLanguageCodes.length + 1; + } + + #extractLocalizedNames(concept, preferredLanguageCodes) { + const names = Array.isArray(concept?.names) ? concept.names : []; + const unique = new Map(); + + for (const item of names) { + const value = item?.name; + if (!value || typeof value !== 'string') { + continue; + } + + const language = typeof item?.locale === 'string' && item.locale.trim() ? item.locale.trim().toLowerCase() : null; + const key = `${language || ''}|${value}`; + if (unique.has(key)) { + continue; + } + + unique.set(key, { + language: language || undefined, + value, + localePreferred: item?.locale_preferred === true, + nameType: item?.name_type || '' + }); + } + + const designation = Array.from(unique.values()) + .sort((a, b) => { + const rankDiff = this.#languageRank(a.language, preferredLanguageCodes) - this.#languageRank(b.language, preferredLanguageCodes); + if (rankDiff !== 0) { + return rankDiff; + } + if (a.localePreferred !== b.localePreferred) { + return a.localePreferred ? -1 : 1; + } + const aFs = /fully\s*-?\s*specified/i.test(a.nameType); + const bFs = /fully\s*-?\s*specified/i.test(b.nameType); + if (aFs !== bFs) { + return aFs ? -1 : 1; + } + return a.value.localeCompare(b.value); + }) + .map(({ language, value }) => ({ + language, + value + })); + + return { + display: designation.length > 0 ? designation[0].value : null, + designation + }; + } + + #extractLocalizedDefinitions(concept, preferredLanguageCodes) { + const descriptions = Array.isArray(concept?.descriptions) ? concept.descriptions : []; + const unique = new Map(); + + for (const item of descriptions) { + const value = item?.description; + if (!value || typeof value !== 'string') { + continue; + } + + const language = typeof item?.locale === 'string' && item.locale.trim() ? item.locale.trim().toLowerCase() : null; + const key = `${language || ''}|${value}`; + if (unique.has(key)) { + continue; + } + + unique.set(key, { + language: language || undefined, + value, + localePreferred: item?.locale_preferred === true, + descriptionType: item?.description_type || '' + }); + } + + const definitions = Array.from(unique.values()) + .sort((a, b) => { + const rankDiff = this.#languageRank(a.language, preferredLanguageCodes) - this.#languageRank(b.language, preferredLanguageCodes); + if (rankDiff !== 0) { + return rankDiff; + } + if (a.localePreferred !== b.localePreferred) { + return a.localePreferred ? -1 : 1; + } + const aDef = /definition/i.test(a.descriptionType); + const bDef = /definition/i.test(b.descriptionType); + if (aDef !== bDef) { + return aDef ? -1 : 1; + } + return a.value.localeCompare(b.value); + }) + .map(({ language, value }) => ({ + language, + value + })); + + return { + definition: definitions.length > 0 ? definitions[0].value : null, + definitions + }; + } + + async #fetchCollectionSourceKeys(conceptsUrl, defaultOwner) { + const keys = new Map(); + let page = 1; + + // eslint-disable-next-line no-constant-condition + while (true) { + const response = await this.httpClient.get(conceptsUrl, { params: { page, limit: CONCEPT_PAGE_SIZE } }); + const payload = response.data; + + let items = []; + if (Array.isArray(payload)) { + items = payload; + } else if (payload && typeof payload === 'object') { + items = Array.isArray(payload.results) + ? payload.results + : Array.isArray(payload.items) + ? payload.items + : Array.isArray(payload.data) + ? payload.data + : []; + } + + if (!items || items.length === 0) { + break; + } + + for (const concept of items) { + const owner = concept.owner || defaultOwner || null; + const source = concept.source || null; + if (owner && source) { + keys.set(`${owner}|${source}`, { owner, source }); + } + } + + if (items.length < CONCEPT_PAGE_SIZE) { + break; + } + + page += 1; + } + + return Array.from(keys.values()); + } + + async #getSourceCanonicalUrl(owner, source) { + const key = `${owner}|${source}`; + if (this.sourceCanonicalCache.has(key)) { + return this.sourceCanonicalCache.get(key); + } + if (this.pendingSourceCanonicalRequests.has(key)) { + return this.pendingSourceCanonicalRequests.get(key); + } + + const path = `/orgs/${encodeURIComponent(owner)}/sources/${encodeURIComponent(source)}/`; + const pending = (async () => { + try { + const response = await this.httpClient.get(path); + const data = response.data || {}; + const canonicalUrl = data.canonical_url || data.canonicalUrl || data.url || source; + this.sourceCanonicalCache.set(key, canonicalUrl); + return canonicalUrl; + } catch (error) { + this.sourceCanonicalCache.set(key, source); + return source; + } + })(); + + this.pendingSourceCanonicalRequests.set(key, pending); + try { + return await pending; + } finally { + this.pendingSourceCanonicalRequests.delete(key); + } + } + + #matches(json, params) { + for (const [name, value] of Object.entries(params)) { + if (!value) { + continue; + } + + switch (name) { + case 'url': + if ((json.url || '').toLowerCase() !== value) { + return false; + } + break; + case 'system': + if (!json.compose?.include?.some(i => (i.system || '').toLowerCase().includes(value))) { + return false; + } + break; + case 'identifier': { + const identifiers = Array.isArray(json.identifier) ? json.identifier : (json.identifier ? [json.identifier] : []); + const match = identifiers.some(i => (i.system || '').toLowerCase().includes(value) || (i.value || '').toLowerCase().includes(value)); + if (!match) { + return false; + } + break; + } + default: { + const field = json[name]; + if (field == null || !String(field).toLowerCase().includes(value)) { + return false; + } + break; + } + } + } + return true; + } + + async #fetchAllPages(path) { + const results = []; + let page = 1; + + // eslint-disable-next-line no-constant-condition + while (true) { + const response = await this.httpClient.get(path, { params: { page, limit: PAGE_SIZE } }); + const payload = response.data; + + let items = []; + if (Array.isArray(payload)) { + items = payload; + } else if (payload && typeof payload === 'object') { + items = Array.isArray(payload.results) + ? payload.results + : Array.isArray(payload.items) + ? payload.items + : Array.isArray(payload.data) + ? payload.data + : []; + } + + if (!items || items.length === 0) { + break; + } + + results.push(...items); + + if (items.length < PAGE_SIZE) { + break; + } + + page += 1; + } + + return results; + } + + #toIsoDate(value) { + if (!value) { + return null; + } + const date = new Date(value); + if (Number.isNaN(date.getTime())) { + return null; + } + return date.toISOString(); + } + + #normalizeFilter(filter) { + if (typeof filter !== 'string') { + return null; + } + const normalized = filter.trim().toLowerCase(); + return normalized.length > 0 ? normalized : null; + } + + #conceptMatchesFilter(searchableText, code, display, definition, filter, filterMatcher) { + if (!filter) { + return true; + } + + const codeText = code ? String(code).toLowerCase() : ''; + const displayText = display ? String(display).toLowerCase() : ''; + const definitionText = definition ? String(definition).toLowerCase() : ''; + + // FHIR allows terminology-server-defined behavior; guarantee baseline contains matching. + if (codeText.includes(filter) || displayText.includes(filter) || definitionText.includes(filter)) { + return true; + } + + // Preserve token/prefix behavior already provided by SearchFilterText. + return !!(filterMatcher && filterMatcher.passes(searchableText)); } } module.exports = { - AbstractValueSetProvider + OCLValueSetProvider }; \ No newline at end of file From 070272582709d6f70a0e9d1a1447704ea14101ba Mon Sep 17 00:00:00 2001 From: Italo Macedo Date: Sat, 7 Mar 2026 03:42:00 -0300 Subject: [PATCH 06/15] Providers in a modularized structure --- tx/ocl/cache/cache-paths.js | 32 ++ tx/ocl/cache/cache-utils.js | 43 ++ tx/ocl/cm-ocl.js | 101 +--- tx/ocl/cs-ocl.js | 664 ++----------------------- tx/ocl/fingerprint/fingerprint.js | 67 +++ tx/ocl/http/client.js | 31 ++ tx/ocl/http/pagination.js | 98 ++++ tx/ocl/jobs/background-queue.js | 200 ++++++++ tx/ocl/mappers/concept-mapper.js | 66 +++ tx/ocl/model/concept-filter-context.js | 51 ++ tx/ocl/shared/constants.js | 15 + tx/ocl/shared/patches.js | 135 +++++ tx/ocl/vs-ocl.js | 177 ++----- 13 files changed, 813 insertions(+), 867 deletions(-) create mode 100644 tx/ocl/cache/cache-paths.js create mode 100644 tx/ocl/cache/cache-utils.js create mode 100644 tx/ocl/fingerprint/fingerprint.js create mode 100644 tx/ocl/http/client.js create mode 100644 tx/ocl/http/pagination.js create mode 100644 tx/ocl/jobs/background-queue.js create mode 100644 tx/ocl/mappers/concept-mapper.js create mode 100644 tx/ocl/model/concept-filter-context.js create mode 100644 tx/ocl/shared/constants.js create mode 100644 tx/ocl/shared/patches.js diff --git a/tx/ocl/cache/cache-paths.js b/tx/ocl/cache/cache-paths.js new file mode 100644 index 00000000..c587cf66 --- /dev/null +++ b/tx/ocl/cache/cache-paths.js @@ -0,0 +1,32 @@ +const path = require('path'); + +const CACHE_BASE_DIR = path.join(process.cwd(), 'data', 'terminology-cache', 'ocl'); +const CACHE_CS_DIR = path.join(CACHE_BASE_DIR, 'codesystems'); +const CACHE_VS_DIR = path.join(CACHE_BASE_DIR, 'valuesets'); + +function sanitizeFilename(text) { + if (!text || typeof text !== 'string') { + return 'unknown'; + } + return text + .replace(/[^a-zA-Z0-9._-]/g, '_') + .replace(/_+/g, '_') + .substring(0, 200); +} + +function getCacheFilePath(baseDir, canonicalUrl, version = null, paramsKey = null) { + const filename = sanitizeFilename(canonicalUrl) + + (version ? `_${sanitizeFilename(version)}` : '') + + (paramsKey && paramsKey !== 'default' ? `_p_${sanitizeFilename(paramsKey)}` : '') + + '.json'; + + return path.join(baseDir, filename); +} + +module.exports = { + CACHE_BASE_DIR, + CACHE_CS_DIR, + CACHE_VS_DIR, + sanitizeFilename, + getCacheFilePath +}; diff --git a/tx/ocl/cache/cache-utils.js b/tx/ocl/cache/cache-utils.js new file mode 100644 index 00000000..b0a89af5 --- /dev/null +++ b/tx/ocl/cache/cache-utils.js @@ -0,0 +1,43 @@ +const fs = require('fs/promises'); +const fsSync = require('fs'); + +async function ensureCacheDirectories(...dirs) { + for (const dir of dirs) { + if (!dir) { + continue; + } + + try { + await fs.mkdir(dir, { recursive: true }); + } catch (error) { + console.error('[OCL] Failed to create cache directory:', dir, error.message); + } + } +} + +function getColdCacheAgeMs(cacheFilePath, logPrefix = '[OCL]') { + try { + const stats = fsSync.statSync(cacheFilePath); + if (!stats || !Number.isFinite(stats.mtimeMs)) { + return null; + } + + return Math.max(0, Date.now() - stats.mtimeMs); + } catch (error) { + if (error && error.code !== 'ENOENT') { + console.error(`${logPrefix} Failed to inspect cold cache file ${cacheFilePath}: ${error.message}`); + } + return null; + } +} + +function formatCacheAgeMinutes(ageMs) { + const minutes = Math.max(1, Math.round(ageMs / 60000)); + return `${minutes} minute${minutes === 1 ? '' : 's'}`; +} + +module.exports = { + ensureCacheDirectories, + getColdCacheAgeMs, + formatCacheAgeMinutes +}; diff --git a/tx/ocl/cm-ocl.js b/tx/ocl/cm-ocl.js index 23b76517..db0c5fb5 100644 --- a/tx/ocl/cm-ocl.js +++ b/tx/ocl/cm-ocl.js @@ -1,9 +1,9 @@ -const axios = require('axios'); const { AbstractConceptMapProvider } = require('../cm/cm-api'); const { ConceptMap } = require('../library/conceptmap'); +const { PAGE_SIZE } = require('./shared/constants'); +const { createOclHttpClient } = require('./http/client'); +const { fetchAllPages, extractItemsAndNext } = require('./http/pagination'); -const DEFAULT_BASE_URL = 'https://oclapi2.ips.hsl.org.br'; -const PAGE_SIZE = 100; const DEFAULT_MAX_SEARCH_PAGES = 10; class OCLConceptMapProvider extends AbstractConceptMapProvider { @@ -11,26 +11,11 @@ class OCLConceptMapProvider extends AbstractConceptMapProvider { super(); const options = typeof config === 'string' ? { baseUrl: config } : (config || {}); - this.baseUrl = (options.baseUrl || DEFAULT_BASE_URL).replace(/\/+$/, ''); this.org = options.org || null; this.maxSearchPages = options.maxSearchPages || DEFAULT_MAX_SEARCH_PAGES; - - const headers = { - Accept: 'application/json', - 'User-Agent': 'FHIRSmith-OCL-Provider/1.0' - }; - - if (options.token) { - headers.Authorization = options.token.startsWith('Token ') || options.token.startsWith('Bearer ') - ? options.token - : `Token ${options.token}`; - } - - this.httpClient = axios.create({ - baseURL: this.baseUrl, - timeout: options.timeout || 30000, - headers - }); + const http = createOclHttpClient(options); + this.baseUrl = http.baseUrl; + this.httpClient = http.client; this.conceptMapMap = new Map(); this._idMap = new Map(); @@ -366,76 +351,16 @@ class OCLConceptMapProvider extends AbstractConceptMapProvider { } async #fetchAllPages(path, params = {}, maxPages = this.maxSearchPages) { - const results = []; - let page = 1; - let nextPath = path; - let pageCount = 0; - let usePageMode = true; - - while (nextPath && pageCount < maxPages) { - const response = usePageMode - ? await this.httpClient.get(path, { params: { ...params, page, limit: PAGE_SIZE } }) - : await this.httpClient.get(nextPath); - - if (Array.isArray(response.data)) { - results.push(...response.data); - pageCount += 1; - if (response.data.length < PAGE_SIZE) { - break; - } - page += 1; - nextPath = path; - continue; - } - - const { items, next } = this.#extractItemsAndNext(response.data); - results.push(...items); - pageCount += 1; - - if (next) { - usePageMode = false; - nextPath = next; - continue; - } - - if (usePageMode && items.length >= PAGE_SIZE && pageCount < maxPages) { - page += 1; - nextPath = path; - } else { - break; - } - } - - return results; + return await fetchAllPages(this.httpClient, path, { + params, + pageSize: PAGE_SIZE, + maxPages, + baseUrl: this.baseUrl + }); } #extractItemsAndNext(payload) { - if (Array.isArray(payload)) { - return { items: payload, next: null }; - } - - if (!payload || typeof payload !== 'object') { - return { items: [], next: null }; - } - - const items = Array.isArray(payload.results) - ? payload.results - : Array.isArray(payload.items) - ? payload.items - : Array.isArray(payload.data) - ? payload.data - : []; - - const next = payload.next || null; - if (!next) { - return { items, next: null }; - } - - if (next.startsWith(this.baseUrl)) { - return { items, next: next.replace(this.baseUrl, '') }; - } - - return { items, next }; + return extractItemsAndNext(payload, this.baseUrl); } #extractMappingId(url) { diff --git a/tx/ocl/cs-ocl.js b/tx/ocl/cs-ocl.js index a4979f48..cdd13880 100644 --- a/tx/ocl/cs-ocl.js +++ b/tx/ocl/cs-ocl.js @@ -1,468 +1,30 @@ -const axios = require('axios'); const fs = require('fs/promises'); -const fsSync = require('fs'); -const crypto = require('crypto'); -const path = require('path'); const { AbstractCodeSystemProvider } = require('../cs/cs-provider-api'); const { CodeSystemProvider, CodeSystemFactoryProvider, CodeSystemContentMode, FilterExecutionContext } = require('../cs/cs-api'); const { CodeSystem } = require('../library/codesystem'); const { SearchFilterText } = require('../library/designations'); - -const DEFAULT_BASE_URL = 'https://oclapi2.ips.hsl.org.br'; -const PAGE_SIZE = 100; -const CONCEPT_PAGE_SIZE = 1000; -const COLD_CACHE_FRESHNESS_MS = 60 * 60 * 1000; -const OCL_CODESYSTEM_MARKER_EXTENSION = 'http://fhir.org/FHIRsmith/StructureDefinition/ocl-codesystem'; -const OCL_SEARCH_PATCH_FLAG = Symbol.for('fhirsmith.ocl.search.codesystem.code.patch'); - -function hasOCLCodeSystemMarker(resource) { - const extensions = Array.isArray(resource?.extension) ? resource.extension : []; - return extensions.some(ext => ext && ext.url === OCL_CODESYSTEM_MARKER_EXTENSION); -} - -function filterConceptTreeByCode(concepts, wantedCode) { - if (!Array.isArray(concepts) || concepts.length === 0) { - return []; - } - - const matches = []; - for (const concept of concepts) { - if (!concept || typeof concept !== 'object') { - continue; - } - - const childMatches = filterConceptTreeByCode(concept.concept, wantedCode); - const isSelfMatch = concept.code != null && String(concept.code) === wantedCode; - if (!isSelfMatch && childMatches.length === 0) { - continue; - } - - const clone = { ...concept }; - if (childMatches.length > 0) { - clone.concept = childMatches; - } else { - delete clone.concept; - } - matches.push(clone); - } - - return matches; -} - -function filterOCLCodeSystemResourceByCode(resource, code) { - if (!resource || typeof resource !== 'object') { - return resource; - } - - const filteredConcepts = filterConceptTreeByCode(resource.concept, code); - return { - ...resource, - concept: filteredConcepts - }; -} - -function patchSearchWorkerForOCLCodeFiltering() { - let SearchWorker; - try { - SearchWorker = require('../workers/search'); - } catch (_error) { - return; - } - - if (!SearchWorker || !SearchWorker.prototype) { - return; - } - - const proto = SearchWorker.prototype; - if (proto[OCL_SEARCH_PATCH_FLAG] === true || typeof proto.searchCodeSystems !== 'function') { - return; - } - - const originalSearchCodeSystems = proto.searchCodeSystems; - proto.searchCodeSystems = function patchedSearchCodeSystems(params) { - const matches = originalSearchCodeSystems.call(this, params); - const requestedCode = params?.code == null ? '' : String(params.code); - - if (!requestedCode) { - return matches; - } - - const filtered = []; - for (const resource of matches) { - if (!hasOCLCodeSystemMarker(resource)) { - filtered.push(resource); - continue; - } - - const projected = filterOCLCodeSystemResourceByCode(resource, requestedCode); - if (Array.isArray(projected?.concept) && projected.concept.length > 0) { - filtered.push(projected); - } - } - - return filtered; - }; - - Object.defineProperty(proto, OCL_SEARCH_PATCH_FLAG, { - value: true, - writable: false, - configurable: false, - enumerable: false - }); -} +const { PAGE_SIZE, CONCEPT_PAGE_SIZE, COLD_CACHE_FRESHNESS_MS, OCL_CODESYSTEM_MARKER_EXTENSION } = require('./shared/constants'); +const { createOclHttpClient } = require('./http/client'); +const { fetchAllPages, extractItemsAndNext } = require('./http/pagination'); +const { CACHE_CS_DIR, CACHE_VS_DIR, getCacheFilePath } = require('./cache/cache-paths'); +const { ensureCacheDirectories, getColdCacheAgeMs, formatCacheAgeMinutes } = require('./cache/cache-utils'); +const { computeCodeSystemFingerprint } = require('./fingerprint/fingerprint'); +const { OCLBackgroundJobQueue } = require('./jobs/background-queue'); +const { OCLConceptFilterContext } = require('./model/concept-filter-context'); +const { toConceptContext } = require('./mappers/concept-mapper'); +const { patchSearchWorkerForOCLCodeFiltering } = require('./shared/patches'); patchSearchWorkerForOCLCodeFiltering(); -// Cold cache configuration -const CACHE_BASE_DIR = path.join(process.cwd(), 'data', 'terminology-cache', 'ocl'); -const CACHE_CS_DIR = path.join(CACHE_BASE_DIR, 'codesystems'); -const CACHE_VS_DIR = path.join(CACHE_BASE_DIR, 'valuesets'); - -// Cache file utilities -async function ensureCacheDirectories() { - try { - await fs.mkdir(CACHE_CS_DIR, { recursive: true }); - await fs.mkdir(CACHE_VS_DIR, { recursive: true }); - } catch (error) { - console.error('[OCL] Failed to create cache directories:', error.message); - } -} - -// CodeSystem fingerprint computation -function computeCodeSystemFingerprint(concepts) { - if (!Array.isArray(concepts) || concepts.length === 0) { - return null; - } - - // Normalize concepts to deterministic strings - const normalized = concepts - .map(concept => { - if (!concept || !concept.code) { - return null; - } - const code = String(concept.code || ''); - const display = String(concept.display || ''); - const definition = String(concept.definition || ''); - const retired = concept.retired === true ? '1' : '0'; - return `${code}|${display}|${definition}|${retired}`; - }) - .filter(Boolean) - .sort(); - - // Compute SHA256 hash - const hash = crypto.createHash('sha256'); - for (const item of normalized) { - hash.update(item); - hash.update('\n'); - } - return hash.digest('hex'); -} - -function sanitizeFilename(text) { - if (!text || typeof text !== 'string') { - return 'unknown'; - } - return text - .replace(/[^a-zA-Z0-9._-]/g, '_') - .replace(/_+/g, '_') - .substring(0, 200); -} - -function getCacheFilePath(baseDir, canonicalUrl, version = null) { - const filename = sanitizeFilename(canonicalUrl) + (version ? `_${sanitizeFilename(version)}` : '') + '.json'; - return path.join(baseDir, filename); -} - -function getColdCacheAgeMs(cacheFilePath) { - try { - const stats = fsSync.statSync(cacheFilePath); - if (!stats || !Number.isFinite(stats.mtimeMs)) { - return null; - } - return Math.max(0, Date.now() - stats.mtimeMs); - } catch (error) { - if (error && error.code !== 'ENOENT') { - console.error(`[OCL] Failed to inspect cold cache file ${cacheFilePath}: ${error.message}`); - } - return null; - } -} - -function formatCacheAgeMinutes(ageMs) { - const minutes = Math.max(1, Math.round(ageMs / 60000)); - return `${minutes} minute${minutes === 1 ? '' : 's'}`; -} - -class OCLConceptFilterContext { - constructor() { - this.concepts = []; - this.currentIndex = -1; - } - - add(concept, rating = 0) { - this.concepts.push({ concept, rating }); - } - - sort() { - this.concepts.sort((a, b) => b.rating - a.rating); - } - - size() { - return this.concepts.length; - } - - hasMore() { - return this.currentIndex + 1 < this.concepts.length; - } - - next() { - if (!this.hasMore()) { - return null; - } - this.currentIndex += 1; - return this.concepts[this.currentIndex].concept; - } - - reset() { - this.currentIndex = -1; - } - - findConceptByCode(code) { - for (const item of this.concepts) { - if (item.concept && item.concept.code === code) { - return item.concept; - } - } - return null; - } - - containsConcept(concept) { - return this.concepts.some(item => item.concept === concept); - } -} - -class OCLBackgroundJobQueue { - static MAX_CONCURRENT = 2; - static HEARTBEAT_INTERVAL_MS = 30000; - static UNKNOWN_JOB_SIZE = Number.MAX_SAFE_INTEGER; - static pendingJobs = []; - static activeCount = 0; - static queuedOrRunningKeys = new Set(); - static activeJobs = new Map(); - static heartbeatTimer = null; - static enqueueSequence = 0; - - static enqueue(jobKey, jobType, runJob, options = {}) { - if (!jobKey || typeof runJob !== 'function') { - return false; - } - - if (this.queuedOrRunningKeys.has(jobKey)) { - return false; - } - - this.queuedOrRunningKeys.add(jobKey); - const resolveAndEnqueue = async () => { - const resolvedSize = await this.#resolveJobSize(options); - const normalizedSize = this.#normalizeJobSize(resolvedSize); - this.#insertPendingJobOrdered({ - jobKey, - jobType: jobType || 'background-job', - jobId: options?.jobId || jobKey, - jobSize: normalizedSize, - getProgress: typeof options?.getProgress === 'function' ? options.getProgress : null, - runJob, - enqueueOrder: this.enqueueSequence++ - }); - this.ensureHeartbeatRunning(); - console.log(`[OCL] ${jobType || 'Background job'} enqueued: ${jobKey} (size=${normalizedSize}, queue=${this.pendingJobs.length}, active=${this.activeCount})`); - this.processNext(); - }; - - Promise.resolve() - .then(resolveAndEnqueue) - .catch((error) => { - this.queuedOrRunningKeys.delete(jobKey); - const message = error && error.message ? error.message : String(error); - console.error(`[OCL] Failed to enqueue background job: ${jobType || 'background-job'} ${jobKey}: ${message}`); - }); - - return true; - } - - static async #resolveJobSize(options = {}) { - if (typeof options?.resolveJobSize === 'function') { - try { - return await options.resolveJobSize(); - } catch (_error) { - return this.UNKNOWN_JOB_SIZE; - } - } - - if (options && Object.prototype.hasOwnProperty.call(options, 'jobSize')) { - return options.jobSize; - } - - return this.UNKNOWN_JOB_SIZE; - } - - static #normalizeJobSize(jobSize) { - const parsed = Number.parseInt(jobSize, 10); - if (!Number.isFinite(parsed) || parsed < 0) { - return this.UNKNOWN_JOB_SIZE; - } - return parsed; - } - - static #insertPendingJobOrdered(job) { - let index = this.pendingJobs.findIndex(existing => { - if (existing.jobSize === job.jobSize) { - return existing.enqueueOrder > job.enqueueOrder; - } - return existing.jobSize > job.jobSize; - }); - - if (index < 0) { - index = this.pendingJobs.length; - } - - this.pendingJobs.splice(index, 0, job); - } - - static isQueuedOrRunning(jobKey) { - return this.queuedOrRunningKeys.has(jobKey); - } - - static ensureHeartbeatRunning() { - if (this.heartbeatTimer) { - return; - } - - this.heartbeatTimer = setInterval(() => { - this.logHeartbeat(); - }, this.HEARTBEAT_INTERVAL_MS); - - // Do not keep the process alive only for telemetry logs. - if (typeof this.heartbeatTimer.unref === 'function') { - this.heartbeatTimer.unref(); - } - } - - static logHeartbeat() { - const activeJobs = Array.from(this.activeJobs.values()); - const lines = [ - '[OCL] OCL background status:', - ` active jobs: ${activeJobs.length}`, - ` queued jobs: ${this.pendingJobs.length}` - ]; - - activeJobs.forEach((job, index) => { - lines.push(''); - lines.push(` job ${index + 1}:`); - lines.push(` type: ${job.jobType || 'background-job'}`); - lines.push(` id: ${job.jobId || job.jobKey}`); - lines.push(` size: ${job.jobSize}`); - lines.push(` progress: ${this.formatProgress(job.getProgress)}`); - }); - - console.log(lines.join('\n')); - } - - static formatProgress(getProgress) { - if (typeof getProgress !== 'function') { - return 'unknown'; - } - - try { - const progress = getProgress(); - if (typeof progress === 'number' && Number.isFinite(progress)) { - const bounded = Math.max(0, Math.min(100, progress)); - return `${Math.round(bounded)}%`; - } - - if (progress && typeof progress === 'object') { - if (typeof progress.percentage === 'number' && Number.isFinite(progress.percentage)) { - const bounded = Math.max(0, Math.min(100, progress.percentage)); - return `${Math.round(bounded)}%`; - } - - if ( - typeof progress.processed === 'number' && - Number.isFinite(progress.processed) && - typeof progress.total === 'number' && - Number.isFinite(progress.total) && - progress.total > 0 - ) { - const ratio = progress.processed / progress.total; - const bounded = Math.max(0, Math.min(100, ratio * 100)); - return `${Math.round(bounded)}%`; - } - } - } catch (error) { - return 'unknown'; - } - - return 'unknown'; - } - - static processNext() { - while (this.activeCount < this.MAX_CONCURRENT && this.pendingJobs.length > 0) { - const job = this.pendingJobs.shift(); - this.activeCount += 1; - this.activeJobs.set(job.jobKey, { - jobKey: job.jobKey, - jobType: job.jobType, - jobId: job.jobId || job.jobKey, - jobSize: job.jobSize, - getProgress: job.getProgress || null, - startedAt: Date.now() - }); - console.log(`[OCL] Background job started: ${job.jobType} ${job.jobKey} (size=${job.jobSize}, queue=${this.pendingJobs.length}, active=${this.activeCount})`); - - Promise.resolve() - .then(() => job.runJob()) - .then(() => { - console.log(`[OCL] Background job completed: ${job.jobType} ${job.jobKey}`); - }) - .catch((error) => { - const message = error && error.message ? error.message : String(error); - console.error(`[OCL] Background job failed: ${job.jobType} ${job.jobKey}: ${message}`); - }) - .finally(() => { - this.activeCount -= 1; - this.queuedOrRunningKeys.delete(job.jobKey); - this.activeJobs.delete(job.jobKey); - console.log(`[OCL] Background queue status: queue=${this.pendingJobs.length}, active=${this.activeCount}`); - this.processNext(); - }); - } - } -} - class OCLCodeSystemProvider extends AbstractCodeSystemProvider { constructor(config = {}) { super(); const options = typeof config === 'string' ? { baseUrl: config } : (config || {}); - this.baseUrl = (options.baseUrl || DEFAULT_BASE_URL).replace(/\/+$/, ''); this.org = options.org || null; - - const headers = { - Accept: 'application/json', - 'User-Agent': 'FHIRSmith-OCL-Provider/1.0' - }; - - if (options.token) { - headers.Authorization = options.token.startsWith('Token ') || options.token.startsWith('Bearer ') - ? options.token - : `Token ${options.token}`; - } - - this.httpClient = axios.create({ - baseURL: this.baseUrl, - timeout: options.timeout || 30000, - headers - }); + const http = createOclHttpClient(options); + this.baseUrl = http.baseUrl; + this.httpClient = http.client; this._codeSystemsByCanonical = new Map(); this._idToCodeSystem = new Map(); @@ -954,82 +516,24 @@ class OCLCodeSystemProvider extends AbstractCodeSystemProvider { } async #fetchAllPages(path) { - const results = []; - let page = 1; - let nextPath = path; - let usePageMode = true; - - while (nextPath) { - try { - const response = usePageMode - ? await this.httpClient.get(path, { params: { page, limit: PAGE_SIZE } }) - : await this.httpClient.get(nextPath); - - if (Array.isArray(response.data)) { - results.push(...response.data); - if (response.data.length < PAGE_SIZE) { - break; - } - page += 1; - nextPath = path; - continue; - } - - const { items, next } = this.#extractItemsAndNext(response.data); - results.push(...items); - - if (next) { - usePageMode = false; - nextPath = next; - continue; - } - - if (usePageMode && items.length >= PAGE_SIZE) { - page += 1; - nextPath = path; - } else { - break; - } - } catch (error) { - console.error(`[OCL] Fetch error on page ${page}:`, error.message); - if (error.response) { - console.error(`[OCL] HTTP ${error.response.status}: ${error.response.statusText}`); - console.error(`[OCL] Response:`, error.response.data); - } - throw error; + try { + return await fetchAllPages(this.httpClient, path, { + pageSize: PAGE_SIZE, + baseUrl: this.baseUrl, + logger: console, + loggerPrefix: '[OCL]' + }); + } catch (error) { + if (error.response) { + console.error(`[OCL] HTTP ${error.response.status}: ${error.response.statusText}`); + console.error('[OCL] Response:', error.response.data); } + throw error; } - - return results; } #extractItemsAndNext(payload) { - if (Array.isArray(payload)) { - return { items: payload, next: null }; - } - - if (!payload || typeof payload !== 'object') { - return { items: [], next: null }; - } - - const items = Array.isArray(payload.results) - ? payload.results - : Array.isArray(payload.items) - ? payload.items - : Array.isArray(payload.data) - ? payload.data - : []; - - const next = payload.next || null; - if (!next) { - return { items, next: null }; - } - - if (next.startsWith(this.baseUrl)) { - return { items, next: next.replace(this.baseUrl, '') }; - } - - return { items, next }; + return extractItemsAndNext(payload, this.baseUrl); } #toIsoDate(value) { @@ -1641,62 +1145,7 @@ class OCLSourceCodeSystemProvider extends CodeSystemProvider { } #toConceptContext(concept) { - if (!concept || typeof concept !== 'object') { - return null; - } - - const code = concept.code || concept.id || null; - if (!code) { - return null; - } - - return { - code, - display: concept.display_name || concept.display || concept.name || null, - definition: concept.description || concept.definition || null, - retired: concept.retired === true, - designations: this.#extractDesignations(concept) - }; - } - - #extractDesignations(concept) { - const result = []; - const seen = new Set(); - - const add = (language, value) => { - const text = typeof value === 'string' ? value.trim() : ''; - if (!text) { - return; - } - const lang = typeof language === 'string' ? language.trim() : ''; - const key = `${lang}|${text}`; - if (seen.has(key)) { - return; - } - seen.add(key); - result.push({ language: lang, value: text }); - }; - - if (Array.isArray(concept.names)) { - for (const entry of concept.names) { - if (!entry || typeof entry !== 'object') { - continue; - } - add(entry.locale || entry.language || entry.lang || '', entry.name || entry.display_name || entry.display || entry.value || entry.term); - } - } - - if (concept.display_name || concept.display || concept.name) { - add(concept.locale || concept.default_locale || concept.language || '', concept.display_name || concept.display || concept.name); - } - - if (concept.locale_display_names && typeof concept.locale_display_names === 'object') { - for (const [lang, value] of Object.entries(concept.locale_display_names)) { - add(lang, value); - } - } - - return result; + return toConceptContext(concept); } } @@ -1782,7 +1231,7 @@ class OCLSourceCodeSystemFactory extends CodeSystemFactoryProvider { const cacheFilePath = getCacheFilePath(CACHE_CS_DIR, canonicalUrl, version); try { - await ensureCacheDirectories(); + await ensureCacheDirectories(CACHE_CS_DIR, CACHE_VS_DIR); const fingerprint = computeCodeSystemFingerprint(concepts); const cacheData = { @@ -2199,62 +1648,7 @@ class OCLSourceCodeSystemFactory extends CodeSystemFactoryProvider { } #toConceptContext(concept) { - if (!concept || typeof concept !== 'object') { - return null; - } - - const code = concept.code || concept.id || null; - if (!code) { - return null; - } - - return { - code, - display: concept.display_name || concept.display || concept.name || null, - definition: concept.description || concept.definition || null, - retired: concept.retired === true, - designations: this.#extractDesignations(concept) - }; - } - - #extractDesignations(concept) { - const result = []; - const seen = new Set(); - - const add = (language, value) => { - const text = typeof value === 'string' ? value.trim() : ''; - if (!text) { - return; - } - const lang = typeof language === 'string' ? language.trim() : ''; - const key = `${lang}|${text}`; - if (seen.has(key)) { - return; - } - seen.add(key); - result.push({ language: lang, value: text }); - }; - - if (Array.isArray(concept.names)) { - for (const entry of concept.names) { - if (!entry || typeof entry !== 'object') { - continue; - } - add(entry.locale || entry.language || entry.lang || '', entry.name || entry.display_name || entry.display || entry.value || entry.term); - } - } - - if (concept.display_name || concept.display || concept.name) { - add(concept.locale || concept.default_locale || concept.language || '', concept.display_name || concept.display || concept.name); - } - - if (concept.locale_display_names && typeof concept.locale_display_names === 'object') { - for (const [lang, value] of Object.entries(concept.locale_display_names)) { - add(lang, value); - } - } - - return result; + return toConceptContext(concept); } system() { diff --git a/tx/ocl/fingerprint/fingerprint.js b/tx/ocl/fingerprint/fingerprint.js new file mode 100644 index 00000000..4a18c158 --- /dev/null +++ b/tx/ocl/fingerprint/fingerprint.js @@ -0,0 +1,67 @@ +const crypto = require('crypto'); + +function hashSortedLines(lines) { + const hash = crypto.createHash('sha256'); + for (const line of lines.sort()) { + hash.update(line); + hash.update('\n'); + } + return hash.digest('hex'); +} + +function computeCodeSystemFingerprint(concepts) { + if (!Array.isArray(concepts) || concepts.length === 0) { + return null; + } + + const normalized = concepts + .map(concept => { + if (!concept || !concept.code) { + return null; + } + + const code = String(concept.code || ''); + const display = String(concept.display || ''); + const definition = String(concept.definition || ''); + const retired = concept.retired === true ? '1' : '0'; + return `${code}|${display}|${definition}|${retired}`; + }) + .filter(Boolean); + + if (normalized.length === 0) { + return null; + } + + return hashSortedLines(normalized); +} + +function computeValueSetExpansionFingerprint(expansion) { + if (!expansion || !Array.isArray(expansion.contains) || expansion.contains.length === 0) { + return null; + } + + const normalized = expansion.contains + .map(entry => { + if (!entry || !entry.code) { + return null; + } + + const system = String(entry.system || ''); + const code = String(entry.code || ''); + const display = String(entry.display || ''); + const inactive = entry.inactive === true ? '1' : '0'; + return `${system}|${code}|${display}|${inactive}`; + }) + .filter(Boolean); + + if (normalized.length === 0) { + return null; + } + + return hashSortedLines(normalized); +} + +module.exports = { + computeCodeSystemFingerprint, + computeValueSetExpansionFingerprint +}; diff --git a/tx/ocl/http/client.js b/tx/ocl/http/client.js new file mode 100644 index 00000000..7ae7ecb2 --- /dev/null +++ b/tx/ocl/http/client.js @@ -0,0 +1,31 @@ +const axios = require('axios'); +const { DEFAULT_BASE_URL } = require('../shared/constants'); + +function createOclHttpClient(config = {}) { + const options = typeof config === 'string' ? { baseUrl: config } : (config || {}); + + const baseUrl = (options.baseUrl || DEFAULT_BASE_URL).replace(/\/+$/, ''); + const headers = { + Accept: 'application/json', + 'User-Agent': 'FHIRSmith-OCL-Provider/1.0' + }; + + if (options.token) { + headers.Authorization = options.token.startsWith('Token ') || options.token.startsWith('Bearer ') + ? options.token + : `Token ${options.token}`; + } + + return { + baseUrl, + client: axios.create({ + baseURL: baseUrl, + timeout: options.timeout || 30000, + headers + }) + }; +} + +module.exports = { + createOclHttpClient +}; diff --git a/tx/ocl/http/pagination.js b/tx/ocl/http/pagination.js new file mode 100644 index 00000000..7ea7f7b4 --- /dev/null +++ b/tx/ocl/http/pagination.js @@ -0,0 +1,98 @@ +const { PAGE_SIZE } = require('../shared/constants'); + +function extractItemsAndNext(payload, baseUrl = null) { + if (Array.isArray(payload)) { + return { items: payload, next: null }; + } + + if (!payload || typeof payload !== 'object') { + return { items: [], next: null }; + } + + const items = Array.isArray(payload.results) + ? payload.results + : Array.isArray(payload.items) + ? payload.items + : Array.isArray(payload.data) + ? payload.data + : []; + + const next = payload.next || null; + if (!next) { + return { items, next: null }; + } + + if (baseUrl && typeof next === 'string' && next.startsWith(baseUrl)) { + return { items, next: next.replace(baseUrl, '') }; + } + + return { items, next }; +} + +async function fetchAllPages(httpClient, path, options = {}) { + const { + params = {}, + pageSize = PAGE_SIZE, + maxPages = Number.MAX_SAFE_INTEGER, + baseUrl = null, + useNextLinks = true, + logger = null, + loggerPrefix = '[OCL]' + } = options; + + const results = []; + let page = 1; + let nextPath = path; + let pageCount = 0; + let usePageMode = true; + + while (nextPath && pageCount < maxPages) { + try { + const response = usePageMode + ? await httpClient.get(path, { params: { ...params, page, limit: pageSize } }) + : await httpClient.get(nextPath); + + if (Array.isArray(response.data)) { + results.push(...response.data); + pageCount += 1; + + if (response.data.length < pageSize) { + break; + } + + page += 1; + nextPath = path; + continue; + } + + const { items, next } = extractItemsAndNext(response.data, baseUrl); + results.push(...items); + pageCount += 1; + + if (useNextLinks && next) { + usePageMode = false; + nextPath = next; + continue; + } + + if (usePageMode && items.length >= pageSize && pageCount < maxPages) { + page += 1; + nextPath = path; + } else { + break; + } + } catch (error) { + if (logger && typeof logger.error === 'function') { + logger.error(`${loggerPrefix} Fetch error on page ${page}:`, error.message); + } + throw error; + } + } + + return results; +} + +module.exports = { + extractItemsAndNext, + fetchAllPages +}; diff --git a/tx/ocl/jobs/background-queue.js b/tx/ocl/jobs/background-queue.js new file mode 100644 index 00000000..060b93b4 --- /dev/null +++ b/tx/ocl/jobs/background-queue.js @@ -0,0 +1,200 @@ +class OCLBackgroundJobQueue { + static MAX_CONCURRENT = 2; + static HEARTBEAT_INTERVAL_MS = 30000; + static UNKNOWN_JOB_SIZE = Number.MAX_SAFE_INTEGER; + static pendingJobs = []; + static activeCount = 0; + static queuedOrRunningKeys = new Set(); + static activeJobs = new Map(); + static heartbeatTimer = null; + static enqueueSequence = 0; + + static enqueue(jobKey, jobType, runJob, options = {}) { + if (!jobKey || typeof runJob !== 'function') { + return false; + } + + if (this.queuedOrRunningKeys.has(jobKey)) { + return false; + } + + this.queuedOrRunningKeys.add(jobKey); + const resolveAndEnqueue = async () => { + const resolvedSize = await this.#resolveJobSize(options); + const normalizedSize = this.#normalizeJobSize(resolvedSize); + this.#insertPendingJobOrdered({ + jobKey, + jobType: jobType || 'background-job', + jobId: options?.jobId || jobKey, + jobSize: normalizedSize, + getProgress: typeof options?.getProgress === 'function' ? options.getProgress : null, + runJob, + enqueueOrder: this.enqueueSequence++ + }); + this.ensureHeartbeatRunning(); + console.log(`[OCL] ${jobType || 'Background job'} enqueued: ${jobKey} (size=${normalizedSize}, queue=${this.pendingJobs.length}, active=${this.activeCount})`); + this.processNext(); + }; + + Promise.resolve() + .then(resolveAndEnqueue) + .catch((error) => { + this.queuedOrRunningKeys.delete(jobKey); + const message = error && error.message ? error.message : String(error); + console.error(`[OCL] Failed to enqueue background job: ${jobType || 'background-job'} ${jobKey}: ${message}`); + }); + + return true; + } + + static async #resolveJobSize(options = {}) { + if (typeof options?.resolveJobSize === 'function') { + try { + return await options.resolveJobSize(); + } catch (_error) { + return this.UNKNOWN_JOB_SIZE; + } + } + + if (options && Object.prototype.hasOwnProperty.call(options, 'jobSize')) { + return options.jobSize; + } + + return this.UNKNOWN_JOB_SIZE; + } + + static #normalizeJobSize(jobSize) { + const parsed = Number.parseInt(jobSize, 10); + if (!Number.isFinite(parsed) || parsed < 0) { + return this.UNKNOWN_JOB_SIZE; + } + return parsed; + } + + static #insertPendingJobOrdered(job) { + let index = this.pendingJobs.findIndex(existing => { + if (existing.jobSize === job.jobSize) { + return existing.enqueueOrder > job.enqueueOrder; + } + return existing.jobSize > job.jobSize; + }); + + if (index < 0) { + index = this.pendingJobs.length; + } + + this.pendingJobs.splice(index, 0, job); + } + + static isQueuedOrRunning(jobKey) { + return this.queuedOrRunningKeys.has(jobKey); + } + + static ensureHeartbeatRunning() { + if (this.heartbeatTimer) { + return; + } + + this.heartbeatTimer = setInterval(() => { + this.logHeartbeat(); + }, this.HEARTBEAT_INTERVAL_MS); + + if (typeof this.heartbeatTimer.unref === 'function') { + this.heartbeatTimer.unref(); + } + } + + static logHeartbeat() { + const activeJobs = Array.from(this.activeJobs.values()); + const lines = [ + '[OCL] OCL background status:', + ` active jobs: ${activeJobs.length}`, + ` queued jobs: ${this.pendingJobs.length}` + ]; + + activeJobs.forEach((job, index) => { + lines.push(''); + lines.push(` job ${index + 1}:`); + lines.push(` type: ${job.jobType || 'background-job'}`); + lines.push(` id: ${job.jobId || job.jobKey}`); + lines.push(` size: ${job.jobSize}`); + lines.push(` progress: ${this.formatProgress(job.getProgress)}`); + }); + + console.log(lines.join('\n')); + } + + static formatProgress(getProgress) { + if (typeof getProgress !== 'function') { + return 'unknown'; + } + + try { + const progress = getProgress(); + if (typeof progress === 'number' && Number.isFinite(progress)) { + const bounded = Math.max(0, Math.min(100, progress)); + return `${Math.round(bounded)}%`; + } + + if (progress && typeof progress === 'object') { + if (typeof progress.percentage === 'number' && Number.isFinite(progress.percentage)) { + const bounded = Math.max(0, Math.min(100, progress.percentage)); + return `${Math.round(bounded)}%`; + } + + if ( + typeof progress.processed === 'number' && + Number.isFinite(progress.processed) && + typeof progress.total === 'number' && + Number.isFinite(progress.total) && + progress.total > 0 + ) { + const ratio = progress.processed / progress.total; + const bounded = Math.max(0, Math.min(100, ratio * 100)); + return `${Math.round(bounded)}%`; + } + } + } catch (_error) { + return 'unknown'; + } + + return 'unknown'; + } + + static processNext() { + while (this.activeCount < this.MAX_CONCURRENT && this.pendingJobs.length > 0) { + const job = this.pendingJobs.shift(); + this.activeCount += 1; + this.activeJobs.set(job.jobKey, { + jobKey: job.jobKey, + jobType: job.jobType, + jobId: job.jobId || job.jobKey, + jobSize: job.jobSize, + getProgress: job.getProgress || null, + startedAt: Date.now() + }); + console.log(`[OCL] Background job started: ${job.jobType} ${job.jobKey} (size=${job.jobSize}, queue=${this.pendingJobs.length}, active=${this.activeCount})`); + + Promise.resolve() + .then(() => job.runJob()) + .then(() => { + console.log(`[OCL] Background job completed: ${job.jobType} ${job.jobKey}`); + }) + .catch((error) => { + const message = error && error.message ? error.message : String(error); + console.error(`[OCL] Background job failed: ${job.jobType} ${job.jobKey}: ${message}`); + }) + .finally(() => { + this.activeCount -= 1; + this.queuedOrRunningKeys.delete(job.jobKey); + this.activeJobs.delete(job.jobKey); + console.log(`[OCL] Background queue status: queue=${this.pendingJobs.length}, active=${this.activeCount}`); + this.processNext(); + }); + } + } +} + +module.exports = { + OCLBackgroundJobQueue +}; diff --git a/tx/ocl/mappers/concept-mapper.js b/tx/ocl/mappers/concept-mapper.js new file mode 100644 index 00000000..e350deb0 --- /dev/null +++ b/tx/ocl/mappers/concept-mapper.js @@ -0,0 +1,66 @@ +function toConceptContext(concept) { + if (!concept || typeof concept !== 'object') { + return null; + } + + const code = concept.code || concept.id || null; + if (!code) { + return null; + } + + return { + code, + display: concept.display_name || concept.display || concept.name || null, + definition: concept.description || concept.definition || null, + retired: concept.retired === true, + designations: extractDesignations(concept) + }; +} + +function extractDesignations(concept) { + const result = []; + const seen = new Set(); + + const add = (language, value) => { + const text = typeof value === 'string' ? value.trim() : ''; + if (!text) { + return; + } + + const lang = typeof language === 'string' ? language.trim() : ''; + const key = `${lang}|${text}`; + if (seen.has(key)) { + return; + } + + seen.add(key); + result.push({ language: lang, value: text }); + }; + + if (Array.isArray(concept.names)) { + for (const entry of concept.names) { + if (!entry || typeof entry !== 'object') { + continue; + } + + add(entry.locale || entry.language || entry.lang || '', entry.name || entry.display_name || entry.display || entry.value || entry.term); + } + } + + if (concept.display_name || concept.display || concept.name) { + add(concept.locale || concept.default_locale || concept.language || '', concept.display_name || concept.display || concept.name); + } + + if (concept.locale_display_names && typeof concept.locale_display_names === 'object') { + for (const [lang, value] of Object.entries(concept.locale_display_names)) { + add(lang, value); + } + } + + return result; +} + +module.exports = { + toConceptContext, + extractDesignations +}; diff --git a/tx/ocl/model/concept-filter-context.js b/tx/ocl/model/concept-filter-context.js new file mode 100644 index 00000000..330ac2a7 --- /dev/null +++ b/tx/ocl/model/concept-filter-context.js @@ -0,0 +1,51 @@ +class OCLConceptFilterContext { + constructor() { + this.concepts = []; + this.currentIndex = -1; + } + + add(concept, rating = 0) { + this.concepts.push({ concept, rating }); + } + + sort() { + this.concepts.sort((a, b) => b.rating - a.rating); + } + + size() { + return this.concepts.length; + } + + hasMore() { + return this.currentIndex + 1 < this.concepts.length; + } + + next() { + if (!this.hasMore()) { + return null; + } + this.currentIndex += 1; + return this.concepts[this.currentIndex].concept; + } + + reset() { + this.currentIndex = -1; + } + + findConceptByCode(code) { + for (const item of this.concepts) { + if (item.concept && item.concept.code === code) { + return item.concept; + } + } + return null; + } + + containsConcept(concept) { + return this.concepts.some(item => item.concept === concept); + } +} + +module.exports = { + OCLConceptFilterContext +}; diff --git a/tx/ocl/shared/constants.js b/tx/ocl/shared/constants.js new file mode 100644 index 00000000..5b338687 --- /dev/null +++ b/tx/ocl/shared/constants.js @@ -0,0 +1,15 @@ +const DEFAULT_BASE_URL = 'https://api.openconceptlab.org'; +const PAGE_SIZE = 100; +const CONCEPT_PAGE_SIZE = 1000; +const FILTERED_CONCEPT_PAGE_SIZE = 200; +const COLD_CACHE_FRESHNESS_MS = 60 * 60 * 1000; +const OCL_CODESYSTEM_MARKER_EXTENSION = 'http://fhir.org/FHIRsmith/StructureDefinition/ocl-codesystem'; + +module.exports = { + DEFAULT_BASE_URL, + PAGE_SIZE, + CONCEPT_PAGE_SIZE, + FILTERED_CONCEPT_PAGE_SIZE, + COLD_CACHE_FRESHNESS_MS, + OCL_CODESYSTEM_MARKER_EXTENSION +}; diff --git a/tx/ocl/shared/patches.js b/tx/ocl/shared/patches.js new file mode 100644 index 00000000..14a235ab --- /dev/null +++ b/tx/ocl/shared/patches.js @@ -0,0 +1,135 @@ +const { OCL_CODESYSTEM_MARKER_EXTENSION } = require('./constants'); + +const OCL_SEARCH_PATCH_FLAG = Symbol.for('fhirsmith.ocl.search.codesystem.code.patch'); +const TXPARAMS_HASH_PATCH_FLAG = Symbol.for('fhirsmith.ocl.txparameters.hash.filter.patch'); + +function hasOCLCodeSystemMarker(resource) { + const extensions = Array.isArray(resource?.extension) ? resource.extension : []; + return extensions.some(ext => ext && ext.url === OCL_CODESYSTEM_MARKER_EXTENSION); +} + +function filterConceptTreeByCode(concepts, wantedCode) { + if (!Array.isArray(concepts) || concepts.length === 0) { + return []; + } + + const matches = []; + for (const concept of concepts) { + if (!concept || typeof concept !== 'object') { + continue; + } + + const childMatches = filterConceptTreeByCode(concept.concept, wantedCode); + const isSelfMatch = concept.code != null && String(concept.code) === wantedCode; + if (!isSelfMatch && childMatches.length === 0) { + continue; + } + + const clone = { ...concept }; + if (childMatches.length > 0) { + clone.concept = childMatches; + } else { + delete clone.concept; + } + matches.push(clone); + } + + return matches; +} + +function filterOCLCodeSystemResourceByCode(resource, code) { + if (!resource || typeof resource !== 'object') { + return resource; + } + + const filteredConcepts = filterConceptTreeByCode(resource.concept, code); + return { + ...resource, + concept: filteredConcepts + }; +} + +function patchSearchWorkerForOCLCodeFiltering() { + let SearchWorker; + try { + SearchWorker = require('../../workers/search'); + } catch (_error) { + return; + } + + if (!SearchWorker || !SearchWorker.prototype) { + return; + } + + const proto = SearchWorker.prototype; + if (proto[OCL_SEARCH_PATCH_FLAG] === true || typeof proto.searchCodeSystems !== 'function') { + return; + } + + const originalSearchCodeSystems = proto.searchCodeSystems; + proto.searchCodeSystems = function patchedSearchCodeSystems(params) { + const matches = originalSearchCodeSystems.call(this, params); + const requestedCode = params?.code == null ? '' : String(params.code); + + if (!requestedCode) { + return matches; + } + + const filtered = []; + for (const resource of matches) { + if (!hasOCLCodeSystemMarker(resource)) { + filtered.push(resource); + continue; + } + + const projected = filterOCLCodeSystemResourceByCode(resource, requestedCode); + if (Array.isArray(projected?.concept) && projected.concept.length > 0) { + filtered.push(projected); + } + } + + return filtered; + }; + + Object.defineProperty(proto, OCL_SEARCH_PATCH_FLAG, { + value: true, + writable: false, + configurable: false, + enumerable: false + }); +} + +function normalizeFilterForCacheKey(filter) { + if (typeof filter !== 'string') { + return ''; + } + + return filter.trim().toLowerCase(); +} + +function ensureTxParametersHashIncludesFilter(TxParameters) { + const proto = TxParameters && TxParameters.prototype; + if (!proto || proto[TXPARAMS_HASH_PATCH_FLAG] === true || typeof proto.hashSource !== 'function') { + return; + } + + const originalHashSource = proto.hashSource; + proto.hashSource = function hashSourceWithFilter() { + const base = originalHashSource.call(this); + const normalizedFilter = normalizeFilterForCacheKey(this.filter); + return `${base}|filter=${normalizedFilter}`; + }; + + Object.defineProperty(proto, TXPARAMS_HASH_PATCH_FLAG, { + value: true, + writable: false, + configurable: false, + enumerable: false + }); +} + +module.exports = { + patchSearchWorkerForOCLCodeFiltering, + ensureTxParametersHashIncludesFilter, + normalizeFilterForCacheKey +}; diff --git a/tx/ocl/vs-ocl.js b/tx/ocl/vs-ocl.js index 8aabad71..18733fb5 100644 --- a/tx/ocl/vs-ocl.js +++ b/tx/ocl/vs-ocl.js @@ -1,6 +1,4 @@ -const axios = require('axios'); const fs = require('fs/promises'); -const fsSync = require('fs'); const crypto = require('crypto'); const path = require('path'); const { AbstractValueSetProvider } = require('../vs/vs-api'); @@ -9,151 +7,24 @@ const ValueSet = require('../library/valueset'); const { SearchFilterText } = require('../library/designations'); const { TxParameters } = require('../params'); const { OCLSourceCodeSystemFactory, OCLBackgroundJobQueue } = require('./cs-ocl'); +const { PAGE_SIZE, CONCEPT_PAGE_SIZE, FILTERED_CONCEPT_PAGE_SIZE, COLD_CACHE_FRESHNESS_MS } = require('./shared/constants'); +const { createOclHttpClient } = require('./http/client'); +const { CACHE_VS_DIR, getCacheFilePath } = require('./cache/cache-paths'); +const { ensureCacheDirectories, getColdCacheAgeMs, formatCacheAgeMinutes } = require('./cache/cache-utils'); +const { computeValueSetExpansionFingerprint } = require('./fingerprint/fingerprint'); +const { ensureTxParametersHashIncludesFilter } = require('./shared/patches'); -const DEFAULT_BASE_URL = 'https://oclapi2.ips.hsl.org.br'; -const PAGE_SIZE = 100; -const CONCEPT_PAGE_SIZE = 1000; -const FILTERED_CONCEPT_PAGE_SIZE = 200; -const COLD_CACHE_FRESHNESS_MS = 60 * 60 * 1000; - -// Cold cache configuration (import from cs-ocl) -const CACHE_BASE_DIR = path.join(process.cwd(), 'data', 'terminology-cache', 'ocl'); -const CACHE_VS_DIR = path.join(CACHE_BASE_DIR, 'valuesets'); -const TXPARAMS_HASH_PATCH_FLAG = Symbol.for('fhirsmith.ocl.txparameters.hash.filter.patch'); - -function normalizeFilterForCacheKey(filter) { - if (typeof filter !== 'string') { - return ''; - } - - // OCL filter matching is case-insensitive and trims surrounding whitespace. - return filter.trim().toLowerCase(); -} - -function ensureTxParametersHashIncludesFilter() { - const proto = TxParameters && TxParameters.prototype; - if (!proto || proto[TXPARAMS_HASH_PATCH_FLAG] === true || typeof proto.hashSource !== 'function') { - return; - } - - const originalHashSource = proto.hashSource; - proto.hashSource = function hashSourceWithFilter() { - const base = originalHashSource.call(this); - const normalizedFilter = normalizeFilterForCacheKey(this.filter); - return `${base}|filter=${normalizedFilter}`; - }; - - Object.defineProperty(proto, TXPARAMS_HASH_PATCH_FLAG, { - value: true, - writable: false, - configurable: false, - enumerable: false - }); -} - -ensureTxParametersHashIncludesFilter(); - -// Cache file utilities -async function ensureCacheDirectories() { - try { - await fs.mkdir(CACHE_VS_DIR, { recursive: true }); - } catch (error) { - console.error('[OCL-ValueSet] Failed to create cache directories:', error.message); - } -} - -// ValueSet expansion fingerprint computation -function computeValueSetExpansionFingerprint(expansion) { - if (!expansion || !Array.isArray(expansion.contains) || expansion.contains.length === 0) { - return null; - } - - // Normalize expansion entries to deterministic strings - const normalized = expansion.contains - .map(entry => { - if (!entry || !entry.code) { - return null; - } - const system = String(entry.system || ''); - const code = String(entry.code || ''); - const display = String(entry.display || ''); - const inactive = entry.inactive === true ? '1' : '0'; - return `${system}|${code}|${display}|${inactive}`; - }) - .filter(Boolean) - .sort(); - - // Compute SHA256 hash - const hash = crypto.createHash('sha256'); - for (const item of normalized) { - hash.update(item); - hash.update('\n'); - } - return hash.digest('hex'); -} - -function sanitizeFilename(text) { - if (!text || typeof text !== 'string') { - return 'unknown'; - } - return text - .replace(/[^a-zA-Z0-9._-]/g, '_') - .replace(/_+/g, '_') - .substring(0, 200); -} - -function getCacheFilePath(baseDir, canonicalUrl, version = null, paramsKey = 'default') { - const filename = sanitizeFilename(canonicalUrl) - + (version ? `_${sanitizeFilename(version)}` : '') - + (paramsKey && paramsKey !== 'default' ? `_p_${sanitizeFilename(paramsKey)}` : '') - + '.json'; - return path.join(baseDir, filename); -} - -function getColdCacheAgeMs(cacheFilePath) { - try { - const stats = fsSync.statSync(cacheFilePath); - if (!stats || !Number.isFinite(stats.mtimeMs)) { - return null; - } - return Math.max(0, Date.now() - stats.mtimeMs); - } catch (error) { - if (error && error.code !== 'ENOENT') { - console.error(`[OCL-ValueSet] Failed to inspect cold cache file ${cacheFilePath}: ${error.message}`); - } - return null; - } -} - -function formatCacheAgeMinutes(ageMs) { - const minutes = Math.max(1, Math.round(ageMs / 60000)); - return `${minutes} minute${minutes === 1 ? '' : 's'}`; -} +ensureTxParametersHashIncludesFilter(TxParameters); class OCLValueSetProvider extends AbstractValueSetProvider { constructor(config = {}) { super(); const options = typeof config === 'string' ? { baseUrl: config } : (config || {}); - this.baseUrl = (options.baseUrl || DEFAULT_BASE_URL).replace(/\/+$/, ''); this.org = options.org || null; - - const headers = { - Accept: 'application/json', - 'User-Agent': 'FHIRSmith-OCL-Provider/1.0' - }; - - if (options.token) { - headers.Authorization = options.token.startsWith('Token ') || options.token.startsWith('Bearer ') - ? options.token - : `Token ${options.token}`; - } - - this.httpClient = axios.create({ - baseURL: this.baseUrl, - timeout: options.timeout || 30000, - headers - }); + const http = createOclHttpClient(options); + this.baseUrl = http.baseUrl; + this.httpClient = http.client; this.valueSetMap = new Map(); this._idMap = new Map(); @@ -197,16 +68,21 @@ class OCLValueSetProvider extends AbstractValueSetProvider { } const paramsKey = cached.paramsKey || 'default'; - const cacheKey = `${cached.canonicalUrl}|${cached.version || ''}|${paramsKey}`; + const cacheKey = this.#expansionCacheKey( + { url: cached.canonicalUrl, version: cached.version || null }, + paramsKey + ); + const createdAt = cached.timestamp ? new Date(cached.timestamp).getTime() : null; this.backgroundExpansionCache.set(cacheKey, { expansion: cached.expansion, metadataSignature: cached.metadataSignature || null, dependencyChecksums: cached.dependencyChecksums || {}, - createdAt: cached.timestamp ? new Date(cached.timestamp).getTime() : Date.now() + createdAt: Number.isFinite(createdAt) ? createdAt : null }); this.valueSetFingerprints.set(cacheKey, cached.fingerprint || null); loadedCount++; + console.log(`[OCL-ValueSet] Loaded ValueSet from cold cache: ${cached.canonicalUrl}`); } catch (error) { console.error(`[OCL-ValueSet] Failed to load cold cache file ${file}:`, error.message); } @@ -232,7 +108,7 @@ class OCLValueSetProvider extends AbstractValueSetProvider { const cacheFilePath = getCacheFilePath(CACHE_VS_DIR, canonicalUrl, version, paramsKey); try { - await ensureCacheDirectories(); + await ensureCacheDirectories(CACHE_VS_DIR); const fingerprint = computeValueSetExpansionFingerprint(expansion); const cacheData = { @@ -474,7 +350,16 @@ class OCLValueSetProvider extends AbstractValueSetProvider { } #indexValueSet(vs) { - this.#invalidateExpansionCache(vs); + const existing = this.valueSetMap.get(vs.url) + || (vs.version ? this.valueSetMap.get(`${vs.url}|${vs.version}`) : null) + || this._idMap.get(vs.id) + || null; + + // Preserve hydrated cold-cache expansions on first index; invalidate only on replacement. + if (existing && existing !== vs) { + this.#invalidateExpansionCache(vs); + } + this.valueSetMap.set(vs.url, vs); if (vs.version) { this.valueSetMap.set(`${vs.url}|${vs.version}`, vs); @@ -977,7 +862,7 @@ class OCLValueSetProvider extends AbstractValueSetProvider { // Treat cache as fresh when either file mtime or persisted timestamp is recent. const freshnessCandidates = [cacheAgeFromFileMs, cacheAgeFromMetadataMs].filter(age => age != null); const freshestCacheAgeMs = freshnessCandidates.length > 0 ? Math.min(...freshnessCandidates) : null; - if (freshestCacheAgeMs != null && freshestCacheAgeMs < COLD_CACHE_FRESHNESS_MS) { + if (freshestCacheAgeMs != null && freshestCacheAgeMs <= COLD_CACHE_FRESHNESS_MS) { const freshnessSource = cacheAgeFromFileMs != null && cacheAgeFromMetadataMs != null ? 'file+metadata' : cacheAgeFromFileMs != null @@ -995,6 +880,10 @@ class OCLValueSetProvider extends AbstractValueSetProvider { } let queuedJobSize = null; + const warmupAgeText = freshestCacheAgeMs != null + ? formatCacheAgeMinutes(freshestCacheAgeMs) + : 'no cold cache'; + console.log(`[OCL-ValueSet] Enqueueing warm-up for ValueSet ${vs.url} (cold cache age: ${warmupAgeText})`); console.log(`[OCL-ValueSet] ValueSet expansion enqueued: ${cacheKey}`); OCLBackgroundJobQueue.enqueue( jobKey, From b23a88366d411a192fe8da9ab756535ae437f242 Mon Sep 17 00:00:00 2001 From: Italo Macedo Date: Sat, 7 Mar 2026 04:18:20 -0300 Subject: [PATCH 07/15] Fixing empty block --- tx/ocl/cm-ocl.js | 2 ++ 1 file changed, 2 insertions(+) diff --git a/tx/ocl/cm-ocl.js b/tx/ocl/cm-ocl.js index db0c5fb5..255eaabd 100644 --- a/tx/ocl/cm-ocl.js +++ b/tx/ocl/cm-ocl.js @@ -469,6 +469,8 @@ class OCLConceptMapProvider extends AbstractConceptMapProvider { this._sourceUrlsByCanonical.get(canonicalKey).add(resolvedSourceUrl); this._canonicalBySourceUrl.set(this.#norm(resolvedSourceUrl), canonical); } catch (e) { + // Ignore source lookup failures and continue resolving remaining sources. + continue; } } } From 9f6b52875c8313d1eb153e2963e1692ebcd0caa3 Mon Sep 17 00:00:00 2001 From: Italo Macedo Date: Sat, 7 Mar 2026 05:27:11 -0300 Subject: [PATCH 08/15] Readme.MD for OCL Providers. >95% coverage tests --- tests/ocl/ocl-cm.test.js | 243 +++ tests/ocl/ocl-cs-provider-methods.test.js | 235 +++ tests/ocl/ocl-cs.test.js | 291 ++++ tests/ocl/ocl-helpers.test.js | 302 ++++ tests/ocl/ocl-vs-advanced.test.js | 428 +++++ tests/ocl/ocl-vs.test.js | 289 ++++ tx/ocl/README.md | 236 +++ tx/ocl/cache/cache-paths.cjs | 32 + tx/ocl/cache/cache-paths.js | 32 +- tx/ocl/cache/cache-utils.cjs | 43 + tx/ocl/cache/cache-utils.js | 43 +- tx/ocl/cm-ocl.cjs | 531 +++++++ tx/ocl/cm-ocl.js | 537 +------ tx/ocl/cs-ocl.cjs | 1679 ++++++++++++++++++++ tx/ocl/cs-ocl.js | 1679 +------------------- tx/ocl/fingerprint/fingerprint.cjs | 67 + tx/ocl/fingerprint/fingerprint.js | 67 +- tx/ocl/http/client.cjs | 31 + tx/ocl/http/client.js | 31 +- tx/ocl/http/pagination.cjs | 98 ++ tx/ocl/http/pagination.js | 98 +- tx/ocl/jobs/background-queue.cjs | 200 +++ tx/ocl/jobs/background-queue.js | 200 +-- tx/ocl/mappers/concept-mapper.cjs | 66 + tx/ocl/mappers/concept-mapper.js | 66 +- tx/ocl/model/concept-filter-context.cjs | 51 + tx/ocl/model/concept-filter-context.js | 51 +- tx/ocl/shared/constants.cjs | 15 + tx/ocl/shared/constants.js | 15 +- tx/ocl/shared/patches.cjs | 135 ++ tx/ocl/shared/patches.js | 135 +- tx/ocl/vs-ocl.cjs | 1760 +++++++++++++++++++++ tx/ocl/vs-ocl.js | 1760 +-------------------- 33 files changed, 6745 insertions(+), 4701 deletions(-) create mode 100644 tests/ocl/ocl-cm.test.js create mode 100644 tests/ocl/ocl-cs-provider-methods.test.js create mode 100644 tests/ocl/ocl-cs.test.js create mode 100644 tests/ocl/ocl-helpers.test.js create mode 100644 tests/ocl/ocl-vs-advanced.test.js create mode 100644 tests/ocl/ocl-vs.test.js create mode 100644 tx/ocl/README.md create mode 100644 tx/ocl/cache/cache-paths.cjs create mode 100644 tx/ocl/cache/cache-utils.cjs create mode 100644 tx/ocl/cm-ocl.cjs create mode 100644 tx/ocl/cs-ocl.cjs create mode 100644 tx/ocl/fingerprint/fingerprint.cjs create mode 100644 tx/ocl/http/client.cjs create mode 100644 tx/ocl/http/pagination.cjs create mode 100644 tx/ocl/jobs/background-queue.cjs create mode 100644 tx/ocl/mappers/concept-mapper.cjs create mode 100644 tx/ocl/model/concept-filter-context.cjs create mode 100644 tx/ocl/shared/constants.cjs create mode 100644 tx/ocl/shared/patches.cjs create mode 100644 tx/ocl/vs-ocl.cjs diff --git a/tests/ocl/ocl-cm.test.js b/tests/ocl/ocl-cm.test.js new file mode 100644 index 00000000..f4c0387d --- /dev/null +++ b/tests/ocl/ocl-cm.test.js @@ -0,0 +1,243 @@ +const nock = require('nock'); + +const { OCLConceptMapProvider } = require('../../tx/ocl/cm-ocl'); + +describe('OCL ConceptMap integration', () => { + const baseUrl = 'https://ocl.cm.test'; + + beforeEach(() => { + nock.cleanAll(); + jest.restoreAllMocks(); + }); + + afterEach(() => { + nock.cleanAll(); + }); + + function mapping(overrides = {}) { + return { + id: 'map-1', + url: `${baseUrl}/mappings/map-1`, + version: '1.0.0', + map_type: 'SAME-AS', + from_source_url: '/orgs/org-a/sources/src-a/', + to_source_url: '/orgs/org-a/sources/src-b/', + from_concept_code: 'A', + to_concept_code: 'B', + from_concept_name_resolved: 'Alpha', + to_concept_name_resolved: 'Beta', + updated_on: '2026-01-02T03:04:05.000Z', + name: 'Map One', + comment: 'ok', + ...overrides + }; + } + + test('fetchConceptMapById resolves and indexes mapping', async () => { + nock(baseUrl) + .get('/mappings/map-1/') + .reply(200, mapping()); + + const provider = new OCLConceptMapProvider({ baseUrl }); + const cm = await provider.fetchConceptMapById('map-1'); + + expect(cm).toBeTruthy(); + expect(cm.id).toBe('map-1'); + expect(cm.jsonObj.group[0].source).toBe('/orgs/org-a/sources/src-a/'); + expect(cm.jsonObj.group[0].target).toBe('/orgs/org-a/sources/src-b/'); + expect(cm.jsonObj.meta.lastUpdated).toBe('2026-01-02T03:04:05.000Z'); + }); + + test('fetchConceptMap can resolve from canonical url via search and mapping-id extraction', async () => { + nock(baseUrl) + .get('/mappings/map-2/') + .reply(200, mapping({ id: 'map-2', url: `${baseUrl}/mappings/map-2` })) + .get('/mappings/') + .query(true) + .reply(200, { results: [mapping()] }); + + const provider = new OCLConceptMapProvider({ baseUrl }); + + const byIdFromUrl = await provider.fetchConceptMap(`${baseUrl}/mappings/map-2`, null); + expect(byIdFromUrl.id).toBe('map-2'); + + const bySearch = await provider.fetchConceptMap('/orgs/org-a/sources/src-a/', '1.0.0'); + expect(bySearch).toBeNull(); + }); + + test('searchConceptMaps filters by source and target parameters', async () => { + nock(baseUrl) + .get('/mappings/') + .query(true) + .reply(200, { results: [mapping(), mapping({ id: 'map-3', from_source_url: '/orgs/x/s/', to_source_url: '/orgs/y/s/' })] }); + + const provider = new OCLConceptMapProvider({ baseUrl }); + + const results = await provider.searchConceptMaps([ + { name: 'source', value: '/orgs/org-a/sources/src-a/' }, + { name: 'target', value: '/orgs/org-a/sources/src-b/' } + ]); + + expect(results).toHaveLength(1); + expect(results[0].id).toBe('map-1'); + }); + + test('findConceptMapForTranslation uses source candidates and canonical resolution', async () => { + nock(baseUrl) + .get('/sources/') + .query(true) + .twice() + .reply(200, { + results: [ + { + canonical_url: 'http://example.org/cs/source-a', + url: '/orgs/org-a/sources/src-a/' + }, + { + canonical_url: 'http://example.org/cs/source-b', + url: '/orgs/org-a/sources/src-b/' + } + ] + }) + .get('/orgs/org-a/sources/src-a/concepts/A/mappings/') + .query(true) + .reply(200, { results: [mapping()] }) + .get('/orgs/org-a/sources/src-a/') + .reply(200, { + canonical_url: 'http://example.org/cs/source-a', + url: '/orgs/org-a/sources/src-a/' + }) + .get('/orgs/org-a/sources/src-b/') + .reply(200, { + canonical_url: 'http://example.org/cs/source-b', + url: '/orgs/org-a/sources/src-b/' + }); + + const provider = new OCLConceptMapProvider({ baseUrl, maxSearchPages: 2 }); + const conceptMaps = []; + + await provider.findConceptMapForTranslation( + null, + conceptMaps, + 'http://example.org/cs/source-a', + null, + null, + 'http://example.org/cs/source-b', + 'A' + ); + + expect(conceptMaps).toHaveLength(1); + expect(conceptMaps[0].jsonObj.group[0].source).toBe('http://example.org/cs/source-a'); + expect(conceptMaps[0].jsonObj.group[0].target).toBe('http://example.org/cs/source-b'); + }); + + test('assignIds prefixes space and cmCount returns unique map count', async () => { + nock(baseUrl) + .get('/mappings/map-1/') + .reply(200, mapping()); + + const provider = new OCLConceptMapProvider({ baseUrl }); + provider.spaceId = 'space'; + + await provider.fetchConceptMapById('map-1'); + + const ids = new Set(); + provider.assignIds(ids); + + expect(ids.has('ConceptMap/space-map-1')).toBe(true); + expect(provider.cmCount()).toBe(1); + }); + + test('fetchConceptMap returns direct cached hit before network lookup', async () => { + const provider = new OCLConceptMapProvider({ baseUrl }); + const cm = await provider.fetchConceptMapById('map-1').catch(() => null); + expect(cm).toBeNull(); + + const cached = { + id: 'cached-1', + url: 'http://example.org/cached-cm', + version: '1.0.0' + }; + provider.conceptMapMap.set('http://example.org/cached-cm|1.0.0', cached); + const out = await provider.fetchConceptMap('http://example.org/cached-cm', '1.0.0'); + expect(out).toBe(cached); + }); + + test('findConceptMapForTranslation fallback search with empty candidate sets', async () => { + nock(baseUrl) + .get('/mappings/') + .query(true) + .reply(200, { results: [mapping({ id: 'map-fallback' })] }); + + const provider = new OCLConceptMapProvider({ baseUrl }); + const conceptMaps = []; + + await provider.findConceptMapForTranslation( + null, + conceptMaps, + null, + null, + null, + null, + null + ); + + expect(conceptMaps.length).toBeGreaterThanOrEqual(0); + }); + + test('findConceptMapForTranslation uses candidate matching when scope check is strict', async () => { + nock(baseUrl) + .get('/sources/') + .query(true) + .twice() + .reply(200, { + results: [ + { + canonical_url: 'http://example.org/cs/source-a', + url: '/orgs/org-a/sources/src-a/' + }, + { + canonical_url: 'http://example.org/cs/source-b', + url: '/orgs/org-a/sources/src-b/' + } + ] + }) + .get('/orgs/org-a/sources/src-a/concepts/A/mappings/') + .query(true) + .reply(200, { + results: [ + mapping({ + updated_on: 'invalid-date' + }) + ] + }) + .get('/orgs/org-a/sources/src-a/') + .reply(200, { + canonical_url: 'http://example.org/cs/source-a', + url: '/orgs/org-a/sources/src-a/' + }) + .get('/orgs/org-a/sources/src-b/') + .reply(200, { + canonical_url: 'http://example.org/cs/source-b', + url: '/orgs/org-a/sources/src-b/' + }); + + const provider = new OCLConceptMapProvider({ baseUrl }); + const conceptMaps = [{ id: 'already-present' }]; + + await provider.findConceptMapForTranslation( + null, + conceptMaps, + 'http://example.org/cs/source-a', + 'http://scope/strict/source', + 'http://scope/strict/target', + 'http://example.org/cs/source-b', + 'A' + ); + + expect(conceptMaps.length).toBeGreaterThanOrEqual(2); + expect(conceptMaps[1].jsonObj.meta).toBeUndefined(); + + await expect(provider.close()).resolves.toBeUndefined(); + }); +}); diff --git a/tests/ocl/ocl-cs-provider-methods.test.js b/tests/ocl/ocl-cs-provider-methods.test.js new file mode 100644 index 00000000..739cec5f --- /dev/null +++ b/tests/ocl/ocl-cs-provider-methods.test.js @@ -0,0 +1,235 @@ +const nock = require('nock'); + +const { OperationContext } = require('../../tx/operation-context'); +const { Designations } = require('../../tx/library/designations'); +const { OCLSourceCodeSystemFactory, OCLBackgroundJobQueue } = require('../../tx/ocl/cs-ocl'); +const { TestUtilities } = require('../test-utilities'); + +function resetQueueState() { + OCLBackgroundJobQueue.pendingJobs = []; + OCLBackgroundJobQueue.activeCount = 0; + OCLBackgroundJobQueue.queuedOrRunningKeys = new Set(); + OCLBackgroundJobQueue.activeJobs = new Map(); + OCLBackgroundJobQueue.enqueueSequence = 0; + if (OCLBackgroundJobQueue.heartbeatTimer) { + clearInterval(OCLBackgroundJobQueue.heartbeatTimer); + OCLBackgroundJobQueue.heartbeatTimer = null; + } +} + +describe('OCL CodeSystem provider runtime methods', () => { + const baseUrl = 'https://ocl.cs.methods.test'; + const conceptsUrl = `${baseUrl}/orgs/org-a/sources/src1/concepts/`; + let i18n; + let langDefs; + + beforeAll(async () => { + langDefs = await TestUtilities.loadLanguageDefinitions(); + i18n = await TestUtilities.loadTranslations(langDefs); + }); + + beforeEach(() => { + nock.cleanAll(); + OCLSourceCodeSystemFactory.factoriesByKey.clear(); + resetQueueState(); + }); + + afterEach(() => { + nock.cleanAll(); + jest.restoreAllMocks(); + }); + + function createFactoryMeta() { + return { + id: 'src1', + canonicalUrl: 'http://example.org/cs/source-one', + version: '2026.1', + name: 'Source One', + description: 'Desc', + checksum: 'chk-1', + conceptsUrl, + codeSystem: { + jsonObj: { + property: [{ code: 'display' }, { code: 'definition' }, { code: 'inactive' }], + content: 'not-present' + } + } + }; + } + + test('runtime provider methods cover lookup/filter/iteration paths', async () => { + nock(baseUrl) + .get('/orgs/org-a/sources/src1/concepts/C1/') + .reply(200, { + code: 'C1', + display_name: 'Alpha Display', + description: 'Alpha Definition', + retired: false, + names: [{ locale: 'en', name: 'Alpha Display' }] + }) + .get('/orgs/org-a/sources/src1/concepts/C404/') + .reply(404) + .get('/orgs/org-a/sources/src1/concepts/') + .query(q => Number(q.page) === 1 && Number(q.limit) === 1000) + .reply(200, { + results: [ + { code: 'C1', display_name: 'Alpha Display', description: 'Alpha Definition', retired: false }, + { code: 'C2', display_name: 'Beta Display', description: 'Beta Definition', retired: true } + ] + }) + .get('/orgs/org-a/sources/src1/concepts/') + .query(q => Number(q.page) === 2 && Number(q.limit) === 1000) + .reply(200, { results: [] }); + + const factory = new OCLSourceCodeSystemFactory(i18n, require('axios').create({ baseURL: baseUrl }), createFactoryMeta()); + const opContext = new OperationContext('en-US', i18n); + const provider = factory.build(opContext, []); + + expect(provider.system()).toBe('http://example.org/cs/source-one'); + expect(provider.version()).toBe('2026.1'); + expect(provider.description()).toBe('Desc'); + expect(provider.name()).toBe('Source One'); + expect(provider.contentMode()).toBe('not-present'); + expect(provider.totalCount()).toBeGreaterThanOrEqual(-1); + + expect((await provider.locate('')).message).toContain('Empty code'); + const notFound = await provider.locate('C404'); + expect(notFound.context).toBeNull(); + + expect(await provider.code('C1')).toBe('C1'); + expect(await provider.display('C1')).toBe('Alpha Display'); + expect(await provider.definition('C1')).toBe('Alpha Definition'); + expect(await provider.isAbstract('C1')).toBe(false); + expect(await provider.isDeprecated('C1')).toBe(false); + expect(await provider.getStatus('C1')).toBe('active'); + expect(await provider.isInactive('C2')).toBe(true); + + const displays = new Designations(langDefs); + await provider.designations('C1', displays); + expect(displays.designations.length).toBeGreaterThan(0); + + const iter = await provider.iteratorAll(); + const seen = []; + let next = await provider.nextContext(iter); + while (next) { + seen.push(next.code); + next = await provider.nextContext(iter); + } + expect(seen).toEqual(expect.arrayContaining(['C1', 'C2'])); + + expect(await provider.doesFilter('display', '=', 'x')).toBe(true); + expect(await provider.doesFilter('inactive', 'in', 'true,false')).toBe(true); + expect(await provider.doesFilter('unknown', '=', 'x')).toBe(false); + expect(await provider.doesFilter('display', 'contains', 'x')).toBe(false); + + const prep = await provider.getPrepContext(iter); + await provider.searchFilter(prep, 'Alpha', true); + const fromSearch = await provider.executeFilters(prep); + expect(await provider.filterSize(prep, fromSearch[0])).toBeGreaterThanOrEqual(1); + + const inactiveSet = await provider.filter(prep, 'inactive', '=', 'true'); + expect(await provider.filterMore(prep, inactiveSet)).toBe(true); + const concept = await provider.filterConcept(prep, inactiveSet); + expect(concept.code).toBe('C2'); + expect(await provider.filterLocate(prep, inactiveSet, 'C2')).toBeTruthy(); + expect(await provider.filterCheck(prep, inactiveSet, concept)).toBe(true); + + const regexSet = await provider.filter(prep, 'display', 'regex', '^Alpha'); + expect(await provider.filterSize(prep, regexSet)).toBeGreaterThanOrEqual(1); + + const inSet = await provider.filter(prep, 'code', 'in', 'C1,C3'); + expect(await provider.filterSize(prep, inSet)).toBe(1); + + const defSet = await provider.filter(prep, 'definition', '=', 'Alpha Definition'); + expect(await provider.filterSize(prep, defSet)).toBe(1); + + await expect(provider.filter(prep, 'display', 'contains', 'x')).rejects.toThrow('not supported'); + + const exec = await provider.executeFilters(prep); + expect(Array.isArray(exec)).toBe(true); + + await provider.filterFinish(prep); + expect(prep.filters).toHaveLength(0); + + // Exercise #ensureContext promise/wrapper path. + const wrapped = Promise.resolve({ context: { code: 'Z1', display: 'Wrapped', retired: false } }); + expect(await provider.display(wrapped)).toBe('Wrapped'); + }); + + test('factory statics and no-concepts warm load paths are exercised', async () => { + const meta = { + id: 'src-null', + canonicalUrl: 'http://example.org/cs/null', + version: null, + name: 'Null Source', + checksum: null, + conceptsUrl: null, + codeSystem: { jsonObj: { content: 'not-present' } } + }; + + const factory = new OCLSourceCodeSystemFactory(i18n, { get: jest.fn() }, meta); + + expect(factory.defaultVersion()).toBeNull(); + expect(factory.system()).toBe('http://example.org/cs/null'); + expect(factory.name()).toBe('Null Source'); + expect(factory.id()).toBe('src-null'); + expect(factory.iteratable()).toBe(true); + expect(factory.isCompleteNow()).toBe(false); + + const missing = OCLSourceCodeSystemFactory.scheduleBackgroundLoadByKey('http://missing', null, 'x'); + expect(missing).toBe(false); + expect(OCLSourceCodeSystemFactory.checksumForResource('http://missing', null)).toBeNull(); + + OCLSourceCodeSystemFactory.syncCodeSystemResource(null, null, null); + + jest.spyOn(OCLBackgroundJobQueue, 'enqueue').mockImplementation((jobKey, jobType, runJob) => { + Promise.resolve(runJob()).finally(() => { + OCLBackgroundJobQueue.queuedOrRunningKeys.delete(jobKey); + }); + return true; + }); + + factory.scheduleBackgroundLoad('no-concepts'); + await global.TestUtils.waitFor(() => factory.isCompleteNow() === true, 2000); + + const progress = OCLSourceCodeSystemFactory.loadProgress(); + expect(progress.total).toBeGreaterThan(0); + expect(progress.loaded).toBeGreaterThan(0); + }); + + test('scheduleBackgroundLoad exposes queue progress callbacks', async () => { + const meta = { + id: 'src-cb', + canonicalUrl: 'http://example.org/cs/callbacks', + version: '1.0.0', + name: 'Callback Source', + checksum: null, + conceptsUrl: `${baseUrl}/orgs/org-a/sources/src-cb/concepts/`, + codeSystem: { jsonObj: { content: 'not-present' } } + }; + + const client = { + get: jest + .fn() + .mockResolvedValueOnce({ data: { results: [] } }) + .mockResolvedValueOnce({ data: { results: [] }, headers: { 'num-found': 'abc' } }) + }; + + const factory = new OCLSourceCodeSystemFactory(i18n, client, meta); + + const enqueueSpy = jest.spyOn(OCLBackgroundJobQueue, 'enqueue').mockImplementation((jobKey, jobType, runJob, options = {}) => { + if (typeof options.getProgress === 'function') { + options.getProgress(); + } + if (typeof options.resolveJobSize === 'function') { + options.resolveJobSize(); + } + return true; + }); + + factory.scheduleBackgroundLoad('callbacks'); + factory.scheduleBackgroundLoad('callbacks-2'); + expect(enqueueSpy).toHaveBeenCalled(); + expect(factory.currentChecksum()).toBeNull(); + }); +}); diff --git a/tests/ocl/ocl-cs.test.js b/tests/ocl/ocl-cs.test.js new file mode 100644 index 00000000..830e27d3 --- /dev/null +++ b/tests/ocl/ocl-cs.test.js @@ -0,0 +1,291 @@ +const fs = require('fs'); +const fsp = require('fs/promises'); +const path = require('path'); +const nock = require('nock'); + +const { OperationContext } = require('../../tx/operation-context'); +const { OCLCodeSystemProvider, OCLSourceCodeSystemFactory, OCLBackgroundJobQueue } = require('../../tx/ocl/cs-ocl'); +const { CACHE_CS_DIR, getCacheFilePath } = require('../../tx/ocl/cache/cache-paths'); +const { COLD_CACHE_FRESHNESS_MS } = require('../../tx/ocl/shared/constants'); +const { TestUtilities } = require('../test-utilities'); + +function resetQueueState() { + OCLBackgroundJobQueue.pendingJobs = []; + OCLBackgroundJobQueue.activeCount = 0; + OCLBackgroundJobQueue.queuedOrRunningKeys = new Set(); + OCLBackgroundJobQueue.activeJobs = new Map(); + OCLBackgroundJobQueue.enqueueSequence = 0; + if (OCLBackgroundJobQueue.heartbeatTimer) { + clearInterval(OCLBackgroundJobQueue.heartbeatTimer); + OCLBackgroundJobQueue.heartbeatTimer = null; + } +} + +async function clearOclCache() { + await fsp.rm(path.join(process.cwd(), 'data', 'terminology-cache', 'ocl'), { recursive: true, force: true }); +} + +describe('OCL CodeSystem integration', () => { + const baseUrl = 'https://ocl.cs.test'; + let i18n; + + beforeAll(async () => { + i18n = await TestUtilities.loadTranslations(await TestUtilities.loadLanguageDefinitions()); + }); + + beforeEach(async () => { + nock.cleanAll(); + OCLSourceCodeSystemFactory.factoriesByKey.clear(); + resetQueueState(); + await clearOclCache(); + }); + + afterEach(() => { + nock.cleanAll(); + jest.restoreAllMocks(); + }); + + test('metadata discovery and source snapshots are loaded', async () => { + nock(baseUrl) + .get('/orgs/') + .query(true) + .reply(200, { results: [{ id: 'org-a' }] }); + + nock(baseUrl) + .get('/orgs/org-a/sources/') + .query(true) + .reply(200, { + results: [ + { + id: 'src1', + owner: 'org-a', + name: 'Source One', + canonical_url: 'http://example.org/cs/source-one', + version: '2026.1', + concepts_url: '/orgs/org-a/sources/src1/concepts/', + checksums: { standard: 'chk-1' } + } + ] + }); + + const provider = new OCLCodeSystemProvider({ baseUrl }); + const systems = await provider.listCodeSystems('5.0', null); + expect(systems).toHaveLength(1); + expect(systems[0].url).toBe('http://example.org/cs/source-one'); + expect(systems[0].content).toBe('not-present'); + + const metas = provider.getSourceMetas(); + expect(metas).toHaveLength(1); + expect(metas[0].conceptsUrl).toBe(`${baseUrl}/orgs/org-a/sources/src1/concepts/`); + + const ids = new Set(); + provider.assignIds(ids); + expect(Array.from(ids).some(x => x.startsWith('CodeSystem/'))).toBe(true); + await expect(provider.close()).resolves.toBeUndefined(); + }); + + test('getCodeSystemChanges returns staged diffs after refresh', async () => { + nock(baseUrl) + .get('/orgs/') + .query(true) + .times(2) + .reply(200, { results: [{ id: 'org-a' }] }); + + nock(baseUrl) + .get('/orgs/org-a/sources/') + .query(true) + .reply(200, { + results: [{ id: 'src1', owner: 'org-a', canonical_url: 'http://example.org/cs/source-one', version: '1.0.0' }] + }) + .get('/orgs/org-a/sources/') + .query(true) + .reply(200, { + results: [ + { id: 'src1', owner: 'org-a', canonical_url: 'http://example.org/cs/source-one', version: '1.0.1' }, + { id: 'src2', owner: 'org-a', canonical_url: 'http://example.org/cs/source-two', version: '1.0.0' } + ] + }); + + const provider = new OCLCodeSystemProvider({ baseUrl }); + await provider.initialize(); + + const immediate = provider.getCodeSystemChanges('5.0', null); + expect(immediate).toEqual({ added: [], changed: [], deleted: [] }); + + await global.TestUtils.waitFor(() => { + const staged = provider.getCodeSystemChanges('5.0', null); + return staged.added.length === 1 && staged.changed.length === 1; + }, 2000); + }); + + test('factory hydrates from cold cache and skips warm-up while fresh', async () => { + const meta = { + id: 'src1', + canonicalUrl: 'http://example.org/cs/source-one', + version: '1.0.0', + name: 'Source One', + checksum: 'meta-1', + conceptsUrl: `${baseUrl}/orgs/org-a/sources/src1/concepts/`, + codeSystem: { + jsonObj: { + content: 'not-present' + } + } + }; + + await fsp.mkdir(CACHE_CS_DIR, { recursive: true }); + const coldFile = getCacheFilePath(CACHE_CS_DIR, meta.canonicalUrl, meta.version); + await fsp.writeFile(coldFile, JSON.stringify({ + canonicalUrl: meta.canonicalUrl, + version: meta.version, + fingerprint: 'fp-old', + concepts: [ + { code: 'A', display: 'Alpha', retired: false }, + { code: 'B', display: 'Beta', retired: true } + ] + }), 'utf8'); + + const factory = new OCLSourceCodeSystemFactory(i18n, { get: jest.fn() }, meta); + + await global.TestUtils.waitFor(() => factory.isCompleteNow() === true, 2000); + + const opContext = new OperationContext('en-US', i18n); + const provider = factory.build(opContext, []); + + expect(provider.contentMode()).toBe('complete'); + expect(await provider.display('A')).toBe('Alpha'); + expect(await provider.isInactive('B')).toBe(true); + + const enqueueSpy = jest.spyOn(OCLBackgroundJobQueue, 'enqueue'); + factory.scheduleBackgroundLoad('test-fresh-cache'); + expect(enqueueSpy).not.toHaveBeenCalled(); + }); + + test('factory enqueues stale warm-up and replaces cold cache with hot cache', async () => { + const conceptsUrl = `${baseUrl}/orgs/org-a/sources/src1/concepts/`; + const meta = { + id: 'src1', + canonicalUrl: 'http://example.org/cs/source-one', + version: '1.0.0', + name: 'Source One', + checksum: 'meta-2', + conceptsUrl, + codeSystem: { + jsonObj: { + content: 'not-present' + } + } + }; + + await fsp.mkdir(CACHE_CS_DIR, { recursive: true }); + const coldFile = getCacheFilePath(CACHE_CS_DIR, meta.canonicalUrl, meta.version); + await fsp.writeFile(coldFile, JSON.stringify({ + canonicalUrl: meta.canonicalUrl, + version: meta.version, + fingerprint: 'fp-legacy', + concepts: [{ code: 'OLD' }] + }), 'utf8'); + + const staleMs = Date.now() - (COLD_CACHE_FRESHNESS_MS + 120000); + fs.utimesSync(coldFile, new Date(staleMs), new Date(staleMs)); + + nock(baseUrl) + .get('/orgs/org-a/sources/src1/concepts/') + .query(q => Number(q.limit) === 1) + .reply(200, { results: [{ code: 'A' }] }, { num_found: '2' }) + .get('/orgs/org-a/sources/src1/concepts/') + .query(q => Number(q.page) === 1 && Number(q.limit) === 1000) + .reply(200, { + count: 2, + results: [ + { code: 'A', display_name: 'Alpha', retired: false }, + { code: 'B', display_name: 'Beta', retired: false } + ] + }); + + const factory = new OCLSourceCodeSystemFactory(i18n, require('axios').create({ baseURL: baseUrl }), meta); + + // Force queue execution inline for deterministic test behavior. + jest.spyOn(OCLBackgroundJobQueue, 'enqueue').mockImplementation((jobKey, jobType, runJob, options = {}) => { + OCLBackgroundJobQueue.queuedOrRunningKeys.add(jobKey); + Promise.resolve() + .then(async () => { + const size = options.resolveJobSize ? await options.resolveJobSize() : options.jobSize; + await runJob(size); + }) + .finally(() => { + OCLBackgroundJobQueue.queuedOrRunningKeys.delete(jobKey); + }); + return true; + }); + + factory.scheduleBackgroundLoad('stale-cache'); + + await global.TestUtils.waitFor(() => factory.isCompleteNow() === true, 3000); + + const coldData = JSON.parse(await fsp.readFile(coldFile, 'utf8')); + // Existing cold-cache concept remains in shared cache; warm load appends newly fetched concepts. + if (typeof coldData.conceptCount === 'number') { + expect(coldData.conceptCount).toBeGreaterThanOrEqual(2); + } else { + expect(Array.isArray(coldData.concepts)).toBe(true); + expect(coldData.concepts.length).toBeGreaterThanOrEqual(2); + } + expect(coldData.fingerprint).toBeTruthy(); + }); + + test('provider lookup/filter lifecycle is functional for lazy fetches', async () => { + const conceptsUrl = `${baseUrl}/orgs/org-a/sources/src1/concepts/`; + const meta = { + id: 'src1', + canonicalUrl: 'http://example.org/cs/source-one', + version: null, + name: 'Source One', + checksum: 'meta-3', + conceptsUrl, + codeSystem: { + jsonObj: { + property: [{ code: 'display' }, { code: 'inactive' }], + content: 'not-present' + } + } + }; + + nock(baseUrl) + .get('/orgs/org-a/sources/src1/concepts/C3/') + .reply(200, { + code: 'C3', + display_name: 'Gamma term', + description: 'Gamma definition', + retired: false, + names: [{ locale: 'en', name: 'Gamma term' }] + }) + .get('/orgs/org-a/sources/src1/concepts/') + .query(q => Number(q.page) === 1 && Number(q.limit) === 1000) + .reply(200, { + results: [ + { code: 'A1', display_name: 'Alpha', description: 'Alpha definition', retired: false }, + { code: 'B2', display_name: 'Beta', description: 'Beta definition', retired: true } + ] + }) + .get('/orgs/org-a/sources/src1/concepts/') + .query(q => Number(q.page) === 2 && Number(q.limit) === 1000) + .reply(200, { + results: [] + }); + + const factory = new OCLSourceCodeSystemFactory(i18n, require('axios').create({ baseURL: baseUrl }), meta); + const opContext = new OperationContext('en-US', i18n); + const provider = factory.build(opContext, []); + + const located = await provider.locate('C3'); + expect(located.context.code).toBe('C3'); + expect(await provider.display('C3')).toBe('Gamma term'); + + const filterCtx = await provider.getPrepContext(null); + const set = await provider.filter(filterCtx, 'inactive', '=', 'true'); + expect(await provider.filterSize(filterCtx, set)).toBe(1); + + await provider.filterFinish(filterCtx); + }); +}); diff --git a/tests/ocl/ocl-helpers.test.js b/tests/ocl/ocl-helpers.test.js new file mode 100644 index 00000000..5e8d0f90 --- /dev/null +++ b/tests/ocl/ocl-helpers.test.js @@ -0,0 +1,302 @@ +const fs = require('fs'); +const fsp = require('fs/promises'); +const path = require('path'); + +const { createOclHttpClient } = require('../../tx/ocl/http/client'); +const { extractItemsAndNext, fetchAllPages } = require('../../tx/ocl/http/pagination'); +const { sanitizeFilename, getCacheFilePath, CACHE_BASE_DIR } = require('../../tx/ocl/cache/cache-paths'); +const { ensureCacheDirectories, getColdCacheAgeMs, formatCacheAgeMinutes } = require('../../tx/ocl/cache/cache-utils'); +const { computeCodeSystemFingerprint, computeValueSetExpansionFingerprint } = require('../../tx/ocl/fingerprint/fingerprint'); +const { toConceptContext, extractDesignations } = require('../../tx/ocl/mappers/concept-mapper'); +const { OCLConceptFilterContext } = require('../../tx/ocl/model/concept-filter-context'); +const { OCLBackgroundJobQueue } = require('../../tx/ocl/jobs/background-queue'); +const { OCL_CODESYSTEM_MARKER_EXTENSION } = require('../../tx/ocl/shared/constants'); + +function resetQueueState() { + OCLBackgroundJobQueue.pendingJobs = []; + OCLBackgroundJobQueue.activeCount = 0; + OCLBackgroundJobQueue.queuedOrRunningKeys = new Set(); + OCLBackgroundJobQueue.activeJobs = new Map(); + OCLBackgroundJobQueue.enqueueSequence = 0; + if (OCLBackgroundJobQueue.heartbeatTimer) { + clearInterval(OCLBackgroundJobQueue.heartbeatTimer); + OCLBackgroundJobQueue.heartbeatTimer = null; + } +} + +describe('OCL helper modules', () => { + afterEach(() => { + jest.restoreAllMocks(); + resetQueueState(); + }); + + test('createOclHttpClient applies baseUrl and token header', () => { + const out = createOclHttpClient({ baseUrl: 'https://example.org///', token: 'abc' }); + expect(out.baseUrl).toBe('https://example.org'); + expect(out.client.defaults.headers.Authorization).toBe('Token abc'); + + const bearer = createOclHttpClient({ baseUrl: 'https://example.org', token: 'Bearer z' }); + expect(bearer.client.defaults.headers.Authorization).toBe('Bearer z'); + }); + + test('extractItemsAndNext handles arrays, object payloads and baseUrl-relative next links', () => { + expect(extractItemsAndNext([{ id: 1 }])).toEqual({ items: [{ id: 1 }], next: null }); + + const withResults = extractItemsAndNext( + { results: [{ id: 2 }], next: 'https://api.test/sources/?page=2' }, + 'https://api.test' + ); + expect(withResults.items).toHaveLength(1); + expect(withResults.next).toBe('/sources/?page=2'); + + const withItems = extractItemsAndNext({ items: [{ id: 3 }] }); + expect(withItems).toEqual({ items: [{ id: 3 }], next: null }); + }); + + test('fetchAllPages supports page mode and next-link mode', async () => { + const pageCalls = []; + const pageClient = { + get: jest.fn(async (_path, opts) => { + pageCalls.push(opts.params.page); + if (opts.params.page === 1) { + return { data: { results: [{ id: 'a' }, { id: 'b' }] } }; + } + return { data: { results: [{ id: 'c' }] } }; + }) + }; + + const paged = await fetchAllPages(pageClient, '/x', { pageSize: 2 }); + expect(paged.map(x => x.id)).toEqual(['a', 'b', 'c']); + expect(pageCalls).toEqual([1, 2]); + + const nextCalls = []; + const nextClient = { + get: jest.fn(async (pathArg) => { + nextCalls.push(pathArg); + if (pathArg === '/x') { + return { + data: { + results: [{ id: '1' }], + next: 'https://api.test/x?page=2' + } + }; + } + return { + data: { + results: [{ id: '2' }], + next: null + } + }; + }) + }; + + const byNext = await fetchAllPages(nextClient, '/x', { + baseUrl: 'https://api.test', + useNextLinks: true + }); + expect(byNext.map(x => x.id)).toEqual(['1', '2']); + expect(nextCalls).toEqual(['/x', '/x?page=2']); + }); + + test('fetchAllPages logs and rethrows fetch errors', async () => { + const logger = { error: jest.fn() }; + const client = { + get: jest.fn(async () => { + throw new Error('boom'); + }) + }; + + await expect(fetchAllPages(client, '/x', { logger, loggerPrefix: '[T]' })).rejects.toThrow('boom'); + expect(logger.error).toHaveBeenCalled(); + }); + + test('cache path and cache utility helpers behave as expected', async () => { + const s = sanitizeFilename('http://a/b?x=y#z'); + expect(s).toContain('http_a_b_x_y_z'); + + const out = getCacheFilePath(path.join(CACHE_BASE_DIR, 'tmp'), 'http://example.org/vs', '1.0.0', 'f1'); + expect(out.endsWith('.json')).toBe(true); + expect(out).toContain('_p_f1'); + + const dir = path.join(process.cwd(), 'data', 'terminology-cache', 'ocl', 'helper-test'); + await ensureCacheDirectories(dir); + expect(fs.existsSync(dir)).toBe(true); + + const file = path.join(dir, 'age.json'); + await fsp.writeFile(file, '{}', 'utf8'); + const age = getColdCacheAgeMs(file); + expect(age).not.toBeNull(); + expect(formatCacheAgeMinutes(60000)).toBe('1 minute'); + expect(formatCacheAgeMinutes(120000)).toBe('2 minutes'); + + expect(getColdCacheAgeMs(path.join(dir, 'missing.json'))).toBeNull(); + + const mkdirSpy = jest.spyOn(fsp, 'mkdir').mockRejectedValueOnce(new Error('mkdir-fail')); + await expect(ensureCacheDirectories(path.join(dir, 'x'))).resolves.toBeUndefined(); + expect(mkdirSpy).toHaveBeenCalled(); + }); + + test('fingerprints and concept mapper/filter context produce stable outputs', () => { + const csFp1 = computeCodeSystemFingerprint([ + { code: 'b', display: 'B', definition: 'd', retired: false }, + { code: 'a', display: 'A', definition: 'd', retired: true } + ]); + const csFp2 = computeCodeSystemFingerprint([ + { code: 'a', display: 'A', definition: 'd', retired: true }, + { code: 'b', display: 'B', definition: 'd', retired: false } + ]); + expect(csFp1).toBe(csFp2); + expect(computeCodeSystemFingerprint([])).toBeNull(); + expect(computeCodeSystemFingerprint([{ x: 1 }])).toBeNull(); + + const vsFp = computeValueSetExpansionFingerprint({ + contains: [ + { system: 's', code: '1', display: 'one', inactive: false }, + { system: 's', code: '2', display: 'two', inactive: true } + ] + }); + expect(vsFp).toBeTruthy(); + expect(computeValueSetExpansionFingerprint({ contains: [] })).toBeNull(); + expect(computeValueSetExpansionFingerprint(null)).toBeNull(); + + const concept = toConceptContext({ + code: '123', + display_name: 'Main', + description: 'Def', + retired: true, + names: [{ locale: 'en', name: 'Main' }, { locale: 'pt', name: 'Principal' }] + }); + expect(concept.code).toBe('123'); + expect(concept.retired).toBe(true); + expect(toConceptContext(null)).toBeNull(); + expect(toConceptContext({})).toBeNull(); + expect(extractDesignations({ names: [{ locale: 'en', name: 'A' }, { locale: 'en', name: 'A' }] })).toHaveLength(1); + expect(extractDesignations({ locale_display_names: { 'pt-BR': 'Nome' } })).toEqual([ + { language: 'pt-BR', value: 'Nome' } + ]); + + const set = new OCLConceptFilterContext(); + set.add({ code: 'b' }, 1); + set.add({ code: 'a' }, 2); + set.sort(); + expect(set.next().code).toBe('a'); + expect(set.findConceptByCode('b').code).toBe('b'); + set.reset(); + const item = set.next(); + expect(set.containsConcept(item)).toBe(true); + set.next(); + expect(set.next()).toBeNull(); + }); + + test('background queue enforces singleton keys, size ordering and progress formatting', async () => { + OCLBackgroundJobQueue.MAX_CONCURRENT = 0; + + const first = OCLBackgroundJobQueue.enqueue('j1', 'job', async () => {}, { jobSize: 10 }); + const duplicate = OCLBackgroundJobQueue.enqueue('j1', 'job', async () => {}, { jobSize: 1 }); + OCLBackgroundJobQueue.enqueue('j2', 'job', async () => {}, { jobSize: 2 }); + OCLBackgroundJobQueue.enqueue('j3', 'job', async () => {}, { jobSize: 5 }); + + await global.TestUtils.delay(10); + + expect(first).toBe(true); + expect(duplicate).toBe(false); + expect(OCLBackgroundJobQueue.pendingJobs.map(j => j.jobSize)).toEqual([2, 5, 10]); + + expect(OCLBackgroundJobQueue.formatProgress(() => 51.2)).toBe('51%'); + expect(OCLBackgroundJobQueue.formatProgress(() => ({ processed: 25, total: 100 }))).toBe('25%'); + expect(OCLBackgroundJobQueue.formatProgress(() => ({ percentage: 120 }))).toBe('100%'); + expect(OCLBackgroundJobQueue.formatProgress(() => { throw new Error('x'); })).toBe('unknown'); + + const spy = jest.spyOn(console, 'log').mockImplementation(() => {}); + OCLBackgroundJobQueue.logHeartbeat(); + expect(spy).toHaveBeenCalled(); + }); + + test('background queue processes jobs and handles failures', async () => { + OCLBackgroundJobQueue.MAX_CONCURRENT = 1; + const completed = []; + + OCLBackgroundJobQueue.enqueue('qa', 'job', async () => { + completed.push('a'); + }, { jobSize: 1 }); + + OCLBackgroundJobQueue.enqueue('qb', 'job', async () => { + completed.push('b'); + throw new Error('expected'); + }, { jobSize: 2 }); + + await global.TestUtils.waitFor(() => completed.length === 2, 2000); + await global.TestUtils.waitFor(() => OCLBackgroundJobQueue.activeCount === 0, 2000); + expect(OCLBackgroundJobQueue.pendingJobs.length).toBe(0); + }); + + test('patches integrate code filtering and TxParameters hash filter extension', () => { + jest.resetModules(); + + const { patchSearchWorkerForOCLCodeFiltering, ensureTxParametersHashIncludesFilter, normalizeFilterForCacheKey } = require('../../tx/ocl/shared/patches'); + const SearchWorker = require('../../tx/workers/search'); + + const originalSearchCodeSystems = SearchWorker.prototype.searchCodeSystems; + SearchWorker.prototype.searchCodeSystems = function () { + return [ + { + url: 'http://test/cs', + extension: [{ url: OCL_CODESYSTEM_MARKER_EXTENSION, valueBoolean: true }], + concept: [ + { code: 'A', display: 'Alpha', concept: [{ code: 'A1' }] }, + { code: 'B', display: 'Beta' } + ] + }, + { + url: 'http://test/non-ocl', + concept: [{ code: 'X' }] + } + ]; + }; + + try { + patchSearchWorkerForOCLCodeFiltering(); + // idempotence path + patchSearchWorkerForOCLCodeFiltering(); + + const worker = Object.create(SearchWorker.prototype); + const filtered = worker.searchCodeSystems({ code: 'A1' }); + expect(filtered).toHaveLength(2); + expect(filtered[0].concept[0].code).toBe('A'); + expect(filtered[0].concept[0].concept[0].code).toBe('A1'); + + const filteredNone = worker.searchCodeSystems({ code: 'missing' }); + expect(filteredNone).toHaveLength(1); + expect(filteredNone[0].url).toBe('http://test/non-ocl'); + } finally { + SearchWorker.prototype.searchCodeSystems = originalSearchCodeSystems; + } + + class TxParameters { + constructor() { + this.filter = ' TeSt '; + } + + hashSource() { + return 'base'; + } + } + + ensureTxParametersHashIncludesFilter(TxParameters); + const p = new TxParameters(); + expect(p.hashSource()).toBe('base|filter=test'); + expect(normalizeFilterForCacheKey(' ABC ')).toBe('abc'); + }); + + test('patchSearchWorkerForOCLCodeFiltering is safe when worker cannot be loaded', () => { + jest.resetModules(); + jest.isolateModules(() => { + jest.doMock('../../tx/workers/search', () => { + throw new Error('no-worker'); + }, { virtual: true }); + + const { patchSearchWorkerForOCLCodeFiltering } = require('../../tx/ocl/shared/patches'); + expect(() => patchSearchWorkerForOCLCodeFiltering()).not.toThrow(); + }); + }); +}); diff --git a/tests/ocl/ocl-vs-advanced.test.js b/tests/ocl/ocl-vs-advanced.test.js new file mode 100644 index 00000000..28043247 --- /dev/null +++ b/tests/ocl/ocl-vs-advanced.test.js @@ -0,0 +1,428 @@ +const nock = require('nock'); + +const ValueSet = require('../../tx/library/valueset'); +const { OCLValueSetProvider } = require('../../tx/ocl/vs-ocl'); +const { OCLBackgroundJobQueue } = require('../../tx/ocl/cs-ocl'); + +function resetQueueState() { + OCLBackgroundJobQueue.pendingJobs = []; + OCLBackgroundJobQueue.activeCount = 0; + OCLBackgroundJobQueue.queuedOrRunningKeys = new Set(); + OCLBackgroundJobQueue.activeJobs = new Map(); + OCLBackgroundJobQueue.enqueueSequence = 0; + if (OCLBackgroundJobQueue.heartbeatTimer) { + clearInterval(OCLBackgroundJobQueue.heartbeatTimer); + OCLBackgroundJobQueue.heartbeatTimer = null; + } +} + +describe('OCL ValueSet advanced provider behavior', () => { + const baseUrl = 'https://ocl.vs.advanced.test'; + const PAGE_SIZE = 100; + + beforeEach(() => { + nock.cleanAll(); + resetQueueState(); + }); + + afterEach(() => { + nock.cleanAll(); + jest.restoreAllMocks(); + }); + + test('sourcePackage, assignIds with and without spaceId, and search/list methods', async () => { + const provider = new OCLValueSetProvider({ baseUrl, org: 'org-a' }); + provider._initialized = true; + expect(provider.sourcePackage()).toBe(`ocl:${baseUrl}|org=org-a`); + + const vs = new ValueSet({ + resourceType: 'ValueSet', + id: 'vs1', + url: 'http://example.org/vs/1', + version: '1.2.3', + name: 'VS1', + status: 'active' + }, 'R5'); + + provider.valueSetMap.set(vs.url, vs); + provider.valueSetMap.set(`${vs.url}|${vs.version}`, vs); + provider.valueSetMap.set(vs.id, vs); + + const idsNoSpace = new Set(); + provider.assignIds(idsNoSpace); + expect(idsNoSpace.size).toBe(0); + + provider.spaceId = 'S'; + const ids = new Set(); + provider.assignIds(ids); + expect(ids.has('ValueSet/S-vs1')).toBe(true); + + const all = await provider.searchValueSets([]); + expect(all).toHaveLength(1); + expect(provider.vsCount()).toBe(1); + expect(await provider.listAllValueSets()).toEqual(['http://example.org/vs/1']); + }); + + test('fetchValueSet resolves semver major.minor and fetchValueSetById handles prefixed ids', async () => { + const provider = new OCLValueSetProvider({ baseUrl, org: 'org-a' }); + jest.spyOn(OCLBackgroundJobQueue, 'enqueue').mockReturnValue(true); + + const vs = new ValueSet({ + resourceType: 'ValueSet', + id: 'vs1', + url: 'http://example.org/vs/1', + version: '1.2', + name: 'VS1', + status: 'active', + compose: { include: [{ system: 'http://example.org/cs/1' }] } + }, 'R5'); + + provider.valueSetMap.set(vs.url, vs); + provider.valueSetMap.set(`${vs.url}|${vs.version}`, vs); + provider.valueSetMap.set(vs.id, vs); + provider._idMap.set(vs.id, vs); + + const bySemver = await provider.fetchValueSet(vs.url, '1.2.9'); + expect(bySemver).toBeTruthy(); + expect(bySemver.version).toBe('1.2'); + + provider.spaceId = 'X'; + provider._idMap.set('X-vs1', vs); + const byPrefixedId = await provider.fetchValueSetById('X-vs1'); + expect(byPrefixedId).toBeTruthy(); + }); + + test('canonical resolution path uses collection search and semver fallback', async () => { + nock(baseUrl) + .get('/orgs/') + .query(true) + .reply(200, { results: [{ id: 'org-a' }] }) + .get('/orgs/org-a/collections/') + .query(q => q.q === 'my-vs') + .reply(200, { + results: [ + { + id: 'col-vs', + owner: 'org-a', + owner_type: 'Organization', + canonical_url: 'http://example.org/vs/my-vs', + version: '2.1', + name: 'My VS', + concepts_url: '/orgs/org-a/collections/col-vs/concepts/' + } + ] + }) + .get('/orgs/org-a/collections/col-vs/concepts/') + .query(true) + .reply(200, { results: [] }); + + const provider = new OCLValueSetProvider({ baseUrl, org: 'org-a' }); + jest.spyOn(OCLBackgroundJobQueue, 'enqueue').mockReturnValue(true); + + const resolved = await provider.fetchValueSet('http://example.org/vs/my-vs', '2.1.5'); + expect(resolved).toBeTruthy(); + expect(resolved.url).toBe('http://example.org/vs/my-vs'); + expect(resolved.version).toBe('2.1'); + }); + + test('compose include fallback via concepts listing and source canonical lookup', async () => { + nock(baseUrl) + .get('/orgs/') + .query(true) + .reply(200, { results: [{ id: 'org-a' }] }) + .get('/orgs/org-a/collections/') + .query(true) + .reply(200, { + results: [ + { + id: 'col2', + owner: 'org-a', + owner_type: 'Organization', + canonical_url: 'http://example.org/vs/compose', + version: '1.0.0', + name: 'Compose VS', + concepts_url: '/orgs/org-a/collections/col2/concepts/', + expansion_url: '/orgs/org-a/collections/col2/HEAD/expansions/autoexpand-HEAD/' + } + ] + }) + .get('/orgs/org-a/collections/col2/HEAD/expansions/autoexpand-HEAD/') + .reply(500) + .get('/orgs/org-a/collections/col2/concepts/') + .query(q => Number(q.page) === 1 && Number(q.limit) === 1000) + .reply(200, { + results: [ + { owner: 'org-a', source: 'src-a', code: 'A' }, + { owner: 'org-a', source: 'src-b', code: 'B' } + ] + }) + .get('/orgs/org-a/collections/col2/concepts/') + .query(q => Number(q.page) === 2 && Number(q.limit) === 1000) + .reply(200, { results: [] }) + .get('/orgs/org-a/sources/src-a/') + .reply(200, { canonical_url: 'http://example.org/cs/src-a' }) + .get('/orgs/org-a/sources/src-b/') + .reply(200, { canonical_url: 'http://example.org/cs/src-b' }); + + const provider = new OCLValueSetProvider({ baseUrl, org: 'org-a' }); + jest.spyOn(OCLBackgroundJobQueue, 'enqueue').mockReturnValue(true); + + await provider.initialize(); + const vs = await provider.fetchValueSet('http://example.org/vs/compose', '1.0.0'); + expect(vs.jsonObj.compose.include.length).toBe(2); + }); + + test('cached expansion invalidates when metadata signature/dependencies mismatch', async () => { + const provider = new OCLValueSetProvider({ baseUrl, org: 'org-a' }); + jest.spyOn(OCLBackgroundJobQueue, 'enqueue').mockReturnValue(true); + + const vs = new ValueSet({ + resourceType: 'ValueSet', + id: 'vs-invalid', + url: 'http://example.org/vs/invalid', + version: '1.0.0', + name: 'Invalid Cache VS', + status: 'active', + compose: { include: [{ system: 'http://example.org/cs/src-a' }] } + }, 'R5'); + + provider.valueSetMap.set(vs.url, vs); + provider.valueSetMap.set(`${vs.url}|${vs.version}`, vs); + provider.valueSetMap.set(vs.id, vs); + + const cacheKey = `${vs.url}|${vs.version}|default`; + provider.backgroundExpansionCache.set(cacheKey, { + expansion: { contains: [{ system: 'x', code: '1' }] }, + metadataSignature: 'stale-signature', + dependencyChecksums: {}, + createdAt: Date.now() - 7200000 + }); + + const out = await provider.fetchValueSet(vs.url, vs.version); + expect(out).toBeTruthy(); + expect(provider.backgroundExpansionCache.has(cacheKey)).toBe(false); + }); + + test('validation and fallback discovery branches', async () => { + const provider = new OCLValueSetProvider({ baseUrl, org: 'org-a' }); + provider._initialized = true; + + await expect(provider.fetchValueSet('', null)).rejects.toThrow('URL must be a non-empty string'); + await expect(provider.searchValueSets('bad')).rejects.toThrow('Search parameters must be an array'); + + const providerFallback = new OCLValueSetProvider({ baseUrl }); + nock(baseUrl) + .get('/orgs/') + .query(true) + .reply(200, { results: [] }) + .get('/collections/') + .query(true) + .reply(200, { + results: [ + { + id: 'col-fallback', + owner: 'org-a', + owner_type: 'Organization', + canonical_url: 'http://example.org/vs/fallback', + version: '1.0.0', + name: 'Fallback VS' + } + ] + }); + + await providerFallback.initialize(); + expect(providerFallback.vsCount()).toBeGreaterThan(0); + }); + + test('searchValueSets matches system and identifier fields', async () => { + const provider = new OCLValueSetProvider({ baseUrl, org: 'org-a' }); + provider._initialized = true; + + const vs = new ValueSet({ + resourceType: 'ValueSet', + id: 'vs-system-id', + url: 'http://example.org/vs/system-id', + version: '1.0.0', + identifier: [{ system: 'urn:sys', value: 'ABC-123' }], + compose: { include: [{ system: 'http://example.org/cs/target' }] } + }, 'R5'); + + provider.valueSetMap.set(vs.url, vs); + provider.valueSetMap.set(`${vs.url}|${vs.version}`, vs); + + const found = await provider.searchValueSets([ + { name: 'system', value: 'cs/target' }, + { name: 'identifier', value: 'abc-123' } + ]); + + expect(found).toHaveLength(1); + expect(found[0].id).toBe('vs-system-id'); + }); + + test('localized sorting and invalid updated_on handling are exercised', async () => { + nock(baseUrl) + .get('/orgs/') + .query(true) + .reply(200, { results: [{ id: 'org-a' }] }) + .get('/orgs/org-a/collections/') + .query(true) + .reply(200, { + results: [ + { + id: 'col-loc', + owner: 'org-a', + owner_type: 'Organization', + canonical_url: 'http://example.org/vs/localized', + version: '1.0.0', + updated_on: 'invalid-date-value', + concepts_url: '/orgs/org-a/collections/col-loc/concepts/' + } + ] + }) + .get('/orgs/org-a/collections/col-loc/concepts/') + .query(q => Number(q.page) === 1 && Number(q.limit) === 1000) + .reply(200, { + results: [ + { + owner: 'org-a', + source: 'src-a', + code: 'C1' + } + ] + }) + .get('/orgs/org-a/collections/col-loc/concepts/') + .query(q => Number(q.page) === 2 && Number(q.limit) === 1000) + .reply(200, { results: [] }) + .get('/orgs/org-a/collections/col-loc/concepts/') + .query(q => Number(q.page) === 1 && Number(q.limit) === 200 && String(q.verbose) === 'true') + .reply(200, { + results: [ + { + owner: 'org-a', + source: 'src-a', + code: 'C1', + names: [ + { locale: 'pt-BR', name: 'Termo', name_type: 'Synonym' }, + { locale: 'en', name: 'Term', name_type: 'Fully Specified Name', locale_preferred: true }, + { locale: 'en', name: 'Term Variant', name_type: 'Synonym' } + ], + descriptions: [ + { locale: 'pt-BR', description: 'Descricao', description_type: 'Text' }, + { locale: 'en', description: 'Definition EN', description_type: 'Definition', locale_preferred: true }, + { locale: 'en', description: 'Other EN', description_type: 'Note' } + ] + } + ] + }) + .get('/orgs/org-a/sources/src-a/') + .reply(200, { canonical_url: 'http://example.org/cs/src-a' }); + + const provider = new OCLValueSetProvider({ baseUrl, org: 'org-a' }); + jest.spyOn(OCLBackgroundJobQueue, 'enqueue').mockReturnValue(true); + + await provider.initialize(); + const vs = await provider.fetchValueSet('http://example.org/vs/localized', '1.0.0'); + expect(vs.jsonObj.meta).toBeUndefined(); + + const out = await vs.oclFetchConcepts({ + count: 5, + offset: 0, + activeOnly: false, + filter: 'term', + languageCodes: ['en', 'pt-BR'] + }); + + expect(out.contains).toHaveLength(1); + expect(out.contains[0].display).toBe('Term'); + expect(out.contains[0].definition).toBe('Definition EN'); + expect(out.contains[0].designation.length).toBeGreaterThan(1); + expect(out.contains[0].definitions.length).toBeGreaterThan(1); + + await expect(provider.close()).resolves.toBeUndefined(); + }); + + test('warm-up enqueue exposes progress callbacks and remote query chooses longest token', async () => { + nock(baseUrl) + .get('/orgs/') + .query(true) + .reply(200, { results: [{ id: 'org-a' }] }) + .get('/orgs/org-a/collections/') + .query(true) + .reply(200, { + results: [ + { + id: 'col-q', + owner: 'org-a', + owner_type: 'Organization', + canonical_url: 'http://example.org/vs/q', + version: '1.0.0', + concepts_url: '/orgs/org-a/collections/col-q/concepts/' + } + ] + }) + .get('/orgs/org-a/collections/col-q/concepts/') + .query(q => Number(q.page) === 1 && Number(q.limit) === 1000) + .reply(200, { results: [] }) + .get('/orgs/org-a/collections/col-q/concepts/') + .query(q => Number(q.page) === 1 && Number(q.limit) === 200 && String(q.verbose) === 'true' && q.q === 'alphabet') + .reply(200, { + results: [ + { code: 'X1', owner: 'org-a', source: 'src-a', display_name: 'Alphabet term', retired: false } + ] + }); + + const provider = new OCLValueSetProvider({ baseUrl, org: 'org-a' }); + const enqueueSpy = jest.spyOn(OCLBackgroundJobQueue, 'enqueue').mockImplementation((jobKey, jobType, runJob, options = {}) => { + if (typeof options.getProgress === 'function') { + options.getProgress(); + } + if (typeof options.resolveJobSize === 'function') { + options.resolveJobSize(); + } + return true; + }); + + await provider.initialize(); + const vs = await provider.fetchValueSet('http://example.org/vs/q', '1.0.0'); + const out = await vs.oclFetchConcepts({ + count: 10, + offset: 0, + filter: 'abc or alphabet', + activeOnly: false + }); + + expect(enqueueSpy).toHaveBeenCalled(); + expect(out.contains).toHaveLength(1); + }); + + test('collection discovery paginates across PAGE_SIZE boundary', async () => { + const page1 = Array.from({ length: PAGE_SIZE }, (_, i) => ({ + id: `col-p1-${i}`, + owner: 'org-a', + owner_type: 'Organization', + canonical_url: `http://example.org/vs/p1/${i}`, + version: '1.0.0', + updated_on: '2026-02-03T04:05:06.000Z', + concepts_url: `/orgs/org-a/collections/col-p1-${i}/concepts/` + })); + + nock(baseUrl) + .get('/orgs/') + .query(q => Number(q.page) === 1 && Number(q.limit) === PAGE_SIZE) + .reply(200, { results: [{ id: 'org-a' }] }) + .get('/orgs/org-a/collections/') + .query(q => Number(q.page) === 1 && Number(q.limit) === PAGE_SIZE) + .reply(200, { results: page1 }) + .get('/orgs/org-a/collections/') + .query(q => Number(q.page) === 2 && Number(q.limit) === PAGE_SIZE) + .reply(200, { results: [] }); + + const provider = new OCLValueSetProvider({ baseUrl, org: 'org-a' }); + await provider.initialize(); + + expect(provider.vsCount()).toBeGreaterThanOrEqual(PAGE_SIZE); + const found = await provider.searchValueSets([{ name: 'url', value: 'http://example.org/vs/p1/0' }]); + expect(found).toHaveLength(1); + expect(found[0].jsonObj.meta.lastUpdated).toBe('2026-02-03T04:05:06.000Z'); + }); +}); diff --git a/tests/ocl/ocl-vs.test.js b/tests/ocl/ocl-vs.test.js new file mode 100644 index 00000000..1ac86f4d --- /dev/null +++ b/tests/ocl/ocl-vs.test.js @@ -0,0 +1,289 @@ +const fs = require('fs'); +const fsp = require('fs/promises'); +const path = require('path'); +const nock = require('nock'); + +const { OCLValueSetProvider } = require('../../tx/ocl/vs-ocl'); +const { OCLSourceCodeSystemFactory, OCLBackgroundJobQueue } = require('../../tx/ocl/cs-ocl'); +const { CACHE_VS_DIR, getCacheFilePath } = require('../../tx/ocl/cache/cache-paths'); +const { COLD_CACHE_FRESHNESS_MS } = require('../../tx/ocl/shared/constants'); + +function resetQueueState() { + OCLBackgroundJobQueue.pendingJobs = []; + OCLBackgroundJobQueue.activeCount = 0; + OCLBackgroundJobQueue.queuedOrRunningKeys = new Set(); + OCLBackgroundJobQueue.activeJobs = new Map(); + OCLBackgroundJobQueue.enqueueSequence = 0; + if (OCLBackgroundJobQueue.heartbeatTimer) { + clearInterval(OCLBackgroundJobQueue.heartbeatTimer); + OCLBackgroundJobQueue.heartbeatTimer = null; + } +} + +async function clearOclCache() { + await fsp.rm(path.join(process.cwd(), 'data', 'terminology-cache', 'ocl'), { recursive: true, force: true }); +} + +describe('OCL ValueSet integration', () => { + const baseUrl = 'https://ocl.vs.test'; + const conceptsPath = '/orgs/org-a/collections/col1/concepts/'; + const expansionPath = '/orgs/org-a/collections/col1/HEAD/expansions/autoexpand-HEAD/'; + + beforeEach(async () => { + nock.cleanAll(); + OCLSourceCodeSystemFactory.factoriesByKey.clear(); + resetQueueState(); + await clearOclCache(); + }); + + afterEach(() => { + nock.cleanAll(); + jest.restoreAllMocks(); + }); + + function mockDiscovery() { + nock(baseUrl) + .get('/orgs/') + .query(true) + .reply(200, { results: [{ id: 'org-a' }] }) + .get('/orgs/org-a/collections/') + .query(true) + .reply(200, { + results: [ + { + id: 'col1', + owner: 'org-a', + owner_type: 'Organization', + canonical_url: 'http://example.org/vs/one', + version: '1.0.0', + preferred_source: 'http://example.org/cs/source-one', + concepts_url: conceptsPath, + expansion_url: expansionPath + } + ] + }); + } + + test('metadata discovery and cold-cache hydration for expansions', async () => { + await fsp.mkdir(CACHE_VS_DIR, { recursive: true }); + const coldPath = getCacheFilePath(CACHE_VS_DIR, 'http://example.org/vs/one', '1.0.0', 'default'); + await fsp.writeFile(coldPath, JSON.stringify({ + canonicalUrl: 'http://example.org/vs/one', + version: '1.0.0', + paramsKey: 'default', + fingerprint: 'fp-vs', + timestamp: new Date().toISOString(), + expansion: { + contains: [{ system: 'http://example.org/cs/source-one', code: 'A', display: 'Alpha' }] + } + }), 'utf8'); + + mockDiscovery(); + + const provider = new OCLValueSetProvider({ baseUrl, org: 'org-a' }); + await provider.initialize(); + jest.spyOn(OCLBackgroundJobQueue, 'enqueue').mockReturnValue(true); + + const fetched = await provider.fetchValueSet('http://example.org/vs/one', '1.0.0'); + expect(fetched).toBeTruthy(); + expect(fetched.url).toBe('http://example.org/vs/one'); + expect(fetched.oclMeta.conceptsUrl).toBe(`${baseUrl}${conceptsPath}`); + }); + + test('warm-up is skipped when cold cache is <= 1 hour old', async () => { + mockDiscovery(); + + const provider = new OCLValueSetProvider({ baseUrl, org: 'org-a' }); + await provider.initialize(); + + await fsp.mkdir(CACHE_VS_DIR, { recursive: true }); + const coldPath = getCacheFilePath(CACHE_VS_DIR, 'http://example.org/vs/one', '1.0.0', 'default'); + await fsp.writeFile(coldPath, JSON.stringify({ + canonicalUrl: 'http://example.org/vs/one', + version: '1.0.0', + paramsKey: 'default', + fingerprint: 'fp-vs', + timestamp: new Date().toISOString(), + expansion: { + contains: [{ system: 'http://example.org/cs/source-one', code: 'A', display: 'Alpha' }] + }, + metadataSignature: '{"k":"v"}', + dependencyChecksums: {} + }), 'utf8'); + + const enqueueSpy = jest.spyOn(OCLBackgroundJobQueue, 'enqueue'); + const fetched = await provider.fetchValueSet('http://example.org/vs/one', '1.0.0'); + + expect(fetched).toBeTruthy(); + expect(enqueueSpy).not.toHaveBeenCalled(); + }); + + test('stale cache triggers warm-up enqueue, expansion build, fingerprint and disk replacement', async () => { + mockDiscovery(); + + nock(baseUrl) + .get(expansionPath) + .reply(200, { + resolved_source_versions: [ + { + canonical_url: 'http://example.org/cs/source-one', + version: '2026.1' + } + ] + }) + .get(conceptsPath) + .query(q => Number(q.limit) === 1) + .times(2) + .reply(200, { results: [{ code: 'A' }] }, { num_found: '3' }) + .get(conceptsPath) + .query(q => Number(q.page) === 1 && Number(q.limit) === 1000 && String(q.verbose) === 'true') + .reply(200, { + results: [ + { + code: 'A', + display_name: 'Alpha', + definition: 'Alpha definition', + retired: false, + owner: 'org-a', + source: 'src-1', + source_canonical_url: 'http://example.org/cs/source-one', + names: [{ locale: 'en', name: 'Alpha', locale_preferred: true }], + descriptions: [{ locale: 'en', description: 'Alpha definition', locale_preferred: true }] + }, + { + code: 'B', + display_name: 'Beta', + retired: true, + owner: 'org-a', + source: 'src-1', + source_canonical_url: 'http://example.org/cs/source-one', + names: [{ locale: 'pt-BR', name: 'Beta' }] + } + ] + }) + .get(conceptsPath) + .query(q => Number(q.page) === 2 && Number(q.limit) === 1000 && String(q.verbose) === 'true') + .reply(200, { results: [] }); + + const provider = new OCLValueSetProvider({ baseUrl, org: 'org-a' }); + await provider.initialize(); + + await fsp.mkdir(CACHE_VS_DIR, { recursive: true }); + const coldPath = getCacheFilePath(CACHE_VS_DIR, 'http://example.org/vs/one', '1.0.0', 'default'); + await fsp.writeFile(coldPath, JSON.stringify({ + canonicalUrl: 'http://example.org/vs/one', + version: '1.0.0', + paramsKey: 'default', + fingerprint: 'old-fingerprint', + timestamp: new Date(Date.now() - (COLD_CACHE_FRESHNESS_MS + 120000)).toISOString(), + expansion: { + contains: [{ system: 'http://example.org/cs/source-one', code: 'OLD' }] + }, + metadataSignature: null, + dependencyChecksums: {} + }), 'utf8'); + const staleMs = Date.now() - (COLD_CACHE_FRESHNESS_MS + 120000); + fs.utimesSync(coldPath, new Date(staleMs), new Date(staleMs)); + + jest.spyOn(OCLSourceCodeSystemFactory, 'scheduleBackgroundLoadByKey').mockImplementation(() => true); + jest.spyOn(OCLBackgroundJobQueue, 'enqueue').mockImplementation((jobKey, jobType, runJob, options = {}) => { + OCLBackgroundJobQueue.queuedOrRunningKeys.add(jobKey); + Promise.resolve() + .then(async () => { + const size = options.resolveJobSize ? await options.resolveJobSize() : options.jobSize; + await runJob(size); + }) + .finally(() => { + OCLBackgroundJobQueue.queuedOrRunningKeys.delete(jobKey); + }); + return true; + }); + + const vs = await provider.fetchValueSet('http://example.org/vs/one', '1.0.0'); + expect(vs).toBeTruthy(); + + await global.TestUtils.waitFor(async () => { + try { + const data = JSON.parse(await fsp.readFile(coldPath, 'utf8')); + return data.conceptCount === 2; + } catch (_e) { + return false; + } + }, 3000); + + const updated = JSON.parse(await fsp.readFile(coldPath, 'utf8')); + expect(updated.conceptCount).toBe(2); + expect(updated.fingerprint).toBeTruthy(); + }); + + test('filter handling in oclFetchConcepts and cache key behavior for filtered calls', async () => { + mockDiscovery(); + + nock(baseUrl) + .get(conceptsPath) + .query(q => Number(q.page) === 1 && Number(q.limit) === 200 && String(q.verbose) === 'true' && q.q === 'alpha') + .reply(200, { + num_found: 2, + results: [ + { + code: 'A', + display_name: 'Alpha term', + definition: 'Definition alpha', + retired: false, + owner: 'org-a', + source: 'src-1', + source_canonical_url: 'http://example.org/cs/source-one', + names: [{ locale: 'en', name: 'Alpha term', locale_preferred: true }], + descriptions: [{ locale: 'en', description: 'Definition alpha' }] + }, + { + code: 'B', + display_name: 'Beta', + definition: 'No match', + retired: false, + owner: 'org-a', + source: 'src-1', + source_canonical_url: 'http://example.org/cs/source-one' + } + ] + }); + + const provider = new OCLValueSetProvider({ baseUrl, org: 'org-a' }); + await provider.initialize(); + jest.spyOn(OCLBackgroundJobQueue, 'enqueue').mockReturnValue(true); + + const vs = await provider.fetchValueSet('http://example.org/vs/one', '1.0.0'); + const out = await vs.oclFetchConcepts({ + count: 20, + offset: 0, + activeOnly: false, + filter: ' alpha ', + languageCodes: ['pt-BR', 'en'] + }); + + expect(out.contains).toHaveLength(1); + expect(out.contains[0].code).toBe('A'); + expect(out.contains[0].display).toBe('Alpha term'); + }); + + test('fetchValueSetById and search/list methods are deterministic', async () => { + mockDiscovery(); + + const provider = new OCLValueSetProvider({ baseUrl, org: 'org-a' }); + provider.spaceId = 'S'; + await provider.initialize(); + jest.spyOn(OCLBackgroundJobQueue, 'enqueue').mockReturnValue(true); + + const ids = new Set(); + provider.assignIds(ids); + + const byId = await provider.fetchValueSetById('S-col1'); + expect(byId).toBeTruthy(); + + const search = await provider.searchValueSets([{ name: 'url', value: 'http://example.org/vs/one' }]); + expect(search).toHaveLength(1); + + const all = await provider.listAllValueSets(); + expect(all).toContain('http://example.org/vs/one'); + }); +}); diff --git a/tx/ocl/README.md b/tx/ocl/README.md new file mode 100644 index 00000000..c46324c2 --- /dev/null +++ b/tx/ocl/README.md @@ -0,0 +1,236 @@ +# OCL Integration in FHIRsmith + +## Overview +The `tx/ocl` module integrates [Open Concept Lab (OCL)](https://openconceptlab.org/) as a terminology source inside FHIRsmith. + +It provides adapters/providers for: + +- `CodeSystem` from OCL Sources +- `ValueSet` from OCL Collections +- `ConceptMap` from OCL Mappings + +In FHIRsmith terms, these providers are loaded by `tx/library.js` when a source entry starts with `ocl:`. OCL metadata is discovered from OCL APIs, and heavy content (concept lists/expansions) is loaded lazily and warmed in background jobs. + +## Architecture +### Main modules +- `cs-ocl.js` + - `OCLCodeSystemProvider`: discovers OCL sources and publishes CodeSystem metadata. + - `OCLSourceCodeSystemFactory`: creates runtime CodeSystem providers with shared caches, cold-cache hydration, and background full-load jobs. + - `OCLSourceCodeSystemProvider`: runtime concept lookup/filter/search behavior for one source. +- `vs-ocl.js` + - `OCLValueSetProvider`: discovers OCL collections, resolves compose includes, serves ValueSet metadata, and builds cached expansions in background. +- `cm-ocl.js` + - `OCLConceptMapProvider`: resolves OCL mappings for fetch/search/translation candidate discovery. + +### Supporting modules under `tx/ocl` +- `http/client.js`: Axios client creation, base URL normalization, token auth header (`Token` or `Bearer`). +- `http/pagination.js`: OCL pagination helper (`results/items/data`, `next`, page mode fallback). +- `cache/cache-paths.js`: cold-cache directories and canonical URL to file path mapping. +- `cache/cache-utils.js`: cache directory creation, file age detection, friendly age formatting. +- `fingerprint/fingerprint.js`: deterministic SHA-256 fingerprints for CodeSystem concepts and ValueSet expansions. +- `jobs/background-queue.js`: singleton keyed queue, size-priority ordering, max concurrency = 2, heartbeat logging every 30s. +- `mappers/concept-mapper.js`: maps OCL concept payloads to internal concept context shape. +- `model/concept-filter-context.js`: ranked filter result set used by CodeSystem filters. +- `shared/constants.js`: defaults and constants (`PAGE_SIZE`, cache freshness window, etc.). +- `shared/patches.js`: + - patches search worker so `CodeSystem?url=...&code=...` on OCL resources only returns matching concept subtree. + - patches `TxParameters.hashSource()` to include `filter` for expansion cache key differentiation. + +## Runtime flow +### Metadata discovery +CodeSystems (`cs-ocl.js`): +- discover orgs via `/orgs/` +- for each org, discover sources via `/orgs/{org}/sources/` +- fallback to `/sources/` if org listing is unavailable + +ValueSets (`vs-ocl.js`): +- discover orgs via `/orgs/` +- discover collections via `/orgs/{org}/collections/` +- fallback to `/collections/` + +ConceptMaps (`cm-ocl.js`): +- fetch by id via `/mappings/{id}/` +- search via `/mappings/` or `/orgs/{org}/mappings/` + +### Lazy loading +- CodeSystem concepts are not fully loaded at metadata discovery time. +- ValueSet expansions are not built inline by default. +- Missing concept/page/expansion data triggers background warm-up scheduling. + +### Cold cache (disk) +- Base folder: `data/terminology-cache/ocl` +- CodeSystems: `data/terminology-cache/ocl/codesystems` +- ValueSets: `data/terminology-cache/ocl/valuesets` + +On startup/initialization: +- CodeSystem factory hydrates concept cache from cold cache file if present. +- ValueSet provider loads cached expansions from cold cache files. + +Corrupt cache handling: +- JSON parse/read errors are logged and skipped; provider continues without crashing. + +### Hot cache (memory) +- CodeSystems: shared in-memory concept/page caches per factory. +- ValueSets: in-memory expansion cache keyed by ValueSet + params hash. +- ConceptMaps: in-memory map keyed by URL/version/id. + +### Background warm-up jobs +Queue behavior (`OCLBackgroundJobQueue`): +- singleton job key (skip duplicate key) +- max concurrency = `2` +- ordering by `jobSize` (smaller concept count first) +- heartbeat log every `30s` +- progress supports `{ processed, total }` and `%` + +CodeSystem warm-up: +- skipped when cold-cache file age is `<= 1 hour` +- enqueued when stale/no cold cache +- loads all concept pages +- computes fingerprint from full concept content +- if fingerprint changed, replaces cold cache file + +ValueSet warm-up: +- skipped when freshest cold cache age (file mtime or cached timestamp) is `<= 1 hour` +- enqueued when stale/no cold cache +- builds expansion by paging collection concepts +- computes expansion fingerprint +- if fingerprint changed, replaces cold cache file + +### Fingerprint/checksum strategy +- OCL source checksum is treated as informational only in `cs-ocl.js` comments. +- Cache replacement decisions use custom fingerprints from concept/expansion content. +- ValueSet cache validity also checks metadata signature and dependency checksums from referenced CodeSystems. + +### ValueSet expansion filtering +`vs-ocl.js` supports filtered concept retrieval through `valueSet.oclFetchConcepts(...)`: +- local baseline filtering (code/display/definition text contains) +- `SearchFilterText` token behavior +- remote query hint (`q`) generated from normalized filter tokens +- dedicated smaller page size for filtered calls (`FILTERED_CONCEPT_PAGE_SIZE`) + +### `/CodeSystem?url=...&code=...` behavior for OCL +`shared/patches.js` patches search worker to apply concept subtree filtering only for OCL-marked CodeSystems (extension URL `http://fhir.org/FHIRsmith/StructureDefinition/ocl-codesystem`). + +## Configuration +## Activation in FHIRsmith +OCL is activated by adding an OCL source line in the TX library YAML (for example `data/library.yml`): + +```yaml +sources: + - ocl:https://oclapi2.ips.hsl.org.br +``` + +`tx/library.js` loads this via `loadOcl()`. + +### Source syntax +`tx/library.js` parses: + +```text +ocl:|org=|token=|timeout= +``` + +Parsed keys: +- `baseUrl` (required) +- `org` (optional) +- `token` (optional) +- `timeout` (optional positive number) + +Examples: + +```yaml +sources: + - ocl:https://api.openconceptlab.org + - ocl:https://ocl.example.org|org=my-org + - ocl:https://ocl.example.org|org=my-org|token=my-ocl-token|timeout=45000 +``` + +### Config value aliasing +`Library.resolveOclConfigValue()` can resolve symbolic values from top-level YAML `ocl:` object loaded into `this.oclConfig`. + +Example pattern: + +```yaml +ocl: + ocl-base: https://ocl.example.org + ocl-token: Token abc123 + +sources: + - ocl:ocl-base|org=my-org|token=ocl-token +``` + +### Credentials and URLs +- Base URL is required (OCL API root). +- Token is optional, sent as `Authorization`: + - `Token ` if no prefix is provided + - preserved if already `Token ...` or `Bearer ...` + +### Enable/disable integration +- Enable by including at least one `ocl:` source entry in TX library YAML. +- Disable by removing/commenting those `ocl:` entries. +- `modules.tx.enabled` must be true in server config to expose TX endpoints. + +## Cache behavior details +### Startup hydration +- CodeSystem cold cache is loaded when factory is created (`OCLSourceCodeSystemFactory` constructor path). +- ValueSet cold cache is loaded during `OCLValueSetProvider.initialize()`. + +### 1-hour freshness rule +- Fresh threshold constant: `COLD_CACHE_FRESHNESS_MS = 60 * 60 * 1000`. +- If cold cache age is `<= 1 hour`, warm-up scheduling is skipped. + +### Hot cache replacing cold cache +- On successful background refresh, new fingerprint is compared to previous cold-cache fingerprint. +- If changed, cold cache is overwritten with the refreshed in-memory state. + +## Operational notes +### Logs to expect +Prefixes: +- `[OCL]` for CodeSystem/queue/general OCL flow +- `[OCL-ValueSet]` for ValueSet flow + +Typical events: +- fetched org/source/collection counts +- cold cache loaded/saved +- warm-up skipped/enqueued/started/completed +- fingerprint unchanged/changed +- queue status + heartbeat snapshots + +### Troubleshooting hints +- If nothing loads, verify OCL base URL and org visibility. +- If cache never warms, check cold-cache timestamps and 1-hour rule. +- If searches by `code` behave unexpectedly, confirm OCL marker extension is present on CodeSystem resources. +- For ValueSet filter behavior, verify `filter` is included in request path through `TxParameters.hashSource` patch and that filter text normalizes as expected. + +### Known limitations (from current implementation) +- OCL checksums are not relied on for cache invalidation. +- Some source/collection discovery paths depend on OCL endpoint support and can fallback. +- Missing endpoints returning `404` in some concept fetch paths are treated as empty content for graceful degradation. + +## Developer notes +### Where to extend +- Add/adjust OCL HTTP behavior: `tx/ocl/http/*` +- Add cache policy/path behavior: `tx/ocl/cache/*` +- Add queue policy: `tx/ocl/jobs/background-queue.js` +- Add mapping logic from OCL payloads: `tx/ocl/mappers/*` +- Add fingerprint strategy: `tx/ocl/fingerprint/*` +- Add provider-specific behavior: + - CodeSystem: `tx/ocl/cs-ocl.js` + - ValueSet: `tx/ocl/vs-ocl.js` + - ConceptMap: `tx/ocl/cm-ocl.js` + +### Ownership by concern +- HTTP access: `http/client.js`, `http/pagination.js` +- Disk cold cache and pathing: `cache/cache-paths.js`, `cache/cache-utils.js` +- Background jobs and scheduling policy: `jobs/background-queue.js` +- Fingerprints/checksums: `fingerprint/fingerprint.js` and provider compare logic +- Worker/runtime patches: `shared/patches.js` + +### Test strategy for future changes +- Keep provider tests deterministic by mocking OCL HTTP responses and cold-cache files. +- Validate stale/fresh transitions with file mtime control. +- Validate queue behavior with isolated static state reset between tests. +- Track coverage with: + +```bash +npm test -- --runInBand tests/ocl --coverage --collectCoverageFrom="tx/ocl/**/*.js" +``` diff --git a/tx/ocl/cache/cache-paths.cjs b/tx/ocl/cache/cache-paths.cjs new file mode 100644 index 00000000..c587cf66 --- /dev/null +++ b/tx/ocl/cache/cache-paths.cjs @@ -0,0 +1,32 @@ +const path = require('path'); + +const CACHE_BASE_DIR = path.join(process.cwd(), 'data', 'terminology-cache', 'ocl'); +const CACHE_CS_DIR = path.join(CACHE_BASE_DIR, 'codesystems'); +const CACHE_VS_DIR = path.join(CACHE_BASE_DIR, 'valuesets'); + +function sanitizeFilename(text) { + if (!text || typeof text !== 'string') { + return 'unknown'; + } + return text + .replace(/[^a-zA-Z0-9._-]/g, '_') + .replace(/_+/g, '_') + .substring(0, 200); +} + +function getCacheFilePath(baseDir, canonicalUrl, version = null, paramsKey = null) { + const filename = sanitizeFilename(canonicalUrl) + + (version ? `_${sanitizeFilename(version)}` : '') + + (paramsKey && paramsKey !== 'default' ? `_p_${sanitizeFilename(paramsKey)}` : '') + + '.json'; + + return path.join(baseDir, filename); +} + +module.exports = { + CACHE_BASE_DIR, + CACHE_CS_DIR, + CACHE_VS_DIR, + sanitizeFilename, + getCacheFilePath +}; diff --git a/tx/ocl/cache/cache-paths.js b/tx/ocl/cache/cache-paths.js index c587cf66..186eed8c 100644 --- a/tx/ocl/cache/cache-paths.js +++ b/tx/ocl/cache/cache-paths.js @@ -1,32 +1,2 @@ -const path = require('path'); +module.exports = require('./cache-paths.cjs'); -const CACHE_BASE_DIR = path.join(process.cwd(), 'data', 'terminology-cache', 'ocl'); -const CACHE_CS_DIR = path.join(CACHE_BASE_DIR, 'codesystems'); -const CACHE_VS_DIR = path.join(CACHE_BASE_DIR, 'valuesets'); - -function sanitizeFilename(text) { - if (!text || typeof text !== 'string') { - return 'unknown'; - } - return text - .replace(/[^a-zA-Z0-9._-]/g, '_') - .replace(/_+/g, '_') - .substring(0, 200); -} - -function getCacheFilePath(baseDir, canonicalUrl, version = null, paramsKey = null) { - const filename = sanitizeFilename(canonicalUrl) - + (version ? `_${sanitizeFilename(version)}` : '') - + (paramsKey && paramsKey !== 'default' ? `_p_${sanitizeFilename(paramsKey)}` : '') - + '.json'; - - return path.join(baseDir, filename); -} - -module.exports = { - CACHE_BASE_DIR, - CACHE_CS_DIR, - CACHE_VS_DIR, - sanitizeFilename, - getCacheFilePath -}; diff --git a/tx/ocl/cache/cache-utils.cjs b/tx/ocl/cache/cache-utils.cjs new file mode 100644 index 00000000..b0a89af5 --- /dev/null +++ b/tx/ocl/cache/cache-utils.cjs @@ -0,0 +1,43 @@ +const fs = require('fs/promises'); +const fsSync = require('fs'); + +async function ensureCacheDirectories(...dirs) { + for (const dir of dirs) { + if (!dir) { + continue; + } + + try { + await fs.mkdir(dir, { recursive: true }); + } catch (error) { + console.error('[OCL] Failed to create cache directory:', dir, error.message); + } + } +} + +function getColdCacheAgeMs(cacheFilePath, logPrefix = '[OCL]') { + try { + const stats = fsSync.statSync(cacheFilePath); + if (!stats || !Number.isFinite(stats.mtimeMs)) { + return null; + } + + return Math.max(0, Date.now() - stats.mtimeMs); + } catch (error) { + if (error && error.code !== 'ENOENT') { + console.error(`${logPrefix} Failed to inspect cold cache file ${cacheFilePath}: ${error.message}`); + } + return null; + } +} + +function formatCacheAgeMinutes(ageMs) { + const minutes = Math.max(1, Math.round(ageMs / 60000)); + return `${minutes} minute${minutes === 1 ? '' : 's'}`; +} + +module.exports = { + ensureCacheDirectories, + getColdCacheAgeMs, + formatCacheAgeMinutes +}; diff --git a/tx/ocl/cache/cache-utils.js b/tx/ocl/cache/cache-utils.js index b0a89af5..402736aa 100644 --- a/tx/ocl/cache/cache-utils.js +++ b/tx/ocl/cache/cache-utils.js @@ -1,43 +1,2 @@ -const fs = require('fs/promises'); -const fsSync = require('fs'); +module.exports = require('./cache-utils.cjs'); -async function ensureCacheDirectories(...dirs) { - for (const dir of dirs) { - if (!dir) { - continue; - } - - try { - await fs.mkdir(dir, { recursive: true }); - } catch (error) { - console.error('[OCL] Failed to create cache directory:', dir, error.message); - } - } -} - -function getColdCacheAgeMs(cacheFilePath, logPrefix = '[OCL]') { - try { - const stats = fsSync.statSync(cacheFilePath); - if (!stats || !Number.isFinite(stats.mtimeMs)) { - return null; - } - - return Math.max(0, Date.now() - stats.mtimeMs); - } catch (error) { - if (error && error.code !== 'ENOENT') { - console.error(`${logPrefix} Failed to inspect cold cache file ${cacheFilePath}: ${error.message}`); - } - return null; - } -} - -function formatCacheAgeMinutes(ageMs) { - const minutes = Math.max(1, Math.round(ageMs / 60000)); - return `${minutes} minute${minutes === 1 ? '' : 's'}`; -} - -module.exports = { - ensureCacheDirectories, - getColdCacheAgeMs, - formatCacheAgeMinutes -}; diff --git a/tx/ocl/cm-ocl.cjs b/tx/ocl/cm-ocl.cjs new file mode 100644 index 00000000..c91fc486 --- /dev/null +++ b/tx/ocl/cm-ocl.cjs @@ -0,0 +1,531 @@ +const { AbstractConceptMapProvider } = require('../cm/cm-api'); +const { ConceptMap } = require('../library/conceptmap'); +const { PAGE_SIZE } = require('./shared/constants'); +const { createOclHttpClient } = require('./http/client'); +const { fetchAllPages, extractItemsAndNext } = require('./http/pagination'); + +const DEFAULT_MAX_SEARCH_PAGES = 10; + +class OCLConceptMapProvider extends AbstractConceptMapProvider { + constructor(config = {}) { + super(); + const options = typeof config === 'string' ? { baseUrl: config } : (config || {}); + + this.org = options.org || null; + this.maxSearchPages = options.maxSearchPages || DEFAULT_MAX_SEARCH_PAGES; + const http = createOclHttpClient(options); + this.baseUrl = http.baseUrl; + this.httpClient = http.client; + + this.conceptMapMap = new Map(); + this._idMap = new Map(); + this._sourceCandidatesCache = new Map(); + this._sourceUrlsByCanonical = new Map(); + this._canonicalBySourceUrl = new Map(); + } + + assignIds(ids) { + if (!this.spaceId) { + return; + } + + const unique = new Set(this.conceptMapMap.values()); + this._idMap.clear(); + + for (const cm of unique) { + if (!cm.id.startsWith(`${this.spaceId}-`)) { + const nextId = `${this.spaceId}-${cm.id}`; + cm.id = nextId; + cm.jsonObj.id = nextId; + } + this._idMap.set(cm.id, cm); + ids.add(`ConceptMap/${cm.id}`); + } + } + + async fetchConceptMap(url, version) { + this._validateFetchParams(url, version); + + const direct = this.conceptMapMap.get(`${url}|${version}`) || this.conceptMapMap.get(url); + if (direct) { + return direct; + } + + const mappingId = this.#extractMappingId(url); + if (mappingId) { + return await this.fetchConceptMapById(mappingId); + } + + const mappings = await this.#searchMappings({ from_source_url: url }, this.maxSearchPages); + for (const mapping of mappings) { + const cm = this.#toConceptMap(mapping); + if (cm) { + this.#indexConceptMap(cm); + if (cm.url === url && (!version || cm.version === version)) { + return cm; + } + } + } + + return null; + } + + async fetchConceptMapById(id) { + if (this._idMap.has(id)) { + return this._idMap.get(id); + } + + let rawId = id; + if (this.spaceId && id.startsWith(`${this.spaceId}-`)) { + rawId = id.substring(this.spaceId.length + 1); + } + + if (this._idMap.has(rawId)) { + return this._idMap.get(rawId); + } + + const response = await this.httpClient.get(`/mappings/${encodeURIComponent(rawId)}/`); + const cm = this.#toConceptMap(response.data); + if (!cm) { + return null; + } + this.#indexConceptMap(cm); + return cm; + } + + // eslint-disable-next-line no-unused-vars + async searchConceptMaps(searchParams, _elements) { + this._validateSearchParams(searchParams); + + const params = Object.fromEntries(searchParams.map(({ name, value }) => [name, String(value).toLowerCase()])); + const oclParams = {}; + + if (params.source) { + oclParams.from_source_url = params.source; + } + if (params.target) { + oclParams.to_source_url = params.target; + } + + const mappings = await this.#searchMappings(oclParams, this.maxSearchPages); + const results = []; + for (const mapping of mappings) { + const cm = this.#toConceptMap(mapping); + if (!cm) { + continue; + } + this.#indexConceptMap(cm); + if (this.#matches(cm.jsonObj, params)) { + results.push(cm); + } + } + return results; + } + + async findConceptMapForTranslation(opContext, conceptMaps, sourceSystem, sourceScope, targetScope, targetSystem, sourceCode = null) { + const sourceCandidates = await this.#candidateSourceUrls(sourceSystem); + const targetCandidates = await this.#candidateSourceUrls(targetSystem); + + const mappings = []; + const sourcePaths = sourceCandidates.filter(s => String(s || '').startsWith('/orgs/')); + + if (sourceCode && sourcePaths.length > 0) { + for (const sourcePath of sourcePaths) { + const conceptPath = `${this.#normalizeSourcePath(sourcePath)}concepts/${encodeURIComponent(sourceCode)}/mappings/`; + const found = await this.#fetchAllPages(conceptPath, { limit: PAGE_SIZE }, Math.min(2, this.maxSearchPages)); + mappings.push(...found); + } + } + + const searchKeys = new Set(); + const searches = []; + + if (sourceCandidates.length === 0 && targetCandidates.length === 0) { + searches.push({}); + } else if (targetCandidates.length === 0) { + for (const src of sourceCandidates) { + const key = `from:${src}`; + if (!searchKeys.has(key)) { + searchKeys.add(key); + searches.push({ from_source_url: src }); + } + } + } else if (sourceCandidates.length === 0) { + for (const tgt of targetCandidates) { + const key = `to:${tgt}`; + if (!searchKeys.has(key)) { + searchKeys.add(key); + searches.push({ to_source_url: tgt }); + } + } + } else { + for (const src of sourceCandidates) { + for (const tgt of targetCandidates) { + const key = `from:${src}|to:${tgt}`; + if (!searchKeys.has(key)) { + searchKeys.add(key); + searches.push({ from_source_url: src, to_source_url: tgt }); + } + } + } + } + + if (mappings.length === 0) { + for (const search of searches) { + const found = await this.#searchMappings(search, Math.min(2, this.maxSearchPages)); + mappings.push(...found); + } + } + + const sourceUrlsToResolve = new Set(); + for (const mapping of mappings) { + const fromSource = mapping?.from_source_url || mapping?.fromSourceUrl; + const toSource = mapping?.to_source_url || mapping?.toSourceUrl; + if (fromSource) { + sourceUrlsToResolve.add(fromSource); + } + if (toSource) { + sourceUrlsToResolve.add(toSource); + } + } + await this.#ensureCanonicalForSourceUrls(sourceUrlsToResolve); + + const seen = new Set(conceptMaps.map(cm => cm.id || cm.url)); + for (const mapping of mappings) { + const cm = this.#toConceptMap(mapping); + if (!cm) { + continue; + } + this.#indexConceptMap(cm); + + const key = cm.id || cm.url; + if (seen.has(key)) { + continue; + } + + if (this.#matchesTranslationRequest(cm, sourceSystem, sourceScope, targetScope, targetSystem, sourceCandidates, targetCandidates)) { + conceptMaps.push(cm); + seen.add(key); + } + } + } + + cmCount() { + return new Set(this.conceptMapMap.values()).size; + } + + async close() { + } + + #indexConceptMap(cm) { + this.conceptMapMap.set(cm.url, cm); + if (cm.version) { + this.conceptMapMap.set(`${cm.url}|${cm.version}`, cm); + } + this.conceptMapMap.set(cm.id, cm); + this._idMap.set(cm.id, cm); + } + + #toConceptMap(mapping) { + if (!mapping || typeof mapping !== 'object') { + return null; + } + + const id = mapping.id; + if (!id) { + return null; + } + + const url = mapping.url || `${this.baseUrl}/mappings/${id}`; + const source = mapping.from_source_url || mapping.fromSourceUrl || mapping.from_concept_url || mapping.fromConceptUrl || null; + const target = mapping.to_source_url || mapping.toSourceUrl || mapping.to_concept_url || mapping.toConceptUrl || null; + const sourceCode = mapping.from_concept_code || mapping.fromConceptCode; + const targetCode = mapping.to_concept_code || mapping.toConceptCode; + + if (!source || !target || !sourceCode || !targetCode) { + return null; + } + + const sourceDisplay = mapping.from_concept_name_resolved || mapping.fromConceptNameResolved || mapping.from_concept_name || mapping.fromConceptName || null; + const targetDisplay = mapping.to_concept_name_resolved || mapping.toConceptNameResolved || mapping.to_concept_name || mapping.toConceptName || null; + const sourceCanonical = this.#canonicalForSourceUrl(source) || source; + const targetCanonical = this.#canonicalForSourceUrl(target) || target; + + const relationship = this.#toRelationship(mapping.map_type || mapping.mapType); + const lastUpdated = this.#toIsoDate(mapping.updated_on || mapping.updatedOn || mapping.updated_at || mapping.updatedAt); + + const json = { + resourceType: 'ConceptMap', + id, + url, + version: mapping.version || null, + name: `mapping-${id}`, + title: mapping.name || `Mapping ${id}`, + status: 'active', + sourceScopeUri: mapping.from_collection_url || mapping.fromCollectionUrl || source, + targetScopeUri: mapping.to_collection_url || mapping.toCollectionUrl || target, + group: [ + { + source: sourceCanonical, + target: targetCanonical, + element: [ + { + code: sourceCode, + display: sourceDisplay, + target: [ + { + code: targetCode, + display: targetDisplay, + relationship, + comment: mapping.comment || null + } + ] + } + ] + } + ] + }; + + if (lastUpdated) { + json.meta = { lastUpdated }; + } + + return new ConceptMap(json, 'R5'); + } + + #toRelationship(mapType) { + switch ((mapType || '').toUpperCase()) { + case 'SAME-AS': + return 'equivalent'; + case 'NARROWER-THAN': + return 'narrower-than'; + case 'BROADER-THAN': + return 'broader-than'; + case 'NOT-EQUIVALENT': + return 'not-related-to'; + default: + return 'related-to'; + } + } + + #matches(json, params) { + for (const [name, value] of Object.entries(params)) { + if (!value) { + continue; + } + + if (name === 'url') { + if ((json.url || '').toLowerCase() !== value) { + return false; + } + continue; + } + + if (name === 'source') { + const src = json.group?.[0]?.source || ''; + if (!src.toLowerCase().includes(value)) { + return false; + } + continue; + } + + if (name === 'target') { + const tgt = json.group?.[0]?.target || ''; + if (!tgt.toLowerCase().includes(value)) { + return false; + } + continue; + } + + const field = json[name]; + if (field == null || !String(field).toLowerCase().includes(value)) { + return false; + } + } + return true; + } + + async #searchMappings(params = {}, maxPages = this.maxSearchPages) { + const endpoint = this.org ? `/orgs/${encodeURIComponent(this.org)}/mappings/` : '/mappings/'; + return await this.#fetchAllPages(endpoint, params, maxPages); + } + + async #fetchAllPages(path, params = {}, maxPages = this.maxSearchPages) { + return await fetchAllPages(this.httpClient, path, { + params, + pageSize: PAGE_SIZE, + maxPages, + baseUrl: this.baseUrl + }); + } + + #extractItemsAndNext(payload) { + return extractItemsAndNext(payload, this.baseUrl); + } + + #extractMappingId(url) { + if (!url) { + return null; + } + const match = url.match(/\/mappings\/([^/]+)\/?$/i); + return match ? match[1] : null; + } + + async #candidateSourceUrls(systemUrl) { + if (!systemUrl) { + return []; + } + + const cacheKey = this.#norm(systemUrl); + if (this._sourceCandidatesCache.has(cacheKey)) { + return this._sourceCandidatesCache.get(cacheKey); + } + + const result = new Set(); + result.add(systemUrl); + + const canonicalKey = cacheKey; + const byCanonical = this._sourceUrlsByCanonical.get(canonicalKey); + if (byCanonical) { + for (const item of byCanonical) { + result.add(item); + } + } + + const discovered = await this.#resolveSourceCandidatesFromOcl(systemUrl); + for (const item of discovered) { + result.add(item); + } + + const out = Array.from(result); + this._sourceCandidatesCache.set(cacheKey, out); + return out; + } + + async #resolveSourceCandidatesFromOcl(systemUrl) { + const endpoint = this.org ? `/orgs/${encodeURIComponent(this.org)}/sources/` : '/sources/'; + const query = this.#queryTokenFromSystem(systemUrl); + if (!query) { + return []; + } + + const sources = await this.#fetchAllPages(endpoint, { q: query, limit: PAGE_SIZE }, 2); + const targetNorm = this.#norm(systemUrl); + const candidates = new Set(); + + for (const source of sources) { + const canonical = source?.canonical_url || source?.canonicalUrl || null; + const sourceUrl = source?.url || source?.uri || null; + if (!sourceUrl) { + continue; + } + + if (canonical) { + const canonicalKey = this.#norm(canonical); + if (!this._sourceUrlsByCanonical.has(canonicalKey)) { + this._sourceUrlsByCanonical.set(canonicalKey, new Set()); + } + this._sourceUrlsByCanonical.get(canonicalKey).add(sourceUrl); + this._canonicalBySourceUrl.set(this.#norm(sourceUrl), canonical); + + if (canonicalKey === targetNorm) { + candidates.add(sourceUrl); + } + } + + if (this.#norm(sourceUrl) === targetNorm) { + candidates.add(sourceUrl); + } + } + + return Array.from(candidates); + } + + async #ensureCanonicalForSourceUrls(sourceUrls) { + for (const sourceUrl of sourceUrls || []) { + const sourceKey = this.#norm(sourceUrl); + if (!sourceKey || this._canonicalBySourceUrl.has(sourceKey)) { + continue; + } + + const sourcePath = String(sourceUrl || '').trim(); + if (!sourcePath.startsWith('/orgs/')) { + continue; + } + + try { + const response = await this.httpClient.get(sourcePath); + const source = response.data || {}; + const canonical = source.canonical_url || source.canonicalUrl; + const resolvedSourceUrl = source.url || source.uri || sourcePath; + if (!canonical) { + continue; + } + + const canonicalKey = this.#norm(canonical); + if (!this._sourceUrlsByCanonical.has(canonicalKey)) { + this._sourceUrlsByCanonical.set(canonicalKey, new Set()); + } + this._sourceUrlsByCanonical.get(canonicalKey).add(resolvedSourceUrl); + this._canonicalBySourceUrl.set(this.#norm(resolvedSourceUrl), canonical); + } catch (e) { + // Ignore source lookup failures and continue resolving remaining sources. + continue; + } + } + } + + #queryTokenFromSystem(systemUrl) { + const raw = String(systemUrl || '').trim().replace(/\/+$/, ''); + if (!raw) { + return null; + } + const slash = raw.lastIndexOf('/'); + if (slash >= 0 && slash < raw.length - 1) { + return raw.substring(slash + 1); + } + return raw; + } + + #normalizeSourcePath(sourcePath) { + const path = String(sourcePath || '').trim(); + return path.endsWith('/') ? path : `${path}/`; + } + + #canonicalForSourceUrl(sourceUrl) { + return this._canonicalBySourceUrl.get(this.#norm(sourceUrl)) || null; + } + + #matchesTranslationRequest(cm, sourceSystem, sourceScope, targetScope, targetSystem, sourceCandidates, targetCandidates) { + if (cm.providesTranslation(sourceSystem, sourceScope, targetScope, targetSystem)) { + return true; + } + + const group = cm.jsonObj?.group?.[0] || {}; + const groupSource = this.#norm(group.source); + const groupTarget = this.#norm(group.target); + + const sourceOk = !sourceSystem || sourceCandidates.some(s => this.#norm(s) === groupSource); + const targetOk = !targetSystem || targetCandidates.some(s => this.#norm(s) === groupTarget); + return sourceOk && targetOk; + } + + #norm(url) { + return String(url || '').trim().replace(/\/+$/, '').toLowerCase(); + } + + #toIsoDate(value) { + if (!value) { + return null; + } + const date = new Date(value); + if (Number.isNaN(date.getTime())) { + return null; + } + return date.toISOString(); + } +} + +module.exports = { + OCLConceptMapProvider +}; \ No newline at end of file diff --git a/tx/ocl/cm-ocl.js b/tx/ocl/cm-ocl.js index 255eaabd..466c7f93 100644 --- a/tx/ocl/cm-ocl.js +++ b/tx/ocl/cm-ocl.js @@ -1,537 +1,2 @@ -const { AbstractConceptMapProvider } = require('../cm/cm-api'); -const { ConceptMap } = require('../library/conceptmap'); -const { PAGE_SIZE } = require('./shared/constants'); -const { createOclHttpClient } = require('./http/client'); -const { fetchAllPages, extractItemsAndNext } = require('./http/pagination'); +module.exports = require('./cm-ocl.cjs'); -const DEFAULT_MAX_SEARCH_PAGES = 10; - -class OCLConceptMapProvider extends AbstractConceptMapProvider { - constructor(config = {}) { - super(); - const options = typeof config === 'string' ? { baseUrl: config } : (config || {}); - - this.org = options.org || null; - this.maxSearchPages = options.maxSearchPages || DEFAULT_MAX_SEARCH_PAGES; - const http = createOclHttpClient(options); - this.baseUrl = http.baseUrl; - this.httpClient = http.client; - - this.conceptMapMap = new Map(); - this._idMap = new Map(); - this._sourceCandidatesCache = new Map(); - this._sourceUrlsByCanonical = new Map(); - this._canonicalBySourceUrl = new Map(); - } - - assignIds(ids) { - if (!this.spaceId) { - return; - } - - const unique = new Set(this.conceptMapMap.values()); - this._idMap.clear(); - - for (const cm of unique) { - if (!cm.id.startsWith(`${this.spaceId}-`)) { - const nextId = `${this.spaceId}-${cm.id}`; - cm.id = nextId; - cm.jsonObj.id = nextId; - } - this._idMap.set(cm.id, cm); - ids.add(`ConceptMap/${cm.id}`); - } - } - - async fetchConceptMap(url, version) { - this._validateFetchParams(url, version); - - const direct = this.conceptMapMap.get(`${url}|${version}`) || this.conceptMapMap.get(url); - if (direct) { - return direct; - } - - const mappingId = this.#extractMappingId(url); - if (mappingId) { - return await this.fetchConceptMapById(mappingId); - } - - const mappings = await this.#searchMappings({ from_source_url: url }, this.maxSearchPages); - for (const mapping of mappings) { - const cm = this.#toConceptMap(mapping); - if (cm) { - this.#indexConceptMap(cm); - if (cm.url === url && (!version || cm.version === version)) { - return cm; - } - } - } - - return null; - } - - async fetchConceptMapById(id) { - if (this._idMap.has(id)) { - return this._idMap.get(id); - } - - let rawId = id; - if (this.spaceId && id.startsWith(`${this.spaceId}-`)) { - rawId = id.substring(this.spaceId.length + 1); - } - - if (this._idMap.has(rawId)) { - return this._idMap.get(rawId); - } - - const response = await this.httpClient.get(`/mappings/${encodeURIComponent(rawId)}/`); - const cm = this.#toConceptMap(response.data); - if (!cm) { - return null; - } - this.#indexConceptMap(cm); - return cm; - } - - // eslint-disable-next-line no-unused-vars - async searchConceptMaps(searchParams, _elements) { - this._validateSearchParams(searchParams); - - const params = Object.fromEntries(searchParams.map(({ name, value }) => [name, String(value).toLowerCase()])); - const oclParams = {}; - - if (params.source) { - oclParams.from_source_url = params.source; - } - if (params.target) { - oclParams.to_source_url = params.target; - } - - const mappings = await this.#searchMappings(oclParams, this.maxSearchPages); - const results = []; - for (const mapping of mappings) { - const cm = this.#toConceptMap(mapping); - if (!cm) { - continue; - } - this.#indexConceptMap(cm); - if (this.#matches(cm.jsonObj, params)) { - results.push(cm); - } - } - return results; - } - - async findConceptMapForTranslation(opContext, conceptMaps, sourceSystem, sourceScope, targetScope, targetSystem, sourceCode = null) { - const sourceCandidates = await this.#candidateSourceUrls(sourceSystem); - const targetCandidates = await this.#candidateSourceUrls(targetSystem); - - const mappings = []; - const sourcePaths = sourceCandidates.filter(s => String(s || '').startsWith('/orgs/')); - - if (sourceCode && sourcePaths.length > 0) { - for (const sourcePath of sourcePaths) { - const conceptPath = `${this.#normalizeSourcePath(sourcePath)}concepts/${encodeURIComponent(sourceCode)}/mappings/`; - const found = await this.#fetchAllPages(conceptPath, { limit: PAGE_SIZE }, Math.min(2, this.maxSearchPages)); - mappings.push(...found); - } - } - - const searchKeys = new Set(); - const searches = []; - - if (sourceCandidates.length === 0 && targetCandidates.length === 0) { - searches.push({}); - } else if (targetCandidates.length === 0) { - for (const src of sourceCandidates) { - const key = `from:${src}`; - if (!searchKeys.has(key)) { - searchKeys.add(key); - searches.push({ from_source_url: src }); - } - } - } else if (sourceCandidates.length === 0) { - for (const tgt of targetCandidates) { - const key = `to:${tgt}`; - if (!searchKeys.has(key)) { - searchKeys.add(key); - searches.push({ to_source_url: tgt }); - } - } - } else { - for (const src of sourceCandidates) { - for (const tgt of targetCandidates) { - const key = `from:${src}|to:${tgt}`; - if (!searchKeys.has(key)) { - searchKeys.add(key); - searches.push({ from_source_url: src, to_source_url: tgt }); - } - } - } - } - - if (mappings.length === 0) { - for (const search of searches) { - const found = await this.#searchMappings(search, Math.min(2, this.maxSearchPages)); - mappings.push(...found); - } - } - - const sourceUrlsToResolve = new Set(); - for (const mapping of mappings) { - const fromSource = mapping?.from_source_url || mapping?.fromSourceUrl; - const toSource = mapping?.to_source_url || mapping?.toSourceUrl; - if (fromSource) { - sourceUrlsToResolve.add(fromSource); - } - if (toSource) { - sourceUrlsToResolve.add(toSource); - } - } - await this.#ensureCanonicalForSourceUrls(sourceUrlsToResolve); - - const seen = new Set(conceptMaps.map(cm => cm.id || cm.url)); - for (const mapping of mappings) { - const cm = this.#toConceptMap(mapping); - if (!cm) { - continue; - } - this.#indexConceptMap(cm); - - const key = cm.id || cm.url; - if (seen.has(key)) { - continue; - } - - if (this.#matchesTranslationRequest(cm, sourceSystem, sourceScope, targetScope, targetSystem, sourceCandidates, targetCandidates)) { - conceptMaps.push(cm); - seen.add(key); - } - } - } - - cmCount() { - return new Set(this.conceptMapMap.values()).size; - } - - async close() { - } - - #indexConceptMap(cm) { - this.conceptMapMap.set(cm.url, cm); - if (cm.version) { - this.conceptMapMap.set(`${cm.url}|${cm.version}`, cm); - } - this.conceptMapMap.set(cm.id, cm); - this._idMap.set(cm.id, cm); - } - - #toConceptMap(mapping) { - if (!mapping || typeof mapping !== 'object') { - return null; - } - - const id = mapping.id; - if (!id) { - return null; - } - - const url = mapping.url || `${this.baseUrl}/mappings/${id}`; - const source = mapping.from_source_url || mapping.fromSourceUrl || mapping.from_concept_url || mapping.fromConceptUrl || null; - const target = mapping.to_source_url || mapping.toSourceUrl || mapping.to_concept_url || mapping.toConceptUrl || null; - const sourceCode = mapping.from_concept_code || mapping.fromConceptCode; - const targetCode = mapping.to_concept_code || mapping.toConceptCode; - - if (!source || !target || !sourceCode || !targetCode) { - return null; - } - - const sourceDisplay = mapping.from_concept_name_resolved || mapping.fromConceptNameResolved || mapping.from_concept_name || mapping.fromConceptName || null; - const targetDisplay = mapping.to_concept_name_resolved || mapping.toConceptNameResolved || mapping.to_concept_name || mapping.toConceptName || null; - const sourceCanonical = this.#canonicalForSourceUrl(source) || source; - const targetCanonical = this.#canonicalForSourceUrl(target) || target; - - const relationship = this.#toRelationship(mapping.map_type || mapping.mapType); - const lastUpdated = this.#toIsoDate(mapping.updated_on || mapping.updatedOn || mapping.updated_at || mapping.updatedAt); - - const json = { - resourceType: 'ConceptMap', - id, - url, - version: mapping.version || null, - name: `mapping-${id}`, - title: mapping.name || `Mapping ${id}`, - status: 'active', - sourceScopeUri: mapping.from_collection_url || mapping.fromCollectionUrl || source, - targetScopeUri: mapping.to_collection_url || mapping.toCollectionUrl || target, - group: [ - { - source: sourceCanonical, - target: targetCanonical, - element: [ - { - code: sourceCode, - display: sourceDisplay, - target: [ - { - code: targetCode, - display: targetDisplay, - relationship, - comment: mapping.comment || null - } - ] - } - ] - } - ] - }; - - if (lastUpdated) { - json.meta = { lastUpdated }; - } - - return new ConceptMap(json, 'R5'); - } - - #toRelationship(mapType) { - switch ((mapType || '').toUpperCase()) { - case 'SAME-AS': - return 'equivalent'; - case 'NARROWER-THAN': - return 'narrower-than'; - case 'BROADER-THAN': - return 'broader-than'; - case 'NOT-EQUIVALENT': - return 'not-related-to'; - default: - return 'related-to'; - } - } - - #matches(json, params) { - for (const [name, value] of Object.entries(params)) { - if (!value) { - continue; - } - - if (name === 'url') { - if ((json.url || '').toLowerCase() !== value) { - return false; - } - continue; - } - - if (name === 'source') { - const src = json.group?.[0]?.source || ''; - if (!src.toLowerCase().includes(value)) { - return false; - } - continue; - } - - if (name === 'target') { - const tgt = json.group?.[0]?.target || ''; - if (!tgt.toLowerCase().includes(value)) { - return false; - } - continue; - } - - const field = json[name]; - if (field == null || !String(field).toLowerCase().includes(value)) { - return false; - } - } - return true; - } - - async #searchMappings(params = {}, maxPages = this.maxSearchPages) { - const endpoint = this.org ? `/orgs/${encodeURIComponent(this.org)}/mappings/` : '/mappings/'; - return await this.#fetchAllPages(endpoint, params, maxPages); - } - - async #fetchAllPages(path, params = {}, maxPages = this.maxSearchPages) { - return await fetchAllPages(this.httpClient, path, { - params, - pageSize: PAGE_SIZE, - maxPages, - baseUrl: this.baseUrl - }); - } - - #extractItemsAndNext(payload) { - return extractItemsAndNext(payload, this.baseUrl); - } - - #extractMappingId(url) { - if (!url) { - return null; - } - const match = url.match(/\/mappings\/([^/]+)\/?$/i); - return match ? match[1] : null; - } - - async #candidateSourceUrls(systemUrl) { - if (!systemUrl) { - return []; - } - - const cacheKey = this.#norm(systemUrl); - if (this._sourceCandidatesCache.has(cacheKey)) { - return this._sourceCandidatesCache.get(cacheKey); - } - - const result = new Set(); - result.add(systemUrl); - - const canonicalKey = cacheKey; - const byCanonical = this._sourceUrlsByCanonical.get(canonicalKey); - if (byCanonical) { - for (const item of byCanonical) { - result.add(item); - } - } - - const discovered = await this.#resolveSourceCandidatesFromOcl(systemUrl); - for (const item of discovered) { - result.add(item); - } - - const out = Array.from(result); - this._sourceCandidatesCache.set(cacheKey, out); - return out; - } - - async #resolveSourceCandidatesFromOcl(systemUrl) { - const endpoint = this.org ? `/orgs/${encodeURIComponent(this.org)}/sources/` : '/sources/'; - const query = this.#queryTokenFromSystem(systemUrl); - if (!query) { - return []; - } - - const sources = await this.#fetchAllPages(endpoint, { q: query, limit: PAGE_SIZE }, 2); - const targetNorm = this.#norm(systemUrl); - const candidates = new Set(); - - for (const source of sources) { - const canonical = source?.canonical_url || source?.canonicalUrl || null; - const sourceUrl = source?.url || source?.uri || null; - if (!sourceUrl) { - continue; - } - - if (canonical) { - const canonicalKey = this.#norm(canonical); - if (!this._sourceUrlsByCanonical.has(canonicalKey)) { - this._sourceUrlsByCanonical.set(canonicalKey, new Set()); - } - this._sourceUrlsByCanonical.get(canonicalKey).add(sourceUrl); - this._canonicalBySourceUrl.set(this.#norm(sourceUrl), canonical); - - if (canonicalKey === targetNorm) { - candidates.add(sourceUrl); - } - } - - if (this.#norm(sourceUrl) === targetNorm) { - candidates.add(sourceUrl); - } - } - - return Array.from(candidates); - } - - async #ensureCanonicalForSourceUrls(sourceUrls) { - for (const sourceUrl of sourceUrls || []) { - const sourceKey = this.#norm(sourceUrl); - if (!sourceKey || this._canonicalBySourceUrl.has(sourceKey)) { - continue; - } - - const sourcePath = String(sourceUrl || '').trim(); - if (!sourcePath.startsWith('/orgs/')) { - continue; - } - - try { - const response = await this.httpClient.get(sourcePath); - const source = response.data || {}; - const canonical = source.canonical_url || source.canonicalUrl; - const resolvedSourceUrl = source.url || source.uri || sourcePath; - if (!canonical) { - continue; - } - - const canonicalKey = this.#norm(canonical); - if (!this._sourceUrlsByCanonical.has(canonicalKey)) { - this._sourceUrlsByCanonical.set(canonicalKey, new Set()); - } - this._sourceUrlsByCanonical.get(canonicalKey).add(resolvedSourceUrl); - this._canonicalBySourceUrl.set(this.#norm(resolvedSourceUrl), canonical); - } catch (e) { - // Ignore source lookup failures and continue resolving remaining sources. - continue; - } - } - } - - #queryTokenFromSystem(systemUrl) { - const raw = String(systemUrl || '').trim().replace(/\/+$/, ''); - if (!raw) { - return null; - } - const slash = raw.lastIndexOf('/'); - if (slash >= 0 && slash < raw.length - 1) { - return raw.substring(slash + 1); - } - return raw; - } - - #normalizeSourcePath(sourcePath) { - const path = String(sourcePath || '').trim(); - if (!path) { - return '/'; - } - return path.endsWith('/') ? path : `${path}/`; - } - - #canonicalForSourceUrl(sourceUrl) { - if (!sourceUrl) { - return null; - } - return this._canonicalBySourceUrl.get(this.#norm(sourceUrl)) || null; - } - - #matchesTranslationRequest(cm, sourceSystem, sourceScope, targetScope, targetSystem, sourceCandidates, targetCandidates) { - if (cm.providesTranslation(sourceSystem, sourceScope, targetScope, targetSystem)) { - return true; - } - - const group = cm.jsonObj?.group?.[0] || {}; - const groupSource = this.#norm(group.source); - const groupTarget = this.#norm(group.target); - - const sourceOk = !sourceSystem || sourceCandidates.some(s => this.#norm(s) === groupSource); - const targetOk = !targetSystem || targetCandidates.some(s => this.#norm(s) === groupTarget); - return sourceOk && targetOk; - } - - #norm(url) { - return String(url || '').trim().replace(/\/+$/, '').toLowerCase(); - } - - #toIsoDate(value) { - if (!value) { - return null; - } - const date = new Date(value); - if (Number.isNaN(date.getTime())) { - return null; - } - return date.toISOString(); - } -} - -module.exports = { - OCLConceptMapProvider -}; \ No newline at end of file diff --git a/tx/ocl/cs-ocl.cjs b/tx/ocl/cs-ocl.cjs new file mode 100644 index 00000000..cdd13880 --- /dev/null +++ b/tx/ocl/cs-ocl.cjs @@ -0,0 +1,1679 @@ +const fs = require('fs/promises'); +const { AbstractCodeSystemProvider } = require('../cs/cs-provider-api'); +const { CodeSystemProvider, CodeSystemFactoryProvider, CodeSystemContentMode, FilterExecutionContext } = require('../cs/cs-api'); +const { CodeSystem } = require('../library/codesystem'); +const { SearchFilterText } = require('../library/designations'); +const { PAGE_SIZE, CONCEPT_PAGE_SIZE, COLD_CACHE_FRESHNESS_MS, OCL_CODESYSTEM_MARKER_EXTENSION } = require('./shared/constants'); +const { createOclHttpClient } = require('./http/client'); +const { fetchAllPages, extractItemsAndNext } = require('./http/pagination'); +const { CACHE_CS_DIR, CACHE_VS_DIR, getCacheFilePath } = require('./cache/cache-paths'); +const { ensureCacheDirectories, getColdCacheAgeMs, formatCacheAgeMinutes } = require('./cache/cache-utils'); +const { computeCodeSystemFingerprint } = require('./fingerprint/fingerprint'); +const { OCLBackgroundJobQueue } = require('./jobs/background-queue'); +const { OCLConceptFilterContext } = require('./model/concept-filter-context'); +const { toConceptContext } = require('./mappers/concept-mapper'); +const { patchSearchWorkerForOCLCodeFiltering } = require('./shared/patches'); + +patchSearchWorkerForOCLCodeFiltering(); + +class OCLCodeSystemProvider extends AbstractCodeSystemProvider { + constructor(config = {}) { + super(); + const options = typeof config === 'string' ? { baseUrl: config } : (config || {}); + + this.org = options.org || null; + const http = createOclHttpClient(options); + this.baseUrl = http.baseUrl; + this.httpClient = http.client; + + this._codeSystemsByCanonical = new Map(); + this._idToCodeSystem = new Map(); + this.sourceMetaByUrl = new Map(); + this._sourceStateByCanonical = new Map(); + this._usedIds = new Set(); + this._refreshPromise = null; + this._pendingChanges = null; + this._initialized = false; + this._initializePromise = null; + } + + async initialize() { + if (this._initialized) { + return; + } + + if (this._initializePromise) { + await this._initializePromise; + return; + } + + this._initializePromise = (async () => { + try { + const sources = await this.#fetchSourcesForDiscovery(); + console.log(`[OCL] Fetched ${sources.length} sources`); + + const snapshot = this.#buildSourceSnapshot(sources); + this.#applySnapshot(snapshot); + + console.log(`[OCL] Loaded ${this._codeSystemsByCanonical.size} code systems`); + this._initialized = true; + } catch (error) { + console.error(`[OCL] Initialization failed:`, error.message); + if (error.response) { + console.error(`[OCL] HTTP ${error.response.status}: ${error.response.statusText}`); + } + throw error; + } + })(); + + try { + await this._initializePromise; + } finally { + this._initializePromise = null; + } + } + + assignIds(ids) { + this._usedIds.clear(); + for (const cs of this._idToCodeSystem.values()) { + if (!cs.id || ids.has(`CodeSystem/${cs.id}`)) { + cs.id = String(ids.size); + cs.jsonObj.id = cs.id; + } + ids.add(`CodeSystem/${cs.id}`); + this._usedIds.add(cs.id); + this._idToCodeSystem.set(cs.id, cs); + } + } + + // eslint-disable-next-line no-unused-vars + async listCodeSystems(_fhirVersion, _context) { + await this.initialize(); + return Array.from(this._codeSystemsByCanonical.values()); + } + + // eslint-disable-next-line no-unused-vars + async loadCodeSystems(fhirVersion, context) { + return await this.listCodeSystems(fhirVersion, context); + } + + // Called once per minute by provider.updateCodeSystemList(). + // That caller is currently sync, so we stage async fetches and return the latest ready diff. + // eslint-disable-next-line no-unused-vars + getCodeSystemChanges(_fhirVersion, _context) { + if (!this._initialized) { + return this.#emptyChanges(); + } + + this.#scheduleRefresh(); + if (!this._pendingChanges) { + return this.#emptyChanges(); + } + + const out = this._pendingChanges; + this._pendingChanges = null; + return out; + } + + async close() { + } + + getSourceMetas() { + return Array.from(this.sourceMetaByUrl.values()); + } + + #scheduleRefresh() { + if (this._refreshPromise) { + return; + } + + this._refreshPromise = (async () => { + try { + const sources = await this.#fetchSourcesForDiscovery(); + const nextSnapshot = this.#buildSourceSnapshot(sources); + const changes = this.#diffSnapshots(this._sourceStateByCanonical, nextSnapshot); + this.#applySnapshot(nextSnapshot); + this._pendingChanges = changes; + } catch (error) { + console.error('[OCL] Incremental source refresh failed:', error.message); + this._pendingChanges = this.#emptyChanges(); + } finally { + this._refreshPromise = null; + } + })(); + } + + #emptyChanges() { + return { added: [], changed: [], deleted: [] }; + } + + #buildSourceSnapshot(sources) { + const snapshot = new Map(); + for (const source of sources || []) { + const cs = this.#toCodeSystem(source); + if (!cs) { + continue; + } + + const canonicalUrl = cs.url; + const meta = this.#buildSourceMeta(source, cs); + const checksum = this.#sourceChecksum(source); + snapshot.set(canonicalUrl, { cs, meta, checksum }); + } + return snapshot; + } + + async #fetchSourcesForDiscovery() { + const organizations = await this.#fetchOrganizationIds(); + if (organizations.length === 0) { + // Fallback for OCL instances that expose global listing but not org listing. + return await this.#fetchAllPages('/sources/'); + } + + const allSources = []; + const seen = new Set(); + + for (const orgId of organizations) { + const endpoint = `/orgs/${encodeURIComponent(orgId)}/sources/`; + const sources = await this.#fetchAllPages(endpoint); + for (const source of sources) { + if (!source || typeof source !== 'object') { + continue; + } + const key = this.#sourceIdentity(source); + if (seen.has(key)) { + continue; + } + seen.add(key); + allSources.push(source); + } + } + + return allSources; + } + + async #fetchOrganizationIds() { + const endpoint = '/orgs/'; + const orgs = await this.#fetchAllPages(endpoint); + + const ids = []; + const seen = new Set(); + for (const org of orgs || []) { + if (!org || typeof org !== 'object') { + continue; + } + + const id = org.id || org.mnemonic || org.short_code || org.shortCode || org.name || null; + if (!id) { + continue; + } + + const normalized = String(id).trim(); + if (!normalized || seen.has(normalized)) { + continue; + } + + seen.add(normalized); + ids.push(normalized); + } + + if (ids.length === 0 && this.org) { + ids.push(this.org); + } + + return ids; + } + + #sourceIdentity(source) { + if (!source || typeof source !== 'object') { + return '__invalid__'; + } + + const owner = source.owner || ''; + const canonical = source.canonical_url || source.canonicalUrl || ''; + const shortCode = source.short_code || source.shortCode || source.id || source.mnemonic || source.name || ''; + return `${owner}|${canonical}|${shortCode}`; + } + + #applySnapshot(snapshot) { + const previousSnapshot = this._sourceStateByCanonical; + this._codeSystemsByCanonical.clear(); + this._idToCodeSystem.clear(); + this.sourceMetaByUrl.clear(); + this._usedIds.clear(); + + for (const [canonicalUrl, entry] of snapshot.entries()) { + const cs = entry.cs; + const meta = entry.meta; + const previousEntry = previousSnapshot.get(canonicalUrl); + + // Preserve complete-content marker if checksum did not change. + if (previousEntry && previousEntry.checksum === entry.checksum && previousEntry.cs?.jsonObj?.content === CodeSystemContentMode.Complete) { + cs.jsonObj.content = CodeSystemContentMode.Complete; + + // Preserve materialized concepts across metadata refreshes. + if (Array.isArray(previousEntry.cs?.jsonObj?.concept)) { + cs.jsonObj.concept = previousEntry.cs.jsonObj.concept.map(concept => ({ ...concept })); + } + } + + // If a factory already has warm/cold concepts for this system, project them to the new snapshot resource. + OCLSourceCodeSystemFactory.syncCodeSystemResource(canonicalUrl, cs.version || null, cs); + + this.#trackCodeSystemId(cs); + this._codeSystemsByCanonical.set(canonicalUrl, cs); + this._idToCodeSystem.set(cs.id, cs); + this.sourceMetaByUrl.set(canonicalUrl, meta); + } + + this._sourceStateByCanonical = snapshot; + } + + #diffSnapshots(previousSnapshot, nextSnapshot) { + const added = []; + const changed = []; + const deleted = []; + + for (const [canonicalUrl, nextEntry] of nextSnapshot.entries()) { + const previousEntry = previousSnapshot.get(canonicalUrl); + if (!previousEntry) { + added.push(nextEntry.cs); + continue; + } + + // Keep stable ids across revisions so clients don't observe resource id churn. + nextEntry.cs.id = previousEntry.cs.id; + nextEntry.cs.jsonObj.id = previousEntry.cs.id; + + const previousChecksum = previousEntry.checksum || null; + const nextChecksum = nextEntry.checksum || null; + const checksumChanged = previousChecksum !== nextChecksum; + const versionChanged = (previousEntry.cs.version || null) !== (nextEntry.cs.version || null); + if (checksumChanged || versionChanged) { + if (checksumChanged) { + console.log(`[OCL] CodeSystem checksum changed: ${canonicalUrl} (${previousChecksum} -> ${nextChecksum})`); + } + changed.push(nextEntry.cs); + } + } + + for (const [canonicalUrl, previousEntry] of previousSnapshot.entries()) { + if (!nextSnapshot.has(canonicalUrl)) { + deleted.push(previousEntry.cs); + } + } + + return { added, changed, deleted }; + } + + #trackCodeSystemId(cs) { + if (!cs) { + return; + } + + if (!cs.id || this._usedIds.has(cs.id)) { + const raw = cs.id || cs.name || cs.url || 'ocl-cs'; + const base = this.spaceId ? `${this.spaceId}-${raw}` : String(raw); + let candidate = base; + let index = 1; + while (this._usedIds.has(candidate)) { + candidate = `${base}-${index}`; + index += 1; + } + cs.id = candidate; + cs.jsonObj.id = candidate; + } + + this._usedIds.add(cs.id); + } + + #toCodeSystem(source) { + if (!source || typeof source !== 'object') { + return null; + } + + const canonicalUrl = source.canonical_url || source.canonicalUrl || source.url; + if (!canonicalUrl) { + return null; + } + + const id = source.id || source.mnemonic; + if (!id) { + return null; + } + + const lastUpdated = this.#toIsoDate(source.updated_at || source.updatedAt || source.updated_on || source.updatedOn); + + const json = { + resourceType: 'CodeSystem', + id, + url: canonicalUrl, + version: source.version || null, + name: source.name || source.mnemonic || id, + title: source.full_name || source.fullName || source.name || source.mnemonic || id, + status: 'active', + experimental: source.experimental === true, + description: source.description || null, + publisher: source.owner || null, + caseSensitive: source.case_sensitive != null ? source.case_sensitive : (source.caseSensitive != null ? source.caseSensitive : true), + language: source.default_locale || source.defaultLocale || null, + filter: [ + { + code: 'code', + description: 'Match concept code', + operator: ['=', 'in', 'regex'], + value: 'code' + }, + { + code: 'display', + description: 'Match concept display text', + operator: ['=', 'in', 'regex'], + value: 'string' + }, + { + code: 'definition', + description: 'Match concept definition text', + operator: ['=', 'in', 'regex'], + value: 'string' + }, + { + code: 'inactive', + description: 'Match inactive (retired) status', + operator: ['=', 'in'], + value: 'boolean' + } + ], + property: [ + { + code: 'code', + uri: 'http://hl7.org/fhir/concept-properties#code', + description: 'Concept code', + type: 'code' + }, + { + code: 'display', + description: 'Concept display text', + type: 'string' + }, + { + code: 'definition', + description: 'Concept definition text', + type: 'string' + }, + { + code: 'inactive', + uri: 'http://hl7.org/fhir/concept-properties#status', + description: 'Whether concept is inactive (retired)', + type: 'boolean' + } + ], + extension: [ + { + url: OCL_CODESYSTEM_MARKER_EXTENSION, + valueBoolean: true + } + ], + content: 'not-present' + }; + + if (lastUpdated) { + json.meta = { lastUpdated }; + } + + return new CodeSystem(json, 'R5', true); + } + + #buildSourceMeta(source, cs) { + if (!source || !cs) { + return null; + } + + const owner = source.owner || null; + const shortCode = source.short_code || source.shortCode || source.mnemonic || source.id || null; + const canonicalUrl = cs.url; + if (!canonicalUrl) { + return; + } + + const conceptsUrl = this.#normalizePath(source.concepts_url || source.conceptsUrl || this.#buildConceptsPath(source)); + const meta = { + id: source.id || shortCode, + shortCode, + owner, + name: source.name || shortCode || cs.id, + description: source.description || null, + canonicalUrl, + version: source.version || null, + conceptsUrl, + checksum: this.#sourceChecksum(source), + codeSystem: cs + }; + + return meta; + } + + #sourceChecksum(source) { + // NOTE: OCL checksums are NOT reliable for cache invalidation decisions. + // They do not update when concepts are added or modified. + // This checksum is logged for debugging purposes only. + // Cache decisions are based on custom fingerprints computed from concept content. + + if (!source || typeof source !== 'object') { + return null; + } + + const checksums = source.checksums || {}; + const standard = checksums.standard || null; + const smart = checksums.smart || null; + if (standard) { + return String(standard); + } + if (smart) { + return String(smart); + } + + if (source.checksum) { + return String(source.checksum); + } + + const updated = source.updated_at || source.updatedAt || source.updated_on || source.updatedOn || null; + const version = source.version || null; + if (updated || version) { + return `${updated || ''}|${version || ''}`; + } + + return null; + } + + #buildConceptsPath(source) { + if (!source || typeof source !== 'object') { + return null; + } + const owner = source.owner || null; + const sourceId = source.short_code || source.shortCode || source.id || source.mnemonic || null; + if (!owner || !sourceId) { + const sourceUrl = source.url; + if (!sourceUrl || typeof sourceUrl !== 'string') { + return null; + } + const trimmed = sourceUrl.endsWith('/') ? sourceUrl : `${sourceUrl}/`; + return `${trimmed}concepts/`; + } + return `/orgs/${encodeURIComponent(owner)}/sources/${encodeURIComponent(sourceId)}/concepts/`; + } + + #normalizePath(pathValue) { + if (!pathValue) { + return null; + } + if (typeof pathValue !== 'string') { + return null; + } + if (pathValue.startsWith('http://') || pathValue.startsWith('https://')) { + return pathValue; + } + return `${this.baseUrl}${pathValue.startsWith('/') ? '' : '/'}${pathValue}`; + } + + async #fetchAllPages(path) { + try { + return await fetchAllPages(this.httpClient, path, { + pageSize: PAGE_SIZE, + baseUrl: this.baseUrl, + logger: console, + loggerPrefix: '[OCL]' + }); + } catch (error) { + if (error.response) { + console.error(`[OCL] HTTP ${error.response.status}: ${error.response.statusText}`); + console.error('[OCL] Response:', error.response.data); + } + throw error; + } + } + + #extractItemsAndNext(payload) { + return extractItemsAndNext(payload, this.baseUrl); + } + + #toIsoDate(value) { + if (!value) { + return null; + } + const date = new Date(value); + if (Number.isNaN(date.getTime())) { + return null; + } + return date.toISOString(); + } +} + +class OCLSourceCodeSystemProvider extends CodeSystemProvider { + constructor(opContext, supplements, client, meta, sharedCaches = null) { + super(opContext, supplements); + this.httpClient = client; + this.meta = meta; + this.conceptCache = sharedCaches?.conceptCache || new Map(); + this.pageCache = sharedCaches?.pageCache || new Map(); + this.pendingConceptRequests = sharedCaches?.pendingConceptRequests || new Map(); + this.pendingPageRequests = sharedCaches?.pendingPageRequests || new Map(); + this.scheduleBackgroundLoad = typeof sharedCaches?.scheduleBackgroundLoad === 'function' + ? sharedCaches.scheduleBackgroundLoad + : null; + this.isSystemComplete = typeof sharedCaches?.isSystemComplete === 'function' + ? sharedCaches.isSystemComplete + : (() => false); + this.getTotalConceptCount = typeof sharedCaches?.getTotalConceptCount === 'function' + ? sharedCaches.getTotalConceptCount + : (() => -1); + } + + system() { + return this.meta.canonicalUrl; + } + + version() { + return this.meta.version || null; + } + + description() { + return this.meta.description || null; + } + + name() { + return this.meta.name || this.meta.shortCode || this.meta.id || this.system(); + } + + contentMode() { + return this.isSystemComplete() ? CodeSystemContentMode.Complete : CodeSystemContentMode.NotPresent; + } + + totalCount() { + return this.getTotalConceptCount(); + } + + propertyDefinitions() { + return this.meta?.codeSystem?.jsonObj?.property || null; + } + + async code(code) { + const ctxt = await this.#ensureContext(code); + return ctxt ? ctxt.code : null; + } + + async display(code) { + const ctxt = await this.#ensureContext(code); + if (!ctxt) { + return null; + } + if (ctxt.display && this.opContext.langs.isEnglishOrNothing()) { + return ctxt.display; + } + const supp = this._displayFromSupplements(ctxt.code); + return supp || ctxt.display || null; + } + + async definition(code) { + const ctxt = await this.#ensureContext(code); + return ctxt ? ctxt.definition : null; + } + + async isAbstract(code) { + await this.#ensureContext(code); + return false; + } + + async isInactive(code) { + const ctxt = await this.#ensureContext(code); + return ctxt ? ctxt.retired === true : false; + } + + async isDeprecated(code) { + await this.#ensureContext(code); + return false; + } + + async getStatus(code) { + const ctxt = await this.#ensureContext(code); + if (!ctxt) { + return null; + } + return ctxt.retired === true ? 'inactive' : 'active'; + } + + async designations(code, displays) { + const ctxt = await this.#ensureContext(code); + if (ctxt && ctxt.display) { + const hasConceptDesignations = Array.isArray(ctxt.designations) && ctxt.designations.length > 0; + if (hasConceptDesignations) { + for (const d of ctxt.designations) { + if (!d || !d.value) { + continue; + } + displays.addDesignation(true, 'active', d.language || '', CodeSystem.makeUseForDisplay(), d.value); + } + } else { + displays.addDesignation(true, 'active', 'en', CodeSystem.makeUseForDisplay(), ctxt.display); + } + this._listSupplementDesignations(ctxt.code, displays); + } + } + + async locate(code) { + if (!code || typeof code !== 'string') { + return { context: null, message: 'Empty code' }; + } + + if (this.conceptCache.has(code)) { + return { context: this.conceptCache.get(code), message: null }; + } + + if (this.scheduleBackgroundLoad) { + this.scheduleBackgroundLoad('lookup-miss'); + } + + const concept = await this.#fetchConcept(code); + if (!concept) { + return { context: null, message: undefined }; + } + + this.conceptCache.set(code, concept); + return { context: concept, message: null }; + } + + async iterator(code) { + await this.#ensureContext(code); + if (code) { + return null; + } + return { + page: 1, + index: 0, + items: [], + total: -1, + done: false + }; + } + + async iteratorAll() { + return this.iterator(null); + } + + async getPrepContext(iterate) { + return new FilterExecutionContext(iterate); + } + + async doesFilter(prop, op, value) { + if (!prop || !op || value == null) { + return false; + } + + const normalizedProp = String(prop).trim().toLowerCase(); + const normalizedOp = String(op).trim().toLowerCase(); + const supportedOps = ['=', 'in', 'regex']; + if (!supportedOps.includes(normalizedOp)) { + return false; + } + + if (['concept', 'code', 'display', 'definition', 'inactive'].includes(normalizedProp)) { + return true; + } + + const defs = this.propertyDefinitions() || []; + return defs.some(def => def && def.code === normalizedProp); + } + + async searchFilter(filterContext, filter, sort) { + const matcher = this.#toSearchFilterText(filter); + const results = new OCLConceptFilterContext(); + const concepts = await this.#allConceptContexts(); + + for (const concept of concepts) { + const text = this.#conceptSearchText(concept); + const match = matcher.passes(text, true); + if (!match || match.passes !== true) { + continue; + } + + results.add(concept, this.#searchRating(concept, matcher, match.rating)); + } + + if (sort === true) { + results.sort(); + } + + if (!Array.isArray(filterContext.filters)) { + filterContext.filters = []; + } + filterContext.filters.push(results); + return filterContext; + } + + async filter(filterContext, prop, op, value) { + const normalizedProp = String(prop || '').trim().toLowerCase(); + const normalizedOp = String(op || '').trim().toLowerCase(); + + if (!await this.doesFilter(normalizedProp, normalizedOp, value)) { + throw new Error(`Filter ${prop} ${op} is not supported by OCL provider`); + } + + const set = new OCLConceptFilterContext(); + const concepts = await this.#allConceptContexts(); + const matcher = this.#buildPropertyMatcher(normalizedProp, normalizedOp, value); + + for (const concept of concepts) { + if (matcher(concept)) { + set.add(concept, 0); + } + } + + if (!Array.isArray(filterContext.filters)) { + filterContext.filters = []; + } + filterContext.filters.push(set); + return set; + } + + async executeFilters(filterContext) { + return Array.isArray(filterContext?.filters) ? filterContext.filters : []; + } + + // eslint-disable-next-line no-unused-vars + async filterSize(filterContext, set) { + return set ? set.size() : 0; + } + + // eslint-disable-next-line no-unused-vars + async filterMore(filterContext, set) { + return !!set && set.hasMore(); + } + + // eslint-disable-next-line no-unused-vars + async filterConcept(filterContext, set) { + if (!set) { + return null; + } + return set.next(); + } + + // eslint-disable-next-line no-unused-vars + async filterLocate(filterContext, set, code) { + if (!set) { + return `Code '${code}' not found: no filter results`; + } + const concept = set.findConceptByCode(code); + if (concept) { + return concept; + } + return null; + } + + // eslint-disable-next-line no-unused-vars + async filterCheck(filterContext, set, concept) { + if (!set || !concept) { + return false; + } + return set.containsConcept(concept); + } + + async filterFinish(filterContext) { + if (!Array.isArray(filterContext?.filters)) { + return; + } + for (const set of filterContext.filters) { + if (set && typeof set.reset === 'function') { + set.reset(); + } + } + filterContext.filters.length = 0; + } + + async nextContext(iteratorContext) { + if (!iteratorContext || iteratorContext.done) { + return null; + } + + if (iteratorContext.index >= iteratorContext.items.length) { + const pageItems = await this.#fetchConceptPage(iteratorContext.page); + iteratorContext.page += 1; + iteratorContext.index = 0; + iteratorContext.items = pageItems; + + if (!pageItems || pageItems.length === 0) { + iteratorContext.done = true; + return null; + } + } + + const concept = iteratorContext.items[iteratorContext.index]; + iteratorContext.index += 1; + return concept; + } + + async #ensureContext(code) { + if (!code) { + return null; + } + + // Some call paths pass a pending locate() Promise (or its wrapper result) + // instead of a raw code/context; normalize both shapes here. + if (code && typeof code === 'object' && typeof code.then === 'function') { + code = await code; + } + + if (code && typeof code === 'object' && Object.prototype.hasOwnProperty.call(code, 'context')) { + if (!code.context) { + throw new Error(code.message || 'Unknown code'); + } + code = code.context; + } + + if (typeof code === 'string') { + const result = await this.locate(code); + if (!result.context) { + throw new Error(result.message || `Unknown code ${code}`); + } + return result.context; + } + if (code && typeof code === 'object' && code.code) { + return code; + } + throw new Error(`Unknown Type at #ensureContext: ${typeof code}`); + } + + async #fetchConceptPage(page) { + if (!this.meta.conceptsUrl) { + return []; + } + const cacheKey = `${this.meta.conceptsUrl}|p=${page}|l=${CONCEPT_PAGE_SIZE}`; + if (this.pageCache.has(cacheKey)) { + const cached = this.pageCache.get(cacheKey); + return Array.isArray(cached) + ? cached + : Array.isArray(cached?.concepts) + ? cached.concepts + : []; + } + if (this.pendingPageRequests.has(cacheKey)) { + const pendingResult = await this.pendingPageRequests.get(cacheKey); + return Array.isArray(pendingResult) + ? pendingResult + : Array.isArray(pendingResult?.concepts) + ? pendingResult.concepts + : []; + } + + if (this.scheduleBackgroundLoad) { + this.scheduleBackgroundLoad('page-miss'); + } + + const pending = (async () => { + let response; + try { + response = await this.httpClient.get(this.meta.conceptsUrl, { params: { page, limit: CONCEPT_PAGE_SIZE } }); + } catch (error) { + // Some OCL instances return 404 for sources without concept listing endpoints. + // Treat this as an empty page so terminology operations degrade gracefully. + if (error && error.response && error.response.status === 404) { + this.pageCache.set(cacheKey, []); + return []; + } + throw error; + } + const payload = response.data; + const items = Array.isArray(payload) + ? payload + : Array.isArray(payload?.results) + ? payload.results + : Array.isArray(payload?.items) + ? payload.items + : Array.isArray(payload?.data) + ? payload.data + : []; + + const mapped = items.map(item => this.#toConceptContext(item)).filter(Boolean); + this.pageCache.set(cacheKey, mapped); + for (const concept of mapped) { + if (concept && concept.code) { + this.conceptCache.set(concept.code, concept); + } + } + return mapped; + })(); + + this.pendingPageRequests.set(cacheKey, pending); + try { + return await pending; + } finally { + this.pendingPageRequests.delete(cacheKey); + } + } + + async #fetchConcept(code) { + if (!this.meta.conceptsUrl) { + return null; + } + if (this.conceptCache.has(code)) { + return this.conceptCache.get(code); + } + const pendingKey = `${this.meta.conceptsUrl}|c=${code}`; + if (this.pendingConceptRequests.has(pendingKey)) { + return this.pendingConceptRequests.get(pendingKey); + } + + if (this.scheduleBackgroundLoad) { + this.scheduleBackgroundLoad('concept-miss'); + } + + const url = this.#buildConceptUrl(code); + const pending = (async () => { + let response; + try { + response = await this.httpClient.get(url); + } catch (error) { + // Missing concept should be treated as not-found, not as an internal server failure. + if (error && error.response && error.response.status === 404) { + return null; + } + throw error; + } + const concept = this.#toConceptContext(response.data); + if (concept && concept.code) { + this.conceptCache.set(concept.code, concept); + } + return concept; + })(); + + this.pendingConceptRequests.set(pendingKey, pending); + try { + return await pending; + } finally { + this.pendingConceptRequests.delete(pendingKey); + } + } + + async #allConceptContexts() { + const concepts = new Map(); + + for (const concept of this.conceptCache.values()) { + if (concept && concept.code) { + concepts.set(concept.code, concept); + } + } + + // Ensure search can operate even when content is not fully warm-loaded. + const iter = await this.iterator(null); + let concept = await this.nextContext(iter); + while (concept) { + if (concept.code && !concepts.has(concept.code)) { + concepts.set(concept.code, concept); + } + concept = await this.nextContext(iter); + } + + return Array.from(concepts.values()); + } + + #toSearchFilterText(filter) { + if (filter instanceof SearchFilterText) { + return filter; + } + if (typeof filter === 'string') { + return new SearchFilterText(filter); + } + if (filter && typeof filter.filter === 'string') { + return new SearchFilterText(filter.filter); + } + return new SearchFilterText(''); + } + + #conceptSearchText(concept) { + if (!concept || typeof concept !== 'object') { + return ''; + } + + const values = [concept.code, concept.display, concept.definition]; + if (Array.isArray(concept.designations)) { + for (const designation of concept.designations) { + if (designation && designation.value) { + values.push(designation.value); + } + } + } + + return values.filter(Boolean).join(' '); + } + + #searchRating(concept, matcher, baseRating) { + let rating = Number.isFinite(baseRating) ? baseRating : 0; + const term = matcher?.filter || ''; + if (!term) { + return rating; + } + + const code = String(concept?.code || '').toLowerCase(); + const display = String(concept?.display || '').toLowerCase(); + const definition = String(concept?.definition || '').toLowerCase(); + + if (code === term || display === term) { + rating += 100; + } else if (code.startsWith(term) || display.startsWith(term)) { + rating += 50; + } else if (definition.includes(term)) { + rating += 10; + } + + return rating; + } + + #buildPropertyMatcher(prop, op, value) { + if (op === 'regex') { + const regex = new RegExp(String(value), 'i'); + return concept => { + const candidate = this.#valueForFilter(concept, prop); + if (candidate == null) { + return false; + } + return regex.test(String(candidate)); + }; + } + + if (op === 'in') { + const tokens = String(value) + .split(',') + .map(token => token.trim().toLowerCase()) + .filter(Boolean); + return concept => { + const candidate = this.#valueForFilter(concept, prop); + if (candidate == null) { + return false; + } + return tokens.includes(String(candidate).toLowerCase()); + }; + } + + if (prop === 'inactive') { + const expected = this.#toBoolean(value); + return concept => { + const candidate = this.#toBoolean(this.#valueForFilter(concept, prop)); + return candidate === expected; + }; + } + + const expected = String(value).toLowerCase(); + return concept => { + const candidate = this.#valueForFilter(concept, prop); + if (candidate == null) { + return false; + } + return String(candidate).toLowerCase() === expected; + }; + } + + #valueForFilter(concept, prop) { + if (!concept || typeof concept !== 'object') { + return null; + } + + switch (prop) { + case 'concept': + case 'code': + return concept.code || null; + case 'display': + return concept.display || null; + case 'definition': + return concept.definition || null; + case 'inactive': + return concept.retired === true; + default: + return concept[prop] ?? null; + } + } + + #toBoolean(value) { + if (typeof value === 'boolean') { + return value; + } + + const text = String(value || '').trim().toLowerCase(); + return text === 'true' || text === '1' || text === 'yes'; + } + + #buildConceptUrl(code) { + const base = this.meta.conceptsUrl.endsWith('/') ? this.meta.conceptsUrl : `${this.meta.conceptsUrl}/`; + return `${base}${encodeURIComponent(code)}/`; + } + + #toConceptContext(concept) { + return toConceptContext(concept); + } +} + +class OCLSourceCodeSystemFactory extends CodeSystemFactoryProvider { + static factoriesByKey = new Map(); + + static syncCodeSystemResource(system, version = null, codeSystem = null) { + if (!system) { + return; + } + + const key = `${system}|${version || ''}`; + const factory = OCLSourceCodeSystemFactory.factoriesByKey.get(key); + if (!factory) { + return; + } + + factory.#applyConceptsToCodeSystemResource(codeSystem || factory.meta?.codeSystem || null); + } + + constructor(i18n, client, meta) { + super(i18n); + this.httpClient = client; + this.meta = meta; + this.sharedConceptCache = new Map(); + this.sharedPageCache = new Map(); + this.sharedPendingConceptRequests = new Map(); + this.sharedPendingPageRequests = new Map(); + this.isComplete = meta?.codeSystem?.jsonObj?.content === CodeSystemContentMode.Complete; + this.loadedConceptCount = -1; + this.loadedChecksum = meta?.checksum || null; + this.customFingerprint = null; + this.backgroundLoadProgress = { processed: 0, total: null }; + this.materializedConceptList = null; + this.materializedConceptCount = -1; + OCLSourceCodeSystemFactory.factoriesByKey.set(this.#resourceKey(), this); + + // Load cold cache at construction + this.#loadColdCache(); + } + + async #loadColdCache() { + const canonicalUrl = this.system(); + const version = this.version(); + const cacheFilePath = getCacheFilePath(CACHE_CS_DIR, canonicalUrl, version); + + try { + const data = await fs.readFile(cacheFilePath, 'utf-8'); + const cached = JSON.parse(data); + + if (!cached || !cached.concepts || !Array.isArray(cached.concepts)) { + return; + } + + // Restore concepts to cache + for (const concept of cached.concepts) { + if (concept && concept.code) { + this.sharedConceptCache.set(concept.code, concept); + } + } + + this.loadedConceptCount = cached.concepts.length; + this.customFingerprint = cached.fingerprint || null; + this.isComplete = true; + + if (this.meta?.codeSystem?.jsonObj) { + this.meta.codeSystem.jsonObj.content = CodeSystemContentMode.Complete; + } + + this.#applyConceptsToCodeSystemResource(this.meta?.codeSystem || null); + + console.log(`[OCL] Loaded CodeSystem from cold cache: ${canonicalUrl} (${cached.concepts.length} concepts, fingerprint=${this.customFingerprint?.substring(0, 8)})`); + } catch (error) { + if (error.code !== 'ENOENT') { + console.error(`[OCL] Failed to load cold cache for CodeSystem ${canonicalUrl}:`, error.message); + } + } + } + + async #saveColdCache(concepts) { + const canonicalUrl = this.system(); + const version = this.version(); + const cacheFilePath = getCacheFilePath(CACHE_CS_DIR, canonicalUrl, version); + + try { + await ensureCacheDirectories(CACHE_CS_DIR, CACHE_VS_DIR); + + const fingerprint = computeCodeSystemFingerprint(concepts); + const cacheData = { + canonicalUrl, + version, + fingerprint, + timestamp: new Date().toISOString(), + conceptCount: concepts.length, + concepts + }; + + await fs.writeFile(cacheFilePath, JSON.stringify(cacheData, null, 2), 'utf-8'); + console.log(`[OCL] Saved CodeSystem to cold cache: ${canonicalUrl} (${concepts.length} concepts, fingerprint=${fingerprint?.substring(0, 8)})`); + + return fingerprint; + } catch (error) { + console.error(`[OCL] Failed to save cold cache for CodeSystem ${canonicalUrl}:`, error.message); + return null; + } + } + + static scheduleBackgroundLoadByKey(system, version = null, reason = 'valueset-expansion') { + const key = `${system}|${version || ''}`; + const factory = OCLSourceCodeSystemFactory.factoriesByKey.get(key); + if (!factory) { + console.log(`[OCL] CodeSystem load not scheduled (factory unavailable): ${key}`); + return false; + } + factory.scheduleBackgroundLoad(reason); + return true; + } + + static checksumForResource(system, version = null) { + const key = `${system}|${version || ''}`; + const factory = OCLSourceCodeSystemFactory.factoriesByKey.get(key); + if (!factory) { + return null; + } + return factory.currentChecksum(); + } + + static loadProgress() { + let total = 0; + let loaded = 0; + + for (const factory of OCLSourceCodeSystemFactory.factoriesByKey.values()) { + total += 1; + if (factory && factory.isCompleteNow()) { + loaded += 1; + } + } + + const percentage = total > 0 ? (loaded / total) * 100 : 0; + return { loaded, total, percentage }; + } + + defaultVersion() { + return this.meta.version || null; + } + + build(opContext, supplements) { + this.#syncWarmStateWithChecksum(); + this.#applyConceptsToCodeSystemResource(this.meta?.codeSystem || null); + this.recordUse(); + return new OCLSourceCodeSystemProvider(opContext, supplements, this.httpClient, this.meta, { + conceptCache: this.sharedConceptCache, + pageCache: this.sharedPageCache, + pendingConceptRequests: this.sharedPendingConceptRequests, + pendingPageRequests: this.sharedPendingPageRequests, + scheduleBackgroundLoad: reason => this.scheduleBackgroundLoad(reason), + isSystemComplete: () => this.isComplete, + getTotalConceptCount: () => this.loadedConceptCount + }); + } + + scheduleBackgroundLoad(reason = 'request') { + this.#syncWarmStateWithChecksum(); + if (this.isComplete) { + return; + } + + const cacheFilePath = getCacheFilePath(CACHE_CS_DIR, this.system(), this.version()); + const cacheAgeMs = getColdCacheAgeMs(cacheFilePath); + if (cacheAgeMs != null && cacheAgeMs < COLD_CACHE_FRESHNESS_MS) { + console.log(`[OCL] Skipping warm-up for CodeSystem ${this.system()} (cold cache age: ${formatCacheAgeMinutes(cacheAgeMs)})`); + return; + } + + const key = this.#resourceKey(); + const jobKey = `cs:${key}`; + + if (OCLBackgroundJobQueue.isQueuedOrRunning(jobKey)) { + console.log(`[OCL] CodeSystem load already queued or running: ${key}`); + return; + } + + let queuedJobSize = null; + console.log(`[OCL] CodeSystem load enqueued: ${key} (${reason})`); + OCLBackgroundJobQueue.enqueue( + jobKey, + 'CodeSystem load', + async () => { + await this.#runBackgroundLoad(key, queuedJobSize); + }, + { + jobId: this.system(), + getProgress: () => this.#backgroundLoadProgressSnapshot(), + resolveJobSize: async () => { + queuedJobSize = await this.#fetchConceptCountFromHeaders(); + return queuedJobSize; + } + } + ); + } + + async #runBackgroundLoad(key, knownConceptCount = null) { + console.log(`[OCL] CodeSystem load started: ${key}`); + try { + this.backgroundLoadProgress = { processed: 0, total: null }; + const resolvedTotal = Number.isFinite(knownConceptCount) && knownConceptCount >= 0 + ? knownConceptCount + : await this.#fetchConceptCountFromHeaders(); + this.backgroundLoadProgress.total = resolvedTotal; + const count = await this.#loadAllConceptPages(); + this.loadedConceptCount = count; + this.isComplete = true; + this.loadedChecksum = this.meta?.checksum || null; + this.backgroundLoadProgress = { + processed: count, + total: count > 0 ? count : this.backgroundLoadProgress.total + }; + + if (this.meta?.codeSystem?.jsonObj) { + this.meta.codeSystem.jsonObj.content = CodeSystemContentMode.Complete; + } + + this.#applyConceptsToCodeSystemResource(this.meta?.codeSystem || null); + + // Compute custom fingerprint and compare with cold cache + const allConcepts = Array.from(this.sharedConceptCache.values()); + const newFingerprint = computeCodeSystemFingerprint(allConcepts); + + if (this.customFingerprint && newFingerprint === this.customFingerprint) { + console.log(`[OCL] CodeSystem fingerprint unchanged: ${key} (fingerprint=${newFingerprint?.substring(0, 8)})`); + } else { + if (this.customFingerprint) { + console.log(`[OCL] CodeSystem fingerprint changed: ${key} (${this.customFingerprint?.substring(0, 8)} -> ${newFingerprint?.substring(0, 8)})`); + console.log(`[OCL] Replacing cold cache with new hot cache: ${key}`); + } else { + console.log(`[OCL] Computed fingerprint for CodeSystem: ${key} (fingerprint=${newFingerprint?.substring(0, 8)})`); + } + + // Save to cold cache + const savedFingerprint = await this.#saveColdCache(allConcepts); + if (savedFingerprint) { + this.customFingerprint = savedFingerprint; + } + } + + console.log(`[OCL] CodeSystem load completed, marked content=complete: ${key}`); + const progress = OCLSourceCodeSystemFactory.loadProgress(); + console.log(`[OCL] CodeSystem load completed: ${this.system()}. Loaded ${progress.loaded}/${progress.total} CodeSystems (${progress.percentage.toFixed(2)}%)`); + console.log(`[OCL] CodeSystem now available in cache: ${key} (${count} concepts)`); + } catch (error) { + console.error(`[OCL] CodeSystem background load failed: ${key}: ${error.message}`); + } + } + + async #loadAllConceptPages() { + if (!this.meta?.conceptsUrl) { + this.loadedConceptCount = 0; + this.backgroundLoadProgress = { processed: 0, total: 0 }; + return 0; + } + + let page = 1; + let total = 0; + + // eslint-disable-next-line no-constant-condition + while (true) { + const pageData = await this.#fetchAndCacheConceptPage(page); + const concepts = Array.isArray(pageData?.concepts) ? pageData.concepts : []; + if (concepts.length === 0) { + break; + } + total += concepts.length; + this.backgroundLoadProgress.processed = total; + if (concepts.length < CONCEPT_PAGE_SIZE) { + break; + } + page += 1; + } + + return total; + } + + async #fetchAndCacheConceptPage(page) { + const cacheKey = `${this.meta.conceptsUrl}|p=${page}|l=${CONCEPT_PAGE_SIZE}`; + if (this.sharedPageCache.has(cacheKey)) { + const cached = this.sharedPageCache.get(cacheKey); + const concepts = Array.isArray(cached) + ? cached + : Array.isArray(cached?.concepts) + ? cached.concepts + : []; + const reportedTotal = this.#extractTotalFromPayload(cached?.payload || null); + return { concepts, reportedTotal }; + } + + if (this.sharedPendingPageRequests.has(cacheKey)) { + return await this.sharedPendingPageRequests.get(cacheKey); + } + + const pending = (async () => { + let response; + try { + response = await this.httpClient.get(this.meta.conceptsUrl, { params: { page, limit: CONCEPT_PAGE_SIZE } }); + } catch (error) { + if (error && error.response && error.response.status === 404) { + this.sharedPageCache.set(cacheKey, []); + return []; + } + throw error; + } + + const payload = response.data; + const items = Array.isArray(payload) + ? payload + : Array.isArray(payload?.results) + ? payload.results + : Array.isArray(payload?.items) + ? payload.items + : Array.isArray(payload?.data) + ? payload.data + : []; + + const mapped = items + .map(item => this.#toConceptContext(item)) + .filter(Boolean); + + this.sharedPageCache.set(cacheKey, { concepts: mapped, payload }); + for (const concept of mapped) { + if (concept && concept.code) { + this.sharedConceptCache.set(concept.code, concept); + } + } + return { + concepts: mapped, + reportedTotal: this.#extractTotalFromPayload(payload) + }; + })(); + + this.sharedPendingPageRequests.set(cacheKey, pending); + try { + return await pending; + } finally { + this.sharedPendingPageRequests.delete(cacheKey); + } + } + + #syncWarmStateWithChecksum() { + const checksum = this.meta?.checksum || null; + if (this.loadedChecksum == null) { + this.loadedChecksum = checksum; + return; + } + + if (checksum !== this.loadedChecksum) { + this.isComplete = false; + this.loadedConceptCount = -1; + this.backgroundLoadProgress = { processed: 0, total: null }; + this.sharedConceptCache.clear(); + this.sharedPageCache.clear(); + this.loadedChecksum = checksum; + this.materializedConceptList = null; + this.materializedConceptCount = -1; + if (this.meta?.codeSystem?.jsonObj) { + this.meta.codeSystem.jsonObj.content = CodeSystemContentMode.NotPresent; + delete this.meta.codeSystem.jsonObj.concept; + } + console.log(`[OCL] CodeSystem checksum changed, invalidated warm cache: ${this.#resourceKey()}`); + } + } + + #applyConceptsToCodeSystemResource(codeSystem) { + if (!codeSystem || typeof codeSystem !== 'object' || !codeSystem.jsonObj) { + return; + } + + if (this.isComplete !== true) { + delete codeSystem.jsonObj.concept; + return; + } + + const concepts = Array.from(this.sharedConceptCache.values()) + .filter(concept => concept && concept.code); + + if (!Array.isArray(this.materializedConceptList) || this.materializedConceptCount !== concepts.length) { + this.materializedConceptList = concepts + .sort((a, b) => String(a.code).localeCompare(String(b.code))) + .map(concept => { + const fhirConcept = { code: concept.code }; + + if (concept.display) { + fhirConcept.display = concept.display; + } + + if (concept.definition) { + fhirConcept.definition = concept.definition; + } + + if (Array.isArray(concept.designations) && concept.designations.length > 0) { + const designations = concept.designations + .filter(d => d && d.value) + .map(d => ({ + language: d.language || undefined, + value: d.value + })); + + if (designations.length > 0) { + fhirConcept.designation = designations; + } + } + + return fhirConcept; + }); + this.materializedConceptCount = concepts.length; + } + + codeSystem.jsonObj.concept = this.materializedConceptList; + codeSystem.jsonObj.content = CodeSystemContentMode.Complete; + } + + #backgroundLoadProgressSnapshot() { + const processed = this.backgroundLoadProgress?.processed; + const total = this.backgroundLoadProgress?.total; + if ( + typeof processed === 'number' && + Number.isFinite(processed) && + typeof total === 'number' && + Number.isFinite(total) && + total > 0 + ) { + return { processed, total }; + } + return null; + } + + #extractTotalFromPayload(payload) { + if (!payload || typeof payload !== 'object' || Array.isArray(payload)) { + return null; + } + + const candidates = [ + payload.total, + payload.total_count, + payload.totalCount, + payload.num_found, + payload.numFound, + payload.count + ]; + + for (const candidate of candidates) { + if (typeof candidate === 'number' && Number.isFinite(candidate) && candidate >= 0) { + return candidate; + } + } + + return null; + } + + async #fetchConceptCountFromHeaders() { + if (!this.meta?.conceptsUrl) { + return null; + } + + try { + const response = await this.httpClient.get(this.meta.conceptsUrl, { + params: { + limit: 1 + } + }); + return this.#extractNumFoundFromHeaders(response?.headers); + } catch (error) { + return null; + } + } + + #extractNumFoundFromHeaders(headers) { + if (!headers || typeof headers !== 'object') { + return null; + } + + const raw = headers.num_found ?? headers['num-found'] ?? headers.Num_Found ?? null; + const parsed = typeof raw === 'number' ? raw : Number.parseInt(raw, 10); + if (!Number.isFinite(parsed) || parsed < 0) { + return null; + } + return parsed; + } + + #resourceKey() { + return `${this.system()}|${this.version() || ''}`; + } + + currentChecksum() { + this.#syncWarmStateWithChecksum(); + return this.meta?.checksum || this.loadedChecksum || null; + } + + isCompleteNow() { + this.#syncWarmStateWithChecksum(); + return this.isComplete === true; + } + + #toConceptContext(concept) { + return toConceptContext(concept); + } + + system() { + return this.meta.canonicalUrl; + } + + name() { + return this.meta.name || this.meta.shortCode || this.meta.id || this.system(); + } + + version() { + return this.meta.version || null; + } + + id() { + return this.meta.id || this.meta.shortCode || this.system(); + } + + iteratable() { + return true; + } +} + +module.exports = { + OCLCodeSystemProvider, + OCLSourceCodeSystemFactory, + OCLBackgroundJobQueue +}; \ No newline at end of file diff --git a/tx/ocl/cs-ocl.js b/tx/ocl/cs-ocl.js index cdd13880..118116d3 100644 --- a/tx/ocl/cs-ocl.js +++ b/tx/ocl/cs-ocl.js @@ -1,1679 +1,2 @@ -const fs = require('fs/promises'); -const { AbstractCodeSystemProvider } = require('../cs/cs-provider-api'); -const { CodeSystemProvider, CodeSystemFactoryProvider, CodeSystemContentMode, FilterExecutionContext } = require('../cs/cs-api'); -const { CodeSystem } = require('../library/codesystem'); -const { SearchFilterText } = require('../library/designations'); -const { PAGE_SIZE, CONCEPT_PAGE_SIZE, COLD_CACHE_FRESHNESS_MS, OCL_CODESYSTEM_MARKER_EXTENSION } = require('./shared/constants'); -const { createOclHttpClient } = require('./http/client'); -const { fetchAllPages, extractItemsAndNext } = require('./http/pagination'); -const { CACHE_CS_DIR, CACHE_VS_DIR, getCacheFilePath } = require('./cache/cache-paths'); -const { ensureCacheDirectories, getColdCacheAgeMs, formatCacheAgeMinutes } = require('./cache/cache-utils'); -const { computeCodeSystemFingerprint } = require('./fingerprint/fingerprint'); -const { OCLBackgroundJobQueue } = require('./jobs/background-queue'); -const { OCLConceptFilterContext } = require('./model/concept-filter-context'); -const { toConceptContext } = require('./mappers/concept-mapper'); -const { patchSearchWorkerForOCLCodeFiltering } = require('./shared/patches'); +module.exports = require('./cs-ocl.cjs'); -patchSearchWorkerForOCLCodeFiltering(); - -class OCLCodeSystemProvider extends AbstractCodeSystemProvider { - constructor(config = {}) { - super(); - const options = typeof config === 'string' ? { baseUrl: config } : (config || {}); - - this.org = options.org || null; - const http = createOclHttpClient(options); - this.baseUrl = http.baseUrl; - this.httpClient = http.client; - - this._codeSystemsByCanonical = new Map(); - this._idToCodeSystem = new Map(); - this.sourceMetaByUrl = new Map(); - this._sourceStateByCanonical = new Map(); - this._usedIds = new Set(); - this._refreshPromise = null; - this._pendingChanges = null; - this._initialized = false; - this._initializePromise = null; - } - - async initialize() { - if (this._initialized) { - return; - } - - if (this._initializePromise) { - await this._initializePromise; - return; - } - - this._initializePromise = (async () => { - try { - const sources = await this.#fetchSourcesForDiscovery(); - console.log(`[OCL] Fetched ${sources.length} sources`); - - const snapshot = this.#buildSourceSnapshot(sources); - this.#applySnapshot(snapshot); - - console.log(`[OCL] Loaded ${this._codeSystemsByCanonical.size} code systems`); - this._initialized = true; - } catch (error) { - console.error(`[OCL] Initialization failed:`, error.message); - if (error.response) { - console.error(`[OCL] HTTP ${error.response.status}: ${error.response.statusText}`); - } - throw error; - } - })(); - - try { - await this._initializePromise; - } finally { - this._initializePromise = null; - } - } - - assignIds(ids) { - this._usedIds.clear(); - for (const cs of this._idToCodeSystem.values()) { - if (!cs.id || ids.has(`CodeSystem/${cs.id}`)) { - cs.id = String(ids.size); - cs.jsonObj.id = cs.id; - } - ids.add(`CodeSystem/${cs.id}`); - this._usedIds.add(cs.id); - this._idToCodeSystem.set(cs.id, cs); - } - } - - // eslint-disable-next-line no-unused-vars - async listCodeSystems(_fhirVersion, _context) { - await this.initialize(); - return Array.from(this._codeSystemsByCanonical.values()); - } - - // eslint-disable-next-line no-unused-vars - async loadCodeSystems(fhirVersion, context) { - return await this.listCodeSystems(fhirVersion, context); - } - - // Called once per minute by provider.updateCodeSystemList(). - // That caller is currently sync, so we stage async fetches and return the latest ready diff. - // eslint-disable-next-line no-unused-vars - getCodeSystemChanges(_fhirVersion, _context) { - if (!this._initialized) { - return this.#emptyChanges(); - } - - this.#scheduleRefresh(); - if (!this._pendingChanges) { - return this.#emptyChanges(); - } - - const out = this._pendingChanges; - this._pendingChanges = null; - return out; - } - - async close() { - } - - getSourceMetas() { - return Array.from(this.sourceMetaByUrl.values()); - } - - #scheduleRefresh() { - if (this._refreshPromise) { - return; - } - - this._refreshPromise = (async () => { - try { - const sources = await this.#fetchSourcesForDiscovery(); - const nextSnapshot = this.#buildSourceSnapshot(sources); - const changes = this.#diffSnapshots(this._sourceStateByCanonical, nextSnapshot); - this.#applySnapshot(nextSnapshot); - this._pendingChanges = changes; - } catch (error) { - console.error('[OCL] Incremental source refresh failed:', error.message); - this._pendingChanges = this.#emptyChanges(); - } finally { - this._refreshPromise = null; - } - })(); - } - - #emptyChanges() { - return { added: [], changed: [], deleted: [] }; - } - - #buildSourceSnapshot(sources) { - const snapshot = new Map(); - for (const source of sources || []) { - const cs = this.#toCodeSystem(source); - if (!cs) { - continue; - } - - const canonicalUrl = cs.url; - const meta = this.#buildSourceMeta(source, cs); - const checksum = this.#sourceChecksum(source); - snapshot.set(canonicalUrl, { cs, meta, checksum }); - } - return snapshot; - } - - async #fetchSourcesForDiscovery() { - const organizations = await this.#fetchOrganizationIds(); - if (organizations.length === 0) { - // Fallback for OCL instances that expose global listing but not org listing. - return await this.#fetchAllPages('/sources/'); - } - - const allSources = []; - const seen = new Set(); - - for (const orgId of organizations) { - const endpoint = `/orgs/${encodeURIComponent(orgId)}/sources/`; - const sources = await this.#fetchAllPages(endpoint); - for (const source of sources) { - if (!source || typeof source !== 'object') { - continue; - } - const key = this.#sourceIdentity(source); - if (seen.has(key)) { - continue; - } - seen.add(key); - allSources.push(source); - } - } - - return allSources; - } - - async #fetchOrganizationIds() { - const endpoint = '/orgs/'; - const orgs = await this.#fetchAllPages(endpoint); - - const ids = []; - const seen = new Set(); - for (const org of orgs || []) { - if (!org || typeof org !== 'object') { - continue; - } - - const id = org.id || org.mnemonic || org.short_code || org.shortCode || org.name || null; - if (!id) { - continue; - } - - const normalized = String(id).trim(); - if (!normalized || seen.has(normalized)) { - continue; - } - - seen.add(normalized); - ids.push(normalized); - } - - if (ids.length === 0 && this.org) { - ids.push(this.org); - } - - return ids; - } - - #sourceIdentity(source) { - if (!source || typeof source !== 'object') { - return '__invalid__'; - } - - const owner = source.owner || ''; - const canonical = source.canonical_url || source.canonicalUrl || ''; - const shortCode = source.short_code || source.shortCode || source.id || source.mnemonic || source.name || ''; - return `${owner}|${canonical}|${shortCode}`; - } - - #applySnapshot(snapshot) { - const previousSnapshot = this._sourceStateByCanonical; - this._codeSystemsByCanonical.clear(); - this._idToCodeSystem.clear(); - this.sourceMetaByUrl.clear(); - this._usedIds.clear(); - - for (const [canonicalUrl, entry] of snapshot.entries()) { - const cs = entry.cs; - const meta = entry.meta; - const previousEntry = previousSnapshot.get(canonicalUrl); - - // Preserve complete-content marker if checksum did not change. - if (previousEntry && previousEntry.checksum === entry.checksum && previousEntry.cs?.jsonObj?.content === CodeSystemContentMode.Complete) { - cs.jsonObj.content = CodeSystemContentMode.Complete; - - // Preserve materialized concepts across metadata refreshes. - if (Array.isArray(previousEntry.cs?.jsonObj?.concept)) { - cs.jsonObj.concept = previousEntry.cs.jsonObj.concept.map(concept => ({ ...concept })); - } - } - - // If a factory already has warm/cold concepts for this system, project them to the new snapshot resource. - OCLSourceCodeSystemFactory.syncCodeSystemResource(canonicalUrl, cs.version || null, cs); - - this.#trackCodeSystemId(cs); - this._codeSystemsByCanonical.set(canonicalUrl, cs); - this._idToCodeSystem.set(cs.id, cs); - this.sourceMetaByUrl.set(canonicalUrl, meta); - } - - this._sourceStateByCanonical = snapshot; - } - - #diffSnapshots(previousSnapshot, nextSnapshot) { - const added = []; - const changed = []; - const deleted = []; - - for (const [canonicalUrl, nextEntry] of nextSnapshot.entries()) { - const previousEntry = previousSnapshot.get(canonicalUrl); - if (!previousEntry) { - added.push(nextEntry.cs); - continue; - } - - // Keep stable ids across revisions so clients don't observe resource id churn. - nextEntry.cs.id = previousEntry.cs.id; - nextEntry.cs.jsonObj.id = previousEntry.cs.id; - - const previousChecksum = previousEntry.checksum || null; - const nextChecksum = nextEntry.checksum || null; - const checksumChanged = previousChecksum !== nextChecksum; - const versionChanged = (previousEntry.cs.version || null) !== (nextEntry.cs.version || null); - if (checksumChanged || versionChanged) { - if (checksumChanged) { - console.log(`[OCL] CodeSystem checksum changed: ${canonicalUrl} (${previousChecksum} -> ${nextChecksum})`); - } - changed.push(nextEntry.cs); - } - } - - for (const [canonicalUrl, previousEntry] of previousSnapshot.entries()) { - if (!nextSnapshot.has(canonicalUrl)) { - deleted.push(previousEntry.cs); - } - } - - return { added, changed, deleted }; - } - - #trackCodeSystemId(cs) { - if (!cs) { - return; - } - - if (!cs.id || this._usedIds.has(cs.id)) { - const raw = cs.id || cs.name || cs.url || 'ocl-cs'; - const base = this.spaceId ? `${this.spaceId}-${raw}` : String(raw); - let candidate = base; - let index = 1; - while (this._usedIds.has(candidate)) { - candidate = `${base}-${index}`; - index += 1; - } - cs.id = candidate; - cs.jsonObj.id = candidate; - } - - this._usedIds.add(cs.id); - } - - #toCodeSystem(source) { - if (!source || typeof source !== 'object') { - return null; - } - - const canonicalUrl = source.canonical_url || source.canonicalUrl || source.url; - if (!canonicalUrl) { - return null; - } - - const id = source.id || source.mnemonic; - if (!id) { - return null; - } - - const lastUpdated = this.#toIsoDate(source.updated_at || source.updatedAt || source.updated_on || source.updatedOn); - - const json = { - resourceType: 'CodeSystem', - id, - url: canonicalUrl, - version: source.version || null, - name: source.name || source.mnemonic || id, - title: source.full_name || source.fullName || source.name || source.mnemonic || id, - status: 'active', - experimental: source.experimental === true, - description: source.description || null, - publisher: source.owner || null, - caseSensitive: source.case_sensitive != null ? source.case_sensitive : (source.caseSensitive != null ? source.caseSensitive : true), - language: source.default_locale || source.defaultLocale || null, - filter: [ - { - code: 'code', - description: 'Match concept code', - operator: ['=', 'in', 'regex'], - value: 'code' - }, - { - code: 'display', - description: 'Match concept display text', - operator: ['=', 'in', 'regex'], - value: 'string' - }, - { - code: 'definition', - description: 'Match concept definition text', - operator: ['=', 'in', 'regex'], - value: 'string' - }, - { - code: 'inactive', - description: 'Match inactive (retired) status', - operator: ['=', 'in'], - value: 'boolean' - } - ], - property: [ - { - code: 'code', - uri: 'http://hl7.org/fhir/concept-properties#code', - description: 'Concept code', - type: 'code' - }, - { - code: 'display', - description: 'Concept display text', - type: 'string' - }, - { - code: 'definition', - description: 'Concept definition text', - type: 'string' - }, - { - code: 'inactive', - uri: 'http://hl7.org/fhir/concept-properties#status', - description: 'Whether concept is inactive (retired)', - type: 'boolean' - } - ], - extension: [ - { - url: OCL_CODESYSTEM_MARKER_EXTENSION, - valueBoolean: true - } - ], - content: 'not-present' - }; - - if (lastUpdated) { - json.meta = { lastUpdated }; - } - - return new CodeSystem(json, 'R5', true); - } - - #buildSourceMeta(source, cs) { - if (!source || !cs) { - return null; - } - - const owner = source.owner || null; - const shortCode = source.short_code || source.shortCode || source.mnemonic || source.id || null; - const canonicalUrl = cs.url; - if (!canonicalUrl) { - return; - } - - const conceptsUrl = this.#normalizePath(source.concepts_url || source.conceptsUrl || this.#buildConceptsPath(source)); - const meta = { - id: source.id || shortCode, - shortCode, - owner, - name: source.name || shortCode || cs.id, - description: source.description || null, - canonicalUrl, - version: source.version || null, - conceptsUrl, - checksum: this.#sourceChecksum(source), - codeSystem: cs - }; - - return meta; - } - - #sourceChecksum(source) { - // NOTE: OCL checksums are NOT reliable for cache invalidation decisions. - // They do not update when concepts are added or modified. - // This checksum is logged for debugging purposes only. - // Cache decisions are based on custom fingerprints computed from concept content. - - if (!source || typeof source !== 'object') { - return null; - } - - const checksums = source.checksums || {}; - const standard = checksums.standard || null; - const smart = checksums.smart || null; - if (standard) { - return String(standard); - } - if (smart) { - return String(smart); - } - - if (source.checksum) { - return String(source.checksum); - } - - const updated = source.updated_at || source.updatedAt || source.updated_on || source.updatedOn || null; - const version = source.version || null; - if (updated || version) { - return `${updated || ''}|${version || ''}`; - } - - return null; - } - - #buildConceptsPath(source) { - if (!source || typeof source !== 'object') { - return null; - } - const owner = source.owner || null; - const sourceId = source.short_code || source.shortCode || source.id || source.mnemonic || null; - if (!owner || !sourceId) { - const sourceUrl = source.url; - if (!sourceUrl || typeof sourceUrl !== 'string') { - return null; - } - const trimmed = sourceUrl.endsWith('/') ? sourceUrl : `${sourceUrl}/`; - return `${trimmed}concepts/`; - } - return `/orgs/${encodeURIComponent(owner)}/sources/${encodeURIComponent(sourceId)}/concepts/`; - } - - #normalizePath(pathValue) { - if (!pathValue) { - return null; - } - if (typeof pathValue !== 'string') { - return null; - } - if (pathValue.startsWith('http://') || pathValue.startsWith('https://')) { - return pathValue; - } - return `${this.baseUrl}${pathValue.startsWith('/') ? '' : '/'}${pathValue}`; - } - - async #fetchAllPages(path) { - try { - return await fetchAllPages(this.httpClient, path, { - pageSize: PAGE_SIZE, - baseUrl: this.baseUrl, - logger: console, - loggerPrefix: '[OCL]' - }); - } catch (error) { - if (error.response) { - console.error(`[OCL] HTTP ${error.response.status}: ${error.response.statusText}`); - console.error('[OCL] Response:', error.response.data); - } - throw error; - } - } - - #extractItemsAndNext(payload) { - return extractItemsAndNext(payload, this.baseUrl); - } - - #toIsoDate(value) { - if (!value) { - return null; - } - const date = new Date(value); - if (Number.isNaN(date.getTime())) { - return null; - } - return date.toISOString(); - } -} - -class OCLSourceCodeSystemProvider extends CodeSystemProvider { - constructor(opContext, supplements, client, meta, sharedCaches = null) { - super(opContext, supplements); - this.httpClient = client; - this.meta = meta; - this.conceptCache = sharedCaches?.conceptCache || new Map(); - this.pageCache = sharedCaches?.pageCache || new Map(); - this.pendingConceptRequests = sharedCaches?.pendingConceptRequests || new Map(); - this.pendingPageRequests = sharedCaches?.pendingPageRequests || new Map(); - this.scheduleBackgroundLoad = typeof sharedCaches?.scheduleBackgroundLoad === 'function' - ? sharedCaches.scheduleBackgroundLoad - : null; - this.isSystemComplete = typeof sharedCaches?.isSystemComplete === 'function' - ? sharedCaches.isSystemComplete - : (() => false); - this.getTotalConceptCount = typeof sharedCaches?.getTotalConceptCount === 'function' - ? sharedCaches.getTotalConceptCount - : (() => -1); - } - - system() { - return this.meta.canonicalUrl; - } - - version() { - return this.meta.version || null; - } - - description() { - return this.meta.description || null; - } - - name() { - return this.meta.name || this.meta.shortCode || this.meta.id || this.system(); - } - - contentMode() { - return this.isSystemComplete() ? CodeSystemContentMode.Complete : CodeSystemContentMode.NotPresent; - } - - totalCount() { - return this.getTotalConceptCount(); - } - - propertyDefinitions() { - return this.meta?.codeSystem?.jsonObj?.property || null; - } - - async code(code) { - const ctxt = await this.#ensureContext(code); - return ctxt ? ctxt.code : null; - } - - async display(code) { - const ctxt = await this.#ensureContext(code); - if (!ctxt) { - return null; - } - if (ctxt.display && this.opContext.langs.isEnglishOrNothing()) { - return ctxt.display; - } - const supp = this._displayFromSupplements(ctxt.code); - return supp || ctxt.display || null; - } - - async definition(code) { - const ctxt = await this.#ensureContext(code); - return ctxt ? ctxt.definition : null; - } - - async isAbstract(code) { - await this.#ensureContext(code); - return false; - } - - async isInactive(code) { - const ctxt = await this.#ensureContext(code); - return ctxt ? ctxt.retired === true : false; - } - - async isDeprecated(code) { - await this.#ensureContext(code); - return false; - } - - async getStatus(code) { - const ctxt = await this.#ensureContext(code); - if (!ctxt) { - return null; - } - return ctxt.retired === true ? 'inactive' : 'active'; - } - - async designations(code, displays) { - const ctxt = await this.#ensureContext(code); - if (ctxt && ctxt.display) { - const hasConceptDesignations = Array.isArray(ctxt.designations) && ctxt.designations.length > 0; - if (hasConceptDesignations) { - for (const d of ctxt.designations) { - if (!d || !d.value) { - continue; - } - displays.addDesignation(true, 'active', d.language || '', CodeSystem.makeUseForDisplay(), d.value); - } - } else { - displays.addDesignation(true, 'active', 'en', CodeSystem.makeUseForDisplay(), ctxt.display); - } - this._listSupplementDesignations(ctxt.code, displays); - } - } - - async locate(code) { - if (!code || typeof code !== 'string') { - return { context: null, message: 'Empty code' }; - } - - if (this.conceptCache.has(code)) { - return { context: this.conceptCache.get(code), message: null }; - } - - if (this.scheduleBackgroundLoad) { - this.scheduleBackgroundLoad('lookup-miss'); - } - - const concept = await this.#fetchConcept(code); - if (!concept) { - return { context: null, message: undefined }; - } - - this.conceptCache.set(code, concept); - return { context: concept, message: null }; - } - - async iterator(code) { - await this.#ensureContext(code); - if (code) { - return null; - } - return { - page: 1, - index: 0, - items: [], - total: -1, - done: false - }; - } - - async iteratorAll() { - return this.iterator(null); - } - - async getPrepContext(iterate) { - return new FilterExecutionContext(iterate); - } - - async doesFilter(prop, op, value) { - if (!prop || !op || value == null) { - return false; - } - - const normalizedProp = String(prop).trim().toLowerCase(); - const normalizedOp = String(op).trim().toLowerCase(); - const supportedOps = ['=', 'in', 'regex']; - if (!supportedOps.includes(normalizedOp)) { - return false; - } - - if (['concept', 'code', 'display', 'definition', 'inactive'].includes(normalizedProp)) { - return true; - } - - const defs = this.propertyDefinitions() || []; - return defs.some(def => def && def.code === normalizedProp); - } - - async searchFilter(filterContext, filter, sort) { - const matcher = this.#toSearchFilterText(filter); - const results = new OCLConceptFilterContext(); - const concepts = await this.#allConceptContexts(); - - for (const concept of concepts) { - const text = this.#conceptSearchText(concept); - const match = matcher.passes(text, true); - if (!match || match.passes !== true) { - continue; - } - - results.add(concept, this.#searchRating(concept, matcher, match.rating)); - } - - if (sort === true) { - results.sort(); - } - - if (!Array.isArray(filterContext.filters)) { - filterContext.filters = []; - } - filterContext.filters.push(results); - return filterContext; - } - - async filter(filterContext, prop, op, value) { - const normalizedProp = String(prop || '').trim().toLowerCase(); - const normalizedOp = String(op || '').trim().toLowerCase(); - - if (!await this.doesFilter(normalizedProp, normalizedOp, value)) { - throw new Error(`Filter ${prop} ${op} is not supported by OCL provider`); - } - - const set = new OCLConceptFilterContext(); - const concepts = await this.#allConceptContexts(); - const matcher = this.#buildPropertyMatcher(normalizedProp, normalizedOp, value); - - for (const concept of concepts) { - if (matcher(concept)) { - set.add(concept, 0); - } - } - - if (!Array.isArray(filterContext.filters)) { - filterContext.filters = []; - } - filterContext.filters.push(set); - return set; - } - - async executeFilters(filterContext) { - return Array.isArray(filterContext?.filters) ? filterContext.filters : []; - } - - // eslint-disable-next-line no-unused-vars - async filterSize(filterContext, set) { - return set ? set.size() : 0; - } - - // eslint-disable-next-line no-unused-vars - async filterMore(filterContext, set) { - return !!set && set.hasMore(); - } - - // eslint-disable-next-line no-unused-vars - async filterConcept(filterContext, set) { - if (!set) { - return null; - } - return set.next(); - } - - // eslint-disable-next-line no-unused-vars - async filterLocate(filterContext, set, code) { - if (!set) { - return `Code '${code}' not found: no filter results`; - } - const concept = set.findConceptByCode(code); - if (concept) { - return concept; - } - return null; - } - - // eslint-disable-next-line no-unused-vars - async filterCheck(filterContext, set, concept) { - if (!set || !concept) { - return false; - } - return set.containsConcept(concept); - } - - async filterFinish(filterContext) { - if (!Array.isArray(filterContext?.filters)) { - return; - } - for (const set of filterContext.filters) { - if (set && typeof set.reset === 'function') { - set.reset(); - } - } - filterContext.filters.length = 0; - } - - async nextContext(iteratorContext) { - if (!iteratorContext || iteratorContext.done) { - return null; - } - - if (iteratorContext.index >= iteratorContext.items.length) { - const pageItems = await this.#fetchConceptPage(iteratorContext.page); - iteratorContext.page += 1; - iteratorContext.index = 0; - iteratorContext.items = pageItems; - - if (!pageItems || pageItems.length === 0) { - iteratorContext.done = true; - return null; - } - } - - const concept = iteratorContext.items[iteratorContext.index]; - iteratorContext.index += 1; - return concept; - } - - async #ensureContext(code) { - if (!code) { - return null; - } - - // Some call paths pass a pending locate() Promise (or its wrapper result) - // instead of a raw code/context; normalize both shapes here. - if (code && typeof code === 'object' && typeof code.then === 'function') { - code = await code; - } - - if (code && typeof code === 'object' && Object.prototype.hasOwnProperty.call(code, 'context')) { - if (!code.context) { - throw new Error(code.message || 'Unknown code'); - } - code = code.context; - } - - if (typeof code === 'string') { - const result = await this.locate(code); - if (!result.context) { - throw new Error(result.message || `Unknown code ${code}`); - } - return result.context; - } - if (code && typeof code === 'object' && code.code) { - return code; - } - throw new Error(`Unknown Type at #ensureContext: ${typeof code}`); - } - - async #fetchConceptPage(page) { - if (!this.meta.conceptsUrl) { - return []; - } - const cacheKey = `${this.meta.conceptsUrl}|p=${page}|l=${CONCEPT_PAGE_SIZE}`; - if (this.pageCache.has(cacheKey)) { - const cached = this.pageCache.get(cacheKey); - return Array.isArray(cached) - ? cached - : Array.isArray(cached?.concepts) - ? cached.concepts - : []; - } - if (this.pendingPageRequests.has(cacheKey)) { - const pendingResult = await this.pendingPageRequests.get(cacheKey); - return Array.isArray(pendingResult) - ? pendingResult - : Array.isArray(pendingResult?.concepts) - ? pendingResult.concepts - : []; - } - - if (this.scheduleBackgroundLoad) { - this.scheduleBackgroundLoad('page-miss'); - } - - const pending = (async () => { - let response; - try { - response = await this.httpClient.get(this.meta.conceptsUrl, { params: { page, limit: CONCEPT_PAGE_SIZE } }); - } catch (error) { - // Some OCL instances return 404 for sources without concept listing endpoints. - // Treat this as an empty page so terminology operations degrade gracefully. - if (error && error.response && error.response.status === 404) { - this.pageCache.set(cacheKey, []); - return []; - } - throw error; - } - const payload = response.data; - const items = Array.isArray(payload) - ? payload - : Array.isArray(payload?.results) - ? payload.results - : Array.isArray(payload?.items) - ? payload.items - : Array.isArray(payload?.data) - ? payload.data - : []; - - const mapped = items.map(item => this.#toConceptContext(item)).filter(Boolean); - this.pageCache.set(cacheKey, mapped); - for (const concept of mapped) { - if (concept && concept.code) { - this.conceptCache.set(concept.code, concept); - } - } - return mapped; - })(); - - this.pendingPageRequests.set(cacheKey, pending); - try { - return await pending; - } finally { - this.pendingPageRequests.delete(cacheKey); - } - } - - async #fetchConcept(code) { - if (!this.meta.conceptsUrl) { - return null; - } - if (this.conceptCache.has(code)) { - return this.conceptCache.get(code); - } - const pendingKey = `${this.meta.conceptsUrl}|c=${code}`; - if (this.pendingConceptRequests.has(pendingKey)) { - return this.pendingConceptRequests.get(pendingKey); - } - - if (this.scheduleBackgroundLoad) { - this.scheduleBackgroundLoad('concept-miss'); - } - - const url = this.#buildConceptUrl(code); - const pending = (async () => { - let response; - try { - response = await this.httpClient.get(url); - } catch (error) { - // Missing concept should be treated as not-found, not as an internal server failure. - if (error && error.response && error.response.status === 404) { - return null; - } - throw error; - } - const concept = this.#toConceptContext(response.data); - if (concept && concept.code) { - this.conceptCache.set(concept.code, concept); - } - return concept; - })(); - - this.pendingConceptRequests.set(pendingKey, pending); - try { - return await pending; - } finally { - this.pendingConceptRequests.delete(pendingKey); - } - } - - async #allConceptContexts() { - const concepts = new Map(); - - for (const concept of this.conceptCache.values()) { - if (concept && concept.code) { - concepts.set(concept.code, concept); - } - } - - // Ensure search can operate even when content is not fully warm-loaded. - const iter = await this.iterator(null); - let concept = await this.nextContext(iter); - while (concept) { - if (concept.code && !concepts.has(concept.code)) { - concepts.set(concept.code, concept); - } - concept = await this.nextContext(iter); - } - - return Array.from(concepts.values()); - } - - #toSearchFilterText(filter) { - if (filter instanceof SearchFilterText) { - return filter; - } - if (typeof filter === 'string') { - return new SearchFilterText(filter); - } - if (filter && typeof filter.filter === 'string') { - return new SearchFilterText(filter.filter); - } - return new SearchFilterText(''); - } - - #conceptSearchText(concept) { - if (!concept || typeof concept !== 'object') { - return ''; - } - - const values = [concept.code, concept.display, concept.definition]; - if (Array.isArray(concept.designations)) { - for (const designation of concept.designations) { - if (designation && designation.value) { - values.push(designation.value); - } - } - } - - return values.filter(Boolean).join(' '); - } - - #searchRating(concept, matcher, baseRating) { - let rating = Number.isFinite(baseRating) ? baseRating : 0; - const term = matcher?.filter || ''; - if (!term) { - return rating; - } - - const code = String(concept?.code || '').toLowerCase(); - const display = String(concept?.display || '').toLowerCase(); - const definition = String(concept?.definition || '').toLowerCase(); - - if (code === term || display === term) { - rating += 100; - } else if (code.startsWith(term) || display.startsWith(term)) { - rating += 50; - } else if (definition.includes(term)) { - rating += 10; - } - - return rating; - } - - #buildPropertyMatcher(prop, op, value) { - if (op === 'regex') { - const regex = new RegExp(String(value), 'i'); - return concept => { - const candidate = this.#valueForFilter(concept, prop); - if (candidate == null) { - return false; - } - return regex.test(String(candidate)); - }; - } - - if (op === 'in') { - const tokens = String(value) - .split(',') - .map(token => token.trim().toLowerCase()) - .filter(Boolean); - return concept => { - const candidate = this.#valueForFilter(concept, prop); - if (candidate == null) { - return false; - } - return tokens.includes(String(candidate).toLowerCase()); - }; - } - - if (prop === 'inactive') { - const expected = this.#toBoolean(value); - return concept => { - const candidate = this.#toBoolean(this.#valueForFilter(concept, prop)); - return candidate === expected; - }; - } - - const expected = String(value).toLowerCase(); - return concept => { - const candidate = this.#valueForFilter(concept, prop); - if (candidate == null) { - return false; - } - return String(candidate).toLowerCase() === expected; - }; - } - - #valueForFilter(concept, prop) { - if (!concept || typeof concept !== 'object') { - return null; - } - - switch (prop) { - case 'concept': - case 'code': - return concept.code || null; - case 'display': - return concept.display || null; - case 'definition': - return concept.definition || null; - case 'inactive': - return concept.retired === true; - default: - return concept[prop] ?? null; - } - } - - #toBoolean(value) { - if (typeof value === 'boolean') { - return value; - } - - const text = String(value || '').trim().toLowerCase(); - return text === 'true' || text === '1' || text === 'yes'; - } - - #buildConceptUrl(code) { - const base = this.meta.conceptsUrl.endsWith('/') ? this.meta.conceptsUrl : `${this.meta.conceptsUrl}/`; - return `${base}${encodeURIComponent(code)}/`; - } - - #toConceptContext(concept) { - return toConceptContext(concept); - } -} - -class OCLSourceCodeSystemFactory extends CodeSystemFactoryProvider { - static factoriesByKey = new Map(); - - static syncCodeSystemResource(system, version = null, codeSystem = null) { - if (!system) { - return; - } - - const key = `${system}|${version || ''}`; - const factory = OCLSourceCodeSystemFactory.factoriesByKey.get(key); - if (!factory) { - return; - } - - factory.#applyConceptsToCodeSystemResource(codeSystem || factory.meta?.codeSystem || null); - } - - constructor(i18n, client, meta) { - super(i18n); - this.httpClient = client; - this.meta = meta; - this.sharedConceptCache = new Map(); - this.sharedPageCache = new Map(); - this.sharedPendingConceptRequests = new Map(); - this.sharedPendingPageRequests = new Map(); - this.isComplete = meta?.codeSystem?.jsonObj?.content === CodeSystemContentMode.Complete; - this.loadedConceptCount = -1; - this.loadedChecksum = meta?.checksum || null; - this.customFingerprint = null; - this.backgroundLoadProgress = { processed: 0, total: null }; - this.materializedConceptList = null; - this.materializedConceptCount = -1; - OCLSourceCodeSystemFactory.factoriesByKey.set(this.#resourceKey(), this); - - // Load cold cache at construction - this.#loadColdCache(); - } - - async #loadColdCache() { - const canonicalUrl = this.system(); - const version = this.version(); - const cacheFilePath = getCacheFilePath(CACHE_CS_DIR, canonicalUrl, version); - - try { - const data = await fs.readFile(cacheFilePath, 'utf-8'); - const cached = JSON.parse(data); - - if (!cached || !cached.concepts || !Array.isArray(cached.concepts)) { - return; - } - - // Restore concepts to cache - for (const concept of cached.concepts) { - if (concept && concept.code) { - this.sharedConceptCache.set(concept.code, concept); - } - } - - this.loadedConceptCount = cached.concepts.length; - this.customFingerprint = cached.fingerprint || null; - this.isComplete = true; - - if (this.meta?.codeSystem?.jsonObj) { - this.meta.codeSystem.jsonObj.content = CodeSystemContentMode.Complete; - } - - this.#applyConceptsToCodeSystemResource(this.meta?.codeSystem || null); - - console.log(`[OCL] Loaded CodeSystem from cold cache: ${canonicalUrl} (${cached.concepts.length} concepts, fingerprint=${this.customFingerprint?.substring(0, 8)})`); - } catch (error) { - if (error.code !== 'ENOENT') { - console.error(`[OCL] Failed to load cold cache for CodeSystem ${canonicalUrl}:`, error.message); - } - } - } - - async #saveColdCache(concepts) { - const canonicalUrl = this.system(); - const version = this.version(); - const cacheFilePath = getCacheFilePath(CACHE_CS_DIR, canonicalUrl, version); - - try { - await ensureCacheDirectories(CACHE_CS_DIR, CACHE_VS_DIR); - - const fingerprint = computeCodeSystemFingerprint(concepts); - const cacheData = { - canonicalUrl, - version, - fingerprint, - timestamp: new Date().toISOString(), - conceptCount: concepts.length, - concepts - }; - - await fs.writeFile(cacheFilePath, JSON.stringify(cacheData, null, 2), 'utf-8'); - console.log(`[OCL] Saved CodeSystem to cold cache: ${canonicalUrl} (${concepts.length} concepts, fingerprint=${fingerprint?.substring(0, 8)})`); - - return fingerprint; - } catch (error) { - console.error(`[OCL] Failed to save cold cache for CodeSystem ${canonicalUrl}:`, error.message); - return null; - } - } - - static scheduleBackgroundLoadByKey(system, version = null, reason = 'valueset-expansion') { - const key = `${system}|${version || ''}`; - const factory = OCLSourceCodeSystemFactory.factoriesByKey.get(key); - if (!factory) { - console.log(`[OCL] CodeSystem load not scheduled (factory unavailable): ${key}`); - return false; - } - factory.scheduleBackgroundLoad(reason); - return true; - } - - static checksumForResource(system, version = null) { - const key = `${system}|${version || ''}`; - const factory = OCLSourceCodeSystemFactory.factoriesByKey.get(key); - if (!factory) { - return null; - } - return factory.currentChecksum(); - } - - static loadProgress() { - let total = 0; - let loaded = 0; - - for (const factory of OCLSourceCodeSystemFactory.factoriesByKey.values()) { - total += 1; - if (factory && factory.isCompleteNow()) { - loaded += 1; - } - } - - const percentage = total > 0 ? (loaded / total) * 100 : 0; - return { loaded, total, percentage }; - } - - defaultVersion() { - return this.meta.version || null; - } - - build(opContext, supplements) { - this.#syncWarmStateWithChecksum(); - this.#applyConceptsToCodeSystemResource(this.meta?.codeSystem || null); - this.recordUse(); - return new OCLSourceCodeSystemProvider(opContext, supplements, this.httpClient, this.meta, { - conceptCache: this.sharedConceptCache, - pageCache: this.sharedPageCache, - pendingConceptRequests: this.sharedPendingConceptRequests, - pendingPageRequests: this.sharedPendingPageRequests, - scheduleBackgroundLoad: reason => this.scheduleBackgroundLoad(reason), - isSystemComplete: () => this.isComplete, - getTotalConceptCount: () => this.loadedConceptCount - }); - } - - scheduleBackgroundLoad(reason = 'request') { - this.#syncWarmStateWithChecksum(); - if (this.isComplete) { - return; - } - - const cacheFilePath = getCacheFilePath(CACHE_CS_DIR, this.system(), this.version()); - const cacheAgeMs = getColdCacheAgeMs(cacheFilePath); - if (cacheAgeMs != null && cacheAgeMs < COLD_CACHE_FRESHNESS_MS) { - console.log(`[OCL] Skipping warm-up for CodeSystem ${this.system()} (cold cache age: ${formatCacheAgeMinutes(cacheAgeMs)})`); - return; - } - - const key = this.#resourceKey(); - const jobKey = `cs:${key}`; - - if (OCLBackgroundJobQueue.isQueuedOrRunning(jobKey)) { - console.log(`[OCL] CodeSystem load already queued or running: ${key}`); - return; - } - - let queuedJobSize = null; - console.log(`[OCL] CodeSystem load enqueued: ${key} (${reason})`); - OCLBackgroundJobQueue.enqueue( - jobKey, - 'CodeSystem load', - async () => { - await this.#runBackgroundLoad(key, queuedJobSize); - }, - { - jobId: this.system(), - getProgress: () => this.#backgroundLoadProgressSnapshot(), - resolveJobSize: async () => { - queuedJobSize = await this.#fetchConceptCountFromHeaders(); - return queuedJobSize; - } - } - ); - } - - async #runBackgroundLoad(key, knownConceptCount = null) { - console.log(`[OCL] CodeSystem load started: ${key}`); - try { - this.backgroundLoadProgress = { processed: 0, total: null }; - const resolvedTotal = Number.isFinite(knownConceptCount) && knownConceptCount >= 0 - ? knownConceptCount - : await this.#fetchConceptCountFromHeaders(); - this.backgroundLoadProgress.total = resolvedTotal; - const count = await this.#loadAllConceptPages(); - this.loadedConceptCount = count; - this.isComplete = true; - this.loadedChecksum = this.meta?.checksum || null; - this.backgroundLoadProgress = { - processed: count, - total: count > 0 ? count : this.backgroundLoadProgress.total - }; - - if (this.meta?.codeSystem?.jsonObj) { - this.meta.codeSystem.jsonObj.content = CodeSystemContentMode.Complete; - } - - this.#applyConceptsToCodeSystemResource(this.meta?.codeSystem || null); - - // Compute custom fingerprint and compare with cold cache - const allConcepts = Array.from(this.sharedConceptCache.values()); - const newFingerprint = computeCodeSystemFingerprint(allConcepts); - - if (this.customFingerprint && newFingerprint === this.customFingerprint) { - console.log(`[OCL] CodeSystem fingerprint unchanged: ${key} (fingerprint=${newFingerprint?.substring(0, 8)})`); - } else { - if (this.customFingerprint) { - console.log(`[OCL] CodeSystem fingerprint changed: ${key} (${this.customFingerprint?.substring(0, 8)} -> ${newFingerprint?.substring(0, 8)})`); - console.log(`[OCL] Replacing cold cache with new hot cache: ${key}`); - } else { - console.log(`[OCL] Computed fingerprint for CodeSystem: ${key} (fingerprint=${newFingerprint?.substring(0, 8)})`); - } - - // Save to cold cache - const savedFingerprint = await this.#saveColdCache(allConcepts); - if (savedFingerprint) { - this.customFingerprint = savedFingerprint; - } - } - - console.log(`[OCL] CodeSystem load completed, marked content=complete: ${key}`); - const progress = OCLSourceCodeSystemFactory.loadProgress(); - console.log(`[OCL] CodeSystem load completed: ${this.system()}. Loaded ${progress.loaded}/${progress.total} CodeSystems (${progress.percentage.toFixed(2)}%)`); - console.log(`[OCL] CodeSystem now available in cache: ${key} (${count} concepts)`); - } catch (error) { - console.error(`[OCL] CodeSystem background load failed: ${key}: ${error.message}`); - } - } - - async #loadAllConceptPages() { - if (!this.meta?.conceptsUrl) { - this.loadedConceptCount = 0; - this.backgroundLoadProgress = { processed: 0, total: 0 }; - return 0; - } - - let page = 1; - let total = 0; - - // eslint-disable-next-line no-constant-condition - while (true) { - const pageData = await this.#fetchAndCacheConceptPage(page); - const concepts = Array.isArray(pageData?.concepts) ? pageData.concepts : []; - if (concepts.length === 0) { - break; - } - total += concepts.length; - this.backgroundLoadProgress.processed = total; - if (concepts.length < CONCEPT_PAGE_SIZE) { - break; - } - page += 1; - } - - return total; - } - - async #fetchAndCacheConceptPage(page) { - const cacheKey = `${this.meta.conceptsUrl}|p=${page}|l=${CONCEPT_PAGE_SIZE}`; - if (this.sharedPageCache.has(cacheKey)) { - const cached = this.sharedPageCache.get(cacheKey); - const concepts = Array.isArray(cached) - ? cached - : Array.isArray(cached?.concepts) - ? cached.concepts - : []; - const reportedTotal = this.#extractTotalFromPayload(cached?.payload || null); - return { concepts, reportedTotal }; - } - - if (this.sharedPendingPageRequests.has(cacheKey)) { - return await this.sharedPendingPageRequests.get(cacheKey); - } - - const pending = (async () => { - let response; - try { - response = await this.httpClient.get(this.meta.conceptsUrl, { params: { page, limit: CONCEPT_PAGE_SIZE } }); - } catch (error) { - if (error && error.response && error.response.status === 404) { - this.sharedPageCache.set(cacheKey, []); - return []; - } - throw error; - } - - const payload = response.data; - const items = Array.isArray(payload) - ? payload - : Array.isArray(payload?.results) - ? payload.results - : Array.isArray(payload?.items) - ? payload.items - : Array.isArray(payload?.data) - ? payload.data - : []; - - const mapped = items - .map(item => this.#toConceptContext(item)) - .filter(Boolean); - - this.sharedPageCache.set(cacheKey, { concepts: mapped, payload }); - for (const concept of mapped) { - if (concept && concept.code) { - this.sharedConceptCache.set(concept.code, concept); - } - } - return { - concepts: mapped, - reportedTotal: this.#extractTotalFromPayload(payload) - }; - })(); - - this.sharedPendingPageRequests.set(cacheKey, pending); - try { - return await pending; - } finally { - this.sharedPendingPageRequests.delete(cacheKey); - } - } - - #syncWarmStateWithChecksum() { - const checksum = this.meta?.checksum || null; - if (this.loadedChecksum == null) { - this.loadedChecksum = checksum; - return; - } - - if (checksum !== this.loadedChecksum) { - this.isComplete = false; - this.loadedConceptCount = -1; - this.backgroundLoadProgress = { processed: 0, total: null }; - this.sharedConceptCache.clear(); - this.sharedPageCache.clear(); - this.loadedChecksum = checksum; - this.materializedConceptList = null; - this.materializedConceptCount = -1; - if (this.meta?.codeSystem?.jsonObj) { - this.meta.codeSystem.jsonObj.content = CodeSystemContentMode.NotPresent; - delete this.meta.codeSystem.jsonObj.concept; - } - console.log(`[OCL] CodeSystem checksum changed, invalidated warm cache: ${this.#resourceKey()}`); - } - } - - #applyConceptsToCodeSystemResource(codeSystem) { - if (!codeSystem || typeof codeSystem !== 'object' || !codeSystem.jsonObj) { - return; - } - - if (this.isComplete !== true) { - delete codeSystem.jsonObj.concept; - return; - } - - const concepts = Array.from(this.sharedConceptCache.values()) - .filter(concept => concept && concept.code); - - if (!Array.isArray(this.materializedConceptList) || this.materializedConceptCount !== concepts.length) { - this.materializedConceptList = concepts - .sort((a, b) => String(a.code).localeCompare(String(b.code))) - .map(concept => { - const fhirConcept = { code: concept.code }; - - if (concept.display) { - fhirConcept.display = concept.display; - } - - if (concept.definition) { - fhirConcept.definition = concept.definition; - } - - if (Array.isArray(concept.designations) && concept.designations.length > 0) { - const designations = concept.designations - .filter(d => d && d.value) - .map(d => ({ - language: d.language || undefined, - value: d.value - })); - - if (designations.length > 0) { - fhirConcept.designation = designations; - } - } - - return fhirConcept; - }); - this.materializedConceptCount = concepts.length; - } - - codeSystem.jsonObj.concept = this.materializedConceptList; - codeSystem.jsonObj.content = CodeSystemContentMode.Complete; - } - - #backgroundLoadProgressSnapshot() { - const processed = this.backgroundLoadProgress?.processed; - const total = this.backgroundLoadProgress?.total; - if ( - typeof processed === 'number' && - Number.isFinite(processed) && - typeof total === 'number' && - Number.isFinite(total) && - total > 0 - ) { - return { processed, total }; - } - return null; - } - - #extractTotalFromPayload(payload) { - if (!payload || typeof payload !== 'object' || Array.isArray(payload)) { - return null; - } - - const candidates = [ - payload.total, - payload.total_count, - payload.totalCount, - payload.num_found, - payload.numFound, - payload.count - ]; - - for (const candidate of candidates) { - if (typeof candidate === 'number' && Number.isFinite(candidate) && candidate >= 0) { - return candidate; - } - } - - return null; - } - - async #fetchConceptCountFromHeaders() { - if (!this.meta?.conceptsUrl) { - return null; - } - - try { - const response = await this.httpClient.get(this.meta.conceptsUrl, { - params: { - limit: 1 - } - }); - return this.#extractNumFoundFromHeaders(response?.headers); - } catch (error) { - return null; - } - } - - #extractNumFoundFromHeaders(headers) { - if (!headers || typeof headers !== 'object') { - return null; - } - - const raw = headers.num_found ?? headers['num-found'] ?? headers.Num_Found ?? null; - const parsed = typeof raw === 'number' ? raw : Number.parseInt(raw, 10); - if (!Number.isFinite(parsed) || parsed < 0) { - return null; - } - return parsed; - } - - #resourceKey() { - return `${this.system()}|${this.version() || ''}`; - } - - currentChecksum() { - this.#syncWarmStateWithChecksum(); - return this.meta?.checksum || this.loadedChecksum || null; - } - - isCompleteNow() { - this.#syncWarmStateWithChecksum(); - return this.isComplete === true; - } - - #toConceptContext(concept) { - return toConceptContext(concept); - } - - system() { - return this.meta.canonicalUrl; - } - - name() { - return this.meta.name || this.meta.shortCode || this.meta.id || this.system(); - } - - version() { - return this.meta.version || null; - } - - id() { - return this.meta.id || this.meta.shortCode || this.system(); - } - - iteratable() { - return true; - } -} - -module.exports = { - OCLCodeSystemProvider, - OCLSourceCodeSystemFactory, - OCLBackgroundJobQueue -}; \ No newline at end of file diff --git a/tx/ocl/fingerprint/fingerprint.cjs b/tx/ocl/fingerprint/fingerprint.cjs new file mode 100644 index 00000000..4a18c158 --- /dev/null +++ b/tx/ocl/fingerprint/fingerprint.cjs @@ -0,0 +1,67 @@ +const crypto = require('crypto'); + +function hashSortedLines(lines) { + const hash = crypto.createHash('sha256'); + for (const line of lines.sort()) { + hash.update(line); + hash.update('\n'); + } + return hash.digest('hex'); +} + +function computeCodeSystemFingerprint(concepts) { + if (!Array.isArray(concepts) || concepts.length === 0) { + return null; + } + + const normalized = concepts + .map(concept => { + if (!concept || !concept.code) { + return null; + } + + const code = String(concept.code || ''); + const display = String(concept.display || ''); + const definition = String(concept.definition || ''); + const retired = concept.retired === true ? '1' : '0'; + return `${code}|${display}|${definition}|${retired}`; + }) + .filter(Boolean); + + if (normalized.length === 0) { + return null; + } + + return hashSortedLines(normalized); +} + +function computeValueSetExpansionFingerprint(expansion) { + if (!expansion || !Array.isArray(expansion.contains) || expansion.contains.length === 0) { + return null; + } + + const normalized = expansion.contains + .map(entry => { + if (!entry || !entry.code) { + return null; + } + + const system = String(entry.system || ''); + const code = String(entry.code || ''); + const display = String(entry.display || ''); + const inactive = entry.inactive === true ? '1' : '0'; + return `${system}|${code}|${display}|${inactive}`; + }) + .filter(Boolean); + + if (normalized.length === 0) { + return null; + } + + return hashSortedLines(normalized); +} + +module.exports = { + computeCodeSystemFingerprint, + computeValueSetExpansionFingerprint +}; diff --git a/tx/ocl/fingerprint/fingerprint.js b/tx/ocl/fingerprint/fingerprint.js index 4a18c158..74ad6070 100644 --- a/tx/ocl/fingerprint/fingerprint.js +++ b/tx/ocl/fingerprint/fingerprint.js @@ -1,67 +1,2 @@ -const crypto = require('crypto'); +module.exports = require('./fingerprint.cjs'); -function hashSortedLines(lines) { - const hash = crypto.createHash('sha256'); - for (const line of lines.sort()) { - hash.update(line); - hash.update('\n'); - } - return hash.digest('hex'); -} - -function computeCodeSystemFingerprint(concepts) { - if (!Array.isArray(concepts) || concepts.length === 0) { - return null; - } - - const normalized = concepts - .map(concept => { - if (!concept || !concept.code) { - return null; - } - - const code = String(concept.code || ''); - const display = String(concept.display || ''); - const definition = String(concept.definition || ''); - const retired = concept.retired === true ? '1' : '0'; - return `${code}|${display}|${definition}|${retired}`; - }) - .filter(Boolean); - - if (normalized.length === 0) { - return null; - } - - return hashSortedLines(normalized); -} - -function computeValueSetExpansionFingerprint(expansion) { - if (!expansion || !Array.isArray(expansion.contains) || expansion.contains.length === 0) { - return null; - } - - const normalized = expansion.contains - .map(entry => { - if (!entry || !entry.code) { - return null; - } - - const system = String(entry.system || ''); - const code = String(entry.code || ''); - const display = String(entry.display || ''); - const inactive = entry.inactive === true ? '1' : '0'; - return `${system}|${code}|${display}|${inactive}`; - }) - .filter(Boolean); - - if (normalized.length === 0) { - return null; - } - - return hashSortedLines(normalized); -} - -module.exports = { - computeCodeSystemFingerprint, - computeValueSetExpansionFingerprint -}; diff --git a/tx/ocl/http/client.cjs b/tx/ocl/http/client.cjs new file mode 100644 index 00000000..7ae7ecb2 --- /dev/null +++ b/tx/ocl/http/client.cjs @@ -0,0 +1,31 @@ +const axios = require('axios'); +const { DEFAULT_BASE_URL } = require('../shared/constants'); + +function createOclHttpClient(config = {}) { + const options = typeof config === 'string' ? { baseUrl: config } : (config || {}); + + const baseUrl = (options.baseUrl || DEFAULT_BASE_URL).replace(/\/+$/, ''); + const headers = { + Accept: 'application/json', + 'User-Agent': 'FHIRSmith-OCL-Provider/1.0' + }; + + if (options.token) { + headers.Authorization = options.token.startsWith('Token ') || options.token.startsWith('Bearer ') + ? options.token + : `Token ${options.token}`; + } + + return { + baseUrl, + client: axios.create({ + baseURL: baseUrl, + timeout: options.timeout || 30000, + headers + }) + }; +} + +module.exports = { + createOclHttpClient +}; diff --git a/tx/ocl/http/client.js b/tx/ocl/http/client.js index 7ae7ecb2..805c3b60 100644 --- a/tx/ocl/http/client.js +++ b/tx/ocl/http/client.js @@ -1,31 +1,2 @@ -const axios = require('axios'); -const { DEFAULT_BASE_URL } = require('../shared/constants'); +module.exports = require('./client.cjs'); -function createOclHttpClient(config = {}) { - const options = typeof config === 'string' ? { baseUrl: config } : (config || {}); - - const baseUrl = (options.baseUrl || DEFAULT_BASE_URL).replace(/\/+$/, ''); - const headers = { - Accept: 'application/json', - 'User-Agent': 'FHIRSmith-OCL-Provider/1.0' - }; - - if (options.token) { - headers.Authorization = options.token.startsWith('Token ') || options.token.startsWith('Bearer ') - ? options.token - : `Token ${options.token}`; - } - - return { - baseUrl, - client: axios.create({ - baseURL: baseUrl, - timeout: options.timeout || 30000, - headers - }) - }; -} - -module.exports = { - createOclHttpClient -}; diff --git a/tx/ocl/http/pagination.cjs b/tx/ocl/http/pagination.cjs new file mode 100644 index 00000000..7ea7f7b4 --- /dev/null +++ b/tx/ocl/http/pagination.cjs @@ -0,0 +1,98 @@ +const { PAGE_SIZE } = require('../shared/constants'); + +function extractItemsAndNext(payload, baseUrl = null) { + if (Array.isArray(payload)) { + return { items: payload, next: null }; + } + + if (!payload || typeof payload !== 'object') { + return { items: [], next: null }; + } + + const items = Array.isArray(payload.results) + ? payload.results + : Array.isArray(payload.items) + ? payload.items + : Array.isArray(payload.data) + ? payload.data + : []; + + const next = payload.next || null; + if (!next) { + return { items, next: null }; + } + + if (baseUrl && typeof next === 'string' && next.startsWith(baseUrl)) { + return { items, next: next.replace(baseUrl, '') }; + } + + return { items, next }; +} + +async function fetchAllPages(httpClient, path, options = {}) { + const { + params = {}, + pageSize = PAGE_SIZE, + maxPages = Number.MAX_SAFE_INTEGER, + baseUrl = null, + useNextLinks = true, + logger = null, + loggerPrefix = '[OCL]' + } = options; + + const results = []; + let page = 1; + let nextPath = path; + let pageCount = 0; + let usePageMode = true; + + while (nextPath && pageCount < maxPages) { + try { + const response = usePageMode + ? await httpClient.get(path, { params: { ...params, page, limit: pageSize } }) + : await httpClient.get(nextPath); + + if (Array.isArray(response.data)) { + results.push(...response.data); + pageCount += 1; + + if (response.data.length < pageSize) { + break; + } + + page += 1; + nextPath = path; + continue; + } + + const { items, next } = extractItemsAndNext(response.data, baseUrl); + results.push(...items); + pageCount += 1; + + if (useNextLinks && next) { + usePageMode = false; + nextPath = next; + continue; + } + + if (usePageMode && items.length >= pageSize && pageCount < maxPages) { + page += 1; + nextPath = path; + } else { + break; + } + } catch (error) { + if (logger && typeof logger.error === 'function') { + logger.error(`${loggerPrefix} Fetch error on page ${page}:`, error.message); + } + throw error; + } + } + + return results; +} + +module.exports = { + extractItemsAndNext, + fetchAllPages +}; diff --git a/tx/ocl/http/pagination.js b/tx/ocl/http/pagination.js index 7ea7f7b4..314b5295 100644 --- a/tx/ocl/http/pagination.js +++ b/tx/ocl/http/pagination.js @@ -1,98 +1,2 @@ -const { PAGE_SIZE } = require('../shared/constants'); +module.exports = require('./pagination.cjs'); -function extractItemsAndNext(payload, baseUrl = null) { - if (Array.isArray(payload)) { - return { items: payload, next: null }; - } - - if (!payload || typeof payload !== 'object') { - return { items: [], next: null }; - } - - const items = Array.isArray(payload.results) - ? payload.results - : Array.isArray(payload.items) - ? payload.items - : Array.isArray(payload.data) - ? payload.data - : []; - - const next = payload.next || null; - if (!next) { - return { items, next: null }; - } - - if (baseUrl && typeof next === 'string' && next.startsWith(baseUrl)) { - return { items, next: next.replace(baseUrl, '') }; - } - - return { items, next }; -} - -async function fetchAllPages(httpClient, path, options = {}) { - const { - params = {}, - pageSize = PAGE_SIZE, - maxPages = Number.MAX_SAFE_INTEGER, - baseUrl = null, - useNextLinks = true, - logger = null, - loggerPrefix = '[OCL]' - } = options; - - const results = []; - let page = 1; - let nextPath = path; - let pageCount = 0; - let usePageMode = true; - - while (nextPath && pageCount < maxPages) { - try { - const response = usePageMode - ? await httpClient.get(path, { params: { ...params, page, limit: pageSize } }) - : await httpClient.get(nextPath); - - if (Array.isArray(response.data)) { - results.push(...response.data); - pageCount += 1; - - if (response.data.length < pageSize) { - break; - } - - page += 1; - nextPath = path; - continue; - } - - const { items, next } = extractItemsAndNext(response.data, baseUrl); - results.push(...items); - pageCount += 1; - - if (useNextLinks && next) { - usePageMode = false; - nextPath = next; - continue; - } - - if (usePageMode && items.length >= pageSize && pageCount < maxPages) { - page += 1; - nextPath = path; - } else { - break; - } - } catch (error) { - if (logger && typeof logger.error === 'function') { - logger.error(`${loggerPrefix} Fetch error on page ${page}:`, error.message); - } - throw error; - } - } - - return results; -} - -module.exports = { - extractItemsAndNext, - fetchAllPages -}; diff --git a/tx/ocl/jobs/background-queue.cjs b/tx/ocl/jobs/background-queue.cjs new file mode 100644 index 00000000..060b93b4 --- /dev/null +++ b/tx/ocl/jobs/background-queue.cjs @@ -0,0 +1,200 @@ +class OCLBackgroundJobQueue { + static MAX_CONCURRENT = 2; + static HEARTBEAT_INTERVAL_MS = 30000; + static UNKNOWN_JOB_SIZE = Number.MAX_SAFE_INTEGER; + static pendingJobs = []; + static activeCount = 0; + static queuedOrRunningKeys = new Set(); + static activeJobs = new Map(); + static heartbeatTimer = null; + static enqueueSequence = 0; + + static enqueue(jobKey, jobType, runJob, options = {}) { + if (!jobKey || typeof runJob !== 'function') { + return false; + } + + if (this.queuedOrRunningKeys.has(jobKey)) { + return false; + } + + this.queuedOrRunningKeys.add(jobKey); + const resolveAndEnqueue = async () => { + const resolvedSize = await this.#resolveJobSize(options); + const normalizedSize = this.#normalizeJobSize(resolvedSize); + this.#insertPendingJobOrdered({ + jobKey, + jobType: jobType || 'background-job', + jobId: options?.jobId || jobKey, + jobSize: normalizedSize, + getProgress: typeof options?.getProgress === 'function' ? options.getProgress : null, + runJob, + enqueueOrder: this.enqueueSequence++ + }); + this.ensureHeartbeatRunning(); + console.log(`[OCL] ${jobType || 'Background job'} enqueued: ${jobKey} (size=${normalizedSize}, queue=${this.pendingJobs.length}, active=${this.activeCount})`); + this.processNext(); + }; + + Promise.resolve() + .then(resolveAndEnqueue) + .catch((error) => { + this.queuedOrRunningKeys.delete(jobKey); + const message = error && error.message ? error.message : String(error); + console.error(`[OCL] Failed to enqueue background job: ${jobType || 'background-job'} ${jobKey}: ${message}`); + }); + + return true; + } + + static async #resolveJobSize(options = {}) { + if (typeof options?.resolveJobSize === 'function') { + try { + return await options.resolveJobSize(); + } catch (_error) { + return this.UNKNOWN_JOB_SIZE; + } + } + + if (options && Object.prototype.hasOwnProperty.call(options, 'jobSize')) { + return options.jobSize; + } + + return this.UNKNOWN_JOB_SIZE; + } + + static #normalizeJobSize(jobSize) { + const parsed = Number.parseInt(jobSize, 10); + if (!Number.isFinite(parsed) || parsed < 0) { + return this.UNKNOWN_JOB_SIZE; + } + return parsed; + } + + static #insertPendingJobOrdered(job) { + let index = this.pendingJobs.findIndex(existing => { + if (existing.jobSize === job.jobSize) { + return existing.enqueueOrder > job.enqueueOrder; + } + return existing.jobSize > job.jobSize; + }); + + if (index < 0) { + index = this.pendingJobs.length; + } + + this.pendingJobs.splice(index, 0, job); + } + + static isQueuedOrRunning(jobKey) { + return this.queuedOrRunningKeys.has(jobKey); + } + + static ensureHeartbeatRunning() { + if (this.heartbeatTimer) { + return; + } + + this.heartbeatTimer = setInterval(() => { + this.logHeartbeat(); + }, this.HEARTBEAT_INTERVAL_MS); + + if (typeof this.heartbeatTimer.unref === 'function') { + this.heartbeatTimer.unref(); + } + } + + static logHeartbeat() { + const activeJobs = Array.from(this.activeJobs.values()); + const lines = [ + '[OCL] OCL background status:', + ` active jobs: ${activeJobs.length}`, + ` queued jobs: ${this.pendingJobs.length}` + ]; + + activeJobs.forEach((job, index) => { + lines.push(''); + lines.push(` job ${index + 1}:`); + lines.push(` type: ${job.jobType || 'background-job'}`); + lines.push(` id: ${job.jobId || job.jobKey}`); + lines.push(` size: ${job.jobSize}`); + lines.push(` progress: ${this.formatProgress(job.getProgress)}`); + }); + + console.log(lines.join('\n')); + } + + static formatProgress(getProgress) { + if (typeof getProgress !== 'function') { + return 'unknown'; + } + + try { + const progress = getProgress(); + if (typeof progress === 'number' && Number.isFinite(progress)) { + const bounded = Math.max(0, Math.min(100, progress)); + return `${Math.round(bounded)}%`; + } + + if (progress && typeof progress === 'object') { + if (typeof progress.percentage === 'number' && Number.isFinite(progress.percentage)) { + const bounded = Math.max(0, Math.min(100, progress.percentage)); + return `${Math.round(bounded)}%`; + } + + if ( + typeof progress.processed === 'number' && + Number.isFinite(progress.processed) && + typeof progress.total === 'number' && + Number.isFinite(progress.total) && + progress.total > 0 + ) { + const ratio = progress.processed / progress.total; + const bounded = Math.max(0, Math.min(100, ratio * 100)); + return `${Math.round(bounded)}%`; + } + } + } catch (_error) { + return 'unknown'; + } + + return 'unknown'; + } + + static processNext() { + while (this.activeCount < this.MAX_CONCURRENT && this.pendingJobs.length > 0) { + const job = this.pendingJobs.shift(); + this.activeCount += 1; + this.activeJobs.set(job.jobKey, { + jobKey: job.jobKey, + jobType: job.jobType, + jobId: job.jobId || job.jobKey, + jobSize: job.jobSize, + getProgress: job.getProgress || null, + startedAt: Date.now() + }); + console.log(`[OCL] Background job started: ${job.jobType} ${job.jobKey} (size=${job.jobSize}, queue=${this.pendingJobs.length}, active=${this.activeCount})`); + + Promise.resolve() + .then(() => job.runJob()) + .then(() => { + console.log(`[OCL] Background job completed: ${job.jobType} ${job.jobKey}`); + }) + .catch((error) => { + const message = error && error.message ? error.message : String(error); + console.error(`[OCL] Background job failed: ${job.jobType} ${job.jobKey}: ${message}`); + }) + .finally(() => { + this.activeCount -= 1; + this.queuedOrRunningKeys.delete(job.jobKey); + this.activeJobs.delete(job.jobKey); + console.log(`[OCL] Background queue status: queue=${this.pendingJobs.length}, active=${this.activeCount}`); + this.processNext(); + }); + } + } +} + +module.exports = { + OCLBackgroundJobQueue +}; diff --git a/tx/ocl/jobs/background-queue.js b/tx/ocl/jobs/background-queue.js index 060b93b4..5bdea4a4 100644 --- a/tx/ocl/jobs/background-queue.js +++ b/tx/ocl/jobs/background-queue.js @@ -1,200 +1,2 @@ -class OCLBackgroundJobQueue { - static MAX_CONCURRENT = 2; - static HEARTBEAT_INTERVAL_MS = 30000; - static UNKNOWN_JOB_SIZE = Number.MAX_SAFE_INTEGER; - static pendingJobs = []; - static activeCount = 0; - static queuedOrRunningKeys = new Set(); - static activeJobs = new Map(); - static heartbeatTimer = null; - static enqueueSequence = 0; +module.exports = require('./background-queue.cjs'); - static enqueue(jobKey, jobType, runJob, options = {}) { - if (!jobKey || typeof runJob !== 'function') { - return false; - } - - if (this.queuedOrRunningKeys.has(jobKey)) { - return false; - } - - this.queuedOrRunningKeys.add(jobKey); - const resolveAndEnqueue = async () => { - const resolvedSize = await this.#resolveJobSize(options); - const normalizedSize = this.#normalizeJobSize(resolvedSize); - this.#insertPendingJobOrdered({ - jobKey, - jobType: jobType || 'background-job', - jobId: options?.jobId || jobKey, - jobSize: normalizedSize, - getProgress: typeof options?.getProgress === 'function' ? options.getProgress : null, - runJob, - enqueueOrder: this.enqueueSequence++ - }); - this.ensureHeartbeatRunning(); - console.log(`[OCL] ${jobType || 'Background job'} enqueued: ${jobKey} (size=${normalizedSize}, queue=${this.pendingJobs.length}, active=${this.activeCount})`); - this.processNext(); - }; - - Promise.resolve() - .then(resolveAndEnqueue) - .catch((error) => { - this.queuedOrRunningKeys.delete(jobKey); - const message = error && error.message ? error.message : String(error); - console.error(`[OCL] Failed to enqueue background job: ${jobType || 'background-job'} ${jobKey}: ${message}`); - }); - - return true; - } - - static async #resolveJobSize(options = {}) { - if (typeof options?.resolveJobSize === 'function') { - try { - return await options.resolveJobSize(); - } catch (_error) { - return this.UNKNOWN_JOB_SIZE; - } - } - - if (options && Object.prototype.hasOwnProperty.call(options, 'jobSize')) { - return options.jobSize; - } - - return this.UNKNOWN_JOB_SIZE; - } - - static #normalizeJobSize(jobSize) { - const parsed = Number.parseInt(jobSize, 10); - if (!Number.isFinite(parsed) || parsed < 0) { - return this.UNKNOWN_JOB_SIZE; - } - return parsed; - } - - static #insertPendingJobOrdered(job) { - let index = this.pendingJobs.findIndex(existing => { - if (existing.jobSize === job.jobSize) { - return existing.enqueueOrder > job.enqueueOrder; - } - return existing.jobSize > job.jobSize; - }); - - if (index < 0) { - index = this.pendingJobs.length; - } - - this.pendingJobs.splice(index, 0, job); - } - - static isQueuedOrRunning(jobKey) { - return this.queuedOrRunningKeys.has(jobKey); - } - - static ensureHeartbeatRunning() { - if (this.heartbeatTimer) { - return; - } - - this.heartbeatTimer = setInterval(() => { - this.logHeartbeat(); - }, this.HEARTBEAT_INTERVAL_MS); - - if (typeof this.heartbeatTimer.unref === 'function') { - this.heartbeatTimer.unref(); - } - } - - static logHeartbeat() { - const activeJobs = Array.from(this.activeJobs.values()); - const lines = [ - '[OCL] OCL background status:', - ` active jobs: ${activeJobs.length}`, - ` queued jobs: ${this.pendingJobs.length}` - ]; - - activeJobs.forEach((job, index) => { - lines.push(''); - lines.push(` job ${index + 1}:`); - lines.push(` type: ${job.jobType || 'background-job'}`); - lines.push(` id: ${job.jobId || job.jobKey}`); - lines.push(` size: ${job.jobSize}`); - lines.push(` progress: ${this.formatProgress(job.getProgress)}`); - }); - - console.log(lines.join('\n')); - } - - static formatProgress(getProgress) { - if (typeof getProgress !== 'function') { - return 'unknown'; - } - - try { - const progress = getProgress(); - if (typeof progress === 'number' && Number.isFinite(progress)) { - const bounded = Math.max(0, Math.min(100, progress)); - return `${Math.round(bounded)}%`; - } - - if (progress && typeof progress === 'object') { - if (typeof progress.percentage === 'number' && Number.isFinite(progress.percentage)) { - const bounded = Math.max(0, Math.min(100, progress.percentage)); - return `${Math.round(bounded)}%`; - } - - if ( - typeof progress.processed === 'number' && - Number.isFinite(progress.processed) && - typeof progress.total === 'number' && - Number.isFinite(progress.total) && - progress.total > 0 - ) { - const ratio = progress.processed / progress.total; - const bounded = Math.max(0, Math.min(100, ratio * 100)); - return `${Math.round(bounded)}%`; - } - } - } catch (_error) { - return 'unknown'; - } - - return 'unknown'; - } - - static processNext() { - while (this.activeCount < this.MAX_CONCURRENT && this.pendingJobs.length > 0) { - const job = this.pendingJobs.shift(); - this.activeCount += 1; - this.activeJobs.set(job.jobKey, { - jobKey: job.jobKey, - jobType: job.jobType, - jobId: job.jobId || job.jobKey, - jobSize: job.jobSize, - getProgress: job.getProgress || null, - startedAt: Date.now() - }); - console.log(`[OCL] Background job started: ${job.jobType} ${job.jobKey} (size=${job.jobSize}, queue=${this.pendingJobs.length}, active=${this.activeCount})`); - - Promise.resolve() - .then(() => job.runJob()) - .then(() => { - console.log(`[OCL] Background job completed: ${job.jobType} ${job.jobKey}`); - }) - .catch((error) => { - const message = error && error.message ? error.message : String(error); - console.error(`[OCL] Background job failed: ${job.jobType} ${job.jobKey}: ${message}`); - }) - .finally(() => { - this.activeCount -= 1; - this.queuedOrRunningKeys.delete(job.jobKey); - this.activeJobs.delete(job.jobKey); - console.log(`[OCL] Background queue status: queue=${this.pendingJobs.length}, active=${this.activeCount}`); - this.processNext(); - }); - } - } -} - -module.exports = { - OCLBackgroundJobQueue -}; diff --git a/tx/ocl/mappers/concept-mapper.cjs b/tx/ocl/mappers/concept-mapper.cjs new file mode 100644 index 00000000..e350deb0 --- /dev/null +++ b/tx/ocl/mappers/concept-mapper.cjs @@ -0,0 +1,66 @@ +function toConceptContext(concept) { + if (!concept || typeof concept !== 'object') { + return null; + } + + const code = concept.code || concept.id || null; + if (!code) { + return null; + } + + return { + code, + display: concept.display_name || concept.display || concept.name || null, + definition: concept.description || concept.definition || null, + retired: concept.retired === true, + designations: extractDesignations(concept) + }; +} + +function extractDesignations(concept) { + const result = []; + const seen = new Set(); + + const add = (language, value) => { + const text = typeof value === 'string' ? value.trim() : ''; + if (!text) { + return; + } + + const lang = typeof language === 'string' ? language.trim() : ''; + const key = `${lang}|${text}`; + if (seen.has(key)) { + return; + } + + seen.add(key); + result.push({ language: lang, value: text }); + }; + + if (Array.isArray(concept.names)) { + for (const entry of concept.names) { + if (!entry || typeof entry !== 'object') { + continue; + } + + add(entry.locale || entry.language || entry.lang || '', entry.name || entry.display_name || entry.display || entry.value || entry.term); + } + } + + if (concept.display_name || concept.display || concept.name) { + add(concept.locale || concept.default_locale || concept.language || '', concept.display_name || concept.display || concept.name); + } + + if (concept.locale_display_names && typeof concept.locale_display_names === 'object') { + for (const [lang, value] of Object.entries(concept.locale_display_names)) { + add(lang, value); + } + } + + return result; +} + +module.exports = { + toConceptContext, + extractDesignations +}; diff --git a/tx/ocl/mappers/concept-mapper.js b/tx/ocl/mappers/concept-mapper.js index e350deb0..b0fc7908 100644 --- a/tx/ocl/mappers/concept-mapper.js +++ b/tx/ocl/mappers/concept-mapper.js @@ -1,66 +1,2 @@ -function toConceptContext(concept) { - if (!concept || typeof concept !== 'object') { - return null; - } +module.exports = require('./concept-mapper.cjs'); - const code = concept.code || concept.id || null; - if (!code) { - return null; - } - - return { - code, - display: concept.display_name || concept.display || concept.name || null, - definition: concept.description || concept.definition || null, - retired: concept.retired === true, - designations: extractDesignations(concept) - }; -} - -function extractDesignations(concept) { - const result = []; - const seen = new Set(); - - const add = (language, value) => { - const text = typeof value === 'string' ? value.trim() : ''; - if (!text) { - return; - } - - const lang = typeof language === 'string' ? language.trim() : ''; - const key = `${lang}|${text}`; - if (seen.has(key)) { - return; - } - - seen.add(key); - result.push({ language: lang, value: text }); - }; - - if (Array.isArray(concept.names)) { - for (const entry of concept.names) { - if (!entry || typeof entry !== 'object') { - continue; - } - - add(entry.locale || entry.language || entry.lang || '', entry.name || entry.display_name || entry.display || entry.value || entry.term); - } - } - - if (concept.display_name || concept.display || concept.name) { - add(concept.locale || concept.default_locale || concept.language || '', concept.display_name || concept.display || concept.name); - } - - if (concept.locale_display_names && typeof concept.locale_display_names === 'object') { - for (const [lang, value] of Object.entries(concept.locale_display_names)) { - add(lang, value); - } - } - - return result; -} - -module.exports = { - toConceptContext, - extractDesignations -}; diff --git a/tx/ocl/model/concept-filter-context.cjs b/tx/ocl/model/concept-filter-context.cjs new file mode 100644 index 00000000..330ac2a7 --- /dev/null +++ b/tx/ocl/model/concept-filter-context.cjs @@ -0,0 +1,51 @@ +class OCLConceptFilterContext { + constructor() { + this.concepts = []; + this.currentIndex = -1; + } + + add(concept, rating = 0) { + this.concepts.push({ concept, rating }); + } + + sort() { + this.concepts.sort((a, b) => b.rating - a.rating); + } + + size() { + return this.concepts.length; + } + + hasMore() { + return this.currentIndex + 1 < this.concepts.length; + } + + next() { + if (!this.hasMore()) { + return null; + } + this.currentIndex += 1; + return this.concepts[this.currentIndex].concept; + } + + reset() { + this.currentIndex = -1; + } + + findConceptByCode(code) { + for (const item of this.concepts) { + if (item.concept && item.concept.code === code) { + return item.concept; + } + } + return null; + } + + containsConcept(concept) { + return this.concepts.some(item => item.concept === concept); + } +} + +module.exports = { + OCLConceptFilterContext +}; diff --git a/tx/ocl/model/concept-filter-context.js b/tx/ocl/model/concept-filter-context.js index 330ac2a7..046dcd2b 100644 --- a/tx/ocl/model/concept-filter-context.js +++ b/tx/ocl/model/concept-filter-context.js @@ -1,51 +1,2 @@ -class OCLConceptFilterContext { - constructor() { - this.concepts = []; - this.currentIndex = -1; - } +module.exports = require('./concept-filter-context.cjs'); - add(concept, rating = 0) { - this.concepts.push({ concept, rating }); - } - - sort() { - this.concepts.sort((a, b) => b.rating - a.rating); - } - - size() { - return this.concepts.length; - } - - hasMore() { - return this.currentIndex + 1 < this.concepts.length; - } - - next() { - if (!this.hasMore()) { - return null; - } - this.currentIndex += 1; - return this.concepts[this.currentIndex].concept; - } - - reset() { - this.currentIndex = -1; - } - - findConceptByCode(code) { - for (const item of this.concepts) { - if (item.concept && item.concept.code === code) { - return item.concept; - } - } - return null; - } - - containsConcept(concept) { - return this.concepts.some(item => item.concept === concept); - } -} - -module.exports = { - OCLConceptFilterContext -}; diff --git a/tx/ocl/shared/constants.cjs b/tx/ocl/shared/constants.cjs new file mode 100644 index 00000000..5b338687 --- /dev/null +++ b/tx/ocl/shared/constants.cjs @@ -0,0 +1,15 @@ +const DEFAULT_BASE_URL = 'https://api.openconceptlab.org'; +const PAGE_SIZE = 100; +const CONCEPT_PAGE_SIZE = 1000; +const FILTERED_CONCEPT_PAGE_SIZE = 200; +const COLD_CACHE_FRESHNESS_MS = 60 * 60 * 1000; +const OCL_CODESYSTEM_MARKER_EXTENSION = 'http://fhir.org/FHIRsmith/StructureDefinition/ocl-codesystem'; + +module.exports = { + DEFAULT_BASE_URL, + PAGE_SIZE, + CONCEPT_PAGE_SIZE, + FILTERED_CONCEPT_PAGE_SIZE, + COLD_CACHE_FRESHNESS_MS, + OCL_CODESYSTEM_MARKER_EXTENSION +}; diff --git a/tx/ocl/shared/constants.js b/tx/ocl/shared/constants.js index 5b338687..9680d3d1 100644 --- a/tx/ocl/shared/constants.js +++ b/tx/ocl/shared/constants.js @@ -1,15 +1,2 @@ -const DEFAULT_BASE_URL = 'https://api.openconceptlab.org'; -const PAGE_SIZE = 100; -const CONCEPT_PAGE_SIZE = 1000; -const FILTERED_CONCEPT_PAGE_SIZE = 200; -const COLD_CACHE_FRESHNESS_MS = 60 * 60 * 1000; -const OCL_CODESYSTEM_MARKER_EXTENSION = 'http://fhir.org/FHIRsmith/StructureDefinition/ocl-codesystem'; +module.exports = require('./constants.cjs'); -module.exports = { - DEFAULT_BASE_URL, - PAGE_SIZE, - CONCEPT_PAGE_SIZE, - FILTERED_CONCEPT_PAGE_SIZE, - COLD_CACHE_FRESHNESS_MS, - OCL_CODESYSTEM_MARKER_EXTENSION -}; diff --git a/tx/ocl/shared/patches.cjs b/tx/ocl/shared/patches.cjs new file mode 100644 index 00000000..14a235ab --- /dev/null +++ b/tx/ocl/shared/patches.cjs @@ -0,0 +1,135 @@ +const { OCL_CODESYSTEM_MARKER_EXTENSION } = require('./constants'); + +const OCL_SEARCH_PATCH_FLAG = Symbol.for('fhirsmith.ocl.search.codesystem.code.patch'); +const TXPARAMS_HASH_PATCH_FLAG = Symbol.for('fhirsmith.ocl.txparameters.hash.filter.patch'); + +function hasOCLCodeSystemMarker(resource) { + const extensions = Array.isArray(resource?.extension) ? resource.extension : []; + return extensions.some(ext => ext && ext.url === OCL_CODESYSTEM_MARKER_EXTENSION); +} + +function filterConceptTreeByCode(concepts, wantedCode) { + if (!Array.isArray(concepts) || concepts.length === 0) { + return []; + } + + const matches = []; + for (const concept of concepts) { + if (!concept || typeof concept !== 'object') { + continue; + } + + const childMatches = filterConceptTreeByCode(concept.concept, wantedCode); + const isSelfMatch = concept.code != null && String(concept.code) === wantedCode; + if (!isSelfMatch && childMatches.length === 0) { + continue; + } + + const clone = { ...concept }; + if (childMatches.length > 0) { + clone.concept = childMatches; + } else { + delete clone.concept; + } + matches.push(clone); + } + + return matches; +} + +function filterOCLCodeSystemResourceByCode(resource, code) { + if (!resource || typeof resource !== 'object') { + return resource; + } + + const filteredConcepts = filterConceptTreeByCode(resource.concept, code); + return { + ...resource, + concept: filteredConcepts + }; +} + +function patchSearchWorkerForOCLCodeFiltering() { + let SearchWorker; + try { + SearchWorker = require('../../workers/search'); + } catch (_error) { + return; + } + + if (!SearchWorker || !SearchWorker.prototype) { + return; + } + + const proto = SearchWorker.prototype; + if (proto[OCL_SEARCH_PATCH_FLAG] === true || typeof proto.searchCodeSystems !== 'function') { + return; + } + + const originalSearchCodeSystems = proto.searchCodeSystems; + proto.searchCodeSystems = function patchedSearchCodeSystems(params) { + const matches = originalSearchCodeSystems.call(this, params); + const requestedCode = params?.code == null ? '' : String(params.code); + + if (!requestedCode) { + return matches; + } + + const filtered = []; + for (const resource of matches) { + if (!hasOCLCodeSystemMarker(resource)) { + filtered.push(resource); + continue; + } + + const projected = filterOCLCodeSystemResourceByCode(resource, requestedCode); + if (Array.isArray(projected?.concept) && projected.concept.length > 0) { + filtered.push(projected); + } + } + + return filtered; + }; + + Object.defineProperty(proto, OCL_SEARCH_PATCH_FLAG, { + value: true, + writable: false, + configurable: false, + enumerable: false + }); +} + +function normalizeFilterForCacheKey(filter) { + if (typeof filter !== 'string') { + return ''; + } + + return filter.trim().toLowerCase(); +} + +function ensureTxParametersHashIncludesFilter(TxParameters) { + const proto = TxParameters && TxParameters.prototype; + if (!proto || proto[TXPARAMS_HASH_PATCH_FLAG] === true || typeof proto.hashSource !== 'function') { + return; + } + + const originalHashSource = proto.hashSource; + proto.hashSource = function hashSourceWithFilter() { + const base = originalHashSource.call(this); + const normalizedFilter = normalizeFilterForCacheKey(this.filter); + return `${base}|filter=${normalizedFilter}`; + }; + + Object.defineProperty(proto, TXPARAMS_HASH_PATCH_FLAG, { + value: true, + writable: false, + configurable: false, + enumerable: false + }); +} + +module.exports = { + patchSearchWorkerForOCLCodeFiltering, + ensureTxParametersHashIncludesFilter, + normalizeFilterForCacheKey +}; diff --git a/tx/ocl/shared/patches.js b/tx/ocl/shared/patches.js index 14a235ab..1771e438 100644 --- a/tx/ocl/shared/patches.js +++ b/tx/ocl/shared/patches.js @@ -1,135 +1,2 @@ -const { OCL_CODESYSTEM_MARKER_EXTENSION } = require('./constants'); +module.exports = require('./patches.cjs'); -const OCL_SEARCH_PATCH_FLAG = Symbol.for('fhirsmith.ocl.search.codesystem.code.patch'); -const TXPARAMS_HASH_PATCH_FLAG = Symbol.for('fhirsmith.ocl.txparameters.hash.filter.patch'); - -function hasOCLCodeSystemMarker(resource) { - const extensions = Array.isArray(resource?.extension) ? resource.extension : []; - return extensions.some(ext => ext && ext.url === OCL_CODESYSTEM_MARKER_EXTENSION); -} - -function filterConceptTreeByCode(concepts, wantedCode) { - if (!Array.isArray(concepts) || concepts.length === 0) { - return []; - } - - const matches = []; - for (const concept of concepts) { - if (!concept || typeof concept !== 'object') { - continue; - } - - const childMatches = filterConceptTreeByCode(concept.concept, wantedCode); - const isSelfMatch = concept.code != null && String(concept.code) === wantedCode; - if (!isSelfMatch && childMatches.length === 0) { - continue; - } - - const clone = { ...concept }; - if (childMatches.length > 0) { - clone.concept = childMatches; - } else { - delete clone.concept; - } - matches.push(clone); - } - - return matches; -} - -function filterOCLCodeSystemResourceByCode(resource, code) { - if (!resource || typeof resource !== 'object') { - return resource; - } - - const filteredConcepts = filterConceptTreeByCode(resource.concept, code); - return { - ...resource, - concept: filteredConcepts - }; -} - -function patchSearchWorkerForOCLCodeFiltering() { - let SearchWorker; - try { - SearchWorker = require('../../workers/search'); - } catch (_error) { - return; - } - - if (!SearchWorker || !SearchWorker.prototype) { - return; - } - - const proto = SearchWorker.prototype; - if (proto[OCL_SEARCH_PATCH_FLAG] === true || typeof proto.searchCodeSystems !== 'function') { - return; - } - - const originalSearchCodeSystems = proto.searchCodeSystems; - proto.searchCodeSystems = function patchedSearchCodeSystems(params) { - const matches = originalSearchCodeSystems.call(this, params); - const requestedCode = params?.code == null ? '' : String(params.code); - - if (!requestedCode) { - return matches; - } - - const filtered = []; - for (const resource of matches) { - if (!hasOCLCodeSystemMarker(resource)) { - filtered.push(resource); - continue; - } - - const projected = filterOCLCodeSystemResourceByCode(resource, requestedCode); - if (Array.isArray(projected?.concept) && projected.concept.length > 0) { - filtered.push(projected); - } - } - - return filtered; - }; - - Object.defineProperty(proto, OCL_SEARCH_PATCH_FLAG, { - value: true, - writable: false, - configurable: false, - enumerable: false - }); -} - -function normalizeFilterForCacheKey(filter) { - if (typeof filter !== 'string') { - return ''; - } - - return filter.trim().toLowerCase(); -} - -function ensureTxParametersHashIncludesFilter(TxParameters) { - const proto = TxParameters && TxParameters.prototype; - if (!proto || proto[TXPARAMS_HASH_PATCH_FLAG] === true || typeof proto.hashSource !== 'function') { - return; - } - - const originalHashSource = proto.hashSource; - proto.hashSource = function hashSourceWithFilter() { - const base = originalHashSource.call(this); - const normalizedFilter = normalizeFilterForCacheKey(this.filter); - return `${base}|filter=${normalizedFilter}`; - }; - - Object.defineProperty(proto, TXPARAMS_HASH_PATCH_FLAG, { - value: true, - writable: false, - configurable: false, - enumerable: false - }); -} - -module.exports = { - patchSearchWorkerForOCLCodeFiltering, - ensureTxParametersHashIncludesFilter, - normalizeFilterForCacheKey -}; diff --git a/tx/ocl/vs-ocl.cjs b/tx/ocl/vs-ocl.cjs new file mode 100644 index 00000000..56682799 --- /dev/null +++ b/tx/ocl/vs-ocl.cjs @@ -0,0 +1,1760 @@ +const fs = require('fs/promises'); +const crypto = require('crypto'); +const path = require('path'); +const { AbstractValueSetProvider } = require('../vs/vs-api'); +const { VersionUtilities } = require('../../library/version-utilities'); +const ValueSet = require('../library/valueset'); +const { SearchFilterText } = require('../library/designations'); +const { TxParameters } = require('../params'); +const { OCLSourceCodeSystemFactory, OCLBackgroundJobQueue } = require('./cs-ocl'); +const { PAGE_SIZE, CONCEPT_PAGE_SIZE, FILTERED_CONCEPT_PAGE_SIZE, COLD_CACHE_FRESHNESS_MS } = require('./shared/constants'); +const { createOclHttpClient } = require('./http/client'); +const { CACHE_VS_DIR, getCacheFilePath } = require('./cache/cache-paths'); +const { ensureCacheDirectories, getColdCacheAgeMs, formatCacheAgeMinutes } = require('./cache/cache-utils'); +const { computeValueSetExpansionFingerprint } = require('./fingerprint/fingerprint'); +const { ensureTxParametersHashIncludesFilter } = require('./shared/patches'); + +ensureTxParametersHashIncludesFilter(TxParameters); + +class OCLValueSetProvider extends AbstractValueSetProvider { + constructor(config = {}) { + super(); + const options = typeof config === 'string' ? { baseUrl: config } : (config || {}); + + this.org = options.org || null; + const http = createOclHttpClient(options); + this.baseUrl = http.baseUrl; + this.httpClient = http.client; + + this.valueSetMap = new Map(); + this._idMap = new Map(); + this.collectionMeta = new Map(); + this.sourceCanonicalCache = new Map(); + this.collectionConceptPageCache = new Map(); + this.pendingCollectionConceptPageRequests = new Map(); + this.collectionSourcesCache = new Map(); + this.pendingCollectionSourcesRequests = new Map(); + this.pendingSourceCanonicalRequests = new Map(); + this.collectionByCanonicalCache = new Map(); + this.pendingCollectionByCanonicalRequests = new Map(); + this._composePromises = new Map(); + this.backgroundExpansionCache = new Map(); + this.backgroundExpansionProgress = new Map(); + this.valueSetFingerprints = new Map(); + this._initialized = false; + this._initializePromise = null; + this.sourcePackageCode = this.org + ? `ocl:${this.baseUrl}|org=${this.org}` + : `ocl:${this.baseUrl}`; + } + + async #loadColdCacheForValueSets() { + try { + const files = await fs.readdir(CACHE_VS_DIR); + let loadedCount = 0; + + for (const file of files) { + if (!file.endsWith('.json')) { + continue; + } + + try { + const filePath = path.join(CACHE_VS_DIR, file); + const data = await fs.readFile(filePath, 'utf-8'); + const cached = JSON.parse(data); + + if (!cached || !cached.canonicalUrl || !cached.expansion) { + continue; + } + + const paramsKey = cached.paramsKey || 'default'; + const cacheKey = this.#expansionCacheKey( + { url: cached.canonicalUrl, version: cached.version || null }, + paramsKey + ); + const createdAt = cached.timestamp ? new Date(cached.timestamp).getTime() : null; + this.backgroundExpansionCache.set(cacheKey, { + expansion: cached.expansion, + metadataSignature: cached.metadataSignature || null, + dependencyChecksums: cached.dependencyChecksums || {}, + createdAt: Number.isFinite(createdAt) ? createdAt : null + }); + + this.valueSetFingerprints.set(cacheKey, cached.fingerprint || null); + loadedCount++; + console.log(`[OCL-ValueSet] Loaded ValueSet from cold cache: ${cached.canonicalUrl}`); + } catch (error) { + console.error(`[OCL-ValueSet] Failed to load cold cache file ${file}:`, error.message); + } + } + + if (loadedCount > 0) { + console.log(`[OCL-ValueSet] Loaded ${loadedCount} ValueSet expansions from cold cache`); + } + } catch (error) { + if (error.code !== 'ENOENT') { + console.error('[OCL-ValueSet] Failed to load cold cache:', error.message); + } + } + } + + async #saveColdCacheForValueSet(vs, expansion, metadataSignature, dependencyChecksums, paramsKey = 'default') { + const canonicalUrl = vs?.url; + const version = vs?.version || null; + if (!canonicalUrl || !expansion) { + return null; + } + + const cacheFilePath = getCacheFilePath(CACHE_VS_DIR, canonicalUrl, version, paramsKey); + + try { + await ensureCacheDirectories(CACHE_VS_DIR); + + const fingerprint = computeValueSetExpansionFingerprint(expansion); + const cacheData = { + canonicalUrl, + version, + paramsKey, + fingerprint, + timestamp: new Date().toISOString(), + conceptCount: expansion.contains?.length || 0, + expansion, + metadataSignature, + dependencyChecksums + }; + + await fs.writeFile(cacheFilePath, JSON.stringify(cacheData, null, 2), 'utf-8'); + console.log(`[OCL-ValueSet] Saved ValueSet expansion to cold cache: ${canonicalUrl} (${expansion.contains?.length || 0} concepts, fingerprint=${fingerprint?.substring(0, 8)})`); + + return fingerprint; + } catch (error) { + console.error(`[OCL-ValueSet] Failed to save cold cache for ValueSet ${canonicalUrl}:`, error.message); + return null; + } + } + + sourcePackage() { + return this.sourcePackageCode; + } + + async initialize() { + if (this._initialized) { + return; + } + + if (this._initializePromise) { + await this._initializePromise; + return; + } + + this._initializePromise = (async () => { + try { + // Load cold cache first + await this.#loadColdCacheForValueSets(); + + const collections = await this.#fetchCollectionsForDiscovery(); + console.log(`[OCL-ValueSet] Fetched ${collections.length} collections`); + + for (const collection of collections) { + const valueSet = this.#toValueSet(collection); + if (!valueSet) { + continue; + } + this.#indexValueSet(valueSet); + } + + console.log(`[OCL-ValueSet] Loaded ${this.valueSetMap.size} value sets`); + this._initialized = true; + } catch (error) { + console.error(`[OCL-ValueSet] Initialization failed:`, error.message); + if (error.response) { + console.error(`[OCL-ValueSet] HTTP ${error.response.status}: ${error.response.statusText}`); + } + throw error; + } + })(); + + try { + await this._initializePromise; + } finally { + this._initializePromise = null; + } + } + + assignIds(ids) { + if (!this.spaceId) { + return; + } + + const unique = new Set(this.valueSetMap.values()); + this._idMap.clear(); + + for (const vs of unique) { + if (!vs.id.startsWith(`${this.spaceId}-`)) { + const nextId = `${this.spaceId}-${vs.id}`; + vs.id = nextId; + vs.jsonObj.id = nextId; + } + this._idMap.set(vs.id, vs); + ids.add(`ValueSet/${vs.id}`); + } + } + + async fetchValueSet(url, version) { + this._validateFetchParams(url, version); + + let key = `${url}|${version}`; + if (this.valueSetMap.has(key)) { + const vs = this.valueSetMap.get(key); + await this.#ensureComposeIncludes(vs); + this.#clearInlineExpansion(vs); + this.#scheduleBackgroundExpansion(vs, { reason: 'fetch-valueset' }); + return vs; + } + + if (version && VersionUtilities.isSemVer(version)) { + const majorMinor = VersionUtilities.getMajMin(version); + if (majorMinor) { + key = `${url}|${majorMinor}`; + if (this.valueSetMap.has(key)) { + const vs = this.valueSetMap.get(key); + await this.#ensureComposeIncludes(vs); + this.#clearInlineExpansion(vs); + this.#scheduleBackgroundExpansion(vs, { reason: 'fetch-valueset-mm' }); + return vs; + } + } + } + + if (this.valueSetMap.has(url)) { + const vs = this.valueSetMap.get(url); + await this.#ensureComposeIncludes(vs); + this.#clearInlineExpansion(vs); + this.#scheduleBackgroundExpansion(vs, { reason: 'fetch-valueset-url' }); + return vs; + } + + const resolved = await this.#resolveValueSetByCanonical(url, version); + if (resolved) { + await this.#ensureComposeIncludes(resolved); + this.#clearInlineExpansion(resolved); + this.#scheduleBackgroundExpansion(resolved, { reason: 'fetch-valueset-resolved' }); + return resolved; + } + + await this.initialize(); + + key = `${url}|${version}`; + if (this.valueSetMap.has(key)) { + const vs = this.valueSetMap.get(key); + await this.#ensureComposeIncludes(vs); + this.#clearInlineExpansion(vs); + this.#scheduleBackgroundExpansion(vs, { reason: 'fetch-valueset-init' }); + return vs; + } + + if (version && VersionUtilities.isSemVer(version)) { + const majorMinor = VersionUtilities.getMajMin(version); + if (majorMinor) { + key = `${url}|${majorMinor}`; + if (this.valueSetMap.has(key)) { + const vs = this.valueSetMap.get(key); + await this.#ensureComposeIncludes(vs); + this.#clearInlineExpansion(vs); + this.#scheduleBackgroundExpansion(vs, { reason: 'fetch-valueset-init-mm' }); + return vs; + } + } + } + + if (this.valueSetMap.has(url)) { + const vs = this.valueSetMap.get(url); + await this.#ensureComposeIncludes(vs); + this.#clearInlineExpansion(vs); + this.#scheduleBackgroundExpansion(vs, { reason: 'fetch-valueset-init-url' }); + return vs; + } + + return null; + } + + async fetchValueSetById(id) { + const local = this.#getLocalValueSetById(id); + if (local) { + await this.#ensureComposeIncludes(local); + this.#clearInlineExpansion(local); + this.#scheduleBackgroundExpansion(local, { reason: 'fetch-valueset-by-id' }); + return local; + } + + await this.initialize(); + + const vs = this.#getLocalValueSetById(id); + await this.#ensureComposeIncludes(vs); + this.#clearInlineExpansion(vs); + this.#scheduleBackgroundExpansion(vs, { reason: 'fetch-valueset-by-id-init' }); + return vs; + } + + #clearInlineExpansion(vs) { + if (!vs || !vs.jsonObj || !vs.jsonObj.expansion) { + return; + } + delete vs.jsonObj.expansion; + } + + #getLocalValueSetById(id) { + if (this._idMap.has(id)) { + return this._idMap.get(id); + } + + if (this.spaceId && id.startsWith(`${this.spaceId}-`)) { + const unprefixed = id.substring(this.spaceId.length + 1); + return this._idMap.get(id) || this._idMap.get(unprefixed) || this.valueSetMap.get(unprefixed) || null; + } + + return this._idMap.get(id) || this.valueSetMap.get(id) || null; + } + + // eslint-disable-next-line no-unused-vars + async searchValueSets(searchParams, _elements) { + await this.initialize(); + this._validateSearchParams(searchParams); + + const params = Object.fromEntries(searchParams.map(({ name, value }) => [name, String(value).toLowerCase()])); + const values = Array.from(new Set(this.valueSetMap.values())); + + if (Object.keys(params).length === 0) { + return values; + } + + return values.filter(vs => this.#matches(vs.jsonObj, params)); + } + + vsCount() { + return new Set(this.valueSetMap.values()).size; + } + + async listAllValueSets() { + await this.initialize(); + const urls = new Set(); + for (const vs of this.valueSetMap.values()) { + if (vs && vs.url) { + urls.add(vs.url); + } + } + return Array.from(urls); + } + + async close() { + } + + #indexValueSet(vs) { + const existing = this.valueSetMap.get(vs.url) + || (vs.version ? this.valueSetMap.get(`${vs.url}|${vs.version}`) : null) + || this._idMap.get(vs.id) + || null; + + // Preserve hydrated cold-cache expansions on first index; invalidate only on replacement. + if (existing && existing !== vs) { + this.#invalidateExpansionCache(vs); + } + + this.valueSetMap.set(vs.url, vs); + if (vs.version) { + this.valueSetMap.set(`${vs.url}|${vs.version}`, vs); + } + this.valueSetMap.set(vs.id, vs); + this._idMap.set(vs.id, vs); + } + + #toValueSet(collection) { + if (!collection || typeof collection !== 'object') { + return null; + } + + const canonicalUrl = collection.canonical_url || collection.canonicalUrl || collection.url; + const id = collection.id; + if (!canonicalUrl || !id) { + return null; + } + + const preferredSource = collection.preferred_source || collection.preferredSource || null; + const json = { + resourceType: 'ValueSet', + id, + url: canonicalUrl, + version: collection.version || null, + name: collection.name || id, + title: collection.full_name || collection.fullName || collection.name || id, + status: 'active', + experimental: collection.experimental === true, + immutable: collection.immutable === true, + description: collection.description || null, + publisher: collection.publisher || collection.owner || null, + language: collection.default_locale || collection.defaultLocale || null + }; + + const lastUpdated = this.#toIsoDate(collection.updated_on || collection.updatedOn || collection.updated_at || collection.updatedAt); + if (lastUpdated) { + json.meta = { lastUpdated }; + } + + if (preferredSource) { + json.compose = { + include: [{ system: preferredSource }] + }; + } + + const conceptsUrl = this.#normalizePath( + collection.concepts_url || collection.conceptsUrl || this.#buildCollectionConceptsPath(collection) + ); + const expansionUrl = this.#normalizePath( + collection.expansion_url || collection.expansionUrl || this.#buildCollectionExpansionPath(collection) + ); + + const meta = { + collectionId: collection.id || collection.short_code || collection.shortCode || null, + conceptsUrl, + expansionUrl, + preferredSource, + owner: collection.owner || null, + ownerType: collection.owner_type || collection.ownerType || null + }; + + this.#storeCollectionMeta(id, canonicalUrl, meta); + + const valueSet = new ValueSet(json, 'R5'); + this.#attachOclHelpers(valueSet, meta); + return valueSet; + } + + #attachOclHelpers(valueSet, meta) { + if (!valueSet || !meta) { + return; + } + + valueSet.oclMeta = meta; + valueSet.oclFetchConcepts = async ({ count, offset, activeOnly, filter, languageCodes }) => { + return this.#fetchCollectionConcepts(meta, { + count, + offset, + activeOnly, + filter: typeof filter === 'string' ? filter : null, + languageCodes: Array.isArray(languageCodes) ? languageCodes : [], + fallbackSystem: meta.preferredSource || valueSet.url + }); + }; + } + + #storeCollectionMeta(id, url, meta) { + if (!meta || (!meta.conceptsUrl && !meta.expansionUrl)) { + return; + } + if (id) { + this.collectionMeta.set(id, meta); + } + if (url) { + this.collectionMeta.set(url, meta); + } + } + + #normalizePath(pathValue) { + if (!pathValue) { + return null; + } + if (typeof pathValue !== 'string') { + return null; + } + if (pathValue.startsWith('http://') || pathValue.startsWith('https://')) { + return pathValue; + } + return `${this.baseUrl}${pathValue.startsWith('/') ? '' : '/'}${pathValue}`; + } + + #buildCollectionConceptsPath(collection) { + if (!collection || typeof collection !== 'object') { + return null; + } + const owner = collection.owner || null; + const ownerType = collection.owner_type || collection.ownerType || null; + const id = collection.id || collection.short_code || collection.shortCode || null; + if (!owner || !id || ownerType !== 'Organization') { + return null; + } + return `/orgs/${encodeURIComponent(owner)}/collections/${encodeURIComponent(id)}/concepts/`; + } + + #buildCollectionExpansionPath(collection) { + if (!collection || typeof collection !== 'object') { + return null; + } + const owner = collection.owner || null; + const ownerType = collection.owner_type || collection.ownerType || null; + const id = collection.id || collection.short_code || collection.shortCode || null; + if (!owner || !id || ownerType !== 'Organization') { + return null; + } + return `/orgs/${encodeURIComponent(owner)}/collections/${encodeURIComponent(id)}/HEAD/expansions/autoexpand-HEAD/`; + } + + #getCollectionMeta(vs) { + if (!vs) { + return null; + } + return this.collectionMeta.get(vs.id) || this.collectionMeta.get(vs.url) || null; + } + + async #ensureComposeIncludes(vs) { + if (!vs || !vs.jsonObj) { + return; + } + if (vs.jsonObj.compose && Array.isArray(vs.jsonObj.compose.include) && vs.jsonObj.compose.include.length > 0) { + return; + } + + const meta = this.#getCollectionMeta(vs); + if (!meta || !meta.conceptsUrl) { + return; + } + + const composeKey = vs.id || vs.url; + if (this._composePromises.has(composeKey)) { + await this._composePromises.get(composeKey); + return; + } + + const promise = (async () => { + const sources = await this.#fetchCollectionSources(meta); + if (!sources || sources.length === 0) { + return; + } + + vs.jsonObj.compose = { + include: sources.map(source => ({ + system: source.system, + version: source.version || undefined + })) + }; + })(); + + this._composePromises.set(composeKey, promise); + try { + await promise; + } finally { + this._composePromises.delete(composeKey); + } + } + + async #fetchCollectionSources(meta) { + const sourcesCacheKey = `${meta.owner || ''}|${meta.collectionId || ''}|${meta.conceptsUrl || ''}|${meta.expansionUrl || ''}`; + if (this.collectionSourcesCache.has(sourcesCacheKey)) { + return this.collectionSourcesCache.get(sourcesCacheKey); + } + if (this.pendingCollectionSourcesRequests.has(sourcesCacheKey)) { + return this.pendingCollectionSourcesRequests.get(sourcesCacheKey); + } + + const pending = (async () => { + const sources = []; + const seen = new Set(); + + if (meta.expansionUrl) { + try { + const response = await this.httpClient.get(meta.expansionUrl); + const resolved = Array.isArray(response.data?.resolved_source_versions) + ? response.data.resolved_source_versions + : []; + + for (const entry of resolved) { + const system = entry.canonical_url || entry.canonicalUrl || null; + const owner = entry.owner || meta.owner || null; + const shortCode = entry.short_code || entry.shortCode || entry.id || null; + const version = entry.version || null; + + const systemUrl = system || (owner && shortCode ? await this.#getSourceCanonicalUrl(owner, shortCode) : null); + if (systemUrl && !seen.has(systemUrl)) { + seen.add(systemUrl); + sources.push({ system: systemUrl, version }); + } + } + } catch (error) { + // fall through to concepts listing + } + } + + if (sources.length > 0) { + return sources; + } + + if (!meta.conceptsUrl) { + return sources; + } + + const sourceKeys = await this.#fetchCollectionSourceKeys(meta.conceptsUrl, meta.owner || null); + for (const { owner, source } of sourceKeys) { + const systemUrl = await this.#getSourceCanonicalUrl(owner, source); + if (systemUrl && !seen.has(systemUrl)) { + seen.add(systemUrl); + sources.push({ system: systemUrl }); + } + } + + if (sources.length === 0 && meta.preferredSource) { + sources.push({ system: meta.preferredSource }); + } + + this.collectionSourcesCache.set(sourcesCacheKey, sources); + return sources; + })(); + + this.pendingCollectionSourcesRequests.set(sourcesCacheKey, pending); + try { + return await pending; + } finally { + this.pendingCollectionSourcesRequests.delete(sourcesCacheKey); + } + } + + async #fetchCollectionConcepts(meta, options) { + if (!meta || !meta.conceptsUrl) { + return { contains: [], total: 0 }; + } + + const count = Number.isInteger(options?.count) ? options.count : CONCEPT_PAGE_SIZE; + const offset = Number.isInteger(options?.offset) ? options.offset : 0; + const activeOnly = options?.activeOnly === true; + const filter = this.#normalizeFilter(options?.filter); + const filterMatcher = filter ? new SearchFilterText(filter) : null; + const remoteQuery = this.#buildRemoteQuery(filter); + const fallbackSystem = options?.fallbackSystem || null; + const preferredLanguageCodes = this.#normalizeLanguageCodes(options?.languageCodes); + const effectiveLanguageCodes = preferredLanguageCodes.length > 0 ? preferredLanguageCodes : ['en']; + + if (count <= 0) { + return { contains: [], total: 0 }; + } + + const hasFilter = !!filter; + const limit = hasFilter + ? Math.min(FILTERED_CONCEPT_PAGE_SIZE, CONCEPT_PAGE_SIZE) + : CONCEPT_PAGE_SIZE; + let page = Math.floor(Math.max(0, offset) / limit) + 1; + let skip = Math.max(0, offset) % limit; + let remaining = count; + const contains = []; + let reportedTotal = null; + + while (remaining > 0) { + const pageData = await this.#fetchConceptPage(meta.conceptsUrl, page, limit, remoteQuery); + const pageItems = Array.isArray(pageData?.items) ? pageData.items : []; + if ( + reportedTotal == null && + typeof pageData?.reportedTotal === 'number' && + Number.isFinite(pageData.reportedTotal) && + pageData.reportedTotal >= 0 + ) { + reportedTotal = pageData.reportedTotal; + } + + if (!pageItems || pageItems.length === 0) { + break; + } + + const slice = pageItems.slice(skip); + skip = 0; + + for (const concept of slice) { + if (remaining <= 0) { + break; + } + if (activeOnly && concept.retired === true) { + continue; + } + + const localizedNames = this.#extractLocalizedNames(concept, effectiveLanguageCodes); + const localizedDefinitions = this.#extractLocalizedDefinitions(concept, effectiveLanguageCodes); + + const display = localizedNames.display || concept.display_name || concept.display || concept.name || null; + const definition = localizedDefinitions.definition || concept.definition || concept.description || concept.concept_class || null; + const code = concept.code || concept.id || null; + const searchableText = [ + code, + display, + definition, + ...localizedNames.designation.map(d => d.value), + ...localizedDefinitions.definitions.map(d => d.value) + ].filter(Boolean).join(' '); + if (!this.#conceptMatchesFilter(searchableText, code, display, definition, filter, filterMatcher)) { + continue; + } + + if (!code) { + continue; + } + + const owner = concept.owner || meta.owner || null; + const source = concept.source || null; + const conceptCanonical = concept.source_canonical_url || concept.sourceCanonicalUrl || null; + const system = conceptCanonical || (owner && source + ? await this.#getSourceCanonicalUrl(owner, source) + : fallbackSystem); + + contains.push({ + system: system || fallbackSystem, + code, + display, + definition: definition || undefined, + designation: localizedNames.designation, + definitions: localizedDefinitions.definitions, + inactive: concept.retired === true ? true : undefined + }); + remaining -= 1; + } + + if (pageItems.length < limit) { + break; + } + + page += 1; + } + + return { contains, total: contains.length, reportedTotal }; + } + + async #resolveValueSetByCanonical(url, version) { + const canonicalUrl = typeof url === 'string' ? url.trim() : ''; + if (!canonicalUrl) { + return null; + } + + const collection = await this.#findCollectionByCanonical(canonicalUrl, version); + if (!collection) { + return null; + } + + const valueSet = this.#toValueSet(collection); + if (!valueSet) { + return null; + } + + this.#indexValueSet(valueSet); + return valueSet; + } + + #valueSetBaseKey(vs) { + if (!vs || !vs.url) { + return null; + } + return `${vs.url}|${vs.version || ''}`; + } + + #expansionParamsKey(params) { + if (!params || typeof params !== 'object') { + return 'default'; + } + + try { + const normalized = Object.keys(params) + .sort() + .reduce((acc, key) => { + if (key === 'tx-resource' || key === 'valueSet') { + return acc; + } + acc[key] = params[key]; + return acc; + }, {}); + + const json = JSON.stringify(normalized); + if (!json || json === '{}') { + return 'default'; + } + return crypto.createHash('sha256').update(json).digest('hex').substring(0, 16); + } catch (error) { + return 'default'; + } + } + + #expansionCacheKey(vs, paramsKey) { + const base = this.#valueSetBaseKey(vs); + if (!base) { + return null; + } + return `${base}|${paramsKey || 'default'}`; + } + + #invalidateExpansionCache(vs) { + const base = this.#valueSetBaseKey(vs); + if (!base) { + return; + } + + for (const key of this.backgroundExpansionCache.keys()) { + if (key.startsWith(`${base}|`)) { + this.backgroundExpansionCache.delete(key); + } + } + } + + #applyCachedExpansion(vs, paramsKey) { + if (!vs || !vs.jsonObj) { + return; + } + + const cacheKey = this.#expansionCacheKey(vs, paramsKey); + if (!cacheKey) { + return; + } + + const cached = this.backgroundExpansionCache.get(cacheKey); + if (!cached || !cached.expansion) { + return; + } + + if (!this.#isCachedExpansionValid(vs, cached)) { + this.backgroundExpansionCache.delete(cacheKey); + if (vs.jsonObj.expansion) { + delete vs.jsonObj.expansion; + } + console.log(`[OCL-ValueSet] Cached ValueSet expansion invalidated: ${cacheKey}`); + return; + } + + if (vs.jsonObj.expansion) { + return; + } + + vs.jsonObj.expansion = structuredClone(cached.expansion); + console.log(`[OCL-ValueSet] ValueSet expansion restored from cache: ${cacheKey}`); + } + + #scheduleBackgroundExpansion(vs, options = {}) { + if (!vs || !vs.jsonObj) { + return; + } + + const paramsKey = this.#expansionParamsKey(options.params || null); + const cacheKey = this.#expansionCacheKey(vs, paramsKey); + if (!cacheKey) { + return; + } + + const cached = this.backgroundExpansionCache.get(cacheKey); + if (cached && !this.#isCachedExpansionValid(vs, cached)) { + this.backgroundExpansionCache.delete(cacheKey); + if (vs.jsonObj.expansion) { + delete vs.jsonObj.expansion; + } + console.log(`[OCL-ValueSet] Cached ValueSet expansion invalidated: ${cacheKey}`); + } + + if (vs.jsonObj.expansion) { + return; + } + + const cacheFilePath = getCacheFilePath(CACHE_VS_DIR, vs.url, vs.version || null, paramsKey); + const cacheAgeFromFileMs = getColdCacheAgeMs(cacheFilePath); + const persistedCache = this.backgroundExpansionCache.get(cacheKey); + const cacheAgeFromMetadataMs = Number.isFinite(persistedCache?.createdAt) + ? Math.max(0, Date.now() - persistedCache.createdAt) + : null; + + // Treat cache as fresh when either file mtime or persisted timestamp is recent. + const freshnessCandidates = [cacheAgeFromFileMs, cacheAgeFromMetadataMs].filter(age => age != null); + const freshestCacheAgeMs = freshnessCandidates.length > 0 ? Math.min(...freshnessCandidates) : null; + if (freshestCacheAgeMs != null && freshestCacheAgeMs <= COLD_CACHE_FRESHNESS_MS) { + const freshnessSource = cacheAgeFromFileMs != null && cacheAgeFromMetadataMs != null + ? 'file+metadata' + : cacheAgeFromFileMs != null + ? 'file' + : 'metadata'; + console.log(`[OCL-ValueSet] Skipping warm-up for ValueSet ${vs.url} (cold cache age: ${formatCacheAgeMinutes(freshestCacheAgeMs)})`); + console.log(`[OCL-ValueSet] ValueSet cold cache is fresh, not enqueueing warm-up job (${cacheKey}, source=${freshnessSource})`); + return; + } + + const jobKey = `vs:${cacheKey}`; + if (OCLBackgroundJobQueue.isQueuedOrRunning(jobKey)) { + console.log(`[OCL-ValueSet] ValueSet expansion already queued or running: ${cacheKey}`); + return; + } + + let queuedJobSize = null; + const warmupAgeText = freshestCacheAgeMs != null + ? formatCacheAgeMinutes(freshestCacheAgeMs) + : 'no cold cache'; + console.log(`[OCL-ValueSet] Enqueueing warm-up for ValueSet ${vs.url} (cold cache age: ${warmupAgeText})`); + console.log(`[OCL-ValueSet] ValueSet expansion enqueued: ${cacheKey}`); + OCLBackgroundJobQueue.enqueue( + jobKey, + 'ValueSet expansion', + async () => { + await this.#runBackgroundExpansion(vs, cacheKey, paramsKey, queuedJobSize); + }, + { + jobId: vs.url || cacheKey, + getProgress: () => this.#backgroundExpansionProgressSnapshot(cacheKey), + resolveJobSize: async () => { + const meta = this.#getCollectionMeta(vs); + queuedJobSize = await this.#fetchConceptCountFromHeaders(meta?.conceptsUrl || null); + return queuedJobSize; + } + } + ); + } + + async #runBackgroundExpansion(vs, cacheKey, paramsKey = 'default', knownConceptCount = null) { + console.log(`[OCL-ValueSet] ValueSet expansion started: ${cacheKey}`); + const progressState = { processed: 0, total: null }; + this.backgroundExpansionProgress.set(cacheKey, progressState); + try { + await this.#ensureComposeIncludes(vs); + + const meta = this.#getCollectionMeta(vs); + const resolvedTotal = Number.isFinite(knownConceptCount) && knownConceptCount >= 0 + ? knownConceptCount + : await this.#fetchConceptCountFromHeaders(meta?.conceptsUrl || null); + progressState.total = resolvedTotal; + const sources = meta ? await this.#fetchCollectionSources(meta) : []; + for (const source of sources || []) { + OCLSourceCodeSystemFactory.scheduleBackgroundLoadByKey( + source.system, + source.version || null, + 'valueset-expansion' + ); + } + + const expansion = await this.#buildBackgroundExpansion(vs, progressState); + if (!expansion) { + return; + } + + progressState.processed = expansion.total || progressState.processed; + if (typeof progressState.total !== 'number' || !Number.isFinite(progressState.total) || progressState.total <= 0) { + progressState.total = expansion.total || 0; + } + + const metadataSignature = this.#valueSetMetadataSignature(vs); + const dependencyChecksums = this.#valueSetDependencyChecksums(vs); + + // Compute custom fingerprint and compare with cold cache + const newFingerprint = computeValueSetExpansionFingerprint(expansion); + const oldFingerprint = this.valueSetFingerprints.get(cacheKey); + + if (oldFingerprint && newFingerprint === oldFingerprint) { + console.log(`[OCL-ValueSet] ValueSet expansion fingerprint unchanged: ${cacheKey} (fingerprint=${newFingerprint?.substring(0, 8)})`); + } else { + if (oldFingerprint) { + console.log(`[OCL-ValueSet] ValueSet expansion fingerprint changed: ${cacheKey} (${oldFingerprint?.substring(0, 8)} -> ${newFingerprint?.substring(0, 8)})`); + console.log(`[OCL-ValueSet] Replacing cold cache with new hot cache: ${cacheKey}`); + } else { + console.log(`[OCL-ValueSet] Computed fingerprint for ValueSet expansion: ${cacheKey} (fingerprint=${newFingerprint?.substring(0, 8)})`); + } + + // Save to cold cache + const savedFingerprint = await this.#saveColdCacheForValueSet(vs, expansion, metadataSignature, dependencyChecksums, paramsKey); + if (savedFingerprint) { + this.valueSetFingerprints.set(cacheKey, savedFingerprint); + } + } + + this.backgroundExpansionCache.set(cacheKey, { + expansion, + metadataSignature, + dependencyChecksums, + createdAt: Date.now() + }); + // Keep expansions in provider-managed cache only. + // Inline expansion on ValueSet bypasses $expand filtering in worker pipeline. + + console.log(`[OCL-ValueSet] ValueSet expansion completed and cached: ${cacheKey}`); + console.log(`[OCL-ValueSet] ValueSet now available in cache: ${cacheKey}`); + } catch (error) { + console.error(`[OCL-ValueSet] ValueSet background expansion failed: ${cacheKey}: ${error.message}`); + } finally { + this.backgroundExpansionProgress.delete(cacheKey); + } + } + + async #buildBackgroundExpansion(vs, progressState = null) { + const meta = this.#getCollectionMeta(vs); + if (!meta || !meta.conceptsUrl) { + return null; + } + + const contains = []; + let offset = 0; + + // Pull all concepts in fixed-size pages until exhausted. + // eslint-disable-next-line no-constant-condition + while (true) { + const batch = await this.#fetchCollectionConcepts(meta, { + count: CONCEPT_PAGE_SIZE, + offset, + activeOnly: false, + filter: null, + languageCodes: [] + }); + + const entries = Array.isArray(batch?.contains) ? batch.contains : []; + if (entries.length === 0) { + break; + } + + for (const entry of entries) { + if (!entry?.system || !entry?.code) { + continue; + } + + const out = { + system: entry.system, + code: entry.code + }; + if (entry.display) { + out.display = entry.display; + } + if (entry.definition) { + out.definition = entry.definition; + } + if (entry.inactive === true) { + out.inactive = true; + } + if (Array.isArray(entry.designation) && entry.designation.length > 0) { + out.designation = entry.designation + .filter(d => d && d.value) + .map(d => ({ + language: d.language, + value: d.value + })); + } + contains.push(out); + } + + if (progressState) { + progressState.processed = contains.length; + } + + if (entries.length < CONCEPT_PAGE_SIZE) { + break; + } + offset += entries.length; + } + + return { + timestamp: new Date().toISOString(), + identifier: `urn:uuid:${crypto.randomUUID()}`, + total: contains.length, + contains + }; + } + + async #findCollectionByCanonical(canonicalUrl, version) { + const lookupKey = `${this.org || '*'}|${canonicalUrl}|${version || ''}`; + if (this.collectionByCanonicalCache.has(lookupKey)) { + return this.collectionByCanonicalCache.get(lookupKey); + } + if (this.pendingCollectionByCanonicalRequests.has(lookupKey)) { + return this.pendingCollectionByCanonicalRequests.get(lookupKey); + } + + const token = this.#canonicalToken(canonicalUrl); + if (!token) { + return null; + } + + const pending = (async () => { + const organizations = await this.#fetchOrganizationIds(); + const endpoints = organizations.length > 0 + ? organizations.map(orgId => `/orgs/${encodeURIComponent(orgId)}/collections/`) + : ['/collections/']; + + const exactMatches = []; + for (const endpoint of endpoints) { + const response = await this.httpClient.get(endpoint, { + params: { + q: token, + page: 1, + limit: PAGE_SIZE + } + }); + + const payload = response.data; + const items = Array.isArray(payload) + ? payload + : Array.isArray(payload?.results) + ? payload.results + : Array.isArray(payload?.items) + ? payload.items + : Array.isArray(payload?.data) + ? payload.data + : []; + + for (const item of items) { + const itemCanonical = item?.canonical_url || item?.canonicalUrl || item?.url || null; + if (itemCanonical === canonicalUrl) { + exactMatches.push(item); + } + } + } + + let match = null; + if (exactMatches.length > 0) { + if (!version) { + match = exactMatches[0]; + } else { + const exactVersion = exactMatches.find(item => (item.version || null) === version); + if (exactVersion) { + match = exactVersion; + } else if (VersionUtilities.isSemVer(version)) { + const majorMinor = VersionUtilities.getMajMin(version); + if (majorMinor) { + const majorMinorMatch = exactMatches.find(item => (item.version || null) === majorMinor); + if (majorMinorMatch) { + match = majorMinorMatch; + } + } + } + } + } + + this.collectionByCanonicalCache.set(lookupKey, match); + return match; + })(); + + this.pendingCollectionByCanonicalRequests.set(lookupKey, pending); + try { + return await pending; + } finally { + this.pendingCollectionByCanonicalRequests.delete(lookupKey); + } + } + + #valueSetMetadataSignature(vs) { + const meta = this.#getCollectionMeta(vs); + const payload = { + url: vs?.url || null, + version: vs?.version || null, + lastUpdated: vs?.jsonObj?.meta?.lastUpdated || null, + collectionId: meta?.collectionId || null, + conceptsUrl: meta?.conceptsUrl || null, + expansionUrl: meta?.expansionUrl || null, + preferredSource: meta?.preferredSource || null + }; + return JSON.stringify(payload); + } + + #valueSetDependencyChecksums(vs) { + const include = Array.isArray(vs?.jsonObj?.compose?.include) ? vs.jsonObj.compose.include : []; + const checksums = {}; + for (const item of include) { + const system = item?.system || null; + if (!system) { + continue; + } + const version = item?.version || null; + const key = `${system}|${version || ''}`; + checksums[key] = OCLSourceCodeSystemFactory.checksumForResource(system, version); + } + return checksums; + } + + #isCachedExpansionValid(vs, cached) { + if (!cached || typeof cached !== 'object') { + return false; + } + + if (cached.metadataSignature !== this.#valueSetMetadataSignature(vs)) { + return false; + } + + const currentDeps = this.#valueSetDependencyChecksums(vs); + const cachedDeps = cached.dependencyChecksums || {}; + const currentKeys = Object.keys(currentDeps).sort(); + const cachedKeys = Object.keys(cachedDeps).sort(); + + if (currentKeys.length !== cachedKeys.length) { + return false; + } + + for (let i = 0; i < currentKeys.length; i++) { + if (currentKeys[i] !== cachedKeys[i]) { + return false; + } + if ((currentDeps[currentKeys[i]] || null) !== (cachedDeps[cachedKeys[i]] || null)) { + return false; + } + } + + return true; + } + + async #fetchCollectionsForDiscovery() { + const organizations = await this.#fetchOrganizationIds(); + if (organizations.length === 0) { + // Fallback for OCL instances that expose global listing but not org listing. + return await this.#fetchAllPages('/collections/'); + } + + const allCollections = []; + const seen = new Set(); + + for (const orgId of organizations) { + const endpoint = `/orgs/${encodeURIComponent(orgId)}/collections/`; + const collections = await this.#fetchAllPages(endpoint); + for (const collection of collections) { + if (!collection || typeof collection !== 'object') { + continue; + } + const key = this.#collectionIdentity(collection); + if (seen.has(key)) { + continue; + } + seen.add(key); + allCollections.push(collection); + } + } + + return allCollections; + } + + async #fetchOrganizationIds() { + const endpoint = '/orgs/'; + console.log(`[OCL-ValueSet] Loading organizations from: ${this.baseUrl}${endpoint}`); + const orgs = await this.#fetchAllPages(endpoint); + + const ids = []; + const seen = new Set(); + for (const org of orgs || []) { + if (!org || typeof org !== 'object') { + continue; + } + + const id = org.id || org.mnemonic || org.short_code || org.shortCode || org.name || null; + if (!id) { + continue; + } + + const normalized = String(id).trim(); + if (!normalized || seen.has(normalized)) { + continue; + } + + seen.add(normalized); + ids.push(normalized); + } + + if (ids.length === 0 && this.org) { + ids.push(this.org); + } + + return ids; + } + + #collectionIdentity(collection) { + if (!collection || typeof collection !== 'object') { + return '__invalid__'; + } + + const owner = collection.owner || ''; + const canonical = collection.canonical_url || collection.canonicalUrl || ''; + const id = collection.id || collection.short_code || collection.shortCode || collection.name || ''; + return `${owner}|${canonical}|${id}`; + } + + #canonicalToken(canonicalUrl) { + if (!canonicalUrl || typeof canonicalUrl !== 'string') { + return null; + } + + const trimmed = canonicalUrl.trim(); + if (!trimmed) { + return null; + } + + const parts = trimmed.replace(/\/+$/, '').split('/').filter(Boolean); + if (parts.length === 0) { + return null; + } + + return parts[parts.length - 1]; + } + + async #fetchConceptPage(conceptsUrl, page, limit, filter = null) { + try { + const cacheKey = `${conceptsUrl}|p=${page}|l=${limit}|q=${filter || ''}|verbose=1`; + if (this.collectionConceptPageCache.has(cacheKey)) { + const cached = this.collectionConceptPageCache.get(cacheKey); + const items = Array.isArray(cached) + ? cached + : Array.isArray(cached?.items) + ? cached.items + : []; + return { + items, + reportedTotal: this.#extractTotalFromPayload(cached?.payload || null) + }; + } + if (this.pendingCollectionConceptPageRequests.has(cacheKey)) { + return this.pendingCollectionConceptPageRequests.get(cacheKey); + } + + const pending = (async () => { + const params = { page, limit, verbose: true }; + if (filter) { + params.q = filter; + } + const response = await this.httpClient.get(conceptsUrl, { params }); + const payload = response.data; + const items = Array.isArray(payload) + ? payload + : Array.isArray(payload?.results) + ? payload.results + : Array.isArray(payload?.items) + ? payload.items + : Array.isArray(payload?.data) + ? payload.data + : []; + + this.collectionConceptPageCache.set(cacheKey, { items, payload }); + return { + items, + reportedTotal: this.#extractTotalFromPayload(payload) + }; + })(); + + this.pendingCollectionConceptPageRequests.set(cacheKey, pending); + try { + return await pending; + } finally { + this.pendingCollectionConceptPageRequests.delete(cacheKey); + } + } catch (error) { + console.error(`[OCL-ValueSet] #fetchConceptPage ERROR: ${error.message}`); + throw error; + } + } + + #buildRemoteQuery(filter) { + if (!filter || typeof filter !== 'string') { + return null; + } + + const tokens = filter + .split(/\s+or\s+|\||&|\s+/i) + .map(t => t.trim()) + .filter(Boolean) + .map(t => (t.startsWith('-') || t.startsWith('!')) ? t.substring(1) : t) + .map(t => t.replace(/[%*?]/g, '')) + .map(t => t.replace(/[^\p{L}\p{N}]+/gu, '')) + .filter(t => t.length >= 3); + + if (tokens.length === 0) { + return null; + } + + tokens.sort((a, b) => b.length - a.length); + return tokens[0]; + } + + #normalizeLanguageCodes(languageCodes) { + if (!Array.isArray(languageCodes)) { + return []; + } + + const normalized = []; + for (const code of languageCodes) { + if (!code || typeof code !== 'string') { + continue; + } + normalized.push(code.toLowerCase()); + } + return normalized; + } + + #backgroundExpansionProgressSnapshot(cacheKey) { + const progress = this.backgroundExpansionProgress.get(cacheKey); + if (!progress) { + return null; + } + + const processed = progress.processed; + const total = progress.total; + if ( + typeof processed === 'number' && + Number.isFinite(processed) && + typeof total === 'number' && + Number.isFinite(total) && + total > 0 + ) { + return { processed, total }; + } + + return null; + } + + #extractTotalFromPayload(payload) { + if (!payload || typeof payload !== 'object' || Array.isArray(payload)) { + return null; + } + + const candidates = [ + payload.total, + payload.total_count, + payload.totalCount, + payload.num_found, + payload.numFound, + payload.count + ]; + + for (const candidate of candidates) { + if (typeof candidate === 'number' && Number.isFinite(candidate) && candidate >= 0) { + return candidate; + } + } + + return null; + } + + async #fetchConceptCountFromHeaders(conceptsUrl) { + if (!conceptsUrl) { + return null; + } + + try { + const response = await this.httpClient.get(conceptsUrl, { + params: { + limit: 1 + } + }); + return this.#extractNumFoundFromHeaders(response?.headers); + } catch (error) { + return null; + } + } + + #extractNumFoundFromHeaders(headers) { + if (!headers || typeof headers !== 'object') { + return null; + } + + const raw = headers.num_found ?? headers['num-found'] ?? headers.Num_Found ?? null; + const parsed = typeof raw === 'number' ? raw : Number.parseInt(raw, 10); + if (!Number.isFinite(parsed) || parsed < 0) { + return null; + } + return parsed; + } + + #languageRank(languageCode, preferredLanguageCodes) { + if (!languageCode) { + return 1000; + } + + const normalized = String(languageCode).toLowerCase(); + for (let i = 0; i < preferredLanguageCodes.length; i++) { + const preferred = preferredLanguageCodes[i]; + if (normalized === preferred || normalized.startsWith(`${preferred}-`) || preferred.startsWith(`${normalized}-`)) { + return i; + } + } + + if (normalized === 'en' || normalized.startsWith('en-')) { + return preferredLanguageCodes.length; + } + + return preferredLanguageCodes.length + 1; + } + + #extractLocalizedNames(concept, preferredLanguageCodes) { + const names = Array.isArray(concept?.names) ? concept.names : []; + const unique = new Map(); + + for (const item of names) { + const value = item?.name; + if (!value || typeof value !== 'string') { + continue; + } + + const language = typeof item?.locale === 'string' && item.locale.trim() ? item.locale.trim().toLowerCase() : null; + const key = `${language || ''}|${value}`; + if (unique.has(key)) { + continue; + } + + unique.set(key, { + language: language || undefined, + value, + localePreferred: item?.locale_preferred === true, + nameType: item?.name_type || '' + }); + } + + const designation = Array.from(unique.values()) + .sort((a, b) => { + const rankDiff = this.#languageRank(a.language, preferredLanguageCodes) - this.#languageRank(b.language, preferredLanguageCodes); + if (rankDiff !== 0) { + return rankDiff; + } + if (a.localePreferred !== b.localePreferred) { + return a.localePreferred ? -1 : 1; + } + const aFs = /fully\s*-?\s*specified/i.test(a.nameType); + const bFs = /fully\s*-?\s*specified/i.test(b.nameType); + if (aFs !== bFs) { + return aFs ? -1 : 1; + } + return a.value.localeCompare(b.value); + }) + .map(({ language, value }) => ({ + language, + value + })); + + return { + display: designation.length > 0 ? designation[0].value : null, + designation + }; + } + + #extractLocalizedDefinitions(concept, preferredLanguageCodes) { + const descriptions = Array.isArray(concept?.descriptions) ? concept.descriptions : []; + const unique = new Map(); + + for (const item of descriptions) { + const value = item?.description; + if (!value || typeof value !== 'string') { + continue; + } + + const language = typeof item?.locale === 'string' && item.locale.trim() ? item.locale.trim().toLowerCase() : null; + const key = `${language || ''}|${value}`; + if (unique.has(key)) { + continue; + } + + unique.set(key, { + language: language || undefined, + value, + localePreferred: item?.locale_preferred === true, + descriptionType: item?.description_type || '' + }); + } + + const definitions = Array.from(unique.values()) + .sort((a, b) => { + const rankDiff = this.#languageRank(a.language, preferredLanguageCodes) - this.#languageRank(b.language, preferredLanguageCodes); + if (rankDiff !== 0) { + return rankDiff; + } + if (a.localePreferred !== b.localePreferred) { + return a.localePreferred ? -1 : 1; + } + const aDef = /definition/i.test(a.descriptionType); + const bDef = /definition/i.test(b.descriptionType); + if (aDef !== bDef) { + return aDef ? -1 : 1; + } + return a.value.localeCompare(b.value); + }) + .map(({ language, value }) => ({ + language, + value + })); + + return { + definition: definitions.length > 0 ? definitions[0].value : null, + definitions + }; + } + + async #fetchCollectionSourceKeys(conceptsUrl, defaultOwner) { + const keys = new Map(); + let page = 1; + + // eslint-disable-next-line no-constant-condition + while (true) { + const response = await this.httpClient.get(conceptsUrl, { params: { page, limit: CONCEPT_PAGE_SIZE } }); + const payload = response.data; + + let items = []; + if (Array.isArray(payload)) { + items = payload; + } else if (payload && typeof payload === 'object') { + items = Array.isArray(payload.results) + ? payload.results + : Array.isArray(payload.items) + ? payload.items + : Array.isArray(payload.data) + ? payload.data + : []; + } + + if (!items || items.length === 0) { + break; + } + + for (const concept of items) { + const owner = concept.owner || defaultOwner || null; + const source = concept.source || null; + if (owner && source) { + keys.set(`${owner}|${source}`, { owner, source }); + } + } + + if (items.length < CONCEPT_PAGE_SIZE) { + break; + } + + page += 1; + } + + return Array.from(keys.values()); + } + + async #getSourceCanonicalUrl(owner, source) { + const key = `${owner}|${source}`; + if (this.sourceCanonicalCache.has(key)) { + return this.sourceCanonicalCache.get(key); + } + if (this.pendingSourceCanonicalRequests.has(key)) { + return this.pendingSourceCanonicalRequests.get(key); + } + + const path = `/orgs/${encodeURIComponent(owner)}/sources/${encodeURIComponent(source)}/`; + const pending = (async () => { + try { + const response = await this.httpClient.get(path); + const data = response.data || {}; + const canonicalUrl = data.canonical_url || data.canonicalUrl || data.url || source; + this.sourceCanonicalCache.set(key, canonicalUrl); + return canonicalUrl; + } catch (error) { + this.sourceCanonicalCache.set(key, source); + return source; + } + })(); + + this.pendingSourceCanonicalRequests.set(key, pending); + try { + return await pending; + } finally { + this.pendingSourceCanonicalRequests.delete(key); + } + } + + #matches(json, params) { + for (const [name, value] of Object.entries(params)) { + if (!value) { + continue; + } + + switch (name) { + case 'url': + if ((json.url || '').toLowerCase() !== value) { + return false; + } + break; + case 'system': + if (!json.compose?.include?.some(i => (i.system || '').toLowerCase().includes(value))) { + return false; + } + break; + case 'identifier': { + const identifiers = Array.isArray(json.identifier) ? json.identifier : (json.identifier ? [json.identifier] : []); + const match = identifiers.some(i => (i.system || '').toLowerCase().includes(value) || (i.value || '').toLowerCase().includes(value)); + if (!match) { + return false; + } + break; + } + default: { + const field = json[name]; + if (field == null || !String(field).toLowerCase().includes(value)) { + return false; + } + break; + } + } + } + return true; + } + + async #fetchAllPages(path) { + const results = []; + let page = 1; + + // eslint-disable-next-line no-constant-condition + while (true) { + const response = await this.httpClient.get(path, { params: { page, limit: PAGE_SIZE } }); + const payload = response.data; + + let items = []; + if (Array.isArray(payload)) { + items = payload; + } else if (payload && typeof payload === 'object') { + items = Array.isArray(payload.results) + ? payload.results + : Array.isArray(payload.items) + ? payload.items + : Array.isArray(payload.data) + ? payload.data + : []; + } + + if (!items || items.length === 0) { + break; + } + + results.push(...items); + + if (items.length < PAGE_SIZE) { + break; + } + + page += 1; + } + + return results; + } + + #toIsoDate(value) { + if (!value) { + return null; + } + const date = new Date(value); + if (Number.isNaN(date.getTime())) { + return null; + } + return date.toISOString(); + } + + #normalizeFilter(filter) { + if (typeof filter !== 'string') { + return null; + } + const normalized = filter.trim().toLowerCase(); + return normalized.length > 0 ? normalized : null; + } + + #conceptMatchesFilter(searchableText, code, display, definition, filter, filterMatcher) { + if (!filter) { + return true; + } + + const codeText = code ? String(code).toLowerCase() : ''; + const displayText = display ? String(display).toLowerCase() : ''; + const definitionText = definition ? String(definition).toLowerCase() : ''; + + // FHIR allows terminology-server-defined behavior; guarantee baseline contains matching. + if (codeText.includes(filter) || displayText.includes(filter) || definitionText.includes(filter)) { + return true; + } + + // Preserve token/prefix behavior already provided by SearchFilterText. + return !!(filterMatcher && filterMatcher.passes(searchableText)); + } +} + +module.exports = { + OCLValueSetProvider +}; \ No newline at end of file diff --git a/tx/ocl/vs-ocl.js b/tx/ocl/vs-ocl.js index 18733fb5..3287b644 100644 --- a/tx/ocl/vs-ocl.js +++ b/tx/ocl/vs-ocl.js @@ -1,1760 +1,2 @@ -const fs = require('fs/promises'); -const crypto = require('crypto'); -const path = require('path'); -const { AbstractValueSetProvider } = require('../vs/vs-api'); -const { VersionUtilities } = require('../../library/version-utilities'); -const ValueSet = require('../library/valueset'); -const { SearchFilterText } = require('../library/designations'); -const { TxParameters } = require('../params'); -const { OCLSourceCodeSystemFactory, OCLBackgroundJobQueue } = require('./cs-ocl'); -const { PAGE_SIZE, CONCEPT_PAGE_SIZE, FILTERED_CONCEPT_PAGE_SIZE, COLD_CACHE_FRESHNESS_MS } = require('./shared/constants'); -const { createOclHttpClient } = require('./http/client'); -const { CACHE_VS_DIR, getCacheFilePath } = require('./cache/cache-paths'); -const { ensureCacheDirectories, getColdCacheAgeMs, formatCacheAgeMinutes } = require('./cache/cache-utils'); -const { computeValueSetExpansionFingerprint } = require('./fingerprint/fingerprint'); -const { ensureTxParametersHashIncludesFilter } = require('./shared/patches'); +module.exports = require('./vs-ocl.cjs'); -ensureTxParametersHashIncludesFilter(TxParameters); - -class OCLValueSetProvider extends AbstractValueSetProvider { - constructor(config = {}) { - super(); - const options = typeof config === 'string' ? { baseUrl: config } : (config || {}); - - this.org = options.org || null; - const http = createOclHttpClient(options); - this.baseUrl = http.baseUrl; - this.httpClient = http.client; - - this.valueSetMap = new Map(); - this._idMap = new Map(); - this.collectionMeta = new Map(); - this.sourceCanonicalCache = new Map(); - this.collectionConceptPageCache = new Map(); - this.pendingCollectionConceptPageRequests = new Map(); - this.collectionSourcesCache = new Map(); - this.pendingCollectionSourcesRequests = new Map(); - this.pendingSourceCanonicalRequests = new Map(); - this.collectionByCanonicalCache = new Map(); - this.pendingCollectionByCanonicalRequests = new Map(); - this._composePromises = new Map(); - this.backgroundExpansionCache = new Map(); - this.backgroundExpansionProgress = new Map(); - this.valueSetFingerprints = new Map(); - this._initialized = false; - this._initializePromise = null; - this.sourcePackageCode = this.org - ? `ocl:${this.baseUrl}|org=${this.org}` - : `ocl:${this.baseUrl}`; - } - - async #loadColdCacheForValueSets() { - try { - const files = await fs.readdir(CACHE_VS_DIR); - let loadedCount = 0; - - for (const file of files) { - if (!file.endsWith('.json')) { - continue; - } - - try { - const filePath = path.join(CACHE_VS_DIR, file); - const data = await fs.readFile(filePath, 'utf-8'); - const cached = JSON.parse(data); - - if (!cached || !cached.canonicalUrl || !cached.expansion) { - continue; - } - - const paramsKey = cached.paramsKey || 'default'; - const cacheKey = this.#expansionCacheKey( - { url: cached.canonicalUrl, version: cached.version || null }, - paramsKey - ); - const createdAt = cached.timestamp ? new Date(cached.timestamp).getTime() : null; - this.backgroundExpansionCache.set(cacheKey, { - expansion: cached.expansion, - metadataSignature: cached.metadataSignature || null, - dependencyChecksums: cached.dependencyChecksums || {}, - createdAt: Number.isFinite(createdAt) ? createdAt : null - }); - - this.valueSetFingerprints.set(cacheKey, cached.fingerprint || null); - loadedCount++; - console.log(`[OCL-ValueSet] Loaded ValueSet from cold cache: ${cached.canonicalUrl}`); - } catch (error) { - console.error(`[OCL-ValueSet] Failed to load cold cache file ${file}:`, error.message); - } - } - - if (loadedCount > 0) { - console.log(`[OCL-ValueSet] Loaded ${loadedCount} ValueSet expansions from cold cache`); - } - } catch (error) { - if (error.code !== 'ENOENT') { - console.error('[OCL-ValueSet] Failed to load cold cache:', error.message); - } - } - } - - async #saveColdCacheForValueSet(vs, expansion, metadataSignature, dependencyChecksums, paramsKey = 'default') { - const canonicalUrl = vs?.url; - const version = vs?.version || null; - if (!canonicalUrl || !expansion) { - return null; - } - - const cacheFilePath = getCacheFilePath(CACHE_VS_DIR, canonicalUrl, version, paramsKey); - - try { - await ensureCacheDirectories(CACHE_VS_DIR); - - const fingerprint = computeValueSetExpansionFingerprint(expansion); - const cacheData = { - canonicalUrl, - version, - paramsKey, - fingerprint, - timestamp: new Date().toISOString(), - conceptCount: expansion.contains?.length || 0, - expansion, - metadataSignature, - dependencyChecksums - }; - - await fs.writeFile(cacheFilePath, JSON.stringify(cacheData, null, 2), 'utf-8'); - console.log(`[OCL-ValueSet] Saved ValueSet expansion to cold cache: ${canonicalUrl} (${expansion.contains?.length || 0} concepts, fingerprint=${fingerprint?.substring(0, 8)})`); - - return fingerprint; - } catch (error) { - console.error(`[OCL-ValueSet] Failed to save cold cache for ValueSet ${canonicalUrl}:`, error.message); - return null; - } - } - - sourcePackage() { - return this.sourcePackageCode; - } - - async initialize() { - if (this._initialized) { - return; - } - - if (this._initializePromise) { - await this._initializePromise; - return; - } - - this._initializePromise = (async () => { - try { - // Load cold cache first - await this.#loadColdCacheForValueSets(); - - const collections = await this.#fetchCollectionsForDiscovery(); - console.log(`[OCL-ValueSet] Fetched ${collections.length} collections`); - - for (const collection of collections) { - const valueSet = this.#toValueSet(collection); - if (!valueSet) { - continue; - } - this.#indexValueSet(valueSet); - } - - console.log(`[OCL-ValueSet] Loaded ${this.valueSetMap.size} value sets`); - this._initialized = true; - } catch (error) { - console.error(`[OCL-ValueSet] Initialization failed:`, error.message); - if (error.response) { - console.error(`[OCL-ValueSet] HTTP ${error.response.status}: ${error.response.statusText}`); - } - throw error; - } - })(); - - try { - await this._initializePromise; - } finally { - this._initializePromise = null; - } - } - - assignIds(ids) { - if (!this.spaceId) { - return; - } - - const unique = new Set(this.valueSetMap.values()); - this._idMap.clear(); - - for (const vs of unique) { - if (!vs.id.startsWith(`${this.spaceId}-`)) { - const nextId = `${this.spaceId}-${vs.id}`; - vs.id = nextId; - vs.jsonObj.id = nextId; - } - this._idMap.set(vs.id, vs); - ids.add(`ValueSet/${vs.id}`); - } - } - - async fetchValueSet(url, version) { - this._validateFetchParams(url, version); - - let key = `${url}|${version}`; - if (this.valueSetMap.has(key)) { - const vs = this.valueSetMap.get(key); - await this.#ensureComposeIncludes(vs); - this.#clearInlineExpansion(vs); - this.#scheduleBackgroundExpansion(vs, { reason: 'fetch-valueset' }); - return vs; - } - - if (version && VersionUtilities.isSemVer(version)) { - const majorMinor = VersionUtilities.getMajMin(version); - if (majorMinor) { - key = `${url}|${majorMinor}`; - if (this.valueSetMap.has(key)) { - const vs = this.valueSetMap.get(key); - await this.#ensureComposeIncludes(vs); - this.#clearInlineExpansion(vs); - this.#scheduleBackgroundExpansion(vs, { reason: 'fetch-valueset-mm' }); - return vs; - } - } - } - - if (this.valueSetMap.has(url)) { - const vs = this.valueSetMap.get(url); - await this.#ensureComposeIncludes(vs); - this.#clearInlineExpansion(vs); - this.#scheduleBackgroundExpansion(vs, { reason: 'fetch-valueset-url' }); - return vs; - } - - const resolved = await this.#resolveValueSetByCanonical(url, version); - if (resolved) { - await this.#ensureComposeIncludes(resolved); - this.#clearInlineExpansion(resolved); - this.#scheduleBackgroundExpansion(resolved, { reason: 'fetch-valueset-resolved' }); - return resolved; - } - - await this.initialize(); - - key = `${url}|${version}`; - if (this.valueSetMap.has(key)) { - const vs = this.valueSetMap.get(key); - await this.#ensureComposeIncludes(vs); - this.#clearInlineExpansion(vs); - this.#scheduleBackgroundExpansion(vs, { reason: 'fetch-valueset-init' }); - return vs; - } - - if (version && VersionUtilities.isSemVer(version)) { - const majorMinor = VersionUtilities.getMajMin(version); - if (majorMinor) { - key = `${url}|${majorMinor}`; - if (this.valueSetMap.has(key)) { - const vs = this.valueSetMap.get(key); - await this.#ensureComposeIncludes(vs); - this.#clearInlineExpansion(vs); - this.#scheduleBackgroundExpansion(vs, { reason: 'fetch-valueset-init-mm' }); - return vs; - } - } - } - - if (this.valueSetMap.has(url)) { - const vs = this.valueSetMap.get(url); - await this.#ensureComposeIncludes(vs); - this.#clearInlineExpansion(vs); - this.#scheduleBackgroundExpansion(vs, { reason: 'fetch-valueset-init-url' }); - return vs; - } - - return null; - } - - async fetchValueSetById(id) { - const local = this.#getLocalValueSetById(id); - if (local) { - await this.#ensureComposeIncludes(local); - this.#clearInlineExpansion(local); - this.#scheduleBackgroundExpansion(local, { reason: 'fetch-valueset-by-id' }); - return local; - } - - await this.initialize(); - - const vs = this.#getLocalValueSetById(id); - await this.#ensureComposeIncludes(vs); - this.#clearInlineExpansion(vs); - this.#scheduleBackgroundExpansion(vs, { reason: 'fetch-valueset-by-id-init' }); - return vs; - } - - #clearInlineExpansion(vs) { - if (!vs || !vs.jsonObj || !vs.jsonObj.expansion) { - return; - } - delete vs.jsonObj.expansion; - } - - #getLocalValueSetById(id) { - if (this._idMap.has(id)) { - return this._idMap.get(id); - } - - if (this.spaceId && id.startsWith(`${this.spaceId}-`)) { - const unprefixed = id.substring(this.spaceId.length + 1); - return this._idMap.get(id) || this._idMap.get(unprefixed) || this.valueSetMap.get(unprefixed) || null; - } - - return this._idMap.get(id) || this.valueSetMap.get(id) || null; - } - - // eslint-disable-next-line no-unused-vars - async searchValueSets(searchParams, _elements) { - await this.initialize(); - this._validateSearchParams(searchParams); - - const params = Object.fromEntries(searchParams.map(({ name, value }) => [name, String(value).toLowerCase()])); - const values = Array.from(new Set(this.valueSetMap.values())); - - if (Object.keys(params).length === 0) { - return values; - } - - return values.filter(vs => this.#matches(vs.jsonObj, params)); - } - - vsCount() { - return new Set(this.valueSetMap.values()).size; - } - - async listAllValueSets() { - await this.initialize(); - const urls = new Set(); - for (const vs of this.valueSetMap.values()) { - if (vs && vs.url) { - urls.add(vs.url); - } - } - return Array.from(urls); - } - - async close() { - } - - #indexValueSet(vs) { - const existing = this.valueSetMap.get(vs.url) - || (vs.version ? this.valueSetMap.get(`${vs.url}|${vs.version}`) : null) - || this._idMap.get(vs.id) - || null; - - // Preserve hydrated cold-cache expansions on first index; invalidate only on replacement. - if (existing && existing !== vs) { - this.#invalidateExpansionCache(vs); - } - - this.valueSetMap.set(vs.url, vs); - if (vs.version) { - this.valueSetMap.set(`${vs.url}|${vs.version}`, vs); - } - this.valueSetMap.set(vs.id, vs); - this._idMap.set(vs.id, vs); - } - - #toValueSet(collection) { - if (!collection || typeof collection !== 'object') { - return null; - } - - const canonicalUrl = collection.canonical_url || collection.canonicalUrl || collection.url; - const id = collection.id; - if (!canonicalUrl || !id) { - return null; - } - - const preferredSource = collection.preferred_source || collection.preferredSource || null; - const json = { - resourceType: 'ValueSet', - id, - url: canonicalUrl, - version: collection.version || null, - name: collection.name || id, - title: collection.full_name || collection.fullName || collection.name || id, - status: 'active', - experimental: collection.experimental === true, - immutable: collection.immutable === true, - description: collection.description || null, - publisher: collection.publisher || collection.owner || null, - language: collection.default_locale || collection.defaultLocale || null - }; - - const lastUpdated = this.#toIsoDate(collection.updated_on || collection.updatedOn || collection.updated_at || collection.updatedAt); - if (lastUpdated) { - json.meta = { lastUpdated }; - } - - if (preferredSource) { - json.compose = { - include: [{ system: preferredSource }] - }; - } - - const conceptsUrl = this.#normalizePath( - collection.concepts_url || collection.conceptsUrl || this.#buildCollectionConceptsPath(collection) - ); - const expansionUrl = this.#normalizePath( - collection.expansion_url || collection.expansionUrl || this.#buildCollectionExpansionPath(collection) - ); - - const meta = { - collectionId: collection.id || collection.short_code || collection.shortCode || null, - conceptsUrl, - expansionUrl, - preferredSource, - owner: collection.owner || null, - ownerType: collection.owner_type || collection.ownerType || null - }; - - this.#storeCollectionMeta(id, canonicalUrl, meta); - - const valueSet = new ValueSet(json, 'R5'); - this.#attachOclHelpers(valueSet, meta); - return valueSet; - } - - #attachOclHelpers(valueSet, meta) { - if (!valueSet || !meta) { - return; - } - - valueSet.oclMeta = meta; - valueSet.oclFetchConcepts = async ({ count, offset, activeOnly, filter, languageCodes }) => { - return this.#fetchCollectionConcepts(meta, { - count, - offset, - activeOnly, - filter: typeof filter === 'string' ? filter : null, - languageCodes: Array.isArray(languageCodes) ? languageCodes : [], - fallbackSystem: meta.preferredSource || valueSet.url - }); - }; - } - - #storeCollectionMeta(id, url, meta) { - if (!meta || (!meta.conceptsUrl && !meta.expansionUrl)) { - return; - } - if (id) { - this.collectionMeta.set(id, meta); - } - if (url) { - this.collectionMeta.set(url, meta); - } - } - - #normalizePath(pathValue) { - if (!pathValue) { - return null; - } - if (typeof pathValue !== 'string') { - return null; - } - if (pathValue.startsWith('http://') || pathValue.startsWith('https://')) { - return pathValue; - } - return `${this.baseUrl}${pathValue.startsWith('/') ? '' : '/'}${pathValue}`; - } - - #buildCollectionConceptsPath(collection) { - if (!collection || typeof collection !== 'object') { - return null; - } - const owner = collection.owner || null; - const ownerType = collection.owner_type || collection.ownerType || null; - const id = collection.id || collection.short_code || collection.shortCode || null; - if (!owner || !id || ownerType !== 'Organization') { - return null; - } - return `/orgs/${encodeURIComponent(owner)}/collections/${encodeURIComponent(id)}/concepts/`; - } - - #buildCollectionExpansionPath(collection) { - if (!collection || typeof collection !== 'object') { - return null; - } - const owner = collection.owner || null; - const ownerType = collection.owner_type || collection.ownerType || null; - const id = collection.id || collection.short_code || collection.shortCode || null; - if (!owner || !id || ownerType !== 'Organization') { - return null; - } - return `/orgs/${encodeURIComponent(owner)}/collections/${encodeURIComponent(id)}/HEAD/expansions/autoexpand-HEAD/`; - } - - #getCollectionMeta(vs) { - if (!vs) { - return null; - } - return this.collectionMeta.get(vs.id) || this.collectionMeta.get(vs.url) || null; - } - - async #ensureComposeIncludes(vs) { - if (!vs || !vs.jsonObj) { - return; - } - if (vs.jsonObj.compose && Array.isArray(vs.jsonObj.compose.include) && vs.jsonObj.compose.include.length > 0) { - return; - } - - const meta = this.#getCollectionMeta(vs); - if (!meta || !meta.conceptsUrl) { - return; - } - - const composeKey = vs.id || vs.url; - if (this._composePromises.has(composeKey)) { - await this._composePromises.get(composeKey); - return; - } - - const promise = (async () => { - const sources = await this.#fetchCollectionSources(meta); - if (!sources || sources.length === 0) { - return; - } - - vs.jsonObj.compose = { - include: sources.map(source => ({ - system: source.system, - version: source.version || undefined - })) - }; - })(); - - this._composePromises.set(composeKey, promise); - try { - await promise; - } finally { - this._composePromises.delete(composeKey); - } - } - - async #fetchCollectionSources(meta) { - const sourcesCacheKey = `${meta.owner || ''}|${meta.collectionId || ''}|${meta.conceptsUrl || ''}|${meta.expansionUrl || ''}`; - if (this.collectionSourcesCache.has(sourcesCacheKey)) { - return this.collectionSourcesCache.get(sourcesCacheKey); - } - if (this.pendingCollectionSourcesRequests.has(sourcesCacheKey)) { - return this.pendingCollectionSourcesRequests.get(sourcesCacheKey); - } - - const pending = (async () => { - const sources = []; - const seen = new Set(); - - if (meta.expansionUrl) { - try { - const response = await this.httpClient.get(meta.expansionUrl); - const resolved = Array.isArray(response.data?.resolved_source_versions) - ? response.data.resolved_source_versions - : []; - - for (const entry of resolved) { - const system = entry.canonical_url || entry.canonicalUrl || null; - const owner = entry.owner || meta.owner || null; - const shortCode = entry.short_code || entry.shortCode || entry.id || null; - const version = entry.version || null; - - const systemUrl = system || (owner && shortCode ? await this.#getSourceCanonicalUrl(owner, shortCode) : null); - if (systemUrl && !seen.has(systemUrl)) { - seen.add(systemUrl); - sources.push({ system: systemUrl, version }); - } - } - } catch (error) { - // fall through to concepts listing - } - } - - if (sources.length > 0) { - return sources; - } - - if (!meta.conceptsUrl) { - return sources; - } - - const sourceKeys = await this.#fetchCollectionSourceKeys(meta.conceptsUrl, meta.owner || null); - for (const { owner, source } of sourceKeys) { - const systemUrl = await this.#getSourceCanonicalUrl(owner, source); - if (systemUrl && !seen.has(systemUrl)) { - seen.add(systemUrl); - sources.push({ system: systemUrl }); - } - } - - if (sources.length === 0 && meta.preferredSource) { - sources.push({ system: meta.preferredSource }); - } - - this.collectionSourcesCache.set(sourcesCacheKey, sources); - return sources; - })(); - - this.pendingCollectionSourcesRequests.set(sourcesCacheKey, pending); - try { - return await pending; - } finally { - this.pendingCollectionSourcesRequests.delete(sourcesCacheKey); - } - } - - async #fetchCollectionConcepts(meta, options) { - if (!meta || !meta.conceptsUrl) { - return { contains: [], total: 0 }; - } - - const count = Number.isInteger(options?.count) ? options.count : CONCEPT_PAGE_SIZE; - const offset = Number.isInteger(options?.offset) ? options.offset : 0; - const activeOnly = options?.activeOnly === true; - const filter = this.#normalizeFilter(options?.filter); - const filterMatcher = filter ? new SearchFilterText(filter) : null; - const remoteQuery = this.#buildRemoteQuery(filter); - const fallbackSystem = options?.fallbackSystem || null; - const preferredLanguageCodes = this.#normalizeLanguageCodes(options?.languageCodes); - const effectiveLanguageCodes = preferredLanguageCodes.length > 0 ? preferredLanguageCodes : ['en']; - - if (count <= 0) { - return { contains: [], total: 0 }; - } - - const hasFilter = !!filter; - const limit = hasFilter - ? Math.min(FILTERED_CONCEPT_PAGE_SIZE, CONCEPT_PAGE_SIZE) - : CONCEPT_PAGE_SIZE; - let page = Math.floor(Math.max(0, offset) / limit) + 1; - let skip = Math.max(0, offset) % limit; - let remaining = count; - const contains = []; - let reportedTotal = null; - - while (remaining > 0) { - const pageData = await this.#fetchConceptPage(meta.conceptsUrl, page, limit, remoteQuery); - const pageItems = Array.isArray(pageData?.items) ? pageData.items : []; - if ( - reportedTotal == null && - typeof pageData?.reportedTotal === 'number' && - Number.isFinite(pageData.reportedTotal) && - pageData.reportedTotal >= 0 - ) { - reportedTotal = pageData.reportedTotal; - } - - if (!pageItems || pageItems.length === 0) { - break; - } - - const slice = pageItems.slice(skip); - skip = 0; - - for (const concept of slice) { - if (remaining <= 0) { - break; - } - if (activeOnly && concept.retired === true) { - continue; - } - - const localizedNames = this.#extractLocalizedNames(concept, effectiveLanguageCodes); - const localizedDefinitions = this.#extractLocalizedDefinitions(concept, effectiveLanguageCodes); - - const display = localizedNames.display || concept.display_name || concept.display || concept.name || null; - const definition = localizedDefinitions.definition || concept.definition || concept.description || concept.concept_class || null; - const code = concept.code || concept.id || null; - const searchableText = [ - code, - display, - definition, - ...localizedNames.designation.map(d => d.value), - ...localizedDefinitions.definitions.map(d => d.value) - ].filter(Boolean).join(' '); - if (filterMatcher && !this.#conceptMatchesFilter(searchableText, code, display, definition, filter, filterMatcher)) { - continue; - } - - if (!code) { - continue; - } - - const owner = concept.owner || meta.owner || null; - const source = concept.source || null; - const conceptCanonical = concept.source_canonical_url || concept.sourceCanonicalUrl || null; - const system = conceptCanonical || (owner && source - ? await this.#getSourceCanonicalUrl(owner, source) - : fallbackSystem); - - contains.push({ - system: system || fallbackSystem, - code, - display, - definition: definition || undefined, - designation: localizedNames.designation, - definitions: localizedDefinitions.definitions, - inactive: concept.retired === true ? true : undefined - }); - remaining -= 1; - } - - if (pageItems.length < limit) { - break; - } - - page += 1; - } - - return { contains, total: contains.length, reportedTotal }; - } - - async #resolveValueSetByCanonical(url, version) { - const canonicalUrl = typeof url === 'string' ? url.trim() : ''; - if (!canonicalUrl) { - return null; - } - - const collection = await this.#findCollectionByCanonical(canonicalUrl, version); - if (!collection) { - return null; - } - - const valueSet = this.#toValueSet(collection); - if (!valueSet) { - return null; - } - - this.#indexValueSet(valueSet); - return valueSet; - } - - #valueSetBaseKey(vs) { - if (!vs || !vs.url) { - return null; - } - return `${vs.url}|${vs.version || ''}`; - } - - #expansionParamsKey(params) { - if (!params || typeof params !== 'object') { - return 'default'; - } - - try { - const normalized = Object.keys(params) - .sort() - .reduce((acc, key) => { - if (key === 'tx-resource' || key === 'valueSet') { - return acc; - } - acc[key] = params[key]; - return acc; - }, {}); - - const json = JSON.stringify(normalized); - if (!json || json === '{}') { - return 'default'; - } - return crypto.createHash('sha256').update(json).digest('hex').substring(0, 16); - } catch (error) { - return 'default'; - } - } - - #expansionCacheKey(vs, paramsKey) { - const base = this.#valueSetBaseKey(vs); - if (!base) { - return null; - } - return `${base}|${paramsKey || 'default'}`; - } - - #invalidateExpansionCache(vs) { - const base = this.#valueSetBaseKey(vs); - if (!base) { - return; - } - - for (const key of this.backgroundExpansionCache.keys()) { - if (key.startsWith(`${base}|`)) { - this.backgroundExpansionCache.delete(key); - } - } - } - - #applyCachedExpansion(vs, paramsKey) { - if (!vs || !vs.jsonObj) { - return; - } - - const cacheKey = this.#expansionCacheKey(vs, paramsKey); - if (!cacheKey) { - return; - } - - const cached = this.backgroundExpansionCache.get(cacheKey); - if (!cached || !cached.expansion) { - return; - } - - if (!this.#isCachedExpansionValid(vs, cached)) { - this.backgroundExpansionCache.delete(cacheKey); - if (vs.jsonObj.expansion) { - delete vs.jsonObj.expansion; - } - console.log(`[OCL-ValueSet] Cached ValueSet expansion invalidated: ${cacheKey}`); - return; - } - - if (vs.jsonObj.expansion) { - return; - } - - vs.jsonObj.expansion = structuredClone(cached.expansion); - console.log(`[OCL-ValueSet] ValueSet expansion restored from cache: ${cacheKey}`); - } - - #scheduleBackgroundExpansion(vs, options = {}) { - if (!vs || !vs.jsonObj) { - return; - } - - const paramsKey = this.#expansionParamsKey(options.params || null); - const cacheKey = this.#expansionCacheKey(vs, paramsKey); - if (!cacheKey) { - return; - } - - const cached = this.backgroundExpansionCache.get(cacheKey); - if (cached && !this.#isCachedExpansionValid(vs, cached)) { - this.backgroundExpansionCache.delete(cacheKey); - if (vs.jsonObj.expansion) { - delete vs.jsonObj.expansion; - } - console.log(`[OCL-ValueSet] Cached ValueSet expansion invalidated: ${cacheKey}`); - } - - if (vs.jsonObj.expansion) { - return; - } - - const cacheFilePath = getCacheFilePath(CACHE_VS_DIR, vs.url, vs.version || null, paramsKey); - const cacheAgeFromFileMs = getColdCacheAgeMs(cacheFilePath); - const persistedCache = this.backgroundExpansionCache.get(cacheKey); - const cacheAgeFromMetadataMs = Number.isFinite(persistedCache?.createdAt) - ? Math.max(0, Date.now() - persistedCache.createdAt) - : null; - - // Treat cache as fresh when either file mtime or persisted timestamp is recent. - const freshnessCandidates = [cacheAgeFromFileMs, cacheAgeFromMetadataMs].filter(age => age != null); - const freshestCacheAgeMs = freshnessCandidates.length > 0 ? Math.min(...freshnessCandidates) : null; - if (freshestCacheAgeMs != null && freshestCacheAgeMs <= COLD_CACHE_FRESHNESS_MS) { - const freshnessSource = cacheAgeFromFileMs != null && cacheAgeFromMetadataMs != null - ? 'file+metadata' - : cacheAgeFromFileMs != null - ? 'file' - : 'metadata'; - console.log(`[OCL-ValueSet] Skipping warm-up for ValueSet ${vs.url} (cold cache age: ${formatCacheAgeMinutes(freshestCacheAgeMs)})`); - console.log(`[OCL-ValueSet] ValueSet cold cache is fresh, not enqueueing warm-up job (${cacheKey}, source=${freshnessSource})`); - return; - } - - const jobKey = `vs:${cacheKey}`; - if (OCLBackgroundJobQueue.isQueuedOrRunning(jobKey)) { - console.log(`[OCL-ValueSet] ValueSet expansion already queued or running: ${cacheKey}`); - return; - } - - let queuedJobSize = null; - const warmupAgeText = freshestCacheAgeMs != null - ? formatCacheAgeMinutes(freshestCacheAgeMs) - : 'no cold cache'; - console.log(`[OCL-ValueSet] Enqueueing warm-up for ValueSet ${vs.url} (cold cache age: ${warmupAgeText})`); - console.log(`[OCL-ValueSet] ValueSet expansion enqueued: ${cacheKey}`); - OCLBackgroundJobQueue.enqueue( - jobKey, - 'ValueSet expansion', - async () => { - await this.#runBackgroundExpansion(vs, cacheKey, paramsKey, queuedJobSize); - }, - { - jobId: vs.url || cacheKey, - getProgress: () => this.#backgroundExpansionProgressSnapshot(cacheKey), - resolveJobSize: async () => { - const meta = this.#getCollectionMeta(vs); - queuedJobSize = await this.#fetchConceptCountFromHeaders(meta?.conceptsUrl || null); - return queuedJobSize; - } - } - ); - } - - async #runBackgroundExpansion(vs, cacheKey, paramsKey = 'default', knownConceptCount = null) { - console.log(`[OCL-ValueSet] ValueSet expansion started: ${cacheKey}`); - const progressState = { processed: 0, total: null }; - this.backgroundExpansionProgress.set(cacheKey, progressState); - try { - await this.#ensureComposeIncludes(vs); - - const meta = this.#getCollectionMeta(vs); - const resolvedTotal = Number.isFinite(knownConceptCount) && knownConceptCount >= 0 - ? knownConceptCount - : await this.#fetchConceptCountFromHeaders(meta?.conceptsUrl || null); - progressState.total = resolvedTotal; - const sources = meta ? await this.#fetchCollectionSources(meta) : []; - for (const source of sources || []) { - OCLSourceCodeSystemFactory.scheduleBackgroundLoadByKey( - source.system, - source.version || null, - 'valueset-expansion' - ); - } - - const expansion = await this.#buildBackgroundExpansion(vs, progressState); - if (!expansion) { - return; - } - - progressState.processed = expansion.total || progressState.processed; - if (typeof progressState.total !== 'number' || !Number.isFinite(progressState.total) || progressState.total <= 0) { - progressState.total = expansion.total || 0; - } - - const metadataSignature = this.#valueSetMetadataSignature(vs); - const dependencyChecksums = this.#valueSetDependencyChecksums(vs); - - // Compute custom fingerprint and compare with cold cache - const newFingerprint = computeValueSetExpansionFingerprint(expansion); - const oldFingerprint = this.valueSetFingerprints.get(cacheKey); - - if (oldFingerprint && newFingerprint === oldFingerprint) { - console.log(`[OCL-ValueSet] ValueSet expansion fingerprint unchanged: ${cacheKey} (fingerprint=${newFingerprint?.substring(0, 8)})`); - } else { - if (oldFingerprint) { - console.log(`[OCL-ValueSet] ValueSet expansion fingerprint changed: ${cacheKey} (${oldFingerprint?.substring(0, 8)} -> ${newFingerprint?.substring(0, 8)})`); - console.log(`[OCL-ValueSet] Replacing cold cache with new hot cache: ${cacheKey}`); - } else { - console.log(`[OCL-ValueSet] Computed fingerprint for ValueSet expansion: ${cacheKey} (fingerprint=${newFingerprint?.substring(0, 8)})`); - } - - // Save to cold cache - const savedFingerprint = await this.#saveColdCacheForValueSet(vs, expansion, metadataSignature, dependencyChecksums, paramsKey); - if (savedFingerprint) { - this.valueSetFingerprints.set(cacheKey, savedFingerprint); - } - } - - this.backgroundExpansionCache.set(cacheKey, { - expansion, - metadataSignature, - dependencyChecksums, - createdAt: Date.now() - }); - // Keep expansions in provider-managed cache only. - // Inline expansion on ValueSet bypasses $expand filtering in worker pipeline. - - console.log(`[OCL-ValueSet] ValueSet expansion completed and cached: ${cacheKey}`); - console.log(`[OCL-ValueSet] ValueSet now available in cache: ${cacheKey}`); - } catch (error) { - console.error(`[OCL-ValueSet] ValueSet background expansion failed: ${cacheKey}: ${error.message}`); - } finally { - this.backgroundExpansionProgress.delete(cacheKey); - } - } - - async #buildBackgroundExpansion(vs, progressState = null) { - const meta = this.#getCollectionMeta(vs); - if (!meta || !meta.conceptsUrl) { - return null; - } - - const contains = []; - let offset = 0; - - // Pull all concepts in fixed-size pages until exhausted. - // eslint-disable-next-line no-constant-condition - while (true) { - const batch = await this.#fetchCollectionConcepts(meta, { - count: CONCEPT_PAGE_SIZE, - offset, - activeOnly: false, - filter: null, - languageCodes: [] - }); - - const entries = Array.isArray(batch?.contains) ? batch.contains : []; - if (entries.length === 0) { - break; - } - - for (const entry of entries) { - if (!entry?.system || !entry?.code) { - continue; - } - - const out = { - system: entry.system, - code: entry.code - }; - if (entry.display) { - out.display = entry.display; - } - if (entry.definition) { - out.definition = entry.definition; - } - if (entry.inactive === true) { - out.inactive = true; - } - if (Array.isArray(entry.designation) && entry.designation.length > 0) { - out.designation = entry.designation - .filter(d => d && d.value) - .map(d => ({ - language: d.language, - value: d.value - })); - } - contains.push(out); - } - - if (progressState) { - progressState.processed = contains.length; - } - - if (entries.length < CONCEPT_PAGE_SIZE) { - break; - } - offset += entries.length; - } - - return { - timestamp: new Date().toISOString(), - identifier: `urn:uuid:${crypto.randomUUID()}`, - total: contains.length, - contains - }; - } - - async #findCollectionByCanonical(canonicalUrl, version) { - const lookupKey = `${this.org || '*'}|${canonicalUrl}|${version || ''}`; - if (this.collectionByCanonicalCache.has(lookupKey)) { - return this.collectionByCanonicalCache.get(lookupKey); - } - if (this.pendingCollectionByCanonicalRequests.has(lookupKey)) { - return this.pendingCollectionByCanonicalRequests.get(lookupKey); - } - - const token = this.#canonicalToken(canonicalUrl); - if (!token) { - return null; - } - - const pending = (async () => { - const organizations = await this.#fetchOrganizationIds(); - const endpoints = organizations.length > 0 - ? organizations.map(orgId => `/orgs/${encodeURIComponent(orgId)}/collections/`) - : ['/collections/']; - - const exactMatches = []; - for (const endpoint of endpoints) { - const response = await this.httpClient.get(endpoint, { - params: { - q: token, - page: 1, - limit: PAGE_SIZE - } - }); - - const payload = response.data; - const items = Array.isArray(payload) - ? payload - : Array.isArray(payload?.results) - ? payload.results - : Array.isArray(payload?.items) - ? payload.items - : Array.isArray(payload?.data) - ? payload.data - : []; - - for (const item of items) { - const itemCanonical = item?.canonical_url || item?.canonicalUrl || item?.url || null; - if (itemCanonical === canonicalUrl) { - exactMatches.push(item); - } - } - } - - let match = null; - if (exactMatches.length > 0) { - if (!version) { - match = exactMatches[0]; - } else { - const exactVersion = exactMatches.find(item => (item.version || null) === version); - if (exactVersion) { - match = exactVersion; - } else if (VersionUtilities.isSemVer(version)) { - const majorMinor = VersionUtilities.getMajMin(version); - if (majorMinor) { - const majorMinorMatch = exactMatches.find(item => (item.version || null) === majorMinor); - if (majorMinorMatch) { - match = majorMinorMatch; - } - } - } - } - } - - this.collectionByCanonicalCache.set(lookupKey, match); - return match; - })(); - - this.pendingCollectionByCanonicalRequests.set(lookupKey, pending); - try { - return await pending; - } finally { - this.pendingCollectionByCanonicalRequests.delete(lookupKey); - } - } - - #valueSetMetadataSignature(vs) { - const meta = this.#getCollectionMeta(vs); - const payload = { - url: vs?.url || null, - version: vs?.version || null, - lastUpdated: vs?.jsonObj?.meta?.lastUpdated || null, - collectionId: meta?.collectionId || null, - conceptsUrl: meta?.conceptsUrl || null, - expansionUrl: meta?.expansionUrl || null, - preferredSource: meta?.preferredSource || null - }; - return JSON.stringify(payload); - } - - #valueSetDependencyChecksums(vs) { - const include = Array.isArray(vs?.jsonObj?.compose?.include) ? vs.jsonObj.compose.include : []; - const checksums = {}; - for (const item of include) { - const system = item?.system || null; - if (!system) { - continue; - } - const version = item?.version || null; - const key = `${system}|${version || ''}`; - checksums[key] = OCLSourceCodeSystemFactory.checksumForResource(system, version); - } - return checksums; - } - - #isCachedExpansionValid(vs, cached) { - if (!cached || typeof cached !== 'object') { - return false; - } - - if (cached.metadataSignature !== this.#valueSetMetadataSignature(vs)) { - return false; - } - - const currentDeps = this.#valueSetDependencyChecksums(vs); - const cachedDeps = cached.dependencyChecksums || {}; - const currentKeys = Object.keys(currentDeps).sort(); - const cachedKeys = Object.keys(cachedDeps).sort(); - - if (currentKeys.length !== cachedKeys.length) { - return false; - } - - for (let i = 0; i < currentKeys.length; i++) { - if (currentKeys[i] !== cachedKeys[i]) { - return false; - } - if ((currentDeps[currentKeys[i]] || null) !== (cachedDeps[cachedKeys[i]] || null)) { - return false; - } - } - - return true; - } - - async #fetchCollectionsForDiscovery() { - const organizations = await this.#fetchOrganizationIds(); - if (organizations.length === 0) { - // Fallback for OCL instances that expose global listing but not org listing. - return await this.#fetchAllPages('/collections/'); - } - - const allCollections = []; - const seen = new Set(); - - for (const orgId of organizations) { - const endpoint = `/orgs/${encodeURIComponent(orgId)}/collections/`; - const collections = await this.#fetchAllPages(endpoint); - for (const collection of collections) { - if (!collection || typeof collection !== 'object') { - continue; - } - const key = this.#collectionIdentity(collection); - if (seen.has(key)) { - continue; - } - seen.add(key); - allCollections.push(collection); - } - } - - return allCollections; - } - - async #fetchOrganizationIds() { - const endpoint = '/orgs/'; - console.log(`[OCL-ValueSet] Loading organizations from: ${this.baseUrl}${endpoint}`); - const orgs = await this.#fetchAllPages(endpoint); - - const ids = []; - const seen = new Set(); - for (const org of orgs || []) { - if (!org || typeof org !== 'object') { - continue; - } - - const id = org.id || org.mnemonic || org.short_code || org.shortCode || org.name || null; - if (!id) { - continue; - } - - const normalized = String(id).trim(); - if (!normalized || seen.has(normalized)) { - continue; - } - - seen.add(normalized); - ids.push(normalized); - } - - if (ids.length === 0 && this.org) { - ids.push(this.org); - } - - return ids; - } - - #collectionIdentity(collection) { - if (!collection || typeof collection !== 'object') { - return '__invalid__'; - } - - const owner = collection.owner || ''; - const canonical = collection.canonical_url || collection.canonicalUrl || ''; - const id = collection.id || collection.short_code || collection.shortCode || collection.name || ''; - return `${owner}|${canonical}|${id}`; - } - - #canonicalToken(canonicalUrl) { - if (!canonicalUrl || typeof canonicalUrl !== 'string') { - return null; - } - - const trimmed = canonicalUrl.trim(); - if (!trimmed) { - return null; - } - - const parts = trimmed.replace(/\/+$/, '').split('/').filter(Boolean); - if (parts.length === 0) { - return null; - } - - return parts[parts.length - 1]; - } - - async #fetchConceptPage(conceptsUrl, page, limit, filter = null) { - try { - const cacheKey = `${conceptsUrl}|p=${page}|l=${limit}|q=${filter || ''}|verbose=1`; - if (this.collectionConceptPageCache.has(cacheKey)) { - const cached = this.collectionConceptPageCache.get(cacheKey); - const items = Array.isArray(cached) - ? cached - : Array.isArray(cached?.items) - ? cached.items - : []; - return { - items, - reportedTotal: this.#extractTotalFromPayload(cached?.payload || null) - }; - } - if (this.pendingCollectionConceptPageRequests.has(cacheKey)) { - return this.pendingCollectionConceptPageRequests.get(cacheKey); - } - - const pending = (async () => { - const params = { page, limit, verbose: true }; - if (filter) { - params.q = filter; - } - const response = await this.httpClient.get(conceptsUrl, { params }); - const payload = response.data; - const items = Array.isArray(payload) - ? payload - : Array.isArray(payload?.results) - ? payload.results - : Array.isArray(payload?.items) - ? payload.items - : Array.isArray(payload?.data) - ? payload.data - : []; - - this.collectionConceptPageCache.set(cacheKey, { items, payload }); - return { - items, - reportedTotal: this.#extractTotalFromPayload(payload) - }; - })(); - - this.pendingCollectionConceptPageRequests.set(cacheKey, pending); - try { - return await pending; - } finally { - this.pendingCollectionConceptPageRequests.delete(cacheKey); - } - } catch (error) { - console.error(`[OCL-ValueSet] #fetchConceptPage ERROR: ${error.message}`); - throw error; - } - } - - #buildRemoteQuery(filter) { - if (!filter || typeof filter !== 'string') { - return null; - } - - const tokens = filter - .split(/\s+or\s+|\||&|\s+/i) - .map(t => t.trim()) - .filter(Boolean) - .map(t => (t.startsWith('-') || t.startsWith('!')) ? t.substring(1) : t) - .map(t => t.replace(/[%*?]/g, '')) - .map(t => t.replace(/[^\p{L}\p{N}]+/gu, '')) - .filter(t => t.length >= 3); - - if (tokens.length === 0) { - return null; - } - - tokens.sort((a, b) => b.length - a.length); - return tokens[0]; - } - - #normalizeLanguageCodes(languageCodes) { - if (!Array.isArray(languageCodes)) { - return []; - } - - const normalized = []; - for (const code of languageCodes) { - if (!code || typeof code !== 'string') { - continue; - } - normalized.push(code.toLowerCase()); - } - return normalized; - } - - #backgroundExpansionProgressSnapshot(cacheKey) { - const progress = this.backgroundExpansionProgress.get(cacheKey); - if (!progress) { - return null; - } - - const processed = progress.processed; - const total = progress.total; - if ( - typeof processed === 'number' && - Number.isFinite(processed) && - typeof total === 'number' && - Number.isFinite(total) && - total > 0 - ) { - return { processed, total }; - } - - return null; - } - - #extractTotalFromPayload(payload) { - if (!payload || typeof payload !== 'object' || Array.isArray(payload)) { - return null; - } - - const candidates = [ - payload.total, - payload.total_count, - payload.totalCount, - payload.num_found, - payload.numFound, - payload.count - ]; - - for (const candidate of candidates) { - if (typeof candidate === 'number' && Number.isFinite(candidate) && candidate >= 0) { - return candidate; - } - } - - return null; - } - - async #fetchConceptCountFromHeaders(conceptsUrl) { - if (!conceptsUrl) { - return null; - } - - try { - const response = await this.httpClient.get(conceptsUrl, { - params: { - limit: 1 - } - }); - return this.#extractNumFoundFromHeaders(response?.headers); - } catch (error) { - return null; - } - } - - #extractNumFoundFromHeaders(headers) { - if (!headers || typeof headers !== 'object') { - return null; - } - - const raw = headers.num_found ?? headers['num-found'] ?? headers.Num_Found ?? null; - const parsed = typeof raw === 'number' ? raw : Number.parseInt(raw, 10); - if (!Number.isFinite(parsed) || parsed < 0) { - return null; - } - return parsed; - } - - #languageRank(languageCode, preferredLanguageCodes) { - if (!languageCode) { - return 1000; - } - - const normalized = String(languageCode).toLowerCase(); - for (let i = 0; i < preferredLanguageCodes.length; i++) { - const preferred = preferredLanguageCodes[i]; - if (normalized === preferred || normalized.startsWith(`${preferred}-`) || preferred.startsWith(`${normalized}-`)) { - return i; - } - } - - if (normalized === 'en' || normalized.startsWith('en-')) { - return preferredLanguageCodes.length; - } - - return preferredLanguageCodes.length + 1; - } - - #extractLocalizedNames(concept, preferredLanguageCodes) { - const names = Array.isArray(concept?.names) ? concept.names : []; - const unique = new Map(); - - for (const item of names) { - const value = item?.name; - if (!value || typeof value !== 'string') { - continue; - } - - const language = typeof item?.locale === 'string' && item.locale.trim() ? item.locale.trim().toLowerCase() : null; - const key = `${language || ''}|${value}`; - if (unique.has(key)) { - continue; - } - - unique.set(key, { - language: language || undefined, - value, - localePreferred: item?.locale_preferred === true, - nameType: item?.name_type || '' - }); - } - - const designation = Array.from(unique.values()) - .sort((a, b) => { - const rankDiff = this.#languageRank(a.language, preferredLanguageCodes) - this.#languageRank(b.language, preferredLanguageCodes); - if (rankDiff !== 0) { - return rankDiff; - } - if (a.localePreferred !== b.localePreferred) { - return a.localePreferred ? -1 : 1; - } - const aFs = /fully\s*-?\s*specified/i.test(a.nameType); - const bFs = /fully\s*-?\s*specified/i.test(b.nameType); - if (aFs !== bFs) { - return aFs ? -1 : 1; - } - return a.value.localeCompare(b.value); - }) - .map(({ language, value }) => ({ - language, - value - })); - - return { - display: designation.length > 0 ? designation[0].value : null, - designation - }; - } - - #extractLocalizedDefinitions(concept, preferredLanguageCodes) { - const descriptions = Array.isArray(concept?.descriptions) ? concept.descriptions : []; - const unique = new Map(); - - for (const item of descriptions) { - const value = item?.description; - if (!value || typeof value !== 'string') { - continue; - } - - const language = typeof item?.locale === 'string' && item.locale.trim() ? item.locale.trim().toLowerCase() : null; - const key = `${language || ''}|${value}`; - if (unique.has(key)) { - continue; - } - - unique.set(key, { - language: language || undefined, - value, - localePreferred: item?.locale_preferred === true, - descriptionType: item?.description_type || '' - }); - } - - const definitions = Array.from(unique.values()) - .sort((a, b) => { - const rankDiff = this.#languageRank(a.language, preferredLanguageCodes) - this.#languageRank(b.language, preferredLanguageCodes); - if (rankDiff !== 0) { - return rankDiff; - } - if (a.localePreferred !== b.localePreferred) { - return a.localePreferred ? -1 : 1; - } - const aDef = /definition/i.test(a.descriptionType); - const bDef = /definition/i.test(b.descriptionType); - if (aDef !== bDef) { - return aDef ? -1 : 1; - } - return a.value.localeCompare(b.value); - }) - .map(({ language, value }) => ({ - language, - value - })); - - return { - definition: definitions.length > 0 ? definitions[0].value : null, - definitions - }; - } - - async #fetchCollectionSourceKeys(conceptsUrl, defaultOwner) { - const keys = new Map(); - let page = 1; - - // eslint-disable-next-line no-constant-condition - while (true) { - const response = await this.httpClient.get(conceptsUrl, { params: { page, limit: CONCEPT_PAGE_SIZE } }); - const payload = response.data; - - let items = []; - if (Array.isArray(payload)) { - items = payload; - } else if (payload && typeof payload === 'object') { - items = Array.isArray(payload.results) - ? payload.results - : Array.isArray(payload.items) - ? payload.items - : Array.isArray(payload.data) - ? payload.data - : []; - } - - if (!items || items.length === 0) { - break; - } - - for (const concept of items) { - const owner = concept.owner || defaultOwner || null; - const source = concept.source || null; - if (owner && source) { - keys.set(`${owner}|${source}`, { owner, source }); - } - } - - if (items.length < CONCEPT_PAGE_SIZE) { - break; - } - - page += 1; - } - - return Array.from(keys.values()); - } - - async #getSourceCanonicalUrl(owner, source) { - const key = `${owner}|${source}`; - if (this.sourceCanonicalCache.has(key)) { - return this.sourceCanonicalCache.get(key); - } - if (this.pendingSourceCanonicalRequests.has(key)) { - return this.pendingSourceCanonicalRequests.get(key); - } - - const path = `/orgs/${encodeURIComponent(owner)}/sources/${encodeURIComponent(source)}/`; - const pending = (async () => { - try { - const response = await this.httpClient.get(path); - const data = response.data || {}; - const canonicalUrl = data.canonical_url || data.canonicalUrl || data.url || source; - this.sourceCanonicalCache.set(key, canonicalUrl); - return canonicalUrl; - } catch (error) { - this.sourceCanonicalCache.set(key, source); - return source; - } - })(); - - this.pendingSourceCanonicalRequests.set(key, pending); - try { - return await pending; - } finally { - this.pendingSourceCanonicalRequests.delete(key); - } - } - - #matches(json, params) { - for (const [name, value] of Object.entries(params)) { - if (!value) { - continue; - } - - switch (name) { - case 'url': - if ((json.url || '').toLowerCase() !== value) { - return false; - } - break; - case 'system': - if (!json.compose?.include?.some(i => (i.system || '').toLowerCase().includes(value))) { - return false; - } - break; - case 'identifier': { - const identifiers = Array.isArray(json.identifier) ? json.identifier : (json.identifier ? [json.identifier] : []); - const match = identifiers.some(i => (i.system || '').toLowerCase().includes(value) || (i.value || '').toLowerCase().includes(value)); - if (!match) { - return false; - } - break; - } - default: { - const field = json[name]; - if (field == null || !String(field).toLowerCase().includes(value)) { - return false; - } - break; - } - } - } - return true; - } - - async #fetchAllPages(path) { - const results = []; - let page = 1; - - // eslint-disable-next-line no-constant-condition - while (true) { - const response = await this.httpClient.get(path, { params: { page, limit: PAGE_SIZE } }); - const payload = response.data; - - let items = []; - if (Array.isArray(payload)) { - items = payload; - } else if (payload && typeof payload === 'object') { - items = Array.isArray(payload.results) - ? payload.results - : Array.isArray(payload.items) - ? payload.items - : Array.isArray(payload.data) - ? payload.data - : []; - } - - if (!items || items.length === 0) { - break; - } - - results.push(...items); - - if (items.length < PAGE_SIZE) { - break; - } - - page += 1; - } - - return results; - } - - #toIsoDate(value) { - if (!value) { - return null; - } - const date = new Date(value); - if (Number.isNaN(date.getTime())) { - return null; - } - return date.toISOString(); - } - - #normalizeFilter(filter) { - if (typeof filter !== 'string') { - return null; - } - const normalized = filter.trim().toLowerCase(); - return normalized.length > 0 ? normalized : null; - } - - #conceptMatchesFilter(searchableText, code, display, definition, filter, filterMatcher) { - if (!filter) { - return true; - } - - const codeText = code ? String(code).toLowerCase() : ''; - const displayText = display ? String(display).toLowerCase() : ''; - const definitionText = definition ? String(definition).toLowerCase() : ''; - - // FHIR allows terminology-server-defined behavior; guarantee baseline contains matching. - if (codeText.includes(filter) || displayText.includes(filter) || definitionText.includes(filter)) { - return true; - } - - // Preserve token/prefix behavior already provided by SearchFilterText. - return !!(filterMatcher && filterMatcher.passes(searchableText)); - } -} - -module.exports = { - OCLValueSetProvider -}; \ No newline at end of file From 8683120e1dae49a4078c5c2265499f639dcb9654 Mon Sep 17 00:00:00 2001 From: Italo Macedo Date: Sat, 7 Mar 2026 07:04:17 -0300 Subject: [PATCH 09/15] Several fixes in features that regressed after tests were added --- tests/ocl/ocl-vs.test.js | 150 +++++++++++++++++++++++++++++++++++++- tx/ocl/cs-ocl.cjs | 109 ++++++++++++++++++++++++--- tx/ocl/shared/patches.cjs | 89 ++++++++++++++++++++++ tx/ocl/vs-ocl.cjs | 130 +++++++++++++++++++++++++++------ 4 files changed, 443 insertions(+), 35 deletions(-) diff --git a/tests/ocl/ocl-vs.test.js b/tests/ocl/ocl-vs.test.js index 1ac86f4d..a69202ac 100644 --- a/tests/ocl/ocl-vs.test.js +++ b/tests/ocl/ocl-vs.test.js @@ -7,6 +7,8 @@ const { OCLValueSetProvider } = require('../../tx/ocl/vs-ocl'); const { OCLSourceCodeSystemFactory, OCLBackgroundJobQueue } = require('../../tx/ocl/cs-ocl'); const { CACHE_VS_DIR, getCacheFilePath } = require('../../tx/ocl/cache/cache-paths'); const { COLD_CACHE_FRESHNESS_MS } = require('../../tx/ocl/shared/constants'); +const { ValueSetExpander } = require('../../tx/workers/expand'); +const { patchValueSetExpandWholeSystemForOcl } = require('../../tx/ocl/shared/patches'); function resetQueueState() { OCLBackgroundJobQueue.pendingJobs = []; @@ -61,6 +63,16 @@ describe('OCL ValueSet integration', () => { expansion_url: expansionPath } ] + }) + .get(expansionPath) + .times(20) + .reply(200, { + resolved_source_versions: [ + { + canonical_url: 'http://example.org/cs/source-one', + version: '1.0.0' + } + ] }); } @@ -136,7 +148,7 @@ describe('OCL ValueSet integration', () => { .times(2) .reply(200, { results: [{ code: 'A' }] }, { num_found: '3' }) .get(conceptsPath) - .query(q => Number(q.page) === 1 && Number(q.limit) === 1000 && String(q.verbose) === 'true') + .query(q => Number(q.page) === 1 && Number(q.limit) === 1000) .reply(200, { results: [ { @@ -162,7 +174,7 @@ describe('OCL ValueSet integration', () => { ] }) .get(conceptsPath) - .query(q => Number(q.page) === 2 && Number(q.limit) === 1000 && String(q.verbose) === 'true') + .query(q => Number(q.page) === 2 && Number(q.limit) === 1000) .reply(200, { results: [] }); const provider = new OCLValueSetProvider({ baseUrl, org: 'org-a' }); @@ -286,4 +298,138 @@ describe('OCL ValueSet integration', () => { const all = await provider.listAllValueSets(); expect(all).toContain('http://example.org/vs/one'); }); + + test('regression: unfiltered whole-system include retries with bounded paging for OCL', async () => { + patchValueSetExpandWholeSystemForOcl(); + + const cs = { + specialEnumeration: () => null, + isNotClosed: () => false, + iterator: async () => ({ total: 5001 }), + nextContext: (() => { + let emitted = false; + return async () => { + if (emitted) { + return null; + } + emitted = true; + return { code: 'A' }; + }; + })(), + system: async () => 'http://example.org/cs/source-one', + version: async () => '1.0.0' + }; + + const expander = new ValueSetExpander({ + internalLimit: 10000, + externalLimit: 1000, + opContext: { log: () => {}, diagnostics: () => '' }, + deadCheck: () => {}, + findCodeSystem: async () => cs, + checkSupplements: () => {}, + i18n: { languageDefinitions: {}, translate: (_k, _langs, args) => `too costly ${args?.join(' ') || ''}` }, + provider: { getFhirVersion: () => '5.0.0' }, + languages: { parse: () => null } + }, { + hasDesignations: false, + designations: [], + httpLanguages: null + }); + + expander.valueSet = { oclFetchConcepts: async () => ({ contains: [] }) }; + expander.limitCount = 1000; + expander.count = -1; + expander.offset = -1; + expander.hasExclusions = false; + expander.requiredSupplements = new Set(); + expander.usedSupplements = new Set(); + expander.map = new Map(); + expander.fullList = []; + expander.rootList = []; + expander.addToTotal = jest.fn(); + expander.passesFilters = jest.fn(async () => true); + expander.includeCodeAndDescendants = jest.fn(async () => 1); + expander.checkProviderCanonicalStatus = jest.fn(); + expander.addParamUri = jest.fn(); + expander.addParamInt = jest.fn(); + + await expect(expander.includeCodes( + { system: 'http://example.org/cs/source-one' }, + 'ValueSet.compose.include[0]', + { vurl: 'http://example.org/vs/one|1.0.0', url: 'http://example.org/vs/one' }, + { include: [{ system: 'http://example.org/cs/source-one' }] }, + { isNull: true }, + {}, + false, + { value: false } + )).resolves.toBeUndefined(); + + expect(expander.addParamInt).toHaveBeenCalledWith(expect.any(Object), 'offset', 0); + expect(expander.addParamInt).toHaveBeenCalledWith(expect.any(Object), 'count', 1000); + expect(expander.includeCodeAndDescendants).toHaveBeenCalled(); + }); + + test('filtered whole-system include keeps default too-costly behavior', async () => { + patchValueSetExpandWholeSystemForOcl(); + + const cs = { + specialEnumeration: () => null, + isNotClosed: () => false, + iterator: async () => ({ total: 5001 }), + nextContext: async () => null, + system: async () => 'http://example.org/cs/source-one', + version: async () => '1.0.0' + }; + + const expander = new ValueSetExpander({ + internalLimit: 10000, + externalLimit: 1000, + opContext: { log: () => {}, diagnostics: () => '' }, + deadCheck: () => {}, + findCodeSystem: async () => cs, + checkSupplements: () => {}, + i18n: { languageDefinitions: {}, translate: (_k, _langs, args) => `too costly ${args?.join(' ') || ''}` }, + provider: { getFhirVersion: () => '5.0.0' }, + languages: { parse: () => null } + }, { + hasDesignations: false, + designations: [], + httpLanguages: null + }); + + expander.valueSet = {}; + expander.limitCount = 1000; + expander.count = -1; + expander.offset = -1; + expander.hasExclusions = false; + expander.requiredSupplements = new Set(); + expander.usedSupplements = new Set(); + expander.map = new Map(); + expander.fullList = []; + expander.rootList = []; + expander.passesFilters = jest.fn(async () => true); + expander.includeCodeAndDescendants = jest.fn(async () => 1); + expander.checkProviderCanonicalStatus = jest.fn(); + expander.addParamUri = jest.fn(); + expander.addParamInt = jest.fn(); + + let thrown = null; + try { + await expander.includeCodes( + { system: 'http://example.org/cs/source-one' }, + 'ValueSet.compose.include[0]', + { vurl: 'http://example.org/vs/one|1.0.0', url: 'http://example.org/vs/one' }, + { include: [{ system: 'http://example.org/cs/source-one' }] }, + { isNull: true }, + {}, + false, + { value: false } + ); + } catch (error) { + thrown = error; + } + + expect(thrown).toBeTruthy(); + expect(thrown.cause).toBe('too-costly'); + }); }); diff --git a/tx/ocl/cs-ocl.cjs b/tx/ocl/cs-ocl.cjs index cdd13880..7ad48fff 100644 --- a/tx/ocl/cs-ocl.cjs +++ b/tx/ocl/cs-ocl.cjs @@ -16,6 +16,20 @@ const { patchSearchWorkerForOCLCodeFiltering } = require('./shared/patches'); patchSearchWorkerForOCLCodeFiltering(); +function normalizeCanonicalSystem(system) { + if (typeof system !== 'string') { + return system; + } + + const trimmed = system.trim(); + if (!trimmed) { + return trimmed; + } + + // Treat canonical URLs with and without trailing slash as equivalent. + return trimmed.replace(/\/+$/, ''); +} + class OCLCodeSystemProvider extends AbstractCodeSystemProvider { constructor(config = {}) { super(); @@ -230,7 +244,7 @@ class OCLCodeSystemProvider extends AbstractCodeSystemProvider { } const owner = source.owner || ''; - const canonical = source.canonical_url || source.canonicalUrl || ''; + const canonical = normalizeCanonicalSystem(source.canonical_url || source.canonicalUrl || ''); const shortCode = source.short_code || source.shortCode || source.id || source.mnemonic || source.name || ''; return `${owner}|${canonical}|${shortCode}`; } @@ -332,7 +346,7 @@ class OCLCodeSystemProvider extends AbstractCodeSystemProvider { return null; } - const canonicalUrl = source.canonical_url || source.canonicalUrl || source.url; + const canonicalUrl = normalizeCanonicalSystem(source.canonical_url || source.canonicalUrl || source.url); if (!canonicalUrl) { return null; } @@ -585,7 +599,20 @@ class OCLSourceCodeSystemProvider extends CodeSystemProvider { } contentMode() { - return this.isSystemComplete() ? CodeSystemContentMode.Complete : CodeSystemContentMode.NotPresent; + if (this.isSystemComplete()) { + return CodeSystemContentMode.Complete; + } + + // OCL CodeSystems are lazily materialized. Even when metadata is still + // warming up, concepts remain fetchable through the source concepts URL. + // Report at least fragment support so $expand does not fail early with + // "has no content" before lazy retrieval/hydration can run. + const hasRealCodeSystemResource = this.meta?.codeSystem instanceof CodeSystem; + if (hasRealCodeSystemResource && (this.meta?.conceptsUrl || this.conceptCache.size > 0 || this.pageCache.size > 0)) { + return CodeSystemContentMode.Fragment; + } + + return CodeSystemContentMode.NotPresent; } totalCount() { @@ -1152,13 +1179,64 @@ class OCLSourceCodeSystemProvider extends CodeSystemProvider { class OCLSourceCodeSystemFactory extends CodeSystemFactoryProvider { static factoriesByKey = new Map(); + static #normalizeSystem(system) { + return normalizeCanonicalSystem(system); + } + + static hasFactory(system, version = null) { + return !!OCLSourceCodeSystemFactory.#findFactory(system, version); + } + + static hasExactFactory(system, version = null) { + const normalizedSystem = OCLSourceCodeSystemFactory.#normalizeSystem(system); + if (!normalizedSystem) { + return false; + } + + const exactKey = `${normalizedSystem}|${version || ''}`; + return OCLSourceCodeSystemFactory.factoriesByKey.has(exactKey); + } + + static #findFactory(system, version = null) { + const normalizedSystem = OCLSourceCodeSystemFactory.#normalizeSystem(system); + if (!normalizedSystem) { + return null; + } + + const exactKey = `${normalizedSystem}|${version || ''}`; + const exact = OCLSourceCodeSystemFactory.factoriesByKey.get(exactKey); + if (exact) { + return exact; + } + + // When caller version does not match the registered one (or is absent), + // still reuse the factory for the same canonical system. + for (const [key, factory] of OCLSourceCodeSystemFactory.factoriesByKey.entries()) { + if (!factory) { + continue; + } + + const separatorIndex = key.lastIndexOf('|'); + if (separatorIndex < 0) { + continue; + } + + const keySystem = OCLSourceCodeSystemFactory.#normalizeSystem(key.substring(0, separatorIndex)); + if (keySystem === normalizedSystem) { + return factory; + } + } + + return null; + } + static syncCodeSystemResource(system, version = null, codeSystem = null) { - if (!system) { + const normalizedSystem = OCLSourceCodeSystemFactory.#normalizeSystem(system); + if (!normalizedSystem) { return; } - const key = `${system}|${version || ''}`; - const factory = OCLSourceCodeSystemFactory.factoriesByKey.get(key); + const factory = OCLSourceCodeSystemFactory.#findFactory(normalizedSystem, version); if (!factory) { return; } @@ -1182,6 +1260,11 @@ class OCLSourceCodeSystemFactory extends CodeSystemFactoryProvider { this.materializedConceptList = null; this.materializedConceptCount = -1; OCLSourceCodeSystemFactory.factoriesByKey.set(this.#resourceKey(), this); + + const unversionedKey = `${this.system()}|`; + if (!OCLSourceCodeSystemFactory.factoriesByKey.has(unversionedKey)) { + OCLSourceCodeSystemFactory.factoriesByKey.set(unversionedKey, this); + } // Load cold cache at construction this.#loadColdCache(); @@ -1254,8 +1337,9 @@ class OCLSourceCodeSystemFactory extends CodeSystemFactoryProvider { } static scheduleBackgroundLoadByKey(system, version = null, reason = 'valueset-expansion') { - const key = `${system}|${version || ''}`; - const factory = OCLSourceCodeSystemFactory.factoriesByKey.get(key); + const normalizedSystem = OCLSourceCodeSystemFactory.#normalizeSystem(system); + const key = `${normalizedSystem}|${version || ''}`; + const factory = OCLSourceCodeSystemFactory.#findFactory(normalizedSystem, version); if (!factory) { console.log(`[OCL] CodeSystem load not scheduled (factory unavailable): ${key}`); return false; @@ -1265,8 +1349,8 @@ class OCLSourceCodeSystemFactory extends CodeSystemFactoryProvider { } static checksumForResource(system, version = null) { - const key = `${system}|${version || ''}`; - const factory = OCLSourceCodeSystemFactory.factoriesByKey.get(key); + const normalizedSystem = OCLSourceCodeSystemFactory.#normalizeSystem(system); + const factory = OCLSourceCodeSystemFactory.#findFactory(normalizedSystem, version); if (!factory) { return null; } @@ -1634,7 +1718,8 @@ class OCLSourceCodeSystemFactory extends CodeSystemFactoryProvider { } #resourceKey() { - return `${this.system()}|${this.version() || ''}`; + const normalizedSystem = OCLSourceCodeSystemFactory.#normalizeSystem(this.system()); + return `${normalizedSystem}|${this.version() || ''}`; } currentChecksum() { @@ -1652,7 +1737,7 @@ class OCLSourceCodeSystemFactory extends CodeSystemFactoryProvider { } system() { - return this.meta.canonicalUrl; + return normalizeCanonicalSystem(this.meta.canonicalUrl); } name() { diff --git a/tx/ocl/shared/patches.cjs b/tx/ocl/shared/patches.cjs index 14a235ab..eae4750b 100644 --- a/tx/ocl/shared/patches.cjs +++ b/tx/ocl/shared/patches.cjs @@ -2,6 +2,8 @@ const { OCL_CODESYSTEM_MARKER_EXTENSION } = require('./constants'); const OCL_SEARCH_PATCH_FLAG = Symbol.for('fhirsmith.ocl.search.codesystem.code.patch'); const TXPARAMS_HASH_PATCH_FLAG = Symbol.for('fhirsmith.ocl.txparameters.hash.filter.patch'); +const OCL_EXPAND_WHOLE_SYSTEM_PATCH_FLAG = Symbol.for('fhirsmith.ocl.expand.whole-system.patch'); +const OCL_EXPAND_WHOLE_SYSTEM_RETRY_FLAG = Symbol.for('fhirsmith.ocl.expand.whole-system.retry'); function hasOCLCodeSystemMarker(resource) { const extensions = Array.isArray(resource?.extension) ? resource.extension : []; @@ -128,8 +130,95 @@ function ensureTxParametersHashIncludesFilter(TxParameters) { }); } +function isOclWholeSystemRetryCandidate(expander, cset, filter) { + if (!expander || !cset) { + return false; + } + + // Only touch the OCL ValueSet path where OCL helpers are attached. + const sourceValueSet = expander?.valueSet; + if (!sourceValueSet || typeof sourceValueSet.oclFetchConcepts !== 'function') { + return false; + } + + // This fallback is only for whole-system, unfiltered includes. + if (!cset.system || cset.concept || cset.filter) { + return false; + } + if (!filter || filter.isNull !== true) { + return false; + } + + // Only engage when request did not ask for an explicit page and we have a limit. + if (!(expander.count < 0 && expander.offset < 0 && expander.limitCount > 0)) { + return false; + } + + // Prevent recursive retries for the same include call path. + if (expander[OCL_EXPAND_WHOLE_SYSTEM_RETRY_FLAG] === true) { + return false; + } + + return true; +} + +function patchValueSetExpandWholeSystemForOcl() { + let expandModule; + try { + expandModule = require('../../workers/expand'); + } catch (_error) { + return; + } + + const ValueSetExpander = expandModule?.ValueSetExpander; + const proto = ValueSetExpander && ValueSetExpander.prototype; + if (!proto || proto[OCL_EXPAND_WHOLE_SYSTEM_PATCH_FLAG] === true || typeof proto.includeCodes !== 'function') { + return; + } + + const originalIncludeCodes = proto.includeCodes; + proto.includeCodes = async function patchedIncludeCodes(cset, path, vsSrc, compose, filter, expansion, excludeInactive, notClosed) { + try { + return await originalIncludeCodes.call(this, cset, path, vsSrc, compose, filter, expansion, excludeInactive, notClosed); + } catch (error) { + if (!isOclWholeSystemRetryCandidate(this, cset, filter)) { + throw error; + } + + const prevCount = this.count; + const prevOffset = this.offset; + this.count = this.limitCount; + this.offset = 0; + + // Mirror effective pagination into expansion parameters for transparency. + if (expansion && typeof this.addParamInt === 'function') { + this.addParamInt(expansion, 'offset', this.offset); + this.addParamInt(expansion, 'count', this.count); + expansion.offset = this.offset; + } + + this[OCL_EXPAND_WHOLE_SYSTEM_RETRY_FLAG] = true; + try { + return await originalIncludeCodes.call(this, cset, path, vsSrc, compose, filter, expansion, excludeInactive, notClosed); + } finally { + this[OCL_EXPAND_WHOLE_SYSTEM_RETRY_FLAG] = false; + this.count = prevCount; + this.offset = prevOffset; + } + } + }; + + Object.defineProperty(proto, OCL_EXPAND_WHOLE_SYSTEM_PATCH_FLAG, { + value: true, + writable: false, + configurable: false, + enumerable: false + }); +} + module.exports = { patchSearchWorkerForOCLCodeFiltering, ensureTxParametersHashIncludesFilter, + patchValueSetExpandWholeSystemForOcl, normalizeFilterForCacheKey }; diff --git a/tx/ocl/vs-ocl.cjs b/tx/ocl/vs-ocl.cjs index 56682799..6b33fc12 100644 --- a/tx/ocl/vs-ocl.cjs +++ b/tx/ocl/vs-ocl.cjs @@ -12,9 +12,24 @@ const { createOclHttpClient } = require('./http/client'); const { CACHE_VS_DIR, getCacheFilePath } = require('./cache/cache-paths'); const { ensureCacheDirectories, getColdCacheAgeMs, formatCacheAgeMinutes } = require('./cache/cache-utils'); const { computeValueSetExpansionFingerprint } = require('./fingerprint/fingerprint'); -const { ensureTxParametersHashIncludesFilter } = require('./shared/patches'); +const { ensureTxParametersHashIncludesFilter, patchValueSetExpandWholeSystemForOcl } = require('./shared/patches'); ensureTxParametersHashIncludesFilter(TxParameters); +patchValueSetExpandWholeSystemForOcl(); + +function normalizeCanonicalSystem(system) { + if (typeof system !== 'string') { + return system; + } + + const trimmed = system.trim(); + if (!trimmed) { + return trimmed; + } + + // Treat canonical URLs with and without trailing slash as equivalent. + return trimmed.replace(/\/+$/, ''); +} class OCLValueSetProvider extends AbstractValueSetProvider { constructor(config = {}) { @@ -379,7 +394,7 @@ class OCLValueSetProvider extends AbstractValueSetProvider { return null; } - const preferredSource = collection.preferred_source || collection.preferredSource || null; + const preferredSource = normalizeCanonicalSystem(collection.preferred_source || collection.preferredSource || null); const json = { resourceType: 'ValueSet', id, @@ -509,14 +524,8 @@ class OCLValueSetProvider extends AbstractValueSetProvider { if (!vs || !vs.jsonObj) { return; } - if (vs.jsonObj.compose && Array.isArray(vs.jsonObj.compose.include) && vs.jsonObj.compose.include.length > 0) { - return; - } const meta = this.#getCollectionMeta(vs); - if (!meta || !meta.conceptsUrl) { - return; - } const composeKey = vs.id || vs.url; if (this._composePromises.has(composeKey)) { @@ -525,17 +534,32 @@ class OCLValueSetProvider extends AbstractValueSetProvider { } const promise = (async () => { - const sources = await this.#fetchCollectionSources(meta); - if (!sources || sources.length === 0) { - return; + const existingInclude = Array.isArray(vs?.jsonObj?.compose?.include) + ? vs.jsonObj.compose.include + : []; + + // Always normalize existing compose entries first because discovery metadata + // can carry non-canonical preferred_source values. + const include = this.#normalizeComposeIncludes(existingInclude); + + // Reconcile with collection-resolved sources whenever available so $expand + // and direct CodeSystem lookups share the same canonical registry keys. + if (meta && (meta.conceptsUrl || meta.expansionUrl)) { + const sources = await this.#fetchCollectionSources(meta); + if (Array.isArray(sources) && sources.length > 0) { + include.push(...this.#normalizeComposeIncludes(sources)); + } } - vs.jsonObj.compose = { - include: sources.map(source => ({ - system: source.system, - version: source.version || undefined - })) - }; + // Preferred source is a fallback only when no resolvable include was found. + if (include.length === 0 && meta?.preferredSource) { + include.push(...this.#normalizeComposeIncludes([{ system: meta.preferredSource }])); + } + + const deduped = this.#dedupeComposeIncludes(include); + if (deduped.length > 0) { + vs.jsonObj.compose = { include: deduped }; + } })(); this._composePromises.set(composeKey, promise); @@ -546,6 +570,67 @@ class OCLValueSetProvider extends AbstractValueSetProvider { } } + #normalizeComposeIncludes(includeEntries) { + if (!Array.isArray(includeEntries) || includeEntries.length === 0) { + return []; + } + + const normalized = []; + for (const entry of includeEntries) { + if (!entry || typeof entry !== 'object') { + continue; + } + + const system = normalizeCanonicalSystem(entry.system); + if (!system) { + continue; + } + + let version = entry.version || null; + const hasAnyFactory = OCLSourceCodeSystemFactory.hasFactory(system, null); + const hasExactFactory = OCLSourceCodeSystemFactory.hasExactFactory(system, version); + + // If include.version does not match the registered OCL factory key, + // omit it so Provider lookup can reuse the already loaded canonical factory. + if (version && hasAnyFactory && !hasExactFactory) { + version = null; + } + + normalized.push({ + system, + version: version || undefined + }); + } + + return normalized; + } + + #dedupeComposeIncludes(includeEntries) { + const deduped = []; + const seen = new Set(); + + for (const include of includeEntries || []) { + const system = normalizeCanonicalSystem(include?.system); + if (!system) { + continue; + } + + const version = include?.version || ''; + const key = `${system}|${version}`; + if (seen.has(key)) { + continue; + } + + seen.add(key); + deduped.push({ + system, + version: version || undefined + }); + } + + return deduped; + } + async #fetchCollectionSources(meta) { const sourcesCacheKey = `${meta.owner || ''}|${meta.collectionId || ''}|${meta.conceptsUrl || ''}|${meta.expansionUrl || ''}`; if (this.collectionSourcesCache.has(sourcesCacheKey)) { @@ -572,7 +657,7 @@ class OCLValueSetProvider extends AbstractValueSetProvider { const shortCode = entry.short_code || entry.shortCode || entry.id || null; const version = entry.version || null; - const systemUrl = system || (owner && shortCode ? await this.#getSourceCanonicalUrl(owner, shortCode) : null); + const systemUrl = normalizeCanonicalSystem(system || (owner && shortCode ? await this.#getSourceCanonicalUrl(owner, shortCode) : null)); if (systemUrl && !seen.has(systemUrl)) { seen.add(systemUrl); sources.push({ system: systemUrl, version }); @@ -593,7 +678,7 @@ class OCLValueSetProvider extends AbstractValueSetProvider { const sourceKeys = await this.#fetchCollectionSourceKeys(meta.conceptsUrl, meta.owner || null); for (const { owner, source } of sourceKeys) { - const systemUrl = await this.#getSourceCanonicalUrl(owner, source); + const systemUrl = normalizeCanonicalSystem(await this.#getSourceCanonicalUrl(owner, source)); if (systemUrl && !seen.has(systemUrl)) { seen.add(systemUrl); sources.push({ system: systemUrl }); @@ -601,7 +686,10 @@ class OCLValueSetProvider extends AbstractValueSetProvider { } if (sources.length === 0 && meta.preferredSource) { - sources.push({ system: meta.preferredSource }); + const preferredSource = normalizeCanonicalSystem(meta.preferredSource); + if (preferredSource) { + sources.push({ system: preferredSource }); + } } this.collectionSourcesCache.set(sourcesCacheKey, sources); @@ -1147,7 +1235,7 @@ class OCLValueSetProvider extends AbstractValueSetProvider { const include = Array.isArray(vs?.jsonObj?.compose?.include) ? vs.jsonObj.compose.include : []; const checksums = {}; for (const item of include) { - const system = item?.system || null; + const system = normalizeCanonicalSystem(item?.system || null); if (!system) { continue; } From 9839e99cb332eb7b3ce8a6dfb72d2e19f9cdac61 Mon Sep 17 00:00:00 2001 From: Italo Macedo Date: Sat, 7 Mar 2026 16:03:25 -0300 Subject: [PATCH 10/15] Fixed display strict validation when informed --- tests/tx/validate.test.js | 30 +++++++++++++++++++++ tx/workers/validate.js | 57 ++++++++++++++++++++++++++++++++++++--- 2 files changed, 83 insertions(+), 4 deletions(-) diff --git a/tests/tx/validate.test.js b/tests/tx/validate.test.js index 7698ef04..01b73df5 100644 --- a/tests/tx/validate.test.js +++ b/tests/tx/validate.test.js @@ -467,5 +467,35 @@ describe('ValidateWorker', () => { resourceType: 'Parameters' })); }); + + test('POST codeableConcept validates same concept as GET url+code', async () => { + const { req, res } = createMockReqRes('POST', {}, { + resourceType: 'Parameters', + parameter: [ + { + name: 'codeableConcept', + valueCodeableConcept: { + coding: [ + { + system: 'http://hl7.org/fhir/administrative-gender', + code: 'male', + display: 'Not The Preferred Display' + } + ] + } + }, + { name: 'displayLanguage', valueString: 'pt-BR' } + ] + }); + + await worker.handleCodeSystem(req, res); + + const payload = res.json.mock.calls[0][0]; + const resultParam = payload.parameter.find(p => p.name === 'result'); + expect(resultParam.valueBoolean).toBe(true); + + const ccParam = payload.parameter.find(p => p.name === 'codeableConcept'); + expect(ccParam.valueCodeableConcept.coding[0].display).toBe('Not The Preferred Display'); + }); }); }); diff --git a/tx/workers/validate.js b/tx/workers/validate.js index 83008569..9b5b22a0 100644 --- a/tx/workers/validate.js +++ b/tx/workers/validate.js @@ -1892,10 +1892,14 @@ class ValidateWorker extends TerminologyWorker { throw new Issue('error', 'invalid', null, null, 'Unable to find code to validate (looked for coding | codeableConcept | code in parameters)', null, 400).handleAsOO(400); } + // For CodeSystem/$validate-code, align POST codeableConcept with GET url+code behavior + // unless display was explicitly provided as an operation parameter. + const codedForValidation = this.normalizeCodeSystemInput(coded, mode, params); + // Get the CodeSystem - from parameter or by url - const codeSystem = await this.resolveCodeSystem(params, txp, coded?.coding?.[0] ?? null, mode); + const codeSystem = await this.resolveCodeSystem(params, txp, codedForValidation?.coding?.[0] ?? null, mode); if (!codeSystem) { - if (!coded?.coding?.[0].system) { + if (!codedForValidation?.coding?.[0].system) { let msg = this.i18n.translate('Coding_has_no_system__cannot_validate', txp.HTTPLanguages, []); throw new Issue('warning', 'invalid', mode.issuePath, 'Coding_has_no_system__cannot_validate', msg, 'invalid-data'); } else { @@ -1907,7 +1911,10 @@ class ValidateWorker extends TerminologyWorker { } // Perform validation - const result = await this.doValidationCS(coded, codeSystem, txp, mode); + const result = await this.doValidationCS(codedForValidation, codeSystem, txp, mode); + if (codedForValidation !== coded) { + this.restoreOriginalCodeableConcept(result, coded); + } if (req) { req.logInfo = this.usedSources.join("|") + txp.logInfo(); } @@ -1966,8 +1973,13 @@ class ValidateWorker extends TerminologyWorker { 'Unable to find code to validate (looked for coding | codeableConcept | code in parameters =codingX:Coding)')); } + const codedForValidation = this.normalizeCodeSystemInput(coded, mode, params); + // Perform validation - const result = await this.doValidationCS(coded, csp, txp, mode); + const result = await this.doValidationCS(codedForValidation, csp, txp, mode); + if (codedForValidation !== coded) { + this.restoreOriginalCodeableConcept(result, coded); + } req.logInfo = this.usedSources.join("|") + txp.logInfo(); return res.json(result); @@ -2245,6 +2257,43 @@ class ValidateWorker extends TerminologyWorker { return null; } + normalizeCodeSystemInput(coded, mode, params) { + if (!coded || mode?.mode !== 'codeableConcept') { + return coded; + } + + // If display is explicitly requested as an operation parameter, preserve strict display validation. + if (this.getStringParam(params, 'display')) { + return coded; + } + + if (!Array.isArray(coded.coding) || coded.coding.length === 0) { + return coded; + } + + const normalized = { + ...coded, + coding: coded.coding.map(c => { + const nc = {}; + if (c?.system !== undefined) nc.system = c.system; + if (c?.code !== undefined) nc.code = c.code; + if (c?.version !== undefined) nc.version = c.version; + return nc; + }) + }; + return normalized; + } + + restoreOriginalCodeableConcept(result, originalCoded) { + if (!result?.parameter || !originalCoded) { + return; + } + const ccParam = result.parameter.find(p => p.name === 'codeableConcept'); + if (ccParam && ccParam.valueCodeableConcept) { + ccParam.valueCodeableConcept = originalCoded; + } + } + // ========== Validation Logic ========== /** From 4d20b9366348475a90c12b74518c9c8043c54c59 Mon Sep 17 00:00:00 2001 From: Italo Macedo Date: Sat, 7 Mar 2026 16:35:47 -0300 Subject: [PATCH 11/15] Fixing divergences in --- tests/cs/cs-loinc.test.js | 12 ++++++++++++ tests/tx/lookup.test.js | 36 ++++++++++++++++++++++++++++++++++++ tx/cs/cs-loinc.js | 7 +++++-- tx/workers/lookup.js | 21 +++++++++++++-------- 4 files changed, 66 insertions(+), 10 deletions(-) diff --git a/tests/cs/cs-loinc.test.js b/tests/cs/cs-loinc.test.js index 1c915615..625feb99 100644 --- a/tests/cs/cs-loinc.test.js +++ b/tests/cs/cs-loinc.test.js @@ -526,6 +526,18 @@ describe('LOINC Provider', () => { } }); + test('should not throw when display lookup runs with non-English context', async () => { + const ptContext = new OperationContext('pt-BR', opContext.i18n); + const ptProvider = await factory.build(ptContext, []); + + try { + const testCode = expectedResults.basic.knownCodes[0]; + await expect(ptProvider.display(testCode)).resolves.toBeDefined(); + } finally { + ptProvider.close(); + } + }); + test('should return correct code for context', async () => { const testCode = expectedResults.basic.knownCodes[0]; const result = await provider.locate(testCode); diff --git a/tests/tx/lookup.test.js b/tests/tx/lookup.test.js index 987a1d68..61f44deb 100644 --- a/tests/tx/lookup.test.js +++ b/tests/tx/lookup.test.js @@ -221,6 +221,42 @@ describe('Lookup Worker', () => { expect(response.status).toBe(200); }); + + test('should produce same result as GET for equivalent coding+version+displayLanguage inputs', async () => { + const getResponse = await request(app) + .get('/tx/r5/CodeSystem/$lookup') + .query({ + system: 'http://hl7.org/fhir/administrative-gender', + code: 'male', + displayLanguage: 'pt', + version: 'does-not-exist', + diagnostics: 'true' + }) + .set('Accept', 'application/json'); + + const postResponse = await request(app) + .post('/tx/r5/CodeSystem/$lookup') + .set('Accept', 'application/json') + .set('Content-Type', 'application/json') + .send({ + resourceType: 'Parameters', + parameter: [ + { + name: 'coding', + valueCoding: { + system: 'http://hl7.org/fhir/administrative-gender', + code: 'male' + } + }, + { name: 'displayLanguage', valueCode: 'pt' }, + { name: 'version', valueString: 'does-not-exist' }, + { name: 'diagnostics', valueBoolean: true } + ] + }); + + expect(postResponse.status).toBe(getResponse.status); + expect(postResponse.body).toEqual(getResponse.body); + }); }); describe('GET /tx/r5/CodeSystem/:id/$lookup', () => { diff --git a/tx/cs/cs-loinc.js b/tx/cs/cs-loinc.js index d1785bb5..efa6b9eb 100644 --- a/tx/cs/cs-loinc.js +++ b/tx/cs/cs-loinc.js @@ -203,9 +203,12 @@ class LoincServices extends BaseCSServices { // Use language-aware display logic if (this.opContext.langs && !this.opContext.langs.isEnglishOrNothing()) { const displays = await this.#getDisplaysForContext(ctxt, this.opContext.langs); + const requestedLanguages = Array.isArray(this.opContext.langs.languages) + ? this.opContext.langs.languages + : (Array.isArray(this.opContext.langs.langs) ? this.opContext.langs.langs : []); // Try to find exact language match - for (const lang of this.opContext.langs.langs) { + for (const lang of requestedLanguages) { for (const display of displays) { if (lang.matches(display.language, true)) { return display.value; @@ -214,7 +217,7 @@ class LoincServices extends BaseCSServices { } // Try partial language match - for (const lang of this.opContext.langs.langs) { + for (const lang of requestedLanguages) { for (const display of displays) { if (lang.matches(display.language, false)) { return display.value; diff --git a/tx/workers/lookup.js b/tx/workers/lookup.js index 3373a511..e8cacfbe 100644 --- a/tx/workers/lookup.js +++ b/tx/workers/lookup.js @@ -106,11 +106,17 @@ class LookupWorker extends TerminologyWorker { // Determine how the code system is identified let csProvider; + let system; + let version; let code; if (params.has('coding')) { // Coding parameter provided - extract system, version, code const coding = params.get('coding'); + system = coding.system; + version = coding.version || params.get('version') || ''; + code = coding.code; + if (!coding.system) { return res.status(400).json(this.operationOutcome('error', 'invalid', 'Coding parameter must include a system')); @@ -120,16 +126,10 @@ class LookupWorker extends TerminologyWorker { 'Coding parameter must include a code')); } - // Allow complete or fragment content modes, nullOk = true to handle not-found ourselves - csProvider = await this.findCodeSystem(coding.system, coding.version || '', txp, ['complete', 'fragment'], true); - this.seeSourceProvider(csProvider, coding.system); - code = coding.code; - } else if (params.has('system') && params.has('code')) { // system + code parameters - csProvider = await this.findCodeSystem(params.get('system'), params.get('version') || '', txp, ['complete', 'fragment'], - null, true, false, false, txp.supplements); - this.seeSourceProvider(csProvider, params.get('system')); + system = params.get('system'); + version = params.get('version') || ''; code = params.get('code'); } else { @@ -137,6 +137,11 @@ class LookupWorker extends TerminologyWorker { 'Must provide either coding parameter, or system and code parameters')); } + // Use one common resolution path so equivalent GET/POST inputs behave identically. + csProvider = await this.findCodeSystem(system, version, txp, ['complete', 'fragment'], + null, true, false, false, txp.supplements); + this.seeSourceProvider(csProvider, system); + if (!csProvider) { const systemUrl = params.system || params.coding?.system; const versionStr = params.version || params.coding?.version; From e3ef6cf88cf5519b974a93dc05bdad2b11554fae Mon Sep 17 00:00:00 2001 From: Italo Macedo Date: Sat, 7 Mar 2026 16:51:27 -0300 Subject: [PATCH 12/15] fix(tx/validate): align POST CodeSystem/-code normalization with GET semantics The POST Parameters form of CodeSystem/-code was not internally equivalent to the GET query form for logically identical requests. GET behavior is the reference and was already correct, but POST could bypass the same display/code validation outcome. Why this change was needed: - Equivalent requests were producing different results depending on transport shape. - GET with system+code+display+version correctly invalidated mismatched display. - POST with parameter[coding] + parameter[display|version|displayLanguage] could take a different normalization path and not carry operation-level display/version into the internal Coding used for validation. - This created inconsistent validation behavior and made client behavior dependent on request encoding instead of terminology semantics. Root cause: - CodeSystem input normalization handled codeableConcept adjustments, but did not fully normalize the coding mode to merge operation parameters into the same internal model used by GET-style inputs. What was changed: - In CodeSystem normalization, for mode=coding: - map operation system/url to coding.system when missing - map operation code to coding.code when missing - map operation version to coding.version when missing - apply operation display to coding.display (so display checks match GET behavior) - Kept GET behavior unchanged as required. - Added regression test proving GET and POST(coding) produce identical result/message for equivalent parameters, with GET treated as source of truth. Why files outside tx/ocl changed: - The inconsistency is in shared -code request parsing/normalization logic (tx/workers/validate.js), not in OCL cache/data assets. - The safety net belongs in shared tx tests (tests/tx/validate.test.js) to prevent future regressions across all providers, including OCL-backed flows. --- tests/tx/validate.test.js | 49 +++++++++++++++++++++++++++++++++++++++ tx/workers/validate.js | 35 +++++++++++++++++++++++++++- 2 files changed, 83 insertions(+), 1 deletion(-) diff --git a/tests/tx/validate.test.js b/tests/tx/validate.test.js index 01b73df5..dcd655ce 100644 --- a/tests/tx/validate.test.js +++ b/tests/tx/validate.test.js @@ -117,6 +117,18 @@ function getCodeSystem() { ); } +function readPrimitiveParam(parameters, name) { + const p = (parameters.parameter || []).find(param => param.name === name); + if (!p) { + return undefined; + } + if (p.valueBoolean !== undefined) return p.valueBoolean; + if (p.valueString !== undefined) return p.valueString; + if (p.valueCode !== undefined) return p.valueCode; + if (p.valueUri !== undefined) return p.valueUri; + return undefined; +} + describe('ValidateWorker', () => { let worker; let opContext; @@ -497,5 +509,42 @@ describe('ValidateWorker', () => { const ccParam = payload.parameter.find(p => p.name === 'codeableConcept'); expect(ccParam.valueCodeableConcept.coding[0].display).toBe('Not The Preferred Display'); }); + + test('CodeSystem validate-code GET and POST(coding) stay equivalent for display/version/language', async () => { + const getCall = createMockReqRes('GET', { + system: 'http://hl7.org/fhir/administrative-gender', + code: 'male', + displayLanguage: 'en', + version: '5.0.0', + display: 'History of Immunization notes' + }); + + const postCall = createMockReqRes('POST', {}, { + resourceType: 'Parameters', + parameter: [ + { + name: 'coding', + valueCoding: { + system: 'http://hl7.org/fhir/administrative-gender', + code: 'male' + } + }, + { name: 'displayLanguage', valueCode: 'en' }, + { name: 'version', valueString: '5.0.0' }, + { name: 'display', valueString: 'History of Immunization notes' } + ] + }); + + await worker.handleCodeSystem(getCall.req, getCall.res); + await worker.handleCodeSystem(postCall.req, postCall.res); + + const getPayload = getCall.res.json.mock.calls[0][0]; + const postPayload = postCall.res.json.mock.calls[0][0]; + + // GET is the source of truth for equivalent POST normalization. + expect(readPrimitiveParam(postPayload, 'result')).toBe(readPrimitiveParam(getPayload, 'result')); + expect(readPrimitiveParam(postPayload, 'message')).toBe(readPrimitiveParam(getPayload, 'message')); + expect(readPrimitiveParam(getPayload, 'result')).toBe(false); + }); }); }); diff --git a/tx/workers/validate.js b/tx/workers/validate.js index 9b5b22a0..33418ec6 100644 --- a/tx/workers/validate.js +++ b/tx/workers/validate.js @@ -2258,7 +2258,40 @@ class ValidateWorker extends TerminologyWorker { } normalizeCodeSystemInput(coded, mode, params) { - if (!coded || mode?.mode !== 'codeableConcept') { + if (!coded) { + return coded; + } + + // Keep POST Coding + operation params equivalent to GET ?system&code&display&version. + if (mode?.mode === 'coding' && Array.isArray(coded.coding) && coded.coding.length > 0) { + const [first, ...rest] = coded.coding; + const normalizedCoding = {...first}; + + const opSystem = this.getStringParam(params, 'system') || this.getStringParam(params, 'url'); + const opCode = this.getStringParam(params, 'code'); + const opVersion = this.getStringParam(params, 'version'); + const opDisplay = this.getStringParam(params, 'display'); + + if (!normalizedCoding.system && opSystem) { + normalizedCoding.system = opSystem; + } + if (!normalizedCoding.code && opCode) { + normalizedCoding.code = opCode; + } + if (!normalizedCoding.version && opVersion) { + normalizedCoding.version = opVersion; + } + if (opDisplay) { + normalizedCoding.display = opDisplay; + } + + return { + ...coded, + coding: [normalizedCoding, ...rest] + }; + } + + if (mode?.mode !== 'codeableConcept') { return coded; } From 7ae7bd14ecc7035e334c5e9f41db425776a941fa Mon Sep 17 00:00:00 2001 From: Italo Macedo Date: Sat, 7 Mar 2026 17:06:12 -0300 Subject: [PATCH 13/15] Fixed cold cache mock for test execution --- tx/ocl/cs-ocl.cjs | 16 +++++++++++++--- 1 file changed, 13 insertions(+), 3 deletions(-) diff --git a/tx/ocl/cs-ocl.cjs b/tx/ocl/cs-ocl.cjs index 7ad48fff..75afab31 100644 --- a/tx/ocl/cs-ocl.cjs +++ b/tx/ocl/cs-ocl.cjs @@ -1393,12 +1393,22 @@ class OCLSourceCodeSystemFactory extends CodeSystemFactoryProvider { scheduleBackgroundLoad(reason = 'request') { this.#syncWarmStateWithChecksum(); + const cacheFilePath = getCacheFilePath(CACHE_CS_DIR, this.system(), this.version()); + const cacheAgeMs = getColdCacheAgeMs(cacheFilePath); + + // If warm state is complete but cold cache is stale, force a refresh run. + // This keeps warm data available while ensuring stale cold-cache files are replaced. if (this.isComplete) { - return; + if (cacheAgeMs == null || cacheAgeMs < COLD_CACHE_FRESHNESS_MS) { + return; + } + + this.isComplete = false; + if (this.meta?.codeSystem?.jsonObj?.content === CodeSystemContentMode.Complete) { + this.meta.codeSystem.jsonObj.content = CodeSystemContentMode.Fragment; + } } - const cacheFilePath = getCacheFilePath(CACHE_CS_DIR, this.system(), this.version()); - const cacheAgeMs = getColdCacheAgeMs(cacheFilePath); if (cacheAgeMs != null && cacheAgeMs < COLD_CACHE_FRESHNESS_MS) { console.log(`[OCL] Skipping warm-up for CodeSystem ${this.system()} (cold cache age: ${formatCacheAgeMinutes(cacheAgeMs)})`); return; From a6fd4a6ba9cc9bc1f8f831c78fcebf55a68296d4 Mon Sep 17 00:00:00 2001 From: Italo Macedo Date: Sat, 7 Mar 2026 21:59:18 -0300 Subject: [PATCH 14/15] Reverting TX Workers modification --- tx/workers/lookup.js | 21 ++++------ tx/workers/validate.js | 90 ++---------------------------------------- 2 files changed, 12 insertions(+), 99 deletions(-) diff --git a/tx/workers/lookup.js b/tx/workers/lookup.js index e8cacfbe..3373a511 100644 --- a/tx/workers/lookup.js +++ b/tx/workers/lookup.js @@ -106,17 +106,11 @@ class LookupWorker extends TerminologyWorker { // Determine how the code system is identified let csProvider; - let system; - let version; let code; if (params.has('coding')) { // Coding parameter provided - extract system, version, code const coding = params.get('coding'); - system = coding.system; - version = coding.version || params.get('version') || ''; - code = coding.code; - if (!coding.system) { return res.status(400).json(this.operationOutcome('error', 'invalid', 'Coding parameter must include a system')); @@ -126,10 +120,16 @@ class LookupWorker extends TerminologyWorker { 'Coding parameter must include a code')); } + // Allow complete or fragment content modes, nullOk = true to handle not-found ourselves + csProvider = await this.findCodeSystem(coding.system, coding.version || '', txp, ['complete', 'fragment'], true); + this.seeSourceProvider(csProvider, coding.system); + code = coding.code; + } else if (params.has('system') && params.has('code')) { // system + code parameters - system = params.get('system'); - version = params.get('version') || ''; + csProvider = await this.findCodeSystem(params.get('system'), params.get('version') || '', txp, ['complete', 'fragment'], + null, true, false, false, txp.supplements); + this.seeSourceProvider(csProvider, params.get('system')); code = params.get('code'); } else { @@ -137,11 +137,6 @@ class LookupWorker extends TerminologyWorker { 'Must provide either coding parameter, or system and code parameters')); } - // Use one common resolution path so equivalent GET/POST inputs behave identically. - csProvider = await this.findCodeSystem(system, version, txp, ['complete', 'fragment'], - null, true, false, false, txp.supplements); - this.seeSourceProvider(csProvider, system); - if (!csProvider) { const systemUrl = params.system || params.coding?.system; const versionStr = params.version || params.coding?.version; diff --git a/tx/workers/validate.js b/tx/workers/validate.js index 33418ec6..83008569 100644 --- a/tx/workers/validate.js +++ b/tx/workers/validate.js @@ -1892,14 +1892,10 @@ class ValidateWorker extends TerminologyWorker { throw new Issue('error', 'invalid', null, null, 'Unable to find code to validate (looked for coding | codeableConcept | code in parameters)', null, 400).handleAsOO(400); } - // For CodeSystem/$validate-code, align POST codeableConcept with GET url+code behavior - // unless display was explicitly provided as an operation parameter. - const codedForValidation = this.normalizeCodeSystemInput(coded, mode, params); - // Get the CodeSystem - from parameter or by url - const codeSystem = await this.resolveCodeSystem(params, txp, codedForValidation?.coding?.[0] ?? null, mode); + const codeSystem = await this.resolveCodeSystem(params, txp, coded?.coding?.[0] ?? null, mode); if (!codeSystem) { - if (!codedForValidation?.coding?.[0].system) { + if (!coded?.coding?.[0].system) { let msg = this.i18n.translate('Coding_has_no_system__cannot_validate', txp.HTTPLanguages, []); throw new Issue('warning', 'invalid', mode.issuePath, 'Coding_has_no_system__cannot_validate', msg, 'invalid-data'); } else { @@ -1911,10 +1907,7 @@ class ValidateWorker extends TerminologyWorker { } // Perform validation - const result = await this.doValidationCS(codedForValidation, codeSystem, txp, mode); - if (codedForValidation !== coded) { - this.restoreOriginalCodeableConcept(result, coded); - } + const result = await this.doValidationCS(coded, codeSystem, txp, mode); if (req) { req.logInfo = this.usedSources.join("|") + txp.logInfo(); } @@ -1973,13 +1966,8 @@ class ValidateWorker extends TerminologyWorker { 'Unable to find code to validate (looked for coding | codeableConcept | code in parameters =codingX:Coding)')); } - const codedForValidation = this.normalizeCodeSystemInput(coded, mode, params); - // Perform validation - const result = await this.doValidationCS(codedForValidation, csp, txp, mode); - if (codedForValidation !== coded) { - this.restoreOriginalCodeableConcept(result, coded); - } + const result = await this.doValidationCS(coded, csp, txp, mode); req.logInfo = this.usedSources.join("|") + txp.logInfo(); return res.json(result); @@ -2257,76 +2245,6 @@ class ValidateWorker extends TerminologyWorker { return null; } - normalizeCodeSystemInput(coded, mode, params) { - if (!coded) { - return coded; - } - - // Keep POST Coding + operation params equivalent to GET ?system&code&display&version. - if (mode?.mode === 'coding' && Array.isArray(coded.coding) && coded.coding.length > 0) { - const [first, ...rest] = coded.coding; - const normalizedCoding = {...first}; - - const opSystem = this.getStringParam(params, 'system') || this.getStringParam(params, 'url'); - const opCode = this.getStringParam(params, 'code'); - const opVersion = this.getStringParam(params, 'version'); - const opDisplay = this.getStringParam(params, 'display'); - - if (!normalizedCoding.system && opSystem) { - normalizedCoding.system = opSystem; - } - if (!normalizedCoding.code && opCode) { - normalizedCoding.code = opCode; - } - if (!normalizedCoding.version && opVersion) { - normalizedCoding.version = opVersion; - } - if (opDisplay) { - normalizedCoding.display = opDisplay; - } - - return { - ...coded, - coding: [normalizedCoding, ...rest] - }; - } - - if (mode?.mode !== 'codeableConcept') { - return coded; - } - - // If display is explicitly requested as an operation parameter, preserve strict display validation. - if (this.getStringParam(params, 'display')) { - return coded; - } - - if (!Array.isArray(coded.coding) || coded.coding.length === 0) { - return coded; - } - - const normalized = { - ...coded, - coding: coded.coding.map(c => { - const nc = {}; - if (c?.system !== undefined) nc.system = c.system; - if (c?.code !== undefined) nc.code = c.code; - if (c?.version !== undefined) nc.version = c.version; - return nc; - }) - }; - return normalized; - } - - restoreOriginalCodeableConcept(result, originalCoded) { - if (!result?.parameter || !originalCoded) { - return; - } - const ccParam = result.parameter.find(p => p.name === 'codeableConcept'); - if (ccParam && ccParam.valueCodeableConcept) { - ccParam.valueCodeableConcept = originalCoded; - } - } - // ========== Validation Logic ========== /** From 3dd545c9fce339b7c458964e06278c8e40c44ba1 Mon Sep 17 00:00:00 2001 From: Italo Macedo Date: Sat, 7 Mar 2026 22:07:57 -0300 Subject: [PATCH 15/15] Reverting tests for lookup and validate --- tests/tx/lookup.test.js | 36 ------------------ tests/tx/validate.test.js | 79 --------------------------------------- 2 files changed, 115 deletions(-) diff --git a/tests/tx/lookup.test.js b/tests/tx/lookup.test.js index 61f44deb..987a1d68 100644 --- a/tests/tx/lookup.test.js +++ b/tests/tx/lookup.test.js @@ -221,42 +221,6 @@ describe('Lookup Worker', () => { expect(response.status).toBe(200); }); - - test('should produce same result as GET for equivalent coding+version+displayLanguage inputs', async () => { - const getResponse = await request(app) - .get('/tx/r5/CodeSystem/$lookup') - .query({ - system: 'http://hl7.org/fhir/administrative-gender', - code: 'male', - displayLanguage: 'pt', - version: 'does-not-exist', - diagnostics: 'true' - }) - .set('Accept', 'application/json'); - - const postResponse = await request(app) - .post('/tx/r5/CodeSystem/$lookup') - .set('Accept', 'application/json') - .set('Content-Type', 'application/json') - .send({ - resourceType: 'Parameters', - parameter: [ - { - name: 'coding', - valueCoding: { - system: 'http://hl7.org/fhir/administrative-gender', - code: 'male' - } - }, - { name: 'displayLanguage', valueCode: 'pt' }, - { name: 'version', valueString: 'does-not-exist' }, - { name: 'diagnostics', valueBoolean: true } - ] - }); - - expect(postResponse.status).toBe(getResponse.status); - expect(postResponse.body).toEqual(getResponse.body); - }); }); describe('GET /tx/r5/CodeSystem/:id/$lookup', () => { diff --git a/tests/tx/validate.test.js b/tests/tx/validate.test.js index dcd655ce..7698ef04 100644 --- a/tests/tx/validate.test.js +++ b/tests/tx/validate.test.js @@ -117,18 +117,6 @@ function getCodeSystem() { ); } -function readPrimitiveParam(parameters, name) { - const p = (parameters.parameter || []).find(param => param.name === name); - if (!p) { - return undefined; - } - if (p.valueBoolean !== undefined) return p.valueBoolean; - if (p.valueString !== undefined) return p.valueString; - if (p.valueCode !== undefined) return p.valueCode; - if (p.valueUri !== undefined) return p.valueUri; - return undefined; -} - describe('ValidateWorker', () => { let worker; let opContext; @@ -479,72 +467,5 @@ describe('ValidateWorker', () => { resourceType: 'Parameters' })); }); - - test('POST codeableConcept validates same concept as GET url+code', async () => { - const { req, res } = createMockReqRes('POST', {}, { - resourceType: 'Parameters', - parameter: [ - { - name: 'codeableConcept', - valueCodeableConcept: { - coding: [ - { - system: 'http://hl7.org/fhir/administrative-gender', - code: 'male', - display: 'Not The Preferred Display' - } - ] - } - }, - { name: 'displayLanguage', valueString: 'pt-BR' } - ] - }); - - await worker.handleCodeSystem(req, res); - - const payload = res.json.mock.calls[0][0]; - const resultParam = payload.parameter.find(p => p.name === 'result'); - expect(resultParam.valueBoolean).toBe(true); - - const ccParam = payload.parameter.find(p => p.name === 'codeableConcept'); - expect(ccParam.valueCodeableConcept.coding[0].display).toBe('Not The Preferred Display'); - }); - - test('CodeSystem validate-code GET and POST(coding) stay equivalent for display/version/language', async () => { - const getCall = createMockReqRes('GET', { - system: 'http://hl7.org/fhir/administrative-gender', - code: 'male', - displayLanguage: 'en', - version: '5.0.0', - display: 'History of Immunization notes' - }); - - const postCall = createMockReqRes('POST', {}, { - resourceType: 'Parameters', - parameter: [ - { - name: 'coding', - valueCoding: { - system: 'http://hl7.org/fhir/administrative-gender', - code: 'male' - } - }, - { name: 'displayLanguage', valueCode: 'en' }, - { name: 'version', valueString: '5.0.0' }, - { name: 'display', valueString: 'History of Immunization notes' } - ] - }); - - await worker.handleCodeSystem(getCall.req, getCall.res); - await worker.handleCodeSystem(postCall.req, postCall.res); - - const getPayload = getCall.res.json.mock.calls[0][0]; - const postPayload = postCall.res.json.mock.calls[0][0]; - - // GET is the source of truth for equivalent POST normalization. - expect(readPrimitiveParam(postPayload, 'result')).toBe(readPrimitiveParam(getPayload, 'result')); - expect(readPrimitiveParam(postPayload, 'message')).toBe(readPrimitiveParam(getPayload, 'message')); - expect(readPrimitiveParam(getPayload, 'result')).toBe(false); - }); }); });