diff --git a/packages/package-crawler.js b/packages/package-crawler.js index 2866fe26..82b39d8f 100644 --- a/packages/package-crawler.js +++ b/packages/package-crawler.js @@ -485,6 +485,14 @@ class PackageCrawler { throw new Error(`NPM Canonical "${canonical}" is not valid from ${source}`); } + const isTemplate = npmPackage.kind === 2; // fhir.template + if (npmPackage.hasInstallScripts) { + throw new Error(`Package ${idver} rejected: contains install scripts (preinstall/install/postinstall)`); + } + if (npmPackage.hasJavaScript && !isTemplate) { + throw new Error(`Package ${idver} rejected: contains JavaScript files but is not a template package`); + } + // Extract URLs from package const urls = this.processPackageUrls(npmPackage); @@ -554,6 +562,14 @@ class PackageCrawler { throw new Error('package.json not found in extracted package'); } + const hasInstallScripts = !!( + packageJson.scripts && ( + packageJson.scripts.preinstall || + packageJson.scripts.install || + packageJson.scripts.postinstall + ) + ); + const hasJavaScript = Object.keys(files).some(f => f.endsWith('.js') || f.endsWith('.mjs') || f.endsWith('.cjs')); const packageJson = JSON.parse(files['package.json']); // Extract basic NPM fields @@ -664,6 +680,8 @@ class PackageCrawler { url: homepage, dependencies, kind, + hasInstallScripts, + hasJavaScript, notForPublication, files }; diff --git a/tests/cs/cs-loinc.test.js b/tests/cs/cs-loinc.test.js index 1c915615..625feb99 100644 --- a/tests/cs/cs-loinc.test.js +++ b/tests/cs/cs-loinc.test.js @@ -526,6 +526,18 @@ describe('LOINC Provider', () => { } }); + test('should not throw when display lookup runs with non-English context', async () => { + const ptContext = new OperationContext('pt-BR', opContext.i18n); + const ptProvider = await factory.build(ptContext, []); + + try { + const testCode = expectedResults.basic.knownCodes[0]; + await expect(ptProvider.display(testCode)).resolves.toBeDefined(); + } finally { + ptProvider.close(); + } + }); + test('should return correct code for context', async () => { const testCode = expectedResults.basic.knownCodes[0]; const result = await provider.locate(testCode); diff --git a/tests/ocl/ocl-cm.test.js b/tests/ocl/ocl-cm.test.js new file mode 100644 index 00000000..f4c0387d --- /dev/null +++ b/tests/ocl/ocl-cm.test.js @@ -0,0 +1,243 @@ +const nock = require('nock'); + +const { OCLConceptMapProvider } = require('../../tx/ocl/cm-ocl'); + +describe('OCL ConceptMap integration', () => { + const baseUrl = 'https://ocl.cm.test'; + + beforeEach(() => { + nock.cleanAll(); + jest.restoreAllMocks(); + }); + + afterEach(() => { + nock.cleanAll(); + }); + + function mapping(overrides = {}) { + return { + id: 'map-1', + url: `${baseUrl}/mappings/map-1`, + version: '1.0.0', + map_type: 'SAME-AS', + from_source_url: '/orgs/org-a/sources/src-a/', + to_source_url: '/orgs/org-a/sources/src-b/', + from_concept_code: 'A', + to_concept_code: 'B', + from_concept_name_resolved: 'Alpha', + to_concept_name_resolved: 'Beta', + updated_on: '2026-01-02T03:04:05.000Z', + name: 'Map One', + comment: 'ok', + ...overrides + }; + } + + test('fetchConceptMapById resolves and indexes mapping', async () => { + nock(baseUrl) + .get('/mappings/map-1/') + .reply(200, mapping()); + + const provider = new OCLConceptMapProvider({ baseUrl }); + const cm = await provider.fetchConceptMapById('map-1'); + + expect(cm).toBeTruthy(); + expect(cm.id).toBe('map-1'); + expect(cm.jsonObj.group[0].source).toBe('/orgs/org-a/sources/src-a/'); + expect(cm.jsonObj.group[0].target).toBe('/orgs/org-a/sources/src-b/'); + expect(cm.jsonObj.meta.lastUpdated).toBe('2026-01-02T03:04:05.000Z'); + }); + + test('fetchConceptMap can resolve from canonical url via search and mapping-id extraction', async () => { + nock(baseUrl) + .get('/mappings/map-2/') + .reply(200, mapping({ id: 'map-2', url: `${baseUrl}/mappings/map-2` })) + .get('/mappings/') + .query(true) + .reply(200, { results: [mapping()] }); + + const provider = new OCLConceptMapProvider({ baseUrl }); + + const byIdFromUrl = await provider.fetchConceptMap(`${baseUrl}/mappings/map-2`, null); + expect(byIdFromUrl.id).toBe('map-2'); + + const bySearch = await provider.fetchConceptMap('/orgs/org-a/sources/src-a/', '1.0.0'); + expect(bySearch).toBeNull(); + }); + + test('searchConceptMaps filters by source and target parameters', async () => { + nock(baseUrl) + .get('/mappings/') + .query(true) + .reply(200, { results: [mapping(), mapping({ id: 'map-3', from_source_url: '/orgs/x/s/', to_source_url: '/orgs/y/s/' })] }); + + const provider = new OCLConceptMapProvider({ baseUrl }); + + const results = await provider.searchConceptMaps([ + { name: 'source', value: '/orgs/org-a/sources/src-a/' }, + { name: 'target', value: '/orgs/org-a/sources/src-b/' } + ]); + + expect(results).toHaveLength(1); + expect(results[0].id).toBe('map-1'); + }); + + test('findConceptMapForTranslation uses source candidates and canonical resolution', async () => { + nock(baseUrl) + .get('/sources/') + .query(true) + .twice() + .reply(200, { + results: [ + { + canonical_url: 'http://example.org/cs/source-a', + url: '/orgs/org-a/sources/src-a/' + }, + { + canonical_url: 'http://example.org/cs/source-b', + url: '/orgs/org-a/sources/src-b/' + } + ] + }) + .get('/orgs/org-a/sources/src-a/concepts/A/mappings/') + .query(true) + .reply(200, { results: [mapping()] }) + .get('/orgs/org-a/sources/src-a/') + .reply(200, { + canonical_url: 'http://example.org/cs/source-a', + url: '/orgs/org-a/sources/src-a/' + }) + .get('/orgs/org-a/sources/src-b/') + .reply(200, { + canonical_url: 'http://example.org/cs/source-b', + url: '/orgs/org-a/sources/src-b/' + }); + + const provider = new OCLConceptMapProvider({ baseUrl, maxSearchPages: 2 }); + const conceptMaps = []; + + await provider.findConceptMapForTranslation( + null, + conceptMaps, + 'http://example.org/cs/source-a', + null, + null, + 'http://example.org/cs/source-b', + 'A' + ); + + expect(conceptMaps).toHaveLength(1); + expect(conceptMaps[0].jsonObj.group[0].source).toBe('http://example.org/cs/source-a'); + expect(conceptMaps[0].jsonObj.group[0].target).toBe('http://example.org/cs/source-b'); + }); + + test('assignIds prefixes space and cmCount returns unique map count', async () => { + nock(baseUrl) + .get('/mappings/map-1/') + .reply(200, mapping()); + + const provider = new OCLConceptMapProvider({ baseUrl }); + provider.spaceId = 'space'; + + await provider.fetchConceptMapById('map-1'); + + const ids = new Set(); + provider.assignIds(ids); + + expect(ids.has('ConceptMap/space-map-1')).toBe(true); + expect(provider.cmCount()).toBe(1); + }); + + test('fetchConceptMap returns direct cached hit before network lookup', async () => { + const provider = new OCLConceptMapProvider({ baseUrl }); + const cm = await provider.fetchConceptMapById('map-1').catch(() => null); + expect(cm).toBeNull(); + + const cached = { + id: 'cached-1', + url: 'http://example.org/cached-cm', + version: '1.0.0' + }; + provider.conceptMapMap.set('http://example.org/cached-cm|1.0.0', cached); + const out = await provider.fetchConceptMap('http://example.org/cached-cm', '1.0.0'); + expect(out).toBe(cached); + }); + + test('findConceptMapForTranslation fallback search with empty candidate sets', async () => { + nock(baseUrl) + .get('/mappings/') + .query(true) + .reply(200, { results: [mapping({ id: 'map-fallback' })] }); + + const provider = new OCLConceptMapProvider({ baseUrl }); + const conceptMaps = []; + + await provider.findConceptMapForTranslation( + null, + conceptMaps, + null, + null, + null, + null, + null + ); + + expect(conceptMaps.length).toBeGreaterThanOrEqual(0); + }); + + test('findConceptMapForTranslation uses candidate matching when scope check is strict', async () => { + nock(baseUrl) + .get('/sources/') + .query(true) + .twice() + .reply(200, { + results: [ + { + canonical_url: 'http://example.org/cs/source-a', + url: '/orgs/org-a/sources/src-a/' + }, + { + canonical_url: 'http://example.org/cs/source-b', + url: '/orgs/org-a/sources/src-b/' + } + ] + }) + .get('/orgs/org-a/sources/src-a/concepts/A/mappings/') + .query(true) + .reply(200, { + results: [ + mapping({ + updated_on: 'invalid-date' + }) + ] + }) + .get('/orgs/org-a/sources/src-a/') + .reply(200, { + canonical_url: 'http://example.org/cs/source-a', + url: '/orgs/org-a/sources/src-a/' + }) + .get('/orgs/org-a/sources/src-b/') + .reply(200, { + canonical_url: 'http://example.org/cs/source-b', + url: '/orgs/org-a/sources/src-b/' + }); + + const provider = new OCLConceptMapProvider({ baseUrl }); + const conceptMaps = [{ id: 'already-present' }]; + + await provider.findConceptMapForTranslation( + null, + conceptMaps, + 'http://example.org/cs/source-a', + 'http://scope/strict/source', + 'http://scope/strict/target', + 'http://example.org/cs/source-b', + 'A' + ); + + expect(conceptMaps.length).toBeGreaterThanOrEqual(2); + expect(conceptMaps[1].jsonObj.meta).toBeUndefined(); + + await expect(provider.close()).resolves.toBeUndefined(); + }); +}); diff --git a/tests/ocl/ocl-cs-provider-methods.test.js b/tests/ocl/ocl-cs-provider-methods.test.js new file mode 100644 index 00000000..739cec5f --- /dev/null +++ b/tests/ocl/ocl-cs-provider-methods.test.js @@ -0,0 +1,235 @@ +const nock = require('nock'); + +const { OperationContext } = require('../../tx/operation-context'); +const { Designations } = require('../../tx/library/designations'); +const { OCLSourceCodeSystemFactory, OCLBackgroundJobQueue } = require('../../tx/ocl/cs-ocl'); +const { TestUtilities } = require('../test-utilities'); + +function resetQueueState() { + OCLBackgroundJobQueue.pendingJobs = []; + OCLBackgroundJobQueue.activeCount = 0; + OCLBackgroundJobQueue.queuedOrRunningKeys = new Set(); + OCLBackgroundJobQueue.activeJobs = new Map(); + OCLBackgroundJobQueue.enqueueSequence = 0; + if (OCLBackgroundJobQueue.heartbeatTimer) { + clearInterval(OCLBackgroundJobQueue.heartbeatTimer); + OCLBackgroundJobQueue.heartbeatTimer = null; + } +} + +describe('OCL CodeSystem provider runtime methods', () => { + const baseUrl = 'https://ocl.cs.methods.test'; + const conceptsUrl = `${baseUrl}/orgs/org-a/sources/src1/concepts/`; + let i18n; + let langDefs; + + beforeAll(async () => { + langDefs = await TestUtilities.loadLanguageDefinitions(); + i18n = await TestUtilities.loadTranslations(langDefs); + }); + + beforeEach(() => { + nock.cleanAll(); + OCLSourceCodeSystemFactory.factoriesByKey.clear(); + resetQueueState(); + }); + + afterEach(() => { + nock.cleanAll(); + jest.restoreAllMocks(); + }); + + function createFactoryMeta() { + return { + id: 'src1', + canonicalUrl: 'http://example.org/cs/source-one', + version: '2026.1', + name: 'Source One', + description: 'Desc', + checksum: 'chk-1', + conceptsUrl, + codeSystem: { + jsonObj: { + property: [{ code: 'display' }, { code: 'definition' }, { code: 'inactive' }], + content: 'not-present' + } + } + }; + } + + test('runtime provider methods cover lookup/filter/iteration paths', async () => { + nock(baseUrl) + .get('/orgs/org-a/sources/src1/concepts/C1/') + .reply(200, { + code: 'C1', + display_name: 'Alpha Display', + description: 'Alpha Definition', + retired: false, + names: [{ locale: 'en', name: 'Alpha Display' }] + }) + .get('/orgs/org-a/sources/src1/concepts/C404/') + .reply(404) + .get('/orgs/org-a/sources/src1/concepts/') + .query(q => Number(q.page) === 1 && Number(q.limit) === 1000) + .reply(200, { + results: [ + { code: 'C1', display_name: 'Alpha Display', description: 'Alpha Definition', retired: false }, + { code: 'C2', display_name: 'Beta Display', description: 'Beta Definition', retired: true } + ] + }) + .get('/orgs/org-a/sources/src1/concepts/') + .query(q => Number(q.page) === 2 && Number(q.limit) === 1000) + .reply(200, { results: [] }); + + const factory = new OCLSourceCodeSystemFactory(i18n, require('axios').create({ baseURL: baseUrl }), createFactoryMeta()); + const opContext = new OperationContext('en-US', i18n); + const provider = factory.build(opContext, []); + + expect(provider.system()).toBe('http://example.org/cs/source-one'); + expect(provider.version()).toBe('2026.1'); + expect(provider.description()).toBe('Desc'); + expect(provider.name()).toBe('Source One'); + expect(provider.contentMode()).toBe('not-present'); + expect(provider.totalCount()).toBeGreaterThanOrEqual(-1); + + expect((await provider.locate('')).message).toContain('Empty code'); + const notFound = await provider.locate('C404'); + expect(notFound.context).toBeNull(); + + expect(await provider.code('C1')).toBe('C1'); + expect(await provider.display('C1')).toBe('Alpha Display'); + expect(await provider.definition('C1')).toBe('Alpha Definition'); + expect(await provider.isAbstract('C1')).toBe(false); + expect(await provider.isDeprecated('C1')).toBe(false); + expect(await provider.getStatus('C1')).toBe('active'); + expect(await provider.isInactive('C2')).toBe(true); + + const displays = new Designations(langDefs); + await provider.designations('C1', displays); + expect(displays.designations.length).toBeGreaterThan(0); + + const iter = await provider.iteratorAll(); + const seen = []; + let next = await provider.nextContext(iter); + while (next) { + seen.push(next.code); + next = await provider.nextContext(iter); + } + expect(seen).toEqual(expect.arrayContaining(['C1', 'C2'])); + + expect(await provider.doesFilter('display', '=', 'x')).toBe(true); + expect(await provider.doesFilter('inactive', 'in', 'true,false')).toBe(true); + expect(await provider.doesFilter('unknown', '=', 'x')).toBe(false); + expect(await provider.doesFilter('display', 'contains', 'x')).toBe(false); + + const prep = await provider.getPrepContext(iter); + await provider.searchFilter(prep, 'Alpha', true); + const fromSearch = await provider.executeFilters(prep); + expect(await provider.filterSize(prep, fromSearch[0])).toBeGreaterThanOrEqual(1); + + const inactiveSet = await provider.filter(prep, 'inactive', '=', 'true'); + expect(await provider.filterMore(prep, inactiveSet)).toBe(true); + const concept = await provider.filterConcept(prep, inactiveSet); + expect(concept.code).toBe('C2'); + expect(await provider.filterLocate(prep, inactiveSet, 'C2')).toBeTruthy(); + expect(await provider.filterCheck(prep, inactiveSet, concept)).toBe(true); + + const regexSet = await provider.filter(prep, 'display', 'regex', '^Alpha'); + expect(await provider.filterSize(prep, regexSet)).toBeGreaterThanOrEqual(1); + + const inSet = await provider.filter(prep, 'code', 'in', 'C1,C3'); + expect(await provider.filterSize(prep, inSet)).toBe(1); + + const defSet = await provider.filter(prep, 'definition', '=', 'Alpha Definition'); + expect(await provider.filterSize(prep, defSet)).toBe(1); + + await expect(provider.filter(prep, 'display', 'contains', 'x')).rejects.toThrow('not supported'); + + const exec = await provider.executeFilters(prep); + expect(Array.isArray(exec)).toBe(true); + + await provider.filterFinish(prep); + expect(prep.filters).toHaveLength(0); + + // Exercise #ensureContext promise/wrapper path. + const wrapped = Promise.resolve({ context: { code: 'Z1', display: 'Wrapped', retired: false } }); + expect(await provider.display(wrapped)).toBe('Wrapped'); + }); + + test('factory statics and no-concepts warm load paths are exercised', async () => { + const meta = { + id: 'src-null', + canonicalUrl: 'http://example.org/cs/null', + version: null, + name: 'Null Source', + checksum: null, + conceptsUrl: null, + codeSystem: { jsonObj: { content: 'not-present' } } + }; + + const factory = new OCLSourceCodeSystemFactory(i18n, { get: jest.fn() }, meta); + + expect(factory.defaultVersion()).toBeNull(); + expect(factory.system()).toBe('http://example.org/cs/null'); + expect(factory.name()).toBe('Null Source'); + expect(factory.id()).toBe('src-null'); + expect(factory.iteratable()).toBe(true); + expect(factory.isCompleteNow()).toBe(false); + + const missing = OCLSourceCodeSystemFactory.scheduleBackgroundLoadByKey('http://missing', null, 'x'); + expect(missing).toBe(false); + expect(OCLSourceCodeSystemFactory.checksumForResource('http://missing', null)).toBeNull(); + + OCLSourceCodeSystemFactory.syncCodeSystemResource(null, null, null); + + jest.spyOn(OCLBackgroundJobQueue, 'enqueue').mockImplementation((jobKey, jobType, runJob) => { + Promise.resolve(runJob()).finally(() => { + OCLBackgroundJobQueue.queuedOrRunningKeys.delete(jobKey); + }); + return true; + }); + + factory.scheduleBackgroundLoad('no-concepts'); + await global.TestUtils.waitFor(() => factory.isCompleteNow() === true, 2000); + + const progress = OCLSourceCodeSystemFactory.loadProgress(); + expect(progress.total).toBeGreaterThan(0); + expect(progress.loaded).toBeGreaterThan(0); + }); + + test('scheduleBackgroundLoad exposes queue progress callbacks', async () => { + const meta = { + id: 'src-cb', + canonicalUrl: 'http://example.org/cs/callbacks', + version: '1.0.0', + name: 'Callback Source', + checksum: null, + conceptsUrl: `${baseUrl}/orgs/org-a/sources/src-cb/concepts/`, + codeSystem: { jsonObj: { content: 'not-present' } } + }; + + const client = { + get: jest + .fn() + .mockResolvedValueOnce({ data: { results: [] } }) + .mockResolvedValueOnce({ data: { results: [] }, headers: { 'num-found': 'abc' } }) + }; + + const factory = new OCLSourceCodeSystemFactory(i18n, client, meta); + + const enqueueSpy = jest.spyOn(OCLBackgroundJobQueue, 'enqueue').mockImplementation((jobKey, jobType, runJob, options = {}) => { + if (typeof options.getProgress === 'function') { + options.getProgress(); + } + if (typeof options.resolveJobSize === 'function') { + options.resolveJobSize(); + } + return true; + }); + + factory.scheduleBackgroundLoad('callbacks'); + factory.scheduleBackgroundLoad('callbacks-2'); + expect(enqueueSpy).toHaveBeenCalled(); + expect(factory.currentChecksum()).toBeNull(); + }); +}); diff --git a/tests/ocl/ocl-cs.test.js b/tests/ocl/ocl-cs.test.js new file mode 100644 index 00000000..830e27d3 --- /dev/null +++ b/tests/ocl/ocl-cs.test.js @@ -0,0 +1,291 @@ +const fs = require('fs'); +const fsp = require('fs/promises'); +const path = require('path'); +const nock = require('nock'); + +const { OperationContext } = require('../../tx/operation-context'); +const { OCLCodeSystemProvider, OCLSourceCodeSystemFactory, OCLBackgroundJobQueue } = require('../../tx/ocl/cs-ocl'); +const { CACHE_CS_DIR, getCacheFilePath } = require('../../tx/ocl/cache/cache-paths'); +const { COLD_CACHE_FRESHNESS_MS } = require('../../tx/ocl/shared/constants'); +const { TestUtilities } = require('../test-utilities'); + +function resetQueueState() { + OCLBackgroundJobQueue.pendingJobs = []; + OCLBackgroundJobQueue.activeCount = 0; + OCLBackgroundJobQueue.queuedOrRunningKeys = new Set(); + OCLBackgroundJobQueue.activeJobs = new Map(); + OCLBackgroundJobQueue.enqueueSequence = 0; + if (OCLBackgroundJobQueue.heartbeatTimer) { + clearInterval(OCLBackgroundJobQueue.heartbeatTimer); + OCLBackgroundJobQueue.heartbeatTimer = null; + } +} + +async function clearOclCache() { + await fsp.rm(path.join(process.cwd(), 'data', 'terminology-cache', 'ocl'), { recursive: true, force: true }); +} + +describe('OCL CodeSystem integration', () => { + const baseUrl = 'https://ocl.cs.test'; + let i18n; + + beforeAll(async () => { + i18n = await TestUtilities.loadTranslations(await TestUtilities.loadLanguageDefinitions()); + }); + + beforeEach(async () => { + nock.cleanAll(); + OCLSourceCodeSystemFactory.factoriesByKey.clear(); + resetQueueState(); + await clearOclCache(); + }); + + afterEach(() => { + nock.cleanAll(); + jest.restoreAllMocks(); + }); + + test('metadata discovery and source snapshots are loaded', async () => { + nock(baseUrl) + .get('/orgs/') + .query(true) + .reply(200, { results: [{ id: 'org-a' }] }); + + nock(baseUrl) + .get('/orgs/org-a/sources/') + .query(true) + .reply(200, { + results: [ + { + id: 'src1', + owner: 'org-a', + name: 'Source One', + canonical_url: 'http://example.org/cs/source-one', + version: '2026.1', + concepts_url: '/orgs/org-a/sources/src1/concepts/', + checksums: { standard: 'chk-1' } + } + ] + }); + + const provider = new OCLCodeSystemProvider({ baseUrl }); + const systems = await provider.listCodeSystems('5.0', null); + expect(systems).toHaveLength(1); + expect(systems[0].url).toBe('http://example.org/cs/source-one'); + expect(systems[0].content).toBe('not-present'); + + const metas = provider.getSourceMetas(); + expect(metas).toHaveLength(1); + expect(metas[0].conceptsUrl).toBe(`${baseUrl}/orgs/org-a/sources/src1/concepts/`); + + const ids = new Set(); + provider.assignIds(ids); + expect(Array.from(ids).some(x => x.startsWith('CodeSystem/'))).toBe(true); + await expect(provider.close()).resolves.toBeUndefined(); + }); + + test('getCodeSystemChanges returns staged diffs after refresh', async () => { + nock(baseUrl) + .get('/orgs/') + .query(true) + .times(2) + .reply(200, { results: [{ id: 'org-a' }] }); + + nock(baseUrl) + .get('/orgs/org-a/sources/') + .query(true) + .reply(200, { + results: [{ id: 'src1', owner: 'org-a', canonical_url: 'http://example.org/cs/source-one', version: '1.0.0' }] + }) + .get('/orgs/org-a/sources/') + .query(true) + .reply(200, { + results: [ + { id: 'src1', owner: 'org-a', canonical_url: 'http://example.org/cs/source-one', version: '1.0.1' }, + { id: 'src2', owner: 'org-a', canonical_url: 'http://example.org/cs/source-two', version: '1.0.0' } + ] + }); + + const provider = new OCLCodeSystemProvider({ baseUrl }); + await provider.initialize(); + + const immediate = provider.getCodeSystemChanges('5.0', null); + expect(immediate).toEqual({ added: [], changed: [], deleted: [] }); + + await global.TestUtils.waitFor(() => { + const staged = provider.getCodeSystemChanges('5.0', null); + return staged.added.length === 1 && staged.changed.length === 1; + }, 2000); + }); + + test('factory hydrates from cold cache and skips warm-up while fresh', async () => { + const meta = { + id: 'src1', + canonicalUrl: 'http://example.org/cs/source-one', + version: '1.0.0', + name: 'Source One', + checksum: 'meta-1', + conceptsUrl: `${baseUrl}/orgs/org-a/sources/src1/concepts/`, + codeSystem: { + jsonObj: { + content: 'not-present' + } + } + }; + + await fsp.mkdir(CACHE_CS_DIR, { recursive: true }); + const coldFile = getCacheFilePath(CACHE_CS_DIR, meta.canonicalUrl, meta.version); + await fsp.writeFile(coldFile, JSON.stringify({ + canonicalUrl: meta.canonicalUrl, + version: meta.version, + fingerprint: 'fp-old', + concepts: [ + { code: 'A', display: 'Alpha', retired: false }, + { code: 'B', display: 'Beta', retired: true } + ] + }), 'utf8'); + + const factory = new OCLSourceCodeSystemFactory(i18n, { get: jest.fn() }, meta); + + await global.TestUtils.waitFor(() => factory.isCompleteNow() === true, 2000); + + const opContext = new OperationContext('en-US', i18n); + const provider = factory.build(opContext, []); + + expect(provider.contentMode()).toBe('complete'); + expect(await provider.display('A')).toBe('Alpha'); + expect(await provider.isInactive('B')).toBe(true); + + const enqueueSpy = jest.spyOn(OCLBackgroundJobQueue, 'enqueue'); + factory.scheduleBackgroundLoad('test-fresh-cache'); + expect(enqueueSpy).not.toHaveBeenCalled(); + }); + + test('factory enqueues stale warm-up and replaces cold cache with hot cache', async () => { + const conceptsUrl = `${baseUrl}/orgs/org-a/sources/src1/concepts/`; + const meta = { + id: 'src1', + canonicalUrl: 'http://example.org/cs/source-one', + version: '1.0.0', + name: 'Source One', + checksum: 'meta-2', + conceptsUrl, + codeSystem: { + jsonObj: { + content: 'not-present' + } + } + }; + + await fsp.mkdir(CACHE_CS_DIR, { recursive: true }); + const coldFile = getCacheFilePath(CACHE_CS_DIR, meta.canonicalUrl, meta.version); + await fsp.writeFile(coldFile, JSON.stringify({ + canonicalUrl: meta.canonicalUrl, + version: meta.version, + fingerprint: 'fp-legacy', + concepts: [{ code: 'OLD' }] + }), 'utf8'); + + const staleMs = Date.now() - (COLD_CACHE_FRESHNESS_MS + 120000); + fs.utimesSync(coldFile, new Date(staleMs), new Date(staleMs)); + + nock(baseUrl) + .get('/orgs/org-a/sources/src1/concepts/') + .query(q => Number(q.limit) === 1) + .reply(200, { results: [{ code: 'A' }] }, { num_found: '2' }) + .get('/orgs/org-a/sources/src1/concepts/') + .query(q => Number(q.page) === 1 && Number(q.limit) === 1000) + .reply(200, { + count: 2, + results: [ + { code: 'A', display_name: 'Alpha', retired: false }, + { code: 'B', display_name: 'Beta', retired: false } + ] + }); + + const factory = new OCLSourceCodeSystemFactory(i18n, require('axios').create({ baseURL: baseUrl }), meta); + + // Force queue execution inline for deterministic test behavior. + jest.spyOn(OCLBackgroundJobQueue, 'enqueue').mockImplementation((jobKey, jobType, runJob, options = {}) => { + OCLBackgroundJobQueue.queuedOrRunningKeys.add(jobKey); + Promise.resolve() + .then(async () => { + const size = options.resolveJobSize ? await options.resolveJobSize() : options.jobSize; + await runJob(size); + }) + .finally(() => { + OCLBackgroundJobQueue.queuedOrRunningKeys.delete(jobKey); + }); + return true; + }); + + factory.scheduleBackgroundLoad('stale-cache'); + + await global.TestUtils.waitFor(() => factory.isCompleteNow() === true, 3000); + + const coldData = JSON.parse(await fsp.readFile(coldFile, 'utf8')); + // Existing cold-cache concept remains in shared cache; warm load appends newly fetched concepts. + if (typeof coldData.conceptCount === 'number') { + expect(coldData.conceptCount).toBeGreaterThanOrEqual(2); + } else { + expect(Array.isArray(coldData.concepts)).toBe(true); + expect(coldData.concepts.length).toBeGreaterThanOrEqual(2); + } + expect(coldData.fingerprint).toBeTruthy(); + }); + + test('provider lookup/filter lifecycle is functional for lazy fetches', async () => { + const conceptsUrl = `${baseUrl}/orgs/org-a/sources/src1/concepts/`; + const meta = { + id: 'src1', + canonicalUrl: 'http://example.org/cs/source-one', + version: null, + name: 'Source One', + checksum: 'meta-3', + conceptsUrl, + codeSystem: { + jsonObj: { + property: [{ code: 'display' }, { code: 'inactive' }], + content: 'not-present' + } + } + }; + + nock(baseUrl) + .get('/orgs/org-a/sources/src1/concepts/C3/') + .reply(200, { + code: 'C3', + display_name: 'Gamma term', + description: 'Gamma definition', + retired: false, + names: [{ locale: 'en', name: 'Gamma term' }] + }) + .get('/orgs/org-a/sources/src1/concepts/') + .query(q => Number(q.page) === 1 && Number(q.limit) === 1000) + .reply(200, { + results: [ + { code: 'A1', display_name: 'Alpha', description: 'Alpha definition', retired: false }, + { code: 'B2', display_name: 'Beta', description: 'Beta definition', retired: true } + ] + }) + .get('/orgs/org-a/sources/src1/concepts/') + .query(q => Number(q.page) === 2 && Number(q.limit) === 1000) + .reply(200, { + results: [] + }); + + const factory = new OCLSourceCodeSystemFactory(i18n, require('axios').create({ baseURL: baseUrl }), meta); + const opContext = new OperationContext('en-US', i18n); + const provider = factory.build(opContext, []); + + const located = await provider.locate('C3'); + expect(located.context.code).toBe('C3'); + expect(await provider.display('C3')).toBe('Gamma term'); + + const filterCtx = await provider.getPrepContext(null); + const set = await provider.filter(filterCtx, 'inactive', '=', 'true'); + expect(await provider.filterSize(filterCtx, set)).toBe(1); + + await provider.filterFinish(filterCtx); + }); +}); diff --git a/tests/ocl/ocl-helpers.test.js b/tests/ocl/ocl-helpers.test.js new file mode 100644 index 00000000..5e8d0f90 --- /dev/null +++ b/tests/ocl/ocl-helpers.test.js @@ -0,0 +1,302 @@ +const fs = require('fs'); +const fsp = require('fs/promises'); +const path = require('path'); + +const { createOclHttpClient } = require('../../tx/ocl/http/client'); +const { extractItemsAndNext, fetchAllPages } = require('../../tx/ocl/http/pagination'); +const { sanitizeFilename, getCacheFilePath, CACHE_BASE_DIR } = require('../../tx/ocl/cache/cache-paths'); +const { ensureCacheDirectories, getColdCacheAgeMs, formatCacheAgeMinutes } = require('../../tx/ocl/cache/cache-utils'); +const { computeCodeSystemFingerprint, computeValueSetExpansionFingerprint } = require('../../tx/ocl/fingerprint/fingerprint'); +const { toConceptContext, extractDesignations } = require('../../tx/ocl/mappers/concept-mapper'); +const { OCLConceptFilterContext } = require('../../tx/ocl/model/concept-filter-context'); +const { OCLBackgroundJobQueue } = require('../../tx/ocl/jobs/background-queue'); +const { OCL_CODESYSTEM_MARKER_EXTENSION } = require('../../tx/ocl/shared/constants'); + +function resetQueueState() { + OCLBackgroundJobQueue.pendingJobs = []; + OCLBackgroundJobQueue.activeCount = 0; + OCLBackgroundJobQueue.queuedOrRunningKeys = new Set(); + OCLBackgroundJobQueue.activeJobs = new Map(); + OCLBackgroundJobQueue.enqueueSequence = 0; + if (OCLBackgroundJobQueue.heartbeatTimer) { + clearInterval(OCLBackgroundJobQueue.heartbeatTimer); + OCLBackgroundJobQueue.heartbeatTimer = null; + } +} + +describe('OCL helper modules', () => { + afterEach(() => { + jest.restoreAllMocks(); + resetQueueState(); + }); + + test('createOclHttpClient applies baseUrl and token header', () => { + const out = createOclHttpClient({ baseUrl: 'https://example.org///', token: 'abc' }); + expect(out.baseUrl).toBe('https://example.org'); + expect(out.client.defaults.headers.Authorization).toBe('Token abc'); + + const bearer = createOclHttpClient({ baseUrl: 'https://example.org', token: 'Bearer z' }); + expect(bearer.client.defaults.headers.Authorization).toBe('Bearer z'); + }); + + test('extractItemsAndNext handles arrays, object payloads and baseUrl-relative next links', () => { + expect(extractItemsAndNext([{ id: 1 }])).toEqual({ items: [{ id: 1 }], next: null }); + + const withResults = extractItemsAndNext( + { results: [{ id: 2 }], next: 'https://api.test/sources/?page=2' }, + 'https://api.test' + ); + expect(withResults.items).toHaveLength(1); + expect(withResults.next).toBe('/sources/?page=2'); + + const withItems = extractItemsAndNext({ items: [{ id: 3 }] }); + expect(withItems).toEqual({ items: [{ id: 3 }], next: null }); + }); + + test('fetchAllPages supports page mode and next-link mode', async () => { + const pageCalls = []; + const pageClient = { + get: jest.fn(async (_path, opts) => { + pageCalls.push(opts.params.page); + if (opts.params.page === 1) { + return { data: { results: [{ id: 'a' }, { id: 'b' }] } }; + } + return { data: { results: [{ id: 'c' }] } }; + }) + }; + + const paged = await fetchAllPages(pageClient, '/x', { pageSize: 2 }); + expect(paged.map(x => x.id)).toEqual(['a', 'b', 'c']); + expect(pageCalls).toEqual([1, 2]); + + const nextCalls = []; + const nextClient = { + get: jest.fn(async (pathArg) => { + nextCalls.push(pathArg); + if (pathArg === '/x') { + return { + data: { + results: [{ id: '1' }], + next: 'https://api.test/x?page=2' + } + }; + } + return { + data: { + results: [{ id: '2' }], + next: null + } + }; + }) + }; + + const byNext = await fetchAllPages(nextClient, '/x', { + baseUrl: 'https://api.test', + useNextLinks: true + }); + expect(byNext.map(x => x.id)).toEqual(['1', '2']); + expect(nextCalls).toEqual(['/x', '/x?page=2']); + }); + + test('fetchAllPages logs and rethrows fetch errors', async () => { + const logger = { error: jest.fn() }; + const client = { + get: jest.fn(async () => { + throw new Error('boom'); + }) + }; + + await expect(fetchAllPages(client, '/x', { logger, loggerPrefix: '[T]' })).rejects.toThrow('boom'); + expect(logger.error).toHaveBeenCalled(); + }); + + test('cache path and cache utility helpers behave as expected', async () => { + const s = sanitizeFilename('http://a/b?x=y#z'); + expect(s).toContain('http_a_b_x_y_z'); + + const out = getCacheFilePath(path.join(CACHE_BASE_DIR, 'tmp'), 'http://example.org/vs', '1.0.0', 'f1'); + expect(out.endsWith('.json')).toBe(true); + expect(out).toContain('_p_f1'); + + const dir = path.join(process.cwd(), 'data', 'terminology-cache', 'ocl', 'helper-test'); + await ensureCacheDirectories(dir); + expect(fs.existsSync(dir)).toBe(true); + + const file = path.join(dir, 'age.json'); + await fsp.writeFile(file, '{}', 'utf8'); + const age = getColdCacheAgeMs(file); + expect(age).not.toBeNull(); + expect(formatCacheAgeMinutes(60000)).toBe('1 minute'); + expect(formatCacheAgeMinutes(120000)).toBe('2 minutes'); + + expect(getColdCacheAgeMs(path.join(dir, 'missing.json'))).toBeNull(); + + const mkdirSpy = jest.spyOn(fsp, 'mkdir').mockRejectedValueOnce(new Error('mkdir-fail')); + await expect(ensureCacheDirectories(path.join(dir, 'x'))).resolves.toBeUndefined(); + expect(mkdirSpy).toHaveBeenCalled(); + }); + + test('fingerprints and concept mapper/filter context produce stable outputs', () => { + const csFp1 = computeCodeSystemFingerprint([ + { code: 'b', display: 'B', definition: 'd', retired: false }, + { code: 'a', display: 'A', definition: 'd', retired: true } + ]); + const csFp2 = computeCodeSystemFingerprint([ + { code: 'a', display: 'A', definition: 'd', retired: true }, + { code: 'b', display: 'B', definition: 'd', retired: false } + ]); + expect(csFp1).toBe(csFp2); + expect(computeCodeSystemFingerprint([])).toBeNull(); + expect(computeCodeSystemFingerprint([{ x: 1 }])).toBeNull(); + + const vsFp = computeValueSetExpansionFingerprint({ + contains: [ + { system: 's', code: '1', display: 'one', inactive: false }, + { system: 's', code: '2', display: 'two', inactive: true } + ] + }); + expect(vsFp).toBeTruthy(); + expect(computeValueSetExpansionFingerprint({ contains: [] })).toBeNull(); + expect(computeValueSetExpansionFingerprint(null)).toBeNull(); + + const concept = toConceptContext({ + code: '123', + display_name: 'Main', + description: 'Def', + retired: true, + names: [{ locale: 'en', name: 'Main' }, { locale: 'pt', name: 'Principal' }] + }); + expect(concept.code).toBe('123'); + expect(concept.retired).toBe(true); + expect(toConceptContext(null)).toBeNull(); + expect(toConceptContext({})).toBeNull(); + expect(extractDesignations({ names: [{ locale: 'en', name: 'A' }, { locale: 'en', name: 'A' }] })).toHaveLength(1); + expect(extractDesignations({ locale_display_names: { 'pt-BR': 'Nome' } })).toEqual([ + { language: 'pt-BR', value: 'Nome' } + ]); + + const set = new OCLConceptFilterContext(); + set.add({ code: 'b' }, 1); + set.add({ code: 'a' }, 2); + set.sort(); + expect(set.next().code).toBe('a'); + expect(set.findConceptByCode('b').code).toBe('b'); + set.reset(); + const item = set.next(); + expect(set.containsConcept(item)).toBe(true); + set.next(); + expect(set.next()).toBeNull(); + }); + + test('background queue enforces singleton keys, size ordering and progress formatting', async () => { + OCLBackgroundJobQueue.MAX_CONCURRENT = 0; + + const first = OCLBackgroundJobQueue.enqueue('j1', 'job', async () => {}, { jobSize: 10 }); + const duplicate = OCLBackgroundJobQueue.enqueue('j1', 'job', async () => {}, { jobSize: 1 }); + OCLBackgroundJobQueue.enqueue('j2', 'job', async () => {}, { jobSize: 2 }); + OCLBackgroundJobQueue.enqueue('j3', 'job', async () => {}, { jobSize: 5 }); + + await global.TestUtils.delay(10); + + expect(first).toBe(true); + expect(duplicate).toBe(false); + expect(OCLBackgroundJobQueue.pendingJobs.map(j => j.jobSize)).toEqual([2, 5, 10]); + + expect(OCLBackgroundJobQueue.formatProgress(() => 51.2)).toBe('51%'); + expect(OCLBackgroundJobQueue.formatProgress(() => ({ processed: 25, total: 100 }))).toBe('25%'); + expect(OCLBackgroundJobQueue.formatProgress(() => ({ percentage: 120 }))).toBe('100%'); + expect(OCLBackgroundJobQueue.formatProgress(() => { throw new Error('x'); })).toBe('unknown'); + + const spy = jest.spyOn(console, 'log').mockImplementation(() => {}); + OCLBackgroundJobQueue.logHeartbeat(); + expect(spy).toHaveBeenCalled(); + }); + + test('background queue processes jobs and handles failures', async () => { + OCLBackgroundJobQueue.MAX_CONCURRENT = 1; + const completed = []; + + OCLBackgroundJobQueue.enqueue('qa', 'job', async () => { + completed.push('a'); + }, { jobSize: 1 }); + + OCLBackgroundJobQueue.enqueue('qb', 'job', async () => { + completed.push('b'); + throw new Error('expected'); + }, { jobSize: 2 }); + + await global.TestUtils.waitFor(() => completed.length === 2, 2000); + await global.TestUtils.waitFor(() => OCLBackgroundJobQueue.activeCount === 0, 2000); + expect(OCLBackgroundJobQueue.pendingJobs.length).toBe(0); + }); + + test('patches integrate code filtering and TxParameters hash filter extension', () => { + jest.resetModules(); + + const { patchSearchWorkerForOCLCodeFiltering, ensureTxParametersHashIncludesFilter, normalizeFilterForCacheKey } = require('../../tx/ocl/shared/patches'); + const SearchWorker = require('../../tx/workers/search'); + + const originalSearchCodeSystems = SearchWorker.prototype.searchCodeSystems; + SearchWorker.prototype.searchCodeSystems = function () { + return [ + { + url: 'http://test/cs', + extension: [{ url: OCL_CODESYSTEM_MARKER_EXTENSION, valueBoolean: true }], + concept: [ + { code: 'A', display: 'Alpha', concept: [{ code: 'A1' }] }, + { code: 'B', display: 'Beta' } + ] + }, + { + url: 'http://test/non-ocl', + concept: [{ code: 'X' }] + } + ]; + }; + + try { + patchSearchWorkerForOCLCodeFiltering(); + // idempotence path + patchSearchWorkerForOCLCodeFiltering(); + + const worker = Object.create(SearchWorker.prototype); + const filtered = worker.searchCodeSystems({ code: 'A1' }); + expect(filtered).toHaveLength(2); + expect(filtered[0].concept[0].code).toBe('A'); + expect(filtered[0].concept[0].concept[0].code).toBe('A1'); + + const filteredNone = worker.searchCodeSystems({ code: 'missing' }); + expect(filteredNone).toHaveLength(1); + expect(filteredNone[0].url).toBe('http://test/non-ocl'); + } finally { + SearchWorker.prototype.searchCodeSystems = originalSearchCodeSystems; + } + + class TxParameters { + constructor() { + this.filter = ' TeSt '; + } + + hashSource() { + return 'base'; + } + } + + ensureTxParametersHashIncludesFilter(TxParameters); + const p = new TxParameters(); + expect(p.hashSource()).toBe('base|filter=test'); + expect(normalizeFilterForCacheKey(' ABC ')).toBe('abc'); + }); + + test('patchSearchWorkerForOCLCodeFiltering is safe when worker cannot be loaded', () => { + jest.resetModules(); + jest.isolateModules(() => { + jest.doMock('../../tx/workers/search', () => { + throw new Error('no-worker'); + }, { virtual: true }); + + const { patchSearchWorkerForOCLCodeFiltering } = require('../../tx/ocl/shared/patches'); + expect(() => patchSearchWorkerForOCLCodeFiltering()).not.toThrow(); + }); + }); +}); diff --git a/tests/ocl/ocl-vs-advanced.test.js b/tests/ocl/ocl-vs-advanced.test.js new file mode 100644 index 00000000..28043247 --- /dev/null +++ b/tests/ocl/ocl-vs-advanced.test.js @@ -0,0 +1,428 @@ +const nock = require('nock'); + +const ValueSet = require('../../tx/library/valueset'); +const { OCLValueSetProvider } = require('../../tx/ocl/vs-ocl'); +const { OCLBackgroundJobQueue } = require('../../tx/ocl/cs-ocl'); + +function resetQueueState() { + OCLBackgroundJobQueue.pendingJobs = []; + OCLBackgroundJobQueue.activeCount = 0; + OCLBackgroundJobQueue.queuedOrRunningKeys = new Set(); + OCLBackgroundJobQueue.activeJobs = new Map(); + OCLBackgroundJobQueue.enqueueSequence = 0; + if (OCLBackgroundJobQueue.heartbeatTimer) { + clearInterval(OCLBackgroundJobQueue.heartbeatTimer); + OCLBackgroundJobQueue.heartbeatTimer = null; + } +} + +describe('OCL ValueSet advanced provider behavior', () => { + const baseUrl = 'https://ocl.vs.advanced.test'; + const PAGE_SIZE = 100; + + beforeEach(() => { + nock.cleanAll(); + resetQueueState(); + }); + + afterEach(() => { + nock.cleanAll(); + jest.restoreAllMocks(); + }); + + test('sourcePackage, assignIds with and without spaceId, and search/list methods', async () => { + const provider = new OCLValueSetProvider({ baseUrl, org: 'org-a' }); + provider._initialized = true; + expect(provider.sourcePackage()).toBe(`ocl:${baseUrl}|org=org-a`); + + const vs = new ValueSet({ + resourceType: 'ValueSet', + id: 'vs1', + url: 'http://example.org/vs/1', + version: '1.2.3', + name: 'VS1', + status: 'active' + }, 'R5'); + + provider.valueSetMap.set(vs.url, vs); + provider.valueSetMap.set(`${vs.url}|${vs.version}`, vs); + provider.valueSetMap.set(vs.id, vs); + + const idsNoSpace = new Set(); + provider.assignIds(idsNoSpace); + expect(idsNoSpace.size).toBe(0); + + provider.spaceId = 'S'; + const ids = new Set(); + provider.assignIds(ids); + expect(ids.has('ValueSet/S-vs1')).toBe(true); + + const all = await provider.searchValueSets([]); + expect(all).toHaveLength(1); + expect(provider.vsCount()).toBe(1); + expect(await provider.listAllValueSets()).toEqual(['http://example.org/vs/1']); + }); + + test('fetchValueSet resolves semver major.minor and fetchValueSetById handles prefixed ids', async () => { + const provider = new OCLValueSetProvider({ baseUrl, org: 'org-a' }); + jest.spyOn(OCLBackgroundJobQueue, 'enqueue').mockReturnValue(true); + + const vs = new ValueSet({ + resourceType: 'ValueSet', + id: 'vs1', + url: 'http://example.org/vs/1', + version: '1.2', + name: 'VS1', + status: 'active', + compose: { include: [{ system: 'http://example.org/cs/1' }] } + }, 'R5'); + + provider.valueSetMap.set(vs.url, vs); + provider.valueSetMap.set(`${vs.url}|${vs.version}`, vs); + provider.valueSetMap.set(vs.id, vs); + provider._idMap.set(vs.id, vs); + + const bySemver = await provider.fetchValueSet(vs.url, '1.2.9'); + expect(bySemver).toBeTruthy(); + expect(bySemver.version).toBe('1.2'); + + provider.spaceId = 'X'; + provider._idMap.set('X-vs1', vs); + const byPrefixedId = await provider.fetchValueSetById('X-vs1'); + expect(byPrefixedId).toBeTruthy(); + }); + + test('canonical resolution path uses collection search and semver fallback', async () => { + nock(baseUrl) + .get('/orgs/') + .query(true) + .reply(200, { results: [{ id: 'org-a' }] }) + .get('/orgs/org-a/collections/') + .query(q => q.q === 'my-vs') + .reply(200, { + results: [ + { + id: 'col-vs', + owner: 'org-a', + owner_type: 'Organization', + canonical_url: 'http://example.org/vs/my-vs', + version: '2.1', + name: 'My VS', + concepts_url: '/orgs/org-a/collections/col-vs/concepts/' + } + ] + }) + .get('/orgs/org-a/collections/col-vs/concepts/') + .query(true) + .reply(200, { results: [] }); + + const provider = new OCLValueSetProvider({ baseUrl, org: 'org-a' }); + jest.spyOn(OCLBackgroundJobQueue, 'enqueue').mockReturnValue(true); + + const resolved = await provider.fetchValueSet('http://example.org/vs/my-vs', '2.1.5'); + expect(resolved).toBeTruthy(); + expect(resolved.url).toBe('http://example.org/vs/my-vs'); + expect(resolved.version).toBe('2.1'); + }); + + test('compose include fallback via concepts listing and source canonical lookup', async () => { + nock(baseUrl) + .get('/orgs/') + .query(true) + .reply(200, { results: [{ id: 'org-a' }] }) + .get('/orgs/org-a/collections/') + .query(true) + .reply(200, { + results: [ + { + id: 'col2', + owner: 'org-a', + owner_type: 'Organization', + canonical_url: 'http://example.org/vs/compose', + version: '1.0.0', + name: 'Compose VS', + concepts_url: '/orgs/org-a/collections/col2/concepts/', + expansion_url: '/orgs/org-a/collections/col2/HEAD/expansions/autoexpand-HEAD/' + } + ] + }) + .get('/orgs/org-a/collections/col2/HEAD/expansions/autoexpand-HEAD/') + .reply(500) + .get('/orgs/org-a/collections/col2/concepts/') + .query(q => Number(q.page) === 1 && Number(q.limit) === 1000) + .reply(200, { + results: [ + { owner: 'org-a', source: 'src-a', code: 'A' }, + { owner: 'org-a', source: 'src-b', code: 'B' } + ] + }) + .get('/orgs/org-a/collections/col2/concepts/') + .query(q => Number(q.page) === 2 && Number(q.limit) === 1000) + .reply(200, { results: [] }) + .get('/orgs/org-a/sources/src-a/') + .reply(200, { canonical_url: 'http://example.org/cs/src-a' }) + .get('/orgs/org-a/sources/src-b/') + .reply(200, { canonical_url: 'http://example.org/cs/src-b' }); + + const provider = new OCLValueSetProvider({ baseUrl, org: 'org-a' }); + jest.spyOn(OCLBackgroundJobQueue, 'enqueue').mockReturnValue(true); + + await provider.initialize(); + const vs = await provider.fetchValueSet('http://example.org/vs/compose', '1.0.0'); + expect(vs.jsonObj.compose.include.length).toBe(2); + }); + + test('cached expansion invalidates when metadata signature/dependencies mismatch', async () => { + const provider = new OCLValueSetProvider({ baseUrl, org: 'org-a' }); + jest.spyOn(OCLBackgroundJobQueue, 'enqueue').mockReturnValue(true); + + const vs = new ValueSet({ + resourceType: 'ValueSet', + id: 'vs-invalid', + url: 'http://example.org/vs/invalid', + version: '1.0.0', + name: 'Invalid Cache VS', + status: 'active', + compose: { include: [{ system: 'http://example.org/cs/src-a' }] } + }, 'R5'); + + provider.valueSetMap.set(vs.url, vs); + provider.valueSetMap.set(`${vs.url}|${vs.version}`, vs); + provider.valueSetMap.set(vs.id, vs); + + const cacheKey = `${vs.url}|${vs.version}|default`; + provider.backgroundExpansionCache.set(cacheKey, { + expansion: { contains: [{ system: 'x', code: '1' }] }, + metadataSignature: 'stale-signature', + dependencyChecksums: {}, + createdAt: Date.now() - 7200000 + }); + + const out = await provider.fetchValueSet(vs.url, vs.version); + expect(out).toBeTruthy(); + expect(provider.backgroundExpansionCache.has(cacheKey)).toBe(false); + }); + + test('validation and fallback discovery branches', async () => { + const provider = new OCLValueSetProvider({ baseUrl, org: 'org-a' }); + provider._initialized = true; + + await expect(provider.fetchValueSet('', null)).rejects.toThrow('URL must be a non-empty string'); + await expect(provider.searchValueSets('bad')).rejects.toThrow('Search parameters must be an array'); + + const providerFallback = new OCLValueSetProvider({ baseUrl }); + nock(baseUrl) + .get('/orgs/') + .query(true) + .reply(200, { results: [] }) + .get('/collections/') + .query(true) + .reply(200, { + results: [ + { + id: 'col-fallback', + owner: 'org-a', + owner_type: 'Organization', + canonical_url: 'http://example.org/vs/fallback', + version: '1.0.0', + name: 'Fallback VS' + } + ] + }); + + await providerFallback.initialize(); + expect(providerFallback.vsCount()).toBeGreaterThan(0); + }); + + test('searchValueSets matches system and identifier fields', async () => { + const provider = new OCLValueSetProvider({ baseUrl, org: 'org-a' }); + provider._initialized = true; + + const vs = new ValueSet({ + resourceType: 'ValueSet', + id: 'vs-system-id', + url: 'http://example.org/vs/system-id', + version: '1.0.0', + identifier: [{ system: 'urn:sys', value: 'ABC-123' }], + compose: { include: [{ system: 'http://example.org/cs/target' }] } + }, 'R5'); + + provider.valueSetMap.set(vs.url, vs); + provider.valueSetMap.set(`${vs.url}|${vs.version}`, vs); + + const found = await provider.searchValueSets([ + { name: 'system', value: 'cs/target' }, + { name: 'identifier', value: 'abc-123' } + ]); + + expect(found).toHaveLength(1); + expect(found[0].id).toBe('vs-system-id'); + }); + + test('localized sorting and invalid updated_on handling are exercised', async () => { + nock(baseUrl) + .get('/orgs/') + .query(true) + .reply(200, { results: [{ id: 'org-a' }] }) + .get('/orgs/org-a/collections/') + .query(true) + .reply(200, { + results: [ + { + id: 'col-loc', + owner: 'org-a', + owner_type: 'Organization', + canonical_url: 'http://example.org/vs/localized', + version: '1.0.0', + updated_on: 'invalid-date-value', + concepts_url: '/orgs/org-a/collections/col-loc/concepts/' + } + ] + }) + .get('/orgs/org-a/collections/col-loc/concepts/') + .query(q => Number(q.page) === 1 && Number(q.limit) === 1000) + .reply(200, { + results: [ + { + owner: 'org-a', + source: 'src-a', + code: 'C1' + } + ] + }) + .get('/orgs/org-a/collections/col-loc/concepts/') + .query(q => Number(q.page) === 2 && Number(q.limit) === 1000) + .reply(200, { results: [] }) + .get('/orgs/org-a/collections/col-loc/concepts/') + .query(q => Number(q.page) === 1 && Number(q.limit) === 200 && String(q.verbose) === 'true') + .reply(200, { + results: [ + { + owner: 'org-a', + source: 'src-a', + code: 'C1', + names: [ + { locale: 'pt-BR', name: 'Termo', name_type: 'Synonym' }, + { locale: 'en', name: 'Term', name_type: 'Fully Specified Name', locale_preferred: true }, + { locale: 'en', name: 'Term Variant', name_type: 'Synonym' } + ], + descriptions: [ + { locale: 'pt-BR', description: 'Descricao', description_type: 'Text' }, + { locale: 'en', description: 'Definition EN', description_type: 'Definition', locale_preferred: true }, + { locale: 'en', description: 'Other EN', description_type: 'Note' } + ] + } + ] + }) + .get('/orgs/org-a/sources/src-a/') + .reply(200, { canonical_url: 'http://example.org/cs/src-a' }); + + const provider = new OCLValueSetProvider({ baseUrl, org: 'org-a' }); + jest.spyOn(OCLBackgroundJobQueue, 'enqueue').mockReturnValue(true); + + await provider.initialize(); + const vs = await provider.fetchValueSet('http://example.org/vs/localized', '1.0.0'); + expect(vs.jsonObj.meta).toBeUndefined(); + + const out = await vs.oclFetchConcepts({ + count: 5, + offset: 0, + activeOnly: false, + filter: 'term', + languageCodes: ['en', 'pt-BR'] + }); + + expect(out.contains).toHaveLength(1); + expect(out.contains[0].display).toBe('Term'); + expect(out.contains[0].definition).toBe('Definition EN'); + expect(out.contains[0].designation.length).toBeGreaterThan(1); + expect(out.contains[0].definitions.length).toBeGreaterThan(1); + + await expect(provider.close()).resolves.toBeUndefined(); + }); + + test('warm-up enqueue exposes progress callbacks and remote query chooses longest token', async () => { + nock(baseUrl) + .get('/orgs/') + .query(true) + .reply(200, { results: [{ id: 'org-a' }] }) + .get('/orgs/org-a/collections/') + .query(true) + .reply(200, { + results: [ + { + id: 'col-q', + owner: 'org-a', + owner_type: 'Organization', + canonical_url: 'http://example.org/vs/q', + version: '1.0.0', + concepts_url: '/orgs/org-a/collections/col-q/concepts/' + } + ] + }) + .get('/orgs/org-a/collections/col-q/concepts/') + .query(q => Number(q.page) === 1 && Number(q.limit) === 1000) + .reply(200, { results: [] }) + .get('/orgs/org-a/collections/col-q/concepts/') + .query(q => Number(q.page) === 1 && Number(q.limit) === 200 && String(q.verbose) === 'true' && q.q === 'alphabet') + .reply(200, { + results: [ + { code: 'X1', owner: 'org-a', source: 'src-a', display_name: 'Alphabet term', retired: false } + ] + }); + + const provider = new OCLValueSetProvider({ baseUrl, org: 'org-a' }); + const enqueueSpy = jest.spyOn(OCLBackgroundJobQueue, 'enqueue').mockImplementation((jobKey, jobType, runJob, options = {}) => { + if (typeof options.getProgress === 'function') { + options.getProgress(); + } + if (typeof options.resolveJobSize === 'function') { + options.resolveJobSize(); + } + return true; + }); + + await provider.initialize(); + const vs = await provider.fetchValueSet('http://example.org/vs/q', '1.0.0'); + const out = await vs.oclFetchConcepts({ + count: 10, + offset: 0, + filter: 'abc or alphabet', + activeOnly: false + }); + + expect(enqueueSpy).toHaveBeenCalled(); + expect(out.contains).toHaveLength(1); + }); + + test('collection discovery paginates across PAGE_SIZE boundary', async () => { + const page1 = Array.from({ length: PAGE_SIZE }, (_, i) => ({ + id: `col-p1-${i}`, + owner: 'org-a', + owner_type: 'Organization', + canonical_url: `http://example.org/vs/p1/${i}`, + version: '1.0.0', + updated_on: '2026-02-03T04:05:06.000Z', + concepts_url: `/orgs/org-a/collections/col-p1-${i}/concepts/` + })); + + nock(baseUrl) + .get('/orgs/') + .query(q => Number(q.page) === 1 && Number(q.limit) === PAGE_SIZE) + .reply(200, { results: [{ id: 'org-a' }] }) + .get('/orgs/org-a/collections/') + .query(q => Number(q.page) === 1 && Number(q.limit) === PAGE_SIZE) + .reply(200, { results: page1 }) + .get('/orgs/org-a/collections/') + .query(q => Number(q.page) === 2 && Number(q.limit) === PAGE_SIZE) + .reply(200, { results: [] }); + + const provider = new OCLValueSetProvider({ baseUrl, org: 'org-a' }); + await provider.initialize(); + + expect(provider.vsCount()).toBeGreaterThanOrEqual(PAGE_SIZE); + const found = await provider.searchValueSets([{ name: 'url', value: 'http://example.org/vs/p1/0' }]); + expect(found).toHaveLength(1); + expect(found[0].jsonObj.meta.lastUpdated).toBe('2026-02-03T04:05:06.000Z'); + }); +}); diff --git a/tests/ocl/ocl-vs.test.js b/tests/ocl/ocl-vs.test.js new file mode 100644 index 00000000..a69202ac --- /dev/null +++ b/tests/ocl/ocl-vs.test.js @@ -0,0 +1,435 @@ +const fs = require('fs'); +const fsp = require('fs/promises'); +const path = require('path'); +const nock = require('nock'); + +const { OCLValueSetProvider } = require('../../tx/ocl/vs-ocl'); +const { OCLSourceCodeSystemFactory, OCLBackgroundJobQueue } = require('../../tx/ocl/cs-ocl'); +const { CACHE_VS_DIR, getCacheFilePath } = require('../../tx/ocl/cache/cache-paths'); +const { COLD_CACHE_FRESHNESS_MS } = require('../../tx/ocl/shared/constants'); +const { ValueSetExpander } = require('../../tx/workers/expand'); +const { patchValueSetExpandWholeSystemForOcl } = require('../../tx/ocl/shared/patches'); + +function resetQueueState() { + OCLBackgroundJobQueue.pendingJobs = []; + OCLBackgroundJobQueue.activeCount = 0; + OCLBackgroundJobQueue.queuedOrRunningKeys = new Set(); + OCLBackgroundJobQueue.activeJobs = new Map(); + OCLBackgroundJobQueue.enqueueSequence = 0; + if (OCLBackgroundJobQueue.heartbeatTimer) { + clearInterval(OCLBackgroundJobQueue.heartbeatTimer); + OCLBackgroundJobQueue.heartbeatTimer = null; + } +} + +async function clearOclCache() { + await fsp.rm(path.join(process.cwd(), 'data', 'terminology-cache', 'ocl'), { recursive: true, force: true }); +} + +describe('OCL ValueSet integration', () => { + const baseUrl = 'https://ocl.vs.test'; + const conceptsPath = '/orgs/org-a/collections/col1/concepts/'; + const expansionPath = '/orgs/org-a/collections/col1/HEAD/expansions/autoexpand-HEAD/'; + + beforeEach(async () => { + nock.cleanAll(); + OCLSourceCodeSystemFactory.factoriesByKey.clear(); + resetQueueState(); + await clearOclCache(); + }); + + afterEach(() => { + nock.cleanAll(); + jest.restoreAllMocks(); + }); + + function mockDiscovery() { + nock(baseUrl) + .get('/orgs/') + .query(true) + .reply(200, { results: [{ id: 'org-a' }] }) + .get('/orgs/org-a/collections/') + .query(true) + .reply(200, { + results: [ + { + id: 'col1', + owner: 'org-a', + owner_type: 'Organization', + canonical_url: 'http://example.org/vs/one', + version: '1.0.0', + preferred_source: 'http://example.org/cs/source-one', + concepts_url: conceptsPath, + expansion_url: expansionPath + } + ] + }) + .get(expansionPath) + .times(20) + .reply(200, { + resolved_source_versions: [ + { + canonical_url: 'http://example.org/cs/source-one', + version: '1.0.0' + } + ] + }); + } + + test('metadata discovery and cold-cache hydration for expansions', async () => { + await fsp.mkdir(CACHE_VS_DIR, { recursive: true }); + const coldPath = getCacheFilePath(CACHE_VS_DIR, 'http://example.org/vs/one', '1.0.0', 'default'); + await fsp.writeFile(coldPath, JSON.stringify({ + canonicalUrl: 'http://example.org/vs/one', + version: '1.0.0', + paramsKey: 'default', + fingerprint: 'fp-vs', + timestamp: new Date().toISOString(), + expansion: { + contains: [{ system: 'http://example.org/cs/source-one', code: 'A', display: 'Alpha' }] + } + }), 'utf8'); + + mockDiscovery(); + + const provider = new OCLValueSetProvider({ baseUrl, org: 'org-a' }); + await provider.initialize(); + jest.spyOn(OCLBackgroundJobQueue, 'enqueue').mockReturnValue(true); + + const fetched = await provider.fetchValueSet('http://example.org/vs/one', '1.0.0'); + expect(fetched).toBeTruthy(); + expect(fetched.url).toBe('http://example.org/vs/one'); + expect(fetched.oclMeta.conceptsUrl).toBe(`${baseUrl}${conceptsPath}`); + }); + + test('warm-up is skipped when cold cache is <= 1 hour old', async () => { + mockDiscovery(); + + const provider = new OCLValueSetProvider({ baseUrl, org: 'org-a' }); + await provider.initialize(); + + await fsp.mkdir(CACHE_VS_DIR, { recursive: true }); + const coldPath = getCacheFilePath(CACHE_VS_DIR, 'http://example.org/vs/one', '1.0.0', 'default'); + await fsp.writeFile(coldPath, JSON.stringify({ + canonicalUrl: 'http://example.org/vs/one', + version: '1.0.0', + paramsKey: 'default', + fingerprint: 'fp-vs', + timestamp: new Date().toISOString(), + expansion: { + contains: [{ system: 'http://example.org/cs/source-one', code: 'A', display: 'Alpha' }] + }, + metadataSignature: '{"k":"v"}', + dependencyChecksums: {} + }), 'utf8'); + + const enqueueSpy = jest.spyOn(OCLBackgroundJobQueue, 'enqueue'); + const fetched = await provider.fetchValueSet('http://example.org/vs/one', '1.0.0'); + + expect(fetched).toBeTruthy(); + expect(enqueueSpy).not.toHaveBeenCalled(); + }); + + test('stale cache triggers warm-up enqueue, expansion build, fingerprint and disk replacement', async () => { + mockDiscovery(); + + nock(baseUrl) + .get(expansionPath) + .reply(200, { + resolved_source_versions: [ + { + canonical_url: 'http://example.org/cs/source-one', + version: '2026.1' + } + ] + }) + .get(conceptsPath) + .query(q => Number(q.limit) === 1) + .times(2) + .reply(200, { results: [{ code: 'A' }] }, { num_found: '3' }) + .get(conceptsPath) + .query(q => Number(q.page) === 1 && Number(q.limit) === 1000) + .reply(200, { + results: [ + { + code: 'A', + display_name: 'Alpha', + definition: 'Alpha definition', + retired: false, + owner: 'org-a', + source: 'src-1', + source_canonical_url: 'http://example.org/cs/source-one', + names: [{ locale: 'en', name: 'Alpha', locale_preferred: true }], + descriptions: [{ locale: 'en', description: 'Alpha definition', locale_preferred: true }] + }, + { + code: 'B', + display_name: 'Beta', + retired: true, + owner: 'org-a', + source: 'src-1', + source_canonical_url: 'http://example.org/cs/source-one', + names: [{ locale: 'pt-BR', name: 'Beta' }] + } + ] + }) + .get(conceptsPath) + .query(q => Number(q.page) === 2 && Number(q.limit) === 1000) + .reply(200, { results: [] }); + + const provider = new OCLValueSetProvider({ baseUrl, org: 'org-a' }); + await provider.initialize(); + + await fsp.mkdir(CACHE_VS_DIR, { recursive: true }); + const coldPath = getCacheFilePath(CACHE_VS_DIR, 'http://example.org/vs/one', '1.0.0', 'default'); + await fsp.writeFile(coldPath, JSON.stringify({ + canonicalUrl: 'http://example.org/vs/one', + version: '1.0.0', + paramsKey: 'default', + fingerprint: 'old-fingerprint', + timestamp: new Date(Date.now() - (COLD_CACHE_FRESHNESS_MS + 120000)).toISOString(), + expansion: { + contains: [{ system: 'http://example.org/cs/source-one', code: 'OLD' }] + }, + metadataSignature: null, + dependencyChecksums: {} + }), 'utf8'); + const staleMs = Date.now() - (COLD_CACHE_FRESHNESS_MS + 120000); + fs.utimesSync(coldPath, new Date(staleMs), new Date(staleMs)); + + jest.spyOn(OCLSourceCodeSystemFactory, 'scheduleBackgroundLoadByKey').mockImplementation(() => true); + jest.spyOn(OCLBackgroundJobQueue, 'enqueue').mockImplementation((jobKey, jobType, runJob, options = {}) => { + OCLBackgroundJobQueue.queuedOrRunningKeys.add(jobKey); + Promise.resolve() + .then(async () => { + const size = options.resolveJobSize ? await options.resolveJobSize() : options.jobSize; + await runJob(size); + }) + .finally(() => { + OCLBackgroundJobQueue.queuedOrRunningKeys.delete(jobKey); + }); + return true; + }); + + const vs = await provider.fetchValueSet('http://example.org/vs/one', '1.0.0'); + expect(vs).toBeTruthy(); + + await global.TestUtils.waitFor(async () => { + try { + const data = JSON.parse(await fsp.readFile(coldPath, 'utf8')); + return data.conceptCount === 2; + } catch (_e) { + return false; + } + }, 3000); + + const updated = JSON.parse(await fsp.readFile(coldPath, 'utf8')); + expect(updated.conceptCount).toBe(2); + expect(updated.fingerprint).toBeTruthy(); + }); + + test('filter handling in oclFetchConcepts and cache key behavior for filtered calls', async () => { + mockDiscovery(); + + nock(baseUrl) + .get(conceptsPath) + .query(q => Number(q.page) === 1 && Number(q.limit) === 200 && String(q.verbose) === 'true' && q.q === 'alpha') + .reply(200, { + num_found: 2, + results: [ + { + code: 'A', + display_name: 'Alpha term', + definition: 'Definition alpha', + retired: false, + owner: 'org-a', + source: 'src-1', + source_canonical_url: 'http://example.org/cs/source-one', + names: [{ locale: 'en', name: 'Alpha term', locale_preferred: true }], + descriptions: [{ locale: 'en', description: 'Definition alpha' }] + }, + { + code: 'B', + display_name: 'Beta', + definition: 'No match', + retired: false, + owner: 'org-a', + source: 'src-1', + source_canonical_url: 'http://example.org/cs/source-one' + } + ] + }); + + const provider = new OCLValueSetProvider({ baseUrl, org: 'org-a' }); + await provider.initialize(); + jest.spyOn(OCLBackgroundJobQueue, 'enqueue').mockReturnValue(true); + + const vs = await provider.fetchValueSet('http://example.org/vs/one', '1.0.0'); + const out = await vs.oclFetchConcepts({ + count: 20, + offset: 0, + activeOnly: false, + filter: ' alpha ', + languageCodes: ['pt-BR', 'en'] + }); + + expect(out.contains).toHaveLength(1); + expect(out.contains[0].code).toBe('A'); + expect(out.contains[0].display).toBe('Alpha term'); + }); + + test('fetchValueSetById and search/list methods are deterministic', async () => { + mockDiscovery(); + + const provider = new OCLValueSetProvider({ baseUrl, org: 'org-a' }); + provider.spaceId = 'S'; + await provider.initialize(); + jest.spyOn(OCLBackgroundJobQueue, 'enqueue').mockReturnValue(true); + + const ids = new Set(); + provider.assignIds(ids); + + const byId = await provider.fetchValueSetById('S-col1'); + expect(byId).toBeTruthy(); + + const search = await provider.searchValueSets([{ name: 'url', value: 'http://example.org/vs/one' }]); + expect(search).toHaveLength(1); + + const all = await provider.listAllValueSets(); + expect(all).toContain('http://example.org/vs/one'); + }); + + test('regression: unfiltered whole-system include retries with bounded paging for OCL', async () => { + patchValueSetExpandWholeSystemForOcl(); + + const cs = { + specialEnumeration: () => null, + isNotClosed: () => false, + iterator: async () => ({ total: 5001 }), + nextContext: (() => { + let emitted = false; + return async () => { + if (emitted) { + return null; + } + emitted = true; + return { code: 'A' }; + }; + })(), + system: async () => 'http://example.org/cs/source-one', + version: async () => '1.0.0' + }; + + const expander = new ValueSetExpander({ + internalLimit: 10000, + externalLimit: 1000, + opContext: { log: () => {}, diagnostics: () => '' }, + deadCheck: () => {}, + findCodeSystem: async () => cs, + checkSupplements: () => {}, + i18n: { languageDefinitions: {}, translate: (_k, _langs, args) => `too costly ${args?.join(' ') || ''}` }, + provider: { getFhirVersion: () => '5.0.0' }, + languages: { parse: () => null } + }, { + hasDesignations: false, + designations: [], + httpLanguages: null + }); + + expander.valueSet = { oclFetchConcepts: async () => ({ contains: [] }) }; + expander.limitCount = 1000; + expander.count = -1; + expander.offset = -1; + expander.hasExclusions = false; + expander.requiredSupplements = new Set(); + expander.usedSupplements = new Set(); + expander.map = new Map(); + expander.fullList = []; + expander.rootList = []; + expander.addToTotal = jest.fn(); + expander.passesFilters = jest.fn(async () => true); + expander.includeCodeAndDescendants = jest.fn(async () => 1); + expander.checkProviderCanonicalStatus = jest.fn(); + expander.addParamUri = jest.fn(); + expander.addParamInt = jest.fn(); + + await expect(expander.includeCodes( + { system: 'http://example.org/cs/source-one' }, + 'ValueSet.compose.include[0]', + { vurl: 'http://example.org/vs/one|1.0.0', url: 'http://example.org/vs/one' }, + { include: [{ system: 'http://example.org/cs/source-one' }] }, + { isNull: true }, + {}, + false, + { value: false } + )).resolves.toBeUndefined(); + + expect(expander.addParamInt).toHaveBeenCalledWith(expect.any(Object), 'offset', 0); + expect(expander.addParamInt).toHaveBeenCalledWith(expect.any(Object), 'count', 1000); + expect(expander.includeCodeAndDescendants).toHaveBeenCalled(); + }); + + test('filtered whole-system include keeps default too-costly behavior', async () => { + patchValueSetExpandWholeSystemForOcl(); + + const cs = { + specialEnumeration: () => null, + isNotClosed: () => false, + iterator: async () => ({ total: 5001 }), + nextContext: async () => null, + system: async () => 'http://example.org/cs/source-one', + version: async () => '1.0.0' + }; + + const expander = new ValueSetExpander({ + internalLimit: 10000, + externalLimit: 1000, + opContext: { log: () => {}, diagnostics: () => '' }, + deadCheck: () => {}, + findCodeSystem: async () => cs, + checkSupplements: () => {}, + i18n: { languageDefinitions: {}, translate: (_k, _langs, args) => `too costly ${args?.join(' ') || ''}` }, + provider: { getFhirVersion: () => '5.0.0' }, + languages: { parse: () => null } + }, { + hasDesignations: false, + designations: [], + httpLanguages: null + }); + + expander.valueSet = {}; + expander.limitCount = 1000; + expander.count = -1; + expander.offset = -1; + expander.hasExclusions = false; + expander.requiredSupplements = new Set(); + expander.usedSupplements = new Set(); + expander.map = new Map(); + expander.fullList = []; + expander.rootList = []; + expander.passesFilters = jest.fn(async () => true); + expander.includeCodeAndDescendants = jest.fn(async () => 1); + expander.checkProviderCanonicalStatus = jest.fn(); + expander.addParamUri = jest.fn(); + expander.addParamInt = jest.fn(); + + let thrown = null; + try { + await expander.includeCodes( + { system: 'http://example.org/cs/source-one' }, + 'ValueSet.compose.include[0]', + { vurl: 'http://example.org/vs/one|1.0.0', url: 'http://example.org/vs/one' }, + { include: [{ system: 'http://example.org/cs/source-one' }] }, + { isNull: true }, + {}, + false, + { value: false } + ); + } catch (error) { + thrown = error; + } + + expect(thrown).toBeTruthy(); + expect(thrown.cause).toBe('too-costly'); + }); +}); diff --git a/tx/cs/cs-loinc.js b/tx/cs/cs-loinc.js index d1785bb5..efa6b9eb 100644 --- a/tx/cs/cs-loinc.js +++ b/tx/cs/cs-loinc.js @@ -203,9 +203,12 @@ class LoincServices extends BaseCSServices { // Use language-aware display logic if (this.opContext.langs && !this.opContext.langs.isEnglishOrNothing()) { const displays = await this.#getDisplaysForContext(ctxt, this.opContext.langs); + const requestedLanguages = Array.isArray(this.opContext.langs.languages) + ? this.opContext.langs.languages + : (Array.isArray(this.opContext.langs.langs) ? this.opContext.langs.langs : []); // Try to find exact language match - for (const lang of this.opContext.langs.langs) { + for (const lang of requestedLanguages) { for (const display of displays) { if (lang.matches(display.language, true)) { return display.value; @@ -214,7 +217,7 @@ class LoincServices extends BaseCSServices { } // Try partial language match - for (const lang of this.opContext.langs.langs) { + for (const lang of requestedLanguages) { for (const display of displays) { if (lang.matches(display.language, false)) { return display.value; diff --git a/tx/cs/cs-provider-api.js b/tx/cs/cs-provider-api.js index 3ae74369..cc7131e0 100644 --- a/tx/cs/cs-provider-api.js +++ b/tx/cs/cs-provider-api.js @@ -20,7 +20,13 @@ class AbstractCodeSystemProvider { } /** - * Returns the list of CodeSystems this provider provides + * Returns the list of CodeSystems this provider provides. This is called once at start up. + * The code systems should be fully loaded; lazy loading code systems is not considered good + * for engineering. + * + * + * Note that unlike value sets, which are accessed from the provider on the fly, code systems + * are all preloaded into the kernel (e.g. provider) at start up * * @param {string} fhirVersion - The FHIRVersion in scope - if relevant (there's always a stated version, though R5 is always used) * @param {string} context - The client's stated context - if provided. @@ -32,6 +38,24 @@ class AbstractCodeSystemProvider { throw new Error('listCodeSystems must be implemented by AbstractCodeSystemProvider subclass'); } + /** + * This is called once a minute to update the code system list that the provider maintains. + * + * return an object that has three Map: {added, changed, deleted} + * + * these use the same key as the + * + * code systems are identified by url and version + * + * @param fhirVersion + * @param context + * @returns {Promise} + */ + // eslint-disable-next-line no-unused-vars + async getCodeSystemChanges(fhirVersion, context){ + return null; + } + async close() { } diff --git a/tx/cs/cs-provider-list.js b/tx/cs/cs-provider-list.js index 9fb55891..019c457f 100644 --- a/tx/cs/cs-provider-list.js +++ b/tx/cs/cs-provider-list.js @@ -7,7 +7,7 @@ class ListCodeSystemProvider extends AbstractCodeSystemProvider { /** * {Map} A list of code system factories that contains all the preloaded native code systems */ - codeSystems = new Map(); + codeSystems = []; /** * ensure that the ids on the code systems are unique, if they are @@ -17,7 +17,7 @@ class ListCodeSystemProvider extends AbstractCodeSystemProvider { */ // eslint-disable-next-line no-unused-vars assignIds(ids) { - for (const cs of this.codeSystems.values()) { + for (const cs of this.codeSystems) { if (!cs.id || ids.has("CodeSystem/"+cs.id)) { cs.id = ""+ids.size; } diff --git a/tx/library.js b/tx/library.js index c1a23597..56def726 100644 --- a/tx/library.js +++ b/tx/library.js @@ -31,6 +31,9 @@ const { Provider } = require("./provider"); const {I18nSupport} = require("../library/i18nsupport"); const folders = require('../library/folder-setup'); const {VSACValueSetProvider} = require("./vs/vs-vsac"); +const { OCLCodeSystemProvider, OCLSourceCodeSystemFactory } = require('./ocl/cs-ocl'); +const { OCLValueSetProvider } = require('./ocl/vs-ocl'); +const { OCLConceptMapProvider } = require('./ocl/cm-ocl'); /** * This class holds all the loaded content ready for processing @@ -95,6 +98,8 @@ class Library { this.codeSystemProviders = []; this.valueSetProviders = []; this.conceptMapProviders = []; + this.oclProviderSets = new Map(); + this.oclConfig = {}; // Create package manager for FHIR packages const packageServers = ['https://packages2.fhir.org/packages']; @@ -154,6 +159,7 @@ class Library { const yamlContent = await fs.readFile(yamlPath, 'utf8'); const config = yaml.parse(yamlContent); this.baseUrl = config.base.url; + this.oclConfig = config.ocl && typeof config.ocl === 'object' ? config.ocl : {}; this.log.info('Fetching Data from '+this.baseUrl); @@ -277,12 +283,127 @@ class Library { case 'url/cs': await this.loadUrl(packageManager, details, isDefault, mode, true); break; + + case 'ocl': + await this.loadOcl(details, isDefault, mode); + break; default: throw new Error(`Unknown source type: ${type}`); } } + parseOclConfig(details) { + const text = String(details || '').trim(); + if (!text) { + throw new Error('OCL source requires details, e.g. ocl:https://ocl.example.org'); + } + + const parts = text.split('|').map(p => p.trim()).filter(Boolean); + const baseUrl = this.resolveOclConfigValue(parts[0]); + if (!baseUrl) { + throw new Error('OCL source requires a base URL'); + } + + const config = { baseUrl }; + for (let i = 1; i < parts.length; i++) { + const part = parts[i]; + const eq = part.indexOf('='); + if (eq === -1) { + continue; + } + const key = part.substring(0, eq).trim().toLowerCase(); + const value = this.resolveOclConfigValue(part.substring(eq + 1).trim()); + if (!value) { + continue; + } + if (key === 'org') { + config.org = value; + } else if (key === 'token') { + config.token = value; + } else if (key === 'timeout') { + const timeout = Number(value); + if (Number.isFinite(timeout) && timeout > 0) { + config.timeout = timeout; + } + } + } + + return config; + } + + resolveOclConfigValue(value) { + const text = String(value || '').trim(); + if (!text) { + return text; + } + + if (this.oclConfig && Object.hasOwn(this.oclConfig, text)) { + const resolved = this.oclConfig[text]; + if (typeof resolved === 'string') { + return resolved.trim(); + } + } + + return text; + } + + async loadOcl(details, isDefault, mode) { + const config = this.parseOclConfig(details); + const cacheKey = `${config.baseUrl}|${config.org || ''}`; + + let providerSet = this.oclProviderSets.get(cacheKey); + if (!providerSet) { + const codeSystemProvider = new OCLCodeSystemProvider(config); + const valueSetProvider = new OCLValueSetProvider(config); + const conceptMapProvider = new OCLConceptMapProvider(config); + providerSet = { + config, + codeSystemProvider, + valueSetProvider, + conceptMapProvider, + csRegistered: false, + factoriesRegistered: false, + vsRegistered: false, + cmRegistered: false + }; + this.oclProviderSets.set(cacheKey, providerSet); + } + + if (mode === 'fetch') { + return; + } + + if (mode === 'cs') { + if (!providerSet.csRegistered) { + this.codeSystemProviders.push(providerSet.codeSystemProvider); + providerSet.csRegistered = true; + } + + if (!providerSet.factoriesRegistered) { + await providerSet.codeSystemProvider.listCodeSystems('5.0', null); + const metas = providerSet.codeSystemProvider.getSourceMetas(); + for (const meta of metas) { + const factory = new OCLSourceCodeSystemFactory(this.i18n, providerSet.codeSystemProvider.httpClient, meta); + this.registerProvider(`ocl:${config.baseUrl}`, factory, isDefault); + } + providerSet.factoriesRegistered = true; + } + return; + } + + if (mode === 'npm') { + if (!providerSet.vsRegistered) { + this.valueSetProviders.push(providerSet.valueSetProvider); + providerSet.vsRegistered = true; + } + if (!providerSet.cmRegistered) { + this.conceptMapProviders.push(providerSet.conceptMapProvider); + providerSet.cmRegistered = true; + } + } + } + async loadInternal(details, isDefault, mode) { if (isDefault) { throw new Error("Default is not supported for internal code system providers"); @@ -464,13 +585,7 @@ class Library { for (const resource of resources) { const cs = new CodeSystem(await contentLoader.loadFile(resource, contentLoader.fhirVersion())); cs.sourcePackage = contentLoader.pid(); - const existing = cp.codeSystems.get(cs.url); - if (!existing || cs.isMoreRecent(existing)) { - cp.codeSystems.set(cs.url, cs); - } - if (cs.version) { - cp.codeSystems.set(cs.vurl, cs); - } + cp.codeSystems.push(cs); csc++; } this.codeSystemProviders.push(cp); @@ -686,10 +801,12 @@ class Library { } + provider.codeSystemProviders = this.codeSystemProviders; + provider.context = context; for (const cp of this.codeSystemProviders) { - const csMap = await cp.listCodeSystems(fhirVersion, context); - for (const [key, value] of csMap) { - provider.codeSystems.set(key, value); + const csList = await cp.listCodeSystems(fhirVersion, context); + for (const cs of csList) { + provider.addCodeSystem(cs); } } // Don't clone valueSetProviders yet - we'll build it with correct order diff --git a/tx/library/canonical-resource.js b/tx/library/canonical-resource.js index 6098d876..112a221d 100644 --- a/tx/library/canonical-resource.js +++ b/tx/library/canonical-resource.js @@ -111,7 +111,12 @@ class CanonicalResource { const fmt = this.versionAlgorithm() || other.versionAlgorithm() || this.guessVersionAlgorithmFromVersion(this.version); switch (fmt) { case 'semver': - return VersionUtilities.isThisOrLater(other.version, this.version, VersionPrecision.PATCH); + try { + return VersionUtilities.isThisOrLater(other.version, this.version, VersionPrecision.PATCH); + } catch (error) { + // other is not semver. Not much we can do + return false; + } case 'date': return this.dateIsMoreRecent(this.version, other.version); case 'integer': diff --git a/tx/ocl/README.md b/tx/ocl/README.md new file mode 100644 index 00000000..c46324c2 --- /dev/null +++ b/tx/ocl/README.md @@ -0,0 +1,236 @@ +# OCL Integration in FHIRsmith + +## Overview +The `tx/ocl` module integrates [Open Concept Lab (OCL)](https://openconceptlab.org/) as a terminology source inside FHIRsmith. + +It provides adapters/providers for: + +- `CodeSystem` from OCL Sources +- `ValueSet` from OCL Collections +- `ConceptMap` from OCL Mappings + +In FHIRsmith terms, these providers are loaded by `tx/library.js` when a source entry starts with `ocl:`. OCL metadata is discovered from OCL APIs, and heavy content (concept lists/expansions) is loaded lazily and warmed in background jobs. + +## Architecture +### Main modules +- `cs-ocl.js` + - `OCLCodeSystemProvider`: discovers OCL sources and publishes CodeSystem metadata. + - `OCLSourceCodeSystemFactory`: creates runtime CodeSystem providers with shared caches, cold-cache hydration, and background full-load jobs. + - `OCLSourceCodeSystemProvider`: runtime concept lookup/filter/search behavior for one source. +- `vs-ocl.js` + - `OCLValueSetProvider`: discovers OCL collections, resolves compose includes, serves ValueSet metadata, and builds cached expansions in background. +- `cm-ocl.js` + - `OCLConceptMapProvider`: resolves OCL mappings for fetch/search/translation candidate discovery. + +### Supporting modules under `tx/ocl` +- `http/client.js`: Axios client creation, base URL normalization, token auth header (`Token` or `Bearer`). +- `http/pagination.js`: OCL pagination helper (`results/items/data`, `next`, page mode fallback). +- `cache/cache-paths.js`: cold-cache directories and canonical URL to file path mapping. +- `cache/cache-utils.js`: cache directory creation, file age detection, friendly age formatting. +- `fingerprint/fingerprint.js`: deterministic SHA-256 fingerprints for CodeSystem concepts and ValueSet expansions. +- `jobs/background-queue.js`: singleton keyed queue, size-priority ordering, max concurrency = 2, heartbeat logging every 30s. +- `mappers/concept-mapper.js`: maps OCL concept payloads to internal concept context shape. +- `model/concept-filter-context.js`: ranked filter result set used by CodeSystem filters. +- `shared/constants.js`: defaults and constants (`PAGE_SIZE`, cache freshness window, etc.). +- `shared/patches.js`: + - patches search worker so `CodeSystem?url=...&code=...` on OCL resources only returns matching concept subtree. + - patches `TxParameters.hashSource()` to include `filter` for expansion cache key differentiation. + +## Runtime flow +### Metadata discovery +CodeSystems (`cs-ocl.js`): +- discover orgs via `/orgs/` +- for each org, discover sources via `/orgs/{org}/sources/` +- fallback to `/sources/` if org listing is unavailable + +ValueSets (`vs-ocl.js`): +- discover orgs via `/orgs/` +- discover collections via `/orgs/{org}/collections/` +- fallback to `/collections/` + +ConceptMaps (`cm-ocl.js`): +- fetch by id via `/mappings/{id}/` +- search via `/mappings/` or `/orgs/{org}/mappings/` + +### Lazy loading +- CodeSystem concepts are not fully loaded at metadata discovery time. +- ValueSet expansions are not built inline by default. +- Missing concept/page/expansion data triggers background warm-up scheduling. + +### Cold cache (disk) +- Base folder: `data/terminology-cache/ocl` +- CodeSystems: `data/terminology-cache/ocl/codesystems` +- ValueSets: `data/terminology-cache/ocl/valuesets` + +On startup/initialization: +- CodeSystem factory hydrates concept cache from cold cache file if present. +- ValueSet provider loads cached expansions from cold cache files. + +Corrupt cache handling: +- JSON parse/read errors are logged and skipped; provider continues without crashing. + +### Hot cache (memory) +- CodeSystems: shared in-memory concept/page caches per factory. +- ValueSets: in-memory expansion cache keyed by ValueSet + params hash. +- ConceptMaps: in-memory map keyed by URL/version/id. + +### Background warm-up jobs +Queue behavior (`OCLBackgroundJobQueue`): +- singleton job key (skip duplicate key) +- max concurrency = `2` +- ordering by `jobSize` (smaller concept count first) +- heartbeat log every `30s` +- progress supports `{ processed, total }` and `%` + +CodeSystem warm-up: +- skipped when cold-cache file age is `<= 1 hour` +- enqueued when stale/no cold cache +- loads all concept pages +- computes fingerprint from full concept content +- if fingerprint changed, replaces cold cache file + +ValueSet warm-up: +- skipped when freshest cold cache age (file mtime or cached timestamp) is `<= 1 hour` +- enqueued when stale/no cold cache +- builds expansion by paging collection concepts +- computes expansion fingerprint +- if fingerprint changed, replaces cold cache file + +### Fingerprint/checksum strategy +- OCL source checksum is treated as informational only in `cs-ocl.js` comments. +- Cache replacement decisions use custom fingerprints from concept/expansion content. +- ValueSet cache validity also checks metadata signature and dependency checksums from referenced CodeSystems. + +### ValueSet expansion filtering +`vs-ocl.js` supports filtered concept retrieval through `valueSet.oclFetchConcepts(...)`: +- local baseline filtering (code/display/definition text contains) +- `SearchFilterText` token behavior +- remote query hint (`q`) generated from normalized filter tokens +- dedicated smaller page size for filtered calls (`FILTERED_CONCEPT_PAGE_SIZE`) + +### `/CodeSystem?url=...&code=...` behavior for OCL +`shared/patches.js` patches search worker to apply concept subtree filtering only for OCL-marked CodeSystems (extension URL `http://fhir.org/FHIRsmith/StructureDefinition/ocl-codesystem`). + +## Configuration +## Activation in FHIRsmith +OCL is activated by adding an OCL source line in the TX library YAML (for example `data/library.yml`): + +```yaml +sources: + - ocl:https://oclapi2.ips.hsl.org.br +``` + +`tx/library.js` loads this via `loadOcl()`. + +### Source syntax +`tx/library.js` parses: + +```text +ocl:|org=|token=|timeout= +``` + +Parsed keys: +- `baseUrl` (required) +- `org` (optional) +- `token` (optional) +- `timeout` (optional positive number) + +Examples: + +```yaml +sources: + - ocl:https://api.openconceptlab.org + - ocl:https://ocl.example.org|org=my-org + - ocl:https://ocl.example.org|org=my-org|token=my-ocl-token|timeout=45000 +``` + +### Config value aliasing +`Library.resolveOclConfigValue()` can resolve symbolic values from top-level YAML `ocl:` object loaded into `this.oclConfig`. + +Example pattern: + +```yaml +ocl: + ocl-base: https://ocl.example.org + ocl-token: Token abc123 + +sources: + - ocl:ocl-base|org=my-org|token=ocl-token +``` + +### Credentials and URLs +- Base URL is required (OCL API root). +- Token is optional, sent as `Authorization`: + - `Token ` if no prefix is provided + - preserved if already `Token ...` or `Bearer ...` + +### Enable/disable integration +- Enable by including at least one `ocl:` source entry in TX library YAML. +- Disable by removing/commenting those `ocl:` entries. +- `modules.tx.enabled` must be true in server config to expose TX endpoints. + +## Cache behavior details +### Startup hydration +- CodeSystem cold cache is loaded when factory is created (`OCLSourceCodeSystemFactory` constructor path). +- ValueSet cold cache is loaded during `OCLValueSetProvider.initialize()`. + +### 1-hour freshness rule +- Fresh threshold constant: `COLD_CACHE_FRESHNESS_MS = 60 * 60 * 1000`. +- If cold cache age is `<= 1 hour`, warm-up scheduling is skipped. + +### Hot cache replacing cold cache +- On successful background refresh, new fingerprint is compared to previous cold-cache fingerprint. +- If changed, cold cache is overwritten with the refreshed in-memory state. + +## Operational notes +### Logs to expect +Prefixes: +- `[OCL]` for CodeSystem/queue/general OCL flow +- `[OCL-ValueSet]` for ValueSet flow + +Typical events: +- fetched org/source/collection counts +- cold cache loaded/saved +- warm-up skipped/enqueued/started/completed +- fingerprint unchanged/changed +- queue status + heartbeat snapshots + +### Troubleshooting hints +- If nothing loads, verify OCL base URL and org visibility. +- If cache never warms, check cold-cache timestamps and 1-hour rule. +- If searches by `code` behave unexpectedly, confirm OCL marker extension is present on CodeSystem resources. +- For ValueSet filter behavior, verify `filter` is included in request path through `TxParameters.hashSource` patch and that filter text normalizes as expected. + +### Known limitations (from current implementation) +- OCL checksums are not relied on for cache invalidation. +- Some source/collection discovery paths depend on OCL endpoint support and can fallback. +- Missing endpoints returning `404` in some concept fetch paths are treated as empty content for graceful degradation. + +## Developer notes +### Where to extend +- Add/adjust OCL HTTP behavior: `tx/ocl/http/*` +- Add cache policy/path behavior: `tx/ocl/cache/*` +- Add queue policy: `tx/ocl/jobs/background-queue.js` +- Add mapping logic from OCL payloads: `tx/ocl/mappers/*` +- Add fingerprint strategy: `tx/ocl/fingerprint/*` +- Add provider-specific behavior: + - CodeSystem: `tx/ocl/cs-ocl.js` + - ValueSet: `tx/ocl/vs-ocl.js` + - ConceptMap: `tx/ocl/cm-ocl.js` + +### Ownership by concern +- HTTP access: `http/client.js`, `http/pagination.js` +- Disk cold cache and pathing: `cache/cache-paths.js`, `cache/cache-utils.js` +- Background jobs and scheduling policy: `jobs/background-queue.js` +- Fingerprints/checksums: `fingerprint/fingerprint.js` and provider compare logic +- Worker/runtime patches: `shared/patches.js` + +### Test strategy for future changes +- Keep provider tests deterministic by mocking OCL HTTP responses and cold-cache files. +- Validate stale/fresh transitions with file mtime control. +- Validate queue behavior with isolated static state reset between tests. +- Track coverage with: + +```bash +npm test -- --runInBand tests/ocl --coverage --collectCoverageFrom="tx/ocl/**/*.js" +``` diff --git a/tx/ocl/cache/cache-paths.cjs b/tx/ocl/cache/cache-paths.cjs new file mode 100644 index 00000000..c587cf66 --- /dev/null +++ b/tx/ocl/cache/cache-paths.cjs @@ -0,0 +1,32 @@ +const path = require('path'); + +const CACHE_BASE_DIR = path.join(process.cwd(), 'data', 'terminology-cache', 'ocl'); +const CACHE_CS_DIR = path.join(CACHE_BASE_DIR, 'codesystems'); +const CACHE_VS_DIR = path.join(CACHE_BASE_DIR, 'valuesets'); + +function sanitizeFilename(text) { + if (!text || typeof text !== 'string') { + return 'unknown'; + } + return text + .replace(/[^a-zA-Z0-9._-]/g, '_') + .replace(/_+/g, '_') + .substring(0, 200); +} + +function getCacheFilePath(baseDir, canonicalUrl, version = null, paramsKey = null) { + const filename = sanitizeFilename(canonicalUrl) + + (version ? `_${sanitizeFilename(version)}` : '') + + (paramsKey && paramsKey !== 'default' ? `_p_${sanitizeFilename(paramsKey)}` : '') + + '.json'; + + return path.join(baseDir, filename); +} + +module.exports = { + CACHE_BASE_DIR, + CACHE_CS_DIR, + CACHE_VS_DIR, + sanitizeFilename, + getCacheFilePath +}; diff --git a/tx/ocl/cache/cache-paths.js b/tx/ocl/cache/cache-paths.js new file mode 100644 index 00000000..186eed8c --- /dev/null +++ b/tx/ocl/cache/cache-paths.js @@ -0,0 +1,2 @@ +module.exports = require('./cache-paths.cjs'); + diff --git a/tx/ocl/cache/cache-utils.cjs b/tx/ocl/cache/cache-utils.cjs new file mode 100644 index 00000000..b0a89af5 --- /dev/null +++ b/tx/ocl/cache/cache-utils.cjs @@ -0,0 +1,43 @@ +const fs = require('fs/promises'); +const fsSync = require('fs'); + +async function ensureCacheDirectories(...dirs) { + for (const dir of dirs) { + if (!dir) { + continue; + } + + try { + await fs.mkdir(dir, { recursive: true }); + } catch (error) { + console.error('[OCL] Failed to create cache directory:', dir, error.message); + } + } +} + +function getColdCacheAgeMs(cacheFilePath, logPrefix = '[OCL]') { + try { + const stats = fsSync.statSync(cacheFilePath); + if (!stats || !Number.isFinite(stats.mtimeMs)) { + return null; + } + + return Math.max(0, Date.now() - stats.mtimeMs); + } catch (error) { + if (error && error.code !== 'ENOENT') { + console.error(`${logPrefix} Failed to inspect cold cache file ${cacheFilePath}: ${error.message}`); + } + return null; + } +} + +function formatCacheAgeMinutes(ageMs) { + const minutes = Math.max(1, Math.round(ageMs / 60000)); + return `${minutes} minute${minutes === 1 ? '' : 's'}`; +} + +module.exports = { + ensureCacheDirectories, + getColdCacheAgeMs, + formatCacheAgeMinutes +}; diff --git a/tx/ocl/cache/cache-utils.js b/tx/ocl/cache/cache-utils.js new file mode 100644 index 00000000..402736aa --- /dev/null +++ b/tx/ocl/cache/cache-utils.js @@ -0,0 +1,2 @@ +module.exports = require('./cache-utils.cjs'); + diff --git a/tx/ocl/cm-ocl.cjs b/tx/ocl/cm-ocl.cjs new file mode 100644 index 00000000..c91fc486 --- /dev/null +++ b/tx/ocl/cm-ocl.cjs @@ -0,0 +1,531 @@ +const { AbstractConceptMapProvider } = require('../cm/cm-api'); +const { ConceptMap } = require('../library/conceptmap'); +const { PAGE_SIZE } = require('./shared/constants'); +const { createOclHttpClient } = require('./http/client'); +const { fetchAllPages, extractItemsAndNext } = require('./http/pagination'); + +const DEFAULT_MAX_SEARCH_PAGES = 10; + +class OCLConceptMapProvider extends AbstractConceptMapProvider { + constructor(config = {}) { + super(); + const options = typeof config === 'string' ? { baseUrl: config } : (config || {}); + + this.org = options.org || null; + this.maxSearchPages = options.maxSearchPages || DEFAULT_MAX_SEARCH_PAGES; + const http = createOclHttpClient(options); + this.baseUrl = http.baseUrl; + this.httpClient = http.client; + + this.conceptMapMap = new Map(); + this._idMap = new Map(); + this._sourceCandidatesCache = new Map(); + this._sourceUrlsByCanonical = new Map(); + this._canonicalBySourceUrl = new Map(); + } + + assignIds(ids) { + if (!this.spaceId) { + return; + } + + const unique = new Set(this.conceptMapMap.values()); + this._idMap.clear(); + + for (const cm of unique) { + if (!cm.id.startsWith(`${this.spaceId}-`)) { + const nextId = `${this.spaceId}-${cm.id}`; + cm.id = nextId; + cm.jsonObj.id = nextId; + } + this._idMap.set(cm.id, cm); + ids.add(`ConceptMap/${cm.id}`); + } + } + + async fetchConceptMap(url, version) { + this._validateFetchParams(url, version); + + const direct = this.conceptMapMap.get(`${url}|${version}`) || this.conceptMapMap.get(url); + if (direct) { + return direct; + } + + const mappingId = this.#extractMappingId(url); + if (mappingId) { + return await this.fetchConceptMapById(mappingId); + } + + const mappings = await this.#searchMappings({ from_source_url: url }, this.maxSearchPages); + for (const mapping of mappings) { + const cm = this.#toConceptMap(mapping); + if (cm) { + this.#indexConceptMap(cm); + if (cm.url === url && (!version || cm.version === version)) { + return cm; + } + } + } + + return null; + } + + async fetchConceptMapById(id) { + if (this._idMap.has(id)) { + return this._idMap.get(id); + } + + let rawId = id; + if (this.spaceId && id.startsWith(`${this.spaceId}-`)) { + rawId = id.substring(this.spaceId.length + 1); + } + + if (this._idMap.has(rawId)) { + return this._idMap.get(rawId); + } + + const response = await this.httpClient.get(`/mappings/${encodeURIComponent(rawId)}/`); + const cm = this.#toConceptMap(response.data); + if (!cm) { + return null; + } + this.#indexConceptMap(cm); + return cm; + } + + // eslint-disable-next-line no-unused-vars + async searchConceptMaps(searchParams, _elements) { + this._validateSearchParams(searchParams); + + const params = Object.fromEntries(searchParams.map(({ name, value }) => [name, String(value).toLowerCase()])); + const oclParams = {}; + + if (params.source) { + oclParams.from_source_url = params.source; + } + if (params.target) { + oclParams.to_source_url = params.target; + } + + const mappings = await this.#searchMappings(oclParams, this.maxSearchPages); + const results = []; + for (const mapping of mappings) { + const cm = this.#toConceptMap(mapping); + if (!cm) { + continue; + } + this.#indexConceptMap(cm); + if (this.#matches(cm.jsonObj, params)) { + results.push(cm); + } + } + return results; + } + + async findConceptMapForTranslation(opContext, conceptMaps, sourceSystem, sourceScope, targetScope, targetSystem, sourceCode = null) { + const sourceCandidates = await this.#candidateSourceUrls(sourceSystem); + const targetCandidates = await this.#candidateSourceUrls(targetSystem); + + const mappings = []; + const sourcePaths = sourceCandidates.filter(s => String(s || '').startsWith('/orgs/')); + + if (sourceCode && sourcePaths.length > 0) { + for (const sourcePath of sourcePaths) { + const conceptPath = `${this.#normalizeSourcePath(sourcePath)}concepts/${encodeURIComponent(sourceCode)}/mappings/`; + const found = await this.#fetchAllPages(conceptPath, { limit: PAGE_SIZE }, Math.min(2, this.maxSearchPages)); + mappings.push(...found); + } + } + + const searchKeys = new Set(); + const searches = []; + + if (sourceCandidates.length === 0 && targetCandidates.length === 0) { + searches.push({}); + } else if (targetCandidates.length === 0) { + for (const src of sourceCandidates) { + const key = `from:${src}`; + if (!searchKeys.has(key)) { + searchKeys.add(key); + searches.push({ from_source_url: src }); + } + } + } else if (sourceCandidates.length === 0) { + for (const tgt of targetCandidates) { + const key = `to:${tgt}`; + if (!searchKeys.has(key)) { + searchKeys.add(key); + searches.push({ to_source_url: tgt }); + } + } + } else { + for (const src of sourceCandidates) { + for (const tgt of targetCandidates) { + const key = `from:${src}|to:${tgt}`; + if (!searchKeys.has(key)) { + searchKeys.add(key); + searches.push({ from_source_url: src, to_source_url: tgt }); + } + } + } + } + + if (mappings.length === 0) { + for (const search of searches) { + const found = await this.#searchMappings(search, Math.min(2, this.maxSearchPages)); + mappings.push(...found); + } + } + + const sourceUrlsToResolve = new Set(); + for (const mapping of mappings) { + const fromSource = mapping?.from_source_url || mapping?.fromSourceUrl; + const toSource = mapping?.to_source_url || mapping?.toSourceUrl; + if (fromSource) { + sourceUrlsToResolve.add(fromSource); + } + if (toSource) { + sourceUrlsToResolve.add(toSource); + } + } + await this.#ensureCanonicalForSourceUrls(sourceUrlsToResolve); + + const seen = new Set(conceptMaps.map(cm => cm.id || cm.url)); + for (const mapping of mappings) { + const cm = this.#toConceptMap(mapping); + if (!cm) { + continue; + } + this.#indexConceptMap(cm); + + const key = cm.id || cm.url; + if (seen.has(key)) { + continue; + } + + if (this.#matchesTranslationRequest(cm, sourceSystem, sourceScope, targetScope, targetSystem, sourceCandidates, targetCandidates)) { + conceptMaps.push(cm); + seen.add(key); + } + } + } + + cmCount() { + return new Set(this.conceptMapMap.values()).size; + } + + async close() { + } + + #indexConceptMap(cm) { + this.conceptMapMap.set(cm.url, cm); + if (cm.version) { + this.conceptMapMap.set(`${cm.url}|${cm.version}`, cm); + } + this.conceptMapMap.set(cm.id, cm); + this._idMap.set(cm.id, cm); + } + + #toConceptMap(mapping) { + if (!mapping || typeof mapping !== 'object') { + return null; + } + + const id = mapping.id; + if (!id) { + return null; + } + + const url = mapping.url || `${this.baseUrl}/mappings/${id}`; + const source = mapping.from_source_url || mapping.fromSourceUrl || mapping.from_concept_url || mapping.fromConceptUrl || null; + const target = mapping.to_source_url || mapping.toSourceUrl || mapping.to_concept_url || mapping.toConceptUrl || null; + const sourceCode = mapping.from_concept_code || mapping.fromConceptCode; + const targetCode = mapping.to_concept_code || mapping.toConceptCode; + + if (!source || !target || !sourceCode || !targetCode) { + return null; + } + + const sourceDisplay = mapping.from_concept_name_resolved || mapping.fromConceptNameResolved || mapping.from_concept_name || mapping.fromConceptName || null; + const targetDisplay = mapping.to_concept_name_resolved || mapping.toConceptNameResolved || mapping.to_concept_name || mapping.toConceptName || null; + const sourceCanonical = this.#canonicalForSourceUrl(source) || source; + const targetCanonical = this.#canonicalForSourceUrl(target) || target; + + const relationship = this.#toRelationship(mapping.map_type || mapping.mapType); + const lastUpdated = this.#toIsoDate(mapping.updated_on || mapping.updatedOn || mapping.updated_at || mapping.updatedAt); + + const json = { + resourceType: 'ConceptMap', + id, + url, + version: mapping.version || null, + name: `mapping-${id}`, + title: mapping.name || `Mapping ${id}`, + status: 'active', + sourceScopeUri: mapping.from_collection_url || mapping.fromCollectionUrl || source, + targetScopeUri: mapping.to_collection_url || mapping.toCollectionUrl || target, + group: [ + { + source: sourceCanonical, + target: targetCanonical, + element: [ + { + code: sourceCode, + display: sourceDisplay, + target: [ + { + code: targetCode, + display: targetDisplay, + relationship, + comment: mapping.comment || null + } + ] + } + ] + } + ] + }; + + if (lastUpdated) { + json.meta = { lastUpdated }; + } + + return new ConceptMap(json, 'R5'); + } + + #toRelationship(mapType) { + switch ((mapType || '').toUpperCase()) { + case 'SAME-AS': + return 'equivalent'; + case 'NARROWER-THAN': + return 'narrower-than'; + case 'BROADER-THAN': + return 'broader-than'; + case 'NOT-EQUIVALENT': + return 'not-related-to'; + default: + return 'related-to'; + } + } + + #matches(json, params) { + for (const [name, value] of Object.entries(params)) { + if (!value) { + continue; + } + + if (name === 'url') { + if ((json.url || '').toLowerCase() !== value) { + return false; + } + continue; + } + + if (name === 'source') { + const src = json.group?.[0]?.source || ''; + if (!src.toLowerCase().includes(value)) { + return false; + } + continue; + } + + if (name === 'target') { + const tgt = json.group?.[0]?.target || ''; + if (!tgt.toLowerCase().includes(value)) { + return false; + } + continue; + } + + const field = json[name]; + if (field == null || !String(field).toLowerCase().includes(value)) { + return false; + } + } + return true; + } + + async #searchMappings(params = {}, maxPages = this.maxSearchPages) { + const endpoint = this.org ? `/orgs/${encodeURIComponent(this.org)}/mappings/` : '/mappings/'; + return await this.#fetchAllPages(endpoint, params, maxPages); + } + + async #fetchAllPages(path, params = {}, maxPages = this.maxSearchPages) { + return await fetchAllPages(this.httpClient, path, { + params, + pageSize: PAGE_SIZE, + maxPages, + baseUrl: this.baseUrl + }); + } + + #extractItemsAndNext(payload) { + return extractItemsAndNext(payload, this.baseUrl); + } + + #extractMappingId(url) { + if (!url) { + return null; + } + const match = url.match(/\/mappings\/([^/]+)\/?$/i); + return match ? match[1] : null; + } + + async #candidateSourceUrls(systemUrl) { + if (!systemUrl) { + return []; + } + + const cacheKey = this.#norm(systemUrl); + if (this._sourceCandidatesCache.has(cacheKey)) { + return this._sourceCandidatesCache.get(cacheKey); + } + + const result = new Set(); + result.add(systemUrl); + + const canonicalKey = cacheKey; + const byCanonical = this._sourceUrlsByCanonical.get(canonicalKey); + if (byCanonical) { + for (const item of byCanonical) { + result.add(item); + } + } + + const discovered = await this.#resolveSourceCandidatesFromOcl(systemUrl); + for (const item of discovered) { + result.add(item); + } + + const out = Array.from(result); + this._sourceCandidatesCache.set(cacheKey, out); + return out; + } + + async #resolveSourceCandidatesFromOcl(systemUrl) { + const endpoint = this.org ? `/orgs/${encodeURIComponent(this.org)}/sources/` : '/sources/'; + const query = this.#queryTokenFromSystem(systemUrl); + if (!query) { + return []; + } + + const sources = await this.#fetchAllPages(endpoint, { q: query, limit: PAGE_SIZE }, 2); + const targetNorm = this.#norm(systemUrl); + const candidates = new Set(); + + for (const source of sources) { + const canonical = source?.canonical_url || source?.canonicalUrl || null; + const sourceUrl = source?.url || source?.uri || null; + if (!sourceUrl) { + continue; + } + + if (canonical) { + const canonicalKey = this.#norm(canonical); + if (!this._sourceUrlsByCanonical.has(canonicalKey)) { + this._sourceUrlsByCanonical.set(canonicalKey, new Set()); + } + this._sourceUrlsByCanonical.get(canonicalKey).add(sourceUrl); + this._canonicalBySourceUrl.set(this.#norm(sourceUrl), canonical); + + if (canonicalKey === targetNorm) { + candidates.add(sourceUrl); + } + } + + if (this.#norm(sourceUrl) === targetNorm) { + candidates.add(sourceUrl); + } + } + + return Array.from(candidates); + } + + async #ensureCanonicalForSourceUrls(sourceUrls) { + for (const sourceUrl of sourceUrls || []) { + const sourceKey = this.#norm(sourceUrl); + if (!sourceKey || this._canonicalBySourceUrl.has(sourceKey)) { + continue; + } + + const sourcePath = String(sourceUrl || '').trim(); + if (!sourcePath.startsWith('/orgs/')) { + continue; + } + + try { + const response = await this.httpClient.get(sourcePath); + const source = response.data || {}; + const canonical = source.canonical_url || source.canonicalUrl; + const resolvedSourceUrl = source.url || source.uri || sourcePath; + if (!canonical) { + continue; + } + + const canonicalKey = this.#norm(canonical); + if (!this._sourceUrlsByCanonical.has(canonicalKey)) { + this._sourceUrlsByCanonical.set(canonicalKey, new Set()); + } + this._sourceUrlsByCanonical.get(canonicalKey).add(resolvedSourceUrl); + this._canonicalBySourceUrl.set(this.#norm(resolvedSourceUrl), canonical); + } catch (e) { + // Ignore source lookup failures and continue resolving remaining sources. + continue; + } + } + } + + #queryTokenFromSystem(systemUrl) { + const raw = String(systemUrl || '').trim().replace(/\/+$/, ''); + if (!raw) { + return null; + } + const slash = raw.lastIndexOf('/'); + if (slash >= 0 && slash < raw.length - 1) { + return raw.substring(slash + 1); + } + return raw; + } + + #normalizeSourcePath(sourcePath) { + const path = String(sourcePath || '').trim(); + return path.endsWith('/') ? path : `${path}/`; + } + + #canonicalForSourceUrl(sourceUrl) { + return this._canonicalBySourceUrl.get(this.#norm(sourceUrl)) || null; + } + + #matchesTranslationRequest(cm, sourceSystem, sourceScope, targetScope, targetSystem, sourceCandidates, targetCandidates) { + if (cm.providesTranslation(sourceSystem, sourceScope, targetScope, targetSystem)) { + return true; + } + + const group = cm.jsonObj?.group?.[0] || {}; + const groupSource = this.#norm(group.source); + const groupTarget = this.#norm(group.target); + + const sourceOk = !sourceSystem || sourceCandidates.some(s => this.#norm(s) === groupSource); + const targetOk = !targetSystem || targetCandidates.some(s => this.#norm(s) === groupTarget); + return sourceOk && targetOk; + } + + #norm(url) { + return String(url || '').trim().replace(/\/+$/, '').toLowerCase(); + } + + #toIsoDate(value) { + if (!value) { + return null; + } + const date = new Date(value); + if (Number.isNaN(date.getTime())) { + return null; + } + return date.toISOString(); + } +} + +module.exports = { + OCLConceptMapProvider +}; \ No newline at end of file diff --git a/tx/ocl/cm-ocl.js b/tx/ocl/cm-ocl.js index ccf0f362..466c7f93 100644 --- a/tx/ocl/cm-ocl.js +++ b/tx/ocl/cm-ocl.js @@ -1,106 +1,2 @@ -/** - * Abstract base class for Concept Map providers - * Defines the interface that all Concept Map providers must implement - */ -// eslint-disable-next-line no-unused-vars -class OCLConceptMapProvider { - /** - * {int} Unique number assigned to this provider - */ - spaceId; +module.exports = require('./cm-ocl.cjs'); - /** - * ensure that the ids on the Concept Maps are unique, if they are - * in the global namespace - * - * @param {Set} ids - */ - // eslint-disable-next-line no-unused-vars - assignIds(ids) { - throw new Error('assignIds must be implemented by AbstractConceptMapProvider subclass'); - } - - /** - * Fetches a specific Concept Map by URL and version - * @param {string} url - The URL/identifier of the Concept Map - * @param {string} version - The version of the Concept Map - * @returns {Promise} The requested Concept Map - * @throws {Error} Must be implemented by subclasses - */ - // eslint-disable-next-line no-unused-vars - async fetchConceptMap(url, version) { - throw new Error('fetchConceptMap must be implemented by subclass'); - } - - /** - * Fetches a specific Concept Map by id. ConceptMap providers must enforce that Concept Map ids are unique - * either globally (as enforced by assignIds) or in their space - * - * @param {string} id - The id of the Concept Map - * @returns {Promise} The requested Concept Map - * @throws {Error} Must be implemented by subclasses - */ - // eslint-disable-next-line no-unused-vars - async fetchConceptMapById(id) { - throw new Error('fetchConceptMapById must be implemented by subclass'); - } - - /** - * Searches for Concept Maps based on provided criteria - * @param {Array<{name: string, value: string}>} searchParams - List of name/value pairs for search criteria - * @returns {Promise>} List of matching Concept Maps - * @throws {Error} Must be implemented by subclasses - */ - // eslint-disable-next-line no-unused-vars - async searchConceptMaps(searchParams, elements = null) { - throw new Error('searchConceptMaps must be implemented by subclass'); - } - - /** - * Validates search parameters - * @param {Array<{name: string, value: string}>} searchParams - Search parameters to validate - * @protected - */ - _validateSearchParams(searchParams) { - if (!Array.isArray(searchParams)) { - throw new Error('Search parameters must be an array'); - } - - for (const param of searchParams) { - if (!param || typeof param !== 'object') { - throw new Error('Each search parameter must be an object'); - } - if (typeof param.name !== 'string' || typeof param.value !== 'string') { - throw new Error('Search parameter must have string name and value properties'); - } - } - } - - /** - * Validates URL and version parameters - * @param {string} url - URL to validate - * @param {string} version - Version to validate - * @protected - */ - _validateFetchParams(url, version) { - if (typeof url !== 'string' || !url.trim()) { - throw new Error('URL must be a non-empty string'); - } - if (version != null && typeof version !== 'string') { - throw new Error('Version must be a string'); - } - } - - // eslint-disable-next-line no-unused-vars - async findConceptMapForTranslation(opContext, conceptMaps, sourceSystem, sourceScope, targetScope, targetSystem) { - // nothing - } - - cmCount() { - return 0; - } -} - -module.exports = { - AbstractConceptMapProvider -}; \ No newline at end of file diff --git a/tx/ocl/cs-ocl.cjs b/tx/ocl/cs-ocl.cjs new file mode 100644 index 00000000..75afab31 --- /dev/null +++ b/tx/ocl/cs-ocl.cjs @@ -0,0 +1,1774 @@ +const fs = require('fs/promises'); +const { AbstractCodeSystemProvider } = require('../cs/cs-provider-api'); +const { CodeSystemProvider, CodeSystemFactoryProvider, CodeSystemContentMode, FilterExecutionContext } = require('../cs/cs-api'); +const { CodeSystem } = require('../library/codesystem'); +const { SearchFilterText } = require('../library/designations'); +const { PAGE_SIZE, CONCEPT_PAGE_SIZE, COLD_CACHE_FRESHNESS_MS, OCL_CODESYSTEM_MARKER_EXTENSION } = require('./shared/constants'); +const { createOclHttpClient } = require('./http/client'); +const { fetchAllPages, extractItemsAndNext } = require('./http/pagination'); +const { CACHE_CS_DIR, CACHE_VS_DIR, getCacheFilePath } = require('./cache/cache-paths'); +const { ensureCacheDirectories, getColdCacheAgeMs, formatCacheAgeMinutes } = require('./cache/cache-utils'); +const { computeCodeSystemFingerprint } = require('./fingerprint/fingerprint'); +const { OCLBackgroundJobQueue } = require('./jobs/background-queue'); +const { OCLConceptFilterContext } = require('./model/concept-filter-context'); +const { toConceptContext } = require('./mappers/concept-mapper'); +const { patchSearchWorkerForOCLCodeFiltering } = require('./shared/patches'); + +patchSearchWorkerForOCLCodeFiltering(); + +function normalizeCanonicalSystem(system) { + if (typeof system !== 'string') { + return system; + } + + const trimmed = system.trim(); + if (!trimmed) { + return trimmed; + } + + // Treat canonical URLs with and without trailing slash as equivalent. + return trimmed.replace(/\/+$/, ''); +} + +class OCLCodeSystemProvider extends AbstractCodeSystemProvider { + constructor(config = {}) { + super(); + const options = typeof config === 'string' ? { baseUrl: config } : (config || {}); + + this.org = options.org || null; + const http = createOclHttpClient(options); + this.baseUrl = http.baseUrl; + this.httpClient = http.client; + + this._codeSystemsByCanonical = new Map(); + this._idToCodeSystem = new Map(); + this.sourceMetaByUrl = new Map(); + this._sourceStateByCanonical = new Map(); + this._usedIds = new Set(); + this._refreshPromise = null; + this._pendingChanges = null; + this._initialized = false; + this._initializePromise = null; + } + + async initialize() { + if (this._initialized) { + return; + } + + if (this._initializePromise) { + await this._initializePromise; + return; + } + + this._initializePromise = (async () => { + try { + const sources = await this.#fetchSourcesForDiscovery(); + console.log(`[OCL] Fetched ${sources.length} sources`); + + const snapshot = this.#buildSourceSnapshot(sources); + this.#applySnapshot(snapshot); + + console.log(`[OCL] Loaded ${this._codeSystemsByCanonical.size} code systems`); + this._initialized = true; + } catch (error) { + console.error(`[OCL] Initialization failed:`, error.message); + if (error.response) { + console.error(`[OCL] HTTP ${error.response.status}: ${error.response.statusText}`); + } + throw error; + } + })(); + + try { + await this._initializePromise; + } finally { + this._initializePromise = null; + } + } + + assignIds(ids) { + this._usedIds.clear(); + for (const cs of this._idToCodeSystem.values()) { + if (!cs.id || ids.has(`CodeSystem/${cs.id}`)) { + cs.id = String(ids.size); + cs.jsonObj.id = cs.id; + } + ids.add(`CodeSystem/${cs.id}`); + this._usedIds.add(cs.id); + this._idToCodeSystem.set(cs.id, cs); + } + } + + // eslint-disable-next-line no-unused-vars + async listCodeSystems(_fhirVersion, _context) { + await this.initialize(); + return Array.from(this._codeSystemsByCanonical.values()); + } + + // eslint-disable-next-line no-unused-vars + async loadCodeSystems(fhirVersion, context) { + return await this.listCodeSystems(fhirVersion, context); + } + + // Called once per minute by provider.updateCodeSystemList(). + // That caller is currently sync, so we stage async fetches and return the latest ready diff. + // eslint-disable-next-line no-unused-vars + getCodeSystemChanges(_fhirVersion, _context) { + if (!this._initialized) { + return this.#emptyChanges(); + } + + this.#scheduleRefresh(); + if (!this._pendingChanges) { + return this.#emptyChanges(); + } + + const out = this._pendingChanges; + this._pendingChanges = null; + return out; + } + + async close() { + } + + getSourceMetas() { + return Array.from(this.sourceMetaByUrl.values()); + } + + #scheduleRefresh() { + if (this._refreshPromise) { + return; + } + + this._refreshPromise = (async () => { + try { + const sources = await this.#fetchSourcesForDiscovery(); + const nextSnapshot = this.#buildSourceSnapshot(sources); + const changes = this.#diffSnapshots(this._sourceStateByCanonical, nextSnapshot); + this.#applySnapshot(nextSnapshot); + this._pendingChanges = changes; + } catch (error) { + console.error('[OCL] Incremental source refresh failed:', error.message); + this._pendingChanges = this.#emptyChanges(); + } finally { + this._refreshPromise = null; + } + })(); + } + + #emptyChanges() { + return { added: [], changed: [], deleted: [] }; + } + + #buildSourceSnapshot(sources) { + const snapshot = new Map(); + for (const source of sources || []) { + const cs = this.#toCodeSystem(source); + if (!cs) { + continue; + } + + const canonicalUrl = cs.url; + const meta = this.#buildSourceMeta(source, cs); + const checksum = this.#sourceChecksum(source); + snapshot.set(canonicalUrl, { cs, meta, checksum }); + } + return snapshot; + } + + async #fetchSourcesForDiscovery() { + const organizations = await this.#fetchOrganizationIds(); + if (organizations.length === 0) { + // Fallback for OCL instances that expose global listing but not org listing. + return await this.#fetchAllPages('/sources/'); + } + + const allSources = []; + const seen = new Set(); + + for (const orgId of organizations) { + const endpoint = `/orgs/${encodeURIComponent(orgId)}/sources/`; + const sources = await this.#fetchAllPages(endpoint); + for (const source of sources) { + if (!source || typeof source !== 'object') { + continue; + } + const key = this.#sourceIdentity(source); + if (seen.has(key)) { + continue; + } + seen.add(key); + allSources.push(source); + } + } + + return allSources; + } + + async #fetchOrganizationIds() { + const endpoint = '/orgs/'; + const orgs = await this.#fetchAllPages(endpoint); + + const ids = []; + const seen = new Set(); + for (const org of orgs || []) { + if (!org || typeof org !== 'object') { + continue; + } + + const id = org.id || org.mnemonic || org.short_code || org.shortCode || org.name || null; + if (!id) { + continue; + } + + const normalized = String(id).trim(); + if (!normalized || seen.has(normalized)) { + continue; + } + + seen.add(normalized); + ids.push(normalized); + } + + if (ids.length === 0 && this.org) { + ids.push(this.org); + } + + return ids; + } + + #sourceIdentity(source) { + if (!source || typeof source !== 'object') { + return '__invalid__'; + } + + const owner = source.owner || ''; + const canonical = normalizeCanonicalSystem(source.canonical_url || source.canonicalUrl || ''); + const shortCode = source.short_code || source.shortCode || source.id || source.mnemonic || source.name || ''; + return `${owner}|${canonical}|${shortCode}`; + } + + #applySnapshot(snapshot) { + const previousSnapshot = this._sourceStateByCanonical; + this._codeSystemsByCanonical.clear(); + this._idToCodeSystem.clear(); + this.sourceMetaByUrl.clear(); + this._usedIds.clear(); + + for (const [canonicalUrl, entry] of snapshot.entries()) { + const cs = entry.cs; + const meta = entry.meta; + const previousEntry = previousSnapshot.get(canonicalUrl); + + // Preserve complete-content marker if checksum did not change. + if (previousEntry && previousEntry.checksum === entry.checksum && previousEntry.cs?.jsonObj?.content === CodeSystemContentMode.Complete) { + cs.jsonObj.content = CodeSystemContentMode.Complete; + + // Preserve materialized concepts across metadata refreshes. + if (Array.isArray(previousEntry.cs?.jsonObj?.concept)) { + cs.jsonObj.concept = previousEntry.cs.jsonObj.concept.map(concept => ({ ...concept })); + } + } + + // If a factory already has warm/cold concepts for this system, project them to the new snapshot resource. + OCLSourceCodeSystemFactory.syncCodeSystemResource(canonicalUrl, cs.version || null, cs); + + this.#trackCodeSystemId(cs); + this._codeSystemsByCanonical.set(canonicalUrl, cs); + this._idToCodeSystem.set(cs.id, cs); + this.sourceMetaByUrl.set(canonicalUrl, meta); + } + + this._sourceStateByCanonical = snapshot; + } + + #diffSnapshots(previousSnapshot, nextSnapshot) { + const added = []; + const changed = []; + const deleted = []; + + for (const [canonicalUrl, nextEntry] of nextSnapshot.entries()) { + const previousEntry = previousSnapshot.get(canonicalUrl); + if (!previousEntry) { + added.push(nextEntry.cs); + continue; + } + + // Keep stable ids across revisions so clients don't observe resource id churn. + nextEntry.cs.id = previousEntry.cs.id; + nextEntry.cs.jsonObj.id = previousEntry.cs.id; + + const previousChecksum = previousEntry.checksum || null; + const nextChecksum = nextEntry.checksum || null; + const checksumChanged = previousChecksum !== nextChecksum; + const versionChanged = (previousEntry.cs.version || null) !== (nextEntry.cs.version || null); + if (checksumChanged || versionChanged) { + if (checksumChanged) { + console.log(`[OCL] CodeSystem checksum changed: ${canonicalUrl} (${previousChecksum} -> ${nextChecksum})`); + } + changed.push(nextEntry.cs); + } + } + + for (const [canonicalUrl, previousEntry] of previousSnapshot.entries()) { + if (!nextSnapshot.has(canonicalUrl)) { + deleted.push(previousEntry.cs); + } + } + + return { added, changed, deleted }; + } + + #trackCodeSystemId(cs) { + if (!cs) { + return; + } + + if (!cs.id || this._usedIds.has(cs.id)) { + const raw = cs.id || cs.name || cs.url || 'ocl-cs'; + const base = this.spaceId ? `${this.spaceId}-${raw}` : String(raw); + let candidate = base; + let index = 1; + while (this._usedIds.has(candidate)) { + candidate = `${base}-${index}`; + index += 1; + } + cs.id = candidate; + cs.jsonObj.id = candidate; + } + + this._usedIds.add(cs.id); + } + + #toCodeSystem(source) { + if (!source || typeof source !== 'object') { + return null; + } + + const canonicalUrl = normalizeCanonicalSystem(source.canonical_url || source.canonicalUrl || source.url); + if (!canonicalUrl) { + return null; + } + + const id = source.id || source.mnemonic; + if (!id) { + return null; + } + + const lastUpdated = this.#toIsoDate(source.updated_at || source.updatedAt || source.updated_on || source.updatedOn); + + const json = { + resourceType: 'CodeSystem', + id, + url: canonicalUrl, + version: source.version || null, + name: source.name || source.mnemonic || id, + title: source.full_name || source.fullName || source.name || source.mnemonic || id, + status: 'active', + experimental: source.experimental === true, + description: source.description || null, + publisher: source.owner || null, + caseSensitive: source.case_sensitive != null ? source.case_sensitive : (source.caseSensitive != null ? source.caseSensitive : true), + language: source.default_locale || source.defaultLocale || null, + filter: [ + { + code: 'code', + description: 'Match concept code', + operator: ['=', 'in', 'regex'], + value: 'code' + }, + { + code: 'display', + description: 'Match concept display text', + operator: ['=', 'in', 'regex'], + value: 'string' + }, + { + code: 'definition', + description: 'Match concept definition text', + operator: ['=', 'in', 'regex'], + value: 'string' + }, + { + code: 'inactive', + description: 'Match inactive (retired) status', + operator: ['=', 'in'], + value: 'boolean' + } + ], + property: [ + { + code: 'code', + uri: 'http://hl7.org/fhir/concept-properties#code', + description: 'Concept code', + type: 'code' + }, + { + code: 'display', + description: 'Concept display text', + type: 'string' + }, + { + code: 'definition', + description: 'Concept definition text', + type: 'string' + }, + { + code: 'inactive', + uri: 'http://hl7.org/fhir/concept-properties#status', + description: 'Whether concept is inactive (retired)', + type: 'boolean' + } + ], + extension: [ + { + url: OCL_CODESYSTEM_MARKER_EXTENSION, + valueBoolean: true + } + ], + content: 'not-present' + }; + + if (lastUpdated) { + json.meta = { lastUpdated }; + } + + return new CodeSystem(json, 'R5', true); + } + + #buildSourceMeta(source, cs) { + if (!source || !cs) { + return null; + } + + const owner = source.owner || null; + const shortCode = source.short_code || source.shortCode || source.mnemonic || source.id || null; + const canonicalUrl = cs.url; + if (!canonicalUrl) { + return; + } + + const conceptsUrl = this.#normalizePath(source.concepts_url || source.conceptsUrl || this.#buildConceptsPath(source)); + const meta = { + id: source.id || shortCode, + shortCode, + owner, + name: source.name || shortCode || cs.id, + description: source.description || null, + canonicalUrl, + version: source.version || null, + conceptsUrl, + checksum: this.#sourceChecksum(source), + codeSystem: cs + }; + + return meta; + } + + #sourceChecksum(source) { + // NOTE: OCL checksums are NOT reliable for cache invalidation decisions. + // They do not update when concepts are added or modified. + // This checksum is logged for debugging purposes only. + // Cache decisions are based on custom fingerprints computed from concept content. + + if (!source || typeof source !== 'object') { + return null; + } + + const checksums = source.checksums || {}; + const standard = checksums.standard || null; + const smart = checksums.smart || null; + if (standard) { + return String(standard); + } + if (smart) { + return String(smart); + } + + if (source.checksum) { + return String(source.checksum); + } + + const updated = source.updated_at || source.updatedAt || source.updated_on || source.updatedOn || null; + const version = source.version || null; + if (updated || version) { + return `${updated || ''}|${version || ''}`; + } + + return null; + } + + #buildConceptsPath(source) { + if (!source || typeof source !== 'object') { + return null; + } + const owner = source.owner || null; + const sourceId = source.short_code || source.shortCode || source.id || source.mnemonic || null; + if (!owner || !sourceId) { + const sourceUrl = source.url; + if (!sourceUrl || typeof sourceUrl !== 'string') { + return null; + } + const trimmed = sourceUrl.endsWith('/') ? sourceUrl : `${sourceUrl}/`; + return `${trimmed}concepts/`; + } + return `/orgs/${encodeURIComponent(owner)}/sources/${encodeURIComponent(sourceId)}/concepts/`; + } + + #normalizePath(pathValue) { + if (!pathValue) { + return null; + } + if (typeof pathValue !== 'string') { + return null; + } + if (pathValue.startsWith('http://') || pathValue.startsWith('https://')) { + return pathValue; + } + return `${this.baseUrl}${pathValue.startsWith('/') ? '' : '/'}${pathValue}`; + } + + async #fetchAllPages(path) { + try { + return await fetchAllPages(this.httpClient, path, { + pageSize: PAGE_SIZE, + baseUrl: this.baseUrl, + logger: console, + loggerPrefix: '[OCL]' + }); + } catch (error) { + if (error.response) { + console.error(`[OCL] HTTP ${error.response.status}: ${error.response.statusText}`); + console.error('[OCL] Response:', error.response.data); + } + throw error; + } + } + + #extractItemsAndNext(payload) { + return extractItemsAndNext(payload, this.baseUrl); + } + + #toIsoDate(value) { + if (!value) { + return null; + } + const date = new Date(value); + if (Number.isNaN(date.getTime())) { + return null; + } + return date.toISOString(); + } +} + +class OCLSourceCodeSystemProvider extends CodeSystemProvider { + constructor(opContext, supplements, client, meta, sharedCaches = null) { + super(opContext, supplements); + this.httpClient = client; + this.meta = meta; + this.conceptCache = sharedCaches?.conceptCache || new Map(); + this.pageCache = sharedCaches?.pageCache || new Map(); + this.pendingConceptRequests = sharedCaches?.pendingConceptRequests || new Map(); + this.pendingPageRequests = sharedCaches?.pendingPageRequests || new Map(); + this.scheduleBackgroundLoad = typeof sharedCaches?.scheduleBackgroundLoad === 'function' + ? sharedCaches.scheduleBackgroundLoad + : null; + this.isSystemComplete = typeof sharedCaches?.isSystemComplete === 'function' + ? sharedCaches.isSystemComplete + : (() => false); + this.getTotalConceptCount = typeof sharedCaches?.getTotalConceptCount === 'function' + ? sharedCaches.getTotalConceptCount + : (() => -1); + } + + system() { + return this.meta.canonicalUrl; + } + + version() { + return this.meta.version || null; + } + + description() { + return this.meta.description || null; + } + + name() { + return this.meta.name || this.meta.shortCode || this.meta.id || this.system(); + } + + contentMode() { + if (this.isSystemComplete()) { + return CodeSystemContentMode.Complete; + } + + // OCL CodeSystems are lazily materialized. Even when metadata is still + // warming up, concepts remain fetchable through the source concepts URL. + // Report at least fragment support so $expand does not fail early with + // "has no content" before lazy retrieval/hydration can run. + const hasRealCodeSystemResource = this.meta?.codeSystem instanceof CodeSystem; + if (hasRealCodeSystemResource && (this.meta?.conceptsUrl || this.conceptCache.size > 0 || this.pageCache.size > 0)) { + return CodeSystemContentMode.Fragment; + } + + return CodeSystemContentMode.NotPresent; + } + + totalCount() { + return this.getTotalConceptCount(); + } + + propertyDefinitions() { + return this.meta?.codeSystem?.jsonObj?.property || null; + } + + async code(code) { + const ctxt = await this.#ensureContext(code); + return ctxt ? ctxt.code : null; + } + + async display(code) { + const ctxt = await this.#ensureContext(code); + if (!ctxt) { + return null; + } + if (ctxt.display && this.opContext.langs.isEnglishOrNothing()) { + return ctxt.display; + } + const supp = this._displayFromSupplements(ctxt.code); + return supp || ctxt.display || null; + } + + async definition(code) { + const ctxt = await this.#ensureContext(code); + return ctxt ? ctxt.definition : null; + } + + async isAbstract(code) { + await this.#ensureContext(code); + return false; + } + + async isInactive(code) { + const ctxt = await this.#ensureContext(code); + return ctxt ? ctxt.retired === true : false; + } + + async isDeprecated(code) { + await this.#ensureContext(code); + return false; + } + + async getStatus(code) { + const ctxt = await this.#ensureContext(code); + if (!ctxt) { + return null; + } + return ctxt.retired === true ? 'inactive' : 'active'; + } + + async designations(code, displays) { + const ctxt = await this.#ensureContext(code); + if (ctxt && ctxt.display) { + const hasConceptDesignations = Array.isArray(ctxt.designations) && ctxt.designations.length > 0; + if (hasConceptDesignations) { + for (const d of ctxt.designations) { + if (!d || !d.value) { + continue; + } + displays.addDesignation(true, 'active', d.language || '', CodeSystem.makeUseForDisplay(), d.value); + } + } else { + displays.addDesignation(true, 'active', 'en', CodeSystem.makeUseForDisplay(), ctxt.display); + } + this._listSupplementDesignations(ctxt.code, displays); + } + } + + async locate(code) { + if (!code || typeof code !== 'string') { + return { context: null, message: 'Empty code' }; + } + + if (this.conceptCache.has(code)) { + return { context: this.conceptCache.get(code), message: null }; + } + + if (this.scheduleBackgroundLoad) { + this.scheduleBackgroundLoad('lookup-miss'); + } + + const concept = await this.#fetchConcept(code); + if (!concept) { + return { context: null, message: undefined }; + } + + this.conceptCache.set(code, concept); + return { context: concept, message: null }; + } + + async iterator(code) { + await this.#ensureContext(code); + if (code) { + return null; + } + return { + page: 1, + index: 0, + items: [], + total: -1, + done: false + }; + } + + async iteratorAll() { + return this.iterator(null); + } + + async getPrepContext(iterate) { + return new FilterExecutionContext(iterate); + } + + async doesFilter(prop, op, value) { + if (!prop || !op || value == null) { + return false; + } + + const normalizedProp = String(prop).trim().toLowerCase(); + const normalizedOp = String(op).trim().toLowerCase(); + const supportedOps = ['=', 'in', 'regex']; + if (!supportedOps.includes(normalizedOp)) { + return false; + } + + if (['concept', 'code', 'display', 'definition', 'inactive'].includes(normalizedProp)) { + return true; + } + + const defs = this.propertyDefinitions() || []; + return defs.some(def => def && def.code === normalizedProp); + } + + async searchFilter(filterContext, filter, sort) { + const matcher = this.#toSearchFilterText(filter); + const results = new OCLConceptFilterContext(); + const concepts = await this.#allConceptContexts(); + + for (const concept of concepts) { + const text = this.#conceptSearchText(concept); + const match = matcher.passes(text, true); + if (!match || match.passes !== true) { + continue; + } + + results.add(concept, this.#searchRating(concept, matcher, match.rating)); + } + + if (sort === true) { + results.sort(); + } + + if (!Array.isArray(filterContext.filters)) { + filterContext.filters = []; + } + filterContext.filters.push(results); + return filterContext; + } + + async filter(filterContext, prop, op, value) { + const normalizedProp = String(prop || '').trim().toLowerCase(); + const normalizedOp = String(op || '').trim().toLowerCase(); + + if (!await this.doesFilter(normalizedProp, normalizedOp, value)) { + throw new Error(`Filter ${prop} ${op} is not supported by OCL provider`); + } + + const set = new OCLConceptFilterContext(); + const concepts = await this.#allConceptContexts(); + const matcher = this.#buildPropertyMatcher(normalizedProp, normalizedOp, value); + + for (const concept of concepts) { + if (matcher(concept)) { + set.add(concept, 0); + } + } + + if (!Array.isArray(filterContext.filters)) { + filterContext.filters = []; + } + filterContext.filters.push(set); + return set; + } + + async executeFilters(filterContext) { + return Array.isArray(filterContext?.filters) ? filterContext.filters : []; + } + + // eslint-disable-next-line no-unused-vars + async filterSize(filterContext, set) { + return set ? set.size() : 0; + } + + // eslint-disable-next-line no-unused-vars + async filterMore(filterContext, set) { + return !!set && set.hasMore(); + } + + // eslint-disable-next-line no-unused-vars + async filterConcept(filterContext, set) { + if (!set) { + return null; + } + return set.next(); + } + + // eslint-disable-next-line no-unused-vars + async filterLocate(filterContext, set, code) { + if (!set) { + return `Code '${code}' not found: no filter results`; + } + const concept = set.findConceptByCode(code); + if (concept) { + return concept; + } + return null; + } + + // eslint-disable-next-line no-unused-vars + async filterCheck(filterContext, set, concept) { + if (!set || !concept) { + return false; + } + return set.containsConcept(concept); + } + + async filterFinish(filterContext) { + if (!Array.isArray(filterContext?.filters)) { + return; + } + for (const set of filterContext.filters) { + if (set && typeof set.reset === 'function') { + set.reset(); + } + } + filterContext.filters.length = 0; + } + + async nextContext(iteratorContext) { + if (!iteratorContext || iteratorContext.done) { + return null; + } + + if (iteratorContext.index >= iteratorContext.items.length) { + const pageItems = await this.#fetchConceptPage(iteratorContext.page); + iteratorContext.page += 1; + iteratorContext.index = 0; + iteratorContext.items = pageItems; + + if (!pageItems || pageItems.length === 0) { + iteratorContext.done = true; + return null; + } + } + + const concept = iteratorContext.items[iteratorContext.index]; + iteratorContext.index += 1; + return concept; + } + + async #ensureContext(code) { + if (!code) { + return null; + } + + // Some call paths pass a pending locate() Promise (or its wrapper result) + // instead of a raw code/context; normalize both shapes here. + if (code && typeof code === 'object' && typeof code.then === 'function') { + code = await code; + } + + if (code && typeof code === 'object' && Object.prototype.hasOwnProperty.call(code, 'context')) { + if (!code.context) { + throw new Error(code.message || 'Unknown code'); + } + code = code.context; + } + + if (typeof code === 'string') { + const result = await this.locate(code); + if (!result.context) { + throw new Error(result.message || `Unknown code ${code}`); + } + return result.context; + } + if (code && typeof code === 'object' && code.code) { + return code; + } + throw new Error(`Unknown Type at #ensureContext: ${typeof code}`); + } + + async #fetchConceptPage(page) { + if (!this.meta.conceptsUrl) { + return []; + } + const cacheKey = `${this.meta.conceptsUrl}|p=${page}|l=${CONCEPT_PAGE_SIZE}`; + if (this.pageCache.has(cacheKey)) { + const cached = this.pageCache.get(cacheKey); + return Array.isArray(cached) + ? cached + : Array.isArray(cached?.concepts) + ? cached.concepts + : []; + } + if (this.pendingPageRequests.has(cacheKey)) { + const pendingResult = await this.pendingPageRequests.get(cacheKey); + return Array.isArray(pendingResult) + ? pendingResult + : Array.isArray(pendingResult?.concepts) + ? pendingResult.concepts + : []; + } + + if (this.scheduleBackgroundLoad) { + this.scheduleBackgroundLoad('page-miss'); + } + + const pending = (async () => { + let response; + try { + response = await this.httpClient.get(this.meta.conceptsUrl, { params: { page, limit: CONCEPT_PAGE_SIZE } }); + } catch (error) { + // Some OCL instances return 404 for sources without concept listing endpoints. + // Treat this as an empty page so terminology operations degrade gracefully. + if (error && error.response && error.response.status === 404) { + this.pageCache.set(cacheKey, []); + return []; + } + throw error; + } + const payload = response.data; + const items = Array.isArray(payload) + ? payload + : Array.isArray(payload?.results) + ? payload.results + : Array.isArray(payload?.items) + ? payload.items + : Array.isArray(payload?.data) + ? payload.data + : []; + + const mapped = items.map(item => this.#toConceptContext(item)).filter(Boolean); + this.pageCache.set(cacheKey, mapped); + for (const concept of mapped) { + if (concept && concept.code) { + this.conceptCache.set(concept.code, concept); + } + } + return mapped; + })(); + + this.pendingPageRequests.set(cacheKey, pending); + try { + return await pending; + } finally { + this.pendingPageRequests.delete(cacheKey); + } + } + + async #fetchConcept(code) { + if (!this.meta.conceptsUrl) { + return null; + } + if (this.conceptCache.has(code)) { + return this.conceptCache.get(code); + } + const pendingKey = `${this.meta.conceptsUrl}|c=${code}`; + if (this.pendingConceptRequests.has(pendingKey)) { + return this.pendingConceptRequests.get(pendingKey); + } + + if (this.scheduleBackgroundLoad) { + this.scheduleBackgroundLoad('concept-miss'); + } + + const url = this.#buildConceptUrl(code); + const pending = (async () => { + let response; + try { + response = await this.httpClient.get(url); + } catch (error) { + // Missing concept should be treated as not-found, not as an internal server failure. + if (error && error.response && error.response.status === 404) { + return null; + } + throw error; + } + const concept = this.#toConceptContext(response.data); + if (concept && concept.code) { + this.conceptCache.set(concept.code, concept); + } + return concept; + })(); + + this.pendingConceptRequests.set(pendingKey, pending); + try { + return await pending; + } finally { + this.pendingConceptRequests.delete(pendingKey); + } + } + + async #allConceptContexts() { + const concepts = new Map(); + + for (const concept of this.conceptCache.values()) { + if (concept && concept.code) { + concepts.set(concept.code, concept); + } + } + + // Ensure search can operate even when content is not fully warm-loaded. + const iter = await this.iterator(null); + let concept = await this.nextContext(iter); + while (concept) { + if (concept.code && !concepts.has(concept.code)) { + concepts.set(concept.code, concept); + } + concept = await this.nextContext(iter); + } + + return Array.from(concepts.values()); + } + + #toSearchFilterText(filter) { + if (filter instanceof SearchFilterText) { + return filter; + } + if (typeof filter === 'string') { + return new SearchFilterText(filter); + } + if (filter && typeof filter.filter === 'string') { + return new SearchFilterText(filter.filter); + } + return new SearchFilterText(''); + } + + #conceptSearchText(concept) { + if (!concept || typeof concept !== 'object') { + return ''; + } + + const values = [concept.code, concept.display, concept.definition]; + if (Array.isArray(concept.designations)) { + for (const designation of concept.designations) { + if (designation && designation.value) { + values.push(designation.value); + } + } + } + + return values.filter(Boolean).join(' '); + } + + #searchRating(concept, matcher, baseRating) { + let rating = Number.isFinite(baseRating) ? baseRating : 0; + const term = matcher?.filter || ''; + if (!term) { + return rating; + } + + const code = String(concept?.code || '').toLowerCase(); + const display = String(concept?.display || '').toLowerCase(); + const definition = String(concept?.definition || '').toLowerCase(); + + if (code === term || display === term) { + rating += 100; + } else if (code.startsWith(term) || display.startsWith(term)) { + rating += 50; + } else if (definition.includes(term)) { + rating += 10; + } + + return rating; + } + + #buildPropertyMatcher(prop, op, value) { + if (op === 'regex') { + const regex = new RegExp(String(value), 'i'); + return concept => { + const candidate = this.#valueForFilter(concept, prop); + if (candidate == null) { + return false; + } + return regex.test(String(candidate)); + }; + } + + if (op === 'in') { + const tokens = String(value) + .split(',') + .map(token => token.trim().toLowerCase()) + .filter(Boolean); + return concept => { + const candidate = this.#valueForFilter(concept, prop); + if (candidate == null) { + return false; + } + return tokens.includes(String(candidate).toLowerCase()); + }; + } + + if (prop === 'inactive') { + const expected = this.#toBoolean(value); + return concept => { + const candidate = this.#toBoolean(this.#valueForFilter(concept, prop)); + return candidate === expected; + }; + } + + const expected = String(value).toLowerCase(); + return concept => { + const candidate = this.#valueForFilter(concept, prop); + if (candidate == null) { + return false; + } + return String(candidate).toLowerCase() === expected; + }; + } + + #valueForFilter(concept, prop) { + if (!concept || typeof concept !== 'object') { + return null; + } + + switch (prop) { + case 'concept': + case 'code': + return concept.code || null; + case 'display': + return concept.display || null; + case 'definition': + return concept.definition || null; + case 'inactive': + return concept.retired === true; + default: + return concept[prop] ?? null; + } + } + + #toBoolean(value) { + if (typeof value === 'boolean') { + return value; + } + + const text = String(value || '').trim().toLowerCase(); + return text === 'true' || text === '1' || text === 'yes'; + } + + #buildConceptUrl(code) { + const base = this.meta.conceptsUrl.endsWith('/') ? this.meta.conceptsUrl : `${this.meta.conceptsUrl}/`; + return `${base}${encodeURIComponent(code)}/`; + } + + #toConceptContext(concept) { + return toConceptContext(concept); + } +} + +class OCLSourceCodeSystemFactory extends CodeSystemFactoryProvider { + static factoriesByKey = new Map(); + + static #normalizeSystem(system) { + return normalizeCanonicalSystem(system); + } + + static hasFactory(system, version = null) { + return !!OCLSourceCodeSystemFactory.#findFactory(system, version); + } + + static hasExactFactory(system, version = null) { + const normalizedSystem = OCLSourceCodeSystemFactory.#normalizeSystem(system); + if (!normalizedSystem) { + return false; + } + + const exactKey = `${normalizedSystem}|${version || ''}`; + return OCLSourceCodeSystemFactory.factoriesByKey.has(exactKey); + } + + static #findFactory(system, version = null) { + const normalizedSystem = OCLSourceCodeSystemFactory.#normalizeSystem(system); + if (!normalizedSystem) { + return null; + } + + const exactKey = `${normalizedSystem}|${version || ''}`; + const exact = OCLSourceCodeSystemFactory.factoriesByKey.get(exactKey); + if (exact) { + return exact; + } + + // When caller version does not match the registered one (or is absent), + // still reuse the factory for the same canonical system. + for (const [key, factory] of OCLSourceCodeSystemFactory.factoriesByKey.entries()) { + if (!factory) { + continue; + } + + const separatorIndex = key.lastIndexOf('|'); + if (separatorIndex < 0) { + continue; + } + + const keySystem = OCLSourceCodeSystemFactory.#normalizeSystem(key.substring(0, separatorIndex)); + if (keySystem === normalizedSystem) { + return factory; + } + } + + return null; + } + + static syncCodeSystemResource(system, version = null, codeSystem = null) { + const normalizedSystem = OCLSourceCodeSystemFactory.#normalizeSystem(system); + if (!normalizedSystem) { + return; + } + + const factory = OCLSourceCodeSystemFactory.#findFactory(normalizedSystem, version); + if (!factory) { + return; + } + + factory.#applyConceptsToCodeSystemResource(codeSystem || factory.meta?.codeSystem || null); + } + + constructor(i18n, client, meta) { + super(i18n); + this.httpClient = client; + this.meta = meta; + this.sharedConceptCache = new Map(); + this.sharedPageCache = new Map(); + this.sharedPendingConceptRequests = new Map(); + this.sharedPendingPageRequests = new Map(); + this.isComplete = meta?.codeSystem?.jsonObj?.content === CodeSystemContentMode.Complete; + this.loadedConceptCount = -1; + this.loadedChecksum = meta?.checksum || null; + this.customFingerprint = null; + this.backgroundLoadProgress = { processed: 0, total: null }; + this.materializedConceptList = null; + this.materializedConceptCount = -1; + OCLSourceCodeSystemFactory.factoriesByKey.set(this.#resourceKey(), this); + + const unversionedKey = `${this.system()}|`; + if (!OCLSourceCodeSystemFactory.factoriesByKey.has(unversionedKey)) { + OCLSourceCodeSystemFactory.factoriesByKey.set(unversionedKey, this); + } + + // Load cold cache at construction + this.#loadColdCache(); + } + + async #loadColdCache() { + const canonicalUrl = this.system(); + const version = this.version(); + const cacheFilePath = getCacheFilePath(CACHE_CS_DIR, canonicalUrl, version); + + try { + const data = await fs.readFile(cacheFilePath, 'utf-8'); + const cached = JSON.parse(data); + + if (!cached || !cached.concepts || !Array.isArray(cached.concepts)) { + return; + } + + // Restore concepts to cache + for (const concept of cached.concepts) { + if (concept && concept.code) { + this.sharedConceptCache.set(concept.code, concept); + } + } + + this.loadedConceptCount = cached.concepts.length; + this.customFingerprint = cached.fingerprint || null; + this.isComplete = true; + + if (this.meta?.codeSystem?.jsonObj) { + this.meta.codeSystem.jsonObj.content = CodeSystemContentMode.Complete; + } + + this.#applyConceptsToCodeSystemResource(this.meta?.codeSystem || null); + + console.log(`[OCL] Loaded CodeSystem from cold cache: ${canonicalUrl} (${cached.concepts.length} concepts, fingerprint=${this.customFingerprint?.substring(0, 8)})`); + } catch (error) { + if (error.code !== 'ENOENT') { + console.error(`[OCL] Failed to load cold cache for CodeSystem ${canonicalUrl}:`, error.message); + } + } + } + + async #saveColdCache(concepts) { + const canonicalUrl = this.system(); + const version = this.version(); + const cacheFilePath = getCacheFilePath(CACHE_CS_DIR, canonicalUrl, version); + + try { + await ensureCacheDirectories(CACHE_CS_DIR, CACHE_VS_DIR); + + const fingerprint = computeCodeSystemFingerprint(concepts); + const cacheData = { + canonicalUrl, + version, + fingerprint, + timestamp: new Date().toISOString(), + conceptCount: concepts.length, + concepts + }; + + await fs.writeFile(cacheFilePath, JSON.stringify(cacheData, null, 2), 'utf-8'); + console.log(`[OCL] Saved CodeSystem to cold cache: ${canonicalUrl} (${concepts.length} concepts, fingerprint=${fingerprint?.substring(0, 8)})`); + + return fingerprint; + } catch (error) { + console.error(`[OCL] Failed to save cold cache for CodeSystem ${canonicalUrl}:`, error.message); + return null; + } + } + + static scheduleBackgroundLoadByKey(system, version = null, reason = 'valueset-expansion') { + const normalizedSystem = OCLSourceCodeSystemFactory.#normalizeSystem(system); + const key = `${normalizedSystem}|${version || ''}`; + const factory = OCLSourceCodeSystemFactory.#findFactory(normalizedSystem, version); + if (!factory) { + console.log(`[OCL] CodeSystem load not scheduled (factory unavailable): ${key}`); + return false; + } + factory.scheduleBackgroundLoad(reason); + return true; + } + + static checksumForResource(system, version = null) { + const normalizedSystem = OCLSourceCodeSystemFactory.#normalizeSystem(system); + const factory = OCLSourceCodeSystemFactory.#findFactory(normalizedSystem, version); + if (!factory) { + return null; + } + return factory.currentChecksum(); + } + + static loadProgress() { + let total = 0; + let loaded = 0; + + for (const factory of OCLSourceCodeSystemFactory.factoriesByKey.values()) { + total += 1; + if (factory && factory.isCompleteNow()) { + loaded += 1; + } + } + + const percentage = total > 0 ? (loaded / total) * 100 : 0; + return { loaded, total, percentage }; + } + + defaultVersion() { + return this.meta.version || null; + } + + build(opContext, supplements) { + this.#syncWarmStateWithChecksum(); + this.#applyConceptsToCodeSystemResource(this.meta?.codeSystem || null); + this.recordUse(); + return new OCLSourceCodeSystemProvider(opContext, supplements, this.httpClient, this.meta, { + conceptCache: this.sharedConceptCache, + pageCache: this.sharedPageCache, + pendingConceptRequests: this.sharedPendingConceptRequests, + pendingPageRequests: this.sharedPendingPageRequests, + scheduleBackgroundLoad: reason => this.scheduleBackgroundLoad(reason), + isSystemComplete: () => this.isComplete, + getTotalConceptCount: () => this.loadedConceptCount + }); + } + + scheduleBackgroundLoad(reason = 'request') { + this.#syncWarmStateWithChecksum(); + const cacheFilePath = getCacheFilePath(CACHE_CS_DIR, this.system(), this.version()); + const cacheAgeMs = getColdCacheAgeMs(cacheFilePath); + + // If warm state is complete but cold cache is stale, force a refresh run. + // This keeps warm data available while ensuring stale cold-cache files are replaced. + if (this.isComplete) { + if (cacheAgeMs == null || cacheAgeMs < COLD_CACHE_FRESHNESS_MS) { + return; + } + + this.isComplete = false; + if (this.meta?.codeSystem?.jsonObj?.content === CodeSystemContentMode.Complete) { + this.meta.codeSystem.jsonObj.content = CodeSystemContentMode.Fragment; + } + } + + if (cacheAgeMs != null && cacheAgeMs < COLD_CACHE_FRESHNESS_MS) { + console.log(`[OCL] Skipping warm-up for CodeSystem ${this.system()} (cold cache age: ${formatCacheAgeMinutes(cacheAgeMs)})`); + return; + } + + const key = this.#resourceKey(); + const jobKey = `cs:${key}`; + + if (OCLBackgroundJobQueue.isQueuedOrRunning(jobKey)) { + console.log(`[OCL] CodeSystem load already queued or running: ${key}`); + return; + } + + let queuedJobSize = null; + console.log(`[OCL] CodeSystem load enqueued: ${key} (${reason})`); + OCLBackgroundJobQueue.enqueue( + jobKey, + 'CodeSystem load', + async () => { + await this.#runBackgroundLoad(key, queuedJobSize); + }, + { + jobId: this.system(), + getProgress: () => this.#backgroundLoadProgressSnapshot(), + resolveJobSize: async () => { + queuedJobSize = await this.#fetchConceptCountFromHeaders(); + return queuedJobSize; + } + } + ); + } + + async #runBackgroundLoad(key, knownConceptCount = null) { + console.log(`[OCL] CodeSystem load started: ${key}`); + try { + this.backgroundLoadProgress = { processed: 0, total: null }; + const resolvedTotal = Number.isFinite(knownConceptCount) && knownConceptCount >= 0 + ? knownConceptCount + : await this.#fetchConceptCountFromHeaders(); + this.backgroundLoadProgress.total = resolvedTotal; + const count = await this.#loadAllConceptPages(); + this.loadedConceptCount = count; + this.isComplete = true; + this.loadedChecksum = this.meta?.checksum || null; + this.backgroundLoadProgress = { + processed: count, + total: count > 0 ? count : this.backgroundLoadProgress.total + }; + + if (this.meta?.codeSystem?.jsonObj) { + this.meta.codeSystem.jsonObj.content = CodeSystemContentMode.Complete; + } + + this.#applyConceptsToCodeSystemResource(this.meta?.codeSystem || null); + + // Compute custom fingerprint and compare with cold cache + const allConcepts = Array.from(this.sharedConceptCache.values()); + const newFingerprint = computeCodeSystemFingerprint(allConcepts); + + if (this.customFingerprint && newFingerprint === this.customFingerprint) { + console.log(`[OCL] CodeSystem fingerprint unchanged: ${key} (fingerprint=${newFingerprint?.substring(0, 8)})`); + } else { + if (this.customFingerprint) { + console.log(`[OCL] CodeSystem fingerprint changed: ${key} (${this.customFingerprint?.substring(0, 8)} -> ${newFingerprint?.substring(0, 8)})`); + console.log(`[OCL] Replacing cold cache with new hot cache: ${key}`); + } else { + console.log(`[OCL] Computed fingerprint for CodeSystem: ${key} (fingerprint=${newFingerprint?.substring(0, 8)})`); + } + + // Save to cold cache + const savedFingerprint = await this.#saveColdCache(allConcepts); + if (savedFingerprint) { + this.customFingerprint = savedFingerprint; + } + } + + console.log(`[OCL] CodeSystem load completed, marked content=complete: ${key}`); + const progress = OCLSourceCodeSystemFactory.loadProgress(); + console.log(`[OCL] CodeSystem load completed: ${this.system()}. Loaded ${progress.loaded}/${progress.total} CodeSystems (${progress.percentage.toFixed(2)}%)`); + console.log(`[OCL] CodeSystem now available in cache: ${key} (${count} concepts)`); + } catch (error) { + console.error(`[OCL] CodeSystem background load failed: ${key}: ${error.message}`); + } + } + + async #loadAllConceptPages() { + if (!this.meta?.conceptsUrl) { + this.loadedConceptCount = 0; + this.backgroundLoadProgress = { processed: 0, total: 0 }; + return 0; + } + + let page = 1; + let total = 0; + + // eslint-disable-next-line no-constant-condition + while (true) { + const pageData = await this.#fetchAndCacheConceptPage(page); + const concepts = Array.isArray(pageData?.concepts) ? pageData.concepts : []; + if (concepts.length === 0) { + break; + } + total += concepts.length; + this.backgroundLoadProgress.processed = total; + if (concepts.length < CONCEPT_PAGE_SIZE) { + break; + } + page += 1; + } + + return total; + } + + async #fetchAndCacheConceptPage(page) { + const cacheKey = `${this.meta.conceptsUrl}|p=${page}|l=${CONCEPT_PAGE_SIZE}`; + if (this.sharedPageCache.has(cacheKey)) { + const cached = this.sharedPageCache.get(cacheKey); + const concepts = Array.isArray(cached) + ? cached + : Array.isArray(cached?.concepts) + ? cached.concepts + : []; + const reportedTotal = this.#extractTotalFromPayload(cached?.payload || null); + return { concepts, reportedTotal }; + } + + if (this.sharedPendingPageRequests.has(cacheKey)) { + return await this.sharedPendingPageRequests.get(cacheKey); + } + + const pending = (async () => { + let response; + try { + response = await this.httpClient.get(this.meta.conceptsUrl, { params: { page, limit: CONCEPT_PAGE_SIZE } }); + } catch (error) { + if (error && error.response && error.response.status === 404) { + this.sharedPageCache.set(cacheKey, []); + return []; + } + throw error; + } + + const payload = response.data; + const items = Array.isArray(payload) + ? payload + : Array.isArray(payload?.results) + ? payload.results + : Array.isArray(payload?.items) + ? payload.items + : Array.isArray(payload?.data) + ? payload.data + : []; + + const mapped = items + .map(item => this.#toConceptContext(item)) + .filter(Boolean); + + this.sharedPageCache.set(cacheKey, { concepts: mapped, payload }); + for (const concept of mapped) { + if (concept && concept.code) { + this.sharedConceptCache.set(concept.code, concept); + } + } + return { + concepts: mapped, + reportedTotal: this.#extractTotalFromPayload(payload) + }; + })(); + + this.sharedPendingPageRequests.set(cacheKey, pending); + try { + return await pending; + } finally { + this.sharedPendingPageRequests.delete(cacheKey); + } + } + + #syncWarmStateWithChecksum() { + const checksum = this.meta?.checksum || null; + if (this.loadedChecksum == null) { + this.loadedChecksum = checksum; + return; + } + + if (checksum !== this.loadedChecksum) { + this.isComplete = false; + this.loadedConceptCount = -1; + this.backgroundLoadProgress = { processed: 0, total: null }; + this.sharedConceptCache.clear(); + this.sharedPageCache.clear(); + this.loadedChecksum = checksum; + this.materializedConceptList = null; + this.materializedConceptCount = -1; + if (this.meta?.codeSystem?.jsonObj) { + this.meta.codeSystem.jsonObj.content = CodeSystemContentMode.NotPresent; + delete this.meta.codeSystem.jsonObj.concept; + } + console.log(`[OCL] CodeSystem checksum changed, invalidated warm cache: ${this.#resourceKey()}`); + } + } + + #applyConceptsToCodeSystemResource(codeSystem) { + if (!codeSystem || typeof codeSystem !== 'object' || !codeSystem.jsonObj) { + return; + } + + if (this.isComplete !== true) { + delete codeSystem.jsonObj.concept; + return; + } + + const concepts = Array.from(this.sharedConceptCache.values()) + .filter(concept => concept && concept.code); + + if (!Array.isArray(this.materializedConceptList) || this.materializedConceptCount !== concepts.length) { + this.materializedConceptList = concepts + .sort((a, b) => String(a.code).localeCompare(String(b.code))) + .map(concept => { + const fhirConcept = { code: concept.code }; + + if (concept.display) { + fhirConcept.display = concept.display; + } + + if (concept.definition) { + fhirConcept.definition = concept.definition; + } + + if (Array.isArray(concept.designations) && concept.designations.length > 0) { + const designations = concept.designations + .filter(d => d && d.value) + .map(d => ({ + language: d.language || undefined, + value: d.value + })); + + if (designations.length > 0) { + fhirConcept.designation = designations; + } + } + + return fhirConcept; + }); + this.materializedConceptCount = concepts.length; + } + + codeSystem.jsonObj.concept = this.materializedConceptList; + codeSystem.jsonObj.content = CodeSystemContentMode.Complete; + } + + #backgroundLoadProgressSnapshot() { + const processed = this.backgroundLoadProgress?.processed; + const total = this.backgroundLoadProgress?.total; + if ( + typeof processed === 'number' && + Number.isFinite(processed) && + typeof total === 'number' && + Number.isFinite(total) && + total > 0 + ) { + return { processed, total }; + } + return null; + } + + #extractTotalFromPayload(payload) { + if (!payload || typeof payload !== 'object' || Array.isArray(payload)) { + return null; + } + + const candidates = [ + payload.total, + payload.total_count, + payload.totalCount, + payload.num_found, + payload.numFound, + payload.count + ]; + + for (const candidate of candidates) { + if (typeof candidate === 'number' && Number.isFinite(candidate) && candidate >= 0) { + return candidate; + } + } + + return null; + } + + async #fetchConceptCountFromHeaders() { + if (!this.meta?.conceptsUrl) { + return null; + } + + try { + const response = await this.httpClient.get(this.meta.conceptsUrl, { + params: { + limit: 1 + } + }); + return this.#extractNumFoundFromHeaders(response?.headers); + } catch (error) { + return null; + } + } + + #extractNumFoundFromHeaders(headers) { + if (!headers || typeof headers !== 'object') { + return null; + } + + const raw = headers.num_found ?? headers['num-found'] ?? headers.Num_Found ?? null; + const parsed = typeof raw === 'number' ? raw : Number.parseInt(raw, 10); + if (!Number.isFinite(parsed) || parsed < 0) { + return null; + } + return parsed; + } + + #resourceKey() { + const normalizedSystem = OCLSourceCodeSystemFactory.#normalizeSystem(this.system()); + return `${normalizedSystem}|${this.version() || ''}`; + } + + currentChecksum() { + this.#syncWarmStateWithChecksum(); + return this.meta?.checksum || this.loadedChecksum || null; + } + + isCompleteNow() { + this.#syncWarmStateWithChecksum(); + return this.isComplete === true; + } + + #toConceptContext(concept) { + return toConceptContext(concept); + } + + system() { + return normalizeCanonicalSystem(this.meta.canonicalUrl); + } + + name() { + return this.meta.name || this.meta.shortCode || this.meta.id || this.system(); + } + + version() { + return this.meta.version || null; + } + + id() { + return this.meta.id || this.meta.shortCode || this.system(); + } + + iteratable() { + return true; + } +} + +module.exports = { + OCLCodeSystemProvider, + OCLSourceCodeSystemFactory, + OCLBackgroundJobQueue +}; \ No newline at end of file diff --git a/tx/ocl/cs-ocl.js b/tx/ocl/cs-ocl.js index 6e008525..118116d3 100644 --- a/tx/ocl/cs-ocl.js +++ b/tx/ocl/cs-ocl.js @@ -1,39 +1,2 @@ -/** - * Abstract base class for value set providers - * Defines the interface that all value set providers must implement - */ -// eslint-disable-next-line no-unused-vars -class OCLCodeSystemProvider { - /** - * {int} Unique number assigned to this provider - */ - spaceId; +module.exports = require('./cs-ocl.cjs'); - /** - * ensure that the ids on the code systems are unique, if they are - * in the global namespace - * - * @param {Set} ids - */ - // eslint-disable-next-line no-unused-vars - assignIds(ids) { - throw new Error('assignIds must be implemented by subclass'); - } - - /** - * Returns the list of CodeSystems this provider provides - * - * @param {string} fhirVersion - The FHIRVersion in scope - if relevant (there's always a stated version, though R5 is always used) - * @param {string} context - The client's stated context - if provided. - * @returns {Map} The list of CodeSystems - * @throws {Error} Must be implemented by subclasses - */ - // eslint-disable-next-line no-unused-vars - async listCodeSystems(fhirVersion, context) { - throw new Error('listCodeSystems must be implemented by AbstractCodeSystemProvider subclass'); - } -} - -module.exports = { - AbstractCodeSystemProvider -}; \ No newline at end of file diff --git a/tx/ocl/fingerprint/fingerprint.cjs b/tx/ocl/fingerprint/fingerprint.cjs new file mode 100644 index 00000000..4a18c158 --- /dev/null +++ b/tx/ocl/fingerprint/fingerprint.cjs @@ -0,0 +1,67 @@ +const crypto = require('crypto'); + +function hashSortedLines(lines) { + const hash = crypto.createHash('sha256'); + for (const line of lines.sort()) { + hash.update(line); + hash.update('\n'); + } + return hash.digest('hex'); +} + +function computeCodeSystemFingerprint(concepts) { + if (!Array.isArray(concepts) || concepts.length === 0) { + return null; + } + + const normalized = concepts + .map(concept => { + if (!concept || !concept.code) { + return null; + } + + const code = String(concept.code || ''); + const display = String(concept.display || ''); + const definition = String(concept.definition || ''); + const retired = concept.retired === true ? '1' : '0'; + return `${code}|${display}|${definition}|${retired}`; + }) + .filter(Boolean); + + if (normalized.length === 0) { + return null; + } + + return hashSortedLines(normalized); +} + +function computeValueSetExpansionFingerprint(expansion) { + if (!expansion || !Array.isArray(expansion.contains) || expansion.contains.length === 0) { + return null; + } + + const normalized = expansion.contains + .map(entry => { + if (!entry || !entry.code) { + return null; + } + + const system = String(entry.system || ''); + const code = String(entry.code || ''); + const display = String(entry.display || ''); + const inactive = entry.inactive === true ? '1' : '0'; + return `${system}|${code}|${display}|${inactive}`; + }) + .filter(Boolean); + + if (normalized.length === 0) { + return null; + } + + return hashSortedLines(normalized); +} + +module.exports = { + computeCodeSystemFingerprint, + computeValueSetExpansionFingerprint +}; diff --git a/tx/ocl/fingerprint/fingerprint.js b/tx/ocl/fingerprint/fingerprint.js new file mode 100644 index 00000000..74ad6070 --- /dev/null +++ b/tx/ocl/fingerprint/fingerprint.js @@ -0,0 +1,2 @@ +module.exports = require('./fingerprint.cjs'); + diff --git a/tx/ocl/http/client.cjs b/tx/ocl/http/client.cjs new file mode 100644 index 00000000..7ae7ecb2 --- /dev/null +++ b/tx/ocl/http/client.cjs @@ -0,0 +1,31 @@ +const axios = require('axios'); +const { DEFAULT_BASE_URL } = require('../shared/constants'); + +function createOclHttpClient(config = {}) { + const options = typeof config === 'string' ? { baseUrl: config } : (config || {}); + + const baseUrl = (options.baseUrl || DEFAULT_BASE_URL).replace(/\/+$/, ''); + const headers = { + Accept: 'application/json', + 'User-Agent': 'FHIRSmith-OCL-Provider/1.0' + }; + + if (options.token) { + headers.Authorization = options.token.startsWith('Token ') || options.token.startsWith('Bearer ') + ? options.token + : `Token ${options.token}`; + } + + return { + baseUrl, + client: axios.create({ + baseURL: baseUrl, + timeout: options.timeout || 30000, + headers + }) + }; +} + +module.exports = { + createOclHttpClient +}; diff --git a/tx/ocl/http/client.js b/tx/ocl/http/client.js new file mode 100644 index 00000000..805c3b60 --- /dev/null +++ b/tx/ocl/http/client.js @@ -0,0 +1,2 @@ +module.exports = require('./client.cjs'); + diff --git a/tx/ocl/http/pagination.cjs b/tx/ocl/http/pagination.cjs new file mode 100644 index 00000000..7ea7f7b4 --- /dev/null +++ b/tx/ocl/http/pagination.cjs @@ -0,0 +1,98 @@ +const { PAGE_SIZE } = require('../shared/constants'); + +function extractItemsAndNext(payload, baseUrl = null) { + if (Array.isArray(payload)) { + return { items: payload, next: null }; + } + + if (!payload || typeof payload !== 'object') { + return { items: [], next: null }; + } + + const items = Array.isArray(payload.results) + ? payload.results + : Array.isArray(payload.items) + ? payload.items + : Array.isArray(payload.data) + ? payload.data + : []; + + const next = payload.next || null; + if (!next) { + return { items, next: null }; + } + + if (baseUrl && typeof next === 'string' && next.startsWith(baseUrl)) { + return { items, next: next.replace(baseUrl, '') }; + } + + return { items, next }; +} + +async function fetchAllPages(httpClient, path, options = {}) { + const { + params = {}, + pageSize = PAGE_SIZE, + maxPages = Number.MAX_SAFE_INTEGER, + baseUrl = null, + useNextLinks = true, + logger = null, + loggerPrefix = '[OCL]' + } = options; + + const results = []; + let page = 1; + let nextPath = path; + let pageCount = 0; + let usePageMode = true; + + while (nextPath && pageCount < maxPages) { + try { + const response = usePageMode + ? await httpClient.get(path, { params: { ...params, page, limit: pageSize } }) + : await httpClient.get(nextPath); + + if (Array.isArray(response.data)) { + results.push(...response.data); + pageCount += 1; + + if (response.data.length < pageSize) { + break; + } + + page += 1; + nextPath = path; + continue; + } + + const { items, next } = extractItemsAndNext(response.data, baseUrl); + results.push(...items); + pageCount += 1; + + if (useNextLinks && next) { + usePageMode = false; + nextPath = next; + continue; + } + + if (usePageMode && items.length >= pageSize && pageCount < maxPages) { + page += 1; + nextPath = path; + } else { + break; + } + } catch (error) { + if (logger && typeof logger.error === 'function') { + logger.error(`${loggerPrefix} Fetch error on page ${page}:`, error.message); + } + throw error; + } + } + + return results; +} + +module.exports = { + extractItemsAndNext, + fetchAllPages +}; diff --git a/tx/ocl/http/pagination.js b/tx/ocl/http/pagination.js new file mode 100644 index 00000000..314b5295 --- /dev/null +++ b/tx/ocl/http/pagination.js @@ -0,0 +1,2 @@ +module.exports = require('./pagination.cjs'); + diff --git a/tx/ocl/jobs/background-queue.cjs b/tx/ocl/jobs/background-queue.cjs new file mode 100644 index 00000000..060b93b4 --- /dev/null +++ b/tx/ocl/jobs/background-queue.cjs @@ -0,0 +1,200 @@ +class OCLBackgroundJobQueue { + static MAX_CONCURRENT = 2; + static HEARTBEAT_INTERVAL_MS = 30000; + static UNKNOWN_JOB_SIZE = Number.MAX_SAFE_INTEGER; + static pendingJobs = []; + static activeCount = 0; + static queuedOrRunningKeys = new Set(); + static activeJobs = new Map(); + static heartbeatTimer = null; + static enqueueSequence = 0; + + static enqueue(jobKey, jobType, runJob, options = {}) { + if (!jobKey || typeof runJob !== 'function') { + return false; + } + + if (this.queuedOrRunningKeys.has(jobKey)) { + return false; + } + + this.queuedOrRunningKeys.add(jobKey); + const resolveAndEnqueue = async () => { + const resolvedSize = await this.#resolveJobSize(options); + const normalizedSize = this.#normalizeJobSize(resolvedSize); + this.#insertPendingJobOrdered({ + jobKey, + jobType: jobType || 'background-job', + jobId: options?.jobId || jobKey, + jobSize: normalizedSize, + getProgress: typeof options?.getProgress === 'function' ? options.getProgress : null, + runJob, + enqueueOrder: this.enqueueSequence++ + }); + this.ensureHeartbeatRunning(); + console.log(`[OCL] ${jobType || 'Background job'} enqueued: ${jobKey} (size=${normalizedSize}, queue=${this.pendingJobs.length}, active=${this.activeCount})`); + this.processNext(); + }; + + Promise.resolve() + .then(resolveAndEnqueue) + .catch((error) => { + this.queuedOrRunningKeys.delete(jobKey); + const message = error && error.message ? error.message : String(error); + console.error(`[OCL] Failed to enqueue background job: ${jobType || 'background-job'} ${jobKey}: ${message}`); + }); + + return true; + } + + static async #resolveJobSize(options = {}) { + if (typeof options?.resolveJobSize === 'function') { + try { + return await options.resolveJobSize(); + } catch (_error) { + return this.UNKNOWN_JOB_SIZE; + } + } + + if (options && Object.prototype.hasOwnProperty.call(options, 'jobSize')) { + return options.jobSize; + } + + return this.UNKNOWN_JOB_SIZE; + } + + static #normalizeJobSize(jobSize) { + const parsed = Number.parseInt(jobSize, 10); + if (!Number.isFinite(parsed) || parsed < 0) { + return this.UNKNOWN_JOB_SIZE; + } + return parsed; + } + + static #insertPendingJobOrdered(job) { + let index = this.pendingJobs.findIndex(existing => { + if (existing.jobSize === job.jobSize) { + return existing.enqueueOrder > job.enqueueOrder; + } + return existing.jobSize > job.jobSize; + }); + + if (index < 0) { + index = this.pendingJobs.length; + } + + this.pendingJobs.splice(index, 0, job); + } + + static isQueuedOrRunning(jobKey) { + return this.queuedOrRunningKeys.has(jobKey); + } + + static ensureHeartbeatRunning() { + if (this.heartbeatTimer) { + return; + } + + this.heartbeatTimer = setInterval(() => { + this.logHeartbeat(); + }, this.HEARTBEAT_INTERVAL_MS); + + if (typeof this.heartbeatTimer.unref === 'function') { + this.heartbeatTimer.unref(); + } + } + + static logHeartbeat() { + const activeJobs = Array.from(this.activeJobs.values()); + const lines = [ + '[OCL] OCL background status:', + ` active jobs: ${activeJobs.length}`, + ` queued jobs: ${this.pendingJobs.length}` + ]; + + activeJobs.forEach((job, index) => { + lines.push(''); + lines.push(` job ${index + 1}:`); + lines.push(` type: ${job.jobType || 'background-job'}`); + lines.push(` id: ${job.jobId || job.jobKey}`); + lines.push(` size: ${job.jobSize}`); + lines.push(` progress: ${this.formatProgress(job.getProgress)}`); + }); + + console.log(lines.join('\n')); + } + + static formatProgress(getProgress) { + if (typeof getProgress !== 'function') { + return 'unknown'; + } + + try { + const progress = getProgress(); + if (typeof progress === 'number' && Number.isFinite(progress)) { + const bounded = Math.max(0, Math.min(100, progress)); + return `${Math.round(bounded)}%`; + } + + if (progress && typeof progress === 'object') { + if (typeof progress.percentage === 'number' && Number.isFinite(progress.percentage)) { + const bounded = Math.max(0, Math.min(100, progress.percentage)); + return `${Math.round(bounded)}%`; + } + + if ( + typeof progress.processed === 'number' && + Number.isFinite(progress.processed) && + typeof progress.total === 'number' && + Number.isFinite(progress.total) && + progress.total > 0 + ) { + const ratio = progress.processed / progress.total; + const bounded = Math.max(0, Math.min(100, ratio * 100)); + return `${Math.round(bounded)}%`; + } + } + } catch (_error) { + return 'unknown'; + } + + return 'unknown'; + } + + static processNext() { + while (this.activeCount < this.MAX_CONCURRENT && this.pendingJobs.length > 0) { + const job = this.pendingJobs.shift(); + this.activeCount += 1; + this.activeJobs.set(job.jobKey, { + jobKey: job.jobKey, + jobType: job.jobType, + jobId: job.jobId || job.jobKey, + jobSize: job.jobSize, + getProgress: job.getProgress || null, + startedAt: Date.now() + }); + console.log(`[OCL] Background job started: ${job.jobType} ${job.jobKey} (size=${job.jobSize}, queue=${this.pendingJobs.length}, active=${this.activeCount})`); + + Promise.resolve() + .then(() => job.runJob()) + .then(() => { + console.log(`[OCL] Background job completed: ${job.jobType} ${job.jobKey}`); + }) + .catch((error) => { + const message = error && error.message ? error.message : String(error); + console.error(`[OCL] Background job failed: ${job.jobType} ${job.jobKey}: ${message}`); + }) + .finally(() => { + this.activeCount -= 1; + this.queuedOrRunningKeys.delete(job.jobKey); + this.activeJobs.delete(job.jobKey); + console.log(`[OCL] Background queue status: queue=${this.pendingJobs.length}, active=${this.activeCount}`); + this.processNext(); + }); + } + } +} + +module.exports = { + OCLBackgroundJobQueue +}; diff --git a/tx/ocl/jobs/background-queue.js b/tx/ocl/jobs/background-queue.js new file mode 100644 index 00000000..5bdea4a4 --- /dev/null +++ b/tx/ocl/jobs/background-queue.js @@ -0,0 +1,2 @@ +module.exports = require('./background-queue.cjs'); + diff --git a/tx/ocl/mappers/concept-mapper.cjs b/tx/ocl/mappers/concept-mapper.cjs new file mode 100644 index 00000000..e350deb0 --- /dev/null +++ b/tx/ocl/mappers/concept-mapper.cjs @@ -0,0 +1,66 @@ +function toConceptContext(concept) { + if (!concept || typeof concept !== 'object') { + return null; + } + + const code = concept.code || concept.id || null; + if (!code) { + return null; + } + + return { + code, + display: concept.display_name || concept.display || concept.name || null, + definition: concept.description || concept.definition || null, + retired: concept.retired === true, + designations: extractDesignations(concept) + }; +} + +function extractDesignations(concept) { + const result = []; + const seen = new Set(); + + const add = (language, value) => { + const text = typeof value === 'string' ? value.trim() : ''; + if (!text) { + return; + } + + const lang = typeof language === 'string' ? language.trim() : ''; + const key = `${lang}|${text}`; + if (seen.has(key)) { + return; + } + + seen.add(key); + result.push({ language: lang, value: text }); + }; + + if (Array.isArray(concept.names)) { + for (const entry of concept.names) { + if (!entry || typeof entry !== 'object') { + continue; + } + + add(entry.locale || entry.language || entry.lang || '', entry.name || entry.display_name || entry.display || entry.value || entry.term); + } + } + + if (concept.display_name || concept.display || concept.name) { + add(concept.locale || concept.default_locale || concept.language || '', concept.display_name || concept.display || concept.name); + } + + if (concept.locale_display_names && typeof concept.locale_display_names === 'object') { + for (const [lang, value] of Object.entries(concept.locale_display_names)) { + add(lang, value); + } + } + + return result; +} + +module.exports = { + toConceptContext, + extractDesignations +}; diff --git a/tx/ocl/mappers/concept-mapper.js b/tx/ocl/mappers/concept-mapper.js new file mode 100644 index 00000000..b0fc7908 --- /dev/null +++ b/tx/ocl/mappers/concept-mapper.js @@ -0,0 +1,2 @@ +module.exports = require('./concept-mapper.cjs'); + diff --git a/tx/ocl/model/concept-filter-context.cjs b/tx/ocl/model/concept-filter-context.cjs new file mode 100644 index 00000000..330ac2a7 --- /dev/null +++ b/tx/ocl/model/concept-filter-context.cjs @@ -0,0 +1,51 @@ +class OCLConceptFilterContext { + constructor() { + this.concepts = []; + this.currentIndex = -1; + } + + add(concept, rating = 0) { + this.concepts.push({ concept, rating }); + } + + sort() { + this.concepts.sort((a, b) => b.rating - a.rating); + } + + size() { + return this.concepts.length; + } + + hasMore() { + return this.currentIndex + 1 < this.concepts.length; + } + + next() { + if (!this.hasMore()) { + return null; + } + this.currentIndex += 1; + return this.concepts[this.currentIndex].concept; + } + + reset() { + this.currentIndex = -1; + } + + findConceptByCode(code) { + for (const item of this.concepts) { + if (item.concept && item.concept.code === code) { + return item.concept; + } + } + return null; + } + + containsConcept(concept) { + return this.concepts.some(item => item.concept === concept); + } +} + +module.exports = { + OCLConceptFilterContext +}; diff --git a/tx/ocl/model/concept-filter-context.js b/tx/ocl/model/concept-filter-context.js new file mode 100644 index 00000000..046dcd2b --- /dev/null +++ b/tx/ocl/model/concept-filter-context.js @@ -0,0 +1,2 @@ +module.exports = require('./concept-filter-context.cjs'); + diff --git a/tx/ocl/shared/constants.cjs b/tx/ocl/shared/constants.cjs new file mode 100644 index 00000000..5b338687 --- /dev/null +++ b/tx/ocl/shared/constants.cjs @@ -0,0 +1,15 @@ +const DEFAULT_BASE_URL = 'https://api.openconceptlab.org'; +const PAGE_SIZE = 100; +const CONCEPT_PAGE_SIZE = 1000; +const FILTERED_CONCEPT_PAGE_SIZE = 200; +const COLD_CACHE_FRESHNESS_MS = 60 * 60 * 1000; +const OCL_CODESYSTEM_MARKER_EXTENSION = 'http://fhir.org/FHIRsmith/StructureDefinition/ocl-codesystem'; + +module.exports = { + DEFAULT_BASE_URL, + PAGE_SIZE, + CONCEPT_PAGE_SIZE, + FILTERED_CONCEPT_PAGE_SIZE, + COLD_CACHE_FRESHNESS_MS, + OCL_CODESYSTEM_MARKER_EXTENSION +}; diff --git a/tx/ocl/shared/constants.js b/tx/ocl/shared/constants.js new file mode 100644 index 00000000..9680d3d1 --- /dev/null +++ b/tx/ocl/shared/constants.js @@ -0,0 +1,2 @@ +module.exports = require('./constants.cjs'); + diff --git a/tx/ocl/shared/patches.cjs b/tx/ocl/shared/patches.cjs new file mode 100644 index 00000000..eae4750b --- /dev/null +++ b/tx/ocl/shared/patches.cjs @@ -0,0 +1,224 @@ +const { OCL_CODESYSTEM_MARKER_EXTENSION } = require('./constants'); + +const OCL_SEARCH_PATCH_FLAG = Symbol.for('fhirsmith.ocl.search.codesystem.code.patch'); +const TXPARAMS_HASH_PATCH_FLAG = Symbol.for('fhirsmith.ocl.txparameters.hash.filter.patch'); +const OCL_EXPAND_WHOLE_SYSTEM_PATCH_FLAG = Symbol.for('fhirsmith.ocl.expand.whole-system.patch'); +const OCL_EXPAND_WHOLE_SYSTEM_RETRY_FLAG = Symbol.for('fhirsmith.ocl.expand.whole-system.retry'); + +function hasOCLCodeSystemMarker(resource) { + const extensions = Array.isArray(resource?.extension) ? resource.extension : []; + return extensions.some(ext => ext && ext.url === OCL_CODESYSTEM_MARKER_EXTENSION); +} + +function filterConceptTreeByCode(concepts, wantedCode) { + if (!Array.isArray(concepts) || concepts.length === 0) { + return []; + } + + const matches = []; + for (const concept of concepts) { + if (!concept || typeof concept !== 'object') { + continue; + } + + const childMatches = filterConceptTreeByCode(concept.concept, wantedCode); + const isSelfMatch = concept.code != null && String(concept.code) === wantedCode; + if (!isSelfMatch && childMatches.length === 0) { + continue; + } + + const clone = { ...concept }; + if (childMatches.length > 0) { + clone.concept = childMatches; + } else { + delete clone.concept; + } + matches.push(clone); + } + + return matches; +} + +function filterOCLCodeSystemResourceByCode(resource, code) { + if (!resource || typeof resource !== 'object') { + return resource; + } + + const filteredConcepts = filterConceptTreeByCode(resource.concept, code); + return { + ...resource, + concept: filteredConcepts + }; +} + +function patchSearchWorkerForOCLCodeFiltering() { + let SearchWorker; + try { + SearchWorker = require('../../workers/search'); + } catch (_error) { + return; + } + + if (!SearchWorker || !SearchWorker.prototype) { + return; + } + + const proto = SearchWorker.prototype; + if (proto[OCL_SEARCH_PATCH_FLAG] === true || typeof proto.searchCodeSystems !== 'function') { + return; + } + + const originalSearchCodeSystems = proto.searchCodeSystems; + proto.searchCodeSystems = function patchedSearchCodeSystems(params) { + const matches = originalSearchCodeSystems.call(this, params); + const requestedCode = params?.code == null ? '' : String(params.code); + + if (!requestedCode) { + return matches; + } + + const filtered = []; + for (const resource of matches) { + if (!hasOCLCodeSystemMarker(resource)) { + filtered.push(resource); + continue; + } + + const projected = filterOCLCodeSystemResourceByCode(resource, requestedCode); + if (Array.isArray(projected?.concept) && projected.concept.length > 0) { + filtered.push(projected); + } + } + + return filtered; + }; + + Object.defineProperty(proto, OCL_SEARCH_PATCH_FLAG, { + value: true, + writable: false, + configurable: false, + enumerable: false + }); +} + +function normalizeFilterForCacheKey(filter) { + if (typeof filter !== 'string') { + return ''; + } + + return filter.trim().toLowerCase(); +} + +function ensureTxParametersHashIncludesFilter(TxParameters) { + const proto = TxParameters && TxParameters.prototype; + if (!proto || proto[TXPARAMS_HASH_PATCH_FLAG] === true || typeof proto.hashSource !== 'function') { + return; + } + + const originalHashSource = proto.hashSource; + proto.hashSource = function hashSourceWithFilter() { + const base = originalHashSource.call(this); + const normalizedFilter = normalizeFilterForCacheKey(this.filter); + return `${base}|filter=${normalizedFilter}`; + }; + + Object.defineProperty(proto, TXPARAMS_HASH_PATCH_FLAG, { + value: true, + writable: false, + configurable: false, + enumerable: false + }); +} + +function isOclWholeSystemRetryCandidate(expander, cset, filter) { + if (!expander || !cset) { + return false; + } + + // Only touch the OCL ValueSet path where OCL helpers are attached. + const sourceValueSet = expander?.valueSet; + if (!sourceValueSet || typeof sourceValueSet.oclFetchConcepts !== 'function') { + return false; + } + + // This fallback is only for whole-system, unfiltered includes. + if (!cset.system || cset.concept || cset.filter) { + return false; + } + if (!filter || filter.isNull !== true) { + return false; + } + + // Only engage when request did not ask for an explicit page and we have a limit. + if (!(expander.count < 0 && expander.offset < 0 && expander.limitCount > 0)) { + return false; + } + + // Prevent recursive retries for the same include call path. + if (expander[OCL_EXPAND_WHOLE_SYSTEM_RETRY_FLAG] === true) { + return false; + } + + return true; +} + +function patchValueSetExpandWholeSystemForOcl() { + let expandModule; + try { + expandModule = require('../../workers/expand'); + } catch (_error) { + return; + } + + const ValueSetExpander = expandModule?.ValueSetExpander; + const proto = ValueSetExpander && ValueSetExpander.prototype; + if (!proto || proto[OCL_EXPAND_WHOLE_SYSTEM_PATCH_FLAG] === true || typeof proto.includeCodes !== 'function') { + return; + } + + const originalIncludeCodes = proto.includeCodes; + proto.includeCodes = async function patchedIncludeCodes(cset, path, vsSrc, compose, filter, expansion, excludeInactive, notClosed) { + try { + return await originalIncludeCodes.call(this, cset, path, vsSrc, compose, filter, expansion, excludeInactive, notClosed); + } catch (error) { + if (!isOclWholeSystemRetryCandidate(this, cset, filter)) { + throw error; + } + + const prevCount = this.count; + const prevOffset = this.offset; + this.count = this.limitCount; + this.offset = 0; + + // Mirror effective pagination into expansion parameters for transparency. + if (expansion && typeof this.addParamInt === 'function') { + this.addParamInt(expansion, 'offset', this.offset); + this.addParamInt(expansion, 'count', this.count); + expansion.offset = this.offset; + } + + this[OCL_EXPAND_WHOLE_SYSTEM_RETRY_FLAG] = true; + try { + return await originalIncludeCodes.call(this, cset, path, vsSrc, compose, filter, expansion, excludeInactive, notClosed); + } finally { + this[OCL_EXPAND_WHOLE_SYSTEM_RETRY_FLAG] = false; + this.count = prevCount; + this.offset = prevOffset; + } + } + }; + + Object.defineProperty(proto, OCL_EXPAND_WHOLE_SYSTEM_PATCH_FLAG, { + value: true, + writable: false, + configurable: false, + enumerable: false + }); +} + +module.exports = { + patchSearchWorkerForOCLCodeFiltering, + ensureTxParametersHashIncludesFilter, + patchValueSetExpandWholeSystemForOcl, + normalizeFilterForCacheKey +}; diff --git a/tx/ocl/shared/patches.js b/tx/ocl/shared/patches.js new file mode 100644 index 00000000..1771e438 --- /dev/null +++ b/tx/ocl/shared/patches.js @@ -0,0 +1,2 @@ +module.exports = require('./patches.cjs'); + diff --git a/tx/ocl/vs-ocl.cjs b/tx/ocl/vs-ocl.cjs new file mode 100644 index 00000000..6b33fc12 --- /dev/null +++ b/tx/ocl/vs-ocl.cjs @@ -0,0 +1,1848 @@ +const fs = require('fs/promises'); +const crypto = require('crypto'); +const path = require('path'); +const { AbstractValueSetProvider } = require('../vs/vs-api'); +const { VersionUtilities } = require('../../library/version-utilities'); +const ValueSet = require('../library/valueset'); +const { SearchFilterText } = require('../library/designations'); +const { TxParameters } = require('../params'); +const { OCLSourceCodeSystemFactory, OCLBackgroundJobQueue } = require('./cs-ocl'); +const { PAGE_SIZE, CONCEPT_PAGE_SIZE, FILTERED_CONCEPT_PAGE_SIZE, COLD_CACHE_FRESHNESS_MS } = require('./shared/constants'); +const { createOclHttpClient } = require('./http/client'); +const { CACHE_VS_DIR, getCacheFilePath } = require('./cache/cache-paths'); +const { ensureCacheDirectories, getColdCacheAgeMs, formatCacheAgeMinutes } = require('./cache/cache-utils'); +const { computeValueSetExpansionFingerprint } = require('./fingerprint/fingerprint'); +const { ensureTxParametersHashIncludesFilter, patchValueSetExpandWholeSystemForOcl } = require('./shared/patches'); + +ensureTxParametersHashIncludesFilter(TxParameters); +patchValueSetExpandWholeSystemForOcl(); + +function normalizeCanonicalSystem(system) { + if (typeof system !== 'string') { + return system; + } + + const trimmed = system.trim(); + if (!trimmed) { + return trimmed; + } + + // Treat canonical URLs with and without trailing slash as equivalent. + return trimmed.replace(/\/+$/, ''); +} + +class OCLValueSetProvider extends AbstractValueSetProvider { + constructor(config = {}) { + super(); + const options = typeof config === 'string' ? { baseUrl: config } : (config || {}); + + this.org = options.org || null; + const http = createOclHttpClient(options); + this.baseUrl = http.baseUrl; + this.httpClient = http.client; + + this.valueSetMap = new Map(); + this._idMap = new Map(); + this.collectionMeta = new Map(); + this.sourceCanonicalCache = new Map(); + this.collectionConceptPageCache = new Map(); + this.pendingCollectionConceptPageRequests = new Map(); + this.collectionSourcesCache = new Map(); + this.pendingCollectionSourcesRequests = new Map(); + this.pendingSourceCanonicalRequests = new Map(); + this.collectionByCanonicalCache = new Map(); + this.pendingCollectionByCanonicalRequests = new Map(); + this._composePromises = new Map(); + this.backgroundExpansionCache = new Map(); + this.backgroundExpansionProgress = new Map(); + this.valueSetFingerprints = new Map(); + this._initialized = false; + this._initializePromise = null; + this.sourcePackageCode = this.org + ? `ocl:${this.baseUrl}|org=${this.org}` + : `ocl:${this.baseUrl}`; + } + + async #loadColdCacheForValueSets() { + try { + const files = await fs.readdir(CACHE_VS_DIR); + let loadedCount = 0; + + for (const file of files) { + if (!file.endsWith('.json')) { + continue; + } + + try { + const filePath = path.join(CACHE_VS_DIR, file); + const data = await fs.readFile(filePath, 'utf-8'); + const cached = JSON.parse(data); + + if (!cached || !cached.canonicalUrl || !cached.expansion) { + continue; + } + + const paramsKey = cached.paramsKey || 'default'; + const cacheKey = this.#expansionCacheKey( + { url: cached.canonicalUrl, version: cached.version || null }, + paramsKey + ); + const createdAt = cached.timestamp ? new Date(cached.timestamp).getTime() : null; + this.backgroundExpansionCache.set(cacheKey, { + expansion: cached.expansion, + metadataSignature: cached.metadataSignature || null, + dependencyChecksums: cached.dependencyChecksums || {}, + createdAt: Number.isFinite(createdAt) ? createdAt : null + }); + + this.valueSetFingerprints.set(cacheKey, cached.fingerprint || null); + loadedCount++; + console.log(`[OCL-ValueSet] Loaded ValueSet from cold cache: ${cached.canonicalUrl}`); + } catch (error) { + console.error(`[OCL-ValueSet] Failed to load cold cache file ${file}:`, error.message); + } + } + + if (loadedCount > 0) { + console.log(`[OCL-ValueSet] Loaded ${loadedCount} ValueSet expansions from cold cache`); + } + } catch (error) { + if (error.code !== 'ENOENT') { + console.error('[OCL-ValueSet] Failed to load cold cache:', error.message); + } + } + } + + async #saveColdCacheForValueSet(vs, expansion, metadataSignature, dependencyChecksums, paramsKey = 'default') { + const canonicalUrl = vs?.url; + const version = vs?.version || null; + if (!canonicalUrl || !expansion) { + return null; + } + + const cacheFilePath = getCacheFilePath(CACHE_VS_DIR, canonicalUrl, version, paramsKey); + + try { + await ensureCacheDirectories(CACHE_VS_DIR); + + const fingerprint = computeValueSetExpansionFingerprint(expansion); + const cacheData = { + canonicalUrl, + version, + paramsKey, + fingerprint, + timestamp: new Date().toISOString(), + conceptCount: expansion.contains?.length || 0, + expansion, + metadataSignature, + dependencyChecksums + }; + + await fs.writeFile(cacheFilePath, JSON.stringify(cacheData, null, 2), 'utf-8'); + console.log(`[OCL-ValueSet] Saved ValueSet expansion to cold cache: ${canonicalUrl} (${expansion.contains?.length || 0} concepts, fingerprint=${fingerprint?.substring(0, 8)})`); + + return fingerprint; + } catch (error) { + console.error(`[OCL-ValueSet] Failed to save cold cache for ValueSet ${canonicalUrl}:`, error.message); + return null; + } + } + + sourcePackage() { + return this.sourcePackageCode; + } + + async initialize() { + if (this._initialized) { + return; + } + + if (this._initializePromise) { + await this._initializePromise; + return; + } + + this._initializePromise = (async () => { + try { + // Load cold cache first + await this.#loadColdCacheForValueSets(); + + const collections = await this.#fetchCollectionsForDiscovery(); + console.log(`[OCL-ValueSet] Fetched ${collections.length} collections`); + + for (const collection of collections) { + const valueSet = this.#toValueSet(collection); + if (!valueSet) { + continue; + } + this.#indexValueSet(valueSet); + } + + console.log(`[OCL-ValueSet] Loaded ${this.valueSetMap.size} value sets`); + this._initialized = true; + } catch (error) { + console.error(`[OCL-ValueSet] Initialization failed:`, error.message); + if (error.response) { + console.error(`[OCL-ValueSet] HTTP ${error.response.status}: ${error.response.statusText}`); + } + throw error; + } + })(); + + try { + await this._initializePromise; + } finally { + this._initializePromise = null; + } + } + + assignIds(ids) { + if (!this.spaceId) { + return; + } + + const unique = new Set(this.valueSetMap.values()); + this._idMap.clear(); + + for (const vs of unique) { + if (!vs.id.startsWith(`${this.spaceId}-`)) { + const nextId = `${this.spaceId}-${vs.id}`; + vs.id = nextId; + vs.jsonObj.id = nextId; + } + this._idMap.set(vs.id, vs); + ids.add(`ValueSet/${vs.id}`); + } + } + + async fetchValueSet(url, version) { + this._validateFetchParams(url, version); + + let key = `${url}|${version}`; + if (this.valueSetMap.has(key)) { + const vs = this.valueSetMap.get(key); + await this.#ensureComposeIncludes(vs); + this.#clearInlineExpansion(vs); + this.#scheduleBackgroundExpansion(vs, { reason: 'fetch-valueset' }); + return vs; + } + + if (version && VersionUtilities.isSemVer(version)) { + const majorMinor = VersionUtilities.getMajMin(version); + if (majorMinor) { + key = `${url}|${majorMinor}`; + if (this.valueSetMap.has(key)) { + const vs = this.valueSetMap.get(key); + await this.#ensureComposeIncludes(vs); + this.#clearInlineExpansion(vs); + this.#scheduleBackgroundExpansion(vs, { reason: 'fetch-valueset-mm' }); + return vs; + } + } + } + + if (this.valueSetMap.has(url)) { + const vs = this.valueSetMap.get(url); + await this.#ensureComposeIncludes(vs); + this.#clearInlineExpansion(vs); + this.#scheduleBackgroundExpansion(vs, { reason: 'fetch-valueset-url' }); + return vs; + } + + const resolved = await this.#resolveValueSetByCanonical(url, version); + if (resolved) { + await this.#ensureComposeIncludes(resolved); + this.#clearInlineExpansion(resolved); + this.#scheduleBackgroundExpansion(resolved, { reason: 'fetch-valueset-resolved' }); + return resolved; + } + + await this.initialize(); + + key = `${url}|${version}`; + if (this.valueSetMap.has(key)) { + const vs = this.valueSetMap.get(key); + await this.#ensureComposeIncludes(vs); + this.#clearInlineExpansion(vs); + this.#scheduleBackgroundExpansion(vs, { reason: 'fetch-valueset-init' }); + return vs; + } + + if (version && VersionUtilities.isSemVer(version)) { + const majorMinor = VersionUtilities.getMajMin(version); + if (majorMinor) { + key = `${url}|${majorMinor}`; + if (this.valueSetMap.has(key)) { + const vs = this.valueSetMap.get(key); + await this.#ensureComposeIncludes(vs); + this.#clearInlineExpansion(vs); + this.#scheduleBackgroundExpansion(vs, { reason: 'fetch-valueset-init-mm' }); + return vs; + } + } + } + + if (this.valueSetMap.has(url)) { + const vs = this.valueSetMap.get(url); + await this.#ensureComposeIncludes(vs); + this.#clearInlineExpansion(vs); + this.#scheduleBackgroundExpansion(vs, { reason: 'fetch-valueset-init-url' }); + return vs; + } + + return null; + } + + async fetchValueSetById(id) { + const local = this.#getLocalValueSetById(id); + if (local) { + await this.#ensureComposeIncludes(local); + this.#clearInlineExpansion(local); + this.#scheduleBackgroundExpansion(local, { reason: 'fetch-valueset-by-id' }); + return local; + } + + await this.initialize(); + + const vs = this.#getLocalValueSetById(id); + await this.#ensureComposeIncludes(vs); + this.#clearInlineExpansion(vs); + this.#scheduleBackgroundExpansion(vs, { reason: 'fetch-valueset-by-id-init' }); + return vs; + } + + #clearInlineExpansion(vs) { + if (!vs || !vs.jsonObj || !vs.jsonObj.expansion) { + return; + } + delete vs.jsonObj.expansion; + } + + #getLocalValueSetById(id) { + if (this._idMap.has(id)) { + return this._idMap.get(id); + } + + if (this.spaceId && id.startsWith(`${this.spaceId}-`)) { + const unprefixed = id.substring(this.spaceId.length + 1); + return this._idMap.get(id) || this._idMap.get(unprefixed) || this.valueSetMap.get(unprefixed) || null; + } + + return this._idMap.get(id) || this.valueSetMap.get(id) || null; + } + + // eslint-disable-next-line no-unused-vars + async searchValueSets(searchParams, _elements) { + await this.initialize(); + this._validateSearchParams(searchParams); + + const params = Object.fromEntries(searchParams.map(({ name, value }) => [name, String(value).toLowerCase()])); + const values = Array.from(new Set(this.valueSetMap.values())); + + if (Object.keys(params).length === 0) { + return values; + } + + return values.filter(vs => this.#matches(vs.jsonObj, params)); + } + + vsCount() { + return new Set(this.valueSetMap.values()).size; + } + + async listAllValueSets() { + await this.initialize(); + const urls = new Set(); + for (const vs of this.valueSetMap.values()) { + if (vs && vs.url) { + urls.add(vs.url); + } + } + return Array.from(urls); + } + + async close() { + } + + #indexValueSet(vs) { + const existing = this.valueSetMap.get(vs.url) + || (vs.version ? this.valueSetMap.get(`${vs.url}|${vs.version}`) : null) + || this._idMap.get(vs.id) + || null; + + // Preserve hydrated cold-cache expansions on first index; invalidate only on replacement. + if (existing && existing !== vs) { + this.#invalidateExpansionCache(vs); + } + + this.valueSetMap.set(vs.url, vs); + if (vs.version) { + this.valueSetMap.set(`${vs.url}|${vs.version}`, vs); + } + this.valueSetMap.set(vs.id, vs); + this._idMap.set(vs.id, vs); + } + + #toValueSet(collection) { + if (!collection || typeof collection !== 'object') { + return null; + } + + const canonicalUrl = collection.canonical_url || collection.canonicalUrl || collection.url; + const id = collection.id; + if (!canonicalUrl || !id) { + return null; + } + + const preferredSource = normalizeCanonicalSystem(collection.preferred_source || collection.preferredSource || null); + const json = { + resourceType: 'ValueSet', + id, + url: canonicalUrl, + version: collection.version || null, + name: collection.name || id, + title: collection.full_name || collection.fullName || collection.name || id, + status: 'active', + experimental: collection.experimental === true, + immutable: collection.immutable === true, + description: collection.description || null, + publisher: collection.publisher || collection.owner || null, + language: collection.default_locale || collection.defaultLocale || null + }; + + const lastUpdated = this.#toIsoDate(collection.updated_on || collection.updatedOn || collection.updated_at || collection.updatedAt); + if (lastUpdated) { + json.meta = { lastUpdated }; + } + + if (preferredSource) { + json.compose = { + include: [{ system: preferredSource }] + }; + } + + const conceptsUrl = this.#normalizePath( + collection.concepts_url || collection.conceptsUrl || this.#buildCollectionConceptsPath(collection) + ); + const expansionUrl = this.#normalizePath( + collection.expansion_url || collection.expansionUrl || this.#buildCollectionExpansionPath(collection) + ); + + const meta = { + collectionId: collection.id || collection.short_code || collection.shortCode || null, + conceptsUrl, + expansionUrl, + preferredSource, + owner: collection.owner || null, + ownerType: collection.owner_type || collection.ownerType || null + }; + + this.#storeCollectionMeta(id, canonicalUrl, meta); + + const valueSet = new ValueSet(json, 'R5'); + this.#attachOclHelpers(valueSet, meta); + return valueSet; + } + + #attachOclHelpers(valueSet, meta) { + if (!valueSet || !meta) { + return; + } + + valueSet.oclMeta = meta; + valueSet.oclFetchConcepts = async ({ count, offset, activeOnly, filter, languageCodes }) => { + return this.#fetchCollectionConcepts(meta, { + count, + offset, + activeOnly, + filter: typeof filter === 'string' ? filter : null, + languageCodes: Array.isArray(languageCodes) ? languageCodes : [], + fallbackSystem: meta.preferredSource || valueSet.url + }); + }; + } + + #storeCollectionMeta(id, url, meta) { + if (!meta || (!meta.conceptsUrl && !meta.expansionUrl)) { + return; + } + if (id) { + this.collectionMeta.set(id, meta); + } + if (url) { + this.collectionMeta.set(url, meta); + } + } + + #normalizePath(pathValue) { + if (!pathValue) { + return null; + } + if (typeof pathValue !== 'string') { + return null; + } + if (pathValue.startsWith('http://') || pathValue.startsWith('https://')) { + return pathValue; + } + return `${this.baseUrl}${pathValue.startsWith('/') ? '' : '/'}${pathValue}`; + } + + #buildCollectionConceptsPath(collection) { + if (!collection || typeof collection !== 'object') { + return null; + } + const owner = collection.owner || null; + const ownerType = collection.owner_type || collection.ownerType || null; + const id = collection.id || collection.short_code || collection.shortCode || null; + if (!owner || !id || ownerType !== 'Organization') { + return null; + } + return `/orgs/${encodeURIComponent(owner)}/collections/${encodeURIComponent(id)}/concepts/`; + } + + #buildCollectionExpansionPath(collection) { + if (!collection || typeof collection !== 'object') { + return null; + } + const owner = collection.owner || null; + const ownerType = collection.owner_type || collection.ownerType || null; + const id = collection.id || collection.short_code || collection.shortCode || null; + if (!owner || !id || ownerType !== 'Organization') { + return null; + } + return `/orgs/${encodeURIComponent(owner)}/collections/${encodeURIComponent(id)}/HEAD/expansions/autoexpand-HEAD/`; + } + + #getCollectionMeta(vs) { + if (!vs) { + return null; + } + return this.collectionMeta.get(vs.id) || this.collectionMeta.get(vs.url) || null; + } + + async #ensureComposeIncludes(vs) { + if (!vs || !vs.jsonObj) { + return; + } + + const meta = this.#getCollectionMeta(vs); + + const composeKey = vs.id || vs.url; + if (this._composePromises.has(composeKey)) { + await this._composePromises.get(composeKey); + return; + } + + const promise = (async () => { + const existingInclude = Array.isArray(vs?.jsonObj?.compose?.include) + ? vs.jsonObj.compose.include + : []; + + // Always normalize existing compose entries first because discovery metadata + // can carry non-canonical preferred_source values. + const include = this.#normalizeComposeIncludes(existingInclude); + + // Reconcile with collection-resolved sources whenever available so $expand + // and direct CodeSystem lookups share the same canonical registry keys. + if (meta && (meta.conceptsUrl || meta.expansionUrl)) { + const sources = await this.#fetchCollectionSources(meta); + if (Array.isArray(sources) && sources.length > 0) { + include.push(...this.#normalizeComposeIncludes(sources)); + } + } + + // Preferred source is a fallback only when no resolvable include was found. + if (include.length === 0 && meta?.preferredSource) { + include.push(...this.#normalizeComposeIncludes([{ system: meta.preferredSource }])); + } + + const deduped = this.#dedupeComposeIncludes(include); + if (deduped.length > 0) { + vs.jsonObj.compose = { include: deduped }; + } + })(); + + this._composePromises.set(composeKey, promise); + try { + await promise; + } finally { + this._composePromises.delete(composeKey); + } + } + + #normalizeComposeIncludes(includeEntries) { + if (!Array.isArray(includeEntries) || includeEntries.length === 0) { + return []; + } + + const normalized = []; + for (const entry of includeEntries) { + if (!entry || typeof entry !== 'object') { + continue; + } + + const system = normalizeCanonicalSystem(entry.system); + if (!system) { + continue; + } + + let version = entry.version || null; + const hasAnyFactory = OCLSourceCodeSystemFactory.hasFactory(system, null); + const hasExactFactory = OCLSourceCodeSystemFactory.hasExactFactory(system, version); + + // If include.version does not match the registered OCL factory key, + // omit it so Provider lookup can reuse the already loaded canonical factory. + if (version && hasAnyFactory && !hasExactFactory) { + version = null; + } + + normalized.push({ + system, + version: version || undefined + }); + } + + return normalized; + } + + #dedupeComposeIncludes(includeEntries) { + const deduped = []; + const seen = new Set(); + + for (const include of includeEntries || []) { + const system = normalizeCanonicalSystem(include?.system); + if (!system) { + continue; + } + + const version = include?.version || ''; + const key = `${system}|${version}`; + if (seen.has(key)) { + continue; + } + + seen.add(key); + deduped.push({ + system, + version: version || undefined + }); + } + + return deduped; + } + + async #fetchCollectionSources(meta) { + const sourcesCacheKey = `${meta.owner || ''}|${meta.collectionId || ''}|${meta.conceptsUrl || ''}|${meta.expansionUrl || ''}`; + if (this.collectionSourcesCache.has(sourcesCacheKey)) { + return this.collectionSourcesCache.get(sourcesCacheKey); + } + if (this.pendingCollectionSourcesRequests.has(sourcesCacheKey)) { + return this.pendingCollectionSourcesRequests.get(sourcesCacheKey); + } + + const pending = (async () => { + const sources = []; + const seen = new Set(); + + if (meta.expansionUrl) { + try { + const response = await this.httpClient.get(meta.expansionUrl); + const resolved = Array.isArray(response.data?.resolved_source_versions) + ? response.data.resolved_source_versions + : []; + + for (const entry of resolved) { + const system = entry.canonical_url || entry.canonicalUrl || null; + const owner = entry.owner || meta.owner || null; + const shortCode = entry.short_code || entry.shortCode || entry.id || null; + const version = entry.version || null; + + const systemUrl = normalizeCanonicalSystem(system || (owner && shortCode ? await this.#getSourceCanonicalUrl(owner, shortCode) : null)); + if (systemUrl && !seen.has(systemUrl)) { + seen.add(systemUrl); + sources.push({ system: systemUrl, version }); + } + } + } catch (error) { + // fall through to concepts listing + } + } + + if (sources.length > 0) { + return sources; + } + + if (!meta.conceptsUrl) { + return sources; + } + + const sourceKeys = await this.#fetchCollectionSourceKeys(meta.conceptsUrl, meta.owner || null); + for (const { owner, source } of sourceKeys) { + const systemUrl = normalizeCanonicalSystem(await this.#getSourceCanonicalUrl(owner, source)); + if (systemUrl && !seen.has(systemUrl)) { + seen.add(systemUrl); + sources.push({ system: systemUrl }); + } + } + + if (sources.length === 0 && meta.preferredSource) { + const preferredSource = normalizeCanonicalSystem(meta.preferredSource); + if (preferredSource) { + sources.push({ system: preferredSource }); + } + } + + this.collectionSourcesCache.set(sourcesCacheKey, sources); + return sources; + })(); + + this.pendingCollectionSourcesRequests.set(sourcesCacheKey, pending); + try { + return await pending; + } finally { + this.pendingCollectionSourcesRequests.delete(sourcesCacheKey); + } + } + + async #fetchCollectionConcepts(meta, options) { + if (!meta || !meta.conceptsUrl) { + return { contains: [], total: 0 }; + } + + const count = Number.isInteger(options?.count) ? options.count : CONCEPT_PAGE_SIZE; + const offset = Number.isInteger(options?.offset) ? options.offset : 0; + const activeOnly = options?.activeOnly === true; + const filter = this.#normalizeFilter(options?.filter); + const filterMatcher = filter ? new SearchFilterText(filter) : null; + const remoteQuery = this.#buildRemoteQuery(filter); + const fallbackSystem = options?.fallbackSystem || null; + const preferredLanguageCodes = this.#normalizeLanguageCodes(options?.languageCodes); + const effectiveLanguageCodes = preferredLanguageCodes.length > 0 ? preferredLanguageCodes : ['en']; + + if (count <= 0) { + return { contains: [], total: 0 }; + } + + const hasFilter = !!filter; + const limit = hasFilter + ? Math.min(FILTERED_CONCEPT_PAGE_SIZE, CONCEPT_PAGE_SIZE) + : CONCEPT_PAGE_SIZE; + let page = Math.floor(Math.max(0, offset) / limit) + 1; + let skip = Math.max(0, offset) % limit; + let remaining = count; + const contains = []; + let reportedTotal = null; + + while (remaining > 0) { + const pageData = await this.#fetchConceptPage(meta.conceptsUrl, page, limit, remoteQuery); + const pageItems = Array.isArray(pageData?.items) ? pageData.items : []; + if ( + reportedTotal == null && + typeof pageData?.reportedTotal === 'number' && + Number.isFinite(pageData.reportedTotal) && + pageData.reportedTotal >= 0 + ) { + reportedTotal = pageData.reportedTotal; + } + + if (!pageItems || pageItems.length === 0) { + break; + } + + const slice = pageItems.slice(skip); + skip = 0; + + for (const concept of slice) { + if (remaining <= 0) { + break; + } + if (activeOnly && concept.retired === true) { + continue; + } + + const localizedNames = this.#extractLocalizedNames(concept, effectiveLanguageCodes); + const localizedDefinitions = this.#extractLocalizedDefinitions(concept, effectiveLanguageCodes); + + const display = localizedNames.display || concept.display_name || concept.display || concept.name || null; + const definition = localizedDefinitions.definition || concept.definition || concept.description || concept.concept_class || null; + const code = concept.code || concept.id || null; + const searchableText = [ + code, + display, + definition, + ...localizedNames.designation.map(d => d.value), + ...localizedDefinitions.definitions.map(d => d.value) + ].filter(Boolean).join(' '); + if (!this.#conceptMatchesFilter(searchableText, code, display, definition, filter, filterMatcher)) { + continue; + } + + if (!code) { + continue; + } + + const owner = concept.owner || meta.owner || null; + const source = concept.source || null; + const conceptCanonical = concept.source_canonical_url || concept.sourceCanonicalUrl || null; + const system = conceptCanonical || (owner && source + ? await this.#getSourceCanonicalUrl(owner, source) + : fallbackSystem); + + contains.push({ + system: system || fallbackSystem, + code, + display, + definition: definition || undefined, + designation: localizedNames.designation, + definitions: localizedDefinitions.definitions, + inactive: concept.retired === true ? true : undefined + }); + remaining -= 1; + } + + if (pageItems.length < limit) { + break; + } + + page += 1; + } + + return { contains, total: contains.length, reportedTotal }; + } + + async #resolveValueSetByCanonical(url, version) { + const canonicalUrl = typeof url === 'string' ? url.trim() : ''; + if (!canonicalUrl) { + return null; + } + + const collection = await this.#findCollectionByCanonical(canonicalUrl, version); + if (!collection) { + return null; + } + + const valueSet = this.#toValueSet(collection); + if (!valueSet) { + return null; + } + + this.#indexValueSet(valueSet); + return valueSet; + } + + #valueSetBaseKey(vs) { + if (!vs || !vs.url) { + return null; + } + return `${vs.url}|${vs.version || ''}`; + } + + #expansionParamsKey(params) { + if (!params || typeof params !== 'object') { + return 'default'; + } + + try { + const normalized = Object.keys(params) + .sort() + .reduce((acc, key) => { + if (key === 'tx-resource' || key === 'valueSet') { + return acc; + } + acc[key] = params[key]; + return acc; + }, {}); + + const json = JSON.stringify(normalized); + if (!json || json === '{}') { + return 'default'; + } + return crypto.createHash('sha256').update(json).digest('hex').substring(0, 16); + } catch (error) { + return 'default'; + } + } + + #expansionCacheKey(vs, paramsKey) { + const base = this.#valueSetBaseKey(vs); + if (!base) { + return null; + } + return `${base}|${paramsKey || 'default'}`; + } + + #invalidateExpansionCache(vs) { + const base = this.#valueSetBaseKey(vs); + if (!base) { + return; + } + + for (const key of this.backgroundExpansionCache.keys()) { + if (key.startsWith(`${base}|`)) { + this.backgroundExpansionCache.delete(key); + } + } + } + + #applyCachedExpansion(vs, paramsKey) { + if (!vs || !vs.jsonObj) { + return; + } + + const cacheKey = this.#expansionCacheKey(vs, paramsKey); + if (!cacheKey) { + return; + } + + const cached = this.backgroundExpansionCache.get(cacheKey); + if (!cached || !cached.expansion) { + return; + } + + if (!this.#isCachedExpansionValid(vs, cached)) { + this.backgroundExpansionCache.delete(cacheKey); + if (vs.jsonObj.expansion) { + delete vs.jsonObj.expansion; + } + console.log(`[OCL-ValueSet] Cached ValueSet expansion invalidated: ${cacheKey}`); + return; + } + + if (vs.jsonObj.expansion) { + return; + } + + vs.jsonObj.expansion = structuredClone(cached.expansion); + console.log(`[OCL-ValueSet] ValueSet expansion restored from cache: ${cacheKey}`); + } + + #scheduleBackgroundExpansion(vs, options = {}) { + if (!vs || !vs.jsonObj) { + return; + } + + const paramsKey = this.#expansionParamsKey(options.params || null); + const cacheKey = this.#expansionCacheKey(vs, paramsKey); + if (!cacheKey) { + return; + } + + const cached = this.backgroundExpansionCache.get(cacheKey); + if (cached && !this.#isCachedExpansionValid(vs, cached)) { + this.backgroundExpansionCache.delete(cacheKey); + if (vs.jsonObj.expansion) { + delete vs.jsonObj.expansion; + } + console.log(`[OCL-ValueSet] Cached ValueSet expansion invalidated: ${cacheKey}`); + } + + if (vs.jsonObj.expansion) { + return; + } + + const cacheFilePath = getCacheFilePath(CACHE_VS_DIR, vs.url, vs.version || null, paramsKey); + const cacheAgeFromFileMs = getColdCacheAgeMs(cacheFilePath); + const persistedCache = this.backgroundExpansionCache.get(cacheKey); + const cacheAgeFromMetadataMs = Number.isFinite(persistedCache?.createdAt) + ? Math.max(0, Date.now() - persistedCache.createdAt) + : null; + + // Treat cache as fresh when either file mtime or persisted timestamp is recent. + const freshnessCandidates = [cacheAgeFromFileMs, cacheAgeFromMetadataMs].filter(age => age != null); + const freshestCacheAgeMs = freshnessCandidates.length > 0 ? Math.min(...freshnessCandidates) : null; + if (freshestCacheAgeMs != null && freshestCacheAgeMs <= COLD_CACHE_FRESHNESS_MS) { + const freshnessSource = cacheAgeFromFileMs != null && cacheAgeFromMetadataMs != null + ? 'file+metadata' + : cacheAgeFromFileMs != null + ? 'file' + : 'metadata'; + console.log(`[OCL-ValueSet] Skipping warm-up for ValueSet ${vs.url} (cold cache age: ${formatCacheAgeMinutes(freshestCacheAgeMs)})`); + console.log(`[OCL-ValueSet] ValueSet cold cache is fresh, not enqueueing warm-up job (${cacheKey}, source=${freshnessSource})`); + return; + } + + const jobKey = `vs:${cacheKey}`; + if (OCLBackgroundJobQueue.isQueuedOrRunning(jobKey)) { + console.log(`[OCL-ValueSet] ValueSet expansion already queued or running: ${cacheKey}`); + return; + } + + let queuedJobSize = null; + const warmupAgeText = freshestCacheAgeMs != null + ? formatCacheAgeMinutes(freshestCacheAgeMs) + : 'no cold cache'; + console.log(`[OCL-ValueSet] Enqueueing warm-up for ValueSet ${vs.url} (cold cache age: ${warmupAgeText})`); + console.log(`[OCL-ValueSet] ValueSet expansion enqueued: ${cacheKey}`); + OCLBackgroundJobQueue.enqueue( + jobKey, + 'ValueSet expansion', + async () => { + await this.#runBackgroundExpansion(vs, cacheKey, paramsKey, queuedJobSize); + }, + { + jobId: vs.url || cacheKey, + getProgress: () => this.#backgroundExpansionProgressSnapshot(cacheKey), + resolveJobSize: async () => { + const meta = this.#getCollectionMeta(vs); + queuedJobSize = await this.#fetchConceptCountFromHeaders(meta?.conceptsUrl || null); + return queuedJobSize; + } + } + ); + } + + async #runBackgroundExpansion(vs, cacheKey, paramsKey = 'default', knownConceptCount = null) { + console.log(`[OCL-ValueSet] ValueSet expansion started: ${cacheKey}`); + const progressState = { processed: 0, total: null }; + this.backgroundExpansionProgress.set(cacheKey, progressState); + try { + await this.#ensureComposeIncludes(vs); + + const meta = this.#getCollectionMeta(vs); + const resolvedTotal = Number.isFinite(knownConceptCount) && knownConceptCount >= 0 + ? knownConceptCount + : await this.#fetchConceptCountFromHeaders(meta?.conceptsUrl || null); + progressState.total = resolvedTotal; + const sources = meta ? await this.#fetchCollectionSources(meta) : []; + for (const source of sources || []) { + OCLSourceCodeSystemFactory.scheduleBackgroundLoadByKey( + source.system, + source.version || null, + 'valueset-expansion' + ); + } + + const expansion = await this.#buildBackgroundExpansion(vs, progressState); + if (!expansion) { + return; + } + + progressState.processed = expansion.total || progressState.processed; + if (typeof progressState.total !== 'number' || !Number.isFinite(progressState.total) || progressState.total <= 0) { + progressState.total = expansion.total || 0; + } + + const metadataSignature = this.#valueSetMetadataSignature(vs); + const dependencyChecksums = this.#valueSetDependencyChecksums(vs); + + // Compute custom fingerprint and compare with cold cache + const newFingerprint = computeValueSetExpansionFingerprint(expansion); + const oldFingerprint = this.valueSetFingerprints.get(cacheKey); + + if (oldFingerprint && newFingerprint === oldFingerprint) { + console.log(`[OCL-ValueSet] ValueSet expansion fingerprint unchanged: ${cacheKey} (fingerprint=${newFingerprint?.substring(0, 8)})`); + } else { + if (oldFingerprint) { + console.log(`[OCL-ValueSet] ValueSet expansion fingerprint changed: ${cacheKey} (${oldFingerprint?.substring(0, 8)} -> ${newFingerprint?.substring(0, 8)})`); + console.log(`[OCL-ValueSet] Replacing cold cache with new hot cache: ${cacheKey}`); + } else { + console.log(`[OCL-ValueSet] Computed fingerprint for ValueSet expansion: ${cacheKey} (fingerprint=${newFingerprint?.substring(0, 8)})`); + } + + // Save to cold cache + const savedFingerprint = await this.#saveColdCacheForValueSet(vs, expansion, metadataSignature, dependencyChecksums, paramsKey); + if (savedFingerprint) { + this.valueSetFingerprints.set(cacheKey, savedFingerprint); + } + } + + this.backgroundExpansionCache.set(cacheKey, { + expansion, + metadataSignature, + dependencyChecksums, + createdAt: Date.now() + }); + // Keep expansions in provider-managed cache only. + // Inline expansion on ValueSet bypasses $expand filtering in worker pipeline. + + console.log(`[OCL-ValueSet] ValueSet expansion completed and cached: ${cacheKey}`); + console.log(`[OCL-ValueSet] ValueSet now available in cache: ${cacheKey}`); + } catch (error) { + console.error(`[OCL-ValueSet] ValueSet background expansion failed: ${cacheKey}: ${error.message}`); + } finally { + this.backgroundExpansionProgress.delete(cacheKey); + } + } + + async #buildBackgroundExpansion(vs, progressState = null) { + const meta = this.#getCollectionMeta(vs); + if (!meta || !meta.conceptsUrl) { + return null; + } + + const contains = []; + let offset = 0; + + // Pull all concepts in fixed-size pages until exhausted. + // eslint-disable-next-line no-constant-condition + while (true) { + const batch = await this.#fetchCollectionConcepts(meta, { + count: CONCEPT_PAGE_SIZE, + offset, + activeOnly: false, + filter: null, + languageCodes: [] + }); + + const entries = Array.isArray(batch?.contains) ? batch.contains : []; + if (entries.length === 0) { + break; + } + + for (const entry of entries) { + if (!entry?.system || !entry?.code) { + continue; + } + + const out = { + system: entry.system, + code: entry.code + }; + if (entry.display) { + out.display = entry.display; + } + if (entry.definition) { + out.definition = entry.definition; + } + if (entry.inactive === true) { + out.inactive = true; + } + if (Array.isArray(entry.designation) && entry.designation.length > 0) { + out.designation = entry.designation + .filter(d => d && d.value) + .map(d => ({ + language: d.language, + value: d.value + })); + } + contains.push(out); + } + + if (progressState) { + progressState.processed = contains.length; + } + + if (entries.length < CONCEPT_PAGE_SIZE) { + break; + } + offset += entries.length; + } + + return { + timestamp: new Date().toISOString(), + identifier: `urn:uuid:${crypto.randomUUID()}`, + total: contains.length, + contains + }; + } + + async #findCollectionByCanonical(canonicalUrl, version) { + const lookupKey = `${this.org || '*'}|${canonicalUrl}|${version || ''}`; + if (this.collectionByCanonicalCache.has(lookupKey)) { + return this.collectionByCanonicalCache.get(lookupKey); + } + if (this.pendingCollectionByCanonicalRequests.has(lookupKey)) { + return this.pendingCollectionByCanonicalRequests.get(lookupKey); + } + + const token = this.#canonicalToken(canonicalUrl); + if (!token) { + return null; + } + + const pending = (async () => { + const organizations = await this.#fetchOrganizationIds(); + const endpoints = organizations.length > 0 + ? organizations.map(orgId => `/orgs/${encodeURIComponent(orgId)}/collections/`) + : ['/collections/']; + + const exactMatches = []; + for (const endpoint of endpoints) { + const response = await this.httpClient.get(endpoint, { + params: { + q: token, + page: 1, + limit: PAGE_SIZE + } + }); + + const payload = response.data; + const items = Array.isArray(payload) + ? payload + : Array.isArray(payload?.results) + ? payload.results + : Array.isArray(payload?.items) + ? payload.items + : Array.isArray(payload?.data) + ? payload.data + : []; + + for (const item of items) { + const itemCanonical = item?.canonical_url || item?.canonicalUrl || item?.url || null; + if (itemCanonical === canonicalUrl) { + exactMatches.push(item); + } + } + } + + let match = null; + if (exactMatches.length > 0) { + if (!version) { + match = exactMatches[0]; + } else { + const exactVersion = exactMatches.find(item => (item.version || null) === version); + if (exactVersion) { + match = exactVersion; + } else if (VersionUtilities.isSemVer(version)) { + const majorMinor = VersionUtilities.getMajMin(version); + if (majorMinor) { + const majorMinorMatch = exactMatches.find(item => (item.version || null) === majorMinor); + if (majorMinorMatch) { + match = majorMinorMatch; + } + } + } + } + } + + this.collectionByCanonicalCache.set(lookupKey, match); + return match; + })(); + + this.pendingCollectionByCanonicalRequests.set(lookupKey, pending); + try { + return await pending; + } finally { + this.pendingCollectionByCanonicalRequests.delete(lookupKey); + } + } + + #valueSetMetadataSignature(vs) { + const meta = this.#getCollectionMeta(vs); + const payload = { + url: vs?.url || null, + version: vs?.version || null, + lastUpdated: vs?.jsonObj?.meta?.lastUpdated || null, + collectionId: meta?.collectionId || null, + conceptsUrl: meta?.conceptsUrl || null, + expansionUrl: meta?.expansionUrl || null, + preferredSource: meta?.preferredSource || null + }; + return JSON.stringify(payload); + } + + #valueSetDependencyChecksums(vs) { + const include = Array.isArray(vs?.jsonObj?.compose?.include) ? vs.jsonObj.compose.include : []; + const checksums = {}; + for (const item of include) { + const system = normalizeCanonicalSystem(item?.system || null); + if (!system) { + continue; + } + const version = item?.version || null; + const key = `${system}|${version || ''}`; + checksums[key] = OCLSourceCodeSystemFactory.checksumForResource(system, version); + } + return checksums; + } + + #isCachedExpansionValid(vs, cached) { + if (!cached || typeof cached !== 'object') { + return false; + } + + if (cached.metadataSignature !== this.#valueSetMetadataSignature(vs)) { + return false; + } + + const currentDeps = this.#valueSetDependencyChecksums(vs); + const cachedDeps = cached.dependencyChecksums || {}; + const currentKeys = Object.keys(currentDeps).sort(); + const cachedKeys = Object.keys(cachedDeps).sort(); + + if (currentKeys.length !== cachedKeys.length) { + return false; + } + + for (let i = 0; i < currentKeys.length; i++) { + if (currentKeys[i] !== cachedKeys[i]) { + return false; + } + if ((currentDeps[currentKeys[i]] || null) !== (cachedDeps[cachedKeys[i]] || null)) { + return false; + } + } + + return true; + } + + async #fetchCollectionsForDiscovery() { + const organizations = await this.#fetchOrganizationIds(); + if (organizations.length === 0) { + // Fallback for OCL instances that expose global listing but not org listing. + return await this.#fetchAllPages('/collections/'); + } + + const allCollections = []; + const seen = new Set(); + + for (const orgId of organizations) { + const endpoint = `/orgs/${encodeURIComponent(orgId)}/collections/`; + const collections = await this.#fetchAllPages(endpoint); + for (const collection of collections) { + if (!collection || typeof collection !== 'object') { + continue; + } + const key = this.#collectionIdentity(collection); + if (seen.has(key)) { + continue; + } + seen.add(key); + allCollections.push(collection); + } + } + + return allCollections; + } + + async #fetchOrganizationIds() { + const endpoint = '/orgs/'; + console.log(`[OCL-ValueSet] Loading organizations from: ${this.baseUrl}${endpoint}`); + const orgs = await this.#fetchAllPages(endpoint); + + const ids = []; + const seen = new Set(); + for (const org of orgs || []) { + if (!org || typeof org !== 'object') { + continue; + } + + const id = org.id || org.mnemonic || org.short_code || org.shortCode || org.name || null; + if (!id) { + continue; + } + + const normalized = String(id).trim(); + if (!normalized || seen.has(normalized)) { + continue; + } + + seen.add(normalized); + ids.push(normalized); + } + + if (ids.length === 0 && this.org) { + ids.push(this.org); + } + + return ids; + } + + #collectionIdentity(collection) { + if (!collection || typeof collection !== 'object') { + return '__invalid__'; + } + + const owner = collection.owner || ''; + const canonical = collection.canonical_url || collection.canonicalUrl || ''; + const id = collection.id || collection.short_code || collection.shortCode || collection.name || ''; + return `${owner}|${canonical}|${id}`; + } + + #canonicalToken(canonicalUrl) { + if (!canonicalUrl || typeof canonicalUrl !== 'string') { + return null; + } + + const trimmed = canonicalUrl.trim(); + if (!trimmed) { + return null; + } + + const parts = trimmed.replace(/\/+$/, '').split('/').filter(Boolean); + if (parts.length === 0) { + return null; + } + + return parts[parts.length - 1]; + } + + async #fetchConceptPage(conceptsUrl, page, limit, filter = null) { + try { + const cacheKey = `${conceptsUrl}|p=${page}|l=${limit}|q=${filter || ''}|verbose=1`; + if (this.collectionConceptPageCache.has(cacheKey)) { + const cached = this.collectionConceptPageCache.get(cacheKey); + const items = Array.isArray(cached) + ? cached + : Array.isArray(cached?.items) + ? cached.items + : []; + return { + items, + reportedTotal: this.#extractTotalFromPayload(cached?.payload || null) + }; + } + if (this.pendingCollectionConceptPageRequests.has(cacheKey)) { + return this.pendingCollectionConceptPageRequests.get(cacheKey); + } + + const pending = (async () => { + const params = { page, limit, verbose: true }; + if (filter) { + params.q = filter; + } + const response = await this.httpClient.get(conceptsUrl, { params }); + const payload = response.data; + const items = Array.isArray(payload) + ? payload + : Array.isArray(payload?.results) + ? payload.results + : Array.isArray(payload?.items) + ? payload.items + : Array.isArray(payload?.data) + ? payload.data + : []; + + this.collectionConceptPageCache.set(cacheKey, { items, payload }); + return { + items, + reportedTotal: this.#extractTotalFromPayload(payload) + }; + })(); + + this.pendingCollectionConceptPageRequests.set(cacheKey, pending); + try { + return await pending; + } finally { + this.pendingCollectionConceptPageRequests.delete(cacheKey); + } + } catch (error) { + console.error(`[OCL-ValueSet] #fetchConceptPage ERROR: ${error.message}`); + throw error; + } + } + + #buildRemoteQuery(filter) { + if (!filter || typeof filter !== 'string') { + return null; + } + + const tokens = filter + .split(/\s+or\s+|\||&|\s+/i) + .map(t => t.trim()) + .filter(Boolean) + .map(t => (t.startsWith('-') || t.startsWith('!')) ? t.substring(1) : t) + .map(t => t.replace(/[%*?]/g, '')) + .map(t => t.replace(/[^\p{L}\p{N}]+/gu, '')) + .filter(t => t.length >= 3); + + if (tokens.length === 0) { + return null; + } + + tokens.sort((a, b) => b.length - a.length); + return tokens[0]; + } + + #normalizeLanguageCodes(languageCodes) { + if (!Array.isArray(languageCodes)) { + return []; + } + + const normalized = []; + for (const code of languageCodes) { + if (!code || typeof code !== 'string') { + continue; + } + normalized.push(code.toLowerCase()); + } + return normalized; + } + + #backgroundExpansionProgressSnapshot(cacheKey) { + const progress = this.backgroundExpansionProgress.get(cacheKey); + if (!progress) { + return null; + } + + const processed = progress.processed; + const total = progress.total; + if ( + typeof processed === 'number' && + Number.isFinite(processed) && + typeof total === 'number' && + Number.isFinite(total) && + total > 0 + ) { + return { processed, total }; + } + + return null; + } + + #extractTotalFromPayload(payload) { + if (!payload || typeof payload !== 'object' || Array.isArray(payload)) { + return null; + } + + const candidates = [ + payload.total, + payload.total_count, + payload.totalCount, + payload.num_found, + payload.numFound, + payload.count + ]; + + for (const candidate of candidates) { + if (typeof candidate === 'number' && Number.isFinite(candidate) && candidate >= 0) { + return candidate; + } + } + + return null; + } + + async #fetchConceptCountFromHeaders(conceptsUrl) { + if (!conceptsUrl) { + return null; + } + + try { + const response = await this.httpClient.get(conceptsUrl, { + params: { + limit: 1 + } + }); + return this.#extractNumFoundFromHeaders(response?.headers); + } catch (error) { + return null; + } + } + + #extractNumFoundFromHeaders(headers) { + if (!headers || typeof headers !== 'object') { + return null; + } + + const raw = headers.num_found ?? headers['num-found'] ?? headers.Num_Found ?? null; + const parsed = typeof raw === 'number' ? raw : Number.parseInt(raw, 10); + if (!Number.isFinite(parsed) || parsed < 0) { + return null; + } + return parsed; + } + + #languageRank(languageCode, preferredLanguageCodes) { + if (!languageCode) { + return 1000; + } + + const normalized = String(languageCode).toLowerCase(); + for (let i = 0; i < preferredLanguageCodes.length; i++) { + const preferred = preferredLanguageCodes[i]; + if (normalized === preferred || normalized.startsWith(`${preferred}-`) || preferred.startsWith(`${normalized}-`)) { + return i; + } + } + + if (normalized === 'en' || normalized.startsWith('en-')) { + return preferredLanguageCodes.length; + } + + return preferredLanguageCodes.length + 1; + } + + #extractLocalizedNames(concept, preferredLanguageCodes) { + const names = Array.isArray(concept?.names) ? concept.names : []; + const unique = new Map(); + + for (const item of names) { + const value = item?.name; + if (!value || typeof value !== 'string') { + continue; + } + + const language = typeof item?.locale === 'string' && item.locale.trim() ? item.locale.trim().toLowerCase() : null; + const key = `${language || ''}|${value}`; + if (unique.has(key)) { + continue; + } + + unique.set(key, { + language: language || undefined, + value, + localePreferred: item?.locale_preferred === true, + nameType: item?.name_type || '' + }); + } + + const designation = Array.from(unique.values()) + .sort((a, b) => { + const rankDiff = this.#languageRank(a.language, preferredLanguageCodes) - this.#languageRank(b.language, preferredLanguageCodes); + if (rankDiff !== 0) { + return rankDiff; + } + if (a.localePreferred !== b.localePreferred) { + return a.localePreferred ? -1 : 1; + } + const aFs = /fully\s*-?\s*specified/i.test(a.nameType); + const bFs = /fully\s*-?\s*specified/i.test(b.nameType); + if (aFs !== bFs) { + return aFs ? -1 : 1; + } + return a.value.localeCompare(b.value); + }) + .map(({ language, value }) => ({ + language, + value + })); + + return { + display: designation.length > 0 ? designation[0].value : null, + designation + }; + } + + #extractLocalizedDefinitions(concept, preferredLanguageCodes) { + const descriptions = Array.isArray(concept?.descriptions) ? concept.descriptions : []; + const unique = new Map(); + + for (const item of descriptions) { + const value = item?.description; + if (!value || typeof value !== 'string') { + continue; + } + + const language = typeof item?.locale === 'string' && item.locale.trim() ? item.locale.trim().toLowerCase() : null; + const key = `${language || ''}|${value}`; + if (unique.has(key)) { + continue; + } + + unique.set(key, { + language: language || undefined, + value, + localePreferred: item?.locale_preferred === true, + descriptionType: item?.description_type || '' + }); + } + + const definitions = Array.from(unique.values()) + .sort((a, b) => { + const rankDiff = this.#languageRank(a.language, preferredLanguageCodes) - this.#languageRank(b.language, preferredLanguageCodes); + if (rankDiff !== 0) { + return rankDiff; + } + if (a.localePreferred !== b.localePreferred) { + return a.localePreferred ? -1 : 1; + } + const aDef = /definition/i.test(a.descriptionType); + const bDef = /definition/i.test(b.descriptionType); + if (aDef !== bDef) { + return aDef ? -1 : 1; + } + return a.value.localeCompare(b.value); + }) + .map(({ language, value }) => ({ + language, + value + })); + + return { + definition: definitions.length > 0 ? definitions[0].value : null, + definitions + }; + } + + async #fetchCollectionSourceKeys(conceptsUrl, defaultOwner) { + const keys = new Map(); + let page = 1; + + // eslint-disable-next-line no-constant-condition + while (true) { + const response = await this.httpClient.get(conceptsUrl, { params: { page, limit: CONCEPT_PAGE_SIZE } }); + const payload = response.data; + + let items = []; + if (Array.isArray(payload)) { + items = payload; + } else if (payload && typeof payload === 'object') { + items = Array.isArray(payload.results) + ? payload.results + : Array.isArray(payload.items) + ? payload.items + : Array.isArray(payload.data) + ? payload.data + : []; + } + + if (!items || items.length === 0) { + break; + } + + for (const concept of items) { + const owner = concept.owner || defaultOwner || null; + const source = concept.source || null; + if (owner && source) { + keys.set(`${owner}|${source}`, { owner, source }); + } + } + + if (items.length < CONCEPT_PAGE_SIZE) { + break; + } + + page += 1; + } + + return Array.from(keys.values()); + } + + async #getSourceCanonicalUrl(owner, source) { + const key = `${owner}|${source}`; + if (this.sourceCanonicalCache.has(key)) { + return this.sourceCanonicalCache.get(key); + } + if (this.pendingSourceCanonicalRequests.has(key)) { + return this.pendingSourceCanonicalRequests.get(key); + } + + const path = `/orgs/${encodeURIComponent(owner)}/sources/${encodeURIComponent(source)}/`; + const pending = (async () => { + try { + const response = await this.httpClient.get(path); + const data = response.data || {}; + const canonicalUrl = data.canonical_url || data.canonicalUrl || data.url || source; + this.sourceCanonicalCache.set(key, canonicalUrl); + return canonicalUrl; + } catch (error) { + this.sourceCanonicalCache.set(key, source); + return source; + } + })(); + + this.pendingSourceCanonicalRequests.set(key, pending); + try { + return await pending; + } finally { + this.pendingSourceCanonicalRequests.delete(key); + } + } + + #matches(json, params) { + for (const [name, value] of Object.entries(params)) { + if (!value) { + continue; + } + + switch (name) { + case 'url': + if ((json.url || '').toLowerCase() !== value) { + return false; + } + break; + case 'system': + if (!json.compose?.include?.some(i => (i.system || '').toLowerCase().includes(value))) { + return false; + } + break; + case 'identifier': { + const identifiers = Array.isArray(json.identifier) ? json.identifier : (json.identifier ? [json.identifier] : []); + const match = identifiers.some(i => (i.system || '').toLowerCase().includes(value) || (i.value || '').toLowerCase().includes(value)); + if (!match) { + return false; + } + break; + } + default: { + const field = json[name]; + if (field == null || !String(field).toLowerCase().includes(value)) { + return false; + } + break; + } + } + } + return true; + } + + async #fetchAllPages(path) { + const results = []; + let page = 1; + + // eslint-disable-next-line no-constant-condition + while (true) { + const response = await this.httpClient.get(path, { params: { page, limit: PAGE_SIZE } }); + const payload = response.data; + + let items = []; + if (Array.isArray(payload)) { + items = payload; + } else if (payload && typeof payload === 'object') { + items = Array.isArray(payload.results) + ? payload.results + : Array.isArray(payload.items) + ? payload.items + : Array.isArray(payload.data) + ? payload.data + : []; + } + + if (!items || items.length === 0) { + break; + } + + results.push(...items); + + if (items.length < PAGE_SIZE) { + break; + } + + page += 1; + } + + return results; + } + + #toIsoDate(value) { + if (!value) { + return null; + } + const date = new Date(value); + if (Number.isNaN(date.getTime())) { + return null; + } + return date.toISOString(); + } + + #normalizeFilter(filter) { + if (typeof filter !== 'string') { + return null; + } + const normalized = filter.trim().toLowerCase(); + return normalized.length > 0 ? normalized : null; + } + + #conceptMatchesFilter(searchableText, code, display, definition, filter, filterMatcher) { + if (!filter) { + return true; + } + + const codeText = code ? String(code).toLowerCase() : ''; + const displayText = display ? String(display).toLowerCase() : ''; + const definitionText = definition ? String(definition).toLowerCase() : ''; + + // FHIR allows terminology-server-defined behavior; guarantee baseline contains matching. + if (codeText.includes(filter) || displayText.includes(filter) || definitionText.includes(filter)) { + return true; + } + + // Preserve token/prefix behavior already provided by SearchFilterText. + return !!(filterMatcher && filterMatcher.passes(searchableText)); + } +} + +module.exports = { + OCLValueSetProvider +}; \ No newline at end of file diff --git a/tx/ocl/vs-ocl.js b/tx/ocl/vs-ocl.js index bfd6349f..3287b644 100644 --- a/tx/ocl/vs-ocl.js +++ b/tx/ocl/vs-ocl.js @@ -1,105 +1,2 @@ -/** - * Abstract base class for value set providers - * Defines the interface that all value set providers must implement - */ -// eslint-disable-next-line no-unused-vars -class OCLValueSetProvider { - /** - * {int} Unique number assigned to this provider - */ - spaceId; +module.exports = require('./vs-ocl.cjs'); - /** - * ensure that the ids on the value sets are unique, if they are - * in the global namespace - * - * @param {Set} ids - */ - // eslint-disable-next-line no-unused-vars - assignIds(ids) { - throw new Error('assignIds must be implemented by AbstractValueSetProvider subclass'); - } - - /** - * Fetches a specific value set by URL and version - * @param {string} url - The URL/identifier of the value set - * @param {string} version - The version of the value set - * @returns {Promise} The requested value set - * @throws {Error} Must be implemented by subclasses - */ - // eslint-disable-next-line no-unused-vars - async fetchValueSet(url, version) { - throw new Error('fetchValueSet must be implemented by subclass'); - } - - /** - * Fetches a specific value set by id. ValueSet providers must enforce that value set ids are unique - * either globally (as enforced by assignIds) or in their space - * - * @param {string} id - The id of the value set - * @returns {Promise} The requested value set - * @throws {Error} Must be implemented by subclasses - */ - // eslint-disable-next-line no-unused-vars - async fetchValueSetById(id) { - throw new Error('fetchValueSetById must be implemented by subclass'); - } - - /** - * Searches for value sets based on provided criteria - * @param {Array<{name: string, value: string}>} searchParams - List of name/value pairs for search criteria - * @returns {Promise>} List of matching value sets - * @throws {Error} Must be implemented by subclasses - */ - // eslint-disable-next-line no-unused-vars - async searchValueSets(searchParams, elements = null) { - throw new Error('searchValueSets must be implemented by subclass'); - } - - /** - * - * @returns {number} total number of value sets - */ - vsCount() { - return 0; - } - - /** - * Validates search parameters - * @param {Array<{name: string, value: string}>} searchParams - Search parameters to validate - * @protected - */ - _validateSearchParams(searchParams) { - if (!Array.isArray(searchParams)) { - throw new Error('Search parameters must be an array'); - } - - for (const param of searchParams) { - if (!param || typeof param !== 'object') { - throw new Error('Each search parameter must be an object'); - } - if (typeof param.name !== 'string' || typeof param.value !== 'string') { - throw new Error('Search parameter must have string name and value properties'); - } - } - } - - /** - * Validates URL and version parameters - * @param {string} url - URL to validate - * @param {string} version - Version to validate - * @protected - */ - _validateFetchParams(url, version) { - if (typeof url !== 'string' || !url.trim()) { - throw new Error('URL must be a non-empty string'); - } - if (version != null && typeof version !== 'string') { - throw new Error('Version must be a string'); - } - } -} - -module.exports = { - AbstractValueSetProvider -}; \ No newline at end of file diff --git a/tx/provider.js b/tx/provider.js index c31d420a..deca4981 100644 --- a/tx/provider.js +++ b/tx/provider.js @@ -23,6 +23,7 @@ const {PackageConceptMapProvider} = require("./cm/cm-package"); class Provider { i18n; fhirVersion; + context; /** * {Map} A list of code system factories that contains all the preloaded native code systems @@ -34,6 +35,11 @@ class Provider { */ codeSystems; + /** + * {List} code system providers, for maintaing the code system list + */ + codeSystemProviders + /** * {List} A list of value set providers that know how to provide value sets by request */ @@ -427,6 +433,47 @@ class Provider { return false; } + updateCodeSystemList() { + for (let csp of this.codeSystemProviders) { + let changes = csp.getCodeSystemChanges(this.fhirVersion, this.context); + if (changes) { + for (let cs of changes.added || []) { + this.addCodeSystem(cs); + } + for (let cs of changes.changed || []) { + this.addCodeSystem(cs); + } + for (let cs of changes.deleted || []) { + this.deleteCodeSystem(cs); + } + } + } + } + + addCodeSystem(cs) { + const existing = this.codeSystems.get(cs.url); + if (!existing || cs.isMoreRecent(existing)) { + this.codeSystems.set(cs.url, cs); + } + if (cs.version) { + this.codeSystems.set(cs.vurl, cs); + } + } + + deleteCodeSystem(cs) { + this.codeSystems.delete(cs.vurl); + this.codeSystems.delete(cs.url); + let existing = null; + for (let t of this.codeSystems.values()) { + if (!existing || t.isMoreRecent(existing)) { + existing = t; + } + } + if (existing) { + this.codeSystems.set(cs.url, cs); + } + } + } module.exports = { Provider }; diff --git a/tx/tx-html.js b/tx/tx-html.js index 2585d4df..6f115bb1 100644 --- a/tx/tx-html.js +++ b/tx/tx-html.js @@ -1246,7 +1246,7 @@ class TxHtmlRenderer { } buildSourceOptions(provider) { - let result = ''; + let result = ''; result += ``; for (let sp of provider.listValueSetSourceCodes()) { result += ``; diff --git a/tx/tx.js b/tx/tx.js index 84b4108c..241bbb1e 100644 --- a/tx/tx.js +++ b/tx/tx.js @@ -247,6 +247,14 @@ class TXModule { if (this.stats) { this.stats.addTask("Client Cache", "5 min"); } + this.timers.push(setInterval(async () => { + try { + await endpointInfo.provider.updateCodeSystemList(); + } catch (error) { + this.log.error(`Error updating CodeSystem list for ${endpointPath}: ${error.message}`); + } + }, 60 * 1000)); + this.log.info(`CodeSystem list update scheduled for ${endpointPath}`); this.timers.push(setInterval(() => { endpointInfo.resourceCache.prune(cacheTimeoutMs); }, pruneIntervalMs)); diff --git a/tx/workers/search.js b/tx/workers/search.js index 832741d5..6cd8579b 100644 --- a/tx/workers/search.js +++ b/tx/workers/search.js @@ -146,9 +146,7 @@ class SearchWorker extends TerminologyWorker { for (const [key, cs] of this.provider.codeSystems) { this.deadCheck('searchCodeSystems'); - if (cs.url == 'http://www.cms.gov/Medicare/Coding/HCPCSReleaseCodeSets') { - console.log("debug"); - } + if (key == cs.vurl) { const json = cs.jsonObj;