diff --git a/src/cli/app.js b/src/cli/app.js index 236ecc6e..6dc82903 100644 --- a/src/cli/app.js +++ b/src/cli/app.js @@ -99,7 +99,11 @@ async function mcodeApp(Client, fromDate, toDate, pathToConfig, pathToRunLogs, d const { notificationInfo } = config; if (notificationInfo) { const notificationErrors = zipErrors(totalExtractionErrors); - await sendEmailNotification(notificationInfo, notificationErrors, debug); + try { + await sendEmailNotification(notificationInfo, notificationErrors, debug); + } catch (e) { + logger.error(e.message); + } } // A run is successful and should be logged when both extraction finishes without fatal errors // and messages are posted without fatal errors @@ -115,8 +119,14 @@ async function mcodeApp(Client, fromDate, toDate, pathToConfig, pathToRunLogs, d const patientConfig = config.extractors.find((e) => e.type === 'CSVPatientExtractor'); if (patientConfig && ('constructorArgs' in patientConfig && 'mask' in patientConfig.constructorArgs)) { if (patientConfig.constructorArgs.mask.includes('mrn')) { - extractedData.forEach((bundle) => { - maskMRN(bundle); + extractedData.forEach((bundle, i) => { + // NOTE: This may fail to mask MRN-related properties on non-patient resources + // Need to investigate further. + try { + maskMRN(bundle); + } catch (e) { + logger.error(`Bundle ${i + 1}: ${e.message}`); + } }); } } diff --git a/src/extractors/CSVPatientExtractor.js b/src/extractors/CSVPatientExtractor.js index b1fb39fb..4ec9eba0 100644 --- a/src/extractors/CSVPatientExtractor.js +++ b/src/extractors/CSVPatientExtractor.js @@ -1,9 +1,11 @@ +const _ = require('lodash'); const { generateMcodeResources } = require('../templates'); const { BaseCSVExtractor } = require('./BaseCSVExtractor'); const { getEthnicityDisplay, getRaceCodesystem, getRaceDisplay, maskPatientData } = require('../helpers/patientUtils'); +const { getEmptyBundle } = require('../helpers/fhirUtils'); const logger = require('../helpers/logger'); const { CSVPatientSchema } = require('../helpers/schemas/csv'); @@ -55,6 +57,10 @@ class CSVPatientExtractor extends BaseCSVExtractor { async get({ mrn }) { // 1. Get all relevant data and do necessary post-processing const patientData = await this.getPatientData(mrn); + if (_.isEmpty(patientData)) { + logger.warn('No patient data found for this patient'); + return getEmptyBundle(); + } // 2. Format data for research study and research subject const packagedPatientData = joinAndReformatData(patientData); diff --git a/src/modules/CSVModule.js b/src/modules/CSVModule.js index 983223ee..28881c98 100644 --- a/src/modules/CSVModule.js +++ b/src/modules/CSVModule.js @@ -13,7 +13,10 @@ class CSVModule { // return all rows if key and value aren't provided if (!key && !value) return this.data; let result = this.data.filter((d) => d[key] === value); - if (result.length === 0) throw new ReferenceError(`CSV Record with provided key '${key}' and value was not found`); + if (result.length === 0) { + logger.warn(`CSV Record with provided key '${key}' and value was not found`); + return result; + } // If fromDate and toDate is provided, filter out all results that fall outside that timespan if (fromDate && moment(fromDate).isValid()) result = result.filter((r) => !(r.dateRecorded && moment(fromDate).isAfter(r.dateRecorded))); diff --git a/test/cli/fixtures/example-clinical-trial-info.csv b/test/cli/fixtures/example-clinical-trial-info.csv new file mode 100644 index 00000000..be74ba37 --- /dev/null +++ b/test/cli/fixtures/example-clinical-trial-info.csv @@ -0,0 +1,4 @@ +mrn,trialSubjectID,enrollmentStatus,trialResearchID,trialStatus,trialResearchSystem +123,subjectId-1,potential-candidate,researchId-1,approved,system-1 +456,subjectId-2,on-study-intervention,researchId-1,completed,system-2 +789,subjectId-3,on-study-observation,researchId-2,active, diff --git a/test/cli/mcodeExtraction.test.js b/test/cli/mcodeExtraction.test.js index 9bcdedf4..4cc33215 100644 --- a/test/cli/mcodeExtraction.test.js +++ b/test/cli/mcodeExtraction.test.js @@ -43,7 +43,7 @@ describe('mcodeExtraction', () => { expect(extractedData).toHaveLength(0); }); - it('should result in a successful extraction when non fatal errors are encountered in the client\'s get method', async () => { + it('should succeed in extraction when CSV files do not have data for all patients', async () => { const testConfig = { extractors: [ { @@ -69,10 +69,34 @@ describe('mcodeExtraction', () => { const { extractedData, successfulExtraction, totalExtractionErrors } = await extractDataForPatients(testPatientIds, testClient, testFromDate, testToDate); expect(successfulExtraction).toEqual(true); + // Should have data for 3 patients and 0 errors + expect(extractedData).toHaveLength(3); + const flatErrors = flattenErrorValues(totalExtractionErrors); + expect(flatErrors).toHaveLength(0); + }); + it('should result in a successful extraction when non fatal errors are encountered in the client\'s get method', async () => { + const testConfig = { + extractors: [ + // Should fail when this extractor is run without patient data in context + { + label: 'CTI', + type: 'CSVClinicalTrialInformationExtractor', + constructorArgs: { + filePath: path.join(__dirname, './fixtures/example-clinical-trial-info.csv'), + }, + }, + ], + }; + const testClient = new MCODEClient(testConfig); + await testClient.init(); + + const { extractedData, successfulExtraction, totalExtractionErrors } = await extractDataForPatients(testPatientIds, testClient, testFromDate, testToDate); + expect(successfulExtraction).toEqual(true); + // Should have three (empty) bundles for patients and an error for each patient when extracting CTI expect(extractedData).toHaveLength(3); const flatErrors = flattenErrorValues(totalExtractionErrors); - expect(flatErrors).toHaveLength(1); + expect(flatErrors).toHaveLength(3); }); }); }); diff --git a/test/modules/CSVModule.test.js b/test/modules/CSVModule.test.js index b0757fa3..477491cf 100644 --- a/test/modules/CSVModule.test.js +++ b/test/modules/CSVModule.test.js @@ -31,10 +31,7 @@ test('Returns data with recordedDate before specified to date', async () => { expect(data).toHaveLength(1); }); -test('Invalid MRN', async () => { - try { - await csvModule.get('mrn', INVALID_MRN); - } catch (e) { - expect(e).toEqual(ReferenceError('CSV Record with provided key \'mrn\' and value was not found')); - } +test('Should return an empty array when key-value pair does not exist', async () => { + const data = await csvModule.get('mrn', INVALID_MRN); + expect(data).toEqual([]); }); diff --git a/test/sample-client-data/cancer-disease-status-information.csv b/test/sample-client-data/cancer-disease-status-information.csv index b069ba01..fd9a011a 100644 --- a/test/sample-client-data/cancer-disease-status-information.csv +++ b/test/sample-client-data/cancer-disease-status-information.csv @@ -1,5 +1,4 @@ mrn,conditionId,diseaseStatusCode,diseaseStatusText,dateOfObservation,evidence,observationStatus,dateRecorded 123,conditionId-1,268910001,responding,2019-12-02,363679005|252416005,preliminary,2020-01-10 -456,conditionId-2,359746009,stable,2020-01-12,363679005,amended,2020-01-12 789,conditionId-2,709137006,not evaluated,2020-04-22,,final,2020-06-10 123,conditionId-1,not-asked,,2020-01-12,,, diff --git a/test/sample-client-data/patient-mrns.csv b/test/sample-client-data/patient-mrns.csv index e48fff3b..11cd708c 100644 --- a/test/sample-client-data/patient-mrns.csv +++ b/test/sample-client-data/patient-mrns.csv @@ -2,3 +2,4 @@ mrn 123 456 789 +1011