From 3bc7a388a335b89f792475f3e2c0ca02d6492709 Mon Sep 17 00:00:00 2001 From: Dylan Phelan Date: Tue, 13 Apr 2021 16:20:25 -0400 Subject: [PATCH 1/3] updates to CSVModule logic --- src/modules/CSVModule.js | 5 ++++- test/modules/CSVModule.test.js | 9 +++------ .../cancer-disease-status-information.csv | 1 - test/sample-client-data/patient-mrns.csv | 1 + 4 files changed, 8 insertions(+), 8 deletions(-) diff --git a/src/modules/CSVModule.js b/src/modules/CSVModule.js index 983223ee..28881c98 100644 --- a/src/modules/CSVModule.js +++ b/src/modules/CSVModule.js @@ -13,7 +13,10 @@ class CSVModule { // return all rows if key and value aren't provided if (!key && !value) return this.data; let result = this.data.filter((d) => d[key] === value); - if (result.length === 0) throw new ReferenceError(`CSV Record with provided key '${key}' and value was not found`); + if (result.length === 0) { + logger.warn(`CSV Record with provided key '${key}' and value was not found`); + return result; + } // If fromDate and toDate is provided, filter out all results that fall outside that timespan if (fromDate && moment(fromDate).isValid()) result = result.filter((r) => !(r.dateRecorded && moment(fromDate).isAfter(r.dateRecorded))); diff --git a/test/modules/CSVModule.test.js b/test/modules/CSVModule.test.js index b0757fa3..477491cf 100644 --- a/test/modules/CSVModule.test.js +++ b/test/modules/CSVModule.test.js @@ -31,10 +31,7 @@ test('Returns data with recordedDate before specified to date', async () => { expect(data).toHaveLength(1); }); -test('Invalid MRN', async () => { - try { - await csvModule.get('mrn', INVALID_MRN); - } catch (e) { - expect(e).toEqual(ReferenceError('CSV Record with provided key \'mrn\' and value was not found')); - } +test('Should return an empty array when key-value pair does not exist', async () => { + const data = await csvModule.get('mrn', INVALID_MRN); + expect(data).toEqual([]); }); diff --git a/test/sample-client-data/cancer-disease-status-information.csv b/test/sample-client-data/cancer-disease-status-information.csv index b069ba01..fd9a011a 100644 --- a/test/sample-client-data/cancer-disease-status-information.csv +++ b/test/sample-client-data/cancer-disease-status-information.csv @@ -1,5 +1,4 @@ mrn,conditionId,diseaseStatusCode,diseaseStatusText,dateOfObservation,evidence,observationStatus,dateRecorded 123,conditionId-1,268910001,responding,2019-12-02,363679005|252416005,preliminary,2020-01-10 -456,conditionId-2,359746009,stable,2020-01-12,363679005,amended,2020-01-12 789,conditionId-2,709137006,not evaluated,2020-04-22,,final,2020-06-10 123,conditionId-1,not-asked,,2020-01-12,,, diff --git a/test/sample-client-data/patient-mrns.csv b/test/sample-client-data/patient-mrns.csv index e48fff3b..11cd708c 100644 --- a/test/sample-client-data/patient-mrns.csv +++ b/test/sample-client-data/patient-mrns.csv @@ -2,3 +2,4 @@ mrn 123 456 789 +1011 From cecf525c8cabef734ff1a227d3ed18c25e93c4cd Mon Sep 17 00:00:00 2001 From: Dylan Phelan Date: Tue, 13 Apr 2021 16:28:01 -0400 Subject: [PATCH 2/3] updates to mcodeExtraction test --- .../fixtures/example-clinical-trial-info.csv | 4 +++ test/cli/mcodeExtraction.test.js | 28 +++++++++++++++++-- 2 files changed, 30 insertions(+), 2 deletions(-) create mode 100644 test/cli/fixtures/example-clinical-trial-info.csv diff --git a/test/cli/fixtures/example-clinical-trial-info.csv b/test/cli/fixtures/example-clinical-trial-info.csv new file mode 100644 index 00000000..be74ba37 --- /dev/null +++ b/test/cli/fixtures/example-clinical-trial-info.csv @@ -0,0 +1,4 @@ +mrn,trialSubjectID,enrollmentStatus,trialResearchID,trialStatus,trialResearchSystem +123,subjectId-1,potential-candidate,researchId-1,approved,system-1 +456,subjectId-2,on-study-intervention,researchId-1,completed,system-2 +789,subjectId-3,on-study-observation,researchId-2,active, diff --git a/test/cli/mcodeExtraction.test.js b/test/cli/mcodeExtraction.test.js index 9bcdedf4..4cc33215 100644 --- a/test/cli/mcodeExtraction.test.js +++ b/test/cli/mcodeExtraction.test.js @@ -43,7 +43,7 @@ describe('mcodeExtraction', () => { expect(extractedData).toHaveLength(0); }); - it('should result in a successful extraction when non fatal errors are encountered in the client\'s get method', async () => { + it('should succeed in extraction when CSV files do not have data for all patients', async () => { const testConfig = { extractors: [ { @@ -69,10 +69,34 @@ describe('mcodeExtraction', () => { const { extractedData, successfulExtraction, totalExtractionErrors } = await extractDataForPatients(testPatientIds, testClient, testFromDate, testToDate); expect(successfulExtraction).toEqual(true); + // Should have data for 3 patients and 0 errors + expect(extractedData).toHaveLength(3); + const flatErrors = flattenErrorValues(totalExtractionErrors); + expect(flatErrors).toHaveLength(0); + }); + it('should result in a successful extraction when non fatal errors are encountered in the client\'s get method', async () => { + const testConfig = { + extractors: [ + // Should fail when this extractor is run without patient data in context + { + label: 'CTI', + type: 'CSVClinicalTrialInformationExtractor', + constructorArgs: { + filePath: path.join(__dirname, './fixtures/example-clinical-trial-info.csv'), + }, + }, + ], + }; + const testClient = new MCODEClient(testConfig); + await testClient.init(); + + const { extractedData, successfulExtraction, totalExtractionErrors } = await extractDataForPatients(testPatientIds, testClient, testFromDate, testToDate); + expect(successfulExtraction).toEqual(true); + // Should have three (empty) bundles for patients and an error for each patient when extracting CTI expect(extractedData).toHaveLength(3); const flatErrors = flattenErrorValues(totalExtractionErrors); - expect(flatErrors).toHaveLength(1); + expect(flatErrors).toHaveLength(3); }); }); }); From 13f42b004f0adf3a13808289c87ffeabf8007dd7 Mon Sep 17 00:00:00 2001 From: Dylan Phelan Date: Thu, 15 Apr 2021 10:43:56 -0400 Subject: [PATCH 3/3] Better error handling for masked MRN's, failed email setup, and patient data cannot be found --- src/cli/app.js | 16 +++++++++++++--- src/extractors/CSVPatientExtractor.js | 6 ++++++ 2 files changed, 19 insertions(+), 3 deletions(-) diff --git a/src/cli/app.js b/src/cli/app.js index 236ecc6e..6dc82903 100644 --- a/src/cli/app.js +++ b/src/cli/app.js @@ -99,7 +99,11 @@ async function mcodeApp(Client, fromDate, toDate, pathToConfig, pathToRunLogs, d const { notificationInfo } = config; if (notificationInfo) { const notificationErrors = zipErrors(totalExtractionErrors); - await sendEmailNotification(notificationInfo, notificationErrors, debug); + try { + await sendEmailNotification(notificationInfo, notificationErrors, debug); + } catch (e) { + logger.error(e.message); + } } // A run is successful and should be logged when both extraction finishes without fatal errors // and messages are posted without fatal errors @@ -115,8 +119,14 @@ async function mcodeApp(Client, fromDate, toDate, pathToConfig, pathToRunLogs, d const patientConfig = config.extractors.find((e) => e.type === 'CSVPatientExtractor'); if (patientConfig && ('constructorArgs' in patientConfig && 'mask' in patientConfig.constructorArgs)) { if (patientConfig.constructorArgs.mask.includes('mrn')) { - extractedData.forEach((bundle) => { - maskMRN(bundle); + extractedData.forEach((bundle, i) => { + // NOTE: This may fail to mask MRN-related properties on non-patient resources + // Need to investigate further. + try { + maskMRN(bundle); + } catch (e) { + logger.error(`Bundle ${i + 1}: ${e.message}`); + } }); } } diff --git a/src/extractors/CSVPatientExtractor.js b/src/extractors/CSVPatientExtractor.js index b1fb39fb..4ec9eba0 100644 --- a/src/extractors/CSVPatientExtractor.js +++ b/src/extractors/CSVPatientExtractor.js @@ -1,9 +1,11 @@ +const _ = require('lodash'); const { generateMcodeResources } = require('../templates'); const { BaseCSVExtractor } = require('./BaseCSVExtractor'); const { getEthnicityDisplay, getRaceCodesystem, getRaceDisplay, maskPatientData } = require('../helpers/patientUtils'); +const { getEmptyBundle } = require('../helpers/fhirUtils'); const logger = require('../helpers/logger'); const { CSVPatientSchema } = require('../helpers/schemas/csv'); @@ -55,6 +57,10 @@ class CSVPatientExtractor extends BaseCSVExtractor { async get({ mrn }) { // 1. Get all relevant data and do necessary post-processing const patientData = await this.getPatientData(mrn); + if (_.isEmpty(patientData)) { + logger.warn('No patient data found for this patient'); + return getEmptyBundle(); + } // 2. Format data for research study and research subject const packagedPatientData = joinAndReformatData(patientData);