From 900b9bf0843c4a4dd310c6e8c67250680621610d Mon Sep 17 00:00:00 2001 From: Dylan Phelan Date: Thu, 16 Dec 2021 15:01:35 -0500 Subject: [PATCH 1/6] initial thoughts on the improving dataDirectory approach --- src/application/app.js | 5 +++-- src/helpers/appUtils.js | 14 ++++++++++++++ 2 files changed, 17 insertions(+), 2 deletions(-) diff --git a/src/application/app.js b/src/application/app.js index 5cf49533..3b0ec8cf 100644 --- a/src/application/app.js +++ b/src/application/app.js @@ -3,7 +3,7 @@ const logger = require('../helpers/logger'); const { RunInstanceLogger } = require('./tools/RunInstanceLogger'); const { sendEmailNotification, zipErrors } = require('./tools/emailNotifications'); const { extractDataForPatients } = require('./tools/mcodeExtraction'); -const { parsePatientIds } = require('../helpers/appUtils'); +const { buildPatientCSVPath, parsePatientIds } = require('../helpers/appUtils'); const { validateConfig } = require('../helpers/configUtils'); function checkInputAndConfig(config, fromDate, toDate) { @@ -30,7 +30,8 @@ async function mcodeApp(Client, fromDate, toDate, config, pathToRunLogs, debug, await mcodeClient.init(); // Parse CSV for list of patient mrns - const patientIds = parsePatientIds(config.patientIdCsvPath); + const patientCSVPath = buildPatientCSVPath(config); + const patientIds = parsePatientIds(patientCSVPath); // Get RunInstanceLogger for recording new runs and inferring dates from previous runs const runLogger = allEntries ? null : new RunInstanceLogger(pathToRunLogs); diff --git a/src/helpers/appUtils.js b/src/helpers/appUtils.js index 7ab38a6c..5024a7be 100644 --- a/src/helpers/appUtils.js +++ b/src/helpers/appUtils.js @@ -1,6 +1,7 @@ const fs = require('fs'); const path = require('path'); const { csvParse } = require('./csvParsingUtils'); +const logger = require('./logger'); /** * Parses a provided CSV with MRN column into string array of IDs @@ -22,6 +23,19 @@ function parsePatientIds(pathToCSV) { return patientIds; } +function buildPatientCSVPath(config) { + try { + const patientIdCsvPath = path.resolve(config.patientIdCsvPath); + return patientIdCsvPath; + } catch (e) { + if (config.commonExtractorArgs.dataDirectory) { + logger.error(`Could not resolve ${config.patientIdCsvPath}; even with a dataDirectory, the config.patientIdCsvPath variable needs to be a resolvable path to the patientID file on disk.`); + } + throw e; + } +} + module.exports = { + buildPatientCSVPath, parsePatientIds, }; From ee8fa508560723b814d336fa392463ab9dbc9721 Mon Sep 17 00:00:00 2001 From: Dylan Phelan Date: Fri, 17 Dec 2021 15:23:53 -0500 Subject: [PATCH 2/6] Improved approach thanks to JA --- src/application/app.js | 5 ++--- src/helpers/appUtils.js | 43 ++++++++++++++++++++--------------------- 2 files changed, 23 insertions(+), 25 deletions(-) diff --git a/src/application/app.js b/src/application/app.js index 3b0ec8cf..56c39364 100644 --- a/src/application/app.js +++ b/src/application/app.js @@ -3,7 +3,7 @@ const logger = require('../helpers/logger'); const { RunInstanceLogger } = require('./tools/RunInstanceLogger'); const { sendEmailNotification, zipErrors } = require('./tools/emailNotifications'); const { extractDataForPatients } = require('./tools/mcodeExtraction'); -const { buildPatientCSVPath, parsePatientIds } = require('../helpers/appUtils'); +const { parsePatientIds } = require('../helpers/appUtils'); const { validateConfig } = require('../helpers/configUtils'); function checkInputAndConfig(config, fromDate, toDate) { @@ -30,8 +30,7 @@ async function mcodeApp(Client, fromDate, toDate, config, pathToRunLogs, debug, await mcodeClient.init(); // Parse CSV for list of patient mrns - const patientCSVPath = buildPatientCSVPath(config); - const patientIds = parsePatientIds(patientCSVPath); + const patientIds = parsePatientIds(config); // Get RunInstanceLogger for recording new runs and inferring dates from previous runs const runLogger = allEntries ? null : new RunInstanceLogger(pathToRunLogs); diff --git a/src/helpers/appUtils.js b/src/helpers/appUtils.js index 5024a7be..55221c62 100644 --- a/src/helpers/appUtils.js +++ b/src/helpers/appUtils.js @@ -4,38 +4,37 @@ const { csvParse } = require('./csvParsingUtils'); const logger = require('./logger'); /** - * Parses a provided CSV with MRN column into string array of IDs + * Loads the patientIdCSV data from disk, with some helpful hints logged in case of failure * - * @param {string} pathToCSV filePath to the CSV content to be parsed to get IDs * @returns array of parsed IDs from the CSV */ -function parsePatientIds(pathToCSV) { - // Parse CSV for list of patient IDs - const patientIdsCsvPath = path.resolve(pathToCSV); - const patientIds = csvParse(fs.readFileSync(patientIdsCsvPath, 'utf8')).map((row) => { - if (!row.mrn) { - throw new Error(`${pathToCSV} has no "mrn" column`); - } - - return row.mrn; - }); - - return patientIds; -} - -function buildPatientCSVPath(config) { +function getPatientIdCSVData(patientIdCsvPath, commonExtractorArgs) { try { - const patientIdCsvPath = path.resolve(config.patientIdCsvPath); - return patientIdCsvPath; + const patientIdsCsvPath = path.resolve(patientIdCsvPath); + return fs.readFileSync(patientIdsCsvPath, 'utf8'); } catch (e) { - if (config.commonExtractorArgs.dataDirectory) { - logger.error(`Could not resolve ${config.patientIdCsvPath}; even with a dataDirectory, the config.patientIdCsvPath variable needs to be a resolvable path to the patientID file on disk.`); + if (commonExtractorArgs && commonExtractorArgs.dataDirectory) { + logger.error(`Could not resolve ${patientIdCsvPath}; even with a dataDirectory, the config.patientIdCsvPath variable needs to be a resolvable path to the patientID file on disk.`); } throw e; } } +/** + * Parses a provided CSV with MRN column into string array of IDs + * + * @returns array of parsed IDs from the CSV + */ +function parsePatientIds({ patientIdCsvPath, commonExtractorArgs }) { + const csvData = getPatientIdCSVData(patientIdCsvPath, commonExtractorArgs); + return csvParse(csvData).map((row) => { + if (!row.mrn) { + throw new Error(`${patientIdCsvPath} has no "mrn" column`); + } + return row.mrn; + }); +} + module.exports = { - buildPatientCSVPath, parsePatientIds, }; From da33942db395760c3b6b9462a11b1f9d8f72ce9a Mon Sep 17 00:00:00 2001 From: Dylan Phelan Date: Fri, 17 Dec 2021 15:28:52 -0500 Subject: [PATCH 3/6] fixed tests --- test/helpers/appUtils.test.js | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/test/helpers/appUtils.test.js b/test/helpers/appUtils.test.js index 97fd61ef..0ac4350b 100644 --- a/test/helpers/appUtils.test.js +++ b/test/helpers/appUtils.test.js @@ -11,7 +11,7 @@ describe('appUtils', () => { describe('parsePatientIds', () => { test('valid path should parse content', () => { const expectedIds = ['123', '456', '789']; - const ids = parsePatientIds(MOCK_VALID_ID_CSV); + const ids = parsePatientIds({ patientIdCsvPath: MOCK_VALID_ID_CSV }); // Should get every MRN expect(ids).toHaveLength(expectedIds.length); @@ -20,7 +20,7 @@ describe('appUtils', () => { test('valid path to CSV with BOM should parse content', () => { const expectedIds = ['123', '456', '789']; - const ids = parsePatientIds(MOCK_VALID_ID_CSV_WITH_BOM); + const ids = parsePatientIds({ patientIdCsvPath: MOCK_VALID_ID_CSV_WITH_BOM }); // Should get every MRN and correctly parse with BOM expect(ids).toHaveLength(expectedIds.length); @@ -28,7 +28,7 @@ describe('appUtils', () => { }); test('invalid path should throw error', () => { - expect(() => parsePatientIds(MOCK_INVALID_ID_CSV)).toThrowError(); + expect(() => parsePatientIds({ patientIdCsvPath: MOCK_INVALID_ID_CSV })).toThrowError(); }); }); }); From 6f3e91e849793be0e86790533f9faab8bdc70175 Mon Sep 17 00:00:00 2001 From: Dylan Phelan Date: Fri, 17 Dec 2021 15:34:40 -0500 Subject: [PATCH 4/6] reverting changes to parsePatientIds fn-contract --- src/application/app.js | 3 ++- src/helpers/appUtils.js | 8 ++++---- test/helpers/appUtils.test.js | 6 +++--- 3 files changed, 9 insertions(+), 8 deletions(-) diff --git a/src/application/app.js b/src/application/app.js index 56c39364..26357fae 100644 --- a/src/application/app.js +++ b/src/application/app.js @@ -30,7 +30,8 @@ async function mcodeApp(Client, fromDate, toDate, config, pathToRunLogs, debug, await mcodeClient.init(); // Parse CSV for list of patient mrns - const patientIds = parsePatientIds(config); + const dataDirectory = config.commonExtractorArgs && config.commonExtractorArgs.dataDirectory; + const patientIds = parsePatientIds(config.patientIdCsvPath, dataDirectory); // Get RunInstanceLogger for recording new runs and inferring dates from previous runs const runLogger = allEntries ? null : new RunInstanceLogger(pathToRunLogs); diff --git a/src/helpers/appUtils.js b/src/helpers/appUtils.js index 55221c62..c39ca9b2 100644 --- a/src/helpers/appUtils.js +++ b/src/helpers/appUtils.js @@ -8,12 +8,12 @@ const logger = require('./logger'); * * @returns array of parsed IDs from the CSV */ -function getPatientIdCSVData(patientIdCsvPath, commonExtractorArgs) { +function getPatientIdCSVData(patientIdCsvPath, dataDirectory) { try { const patientIdsCsvPath = path.resolve(patientIdCsvPath); return fs.readFileSync(patientIdsCsvPath, 'utf8'); } catch (e) { - if (commonExtractorArgs && commonExtractorArgs.dataDirectory) { + if (dataDirectory) { logger.error(`Could not resolve ${patientIdCsvPath}; even with a dataDirectory, the config.patientIdCsvPath variable needs to be a resolvable path to the patientID file on disk.`); } throw e; @@ -25,8 +25,8 @@ function getPatientIdCSVData(patientIdCsvPath, commonExtractorArgs) { * * @returns array of parsed IDs from the CSV */ -function parsePatientIds({ patientIdCsvPath, commonExtractorArgs }) { - const csvData = getPatientIdCSVData(patientIdCsvPath, commonExtractorArgs); +function parsePatientIds(patientIdCsvPath, dataDirectory) { + const csvData = getPatientIdCSVData(patientIdCsvPath, dataDirectory); return csvParse(csvData).map((row) => { if (!row.mrn) { throw new Error(`${patientIdCsvPath} has no "mrn" column`); diff --git a/test/helpers/appUtils.test.js b/test/helpers/appUtils.test.js index 0ac4350b..97fd61ef 100644 --- a/test/helpers/appUtils.test.js +++ b/test/helpers/appUtils.test.js @@ -11,7 +11,7 @@ describe('appUtils', () => { describe('parsePatientIds', () => { test('valid path should parse content', () => { const expectedIds = ['123', '456', '789']; - const ids = parsePatientIds({ patientIdCsvPath: MOCK_VALID_ID_CSV }); + const ids = parsePatientIds(MOCK_VALID_ID_CSV); // Should get every MRN expect(ids).toHaveLength(expectedIds.length); @@ -20,7 +20,7 @@ describe('appUtils', () => { test('valid path to CSV with BOM should parse content', () => { const expectedIds = ['123', '456', '789']; - const ids = parsePatientIds({ patientIdCsvPath: MOCK_VALID_ID_CSV_WITH_BOM }); + const ids = parsePatientIds(MOCK_VALID_ID_CSV_WITH_BOM); // Should get every MRN and correctly parse with BOM expect(ids).toHaveLength(expectedIds.length); @@ -28,7 +28,7 @@ describe('appUtils', () => { }); test('invalid path should throw error', () => { - expect(() => parsePatientIds({ patientIdCsvPath: MOCK_INVALID_ID_CSV })).toThrowError(); + expect(() => parsePatientIds(MOCK_INVALID_ID_CSV)).toThrowError(); }); }); }); From b41bd6fdba9df5df237e9b62900970b29651c2ef Mon Sep 17 00:00:00 2001 From: Dylan Phelan Date: Fri, 17 Dec 2021 15:37:48 -0500 Subject: [PATCH 5/6] g --- src/helpers/appUtils.js | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/helpers/appUtils.js b/src/helpers/appUtils.js index c39ca9b2..e6b62e13 100644 --- a/src/helpers/appUtils.js +++ b/src/helpers/appUtils.js @@ -23,6 +23,8 @@ function getPatientIdCSVData(patientIdCsvPath, dataDirectory) { /** * Parses a provided CSV with MRN column into string array of IDs * + * @param {string} patientIdCsvPath filePath to the CSV content to be parsed to get IDs + * @param {string} dataDirectory optional argument for if a dataDirectory was specified by the config * @returns array of parsed IDs from the CSV */ function parsePatientIds(patientIdCsvPath, dataDirectory) { From 57aaed882d6d014adef61d8accbb635b10e77bec Mon Sep 17 00:00:00 2001 From: Dylan Phelan Date: Fri, 17 Dec 2021 15:38:03 -0500 Subject: [PATCH 6/6] updated fn descriptions --- src/helpers/appUtils.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/helpers/appUtils.js b/src/helpers/appUtils.js index e6b62e13..b0a78d17 100644 --- a/src/helpers/appUtils.js +++ b/src/helpers/appUtils.js @@ -6,7 +6,7 @@ const logger = require('./logger'); /** * Loads the patientIdCSV data from disk, with some helpful hints logged in case of failure * - * @returns array of parsed IDs from the CSV + * @returns file corresponding to the patient data */ function getPatientIdCSVData(patientIdCsvPath, dataDirectory) { try {