From 12b528353ec1401806617c6a6f2186040ecb63e3 Mon Sep 17 00:00:00 2001 From: julianxcarter Date: Thu, 7 Oct 2021 14:34:22 -0400 Subject: [PATCH 1/3] All option for masking fields added to config --- README.md | 12 ++++++++++++ src/extractors/CSVPatientExtractor.js | 26 ++++++++++++++++++++++++-- src/extractors/FHIRPatientExtractor.js | 24 +++++++++++++++++++++++- src/helpers/configUtils.js | 4 ++++ src/helpers/schemas/config.schema.json | 16 ++++++++++++---- 5 files changed, 75 insertions(+), 7 deletions(-) diff --git a/README.md b/README.md index a8abbb63..f4387829 100644 --- a/README.md +++ b/README.md @@ -148,6 +148,18 @@ To mask a property, provide an array of the properties to mask in the `construct } ``` +Alternatively, providing a string with a value of `all` in the `constructorArgs` of the Patient extractor will mask all of the supported properties listed above. The following configuration can be used to mask all properties of the `Patient` resource, rather than listing each individual property: + +```bash +{ + "label": "patient", + "type": "CSVPatientExtractor", + "constructorArgs": { + "filePath": "./data/patient-information.csv" + "mask": "all" + } +} +``` ### Extraction Date Range The mCODE Extraction Client will extract all data that is provided in the CSV files by default, regardless of any dates associated with each row of data. It is recommended that any required date filtering is performed outside of the scope of this client. diff --git a/src/extractors/CSVPatientExtractor.js b/src/extractors/CSVPatientExtractor.js index 14315f4b..f3757204 100644 --- a/src/extractors/CSVPatientExtractor.js +++ b/src/extractors/CSVPatientExtractor.js @@ -9,6 +9,26 @@ const { getEmptyBundle } = require('../helpers/fhirUtils'); const { CSVPatientSchema } = require('../helpers/schemas/csv'); const logger = require('../helpers/logger'); +const ALL_SUPPORTED_MASK_FIELDS = [ + 'gender', + 'mrn', + 'name', + 'address', + 'birthDate', + 'language', + 'ethnicity', + 'birthsex', + 'race', + 'telecom', + 'multipleBirth', + 'photo', + 'contact', + 'generalPractitioner', + 'managingOrganization', + 'link', +]; + + function joinAndReformatData(patientData) { logger.debug('Reformatting patient data from CSV into template format'); // No join needed, just a reformatting @@ -87,8 +107,10 @@ class CSVPatientExtractor extends BaseCSVExtractor { // 3. Generate FHIR Resources const bundle = generateMcodeResources('Patient', packagedPatientData); - // mask fields in the patient data if specified in mask array - if (this.mask.length > 0) maskPatientData(bundle, this.mask); + // mask specified fields in the patient data + if (typeof this.mask === 'string') { + maskPatientData(bundle, ALL_SUPPORTED_MASK_FIELDS); + } else if (this.mask.length > 0) maskPatientData(bundle, this.mask); return bundle; } } diff --git a/src/extractors/FHIRPatientExtractor.js b/src/extractors/FHIRPatientExtractor.js index 0ac7f262..76178d41 100644 --- a/src/extractors/FHIRPatientExtractor.js +++ b/src/extractors/FHIRPatientExtractor.js @@ -1,6 +1,25 @@ const { BaseFHIRExtractor } = require('./BaseFHIRExtractor'); const { maskPatientData } = require('../helpers/patientUtils.js'); +const ALL_SUPPORTED_MASK_FIELDS = [ + 'gender', + 'mrn', + 'name', + 'address', + 'birthDate', + 'language', + 'ethnicity', + 'birthsex', + 'race', + 'telecom', + 'multipleBirth', + 'photo', + 'contact', + 'generalPractitioner', + 'managingOrganization', + 'link', +]; + class FHIRPatientExtractor extends BaseFHIRExtractor { constructor({ baseFhirUrl, requestHeaders, version, mask = [] }) { super({ baseFhirUrl, requestHeaders, version }); @@ -18,7 +37,10 @@ class FHIRPatientExtractor extends BaseFHIRExtractor { async get(argumentObject) { const bundle = await super.get(argumentObject); - if (this.mask.length > 0) maskPatientData(bundle, this.mask); + // mask specified fields in the patient data + if (typeof this.mask === 'string') { + maskPatientData(bundle, ALL_SUPPORTED_MASK_FIELDS); + } else if (this.mask.length > 0) maskPatientData(bundle, this.mask); return bundle; } } diff --git a/src/helpers/configUtils.js b/src/helpers/configUtils.js index e0494cde..486fe7e0 100644 --- a/src/helpers/configUtils.js +++ b/src/helpers/configUtils.js @@ -34,6 +34,10 @@ ajv.addFormat('email-with-name', { return emailRegex.test(email.trim().split(' ').pop()); }, }); +ajv.addFormat('mask-all', { + type: 'string', + validate: (string) => string === 'all', +}); const validator = ajv.addSchema(configSchema, 'config'); diff --git a/src/helpers/schemas/config.schema.json b/src/helpers/schemas/config.schema.json index be659c9c..d942b482 100644 --- a/src/helpers/schemas/config.schema.json +++ b/src/helpers/schemas/config.schema.json @@ -128,10 +128,18 @@ }, "mask": { "title": "Masked Fields", - "type": "array", - "items": { - "type": "string" - } + "oneOf": [ + { + "type": "string", + "format": "mask-all" + }, + { + "type": "array", + "items": { + "type": "string" + } + } + ] } } } From 9e8b0b492e04959f418ad37048261d3129609b2f Mon Sep 17 00:00:00 2001 From: julianxcarter Date: Thu, 7 Oct 2021 15:04:01 -0400 Subject: [PATCH 2/3] One more round of 'all' validation --- src/extractors/CSVPatientExtractor.js | 2 +- src/extractors/FHIRPatientExtractor.js | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/extractors/CSVPatientExtractor.js b/src/extractors/CSVPatientExtractor.js index f3757204..6aa14b8d 100644 --- a/src/extractors/CSVPatientExtractor.js +++ b/src/extractors/CSVPatientExtractor.js @@ -108,7 +108,7 @@ class CSVPatientExtractor extends BaseCSVExtractor { const bundle = generateMcodeResources('Patient', packagedPatientData); // mask specified fields in the patient data - if (typeof this.mask === 'string') { + if (typeof this.mask === 'string' && this.mask === 'all') { maskPatientData(bundle, ALL_SUPPORTED_MASK_FIELDS); } else if (this.mask.length > 0) maskPatientData(bundle, this.mask); return bundle; diff --git a/src/extractors/FHIRPatientExtractor.js b/src/extractors/FHIRPatientExtractor.js index 76178d41..04111fd8 100644 --- a/src/extractors/FHIRPatientExtractor.js +++ b/src/extractors/FHIRPatientExtractor.js @@ -38,7 +38,7 @@ class FHIRPatientExtractor extends BaseFHIRExtractor { async get(argumentObject) { const bundle = await super.get(argumentObject); // mask specified fields in the patient data - if (typeof this.mask === 'string') { + if (typeof this.mask === 'string' && this.mask === 'all') { maskPatientData(bundle, ALL_SUPPORTED_MASK_FIELDS); } else if (this.mask.length > 0) maskPatientData(bundle, this.mask); return bundle; From ec7ef8c4d118ddb3c8788817ac53e3175a55c8bb Mon Sep 17 00:00:00 2001 From: julianxcarter Date: Tue, 12 Oct 2021 15:25:12 -0400 Subject: [PATCH 3/3] maskAll flag added to maskPatientData, unit test added --- src/extractors/CSVPatientExtractor.js | 22 +--------------------- src/extractors/FHIRPatientExtractor.js | 21 +-------------------- src/helpers/patientUtils.js | 7 +++++-- test/helpers/patientUtils.test.js | 6 ++++++ 4 files changed, 13 insertions(+), 43 deletions(-) diff --git a/src/extractors/CSVPatientExtractor.js b/src/extractors/CSVPatientExtractor.js index 6aa14b8d..f2e66e8a 100644 --- a/src/extractors/CSVPatientExtractor.js +++ b/src/extractors/CSVPatientExtractor.js @@ -9,26 +9,6 @@ const { getEmptyBundle } = require('../helpers/fhirUtils'); const { CSVPatientSchema } = require('../helpers/schemas/csv'); const logger = require('../helpers/logger'); -const ALL_SUPPORTED_MASK_FIELDS = [ - 'gender', - 'mrn', - 'name', - 'address', - 'birthDate', - 'language', - 'ethnicity', - 'birthsex', - 'race', - 'telecom', - 'multipleBirth', - 'photo', - 'contact', - 'generalPractitioner', - 'managingOrganization', - 'link', -]; - - function joinAndReformatData(patientData) { logger.debug('Reformatting patient data from CSV into template format'); // No join needed, just a reformatting @@ -109,7 +89,7 @@ class CSVPatientExtractor extends BaseCSVExtractor { // mask specified fields in the patient data if (typeof this.mask === 'string' && this.mask === 'all') { - maskPatientData(bundle, ALL_SUPPORTED_MASK_FIELDS); + maskPatientData(bundle, [], true); } else if (this.mask.length > 0) maskPatientData(bundle, this.mask); return bundle; } diff --git a/src/extractors/FHIRPatientExtractor.js b/src/extractors/FHIRPatientExtractor.js index 04111fd8..f56a34e5 100644 --- a/src/extractors/FHIRPatientExtractor.js +++ b/src/extractors/FHIRPatientExtractor.js @@ -1,25 +1,6 @@ const { BaseFHIRExtractor } = require('./BaseFHIRExtractor'); const { maskPatientData } = require('../helpers/patientUtils.js'); -const ALL_SUPPORTED_MASK_FIELDS = [ - 'gender', - 'mrn', - 'name', - 'address', - 'birthDate', - 'language', - 'ethnicity', - 'birthsex', - 'race', - 'telecom', - 'multipleBirth', - 'photo', - 'contact', - 'generalPractitioner', - 'managingOrganization', - 'link', -]; - class FHIRPatientExtractor extends BaseFHIRExtractor { constructor({ baseFhirUrl, requestHeaders, version, mask = [] }) { super({ baseFhirUrl, requestHeaders, version }); @@ -39,7 +20,7 @@ class FHIRPatientExtractor extends BaseFHIRExtractor { const bundle = await super.get(argumentObject); // mask specified fields in the patient data if (typeof this.mask === 'string' && this.mask === 'all') { - maskPatientData(bundle, ALL_SUPPORTED_MASK_FIELDS); + maskPatientData(bundle, [], true); } else if (this.mask.length > 0) maskPatientData(bundle, this.mask); return bundle; } diff --git a/src/helpers/patientUtils.js b/src/helpers/patientUtils.js index 34a18226..7224c5c2 100644 --- a/src/helpers/patientUtils.js +++ b/src/helpers/patientUtils.js @@ -81,8 +81,9 @@ function getPatientName(name) { * 'gender','mrn','name','address','birthDate','language','ethnicity','birthsex', * 'race', 'telecom', 'multipleBirth', 'photo', 'contact', 'generalPractitioner', * 'managingOrganization', and 'link' + * @param {Boolean} maskAll indicates that all supported fields should be masked, defaults to false */ -function maskPatientData(bundle, mask) { +function maskPatientData(bundle, mask, maskAll = false) { // get Patient resource from bundle const patient = fhirpath.evaluate( bundle, @@ -109,7 +110,9 @@ function maskPatientData(bundle, mask) { ]; const masked = extensionArr(dataAbsentReasonExtension('masked')); - mask.forEach((field) => { + const maskingFields = maskAll ? validFields : mask; + + maskingFields.forEach((field) => { if (!validFields.includes(field)) { throw Error(`'${field}' is not a field that can be masked. Patient will only be extracted if all mask fields are valid. Valid fields include: Valid fields include: ${validFields.join(', ')}`); } diff --git a/test/helpers/patientUtils.test.js b/test/helpers/patientUtils.test.js index abc9481f..ecd5174e 100644 --- a/test/helpers/patientUtils.test.js +++ b/test/helpers/patientUtils.test.js @@ -113,6 +113,12 @@ describe('PatientUtils', () => { expect(bundle).toEqual(exampleMaskedPatient); }); + test('bundle should be modified to have dataAbsentReason for all fields when the maskAll flag is provided', () => { + const bundle = _.cloneDeep(examplePatient); + maskPatientData(bundle, [], true); + expect(bundle).toEqual(exampleMaskedPatient); + }); + test('should mask gender even if it only had an extension', () => { const bundle = _.cloneDeep(examplePatient); delete bundle.entry[0].resource.gender;