From 2ce6abf5db1956458d410b1bab76a73002d5c84f Mon Sep 17 00:00:00 2001 From: Dylan Phelan Date: Wed, 14 Jul 2021 14:12:30 -0400 Subject: [PATCH 1/4] added logging to the CSVURLModule; tested with local config --- src/modules/CSVURLModule.js | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/src/modules/CSVURLModule.js b/src/modules/CSVURLModule.js index 32e37d58..15f9fc8b 100644 --- a/src/modules/CSVURLModule.js +++ b/src/modules/CSVURLModule.js @@ -16,13 +16,22 @@ class CSVURLModule { // If data is already cached, this function does nothing async fillDataCache() { if (!this.data) { - const csvData = await axios.get(this.url).then((res) => res.data); + logger.debug('Filling the data cache of CSVURLModule'); + const csvData = await axios.get(this.url) + .then((res) => res.data) + .catch((e) => { + logger.error('Error occurred when getting CSV data using url'); + throw e; + }); + logger.debug('Web request successful'); // Parse then normalize the data const parsedData = parse(csvData, { columns: (header) => header.map((column) => stringNormalizer(column)), bom: true, }); + logger.debug('Data parsing successful'); this.data = normalizeEmptyValues(parsedData, this.unalterableColumns); + logger.debug('Normalization of empty values successful'); } } From 47d0526621a6bd71c6eb1f150923ee7c95f035ba Mon Sep 17 00:00:00 2001 From: Dylan Phelan Date: Thu, 15 Jul 2021 12:44:12 -0400 Subject: [PATCH 2/4] DM1 --- src/helpers/csvParsingUtils.js | 1 + src/modules/CSVURLModule.js | 1 - 2 files changed, 1 insertion(+), 1 deletion(-) diff --git a/src/helpers/csvParsingUtils.js b/src/helpers/csvParsingUtils.js index f831a122..8670e5f2 100644 --- a/src/helpers/csvParsingUtils.js +++ b/src/helpers/csvParsingUtils.js @@ -7,6 +7,7 @@ function stringNormalizer(str) { // For translating null/nil-like values into empty strings function normalizeEmptyValues(data, unalterableColumns = []) { + logger.debug('Checking for empty CSV values to normalize'); const EMPTY_VALUES = ['null', 'nil'].map(stringNormalizer); const normalizedUnalterableColumns = unalterableColumns.map(stringNormalizer); // Flag tracking if empty values were normalized or not. diff --git a/src/modules/CSVURLModule.js b/src/modules/CSVURLModule.js index 15f9fc8b..9b20fbd1 100644 --- a/src/modules/CSVURLModule.js +++ b/src/modules/CSVURLModule.js @@ -31,7 +31,6 @@ class CSVURLModule { }); logger.debug('Data parsing successful'); this.data = normalizeEmptyValues(parsedData, this.unalterableColumns); - logger.debug('Normalization of empty values successful'); } } From f9f3f1e01a3c497fd0d70f39573b1f730bc4bf12 Mon Sep 17 00:00:00 2001 From: Dylan Phelan Date: Mon, 19 Jul 2021 13:36:46 -0400 Subject: [PATCH 3/4] Addressing Julia's comments on uncaught promise rejections in validation --- src/client/BaseClient.js | 18 +++++++++++------- src/modules/CSVURLModule.js | 4 ++-- 2 files changed, 13 insertions(+), 9 deletions(-) diff --git a/src/client/BaseClient.js b/src/client/BaseClient.js index b86da8a3..05389a55 100644 --- a/src/client/BaseClient.js +++ b/src/client/BaseClient.js @@ -43,19 +43,23 @@ class BaseClient { // Using Reduce to compute the validity of all extractors const allExtractorsValid = await this.extractors.reduce(async (curExtractorsValid, curExtractor) => { const { name } = curExtractor.constructor; - if (curExtractor.validate) { logger.debug(`Validating ${name}`); - const isExtractorValid = await curExtractor.validate(); - if (isExtractorValid) { - logger.debug(`Extractor ${name} PASSED CSV validation`); - } else { + try { + const isExtractorValid = await curExtractor.validate(); + if (isExtractorValid) { + logger.debug(`Extractor ${name} PASSED CSV validation`); + } else { + logger.warn(`Extractor ${name} FAILED CSV validation`); + } + return ((await curExtractorsValid) && isExtractorValid); + } catch (e) { logger.warn(`Extractor ${name} FAILED CSV validation`); + return false; } - return (curExtractorsValid && isExtractorValid); } return curExtractorsValid; - }, true); + }, Promise.resolve(true)); if (allExtractorsValid) { logger.info('Validation succeeded'); diff --git a/src/modules/CSVURLModule.js b/src/modules/CSVURLModule.js index 9b20fbd1..8beba716 100644 --- a/src/modules/CSVURLModule.js +++ b/src/modules/CSVURLModule.js @@ -20,7 +20,7 @@ class CSVURLModule { const csvData = await axios.get(this.url) .then((res) => res.data) .catch((e) => { - logger.error('Error occurred when getting CSV data using url'); + logger.error('Error occurred when making a connection to this url'); throw e; }); logger.debug('Web request successful'); @@ -29,7 +29,7 @@ class CSVURLModule { columns: (header) => header.map((column) => stringNormalizer(column)), bom: true, }); - logger.debug('Data parsing successful'); + logger.debug('CSV Data parsing successful'); this.data = normalizeEmptyValues(parsedData, this.unalterableColumns); } } From 9c70d16e46c6012f0820a3e02e168fd28b0d48ca Mon Sep 17 00:00:00 2001 From: Dylan Phelan Date: Mon, 19 Jul 2021 16:47:42 -0400 Subject: [PATCH 4/4] Update src/client/BaseClient.js Co-authored-by: Julia Afeltra <30803904+jafeltra@users.noreply.github.com> --- src/client/BaseClient.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/client/BaseClient.js b/src/client/BaseClient.js index 05389a55..feb5b8bf 100644 --- a/src/client/BaseClient.js +++ b/src/client/BaseClient.js @@ -54,7 +54,7 @@ class BaseClient { } return ((await curExtractorsValid) && isExtractorValid); } catch (e) { - logger.warn(`Extractor ${name} FAILED CSV validation`); + logger.warn(`Extractor ${name} could not validate. Encountered the following error: ${e.message}`); return false; } }