diff --git a/src/client/BaseClient.js b/src/client/BaseClient.js index b86da8a3..feb5b8bf 100644 --- a/src/client/BaseClient.js +++ b/src/client/BaseClient.js @@ -43,19 +43,23 @@ class BaseClient { // Using Reduce to compute the validity of all extractors const allExtractorsValid = await this.extractors.reduce(async (curExtractorsValid, curExtractor) => { const { name } = curExtractor.constructor; - if (curExtractor.validate) { logger.debug(`Validating ${name}`); - const isExtractorValid = await curExtractor.validate(); - if (isExtractorValid) { - logger.debug(`Extractor ${name} PASSED CSV validation`); - } else { - logger.warn(`Extractor ${name} FAILED CSV validation`); + try { + const isExtractorValid = await curExtractor.validate(); + if (isExtractorValid) { + logger.debug(`Extractor ${name} PASSED CSV validation`); + } else { + logger.warn(`Extractor ${name} FAILED CSV validation`); + } + return ((await curExtractorsValid) && isExtractorValid); + } catch (e) { + logger.warn(`Extractor ${name} could not validate. Encountered the following error: ${e.message}`); + return false; } - return (curExtractorsValid && isExtractorValid); } return curExtractorsValid; - }, true); + }, Promise.resolve(true)); if (allExtractorsValid) { logger.info('Validation succeeded'); diff --git a/src/helpers/csvParsingUtils.js b/src/helpers/csvParsingUtils.js index f831a122..8670e5f2 100644 --- a/src/helpers/csvParsingUtils.js +++ b/src/helpers/csvParsingUtils.js @@ -7,6 +7,7 @@ function stringNormalizer(str) { // For translating null/nil-like values into empty strings function normalizeEmptyValues(data, unalterableColumns = []) { + logger.debug('Checking for empty CSV values to normalize'); const EMPTY_VALUES = ['null', 'nil'].map(stringNormalizer); const normalizedUnalterableColumns = unalterableColumns.map(stringNormalizer); // Flag tracking if empty values were normalized or not. diff --git a/src/modules/CSVURLModule.js b/src/modules/CSVURLModule.js index 32e37d58..8beba716 100644 --- a/src/modules/CSVURLModule.js +++ b/src/modules/CSVURLModule.js @@ -16,12 +16,20 @@ class CSVURLModule { // If data is already cached, this function does nothing async fillDataCache() { if (!this.data) { - const csvData = await axios.get(this.url).then((res) => res.data); + logger.debug('Filling the data cache of CSVURLModule'); + const csvData = await axios.get(this.url) + .then((res) => res.data) + .catch((e) => { + logger.error('Error occurred when making a connection to this url'); + throw e; + }); + logger.debug('Web request successful'); // Parse then normalize the data const parsedData = parse(csvData, { columns: (header) => header.map((column) => stringNormalizer(column)), bom: true, }); + logger.debug('CSV Data parsing successful'); this.data = normalizeEmptyValues(parsedData, this.unalterableColumns); } }