From 7e346853046e945cc76c242264919b3278914822 Mon Sep 17 00:00:00 2001 From: David Randolph Phillips Date: Wed, 29 Mar 2023 15:24:19 -0400 Subject: [PATCH 1/4] [BI-1583] Check for 'Data' worksheet in import Excel file --- .../importer/services/FileImportService.java | 2 +- .../services/TraitUploadService.java | 2 +- .../services/parsers/ParsingExceptionType.java | 4 +++- .../parsers/trait/TraitFileParser.java | 2 +- .../breedinginsight/utilities/FileUtil.java | 18 ++++++++++++++---- 5 files changed, 20 insertions(+), 8 deletions(-) diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/FileImportService.java b/src/main/java/org/breedinginsight/brapps/importer/services/FileImportService.java index 19bdbd1cb..cbc6f6620 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/FileImportService.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/FileImportService.java @@ -162,7 +162,7 @@ private Table parseUploadedFile(CompletedFileUpload file) throws UnsupportedType //TODO: Allow them to pass in header row index in the future df = FileUtil.parseTableFromExcel(file.getInputStream(), 0); } catch (IOException | ParsingException e) { - throw new HttpStatusException(HttpStatus.BAD_REQUEST, "Error parsing excel: " + e.getMessage()); + throw new HttpStatusException(HttpStatus.BAD_REQUEST, String.format("Error(s) detected in file, %s. %s. Import cannot proceed.", file.getFilename(), e.getMessage())); } } else { throw new UnsupportedTypeException("Unsupported mime type"); diff --git a/src/main/java/org/breedinginsight/services/TraitUploadService.java b/src/main/java/org/breedinginsight/services/TraitUploadService.java index 8bb4712d8..414baead7 100644 --- a/src/main/java/org/breedinginsight/services/TraitUploadService.java +++ b/src/main/java/org/breedinginsight/services/TraitUploadService.java @@ -116,7 +116,7 @@ public ProgramUpload updateTraitUpload(UUID programId, CompletedFileUpload file, traits = parser.parseExcel(new BOMInputStream(file.getInputStream(), false)); } catch(IOException | ParsingException e) { log.error(e.getMessage()); - throw new HttpStatusException(HttpStatus.BAD_REQUEST, "Error parsing excel: " + e.getMessage()); + throw new HttpStatusException( HttpStatus.BAD_REQUEST, String.format("Error(s) detected in file, %s. %s.", file.getFilename(), e.getMessage()) ); } } else { throw new UnsupportedTypeException("Unsupported mime type"); diff --git a/src/main/java/org/breedinginsight/services/parsers/ParsingExceptionType.java b/src/main/java/org/breedinginsight/services/parsers/ParsingExceptionType.java index d7d33ae03..fcae51935 100644 --- a/src/main/java/org/breedinginsight/services/parsers/ParsingExceptionType.java +++ b/src/main/java/org/breedinginsight/services/parsers/ParsingExceptionType.java @@ -16,6 +16,8 @@ */ package org.breedinginsight.services.parsers; +import org.breedinginsight.utilities.FileUtil; + public enum ParsingExceptionType { MISSING_COLUMN_NAMES("Missing column names row"), @@ -23,7 +25,7 @@ public enum ParsingExceptionType { DUPLICATE_COLUMN_NAMES("Found duplicate column names"), MISSING_EXPECTED_COLUMNS("Missing expected columns"), ERROR_READING_FILE("Error reading file"), - MISSING_SHEET("Missing sheet Template"), + MISSING_SHEET( String.format("Worksheet titled '%s' is missing", FileUtil.EXCEL_DATA_SHEET_NAME) ), EMPTY_ROW("Empty row"), INVALID_TRAIT_STATUS("Invalid trait status value"), INVALID_SCALE_CLASS("Invalid scale class value"), diff --git a/src/main/java/org/breedinginsight/services/parsers/trait/TraitFileParser.java b/src/main/java/org/breedinginsight/services/parsers/trait/TraitFileParser.java index 906115f3d..98a0c71d3 100644 --- a/src/main/java/org/breedinginsight/services/parsers/trait/TraitFileParser.java +++ b/src/main/java/org/breedinginsight/services/parsers/trait/TraitFileParser.java @@ -58,7 +58,7 @@ public class TraitFileParser { private static final String LIST_DELIMITER = ";"; private static final String CATEGORY_DELIMITER = "="; - private static final String EXCEL_DATA_SHEET_NAME = "Template"; + private static final String EXCEL_DATA_SHEET_NAME = "Data"; private static final String TRAIT_STATUS_ACTIVE = "active"; private static final String TRAIT_STATUS_ARCHIVED = "archived"; diff --git a/src/main/java/org/breedinginsight/utilities/FileUtil.java b/src/main/java/org/breedinginsight/utilities/FileUtil.java index a52ff1ee5..e04a332fc 100644 --- a/src/main/java/org/breedinginsight/utilities/FileUtil.java +++ b/src/main/java/org/breedinginsight/utilities/FileUtil.java @@ -33,8 +33,13 @@ import java.util.*; import java.util.stream.Collectors; + @Slf4j public class FileUtil { + public static final String EXCEL_DATA_SHEET_NAME = "Data"; + // For backward compatibility + private static final String OLD_GERMPLASM_EXCEL_DATA_SHEET_NAME = "Germplasm Import"; + private static final String OLD_EXPERIMENT_EXCEL_DATA_SHEET_NAME = "Experiment Data"; public static Table parseTableFromExcel(InputStream inputStream, Integer headerRowIndex) throws ParsingException { @@ -46,10 +51,15 @@ public static Table parseTableFromExcel(InputStream inputStream, Integer headerR throw new ParsingException(ParsingExceptionType.ERROR_READING_FILE); } - List sheets = new ArrayList<>(); - workbook.sheetIterator().forEachRemaining(sheets::add); - //TODO: Gets the last sheet for now, do we want to allow them to specify which sheet to use? - Sheet sheet = workbook.getSheetAt(sheets.size() - 1); + Sheet sheet = workbook.getSheet(EXCEL_DATA_SHEET_NAME); + + //For backward compatibility allow old sheet names + if( sheet == null){ sheet = workbook.getSheet(OLD_GERMPLASM_EXCEL_DATA_SHEET_NAME); } + if( sheet == null){ sheet = workbook.getSheet(OLD_EXPERIMENT_EXCEL_DATA_SHEET_NAME); } + + if (sheet == null) { + throw new ParsingException(ParsingExceptionType.MISSING_SHEET); + } Iterator rowIterator = sheet.rowIterator(); // Get into format tablesaw can use From e266aa9579a67ee82c262bade6cb03fff665a4a8 Mon Sep 17 00:00:00 2001 From: David Randolph Phillips Date: Thu, 30 Mar 2023 11:23:52 -0400 Subject: [PATCH 2/4] [BI-1583] backwards compatibility or ontology/trait import --- .../services/parsers/trait/TraitFileParser.java | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/src/main/java/org/breedinginsight/services/parsers/trait/TraitFileParser.java b/src/main/java/org/breedinginsight/services/parsers/trait/TraitFileParser.java index 98a0c71d3..698859b08 100644 --- a/src/main/java/org/breedinginsight/services/parsers/trait/TraitFileParser.java +++ b/src/main/java/org/breedinginsight/services/parsers/trait/TraitFileParser.java @@ -59,6 +59,8 @@ public class TraitFileParser { private static final String LIST_DELIMITER = ";"; private static final String CATEGORY_DELIMITER = "="; private static final String EXCEL_DATA_SHEET_NAME = "Data"; + private static final String OLD_EXCEL_DATA_SHEET_NAME = "Template"; + private static final String TRAIT_STATUS_ACTIVE = "active"; private static final String TRAIT_STATUS_ARCHIVED = "archived"; @@ -82,8 +84,14 @@ public List parseExcel(@NonNull InputStream inputStream) throws ParsingEx log.error(e.getMessage()); throw new ParsingException(ParsingExceptionType.ERROR_READING_FILE); } - + // TODO accept a "Template" sheet Sheet sheet = workbook.getSheet(EXCEL_DATA_SHEET_NAME); + + // accept the old sheet name ("template") for backwards compatability + if (sheet == null){ + sheet = workbook.getSheet(OLD_EXCEL_DATA_SHEET_NAME); + } + if (sheet == null) { throw new ParsingException(ParsingExceptionType.MISSING_SHEET); } From 16741158bcdfb3e9da46bd987ae177feb7c24be0 Mon Sep 17 00:00:00 2001 From: David Randolph Phillips Date: Thu, 30 Mar 2023 15:36:57 -0400 Subject: [PATCH 3/4] [BI-1583] Improved Error message. Removed unneeded 'TODO' --- .../java/org/breedinginsight/services/TraitUploadService.java | 2 +- .../breedinginsight/services/parsers/trait/TraitFileParser.java | 1 - 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/src/main/java/org/breedinginsight/services/TraitUploadService.java b/src/main/java/org/breedinginsight/services/TraitUploadService.java index 414baead7..8c49c303d 100644 --- a/src/main/java/org/breedinginsight/services/TraitUploadService.java +++ b/src/main/java/org/breedinginsight/services/TraitUploadService.java @@ -116,7 +116,7 @@ public ProgramUpload updateTraitUpload(UUID programId, CompletedFileUpload file, traits = parser.parseExcel(new BOMInputStream(file.getInputStream(), false)); } catch(IOException | ParsingException e) { log.error(e.getMessage()); - throw new HttpStatusException( HttpStatus.BAD_REQUEST, String.format("Error(s) detected in file, %s. %s.", file.getFilename(), e.getMessage()) ); + throw new HttpStatusException( HttpStatus.BAD_REQUEST, String.format("Error(s) detected in file, %s. %s. Import cannot proceed.", file.getFilename(), e.getMessage()) ); } } else { throw new UnsupportedTypeException("Unsupported mime type"); diff --git a/src/main/java/org/breedinginsight/services/parsers/trait/TraitFileParser.java b/src/main/java/org/breedinginsight/services/parsers/trait/TraitFileParser.java index 698859b08..3d81eac24 100644 --- a/src/main/java/org/breedinginsight/services/parsers/trait/TraitFileParser.java +++ b/src/main/java/org/breedinginsight/services/parsers/trait/TraitFileParser.java @@ -84,7 +84,6 @@ public List parseExcel(@NonNull InputStream inputStream) throws ParsingEx log.error(e.getMessage()); throw new ParsingException(ParsingExceptionType.ERROR_READING_FILE); } - // TODO accept a "Template" sheet Sheet sheet = workbook.getSheet(EXCEL_DATA_SHEET_NAME); // accept the old sheet name ("template") for backwards compatability From 3e71060ab1a131c2dd956528f5707f5cec7ec432 Mon Sep 17 00:00:00 2001 From: David Randolph Phillips Date: Thu, 6 Apr 2023 15:43:06 -0400 Subject: [PATCH 4/4] [BI-1583] fixed unit test --- .../breedinginsight/utilities/response/FileUtilUnitTest.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/test/java/org/breedinginsight/utilities/response/FileUtilUnitTest.java b/src/test/java/org/breedinginsight/utilities/response/FileUtilUnitTest.java index ad36d4f64..14d00a25c 100644 --- a/src/test/java/org/breedinginsight/utilities/response/FileUtilUnitTest.java +++ b/src/test/java/org/breedinginsight/utilities/response/FileUtilUnitTest.java @@ -80,7 +80,7 @@ void writeExcelCheckColumns() { row.put("Test C", "C"); data.add(row); - InputStream inputStream = ExcelWriter.writeToInputStream("SheetName", columns, data, FileType.XLSX); + InputStream inputStream = ExcelWriter.writeToInputStream("Data", columns, data, FileType.XLSX); Table resultTable = FileUtil.parseTableFromExcel(inputStream, 0); assertEquals(1, resultTable.rowCount(), "Wrong number of rows were exported");