BI-1183 - Import Validations: Ontology-specific observation criteria

src/main/java/org/breedinginsight/brapps/importer/controllers/ImportController.java

@@ -197,7 +197,7 @@ public HttpResponse<Response<DataResponse<ImportMapping>>> getSystemMappings(@Nu
         if (StringUtils.isBlank(importName)){
             result = fileImportService.getAllSystemMappings(actingUser);
         } else {
-            result = fileImportService.getSystemMappingByName(actingUser, importName);
+            result = fileImportService.getSystemMappingByName(importName);
         }
 
         List<Status> metadataStatus = new ArrayList<>();

src/main/java/org/breedinginsight/brapps/importer/daos/ImportMappingDAO.java

@@ -71,7 +71,7 @@ public List<ImportMapping> getAllProgramMappings(UUID programId, Boolean draft)
 
         List<ImportMapping> mappings = new ArrayList<>();
         for (Record record: records) {
-            mappings.add(ImportMapping.parseSQLRecord(record));
+            mappings.add(parseBrAPIImportMapping(ImportMapping.parseSQLRecord(record)));
         }
         return mappings;
     }
@@ -87,7 +87,7 @@ public List<ImportMapping> getProgramMappingsByName(UUID programId, String name)
 
         List<ImportMapping> mappings = new ArrayList<>();
         for (Record record: records) {
-            mappings.add(ImportMapping.parseSQLRecord(record));
+            mappings.add(parseBrAPIImportMapping(ImportMapping.parseSQLRecord(record)));
         }
         return mappings;
     }
@@ -102,7 +102,7 @@ public List<ImportMapping> getAllSystemMappings() {
 
         List<ImportMapping> mappings = new ArrayList<>();
         for (Record record: records) {
-            mappings.add(ImportMapping.parseSQLRecord(record));
+            mappings.add(parseBrAPIImportMapping(ImportMapping.parseSQLRecord(record)));
         }
         return mappings;
     }
@@ -119,7 +119,7 @@ public List<ImportMapping> getSystemMappingByName(String name) {
 
         List<ImportMapping> mappings = new ArrayList<>();
         for (Record record: records) {
-            mappings.add(ImportMapping.parseSQLRecord(record));
+            mappings.add(parseBrAPIImportMapping(ImportMapping.parseSQLRecord(record)));
         }
         return mappings;
     }

src/main/java/org/breedinginsight/brapps/importer/model/imports/experimentObservation/ExperimentImportService.java

@@ -68,7 +68,7 @@ public ImportPreviewResponse process(List<BrAPIImport> brAPIImports, Table data,
 
         ImportPreviewResponse response = null;
         List<Processor> processors = List.of(experimentProcessorProvider.get());
-        response = processorManagerProvider.get().process(brAPIImports, processors, program, upload, user, commit);
+        response = processorManagerProvider.get().process(brAPIImports, processors, data, program, upload, user, commit);
         return response;
 
     }

src/main/java/org/breedinginsight/brapps/importer/model/imports/germplasm/GermplasmImportService.java

@@ -70,7 +70,7 @@ public ImportPreviewResponse process(List<BrAPIImport> brAPIImports, Table data,
 
         ImportPreviewResponse response = null;
         List<Processor> processors = List.of(germplasmProcessorProvider.get());
-        response = processorManagerProvider.get().process(brAPIImports, processors, program, upload, user, commit);
+        response = processorManagerProvider.get().process(brAPIImports, processors, data, program, upload, user, commit);
         return response;
     }
 }

src/main/java/org/breedinginsight/brapps/importer/services/FileImportService.java

@@ -104,7 +104,7 @@ public List<ImportConfigResponse> getAllImportTypeConfigs() {
         Saves the file for the mapping record
      */
     public ImportMapping createMapping(UUID programId, AuthenticatedUser actingUser, CompletedFileUpload file) throws
-            DoesNotExistException, AuthorizationException, UnsupportedTypeException {
+            DoesNotExistException, AuthorizationException, UnsupportedTypeException, HttpStatusException {
 
         Program program = validateRequest(programId, actingUser);
 
@@ -530,7 +530,7 @@ public List<ImportMapping> getAllSystemMappings(AuthenticatedUser actingUser) {
         return importMappings;
     }
 
-    public List<ImportMapping> getSystemMappingByName(AuthenticatedUser actingUser, String name) {
+    public List<ImportMapping> getSystemMappingByName(String name) {
         List<ImportMapping> importMappings = importMappingDAO.getSystemMappingByName(name);
         return importMappings;
     }

src/main/java/org/breedinginsight/brapps/importer/services/FileMappingUtil.java

@@ -17,21 +17,38 @@
 
 package org.breedinginsight.brapps.importer.services;
 
+import io.micronaut.http.server.exceptions.InternalServerException;
 import org.apache.commons.lang3.tuple.MutablePair;
 import org.apache.commons.lang3.tuple.Pair;
+import org.breedinginsight.brapi.v2.dao.BrAPIGermplasmDAO;
+import org.breedinginsight.brapps.importer.daos.*;
 import org.breedinginsight.brapps.importer.model.config.MappedImportRelation;
+import org.breedinginsight.brapps.importer.model.mapping.ImportMapping;
+import org.breedinginsight.brapps.importer.model.mapping.MappingField;
+import org.jooq.DSLContext;
 import tech.tablesaw.api.Row;
 import tech.tablesaw.api.Table;
+import tech.tablesaw.columns.Column;
 
+import javax.inject.Inject;
 import javax.inject.Singleton;
 import java.util.ArrayList;
 import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
+import java.util.stream.Collectors;
 
 @Singleton
 public class FileMappingUtil {
 
+    public static final String EXPERIMENT_TEMPLATE_NAME = "ExperimentsTemplateMap";
+    private FileImportService fileImportService;
+
+    @Inject
+    public FileMappingUtil(FileImportService fileImportService) {
+        this.fileImportService = fileImportService;
+    }
+
     // Returns a list of integers to identify the target row of the relationship. -1 if no relationship was found
     public List<Pair<Integer, String>> findFileRelationships(Table data, List<MappedImportRelation> importRelations) {
 
@@ -63,4 +80,27 @@ public List<Pair<Integer, String>> findFileRelationships(Table data, List<Mapped
 
         return targetIndexList;
     }
+    public List<Column<?>> getDynamicColumns(Table data, String templateName) {
+        List<ImportMapping> result = fileImportService.getSystemMappingByName(templateName);
+
+        if (result.isEmpty()) {
+            throw new InternalServerException("System mapping does not exist");
+        }
+
+        ImportMapping mapping = result.get(0);
+        List<MappingField> config = mapping.getMappingConfig();
+        List<String> columnNames = new ArrayList<>();
+
+        for (MappingField field : config) {
+            if (field.getValue() != null) {
+                columnNames.add(field.getValue().getFileFieldName());
+            }
+        }
+
+        List<String> differences = data.columnNames().stream()
+                .filter(col -> !columnNames.contains(col))
+                .collect(Collectors.toList());
+
+        return data.columns(differences.toArray(String[]::new));
+    }
 }

src/main/java/org/breedinginsight/brapps/importer/services/processors/ExperimentProcessor.java

@@ -44,19 +44,33 @@
 import org.breedinginsight.brapps.importer.model.response.ImportPreviewStatistics;
 import org.breedinginsight.brapps.importer.model.response.PendingImportObject;
 import org.breedinginsight.brapps.importer.services.ExternalReferenceSource;
+import org.breedinginsight.brapps.importer.services.FileMappingUtil;
+import org.breedinginsight.dao.db.tables.pojos.TraitEntity;
 import org.breedinginsight.model.Program;
+import org.breedinginsight.model.Scale;
+import org.breedinginsight.model.Trait;
 import org.breedinginsight.model.User;
+import org.breedinginsight.services.OntologyService;
+import org.breedinginsight.services.exceptions.DoesNotExistException;
 import org.breedinginsight.services.exceptions.MissingRequiredInfoException;
 import org.breedinginsight.services.exceptions.ValidatorException;
 import org.breedinginsight.utilities.Utilities;
 import org.jooq.DSLContext;
+import tech.tablesaw.api.Table;
+import tech.tablesaw.columns.Column;
 
 import javax.inject.Inject;
+import java.math.BigDecimal;
 import java.math.BigInteger;
+import java.time.format.DateTimeFormatter;
+import java.time.format.DateTimeParseException;
 import java.util.*;
+import java.util.function.Function;
 import java.util.function.Supplier;
 import java.util.stream.Collectors;
 
+import static org.breedinginsight.brapps.importer.services.FileMappingUtil.EXPERIMENT_TEMPLATE_NAME;
+
 @Slf4j
 @Prototype
 public class ExperimentProcessor implements Processor {
@@ -70,14 +84,15 @@ public class ExperimentProcessor implements Processor {
     @Property(name = "brapi.server.reference-source")
     private String BRAPI_REFERENCE_SOURCE;
 
-
-    private final DSLContext dsl;
-    private final BrAPITrialDAO brapiTrialDAO;
-    private final BrAPILocationDAO brAPILocationDAO;
-    private final BrAPIStudyDAO brAPIStudyDAO;
-    private final BrAPIObservationUnitDAO brAPIObservationUnitDAO;
-    private final BrAPISeasonDAO brAPISeasonDAO;
-    private final BrAPIGermplasmDAO brAPIGermplasmDAO;
+    private DSLContext dsl;
+    private BrAPITrialDAO brapiTrialDAO;
+    private BrAPILocationDAO brAPILocationDAO;
+    private BrAPIStudyDAO brAPIStudyDAO;
+    private BrAPIObservationUnitDAO brAPIObservationUnitDAO;
+    private BrAPISeasonDAO brAPISeasonDAO;
+    private BrAPIGermplasmDAO brAPIGermplasmDAO;
+    private OntologyService ontologyService;
+    private FileMappingUtil fileMappingUtil;
 
     // used to make the yearsToSeasonDbId() function more efficient
     private final Map<String, String > yearToSeasonDbIdCache = new HashMap<>();
@@ -102,14 +117,18 @@ public ExperimentProcessor(DSLContext dsl,
                                BrAPIStudyDAO brAPIStudyDAO,
                                BrAPIObservationUnitDAO brAPIObservationUnitDAO,
                                BrAPISeasonDAO brAPISeasonDAO,
-                               BrAPIGermplasmDAO brAPIGermplasmDAO) {
+                               BrAPIGermplasmDAO brAPIGermplasmDAO,
+                               OntologyService ontologyService,
+                               FileMappingUtil fileMappingUtil) {
         this.dsl = dsl;
         this.brapiTrialDAO = brapiTrialDAO;
         this.brAPILocationDAO = brAPILocationDAO;
         this.brAPIStudyDAO = brAPIStudyDAO;
         this.brAPIObservationUnitDAO = brAPIObservationUnitDAO;
         this.brAPISeasonDAO = brAPISeasonDAO;
         this.brAPIGermplasmDAO = brAPIGermplasmDAO;
+        this.ontologyService = ontologyService;
+        this.fileMappingUtil = fileMappingUtil;
     }
 
     /**
@@ -146,12 +165,54 @@ public void getExistingBrapiData(List<BrAPIImport> importRows, Program program)
     public Map<String, ImportPreviewStatistics> process(
             List<BrAPIImport> importRows,
             Map<Integer, PendingImport> mappedBrAPIImport,
+            Table data,
             Program program,
             User user,
             boolean commit) throws ValidatorException, MissingRequiredInfoException {
 
         ValidationErrors validationErrors = new ValidationErrors();
 
+        // Get dynamic phenotype columns for processing
+        List<Column<?>> phenotypeCols = fileMappingUtil.getDynamicColumns(data, EXPERIMENT_TEMPLATE_NAME);
+        List<String> varNames = phenotypeCols.stream().map(Column::name).collect(Collectors.toList());
+
+        // Lookup all traits in system for program, maybe eventually add a variable search in ontology service
+        List<Trait> traits = null;
+        try {
+            traits = ontologyService.getTraitsByProgramId(program.getId(), true);
+        } catch (DoesNotExistException e) {
+            log.error(e.getMessage(), e);
+            throw new InternalServerException(e.toString(), e);
+        }
+
+        // filter out just traits specified in file
+        List<Trait> filteredTraits = traits.stream()
+                .filter(e -> varNames.contains(e.getObservationVariableName()))
+                .collect(Collectors.toList());
+
+        // check that all specified ontology terms were found
+        if (filteredTraits.size() != varNames.size()) {
+            List<String> returnedVarNames = filteredTraits.stream().map(TraitEntity::getObservationVariableName)
+                    .collect(Collectors.toList());
+            List<String> differences = varNames.stream()
+                    .filter(var -> !returnedVarNames.contains(var))
+                    .collect(Collectors.toList());
+            throw new HttpStatusException(HttpStatus.UNPROCESSABLE_ENTITY,
+                    "Ontology term(s) not found: " + String.join(", ", differences));
+        }
+
+        // Perform ontology validations on each observation value in phenotype column
+        Map<String, Trait> colVarMap = filteredTraits.stream()
+                .collect(Collectors.toMap(Trait::getObservationVariableName, Function.identity()));
+
+        for (Column<?> column : phenotypeCols) {
+            for (int i=0; i < column.size(); i++) {
+                String value = column.getString(i);
+                String colName = column.name();
+                validateObservationValue(colVarMap.get(colName), value, colName, validationErrors, i);
+            }
+        }
+
         // add "New" pending data to the BrapiData objects
         getNewBrapiData(importRows, program, commit);
 
@@ -708,6 +769,74 @@ private String simpleStudyName(String scopedName){
         return scopedName.replaceFirst(" \\[.*\\]", "");
     }
 
+    private void validateObservationValue(Trait variable, String value,
+                                          String columnHeader, ValidationErrors validationErrors, int row) {
+        if(StringUtils.isBlank(value)) {
+            log.debug(String.format("skipping validation of observation because there is no value.\n\tvariable: %s\n\trow: %d", variable.getObservationVariableName(), row));
+            return;
+        }
+
+        switch(variable.getScale().getDataType()) {
+            case NUMERICAL:
+                Optional<BigDecimal> number = validNumericValue(value);
+                if (number.isEmpty()) {
+                    addRowError(columnHeader, "Non-numeric text detected detected", validationErrors, row);
+                }
+                else if (!validNumericRange(number.get(), variable.getScale())) {
+                    addRowError(columnHeader, "Value outside of min/max range detected", validationErrors, row);
+                }
+                break;
+            case DATE:
+                if (!validDateValue(value)) {
+                    addRowError(columnHeader, "Incorrect date format detected. Expected YYYY-MM-DD", validationErrors, row);
+                }
+                break;
+            case ORDINAL:
+                if (!validCategory(variable.getScale().getCategories(), value)) {
+                    addRowError(columnHeader, "Undefined ordinal category detected", validationErrors, row);
+                }
+                break;
+            case NOMINAL:
+                if (!validCategory(variable.getScale().getCategories(), value)) {
+                    addRowError(columnHeader, "Undefined nominal category detected", validationErrors, row);
+                }
+                break;
+            default:
+                break;
+        }
+
+    }
+    private Optional<BigDecimal> validNumericValue(String value) {
+        BigDecimal number;
+        try {
+            number = new BigDecimal(value);
+        } catch (NumberFormatException e) {
+            return Optional.empty();
+        }
+        return Optional.of(number);
+    }
+
+    private boolean validNumericRange(BigDecimal value, Scale validValues) {
+        // account for empty min or max in valid determination
+        return (validValues.getValidValueMin() == null || value.compareTo(BigDecimal.valueOf(validValues.getValidValueMin())) >= 0) &&
+               (validValues.getValidValueMax() == null || value.compareTo(BigDecimal.valueOf(validValues.getValidValueMax())) <= 0);
+    }
+
+    private boolean validDateValue(String value) {
+        DateTimeFormatter formatter = DateTimeFormatter.ISO_DATE;
+        try {
+            formatter.parse(value);
+        } catch (DateTimeParseException e) {
+            return false;
+        }
+        return true;
+    }
+
+    private boolean validCategory(List<BrAPIScaleValidValuesCategories> categories, String value) {
+        Set<String> categoryValues = categories.stream().map(category -> category.getValue().toLowerCase()).collect(Collectors.toSet());
+        return categoryValues.contains(value.toLowerCase());
+    }
+
     /**
      * Converts year String to SeasonDbId
      * <br>

src/main/java/org/breedinginsight/brapps/importer/services/processors/GermplasmProcessor.java

@@ -44,6 +44,7 @@
 import org.breedinginsight.model.User;
 import org.breedinginsight.services.exceptions.ValidatorException;
 import org.jooq.DSLContext;
+import tech.tablesaw.api.Table;
 
 import javax.inject.Inject;
 import java.math.BigInteger;
@@ -205,7 +206,8 @@ public void getExistingBrapiData(List<BrAPIImport> importRows, Program program)
 
     @Override
     public Map<String, ImportPreviewStatistics> process(List<BrAPIImport> importRows,
-                        Map<Integer, PendingImport> mappedBrAPIImport, Program program, User user, boolean commit) throws ValidatorException {
+                                                        Map<Integer, PendingImport> mappedBrAPIImport, Table data,
+                                                        Program program, User user, boolean commit) throws ValidatorException {
 
         // Method for generating accession number
         String germplasmSequenceName = program.getGermplasmSequence();

src/main/java/org/breedinginsight/brapps/importer/services/processors/LocationProcessor.java

@@ -31,6 +31,7 @@
 import org.breedinginsight.model.Program;
 import org.breedinginsight.model.User;
 import org.breedinginsight.services.exceptions.ValidatorException;
+import tech.tablesaw.api.Table;
 
 import javax.inject.Inject;
 import java.util.ArrayList;
@@ -74,7 +75,8 @@ public void getExistingBrapiData(List<BrAPIImport> importRows, Program program)
 
     @Override
     public Map<String, ImportPreviewStatistics> process(List<BrAPIImport> importRows,
-                Map<Integer, PendingImport> mappedBrAPIImport, Program program, User user, boolean commit) throws ValidatorException {
+                                                        Map<Integer, PendingImport> mappedBrAPIImport, Table data,
+                                                        Program program, User user, boolean commit) throws ValidatorException {
 
         for (int i = 0; i < importRows.size(); i++) {
             BrAPIImport brapiImport = importRows.get(i);