diff --git a/src/main/java/org/breedinginsight/brapps/importer/controllers/ImportController.java b/src/main/java/org/breedinginsight/brapps/importer/controllers/ImportController.java index 8475fdd17..e0a61117a 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/controllers/ImportController.java +++ b/src/main/java/org/breedinginsight/brapps/importer/controllers/ImportController.java @@ -197,7 +197,7 @@ public HttpResponse>> getSystemMappings(@Nu if (StringUtils.isBlank(importName)){ result = fileImportService.getAllSystemMappings(actingUser); } else { - result = fileImportService.getSystemMappingByName(actingUser, importName); + result = fileImportService.getSystemMappingByName(importName); } List metadataStatus = new ArrayList<>(); diff --git a/src/main/java/org/breedinginsight/brapps/importer/daos/ImportMappingDAO.java b/src/main/java/org/breedinginsight/brapps/importer/daos/ImportMappingDAO.java index ef36344ec..8d54b3cbe 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/daos/ImportMappingDAO.java +++ b/src/main/java/org/breedinginsight/brapps/importer/daos/ImportMappingDAO.java @@ -71,7 +71,7 @@ public List getAllProgramMappings(UUID programId, Boolean draft) List mappings = new ArrayList<>(); for (Record record: records) { - mappings.add(ImportMapping.parseSQLRecord(record)); + mappings.add(parseBrAPIImportMapping(ImportMapping.parseSQLRecord(record))); } return mappings; } @@ -87,7 +87,7 @@ public List getProgramMappingsByName(UUID programId, String name) List mappings = new ArrayList<>(); for (Record record: records) { - mappings.add(ImportMapping.parseSQLRecord(record)); + mappings.add(parseBrAPIImportMapping(ImportMapping.parseSQLRecord(record))); } return mappings; } @@ -102,7 +102,7 @@ public List getAllSystemMappings() { List mappings = new ArrayList<>(); for (Record record: records) { - mappings.add(ImportMapping.parseSQLRecord(record)); + mappings.add(parseBrAPIImportMapping(ImportMapping.parseSQLRecord(record))); } return mappings; } @@ -119,7 +119,7 @@ public List getSystemMappingByName(String name) { List mappings = new ArrayList<>(); for (Record record: records) { - mappings.add(ImportMapping.parseSQLRecord(record)); + mappings.add(parseBrAPIImportMapping(ImportMapping.parseSQLRecord(record))); } return mappings; } diff --git a/src/main/java/org/breedinginsight/brapps/importer/model/imports/experimentObservation/ExperimentImportService.java b/src/main/java/org/breedinginsight/brapps/importer/model/imports/experimentObservation/ExperimentImportService.java index 4340ac262..19c4a3c22 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/model/imports/experimentObservation/ExperimentImportService.java +++ b/src/main/java/org/breedinginsight/brapps/importer/model/imports/experimentObservation/ExperimentImportService.java @@ -68,7 +68,7 @@ public ImportPreviewResponse process(List brAPIImports, Table data, ImportPreviewResponse response = null; List processors = List.of(experimentProcessorProvider.get()); - response = processorManagerProvider.get().process(brAPIImports, processors, program, upload, user, commit); + response = processorManagerProvider.get().process(brAPIImports, processors, data, program, upload, user, commit); return response; } diff --git a/src/main/java/org/breedinginsight/brapps/importer/model/imports/germplasm/GermplasmImportService.java b/src/main/java/org/breedinginsight/brapps/importer/model/imports/germplasm/GermplasmImportService.java index a99673275..cadd379a0 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/model/imports/germplasm/GermplasmImportService.java +++ b/src/main/java/org/breedinginsight/brapps/importer/model/imports/germplasm/GermplasmImportService.java @@ -70,7 +70,7 @@ public ImportPreviewResponse process(List brAPIImports, Table data, ImportPreviewResponse response = null; List processors = List.of(germplasmProcessorProvider.get()); - response = processorManagerProvider.get().process(brAPIImports, processors, program, upload, user, commit); + response = processorManagerProvider.get().process(brAPIImports, processors, data, program, upload, user, commit); return response; } } diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/FileImportService.java b/src/main/java/org/breedinginsight/brapps/importer/services/FileImportService.java index f253d7873..cf9c6b916 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/FileImportService.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/FileImportService.java @@ -104,7 +104,7 @@ public List getAllImportTypeConfigs() { Saves the file for the mapping record */ public ImportMapping createMapping(UUID programId, AuthenticatedUser actingUser, CompletedFileUpload file) throws - DoesNotExistException, AuthorizationException, UnsupportedTypeException { + DoesNotExistException, AuthorizationException, UnsupportedTypeException, HttpStatusException { Program program = validateRequest(programId, actingUser); @@ -530,7 +530,7 @@ public List getAllSystemMappings(AuthenticatedUser actingUser) { return importMappings; } - public List getSystemMappingByName(AuthenticatedUser actingUser, String name) { + public List getSystemMappingByName(String name) { List importMappings = importMappingDAO.getSystemMappingByName(name); return importMappings; } diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/FileMappingUtil.java b/src/main/java/org/breedinginsight/brapps/importer/services/FileMappingUtil.java index 076bf30be..b87c2a62c 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/FileMappingUtil.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/FileMappingUtil.java @@ -17,21 +17,38 @@ package org.breedinginsight.brapps.importer.services; +import io.micronaut.http.server.exceptions.InternalServerException; import org.apache.commons.lang3.tuple.MutablePair; import org.apache.commons.lang3.tuple.Pair; +import org.breedinginsight.brapi.v2.dao.BrAPIGermplasmDAO; +import org.breedinginsight.brapps.importer.daos.*; import org.breedinginsight.brapps.importer.model.config.MappedImportRelation; +import org.breedinginsight.brapps.importer.model.mapping.ImportMapping; +import org.breedinginsight.brapps.importer.model.mapping.MappingField; +import org.jooq.DSLContext; import tech.tablesaw.api.Row; import tech.tablesaw.api.Table; +import tech.tablesaw.columns.Column; +import javax.inject.Inject; import javax.inject.Singleton; import java.util.ArrayList; import java.util.HashMap; import java.util.List; import java.util.Map; +import java.util.stream.Collectors; @Singleton public class FileMappingUtil { + public static final String EXPERIMENT_TEMPLATE_NAME = "ExperimentsTemplateMap"; + private FileImportService fileImportService; + + @Inject + public FileMappingUtil(FileImportService fileImportService) { + this.fileImportService = fileImportService; + } + // Returns a list of integers to identify the target row of the relationship. -1 if no relationship was found public List> findFileRelationships(Table data, List importRelations) { @@ -63,4 +80,27 @@ public List> findFileRelationships(Table data, List> getDynamicColumns(Table data, String templateName) { + List result = fileImportService.getSystemMappingByName(templateName); + + if (result.isEmpty()) { + throw new InternalServerException("System mapping does not exist"); + } + + ImportMapping mapping = result.get(0); + List config = mapping.getMappingConfig(); + List columnNames = new ArrayList<>(); + + for (MappingField field : config) { + if (field.getValue() != null) { + columnNames.add(field.getValue().getFileFieldName()); + } + } + + List differences = data.columnNames().stream() + .filter(col -> !columnNames.contains(col)) + .collect(Collectors.toList()); + + return data.columns(differences.toArray(String[]::new)); + } } diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/ExperimentProcessor.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/ExperimentProcessor.java index 1858d2b14..574bbb83d 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/ExperimentProcessor.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/ExperimentProcessor.java @@ -44,19 +44,33 @@ import org.breedinginsight.brapps.importer.model.response.ImportPreviewStatistics; import org.breedinginsight.brapps.importer.model.response.PendingImportObject; import org.breedinginsight.brapps.importer.services.ExternalReferenceSource; +import org.breedinginsight.brapps.importer.services.FileMappingUtil; +import org.breedinginsight.dao.db.tables.pojos.TraitEntity; import org.breedinginsight.model.Program; +import org.breedinginsight.model.Scale; +import org.breedinginsight.model.Trait; import org.breedinginsight.model.User; +import org.breedinginsight.services.OntologyService; +import org.breedinginsight.services.exceptions.DoesNotExistException; import org.breedinginsight.services.exceptions.MissingRequiredInfoException; import org.breedinginsight.services.exceptions.ValidatorException; import org.breedinginsight.utilities.Utilities; import org.jooq.DSLContext; +import tech.tablesaw.api.Table; +import tech.tablesaw.columns.Column; import javax.inject.Inject; +import java.math.BigDecimal; import java.math.BigInteger; +import java.time.format.DateTimeFormatter; +import java.time.format.DateTimeParseException; import java.util.*; +import java.util.function.Function; import java.util.function.Supplier; import java.util.stream.Collectors; +import static org.breedinginsight.brapps.importer.services.FileMappingUtil.EXPERIMENT_TEMPLATE_NAME; + @Slf4j @Prototype public class ExperimentProcessor implements Processor { @@ -70,14 +84,15 @@ public class ExperimentProcessor implements Processor { @Property(name = "brapi.server.reference-source") private String BRAPI_REFERENCE_SOURCE; - - private final DSLContext dsl; - private final BrAPITrialDAO brapiTrialDAO; - private final BrAPILocationDAO brAPILocationDAO; - private final BrAPIStudyDAO brAPIStudyDAO; - private final BrAPIObservationUnitDAO brAPIObservationUnitDAO; - private final BrAPISeasonDAO brAPISeasonDAO; - private final BrAPIGermplasmDAO brAPIGermplasmDAO; + private DSLContext dsl; + private BrAPITrialDAO brapiTrialDAO; + private BrAPILocationDAO brAPILocationDAO; + private BrAPIStudyDAO brAPIStudyDAO; + private BrAPIObservationUnitDAO brAPIObservationUnitDAO; + private BrAPISeasonDAO brAPISeasonDAO; + private BrAPIGermplasmDAO brAPIGermplasmDAO; + private OntologyService ontologyService; + private FileMappingUtil fileMappingUtil; // used to make the yearsToSeasonDbId() function more efficient private final Map yearToSeasonDbIdCache = new HashMap<>(); @@ -102,7 +117,9 @@ public ExperimentProcessor(DSLContext dsl, BrAPIStudyDAO brAPIStudyDAO, BrAPIObservationUnitDAO brAPIObservationUnitDAO, BrAPISeasonDAO brAPISeasonDAO, - BrAPIGermplasmDAO brAPIGermplasmDAO) { + BrAPIGermplasmDAO brAPIGermplasmDAO, + OntologyService ontologyService, + FileMappingUtil fileMappingUtil) { this.dsl = dsl; this.brapiTrialDAO = brapiTrialDAO; this.brAPILocationDAO = brAPILocationDAO; @@ -110,6 +127,8 @@ public ExperimentProcessor(DSLContext dsl, this.brAPIObservationUnitDAO = brAPIObservationUnitDAO; this.brAPISeasonDAO = brAPISeasonDAO; this.brAPIGermplasmDAO = brAPIGermplasmDAO; + this.ontologyService = ontologyService; + this.fileMappingUtil = fileMappingUtil; } /** @@ -146,12 +165,54 @@ public void getExistingBrapiData(List importRows, Program program) public Map process( List importRows, Map mappedBrAPIImport, + Table data, Program program, User user, boolean commit) throws ValidatorException, MissingRequiredInfoException { ValidationErrors validationErrors = new ValidationErrors(); + // Get dynamic phenotype columns for processing + List> phenotypeCols = fileMappingUtil.getDynamicColumns(data, EXPERIMENT_TEMPLATE_NAME); + List varNames = phenotypeCols.stream().map(Column::name).collect(Collectors.toList()); + + // Lookup all traits in system for program, maybe eventually add a variable search in ontology service + List traits = null; + try { + traits = ontologyService.getTraitsByProgramId(program.getId(), true); + } catch (DoesNotExistException e) { + log.error(e.getMessage(), e); + throw new InternalServerException(e.toString(), e); + } + + // filter out just traits specified in file + List filteredTraits = traits.stream() + .filter(e -> varNames.contains(e.getObservationVariableName())) + .collect(Collectors.toList()); + + // check that all specified ontology terms were found + if (filteredTraits.size() != varNames.size()) { + List returnedVarNames = filteredTraits.stream().map(TraitEntity::getObservationVariableName) + .collect(Collectors.toList()); + List differences = varNames.stream() + .filter(var -> !returnedVarNames.contains(var)) + .collect(Collectors.toList()); + throw new HttpStatusException(HttpStatus.UNPROCESSABLE_ENTITY, + "Ontology term(s) not found: " + String.join(", ", differences)); + } + + // Perform ontology validations on each observation value in phenotype column + Map colVarMap = filteredTraits.stream() + .collect(Collectors.toMap(Trait::getObservationVariableName, Function.identity())); + + for (Column column : phenotypeCols) { + for (int i=0; i < column.size(); i++) { + String value = column.getString(i); + String colName = column.name(); + validateObservationValue(colVarMap.get(colName), value, colName, validationErrors, i); + } + } + // add "New" pending data to the BrapiData objects getNewBrapiData(importRows, program, commit); @@ -708,6 +769,74 @@ private String simpleStudyName(String scopedName){ return scopedName.replaceFirst(" \\[.*\\]", ""); } + private void validateObservationValue(Trait variable, String value, + String columnHeader, ValidationErrors validationErrors, int row) { + if(StringUtils.isBlank(value)) { + log.debug(String.format("skipping validation of observation because there is no value.\n\tvariable: %s\n\trow: %d", variable.getObservationVariableName(), row)); + return; + } + + switch(variable.getScale().getDataType()) { + case NUMERICAL: + Optional number = validNumericValue(value); + if (number.isEmpty()) { + addRowError(columnHeader, "Non-numeric text detected detected", validationErrors, row); + } + else if (!validNumericRange(number.get(), variable.getScale())) { + addRowError(columnHeader, "Value outside of min/max range detected", validationErrors, row); + } + break; + case DATE: + if (!validDateValue(value)) { + addRowError(columnHeader, "Incorrect date format detected. Expected YYYY-MM-DD", validationErrors, row); + } + break; + case ORDINAL: + if (!validCategory(variable.getScale().getCategories(), value)) { + addRowError(columnHeader, "Undefined ordinal category detected", validationErrors, row); + } + break; + case NOMINAL: + if (!validCategory(variable.getScale().getCategories(), value)) { + addRowError(columnHeader, "Undefined nominal category detected", validationErrors, row); + } + break; + default: + break; + } + + } + private Optional validNumericValue(String value) { + BigDecimal number; + try { + number = new BigDecimal(value); + } catch (NumberFormatException e) { + return Optional.empty(); + } + return Optional.of(number); + } + + private boolean validNumericRange(BigDecimal value, Scale validValues) { + // account for empty min or max in valid determination + return (validValues.getValidValueMin() == null || value.compareTo(BigDecimal.valueOf(validValues.getValidValueMin())) >= 0) && + (validValues.getValidValueMax() == null || value.compareTo(BigDecimal.valueOf(validValues.getValidValueMax())) <= 0); + } + + private boolean validDateValue(String value) { + DateTimeFormatter formatter = DateTimeFormatter.ISO_DATE; + try { + formatter.parse(value); + } catch (DateTimeParseException e) { + return false; + } + return true; + } + + private boolean validCategory(List categories, String value) { + Set categoryValues = categories.stream().map(category -> category.getValue().toLowerCase()).collect(Collectors.toSet()); + return categoryValues.contains(value.toLowerCase()); + } + /** * Converts year String to SeasonDbId *
diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/GermplasmProcessor.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/GermplasmProcessor.java index de32e6b70..1f97df9d8 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/GermplasmProcessor.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/GermplasmProcessor.java @@ -44,6 +44,7 @@ import org.breedinginsight.model.User; import org.breedinginsight.services.exceptions.ValidatorException; import org.jooq.DSLContext; +import tech.tablesaw.api.Table; import javax.inject.Inject; import java.math.BigInteger; @@ -205,7 +206,8 @@ public void getExistingBrapiData(List importRows, Program program) @Override public Map process(List importRows, - Map mappedBrAPIImport, Program program, User user, boolean commit) throws ValidatorException { + Map mappedBrAPIImport, Table data, + Program program, User user, boolean commit) throws ValidatorException { // Method for generating accession number String germplasmSequenceName = program.getGermplasmSequence(); diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/LocationProcessor.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/LocationProcessor.java index 730f162bd..820fba4ab 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/LocationProcessor.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/LocationProcessor.java @@ -31,6 +31,7 @@ import org.breedinginsight.model.Program; import org.breedinginsight.model.User; import org.breedinginsight.services.exceptions.ValidatorException; +import tech.tablesaw.api.Table; import javax.inject.Inject; import java.util.ArrayList; @@ -74,7 +75,8 @@ public void getExistingBrapiData(List importRows, Program program) @Override public Map process(List importRows, - Map mappedBrAPIImport, Program program, User user, boolean commit) throws ValidatorException { + Map mappedBrAPIImport, Table data, + Program program, User user, boolean commit) throws ValidatorException { for (int i = 0; i < importRows.size(); i++) { BrAPIImport brapiImport = importRows.get(i); diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/ObservationProcessor.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/ObservationProcessor.java index 0a2b59108..8dc9e147d 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/ObservationProcessor.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/ObservationProcessor.java @@ -38,6 +38,7 @@ import org.breedinginsight.model.Program; import org.breedinginsight.model.User; import org.breedinginsight.services.exceptions.ValidatorException; +import tech.tablesaw.api.Table; import javax.inject.Inject; import java.time.ZoneOffset; @@ -142,7 +143,8 @@ private void getDependentDbIds(List importRows, Program program) { @Override public Map process(List importRows, - Map mappedBrAPIImport, Program program, User user, boolean commit) throws ValidatorException { + Map mappedBrAPIImport, Table data, + Program program, User user, boolean commit) throws ValidatorException { if (!importRows.isEmpty() && importRows.get(0).getObservations() != null) { diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/ObservationUnitProcessor.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/ObservationUnitProcessor.java index e024fc121..ff283f58c 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/ObservationUnitProcessor.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/ObservationUnitProcessor.java @@ -35,6 +35,7 @@ import org.breedinginsight.model.Program; import org.breedinginsight.model.User; import org.breedinginsight.services.exceptions.ValidatorException; +import tech.tablesaw.api.Table; import javax.inject.Inject; import java.util.*; @@ -88,7 +89,8 @@ private void getExistingStudyObjects() { @Override public Map process(List importRows, - Map mappedBrAPIImport, Program program, User user, boolean commit) { + Map mappedBrAPIImport, Table data, + Program program, User user, boolean commit) { for (int i = 0; i < importRows.size(); i++) { BrAPIImport brapiImport = importRows.get(i); diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/Processor.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/Processor.java index 2ab32564f..9c72ea4f4 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/Processor.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/Processor.java @@ -25,6 +25,7 @@ import org.breedinginsight.model.User; import org.breedinginsight.services.exceptions.MissingRequiredInfoException; import org.breedinginsight.services.exceptions.ValidatorException; +import tech.tablesaw.api.Table; import java.util.List; import java.util.Map; @@ -44,13 +45,15 @@ public interface Processor { * Return stats on number of new & existing objects * @param importRows * @param mappedBrAPIImport + * @param data * @param program * @return * @throws ValidatorException */ Map process(List importRows, - Map mappedBrAPIImport, - Program program, User user, boolean commit) throws ValidatorException, MissingRequiredInfoException; + Map mappedBrAPIImport, Table data, + Program program, User user, boolean commit) + throws ValidatorException, MissingRequiredInfoException; /** * Given mapped brapi import with updates from prior dependencies, check if have everything needed diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/ProcessorManager.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/ProcessorManager.java index c8bad7218..cc281df34 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/ProcessorManager.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/ProcessorManager.java @@ -29,6 +29,7 @@ import org.breedinginsight.model.User; import org.breedinginsight.services.exceptions.MissingRequiredInfoException; import org.breedinginsight.services.exceptions.ValidatorException; +import tech.tablesaw.api.Table; import javax.inject.Inject; import java.util.ArrayList; @@ -52,7 +53,7 @@ public ProcessorManager(ImportStatusService statusService) { this.statusService = statusService; } - public ImportPreviewResponse process(List importRows, List processors, Program program, ImportUpload upload, User user, boolean commit) throws ValidatorException, ApiException, MissingRequiredInfoException { + public ImportPreviewResponse process(List importRows, List processors, Table data, Program program, ImportUpload upload, User user, boolean commit) throws ValidatorException, ApiException, MissingRequiredInfoException { this.processors = processors; @@ -60,7 +61,7 @@ public ImportPreviewResponse process(List importRows, List stats = processor.process(importRows, mappedBrAPIImport, program, user, commit); + Map stats = processor.process(importRows, mappedBrAPIImport, data, program, user, commit); statistics.putAll(stats); } diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/StudyProcessor.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/StudyProcessor.java index 39cca38a8..4a041cad9 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/StudyProcessor.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/StudyProcessor.java @@ -35,6 +35,7 @@ import org.breedinginsight.model.Program; import org.breedinginsight.model.User; import org.breedinginsight.services.exceptions.ValidatorException; +import tech.tablesaw.api.Table; import javax.inject.Inject; import java.util.*; @@ -79,7 +80,8 @@ public void getExistingBrapiData(List importRows, Program program) @Override public Map process(List importRows, - Map mappedBrAPIImport, Program program, User user, boolean commit) { + Map mappedBrAPIImport, Table data, + Program program, User user, boolean commit) { for (int i = 0; i < importRows.size(); i++) { BrAPIImport brapiImport = importRows.get(i); diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/TrialProcessor.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/TrialProcessor.java index 67a29adbb..18ee6d37c 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/TrialProcessor.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/TrialProcessor.java @@ -31,6 +31,7 @@ import org.breedinginsight.model.Program; import org.breedinginsight.model.User; import org.breedinginsight.services.exceptions.ValidatorException; +import tech.tablesaw.api.Table; import javax.inject.Inject; import java.util.ArrayList; @@ -74,7 +75,8 @@ public void getExistingBrapiData(List importRows, Program program) @Override public Map process(List importRows, - Map mappedBrAPIImport, Program program, User user, boolean commit) throws ValidatorException { + Map mappedBrAPIImport, Table data, + Program program, User user, boolean commit) throws ValidatorException { getExistingBrapiData(importRows, program); diff --git a/src/main/java/org/breedinginsight/daos/ObservationDAO.java b/src/main/java/org/breedinginsight/daos/ObservationDAO.java index 9404937b5..76e6c284c 100644 --- a/src/main/java/org/breedinginsight/daos/ObservationDAO.java +++ b/src/main/java/org/breedinginsight/daos/ObservationDAO.java @@ -20,6 +20,7 @@ import lombok.extern.slf4j.Slf4j; import org.apache.commons.lang3.tuple.Pair; import org.brapi.client.v2.ApiResponse; +import org.brapi.client.v2.BrAPIClient; import org.brapi.client.v2.model.exceptions.ApiException; import org.brapi.client.v2.model.queryParams.phenotype.ObservationQueryParams; import org.brapi.client.v2.modules.phenotype.ObservationsApi; @@ -44,22 +45,24 @@ @Singleton @Slf4j public class ObservationDAO { - private BrAPIProvider brAPIProvider; private final BrAPIDAOUtil brAPIDAOUtil; + private ProgramDAO programDAO; @Inject - public ObservationDAO(BrAPIProvider brAPIProvider, BrAPIDAOUtil brAPIDAOUtil) { - this.brAPIProvider = brAPIProvider; + public ObservationDAO(BrAPIDAOUtil brAPIDAOUtil, ProgramDAO programDAO) { this.brAPIDAOUtil = brAPIDAOUtil; + this.programDAO = programDAO; } - public List getObservationsByVariableDbId(String observationVariableDbId) { + public List getObservationsByVariableDbId(String observationVariableDbId, UUID programId) { ApiResponse brapiObservations; ObservationQueryParams observationsRequest = new ObservationQueryParams() .observationVariableDbId(observationVariableDbId); try { - brapiObservations = brAPIProvider.getObservationsAPI(BrAPIClientType.PHENO).observationsGet(observationsRequest); + BrAPIClient client = programDAO.getCoreClient(programId); + ObservationsApi api = new ObservationsApi(client); + brapiObservations = api.observationsGet(observationsRequest); } catch (ApiException e) { log.warn(Utilities.generateApiExceptionLogMessage(e)); throw new InternalServerException("Error making BrAPI call", e); @@ -69,16 +72,17 @@ public List getObservationsByVariableDbId(String observationVa } // search by ObservationVariableDbIds - public List getObservationsByVariableDbIds(List observationVariableDbIds) { + public List getObservationsByVariableDbIds(List observationVariableDbIds, UUID programId) { try { BrAPIObservationSearchRequest request = new BrAPIObservationSearchRequest() .observationVariableDbIds(observationVariableDbIds); - ObservationsApi api = brAPIProvider.getObservationsAPI(BrAPIClientType.PHENO); + BrAPIClient client = programDAO.getCoreClient(programId); + ObservationsApi api = new ObservationsApi(client); return brAPIDAOUtil.search( api::searchObservationsPost, - this::searchObservationsSearchResultsDbIdGet, + api::searchObservationsSearchResultsDbIdGet, request ); } catch (ApiException e) { @@ -87,17 +91,18 @@ public List getObservationsByVariableDbIds(List observ } - public List getObservationsByVariableAndBrAPIProgram(String brapiProgramId, List observationVariableDbIds) { + public List getObservationsByVariableAndBrAPIProgram(String brapiProgramId, UUID programId, List observationVariableDbIds) { try { BrAPIObservationSearchRequest request = new BrAPIObservationSearchRequest() .observationVariableDbIds(observationVariableDbIds) .programDbIds(List.of(brapiProgramId)); - ObservationsApi api = brAPIProvider.getObservationsAPI(BrAPIClientType.PHENO); + BrAPIClient client = programDAO.getCoreClient(programId); + ObservationsApi api = new ObservationsApi(client); return brAPIDAOUtil.search( api::searchObservationsPost, - this::searchObservationsSearchResultsDbIdGet, + api::searchObservationsSearchResultsDbIdGet, request ); } catch (ApiException e) { @@ -106,10 +111,4 @@ public List getObservationsByVariableAndBrAPIProgram(String br } - private ApiResponse, Optional>> - searchObservationsSearchResultsDbIdGet(String searchResultsDbId, Integer page, Integer pageSize) throws ApiException { - ObservationsApi api = brAPIProvider.getObservationsAPI(BrAPIClientType.PHENO); - return api.searchObservationsSearchResultsDbIdGet(APPLICATION_JSON, searchResultsDbId, page, pageSize); - } - } diff --git a/src/main/java/org/breedinginsight/daos/TraitDAO.java b/src/main/java/org/breedinginsight/daos/TraitDAO.java index c0cd1b5b7..4a14f94f0 100644 --- a/src/main/java/org/breedinginsight/daos/TraitDAO.java +++ b/src/main/java/org/breedinginsight/daos/TraitDAO.java @@ -28,13 +28,14 @@ public interface TraitDAO extends DAO { // could be more efficient to do a single get instead of search in saved search case but less code this way // and search stuff is working in breedbase - List getObservationsForTrait(UUID traitId); - List getObservationsForTraits(List traitIds); + List getObservationsForTrait(UUID traitId, UUID programId); - List getObservationsForTraitsByBrAPIProgram(String brapiProgramId, List traitIds); + List getObservationsForTraits(List traitIds, UUID programId); - List searchVariables(List variableIds); + List getObservationsForTraitsByBrAPIProgram(String brapiProgramId, UUID programId, List traitIds); + + List searchVariables(List variableIds, UUID programId); Optional getTrait(UUID programId, UUID traitId); diff --git a/src/main/java/org/breedinginsight/daos/cache/ProgramCache.java b/src/main/java/org/breedinginsight/daos/cache/ProgramCache.java index 6dd2f78aa..be7168c0d 100644 --- a/src/main/java/org/breedinginsight/daos/cache/ProgramCache.java +++ b/src/main/java/org/breedinginsight/daos/cache/ProgramCache.java @@ -38,8 +38,6 @@ public class ProgramCache { private final RedissonClient connection; private final Gson gson; private final FetchFunction> fetchMethod; - private final Map programSemaphore = new HashMap<>(); - private final Map programQueueSemaphore = new HashMap<>(); private Class type; private final Executor executor = Executors.newCachedThreadPool(); @@ -58,17 +56,13 @@ public void populate(List keys) { public void populate(@NotNull UUID key) { String cacheKey = generateCacheKey(key); - if (!programSemaphore.containsKey(cacheKey)) { - RSemaphore semaphore = connection.getSemaphore(cacheKey+":semaphore"); - semaphore.trySetPermits(1); - programSemaphore.put(cacheKey, semaphore); - - RSemaphore queueSemaphore = connection.getSemaphore(cacheKey+":semaphore:queue"); - queueSemaphore.trySetPermits(1); - programQueueSemaphore.put(cacheKey, queueSemaphore); - } + RSemaphore semaphore = connection.getSemaphore(cacheKey+":semaphore"); + semaphore.trySetPermits(1); + + RSemaphore queueSemaphore = connection.getSemaphore(cacheKey+":semaphore:queue"); + queueSemaphore.trySetPermits(1); - boolean acquired = programSemaphore.get(cacheKey).tryAcquire(); + boolean acquired = semaphore.tryAcquire(); boolean refresh = true; if(!acquired) { @@ -77,16 +71,19 @@ public void populate(@NotNull UUID key) { If there is already a thread in line, let this thread finish as the next refresh will pick up data persisted by this thread */ - if(programQueueSemaphore.get(cacheKey).tryAcquire()) { + if(queueSemaphore.tryAcquire()) { + log.debug("Attempting to refresh"); try { // block until we get the green light to refresh the cache - programSemaphore.get(cacheKey).acquire(); + semaphore.acquire(); // and let go of our hold on the refresh queue - programQueueSemaphore.get(cacheKey).release(); log.debug("repopulating cache for " + cacheKey); } catch (InterruptedException e) { log.error("Error acquiring lock to refresh "+cacheKey, e); throw new RuntimeException(e); + } finally { + log.debug("Released queue semaphore: "+cacheKey); + queueSemaphore.release(); } } else { log.debug("A refresh is queued up for key: "+cacheKey+", leaving"); @@ -120,8 +117,9 @@ public void populate(@NotNull UUID key) { invalidate(key); throw new InternalServerException(e.getMessage(), e); } finally { + log.debug("Releasing semaphore: " + cacheKey); connection.getAtomicLong(cacheKey+":refreshing").set(0); - programSemaphore.get(cacheKey).release(); + semaphore.release(); } }); } @@ -138,15 +136,22 @@ public void invalidate(@NotNull UUID key) { public Map get(UUID key) throws ApiException { String cacheKey = generateCacheKey(key); log.debug("Getting for key: " + cacheKey); - - try { - if (!connection.getBucket(cacheKey).isExists()) { + if (!connection.getBucket(cacheKey).isExists()) { + RSemaphore semaphore = connection.getSemaphore(cacheKey + ":semaphore"); + try { log.debug("cache miss, populating"); populate(key); //block until any updates are done - programSemaphore.get(cacheKey).acquire(); - programSemaphore.get(cacheKey).release(); + semaphore.acquire(); + log.debug("Cache loading done!!!!"); + } catch(Exception e){ + throw new ApiException(e); + } finally { + semaphore.release(); } + } + + try { return deserialize(connection.getMap(cacheKey)); } catch (Exception e) { throw new ApiException(e); diff --git a/src/main/java/org/breedinginsight/daos/impl/ProgramDAOImpl.java b/src/main/java/org/breedinginsight/daos/impl/ProgramDAOImpl.java index ae3b6f699..ed11d265a 100644 --- a/src/main/java/org/breedinginsight/daos/impl/ProgramDAOImpl.java +++ b/src/main/java/org/breedinginsight/daos/impl/ProgramDAOImpl.java @@ -279,7 +279,8 @@ public BrAPIProgram getProgramBrAPI(Program program) { .externalReferenceID(program.getId().toString()) .externalReferenceSource(referenceSource); - ProgramsApi programsApi = brAPIProvider.getProgramsAPI(BrAPIClientType.CORE); + BrAPIClient client = getCoreClient(program.getId()); + ProgramsApi programsApi = new ProgramsApi(client); // Get existing brapi program ApiResponse brApiPrograms; try { diff --git a/src/main/java/org/breedinginsight/daos/impl/TraitDAOImpl.java b/src/main/java/org/breedinginsight/daos/impl/TraitDAOImpl.java index e86944c25..e221e568a 100644 --- a/src/main/java/org/breedinginsight/daos/impl/TraitDAOImpl.java +++ b/src/main/java/org/breedinginsight/daos/impl/TraitDAOImpl.java @@ -26,6 +26,7 @@ import io.micronaut.scheduling.annotation.Scheduled; import lombok.extern.slf4j.Slf4j; import org.brapi.client.v2.ApiResponse; +import org.brapi.client.v2.BrAPIClient; import org.brapi.client.v2.model.exceptions.ApiException; import org.brapi.client.v2.model.queryParams.phenotype.VariableQueryParams; import org.brapi.client.v2.modules.phenotype.ObservationVariablesApi; @@ -251,18 +252,18 @@ public Optional getTraitFull(UUID programId, UUID traitId){ // could be more efficient to do a single get instead of search in saved search case but less code this way // and search stuff is working in breedbase @Override - public List getObservationsForTrait(UUID traitId) { - return getObservationsForTraits(Stream.of(traitId).collect(Collectors.toList())); + public List getObservationsForTrait(UUID traitId, UUID programId) { + return getObservationsForTraits(Stream.of(traitId).collect(Collectors.toList()), programId); } @Override - public List getObservationsForTraits(List traitIds) { + public List getObservationsForTraits(List traitIds, UUID programId) { List ids = traitIds.stream() .map(UUID::toString) .collect(Collectors.toList()); - List variables = searchVariables(ids); + List variables = searchVariables(ids, programId); // TODO: make sure have all expected external references if (variables.size() != ids.size()) { @@ -272,17 +273,17 @@ public List getObservationsForTraits(List traitIds) { List brapiVariableIds = variables.stream() .map(BrAPIObservationVariable::getObservationVariableDbId).collect(Collectors.toList()); - return observationDao.getObservationsByVariableDbIds(brapiVariableIds); + return observationDao.getObservationsByVariableDbIds(brapiVariableIds, programId); } @Override - public List getObservationsForTraitsByBrAPIProgram(String brapiProgramId, List traitIds) { + public List getObservationsForTraitsByBrAPIProgram(String brapiProgramId, UUID programId, List traitIds) { List ids = traitIds.stream() .map(UUID::toString) .collect(Collectors.toList()); - List variables = searchVariables(ids); + List variables = searchVariables(ids, programId); // TODO: make sure have all expected external references if (variables.size() != ids.size()) { @@ -292,18 +293,20 @@ public List getObservationsForTraitsByBrAPIProgram(String brap List brapiVariableIds = variables.stream() .map(BrAPIObservationVariable::getObservationVariableDbId).collect(Collectors.toList()); - return observationDao.getObservationsByVariableAndBrAPIProgram(brapiProgramId, brapiVariableIds); + return observationDao.getObservationsByVariableAndBrAPIProgram(brapiProgramId, programId, brapiVariableIds); } @Override - public List searchVariables(List variableIds) { + public List searchVariables(List variableIds, UUID programId) { if (variableIds == null || variableIds.size() == 0) return new ArrayList<>(); try { BrAPIObservationVariableSearchRequest request = new BrAPIObservationVariableSearchRequest() .externalReferenceIDs(variableIds); - ObservationVariablesApi api = brAPIProvider.getVariablesAPI(PHENO); + BrAPIClient client = programDAO.getCoreClient(programId); + ObservationVariablesApi api = new ObservationVariablesApi(client); + return brAPIDAOUtil.search( api::searchVariablesPost, api::searchVariablesSearchResultsDbIdGet, diff --git a/src/main/java/org/breedinginsight/services/OntologyService.java b/src/main/java/org/breedinginsight/services/OntologyService.java index 537d1c127..a801115d8 100644 --- a/src/main/java/org/breedinginsight/services/OntologyService.java +++ b/src/main/java/org/breedinginsight/services/OntologyService.java @@ -141,7 +141,7 @@ private Boolean ontologyIsEditable(ProgramSharedOntologyEntity sharedOntologyEnt BrAPIProgram brAPIProgram = programDAO.getProgramBrAPI(program.get(0)); // Get all observations for the ontology - return traitDAO.getObservationsForTraitsByBrAPIProgram(brAPIProgram.getProgramDbId(), traitIds).isEmpty(); + return traitDAO.getObservationsForTraitsByBrAPIProgram(brAPIProgram.getProgramDbId(), program.get(0).getId(), traitIds).isEmpty(); } else { return true; } diff --git a/src/main/java/org/breedinginsight/services/TraitService.java b/src/main/java/org/breedinginsight/services/TraitService.java index 2a8399e45..f2b2953aa 100644 --- a/src/main/java/org/breedinginsight/services/TraitService.java +++ b/src/main/java/org/breedinginsight/services/TraitService.java @@ -137,7 +137,7 @@ public Editable getEditable(UUID programId, UUID traitId) { throw new HttpStatusException(HttpStatus.NOT_FOUND, "traitId does not exist"); } - List observations = traitDAO.getObservationsForTrait(traitId); + List observations = traitDAO.getObservationsForTrait(traitId, programId); return Editable.builder().editable(observations.isEmpty()).build(); } @@ -380,7 +380,7 @@ public List updateTraits(UUID programId, List traits, Authenticate .map(trait -> trait.getId()) .collect(Collectors.toList()); - if (!traitDAO.getObservationsForTraits(ids).isEmpty()) { + if (!traitDAO.getObservationsForTraits(ids, program.getId()).isEmpty()) { throw new HttpStatusException(HttpStatus.METHOD_NOT_ALLOWED, "Observations exist for trait, cannot edit"); } diff --git a/src/main/java/org/breedinginsight/utilities/BrAPIDAOUtil.java b/src/main/java/org/breedinginsight/utilities/BrAPIDAOUtil.java index a669c7dae..6a302638d 100644 --- a/src/main/java/org/breedinginsight/utilities/BrAPIDAOUtil.java +++ b/src/main/java/org/breedinginsight/utilities/BrAPIDAOUtil.java @@ -22,6 +22,7 @@ import io.reactivex.functions.Consumer; import io.reactivex.functions.Function; import io.reactivex.functions.Function3; +import io.reactivex.functions.Function4; import lombok.extern.slf4j.Slf4j; import org.apache.commons.lang3.tuple.Pair; import org.brapi.client.v2.ApiResponse; @@ -35,6 +36,8 @@ import java.util.List; import java.util.Optional; +import static org.brapi.v2.model.BrAPIWSMIMEDataTypes.APPLICATION_JSON; + @Singleton @Slf4j public class BrAPIDAOUtil { @@ -47,12 +50,24 @@ public class BrAPIDAOUtil { private int pageSize; @Property(name = "brapi.post-group-size") private int postGroupSize; + public List search(Function, Optional>>> searchMethod, + Function3, Optional>>> searchGetMethod, + U searchBody + ) throws ApiException { + return searchInternal(searchMethod, searchGetMethod, null, searchBody); + } public List search(Function, Optional>>> searchMethod, - Function3, Optional>>> searchGetMethod, - U searchBody + Function4, Optional>>> searchGetMethod, + U searchBody ) throws ApiException { + return searchInternal(searchMethod, null, searchGetMethod, searchBody); + } + private List searchInternal(Function, Optional>>> searchMethod, + Function3, Optional>>> searchGetMethod, + Function4, Optional>>> searchGetMethodWithMimeType, + U searchBody) throws ApiException { try { List listResult = new ArrayList<>(); //NOTE: Because of the way Breedbase implements BrAPI searches, the page size is initially set to an @@ -90,7 +105,8 @@ public List search(Funct while (!searchFinished) { BrAPIAcceptedSearchResponse searchResult = response.getBody().getRight().get(); - ApiResponse, Optional>> searchGetResponse = searchGetMethod.apply(searchResult.getResult().getSearchResultsDbId(), currentPage, pageSize); + ApiResponse, Optional>> searchGetResponse = + searchGetResponse(searchGetMethod, searchGetMethodWithMimeType, searchResult, currentPage); if (searchGetResponse.getBody().getLeft().isPresent()) { searchFinished = true; BrAPIResponse listResponse = (BrAPIResponse) searchGetResponse.getBody().getLeft().get(); @@ -99,11 +115,11 @@ public List search(Funct if(hasMorePages(listResponse)) { currentPage++; int totalPages = listResponse.getMetadata() - .getPagination() - .getTotalPages(); + .getPagination() + .getTotalPages(); while (currentPage < totalPages) { - searchGetResponse = searchGetMethod.apply(searchResult.getResult().getSearchResultsDbId(), currentPage, pageSize); + searchGetResponse = searchGetResponse(searchGetMethod, searchGetMethodWithMimeType, searchResult, currentPage); if (searchGetResponse.getBody().getLeft().isPresent()) { listResult.addAll(getListResult(searchGetResponse)); } @@ -131,6 +147,14 @@ public List search(Funct } } + private ApiResponse, Optional>> searchGetResponse(Function3, Optional>>> searchGetMethod, + Function4, Optional>>> searchGetMethodWithMimeType, + BrAPIAcceptedSearchResponse searchResult, + int currentPage) throws Exception{ + return searchGetMethod != null ? searchGetMethod.apply(searchResult.getResult().getSearchResultsDbId(), currentPage, pageSize) : + searchGetMethodWithMimeType.apply(APPLICATION_JSON, searchResult.getResult().getSearchResultsDbId(), currentPage, pageSize); + } + private boolean hasMorePages(BrAPIResponse listResponse) { return listResponse.getMetadata() != null && listResponse.getMetadata().getPagination() != null diff --git a/src/main/java/org/breedinginsight/utilities/FileUtil.java b/src/main/java/org/breedinginsight/utilities/FileUtil.java index 542220051..6427bca49 100644 --- a/src/main/java/org/breedinginsight/utilities/FileUtil.java +++ b/src/main/java/org/breedinginsight/utilities/FileUtil.java @@ -85,9 +85,13 @@ public static Table parseTableFromExcel(InputStream inputStream, Integer headerR // Read from the excel header row to get proper order Table table = Table.create(); Iterator headerIterator = headerRow.cellIterator(); + HashSet colNames = new HashSet<>(); while (headerIterator.hasNext()) { Cell cell = headerIterator.next(); StringColumn column = StringColumn.create(formatter.formatCellValue(cell), columns.get(formatter.formatCellValue(cell))); + if (!colNames.add(column.name())) { + throw new ParsingException(ParsingExceptionType.DUPLICATE_COLUMN_NAMES); + } table.addColumns(column); } diff --git a/src/main/resources/db/migration/V1.0.8__update_experiment_obs_id_template_system_mapping.sql b/src/main/resources/db/migration/V1.0.8__update_experiment_obs_id_template_system_mapping.sql new file mode 100644 index 000000000..d1ac21bd2 --- /dev/null +++ b/src/main/resources/db/migration/V1.0.8__update_experiment_obs_id_template_system_mapping.sql @@ -0,0 +1,26 @@ +/* + * See the NOTICE file distributed with this work for additional information + * regarding copyright ownership. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +DO $$ + +BEGIN +update importer_mapping +set + mapping = '[{"id": "726a9f10-4892-4204-9e52-bd2b1d735f65", "value": {"fileFieldName": "Germplasm Name"}, "objectId": "germplasmName"}, {"id": "98774e20-6f89-4d6a-a7c9-f88887228ed6", "value": {"fileFieldName": "Germplasm GID"}, "objectId": "gid"}, {"id": "880ef0c9-4e3e-42d4-9edc-667684a91889", "value": {"fileFieldName": "Test (T) or Check (C )"}, "objectId": "test_or_check"}, {"id": "b693eca7-efcd-4518-a9d3-db0b037a76ee", "value": {"fileFieldName": "Exp Title"}, "objectId": "exp_title"}, {"id": "df340215-db6e-4219-a3b7-119f297b81c3", "value": {"fileFieldName": "Exp Description"}, "objectId": "expDescription"}, {"id": "9ca7cc81-562c-43a7-989a-41da309f603d", "value": {"fileFieldName": "Exp Unit"}, "objectId": "expUnit"}, {"id": "27215777-c8f9-4fe7-a7ac-918d6168b0dd", "value": {"fileFieldName": "Exp Type"}, "objectId": "expType"}, {"id": "19d220e2-dff0-4a3a-ad6e-32f4d8602b5c", "value": {"fileFieldName": "Env"}, "objectId": "env"}, {"id": "861518b9-5c9e-4fe5-b31e-baf16e27155d", "value": {"fileFieldName": "Env Location"}, "objectId": "envLocation"}, {"id": "667355c3-dae1-4a64-94c8-ac2d543bd474", "value": {"fileFieldName": "Env Year"}, "objectId": "envYear"}, {"id": "ad11f2df-c5b4-4a05-8e52-c57625140061", "value": {"fileFieldName": "Exp Unit ID"}, "objectId": "expUnitId"}, {"id": "639b40ec-20f8-4659-8464-6a4be997ac7a", "value": {"fileFieldName": "Exp Replicate #"}, "objectId": "expReplicateNo"}, {"id": "2a62a80f-d8ba-42c4-9997-3b4ac8a965aa", "value": {"fileFieldName": "Exp Block #"}, "objectId": "expBlockNo"}, {"id": "f3e7de69-21ad-4cda-b1cc-a5e1987fb931", "value": {"fileFieldName": "Row"}, "objectId": "row"}, {"id": "251c5bbd-fc4d-4371-a4ce-4e2686f6837e", "value": {"fileFieldName": "Column"}, "objectId": "column"}, {"id": "ce5f61f2-f1de-45a4-8baf-e2471a5d863d", "value": {"fileFieldName": "Treatment Factors"}, "objectId": "treatmentFactors"}, {"id": "c5b8276f-e777-4385-a80f-5199abe63aac", "value": {"fileFieldName": "ObsUnitID"}, "objectId": "ObsUnitID"}]', + file = '[{"Env": "New Study", "Row": 4, "Column": 5, "Env Year": 2012, "Exp Type": "phenotyping", "Exp Unit": "plot", "Exp Title": "New Trial", "ObsUnitID": "", "Exp Block #": 2, "Exp Unit ID": 3, "Env Location": "New Location", "Germplasm GID": 1, "Germplasm Name": "Test", "Exp Description": "A new trial", "Exp Replicate #": 0, "Treatment Factors": "Jam application", "Test (T) or Check (C )": true}]' +where import_type_id = 'ExperimentImport'; +END $$; \ No newline at end of file diff --git a/src/test/java/org/breedinginsight/api/v1/controller/brapi/BrAPIServiceFilterIntegrationTest.java b/src/test/java/org/breedinginsight/api/v1/controller/brapi/BrAPIServiceFilterIntegrationTest.java index 35d120933..90e624022 100644 --- a/src/test/java/org/breedinginsight/api/v1/controller/brapi/BrAPIServiceFilterIntegrationTest.java +++ b/src/test/java/org/breedinginsight/api/v1/controller/brapi/BrAPIServiceFilterIntegrationTest.java @@ -262,21 +262,14 @@ public void urlChangesForDifferentRequestsCallTwo() { public void getTraitSingleEditableUsesFilter() { String phenoUrl = "http://getTraitSingle" + UUID.randomUUID().toString() + "/brapi/v2"; - ProgramBrAPIEndpoints programBrAPIEndpoints = getBrAPIEndpoints(null, phenoUrl, null); + ProgramBrAPIEndpoints programBrAPIEndpoints = getBrAPIEndpoints(phenoUrl, phenoUrl, phenoUrl); reset(programService); when(programService.getBrapiEndpoints(any(UUID.class))).thenReturn(programBrAPIEndpoints); when(programService.exists(any(UUID.class))).thenReturn(true); - validProgram.setBrapiUrl(phenoUrl); - programDAO.update(validProgram); - - when(brAPIClientProvider.getClient(BrAPIClientType.PHENO)).thenAnswer((Answer) invocation -> { - // Create a spy on our real brapi client to see what url was ultimately used - BrAPIClient realBrAPIClient = (BrAPIClient) invocation.callRealMethod(); - brAPIClient.setBasePath(realBrAPIClient.getBasePath()); - return brAPIClient; - }); + brAPIClient.setBasePath(phenoUrl); + doReturn(brAPIClient).when(programDAO).getCoreClient(any(UUID.class)); Flowable> call = client.exchange( GET("/programs/" + validProgram.getId() + "/traits/" + validVariable.getId()+"/editable")