diff --git a/src/main/java/org/breedinginsight/brapi/v2/services/BrAPITrialService.java b/src/main/java/org/breedinginsight/brapi/v2/services/BrAPITrialService.java index 4faa89dba..c66671355 100644 --- a/src/main/java/org/breedinginsight/brapi/v2/services/BrAPITrialService.java +++ b/src/main/java/org/breedinginsight/brapi/v2/services/BrAPITrialService.java @@ -17,6 +17,7 @@ import org.breedinginsight.brapps.importer.model.exports.FileType; import org.breedinginsight.brapps.importer.model.imports.experimentObservation.ExperimentObservation; import org.breedinginsight.brapps.importer.services.ExternalReferenceSource; +import org.breedinginsight.brapps.importer.services.FileMappingUtil; import org.breedinginsight.model.BrAPIConstants; import org.breedinginsight.model.Column; import org.breedinginsight.model.DownloadFile; @@ -46,6 +47,7 @@ public class BrAPITrialService { private final BrAPISeasonDAO seasonDAO; private final BrAPIObservationUnitDAO ouDAO; private final BrAPIGermplasmDAO germplasmDAO; + private final FileMappingUtil fileMappingUtil; @Inject public BrAPITrialService(@Property(name = "brapi.server.reference-source") String referenceSource, @@ -56,7 +58,8 @@ public BrAPITrialService(@Property(name = "brapi.server.reference-source") Strin BrAPIStudyDAO studyDAO, BrAPISeasonDAO seasonDAO, BrAPIObservationUnitDAO ouDAO, - BrAPIGermplasmDAO germplasmDAO) { + BrAPIGermplasmDAO germplasmDAO, + FileMappingUtil fileMappingUtil) { this.referenceSource = referenceSource; this.trialDAO = trialDAO; @@ -67,6 +70,7 @@ public BrAPITrialService(@Property(name = "brapi.server.reference-source") Strin this.seasonDAO = seasonDAO; this.ouDAO = ouDAO; this.germplasmDAO = germplasmDAO; + this.fileMappingUtil = fileMappingUtil; } public List getExperiments(UUID programId) throws ApiException, DoesNotExistException { @@ -275,7 +279,10 @@ public List getDatasetObsVars(String datasetId, Progra String listDbId = lists.get(0).getListDbId(); BrAPIListsSingleResponse list = listDAO.getListById(listDbId, program.getId()); List obsVarNames = list.getResult().getData(); - return obsVarDAO.getVariableByName(obsVarNames, program.getId()); + List obsVars = obsVarDAO.getVariableByName(obsVarNames, program.getId()); + + // sort the obsVars to match the order stored in the dataset list + return fileMappingUtil.sortByField(obsVarNames, obsVars, BrAPIObservationVariable::getObservationVariableName); } public BrAPITrial getExperiment(Program program, UUID experimentId) throws ApiException { diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/FileMappingUtil.java b/src/main/java/org/breedinginsight/brapps/importer/services/FileMappingUtil.java index b70c11f30..e26f375c5 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/FileMappingUtil.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/FileMappingUtil.java @@ -17,23 +17,19 @@ package org.breedinginsight.brapps.importer.services; -import io.micronaut.http.server.exceptions.InternalServerException; +import io.reactivex.functions.Function; import org.apache.commons.lang3.tuple.MutablePair; import org.apache.commons.lang3.tuple.Pair; -import org.breedinginsight.brapi.v2.dao.BrAPIGermplasmDAO; -import org.breedinginsight.brapps.importer.daos.*; import org.breedinginsight.brapps.importer.model.config.MappedImportRelation; -import org.breedinginsight.brapps.importer.model.mapping.ImportMapping; -import org.breedinginsight.brapps.importer.model.mapping.MappingField; -import org.jooq.DSLContext; import tech.tablesaw.api.Row; import tech.tablesaw.api.Table; -import tech.tablesaw.columns.Column; import javax.inject.Inject; import javax.inject.Singleton; -import java.util.*; -import java.util.stream.Collectors; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; @Singleton public class FileMappingUtil { @@ -41,6 +37,7 @@ public class FileMappingUtil { public static final String EXPERIMENT_TEMPLATE_NAME = "ExperimentsTemplateMap"; private FileImportService fileImportService; + @Inject public FileMappingUtil(FileImportService fileImportService) { this.fileImportService = fileImportService; @@ -77,27 +74,23 @@ public List> findFileRelationships(Table data, List> getDynamicColumns(Table data, String templateName) { - List result = fileImportService.getSystemMappingByName(templateName); - if (result.isEmpty()) { - throw new InternalServerException("System mapping does not exist"); + public List sortByField(List sortedFields, List unsortedItems, Function fieldGetter) { + HashMap sortOrder = new HashMap<>(); + for (int i = 0; i < sortedFields.size(); i++) { + sortOrder.put(sortedFields.get(i), i); } - ImportMapping mapping = result.get(0); - List config = mapping.getMappingConfig(); - List columnNames = new ArrayList<>(); - - for (MappingField field : config) { - if (field.getValue() != null) { - columnNames.add(field.getValue().getFileFieldName()); + unsortedItems.sort((i1, i2) -> { + try { + String field1 = fieldGetter.apply(i1); + String field2 = fieldGetter.apply(i2); + return Integer.compare(sortOrder.get(field1), sortOrder.get(field2)); + } catch (Exception e) { + throw new RuntimeException(e); } - } - - List differences = data.columnNames().stream() - .filter(col -> !columnNames.contains(col)) - .collect(Collectors.toList()); + }); - return data.columns(differences.toArray(String[]::new)); + return unsortedItems; } } diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/ExperimentProcessor.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/ExperimentProcessor.java index d1210b138..f9c79bdd9 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/ExperimentProcessor.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/ExperimentProcessor.java @@ -195,6 +195,7 @@ public void getExistingBrapiData(List importRows, Program program) */ @Override public Map process( + ImportUpload upload, List importRows, Map mappedBrAPIImport, Table data, @@ -206,7 +207,7 @@ public Map process( ValidationErrors validationErrors = new ValidationErrors(); // Get dynamic phenotype columns for processing - List> dynamicCols = fileMappingUtil.getDynamicColumns(data, EXPERIMENT_TEMPLATE_NAME); + List> dynamicCols = data.columns(upload.getDynamicColumnNames()); List> phenotypeCols = new ArrayList<>(); List> timestampCols = new ArrayList<>(); for (Column dynamicCol : dynamicCols) { @@ -436,7 +437,9 @@ private List verifyTraits(UUID programId, List> phenotypeCols, "Timestamp column(s) lack corresponding phenotype column(s): " + String.join(COMMA_DELIMITER, unmatchedTimestamps)); } - return filteredTraits; + // sort the verified traits to match the order of the trait columns + List phenotypeColNames = phenotypeCols.stream().map(Column::name).collect(Collectors.toList()); + return fileMappingUtil.sortByField(phenotypeColNames, filteredTraits, TraitEntity::getObservationVariableName); } private List fetchFileTraits(UUID programId, Collection varNames) { diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/GermplasmProcessor.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/GermplasmProcessor.java index cc4d8f476..326c03657 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/GermplasmProcessor.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/GermplasmProcessor.java @@ -16,9 +16,6 @@ */ package org.breedinginsight.brapps.importer.services.processors; -import com.google.gson.JsonElement; -import com.google.gson.JsonObject; -import com.google.gson.JsonParser; import io.micronaut.context.annotation.Property; import io.micronaut.context.annotation.Prototype; import io.micronaut.http.HttpStatus; @@ -30,7 +27,6 @@ import org.brapi.v2.model.core.BrAPIListSummary; import org.brapi.v2.model.core.request.BrAPIListNewRequest; import org.brapi.v2.model.germ.BrAPIGermplasm; -import org.brapi.v2.model.germ.BrAPIGermplasmSynonyms; import org.breedinginsight.api.model.v1.response.ValidationError; import org.breedinginsight.api.model.v1.response.ValidationErrors; import org.breedinginsight.brapi.v2.constants.BrAPIAdditionalInfoFields; @@ -221,7 +217,7 @@ public void getExistingBrapiData(List importRows, Program program) } @Override - public Map process(List importRows, + public Map process(ImportUpload upload, List importRows, Map mappedBrAPIImport, Table data, Program program, User user, boolean commit) throws ValidatorException { diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/LocationProcessor.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/LocationProcessor.java index cd2debb1e..5a3d5af0a 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/LocationProcessor.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/LocationProcessor.java @@ -78,7 +78,7 @@ public void getExistingBrapiData(List importRows, Program program) } @Override - public Map process(List importRows, + public Map process(ImportUpload upload, List importRows, Map mappedBrAPIImport, Table data, Program program, User user, boolean commit) throws ValidatorException { diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/ObservationProcessor.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/ObservationProcessor.java index 5a8f0c907..693dda6e1 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/ObservationProcessor.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/ObservationProcessor.java @@ -142,7 +142,7 @@ private void getDependentDbIds(List importRows, Program program) { } @Override - public Map process(List importRows, + public Map process(ImportUpload upload, List importRows, Map mappedBrAPIImport, Table data, Program program, User user, boolean commit) throws ValidatorException { diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/ObservationUnitProcessor.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/ObservationUnitProcessor.java index ff283f58c..daad1eecd 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/ObservationUnitProcessor.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/ObservationUnitProcessor.java @@ -88,7 +88,7 @@ private void getExistingStudyObjects() { } @Override - public Map process(List importRows, + public Map process(ImportUpload upload, List importRows, Map mappedBrAPIImport, Table data, Program program, User user, boolean commit) { diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/Processor.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/Processor.java index 26ba7a1ac..e4c2e067a 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/Processor.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/Processor.java @@ -43,6 +43,8 @@ public interface Processor { /** * Update mappedBrAPIImport mapping with PendingImport data for brapi object based on new and existing objects. * Return stats on number of new & existing objects + * + * @param upload * @param importRows * @param mappedBrAPIImport * @param data @@ -50,7 +52,7 @@ public interface Processor { * @return * @throws ValidatorException */ - Map process(List importRows, + Map process(ImportUpload upload, List importRows, Map mappedBrAPIImport, Table data, Program program, User user, boolean commit) throws ValidatorException, MissingRequiredInfoException, ApiException; diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/ProcessorManager.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/ProcessorManager.java index 1e4361673..1f61dabe8 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/ProcessorManager.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/ProcessorManager.java @@ -62,7 +62,7 @@ public ImportPreviewResponse process(List importRows, List stats = processor.process(importRows, mappedBrAPIImport, data, program, user, commit); + Map stats = processor.process(upload, importRows, mappedBrAPIImport, data, program, user, commit); statistics.putAll(stats); } diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/StudyProcessor.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/StudyProcessor.java index 8576b01fa..398eb2d88 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/StudyProcessor.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/StudyProcessor.java @@ -21,7 +21,6 @@ import io.micronaut.http.exceptions.HttpStatusException; import io.micronaut.http.server.exceptions.InternalServerException; import org.brapi.client.v2.model.exceptions.ApiException; -import org.brapi.v2.model.core.BrAPILocation; import org.brapi.v2.model.core.BrAPIStudy; import org.brapi.v2.model.core.BrAPITrial; import org.breedinginsight.brapps.importer.daos.BrAPIStudyDAO; @@ -80,7 +79,7 @@ public void getExistingBrapiData(List importRows, Program program) } @Override - public Map process(List importRows, + public Map process(ImportUpload upload, List importRows, Map mappedBrAPIImport, Table data, Program program, User user, boolean commit) { diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/TrialProcessor.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/TrialProcessor.java index 71fa6d04c..bce850f51 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/TrialProcessor.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/TrialProcessor.java @@ -74,7 +74,7 @@ public void getExistingBrapiData(List importRows, Program program) } @Override - public Map process(List importRows, + public Map process(ImportUpload upload, List importRows, Map mappedBrAPIImport, Table data, Program program, User user, boolean commit) throws ValidatorException {