From 6d4d6c237fcba962ca3dd075da282aeb20a0180f Mon Sep 17 00:00:00 2001 From: dmeidlin <14339308+dmeidlin@users.noreply.github.com> Date: Wed, 16 Aug 2023 11:55:45 -0400 Subject: [PATCH 01/14] add dataset xref to new OU on exp import --- .../ExperimentObservation.java | 18 +++++++++++------- .../processors/ExperimentProcessor.java | 6 +++++- 2 files changed, 16 insertions(+), 8 deletions(-) diff --git a/src/main/java/org/breedinginsight/brapps/importer/model/imports/experimentObservation/ExperimentObservation.java b/src/main/java/org/breedinginsight/brapps/importer/model/imports/experimentObservation/ExperimentObservation.java index 4159d6008..f98e39d4b 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/model/imports/experimentObservation/ExperimentObservation.java +++ b/src/main/java/org/breedinginsight/brapps/importer/model/imports/experimentObservation/ExperimentObservation.java @@ -223,6 +223,7 @@ public BrAPIObservationUnit constructBrAPIObservationUnit( String germplasmName, String referenceSource, UUID trialID, + UUID datasetId, UUID studyID, UUID id ) { @@ -232,7 +233,7 @@ public BrAPIObservationUnit constructBrAPIObservationUnit( observationUnit.setObservationUnitName(Utilities.appendProgramKey(getExpUnitId(), program.getKey(), seqVal)); // Set external reference - observationUnit.setExternalReferences(getObsUnitExternalReferences(program, referenceSource, trialID, studyID, id)); + observationUnit.setExternalReferences(getObsUnitExternalReferences(program, referenceSource, trialID, datasetId, studyID, id)); } else { observationUnit.setObservationUnitName(getExpUnitId()); } @@ -345,13 +346,16 @@ public BrAPIObservation constructBrAPIObservation( } private List getBrAPIExternalReferences( - Program program, String referenceSourceBaseName, UUID trialId, UUID studyId, UUID obsUnitId, UUID observationId) { + Program program, String referenceSourceBaseName, UUID trialId, UUID datasetId, UUID studyId, UUID obsUnitId, UUID observationId) { List refs = new ArrayList<>(); addReference(refs, program.getId(), referenceSourceBaseName, ExternalReferenceSource.PROGRAMS); if (trialId != null) { addReference(refs, trialId, referenceSourceBaseName, ExternalReferenceSource.TRIALS); } + if (datasetId != null) { + addReference(refs, datasetId, referenceSourceBaseName, ExternalReferenceSource.DATASET); + } if (studyId != null) { addReference(refs, studyId, referenceSourceBaseName, ExternalReferenceSource.STUDIES); } @@ -367,22 +371,22 @@ private List getBrAPIExternalReferences( private List getTrialExternalReferences( Program program, String referenceSourceBaseName, UUID trialId) { - return getBrAPIExternalReferences(program, referenceSourceBaseName, trialId, null, null, null); + return getBrAPIExternalReferences(program, referenceSourceBaseName, trialId, null, null, null, null); } private List getStudyExternalReferences( Program program, String referenceSourceBaseName, UUID trialId, UUID studyId) { - return getBrAPIExternalReferences(program, referenceSourceBaseName, trialId, studyId, null, null); + return getBrAPIExternalReferences(program, referenceSourceBaseName, trialId, null, studyId, null, null); } private List getObsUnitExternalReferences( - Program program, String referenceSourceBaseName, UUID trialId, UUID studyId, UUID obsUnitId) { - return getBrAPIExternalReferences(program, referenceSourceBaseName, trialId, studyId, obsUnitId, null); + Program program, String referenceSourceBaseName, UUID trialId, UUID datasetId, UUID studyId, UUID obsUnitId) { + return getBrAPIExternalReferences(program, referenceSourceBaseName, trialId, datasetId, null, obsUnitId, null); } private List getObservationExternalReferences( Program program, String referenceSourceBaseName, UUID trialId, UUID studyId, UUID obsUnitId, UUID observationId) { - return getBrAPIExternalReferences(program, referenceSourceBaseName, trialId, studyId, obsUnitId, observationId); + return getBrAPIExternalReferences(program, referenceSourceBaseName, trialId, null, studyId, obsUnitId, observationId); } diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/ExperimentProcessor.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/ExperimentProcessor.java index dcc3339c6..88209ee57 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/ExperimentProcessor.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/ExperimentProcessor.java @@ -41,6 +41,7 @@ import org.breedinginsight.brapi.v2.dao.BrAPIGermplasmDAO; import org.breedinginsight.brapps.importer.daos.*; import org.breedinginsight.brapps.importer.model.ImportUpload; +import org.breedinginsight.brapps.importer.model.base.AdditionalInfo; import org.breedinginsight.brapps.importer.model.imports.BrAPIImport; import org.breedinginsight.brapps.importer.model.imports.PendingImport; import org.breedinginsight.brapps.importer.model.imports.experimentObservation.ExperimentObservation; @@ -805,10 +806,13 @@ private PendingImportObject fetchOrCreateObsUnitPIO(Progra } PendingImportObject trialPIO = this.trialByNameNoScope.get(importRow.getExpTitle()); UUID trialID = trialPIO.getId(); + UUID datasetId = UUID.fromString(trialPIO.getBrAPIObject() + .getAdditionalInfo().getAsJsonObject() + .get(BrAPIAdditionalInfoFields.OBSERVATION_DATASET_ID).getAsString()); PendingImportObject studyPIO = this.studyByNameNoScope.get(importRow.getEnv()); UUID studyID = studyPIO.getId(); UUID id = UUID.randomUUID(); - BrAPIObservationUnit newObservationUnit = importRow.constructBrAPIObservationUnit(program, envSeqValue, commit, germplasmName, BRAPI_REFERENCE_SOURCE, trialID, studyID, id); + BrAPIObservationUnit newObservationUnit = importRow.constructBrAPIObservationUnit(program, envSeqValue, commit, germplasmName, BRAPI_REFERENCE_SOURCE, trialID, datasetId, studyID, id); pio = new PendingImportObject<>(ImportObjectState.NEW, newObservationUnit, id); this.observationUnitByNameNoScope.put(key, pio); } From 54c0e40c912554a1d9c960f65e250f6ba1966e8b Mon Sep 17 00:00:00 2001 From: dmeidlin <14339308+dmeidlin@users.noreply.github.com> Date: Fri, 18 Aug 2023 12:06:23 -0400 Subject: [PATCH 02/14] create flyway migration to assign dataset id to ous --- .../daos/BrAPIObservationUnitDAO.java | 14 +++ .../ExperimentObservation.java | 2 +- ...V1_0_13__Update_BrAPI_Locations_XRefs.java | 20 +-- .../V1_0_15__Add_OU_Dataset_Xrefs.java | 115 ++++++++++++++++++ .../breedinginsight/utilities/Utilities.java | 23 ++++ 5 files changed, 154 insertions(+), 20 deletions(-) create mode 100644 src/main/java/org/breedinginsight/db/migration/V1_0_15__Add_OU_Dataset_Xrefs.java diff --git a/src/main/java/org/breedinginsight/brapps/importer/daos/BrAPIObservationUnitDAO.java b/src/main/java/org/breedinginsight/brapps/importer/daos/BrAPIObservationUnitDAO.java index 02347e62d..75691a7f3 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/daos/BrAPIObservationUnitDAO.java +++ b/src/main/java/org/breedinginsight/brapps/importer/daos/BrAPIObservationUnitDAO.java @@ -23,6 +23,7 @@ import com.google.gson.reflect.TypeToken; import io.micronaut.context.annotation.Property; import org.brapi.client.v2.JSON; +import io.micronaut.http.server.exceptions.InternalServerException; import org.brapi.client.v2.model.exceptions.ApiException; import org.brapi.client.v2.modules.phenotype.ObservationUnitsApi; import org.brapi.v2.model.germ.BrAPIGermplasm; @@ -97,6 +98,19 @@ public List createBrAPIObservationUnits(List getObservationUnitsById(Collection observationUnitExternalIds, Program program) throws ApiException { if(observationUnitExternalIds.isEmpty()) { return Collections.emptyList(); diff --git a/src/main/java/org/breedinginsight/brapps/importer/model/imports/experimentObservation/ExperimentObservation.java b/src/main/java/org/breedinginsight/brapps/importer/model/imports/experimentObservation/ExperimentObservation.java index f98e39d4b..727e34caf 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/model/imports/experimentObservation/ExperimentObservation.java +++ b/src/main/java/org/breedinginsight/brapps/importer/model/imports/experimentObservation/ExperimentObservation.java @@ -381,7 +381,7 @@ private List getStudyExternalReferences( private List getObsUnitExternalReferences( Program program, String referenceSourceBaseName, UUID trialId, UUID datasetId, UUID studyId, UUID obsUnitId) { - return getBrAPIExternalReferences(program, referenceSourceBaseName, trialId, datasetId, null, obsUnitId, null); + return getBrAPIExternalReferences(program, referenceSourceBaseName, trialId, datasetId, studyId, obsUnitId, null); } private List getObservationExternalReferences( diff --git a/src/main/java/org/breedinginsight/db/migration/V1_0_13__Update_BrAPI_Locations_XRefs.java b/src/main/java/org/breedinginsight/db/migration/V1_0_13__Update_BrAPI_Locations_XRefs.java index 6effc5138..fadc89421 100644 --- a/src/main/java/org/breedinginsight/db/migration/V1_0_13__Update_BrAPI_Locations_XRefs.java +++ b/src/main/java/org/breedinginsight/db/migration/V1_0_13__Update_BrAPI_Locations_XRefs.java @@ -49,7 +49,7 @@ public void migrate(Context context) throws Exception { String referenceSource = placeholders.get(BRAPI_REFERENCE_SOURCE_KEY); // Get all the programs - List programs = getAllPrograms(context, defaultUrl); + List programs = Utilities.getAllProgramsFlyway(context, defaultUrl); Map locationsApiForProgram = new HashMap<>(); for (Program program : programs) { BrAPIClient client = new BrAPIClient(program.getBrapiUrl(), 240000); @@ -103,22 +103,4 @@ private List getAllLocations(Context context) throws SQLExcepti } return locations; } - - private List getAllPrograms(Context context, String defaultUrl) throws Exception { - List programs = new ArrayList<>(); - try (Statement select = context.getConnection().createStatement()) { - try (ResultSet rows = select.executeQuery("SELECT id, brapi_url, key FROM program where active = true ORDER BY id")) { - while (rows.next()) { - Program program = new Program(); - program.setId(UUID.fromString(rows.getString(1))); - String brapi_url = rows.getString(2); - if (brapi_url == null) brapi_url = defaultUrl; - program.setBrapiUrl(brapi_url); - program.setKey(rows.getString(3)); - programs.add(program); - } - } - } - return programs; - } } \ No newline at end of file diff --git a/src/main/java/org/breedinginsight/db/migration/V1_0_15__Add_OU_Dataset_Xrefs.java b/src/main/java/org/breedinginsight/db/migration/V1_0_15__Add_OU_Dataset_Xrefs.java new file mode 100644 index 000000000..a2c51a314 --- /dev/null +++ b/src/main/java/org/breedinginsight/db/migration/V1_0_15__Add_OU_Dataset_Xrefs.java @@ -0,0 +1,115 @@ +/* + * See the NOTICE file distributed with this work for additional information + * regarding copyright ownership. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.breedinginsight.db.migration; + +import io.micronaut.context.annotation.ConfigurationInject; +import lombok.extern.slf4j.Slf4j; +import org.brapi.client.v2.ApiResponse; +import org.brapi.client.v2.BrAPIClient; +import org.brapi.client.v2.model.queryParams.core.TrialQueryParams; +import org.brapi.client.v2.model.queryParams.phenotype.ObservationUnitQueryParams; +import org.brapi.client.v2.modules.core.TrialsApi; +import org.brapi.client.v2.modules.phenotype.ObservationUnitsApi; +import org.brapi.v2.model.BrAPIExternalReference; +import org.brapi.v2.model.core.BrAPITrial; +import org.brapi.v2.model.core.response.BrAPITrialListResponse; +import org.brapi.v2.model.pheno.BrAPIObservationUnit; +import org.brapi.v2.model.pheno.response.BrAPIObservationUnitListResponse; +import org.breedinginsight.brapi.v2.constants.BrAPIAdditionalInfoFields; +import org.breedinginsight.brapps.importer.daos.BrAPIObservationUnitDAO; +import org.breedinginsight.brapps.importer.daos.BrAPITrialDAO; +import org.breedinginsight.brapps.importer.services.ExternalReferenceSource; +import org.breedinginsight.model.Program; +import org.breedinginsight.utilities.Utilities; +import org.flywaydb.core.api.migration.BaseJavaMigration; +import org.flywaydb.core.api.migration.Context; + +import javax.inject.Inject; +import java.util.*; +import java.util.stream.Collectors; + +@Slf4j +public class V1_0_15__Add_OU_Dataset_Xrefs extends BaseJavaMigration { + final private String DEFAULT_URL_KEY = "default-url"; + final private String BRAPI_REFERENCE_SOURCE_KEY = "brapi-reference-source"; + private final BrAPITrialDAO trialDAO; + private final BrAPIObservationUnitDAO ouDAO; + + @Inject + V1_0_15__Add_OU_Dataset_Xrefs(BrAPITrialDAO trialDAO, BrAPIObservationUnitDAO ouDAO) { + this.trialDAO = trialDAO; + this.ouDAO = ouDAO; + } + + public void migrate(Context context) throws Exception { + Map placeholders = context.getConfiguration().getPlaceholders(); + String defaultUrl = placeholders.get(DEFAULT_URL_KEY); + String referenceSource = placeholders.get(BRAPI_REFERENCE_SOURCE_KEY); + String programReferenceSource = Utilities.generateReferenceSource(referenceSource, ExternalReferenceSource.PROGRAMS); + String trialReferenceSource = Utilities.generateReferenceSource(referenceSource, ExternalReferenceSource.TRIALS); + String datasetReferenceSource = Utilities.generateReferenceSource(referenceSource, ExternalReferenceSource.DATASET); + + + // Get all the programs + List programs = Utilities.getAllProgramsFlyway(context, defaultUrl); + + // For each program, update any observation units created via Deltabreed + for (Program program : programs) { + + // Get the Deltabreed-generated experiments for the program + List experiments = trialDAO.getTrials(program.getId()).stream().filter(trial -> { + List xrefs = trial.getExternalReferences(); + Optional programRef = Utilities.getExternalReference(xrefs,programReferenceSource); + return trial.getAdditionalInfo().getAsJsonObject().has(BrAPIAdditionalInfoFields.OBSERVATION_DATASET_ID) && + programRef.isPresent() && + program.getId().equals(UUID.fromString(programRef.get().getReferenceID())); + }).collect(Collectors.toList()); + + Map ExpIdByDbId = new HashMap<>(); + experiments.stream().forEach(exp -> { + Optional expRef = Utilities.getExternalReference(exp.getExternalReferences(), trialReferenceSource); + if (expRef.isPresent()) { + ExpIdByDbId.put(expRef.get().getReferenceID(), exp.getTrialDbId()); + } + }); + + for (BrAPITrial exp : experiments) { + + ouDAO.getObservationUnitsForTrialDbId(program.getId(), exp.getTrialDbId()) + .stream().filter(ou -> { + + // For each experiment, fetch the observation units that need a dataset reference + List xrefs = ou.getExternalReferences(); + Optional expRef = Utilities.getExternalReference(xrefs, trialReferenceSource); + Optional datasetRef = Utilities.getExternalReference(xrefs, datasetReferenceSource); + return datasetRef.isEmpty() && + expRef.isPresent() && + ExpIdByDbId.get(exp.getTrialDbId()).equals(expRef.get().getReferenceID()); + }).forEach(ou -> { + + // Assign the experiment Observation Dataset id to the observation units + BrAPIExternalReference datasetRef = new BrAPIExternalReference() + .referenceSource(datasetReferenceSource) + .referenceID(exp.getAdditionalInfo().getAsJsonObject().get(BrAPIAdditionalInfoFields.OBSERVATION_DATASET_ID).getAsString()); + ou.getExternalReferences().add(datasetRef); + ouDAO.updateBrAPIObservationUnit(ou.getObservationUnitDbId(), ou, program.getId()); + }); + } + } + } +} diff --git a/src/main/java/org/breedinginsight/utilities/Utilities.java b/src/main/java/org/breedinginsight/utilities/Utilities.java index 9ce3a1b9e..85693bbc7 100644 --- a/src/main/java/org/breedinginsight/utilities/Utilities.java +++ b/src/main/java/org/breedinginsight/utilities/Utilities.java @@ -21,7 +21,12 @@ import org.brapi.client.v2.model.exceptions.ApiException; import org.brapi.v2.model.BrAPIExternalReference; import org.breedinginsight.brapps.importer.services.ExternalReferenceSource; +import org.breedinginsight.model.Program; +import org.flywaydb.core.api.migration.Context; +import java.sql.ResultSet; +import java.sql.Statement; +import java.util.ArrayList; import java.util.List; import java.util.Optional; import java.util.UUID; @@ -181,6 +186,24 @@ public static String makePortableFilename(String name) { return sb.toString(); } + public static List getAllProgramsFlyway(Context context, String defaultUrl) throws Exception { + List programs = new ArrayList<>(); + try (Statement select = context.getConnection().createStatement()) { + try (ResultSet rows = select.executeQuery("SELECT id, brapi_url, key FROM program where active = true ORDER BY id")) { + while (rows.next()) { + Program program = new Program(); + program.setId(UUID.fromString(rows.getString(1))); + String brapi_url = rows.getString(2); + if (brapi_url == null) brapi_url = defaultUrl; + program.setBrapiUrl(brapi_url); + program.setKey(rows.getString(3)); + programs.add(program); + } + } + } + return programs; + } + private static boolean isSafeChar(char c) { // Check if c is in the portable filename character set. // See https://pubs.opengroup.org/onlinepubs/9699919799/basedefs/V1_chap03.html#tag_03_282 From b03087edd759454d092695e0deb29a9d24b05ed0 Mon Sep 17 00:00:00 2001 From: dmeidlin <14339308+dmeidlin@users.noreply.github.com> Date: Mon, 21 Aug 2023 11:48:56 -0400 Subject: [PATCH 03/14] code clean up --- .../V1_0_15__Add_OU_Dataset_Xrefs.java | 20 ++++--------------- 1 file changed, 4 insertions(+), 16 deletions(-) diff --git a/src/main/java/org/breedinginsight/db/migration/V1_0_15__Add_OU_Dataset_Xrefs.java b/src/main/java/org/breedinginsight/db/migration/V1_0_15__Add_OU_Dataset_Xrefs.java index a2c51a314..29fd5ee6f 100644 --- a/src/main/java/org/breedinginsight/db/migration/V1_0_15__Add_OU_Dataset_Xrefs.java +++ b/src/main/java/org/breedinginsight/db/migration/V1_0_15__Add_OU_Dataset_Xrefs.java @@ -17,19 +17,9 @@ package org.breedinginsight.db.migration; -import io.micronaut.context.annotation.ConfigurationInject; import lombok.extern.slf4j.Slf4j; -import org.brapi.client.v2.ApiResponse; -import org.brapi.client.v2.BrAPIClient; -import org.brapi.client.v2.model.queryParams.core.TrialQueryParams; -import org.brapi.client.v2.model.queryParams.phenotype.ObservationUnitQueryParams; -import org.brapi.client.v2.modules.core.TrialsApi; -import org.brapi.client.v2.modules.phenotype.ObservationUnitsApi; import org.brapi.v2.model.BrAPIExternalReference; import org.brapi.v2.model.core.BrAPITrial; -import org.brapi.v2.model.core.response.BrAPITrialListResponse; -import org.brapi.v2.model.pheno.BrAPIObservationUnit; -import org.brapi.v2.model.pheno.response.BrAPIObservationUnitListResponse; import org.breedinginsight.brapi.v2.constants.BrAPIAdditionalInfoFields; import org.breedinginsight.brapps.importer.daos.BrAPIObservationUnitDAO; import org.breedinginsight.brapps.importer.daos.BrAPITrialDAO; @@ -45,8 +35,6 @@ @Slf4j public class V1_0_15__Add_OU_Dataset_Xrefs extends BaseJavaMigration { - final private String DEFAULT_URL_KEY = "default-url"; - final private String BRAPI_REFERENCE_SOURCE_KEY = "brapi-reference-source"; private final BrAPITrialDAO trialDAO; private final BrAPIObservationUnitDAO ouDAO; @@ -58,7 +46,9 @@ public class V1_0_15__Add_OU_Dataset_Xrefs extends BaseJavaMigration { public void migrate(Context context) throws Exception { Map placeholders = context.getConfiguration().getPlaceholders(); + String DEFAULT_URL_KEY = "default-url"; String defaultUrl = placeholders.get(DEFAULT_URL_KEY); + String BRAPI_REFERENCE_SOURCE_KEY = "brapi-reference-source"; String referenceSource = placeholders.get(BRAPI_REFERENCE_SOURCE_KEY); String programReferenceSource = Utilities.generateReferenceSource(referenceSource, ExternalReferenceSource.PROGRAMS); String trialReferenceSource = Utilities.generateReferenceSource(referenceSource, ExternalReferenceSource.TRIALS); @@ -81,11 +71,9 @@ public void migrate(Context context) throws Exception { }).collect(Collectors.toList()); Map ExpIdByDbId = new HashMap<>(); - experiments.stream().forEach(exp -> { + experiments.forEach(exp -> { Optional expRef = Utilities.getExternalReference(exp.getExternalReferences(), trialReferenceSource); - if (expRef.isPresent()) { - ExpIdByDbId.put(expRef.get().getReferenceID(), exp.getTrialDbId()); - } + expRef.ifPresent(brAPIExternalReference -> ExpIdByDbId.put(brAPIExternalReference.getReferenceID(), exp.getTrialDbId())); }); for (BrAPITrial exp : experiments) { From 0c6408b3cfd37df2036da63ada6af530c743fac5 Mon Sep 17 00:00:00 2001 From: dmeidlin <14339308+dmeidlin@users.noreply.github.com> Date: Tue, 22 Aug 2023 13:43:47 -0400 Subject: [PATCH 04/14] remove dependency injection from flyway migration --- .../V1_0_15__Add_OU_Dataset_Xrefs.java | 144 +++++++++++++----- 1 file changed, 102 insertions(+), 42 deletions(-) diff --git a/src/main/java/org/breedinginsight/db/migration/V1_0_15__Add_OU_Dataset_Xrefs.java b/src/main/java/org/breedinginsight/db/migration/V1_0_15__Add_OU_Dataset_Xrefs.java index 29fd5ee6f..63c9325e9 100644 --- a/src/main/java/org/breedinginsight/db/migration/V1_0_15__Add_OU_Dataset_Xrefs.java +++ b/src/main/java/org/breedinginsight/db/migration/V1_0_15__Add_OU_Dataset_Xrefs.java @@ -17,12 +17,23 @@ package org.breedinginsight.db.migration; +import io.micronaut.http.server.exceptions.InternalServerException; import lombok.extern.slf4j.Slf4j; +import org.brapi.client.v2.ApiResponse; +import org.brapi.client.v2.BrAPIClient; +import org.brapi.client.v2.model.queryParams.core.TrialQueryParams; +import org.brapi.client.v2.model.queryParams.phenotype.ObservationUnitQueryParams; +import org.brapi.client.v2.modules.core.TrialsApi; +import org.brapi.client.v2.modules.phenotype.ObservationUnitsApi; import org.brapi.v2.model.BrAPIExternalReference; import org.brapi.v2.model.core.BrAPITrial; +import org.brapi.v2.model.core.response.BrAPITrialListResponse; +import org.brapi.v2.model.pheno.BrAPIObservationUnit; +import org.brapi.v2.model.pheno.response.BrAPIObservationUnitListResponse; import org.breedinginsight.brapi.v2.constants.BrAPIAdditionalInfoFields; import org.breedinginsight.brapps.importer.daos.BrAPIObservationUnitDAO; import org.breedinginsight.brapps.importer.daos.BrAPITrialDAO; +import org.breedinginsight.brapps.importer.daos.impl.BrAPITrialDAOImpl; import org.breedinginsight.brapps.importer.services.ExternalReferenceSource; import org.breedinginsight.model.Program; import org.breedinginsight.utilities.Utilities; @@ -35,14 +46,7 @@ @Slf4j public class V1_0_15__Add_OU_Dataset_Xrefs extends BaseJavaMigration { - private final BrAPITrialDAO trialDAO; - private final BrAPIObservationUnitDAO ouDAO; - @Inject - V1_0_15__Add_OU_Dataset_Xrefs(BrAPITrialDAO trialDAO, BrAPIObservationUnitDAO ouDAO) { - this.trialDAO = trialDAO; - this.ouDAO = ouDAO; - } public void migrate(Context context) throws Exception { Map placeholders = context.getConfiguration().getPlaceholders(); @@ -60,43 +64,99 @@ public void migrate(Context context) throws Exception { // For each program, update any observation units created via Deltabreed for (Program program : programs) { + BrAPIClient client = new BrAPIClient(program.getBrapiUrl(), 240000); // Get the Deltabreed-generated experiments for the program - List experiments = trialDAO.getTrials(program.getId()).stream().filter(trial -> { - List xrefs = trial.getExternalReferences(); - Optional programRef = Utilities.getExternalReference(xrefs,programReferenceSource); - return trial.getAdditionalInfo().getAsJsonObject().has(BrAPIAdditionalInfoFields.OBSERVATION_DATASET_ID) && - programRef.isPresent() && - program.getId().equals(UUID.fromString(programRef.get().getReferenceID())); - }).collect(Collectors.toList()); - - Map ExpIdByDbId = new HashMap<>(); - experiments.forEach(exp -> { - Optional expRef = Utilities.getExternalReference(exp.getExternalReferences(), trialReferenceSource); - expRef.ifPresent(brAPIExternalReference -> ExpIdByDbId.put(brAPIExternalReference.getReferenceID(), exp.getTrialDbId())); - }); - - for (BrAPITrial exp : experiments) { - - ouDAO.getObservationUnitsForTrialDbId(program.getId(), exp.getTrialDbId()) - .stream().filter(ou -> { - - // For each experiment, fetch the observation units that need a dataset reference - List xrefs = ou.getExternalReferences(); - Optional expRef = Utilities.getExternalReference(xrefs, trialReferenceSource); - Optional datasetRef = Utilities.getExternalReference(xrefs, datasetReferenceSource); - return datasetRef.isEmpty() && - expRef.isPresent() && - ExpIdByDbId.get(exp.getTrialDbId()).equals(expRef.get().getReferenceID()); - }).forEach(ou -> { - - // Assign the experiment Observation Dataset id to the observation units - BrAPIExternalReference datasetRef = new BrAPIExternalReference() - .referenceSource(datasetReferenceSource) - .referenceID(exp.getAdditionalInfo().getAsJsonObject().get(BrAPIAdditionalInfoFields.OBSERVATION_DATASET_ID).getAsString()); - ou.getExternalReferences().add(datasetRef); - ouDAO.updateBrAPIObservationUnit(ou.getObservationUnitDbId(), ou, program.getId()); - }); + TrialsApi trialsApi = new TrialsApi(client); + TrialQueryParams trialQueryParams = new TrialQueryParams(); + trialQueryParams.externalReferenceSource(programReferenceSource); + trialQueryParams.externalReferenceID(program.getId().toString()); + trialQueryParams.page(0); + trialQueryParams.pageSize(1000); + ApiResponse trialsResponse = trialsApi.trialsGet(trialQueryParams); + + boolean trialsDone = trialsResponse.getBody() == null || trialsResponse.getBody().getMetadata().getPagination().getTotalCount() == 0 || trialsResponse.getBody().getResult() == null; + while(!trialsDone) { + List experiments = trialsResponse.getBody().getResult().getData().stream().filter(trial -> { + List xrefs = trial.getExternalReferences(); + Optional programRef = Utilities.getExternalReference(xrefs,programReferenceSource); + return trial.getAdditionalInfo().getAsJsonObject().has(BrAPIAdditionalInfoFields.OBSERVATION_DATASET_ID) && + programRef.isPresent() && + program.getId().equals(UUID.fromString(programRef.get().getReferenceID())); + }).collect(Collectors.toList()); + + Map ExpIdByDbId = new HashMap<>(); + experiments.forEach(exp -> { + Optional expRef = Utilities.getExternalReference(exp.getExternalReferences(), trialReferenceSource); + expRef.ifPresent(brAPIExternalReference -> ExpIdByDbId.put(exp.getTrialDbId(), brAPIExternalReference.getReferenceID())); + }); + + for (BrAPITrial exp : experiments) { + + // Fetch all observation units for an experiment + ObservationUnitsApi ousApi = new ObservationUnitsApi(client); + ObservationUnitQueryParams ouQueryParams = new ObservationUnitQueryParams(); + ouQueryParams.externalReferenceSource(trialReferenceSource); + ouQueryParams.externalReferenceID(ExpIdByDbId.get(exp.getTrialDbId())); + ouQueryParams.page(0); + ouQueryParams.pageSize(1000); + ApiResponse ousResponse = ousApi.observationunitsGet(ouQueryParams); + + boolean ousDone = ousResponse.getBody() == null || ousResponse.getBody().getMetadata().getPagination().getTotalCount() == 0 || ousResponse.getBody().getResult() == null; + while(!ousDone) { + ousResponse.getBody().getResult().getData() + .stream().filter(ou -> { + + // Find the observation units that need a dataset reference + List xrefs = ou.getExternalReferences(); + Optional expRef = Utilities.getExternalReference(xrefs, trialReferenceSource); + Optional datasetRef = Utilities.getExternalReference(xrefs, datasetReferenceSource); + return datasetRef.isEmpty() && + expRef.isPresent() && + ExpIdByDbId.get(exp.getTrialDbId()).equals(expRef.get().getReferenceID()); + }).forEach(ou -> { + + // Assign the experiment Observation Dataset id to the observation units + BrAPIExternalReference datasetRef = new BrAPIExternalReference() + .referenceSource(datasetReferenceSource) + .referenceID(exp.getAdditionalInfo().getAsJsonObject().get(BrAPIAdditionalInfoFields.OBSERVATION_DATASET_ID).getAsString()); + ou.getExternalReferences().add(datasetRef); + try { + + // Send the updated observation unit back to the brapi server + BrAPIObservationUnit updatedOu = ousApi.observationunitsObservationUnitDbIdPut(ou.getObservationUnitDbId(), ou).getBody().getResult(); + + // Verify that the observation unit was updated at the server + boolean isUpdated = updatedOu.getExternalReferences().stream().anyMatch(xref -> { + return xref.getReferenceSource().equals(datasetReferenceSource) && + xref.getReferenceID().equals(exp.getAdditionalInfo().getAsJsonObject().get(BrAPIAdditionalInfoFields.OBSERVATION_DATASET_ID).getAsString()); + }); + if (!isUpdated) { + throw new Exception("Observation unit returned from brapi server was not updated. Check your brapi server."); + } + } catch(Exception e) { + log.error(e.getMessage(), e); + throw new InternalServerException(e.toString(), e); + } + }); + + // Fetch the next page of observation units for this experiment + if(ousResponse.getBody().getMetadata().getPagination().getCurrentPage() + 1 == ousResponse.getBody().getMetadata().getPagination().getTotalPages()) { + ousDone = true; + } else { + ouQueryParams.page(ouQueryParams.page() + 1); + ousResponse = ousApi.observationunitsGet(ouQueryParams); + } + } + } + + // Fetch the next page of experiments for this program + if(trialsResponse.getBody().getMetadata().getPagination().getCurrentPage() + 1 == trialsResponse.getBody().getMetadata().getPagination().getTotalPages()) { + trialsDone = true; + } else { + trialQueryParams.page(trialQueryParams.page() + 1); + trialsResponse = trialsApi.trialsGet(trialQueryParams); + } } } } From 614fe2186a6b0d6483ae974582ed0f7a234b750a Mon Sep 17 00:00:00 2001 From: dmeidlin <14339308+dmeidlin@users.noreply.github.com> Date: Wed, 23 Aug 2023 10:33:46 -0400 Subject: [PATCH 05/14] add log statements --- .../V1_0_15__Add_OU_Dataset_Xrefs.java | 17 ++++++++++++++++- 1 file changed, 16 insertions(+), 1 deletion(-) diff --git a/src/main/java/org/breedinginsight/db/migration/V1_0_15__Add_OU_Dataset_Xrefs.java b/src/main/java/org/breedinginsight/db/migration/V1_0_15__Add_OU_Dataset_Xrefs.java index 63c9325e9..79e57b6a9 100644 --- a/src/main/java/org/breedinginsight/db/migration/V1_0_15__Add_OU_Dataset_Xrefs.java +++ b/src/main/java/org/breedinginsight/db/migration/V1_0_15__Add_OU_Dataset_Xrefs.java @@ -56,15 +56,16 @@ public void migrate(Context context) throws Exception { String referenceSource = placeholders.get(BRAPI_REFERENCE_SOURCE_KEY); String programReferenceSource = Utilities.generateReferenceSource(referenceSource, ExternalReferenceSource.PROGRAMS); String trialReferenceSource = Utilities.generateReferenceSource(referenceSource, ExternalReferenceSource.TRIALS); + String observationUnitReferenceSource = Utilities.generateReferenceSource(referenceSource, ExternalReferenceSource.OBSERVATION_UNITS); String datasetReferenceSource = Utilities.generateReferenceSource(referenceSource, ExternalReferenceSource.DATASET); - // Get all the programs List programs = Utilities.getAllProgramsFlyway(context, defaultUrl); // For each program, update any observation units created via Deltabreed for (Program program : programs) { BrAPIClient client = new BrAPIClient(program.getBrapiUrl(), 240000); + log.debug("Migrating observation units for programId: " + program.getId()); // Get the Deltabreed-generated experiments for the program TrialsApi trialsApi = new TrialsApi(client); @@ -77,6 +78,10 @@ public void migrate(Context context) throws Exception { boolean trialsDone = trialsResponse.getBody() == null || trialsResponse.getBody().getMetadata().getPagination().getTotalCount() == 0 || trialsResponse.getBody().getResult() == null; while(!trialsDone) { + log.debug(String.format("processing page %d of %d of experiments for programId: %s", + trialsResponse.getBody().getMetadata().getPagination().getCurrentPage()+1, + trialsResponse.getBody().getMetadata().getPagination().getTotalPages(), + program.getId())); List experiments = trialsResponse.getBody().getResult().getData().stream().filter(trial -> { List xrefs = trial.getExternalReferences(); Optional programRef = Utilities.getExternalReference(xrefs,programReferenceSource); @@ -104,6 +109,11 @@ public void migrate(Context context) throws Exception { boolean ousDone = ousResponse.getBody() == null || ousResponse.getBody().getMetadata().getPagination().getTotalCount() == 0 || ousResponse.getBody().getResult() == null; while(!ousDone) { + log.debug(String.format("processing page %d of %d of observation units for experiment: %s", + ousResponse.getBody().getMetadata().getPagination().getCurrentPage()+1, + ousResponse.getBody().getMetadata().getPagination().getTotalPages(), + exp.getTrialName())); + ousResponse.getBody().getResult().getData() .stream().filter(ou -> { @@ -124,9 +134,14 @@ public void migrate(Context context) throws Exception { try { // Send the updated observation unit back to the brapi server + log.debug(String.format("Sending observation unit (id: %s) with observation dataset (id: %s) to %s", + Utilities.getExternalReference(ou.getExternalReferences(), observationUnitReferenceSource), + datasetRef.getReferenceID(), + defaultUrl)); BrAPIObservationUnit updatedOu = ousApi.observationunitsObservationUnitDbIdPut(ou.getObservationUnitDbId(), ou).getBody().getResult(); // Verify that the observation unit was updated at the server + log.debug(updatedOu.getExternalReferences().toString()); boolean isUpdated = updatedOu.getExternalReferences().stream().anyMatch(xref -> { return xref.getReferenceSource().equals(datasetReferenceSource) && xref.getReferenceID().equals(exp.getAdditionalInfo().getAsJsonObject().get(BrAPIAdditionalInfoFields.OBSERVATION_DATASET_ID).getAsString()); From 13e0aa7bec01239c9d8e66025bc28c7383158758 Mon Sep 17 00:00:00 2001 From: dmeidlin <14339308+dmeidlin@users.noreply.github.com> Date: Wed, 23 Aug 2023 17:30:49 -0400 Subject: [PATCH 06/14] update BrAPITrialService#getDatasetData to fetch OUs by dataset id --- .../brapi/v2/services/BrAPITrialService.java | 12 ++++------ .../daos/BrAPIObservationUnitDAO.java | 22 ++++++++++++++++++- 2 files changed, 25 insertions(+), 9 deletions(-) diff --git a/src/main/java/org/breedinginsight/brapi/v2/services/BrAPITrialService.java b/src/main/java/org/breedinginsight/brapi/v2/services/BrAPITrialService.java index 182882ff5..8c002d27c 100644 --- a/src/main/java/org/breedinginsight/brapi/v2/services/BrAPITrialService.java +++ b/src/main/java/org/breedinginsight/brapi/v2/services/BrAPITrialService.java @@ -296,18 +296,14 @@ private StreamedFile zipFiles(List files) throws IOException { } public Dataset getDatasetData(Program program, UUID experimentId, UUID datsetId, Boolean stats) throws ApiException, DoesNotExistException { - BrAPITrial experiment = this.getExperiment(program, experimentId); - - // TODO: Once BI-1831 is complete and OUs in a dataset can be identified using the datasetId stored as a xref - // the expOUs needs to be replaced with datasetOUs, as was done with datasetObsVars - List expOUs = ouDAO.getObservationUnitsForTrialDbId(program.getId(), experiment.getTrialDbId(), true); + List datasetOUs = ouDAO.getObservationUnitsForDataset(datsetId.toString(), program); List datasetObsVars = getDatasetObsVars(datsetId.toString(), program); - List ouDbIds = expOUs.stream().map(BrAPIObservationUnit::getObservationUnitDbId).collect(Collectors.toList()); + List ouDbIds = datasetOUs.stream().map(BrAPIObservationUnit::getObservationUnitDbId).collect(Collectors.toList()); List obsVarDbIds = datasetObsVars.stream().map(BrAPIObservationVariable::getObservationVariableDbId).collect(Collectors.toList()); List data = observationDAO.getObservationsByObservationUnitsAndVariables(ouDbIds, obsVarDbIds, program); - Dataset dataset = new Dataset(experimentId.toString(), data, expOUs, datasetObsVars); + Dataset dataset = new Dataset(experimentId.toString(), data, datasetOUs, datasetObsVars); if (stats) { - Integer ouCount = expOUs.size(); + Integer ouCount = datasetOUs.size(); Integer obsVarCount = datasetObsVars.size(); Integer obsCount = ouCount * obsVarCount; Integer obsDataCount = data.size(); diff --git a/src/main/java/org/breedinginsight/brapps/importer/daos/BrAPIObservationUnitDAO.java b/src/main/java/org/breedinginsight/brapps/importer/daos/BrAPIObservationUnitDAO.java index 75691a7f3..65d81712b 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/daos/BrAPIObservationUnitDAO.java +++ b/src/main/java/org/breedinginsight/brapps/importer/daos/BrAPIObservationUnitDAO.java @@ -26,6 +26,7 @@ import io.micronaut.http.server.exceptions.InternalServerException; import org.brapi.client.v2.model.exceptions.ApiException; import org.brapi.client.v2.modules.phenotype.ObservationUnitsApi; +import org.brapi.v2.model.BrAPIExternalReference; import org.brapi.v2.model.germ.BrAPIGermplasm; import org.brapi.v2.model.pheno.BrAPIObservationUnit; import org.brapi.v2.model.pheno.request.BrAPIObservationUnitSearchRequest; @@ -43,6 +44,7 @@ import org.breedinginsight.services.brapi.BrAPIEndpointProvider; import org.breedinginsight.services.exceptions.DoesNotExistException; import org.breedinginsight.utilities.BrAPIDAOUtil; +import org.breedinginsight.utilities.Utilities; import javax.inject.Inject; import javax.inject.Singleton; @@ -65,7 +67,13 @@ public class BrAPIObservationUnitDAO { private final Type treatmentlistType = new TypeToken>(){}.getType(); @Inject - public BrAPIObservationUnitDAO(ProgramDAO programDAO, ImportDAO importDAO, BrAPIDAOUtil brAPIDAOUtil, BrAPIEndpointProvider brAPIEndpointProvider, BrAPIGermplasmService germplasmService, ProgramService programService, @Property(name = "brapi.server.reference-source") String referenceSource) { + public BrAPIObservationUnitDAO(ProgramDAO programDAO, + ImportDAO importDAO, + BrAPIDAOUtil brAPIDAOUtil, + BrAPIEndpointProvider brAPIEndpointProvider, + BrAPIGermplasmService germplasmService, + ProgramService programService, + @Property(name = "brapi.server.reference-source") String referenceSource) { this.programDAO = programDAO; this.importDAO = importDAO; this.brAPIDAOUtil = brAPIDAOUtil; @@ -138,6 +146,18 @@ public List getObservationUnitsForTrialDbId(@NotNull UUID return getObservationUnitsForTrialDbId(programId, trialDbId, false); } + public List getObservationUnitsForDataset(@NotNull String datasetId, @NotNull Program program) throws ApiException { + String datasetReferenceSource = Utilities.generateReferenceSource(referenceSource, ExternalReferenceSource.DATASET); + BrAPIObservationUnitSearchRequest ouSearchRequest = new BrAPIObservationUnitSearchRequest(); + ouSearchRequest.programDbIds(List.of(program.getBrapiProgram().getProgramDbId())); + ouSearchRequest.externalReferenceSources(List.of(datasetReferenceSource)); + ouSearchRequest.externalReferenceIDs(List.of(datasetId)); + ObservationUnitsApi api = brAPIEndpointProvider.get(programDAO.getCoreClient(program.getId()), ObservationUnitsApi.class); + return brAPIDAOUtil.search(api::searchObservationunitsPost, + api::searchObservationunitsSearchResultsDbIdGet, + ouSearchRequest); + } + public List getObservationUnitsForTrialDbId(@NotNull UUID programId, @NotNull String trialDbId, boolean withGID) throws ApiException, DoesNotExistException { Program program = programService.getById(programId).orElseThrow(() -> new DoesNotExistException("Program id does not exist")); From 5a0d79a1a996e0c70282461ea6cb3004f0e3edd0 Mon Sep 17 00:00:00 2001 From: dmeidlin <14339308+dmeidlin@users.noreply.github.com> Date: Mon, 28 Aug 2023 11:56:25 -0400 Subject: [PATCH 07/14] add commit flag to adding obs unit xref --- .../services/processors/ExperimentProcessor.java | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/ExperimentProcessor.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/ExperimentProcessor.java index 88209ee57..c9e7f5aa0 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/ExperimentProcessor.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/ExperimentProcessor.java @@ -806,9 +806,12 @@ private PendingImportObject fetchOrCreateObsUnitPIO(Progra } PendingImportObject trialPIO = this.trialByNameNoScope.get(importRow.getExpTitle()); UUID trialID = trialPIO.getId(); - UUID datasetId = UUID.fromString(trialPIO.getBrAPIObject() - .getAdditionalInfo().getAsJsonObject() - .get(BrAPIAdditionalInfoFields.OBSERVATION_DATASET_ID).getAsString()); + UUID datasetId = null; + if (commit) { + datasetId = UUID.fromString(trialPIO.getBrAPIObject() + .getAdditionalInfo().getAsJsonObject() + .get(BrAPIAdditionalInfoFields.OBSERVATION_DATASET_ID).getAsString()); + } PendingImportObject studyPIO = this.studyByNameNoScope.get(importRow.getEnv()); UUID studyID = studyPIO.getId(); UUID id = UUID.randomUUID(); From 035af2abe7967a3adcfb5c6f8ee5404e5bc3c3a7 Mon Sep 17 00:00:00 2001 From: dmeidlin <14339308+dmeidlin@users.noreply.github.com> Date: Mon, 28 Aug 2023 15:32:23 -0400 Subject: [PATCH 08/14] change log debug statement --- .../db/migration/V1_0_15__Add_OU_Dataset_Xrefs.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/java/org/breedinginsight/db/migration/V1_0_15__Add_OU_Dataset_Xrefs.java b/src/main/java/org/breedinginsight/db/migration/V1_0_15__Add_OU_Dataset_Xrefs.java index 79e57b6a9..abd00c749 100644 --- a/src/main/java/org/breedinginsight/db/migration/V1_0_15__Add_OU_Dataset_Xrefs.java +++ b/src/main/java/org/breedinginsight/db/migration/V1_0_15__Add_OU_Dataset_Xrefs.java @@ -141,11 +141,11 @@ public void migrate(Context context) throws Exception { BrAPIObservationUnit updatedOu = ousApi.observationunitsObservationUnitDbIdPut(ou.getObservationUnitDbId(), ou).getBody().getResult(); // Verify that the observation unit was updated at the server - log.debug(updatedOu.getExternalReferences().toString()); boolean isUpdated = updatedOu.getExternalReferences().stream().anyMatch(xref -> { return xref.getReferenceSource().equals(datasetReferenceSource) && xref.getReferenceID().equals(exp.getAdditionalInfo().getAsJsonObject().get(BrAPIAdditionalInfoFields.OBSERVATION_DATASET_ID).getAsString()); }); + log.debug("Updating observation unit successful: " + String.valueOf(isUpdated)); if (!isUpdated) { throw new Exception("Observation unit returned from brapi server was not updated. Check your brapi server."); } From 5ff2f12172b9a7ffdf5ba9828680a6c2012bccba Mon Sep 17 00:00:00 2001 From: dmeidlin <14339308+dmeidlin@users.noreply.github.com> Date: Wed, 30 Aug 2023 12:32:21 -0400 Subject: [PATCH 09/14] update list of observation units --- .../V1_0_15__Add_OU_Dataset_Xrefs.java | 54 +++++++++++-------- 1 file changed, 31 insertions(+), 23 deletions(-) diff --git a/src/main/java/org/breedinginsight/db/migration/V1_0_15__Add_OU_Dataset_Xrefs.java b/src/main/java/org/breedinginsight/db/migration/V1_0_15__Add_OU_Dataset_Xrefs.java index abd00c749..139d834fd 100644 --- a/src/main/java/org/breedinginsight/db/migration/V1_0_15__Add_OU_Dataset_Xrefs.java +++ b/src/main/java/org/breedinginsight/db/migration/V1_0_15__Add_OU_Dataset_Xrefs.java @@ -30,6 +30,7 @@ import org.brapi.v2.model.core.response.BrAPITrialListResponse; import org.brapi.v2.model.pheno.BrAPIObservationUnit; import org.brapi.v2.model.pheno.response.BrAPIObservationUnitListResponse; +import org.brapi.v2.model.pheno.response.BrAPIObservationUnitListResponseResult; import org.breedinginsight.brapi.v2.constants.BrAPIAdditionalInfoFields; import org.breedinginsight.brapps.importer.daos.BrAPIObservationUnitDAO; import org.breedinginsight.brapps.importer.daos.BrAPITrialDAO; @@ -113,7 +114,7 @@ public void migrate(Context context) throws Exception { ousResponse.getBody().getMetadata().getPagination().getCurrentPage()+1, ousResponse.getBody().getMetadata().getPagination().getTotalPages(), exp.getTrialName())); - + Map mutatedOuByDbId = new HashMap<>(); ousResponse.getBody().getResult().getData() .stream().filter(ou -> { @@ -131,29 +132,36 @@ public void migrate(Context context) throws Exception { .referenceSource(datasetReferenceSource) .referenceID(exp.getAdditionalInfo().getAsJsonObject().get(BrAPIAdditionalInfoFields.OBSERVATION_DATASET_ID).getAsString()); ou.getExternalReferences().add(datasetRef); - try { - - // Send the updated observation unit back to the brapi server - log.debug(String.format("Sending observation unit (id: %s) with observation dataset (id: %s) to %s", - Utilities.getExternalReference(ou.getExternalReferences(), observationUnitReferenceSource), - datasetRef.getReferenceID(), - defaultUrl)); - BrAPIObservationUnit updatedOu = ousApi.observationunitsObservationUnitDbIdPut(ou.getObservationUnitDbId(), ou).getBody().getResult(); - - // Verify that the observation unit was updated at the server - boolean isUpdated = updatedOu.getExternalReferences().stream().anyMatch(xref -> { - return xref.getReferenceSource().equals(datasetReferenceSource) && - xref.getReferenceID().equals(exp.getAdditionalInfo().getAsJsonObject().get(BrAPIAdditionalInfoFields.OBSERVATION_DATASET_ID).getAsString()); - }); - log.debug("Updating observation unit successful: " + String.valueOf(isUpdated)); - if (!isUpdated) { - throw new Exception("Observation unit returned from brapi server was not updated. Check your brapi server."); - } - } catch(Exception e) { - log.error(e.getMessage(), e); - throw new InternalServerException(e.toString(), e); - } + mutatedOuByDbId.put(ou.getObservationUnitDbId(), ou); + }); + try { + + // Send the updated observation unit back to the brapi server + mutatedOuByDbId.values().stream().forEach(ou -> { + log.debug(String.format("Sending observation unit (id: %s) with observation dataset (id: %s) to %s", + Utilities.getExternalReference(ou.getExternalReferences(), observationUnitReferenceSource), + Utilities.getExternalReference(ou.getExternalReferences(), datasetReferenceSource), + defaultUrl)); + }); + List updatedOus = ousApi.observationunitsPut(mutatedOuByDbId).getBody().getResult().getData(); + + // Verify that the observation unit was updated at the server + boolean updateFailed = updatedOus.stream().anyMatch(updatedOu -> { + return !updatedOu.getExternalReferences().stream().anyMatch(xref -> { + return xref.getReferenceSource().equals(datasetReferenceSource) && + xref.getReferenceID().equals(exp.getAdditionalInfo().getAsJsonObject().get(BrAPIAdditionalInfoFields.OBSERVATION_DATASET_ID).getAsString()); }); + }); + log.debug("Updating observation unit successful: " + String.valueOf(!updateFailed)); + if (updateFailed) { + throw new Exception("Observation unit returned from brapi server was not updated. Check your brapi server."); + } + } catch(Exception e) { + log.error(e.getMessage(), e); + throw new InternalServerException(e.toString(), e); + } + + // Fetch the next page of observation units for this experiment if(ousResponse.getBody().getMetadata().getPagination().getCurrentPage() + 1 == ousResponse.getBody().getMetadata().getPagination().getTotalPages()) { From e57fef304bc065f96649d478a78f2dbbe67107e9 Mon Sep 17 00:00:00 2001 From: dmeidlin <14339308+dmeidlin@users.noreply.github.com> Date: Wed, 30 Aug 2023 16:21:34 -0400 Subject: [PATCH 10/14] revert single ou put --- .../V1_0_15__Add_OU_Dataset_Xrefs.java | 53 ++++++++----------- 1 file changed, 23 insertions(+), 30 deletions(-) diff --git a/src/main/java/org/breedinginsight/db/migration/V1_0_15__Add_OU_Dataset_Xrefs.java b/src/main/java/org/breedinginsight/db/migration/V1_0_15__Add_OU_Dataset_Xrefs.java index 139d834fd..30cbf9c6f 100644 --- a/src/main/java/org/breedinginsight/db/migration/V1_0_15__Add_OU_Dataset_Xrefs.java +++ b/src/main/java/org/breedinginsight/db/migration/V1_0_15__Add_OU_Dataset_Xrefs.java @@ -132,37 +132,30 @@ public void migrate(Context context) throws Exception { .referenceSource(datasetReferenceSource) .referenceID(exp.getAdditionalInfo().getAsJsonObject().get(BrAPIAdditionalInfoFields.OBSERVATION_DATASET_ID).getAsString()); ou.getExternalReferences().add(datasetRef); - mutatedOuByDbId.put(ou.getObservationUnitDbId(), ou); + try { + + // Send the updated observation unit back to the brapi server + log.debug(String.format("Sending observation unit (id: %s) with observation dataset (id: %s) to %s", + Utilities.getExternalReference(ou.getExternalReferences(), observationUnitReferenceSource), + datasetRef.getReferenceID(), + defaultUrl)); + BrAPIObservationUnit updatedOu = ousApi.observationunitsObservationUnitDbIdPut(ou.getObservationUnitDbId(), ou).getBody().getResult(); + + // Verify that the observation unit was updated at the server + boolean isUpdated = updatedOu.getExternalReferences().stream().anyMatch(xref -> { + return xref.getReferenceSource().equals(datasetReferenceSource) && + xref.getReferenceID().equals(exp.getAdditionalInfo().getAsJsonObject().get(BrAPIAdditionalInfoFields.OBSERVATION_DATASET_ID).getAsString()); + }); + log.debug("Updating observation unit successful: " + String.valueOf(isUpdated)); + if (!isUpdated) { + throw new Exception("Observation unit returned from brapi server was not updated. Check your brapi server."); + } + } catch(Exception e) { + log.error(e.getMessage(), e); + throw new InternalServerException(e.toString(), e); + } }); - try { - - // Send the updated observation unit back to the brapi server - mutatedOuByDbId.values().stream().forEach(ou -> { - log.debug(String.format("Sending observation unit (id: %s) with observation dataset (id: %s) to %s", - Utilities.getExternalReference(ou.getExternalReferences(), observationUnitReferenceSource), - Utilities.getExternalReference(ou.getExternalReferences(), datasetReferenceSource), - defaultUrl)); - }); - List updatedOus = ousApi.observationunitsPut(mutatedOuByDbId).getBody().getResult().getData(); - - // Verify that the observation unit was updated at the server - boolean updateFailed = updatedOus.stream().anyMatch(updatedOu -> { - return !updatedOu.getExternalReferences().stream().anyMatch(xref -> { - return xref.getReferenceSource().equals(datasetReferenceSource) && - xref.getReferenceID().equals(exp.getAdditionalInfo().getAsJsonObject().get(BrAPIAdditionalInfoFields.OBSERVATION_DATASET_ID).getAsString()); - }); - }); - log.debug("Updating observation unit successful: " + String.valueOf(!updateFailed)); - if (updateFailed) { - throw new Exception("Observation unit returned from brapi server was not updated. Check your brapi server."); - } - } catch(Exception e) { - log.error(e.getMessage(), e); - throw new InternalServerException(e.toString(), e); - } - - - + // Fetch the next page of observation units for this experiment if(ousResponse.getBody().getMetadata().getPagination().getCurrentPage() + 1 == ousResponse.getBody().getMetadata().getPagination().getTotalPages()) { ousDone = true; From 35412fef8cb9fb08b47b6978141f011fc973b1f5 Mon Sep 17 00:00:00 2001 From: dmeidlin <14339308+dmeidlin@users.noreply.github.com> Date: Thu, 31 Aug 2023 14:03:35 -0400 Subject: [PATCH 11/14] add TODO --- .../db/migration/V1_0_15__Add_OU_Dataset_Xrefs.java | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/src/main/java/org/breedinginsight/db/migration/V1_0_15__Add_OU_Dataset_Xrefs.java b/src/main/java/org/breedinginsight/db/migration/V1_0_15__Add_OU_Dataset_Xrefs.java index 30cbf9c6f..bcbd9d5e6 100644 --- a/src/main/java/org/breedinginsight/db/migration/V1_0_15__Add_OU_Dataset_Xrefs.java +++ b/src/main/java/org/breedinginsight/db/migration/V1_0_15__Add_OU_Dataset_Xrefs.java @@ -30,18 +30,13 @@ import org.brapi.v2.model.core.response.BrAPITrialListResponse; import org.brapi.v2.model.pheno.BrAPIObservationUnit; import org.brapi.v2.model.pheno.response.BrAPIObservationUnitListResponse; -import org.brapi.v2.model.pheno.response.BrAPIObservationUnitListResponseResult; import org.breedinginsight.brapi.v2.constants.BrAPIAdditionalInfoFields; -import org.breedinginsight.brapps.importer.daos.BrAPIObservationUnitDAO; -import org.breedinginsight.brapps.importer.daos.BrAPITrialDAO; -import org.breedinginsight.brapps.importer.daos.impl.BrAPITrialDAOImpl; import org.breedinginsight.brapps.importer.services.ExternalReferenceSource; import org.breedinginsight.model.Program; import org.breedinginsight.utilities.Utilities; import org.flywaydb.core.api.migration.BaseJavaMigration; import org.flywaydb.core.api.migration.Context; -import javax.inject.Inject; import java.util.*; import java.util.stream.Collectors; @@ -114,7 +109,6 @@ public void migrate(Context context) throws Exception { ousResponse.getBody().getMetadata().getPagination().getCurrentPage()+1, ousResponse.getBody().getMetadata().getPagination().getTotalPages(), exp.getTrialName())); - Map mutatedOuByDbId = new HashMap<>(); ousResponse.getBody().getResult().getData() .stream().filter(ou -> { @@ -142,14 +136,20 @@ public void migrate(Context context) throws Exception { BrAPIObservationUnit updatedOu = ousApi.observationunitsObservationUnitDbIdPut(ou.getObservationUnitDbId(), ou).getBody().getResult(); // Verify that the observation unit was updated at the server + // TODO: Breedbase does not return the updated unit in the successful response, and fetching a single unit via GET returns a an error + // parsing the date time. Once these bugs are fixed in Breedbase, uncomment the verification step below. + /* boolean isUpdated = updatedOu.getExternalReferences().stream().anyMatch(xref -> { return xref.getReferenceSource().equals(datasetReferenceSource) && xref.getReferenceID().equals(exp.getAdditionalInfo().getAsJsonObject().get(BrAPIAdditionalInfoFields.OBSERVATION_DATASET_ID).getAsString()); }); + log.debug("Updating observation unit successful: " + String.valueOf(isUpdated)); if (!isUpdated) { throw new Exception("Observation unit returned from brapi server was not updated. Check your brapi server."); } + + */ } catch(Exception e) { log.error(e.getMessage(), e); throw new InternalServerException(e.toString(), e); From 139072a06acf9b9e5592105d6ee72028598e075a Mon Sep 17 00:00:00 2001 From: dmeidlin <14339308+dmeidlin@users.noreply.github.com> Date: Wed, 13 Sep 2023 13:45:40 -0400 Subject: [PATCH 12/14] Update src/main/java/org/breedinginsight/db/migration/V1_0_15__Add_OU_Dataset_Xrefs.java Co-authored-by: Tim Parsons --- .../V1_0_15__Add_OU_Dataset_Xrefs.java | 46 +++++++------------ 1 file changed, 17 insertions(+), 29 deletions(-) diff --git a/src/main/java/org/breedinginsight/db/migration/V1_0_15__Add_OU_Dataset_Xrefs.java b/src/main/java/org/breedinginsight/db/migration/V1_0_15__Add_OU_Dataset_Xrefs.java index bcbd9d5e6..66b3259a1 100644 --- a/src/main/java/org/breedinginsight/db/migration/V1_0_15__Add_OU_Dataset_Xrefs.java +++ b/src/main/java/org/breedinginsight/db/migration/V1_0_15__Add_OU_Dataset_Xrefs.java @@ -63,8 +63,11 @@ public void migrate(Context context) throws Exception { BrAPIClient client = new BrAPIClient(program.getBrapiUrl(), 240000); log.debug("Migrating observation units for programId: " + program.getId()); + Map ousToUpdate = new HashMap<>(); + // Get the Deltabreed-generated experiments for the program TrialsApi trialsApi = new TrialsApi(client); + ObservationUnitsApi ousApi = new ObservationUnitsApi(client); TrialQueryParams trialQueryParams = new TrialQueryParams(); trialQueryParams.externalReferenceSource(programReferenceSource); trialQueryParams.externalReferenceID(program.getId().toString()); @@ -92,10 +95,10 @@ public void migrate(Context context) throws Exception { expRef.ifPresent(brAPIExternalReference -> ExpIdByDbId.put(exp.getTrialDbId(), brAPIExternalReference.getReferenceID())); }); + for (BrAPITrial exp : experiments) { // Fetch all observation units for an experiment - ObservationUnitsApi ousApi = new ObservationUnitsApi(client); ObservationUnitQueryParams ouQueryParams = new ObservationUnitQueryParams(); ouQueryParams.externalReferenceSource(trialReferenceSource); ouQueryParams.externalReferenceID(ExpIdByDbId.get(exp.getTrialDbId())); @@ -126,34 +129,10 @@ public void migrate(Context context) throws Exception { .referenceSource(datasetReferenceSource) .referenceID(exp.getAdditionalInfo().getAsJsonObject().get(BrAPIAdditionalInfoFields.OBSERVATION_DATASET_ID).getAsString()); ou.getExternalReferences().add(datasetRef); - try { - - // Send the updated observation unit back to the brapi server - log.debug(String.format("Sending observation unit (id: %s) with observation dataset (id: %s) to %s", - Utilities.getExternalReference(ou.getExternalReferences(), observationUnitReferenceSource), - datasetRef.getReferenceID(), - defaultUrl)); - BrAPIObservationUnit updatedOu = ousApi.observationunitsObservationUnitDbIdPut(ou.getObservationUnitDbId(), ou).getBody().getResult(); - - // Verify that the observation unit was updated at the server - // TODO: Breedbase does not return the updated unit in the successful response, and fetching a single unit via GET returns a an error - // parsing the date time. Once these bugs are fixed in Breedbase, uncomment the verification step below. - /* - boolean isUpdated = updatedOu.getExternalReferences().stream().anyMatch(xref -> { - return xref.getReferenceSource().equals(datasetReferenceSource) && - xref.getReferenceID().equals(exp.getAdditionalInfo().getAsJsonObject().get(BrAPIAdditionalInfoFields.OBSERVATION_DATASET_ID).getAsString()); - }); - - log.debug("Updating observation unit successful: " + String.valueOf(isUpdated)); - if (!isUpdated) { - throw new Exception("Observation unit returned from brapi server was not updated. Check your brapi server."); - } - - */ - } catch(Exception e) { - log.error(e.getMessage(), e); - throw new InternalServerException(e.toString(), e); - } + log.debug(String.format("Adding observation unit (id: %s) with observation dataset (id: %s) to update list", + Utilities.getExternalReference(ou.getExternalReferences(), observationUnitReferenceSource), + datasetRef.getReferenceID())); + ousToUpdate.put(ou.getObservationUnitDbId(), ou); }); // Fetch the next page of observation units for this experiment @@ -174,6 +153,15 @@ public void migrate(Context context) throws Exception { trialsResponse = trialsApi.trialsGet(trialQueryParams); } } + + try { + log.debug(String.format("Update OUs for program: %s at url: %s", program.getId(), program.getBrapiUrl())); + ousApi.observationunitsPut(ousToUpdate); + } catch(Exception e) { + log.error(e.getMessage(), e); + throw new InternalServerException(e.toString(), e); + } + } } } } From b6f4117049d8b1092ad23be3e300e510388842ea Mon Sep 17 00:00:00 2001 From: dmeidlin <14339308+dmeidlin@users.noreply.github.com> Date: Wed, 13 Sep 2023 14:26:51 -0400 Subject: [PATCH 13/14] add doc comments for Utilities#getAllProgramsFlyway --- src/main/java/org/breedinginsight/utilities/Utilities.java | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/src/main/java/org/breedinginsight/utilities/Utilities.java b/src/main/java/org/breedinginsight/utilities/Utilities.java index 85693bbc7..7aa390635 100644 --- a/src/main/java/org/breedinginsight/utilities/Utilities.java +++ b/src/main/java/org/breedinginsight/utilities/Utilities.java @@ -186,6 +186,12 @@ public static String makePortableFilename(String name) { return sb.toString(); } + /** + * For only the context of a specific flyway migration, return a list of all Deltabreed programs. + * @param context the context relevant to a Java-based migration + * @param defaultUrl the url for the default BrAPI service + * @return a list of all Deltabreed programs + */ public static List getAllProgramsFlyway(Context context, String defaultUrl) throws Exception { List programs = new ArrayList<>(); try (Statement select = context.getConnection().createStatement()) { From 01515bfcc3e89bb02e417d8951bc7f3c1a96f79e Mon Sep 17 00:00:00 2001 From: dmeidlin <14339308+dmeidlin@users.noreply.github.com> Date: Thu, 14 Sep 2023 12:01:58 -0400 Subject: [PATCH 14/14] fix typo --- .../V1_0_15__Add_OU_Dataset_Xrefs.java | 24 +++++++++---------- 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/src/main/java/org/breedinginsight/db/migration/V1_0_15__Add_OU_Dataset_Xrefs.java b/src/main/java/org/breedinginsight/db/migration/V1_0_15__Add_OU_Dataset_Xrefs.java index 66b3259a1..d691629ec 100644 --- a/src/main/java/org/breedinginsight/db/migration/V1_0_15__Add_OU_Dataset_Xrefs.java +++ b/src/main/java/org/breedinginsight/db/migration/V1_0_15__Add_OU_Dataset_Xrefs.java @@ -76,14 +76,14 @@ public void migrate(Context context) throws Exception { ApiResponse trialsResponse = trialsApi.trialsGet(trialQueryParams); boolean trialsDone = trialsResponse.getBody() == null || trialsResponse.getBody().getMetadata().getPagination().getTotalCount() == 0 || trialsResponse.getBody().getResult() == null; - while(!trialsDone) { + while (!trialsDone) { log.debug(String.format("processing page %d of %d of experiments for programId: %s", - trialsResponse.getBody().getMetadata().getPagination().getCurrentPage()+1, + trialsResponse.getBody().getMetadata().getPagination().getCurrentPage() + 1, trialsResponse.getBody().getMetadata().getPagination().getTotalPages(), program.getId())); List experiments = trialsResponse.getBody().getResult().getData().stream().filter(trial -> { List xrefs = trial.getExternalReferences(); - Optional programRef = Utilities.getExternalReference(xrefs,programReferenceSource); + Optional programRef = Utilities.getExternalReference(xrefs, programReferenceSource); return trial.getAdditionalInfo().getAsJsonObject().has(BrAPIAdditionalInfoFields.OBSERVATION_DATASET_ID) && programRef.isPresent() && program.getId().equals(UUID.fromString(programRef.get().getReferenceID())); @@ -107,9 +107,9 @@ public void migrate(Context context) throws Exception { ApiResponse ousResponse = ousApi.observationunitsGet(ouQueryParams); boolean ousDone = ousResponse.getBody() == null || ousResponse.getBody().getMetadata().getPagination().getTotalCount() == 0 || ousResponse.getBody().getResult() == null; - while(!ousDone) { + while (!ousDone) { log.debug(String.format("processing page %d of %d of observation units for experiment: %s", - ousResponse.getBody().getMetadata().getPagination().getCurrentPage()+1, + ousResponse.getBody().getMetadata().getPagination().getCurrentPage() + 1, ousResponse.getBody().getMetadata().getPagination().getTotalPages(), exp.getTrialName())); ousResponse.getBody().getResult().getData() @@ -130,13 +130,13 @@ public void migrate(Context context) throws Exception { .referenceID(exp.getAdditionalInfo().getAsJsonObject().get(BrAPIAdditionalInfoFields.OBSERVATION_DATASET_ID).getAsString()); ou.getExternalReferences().add(datasetRef); log.debug(String.format("Adding observation unit (id: %s) with observation dataset (id: %s) to update list", - Utilities.getExternalReference(ou.getExternalReferences(), observationUnitReferenceSource), - datasetRef.getReferenceID())); + Utilities.getExternalReference(ou.getExternalReferences(), observationUnitReferenceSource), + datasetRef.getReferenceID())); ousToUpdate.put(ou.getObservationUnitDbId(), ou); }); - + // Fetch the next page of observation units for this experiment - if(ousResponse.getBody().getMetadata().getPagination().getCurrentPage() + 1 == ousResponse.getBody().getMetadata().getPagination().getTotalPages()) { + if (ousResponse.getBody().getMetadata().getPagination().getCurrentPage() + 1 == ousResponse.getBody().getMetadata().getPagination().getTotalPages()) { ousDone = true; } else { ouQueryParams.page(ouQueryParams.page() + 1); @@ -146,7 +146,7 @@ public void migrate(Context context) throws Exception { } // Fetch the next page of experiments for this program - if(trialsResponse.getBody().getMetadata().getPagination().getCurrentPage() + 1 == trialsResponse.getBody().getMetadata().getPagination().getTotalPages()) { + if (trialsResponse.getBody().getMetadata().getPagination().getCurrentPage() + 1 == trialsResponse.getBody().getMetadata().getPagination().getTotalPages()) { trialsDone = true; } else { trialQueryParams.page(trialQueryParams.page() + 1); @@ -157,11 +157,11 @@ public void migrate(Context context) throws Exception { try { log.debug(String.format("Update OUs for program: %s at url: %s", program.getId(), program.getBrapiUrl())); ousApi.observationunitsPut(ousToUpdate); - } catch(Exception e) { + } catch (Exception e) { log.error(e.getMessage(), e); throw new InternalServerException(e.toString(), e); } - } + } } }