diff --git a/src/main/java/org/breedinginsight/brapi/v2/services/BrAPITrialService.java b/src/main/java/org/breedinginsight/brapi/v2/services/BrAPITrialService.java index 182882ff5..8c002d27c 100644 --- a/src/main/java/org/breedinginsight/brapi/v2/services/BrAPITrialService.java +++ b/src/main/java/org/breedinginsight/brapi/v2/services/BrAPITrialService.java @@ -296,18 +296,14 @@ private StreamedFile zipFiles(List files) throws IOException { } public Dataset getDatasetData(Program program, UUID experimentId, UUID datsetId, Boolean stats) throws ApiException, DoesNotExistException { - BrAPITrial experiment = this.getExperiment(program, experimentId); - - // TODO: Once BI-1831 is complete and OUs in a dataset can be identified using the datasetId stored as a xref - // the expOUs needs to be replaced with datasetOUs, as was done with datasetObsVars - List expOUs = ouDAO.getObservationUnitsForTrialDbId(program.getId(), experiment.getTrialDbId(), true); + List datasetOUs = ouDAO.getObservationUnitsForDataset(datsetId.toString(), program); List datasetObsVars = getDatasetObsVars(datsetId.toString(), program); - List ouDbIds = expOUs.stream().map(BrAPIObservationUnit::getObservationUnitDbId).collect(Collectors.toList()); + List ouDbIds = datasetOUs.stream().map(BrAPIObservationUnit::getObservationUnitDbId).collect(Collectors.toList()); List obsVarDbIds = datasetObsVars.stream().map(BrAPIObservationVariable::getObservationVariableDbId).collect(Collectors.toList()); List data = observationDAO.getObservationsByObservationUnitsAndVariables(ouDbIds, obsVarDbIds, program); - Dataset dataset = new Dataset(experimentId.toString(), data, expOUs, datasetObsVars); + Dataset dataset = new Dataset(experimentId.toString(), data, datasetOUs, datasetObsVars); if (stats) { - Integer ouCount = expOUs.size(); + Integer ouCount = datasetOUs.size(); Integer obsVarCount = datasetObsVars.size(); Integer obsCount = ouCount * obsVarCount; Integer obsDataCount = data.size(); diff --git a/src/main/java/org/breedinginsight/brapps/importer/daos/BrAPIObservationUnitDAO.java b/src/main/java/org/breedinginsight/brapps/importer/daos/BrAPIObservationUnitDAO.java index 02347e62d..65d81712b 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/daos/BrAPIObservationUnitDAO.java +++ b/src/main/java/org/breedinginsight/brapps/importer/daos/BrAPIObservationUnitDAO.java @@ -23,8 +23,10 @@ import com.google.gson.reflect.TypeToken; import io.micronaut.context.annotation.Property; import org.brapi.client.v2.JSON; +import io.micronaut.http.server.exceptions.InternalServerException; import org.brapi.client.v2.model.exceptions.ApiException; import org.brapi.client.v2.modules.phenotype.ObservationUnitsApi; +import org.brapi.v2.model.BrAPIExternalReference; import org.brapi.v2.model.germ.BrAPIGermplasm; import org.brapi.v2.model.pheno.BrAPIObservationUnit; import org.brapi.v2.model.pheno.request.BrAPIObservationUnitSearchRequest; @@ -42,6 +44,7 @@ import org.breedinginsight.services.brapi.BrAPIEndpointProvider; import org.breedinginsight.services.exceptions.DoesNotExistException; import org.breedinginsight.utilities.BrAPIDAOUtil; +import org.breedinginsight.utilities.Utilities; import javax.inject.Inject; import javax.inject.Singleton; @@ -64,7 +67,13 @@ public class BrAPIObservationUnitDAO { private final Type treatmentlistType = new TypeToken>(){}.getType(); @Inject - public BrAPIObservationUnitDAO(ProgramDAO programDAO, ImportDAO importDAO, BrAPIDAOUtil brAPIDAOUtil, BrAPIEndpointProvider brAPIEndpointProvider, BrAPIGermplasmService germplasmService, ProgramService programService, @Property(name = "brapi.server.reference-source") String referenceSource) { + public BrAPIObservationUnitDAO(ProgramDAO programDAO, + ImportDAO importDAO, + BrAPIDAOUtil brAPIDAOUtil, + BrAPIEndpointProvider brAPIEndpointProvider, + BrAPIGermplasmService germplasmService, + ProgramService programService, + @Property(name = "brapi.server.reference-source") String referenceSource) { this.programDAO = programDAO; this.importDAO = importDAO; this.brAPIDAOUtil = brAPIDAOUtil; @@ -97,6 +106,19 @@ public List createBrAPIObservationUnits(List getObservationUnitsById(Collection observationUnitExternalIds, Program program) throws ApiException { if(observationUnitExternalIds.isEmpty()) { return Collections.emptyList(); @@ -124,6 +146,18 @@ public List getObservationUnitsForTrialDbId(@NotNull UUID return getObservationUnitsForTrialDbId(programId, trialDbId, false); } + public List getObservationUnitsForDataset(@NotNull String datasetId, @NotNull Program program) throws ApiException { + String datasetReferenceSource = Utilities.generateReferenceSource(referenceSource, ExternalReferenceSource.DATASET); + BrAPIObservationUnitSearchRequest ouSearchRequest = new BrAPIObservationUnitSearchRequest(); + ouSearchRequest.programDbIds(List.of(program.getBrapiProgram().getProgramDbId())); + ouSearchRequest.externalReferenceSources(List.of(datasetReferenceSource)); + ouSearchRequest.externalReferenceIDs(List.of(datasetId)); + ObservationUnitsApi api = brAPIEndpointProvider.get(programDAO.getCoreClient(program.getId()), ObservationUnitsApi.class); + return brAPIDAOUtil.search(api::searchObservationunitsPost, + api::searchObservationunitsSearchResultsDbIdGet, + ouSearchRequest); + } + public List getObservationUnitsForTrialDbId(@NotNull UUID programId, @NotNull String trialDbId, boolean withGID) throws ApiException, DoesNotExistException { Program program = programService.getById(programId).orElseThrow(() -> new DoesNotExistException("Program id does not exist")); diff --git a/src/main/java/org/breedinginsight/brapps/importer/model/imports/experimentObservation/ExperimentObservation.java b/src/main/java/org/breedinginsight/brapps/importer/model/imports/experimentObservation/ExperimentObservation.java index 4159d6008..727e34caf 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/model/imports/experimentObservation/ExperimentObservation.java +++ b/src/main/java/org/breedinginsight/brapps/importer/model/imports/experimentObservation/ExperimentObservation.java @@ -223,6 +223,7 @@ public BrAPIObservationUnit constructBrAPIObservationUnit( String germplasmName, String referenceSource, UUID trialID, + UUID datasetId, UUID studyID, UUID id ) { @@ -232,7 +233,7 @@ public BrAPIObservationUnit constructBrAPIObservationUnit( observationUnit.setObservationUnitName(Utilities.appendProgramKey(getExpUnitId(), program.getKey(), seqVal)); // Set external reference - observationUnit.setExternalReferences(getObsUnitExternalReferences(program, referenceSource, trialID, studyID, id)); + observationUnit.setExternalReferences(getObsUnitExternalReferences(program, referenceSource, trialID, datasetId, studyID, id)); } else { observationUnit.setObservationUnitName(getExpUnitId()); } @@ -345,13 +346,16 @@ public BrAPIObservation constructBrAPIObservation( } private List getBrAPIExternalReferences( - Program program, String referenceSourceBaseName, UUID trialId, UUID studyId, UUID obsUnitId, UUID observationId) { + Program program, String referenceSourceBaseName, UUID trialId, UUID datasetId, UUID studyId, UUID obsUnitId, UUID observationId) { List refs = new ArrayList<>(); addReference(refs, program.getId(), referenceSourceBaseName, ExternalReferenceSource.PROGRAMS); if (trialId != null) { addReference(refs, trialId, referenceSourceBaseName, ExternalReferenceSource.TRIALS); } + if (datasetId != null) { + addReference(refs, datasetId, referenceSourceBaseName, ExternalReferenceSource.DATASET); + } if (studyId != null) { addReference(refs, studyId, referenceSourceBaseName, ExternalReferenceSource.STUDIES); } @@ -367,22 +371,22 @@ private List getBrAPIExternalReferences( private List getTrialExternalReferences( Program program, String referenceSourceBaseName, UUID trialId) { - return getBrAPIExternalReferences(program, referenceSourceBaseName, trialId, null, null, null); + return getBrAPIExternalReferences(program, referenceSourceBaseName, trialId, null, null, null, null); } private List getStudyExternalReferences( Program program, String referenceSourceBaseName, UUID trialId, UUID studyId) { - return getBrAPIExternalReferences(program, referenceSourceBaseName, trialId, studyId, null, null); + return getBrAPIExternalReferences(program, referenceSourceBaseName, trialId, null, studyId, null, null); } private List getObsUnitExternalReferences( - Program program, String referenceSourceBaseName, UUID trialId, UUID studyId, UUID obsUnitId) { - return getBrAPIExternalReferences(program, referenceSourceBaseName, trialId, studyId, obsUnitId, null); + Program program, String referenceSourceBaseName, UUID trialId, UUID datasetId, UUID studyId, UUID obsUnitId) { + return getBrAPIExternalReferences(program, referenceSourceBaseName, trialId, datasetId, studyId, obsUnitId, null); } private List getObservationExternalReferences( Program program, String referenceSourceBaseName, UUID trialId, UUID studyId, UUID obsUnitId, UUID observationId) { - return getBrAPIExternalReferences(program, referenceSourceBaseName, trialId, studyId, obsUnitId, observationId); + return getBrAPIExternalReferences(program, referenceSourceBaseName, trialId, null, studyId, obsUnitId, observationId); } diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/ExperimentProcessor.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/ExperimentProcessor.java index dcc3339c6..c9e7f5aa0 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/ExperimentProcessor.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/ExperimentProcessor.java @@ -41,6 +41,7 @@ import org.breedinginsight.brapi.v2.dao.BrAPIGermplasmDAO; import org.breedinginsight.brapps.importer.daos.*; import org.breedinginsight.brapps.importer.model.ImportUpload; +import org.breedinginsight.brapps.importer.model.base.AdditionalInfo; import org.breedinginsight.brapps.importer.model.imports.BrAPIImport; import org.breedinginsight.brapps.importer.model.imports.PendingImport; import org.breedinginsight.brapps.importer.model.imports.experimentObservation.ExperimentObservation; @@ -805,10 +806,16 @@ private PendingImportObject fetchOrCreateObsUnitPIO(Progra } PendingImportObject trialPIO = this.trialByNameNoScope.get(importRow.getExpTitle()); UUID trialID = trialPIO.getId(); + UUID datasetId = null; + if (commit) { + datasetId = UUID.fromString(trialPIO.getBrAPIObject() + .getAdditionalInfo().getAsJsonObject() + .get(BrAPIAdditionalInfoFields.OBSERVATION_DATASET_ID).getAsString()); + } PendingImportObject studyPIO = this.studyByNameNoScope.get(importRow.getEnv()); UUID studyID = studyPIO.getId(); UUID id = UUID.randomUUID(); - BrAPIObservationUnit newObservationUnit = importRow.constructBrAPIObservationUnit(program, envSeqValue, commit, germplasmName, BRAPI_REFERENCE_SOURCE, trialID, studyID, id); + BrAPIObservationUnit newObservationUnit = importRow.constructBrAPIObservationUnit(program, envSeqValue, commit, germplasmName, BRAPI_REFERENCE_SOURCE, trialID, datasetId, studyID, id); pio = new PendingImportObject<>(ImportObjectState.NEW, newObservationUnit, id); this.observationUnitByNameNoScope.put(key, pio); } diff --git a/src/main/java/org/breedinginsight/db/migration/V1_0_13__Update_BrAPI_Locations_XRefs.java b/src/main/java/org/breedinginsight/db/migration/V1_0_13__Update_BrAPI_Locations_XRefs.java index 6effc5138..fadc89421 100644 --- a/src/main/java/org/breedinginsight/db/migration/V1_0_13__Update_BrAPI_Locations_XRefs.java +++ b/src/main/java/org/breedinginsight/db/migration/V1_0_13__Update_BrAPI_Locations_XRefs.java @@ -49,7 +49,7 @@ public void migrate(Context context) throws Exception { String referenceSource = placeholders.get(BRAPI_REFERENCE_SOURCE_KEY); // Get all the programs - List programs = getAllPrograms(context, defaultUrl); + List programs = Utilities.getAllProgramsFlyway(context, defaultUrl); Map locationsApiForProgram = new HashMap<>(); for (Program program : programs) { BrAPIClient client = new BrAPIClient(program.getBrapiUrl(), 240000); @@ -103,22 +103,4 @@ private List getAllLocations(Context context) throws SQLExcepti } return locations; } - - private List getAllPrograms(Context context, String defaultUrl) throws Exception { - List programs = new ArrayList<>(); - try (Statement select = context.getConnection().createStatement()) { - try (ResultSet rows = select.executeQuery("SELECT id, brapi_url, key FROM program where active = true ORDER BY id")) { - while (rows.next()) { - Program program = new Program(); - program.setId(UUID.fromString(rows.getString(1))); - String brapi_url = rows.getString(2); - if (brapi_url == null) brapi_url = defaultUrl; - program.setBrapiUrl(brapi_url); - program.setKey(rows.getString(3)); - programs.add(program); - } - } - } - return programs; - } } \ No newline at end of file diff --git a/src/main/java/org/breedinginsight/db/migration/V1_0_15__Add_OU_Dataset_Xrefs.java b/src/main/java/org/breedinginsight/db/migration/V1_0_15__Add_OU_Dataset_Xrefs.java new file mode 100644 index 000000000..d691629ec --- /dev/null +++ b/src/main/java/org/breedinginsight/db/migration/V1_0_15__Add_OU_Dataset_Xrefs.java @@ -0,0 +1,167 @@ +/* + * See the NOTICE file distributed with this work for additional information + * regarding copyright ownership. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.breedinginsight.db.migration; + +import io.micronaut.http.server.exceptions.InternalServerException; +import lombok.extern.slf4j.Slf4j; +import org.brapi.client.v2.ApiResponse; +import org.brapi.client.v2.BrAPIClient; +import org.brapi.client.v2.model.queryParams.core.TrialQueryParams; +import org.brapi.client.v2.model.queryParams.phenotype.ObservationUnitQueryParams; +import org.brapi.client.v2.modules.core.TrialsApi; +import org.brapi.client.v2.modules.phenotype.ObservationUnitsApi; +import org.brapi.v2.model.BrAPIExternalReference; +import org.brapi.v2.model.core.BrAPITrial; +import org.brapi.v2.model.core.response.BrAPITrialListResponse; +import org.brapi.v2.model.pheno.BrAPIObservationUnit; +import org.brapi.v2.model.pheno.response.BrAPIObservationUnitListResponse; +import org.breedinginsight.brapi.v2.constants.BrAPIAdditionalInfoFields; +import org.breedinginsight.brapps.importer.services.ExternalReferenceSource; +import org.breedinginsight.model.Program; +import org.breedinginsight.utilities.Utilities; +import org.flywaydb.core.api.migration.BaseJavaMigration; +import org.flywaydb.core.api.migration.Context; + +import java.util.*; +import java.util.stream.Collectors; + +@Slf4j +public class V1_0_15__Add_OU_Dataset_Xrefs extends BaseJavaMigration { + + + public void migrate(Context context) throws Exception { + Map placeholders = context.getConfiguration().getPlaceholders(); + String DEFAULT_URL_KEY = "default-url"; + String defaultUrl = placeholders.get(DEFAULT_URL_KEY); + String BRAPI_REFERENCE_SOURCE_KEY = "brapi-reference-source"; + String referenceSource = placeholders.get(BRAPI_REFERENCE_SOURCE_KEY); + String programReferenceSource = Utilities.generateReferenceSource(referenceSource, ExternalReferenceSource.PROGRAMS); + String trialReferenceSource = Utilities.generateReferenceSource(referenceSource, ExternalReferenceSource.TRIALS); + String observationUnitReferenceSource = Utilities.generateReferenceSource(referenceSource, ExternalReferenceSource.OBSERVATION_UNITS); + String datasetReferenceSource = Utilities.generateReferenceSource(referenceSource, ExternalReferenceSource.DATASET); + + // Get all the programs + List programs = Utilities.getAllProgramsFlyway(context, defaultUrl); + + // For each program, update any observation units created via Deltabreed + for (Program program : programs) { + BrAPIClient client = new BrAPIClient(program.getBrapiUrl(), 240000); + log.debug("Migrating observation units for programId: " + program.getId()); + + Map ousToUpdate = new HashMap<>(); + + // Get the Deltabreed-generated experiments for the program + TrialsApi trialsApi = new TrialsApi(client); + ObservationUnitsApi ousApi = new ObservationUnitsApi(client); + TrialQueryParams trialQueryParams = new TrialQueryParams(); + trialQueryParams.externalReferenceSource(programReferenceSource); + trialQueryParams.externalReferenceID(program.getId().toString()); + trialQueryParams.page(0); + trialQueryParams.pageSize(1000); + ApiResponse trialsResponse = trialsApi.trialsGet(trialQueryParams); + + boolean trialsDone = trialsResponse.getBody() == null || trialsResponse.getBody().getMetadata().getPagination().getTotalCount() == 0 || trialsResponse.getBody().getResult() == null; + while (!trialsDone) { + log.debug(String.format("processing page %d of %d of experiments for programId: %s", + trialsResponse.getBody().getMetadata().getPagination().getCurrentPage() + 1, + trialsResponse.getBody().getMetadata().getPagination().getTotalPages(), + program.getId())); + List experiments = trialsResponse.getBody().getResult().getData().stream().filter(trial -> { + List xrefs = trial.getExternalReferences(); + Optional programRef = Utilities.getExternalReference(xrefs, programReferenceSource); + return trial.getAdditionalInfo().getAsJsonObject().has(BrAPIAdditionalInfoFields.OBSERVATION_DATASET_ID) && + programRef.isPresent() && + program.getId().equals(UUID.fromString(programRef.get().getReferenceID())); + }).collect(Collectors.toList()); + + Map ExpIdByDbId = new HashMap<>(); + experiments.forEach(exp -> { + Optional expRef = Utilities.getExternalReference(exp.getExternalReferences(), trialReferenceSource); + expRef.ifPresent(brAPIExternalReference -> ExpIdByDbId.put(exp.getTrialDbId(), brAPIExternalReference.getReferenceID())); + }); + + + for (BrAPITrial exp : experiments) { + + // Fetch all observation units for an experiment + ObservationUnitQueryParams ouQueryParams = new ObservationUnitQueryParams(); + ouQueryParams.externalReferenceSource(trialReferenceSource); + ouQueryParams.externalReferenceID(ExpIdByDbId.get(exp.getTrialDbId())); + ouQueryParams.page(0); + ouQueryParams.pageSize(1000); + ApiResponse ousResponse = ousApi.observationunitsGet(ouQueryParams); + + boolean ousDone = ousResponse.getBody() == null || ousResponse.getBody().getMetadata().getPagination().getTotalCount() == 0 || ousResponse.getBody().getResult() == null; + while (!ousDone) { + log.debug(String.format("processing page %d of %d of observation units for experiment: %s", + ousResponse.getBody().getMetadata().getPagination().getCurrentPage() + 1, + ousResponse.getBody().getMetadata().getPagination().getTotalPages(), + exp.getTrialName())); + ousResponse.getBody().getResult().getData() + .stream().filter(ou -> { + + // Find the observation units that need a dataset reference + List xrefs = ou.getExternalReferences(); + Optional expRef = Utilities.getExternalReference(xrefs, trialReferenceSource); + Optional datasetRef = Utilities.getExternalReference(xrefs, datasetReferenceSource); + return datasetRef.isEmpty() && + expRef.isPresent() && + ExpIdByDbId.get(exp.getTrialDbId()).equals(expRef.get().getReferenceID()); + }).forEach(ou -> { + + // Assign the experiment Observation Dataset id to the observation units + BrAPIExternalReference datasetRef = new BrAPIExternalReference() + .referenceSource(datasetReferenceSource) + .referenceID(exp.getAdditionalInfo().getAsJsonObject().get(BrAPIAdditionalInfoFields.OBSERVATION_DATASET_ID).getAsString()); + ou.getExternalReferences().add(datasetRef); + log.debug(String.format("Adding observation unit (id: %s) with observation dataset (id: %s) to update list", + Utilities.getExternalReference(ou.getExternalReferences(), observationUnitReferenceSource), + datasetRef.getReferenceID())); + ousToUpdate.put(ou.getObservationUnitDbId(), ou); + }); + + // Fetch the next page of observation units for this experiment + if (ousResponse.getBody().getMetadata().getPagination().getCurrentPage() + 1 == ousResponse.getBody().getMetadata().getPagination().getTotalPages()) { + ousDone = true; + } else { + ouQueryParams.page(ouQueryParams.page() + 1); + ousResponse = ousApi.observationunitsGet(ouQueryParams); + } + } + } + + // Fetch the next page of experiments for this program + if (trialsResponse.getBody().getMetadata().getPagination().getCurrentPage() + 1 == trialsResponse.getBody().getMetadata().getPagination().getTotalPages()) { + trialsDone = true; + } else { + trialQueryParams.page(trialQueryParams.page() + 1); + trialsResponse = trialsApi.trialsGet(trialQueryParams); + } + } + + try { + log.debug(String.format("Update OUs for program: %s at url: %s", program.getId(), program.getBrapiUrl())); + ousApi.observationunitsPut(ousToUpdate); + } catch (Exception e) { + log.error(e.getMessage(), e); + throw new InternalServerException(e.toString(), e); + } + + } + } +} diff --git a/src/main/java/org/breedinginsight/utilities/Utilities.java b/src/main/java/org/breedinginsight/utilities/Utilities.java index 9ce3a1b9e..7aa390635 100644 --- a/src/main/java/org/breedinginsight/utilities/Utilities.java +++ b/src/main/java/org/breedinginsight/utilities/Utilities.java @@ -21,7 +21,12 @@ import org.brapi.client.v2.model.exceptions.ApiException; import org.brapi.v2.model.BrAPIExternalReference; import org.breedinginsight.brapps.importer.services.ExternalReferenceSource; +import org.breedinginsight.model.Program; +import org.flywaydb.core.api.migration.Context; +import java.sql.ResultSet; +import java.sql.Statement; +import java.util.ArrayList; import java.util.List; import java.util.Optional; import java.util.UUID; @@ -181,6 +186,30 @@ public static String makePortableFilename(String name) { return sb.toString(); } + /** + * For only the context of a specific flyway migration, return a list of all Deltabreed programs. + * @param context the context relevant to a Java-based migration + * @param defaultUrl the url for the default BrAPI service + * @return a list of all Deltabreed programs + */ + public static List getAllProgramsFlyway(Context context, String defaultUrl) throws Exception { + List programs = new ArrayList<>(); + try (Statement select = context.getConnection().createStatement()) { + try (ResultSet rows = select.executeQuery("SELECT id, brapi_url, key FROM program where active = true ORDER BY id")) { + while (rows.next()) { + Program program = new Program(); + program.setId(UUID.fromString(rows.getString(1))); + String brapi_url = rows.getString(2); + if (brapi_url == null) brapi_url = defaultUrl; + program.setBrapiUrl(brapi_url); + program.setKey(rows.getString(3)); + programs.add(program); + } + } + } + return programs; + } + private static boolean isSafeChar(char c) { // Check if c is in the portable filename character set. // See https://pubs.opengroup.org/onlinepubs/9699919799/basedefs/V1_chap03.html#tag_03_282