From 6bb52501442ee5f6899bb658782b34180dc0ac8f Mon Sep 17 00:00:00 2001 From: David Randolph Phillips Date: Tue, 13 Sep 2022 10:56:14 -0400 Subject: [PATCH 1/9] [Bi-1594] TrialId was being gernerated and saved inconsistently --- .../processors/ExperimentProcessor.java | 25 +++++++++++-------- 1 file changed, 14 insertions(+), 11 deletions(-) diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/ExperimentProcessor.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/ExperimentProcessor.java index 7718bfa8a..5de29aba2 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/ExperimentProcessor.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/ExperimentProcessor.java @@ -606,20 +606,12 @@ private Map> initialize_studyByNameNoSco ExperimentObservation experimentObservation = experimentImportRows.get(0); PendingImportObject trial = this.trialByNameNoScope.get(experimentObservation.getExpTitle()); - List experimentRefs = trial.getBrAPIObject().getExternalReferences(); - Optional experimentIDRef = experimentRefs.stream() - .filter(this::isTrialRefSource) - .findFirst(); - if( experimentIDRef.isEmpty()){ - throw new InternalServerException("An Experiment ID was not found any of the external references"); - } - //get experimentID - String experimentIDStr = experimentIDRef.get().getReferenceID(); + UUID experimentId = trial.getId(); List existingStudies; try { - existingStudies = brAPIStudyDAO.getStudiesByExperimentID(UUID.fromString(experimentIDStr), program); + existingStudies = brAPIStudyDAO.getStudiesByExperimentID(experimentId, program); existingStudies.forEach(existingStudy -> { //Swap season DbId with year String String seasonId = existingStudy.getSeasons().get(0); @@ -672,9 +664,20 @@ private Map> initialize_trialByNameNoSco existingTrials = brapiTrialDAO.getTrialByName(uniqueTrialNames, program); existingTrials.forEach(existingTrial -> { existingTrial.setTrialName(Utilities.removeProgramKey(existingTrial.getTrialName(), program.getKey())); + + //get TrialId from existingTrial + List experimentRefs = existingTrial.getExternalReferences(); + Optional experimentIDRef = experimentRefs.stream() + .filter(this::isTrialRefSource) + .findFirst(); + if( experimentIDRef.isEmpty()){ + throw new InternalServerException("An Experiment ID was not found any of the external references"); + } + UUID experimentId = UUID.fromString( experimentIDRef.get().getReferenceID() ); + trialByNameNoScope.put( existingTrial.getTrialName(), - new PendingImportObject<>(ImportObjectState.EXISTING, existingTrial)); + new PendingImportObject<>(ImportObjectState.EXISTING, existingTrial, experimentId)); }); return trialByNameNoScope; } catch (ApiException e) { From 195598d10a2986d31ddd088326dfa89a9cd32509 Mon Sep 17 00:00:00 2001 From: David Randolph Phillips Date: Tue, 20 Sep 2022 08:48:42 -0400 Subject: [PATCH 2/9] [BI-1594] fix bug in the experiment sequence (and observation unit sequence) --- .../ExperimentObservation.java | 6 ++-- .../processors/ExperimentProcessor.java | 35 +++++++++++-------- 2 files changed, 24 insertions(+), 17 deletions(-) diff --git a/src/main/java/org/breedinginsight/brapps/importer/model/imports/experimentObservation/ExperimentObservation.java b/src/main/java/org/breedinginsight/brapps/importer/model/imports/experimentObservation/ExperimentObservation.java index cb64a5a9a..0e87e9012 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/model/imports/experimentObservation/ExperimentObservation.java +++ b/src/main/java/org/breedinginsight/brapps/importer/model/imports/experimentObservation/ExperimentObservation.java @@ -179,7 +179,7 @@ public BrAPIStudy constructBrAPIStudy( public BrAPIObservationUnit constructBrAPIObservationUnit( Program program, - Supplier nextVal, + String nextObjsUnitSeqVal, boolean commit, String germplasmName, String referenceSource, @@ -190,7 +190,7 @@ public BrAPIObservationUnit constructBrAPIObservationUnit( BrAPIObservationUnit observationUnit = new BrAPIObservationUnit(); if( commit){ - observationUnit.setObservationUnitName( Utilities.appendProgramKey(getExpUnitId(), program.getKey(), nextVal.get().toString())); + observationUnit.setObservationUnitName( Utilities.appendProgramKey(getExpUnitId(), program.getKey(), nextObjsUnitSeqVal) ); // Set external reference observationUnit.setExternalReferences(getObsUnitExternalReferences(program, referenceSource, trialID, studyID, id)); @@ -208,7 +208,7 @@ public BrAPIObservationUnit constructBrAPIObservationUnit( BrAPIObservationUnitPosition position = new BrAPIObservationUnitPosition(); BrAPIObservationUnitLevelRelationship level = new BrAPIObservationUnitLevelRelationship(); level.setLevelName("plot"); //BreedBase only accepts "plot" or "plant" - level.setLevelCode( getExpUnitId() ); + level.setLevelCode( Utilities.appendProgramKey(getExpUnitId(), program.getKey(), nextObjsUnitSeqVal) ); position.setObservationLevel(level); observationUnit.putAdditionalInfoItem(BrAPIAdditionalInfoFields.OBSERVATION_LEVEL, getExpUnit()); diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/ExperimentProcessor.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/ExperimentProcessor.java index 5de29aba2..a57817f86 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/ExperimentProcessor.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/ExperimentProcessor.java @@ -33,6 +33,7 @@ import org.brapi.v2.model.pheno.*; import org.breedinginsight.api.model.v1.response.ValidationError; import org.breedinginsight.api.model.v1.response.ValidationErrors; +import org.breedinginsight.brapi.v2.constants.BrAPIAdditionalInfoFields; import org.breedinginsight.brapi.v2.dao.BrAPIGermplasmDAO; import org.breedinginsight.brapps.importer.daos.*; import org.breedinginsight.brapps.importer.model.ImportUpload; @@ -199,23 +200,24 @@ private void getNewBrapiData(List importRows, Program program, bool } Supplier obsUnitNextVal = () -> dsl.nextval(obsUnitSequenceName.toLowerCase()); - // It is assumed, at this point, that there is only one experiment per import-file - String expSeqValue = null; - if (commit) { - String expUnitSequenceName = program.getExpSequence(); - if (expUnitSequenceName == null) { - log.error(String.format("Program, %s, is missing a value in the exp sequence column.", program.getName())); - throw new HttpStatusException(HttpStatus.UNPROCESSABLE_ENTITY, "Program is not properly configured for experiment import"); - } - expSeqValue = dsl.nextval(expUnitSequenceName.toLowerCase()).toString(); + String expSequenceName = program.getObsUnitSequence(); + if (expSequenceName == null) { + log.error(String.format("Program, %s, is missing a value in the evp sequence column.", program.getName())); + throw new HttpStatusException(HttpStatus.UNPROCESSABLE_ENTITY, "Program is not properly configured for observation unit import"); } + Supplier expNextVal = () -> dsl.nextval(expSequenceName.toLowerCase()); for (BrAPIImport row : importRows) { ExperimentObservation importRow = (ExperimentObservation) row; - PendingImportObject trialPIO = createTrialPIO(program, commit, importRow, expSeqValue); + PendingImportObject trialPIO = createTrialPIO(program, commit, importRow, expNextVal); this.trialByNameNoScope.put(importRow.getExpTitle(), trialPIO); + String expSeqValue = null; + if(commit) { + expSeqValue = trialPIO.getBrAPIObject().getAdditionalInfo().get(BrAPIAdditionalInfoFields.EXPERIMENT_NUMBER).getAsString(); + } + PendingImportObject locationPIO = createLocationPIO(importRow); this.locationByName.put(importRow.getEnvLocation(), locationPIO); @@ -396,13 +398,14 @@ private PendingImportObject createObsUnitPIO(Program progr PendingImportObject studyPIO = this.studyByNameNoScope.get(importRow.getEnv()); UUID studyID = studyPIO.getId(); UUID id = UUID.randomUUID(); - BrAPIObservationUnit newObservationUnit = importRow.constructBrAPIObservationUnit(program, obsUnitNextVal, commit, germplasmName, BRAPI_REFERENCE_SOURCE, trialID, studyID, id); + String obsUnitSeqValue = obsUnitNextVal.get().toString(); + BrAPIObservationUnit newObservationUnit = importRow.constructBrAPIObservationUnit(program, obsUnitSeqValue, commit, germplasmName, BRAPI_REFERENCE_SOURCE, trialID, studyID, id); pio = new PendingImportObject<>(ImportObjectState.NEW, newObservationUnit); } return pio; } - private PendingImportObject createStudyPIO(Program program, boolean commit, String expSeqenceValue, ExperimentObservation importRow) { + private PendingImportObject createStudyPIO(Program program, boolean commit, String expSequenceValue, ExperimentObservation importRow) { PendingImportObject pio = null; if( studyByNameNoScope.containsKey( importRow.getEnv()) ) { pio = studyByNameNoScope.get( importRow.getEnv() ) ; @@ -411,7 +414,7 @@ private PendingImportObject createStudyPIO(Program program, boolean PendingImportObject trialPIO = this.trialByNameNoScope.get(importRow.getExpTitle()); UUID trialID = trialPIO.getId(); UUID id = UUID.randomUUID(); - BrAPIStudy newStudy = importRow.constructBrAPIStudy(program, commit, BRAPI_REFERENCE_SOURCE, expSeqenceValue, trialID, id); + BrAPIStudy newStudy = importRow.constructBrAPIStudy(program, commit, BRAPI_REFERENCE_SOURCE, expSequenceValue, trialID, id); if( commit) { String seasonID = this.yearsToSeasonDbId(newStudy.getSeasons(), program.getId()); @@ -435,13 +438,17 @@ private PendingImportObject createLocationPIO(ExperimentObservati return pio; } - private PendingImportObject createTrialPIO(Program program, boolean commit, ExperimentObservation importRow, String expSeqValue) { + private PendingImportObject createTrialPIO(Program program, boolean commit, ExperimentObservation importRow, Supplier expNextVal) { PendingImportObject pio = null; if( trialByNameNoScope.containsKey( importRow.getExpTitle()) ) { pio = trialByNameNoScope.get( importRow.getExpTitle() ) ; } else { UUID id = UUID.randomUUID(); + String expSeqValue = null; + if(commit){ + expSeqValue = expNextVal.get().toString(); + } BrAPITrial newTrial = importRow.constructBrAPITrial(program, commit, BRAPI_REFERENCE_SOURCE, id, expSeqValue); pio = new PendingImportObject<>(ImportObjectState.NEW, newTrial, id); } From ed57611e6d2e210e88458a4b73ad176a42a7a444 Mon Sep 17 00:00:00 2001 From: David Randolph Phillips Date: Tue, 20 Sep 2022 11:48:17 -0400 Subject: [PATCH 3/9] [BI-1594] append environment sequence number to Exp Unit ID --- .../constants/BrAPIAdditionalInfoFields.java | 1 + .../ExperimentObservation.java | 15 +++++--- .../processors/ExperimentProcessor.java | 27 ++++++++++----- .../org/breedinginsight/model/Program.java | 1 + .../services/ProgramService.java | 7 ++++ .../V1.0.5__create_environment_sequence.sql | 34 +++++++++++++++++++ 6 files changed, 72 insertions(+), 13 deletions(-) create mode 100644 src/main/resources/db/migration/V1.0.5__create_environment_sequence.sql diff --git a/src/main/java/org/breedinginsight/brapi/v2/constants/BrAPIAdditionalInfoFields.java b/src/main/java/org/breedinginsight/brapi/v2/constants/BrAPIAdditionalInfoFields.java index 3a51d8ff8..3968eb7e0 100644 --- a/src/main/java/org/breedinginsight/brapi/v2/constants/BrAPIAdditionalInfoFields.java +++ b/src/main/java/org/breedinginsight/brapi/v2/constants/BrAPIAdditionalInfoFields.java @@ -36,4 +36,5 @@ public final class BrAPIAdditionalInfoFields { public static final String OBSERVATION_LEVEL = "observationLevel"; public static final String EXPERIMENT_TYPE = "experimentType"; public static final String EXPERIMENT_NUMBER = "experimentNumber"; + public static final String ENVIRONMENT_NUMBER = "environmentNumber"; } diff --git a/src/main/java/org/breedinginsight/brapps/importer/model/imports/experimentObservation/ExperimentObservation.java b/src/main/java/org/breedinginsight/brapps/importer/model/imports/experimentObservation/ExperimentObservation.java index 0e87e9012..a40819208 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/model/imports/experimentObservation/ExperimentObservation.java +++ b/src/main/java/org/breedinginsight/brapps/importer/model/imports/experimentObservation/ExperimentObservation.java @@ -146,7 +146,8 @@ public BrAPIStudy constructBrAPIStudy( String referenceSource, String expSequenceValue, UUID trialId, - UUID id) { + UUID id, + Supplier envNextVal) { BrAPIStudy study = new BrAPIStudy(); if ( commit ){ study.setStudyName(Utilities.appendProgramKey(getEnv(), program.getKey(), expSequenceValue)); @@ -173,13 +174,17 @@ public BrAPIStudy constructBrAPIStudy( design.setPUI(designType); design.setDescription(designType); study.setExperimentalDesign(design); - + String envSequenceValue = null; + if( commit ){ + envSequenceValue = envNextVal.get().toString(); + } + study.putAdditionalInfoItem( BrAPIAdditionalInfoFields.ENVIRONMENT_NUMBER, envSequenceValue); return study; } public BrAPIObservationUnit constructBrAPIObservationUnit( Program program, - String nextObjsUnitSeqVal, + String seqVal, boolean commit, String germplasmName, String referenceSource, @@ -190,7 +195,7 @@ public BrAPIObservationUnit constructBrAPIObservationUnit( BrAPIObservationUnit observationUnit = new BrAPIObservationUnit(); if( commit){ - observationUnit.setObservationUnitName( Utilities.appendProgramKey(getExpUnitId(), program.getKey(), nextObjsUnitSeqVal) ); + observationUnit.setObservationUnitName( Utilities.appendProgramKey(getExpUnitId(), program.getKey(), seqVal) ); // Set external reference observationUnit.setExternalReferences(getObsUnitExternalReferences(program, referenceSource, trialID, studyID, id)); @@ -208,7 +213,7 @@ public BrAPIObservationUnit constructBrAPIObservationUnit( BrAPIObservationUnitPosition position = new BrAPIObservationUnitPosition(); BrAPIObservationUnitLevelRelationship level = new BrAPIObservationUnitLevelRelationship(); level.setLevelName("plot"); //BreedBase only accepts "plot" or "plant" - level.setLevelCode( Utilities.appendProgramKey(getExpUnitId(), program.getKey(), nextObjsUnitSeqVal) ); + level.setLevelCode( Utilities.appendProgramKey(getExpUnitId(), program.getKey(), seqVal) ); position.setObservationLevel(level); observationUnit.putAdditionalInfoItem(BrAPIAdditionalInfoFields.OBSERVATION_LEVEL, getExpUnit()); diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/ExperimentProcessor.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/ExperimentProcessor.java index a57817f86..89da02386 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/ExperimentProcessor.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/ExperimentProcessor.java @@ -202,11 +202,18 @@ private void getNewBrapiData(List importRows, Program program, bool String expSequenceName = program.getObsUnitSequence(); if (expSequenceName == null) { - log.error(String.format("Program, %s, is missing a value in the evp sequence column.", program.getName())); + log.error(String.format("Program, %s, is missing a value in the exp sequence column.", program.getName())); throw new HttpStatusException(HttpStatus.UNPROCESSABLE_ENTITY, "Program is not properly configured for observation unit import"); } Supplier expNextVal = () -> dsl.nextval(expSequenceName.toLowerCase()); + String envSequenceName = program.getEnvSequence(); + if (envSequenceName == null) { + log.error(String.format("Program, %s, is missing a value in the env sequence column.", program.getName())); + throw new HttpStatusException(HttpStatus.UNPROCESSABLE_ENTITY, "Program is not properly configured for environment import"); + } + Supplier envNextVal = () -> dsl.nextval(envSequenceName.toLowerCase()); + for (BrAPIImport row : importRows) { ExperimentObservation importRow = (ExperimentObservation) row; @@ -221,10 +228,15 @@ private void getNewBrapiData(List importRows, Program program, bool PendingImportObject locationPIO = createLocationPIO(importRow); this.locationByName.put(importRow.getEnvLocation(), locationPIO); - PendingImportObject studyPIO = createStudyPIO(program, commit, expSeqValue, importRow); + PendingImportObject studyPIO = createStudyPIO(program, commit, expSeqValue, importRow, envNextVal); this.studyByNameNoScope.put(importRow.getEnv(), studyPIO); - PendingImportObject obsUnitPIO = createObsUnitPIO(program, commit, obsUnitNextVal, importRow); + String envSeqValue = null; + if(commit) { + envSeqValue = studyPIO.getBrAPIObject().getAdditionalInfo().get(BrAPIAdditionalInfoFields.ENVIRONMENT_NUMBER).getAsString(); + } + + PendingImportObject obsUnitPIO = createObsUnitPIO(program, commit, envSeqValue, importRow); String key = createObservationUnitKey(importRow); this.observationUnitByNameNoScope.put(key, obsUnitPIO); } @@ -383,7 +395,7 @@ private PendingImportObject getGidPOI(ExperimentObservation impo } } - private PendingImportObject createObsUnitPIO(Program program, boolean commit, Supplier obsUnitNextVal, ExperimentObservation importRow) { + private PendingImportObject createObsUnitPIO(Program program, boolean commit, String seqValue, ExperimentObservation importRow) { PendingImportObject pio = null; if( this.observationUnitByNameNoScope.containsKey( createObservationUnitKey( importRow ) ) ) { pio = observationUnitByNameNoScope.get( createObservationUnitKey( importRow ) ) ; @@ -398,14 +410,13 @@ private PendingImportObject createObsUnitPIO(Program progr PendingImportObject studyPIO = this.studyByNameNoScope.get(importRow.getEnv()); UUID studyID = studyPIO.getId(); UUID id = UUID.randomUUID(); - String obsUnitSeqValue = obsUnitNextVal.get().toString(); - BrAPIObservationUnit newObservationUnit = importRow.constructBrAPIObservationUnit(program, obsUnitSeqValue, commit, germplasmName, BRAPI_REFERENCE_SOURCE, trialID, studyID, id); + BrAPIObservationUnit newObservationUnit = importRow.constructBrAPIObservationUnit(program, seqValue, commit, germplasmName, BRAPI_REFERENCE_SOURCE, trialID, studyID, id); pio = new PendingImportObject<>(ImportObjectState.NEW, newObservationUnit); } return pio; } - private PendingImportObject createStudyPIO(Program program, boolean commit, String expSequenceValue, ExperimentObservation importRow) { + private PendingImportObject createStudyPIO(Program program, boolean commit, String expSequenceValue, ExperimentObservation importRow, Supplier envNextVal) { PendingImportObject pio = null; if( studyByNameNoScope.containsKey( importRow.getEnv()) ) { pio = studyByNameNoScope.get( importRow.getEnv() ) ; @@ -414,7 +425,7 @@ private PendingImportObject createStudyPIO(Program program, boolean PendingImportObject trialPIO = this.trialByNameNoScope.get(importRow.getExpTitle()); UUID trialID = trialPIO.getId(); UUID id = UUID.randomUUID(); - BrAPIStudy newStudy = importRow.constructBrAPIStudy(program, commit, BRAPI_REFERENCE_SOURCE, expSequenceValue, trialID, id); + BrAPIStudy newStudy = importRow.constructBrAPIStudy(program, commit, BRAPI_REFERENCE_SOURCE, expSequenceValue, trialID, id, envNextVal); if( commit) { String seasonID = this.yearsToSeasonDbId(newStudy.getSeasons(), program.getId()); diff --git a/src/main/java/org/breedinginsight/model/Program.java b/src/main/java/org/breedinginsight/model/Program.java index 9ffbb6d60..07e657f96 100644 --- a/src/main/java/org/breedinginsight/model/Program.java +++ b/src/main/java/org/breedinginsight/model/Program.java @@ -92,6 +92,7 @@ public static Program parseSQLRecord(Record record, ProgramTable programTable) { .germplasmSequence(record.getValue(programTable.GERMPLASM_SEQUENCE)) .expSequence( record.getValue(programTable.EXP_SEQUENCE)) .obsUnitSequence(( record.getValue(programTable.OBS_UNIT_SEQUENCE))) + .envSequence(( record.getValue(programTable.ENV_SEQUENCE))) .build(); return program; diff --git a/src/main/java/org/breedinginsight/services/ProgramService.java b/src/main/java/org/breedinginsight/services/ProgramService.java index 70f218049..43d61c2bb 100644 --- a/src/main/java/org/breedinginsight/services/ProgramService.java +++ b/src/main/java/org/breedinginsight/services/ProgramService.java @@ -61,6 +61,8 @@ public class ProgramService { private static final String GERMPLASM_SEQUENCE_TEMPLATE = "%s_germplasm_sequence"; private static final String OBS_UNIT_SEQUENCE_TEMPLATE = "%s_obs_unit_sequence"; private static final String EXP_SEQUENCE_TEMPLATE = "%s_exp_sequence"; + private static final String ENV_SEQUENCE_TEMPLATE = "%s_env_sequence"; + @Inject public ProgramService(ProgramDAO dao, ProgramOntologyDAO programOntologyDAO, ProgramObservationLevelDAO programObservationLevelDAO, @@ -171,6 +173,10 @@ public Program create(ProgramRequest programRequest, AuthenticatedUser actingUse String obs_unit_sequence_name = String.format(OBS_UNIT_SEQUENCE_TEMPLATE, programRequest.getKey()).toLowerCase(); dsl.createSequence(obs_unit_sequence_name).execute(); + // Create env sequence + String env_sequence_name = String.format(ENV_SEQUENCE_TEMPLATE, programRequest.getKey()).toLowerCase(); + dsl.createSequence(env_sequence_name).execute(); + // Parse and create the program object ProgramEntity programEntity = ProgramEntity.builder() .name(programRequest.getName()) @@ -182,6 +188,7 @@ public Program create(ProgramRequest programRequest, AuthenticatedUser actingUse .key(programRequest.getKey()) .germplasmSequence(germplasm_sequence_name) .expSequence( exp_sequence_name ) + .envSequence( env_sequence_name ) .obsUnitSequence( obs_unit_sequence_name ) .createdBy(actingUser.getId()) .updatedBy(actingUser.getId()) diff --git a/src/main/resources/db/migration/V1.0.5__create_environment_sequence.sql b/src/main/resources/db/migration/V1.0.5__create_environment_sequence.sql new file mode 100644 index 000000000..1f6beb640 --- /dev/null +++ b/src/main/resources/db/migration/V1.0.5__create_environment_sequence.sql @@ -0,0 +1,34 @@ +/* + * See the NOTICE file distributed with this work for additional information + * regarding copyright ownership. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +alter table program add column env_sequence text; + +do +$$ +declare +f record; +begin +for f in select * from program +loop + if f.key is NULL then + RAISE EXCEPTION 'Programs must have a key associated with them'; + end if; + execute format('create sequence %s_env_sequence',f.key); + update program set env_sequence = format('%s_env_sequence', f.key) where id = f.id; +end loop; +end; +$$ \ No newline at end of file From 203082c9d5b9288b86e8f93f91c1096464c081e3 Mon Sep 17 00:00:00 2001 From: David Randolph Phillips Date: Wed, 21 Sep 2022 12:14:52 -0400 Subject: [PATCH 4/9] [BI-1594] defensive code --- .../importer/services/processors/ExperimentProcessor.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/ExperimentProcessor.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/ExperimentProcessor.java index 89da02386..f04af15f5 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/ExperimentProcessor.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/ExperimentProcessor.java @@ -564,7 +564,7 @@ private void updateStudyDependencyValues(Map mappedBrAPI private void updateStudyLocationDbId(BrAPILocation location) { this.studyByNameNoScope.values().stream() - .filter(study -> study.getBrAPIObject().getLocationName().equals(location.getLocationName())) + .filter(study -> location.getLocationName().equals( study.getBrAPIObject().getLocationName() )) .forEach(study -> study.getBrAPIObject().setLocationDbId(location.getLocationDbId())); } From c8644183e1a2169607f2edefb6b42d20ede15667 Mon Sep 17 00:00:00 2001 From: David Randolph Phillips Date: Thu, 29 Sep 2022 14:25:24 -0400 Subject: [PATCH 5/9] [BI-1594] Removed obs_unit_squences --- .../processors/ExperimentProcessor.java | 9 +---- .../org/breedinginsight/model/Program.java | 1 - .../services/ProgramService.java | 1 - ...> V1.0.6__create_environment_sequence.sql} | 0 .../V1.0.7__delete_obs_unit_sequence.sql | 33 +++++++++++++++++++ 5 files changed, 34 insertions(+), 10 deletions(-) rename src/main/resources/db/migration/{V1.0.5__create_environment_sequence.sql => V1.0.6__create_environment_sequence.sql} (100%) create mode 100644 src/main/resources/db/migration/V1.0.7__delete_obs_unit_sequence.sql diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/ExperimentProcessor.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/ExperimentProcessor.java index f04af15f5..ae4fba05d 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/ExperimentProcessor.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/ExperimentProcessor.java @@ -193,14 +193,7 @@ public Map process( private void getNewBrapiData(List importRows, Program program, boolean commit) { - String obsUnitSequenceName = program.getObsUnitSequence(); - if (obsUnitSequenceName == null) { - log.error(String.format("Program, %s, is missing a value in the obsUnit sequence column.", program.getName())); - throw new HttpStatusException(HttpStatus.UNPROCESSABLE_ENTITY, "Program is not properly configured for observation unit import"); - } - Supplier obsUnitNextVal = () -> dsl.nextval(obsUnitSequenceName.toLowerCase()); - - String expSequenceName = program.getObsUnitSequence(); + String expSequenceName = program.getExpSequence(); if (expSequenceName == null) { log.error(String.format("Program, %s, is missing a value in the exp sequence column.", program.getName())); throw new HttpStatusException(HttpStatus.UNPROCESSABLE_ENTITY, "Program is not properly configured for observation unit import"); diff --git a/src/main/java/org/breedinginsight/model/Program.java b/src/main/java/org/breedinginsight/model/Program.java index 07e657f96..b2aec14cf 100644 --- a/src/main/java/org/breedinginsight/model/Program.java +++ b/src/main/java/org/breedinginsight/model/Program.java @@ -91,7 +91,6 @@ public static Program parseSQLRecord(Record record, ProgramTable programTable) { .active(record.getValue(programTable.ACTIVE)) .germplasmSequence(record.getValue(programTable.GERMPLASM_SEQUENCE)) .expSequence( record.getValue(programTable.EXP_SEQUENCE)) - .obsUnitSequence(( record.getValue(programTable.OBS_UNIT_SEQUENCE))) .envSequence(( record.getValue(programTable.ENV_SEQUENCE))) .build(); diff --git a/src/main/java/org/breedinginsight/services/ProgramService.java b/src/main/java/org/breedinginsight/services/ProgramService.java index 43d61c2bb..a1942c073 100644 --- a/src/main/java/org/breedinginsight/services/ProgramService.java +++ b/src/main/java/org/breedinginsight/services/ProgramService.java @@ -189,7 +189,6 @@ public Program create(ProgramRequest programRequest, AuthenticatedUser actingUse .germplasmSequence(germplasm_sequence_name) .expSequence( exp_sequence_name ) .envSequence( env_sequence_name ) - .obsUnitSequence( obs_unit_sequence_name ) .createdBy(actingUser.getId()) .updatedBy(actingUser.getId()) .build(); diff --git a/src/main/resources/db/migration/V1.0.5__create_environment_sequence.sql b/src/main/resources/db/migration/V1.0.6__create_environment_sequence.sql similarity index 100% rename from src/main/resources/db/migration/V1.0.5__create_environment_sequence.sql rename to src/main/resources/db/migration/V1.0.6__create_environment_sequence.sql diff --git a/src/main/resources/db/migration/V1.0.7__delete_obs_unit_sequence.sql b/src/main/resources/db/migration/V1.0.7__delete_obs_unit_sequence.sql new file mode 100644 index 000000000..af85cee96 --- /dev/null +++ b/src/main/resources/db/migration/V1.0.7__delete_obs_unit_sequence.sql @@ -0,0 +1,33 @@ +/* + * See the NOTICE file distributed with this work for additional information + * regarding copyright ownership. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +alter table program drop column obs_unit_sequence; + +do +$$ +declare +f record; +begin +for f in select * from program +loop + if f.key is NULL then + RAISE EXCEPTION 'Programs must have a key associated with them'; + end if; + execute format('drop sequence if exists %s_obs_unit_sequence',f.key); +end loop; +end; +$$ \ No newline at end of file From 28fd7527e9a9d2c5d0e33b80f7309df5781a1fd8 Mon Sep 17 00:00:00 2001 From: David Randolph Phillips Date: Thu, 29 Sep 2022 16:17:01 -0400 Subject: [PATCH 6/9] [BI-1594] updated migrations with the 'IF NOT EXISTS' parameter --- .../db/migration/V1.0.3__create_obs_unit_sequence.sql | 4 ++-- .../db/migration/V1.0.4__create_experiment_sequence.sql | 4 ++-- .../db/migration/V1.0.6__create_environment_sequence.sql | 4 ++-- 3 files changed, 6 insertions(+), 6 deletions(-) diff --git a/src/main/resources/db/migration/V1.0.3__create_obs_unit_sequence.sql b/src/main/resources/db/migration/V1.0.3__create_obs_unit_sequence.sql index f0bed3308..0981ec7a6 100644 --- a/src/main/resources/db/migration/V1.0.3__create_obs_unit_sequence.sql +++ b/src/main/resources/db/migration/V1.0.3__create_obs_unit_sequence.sql @@ -15,7 +15,7 @@ * limitations under the License. */ -alter table program add column obs_unit_sequence text; +alter table program add column IF NOT EXISTS obs_unit_sequence text; do $$ @@ -27,7 +27,7 @@ loop if f.key is NULL then RAISE EXCEPTION 'Programs must have a key associated with them'; end if; - execute format('create sequence %s_obs_unit_sequence',f.key); + execute format('create sequence IF NOT EXISTS %s_obs_unit_sequence',f.key); update program set obs_unit_sequence = format('%s_obs_unit_sequence', f.key) where id = f.id; end loop; end; diff --git a/src/main/resources/db/migration/V1.0.4__create_experiment_sequence.sql b/src/main/resources/db/migration/V1.0.4__create_experiment_sequence.sql index 1d2b97e83..c5ce9461d 100644 --- a/src/main/resources/db/migration/V1.0.4__create_experiment_sequence.sql +++ b/src/main/resources/db/migration/V1.0.4__create_experiment_sequence.sql @@ -15,7 +15,7 @@ * limitations under the License. */ -alter table program add column exp_sequence text; +alter table program add column IF NOT EXISTS exp_sequence text; do $$ @@ -27,7 +27,7 @@ loop if f.key is NULL then RAISE EXCEPTION 'Programs must have a key associated with them'; end if; - execute format('create sequence %s_exp_sequence',f.key); + execute format('create sequence IF NOT EXISTS %s_exp_sequence',f.key); update program set exp_sequence = format('%s_exp_sequence', f.key) where id = f.id; end loop; end; diff --git a/src/main/resources/db/migration/V1.0.6__create_environment_sequence.sql b/src/main/resources/db/migration/V1.0.6__create_environment_sequence.sql index 1f6beb640..1bc60dfc7 100644 --- a/src/main/resources/db/migration/V1.0.6__create_environment_sequence.sql +++ b/src/main/resources/db/migration/V1.0.6__create_environment_sequence.sql @@ -15,7 +15,7 @@ * limitations under the License. */ -alter table program add column env_sequence text; +alter table program add column IF NOT EXISTS env_sequence text; do $$ @@ -27,7 +27,7 @@ loop if f.key is NULL then RAISE EXCEPTION 'Programs must have a key associated with them'; end if; - execute format('create sequence %s_env_sequence',f.key); + execute format('create sequence IF NOT EXISTS %s_env_sequence',f.key); update program set env_sequence = format('%s_env_sequence', f.key) where id = f.id; end loop; end; From ee8f5c80c309a5ec8871935c57ca05f8b2c0393a Mon Sep 17 00:00:00 2001 From: David Randolph Phillips Date: Wed, 5 Oct 2022 12:44:16 -0400 Subject: [PATCH 7/9] [BI-154] fixed bug - could not save a new study with a new location (added locationDbId to new study) --- .../services/processors/ExperimentProcessor.java | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/ExperimentProcessor.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/ExperimentProcessor.java index ae4fba05d..a22890f19 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/ExperimentProcessor.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/ExperimentProcessor.java @@ -474,7 +474,6 @@ public void postBrapiData(Map mappedBrAPIImport, Program try { List createdTrials = new ArrayList<>(brapiTrialDAO.createBrAPITrial(newTrials, program.getId(), upload)); - // set the DbId to the for each newly created trial for ( BrAPITrial createdTrial: createdTrials ) { String createdTrialName = createdTrial.getTrialName(); @@ -484,7 +483,16 @@ public void postBrapiData(Map mappedBrAPIImport, Program String dbid = createdTrial.getTrialDbId(); listedTrial.setTrialDbId( dbid ); } - brAPILocationDAO.createBrAPILocation(newLocations, program.getId(), upload); + + List createdLocations = new ArrayList<>(brAPILocationDAO.createBrAPILocation(newLocations, program.getId(), upload)); + // set the DbId to the for each newly created trial + for ( BrAPILocation createdLocation : createdLocations){ + String createdLocationName = createdLocation.getLocationName(); + PendingImportObject pi = this.locationByName.get(createdLocationName); + BrAPILocation listedLocation = pi.getBrAPIObject(); + String dbid = createdLocation.getLocationDbId(); + listedLocation.setLocationDbId(dbid); + } updateStudyDependencyValues(mappedBrAPIImport,program.getKey()); List createdStudies = new ArrayList<>(); From f4b8b2822c0cbe3cfcb66e20ae8d996f5321b05f Mon Sep 17 00:00:00 2001 From: David Randolph Phillips Date: Fri, 7 Oct 2022 13:51:27 -0400 Subject: [PATCH 8/9] [BI-1594] restore V1.0.3__create_obs_unit_sequence.sql --- .../db/migration/V1.0.3__create_obs_unit_sequence.sql | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/main/resources/db/migration/V1.0.3__create_obs_unit_sequence.sql b/src/main/resources/db/migration/V1.0.3__create_obs_unit_sequence.sql index 0981ec7a6..f0bed3308 100644 --- a/src/main/resources/db/migration/V1.0.3__create_obs_unit_sequence.sql +++ b/src/main/resources/db/migration/V1.0.3__create_obs_unit_sequence.sql @@ -15,7 +15,7 @@ * limitations under the License. */ -alter table program add column IF NOT EXISTS obs_unit_sequence text; +alter table program add column obs_unit_sequence text; do $$ @@ -27,7 +27,7 @@ loop if f.key is NULL then RAISE EXCEPTION 'Programs must have a key associated with them'; end if; - execute format('create sequence IF NOT EXISTS %s_obs_unit_sequence',f.key); + execute format('create sequence %s_obs_unit_sequence',f.key); update program set obs_unit_sequence = format('%s_obs_unit_sequence', f.key) where id = f.id; end loop; end; From b017a5d4fbe1bbf47b5376dd2475c3f4401257a1 Mon Sep 17 00:00:00 2001 From: David Randolph Phillips Date: Fri, 7 Oct 2022 14:04:56 -0400 Subject: [PATCH 9/9] [BI-1594] Only add the envSequenceValue to the study if commiting. Improve ProgramCahsTest.getMethodDoesNotWaitForRefresh() --- .../imports/experimentObservation/ExperimentObservation.java | 2 +- .../org/breedinginsight/brapi/v2/ProgramCacheUnitTest.java | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/src/main/java/org/breedinginsight/brapps/importer/model/imports/experimentObservation/ExperimentObservation.java b/src/main/java/org/breedinginsight/brapps/importer/model/imports/experimentObservation/ExperimentObservation.java index a40819208..468782d47 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/model/imports/experimentObservation/ExperimentObservation.java +++ b/src/main/java/org/breedinginsight/brapps/importer/model/imports/experimentObservation/ExperimentObservation.java @@ -177,8 +177,8 @@ public BrAPIStudy constructBrAPIStudy( String envSequenceValue = null; if( commit ){ envSequenceValue = envNextVal.get().toString(); + study.putAdditionalInfoItem( BrAPIAdditionalInfoFields.ENVIRONMENT_NUMBER, envSequenceValue); } - study.putAdditionalInfoItem( BrAPIAdditionalInfoFields.ENVIRONMENT_NUMBER, envSequenceValue); return study; } diff --git a/src/test/java/org/breedinginsight/brapi/v2/ProgramCacheUnitTest.java b/src/test/java/org/breedinginsight/brapi/v2/ProgramCacheUnitTest.java index 090ddafce..cb72cca48 100644 --- a/src/test/java/org/breedinginsight/brapi/v2/ProgramCacheUnitTest.java +++ b/src/test/java/org/breedinginsight/brapi/v2/ProgramCacheUnitTest.java @@ -139,7 +139,7 @@ public void getMethodDoesNotWaitForRefresh() { List newList = new ArrayList<>(); newList.add(new BrAPIGermplasm()); mockBrAPI.put(programId, new ArrayList<>(newList)); - ProgramCache cache = new ProgramCache<>((UUID id) -> mockFetch(id, waitTime), List.of(programId)); + ProgramCache cache = new ProgramCache<>((UUID id) -> mockFetch(id, waitTime*2), List.of(programId)); Callable> postFunction = () -> mockPost(programId, new ArrayList<>(newList)); // Get waits for initial fetch @@ -152,7 +152,7 @@ public void getMethodDoesNotWaitForRefresh() { assertEquals(1, cachedGermplasm.size(), "Get method seemed to have waited for refresh method"); // Now wait for the fetch after the post to finish - Thread.sleep(waitTime*2); + Thread.sleep(waitTime*3); cachedGermplasm = cache.get(programId); assertEquals(2, cachedGermplasm.size(), "Get method did not get updated germplasm"); }