diff --git a/src/main/java/org/breedinginsight/brapi/v2/constants/BrAPIAdditionalInfoFields.java b/src/main/java/org/breedinginsight/brapi/v2/constants/BrAPIAdditionalInfoFields.java index 3a51d8ff8..3968eb7e0 100644 --- a/src/main/java/org/breedinginsight/brapi/v2/constants/BrAPIAdditionalInfoFields.java +++ b/src/main/java/org/breedinginsight/brapi/v2/constants/BrAPIAdditionalInfoFields.java @@ -36,4 +36,5 @@ public final class BrAPIAdditionalInfoFields { public static final String OBSERVATION_LEVEL = "observationLevel"; public static final String EXPERIMENT_TYPE = "experimentType"; public static final String EXPERIMENT_NUMBER = "experimentNumber"; + public static final String ENVIRONMENT_NUMBER = "environmentNumber"; } diff --git a/src/main/java/org/breedinginsight/brapps/importer/model/imports/experimentObservation/ExperimentObservation.java b/src/main/java/org/breedinginsight/brapps/importer/model/imports/experimentObservation/ExperimentObservation.java index cb64a5a9a..468782d47 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/model/imports/experimentObservation/ExperimentObservation.java +++ b/src/main/java/org/breedinginsight/brapps/importer/model/imports/experimentObservation/ExperimentObservation.java @@ -146,7 +146,8 @@ public BrAPIStudy constructBrAPIStudy( String referenceSource, String expSequenceValue, UUID trialId, - UUID id) { + UUID id, + Supplier envNextVal) { BrAPIStudy study = new BrAPIStudy(); if ( commit ){ study.setStudyName(Utilities.appendProgramKey(getEnv(), program.getKey(), expSequenceValue)); @@ -173,13 +174,17 @@ public BrAPIStudy constructBrAPIStudy( design.setPUI(designType); design.setDescription(designType); study.setExperimentalDesign(design); - + String envSequenceValue = null; + if( commit ){ + envSequenceValue = envNextVal.get().toString(); + study.putAdditionalInfoItem( BrAPIAdditionalInfoFields.ENVIRONMENT_NUMBER, envSequenceValue); + } return study; } public BrAPIObservationUnit constructBrAPIObservationUnit( Program program, - Supplier nextVal, + String seqVal, boolean commit, String germplasmName, String referenceSource, @@ -190,7 +195,7 @@ public BrAPIObservationUnit constructBrAPIObservationUnit( BrAPIObservationUnit observationUnit = new BrAPIObservationUnit(); if( commit){ - observationUnit.setObservationUnitName( Utilities.appendProgramKey(getExpUnitId(), program.getKey(), nextVal.get().toString())); + observationUnit.setObservationUnitName( Utilities.appendProgramKey(getExpUnitId(), program.getKey(), seqVal) ); // Set external reference observationUnit.setExternalReferences(getObsUnitExternalReferences(program, referenceSource, trialID, studyID, id)); @@ -208,7 +213,7 @@ public BrAPIObservationUnit constructBrAPIObservationUnit( BrAPIObservationUnitPosition position = new BrAPIObservationUnitPosition(); BrAPIObservationUnitLevelRelationship level = new BrAPIObservationUnitLevelRelationship(); level.setLevelName("plot"); //BreedBase only accepts "plot" or "plant" - level.setLevelCode( getExpUnitId() ); + level.setLevelCode( Utilities.appendProgramKey(getExpUnitId(), program.getKey(), seqVal) ); position.setObservationLevel(level); observationUnit.putAdditionalInfoItem(BrAPIAdditionalInfoFields.OBSERVATION_LEVEL, getExpUnit()); diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/ExperimentProcessor.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/ExperimentProcessor.java index 7718bfa8a..a22890f19 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/ExperimentProcessor.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/ExperimentProcessor.java @@ -33,6 +33,7 @@ import org.brapi.v2.model.pheno.*; import org.breedinginsight.api.model.v1.response.ValidationError; import org.breedinginsight.api.model.v1.response.ValidationErrors; +import org.breedinginsight.brapi.v2.constants.BrAPIAdditionalInfoFields; import org.breedinginsight.brapi.v2.dao.BrAPIGermplasmDAO; import org.breedinginsight.brapps.importer.daos.*; import org.breedinginsight.brapps.importer.model.ImportUpload; @@ -192,37 +193,43 @@ public Map process( private void getNewBrapiData(List importRows, Program program, boolean commit) { - String obsUnitSequenceName = program.getObsUnitSequence(); - if (obsUnitSequenceName == null) { - log.error(String.format("Program, %s, is missing a value in the obsUnit sequence column.", program.getName())); + String expSequenceName = program.getExpSequence(); + if (expSequenceName == null) { + log.error(String.format("Program, %s, is missing a value in the exp sequence column.", program.getName())); throw new HttpStatusException(HttpStatus.UNPROCESSABLE_ENTITY, "Program is not properly configured for observation unit import"); } - Supplier obsUnitNextVal = () -> dsl.nextval(obsUnitSequenceName.toLowerCase()); + Supplier expNextVal = () -> dsl.nextval(expSequenceName.toLowerCase()); - // It is assumed, at this point, that there is only one experiment per import-file - String expSeqValue = null; - if (commit) { - String expUnitSequenceName = program.getExpSequence(); - if (expUnitSequenceName == null) { - log.error(String.format("Program, %s, is missing a value in the exp sequence column.", program.getName())); - throw new HttpStatusException(HttpStatus.UNPROCESSABLE_ENTITY, "Program is not properly configured for experiment import"); - } - expSeqValue = dsl.nextval(expUnitSequenceName.toLowerCase()).toString(); + String envSequenceName = program.getEnvSequence(); + if (envSequenceName == null) { + log.error(String.format("Program, %s, is missing a value in the env sequence column.", program.getName())); + throw new HttpStatusException(HttpStatus.UNPROCESSABLE_ENTITY, "Program is not properly configured for environment import"); } + Supplier envNextVal = () -> dsl.nextval(envSequenceName.toLowerCase()); for (BrAPIImport row : importRows) { ExperimentObservation importRow = (ExperimentObservation) row; - PendingImportObject trialPIO = createTrialPIO(program, commit, importRow, expSeqValue); + PendingImportObject trialPIO = createTrialPIO(program, commit, importRow, expNextVal); this.trialByNameNoScope.put(importRow.getExpTitle(), trialPIO); + String expSeqValue = null; + if(commit) { + expSeqValue = trialPIO.getBrAPIObject().getAdditionalInfo().get(BrAPIAdditionalInfoFields.EXPERIMENT_NUMBER).getAsString(); + } + PendingImportObject locationPIO = createLocationPIO(importRow); this.locationByName.put(importRow.getEnvLocation(), locationPIO); - PendingImportObject studyPIO = createStudyPIO(program, commit, expSeqValue, importRow); + PendingImportObject studyPIO = createStudyPIO(program, commit, expSeqValue, importRow, envNextVal); this.studyByNameNoScope.put(importRow.getEnv(), studyPIO); - PendingImportObject obsUnitPIO = createObsUnitPIO(program, commit, obsUnitNextVal, importRow); + String envSeqValue = null; + if(commit) { + envSeqValue = studyPIO.getBrAPIObject().getAdditionalInfo().get(BrAPIAdditionalInfoFields.ENVIRONMENT_NUMBER).getAsString(); + } + + PendingImportObject obsUnitPIO = createObsUnitPIO(program, commit, envSeqValue, importRow); String key = createObservationUnitKey(importRow); this.observationUnitByNameNoScope.put(key, obsUnitPIO); } @@ -381,7 +388,7 @@ private PendingImportObject getGidPOI(ExperimentObservation impo } } - private PendingImportObject createObsUnitPIO(Program program, boolean commit, Supplier obsUnitNextVal, ExperimentObservation importRow) { + private PendingImportObject createObsUnitPIO(Program program, boolean commit, String seqValue, ExperimentObservation importRow) { PendingImportObject pio = null; if( this.observationUnitByNameNoScope.containsKey( createObservationUnitKey( importRow ) ) ) { pio = observationUnitByNameNoScope.get( createObservationUnitKey( importRow ) ) ; @@ -396,13 +403,13 @@ private PendingImportObject createObsUnitPIO(Program progr PendingImportObject studyPIO = this.studyByNameNoScope.get(importRow.getEnv()); UUID studyID = studyPIO.getId(); UUID id = UUID.randomUUID(); - BrAPIObservationUnit newObservationUnit = importRow.constructBrAPIObservationUnit(program, obsUnitNextVal, commit, germplasmName, BRAPI_REFERENCE_SOURCE, trialID, studyID, id); + BrAPIObservationUnit newObservationUnit = importRow.constructBrAPIObservationUnit(program, seqValue, commit, germplasmName, BRAPI_REFERENCE_SOURCE, trialID, studyID, id); pio = new PendingImportObject<>(ImportObjectState.NEW, newObservationUnit); } return pio; } - private PendingImportObject createStudyPIO(Program program, boolean commit, String expSeqenceValue, ExperimentObservation importRow) { + private PendingImportObject createStudyPIO(Program program, boolean commit, String expSequenceValue, ExperimentObservation importRow, Supplier envNextVal) { PendingImportObject pio = null; if( studyByNameNoScope.containsKey( importRow.getEnv()) ) { pio = studyByNameNoScope.get( importRow.getEnv() ) ; @@ -411,7 +418,7 @@ private PendingImportObject createStudyPIO(Program program, boolean PendingImportObject trialPIO = this.trialByNameNoScope.get(importRow.getExpTitle()); UUID trialID = trialPIO.getId(); UUID id = UUID.randomUUID(); - BrAPIStudy newStudy = importRow.constructBrAPIStudy(program, commit, BRAPI_REFERENCE_SOURCE, expSeqenceValue, trialID, id); + BrAPIStudy newStudy = importRow.constructBrAPIStudy(program, commit, BRAPI_REFERENCE_SOURCE, expSequenceValue, trialID, id, envNextVal); if( commit) { String seasonID = this.yearsToSeasonDbId(newStudy.getSeasons(), program.getId()); @@ -435,13 +442,17 @@ private PendingImportObject createLocationPIO(ExperimentObservati return pio; } - private PendingImportObject createTrialPIO(Program program, boolean commit, ExperimentObservation importRow, String expSeqValue) { + private PendingImportObject createTrialPIO(Program program, boolean commit, ExperimentObservation importRow, Supplier expNextVal) { PendingImportObject pio = null; if( trialByNameNoScope.containsKey( importRow.getExpTitle()) ) { pio = trialByNameNoScope.get( importRow.getExpTitle() ) ; } else { UUID id = UUID.randomUUID(); + String expSeqValue = null; + if(commit){ + expSeqValue = expNextVal.get().toString(); + } BrAPITrial newTrial = importRow.constructBrAPITrial(program, commit, BRAPI_REFERENCE_SOURCE, id, expSeqValue); pio = new PendingImportObject<>(ImportObjectState.NEW, newTrial, id); } @@ -463,7 +474,6 @@ public void postBrapiData(Map mappedBrAPIImport, Program try { List createdTrials = new ArrayList<>(brapiTrialDAO.createBrAPITrial(newTrials, program.getId(), upload)); - // set the DbId to the for each newly created trial for ( BrAPITrial createdTrial: createdTrials ) { String createdTrialName = createdTrial.getTrialName(); @@ -473,7 +483,16 @@ public void postBrapiData(Map mappedBrAPIImport, Program String dbid = createdTrial.getTrialDbId(); listedTrial.setTrialDbId( dbid ); } - brAPILocationDAO.createBrAPILocation(newLocations, program.getId(), upload); + + List createdLocations = new ArrayList<>(brAPILocationDAO.createBrAPILocation(newLocations, program.getId(), upload)); + // set the DbId to the for each newly created trial + for ( BrAPILocation createdLocation : createdLocations){ + String createdLocationName = createdLocation.getLocationName(); + PendingImportObject pi = this.locationByName.get(createdLocationName); + BrAPILocation listedLocation = pi.getBrAPIObject(); + String dbid = createdLocation.getLocationDbId(); + listedLocation.setLocationDbId(dbid); + } updateStudyDependencyValues(mappedBrAPIImport,program.getKey()); List createdStudies = new ArrayList<>(); @@ -546,7 +565,7 @@ private void updateStudyDependencyValues(Map mappedBrAPI private void updateStudyLocationDbId(BrAPILocation location) { this.studyByNameNoScope.values().stream() - .filter(study -> study.getBrAPIObject().getLocationName().equals(location.getLocationName())) + .filter(study -> location.getLocationName().equals( study.getBrAPIObject().getLocationName() )) .forEach(study -> study.getBrAPIObject().setLocationDbId(location.getLocationDbId())); } @@ -606,20 +625,12 @@ private Map> initialize_studyByNameNoSco ExperimentObservation experimentObservation = experimentImportRows.get(0); PendingImportObject trial = this.trialByNameNoScope.get(experimentObservation.getExpTitle()); - List experimentRefs = trial.getBrAPIObject().getExternalReferences(); - Optional experimentIDRef = experimentRefs.stream() - .filter(this::isTrialRefSource) - .findFirst(); - if( experimentIDRef.isEmpty()){ - throw new InternalServerException("An Experiment ID was not found any of the external references"); - } - //get experimentID - String experimentIDStr = experimentIDRef.get().getReferenceID(); + UUID experimentId = trial.getId(); List existingStudies; try { - existingStudies = brAPIStudyDAO.getStudiesByExperimentID(UUID.fromString(experimentIDStr), program); + existingStudies = brAPIStudyDAO.getStudiesByExperimentID(experimentId, program); existingStudies.forEach(existingStudy -> { //Swap season DbId with year String String seasonId = existingStudy.getSeasons().get(0); @@ -672,9 +683,20 @@ private Map> initialize_trialByNameNoSco existingTrials = brapiTrialDAO.getTrialByName(uniqueTrialNames, program); existingTrials.forEach(existingTrial -> { existingTrial.setTrialName(Utilities.removeProgramKey(existingTrial.getTrialName(), program.getKey())); + + //get TrialId from existingTrial + List experimentRefs = existingTrial.getExternalReferences(); + Optional experimentIDRef = experimentRefs.stream() + .filter(this::isTrialRefSource) + .findFirst(); + if( experimentIDRef.isEmpty()){ + throw new InternalServerException("An Experiment ID was not found any of the external references"); + } + UUID experimentId = UUID.fromString( experimentIDRef.get().getReferenceID() ); + trialByNameNoScope.put( existingTrial.getTrialName(), - new PendingImportObject<>(ImportObjectState.EXISTING, existingTrial)); + new PendingImportObject<>(ImportObjectState.EXISTING, existingTrial, experimentId)); }); return trialByNameNoScope; } catch (ApiException e) { diff --git a/src/main/java/org/breedinginsight/model/Program.java b/src/main/java/org/breedinginsight/model/Program.java index 9ffbb6d60..b2aec14cf 100644 --- a/src/main/java/org/breedinginsight/model/Program.java +++ b/src/main/java/org/breedinginsight/model/Program.java @@ -91,7 +91,7 @@ public static Program parseSQLRecord(Record record, ProgramTable programTable) { .active(record.getValue(programTable.ACTIVE)) .germplasmSequence(record.getValue(programTable.GERMPLASM_SEQUENCE)) .expSequence( record.getValue(programTable.EXP_SEQUENCE)) - .obsUnitSequence(( record.getValue(programTable.OBS_UNIT_SEQUENCE))) + .envSequence(( record.getValue(programTable.ENV_SEQUENCE))) .build(); return program; diff --git a/src/main/java/org/breedinginsight/services/ProgramService.java b/src/main/java/org/breedinginsight/services/ProgramService.java index 70f218049..a1942c073 100644 --- a/src/main/java/org/breedinginsight/services/ProgramService.java +++ b/src/main/java/org/breedinginsight/services/ProgramService.java @@ -61,6 +61,8 @@ public class ProgramService { private static final String GERMPLASM_SEQUENCE_TEMPLATE = "%s_germplasm_sequence"; private static final String OBS_UNIT_SEQUENCE_TEMPLATE = "%s_obs_unit_sequence"; private static final String EXP_SEQUENCE_TEMPLATE = "%s_exp_sequence"; + private static final String ENV_SEQUENCE_TEMPLATE = "%s_env_sequence"; + @Inject public ProgramService(ProgramDAO dao, ProgramOntologyDAO programOntologyDAO, ProgramObservationLevelDAO programObservationLevelDAO, @@ -171,6 +173,10 @@ public Program create(ProgramRequest programRequest, AuthenticatedUser actingUse String obs_unit_sequence_name = String.format(OBS_UNIT_SEQUENCE_TEMPLATE, programRequest.getKey()).toLowerCase(); dsl.createSequence(obs_unit_sequence_name).execute(); + // Create env sequence + String env_sequence_name = String.format(ENV_SEQUENCE_TEMPLATE, programRequest.getKey()).toLowerCase(); + dsl.createSequence(env_sequence_name).execute(); + // Parse and create the program object ProgramEntity programEntity = ProgramEntity.builder() .name(programRequest.getName()) @@ -182,7 +188,7 @@ public Program create(ProgramRequest programRequest, AuthenticatedUser actingUse .key(programRequest.getKey()) .germplasmSequence(germplasm_sequence_name) .expSequence( exp_sequence_name ) - .obsUnitSequence( obs_unit_sequence_name ) + .envSequence( env_sequence_name ) .createdBy(actingUser.getId()) .updatedBy(actingUser.getId()) .build(); diff --git a/src/main/resources/db/migration/V1.0.4__create_experiment_sequence.sql b/src/main/resources/db/migration/V1.0.4__create_experiment_sequence.sql index 1d2b97e83..c5ce9461d 100644 --- a/src/main/resources/db/migration/V1.0.4__create_experiment_sequence.sql +++ b/src/main/resources/db/migration/V1.0.4__create_experiment_sequence.sql @@ -15,7 +15,7 @@ * limitations under the License. */ -alter table program add column exp_sequence text; +alter table program add column IF NOT EXISTS exp_sequence text; do $$ @@ -27,7 +27,7 @@ loop if f.key is NULL then RAISE EXCEPTION 'Programs must have a key associated with them'; end if; - execute format('create sequence %s_exp_sequence',f.key); + execute format('create sequence IF NOT EXISTS %s_exp_sequence',f.key); update program set exp_sequence = format('%s_exp_sequence', f.key) where id = f.id; end loop; end; diff --git a/src/main/resources/db/migration/V1.0.6__create_environment_sequence.sql b/src/main/resources/db/migration/V1.0.6__create_environment_sequence.sql new file mode 100644 index 000000000..1bc60dfc7 --- /dev/null +++ b/src/main/resources/db/migration/V1.0.6__create_environment_sequence.sql @@ -0,0 +1,34 @@ +/* + * See the NOTICE file distributed with this work for additional information + * regarding copyright ownership. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +alter table program add column IF NOT EXISTS env_sequence text; + +do +$$ +declare +f record; +begin +for f in select * from program +loop + if f.key is NULL then + RAISE EXCEPTION 'Programs must have a key associated with them'; + end if; + execute format('create sequence IF NOT EXISTS %s_env_sequence',f.key); + update program set env_sequence = format('%s_env_sequence', f.key) where id = f.id; +end loop; +end; +$$ \ No newline at end of file diff --git a/src/main/resources/db/migration/V1.0.7__delete_obs_unit_sequence.sql b/src/main/resources/db/migration/V1.0.7__delete_obs_unit_sequence.sql new file mode 100644 index 000000000..af85cee96 --- /dev/null +++ b/src/main/resources/db/migration/V1.0.7__delete_obs_unit_sequence.sql @@ -0,0 +1,33 @@ +/* + * See the NOTICE file distributed with this work for additional information + * regarding copyright ownership. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +alter table program drop column obs_unit_sequence; + +do +$$ +declare +f record; +begin +for f in select * from program +loop + if f.key is NULL then + RAISE EXCEPTION 'Programs must have a key associated with them'; + end if; + execute format('drop sequence if exists %s_obs_unit_sequence',f.key); +end loop; +end; +$$ \ No newline at end of file diff --git a/src/test/java/org/breedinginsight/brapi/v2/ProgramCacheUnitTest.java b/src/test/java/org/breedinginsight/brapi/v2/ProgramCacheUnitTest.java index 090ddafce..cb72cca48 100644 --- a/src/test/java/org/breedinginsight/brapi/v2/ProgramCacheUnitTest.java +++ b/src/test/java/org/breedinginsight/brapi/v2/ProgramCacheUnitTest.java @@ -139,7 +139,7 @@ public void getMethodDoesNotWaitForRefresh() { List newList = new ArrayList<>(); newList.add(new BrAPIGermplasm()); mockBrAPI.put(programId, new ArrayList<>(newList)); - ProgramCache cache = new ProgramCache<>((UUID id) -> mockFetch(id, waitTime), List.of(programId)); + ProgramCache cache = new ProgramCache<>((UUID id) -> mockFetch(id, waitTime*2), List.of(programId)); Callable> postFunction = () -> mockPost(programId, new ArrayList<>(newList)); // Get waits for initial fetch @@ -152,7 +152,7 @@ public void getMethodDoesNotWaitForRefresh() { assertEquals(1, cachedGermplasm.size(), "Get method seemed to have waited for refresh method"); // Now wait for the fetch after the post to finish - Thread.sleep(waitTime*2); + Thread.sleep(waitTime*3); cachedGermplasm = cache.get(programId); assertEquals(2, cachedGermplasm.size(), "Get method did not get updated germplasm"); }