From cd56b58f997722deabc744098ea9d2b64aa61473 Mon Sep 17 00:00:00 2001 From: David Randolph Phillips Date: Tue, 11 Jun 2024 13:36:56 -0400 Subject: [PATCH 1/3] [BI-2093] WIP --- .../brapi/v2/dao/BrAPIObservationDAO.java | 81 ++++++++++++++++--- .../processors/ExperimentProcessor.java | 34 +++----- .../daos/cache/ProgramCache.java | 8 +- 3 files changed, 85 insertions(+), 38 deletions(-) diff --git a/src/main/java/org/breedinginsight/brapi/v2/dao/BrAPIObservationDAO.java b/src/main/java/org/breedinginsight/brapi/v2/dao/BrAPIObservationDAO.java index 92b45633c..af06352ca 100644 --- a/src/main/java/org/breedinginsight/brapi/v2/dao/BrAPIObservationDAO.java +++ b/src/main/java/org/breedinginsight/brapi/v2/dao/BrAPIObservationDAO.java @@ -250,10 +250,54 @@ public BrAPIObservation updateBrAPIObservation(String dbId, BrAPIObservation obs ObservationsApi api = brAPIEndpointProvider.get(programDAO.getCoreClient(programId), ObservationsApi.class); var program = programDAO.fetchOneById(programId); try { + Callable> postFunction = () -> { + ApiResponse response = api.observationsObservationDbIdPut(dbId, observation); - if (response == null) - { + if (response == null) { + throw new ApiException("Response is null", 0, null, null); + } + BrAPIObservationSingleResponse body = response.getBody(); + if (body == null) { + throw new ApiException("Response is missing body", 0, response.getHeaders(), null); + } + BrAPIObservation updatedObservation = body.getResult(); + if (updatedObservation == null) { + throw new ApiException("Response body is missing result", 0, response.getHeaders(), response.getBody().toString()); + } + return processObservationsForCache(List.of(updatedObservation), program.getKey()); + }; + // returns ListArray<> = postFunction.call().values() + return programObservationCache.post(programId, postFunction).get(0); + } catch (ApiException e) { + log.error(Utilities.generateApiExceptionLogMessage(e)); + throw new InternalServerException("Unknown error has occurred: " + e.getMessage(), e); + } catch (Exception e) { + throw new InternalServerException("Unknown error has occurred: " + e.getMessage(), e); + } + } + + public void updateBrAPIObservation(Map mutatedObservationByDbId, UUID programId) throws ApiException { + ObservationsApi api = brAPIEndpointProvider.get(programDAO.getCoreClient(programId), ObservationsApi.class); + var program = programDAO.fetchOneById(programId); + + List updatedObservations = new ArrayList<>(); + try { + Map updatedObservationsByDbId = new HashMap<>(); + for (Map.Entry entry : mutatedObservationByDbId.entrySet()) { + String dbId = entry.getKey(); + BrAPIObservation observation = entry.getValue(); + if (observation == null) { + throw new Exception("Null observation"); + } + + ApiResponse response = null; + try { + response = api.observationsObservationDbIdPut(dbId, observation); + } catch (ApiException e) { + throw new RuntimeException(e); + } + if (response == null) { throw new ApiException("Response is null", 0, null, null); } BrAPIObservationSingleResponse body = response.getBody(); @@ -264,14 +308,29 @@ public BrAPIObservation updateBrAPIObservation(String dbId, BrAPIObservation obs if (updatedObservation == null) { throw new ApiException("Response body is missing result", 0, response.getHeaders(), response.getBody().toString()); } - return processObservationsForCache(List.of(updatedObservation), program.getKey()); - }; - return programObservationCache.post(programId, postFunction).get(0); - } catch (ApiException e) { - log.error(Utilities.generateApiExceptionLogMessage(e)); - throw new InternalServerException("Unknown error has occurred: " + e.getMessage(), e); - } catch (Exception e) { - throw new InternalServerException("Unknown error has occurred: " + e.getMessage(), e); - } + updatedObservations.add(updatedObservation); + + if (!Objects.equals(observation.getValue(), updatedObservation.getValue()) + || !Objects.equals(observation.getObservationTimeStamp(), updatedObservation.getObservationTimeStamp())) { + String message; + if (!Objects.equals(observation.getValue(), updatedObservation.getValue())) { + message = String.format("Updated observation, %s, from BrAPI service does not match requested update %s.", updatedObservation.getValue(), observation.getValue()); + } else { + message = String.format("Updated observation timestamp, %s, from BrAPI service does not match requested update timestamp %s.", updatedObservation.getObservationTimeStamp(), observation.getObservationTimeStamp()); + } + throw new Exception(message); + } + } + + } catch (ApiException e) { + log.error("Error updating observation: " + Utilities.generateApiExceptionLogMessage(e), e); + throw new InternalServerException("Error saving experiment import", e); + } catch (Exception e) { + log.error("Error updating observation: ", e); + throw new InternalServerException(e.getMessage(), e); + } + processObservationsForCache(updatedObservations, program.getKey()); + programObservationCache.populate(programId); + return; } } diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/ExperimentProcessor.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/ExperimentProcessor.java index c0f5c4c1f..d67cebdfc 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/ExperimentProcessor.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/ExperimentProcessor.java @@ -300,13 +300,16 @@ public void validateDependencies(Map mappedBrAPIImport) } @Override - public void postBrapiData(Map mappedBrAPIImport, Program program, ImportUpload upload) { + public void postBrapiData(Map mappedBrAPIImport, Program program, ImportUpload upload) { log.debug("starting post of experiment data to BrAPI server"); List newTrials = ProcessorData.getNewObjects(this.trialByNameNoScope); Map mutatedTrialsById = ProcessorData .getMutationsByObjectId(trialByNameNoScope, BrAPITrial::getTrialDbId); + //getMutationsByObjectId() will return a HashMap of just the mutated Observations, + // key = BrAPIObservation::getObservationDbId + // value = mutated BrAPIObservation Map mutatedObservationByDbId = ProcessorData .getMutationsByObjectId(observationByHash, BrAPIObservation::getObservationDbId); @@ -420,6 +423,7 @@ public void postBrapiData(Map mappedBrAPIImport, Program obsVarIds.addAll(newObsVarIds); dataset.setData(obsVarIds); brAPIListDAO.updateBrAPIList(id, dataset, program.getId()); + // Map updatedObservationByDbId = brAPIObservationDAO.updateBrAPIObservation( mutatedObservationByDbId, program.getId() ); } catch (ApiException e) { log.error("Error updating dataset observation variables: " + Utilities.generateApiExceptionLogMessage(e), e); throw new InternalServerException("Error saving experiment import", e); @@ -428,36 +432,16 @@ public void postBrapiData(Map mappedBrAPIImport, Program throw new InternalServerException(e.getMessage(), e); } }); - - mutatedObservationByDbId.forEach((id, observation) -> { - try { - if (observation == null) { - throw new Exception("Null observation"); - } - BrAPIObservation updatedObs = brAPIObservationDAO.updateBrAPIObservation(id, observation, program.getId()); - - if (updatedObs == null) { - throw new Exception("Null updated observation"); - } - - if (!Objects.equals(observation.getValue(), updatedObs.getValue()) - || !Objects.equals(observation.getObservationTimeStamp(), updatedObs.getObservationTimeStamp())) { - String message; - if(!Objects.equals(observation.getValue(), updatedObs.getValue())) { - message = String.format("Updated observation, %s, from BrAPI service does not match requested update %s.", updatedObs.getValue(), observation.getValue()); - } else { - message = String.format("Updated observation timestamp, %s, from BrAPI service does not match requested update timestamp %s.", updatedObs.getObservationTimeStamp(), observation.getObservationTimeStamp()); - } - throw new Exception(message); - } - } catch (ApiException e) { + try { + brAPIObservationDAO.updateBrAPIObservation( mutatedObservationByDbId, program.getId() ); + } catch (ApiException e) { log.error("Error updating observation: " + Utilities.generateApiExceptionLogMessage(e), e); throw new InternalServerException("Error saving experiment import", e); } catch (Exception e) { log.error("Error updating observation: ", e); throw new InternalServerException(e.getMessage(), e); } - }); + log.debug("experiment import complete"); } diff --git a/src/main/java/org/breedinginsight/daos/cache/ProgramCache.java b/src/main/java/org/breedinginsight/daos/cache/ProgramCache.java index d231116ae..0e41a6b17 100644 --- a/src/main/java/org/breedinginsight/daos/cache/ProgramCache.java +++ b/src/main/java/org/breedinginsight/daos/cache/ProgramCache.java @@ -168,6 +168,11 @@ private Map deserialize(Map cachedVals) { } public List post(UUID key, Callable> postMethod) throws Exception { + log.debug("posting for key: " + generateCacheKey(key)); + Map response = post_map(key,postMethod); + return new ArrayList<>(response.values()); + } + public Map post_map(UUID key, Callable> postMethod) throws Exception { log.debug("posting for key: " + generateCacheKey(key)); Map response = null; try { @@ -181,14 +186,13 @@ public List post(UUID key, Callable> postMethod) throws Except } populate(key); - return new ArrayList<>(response.values()); + return response; } catch (Exception e) { log.error("Error posting data and populating the cache", e); invalidate(key); throw e; } } - public boolean isRefreshing(UUID key) { RAtomicLong isRefreshing = connection.getAtomicLong(generateCacheKey(key) + ":refreshing"); From 8b7e49ffb8c00f560e80c6b40c8484cab1a4769a Mon Sep 17 00:00:00 2001 From: David Randolph Phillips Date: Thu, 20 Jun 2024 09:11:17 -0400 Subject: [PATCH 2/3] [BI-2093] Added comments, tightened code --- .../breedinginsight/brapi/v2/dao/BrAPIObservationDAO.java | 3 +++ .../java/org/breedinginsight/daos/cache/ProgramCache.java | 8 ++------ 2 files changed, 5 insertions(+), 6 deletions(-) diff --git a/src/main/java/org/breedinginsight/brapi/v2/dao/BrAPIObservationDAO.java b/src/main/java/org/breedinginsight/brapi/v2/dao/BrAPIObservationDAO.java index af06352ca..fc9970b75 100644 --- a/src/main/java/org/breedinginsight/brapi/v2/dao/BrAPIObservationDAO.java +++ b/src/main/java/org/breedinginsight/brapi/v2/dao/BrAPIObservationDAO.java @@ -277,6 +277,9 @@ public BrAPIObservation updateBrAPIObservation(String dbId, BrAPIObservation obs } } + // This method overloads updateBrAPIObservation(String dbId, BrAPIObservation observation, UUID programId) + // It was added to increase efficiency. It insures that ProgramCache.populate() is called only once + // not once per observation. public void updateBrAPIObservation(Map mutatedObservationByDbId, UUID programId) throws ApiException { ObservationsApi api = brAPIEndpointProvider.get(programDAO.getCoreClient(programId), ObservationsApi.class); var program = programDAO.fetchOneById(programId); diff --git a/src/main/java/org/breedinginsight/daos/cache/ProgramCache.java b/src/main/java/org/breedinginsight/daos/cache/ProgramCache.java index 0e41a6b17..d231116ae 100644 --- a/src/main/java/org/breedinginsight/daos/cache/ProgramCache.java +++ b/src/main/java/org/breedinginsight/daos/cache/ProgramCache.java @@ -168,11 +168,6 @@ private Map deserialize(Map cachedVals) { } public List post(UUID key, Callable> postMethod) throws Exception { - log.debug("posting for key: " + generateCacheKey(key)); - Map response = post_map(key,postMethod); - return new ArrayList<>(response.values()); - } - public Map post_map(UUID key, Callable> postMethod) throws Exception { log.debug("posting for key: " + generateCacheKey(key)); Map response = null; try { @@ -186,13 +181,14 @@ public Map post_map(UUID key, Callable> postMethod) th } populate(key); - return response; + return new ArrayList<>(response.values()); } catch (Exception e) { log.error("Error posting data and populating the cache", e); invalidate(key); throw e; } } + public boolean isRefreshing(UUID key) { RAtomicLong isRefreshing = connection.getAtomicLong(generateCacheKey(key) + ":refreshing"); From 2086a1788104c57660569c545f5d9f803dd5dafc Mon Sep 17 00:00:00 2001 From: David Randolph Phillips Date: Thu, 20 Jun 2024 11:34:13 -0400 Subject: [PATCH 3/3] [BI-2093] Clean up code --- .../brapps/importer/services/processors/ExperimentProcessor.java | 1 - 1 file changed, 1 deletion(-) diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/ExperimentProcessor.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/ExperimentProcessor.java index d67cebdfc..4e4aaf9cc 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/ExperimentProcessor.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/ExperimentProcessor.java @@ -423,7 +423,6 @@ public void postBrapiData(Map mappedBrAPIImport, Program obsVarIds.addAll(newObsVarIds); dataset.setData(obsVarIds); brAPIListDAO.updateBrAPIList(id, dataset, program.getId()); - // Map updatedObservationByDbId = brAPIObservationDAO.updateBrAPIObservation( mutatedObservationByDbId, program.getId() ); } catch (ApiException e) { log.error("Error updating dataset observation variables: " + Utilities.generateApiExceptionLogMessage(e), e); throw new InternalServerException("Error saving experiment import", e);