From 311ff826919f7ce563f00d4814b334c6e8f7bdd7 Mon Sep 17 00:00:00 2001 From: timparsons Date: Fri, 15 Sep 2023 16:17:47 -0400 Subject: [PATCH 1/3] [BI-1909] small fix for changes made in the brapi-java-server Changed the database container to be created once at the beginning of running unit tests, and reused throughout all unit tests. Updated the AfterAll method to reset the database before the next test. --- .../org/breedinginsight/DatabaseTest.java | 80 +++++++++++++------ .../TraitControllerIntegrationTest.java | 4 +- 2 files changed, 57 insertions(+), 27 deletions(-) diff --git a/src/test/java/org/breedinginsight/DatabaseTest.java b/src/test/java/org/breedinginsight/DatabaseTest.java index a117c45cb..11be7bf8f 100644 --- a/src/test/java/org/breedinginsight/DatabaseTest.java +++ b/src/test/java/org/breedinginsight/DatabaseTest.java @@ -31,10 +31,12 @@ import org.testcontainers.images.PullPolicy; import javax.annotation.Nonnull; +import java.sql.Connection; +import java.sql.DriverManager; +import java.sql.SQLException; import java.time.Duration; import java.time.temporal.ChronoUnit; -import java.util.HashMap; -import java.util.Map; +import java.util.*; @Slf4j public class DatabaseTest implements TestPropertyProvider { @@ -43,36 +45,43 @@ public class DatabaseTest implements TestPropertyProvider { private static final String dbPassword = "postgres"; @Getter - private GenericContainer dbContainer; + private static GenericContainer dbContainer; @Getter - private Network network; + private static Network network; @Getter - private GenericContainer redisContainer; + private static GenericContainer redisContainer; @Getter private RedissonClient redisConnection; - @SneakyThrows +// @SneakyThrows public DatabaseTest() { - network = Network.newNetwork(); - dbContainer = new GenericContainer<>("postgres:11.4") - .withNetwork(network) - .withNetworkAliases("testdb") - .withImagePullPolicy(PullPolicy.defaultPolicy()) - .withExposedPorts(5432) - .withEnv("POSTGRES_DB", dbName) - .withEnv("POSTGRES_PASSWORD", dbPassword) - .waitingFor(Wait.forLogMessage(".*LOG: database system is ready to accept connections.*", 1).withStartupTimeout(Duration.of(2, ChronoUnit.MINUTES))); - dbContainer.start(); - redisContainer = new GenericContainer<>("redis") - .withNetwork(network) - .withNetworkAliases("redis") - .withImagePullPolicy(PullPolicy.defaultPolicy()) - .withExposedPorts(6379) - .waitingFor(Wait.forListeningPort()); - redisContainer.start(); + if(network == null) { + network = Network.newNetwork(); + } + if(dbContainer == null) { + dbContainer = new GenericContainer<>("postgres:11.4") + .withNetwork(network) + .withNetworkAliases("testdb") + .withImagePullPolicy(PullPolicy.defaultPolicy()) + .withExposedPorts(5432) + .withEnv("POSTGRES_DB", dbName) + .withEnv("POSTGRES_PASSWORD", dbPassword) + .waitingFor(Wait.forLogMessage(".*LOG: database system is ready to accept connections.*", 1) + .withStartupTimeout(Duration.of(2, ChronoUnit.MINUTES))); + dbContainer.start(); + } + if(redisContainer == null) { + redisContainer = new GenericContainer<>("redis") + .withNetwork(network) + .withNetworkAliases("redis") + .withImagePullPolicy(PullPolicy.defaultPolicy()) + .withExposedPorts(6379) + .waitingFor(Wait.forListeningPort()); + redisContainer.start(); + } Integer redisContainerPort = redisContainer.getMappedPort(6379); String redisContainerIp = redisContainer.getContainerIpAddress(); @@ -111,8 +120,27 @@ public Map getProperties() { @SneakyThrows @AfterAll public void stopContainers() { - redisContainer.stop(); - dbContainer.stop(); - network.close(); + if(redisContainer != null) { + redisConnection.getKeys() + .flushall(); + } + if(dbContainer != null) { + resetDb(dbName); + resetDb("postgres"); + } + } + + private void resetDb(String name) { + log.debug("resetting db: " + name); + try (Connection con = DriverManager. + getConnection(String.format("jdbc:postgresql://%s:%s/"+name, + dbContainer.getContainerIpAddress(), dbContainer.getMappedPort(5432)), + "postgres", dbPassword)) { + + con.prepareStatement("drop schema public CASCADE").execute(); + con.prepareStatement("create schema public").execute(); + } catch (SQLException e) { + log.error("Error during reset", e); + } } } diff --git a/src/test/java/org/breedinginsight/api/v1/controller/TraitControllerIntegrationTest.java b/src/test/java/org/breedinginsight/api/v1/controller/TraitControllerIntegrationTest.java index 89178c965..9f43c37b8 100644 --- a/src/test/java/org/breedinginsight/api/v1/controller/TraitControllerIntegrationTest.java +++ b/src/test/java/org/breedinginsight/api/v1/controller/TraitControllerIntegrationTest.java @@ -1104,7 +1104,9 @@ public void checkTraitFullResponse(JsonObject traitJson, Trait trait) { assertEquals(trait.getScale().getDataType().toString(), scaleJson.get("dataType").getAsString(), "Scale data types don't match"); List jsonCategories = new ArrayList<>(); - scaleJson.get("categories").getAsJsonArray().iterator().forEachRemaining(element -> jsonCategories.add(element.getAsJsonObject())); + if(scaleJson.has("categories")) { + scaleJson.get("categories").getAsJsonArray().iterator().forEachRemaining(element -> jsonCategories.add(element.getAsJsonObject())); + } Collections.sort(jsonCategories, Comparator.comparing((x) -> x.get("label").getAsString())); List brApiScaleCategories = new ArrayList<>(trait.getScale().getCategories()); Collections.sort(brApiScaleCategories, Comparator.comparing(BrAPIScaleValidValuesCategories::getLabel)); From 8de2b1f2e6cddc72d773d8aa38ac6983c2fc8aef Mon Sep 17 00:00:00 2001 From: timparsons Date: Tue, 19 Sep 2023 14:29:39 -0400 Subject: [PATCH 2/3] [BI-1909] adding log messages to dataset fetch code --- .../brapi/v2/services/BrAPITrialService.java | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/src/main/java/org/breedinginsight/brapi/v2/services/BrAPITrialService.java b/src/main/java/org/breedinginsight/brapi/v2/services/BrAPITrialService.java index 8c002d27c..b90f7ef8e 100644 --- a/src/main/java/org/breedinginsight/brapi/v2/services/BrAPITrialService.java +++ b/src/main/java/org/breedinginsight/brapi/v2/services/BrAPITrialService.java @@ -92,7 +92,9 @@ public BrAPITrial getTrialDataByUUID(UUID programId, UUID trialId, boolean stats //Remove the [program key] from the trial name trial.setTrialName( Utilities.removeUnknownProgramKey( trial.getTrialName()) ); if( stats ){ + log.debug("fetching experiment: " + trialId + " stats"); int environmentsCount = 1; // For now this is hardcoded to 1, because we are only supporting one environment per experiment + log.debug("fetching observation units for experiment: " + trialId); long germplasmCount = countGermplasm(programId, trial.getTrialDbId()); trial.putAdditionalInfoItem("environmentsCount", environmentsCount); trial.putAdditionalInfoItem("germplasmCount", germplasmCount); @@ -296,11 +298,16 @@ private StreamedFile zipFiles(List files) throws IOException { } public Dataset getDatasetData(Program program, UUID experimentId, UUID datsetId, Boolean stats) throws ApiException, DoesNotExistException { + log.debug("fetching dataset: " + datsetId + " for experiment: " + experimentId + ". including stats: " + stats); + log.debug("fetching observationUnits for dataset: " + datsetId); List datasetOUs = ouDAO.getObservationUnitsForDataset(datsetId.toString(), program); + log.debug("fetching dataset variables dataset: " + datsetId); List datasetObsVars = getDatasetObsVars(datsetId.toString(), program); List ouDbIds = datasetOUs.stream().map(BrAPIObservationUnit::getObservationUnitDbId).collect(Collectors.toList()); List obsVarDbIds = datasetObsVars.stream().map(BrAPIObservationVariable::getObservationVariableDbId).collect(Collectors.toList()); + log.debug("fetching observations for dataset: " + datsetId); List data = observationDAO.getObservationsByObservationUnitsAndVariables(ouDbIds, obsVarDbIds, program); + log.debug("building dataset object for dataset: " + datsetId); Dataset dataset = new Dataset(experimentId.toString(), data, datasetOUs, datasetObsVars); if (stats) { Integer ouCount = datasetOUs.size(); From 294172c01f1704fdcd749bd21c57d55a4d523182 Mon Sep 17 00:00:00 2001 From: timparsons Date: Thu, 21 Sep 2023 15:40:24 -0400 Subject: [PATCH 3/3] [BI-1909] cleanup and renaming method --- src/test/java/org/breedinginsight/DatabaseTest.java | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/src/test/java/org/breedinginsight/DatabaseTest.java b/src/test/java/org/breedinginsight/DatabaseTest.java index 11be7bf8f..540955bcf 100644 --- a/src/test/java/org/breedinginsight/DatabaseTest.java +++ b/src/test/java/org/breedinginsight/DatabaseTest.java @@ -56,7 +56,6 @@ public class DatabaseTest implements TestPropertyProvider { @Getter private RedissonClient redisConnection; -// @SneakyThrows public DatabaseTest() { if(network == null) { network = Network.newNetwork(); @@ -119,7 +118,7 @@ public Map getProperties() { @SneakyThrows @AfterAll - public void stopContainers() { + public void resetContainers() { if(redisContainer != null) { redisConnection.getKeys() .flushall();