Skip to content

[BI-1638] - Cache germplasm data immediately after saving to BrAPI #218

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
timparsons merged 1 commit into from
Oct 17, 2022
Merged

[BI-1638] - Cache germplasm data immediately after saving to BrAPI #218

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
6 changes: 5 additions & 1 deletion src/main/java/org/breedinginsight/brapi/v2/dao/BrAPIGermplasmDAO.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -219,8 +219,12 @@ private Map<String,BrAPIGermplasm> processGermplasmForDisplay(List<BrAPIGermplas

public List<BrAPIGermplasm> importBrAPIGermplasm(List<BrAPIGermplasm> brAPIGermplasmList, UUID programId, ImportUpload upload) throws ApiException {
GermplasmApi api = new GermplasmApi(programDAO.getCoreClient(programId));
var program = programDAO.fetchOneById(programId);
try {
Callable<List<BrAPIGermplasm>> postFunction = () -> brAPIDAOUtil.post(brAPIGermplasmList, upload, api::germplasmPost, importDAO::update);
Callable<Map<String, BrAPIGermplasm>> postFunction = () -> {
List<BrAPIGermplasm> postResponse = brAPIDAOUtil.post(brAPIGermplasmList, upload, api::germplasmPost, importDAO::update);
return processGermplasmForDisplay(postResponse, program.getKey());
};
return programGermplasmCache.post(programId, postFunction);
} catch (ApiException e) {
throw e;
Expand Down
14 changes: 11 additions & 3 deletions src/main/java/org/breedinginsight/brapi/v2/dao/ProgramCache.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -129,9 +129,17 @@ private void updateCache(UUID programId) {
}
}

public List<R> post(UUID programId, Callable<List<R>> postMethod) throws Exception {
List<R> response = postMethod.call();
public List<R> post(UUID programId, Callable<Map<K, R>> postMethod) throws Exception {
Map<K, R> response = postMethod.call();

Map<K, R> map = cache.getIfPresent(programId);
if(map != null) {
map.putAll(response);
} else {
cache.put(programId, response);
}

updateCache(programId);
return response;
return new ArrayList<>(response.values());
}
}
36 changes: 19 additions & 17 deletions src/test/java/org/breedinginsight/brapi/v2/ProgramCacheUnitTest.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -20,6 +20,7 @@
import javax.inject.Inject;
import java.util.*;
import java.util.concurrent.Callable;
import java.util.concurrent.atomic.AtomicInteger;
import java.util.stream.Collectors;

import static org.junit.jupiter.api.Assertions.*;
Expand All @@ -36,7 +37,7 @@ public class ProgramCacheUnitTest {
// POSTing refresh tracking separate for each program
// Cache refresh failure invalidates cache

Integer fetchCount = 0;
AtomicInteger fetchCount = new AtomicInteger(0);
Integer waitTime = 1000;
Map<UUID, List<BrAPIGermplasm>> mockBrAPI = new HashMap<>();

Expand All @@ -53,26 +54,28 @@ void setup() {}
void setupNextTest() {
// There is some thread sleeping used in this testing, wait for all processes to clean out between tests
Thread.sleep(5000);
fetchCount = 0;
fetchCount.set(0);
mockBrAPI = new HashMap<>();
}

@SneakyThrows
public Map<String, BrAPIGermplasm> mockFetch(UUID programId, Integer sleepTime) {
fetchCount += 1;
fetchCount.incrementAndGet();
Thread.sleep(sleepTime);
return mockBrAPI.containsKey(programId) ? new HashMap<>(mockBrAPI.get(programId).stream().collect(Collectors.toMap(germplasm -> UUID.randomUUID().toString(), germplasm -> germplasm))) : new HashMap<>();
}

@SneakyThrows
public List<BrAPIGermplasm> mockPost(UUID programId, List<BrAPIGermplasm> germplasm) {
public Map<String, BrAPIGermplasm> mockPost(UUID programId, List<BrAPIGermplasm> germplasm) {
if (!mockBrAPI.containsKey(programId)) {
mockBrAPI.put(programId, germplasm);
} else {
List<BrAPIGermplasm> allGermplasm = mockBrAPI.get(programId);
allGermplasm.addAll(germplasm);
}
return germplasm;
Map<String, BrAPIGermplasm> germMap = new HashMap<>();
germplasm.forEach(brAPIGermplasm -> germMap.put(brAPIGermplasm.getGermplasmDbId(), brAPIGermplasm));
return germMap;
}

@Test
Expand All @@ -85,11 +88,11 @@ public void populatedRefreshQueueSkipsRefresh() {
int currPost = 0;
while (currPost < numPost) {
List<BrAPIGermplasm> newList = new ArrayList<>();
newList.add(new BrAPIGermplasm());
newList.add(new BrAPIGermplasm().germplasmDbId(UUID.randomUUID().toString()));
cache.post(programId, () -> mockPost(programId, new ArrayList<>(newList)));
currPost += 1;
}
assertTrue(fetchCount < numPost, "A fetch call was made for every post. It shouldn't.");
assertTrue(fetchCount.get() < numPost, "A fetch call was made for every post. It shouldn't.");
assertEquals(1, mockBrAPI.size(), "More than one program existed in mocked brapi db.");
assertEquals(numPost, mockBrAPI.get(programId).size(), "Wrong number of germplasm in db");
Map<String, BrAPIGermplasm> cachedGermplasm = cache.get(programId);
Expand All @@ -106,13 +109,12 @@ public void programRefreshesSeparated() {
while (currPost < numPost) {
UUID id = UUID.randomUUID();
List<BrAPIGermplasm> newList = new ArrayList<>();
newList.add(new BrAPIGermplasm());
newList.add(new BrAPIGermplasm().germplasmDbId(UUID.randomUUID().toString()));
cache.post(id, () -> mockPost(id, new ArrayList<>(newList)));
// This doesn't have to do with our code, our mock function is just tripping over itself trying to update the number of fetches
Thread.sleep(waitTime/5);
currPost += 1;
}
assertEquals(numPost, fetchCount, "A fetch call should have been made for every post");
Thread.sleep(waitTime);
assertEquals(numPost, fetchCount.get(), "A fetch call should have been made for every post");
assertEquals(numPost, mockBrAPI.size(), "Less programs existed than existed in mock brapi db.");
for (UUID key: mockBrAPI.keySet()) {
assertEquals(1, mockBrAPI.get(key).size(), "Wrong number of germplasm in db");
Expand All @@ -128,19 +130,19 @@ public void initialGetMethodWaitsForLoad() {
ProgramCache<String, BrAPIGermplasm> cache = new ProgramCache<>((UUID id) -> mockFetch(id, waitTime), List.of(programId));
cache.get(programId);
// Our fetch method should have only been called once for the initial loading
assertEquals(1, fetchCount, "Fetch method was called on get");
assertEquals(1, fetchCount.get(), "Fetch method was called on get");
}

@Test
@SneakyThrows
public void getMethodDoesNotWaitForRefresh() {
public void postTriggersRefresh() {
// Test that the get method does not wait for a refresh when there is data present
UUID programId = UUID.randomUUID();
List<BrAPIGermplasm> newList = new ArrayList<>();
newList.add(new BrAPIGermplasm());
newList.add(new BrAPIGermplasm().germplasmDbId(UUID.randomUUID().toString()));
mockBrAPI.put(programId, new ArrayList<>(newList));
ProgramCache<String, BrAPIGermplasm> cache = new ProgramCache<>((UUID id) -> mockFetch(id, waitTime*2), List.of(programId));
Callable<List<BrAPIGermplasm>> postFunction = () -> mockPost(programId, new ArrayList<>(newList));
Callable<Map<String, BrAPIGermplasm>> postFunction = () -> mockPost(programId, new ArrayList<>(newList));

// Get waits for initial fetch
Map<String, BrAPIGermplasm> cachedGermplasm = cache.get(programId);
Expand All @@ -149,7 +151,7 @@ public void getMethodDoesNotWaitForRefresh() {
// Now post another object and call get immediately to see that it returns the old data
cache.post(programId, postFunction);
cachedGermplasm = cache.get(programId);
assertEquals(1, cachedGermplasm.size(), "Get method seemed to have waited for refresh method");
assertEquals(2, cachedGermplasm.size(), "Get post method didn't insert the new data");

// Now wait for the fetch after the post to finish
Thread.sleep(waitTime*3);
Expand All @@ -167,7 +169,7 @@ public void refreshErrorInvalidatesCache() {
// Set starter data
UUID programId = UUID.randomUUID();
List<BrAPIGermplasm> newList = new ArrayList<>();
newList.add(new BrAPIGermplasm());
newList.add(new BrAPIGermplasm().germplasmDbId(UUID.randomUUID().toString()));
mockBrAPI.put(programId, new ArrayList<>(newList));

// Mock our method
Expand Down