Skip to content
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,7 @@
import io.micronaut.http.HttpStatus;
import io.micronaut.http.MediaType;
import io.micronaut.http.annotation.*;
import io.micronaut.http.server.exceptions.InternalServerException;
import io.micronaut.http.server.types.files.StreamedFile;
import io.micronaut.security.annotation.Secured;
import io.micronaut.security.rules.SecurityRule;
Expand All @@ -21,6 +22,7 @@
import org.breedinginsight.brapi.v2.model.response.mappers.GermplasmQueryMapper;
import org.breedinginsight.brapi.v2.services.BrAPIGermplasmService;
import org.breedinginsight.model.DownloadFile;
import org.breedinginsight.services.exceptions.DoesNotExistException;
import org.breedinginsight.utilities.response.ResponseUtils;

import javax.inject.Inject;
Expand Down Expand Up @@ -79,4 +81,23 @@ public HttpResponse<StreamedFile> germplasmListExport(
return response;
}
}

@Get("/${micronaut.bi.api.version}/programs/{programId}" + BrapiVersion.BRAPI_V2 + "/germplasm/{germplasmId}")
@Produces(MediaType.APPLICATION_JSON)
@ProgramSecured(roleGroups = {ProgramSecuredRoleGroup.ALL})
public HttpResponse<Response<BrAPIGermplasm>> getSingleGermplasm(
@PathVariable("programId") UUID programId,
@PathVariable("germplasmId") String germplasmId) {
try {
log.debug("fetching germ id:" + germplasmId +" for program: " + programId);
Response<BrAPIGermplasm> response = new Response(germplasmService.getGermplasmByUUID(programId, germplasmId));
return HttpResponse.ok(response);
} catch (InternalServerException e) {
log.info(e.getMessage(), e);
return HttpResponse.status(HttpStatus.INTERNAL_SERVER_ERROR, "Error retrieving germplasm");
} catch (DoesNotExistException e) {
log.info(e.getMessage(), e);
return HttpResponse.status(HttpStatus.NOT_FOUND, "Germplasm not found");
}
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -19,4 +19,5 @@

public final class BrAPIAdditionalInfoFields {
public static final String GERMPLASM_RAW_PEDIGREE = "rawPedigree";
public static final String GERMPLASM_PEDIGREE_BY_NAME = "pedigreeByName";
}
Original file line number Diff line number Diff line change
Expand Up @@ -24,13 +24,15 @@
import lombok.extern.slf4j.Slf4j;
import org.brapi.client.v2.model.exceptions.ApiException;
import org.brapi.client.v2.modules.germplasm.GermplasmApi;
import org.brapi.v2.model.BrAPIExternalReference;
import org.brapi.v2.model.germ.BrAPIGermplasm;
import org.brapi.v2.model.germ.request.BrAPIGermplasmSearchRequest;
import org.breedinginsight.brapi.v2.constants.BrAPIAdditionalInfoFields;
import org.breedinginsight.brapps.importer.daos.ImportDAO;
import org.breedinginsight.brapps.importer.model.ImportUpload;
import org.breedinginsight.daos.ProgramDAO;
import org.breedinginsight.model.Program;
import org.breedinginsight.services.exceptions.DoesNotExistException;
import org.breedinginsight.utilities.BrAPIDAOUtil;
import org.breedinginsight.utilities.Utilities;

Expand All @@ -54,7 +56,7 @@ public class BrAPIGermplasmDAO {
@Property(name = "brapi.server.reference-source")
private String referenceSource;

ProgramCache<BrAPIGermplasm> programGermplasmCache;
ProgramCache<String, BrAPIGermplasm> programGermplasmCache;

@Inject
public BrAPIGermplasmDAO(ProgramDAO programDAO, ImportDAO importDAO) {
Expand All @@ -76,7 +78,7 @@ private void setup() {
* @throws ApiException
*/
public List<BrAPIGermplasm> getGermplasm(UUID programId) throws ApiException {
return programGermplasmCache.get(programId);
return new ArrayList<>(programGermplasmCache.get(programId).values());
}

/**
Expand All @@ -87,7 +89,8 @@ public List<BrAPIGermplasm> getGermplasm(UUID programId) throws ApiException {
*/
public List<BrAPIGermplasm> getRawGermplasm(UUID programId) throws ApiException {
Program program = new Program(programDAO.fetchOneById(programId));
return programGermplasmCache.get(programId).stream().map(germplasm -> {
List<BrAPIGermplasm> cacheList = new ArrayList<>(programGermplasmCache.get(programId).values());
return cacheList.stream().map(germplasm -> {
germplasm.setGermplasmName(Utilities.appendProgramKey(germplasm.getDefaultDisplayName(), program.getKey(), germplasm.getAccessionNumber()));
if(germplasm.getAdditionalInfo() != null && germplasm.getAdditionalInfo()
.has(BrAPIAdditionalInfoFields.GERMPLASM_RAW_PEDIGREE)) {
Expand All @@ -98,7 +101,14 @@ public List<BrAPIGermplasm> getRawGermplasm(UUID programId) throws ApiException
}).collect(Collectors.toList());
}

private List<BrAPIGermplasm> fetchProgramGermplasm(UUID programId) throws ApiException {

/**
* Fetch formatted germplasm for this program
* @param programId
* @return Map<Key = string representing germplasm UUID, value = formatted BrAPIGermplasm>
* @throws ApiException
*/
private Map<String, BrAPIGermplasm> fetchProgramGermplasm(UUID programId) throws ApiException {
GermplasmApi api = new GermplasmApi(programDAO.getCoreClient(programId));

// Set query params and make call
Expand All @@ -112,8 +122,15 @@ private List<BrAPIGermplasm> fetchProgramGermplasm(UUID programId) throws ApiExc
));
}

private List<BrAPIGermplasm> processGermplasmForDisplay(List<BrAPIGermplasm> programGermplasm) {
/**
* Process germplasm into a format for display
* @param programGermplasm
* @return Map<Key = string representing germplasm UUID, value = formatted BrAPIGermplasm>
* @throws ApiException
*/
private Map<String,BrAPIGermplasm> processGermplasmForDisplay(List<BrAPIGermplasm> programGermplasm) {
// Process the germplasm
Map<String, BrAPIGermplasm> programGermplasmMap = new HashMap<>();
log.debug("processing germ for display: " + programGermplasm);
Map<String, BrAPIGermplasm> programGermplasmByFullName = new HashMap<>();
for (BrAPIGermplasm germplasm: programGermplasm) {
Expand All @@ -139,22 +156,32 @@ private List<BrAPIGermplasm> processGermplasmForDisplay(List<BrAPIGermplasm> pro
additionalInfo.addProperty(BrAPIAdditionalInfoFields.GERMPLASM_RAW_PEDIGREE, germplasm.getPedigree());

String newPedigreeString = "";
String namePedigreeString = "";
List<String> parents = Arrays.asList(germplasm.getPedigree().split("/"));
if (parents.size() >= 1) {
if (programGermplasmByFullName.containsKey(parents.get(0))) {
newPedigreeString = programGermplasmByFullName.get(parents.get(0)).getAccessionNumber();
namePedigreeString = programGermplasmByFullName.get(parents.get(0)).getDefaultDisplayName();
}
}
if (parents.size() == 2) {
if (programGermplasmByFullName.containsKey(parents.get(1))) {
newPedigreeString += "/" + programGermplasmByFullName.get(parents.get(1)).getAccessionNumber();
namePedigreeString += "/" + programGermplasmByFullName.get(parents.get(1)).getDefaultDisplayName();
}
}
//For use in individual germplasm display
additionalInfo.addProperty(BrAPIAdditionalInfoFields.GERMPLASM_PEDIGREE_BY_NAME, namePedigreeString);

germplasm.setPedigree(newPedigreeString);
}

BrAPIExternalReference extRef = germplasm.getExternalReferences().stream().filter(reference -> referenceSource.equals(reference.getReferenceSource())).findFirst().orElseThrow(() -> new IllegalStateException("No BI external reference found"));
String germplasmId = extRef.getReferenceID();
programGermplasmMap.put(germplasmId, germplasm);
}

return programGermplasm;
return programGermplasmMap;
}

public List<BrAPIGermplasm> importBrAPIGermplasm(List<BrAPIGermplasm> brAPIGermplasmList, UUID programId, ImportUpload upload) throws ApiException {
Expand All @@ -176,4 +203,17 @@ public List<BrAPIGermplasm> getGermplasmByRawName(List<String> germplasmNames, U
.filter(brAPIGermplasm -> germplasmNames.contains(Utilities.appendProgramKey(brAPIGermplasm.getGermplasmName(),program.getKey(),brAPIGermplasm.getAccessionNumber())))
.collect(Collectors.toList());
}

public BrAPIGermplasm getGermplasmByUUID(String germplasmId, UUID programId) throws ApiException, DoesNotExistException {
Map<String, BrAPIGermplasm> cache = programGermplasmCache.get(programId);
BrAPIGermplasm germplasm = null;
if (cache != null) {
germplasm = cache.get(germplasmId);
}
if (germplasm == null) {
throw new DoesNotExistException("UUID for this germplasm does not exist");
}
return germplasm;
}

}
31 changes: 19 additions & 12 deletions src/main/java/org/breedinginsight/brapi/v2/dao/ProgramCache.java
Original file line number Diff line number Diff line change
Expand Up @@ -30,20 +30,25 @@
import java.util.concurrent.*;
import java.util.stream.Collectors;

/**
*
* @param <K> key/id of object
* @param <R> object
*/
@Slf4j
public class ProgramCache<R> {
public class ProgramCache<K, R> {

private final FetchFunction<UUID, List<R>> fetchMethod;
private final FetchFunction<UUID, Map<K, R>> fetchMethod;
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Not clearly explained that K should be the object id, it took me a bit to figure out what K was supposed to be. Maybe a class header comment for explanation?

private final Map<UUID, Semaphore> programSemaphore = new HashMap<>();
private final Cloner cloner;

private final Executor executor = Executors.newCachedThreadPool();
private final LoadingCache<UUID, List<R>> cache = CacheBuilder.newBuilder()
private final LoadingCache<UUID, Map<K, R>> cache = CacheBuilder.newBuilder()
.build(new CacheLoader<>() {
@Override
public List<R> load(@NotNull UUID programId) throws Exception {
public Map<K, R> load(@NotNull UUID programId) throws Exception {
try {
List<R> values = fetchMethod.apply(programId);
Map<K, R> values = fetchMethod.apply(programId);
log.debug("cache loading complete.\nprogramId: " + programId);
return values;
} catch (Exception e) {
Expand All @@ -54,7 +59,7 @@ public List<R> load(@NotNull UUID programId) throws Exception {
}
});

public ProgramCache(FetchFunction<UUID, List<R>> fetchMethod, List<UUID> keys) {
public ProgramCache(FetchFunction<UUID, Map<K, R>> fetchMethod, List<UUID> keys) {
this.fetchMethod = fetchMethod;
this.cloner = new Cloner();
// Populate cache on start up
Expand All @@ -63,27 +68,29 @@ public ProgramCache(FetchFunction<UUID, List<R>> fetchMethod, List<UUID> keys) {
}
}

public ProgramCache(FetchFunction<UUID, List<R>> fetchMethod) {
public ProgramCache(FetchFunction<UUID, Map<K, R>> fetchMethod) {
this.fetchMethod = fetchMethod;
this.cloner = new Cloner();
}

public List<R> get(UUID programId) throws ApiException {
public Map<K, R> get(UUID programId) throws ApiException {
try {
// This will get current cache data, or wait for the refresh to finish if there is no cache data.
// TODO: Do we want to wait for a refresh method if it is running? Returns current data right now, even if old
if (!programSemaphore.containsKey(programId) || cache.getIfPresent(programId) == null) {
// If the cache is missing, refresh and get
log.trace("cache miss, fetching from source.\nprogramId: " + programId);
updateCache(programId);
List<R> result = new ArrayList<>(cache.get(programId));
result = result.stream().map(cloner::deepClone).collect(Collectors.toList());
Map<K, R> result = new HashMap<>(cache.get(programId));
result = result.entrySet().stream().map(cloner::deepClone)
.collect(Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue));
return result;
} else {
log.trace("cache contains records for the program.\nprogramId: " + programId);
// Most cases where the cache is populated
List<R> result = new ArrayList<>(cache.get(programId));
result = result.stream().map(cloner::deepClone).collect(Collectors.toList());
Map<K, R> result = new HashMap<>(cache.get(programId));
result = result.entrySet().stream().map(cloner::deepClone)
.collect(Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue));
return result;
}
} catch (ExecutionException e) {
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -50,14 +50,21 @@ public BrAPIGermplasmService(BrAPIListDAO brAPIListDAO, ProgramService programSe
}

public List<BrAPIGermplasm> getGermplasm(UUID programId) throws ApiException {
List<BrAPIGermplasm> germplasmList;
try {
return germplasmDAO.getGermplasm(programId);
} catch (ApiException e) {
throw new InternalServerException(e.getMessage(), e);
}
}

public BrAPIGermplasm getGermplasmByUUID(UUID programId, String germplasmId) throws DoesNotExistException {
try {
return germplasmDAO.getGermplasmByUUID(germplasmId, programId);
} catch (ApiException e) {
throw new InternalServerException(e.getMessage(), e);
}
}

public List<BrAPIListSummary> getGermplasmListsByProgramId(UUID programId, HttpRequest<String> request) throws DoesNotExistException, ApiException {

if (!programService.exists(programId)) {
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -20,6 +20,7 @@
import javax.inject.Inject;
import java.util.*;
import java.util.concurrent.Callable;
import java.util.stream.Collectors;

import static org.junit.jupiter.api.Assertions.*;
import static org.mockito.ArgumentMatchers.any;
Expand Down Expand Up @@ -57,10 +58,10 @@ void setupNextTest() {
}

@SneakyThrows
public List<BrAPIGermplasm> mockFetch(UUID programId, Integer sleepTime) {
public Map<String, BrAPIGermplasm> mockFetch(UUID programId, Integer sleepTime) {
fetchCount += 1;
Thread.sleep(sleepTime);
return mockBrAPI.containsKey(programId) ? new ArrayList<>(mockBrAPI.get(programId)) : new ArrayList<>();
return mockBrAPI.containsKey(programId) ? new HashMap<>(mockBrAPI.get(programId).stream().collect(Collectors.toMap(germplasm -> UUID.randomUUID().toString(), germplasm -> germplasm))) : new HashMap<>();
}

@SneakyThrows
Expand All @@ -78,7 +79,7 @@ public List<BrAPIGermplasm> mockPost(UUID programId, List<BrAPIGermplasm> germpl
@SneakyThrows
public void populatedRefreshQueueSkipsRefresh() {
// Make a lot of post calls and just how many times the fetch method is called
ProgramCache<BrAPIGermplasm> cache = new ProgramCache<>((UUID id) -> mockFetch(id, waitTime));
ProgramCache<String, BrAPIGermplasm> cache = new ProgramCache<>((UUID id) -> mockFetch(id, waitTime));
UUID programId = UUID.randomUUID();
int numPost = 10;
int currPost = 0;
Expand All @@ -91,15 +92,15 @@ public void populatedRefreshQueueSkipsRefresh() {
assertTrue(fetchCount < numPost, "A fetch call was made for every post. It shouldn't.");
assertEquals(1, mockBrAPI.size(), "More than one program existed in mocked brapi db.");
assertEquals(numPost, mockBrAPI.get(programId).size(), "Wrong number of germplasm in db");
List<BrAPIGermplasm> cachedGermplasm = cache.get(programId);
Map<String, BrAPIGermplasm> cachedGermplasm = cache.get(programId);
assertEquals(numPost, cachedGermplasm.size(), "Wrong number of germplasm in cache");
}

@Test
@SneakyThrows
public void programRefreshesSeparated() {
// Make a lot of post calls on different programs to check that they don't wait for each other
ProgramCache<BrAPIGermplasm> cache = new ProgramCache<>((UUID id) -> mockFetch(id, waitTime));
ProgramCache<String, BrAPIGermplasm> cache = new ProgramCache<>((UUID id) -> mockFetch(id, waitTime));
int numPost = 10;
int currPost = 0;
while (currPost < numPost) {
Expand All @@ -124,7 +125,7 @@ public void programRefreshesSeparated() {
public void initialGetMethodWaitsForLoad() {
// Test that the get method waits for an ongoing refresh to finish when there isn't any day
UUID programId = UUID.randomUUID();
ProgramCache<BrAPIGermplasm> cache = new ProgramCache<>((UUID id) -> mockFetch(id, waitTime), List.of(programId));
ProgramCache<String, BrAPIGermplasm> cache = new ProgramCache<>((UUID id) -> mockFetch(id, waitTime), List.of(programId));
cache.get(programId);
// Our fetch method should have only been called once for the initial loading
assertEquals(1, fetchCount, "Fetch method was called on get");
Expand All @@ -138,11 +139,11 @@ public void getMethodDoesNotWaitForRefresh() {
List<BrAPIGermplasm> newList = new ArrayList<>();
newList.add(new BrAPIGermplasm());
mockBrAPI.put(programId, new ArrayList<>(newList));
ProgramCache<BrAPIGermplasm> cache = new ProgramCache<>((UUID id) -> mockFetch(id, waitTime), List.of(programId));
ProgramCache<String, BrAPIGermplasm> cache = new ProgramCache<>((UUID id) -> mockFetch(id, waitTime), List.of(programId));
Callable<List<BrAPIGermplasm>> postFunction = () -> mockPost(programId, new ArrayList<>(newList));

// Get waits for initial fetch
List<BrAPIGermplasm> cachedGermplasm = cache.get(programId);
Map<String, BrAPIGermplasm> cachedGermplasm = cache.get(programId);
assertEquals(1, cachedGermplasm.size(), "Initial germplasm not as expected");

// Now post another object and call get immediately to see that it returns the old data
Expand Down Expand Up @@ -173,10 +174,10 @@ public void refreshErrorInvalidatesCache() {
ProgramCacheUnitTest mockTest = spy(this);

// Start cache
ProgramCache<BrAPIGermplasm> cache = new ProgramCache<>((UUID id) -> mockTest.mockFetch(programId, waitTime), List.of(programId));
ProgramCache<String, BrAPIGermplasm> cache = new ProgramCache<>((UUID id) -> mockTest.mockFetch(programId, waitTime), List.of(programId));

// Get waits for initial fetch
List<BrAPIGermplasm> cachedGermplasm = cache.get(programId);
Map<String, BrAPIGermplasm> cachedGermplasm = cache.get(programId);
assertEquals(1, cachedGermplasm.size(), "Initial germplasm not as expected");

// Change our fetch method to throw an error now
Expand Down