Skip to content

BI-1915 - Export Ontology File #310

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
davedrp merged 6 commits into from
Nov 18, 2023
Merged

BI-1915 - Export Ontology File #310

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
6 commits
Select commit Hold shift + click to select a range
38c6a65
[BI-1915] ontology export/download
davedrp Oct 27, 2023
fe5aed9
[BI-1915]fixed text for scale class DATE
davedrp Nov 1, 2023
86189b4
Merge branch 'develop' into feature/BI-1915
davedrp Nov 8, 2023
fe6eb47
[BI-1915] Fixed IntellaJ 'problems'
davedrp Nov 8, 2023
7582bd0
[BI-1915] only include caigories in ontology export, if the scale cla…
davedrp Nov 8, 2023
23dff7d
Merge branch 'develop' into feature/BI-1915
davedrp Nov 17, 2023
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
37 changes: 30 additions & 7 deletions src/main/java/org/breedinginsight/api/v1/controller/TraitController.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -17,10 +17,12 @@

package org.breedinginsight.api.v1.controller;

import io.micronaut.http.HttpHeaders;
import io.micronaut.http.HttpResponse;
import io.micronaut.http.HttpStatus;
import io.micronaut.http.MediaType;
import io.micronaut.http.annotation.*;
import io.micronaut.http.server.types.files.StreamedFile;
import lombok.extern.slf4j.Slf4j;
import org.breedinginsight.api.auth.*;
import org.breedinginsight.api.model.v1.request.query.QueryParams;
Expand All @@ -35,6 +37,8 @@
import org.breedinginsight.api.model.v1.validators.QueryValid;
import org.breedinginsight.api.model.v1.validators.SearchValid;
import org.breedinginsight.api.v1.controller.metadata.AddMetadata;
import org.breedinginsight.brapps.importer.model.exports.FileType;
import org.breedinginsight.model.DownloadFile;
import org.breedinginsight.model.Editable;
import org.breedinginsight.model.Program;
import org.breedinginsight.model.Trait;
Expand All @@ -52,16 +56,15 @@
import java.util.List;
import java.util.Optional;
import java.util.UUID;
import java.util.concurrent.atomic.AtomicReference;

@Slf4j
@Controller("/${micronaut.bi.api.version}")
public class TraitController {

private TraitService traitService;
private SecurityService securityService;
private TraitQueryMapper traitQueryMapper;
private OntologyService ontologyService;
private final TraitService traitService;
private final SecurityService securityService;
private final TraitQueryMapper traitQueryMapper;
private final OntologyService ontologyService;

@Inject
public TraitController(TraitService traitService, SecurityService securityService,
Expand All @@ -88,6 +91,26 @@ public HttpResponse<Response<DataResponse<Trait>>> getTraits(
}
}

@Get("/programs/{programId}/traits/export{?fileExtension,isActive}")
@Produces(value = "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet")
@ProgramSecured(roleGroups = {ProgramSecuredRoleGroup.ALL})
public HttpResponse getTraitsExport(
@PathVariable("programId") UUID programId, @QueryValue(defaultValue = "XLSX") String fileExtension, @QueryValue(defaultValue = "true") Boolean isActive) {
String downloadErrorMessage = "An error occurred while generating the download file. Contact the development team at bidevteam@cornell.edu.";
try {
FileType extension = Enum.valueOf(FileType.class, fileExtension);
DownloadFile ontologyFile = ontologyService.exportOntology(programId, extension, isActive);
HttpResponse<StreamedFile> ontologyExport = HttpResponse.ok(ontologyFile.getStreamedFile()).header(HttpHeaders.CONTENT_DISPOSITION, "attachment;filename="+ontologyFile.getFileName()+extension.getExtension());
return ontologyExport;
}
catch (Exception e) {
log.info(e.getMessage(), e);
e.printStackTrace();
HttpResponse response = HttpResponse.status(HttpStatus.INTERNAL_SERVER_ERROR, downloadErrorMessage).contentType(MediaType.TEXT_PLAIN).body(downloadErrorMessage);
return response;
}
}

@Post("/programs/{programId}/traits/search{?queryParams*}")
@Produces(MediaType.APPLICATION_JSON)
@ProgramSecured(roleGroups = {ProgramSecuredRoleGroup.ALL})
Expand Down Expand Up @@ -140,7 +163,7 @@ public HttpResponse<Response<Editable>> getTraitEditable(@PathVariable UUID prog
@Post("/programs/{programId}/traits")
@Produces(MediaType.APPLICATION_JSON)
@ProgramSecured(roles = {ProgramSecuredRole.BREEDER})
public HttpResponse<Response<DataResponse<Trait>>> createTraits(@PathVariable UUID programId, @Body @Valid List<Trait> traits) {
public HttpResponse createTraits(@PathVariable UUID programId, @Body @Valid List<Trait> traits) {
AuthenticatedUser actingUser = securityService.getUser();
try {
List<Trait> createdTraits = ontologyService.createTraits(programId, traits, actingUser, true);
Expand Down Expand Up @@ -168,7 +191,7 @@ public HttpResponse<Response<DataResponse<Trait>>> createTraits(@PathVariable UU
@Put("/programs/{programId}/traits")
@Produces(MediaType.APPLICATION_JSON)
@ProgramSecured(roles = {ProgramSecuredRole.BREEDER})
public HttpResponse<Response<DataResponse<Trait>>> updateTraits(@PathVariable UUID programId, @Body @Valid List<Trait> traits) {
public HttpResponse updateTraits(@PathVariable UUID programId, @Body @Valid List<Trait> traits) {
AuthenticatedUser actingUser = securityService.getUser();
try {
List<Trait> updatedTraits = ontologyService.updateTraits(programId, traits, actingUser);
Expand Down
34 changes: 34 additions & 0 deletions src/main/java/org/breedinginsight/brapps/importer/model/imports/DataTypeTranslator.java
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,34 @@
package org.breedinginsight.brapps.importer.model.imports;

import org.breedinginsight.dao.db.enums.DataType;

import java.util.Map;
import java.util.Optional;

public class DataTypeTranslator {

private static final Map<String, DataType> displayToTypeMap = Map.of(
"Date", DataType.DATE,
"Numerical", DataType.NUMERICAL,
"Nominal", DataType.NOMINAL,
"Ordinal", DataType.ORDINAL,
"Text", DataType.TEXT);


public static Optional<DataType> getTypeFromUserDisplayName(String displayName) {
return Optional.ofNullable(displayToTypeMap.get(displayName));
}

public static String getDisplayNameFromType(DataType type){
String displayName = "";
if(type!=null) {
for (String key : displayToTypeMap.keySet()) {
if (type == displayToTypeMap.get(key)) {
displayName = key;
break;
}
}
}
return displayName;
}
}
13 changes: 13 additions & 0 deletions src/main/java/org/breedinginsight/brapps/importer/model/imports/TermTypeTranslator.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -14,4 +14,17 @@ public class TermTypeTranslator {
public static Optional<TermType> getTermTypeFromUserDisplayName(String userDisplayName) {
return Optional.ofNullable(userDisplayToTermTypeMap.get(userDisplayName));
}

public static String getDisplayNameFromTermType(TermType termType){
String displayName = "";
if(termType!=null) {
for (String key : userDisplayToTermTypeMap.keySet()) {
if (termType == userDisplayToTermTypeMap.get(key)) {
displayName = key;
break;
}
}
}
return displayName;
}
}
157 changes: 143 additions & 14 deletions src/main/java/org/breedinginsight/services/OntologyService.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -4,32 +4,47 @@
import io.micronaut.http.exceptions.HttpStatusException;
import io.micronaut.http.multipart.CompletedFileUpload;
import io.micronaut.http.server.exceptions.InternalServerException;
import io.micronaut.http.server.types.files.StreamedFile;
import org.apache.commons.lang3.StringUtils;
import org.brapi.v2.model.core.BrAPIProgram;
import org.brapi.v2.model.pheno.BrAPIScaleValidValuesCategories;
import org.breedinginsight.api.auth.AuthenticatedUser;
import org.breedinginsight.api.model.v1.request.SharedOntologyProgramRequest;
import org.breedinginsight.api.model.v1.response.ValidationError;
import org.breedinginsight.api.model.v1.response.ValidationErrors;
import org.breedinginsight.brapps.importer.model.exports.FileType;
import org.breedinginsight.brapps.importer.model.imports.DataTypeTranslator;
import org.breedinginsight.brapps.importer.model.imports.TermTypeTranslator;
import org.breedinginsight.dao.db.enums.DataType;
import org.breedinginsight.dao.db.tables.pojos.ProgramSharedOntologyEntity;
import org.breedinginsight.dao.db.tables.pojos.TraitEntity;
import org.breedinginsight.daos.ProgramDAO;
import org.breedinginsight.daos.ProgramOntologyDAO;
import org.breedinginsight.daos.TraitDAO;
import org.breedinginsight.model.*;
import org.breedinginsight.services.exceptions.*;
import org.breedinginsight.services.parsers.trait.TraitFileColumns;
import org.breedinginsight.services.writers.CSVWriter;
import org.breedinginsight.services.writers.ExcelWriter;

import org.jooq.exception.IOException;

import javax.inject.Inject;
import javax.inject.Singleton;
import javax.validation.constraints.NotNull;
import java.time.OffsetDateTime;
import java.time.format.DateTimeFormatter;
import java.util.*;
import java.util.stream.Collectors;

@Singleton
public class OntologyService {

private ProgramDAO programDAO;
private ProgramOntologyDAO programOntologyDAO;
private TraitDAO traitDAO;
private TraitService traitService;
private TraitUploadService traitUploadService;
private final ProgramDAO programDAO;
private final ProgramOntologyDAO programOntologyDAO;
private final TraitDAO traitDAO;
private final TraitService traitService;
private final TraitUploadService traitUploadService;

@Inject
public OntologyService(ProgramDAO programDAO, ProgramOntologyDAO programOntologyDAO, TraitDAO traitDAO, TraitService traitService, TraitUploadService traitUploadService) {
Expand All @@ -46,7 +61,7 @@ public OntologyService(ProgramDAO programDAO, ProgramOntologyDAO programOntology
* @param sharedOnly -- True = return only shared programs, False = get all shareable programs
* @return List<SharedOntologyProgram>
*/
public List getSharedOntology(@NotNull UUID programId, @NotNull Boolean sharedOnly) throws DoesNotExistException {
public List<SharedOntology> getSharedOntology(@NotNull UUID programId, @NotNull Boolean sharedOnly) throws DoesNotExistException {

// Get program with that id
Program program = getProgram(programId);
Expand Down Expand Up @@ -87,7 +102,7 @@ public List getSharedOntology(@NotNull UUID programId, @NotNull Boolean sharedOn
unsharableIds.addAll(shareTargetIds);
formattedPrograms.addAll(matchingPrograms.stream()
.filter(matchingProgram -> !unsharableIds.contains(matchingProgram.getId()))
.map(matchingProgram -> formatResponse(matchingProgram))
.map(this::formatResponse)
.collect(Collectors.toList()));
}

Expand All @@ -102,7 +117,7 @@ private List<SharedOntology> getSharedProgramsFormatted(Program program) {
sharedOntologies.stream().map(ProgramSharedOntologyEntity::getSharedProgramId).collect(Collectors.toList()));
// Get the programs in a lookup map
Map<UUID, Program> sharedProgramsMap = new HashMap<>();
sharedPrograms.stream().forEach(sharedProgram -> sharedProgramsMap.put(sharedProgram.getId(), sharedProgram));
sharedPrograms.forEach(sharedProgram -> sharedProgramsMap.put(sharedProgram.getId(), sharedProgram));

// Format shared programs response
return sharedOntologies.stream().map(sharedOntology ->
Expand Down Expand Up @@ -157,7 +172,7 @@ private Boolean ontologyIsEditable(ProgramSharedOntologyEntity sharedOntologyEnt
if (sharedOntologyEntity.getActive()) {
// Get all trait ids for the program
List<UUID> traitIds = traitService.getSubscribedOntologyTraits(sharedOntologyEntity.getSharedProgramId()).stream()
.map(trait -> trait.getId())
.map(TraitEntity::getId)
.collect(Collectors.toList());

// Get the brapi program id
Expand Down Expand Up @@ -203,7 +218,7 @@ public List<SharedOntology> shareOntology(@NotNull UUID programId, Authenticated
// Check shareability, same brapi server, same species
List<Program> matchingPrograms = getMatchingPrograms(program);
Set<UUID> matchingProgramsSet = new HashSet<>();
matchingPrograms.stream().forEach(matchingProgram -> matchingProgramsSet.add(matchingProgram.getId()));
matchingPrograms.forEach(matchingProgram -> matchingProgramsSet.add(matchingProgram.getId()));

Set<UUID> shareProgramIdsSet = new HashSet<>();
ValidationErrors validationErrors = new ValidationErrors();
Expand Down Expand Up @@ -249,7 +264,7 @@ public List<SharedOntology> shareOntology(@NotNull UUID programId, Authenticated
*/
public void revokeOntology(@NotNull UUID programId, @NotNull UUID sharedProgramId) throws UnprocessableEntityException, DoesNotExistException {
// Check that program exists
Program program = getProgram(programId);
getProgram(programId);

// Check that shared program exists
Optional<ProgramSharedOntologyEntity> optionalSharedOntology = programOntologyDAO.getSharedOntologyById(programId, sharedProgramId);
Expand Down Expand Up @@ -311,7 +326,7 @@ public SubscribedOntology subscribeOntology(UUID programId, UUID sharingProgramI

public void unsubscribeOntology(UUID programId, UUID sharingProgramId) throws DoesNotExistException, UnprocessableEntityException {
// Check that program exists
Program program = getProgram(programId);
getProgram(programId);

// Check that shared program exists
Optional<ProgramSharedOntologyEntity> optionalSharedOntology = programOntologyDAO.getSharedOntologyById(sharingProgramId, programId);
Expand Down Expand Up @@ -342,6 +357,47 @@ public List<Trait> updateTraits(UUID programId, List<Trait> traits, Authenticate
return traitService.updateTraits(lookupId, traits, actingUser);
}


public DownloadFile exportOntology(UUID programId, FileType fileExtension, boolean isActive) throws IllegalArgumentException, IOException, java.io.IOException {
List<Column> columns = TraitFileColumns.getOrderedColumns();

//Retrieve trait list data
List<Trait> traits = traitDAO.getTraitsFullByProgramId(programId);
//Filter traits for Active or Archived
traits = traits.stream().filter(trait -> trait.getActive()==isActive).collect(Collectors.toList());
//Sort list in default (trait name) order.
traits.sort(Comparator.comparing(trait -> trait.getObservationVariableName().toLowerCase()));

// make file Name
String fileName = makeFileName(programId, isActive);

StreamedFile downloadFile;

//Convert traits list to List<Map<String, Object>> data to pass into file writer
List<Map<String, Object>> processedData = processData(traits);

if (fileExtension == FileType.CSV){
downloadFile = CSVWriter.writeToDownload(columns, processedData, fileExtension);
} else {
downloadFile = ExcelWriter.writeToDownload("Data", columns, processedData, fileExtension);
}
return new DownloadFile(fileName, downloadFile);
}

private String makeFileName(UUID programId, boolean isActive) {
DateTimeFormatter formatter = DateTimeFormatter.ofPattern("yyyy-MM-dd:hh-mm-ssZ");
String timestamp = formatter.format(OffsetDateTime.now());
Program program = null;
try {
program = getProgram(programId);
} catch (DoesNotExistException e) {
e.printStackTrace();
}
String activeOrArchive = isActive ? "Active" : "Archive";
String programName = program==null? "program" : program.getName();
return programName + "_" + activeOrArchive + "_Ontology_" + timestamp;
}

public List<Trait> createTraits(UUID programId, List<Trait> traits, AuthenticatedUser actingUser, Boolean throwDuplicateErrors)
throws DoesNotExistException, ValidatorException, BadRequestException {
UUID lookupId = getSubscribedOntologyProgramId(programId);
Expand Down Expand Up @@ -390,8 +446,8 @@ public UUID getSubscribedOntologyProgramId(UUID programId) throws DoesNotExistEx
}

public List<SubscribedOntology> getSubscribeOntologyOptions(UUID programId) throws DoesNotExistException {

Program program = getProgram(programId);
// Check that program exists
getProgram(programId);

List<ProgramSharedOntologyEntity> sharedOntologies = programOntologyDAO.getSubscriptionOptions(programId);
List<Program> programs = programDAO.get(sharedOntologies.stream().map(ProgramSharedOntologyEntity::getProgramId).collect(Collectors.toList()));
Expand All @@ -408,5 +464,78 @@ public List<SubscribedOntology> getSubscribeOntologyOptions(UUID programId) thro
).collect(Collectors.toList());
return subscriptionOptions;
}
public List<Map<String, Object>> processData(List<Trait> traits) {
List<Map<String, Object>> processedData = new ArrayList<>();

for (Trait trait : traits) {
HashMap<String, Object> row = new HashMap<>();
row.put(TraitFileColumns.NAME.toString(), trait.getObservationVariableName());
row.put(TraitFileColumns.FULL_NAME.toString(), trait.getFullName());
row.put(TraitFileColumns.TERM_TYPE.toString(), TermTypeTranslator.getDisplayNameFromTermType( trait.getTermType() ));
row.put(TraitFileColumns.DESCRIPTION.toString(), trait.getTraitDescription());
//SYNONYMS
String synonymsAsStr = null;
if(trait.getSynonyms() != null) {
synonymsAsStr = String.join("; ", trait.getSynonyms());
}
row.put(TraitFileColumns.SYNONYMS.toString(), synonymsAsStr);
//STATUS
if(trait.getActive()) {
row.put(TraitFileColumns.STATUS.toString(), "active");
} else {
row.put(TraitFileColumns.STATUS.toString(), "archived");

}
//TAGS
String tagsAsStr = null;
if(trait.getTags() != null) {
tagsAsStr = String.join("; ", trait.getTags());
}
row.put(TraitFileColumns.TAGS.toString(), tagsAsStr);

row.put(TraitFileColumns.TRAIT_ENTITY.toString(), trait.getEntity());
row.put(TraitFileColumns.TRAIT_ATTRIBUTE.toString(), trait.getAttribute());
Method method = trait.getMethod();
if(method!=null) {
row.put(TraitFileColumns.METHOD_DESCRIPTION.toString(), method.getDescription());
row.put(TraitFileColumns.METHOD_CLASS.toString(), method.getMethodClass());
row.put(TraitFileColumns.METHOD_FORMULA.toString(), method.getFormula());
}
Scale scale = trait.getScale();
if(scale!=null) {

row.put(TraitFileColumns.SCALE_CLASS.toString(), DataTypeTranslator.getDisplayNameFromType(scale.getDataType()));
row.put(TraitFileColumns.SCALE_NAME.toString(), scale.getScaleName());
row.put(TraitFileColumns.SCALE_DECIMAL_PLACES.toString(), scale.getDecimalPlaces());
row.put(TraitFileColumns.SCALE_LOWER_LIMIT.toString(), scale.getValidValueMin());
row.put(TraitFileColumns.SCALE_UPPER_LIMIT.toString(), scale.getValidValueMax());
//SCALE_CATEGORIES
String categoriesAsStr = makeCategoriesString(scale);

row.put(TraitFileColumns.SCALE_CATEGORIES.toString(), categoriesAsStr);
}
processedData.add(row);
}
return processedData;
}

private String makeCategoriesString(Scale scale) {
String categoriesAsStr = null;
if(scale.getCategories() != null &&
(scale.getDataType()==DataType.ORDINAL || scale.getDataType() == DataType.NOMINAL) ) {
categoriesAsStr = scale.getCategories().stream()
.map(cat -> makeCategoryString(cat, scale.getDataType()) )
.collect(Collectors.joining("; "));
}
return categoriesAsStr;
}

private String makeCategoryString(BrAPIScaleValidValuesCategories category, DataType scaleClass){
StringBuilder stringBuilder = new StringBuilder( category.getValue() );
if( StringUtils.isNotBlank( category.getLabel() ) && scaleClass == DataType.ORDINAL ){
stringBuilder.append("=");
stringBuilder.append( category.getLabel() );
}
return stringBuilder.toString();
}
}
Loading