From 1941b14a40dc3e2ae8c22cb9513358c64d6762f3 Mon Sep 17 00:00:00 2001 From: Jim Myers Date: Wed, 10 Nov 2021 11:00:10 -0500 Subject: [PATCH 01/16] add list/delete, remove tabular restriction, improve error handling --- .../dataverse/AuxiliaryFileServiceBean.java | 74 +++++++++++--- .../edu/harvard/iq/dataverse/api/Access.java | 99 +++++++++++++++++-- 2 files changed, 149 insertions(+), 24 deletions(-) diff --git a/src/main/java/edu/harvard/iq/dataverse/AuxiliaryFileServiceBean.java b/src/main/java/edu/harvard/iq/dataverse/AuxiliaryFileServiceBean.java index 0643c3622d4..59c06b6e3c3 100644 --- a/src/main/java/edu/harvard/iq/dataverse/AuxiliaryFileServiceBean.java +++ b/src/main/java/edu/harvard/iq/dataverse/AuxiliaryFileServiceBean.java @@ -4,10 +4,13 @@ import edu.harvard.iq.dataverse.dataaccess.StorageIO; import edu.harvard.iq.dataverse.util.FileUtil; import edu.harvard.iq.dataverse.util.SystemConfig; + +import java.io.FileNotFoundException; import java.io.IOException; import java.io.InputStream; import java.security.DigestInputStream; import java.security.MessageDigest; +import java.security.NoSuchAlgorithmException; import java.util.ArrayList; import java.util.List; import java.util.logging.Logger; @@ -18,6 +21,10 @@ import javax.persistence.PersistenceContext; import javax.persistence.Query; import javax.persistence.TypedQuery; +import javax.ws.rs.ClientErrorException; +import javax.ws.rs.InternalServerErrorException; +import javax.ws.rs.core.Response; + import org.apache.tika.Tika; /** @@ -62,8 +69,8 @@ public AuxiliaryFile save(AuxiliaryFile auxiliaryFile) { * @return success boolean - returns whether the save was successful */ public AuxiliaryFile processAuxiliaryFile(InputStream fileInputStream, DataFile dataFile, String formatTag, String formatVersion, String origin, boolean isPublic, String type) { - - StorageIO storageIO =null; + + StorageIO storageIO = null; AuxiliaryFile auxFile = new AuxiliaryFile(); String auxExtension = formatTag + "_" + formatVersion; try { @@ -73,12 +80,20 @@ public AuxiliaryFile processAuxiliaryFile(InputStream fileInputStream, DataFile // If the db fails for any reason, then rollback // by removing the auxfile from storage. storageIO = dataFile.getStorageIO(); - MessageDigest md = MessageDigest.getInstance(systemConfig.getFileFixityChecksumAlgorithm().toString()); - DigestInputStream di - = new DigestInputStream(fileInputStream, md); - - storageIO.saveInputStreamAsAux(fileInputStream, auxExtension); - auxFile.setChecksum(FileUtil.checksumDigestToString(di.getMessageDigest().digest()) ); + if (storageIO.isAuxObjectCached(auxExtension)) { + throw new ClientErrorException("Auxiliary file already exists", Response.Status.CONFLICT); + } + MessageDigest md; + try { + md = MessageDigest.getInstance(systemConfig.getFileFixityChecksumAlgorithm().toString()); + } catch (NoSuchAlgorithmException e) { + logger.severe("NoSuchAlgorithmException for system fixity algorithm: " + systemConfig.getFileFixityChecksumAlgorithm().toString()); + throw new InternalServerErrorException(); + } + DigestInputStream di = new DigestInputStream(fileInputStream, md); + + storageIO.saveInputStreamAsAux(fileInputStream, auxExtension); + auxFile.setChecksum(FileUtil.checksumDigestToString(di.getMessageDigest().digest())); Tika tika = new Tika(); auxFile.setContentType(tika.detect(storageIO.getAuxFileAsInputStream(auxExtension))); @@ -87,20 +102,20 @@ public AuxiliaryFile processAuxiliaryFile(InputStream fileInputStream, DataFile auxFile.setOrigin(origin); auxFile.setIsPublic(isPublic); auxFile.setType(type); - auxFile.setDataFile(dataFile); + auxFile.setDataFile(dataFile); auxFile.setFileSize(storageIO.getAuxObjectSize(auxExtension)); auxFile = save(auxFile); } catch (IOException ioex) { - logger.info("IO Exception trying to save auxiliary file: " + ioex.getMessage()); - return null; - } catch (Exception e) { + logger.severe("IO Exception trying to save auxiliary file: " + ioex.getMessage()); + throw new InternalServerErrorException(); + } catch (RuntimeException e) { // If anything fails during database insert, remove file from storage try { storageIO.deleteAuxObject(auxExtension); - } catch(IOException ioex) { - logger.info("IO Exception trying remove auxiliary file in exception handler: " + ioex.getMessage()); - return null; + } catch (IOException ioex) { + logger.warning("IO Exception trying remove auxiliary file in exception handler: " + ioex.getMessage()); } + throw e; } return auxFile; } @@ -119,6 +134,35 @@ public AuxiliaryFile lookupAuxiliaryFile(DataFile dataFile, String formatTag, St return null; } } + + @SuppressWarnings("unchecked") + public List listAuxiliaryFiles(DataFile dataFile, String origin) { + + Query query = em.createQuery("select object(o) from AuxiliaryFile as o where o.dataFile.id = :dataFileId and o.origin = :origin"); + + query.setParameter("dataFileId", dataFile.getId()); + query.setParameter("origin", origin); + try { + List retVal = (List)query.getResultList(); + return retVal; + } catch(Exception ex) { + return null; + } + } + + public void deleteAuxiliaryFile(DataFile dataFile, String formatTag, String formatVersion) throws IOException { + AuxiliaryFile af = lookupAuxiliaryFile(dataFile, formatTag, formatVersion); + if (af == null) { + throw new FileNotFoundException(); + } + em.remove(af); + StorageIO storageIO; + storageIO = dataFile.getStorageIO(); + String auxExtension = formatTag + "_" + formatVersion; + if (storageIO.isAuxObjectCached(auxExtension)) { + storageIO.deleteAuxObject(auxExtension); + } + } public List findAuxiliaryFiles(DataFile dataFile) { TypedQuery query = em.createNamedQuery("AuxiliaryFile.findAuxiliaryFiles", AuxiliaryFile.class); diff --git a/src/main/java/edu/harvard/iq/dataverse/api/Access.java b/src/main/java/edu/harvard/iq/dataverse/api/Access.java index 71a4443ecc9..893e0eda839 100644 --- a/src/main/java/edu/harvard/iq/dataverse/api/Access.java +++ b/src/main/java/edu/harvard/iq/dataverse/api/Access.java @@ -74,6 +74,7 @@ import java.io.ByteArrayOutputStream; import java.io.File; import java.io.FileInputStream; +import java.io.FileNotFoundException; import java.io.IOException; import java.io.OutputStream; import java.sql.Timestamp; @@ -119,6 +120,7 @@ import java.util.stream.Stream; import javax.json.JsonObjectBuilder; import javax.ws.rs.RedirectionException; +import javax.ws.rs.ServerErrorException; import javax.ws.rs.core.MediaType; import static javax.ws.rs.core.Response.Status.FORBIDDEN; import org.glassfish.jersey.media.multipart.FormDataParam; @@ -538,6 +540,49 @@ public String dataVariableMetadataDDI(@PathParam("varId") Long varId, @QueryPara return retValue; } + + /* + * GET method for retrieving various auxiliary files associated with + * a tabular datafile. + */ + + @Path("datafile/{fileId}/auxiliary/{origin}") + @GET + public Response listDatafileMetadataAux(@PathParam("fileId") String fileId, + @PathParam("origin") String origin, + @QueryParam("key") String apiToken, + @Context UriInfo uriInfo, + @Context HttpHeaders headers, + @Context HttpServletResponse response) throws ServiceUnavailableException { + + DataFile df = findDataFileOrDieWrapper(fileId); + + if (apiToken == null || apiToken.equals("")) { + apiToken = headers.getHeaderString(API_KEY_HEADER); + } + + List auxFileList = auxiliaryFileService.listAuxiliaryFiles(df, origin); + + if (auxFileList == null || auxFileList.isEmpty()) { + throw new NotFoundException("No Auxiliary files exist for datafile " + fileId + " and the specified origin"); + } + boolean isAccessAllowed = isAccessAuthorized(df, apiToken); + JsonArrayBuilder jab = Json.createArrayBuilder(); + auxFileList.forEach(auxFile -> { + if (isAccessAllowed || auxFile.getIsPublic()) { + JsonObjectBuilder job = Json.createObjectBuilder(); + job.add("formatTag", auxFile.getFormatTag()); + job.add("formatVersion", auxFile.getFormatVersion()); + job.add("fileSize", auxFile.getFileSize()); + job.add("contentType", auxFile.getContentType()); + job.add("isPublic", auxFile.getIsPublic()); + job.add("type", auxFile.getType()); + jab.add(job); + } + }); + return ok(jab); + } + /* * GET method for retrieving various auxiliary files associated with * a tabular datafile. @@ -563,10 +608,6 @@ public DownloadInstance downloadAuxiliaryFile(@PathParam("fileId") String fileId DownloadInfo dInfo = new DownloadInfo(df); boolean publiclyAvailable = false; - if (!df.isTabularData()) { - throw new BadRequestException("tabular data required"); - } - DownloadInstance downloadInstance; AuxiliaryFile auxFile = null; @@ -1248,12 +1289,8 @@ public Response saveAuxiliaryFileWithVersion(@PathParam("fileId") Long fileId, return error(FORBIDDEN, "User not authorized to edit the dataset."); } - if (!dataFile.isTabularData()) { - return error(BAD_REQUEST, "Not a tabular DataFile (db id=" + fileId + ")"); - } - AuxiliaryFile saved = auxiliaryFileService.processAuxiliaryFile(fileInputStream, dataFile, formatTag, formatVersion, origin, isPublic, type); - + if (saved!=null) { return ok(json(saved)); } else { @@ -1263,6 +1300,50 @@ public Response saveAuxiliaryFileWithVersion(@PathParam("fileId") Long fileId, + /** + * + * @param fileId + * @param formatTag + * @param formatVersion + * @param origin + * @param isPublic + * @param fileInputStream + * @param contentDispositionHeader + * @param formDataBodyPart + * @return + */ + @Path("datafile/{fileId}/auxiliary/{formatTag}/{formatVersion}") + @DELETE + public Response deleteAuxiliaryFileWithVersion(@PathParam("fileId") Long fileId, + @PathParam("formatTag") String formatTag, + @PathParam("formatVersion") String formatVersion) { + AuthenticatedUser authenticatedUser; + try { + authenticatedUser = findAuthenticatedUserOrDie(); + } catch (WrappedResponse ex) { + return error(FORBIDDEN, "Authorized users only."); + } + + DataFile dataFile = dataFileService.find(fileId); + if (dataFile == null) { + return error(BAD_REQUEST, "File not found based on id " + fileId + "."); + } + + if (!permissionService.userOn(authenticatedUser, dataFile.getOwner()).has(Permission.EditDataset)) { + return error(FORBIDDEN, "User not authorized to edit the dataset."); + } + + try { + auxiliaryFileService.deleteAuxiliaryFile(dataFile, formatTag, formatVersion); + } catch (FileNotFoundException e) { + throw new NotFoundException(); + } catch(IOException io) { + throw new ServerErrorException("IO Exception trying remove auxiliary file", Response.Status.INTERNAL_SERVER_ERROR, io); + } + + return ok("Auxiliary file deleted."); + } + /** From d179fa4838c5835f2da1db3d4bce7c5358c1d6b5 Mon Sep 17 00:00:00 2001 From: Jim Myers Date: Wed, 10 Nov 2021 11:17:00 -0500 Subject: [PATCH 02/16] document --- .../source/developers/aux-file-support.rst | 36 +++++++++++++++++-- 1 file changed, 34 insertions(+), 2 deletions(-) diff --git a/doc/sphinx-guides/source/developers/aux-file-support.rst b/doc/sphinx-guides/source/developers/aux-file-support.rst index 42b9aebf975..33e3dbaa444 100644 --- a/doc/sphinx-guides/source/developers/aux-file-support.rst +++ b/doc/sphinx-guides/source/developers/aux-file-support.rst @@ -21,7 +21,7 @@ To add an auxiliary file, specify the primary key of the datafile (FILE_ID), and You should expect a 200 ("OK") response and JSON with information about your newly uploaded auxiliary file. -Downloading an Auxiliary File that belongs to a Datafile +Downloading an Auxiliary File that belongs to a Datafile -------------------------------------------------------- To download an auxiliary file, use the primary key of the datafile, and the formatTag and formatVersion (if applicable) associated with the auxiliary file: @@ -33,5 +33,37 @@ formatTag and formatVersion (if applicable) associated with the auxiliary file: export FILE_ID='12345' export FORMAT_TAG='dpJson' export FORMAT_VERSION='v1' - + curl "$SERVER_URL/api/access/datafile/$FILE_ID/auxiliary/$FORMAT_TAG/$FORMAT_VERSION" + +Listing Auxiliary Files for a Datafile by Origin +------------------------------------------------ +To list auxiliary files, specify the primary key of the datafile (FILE_ID), and the origin associated with the auxiliary files to list (the application/entity that created them). + +.. code-block:: bash + + export API_TOKEN=xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx + export FILE_ID='12345' + export SERVER_URL=https://demo.dataverse.org + export ORIGIN='app1' + + curl "$SERVER_URL/api/access/datafile/$FILE_ID/auxiliary/$ORIGIN" + +You should expect a 200 ("OK") response and a JSON array with objects representing the auxiliary files found, or a 404 /Not Found response if no auxiliary files exist with that origin. + +Deleting an Auxiliary File that belongs to a Datafile +----------------------------------------------------- +To delete an auxiliary file, use the primary key of the datafile, and the +formatTag and formatVersion (if applicable) associated with the auxiliary file: + +.. code-block:: bash + + export API_TOKEN=xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx + export SERVER_URL=https://demo.dataverse.org + export FILE_ID='12345' + export FORMAT_TAG='dpJson' + export FORMAT_VERSION='v1' + + curl -X DELETE "$SERVER_URL/api/access/datafile/$FILE_ID/auxiliary/$FORMAT_TAG/$FORMAT_VERSION" + + \ No newline at end of file From db9e08a03598a3f1ba02140e99c416d1501e1ee1 Mon Sep 17 00:00:00 2001 From: Jim Myers Date: Wed, 10 Nov 2021 11:21:53 -0500 Subject: [PATCH 03/16] release note draft --- doc/release-notes/8235-auxiliaryfileAPIenhancements.md | 9 +++++++++ 1 file changed, 9 insertions(+) create mode 100644 doc/release-notes/8235-auxiliaryfileAPIenhancements.md diff --git a/doc/release-notes/8235-auxiliaryfileAPIenhancements.md b/doc/release-notes/8235-auxiliaryfileAPIenhancements.md new file mode 100644 index 00000000000..bcd43963fac --- /dev/null +++ b/doc/release-notes/8235-auxiliaryfileAPIenhancements.md @@ -0,0 +1,9 @@ +### Auxiliary File API Enhancements + +This release includes updates to the Auxiliary File API: +- Auxiliary files can now also be associated with non-tabular files +- Improved error reporting +- The API will block attempts to create a duplicate auxiliary file +- Delete and list-by-original calls have been added + +Please note that the auxiliary files feature is experimental and is designed to support integration with tools from the [OpenDP Project](https://opendp.org). If the API endpoints are not needed they can be blocked. From 7464a1ddc6d092b55a6611595194987153c9aa1e Mon Sep 17 00:00:00 2001 From: Danny Brooke Date: Mon, 15 Nov 2021 10:36:46 -0500 Subject: [PATCH 04/16] adding major use cases reminder --- doc/release-notes/8235-auxiliaryfileAPIenhancements.md | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/doc/release-notes/8235-auxiliaryfileAPIenhancements.md b/doc/release-notes/8235-auxiliaryfileAPIenhancements.md index bcd43963fac..6c02c3d0e93 100644 --- a/doc/release-notes/8235-auxiliaryfileAPIenhancements.md +++ b/doc/release-notes/8235-auxiliaryfileAPIenhancements.md @@ -7,3 +7,7 @@ This release includes updates to the Auxiliary File API: - Delete and list-by-original calls have been added Please note that the auxiliary files feature is experimental and is designed to support integration with tools from the [OpenDP Project](https://opendp.org). If the API endpoints are not needed they can be blocked. + +### Major Use Cases + +(note for release time - expand on the items above, as use cases) \ No newline at end of file From 871c83c7abf6b8c2231d36444442c5a607549206 Mon Sep 17 00:00:00 2001 From: Philip Durbin Date: Tue, 16 Nov 2021 10:40:46 -0500 Subject: [PATCH 05/16] link to OpenDP website #8235 --- doc/sphinx-guides/source/developers/aux-file-support.rst | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/doc/sphinx-guides/source/developers/aux-file-support.rst b/doc/sphinx-guides/source/developers/aux-file-support.rst index 33e3dbaa444..13433711c48 100644 --- a/doc/sphinx-guides/source/developers/aux-file-support.rst +++ b/doc/sphinx-guides/source/developers/aux-file-support.rst @@ -1,7 +1,7 @@ Auxiliary File Support ====================== -Auxiliary file support is experimental and as such, related APIs may be added, changed or removed without standard backward compatibility. Auxiliary files in the Dataverse Software are being added to support depositing and downloading differentially private metadata, as part of the OpenDP project (opendp.org). In future versions, this approach will likely become more broadly used and supported. +Auxiliary file support is experimental and as such, related APIs may be added, changed or removed without standard backward compatibility. Auxiliary files in the Dataverse Software are being added to support depositing and downloading differentially private metadata, as part of the `OpenDP project `_. In future versions, this approach will likely become more broadly used and supported. Adding an Auxiliary File to a Datafile -------------------------------------- @@ -66,4 +66,4 @@ formatTag and formatVersion (if applicable) associated with the auxiliary file: curl -X DELETE "$SERVER_URL/api/access/datafile/$FILE_ID/auxiliary/$FORMAT_TAG/$FORMAT_VERSION" - \ No newline at end of file + From 59d0cbb958e000560b2fff8f08c0defaf8f11a33 Mon Sep 17 00:00:00 2001 From: Jim Myers Date: Tue, 16 Nov 2021 14:36:03 -0500 Subject: [PATCH 06/16] re-add "list aux files by origin" method dropped by dedd9f8 merge #8235 --- .../edu/harvard/iq/dataverse/api/Access.java | 42 +++++++++++++++++++ 1 file changed, 42 insertions(+) diff --git a/src/main/java/edu/harvard/iq/dataverse/api/Access.java b/src/main/java/edu/harvard/iq/dataverse/api/Access.java index 94e4f62cad4..2c0d8e14659 100644 --- a/src/main/java/edu/harvard/iq/dataverse/api/Access.java +++ b/src/main/java/edu/harvard/iq/dataverse/api/Access.java @@ -512,6 +512,48 @@ public String tabularDatafileMetadataDDI(@PathParam("fileId") String fileId, @Q return retValue; } + + /* + * GET method for retrieving various auxiliary files associated with + * a tabular datafile. + */ + @Path("datafile/{fileId}/auxiliary/{origin}") + @GET + public Response listDatafileMetadataAux(@PathParam("fileId") String fileId, + @PathParam("origin") String origin, + @QueryParam("key") String apiToken, + @Context UriInfo uriInfo, + @Context HttpHeaders headers, + @Context HttpServletResponse response) throws ServiceUnavailableException { + + DataFile df = findDataFileOrDieWrapper(fileId); + + if (apiToken == null || apiToken.equals("")) { + apiToken = headers.getHeaderString(API_KEY_HEADER); + } + + List auxFileList = auxiliaryFileService.listAuxiliaryFiles(df, origin); + + if (auxFileList == null || auxFileList.isEmpty()) { + throw new NotFoundException("No Auxiliary files exist for datafile " + fileId + " and the specified origin"); + } + boolean isAccessAllowed = isAccessAuthorized(df, apiToken); + JsonArrayBuilder jab = Json.createArrayBuilder(); + auxFileList.forEach(auxFile -> { + if (isAccessAllowed || auxFile.getIsPublic()) { + JsonObjectBuilder job = Json.createObjectBuilder(); + job.add("formatTag", auxFile.getFormatTag()); + job.add("formatVersion", auxFile.getFormatVersion()); + job.add("fileSize", auxFile.getFileSize()); + job.add("contentType", auxFile.getContentType()); + job.add("isPublic", auxFile.getIsPublic()); + job.add("type", auxFile.getType()); + jab.add(job); + } + }); + return ok(jab); + } + /* * GET method for retrieving various auxiliary files associated with * a tabular datafile. From b66fb55a7288008969217cf6209aaefe27822945 Mon Sep 17 00:00:00 2001 From: Philip Durbin Date: Tue, 16 Nov 2021 14:49:01 -0500 Subject: [PATCH 07/16] add tests for "delete aux file" and "list aux by origin" #8235 --- .../iq/dataverse/api/AuxiliaryFilesIT.java | 36 +++++++++++++++++++ .../edu/harvard/iq/dataverse/api/UtilIT.java | 22 ++++++++++++ 2 files changed, 58 insertions(+) diff --git a/src/test/java/edu/harvard/iq/dataverse/api/AuxiliaryFilesIT.java b/src/test/java/edu/harvard/iq/dataverse/api/AuxiliaryFilesIT.java index 4cc688c8758..68deb8ac4b3 100644 --- a/src/test/java/edu/harvard/iq/dataverse/api/AuxiliaryFilesIT.java +++ b/src/test/java/edu/harvard/iq/dataverse/api/AuxiliaryFilesIT.java @@ -9,6 +9,7 @@ import java.nio.file.Paths; import static javax.ws.rs.core.Response.Status.CREATED; import static javax.ws.rs.core.Response.Status.FORBIDDEN; +import static javax.ws.rs.core.Response.Status.NOT_FOUND; import static javax.ws.rs.core.Response.Status.OK; import static org.hamcrest.CoreMatchers.equalTo; import org.junit.Assert; @@ -204,6 +205,22 @@ public void testUploadAuxFiles() throws IOException { .body("data.type", equalTo(null)) .body("data.contentType", equalTo("text/plain")); + // file with an origin + Path pathToAuxFileOrigin1 = Paths.get(java.nio.file.Files.createTempDirectory(null) + File.separator + "file1.txt"); + String contentOfOrigin1 = "This file has an origin."; + java.nio.file.Files.write(pathToAuxFileOrigin1, contentOfOrigin1.getBytes()); + String formatTagOrigin1 = "origin1"; + String formatVersionOrigin1 = "0.1"; + String mimeTypeOrigin1 = "text/plain"; + String typeOrigin1 = "someType"; + String origin1 = "myApp"; + Response uploadAuxFileOrigin1 = UtilIT.uploadAuxFile(fileId, pathToAuxFileOrigin1.toString(), formatTagOrigin1, formatVersionOrigin1, mimeTypeOrigin1, true, typeOrigin1, origin1, apiToken); + uploadAuxFileOrigin1.prettyPrint(); + uploadAuxFileOrigin1.then().assertThat() + .statusCode(OK.getStatusCode()) + .body("data.type", equalTo("someType")) + .body("data.contentType", equalTo("text/plain")); + // Download JSON aux file. Response downloadAuxFileJson = UtilIT.downloadAuxFile(fileId, formatTagJson, formatVersionJson, apiToken); downloadAuxFileJson.then().assertThat().statusCode(OK.getStatusCode()); @@ -252,5 +269,24 @@ public void testUploadAuxFiles() throws IOException { Response failToDownloadAuxFileJsonPostPublish = UtilIT.downloadAuxFile(fileId, formatTagJson, formatVersionJson, apiTokenNoPrivs); failToDownloadAuxFileJsonPostPublish.then().assertThat().statusCode(OK.getStatusCode()); + Response listAuxFilesOrigin1 = UtilIT.listAuxFilesByOrigin(fileId, origin1, apiToken); + listAuxFilesOrigin1.prettyPrint(); + listAuxFilesOrigin1.then().assertThat() + .statusCode(OK.getStatusCode()) + .body("data[0].formatTag", equalTo("origin1")) + .body("data[0].formatVersion", equalTo("0.1")) + .body("data[0].fileSize", equalTo(24)) + .body("data[0].contentType", equalTo("text/plain")) + .body("data[0].isPublic", equalTo(true)) + .body("data[0].type", equalTo("someType")); + + Response deleteAuxFileOrigin1 = UtilIT.deleteAuxFile(fileId, formatTagOrigin1, formatVersionOrigin1, apiToken); + deleteAuxFileOrigin1.prettyPrint(); + deleteAuxFileOrigin1.then().assertThat().statusCode(OK.getStatusCode()); + + Response listAuxFilesOrigin1NowGone = UtilIT.listAuxFilesByOrigin(fileId, origin1, apiToken); + listAuxFilesOrigin1NowGone.prettyPrint(); + listAuxFilesOrigin1NowGone.then().assertThat().statusCode(NOT_FOUND.getStatusCode()); + } } diff --git a/src/test/java/edu/harvard/iq/dataverse/api/UtilIT.java b/src/test/java/edu/harvard/iq/dataverse/api/UtilIT.java index 03145f4c01b..b28068b5f75 100644 --- a/src/test/java/edu/harvard/iq/dataverse/api/UtilIT.java +++ b/src/test/java/edu/harvard/iq/dataverse/api/UtilIT.java @@ -649,6 +649,11 @@ static Response uploadFileViaNative(String datasetId, String pathToFile, String } static Response uploadAuxFile(Long fileId, String pathToFile, String formatTag, String formatVersion, String mimeType, boolean isPublic, String type, String apiToken) { + String nullOrigin = null; + return uploadAuxFile(fileId, pathToFile, formatTag, formatVersion, mimeType, isPublic, type, nullOrigin, apiToken); + } + + static Response uploadAuxFile(Long fileId, String pathToFile, String formatTag, String formatVersion, String mimeType, boolean isPublic, String type, String origin, String apiToken) { RequestSpecification requestSpecification = given() .header(API_TOKEN_HTTP_HEADER, apiToken) .multiPart("file", new File(pathToFile), mimeType) @@ -656,6 +661,9 @@ static Response uploadAuxFile(Long fileId, String pathToFile, String formatTag, if (type != null) { requestSpecification.multiPart("type", type); } + if (origin != null) { + requestSpecification.multiPart("origin", origin); + } return requestSpecification.post("/api/access/datafile/" + fileId + "/auxiliary/" + formatTag + "/" + formatVersion); } @@ -666,6 +674,20 @@ static Response downloadAuxFile(Long fileId, String formatTag, String formatVers return response; } + static Response listAuxFilesByOrigin(Long fileId, String origin, String apiToken) { + Response response = given() + .header(API_TOKEN_HTTP_HEADER, apiToken) + .get("/api/access/datafile/" + fileId + "/auxiliary/" + origin); + return response; + } + + static Response deleteAuxFile(Long fileId, String formatTag, String formatVersion, String apiToken) { + Response response = given() + .header(API_TOKEN_HTTP_HEADER, apiToken) + .delete("/api/access/datafile/" + fileId + "/auxiliary/" + formatTag + "/" + formatVersion); + return response; + } + static Response getCrawlableFileAccess(String datasetId, String folderName, String apiToken) { RequestSpecification requestSpecification = given() .header(API_TOKEN_HTTP_HEADER, apiToken); From 90c94b56d6ecedc80832310cf0984f387ea77a7b Mon Sep 17 00:00:00 2001 From: Philip Durbin Date: Tue, 16 Nov 2021 15:17:03 -0500 Subject: [PATCH 08/16] doc tweaks #8235 --- doc/sphinx-guides/source/developers/aux-file-support.rst | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/doc/sphinx-guides/source/developers/aux-file-support.rst b/doc/sphinx-guides/source/developers/aux-file-support.rst index 13433711c48..c92435796e1 100644 --- a/doc/sphinx-guides/source/developers/aux-file-support.rst +++ b/doc/sphinx-guides/source/developers/aux-file-support.rst @@ -16,12 +16,12 @@ To add an auxiliary file, specify the primary key of the datafile (FILE_ID), and export FORMAT_VERSION='v1' export TYPE='DP' export SERVER_URL=https://demo.dataverse.org - + curl -H X-Dataverse-key:$API_TOKEN -X POST -F "file=@$FILENAME" -F 'origin=myApp' -F 'isPublic=true' -F "type=$TYPE" "$SERVER_URL/api/access/datafile/$FILE_ID/auxiliary/$FORMAT_TAG/$FORMAT_VERSION" You should expect a 200 ("OK") response and JSON with information about your newly uploaded auxiliary file. -Downloading an Auxiliary File that belongs to a Datafile +Downloading an Auxiliary File that Belongs to a Datafile -------------------------------------------------------- To download an auxiliary file, use the primary key of the datafile, and the formatTag and formatVersion (if applicable) associated with the auxiliary file: @@ -49,9 +49,9 @@ To list auxiliary files, specify the primary key of the datafile (FILE_ID), and curl "$SERVER_URL/api/access/datafile/$FILE_ID/auxiliary/$ORIGIN" -You should expect a 200 ("OK") response and a JSON array with objects representing the auxiliary files found, or a 404 /Not Found response if no auxiliary files exist with that origin. +You should expect a 200 ("OK") response and a JSON array with objects representing the auxiliary files found, or a 404/Not Found response if no auxiliary files exist with that origin. -Deleting an Auxiliary File that belongs to a Datafile +Deleting an Auxiliary File that Belongs to a Datafile ----------------------------------------------------- To delete an auxiliary file, use the primary key of the datafile, and the formatTag and formatVersion (if applicable) associated with the auxiliary file: From 081a6a1cf7cb84089369fb73f0f88b5a88cd3e45 Mon Sep 17 00:00:00 2001 From: Jim Myers Date: Fri, 19 Nov 2021 11:29:29 -0500 Subject: [PATCH 09/16] updates per review of #8237 --- .../harvard/iq/dataverse/AuxiliaryFile.java | 5 ++- .../dataverse/AuxiliaryFileServiceBean.java | 36 ++++++++++--------- .../edu/harvard/iq/dataverse/api/Access.java | 34 +++++++++++++----- 3 files changed, 49 insertions(+), 26 deletions(-) diff --git a/src/main/java/edu/harvard/iq/dataverse/AuxiliaryFile.java b/src/main/java/edu/harvard/iq/dataverse/AuxiliaryFile.java index cbc29217c7f..a7a89934f47 100644 --- a/src/main/java/edu/harvard/iq/dataverse/AuxiliaryFile.java +++ b/src/main/java/edu/harvard/iq/dataverse/AuxiliaryFile.java @@ -29,7 +29,10 @@ @NamedQuery(name = "AuxiliaryFile.findAuxiliaryFilesByType", query = "select object(o) from AuxiliaryFile as o where o.dataFile.id = :dataFileId and o.type = :type"), @NamedQuery(name = "AuxiliaryFile.findAuxiliaryFilesWithoutType", - query = "select object(o) from AuxiliaryFile as o where o.dataFile.id = :dataFileId and o.type is null"),}) + query = "select object(o) from AuxiliaryFile as o where o.dataFile.id = :dataFileId and o.type is null"), + @NamedQuery(name = "AuxiliaryFile.findAuxiliaryFilesByOrigin", + query = "select object(o) from AuxiliaryFile as o where o.dataFile.id = :dataFileId and o.origin = :origin"), +}) @NamedNativeQueries({ @NamedNativeQuery(name = "AuxiliaryFile.findAuxiliaryFileTypes", query = "select distinct type from auxiliaryfile where datafile_id = ?1") diff --git a/src/main/java/edu/harvard/iq/dataverse/AuxiliaryFileServiceBean.java b/src/main/java/edu/harvard/iq/dataverse/AuxiliaryFileServiceBean.java index 59c06b6e3c3..149d9e18300 100644 --- a/src/main/java/edu/harvard/iq/dataverse/AuxiliaryFileServiceBean.java +++ b/src/main/java/edu/harvard/iq/dataverse/AuxiliaryFileServiceBean.java @@ -18,6 +18,7 @@ import javax.ejb.Stateless; import javax.inject.Named; import javax.persistence.EntityManager; +import javax.persistence.NoResultException; import javax.persistence.PersistenceContext; import javax.persistence.Query; import javax.persistence.TypedQuery; @@ -130,24 +131,25 @@ public AuxiliaryFile lookupAuxiliaryFile(DataFile dataFile, String formatTag, St try { AuxiliaryFile retVal = (AuxiliaryFile)query.getSingleResult(); return retVal; - } catch(Exception ex) { + } catch(NoResultException nr) { return null; } } - @SuppressWarnings("unchecked") - public List listAuxiliaryFiles(DataFile dataFile, String origin) { + + public List findAuxiliaryFiles(DataFile dataFile, String origin) { - Query query = em.createQuery("select object(o) from AuxiliaryFile as o where o.dataFile.id = :dataFileId and o.origin = :origin"); - - query.setParameter("dataFileId", dataFile.getId()); - query.setParameter("origin", origin); - try { - List retVal = (List)query.getResultList(); - return retVal; - } catch(Exception ex) { - return null; + TypedQuery query; + if (origin == null) { + query = em.createNamedQuery("AuxiliaryFile.findAuxiliaryFiles", AuxiliaryFile.class); + } else { + query = em.createNamedQuery("AuxiliaryFile.findAuxiliaryFilesByOrigin", AuxiliaryFile.class); + query.setParameter("origin", origin); } + query.setParameter("dataFileId", dataFile.getId()); + + List retVal = (List)query.getResultList(); + return retVal; } public void deleteAuxiliaryFile(DataFile dataFile, String formatTag, String formatVersion) throws IOException { @@ -165,7 +167,7 @@ public void deleteAuxiliaryFile(DataFile dataFile, String formatTag, String form } public List findAuxiliaryFiles(DataFile dataFile) { - TypedQuery query = em.createNamedQuery("AuxiliaryFile.findAuxiliaryFiles", AuxiliaryFile.class); + TypedQuery query = em.createNamedQuery("AuxiliaryFile.findAuxiliaryFiles", AuxiliaryFile.class); query.setParameter("dataFileId", dataFile.getId()); return query.getResultList(); } @@ -195,13 +197,13 @@ public List findAuxiliaryFileTypes(DataFile dataFile, boolean inBundle) } public List findAuxiliaryFileTypes(DataFile dataFile) { - Query query = em.createNamedQuery("AuxiliaryFile.findAuxiliaryFileTypes"); + TypedQuery query = em.createNamedQuery("AuxiliaryFile.findAuxiliaryFileTypes", String.class); query.setParameter(1, dataFile.getId()); return query.getResultList(); } public List findAuxiliaryFilesByType(DataFile dataFile, String typeString) { - TypedQuery query = em.createNamedQuery("AuxiliaryFile.findAuxiliaryFilesByType", AuxiliaryFile.class); + TypedQuery query = em.createNamedQuery("AuxiliaryFile.findAuxiliaryFilesByType", AuxiliaryFile.class); query.setParameter("dataFileId", dataFile.getId()); query.setParameter("type", typeString); return query.getResultList(); @@ -211,7 +213,7 @@ public List findOtherAuxiliaryFiles(DataFile dataFile) { List otherAuxFiles = new ArrayList<>(); List otherTypes = findAuxiliaryFileTypes(dataFile, false); for (String typeString : otherTypes) { - TypedQuery query = em.createNamedQuery("AuxiliaryFile.findAuxiliaryFilesByType", AuxiliaryFile.class); + TypedQuery query = em.createNamedQuery("AuxiliaryFile.findAuxiliaryFilesByType", AuxiliaryFile.class); query.setParameter("dataFileId", dataFile.getId()); query.setParameter("type", typeString); List auxFiles = query.getResultList(); @@ -222,7 +224,7 @@ public List findOtherAuxiliaryFiles(DataFile dataFile) { } public List findAuxiliaryFilesWithoutType(DataFile dataFile) { - Query query = em.createNamedQuery("AuxiliaryFile.findAuxiliaryFilesWithoutType", AuxiliaryFile.class); + TypedQuery query = em.createNamedQuery("AuxiliaryFile.findAuxiliaryFilesWithoutType", AuxiliaryFile.class); query.setParameter("dataFileId", dataFile.getId()); return query.getResultList(); } diff --git a/src/main/java/edu/harvard/iq/dataverse/api/Access.java b/src/main/java/edu/harvard/iq/dataverse/api/Access.java index 893e0eda839..f37eecf1458 100644 --- a/src/main/java/edu/harvard/iq/dataverse/api/Access.java +++ b/src/main/java/edu/harvard/iq/dataverse/api/Access.java @@ -67,6 +67,7 @@ import edu.harvard.iq.dataverse.util.FileUtil; import edu.harvard.iq.dataverse.util.StringUtil; import edu.harvard.iq.dataverse.util.SystemConfig; +import edu.harvard.iq.dataverse.util.json.NullSafeJsonBuilder; import java.util.logging.Logger; import javax.ejb.EJB; @@ -540,37 +541,54 @@ public String dataVariableMetadataDDI(@PathParam("varId") Long varId, @QueryPara return retValue; } - + /* - * GET method for retrieving various auxiliary files associated with + * GET method for retrieving a list of auxiliary files associated with * a tabular datafile. */ - @Path("datafile/{fileId}/auxiliary/{origin}") + @Path("datafile/{fileId}/auxiliary") @GET public Response listDatafileMetadataAux(@PathParam("fileId") String fileId, + @QueryParam("key") String apiToken, + @Context UriInfo uriInfo, + @Context HttpHeaders headers, + @Context HttpServletResponse response) throws ServiceUnavailableException { + return listAuxiliaryFiles(fileId, null, apiToken, uriInfo, headers, response); + } + /* + * GET method for retrieving a list auxiliary files associated with + * a tabular datafile and having the specified origin. + */ + + @Path("datafile/{fileId}/auxiliary/{origin}") + @GET + public Response listDatafileMetadataAuxByOrigin(@PathParam("fileId") String fileId, @PathParam("origin") String origin, @QueryParam("key") String apiToken, @Context UriInfo uriInfo, @Context HttpHeaders headers, @Context HttpServletResponse response) throws ServiceUnavailableException { - - DataFile df = findDataFileOrDieWrapper(fileId); + return listAuxiliaryFiles(fileId, origin, apiToken, uriInfo, headers, response); + } + + private Response listAuxiliaryFiles(String fileId, String origin, String apiToken, UriInfo uriInfo, HttpHeaders headers, HttpServletResponse response) { + DataFile df = findDataFileOrDieWrapper(fileId); if (apiToken == null || apiToken.equals("")) { apiToken = headers.getHeaderString(API_KEY_HEADER); } - List auxFileList = auxiliaryFileService.listAuxiliaryFiles(df, origin); + List auxFileList = auxiliaryFileService.findAuxiliaryFiles(df, origin); if (auxFileList == null || auxFileList.isEmpty()) { - throw new NotFoundException("No Auxiliary files exist for datafile " + fileId + " and the specified origin"); + throw new NotFoundException("No Auxiliary files exist for datafile " + fileId + (origin==null ? "": " and the specified origin")); } boolean isAccessAllowed = isAccessAuthorized(df, apiToken); JsonArrayBuilder jab = Json.createArrayBuilder(); auxFileList.forEach(auxFile -> { if (isAccessAllowed || auxFile.getIsPublic()) { - JsonObjectBuilder job = Json.createObjectBuilder(); + NullSafeJsonBuilder job = NullSafeJsonBuilder.jsonObjectBuilder(); job.add("formatTag", auxFile.getFormatTag()); job.add("formatVersion", auxFile.getFormatVersion()); job.add("fileSize", auxFile.getFileSize()); From c4599e6d9fcee0b1b1042302840a15da5efa70ef Mon Sep 17 00:00:00 2001 From: Jim Myers Date: Fri, 19 Nov 2021 12:41:37 -0500 Subject: [PATCH 10/16] fix tests --- .../iq/dataverse/AuxiliaryFileServiceBeanTest.java | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/src/test/java/edu/harvard/iq/dataverse/AuxiliaryFileServiceBeanTest.java b/src/test/java/edu/harvard/iq/dataverse/AuxiliaryFileServiceBeanTest.java index 55c1dfb9b20..ad97eba137c 100644 --- a/src/test/java/edu/harvard/iq/dataverse/AuxiliaryFileServiceBeanTest.java +++ b/src/test/java/edu/harvard/iq/dataverse/AuxiliaryFileServiceBeanTest.java @@ -4,11 +4,15 @@ import java.util.List; import javax.persistence.EntityManager; import javax.persistence.Query; +import javax.persistence.TypedQuery; + import static org.junit.Assert.assertEquals; import org.junit.Test; import org.junit.Before; import org.junit.jupiter.api.extension.ExtendWith; import org.mockito.ArgumentMatchers; +import org.mockito.Matchers; + import static org.mockito.Mockito.mock; import static org.mockito.Mockito.when; import org.mockito.junit.jupiter.MockitoExtension; @@ -18,7 +22,7 @@ public class AuxiliaryFileServiceBeanTest { EntityManager em; AuxiliaryFileServiceBean svc; - Query query; + TypedQuery query; List types; DataFile dataFile; @@ -26,7 +30,7 @@ public class AuxiliaryFileServiceBeanTest { public void setup() { svc = new AuxiliaryFileServiceBean(); svc.em = mock(EntityManager.class); - query = mock(Query.class); + query = mock(TypedQuery.class); types = Arrays.asList("DP", "GEOSPATIAL", "NEXT_BIG_THING", "FUTURE_FUN"); dataFile = new DataFile(); dataFile.setId(42l); @@ -36,7 +40,7 @@ public void setup() { public void testFindAuxiliaryFileTypesInBundleFalse() { System.out.println("testFindAuxiliaryFileTypesInBundleFalse"); boolean inBundle = false; - when(this.svc.em.createNamedQuery(ArgumentMatchers.anyString())).thenReturn(query); + when(this.svc.em.createNamedQuery(ArgumentMatchers.anyString(),ArgumentMatchers.>any())).thenReturn(query); when(query.getResultList()).thenReturn(types); List result = svc.findAuxiliaryFileTypes(dataFile, inBundle); // None of these are in the bundle. @@ -48,7 +52,7 @@ public void testFindAuxiliaryFileTypesInBundleFalse() { public void testFindAuxiliaryFileTypesInBundleTrue() { System.out.println("testFindAuxiliaryFileTypesInBundleTrue"); boolean inBundle = true; - when(this.svc.em.createNamedQuery(ArgumentMatchers.anyString())).thenReturn(query); + when(this.svc.em.createNamedQuery(ArgumentMatchers.anyString(),ArgumentMatchers.>any())).thenReturn(query); when(query.getResultList()).thenReturn(types); List result = svc.findAuxiliaryFileTypes(dataFile, inBundle); // DP is in the bundle. From 6df3875815d618ade5acf990291ac536a433401b Mon Sep 17 00:00:00 2001 From: Jim Myers Date: Fri, 19 Nov 2021 13:24:35 -0500 Subject: [PATCH 11/16] add tests of new functionality --- .../harvard/iq/dataverse/api/AuxiliaryFilesIT.java | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/src/test/java/edu/harvard/iq/dataverse/api/AuxiliaryFilesIT.java b/src/test/java/edu/harvard/iq/dataverse/api/AuxiliaryFilesIT.java index 68deb8ac4b3..0f5dcf8b3be 100644 --- a/src/test/java/edu/harvard/iq/dataverse/api/AuxiliaryFilesIT.java +++ b/src/test/java/edu/harvard/iq/dataverse/api/AuxiliaryFilesIT.java @@ -7,6 +7,7 @@ import java.io.IOException; import java.nio.file.Path; import java.nio.file.Paths; +import static javax.ws.rs.core.Response.Status.CONFLICT; import static javax.ws.rs.core.Response.Status.CREATED; import static javax.ws.rs.core.Response.Status.FORBIDDEN; import static javax.ws.rs.core.Response.Status.NOT_FOUND; @@ -83,6 +84,10 @@ public void testUploadAuxFiles() throws IOException { .body("data.type", equalTo("DP")) // FIXME: application/json would be better .body("data.contentType", equalTo("text/plain")); + Response uploadAuxFileJsonAgain = UtilIT.uploadAuxFile(fileId, pathToAuxFileJson.toString(), formatTagJson, formatVersionJson, mimeTypeJson, true, dpType, apiToken); + uploadAuxFileJsonAgain.prettyPrint(); + uploadAuxFileJsonAgain.then().assertThat() + .statusCode(CONFLICT.getStatusCode()); // XML aux file Path pathToAuxFileXml = Paths.get(java.nio.file.Files.createTempDirectory(null) + File.separator + "data.xml"); @@ -279,6 +284,13 @@ public void testUploadAuxFiles() throws IOException { .body("data[0].contentType", equalTo("text/plain")) .body("data[0].isPublic", equalTo(true)) .body("data[0].type", equalTo("someType")); + + Response listAllAuxFiles = UtilIT.listAuxFilesByOrigin(fileId, origin1, apiToken); + listAllAuxFiles.prettyPrint(); + listAllAuxFiles.then().assertThat() + .statusCode(OK.getStatusCode()) + .body("data.size()", equalTo(8)); + Response deleteAuxFileOrigin1 = UtilIT.deleteAuxFile(fileId, formatTagOrigin1, formatVersionOrigin1, apiToken); deleteAuxFileOrigin1.prettyPrint(); From 2f06d0dd5273220df8af75e4428d6d2087a2ce91 Mon Sep 17 00:00:00 2001 From: Jim Myers Date: Fri, 19 Nov 2021 13:30:27 -0500 Subject: [PATCH 12/16] remove unnecessary cast --- .../java/edu/harvard/iq/dataverse/AuxiliaryFileServiceBean.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/java/edu/harvard/iq/dataverse/AuxiliaryFileServiceBean.java b/src/main/java/edu/harvard/iq/dataverse/AuxiliaryFileServiceBean.java index 149d9e18300..d984484357b 100644 --- a/src/main/java/edu/harvard/iq/dataverse/AuxiliaryFileServiceBean.java +++ b/src/main/java/edu/harvard/iq/dataverse/AuxiliaryFileServiceBean.java @@ -148,7 +148,7 @@ public List findAuxiliaryFiles(DataFile dataFile, String origin) } query.setParameter("dataFileId", dataFile.getId()); - List retVal = (List)query.getResultList(); + List retVal = query.getResultList(); return retVal; } From 008914d215f70f0f956b97853c34ce973d7f6269 Mon Sep 17 00:00:00 2001 From: qqmyers Date: Fri, 19 Nov 2021 19:44:13 -0500 Subject: [PATCH 13/16] fix checksum, only delete file for server errors --- .../edu/harvard/iq/dataverse/AuxiliaryFileServiceBean.java | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/src/main/java/edu/harvard/iq/dataverse/AuxiliaryFileServiceBean.java b/src/main/java/edu/harvard/iq/dataverse/AuxiliaryFileServiceBean.java index d984484357b..fe06b6074e7 100644 --- a/src/main/java/edu/harvard/iq/dataverse/AuxiliaryFileServiceBean.java +++ b/src/main/java/edu/harvard/iq/dataverse/AuxiliaryFileServiceBean.java @@ -24,6 +24,7 @@ import javax.persistence.TypedQuery; import javax.ws.rs.ClientErrorException; import javax.ws.rs.InternalServerErrorException; +import javax.ws.rs.ServerErrorException; import javax.ws.rs.core.Response; import org.apache.tika.Tika; @@ -93,7 +94,7 @@ public AuxiliaryFile processAuxiliaryFile(InputStream fileInputStream, DataFile } DigestInputStream di = new DigestInputStream(fileInputStream, md); - storageIO.saveInputStreamAsAux(fileInputStream, auxExtension); + storageIO.saveInputStreamAsAux(di, auxExtension); auxFile.setChecksum(FileUtil.checksumDigestToString(di.getMessageDigest().digest())); Tika tika = new Tika(); @@ -109,7 +110,7 @@ public AuxiliaryFile processAuxiliaryFile(InputStream fileInputStream, DataFile } catch (IOException ioex) { logger.severe("IO Exception trying to save auxiliary file: " + ioex.getMessage()); throw new InternalServerErrorException(); - } catch (RuntimeException e) { + } catch (ServerErrorException e) { // If anything fails during database insert, remove file from storage try { storageIO.deleteAuxObject(auxExtension); From 194ef8a3c8738570cc2c3595e13cf81066590c97 Mon Sep 17 00:00:00 2001 From: qqmyers Date: Fri, 19 Nov 2021 19:44:21 -0500 Subject: [PATCH 14/16] test fixes --- .../edu/harvard/iq/dataverse/api/AuxiliaryFilesIT.java | 2 +- src/test/java/edu/harvard/iq/dataverse/api/UtilIT.java | 8 ++++++++ 2 files changed, 9 insertions(+), 1 deletion(-) diff --git a/src/test/java/edu/harvard/iq/dataverse/api/AuxiliaryFilesIT.java b/src/test/java/edu/harvard/iq/dataverse/api/AuxiliaryFilesIT.java index 0f5dcf8b3be..a64df2c48db 100644 --- a/src/test/java/edu/harvard/iq/dataverse/api/AuxiliaryFilesIT.java +++ b/src/test/java/edu/harvard/iq/dataverse/api/AuxiliaryFilesIT.java @@ -285,7 +285,7 @@ public void testUploadAuxFiles() throws IOException { .body("data[0].isPublic", equalTo(true)) .body("data[0].type", equalTo("someType")); - Response listAllAuxFiles = UtilIT.listAuxFilesByOrigin(fileId, origin1, apiToken); + Response listAllAuxFiles = UtilIT.listAllAuxFiles(fileId, apiToken); listAllAuxFiles.prettyPrint(); listAllAuxFiles.then().assertThat() .statusCode(OK.getStatusCode()) diff --git a/src/test/java/edu/harvard/iq/dataverse/api/UtilIT.java b/src/test/java/edu/harvard/iq/dataverse/api/UtilIT.java index b28068b5f75..9e900a945be 100644 --- a/src/test/java/edu/harvard/iq/dataverse/api/UtilIT.java +++ b/src/test/java/edu/harvard/iq/dataverse/api/UtilIT.java @@ -681,6 +681,14 @@ static Response listAuxFilesByOrigin(Long fileId, String origin, String apiToken return response; } + static Response listAllAuxFiles(Long fileId, String apiToken) { + Response response = given() + .header(API_TOKEN_HTTP_HEADER, apiToken) + .get("/api/access/datafile/" + fileId + "/auxiliary"); + return response; + } + + static Response deleteAuxFile(Long fileId, String formatTag, String formatVersion, String apiToken) { Response response = given() .header(API_TOKEN_HTTP_HEADER, apiToken) From b06657ff4a406cd00b868b3d40effef2cb49fd2a Mon Sep 17 00:00:00 2001 From: qqmyers Date: Fri, 19 Nov 2021 19:46:37 -0500 Subject: [PATCH 15/16] update notes --- doc/release-notes/8235-auxiliaryfileAPIenhancements.md | 1 + 1 file changed, 1 insertion(+) diff --git a/doc/release-notes/8235-auxiliaryfileAPIenhancements.md b/doc/release-notes/8235-auxiliaryfileAPIenhancements.md index 6c02c3d0e93..b83f10be948 100644 --- a/doc/release-notes/8235-auxiliaryfileAPIenhancements.md +++ b/doc/release-notes/8235-auxiliaryfileAPIenhancements.md @@ -5,6 +5,7 @@ This release includes updates to the Auxiliary File API: - Improved error reporting - The API will block attempts to create a duplicate auxiliary file - Delete and list-by-original calls have been added +- Bug fix: correct checksum recorded for aux file Please note that the auxiliary files feature is experimental and is designed to support integration with tools from the [OpenDP Project](https://opendp.org). If the API endpoints are not needed they can be blocked. From 1c95a956a482adf34b5ee2389e3236593404689f Mon Sep 17 00:00:00 2001 From: Philip Durbin Date: Mon, 22 Nov 2021 15:03:30 -0500 Subject: [PATCH 16/16] give expected file extension when downloading auxiliary files #8241 --- doc/release-notes/8241-ext.md | 3 +++ .../source/developers/aux-file-support.rst | 4 +++- .../edu/harvard/iq/dataverse/AuxiliaryFile.java | 15 +++++++++++++++ .../iq/dataverse/AuxiliaryFileServiceBean.java | 4 +++- .../edu/harvard/iq/dataverse/api/Access.java | 9 ++++++++- .../iq/dataverse/api/DownloadInstanceWriter.java | 9 ++++++++- .../db/migration/V5.8.0.2__8241-ext.sql | 1 + .../iq/dataverse/api/AuxiliaryFilesIT.java | 8 +++----- .../api/DownloadInstanceWriterTest.java | 16 ++++++++++++++++ 9 files changed, 60 insertions(+), 9 deletions(-) create mode 100644 doc/release-notes/8241-ext.md create mode 100644 src/main/resources/db/migration/V5.8.0.2__8241-ext.sql diff --git a/doc/release-notes/8241-ext.md b/doc/release-notes/8241-ext.md new file mode 100644 index 00000000000..9556c7c981b --- /dev/null +++ b/doc/release-notes/8241-ext.md @@ -0,0 +1,3 @@ +When you download an auxiliary file, the file extension will now be based on the extension of the file you uploaded, if it had one. + +Auxiliary files uploaded previously do not have the filename saved and will have a file extension based on detected content type (MIME type), if any. diff --git a/doc/sphinx-guides/source/developers/aux-file-support.rst b/doc/sphinx-guides/source/developers/aux-file-support.rst index c92435796e1..8245fcac204 100644 --- a/doc/sphinx-guides/source/developers/aux-file-support.rst +++ b/doc/sphinx-guides/source/developers/aux-file-support.rst @@ -35,7 +35,9 @@ formatTag and formatVersion (if applicable) associated with the auxiliary file: export FORMAT_VERSION='v1' curl "$SERVER_URL/api/access/datafile/$FILE_ID/auxiliary/$FORMAT_TAG/$FORMAT_VERSION" - + +The file extension will be based on the file extension originally uploaded (but converted to lower case) or in the case of no file extension, a best guess will be made based on the content type (MIME type). + Listing Auxiliary Files for a Datafile by Origin ------------------------------------------------ To list auxiliary files, specify the primary key of the datafile (FILE_ID), and the origin associated with the auxiliary files to list (the application/entity that created them). diff --git a/src/main/java/edu/harvard/iq/dataverse/AuxiliaryFile.java b/src/main/java/edu/harvard/iq/dataverse/AuxiliaryFile.java index a7a89934f47..90b2c937202 100644 --- a/src/main/java/edu/harvard/iq/dataverse/AuxiliaryFile.java +++ b/src/main/java/edu/harvard/iq/dataverse/AuxiliaryFile.java @@ -4,6 +4,7 @@ import edu.harvard.iq.dataverse.util.BundleUtil; import java.io.Serializable; import java.util.MissingResourceException; +import javax.persistence.Column; import javax.persistence.Entity; import javax.persistence.GeneratedValue; import javax.persistence.GenerationType; @@ -66,6 +67,12 @@ public class AuxiliaryFile implements Serializable { private String checksum; + /** + * filename can be null because it was never required originally. + */ + @Column(nullable = true) + private String filename; + /** * A way of grouping similar auxiliary files together. The type could be * "DP" for "Differentially Private Statistics", for example. @@ -160,4 +167,12 @@ public String getTypeFriendly() { } } + public String getFilename() { + return filename; + } + + public void setFilename(String filename) { + this.filename = filename; + } + } diff --git a/src/main/java/edu/harvard/iq/dataverse/AuxiliaryFileServiceBean.java b/src/main/java/edu/harvard/iq/dataverse/AuxiliaryFileServiceBean.java index fe06b6074e7..189d70c0114 100644 --- a/src/main/java/edu/harvard/iq/dataverse/AuxiliaryFileServiceBean.java +++ b/src/main/java/edu/harvard/iq/dataverse/AuxiliaryFileServiceBean.java @@ -68,9 +68,10 @@ public AuxiliaryFile save(AuxiliaryFile auxiliaryFile) { * @param isPublic boolean - is this file available to any user? * @param type how to group the files such as "DP" for "Differentially * Private Statistics". + * @param filename name of the file * @return success boolean - returns whether the save was successful */ - public AuxiliaryFile processAuxiliaryFile(InputStream fileInputStream, DataFile dataFile, String formatTag, String formatVersion, String origin, boolean isPublic, String type) { + public AuxiliaryFile processAuxiliaryFile(InputStream fileInputStream, DataFile dataFile, String formatTag, String formatVersion, String origin, boolean isPublic, String type, String filename) { StorageIO storageIO = null; AuxiliaryFile auxFile = new AuxiliaryFile(); @@ -106,6 +107,7 @@ public AuxiliaryFile processAuxiliaryFile(InputStream fileInputStream, DataFile auxFile.setType(type); auxFile.setDataFile(dataFile); auxFile.setFileSize(storageIO.getAuxObjectSize(auxExtension)); + auxFile.setFilename(filename); auxFile = save(auxFile); } catch (IOException ioex) { logger.severe("IO Exception trying to save auxiliary file: " + ioex.getMessage()); diff --git a/src/main/java/edu/harvard/iq/dataverse/api/Access.java b/src/main/java/edu/harvard/iq/dataverse/api/Access.java index 49880b0b498..23457df9343 100644 --- a/src/main/java/edu/harvard/iq/dataverse/api/Access.java +++ b/src/main/java/edu/harvard/iq/dataverse/api/Access.java @@ -125,6 +125,7 @@ import javax.ws.rs.core.MediaType; import static javax.ws.rs.core.Response.Status.FORBIDDEN; import static javax.ws.rs.core.Response.Status.UNAUTHORIZED; +import org.glassfish.jersey.media.multipart.FormDataContentDisposition; import org.glassfish.jersey.media.multipart.FormDataParam; /* @@ -1261,6 +1262,7 @@ public Response saveAuxiliaryFileWithVersion(@PathParam("fileId") Long fileId, @FormDataParam("origin") String origin, @FormDataParam("isPublic") boolean isPublic, @FormDataParam("type") String type, + @FormDataParam("file") FormDataContentDisposition contentDispositionHeader, @FormDataParam("file") InputStream fileInputStream ) { @@ -1280,7 +1282,12 @@ public Response saveAuxiliaryFileWithVersion(@PathParam("fileId") Long fileId, return error(FORBIDDEN, "User not authorized to edit the dataset."); } - AuxiliaryFile saved = auxiliaryFileService.processAuxiliaryFile(fileInputStream, dataFile, formatTag, formatVersion, origin, isPublic, type); + String filename = null; + if (contentDispositionHeader != null) { + filename = contentDispositionHeader.getFileName(); + } + + AuxiliaryFile saved = auxiliaryFileService.processAuxiliaryFile(fileInputStream, dataFile, formatTag, formatVersion, origin, isPublic, type, filename); if (saved!=null) { return ok(json(saved)); diff --git a/src/main/java/edu/harvard/iq/dataverse/api/DownloadInstanceWriter.java b/src/main/java/edu/harvard/iq/dataverse/api/DownloadInstanceWriter.java index 80b3f988953..2d0964a4e2a 100644 --- a/src/main/java/edu/harvard/iq/dataverse/api/DownloadInstanceWriter.java +++ b/src/main/java/edu/harvard/iq/dataverse/api/DownloadInstanceWriter.java @@ -577,11 +577,18 @@ private boolean isAuxiliaryObjectCached(StorageIO storageIO, String auxiliaryTag } } - private String getFileExtension(AuxiliaryFile auxFile) { + public String getFileExtension(AuxiliaryFile auxFile) { String fileExtension = ""; if (auxFile == null) { return fileExtension; } + String filename = auxFile.getFilename(); + if (filename != null) { + String extension = FileUtil.getFileExtension(filename); + if (extension != null) { + return "." + extension; + } + } String contentType = auxFile.getContentType(); if (contentType != null) { MimeTypes allTypes = MimeTypes.getDefaultMimeTypes(); diff --git a/src/main/resources/db/migration/V5.8.0.2__8241-ext.sql b/src/main/resources/db/migration/V5.8.0.2__8241-ext.sql new file mode 100644 index 00000000000..9bf1c38fb2f --- /dev/null +++ b/src/main/resources/db/migration/V5.8.0.2__8241-ext.sql @@ -0,0 +1 @@ +ALTER TABLE auxiliaryfile ADD COLUMN IF NOT EXISTS filename VARCHAR(255); diff --git a/src/test/java/edu/harvard/iq/dataverse/api/AuxiliaryFilesIT.java b/src/test/java/edu/harvard/iq/dataverse/api/AuxiliaryFilesIT.java index a64df2c48db..96e4bcc87a6 100644 --- a/src/test/java/edu/harvard/iq/dataverse/api/AuxiliaryFilesIT.java +++ b/src/test/java/edu/harvard/iq/dataverse/api/AuxiliaryFilesIT.java @@ -229,14 +229,12 @@ public void testUploadAuxFiles() throws IOException { // Download JSON aux file. Response downloadAuxFileJson = UtilIT.downloadAuxFile(fileId, formatTagJson, formatVersionJson, apiToken); downloadAuxFileJson.then().assertThat().statusCode(OK.getStatusCode()); - // FIXME: This should be ".json" instead of ".txt" - Assert.assertEquals("attachment; filename=\"data.tab.dpJson_0.1.txt\"", downloadAuxFileJson.header("Content-disposition")); + Assert.assertEquals("attachment; filename=\"data.tab.dpJson_0.1.json\"", downloadAuxFileJson.header("Content-disposition")); // Download XML aux file. Response downloadAuxFileXml = UtilIT.downloadAuxFile(fileId, formatTagXml, formatVersionXml, apiToken); downloadAuxFileXml.then().assertThat().statusCode(OK.getStatusCode()); - // FIXME: This should be ".xml" instead of ".txt" - Assert.assertEquals("attachment; filename=\"data.tab.dpXml_0.1.txt\"", downloadAuxFileXml.header("Content-disposition")); + Assert.assertEquals("attachment; filename=\"data.tab.dpXml_0.1.xml\"", downloadAuxFileXml.header("Content-disposition")); // Download PDF aux file. Response downloadAuxFilePdf = UtilIT.downloadAuxFile(fileId, formatTagPdf, formatVersionPdf, apiToken); @@ -246,7 +244,7 @@ public void testUploadAuxFiles() throws IOException { // Download Markdown aux file. Response downloadAuxFileMd = UtilIT.downloadAuxFile(fileId, formatTagMd, formatVersionMd, apiToken); downloadAuxFileMd.then().assertThat().statusCode(OK.getStatusCode()); - Assert.assertEquals("attachment; filename=\"data.tab.README_0.1.txt\"", downloadAuxFileMd.header("Content-disposition")); + Assert.assertEquals("attachment; filename=\"data.tab.README_0.1.md\"", downloadAuxFileMd.header("Content-disposition")); Response createUserNoPrivs = UtilIT.createRandomUser(); createUserNoPrivs.then().assertThat().statusCode(OK.getStatusCode()); diff --git a/src/test/java/edu/harvard/iq/dataverse/api/DownloadInstanceWriterTest.java b/src/test/java/edu/harvard/iq/dataverse/api/DownloadInstanceWriterTest.java index 6de52951077..d7bdc28c8b0 100644 --- a/src/test/java/edu/harvard/iq/dataverse/api/DownloadInstanceWriterTest.java +++ b/src/test/java/edu/harvard/iq/dataverse/api/DownloadInstanceWriterTest.java @@ -1,5 +1,6 @@ package edu.harvard.iq.dataverse.api; +import edu.harvard.iq.dataverse.AuxiliaryFile; import edu.harvard.iq.dataverse.dataaccess.Range; import java.util.List; import static org.junit.Assert.assertEquals; @@ -145,4 +146,19 @@ public void testGetRanges0to0and90toNull() { assertNotNull(expectedException); } + @Test + public void testGetAuxFileExtension() { + AuxiliaryFile auxFile = null; + assertEquals("", diw.getFileExtension(auxFile)); + auxFile = new AuxiliaryFile(); + assertEquals("", diw.getFileExtension(auxFile)); + auxFile.setContentType("text/plain"); + assertEquals(".txt", diw.getFileExtension(auxFile)); + auxFile.setFilename("foo.json"); + assertEquals(".json", diw.getFileExtension(auxFile)); + auxFile.setFilename("MANIFEST.TXT"); + // .TXT becomes .txt due to FileUtil.getFileExtension doing toLowerCase + assertEquals(".txt", diw.getFileExtension(auxFile)); + } + }