diff --git a/doc/release-notes/8235-auxiliaryfileAPIenhancements.md b/doc/release-notes/8235-auxiliaryfileAPIenhancements.md new file mode 100644 index 00000000000..b83f10be948 --- /dev/null +++ b/doc/release-notes/8235-auxiliaryfileAPIenhancements.md @@ -0,0 +1,14 @@ +### Auxiliary File API Enhancements + +This release includes updates to the Auxiliary File API: +- Auxiliary files can now also be associated with non-tabular files +- Improved error reporting +- The API will block attempts to create a duplicate auxiliary file +- Delete and list-by-original calls have been added +- Bug fix: correct checksum recorded for aux file + +Please note that the auxiliary files feature is experimental and is designed to support integration with tools from the [OpenDP Project](https://opendp.org). If the API endpoints are not needed they can be blocked. + +### Major Use Cases + +(note for release time - expand on the items above, as use cases) \ No newline at end of file diff --git a/doc/release-notes/8241-ext.md b/doc/release-notes/8241-ext.md new file mode 100644 index 00000000000..9556c7c981b --- /dev/null +++ b/doc/release-notes/8241-ext.md @@ -0,0 +1,3 @@ +When you download an auxiliary file, the file extension will now be based on the extension of the file you uploaded, if it had one. + +Auxiliary files uploaded previously do not have the filename saved and will have a file extension based on detected content type (MIME type), if any. diff --git a/doc/sphinx-guides/source/developers/aux-file-support.rst b/doc/sphinx-guides/source/developers/aux-file-support.rst index 42b9aebf975..8245fcac204 100644 --- a/doc/sphinx-guides/source/developers/aux-file-support.rst +++ b/doc/sphinx-guides/source/developers/aux-file-support.rst @@ -1,7 +1,7 @@ Auxiliary File Support ====================== -Auxiliary file support is experimental and as such, related APIs may be added, changed or removed without standard backward compatibility. Auxiliary files in the Dataverse Software are being added to support depositing and downloading differentially private metadata, as part of the OpenDP project (opendp.org). In future versions, this approach will likely become more broadly used and supported. +Auxiliary file support is experimental and as such, related APIs may be added, changed or removed without standard backward compatibility. Auxiliary files in the Dataverse Software are being added to support depositing and downloading differentially private metadata, as part of the `OpenDP project `_. In future versions, this approach will likely become more broadly used and supported. Adding an Auxiliary File to a Datafile -------------------------------------- @@ -16,12 +16,12 @@ To add an auxiliary file, specify the primary key of the datafile (FILE_ID), and export FORMAT_VERSION='v1' export TYPE='DP' export SERVER_URL=https://demo.dataverse.org - + curl -H X-Dataverse-key:$API_TOKEN -X POST -F "file=@$FILENAME" -F 'origin=myApp' -F 'isPublic=true' -F "type=$TYPE" "$SERVER_URL/api/access/datafile/$FILE_ID/auxiliary/$FORMAT_TAG/$FORMAT_VERSION" You should expect a 200 ("OK") response and JSON with information about your newly uploaded auxiliary file. -Downloading an Auxiliary File that belongs to a Datafile +Downloading an Auxiliary File that Belongs to a Datafile -------------------------------------------------------- To download an auxiliary file, use the primary key of the datafile, and the formatTag and formatVersion (if applicable) associated with the auxiliary file: @@ -33,5 +33,39 @@ formatTag and formatVersion (if applicable) associated with the auxiliary file: export FILE_ID='12345' export FORMAT_TAG='dpJson' export FORMAT_VERSION='v1' - + curl "$SERVER_URL/api/access/datafile/$FILE_ID/auxiliary/$FORMAT_TAG/$FORMAT_VERSION" + +The file extension will be based on the file extension originally uploaded (but converted to lower case) or in the case of no file extension, a best guess will be made based on the content type (MIME type). + +Listing Auxiliary Files for a Datafile by Origin +------------------------------------------------ +To list auxiliary files, specify the primary key of the datafile (FILE_ID), and the origin associated with the auxiliary files to list (the application/entity that created them). + +.. code-block:: bash + + export API_TOKEN=xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx + export FILE_ID='12345' + export SERVER_URL=https://demo.dataverse.org + export ORIGIN='app1' + + curl "$SERVER_URL/api/access/datafile/$FILE_ID/auxiliary/$ORIGIN" + +You should expect a 200 ("OK") response and a JSON array with objects representing the auxiliary files found, or a 404/Not Found response if no auxiliary files exist with that origin. + +Deleting an Auxiliary File that Belongs to a Datafile +----------------------------------------------------- +To delete an auxiliary file, use the primary key of the datafile, and the +formatTag and formatVersion (if applicable) associated with the auxiliary file: + +.. code-block:: bash + + export API_TOKEN=xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx + export SERVER_URL=https://demo.dataverse.org + export FILE_ID='12345' + export FORMAT_TAG='dpJson' + export FORMAT_VERSION='v1' + + curl -X DELETE "$SERVER_URL/api/access/datafile/$FILE_ID/auxiliary/$FORMAT_TAG/$FORMAT_VERSION" + + diff --git a/src/main/java/edu/harvard/iq/dataverse/AuxiliaryFile.java b/src/main/java/edu/harvard/iq/dataverse/AuxiliaryFile.java index cbc29217c7f..90b2c937202 100644 --- a/src/main/java/edu/harvard/iq/dataverse/AuxiliaryFile.java +++ b/src/main/java/edu/harvard/iq/dataverse/AuxiliaryFile.java @@ -4,6 +4,7 @@ import edu.harvard.iq.dataverse.util.BundleUtil; import java.io.Serializable; import java.util.MissingResourceException; +import javax.persistence.Column; import javax.persistence.Entity; import javax.persistence.GeneratedValue; import javax.persistence.GenerationType; @@ -29,7 +30,10 @@ @NamedQuery(name = "AuxiliaryFile.findAuxiliaryFilesByType", query = "select object(o) from AuxiliaryFile as o where o.dataFile.id = :dataFileId and o.type = :type"), @NamedQuery(name = "AuxiliaryFile.findAuxiliaryFilesWithoutType", - query = "select object(o) from AuxiliaryFile as o where o.dataFile.id = :dataFileId and o.type is null"),}) + query = "select object(o) from AuxiliaryFile as o where o.dataFile.id = :dataFileId and o.type is null"), + @NamedQuery(name = "AuxiliaryFile.findAuxiliaryFilesByOrigin", + query = "select object(o) from AuxiliaryFile as o where o.dataFile.id = :dataFileId and o.origin = :origin"), +}) @NamedNativeQueries({ @NamedNativeQuery(name = "AuxiliaryFile.findAuxiliaryFileTypes", query = "select distinct type from auxiliaryfile where datafile_id = ?1") @@ -63,6 +67,12 @@ public class AuxiliaryFile implements Serializable { private String checksum; + /** + * filename can be null because it was never required originally. + */ + @Column(nullable = true) + private String filename; + /** * A way of grouping similar auxiliary files together. The type could be * "DP" for "Differentially Private Statistics", for example. @@ -157,4 +167,12 @@ public String getTypeFriendly() { } } + public String getFilename() { + return filename; + } + + public void setFilename(String filename) { + this.filename = filename; + } + } diff --git a/src/main/java/edu/harvard/iq/dataverse/AuxiliaryFileServiceBean.java b/src/main/java/edu/harvard/iq/dataverse/AuxiliaryFileServiceBean.java index 0643c3622d4..189d70c0114 100644 --- a/src/main/java/edu/harvard/iq/dataverse/AuxiliaryFileServiceBean.java +++ b/src/main/java/edu/harvard/iq/dataverse/AuxiliaryFileServiceBean.java @@ -4,10 +4,13 @@ import edu.harvard.iq.dataverse.dataaccess.StorageIO; import edu.harvard.iq.dataverse.util.FileUtil; import edu.harvard.iq.dataverse.util.SystemConfig; + +import java.io.FileNotFoundException; import java.io.IOException; import java.io.InputStream; import java.security.DigestInputStream; import java.security.MessageDigest; +import java.security.NoSuchAlgorithmException; import java.util.ArrayList; import java.util.List; import java.util.logging.Logger; @@ -15,9 +18,15 @@ import javax.ejb.Stateless; import javax.inject.Named; import javax.persistence.EntityManager; +import javax.persistence.NoResultException; import javax.persistence.PersistenceContext; import javax.persistence.Query; import javax.persistence.TypedQuery; +import javax.ws.rs.ClientErrorException; +import javax.ws.rs.InternalServerErrorException; +import javax.ws.rs.ServerErrorException; +import javax.ws.rs.core.Response; + import org.apache.tika.Tika; /** @@ -59,11 +68,12 @@ public AuxiliaryFile save(AuxiliaryFile auxiliaryFile) { * @param isPublic boolean - is this file available to any user? * @param type how to group the files such as "DP" for "Differentially * Private Statistics". + * @param filename name of the file * @return success boolean - returns whether the save was successful */ - public AuxiliaryFile processAuxiliaryFile(InputStream fileInputStream, DataFile dataFile, String formatTag, String formatVersion, String origin, boolean isPublic, String type) { - - StorageIO storageIO =null; + public AuxiliaryFile processAuxiliaryFile(InputStream fileInputStream, DataFile dataFile, String formatTag, String formatVersion, String origin, boolean isPublic, String type, String filename) { + + StorageIO storageIO = null; AuxiliaryFile auxFile = new AuxiliaryFile(); String auxExtension = formatTag + "_" + formatVersion; try { @@ -73,12 +83,20 @@ public AuxiliaryFile processAuxiliaryFile(InputStream fileInputStream, DataFile // If the db fails for any reason, then rollback // by removing the auxfile from storage. storageIO = dataFile.getStorageIO(); - MessageDigest md = MessageDigest.getInstance(systemConfig.getFileFixityChecksumAlgorithm().toString()); - DigestInputStream di - = new DigestInputStream(fileInputStream, md); - - storageIO.saveInputStreamAsAux(fileInputStream, auxExtension); - auxFile.setChecksum(FileUtil.checksumDigestToString(di.getMessageDigest().digest()) ); + if (storageIO.isAuxObjectCached(auxExtension)) { + throw new ClientErrorException("Auxiliary file already exists", Response.Status.CONFLICT); + } + MessageDigest md; + try { + md = MessageDigest.getInstance(systemConfig.getFileFixityChecksumAlgorithm().toString()); + } catch (NoSuchAlgorithmException e) { + logger.severe("NoSuchAlgorithmException for system fixity algorithm: " + systemConfig.getFileFixityChecksumAlgorithm().toString()); + throw new InternalServerErrorException(); + } + DigestInputStream di = new DigestInputStream(fileInputStream, md); + + storageIO.saveInputStreamAsAux(di, auxExtension); + auxFile.setChecksum(FileUtil.checksumDigestToString(di.getMessageDigest().digest())); Tika tika = new Tika(); auxFile.setContentType(tika.detect(storageIO.getAuxFileAsInputStream(auxExtension))); @@ -87,20 +105,21 @@ public AuxiliaryFile processAuxiliaryFile(InputStream fileInputStream, DataFile auxFile.setOrigin(origin); auxFile.setIsPublic(isPublic); auxFile.setType(type); - auxFile.setDataFile(dataFile); + auxFile.setDataFile(dataFile); auxFile.setFileSize(storageIO.getAuxObjectSize(auxExtension)); + auxFile.setFilename(filename); auxFile = save(auxFile); } catch (IOException ioex) { - logger.info("IO Exception trying to save auxiliary file: " + ioex.getMessage()); - return null; - } catch (Exception e) { + logger.severe("IO Exception trying to save auxiliary file: " + ioex.getMessage()); + throw new InternalServerErrorException(); + } catch (ServerErrorException e) { // If anything fails during database insert, remove file from storage try { storageIO.deleteAuxObject(auxExtension); - } catch(IOException ioex) { - logger.info("IO Exception trying remove auxiliary file in exception handler: " + ioex.getMessage()); - return null; + } catch (IOException ioex) { + logger.warning("IO Exception trying remove auxiliary file in exception handler: " + ioex.getMessage()); } + throw e; } return auxFile; } @@ -115,13 +134,43 @@ public AuxiliaryFile lookupAuxiliaryFile(DataFile dataFile, String formatTag, St try { AuxiliaryFile retVal = (AuxiliaryFile)query.getSingleResult(); return retVal; - } catch(Exception ex) { + } catch(NoResultException nr) { return null; } } + + + public List findAuxiliaryFiles(DataFile dataFile, String origin) { + + TypedQuery query; + if (origin == null) { + query = em.createNamedQuery("AuxiliaryFile.findAuxiliaryFiles", AuxiliaryFile.class); + } else { + query = em.createNamedQuery("AuxiliaryFile.findAuxiliaryFilesByOrigin", AuxiliaryFile.class); + query.setParameter("origin", origin); + } + query.setParameter("dataFileId", dataFile.getId()); + + List retVal = query.getResultList(); + return retVal; + } + + public void deleteAuxiliaryFile(DataFile dataFile, String formatTag, String formatVersion) throws IOException { + AuxiliaryFile af = lookupAuxiliaryFile(dataFile, formatTag, formatVersion); + if (af == null) { + throw new FileNotFoundException(); + } + em.remove(af); + StorageIO storageIO; + storageIO = dataFile.getStorageIO(); + String auxExtension = formatTag + "_" + formatVersion; + if (storageIO.isAuxObjectCached(auxExtension)) { + storageIO.deleteAuxObject(auxExtension); + } + } public List findAuxiliaryFiles(DataFile dataFile) { - TypedQuery query = em.createNamedQuery("AuxiliaryFile.findAuxiliaryFiles", AuxiliaryFile.class); + TypedQuery query = em.createNamedQuery("AuxiliaryFile.findAuxiliaryFiles", AuxiliaryFile.class); query.setParameter("dataFileId", dataFile.getId()); return query.getResultList(); } @@ -151,13 +200,13 @@ public List findAuxiliaryFileTypes(DataFile dataFile, boolean inBundle) } public List findAuxiliaryFileTypes(DataFile dataFile) { - Query query = em.createNamedQuery("AuxiliaryFile.findAuxiliaryFileTypes"); + TypedQuery query = em.createNamedQuery("AuxiliaryFile.findAuxiliaryFileTypes", String.class); query.setParameter(1, dataFile.getId()); return query.getResultList(); } public List findAuxiliaryFilesByType(DataFile dataFile, String typeString) { - TypedQuery query = em.createNamedQuery("AuxiliaryFile.findAuxiliaryFilesByType", AuxiliaryFile.class); + TypedQuery query = em.createNamedQuery("AuxiliaryFile.findAuxiliaryFilesByType", AuxiliaryFile.class); query.setParameter("dataFileId", dataFile.getId()); query.setParameter("type", typeString); return query.getResultList(); @@ -167,7 +216,7 @@ public List findOtherAuxiliaryFiles(DataFile dataFile) { List otherAuxFiles = new ArrayList<>(); List otherTypes = findAuxiliaryFileTypes(dataFile, false); for (String typeString : otherTypes) { - TypedQuery query = em.createNamedQuery("AuxiliaryFile.findAuxiliaryFilesByType", AuxiliaryFile.class); + TypedQuery query = em.createNamedQuery("AuxiliaryFile.findAuxiliaryFilesByType", AuxiliaryFile.class); query.setParameter("dataFileId", dataFile.getId()); query.setParameter("type", typeString); List auxFiles = query.getResultList(); @@ -178,7 +227,7 @@ public List findOtherAuxiliaryFiles(DataFile dataFile) { } public List findAuxiliaryFilesWithoutType(DataFile dataFile) { - Query query = em.createNamedQuery("AuxiliaryFile.findAuxiliaryFilesWithoutType", AuxiliaryFile.class); + TypedQuery query = em.createNamedQuery("AuxiliaryFile.findAuxiliaryFilesWithoutType", AuxiliaryFile.class); query.setParameter("dataFileId", dataFile.getId()); return query.getResultList(); } diff --git a/src/main/java/edu/harvard/iq/dataverse/api/Access.java b/src/main/java/edu/harvard/iq/dataverse/api/Access.java index d171658fe2d..23457df9343 100644 --- a/src/main/java/edu/harvard/iq/dataverse/api/Access.java +++ b/src/main/java/edu/harvard/iq/dataverse/api/Access.java @@ -67,6 +67,7 @@ import edu.harvard.iq.dataverse.util.FileUtil; import edu.harvard.iq.dataverse.util.StringUtil; import edu.harvard.iq.dataverse.util.SystemConfig; +import edu.harvard.iq.dataverse.util.json.NullSafeJsonBuilder; import java.util.logging.Logger; import javax.ejb.EJB; @@ -74,6 +75,7 @@ import java.io.ByteArrayOutputStream; import java.io.File; import java.io.FileInputStream; +import java.io.FileNotFoundException; import java.io.IOException; import java.io.OutputStream; import java.sql.Timestamp; @@ -119,9 +121,11 @@ import java.util.stream.Stream; import javax.json.JsonObjectBuilder; import javax.ws.rs.RedirectionException; +import javax.ws.rs.ServerErrorException; import javax.ws.rs.core.MediaType; import static javax.ws.rs.core.Response.Status.FORBIDDEN; import static javax.ws.rs.core.Response.Status.UNAUTHORIZED; +import org.glassfish.jersey.media.multipart.FormDataContentDisposition; import org.glassfish.jersey.media.multipart.FormDataParam; /* @@ -511,6 +515,66 @@ public String tabularDatafileMetadataDDI(@PathParam("fileId") String fileId, @Q return retValue; } + + /* + * GET method for retrieving a list of auxiliary files associated with + * a tabular datafile. + */ + + @Path("datafile/{fileId}/auxiliary") + @GET + public Response listDatafileMetadataAux(@PathParam("fileId") String fileId, + @QueryParam("key") String apiToken, + @Context UriInfo uriInfo, + @Context HttpHeaders headers, + @Context HttpServletResponse response) throws ServiceUnavailableException { + return listAuxiliaryFiles(fileId, null, apiToken, uriInfo, headers, response); + } + /* + * GET method for retrieving a list auxiliary files associated with + * a tabular datafile and having the specified origin. + */ + + @Path("datafile/{fileId}/auxiliary/{origin}") + @GET + public Response listDatafileMetadataAuxByOrigin(@PathParam("fileId") String fileId, + @PathParam("origin") String origin, + @QueryParam("key") String apiToken, + @Context UriInfo uriInfo, + @Context HttpHeaders headers, + @Context HttpServletResponse response) throws ServiceUnavailableException { + return listAuxiliaryFiles(fileId, origin, apiToken, uriInfo, headers, response); + } + + private Response listAuxiliaryFiles(String fileId, String origin, String apiToken, UriInfo uriInfo, HttpHeaders headers, HttpServletResponse response) { + DataFile df = findDataFileOrDieWrapper(fileId); + + if (apiToken == null || apiToken.equals("")) { + apiToken = headers.getHeaderString(API_KEY_HEADER); + } + + List auxFileList = auxiliaryFileService.findAuxiliaryFiles(df, origin); + + if (auxFileList == null || auxFileList.isEmpty()) { + throw new NotFoundException("No Auxiliary files exist for datafile " + fileId + (origin==null ? "": " and the specified origin")); + } + boolean isAccessAllowed = isAccessAuthorized(df, apiToken); + JsonArrayBuilder jab = Json.createArrayBuilder(); + auxFileList.forEach(auxFile -> { + if (isAccessAllowed || auxFile.getIsPublic()) { + NullSafeJsonBuilder job = NullSafeJsonBuilder.jsonObjectBuilder(); + job.add("formatTag", auxFile.getFormatTag()); + job.add("formatVersion", auxFile.getFormatVersion()); + job.add("fileSize", auxFile.getFileSize()); + job.add("contentType", auxFile.getContentType()); + job.add("isPublic", auxFile.getIsPublic()); + job.add("type", auxFile.getType()); + jab.add(job); + } + }); + return ok(jab); + } + /* * GET method for retrieving various auxiliary files associated with * a tabular datafile. @@ -536,10 +600,6 @@ public DownloadInstance downloadAuxiliaryFile(@PathParam("fileId") String fileId DownloadInfo dInfo = new DownloadInfo(df); boolean publiclyAvailable = false; - if (!df.isTabularData()) { - throw new BadRequestException("tabular data required"); - } - DownloadInstance downloadInstance; AuxiliaryFile auxFile = null; @@ -1202,6 +1262,7 @@ public Response saveAuxiliaryFileWithVersion(@PathParam("fileId") Long fileId, @FormDataParam("origin") String origin, @FormDataParam("isPublic") boolean isPublic, @FormDataParam("type") String type, + @FormDataParam("file") FormDataContentDisposition contentDispositionHeader, @FormDataParam("file") InputStream fileInputStream ) { @@ -1221,12 +1282,13 @@ public Response saveAuxiliaryFileWithVersion(@PathParam("fileId") Long fileId, return error(FORBIDDEN, "User not authorized to edit the dataset."); } - if (!dataFile.isTabularData()) { - return error(BAD_REQUEST, "Not a tabular DataFile (db id=" + fileId + ")"); + String filename = null; + if (contentDispositionHeader != null) { + filename = contentDispositionHeader.getFileName(); } - AuxiliaryFile saved = auxiliaryFileService.processAuxiliaryFile(fileInputStream, dataFile, formatTag, formatVersion, origin, isPublic, type); - + AuxiliaryFile saved = auxiliaryFileService.processAuxiliaryFile(fileInputStream, dataFile, formatTag, formatVersion, origin, isPublic, type, filename); + if (saved!=null) { return ok(json(saved)); } else { @@ -1236,6 +1298,50 @@ public Response saveAuxiliaryFileWithVersion(@PathParam("fileId") Long fileId, + /** + * + * @param fileId + * @param formatTag + * @param formatVersion + * @param origin + * @param isPublic + * @param fileInputStream + * @param contentDispositionHeader + * @param formDataBodyPart + * @return + */ + @Path("datafile/{fileId}/auxiliary/{formatTag}/{formatVersion}") + @DELETE + public Response deleteAuxiliaryFileWithVersion(@PathParam("fileId") Long fileId, + @PathParam("formatTag") String formatTag, + @PathParam("formatVersion") String formatVersion) { + AuthenticatedUser authenticatedUser; + try { + authenticatedUser = findAuthenticatedUserOrDie(); + } catch (WrappedResponse ex) { + return error(FORBIDDEN, "Authorized users only."); + } + + DataFile dataFile = dataFileService.find(fileId); + if (dataFile == null) { + return error(BAD_REQUEST, "File not found based on id " + fileId + "."); + } + + if (!permissionService.userOn(authenticatedUser, dataFile.getOwner()).has(Permission.EditDataset)) { + return error(FORBIDDEN, "User not authorized to edit the dataset."); + } + + try { + auxiliaryFileService.deleteAuxiliaryFile(dataFile, formatTag, formatVersion); + } catch (FileNotFoundException e) { + throw new NotFoundException(); + } catch(IOException io) { + throw new ServerErrorException("IO Exception trying remove auxiliary file", Response.Status.INTERNAL_SERVER_ERROR, io); + } + + return ok("Auxiliary file deleted."); + } + /** diff --git a/src/main/java/edu/harvard/iq/dataverse/api/DownloadInstanceWriter.java b/src/main/java/edu/harvard/iq/dataverse/api/DownloadInstanceWriter.java index 80b3f988953..2d0964a4e2a 100644 --- a/src/main/java/edu/harvard/iq/dataverse/api/DownloadInstanceWriter.java +++ b/src/main/java/edu/harvard/iq/dataverse/api/DownloadInstanceWriter.java @@ -577,11 +577,18 @@ private boolean isAuxiliaryObjectCached(StorageIO storageIO, String auxiliaryTag } } - private String getFileExtension(AuxiliaryFile auxFile) { + public String getFileExtension(AuxiliaryFile auxFile) { String fileExtension = ""; if (auxFile == null) { return fileExtension; } + String filename = auxFile.getFilename(); + if (filename != null) { + String extension = FileUtil.getFileExtension(filename); + if (extension != null) { + return "." + extension; + } + } String contentType = auxFile.getContentType(); if (contentType != null) { MimeTypes allTypes = MimeTypes.getDefaultMimeTypes(); diff --git a/src/main/resources/db/migration/V5.8.0.2__8241-ext.sql b/src/main/resources/db/migration/V5.8.0.2__8241-ext.sql new file mode 100644 index 00000000000..9bf1c38fb2f --- /dev/null +++ b/src/main/resources/db/migration/V5.8.0.2__8241-ext.sql @@ -0,0 +1 @@ +ALTER TABLE auxiliaryfile ADD COLUMN IF NOT EXISTS filename VARCHAR(255); diff --git a/src/test/java/edu/harvard/iq/dataverse/AuxiliaryFileServiceBeanTest.java b/src/test/java/edu/harvard/iq/dataverse/AuxiliaryFileServiceBeanTest.java index 55c1dfb9b20..ad97eba137c 100644 --- a/src/test/java/edu/harvard/iq/dataverse/AuxiliaryFileServiceBeanTest.java +++ b/src/test/java/edu/harvard/iq/dataverse/AuxiliaryFileServiceBeanTest.java @@ -4,11 +4,15 @@ import java.util.List; import javax.persistence.EntityManager; import javax.persistence.Query; +import javax.persistence.TypedQuery; + import static org.junit.Assert.assertEquals; import org.junit.Test; import org.junit.Before; import org.junit.jupiter.api.extension.ExtendWith; import org.mockito.ArgumentMatchers; +import org.mockito.Matchers; + import static org.mockito.Mockito.mock; import static org.mockito.Mockito.when; import org.mockito.junit.jupiter.MockitoExtension; @@ -18,7 +22,7 @@ public class AuxiliaryFileServiceBeanTest { EntityManager em; AuxiliaryFileServiceBean svc; - Query query; + TypedQuery query; List types; DataFile dataFile; @@ -26,7 +30,7 @@ public class AuxiliaryFileServiceBeanTest { public void setup() { svc = new AuxiliaryFileServiceBean(); svc.em = mock(EntityManager.class); - query = mock(Query.class); + query = mock(TypedQuery.class); types = Arrays.asList("DP", "GEOSPATIAL", "NEXT_BIG_THING", "FUTURE_FUN"); dataFile = new DataFile(); dataFile.setId(42l); @@ -36,7 +40,7 @@ public void setup() { public void testFindAuxiliaryFileTypesInBundleFalse() { System.out.println("testFindAuxiliaryFileTypesInBundleFalse"); boolean inBundle = false; - when(this.svc.em.createNamedQuery(ArgumentMatchers.anyString())).thenReturn(query); + when(this.svc.em.createNamedQuery(ArgumentMatchers.anyString(),ArgumentMatchers.>any())).thenReturn(query); when(query.getResultList()).thenReturn(types); List result = svc.findAuxiliaryFileTypes(dataFile, inBundle); // None of these are in the bundle. @@ -48,7 +52,7 @@ public void testFindAuxiliaryFileTypesInBundleFalse() { public void testFindAuxiliaryFileTypesInBundleTrue() { System.out.println("testFindAuxiliaryFileTypesInBundleTrue"); boolean inBundle = true; - when(this.svc.em.createNamedQuery(ArgumentMatchers.anyString())).thenReturn(query); + when(this.svc.em.createNamedQuery(ArgumentMatchers.anyString(),ArgumentMatchers.>any())).thenReturn(query); when(query.getResultList()).thenReturn(types); List result = svc.findAuxiliaryFileTypes(dataFile, inBundle); // DP is in the bundle. diff --git a/src/test/java/edu/harvard/iq/dataverse/api/AuxiliaryFilesIT.java b/src/test/java/edu/harvard/iq/dataverse/api/AuxiliaryFilesIT.java index 4cc688c8758..96e4bcc87a6 100644 --- a/src/test/java/edu/harvard/iq/dataverse/api/AuxiliaryFilesIT.java +++ b/src/test/java/edu/harvard/iq/dataverse/api/AuxiliaryFilesIT.java @@ -7,8 +7,10 @@ import java.io.IOException; import java.nio.file.Path; import java.nio.file.Paths; +import static javax.ws.rs.core.Response.Status.CONFLICT; import static javax.ws.rs.core.Response.Status.CREATED; import static javax.ws.rs.core.Response.Status.FORBIDDEN; +import static javax.ws.rs.core.Response.Status.NOT_FOUND; import static javax.ws.rs.core.Response.Status.OK; import static org.hamcrest.CoreMatchers.equalTo; import org.junit.Assert; @@ -82,6 +84,10 @@ public void testUploadAuxFiles() throws IOException { .body("data.type", equalTo("DP")) // FIXME: application/json would be better .body("data.contentType", equalTo("text/plain")); + Response uploadAuxFileJsonAgain = UtilIT.uploadAuxFile(fileId, pathToAuxFileJson.toString(), formatTagJson, formatVersionJson, mimeTypeJson, true, dpType, apiToken); + uploadAuxFileJsonAgain.prettyPrint(); + uploadAuxFileJsonAgain.then().assertThat() + .statusCode(CONFLICT.getStatusCode()); // XML aux file Path pathToAuxFileXml = Paths.get(java.nio.file.Files.createTempDirectory(null) + File.separator + "data.xml"); @@ -204,17 +210,31 @@ public void testUploadAuxFiles() throws IOException { .body("data.type", equalTo(null)) .body("data.contentType", equalTo("text/plain")); + // file with an origin + Path pathToAuxFileOrigin1 = Paths.get(java.nio.file.Files.createTempDirectory(null) + File.separator + "file1.txt"); + String contentOfOrigin1 = "This file has an origin."; + java.nio.file.Files.write(pathToAuxFileOrigin1, contentOfOrigin1.getBytes()); + String formatTagOrigin1 = "origin1"; + String formatVersionOrigin1 = "0.1"; + String mimeTypeOrigin1 = "text/plain"; + String typeOrigin1 = "someType"; + String origin1 = "myApp"; + Response uploadAuxFileOrigin1 = UtilIT.uploadAuxFile(fileId, pathToAuxFileOrigin1.toString(), formatTagOrigin1, formatVersionOrigin1, mimeTypeOrigin1, true, typeOrigin1, origin1, apiToken); + uploadAuxFileOrigin1.prettyPrint(); + uploadAuxFileOrigin1.then().assertThat() + .statusCode(OK.getStatusCode()) + .body("data.type", equalTo("someType")) + .body("data.contentType", equalTo("text/plain")); + // Download JSON aux file. Response downloadAuxFileJson = UtilIT.downloadAuxFile(fileId, formatTagJson, formatVersionJson, apiToken); downloadAuxFileJson.then().assertThat().statusCode(OK.getStatusCode()); - // FIXME: This should be ".json" instead of ".txt" - Assert.assertEquals("attachment; filename=\"data.tab.dpJson_0.1.txt\"", downloadAuxFileJson.header("Content-disposition")); + Assert.assertEquals("attachment; filename=\"data.tab.dpJson_0.1.json\"", downloadAuxFileJson.header("Content-disposition")); // Download XML aux file. Response downloadAuxFileXml = UtilIT.downloadAuxFile(fileId, formatTagXml, formatVersionXml, apiToken); downloadAuxFileXml.then().assertThat().statusCode(OK.getStatusCode()); - // FIXME: This should be ".xml" instead of ".txt" - Assert.assertEquals("attachment; filename=\"data.tab.dpXml_0.1.txt\"", downloadAuxFileXml.header("Content-disposition")); + Assert.assertEquals("attachment; filename=\"data.tab.dpXml_0.1.xml\"", downloadAuxFileXml.header("Content-disposition")); // Download PDF aux file. Response downloadAuxFilePdf = UtilIT.downloadAuxFile(fileId, formatTagPdf, formatVersionPdf, apiToken); @@ -224,7 +244,7 @@ public void testUploadAuxFiles() throws IOException { // Download Markdown aux file. Response downloadAuxFileMd = UtilIT.downloadAuxFile(fileId, formatTagMd, formatVersionMd, apiToken); downloadAuxFileMd.then().assertThat().statusCode(OK.getStatusCode()); - Assert.assertEquals("attachment; filename=\"data.tab.README_0.1.txt\"", downloadAuxFileMd.header("Content-disposition")); + Assert.assertEquals("attachment; filename=\"data.tab.README_0.1.md\"", downloadAuxFileMd.header("Content-disposition")); Response createUserNoPrivs = UtilIT.createRandomUser(); createUserNoPrivs.then().assertThat().statusCode(OK.getStatusCode()); @@ -252,5 +272,31 @@ public void testUploadAuxFiles() throws IOException { Response failToDownloadAuxFileJsonPostPublish = UtilIT.downloadAuxFile(fileId, formatTagJson, formatVersionJson, apiTokenNoPrivs); failToDownloadAuxFileJsonPostPublish.then().assertThat().statusCode(OK.getStatusCode()); + Response listAuxFilesOrigin1 = UtilIT.listAuxFilesByOrigin(fileId, origin1, apiToken); + listAuxFilesOrigin1.prettyPrint(); + listAuxFilesOrigin1.then().assertThat() + .statusCode(OK.getStatusCode()) + .body("data[0].formatTag", equalTo("origin1")) + .body("data[0].formatVersion", equalTo("0.1")) + .body("data[0].fileSize", equalTo(24)) + .body("data[0].contentType", equalTo("text/plain")) + .body("data[0].isPublic", equalTo(true)) + .body("data[0].type", equalTo("someType")); + + Response listAllAuxFiles = UtilIT.listAllAuxFiles(fileId, apiToken); + listAllAuxFiles.prettyPrint(); + listAllAuxFiles.then().assertThat() + .statusCode(OK.getStatusCode()) + .body("data.size()", equalTo(8)); + + + Response deleteAuxFileOrigin1 = UtilIT.deleteAuxFile(fileId, formatTagOrigin1, formatVersionOrigin1, apiToken); + deleteAuxFileOrigin1.prettyPrint(); + deleteAuxFileOrigin1.then().assertThat().statusCode(OK.getStatusCode()); + + Response listAuxFilesOrigin1NowGone = UtilIT.listAuxFilesByOrigin(fileId, origin1, apiToken); + listAuxFilesOrigin1NowGone.prettyPrint(); + listAuxFilesOrigin1NowGone.then().assertThat().statusCode(NOT_FOUND.getStatusCode()); + } } diff --git a/src/test/java/edu/harvard/iq/dataverse/api/DownloadInstanceWriterTest.java b/src/test/java/edu/harvard/iq/dataverse/api/DownloadInstanceWriterTest.java index 6de52951077..d7bdc28c8b0 100644 --- a/src/test/java/edu/harvard/iq/dataverse/api/DownloadInstanceWriterTest.java +++ b/src/test/java/edu/harvard/iq/dataverse/api/DownloadInstanceWriterTest.java @@ -1,5 +1,6 @@ package edu.harvard.iq.dataverse.api; +import edu.harvard.iq.dataverse.AuxiliaryFile; import edu.harvard.iq.dataverse.dataaccess.Range; import java.util.List; import static org.junit.Assert.assertEquals; @@ -145,4 +146,19 @@ public void testGetRanges0to0and90toNull() { assertNotNull(expectedException); } + @Test + public void testGetAuxFileExtension() { + AuxiliaryFile auxFile = null; + assertEquals("", diw.getFileExtension(auxFile)); + auxFile = new AuxiliaryFile(); + assertEquals("", diw.getFileExtension(auxFile)); + auxFile.setContentType("text/plain"); + assertEquals(".txt", diw.getFileExtension(auxFile)); + auxFile.setFilename("foo.json"); + assertEquals(".json", diw.getFileExtension(auxFile)); + auxFile.setFilename("MANIFEST.TXT"); + // .TXT becomes .txt due to FileUtil.getFileExtension doing toLowerCase + assertEquals(".txt", diw.getFileExtension(auxFile)); + } + } diff --git a/src/test/java/edu/harvard/iq/dataverse/api/UtilIT.java b/src/test/java/edu/harvard/iq/dataverse/api/UtilIT.java index 03145f4c01b..9e900a945be 100644 --- a/src/test/java/edu/harvard/iq/dataverse/api/UtilIT.java +++ b/src/test/java/edu/harvard/iq/dataverse/api/UtilIT.java @@ -649,6 +649,11 @@ static Response uploadFileViaNative(String datasetId, String pathToFile, String } static Response uploadAuxFile(Long fileId, String pathToFile, String formatTag, String formatVersion, String mimeType, boolean isPublic, String type, String apiToken) { + String nullOrigin = null; + return uploadAuxFile(fileId, pathToFile, formatTag, formatVersion, mimeType, isPublic, type, nullOrigin, apiToken); + } + + static Response uploadAuxFile(Long fileId, String pathToFile, String formatTag, String formatVersion, String mimeType, boolean isPublic, String type, String origin, String apiToken) { RequestSpecification requestSpecification = given() .header(API_TOKEN_HTTP_HEADER, apiToken) .multiPart("file", new File(pathToFile), mimeType) @@ -656,6 +661,9 @@ static Response uploadAuxFile(Long fileId, String pathToFile, String formatTag, if (type != null) { requestSpecification.multiPart("type", type); } + if (origin != null) { + requestSpecification.multiPart("origin", origin); + } return requestSpecification.post("/api/access/datafile/" + fileId + "/auxiliary/" + formatTag + "/" + formatVersion); } @@ -666,6 +674,28 @@ static Response downloadAuxFile(Long fileId, String formatTag, String formatVers return response; } + static Response listAuxFilesByOrigin(Long fileId, String origin, String apiToken) { + Response response = given() + .header(API_TOKEN_HTTP_HEADER, apiToken) + .get("/api/access/datafile/" + fileId + "/auxiliary/" + origin); + return response; + } + + static Response listAllAuxFiles(Long fileId, String apiToken) { + Response response = given() + .header(API_TOKEN_HTTP_HEADER, apiToken) + .get("/api/access/datafile/" + fileId + "/auxiliary"); + return response; + } + + + static Response deleteAuxFile(Long fileId, String formatTag, String formatVersion, String apiToken) { + Response response = given() + .header(API_TOKEN_HTTP_HEADER, apiToken) + .delete("/api/access/datafile/" + fileId + "/auxiliary/" + formatTag + "/" + formatVersion); + return response; + } + static Response getCrawlableFileAccess(String datasetId, String folderName, String apiToken) { RequestSpecification requestSpecification = given() .header(API_TOKEN_HTTP_HEADER, apiToken);