diff --git a/conf/docker-aio/run-test-suite.sh b/conf/docker-aio/run-test-suite.sh index 47a4c3b9576..bf0683fdbd4 100755 --- a/conf/docker-aio/run-test-suite.sh +++ b/conf/docker-aio/run-test-suite.sh @@ -8,4 +8,4 @@ fi # Please note the "dataverse.test.baseurl" is set to run for "all-in-one" Docker environment. # TODO: Rather than hard-coding the list of "IT" classes here, add a profile to pom.xml. -source maven/maven.sh && mvn test -Dtest=DataversesIT,DatasetsIT,SwordIT,AdminIT,BuiltinUsersIT,UsersIT,UtilIT,ConfirmEmailIT,FileMetadataIT,FilesIT,SearchIT,InReviewWorkflowIT,HarvestingServerIT,MoveIT,MakeDataCountApiIT,FileTypeDetectionIT,EditDDIIT,ExternalToolsIT,AccessIT,DuplicateFilesIT,DownloadFilesIT,LinkIT,DeleteUsersIT,DeactivateUsersIT -Ddataverse.test.baseurl=$dvurl +source maven/maven.sh && mvn test -Dtest=DataversesIT,DatasetsIT,SwordIT,AdminIT,BuiltinUsersIT,UsersIT,UtilIT,ConfirmEmailIT,FileMetadataIT,FilesIT,SearchIT,InReviewWorkflowIT,HarvestingServerIT,MoveIT,MakeDataCountApiIT,FileTypeDetectionIT,EditDDIIT,ExternalToolsIT,AccessIT,DuplicateFilesIT,DownloadFilesIT,LinkIT,DeleteUsersIT,DeactivateUsersIT,AuxiliaryFilesIT -Ddataverse.test.baseurl=$dvurl diff --git a/doc/release-notes/7400-opendp-download.md b/doc/release-notes/7400-opendp-download.md new file mode 100644 index 00000000000..64e33250e1a --- /dev/null +++ b/doc/release-notes/7400-opendp-download.md @@ -0,0 +1,12 @@ +Auxiliary Files can now be downloaded from the web interface. + +- Aux files uploaded as type=DP appear under "Differentially Private Statistics" under file level download. The rest appear under "Other Auxiliary Files". + +In addition, related changes were made, including the following: + +- New tooltip over the lock indicating if you have been granted access to a restricted file or not. +- When downloading individual files, you will see "Restricted with Access Granted" or just "Restricted" (followed by "Users may not request access to files.") as appropriate. +- When downloading individual files, instead of "Download" you should expect to see the file type such as "JPEG Image" or "Original File Format" if the type is unknown. +- Downloaded aux files now have a file extension if it can be determined. + +Please note that the auxiliary files feature is experimental and if you don't need it, its API endpoints can be blocked. diff --git a/doc/sphinx-guides/source/developers/aux-file-support.rst b/doc/sphinx-guides/source/developers/aux-file-support.rst index 260c4493590..20b08bb9456 100644 --- a/doc/sphinx-guides/source/developers/aux-file-support.rst +++ b/doc/sphinx-guides/source/developers/aux-file-support.rst @@ -1,11 +1,11 @@ Auxiliary File Support ====================== -Auxiliary file support is experimental. Auxiliary files in the Dataverse Software are being added to support depositing and downloading differentially private metadata, as part of the OpenDP project (OpenDP.io). In future versions, this approach may become more broadly used and supported. +Auxiliary file support is experimental and as such, related APIs may be added, changed or removed without standard backward compatibility. Auxiliary files in the Dataverse Software are being added to support depositing and downloading differentially private metadata, as part of the OpenDP project (opendp.org). In future versions, this approach will likely become more broadly used and supported. Adding an Auxiliary File to a Datafile -------------------------------------- -To add an auxiliary file, specify the primary key of the datafile (FILE_ID), and the formatTag and formatVersion (if applicable) associated with the auxiliary file. There are two form parameters. "Origin" specifies the application/entity that created the auxiliary file, an "isPublic" controls access to downloading the file. If "isPublic" is true, any user can download the file, else, access authorization is based on the access rules as defined for the DataFile itself. +To add an auxiliary file, specify the primary key of the datafile (FILE_ID), and the formatTag and formatVersion (if applicable) associated with the auxiliary file. There are multiple form parameters. "Origin" specifies the application/entity that created the auxiliary file, and "isPublic" controls access to downloading the file. If "isPublic" is true, any user can download the file if the dataset has been published, else, access authorization is based on the access rules as defined for the DataFile itself. The "type" parameter is used to group similar auxiliary files in the UI. Currently, auxiliary files with type "DP" appear under "Differentially Private Statistics", while all other auxiliary files appear under "Other Auxiliary Files". .. code-block:: bash @@ -14,9 +14,10 @@ To add an auxiliary file, specify the primary key of the datafile (FILE_ID), and export FILE_ID='12345' export FORMAT_TAG='dpJson' export FORMAT_VERSION='v1' + export TYPE='DP' export SERVER_URL=https://demo.dataverse.org - curl -H X-Dataverse-key:$API_TOKEN -X POST -F "file=@$FILENAME" -F 'origin=myApp' -F 'isPublic=true' "$SERVER_URL/api/access/datafile/$FILE_ID/metadata/$FORMAT_TAG/$FORMAT_VERSION" + curl -H X-Dataverse-key:$API_TOKEN -X POST -F "file=@$FILENAME" -F 'origin=myApp' -F 'isPublic=true' -F "type=$TYPE" "$SERVER_URL/api/access/datafile/$FILE_ID/metadata/$FORMAT_TAG/$FORMAT_VERSION" You should expect a 200 ("OK") response and JSON with information about your newly uploaded auxiliary file. @@ -33,4 +34,4 @@ formatTag and formatVersion (if applicable) associated with the auxiliary file: export FORMAT_TAG='dpJson' export FORMAT_VERSION='v1' - curl "$SERVER_URL/api/access/datafile/$FILE_ID/$FORMAT_TAG/$FORMAT_VERSION" + curl "$SERVER_URL/api/access/datafile/$FILE_ID/metadata/$FORMAT_TAG/$FORMAT_VERSION" diff --git a/doc/sphinx-guides/source/user/dataset-management.rst b/doc/sphinx-guides/source/user/dataset-management.rst index 1d5f1344c6c..db7c813b8c2 100755 --- a/doc/sphinx-guides/source/user/dataset-management.rst +++ b/doc/sphinx-guides/source/user/dataset-management.rst @@ -179,6 +179,8 @@ Additional download options available for tabular data (found in the same drop-d - Data File Citation (currently in either RIS, EndNote XML, or BibTeX format); - All of the above, as a zipped bundle. +Differentially Private (DP) Metadata can also be accessed for restricted tabular files if the data depositor has created a DP Metadata Release. See :ref:`dp-release-create` for more information. + Astronomy (FITS) ---------------- @@ -210,6 +212,8 @@ Restricted Files When you restrict a file it cannot be downloaded unless permission has been granted. +Differentially Private (DP) Metadata can be accessed for restricted tabular files if the data depositor has created a DP Metadata Release. See :ref:`dp-release-create` for more information. + See also :ref:`terms-of-access` and :ref:`permissions`. Edit Files @@ -302,6 +306,23 @@ If you restrict any files in your dataset, you will be prompted by a pop-up to e See also :ref:`restricted-files`. +.. _dp-release-create: + +Creating and Depositing Differentially Private Metadata (Experimental) +---------------------------------------------------------------------- + +Through an integration with tools from the OpenDP Project (opendp.org), the Dataverse Software offers an experimental workflow that allows a data depositor to create and deposit Differentially Private (DP) Metadata files, which can then be used for exploratory data analysis. This workflow allows researchers to view the DP metadata for a tabular file, determine whether or not the file contains useful information, and then make an informed decision about whether or not to request access to the original file. + +If this integration has been enabled in your Dataverse installation, you can follow these steps to create a DP Metadata Release and make it available to researchers, while still keeping the files themselves restricted and able to be accessed after a successful access request. + +- Deposit a tabular file and let the ingest process complete +- Restrict the File +- In the kebab next to the file on the dataset page, or from the "Edit Files" dropdown on the file page, click "OpenDP Tool" +- Go through the process to create a DP Metadata Release in the OpenDP tool, and at the end of the process deposit the DP Metadata Release back to the Dataverse installation +- Publish the Dataset + +Once the dataset is published, users will be able to request access using the normal process, but will also have the option to download DP Statistics in order to get more information about the file. + Guestbook --------- diff --git a/doc/sphinx-guides/source/user/find-use-data.rst b/doc/sphinx-guides/source/user/find-use-data.rst index 97b7694b9f2..42e1a2b23d4 100755 --- a/doc/sphinx-guides/source/user/find-use-data.rst +++ b/doc/sphinx-guides/source/user/find-use-data.rst @@ -153,6 +153,19 @@ Explore Data Some file types and datasets offer data exploration options if external tools have been installed. The tools are described in the :doc:`/admin/external-tools` section of the Admin Guide. +Exploratory Data Analysis Using Differentially Private Metadata (Experimental) +------------------------------------------------------------------------------ + +Through an integration with tools from the OpenDP Project (opendp.org), the Dataverse Software offers an experimental workflow that allows a data depositor to create and deposit Differentially Private (DP) Metadata files, which can then be used for exploratory data analysis. This workflow allows researchers to view the DP metadata for a tabular file, determine whether or not the file contains useful information, and then make an informed decision about whether or not to request access to the original file. + +If the data depositor has made available DP metadata for one or more files in their dataset, these access options will appear on the access dropdown on both the Dataset Page and the File Page. These access options will be available even if a file is restricted. Three types of DP metadata will be available: + +- .PDF +- .XML +- .JSON + +For more information about how data depositors can enable access using the OpenDP tool, visit the :doc:`/user/dataset-management` section of the User Guide. + .. |image-file-tree-view| image:: ./img/file-tree-view.png :class: img-responsive .. |image-file-search-facets| image:: ./img/file-search-facets.png diff --git a/src/main/java/edu/harvard/iq/dataverse/AuxiliaryFile.java b/src/main/java/edu/harvard/iq/dataverse/AuxiliaryFile.java index 957a7cc93bf..cbc29217c7f 100644 --- a/src/main/java/edu/harvard/iq/dataverse/AuxiliaryFile.java +++ b/src/main/java/edu/harvard/iq/dataverse/AuxiliaryFile.java @@ -1,13 +1,19 @@ package edu.harvard.iq.dataverse; +import edu.harvard.iq.dataverse.util.BundleUtil; import java.io.Serializable; +import java.util.MissingResourceException; import javax.persistence.Entity; import javax.persistence.GeneratedValue; import javax.persistence.GenerationType; import javax.persistence.Id; import javax.persistence.JoinColumn; import javax.persistence.ManyToOne; +import javax.persistence.NamedNativeQueries; +import javax.persistence.NamedNativeQuery; +import javax.persistence.NamedQueries; +import javax.persistence.NamedQuery; /** * @@ -15,6 +21,19 @@ * Represents a generic file that is associated with a dataFile. * This is a data representation of a physical file in StorageIO */ +@NamedQueries({ + @NamedQuery(name = "AuxiliaryFile.lookupAuxiliaryFile", + query = "select object(o) from AuxiliaryFile as o where o.dataFile.id = :dataFileId and o.formatTag = :formatTag and o.formatVersion = :formatVersion"), + @NamedQuery(name = "AuxiliaryFile.findAuxiliaryFiles", + query = "select object(o) from AuxiliaryFile as o where o.dataFile.id = :dataFileId"), + @NamedQuery(name = "AuxiliaryFile.findAuxiliaryFilesByType", + query = "select object(o) from AuxiliaryFile as o where o.dataFile.id = :dataFileId and o.type = :type"), + @NamedQuery(name = "AuxiliaryFile.findAuxiliaryFilesWithoutType", + query = "select object(o) from AuxiliaryFile as o where o.dataFile.id = :dataFileId and o.type is null"),}) +@NamedNativeQueries({ + @NamedNativeQuery(name = "AuxiliaryFile.findAuxiliaryFileTypes", + query = "select distinct type from auxiliaryfile where datafile_id = ?1") +}) @Entity public class AuxiliaryFile implements Serializable { @@ -44,6 +63,12 @@ public class AuxiliaryFile implements Serializable { private String checksum; + /** + * A way of grouping similar auxiliary files together. The type could be + * "DP" for "Differentially Private Statistics", for example. + */ + private String type; + public Long getId() { return id; } @@ -115,6 +140,21 @@ public String getChecksum() { public void setChecksum(String checksum) { this.checksum = checksum; } - - + + public String getType() { + return type; + } + + public void setType(String type) { + this.type = type; + } + + public String getTypeFriendly() { + try { + return BundleUtil.getStringFromPropertyFile("file.auxfiles.types." + type, "Bundle"); + } catch (MissingResourceException ex) { + return null; + } + } + } diff --git a/src/main/java/edu/harvard/iq/dataverse/AuxiliaryFileServiceBean.java b/src/main/java/edu/harvard/iq/dataverse/AuxiliaryFileServiceBean.java index 4f97c146e7b..0643c3622d4 100644 --- a/src/main/java/edu/harvard/iq/dataverse/AuxiliaryFileServiceBean.java +++ b/src/main/java/edu/harvard/iq/dataverse/AuxiliaryFileServiceBean.java @@ -8,6 +8,8 @@ import java.io.InputStream; import java.security.DigestInputStream; import java.security.MessageDigest; +import java.util.ArrayList; +import java.util.List; import java.util.logging.Logger; import javax.ejb.EJB; import javax.ejb.Stateless; @@ -15,6 +17,7 @@ import javax.persistence.EntityManager; import javax.persistence.PersistenceContext; import javax.persistence.Query; +import javax.persistence.TypedQuery; import org.apache.tika.Tika; /** @@ -28,7 +31,7 @@ public class AuxiliaryFileServiceBean implements java.io.Serializable { private static final Logger logger = Logger.getLogger(AuxiliaryFileServiceBean.class.getCanonicalName()); @PersistenceContext(unitName = "VDCNet-ejbPU") - private EntityManager em; + protected EntityManager em; @EJB private SystemConfig systemConfig; @@ -54,9 +57,11 @@ public AuxiliaryFile save(AuxiliaryFile auxiliaryFile) { * @param formatVersion - to distinguish between multiple versions of a file * @param origin - name of the tool/system that created the file * @param isPublic boolean - is this file available to any user? + * @param type how to group the files such as "DP" for "Differentially + * Private Statistics". * @return success boolean - returns whether the save was successful */ - public AuxiliaryFile processAuxiliaryFile(InputStream fileInputStream, DataFile dataFile, String formatTag, String formatVersion, String origin, boolean isPublic) { + public AuxiliaryFile processAuxiliaryFile(InputStream fileInputStream, DataFile dataFile, String formatTag, String formatVersion, String origin, boolean isPublic, String type) { StorageIO storageIO =null; AuxiliaryFile auxFile = new AuxiliaryFile(); @@ -81,6 +86,7 @@ public AuxiliaryFile processAuxiliaryFile(InputStream fileInputStream, DataFile auxFile.setFormatVersion(formatVersion); auxFile.setOrigin(origin); auxFile.setIsPublic(isPublic); + auxFile.setType(type); auxFile.setDataFile(dataFile); auxFile.setFileSize(storageIO.getAuxObjectSize(auxExtension)); auxFile = save(auxFile); @@ -101,7 +107,7 @@ public AuxiliaryFile processAuxiliaryFile(InputStream fileInputStream, DataFile public AuxiliaryFile lookupAuxiliaryFile(DataFile dataFile, String formatTag, String formatVersion) { - Query query = em.createQuery("select object(o) from AuxiliaryFile as o where o.dataFile.id = :dataFileId and o.formatTag = :formatTag and o.formatVersion = :formatVersion"); + Query query = em.createNamedQuery("AuxiliaryFile.lookupAuxiliaryFile"); query.setParameter("dataFileId", dataFile.getId()); query.setParameter("formatTag", formatTag); @@ -114,4 +120,73 @@ public AuxiliaryFile lookupAuxiliaryFile(DataFile dataFile, String formatTag, St } } + public List findAuxiliaryFiles(DataFile dataFile) { + TypedQuery query = em.createNamedQuery("AuxiliaryFile.findAuxiliaryFiles", AuxiliaryFile.class); + query.setParameter("dataFileId", dataFile.getId()); + return query.getResultList(); + } + + /** + * @param inBundle If true, only return types that are in the bundle. If + * false, only return types that are not in the bundle. + */ + public List findAuxiliaryFileTypes(DataFile dataFile, boolean inBundle) { + List allTypes = findAuxiliaryFileTypes(dataFile); + List typesInBundle = new ArrayList<>(); + List typeNotInBundle = new ArrayList<>(); + for (String type : allTypes) { + // Check if type is in the bundle. + String friendlyType = getFriendlyNameForType(type); + if (friendlyType != null) { + typesInBundle.add(type); + } else { + typeNotInBundle.add(type); + } + } + if (inBundle) { + return typesInBundle; + } else { + return typeNotInBundle; + } + } + + public List findAuxiliaryFileTypes(DataFile dataFile) { + Query query = em.createNamedQuery("AuxiliaryFile.findAuxiliaryFileTypes"); + query.setParameter(1, dataFile.getId()); + return query.getResultList(); + } + + public List findAuxiliaryFilesByType(DataFile dataFile, String typeString) { + TypedQuery query = em.createNamedQuery("AuxiliaryFile.findAuxiliaryFilesByType", AuxiliaryFile.class); + query.setParameter("dataFileId", dataFile.getId()); + query.setParameter("type", typeString); + return query.getResultList(); + } + + public List findOtherAuxiliaryFiles(DataFile dataFile) { + List otherAuxFiles = new ArrayList<>(); + List otherTypes = findAuxiliaryFileTypes(dataFile, false); + for (String typeString : otherTypes) { + TypedQuery query = em.createNamedQuery("AuxiliaryFile.findAuxiliaryFilesByType", AuxiliaryFile.class); + query.setParameter("dataFileId", dataFile.getId()); + query.setParameter("type", typeString); + List auxFiles = query.getResultList(); + otherAuxFiles.addAll(auxFiles); + } + otherAuxFiles.addAll(findAuxiliaryFilesWithoutType(dataFile)); + return otherAuxFiles; + } + + public List findAuxiliaryFilesWithoutType(DataFile dataFile) { + Query query = em.createNamedQuery("AuxiliaryFile.findAuxiliaryFilesWithoutType", AuxiliaryFile.class); + query.setParameter("dataFileId", dataFile.getId()); + return query.getResultList(); + } + + public String getFriendlyNameForType(String type) { + AuxiliaryFile auxFile = new AuxiliaryFile(); + auxFile.setType(type); + return auxFile.getTypeFriendly(); + } + } diff --git a/src/main/java/edu/harvard/iq/dataverse/FileDownloadServiceBean.java b/src/main/java/edu/harvard/iq/dataverse/FileDownloadServiceBean.java index 4b0272d4a36..6f7d62924ac 100644 --- a/src/main/java/edu/harvard/iq/dataverse/FileDownloadServiceBean.java +++ b/src/main/java/edu/harvard/iq/dataverse/FileDownloadServiceBean.java @@ -280,6 +280,14 @@ private void redirectToBatchDownloadAPI(String multiFileString, Boolean download redirectToBatchDownloadAPI(multiFileString, true, downloadOriginal); } + public void redirectToAuxFileDownloadAPI(Long fileId, String formatTag, String formatVersion) { + String fileDownloadUrl = "/api/access/datafile/" + fileId + "/metadata/" + formatTag + "/" + formatVersion; + try { + FacesContext.getCurrentInstance().getExternalContext().redirect(fileDownloadUrl); + } catch (IOException ex) { + logger.info("Failed to issue a redirect to aux file download url (" + fileDownloadUrl + "): " + ex); + } + } /** * Launch an "explore" tool which is a type of ExternalTool such as diff --git a/src/main/java/edu/harvard/iq/dataverse/api/Access.java b/src/main/java/edu/harvard/iq/dataverse/api/Access.java index 6fc2f066b36..51dbfa6fb58 100644 --- a/src/main/java/edu/harvard/iq/dataverse/api/Access.java +++ b/src/main/java/edu/harvard/iq/dataverse/api/Access.java @@ -116,6 +116,9 @@ import javax.ws.rs.core.StreamingOutput; import static edu.harvard.iq.dataverse.util.json.JsonPrinter.json; import java.net.URISyntaxException; +import java.util.stream.Collectors; +import java.util.stream.Stream; +import javax.json.JsonObjectBuilder; import javax.ws.rs.RedirectionException; import javax.ws.rs.core.MediaType; import static javax.ws.rs.core.Response.Status.FORBIDDEN; @@ -541,6 +544,8 @@ public String dataVariableMetadataDDI(@PathParam("varId") Long varId, @QueryPara /* * GET method for retrieving various auxiliary files associated with * a tabular datafile. + * + * TODO: Consider removing "metadata" from the path. */ @Path("datafile/{fileId}/metadata/{formatTag}/{formatVersion}") @@ -587,8 +592,9 @@ public DownloadInstance tabularDatafileMetadataAux(@PathParam("fileId") String f if (auxFile == null) { throw new NotFoundException("Auxiliary metadata format "+formatTag+" is not available for datafile "+fileId); } - - if (auxFile.getIsPublic()) { + + // Don't consider aux file public unless data file is published. + if (auxFile.getIsPublic() && df.getPublicationDate() != null) { publiclyAvailable = true; } downloadInstance = new DownloadInstance(dInfo); @@ -1158,10 +1164,13 @@ private String getWebappImageResource(String imageName) { * @param formatVersion * @param origin * @param isPublic + * @param type * @param fileInputStream * @param contentDispositionHeader * @param formDataBodyPart * @return + * + * TODO: Consider removing "metadata" from the path. */ @Path("datafile/{fileId}/metadata/{formatTag}/{formatVersion}") @POST @@ -1172,6 +1181,7 @@ public Response saveAuxiliaryFileWithVersion(@PathParam("fileId") Long fileId, @PathParam("formatVersion") String formatVersion, @FormDataParam("origin") String origin, @FormDataParam("isPublic") boolean isPublic, + @FormDataParam("type") String type, @FormDataParam("file") InputStream fileInputStream ) { @@ -1194,9 +1204,8 @@ public Response saveAuxiliaryFileWithVersion(@PathParam("fileId") Long fileId, if (!dataFile.isTabularData()) { return error(BAD_REQUEST, "Not a tabular DataFile (db id=" + fileId + ")"); } - - AuxiliaryFile saved = auxiliaryFileService.processAuxiliaryFile(fileInputStream, dataFile, formatTag, formatVersion, origin, isPublic); + AuxiliaryFile saved = auxiliaryFileService.processAuxiliaryFile(fileInputStream, dataFile, formatTag, formatVersion, origin, isPublic, type); if (saved!=null) { return ok(json(saved)); diff --git a/src/main/java/edu/harvard/iq/dataverse/api/DownloadInstanceWriter.java b/src/main/java/edu/harvard/iq/dataverse/api/DownloadInstanceWriter.java index 39bcbabbc99..46dc0512575 100644 --- a/src/main/java/edu/harvard/iq/dataverse/api/DownloadInstanceWriter.java +++ b/src/main/java/edu/harvard/iq/dataverse/api/DownloadInstanceWriter.java @@ -6,6 +6,7 @@ package edu.harvard.iq.dataverse.api; +import edu.harvard.iq.dataverse.AuxiliaryFile; import java.lang.reflect.Type; import java.lang.annotation.Annotation; import java.io.InputStream; @@ -42,6 +43,9 @@ import javax.ws.rs.NotFoundException; import javax.ws.rs.RedirectionException; import javax.ws.rs.ServiceUnavailableException; +import org.apache.tika.mime.MimeType; +import org.apache.tika.mime.MimeTypeException; +import org.apache.tika.mime.MimeTypes; /** * @@ -238,7 +242,8 @@ public void writeTo(DownloadInstance di, Class clazz, Type type, Annotation[] } long auxFileSize = di.getAuxiliaryFile().getFileSize(); InputStreamIO auxStreamIO = new InputStreamIO(storageIO.getAuxFileAsInputStream(auxTag), auxFileSize); - auxStreamIO.setFileName(storageIO.getFileName() + "." + auxTag); + String fileExtension = getFileExtension(di.getAuxiliaryFile()); + auxStreamIO.setFileName(storageIO.getFileName() + "." + auxTag + fileExtension); auxStreamIO.setMimeType(di.getAuxiliaryFile().getContentType()); storageIO = auxStreamIO; @@ -388,7 +393,24 @@ public void writeTo(DownloadInstance di, Class clazz, Type type, Annotation[] throw new NotFoundException(); } - + + private String getFileExtension(AuxiliaryFile auxFile) { + String fileExtension = ""; + if (auxFile == null) { + return fileExtension; + } + String contentType = auxFile.getContentType(); + if (contentType != null) { + MimeTypes allTypes = MimeTypes.getDefaultMimeTypes(); + try { + MimeType mimeType = allTypes.forName(contentType); + fileExtension = mimeType.getExtension(); + } catch (MimeTypeException ex) { + } + } + return fileExtension; + } + private boolean isThumbnailDownload(DownloadInstance downloadInstance) { if (downloadInstance == null) return false; diff --git a/src/main/java/edu/harvard/iq/dataverse/util/json/JsonPrinter.java b/src/main/java/edu/harvard/iq/dataverse/util/json/JsonPrinter.java index 3a1d61c6554..4302d932f3c 100644 --- a/src/main/java/edu/harvard/iq/dataverse/util/json/JsonPrinter.java +++ b/src/main/java/edu/harvard/iq/dataverse/util/json/JsonPrinter.java @@ -564,6 +564,8 @@ public static JsonObjectBuilder json(AuxiliaryFile auxFile) { .add("formatVersion", auxFile.getFormatVersion()) // "label" is the filename .add("origin", auxFile.getOrigin()) .add("isPublic", auxFile.getIsPublic()) + .add("type", auxFile.getType()) + .add("contentType", auxFile.getContentType()) .add("fileSize", auxFile.getFileSize()) .add("checksum", auxFile.getChecksum()) .add("dataFile", JsonPrinter.json(auxFile.getDataFile())); diff --git a/src/main/java/propertyFiles/Bundle.properties b/src/main/java/propertyFiles/Bundle.properties index ab5352c8efd..6b8ab6edcad 100644 --- a/src/main/java/propertyFiles/Bundle.properties +++ b/src/main/java/propertyFiles/Bundle.properties @@ -10,7 +10,7 @@ files=Files file=File public=Public restricted=Restricted -restrictedaccess=Restricted Access +restrictedaccess=Restricted with Access Granted find=Find search=Search language=Language @@ -1585,6 +1585,7 @@ file.metaData.checksum.copy=Click to copy file.metaData.dataFile.dataTab.unf=UNF file.metaData.dataFile.dataTab.variables=Variables file.metaData.dataFile.dataTab.observations=Observations +file.metaData.fileAccess=File Access: file.addDescription=Add file description... file.tags=Tags file.editTags=Edit Tags @@ -1620,10 +1621,11 @@ file.ingestFailed.header=Upload Completed with Errors file.ingestFailed.message=Tabular data ingest failed. file.downloadBtn.format.all=All File Formats + Information file.downloadBtn.format.tab=Tab-Delimited -file.downloadBtn.format.original=Original File Format ({0}) -file.downloadBtn.format.rdata=RData Format +file.downloadBtn.format.original={0} (Original File Format) +file.downloadBtn.format.rdata=RData file.downloadBtn.format.var=Variable Metadata file.downloadBtn.format.citation=Data File Citation +file.download.filetype.unknown=Original File Format file.more.information.link=Link to more file information for file.requestAccess=Request Access file.requestAccess.dialog.msg=You need to Log In to request access to this file. @@ -1805,6 +1807,13 @@ file.results.btn.sort.option.size=Size file.results.btn.sort.option.type=Type file.compute.fileAccessDenied=This file is restricted and you may not compute on it because you have not been granted access. file.configure.Button=Configure + +file.auxfiles.download.header=Download Auxiliary Files +# These types correspond to the AuxiliaryFile.Type enum. +file.auxfiles.types.DP=Differentially Private Statistics +# Add more types here +file.auxfiles.unspecifiedTypes=Other Auxiliary Files + # dataset-widgets.xhtml dataset.widgets.title=Dataset Thumbnail + Widgets dataset.widgets.notPublished.why.header=Why Use Widgets? diff --git a/src/main/resources/db/migration/V5.4.1.1__7400-opendp-download.sql b/src/main/resources/db/migration/V5.4.1.1__7400-opendp-download.sql new file mode 100644 index 00000000000..de1a18b1e01 --- /dev/null +++ b/src/main/resources/db/migration/V5.4.1.1__7400-opendp-download.sql @@ -0,0 +1 @@ +ALTER TABLE auxiliaryfile ADD COLUMN IF NOT EXISTS type character varying(255); diff --git a/src/main/webapp/dataset.xhtml b/src/main/webapp/dataset.xhtml index 0198e303b06..f53950d2ecf 100644 --- a/src/main/webapp/dataset.xhtml +++ b/src/main/webapp/dataset.xhtml @@ -37,10 +37,10 @@ or (DatasetPage.workingVersion.deaccessioned and DatasetPage.canUpdateDataset()))}"/> - + + diff --git a/src/main/webapp/file-download-button-fragment.xhtml b/src/main/webapp/file-download-button-fragment.xhtml index 85fe60863b4..ac8f7c28c40 100644 --- a/src/main/webapp/file-download-button-fragment.xhtml +++ b/src/main/webapp/file-download-button-fragment.xhtml @@ -10,10 +10,48 @@ xmlns:cc="http://java.sun.com/jsf/composite" xmlns:o="http://omnifaces.org/ui" xmlns:iqbs="http://xmlns.jcp.org/jsf/composite/iqbs"> + + +
  • File Access
    • +
  • + + +
    • - + + + + #{fileMetadata.dataFile.fileAccessRequesters.contains(dataverseSession.user) ? bundle['file.accessRequested'] : bundle['file.requestAccess']} + + +
  • + + #{bundle['file.requestAccess']} + +
    • +     + +
  • #{bundle['file.dataFilesTab.terms.list.termsOfAccess.requestAccess.notRequest']}
    • +     + + + +
  • #{bundle['file.accessBtn.header.download']}
    • +
  • - #{bundle.download} + #{fileMetadata.dataFile.friendlyType == 'Unknown' ? bundle['file.download.filetype.unknown'] : fileMetadata.dataFile.friendlyType} - #{bundle.download} + #{fileMetadata.dataFile.friendlyType == 'Unknown' ? bundle['file.download.filetype.unknown'] : fileMetadata.dataFile.friendlyType} - #{bundle.download} + #{fileMetadata.dataFile.friendlyType == 'Unknown' ? bundle['file.download.filetype.unknown'] : fileMetadata.dataFile.friendlyType} - #{bundle.download} + #{fileMetadata.dataFile.friendlyType == 'Unknown' ? bundle['file.download.filetype.unknown'] : fileMetadata.dataFile.friendlyType}
    • - +
  • @@ -131,93 +169,95 @@
    • +     + + + + + +
  • Download Metadata
    • + + +
  • + + #{bundle['file.downloadBtn.format.var']} + + + #{bundle['file.downloadBtn.format.var']} + +
    • +     + + +
  • + #{bundle['file.downloadBtn.format.citation']} +
    • - - #{bundle['file.downloadBtn.format.var']} - - - #{bundle['file.downloadBtn.format.var']} - + + + +
      • +
    • + + + +
      • +
    • + + +
      • +
    +
    • + + + + + + + +
  • #{bundle['file.auxfiles.download.header']}
    • + +
  • - #{bundle['file.downloadBtn.format.citation']} + #{auxiliaryFileServiceBean.getFriendlyNameForType(auxFileType)}
      -
    • - - - -
      • -
    • - - - -
      • -
    • - - - -
      • + +
    • + + + +
      • +
    • -     - + + +
  • + #{bundle['file.auxfiles.unspecifiedTypes']} +
      + +
    • + + + +
      • +       +
    +
    • - - - - -
  • #{bundle['file.accessBtn.header.download']}
    • - - - - - - #{fileMetadata.dataFile.fileAccessRequesters.contains(dataverseSession.user) ? bundle['file.accessRequested'] : bundle['file.requestAccess']} - - - -
  • - - #{bundle['file.requestAccess']} - -
    • -     - + + diff --git a/src/main/webapp/file-info-fragment.xhtml b/src/main/webapp/file-info-fragment.xhtml index 9a2add76147..542b9a37e6f 100644 --- a/src/main/webapp/file-info-fragment.xhtml +++ b/src/main/webapp/file-info-fragment.xhtml @@ -35,10 +35,10 @@ -
    +
    -
    +
    diff --git a/src/main/webapp/file.xhtml b/src/main/webapp/file.xhtml index 06ff420a9d1..0e4931e7086 100644 --- a/src/main/webapp/file.xhtml +++ b/src/main/webapp/file.xhtml @@ -18,10 +18,10 @@ - + + @@ -61,8 +61,8 @@
    - - + + diff --git a/src/main/webapp/resources/css/structure.css b/src/main/webapp/resources/css/structure.css index b0cc69f8628..0ffc336387e 100644 --- a/src/main/webapp/resources/css/structure.css +++ b/src/main/webapp/resources/css/structure.css @@ -757,6 +757,23 @@ div[id$="filesTable"] thead[id$="filesTable_head"] th.ui-selection-column .ui-ch word-break: break-word; } +/* REQUEST ACCESS DOWNLOAD OPTION LINK */ +div[id$="requestPanel"].iq-dropdown-list-item {display:list-item !important;} +div[id$="requestPanel"].iq-dropdown-list-item>a.ui-commandlink{display:block;padding:3px 20px;clear:both;font-weight:400;line-height:1.42857143;color:#333;white-space:nowrap} +div[id$="requestPanel"].iq-dropdown-list-item>a.ui-commandlink:focus, +div[id$="requestPanel"].iq-dropdown-list-item>a.ui-commandlink:hover{color:#262626;text-decoration:none;background-color:#f5f5f5} +div[id$="requestPanel"].iq-dropdown-list-item>a.ui-commandlink.active, +div[id$="requestPanel"].iq-dropdown-list-item>a.ui-commandlink.active:focus, +div[id$="requestPanel"].iq-dropdown-list-item>a.ui-commandlink.active:hover{color:#fff;text-decoration:none;background-color:#337ab7;outline:0} +div[id$="requestPanel"].iq-dropdown-list-item.disabled>span.ui-commandlink.ui-state-disabled{display:block;padding:3px 20px;clear:both;font-weight:400;line-height:1.42857143;white-space:nowrap} +div[id$="requestPanel"].iq-dropdown-list-item.disabled>span.ui-commandlink.ui-state-disabled, +div[id$="requestPanel"].iq-dropdown-list-item.disabled>span.ui-commandlink.ui-state-disabled:focus, +div[id$="requestPanel"].iq-dropdown-list-item.disabled>span.ui-commandlink.ui-state-disabled:hover{background-color:transparent;color:#777;} +div[id$="requestPanel"].iq-dropdown-list-item.disabled>span.ui-commandlink.ui-state-disabled:focus, +div[id$="requestPanel"].iq-dropdown-list-item.disabled>span.ui-commandlink.ui-state-disabled:hover{text-decoration:none;background-color:transparent;background-image:none;filter:progid:DXImageTransform.Microsoft.gradient(enabled=false)} +div[id$="requestPanel"].iq-dropdown-list-item.disabled{cursor:not-allowed;} +div[id$="requestPanel"].iq-dropdown-list-item.disabled>span.ui-commandlink.ui-state-disabled{pointer-events:none;} + /* PAGINATION */ div[id$="filesTable"] .ui-paginator { font-weight: normal; @@ -899,6 +916,7 @@ span.ui-autocomplete input.ui-autocomplete-input {width:100%;} /* Overwrite Bootstrap */ .btn, .input-group-btn {white-space: normal !important;} +.dropdown-item-text {display: block;padding: 3px 20px;color: #212529;} @media print { a[href]:after { diff --git a/src/main/webapp/search-include-fragment.xhtml b/src/main/webapp/search-include-fragment.xhtml index 0167a53ca48..f70b321cea4 100644 --- a/src/main/webapp/search-include-fragment.xhtml +++ b/src/main/webapp/search-include-fragment.xhtml @@ -581,8 +581,8 @@ - - + + diff --git a/src/test/java/edu/harvard/iq/dataverse/AuxiliaryFileServiceBeanTest.java b/src/test/java/edu/harvard/iq/dataverse/AuxiliaryFileServiceBeanTest.java new file mode 100644 index 00000000000..55c1dfb9b20 --- /dev/null +++ b/src/test/java/edu/harvard/iq/dataverse/AuxiliaryFileServiceBeanTest.java @@ -0,0 +1,59 @@ +package edu.harvard.iq.dataverse; + +import java.util.Arrays; +import java.util.List; +import javax.persistence.EntityManager; +import javax.persistence.Query; +import static org.junit.Assert.assertEquals; +import org.junit.Test; +import org.junit.Before; +import org.junit.jupiter.api.extension.ExtendWith; +import org.mockito.ArgumentMatchers; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; +import org.mockito.junit.jupiter.MockitoExtension; + +@ExtendWith(MockitoExtension.class) +public class AuxiliaryFileServiceBeanTest { + + EntityManager em; + AuxiliaryFileServiceBean svc; + Query query; + List types; + DataFile dataFile; + + @Before + public void setup() { + svc = new AuxiliaryFileServiceBean(); + svc.em = mock(EntityManager.class); + query = mock(Query.class); + types = Arrays.asList("DP", "GEOSPATIAL", "NEXT_BIG_THING", "FUTURE_FUN"); + dataFile = new DataFile(); + dataFile.setId(42l); + } + + @Test + public void testFindAuxiliaryFileTypesInBundleFalse() { + System.out.println("testFindAuxiliaryFileTypesInBundleFalse"); + boolean inBundle = false; + when(this.svc.em.createNamedQuery(ArgumentMatchers.anyString())).thenReturn(query); + when(query.getResultList()).thenReturn(types); + List result = svc.findAuxiliaryFileTypes(dataFile, inBundle); + // None of these are in the bundle. + List expected = Arrays.asList("GEOSPATIAL", "NEXT_BIG_THING", "FUTURE_FUN"); + assertEquals(expected, result); + } + + @Test + public void testFindAuxiliaryFileTypesInBundleTrue() { + System.out.println("testFindAuxiliaryFileTypesInBundleTrue"); + boolean inBundle = true; + when(this.svc.em.createNamedQuery(ArgumentMatchers.anyString())).thenReturn(query); + when(query.getResultList()).thenReturn(types); + List result = svc.findAuxiliaryFileTypes(dataFile, inBundle); + // DP is in the bundle. + List expected = Arrays.asList("DP"); + assertEquals(expected, result); + } + +} diff --git a/src/test/java/edu/harvard/iq/dataverse/api/AccessIT.java b/src/test/java/edu/harvard/iq/dataverse/api/AccessIT.java index 4fb1271c8c9..1945804c897 100644 --- a/src/test/java/edu/harvard/iq/dataverse/api/AccessIT.java +++ b/src/test/java/edu/harvard/iq/dataverse/api/AccessIT.java @@ -103,6 +103,10 @@ public static void setUp() throws InterruptedException { Response createDatasetResponse = UtilIT.createDatasetViaNativeApi(dataverseAlias, pathToJsonFile, apiToken); createDatasetResponse.prettyPrint(); datasetId = JsonPath.from(createDatasetResponse.body().asString()).getInt("data.id"); + + Response allowAccessRequests = UtilIT.allowAccessRequests(datasetId.toString(), true, apiToken); + allowAccessRequests.prettyPrint(); + allowAccessRequests.then().assertThat().statusCode(200); basicFileName = "004.txt"; String basicPathToFile = "scripts/search/data/replace_test/" + basicFileName; @@ -176,21 +180,19 @@ public void testSaveAuxiliaryFileWithVersion() throws IOException { System.out.println("Add aux file with update"); String mimeType = null; String pathToFile = "scripts/search/data/tabular/1char"; - Response response = given() - .header(API_TOKEN_HTTP_HEADER, apiToken) - .multiPart("file", new File(pathToFile), mimeType) - .post("/api/access/datafile/" + tabFile1Id + "/metadata/dpJSON/v1"); - response.prettyPrint(); - assertEquals(200, response.getStatusCode()); + String formatTag = "dpJSON"; + String formatVersion = "v1"; + + Response uploadResponse = UtilIT.uploadAuxFile(tabFile3IdRestricted.longValue(), pathToFile, formatTag, formatVersion, mimeType, true, null, apiToken); + uploadResponse.prettyPrint(); + uploadResponse.then().assertThat().statusCode(OK.getStatusCode()); + System.out.println("Downloading Aux file that was just added"); - response = given() - .header(API_TOKEN_HTTP_HEADER, apiToken) - .get("/api/access/datafile/" + tabFile1Id + "/metadata/dpJSON/v1"); - - String dataStr = response.prettyPrint(); - assertEquals(dataStr,"a\n"); - assertEquals(200, response.getStatusCode()); - } + Response downloadResponse = UtilIT.downloadAuxFile(tabFile3IdRestricted.longValue(), formatTag, formatVersion, apiToken); + downloadResponse.then().assertThat().statusCode(OK.getStatusCode()); + String dataStr = downloadResponse.prettyPrint(); + assertEquals(dataStr, "a\n"); + } //This test does a lot of testing of non-original downloads as well @Test diff --git a/src/test/java/edu/harvard/iq/dataverse/api/AuxiliaryFilesIT.java b/src/test/java/edu/harvard/iq/dataverse/api/AuxiliaryFilesIT.java new file mode 100644 index 00000000000..4cc688c8758 --- /dev/null +++ b/src/test/java/edu/harvard/iq/dataverse/api/AuxiliaryFilesIT.java @@ -0,0 +1,256 @@ +package edu.harvard.iq.dataverse.api; + +import com.jayway.restassured.RestAssured; +import com.jayway.restassured.path.json.JsonPath; +import com.jayway.restassured.response.Response; +import java.io.File; +import java.io.IOException; +import java.nio.file.Path; +import java.nio.file.Paths; +import static javax.ws.rs.core.Response.Status.CREATED; +import static javax.ws.rs.core.Response.Status.FORBIDDEN; +import static javax.ws.rs.core.Response.Status.OK; +import static org.hamcrest.CoreMatchers.equalTo; +import org.junit.Assert; +import static org.junit.Assert.assertTrue; +import org.junit.BeforeClass; +import org.junit.Test; + +public class AuxiliaryFilesIT { + + @BeforeClass + public static void setUp() { + RestAssured.baseURI = UtilIT.getRestAssuredBaseUri(); + } + + @Test + public void testUploadAuxFiles() throws IOException { + Response createUser = UtilIT.createRandomUser(); + createUser.then().assertThat().statusCode(OK.getStatusCode()); + String apiToken = UtilIT.getApiTokenFromResponse(createUser); + + Response createDataverseResponse = UtilIT.createRandomDataverse(apiToken); + createDataverseResponse.prettyPrint(); + createDataverseResponse.then().assertThat() + .statusCode(CREATED.getStatusCode()); + + String dataverseAlias = UtilIT.getAliasFromResponse(createDataverseResponse); + + Response createDataset = UtilIT.createRandomDatasetViaNativeApi(dataverseAlias, apiToken); + createDataset.prettyPrint(); + createDataset.then().assertThat() + .statusCode(CREATED.getStatusCode()); + + Integer datasetId = UtilIT.getDatasetIdFromResponse(createDataset); + String datasetPid = UtilIT.getDatasetPersistentIdFromResponse(createDataset); + + Path pathToDataFile = Paths.get(java.nio.file.Files.createTempDirectory(null) + File.separator + "data.csv"); + String contentOfCsv = "" + + "name,pounds,species\n" + + "Marshall,40,dog\n" + + "Tiger,17,cat\n" + + "Panther,21,cat\n"; + java.nio.file.Files.write(pathToDataFile, contentOfCsv.getBytes()); + + Response uploadFile = UtilIT.uploadFileViaNative(datasetId.toString(), pathToDataFile.toString(), apiToken); + uploadFile.prettyPrint(); + uploadFile.then().assertThat() + .statusCode(OK.getStatusCode()) + .body("data.files[0].label", equalTo("data.csv")); + + Long fileId = JsonPath.from(uploadFile.body().asString()).getLong("data.files[0].dataFile.id"); + + assertTrue("Failed test if Ingest Lock exceeds max duration " + pathToDataFile, UtilIT.sleepForLock(datasetId.longValue(), "Ingest", apiToken, UtilIT.MAXIMUM_INGEST_LOCK_DURATION)); + + Response restrictFile = UtilIT.restrictFile(fileId.toString(), true, apiToken); + restrictFile.prettyPrint(); + restrictFile.then().assertThat().statusCode(OK.getStatusCode()); + + String dpType = "DP"; + + // JSON aux file + Path pathToAuxFileJson = Paths.get(java.nio.file.Files.createTempDirectory(null) + File.separator + "data.json"); + String contentOfJson = "{}"; + java.nio.file.Files.write(pathToAuxFileJson, contentOfJson.getBytes()); + String formatTagJson = "dpJson"; + String formatVersionJson = "0.1"; + String mimeTypeJson = "application/json"; + Response uploadAuxFileJson = UtilIT.uploadAuxFile(fileId, pathToAuxFileJson.toString(), formatTagJson, formatVersionJson, mimeTypeJson, true, dpType, apiToken); + uploadAuxFileJson.prettyPrint(); + uploadAuxFileJson.then().assertThat() + .statusCode(OK.getStatusCode()) + .body("data.type", equalTo("DP")) + // FIXME: application/json would be better + .body("data.contentType", equalTo("text/plain")); + + // XML aux file + Path pathToAuxFileXml = Paths.get(java.nio.file.Files.createTempDirectory(null) + File.separator + "data.xml"); + String contentOfXml = ""; + java.nio.file.Files.write(pathToAuxFileXml, contentOfXml.getBytes()); + String formatTagXml = "dpXml"; + String formatVersionXml = "0.1"; + String mimeTypeXml = "application/xml"; + Response uploadAuxFileXml = UtilIT.uploadAuxFile(fileId, pathToAuxFileXml.toString(), formatTagXml, formatVersionXml, mimeTypeXml, true, dpType, apiToken); + uploadAuxFileXml.prettyPrint(); + uploadAuxFileXml.then().assertThat() + .statusCode(OK.getStatusCode()) + .body("data.type", equalTo("DP")) + // FIXME: application/xml would be better + .body("data.contentType", equalTo("text/plain")); + + // PDF aux file + Path pathToAuxFilePdf = Paths.get(java.nio.file.Files.createTempDirectory(null) + File.separator + "data.pdf"); + // PDF content from https://stackoverflow.com/questions/17279712/what-is-the-smallest-possible-valid-pdf/32142316#32142316 + String contentOfPdf = "%PDF-1.2 \n" + + "9 0 obj\n" + + "<<\n" + + ">>\n" + + "stream\n" + + "BT/ 9 Tf(Test)' ET\n" + + "endstream\n" + + "endobj\n" + + "4 0 obj\n" + + "<<\n" + + "/Type /Page\n" + + "/Parent 5 0 R\n" + + "/Contents 9 0 R\n" + + ">>\n" + + "endobj\n" + + "5 0 obj\n" + + "<<\n" + + "/Kids [4 0 R ]\n" + + "/Count 1\n" + + "/Type /Pages\n" + + "/MediaBox [ 0 0 99 9 ]\n" + + ">>\n" + + "endobj\n" + + "3 0 obj\n" + + "<<\n" + + "/Pages 5 0 R\n" + + "/Type /Catalog\n" + + ">>\n" + + "endobj\n" + + "trailer\n" + + "<<\n" + + "/Root 3 0 R\n" + + ">>\n" + + "%%EOF"; + java.nio.file.Files.write(pathToAuxFilePdf, contentOfPdf.getBytes()); + String formatTagPdf = "dpPdf"; + String formatVersionPdf = "0.1"; + String mimeTypePdf = "application/pdf"; + Response uploadAuxFilePdf = UtilIT.uploadAuxFile(fileId, pathToAuxFilePdf.toString(), formatTagPdf, formatVersionPdf, mimeTypePdf, true, dpType, apiToken); + uploadAuxFilePdf.prettyPrint(); + uploadAuxFilePdf.then().assertThat() + .statusCode(OK.getStatusCode()) + .body("data.type", equalTo("DP")) + .body("data.contentType", equalTo(mimeTypePdf)); + + // Non-DP aux file, no type specified + Path pathToAuxFileMd = Paths.get(java.nio.file.Files.createTempDirectory(null) + File.separator + "README.md"); + String contentOfMd = "This is my README."; + java.nio.file.Files.write(pathToAuxFileMd, contentOfMd.getBytes()); + String formatTagMd = "README"; + String formatVersionMd = "0.1"; + String mimeTypeMd = "application/xml"; + Response uploadAuxFileMd = UtilIT.uploadAuxFile(fileId, pathToAuxFileMd.toString(), formatTagMd, formatVersionMd, mimeTypeMd, true, null, apiToken); + uploadAuxFileMd.prettyPrint(); + uploadAuxFileMd.then().assertThat() + .statusCode(OK.getStatusCode()) + .body("data.type", equalTo(null)) + .body("data.contentType", equalTo("text/plain")); + + // Unknown type + Path pathToAuxFileTxt = Paths.get(java.nio.file.Files.createTempDirectory(null) + File.separator + "nbt.txt"); + String contentOfTxt = "It's gonna be huge."; + java.nio.file.Files.write(pathToAuxFileTxt, contentOfTxt.getBytes()); + String formatTagTxt = "gonnaBeHuge"; + String formatVersionTxt = "0.1"; + String mimeTypeTxt = "text/plain"; + String typeTxt = "NEXT_BIG_THING"; + Response uploadAuxFileTxt = UtilIT.uploadAuxFile(fileId, pathToAuxFileTxt.toString(), formatTagTxt, formatVersionTxt, mimeTypeTxt, true, typeTxt, apiToken); + uploadAuxFileTxt.prettyPrint(); + uploadAuxFileTxt.then().assertThat() + .statusCode(OK.getStatusCode()) + // .body("data.type", equalTo("Other Auxiliary Files")); + .body("data.type", equalTo("NEXT_BIG_THING")); + + // Another Unknown type + Path pathToAuxFileTxt2 = Paths.get(java.nio.file.Files.createTempDirectory(null) + File.separator + "future.txt"); + String contentOfTxt2 = "The future's so bright I have to wear shades."; + java.nio.file.Files.write(pathToAuxFileTxt2, contentOfTxt2.getBytes()); + String formatTagTxt2 = "soBright"; + String formatVersionTxt2 = "0.1"; + String mimeTypeTxt2 = "text/plain"; + String typeTxt2 = "FUTURE_FUN"; + Response uploadAuxFileTxt2 = UtilIT.uploadAuxFile(fileId, pathToAuxFileTxt2.toString(), formatTagTxt2, formatVersionTxt2, mimeTypeTxt2, true, typeTxt2, apiToken); + uploadAuxFileTxt2.prettyPrint(); + uploadAuxFileTxt2.then().assertThat() + .statusCode(OK.getStatusCode()) + // .body("data.type", equalTo("Other Auxiliary Files")); + .body("data.type", equalTo("FUTURE_FUN")); + + // rst aux file with public=false + Path pathToAuxFileRst = Paths.get(java.nio.file.Files.createTempDirectory(null) + File.separator + "nonpublic.rst"); + String contentOfRst = "Nonpublic stuff in here."; + java.nio.file.Files.write(pathToAuxFileRst, contentOfRst.getBytes()); + String formatTagRst = "nonPublic"; + String formatVersionRst = "0.1"; + String mimeTypeRst = "text/plain"; + Response uploadAuxFileRst = UtilIT.uploadAuxFile(fileId, pathToAuxFileRst.toString(), formatTagRst, formatVersionRst, mimeTypeRst, false, null, apiToken); + uploadAuxFileRst.prettyPrint(); + uploadAuxFileRst.then().assertThat() + .statusCode(OK.getStatusCode()) + .body("data.type", equalTo(null)) + .body("data.contentType", equalTo("text/plain")); + + // Download JSON aux file. + Response downloadAuxFileJson = UtilIT.downloadAuxFile(fileId, formatTagJson, formatVersionJson, apiToken); + downloadAuxFileJson.then().assertThat().statusCode(OK.getStatusCode()); + // FIXME: This should be ".json" instead of ".txt" + Assert.assertEquals("attachment; filename=\"data.tab.dpJson_0.1.txt\"", downloadAuxFileJson.header("Content-disposition")); + + // Download XML aux file. + Response downloadAuxFileXml = UtilIT.downloadAuxFile(fileId, formatTagXml, formatVersionXml, apiToken); + downloadAuxFileXml.then().assertThat().statusCode(OK.getStatusCode()); + // FIXME: This should be ".xml" instead of ".txt" + Assert.assertEquals("attachment; filename=\"data.tab.dpXml_0.1.txt\"", downloadAuxFileXml.header("Content-disposition")); + + // Download PDF aux file. + Response downloadAuxFilePdf = UtilIT.downloadAuxFile(fileId, formatTagPdf, formatVersionPdf, apiToken); + downloadAuxFilePdf.then().assertThat().statusCode(OK.getStatusCode()); + Assert.assertEquals("attachment; filename=\"data.tab.dpPdf_0.1.pdf\"", downloadAuxFilePdf.header("Content-disposition")); + + // Download Markdown aux file. + Response downloadAuxFileMd = UtilIT.downloadAuxFile(fileId, formatTagMd, formatVersionMd, apiToken); + downloadAuxFileMd.then().assertThat().statusCode(OK.getStatusCode()); + Assert.assertEquals("attachment; filename=\"data.tab.README_0.1.txt\"", downloadAuxFileMd.header("Content-disposition")); + + Response createUserNoPrivs = UtilIT.createRandomUser(); + createUserNoPrivs.then().assertThat().statusCode(OK.getStatusCode()); + String apiTokenNoPrivs = UtilIT.getApiTokenFromResponse(createUserNoPrivs); + + // This fails because the dataset hasn't been published. + Response failToDownloadAuxFileJson = UtilIT.downloadAuxFile(fileId, formatTagJson, formatVersionJson, apiTokenNoPrivs); + failToDownloadAuxFileJson.then().assertThat().statusCode(FORBIDDEN.getStatusCode()); + + Response failToDownloadAuxFileRstBeforePublish = UtilIT.downloadAuxFile(fileId, formatTagRst, formatVersionRst, apiTokenNoPrivs); + failToDownloadAuxFileRstBeforePublish.then().assertThat().statusCode(FORBIDDEN.getStatusCode()); + + UtilIT.publishDataverseViaNativeApi(dataverseAlias, apiToken).then().assertThat().statusCode(OK.getStatusCode()); + + UtilIT.publishDatasetViaNativeApi(datasetPid, "major", apiToken).then().assertThat().statusCode(OK.getStatusCode()); + + // isPublic=false so publishing the dataset doesn't let a "no privs" account download the file. + Response failToDownloadAuxFileRstAfterPublish = UtilIT.downloadAuxFile(fileId, formatTagRst, formatVersionRst, apiTokenNoPrivs); + failToDownloadAuxFileRstAfterPublish.then().assertThat().statusCode(FORBIDDEN.getStatusCode()); + + Response creatorCanDownloadNonPublicAuxFile = UtilIT.downloadAuxFile(fileId, formatTagRst, formatVersionRst, apiToken); + creatorCanDownloadNonPublicAuxFile.then().assertThat().statusCode(OK.getStatusCode()); + + // This succeeds now that the dataset has been published. + Response failToDownloadAuxFileJsonPostPublish = UtilIT.downloadAuxFile(fileId, formatTagJson, formatVersionJson, apiTokenNoPrivs); + failToDownloadAuxFileJsonPostPublish.then().assertThat().statusCode(OK.getStatusCode()); + + } +} diff --git a/src/test/java/edu/harvard/iq/dataverse/api/UtilIT.java b/src/test/java/edu/harvard/iq/dataverse/api/UtilIT.java index d00045679f9..56deaef569b 100644 --- a/src/test/java/edu/harvard/iq/dataverse/api/UtilIT.java +++ b/src/test/java/edu/harvard/iq/dataverse/api/UtilIT.java @@ -642,7 +642,25 @@ static Response uploadFileViaNative(String datasetId, String pathToFile, String } return requestSpecification.post("/api/datasets/" + datasetId + "/add"); } - + + static Response uploadAuxFile(Long fileId, String pathToFile, String formatTag, String formatVersion, String mimeType, boolean isPublic, String type, String apiToken) { + RequestSpecification requestSpecification = given() + .header(API_TOKEN_HTTP_HEADER, apiToken) + .multiPart("file", new File(pathToFile), mimeType) + .multiPart("isPublic", isPublic); + if (type != null) { + requestSpecification.multiPart("type", type); + } + return requestSpecification.post("/api/access/datafile/" + fileId + "/metadata/" + formatTag + "/" + formatVersion); + } + + static Response downloadAuxFile(Long fileId, String formatTag, String formatVersion, String apiToken) { + Response response = given() + .header(API_TOKEN_HTTP_HEADER, apiToken) + .get("/api/access/datafile/" + fileId + "/metadata/" + formatTag + "/" + formatVersion); + return response; + } + static Response getCrawlableFileAccess(String datasetId, String folderName, String apiToken) { RequestSpecification requestSpecification = given() .header(API_TOKEN_HTTP_HEADER, apiToken);