From f41747d2684697c0bd02fbbb15fc4bcbfc578e0e Mon Sep 17 00:00:00 2001 From: matthew-a-dunlap Date: Thu, 1 Nov 2018 16:24:24 -0400 Subject: [PATCH 01/32] Pull new dcm package file and zip checksum #4949 --- .../iq/dataverse/S3PackageImporter.java | 24 ++++++++++++++----- 1 file changed, 18 insertions(+), 6 deletions(-) diff --git a/src/main/java/edu/harvard/iq/dataverse/S3PackageImporter.java b/src/main/java/edu/harvard/iq/dataverse/S3PackageImporter.java index f5310119150..0032016014b 100644 --- a/src/main/java/edu/harvard/iq/dataverse/S3PackageImporter.java +++ b/src/main/java/edu/harvard/iq/dataverse/S3PackageImporter.java @@ -142,15 +142,27 @@ public DataFile createPackageDataFile(Dataset dataset, String folderName, long t FileUtil.generateStorageIdentifier(packageFile); - String dvBucketName = System.getProperty("dataverse.files.s3-bucket-name"); String dvDatasetKey = getS3DatasetKey(dataset); - S3Object s3object = null; + S3Object s3FilesSha = null; + + //String fileName = folderName.substring(folderName.lastIndexOf('/') + 1); + //FK2/BST918/package_FK2BST918.zip + + //This is a brittle calculation, changes of the dcm post_upload script will blow this up + String rootPackageName = "package_" + folderName.replace("/", ""); + + //getting the name of the .sha file via substring, ${packageName}.sha + logger.log(Level.INFO, "shaname {0}", new Object[]{rootPackageName + ".sha"}); + - s3object = s3.getObject(new GetObjectRequest(dvBucketName, dvDatasetKey+"/files.sha")); + s3FilesSha = s3.getObject(new GetObjectRequest(dvBucketName, dvDatasetKey + "/" + rootPackageName + ".sha")); - InputStream in = s3object.getObjectContent(); - String checksumVal = FileUtil.CalculateChecksum(in, packageFile.getChecksumType()); + InputStream inSha = s3FilesSha.getObjectContent(); + String checksumVal = FileUtil.CalculateChecksum(inSha, packageFile.getChecksumType()); + + logger.log(Level.INFO, "Checksum value for the package in Dataset {0} is: {1}", + new Object[]{dataset.getIdentifier(), checksumVal}); packageFile.setChecksumValue(checksumVal); @@ -165,7 +177,7 @@ public DataFile createPackageDataFile(Dataset dataset, String folderName, long t // set metadata and add to latest version FileMetadata fmd = new FileMetadata(); - fmd.setLabel(folderName.substring(folderName.lastIndexOf('/') + 1)); + fmd.setLabel(rootPackageName + ".zip"); fmd.setDataFile(packageFile); packageFile.getFileMetadatas().add(fmd); From c8f80f092b62248ab01fa018d24ae67c921af8ee Mon Sep 17 00:00:00 2001 From: matthew-a-dunlap Date: Fri, 2 Nov 2018 17:45:18 -0400 Subject: [PATCH 02/32] Pull checksum from dcm uploaded .sha #5053 --- .../iq/dataverse/S3PackageImporter.java | 52 ++++++++++++++++--- 1 file changed, 44 insertions(+), 8 deletions(-) diff --git a/src/main/java/edu/harvard/iq/dataverse/S3PackageImporter.java b/src/main/java/edu/harvard/iq/dataverse/S3PackageImporter.java index 0032016014b..d7c17361cae 100644 --- a/src/main/java/edu/harvard/iq/dataverse/S3PackageImporter.java +++ b/src/main/java/edu/harvard/iq/dataverse/S3PackageImporter.java @@ -23,8 +23,11 @@ import edu.harvard.iq.dataverse.settings.SettingsServiceBean; import edu.harvard.iq.dataverse.util.FileUtil; import static edu.harvard.iq.dataverse.util.json.NullSafeJsonBuilder.jsonObjectBuilder; +import java.io.BufferedReader; import java.io.IOException; import java.io.InputStream; +import java.io.InputStreamReader; +import java.nio.charset.StandardCharsets; import java.sql.Timestamp; import java.util.ArrayList; import java.util.Date; @@ -36,6 +39,7 @@ import javax.inject.Named; import javax.json.JsonObject; import javax.json.JsonObjectBuilder; +import org.apache.commons.io.IOUtils; /** * This class is for importing files added to s3 outside of dataverse. @@ -144,10 +148,6 @@ public DataFile createPackageDataFile(Dataset dataset, String folderName, long t String dvBucketName = System.getProperty("dataverse.files.s3-bucket-name"); String dvDatasetKey = getS3DatasetKey(dataset); - S3Object s3FilesSha = null; - - //String fileName = folderName.substring(folderName.lastIndexOf('/') + 1); - //FK2/BST918/package_FK2BST918.zip //This is a brittle calculation, changes of the dcm post_upload script will blow this up String rootPackageName = "package_" + folderName.replace("/", ""); @@ -155,11 +155,47 @@ public DataFile createPackageDataFile(Dataset dataset, String folderName, long t //getting the name of the .sha file via substring, ${packageName}.sha logger.log(Level.INFO, "shaname {0}", new Object[]{rootPackageName + ".sha"}); - - s3FilesSha = s3.getObject(new GetObjectRequest(dvBucketName, dvDatasetKey + "/" + rootPackageName + ".sha")); + S3Object s3FilesSha = s3.getObject(new GetObjectRequest(dvBucketName, dvDatasetKey + "/" + rootPackageName + ".sha")); - InputStream inSha = s3FilesSha.getObjectContent(); - String checksumVal = FileUtil.CalculateChecksum(inSha, packageFile.getChecksumType()); + + InputStreamReader str = new InputStreamReader(s3FilesSha.getObjectContent()); + BufferedReader reader = new BufferedReader(str); + String checksumVal = ""; + try { + String line; + while((line = reader.readLine()) != null && checksumVal.isEmpty()) { + logger.log(Level.INFO, "line {0}", new Object[]{line}); + String[] splitLine = line.split(" "); + //logger.log(Level.INFO, "root package name {0}", new Object[]{rootPackageName + ".zip"}); + //logger.log(Level.INFO, "splitline {0} | {1}", new Object[]{splitLine[0], splitLine[1]}); + + //the sha file should only contain one line, but incase it doesn't we will check for the one for our zip + if(splitLine[1].contains(rootPackageName + ".zip")) { + checksumVal = splitLine[0]; + logger.log(Level.INFO, "checksumVal found {0}", new Object[]{checksumVal}); + } + } + if(checksumVal.isEmpty()) { + logger.log(Level.SEVERE, "No checksum found for uploaded DCM S3 zip on dataset {0}", new Object[]{dataset.getIdentifier()}); + } + } catch (IOException ex){ + logger.log(Level.SEVERE, "Error parsing DCM s3 checksum file on dataset {0} . Error: {1} ", new Object[]{dataset.getIdentifier(), ex}); + } finally { + try { + str.close(); + reader.close(); + } catch (IOException ex) { + logger.log(Level.WARNING, "errors closing s3 DCM object reader stream: {0}", new Object[]{ex}); + } + + } + + //MAD: CLOSE THINGS + + //String shaFileContents = IOUtils.toString(s3FilesSha.getObjectContent(), StandardCharsets.UTF_8); + + //We parse the checksum from the passed sha 1 + //String checksumVal = shaFileContents. logger.log(Level.INFO, "Checksum value for the package in Dataset {0} is: {1}", new Object[]{dataset.getIdentifier(), checksumVal}); From 4b55ebaf961b45ce3520fcfe7a800d3043fdfb59 Mon Sep 17 00:00:00 2001 From: matthew-a-dunlap Date: Fri, 2 Nov 2018 17:53:19 -0400 Subject: [PATCH 03/32] Revert "Pull checksum from dcm uploaded .sha #5053" This reverts commit c8f80f092b62248ab01fa018d24ae67c921af8ee. --- .../iq/dataverse/S3PackageImporter.java | 52 +++---------------- 1 file changed, 8 insertions(+), 44 deletions(-) diff --git a/src/main/java/edu/harvard/iq/dataverse/S3PackageImporter.java b/src/main/java/edu/harvard/iq/dataverse/S3PackageImporter.java index d7c17361cae..0032016014b 100644 --- a/src/main/java/edu/harvard/iq/dataverse/S3PackageImporter.java +++ b/src/main/java/edu/harvard/iq/dataverse/S3PackageImporter.java @@ -23,11 +23,8 @@ import edu.harvard.iq.dataverse.settings.SettingsServiceBean; import edu.harvard.iq.dataverse.util.FileUtil; import static edu.harvard.iq.dataverse.util.json.NullSafeJsonBuilder.jsonObjectBuilder; -import java.io.BufferedReader; import java.io.IOException; import java.io.InputStream; -import java.io.InputStreamReader; -import java.nio.charset.StandardCharsets; import java.sql.Timestamp; import java.util.ArrayList; import java.util.Date; @@ -39,7 +36,6 @@ import javax.inject.Named; import javax.json.JsonObject; import javax.json.JsonObjectBuilder; -import org.apache.commons.io.IOUtils; /** * This class is for importing files added to s3 outside of dataverse. @@ -148,6 +144,10 @@ public DataFile createPackageDataFile(Dataset dataset, String folderName, long t String dvBucketName = System.getProperty("dataverse.files.s3-bucket-name"); String dvDatasetKey = getS3DatasetKey(dataset); + S3Object s3FilesSha = null; + + //String fileName = folderName.substring(folderName.lastIndexOf('/') + 1); + //FK2/BST918/package_FK2BST918.zip //This is a brittle calculation, changes of the dcm post_upload script will blow this up String rootPackageName = "package_" + folderName.replace("/", ""); @@ -155,47 +155,11 @@ public DataFile createPackageDataFile(Dataset dataset, String folderName, long t //getting the name of the .sha file via substring, ${packageName}.sha logger.log(Level.INFO, "shaname {0}", new Object[]{rootPackageName + ".sha"}); - S3Object s3FilesSha = s3.getObject(new GetObjectRequest(dvBucketName, dvDatasetKey + "/" + rootPackageName + ".sha")); - - - InputStreamReader str = new InputStreamReader(s3FilesSha.getObjectContent()); - BufferedReader reader = new BufferedReader(str); - String checksumVal = ""; - try { - String line; - while((line = reader.readLine()) != null && checksumVal.isEmpty()) { - logger.log(Level.INFO, "line {0}", new Object[]{line}); - String[] splitLine = line.split(" "); - //logger.log(Level.INFO, "root package name {0}", new Object[]{rootPackageName + ".zip"}); - //logger.log(Level.INFO, "splitline {0} | {1}", new Object[]{splitLine[0], splitLine[1]}); - - //the sha file should only contain one line, but incase it doesn't we will check for the one for our zip - if(splitLine[1].contains(rootPackageName + ".zip")) { - checksumVal = splitLine[0]; - logger.log(Level.INFO, "checksumVal found {0}", new Object[]{checksumVal}); - } - } - if(checksumVal.isEmpty()) { - logger.log(Level.SEVERE, "No checksum found for uploaded DCM S3 zip on dataset {0}", new Object[]{dataset.getIdentifier()}); - } - } catch (IOException ex){ - logger.log(Level.SEVERE, "Error parsing DCM s3 checksum file on dataset {0} . Error: {1} ", new Object[]{dataset.getIdentifier(), ex}); - } finally { - try { - str.close(); - reader.close(); - } catch (IOException ex) { - logger.log(Level.WARNING, "errors closing s3 DCM object reader stream: {0}", new Object[]{ex}); - } - - } - - //MAD: CLOSE THINGS - - //String shaFileContents = IOUtils.toString(s3FilesSha.getObjectContent(), StandardCharsets.UTF_8); - //We parse the checksum from the passed sha 1 - //String checksumVal = shaFileContents. + s3FilesSha = s3.getObject(new GetObjectRequest(dvBucketName, dvDatasetKey + "/" + rootPackageName + ".sha")); + + InputStream inSha = s3FilesSha.getObjectContent(); + String checksumVal = FileUtil.CalculateChecksum(inSha, packageFile.getChecksumType()); logger.log(Level.INFO, "Checksum value for the package in Dataset {0} is: {1}", new Object[]{dataset.getIdentifier(), checksumVal}); From cbea0c6339093e4e214c82f192bd712e17c8b453 Mon Sep 17 00:00:00 2001 From: matthew-a-dunlap Date: Fri, 2 Nov 2018 17:54:39 -0400 Subject: [PATCH 04/32] Pull checksum from dcm uploaded .sha #4949 --- .../iq/dataverse/S3PackageImporter.java | 52 ++++++++++++++++--- 1 file changed, 44 insertions(+), 8 deletions(-) diff --git a/src/main/java/edu/harvard/iq/dataverse/S3PackageImporter.java b/src/main/java/edu/harvard/iq/dataverse/S3PackageImporter.java index 0032016014b..d7c17361cae 100644 --- a/src/main/java/edu/harvard/iq/dataverse/S3PackageImporter.java +++ b/src/main/java/edu/harvard/iq/dataverse/S3PackageImporter.java @@ -23,8 +23,11 @@ import edu.harvard.iq.dataverse.settings.SettingsServiceBean; import edu.harvard.iq.dataverse.util.FileUtil; import static edu.harvard.iq.dataverse.util.json.NullSafeJsonBuilder.jsonObjectBuilder; +import java.io.BufferedReader; import java.io.IOException; import java.io.InputStream; +import java.io.InputStreamReader; +import java.nio.charset.StandardCharsets; import java.sql.Timestamp; import java.util.ArrayList; import java.util.Date; @@ -36,6 +39,7 @@ import javax.inject.Named; import javax.json.JsonObject; import javax.json.JsonObjectBuilder; +import org.apache.commons.io.IOUtils; /** * This class is for importing files added to s3 outside of dataverse. @@ -144,10 +148,6 @@ public DataFile createPackageDataFile(Dataset dataset, String folderName, long t String dvBucketName = System.getProperty("dataverse.files.s3-bucket-name"); String dvDatasetKey = getS3DatasetKey(dataset); - S3Object s3FilesSha = null; - - //String fileName = folderName.substring(folderName.lastIndexOf('/') + 1); - //FK2/BST918/package_FK2BST918.zip //This is a brittle calculation, changes of the dcm post_upload script will blow this up String rootPackageName = "package_" + folderName.replace("/", ""); @@ -155,11 +155,47 @@ public DataFile createPackageDataFile(Dataset dataset, String folderName, long t //getting the name of the .sha file via substring, ${packageName}.sha logger.log(Level.INFO, "shaname {0}", new Object[]{rootPackageName + ".sha"}); - - s3FilesSha = s3.getObject(new GetObjectRequest(dvBucketName, dvDatasetKey + "/" + rootPackageName + ".sha")); + S3Object s3FilesSha = s3.getObject(new GetObjectRequest(dvBucketName, dvDatasetKey + "/" + rootPackageName + ".sha")); - InputStream inSha = s3FilesSha.getObjectContent(); - String checksumVal = FileUtil.CalculateChecksum(inSha, packageFile.getChecksumType()); + + InputStreamReader str = new InputStreamReader(s3FilesSha.getObjectContent()); + BufferedReader reader = new BufferedReader(str); + String checksumVal = ""; + try { + String line; + while((line = reader.readLine()) != null && checksumVal.isEmpty()) { + logger.log(Level.INFO, "line {0}", new Object[]{line}); + String[] splitLine = line.split(" "); + //logger.log(Level.INFO, "root package name {0}", new Object[]{rootPackageName + ".zip"}); + //logger.log(Level.INFO, "splitline {0} | {1}", new Object[]{splitLine[0], splitLine[1]}); + + //the sha file should only contain one line, but incase it doesn't we will check for the one for our zip + if(splitLine[1].contains(rootPackageName + ".zip")) { + checksumVal = splitLine[0]; + logger.log(Level.INFO, "checksumVal found {0}", new Object[]{checksumVal}); + } + } + if(checksumVal.isEmpty()) { + logger.log(Level.SEVERE, "No checksum found for uploaded DCM S3 zip on dataset {0}", new Object[]{dataset.getIdentifier()}); + } + } catch (IOException ex){ + logger.log(Level.SEVERE, "Error parsing DCM s3 checksum file on dataset {0} . Error: {1} ", new Object[]{dataset.getIdentifier(), ex}); + } finally { + try { + str.close(); + reader.close(); + } catch (IOException ex) { + logger.log(Level.WARNING, "errors closing s3 DCM object reader stream: {0}", new Object[]{ex}); + } + + } + + //MAD: CLOSE THINGS + + //String shaFileContents = IOUtils.toString(s3FilesSha.getObjectContent(), StandardCharsets.UTF_8); + + //We parse the checksum from the passed sha 1 + //String checksumVal = shaFileContents. logger.log(Level.INFO, "Checksum value for the package in Dataset {0} is: {1}", new Object[]{dataset.getIdentifier(), checksumVal}); From 9284ba9d1081b1d2a4789d43668d86bdda2e2346 Mon Sep 17 00:00:00 2001 From: matthew-a-dunlap Date: Wed, 7 Nov 2018 13:51:28 -0500 Subject: [PATCH 05/32] Move file info to new fragment #4949 This move allows that code to be reused in the new package popup This also contains a bit of render logic around s3 package UI --- src/main/webapp/filesFragment.xhtml | 91 ++---------------------- src/main/webapp/filesInfoFragment.xhtml | 94 +++++++++++++++++++++++++ 2 files changed, 101 insertions(+), 84 deletions(-) create mode 100644 src/main/webapp/filesInfoFragment.xhtml diff --git a/src/main/webapp/filesFragment.xhtml b/src/main/webapp/filesFragment.xhtml index 6e959fb4bf5..d66118e91b0 100644 --- a/src/main/webapp/filesFragment.xhtml +++ b/src/main/webapp/filesFragment.xhtml @@ -152,88 +152,10 @@ - -
- - - - - - - - - - - - - - - - -
- -
-
- -
-
- - - - - - - #{fileMetadata.label} - - - - - #{fileMetadata.label} - - - - -
- - - - - - - - - - - -
- -
-
- -
- - - -
- -
- -
+ + + -
- - - - - - - - - - -
- @@ -375,7 +297,8 @@ -
+
@@ -388,8 +311,8 @@
- -
+
diff --git a/src/main/webapp/filesInfoFragment.xhtml b/src/main/webapp/filesInfoFragment.xhtml new file mode 100644 index 00000000000..8de06d90702 --- /dev/null +++ b/src/main/webapp/filesInfoFragment.xhtml @@ -0,0 +1,94 @@ + + + + +
+ + + + + + + + + + + + + + + + +
+ +
+
+ +
+
+ + + + + + + #{fileMetadata.label} + + + + + #{fileMetadata.label} + + + + +
+ + + + + + + + + + + +
+ +
+
+ +
+ + + +
+ +
+ +
+ +
+ + + + + + + + + + +
+ + \ No newline at end of file From 0cdc100d953ea03c9eb2b70f4df741ccfd733d0e Mon Sep 17 00:00:00 2001 From: matthew-a-dunlap Date: Thu, 8 Nov 2018 18:25:09 -0500 Subject: [PATCH 06/32] Generate s3 identifier on import #4949 There was a bug in the previous story where the imported dcm s3 files did not have an S3 identifier. --- .../edu/harvard/iq/dataverse/S3PackageImporter.java | 3 ++- .../java/edu/harvard/iq/dataverse/util/FileUtil.java | 12 +++++++++++- 2 files changed, 13 insertions(+), 2 deletions(-) diff --git a/src/main/java/edu/harvard/iq/dataverse/S3PackageImporter.java b/src/main/java/edu/harvard/iq/dataverse/S3PackageImporter.java index d7c17361cae..e74850156e7 100644 --- a/src/main/java/edu/harvard/iq/dataverse/S3PackageImporter.java +++ b/src/main/java/edu/harvard/iq/dataverse/S3PackageImporter.java @@ -63,6 +63,7 @@ public class S3PackageImporter extends AbstractApiBean implements java.io.Serial @EJB EjbDataverseEngine commandEngine; + //Copies from another s3 bucket to our own public void copyFromS3(Dataset dataset, String s3ImportPath) throws IOException { try { s3 = AmazonS3ClientBuilder.standard().defaultClient(); @@ -144,7 +145,7 @@ public DataFile createPackageDataFile(Dataset dataset, String folderName, long t DataFile packageFile = new DataFile(DataFileServiceBean.MIME_TYPE_PACKAGE_FILE); packageFile.setChecksumType(DataFile.ChecksumType.SHA1); - FileUtil.generateStorageIdentifier(packageFile); + FileUtil.generateS3StorageIdentifier(packageFile); String dvBucketName = System.getProperty("dataverse.files.s3-bucket-name"); String dvDatasetKey = getS3DatasetKey(dataset); diff --git a/src/main/java/edu/harvard/iq/dataverse/util/FileUtil.java b/src/main/java/edu/harvard/iq/dataverse/util/FileUtil.java index 48fbfb5e37d..e5cdf4c856d 100644 --- a/src/main/java/edu/harvard/iq/dataverse/util/FileUtil.java +++ b/src/main/java/edu/harvard/iq/dataverse/util/FileUtil.java @@ -28,6 +28,7 @@ import edu.harvard.iq.dataverse.FileMetadata; import edu.harvard.iq.dataverse.TermsOfUseAndAccess; import edu.harvard.iq.dataverse.dataaccess.ImageThumbConverter; +import static edu.harvard.iq.dataverse.dataaccess.S3AccessIO.S3_IDENTIFIER_PREFIX; import edu.harvard.iq.dataverse.dataset.DatasetThumbnail; import edu.harvard.iq.dataverse.datasetutility.FileExceedsMaxSizeException; import static edu.harvard.iq.dataverse.datasetutility.FileSizeChecker.bytesToHumanReadable; @@ -91,7 +92,7 @@ public class FileUtil implements java.io.Serializable { private static final String[] TABULAR_DATA_FORMAT_SET = {"POR", "SAV", "DTA", "RDA"}; private static Map STATISTICAL_FILE_EXTENSION = new HashMap(); - + /* * The following are Stata, SAS and SPSS syntax/control cards: * These are recognized as text files (because they are!) so @@ -1132,6 +1133,15 @@ public static String getFilesTempDirectory() { return filesTempDirectory; } + //MAD: Refactor this again so it reuses code from S3AccessIO. + //Also maybe refactor the generic named one to reflect that it creates fileAccessIO on its own. + public static void generateS3StorageIdentifier(DataFile dataFile) { + String storageId = generateStorageIdentifier(); + String bucketName = System.getProperty("dataverse.files.s3-bucket-name"); + storageId = S3_IDENTIFIER_PREFIX + "://" + bucketName + ":" + storageId; + dataFile.setStorageIdentifier(storageId); + } + public static void generateStorageIdentifier(DataFile dataFile) { dataFile.setStorageIdentifier(generateStorageIdentifier()); } From f391e6334663641bb4b274ddf9f2c7f56be6b5db Mon Sep 17 00:00:00 2001 From: matthew-a-dunlap Date: Wed, 14 Nov 2018 19:22:57 -0500 Subject: [PATCH 07/32] Popup with s3 package download url #4949 --- src/main/java/Bundle.properties | 4 ++ .../iq/dataverse/S3PackageImporter.java | 15 ++++--- .../dataverse/S3PackagePopupFragmentBean.java | 45 +++++++++++++++++++ .../dataverse/api/DownloadInstanceWriter.java | 3 ++ .../harvard/iq/dataverse/util/FileUtil.java | 9 ++-- src/main/webapp/dataset.xhtml | 14 ++++-- .../file-download-button-fragment.xhtml | 24 +++++++++- src/main/webapp/file.xhtml | 11 +++++ .../s3-package-download-popup-fragment.xhtml | 42 +++++++++++++++++ 9 files changed, 150 insertions(+), 17 deletions(-) create mode 100644 src/main/java/edu/harvard/iq/dataverse/S3PackagePopupFragmentBean.java create mode 100644 src/main/webapp/s3-package-download-popup-fragment.xhtml diff --git a/src/main/java/Bundle.properties b/src/main/java/Bundle.properties index 8f94e25199c..8a332cdde5b 100755 --- a/src/main/java/Bundle.properties +++ b/src/main/java/Bundle.properties @@ -70,6 +70,10 @@ toggleNavigation=Toggle navigation defaultBody=Default Body filter=Filter +s3packagedownload.title=Download Package File +s3packagedownload.instructions=Download this package file by copying and pasting the URL below into Wget or a download manager. Use of a web browser for this download is not recommended. User Guide - Downloading Package Files +s3packagedownload.urlHeader=Download URL + # dataverse_header.xhtml header.status.header=Status header.search.title=Search all dataverses... diff --git a/src/main/java/edu/harvard/iq/dataverse/S3PackageImporter.java b/src/main/java/edu/harvard/iq/dataverse/S3PackageImporter.java index e74850156e7..f423ca768be 100644 --- a/src/main/java/edu/harvard/iq/dataverse/S3PackageImporter.java +++ b/src/main/java/edu/harvard/iq/dataverse/S3PackageImporter.java @@ -145,14 +145,12 @@ public DataFile createPackageDataFile(Dataset dataset, String folderName, long t DataFile packageFile = new DataFile(DataFileServiceBean.MIME_TYPE_PACKAGE_FILE); packageFile.setChecksumType(DataFile.ChecksumType.SHA1); - FileUtil.generateS3StorageIdentifier(packageFile); - + //This is a brittle calculation, changes of the dcm post_upload script will blow this up + String rootPackageName = "package_" + folderName.replace("/", ""); + String dvBucketName = System.getProperty("dataverse.files.s3-bucket-name"); String dvDatasetKey = getS3DatasetKey(dataset); - //This is a brittle calculation, changes of the dcm post_upload script will blow this up - String rootPackageName = "package_" + folderName.replace("/", ""); - //getting the name of the .sha file via substring, ${packageName}.sha logger.log(Level.INFO, "shaname {0}", new Object[]{rootPackageName + ".sha"}); @@ -211,18 +209,21 @@ public DataFile createPackageDataFile(Dataset dataset, String folderName, long t dataset.getFiles().add(packageFile); packageFile.setIngestDone(); - + // set metadata and add to latest version + // Set early so we can generate the storage id with the info FileMetadata fmd = new FileMetadata(); fmd.setLabel(rootPackageName + ".zip"); fmd.setDataFile(packageFile); packageFile.getFileMetadatas().add(fmd); if (dataset.getLatestVersion().getFileMetadatas() == null) dataset.getLatestVersion().setFileMetadatas(new ArrayList<>()); - + dataset.getLatestVersion().getFileMetadatas().add(fmd); fmd.setDatasetVersion(dataset.getLatestVersion()); + FileUtil.generateS3PackageStorageIdentifier(packageFile); + GlobalIdServiceBean idServiceBean = GlobalIdServiceBean.getBean(packageFile.getProtocol(), commandEngine.getContext()); if (packageFile.getIdentifier() == null || packageFile.getIdentifier().isEmpty()) { String packageIdentifier = dataFileServiceBean.generateDataFileIdentifier(packageFile, idServiceBean); diff --git a/src/main/java/edu/harvard/iq/dataverse/S3PackagePopupFragmentBean.java b/src/main/java/edu/harvard/iq/dataverse/S3PackagePopupFragmentBean.java new file mode 100644 index 00000000000..dfcbc952b66 --- /dev/null +++ b/src/main/java/edu/harvard/iq/dataverse/S3PackagePopupFragmentBean.java @@ -0,0 +1,45 @@ +/* + * To change this license header, choose License Headers in Project Properties. + * To change this template file, choose Tools | Templates + * and open the template in the editor. + */ +//MAD: This and s3importer should probably be in their own package +package edu.harvard.iq.dataverse; + +import edu.harvard.iq.dataverse.dataaccess.DataAccess; +import edu.harvard.iq.dataverse.dataaccess.S3AccessIO; +import edu.harvard.iq.dataverse.dataaccess.StorageIO; +import java.io.IOException; +import javax.faces.view.ViewScoped; +import javax.inject.Named; + +/** + * + * @author matthew + */ + +@ViewScoped +@Named +public class S3PackagePopupFragmentBean implements java.io.Serializable { + + DataFile df; + + public void setDataFile(DataFile dataFile) { + df = dataFile; + } + + public DataFile getDataFile() { + return df; + } + +// public String provideS3OneTimeUrl() throws IOException { +// if(df != null ) { +// StorageIO storageIO = DataAccess.getStorageIO(df); +// return ((S3AccessIO)storageIO).generateTemporaryS3Url(); +// } else { +// return ""; +// } +// +// } +} + \ No newline at end of file diff --git a/src/main/java/edu/harvard/iq/dataverse/api/DownloadInstanceWriter.java b/src/main/java/edu/harvard/iq/dataverse/api/DownloadInstanceWriter.java index fab2538832e..0e4471c536b 100644 --- a/src/main/java/edu/harvard/iq/dataverse/api/DownloadInstanceWriter.java +++ b/src/main/java/edu/harvard/iq/dataverse/api/DownloadInstanceWriter.java @@ -4,6 +4,7 @@ * and open the template in the editor. */ +//MAD: WHY DOES THIS LIVE IN API package edu.harvard.iq.dataverse.api; import java.lang.reflect.Type; @@ -33,6 +34,7 @@ import java.net.URISyntaxException; import java.util.ArrayList; import java.util.List; +import java.util.logging.Level; import java.util.logging.Logger; import javax.ws.rs.NotFoundException; import javax.ws.rs.RedirectionException; @@ -75,6 +77,7 @@ public void writeTo(DownloadInstance di, Class clazz, Type type, Annotation[] storageIO.open(); } catch (IOException ioex) { //throw new WebApplicationException(Response.Status.SERVICE_UNAVAILABLE); + logger.log(Level.INFO, "Datafile {0}: Failed to locate and/or open physical file. Error message: {1}", new Object[]{dataFile.getId(), ioex.getLocalizedMessage()}); throw new NotFoundException("Datafile "+dataFile.getId()+": Failed to locate and/or open physical file."); } diff --git a/src/main/java/edu/harvard/iq/dataverse/util/FileUtil.java b/src/main/java/edu/harvard/iq/dataverse/util/FileUtil.java index e5cdf4c856d..595b633a9a9 100644 --- a/src/main/java/edu/harvard/iq/dataverse/util/FileUtil.java +++ b/src/main/java/edu/harvard/iq/dataverse/util/FileUtil.java @@ -1133,12 +1133,11 @@ public static String getFilesTempDirectory() { return filesTempDirectory; } - //MAD: Refactor this again so it reuses code from S3AccessIO. - //Also maybe refactor the generic named one to reflect that it creates fileAccessIO on its own. - public static void generateS3StorageIdentifier(DataFile dataFile) { - String storageId = generateStorageIdentifier(); + //OLD MAD: Refactor this again so it reuses code from S3AccessIO. + //OLD MAD: Also maybe refactor the generic named one to reflect that it creates fileAccessIO on its own. + public static void generateS3PackageStorageIdentifier(DataFile dataFile) { String bucketName = System.getProperty("dataverse.files.s3-bucket-name"); - storageId = S3_IDENTIFIER_PREFIX + "://" + bucketName + ":" + storageId; + String storageId = S3_IDENTIFIER_PREFIX + "://" + bucketName + ":" + dataFile.getFileMetadata().getLabel(); dataFile.setStorageIdentifier(storageId); } diff --git a/src/main/webapp/dataset.xhtml b/src/main/webapp/dataset.xhtml index 8b2c0878dd0..d7a34ef2a58 100755 --- a/src/main/webapp/dataset.xhtml +++ b/src/main/webapp/dataset.xhtml @@ -1165,7 +1165,6 @@ - @@ -1173,9 +1172,18 @@ - - + + + + + + + + + + + diff --git a/src/main/webapp/file-download-button-fragment.xhtml b/src/main/webapp/file-download-button-fragment.xhtml index a08acdf74e8..c36561278da 100644 --- a/src/main/webapp/file-download-button-fragment.xhtml +++ b/src/main/webapp/file-download-button-fragment.xhtml @@ -112,8 +112,28 @@ #{bundle['file.compute']} - - + + + + + + + S3 Download #{fileMetadata.dataFile} + + + + +
+ + + + + + + + + + + diff --git a/src/main/webapp/s3-package-download-popup-fragment.xhtml b/src/main/webapp/s3-package-download-popup-fragment.xhtml new file mode 100644 index 00000000000..9fc65ac80e9 --- /dev/null +++ b/src/main/webapp/s3-package-download-popup-fragment.xhtml @@ -0,0 +1,42 @@ + + + + + + + + + + + +
+ + + + + +
+
+
#{systemConfig.getDataverseSiteUrl()}/api/access/datafile/#{s3PackagePopupFragmentBean.dataFile.id}
+
+ + +
+ From 36b864864835895d73970f11fd633f259d380153 Mon Sep 17 00:00:00 2001 From: matthew-a-dunlap Date: Thu, 15 Nov 2018 18:12:56 -0500 Subject: [PATCH 08/32] Launch s3Package popup from guestbook #4949 --- .../iq/dataverse/FileDownloadHelper.java | 17 +++++++++- .../file-download-button-fragment.xhtml | 33 +++++++++++-------- .../webapp/file-download-popup-fragment.xhtml | 14 ++++++-- 3 files changed, 48 insertions(+), 16 deletions(-) diff --git a/src/main/java/edu/harvard/iq/dataverse/FileDownloadHelper.java b/src/main/java/edu/harvard/iq/dataverse/FileDownloadHelper.java index befdc3a9eeb..476a3bf97e3 100644 --- a/src/main/java/edu/harvard/iq/dataverse/FileDownloadHelper.java +++ b/src/main/java/edu/harvard/iq/dataverse/FileDownloadHelper.java @@ -297,7 +297,22 @@ public void writeGuestbookAndLaunchExploreTool(GuestbookResponse guestbookRespon } fileDownloadService.explore(guestbookResponse, fmd, externalTool); requestContext.execute("PF('downloadPopup').hide()"); - } + } + + public void writeGuestbookAndLaunchS3PackagePopup(GuestbookResponse guestbookResponse) { + RequestContext requestContext = RequestContext.getCurrentInstance(); + boolean valid = validateGuestbookResponse(guestbookResponse); + + if (!valid) { + JH.addMessage(FacesMessage.SEVERITY_ERROR, JH.localize("dataset.message.validationError")); + } else { + requestContext.execute("PF('downloadPopup').hide()"); + requestContext.execute("PF('downloadS3PackagePopup').show()"); + requestContext.execute("handleResizeDialog('downloadS3PackagePopup')"); + + fileDownloadService.writeGuestbookResponseRecord(guestbookResponse); + } + } public String startWorldMapDownloadLink(GuestbookResponse guestbookResponse, FileMetadata fmd){ diff --git a/src/main/webapp/file-download-button-fragment.xhtml b/src/main/webapp/file-download-button-fragment.xhtml index c36561278da..a7206922824 100644 --- a/src/main/webapp/file-download-button-fragment.xhtml +++ b/src/main/webapp/file-download-button-fragment.xhtml @@ -112,25 +112,31 @@ #{bundle['file.compute']} - - - - - - - + + S3 Download #{fileMetadata.dataFile} - + + + + #{bundle.download} + #{bundle.download} -
+ @@ -172,7 +174,15 @@ update="guestbookUIFragment"> #{bundle['acceptTerms']} + + + + + +
\ No newline at end of file From e7d7dcd8971e80945b78d361eb37b1cf6f3b7a2a Mon Sep 17 00:00:00 2001 From: matthew-a-dunlap Date: Mon, 19 Nov 2018 18:30:33 -0500 Subject: [PATCH 09/32] dcm s3 file page start #4949 --- .../harvard/iq/dataverse/DataCiteRESTfullClient.java | 1 + .../java/edu/harvard/iq/dataverse/DatasetPage.java | 6 +++++- .../edu/harvard/iq/dataverse/S3PackageImporter.java | 2 +- src/main/webapp/file-download-button-fragment.xhtml | 11 ++++++----- src/main/webapp/file.xhtml | 10 +++++++--- 5 files changed, 20 insertions(+), 10 deletions(-) diff --git a/src/main/java/edu/harvard/iq/dataverse/DataCiteRESTfullClient.java b/src/main/java/edu/harvard/iq/dataverse/DataCiteRESTfullClient.java index a8ddc3b06e9..d128996c91e 100644 --- a/src/main/java/edu/harvard/iq/dataverse/DataCiteRESTfullClient.java +++ b/src/main/java/edu/harvard/iq/dataverse/DataCiteRESTfullClient.java @@ -219,6 +219,7 @@ public String inactiveDataset(String doi) { } } + //MAD: This shouldn't be hardcoded in our source.... public static void main(String[] args) throws Exception { String doi = "10.5072/DVN/274533"; DataCiteRESTfullClient client = new DataCiteRESTfullClient("https://mds.test.datacite.org", "DATACITE.HARVARD", "DVNapitest"); diff --git a/src/main/java/edu/harvard/iq/dataverse/DatasetPage.java b/src/main/java/edu/harvard/iq/dataverse/DatasetPage.java index 84462e3915e..5e762000386 100644 --- a/src/main/java/edu/harvard/iq/dataverse/DatasetPage.java +++ b/src/main/java/edu/harvard/iq/dataverse/DatasetPage.java @@ -1,6 +1,7 @@ package edu.harvard.iq.dataverse; import edu.harvard.iq.dataverse.provenance.ProvPopupFragmentBean; +import edu.harvard.iq.dataverse.S3PackagePopupFragmentBean; import edu.harvard.iq.dataverse.api.AbstractApiBean; import edu.harvard.iq.dataverse.authorization.AuthenticationServiceBean; import edu.harvard.iq.dataverse.authorization.Permission; @@ -196,7 +197,9 @@ public enum DisplayMode { SettingsWrapper settingsWrapper; @Inject ProvPopupFragmentBean provPopupFragmentBean; - +//MAD: Does this even do anything? Does the prov one for that matter? +// @Inject +// S3PackagePopupFragmentBean s3PopupFragmentBean; private Dataset dataset = new Dataset(); private EditMode editMode; @@ -1470,6 +1473,7 @@ private String init(boolean initFull) { logger.fine("Checking if rsync support is enabled."); if (DataCaptureModuleUtil.rsyncSupportEnabled(settingsWrapper.getValueForKey(SettingsServiceBean.Key.UploadMethods))) { try { +//MAD: I think this is the spot that keeps checking for rsync even though its gone thru the process ScriptRequestResponse scriptRequestResponse = commandEngine.submit(new RequestRsyncScriptCommand(dvRequestService.getDataverseRequest(), dataset)); logger.fine("script: " + scriptRequestResponse.getScript()); if(scriptRequestResponse.getScript()!=null && !scriptRequestResponse.getScript().isEmpty()){ diff --git a/src/main/java/edu/harvard/iq/dataverse/S3PackageImporter.java b/src/main/java/edu/harvard/iq/dataverse/S3PackageImporter.java index f423ca768be..806b6d32ef8 100644 --- a/src/main/java/edu/harvard/iq/dataverse/S3PackageImporter.java +++ b/src/main/java/edu/harvard/iq/dataverse/S3PackageImporter.java @@ -189,7 +189,7 @@ public DataFile createPackageDataFile(Dataset dataset, String folderName, long t } - //MAD: CLOSE THINGS + //MAD: CLOSE THINGS !! //String shaFileContents = IOUtils.toString(s3FilesSha.getObjectContent(), StandardCharsets.UTF_8); diff --git a/src/main/webapp/file-download-button-fragment.xhtml b/src/main/webapp/file-download-button-fragment.xhtml index a7206922824..20c12ad5ba4 100644 --- a/src/main/webapp/file-download-button-fragment.xhtml +++ b/src/main/webapp/file-download-button-fragment.xhtml @@ -113,8 +113,9 @@ #{bundle['file.compute']} + @@ -150,7 +151,7 @@
+ and (!FilePage.fileMetadata.dataFile.filePackage or + FilePage.fileMetadata.dataFile.filePackage and FilePage.fileMetadata.getDataFile().getOwner().getStorageIdentifier().startsWith('s3://'))}"> + @@ -362,7 +365,7 @@ + rendered="#{settingsWrapper.rsyncDownload and FilePage.fileMetadata.dataFile.filePackage and !FilePage.fileMetadata.getDataFile().getOwner().getStorageIdentifier().startsWith('s3://')}"> @@ -551,7 +554,8 @@
- + From 756de2b56e87bcb6a744416b0396c724ab6db680 Mon Sep 17 00:00:00 2001 From: matthew-a-dunlap Date: Tue, 27 Nov 2018 19:18:33 -0500 Subject: [PATCH 10/32] Share file entry code from replace and s3 #4949 Wired up for all flows, and multiple update bugs fixed. --- .../iq/dataverse/EditDatafilesPage.java | 13 +++ .../iq/dataverse/FileDownloadServiceBean.java | 8 ++ .../dataverse/S3PackagePopupFragmentBean.java | 14 +-- src/main/webapp/editdatafiles.xhtml | 53 +---------- .../file-download-button-fragment.xhtml | 17 ++-- .../webapp/file-download-popup-fragment.xhtml | 8 +- src/main/webapp/filesFragment.xhtml | 84 ++++++++++++++++- src/main/webapp/filesInfoFragment.xhtml | 94 ------------------- src/main/webapp/individualFileTable.xhtml | 60 ++++++++++++ .../s3-package-download-popup-fragment.xhtml | 24 ++--- 10 files changed, 192 insertions(+), 183 deletions(-) delete mode 100644 src/main/webapp/filesInfoFragment.xhtml create mode 100644 src/main/webapp/individualFileTable.xhtml diff --git a/src/main/java/edu/harvard/iq/dataverse/EditDatafilesPage.java b/src/main/java/edu/harvard/iq/dataverse/EditDatafilesPage.java index 3a20a08068c..176360ec49b 100644 --- a/src/main/java/edu/harvard/iq/dataverse/EditDatafilesPage.java +++ b/src/main/java/edu/harvard/iq/dataverse/EditDatafilesPage.java @@ -180,6 +180,7 @@ public String getSelectedFileIds() { return selectedFileIdsString; } + //If you change this the getFileMetadataToReplace should be changed in kind public DataFile getFileToReplace(){ if (!this.isFileReplaceOperation()){ return null; @@ -190,6 +191,18 @@ public DataFile getFileToReplace(){ return this.fileReplacePageHelper.getFileToReplace(); } + //MAD: This is an attempt to use the same ui for all file cards. + //We want that fragment to use fileMetadata because its easier to get from that to the needed info + public FileMetadata getFileMetadataToReplace(){ + if (!this.isFileReplaceOperation()){ + return null; + } + if (this.fileReplacePageHelper == null){ + return null; + } + return this.fileReplacePageHelper.getFileToReplace().getFileMetadata(); + } + public void setSelectedFileIds(String selectedFileIds) { selectedFileIdsString = selectedFileIds; } diff --git a/src/main/java/edu/harvard/iq/dataverse/FileDownloadServiceBean.java b/src/main/java/edu/harvard/iq/dataverse/FileDownloadServiceBean.java index 73458433bb8..cc8f895b370 100644 --- a/src/main/java/edu/harvard/iq/dataverse/FileDownloadServiceBean.java +++ b/src/main/java/edu/harvard/iq/dataverse/FileDownloadServiceBean.java @@ -145,6 +145,14 @@ public void writeGuestbookAndStartFileDownload(GuestbookResponse guestbookRespon logger.fine("issued file download redirect for datafile "+guestbookResponse.getDataFile().getId()); } + //MAD: I'm pretty sure we should filter out draft + public void writeGuestbookResponseRecord(GuestbookResponse guestbookResponse, FileMetadata fileMetadata, String format) { + if(!fileMetadata.getDatasetVersion().isDraft()){ + guestbookResponse = guestbookResponseService.modifyDatafileAndFormat(guestbookResponse, fileMetadata, format); + writeGuestbookResponseRecord(guestbookResponse); + } + } + public void writeGuestbookResponseRecord(GuestbookResponse guestbookResponse) { try { diff --git a/src/main/java/edu/harvard/iq/dataverse/S3PackagePopupFragmentBean.java b/src/main/java/edu/harvard/iq/dataverse/S3PackagePopupFragmentBean.java index dfcbc952b66..d268726e3ee 100644 --- a/src/main/java/edu/harvard/iq/dataverse/S3PackagePopupFragmentBean.java +++ b/src/main/java/edu/harvard/iq/dataverse/S3PackagePopupFragmentBean.java @@ -6,10 +6,6 @@ //MAD: This and s3importer should probably be in their own package package edu.harvard.iq.dataverse; -import edu.harvard.iq.dataverse.dataaccess.DataAccess; -import edu.harvard.iq.dataverse.dataaccess.S3AccessIO; -import edu.harvard.iq.dataverse.dataaccess.StorageIO; -import java.io.IOException; import javax.faces.view.ViewScoped; import javax.inject.Named; @@ -22,14 +18,14 @@ @Named public class S3PackagePopupFragmentBean implements java.io.Serializable { - DataFile df; + FileMetadata fm; - public void setDataFile(DataFile dataFile) { - df = dataFile; + public void setFileMetadata(FileMetadata fileMetadata) { + fm = fileMetadata; } - public DataFile getDataFile() { - return df; + public FileMetadata getFileMetadata() { + return fm; } // public String provideS3OneTimeUrl() throws IOException { diff --git a/src/main/webapp/editdatafiles.xhtml b/src/main/webapp/editdatafiles.xhtml index aab4541229e..fcf19e8bf8f 100755 --- a/src/main/webapp/editdatafiles.xhtml +++ b/src/main/webapp/editdatafiles.xhtml @@ -38,58 +38,15 @@ - - -
-
-
-
- - - - - -
- -
-
- -
-
-
-
-
- #{fileToReplace.fileMetadata.label} -
-
- #{fileToReplace.friendlyType} - - #{fileToReplace.filesize} KB - - #{fileToReplace.fileMetadata.getFileDateToDisplay()} -
#{fileToReplace.checksumType}: #{fileToReplace.checksumValue}
-
-
- -
- -
- - - - - - - - - - -
-
-
+ + + + +
diff --git a/src/main/webapp/file-download-button-fragment.xhtml b/src/main/webapp/file-download-button-fragment.xhtml index 20c12ad5ba4..97a1391a669 100644 --- a/src/main/webapp/file-download-button-fragment.xhtml +++ b/src/main/webapp/file-download-button-fragment.xhtml @@ -113,18 +113,18 @@ #{bundle['file.compute']} - - - + + - S3 Download #{fileMetadata.dataFile} + #{bundle.download} + #{bundle.download} @@ -141,7 +142,7 @@ diff --git a/src/main/webapp/file-download-popup-fragment.xhtml b/src/main/webapp/file-download-popup-fragment.xhtml index 21c0b954b63..ea8da2a9b3d 100644 --- a/src/main/webapp/file-download-popup-fragment.xhtml +++ b/src/main/webapp/file-download-popup-fragment.xhtml @@ -176,11 +176,11 @@ - - - + + + #{bundle['acceptTerms']} diff --git a/src/main/webapp/filesFragment.xhtml b/src/main/webapp/filesFragment.xhtml index d66118e91b0..51088665c96 100644 --- a/src/main/webapp/filesFragment.xhtml +++ b/src/main/webapp/filesFragment.xhtml @@ -153,8 +153,88 @@ - - + +
+ + + + + + + + + + + + + + + + +
+ +
+
+ +
+
+ + + + + + + #{fileMetadata.label} + + + + + #{fileMetadata.label} + + + + +
+ + + + + + + + + + + +
+ +
+
+ +
+ + + +
+ +
+ +
+ +
+ + + + + + + + + + +
+ diff --git a/src/main/webapp/filesInfoFragment.xhtml b/src/main/webapp/filesInfoFragment.xhtml deleted file mode 100644 index 8de06d90702..00000000000 --- a/src/main/webapp/filesInfoFragment.xhtml +++ /dev/null @@ -1,94 +0,0 @@ - - - - -
- - - - - - - - - - - - - - - - -
- -
-
- -
-
- - - - - - - #{fileMetadata.label} - - - - - #{fileMetadata.label} - - - - -
- - - - - - - - - - - -
- -
-
- -
- - - -
- -
- -
- -
- - - - - - - - - - -
- - \ No newline at end of file diff --git a/src/main/webapp/individualFileTable.xhtml b/src/main/webapp/individualFileTable.xhtml new file mode 100644 index 00000000000..a9e2cde4762 --- /dev/null +++ b/src/main/webapp/individualFileTable.xhtml @@ -0,0 +1,60 @@ + + +
+
+
+ + + + + +
+ +
+
+ +
+
+
+
+
+ #{fileMetadata.label} +
+
+ #{fileMetadata.dataFile.friendlyType} + - #{fileMetadata.dataFile.filesize} KB + - #{fileMetadata.fileDateToDisplay} +
#{fileMetadata.dataFile.checksumType}: #{fileMetadata.dataFile.checksumValue}
+
+
+ +
+ +
+ + + + + + + + + + +
+
+
+ + \ No newline at end of file diff --git a/src/main/webapp/s3-package-download-popup-fragment.xhtml b/src/main/webapp/s3-package-download-popup-fragment.xhtml index 9fc65ac80e9..92bae40616c 100644 --- a/src/main/webapp/s3-package-download-popup-fragment.xhtml +++ b/src/main/webapp/s3-package-download-popup-fragment.xhtml @@ -9,23 +9,10 @@ xmlns:iqbs="http://xmlns.jcp.org/jsf/composite/iqbs"> - - - - - - - - +
+ + +
@@ -34,9 +21,10 @@
-
#{systemConfig.getDataverseSiteUrl()}/api/access/datafile/#{s3PackagePopupFragmentBean.dataFile.id}
+
#{systemConfig.getDataverseSiteUrl()}/api/access/datafile/#{s3PackagePopupFragmentBean.fileMetadata.dataFile.id}
+
From 23196c005a49f98d480016b230f02273069b51f4 Mon Sep 17 00:00:00 2001 From: matthew-a-dunlap Date: Thu, 29 Nov 2018 15:17:48 -0500 Subject: [PATCH 11/32] dcm s3 download code cleanup #4949 --- .../iq/dataverse/DataCiteRESTfullClient.java | 1 - .../edu/harvard/iq/dataverse/DatasetPage.java | 5 +--- .../iq/dataverse/EditDatafilesPage.java | 13 --------- .../iq/dataverse/FileDownloadHelper.java | 6 ++-- .../iq/dataverse/FileDownloadServiceBean.java | 3 -- .../iq/dataverse/S3PackageImporter.java | 29 ++++--------------- .../dataverse/S3PackagePopupFragmentBean.java | 11 +------ .../dataverse/api/DownloadInstanceWriter.java | 1 - .../harvard/iq/dataverse/util/FileUtil.java | 2 -- src/main/webapp/WEB-INF/web.xml | 6 ++++ src/main/webapp/dataset.xhtml | 1 - src/main/webapp/editdatafiles.xhtml | 4 +-- .../file-download-button-fragment.xhtml | 2 -- src/main/webapp/file.xhtml | 4 --- .../s3-package-download-popup-fragment.xhtml | 2 +- 15 files changed, 18 insertions(+), 72 deletions(-) diff --git a/src/main/java/edu/harvard/iq/dataverse/DataCiteRESTfullClient.java b/src/main/java/edu/harvard/iq/dataverse/DataCiteRESTfullClient.java index d128996c91e..a8ddc3b06e9 100644 --- a/src/main/java/edu/harvard/iq/dataverse/DataCiteRESTfullClient.java +++ b/src/main/java/edu/harvard/iq/dataverse/DataCiteRESTfullClient.java @@ -219,7 +219,6 @@ public String inactiveDataset(String doi) { } } - //MAD: This shouldn't be hardcoded in our source.... public static void main(String[] args) throws Exception { String doi = "10.5072/DVN/274533"; DataCiteRESTfullClient client = new DataCiteRESTfullClient("https://mds.test.datacite.org", "DATACITE.HARVARD", "DVNapitest"); diff --git a/src/main/java/edu/harvard/iq/dataverse/DatasetPage.java b/src/main/java/edu/harvard/iq/dataverse/DatasetPage.java index 5e762000386..67cde84ed58 100644 --- a/src/main/java/edu/harvard/iq/dataverse/DatasetPage.java +++ b/src/main/java/edu/harvard/iq/dataverse/DatasetPage.java @@ -197,9 +197,6 @@ public enum DisplayMode { SettingsWrapper settingsWrapper; @Inject ProvPopupFragmentBean provPopupFragmentBean; -//MAD: Does this even do anything? Does the prov one for that matter? -// @Inject -// S3PackagePopupFragmentBean s3PopupFragmentBean; private Dataset dataset = new Dataset(); private EditMode editMode; @@ -1473,7 +1470,7 @@ private String init(boolean initFull) { logger.fine("Checking if rsync support is enabled."); if (DataCaptureModuleUtil.rsyncSupportEnabled(settingsWrapper.getValueForKey(SettingsServiceBean.Key.UploadMethods))) { try { -//MAD: I think this is the spot that keeps checking for rsync even though its gone thru the process + //TODO: This check could be improved to check the existence of rsync data before requesting a script ScriptRequestResponse scriptRequestResponse = commandEngine.submit(new RequestRsyncScriptCommand(dvRequestService.getDataverseRequest(), dataset)); logger.fine("script: " + scriptRequestResponse.getScript()); if(scriptRequestResponse.getScript()!=null && !scriptRequestResponse.getScript().isEmpty()){ diff --git a/src/main/java/edu/harvard/iq/dataverse/EditDatafilesPage.java b/src/main/java/edu/harvard/iq/dataverse/EditDatafilesPage.java index 176360ec49b..3a20a08068c 100644 --- a/src/main/java/edu/harvard/iq/dataverse/EditDatafilesPage.java +++ b/src/main/java/edu/harvard/iq/dataverse/EditDatafilesPage.java @@ -180,7 +180,6 @@ public String getSelectedFileIds() { return selectedFileIdsString; } - //If you change this the getFileMetadataToReplace should be changed in kind public DataFile getFileToReplace(){ if (!this.isFileReplaceOperation()){ return null; @@ -191,18 +190,6 @@ public DataFile getFileToReplace(){ return this.fileReplacePageHelper.getFileToReplace(); } - //MAD: This is an attempt to use the same ui for all file cards. - //We want that fragment to use fileMetadata because its easier to get from that to the needed info - public FileMetadata getFileMetadataToReplace(){ - if (!this.isFileReplaceOperation()){ - return null; - } - if (this.fileReplacePageHelper == null){ - return null; - } - return this.fileReplacePageHelper.getFileToReplace().getFileMetadata(); - } - public void setSelectedFileIds(String selectedFileIds) { selectedFileIdsString = selectedFileIds; } diff --git a/src/main/java/edu/harvard/iq/dataverse/FileDownloadHelper.java b/src/main/java/edu/harvard/iq/dataverse/FileDownloadHelper.java index 476a3bf97e3..16e7ca4ee05 100644 --- a/src/main/java/edu/harvard/iq/dataverse/FileDownloadHelper.java +++ b/src/main/java/edu/harvard/iq/dataverse/FileDownloadHelper.java @@ -309,7 +309,7 @@ public void writeGuestbookAndLaunchS3PackagePopup(GuestbookResponse guestbookRes requestContext.execute("PF('downloadPopup').hide()"); requestContext.execute("PF('downloadS3PackagePopup').show()"); requestContext.execute("handleResizeDialog('downloadS3PackagePopup')"); - +//MAD: Do I need to check this is draft? Also am I passing that this is s3? fileDownloadService.writeGuestbookResponseRecord(guestbookResponse); } } @@ -351,10 +351,8 @@ public void clearRequestAccessFiles(){ public void addMultipleFilesForRequestAccess(DataFile dataFile) { this.filesForRequestAccess.add(dataFile); - } + } - - private String selectedFileId = null; public String getSelectedFileId() { diff --git a/src/main/java/edu/harvard/iq/dataverse/FileDownloadServiceBean.java b/src/main/java/edu/harvard/iq/dataverse/FileDownloadServiceBean.java index cc8f895b370..18dbe1c49f5 100644 --- a/src/main/java/edu/harvard/iq/dataverse/FileDownloadServiceBean.java +++ b/src/main/java/edu/harvard/iq/dataverse/FileDownloadServiceBean.java @@ -145,7 +145,6 @@ public void writeGuestbookAndStartFileDownload(GuestbookResponse guestbookRespon logger.fine("issued file download redirect for datafile "+guestbookResponse.getDataFile().getId()); } - //MAD: I'm pretty sure we should filter out draft public void writeGuestbookResponseRecord(GuestbookResponse guestbookResponse, FileMetadata fileMetadata, String format) { if(!fileMetadata.getDatasetVersion().isDraft()){ guestbookResponse = guestbookResponseService.modifyDatafileAndFormat(guestbookResponse, fileMetadata, format); @@ -154,13 +153,11 @@ public void writeGuestbookResponseRecord(GuestbookResponse guestbookResponse, Fi } public void writeGuestbookResponseRecord(GuestbookResponse guestbookResponse) { - try { CreateGuestbookResponseCommand cmd = new CreateGuestbookResponseCommand(dvRequestService.getDataverseRequest(), guestbookResponse, guestbookResponse.getDataset()); commandEngine.submit(cmd); } catch (CommandException e) { //if an error occurs here then download won't happen no need for response recs... - } } diff --git a/src/main/java/edu/harvard/iq/dataverse/S3PackageImporter.java b/src/main/java/edu/harvard/iq/dataverse/S3PackageImporter.java index 806b6d32ef8..4fa6b8b944b 100644 --- a/src/main/java/edu/harvard/iq/dataverse/S3PackageImporter.java +++ b/src/main/java/edu/harvard/iq/dataverse/S3PackageImporter.java @@ -21,13 +21,10 @@ import edu.harvard.iq.dataverse.engine.command.DataverseRequest; import edu.harvard.iq.dataverse.engine.command.exception.IllegalCommandException; import edu.harvard.iq.dataverse.settings.SettingsServiceBean; -import edu.harvard.iq.dataverse.util.FileUtil; -import static edu.harvard.iq.dataverse.util.json.NullSafeJsonBuilder.jsonObjectBuilder; +import edu.harvard.iq.dataverse.util.FileUtil;; import java.io.BufferedReader; import java.io.IOException; -import java.io.InputStream; import java.io.InputStreamReader; -import java.nio.charset.StandardCharsets; import java.sql.Timestamp; import java.util.ArrayList; import java.util.Date; @@ -37,9 +34,6 @@ import javax.ejb.EJB; import javax.ejb.Stateless; import javax.inject.Named; -import javax.json.JsonObject; -import javax.json.JsonObjectBuilder; -import org.apache.commons.io.IOUtils; /** * This class is for importing files added to s3 outside of dataverse. @@ -63,7 +57,7 @@ public class S3PackageImporter extends AbstractApiBean implements java.io.Serial @EJB EjbDataverseEngine commandEngine; - //Copies from another s3 bucket to our own + //Coppies from another s3 bucket to our own public void copyFromS3(Dataset dataset, String s3ImportPath) throws IOException { try { s3 = AmazonS3ClientBuilder.standard().defaultClient(); @@ -72,11 +66,7 @@ public void copyFromS3(Dataset dataset, String s3ImportPath) throws IOException "Cannot instantiate a S3 client using; check your AWS credentials and region", e); } - - JsonObjectBuilder bld = jsonObjectBuilder(); - - String fileMode = FileRecordWriter.FILE_MODE_PACKAGE_FILE; - + String dcmBucketName = System.getProperty("dataverse.files.dcm-s3-bucket-name"); String dcmDatasetKey = s3ImportPath; String dvBucketName = System.getProperty("dataverse.files.s3-bucket-name"); @@ -165,10 +155,8 @@ public DataFile createPackageDataFile(Dataset dataset, String folderName, long t while((line = reader.readLine()) != null && checksumVal.isEmpty()) { logger.log(Level.INFO, "line {0}", new Object[]{line}); String[] splitLine = line.split(" "); - //logger.log(Level.INFO, "root package name {0}", new Object[]{rootPackageName + ".zip"}); - //logger.log(Level.INFO, "splitline {0} | {1}", new Object[]{splitLine[0], splitLine[1]}); - - //the sha file should only contain one line, but incase it doesn't we will check for the one for our zip + + //the sha file should only contain one entry, but incase it doesn't we will check for the one for our zip if(splitLine[1].contains(rootPackageName + ".zip")) { checksumVal = splitLine[0]; logger.log(Level.INFO, "checksumVal found {0}", new Object[]{checksumVal}); @@ -189,13 +177,6 @@ public DataFile createPackageDataFile(Dataset dataset, String folderName, long t } - //MAD: CLOSE THINGS !! - - //String shaFileContents = IOUtils.toString(s3FilesSha.getObjectContent(), StandardCharsets.UTF_8); - - //We parse the checksum from the passed sha 1 - //String checksumVal = shaFileContents. - logger.log(Level.INFO, "Checksum value for the package in Dataset {0} is: {1}", new Object[]{dataset.getIdentifier(), checksumVal}); diff --git a/src/main/java/edu/harvard/iq/dataverse/S3PackagePopupFragmentBean.java b/src/main/java/edu/harvard/iq/dataverse/S3PackagePopupFragmentBean.java index d268726e3ee..eadacfbe536 100644 --- a/src/main/java/edu/harvard/iq/dataverse/S3PackagePopupFragmentBean.java +++ b/src/main/java/edu/harvard/iq/dataverse/S3PackagePopupFragmentBean.java @@ -3,7 +3,7 @@ * To change this template file, choose Tools | Templates * and open the template in the editor. */ -//MAD: This and s3importer should probably be in their own package + package edu.harvard.iq.dataverse; import javax.faces.view.ViewScoped; @@ -28,14 +28,5 @@ public FileMetadata getFileMetadata() { return fm; } -// public String provideS3OneTimeUrl() throws IOException { -// if(df != null ) { -// StorageIO storageIO = DataAccess.getStorageIO(df); -// return ((S3AccessIO)storageIO).generateTemporaryS3Url(); -// } else { -// return ""; -// } -// -// } } \ No newline at end of file diff --git a/src/main/java/edu/harvard/iq/dataverse/api/DownloadInstanceWriter.java b/src/main/java/edu/harvard/iq/dataverse/api/DownloadInstanceWriter.java index 0e4471c536b..399e9a081df 100644 --- a/src/main/java/edu/harvard/iq/dataverse/api/DownloadInstanceWriter.java +++ b/src/main/java/edu/harvard/iq/dataverse/api/DownloadInstanceWriter.java @@ -4,7 +4,6 @@ * and open the template in the editor. */ -//MAD: WHY DOES THIS LIVE IN API package edu.harvard.iq.dataverse.api; import java.lang.reflect.Type; diff --git a/src/main/java/edu/harvard/iq/dataverse/util/FileUtil.java b/src/main/java/edu/harvard/iq/dataverse/util/FileUtil.java index 595b633a9a9..38f17e78b9b 100644 --- a/src/main/java/edu/harvard/iq/dataverse/util/FileUtil.java +++ b/src/main/java/edu/harvard/iq/dataverse/util/FileUtil.java @@ -1133,8 +1133,6 @@ public static String getFilesTempDirectory() { return filesTempDirectory; } - //OLD MAD: Refactor this again so it reuses code from S3AccessIO. - //OLD MAD: Also maybe refactor the generic named one to reflect that it creates fileAccessIO on its own. public static void generateS3PackageStorageIdentifier(DataFile dataFile) { String bucketName = System.getProperty("dataverse.files.s3-bucket-name"); String storageId = S3_IDENTIFIER_PREFIX + "://" + bucketName + ":" + dataFile.getFileMetadata().getLabel(); diff --git a/src/main/webapp/WEB-INF/web.xml b/src/main/webapp/WEB-INF/web.xml index 5ef5faf85f6..cb7c6ffdf17 100644 --- a/src/main/webapp/WEB-INF/web.xml +++ b/src/main/webapp/WEB-INF/web.xml @@ -22,6 +22,12 @@ org.jboss.weld.context.conversation.lazy false + + + javax.faces.PROJECT_STAGE + + Production + primefaces.THEME bootstrap diff --git a/src/main/webapp/dataset.xhtml b/src/main/webapp/dataset.xhtml index d7a34ef2a58..6e15fd588c5 100755 --- a/src/main/webapp/dataset.xhtml +++ b/src/main/webapp/dataset.xhtml @@ -1173,7 +1173,6 @@ - diff --git a/src/main/webapp/editdatafiles.xhtml b/src/main/webapp/editdatafiles.xhtml index fcf19e8bf8f..2f3898fe636 100755 --- a/src/main/webapp/editdatafiles.xhtml +++ b/src/main/webapp/editdatafiles.xhtml @@ -43,8 +43,8 @@ #{bundle['file.replace.original']} - - + +
diff --git a/src/main/webapp/file-download-button-fragment.xhtml b/src/main/webapp/file-download-button-fragment.xhtml index 97a1391a669..515a94e128f 100644 --- a/src/main/webapp/file-download-button-fragment.xhtml +++ b/src/main/webapp/file-download-button-fragment.xhtml @@ -113,8 +113,6 @@ #{bundle['file.compute']} - - - @@ -554,8 +552,6 @@ - diff --git a/src/main/webapp/s3-package-download-popup-fragment.xhtml b/src/main/webapp/s3-package-download-popup-fragment.xhtml index 92bae40616c..76f7608efae 100644 --- a/src/main/webapp/s3-package-download-popup-fragment.xhtml +++ b/src/main/webapp/s3-package-download-popup-fragment.xhtml @@ -8,7 +8,7 @@ xmlns:jsf="http://xmlns.jcp.org/jsf" xmlns:iqbs="http://xmlns.jcp.org/jsf/composite/iqbs"> - +
From 8827430e37d018a36fe2e6787ea89e80ec4e42b8 Mon Sep 17 00:00:00 2001 From: Michael Heppler Date: Thu, 29 Nov 2018 18:17:08 -0500 Subject: [PATCH 12/32] Cleaned up UI of new popup for S3 download [ref #4949] --- src/main/java/Bundle.properties | 7 ++-- src/main/webapp/dataset.xhtml | 2 +- src/main/webapp/editdatafiles.xhtml | 12 ++++--- .../file-download-button-fragment.xhtml | 9 +++-- src/main/webapp/individualFileTable.xhtml | 10 +++--- .../s3-package-download-popup-fragment.xhtml | 33 +++++++++++-------- 6 files changed, 41 insertions(+), 32 deletions(-) diff --git a/src/main/java/Bundle.properties b/src/main/java/Bundle.properties index 8a332cdde5b..d3e471dc3dc 100755 --- a/src/main/java/Bundle.properties +++ b/src/main/java/Bundle.properties @@ -70,10 +70,6 @@ toggleNavigation=Toggle navigation defaultBody=Default Body filter=Filter -s3packagedownload.title=Download Package File -s3packagedownload.instructions=Download this package file by copying and pasting the URL below into Wget or a download manager. Use of a web browser for this download is not recommended. User Guide - Downloading Package Files -s3packagedownload.urlHeader=Download URL - # dataverse_header.xhtml header.status.header=Status header.search.title=Search all dataverses... @@ -1620,6 +1616,9 @@ file.dataset.allFiles=All Files from this Dataset file.downloadDialog.header=Dataset Terms file.downloadDialog.tip=Please confirm and/or complete the information needed below in order to continue. file.requestAccessTermsDialog.tip=Please confirm and/or complete the information needed below in order to request access to files in this dataset. +s3packagedownload.title=Package File Download +s3packagedownload.instructions=Use the Download URL in a Wget command or a download manager to download this package file. Download via web browser is not recommended. User Guide - Downloading Package Files +s3packagedownload.urlHeader=Download URL file.search.placeholder=Search this dataset... file.results.btn.sort=Sort file.results.btn.sort.option.nameAZ=Name (A-Z) diff --git a/src/main/webapp/dataset.xhtml b/src/main/webapp/dataset.xhtml index 6e15fd588c5..7cbe0ad4483 100755 --- a/src/main/webapp/dataset.xhtml +++ b/src/main/webapp/dataset.xhtml @@ -1173,7 +1173,7 @@ - + diff --git a/src/main/webapp/editdatafiles.xhtml b/src/main/webapp/editdatafiles.xhtml index 2f3898fe636..32406ce430d 100755 --- a/src/main/webapp/editdatafiles.xhtml +++ b/src/main/webapp/editdatafiles.xhtml @@ -42,11 +42,13 @@ - - - - - +
+ + + + + +
diff --git a/src/main/webapp/file-download-button-fragment.xhtml b/src/main/webapp/file-download-button-fragment.xhtml index 515a94e128f..1cbad980964 100644 --- a/src/main/webapp/file-download-button-fragment.xhtml +++ b/src/main/webapp/file-download-button-fragment.xhtml @@ -112,9 +112,12 @@ #{bundle['file.compute']} + + - @@ -122,7 +125,7 @@ - #{bundle.download} + #{bundle.download} S3 -
-
+
+
- + @@ -27,10 +27,10 @@
-
+
- #{fileMetadata.label} + #{fileMetadata.label}
#{fileMetadata.dataFile.friendlyType} diff --git a/src/main/webapp/s3-package-download-popup-fragment.xhtml b/src/main/webapp/s3-package-download-popup-fragment.xhtml index 76f7608efae..2b97933d4aa 100644 --- a/src/main/webapp/s3-package-download-popup-fragment.xhtml +++ b/src/main/webapp/s3-package-download-popup-fragment.xhtml @@ -10,21 +10,26 @@
- - - -
- - - - - +
+

+ + + + + +

-
-
#{systemConfig.getDataverseSiteUrl()}/api/access/datafile/#{s3PackagePopupFragmentBean.fileMetadata.dataFile.id}
-
- - +
+ + + +
+
+ +

#{systemConfig.getDataverseSiteUrl()}/api/access/datafile/#{s3PackagePopupFragmentBean.fileMetadata.dataFile.id}

+
+
+
From 1802ef0fea15c3348ac0d1ca9d6b995ffc231702 Mon Sep 17 00:00:00 2001 From: Michael Heppler Date: Fri, 30 Nov 2018 10:00:17 -0500 Subject: [PATCH 13/32] Added placeholder User Guide section for S3 download, and link from popup, also removed dev code [ref #4949] --- doc/sphinx-guides/source/user/find-use-data.rst | 8 ++++++++ src/main/java/Bundle.properties | 2 +- src/main/webapp/file-download-button-fragment.xhtml | 9 +++------ 3 files changed, 12 insertions(+), 7 deletions(-) diff --git a/doc/sphinx-guides/source/user/find-use-data.rst b/doc/sphinx-guides/source/user/find-use-data.rst index 4d57afa11ec..cced917ef01 100755 --- a/doc/sphinx-guides/source/user/find-use-data.rst +++ b/doc/sphinx-guides/source/user/find-use-data.rst @@ -110,6 +110,14 @@ In a dataset containing a Dataverse Package, at the bottom of the dataset page, After you've downloaded the Dataverse Package, you may want to double-check that your download went perfectly. Under **Verify Data**, you'll find a command that you can run in your terminal that will initiate a checksum to ensure that the data you downloaded matches the data in Dataverse precisely. This way, you can ensure the integrity of the data you're working with. +.. _package_download_url: + +Downloading a Dataverse Package via URL +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +[[ ADD TEXT HERE ]] + + Explore Data ------------ diff --git a/src/main/java/Bundle.properties b/src/main/java/Bundle.properties index d3e471dc3dc..26f6dc9b304 100755 --- a/src/main/java/Bundle.properties +++ b/src/main/java/Bundle.properties @@ -1617,7 +1617,7 @@ file.downloadDialog.header=Dataset Terms file.downloadDialog.tip=Please confirm and/or complete the information needed below in order to continue. file.requestAccessTermsDialog.tip=Please confirm and/or complete the information needed below in order to request access to files in this dataset. s3packagedownload.title=Package File Download -s3packagedownload.instructions=Use the Download URL in a Wget command or a download manager to download this package file. Download via web browser is not recommended. User Guide - Downloading Package Files +s3packagedownload.instructions=Use the Download URL in a Wget command or a download manager to download this package file. Download via web browser is not recommended. User Guide - Downloading a Dataverse Package via URL s3packagedownload.urlHeader=Download URL file.search.placeholder=Search this dataset... file.results.btn.sort=Sort diff --git a/src/main/webapp/file-download-button-fragment.xhtml b/src/main/webapp/file-download-button-fragment.xhtml index 1cbad980964..c435e3108ab 100644 --- a/src/main/webapp/file-download-button-fragment.xhtml +++ b/src/main/webapp/file-download-button-fragment.xhtml @@ -113,11 +113,8 @@ #{bundle['file.compute']} - - - @@ -125,7 +122,7 @@ - #{bundle.download} S3 + #{bundle.download} ()); - - dataset.getLatestVersion().getFileMetadatas().add(fmd); - fmd.setDatasetVersion(dataset.getLatestVersion()); - - FileUtil.generateS3PackageStorageIdentifier(packageFile); - - GlobalIdServiceBean idServiceBean = GlobalIdServiceBean.getBean(packageFile.getProtocol(), commandEngine.getContext()); - if (packageFile.getIdentifier() == null || packageFile.getIdentifier().isEmpty()) { - String packageIdentifier = dataFileServiceBean.generateDataFileIdentifier(packageFile, idServiceBean); - packageFile.setIdentifier(packageIdentifier); - } - - String nonNullDefaultIfKeyNotFound = ""; - String protocol = commandEngine.getContext().settings().getValueForKey(SettingsServiceBean.Key.Protocol, nonNullDefaultIfKeyNotFound); - String authority = commandEngine.getContext().settings().getValueForKey(SettingsServiceBean.Key.Authority, nonNullDefaultIfKeyNotFound); + packageFile.setChecksumValue(checksumVal); - if (packageFile.getProtocol() == null) { - packageFile.setProtocol(protocol); - } - if (packageFile.getAuthority() == null) { - packageFile.setAuthority(authority); - } + packageFile.setFilesize(totalSize); + packageFile.setModificationTime(new Timestamp(new Date().getTime())); + packageFile.setCreateDate(new Timestamp(new Date().getTime())); + packageFile.setPermissionModificationTime(new Timestamp(new Date().getTime())); + packageFile.setOwner(dataset); + dataset.getFiles().add(packageFile); - if (!packageFile.isIdentifierRegistered()) { - String doiRetString = ""; - idServiceBean = GlobalIdServiceBean.getBean(commandEngine.getContext()); - try { - doiRetString = idServiceBean.createIdentifier(packageFile); - } catch (Throwable e) { - - } + packageFile.setIngestDone(); + + // set metadata and add to latest version + // Set early so we can generate the storage id with the info + FileMetadata fmd = new FileMetadata(); + fmd.setLabel(rootPackageName + ".zip"); + + fmd.setDataFile(packageFile); + packageFile.getFileMetadatas().add(fmd); + if (dataset.getLatestVersion().getFileMetadatas() == null) dataset.getLatestVersion().setFileMetadatas(new ArrayList<>()); + + dataset.getLatestVersion().getFileMetadatas().add(fmd); + fmd.setDatasetVersion(dataset.getLatestVersion()); + + FileUtil.generateS3PackageStorageIdentifier(packageFile); + + GlobalIdServiceBean idServiceBean = GlobalIdServiceBean.getBean(packageFile.getProtocol(), commandEngine.getContext()); + if (packageFile.getIdentifier() == null || packageFile.getIdentifier().isEmpty()) { + String packageIdentifier = dataFileServiceBean.generateDataFileIdentifier(packageFile, idServiceBean); + packageFile.setIdentifier(packageIdentifier); + } + + String nonNullDefaultIfKeyNotFound = ""; + String protocol = commandEngine.getContext().settings().getValueForKey(SettingsServiceBean.Key.Protocol, nonNullDefaultIfKeyNotFound); + String authority = commandEngine.getContext().settings().getValueForKey(SettingsServiceBean.Key.Authority, nonNullDefaultIfKeyNotFound); + + if (packageFile.getProtocol() == null) { + packageFile.setProtocol(protocol); + } + if (packageFile.getAuthority() == null) { + packageFile.setAuthority(authority); + } + + if (!packageFile.isIdentifierRegistered()) { + String doiRetString = ""; + idServiceBean = GlobalIdServiceBean.getBean(commandEngine.getContext()); + try { + doiRetString = idServiceBean.createIdentifier(packageFile); + } catch (Throwable e) { - // Check return value to make sure registration succeeded - if (!idServiceBean.registerWhenPublished() && doiRetString.contains(packageFile.getIdentifier())) { - packageFile.setIdentifierRegistered(true); - packageFile.setGlobalIdCreateTime(new Date()); - } } + // Check return value to make sure registration succeeded + if (!idServiceBean.registerWhenPublished() && doiRetString.contains(packageFile.getIdentifier())) { + packageFile.setIdentifierRegistered(true); + packageFile.setGlobalIdCreateTime(new Date()); + } + } + return packageFile; } diff --git a/src/main/webapp/individualFileTable.xhtml b/src/main/webapp/individualFileTable.xhtml index 5afbda7ac75..2add73eab3a 100644 --- a/src/main/webapp/individualFileTable.xhtml +++ b/src/main/webapp/individualFileTable.xhtml @@ -34,8 +34,8 @@
#{fileMetadata.dataFile.friendlyType} - - #{fileMetadata.dataFile.filesize} KB - - #{fileMetadata.fileDateToDisplay} + - #{fileMetadata.dataFile.friendlySize} + - #{fileMetadata.fileDateToDisplay}
#{fileMetadata.dataFile.checksumType}: #{fileMetadata.dataFile.checksumValue}
From dbc13d51299860c048cb634fcec4b98194cf0e25 Mon Sep 17 00:00:00 2001 From: matthew-a-dunlap Date: Fri, 30 Nov 2018 13:13:52 -0500 Subject: [PATCH 16/32] Present correct script name dcms3 #4949 --- .../iq/dataverse/datacapturemodule/DataCaptureModuleUtil.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/java/edu/harvard/iq/dataverse/datacapturemodule/DataCaptureModuleUtil.java b/src/main/java/edu/harvard/iq/dataverse/datacapturemodule/DataCaptureModuleUtil.java index 92f8252f45c..1aa384d205e 100644 --- a/src/main/java/edu/harvard/iq/dataverse/datacapturemodule/DataCaptureModuleUtil.java +++ b/src/main/java/edu/harvard/iq/dataverse/datacapturemodule/DataCaptureModuleUtil.java @@ -77,7 +77,7 @@ public static String getMessageFromException(DataCaptureModuleException ex) { } public static String getScriptName(DatasetVersion datasetVersion) { - return "upload-" + datasetVersion.getDataset().getIdentifier() + ".bash"; + return "upload-" + datasetVersion.getDataset().getIdentifier().replace("/", "_") + ".bash"; } } From 02a662228cf49e2a5340cac9448b5dd55c6167a6 Mon Sep 17 00:00:00 2001 From: matthew-a-dunlap Date: Fri, 30 Nov 2018 13:26:21 -0500 Subject: [PATCH 17/32] dataset w. package not look for rsync script #4949 --- src/main/java/edu/harvard/iq/dataverse/DatasetPage.java | 4 ++-- src/main/java/edu/harvard/iq/dataverse/EditDatafilesPage.java | 3 ++- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/src/main/java/edu/harvard/iq/dataverse/DatasetPage.java b/src/main/java/edu/harvard/iq/dataverse/DatasetPage.java index 67cde84ed58..d9949a36f0e 100644 --- a/src/main/java/edu/harvard/iq/dataverse/DatasetPage.java +++ b/src/main/java/edu/harvard/iq/dataverse/DatasetPage.java @@ -1468,9 +1468,9 @@ private String init(boolean initFull) { this.guestbookResponse = guestbookResponseService.initGuestbookResponseForFragment(workingVersion, null, session); this.getFileDownloadHelper().setGuestbookResponse(guestbookResponse); logger.fine("Checking if rsync support is enabled."); - if (DataCaptureModuleUtil.rsyncSupportEnabled(settingsWrapper.getValueForKey(SettingsServiceBean.Key.UploadMethods))) { + if (DataCaptureModuleUtil.rsyncSupportEnabled(settingsWrapper.getValueForKey(SettingsServiceBean.Key.UploadMethods)) + && dataset.getFiles().isEmpty()) { //only check for rsync if no files exist try { - //TODO: This check could be improved to check the existence of rsync data before requesting a script ScriptRequestResponse scriptRequestResponse = commandEngine.submit(new RequestRsyncScriptCommand(dvRequestService.getDataverseRequest(), dataset)); logger.fine("script: " + scriptRequestResponse.getScript()); if(scriptRequestResponse.getScript()!=null && !scriptRequestResponse.getScript().isEmpty()){ diff --git a/src/main/java/edu/harvard/iq/dataverse/EditDatafilesPage.java b/src/main/java/edu/harvard/iq/dataverse/EditDatafilesPage.java index 3a20a08068c..0d52c45b1e4 100644 --- a/src/main/java/edu/harvard/iq/dataverse/EditDatafilesPage.java +++ b/src/main/java/edu/harvard/iq/dataverse/EditDatafilesPage.java @@ -1728,7 +1728,8 @@ public void setHasRsyncScript(Boolean hasRsyncScript) { private void setUpRsync() { logger.fine("setUpRsync called..."); - if (DataCaptureModuleUtil.rsyncSupportEnabled(settingsWrapper.getValueForKey(SettingsServiceBean.Key.UploadMethods))) { + if (DataCaptureModuleUtil.rsyncSupportEnabled(settingsWrapper.getValueForKey(SettingsServiceBean.Key.UploadMethods)) + && dataset.getFiles().isEmpty()) { //only check for rsync if no files exist try { ScriptRequestResponse scriptRequestResponse = commandEngine.submit(new RequestRsyncScriptCommand(dvRequestService.getDataverseRequest(), dataset)); logger.fine("script: " + scriptRequestResponse.getScript()); From 42e59774bd29bf0d60ffc68571c9196634135f54 Mon Sep 17 00:00:00 2001 From: matthew-a-dunlap Date: Fri, 30 Nov 2018 14:21:16 -0500 Subject: [PATCH 18/32] code cleanup and better bundle naming #4949 --- src/main/java/Bundle.properties | 7 ++++--- src/main/webapp/WEB-INF/web.xml | 2 +- src/main/webapp/dataset.xhtml | 2 +- src/main/webapp/file.xhtml | 2 +- src/main/webapp/s3-package-download-popup-fragment.xhtml | 4 ++-- 5 files changed, 9 insertions(+), 8 deletions(-) diff --git a/src/main/java/Bundle.properties b/src/main/java/Bundle.properties index 4966513445c..4e56b012024 100755 --- a/src/main/java/Bundle.properties +++ b/src/main/java/Bundle.properties @@ -1617,9 +1617,6 @@ file.downloadDialog.header=Dataset Terms file.downloadDialog.tip=Please confirm and/or complete the information needed below in order to continue. file.requestAccessTermsDialog.tip=Please confirm and/or complete the information needed below in order to request access to files in this dataset. file.requestAccess.notAllowed=Requests for access are not accepted on the Dataset. -s3packagedownload.title=Package File Download -s3packagedownload.instructions=Use the Download URL in a Wget command or a download manager to download this package file. Download via web browser is not recommended. User Guide - Downloading a Dataverse Package via URL -s3packagedownload.urlHeader=Download URL file.search.placeholder=Search this dataset... file.results.btn.sort=Sort @@ -2143,3 +2140,7 @@ permission.DownloadFile.desc=Download a file permission.ViewUnpublishedDataset.desc=View an unpublished dataset and its files permission.ViewUnpublishedDataverse.desc=View an unpublished dataverse permission.AddDataset.desc=Add a dataset to a dataverse + +s3PackageDownload.title=Package File Download +s3PackageDownload.instructions=Use the Download URL in a Wget command or a download manager to download this package file. Download via web browser is not recommended. User Guide - Downloading a Dataverse Package via URL +s3PackageDownload.urlHeader=Download URL \ No newline at end of file diff --git a/src/main/webapp/WEB-INF/web.xml b/src/main/webapp/WEB-INF/web.xml index cb7c6ffdf17..129a5353616 100644 --- a/src/main/webapp/WEB-INF/web.xml +++ b/src/main/webapp/WEB-INF/web.xml @@ -22,7 +22,7 @@ org.jboss.weld.context.conversation.lazy false - + javax.faces.PROJECT_STAGE diff --git a/src/main/webapp/dataset.xhtml b/src/main/webapp/dataset.xhtml index 7cbe0ad4483..b64d43197a8 100755 --- a/src/main/webapp/dataset.xhtml +++ b/src/main/webapp/dataset.xhtml @@ -1173,7 +1173,7 @@ - + diff --git a/src/main/webapp/file.xhtml b/src/main/webapp/file.xhtml index 08add9d5431..f2e25ee9589 100644 --- a/src/main/webapp/file.xhtml +++ b/src/main/webapp/file.xhtml @@ -552,7 +552,7 @@
- + diff --git a/src/main/webapp/s3-package-download-popup-fragment.xhtml b/src/main/webapp/s3-package-download-popup-fragment.xhtml index 2b97933d4aa..836736134e1 100644 --- a/src/main/webapp/s3-package-download-popup-fragment.xhtml +++ b/src/main/webapp/s3-package-download-popup-fragment.xhtml @@ -13,7 +13,7 @@

- + @@ -25,7 +25,7 @@

- +

#{systemConfig.getDataverseSiteUrl()}/api/access/datafile/#{s3PackagePopupFragmentBean.fileMetadata.dataFile.id}

From ddf52d1936d8dbf2d89ad4ebc0f40dda247c6185 Mon Sep 17 00:00:00 2001 From: Derek Murphy Date: Fri, 30 Nov 2018 14:36:55 -0500 Subject: [PATCH 19/32] Package Download User Guide Section [#4949] Added the section "Downloading a Dataverse Package via URL". This should cover this feature for the User Guide. --- doc/sphinx-guides/source/user/find-use-data.rst | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/doc/sphinx-guides/source/user/find-use-data.rst b/doc/sphinx-guides/source/user/find-use-data.rst index d663c10e4b5..0bf11d90ce3 100755 --- a/doc/sphinx-guides/source/user/find-use-data.rst +++ b/doc/sphinx-guides/source/user/find-use-data.rst @@ -115,8 +115,7 @@ After you've downloaded the Dataverse Package, you may want to double-check that Downloading a Dataverse Package via URL ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ -[[ ADD TEXT HERE ]] - +Dataverse Packages are typically used to represent extremely large files or bundles containing a large number of files. Dataverse Packages are often too large to be reliably downloaded using a web browser. When you click to download a Dataverse Package, instead of automatically initiating the download in your web browser, Dataverse displays a plaintext URL for the location of the file. To ensure a fast, reliable, unbroken download, we recommend using `GNU Wget `_ in a command line terminal or using a download manager software of your choice. If you try to simply paste the URL into your web browser then the download may overwhelm your browser, resulting in an interrupted, timed out, or otherwise failed download. Explore Data ------------ From 1a53227a8482b95f6db9a4552f775c705c32089d Mon Sep 17 00:00:00 2001 From: Derek Murphy Date: Fri, 30 Nov 2018 14:56:27 -0500 Subject: [PATCH 20/32] Update to docs - Wget isn't magic [#4949] Based on Phil's review, changed the docs to no longer wildly overestimate Wget's capabilities ;) --- doc/sphinx-guides/source/user/find-use-data.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/sphinx-guides/source/user/find-use-data.rst b/doc/sphinx-guides/source/user/find-use-data.rst index 0bf11d90ce3..6c59ebc2987 100755 --- a/doc/sphinx-guides/source/user/find-use-data.rst +++ b/doc/sphinx-guides/source/user/find-use-data.rst @@ -115,7 +115,7 @@ After you've downloaded the Dataverse Package, you may want to double-check that Downloading a Dataverse Package via URL ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ -Dataverse Packages are typically used to represent extremely large files or bundles containing a large number of files. Dataverse Packages are often too large to be reliably downloaded using a web browser. When you click to download a Dataverse Package, instead of automatically initiating the download in your web browser, Dataverse displays a plaintext URL for the location of the file. To ensure a fast, reliable, unbroken download, we recommend using `GNU Wget `_ in a command line terminal or using a download manager software of your choice. If you try to simply paste the URL into your web browser then the download may overwhelm your browser, resulting in an interrupted, timed out, or otherwise failed download. +Dataverse Packages are typically used to represent extremely large files or bundles containing a large number of files. Dataverse Packages are often too large to be reliably downloaded using a web browser. When you click to download a Dataverse Package, instead of automatically initiating the download in your web browser, Dataverse displays a plaintext URL for the location of the file. To ensure a reliable, resumable download, we recommend using `GNU Wget `_ in a command line terminal or using a download manager software of your choice. If you try to simply paste the URL into your web browser then the download may overwhelm your browser, resulting in an interrupted, timed out, or otherwise failed download. Explore Data ------------ From 1db38db82eab8fbf39aa3b3041c8fd28c830bf89 Mon Sep 17 00:00:00 2001 From: matthew-a-dunlap Date: Fri, 30 Nov 2018 16:48:36 -0500 Subject: [PATCH 21/32] dcms3 download only with downloadMethod #4949 --- .../java/edu/harvard/iq/dataverse/util/SystemConfig.java | 9 ++++++++- src/main/webapp/file-download-button-fragment.xhtml | 8 ++++---- src/main/webapp/file.xhtml | 5 +++-- src/main/webapp/filesFragment.xhtml | 9 +++++---- 4 files changed, 20 insertions(+), 11 deletions(-) diff --git a/src/main/java/edu/harvard/iq/dataverse/util/SystemConfig.java b/src/main/java/edu/harvard/iq/dataverse/util/SystemConfig.java index ccf5bbef19e..f56679a0229 100644 --- a/src/main/java/edu/harvard/iq/dataverse/util/SystemConfig.java +++ b/src/main/java/edu/harvard/iq/dataverse/util/SystemConfig.java @@ -893,7 +893,7 @@ public enum FileDownloadMethods { * go through Glassfish. */ RSYNC("rsal/rsync"), - NATIVE("NATIVE"); + NATIVE("native/http"); private final String text; private FileDownloadMethods(final String text) { @@ -1011,6 +1011,13 @@ public boolean isRsyncDownload() return downloadMethods !=null && downloadMethods.toLowerCase().contains(SystemConfig.FileDownloadMethods.RSYNC.toString()); } + public boolean isHTTPDownload() + { + String downloadMethods = settingsService.getValueForKey(SettingsServiceBean.Key.DownloadMethods); + logger.warning("Download Methods:" + downloadMethods); + return downloadMethods !=null && downloadMethods.toLowerCase().contains(SystemConfig.FileDownloadMethods.NATIVE.toString()); + } + private Boolean getUploadMethodAvailable(String method){ String uploadMethods = settingsService.getValueForKey(SettingsServiceBean.Key.UploadMethods); if (uploadMethods==null){ diff --git a/src/main/webapp/file-download-button-fragment.xhtml b/src/main/webapp/file-download-button-fragment.xhtml index c435e3108ab..68cd3f402a0 100644 --- a/src/main/webapp/file-download-button-fragment.xhtml +++ b/src/main/webapp/file-download-button-fragment.xhtml @@ -114,7 +114,7 @@ @@ -126,7 +126,7 @@ + FilePage.fileMetadata.dataFile.filePackage and systemConfig.HTTPDownload)}"> @@ -363,7 +363,8 @@ + rendered="#{settingsWrapper.rsyncDownload and FilePage.fileMetadata.dataFile.filePackage and systemConfig.rsyncDownload + and !FilePage.fileMetadata.getDataFile().getOwner().getStorageIdentifier().startsWith('s3://') }"> diff --git a/src/main/webapp/filesFragment.xhtml b/src/main/webapp/filesFragment.xhtml index 51088665c96..71c7a78f9c4 100644 --- a/src/main/webapp/filesFragment.xhtml +++ b/src/main/webapp/filesFragment.xhtml @@ -378,7 +378,7 @@
+ fileMetadata.dataFile.filePackage and systemConfig.HTTPDownload}"> @@ -391,8 +391,9 @@
-
+
@@ -401,5 +402,5 @@
- + From a6d4f96fcd35400ebd8bf39d63bb568774287c53 Mon Sep 17 00:00:00 2001 From: matthew-a-dunlap Date: Fri, 30 Nov 2018 18:05:58 -0500 Subject: [PATCH 22/32] remove s3 naming from package code #4949 in the future this code may be used for non-s3 packages and so should be more agnostic. --- src/main/java/Bundle.properties | 6 +++--- .../edu/harvard/iq/dataverse/DatasetPage.java | 2 +- .../harvard/iq/dataverse/FileDownloadHelper.java | 8 ++++---- ...ntBean.java => PackagePopupFragmentBean.java} | 2 +- src/main/webapp/WEB-INF/web.xml | 3 ++- src/main/webapp/dataset.xhtml | 4 ++-- .../webapp/file-download-button-fragment.xhtml | 16 ++++++++-------- .../webapp/file-download-popup-fragment.xhtml | 8 ++++---- src/main/webapp/file.xhtml | 4 ++-- ...tml => package-download-popup-fragment.xhtml} | 12 ++++++------ 10 files changed, 33 insertions(+), 32 deletions(-) rename src/main/java/edu/harvard/iq/dataverse/{S3PackagePopupFragmentBean.java => PackagePopupFragmentBean.java} (87%) rename src/main/webapp/{s3-package-download-popup-fragment.xhtml => package-download-popup-fragment.xhtml} (71%) diff --git a/src/main/java/Bundle.properties b/src/main/java/Bundle.properties index 4e56b012024..a5daebebf59 100755 --- a/src/main/java/Bundle.properties +++ b/src/main/java/Bundle.properties @@ -2141,6 +2141,6 @@ permission.ViewUnpublishedDataset.desc=View an unpublished dataset and its files permission.ViewUnpublishedDataverse.desc=View an unpublished dataverse permission.AddDataset.desc=Add a dataset to a dataverse -s3PackageDownload.title=Package File Download -s3PackageDownload.instructions=Use the Download URL in a Wget command or a download manager to download this package file. Download via web browser is not recommended. User Guide - Downloading a Dataverse Package via URL -s3PackageDownload.urlHeader=Download URL \ No newline at end of file +packageDownload.title=Package File Download +packageDownload.instructions=Use the Download URL in a Wget command or a download manager to download this package file. Download via web browser is not recommended. User Guide - Downloading a Dataverse Package via URL +packageDownload.urlHeader=Download URL \ No newline at end of file diff --git a/src/main/java/edu/harvard/iq/dataverse/DatasetPage.java b/src/main/java/edu/harvard/iq/dataverse/DatasetPage.java index d9949a36f0e..e7aa01bc3d4 100644 --- a/src/main/java/edu/harvard/iq/dataverse/DatasetPage.java +++ b/src/main/java/edu/harvard/iq/dataverse/DatasetPage.java @@ -1,7 +1,7 @@ package edu.harvard.iq.dataverse; import edu.harvard.iq.dataverse.provenance.ProvPopupFragmentBean; -import edu.harvard.iq.dataverse.S3PackagePopupFragmentBean; +import edu.harvard.iq.dataverse.PackagePopupFragmentBean; import edu.harvard.iq.dataverse.api.AbstractApiBean; import edu.harvard.iq.dataverse.authorization.AuthenticationServiceBean; import edu.harvard.iq.dataverse.authorization.Permission; diff --git a/src/main/java/edu/harvard/iq/dataverse/FileDownloadHelper.java b/src/main/java/edu/harvard/iq/dataverse/FileDownloadHelper.java index c9da67e26e4..e93385ae7a7 100644 --- a/src/main/java/edu/harvard/iq/dataverse/FileDownloadHelper.java +++ b/src/main/java/edu/harvard/iq/dataverse/FileDownloadHelper.java @@ -299,7 +299,7 @@ public void writeGuestbookAndLaunchExploreTool(GuestbookResponse guestbookRespon requestContext.execute("PF('downloadPopup').hide()"); } - public void writeGuestbookAndLaunchS3PackagePopup(GuestbookResponse guestbookResponse) { + public void writeGuestbookAndLaunchPackagePopup(GuestbookResponse guestbookResponse) { RequestContext requestContext = RequestContext.getCurrentInstance(); boolean valid = validateGuestbookResponse(guestbookResponse); @@ -307,9 +307,9 @@ public void writeGuestbookAndLaunchS3PackagePopup(GuestbookResponse guestbookRes JH.addMessage(FacesMessage.SEVERITY_ERROR, JH.localize("dataset.message.validationError")); } else { requestContext.execute("PF('downloadPopup').hide()"); - requestContext.execute("PF('downloadS3PackagePopup').show()"); - requestContext.execute("handleResizeDialog('downloadS3PackagePopup')"); -//MAD: Do I need to check this is draft? Also am I passing that this is s3? + requestContext.execute("PF('downloadPackagePopup').show()"); + requestContext.execute("handleResizeDialog('downloadPackagePopup')"); + fileDownloadService.writeGuestbookResponseRecord(guestbookResponse); } } diff --git a/src/main/java/edu/harvard/iq/dataverse/S3PackagePopupFragmentBean.java b/src/main/java/edu/harvard/iq/dataverse/PackagePopupFragmentBean.java similarity index 87% rename from src/main/java/edu/harvard/iq/dataverse/S3PackagePopupFragmentBean.java rename to src/main/java/edu/harvard/iq/dataverse/PackagePopupFragmentBean.java index eadacfbe536..fac2abeddb8 100644 --- a/src/main/java/edu/harvard/iq/dataverse/S3PackagePopupFragmentBean.java +++ b/src/main/java/edu/harvard/iq/dataverse/PackagePopupFragmentBean.java @@ -16,7 +16,7 @@ @ViewScoped @Named -public class S3PackagePopupFragmentBean implements java.io.Serializable { +public class PackagePopupFragmentBean implements java.io.Serializable { FileMetadata fm; diff --git a/src/main/webapp/WEB-INF/web.xml b/src/main/webapp/WEB-INF/web.xml index 129a5353616..24d19cbd909 100644 --- a/src/main/webapp/WEB-INF/web.xml +++ b/src/main/webapp/WEB-INF/web.xml @@ -22,7 +22,8 @@ org.jboss.weld.context.conversation.lazy false - + javax.faces.PROJECT_STAGE diff --git a/src/main/webapp/dataset.xhtml b/src/main/webapp/dataset.xhtml index b64d43197a8..117be978f52 100755 --- a/src/main/webapp/dataset.xhtml +++ b/src/main/webapp/dataset.xhtml @@ -1173,9 +1173,9 @@ - + - + diff --git a/src/main/webapp/file-download-button-fragment.xhtml b/src/main/webapp/file-download-button-fragment.xhtml index 68cd3f402a0..f8f7f50a532 100644 --- a/src/main/webapp/file-download-button-fragment.xhtml +++ b/src/main/webapp/file-download-button-fragment.xhtml @@ -117,11 +117,11 @@ and fileMetadata.dataFile.filePackage and systemConfig.HTTPDownload}" type="button" styleClass="btn btn-default" process="@this" disabled="#{(fileMetadata.dataFile.ingestInProgress or lockedFromDownload) ? 'disabled' : ''}" - update="@widgetVar(downloadS3PackagePopup)" oncomplete="PF('downloadS3PackagePopup').show();handleResizeDialog('downloadS3PackagePopup');"> - - - - + update="@widgetVar(downloadPackagePopup)" oncomplete="PF('downloadPackagePopup').show();handleResizeDialog('downloadPackagePopup');"> + + + + #{bundle.download} @@ -130,10 +130,10 @@ type="button" styleClass="btn btn-default" process="@this" disabled="#{(fileMetadata.dataFile.ingestInProgress or lockedFromDownload) ? 'disabled' : ''}" - action="#{guestbookResponseService.modifyDatafileAndFormat(guestbookResponse, fileMetadata, 's3Package')}" + action="#{guestbookResponseService.modifyDatafileAndFormat(guestbookResponse, fileMetadata, 'package')}" update="@widgetVar(downloadPopup)" oncomplete="PF('downloadPopup').show();handleResizeDialog('downloadPopup');"> - - + + #{bundle.download} diff --git a/src/main/webapp/file-download-popup-fragment.xhtml b/src/main/webapp/file-download-popup-fragment.xhtml index ea8da2a9b3d..a3c0d5e8e56 100644 --- a/src/main/webapp/file-download-popup-fragment.xhtml +++ b/src/main/webapp/file-download-popup-fragment.xhtml @@ -150,7 +150,7 @@ @@ -176,9 +176,9 @@ - - + + #{bundle['acceptTerms']} diff --git a/src/main/webapp/file.xhtml b/src/main/webapp/file.xhtml index f310a27a401..ac4c3c1b780 100644 --- a/src/main/webapp/file.xhtml +++ b/src/main/webapp/file.xhtml @@ -553,8 +553,8 @@
- - + + diff --git a/src/main/webapp/s3-package-download-popup-fragment.xhtml b/src/main/webapp/package-download-popup-fragment.xhtml similarity index 71% rename from src/main/webapp/s3-package-download-popup-fragment.xhtml rename to src/main/webapp/package-download-popup-fragment.xhtml index 836736134e1..3900b2f7f9a 100644 --- a/src/main/webapp/s3-package-download-popup-fragment.xhtml +++ b/src/main/webapp/package-download-popup-fragment.xhtml @@ -9,11 +9,11 @@ xmlns:iqbs="http://xmlns.jcp.org/jsf/composite/iqbs"> -
+

- + @@ -21,15 +21,15 @@

- +
- -

#{systemConfig.getDataverseSiteUrl()}/api/access/datafile/#{s3PackagePopupFragmentBean.fileMetadata.dataFile.id}

+ +

#{systemConfig.getDataverseSiteUrl()}/api/access/datafile/#{packagePopupFragmentBean.fileMetadata.dataFile.id}

- +
From 1938862698c9f39a50ee7cc08e5231ba3e5ce262 Mon Sep 17 00:00:00 2001 From: matthew-a-dunlap Date: Tue, 4 Dec 2018 17:55:35 -0500 Subject: [PATCH 23/32] move&format dcm s3 docs Editing still needed --- .../source/developers/big-data-support.rst | 136 ++++++++++++++++++ 1 file changed, 136 insertions(+) diff --git a/doc/sphinx-guides/source/developers/big-data-support.rst b/doc/sphinx-guides/source/developers/big-data-support.rst index 567fb99ec7a..efe5092c2d4 100644 --- a/doc/sphinx-guides/source/developers/big-data-support.rst +++ b/doc/sphinx-guides/source/developers/big-data-support.rst @@ -101,6 +101,142 @@ At this point you should be able to download a placeholder rsync script. Dataver Now the files are in place and you need to send JSON to Dataverse with a success or failure message as described above. Make a copy of ``doc/sphinx-guides/source/_static/installation/files/root/big-data-support/checksumValidationSuccess.json`` and put the identifier in place such as "X1METO" under "uploadFolder"). Then use curl as described above to send the JSON. +Steps to set up a DCM via Docker for Development +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +If you need a fully operating DCM client for development purposes, these steps will guide you to setting one up. This includes steps to set up the DCM on S3 variant. + +Docker Image Set-up +^^^^^^^^^^^^^^^^^^^ + +- Install docker if you do not have it +- Here are the steps copied out of docker-aio/readme.md & docker-dcm/readme.txt . If the info on this page is consolidated to a doc, it may be good to look back at the guides themselves : + + - `cd conf/docker-aio` and run `./0prep_deps.sh` to create Glassfish and Solr tarballs in conf/docker-aio/dv/deps. + - run `./1prep.sh` + - build the docker image: `docker build -t dv0 -f c7.dockerfile .` + - `cd ../docker-dcm` and run `./0prep.sh` + - build dcm/dv0dcm images with docker-compose: `docker-compose -f docker-compose.yml build` + - start containers: `docker-compose -f docker-compose.yml up -d` + - wait for container to show "healthy" (aka - `docker ps`), then wait another 4-5 minutes (even though it shows healthy, glassfish is still standing itself up), then run dataverse app installation: `docker exec -it dvsrv /opt/dv/install.bash` + - configure dataverse application to use DCM (run from outside the container): `docker exec -it dvsrv /opt/dv/configure_dcm.sh` + - The dataverse installation is accessible at `http://localhost:8084`. + - You may need to change the DoiProvider inside dvsrv (ezid does not work): + + - `curl -X DELETE -d EZID "localhost:8080/api/admin/settings/:DoiProvider"` + - `curl -X PUT -d DataCite "localhost:8080/api/admin/settings/:DoiProvider"` + - also change the doi.baseUrlString, doi.username, doi.password + +Setting up the S3 DCM Variant +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +- Before: Make sure to do the "To run rsync posix in docker" steps first. +- Before: the default bucket for dcm to hold files in s3 is named test-dcm. It is coded into post_upload.bash (line 30). + + - This bucket should exist on our testing environment, but if it is not there go and create it through the browser ui + +- `docker cp data-capture-module/scn/post_upload_s3.bash dcmsrv:/opt/dcm/scn/post_upload.bash` +- Install aws on dcmsrv and symlink it + + - `pip install awscli --upgrade --user` + - `export PATH=~/.local/bin:$PATH` + +- Configure aws bucket on dcmsrv + + - you need a credentials files in ~/.aws + + - `mkdir ~/.aws` + - `yum install nano` (or use a different editor below) + - `nano ~/.aws/credentials` + + - Structure like: + + - `[default]` + - `aws_access_key_id = ` + - `aws_secret_access_key =` + +- Dataverse configuration (on dvsrv): + + - Set s3 as the storage driver + + - `cd /opt/glassfish4/bin/` + - `./asadmin delete-jvm-options "\-Ddataverse.files.storage-driver-id=file"` + - `./asadmin create-jvm-options "\-Ddataverse.files.storage-driver-id=s3"` + + - Set up aws configs for dataverse + + - you may need this: `yum install awscli` + - `mkdir ~/.aws` + - `yum install nano` (or use a different editor below) + - `nano ~/.aws/credentials` + + - Structure like: + + - `[default]` + - `aws_access_key_id = ` + - `aws_secret_access_key =` + + - ALSO: create a region file + + - `nano ~/.aws/config` + - contents: + + - `[default]` + - `region = us-east-1` + + - Add the bucket names to dataverse + + - s3 bucket for dataverse + + - `/usr/local/glassfish4/glassfish/bin/asadmin create-jvm-options "-Ddataverse.files.s3-bucket-name=iqsstestdcmbucket"` + + - s3 bucket for dcm (as dataverse needs to do the copy over) + + - `/usr/local/glassfish4/glassfish/bin/asadmin create-jvm-options "-Ddataverse.files.dcm-s3-bucket-name=test-dcm"` + + - [set download method to be http] + +Using the DCM Docker Containers +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +- Create dataset and download rsync upload script +- Upload script to dcm_client (if needed, you can probably do all the actions for create/download inside dcm_client) + + - `docker cp ~/Downloads/upload-FK2_NN49YM.bash dcm_client:/tmp` + +- Create a folder of files to upload (you just need something...) +- Run script + + - `bash ./upload-FK2_NN49YM.bash` + +- manually run post_upload.bash on dcmsrv + + - `bash ./opt/dcm/scn/post_upload.bash` + +Additional DCM docker development tips +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +- You can completely blow away all the docker images with these commands (including non dcm ones!) + + - `docker stop dvsrv` + - `docker stop dcm_client` + - `docker stop dcmsrv` + - `docker rm $(docker ps -a -q)` + - `docker rmi $(docker images -q)` + +- There are a few logs to tail + + - dvsrv : `tail -n 2000 -f /opt/glassfish4/glassfish/domains/domain1/logs/server.log` + - dcmsrv : `tail -n 2000 -f /var/log/lighttpd/breakage.log` + - dcmsrv : `tail -n 2000 -f /var/log/lighttpd/access.log` + +- I have attached a script for redeploying dataverse into your docker image if you just want to update the code. You'll probably need to tweak the paths [deploy_dataverse_into_docker.sh.zip](https://github.com/IQSS/dataverse/files/2237096/deploy_dataverse_into_docker.sh.zip) + + - This script is also currently broken by the installer bug + +- Note that by default the docker container will stop running if the process it is following is turned off. For example flask with dcmsrv. You can get around this by having the script being followed never close (e.g. sleep infinity) https://stackoverflow.com/questions/31870222/how-can-i-keep-container-running-on-kubernetes +- You may have to restart the glassfish domain occasionally to deal with memory filling up. If deployment is getting reallllllly slow, its a good time. + Troubleshooting ~~~~~~~~~~~~~~~ From f109aaf09131f88fd7e9d743446237bb6d3ff168 Mon Sep 17 00:00:00 2001 From: matthew-a-dunlap Date: Wed, 5 Dec 2018 12:36:12 -0500 Subject: [PATCH 24/32] Switch file info frag name #4949 --- src/main/webapp/editdatafiles.xhtml | 2 +- ...individualFileTable.xhtml => file-info-fragment.xhtml.xhtml} | 0 src/main/webapp/package-download-popup-fragment.xhtml | 2 +- 3 files changed, 2 insertions(+), 2 deletions(-) rename src/main/webapp/{individualFileTable.xhtml => file-info-fragment.xhtml.xhtml} (100%) diff --git a/src/main/webapp/editdatafiles.xhtml b/src/main/webapp/editdatafiles.xhtml index 32406ce430d..db847210924 100755 --- a/src/main/webapp/editdatafiles.xhtml +++ b/src/main/webapp/editdatafiles.xhtml @@ -43,7 +43,7 @@ #{bundle['file.replace.original']}
- + diff --git a/src/main/webapp/individualFileTable.xhtml b/src/main/webapp/file-info-fragment.xhtml.xhtml similarity index 100% rename from src/main/webapp/individualFileTable.xhtml rename to src/main/webapp/file-info-fragment.xhtml.xhtml diff --git a/src/main/webapp/package-download-popup-fragment.xhtml b/src/main/webapp/package-download-popup-fragment.xhtml index 3900b2f7f9a..e1e229d5113 100644 --- a/src/main/webapp/package-download-popup-fragment.xhtml +++ b/src/main/webapp/package-download-popup-fragment.xhtml @@ -20,7 +20,7 @@

- +
From 5e0a0fbd54b73d867f1425d511d6c35e7b59b362 Mon Sep 17 00:00:00 2001 From: matthew-a-dunlap Date: Wed, 5 Dec 2018 12:36:49 -0500 Subject: [PATCH 25/32] code cleanup and test fixing #4949 --- .../java/edu/harvard/iq/dataverse/S3PackageImporter.java | 6 +++--- .../java/edu/harvard/iq/dataverse/util/SystemConfig.java | 6 ++---- .../datacapturemodule/DataCaptureModuleUtilTest.java | 6 +++--- 3 files changed, 8 insertions(+), 10 deletions(-) diff --git a/src/main/java/edu/harvard/iq/dataverse/S3PackageImporter.java b/src/main/java/edu/harvard/iq/dataverse/S3PackageImporter.java index ef7820d35cf..953fbeeb820 100644 --- a/src/main/java/edu/harvard/iq/dataverse/S3PackageImporter.java +++ b/src/main/java/edu/harvard/iq/dataverse/S3PackageImporter.java @@ -155,13 +155,13 @@ public DataFile createPackageDataFile(Dataset dataset, String folderName, long t try { String line; while((line = reader.readLine()) != null && checksumVal.isEmpty()) { - logger.log(Level.INFO, "line {0}", new Object[]{line}); + logger.log(Level.FINE, "line {0}", new Object[]{line}); String[] splitLine = line.split(" "); //the sha file should only contain one entry, but incase it doesn't we will check for the one for our zip if(splitLine[1].contains(rootPackageName + ".zip")) { checksumVal = splitLine[0]; - logger.log(Level.INFO, "checksumVal found {0}", new Object[]{checksumVal}); + logger.log(Level.FINE, "checksumVal found {0}", new Object[]{checksumVal}); } } if(checksumVal.isEmpty()) { @@ -179,7 +179,7 @@ public DataFile createPackageDataFile(Dataset dataset, String folderName, long t } - logger.log(Level.INFO, "Checksum value for the package in Dataset {0} is: {1}", + logger.log(Level.FINE, "Checksum value for the package in Dataset {0} is: {1}", new Object[]{dataset.getIdentifier(), checksumVal}); packageFile.setChecksumValue(checksumVal); diff --git a/src/main/java/edu/harvard/iq/dataverse/util/SystemConfig.java b/src/main/java/edu/harvard/iq/dataverse/util/SystemConfig.java index f56679a0229..fd059640780 100644 --- a/src/main/java/edu/harvard/iq/dataverse/util/SystemConfig.java +++ b/src/main/java/edu/harvard/iq/dataverse/util/SystemConfig.java @@ -1005,14 +1005,12 @@ public boolean isRsyncOnly(){ } } - public boolean isRsyncDownload() - { + public boolean isRsyncDownload() { String downloadMethods = settingsService.getValueForKey(SettingsServiceBean.Key.DownloadMethods); return downloadMethods !=null && downloadMethods.toLowerCase().contains(SystemConfig.FileDownloadMethods.RSYNC.toString()); } - public boolean isHTTPDownload() - { + public boolean isHTTPDownload() { String downloadMethods = settingsService.getValueForKey(SettingsServiceBean.Key.DownloadMethods); logger.warning("Download Methods:" + downloadMethods); return downloadMethods !=null && downloadMethods.toLowerCase().contains(SystemConfig.FileDownloadMethods.NATIVE.toString()); diff --git a/src/test/java/edu/harvard/iq/dataverse/datacapturemodule/DataCaptureModuleUtilTest.java b/src/test/java/edu/harvard/iq/dataverse/datacapturemodule/DataCaptureModuleUtilTest.java index 17f67c31b7f..a00daef63c2 100644 --- a/src/test/java/edu/harvard/iq/dataverse/datacapturemodule/DataCaptureModuleUtilTest.java +++ b/src/test/java/edu/harvard/iq/dataverse/datacapturemodule/DataCaptureModuleUtilTest.java @@ -28,9 +28,9 @@ public void testRsyncSupportEnabled() { assertEquals(false, DataCaptureModuleUtil.rsyncSupportEnabled(null)); assertEquals(true, DataCaptureModuleUtil.rsyncSupportEnabled("dcm/rsync+ssh")); // Comma sepratated lists of upload methods are supported. - assertEquals(false, DataCaptureModuleUtil.rsyncSupportEnabled("NATIVE:dcm/rsync+ssh")); - assertEquals(true, DataCaptureModuleUtil.rsyncSupportEnabled("NATIVE,dcm/rsync+ssh")); - assertEquals(false, DataCaptureModuleUtil.rsyncSupportEnabled("NATIVE")); + assertEquals(false, DataCaptureModuleUtil.rsyncSupportEnabled("native/http:dcm/rsync+ssh")); + assertEquals(true, DataCaptureModuleUtil.rsyncSupportEnabled("native/http,dcm/rsync+ssh")); + assertEquals(false, DataCaptureModuleUtil.rsyncSupportEnabled("native/http")); assertEquals(false, DataCaptureModuleUtil.rsyncSupportEnabled("junk")); } From ea766176b2a845d840050741a05b28e161214509 Mon Sep 17 00:00:00 2001 From: matthew-a-dunlap Date: Wed, 5 Dec 2018 12:37:00 -0500 Subject: [PATCH 26/32] More doc cleanup #4949 --- .../source/developers/big-data-support.rst | 161 +++++++++--------- 1 file changed, 79 insertions(+), 82 deletions(-) diff --git a/doc/sphinx-guides/source/developers/big-data-support.rst b/doc/sphinx-guides/source/developers/big-data-support.rst index efe5092c2d4..32b34832968 100644 --- a/doc/sphinx-guides/source/developers/big-data-support.rst +++ b/doc/sphinx-guides/source/developers/big-data-support.rst @@ -18,9 +18,10 @@ Install a DCM Installation instructions can be found at https://github.com/sbgrid/data-capture-module . Note that a shared filesystem (posix or AWS S3) between Dataverse and your DCM is required. You cannot use a DCM with Swift at this point in time. -Please note that S3 support for DCM is highly experimental. Files can be uploaded to S3 but they cannot be downloaded until https://github.com/IQSS/dataverse/issues/4949 is worked on. If you want to play around with S3 support for DCM, you must configure a JVM option called ``dataverse.files.dcm-s3-bucket-name`` which is a holding area for uploaded files that have not yet passed checksum validation. Search for that JVM option at https://github.com/IQSS/dataverse/issues/4703 for commands on setting that JVM option and related setup. Note that because that GitHub issue has so many comments you will need to click "Load more" where it says "hidden items". FIXME: Document all of this properly. +Please note that S3 support for DCM is highly experimental. Files can be uploaded to S3 but they cannot be downloaded until https://github.com/IQSS/dataverse/issues/4949 is worked on. If you want to play around with S3 support for DCM, you must configure a JVM option called ``dataverse.files.dcm-s3-bucket-name`` which is a holding area for uploaded files that have not yet passed checksum validation. Search for that JVM option at https://github.com/IQSS/dataverse/issues/4703 for commands on setting that JVM option and related setup. Note that because that GitHub issue has so many comments you will need to click "Load more" where it says "hidden items". -. FIXME: Explain what ``dataverse.files.dcm-s3-bucket-name`` is for and what it has to do with ``dataverse.files.s3-bucket-name``. +.. FIXME: Document all of this properly. +.. FIXME: Explain what ``dataverse.files.dcm-s3-bucket-name`` is for and what it has to do with ``dataverse.files.s3-bucket-name``. Once you have installed a DCM, you will need to configure two database settings on the Dataverse side. These settings are documented in the :doc:`/installation/config` section of the Installation Guide: @@ -61,7 +62,6 @@ Steps to set up a DCM mock for Development Install Flask. - Download and run the mock. You will be cloning the https://github.com/sbgrid/data-capture-module repo. - ``git clone git://github.com/sbgrid/data-capture-module.git`` @@ -101,6 +101,13 @@ At this point you should be able to download a placeholder rsync script. Dataver Now the files are in place and you need to send JSON to Dataverse with a success or failure message as described above. Make a copy of ``doc/sphinx-guides/source/_static/installation/files/root/big-data-support/checksumValidationSuccess.json`` and put the identifier in place such as "X1METO" under "uploadFolder"). Then use curl as described above to send the JSON. +Troubleshooting +~~~~~~~~~~~~~~~ + +The following low level command should only be used when troubleshooting the "import" code a DCM uses but is documented here for completeness. + +``curl -H "X-Dataverse-key: $API_TOKEN" -X POST "$DV_BASE_URL/api/batch/jobs/import/datasets/files/$DATASET_DB_ID?uploadFolder=$UPLOAD_FOLDER&totalSize=$TOTAL_SIZE"`` + Steps to set up a DCM via Docker for Development ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ @@ -110,125 +117,122 @@ Docker Image Set-up ^^^^^^^^^^^^^^^^^^^ - Install docker if you do not have it -- Here are the steps copied out of docker-aio/readme.md & docker-dcm/readme.txt . If the info on this page is consolidated to a doc, it may be good to look back at the guides themselves : - - - `cd conf/docker-aio` and run `./0prep_deps.sh` to create Glassfish and Solr tarballs in conf/docker-aio/dv/deps. - - run `./1prep.sh` - - build the docker image: `docker build -t dv0 -f c7.dockerfile .` - - `cd ../docker-dcm` and run `./0prep.sh` - - build dcm/dv0dcm images with docker-compose: `docker-compose -f docker-compose.yml build` - - start containers: `docker-compose -f docker-compose.yml up -d` - - wait for container to show "healthy" (aka - `docker ps`), then wait another 4-5 minutes (even though it shows healthy, glassfish is still standing itself up), then run dataverse app installation: `docker exec -it dvsrv /opt/dv/install.bash` - - configure dataverse application to use DCM (run from outside the container): `docker exec -it dvsrv /opt/dv/configure_dcm.sh` - - The dataverse installation is accessible at `http://localhost:8084`. +- Follow these steps (extracted from ``docker-aio/readme.md`` & ``docker-dcm/readme.txt``) : + + - ``cd conf/docker-aio`` and run ``./0prep_deps.sh`` to create Glassfish and Solr tarballs in conf/docker-aio/dv/deps. + - Run ``./1prep.sh`` + - Build the docker image: ``docker build -t dv0 -f c7.dockerfile .`` + - ``cd ../docker-dcm`` and run ``./0prep.sh`` + - Build dcm/dv0dcm images with docker-compose: ``docker-compose -f docker-compose.yml build`` + - Start containers: ``docker-compose -f docker-compose.yml up -d`` + - Wait for container to show "healthy" (aka - ``docker ps``), then wait another 5 minutes (even though it shows healthy, glassfish is still standing itself up). Then run Dataverse app installation: ``docker exec -it dvsrv /opt/dv/install.bash`` + - Configure Dataverse application to use DCM (run from outside the container): ``docker exec -it dvsrv /opt/dv/configure_dcm.sh`` + - The Dataverse installation is accessible at ``http://localhost:8084``. - You may need to change the DoiProvider inside dvsrv (ezid does not work): - - `curl -X DELETE -d EZID "localhost:8080/api/admin/settings/:DoiProvider"` - - `curl -X PUT -d DataCite "localhost:8080/api/admin/settings/:DoiProvider"` - - also change the doi.baseUrlString, doi.username, doi.password + - ``curl -X DELETE -d EZID "localhost:8080/api/admin/settings/:DoiProvider"`` + - ``curl -X PUT -d DataCite "localhost:8080/api/admin/settings/:DoiProvider"`` + - Also change the doi.baseUrlString, doi.username, doi.password -Setting up the S3 DCM Variant -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - -- Before: Make sure to do the "To run rsync posix in docker" steps first. -- Before: the default bucket for dcm to hold files in s3 is named test-dcm. It is coded into post_upload.bash (line 30). - - - This bucket should exist on our testing environment, but if it is not there go and create it through the browser ui - -- `docker cp data-capture-module/scn/post_upload_s3.bash dcmsrv:/opt/dcm/scn/post_upload.bash` -- Install aws on dcmsrv and symlink it +Optional steps for setting up the S3 Docker DCM Variant +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - - `pip install awscli --upgrade --user` - - `export PATH=~/.local/bin:$PATH` +- Before: the default bucket for DCM to hold files in S3 is named test-dcm. It is coded into `post_upload.bash` (line 30). Change to a different bucket if needed. +- Move the S3 variant of the rsync processing script into docker: ``docker cp data-capture-module/scn/post_upload_s3.bash dcmsrv:/opt/dcm/scn/post_upload.bash`` +- Install AWS on dcmsrv and symlink it +.. Delete these after testing +.. - ``pip install awscli --upgrade --user`` +.. - ``export PATH=~/.local/bin:$PATH`` +- Add AWS bucket info to dcmsrv -- Configure aws bucket on dcmsrv + - You need a credentials files in ~/.aws - - you need a credentials files in ~/.aws - - - `mkdir ~/.aws` - - `yum install nano` (or use a different editor below) - - `nano ~/.aws/credentials` + - ``mkdir ~/.aws`` + - ``yum install nano`` (or use a different editor below) + - ``nano ~/.aws/credentials`` - Structure like: - - `[default]` - - `aws_access_key_id = ` - - `aws_secret_access_key =` + - ``[default]`` + - ``aws_access_key_id =`` + - ``aws_secret_access_key =`` - Dataverse configuration (on dvsrv): - - Set s3 as the storage driver + - Set S3 as the storage driver - - `cd /opt/glassfish4/bin/` - - `./asadmin delete-jvm-options "\-Ddataverse.files.storage-driver-id=file"` - - `./asadmin create-jvm-options "\-Ddataverse.files.storage-driver-id=s3"` + - ``cd /opt/glassfish4/bin/`` + - ``./asadmin delete-jvm-options "\-Ddataverse.files.storage-driver-id=file"`` + - ``./asadmin create-jvm-options "\-Ddataverse.files.storage-driver-id=s3"`` - - Set up aws configs for dataverse + - Add AWS bucket info to Dataverse - - you may need this: `yum install awscli` - - `mkdir ~/.aws` - - `yum install nano` (or use a different editor below) - - `nano ~/.aws/credentials` + - You may need this: ``yum install awscli`` + - ``mkdir ~/.aws`` + - ``yum install nano`` (or use a different editor below) + - ``nano ~/.aws/credentials`` - Structure like: - - `[default]` - - `aws_access_key_id = ` - - `aws_secret_access_key =` + - ``[default]`` + - ``aws_access_key_id =`` + - ``aws_secret_access_key =`` - ALSO: create a region file - - `nano ~/.aws/config` - - contents: + - ``nano ~/.aws/config`` + - Contents: - - `[default]` - - `region = us-east-1` + - ``[default]`` + - ``region = us-east-1`` - - Add the bucket names to dataverse + - Add the S3 bucket names to Dataverse - - s3 bucket for dataverse + - S3 bucket for Dataverse - - `/usr/local/glassfish4/glassfish/bin/asadmin create-jvm-options "-Ddataverse.files.s3-bucket-name=iqsstestdcmbucket"` + - ``/usr/local/glassfish4/glassfish/bin/asadmin create-jvm-options "-Ddataverse.files.s3-bucket-name=iqsstestdcmbucket"`` - - s3 bucket for dcm (as dataverse needs to do the copy over) + - S3 bucket for DCM (as Dataverse needs to do the copy over) - - `/usr/local/glassfish4/glassfish/bin/asadmin create-jvm-options "-Ddataverse.files.dcm-s3-bucket-name=test-dcm"` + - ``/usr/local/glassfish4/glassfish/bin/asadmin create-jvm-options "-Ddataverse.files.dcm-s3-bucket-name=test-dcm"`` - - [set download method to be http] + - Set download method to be HTTP, as DCM downloads through S3 are over this protocol ``curl -X PUT "http://localhost:8080/api/admin/settings/:DownloadMethods" -d "native/http"`` Using the DCM Docker Containers ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ -- Create dataset and download rsync upload script +For using these commands, you will need to connect to the shell prompt inside various containers (e.g. ``docker exec -it dvsrv /bin/bash``) + +- Create a dataset and download rsync upload script - Upload script to dcm_client (if needed, you can probably do all the actions for create/download inside dcm_client) - - `docker cp ~/Downloads/upload-FK2_NN49YM.bash dcm_client:/tmp` + - ``docker cp ~/Downloads/upload-FK2_NN49YM.bash dcm_client:/tmp`` -- Create a folder of files to upload (you just need something...) +- Create a folder of files to upload (files can be empty) - Run script - - `bash ./upload-FK2_NN49YM.bash` + - e.g. ``bash ./upload-FK2_NN49YM.bash`` -- manually run post_upload.bash on dcmsrv +- Manually run post_upload.bash on dcmsrv - - `bash ./opt/dcm/scn/post_upload.bash` + - ``bash ./opt/dcm/scn/post_upload.bash`` Additional DCM docker development tips ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ -- You can completely blow away all the docker images with these commands (including non dcm ones!) +- You can completely blow away all the docker images with these commands (including non DCM ones!) - - `docker stop dvsrv` - - `docker stop dcm_client` - - `docker stop dcmsrv` - - `docker rm $(docker ps -a -q)` - - `docker rmi $(docker images -q)` + - ``docker stop dvsrv`` + - ``docker stop dcm_client`` + - ``docker stop dcmsrv`` + - ``docker rm $(docker ps -a -q)`` + - ``docker rmi $(docker images -q)`` - There are a few logs to tail - - dvsrv : `tail -n 2000 -f /opt/glassfish4/glassfish/domains/domain1/logs/server.log` - - dcmsrv : `tail -n 2000 -f /var/log/lighttpd/breakage.log` - - dcmsrv : `tail -n 2000 -f /var/log/lighttpd/access.log` + - dvsrv : ``tail -n 2000 -f /opt/glassfish4/glassfish/domains/domain1/logs/server.log`` + - dcmsrv : ``tail -n 2000 -f /var/log/lighttpd/breakage.log`` + - dcmsrv : ``tail -n 2000 -f /var/log/lighttpd/access.log`` - I have attached a script for redeploying dataverse into your docker image if you just want to update the code. You'll probably need to tweak the paths [deploy_dataverse_into_docker.sh.zip](https://github.com/IQSS/dataverse/files/2237096/deploy_dataverse_into_docker.sh.zip) @@ -237,13 +241,6 @@ Additional DCM docker development tips - Note that by default the docker container will stop running if the process it is following is turned off. For example flask with dcmsrv. You can get around this by having the script being followed never close (e.g. sleep infinity) https://stackoverflow.com/questions/31870222/how-can-i-keep-container-running-on-kubernetes - You may have to restart the glassfish domain occasionally to deal with memory filling up. If deployment is getting reallllllly slow, its a good time. -Troubleshooting -~~~~~~~~~~~~~~~ - -The following low level command should only be used when troubleshooting the "import" code a DCM uses but is documented here for completeness. - -``curl -H "X-Dataverse-key: $API_TOKEN" -X POST "$DV_BASE_URL/api/batch/jobs/import/datasets/files/$DATASET_DB_ID?uploadFolder=$UPLOAD_FOLDER&totalSize=$TOTAL_SIZE"`` - Repository Storage Abstraction Layer (RSAL) ------------------------------------------- From 9a47958dad43c52dd743c5b1fee342067c9dc9d0 Mon Sep 17 00:00:00 2001 From: matthew-a-dunlap Date: Wed, 5 Dec 2018 14:22:10 -0500 Subject: [PATCH 27/32] Bump DCM version #4949 --- conf/docker-dcm/0prep.sh | 3 +-- conf/docker-dcm/dcmsrv.dockerfile | 3 ++- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/conf/docker-dcm/0prep.sh b/conf/docker-dcm/0prep.sh index a77f8775495..8f87cd141ec 100755 --- a/conf/docker-dcm/0prep.sh +++ b/conf/docker-dcm/0prep.sh @@ -1,4 +1,3 @@ #!/bin/sh - -wget https://github.com/sbgrid/data-capture-module/releases/download/0.3/dcm-0.3-0.noarch.rpm +wget https://github.com/sbgrid/data-capture-module/releases/download/0.3/dcm-0.5-0.noarch.rpm diff --git a/conf/docker-dcm/dcmsrv.dockerfile b/conf/docker-dcm/dcmsrv.dockerfile index 02d88b6c3a2..4ec6fb86c06 100644 --- a/conf/docker-dcm/dcmsrv.dockerfile +++ b/conf/docker-dcm/dcmsrv.dockerfile @@ -1,13 +1,14 @@ # build from repo root FROM centos:6 RUN yum install -y epel-release -ARG RPMFILE=dcm-0.3-0.noarch.rpm +ARG RPMFILE=dcm-0.5-0.noarch.rpm COPY ${RPMFILE} /tmp/ COPY bashrc /root/.bashrc COPY test_install.sh /root/ RUN yum localinstall -y /tmp/${RPMFILE} RUN pip install -r /opt/dcm/requirements.txt RUN pip install awscli==1.15.75 +run export PATH=~/.local/bin:$PATH RUN /root/test_install.sh COPY rq-init-d /etc/init.d/rq RUN useradd glassfish From a170d7815014b876c9cfa2182a0f638f53d3e2f2 Mon Sep 17 00:00:00 2001 From: matthew-a-dunlap Date: Wed, 5 Dec 2018 14:25:03 -0500 Subject: [PATCH 28/32] Minor doc fix #4949 --- doc/sphinx-guides/source/developers/big-data-support.rst | 3 --- 1 file changed, 3 deletions(-) diff --git a/doc/sphinx-guides/source/developers/big-data-support.rst b/doc/sphinx-guides/source/developers/big-data-support.rst index 32b34832968..3a234687a4a 100644 --- a/doc/sphinx-guides/source/developers/big-data-support.rst +++ b/doc/sphinx-guides/source/developers/big-data-support.rst @@ -140,9 +140,6 @@ Optional steps for setting up the S3 Docker DCM Variant - Before: the default bucket for DCM to hold files in S3 is named test-dcm. It is coded into `post_upload.bash` (line 30). Change to a different bucket if needed. - Move the S3 variant of the rsync processing script into docker: ``docker cp data-capture-module/scn/post_upload_s3.bash dcmsrv:/opt/dcm/scn/post_upload.bash`` - Install AWS on dcmsrv and symlink it -.. Delete these after testing -.. - ``pip install awscli --upgrade --user`` -.. - ``export PATH=~/.local/bin:$PATH`` - Add AWS bucket info to dcmsrv - You need a credentials files in ~/.aws From c891407d388957a8decbaa57783e41432668d437 Mon Sep 17 00:00:00 2001 From: matthew-a-dunlap Date: Wed, 5 Dec 2018 14:33:16 -0500 Subject: [PATCH 29/32] Another minor doc change #4949 --- doc/sphinx-guides/source/developers/big-data-support.rst | 4 ---- 1 file changed, 4 deletions(-) diff --git a/doc/sphinx-guides/source/developers/big-data-support.rst b/doc/sphinx-guides/source/developers/big-data-support.rst index 3a234687a4a..536ddf6f982 100644 --- a/doc/sphinx-guides/source/developers/big-data-support.rst +++ b/doc/sphinx-guides/source/developers/big-data-support.rst @@ -231,10 +231,6 @@ Additional DCM docker development tips - dcmsrv : ``tail -n 2000 -f /var/log/lighttpd/breakage.log`` - dcmsrv : ``tail -n 2000 -f /var/log/lighttpd/access.log`` -- I have attached a script for redeploying dataverse into your docker image if you just want to update the code. You'll probably need to tweak the paths [deploy_dataverse_into_docker.sh.zip](https://github.com/IQSS/dataverse/files/2237096/deploy_dataverse_into_docker.sh.zip) - - - This script is also currently broken by the installer bug - - Note that by default the docker container will stop running if the process it is following is turned off. For example flask with dcmsrv. You can get around this by having the script being followed never close (e.g. sleep infinity) https://stackoverflow.com/questions/31870222/how-can-i-keep-container-running-on-kubernetes - You may have to restart the glassfish domain occasionally to deal with memory filling up. If deployment is getting reallllllly slow, its a good time. From 4432cae59c26993a3cdb8a8ef341a2f28ae6ae5c Mon Sep 17 00:00:00 2001 From: matthew-a-dunlap Date: Mon, 10 Dec 2018 14:38:32 -0500 Subject: [PATCH 30/32] Fix refactor double extension #4949 --- .../{file-info-fragment.xhtml.xhtml => file-info-fragment.xhtml} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename src/main/webapp/{file-info-fragment.xhtml.xhtml => file-info-fragment.xhtml} (100%) diff --git a/src/main/webapp/file-info-fragment.xhtml.xhtml b/src/main/webapp/file-info-fragment.xhtml similarity index 100% rename from src/main/webapp/file-info-fragment.xhtml.xhtml rename to src/main/webapp/file-info-fragment.xhtml From 55371757ddbf28d11416c9da36344bab4bd28a0c Mon Sep 17 00:00:00 2001 From: matthew-a-dunlap Date: Thu, 13 Dec 2018 17:06:54 -0500 Subject: [PATCH 31/32] dcms3 code review docfixes #4949 includes the latex/pdf fix --- .../source/developers/big-data-support.rst | 48 +++++++------------ 1 file changed, 18 insertions(+), 30 deletions(-) diff --git a/doc/sphinx-guides/source/developers/big-data-support.rst b/doc/sphinx-guides/source/developers/big-data-support.rst index 536ddf6f982..b3241d513f8 100644 --- a/doc/sphinx-guides/source/developers/big-data-support.rst +++ b/doc/sphinx-guides/source/developers/big-data-support.rst @@ -18,9 +18,6 @@ Install a DCM Installation instructions can be found at https://github.com/sbgrid/data-capture-module . Note that a shared filesystem (posix or AWS S3) between Dataverse and your DCM is required. You cannot use a DCM with Swift at this point in time. -Please note that S3 support for DCM is highly experimental. Files can be uploaded to S3 but they cannot be downloaded until https://github.com/IQSS/dataverse/issues/4949 is worked on. If you want to play around with S3 support for DCM, you must configure a JVM option called ``dataverse.files.dcm-s3-bucket-name`` which is a holding area for uploaded files that have not yet passed checksum validation. Search for that JVM option at https://github.com/IQSS/dataverse/issues/4703 for commands on setting that JVM option and related setup. Note that because that GitHub issue has so many comments you will need to click "Load more" where it says "hidden items". - -.. FIXME: Document all of this properly. .. FIXME: Explain what ``dataverse.files.dcm-s3-bucket-name`` is for and what it has to do with ``dataverse.files.s3-bucket-name``. Once you have installed a DCM, you will need to configure two database settings on the Dataverse side. These settings are documented in the :doc:`/installation/config` section of the Installation Guide: @@ -137,22 +134,18 @@ Docker Image Set-up Optional steps for setting up the S3 Docker DCM Variant ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ -- Before: the default bucket for DCM to hold files in S3 is named test-dcm. It is coded into `post_upload.bash` (line 30). Change to a different bucket if needed. -- Move the S3 variant of the rsync processing script into docker: ``docker cp data-capture-module/scn/post_upload_s3.bash dcmsrv:/opt/dcm/scn/post_upload.bash`` -- Install AWS on dcmsrv and symlink it +- Before: the default bucket for DCM to hold files in S3 is named test-dcm. It is coded into `post_upload_s3.bash` (line 30). Change to a different bucket if needed. - Add AWS bucket info to dcmsrv - You need a credentials files in ~/.aws - ``mkdir ~/.aws`` - ``yum install nano`` (or use a different editor below) - - ``nano ~/.aws/credentials`` - - - Structure like: + - ``nano ~/.aws/credentials`` and add these contents with your keys: - - ``[default]`` - - ``aws_access_key_id =`` - - ``aws_secret_access_key =`` + - ``[default]`` + - ``aws_access_key_id =`` + - ``aws_secret_access_key =`` - Dataverse configuration (on dvsrv): @@ -164,24 +157,18 @@ Optional steps for setting up the S3 Docker DCM Variant - Add AWS bucket info to Dataverse - - You may need this: ``yum install awscli`` - ``mkdir ~/.aws`` - ``yum install nano`` (or use a different editor below) - - ``nano ~/.aws/credentials`` - - - Structure like: - - - ``[default]`` - - ``aws_access_key_id =`` - - ``aws_secret_access_key =`` + - ``nano ~/.aws/credentials`` and add these contents with your keys: - - ALSO: create a region file + - ``[default]`` + - ``aws_access_key_id =`` + - ``aws_secret_access_key =`` - - ``nano ~/.aws/config`` - - Contents: + - ALSO: ``nano ~/.aws/config`` to create a region file. Add these contents: - - ``[default]`` - - ``region = us-east-1`` + - ``[default]`` + - ``region = us-east-1`` - Add the S3 bucket names to Dataverse @@ -210,9 +197,10 @@ For using these commands, you will need to connect to the shell prompt inside va - e.g. ``bash ./upload-FK2_NN49YM.bash`` -- Manually run post_upload.bash on dcmsrv +- Manually run post upload script on dcmsrv - - ``bash ./opt/dcm/scn/post_upload.bash`` + - for posix implementation: ``bash ./opt/dcm/scn/post_upload.bash`` + - for S3 implementation: ``bash ./opt/dcm/scn/post_upload_s3.bash`` Additional DCM docker development tips ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ @@ -347,7 +335,7 @@ Available Steps Dataverse has an internal step provider, whose id is ``:internal``. It offers the following steps: log -+++ +^^^ A step that writes data about the current workflow invocation to the instance log. It also writes the messages in its ``parameters`` map. @@ -364,7 +352,7 @@ A step that writes data about the current workflow invocation to the instance lo pause -+++++ +^^^^^ A step that pauses the workflow. The workflow is paused until a POST request is sent to ``/api/workflows/{invocation-id}``. @@ -377,7 +365,7 @@ A step that pauses the workflow. The workflow is paused until a POST request is http/sr -+++++++ +^^^^^^^ A step that sends a HTTP request to an external system, and then waits for a response. The response has to match a regular expression specified in the step parameters. The url, content type, and message body can use data from the workflow context, using a simple markup language. This step has specific parameters for rollback. From bf31f9e9c433d7caeaaf491af471f99ffe1c20ef Mon Sep 17 00:00:00 2001 From: matthew-a-dunlap Date: Thu, 13 Dec 2018 18:18:07 -0500 Subject: [PATCH 32/32] dcms3 install fixes and enable restrict #4949 --- conf/docker-aio/c7.dockerfile | 2 +- conf/docker-dcm/0prep.sh | 2 +- doc/sphinx-guides/source/developers/big-data-support.rst | 2 +- src/main/webapp/filesFragment.xhtml | 5 +++-- 4 files changed, 6 insertions(+), 5 deletions(-) diff --git a/conf/docker-aio/c7.dockerfile b/conf/docker-aio/c7.dockerfile index 649fe74f673..ed73ab02d2a 100644 --- a/conf/docker-aio/c7.dockerfile +++ b/conf/docker-aio/c7.dockerfile @@ -3,7 +3,7 @@ FROM centos:7 RUN yum install -y https://download.postgresql.org/pub/repos/yum/9.6/redhat/rhel-7-x86_64/pgdg-centos96-9.6-3.noarch.rpm #RUN yum install -y java-1.8.0-openjdk-headless postgresql-server sudo epel-release unzip perl curl httpd RUN yum install -y java-1.8.0-openjdk-devel postgresql96-server sudo epel-release unzip perl curl httpd -RUN yum install -y jq lsof +RUN yum install -y jq lsof awscli # copy and unpack dependencies (solr, glassfish) COPY dv /tmp/dv diff --git a/conf/docker-dcm/0prep.sh b/conf/docker-dcm/0prep.sh index 8f87cd141ec..98cff3c805a 100755 --- a/conf/docker-dcm/0prep.sh +++ b/conf/docker-dcm/0prep.sh @@ -1,3 +1,3 @@ #!/bin/sh -wget https://github.com/sbgrid/data-capture-module/releases/download/0.3/dcm-0.5-0.noarch.rpm +wget https://github.com/sbgrid/data-capture-module/releases/download/0.5/dcm-0.5-0.noarch.rpm diff --git a/doc/sphinx-guides/source/developers/big-data-support.rst b/doc/sphinx-guides/source/developers/big-data-support.rst index b3241d513f8..e0d0b4ffd25 100644 --- a/doc/sphinx-guides/source/developers/big-data-support.rst +++ b/doc/sphinx-guides/source/developers/big-data-support.rst @@ -165,7 +165,7 @@ Optional steps for setting up the S3 Docker DCM Variant - ``aws_access_key_id =`` - ``aws_secret_access_key =`` - - ALSO: ``nano ~/.aws/config`` to create a region file. Add these contents: + - Also: ``nano ~/.aws/config`` to create a region file. Add these contents: - ``[default]`` - ``region = us-east-1`` diff --git a/src/main/webapp/filesFragment.xhtml b/src/main/webapp/filesFragment.xhtml index c9e575c9bbd..0b5b11cfe4f 100644 --- a/src/main/webapp/filesFragment.xhtml +++ b/src/main/webapp/filesFragment.xhtml @@ -258,8 +258,9 @@ - -
  • + + +