-
All Files datasets added file downloads
+
+
+
+
+
+
Datasets
+
All Activity
+
Past 30 Days
+
+
+
+
-
-
Past 30 Days datasets added file downloads
+
+
+
+
+
+
Files
+
All Activity
+
Past 30 Days
+
+
+
+
-
+
@@ -154,7 +253,8 @@
//switch baseUrl to point to a different server than the local box
var baseUrl = "";
- var thumbBaseURL = baseUrl + "/api/datasets/:persistentId/thumbnail" + "?persistentId=";
+ // NOTE: REMOVED THUMBNAILS
+ // var thumbBaseURL = baseUrl + "/api/datasets/:persistentId/thumbnail" + "?persistentId=";
var metricBaseUrl = baseUrl + "/api/info/metrics/";
var simpleCatSearch = baseUrl + "/api/search?q=*&type=dataset&sort=dateSort&order=desc&fq=categoryOfDataverse:";
@@ -162,22 +262,33 @@
$.get(simpleCatSearch + fqString + "&per_page=" + resultCount, function(jData) {
var resultHtml = "";
jData.data.items.forEach(function(item){
- var date = new Date(item.published_at);
- resultHtml += "
";
- resultHtml += "
";
- resultHtml += "
";
- resultHtml += "
";
+ var options = { year: 'numeric', month: 'short', day: 'numeric' };
+ var date = new Date(item.published_at).toLocaleString('en-US', options);
+ resultHtml += "
";
+ // NOTE: REMOVED THUMBNAILS
+ // resultHtml += "
";
+ // NOTE: REMOVED THUMBNAILS... in next line, change grid layout class to `col-xs-9 col-md-10`
+ resultHtml += "
";
+ resultHtml += "
";
});
document.getElementById(elm).innerHTML = resultHtml;
});
}
//For metrics that return simple json with a count
+ //Can take a single element or an array of elements
function queryMetricSimple(metricRelPath, month, elm) {
$.get(metricBaseUrl + metricRelPath + month, function(jData) {
var resultCount = jData.data.count;
- document.getElementById(elm).innerHTML = resultCount;
+ if(Array.isArray(elm)) {
+ elm.forEach(function(e) {
+ document.getElementById(e).innerHTML = resultCount.toLocaleString('en');
+ });
+ } else {
+ document.getElementById(elm).innerHTML = resultCount.toLocaleString('en');
+ }
});
}
@@ -185,38 +296,62 @@
$.get(metricBaseUrl + metricRelPath + month, function(jData) {
var resultCount = jData.data.count;
var roundedDatasetCount = Math.floor(resultCount / 100) * 100;
- var dynamicSearchPlaceholder = "Search over " + roundedDatasetCount + " datasets...";
+ var dynamicSearchPlaceholder = "Search over " + roundedDatasetCount.toLocaleString('en') + " datasets...";
$("#inputDataverseSearch").attr({"placeholder" : dynamicSearchPlaceholder});
});
}
-
- function querySubject(elm) {
+
+ function querySubjectDataverseDataset(elm) {
+ var dvArray = [];
+ var fullArray = [];
$.get(metricBaseUrl + "dataverses/bySubject", function(jData) {
- var subArray = [];
- var resultHtml = "";
jData.data.forEach(function(item) {
- if(item.subject !== "N/A")
- subArray.push([item.subject, item.count]);
+ if(item.subject !== "N/A" && item.subject !== "Other") {
+ dvArray.push([item.subject, item.count]);
+ }
});
- subArray.sort();
- subArray.forEach(function(subject){
- resultHtml += "
" + subject[0] + " " + subject[1] + "
";
+ $.get(metricBaseUrl + "datasets/bySubject?dataLocation=all", function(jData) {
+ var resultHtml = "";
+ jData.data.forEach(function(item) {
+ if(item.subject !== "N/A" && item.subject !== "Other") {
+ var dvCount = 0;
+ for (var dvi = 0; dvi < dvArray.length; dvi++) {
+ if(item.subject === dvArray[dvi][0]) {
+ dvCount = dvArray[dvi][1];
+ break;
+ }
+ }
+ fullArray.push([item.subject, (item.count + dvCount -1).toLocaleString('en')]); //subtract 1 to remove root dv from counts
+ }
+ });
+ fullArray.sort();
+ fullArray.forEach(function(subject){
+ // NOTE: The alias of the root dataverse will need to be configured in this URL
+ resultHtml += "
" + subject[0] + " " + subject[1] + "
";
+ });
+ document.getElementById(elm).innerHTML = resultHtml;
});
- document.getElementById(elm).innerHTML = resultHtml;
});
}
+ queryMetricSimple("datasets", "?dataLocation=all", ["headAllTimeAllDatasetsValue", "activityAllTimeAllDatasetsValue"]);
+ queryMetricSimple("downloads", "", ["headAllTimeAllDownloadsValue", "activityAllTimeAllDownloadsValue"]);
+ queryMetricSimple("dataverses", "", "headAllTimeAllDataversesValue");
+
writeRecentDatasetsInDataverses("(Journal)" , 3, "journals");
- writeRecentDatasetsInDataverses("(\"Research+Project\"%20OR%20Researcher%20OR%20\"Research+Group\")" , 3, "researchers");
-
- queryMetricSimple("datasets", "", "activityAllTimeDatasetsValue");
- queryMetricSimple("downloads", "", "activityAllTimeFilesValue");
+ writeRecentDatasetsInDataverses("(\"Research+Project\"%20OR%20Researcher%20OR%20\"Research+Group\")" , 6, "researchers");
- queryMetricSimple("datasets/pastDays", "/30", "activity30DaysDatasetsValue");
- queryMetricSimple("downloads/pastDays", "/30", "activity30DaysFilesValue");
-
- querySubject("dataversesBySubject");
- queryMetricSearch("datasets", "");
+ queryMetricSimple("datasets/pastDays", "/30?dataLocation=all", "activity30DaysAllDatasetsValue");
+ queryMetricSimple("datasets", "?dataLocation=local", "activityAllTimeDepositedDatasetsValue");
+ queryMetricSimple("datasets/pastDays", "/30?dataLocation=local", "activity30DaysDepositedDatasetsValue");
+ queryMetricSimple("datasets", "?dataLocation=remote", "activityAllTimeHarvestedDatasetsValue");
+ queryMetricSimple("datasets/pastDays", "/30?dataLocation=remote", "activity30DaysHarvestedDatasetsValue");
+
+ queryMetricSimple("downloads/pastDays", "/30", "activity30DaysAllDownloadsValue");
+ queryMetricSimple("files", "", "activityAllTimeDepositedFilesValue");
+ queryMetricSimple("files/pastDays", "/30", "activity30DaysDepositedFilesValue");
+ querySubjectDataverseDataset("dataversesBySubject");
+ queryMetricSearch("datasets", "?dataLocation=all");
//]]>
diff --git a/doc/sphinx-guides/source/admin/integrations.rst b/doc/sphinx-guides/source/admin/integrations.rst
index de4e15efd25..c7924cd7c20 100644
--- a/doc/sphinx-guides/source/admin/integrations.rst
+++ b/doc/sphinx-guides/source/admin/integrations.rst
@@ -70,6 +70,13 @@ Compute Button
The "Compute" button is still highly experimental and has special requirements such as use of a Swift object store, but it is documented under "Setting up Compute" in the :doc:`/installation/config` section of the Installation Guide.
+Whole Tale
+++++++++++
+
+`Whole Tale
`_ enables researchers to analyze data using popular tools including Jupyter and RStudio with the ultimate goal of supporting publishing of reproducible research packages. Users can
+`import data from Dataverse
+`_ via identifier (e.g., DOI, URI, etc) or through the External Tools integration. For installation instructions, see the :doc:`/installation/external-tools` section of this Installation Guide or the `Integration `_ section of the Whole Tale User Guide.
+
Discoverability
---------------
@@ -85,6 +92,23 @@ SHARE
`SHARE `_ is building a free, open, data set about research and scholarly activities across their life cycle. It's possible to add and installation of Dataverse as one of the `sources `_ they include if you contact the SHARE team.
+Research Data Preservation
+--------------------------
+
+Archivematica
++++++++++++++
+
+`Archivematica `_ is an integrated suite of open-source tools for processing digital objects for long-term preservation, developed and maintained by Artefactual Systems Inc. Its configurable workflow is designed to produce system-independent, standards-based Archival Information Packages (AIPs) suitable for long-term storage and management.
+
+Sponsored by the `Ontario Council of University Libraries (OCUL) `_, this technical integration enables users of Archivematica to select datasets from connected Dataverse instances and process them for long-term access and digital preservation. For more information and list of known issues, please refer to Artefactual's `release notes `_, `integration documentation `_, and the `project wiki `_.
+
+DuraCloud/Chronopolis
++++++++++++++++++++++
+
+Dataverse can be configured to submit a copy of published Datasets, packaged as `Research Data Alliance conformant `_ zipped `BagIt `_ bags to the `Chronopolis `_ via `DuraCloud `_
+
+For details on how to configure this integration, look for "DuraCloud/Chronopolis" in the :doc:`/installation/config` section of the Installation Guide.
+
Future Integrations
-------------------
diff --git a/doc/sphinx-guides/source/admin/metadatacustomization.rst b/doc/sphinx-guides/source/admin/metadatacustomization.rst
index 66b9b02fc87..8c13233d5c7 100644
--- a/doc/sphinx-guides/source/admin/metadatacustomization.rst
+++ b/doc/sphinx-guides/source/admin/metadatacustomization.rst
@@ -599,7 +599,7 @@ Once you have enabled a new metadata block you should be able to see the new fie
See the :doc:`/installation/prerequisites/` section of the Installation Guide for a suggested location on disk for the Solr schema file.
-Please note that if you are going to make a pull request updating ``conf/solr/7.3.0/schema.xml`` with fields you have added, you should first load all the custom metadata blocks in ``scripts/api/data/metadatablocks`` (including ones you don't care about) to create a complete list of fields.
+Please note that if you are going to make a pull request updating ``conf/solr/7.3.1/schema.xml`` with fields you have added, you should first load all the custom metadata blocks in ``scripts/api/data/metadatablocks`` (including ones you don't care about) to create a complete list of fields.
Reloading a Metadata Block
--------------------------
diff --git a/doc/sphinx-guides/source/admin/monitoring.rst b/doc/sphinx-guides/source/admin/monitoring.rst
index aa5131d1e8a..8ce5f65c5ca 100644
--- a/doc/sphinx-guides/source/admin/monitoring.rst
+++ b/doc/sphinx-guides/source/admin/monitoring.rst
@@ -58,7 +58,11 @@ HTTP traffic can be monitored from the client side, the server side, or both.
Monitoring HTTP Traffic from the Client Side
++++++++++++++++++++++++++++++++++++++++++++
-HTTP traffic for web clients that have cookies enabled (most browsers) can be tracked by Google Analytics and Piwik (renamed to "Matomo") as explained in the :doc:`/installation/config` section of the Installation Guide under ``:GoogleAnalyticsCode`` and ``:PiwikAnalyticsId``, respectively. You could also embed additional client side monitoring solutions by using a custom footer (``:FooterCustomizationFile``), which is described on the same page.
+HTTP traffic for web clients that have cookies enabled (most browsers) can be tracked by Google Analytics (https://www.google.com/analytics/) and Matomo (formerly "Piwik"; https://matomo.org/) as explained in the :ref:`Web-Analytics-Code` section of the Installation Guide.
+
+To track analytics beyond pageviews, style classes have been added for end user action buttons, which include:
+
+``btn-compute``, ``btn-contact``, ``btn-download``, ``btn-explore``, ``btn-export``, ``btn-preview``, ``btn-request``, ``btn-share``
Monitoring HTTP Traffic from the Server Side
+++++++++++++++++++++++++++++++++++++++++++++
@@ -99,6 +103,11 @@ actionlogrecord
There is a database table called ``actionlogrecord`` that captures events that may be of interest. See https://github.com/IQSS/dataverse/issues/2729 for more discussion around this table.
+Edit Draft Versions Logging
+---------------------------
+
+Changes made to draft versions of datasets are logged in a folder called logs/edit-drafts. See https://github.com/IQSS/dataverse/issues/5145 for more information on this logging.
+
EJB Timers
----------
diff --git a/doc/sphinx-guides/source/api/dataaccess.rst b/doc/sphinx-guides/source/api/dataaccess.rst
index 3e3ad23b2c8..3a91dca5024 100755
--- a/doc/sphinx-guides/source/api/dataaccess.rst
+++ b/doc/sphinx-guides/source/api/dataaccess.rst
@@ -69,7 +69,9 @@ Multiple File ("bundle") download
Returns the files listed, zipped.
-.. note:: If files requested cannot be provided, a 207 status code will be returned indicating that the result was a mixed success.
+.. note:: If the request can only be completed partially - if only *some* of the requested files can be served (because of the permissions and/or size restrictions), the file MANIFEST.TXT included in the zipped bundle will have entries specifying the reasons the missing files could not be downloaded. IN THE FUTURE the API will return a 207 status code to indicate that the result was a partial success. (As of writing this - v.4.11 - this hasn't been implemented yet)
+
+.. note:: If any of the datafiles have the ``DirectoryLabel`` attributes in the corresponding ``FileMetadata`` entries, these will be added as folders to the Zip archive, and the files will be placed in them accordingly.
Parameters:
~~~~~~~~~~~
@@ -84,12 +86,12 @@ original "Saved Original", the proprietary (SPSS, Stata, R, etc.) file fr
============== ===========
-"All Formats" bundled access for Tabular Files.
------------------------------------------------
+"All Formats" bundled download for Tabular Files.
+-------------------------------------------------
``/api/access/datafile/bundle/$id``
-This is convenience packaging method is available for tabular data files.
+This is a convenience packaging method available for tabular data files.
It returns a zipped bundle that contains the data in the following formats:
* Tab-delimited;
diff --git a/doc/sphinx-guides/source/api/metrics.rst b/doc/sphinx-guides/source/api/metrics.rst
index 821b74b0a96..c86a9111e0b 100755
--- a/doc/sphinx-guides/source/api/metrics.rst
+++ b/doc/sphinx-guides/source/api/metrics.rst
@@ -24,7 +24,7 @@ Example: ``curl https://demo.dataverse.org/api/info/metrics/downloads``
To-Month
--------
-Returns a count of various objects in dataverse up to a specified month ``$YYYY-DD`` in YYYY-MM format (i.e. ``2018-01``)::
+Returns a count of various objects in dataverse up to a specified month ``$YYYY-DD`` in YYYY-MM format (e.g. ``2018-01``)::
GET https://$SERVER/api/info/metrics/$type/toMonth/$YYYY-DD
@@ -36,7 +36,7 @@ Example: ``curl https://demo.dataverse.org/api/info/metrics/dataverses/toMonth/2
Past Days
---------
-Returns a count of various objects in dataverse for the past ``$days`` (i.e. ``30``)::
+Returns a count of various objects in dataverse for the past ``$days`` (e.g. ``30``)::
GET https://$SERVER/api/info/metrics/$type/pastDays/$days
@@ -45,8 +45,8 @@ Returns a count of various objects in dataverse for the past ``$days`` (i.e. ``3
Example: ``curl https://demo.dataverse.org/api/info/metrics/datasets/pastDays/30``
-Dataverse Specific Commands
----------------------------
+Dataverse Specific Metrics
+--------------------------
By Subject
~~~~~~~~~~~~~~~
@@ -64,18 +64,41 @@ Returns the number of dataverses by each category::
GET https://$SERVER/api/info/metrics/dataverses/byCategory
-Dataset Specific Commands
--------------------------
+Dataset Specific Metrics
+------------------------
By Subject
-~~~~~~~~~~~~~~~
+~~~~~~~~~~
Returns the number of datasets by each subject::
GET https://$SERVER/api/info/metrics/datasets/bySubject
+
+By Subject, and to Month
+~~~~~~~~~~~~~~~~~~~~~~~~
+
+Returns the number of datasets by each subject, and up to a specified month ``$YYYY-DD`` in YYYY-MM format (e.g. ``2018-01``)::
+
+ GET https://$SERVER/api/info/metrics/datasets/bySubject/toMonth/$YYYY-DD
+
+Example: ``curl https://demo.dataverse.org/api/info/metrics/datasets/bySubject/toMonth/2018-01``
+
.. |CORS| raw:: html
CORS
-
\ No newline at end of file
+
+
+
+Metric Query Parameters
+-----------------------
+
+To further tailor your metric, query parameters can be provided.
+
+dataLocation
+~~~~~~~~~~~~
+
+Specifies whether the metric should query ``local`` data, ``remote`` data (e.g. harvested), or ``all`` data when getting results. Only works for dataset metrics.
+
+Example: ``curl https://demo.dataverse.org/api/info/metrics/datasets/?dataLocation=remote``
diff --git a/doc/sphinx-guides/source/api/search.rst b/doc/sphinx-guides/source/api/search.rst
index b2b37e71425..8ea51b24ce4 100755
--- a/doc/sphinx-guides/source/api/search.rst
+++ b/doc/sphinx-guides/source/api/search.rst
@@ -85,7 +85,11 @@ https://demo.dataverse.org/api/search?q=trees
"file_content_type":"image/png",
"size_in_bytes":8361,
"md5":"0386269a5acb2c57b4eade587ff4db64",
- "dataset_citation":"Spruce, Sabrina, 2016, \"Spruce Goose\", http://dx.doi.org/10.5072/FK2/NFSEHG, Root Dataverse, V1"
+ "file_persistent_id": "doi:10.5072/FK2/XTT5BV/PCCHV7",
+ "dataset_name": "Dataset One",
+ "dataset_id": "32",
+ "dataset_persistent_id": "doi:10.5072/FK2/XTT5BV",
+ "dataset_citation":"Spruce, Sabrina, 2016, \"Spruce Goose\", http://dx.doi.org/10.5072/FK2/XTT5BV, Root Dataverse, V1"
},
{
"name":"Birds",
diff --git a/doc/sphinx-guides/source/conf.py b/doc/sphinx-guides/source/conf.py
index 42ef04f3c8d..f3e954ca1f6 100755
--- a/doc/sphinx-guides/source/conf.py
+++ b/doc/sphinx-guides/source/conf.py
@@ -65,9 +65,9 @@
# built documents.
#
# The short X.Y version.
-version = '4.10.1'
+version = '4.11'
# The full version, including alpha/beta/rc tags.
-release = '4.10.1'
+release = '4.11'
# The language for content autogenerated by Sphinx. Refer to documentation
# for a list of supported languages.
diff --git a/doc/sphinx-guides/source/developers/big-data-support.rst b/doc/sphinx-guides/source/developers/big-data-support.rst
index e0d0b4ffd25..06917073b32 100644
--- a/doc/sphinx-guides/source/developers/big-data-support.rst
+++ b/doc/sphinx-guides/source/developers/big-data-support.rst
@@ -230,7 +230,7 @@ Configuring the RSAL Mock
Info for configuring the RSAL Mock: https://github.com/sbgrid/rsal/tree/master/mocks
-Also, to configure Dataverse to use the new workflow you must do the following (see also the section below on workflows):
+Also, to configure Dataverse to use the new workflow you must do the following (see also the :doc:`workflows` section):
1. Configure the RSAL URL:
@@ -301,98 +301,4 @@ In the GUI, this is called "Local Access". It's where you can compute on files o
``curl http://localhost:8080/api/admin/settings/:LocalDataAccessPath -X PUT -d "/programs/datagrid"``
-Workflows
----------
-Dataverse can perform two sequences of actions when datasets are published: one prior to publishing (marked by a ``PrePublishDataset`` trigger), and one after the publication has succeeded (``PostPublishDataset``). The pre-publish workflow is useful for having an external system prepare a dataset for being publicly accessed (a possibly lengthy activity that requires moving files around, uploading videos to a streaming server, etc.), or to start an approval process. A post-publish workflow might be used for sending notifications about the newly published dataset.
-
-Workflow steps are created using *step providers*. Dataverse ships with an internal step provider that offers some basic functionality, and with the ability to load 3rd party step providers. This allows installations to implement functionality they need without changing the Dataverse source code.
-
-Steps can be internal (say, writing some data to the log) or external. External steps involve Dataverse sending a request to an external system, and waiting for the system to reply. The wait period is arbitrary, and so allows the external system unbounded operation time. This is useful, e.g., for steps that require human intervension, such as manual approval of a dataset publication.
-
-The external system reports the step result back to dataverse, by sending a HTTP ``POST`` command to ``api/workflows/{invocation-id}``. The body of the request is passed to the paused step for further processing.
-
-If a step in a workflow fails, Dataverse make an effort to roll back all the steps that preceeded it. Some actions, such as writing to the log, cannot be rolled back. If such an action has a public external effect (e.g. send an EMail to a mailing list) it is advisable to put it in the post-release workflow.
-
-.. tip::
- For invoking external systems using a REST api, Dataverse's internal step
- provider offers a step for sending and receiving customizable HTTP requests.
- It's called *http/sr*, and is detailed below.
-
-Administration
-~~~~~~~~~~~~~~
-
-A Dataverse instance stores a set of workflows in its database. Workflows can be managed using the ``api/admin/workflows/`` endpoints of the :doc:`/api/native-api`. Sample workflow files are available in ``scripts/api/data/workflows``.
-
-At the moment, defining a workflow for each trigger is done for the entire instance, using the endpoint ``api/admin/workflows/default/«trigger type»``.
-
-In order to prevent unauthorized resuming of workflows, Dataverse maintains a "white list" of IP addresses from which resume requests are honored. This list is maintained using the ``/api/admin/workflows/ip-whitelist`` endpoint of the :doc:`/api/native-api`. By default, Dataverse honors resume requests from localhost only (``127.0.0.1;::1``), so set-ups that use a single server work with no additional configuration.
-
-
-Available Steps
-~~~~~~~~~~~~~~~
-
-Dataverse has an internal step provider, whose id is ``:internal``. It offers the following steps:
-
-log
-^^^
-
-A step that writes data about the current workflow invocation to the instance log. It also writes the messages in its ``parameters`` map.
-
-.. code:: json
-
- {
- "provider":":internal",
- "stepType":"log",
- "parameters": {
- "aMessage": "message content",
- "anotherMessage": "message content, too"
- }
- }
-
-
-pause
-^^^^^
-
-A step that pauses the workflow. The workflow is paused until a POST request is sent to ``/api/workflows/{invocation-id}``.
-
-.. code:: json
-
- {
- "provider":":internal",
- "stepType":"pause"
- }
-
-
-http/sr
-^^^^^^^
-
-A step that sends a HTTP request to an external system, and then waits for a response. The response has to match a regular expression specified in the step parameters. The url, content type, and message body can use data from the workflow context, using a simple markup language. This step has specific parameters for rollback.
-
-.. code:: json
-
- {
- "provider":":internal",
- "stepType":"http/sr",
- "parameters": {
- "url":"http://localhost:5050/dump/${invocationId}",
- "method":"POST",
- "contentType":"text/plain",
- "body":"START RELEASE ${dataset.id} as ${dataset.displayName}",
- "expectedResponse":"OK.*",
- "rollbackUrl":"http://localhost:5050/dump/${invocationId}",
- "rollbackMethod":"DELETE ${dataset.id}"
- }
- }
-
-Available variables are:
-
-* ``invocationId``
-* ``dataset.id``
-* ``dataset.identifier``
-* ``dataset.globalId``
-* ``dataset.displayName``
-* ``dataset.citation``
-* ``minorVersion``
-* ``majorVersion``
-* ``releaseStatus``
diff --git a/doc/sphinx-guides/source/developers/deployment.rst b/doc/sphinx-guides/source/developers/deployment.rst
index fdcb32afded..15892a4fef3 100755
--- a/doc/sphinx-guides/source/developers/deployment.rst
+++ b/doc/sphinx-guides/source/developers/deployment.rst
@@ -70,23 +70,12 @@ Then update the file and replace the values for "aws_access_key_id" and "aws_sec
If you are having trouble configuring the files manually as described above, see https://docs.aws.amazon.com/cli/latest/userguide/cli-chap-getting-started.html which documents the ``aws configure`` command.
-Configure ~/.dataverse/ec2.env
-~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+Configure Ansible File (Optional)
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
-In order to publish datasets you must configure a file at ``~/.dataverse/ec2.env`` and contact DataCite at support@datacite.org to ask for a test username and password.
+In order to configure Dataverse settings such as the password of the dataverseAdmin user, download https://raw.githubusercontent.com/IQSS/dataverse-ansible/master/defaults/main.yml and edit the file to your liking.
-Create a ``.dataverse`` directory in your home directory like this:
-
-``mkdir ~/.dataverse``
-
-Download :download:`ec2.env <../../../../scripts/installer/ec2.env>` and put it in the directory at ``~/.dataverse`` that you just created. From the command line, you can try the command below to move the file into place:
-
-``mv ~/Downloads/ec2.env ~/.dataverse``
-
-Edit the file at ``~/.dataverse/ec2.env`` and fill in username and password from DataCite into the following fields:
-
-- dataverse_doi_username
-- dataverse_doi_password
+You can skip this step if you're fine with the values in the "main.yml" file in the link above.
Download and Run the "Create Instance" Script
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
@@ -103,6 +92,10 @@ You must specify the branch with ``-b`` but you can also specify a non-IQSS git
``bash ~/Downloads/ec2-create-instance.sh -b develop -r https://github.com/scholarsportal/dataverse.git``
+If you configured an Ansible file above and want to make use of it, add ``-g main.yml`` (or whatever you named your file) as in the following example.
+
+``bash ~/Downloads/ec2-create-instance.sh -b develop -r https://github.com/scholarsportal/dataverse.git -b main.yml``
+
Now you will need to wait around 15 minutes until the deployment is finished. Eventually, the output should tell you how to access the installation of Dataverse in a web browser or via ssh. It will also provide instructions on how to delete the instance when you are finished with it. Please be aware that AWS charges per minute for a running instance. You can also delete your instance from https://console.aws.amazon.com/console/home?region=us-east-1 .
Caveats
diff --git a/doc/sphinx-guides/source/developers/dev-environment.rst b/doc/sphinx-guides/source/developers/dev-environment.rst
index 9757ac5dd81..4b884092d7d 100755
--- a/doc/sphinx-guides/source/developers/dev-environment.rst
+++ b/doc/sphinx-guides/source/developers/dev-environment.rst
@@ -86,6 +86,19 @@ To install Glassfish, run the following commands:
``sudo chown -R $USER /usr/local/glassfish4``
+Test Glassfish Startup Time on Mac
+++++++++++++++++++++++++++++++++++
+
+``cd /usr/local/glassfish4/glassfish/bin``
+
+``./asadmin start-domain``
+
+``grep "startup time" /usr/local/glassfish4/glassfish/domains/domain1/logs/server.log``
+
+If you are seeing startup times in the 30 second range (31,584ms for "Felix" for example) please be aware that startup time can be greatly reduced (to less than 1.5 seconds in our testing) if you make a small edit to your ``/etc/hosts`` file as described at https://stackoverflow.com/questions/39636792/jvm-takes-a-long-time-to-resolve-ip-address-for-localhost/39698914#39698914 and https://thoeni.io/post/macos-sierra-java/
+
+Look for a line that says ``127.0.0.1 localhost`` and add a space followed by the output of ``hostname`` which should be something like ``foobar.local`` depending on the name of your Mac. For example, the line would say ``127.0.0.1 localhost foobar.local`` if your Mac's name is "foobar".
+
Install PostgreSQL
~~~~~~~~~~~~~~~~~~
@@ -112,7 +125,7 @@ On Linux, you should just install PostgreSQL from your package manager without w
Install Solr
~~~~~~~~~~~~
-`Solr `_ 7.3.0 is required.
+`Solr `_ 7.3.1 is required.
To install Solr, execute the following commands:
@@ -122,23 +135,23 @@ To install Solr, execute the following commands:
``cd /usr/local/solr``
-``curl -O http://archive.apache.org/dist/lucene/solr/7.3.0/solr-7.3.0.tgz``
+``curl -O http://archive.apache.org/dist/lucene/solr/7.3.1/solr-7.3.1.tgz``
-``tar xvfz solr-7.3.0.tgz``
+``tar xvfz solr-7.3.1.tgz``
-``cd solr-7.3.0/server/solr``
+``cd solr-7.3.1/server/solr``
``cp -r configsets/_default collection1``
-``curl -O https://raw.githubusercontent.com/IQSS/dataverse/develop/conf/solr/7.3.0/schema.xml``
+``curl -O https://raw.githubusercontent.com/IQSS/dataverse/develop/conf/solr/7.3.1/schema.xml``
``mv schema.xml collection1/conf``
-``curl -O https://raw.githubusercontent.com/IQSS/dataverse/develop/conf/solr/7.3.0/solrconfig.xml``
+``curl -O https://raw.githubusercontent.com/IQSS/dataverse/develop/conf/solr/7.3.1/solrconfig.xml``
``mv solrconfig.xml collection1/conf/solrconfig.xml``
-``cd /usr/local/solr/solr-7.3.0``
+``cd /usr/local/solr/solr-7.3.1``
``bin/solr start``
@@ -173,6 +186,8 @@ Run the following command:
``curl http://localhost:8080/api/admin/settings/:DoiProvider -X PUT -d FAKE``
+This will disable DOI registration by using a fake (in-code) DOI provider. Please note that this feature is only available in version >= 4.10 and that at present, the UI will give no indication that the DOIs thus minted are fake.
+
Next Steps
----------
diff --git a/doc/sphinx-guides/source/developers/index.rst b/doc/sphinx-guides/source/developers/index.rst
index 52bde9ee184..6f74d458e07 100755
--- a/doc/sphinx-guides/source/developers/index.rst
+++ b/doc/sphinx-guides/source/developers/index.rst
@@ -31,3 +31,4 @@ Developer Guide
geospatial
selinux
big-data-support
+ workflows
diff --git a/doc/sphinx-guides/source/developers/tips.rst b/doc/sphinx-guides/source/developers/tips.rst
index da6121d74fc..17b84381bd9 100755
--- a/doc/sphinx-guides/source/developers/tips.rst
+++ b/doc/sphinx-guides/source/developers/tips.rst
@@ -36,6 +36,8 @@ Dataverse only works with a specific version of Glassfish (see https://github.co
Launch Netbeans and click "Tools" and then "Servers". Click "Add Server" and select "Glassfish Server" and set the installation location to ``/usr/local/glassfish4``. The default are fine so you can click "Next" and "Finish". To avoid confusing, click "Remove Server" on the newer version of Glassfish that came bundled with Glassfish.
+Please note that if you are on a Mac, Netbeans may be unable to start Glassfish due to proxy settings in Netbeans. Go to the "General" tab in Netbeans preferences and click "Test connection" to see if you are affected. If you get a green checkmark, you're all set. If you get a red exclamation mark, change "Proxy Settings" to "No Proxy" and retest. A more complicated answer having to do with changing network settings is available at https://discussions.apple.com/thread/7680039?answerId=30715103022#30715103022 and the bug is also described at https://netbeans.org/bugzilla/show_bug.cgi?id=268076
+
At this point you can manage Glassfish using Netbeans. Click "Window" and then "Services". Expand "Servers" and right-click Glassfish to stop and then start it so that it appears in the Output window. Note that you can expand "Glassfish" and "Applications" to see if any applications are deployed.
Ensure that Dataverse Will Be Deployed to Glassfish 4.1
@@ -125,7 +127,7 @@ By default, Glassfish reports analytics information. The administration guide su
Solr
----
-Once some dataverses, datasets, and files have been created and indexed, you can experiment with searches directly from Solr at http://localhost:8983/solr/#/collection1/query and look at the JSON output of searches, such as this wildcard search: http://localhost:8983/solr/collection1/select?q=*%3A*&wt=json&indent=true . You can also get JSON output of static fields Solr knows about: http://localhost:8983/solr/schema/fields
+Once some dataverses, datasets, and files have been created and indexed, you can experiment with searches directly from Solr at http://localhost:8983/solr/#/collection1/query and look at the JSON output of searches, such as this wildcard search: http://localhost:8983/solr/collection1/select?q=*%3A*&wt=json&indent=true . You can also get JSON output of static fields Solr knows about: http://localhost:8983/solr/collection1/schema/fields
You can simply double-click "start.jar" rather that running ``java -jar start.jar`` from the command line. Figuring out how to stop Solr after double-clicking it is an exercise for the reader.
diff --git a/doc/sphinx-guides/source/developers/workflows.rst b/doc/sphinx-guides/source/developers/workflows.rst
new file mode 100644
index 00000000000..b9090b86be3
--- /dev/null
+++ b/doc/sphinx-guides/source/developers/workflows.rst
@@ -0,0 +1,130 @@
+Workflows
+================
+
+Dataverse has a flexible workflow mechanism that can be used to trigger actions before and after Dataset publication.
+
+.. contents:: |toctitle|
+ :local:
+
+
+Introduction
+------------
+
+Dataverse can perform two sequences of actions when datasets are published: one prior to publishing (marked by a ``PrePublishDataset`` trigger), and one after the publication has succeeded (``PostPublishDataset``). The pre-publish workflow is useful for having an external system prepare a dataset for being publicly accessed (a possibly lengthy activity that requires moving files around, uploading videos to a streaming server, etc.), or to start an approval process. A post-publish workflow might be used for sending notifications about the newly published dataset.
+
+Workflow steps are created using *step providers*. Dataverse ships with an internal step provider that offers some basic functionality, and with the ability to load 3rd party step providers. This allows installations to implement functionality they need without changing the Dataverse source code.
+
+Steps can be internal (say, writing some data to the log) or external. External steps involve Dataverse sending a request to an external system, and waiting for the system to reply. The wait period is arbitrary, and so allows the external system unbounded operation time. This is useful, e.g., for steps that require human intervension, such as manual approval of a dataset publication.
+
+The external system reports the step result back to dataverse, by sending a HTTP ``POST`` command to ``api/workflows/{invocation-id}``. The body of the request is passed to the paused step for further processing.
+
+If a step in a workflow fails, Dataverse make an effort to roll back all the steps that preceded it. Some actions, such as writing to the log, cannot be rolled back. If such an action has a public external effect (e.g. send an EMail to a mailing list) it is advisable to put it in the post-release workflow.
+
+.. tip::
+ For invoking external systems using a REST api, Dataverse's internal step
+ provider offers a step for sending and receiving customizable HTTP requests.
+ It's called *http/sr*, and is detailed below.
+
+Administration
+~~~~~~~~~~~~~~
+
+A Dataverse instance stores a set of workflows in its database. Workflows can be managed using the ``api/admin/workflows/`` endpoints of the :doc:`/api/native-api`. Sample workflow files are available in ``scripts/api/data/workflows``.
+
+At the moment, defining a workflow for each trigger is done for the entire instance, using the endpoint ``api/admin/workflows/default/«trigger type»``.
+
+In order to prevent unauthorized resuming of workflows, Dataverse maintains a "white list" of IP addresses from which resume requests are honored. This list is maintained using the ``/api/admin/workflows/ip-whitelist`` endpoint of the :doc:`/api/native-api`. By default, Dataverse honors resume requests from localhost only (``127.0.0.1;::1``), so set-ups that use a single server work with no additional configuration.
+
+
+Available Steps
+~~~~~~~~~~~~~~~
+
+Dataverse has an internal step provider, whose id is ``:internal``. It offers the following steps:
+
+log
++++
+
+A step that writes data about the current workflow invocation to the instance log. It also writes the messages in its ``parameters`` map.
+
+.. code:: json
+
+ {
+ "provider":":internal",
+ "stepType":"log",
+ "parameters": {
+ "aMessage": "message content",
+ "anotherMessage": "message content, too"
+ }
+ }
+
+
+pause
++++++
+
+A step that pauses the workflow. The workflow is paused until a POST request is sent to ``/api/workflows/{invocation-id}``.
+
+.. code:: json
+
+ {
+ "provider":":internal",
+ "stepType":"pause"
+ }
+
+
+http/sr
++++++++
+
+A step that sends a HTTP request to an external system, and then waits for a response. The response has to match a regular expression specified in the step parameters. The url, content type, and message body can use data from the workflow context, using a simple markup language. This step has specific parameters for rollback.
+
+.. code:: json
+
+ {
+ "provider":":internal",
+ "stepType":"http/sr",
+ "parameters": {
+ "url":"http://localhost:5050/dump/${invocationId}",
+ "method":"POST",
+ "contentType":"text/plain",
+ "body":"START RELEASE ${dataset.id} as ${dataset.displayName}",
+ "expectedResponse":"OK.*",
+ "rollbackUrl":"http://localhost:5050/dump/${invocationId}",
+ "rollbackMethod":"DELETE ${dataset.id}"
+ }
+ }
+
+Available variables are:
+
+* ``invocationId``
+* ``dataset.id``
+* ``dataset.identifier``
+* ``dataset.globalId``
+* ``dataset.displayName``
+* ``dataset.citation``
+* ``minorVersion``
+* ``majorVersion``
+* ``releaseStatus``
+
+archiver
+++++++++
+
+A step that sends an archival copy of a Dataset Version to a configured archiver, e.g. the DuraCloud interface of Chronopolis. See the `DuraCloud/Chronopolis Integration documentation `_ for further detail.
+
+Note - the example step includes two settings required for any archiver and three (DuraCloud*) that are specific to DuraCloud.
+
+.. code:: json
+
+
+ {
+ "provider":":internal",
+ "stepType":"archiver",
+ "parameters": {
+ "stepName":"archive submission"
+ },
+ "requiredSettings": {
+ ":ArchiverClassName": "string",
+ ":ArchiverSettings": "string",
+ ":DuraCloudHost":"string",
+ ":DuraCloudPort":"string",
+ ":DuraCloudContext":"string"
+ }
+ }
+
diff --git a/doc/sphinx-guides/source/index.rst b/doc/sphinx-guides/source/index.rst
index 7f292b90249..2db24b7a952 100755
--- a/doc/sphinx-guides/source/index.rst
+++ b/doc/sphinx-guides/source/index.rst
@@ -60,11 +60,14 @@ We maintain an email based support service that's free of charge. We
attempt to respond within one business day to all questions and if it
cannot be resolved immediately, we'll let you know what to expect.
-The support email address is
-`support@dataverse.org `__.
+The support email address is `support@dataverse.org `__.
-This is the same address as the Report Issue link. We try to respond
-within one business day.
+**Reporting Issues and Contributing**
+
+Report bugs and add feature requests in `GitHub Issues `__
+or use `GitHub pull requests `__,
+if you have some code, scripts or documentation that you'd like to share.
+If you have a **security issue** to report, please email `security@dataverse.org `__.
Indices and tables
diff --git a/doc/sphinx-guides/source/installation/advanced.rst b/doc/sphinx-guides/source/installation/advanced.rst
index acc2091b4fa..b0264f07238 100644
--- a/doc/sphinx-guides/source/installation/advanced.rst
+++ b/doc/sphinx-guides/source/installation/advanced.rst
@@ -15,6 +15,8 @@ You should be conscious of the following when running multiple Glassfish servers
- Only one Glassfish server can be the dedicated timer server, as explained in the :doc:`/admin/timers` section of the Admin Guide.
- When users upload a logo for their dataverse using the "theme" feature described in the :doc:`/user/dataverse-management` section of the User Guide, these logos are stored only on the Glassfish server the user happend to be on when uploading the logo. By default these logos are written to the directory ``/usr/local/glassfish4/glassfish/domains/domain1/docroot/logos``.
- When a sitemp is created by a Glassfish server it is written to the filesystem of just that Glassfish server. By default the sitemap is written to the directory ``/usr/local/glassfish4/glassfish/domains/domain1/docroot/sitemap``.
+- Dataset draft version logging occurs separately on each Glassfish server. See "Edit Draft Versions Logging" in the :doc:`/admin/monitoring` section of the Admin Guide for details.
+- Password aliases (``db_password_alias``, etc.) are stored per Glassfish server.
Detecting Which Glassfish Server a User Is On
+++++++++++++++++++++++++++++++++++++++++++++
diff --git a/doc/sphinx-guides/source/installation/config.rst b/doc/sphinx-guides/source/installation/config.rst
index 61a16ecd95e..94c16e2f81a 100644
--- a/doc/sphinx-guides/source/installation/config.rst
+++ b/doc/sphinx-guides/source/installation/config.rst
@@ -110,12 +110,16 @@ Persistent Identifiers and Publishing Datasets
Persistent identifiers are a required and integral part of the Dataverse platform. They provide a URL that is guaranteed to resolve to the datasets or files they represent. Dataverse currently supports creating identifiers using DOI and Handle.
-By default, the installer configures a test DOI namespace (10.5072) with DataCite as the registration provider. Please note that as of the release 4.9.3, we can no longer use EZID as the provider. Unlike EZID, DataCite requires that you register for a test account (please contact support@datacite.org). Once you receive the login name and password for the account, configure it in your domain.xml, as the following two JVM options::
+By default, the installer configures a default DOI namespace (10.5072) with DataCite as the registration provider. Please note that as of the release 4.9.3, we can no longer use EZID as the provider. Unlike EZID, DataCite requires that you register for a test account, configured with your own prefix (please contact support@datacite.org). Once you receive the login name, password, and prefix for the account, configure the credentials in your domain.xml, as the following two JVM options::
-Ddoi.username=...
-Ddoi.password=...
-and restart Glassfish. Once this is done, you will be able to publish datasets and files, but the persistent identifiers will not be citable or guaranteed to be preserved. Note that any datasets or files created using the test configuration cannot be directly migrated and would need to be created again once a valid DOI namespace is configured.
+and restart Glassfish. The prefix can be configured via the API (where it is referred to as "Authority"):
+
+``curl -X PUT -d 10.xxxx http://localhost:8080/api/admin/settings/:Authority``
+
+Once this is done, you will be able to publish datasets and files, but the persistent identifiers will not be citable, and they will only resolve from the DataCite test environment (and then only if the Dataverse from which you published them is accessible - DOIs minted from your laptop will not resolve). Note that any datasets or files created using the test configuration cannot be directly migrated and would need to be created again once a valid DOI namespace is configured.
To properly configure persistent identifiers for a production installation, an account and associated namespace must be acquired for a fee from a DOI or HDL provider. **DataCite** (https://www.datacite.org) is the recommended DOI provider (see https://dataverse.org/global-dataverse-community-consortium for more on joining DataCite) but **EZID** (http://ezid.cdlib.org) is an option for the University of California according to https://www.cdlib.org/cdlinfo/2017/08/04/ezid-doi-service-is-evolving/ . **Handle.Net** (https://www.handle.net) is the HDL provider.
@@ -526,6 +530,97 @@ Once you have the location of your custom CSS file, run this curl command to add
``curl -X PUT -d '/var/www/dataverse/branding/custom-stylesheet.css' http://localhost:8080/api/admin/settings/:StyleCustomizationFile``
+.. _Web-Analytics-Code:
+
+Web Analytics Code
+------------------
+
+Your analytics code can be added to your Dataverse installation in a similar fashion to how you brand it, by adding a custom HTML file containing the analytics code snippet and adding the file location to your settings.
+
+Popular analytics providers Google Analytics (https://www.google.com/analytics/) and Matomo (formerly "Piwik"; https://matomo.org/) have been set up with Dataverse. Use the documentation they provide to add the analytics code to your custom HTML file. This allows for more control of your analytics, making it easier to customize what you prefer to track.
+
+Create your own ``analytics-code.html`` file using the analytics code snippet provided by Google or Matomo and place it at ``/var/www/dataverse/branding/analytics-code.html``. Here is an example of what your HTML file should like like:
+
+.. code-block:: none
+
+
+
+Once you have the location of your analytics file, run this curl command to add it to your settings:
+
+``curl -X PUT -d '/var/www/dataverse/branding/analytics-code.html' http://localhost:8080/api/admin/settings/:WebAnalyticsCode``
+
+DuraCloud/Chronopolis Integration
+---------------------------------
+
+It's completely optional to integrate your installation of Dataverse with DuraCloud/Chronopolis but the details are listed here to keep the :doc:`/admin/integrations` section of the Admin Guide shorter.
+
+Dataverse can be configured to submit a copy of published Datasets, packaged as `Research Data Alliance conformant `_ zipped `BagIt `_ bags to the `Chronopolis `_ via `DuraCloud `_
+
+This integration is occurs through customization of an internal Dataverse archiver workflow that can be configured as a PostPublication workflow to submit the bag to Chronopolis' Duracloud interface using your organization's credentials. An admin API call exists that can manually submit previously published Datasets, and prior versions, to a configured archive such as Chronopolis. The workflow leverages new functionality in Dataverse to create a `JSON-LD `_ serialized `OAI-ORE `_ map file, which is also available as a metadata export format in the Dataverse web interface.
+
+At present, the DPNSubmitToArchiveCommand is the only implementation extending the AbstractSubmitToArchiveCommand and using the configurable mechanisms discussed below.
+
+Also note that while the current Chronopolis implementation generates the bag and submits it to the archive's DuraCloud interface, the step to make a 'snapshot' of the space containing the Bag (and verify it's successful submission) are actions a curator must take in the DuraCloud interface.
+
+The minimal configuration to support an archiver integration involves adding a minimum of two Dataverse Keys and any required Glassfish jvm options. The example instructions here are specific to the DuraCloud Archiver\:
+
+\:ArchiverClassName - the fully qualified class to be used for archiving. For example:
+
+``curl http://localhost:8080/api/admin/settings/:ArchiverClassName -X PUT -d "edu.harvard.iq.dataverse.engine.command.impl.DuraCloudSubmitToArchiveCommand"``
+
+\:ArchiverSettings - the archiver class can access required settings including existing Dataverse settings and dynamically defined ones specific to the class. This setting is a comma-separated list of those settings. For example\:
+
+``curl http://localhost:8080/api/admin/settings/:ArchiverSettings -X PUT -d ":DuraCloudHost, :DuraCloudPort, :DuraCloudContext"``
+
+The DPN archiver defines three custom settings, one of which is required (the others have defaults):
+
+\:DuraCloudHost - the URL for your organization's Duracloud site. For example:
+
+``curl http://localhost:8080/api/admin/settings/:DuraCloudHost -X PUT -d "qdr.duracloud.org"``
+
+:DuraCloudPort and :DuraCloudContext are also defined if you are not using the defaults ("443" and "duracloud" respectively). (Note\: these settings are only in effect if they are listed in the \:ArchiverSettings. Otherwise, they will not be passed to the DuraCloud Archiver class.)
+
+Archivers may require glassfish settings as well. For the Chronopolis archiver, the username and password associated with your organization's Chronopolis/DuraCloud account should be configured in Glassfish:
+
+``./asadmin create-jvm-options '-Dduracloud.username=YOUR_USERNAME_HERE'``
+
+``./asadmin create-jvm-options '-Dduracloud.password=YOUR_PASSWORD_HERE'``
+
+**API Call**
+
+Once this configuration is complete, you, as a user with the *PublishDataset* permission, should be able to use the API call to manually submit a DatasetVersion for processing:
+
+``curl -H "X-Dataverse-key: " http://localhost:8080/api/admin/submitDataVersionToArchive/{id}/{version}``
+
+where:
+
+{id} is the DatasetId (or :persistentId with the ?persistentId="\" parameter), and
+
+{version} is the friendly version number, e.g. "1.2".
+
+The submitDataVersionToArchive API (and the workflow discussed below) attempt to archive the dataset version via an archive specific method. For Chronopolis, a DuraCloud space named for the dataset (it's DOI with ':' and '.' replaced with '-') is created and two files are uploaded to it: a version-specific datacite.xml metadata file and a BagIt bag containing the data and an OAI-ORE map file. (The datacite.xml file, stored outside the Bag as well as inside is intended to aid in discovery while the ORE map file is 'complete', containing all user-entered metadata and is intended as an archival record.)
+
+In the Chronopolis case, since the transfer from the DuraCloud front-end to archival storage in Chronopolis can take significant time, it is currently up to the admin/curator to submit a 'snap-shot' of the space within DuraCloud and to monitor its successful transfer. Once transfer is complete the space should be deleted, at which point the Dataverse API call can be used to submit a Bag for other versions of the same Dataset. (The space is reused, so that archival copies of different Dataset versions correspond to different snapshots of the same DuraCloud space.).
+
+**PostPublication Workflow**
+
+To automate the submission of archival copies to an archive as part of publication, one can setup a Dataverse Workflow using the "archiver" workflow step - see the :doc:`/developers/workflows` guide.
+. The archiver step uses the configuration information discussed above including the :ArchiverClassName setting. The workflow step definition should include the set of properties defined in \:ArchiverSettings in the workflow definition.
+
+To active this workflow, one must first install a workflow using the archiver step. A simple workflow that invokes the archiver step configured to submit to DuraCloud as its only action is included in dataverse at /scripts/api/data/workflows/internal-archiver-workflow.json.
+
+Using the Workflow Native API (see the :doc:`/api/native-api` guide) this workflow can be installed using:
+
+``curl -X POST -H 'Content-type: application/json' --upload-file internal-archiver-workflow.json http://localhost:8080/api/admin/workflows``
+
+The workflow id returned in this call (or available by doing a GET of /api/admin/workflows ) can then be submitted as the default PostPublication workflow:
+
+``curl -X PUT -d {id} http://localhost:8080/api/admin/workflows/default/PostPublishDataset``
+
+Once these steps are taken, new publication requests will automatically trigger submission of an archival copy to the specified archiver, Chronopolis' DuraCloud component in this example. For Chronopolis, as when using the API, it is currently the admin's responsibility to snap-shot the DuraCloud space and monitor the result. Failure of the workflow, (e.g. if DuraCloud is unavailable, the configuration is wrong, or the space for this dataset already exists due to a prior publication action or use of the API), will create a failure message but will not affect publication itself.
+
Going Live: Launching Your Production Deployment
------------------------------------------------
@@ -872,6 +967,11 @@ See :ref:`Branding Your Installation` above.
See :ref:`Branding Your Installation` above.
+:WebAnalyticsCode
++++++++++++++++++
+
+See :ref:`Web-Analytics-Code` above.
+
:FooterCopyright
++++++++++++++++
@@ -884,7 +984,7 @@ By default the footer says "Copyright © [YYYY]" but you can add text after the
:DoiProvider
++++++++++++
-As of this writing "DataCite" and "EZID" are the only valid options for production installations. Developers are welcome to use "FAKE". ``:DoiProvider`` is only needed if you are using DOI.
+As of this writing "DataCite" and "EZID" are the only valid options for production installations. Developers using version 4.10 and above are welcome to use the keyword "FAKE" to configure a non-production installation with an non-resolving, in-code provider, which will basically short-circuit the DOI publishing process. ``:DoiProvider`` is only needed if you are using DOI.
``curl -X PUT -d DataCite http://localhost:8080/api/admin/settings/:DoiProvider``
@@ -1126,13 +1226,6 @@ For example, if you want your installation of Dataverse to not attempt to ingest
Limit the number of files in a zip that Dataverse will accept.
-:GoogleAnalyticsCode
-++++++++++++++++++++
-
-Set your Google Analytics Tracking ID thusly:
-
-``curl -X PUT -d 'trackingID' http://localhost:8080/api/admin/settings/:GoogleAnalyticsCode``
-
:SolrHostColonPort
++++++++++++++++++
@@ -1240,34 +1333,6 @@ Here is an example of setting the default auth provider back to ``builtin``:
Set to false to disallow local accounts to be created. See also the sections on :doc:`shibboleth` and :doc:`oauth2`.
-:PiwikAnalyticsId
-++++++++++++++++++++
-
-Site identifier created in your Piwik instance. Example:
-
-``curl -X PUT -d 42 http://localhost:8080/api/admin/settings/:PiwikAnalyticsId``
-
-:PiwikAnalyticsHost
-++++++++++++++++++++
-
-Host FQDN or URL of your Piwik instance before the ``/piwik.php``. Examples:
-
-``curl -X PUT -d stats.domain.tld http://localhost:8080/api/admin/settings/:PiwikAnalyticsHost``
-
-or
-
-``curl -X PUT -d hostname.domain.tld/stats http://localhost:8080/api/admin/settings/:PiwikAnalyticsHost``
-
-:PiwikAnalyticsTrackerFileName
-++++++++++++++++++++++++++++++
-
-Filename for the 'php' and 'js' tracker files used in the Piwik code (piwik.php and piwik.js).
-Sometimes these files are renamed in order to prevent ad-blockers (in the browser) to block the Piwik tracking code.
-This sets the base name (without dot and extension), if not set it defaults to 'piwik'.
-
-``curl -X PUT -d domainstats http://localhost:8080/api/admin/settings/:PiwikAnalyticsTrackerFileName``
-
-
:FileFixityChecksumAlgorithm
++++++++++++++++++++++++++++
diff --git a/doc/sphinx-guides/source/installation/external-tools.rst b/doc/sphinx-guides/source/installation/external-tools.rst
index c5fde7dba82..d47c85c8324 100644
--- a/doc/sphinx-guides/source/installation/external-tools.rst
+++ b/doc/sphinx-guides/source/installation/external-tools.rst
@@ -14,8 +14,12 @@ Support for external tools is just getting off the ground but the following tool
- TwoRavens: a system of interlocking statistical tools for data exploration, analysis, and meta-analysis: http://2ra.vn. See the :doc:`/user/data-exploration/tworavens` section of the User Guide for more information on TwoRavens from the user perspective and the :doc:`r-rapache-tworavens` section of the Installation Guide.
- Data Explorer: a GUI which lists the variables in a tabular data file allowing searching, charting and cross tabulation analysis. See the README.md file at https://github.com/scholarsportal/Dataverse-Data-Explorer for the instructions on adding Data Explorer to your Dataverse; and the :doc:`prerequisites` section of the Installation Guide for the instructions on how to set up **basic R configuration required** (specifically, Dataverse uses R to generate .prep metadata files that are needed to run Data Explorer).
+
+- `Whole Tale `_: a platform for the creation of reproducible research packages that allows users to launch containerized interactive analysis environments based on popular tools such as Jupyter and RStudio. Using this integration, Dataverse users can launch Jupyter and RStudio environments to analyze published datasets. For more information, see the `Whole Tale User Guide `_.
+
- [Your tool here! Please get in touch! :) ]
+
Downloading and Adjusting an External Tool Manifest File
--------------------------------------------------------
@@ -25,13 +29,17 @@ External tools must be expressed in an external tool manifest file, a specific J
.. literalinclude:: ../_static/installation/files/root/external-tools/awesomeTool.json
-``type`` is required and must be ``explore`` or ``configure`` to make the tool appear under a button called "Explore" or "Configure", respectively. Currently external tools only operate on tabular files that have been successfully ingested. (For more on ingest, see the :doc:`/user/tabulardataingest/ingestprocess` of the User Guide.)
+``type`` is required and must be ``explore`` or ``configure`` to make the tool appear under a button called "Explore" or "Configure", respectively.
+
+External tools can operate on any file, including tabular files that have been created by successful ingestion. (For more on ingest, see the :doc:`/user/tabulardataingest/ingestprocess` of the User Guide.) The optional ``contentType`` entry specifies the mimetype a tool works on. (Not providing this parameter makes the tool work on ingested tabular files and is equivalent to specifying the ``contentType`` as "text/tab-separated-values".)
In the example above, a mix of required and optional reserved words appear that can be used to insert dynamic values into tools. The supported values are:
- ``{fileId}`` (required) - The Dataverse database ID of a file the external tool has been launched on.
- ``{siteUrl}`` (optional) - The URL of the Dataverse installation that hosts the file with the fileId above.
- ``{apiToken}`` (optional) - The Dataverse API token of the user launching the external tool, if available.
+- ``{datasetId}`` (optional) - The ID of the dataset containing the file.
+- ``{datasetVersion}`` (optional) - The friendly version number ( or \:draft ) of the dataset version the tool is being launched from.
Making an External Tool Available in Dataverse
----------------------------------------------
diff --git a/doc/sphinx-guides/source/installation/installation-main.rst b/doc/sphinx-guides/source/installation/installation-main.rst
index ba6a56c4e4b..4f10961936b 100755
--- a/doc/sphinx-guides/source/installation/installation-main.rst
+++ b/doc/sphinx-guides/source/installation/installation-main.rst
@@ -69,6 +69,14 @@ This allows the installer to be run in non-interactive mode (with ``./install -y
All the Glassfish configuration tasks performed by the installer are isolated in the shell script ``dvinstall/glassfish-setup.sh`` (as ``asadmin`` commands).
+**IMPORTANT:** As a security measure, the ``glassfish-setup.sh`` script stores passwords as "aliases" rather than plaintext. If you change your database password, for example, you will need to update the alias with ``asadmin update-password-alias db_password_alias``, for example. Here is a list of the password aliases that are set by the installation process and entered into Glassfish's ``domain.xml`` file:
+
+- ``db_password_alias``
+- ``doi_password_alias``
+- ``rserve_password_alias``
+
+Glassfish does not provide up to date documentation but Payara (a fork of Glassfish) does so for more information, please see https://docs.payara.fish/documentation/payara-server/password-aliases/password-alias-asadmin-commands.html
+
**IMPORTANT:** The installer will also ask for an external site URL for Dataverse. It is *imperative* that this value be supplied accurately, or a long list of functions will be inoperable, including:
- email confirmation links
diff --git a/doc/sphinx-guides/source/installation/prerequisites.rst b/doc/sphinx-guides/source/installation/prerequisites.rst
index 06b41e9e7d3..4a77a536724 100644
--- a/doc/sphinx-guides/source/installation/prerequisites.rst
+++ b/doc/sphinx-guides/source/installation/prerequisites.rst
@@ -202,19 +202,19 @@ Become the ``solr`` user and then download and configure Solr::
su - solr
cd /usr/local/solr
- wget https://archive.apache.org/dist/lucene/solr/7.3.0/solr-7.3.0.tgz
- tar xvzf solr-7.3.0.tgz
- cd solr-7.3.0
+ wget https://archive.apache.org/dist/lucene/solr/7.3.1/solr-7.3.1.tgz
+ tar xvzf solr-7.3.1.tgz
+ cd solr-7.3.1
cp -r server/solr/configsets/_default server/solr/collection1
You should already have a "dvinstall.zip" file that you downloaded from https://github.com/IQSS/dataverse/releases . Unzip it into ``/tmp``. Then copy the files into place::
- cp /tmp/dvinstall/schema.xml /usr/local/solr/solr-7.3.0/server/solr/collection1/conf
- cp /tmp/dvinstall/solrconfig.xml /usr/local/solr/solr-7.3.0/server/solr/collection1/conf
+ cp /tmp/dvinstall/schema.xml /usr/local/solr/solr-7.3.1/server/solr/collection1/conf
+ cp /tmp/dvinstall/solrconfig.xml /usr/local/solr/solr-7.3.1/server/solr/collection1/conf
Note: Dataverse has customized Solr to boost results that come from certain indexed elements inside Dataverse, for example prioritizing results from Dataverses over Datasets. If you would like to remove this, edit your ``solrconfig.xml`` and remove the ```` element and its contents. If you have ideas about how this boosting could be improved, feel free to contact us through our Google Group https://groups.google.com/forum/#!forum/dataverse-dev .
-Dataverse requires a change to the ``jetty.xml`` file that ships with Solr. Edit ``/usr/local/solr/solr-7.3.0/server/etc/jetty.xml`` , increasing ``requestHeaderSize`` from ``8192`` to ``102400``
+Dataverse requires a change to the ``jetty.xml`` file that ships with Solr. Edit ``/usr/local/solr/solr-7.3.1/server/etc/jetty.xml`` , increasing ``requestHeaderSize`` from ``8192`` to ``102400``
Solr will warn about needing to increase the number of file descriptors and max processes in a production environment but will still run with defaults. We have increased these values to the recommended levels by adding ulimit -n 65000 to the init script, and the following to ``/etc/security/limits.conf``::
@@ -223,7 +223,7 @@ Solr will warn about needing to increase the number of file descriptors and max
solr soft nofile 65000
solr hard nofile 65000
-On operating systems which use systemd such as RHEL or CentOS 7, you may then add a line like LimitNOFILE=65000 to the systemd unit file, or adjust the limits on a running process using the prlimit tool::
+On operating systems which use systemd such as RHEL or CentOS 7, you may then add a line like LimitNOFILE=65000 for the number of open file descriptors and a line with LimitNPROC=65000 for the max processes to the systemd unit file, or adjust the limits on a running process using the prlimit tool::
# sudo prlimit --pid pid --nofile=65000:65000
@@ -231,7 +231,7 @@ Solr launches asynchronously and attempts to use the ``lsof`` binary to watch fo
Finally, you may start Solr and create the core that will be used to manage search information::
- cd /usr/local/solr/solr-7.3.0
+ cd /usr/local/solr/solr-7.3.1
bin/solr start
bin/solr create_core -c collection1 -d server/solr/collection1/conf/
diff --git a/doc/sphinx-guides/source/installation/shibboleth.rst b/doc/sphinx-guides/source/installation/shibboleth.rst
index d3c0ab49dc8..018b9995b67 100644
--- a/doc/sphinx-guides/source/installation/shibboleth.rst
+++ b/doc/sphinx-guides/source/installation/shibboleth.rst
@@ -196,6 +196,13 @@ attribute-map.xml
By default, some attributes ``/etc/shibboleth/attribute-map.xml`` are commented out. Edit the file to enable them so that all the require attributes come through. You can download a :download:`sample attribute-map.xml file <../_static/installation/files/etc/shibboleth/attribute-map.xml>`.
+Shibboleth and ADFS
+~~~~~~~~~~~~~~~~~~~
+With appropriate configuration, Dataverse and Shibboleth can make use of "single sign on" using Active Directory.
+This requires configuring ``shibd`` and ``httpd`` to load appropriate libraries, and insuring that the attribute mapping matches those provided.
+Example configuration files for :download:`shibboleth2.xml <../_static/installation/files/etc/shibboleth/shibboleth2_adfs.xml>` and :download:`attribute-map.xml <../_static/installation/files/etc/shibboleth/attribute-map_adfs.xml>` may be helpful.
+Note that your ADFS server hostname goes in the file referenced under "MetadataProvider" in your shibboleth2.xml file.
+
Disable or Reconfigure SELinux
------------------------------
diff --git a/doc/sphinx-guides/source/user/dataset-management.rst b/doc/sphinx-guides/source/user/dataset-management.rst
index 51c409cc657..124b62344f4 100755
--- a/doc/sphinx-guides/source/user/dataset-management.rst
+++ b/doc/sphinx-guides/source/user/dataset-management.rst
@@ -139,7 +139,7 @@ Additional download options available for tabular data (found in the same drop-d
- The original file uploaded by the user;
- Saved as R data (if the original file was not in R format);
- Variable Metadata (as a `DDI Codebook `_ XML file);
-- Data File Citation (currently in either RIS or EndNote XML format);
+- Data File Citation (currently in either RIS, EndNote XML, or BibTeX format);
- All of the above, as a zipped bundle.
|image2|
@@ -180,7 +180,9 @@ Metadata found in the header section of `Flexible Image Transport System (FITS)
Compressed Files
----------------
-Compressed files in zip format are unpacked automatically. If it fails to unpack, for whatever reason, it will upload as is. If the number of files inside are more than a set limit (1,000), you will get an error message and the file will uploads as is.
+Compressed files in zip format are unpacked automatically. If it fails to unpack, for whatever reason, it will upload as is. If the number of files inside are more than a set limit (1,000 by default, configurable by the Administrator), you will get an error message and the zip file will uploads as is.
+
+.. note:: If the uploaded zip file contains sub-folders, the names of the folders will be preserved as the ``DirectoryLabel`` attributes in the ``FileMetadata`` objects of the corresponding individual datafiles. As of writing this - v.4.11 - these labels are only used to restore the folder structure in multi-file, zipped download bundles (see :doc:`/api/dataaccess` for more information). In the future folder structure will be supported for organizing files on the dataset page as well.
Support for unpacking tar files will be added when this ticket is closed: https://github.com/IQSS/dataverse/issues/2195.
diff --git a/doc/sphinx-guides/source/versions.rst b/doc/sphinx-guides/source/versions.rst
index 8b5a91936ff..6ee3396ab0a 100755
--- a/doc/sphinx-guides/source/versions.rst
+++ b/doc/sphinx-guides/source/versions.rst
@@ -6,8 +6,9 @@ Dataverse Guides Versions
This list provides a way to refer to previous versions of the Dataverse guides, which we still host. In order to learn more about the updates delivered from one version to another, visit the `Releases `__ page in our GitHub repo.
-- 4.10.1
+- 4.11
+- `4.10.1 `__
- `4.10 `__
- `4.9.4 `__
- `4.9.3 `__
diff --git a/downloads/download.sh b/downloads/download.sh
index 079da2694e6..239d75288b6 100755
--- a/downloads/download.sh
+++ b/downloads/download.sh
@@ -1,5 +1,5 @@
#!/bin/sh
curl -L -O http://download.java.net/glassfish/4.1/release/glassfish-4.1.zip
-curl -L -O https://archive.apache.org/dist/lucene/solr/7.3.0/solr-7.3.0.tgz
+curl -L -O https://archive.apache.org/dist/lucene/solr/7.3.1/solr-7.3.1.tgz
curl -L -O http://search.maven.org/remotecontent?filepath=org/jboss/weld/weld-osgi-bundle/2.2.10.Final/weld-osgi-bundle-2.2.10.Final-glassfish4.jar
curl -s -L http://sourceforge.net/projects/schemaspy/files/schemaspy/SchemaSpy%205.0.0/schemaSpy_5.0.0.jar/download > schemaSpy_5.0.0.jar
diff --git a/pom.xml b/pom.xml
index c2eaba8da54..e21ba43ef80 100644
--- a/pom.xml
+++ b/pom.xml
@@ -7,7 +7,7 @@
-->
edu.harvard.iq
dataverse
- 4.10.1
+ 4.11
war
dataverse
@@ -37,7 +37,11 @@
5.3.1
1.3.1
2.22.0
- 0.8.2
+
+ 0.8.1
+
+ org.apache.commons
+ commons-compress
+ 1.18
+
+
+ org.duracloud
+ common
+ 4.4.6
+
+
+ org.slf4j
+ log4j-over-slf4j
+
+
+ ch.qos.logback
+ logback-classic
+
+
+
+
+ org.duracloud
+ storeclient
+ 4.4.6
+
+
+ org.slf4j
+ log4j-over-slf4j
+
+
+ com.amazonaws
+ aws-java-sdk-sqs
+
+
+ ch.qos.logback
+ logback-classic
+
+
+
+
+
+ commons-cli
+ commons-cli
+ 1.2
+
org.apache.tika
diff --git a/scripts/api/data/workflows/internal-archiver-workflow.json b/scripts/api/data/workflows/internal-archiver-workflow.json
new file mode 100644
index 00000000000..abaf55faad1
--- /dev/null
+++ b/scripts/api/data/workflows/internal-archiver-workflow.json
@@ -0,0 +1,19 @@
+{
+ "name": "Archive submission workflow",
+ "steps": [
+ {
+ "provider": ":internal",
+ "stepType": "archiver",
+ "parameters": {
+ "stepName": "archive submission"
+ },
+ "requiredSettings": {
+ ":ArchiverClassName": "string",
+ ":ArchiverSettings": "string",
+ ":DuraCloudHost": "string",
+ ":DuraCloudPort": "string",
+ ":DuraCloudContext": "string"
+ }
+ }
+ ]
+}
diff --git a/scripts/api/data/zip/test.zip b/scripts/api/data/zip/test.zip
new file mode 100644
index 00000000000..9d94b062ec7
Binary files /dev/null and b/scripts/api/data/zip/test.zip differ
diff --git a/scripts/database/create/create_v4.11.sql b/scripts/database/create/create_v4.11.sql
new file mode 100644
index 00000000000..4ba735c5186
--- /dev/null
+++ b/scripts/database/create/create_v4.11.sql
@@ -0,0 +1,351 @@
+CREATE TABLE DATAVERSETHEME (ID SERIAL NOT NULL, BACKGROUNDCOLOR VARCHAR(255), LINKCOLOR VARCHAR(255), LINKURL VARCHAR(255), LOGO VARCHAR(255), LOGOALIGNMENT VARCHAR(255), LOGOBACKGROUNDCOLOR VARCHAR(255), LOGOFORMAT VARCHAR(255), TAGLINE VARCHAR(255), TEXTCOLOR VARCHAR(255), dataverse_id BIGINT, PRIMARY KEY (ID));
+CREATE INDEX INDEX_DATAVERSETHEME_dataverse_id ON DATAVERSETHEME (dataverse_id);
+CREATE TABLE DATAFILECATEGORY (ID SERIAL NOT NULL, NAME VARCHAR(255) NOT NULL, DATASET_ID BIGINT NOT NULL, PRIMARY KEY (ID));
+CREATE INDEX INDEX_DATAFILECATEGORY_dataset_id ON DATAFILECATEGORY (dataset_id);
+CREATE TABLE ROLEASSIGNMENT (ID SERIAL NOT NULL, ASSIGNEEIDENTIFIER VARCHAR(255) NOT NULL, PRIVATEURLTOKEN VARCHAR(255), DEFINITIONPOINT_ID BIGINT NOT NULL, ROLE_ID BIGINT NOT NULL, PRIMARY KEY (ID));
+CREATE INDEX INDEX_ROLEASSIGNMENT_assigneeidentifier ON ROLEASSIGNMENT (assigneeidentifier);
+CREATE INDEX INDEX_ROLEASSIGNMENT_definitionpoint_id ON ROLEASSIGNMENT (definitionpoint_id);
+CREATE INDEX INDEX_ROLEASSIGNMENT_role_id ON ROLEASSIGNMENT (role_id);
+CREATE TABLE WORKFLOW (ID SERIAL NOT NULL, NAME VARCHAR(255), PRIMARY KEY (ID));
+CREATE TABLE OAISET (ID SERIAL NOT NULL, DEFINITION TEXT, DELETED BOOLEAN, DESCRIPTION TEXT, NAME TEXT, SPEC TEXT, UPDATEINPROGRESS BOOLEAN, VERSION BIGINT, PRIMARY KEY (ID));
+CREATE TABLE DATAVERSELINKINGDATAVERSE (ID SERIAL NOT NULL, LINKCREATETIME TIMESTAMP, DATAVERSE_ID BIGINT NOT NULL, LINKINGDATAVERSE_ID BIGINT NOT NULL, PRIMARY KEY (ID));
+CREATE INDEX INDEX_DATAVERSELINKINGDATAVERSE_dataverse_id ON DATAVERSELINKINGDATAVERSE (dataverse_id);
+CREATE INDEX INDEX_DATAVERSELINKINGDATAVERSE_linkingDataverse_id ON DATAVERSELINKINGDATAVERSE (linkingDataverse_id);
+CREATE TABLE METADATABLOCK (ID SERIAL NOT NULL, DISPLAYNAME VARCHAR(255) NOT NULL, NAME VARCHAR(255) NOT NULL, namespaceuri TEXT, owner_id BIGINT, PRIMARY KEY (ID));
+CREATE INDEX INDEX_METADATABLOCK_name ON METADATABLOCK (name);
+CREATE INDEX INDEX_METADATABLOCK_owner_id ON METADATABLOCK (owner_id);
+CREATE TABLE CONFIRMEMAILDATA (ID SERIAL NOT NULL, CREATED TIMESTAMP NOT NULL, EXPIRES TIMESTAMP NOT NULL, TOKEN VARCHAR(255), AUTHENTICATEDUSER_ID BIGINT NOT NULL UNIQUE, PRIMARY KEY (ID));
+CREATE INDEX INDEX_CONFIRMEMAILDATA_token ON CONFIRMEMAILDATA (token);
+CREATE INDEX INDEX_CONFIRMEMAILDATA_authenticateduser_id ON CONFIRMEMAILDATA (authenticateduser_id);
+CREATE TABLE OAUTH2TOKENDATA (ID SERIAL NOT NULL, ACCESSTOKEN TEXT, EXPIRYDATE TIMESTAMP, OAUTHPROVIDERID VARCHAR(255), RAWRESPONSE TEXT, REFRESHTOKEN VARCHAR(64), SCOPE VARCHAR(64), TOKENTYPE VARCHAR(32), USER_ID BIGINT, PRIMARY KEY (ID));
+CREATE TABLE DVOBJECT (ID SERIAL NOT NULL, DTYPE VARCHAR(31), AUTHORITY VARCHAR(255), CREATEDATE TIMESTAMP NOT NULL, GLOBALIDCREATETIME TIMESTAMP, IDENTIFIER VARCHAR(255), IDENTIFIERREGISTERED BOOLEAN, INDEXTIME TIMESTAMP, MODIFICATIONTIME TIMESTAMP NOT NULL, PERMISSIONINDEXTIME TIMESTAMP, PERMISSIONMODIFICATIONTIME TIMESTAMP, PREVIEWIMAGEAVAILABLE BOOLEAN, PROTOCOL VARCHAR(255), PUBLICATIONDATE TIMESTAMP, STORAGEIDENTIFIER VARCHAR(255), CREATOR_ID BIGINT, OWNER_ID BIGINT, RELEASEUSER_ID BIGINT, PRIMARY KEY (ID));
+CREATE INDEX INDEX_DVOBJECT_dtype ON DVOBJECT (dtype);
+CREATE INDEX INDEX_DVOBJECT_owner_id ON DVOBJECT (owner_id);
+CREATE INDEX INDEX_DVOBJECT_creator_id ON DVOBJECT (creator_id);
+CREATE INDEX INDEX_DVOBJECT_releaseuser_id ON DVOBJECT (releaseuser_id);
+CREATE TABLE DATAVERSE (ID BIGINT NOT NULL, AFFILIATION VARCHAR(255), ALIAS VARCHAR(255) NOT NULL UNIQUE, DATAVERSETYPE VARCHAR(255) NOT NULL, description TEXT, FACETROOT BOOLEAN, GUESTBOOKROOT BOOLEAN, METADATABLOCKROOT BOOLEAN, NAME VARCHAR(255) NOT NULL, PERMISSIONROOT BOOLEAN, TEMPLATEROOT BOOLEAN, THEMEROOT BOOLEAN, DEFAULTCONTRIBUTORROLE_ID BIGINT, DEFAULTTEMPLATE_ID BIGINT, PRIMARY KEY (ID));
+CREATE INDEX INDEX_DATAVERSE_defaultcontributorrole_id ON DATAVERSE (defaultcontributorrole_id);
+CREATE INDEX INDEX_DATAVERSE_defaulttemplate_id ON DATAVERSE (defaulttemplate_id);
+CREATE INDEX INDEX_DATAVERSE_alias ON DATAVERSE (alias);
+CREATE INDEX INDEX_DATAVERSE_affiliation ON DATAVERSE (affiliation);
+CREATE INDEX INDEX_DATAVERSE_dataversetype ON DATAVERSE (dataversetype);
+CREATE INDEX INDEX_DATAVERSE_facetroot ON DATAVERSE (facetroot);
+CREATE INDEX INDEX_DATAVERSE_guestbookroot ON DATAVERSE (guestbookroot);
+CREATE INDEX INDEX_DATAVERSE_metadatablockroot ON DATAVERSE (metadatablockroot);
+CREATE INDEX INDEX_DATAVERSE_templateroot ON DATAVERSE (templateroot);
+CREATE INDEX INDEX_DATAVERSE_permissionroot ON DATAVERSE (permissionroot);
+CREATE INDEX INDEX_DATAVERSE_themeroot ON DATAVERSE (themeroot);
+CREATE TABLE IPV6RANGE (ID BIGINT NOT NULL, BOTTOMA BIGINT, BOTTOMB BIGINT, BOTTOMC BIGINT, BOTTOMD BIGINT, TOPA BIGINT, TOPB BIGINT, TOPC BIGINT, TOPD BIGINT, OWNER_ID BIGINT, PRIMARY KEY (ID));
+CREATE INDEX INDEX_IPV6RANGE_owner_id ON IPV6RANGE (owner_id);
+CREATE TABLE SAVEDSEARCHFILTERQUERY (ID SERIAL NOT NULL, FILTERQUERY TEXT, SAVEDSEARCH_ID BIGINT NOT NULL, PRIMARY KEY (ID));
+CREATE INDEX INDEX_SAVEDSEARCHFILTERQUERY_savedsearch_id ON SAVEDSEARCHFILTERQUERY (savedsearch_id);
+CREATE TABLE STORAGESITE (ID SERIAL NOT NULL, hostname TEXT, name TEXT, PRIMARYSTORAGE BOOLEAN NOT NULL, transferProtocols TEXT, PRIMARY KEY (ID));
+CREATE TABLE DATAVERSEFACET (ID SERIAL NOT NULL, DISPLAYORDER INTEGER, datasetfieldtype_id BIGINT, dataverse_id BIGINT, PRIMARY KEY (ID));
+CREATE INDEX INDEX_DATAVERSEFACET_dataverse_id ON DATAVERSEFACET (dataverse_id);
+CREATE INDEX INDEX_DATAVERSEFACET_datasetfieldtype_id ON DATAVERSEFACET (datasetfieldtype_id);
+CREATE INDEX INDEX_DATAVERSEFACET_displayorder ON DATAVERSEFACET (displayorder);
+CREATE TABLE OAIRECORD (ID SERIAL NOT NULL, GLOBALID VARCHAR(255), LASTUPDATETIME TIMESTAMP, REMOVED BOOLEAN, SETNAME VARCHAR(255), PRIMARY KEY (ID));
+CREATE TABLE DATAVERSEFEATUREDDATAVERSE (ID SERIAL NOT NULL, DISPLAYORDER INTEGER, dataverse_id BIGINT, featureddataverse_id BIGINT, PRIMARY KEY (ID));
+CREATE INDEX INDEX_DATAVERSEFEATUREDDATAVERSE_dataverse_id ON DATAVERSEFEATUREDDATAVERSE (dataverse_id);
+CREATE INDEX INDEX_DATAVERSEFEATUREDDATAVERSE_featureddataverse_id ON DATAVERSEFEATUREDDATAVERSE (featureddataverse_id);
+CREATE INDEX INDEX_DATAVERSEFEATUREDDATAVERSE_displayorder ON DATAVERSEFEATUREDDATAVERSE (displayorder);
+CREATE TABLE HARVESTINGCLIENT (ID SERIAL NOT NULL, ARCHIVEDESCRIPTION TEXT, ARCHIVEURL VARCHAR(255), DELETED BOOLEAN, HARVESTSTYLE VARCHAR(255), HARVESTTYPE VARCHAR(255), HARVESTINGNOW BOOLEAN, HARVESTINGSET VARCHAR(255), HARVESTINGURL VARCHAR(255), METADATAPREFIX VARCHAR(255), NAME VARCHAR(255) NOT NULL UNIQUE, SCHEDULEDAYOFWEEK INTEGER, SCHEDULEHOUROFDAY INTEGER, SCHEDULEPERIOD VARCHAR(255), SCHEDULED BOOLEAN, dataverse_id BIGINT, PRIMARY KEY (ID));
+CREATE INDEX INDEX_HARVESTINGCLIENT_dataverse_id ON HARVESTINGCLIENT (dataverse_id);
+CREATE INDEX INDEX_HARVESTINGCLIENT_harvesttype ON HARVESTINGCLIENT (harvesttype);
+CREATE INDEX INDEX_HARVESTINGCLIENT_harveststyle ON HARVESTINGCLIENT (harveststyle);
+CREATE INDEX INDEX_HARVESTINGCLIENT_harvestingurl ON HARVESTINGCLIENT (harvestingurl);
+CREATE TABLE APITOKEN (ID SERIAL NOT NULL, CREATETIME TIMESTAMP NOT NULL, DISABLED BOOLEAN NOT NULL, EXPIRETIME TIMESTAMP NOT NULL, TOKENSTRING VARCHAR(255) NOT NULL UNIQUE, AUTHENTICATEDUSER_ID BIGINT NOT NULL, PRIMARY KEY (ID));
+CREATE INDEX INDEX_APITOKEN_authenticateduser_id ON APITOKEN (authenticateduser_id);
+CREATE TABLE DATASETFIELDVALUE (ID SERIAL NOT NULL, DISPLAYORDER INTEGER, value TEXT, DATASETFIELD_ID BIGINT NOT NULL, PRIMARY KEY (ID));
+CREATE INDEX INDEX_DATASETFIELDVALUE_datasetfield_id ON DATASETFIELDVALUE (datasetfield_id);
+CREATE TABLE CUSTOMQUESTION (ID SERIAL NOT NULL, DISPLAYORDER INTEGER, HIDDEN BOOLEAN, QUESTIONSTRING VARCHAR(255) NOT NULL, QUESTIONTYPE VARCHAR(255) NOT NULL, REQUIRED BOOLEAN, GUESTBOOK_ID BIGINT NOT NULL, PRIMARY KEY (ID));
+CREATE INDEX INDEX_CUSTOMQUESTION_guestbook_id ON CUSTOMQUESTION (guestbook_id);
+CREATE TABLE VARIABLERANGEITEM (ID SERIAL NOT NULL, VALUE DECIMAL(38), DATAVARIABLE_ID BIGINT NOT NULL, PRIMARY KEY (ID));
+CREATE INDEX INDEX_VARIABLERANGEITEM_datavariable_id ON VARIABLERANGEITEM (datavariable_id);
+CREATE TABLE VARIABLERANGE (ID SERIAL NOT NULL, BEGINVALUE VARCHAR(255), BEGINVALUETYPE INTEGER, ENDVALUE VARCHAR(255), ENDVALUETYPE INTEGER, DATAVARIABLE_ID BIGINT NOT NULL, PRIMARY KEY (ID));
+CREATE INDEX INDEX_VARIABLERANGE_datavariable_id ON VARIABLERANGE (datavariable_id);
+CREATE TABLE SHIBGROUP (ID SERIAL NOT NULL, ATTRIBUTE VARCHAR(255) NOT NULL, NAME VARCHAR(255) NOT NULL, PATTERN VARCHAR(255) NOT NULL, PRIMARY KEY (ID));
+CREATE TABLE WORKFLOWCOMMENT (ID SERIAL NOT NULL, CREATED TIMESTAMP NOT NULL, MESSAGE TEXT, TYPE VARCHAR(255) NOT NULL, AUTHENTICATEDUSER_ID BIGINT, DATASETVERSION_ID BIGINT NOT NULL, PRIMARY KEY (ID));
+CREATE TABLE DATASETFIELD (ID SERIAL NOT NULL, DATASETFIELDTYPE_ID BIGINT NOT NULL, DATASETVERSION_ID BIGINT, PARENTDATASETFIELDCOMPOUNDVALUE_ID BIGINT, TEMPLATE_ID BIGINT, PRIMARY KEY (ID));
+CREATE INDEX INDEX_DATASETFIELD_datasetfieldtype_id ON DATASETFIELD (datasetfieldtype_id);
+CREATE INDEX INDEX_DATASETFIELD_datasetversion_id ON DATASETFIELD (datasetversion_id);
+CREATE INDEX INDEX_DATASETFIELD_parentdatasetfieldcompoundvalue_id ON DATASETFIELD (parentdatasetfieldcompoundvalue_id);
+CREATE INDEX INDEX_DATASETFIELD_template_id ON DATASETFIELD (template_id);
+CREATE TABLE PERSISTEDGLOBALGROUP (ID BIGINT NOT NULL, DTYPE VARCHAR(31), DESCRIPTION VARCHAR(255), DISPLAYNAME VARCHAR(255), PERSISTEDGROUPALIAS VARCHAR(255) UNIQUE, PRIMARY KEY (ID));
+CREATE INDEX INDEX_PERSISTEDGLOBALGROUP_dtype ON PERSISTEDGLOBALGROUP (dtype);
+CREATE TABLE IPV4RANGE (ID BIGINT NOT NULL, BOTTOMASLONG BIGINT, TOPASLONG BIGINT, OWNER_ID BIGINT, PRIMARY KEY (ID));
+CREATE INDEX INDEX_IPV4RANGE_owner_id ON IPV4RANGE (owner_id);
+CREATE TABLE DATASETVERSION (ID SERIAL NOT NULL, UNF VARCHAR(255), ARCHIVALCOPYLOCATION TEXT, ARCHIVENOTE VARCHAR(1000), ARCHIVETIME TIMESTAMP, CREATETIME TIMESTAMP NOT NULL, DEACCESSIONLINK VARCHAR(255), LASTUPDATETIME TIMESTAMP NOT NULL, MINORVERSIONNUMBER BIGINT, RELEASETIME TIMESTAMP, VERSION BIGINT, VERSIONNOTE VARCHAR(1000), VERSIONNUMBER BIGINT, VERSIONSTATE VARCHAR(255), DATASET_ID BIGINT, termsOfUseAndAccess_id BIGINT, PRIMARY KEY (ID));
+CREATE INDEX INDEX_DATASETVERSION_dataset_id ON DATASETVERSION (dataset_id);
+CREATE TABLE METRIC (ID SERIAL NOT NULL, DATALOCATION TEXT, DAYSTRING TEXT, LASTCALLEDDATE TIMESTAMP NOT NULL, NAME VARCHAR(255) NOT NULL, VALUEJSON TEXT, PRIMARY KEY (ID));
+CREATE INDEX INDEX_METRIC_id ON METRIC (id);
+CREATE TABLE USERNOTIFICATION (ID SERIAL NOT NULL, EMAILED BOOLEAN, OBJECTID BIGINT, READNOTIFICATION BOOLEAN, SENDDATE TIMESTAMP, TYPE INTEGER NOT NULL, REQUESTOR_ID BIGINT, USER_ID BIGINT NOT NULL, PRIMARY KEY (ID));
+CREATE INDEX INDEX_USERNOTIFICATION_user_id ON USERNOTIFICATION (user_id);
+CREATE TABLE WORKFLOWSTEPDATA (ID SERIAL NOT NULL, PROVIDERID VARCHAR(255), STEPTYPE VARCHAR(255), PARENT_ID BIGINT, index INTEGER, PRIMARY KEY (ID));
+CREATE TABLE CUSTOMFIELDMAP (ID SERIAL NOT NULL, SOURCEDATASETFIELD VARCHAR(255), SOURCETEMPLATE VARCHAR(255), TARGETDATASETFIELD VARCHAR(255), PRIMARY KEY (ID));
+CREATE INDEX INDEX_CUSTOMFIELDMAP_sourcedatasetfield ON CUSTOMFIELDMAP (sourcedatasetfield);
+CREATE INDEX INDEX_CUSTOMFIELDMAP_sourcetemplate ON CUSTOMFIELDMAP (sourcetemplate);
+CREATE TABLE GUESTBOOK (ID SERIAL NOT NULL, CREATETIME TIMESTAMP NOT NULL, EMAILREQUIRED BOOLEAN, ENABLED BOOLEAN, INSTITUTIONREQUIRED BOOLEAN, NAME VARCHAR(255), NAMEREQUIRED BOOLEAN, POSITIONREQUIRED BOOLEAN, DATAVERSE_ID BIGINT, PRIMARY KEY (ID));
+CREATE TABLE ACTIONLOGRECORD (ID VARCHAR(36) NOT NULL, ACTIONRESULT VARCHAR(255), ACTIONSUBTYPE VARCHAR(255), ACTIONTYPE VARCHAR(255), ENDTIME TIMESTAMP, INFO TEXT, STARTTIME TIMESTAMP, USERIDENTIFIER VARCHAR(255), PRIMARY KEY (ID));
+CREATE INDEX INDEX_ACTIONLOGRECORD_useridentifier ON ACTIONLOGRECORD (useridentifier);
+CREATE INDEX INDEX_ACTIONLOGRECORD_actiontype ON ACTIONLOGRECORD (actiontype);
+CREATE INDEX INDEX_ACTIONLOGRECORD_starttime ON ACTIONLOGRECORD (starttime);
+CREATE TABLE MAPLAYERMETADATA (ID SERIAL NOT NULL, EMBEDMAPLINK VARCHAR(255) NOT NULL, ISJOINLAYER BOOLEAN, JOINDESCRIPTION TEXT, LASTVERIFIEDSTATUS INTEGER, LASTVERIFIEDTIME TIMESTAMP, LAYERLINK VARCHAR(255) NOT NULL, LAYERNAME VARCHAR(255) NOT NULL, MAPIMAGELINK VARCHAR(255), MAPLAYERLINKS TEXT, WORLDMAPUSERNAME VARCHAR(255) NOT NULL, DATASET_ID BIGINT NOT NULL, DATAFILE_ID BIGINT NOT NULL UNIQUE, PRIMARY KEY (ID));
+CREATE INDEX INDEX_MAPLAYERMETADATA_dataset_id ON MAPLAYERMETADATA (dataset_id);
+CREATE TABLE SAVEDSEARCH (ID SERIAL NOT NULL, QUERY TEXT, CREATOR_ID BIGINT NOT NULL, DEFINITIONPOINT_ID BIGINT NOT NULL, PRIMARY KEY (ID));
+CREATE INDEX INDEX_SAVEDSEARCH_definitionpoint_id ON SAVEDSEARCH (definitionpoint_id);
+CREATE INDEX INDEX_SAVEDSEARCH_creator_id ON SAVEDSEARCH (creator_id);
+CREATE TABLE EXPLICITGROUP (ID SERIAL NOT NULL, DESCRIPTION VARCHAR(1024), DISPLAYNAME VARCHAR(255), GROUPALIAS VARCHAR(255) UNIQUE, GROUPALIASINOWNER VARCHAR(255), OWNER_ID BIGINT, PRIMARY KEY (ID));
+CREATE INDEX INDEX_EXPLICITGROUP_owner_id ON EXPLICITGROUP (owner_id);
+CREATE INDEX INDEX_EXPLICITGROUP_groupaliasinowner ON EXPLICITGROUP (groupaliasinowner);
+CREATE TABLE FOREIGNMETADATAFORMATMAPPING (ID SERIAL NOT NULL, DISPLAYNAME VARCHAR(255) NOT NULL, NAME VARCHAR(255) NOT NULL, SCHEMALOCATION VARCHAR(255), STARTELEMENT VARCHAR(255), PRIMARY KEY (ID));
+CREATE INDEX INDEX_FOREIGNMETADATAFORMATMAPPING_name ON FOREIGNMETADATAFORMATMAPPING (name);
+CREATE TABLE EXTERNALTOOL (ID SERIAL NOT NULL, CONTENTTYPE TEXT, DESCRIPTION TEXT, DISPLAYNAME VARCHAR(255) NOT NULL, TOOLPARAMETERS VARCHAR(255) NOT NULL, TOOLURL VARCHAR(255) NOT NULL, TYPE VARCHAR(255) NOT NULL, PRIMARY KEY (ID));
+CREATE TABLE DATASETFIELDDEFAULTVALUE (ID SERIAL NOT NULL, DISPLAYORDER INTEGER, STRVALUE TEXT, DATASETFIELD_ID BIGINT NOT NULL, DEFAULTVALUESET_ID BIGINT NOT NULL, PARENTDATASETFIELDDEFAULTVALUE_ID BIGINT, PRIMARY KEY (ID));
+CREATE INDEX INDEX_DATASETFIELDDEFAULTVALUE_datasetfield_id ON DATASETFIELDDEFAULTVALUE (datasetfield_id);
+CREATE INDEX INDEX_DATASETFIELDDEFAULTVALUE_defaultvalueset_id ON DATASETFIELDDEFAULTVALUE (defaultvalueset_id);
+CREATE INDEX INDEX_DATASETFIELDDEFAULTVALUE_parentdatasetfielddefaultvalue_id ON DATASETFIELDDEFAULTVALUE (parentdatasetfielddefaultvalue_id);
+CREATE INDEX INDEX_DATASETFIELDDEFAULTVALUE_displayorder ON DATASETFIELDDEFAULTVALUE (displayorder);
+CREATE TABLE PENDINGWORKFLOWINVOCATION (INVOCATIONID VARCHAR(255) NOT NULL, DATASETEXTERNALLYRELEASED BOOLEAN, IPADDRESS VARCHAR(255), NEXTMINORVERSIONNUMBER BIGINT, NEXTVERSIONNUMBER BIGINT, PENDINGSTEPIDX INTEGER, TYPEORDINAL INTEGER, USERID VARCHAR(255), WORKFLOW_ID BIGINT, DATASET_ID BIGINT, PRIMARY KEY (INVOCATIONID));
+CREATE TABLE DEFAULTVALUESET (ID SERIAL NOT NULL, NAME VARCHAR(255) NOT NULL, PRIMARY KEY (ID));
+CREATE TABLE AUTHENTICATEDUSER (ID SERIAL NOT NULL, AFFILIATION VARCHAR(255), CREATEDTIME TIMESTAMP NOT NULL, EMAIL VARCHAR(255) NOT NULL UNIQUE, EMAILCONFIRMED TIMESTAMP, FIRSTNAME VARCHAR(255), LASTAPIUSETIME TIMESTAMP, LASTLOGINTIME TIMESTAMP, LASTNAME VARCHAR(255), POSITION VARCHAR(255), SUPERUSER BOOLEAN, USERIDENTIFIER VARCHAR(255) NOT NULL UNIQUE, PRIMARY KEY (ID));
+CREATE TABLE DATATABLE (ID SERIAL NOT NULL, CASEQUANTITY BIGINT, ORIGINALFILEFORMAT VARCHAR(255), ORIGINALFILESIZE BIGINT, ORIGINALFORMATVERSION VARCHAR(255), RECORDSPERCASE BIGINT, UNF VARCHAR(255) NOT NULL, VARQUANTITY BIGINT, DATAFILE_ID BIGINT NOT NULL, PRIMARY KEY (ID));
+CREATE INDEX INDEX_DATATABLE_datafile_id ON DATATABLE (datafile_id);
+CREATE TABLE INGESTREPORT (ID SERIAL NOT NULL, ENDTIME TIMESTAMP, REPORT TEXT, STARTTIME TIMESTAMP, STATUS INTEGER, TYPE INTEGER, DATAFILE_ID BIGINT NOT NULL, PRIMARY KEY (ID));
+CREATE INDEX INDEX_INGESTREPORT_datafile_id ON INGESTREPORT (datafile_id);
+CREATE TABLE AUTHENTICATIONPROVIDERROW (ID VARCHAR(255) NOT NULL, ENABLED BOOLEAN, FACTORYALIAS VARCHAR(255), FACTORYDATA TEXT, SUBTITLE VARCHAR(255), TITLE VARCHAR(255), PRIMARY KEY (ID));
+CREATE INDEX INDEX_AUTHENTICATIONPROVIDERROW_enabled ON AUTHENTICATIONPROVIDERROW (enabled);
+CREATE TABLE FOREIGNMETADATAFIELDMAPPING (ID SERIAL NOT NULL, datasetfieldName TEXT, foreignFieldXPath TEXT, ISATTRIBUTE BOOLEAN, FOREIGNMETADATAFORMATMAPPING_ID BIGINT, PARENTFIELDMAPPING_ID BIGINT, PRIMARY KEY (ID));
+CREATE INDEX INDEX_FOREIGNMETADATAFIELDMAPPING_foreignmetadataformatmapping_id ON FOREIGNMETADATAFIELDMAPPING (foreignmetadataformatmapping_id);
+CREATE INDEX INDEX_FOREIGNMETADATAFIELDMAPPING_foreignfieldxpath ON FOREIGNMETADATAFIELDMAPPING (foreignfieldxpath);
+CREATE INDEX INDEX_FOREIGNMETADATAFIELDMAPPING_parentfieldmapping_id ON FOREIGNMETADATAFIELDMAPPING (parentfieldmapping_id);
+CREATE TABLE FILEMETADATA (ID SERIAL NOT NULL, DESCRIPTION TEXT, DIRECTORYLABEL VARCHAR(255), LABEL VARCHAR(255) NOT NULL, prov_freeform TEXT, RESTRICTED BOOLEAN, VERSION BIGINT, DATAFILE_ID BIGINT NOT NULL, DATASETVERSION_ID BIGINT NOT NULL, PRIMARY KEY (ID));
+CREATE INDEX INDEX_FILEMETADATA_datafile_id ON FILEMETADATA (datafile_id);
+CREATE INDEX INDEX_FILEMETADATA_datasetversion_id ON FILEMETADATA (datasetversion_id);
+CREATE TABLE CUSTOMQUESTIONVALUE (ID SERIAL NOT NULL, DISPLAYORDER INTEGER, VALUESTRING VARCHAR(255) NOT NULL, CUSTOMQUESTION_ID BIGINT NOT NULL, PRIMARY KEY (ID));
+CREATE TABLE SUMMARYSTATISTIC (ID SERIAL NOT NULL, TYPE INTEGER, VALUE VARCHAR(255), DATAVARIABLE_ID BIGINT NOT NULL, PRIMARY KEY (ID));
+CREATE INDEX INDEX_SUMMARYSTATISTIC_datavariable_id ON SUMMARYSTATISTIC (datavariable_id);
+CREATE TABLE worldmapauth_token (ID SERIAL NOT NULL, CREATED TIMESTAMP NOT NULL, HASEXPIRED BOOLEAN NOT NULL, LASTREFRESHTIME TIMESTAMP NOT NULL, MODIFIED TIMESTAMP NOT NULL, TOKEN VARCHAR(255), APPLICATION_ID BIGINT NOT NULL, DATAFILE_ID BIGINT NOT NULL, DATAVERSEUSER_ID BIGINT NOT NULL, PRIMARY KEY (ID));
+CREATE UNIQUE INDEX token_value ON worldmapauth_token (token);
+CREATE INDEX INDEX_worldmapauth_token_application_id ON worldmapauth_token (application_id);
+CREATE INDEX INDEX_worldmapauth_token_datafile_id ON worldmapauth_token (datafile_id);
+CREATE INDEX INDEX_worldmapauth_token_dataverseuser_id ON worldmapauth_token (dataverseuser_id);
+CREATE TABLE PASSWORDRESETDATA (ID SERIAL NOT NULL, CREATED TIMESTAMP NOT NULL, EXPIRES TIMESTAMP NOT NULL, REASON VARCHAR(255), TOKEN VARCHAR(255), BUILTINUSER_ID BIGINT NOT NULL, PRIMARY KEY (ID));
+CREATE INDEX INDEX_PASSWORDRESETDATA_token ON PASSWORDRESETDATA (token);
+CREATE INDEX INDEX_PASSWORDRESETDATA_builtinuser_id ON PASSWORDRESETDATA (builtinuser_id);
+CREATE TABLE CONTROLLEDVOCABULARYVALUE (ID SERIAL NOT NULL, DISPLAYORDER INTEGER, IDENTIFIER VARCHAR(255), STRVALUE TEXT, DATASETFIELDTYPE_ID BIGINT, PRIMARY KEY (ID));
+CREATE INDEX INDEX_CONTROLLEDVOCABULARYVALUE_datasetfieldtype_id ON CONTROLLEDVOCABULARYVALUE (datasetfieldtype_id);
+CREATE INDEX INDEX_CONTROLLEDVOCABULARYVALUE_displayorder ON CONTROLLEDVOCABULARYVALUE (displayorder);
+CREATE TABLE DATASETLINKINGDATAVERSE (ID SERIAL NOT NULL, LINKCREATETIME TIMESTAMP NOT NULL, DATASET_ID BIGINT NOT NULL, LINKINGDATAVERSE_ID BIGINT NOT NULL, PRIMARY KEY (ID));
+CREATE INDEX INDEX_DATASETLINKINGDATAVERSE_dataset_id ON DATASETLINKINGDATAVERSE (dataset_id);
+CREATE INDEX INDEX_DATASETLINKINGDATAVERSE_linkingDataverse_id ON DATASETLINKINGDATAVERSE (linkingDataverse_id);
+CREATE TABLE DATASET (ID BIGINT NOT NULL, FILEACCESSREQUEST BOOLEAN, HARVESTIDENTIFIER VARCHAR(255), LASTEXPORTTIME TIMESTAMP, USEGENERICTHUMBNAIL BOOLEAN, citationDateDatasetFieldType_id BIGINT, harvestingClient_id BIGINT, guestbook_id BIGINT, thumbnailfile_id BIGINT, PRIMARY KEY (ID));
+CREATE INDEX INDEX_DATASET_guestbook_id ON DATASET (guestbook_id);
+CREATE INDEX INDEX_DATASET_thumbnailfile_id ON DATASET (thumbnailfile_id);
+CREATE TABLE CLIENTHARVESTRUN (ID SERIAL NOT NULL, DELETEDDATASETCOUNT BIGINT, FAILEDDATASETCOUNT BIGINT, FINISHTIME TIMESTAMP, HARVESTRESULT INTEGER, HARVESTEDDATASETCOUNT BIGINT, STARTTIME TIMESTAMP, HARVESTINGCLIENT_ID BIGINT NOT NULL, PRIMARY KEY (ID));
+CREATE TABLE worldmapauth_tokentype (ID SERIAL NOT NULL, CONTACTEMAIL VARCHAR(255), CREATED TIMESTAMP NOT NULL, HOSTNAME VARCHAR(255), IPADDRESS VARCHAR(255), MAPITLINK VARCHAR(255) NOT NULL, MD5 VARCHAR(255) NOT NULL, MODIFIED TIMESTAMP NOT NULL, NAME VARCHAR(255) NOT NULL, timeLimitMinutes int default 30, timeLimitSeconds bigint default 1800, PRIMARY KEY (ID));
+CREATE UNIQUE INDEX application_name ON worldmapauth_tokentype (name);
+CREATE TABLE DATAFILETAG (ID SERIAL NOT NULL, TYPE INTEGER NOT NULL, DATAFILE_ID BIGINT NOT NULL, PRIMARY KEY (ID));
+CREATE INDEX INDEX_DATAFILETAG_datafile_id ON DATAFILETAG (datafile_id);
+CREATE TABLE AUTHENTICATEDUSERLOOKUP (ID SERIAL NOT NULL, AUTHENTICATIONPROVIDERID VARCHAR(255), PERSISTENTUSERID VARCHAR(255), AUTHENTICATEDUSER_ID BIGINT NOT NULL UNIQUE, PRIMARY KEY (ID));
+CREATE TABLE INGESTREQUEST (ID SERIAL NOT NULL, CONTROLCARD VARCHAR(255), FORCETYPECHECK BOOLEAN, LABELSFILE VARCHAR(255), TEXTENCODING VARCHAR(255), datafile_id BIGINT, PRIMARY KEY (ID));
+CREATE INDEX INDEX_INGESTREQUEST_datafile_id ON INGESTREQUEST (datafile_id);
+CREATE TABLE SETTING (NAME VARCHAR(255) NOT NULL, CONTENT TEXT, PRIMARY KEY (NAME));
+CREATE TABLE DATAVERSECONTACT (ID SERIAL NOT NULL, CONTACTEMAIL VARCHAR(255) NOT NULL, DISPLAYORDER INTEGER, dataverse_id BIGINT, PRIMARY KEY (ID));
+CREATE INDEX INDEX_DATAVERSECONTACT_dataverse_id ON DATAVERSECONTACT (dataverse_id);
+CREATE INDEX INDEX_DATAVERSECONTACT_contactemail ON DATAVERSECONTACT (contactemail);
+CREATE INDEX INDEX_DATAVERSECONTACT_displayorder ON DATAVERSECONTACT (displayorder);
+CREATE TABLE VARIABLECATEGORY (ID SERIAL NOT NULL, CATORDER INTEGER, FREQUENCY FLOAT, LABEL VARCHAR(255), MISSING BOOLEAN, VALUE VARCHAR(255), DATAVARIABLE_ID BIGINT NOT NULL, PRIMARY KEY (ID));
+CREATE INDEX INDEX_VARIABLECATEGORY_datavariable_id ON VARIABLECATEGORY (datavariable_id);
+CREATE TABLE DATAVARIABLE (ID SERIAL NOT NULL, FACTOR BOOLEAN, FILEENDPOSITION BIGINT, FILEORDER INTEGER, FILESTARTPOSITION BIGINT, FORMAT VARCHAR(255), FORMATCATEGORY VARCHAR(255), INTERVAL INTEGER, LABEL TEXT, NAME VARCHAR(255), NUMBEROFDECIMALPOINTS BIGINT, ORDEREDFACTOR BOOLEAN, RECORDSEGMENTNUMBER BIGINT, TYPE INTEGER, UNF VARCHAR(255), UNIVERSE VARCHAR(255), WEIGHTED BOOLEAN, DATATABLE_ID BIGINT NOT NULL, PRIMARY KEY (ID));
+CREATE INDEX INDEX_DATAVARIABLE_datatable_id ON DATAVARIABLE (datatable_id);
+CREATE TABLE CONTROLLEDVOCABALTERNATE (ID SERIAL NOT NULL, STRVALUE TEXT, CONTROLLEDVOCABULARYVALUE_ID BIGINT NOT NULL, DATASETFIELDTYPE_ID BIGINT NOT NULL, PRIMARY KEY (ID));
+CREATE INDEX INDEX_CONTROLLEDVOCABALTERNATE_controlledvocabularyvalue_id ON CONTROLLEDVOCABALTERNATE (controlledvocabularyvalue_id);
+CREATE INDEX INDEX_CONTROLLEDVOCABALTERNATE_datasetfieldtype_id ON CONTROLLEDVOCABALTERNATE (datasetfieldtype_id);
+CREATE TABLE DATAFILE (ID BIGINT NOT NULL, CHECKSUMTYPE VARCHAR(255) NOT NULL, CHECKSUMVALUE VARCHAR(255) NOT NULL, CONTENTTYPE VARCHAR(255) NOT NULL, FILESIZE BIGINT, INGESTSTATUS CHAR(1), PREVIOUSDATAFILEID BIGINT, prov_entityname TEXT, RESTRICTED BOOLEAN, ROOTDATAFILEID BIGINT NOT NULL, PRIMARY KEY (ID));
+CREATE INDEX INDEX_DATAFILE_ingeststatus ON DATAFILE (ingeststatus);
+CREATE INDEX INDEX_DATAFILE_checksumvalue ON DATAFILE (checksumvalue);
+CREATE INDEX INDEX_DATAFILE_contenttype ON DATAFILE (contenttype);
+CREATE INDEX INDEX_DATAFILE_restricted ON DATAFILE (restricted);
+CREATE TABLE DataverseFieldTypeInputLevel (ID SERIAL NOT NULL, INCLUDE BOOLEAN, REQUIRED BOOLEAN, datasetfieldtype_id BIGINT, dataverse_id BIGINT, PRIMARY KEY (ID));
+CREATE INDEX INDEX_DataverseFieldTypeInputLevel_dataverse_id ON DataverseFieldTypeInputLevel (dataverse_id);
+CREATE INDEX INDEX_DataverseFieldTypeInputLevel_datasetfieldtype_id ON DataverseFieldTypeInputLevel (datasetfieldtype_id);
+CREATE INDEX INDEX_DataverseFieldTypeInputLevel_required ON DataverseFieldTypeInputLevel (required);
+CREATE TABLE BUILTINUSER (ID SERIAL NOT NULL, ENCRYPTEDPASSWORD VARCHAR(255), PASSWORDENCRYPTIONVERSION INTEGER, USERNAME VARCHAR(255) NOT NULL UNIQUE, PRIMARY KEY (ID));
+CREATE INDEX INDEX_BUILTINUSER_userName ON BUILTINUSER (userName);
+CREATE TABLE TERMSOFUSEANDACCESS (ID SERIAL NOT NULL, AVAILABILITYSTATUS TEXT, CITATIONREQUIREMENTS TEXT, CONDITIONS TEXT, CONFIDENTIALITYDECLARATION TEXT, CONTACTFORACCESS TEXT, DATAACCESSPLACE TEXT, DEPOSITORREQUIREMENTS TEXT, DISCLAIMER TEXT, FILEACCESSREQUEST BOOLEAN, LICENSE VARCHAR(255), ORIGINALARCHIVE TEXT, RESTRICTIONS TEXT, SIZEOFCOLLECTION TEXT, SPECIALPERMISSIONS TEXT, STUDYCOMPLETION TEXT, TERMSOFACCESS TEXT, TERMSOFUSE TEXT, PRIMARY KEY (ID));
+CREATE TABLE DOIDATACITEREGISTERCACHE (ID SERIAL NOT NULL, DOI VARCHAR(255) UNIQUE, STATUS VARCHAR(255), URL VARCHAR(255), XML TEXT, PRIMARY KEY (ID));
+CREATE TABLE HARVESTINGDATAVERSECONFIG (ID BIGINT NOT NULL, ARCHIVEDESCRIPTION TEXT, ARCHIVEURL VARCHAR(255), HARVESTSTYLE VARCHAR(255), HARVESTTYPE VARCHAR(255), HARVESTINGSET VARCHAR(255), HARVESTINGURL VARCHAR(255), dataverse_id BIGINT, PRIMARY KEY (ID));
+CREATE INDEX INDEX_HARVESTINGDATAVERSECONFIG_dataverse_id ON HARVESTINGDATAVERSECONFIG (dataverse_id);
+CREATE INDEX INDEX_HARVESTINGDATAVERSECONFIG_harvesttype ON HARVESTINGDATAVERSECONFIG (harvesttype);
+CREATE INDEX INDEX_HARVESTINGDATAVERSECONFIG_harveststyle ON HARVESTINGDATAVERSECONFIG (harveststyle);
+CREATE INDEX INDEX_HARVESTINGDATAVERSECONFIG_harvestingurl ON HARVESTINGDATAVERSECONFIG (harvestingurl);
+CREATE TABLE ALTERNATIVEPERSISTENTIDENTIFIER (ID SERIAL NOT NULL, AUTHORITY VARCHAR(255), GLOBALIDCREATETIME TIMESTAMP, IDENTIFIER VARCHAR(255), IDENTIFIERREGISTERED BOOLEAN, PROTOCOL VARCHAR(255), STORAGELOCATIONDESIGNATOR BOOLEAN, DVOBJECT_ID BIGINT NOT NULL, PRIMARY KEY (ID));
+CREATE TABLE DATASETFIELDCOMPOUNDVALUE (ID SERIAL NOT NULL, DISPLAYORDER INTEGER, PARENTDATASETFIELD_ID BIGINT, PRIMARY KEY (ID));
+CREATE INDEX INDEX_DATASETFIELDCOMPOUNDVALUE_parentdatasetfield_id ON DATASETFIELDCOMPOUNDVALUE (parentdatasetfield_id);
+CREATE TABLE DATASETVERSIONUSER (ID SERIAL NOT NULL, LASTUPDATEDATE TIMESTAMP NOT NULL, authenticatedUser_id BIGINT, datasetversion_id BIGINT, PRIMARY KEY (ID));
+CREATE INDEX INDEX_DATASETVERSIONUSER_authenticateduser_id ON DATASETVERSIONUSER (authenticateduser_id);
+CREATE INDEX INDEX_DATASETVERSIONUSER_datasetversion_id ON DATASETVERSIONUSER (datasetversion_id);
+CREATE TABLE GUESTBOOKRESPONSE (ID SERIAL NOT NULL, DOWNLOADTYPE VARCHAR(255), EMAIL VARCHAR(255), INSTITUTION VARCHAR(255), NAME VARCHAR(255), POSITION VARCHAR(255), RESPONSETIME TIMESTAMP, SESSIONID VARCHAR(255), AUTHENTICATEDUSER_ID BIGINT, DATAFILE_ID BIGINT NOT NULL, DATASET_ID BIGINT NOT NULL, DATASETVERSION_ID BIGINT, GUESTBOOK_ID BIGINT NOT NULL, PRIMARY KEY (ID));
+CREATE INDEX INDEX_GUESTBOOKRESPONSE_guestbook_id ON GUESTBOOKRESPONSE (guestbook_id);
+CREATE INDEX INDEX_GUESTBOOKRESPONSE_datafile_id ON GUESTBOOKRESPONSE (datafile_id);
+CREATE INDEX INDEX_GUESTBOOKRESPONSE_dataset_id ON GUESTBOOKRESPONSE (dataset_id);
+CREATE TABLE CUSTOMQUESTIONRESPONSE (ID SERIAL NOT NULL, response TEXT, CUSTOMQUESTION_ID BIGINT NOT NULL, GUESTBOOKRESPONSE_ID BIGINT NOT NULL, PRIMARY KEY (ID));
+CREATE INDEX INDEX_CUSTOMQUESTIONRESPONSE_guestbookresponse_id ON CUSTOMQUESTIONRESPONSE (guestbookresponse_id);
+CREATE TABLE TEMPLATE (ID SERIAL NOT NULL, CREATETIME TIMESTAMP NOT NULL, NAME VARCHAR(255) NOT NULL, USAGECOUNT BIGINT, DATAVERSE_ID BIGINT, termsOfUseAndAccess_id BIGINT, PRIMARY KEY (ID));
+CREATE INDEX INDEX_TEMPLATE_dataverse_id ON TEMPLATE (dataverse_id);
+CREATE TABLE DATASETLOCK (ID SERIAL NOT NULL, INFO VARCHAR(255), REASON VARCHAR(255) NOT NULL, STARTTIME TIMESTAMP, DATASET_ID BIGINT NOT NULL, USER_ID BIGINT NOT NULL, PRIMARY KEY (ID));
+CREATE INDEX INDEX_DATASETLOCK_user_id ON DATASETLOCK (user_id);
+CREATE INDEX INDEX_DATASETLOCK_dataset_id ON DATASETLOCK (dataset_id);
+CREATE TABLE DATAVERSEROLE (ID SERIAL NOT NULL, ALIAS VARCHAR(255) NOT NULL UNIQUE, DESCRIPTION VARCHAR(255), NAME VARCHAR(255) NOT NULL, PERMISSIONBITS BIGINT, OWNER_ID BIGINT, PRIMARY KEY (ID));
+CREATE INDEX INDEX_DATAVERSEROLE_owner_id ON DATAVERSEROLE (owner_id);
+CREATE INDEX INDEX_DATAVERSEROLE_name ON DATAVERSEROLE (name);
+CREATE INDEX INDEX_DATAVERSEROLE_alias ON DATAVERSEROLE (alias);
+CREATE TABLE DATASETFIELDTYPE (ID SERIAL NOT NULL, ADVANCEDSEARCHFIELDTYPE BOOLEAN, ALLOWCONTROLLEDVOCABULARY BOOLEAN, ALLOWMULTIPLES BOOLEAN, description TEXT, DISPLAYFORMAT VARCHAR(255), DISPLAYONCREATE BOOLEAN, DISPLAYORDER INTEGER, FACETABLE BOOLEAN, FIELDTYPE VARCHAR(255) NOT NULL, name TEXT, REQUIRED BOOLEAN, title TEXT, uri TEXT, VALIDATIONFORMAT VARCHAR(255), WATERMARK VARCHAR(255), METADATABLOCK_ID BIGINT, PARENTDATASETFIELDTYPE_ID BIGINT, PRIMARY KEY (ID));
+CREATE INDEX INDEX_DATASETFIELDTYPE_metadatablock_id ON DATASETFIELDTYPE (metadatablock_id);
+CREATE INDEX INDEX_DATASETFIELDTYPE_parentdatasetfieldtype_id ON DATASETFIELDTYPE (parentdatasetfieldtype_id);
+CREATE TABLE FILEMETADATA_DATAFILECATEGORY (fileCategories_ID BIGINT NOT NULL, fileMetadatas_ID BIGINT NOT NULL, PRIMARY KEY (fileCategories_ID, fileMetadatas_ID));
+CREATE INDEX INDEX_FILEMETADATA_DATAFILECATEGORY_filecategories_id ON FILEMETADATA_DATAFILECATEGORY (filecategories_id);
+CREATE INDEX INDEX_FILEMETADATA_DATAFILECATEGORY_filemetadatas_id ON FILEMETADATA_DATAFILECATEGORY (filemetadatas_id);
+CREATE TABLE dataverse_citationDatasetFieldTypes (dataverse_id BIGINT NOT NULL, citationdatasetfieldtype_id BIGINT NOT NULL, PRIMARY KEY (dataverse_id, citationdatasetfieldtype_id));
+CREATE TABLE dataversesubjects (dataverse_id BIGINT NOT NULL, controlledvocabularyvalue_id BIGINT NOT NULL, PRIMARY KEY (dataverse_id, controlledvocabularyvalue_id));
+CREATE TABLE DATAVERSE_METADATABLOCK (Dataverse_ID BIGINT NOT NULL, metadataBlocks_ID BIGINT NOT NULL, PRIMARY KEY (Dataverse_ID, metadataBlocks_ID));
+CREATE TABLE DATASETFIELD_CONTROLLEDVOCABULARYVALUE (DatasetField_ID BIGINT NOT NULL, controlledVocabularyValues_ID BIGINT NOT NULL, PRIMARY KEY (DatasetField_ID, controlledVocabularyValues_ID));
+CREATE INDEX INDEX_DATASETFIELD_CONTROLLEDVOCABULARYVALUE_datasetfield_id ON DATASETFIELD_CONTROLLEDVOCABULARYVALUE (datasetfield_id);
+CREATE INDEX INDEX_DATASETFIELD_CONTROLLEDVOCABULARYVALUE_controlledvocabularyvalues_id ON DATASETFIELD_CONTROLLEDVOCABULARYVALUE (controlledvocabularyvalues_id);
+CREATE TABLE WorkflowStepData_STEPPARAMETERS (WorkflowStepData_ID BIGINT, STEPPARAMETERS VARCHAR(2048), STEPPARAMETERS_KEY VARCHAR(255));
+CREATE TABLE WorkflowStepData_STEPSETTINGS (WorkflowStepData_ID BIGINT, STEPSETTINGS VARCHAR(2048), STEPSETTINGS_KEY VARCHAR(255));
+CREATE TABLE ExplicitGroup_CONTAINEDROLEASSIGNEES (ExplicitGroup_ID BIGINT, CONTAINEDROLEASSIGNEES VARCHAR(255));
+CREATE TABLE EXPLICITGROUP_AUTHENTICATEDUSER (ExplicitGroup_ID BIGINT NOT NULL, containedAuthenticatedUsers_ID BIGINT NOT NULL, PRIMARY KEY (ExplicitGroup_ID, containedAuthenticatedUsers_ID));
+CREATE TABLE explicitgroup_explicitgroup (explicitgroup_id BIGINT NOT NULL, containedexplicitgroups_id BIGINT NOT NULL, PRIMARY KEY (explicitgroup_id, containedexplicitgroups_id));
+CREATE TABLE PendingWorkflowInvocation_LOCALDATA (PendingWorkflowInvocation_INVOCATIONID VARCHAR(255), LOCALDATA VARCHAR(255), LOCALDATA_KEY VARCHAR(255));
+CREATE TABLE fileaccessrequests (datafile_id BIGINT NOT NULL, authenticated_user_id BIGINT NOT NULL, PRIMARY KEY (datafile_id, authenticated_user_id));
+ALTER TABLE ROLEASSIGNMENT ADD CONSTRAINT UNQ_ROLEASSIGNMENT_0 UNIQUE (assigneeIdentifier, role_id, definitionPoint_id);
+ALTER TABLE DVOBJECT ADD CONSTRAINT UNQ_DVOBJECT_0 UNIQUE (authority,protocol,identifier);
+ALTER TABLE DATASETVERSION ADD CONSTRAINT UNQ_DATASETVERSION_0 UNIQUE (dataset_id,versionnumber,minorversionnumber);
+ALTER TABLE FOREIGNMETADATAFIELDMAPPING ADD CONSTRAINT UNQ_FOREIGNMETADATAFIELDMAPPING_0 UNIQUE (foreignMetadataFormatMapping_id, foreignFieldXpath);
+ALTER TABLE AUTHENTICATEDUSERLOOKUP ADD CONSTRAINT UNQ_AUTHENTICATEDUSERLOOKUP_0 UNIQUE (persistentuserid, authenticationproviderid);
+ALTER TABLE DataverseFieldTypeInputLevel ADD CONSTRAINT UNQ_DataverseFieldTypeInputLevel_0 UNIQUE (dataverse_id, datasetfieldtype_id);
+ALTER TABLE DATAVERSETHEME ADD CONSTRAINT FK_DATAVERSETHEME_dataverse_id FOREIGN KEY (dataverse_id) REFERENCES DVOBJECT (ID);
+ALTER TABLE DATAFILECATEGORY ADD CONSTRAINT FK_DATAFILECATEGORY_DATASET_ID FOREIGN KEY (DATASET_ID) REFERENCES DVOBJECT (ID);
+ALTER TABLE ROLEASSIGNMENT ADD CONSTRAINT FK_ROLEASSIGNMENT_ROLE_ID FOREIGN KEY (ROLE_ID) REFERENCES DATAVERSEROLE (ID);
+ALTER TABLE ROLEASSIGNMENT ADD CONSTRAINT FK_ROLEASSIGNMENT_DEFINITIONPOINT_ID FOREIGN KEY (DEFINITIONPOINT_ID) REFERENCES DVOBJECT (ID);
+ALTER TABLE DATAVERSELINKINGDATAVERSE ADD CONSTRAINT FK_DATAVERSELINKINGDATAVERSE_DATAVERSE_ID FOREIGN KEY (DATAVERSE_ID) REFERENCES DVOBJECT (ID);
+ALTER TABLE DATAVERSELINKINGDATAVERSE ADD CONSTRAINT FK_DATAVERSELINKINGDATAVERSE_LINKINGDATAVERSE_ID FOREIGN KEY (LINKINGDATAVERSE_ID) REFERENCES DVOBJECT (ID);
+ALTER TABLE METADATABLOCK ADD CONSTRAINT FK_METADATABLOCK_owner_id FOREIGN KEY (owner_id) REFERENCES DVOBJECT (ID);
+ALTER TABLE CONFIRMEMAILDATA ADD CONSTRAINT FK_CONFIRMEMAILDATA_AUTHENTICATEDUSER_ID FOREIGN KEY (AUTHENTICATEDUSER_ID) REFERENCES AUTHENTICATEDUSER (ID);
+ALTER TABLE OAUTH2TOKENDATA ADD CONSTRAINT FK_OAUTH2TOKENDATA_USER_ID FOREIGN KEY (USER_ID) REFERENCES AUTHENTICATEDUSER (ID);
+ALTER TABLE DVOBJECT ADD CONSTRAINT FK_DVOBJECT_CREATOR_ID FOREIGN KEY (CREATOR_ID) REFERENCES AUTHENTICATEDUSER (ID);
+ALTER TABLE DVOBJECT ADD CONSTRAINT FK_DVOBJECT_RELEASEUSER_ID FOREIGN KEY (RELEASEUSER_ID) REFERENCES AUTHENTICATEDUSER (ID);
+ALTER TABLE DVOBJECT ADD CONSTRAINT FK_DVOBJECT_OWNER_ID FOREIGN KEY (OWNER_ID) REFERENCES DVOBJECT (ID);
+ALTER TABLE DATAVERSE ADD CONSTRAINT FK_DATAVERSE_DEFAULTTEMPLATE_ID FOREIGN KEY (DEFAULTTEMPLATE_ID) REFERENCES TEMPLATE (ID);
+ALTER TABLE DATAVERSE ADD CONSTRAINT FK_DATAVERSE_ID FOREIGN KEY (ID) REFERENCES DVOBJECT (ID);
+ALTER TABLE DATAVERSE ADD CONSTRAINT FK_DATAVERSE_DEFAULTCONTRIBUTORROLE_ID FOREIGN KEY (DEFAULTCONTRIBUTORROLE_ID) REFERENCES DATAVERSEROLE (ID);
+ALTER TABLE IPV6RANGE ADD CONSTRAINT FK_IPV6RANGE_OWNER_ID FOREIGN KEY (OWNER_ID) REFERENCES PERSISTEDGLOBALGROUP (ID);
+ALTER TABLE SAVEDSEARCHFILTERQUERY ADD CONSTRAINT FK_SAVEDSEARCHFILTERQUERY_SAVEDSEARCH_ID FOREIGN KEY (SAVEDSEARCH_ID) REFERENCES SAVEDSEARCH (ID);
+ALTER TABLE DATAVERSEFACET ADD CONSTRAINT FK_DATAVERSEFACET_datasetfieldtype_id FOREIGN KEY (datasetfieldtype_id) REFERENCES DATASETFIELDTYPE (ID);
+ALTER TABLE DATAVERSEFACET ADD CONSTRAINT FK_DATAVERSEFACET_dataverse_id FOREIGN KEY (dataverse_id) REFERENCES DVOBJECT (ID);
+ALTER TABLE DATAVERSEFEATUREDDATAVERSE ADD CONSTRAINT FK_DATAVERSEFEATUREDDATAVERSE_dataverse_id FOREIGN KEY (dataverse_id) REFERENCES DVOBJECT (ID);
+ALTER TABLE DATAVERSEFEATUREDDATAVERSE ADD CONSTRAINT FK_DATAVERSEFEATUREDDATAVERSE_featureddataverse_id FOREIGN KEY (featureddataverse_id) REFERENCES DVOBJECT (ID);
+ALTER TABLE HARVESTINGCLIENT ADD CONSTRAINT FK_HARVESTINGCLIENT_dataverse_id FOREIGN KEY (dataverse_id) REFERENCES DVOBJECT (ID);
+ALTER TABLE APITOKEN ADD CONSTRAINT FK_APITOKEN_AUTHENTICATEDUSER_ID FOREIGN KEY (AUTHENTICATEDUSER_ID) REFERENCES AUTHENTICATEDUSER (ID);
+ALTER TABLE DATASETFIELDVALUE ADD CONSTRAINT FK_DATASETFIELDVALUE_DATASETFIELD_ID FOREIGN KEY (DATASETFIELD_ID) REFERENCES DATASETFIELD (ID);
+ALTER TABLE CUSTOMQUESTION ADD CONSTRAINT FK_CUSTOMQUESTION_GUESTBOOK_ID FOREIGN KEY (GUESTBOOK_ID) REFERENCES GUESTBOOK (ID);
+ALTER TABLE VARIABLERANGEITEM ADD CONSTRAINT FK_VARIABLERANGEITEM_DATAVARIABLE_ID FOREIGN KEY (DATAVARIABLE_ID) REFERENCES DATAVARIABLE (ID);
+ALTER TABLE VARIABLERANGE ADD CONSTRAINT FK_VARIABLERANGE_DATAVARIABLE_ID FOREIGN KEY (DATAVARIABLE_ID) REFERENCES DATAVARIABLE (ID);
+ALTER TABLE WORKFLOWCOMMENT ADD CONSTRAINT FK_WORKFLOWCOMMENT_AUTHENTICATEDUSER_ID FOREIGN KEY (AUTHENTICATEDUSER_ID) REFERENCES AUTHENTICATEDUSER (ID);
+ALTER TABLE WORKFLOWCOMMENT ADD CONSTRAINT FK_WORKFLOWCOMMENT_DATASETVERSION_ID FOREIGN KEY (DATASETVERSION_ID) REFERENCES DATASETVERSION (ID);
+ALTER TABLE DATASETFIELD ADD CONSTRAINT FK_DATASETFIELD_DATASETFIELDTYPE_ID FOREIGN KEY (DATASETFIELDTYPE_ID) REFERENCES DATASETFIELDTYPE (ID);
+ALTER TABLE DATASETFIELD ADD CONSTRAINT FK_DATASETFIELD_TEMPLATE_ID FOREIGN KEY (TEMPLATE_ID) REFERENCES TEMPLATE (ID);
+ALTER TABLE DATASETFIELD ADD CONSTRAINT FK_DATASETFIELD_DATASETVERSION_ID FOREIGN KEY (DATASETVERSION_ID) REFERENCES DATASETVERSION (ID);
+ALTER TABLE DATASETFIELD ADD CONSTRAINT FK_DATASETFIELD_PARENTDATASETFIELDCOMPOUNDVALUE_ID FOREIGN KEY (PARENTDATASETFIELDCOMPOUNDVALUE_ID) REFERENCES DATASETFIELDCOMPOUNDVALUE (ID);
+ALTER TABLE IPV4RANGE ADD CONSTRAINT FK_IPV4RANGE_OWNER_ID FOREIGN KEY (OWNER_ID) REFERENCES PERSISTEDGLOBALGROUP (ID);
+ALTER TABLE DATASETVERSION ADD CONSTRAINT FK_DATASETVERSION_termsOfUseAndAccess_id FOREIGN KEY (termsOfUseAndAccess_id) REFERENCES TERMSOFUSEANDACCESS (ID);
+ALTER TABLE DATASETVERSION ADD CONSTRAINT FK_DATASETVERSION_DATASET_ID FOREIGN KEY (DATASET_ID) REFERENCES DVOBJECT (ID);
+ALTER TABLE USERNOTIFICATION ADD CONSTRAINT FK_USERNOTIFICATION_USER_ID FOREIGN KEY (USER_ID) REFERENCES AUTHENTICATEDUSER (ID);
+ALTER TABLE USERNOTIFICATION ADD CONSTRAINT FK_USERNOTIFICATION_REQUESTOR_ID FOREIGN KEY (REQUESTOR_ID) REFERENCES AUTHENTICATEDUSER (ID);
+ALTER TABLE WORKFLOWSTEPDATA ADD CONSTRAINT FK_WORKFLOWSTEPDATA_PARENT_ID FOREIGN KEY (PARENT_ID) REFERENCES WORKFLOW (ID);
+ALTER TABLE GUESTBOOK ADD CONSTRAINT FK_GUESTBOOK_DATAVERSE_ID FOREIGN KEY (DATAVERSE_ID) REFERENCES DVOBJECT (ID);
+ALTER TABLE MAPLAYERMETADATA ADD CONSTRAINT FK_MAPLAYERMETADATA_DATAFILE_ID FOREIGN KEY (DATAFILE_ID) REFERENCES DVOBJECT (ID);
+ALTER TABLE MAPLAYERMETADATA ADD CONSTRAINT FK_MAPLAYERMETADATA_DATASET_ID FOREIGN KEY (DATASET_ID) REFERENCES DVOBJECT (ID);
+ALTER TABLE SAVEDSEARCH ADD CONSTRAINT FK_SAVEDSEARCH_DEFINITIONPOINT_ID FOREIGN KEY (DEFINITIONPOINT_ID) REFERENCES DVOBJECT (ID);
+ALTER TABLE SAVEDSEARCH ADD CONSTRAINT FK_SAVEDSEARCH_CREATOR_ID FOREIGN KEY (CREATOR_ID) REFERENCES AUTHENTICATEDUSER (ID);
+ALTER TABLE EXPLICITGROUP ADD CONSTRAINT FK_EXPLICITGROUP_OWNER_ID FOREIGN KEY (OWNER_ID) REFERENCES DVOBJECT (ID);
+ALTER TABLE DATASETFIELDDEFAULTVALUE ADD CONSTRAINT FK_DATASETFIELDDEFAULTVALUE_DEFAULTVALUESET_ID FOREIGN KEY (DEFAULTVALUESET_ID) REFERENCES DEFAULTVALUESET (ID);
+ALTER TABLE DATASETFIELDDEFAULTVALUE ADD CONSTRAINT FK_DATASETFIELDDEFAULTVALUE_DATASETFIELD_ID FOREIGN KEY (DATASETFIELD_ID) REFERENCES DATASETFIELDTYPE (ID);
+ALTER TABLE DATASETFIELDDEFAULTVALUE ADD CONSTRAINT FK_DATASETFIELDDEFAULTVALUE_PARENTDATASETFIELDDEFAULTVALUE_ID FOREIGN KEY (PARENTDATASETFIELDDEFAULTVALUE_ID) REFERENCES DATASETFIELDDEFAULTVALUE (ID);
+ALTER TABLE PENDINGWORKFLOWINVOCATION ADD CONSTRAINT FK_PENDINGWORKFLOWINVOCATION_WORKFLOW_ID FOREIGN KEY (WORKFLOW_ID) REFERENCES WORKFLOW (ID);
+ALTER TABLE PENDINGWORKFLOWINVOCATION ADD CONSTRAINT FK_PENDINGWORKFLOWINVOCATION_DATASET_ID FOREIGN KEY (DATASET_ID) REFERENCES DVOBJECT (ID);
+ALTER TABLE DATATABLE ADD CONSTRAINT FK_DATATABLE_DATAFILE_ID FOREIGN KEY (DATAFILE_ID) REFERENCES DVOBJECT (ID);
+ALTER TABLE INGESTREPORT ADD CONSTRAINT FK_INGESTREPORT_DATAFILE_ID FOREIGN KEY (DATAFILE_ID) REFERENCES DVOBJECT (ID);
+ALTER TABLE FOREIGNMETADATAFIELDMAPPING ADD CONSTRAINT FK_FOREIGNMETADATAFIELDMAPPING_FOREIGNMETADATAFORMATMAPPING_ID FOREIGN KEY (FOREIGNMETADATAFORMATMAPPING_ID) REFERENCES FOREIGNMETADATAFORMATMAPPING (ID);
+ALTER TABLE FOREIGNMETADATAFIELDMAPPING ADD CONSTRAINT FK_FOREIGNMETADATAFIELDMAPPING_PARENTFIELDMAPPING_ID FOREIGN KEY (PARENTFIELDMAPPING_ID) REFERENCES FOREIGNMETADATAFIELDMAPPING (ID);
+ALTER TABLE FILEMETADATA ADD CONSTRAINT FK_FILEMETADATA_DATASETVERSION_ID FOREIGN KEY (DATASETVERSION_ID) REFERENCES DATASETVERSION (ID);
+ALTER TABLE FILEMETADATA ADD CONSTRAINT FK_FILEMETADATA_DATAFILE_ID FOREIGN KEY (DATAFILE_ID) REFERENCES DVOBJECT (ID);
+ALTER TABLE CUSTOMQUESTIONVALUE ADD CONSTRAINT FK_CUSTOMQUESTIONVALUE_CUSTOMQUESTION_ID FOREIGN KEY (CUSTOMQUESTION_ID) REFERENCES CUSTOMQUESTION (ID);
+ALTER TABLE SUMMARYSTATISTIC ADD CONSTRAINT FK_SUMMARYSTATISTIC_DATAVARIABLE_ID FOREIGN KEY (DATAVARIABLE_ID) REFERENCES DATAVARIABLE (ID);
+ALTER TABLE worldmapauth_token ADD CONSTRAINT FK_worldmapauth_token_DATAVERSEUSER_ID FOREIGN KEY (DATAVERSEUSER_ID) REFERENCES AUTHENTICATEDUSER (ID);
+ALTER TABLE worldmapauth_token ADD CONSTRAINT FK_worldmapauth_token_DATAFILE_ID FOREIGN KEY (DATAFILE_ID) REFERENCES DVOBJECT (ID);
+ALTER TABLE worldmapauth_token ADD CONSTRAINT FK_worldmapauth_token_APPLICATION_ID FOREIGN KEY (APPLICATION_ID) REFERENCES worldmapauth_tokentype (ID);
+ALTER TABLE PASSWORDRESETDATA ADD CONSTRAINT FK_PASSWORDRESETDATA_BUILTINUSER_ID FOREIGN KEY (BUILTINUSER_ID) REFERENCES BUILTINUSER (ID);
+ALTER TABLE CONTROLLEDVOCABULARYVALUE ADD CONSTRAINT FK_CONTROLLEDVOCABULARYVALUE_DATASETFIELDTYPE_ID FOREIGN KEY (DATASETFIELDTYPE_ID) REFERENCES DATASETFIELDTYPE (ID);
+ALTER TABLE DATASETLINKINGDATAVERSE ADD CONSTRAINT FK_DATASETLINKINGDATAVERSE_DATASET_ID FOREIGN KEY (DATASET_ID) REFERENCES DVOBJECT (ID);
+ALTER TABLE DATASETLINKINGDATAVERSE ADD CONSTRAINT FK_DATASETLINKINGDATAVERSE_LINKINGDATAVERSE_ID FOREIGN KEY (LINKINGDATAVERSE_ID) REFERENCES DVOBJECT (ID);
+ALTER TABLE DATASET ADD CONSTRAINT FK_DATASET_ID FOREIGN KEY (ID) REFERENCES DVOBJECT (ID);
+ALTER TABLE DATASET ADD CONSTRAINT FK_DATASET_harvestingClient_id FOREIGN KEY (harvestingClient_id) REFERENCES HARVESTINGCLIENT (ID);
+ALTER TABLE DATASET ADD CONSTRAINT FK_DATASET_guestbook_id FOREIGN KEY (guestbook_id) REFERENCES GUESTBOOK (ID);
+ALTER TABLE DATASET ADD CONSTRAINT FK_DATASET_thumbnailfile_id FOREIGN KEY (thumbnailfile_id) REFERENCES DVOBJECT (ID);
+ALTER TABLE DATASET ADD CONSTRAINT FK_DATASET_citationDateDatasetFieldType_id FOREIGN KEY (citationDateDatasetFieldType_id) REFERENCES DATASETFIELDTYPE (ID);
+ALTER TABLE CLIENTHARVESTRUN ADD CONSTRAINT FK_CLIENTHARVESTRUN_HARVESTINGCLIENT_ID FOREIGN KEY (HARVESTINGCLIENT_ID) REFERENCES HARVESTINGCLIENT (ID);
+ALTER TABLE DATAFILETAG ADD CONSTRAINT FK_DATAFILETAG_DATAFILE_ID FOREIGN KEY (DATAFILE_ID) REFERENCES DVOBJECT (ID);
+ALTER TABLE AUTHENTICATEDUSERLOOKUP ADD CONSTRAINT FK_AUTHENTICATEDUSERLOOKUP_AUTHENTICATEDUSER_ID FOREIGN KEY (AUTHENTICATEDUSER_ID) REFERENCES AUTHENTICATEDUSER (ID);
+ALTER TABLE INGESTREQUEST ADD CONSTRAINT FK_INGESTREQUEST_datafile_id FOREIGN KEY (datafile_id) REFERENCES DVOBJECT (ID);
+ALTER TABLE DATAVERSECONTACT ADD CONSTRAINT FK_DATAVERSECONTACT_dataverse_id FOREIGN KEY (dataverse_id) REFERENCES DVOBJECT (ID);
+ALTER TABLE VARIABLECATEGORY ADD CONSTRAINT FK_VARIABLECATEGORY_DATAVARIABLE_ID FOREIGN KEY (DATAVARIABLE_ID) REFERENCES DATAVARIABLE (ID);
+ALTER TABLE DATAVARIABLE ADD CONSTRAINT FK_DATAVARIABLE_DATATABLE_ID FOREIGN KEY (DATATABLE_ID) REFERENCES DATATABLE (ID);
+ALTER TABLE CONTROLLEDVOCABALTERNATE ADD CONSTRAINT FK_CONTROLLEDVOCABALTERNATE_DATASETFIELDTYPE_ID FOREIGN KEY (DATASETFIELDTYPE_ID) REFERENCES DATASETFIELDTYPE (ID);
+ALTER TABLE CONTROLLEDVOCABALTERNATE ADD CONSTRAINT FK_CONTROLLEDVOCABALTERNATE_CONTROLLEDVOCABULARYVALUE_ID FOREIGN KEY (CONTROLLEDVOCABULARYVALUE_ID) REFERENCES CONTROLLEDVOCABULARYVALUE (ID);
+ALTER TABLE DATAFILE ADD CONSTRAINT FK_DATAFILE_ID FOREIGN KEY (ID) REFERENCES DVOBJECT (ID);
+ALTER TABLE DataverseFieldTypeInputLevel ADD CONSTRAINT FK_DataverseFieldTypeInputLevel_datasetfieldtype_id FOREIGN KEY (datasetfieldtype_id) REFERENCES DATASETFIELDTYPE (ID);
+ALTER TABLE DataverseFieldTypeInputLevel ADD CONSTRAINT FK_DataverseFieldTypeInputLevel_dataverse_id FOREIGN KEY (dataverse_id) REFERENCES DVOBJECT (ID);
+ALTER TABLE HARVESTINGDATAVERSECONFIG ADD CONSTRAINT FK_HARVESTINGDATAVERSECONFIG_dataverse_id FOREIGN KEY (dataverse_id) REFERENCES DVOBJECT (ID);
+ALTER TABLE ALTERNATIVEPERSISTENTIDENTIFIER ADD CONSTRAINT FK_ALTERNATIVEPERSISTENTIDENTIFIER_DVOBJECT_ID FOREIGN KEY (DVOBJECT_ID) REFERENCES DVOBJECT (ID);
+ALTER TABLE DATASETFIELDCOMPOUNDVALUE ADD CONSTRAINT FK_DATASETFIELDCOMPOUNDVALUE_PARENTDATASETFIELD_ID FOREIGN KEY (PARENTDATASETFIELD_ID) REFERENCES DATASETFIELD (ID);
+ALTER TABLE DATASETVERSIONUSER ADD CONSTRAINT FK_DATASETVERSIONUSER_authenticatedUser_id FOREIGN KEY (authenticatedUser_id) REFERENCES AUTHENTICATEDUSER (ID);
+ALTER TABLE DATASETVERSIONUSER ADD CONSTRAINT FK_DATASETVERSIONUSER_datasetversion_id FOREIGN KEY (datasetversion_id) REFERENCES DATASETVERSION (ID);
+ALTER TABLE GUESTBOOKRESPONSE ADD CONSTRAINT FK_GUESTBOOKRESPONSE_DATAFILE_ID FOREIGN KEY (DATAFILE_ID) REFERENCES DVOBJECT (ID);
+ALTER TABLE GUESTBOOKRESPONSE ADD CONSTRAINT FK_GUESTBOOKRESPONSE_DATASETVERSION_ID FOREIGN KEY (DATASETVERSION_ID) REFERENCES DATASETVERSION (ID);
+ALTER TABLE GUESTBOOKRESPONSE ADD CONSTRAINT FK_GUESTBOOKRESPONSE_GUESTBOOK_ID FOREIGN KEY (GUESTBOOK_ID) REFERENCES GUESTBOOK (ID);
+ALTER TABLE GUESTBOOKRESPONSE ADD CONSTRAINT FK_GUESTBOOKRESPONSE_AUTHENTICATEDUSER_ID FOREIGN KEY (AUTHENTICATEDUSER_ID) REFERENCES AUTHENTICATEDUSER (ID);
+ALTER TABLE GUESTBOOKRESPONSE ADD CONSTRAINT FK_GUESTBOOKRESPONSE_DATASET_ID FOREIGN KEY (DATASET_ID) REFERENCES DVOBJECT (ID);
+ALTER TABLE CUSTOMQUESTIONRESPONSE ADD CONSTRAINT FK_CUSTOMQUESTIONRESPONSE_CUSTOMQUESTION_ID FOREIGN KEY (CUSTOMQUESTION_ID) REFERENCES CUSTOMQUESTION (ID);
+ALTER TABLE CUSTOMQUESTIONRESPONSE ADD CONSTRAINT FK_CUSTOMQUESTIONRESPONSE_GUESTBOOKRESPONSE_ID FOREIGN KEY (GUESTBOOKRESPONSE_ID) REFERENCES GUESTBOOKRESPONSE (ID);
+ALTER TABLE TEMPLATE ADD CONSTRAINT FK_TEMPLATE_DATAVERSE_ID FOREIGN KEY (DATAVERSE_ID) REFERENCES DVOBJECT (ID);
+ALTER TABLE TEMPLATE ADD CONSTRAINT FK_TEMPLATE_termsOfUseAndAccess_id FOREIGN KEY (termsOfUseAndAccess_id) REFERENCES TERMSOFUSEANDACCESS (ID);
+ALTER TABLE DATASETLOCK ADD CONSTRAINT FK_DATASETLOCK_DATASET_ID FOREIGN KEY (DATASET_ID) REFERENCES DVOBJECT (ID);
+ALTER TABLE DATASETLOCK ADD CONSTRAINT FK_DATASETLOCK_USER_ID FOREIGN KEY (USER_ID) REFERENCES AUTHENTICATEDUSER (ID);
+ALTER TABLE DATAVERSEROLE ADD CONSTRAINT FK_DATAVERSEROLE_OWNER_ID FOREIGN KEY (OWNER_ID) REFERENCES DVOBJECT (ID);
+ALTER TABLE DATASETFIELDTYPE ADD CONSTRAINT FK_DATASETFIELDTYPE_PARENTDATASETFIELDTYPE_ID FOREIGN KEY (PARENTDATASETFIELDTYPE_ID) REFERENCES DATASETFIELDTYPE (ID);
+ALTER TABLE DATASETFIELDTYPE ADD CONSTRAINT FK_DATASETFIELDTYPE_METADATABLOCK_ID FOREIGN KEY (METADATABLOCK_ID) REFERENCES METADATABLOCK (ID);
+ALTER TABLE FILEMETADATA_DATAFILECATEGORY ADD CONSTRAINT FK_FILEMETADATA_DATAFILECATEGORY_fileMetadatas_ID FOREIGN KEY (fileMetadatas_ID) REFERENCES FILEMETADATA (ID);
+ALTER TABLE FILEMETADATA_DATAFILECATEGORY ADD CONSTRAINT FK_FILEMETADATA_DATAFILECATEGORY_fileCategories_ID FOREIGN KEY (fileCategories_ID) REFERENCES DATAFILECATEGORY (ID);
+ALTER TABLE dataverse_citationDatasetFieldTypes ADD CONSTRAINT FK_dataverse_citationDatasetFieldTypes_dataverse_id FOREIGN KEY (dataverse_id) REFERENCES DVOBJECT (ID);
+ALTER TABLE dataverse_citationDatasetFieldTypes ADD CONSTRAINT dataverse_citationDatasetFieldTypes_citationdatasetfieldtype_id FOREIGN KEY (citationdatasetfieldtype_id) REFERENCES DATASETFIELDTYPE (ID);
+ALTER TABLE dataversesubjects ADD CONSTRAINT FK_dataversesubjects_dataverse_id FOREIGN KEY (dataverse_id) REFERENCES DVOBJECT (ID);
+ALTER TABLE dataversesubjects ADD CONSTRAINT FK_dataversesubjects_controlledvocabularyvalue_id FOREIGN KEY (controlledvocabularyvalue_id) REFERENCES CONTROLLEDVOCABULARYVALUE (ID);
+ALTER TABLE DATAVERSE_METADATABLOCK ADD CONSTRAINT FK_DATAVERSE_METADATABLOCK_Dataverse_ID FOREIGN KEY (Dataverse_ID) REFERENCES DVOBJECT (ID);
+ALTER TABLE DATAVERSE_METADATABLOCK ADD CONSTRAINT FK_DATAVERSE_METADATABLOCK_metadataBlocks_ID FOREIGN KEY (metadataBlocks_ID) REFERENCES METADATABLOCK (ID);
+ALTER TABLE DATASETFIELD_CONTROLLEDVOCABULARYVALUE ADD CONSTRAINT FK_DATASETFIELD_CONTROLLEDVOCABULARYVALUE_DatasetField_ID FOREIGN KEY (DatasetField_ID) REFERENCES DATASETFIELD (ID);
+ALTER TABLE DATASETFIELD_CONTROLLEDVOCABULARYVALUE ADD CONSTRAINT DTASETFIELDCONTROLLEDVOCABULARYVALUEcntrolledVocabularyValuesID FOREIGN KEY (controlledVocabularyValues_ID) REFERENCES CONTROLLEDVOCABULARYVALUE (ID);
+ALTER TABLE WorkflowStepData_STEPPARAMETERS ADD CONSTRAINT FK_WorkflowStepData_STEPPARAMETERS_WorkflowStepData_ID FOREIGN KEY (WorkflowStepData_ID) REFERENCES WORKFLOWSTEPDATA (ID);
+ALTER TABLE WorkflowStepData_STEPSETTINGS ADD CONSTRAINT FK_WorkflowStepData_STEPSETTINGS_WorkflowStepData_ID FOREIGN KEY (WorkflowStepData_ID) REFERENCES WORKFLOWSTEPDATA (ID);
+ALTER TABLE ExplicitGroup_CONTAINEDROLEASSIGNEES ADD CONSTRAINT FK_ExplicitGroup_CONTAINEDROLEASSIGNEES_ExplicitGroup_ID FOREIGN KEY (ExplicitGroup_ID) REFERENCES EXPLICITGROUP (ID);
+ALTER TABLE EXPLICITGROUP_AUTHENTICATEDUSER ADD CONSTRAINT FK_EXPLICITGROUP_AUTHENTICATEDUSER_ExplicitGroup_ID FOREIGN KEY (ExplicitGroup_ID) REFERENCES EXPLICITGROUP (ID);
+ALTER TABLE EXPLICITGROUP_AUTHENTICATEDUSER ADD CONSTRAINT EXPLICITGROUP_AUTHENTICATEDUSER_containedAuthenticatedUsers_ID FOREIGN KEY (containedAuthenticatedUsers_ID) REFERENCES AUTHENTICATEDUSER (ID);
+ALTER TABLE explicitgroup_explicitgroup ADD CONSTRAINT FK_explicitgroup_explicitgroup_explicitgroup_id FOREIGN KEY (explicitgroup_id) REFERENCES EXPLICITGROUP (ID);
+ALTER TABLE explicitgroup_explicitgroup ADD CONSTRAINT FK_explicitgroup_explicitgroup_containedexplicitgroups_id FOREIGN KEY (containedexplicitgroups_id) REFERENCES EXPLICITGROUP (ID);
+ALTER TABLE PendingWorkflowInvocation_LOCALDATA ADD CONSTRAINT PndngWrkflwInvocationLOCALDATAPndngWrkflwInvocationINVOCATIONID FOREIGN KEY (PendingWorkflowInvocation_INVOCATIONID) REFERENCES PENDINGWORKFLOWINVOCATION (INVOCATIONID);
+ALTER TABLE fileaccessrequests ADD CONSTRAINT FK_fileaccessrequests_datafile_id FOREIGN KEY (datafile_id) REFERENCES DVOBJECT (ID);
+ALTER TABLE fileaccessrequests ADD CONSTRAINT FK_fileaccessrequests_authenticated_user_id FOREIGN KEY (authenticated_user_id) REFERENCES AUTHENTICATEDUSER (ID);
+CREATE TABLE SEQUENCE (SEQ_NAME VARCHAR(50) NOT NULL, SEQ_COUNT DECIMAL(38), PRIMARY KEY (SEQ_NAME));
+INSERT INTO SEQUENCE(SEQ_NAME, SEQ_COUNT) values ('SEQ_GEN', 0);
diff --git a/scripts/database/releases.txt b/scripts/database/releases.txt
index fd4bda9c2c2..074d1f7275a 100644
--- a/scripts/database/releases.txt
+++ b/scripts/database/releases.txt
@@ -30,3 +30,4 @@ v4.9.3
v4.9.4
v4.10
v4.10.1
+v4.11
diff --git a/scripts/database/upgrades/upgrade_v4.10.1_to_4.11.sql b/scripts/database/upgrades/upgrade_v4.10.1_to_4.11.sql
new file mode 100644
index 00000000000..bac06924f71
--- /dev/null
+++ b/scripts/database/upgrades/upgrade_v4.10.1_to_4.11.sql
@@ -0,0 +1,8 @@
+ALTER TABLE datasetversion ADD COLUMN archivalcopylocation text;
+ALTER TABLE externaltool ADD COLUMN contenttype text NOT NULL default 'text/tab-separated-values';
+TRUNCATE metric;
+ALTER TABLE metric ADD COLUMN dayString text;
+ALTER TABLE metric ADD COLUMN dataLocation text;
+ALTER TABLE metric DROP CONSTRAINT "metric_metricname_key";
+ALTER TABLE metric RENAME COLUMN metricValue TO valueJson;
+ALTER TABLE metric RENAME COLUMN metricName TO name;
\ No newline at end of file
diff --git a/scripts/database/upgrades/upgrade_v4.8.6_to_v4.9.sql b/scripts/database/upgrades/upgrade_v4.8.6_to_v4.9.sql
index 1158b2942a8..5335c9ae270 100644
--- a/scripts/database/upgrades/upgrade_v4.8.6_to_v4.9.sql
+++ b/scripts/database/upgrades/upgrade_v4.8.6_to_v4.9.sql
@@ -62,12 +62,11 @@ ALTER TABLE dvobject DROP COLUMN doiseparator;
--Add new setting into content for shoulder
INSERT INTO setting(name, content)
-VALUES (':Shoulder', (SELECT substring(content, strpos(content,'/')+1) || '/' from setting where name = ':Authority'));
+SELECT ':Shoulder', substring(content, strpos(content,'/')+1) || '/' from setting where name = ':Authority' and strpos(content,'/')>0;
- --strip shoulder from authority setting
- UPDATE setting
- SET content=(SELECT substring(content from 0 for strpos(content,'/'))
- FROM setting
- WHERE name=':Authority' and strpos(content,'/')>0) where name=':Authority';
+--strip shoulder from authority setting if the shoulder exists
+UPDATE setting
+SET content= case when (strpos(content,'/')>0) then substring(content from 0 for strpos(content,'/'))
+else content end where name=':Authority';
update datasetfieldtype set displayformat = '#VALUE' where name in ('alternativeURL', 'keywordVocabularyURI', 'topicClassVocabURI', 'publicationURL', 'producerURL', 'distributorURL');
diff --git a/scripts/installer/Makefile b/scripts/installer/Makefile
index 9c265bb8a1d..62caf35bd05 100644
--- a/scripts/installer/Makefile
+++ b/scripts/installer/Makefile
@@ -70,12 +70,12 @@ ${JHOVE_SCHEMA}: ../../conf/jhove/jhoveConfig.xsd
@mkdir -p ${INSTALLER_ZIP_DIR}
/bin/cp ../../conf/jhove/jhoveConfig.xsd ${INSTALLER_ZIP_DIR}
-${SOLR_SCHEMA}: ../../conf/solr/7.3.0/schema.xml
+${SOLR_SCHEMA}: ../../conf/solr/7.3.1/schema.xml
@echo copying Solr schema file
@mkdir -p ${INSTALLER_ZIP_DIR}
- /bin/cp ../../conf/solr/7.3.0/schema.xml ${INSTALLER_ZIP_DIR}
+ /bin/cp ../../conf/solr/7.3.1/schema.xml ${INSTALLER_ZIP_DIR}
-${SOLR_CONFIG}: ../../conf/solr/7.3.0/solrconfig.xml
+${SOLR_CONFIG}: ../../conf/solr/7.3.1/solrconfig.xml
@echo copying Solr config file
@mkdir -p ${INSTALLER_ZIP_DIR}
- /bin/cp ../../conf/solr/7.3.0/solrconfig.xml ${INSTALLER_ZIP_DIR}
+ /bin/cp ../../conf/solr/7.3.1/solrconfig.xml ${INSTALLER_ZIP_DIR}
diff --git a/scripts/installer/README.txt b/scripts/installer/README.txt
index d402582ed34..b694c36b7ba 100644
--- a/scripts/installer/README.txt
+++ b/scripts/installer/README.txt
@@ -40,7 +40,7 @@ from conf/jhove:
jhove.conf
-SOLR schema and config files, from conf/solr/7.3.0:
+SOLR schema and config files, from conf/solr/7.3.1:
schema.xml
solrconfig.xml
diff --git a/scripts/installer/ec2-create-instance.sh b/scripts/installer/ec2-create-instance.sh
old mode 100644
new mode 100755
index c494ce2be8e..4d83ce2b6e1
--- a/scripts/installer/ec2-create-instance.sh
+++ b/scripts/installer/ec2-create-instance.sh
@@ -7,14 +7,14 @@ REPO_URL='https://github.com/IQSS/dataverse.git'
BRANCH='develop'
usage() {
- echo "Usage: $0 -b -r -e " 1>&2
+ echo "Usage: $0 -b -r -g " 1>&2
echo "default branch is develop"
echo "default repo is https://github.com/IQSS/dataverse"
- echo "default conf file is ~/.dataverse/ec2.env"
+ echo "example group_vars may be retrieved from https://raw.githubusercontent.com/IQSS/dataverse-ansible/master/defaults/main.yml"
exit 1
}
-while getopts ":r:b:e:" o; do
+while getopts ":r:b:g:" o; do
case "${o}" in
r)
REPO_URL=${OPTARG}
@@ -22,8 +22,8 @@ while getopts ":r:b:e:" o; do
b)
BRANCH=${OPTARG}
;;
- e)
- EC2ENV=${OPTARG}
+ g)
+ GRPVRS=${OPTARG}
;;
*)
usage
@@ -32,35 +32,21 @@ while getopts ":r:b:e:" o; do
done
# test for user-supplied conf files
-if [ ! -z "$EC2ENV" ]; then
- CONF=$EC2ENV
-elif [ -f ~/.dataverse/ec2.env ]; then
- echo "using environment variables specified in ~/.dataverse/ec2.env."
- echo "override with -e "
- CONF="$HOME/.dataverse/ec2.env"
-else
- echo "no conf file supplied (-e ) or found at ~/.dataverse/ec2.env."
- echo "running script with defaults. this may or may not be what you want."
+if [ ! -z "$GRPVRS" ]; then
+ GVFILE=$(basename "$GRPVRS")
+ GVARG="-e @$GVFILE"
+ echo "using $GRPVRS for extra vars"
fi
-
-# read environment variables from conf file
-if [ ! -z "$CONF" ];then
- set -a
- echo "reading $CONF"
- source $CONF
- set +a
+
+if [ ! -z "$REPO_URL" ]; then
+ GVARG+=" -e dataverse_repo=$REPO_URL"
+ echo "using $REPO_URL"
fi
-# now build extra-vars string from doi_* env variables
-NL=$'\n'
-extra_vars="dataverse_branch=$BRANCH dataverse_repo=$REPO_URL"
-while IFS='=' read -r name value; do
- if [[ $name == *'doi_'* ]]; then
- extra_var="$name"=${!name}
- extra_var=${extra_var%$NL}
- extra_vars="$extra_vars $extra_var"
- fi
-done < <(env)
+if [ ! -z "$BRANCH" ]; then
+ GVARG+=" -e dataverse_branch=$BRANCH"
+ echo "building $BRANCH"
+fi
AWS_CLI_VERSION=$(aws --version)
if [[ "$?" -ne 0 ]]; then
@@ -113,8 +99,8 @@ echo "Creating EC2 instance"
# TODO: Add some error checking for "ec2 run-instances".
INSTANCE_ID=$(aws ec2 run-instances --image-id $AMI_ID --security-groups $SECURITY_GROUP --count 1 --instance-type $SIZE --key-name $KEY_NAME --query 'Instances[0].InstanceId' --block-device-mappings '[ { "DeviceName": "/dev/sda1", "Ebs": { "DeleteOnTermination": true } } ]' | tr -d \")
echo "Instance ID: "$INSTANCE_ID
-echo "giving instance 15 seconds to wake up..."
-sleep 15
+echo "giving instance 30 seconds to wake up..."
+sleep 30
echo "End creating EC2 instance"
PUBLIC_DNS=$(aws ec2 describe-instances --instance-ids $INSTANCE_ID --query "Reservations[*].Instances[*].[PublicDnsName]" --output text)
@@ -126,6 +112,10 @@ echo "ssh -i $PEM_FILE $USER_AT_HOST"
echo "Please wait at least 15 minutes while the branch \"$BRANCH\" from $REPO_URL is being deployed."
+if [ ! -z "$GRPVRS" ]; then
+ scp -i $PEM_FILE -o 'StrictHostKeyChecking no' -o 'UserKnownHostsFile=/dev/null' -o 'ConnectTimeout=300' $GRPVRS $USER_AT_HOST:$GVFILE
+fi
+
# epel-release is installed first to ensure the latest ansible is installed after
# TODO: Add some error checking for this ssh command.
ssh -T -i $PEM_FILE -o 'StrictHostKeyChecking no' -o 'UserKnownHostsFile=/dev/null' -o 'ConnectTimeout=300' $USER_AT_HOST < /tmp/$1.txt
+ ./asadmin $ASADMIN_OPTS create-password-alias --passwordfile /tmp/$1.txt $1
+ rm /tmp/$1.txt
+ done
+
###
# Add the necessary JVM options:
#
@@ -65,7 +74,7 @@ function preliminary_setup()
./asadmin $ASADMIN_OPTS create-jvm-options "\-Ddataverse.rserve.host=${RSERVE_HOST}"
./asadmin $ASADMIN_OPTS create-jvm-options "\-Ddataverse.rserve.port=${RSERVE_PORT}"
./asadmin $ASADMIN_OPTS create-jvm-options "\-Ddataverse.rserve.user=${RSERVE_USER}"
- ./asadmin $ASADMIN_OPTS create-jvm-options "\-Ddataverse.rserve.password=${RSERVE_PASS}"
+ ./asadmin $ASADMIN_OPTS create-jvm-options '\-Ddataverse.rserve.password=${ALIAS=rserve_password_alias}'
# Data Deposit API options
./asadmin $ASADMIN_OPTS create-jvm-options "\-Ddataverse.fqdn=${HOST_ADDRESS}"
# password reset token timeout in minutes
@@ -76,7 +85,7 @@ function preliminary_setup()
# jvm-options use colons as separators, escape as literal
DOI_BASEURL_ESC=`echo $DOI_BASEURL | sed -e 's/:/\\\:/'`
./asadmin $ASADMIN_OPTS create-jvm-options "\-Ddoi.username=${DOI_USERNAME}"
- ./asadmin $ASADMIN_OPTS create-jvm-options "\-Ddoi.password=${DOI_PASSWORD}"
+ ./asadmin $ASADMIN_OPTS create-jvm-options '\-Ddoi.password=${ALIAS=doi_password_alias}'
./asadmin $ASADMIN_OPTS create-jvm-options "\-Ddoi.baseurlstring=$DOI_BASEURL_ESC"
./asadmin $ASADMIN_OPTS create-jvm-options "-Ddataverse.timerServer=true"
@@ -114,9 +123,11 @@ function final_setup(){
./asadmin $ASADMIN_OPTS create-jdbc-connection-pool --restype javax.sql.DataSource \
--datasourceclassname org.postgresql.ds.PGPoolingDataSource \
- --property create=true:User=$DB_USER:PortNumber=$DB_PORT:databaseName=$DB_NAME:password=$DB_PASS:ServerName=$DB_HOST \
+ --property create=true:User=$DB_USER:PortNumber=$DB_PORT:databaseName=$DB_NAME:ServerName=$DB_HOST \
dvnDbPool
+ ./asadmin $ASADMIN_OPTS set resources.jdbc-connection-pool.dvnDbPool.property.password='${ALIAS=db_password_alias}'
+
###
# Create data sources
./asadmin $ASADMIN_OPTS create-jdbc-resource --connectionpoolid dvnDbPool jdbc/VDCNetDS
diff --git a/scripts/vagrant/setup-solr.sh b/scripts/vagrant/setup-solr.sh
index a591817ce75..6bf96212fc0 100755
--- a/scripts/vagrant/setup-solr.sh
+++ b/scripts/vagrant/setup-solr.sh
@@ -4,12 +4,12 @@ SOLR_USER=solr
SOLR_HOME=/usr/local/solr
mkdir $SOLR_HOME
chown $SOLR_USER:$SOLR_USER $SOLR_HOME
-su $SOLR_USER -s /bin/sh -c "cp /downloads/solr-7.3.0.tgz $SOLR_HOME"
-su $SOLR_USER -s /bin/sh -c "cd $SOLR_HOME && tar xfz solr-7.3.0.tgz"
-su $SOLR_USER -s /bin/sh -c "cd $SOLR_HOME/solr-7.3.0/server/solr && cp -r configsets/_default . && mv _default collection1"
-su $SOLR_USER -s /bin/sh -c "cp /conf/solr/7.3.0/schema.xml $SOLR_HOME/solr-7.3.0/server/solr/collection1/conf/schema.xml"
-su $SOLR_USER -s /bin/sh -c "cp /conf/solr/7.3.0/solrconfig.xml $SOLR_HOME/solr-7.3.0/server/solr/collection1/conf/solrconfig.xml"
-su $SOLR_USER -s /bin/sh -c "cd $SOLR_HOME/solr-7.3.0 && bin/solr start && bin/solr create_core -c collection1 -d server/solr/collection1/conf/"
+su $SOLR_USER -s /bin/sh -c "cp /downloads/solr-7.3.1.tgz $SOLR_HOME"
+su $SOLR_USER -s /bin/sh -c "cd $SOLR_HOME && tar xfz solr-7.3.1.tgz"
+su $SOLR_USER -s /bin/sh -c "cd $SOLR_HOME/solr-7.3.1/server/solr && cp -r configsets/_default . && mv _default collection1"
+su $SOLR_USER -s /bin/sh -c "cp /conf/solr/7.3.1/schema.xml $SOLR_HOME/solr-7.3.1/server/solr/collection1/conf/schema.xml"
+su $SOLR_USER -s /bin/sh -c "cp /conf/solr/7.3.1/solrconfig.xml $SOLR_HOME/solr-7.3.1/server/solr/collection1/conf/solrconfig.xml"
+su $SOLR_USER -s /bin/sh -c "cd $SOLR_HOME/solr-7.3.1 && bin/solr start && bin/solr create_core -c collection1 -d server/solr/collection1/conf/"
cp /dataverse/doc/sphinx-guides/source/_static/installation/files/etc/init.d/solr /etc/init.d/solr
chmod 755 /etc/init.d/solr
/etc/init.d/solr stop
diff --git a/scripts/vagrant/setup.sh b/scripts/vagrant/setup.sh
index 807213bf40a..108ebefd529 100644
--- a/scripts/vagrant/setup.sh
+++ b/scripts/vagrant/setup.sh
@@ -40,7 +40,7 @@ echo "Ensuring Unix user '$SOLR_USER' exists"
useradd $SOLR_USER || :
DOWNLOAD_DIR='/dataverse/downloads'
GLASSFISH_ZIP="$DOWNLOAD_DIR/glassfish-4.1.zip"
-SOLR_TGZ="$DOWNLOAD_DIR/solr-7.3.0.tgz"
+SOLR_TGZ="$DOWNLOAD_DIR/solr-7.3.1.tgz"
WELD_PATCH="$DOWNLOAD_DIR/weld-osgi-bundle-2.2.10.Final-glassfish4.jar"
# The CA certificate bundle files from CentOS are ok. Glassfish's are expired.
GOOD_CACERTS='/etc/pki/ca-trust/extracted/java/cacerts'
diff --git a/src/main/java/Bundle.properties b/src/main/java/Bundle.properties
index d857d6d8026..56688b31a1d 100755
--- a/src/main/java/Bundle.properties
+++ b/src/main/java/Bundle.properties
@@ -71,6 +71,7 @@ defaultBody=Default Body
filter=Filter
# dataverse_header.xhtml
+header.noscript=Please enable JavaScript in your browser. It is required to use most of the features of Dataverse.
header.status.header=Status
header.search.title=Search all dataverses...
header.about=About
@@ -198,7 +199,8 @@ notification.import.filesystem=Dataset {1}, dataset had file checksums added via a batch job.
removeNotification=Remove Notification
groupAndRoles.manageTips=Here is where you can access and manage all the groups you belong to, and the roles you have been assigned.
-user.signup.tip=Why have a Dataverse account? To create your own dataverse and customize it, add datasets, or request access to restricted files.
+user.message.signup.label=Create Account
+user.message.signup.tip=Why have a Dataverse account? To create your own dataverse and customize it, add datasets, or request access to restricted files.
user.signup.otherLogInOptions.tip=You can also create a Dataverse account with one of our other log in options.
user.username.illegal.tip=Between 2-60 characters, and can use "a-z", "0-9", "_" for your username.
user.username=Username
@@ -390,7 +392,7 @@ dashboard.card.metadataexport.message=Dataset metadata export is only available
#harvestclients.xhtml
harvestclients.title=Manage Harvesting Clients
-harvestclients.toptip= - Harvesting can be scheduled to run at a specific time or on demand. Harvesting can be initiated here or via the REST API.
+harvestclients.toptip=Harvesting can be scheduled to run at a specific time or on demand. Harvesting can be initiated here or via the REST API.
harvestclients.noClients.label=No clients are configured.
harvestclients.noClients.why.header=What is Harvesting?
harvestclients.noClients.why.reason1=Harvesting is a process of exchanging metadata with other repositories. As a harvesting client, your Dataverse gathers metadata records from remote sources. These can be other Dataverse instances, or other archives that support OAI-PMH, the standard harvesting protocol.
@@ -478,7 +480,7 @@ harvestclients.newClientDialog.title.edit=Edit Group {0}
#harvestset.xhtml
harvestserver.title=Manage Harvesting Server
-harvestserver.toptip= - Define sets of local datasets that will be available for harvesting by remote clients.
+harvestserver.toptip=Define sets of local datasets that will be available for harvesting by remote clients.
harvestserver.service.label=OAI Server
harvestserver.service.enabled=Enabled
harvestserver.service.disabled=Disabled
@@ -1022,7 +1024,8 @@ dataset.manageTemplates.tab.action.btn.view.dialog.datasetTemplate=Dataset Templ
dataset.manageTemplates.tab.action.btn.view.dialog.datasetTemplate.title=The dataset template which prepopulates info into the form automatically.
dataset.manageTemplates.tab.action.noedit.createdin=Template created at {0}
dataset.manageTemplates.delete.usedAsDefault=This template is the default template for the following dataverse(s). It will be removed as default as well.
-dataset.manageTemplates.info.message.notEmptyTable=Create, clone, edit, view, or delete dataset templates. Create a dataset template to prefill metadata fields with standard values, such as author affiliation, to help users create datasets in this dataverse. You can also add help text directly into the metadata fields to give users more information on what to add to these metadata fields.
+dataset.message.manageTemplates.label=Manage Dataset Templates
+dataset.message.manageTemplates.message=Create a template prefilled with metadata fields standard values, such as Author Affiliation, or add instructions in the metadata fields to give depositors more information on what metadata is expected.
# metadataFragment.xhtml
@@ -1263,8 +1266,10 @@ dataset.asterisk.tip=Asterisks indicate required fields
dataset.message.uploadFiles.label=Upload Dataset Files
dataset.message.uploadFilesSingle.message=For more information about supported file formats, please refer to the User Guide.
dataset.message.uploadFilesMultiple.message=Multiple file upload/download methods are available for this dataset. Once you upload a file using one of these methods, your choice will be locked in for this dataset.
-dataset.message.editMetadata=Edit Dataset Metadata - Add more metadata about this dataset to help others easily find it.
-dataset.message.editTerms=Edit Dataset Terms - Update this dataset's terms of use.
+dataset.message.editMetadata.label=Edit Dataset Metadata
+dataset.message.editMetadata.message=Add more metadata about this dataset to help others easily find it.
+dataset.message.editTerms.label=Edit Dataset Terms
+dataset.message.editTerms.message=Add the terms of use for this dataset to explain how to access and use your data.
dataset.message.locked.editNotAllowedInReview=Dataset cannot be edited due to In Review dataset lock.
dataset.message.locked.downloadNotAllowedInReview=Dataset file(s) may not be downloaded due to In Review dataset lock.
dataset.message.locked.downloadNotAllowed=Dataset file(s) may not be downloaded due to dataset lock.
@@ -1309,6 +1314,11 @@ file.metadata.persistentId=File Persistent ID
file.metadata.persistentId.tip=The unique persistent identifier for a file, which can be a Handle or DOI in Dataverse.
dataset.versionDifferences.termsOfUseAccess=Terms of Use and Access
dataset.versionDifferences.termsOfUseAccessChanged=Terms of Use/Access Changed
+dataset.versionDifferences.metadataBlock=Metadata Block
+dataset.versionDifferences.field=Field
+dataset.versionDifferences.changed=Changed
+dataset.versionDifferences.from=From
+dataset.versionDifferences.to=To
file.viewDiffDialog.restricted=Restricted
dataset.template.tip=Changing the template will clear any fields you may have entered data into.
dataset.noTemplate.label=None
@@ -1564,9 +1574,9 @@ file.dataFilesTab.versions.description.deaccessionedReason=Deaccessioned Reason:
file.dataFilesTab.versions.description.beAccessedAt=The dataset can now be accessed at:
file.dataFilesTab.versions.viewDetails.btn=View Details
file.dataFilesTab.versions.widget.viewMoreInfo=To view more information about the versions of this dataset, and to edit it if this is your dataset, please visit the full version of this dataset at the {2}.
-file.deleteDialog.tip=Are you sure you want to delete this dataset? You cannot undelete this dataset.
+file.deleteDialog.tip=Are you sure you want to delete this dataset and all of its files? You cannot undelete this dataset.
file.deleteDialog.header=Delete Dataset
-file.deleteDraftDialog.tip=Are you sure you want to delete this draft version? You cannot undelete this draft.
+file.deleteDraftDialog.tip=Are you sure you want to delete this draft version? Files will be reverted to the most recently published version. You cannot undelete this draft.
file.deleteDraftDialog.header=Delete Draft Version
file.deleteFileDialog.tip=The file(s) will be deleted after you click on the Save Changes button on the bottom of this page.
file.deleteFileDialog.immediate=The file will be deleted after you click on the Delete button.
@@ -1589,18 +1599,24 @@ file.deaccessionDialog.leaveURL=If applicable, please leave a URL where this dat
file.deaccessionDialog.leaveURL.watermark=Optional dataset site, http://...
file.deaccessionDialog.deaccession.tip=Are you sure you want to deaccession? The selected version(s) will no longer be viewable by the public.
file.deaccessionDialog.deaccessionDataset.tip=Are you sure you want to deaccession this dataset? It will no longer be viewable by the public.
+file.deaccessionDialog.dialog.selectVersion.error=Please select version(s) for deaccessioning.
file.deaccessionDialog.dialog.selectVersion.tip=Please select version(s) for deaccessioning.
file.deaccessionDialog.dialog.selectVersion.header=Please Select Version(s)
file.deaccessionDialog.dialog.reason.tip=Please select reason for deaccessioning.
+file.deaccessionDialog.dialog.reason.error=Please select reason for deaccessioning.
file.deaccessionDialog.dialog.reason.header=Please Select Reason
file.deaccessionDialog.dialog.url.tip=Please enter valid forwarding URL.
+file.deaccessionDialog.dialog.url.error=Please enter valid forwarding URL.
file.deaccessionDialog.dialog.url.header=Invalid URL
file.deaccessionDialog.dialog.textForReason.tip=Please enter text for reason for deaccessioning.
file.deaccessionDialog.dialog.textForReason.header=Enter additional information
+file.deaccessionDialog.dialog.textForReason.error=Please enter text for reason for deaccessioning.
file.deaccessionDialog.dialog.limitChar.tip=Text for reason for deaccessioning may be no longer than 1000 characters.
+file.deaccessionDialog.dialog.limitChar.error=Text for reason for deaccessioning may be no longer than {0} characters.
file.deaccessionDialog.dialog.limitChar.header=Limit 1000 characters
file.viewDiffDialog.header=Version Differences Details
file.viewDiffDialog.dialog.warning=Please select two versions to view the differences.
+file.viewDiffDialog.notAvailable=N/A
file.viewDiffDialog.version=Version
file.viewDiffDialog.lastUpdated=Last Updated
file.viewDiffDialog.fileID=File ID
@@ -1897,6 +1913,10 @@ api.prov.error.freeformMissingJsonKey=The JSON object you send must have a key c
api.prov.error.freeformNoText=No provenance free form text available for this file.
api.prov.error.noDataFileFound=Could not find a file based on ID.
+bagit.sourceOrganization=Dataverse Installation ()
+bagit.sourceOrganizationAddress=
+bagit.sourceOrganizationEmail=
+
#Permission.java
permission.addDataverseDataverse=Add a dataverse within another dataverse
permission.deleteDataset=Delete a dataset draft
@@ -1962,6 +1982,9 @@ dataset.file.uploadWorked=upload worked
#EmailValidator.java
email.invalid=is not a valid email address.
+#URLValidator.java
+url.invalid=is not a valid URL.
+
#HarvestingClientsPage.java
harvest.start.error=Sorry, harvest could not be started for the selected harvesting client configuration (unknown server error).
harvest.delete.error=Selected harvesting client cannot be deleted; unknown exception:
@@ -1997,6 +2020,8 @@ dataset.registered=DatasetRegistered
dataset.registered.msg=Your dataset is now registered.
dataset.notlinked=DatasetNotLinked
dataset.notlinked.msg=There was a problem linking this dataset to yours:
+datasetversion.archive.success=Archival copy of Version successfully submitted
+datasetversion.archive.failure=Error in submitting an archival copy
#ThemeWidgetFragment.java
theme.validateTagline=Tagline must be at most 140 characters.
@@ -2125,6 +2150,7 @@ access.api.revokeAccess.success.for.single.file=File Downloader access has been
access.api.requestList.fileNotFound=Could not find datafile with id {0}.
access.api.requestList.noKey=You must provide a key to get list of access requests for a file.
access.api.requestList.noRequestsFound=There are no access requests for this file {0}.
+access.api.exception.metadata.not.available.for.nontabular.file=This type of metadata is only available for tabular files.
#permission
permission.AddDataverse.label=AddDataverse
diff --git a/src/main/java/Bundle_fr.properties b/src/main/java/Bundle_fr.properties
index 28e24dde98f..eb7bcc63d84 100644
--- a/src/main/java/Bundle_fr.properties
+++ b/src/main/java/Bundle_fr.properties
@@ -2199,4 +2199,4 @@ passwdVal.passwdReq.lowercase=minuscule
passwdVal.passwdReq.letter=lettre
passwdVal.passwdReq.numeral=chiffre
passwdVal.passwdReq.special=caractère spécial
-dataretrieverAPI.noMsgResultsFound=Désolé, aucun résultat n'a été trouvé.
\ No newline at end of file
+dataretrieverAPI.noMsgResultsFound=Désolé, aucun résultat n'a été trouvé.
diff --git a/src/main/java/edu/harvard/iq/dataverse/CustomizationFilesServlet.java b/src/main/java/edu/harvard/iq/dataverse/CustomizationFilesServlet.java
index faaa247920e..713d365ba0f 100644
--- a/src/main/java/edu/harvard/iq/dataverse/CustomizationFilesServlet.java
+++ b/src/main/java/edu/harvard/iq/dataverse/CustomizationFilesServlet.java
@@ -114,6 +114,11 @@ private String getFilePath(String fileTypeParam){
// Style (css)
return settingsService.getValueForKey(SettingsServiceBean.Key.StyleCustomizationFile, nonNullDefaultIfKeyNotFound);
+ } else if (fileTypeParam.equals(CustomizationConstants.fileTypeAnalytics)) {
+
+ // Analytics - appears in head
+ return settingsService.getValueForKey(SettingsServiceBean.Key.WebAnalyticsCode, nonNullDefaultIfKeyNotFound);
+
} else if (fileTypeParam.equals(CustomizationConstants.fileTypeLogo)) {
// Logo for installation - appears in header
diff --git a/src/main/java/edu/harvard/iq/dataverse/DOIDataCiteRegisterService.java b/src/main/java/edu/harvard/iq/dataverse/DOIDataCiteRegisterService.java
index 50f92f81fb5..4b1211590e8 100644
--- a/src/main/java/edu/harvard/iq/dataverse/DOIDataCiteRegisterService.java
+++ b/src/main/java/edu/harvard/iq/dataverse/DOIDataCiteRegisterService.java
@@ -23,6 +23,7 @@
import javax.persistence.EntityManager;
import javax.persistence.PersistenceContext;
import javax.persistence.TypedQuery;
+import org.apache.commons.lang.StringEscapeUtils;
import org.jsoup.Jsoup;
import org.jsoup.nodes.Document;
import org.jsoup.nodes.Element;
@@ -149,7 +150,9 @@ public static String getMetadataFromDvObject(String identifier, Map getValues() {
}
return returnList;
}
+
+ public List getRawValuesList() {
+ List returnList = new ArrayList<>();
+ if (!datasetFieldValues.isEmpty()) {
+ for (DatasetFieldValue dsfv : datasetFieldValues) {
+ returnList.add(dsfv.getUnsanitizedDisplayValue());
+ }
+ } else {
+ for (ControlledVocabularyValue cvv : controlledVocabularyValues) {
+ if (cvv != null && cvv.getStrValue() != null) {
+ returnList.add(cvv.getStrValue());
+ }
+ }
+ }
+ return returnList;
+ }
+
/**
* list of values (as opposed to display values).
* used for passing to solr for indexing
diff --git a/src/main/java/edu/harvard/iq/dataverse/DatasetFieldValue.java b/src/main/java/edu/harvard/iq/dataverse/DatasetFieldValue.java
index cf6c762f3ed..27929dd3a39 100644
--- a/src/main/java/edu/harvard/iq/dataverse/DatasetFieldValue.java
+++ b/src/main/java/edu/harvard/iq/dataverse/DatasetFieldValue.java
@@ -115,6 +115,23 @@ public String getDisplayValue() {
return retVal;
}
+ public String getUnsanitizedDisplayValue() {
+ String retVal = "";
+ if (!StringUtils.isBlank(this.getValue()) && !DatasetField.NA_VALUE.equals(this.getValue())) {
+ String format = this.datasetField.getDatasetFieldType().getDisplayFormat();
+ if (StringUtils.isBlank(format)) {
+ format = "#VALUE";
+ }
+ String value = this.getValue();
+ String displayValue = format
+ .replace("#NAME", this.datasetField.getDatasetFieldType().getTitle() == null ? "" : this.datasetField.getDatasetFieldType().getTitle())
+ .replace("#EMAIL", BundleUtil.getStringFromBundle("dataset.email.hiddenMessage"))
+ .replace("#VALUE", value);
+ retVal = displayValue;
+ }
+ return retVal;
+ }
+
public int getDisplayOrder() {
return displayOrder;
}
diff --git a/src/main/java/edu/harvard/iq/dataverse/DatasetPage.java b/src/main/java/edu/harvard/iq/dataverse/DatasetPage.java
index 2b73ad9a92a..50998e98ba4 100644
--- a/src/main/java/edu/harvard/iq/dataverse/DatasetPage.java
+++ b/src/main/java/edu/harvard/iq/dataverse/DatasetPage.java
@@ -40,6 +40,7 @@
import edu.harvard.iq.dataverse.search.SearchFilesServiceBean;
import edu.harvard.iq.dataverse.search.SortBy;
import edu.harvard.iq.dataverse.settings.SettingsServiceBean;
+import edu.harvard.iq.dataverse.util.ArchiverUtil;
import edu.harvard.iq.dataverse.util.BundleUtil;
import edu.harvard.iq.dataverse.util.FileSortFieldAndOrder;
import edu.harvard.iq.dataverse.util.FileUtil;
@@ -82,6 +83,7 @@
import java.util.logging.Level;
import edu.harvard.iq.dataverse.datasetutility.WorldMapPermissionHelper;
import edu.harvard.iq.dataverse.engine.command.exception.IllegalCommandException;
+import edu.harvard.iq.dataverse.engine.command.impl.AbstractSubmitToArchiveCommand;
import edu.harvard.iq.dataverse.engine.command.impl.CreateNewDatasetCommand;
import edu.harvard.iq.dataverse.engine.command.impl.DeleteDataFileCommand;
import edu.harvard.iq.dataverse.engine.command.impl.GetLatestPublishedDatasetVersionCommand;
@@ -94,6 +96,7 @@
import edu.harvard.iq.dataverse.externaltools.ExternalToolServiceBean;
import edu.harvard.iq.dataverse.export.SchemaDotOrgExporter;
import java.util.Collections;
+import javax.faces.component.UIComponent;
import javax.faces.component.UIInput;
import javax.faces.event.AjaxBehaviorEvent;
@@ -207,6 +210,7 @@ public enum DisplayMode {
private int selectedTabIndex;
private List newFiles = new ArrayList<>();
private DatasetVersion workingVersion;
+ private DatasetVersion clone;
private int releaseRadio = 1;
private int deaccessionRadio = 0;
private int deaccessionReasonRadio = 0;
@@ -747,8 +751,22 @@ public void updateReleasedVersions(){
setReleasedVersionTabList(resetReleasedVersionTabList());
}
+
+ public void clickDeaccessionDataset(){
+ setReleasedVersionTabList(resetReleasedVersionTabList());
+ setRenderDeaccessionPopup(true);
+ }
+
+ private boolean renderDeaccessionPopup = false;
+ public boolean isRenderDeaccessionPopup() {
+ return renderDeaccessionPopup;
+ }
+ public void setRenderDeaccessionPopup(boolean renderDeaccessionPopup) {
+ this.renderDeaccessionPopup = renderDeaccessionPopup;
+ }
+
public void updateSelectedLinkingDV(ValueChangeEvent event) {
linkingDataverseId = (Long) event.getNewValue();
}
@@ -1751,7 +1769,7 @@ public void edit(EditMode editMode) {
dataset = datasetService.find(dataset.getId());
}
workingVersion = dataset.getEditVersion();
-
+ clone = workingVersion.cloneDatasetVersion();
if (editMode == EditMode.INFO) {
// ?
} else if (editMode == EditMode.FILE) {
@@ -1760,10 +1778,10 @@ public void edit(EditMode editMode) {
} else if (editMode.equals(EditMode.METADATA)) {
datasetVersionUI = datasetVersionUI.initDatasetVersionUI(workingVersion, true);
updateDatasetFieldInputLevels();
- JH.addMessage(FacesMessage.SEVERITY_INFO, BundleUtil.getStringFromBundle("dataset.message.editMetadata"));
+ JH.addMessage(FacesMessage.SEVERITY_INFO, BundleUtil.getStringFromBundle("dataset.message.editMetadata.label"), BundleUtil.getStringFromBundle("dataset.message.editMetadata.message"));
//FacesContext.getCurrentInstance().addMessage(null, new FacesMessage(FacesMessage.SEVERITY_INFO, "Edit Dataset Metadata", " - Add more metadata about your dataset to help others easily find it."));
} else if (editMode.equals(EditMode.LICENSE)){
- JH.addMessage(FacesMessage.SEVERITY_INFO, BundleUtil.getStringFromBundle("dataset.message.editTerms"));
+ JH.addMessage(FacesMessage.SEVERITY_INFO, BundleUtil.getStringFromBundle("dataset.message.editTerms.label"), BundleUtil.getStringFromBundle("dataset.message.editTerms.message"));
//FacesContext.getCurrentInstance().addMessage(null, new FacesMessage(FacesMessage.SEVERITY_INFO, "Edit Dataset License and Terms", " - Update your dataset's license and terms of use."));
}
this.readOnly = false;
@@ -1875,6 +1893,7 @@ public String deaccessionVersions() {
logger.severe(ex.getMessage());
JH.addMessage(FacesMessage.SEVERITY_FATAL, BundleUtil.getStringFromBundle("dataset.message.deaccessionFailure"));
}
+ setRenderDeaccessionPopup(false);
JsfHelper.addSuccessMessage(BundleUtil.getStringFromBundle("datasetVersion.message.deaccessionSuccess"));
return returnToDatasetOnly();
}
@@ -2271,21 +2290,16 @@ private List getSuccessMessageArguments() {
}
- public void saveLinkingDataverses() {
-
- if (selectedDataverseForLinking == null) {
- FacesMessage message = new FacesMessage(FacesMessage.SEVERITY_ERROR, "", BundleUtil.getStringFromBundle("dataverse.link.select"));
- FacesContext.getCurrentInstance().addMessage(null, message);
- return;
- }
+ public void saveLinkingDataverses(ActionEvent evt) {
- if(saveLink(selectedDataverseForLinking)){
+ if (saveLink(selectedDataverseForLinking)) {
JsfHelper.addSuccessMessage(BundleUtil.getStringFromBundle("dataset.message.linkSuccess", getSuccessMessageArguments()));
- } else{
+ } else {
FacesMessage message = new FacesMessage(FacesMessage.SEVERITY_INFO, BundleUtil.getStringFromBundle("dataset.notlinked"), linkingDataverseErrorMessage);
FacesContext.getCurrentInstance().addMessage(null, message);
}
- }
+
+ }
private String linkingDataverseErrorMessage = "";
@@ -2557,7 +2571,59 @@ public String saveWithTermsOfUse() {
return save();
}
- public String save() {
+ public void validateDeaccessionReason(FacesContext context, UIComponent toValidate, Object value) {
+
+ UIInput reasonRadio = (UIInput) toValidate.getAttributes().get("reasonRadio");
+ Object reasonRadioValue = reasonRadio.getValue();
+ Integer radioVal = new Integer(reasonRadioValue.toString());
+
+ if (radioVal == 7 && (value == null || value.toString().isEmpty())) {
+ ((UIInput) toValidate).setValid(false);
+ FacesMessage message = new FacesMessage(FacesMessage.SEVERITY_ERROR, "", BundleUtil.getStringFromBundle("file.deaccessionDialog.dialog.textForReason.error"));
+ context.addMessage(toValidate.getClientId(context), message);
+
+ } else {
+ if (value == null || value.toString().length() <= DatasetVersion.VERSION_NOTE_MAX_LENGTH) {
+ ((UIInput) toValidate).setValid(true);
+ } else {
+ ((UIInput) toValidate).setValid(false);
+ Integer lenghtInt = DatasetVersion.VERSION_NOTE_MAX_LENGTH;
+ String lengthString = lenghtInt.toString();
+ String userMsg = BundleUtil.getStringFromBundle("file.deaccessionDialog.dialog.limitChar.error", Arrays.asList(lengthString));
+ FacesMessage message = new FacesMessage(FacesMessage.SEVERITY_ERROR, "", userMsg);
+ context.addMessage(toValidate.getClientId(context), message);
+ }
+ }
+ }
+
+ public void validateForwardURL(FacesContext context, UIComponent toValidate, Object value) {
+
+ if ((value == null || value.toString().isEmpty())) {
+ ((UIInput) toValidate).setValid(true);
+ return;
+ }
+
+ String testVal = value.toString();
+
+ if (!URLValidator.isURLValid(testVal)) {
+ ((UIInput) toValidate).setValid(false);
+ FacesMessage message = new FacesMessage(FacesMessage.SEVERITY_ERROR, BundleUtil.getStringFromBundle("file.deaccessionDialog.dialog.url.error"), BundleUtil.getStringFromBundle("file.deaccessionDialog.dialog.url.error"));
+ context.addMessage(toValidate.getClientId(context), message);
+ return;
+ }
+
+ if (value.toString().length() <= DatasetVersion.ARCHIVE_NOTE_MAX_LENGTH) {
+ ((UIInput) toValidate).setValid(true);
+ } else {
+ ((UIInput) toValidate).setValid(false);
+ FacesMessage message = new FacesMessage(FacesMessage.SEVERITY_ERROR, BundleUtil.getStringFromBundle("file.deaccessionDialog.dialog.url.error"), BundleUtil.getStringFromBundle("file.deaccessionDialog.dialog.url.error"));
+ context.addMessage(toValidate.getClientId(context), message);
+
+ }
+
+ }
+
+ public String save() {
//Before dataset saved, write cached prov freeform to version
if(systemConfig.isProvCollectionEnabled()) {
provPopupFragmentBean.saveStageProvFreeformToLatestVersion();
@@ -2588,7 +2654,7 @@ public String save() {
}
} else {
- cmd = new UpdateDatasetVersionCommand(dataset, dvRequestService.getDataverseRequest(), filesToBeDeleted);
+ cmd = new UpdateDatasetVersionCommand(dataset, dvRequestService.getDataverseRequest(), filesToBeDeleted, clone );
((UpdateDatasetVersionCommand) cmd).setValidateLenient(true);
}
dataset = commandEngine.submit(cmd);
@@ -4324,7 +4390,8 @@ public String getJsonLd() {
return jsonLd;
} else {
logger.fine("No cached schema.org JSON-LD available. Going to the database.");
- return workingVersion.getJsonLd();
+ String jsonLdProduced = workingVersion.getJsonLd();
+ return jsonLdProduced != null ? jsonLdProduced : "";
}
}
return "";
@@ -4337,7 +4404,7 @@ public void selectAllFiles() {
public void clearSelection() {
logger.info("clearSelection called");
- selectedFiles = Collections.EMPTY_LIST;
+ selectedFiles = Collections.emptyList();
}
public void fileListingPaginatorListener(PageEvent event) {
@@ -4350,4 +4417,43 @@ public void refreshPaginator() {
setFilePaginatorPage(dt.getPage());
setRowsPerPage(dt.getRowsToRender());
}
+
+ /**
+ * This method can be called from *.xhtml files to allow archiving of a dataset
+ * version from the user interface. It is not currently (11/18) used in the IQSS/develop
+ * branch, but is used by QDR and is kept here in anticipation of including a
+ * GUI option to archive (already published) versions after other dataset page
+ * changes have been completed.
+ *
+ * @param id - the id of the datasetversion to archive.
+ */
+ public void archiveVersion(Long id) {
+ if (session.getUser() instanceof AuthenticatedUser) {
+ AuthenticatedUser au = ((AuthenticatedUser) session.getUser());
+
+ DatasetVersion dv = datasetVersionService.retrieveDatasetVersionByVersionId(id).getDatasetVersion();
+ String className = settingsService.getValueForKey(SettingsServiceBean.Key.ArchiverClassName);
+ AbstractSubmitToArchiveCommand cmd = ArchiverUtil.createSubmitToArchiveCommand(className, dvRequestService.getDataverseRequest(), dv);
+ if (cmd != null) {
+ try {
+ DatasetVersion version = commandEngine.submit(cmd);
+ logger.info("Archived to " + version.getArchivalCopyLocation());
+ if (version.getArchivalCopyLocation() != null) {
+ resetVersionTabList();
+ this.setVersionTabListForPostLoad(getVersionTabList());
+ JsfHelper.addSuccessMessage(BundleUtil.getStringFromBundle("datasetversion.archive.success"));
+ } else {
+ JsfHelper.addErrorMessage(BundleUtil.getStringFromBundle("datasetversion.archive.failure"));
+ }
+ } catch (CommandException ex) {
+ logger.log(Level.SEVERE, "Unexpected Exception calling submit archive command", ex);
+ JsfHelper.addErrorMessage(BundleUtil.getStringFromBundle("datasetversion.archive.failure"));
+ }
+ } else {
+ logger.log(Level.SEVERE, "Could not find Archiver class: " + className);
+ JsfHelper.addErrorMessage(BundleUtil.getStringFromBundle("datasetversion.archive.failure"));
+
+ }
+ }
+ }
}
diff --git a/src/main/java/edu/harvard/iq/dataverse/DatasetVersion.java b/src/main/java/edu/harvard/iq/dataverse/DatasetVersion.java
index b8f1f636541..5df04bf6eb7 100644
--- a/src/main/java/edu/harvard/iq/dataverse/DatasetVersion.java
+++ b/src/main/java/edu/harvard/iq/dataverse/DatasetVersion.java
@@ -48,6 +48,7 @@
import javax.validation.Validation;
import javax.validation.Validator;
import javax.validation.ValidatorFactory;
+import javax.validation.constraints.Size;
/**
*
@@ -56,6 +57,7 @@
@Entity
@Table(indexes = {@Index(columnList="dataset_id")},
uniqueConstraints = @UniqueConstraint(columnNames = {"dataset_id,versionnumber,minorversionnumber"}))
+@ValidateVersionNote(versionNote = "versionNote", versionState = "versionState")
public class DatasetVersion implements Serializable {
private static final Logger logger = Logger.getLogger(DatasetVersion.class.getCanonicalName());
@@ -103,6 +105,7 @@ public enum License {
private Long versionNumber;
private Long minorVersionNumber;
+ @Size(min=0, max=VERSION_NOTE_MAX_LENGTH)
@Column(length = VERSION_NOTE_MAX_LENGTH)
private String versionNote;
@@ -142,9 +145,15 @@ public enum License {
@Temporal(value = TemporalType.TIMESTAMP)
private Date archiveTime;
+ @Size(min=0, max=ARCHIVE_NOTE_MAX_LENGTH)
@Column(length = ARCHIVE_NOTE_MAX_LENGTH)
+ @ValidateURL()
private String archiveNote;
+ @Column(nullable=true, columnDefinition = "TEXT")
+ private String archivalCopyLocation;
+
+
private String deaccessionLink;
@Transient
@@ -257,6 +266,14 @@ public void setArchiveNote(String note) {
}
this.archiveNote = note;
}
+
+ public String getArchivalCopyLocation() {
+ return archivalCopyLocation;
+ }
+
+ public void setArchivalCopyLocation(String location) {
+ this.archivalCopyLocation = location;
+ }
public String getDeaccessionLink() {
return deaccessionLink;
@@ -514,6 +531,56 @@ public void updateDefaultValuesFromTemplate(Template template) {
this.setTermsOfUseAndAccess(terms);
}
}
+
+ public DatasetVersion cloneDatasetVersion(){
+ DatasetVersion dsv = new DatasetVersion();
+ dsv.setVersionState(this.getPriorVersionState());
+ dsv.setFileMetadatas(new ArrayList<>());
+
+ if (this.getUNF() != null){
+ dsv.setUNF(this.getUNF());
+ }
+
+ if (this.getDatasetFields() != null && !this.getDatasetFields().isEmpty()) {
+ dsv.setDatasetFields(dsv.copyDatasetFields(this.getDatasetFields()));
+ }
+
+ if (this.getTermsOfUseAndAccess()!= null){
+ dsv.setTermsOfUseAndAccess(this.getTermsOfUseAndAccess().copyTermsOfUseAndAccess());
+ } else {
+ TermsOfUseAndAccess terms = new TermsOfUseAndAccess();
+ terms.setDatasetVersion(dsv);
+ terms.setLicense(TermsOfUseAndAccess.License.CC0);
+ dsv.setTermsOfUseAndAccess(terms);
+ }
+
+ for (FileMetadata fm : this.getFileMetadatas()) {
+ FileMetadata newFm = new FileMetadata();
+ // TODO:
+ // the "category" will be removed, shortly.
+ // (replaced by multiple, tag-like categories of
+ // type DataFileCategory) -- L.A. beta 10
+ //newFm.setCategory(fm.getCategory());
+ // yep, these are the new categories:
+ newFm.setCategories(fm.getCategories());
+ newFm.setDescription(fm.getDescription());
+ newFm.setLabel(fm.getLabel());
+ newFm.setDirectoryLabel(fm.getDirectoryLabel());
+ newFm.setRestricted(fm.isRestricted());
+ newFm.setDataFile(fm.getDataFile());
+ newFm.setDatasetVersion(dsv);
+ newFm.setProvFreeForm(fm.getProvFreeForm());
+
+ dsv.getFileMetadatas().add(newFm);
+ }
+
+
+
+
+ dsv.setDataset(this.getDataset());
+ return dsv;
+
+ }
public void initDefaultValues() {
//first clear then initialize - in case values were present
@@ -766,7 +833,7 @@ public List getDatasetAuthors() {
}
return retList;
}
-
+
public List getFunders() {
List retList = new ArrayList<>();
for (DatasetField dsf : this.getDatasetFields()) {
@@ -781,14 +848,11 @@ public List getFunders() {
}
if (subField.getDatasetFieldType().getName().equals(DatasetFieldConstant.contributorType)) {
contributorType = subField.getDisplayValue();
- // TODO: Consider how this will work in French, Chinese, etc.
- String funderString = "Funder";
- if (funderString.equals(contributorType)) {
- addFunder = true;
- }
}
}
- if (addFunder) {
+ //SEK 02/12/2019 move outside loop to prevent contrib type to carry over to next contributor
+ // TODO: Consider how this will work in French, Chinese, etc.
+ if ("Funder".equals(contributorType)) {
retList.add(contributorName);
}
}
@@ -818,46 +882,46 @@ public List getTimePeriodsCovered() {
String start = "";
String end = "";
for (DatasetField subField : timePeriodValue.getChildDatasetFields()) {
- if (subField.getDatasetFieldType().getName()
- .equals(DatasetFieldConstant.timePeriodCoveredStart)) {
- if (subField.isEmptyForDisplay()) {
- start = null;
- } else {
- // we want to use "getValue()", as opposed to "getDisplayValue()" here -
- // as the latter method prepends the value with the word "Start:"!
- start = subField.getValue();
- }
- }
- if (subField.getDatasetFieldType().getName()
- .equals(DatasetFieldConstant.timePeriodCoveredEnd)) {
- if (subField.isEmptyForDisplay()) {
- end = null;
- } else {
- // see the comment above
- end = subField.getValue();
- }
- }
-
- }
- if (start != null && end != null) {
- retList.add(start + "/" + end);
- }
- }
- }
- }
- return retList;
- }
-
- public List getDatesOfCollection() {
- List retList = new ArrayList<>();
- for (DatasetField dsf : this.getDatasetFields()) {
- if (dsf.getDatasetFieldType().getName().equals(DatasetFieldConstant.dateOfCollection)) {
- for (DatasetFieldCompoundValue timePeriodValue : dsf.getDatasetFieldCompoundValues()) {
- String start = "";
- String end = "";
- for (DatasetField subField : timePeriodValue.getChildDatasetFields()) {
- if (subField.getDatasetFieldType().getName()
- .equals(DatasetFieldConstant.dateOfCollectionStart)) {
+ if (subField.getDatasetFieldType().getName()
+ .equals(DatasetFieldConstant.timePeriodCoveredStart)) {
+ if (subField.isEmptyForDisplay()) {
+ start = null;
+ } else {
+ // we want to use "getValue()", as opposed to "getDisplayValue()" here -
+ // as the latter method prepends the value with the word "Start:"!
+ start = subField.getValue();
+ }
+ }
+ if (subField.getDatasetFieldType().getName()
+ .equals(DatasetFieldConstant.timePeriodCoveredEnd)) {
+ if (subField.isEmptyForDisplay()) {
+ end = null;
+ } else {
+ // see the comment above
+ end = subField.getValue();
+ }
+ }
+
+ }
+ if (start != null && end != null) {
+ retList.add(start + "/" + end);
+ }
+ }
+ }
+ }
+ return retList;
+ }
+
+ public List getDatesOfCollection() {
+ List retList = new ArrayList<>();
+ for (DatasetField dsf : this.getDatasetFields()) {
+ if (dsf.getDatasetFieldType().getName().equals(DatasetFieldConstant.dateOfCollection)) {
+ for (DatasetFieldCompoundValue timePeriodValue : dsf.getDatasetFieldCompoundValues()) {
+ String start = "";
+ String end = "";
+ for (DatasetField subField : timePeriodValue.getChildDatasetFields()) {
+ if (subField.getDatasetFieldType().getName()
+ .equals(DatasetFieldConstant.dateOfCollectionStart)) {
if (subField.isEmptyForDisplay()) {
start = null;
} else {
@@ -866,8 +930,8 @@ public List getDatesOfCollection() {
start = subField.getValue();
}
}
- if (subField.getDatasetFieldType().getName()
- .equals(DatasetFieldConstant.dateOfCollectionEnd)) {
+ if (subField.getDatasetFieldType().getName()
+ .equals(DatasetFieldConstant.dateOfCollectionEnd)) {
if (subField.isEmptyForDisplay()) {
end = null;
} else {
@@ -914,79 +978,79 @@ public List getDatasetSubjects() {
* @return List of Strings containing the version's Topic Classifications
*/
public List getTopicClassifications() {
- return getCompoundChildFieldValues(DatasetFieldConstant.topicClassification,
- DatasetFieldConstant.topicClassValue);
- }
-
- /**
- * @return List of Strings containing the version's Kind Of Data entries
- */
- public List getKindOfData() {
- List kod = new ArrayList<>();
- for (DatasetField dsf : this.getDatasetFields()) {
- if (dsf.getDatasetFieldType().getName().equals(DatasetFieldConstant.kindOfData)) {
- kod.addAll(dsf.getValues());
- }
- }
- return kod;
- }
-
- /**
- * @return List of Strings containing the version's language entries
- */
- public List getLanguages() {
- List languages = new ArrayList<>();
- for (DatasetField dsf : this.getDatasetFields()) {
- if (dsf.getDatasetFieldType().getName().equals(DatasetFieldConstant.language)) {
- languages.addAll(dsf.getValues());
- }
- }
- return languages;
- }
-
+ return getCompoundChildFieldValues(DatasetFieldConstant.topicClassification,
+ DatasetFieldConstant.topicClassValue);
+ }
+
+ /**
+ * @return List of Strings containing the version's Kind Of Data entries
+ */
+ public List getKindOfData() {
+ List kod = new ArrayList<>();
+ for (DatasetField dsf : this.getDatasetFields()) {
+ if (dsf.getDatasetFieldType().getName().equals(DatasetFieldConstant.kindOfData)) {
+ kod.addAll(dsf.getValues());
+ }
+ }
+ return kod;
+ }
+
+ /**
+ * @return List of Strings containing the version's language entries
+ */
+ public List getLanguages() {
+ List languages = new ArrayList<>();
+ for (DatasetField dsf : this.getDatasetFields()) {
+ if (dsf.getDatasetFieldType().getName().equals(DatasetFieldConstant.language)) {
+ languages.addAll(dsf.getValues());
+ }
+ }
+ return languages;
+ }
+
// TODO: consider calling the newer getSpatialCoverages method below with the commaSeparated boolean set to true.
- public List getSpatialCoverages() {
- List retList = new ArrayList<>();
- for (DatasetField dsf : this.getDatasetFields()) {
- if (dsf.getDatasetFieldType().getName().equals(DatasetFieldConstant.geographicCoverage)) {
- for (DatasetFieldCompoundValue geoValue : dsf.getDatasetFieldCompoundValues()) {
- List coverage = new ArrayList();
- for (DatasetField subField : geoValue.getChildDatasetFields()) {
- if (subField.getDatasetFieldType().getName()
- .equals(DatasetFieldConstant.country)) {
- if (!subField.isEmptyForDisplay()) {
- } else {
- coverage.add(subField.getValue());
- }
- }
- if (subField.getDatasetFieldType().getName()
- .equals(DatasetFieldConstant.state)) {
- if (!subField.isEmptyForDisplay()) {
- coverage.add(subField.getValue());
- }
- }
- if (subField.getDatasetFieldType().getName()
- .equals(DatasetFieldConstant.city)) {
- if (!subField.isEmptyForDisplay()) {
- coverage.add(subField.getValue());
- }
- }
- if (subField.getDatasetFieldType().getName()
- .equals(DatasetFieldConstant.otherGeographicCoverage)) {
- if (!subField.isEmptyForDisplay()) {
- coverage.add(subField.getValue());
- }
- }
- }
- if (!coverage.isEmpty()) {
- retList.add(String.join(",", coverage));
- }
- }
- }
- }
- return retList;
+ public List getSpatialCoverages() {
+ List retList = new ArrayList<>();
+ for (DatasetField dsf : this.getDatasetFields()) {
+ if (dsf.getDatasetFieldType().getName().equals(DatasetFieldConstant.geographicCoverage)) {
+ for (DatasetFieldCompoundValue geoValue : dsf.getDatasetFieldCompoundValues()) {
+ List coverage = new ArrayList();
+ for (DatasetField subField : geoValue.getChildDatasetFields()) {
+ if (subField.getDatasetFieldType().getName()
+ .equals(DatasetFieldConstant.country)) {
+ if (!subField.isEmptyForDisplay()) {
+ } else {
+ coverage.add(subField.getValue());
+ }
+ }
+ if (subField.getDatasetFieldType().getName()
+ .equals(DatasetFieldConstant.state)) {
+ if (!subField.isEmptyForDisplay()) {
+ coverage.add(subField.getValue());
+ }
+ }
+ if (subField.getDatasetFieldType().getName()
+ .equals(DatasetFieldConstant.city)) {
+ if (!subField.isEmptyForDisplay()) {
+ coverage.add(subField.getValue());
+ }
+ }
+ if (subField.getDatasetFieldType().getName()
+ .equals(DatasetFieldConstant.otherGeographicCoverage)) {
+ if (!subField.isEmptyForDisplay()) {
+ coverage.add(subField.getValue());
+ }
+ }
+ }
+ if (!coverage.isEmpty()) {
+ retList.add(String.join(",", coverage));
+ }
+ }
+ }
+ }
+ return retList;
}
-
+
public List getSpatialCoverages(boolean commaSeparated) {
List retList = new ArrayList<>();
for (DatasetField dsf : this.getDatasetFields()) {
@@ -1056,7 +1120,7 @@ private List sortSpatialCoverage(Map hash) {
public List getKeywords() {
return getCompoundChildFieldValues(DatasetFieldConstant.keyword, DatasetFieldConstant.keywordValue);
}
-
+
public List getRelatedPublications() {
List relatedPublications = new ArrayList<>();
for (DatasetField dsf : this.getDatasetFields()) {
@@ -1083,38 +1147,38 @@ public List getRelatedPublications() {
}
/**
- * @return List of Strings containing the version's Grant Agency(ies)
- */
- public List getUniqueGrantAgencyValues() {
-
- // Since only grant agency names are returned, use distinct() to avoid repeats
- // (e.g. if there are two grants from the same agency)
- return getCompoundChildFieldValues(DatasetFieldConstant.grantNumber, DatasetFieldConstant.grantNumberAgency)
- .stream().distinct().collect(Collectors.toList());
- }
-
- /**
- * @return String containing the version's series title
- */
- public String getSeriesTitle() {
-
- List seriesNames = getCompoundChildFieldValues(DatasetFieldConstant.series,
- DatasetFieldConstant.seriesName);
- if (seriesNames.size() > 1) {
- logger.warning("More than one series title found for datasetVersion: " + this.id);
- }
- if (!seriesNames.isEmpty()) {
- return seriesNames.get(0);
- } else {
- return null;
- }
- }
-
- /**
- * @param parentFieldName
- * compound dataset field A (from DatasetFieldConstant.*)
- * @param childFieldName
- * dataset field B, child field of A (from DatasetFieldConstant.*)
+ * @return List of Strings containing the version's Grant Agency(ies)
+ */
+ public List getUniqueGrantAgencyValues() {
+
+ // Since only grant agency names are returned, use distinct() to avoid repeats
+ // (e.g. if there are two grants from the same agency)
+ return getCompoundChildFieldValues(DatasetFieldConstant.grantNumber, DatasetFieldConstant.grantNumberAgency)
+ .stream().distinct().collect(Collectors.toList());
+ }
+
+ /**
+ * @return String containing the version's series title
+ */
+ public String getSeriesTitle() {
+
+ List seriesNames = getCompoundChildFieldValues(DatasetFieldConstant.series,
+ DatasetFieldConstant.seriesName);
+ if (seriesNames.size() > 1) {
+ logger.warning("More than one series title found for datasetVersion: " + this.id);
+ }
+ if (!seriesNames.isEmpty()) {
+ return seriesNames.get(0);
+ } else {
+ return null;
+ }
+ }
+
+ /**
+ * @param parentFieldName
+ * compound dataset field A (from DatasetFieldConstant.*)
+ * @param childFieldName
+ * dataset field B, child field of A (from DatasetFieldConstant.*)
* @return List of values of the child field
*/
public List getCompoundChildFieldValues(String parentFieldName, String childFieldName) {
@@ -1146,7 +1210,7 @@ public List getDatasetProducerNames(){
for (DatasetFieldCompoundValue authorValue : dsf.getDatasetFieldCompoundValues()) {
for (DatasetField subField : authorValue.getChildDatasetFields()) {
if (subField.getDatasetFieldType().getName().equals(DatasetFieldConstant.producerName)) {
- producerNames.add(subField.getDisplayValue().trim());
+ producerNames.add(subField.getDisplayValue().trim());
}
}
}
@@ -1494,9 +1558,10 @@ public Set validate() {
}
}
}
+
return returnSet;
}
-
+
public List getWorkflowComments() {
return workflowComments;
}
@@ -1635,7 +1700,7 @@ public String getJsonLd() {
}
job.add("keywords", keywords);
-
+
/**
* citation: (multiple) related publication citation and URLs, if
* present.
@@ -1671,7 +1736,7 @@ public String getJsonLd() {
job.add("citation", jsonArray);
}
}
-
+
/**
* temporalCoverage:
* (if available)
@@ -1685,13 +1750,13 @@ public String getJsonLd() {
}
job.add("temporalCoverage", temporalCoverage);
}
-
+
/**
* https://schema.org/version/3.4/ says, "Note that schema.org release
* numbers are not generally included when you use schema.org. In
* contexts (e.g. related standards work) when a particular release
* needs to be cited, this document provides the appropriate URL."
- *
+ *
* For the reason above we decided to take out schemaVersion but we're
* leaving this Javadoc in here to remind us that we made this decision.
* We used to include "https://schema.org/version/3.3" in the output for
@@ -1738,7 +1803,7 @@ public String getJsonLd() {
if (!funderNames.isEmpty()) {
JsonArrayBuilder funderArray = Json.createArrayBuilder();
for (String funderName : funderNames) {
- JsonObjectBuilder funder = Json.createObjectBuilder();
+ JsonObjectBuilder funder = NullSafeJsonBuilder.jsonObjectBuilder();
funder.add("@type", "Organization");
funder.add("name", funderName);
funderArray.add(funder);
diff --git a/src/main/java/edu/harvard/iq/dataverse/DatasetVersionConverter.java b/src/main/java/edu/harvard/iq/dataverse/DatasetVersionConverter.java
new file mode 100644
index 00000000000..79a801fbb55
--- /dev/null
+++ b/src/main/java/edu/harvard/iq/dataverse/DatasetVersionConverter.java
@@ -0,0 +1,44 @@
+/*
+ * To change this license header, choose License Headers in Project Properties.
+ * To change this template file, choose Tools | Templates
+ * and open the template in the editor.
+ */
+package edu.harvard.iq.dataverse;
+
+import javax.ejb.EJB;
+import javax.faces.component.UIComponent;
+import javax.faces.context.FacesContext;
+import javax.faces.convert.Converter;
+import javax.faces.convert.FacesConverter;
+
+/**
+ *
+ * @author skraffmi
+ */
+@FacesConverter("datasetVersionConverter")
+public class DatasetVersionConverter implements Converter {
+
+ @EJB
+ DatasetVersionServiceBean datasetVersionService;
+
+
+ @Override
+ public Object getAsObject(FacesContext context, UIComponent component, String value) {
+ if (value == null || value.equals("")) {
+ return "";
+ } else {
+ return datasetVersionService.find(new Long(value));
+ }
+ }
+
+ @Override
+ public String getAsString(FacesContext context, UIComponent component, Object value) {
+ if (value == null || value.equals("")) {
+ return "";
+ } else {
+ String stringToReturn = ((DatasetVersion) value).getId().toString();
+ return stringToReturn;
+ }
+ }
+
+}
diff --git a/src/main/java/edu/harvard/iq/dataverse/DatasetVersionDifference.java b/src/main/java/edu/harvard/iq/dataverse/DatasetVersionDifference.java
index d1a8a0692ab..e3fe4a96ffe 100644
--- a/src/main/java/edu/harvard/iq/dataverse/DatasetVersionDifference.java
+++ b/src/main/java/edu/harvard/iq/dataverse/DatasetVersionDifference.java
@@ -8,6 +8,7 @@
import java.util.ResourceBundle;
import edu.harvard.iq.dataverse.util.BundleUtil;
import java.util.Arrays;
+import java.util.Date;
/**
*
@@ -784,7 +785,9 @@ private void initDatasetFilesDifferencesList() {
fdr.setLeftColumn(diffLabel);
fdr.setFdi(fdi);
fdr.setFile1Id(replacedFile.getDataFile().getId().toString());
- fdr.setFile2Id(newFile.getDataFile().getId().toString());
+ if (newFile.getDataFile().getId() != null) {
+ fdr.setFile2Id(newFile.getDataFile().getId().toString());
+ }
fdr.setFile1ChecksumType(replacedFile.getDataFile().getChecksumType());
fdr.setFile2ChecksumType(newFile.getDataFile().getChecksumType());
fdr.setFile1ChecksumValue(replacedFile.getDataFile().getChecksumValue());
@@ -1161,6 +1164,172 @@ private datasetFileDifferenceItem selectFileMetadataDiffs(FileMetadata fm1, File
return fdi;
}
+ public String getEditSummaryForLog() {
+
+ String retVal = "";
+
+ retVal = System.lineSeparator() + this.newVersion.getTitle() + " (" + this.originalVersion.getDataset().getIdentifier() + ") was updated " + new Date();
+
+ String valueString = "";
+ String groupString = "";
+
+ //Metadata differences displayed by Metdata block
+ if (!this.detailDataByBlock.isEmpty()) {
+ for (List blocks : detailDataByBlock) {
+ groupString = System.lineSeparator() + " " + BundleUtil.getStringFromBundle("dataset.versionDifferences.metadataBlock") ;
+ String blockDisplay = " " + blocks.get(0)[0].getDatasetFieldType().getMetadataBlock().getDisplayName() + ": " + System.lineSeparator();
+ groupString += blockDisplay;
+ for (DatasetField[] dsfArray : blocks) {
+ valueString = " " + BundleUtil.getStringFromBundle("dataset.versionDifferences.field") + ": ";
+ String title = dsfArray[0].getDatasetFieldType().getTitle();
+ valueString += title;
+ String oldValue = " " + BundleUtil.getStringFromBundle("dataset.versionDifferences.changed") + " " + BundleUtil.getStringFromBundle("dataset.versionDifferences.from") + ": ";
+
+ if (!dsfArray[0].isEmpty()) {
+ if (dsfArray[0].getDatasetFieldType().isPrimitive()) {
+ oldValue += dsfArray[0].getRawValue();
+ } else {
+ oldValue += dsfArray[0].getCompoundRawValue();
+ }
+ }
+ valueString += oldValue;
+
+ String newValue = " " + BundleUtil.getStringFromBundle("dataset.versionDifferences.to") + ": ";
+ if (!dsfArray[1].isEmpty()) {
+ if (dsfArray[1].getDatasetFieldType().isPrimitive()) {
+ newValue += dsfArray[1].getRawValue();
+ } else {
+ newValue += dsfArray[1].getCompoundRawValue();
+ }
+
+ }
+ valueString += newValue;
+ groupString += valueString + System.lineSeparator();
+ }
+ retVal += groupString + System.lineSeparator();
+ }
+ }
+
+ // File Differences
+ String fileDiff = System.lineSeparator() + BundleUtil.getStringFromBundle("file.viewDiffDialog.files.header") + ": " + System.lineSeparator();
+ if(!this.getDatasetFilesDiffList().isEmpty()){
+
+ String itemDiff;
+
+ for (datasetFileDifferenceItem item : this.getDatasetFilesDiffList()) {
+ itemDiff = BundleUtil.getStringFromBundle("file.viewDiffDialog.fileID") + ": " + item.fileId;
+
+ if (item.fileName1 != null || item.fileName2 != null) {
+ itemDiff = System.lineSeparator() + " " + BundleUtil.getStringFromBundle("file.viewDiffDialog.fileName") + ": ";
+ itemDiff += item.fileName1 != null ? item.fileName1 : BundleUtil.getStringFromBundle("file.viewDiffDialog.notAvailable");
+ itemDiff += " : ";
+ itemDiff += item.fileName2 != null ? item.fileName2 : BundleUtil.getStringFromBundle("file.viewDiffDialog.notAvailable") + " ";
+ }
+
+ if (item.fileType1 != null || item.fileType2 != null) {
+ itemDiff += System.lineSeparator() + " " + BundleUtil.getStringFromBundle("file.viewDiffDialog.fileType") + ": ";
+ itemDiff += item.fileType1 != null ? item.fileType1 : BundleUtil.getStringFromBundle("file.viewDiffDialog.notAvailable");
+ itemDiff += " : ";
+ itemDiff += item.fileType2 != null ? item.fileType2 : BundleUtil.getStringFromBundle("file.viewDiffDialog.notAvailable") + " ";
+ }
+
+ if (item.fileSize1 != null || item.fileSize2 != null) {
+ itemDiff += System.lineSeparator() + " " + BundleUtil.getStringFromBundle("file.viewDiffDialog.fileSize") + ": ";
+ itemDiff += item.fileSize1 != null ? item.fileSize1 : BundleUtil.getStringFromBundle("file.viewDiffDialog.notAvailable");
+ itemDiff += " : ";
+ itemDiff += item.fileSize2 != null ? item.fileSize2 : BundleUtil.getStringFromBundle("file.viewDiffDialog.notAvailable") + " ";
+ }
+
+ if (item.fileCat1 != null || item.fileCat2 != null) {
+ itemDiff += System.lineSeparator() + " " + BundleUtil.getStringFromBundle("file.viewDiffDialog.category") + ": ";
+ itemDiff += item.fileCat1 != null ? item.fileCat1 : BundleUtil.getStringFromBundle("file.viewDiffDialog.notAvailable");
+ itemDiff += " : ";
+ itemDiff += item.fileCat2 != null ? item.fileCat2 : BundleUtil.getStringFromBundle("file.viewDiffDialog.notAvailable") + " ";
+ }
+
+ if (item.fileDesc1 != null || item.fileDesc2 != null) {
+ itemDiff += System.lineSeparator() + " " + BundleUtil.getStringFromBundle("file.viewDiffDialog.description") + ": ";
+ itemDiff += item.fileDesc1 != null ? item.fileDesc1 : BundleUtil.getStringFromBundle("file.viewDiffDialog.notAvailable");
+ itemDiff += " : ";
+ itemDiff += item.fileDesc2 != null ? item.fileDesc2 : BundleUtil.getStringFromBundle("file.viewDiffDialog.notAvailable") + " ";
+ }
+
+ if (item.fileProvFree1 != null || item.fileProvFree2 != null) {
+ itemDiff += System.lineSeparator() + " " + BundleUtil.getStringFromBundle("file.viewDiffDialog.provDescription") + ": ";
+ itemDiff += item.fileProvFree1 != null ? item.fileProvFree1 : BundleUtil.getStringFromBundle("file.viewDiffDialog.notAvailable");
+ itemDiff += " : ";
+ itemDiff += item.fileProvFree2 != null ? item.fileProvFree2 : BundleUtil.getStringFromBundle("file.viewDiffDialog.notAvailable") + " ";
+ }
+
+ if (item.fileRest1 != null || item.fileRest2 != null) {
+ itemDiff += System.lineSeparator() + " " + BundleUtil.getStringFromBundle("file.viewDiffDialog.restricted") + ": ";
+ itemDiff += item.fileRest1 != null ? item.fileRest1 : BundleUtil.getStringFromBundle("file.viewDiffDialog.notAvailable");
+ itemDiff += " : ";
+ itemDiff += item.fileRest2 != null ? item.fileRest2 : BundleUtil.getStringFromBundle("file.viewDiffDialog.notAvailable") + " ";
+
+ }
+
+ fileDiff += itemDiff;
+ }
+
+ retVal += fileDiff;
+ }
+
+ String fileReplaced = System.lineSeparator() + BundleUtil.getStringFromBundle("file.viewDiffDialog.filesReplaced")+ ": "+ System.lineSeparator();
+ if(!this.getDatasetFilesReplacementList().isEmpty()){
+ String itemDiff;
+ for (datasetReplaceFileItem item : this.getDatasetFilesReplacementList()) {
+ itemDiff = "";
+ itemDiff = System.lineSeparator() + " " + BundleUtil.getStringFromBundle("file.viewDiffDialog.fileName") + ": ";
+ itemDiff += item.fdi.fileName1 != null ? item.fdi.fileName1 : BundleUtil.getStringFromBundle("file.viewDiffDialog.notAvailable");
+ itemDiff += " : ";
+ itemDiff += item.fdi.fileName2 != null ? item.fdi.fileName2 : BundleUtil.getStringFromBundle("file.viewDiffDialog.notAvailable");
+ itemDiff += System.lineSeparator() + " " + BundleUtil.getStringFromBundle("file.viewDiffDialog.fileType") + ": ";
+ itemDiff += item.fdi.fileType1 != null ? item.fdi.fileType1 : BundleUtil.getStringFromBundle("file.viewDiffDialog.notAvailable");
+ itemDiff += " : ";
+ itemDiff += item.fdi.fileType2 != null ? item.fdi.fileType2 : BundleUtil.getStringFromBundle("file.viewDiffDialog.notAvailable") + " ";
+ itemDiff += System.lineSeparator() + " " + BundleUtil.getStringFromBundle("file.viewDiffDialog.fileSize") + ": ";
+ itemDiff += item.fdi.fileSize1 != null ? item.fdi.fileSize1 : BundleUtil.getStringFromBundle("file.viewDiffDialog.notAvailable");
+ itemDiff += " : ";
+ itemDiff += item.fdi.fileSize2 != null ? item.fdi.fileSize2 : BundleUtil.getStringFromBundle("file.viewDiffDialog.notAvailable") + " ";
+ itemDiff += System.lineSeparator() + " " + BundleUtil.getStringFromBundle("file.viewDiffDialog.category") + ": ";
+ itemDiff += item.fdi.fileCat1 != null ? item.fdi.fileCat1 : BundleUtil.getStringFromBundle("file.viewDiffDialog.notAvailable");
+ itemDiff += " : ";
+ itemDiff += item.fdi.fileCat2 != null ? item.fdi.fileCat2 : BundleUtil.getStringFromBundle("file.viewDiffDialog.notAvailable") + " ";
+ itemDiff += System.lineSeparator() + " " + BundleUtil.getStringFromBundle("file.viewDiffDialog.description") + ": ";
+ itemDiff += item.fdi.fileDesc1 != null ? item.fdi.fileDesc1 : BundleUtil.getStringFromBundle("file.viewDiffDialog.notAvailable");
+ itemDiff += " : ";
+ itemDiff += item.fdi.fileDesc2 != null ? item.fdi.fileDesc2 : BundleUtil.getStringFromBundle("file.viewDiffDialog.notAvailable") + " ";
+ itemDiff += System.lineSeparator() + " " + BundleUtil.getStringFromBundle("file.viewDiffDialog.provDescription") + ": ";
+ itemDiff += item.fdi.fileProvFree1 != null ? item.fdi.fileProvFree1 : BundleUtil.getStringFromBundle("file.viewDiffDialog.notAvailable");
+ itemDiff += " : ";
+ itemDiff += item.fdi.fileProvFree2 != null ? item.fdi.fileProvFree2 : BundleUtil.getStringFromBundle("file.viewDiffDialog.notAvailable") + " ";
+ itemDiff += System.lineSeparator() + " " + BundleUtil.getStringFromBundle("file.viewDiffDialog.restricted") + ": ";
+ itemDiff += item.fdi.fileRest1 != null ? item.fdi.fileRest1 : BundleUtil.getStringFromBundle("file.viewDiffDialog.notAvailable");
+ itemDiff += " : ";
+ itemDiff += item.fdi.fileRest2 != null ? item.fdi.fileRest2 : BundleUtil.getStringFromBundle("file.viewDiffDialog.notAvailable") + " ";
+ fileReplaced += itemDiff;
+ }
+ retVal += fileReplaced;
+ }
+
+ String termsOfUseDiff = System.lineSeparator() + "Terms of Use and Access Changes: "+ System.lineSeparator();
+
+ if (!this.changedTermsAccess.isEmpty()){
+ for (String[] blocks : changedTermsAccess) {
+ String itemDiff = System.lineSeparator() + blocks[0] + " " + BundleUtil.getStringFromBundle("dataset.versionDifferences.changed") + " " + BundleUtil.getStringFromBundle("dataset.versionDifferences.from") + ": ";
+ itemDiff += blocks[1];
+ itemDiff += " " + BundleUtil.getStringFromBundle("dataset.versionDifferences.to") + ": "+ blocks[2];
+ termsOfUseDiff +=itemDiff;
+ }
+ retVal +=termsOfUseDiff;
+ }
+
+
+ return retVal;
+ }
+
+
public class DifferenceSummaryGroup{
private String displayName;
diff --git a/src/main/java/edu/harvard/iq/dataverse/DatasetVersionNoteValidator.java b/src/main/java/edu/harvard/iq/dataverse/DatasetVersionNoteValidator.java
new file mode 100644
index 00000000000..c086fed3b10
--- /dev/null
+++ b/src/main/java/edu/harvard/iq/dataverse/DatasetVersionNoteValidator.java
@@ -0,0 +1,47 @@
+/*
+ * To change this license header, choose License Headers in Project Properties.
+ * To change this template file, choose Tools | Templates
+ * and open the template in the editor.
+ */
+package edu.harvard.iq.dataverse;
+
+import edu.harvard.iq.dataverse.util.BundleUtil;
+import javax.validation.ConstraintValidator;
+import javax.validation.ConstraintValidatorContext;
+
+/**
+ *
+ * @author skraffmi
+ */
+public class DatasetVersionNoteValidator implements ConstraintValidator {
+
+ private String versionState;
+ private String versionNote;
+
+ @Override
+ public void initialize(ValidateVersionNote constraintAnnotation) {
+ versionState = constraintAnnotation.versionState();
+ versionNote = constraintAnnotation.versionNote();
+ }
+
+
+ @Override
+ public boolean isValid(DatasetVersion value, ConstraintValidatorContext context) {
+
+ if (versionState.equals(DatasetVersion.VersionState.DEACCESSIONED) && versionNote.isEmpty()){
+ if (context != null) {
+ context.buildConstraintViolationWithTemplate(value + " " + BundleUtil.getStringFromBundle("file.deaccessionDialog.dialog.textForReason.error")).addConstraintViolation();
+ }
+ return false;
+ }
+ if (versionState.equals(DatasetVersion.VersionState.DEACCESSIONED) && versionNote.length() > DatasetVersion.VERSION_NOTE_MAX_LENGTH){
+ if (context != null) {
+ context.buildConstraintViolationWithTemplate(value + " " + BundleUtil.getStringFromBundle("file.deaccessionDialog.dialog.limitChar.error")).addConstraintViolation();
+ }
+ return false;
+ }
+ return true;
+
+ }
+
+}
diff --git a/src/main/java/edu/harvard/iq/dataverse/DatasetVersionServiceBean.java b/src/main/java/edu/harvard/iq/dataverse/DatasetVersionServiceBean.java
index fe392f5132f..26439db91f0 100644
--- a/src/main/java/edu/harvard/iq/dataverse/DatasetVersionServiceBean.java
+++ b/src/main/java/edu/harvard/iq/dataverse/DatasetVersionServiceBean.java
@@ -5,13 +5,18 @@
import edu.harvard.iq.dataverse.search.IndexServiceBean;
import edu.harvard.iq.dataverse.authorization.AuthenticationServiceBean;
import edu.harvard.iq.dataverse.authorization.users.AuthenticatedUser;
+import static edu.harvard.iq.dataverse.batch.jobs.importer.filesystem.FileRecordJobListener.SEP;
+import edu.harvard.iq.dataverse.batch.util.LoggingUtil;
import edu.harvard.iq.dataverse.search.SolrSearchResult;
import edu.harvard.iq.dataverse.settings.SettingsServiceBean;
import edu.harvard.iq.dataverse.util.BundleUtil;
+import edu.harvard.iq.dataverse.util.MarkupChecker;
import edu.harvard.iq.dataverse.util.SystemConfig;
+import java.text.SimpleDateFormat;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collections;
+import java.util.Date;
import java.util.HashMap;
import java.util.Iterator;
import java.util.List;
@@ -39,6 +44,8 @@
public class DatasetVersionServiceBean implements java.io.Serializable {
private static final Logger logger = Logger.getLogger(DatasetVersionServiceBean.class.getCanonicalName());
+
+ private static final SimpleDateFormat logFormatter = new SimpleDateFormat("yyyy-MM-dd'T'HH-mm-ss");
@EJB
DatasetServiceBean datasetService;
@@ -99,7 +106,7 @@ public String getDifferentVersionMessage(){
if (DatasetVersionServiceBean.this.isVersionAskingForDraft(this.requestedVersion)){
userMsg = BundleUtil.getStringFromBundle("file.viewDiffDialog.msg.draftNotFound");
}else{
- userMsg = BundleUtil.getStringFromBundle("file.viewDiffDialog.msg.versionNotFound", Arrays.asList(this.requestedVersion));
+ userMsg = BundleUtil.getStringFromBundle("file.viewDiffDialog.msg.versionNotFound", Arrays.asList(MarkupChecker.escapeHtml(this.requestedVersion)));
}
if (DatasetVersionServiceBean.this.isVersionAskingForDraft(this.actualVersion)){
@@ -146,7 +153,6 @@ public DatasetVersion find(Object pk) {
}
public DatasetVersion findByFriendlyVersionNumber(Long datasetId, String friendlyVersionNumber) {
- //FIXME: this logic doesn't work
Long majorVersionNumber = null;
Long minorVersionNumber = null;
@@ -163,7 +169,6 @@ public DatasetVersion findByFriendlyVersionNumber(Long datasetId, String friendl
} catch (NumberFormatException n) {
return null;
}
-
if (minorVersionNumber != null) {
String queryStr = "SELECT v from DatasetVersion v where v.dataset.id = :datasetId and v.versionNumber= :majorVersionNumber and v.minorVersionNumber= :minorVersionNumber";
DatasetVersion foundDatasetVersion = null;
@@ -178,17 +183,13 @@ public DatasetVersion findByFriendlyVersionNumber(Long datasetId, String friendl
// DO nothing, just return null.
}
return foundDatasetVersion;
-
}
-
- if (majorVersionNumber == null && minorVersionNumber == null) {
+ if (majorVersionNumber == null && minorVersionNumber == null) {
return null;
-
}
if (majorVersionNumber != null && minorVersionNumber == null) {
-
try {
TypedQuery typedQuery = em.createQuery("SELECT v from DatasetVersion v where v.dataset.id = :datasetId and v.versionNumber= :majorVersionNumber", DatasetVersion.class);
typedQuery.setParameter("datasetId", datasetId);
@@ -204,18 +205,16 @@ public DatasetVersion findByFriendlyVersionNumber(Long datasetId, String friendl
}
}
}
-
return retVal;
-
} catch (javax.persistence.NoResultException e) {
logger.warning("no ds version found: " + datasetId + " " + friendlyVersionNumber);
// DO nothing, just return null.
}
}
-
return null;
}
+
/**
* Parse a Persistent Id and return as 3 strings.
@@ -796,6 +795,19 @@ private void assignDatasetThumbnailByNativeQuery(Long versionId, Long dataFileId
}
}
+ public void writeEditVersionLog(DatasetVersionDifference dvd, AuthenticatedUser au) {
+
+ String logDir = System.getProperty("com.sun.aas.instanceRoot") + SEP + "logs" + SEP + "edit-drafts" + SEP;
+ String identifier = dvd.getOriginalVersion().getDataset().getIdentifier();
+ identifier = identifier.substring(identifier.indexOf("/") + 1);
+ String datasetId = dvd.getOriginalVersion().getDataset().getId().toString();
+ String summary = au.getFirstName() + " " + au.getLastName() + " (" + au.getIdentifier() + ") updated " + dvd.getEditSummaryForLog();
+ String logTimestamp = logFormatter.format(new Date());
+ String fileName = "/edit-draft-" + datasetId + "-" + identifier + "-" + logTimestamp + ".txt";
+ LoggingUtil.saveLogFile(summary, logDir, fileName);
+
+ }
+
public void populateDatasetSearchCard(SolrSearchResult solrSearchResult) {
Long dataverseId = Long.parseLong(solrSearchResult.getParent().get("id"));
Long datasetVersionId = solrSearchResult.getDatasetVersionId();
diff --git a/src/main/java/edu/harvard/iq/dataverse/DataverseLocaleBean.java b/src/main/java/edu/harvard/iq/dataverse/DataverseLocaleBean.java
deleted file mode 100644
index d771990be02..00000000000
--- a/src/main/java/edu/harvard/iq/dataverse/DataverseLocaleBean.java
+++ /dev/null
@@ -1,119 +0,0 @@
-package edu.harvard.iq.dataverse;
-
-import edu.harvard.iq.dataverse.settings.SettingsServiceBean;
-import java.io.IOException;
-import java.io.Serializable;
-import java.util.*;
-import java.util.logging.Logger;
-
-import javax.faces.context.FacesContext;
-import javax.inject.Inject;
-import javax.inject.Named;
-import javax.servlet.http.HttpServletRequest;
-
-import org.json.*;
-
-@Named
-@javax.enterprise.context.SessionScoped
-public class DataverseLocaleBean implements Serializable {
-
- private static final Logger logger = Logger.getLogger(DataverseLocaleBean.class.getCanonicalName());
-
- @Inject
- SettingsWrapper settingsWrapper;
-
- {
- //Noticed that the NullPointerException was thrown from FacesContext.getCurrentInstance() while running the testcases(mvn:package).
- //Reason: the FacesContext is not initialized until the app starts. So, added the below if-condition
- if(FacesContext.getCurrentInstance() == null) {
- localeCode = "en";
- }
- else if (FacesContext.getCurrentInstance().getViewRoot() == null ) {
- localeCode = FacesContext.getCurrentInstance().getExternalContext().getRequestLocale().getLanguage();
- }
- else if (FacesContext.getCurrentInstance().getViewRoot().getLocale().getLanguage() == "en_US") {
- localeCode = "en";
- }
- else {
- localeCode = FacesContext.getCurrentInstance().getViewRoot().getLocale().getLanguage();
- }
- }
-
-
-
- // Map from locale to display name eg en -> English
- private Map dataverseLocales;
-
- private String localeCode;
-
- public void init() {
- dataverseLocales = new LinkedHashMap<>();
- try {
- JSONArray entries = new JSONArray(settingsWrapper.getValueForKey(SettingsServiceBean.Key.Languages, "[]"));
- for (Object obj : entries) {
- JSONObject entry = (JSONObject) obj;
- String locale = entry.getString("locale");
- String title = entry.getString("title");
-
- dataverseLocales.put(locale, title);
- }
- } catch (JSONException e) {
- // TODO Auto-generated catch block
- e.printStackTrace();
- }
- //localeCode = dataverseLocales.keySet().iterator().next();
- /*if (FacesContext.getCurrentInstance().getViewRoot().getLocale().getLanguage() == "en_US") {
- localeCode = "en";
- } else {
- localeCode = FacesContext.getCurrentInstance().getViewRoot().getLocale().getLanguage();
- }*/
- }
-
- public Map getDataverseLocales(){
- return dataverseLocales;
- }
-
- public boolean useLocale() {
- if (dataverseLocales == null) {
- init();
- }
- return dataverseLocales.size() > 1;
- }
-
- public String getLocaleCode() {
- if (localeCode == null) {
- init();
- }
- return localeCode;
- }
-
- public void setLocaleCode(String localeCode) {
- this.localeCode = localeCode;
- }
-
- public String getLocaleTitle() {
- if (dataverseLocales == null) {
- init();
- }
- return dataverseLocales.get(localeCode);
- }
-
- public void countryLocaleCodeChanged(String code) {
- if (dataverseLocales == null) {
- init();
- }
- localeCode = code;
- FacesContext.getCurrentInstance()
- .getViewRoot().setLocale(new Locale(dataverseLocales.get(code)));
- try {
- String url = ((HttpServletRequest) FacesContext.getCurrentInstance()
- .getExternalContext().getRequest()).getHeader("referer");
- FacesContext.getCurrentInstance().getExternalContext().redirect(url);
-
- } catch (IOException ioe) {
- ioe.printStackTrace();
- }
-
- }
-
-}
diff --git a/src/main/java/edu/harvard/iq/dataverse/DataversePage.java b/src/main/java/edu/harvard/iq/dataverse/DataversePage.java
index 0bd67f0f24e..ccac1b35e76 100644
--- a/src/main/java/edu/harvard/iq/dataverse/DataversePage.java
+++ b/src/main/java/edu/harvard/iq/dataverse/DataversePage.java
@@ -46,6 +46,7 @@
import javax.faces.model.SelectItem;
import org.apache.commons.lang.StringEscapeUtils;
import org.apache.commons.lang.StringUtils;
+import org.primefaces.PrimeFaces;
import org.primefaces.event.TransferEvent;
/**
@@ -506,6 +507,7 @@ public void updateInclude(Long mdbId, long dsftId) {
}
}
}
+ PrimeFaces.current().executeScript("scrollAfterUpdate();");
}
public List resetSelectItems(DatasetFieldType typeIn) {
diff --git a/src/main/java/edu/harvard/iq/dataverse/DataverseSession.java b/src/main/java/edu/harvard/iq/dataverse/DataverseSession.java
index 7fc80542daa..2b9f8cbbf60 100644
--- a/src/main/java/edu/harvard/iq/dataverse/DataverseSession.java
+++ b/src/main/java/edu/harvard/iq/dataverse/DataverseSession.java
@@ -6,10 +6,16 @@
import edu.harvard.iq.dataverse.actionlogging.ActionLogServiceBean;
import edu.harvard.iq.dataverse.authorization.users.GuestUser;
import edu.harvard.iq.dataverse.authorization.users.User;
+import java.io.IOException;
import java.io.Serializable;
+import java.util.Locale;
+import java.util.logging.Logger;
import javax.ejb.EJB;
import javax.enterprise.context.SessionScoped;
+import javax.faces.context.FacesContext;
+import javax.inject.Inject;
import javax.inject.Named;
+import javax.servlet.http.HttpServletRequest;
/**
*
@@ -31,6 +37,11 @@ public class DataverseSession implements Serializable{
@EJB
ActionLogServiceBean logSvc;
+ @Inject
+ SettingsWrapper settingsWrapper;
+
+ private static final Logger logger = Logger.getLogger(DataverseSession.class.getCanonicalName());
+
private boolean statusDismissed = false;
public User getUser() {
@@ -60,5 +71,68 @@ public void setStatusDismissed(boolean status) {
public StaticPermissionQuery on( Dataverse d ) {
return permissionsService.userOn(user, d);
}
+
+ // Language Locale methods:
+
+ private String localeCode;
+
+ public String getLocaleCode() {
+ if (localeCode == null) {
+ initLocale();
+ }
+ return localeCode;
+ }
+
+ public void setLocaleCode(String localeCode) {
+ this.localeCode = localeCode;
+ }
+
+ public String getLocaleTitle() {
+ if (localeCode == null) {
+ initLocale();
+ }
+ return settingsWrapper.getConfiguredLocales().get(localeCode);
+ }
+
+ public void initLocale() {
+
+ if(FacesContext.getCurrentInstance() == null) {
+ localeCode = "en";
+ }
+ else if (FacesContext.getCurrentInstance().getViewRoot() == null ) {
+ localeCode = FacesContext.getCurrentInstance().getExternalContext().getRequestLocale().getLanguage();
+ }
+ else if (FacesContext.getCurrentInstance().getViewRoot().getLocale().getLanguage().equals("en_US")) {
+ localeCode = "en";
+ }
+ else {
+ localeCode = FacesContext.getCurrentInstance().getViewRoot().getLocale().getLanguage();
+ }
+
+ logger.fine("init: locale set to "+localeCode);
+ }
+
+ public void updateLocaleInViewRootAndRedirect(String code) {
+
+ localeCode = code;
+ FacesContext.getCurrentInstance().getViewRoot().setLocale(new Locale(code));
+ try {
+ String url = ((HttpServletRequest) FacesContext.getCurrentInstance().getExternalContext().getRequest()).getHeader("referer");
+ FacesContext.getCurrentInstance().getExternalContext().redirect(url);
+ } catch (IOException ioe) {
+ ioe.printStackTrace();
+ }
+ }
+
+ public void updateLocaleInViewRoot() {
+ if (localeCode != null
+ && FacesContext.getCurrentInstance() != null
+ && FacesContext.getCurrentInstance().getViewRoot() != null
+ && !localeCode.equals(FacesContext.getCurrentInstance().getViewRoot().getLocale().getLanguage())) {
+ FacesContext.getCurrentInstance().getViewRoot().setLocale(new Locale(localeCode));
+ }
+ }
+
+
}
diff --git a/src/main/java/edu/harvard/iq/dataverse/EditDatafilesPage.java b/src/main/java/edu/harvard/iq/dataverse/EditDatafilesPage.java
index e4cd115bccb..29896eb5a97 100644
--- a/src/main/java/edu/harvard/iq/dataverse/EditDatafilesPage.java
+++ b/src/main/java/edu/harvard/iq/dataverse/EditDatafilesPage.java
@@ -146,6 +146,7 @@ public enum FileEditMode {
private List newFiles = new ArrayList<>();;
private List uploadedFiles = new ArrayList<>();;
private DatasetVersion workingVersion;
+ private DatasetVersion clone;
private String dropBoxSelection = "";
private String displayCitation;
private boolean datasetUpdateRequired = false;
@@ -474,7 +475,7 @@ public String init() {
workingVersion = dataset.getEditVersion();
-
+ clone = workingVersion.cloneDatasetVersion();
if (workingVersion == null || !workingVersion.isDraft()) {
// Sorry, we couldn't find/obtain a draft version for this dataset!
return permissionsWrapper.notFound();
@@ -1220,7 +1221,7 @@ public String save() {
Command cmd;
try {
- cmd = new UpdateDatasetVersionCommand(dataset, dvRequestService.getDataverseRequest(), filesToBeDeleted);
+ cmd = new UpdateDatasetVersionCommand(dataset, dvRequestService.getDataverseRequest(), filesToBeDeleted, clone);
((UpdateDatasetVersionCommand) cmd).setValidateLenient(true);
dataset = commandEngine.submit(cmd);
diff --git a/src/main/java/edu/harvard/iq/dataverse/FileDownloadServiceBean.java b/src/main/java/edu/harvard/iq/dataverse/FileDownloadServiceBean.java
index d859a5f057e..9c06e3f2a43 100644
--- a/src/main/java/edu/harvard/iq/dataverse/FileDownloadServiceBean.java
+++ b/src/main/java/edu/harvard/iq/dataverse/FileDownloadServiceBean.java
@@ -235,6 +235,10 @@ public void explore(GuestbookResponse guestbookResponse, FileMetadata fmd, Exter
dataFile = guestbookResponse.getDataFile();
}
}
+ //For tools to get the dataset and datasetversion ids, we need a full DataFile object (not a findCheapAndEasy() copy)
+ if(dataFile.getFileMetadata()==null) {
+ dataFile=datafileService.find(dataFile.getId());
+ }
ExternalToolHandler externalToolHandler = new ExternalToolHandler(externalTool, dataFile, apiToken);
// Back when we only had TwoRavens, the downloadType was always "Explore". Now we persist the name of the tool (i.e. "TwoRavens", "Data Explorer", etc.)
guestbookResponse.setDownloadtype(externalTool.getDisplayName());
@@ -411,6 +415,7 @@ public void downloadCitationBibtex(FileMetadata fileMetadata, Dataset dataset, b
//SEK 12/3/2018 changing this to open the json in a new tab.
FacesContext ctx = FacesContext.getCurrentInstance();
HttpServletResponse response = (HttpServletResponse) ctx.getExternalContext().getResponse();
+ // FIXME: BibTeX isn't JSON. Firefox will try to parse it and report "SyntaxError".
response.setContentType("application/json");
String fileNameString;
diff --git a/src/main/java/edu/harvard/iq/dataverse/FilePage.java b/src/main/java/edu/harvard/iq/dataverse/FilePage.java
index 4a7b3ff68b5..1fa2b986e31 100644
--- a/src/main/java/edu/harvard/iq/dataverse/FilePage.java
+++ b/src/main/java/edu/harvard/iq/dataverse/FilePage.java
@@ -837,7 +837,7 @@ public String getPublicDownloadUrl() {
e.printStackTrace();
}
- return FileUtil.getPublicDownloadUrl(systemConfig.getDataverseSiteUrl(), persistentId);
+ return FileUtil.getPublicDownloadUrl(systemConfig.getDataverseSiteUrl(), persistentId, fileId);
}
public List getConfigureTools() {
diff --git a/src/main/java/edu/harvard/iq/dataverse/HarvestingClientsPage.java b/src/main/java/edu/harvard/iq/dataverse/HarvestingClientsPage.java
index 826cb2b37d5..c0d7ed01c8e 100644
--- a/src/main/java/edu/harvard/iq/dataverse/HarvestingClientsPage.java
+++ b/src/main/java/edu/harvard/iq/dataverse/HarvestingClientsPage.java
@@ -23,6 +23,7 @@
import java.util.ArrayList;
import java.util.Arrays;
import java.util.List;
+import java.util.Locale;
import java.util.logging.Level;
import java.util.logging.Logger;
import java.util.regex.Pattern;
diff --git a/src/main/java/edu/harvard/iq/dataverse/ManageTemplatesPage.java b/src/main/java/edu/harvard/iq/dataverse/ManageTemplatesPage.java
index f4ece9cab6b..a4f5d27c38e 100644
--- a/src/main/java/edu/harvard/iq/dataverse/ManageTemplatesPage.java
+++ b/src/main/java/edu/harvard/iq/dataverse/ManageTemplatesPage.java
@@ -93,7 +93,7 @@ public String init() {
templates.add(ct);
}
if (!templates.isEmpty()){
- JH.addMessage(FacesMessage.SEVERITY_INFO, BundleUtil.getStringFromBundle("dataset.manageTemplates.info.message.notEmptyTable"));
+ JH.addMessage(FacesMessage.SEVERITY_INFO, BundleUtil.getStringFromBundle("dataset.message.manageTemplates.label"), BundleUtil.getStringFromBundle("dataset.message.manageTemplates.message"));
}
return null;
}
diff --git a/src/main/java/edu/harvard/iq/dataverse/Metric.java b/src/main/java/edu/harvard/iq/dataverse/Metric.java
index ebcde546002..b1b1a276ceb 100644
--- a/src/main/java/edu/harvard/iq/dataverse/Metric.java
+++ b/src/main/java/edu/harvard/iq/dataverse/Metric.java
@@ -5,6 +5,7 @@
*/
package edu.harvard.iq.dataverse;
+import java.io.IOException;
import java.io.Serializable;
import java.sql.Timestamp;
import java.util.Date;
@@ -17,7 +18,6 @@
import javax.persistence.Table;
import javax.persistence.Temporal;
import javax.persistence.TemporalType;
-import javax.persistence.Transient;
/**
*
@@ -34,34 +34,36 @@ public class Metric implements Serializable {
@Column(nullable = false)
private int id;
- @Column(nullable = false, unique = true)
- private String metricName;
+ @Column(nullable = false)
+ private String name;
@Column(columnDefinition = "TEXT", nullable = false)
- private String metricValue;
+ private String valueJson;
+
+ @Column(columnDefinition = "TEXT", nullable = true)
+ private String dataLocation;
+
+ @Column(columnDefinition = "TEXT", nullable = true)
+ private String dayString;
@Temporal(value = TemporalType.TIMESTAMP)
@Column(nullable = false)
private Date lastCalledDate;
- @Transient
- private static final String separator = "_";
-
@Deprecated
public Metric() {
}
//For monthly and day metrics
- public Metric(String metricTitle, String dayString, String metricValue) {
- this.metricName = generateMetricName(metricTitle, dayString);
- this.metricValue = metricValue;
- this.lastCalledDate = new Timestamp(new Date().getTime());
- }
-
- //For all-time metrics
- public Metric(String metricName, String metricValue) {
- this.metricName = metricName;
- this.metricValue = metricValue;
+
+ public Metric(String name, String dayString, String dataLocation, String value) throws IOException {
+ if(null == name || null == value) {
+ throw new IOException("A created metric must have a metricName and metricValue");
+ }
+ this.name = name;
+ this.valueJson = value;
+ this.dataLocation = dataLocation;
+ this.dayString = dayString;
this.lastCalledDate = new Timestamp(new Date().getTime());
}
@@ -79,30 +81,30 @@ public void setId(int id) {
this.id = id;
}
- public String getMetricDateString() {
- return metricName.substring(metricName.indexOf(separator) + 1);
+ public String getDateString() {
+ return dayString;
}
- public String getMetricTitle() {
- int monthSeperatorIndex = metricName.indexOf(separator);
- if (monthSeperatorIndex >= 0) {
- return metricName.substring(0, monthSeperatorIndex);
- }
- return metricName;
+ public String getDataLocation() {
+ return dataLocation;
+ }
+
+ public String getName() {
+ return name;
}
/**
- * @return the metricValue
+ * @return the valueJson
*/
- public String getMetricValue() {
- return metricValue;
+ public String getValueJson() {
+ return valueJson;
}
/**
- * @param metricValue the metricValue to set
+ * @param metricValue the valueJson to set
*/
- public void setMetricValue(String metricValue) {
- this.metricValue = metricValue;
+ public void setValueJson(String metricValue) {
+ this.valueJson = metricValue;
}
/**
@@ -119,14 +121,4 @@ public void setLastCalledDate(Date calledDate) {
this.lastCalledDate = calledDate;
}
- public static String generateMetricName(String title, String dateString) {
- if (title.contains(separator) || dateString.contains(separator)) {
- throw new IllegalArgumentException("Metric title or date contains character reserved for seperator");
- }
- if (separator.contains("-")) {
- throw new IllegalArgumentException("Metric seperator cannot be '-', value reserved for dates");
- }
- return title + separator + dateString;
- }
-
}
diff --git a/src/main/java/edu/harvard/iq/dataverse/SettingsWrapper.java b/src/main/java/edu/harvard/iq/dataverse/SettingsWrapper.java
index f3efc0a1e7b..b766e2605bf 100644
--- a/src/main/java/edu/harvard/iq/dataverse/SettingsWrapper.java
+++ b/src/main/java/edu/harvard/iq/dataverse/SettingsWrapper.java
@@ -13,11 +13,15 @@
import edu.harvard.iq.dataverse.util.StringUtil;
import edu.harvard.iq.dataverse.util.SystemConfig;
import java.util.HashMap;
+import java.util.LinkedHashMap;
import java.util.Map;
import javax.ejb.EJB;
import javax.faces.view.ViewScoped;
import javax.inject.Named;
import javax.mail.internet.InternetAddress;
+import org.json.JSONArray;
+import org.json.JSONException;
+import org.json.JSONObject;
/**
*
@@ -195,6 +199,43 @@ public Boolean isHasDropBoxKey() {
return !getDropBoxKey().isEmpty();
}
+
+ // Language Locales Configuration:
+
+ // Map from locale to display name eg en -> English
+ private Map configuredLocales;
+
+ public boolean isLocalesConfigured() {
+ if (configuredLocales == null) {
+ initLocaleSettings();
+ }
+ return configuredLocales.size() > 1;
+ }
+ public Map getConfiguredLocales() {
+ if (configuredLocales == null) {
+ initLocaleSettings();
+ }
+ return configuredLocales;
+ }
+
+ private void initLocaleSettings() {
+
+ configuredLocales = new LinkedHashMap<>();
+
+ try {
+ JSONArray entries = new JSONArray(getValueForKey(SettingsServiceBean.Key.Languages, "[]"));
+ for (Object obj : entries) {
+ JSONObject entry = (JSONObject) obj;
+ String locale = entry.getString("locale");
+ String title = entry.getString("title");
+
+ configuredLocales.put(locale, title);
+ }
+ } catch (JSONException e) {
+ //e.printStackTrace();
+ // do we want to know? - probably not
+ }
+ }
}
diff --git a/src/main/java/edu/harvard/iq/dataverse/URLValidator.java b/src/main/java/edu/harvard/iq/dataverse/URLValidator.java
new file mode 100644
index 00000000000..35173da5860
--- /dev/null
+++ b/src/main/java/edu/harvard/iq/dataverse/URLValidator.java
@@ -0,0 +1,42 @@
+package edu.harvard.iq.dataverse;
+
+import edu.harvard.iq.dataverse.util.BundleUtil;
+import java.net.MalformedURLException;
+import java.net.URL;
+import javax.validation.ConstraintValidator;
+import javax.validation.ConstraintValidatorContext;
+
+/**
+ *
+ * @author skraffmi
+ */
+public class URLValidator implements ConstraintValidator {
+
+ @Override
+ public void initialize(ValidateURL constraintAnnotation) {
+
+ }
+
+ @Override
+ public boolean isValid(String value, ConstraintValidatorContext context) {
+
+ boolean valid = isURLValid(value);
+ if (context != null && !valid) {
+ context.buildConstraintViolationWithTemplate(value + " " + BundleUtil.getStringFromBundle("url.invalid")).addConstraintViolation();
+ }
+ return valid;
+ }
+
+ public static boolean isURLValid(String value) {
+ if (value == null || value.isEmpty()) {
+ return true;
+ }
+ try {
+ URL url = new URL(value);
+ } catch (MalformedURLException e) {
+ return false;
+ }
+ return true;
+ }
+
+}
diff --git a/src/main/java/edu/harvard/iq/dataverse/ValidateURL.java b/src/main/java/edu/harvard/iq/dataverse/ValidateURL.java
new file mode 100644
index 00000000000..106876e423e
--- /dev/null
+++ b/src/main/java/edu/harvard/iq/dataverse/ValidateURL.java
@@ -0,0 +1,24 @@
+
+package edu.harvard.iq.dataverse;
+
+import java.lang.annotation.Documented;
+import static java.lang.annotation.ElementType.FIELD;
+import java.lang.annotation.Retention;
+import static java.lang.annotation.RetentionPolicy.RUNTIME;
+import java.lang.annotation.Target;
+import javax.validation.Constraint;
+import javax.validation.Payload;
+
+@Target({FIELD})
+@Retention(RUNTIME)
+@Constraint(validatedBy = {URLValidator.class})
+@Documented
+public @interface ValidateURL {
+
+ String message() default "Failed Validation for Validate URL";
+
+ Class[] groups() default {};
+
+ Class[] payload() default {};
+
+}
diff --git a/src/main/java/edu/harvard/iq/dataverse/ValidateVersionNote.java b/src/main/java/edu/harvard/iq/dataverse/ValidateVersionNote.java
new file mode 100644
index 00000000000..405a7feb52f
--- /dev/null
+++ b/src/main/java/edu/harvard/iq/dataverse/ValidateVersionNote.java
@@ -0,0 +1,39 @@
+/*
+ * To change this license header, choose License Headers in Project Properties.
+ * To change this template file, choose Tools | Templates
+ * and open the template in the editor.
+ */
+package edu.harvard.iq.dataverse;
+
+import java.lang.annotation.Documented;
+import static java.lang.annotation.ElementType.ANNOTATION_TYPE;
+import static java.lang.annotation.ElementType.TYPE;
+import java.lang.annotation.Retention;
+import static java.lang.annotation.RetentionPolicy.RUNTIME;
+import java.lang.annotation.Target;
+import javax.validation.Constraint;
+import javax.validation.Payload;
+
+/**
+ *
+ * @author skraffmi
+ */
+
+
+@Target({TYPE, ANNOTATION_TYPE})
+@Retention(RUNTIME)
+@Constraint(validatedBy = {DatasetVersionNoteValidator.class})
+@Documented
+public @interface ValidateVersionNote {
+
+ String message() default "Failed Validation for DatasetVersionNote";
+
+ Class[] groups() default {};
+
+ Class[] payload() default {};
+
+ String versionNote();
+
+ String versionState();
+
+}
diff --git a/src/main/java/edu/harvard/iq/dataverse/api/Access.java b/src/main/java/edu/harvard/iq/dataverse/api/Access.java
index 20ed63fa789..492e0ffb1fb 100644
--- a/src/main/java/edu/harvard/iq/dataverse/api/Access.java
+++ b/src/main/java/edu/harvard/iq/dataverse/api/Access.java
@@ -107,6 +107,7 @@
import javax.ws.rs.core.Response;
import static javax.ws.rs.core.Response.Status.BAD_REQUEST;
import javax.ws.rs.core.StreamingOutput;
+import static edu.harvard.iq.dataverse.util.json.JsonPrinter.json;
/*
Custom API exceptions [NOT YET IMPLEMENTED]
@@ -388,12 +389,14 @@ public String tabularDatafileMetadataDDI(@PathParam("fileId") String fileId, @Qu
DataFile dataFile = null;
- //httpHeaders.add("Content-disposition", "attachment; filename=\"dataverse_files.zip\"");
- //httpHeaders.add("Content-Type", "application/zip; name=\"dataverse_files.zip\"");
- response.setHeader("Content-disposition", "attachment; filename=\"dataverse_files.zip\"");
-
dataFile = findDataFileOrDieWrapper(fileId);
+ if (!dataFile.isTabularData()) {
+ throw new BadRequestException("tabular data required");
+ }
+
+ response.setHeader("Content-disposition", "attachment; filename=\"dataverse_files.zip\"");
+
String fileName = dataFile.getFileMetadata().getLabel().replaceAll("\\.tab$", "-ddi.xml");
response.setHeader("Content-disposition", "attachment; filename=\""+fileName+"\"");
response.setHeader("Content-Type", "application/xml; name=\""+fileName+"\"");
@@ -476,7 +479,7 @@ public DownloadInstance tabularDatafileMetadataPreprocessed(@PathParam("fileId")
if (df.isTabularData()) {
dInfo.addServiceAvailable(new OptionalAccessService("preprocessed", "application/json", "format=prep", "Preprocessed data in JSON"));
} else {
- throw new ServiceUnavailableException("Preprocessed Content Metadata requested on a non-tabular data file.");
+ throw new BadRequestException("tabular data required");
}
DownloadInstance downloadInstance = new DownloadInstance(dInfo);
if (downloadInstance.checkIfServiceSupportedAndSetConverter("format", "prep")) {
diff --git a/src/main/java/edu/harvard/iq/dataverse/api/Admin.java b/src/main/java/edu/harvard/iq/dataverse/api/Admin.java
index 39e41ebe228..b0c425c85af 100644
--- a/src/main/java/edu/harvard/iq/dataverse/api/Admin.java
+++ b/src/main/java/edu/harvard/iq/dataverse/api/Admin.java
@@ -4,23 +4,23 @@
import edu.harvard.iq.dataverse.DataFileServiceBean;
import edu.harvard.iq.dataverse.Dataset;
import edu.harvard.iq.dataverse.DatasetServiceBean;
+import edu.harvard.iq.dataverse.DatasetVersion;
+import edu.harvard.iq.dataverse.DatasetVersionServiceBean;
import edu.harvard.iq.dataverse.Dataverse;
+import edu.harvard.iq.dataverse.DataverseRequestServiceBean;
import edu.harvard.iq.dataverse.DataverseSession;
import edu.harvard.iq.dataverse.DvObject;
import edu.harvard.iq.dataverse.EMailValidator;
-import edu.harvard.iq.dataverse.RoleAssignment;
+import edu.harvard.iq.dataverse.EjbDataverseEngine;
import edu.harvard.iq.dataverse.GlobalId;
import edu.harvard.iq.dataverse.UserServiceBean;
import edu.harvard.iq.dataverse.actionlogging.ActionLogRecord;
-import static edu.harvard.iq.dataverse.api.AbstractApiBean.error;
import edu.harvard.iq.dataverse.api.dto.RoleDTO;
import edu.harvard.iq.dataverse.authorization.AuthenticatedUserDisplayInfo;
import edu.harvard.iq.dataverse.authorization.AuthenticationProvider;
-import edu.harvard.iq.dataverse.authorization.DataverseRole;
import edu.harvard.iq.dataverse.authorization.UserIdentifier;
import edu.harvard.iq.dataverse.authorization.exceptions.AuthenticationProviderFactoryNotFoundException;
import edu.harvard.iq.dataverse.authorization.exceptions.AuthorizationSetupException;
-import edu.harvard.iq.dataverse.authorization.groups.Group;
import edu.harvard.iq.dataverse.authorization.groups.GroupServiceBean;
import edu.harvard.iq.dataverse.authorization.providers.AuthenticationProviderRow;
import edu.harvard.iq.dataverse.authorization.providers.builtin.BuiltinUser;
@@ -34,6 +34,7 @@
import edu.harvard.iq.dataverse.confirmemail.ConfirmEmailInitResponse;
import edu.harvard.iq.dataverse.dataaccess.DataAccessOption;
import edu.harvard.iq.dataverse.dataaccess.StorageIO;
+import edu.harvard.iq.dataverse.engine.command.impl.AbstractSubmitToArchiveCommand;
import edu.harvard.iq.dataverse.engine.command.impl.PublishDataverseCommand;
import edu.harvard.iq.dataverse.settings.Setting;
import javax.json.Json;
@@ -73,19 +74,19 @@
import edu.harvard.iq.dataverse.dataset.DatasetThumbnail;
import edu.harvard.iq.dataverse.dataset.DatasetUtil;
import edu.harvard.iq.dataverse.engine.command.DataverseRequest;
+import edu.harvard.iq.dataverse.engine.command.exception.CommandException;
import edu.harvard.iq.dataverse.engine.command.impl.RegisterDvObjectCommand;
import edu.harvard.iq.dataverse.ingest.IngestServiceBean;
import edu.harvard.iq.dataverse.settings.SettingsServiceBean;
import edu.harvard.iq.dataverse.userdata.UserListMaker;
import edu.harvard.iq.dataverse.userdata.UserListResult;
+import edu.harvard.iq.dataverse.util.ArchiverUtil;
import edu.harvard.iq.dataverse.util.BundleUtil;
import edu.harvard.iq.dataverse.util.FileUtil;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Date;
-import java.util.HashMap;
-import java.util.ResourceBundle;
import javax.inject.Inject;
import javax.persistence.Query;
import javax.ws.rs.QueryParam;
@@ -115,8 +116,16 @@ public class Admin extends AbstractApiBean {
DataFileServiceBean fileService;
@EJB
DatasetServiceBean datasetService;
+ @EJB
+ DatasetVersionServiceBean datasetversionService;
+ @Inject
+ DataverseRequestServiceBean dvRequestService;
+ @EJB
+ EjbDataverseEngine commandEngine;
@EJB
GroupServiceBean groupService;
+ @EJB
+ SettingsServiceBean settingsService;
// Make the session available
@Inject
@@ -1077,7 +1086,7 @@ public Response validatePassword(String password) {
public Response isOrcidEnabled() {
return authSvc.isOrcidEnabled() ? ok("Orcid is enabled") : ok("no orcid for you.");
}
-
+
@POST
@Path("{id}/reregisterHDLToPID")
public Response reregisterHdlToPID(@PathParam("id") String id) {
@@ -1240,7 +1249,7 @@ public Response updateHashValues(@PathParam("alg") String alg, @QueryParam("num"
}
if (in == null)
logger.warning("Cannot retrieve file.");
- String currentChecksum = FileUtil.CalculateChecksum(in, df.getChecksumType());
+ String currentChecksum = FileUtil.calculateChecksum(in, df.getChecksumType());
if (currentChecksum.equals(df.getChecksumValue())) {
logger.fine("Current checksum for datafile: " + df.getFileMetadata().getLabel() + ", "
+ df.getIdentifier() + " is valid");
@@ -1254,7 +1263,7 @@ public Response updateHashValues(@PathParam("alg") String alg, @QueryParam("num"
}
if (in2 == null)
logger.warning("Cannot retrieve file to calculate new checksum.");
- String newChecksum = FileUtil.CalculateChecksum(in2, cType);
+ String newChecksum = FileUtil.calculateChecksum(in2, cType);
df.setChecksumType(cType);
df.setChecksumValue(newChecksum);
@@ -1294,7 +1303,48 @@ public Response updateHashValues(@PathParam("alg") String alg, @QueryParam("num"
return ok("Datafile rehashing complete." + successes + " of " + rehashed + " files successfully rehashed.");
}
-
+ @GET
+ @Path("/submitDataVersionToArchive/{id}/{version}")
+ public Response submitDatasetVersionToArchive(@PathParam("id") String dsid, @PathParam("version") String versionNumber) {
+
+ try {
+ AuthenticatedUser au = findAuthenticatedUserOrDie();
+ session.setUser(au);
+ Dataset ds = findDatasetOrDie(dsid);
+
+ DatasetVersion dv = datasetversionService.findByFriendlyVersionNumber(ds.getId(), versionNumber);
+ if (dv.getArchivalCopyLocation() == null) {
+ String className = settingsService.getValueForKey(SettingsServiceBean.Key.ArchiverClassName);
+ AbstractSubmitToArchiveCommand cmd = ArchiverUtil.createSubmitToArchiveCommand(className, dvRequestService.getDataverseRequest(), dv);
+ if (cmd != null) {
+ new Thread(new Runnable() {
+ public void run() {
+ try {
+ DatasetVersion dv = commandEngine.submit(cmd);
+ if (dv.getArchivalCopyLocation() != null) {
+ logger.info("DatasetVersion id=" + ds.getGlobalId().toString() + " v" + versionNumber + " submitted to Archive at: "
+ + dv.getArchivalCopyLocation());
+ } else {
+ logger.severe("Error submitting version due to conflict/error at Archive");
+ }
+ } catch (CommandException ex) {
+ logger.log(Level.SEVERE, "Unexpected Exception calling submit archive command", ex);
+ }
+ }
+ }).start();
+ return ok("Archive submission using " + cmd.getClass().getCanonicalName() + " started. Processing can take significant time for large datasets. View log and/or check archive for results.");
+ } else {
+ logger.log(Level.SEVERE, "Could not find Archiver class: " + className);
+ return error(Status.INTERNAL_SERVER_ERROR, "Could not find Archiver class: " + className);
+ }
+ } else {
+ return error(Status.BAD_REQUEST, "Version already archived at: " + dv.getArchivalCopyLocation());
+ }
+ } catch (WrappedResponse e1) {
+ return error(Status.UNAUTHORIZED, "api key required");
+ }
+ }
+
@DELETE
@Path("/clearMetricsCache")
public Response clearMetricsCache() {
diff --git a/src/main/java/edu/harvard/iq/dataverse/api/Metrics.java b/src/main/java/edu/harvard/iq/dataverse/api/Metrics.java
index 6b77f7fa32c..2694252091f 100644
--- a/src/main/java/edu/harvard/iq/dataverse/api/Metrics.java
+++ b/src/main/java/edu/harvard/iq/dataverse/api/Metrics.java
@@ -2,13 +2,17 @@
import edu.harvard.iq.dataverse.Metric;
import edu.harvard.iq.dataverse.metrics.MetricsUtil;
+import java.util.Arrays;
import javax.json.JsonArrayBuilder;
import javax.json.JsonObjectBuilder;
import javax.ws.rs.GET;
import javax.ws.rs.Path;
import javax.ws.rs.PathParam;
+import javax.ws.rs.QueryParam;
+import javax.ws.rs.core.Context;
import javax.ws.rs.core.Response;
import static javax.ws.rs.core.Response.Status.BAD_REQUEST;
+import javax.ws.rs.core.UriInfo;
/**
* API endpoints for various metrics.
@@ -22,60 +26,75 @@
*/
@Path("info/metrics")
public class Metrics extends AbstractApiBean {
- /** Dataverses */
+ /** Dataverses */
+
@GET
@Path("dataverses")
- public Response getDataversesAllTime() {
- return getDataversesToMonth(MetricsUtil.getCurrentMonth());
+ public Response getDataversesAllTime(@Context UriInfo uriInfo) {
+ return getDataversesToMonth(uriInfo, MetricsUtil.getCurrentMonth());
}
@Deprecated //for better path
@GET
@Path("dataverses/toMonth")
- public Response getDataversesToMonthCurrent() {
- return getDataversesToMonth(MetricsUtil.getCurrentMonth());
+ public Response getDataversesToMonthCurrent(@Context UriInfo uriInfo) {
+ return getDataversesToMonth(uriInfo, MetricsUtil.getCurrentMonth());
}
@GET
@Path("dataverses/toMonth/{yyyymm}")
- public Response getDataversesToMonth(@PathParam("yyyymm") String yyyymm) {
+ public Response getDataversesToMonth(@Context UriInfo uriInfo, @PathParam("yyyymm") String yyyymm) {
+ try {
+ errorIfUnrecongizedQueryParamPassed(uriInfo, new String[]{""});
+ } catch (IllegalArgumentException ia) {
+ return allowCors(error(BAD_REQUEST, ia.getLocalizedMessage()));
+ }
+
String metricName = "dataversesToMonth";
try {
String sanitizedyyyymm = MetricsUtil.sanitizeYearMonthUserInput(yyyymm);
- String jsonString = metricsSvc.returnUnexpiredCacheMonthly(metricName, sanitizedyyyymm);
+ String jsonString = metricsSvc.returnUnexpiredCacheMonthly(metricName, sanitizedyyyymm, null);
if (null == jsonString) { //run query and save
Long count = metricsSvc.dataversesToMonth(sanitizedyyyymm);
JsonObjectBuilder jsonObjBuilder = MetricsUtil.countToJson(count);
jsonString = jsonObjBuilder.build().toString();
- metricsSvc.save(new Metric(metricName, sanitizedyyyymm, jsonString), true); //if not using cache save new
+ metricsSvc.save(new Metric(metricName, sanitizedyyyymm, null, jsonString));
}
return allowCors(ok(MetricsUtil.stringToJsonObjectBuilder(jsonString)));
+ //TODO: Eventually the catch in each endpoint should be more specific
+ // and more general errors should be logged.
} catch (Exception ex) {
return allowCors(error(BAD_REQUEST, ex.getLocalizedMessage()));
}
}
-
+
@GET
@Path("dataverses/pastDays/{days}")
- public Response getDataversesPastDays(@PathParam("days") int days) {
+ public Response getDataversesPastDays(@Context UriInfo uriInfo, @PathParam("days") int days) {
+ try {
+ errorIfUnrecongizedQueryParamPassed(uriInfo, new String[]{""});
+ } catch (IllegalArgumentException ia) {
+ return allowCors(error(BAD_REQUEST, ia.getLocalizedMessage()));
+ }
+
String metricName = "dataversesPastDays";
if(days < 1) {
return allowCors(error(BAD_REQUEST, "Invalid parameter for number of days."));
}
try {
- String jsonString = metricsSvc.returnUnexpiredCacheDayBased(metricName, String.valueOf(days));
+ String jsonString = metricsSvc.returnUnexpiredCacheDayBased(metricName, String.valueOf(days), null);
if (null == jsonString) { //run query and save
Long count = metricsSvc.dataversesPastDays(days);
JsonObjectBuilder jsonObjBuilder = MetricsUtil.countToJson(count);
jsonString = jsonObjBuilder.build().toString();
- metricsSvc.save(new Metric(metricName, String.valueOf(days), jsonString), true); //if not using cache save new
+ metricsSvc.save(new Metric(metricName, String.valueOf(days), null, jsonString));
}
return allowCors(ok(MetricsUtil.stringToJsonObjectBuilder(jsonString)));
@@ -87,16 +106,22 @@ public Response getDataversesPastDays(@PathParam("days") int days) {
@GET
@Path("dataverses/byCategory")
- public Response getDataversesByCategory() {
+ public Response getDataversesByCategory(@Context UriInfo uriInfo) {
+ try {
+ errorIfUnrecongizedQueryParamPassed(uriInfo, new String[]{""});
+ } catch (IllegalArgumentException ia) {
+ return allowCors(error(BAD_REQUEST, ia.getLocalizedMessage()));
+ }
+
String metricName = "dataversesByCategory";
try {
- String jsonArrayString = metricsSvc.returnUnexpiredCacheAllTime(metricName);
+ String jsonArrayString = metricsSvc.returnUnexpiredCacheAllTime(metricName, null);
if (null == jsonArrayString) { //run query and save
JsonArrayBuilder jsonArrayBuilder = MetricsUtil.dataversesByCategoryToJson(metricsSvc.dataversesByCategory());
jsonArrayString = jsonArrayBuilder.build().toString();
- metricsSvc.save(new Metric(metricName, jsonArrayString), false);
+ metricsSvc.save(new Metric(metricName, null, null, jsonArrayString));
}
return allowCors(ok(MetricsUtil.stringToJsonArrayBuilder(jsonArrayString)));
@@ -107,16 +132,22 @@ public Response getDataversesByCategory() {
@GET
@Path("dataverses/bySubject")
- public Response getDataversesBySubject() {
+ public Response getDataversesBySubject(@Context UriInfo uriInfo) {
+ try {
+ errorIfUnrecongizedQueryParamPassed(uriInfo, new String[]{""});
+ } catch (IllegalArgumentException ia) {
+ return allowCors(error(BAD_REQUEST, ia.getLocalizedMessage()));
+ }
+
String metricName = "dataversesBySubject";
try {
- String jsonArrayString = metricsSvc.returnUnexpiredCacheAllTime(metricName);
+ String jsonArrayString = metricsSvc.returnUnexpiredCacheAllTime(metricName, null);
if (null == jsonArrayString) { //run query and save
JsonArrayBuilder jsonArrayBuilder = MetricsUtil.dataversesBySubjectToJson(metricsSvc.dataversesBySubject());
jsonArrayString = jsonArrayBuilder.build().toString();
- metricsSvc.save(new Metric(metricName, jsonArrayString), false);
+ metricsSvc.save(new Metric(metricName, null, null, jsonArrayString));
}
return allowCors(ok(MetricsUtil.stringToJsonArrayBuilder(jsonArrayString)));
@@ -129,31 +160,38 @@ public Response getDataversesBySubject() {
@GET
@Path("datasets")
- public Response getDatasetsAllTime() {
- return getDatasetsToMonth(MetricsUtil.getCurrentMonth());
+ public Response getDatasetsAllTime(@Context UriInfo uriInfo, @QueryParam("dataLocation") String dataLocation) {
+ return getDatasetsToMonth(uriInfo, MetricsUtil.getCurrentMonth(), dataLocation);
}
@Deprecated //for better path
@GET
@Path("datasets/toMonth")
- public Response getDatasetsToMonthCurrent() {
- return getDatasetsToMonth(MetricsUtil.getCurrentMonth());
+ public Response getDatasetsToMonthCurrent(@Context UriInfo uriInfo, @QueryParam("dataLocation") String dataLocation) {
+ return getDatasetsToMonth(uriInfo, MetricsUtil.getCurrentMonth(), dataLocation);
}
@GET
@Path("datasets/toMonth/{yyyymm}")
- public Response getDatasetsToMonth(@PathParam("yyyymm") String yyyymm) {
+ public Response getDatasetsToMonth(@Context UriInfo uriInfo, @PathParam("yyyymm") String yyyymm, @QueryParam("dataLocation") String dataLocation) {
+ try {
+ errorIfUnrecongizedQueryParamPassed(uriInfo, new String[]{"dataLocation"});
+ } catch (IllegalArgumentException ia) {
+ return allowCors(error(BAD_REQUEST, ia.getLocalizedMessage()));
+ }
+
String metricName = "datasetsToMonth";
try {
String sanitizedyyyymm = MetricsUtil.sanitizeYearMonthUserInput(yyyymm);
- String jsonString = metricsSvc.returnUnexpiredCacheMonthly(metricName, sanitizedyyyymm);
+ String validDataLocation = MetricsUtil.validateDataLocationStringType(dataLocation);
+ String jsonString = metricsSvc.returnUnexpiredCacheMonthly(metricName, sanitizedyyyymm, validDataLocation);
if (null == jsonString) { //run query and save
- Long count = metricsSvc.datasetsToMonth(sanitizedyyyymm);
+ Long count = metricsSvc.datasetsToMonth(sanitizedyyyymm, validDataLocation);
JsonObjectBuilder jsonObjBuilder = MetricsUtil.countToJson(count);
jsonString = jsonObjBuilder.build().toString();
- metricsSvc.save(new Metric(metricName, sanitizedyyyymm, jsonString), true); //if not using cache save new
+ metricsSvc.save(new Metric(metricName, sanitizedyyyymm, validDataLocation, jsonString));
}
return allowCors(ok(MetricsUtil.stringToJsonObjectBuilder(jsonString)));
@@ -165,20 +203,27 @@ public Response getDatasetsToMonth(@PathParam("yyyymm") String yyyymm) {
@GET
@Path("datasets/pastDays/{days}")
- public Response getDatasetsPastDays(@PathParam("days") int days) {
+ public Response getDatasetsPastDays(@Context UriInfo uriInfo, @PathParam("days") int days, @QueryParam("dataLocation") String dataLocation) {
+ try {
+ errorIfUnrecongizedQueryParamPassed(uriInfo, new String[]{"dataLocation"});
+ } catch (IllegalArgumentException ia) {
+ return allowCors(error(BAD_REQUEST, ia.getLocalizedMessage()));
+ }
+
String metricName = "datasetsPastDays";
if(days < 1) {
return allowCors(error(BAD_REQUEST, "Invalid parameter for number of days."));
}
try {
- String jsonString = metricsSvc.returnUnexpiredCacheDayBased(metricName, String.valueOf(days));
+ String validDataLocation = MetricsUtil.validateDataLocationStringType(dataLocation);
+ String jsonString = metricsSvc.returnUnexpiredCacheDayBased(metricName, String.valueOf(days), validDataLocation);
if (null == jsonString) { //run query and save
- Long count = metricsSvc.datasetsPastDays(days);
+ Long count = metricsSvc.datasetsPastDays(days, validDataLocation);
JsonObjectBuilder jsonObjBuilder = MetricsUtil.countToJson(count);
jsonString = jsonObjBuilder.build().toString();
- metricsSvc.save(new Metric(metricName, String.valueOf(days), jsonString), true); //if not using cache save new
+ metricsSvc.save(new Metric(metricName, String.valueOf(days), validDataLocation, jsonString));
}
return allowCors(ok(MetricsUtil.stringToJsonObjectBuilder(jsonString)));
@@ -190,16 +235,30 @@ public Response getDatasetsPastDays(@PathParam("days") int days) {
@GET
@Path("datasets/bySubject")
- public Response getDatasetsBySubject() {
- String metricName = "datasetsBySubject";
+ public Response getDatasetsBySubject(@Context UriInfo uriInfo, @QueryParam("dataLocation") String dataLocation) {
+ return getDatasetsBySubjectToMonth(uriInfo, MetricsUtil.getCurrentMonth(), dataLocation);
+ }
+
+ @GET
+ @Path("datasets/bySubject/toMonth/{yyyymm}")
+ public Response getDatasetsBySubjectToMonth(@Context UriInfo uriInfo, @PathParam("yyyymm") String yyyymm, @QueryParam("dataLocation") String dataLocation) {
+ try {
+ errorIfUnrecongizedQueryParamPassed(uriInfo, new String[]{"dataLocation"});
+ } catch (IllegalArgumentException ia) {
+ return allowCors(error(BAD_REQUEST, ia.getLocalizedMessage()));
+ }
+
+ String metricName = "datasetsBySubjectToMonth";
try {
- String jsonArrayString = metricsSvc.returnUnexpiredCacheAllTime(metricName);
-
+ String sanitizedyyyymm = MetricsUtil.sanitizeYearMonthUserInput(yyyymm);
+ String validDataLocation = MetricsUtil.validateDataLocationStringType(dataLocation);
+ String jsonArrayString = metricsSvc.returnUnexpiredCacheMonthly(metricName, sanitizedyyyymm, validDataLocation);
+
if (null == jsonArrayString) { //run query and save
- JsonArrayBuilder jsonArrayBuilder = MetricsUtil.datasetsBySubjectToJson(metricsSvc.datasetsBySubject());
+ JsonArrayBuilder jsonArrayBuilder = MetricsUtil.datasetsBySubjectToJson(metricsSvc.datasetsBySubjectToMonth(sanitizedyyyymm, validDataLocation));
jsonArrayString = jsonArrayBuilder.build().toString();
- metricsSvc.save(new Metric(metricName, jsonArrayString), false);
+ metricsSvc.save(new Metric(metricName, sanitizedyyyymm, validDataLocation, jsonArrayString));
}
return allowCors(ok(MetricsUtil.stringToJsonArrayBuilder(jsonArrayString)));
@@ -207,35 +266,41 @@ public Response getDatasetsBySubject() {
return allowCors(error(BAD_REQUEST, ex.getLocalizedMessage()));
}
}
-
+
/** Files */
@GET
@Path("files")
- public Response getFilesAllTime() {
- return getFilesToMonth(MetricsUtil.getCurrentMonth());
+ public Response getFilesAllTime(@Context UriInfo uriInfo) {
+ return getFilesToMonth(uriInfo, MetricsUtil.getCurrentMonth());
}
@Deprecated //for better path
@GET
@Path("files/toMonth")
- public Response getFilesToMonthCurrent() {
- return getFilesToMonth(MetricsUtil.getCurrentMonth());
+ public Response getFilesToMonthCurrent(@Context UriInfo uriInfo) {
+ return getFilesToMonth(uriInfo, MetricsUtil.getCurrentMonth());
}
@GET
@Path("files/toMonth/{yyyymm}")
- public Response getFilesToMonth(@PathParam("yyyymm") String yyyymm) {
+ public Response getFilesToMonth(@Context UriInfo uriInfo, @PathParam("yyyymm") String yyyymm) {
+ try {
+ errorIfUnrecongizedQueryParamPassed(uriInfo, new String[]{""});
+ } catch (IllegalArgumentException ia) {
+ return allowCors(error(BAD_REQUEST, ia.getLocalizedMessage()));
+ }
+
String metricName = "filesToMonth";
try {
String sanitizedyyyymm = MetricsUtil.sanitizeYearMonthUserInput(yyyymm);
- String jsonString = metricsSvc.returnUnexpiredCacheMonthly(metricName, sanitizedyyyymm);
+ String jsonString = metricsSvc.returnUnexpiredCacheMonthly(metricName, sanitizedyyyymm, null);
if (null == jsonString) { //run query and save
Long count = metricsSvc.filesToMonth(sanitizedyyyymm);
JsonObjectBuilder jsonObjBuilder = MetricsUtil.countToJson(count);
jsonString = jsonObjBuilder.build().toString();
- metricsSvc.save(new Metric(metricName, sanitizedyyyymm, jsonString), true); //if not using cache save new
+ metricsSvc.save(new Metric(metricName, sanitizedyyyymm, null, jsonString));
}
return allowCors(ok(MetricsUtil.stringToJsonObjectBuilder(jsonString)));
@@ -246,20 +311,26 @@ public Response getFilesToMonth(@PathParam("yyyymm") String yyyymm) {
@GET
@Path("files/pastDays/{days}")
- public Response getFilesPastDays(@PathParam("days") int days) {
+ public Response getFilesPastDays(@Context UriInfo uriInfo, @PathParam("days") int days) {
+ try {
+ errorIfUnrecongizedQueryParamPassed(uriInfo, new String[]{""});
+ } catch (IllegalArgumentException ia) {
+ return allowCors(error(BAD_REQUEST, ia.getLocalizedMessage()));
+ }
+
String metricName = "filesPastDays";
if(days < 1) {
return allowCors(error(BAD_REQUEST, "Invalid parameter for number of days."));
}
try {
- String jsonString = metricsSvc.returnUnexpiredCacheDayBased(metricName, String.valueOf(days));
+ String jsonString = metricsSvc.returnUnexpiredCacheDayBased(metricName, String.valueOf(days), null);
if (null == jsonString) { //run query and save
Long count = metricsSvc.filesPastDays(days);
JsonObjectBuilder jsonObjBuilder = MetricsUtil.countToJson(count);
jsonString = jsonObjBuilder.build().toString();
- metricsSvc.save(new Metric(metricName, String.valueOf(days), jsonString), true); //if not using cache save new
+ metricsSvc.save(new Metric(metricName, String.valueOf(days), null, jsonString));
}
return allowCors(ok(MetricsUtil.stringToJsonObjectBuilder(jsonString)));
@@ -273,34 +344,43 @@ public Response getFilesPastDays(@PathParam("days") int days) {
@GET
@Path("downloads")
- public Response getDownloadsAllTime() {
- return getDownloadsToMonth(MetricsUtil.getCurrentMonth());
+ public Response getDownloadsAllTime(@Context UriInfo uriInfo) {
+ return getDownloadsToMonth(uriInfo, MetricsUtil.getCurrentMonth());
}
@Deprecated //for better path
@GET
@Path("downloads/toMonth")
- public Response getDownloadsToMonthCurrent() {
- return getDownloadsToMonth(MetricsUtil.getCurrentMonth());
+ public Response getDownloadsToMonthCurrent(@Context UriInfo uriInfo) {
+ return getDownloadsToMonth(uriInfo, MetricsUtil.getCurrentMonth());
}
@GET
@Path("downloads/toMonth/{yyyymm}")
- public Response getDownloadsToMonth(@PathParam("yyyymm") String yyyymm) {
+ public Response getDownloadsToMonth(@Context UriInfo uriInfo, @PathParam("yyyymm") String yyyymm) {
+ try {
+ errorIfUnrecongizedQueryParamPassed(uriInfo, new String[]{""});
+ } catch (IllegalArgumentException ia) {
+ return allowCors(error(BAD_REQUEST, ia.getLocalizedMessage()));
+ }
+
String metricName = "downloadsToMonth";
-
+
try {
+
String sanitizedyyyymm = MetricsUtil.sanitizeYearMonthUserInput(yyyymm);
- String jsonString = metricsSvc.returnUnexpiredCacheMonthly(metricName, sanitizedyyyymm);
+ String jsonString = metricsSvc.returnUnexpiredCacheMonthly(metricName, sanitizedyyyymm, null);
if (null == jsonString) { //run query and save
Long count = metricsSvc.downloadsToMonth(sanitizedyyyymm);
JsonObjectBuilder jsonObjBuilder = MetricsUtil.countToJson(count);
jsonString = jsonObjBuilder.build().toString();
- metricsSvc.save(new Metric(metricName, sanitizedyyyymm, jsonString), true); //if not using cache save new
+ metricsSvc.save(new Metric(metricName, sanitizedyyyymm, null, jsonString));
}
return allowCors(ok(MetricsUtil.stringToJsonObjectBuilder(jsonString)));
+ } catch (IllegalArgumentException ia) {
+ return allowCors(error(BAD_REQUEST, ia.getLocalizedMessage()));
} catch (Exception ex) {
return allowCors(error(BAD_REQUEST, ex.getLocalizedMessage()));
}
@@ -308,20 +388,26 @@ public Response getDownloadsToMonth(@PathParam("yyyymm") String yyyymm) {
@GET
@Path("downloads/pastDays/{days}")
- public Response getDownloadsPastDays(@PathParam("days") int days) {
+ public Response getDownloadsPastDays(@Context UriInfo uriInfo, @PathParam("days") int days) {
+ try {
+ errorIfUnrecongizedQueryParamPassed(uriInfo, new String[]{""});
+ } catch (IllegalArgumentException ia) {
+ return allowCors(error(BAD_REQUEST, ia.getLocalizedMessage()));
+ }
+
String metricName = "downloadsPastDays";
if(days < 1) {
return allowCors(error(BAD_REQUEST, "Invalid parameter for number of days."));
}
try {
- String jsonString = metricsSvc.returnUnexpiredCacheDayBased(metricName, String.valueOf(days));
+ String jsonString = metricsSvc.returnUnexpiredCacheDayBased(metricName, String.valueOf(days), null);
if (null == jsonString) { //run query and save
Long count = metricsSvc.downloadsPastDays(days);
JsonObjectBuilder jsonObjBuilder = MetricsUtil.countToJson(count);
jsonString = jsonObjBuilder.build().toString();
- metricsSvc.save(new Metric(metricName, String.valueOf(days), jsonString), true); //if not using cache save new
+ metricsSvc.save(new Metric(metricName, String.valueOf(days), null, jsonString));
}
return allowCors(ok(MetricsUtil.stringToJsonObjectBuilder(jsonString)));
@@ -331,4 +417,13 @@ public Response getDownloadsPastDays(@PathParam("days") int days) {
}
}
+ private void errorIfUnrecongizedQueryParamPassed(UriInfo uriDetails, String[] allowedQueryParams) throws IllegalArgumentException {
+ for(String theKey : uriDetails.getQueryParameters().keySet()) {
+ if(!Arrays.stream(allowedQueryParams).anyMatch(theKey::equals)) {
+ throw new IllegalArgumentException("queryParameter " + theKey + " not supported for this endpont");
+ }
+ }
+
+ }
+
}
diff --git a/src/main/java/edu/harvard/iq/dataverse/api/errorhandlers/BadRequestExceptionHandler.java b/src/main/java/edu/harvard/iq/dataverse/api/errorhandlers/BadRequestExceptionHandler.java
new file mode 100644
index 00000000000..f6412a871ac
--- /dev/null
+++ b/src/main/java/edu/harvard/iq/dataverse/api/errorhandlers/BadRequestExceptionHandler.java
@@ -0,0 +1,49 @@
+/*
+ * To change this license header, choose License Headers in Project Properties.
+ * To change this template file, choose Tools | Templates
+ * and open the template in the editor.
+ */
+package edu.harvard.iq.dataverse.api.errorhandlers;
+
+import edu.harvard.iq.dataverse.util.BundleUtil;
+import javax.json.Json;
+import javax.servlet.http.HttpServletRequest;
+import javax.ws.rs.BadRequestException;
+import javax.ws.rs.core.Context;
+import javax.ws.rs.core.Response;
+import javax.ws.rs.ext.ExceptionMapper;
+import javax.ws.rs.ext.Provider;
+
+/**
+ *
+ * @author skraffmi
+ */
+@Provider
+public class BadRequestExceptionHandler implements ExceptionMapper {
+
+ @Context
+ HttpServletRequest request;
+
+ @Override
+ public Response toResponse(BadRequestException ex) {
+ System.out.print( ex.getMessage());
+ String uri = request.getRequestURI();
+ String exMessage = ex.getMessage();
+ String outputMessage;
+ if (exMessage != null && exMessage.toLowerCase().startsWith("tabular data required")) {
+ outputMessage = BundleUtil.getStringFromBundle("access.api.exception.metadata.not.available.for.nontabular.file");
+ } else {
+ outputMessage = "Bad Request. The API request cannot be completed with the parameters supplied. Please check your code for typos, or consult our API guide at http://guides.dataverse.org.";
+ }
+ return Response.status(400)
+ .entity( Json.createObjectBuilder()
+ .add("status", "ERROR")
+ .add("code", 400)
+ .add("message", "'" + uri + "' " + outputMessage)
+ .build())
+ .type("application/json").build();
+
+
+ }
+
+}
diff --git a/src/main/java/edu/harvard/iq/dataverse/api/imports/ImportGenericServiceBean.java b/src/main/java/edu/harvard/iq/dataverse/api/imports/ImportGenericServiceBean.java
index b5554816c0a..9b60993b365 100644
--- a/src/main/java/edu/harvard/iq/dataverse/api/imports/ImportGenericServiceBean.java
+++ b/src/main/java/edu/harvard/iq/dataverse/api/imports/ImportGenericServiceBean.java
@@ -411,16 +411,17 @@ private String reassignIdentifierAsGlobalId(String identifierString, DatasetDTO
if (identifierString.startsWith(GlobalId.HDL_RESOLVER_URL)) {
logger.fine("Processing Handle identifier formatted as a resolver URL: "+identifierString);
protocol = GlobalId.HDL_PROTOCOL;
- index1 = GlobalId.HDL_RESOLVER_URL.length() - 1;
+ index1 = GlobalId.HDL_RESOLVER_URL.length() - 1;
+ index2 = identifierString.indexOf("/", index1 + 1);
} else if (identifierString.startsWith(GlobalId.DOI_RESOLVER_URL)) {
logger.fine("Processing DOI identifier formatted as a resolver URL: "+identifierString);
protocol = GlobalId.DOI_PROTOCOL;
index1 = GlobalId.DOI_RESOLVER_URL.length() - 1;
+ index2 = identifierString.indexOf("/", index1 + 1);
} else {
logger.warning("HTTP Url in supplied as the identifier is neither a Handle nor DOI resolver: "+identifierString);
return null;
}
- // index2 was already found as the index of '/' - so it's still good.
} else {
logger.warning("Unknown identifier format: "+identifierString);
return null;
diff --git a/src/main/java/edu/harvard/iq/dataverse/authorization/providers/builtin/DataverseUserPage.java b/src/main/java/edu/harvard/iq/dataverse/authorization/providers/builtin/DataverseUserPage.java
index 09096fe5766..673e18d2b4a 100644
--- a/src/main/java/edu/harvard/iq/dataverse/authorization/providers/builtin/DataverseUserPage.java
+++ b/src/main/java/edu/harvard/iq/dataverse/authorization/providers/builtin/DataverseUserPage.java
@@ -156,7 +156,7 @@ public String init() {
} else {
// in create mode for new user
- JH.addMessage(FacesMessage.SEVERITY_INFO, BundleUtil.getStringFromBundle("user.signup.tip"));
+ JH.addMessage(FacesMessage.SEVERITY_INFO, BundleUtil.getStringFromBundle("user.message.signup.label"), BundleUtil.getStringFromBundle("user.message.signup.tip"));
userDisplayInfo = new AuthenticatedUserDisplayInfo();
return "";
}
diff --git a/src/main/java/edu/harvard/iq/dataverse/batch/jobs/importer/filesystem/FileRecordWriter.java b/src/main/java/edu/harvard/iq/dataverse/batch/jobs/importer/filesystem/FileRecordWriter.java
index 033b895caeb..c82a5bb01eb 100644
--- a/src/main/java/edu/harvard/iq/dataverse/batch/jobs/importer/filesystem/FileRecordWriter.java
+++ b/src/main/java/edu/harvard/iq/dataverse/batch/jobs/importer/filesystem/FileRecordWriter.java
@@ -319,7 +319,7 @@ private DataFile createPackageDataFile(List files) {
// add code to generate the manifest, if not present? -- L.A.
} else {
try {
- packageFile.setChecksumValue(FileUtil.CalculateChecksum(checksumManifestPath, packageFile.getChecksumType()));
+ packageFile.setChecksumValue(FileUtil.calculateChecksum(checksumManifestPath, packageFile.getChecksumType()));
} catch (Exception ex) {
getJobLogger().log(Level.SEVERE, "Failed to calculate checksum (type "+packageFile.getChecksumType()+") "+ex.getMessage());
jobContext.setExitStatus("FAILED");
diff --git a/src/main/java/edu/harvard/iq/dataverse/batch/util/LoggingUtil.java b/src/main/java/edu/harvard/iq/dataverse/batch/util/LoggingUtil.java
index eccb8c0f9eb..b695feda0c2 100644
--- a/src/main/java/edu/harvard/iq/dataverse/batch/util/LoggingUtil.java
+++ b/src/main/java/edu/harvard/iq/dataverse/batch/util/LoggingUtil.java
@@ -25,6 +25,8 @@
import javax.batch.runtime.JobExecution;
import java.io.File;
import java.io.IOException;
+import java.text.SimpleDateFormat;
+import java.util.Date;
import java.util.logging.FileHandler;
import java.util.logging.Formatter;
import java.util.logging.Level;
@@ -37,17 +39,28 @@ public class LoggingUtil {
private static final Logger logger = Logger.getLogger(LoggingUtil.class.getName());
public static void saveJsonLog(String jobJson, String logDir, String jobId) {
- try {
- checkCreateLogDirectory( logDir );
- File dir = new File(logDir);
- if (!dir.exists() && !dir.mkdirs()) {
- logger.log(Level.SEVERE, "Couldn't create directory: " + dir.getAbsolutePath());
- }
- File reportJson = new File(dir.getAbsolutePath() + "/job-" + jobId + ".json");
- FileUtils.writeStringToFile(reportJson, jobJson);
- } catch (Exception e) {
- logger.log(Level.SEVERE, "Error saving json report: " + e.getMessage());
- }
+ try {
+ String fileName = "/job-" + jobId + ".json";
+ saveLogFile(jobJson, logDir, fileName);
+ } catch (Exception e) {
+ logger.log(Level.SEVERE, "Error saving json report: " + e.getMessage());
+ }
+ }
+
+ public static void saveLogFile(String fileContent, String logDir, String fileName) {
+
+ try {
+ checkCreateLogDirectory(logDir);
+ File dir = new File(logDir);
+ if (!dir.exists() && !dir.mkdirs()) {
+ logger.log(Level.SEVERE, "Couldn't create directory: " + dir.getAbsolutePath());
+ }
+ File logFile = new File(dir.getAbsolutePath() + fileName);
+ FileUtils.writeStringToFile(logFile, fileContent);
+ } catch (Exception e) {
+ logger.log(Level.SEVERE, "Error saving log report: " + fileName + " " + e.getMessage());
+ }
+
}
public static ActionLogRecord getActionLogRecord(String userId, JobExecution jobExec, String jobInfo, String jobId) {
diff --git a/src/main/java/edu/harvard/iq/dataverse/customization/CustomizationConstants.java b/src/main/java/edu/harvard/iq/dataverse/customization/CustomizationConstants.java
index 314cefc154d..86bab2e1f60 100644
--- a/src/main/java/edu/harvard/iq/dataverse/customization/CustomizationConstants.java
+++ b/src/main/java/edu/harvard/iq/dataverse/customization/CustomizationConstants.java
@@ -19,6 +19,8 @@ public class CustomizationConstants {
public static String fileTypeStyle = "style";
+ public static String fileTypeAnalytics = "analytics";
+
public static String fileTypeLogo = "logo";
diff --git a/src/main/java/edu/harvard/iq/dataverse/dataaccess/DataFileZipper.java b/src/main/java/edu/harvard/iq/dataverse/dataaccess/DataFileZipper.java
index 0451da95cca..15ff7b5ac99 100644
--- a/src/main/java/edu/harvard/iq/dataverse/dataaccess/DataFileZipper.java
+++ b/src/main/java/edu/harvard/iq/dataverse/dataaccess/DataFileZipper.java
@@ -22,12 +22,13 @@
import java.io.InputStream;
import java.io.IOException;
-import java.util.Iterator;
import edu.harvard.iq.dataverse.DataFile;
import java.io.OutputStream;
import java.util.ArrayList;
+import java.util.HashSet;
import java.util.List;
+import java.util.Set;
import java.util.logging.Logger;
import java.util.zip.ZipEntry;
import java.util.zip.ZipOutputStream;
@@ -41,6 +42,7 @@ public class DataFileZipper {
public static long DEFAULT_ZIPFILE_LIMIT = 100 * 1024 * 1024; // 100MB
private static final Logger logger = Logger.getLogger(DataFileZipper.class.getCanonicalName());
+ private static final String MANIFEST_FILE_NAME = "MANIFEST.TXT";
private OutputStream outputStream = null;
private ZipOutputStream zipOutputStream = null;
@@ -49,16 +51,20 @@ public class DataFileZipper {
private List zippedFilesList = null; // list of successfully zipped files, to update guestbooks and download counts (not yet implemented)
private String fileManifest = "";
+
+ private Set zippedFolders = null;
public DataFileZipper() {
fileNameList = new ArrayList<>();
zippedFilesList = new ArrayList<>();
+ zippedFolders = new HashSet<>();
}
public DataFileZipper(OutputStream outputStream) {
this.outputStream = outputStream;
fileNameList = new ArrayList<>();
zippedFilesList = new ArrayList<>();
+ zippedFolders = new HashSet<>();
}
public void setOutputStream(OutputStream outputStream) {
@@ -133,12 +139,31 @@ public long addFileToZipStream(DataFile dataFile, boolean getOriginal) throws IO
Success = false;
} else {
+ // If any of the files have non-empty DirectoryLabels we'll
+ // use them to re-create the folders in the Zipped bundle:
+ String folderName = dataFile.getFileMetadata().getDirectoryLabel();
+ if (folderName != null) {
+ // If any of the saved folder names start with with slashes,
+ // we want to remove them:
+ // (i.e., ///foo/bar will become foo/bar)
+ while (folderName.startsWith("/")) {
+ folderName = folderName.substring(1);
+ }
+ if (!"".equals(folderName)) {
+ if (!zippedFolders.contains(folderName)) {
+ ZipEntry d = new ZipEntry(folderName + "/");
+ zipOutputStream.putNextEntry(d);
+ zipOutputStream.closeEntry();
+ zippedFolders.add(folderName);
+ }
+ fileName = folderName + "/" + fileName;
+ }
+ }
+
String zipEntryName = checkZipEntryName(fileName);
+
ZipEntry e = new ZipEntry(zipEntryName);
logger.fine("created new zip entry for " + zipEntryName);
- // support for categories: (not yet implemented)
- //String zipEntryDirectoryName = file.getCategory(versionNum);
- //ZipEntry e = new ZipEntry(zipEntryDirectoryName + "/" + zipEntryName);
zipOutputStream.putNextEntry(e);
@@ -189,7 +214,12 @@ public void finalizeZipStream() throws IOException {
}
if (createManifest) {
- ZipEntry e = new ZipEntry("MANIFEST.TXT");
+ String manifestEntry = MANIFEST_FILE_NAME;
+ while (fileNameList.contains(manifestEntry)) {
+ manifestEntry = "0".concat(manifestEntry);
+ }
+
+ ZipEntry e = new ZipEntry(manifestEntry);
zipOutputStream.putNextEntry(e);
zipOutputStream.write(fileManifest.getBytes());
diff --git a/src/main/java/edu/harvard/iq/dataverse/dataaccess/FileAccessIO.java b/src/main/java/edu/harvard/iq/dataverse/dataaccess/FileAccessIO.java
index 93d67be3d63..5b91b7a1904 100644
--- a/src/main/java/edu/harvard/iq/dataverse/dataaccess/FileAccessIO.java
+++ b/src/main/java/edu/harvard/iq/dataverse/dataaccess/FileAccessIO.java
@@ -565,6 +565,8 @@ public boolean accept(Path file) throws IOException {
}
}
+ dirStream.close();
+
return auxItems;
}
diff --git a/src/main/java/edu/harvard/iq/dataverse/dataset/DatasetUtil.java b/src/main/java/edu/harvard/iq/dataverse/dataset/DatasetUtil.java
index 9ac51d92f1b..b844a9d109f 100644
--- a/src/main/java/edu/harvard/iq/dataverse/dataset/DatasetUtil.java
+++ b/src/main/java/edu/harvard/iq/dataverse/dataset/DatasetUtil.java
@@ -22,18 +22,16 @@
import java.io.UnsupportedEncodingException;
import java.nio.channels.FileChannel;
import java.nio.file.Files;
-import java.nio.file.Path;
import java.nio.file.Paths;
-import java.nio.file.StandardCopyOption;
import java.util.ArrayList;
import java.util.List;
import java.util.logging.Logger;
import java.util.Base64;
import java.util.HashMap;
-import java.util.HashSet;
import java.util.Map;
import javax.imageio.ImageIO;
import org.apache.commons.io.IOUtils;
+import static edu.harvard.iq.dataverse.dataaccess.DataAccess.getStorageIO;
public class DatasetUtil {
diff --git a/src/main/java/edu/harvard/iq/dataverse/datasetutility/AddReplaceFileHelper.java b/src/main/java/edu/harvard/iq/dataverse/datasetutility/AddReplaceFileHelper.java
index acbb9211ca1..154399688cf 100644
--- a/src/main/java/edu/harvard/iq/dataverse/datasetutility/AddReplaceFileHelper.java
+++ b/src/main/java/edu/harvard/iq/dataverse/datasetutility/AddReplaceFileHelper.java
@@ -129,6 +129,7 @@ public class AddReplaceFileHelper{
// -----------------------------------
private User user;
private DatasetVersion workingVersion;
+ private DatasetVersion clone;
List initialFileList;
List finalFileList;
@@ -1042,7 +1043,7 @@ private boolean step_030_createNewFilesViaIngest(){
// Load the working version of the Dataset
workingVersion = dataset.getEditVersion();
-
+ clone = workingVersion.cloneDatasetVersion();
try {
initialFileList = FileUtil.createDataFiles(workingVersion,
this.newFileInputStream,
@@ -1439,7 +1440,7 @@ private boolean step_070_run_update_dataset_command(){
}
Command update_cmd;
- update_cmd = new UpdateDatasetVersionCommand(dataset, dvRequest);
+ update_cmd = new UpdateDatasetVersionCommand(dataset, dvRequest, clone);
((UpdateDatasetVersionCommand) update_cmd).setValidateLenient(true);
try {
diff --git a/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/AbstractSubmitToArchiveCommand.java b/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/AbstractSubmitToArchiveCommand.java
new file mode 100644
index 00000000000..976c4983bcd
--- /dev/null
+++ b/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/AbstractSubmitToArchiveCommand.java
@@ -0,0 +1,75 @@
+package edu.harvard.iq.dataverse.engine.command.impl;
+
+import edu.harvard.iq.dataverse.DatasetVersion;
+import edu.harvard.iq.dataverse.DvObject;
+import edu.harvard.iq.dataverse.authorization.Permission;
+import edu.harvard.iq.dataverse.authorization.users.ApiToken;
+import edu.harvard.iq.dataverse.authorization.users.AuthenticatedUser;
+import edu.harvard.iq.dataverse.engine.command.AbstractCommand;
+import edu.harvard.iq.dataverse.engine.command.CommandContext;
+import edu.harvard.iq.dataverse.engine.command.DataverseRequest;
+import edu.harvard.iq.dataverse.engine.command.RequiredPermissions;
+import edu.harvard.iq.dataverse.engine.command.exception.CommandException;
+import edu.harvard.iq.dataverse.settings.SettingsServiceBean;
+import edu.harvard.iq.dataverse.workflow.step.WorkflowStepResult;
+
+import java.util.Date;
+import java.util.HashMap;
+import java.util.Map;
+import java.util.logging.Logger;
+
+@RequiredPermissions(Permission.PublishDataset)
+public abstract class AbstractSubmitToArchiveCommand extends AbstractCommand {
+
+ private final DatasetVersion version;
+ private final Map requestedSettings = new HashMap();
+ private static final Logger logger = Logger.getLogger(AbstractSubmitToArchiveCommand.class.getName());
+
+ public AbstractSubmitToArchiveCommand(DataverseRequest aRequest, DatasetVersion version) {
+ super(aRequest, version.getDataset());
+ this.version = version;
+ }
+
+ @Override
+ public DatasetVersion execute(CommandContext ctxt) throws CommandException {
+
+ String settings = ctxt.settings().getValueForKey(SettingsServiceBean.Key.ArchiverSettings);
+ String[] settingsArray = settings.split(",");
+ for (String setting : settingsArray) {
+ setting = setting.trim();
+ if (!setting.startsWith(":")) {
+ logger.warning("Invalid Archiver Setting: " + setting);
+ } else {
+ requestedSettings.put(setting, ctxt.settings().get(setting));
+ }
+ }
+
+ AuthenticatedUser user = getRequest().getAuthenticatedUser();
+ ApiToken token = ctxt.authentication().findApiTokenByUser(user);
+ if ((token == null) || (token.getExpireTime().before(new Date()))) {
+ token = ctxt.authentication().generateApiTokenForUser(user);
+ }
+ performArchiveSubmission(version, token, requestedSettings);
+ return ctxt.em().merge(version);
+ }
+
+ /**
+ * This method is the only one that should be overwritten by other classes. Note
+ * that this method may be called from the execute method above OR from a
+ * workflow in which execute() is never called and therefore in which all
+ * variables must be sent as method parameters. (Nominally version is set in the
+ * constructor and could be dropped from the parameter list.)
+ *
+ * @param version - the DatasetVersion to archive
+ * @param token - an API Token for the user performing this action
+ * @param requestedSettings - a map of the names/values for settings required by this archiver (sent because this class is not part of the EJB context (by design) and has no direct access to service beans).
+ */
+ abstract public WorkflowStepResult performArchiveSubmission(DatasetVersion version, ApiToken token, Map requestedSetttings);
+
+ @Override
+ public String describe() {
+ return super.describe() + "DatasetVersion: [" + version.getId() + " (v"
+ + version.getFriendlyVersionNumber()+")]";
+ }
+
+}
diff --git a/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/DeleteDatasetVersionCommand.java b/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/DeleteDatasetVersionCommand.java
index c4d53466f82..d7cbd12eb25 100644
--- a/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/DeleteDatasetVersionCommand.java
+++ b/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/DeleteDatasetVersionCommand.java
@@ -27,7 +27,7 @@ public class DeleteDatasetVersionCommand extends AbstractVoidCommand {
private static final Logger logger = Logger.getLogger(DeleteDatasetVersionCommand.class.getCanonicalName());
- private final Dataset doomed;
+ private Dataset doomed;
public DeleteDatasetVersionCommand(DataverseRequest aRequest, Dataset dataset) {
super(aRequest, dataset);
@@ -37,7 +37,7 @@ public DeleteDatasetVersionCommand(DataverseRequest aRequest, Dataset dataset) {
@Override
protected void executeImpl(CommandContext ctxt) throws CommandException {
ctxt.permissions().checkEditDatasetLock(doomed, getRequest(), this);
-
+ doomed = ctxt.em().find(Dataset.class, doomed.getId());
// if you are deleting a dataset that only has 1 draft, we are actually destroying the dataset
if (doomed.getVersions().size() == 1) {
ctxt.engine().submit(new DestroyDatasetCommand(doomed, getRequest()));
diff --git a/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/DuraCloudSubmitToArchiveCommand.java b/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/DuraCloudSubmitToArchiveCommand.java
new file mode 100644
index 00000000000..34bd3234351
--- /dev/null
+++ b/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/DuraCloudSubmitToArchiveCommand.java
@@ -0,0 +1,198 @@
+package edu.harvard.iq.dataverse.engine.command.impl;
+
+import edu.harvard.iq.dataverse.DOIDataCiteRegisterService;
+import edu.harvard.iq.dataverse.DataCitation;
+import edu.harvard.iq.dataverse.Dataset;
+import edu.harvard.iq.dataverse.DatasetVersion;
+import edu.harvard.iq.dataverse.DatasetLock.Reason;
+import edu.harvard.iq.dataverse.authorization.Permission;
+import edu.harvard.iq.dataverse.authorization.users.ApiToken;
+import edu.harvard.iq.dataverse.engine.command.Command;
+import edu.harvard.iq.dataverse.engine.command.DataverseRequest;
+import edu.harvard.iq.dataverse.engine.command.RequiredPermissions;
+import edu.harvard.iq.dataverse.util.bagit.BagGenerator;
+import edu.harvard.iq.dataverse.util.bagit.OREMap;
+import edu.harvard.iq.dataverse.workflow.step.Failure;
+import edu.harvard.iq.dataverse.workflow.step.WorkflowStepResult;
+
+import java.io.IOException;
+import java.io.PipedInputStream;
+import java.io.PipedOutputStream;
+import java.nio.charset.Charset;
+import java.security.DigestInputStream;
+import java.security.MessageDigest;
+import java.security.NoSuchAlgorithmException;
+import java.util.Map;
+import java.util.logging.Logger;
+
+import org.apache.commons.codec.binary.Hex;
+import org.duracloud.client.ContentStore;
+import org.duracloud.client.ContentStoreManager;
+import org.duracloud.client.ContentStoreManagerImpl;
+import org.duracloud.common.model.Credential;
+import org.duracloud.error.ContentStoreException;
+
+@RequiredPermissions(Permission.PublishDataset)
+public class DuraCloudSubmitToArchiveCommand extends AbstractSubmitToArchiveCommand implements Command {
+
+ private static final Logger logger = Logger.getLogger(DuraCloudSubmitToArchiveCommand.class.getName());
+ private static final String DEFAULT_PORT = "443";
+ private static final String DEFAULT_CONTEXT = "durastore";
+ private static final String DURACLOUD_PORT = ":DuraCloudPort";
+ private static final String DURACLOUD_HOST = ":DuraCloudHost";
+ private static final String DURACLOUD_CONTEXT = ":DuraCloudContext";
+
+ public DuraCloudSubmitToArchiveCommand(DataverseRequest aRequest, DatasetVersion version) {
+ super(aRequest, version);
+ }
+
+ @Override
+ public WorkflowStepResult performArchiveSubmission(DatasetVersion dv, ApiToken token, Map requestedSettings) {
+
+ String port = requestedSettings.get(DURACLOUD_PORT) != null ? requestedSettings.get(DURACLOUD_PORT) : DEFAULT_PORT;
+ String dpnContext = requestedSettings.get(DURACLOUD_CONTEXT) != null ? requestedSettings.get(DURACLOUD_CONTEXT) : DEFAULT_CONTEXT;
+ String host = requestedSettings.get(DURACLOUD_HOST);
+ if (host != null) {
+ Dataset dataset = dv.getDataset();
+ if (dataset.getLockFor(Reason.pidRegister) == null) {
+ // Use Duracloud client classes to login
+ ContentStoreManager storeManager = new ContentStoreManagerImpl(host, port, dpnContext);
+ Credential credential = new Credential(System.getProperty("duracloud.username"),
+ System.getProperty("duracloud.password"));
+ storeManager.login(credential);
+
+ String spaceName = dataset.getGlobalId().asString().replace(':', '-').replace('/', '-')
+ .replace('.', '-').toLowerCase();
+
+ ContentStore store;
+ try {
+ /*
+ * If there is a failure in creating a space, it is likely that a prior version
+ * has not been fully processed (snapshot created, archiving completed and files
+ * and space deleted - currently manual operations done at the project's
+ * duracloud website)
+ */
+ store = storeManager.getPrimaryContentStore();
+ // Create space to copy archival files to
+ store.createSpace(spaceName);
+ DataCitation dc = new DataCitation(dv);
+ Map metadata = dc.getDataCiteMetadata();
+ String dataciteXml = DOIDataCiteRegisterService.getMetadataFromDvObject(
+ dv.getDataset().getGlobalId().asString(), metadata, dv.getDataset());
+
+ MessageDigest messageDigest = MessageDigest.getInstance("MD5");
+ try (PipedInputStream dataciteIn = new PipedInputStream(); DigestInputStream digestInputStream = new DigestInputStream(dataciteIn, messageDigest)) {
+ // Add datacite.xml file
+
+ new Thread(new Runnable() {
+ public void run() {
+ try (PipedOutputStream dataciteOut = new PipedOutputStream(dataciteIn)) {
+
+ dataciteOut.write(dataciteXml.getBytes(Charset.forName("utf-8")));
+ dataciteOut.close();
+ } catch (Exception e) {
+ logger.severe("Error creating datacite.xml: " + e.getMessage());
+ // TODO Auto-generated catch block
+ e.printStackTrace();
+ throw new RuntimeException("Error creating datacite.xml: " + e.getMessage());
+ }
+ }
+ }).start();
+
+ String checksum = store.addContent(spaceName, "datacite.xml", digestInputStream, -1l, null, null,
+ null);
+ logger.fine("Content: datacite.xml added with checksum: " + checksum);
+ String localchecksum = Hex.encodeHexString(digestInputStream.getMessageDigest().digest());
+ if (!checksum.equals(localchecksum)) {
+ logger.severe(checksum + " not equal to " + localchecksum);
+ return new Failure("Error in transferring DataCite.xml file to DuraCloud",
+ "DuraCloud Submission Failure: incomplete metadata transfer");
+ }
+
+ // Store BagIt file
+ String fileName = spaceName + "v" + dv.getFriendlyVersionNumber() + ".zip";
+
+ // Add BagIt ZIP file
+ // Although DuraCloud uses SHA-256 internally, it's API uses MD5 to verify the
+ // transfer
+ messageDigest = MessageDigest.getInstance("MD5");
+ try (PipedInputStream in = new PipedInputStream(); DigestInputStream digestInputStream2 = new DigestInputStream(in, messageDigest)) {
+ new Thread(new Runnable() {
+ public void run() {
+ try (PipedOutputStream out = new PipedOutputStream(in)){
+ // Generate bag
+ BagGenerator bagger = new BagGenerator(new OREMap(dv, false), dataciteXml);
+ bagger.setAuthenticationKey(token.getTokenString());
+ bagger.generateBag(out);
+ } catch (Exception e) {
+ logger.severe("Error creating bag: " + e.getMessage());
+ // TODO Auto-generated catch block
+ e.printStackTrace();
+ throw new RuntimeException("Error creating bag: " + e.getMessage());
+ }
+ }
+ }).start();
+
+ checksum = store.addContent(spaceName, fileName, digestInputStream2, -1l, null, null,
+ null);
+ logger.fine("Content: " + fileName + " added with checksum: " + checksum);
+ localchecksum = Hex.encodeHexString(digestInputStream2.getMessageDigest().digest());
+ if (!checksum.equals(localchecksum)) {
+ logger.severe(checksum + " not equal to " + localchecksum);
+ return new Failure("Error in transferring Zip file to DuraCloud",
+ "DuraCloud Submission Failure: incomplete archive transfer");
+ }
+ } catch (RuntimeException rte) {
+ logger.severe(rte.getMessage());
+ return new Failure("Error in generating Bag",
+ "DuraCloud Submission Failure: archive file not created");
+ }
+
+ logger.fine("DuraCloud Submission step: Content Transferred");
+
+ // Document the location of dataset archival copy location (actually the URL
+ // where you can
+ // view it as an admin)
+ StringBuffer sb = new StringBuffer("https://");
+ sb.append(host);
+ if (!port.equals("443")) {
+ sb.append(":" + port);
+ }
+ sb.append("/duradmin/spaces/sm/");
+ sb.append(store.getStoreId());
+ sb.append("/" + spaceName + "/" + fileName);
+ dv.setArchivalCopyLocation(sb.toString());
+ logger.fine("DuraCloud Submission step complete: " + sb.toString());
+ } catch (ContentStoreException | IOException e) {
+ // TODO Auto-generated catch block
+ logger.warning(e.getMessage());
+ e.printStackTrace();
+ return new Failure("Error in transferring file to DuraCloud",
+ "DuraCloud Submission Failure: archive file not transferred");
+ } catch (RuntimeException rte) {
+ logger.severe(rte.getMessage());
+ return new Failure("Error in generating datacite.xml file",
+ "DuraCloud Submission Failure: metadata file not created");
+ }
+ } catch (ContentStoreException e) {
+ logger.warning(e.getMessage());
+ e.printStackTrace();
+ String mesg = "DuraCloud Submission Failure";
+ if (!(1 == dv.getVersion()) || !(0 == dv.getMinorVersionNumber())) {
+ mesg = mesg + ": Prior Version archiving not yet complete?";
+ }
+ return new Failure("Unable to create DuraCloud space with name: " + spaceName, mesg);
+ } catch (NoSuchAlgorithmException e) {
+ logger.severe("MD5 MessageDigest not available!");
+ }
+ } else {
+ logger.warning("DuraCloud Submision Workflow aborted: Dataset locked for pidRegister");
+ return new Failure("Dataset locked");
+ }
+ return WorkflowStepResult.OK;
+ } else {
+ return new Failure("DuraCloud Submission not configured - no \":DuraCloudHost\".");
+ }
+ }
+
+}
diff --git a/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/UpdateDatasetVersionCommand.java b/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/UpdateDatasetVersionCommand.java
index f54ce5d8be5..467ffd734c0 100644
--- a/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/UpdateDatasetVersionCommand.java
+++ b/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/UpdateDatasetVersionCommand.java
@@ -22,15 +22,24 @@ public class UpdateDatasetVersionCommand extends AbstractDatasetCommand
private static final Logger logger = Logger.getLogger(UpdateDatasetVersionCommand.class.getCanonicalName());
private final List filesToDelete;
private boolean validateLenient = false;
+ private final DatasetVersion clone;
public UpdateDatasetVersionCommand(Dataset theDataset, DataverseRequest aRequest) {
super(aRequest, theDataset);
this.filesToDelete = new ArrayList<>();
+ this.clone = null;
}
public UpdateDatasetVersionCommand(Dataset theDataset, DataverseRequest aRequest, List filesToDelete) {
super(aRequest, theDataset);
this.filesToDelete = filesToDelete;
+ this.clone = null;
+ }
+
+ public UpdateDatasetVersionCommand(Dataset theDataset, DataverseRequest aRequest, List filesToDelete, DatasetVersion clone) {
+ super(aRequest, theDataset);
+ this.filesToDelete = filesToDelete;
+ this.clone = clone;
}
public UpdateDatasetVersionCommand(Dataset theDataset, DataverseRequest aRequest, DataFile fileToDelete) {
@@ -38,13 +47,20 @@ public UpdateDatasetVersionCommand(Dataset theDataset, DataverseRequest aRequest
// get the latest file metadata for the file; ensuring that it is a draft version
this.filesToDelete = new ArrayList<>();
+ this.clone = null;
for (FileMetadata fmd : theDataset.getEditVersion().getFileMetadatas()) {
if (fmd.getDataFile().equals(fileToDelete)) {
filesToDelete.add(fmd);
break;
}
}
- }
+ }
+
+ public UpdateDatasetVersionCommand(Dataset theDataset, DataverseRequest aRequest, DatasetVersion clone) {
+ super(aRequest, theDataset);
+ this.filesToDelete = new ArrayList<>();
+ this.clone = clone;
+ }
public boolean isValidateLenient() {
return validateLenient;
@@ -144,7 +160,11 @@ public Dataset execute(CommandContext ctxt) throws CommandException {
updateDatasetUser(ctxt);
ctxt.index().indexDataset(savedDataset, true);
-
+ if (clone != null) {
+ DatasetVersionDifference dvd = new DatasetVersionDifference(editVersion, clone);
+ AuthenticatedUser au = (AuthenticatedUser) getUser();
+ ctxt.datasetVersion().writeEditVersionLog(dvd, au);
+ }
return savedDataset;
}
diff --git a/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/UpdateDvObjectPIDMetadataCommand.java b/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/UpdateDvObjectPIDMetadataCommand.java
index e36fe06b863..7e37241563c 100644
--- a/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/UpdateDvObjectPIDMetadataCommand.java
+++ b/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/UpdateDvObjectPIDMetadataCommand.java
@@ -11,6 +11,7 @@
import edu.harvard.iq.dataverse.engine.command.RequiredPermissions;
import edu.harvard.iq.dataverse.engine.command.exception.CommandException;
import edu.harvard.iq.dataverse.engine.command.exception.PermissionException;
+import edu.harvard.iq.dataverse.settings.SettingsServiceBean;
import edu.harvard.iq.dataverse.util.BundleUtil;
import java.sql.Timestamp;
import java.util.Collections;
@@ -52,12 +53,29 @@ protected void executeImpl(CommandContext ctxt) throws CommandException {
target.setGlobalIdCreateTime(new Timestamp(new Date().getTime()));
ctxt.em().merge(target);
ctxt.em().flush();
+ // When updating, we want to traverse through files even if the dataset itself
+ // didn't need updating.
+ String currentGlobalIdProtocol = ctxt.settings().getValueForKey(SettingsServiceBean.Key.Protocol, "");
+ String dataFilePIDFormat = ctxt.settings().getValueForKey(SettingsServiceBean.Key.DataFilePIDFormat, "DEPENDENT");
+ boolean isFilePIDsEnabled = ctxt.systemConfig().isFilePIDsEnabled();
+ // We will skip trying to update the global identifiers for datafiles if they
+ // aren't being used.
+ // If they are, we need to assure that there's an existing PID or, as when
+ // creating PIDs, that the protocol matches that of the dataset DOI if
+ // we're going to create a DEPENDENT file PID.
+ String protocol = target.getProtocol();
for (DataFile df : target.getFiles()) {
- doiRetString = idServiceBean.publicizeIdentifier(df);
- if (doiRetString) {
- df.setGlobalIdCreateTime(new Timestamp(new Date().getTime()));
- ctxt.em().merge(df);
- ctxt.em().flush();
+ if (isFilePIDsEnabled && // using file PIDs and
+ (!(df.getIdentifier() == null || df.getIdentifier().isEmpty()) || // identifier exists, or
+ currentGlobalIdProtocol.equals(protocol) || // right protocol to create dependent DOIs, or
+ dataFilePIDFormat.equals("INDEPENDENT"))// or independent. TODO(pm) - check authority too
+ ) {
+ doiRetString = idServiceBean.publicizeIdentifier(df);
+ if (doiRetString) {
+ df.setGlobalIdCreateTime(new Timestamp(new Date().getTime()));
+ ctxt.em().merge(df);
+ ctxt.em().flush();
+ }
}
}
} else {
diff --git a/src/main/java/edu/harvard/iq/dataverse/externaltools/ExternalTool.java b/src/main/java/edu/harvard/iq/dataverse/externaltools/ExternalTool.java
index fee92b6c0b9..d3c36675c2e 100644
--- a/src/main/java/edu/harvard/iq/dataverse/externaltools/ExternalTool.java
+++ b/src/main/java/edu/harvard/iq/dataverse/externaltools/ExternalTool.java
@@ -25,6 +25,7 @@ public class ExternalTool implements Serializable {
public static final String TYPE = "type";
public static final String TOOL_URL = "toolUrl";
public static final String TOOL_PARAMETERS = "toolParameters";
+ public static final String CONTENT_TYPE = "contentType";
@Id
@GeneratedValue(strategy = GenerationType.IDENTITY)
@@ -61,6 +62,13 @@ public class ExternalTool implements Serializable {
@Column(nullable = false)
private String toolParameters;
+ /**
+ * The file content type the tool works on. For tabular files, the type text/tab-separated-values should be sent
+ */
+ @Column(nullable = false, columnDefinition = "TEXT")
+ private String contentType;
+
+
/**
* This default constructor is only here to prevent this error at
* deployment:
@@ -75,12 +83,13 @@ public class ExternalTool implements Serializable {
public ExternalTool() {
}
- public ExternalTool(String displayName, String description, Type type, String toolUrl, String toolParameters) {
+ public ExternalTool(String displayName, String description, Type type, String toolUrl, String toolParameters, String contentType) {
this.displayName = displayName;
this.description = description;
this.type = type;
this.toolUrl = toolUrl;
this.toolParameters = toolParameters;
+ this.contentType = contentType;
}
public enum Type {
@@ -155,6 +164,14 @@ public void setToolParameters(String toolParameters) {
this.toolParameters = toolParameters;
}
+ public String getContentType() {
+ return this.contentType;
+ }
+
+ public void setContentType(String contentType) {
+ this.contentType = contentType;
+ }
+
public JsonObjectBuilder toJson() {
JsonObjectBuilder jab = Json.createObjectBuilder();
jab.add("id", getId());
@@ -163,6 +180,7 @@ public JsonObjectBuilder toJson() {
jab.add(TYPE, getType().text);
jab.add(TOOL_URL, getToolUrl());
jab.add(TOOL_PARAMETERS, getToolParameters());
+ jab.add(CONTENT_TYPE, getContentType());
return jab;
}
@@ -174,7 +192,9 @@ public enum ReservedWord {
// from https://swagger.io/specification/#fixed-fields-29 but that's for URLs.
FILE_ID("fileId"),
SITE_URL("siteUrl"),
- API_TOKEN("apiToken");
+ API_TOKEN("apiToken"),
+ DATASET_ID("datasetId"),
+ DATASET_VERSION("datasetVersion");
private final String text;
private final String START = "{";
diff --git a/src/main/java/edu/harvard/iq/dataverse/externaltools/ExternalToolHandler.java b/src/main/java/edu/harvard/iq/dataverse/externaltools/ExternalToolHandler.java
index 9fb5a72d9ba..93ea882c80a 100644
--- a/src/main/java/edu/harvard/iq/dataverse/externaltools/ExternalToolHandler.java
+++ b/src/main/java/edu/harvard/iq/dataverse/externaltools/ExternalToolHandler.java
@@ -1,6 +1,9 @@
package edu.harvard.iq.dataverse.externaltools;
import edu.harvard.iq.dataverse.DataFile;
+import edu.harvard.iq.dataverse.Dataset;
+import edu.harvard.iq.dataverse.DatasetVersion;
+import edu.harvard.iq.dataverse.FileMetadata;
import edu.harvard.iq.dataverse.authorization.users.ApiToken;
import edu.harvard.iq.dataverse.externaltools.ExternalTool.ReservedWord;
import edu.harvard.iq.dataverse.util.SystemConfig;
@@ -24,6 +27,7 @@ public class ExternalToolHandler {
private final ExternalTool externalTool;
private final DataFile dataFile;
+ private final Dataset dataset;
private final ApiToken apiToken;
@@ -42,6 +46,7 @@ public ExternalToolHandler(ExternalTool externalTool, DataFile dataFile, ApiToke
}
this.dataFile = dataFile;
this.apiToken = apiToken;
+ dataset = getDataFile().getFileMetadata().getDatasetVersion().getDataset();
}
public DataFile getDataFile() {
@@ -67,7 +72,7 @@ public String getQueryParametersForUrl() {
String value = queryParam.getString(key);
String param = getQueryParam(key, value);
if (param != null && !param.isEmpty()) {
- params.add(getQueryParam(key, value));
+ params.add(param);
}
});
});
@@ -90,6 +95,20 @@ private String getQueryParam(String key, String value) {
return key + "=" + apiTokenString;
}
break;
+ case DATASET_ID:
+ return key + "=" + dataset.getId();
+ case DATASET_VERSION:
+ String version = null;
+ if (getApiToken() != null) {
+ version = dataset.getLatestVersion().getFriendlyVersionNumber();
+ } else {
+ version = dataset.getLatestVersionForCopy().getFriendlyVersionNumber();
+ }
+ if (("DRAFT").equals(version)) {
+ version = ":draft"; // send the token needed in api calls that can be substituted for a numeric
+ // version.
+ }
+ return key + "=" + version;
default:
break;
}
diff --git a/src/main/java/edu/harvard/iq/dataverse/externaltools/ExternalToolServiceBean.java b/src/main/java/edu/harvard/iq/dataverse/externaltools/ExternalToolServiceBean.java
index 35406a7f22b..eecb2126d4f 100644
--- a/src/main/java/edu/harvard/iq/dataverse/externaltools/ExternalToolServiceBean.java
+++ b/src/main/java/edu/harvard/iq/dataverse/externaltools/ExternalToolServiceBean.java
@@ -1,12 +1,16 @@
package edu.harvard.iq.dataverse.externaltools;
import edu.harvard.iq.dataverse.DataFile;
+import edu.harvard.iq.dataverse.DataFileServiceBean;
import edu.harvard.iq.dataverse.externaltools.ExternalTool.ReservedWord;
+import edu.harvard.iq.dataverse.externaltools.ExternalTool.Type;
+
import static edu.harvard.iq.dataverse.externaltools.ExternalTool.DESCRIPTION;
import static edu.harvard.iq.dataverse.externaltools.ExternalTool.DISPLAY_NAME;
import static edu.harvard.iq.dataverse.externaltools.ExternalTool.TOOL_PARAMETERS;
import static edu.harvard.iq.dataverse.externaltools.ExternalTool.TOOL_URL;
import static edu.harvard.iq.dataverse.externaltools.ExternalTool.TYPE;
+import static edu.harvard.iq.dataverse.externaltools.ExternalTool.CONTENT_TYPE;
import java.io.StringReader;
import java.util.ArrayList;
import java.util.List;
@@ -40,10 +44,14 @@ public List findAll() {
/**
+ * @param type
* @return A list of tools or an empty list.
*/
- public List findByType(ExternalTool.Type type) {
+ public List findByType(Type type) {
+
List externalTools = new ArrayList<>();
+
+ //If contentType==null, get all tools of the given ExternalTool.Type
TypedQuery typedQuery = em.createQuery("SELECT OBJECT(o) FROM ExternalTool AS o WHERE o.type = :type", ExternalTool.class);
typedQuery.setParameter("type", type);
List toolsFromQuery = typedQuery.getResultList();
@@ -53,6 +61,8 @@ public List findByType(ExternalTool.Type type) {
return externalTools;
}
+
+
public ExternalTool findById(long id) {
TypedQuery typedQuery = em.createQuery("SELECT OBJECT(o) FROM ExternalTool AS o WHERE o.id = :id", ExternalTool.class);
typedQuery.setParameter("id", id);
@@ -86,8 +96,11 @@ public ExternalTool save(ExternalTool externalTool) {
*/
public static List findExternalToolsByFile(List allExternalTools, DataFile file) {
List externalTools = new ArrayList<>();
+ //Map tabular data to it's mimetype (the isTabularData() check assures that this code works the same as before, but it may need to change if tabular data is split into subtypes with differing mimetypes)
+ final String contentType = file.isTabularData() ? DataFileServiceBean.MIME_TYPE_TSV_ALT : file.getContentType();
allExternalTools.forEach((externalTool) -> {
- if (file.isTabularData()) {
+ //Match tool and file type
+ if (contentType.equals(externalTool.getContentType())) {
externalTools.add(externalTool);
}
});
@@ -101,9 +114,16 @@ public static ExternalTool parseAddExternalToolManifest(String manifest) {
}
JsonReader jsonReader = Json.createReader(new StringReader(manifest));
JsonObject jsonObject = jsonReader.readObject();
+ //Note: ExternalToolServiceBeanTest tests are dependent on the order of these retrievals
String displayName = getRequiredTopLevelField(jsonObject, DISPLAY_NAME);
String description = getRequiredTopLevelField(jsonObject, DESCRIPTION);
String typeUserInput = getRequiredTopLevelField(jsonObject, TYPE);
+ String contentType = getOptionalTopLevelField(jsonObject, CONTENT_TYPE);
+ //Legacy support - assume tool manifests without any mimetype are for tabular data
+ if(contentType==null) {
+ contentType=DataFileServiceBean.MIME_TYPE_TSV_ALT;
+ }
+
// Allow IllegalArgumentException to bubble up from ExternalTool.Type.fromString
ExternalTool.Type type = ExternalTool.Type.fromString(typeUserInput);
String toolUrl = getRequiredTopLevelField(jsonObject, TOOL_URL);
@@ -125,7 +145,7 @@ public static ExternalTool parseAddExternalToolManifest(String manifest) {
throw new IllegalArgumentException("Required reserved word not found: " + ReservedWord.FILE_ID.toString());
}
String toolParameters = toolParametersObj.toString();
- return new ExternalTool(displayName, description, type, toolUrl, toolParameters);
+ return new ExternalTool(displayName, description, type, toolUrl, toolParameters, contentType);
}
private static String getRequiredTopLevelField(JsonObject jsonObject, String key) {
@@ -135,5 +155,14 @@ private static String getRequiredTopLevelField(JsonObject jsonObject, String key
throw new IllegalArgumentException(key + " is required.");
}
}
+
+ private static String getOptionalTopLevelField(JsonObject jsonObject, String key) {
+ try {
+ return jsonObject.getString(key);
+ } catch (NullPointerException ex) {
+ return null;
+ }
+ }
+
}
diff --git a/src/main/java/edu/harvard/iq/dataverse/metrics/MetricsServiceBean.java b/src/main/java/edu/harvard/iq/dataverse/metrics/MetricsServiceBean.java
index 10f9f7440f2..2b1caa22fe9 100644
--- a/src/main/java/edu/harvard/iq/dataverse/metrics/MetricsServiceBean.java
+++ b/src/main/java/edu/harvard/iq/dataverse/metrics/MetricsServiceBean.java
@@ -1,18 +1,22 @@
package edu.harvard.iq.dataverse.metrics;
import edu.harvard.iq.dataverse.Metric;
+import static edu.harvard.iq.dataverse.metrics.MetricsUtil.*;
import edu.harvard.iq.dataverse.util.SystemConfig;
import java.io.Serializable;
+import java.sql.Timestamp;
import java.text.SimpleDateFormat;
import java.time.LocalDate;
import java.time.LocalDateTime;
import java.time.ZoneId;
import java.util.Date;
import java.util.List;
+import java.util.logging.Level;
import java.util.logging.Logger;
import javax.ejb.EJB;
import javax.ejb.Stateless;
import javax.persistence.EntityManager;
+import javax.persistence.NoResultException;
import javax.persistence.NonUniqueResultException;
import javax.persistence.PersistenceContext;
import javax.persistence.Query;
@@ -29,12 +33,13 @@ public class MetricsServiceBean implements Serializable {
@EJB
SystemConfig systemConfig;
+
/** Dataverses */
/**
* @param yyyymm Month in YYYY-MM format.
*/
- public long dataversesToMonth(String yyyymm) throws Exception {
+ public long dataversesToMonth(String yyyymm) throws Exception {
Query query = em.createNativeQuery(""
+ "select count(dvobject.id)\n"
+ "from dataverse\n"
@@ -42,8 +47,8 @@ public long dataversesToMonth(String yyyymm) throws Exception {
+ "where dvobject.publicationdate is not null\n"
+ "and date_trunc('month', publicationdate) <= to_date('" + yyyymm + "','YYYY-MM');"
);
- logger.fine("query: " + query);
-
+ logger.log(Level.FINE, "Metric query: {0}", query);
+
return (long) query.getSingleResult();
}
@@ -55,9 +60,8 @@ public long dataversesPastDays(int days) throws Exception {
+ "where dvobject.publicationdate is not null\n"
+ "and publicationdate > current_date - interval '"+days+"' day;\n"
);
-
- logger.fine("query: " + query);
-
+ logger.log(Level.FINE, "Metric query: {0}", query);
+
return (long) query.getSingleResult();
}
@@ -70,27 +74,106 @@ public List dataversesByCategory() throws Exception {
+ "group by dataversetype\n"
+ "order by count desc;"
);
-
- logger.fine("query: " + query);
+ logger.log(Level.FINE, "Metric query: {0}", query);
+
return query.getResultList();
}
public List dataversesBySubject() {
Query query = em.createNativeQuery(""
+ "select cvv.strvalue, count(dataverse_id) from dataversesubjects\n"
- + "join controlledvocabularyvalue cvv ON cvv.id = controlledvocabularyvalue_id\n"
+ + "join controlledvocabularyvalue cvv ON cvv.id = controlledvocabularyvalue_id \n"
+ //+ "where dataverse_id != ( select id from dvobject where owner_id is null) \n" //removes root, we decided to do this in the homepage js instead
+ "group by cvv.strvalue\n"
+ "order by count desc;"
-
);
- logger.info("query: " + query);
-
+ logger.log(Level.FINE, "Metric query: {0}", query);
+
return query.getResultList();
}
/** Datasets */
- public List datasetsBySubject() {
+
+ /**
+ * @param yyyymm Month in YYYY-MM format.
+ */
+ public long datasetsToMonth(String yyyymm, String dataLocation) throws Exception {
+ String dataLocationLine = "(date_trunc('month', releasetime) <= to_date('" + yyyymm +"','YYYY-MM') and dataset.harvestingclient_id IS NULL)\n";
+
+ if(!DATA_LOCATION_LOCAL.equals(dataLocation)) { //Default api state is DATA_LOCATION_LOCAL
+ //we have to use createtime for harvest as post dvn3 harvests do not have releasetime populated
+ String harvestBaseLine = "(date_trunc('month', createtime) <= to_date('" + yyyymm +"','YYYY-MM') and dataset.harvestingclient_id IS NOT NULL)\n";
+ if (DATA_LOCATION_REMOTE.equals(dataLocation)) {
+ dataLocationLine = harvestBaseLine; //replace
+ } else if(DATA_LOCATION_ALL.equals(dataLocation)) {
+ dataLocationLine = "(" + dataLocationLine + " OR " + harvestBaseLine + ")\n"; //append
+ }
+ }
+
+ // Note that this SQL line in the code below:
+ // datasetversion.dataset_id || ':' || max(datasetversion.versionnumber + (.1 * datasetversion.minorversionnumber))
+ // behaves somewhat counter-intuitively if the versionnumber and/or
+ // minorversionnumber is/are NULL - it results in an empty string
+ // (NOT the string "{dataset_id}:", in other words). Some harvested
+ // versions do not have version numbers (only the ones harvested from
+ // other Dataverses!) It works fine
+ // for our purposes below, because we are simply counting the selected
+ // lines - i.e. we don't care if some of these lines are empty.
+ // But do not use this notation if you need the values returned to
+ // meaningfully identify the datasets!
+
+ Query query = em.createNativeQuery(
+ "select count(*)\n"
+ +"from (\n"
+ + "select datasetversion.dataset_id || ':' || max(datasetversion.versionnumber + (.1 * datasetversion.minorversionnumber))\n"
+ + "from datasetversion\n"
+ + "join dataset on dataset.id = datasetversion.dataset_id\n"
+ + "where versionstate='RELEASED' \n"
+ + "and \n"
+ + dataLocationLine //be careful about adding more and statements after this line.
+ + "group by dataset_id \n"
+ +") sub_temp"
+ );
+ logger.log(Level.FINE, "Metric query: {0}", query);
+
+ return (long) query.getSingleResult();
+ }
+
+ public List datasetsBySubjectToMonth(String yyyymm, String dataLocation) {
+ // The SQL code below selects the local, non-harvested dataset versions:
+ // A published local datasets may have more than one released version!
+ // So that's why we have to jump through some extra hoops below
+ // in order to select the latest one:
+ String originClause = "(datasetversion.dataset_id || ':' || datasetversion.versionnumber + (.1 * datasetversion.minorversionnumber) in\n" +
+ "(\n" +
+ "select datasetversion.dataset_id || ':' || max(datasetversion.versionnumber + (.1 * datasetversion.minorversionnumber))\n" +
+ " from datasetversion\n" +
+ " join dataset on dataset.id = datasetversion.dataset_id\n" +
+ " where versionstate='RELEASED'\n" +
+ " and dataset.harvestingclient_id is null\n" +
+ " and date_trunc('month', releasetime) <= to_date('" + yyyymm + "','YYYY-MM')\n" +
+ " group by dataset_id\n" +
+ "))\n";
+
+ if(!DATA_LOCATION_LOCAL.equals(dataLocation)) { //Default api state is DATA_LOCATION_LOCAL
+ //we have to use createtime for harvest as post dvn3 harvests do not have releasetime populated
+ // But we can operate on the assumption that all the harvested datasets
+ // are published, and there is always only one version per dataset -
+ // so the query is simpler:
+ String harvestOriginClause = "(\n" +
+ " datasetversion.dataset_id = dataset.id\n" +
+ " AND dataset.harvestingclient_id IS NOT null \n" +
+ " AND date_trunc('month', datasetversion.createtime) <= to_date('" + yyyymm + "','YYYY-MM')\n" +
+ ")\n";
+
+ if (DATA_LOCATION_REMOTE.equals(dataLocation)) {
+ originClause = harvestOriginClause; //replace
+ } else if(DATA_LOCATION_ALL.equals(dataLocation)) {
+ originClause = "(" + originClause + " OR " + harvestOriginClause + ")\n"; //append
+ }
+ }
+
Query query = em.createNativeQuery(""
+ "SELECT strvalue, count(dataset.id)\n"
+ "FROM datasetfield_controlledvocabularyvalue \n"
@@ -98,67 +181,44 @@ public List datasetsBySubject() {
+ "JOIN datasetfield ON datasetfield.id = datasetfield_controlledvocabularyvalue.datasetfield_id\n"
+ "JOIN datasetfieldtype ON datasetfieldtype.id = controlledvocabularyvalue.datasetfieldtype_id\n"
+ "JOIN datasetversion ON datasetversion.id = datasetfield.datasetversion_id\n"
- + "JOIN dvobject ON dvobject.id = datasetversion.dataset_id\n"
+ "JOIN dataset ON dataset.id = datasetversion.dataset_id\n"
+ "WHERE\n"
- + "datasetversion.dataset_id || ':' || datasetversion.versionnumber + (.1 * datasetversion.minorversionnumber) in \n"
- + "(\n"
- + "select datasetversion.dataset_id || ':' || max(datasetversion.versionnumber + (.1 * datasetversion.minorversionnumber)) as max \n"
- + "from datasetversion\n"
- + "join dataset on dataset.id = datasetversion.dataset_id\n"
- + "where versionstate='RELEASED'\n"
- + "and dataset.harvestingclient_id is null\n"
- + "group by dataset_id \n"
- + ")\n"
+ + originClause
+ "AND datasetfieldtype.name = 'subject'\n"
+ "GROUP BY strvalue\n"
+ "ORDER BY count(dataset.id) desc;"
);
- logger.info("query: " + query);
+ logger.log(Level.FINE, "Metric query: {0}", query);
return query.getResultList();
}
- /**
- * @param yyyymm Month in YYYY-MM format.
- */
- public long datasetsToMonth(String yyyymm) throws Exception {
- Query query = em.createNativeQuery(""
- + "select count(*)\n"
- + "from datasetversion\n"
- + "where datasetversion.dataset_id || ':' || datasetversion.versionnumber + (.1 * datasetversion.minorversionnumber) in \n"
- + "(\n"
- + "select datasetversion.dataset_id || ':' || max(datasetversion.versionnumber + (.1 * datasetversion.minorversionnumber)) as max \n"
- + "from datasetversion\n"
- + "join dataset on dataset.id = datasetversion.dataset_id\n"
- + "where versionstate='RELEASED' \n"
- + "and date_trunc('month', releasetime) <= to_date('" + yyyymm + "','YYYY-MM')\n"
- + "and dataset.harvestingclient_id is null\n"
- + "group by dataset_id \n"
- + ");"
- );
- logger.fine("query: " + query);
-
- return (long) query.getSingleResult();
- }
-
- public long datasetsPastDays(int days) throws Exception {
+ public long datasetsPastDays(int days, String dataLocation) throws Exception {
+ String dataLocationLine = "(releasetime > current_date - interval '"+days+"' day and dataset.harvestingclient_id IS NULL)\n";
+
+ if(!DATA_LOCATION_LOCAL.equals(dataLocation)) { //Default api state is DATA_LOCATION_LOCAL
+ //we have to use createtime for harvest as post dvn3 harvests do not have releasetime populated
+ String harvestBaseLine = "(createtime > current_date - interval '"+days+"' day and dataset.harvestingclient_id IS NOT NULL)\n";
+ if (DATA_LOCATION_REMOTE.equals(dataLocation)) {
+ dataLocationLine = harvestBaseLine; //replace
+ } else if(DATA_LOCATION_ALL.equals(dataLocation)) {
+ dataLocationLine += " or " +harvestBaseLine; //append
+ }
+ }
Query query = em.createNativeQuery(
- "select count(*)\n" +
- "from datasetversion\n" +
- "where datasetversion.dataset_id || ':' || datasetversion.versionnumber + (.1 * datasetversion.minorversionnumber) in \n" +
- "(\n" +
- " select datasetversion.dataset_id || ':' || max(datasetversion.versionnumber + (.1 * datasetversion.minorversionnumber)) as max \n" +
- " from datasetversion\n" +
- " join dataset on dataset.id = datasetversion.dataset_id\n" +
- " where versionstate='RELEASED' \n" +
- " and releasetime > current_date - interval '"+days+"' day\n" +
- " and dataset.harvestingclient_id is null\n" +
- " group by dataset_id \n" +
- ");"
+ "select count(*)\n"
+ +"from (\n"
+ + "select datasetversion.dataset_id || ':' || max(datasetversion.versionnumber + (.1 * datasetversion.minorversionnumber)) as max\n"
+ + "from datasetversion\n"
+ + "join dataset on dataset.id = datasetversion.dataset_id\n"
+ + "where versionstate='RELEASED' \n"
+ + "and \n"
+ + dataLocationLine //be careful about adding more and statements after this line.
+ + "group by dataset_id \n"
+ +") sub_temp"
);
- logger.fine("query: " + query);
+ logger.log(Level.FINE, "Metric query: {0}", query);
return (long) query.getSingleResult();
}
@@ -185,7 +245,8 @@ public long filesToMonth(String yyyymm) throws Exception {
+ "group by dataset_id \n"
+ ");"
);
- logger.fine("query: " + query);
+ logger.log(Level.FINE, "Metric query: {0}", query);
+
return (long) query.getSingleResult();
}
@@ -205,25 +266,50 @@ public long filesPastDays(int days) throws Exception {
+ "group by dataset_id \n"
+ ");"
);
-
- logger.fine("query: " + query);
+ logger.log(Level.FINE, "Metric query: {0}", query);
return (long) query.getSingleResult();
}
/** Downloads */
- /**
+ /*
+ * This includes getting historic download without a timestamp if query
+ * is earlier than earliest timestamped record
+ *
* @param yyyymm Month in YYYY-MM format.
*/
public long downloadsToMonth(String yyyymm) throws Exception {
- Query query = em.createNativeQuery(""
- + "select count(id)\n"
- + "from guestbookresponse\n"
- + "where date_trunc('month', responsetime) <= to_date('" + yyyymm + "','YYYY-MM');"
+ Query earlyDateQuery = em.createNativeQuery(""
+ + "select responsetime from guestbookresponse\n"
+ + "ORDER BY responsetime LIMIT 1;"
);
- logger.fine("query: " + query);
- return (long) query.getSingleResult();
+
+ try {
+ Timestamp earlyDateTimestamp = (Timestamp) earlyDateQuery.getSingleResult();
+ Date earliestDate = new Date(earlyDateTimestamp.getTime());
+ SimpleDateFormat formatter2 = new SimpleDateFormat("yyyy-MM");
+ Date dateQueried = formatter2.parse(yyyymm);
+
+ if(!dateQueried.before(earliestDate)) {
+ Query query = em.createNativeQuery(""
+ + "select count(id)\n"
+ + "from guestbookresponse\n"
+ + "where date_trunc('month', responsetime) <= to_date('" + yyyymm + "','YYYY-MM')"
+ + "or responsetime is NULL;" //includes historic guestbook records without date
+ );
+ logger.log(Level.FINE, "Metric query: {0}", query);
+ return (long) query.getSingleResult();
+ }
+ else {
+ //When we query before the earliest dated record, return 0;
+ return 0L;
+ }
+ } catch(NoResultException e) {
+ //If earlyDateQuery.getSingleResult is null, then there are no guestbooks and we can return 0
+ return 0L;
+ }
+
}
public long downloadsPastDays(int days) throws Exception {
@@ -232,37 +318,36 @@ public long downloadsPastDays(int days) throws Exception {
+ "from guestbookresponse\n"
+ "where responsetime > current_date - interval '"+days+"' day;\n"
);
-
- logger.fine("query: " + query);
+ logger.log(Level.FINE, "Metric query: {0}", query);
return (long) query.getSingleResult();
}
/** Helper functions for metric caching */
- public String returnUnexpiredCacheDayBased(String metricName, String days) throws Exception {
- Metric queriedMetric = getMetric(metricName, days);
+ public String returnUnexpiredCacheDayBased(String metricName, String days, String dataLocation) throws Exception {
+ Metric queriedMetric = getMetric(metricName, dataLocation, days);
if (!doWeQueryAgainDayBased(queriedMetric)) {
- return queriedMetric.getMetricValue();
+ return queriedMetric.getValueJson();
}
return null;
}
- public String returnUnexpiredCacheMonthly(String metricName, String yyyymm) throws Exception {
- Metric queriedMetric = getMetric(metricName, yyyymm);
+ public String returnUnexpiredCacheMonthly(String metricName, String yyyymm, String dataLocation) throws Exception {
+ Metric queriedMetric = getMetric(metricName, dataLocation, yyyymm);
if (!doWeQueryAgainMonthly(queriedMetric)) {
- return queriedMetric.getMetricValue();
+ return queriedMetric.getValueJson();
}
return null;
}
- public String returnUnexpiredCacheAllTime(String metricName) throws Exception {
- Metric queriedMetric = getMetric(metricName);
+ public String returnUnexpiredCacheAllTime(String metricName, String dataLocation) throws Exception {
+ Metric queriedMetric = getMetric(metricName, dataLocation, null); //MAD: not passing a date
if (!doWeQueryAgainAllTime(queriedMetric)) {
- return queriedMetric.getMetricValue();
+ return queriedMetric.getValueJson();
}
return null;
}
@@ -291,7 +376,7 @@ public boolean doWeQueryAgainMonthly(Metric queriedMetric) {
return true;
}
- String yyyymm = queriedMetric.getMetricDateString();
+ String yyyymm = queriedMetric.getDateString();
String thisMonthYYYYMM = MetricsUtil.getCurrentMonth();
Date lastCalled = queriedMetric.getLastCalledDate();
@@ -331,13 +416,9 @@ public boolean doWeQueryAgainAllTime(Metric queriedMetric) {
return (todayMinus.after(lastCalled));
}
- public Metric save(Metric newMetric, boolean monthly) throws Exception {
- Metric oldMetric;
- if (monthly) {
- oldMetric = getMetric(newMetric.getMetricTitle(), newMetric.getMetricDateString());
- } else {
- oldMetric = getMetric(newMetric.getMetricTitle());
- }
+ public Metric save(Metric newMetric) throws Exception {
+ Metric oldMetric = getMetric(newMetric.getName(), newMetric.getDataLocation(), newMetric.getDateString());
+
if (oldMetric != null) {
em.remove(oldMetric);
em.flush();
@@ -347,22 +428,33 @@ public Metric save(Metric newMetric, boolean monthly) throws Exception {
}
//This works for date and day based metrics
- public Metric getMetric(String metricTitle, String dayString) throws Exception {
- String searchMetricName = Metric.generateMetricName(metricTitle, dayString);
-
- return getMetric(searchMetricName);
- }
-
- public Metric getMetric(String searchMetricName) throws Exception {
- Query query = em.createQuery("select object(o) from Metric as o where o.metricName = :metricName", Metric.class);
- query.setParameter("metricName", searchMetricName);
+ //It is ok to pass null for dataLocation and dayString
+ public Metric getMetric(String name, String dataLocation, String dayString) throws Exception {
+ Query query = em.createQuery("select object(o) from Metric as o"
+ + " where o.name = :name"
+ + " and o.dataLocation" + (dataLocation == null ? " is null" : " = :dataLocation")
+ + " and o.dayString" + (dayString == null ? " is null" : " = :dayString")
+ , Metric.class);
+ query.setParameter("name", name);
+ if(dataLocation != null){ query.setParameter("dataLocation", dataLocation);}
+ if(dayString != null) {query.setParameter("dayString", dayString);}
+
+ logger.log(Level.FINE, "getMetric query: {0}", query);
+
Metric metric = null;
try {
metric = (Metric) query.getSingleResult();
} catch (javax.persistence.NoResultException nr) {
//do nothing
} catch (NonUniqueResultException nur) {
- throw new Exception("Multiple cached results found for this query. Contact your system administrator.");
+ //duplicates can happen when a new/requeried metric is called twice and saved twice before one can use the cache
+ //this remove all but the 0th index one in that case
+ for(int i = 1; i < query.getResultList().size(); i++) {
+ Metric extraMetric = (Metric) query.getResultList().get(i);
+ em.remove(extraMetric);
+ em.flush();
+ }
+ metric = (Metric) query.getResultList().get(0);
}
return metric;
}
diff --git a/src/main/java/edu/harvard/iq/dataverse/metrics/MetricsUtil.java b/src/main/java/edu/harvard/iq/dataverse/metrics/MetricsUtil.java
index 96a9ef53974..a3e4c68c848 100644
--- a/src/main/java/edu/harvard/iq/dataverse/metrics/MetricsUtil.java
+++ b/src/main/java/edu/harvard/iq/dataverse/metrics/MetricsUtil.java
@@ -1,10 +1,8 @@
package edu.harvard.iq.dataverse.metrics;
import edu.harvard.iq.dataverse.Dataverse;
-import edu.harvard.iq.dataverse.Metric;
import java.io.StringReader;
import java.time.LocalDate;
-import java.time.LocalDateTime;
import java.time.ZoneId;
import java.time.format.DateTimeFormatter;
import java.time.format.DateTimeFormatterBuilder;
@@ -30,6 +28,10 @@ public class MetricsUtil {
private final static String CATEGORY = "category";
private final static String SUBJECT = "subject";
public static String YEAR_AND_MONTH_PATTERN = "yyyy-MM";
+
+ public static final String DATA_LOCATION_LOCAL = "local";
+ public static final String DATA_LOCATION_REMOTE = "remote";
+ public static final String DATA_LOCATION_ALL = "all";
public static JsonObjectBuilder countToJson(long count) {
JsonObjectBuilder job = Json.createObjectBuilder();
@@ -112,6 +114,17 @@ public static String sanitizeYearMonthUserInput(String userInput) throws Excepti
return sanitized;
}
+ public static String validateDataLocationStringType(String dataLocation) throws Exception {
+ if( null == dataLocation || "".equals(dataLocation)) {
+ dataLocation = DATA_LOCATION_LOCAL;
+ }
+ if(!(DATA_LOCATION_LOCAL.equals(dataLocation) || DATA_LOCATION_REMOTE.equals(dataLocation) || DATA_LOCATION_ALL.equals(dataLocation))) {
+ throw new Exception("The inputted data location is not valid");
+ }
+
+ return dataLocation;
+ }
+
public static String getCurrentMonth() {
return LocalDate.now().format(DateTimeFormatter.ofPattern(MetricsUtil.YEAR_AND_MONTH_PATTERN));
}
diff --git a/src/main/java/edu/harvard/iq/dataverse/search/SearchIncludeFragment.java b/src/main/java/edu/harvard/iq/dataverse/search/SearchIncludeFragment.java
index da7a40eecc1..f75671515f5 100644
--- a/src/main/java/edu/harvard/iq/dataverse/search/SearchIncludeFragment.java
+++ b/src/main/java/edu/harvard/iq/dataverse/search/SearchIncludeFragment.java
@@ -1233,7 +1233,7 @@ public void setDisplayCardValues() {
if (dataverse.getId().equals(result.getParentIdAsLong())) {
// definitely NOT linked:
result.setIsInTree(true);
- } else if (result.getParentIdAsLong() == 1L) {
+ } else if (result.getParentIdAsLong() == dataverseService.findRootDataverse().getId()) {
// the object's parent is the root Dv; and the current
// Dv is NOT root... definitely linked:
result.setIsInTree(false);
diff --git a/src/main/java/edu/harvard/iq/dataverse/search/SearchServiceBean.java b/src/main/java/edu/harvard/iq/dataverse/search/SearchServiceBean.java
index 45fa189787a..77a5e3ef563 100644
--- a/src/main/java/edu/harvard/iq/dataverse/search/SearchServiceBean.java
+++ b/src/main/java/edu/harvard/iq/dataverse/search/SearchServiceBean.java
@@ -576,6 +576,10 @@ public SolrQueryResponse search(DataverseRequest dataverseRequest, List clazz = Class.forName(className);
+ if (AbstractSubmitToArchiveCommand.class.isAssignableFrom(clazz)) {
+ Constructor ctor;
+ ctor = clazz.getConstructor(DataverseRequest.class, DatasetVersion.class);
+ return (AbstractSubmitToArchiveCommand) ctor.newInstance(new Object[] { dvr, version });
+ }
+ } catch (Exception e) {
+ logger.warning("Unable to instantiate an Archiver of class: " + className);
+ e.printStackTrace();
+ }
+
+ return null;
+ }
+}
diff --git a/src/main/java/edu/harvard/iq/dataverse/util/BundleUtil.java b/src/main/java/edu/harvard/iq/dataverse/util/BundleUtil.java
index c1c3741e171..1168a4146b1 100644
--- a/src/main/java/edu/harvard/iq/dataverse/util/BundleUtil.java
+++ b/src/main/java/edu/harvard/iq/dataverse/util/BundleUtil.java
@@ -1,7 +1,5 @@
package edu.harvard.iq.dataverse.util;
-import edu.harvard.iq.dataverse.DataverseLocaleBean;
-
import java.io.File;
import java.net.URL;
import java.net.URLClassLoader;
@@ -12,13 +10,13 @@
import java.util.ResourceBundle;
import java.util.logging.Level;
import java.util.logging.Logger;
+import javax.faces.context.FacesContext;
public class BundleUtil {
private static final Logger logger = Logger.getLogger(BundleUtil.class.getCanonicalName());
private static final String defaultBundleFile = "Bundle";
- private static Locale bundle_locale;
public static String getStringFromBundle(String key) {
return getStringFromBundle(key, null);
@@ -26,6 +24,9 @@ public static String getStringFromBundle(String key) {
public static String getStringFromBundle(String key, List arguments) {
ResourceBundle bundle = getResourceBundle(defaultBundleFile );
+ if (bundle == null) {
+ return null;
+ }
return getStringFromBundle(key, arguments, bundle);
}
@@ -63,19 +64,21 @@ private static String getStringFromBundleNoMissingCheck(String key, List
public static String getStringFromPropertyFile(String key, String propertyFileName ) throws MissingResourceException {
ResourceBundle bundle = getResourceBundle(propertyFileName);
+ if (bundle == null) {
+ return null;
+ }
return getStringFromBundleNoMissingCheck(key, null, bundle);
}
- public static ResourceBundle getResourceBundle(String propertyFileName)
- {
- DataverseLocaleBean d = new DataverseLocaleBean();
+ public static ResourceBundle getResourceBundle(String propertyFileName) {
ResourceBundle bundle;
- bundle_locale = new Locale(d.getLocaleCode());
String filesRootDirectory = System.getProperty("dataverse.lang.directory");
+ Locale currentLocale = getCurrentLocale();
+
if (filesRootDirectory == null || filesRootDirectory.isEmpty()) {
- bundle = ResourceBundle.getBundle(propertyFileName, bundle_locale);
+ bundle = ResourceBundle.getBundle(propertyFileName, currentLocale);
} else {
File bundleFileDir = new File(filesRootDirectory);
URL[] urls = null;
@@ -83,12 +86,37 @@ public static ResourceBundle getResourceBundle(String propertyFileName)
urls = new URL[]{bundleFileDir.toURI().toURL()};
} catch (Exception e) {
e.printStackTrace();
+ return null;
}
ClassLoader loader = new URLClassLoader(urls);
- bundle = ResourceBundle.getBundle(propertyFileName, bundle_locale, loader);
+ bundle = ResourceBundle.getBundle(propertyFileName, currentLocale, loader);
}
return bundle ;
}
+
+ public static Locale getCurrentLocale() {
+ if (FacesContext.getCurrentInstance() == null) {
+ String localeEnvVar = System.getenv().get("LANG");
+ if (localeEnvVar != null) {
+ if (localeEnvVar.indexOf('.') > 0) {
+ localeEnvVar = localeEnvVar.substring(0, localeEnvVar.indexOf('.'));
+ }
+ if (!"en_US".equals(localeEnvVar)) {
+ logger.fine("BundleUtil: LOCALE code from the environmental variable is "+localeEnvVar);
+ return new Locale(localeEnvVar);
+ }
+ }
+
+ return new Locale("en");
+ } else if (FacesContext.getCurrentInstance().getViewRoot() == null) {
+ return FacesContext.getCurrentInstance().getExternalContext().getRequestLocale();
+ } else if (FacesContext.getCurrentInstance().getViewRoot().getLocale().getLanguage().equals("en_US")) {
+ return new Locale("en");
+ }
+
+ return FacesContext.getCurrentInstance().getViewRoot().getLocale();
+
+ }
}
diff --git a/src/main/java/edu/harvard/iq/dataverse/util/DateUtil.java b/src/main/java/edu/harvard/iq/dataverse/util/DateUtil.java
index 4009e199ea8..ffde59d6ce5 100644
--- a/src/main/java/edu/harvard/iq/dataverse/util/DateUtil.java
+++ b/src/main/java/edu/harvard/iq/dataverse/util/DateUtil.java
@@ -1,7 +1,5 @@
package edu.harvard.iq.dataverse.util;
-import edu.harvard.iq.dataverse.DataverseLocaleBean;
-
import java.sql.Timestamp;
import java.text.DateFormat;
import java.text.ParseException;
@@ -20,8 +18,7 @@ public static String formatDate(Date dateToformat) {
String formattedDate;
DateFormat dateFormatter;
try {
- DataverseLocaleBean d = new DataverseLocaleBean();
- Locale currentLocale = new Locale(d.getLocaleCode());
+ Locale currentLocale = BundleUtil.getCurrentLocale();
dateFormatter = DateFormat.getDateInstance(DateFormat.DEFAULT, currentLocale);
formattedDate = dateFormatter.format(dateToformat);
return formattedDate;
@@ -48,8 +45,7 @@ public static String formatDate(Timestamp datetimeToformat) {
String formattedDate;
DateFormat dateFormatter;
try {
- DataverseLocaleBean d = new DataverseLocaleBean();
- Locale currentLocale = new Locale(d.getLocaleCode());
+ Locale currentLocale = BundleUtil.getCurrentLocale();
dateFormatter = DateFormat.getDateTimeInstance(
DateFormat.DEFAULT,
DateFormat.LONG,
diff --git a/src/main/java/edu/harvard/iq/dataverse/util/FileUtil.java b/src/main/java/edu/harvard/iq/dataverse/util/FileUtil.java
index 3c890e53db1..64d959773d9 100644
--- a/src/main/java/edu/harvard/iq/dataverse/util/FileUtil.java
+++ b/src/main/java/edu/harvard/iq/dataverse/util/FileUtil.java
@@ -77,6 +77,7 @@
import java.util.zip.GZIPInputStream;
import java.util.zip.ZipEntry;
import java.util.zip.ZipInputStream;
+import static edu.harvard.iq.dataverse.datasetutility.FileSizeChecker.bytesToHumanReadable;
/**
@@ -505,7 +506,7 @@ private static boolean isGraphMLFile(File file) {
}
// from MD5Checksum.java
- public static String CalculateChecksum(String datafile, ChecksumType checksumType) {
+ public static String calculateChecksum(String datafile, ChecksumType checksumType) {
FileInputStream fis = null;
try {
@@ -514,11 +515,11 @@ public static String CalculateChecksum(String datafile, ChecksumType checksumTyp
throw new RuntimeException(ex);
}
- return CalculateChecksum(fis, checksumType);
+ return FileUtil.calculateChecksum(fis, checksumType);
}
// from MD5Checksum.java
- public static String CalculateChecksum(InputStream in, ChecksumType checksumType) {
+ public static String calculateChecksum(InputStream in, ChecksumType checksumType) {
MessageDigest md = null;
try {
// Use "SHA-1" (toString) rather than "SHA1", for example.
@@ -543,10 +544,28 @@ public static String CalculateChecksum(InputStream in, ChecksumType checksumType
}
}
- byte[] mdbytes = md.digest();
+ return checksumDigestToString(md.digest());
+ }
+
+ public static String calculateChecksum(byte[] dataBytes, ChecksumType checksumType) {
+ MessageDigest md = null;
+ try {
+ // Use "SHA-1" (toString) rather than "SHA1", for example.
+ md = MessageDigest.getInstance(checksumType.toString());
+ } catch (NoSuchAlgorithmException e) {
+ throw new RuntimeException(e);
+ }
+
+ md.update(dataBytes);
+
+ return checksumDigestToString(md.digest());
+
+ }
+
+ private static String checksumDigestToString(byte[] digestBytes) {
StringBuilder sb = new StringBuilder("");
- for (int i = 0; i < mdbytes.length; i++) {
- sb.append(Integer.toString((mdbytes[i] & 0xff) + 0x100, 16).substring(1));
+ for (int i = 0; i < digestBytes.length; i++) {
+ sb.append(Integer.toString((digestBytes[i] & 0xff) + 0x100, 16).substring(1));
}
return sb.toString();
}
@@ -1034,7 +1053,7 @@ private static DataFile createSingleDataFile(DatasetVersion version, File tempFi
try {
// We persist "SHA1" rather than "SHA-1".
datafile.setChecksumType(checksumType);
- datafile.setChecksumValue(CalculateChecksum(getFilesTempDirectory() + "/" + datafile.getStorageIdentifier(), datafile.getChecksumType()));
+ datafile.setChecksumValue(calculateChecksum(getFilesTempDirectory() + "/" + datafile.getStorageIdentifier(), datafile.getChecksumType()));
} catch (Exception cksumEx) {
logger.warning("Could not calculate " + checksumType + " signature for the new file " + fileName);
}
@@ -1328,11 +1347,18 @@ public static boolean isPubliclyDownloadable(FileMetadata fileMetadata) {
* This is what the UI displays for "Download URL" on the file landing page
* (DOIs rather than file IDs.
*/
- public static String getPublicDownloadUrl(String dataverseSiteUrl, String persistentId) {
- String path = "/api/access/datafile/:persistentId?persistentId=" + persistentId;
- return dataverseSiteUrl + path;
+ public static String getPublicDownloadUrl(String dataverseSiteUrl, String persistentId, Long fileId) {
+ String path = null;
+ if(persistentId != null) {
+ path = dataverseSiteUrl + "/api/access/datafile/:persistentId?persistentId=" + persistentId;
+ } else if( fileId != null) {
+ path = dataverseSiteUrl + "/api/access/datafile/" + fileId;
+ } else {
+ logger.info("In getPublicDownloadUrl but persistentId & fileId are both null!");
+ }
+ return path;
}
-
+
/**
* The FileDownloadServiceBean operates on file IDs, not DOIs.
*/
diff --git a/src/main/java/edu/harvard/iq/dataverse/util/LocalBundle.java b/src/main/java/edu/harvard/iq/dataverse/util/LocalBundle.java
index 01d1d9afc76..4039883138d 100644
--- a/src/main/java/edu/harvard/iq/dataverse/util/LocalBundle.java
+++ b/src/main/java/edu/harvard/iq/dataverse/util/LocalBundle.java
@@ -1,50 +1,20 @@
package edu.harvard.iq.dataverse.util;
-import edu.harvard.iq.dataverse.DataverseLocaleBean;
-import java.io.File;
-import java.io.FileNotFoundException;
-import java.io.FileReader;
-
-import java.io.IOException;
-import java.net.URL;
-import java.net.URLClassLoader;
import java.util.Enumeration;
-import java.util.Locale;
import java.util.ResourceBundle;
-import java.util.PropertyResourceBundle;
-
-import javax.faces.context.FacesContext;
public class LocalBundle extends ResourceBundle {
private static final String defaultBundleFile = "Bundle";
- private static ResourceBundle bundle;
- private static Locale bundle_locale;
public LocalBundle(){
- DataverseLocaleBean d = new DataverseLocaleBean();
- bundle_locale = new Locale(d.getLocaleCode());
-
- String filesRootDirectory = System.getProperty("dataverse.lang.directory");
-
- if (filesRootDirectory == null || filesRootDirectory.isEmpty()) {
- bundle = ResourceBundle.getBundle(defaultBundleFile, bundle_locale);
- } else {
- File bundleFileDir = new File(filesRootDirectory);
- URL[] urls = null;
- try {
- urls = new URL[]{bundleFileDir.toURI().toURL()};
- } catch (Exception e) {
- e.printStackTrace();
- }
-
- ClassLoader loader = new URLClassLoader(urls);
- bundle = ResourceBundle.getBundle(defaultBundleFile, bundle_locale, loader);
- }
+ ResourceBundle localBundle = BundleUtil.getResourceBundle(defaultBundleFile);
- setParent(bundle);
+ if (localBundle != null) {
+ setParent(localBundle);
+ }
}
@Override
diff --git a/src/main/java/edu/harvard/iq/dataverse/util/bagit/BagGenerator.java b/src/main/java/edu/harvard/iq/dataverse/util/bagit/BagGenerator.java
new file mode 100644
index 00000000000..6b13a194d5f
--- /dev/null
+++ b/src/main/java/edu/harvard/iq/dataverse/util/bagit/BagGenerator.java
@@ -0,0 +1,1062 @@
+package edu.harvard.iq.dataverse.util.bagit;
+
+import java.io.BufferedReader;
+import java.io.ByteArrayInputStream;
+import java.io.File;
+import java.io.FileOutputStream;
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.InputStreamReader;
+import java.io.OutputStream;
+import java.io.PrintWriter;
+import java.io.UnsupportedEncodingException;
+import java.net.MalformedURLException;
+import java.net.URI;
+import java.net.URISyntaxException;
+import java.nio.file.Paths;
+import java.security.KeyManagementException;
+import java.security.KeyStoreException;
+import java.security.NoSuchAlgorithmException;
+import java.text.SimpleDateFormat;
+import java.util.ArrayList;
+import java.util.Calendar;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.Iterator;
+import java.util.LinkedHashMap;
+import java.util.ResourceBundle;
+import java.util.Set;
+import java.util.TreeSet;
+import java.util.Map.Entry;
+import java.util.concurrent.ExecutionException;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Executors;
+import java.util.concurrent.TimeUnit;
+import java.util.logging.Level;
+import java.util.logging.Logger;
+import java.util.zip.ZipEntry;
+
+import org.apache.commons.codec.digest.DigestUtils;
+import org.apache.commons.compress.archivers.zip.ParallelScatterZipCreator;
+import org.apache.commons.compress.archivers.zip.ScatterZipOutputStream;
+import org.apache.commons.compress.archivers.zip.ZipArchiveEntry;
+import org.apache.commons.compress.archivers.zip.ZipArchiveEntryRequest;
+import org.apache.commons.compress.archivers.zip.ZipArchiveOutputStream;
+import org.apache.commons.compress.archivers.zip.ZipFile;
+import org.apache.commons.compress.parallel.InputStreamSupplier;
+import org.apache.commons.compress.utils.IOUtils;
+import org.apache.commons.lang3.StringUtils;
+import org.apache.commons.lang3.text.WordUtils;
+import org.apache.http.client.ClientProtocolException;
+import org.apache.http.client.config.RequestConfig;
+import org.apache.http.client.methods.CloseableHttpResponse;
+import org.apache.http.client.methods.HttpGet;
+import org.apache.http.client.protocol.HttpClientContext;
+import org.apache.http.config.Registry;
+import org.apache.http.config.RegistryBuilder;
+import org.apache.http.conn.socket.ConnectionSocketFactory;
+import org.apache.http.conn.ssl.NoopHostnameVerifier;
+import org.apache.http.conn.ssl.SSLConnectionSocketFactory;
+import org.apache.http.conn.ssl.TrustSelfSignedStrategy;
+import org.apache.http.ssl.SSLContextBuilder;
+
+import org.apache.http.impl.client.CloseableHttpClient;
+import org.apache.http.impl.client.HttpClients;
+import org.apache.http.impl.conn.PoolingHttpClientConnectionManager;
+import org.json.JSONArray;
+import com.google.gson.JsonArray;
+import com.google.gson.JsonElement;
+import com.google.gson.JsonObject;
+import com.google.gson.JsonParser;
+import com.google.gson.JsonPrimitive;
+import com.google.gson.JsonSyntaxException;
+
+import edu.harvard.iq.dataverse.DataFile;
+import edu.harvard.iq.dataverse.DataFile.ChecksumType;
+import edu.harvard.iq.dataverse.util.json.JsonLDTerm;
+
+public class BagGenerator {
+
+ private static final Logger logger = Logger.getLogger(BagGenerator.class.getCanonicalName());
+
+ private ParallelScatterZipCreator scatterZipCreator = null;
+ private ScatterZipOutputStream dirs = null;
+
+ private JsonArray aggregates = null;
+ private ArrayList resourceIndex = null;
+ private Boolean[] resourceUsed = null;
+ private HashMap pidMap = new LinkedHashMap();
+ private HashMap checksumMap = new LinkedHashMap();
+
+ private int timeout = 60;
+ private RequestConfig config = RequestConfig.custom().setConnectTimeout(timeout * 1000)
+ .setConnectionRequestTimeout(timeout * 1000).setSocketTimeout(timeout * 1000).build();
+ private static HttpClientContext localContext = HttpClientContext.create();
+ protected CloseableHttpClient client;
+ private PoolingHttpClientConnectionManager cm = null;
+
+ private ChecksumType hashtype = null;
+ private boolean ignorehashes = false;
+
+ private long dataCount = 0l;
+ private long totalDataSize = 0l;
+ private long maxFileSize = 0l;
+ private Set mimetypes = new TreeSet();
+
+ private String bagID = null;
+ private String bagPath = "/tmp";
+ String bagName = null;
+
+ private String apiKey = null;
+
+ private javax.json.JsonObject oremapObject;
+ private JsonObject aggregation;
+
+ private String dataciteXml;
+
+ private boolean usetemp = false;
+
+ private int numConnections = 8;
+
+ private OREMap oremap;
+
+ static PrintWriter pw = null;
+
+ /**
+ * This BagGenerator creates a BagIt version 1.0
+ * (https://tools.ietf.org/html/draft-kunze-bagit-16) compliant bag that is also
+ * minimally compatible with the Research Data Repository Interoperability WG
+ * Final Recommendations (DOI: 10.15497/RDA00025). It works by parsing the
+ * submitted OAI-ORE Map file, using the metadata therein to create required
+ * BagIt metadata, and using the schema.org/sameAs entries for
+ * AggregatedResources as a way to retrieve these files and store them in the
+ * /data directory within the BagIt structure. The Bag is zipped. File retrieval
+ * and zipping are done in parallel, using a connection pool. The required space
+ * on disk is ~ n+1/n of the final bag size, e.g. 125% of the bag size for a
+ * 4-way parallel zip operation.
+ * @throws Exception
+ * @throws JsonSyntaxException
+ */
+
+ public BagGenerator(OREMap oreMap, String dataciteXml) throws JsonSyntaxException, Exception {
+ this.oremap = oreMap;
+ this.oremapObject = oreMap.getOREMap();
+ //(JsonObject) new JsonParser().parse(oreMap.getOREMap().toString());
+ this.dataciteXml = dataciteXml;
+
+ try {
+ // Using Dataverse, all the URLs to be retrieved should be on the current server, so allowing self-signed certs and not verifying hostnames are useful in testing and
+ // shouldn't be a significant security issue. This should not be allowed for arbitrary OREMap sources.
+ SSLContextBuilder builder = new SSLContextBuilder();
+ try {
+ builder.loadTrustMaterial(null, new TrustSelfSignedStrategy());
+ } catch (KeyStoreException e) {
+ e.printStackTrace();
+ }
+
+ SSLConnectionSocketFactory sslConnectionFactory = new SSLConnectionSocketFactory(builder.build(), NoopHostnameVerifier.INSTANCE);
+
+ Registry registry = RegistryBuilder.create()
+ .register("https", sslConnectionFactory).build();
+ cm = new PoolingHttpClientConnectionManager(registry);
+
+ cm.setDefaultMaxPerRoute(numConnections);
+ cm.setMaxTotal(numConnections > 20 ? numConnections : 20);
+
+ client = HttpClients.custom().setConnectionManager(cm).setDefaultRequestConfig(config).build();
+
+ scatterZipCreator = new ParallelScatterZipCreator(Executors.newFixedThreadPool(numConnections));
+ } catch (NoSuchAlgorithmException | KeyManagementException e) {
+ logger.warning("Aint gonna work");
+ e.printStackTrace();
+ }
+ }
+
+ public void setIgnoreHashes(boolean val) {
+ ignorehashes = val;
+ }
+
+ public static void println(String s) {
+ System.out.println(s);
+ System.out.flush();
+ if (pw != null) {
+ pw.println(s);
+ pw.flush();
+ }
+ return;
+ }
+
+ /*
+ * Full workflow to generate new BagIt bag from ORE Map Url and to write the bag
+ * to the provided output stream (Ex: File OS, FTP OS etc.).
+ *
+ * @return success true/false
+ */
+ public boolean generateBag(OutputStream outputStream) throws Exception {
+ logger.info("Generating: Bag to the Future!");
+
+ File tmp = File.createTempFile("qdr-scatter-dirs", "tmp");
+ dirs = ScatterZipOutputStream.fileBased(tmp);
+ // The oremapObject is javax.json.JsonObject and we need com.google.gson.JsonObject for the aggregation object
+ aggregation = (JsonObject) new JsonParser().parse(oremapObject.getJsonObject(JsonLDTerm.ore("describes").getLabel()).toString());
+
+ bagID = aggregation.get("@id").getAsString() + "v."
+ + aggregation.get(JsonLDTerm.schemaOrg("version").getLabel()).getAsString();
+ try {
+ // Create valid filename from identifier and extend path with
+ // two levels of hash-based subdirs to help distribute files
+ bagName = getValidName(bagID);
+ } catch (Exception e) {
+ logger.severe("Couldn't create valid filename: " + e.getLocalizedMessage());
+ return false;
+ }
+ // Create data dir in bag, also creates parent bagName dir
+ String currentPath = "data/";
+ createDir(currentPath);
+
+ aggregates = aggregation.getAsJsonArray(JsonLDTerm.ore("aggregates").getLabel());
+
+ if (aggregates != null) {
+ // Add container and data entries
+ // Setup global index of the aggregation and all aggregated
+ // resources by Identifier
+ resourceIndex = indexResources(aggregation.get("@id").getAsString(), aggregates);
+ // Setup global list of succeed(true), fail(false), notused
+ // (null) flags
+ resourceUsed = new Boolean[aggregates.size() + 1];
+ // Process current container (the aggregation itself) and its
+ // children
+ processContainer(aggregation, currentPath);
+ }
+ // Create manifest files
+ // pid-mapping.txt - a DataOne recommendation to connect ids and
+ // in-bag path/names
+ StringBuffer pidStringBuffer = new StringBuffer();
+ boolean first = true;
+ for (Entry pidEntry : pidMap.entrySet()) {
+ if (!first) {
+ pidStringBuffer.append("\r\n");
+ } else {
+ first = false;
+ }
+ String path = pidEntry.getValue();
+ pidStringBuffer.append(pidEntry.getKey() + " " + path);
+ }
+ createDir("metadata/");
+ createFileFromString("metadata/pid-mapping.txt", pidStringBuffer.toString());
+ // Hash manifest - a hash manifest is required
+ // by Bagit spec
+ StringBuffer sha1StringBuffer = new StringBuffer();
+ first = true;
+ for (Entry sha1Entry : checksumMap.entrySet()) {
+ if (!first) {
+ sha1StringBuffer.append("\r\n");
+ } else {
+ first = false;
+ }
+ String path = sha1Entry.getKey();
+ sha1StringBuffer.append(sha1Entry.getValue() + " " + path);
+ }
+ if (!(hashtype == null)) {
+ String manifestName = "manifest-";
+ if (hashtype.equals(DataFile.ChecksumType.SHA1)) {
+ manifestName = manifestName + "sha1.txt";
+ } else if (hashtype.equals(DataFile.ChecksumType.SHA256)) {
+ manifestName = manifestName + "sha256.txt";
+ } else if (hashtype.equals(DataFile.ChecksumType.SHA512)) {
+ manifestName = manifestName + "sha512.txt";
+ } else if (hashtype.equals(DataFile.ChecksumType.MD5)) {
+ manifestName = manifestName + "md5.txt";
+ } else {
+ logger.warning("Unsupported Hash type: " + hashtype);
+ }
+ createFileFromString(manifestName, sha1StringBuffer.toString());
+ } else {
+ logger.warning("No Hash values sent - Bag File does not meet BagIT specification requirement");
+ }
+ // bagit.txt - Required by spec
+ createFileFromString("bagit.txt", "BagIt-Version: 1.0\r\nTag-File-Character-Encoding: UTF-8");
+
+ aggregation.addProperty(JsonLDTerm.totalSize.getLabel(), totalDataSize);
+ aggregation.addProperty(JsonLDTerm.fileCount.getLabel(), dataCount);
+ JsonArray mTypes = new JsonArray();
+ for (String mt : mimetypes) {
+ mTypes.add(new JsonPrimitive(mt));
+ }
+ aggregation.add(JsonLDTerm.dcTerms("format").getLabel(), mTypes);
+ aggregation.addProperty(JsonLDTerm.maxFileSize.getLabel(), maxFileSize);
+ // Serialize oremap itself
+ // FixMe - add missing hash values if needed and update context
+ // (read and cache files or read twice?)
+ createFileFromString("metadata/oai-ore.jsonld", oremapObject.toString());
+
+ createFileFromString("metadata/datacite.xml", dataciteXml);
+
+ // Add a bag-info file
+ createFileFromString("bag-info.txt", generateInfoFile());
+
+ logger.fine("Creating bag: " + bagName);
+
+ ZipArchiveOutputStream zipArchiveOutputStream = new ZipArchiveOutputStream(outputStream);
+
+ /*
+ * Add all the waiting contents - dirs created first, then data files are
+ * retrieved via URLs in parallel (defaults to one thread per processor)
+ * directly to the zip file
+ */
+ logger.fine("Starting write");
+ writeTo(zipArchiveOutputStream);
+ logger.fine("Zipfile Written");
+ // Finish
+ zipArchiveOutputStream.close();
+ logger.fine("Closed");
+
+ // Validate oremap - all entries are part of the collection
+ for (int i = 0; i < resourceUsed.length; i++) {
+ Boolean b = resourceUsed[i];
+ if (b == null) {
+ logger.warning("Problem: " + pidMap.get(resourceIndex.get(i)) + " was not used");
+ } else if (!b) {
+ logger.warning("Problem: " + pidMap.get(resourceIndex.get(i)) + " was not included successfully");
+ } else {
+ // Successfully included - now check for hash value and
+ // generate if needed
+ if (i > 0) { // Not root container
+ if (!checksumMap.containsKey(pidMap.get(resourceIndex.get(i)))) {
+
+ if (!childIsContainer(aggregates.get(i - 1).getAsJsonObject()))
+ logger.warning("Missing checksum hash for: " + resourceIndex.get(i));
+ // FixMe - actually generate it before adding the
+ // oremap
+ // to the zip
+ }
+ }
+ }
+
+ }
+
+ logger.info("Created bag: " + bagName);
+ client.close();
+ return true;
+
+ }
+
+ public boolean generateBag(String bagName, boolean temp) {
+ usetemp = temp;
+ FileOutputStream bagFileOS = null;
+ try {
+ File origBagFile = getBagFile(bagName);
+ File bagFile = origBagFile;
+ if (usetemp) {
+ bagFile = new File(bagFile.getAbsolutePath() + ".tmp");
+ logger.fine("Writing to: " + bagFile.getAbsolutePath());
+ }
+ // Create an output stream backed by the file
+ bagFileOS = new FileOutputStream(bagFile);
+ if (generateBag(bagFileOS)) {
+ validateBagFile(bagFile);
+ if (usetemp) {
+ logger.fine("Moving tmp zip");
+ origBagFile.delete();
+ bagFile.renameTo(origBagFile);
+ }
+ return true;
+ } else {
+ return false;
+ }
+ } catch (Exception e) {
+ logger.log(Level.SEVERE,"Bag Exception: ", e);
+ e.printStackTrace();
+ logger.warning("Failure: Processing failure during Bagit file creation");
+ return false;
+ } finally {
+ IOUtils.closeQuietly(bagFileOS);
+ }
+ }
+
+ public void validateBag(String bagId) {
+ logger.info("Validating Bag");
+ ZipFile zf = null;
+ InputStream is = null;
+ try {
+ zf = new ZipFile(getBagFile(bagId));
+ ZipArchiveEntry entry = zf.getEntry(getValidName(bagId) + "/manifest-sha1.txt");
+ if (entry != null) {
+ logger.info("SHA1 hashes used");
+ hashtype = DataFile.ChecksumType.SHA1;
+ } else {
+ entry = zf.getEntry(getValidName(bagId) + "/manifest-sha512.txt");
+ if (entry != null) {
+ logger.info("SHA512 hashes used");
+ hashtype = DataFile.ChecksumType.SHA512;
+ } else {
+ entry = zf.getEntry(getValidName(bagId) + "/manifest-sha256.txt");
+ if (entry != null) {
+ logger.info("SHA256 hashes used");
+ hashtype = DataFile.ChecksumType.SHA256;
+ } else {
+ entry = zf.getEntry(getValidName(bagId) + "/manifest-md5.txt");
+ if (entry != null) {
+ logger.info("MD5 hashes used");
+ hashtype = DataFile.ChecksumType.MD5;
+ }
+ }
+ }
+ }
+ if (entry == null)
+ throw new IOException("No manifest file found");
+ is = zf.getInputStream(entry);
+ BufferedReader br = new BufferedReader(new InputStreamReader(is));
+ String line = br.readLine();
+ while (line != null) {
+ logger.fine("Hash entry: " + line);
+ int breakIndex = line.indexOf(' ');
+ String hash = line.substring(0, breakIndex);
+ String path = line.substring(breakIndex + 1);
+ logger.fine("Adding: " + path + " with hash: " + hash);
+ checksumMap.put(path, hash);
+ line = br.readLine();
+ }
+ IOUtils.closeQuietly(is);
+ logger.info("HashMap Map contains: " + checksumMap.size() + " entries");
+ checkFiles(checksumMap, zf);
+ } catch (IOException io) {
+ logger.log(Level.SEVERE,"Could not validate Hashes", io);
+ } catch (Exception e) {
+ logger.log(Level.SEVERE,"Could not validate Hashes", e);
+ } finally {
+ IOUtils.closeQuietly(zf);
+ }
+ return;
+ }
+
+ public File getBagFile(String bagID) throws Exception {
+
+ String bagPath = Paths.get(getBagPath()).toString();
+ // Create the bag file on disk
+ File parent = new File(bagPath);
+ if (!parent.exists()) {
+ parent.mkdirs();
+ }
+ // Create known-good filename
+ bagName = getValidName(bagID);
+ File bagFile = new File(bagPath, bagName + ".zip");
+ logger.fine("BagPath: " + bagFile.getAbsolutePath());
+ // Create an output stream backed by the file
+ return bagFile;
+ }
+
+ private void validateBagFile(File bagFile) throws IOException {
+ // Run a confirmation test - should verify all files and hashes
+ ZipFile zf = new ZipFile(bagFile);
+ // Check files calculates the hashes and file sizes and reports on
+ // whether hashes are correct
+ checkFiles(checksumMap, zf);
+
+ logger.info("Data Count: " + dataCount);
+ logger.info("Data Size: " + totalDataSize);
+ zf.close();
+ }
+
+ public static String getValidName(String bagName) {
+ // Create known-good filename - no spaces, no file-system separators.
+ return bagName.replaceAll("\\W", "-");
+ }
+
+ private void processContainer(JsonObject item, String currentPath) throws IOException {
+ JsonArray children = getChildren(item);
+ HashSet titles = new HashSet();
+ String title = null;
+ if (item.has(JsonLDTerm.dcTerms("Title").getLabel())) {
+ title = item.get("Title").getAsString();
+ } else if (item.has(JsonLDTerm.schemaOrg("name").getLabel())) {
+ title = item.get(JsonLDTerm.schemaOrg("name").getLabel()).getAsString();
+ }
+
+ currentPath = currentPath + title + "/";
+ int containerIndex = -1;
+ try {
+ createDir(currentPath);
+ // Add containers to pid map and mark as 'used', but no sha1 hash
+ // value
+ containerIndex = getUnusedIndexOf(item.get("@id").getAsString());
+ resourceUsed[containerIndex] = true;
+ pidMap.put(item.get("@id").getAsString(), currentPath);
+
+ } catch (InterruptedException | IOException | ExecutionException e) {
+ e.printStackTrace();
+ logger.severe(e.getMessage());
+ if (containerIndex != -1) {
+ resourceUsed[containerIndex] = false;
+ }
+ throw new IOException("Unable to create bag");
+
+ }
+ for (int i = 0; i < children.size(); i++) {
+
+ // Find the ith child in the overall array of aggregated
+ // resources
+ String childId = children.get(i).getAsString();
+ logger.fine("Processing: " + childId);
+ int index = getUnusedIndexOf(childId);
+ if (resourceUsed[index] != null) {
+ System.out.println("Warning: reusing resource " + index);
+ }
+
+ // Aggregation is at index 0, so need to shift by 1 for aggregates
+ // entries
+ JsonObject child = aggregates.get(index - 1).getAsJsonObject();
+ if (childIsContainer(child)) {
+ // create dir and process children
+ // processContainer will mark this item as used
+ processContainer(child, currentPath);
+ } else {
+ resourceUsed[index] = true;
+ // add item
+ // ToDo
+ String dataUrl = child.get(JsonLDTerm.schemaOrg("sameAs").getLabel()).getAsString();
+ logger.fine("File url: " + dataUrl);
+ String childTitle = child.get(JsonLDTerm.schemaOrg("name").getLabel()).getAsString();
+ if (titles.contains(childTitle)) {
+ logger.warning("**** Multiple items with the same title in: " + currentPath);
+ logger.warning("**** Will cause failure in hash and size validation.");
+ } else {
+ titles.add(childTitle);
+ }
+ String childPath = currentPath + childTitle;
+
+ String childHash = null;
+ if (child.has(JsonLDTerm.checksum.getLabel())) {
+ ChecksumType childHashType = ChecksumType.fromString(
+ child.getAsJsonObject(JsonLDTerm.checksum.getLabel()).get("@type").getAsString());
+ if (hashtype != null && !hashtype.equals(childHashType)) {
+ logger.warning("Multiple hash values in use - not supported");
+ }
+ if (hashtype == null)
+ hashtype = childHashType;
+ childHash = child.getAsJsonObject(JsonLDTerm.checksum.getLabel()).get("@value").getAsString();
+ if (checksumMap.containsValue(childHash)) {
+ // Something else has this hash
+ logger.warning("Duplicate/Collision: " + child.get("@id").getAsString() + " has SHA1 Hash: "
+ + childHash);
+ }
+ checksumMap.put(childPath, childHash);
+ }
+ if ((hashtype == null) | ignorehashes) {
+ // Pick sha512 when ignoring hashes or none exist
+ hashtype = DataFile.ChecksumType.SHA512;
+ }
+ try {
+ if ((childHash == null) | ignorehashes) {
+ // Generate missing hashInputStream inputStream = null;
+ InputStream inputStream = null;
+ try {
+ inputStream = getInputStreamSupplier(dataUrl).get();
+
+ if (hashtype != null) {
+ if (hashtype.equals(DataFile.ChecksumType.SHA1)) {
+ childHash = DigestUtils.sha1Hex(inputStream);
+ } else if (hashtype.equals(DataFile.ChecksumType.SHA256)) {
+ childHash = DigestUtils.sha256Hex(inputStream);
+ } else if (hashtype.equals(DataFile.ChecksumType.SHA512)) {
+ childHash = DigestUtils.sha512Hex(inputStream);
+ } else if (hashtype.equals(DataFile.ChecksumType.MD5)) {
+ childHash = DigestUtils.md5Hex(inputStream);
+ }
+ }
+
+ } catch (IOException e) {
+ // TODO Auto-generated catch block
+ e.printStackTrace();
+ } finally {
+ IOUtils.closeQuietly(inputStream);
+ }
+ if (childHash != null) {
+ JsonObject childHashObject = new JsonObject();
+ childHashObject.addProperty("@type", hashtype.toString());
+ childHashObject.addProperty("@value", childHash);
+ child.add(JsonLDTerm.checksum.getLabel(), (JsonElement) childHashObject);
+
+ checksumMap.put(childPath, childHash);
+ } else {
+ logger.warning("Unable to calculate a " + hashtype + " for " + dataUrl);
+ }
+ }
+ logger.fine("Requesting: " + childPath + " from " + dataUrl);
+ createFileFromURL(childPath, dataUrl);
+ dataCount++;
+ if (dataCount % 1000 == 0) {
+ logger.info("Retrieval in progress: " + dataCount + " files retrieved");
+ }
+ if (child.has(JsonLDTerm.filesize.getLabel())) {
+ Long size = child.get(JsonLDTerm.filesize.getLabel()).getAsLong();
+ totalDataSize += size;
+ if (size > maxFileSize) {
+ maxFileSize = size;
+ }
+ }
+ if (child.has(JsonLDTerm.schemaOrg("fileFormat").getLabel())) {
+ mimetypes.add(child.get(JsonLDTerm.schemaOrg("fileFormat").getLabel()).getAsString());
+ }
+
+ } catch (Exception e) {
+ resourceUsed[index] = false;
+ e.printStackTrace();
+ throw new IOException("Unable to create bag");
+ }
+
+ // Check for nulls!
+ pidMap.put(child.get("@id").getAsString(), childPath);
+
+ }
+ }
+ }
+
+ private int getUnusedIndexOf(String childId) {
+ int index = resourceIndex.indexOf(childId);
+ if (resourceUsed[index] != null) {
+ System.out.println("Warning: reusing resource " + index);
+ }
+
+ while (resourceUsed[index] != null) {
+ int offset = index;
+ index = offset + 1 + resourceIndex.subList(offset + 1, resourceIndex.size()).indexOf(childId);
+ }
+ System.out.println("Using index: " + index);
+ if (index == -1) {
+ logger.severe("Reused ID: " + childId + " not found enough times in resource list");
+ }
+ return index;
+ }
+
+ private ArrayList indexResources(String aggId, JsonArray aggregates) {
+
+ ArrayList l = new ArrayList(aggregates.size() + 1);
+ l.add(aggId);
+ for (int i = 0; i < aggregates.size(); i++) {
+ logger.fine("Indexing : " + i + " " + aggregates.get(i).getAsJsonObject().get("@id").getAsString());
+ l.add(aggregates.get(i).getAsJsonObject().get("@id").getAsString());
+ }
+ logger.fine("Index created for " + aggregates.size() + " entries");
+ return l;
+ }
+
+ private void createDir(final String name) throws IOException, ExecutionException, InterruptedException {
+
+ ZipArchiveEntry archiveEntry = new ZipArchiveEntry(bagName + "/" + name);
+ archiveEntry.setMethod(ZipEntry.DEFLATED);
+ InputStreamSupplier supp = new InputStreamSupplier() {
+ public InputStream get() {
+ return new ByteArrayInputStream(("").getBytes());
+ }
+ };
+
+ addEntry(archiveEntry, supp);
+ }
+
+ private void createFileFromString(final String relPath, final String content)
+ throws IOException, ExecutionException, InterruptedException {
+
+ ZipArchiveEntry archiveEntry = new ZipArchiveEntry(bagName + "/" + relPath);
+ archiveEntry.setMethod(ZipEntry.DEFLATED);
+ InputStreamSupplier supp = new InputStreamSupplier() {
+ public InputStream get() {
+ try {
+ return new ByteArrayInputStream(content.getBytes("UTF-8"));
+ } catch (UnsupportedEncodingException e) {
+ e.printStackTrace();
+ }
+ return null;
+ }
+ };
+
+ addEntry(archiveEntry, supp);
+ }
+
+ private void createFileFromURL(final String relPath, final String uri)
+ throws IOException, ExecutionException, InterruptedException {
+
+ ZipArchiveEntry archiveEntry = new ZipArchiveEntry(bagName + "/" + relPath);
+ archiveEntry.setMethod(ZipEntry.DEFLATED);
+ InputStreamSupplier supp = getInputStreamSupplier(uri);
+ addEntry(archiveEntry, supp);
+ }
+
+ private void checkFiles(HashMap shaMap, ZipFile zf) {
+ ExecutorService executor = Executors.newFixedThreadPool(numConnections);
+ BagValidationJob.setZipFile(zf);
+ BagValidationJob.setBagGenerator(this);
+ logger.fine("Validating hashes for zipped data files");
+ int i = 0;
+ for (Entry entry : shaMap.entrySet()) {
+ BagValidationJob vj = new BagValidationJob(entry.getValue(), entry.getKey());
+ executor.execute(vj);
+ i++;
+ if (i % 1000 == 0) {
+ logger.info("Queuing Hash Validations: " + i);
+ }
+ }
+ logger.fine("All Hash Validations Queued: " + i);
+
+ executor.shutdown();
+ try {
+ while (!executor.awaitTermination(10, TimeUnit.MINUTES)) {
+ logger.fine("Awaiting completion of hash calculations.");
+ }
+ } catch (InterruptedException e) {
+ logger.log(Level.SEVERE,"Hash Calculations interrupted", e);
+ }
+ logger.fine("Hash Validations Completed");
+
+ }
+
+ public void addEntry(ZipArchiveEntry zipArchiveEntry, InputStreamSupplier streamSupplier) throws IOException {
+ if (zipArchiveEntry.isDirectory() && !zipArchiveEntry.isUnixSymlink())
+ dirs.addArchiveEntry(ZipArchiveEntryRequest.createZipArchiveEntryRequest(zipArchiveEntry, streamSupplier));
+ else
+ scatterZipCreator.addArchiveEntry(zipArchiveEntry, streamSupplier);
+ }
+
+ public void writeTo(ZipArchiveOutputStream zipArchiveOutputStream)
+ throws IOException, ExecutionException, InterruptedException {
+ logger.fine("Writing dirs");
+ dirs.writeTo(zipArchiveOutputStream);
+ dirs.close();
+ logger.fine("Dirs written");
+ scatterZipCreator.writeTo(zipArchiveOutputStream);
+ logger.fine("Files written");
+ }
+
+ static final String CRLF = "\r\n";
+
+ private String generateInfoFile() {
+ logger.fine("Generating info file");
+ StringBuffer info = new StringBuffer();
+
+ JsonArray contactsArray = new JsonArray();
+ /* Contact, and it's subfields, are terms from citation.tsv whose mapping to a formal vocabulary and label in the oremap may change
+ * so we need to find the labels used.
+ */
+ JsonLDTerm contactTerm = oremap.getContactTerm();
+ if ((contactTerm != null) && aggregation.has(contactTerm.getLabel())) {
+
+ JsonElement contacts = aggregation.get(contactTerm.getLabel());
+ JsonLDTerm contactNameTerm = oremap.getContactNameTerm();
+ JsonLDTerm contactEmailTerm = oremap.getContactEmailTerm();
+
+ if (contacts.isJsonArray()) {
+ for (int i = 0; i < contactsArray.size(); i++) {
+ info.append("Contact-Name: ");
+ JsonElement person = contactsArray.get(i);
+ if (person.isJsonPrimitive()) {
+ info.append(person.getAsString());
+ info.append(CRLF);
+
+ } else {
+ info.append(((JsonObject) person).get(contactNameTerm.getLabel()).getAsString());
+ info.append(CRLF);
+ if ((contactEmailTerm!=null) &&((JsonObject) person).has(contactEmailTerm.getLabel())) {
+ info.append("Contact-Email: ");
+ info.append(((JsonObject) person).get(contactEmailTerm.getLabel()).getAsString());
+ info.append(CRLF);
+ }
+ }
+ }
+ } else {
+ info.append("Contact-Name: ");
+
+ if (contacts.isJsonPrimitive()) {
+ info.append((String) contacts.getAsString());
+ info.append(CRLF);
+
+ } else {
+ JsonObject person = contacts.getAsJsonObject();
+
+ info.append(person.get(contactNameTerm.getLabel()).getAsString());
+ info.append(CRLF);
+ if ((contactEmailTerm!=null) && (person.has(contactEmailTerm.getLabel()))) {
+ info.append("Contact-Email: ");
+ info.append(person.get(contactEmailTerm.getLabel()).getAsString());
+ info.append(CRLF);
+ }
+ }
+
+ }
+ } else {
+ logger.warning("No contact info available for BagIt Info file");
+ }
+
+ info.append("Source-Organization: " + ResourceBundle.getBundle("Bundle").getString("bagit.sourceOrganization"));
+ // ToDo - make configurable
+ info.append(CRLF);
+
+ info.append("Organization-Address: " + WordUtils.wrap(
+ ResourceBundle.getBundle("Bundle").getString("bagit.sourceOrganizationAddress"), 78, CRLF + " ", true));
+ info.append(CRLF);
+
+ // Not a BagIt standard name
+ info.append(
+ "Organization-Email: " + ResourceBundle.getBundle("Bundle").getString("bagit.sourceOrganizationEmail"));
+ info.append(CRLF);
+
+ info.append("External-Description: ");
+
+ /* Description, and it's subfields, are terms from citation.tsv whose mapping to a formal vocabulary and label in the oremap may change
+ * so we need to find the labels used.
+ */
+ JsonLDTerm descriptionTerm = oremap.getDescriptionTerm();
+ JsonLDTerm descriptionTextTerm = oremap.getDescriptionTextTerm();
+ if (descriptionTerm == null) {
+ logger.warning("No description available for BagIt Info file");
+ } else {
+ info.append(
+ // FixMe - handle description having subfields better
+ WordUtils.wrap(getSingleValue(aggregation.getAsJsonObject(descriptionTerm.getLabel()),
+ descriptionTextTerm.getLabel()), 78, CRLF + " ", true));
+
+ info.append(CRLF);
+ }
+ info.append("Bagging-Date: ");
+ info.append((new SimpleDateFormat("yyyy-MM-dd").format(Calendar.getInstance().getTime())));
+ info.append(CRLF);
+
+ info.append("External-Identifier: ");
+ info.append(aggregation.get("@id").getAsString());
+ info.append(CRLF);
+
+ info.append("Bag-Size: ");
+ info.append(byteCountToDisplaySize(totalDataSize));
+ info.append(CRLF);
+
+ info.append("Payload-Oxum: ");
+ info.append(Long.toString(totalDataSize));
+ info.append(".");
+ info.append(Long.toString(dataCount));
+ info.append(CRLF);
+
+ info.append("Internal-Sender-Identifier: ");
+ String catalog = ResourceBundle.getBundle("Bundle").getString("bagit.sourceOrganization") + " Catalog";
+ if (aggregation.has(JsonLDTerm.schemaOrg("includedInDataCatalog").getLabel())) {
+ catalog = aggregation.get(JsonLDTerm.schemaOrg("includedInDataCatalog").getLabel()).getAsString();
+ }
+ info.append(catalog + ":" + aggregation.get(JsonLDTerm.schemaOrg("name").getLabel()).getAsString());
+ info.append(CRLF);
+
+ return info.toString();
+
+ }
+
+ /**
+ * Kludge - handle when a single string is sent as an array of 1 string and, for
+ * cases where multiple values are sent when only one is expected, create a
+ * concatenated string so that information is not lost.
+ *
+ * @param jsonObject
+ * - the root json object
+ * @param key
+ * - the key to find a value(s) for
+ * @return - a single string
+ */
+ String getSingleValue(JsonObject jsonObject, String key) {
+ String val = "";
+ if (jsonObject.get(key).isJsonPrimitive()) {
+ val = jsonObject.get(key).getAsString();
+ } else if (jsonObject.get(key).isJsonArray()) {
+ Iterator iter = jsonObject.getAsJsonArray(key).iterator();
+ ArrayList stringArray = new ArrayList();
+ while (iter.hasNext()) {
+ stringArray.add(iter.next().getAsString());
+ }
+ if (stringArray.size() > 1) {
+ val = StringUtils.join((String[]) stringArray.toArray(), ",");
+ } else {
+ val = stringArray.get(0);
+ }
+ logger.warning("Multiple values found for: " + key + ": " + val);
+ }
+ return val;
+ }
+
+ // Used in validation
+
+ public void incrementTotalDataSize(long inc) {
+ totalDataSize += inc;
+ }
+
+ public String getHashtype() {
+ return hashtype.toString();
+ }
+
+ // Get's all "Has Part" children, standardized to send an array with 0,1, or
+ // more elements
+ private static JsonArray getChildren(JsonObject parent) {
+ JsonElement o = null;
+ o = parent.get(JsonLDTerm.schemaOrg("hasPart").getLabel());
+ if (o == null) {
+ return new JsonArray();
+ } else {
+ if (o.isJsonArray()) {
+ return (JsonArray) o;
+ } else if (o.isJsonPrimitive()) {
+ JsonArray children = new JsonArray();
+ children.add(o);
+ return (children);
+ }
+ logger.severe("Error finding children: " + o.toString());
+ return new JsonArray();
+ }
+ }
+
+ // Logic to decide if this is a container -
+ // first check for children, then check for source-specific type indicators
+ private static boolean childIsContainer(JsonObject item) {
+ if (getChildren(item).size() != 0) {
+ return true;
+ }
+ // Also check for any indicative type
+ Object o = item.get("@type");
+ if (o != null) {
+ if (o instanceof JSONArray) {
+ // As part of an array
+ for (int i = 0; i < ((JSONArray) o).length(); i++) {
+ String type = ((JSONArray) o).getString(i).trim();
+ if ("http://cet.ncsa.uiuc.edu/2016/Folder".equals(type)) {
+ return true;
+ }
+ }
+ } else if (o instanceof String) {
+ // Or as the only type
+ String type = ((String) o).trim();
+ if ("http://cet.ncsa.uiuc.edu/2016/Folder".equals(type)) {
+ return true;
+ }
+ }
+ }
+ return false;
+ }
+
+ public String getBagPath() {
+ return bagPath;
+ }
+
+ public void setBagPath(String bagPath) {
+ this.bagPath = bagPath;
+ }
+
+ private HttpGet createNewGetRequest(URI url, String returnType) {
+
+ HttpGet request = null;
+
+ if (apiKey != null) {
+ try {
+ String urlString = url.toURL().toString();
+ // Add key as param - check whether it is the only param or not
+ urlString = urlString + ((urlString.indexOf('?') != -1) ? "&key=" : "?key=") + apiKey;
+ request = new HttpGet(new URI(urlString));
+ } catch (MalformedURLException e) {
+ // TODO Auto-generated catch block
+ e.printStackTrace();
+ } catch (URISyntaxException e) {
+ // TODO Auto-generated catch block
+ e.printStackTrace();
+ }
+ } else {
+ request = new HttpGet(url);
+ }
+ if (returnType != null) {
+ request.addHeader("accept", returnType);
+ }
+ return request;
+ }
+
+ InputStreamSupplier getInputStreamSupplier(final String uri) {
+
+ return new InputStreamSupplier() {
+ public InputStream get() {
+ int tries = 0;
+ while (tries < 5) {
+ try {
+ logger.fine("Get # " + tries + " for " + uri);
+ HttpGet getMap = createNewGetRequest(new URI(uri), null);
+ logger.finest("Retrieving " + tries + ": " + uri);
+ CloseableHttpResponse response;
+ response = client.execute(getMap, localContext);
+ if (response.getStatusLine().getStatusCode() == 200) {
+ logger.finest("Retrieved: " + uri);
+ return response.getEntity().getContent();
+ }
+ logger.fine("Status: " + response.getStatusLine().getStatusCode());
+ tries++;
+
+ } catch (ClientProtocolException e) {
+ tries += 5;
+ // TODO Auto-generated catch block
+ e.printStackTrace();
+ } catch (IOException e) {
+ // Retry if this is a potentially temporary error such
+ // as a timeout
+ tries++;
+ logger.log(Level.WARNING,"Attempt# " + tries + " : Unable to retrieve file: " + uri, e);
+ if (tries == 5) {
+ logger.severe("Final attempt failed for " + uri);
+ }
+ e.printStackTrace();
+ } catch (URISyntaxException e) {
+ tries += 5;
+ // TODO Auto-generated catch block
+ e.printStackTrace();
+ }
+ }
+ logger.severe("Could not read: " + uri);
+ return null;
+ }
+ };
+ }
+
+ /**
+ * Adapted from org/apache/commons/io/FileUtils.java change to SI - add 2 digits
+ * of precision
+ */
+ /**
+ * The number of bytes in a kilobyte.
+ */
+ public static final long ONE_KB = 1000;
+
+ /**
+ * The number of bytes in a megabyte.
+ */
+ public static final long ONE_MB = ONE_KB * ONE_KB;
+
+ /**
+ * The number of bytes in a gigabyte.
+ */
+ public static final long ONE_GB = ONE_KB * ONE_MB;
+
+ /**
+ * Returns a human-readable version of the file size, where the input represents
+ * a specific number of bytes.
+ *
+ * @param size
+ * the number of bytes
+ * @return a human-readable display value (includes units)
+ */
+ public static String byteCountToDisplaySize(long size) {
+ String displaySize;
+
+ if (size / ONE_GB > 0) {
+ displaySize = String.valueOf(Math.round(size / (ONE_GB / 100.0d)) / 100.0) + " GB";
+ } else if (size / ONE_MB > 0) {
+ displaySize = String.valueOf(Math.round(size / (ONE_MB / 100.0d)) / 100.0) + " MB";
+ } else if (size / ONE_KB > 0) {
+ displaySize = String.valueOf(Math.round(size / (ONE_KB / 100.0d)) / 100.0) + " KB";
+ } else {
+ displaySize = String.valueOf(size) + " bytes";
+ }
+ return displaySize;
+ }
+
+ public void setAuthenticationKey(String tokenString) {
+ apiKey = tokenString;
+ }
+
+}
\ No newline at end of file
diff --git a/src/main/java/edu/harvard/iq/dataverse/util/bagit/BagValidationJob.java b/src/main/java/edu/harvard/iq/dataverse/util/bagit/BagValidationJob.java
new file mode 100644
index 00000000000..7a32b96f4a0
--- /dev/null
+++ b/src/main/java/edu/harvard/iq/dataverse/util/bagit/BagValidationJob.java
@@ -0,0 +1,123 @@
+package edu.harvard.iq.dataverse.util.bagit;
+/*
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ * @author qqmyers@hotmail.com
+ */
+
+import java.io.IOException;
+import java.io.InputStream;
+import java.util.logging.Logger;
+import java.util.zip.ZipException;
+
+import org.apache.commons.codec.digest.DigestUtils;
+import org.apache.commons.compress.archivers.zip.ZipArchiveEntry;
+import org.apache.commons.compress.archivers.zip.ZipFile;
+
+import edu.harvard.iq.dataverse.DataFile;
+
+import org.apache.commons.compress.utils.IOUtils;
+
+/**
+ * @author Jim
+ *
+ */
+public class BagValidationJob implements Runnable {
+
+ private static final Logger log = Logger.getLogger(BagValidationJob.class.getCanonicalName());
+
+ private static ZipFile zf = null;
+ private static BagGenerator bagGenerator = null;
+
+ private String hash;
+ private String name;
+ private static String hashtype;
+
+ public BagValidationJob(String value, String key) throws IllegalStateException {
+ if (zf == null || bagGenerator == null) {
+ throw new IllegalStateException(
+ "Static Zipfile and BagGenerator must be set before creating ValidationJobs");
+ }
+ hash = value;
+ name = key;
+
+ }
+
+ /*
+ * (non-Javadoc)
+ *
+ * @see java.lang.Runnable#run()
+ */
+ public void run() {
+
+ String realHash = generateFileHash(name, zf);
+ if (hash.equals(realHash)) {
+ log.fine("Valid hash for " + name);
+ } else {
+ log.severe("Invalid " + bagGenerator.getHashtype() + " for " + name);
+ log.fine("As sent: " + hash);
+ log.fine("As calculated: " + realHash);
+ }
+ }
+
+ private String generateFileHash(String name, ZipFile zf) {
+
+ ZipArchiveEntry archiveEntry1 = zf.getEntry(name);
+ // Error check - add file sizes to compare against supplied stats
+
+ long start = System.currentTimeMillis();
+ InputStream inputStream = null;
+ String realHash = null;
+ try {
+ inputStream = zf.getInputStream(archiveEntry1);
+ if (hashtype.equals(DataFile.ChecksumType.SHA1)) {
+ realHash = DigestUtils.sha1Hex(inputStream);
+ } else if (hashtype.equals(DataFile.ChecksumType.SHA256)) {
+ realHash = DigestUtils.sha256Hex(inputStream);
+ } else if (hashtype.equals(DataFile.ChecksumType.SHA512)) {
+ realHash = DigestUtils.sha512Hex(inputStream);
+ } else if (hashtype.equals(DataFile.ChecksumType.MD5)) {
+ realHash = DigestUtils.md5Hex(inputStream);
+ } else {
+ log.warning("Unknown hash type: " + hashtype);
+ }
+
+ } catch (ZipException e) {
+ // TODO Auto-generated catch block
+ e.printStackTrace();
+ } catch (IOException e) {
+ // TODO Auto-generated catch block
+ e.printStackTrace();
+ } finally {
+ IOUtils.closeQuietly(inputStream);
+ }
+ log.fine("Retrieve/compute time = " + (System.currentTimeMillis() - start) + " ms");
+ // Error check - add file sizes to compare against supplied stats
+ bagGenerator.incrementTotalDataSize(archiveEntry1.getSize());
+ return realHash;
+ }
+
+ public static void setZipFile(ZipFile zf) {
+ BagValidationJob.zf = zf;
+ }
+
+ public static void setBagGenerator(BagGenerator bg) {
+ bagGenerator = bg;
+ hashtype = bagGenerator.getHashtype();
+ if (hashtype == null) {
+ log.warning("Null hashtype. Validation will not occur");
+ }
+
+ }
+
+}
diff --git a/src/main/java/edu/harvard/iq/dataverse/util/bagit/OREMap.java b/src/main/java/edu/harvard/iq/dataverse/util/bagit/OREMap.java
index d0b1f28126b..d95d0fe50c0 100644
--- a/src/main/java/edu/harvard/iq/dataverse/util/bagit/OREMap.java
+++ b/src/main/java/edu/harvard/iq/dataverse/util/bagit/OREMap.java
@@ -59,10 +59,10 @@ public JsonObject getOREMap() throws Exception {
Dataset dataset = version.getDataset();
String id = dataset.getGlobalId().asString();
JsonArrayBuilder fileArray = Json.createArrayBuilder();
- //The map describes an aggregation
+ // The map describes an aggregation
JsonObjectBuilder aggBuilder = Json.createObjectBuilder();
List fields = version.getDatasetFields();
- //That has it's own metadata
+ // That has it's own metadata
for (DatasetField field : fields) {
if (!field.isEmpty()) {
DatasetFieldType dfType = field.getDatasetFieldType();
@@ -124,7 +124,7 @@ public JsonObject getOREMap() throws Exception {
aggBuilder.add(fieldName.getLabel(), (valArray.size() != 1) ? valArray : valArray.get(0));
}
}
- //Add metadata related to the Dataset/DatasetVersion
+ // Add metadata related to the Dataset/DatasetVersion
aggBuilder.add("@id", id)
.add("@type",
Json.createArrayBuilder().add(JsonLDTerm.ore("Aggregation").getLabel())
@@ -149,7 +149,7 @@ public JsonObject getOREMap() throws Exception {
addIfNotNull(aggBuilder, JsonLDTerm.conditions, terms.getConditions());
addIfNotNull(aggBuilder, JsonLDTerm.disclaimer, terms.getDisclaimer());
- //Add fileTermsofAccess as an object since it is compound
+ // Add fileTermsofAccess as an object since it is compound
JsonObjectBuilder fAccess = Json.createObjectBuilder();
addIfNotNull(fAccess, JsonLDTerm.termsOfAccess, terms.getTermsOfAccess());
addIfNotNull(fAccess, JsonLDTerm.fileRequestAccess, terms.isFileAccessRequest());
@@ -200,7 +200,7 @@ public JsonObject getOREMap() throws Exception {
// File DOI if it exists
String fileId = null;
String fileSameAs = null;
- if (df.getGlobalId() != null) {
+ if (df.getGlobalId().asString().length() != 0) {
fileId = df.getGlobalId().asString();
fileSameAs = SystemConfig.getDataverseSiteUrlStatic()
+ "/api/access/datafile/:persistentId?persistentId=" + fileId;
@@ -222,7 +222,7 @@ public JsonObject getOREMap() throws Exception {
addIfNotNull(aggRes, JsonLDTerm.rootDataFileId, df.getRootDataFileId());
addIfNotNull(aggRes, JsonLDTerm.previousDataFileId, df.getPreviousDataFileId());
JsonObject checksum = null;
- //Add checksum. RDA recommends SHA-512
+ // Add checksum. RDA recommends SHA-512
if (df.getChecksumType() != null && df.getChecksumValue() != null) {
checksum = Json.createObjectBuilder().add("@type", df.getChecksumType().toString())
.add("@value", df.getChecksumValue()).build();
@@ -234,15 +234,15 @@ public JsonObject getOREMap() throws Exception {
tabTags = jab.build();
}
addIfNotNull(aggRes, JsonLDTerm.tabularTags, tabTags);
- //Add lates resource to the array
+ //Add latest resource to the array
aggResArrayBuilder.add(aggRes.build());
}
- //Build the '@context' object for json-ld based on the localContext entries
+ // Build the '@context' object for json-ld based on the localContext entries
JsonObjectBuilder contextBuilder = Json.createObjectBuilder();
for (Entry e : localContext.entrySet()) {
contextBuilder.add(e.getKey(), e.getValue());
}
- //Now create the overall map object with it's metadata
+ // Now create the overall map object with it's metadata
JsonObject oremap = Json.createObjectBuilder()
.add(JsonLDTerm.dcTerms("modified").getLabel(), LocalDate.now().toString())
.add(JsonLDTerm.dcTerms("creator").getLabel(),
@@ -266,7 +266,7 @@ public JsonObject getOREMap() throws Exception {
* Simple methods to only add an entry to JSON if the value of the term is
* non-null. Methods created for string, JsonValue, boolean, and long
*/
-
+
private void addIfNotNull(JsonObjectBuilder builder, JsonLDTerm key, String value) {
if (value != null) {
builder.add(key.getLabel(), value);
diff --git a/src/main/java/edu/harvard/iq/dataverse/util/json/JsonParser.java b/src/main/java/edu/harvard/iq/dataverse/util/json/JsonParser.java
index bdefddf46da..3b8efa17513 100644
--- a/src/main/java/edu/harvard/iq/dataverse/util/json/JsonParser.java
+++ b/src/main/java/edu/harvard/iq/dataverse/util/json/JsonParser.java
@@ -719,6 +719,12 @@ public WorkflowStepData parseStepData( JsonObject json ) throws JsonParseExcepti
params.keySet().forEach(k -> paramMap.put(k,jsonValueToString(params.get(k))));
wsd.setStepParameters(paramMap);
}
+ if ( json.containsKey("requiredSettings") ) {
+ JsonObject settings = json.getJsonObject("requiredSettings");
+ Map settingsMap = new HashMap<>();
+ settings.keySet().forEach(k -> settingsMap.put(k,jsonValueToString(settings.get(k))));
+ wsd.setStepSettings(settingsMap);
+ }
return wsd;
}
diff --git a/src/main/java/edu/harvard/iq/dataverse/util/json/JsonPrinter.java b/src/main/java/edu/harvard/iq/dataverse/util/json/JsonPrinter.java
index 91123c9d6be..5f81dc554e2 100644
--- a/src/main/java/edu/harvard/iq/dataverse/util/json/JsonPrinter.java
+++ b/src/main/java/edu/harvard/iq/dataverse/util/json/JsonPrinter.java
@@ -221,7 +221,7 @@ public static JsonObjectBuilder json(Workflow wf){
arr.add( jsonObjectBuilder().add("stepType", stp.getStepType())
.add("provider", stp.getProviderId())
.add("parameters", mapToObject(stp.getStepParameters()))
- .add("requiredSettings", mapToObject(stp.getStepParameters())) );
+ .add("requiredSettings", mapToObject(stp.getStepSettings())) );
}
bld.add("steps", arr );
}
diff --git a/src/main/java/edu/harvard/iq/dataverse/workflow/internalspi/ArchivalSubmissionWorkflowStep.java b/src/main/java/edu/harvard/iq/dataverse/workflow/internalspi/ArchivalSubmissionWorkflowStep.java
new file mode 100644
index 00000000000..105af6a00d8
--- /dev/null
+++ b/src/main/java/edu/harvard/iq/dataverse/workflow/internalspi/ArchivalSubmissionWorkflowStep.java
@@ -0,0 +1,72 @@
+package edu.harvard.iq.dataverse.workflow.internalspi;
+
+import edu.harvard.iq.dataverse.DatasetVersion;
+import edu.harvard.iq.dataverse.engine.command.DataverseRequest;
+import edu.harvard.iq.dataverse.engine.command.impl.AbstractSubmitToArchiveCommand;
+import edu.harvard.iq.dataverse.settings.SettingsServiceBean;
+import edu.harvard.iq.dataverse.util.ArchiverUtil;
+import edu.harvard.iq.dataverse.workflow.WorkflowContext;
+import edu.harvard.iq.dataverse.workflow.step.Failure;
+import edu.harvard.iq.dataverse.workflow.step.WorkflowStep;
+import edu.harvard.iq.dataverse.workflow.step.WorkflowStepResult;
+
+import java.lang.reflect.Constructor;
+import java.util.HashMap;
+import java.util.Map;
+import java.util.logging.Level;
+import java.util.logging.Logger;
+
+import javax.servlet.http.HttpServletRequest;
+
+/**
+ * A step that submits a BagIT bag of the newly published dataset version via a
+ * configured archiver.
+ *
+ * @author jimmyers
+ */
+
+public class ArchivalSubmissionWorkflowStep implements WorkflowStep {
+
+ private static final Logger logger = Logger.getLogger(ArchivalSubmissionWorkflowStep.class.getName());
+
+ public ArchivalSubmissionWorkflowStep(Map paramSet) {
+ }
+
+ @Override
+ public WorkflowStepResult run(WorkflowContext context) {
+ Map requestedSettings = new HashMap();
+ Map typedSettings = context.getSettings();
+ for (String setting : (typedSettings.keySet())) {
+ Object val = typedSettings.get(setting);
+ if (val instanceof String) {
+ requestedSettings.put(setting, (String) val);
+ } else if (val instanceof Boolean) {
+ requestedSettings.put(setting, ((Boolean) val).booleanValue() ? "true" : "false");
+ } else if (val instanceof Long) {
+ requestedSettings.put(setting, val.toString());
+ }
+ }
+
+ DataverseRequest dvr = new DataverseRequest(context.getRequest().getAuthenticatedUser(), (HttpServletRequest) null);
+ String className = requestedSettings.get(SettingsServiceBean.Key.ArchiverClassName.toString());
+ AbstractSubmitToArchiveCommand archiveCommand = ArchiverUtil.createSubmitToArchiveCommand(className, dvr, context.getDataset().getReleasedVersion());
+ if (archiveCommand != null) {
+ return (archiveCommand.performArchiveSubmission(context.getDataset().getReleasedVersion(), context.getApiToken(), requestedSettings));
+ } else {
+ logger.severe("No Archiver instance could be created for name: " + className);
+ return new Failure("No Archiver", "Could not create instance of class: " + className);
+ }
+
+ }
+
+ @Override
+ public WorkflowStepResult resume(WorkflowContext context, Map internalData, String externalData) {
+ throw new UnsupportedOperationException("Not supported yet."); // This class does not need to resume.
+ }
+
+ @Override
+ public void rollback(WorkflowContext context, Failure reason) {
+ logger.log(Level.INFO, "rolling back workflow invocation {0}", context.getInvocationId());
+ logger.warning("Manual cleanup of Archive for: " + context.getDataset().getGlobalId().asString() + ", version: " + context.getDataset().getReleasedVersion() + " may be required");
+ }
+}
diff --git a/src/main/java/edu/harvard/iq/dataverse/workflow/internalspi/InternalWorkflowStepSP.java b/src/main/java/edu/harvard/iq/dataverse/workflow/internalspi/InternalWorkflowStepSP.java
index 30ca94f8833..76b7bcd7d44 100644
--- a/src/main/java/edu/harvard/iq/dataverse/workflow/internalspi/InternalWorkflowStepSP.java
+++ b/src/main/java/edu/harvard/iq/dataverse/workflow/internalspi/InternalWorkflowStepSP.java
@@ -19,6 +19,8 @@ public WorkflowStep getStep(String stepType, Map stepParameters)
return new PauseStep(stepParameters);
case "http/sr":
return new HttpSendReceiveClientStep(stepParameters);
+ case "archiver":
+ return new ArchivalSubmissionWorkflowStep(stepParameters);
default:
throw new IllegalArgumentException("Unsupported step type: '" + stepType + "'.");
}
diff --git a/src/main/webapp/confirmemail.xhtml b/src/main/webapp/confirmemail.xhtml
index 1ac88a457c6..2a071c83817 100644
--- a/src/main/webapp/confirmemail.xhtml
+++ b/src/main/webapp/confirmemail.xhtml
@@ -17,6 +17,7 @@
+
@@ -28,4 +29,4 @@
-