#1612 - Add or Replace File via API

doc/sphinx-guides/source/api/native-api.rst

@@ -199,6 +199,170 @@ Delete a Private URL from a dataset (if it exists)::
 
     DELETE http://$SERVER/api/datasets/$id/privateUrl?key=$apiKey
 
+Add a file to an existing Dataset.  Description and tags are optional::
+
+    PUT http://$SERVER/api/datasets/$id/add?key=$apiKey
+
+
+Example python code to add a file.  This may be run by changing these parameters in the sample code:
+
+* ``dataverse_server`` - e.g. https://dataverse.harvard.edu
+* ``api_key`` - See the top of this document for a description
+* ``persistentId`` - Example: ``doi:10.5072/FK2/6XACVA``
+* ``dataset_id`` - Database id of the dataset
+
+In practice, you only need one the ``dataset_id`` or the ``persistentId``. The example below shows both uses.
+
+.. code-block:: python
+
+    from datetime import datetime
+    import json
+    import requests  # http://docs.python-requests.org/en/master/
+
+    # --------------------------------------------------
+    # Update the 4 params below to run this code
+    # --------------------------------------------------
+    dataverse_server = 'https://your dataverse server' # no trailing slash
+    api_key = 'api key'
+    dataset_id = 1  # database id of the dataset
+    persistentId = 'doi:10.5072/FK2/6XACVA' # doi or hdl of the dataset
+
+    # --------------------------------------------------
+    # Prepare "file"
+    # --------------------------------------------------
+    file_content = 'content: %s' % datetime.now()
+    files = {'file': ('sample_file.txt', file_content)}
+
+    # --------------------------------------------------
+    # Using a "jsonData" parameter, add optional description + file tags
+    # --------------------------------------------------
+    params = dict(description='Blue skies!',
+                tags=['Lily', 'Rosemary', 'Jack of Hearts'])
+
+    params_as_json_string = json.dumps(params)
+
+    payload = dict(jsonData=params_as_json_string)
+
+    # --------------------------------------------------
+    # Add file using the Dataset's id
+    # --------------------------------------------------
+    url_dataset_id = '%s/api/datasets/%s/add?key=%s' % (dataverse_server, dataset_id, api_key)
+
+    # -------------------
+    # Make the request
+    # -------------------
+    print '-' * 40
+    print 'making request: %s' % url_dataset_id
+    r = requests.post(url_dataset_id, data=payload, files=files)
+
+    # -------------------
+    # Print the response
+    # -------------------
+    print '-' * 40
+    print r.json()
+    print r.status_code
+
+    # --------------------------------------------------
+    # Add file using the Dataset's persistentId (e.g. doi, hdl, etc)
+    # --------------------------------------------------
+    url_persistent_id = '%s/api/datasets/:persistentId/add?persistentId=%s&key=%s' % (dataverse_server, persistentId, api_key)
+
+    # -------------------
+    # Update the file content to avoid a duplicate file error
+    # -------------------
+    file_content = 'content2: %s' % datetime.now()
+    files = {'file': ('sample_file2.txt', file_content)}
+
+
+    # -------------------
+    # Make the request
+    # -------------------
+    print '-' * 40
+    print 'making request: %s' % url_persistent_id
+    r = requests.post(url_persistent_id, data=payload, files=files)
+
+    # -------------------
+    # Print the response
+    # -------------------
+    print '-' * 40
+    print r.json()
+    print r.status_code
+
+Files
+~~~~~~~~~~~
+
+Replace an existing file where ``id`` is the Database id of the file to replace::
+
+    POST http://$SERVER/api/files/{id}/replace?key=$apiKey
+
+Example python code to replace a file.  This may be run by changing these parameters in the sample code:
+
+* ``dataverse_server`` - e.g. https://dataverse.harvard.edu
+* ``api_key`` - See the top of this document for a description
+* ``file_id`` - Database id of the file to replace (returned in the GET API for a Dataset)
+
+.. code-block:: python
+
+    from datetime import datetime
+    import json
+    import requests  # http://docs.python-requests.org/en/master/
+
+    # --------------------------------------------------
+    # Update params below to run code
+    # --------------------------------------------------
+    dataverse_server = 'http://127.0.0.1:8080' # no trailing slash
+    api_key = 'some key'
+    file_id = 1401  # id of the file to replace
+
+    # --------------------------------------------------
+    # Prepare replacement "file"
+    # --------------------------------------------------
+    file_content = 'content: %s' % datetime.now()
+    files = {'file': ('replacement_file.txt', file_content)}
+
+    # --------------------------------------------------
+    # Using a "jsonData" parameter, add optional description + file tags
+    # --------------------------------------------------
+    params = dict(description='Sunset',
+                tags=['One', 'More', 'Cup of Coffee'])
+
+    # -------------------
+    # IMPORTANT: If the mimetype of the replacement file differs
+    #   from the origina file, the replace will fail
+    #
+    #  e.g. if you try to replace a ".csv" with a ".png" or something similar
+    #
+    #  You can override this with a "forceReplace" parameter
+    # -------------------
+    params['forceReplace'] = True
+
+
+    params_as_json_string = json.dumps(params)
+
+    payload = dict(jsonData=params_as_json_string)
+
+    print 'payload', payload
+    # --------------------------------------------------
+    # Replace file using the id of the file to replace
+    # --------------------------------------------------
+    url_replace = '%s/api/v1/files/%s/replace?key=%s' % (dataverse_server, file_id, api_key)
+
+    # -------------------
+    # Make the request
+    # -------------------
+    print '-' * 40
+    print 'making request: %s' % url_replace
+    r = requests.post(url_replace, data=payload, files=files)
+
+    # -------------------
+    # Print the response
+    # -------------------
+    print '-' * 40
+    print r.json()
+    print r.status_code
+
+
+
 Builtin Users
 ~~~~~~~~~~~~~
 

pom.xml

@@ -403,6 +403,18 @@
             <optional>true</optional>
             <type>jar</type>
         </dependency>
+        <!-- For API File Upload: 1 of 2 -->
+        <dependency>
+           <groupId>org.glassfish.jersey.containers</groupId>
+           <artifactId>jersey-container-servlet</artifactId>
+           <version>2.23.2</version>
+       </dependency>                
+        <!-- For API File Upload: 2 of 2 -->
+       <dependency>
+           <groupId>org.glassfish.jersey.media</groupId>
+           <artifactId>jersey-media-multipart</artifactId>
+           <version>2.23.2</version>
+       </dependency>
     </dependencies>
 
     <build>

scripts/database/upgrades/3354-alt-checksum.sql → scripts/database/upgrades/upgrade_v4.5.1_to_v4.6.sql

@@ -1,7 +1,13 @@
+-- For supporting SHA1 rather than MD5 as a checksum on a per file basis #3354
 ALTER TABLE datafile ADD COLUMN checksumtype character varying(255);
 UPDATE datafile SET checksumtype = 'MD5';
 ALTER TABLE datafile ALTER COLUMN checksumtype SET NOT NULL;
 -- alternate statement for sbgrid.org and others interested in SHA-1 support
 -- note that in the database we use "SHA1" (no hyphen) but the GUI will show "SHA-1"
 --UPDATE datafile SET checksumtype = 'SHA1';
 ALTER TABLE datafile RENAME md5 TO checksumvalue;
+-- For DataFile, file replace functionality:
+ALTER TABLE datafile ADD COLUMN rootdatafileid bigint default -1;
+ALTER TABLE datafile ADD COLUMN previousdatafileid bigint default null;
+-- For existing DataFile objects, update rootDataFileId values:
+UPDATE datafile  SET rootdatafileid = -1;

scripts/search/data/replace_test/003.txt

@@ -0,0 +1 @@
+3

scripts/search/data/replace_test/004.txt

@@ -0,0 +1 @@
+4

scripts/search/data/replace_test/005.txt

@@ -0,0 +1 @@
+5

src/main/java/Bundle.properties

@@ -1427,3 +1427,46 @@ citationFrame.banner.message.here=here
 citationFrame.banner.closeIcon=Close this message, go to dataset
 citationFrame.banner.countdownMessage= This message will close in 
 citationFrame.banner.countdownMessage.seconds=seconds
+
+
+# File metadata error
+file.metadata.datafiletag.not_tabular=You cannot add data file tags to a non-tabular file.
+
+# File Add/Replace operation messages
+file.addreplace.error.dataset_is_null=The dataset cannot be null.
+file.addreplace.error.dataset_id_is_null=The dataset ID cannot be null.
+find.dataset.error.dataset_id_is_null=When accessing a dataset based on persistent id, a {0} query parameter must be present
+find.dataset.error.dataset.not.found.persistentId=Dataset with persistent id {0} not found
+find.dataset.error.dataset.not.found.id=Dataset with id {0} not found
+find.dataset.error.dataset.not.found.bad.id=Bad dataset id number: {0}
+file.addreplace.error.dataset_id_not_found=There was no dataset found for id:
+file.addreplace.error.no_edit_dataset_permission=You do not have permission to edit this dataset.
+file.addreplace.error.filename_is_null=The fileName cannot be null.
+file.addreplace.error.file_content_type_is_null=The file content type cannot be null.
+file.addreplace.error.file_input_stream_is_null=The file upload cannot be null.
+file.addreplace.error.duplicate_file=This file has a duplicate already in the dataset:
+file.addreplace.error.existing_file_to_replace_id_is_null=The id of the existing file to replace cannot be null
+file.addreplace.error.existing_file_to_replace_not_found_by_id=Replacement file not found.  There was no file found for id:
+file.addreplace.error.existing_file_to_replace_is_null=The existing file to replace cannot be null
+file.addreplace.error.existing_file_to_replace_not_in_dataset=The existing file to replace does not belong to this dataset
+file.addreplace.error.existing_file_not_in_latest_published_version=You cannot replace a file that is not in the most recently published Dataset. (The file is unpublished or was deleted from a previous version.)
+file.addreplace.error.replace.new_file_same_as_replacement=The new file contains the same content as the file to be replaced.
+file.addreplace.error.replace.new_file_has_different_content_type=Warning! The new and old file have different content types.
+file.addreplace.error.unpublished_file_cannot_be_replaced=You cannot replace an unpublished file.  Please delete it instead of replacing it.
+file.addreplace.error.ingest_create_file_err=There was an error when trying to add the new file.
+file.addreplace.error.initial_file_list_empty=Sorry! An error occurred and the new file was not added.
+file.addreplace.error.initial_file_list_more_than_one=You cannot replace a single file with multiple files.  The file you uploaded was ingested into multiple files.
+file.addreplace.error.final_file_list_empty=There are no files to add.  (This error should not happen if steps called in sequence....)
+file.addreplace.error.only_replace_operation=This should ONLY be called for file replace operations!!
+file.addreplace.error.failed_to_remove_old_file_from_dataset=Unable to remove old file from new DatasetVersion.
+file.addreplace.error.add.command_engine_error=Failed to update the dataset.  Please contact the administrator. (CommandException)
+file.addreplace.error.add.ejb_exception=Failed to update the dataset.  Please contact the administrator. (EJBException)
+file.addreplace.error.replace.command_engine_error=Failed to update the dataset.  Please contact the administrator. (CommandException)
+file.addreplace.error.replace.ejb_exception=Failed to update the dataset.  Please contact the administrator. (EJBException)
+file.addreplace.error.remove_linked_file.dataset=dataset cannot be null in removeLinkedFileFromDataset
+file.addreplace.error.remove_linked_file.file=file cannot be null in removeLinkedFileFromDataset
+file.addreplace.error.phase2_called_early_no_new_files=There was an error saving the dataset--no new files found.
+file.addreplace.success.add=File successfully added!
+file.addreplace.success.replace=File successfully replaced!
+file.addreplace.error.auth=The API key is invalid.
+file.addreplace.error.invalid_datafile_tag=Not a valid Tabular Tag: