From 910a390e865f562f98c9510d2d9c30bb78a59691 Mon Sep 17 00:00:00 2001 From: qqmyers Date: Wed, 29 Oct 2025 18:57:56 -0400 Subject: [PATCH 1/5] fix error, cleanup, add out-of-band key, add test --- .../edu/harvard/iq/dataverse/api/Admin.java | 4 +- .../harvard/iq/dataverse/api/Datasets.java | 2 +- .../iq/dataverse/dataaccess/StorageIO.java | 14 ++-- .../iq/dataverse/util/json/JsonPrinter.java | 18 ++--- .../dataverse/util/json/JsonPrinterTest.java | 70 +++++++++++++++++++ 5 files changed, 88 insertions(+), 20 deletions(-) diff --git a/src/main/java/edu/harvard/iq/dataverse/api/Admin.java b/src/main/java/edu/harvard/iq/dataverse/api/Admin.java index 3df78648433..e5e40cb8ac8 100644 --- a/src/main/java/edu/harvard/iq/dataverse/api/Admin.java +++ b/src/main/java/edu/harvard/iq/dataverse/api/Admin.java @@ -2198,9 +2198,9 @@ public Response getStorageDriver(@Context ContainerRequestContext crc, @PathPara } if (getEffective != null && getEffective) { - return ok(JsonPrinter.jsonStorageDriver(dataverse.getEffectiveStorageDriverId(), null)); + return ok(JsonPrinter.jsonStorageDriver(dataverse.getEffectiveStorageDriverId())); } else { - return ok(JsonPrinter.jsonStorageDriver(dataverse.getStorageDriverId(), null)); + return ok(JsonPrinter.jsonStorageDriver(dataverse.getStorageDriverId())); } } diff --git a/src/main/java/edu/harvard/iq/dataverse/api/Datasets.java b/src/main/java/edu/harvard/iq/dataverse/api/Datasets.java index 12715d31e77..ea515756915 100644 --- a/src/main/java/edu/harvard/iq/dataverse/api/Datasets.java +++ b/src/main/java/edu/harvard/iq/dataverse/api/Datasets.java @@ -3674,7 +3674,7 @@ public Response getFileStore(@Context ContainerRequestContext crc, @PathParam("i return error(Response.Status.NOT_FOUND, "No such dataset"); } - return ok(JsonPrinter.jsonStorageDriver(dataset.getEffectiveStorageDriverId(), dataset)); + return ok(JsonPrinter.jsonStorageDriver(dataset.getEffectiveStorageDriverId())); } @PUT diff --git a/src/main/java/edu/harvard/iq/dataverse/dataaccess/StorageIO.java b/src/main/java/edu/harvard/iq/dataverse/dataaccess/StorageIO.java index bd59c11c5b1..b009c4e86ec 100644 --- a/src/main/java/edu/harvard/iq/dataverse/dataaccess/StorageIO.java +++ b/src/main/java/edu/harvard/iq/dataverse/dataaccess/StorageIO.java @@ -53,10 +53,11 @@ public abstract class StorageIO { static final String INGEST_SIZE_LIMIT = "ingestsizelimit"; static final String PUBLIC = "public"; - static final String TYPE = "type"; - static final String UPLOAD_REDIRECT = "upload-redirect"; - static final String UPLOAD_OUT_OF_BAND = "upload-out-of-band"; - protected static final String DOWNLOAD_REDIRECT = "download-redirect"; + public static final String TYPE = "type"; + public static final String UPLOAD_REDIRECT = "upload-redirect"; + public static final String UPLOAD_OUT_OF_BAND = "upload-out-of-band"; + public static final String DOWNLOAD_REDIRECT = "download-redirect"; + public static final String LABEL = "label"; public StorageIO() { @@ -675,10 +676,11 @@ public boolean isDataverseAccessible() { return true; } - protected static String getConfigParamForDriver(String driverId, String parameterName) { + public static String getConfigParamForDriver(String driverId, String parameterName) { return getConfigParamForDriver(driverId, parameterName, null); } - protected static String getConfigParamForDriver(String driverId, String parameterName, String defaultValue) { + + public static String getConfigParamForDriver(String driverId, String parameterName, String defaultValue) { return System.getProperty("dataverse.files." + driverId + "." + parameterName, defaultValue); } diff --git a/src/main/java/edu/harvard/iq/dataverse/util/json/JsonPrinter.java b/src/main/java/edu/harvard/iq/dataverse/util/json/JsonPrinter.java index 46a05fc93f2..92a37807e2a 100644 --- a/src/main/java/edu/harvard/iq/dataverse/util/json/JsonPrinter.java +++ b/src/main/java/edu/harvard/iq/dataverse/util/json/JsonPrinter.java @@ -17,6 +17,7 @@ import edu.harvard.iq.dataverse.authorization.users.User; import edu.harvard.iq.dataverse.branding.BrandingUtil; import edu.harvard.iq.dataverse.dataaccess.DataAccess; +import edu.harvard.iq.dataverse.dataaccess.StorageIO; import edu.harvard.iq.dataverse.dataset.DatasetType; import edu.harvard.iq.dataverse.dataset.DatasetUtil; import edu.harvard.iq.dataverse.datasetversionsummaries.*; @@ -1670,19 +1671,14 @@ public static JsonArrayBuilder jsonTemplateInstructions(Map temp return jsonArrayBuilder; } - public static JsonObjectBuilder jsonStorageDriver(String storageDriverId, Dataset dataset) { + public static JsonObjectBuilder jsonStorageDriver(String storageDriverId) { JsonObjectBuilder jsonObjectBuilder = new NullSafeJsonBuilder(); jsonObjectBuilder.add("name", storageDriverId); - jsonObjectBuilder.add("type", DataAccess.getDriverType(storageDriverId)); - jsonObjectBuilder.add("label", DataAccess.getStorageDriverLabelFor(storageDriverId)); - if (dataset != null) { - jsonObjectBuilder.add("directUpload", DataAccess.uploadToDatasetAllowed(dataset, storageDriverId)); - try { - jsonObjectBuilder.add("directDownload", DataAccess.getStorageIO(dataset).downloadRedirectEnabled()); - } catch (IOException ex) { - logger.fine("Failed to get Storage IO for dataset " + ex.getMessage()); - } - } + jsonObjectBuilder.add("type", StorageIO.getConfigParamForDriver(storageDriverId, StorageIO.TYPE)); + jsonObjectBuilder.add("label", StorageIO.getConfigParamForDriver(storageDriverId, StorageIO.LABEL)); + jsonObjectBuilder.add("directUpload", Boolean.parseBoolean(StorageIO.getConfigParamForDriver(storageDriverId, StorageIO.UPLOAD_REDIRECT))); + jsonObjectBuilder.add("directDownload", Boolean.parseBoolean(StorageIO.getConfigParamForDriver(storageDriverId, StorageIO.DOWNLOAD_REDIRECT))); + jsonObjectBuilder.add("uploadOutOfBand", Boolean.parseBoolean(StorageIO.getConfigParamForDriver(storageDriverId, StorageIO.UPLOAD_OUT_OF_BAND))); return jsonObjectBuilder; } diff --git a/src/test/java/edu/harvard/iq/dataverse/util/json/JsonPrinterTest.java b/src/test/java/edu/harvard/iq/dataverse/util/json/JsonPrinterTest.java index c17529fb6ac..94f16553dff 100644 --- a/src/test/java/edu/harvard/iq/dataverse/util/json/JsonPrinterTest.java +++ b/src/test/java/edu/harvard/iq/dataverse/util/json/JsonPrinterTest.java @@ -664,4 +664,74 @@ private DataFile createDatafile(long id) { return datafile; } + + + @Test + public void testJsonStorageDriver() { + // Test with directDownload enabled (true-like values) + System.setProperty("dataverse.files.test-driver.type", "s3"); + System.setProperty("dataverse.files.test-driver.label", "Test Storage Driver"); + System.setProperty("dataverse.files.test-driver.download-redirect", "true"); + System.setProperty("dataverse.files.test-driver.upload-redirect", "true"); + + JsonObject result = JsonPrinter.jsonStorageDriver("test-driver").build(); + + assertEquals("test-driver", result.getString("name")); + assertEquals("s3", result.getString("type")); + assertEquals("Test Storage Driver", result.getString("label")); + assertTrue(result.getBoolean("directUpload")); + assertTrue(result.getBoolean("directDownload")); + assertFalse(result.getBoolean("uploadOutOfBand")); + + // Test with directDownload disabled (false values) + System.setProperty("dataverse.files.test-driver2.type", "file"); + System.setProperty("dataverse.files.test-driver2.label", "Local Storage"); + System.setProperty("dataverse.files.test-driver2.download-redirect", "false"); + System.setProperty("dataverse.files.test-driver2.upload-redirect", "false"); + + JsonObject result2 = JsonPrinter.jsonStorageDriver("test-driver2").build(); + + assertEquals("test-driver2", result2.getString("name")); + assertEquals("file", result2.getString("type")); + assertEquals("Local Storage", result2.getString("label")); + assertFalse(result2.getBoolean("directUpload")); + assertFalse(result2.getBoolean("directDownload")); + assertFalse(result2.getBoolean("uploadOutOfBand")); + + // Test with all caps TRUE and out-of-band + System.setProperty("dataverse.files.test-driver3.type", "swift"); + System.setProperty("dataverse.files.test-driver3.label", "Swift Storage"); + System.setProperty("dataverse.files.test-driver3.download-redirect", "TRUE"); + System.setProperty("dataverse.files.test-driver3.upload-out-of-band", "true"); + + JsonObject result3 = JsonPrinter.jsonStorageDriver("test-driver3").build(); + assertTrue(result3.getBoolean("directDownload")); + assertTrue(result3.getBoolean("uploadOutOfBand")); + + // Test with null/missing properties + System.setProperty("dataverse.files.test-driver4.type", "s3"); + System.setProperty("dataverse.files.test-driver4.label", "Minimal Storage"); + // Not setting download-redirect property + + JsonObject result4 = JsonPrinter.jsonStorageDriver("test-driver4").build(); + assertFalse(result4.getBoolean("directDownload")); + assertFalse(result.getBoolean("directUpload")); + assertFalse(result.getBoolean("uploadOutOfBand")); + + // Clean up system properties + System.clearProperty("dataverse.files.test-driver.type"); + System.clearProperty("dataverse.files.test-driver.label"); + System.clearProperty("dataverse.files.test-driver.download-redirect"); + System.clearProperty("dataverse.files.test-driver.upload-redirect"); + System.clearProperty("dataverse.files.test-driver2.type"); + System.clearProperty("dataverse.files.test-driver2.label"); + System.clearProperty("dataverse.files.test-driver2.download-redirect"); + System.clearProperty("dataverse.files.test-driver2.upload-redirect"); + System.clearProperty("dataverse.files.test-driver3.type"); + System.clearProperty("dataverse.files.test-driver3.label"); + System.clearProperty("dataverse.files.test-driver3.download-redirect"); + System.clearProperty("dataverse.files.test-driver3.upload-out-of-band"); + System.clearProperty("dataverse.files.test-driver4.type"); + System.clearProperty("dataverse.files.test-driver4.label"); + } } From f0635e3ee356d82a820391a2307ab4b221e5cf00 Mon Sep 17 00:00:00 2001 From: qqmyers Date: Wed, 29 Oct 2025 19:00:59 -0400 Subject: [PATCH 2/5] typos --- .../edu/harvard/iq/dataverse/util/json/JsonPrinterTest.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/test/java/edu/harvard/iq/dataverse/util/json/JsonPrinterTest.java b/src/test/java/edu/harvard/iq/dataverse/util/json/JsonPrinterTest.java index 94f16553dff..2f4fda068d4 100644 --- a/src/test/java/edu/harvard/iq/dataverse/util/json/JsonPrinterTest.java +++ b/src/test/java/edu/harvard/iq/dataverse/util/json/JsonPrinterTest.java @@ -715,8 +715,8 @@ public void testJsonStorageDriver() { JsonObject result4 = JsonPrinter.jsonStorageDriver("test-driver4").build(); assertFalse(result4.getBoolean("directDownload")); - assertFalse(result.getBoolean("directUpload")); - assertFalse(result.getBoolean("uploadOutOfBand")); + assertFalse(result4.getBoolean("directUpload")); + assertFalse(result4.getBoolean("uploadOutOfBand")); // Clean up system properties System.clearProperty("dataverse.files.test-driver.type"); From e66421f9cbd53e396771e8cf4fdb69fd98470255 Mon Sep 17 00:00:00 2001 From: qqmyers Date: Thu, 30 Oct 2025 13:32:57 -0400 Subject: [PATCH 3/5] doc updates --- .../11695-11940-change-api-get-storage-driver.md | 12 ++++++++++++ .../11695-change-api-get-storage-driver.md | 12 ------------ .../source/admin/dataverses-datasets.rst | 16 +++++++++------- doc/sphinx-guides/source/api/changelog.rst | 2 +- 4 files changed, 22 insertions(+), 20 deletions(-) create mode 100644 doc/release-notes/11695-11940-change-api-get-storage-driver.md delete mode 100644 doc/release-notes/11695-change-api-get-storage-driver.md diff --git a/doc/release-notes/11695-11940-change-api-get-storage-driver.md b/doc/release-notes/11695-11940-change-api-get-storage-driver.md new file mode 100644 index 00000000000..5b1c8e383be --- /dev/null +++ b/doc/release-notes/11695-11940-change-api-get-storage-driver.md @@ -0,0 +1,12 @@ +## Get Dataset/Dataverse Storage Driver API + +### Changed Json response - breaking change! + +The API for getting the Storage Driver info has been changed/extended. +/api/datasets/{identifier}/storageDriver +/api/admin/dataverse/{dataverse-alias}/storageDriver +Rather tha returning just the name/id of the driver (with the key "message"), the api call now returns a JSONObject with the driver's "name", "type" and "label", and booleas indicating whether the driver has "directUpload", "directDownload", and/or "uploadOutOfBand" enabled. + +This change also affects the /api/admin/dataverse/{dataverse-alias}/storageDriver api call. In addition, this call now supports an optional ?getEffective=true to find the effective storageDriver (the driver that will be used for new datasets in the collection) + +See also [the guides](https://dataverse-guide--11664.org.readthedocs.build/en/11664/api/native-api.html#configure-a-dataset-to-store-all-new-files-in-a-specific-file-store), #11695, and #11664. diff --git a/doc/release-notes/11695-change-api-get-storage-driver.md b/doc/release-notes/11695-change-api-get-storage-driver.md deleted file mode 100644 index aa993232f4d..00000000000 --- a/doc/release-notes/11695-change-api-get-storage-driver.md +++ /dev/null @@ -1,12 +0,0 @@ -## Get Dataset/Dataverse Storage Driver API - -### Changed Json response - breaking change! - -The API for getting the Storage Driver info has been changed/extended. -/api/datasets/{identifier}/storageDriver -/api/admin/dataverse/{dataverse-alias}/storageDriver -changed "message" to "name" and added "type" and "label" - -Also added query param for /api/admin/dataverse/{dataverse-alias}/storageDriver?getEffective=true to recurse the chain of parents to find the effective storageDriver - -See also [the guides](https://dataverse-guide--11664.org.readthedocs.build/en/11664/api/native-api.html#configure-a-dataset-to-store-all-new-files-in-a-specific-file-store), #11695, and #11664. diff --git a/doc/sphinx-guides/source/admin/dataverses-datasets.rst b/doc/sphinx-guides/source/admin/dataverses-datasets.rst index 0fa5bcf69f1..1d97155317d 100644 --- a/doc/sphinx-guides/source/admin/dataverses-datasets.rst +++ b/doc/sphinx-guides/source/admin/dataverses-datasets.rst @@ -56,17 +56,19 @@ To direct new files (uploaded when datasets are created or edited) for all datas (Note that for ``dataverse.files.store1.label=MyLabel``, you should pass ``MyLabel``.) -The current driver can be seen using:: +A store assigned directly to a collection can be seen using:: curl -H "X-Dataverse-key: $API_TOKEN" http://$SERVER/api/admin/dataverse/$dataverse-alias/storageDriver -Or to recurse the chain of parents to find the effective storageDriver:: +This may be null. To get the effective storageDriver for a collection, which may be inherited from a parent collection or be the installation default, you can use:: curl -H "X-Dataverse-key: $API_TOKEN" http://$SERVER/api/admin/dataverse/$dataverse-alias/storageDriver?getEffective=true + +This will never be null. -(Note that for ``dataverse.files.store1.label=MyLabel``, ``store1`` will be returned.) +(Note that for ``dataverse.files.store1.label=MyLabel``, the JSON response will include "name":"store1" and "label":"MyLabel".) -and can be reset to the default store with:: +To delete a store assigned directly to a collection (so that the colllection's effective store is inherted from it's parent or is the global default), use:: curl -H "X-Dataverse-key: $API_TOKEN" -X DELETE http://$SERVER/api/admin/dataverse/$dataverse-alias/storageDriver @@ -261,15 +263,15 @@ To identify invalid data values in specific datasets (if, for example, an attemp Configure a Dataset to Store All New Files in a Specific File Store ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ -Configure a dataset to use a specific file store (this API can only be used by a superuser) :: +Configure an individual dataset to use a specific file store (this API can only be used by a superuser) :: curl -H "X-Dataverse-key: $API_TOKEN" -X PUT -d $storageDriverLabel http://$SERVER/api/datasets/$dataset-id/storageDriver -The current driver can be seen using:: +The effective store can be seen using:: curl http://$SERVER/api/datasets/$dataset-id/storageDriver -It can be reset to the default store as follows (only a superuser can do this) :: +To remove an assigned store, and allow the dataset to inherit the store from it's parent collection, use the following (only a superuser can do this) :: curl -H "X-Dataverse-key: $API_TOKEN" -X DELETE http://$SERVER/api/datasets/$dataset-id/storageDriver diff --git a/doc/sphinx-guides/source/api/changelog.rst b/doc/sphinx-guides/source/api/changelog.rst index d6523bfbdbc..41633414fbb 100644 --- a/doc/sphinx-guides/source/api/changelog.rst +++ b/doc/sphinx-guides/source/api/changelog.rst @@ -10,6 +10,7 @@ This API changelog is experimental and we would love feedback on its usefulness. v6.9 ---- - The POST /api/admin/makeDataCount/{id}/updateCitationsForDataset processing is now asynchronous and the response no longer includes the number of citations. The response can be OK if the request is queued or 503 if the queue is full (default queue size is 1000). +- For GET /api/admin/dataverse/{dataverse-alias}/storageDriver and /api/datasets/{identifier}/storageDriver the driver name is no longer returned in data.message. Instead, it is returned as data.name (along with other information about the storageDriver). v6.8 ---- @@ -17,7 +18,6 @@ v6.8 - For POST /api/files/{id}/metadata passing an empty string ("description":"") or array ("categories":[]) will no longer be ignored. Empty fields will now clear out the values in the file's metadata. To ignore the fields simply do not include them in the JSON string. - For PUT /api/datasets/{id}/editMetadata the query parameter "sourceInternalVersionNumber" has been removed and replaced with "sourceLastUpdateTime" to verify that the data being edited hasn't been modified and isn't stale. - For GET /api/dataverses/$dataverse-alias/links the Json response has changed breaking the backward compatibility of the API. -- For GET /api/admin/dataverse/{dataverse-alias}/storageDriver and /api/datasets/{identifier}/storageDriver the driver name is no longer returned in data.message. This value is now returned in data.name. - For PUT /api/dataverses/$dataverse-alias/inputLevels custom input levels that had been previously set will no longer be deleted. To delete input levels send an empty list (deletes all), then send the new/modified list. - For GET /api/externalTools and /api/externalTools/{id} the responses are now formatted as JSON (previously the toolParameters and allowedApiCalls were a JSON object and array (respectively) that were serialized as JSON strings) and any configured "requirements" are included. From 29e46926fda52ba14ba4c0fb343aa7f46495b116 Mon Sep 17 00:00:00 2001 From: qqmyers Date: Tue, 4 Nov 2025 14:59:44 -0500 Subject: [PATCH 4/5] fix typos per review --- doc/release-notes/11695-11940-change-api-get-storage-driver.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/release-notes/11695-11940-change-api-get-storage-driver.md b/doc/release-notes/11695-11940-change-api-get-storage-driver.md index 5b1c8e383be..8fb05879871 100644 --- a/doc/release-notes/11695-11940-change-api-get-storage-driver.md +++ b/doc/release-notes/11695-11940-change-api-get-storage-driver.md @@ -5,7 +5,7 @@ The API for getting the Storage Driver info has been changed/extended. /api/datasets/{identifier}/storageDriver /api/admin/dataverse/{dataverse-alias}/storageDriver -Rather tha returning just the name/id of the driver (with the key "message"), the api call now returns a JSONObject with the driver's "name", "type" and "label", and booleas indicating whether the driver has "directUpload", "directDownload", and/or "uploadOutOfBand" enabled. +Rather than returning just the name/id of the driver (with the key "message"), the api call now returns a JSONObject with the driver's "name", "type" and "label", and booleans indicating whether the driver has "directUpload", "directDownload", and/or "uploadOutOfBand" enabled. This change also affects the /api/admin/dataverse/{dataverse-alias}/storageDriver api call. In addition, this call now supports an optional ?getEffective=true to find the effective storageDriver (the driver that will be used for new datasets in the collection) From e8e58755c464ca96939ad32ed9bec7edb02321c1 Mon Sep 17 00:00:00 2001 From: qqmyers Date: Tue, 4 Nov 2025 15:05:19 -0500 Subject: [PATCH 5/5] update tests - direct up/down now included for dataverse api call --- src/test/java/edu/harvard/iq/dataverse/api/DataversesIT.java | 4 ++-- src/test/java/edu/harvard/iq/dataverse/api/S3AccessIT.java | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/src/test/java/edu/harvard/iq/dataverse/api/DataversesIT.java b/src/test/java/edu/harvard/iq/dataverse/api/DataversesIT.java index 307af623120..e7a56cb339c 100644 --- a/src/test/java/edu/harvard/iq/dataverse/api/DataversesIT.java +++ b/src/test/java/edu/harvard/iq/dataverse/api/DataversesIT.java @@ -2726,8 +2726,8 @@ public void testGetStorageDriver() { .body("data.name", CoreMatchers.notNullValue()) .body("data.type", CoreMatchers.notNullValue()) .body("data.label", CoreMatchers.notNullValue()) - .body("data.directUpload", CoreMatchers.nullValue()) - .body("data.directDownload", CoreMatchers.nullValue()) + .body("data.directUpload", CoreMatchers.notNullValue()) + .body("data.directDownload", CoreMatchers.notNullValue()) .statusCode(200); // Root without default is undefined diff --git a/src/test/java/edu/harvard/iq/dataverse/api/S3AccessIT.java b/src/test/java/edu/harvard/iq/dataverse/api/S3AccessIT.java index 24625c87ce2..60e83f9f2ba 100644 --- a/src/test/java/edu/harvard/iq/dataverse/api/S3AccessIT.java +++ b/src/test/java/edu/harvard/iq/dataverse/api/S3AccessIT.java @@ -156,7 +156,7 @@ public void testNonDirectUpload() { updatedStorageDriver.then().assertThat() .body("data.type", CoreMatchers.notNullValue()) .body("data.label", CoreMatchers.notNullValue()) - .body("data.directUpload", CoreMatchers.nullValue()) + .body("data.directUpload", CoreMatchers.notNullValue()) .statusCode(200); Response createDatasetResponse = UtilIT.createRandomDatasetViaNativeApi(dataverseAlias, apiToken);