diff --git a/doc/release-notes/11695-11940-change-api-get-storage-driver.md b/doc/release-notes/11695-11940-change-api-get-storage-driver.md new file mode 100644 index 00000000000..8fb05879871 --- /dev/null +++ b/doc/release-notes/11695-11940-change-api-get-storage-driver.md @@ -0,0 +1,12 @@ +## Get Dataset/Dataverse Storage Driver API + +### Changed Json response - breaking change! + +The API for getting the Storage Driver info has been changed/extended. +/api/datasets/{identifier}/storageDriver +/api/admin/dataverse/{dataverse-alias}/storageDriver +Rather than returning just the name/id of the driver (with the key "message"), the api call now returns a JSONObject with the driver's "name", "type" and "label", and booleans indicating whether the driver has "directUpload", "directDownload", and/or "uploadOutOfBand" enabled. + +This change also affects the /api/admin/dataverse/{dataverse-alias}/storageDriver api call. In addition, this call now supports an optional ?getEffective=true to find the effective storageDriver (the driver that will be used for new datasets in the collection) + +See also [the guides](https://dataverse-guide--11664.org.readthedocs.build/en/11664/api/native-api.html#configure-a-dataset-to-store-all-new-files-in-a-specific-file-store), #11695, and #11664. diff --git a/doc/release-notes/11695-change-api-get-storage-driver.md b/doc/release-notes/11695-change-api-get-storage-driver.md deleted file mode 100644 index aa993232f4d..00000000000 --- a/doc/release-notes/11695-change-api-get-storage-driver.md +++ /dev/null @@ -1,12 +0,0 @@ -## Get Dataset/Dataverse Storage Driver API - -### Changed Json response - breaking change! - -The API for getting the Storage Driver info has been changed/extended. -/api/datasets/{identifier}/storageDriver -/api/admin/dataverse/{dataverse-alias}/storageDriver -changed "message" to "name" and added "type" and "label" - -Also added query param for /api/admin/dataverse/{dataverse-alias}/storageDriver?getEffective=true to recurse the chain of parents to find the effective storageDriver - -See also [the guides](https://dataverse-guide--11664.org.readthedocs.build/en/11664/api/native-api.html#configure-a-dataset-to-store-all-new-files-in-a-specific-file-store), #11695, and #11664. diff --git a/doc/sphinx-guides/source/admin/dataverses-datasets.rst b/doc/sphinx-guides/source/admin/dataverses-datasets.rst index 0fa5bcf69f1..1d97155317d 100644 --- a/doc/sphinx-guides/source/admin/dataverses-datasets.rst +++ b/doc/sphinx-guides/source/admin/dataverses-datasets.rst @@ -56,17 +56,19 @@ To direct new files (uploaded when datasets are created or edited) for all datas (Note that for ``dataverse.files.store1.label=MyLabel``, you should pass ``MyLabel``.) -The current driver can be seen using:: +A store assigned directly to a collection can be seen using:: curl -H "X-Dataverse-key: $API_TOKEN" http://$SERVER/api/admin/dataverse/$dataverse-alias/storageDriver -Or to recurse the chain of parents to find the effective storageDriver:: +This may be null. To get the effective storageDriver for a collection, which may be inherited from a parent collection or be the installation default, you can use:: curl -H "X-Dataverse-key: $API_TOKEN" http://$SERVER/api/admin/dataverse/$dataverse-alias/storageDriver?getEffective=true + +This will never be null. -(Note that for ``dataverse.files.store1.label=MyLabel``, ``store1`` will be returned.) +(Note that for ``dataverse.files.store1.label=MyLabel``, the JSON response will include "name":"store1" and "label":"MyLabel".) -and can be reset to the default store with:: +To delete a store assigned directly to a collection (so that the colllection's effective store is inherted from it's parent or is the global default), use:: curl -H "X-Dataverse-key: $API_TOKEN" -X DELETE http://$SERVER/api/admin/dataverse/$dataverse-alias/storageDriver @@ -261,15 +263,15 @@ To identify invalid data values in specific datasets (if, for example, an attemp Configure a Dataset to Store All New Files in a Specific File Store ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ -Configure a dataset to use a specific file store (this API can only be used by a superuser) :: +Configure an individual dataset to use a specific file store (this API can only be used by a superuser) :: curl -H "X-Dataverse-key: $API_TOKEN" -X PUT -d $storageDriverLabel http://$SERVER/api/datasets/$dataset-id/storageDriver -The current driver can be seen using:: +The effective store can be seen using:: curl http://$SERVER/api/datasets/$dataset-id/storageDriver -It can be reset to the default store as follows (only a superuser can do this) :: +To remove an assigned store, and allow the dataset to inherit the store from it's parent collection, use the following (only a superuser can do this) :: curl -H "X-Dataverse-key: $API_TOKEN" -X DELETE http://$SERVER/api/datasets/$dataset-id/storageDriver diff --git a/doc/sphinx-guides/source/api/changelog.rst b/doc/sphinx-guides/source/api/changelog.rst index 89babf3c0f8..4c7a5914b1e 100644 --- a/doc/sphinx-guides/source/api/changelog.rst +++ b/doc/sphinx-guides/source/api/changelog.rst @@ -14,6 +14,7 @@ v6.9 - The way to set per-format size limits for tabular ingest has changed. JSON input is now used. See :ref:`:TabularIngestSizeLimit`. - In the past, the settings API would accept any key and value. This is no longer the case because validation has been added. See :ref:`settings_put_single`, for example. - For GET /api/notifications/all the JSON response has changed breaking the backward compatibility of the API. +- For GET /api/admin/dataverse/{dataverse-alias}/storageDriver and /api/datasets/{identifier}/storageDriver the driver name is no longer returned in data.message. Instead, it is returned as data.name (along with other information about the storageDriver). v6.8 ---- @@ -21,7 +22,6 @@ v6.8 - For POST /api/files/{id}/metadata passing an empty string ("description":"") or array ("categories":[]) will no longer be ignored. Empty fields will now clear out the values in the file's metadata. To ignore the fields simply do not include them in the JSON string. - For PUT /api/datasets/{id}/editMetadata the query parameter "sourceInternalVersionNumber" has been removed and replaced with "sourceLastUpdateTime" to verify that the data being edited hasn't been modified and isn't stale. - For GET /api/dataverses/$dataverse-alias/links the JSON response has changed breaking the backward compatibility of the API. -- For GET /api/admin/dataverse/{dataverse-alias}/storageDriver and /api/datasets/{identifier}/storageDriver the driver name is no longer returned in data.message. This value is now returned in data.name. - For PUT /api/dataverses/$dataverse-alias/inputLevels custom input levels that had been previously set will no longer be deleted. To delete input levels send an empty list (deletes all), then send the new/modified list. - For GET /api/externalTools and /api/externalTools/{id} the responses are now formatted as JSON (previously the toolParameters and allowedApiCalls were a JSON object and array (respectively) that were serialized as JSON strings) and any configured "requirements" are included. diff --git a/src/main/java/edu/harvard/iq/dataverse/api/Admin.java b/src/main/java/edu/harvard/iq/dataverse/api/Admin.java index 75aedb038dc..18f28569d7d 100644 --- a/src/main/java/edu/harvard/iq/dataverse/api/Admin.java +++ b/src/main/java/edu/harvard/iq/dataverse/api/Admin.java @@ -2275,9 +2275,9 @@ public Response getStorageDriver(@Context ContainerRequestContext crc, @PathPara } if (getEffective != null && getEffective) { - return ok(JsonPrinter.jsonStorageDriver(dataverse.getEffectiveStorageDriverId(), null)); + return ok(JsonPrinter.jsonStorageDriver(dataverse.getEffectiveStorageDriverId())); } else { - return ok(JsonPrinter.jsonStorageDriver(dataverse.getStorageDriverId(), null)); + return ok(JsonPrinter.jsonStorageDriver(dataverse.getStorageDriverId())); } } diff --git a/src/main/java/edu/harvard/iq/dataverse/api/Datasets.java b/src/main/java/edu/harvard/iq/dataverse/api/Datasets.java index 4b3db65556c..47e18e8ff34 100644 --- a/src/main/java/edu/harvard/iq/dataverse/api/Datasets.java +++ b/src/main/java/edu/harvard/iq/dataverse/api/Datasets.java @@ -3662,7 +3662,7 @@ public Response getFileStore(@Context ContainerRequestContext crc, @PathParam("i return error(Response.Status.NOT_FOUND, "No such dataset"); } - return ok(JsonPrinter.jsonStorageDriver(dataset.getEffectiveStorageDriverId(), dataset)); + return ok(JsonPrinter.jsonStorageDriver(dataset.getEffectiveStorageDriverId())); } @PUT diff --git a/src/main/java/edu/harvard/iq/dataverse/dataaccess/StorageIO.java b/src/main/java/edu/harvard/iq/dataverse/dataaccess/StorageIO.java index bd59c11c5b1..b009c4e86ec 100644 --- a/src/main/java/edu/harvard/iq/dataverse/dataaccess/StorageIO.java +++ b/src/main/java/edu/harvard/iq/dataverse/dataaccess/StorageIO.java @@ -53,10 +53,11 @@ public abstract class StorageIO { static final String INGEST_SIZE_LIMIT = "ingestsizelimit"; static final String PUBLIC = "public"; - static final String TYPE = "type"; - static final String UPLOAD_REDIRECT = "upload-redirect"; - static final String UPLOAD_OUT_OF_BAND = "upload-out-of-band"; - protected static final String DOWNLOAD_REDIRECT = "download-redirect"; + public static final String TYPE = "type"; + public static final String UPLOAD_REDIRECT = "upload-redirect"; + public static final String UPLOAD_OUT_OF_BAND = "upload-out-of-band"; + public static final String DOWNLOAD_REDIRECT = "download-redirect"; + public static final String LABEL = "label"; public StorageIO() { @@ -675,10 +676,11 @@ public boolean isDataverseAccessible() { return true; } - protected static String getConfigParamForDriver(String driverId, String parameterName) { + public static String getConfigParamForDriver(String driverId, String parameterName) { return getConfigParamForDriver(driverId, parameterName, null); } - protected static String getConfigParamForDriver(String driverId, String parameterName, String defaultValue) { + + public static String getConfigParamForDriver(String driverId, String parameterName, String defaultValue) { return System.getProperty("dataverse.files." + driverId + "." + parameterName, defaultValue); } diff --git a/src/main/java/edu/harvard/iq/dataverse/util/json/JsonPrinter.java b/src/main/java/edu/harvard/iq/dataverse/util/json/JsonPrinter.java index 654154c5b64..aa96d5dd20f 100644 --- a/src/main/java/edu/harvard/iq/dataverse/util/json/JsonPrinter.java +++ b/src/main/java/edu/harvard/iq/dataverse/util/json/JsonPrinter.java @@ -17,6 +17,7 @@ import edu.harvard.iq.dataverse.authorization.users.User; import edu.harvard.iq.dataverse.branding.BrandingUtil; import edu.harvard.iq.dataverse.dataaccess.DataAccess; +import edu.harvard.iq.dataverse.dataaccess.StorageIO; import edu.harvard.iq.dataverse.dataset.DatasetType; import edu.harvard.iq.dataverse.dataset.DatasetUtil; import edu.harvard.iq.dataverse.datasetversionsummaries.*; @@ -1670,19 +1671,14 @@ public static JsonArrayBuilder jsonTemplateInstructions(Map temp return jsonArrayBuilder; } - public static JsonObjectBuilder jsonStorageDriver(String storageDriverId, Dataset dataset) { + public static JsonObjectBuilder jsonStorageDriver(String storageDriverId) { JsonObjectBuilder jsonObjectBuilder = new NullSafeJsonBuilder(); jsonObjectBuilder.add("name", storageDriverId); - jsonObjectBuilder.add("type", DataAccess.getDriverType(storageDriverId)); - jsonObjectBuilder.add("label", DataAccess.getStorageDriverLabelFor(storageDriverId)); - if (dataset != null) { - jsonObjectBuilder.add("directUpload", DataAccess.uploadToDatasetAllowed(dataset, storageDriverId)); - try { - jsonObjectBuilder.add("directDownload", DataAccess.getStorageIO(dataset).downloadRedirectEnabled()); - } catch (IOException ex) { - logger.fine("Failed to get Storage IO for dataset " + ex.getMessage()); - } - } + jsonObjectBuilder.add("type", StorageIO.getConfigParamForDriver(storageDriverId, StorageIO.TYPE)); + jsonObjectBuilder.add("label", StorageIO.getConfigParamForDriver(storageDriverId, StorageIO.LABEL)); + jsonObjectBuilder.add("directUpload", Boolean.parseBoolean(StorageIO.getConfigParamForDriver(storageDriverId, StorageIO.UPLOAD_REDIRECT))); + jsonObjectBuilder.add("directDownload", Boolean.parseBoolean(StorageIO.getConfigParamForDriver(storageDriverId, StorageIO.DOWNLOAD_REDIRECT))); + jsonObjectBuilder.add("uploadOutOfBand", Boolean.parseBoolean(StorageIO.getConfigParamForDriver(storageDriverId, StorageIO.UPLOAD_OUT_OF_BAND))); return jsonObjectBuilder; } diff --git a/src/test/java/edu/harvard/iq/dataverse/api/DataversesIT.java b/src/test/java/edu/harvard/iq/dataverse/api/DataversesIT.java index cfa91e44cc0..1b908f63bfa 100644 --- a/src/test/java/edu/harvard/iq/dataverse/api/DataversesIT.java +++ b/src/test/java/edu/harvard/iq/dataverse/api/DataversesIT.java @@ -2740,8 +2740,8 @@ public void testGetStorageDriver() { .body("data.name", CoreMatchers.notNullValue()) .body("data.type", CoreMatchers.notNullValue()) .body("data.label", CoreMatchers.notNullValue()) - .body("data.directUpload", CoreMatchers.nullValue()) - .body("data.directDownload", CoreMatchers.nullValue()) + .body("data.directUpload", CoreMatchers.notNullValue()) + .body("data.directDownload", CoreMatchers.notNullValue()) .statusCode(200); // Root without default is undefined diff --git a/src/test/java/edu/harvard/iq/dataverse/api/S3AccessIT.java b/src/test/java/edu/harvard/iq/dataverse/api/S3AccessIT.java index 24625c87ce2..60e83f9f2ba 100644 --- a/src/test/java/edu/harvard/iq/dataverse/api/S3AccessIT.java +++ b/src/test/java/edu/harvard/iq/dataverse/api/S3AccessIT.java @@ -156,7 +156,7 @@ public void testNonDirectUpload() { updatedStorageDriver.then().assertThat() .body("data.type", CoreMatchers.notNullValue()) .body("data.label", CoreMatchers.notNullValue()) - .body("data.directUpload", CoreMatchers.nullValue()) + .body("data.directUpload", CoreMatchers.notNullValue()) .statusCode(200); Response createDatasetResponse = UtilIT.createRandomDatasetViaNativeApi(dataverseAlias, apiToken); diff --git a/src/test/java/edu/harvard/iq/dataverse/util/json/JsonPrinterTest.java b/src/test/java/edu/harvard/iq/dataverse/util/json/JsonPrinterTest.java index c17529fb6ac..2f4fda068d4 100644 --- a/src/test/java/edu/harvard/iq/dataverse/util/json/JsonPrinterTest.java +++ b/src/test/java/edu/harvard/iq/dataverse/util/json/JsonPrinterTest.java @@ -664,4 +664,74 @@ private DataFile createDatafile(long id) { return datafile; } + + + @Test + public void testJsonStorageDriver() { + // Test with directDownload enabled (true-like values) + System.setProperty("dataverse.files.test-driver.type", "s3"); + System.setProperty("dataverse.files.test-driver.label", "Test Storage Driver"); + System.setProperty("dataverse.files.test-driver.download-redirect", "true"); + System.setProperty("dataverse.files.test-driver.upload-redirect", "true"); + + JsonObject result = JsonPrinter.jsonStorageDriver("test-driver").build(); + + assertEquals("test-driver", result.getString("name")); + assertEquals("s3", result.getString("type")); + assertEquals("Test Storage Driver", result.getString("label")); + assertTrue(result.getBoolean("directUpload")); + assertTrue(result.getBoolean("directDownload")); + assertFalse(result.getBoolean("uploadOutOfBand")); + + // Test with directDownload disabled (false values) + System.setProperty("dataverse.files.test-driver2.type", "file"); + System.setProperty("dataverse.files.test-driver2.label", "Local Storage"); + System.setProperty("dataverse.files.test-driver2.download-redirect", "false"); + System.setProperty("dataverse.files.test-driver2.upload-redirect", "false"); + + JsonObject result2 = JsonPrinter.jsonStorageDriver("test-driver2").build(); + + assertEquals("test-driver2", result2.getString("name")); + assertEquals("file", result2.getString("type")); + assertEquals("Local Storage", result2.getString("label")); + assertFalse(result2.getBoolean("directUpload")); + assertFalse(result2.getBoolean("directDownload")); + assertFalse(result2.getBoolean("uploadOutOfBand")); + + // Test with all caps TRUE and out-of-band + System.setProperty("dataverse.files.test-driver3.type", "swift"); + System.setProperty("dataverse.files.test-driver3.label", "Swift Storage"); + System.setProperty("dataverse.files.test-driver3.download-redirect", "TRUE"); + System.setProperty("dataverse.files.test-driver3.upload-out-of-band", "true"); + + JsonObject result3 = JsonPrinter.jsonStorageDriver("test-driver3").build(); + assertTrue(result3.getBoolean("directDownload")); + assertTrue(result3.getBoolean("uploadOutOfBand")); + + // Test with null/missing properties + System.setProperty("dataverse.files.test-driver4.type", "s3"); + System.setProperty("dataverse.files.test-driver4.label", "Minimal Storage"); + // Not setting download-redirect property + + JsonObject result4 = JsonPrinter.jsonStorageDriver("test-driver4").build(); + assertFalse(result4.getBoolean("directDownload")); + assertFalse(result4.getBoolean("directUpload")); + assertFalse(result4.getBoolean("uploadOutOfBand")); + + // Clean up system properties + System.clearProperty("dataverse.files.test-driver.type"); + System.clearProperty("dataverse.files.test-driver.label"); + System.clearProperty("dataverse.files.test-driver.download-redirect"); + System.clearProperty("dataverse.files.test-driver.upload-redirect"); + System.clearProperty("dataverse.files.test-driver2.type"); + System.clearProperty("dataverse.files.test-driver2.label"); + System.clearProperty("dataverse.files.test-driver2.download-redirect"); + System.clearProperty("dataverse.files.test-driver2.upload-redirect"); + System.clearProperty("dataverse.files.test-driver3.type"); + System.clearProperty("dataverse.files.test-driver3.label"); + System.clearProperty("dataverse.files.test-driver3.download-redirect"); + System.clearProperty("dataverse.files.test-driver3.upload-out-of-band"); + System.clearProperty("dataverse.files.test-driver4.type"); + System.clearProperty("dataverse.files.test-driver4.label"); + } }