From f31768b0db8142037aeacce6712349cb595bae49 Mon Sep 17 00:00:00 2001 From: Carlos McGregor Date: Mon, 4 Mar 2019 18:31:15 -0500 Subject: [PATCH 1/9] Change from swift.properties config to System Remove function that loads swift.properties --- .../dataverse/dataaccess/SwiftAccessIO.java | 58 +++++-------------- 1 file changed, 15 insertions(+), 43 deletions(-) diff --git a/src/main/java/edu/harvard/iq/dataverse/dataaccess/SwiftAccessIO.java b/src/main/java/edu/harvard/iq/dataverse/dataaccess/SwiftAccessIO.java index 0e23a0a3505..2566e76747d 100644 --- a/src/main/java/edu/harvard/iq/dataverse/dataaccess/SwiftAccessIO.java +++ b/src/main/java/edu/harvard/iq/dataverse/dataaccess/SwiftAccessIO.java @@ -21,9 +21,7 @@ import java.util.ArrayList; import java.util.Collection; import java.util.Formatter; -import java.util.HashMap; import java.util.List; -import java.util.Map; import java.util.Properties; import java.util.logging.Logger; import javax.crypto.Mac; @@ -68,16 +66,15 @@ public SwiftAccessIO(T dvObject, DataAccessRequest req) { private Account account = null; private StoredObject swiftFileObject = null; private Container swiftContainer = null; - //TODO: when swift containers can be private, change this -SF - boolean publicSwiftContainer = true; + private boolean isPublicContainer = Boolean.parseBoolean(System.getProperty("dataverse.file.swift-is-public-container", "true")); + private String swiftFolderPathSeparator = System.getProperty("dataverse.files.swift-folder-path-separator", "_"); - //for hash private static final String HMAC_SHA1_ALGORITHM = "HmacSHA1"; //TODO: should this be dynamically generated based on size of file? //Also, this is in seconds - private static int TEMP_URL_EXPIRES = System.getProperty("dataverse.files.temp_url_expire") != null ? Integer.parseInt(System.getProperty("dataverse.files.temp_url_expire")) : 60; + private static int TEMP_URL_EXPIRES = Integer.parseInt(System.getProperty("dataverse.files.temp_url_expire", "60")); private static int LIST_PAGE_LIMIT = 100; @@ -515,8 +512,7 @@ private StoredObject initializeSwiftFileObject(boolean writeAccess, String auxIt // object! throw new IOException("IO driver mismatch: SwiftAccessIO called on a non-swift stored object."); } else if (this.isWriteAccess) { - Properties p = getSwiftProperties(); - swiftEndPoint = p.getProperty("swift.default.endpoint"); + swiftEndPoint = System.getProperty("dataverse.file.swift-default-endpoint"); // Swift uses this to create pseudo-hierarchical folders String swiftPseudoFolderPathSeparator = "/"; @@ -570,9 +566,7 @@ private StoredObject initializeSwiftFileObject(boolean writeAccess, String auxIt // object! throw new IOException("IO driver mismatch: SwiftAccessIO called on a non-swift stored object."); } else if (this.isWriteAccess) { - Properties p = getSwiftProperties(); - swiftEndPoint = p.getProperty("swift.default.endpoint"); - String swiftFolderPathSeparator = "-"; + swiftEndPoint = System.getProperty("dataverse.file.swift-default-endpoint"); // Swift uses this to create pseudo-hierarchical folders String swiftPseudoFolderPathSeparator = "/"; @@ -621,7 +615,7 @@ private StoredObject initializeSwiftFileObject(boolean writeAccess, String auxIt if (writeAccess) { //creates a private data container swiftContainer.create(); - if (publicSwiftContainer) { + if (isPublicContainer) { try { //creates a public data container this.swiftContainer.makePublic(); @@ -684,29 +678,13 @@ private StoredObject openSwiftAuxFile(boolean writeAccess, String auxItemTag) th return initializeSwiftFileObject(writeAccess, auxItemTag); } - private Properties getSwiftProperties() throws IOException { - if (swiftProperties == null) { - String domainRoot = System.getProperties().getProperty("com.sun.aas.instanceRoot"); - String swiftPropertiesFile = domainRoot + File.separator + "config" + File.separator + "swift.properties"; - swiftProperties = new Properties(); - swiftProperties.load(new FileInputStream(new File(swiftPropertiesFile))); - } - - return swiftProperties; - } - Account authenticateWithSwift(String swiftEndPoint) throws IOException { - - Properties p = getSwiftProperties(); - - // (this will throw an IOException, if the swift properties file - // is missing or corrupted) - String swiftEndPointAuthUrl = p.getProperty("swift.auth_url." + swiftEndPoint); - String swiftEndPointUsername = p.getProperty("swift.username." + swiftEndPoint); - String swiftEndPointSecretKey = p.getProperty("swift.password." + swiftEndPoint); - String swiftEndPointTenantName = p.getProperty("swift.tenant." + swiftEndPoint); - String swiftEndPointAuthMethod = p.getProperty("swift.auth_type." + swiftEndPoint); - String swiftEndPointTenantId = p.getProperty("swift.tenant_id." + swiftEndPoint); + String swiftEndPointAuthUrl = System.getProperty("dataverse.file.swift-auth-url." + swiftEndPoint); + String swiftEndPointUsername = System.getProperty("dataverse.file.swift-username." + swiftEndPoint); + String swiftEndPointSecretKey = System.getProperty("dataverse.file.swift-password." + swiftEndPoint); + String swiftEndPointTenantName = System.getProperty("dataverse.file.swift-tenant." + swiftEndPoint); + String swiftEndPointAuthMethod = System.getProperty("dataverse.file.swift-auth-type." + swiftEndPoint); + String swiftEndPointTenantId = System.getProperty("dataverse.file.swift-tenant-id." + swiftEndPoint); if (swiftEndPointAuthUrl == null || swiftEndPointUsername == null || swiftEndPointSecretKey == null || "".equals(swiftEndPointAuthUrl) || "".equals(swiftEndPointUsername) || "".equals(swiftEndPointSecretKey)) { @@ -775,10 +753,9 @@ private String getSwiftFileURI(StoredObject fileObject) throws IOException { private String hmac = null; public String generateTempUrlSignature(String swiftEndPoint, String containerName, String objectName, int duration) throws IOException { if (hmac == null || isExpiryExpired(generateTempUrlExpiry(duration, System.currentTimeMillis()), duration, System.currentTimeMillis())) { - Properties p = getSwiftProperties(); - String secretKey = p.getProperty("swift.hash_key." + swiftEndPoint); + String secretKey = System.getProperty("dataverse.file.swift-hash-key." + swiftEndPoint); if (secretKey == null) { - throw new IOException("Please input a hash key in swift.properties"); + throw new IOException("Please input a hash key under dataverse.file.swift-hash-key." + swiftEndPoint); } String path = "/v1/" + containerName + "/" + objectName; Long expires = generateTempUrlExpiry(duration, System.currentTimeMillis()); @@ -803,8 +780,7 @@ public long generateTempUrlExpiry(int duration, long currentTime) { private String temporaryUrl = null; private String generateTemporarySwiftUrl(String swiftEndPoint, String containerName, String objectName, int duration) throws IOException { - Properties p = getSwiftProperties(); - String baseUrl = p.getProperty("swift.swift_endpoint." + swiftEndPoint); + String baseUrl = System.getProperty("dataverse.file.swift-swift-endpoint." + swiftEndPoint); String path = "/v1/" + containerName + "/" + objectName; if (temporaryUrl == null || isExpiryExpired(generateTempUrlExpiry(duration, System.currentTimeMillis()), duration, System.currentTimeMillis())) { @@ -832,10 +808,6 @@ public InputStream getAuxFileAsInputStream(String auxItemTag) throws IOException @Override public String getSwiftContainerName() { - String swiftFolderPathSeparator = System.getProperty("dataverse.files.swift-folder-path-separator"); - if (swiftFolderPathSeparator == null) { - swiftFolderPathSeparator = "_"; - } if (dvObject instanceof DataFile) { String authorityNoSlashes = this.getDataFile().getOwner().getAuthorityForFileStorage().replace("/", swiftFolderPathSeparator); return this.getDataFile().getOwner().getProtocolForFileStorage() + swiftFolderPathSeparator From 509b950ecfa2806ff26944563c9298e1163f5a07 Mon Sep 17 00:00:00 2001 From: Carlos McGregor Date: Tue, 5 Mar 2019 17:43:50 -0500 Subject: [PATCH 2/9] Fix typo in property --- .../java/edu/harvard/iq/dataverse/dataaccess/SwiftAccessIO.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/java/edu/harvard/iq/dataverse/dataaccess/SwiftAccessIO.java b/src/main/java/edu/harvard/iq/dataverse/dataaccess/SwiftAccessIO.java index 2566e76747d..fe06a4e4212 100644 --- a/src/main/java/edu/harvard/iq/dataverse/dataaccess/SwiftAccessIO.java +++ b/src/main/java/edu/harvard/iq/dataverse/dataaccess/SwiftAccessIO.java @@ -780,7 +780,7 @@ public long generateTempUrlExpiry(int duration, long currentTime) { private String temporaryUrl = null; private String generateTemporarySwiftUrl(String swiftEndPoint, String containerName, String objectName, int duration) throws IOException { - String baseUrl = System.getProperty("dataverse.file.swift-swift-endpoint." + swiftEndPoint); + String baseUrl = System.getProperty("dataverse.file.swift-endpoint." + swiftEndPoint); String path = "/v1/" + containerName + "/" + objectName; if (temporaryUrl == null || isExpiryExpired(generateTempUrlExpiry(duration, System.currentTimeMillis()), duration, System.currentTimeMillis())) { From f54b410144142b46f6fac8242c742f5244471567 Mon Sep 17 00:00:00 2001 From: Carlos McGregor Date: Fri, 22 Mar 2019 15:07:03 -0400 Subject: [PATCH 3/9] Move swift default endpoint to class variable Add public class variable identifier prefix --- .../dataverse/dataaccess/SwiftAccessIO.java | 26 +++++++++---------- 1 file changed, 13 insertions(+), 13 deletions(-) diff --git a/src/main/java/edu/harvard/iq/dataverse/dataaccess/SwiftAccessIO.java b/src/main/java/edu/harvard/iq/dataverse/dataaccess/SwiftAccessIO.java index fe06a4e4212..102bcd03938 100644 --- a/src/main/java/edu/harvard/iq/dataverse/dataaccess/SwiftAccessIO.java +++ b/src/main/java/edu/harvard/iq/dataverse/dataaccess/SwiftAccessIO.java @@ -6,7 +6,6 @@ import edu.harvard.iq.dataverse.datavariable.DataVariable; import edu.harvard.iq.dataverse.util.StringUtil; import java.io.File; -import java.io.FileInputStream; import java.io.FileNotFoundException; import java.io.IOException; import java.io.InputStream; @@ -62,12 +61,12 @@ public SwiftAccessIO(T dvObject, DataAccessRequest req) { this.setIsLocalFile(false); } - private Properties swiftProperties = null; private Account account = null; private StoredObject swiftFileObject = null; private Container swiftContainer = null; private boolean isPublicContainer = Boolean.parseBoolean(System.getProperty("dataverse.file.swift-is-public-container", "true")); private String swiftFolderPathSeparator = System.getProperty("dataverse.files.swift-folder-path-separator", "_"); + private String swiftDefaultEndpoint = System.getProperty("dataverse.file.swift-default-endpoint"); //for hash private static final String HMAC_SHA1_ALGORITHM = "HmacSHA1"; @@ -76,7 +75,9 @@ public SwiftAccessIO(T dvObject, DataAccessRequest req) { //Also, this is in seconds private static int TEMP_URL_EXPIRES = Integer.parseInt(System.getProperty("dataverse.files.temp_url_expire", "60")); - private static int LIST_PAGE_LIMIT = 100; + private static int LIST_PAGE_LIMIT = 100; + + public static String SWIFT_IDENTIFIER_PREFIX = "swift"; @Override public void open(DataAccessOption... options) throws IOException { @@ -91,13 +92,14 @@ public void open(DataAccessOption... options) throws IOException { } if (dvObject instanceof DataFile) { + String storageIdentifier = dvObject.getStorageIdentifier(); DataFile dataFile = this.getDataFile(); if (req != null && req.getParameter("noVarHeader") != null) { this.setNoVarHeader(true); } - if (dataFile.getStorageIdentifier() == null || "".equals(dataFile.getStorageIdentifier())) { + if (storageIdentifier == null || "".equals(storageIdentifier)) { throw new IOException("Data Access: No local storage identifier defined for this datafile."); } @@ -251,7 +253,7 @@ public Channel openAuxChannel(String auxItemTag, DataAccessOption... options) th } @Override - public boolean isAuxObjectCached(String auxItemTag) throws IOException { + public boolean isAuxObjectCached(String auxItemTag) { StoredObject swiftAuxObject; try { swiftAuxObject = openSwiftAuxFile(auxItemTag); @@ -484,7 +486,7 @@ private StoredObject initializeSwiftFileObject(boolean writeAccess, String auxIt if (dvObject instanceof DataFile) { Dataset owner = this.getDataFile().getOwner(); - if (storageIdentifier.startsWith("swift://")) { + if (storageIdentifier.startsWith(SWIFT_IDENTIFIER_PREFIX + "://")) { // This is a call on an already existing swift object. String[] swiftStorageTokens = storageIdentifier.substring(8).split(":", 3); @@ -512,8 +514,6 @@ private StoredObject initializeSwiftFileObject(boolean writeAccess, String auxIt // object! throw new IOException("IO driver mismatch: SwiftAccessIO called on a non-swift stored object."); } else if (this.isWriteAccess) { - swiftEndPoint = System.getProperty("dataverse.file.swift-default-endpoint"); - // Swift uses this to create pseudo-hierarchical folders String swiftPseudoFolderPathSeparator = "/"; @@ -528,18 +528,18 @@ private StoredObject initializeSwiftFileObject(boolean writeAccess, String auxIt //setSwiftContainerName(swiftFolderPath); //swiftFileName = dataFile.getDisplayName(); //Storage Identifier is now updated after the object is uploaded on Swift. - dvObject.setStorageIdentifier("swift://" + swiftEndPoint + ":" + swiftFolderPath + ":" + swiftFileName); + dvObject.setStorageIdentifier(SWIFT_IDENTIFIER_PREFIX + "://" + swiftDefaultEndpoint + ":" + swiftFolderPath + ":" + swiftFileName); } else { throw new IOException("SwiftAccessIO: unknown access mode."); } } else if (dvObject instanceof Dataset) { Dataset dataset = this.getDataset(); - if (storageIdentifier.startsWith("swift://")) { + if (storageIdentifier.startsWith(SWIFT_IDENTIFIER_PREFIX + "://")) { // This is a call on an already existing swift object. //TODO: determine how storage identifer will give us info - String[] swiftStorageTokens = storageIdentifier.substring(8).split(":", 3); + String[] swiftStorageTokens = storageIdentifier.substring(8).split(":", 3); //number of tokens should be two because there is not main file if (swiftStorageTokens.length != 2) { // bad storage identifier @@ -577,7 +577,7 @@ private StoredObject initializeSwiftFileObject(boolean writeAccess, String auxIt swiftPseudoFolderPathSeparator + dataset.getIdentifierForFileStorage(); swiftFileName = auxItemTag; - dvObject.setStorageIdentifier("swift://" + swiftEndPoint + ":" + swiftFolderPath); + dvObject.setStorageIdentifier(SWIFT_IDENTIFIER_PREFIX + "://" + swiftEndPoint + ":" + swiftFolderPath); } else { throw new IOException("SwiftAccessIO: unknown access mode."); } @@ -604,7 +604,7 @@ private StoredObject initializeSwiftFileObject(boolean writeAccess, String auxIt other swiftContainerName Object Store pseudo-folder can be created, which is not provide by the joss Java swift library as of yet. */ - if (storageIdentifier.startsWith("swift://")) { + if (storageIdentifier.startsWith(SWIFT_IDENTIFIER_PREFIX + "://")) { // An existing swift object; the container must already exist as well. this.swiftContainer = account.getContainer(swiftContainerName); } else { From e2d324d059593f19fd4f242b3ff421e161615263 Mon Sep 17 00:00:00 2001 From: Carlos McGregor Date: Fri, 22 Mar 2019 15:36:36 -0400 Subject: [PATCH 4/9] Fix typo in temp URL expiry time setting --- .../java/edu/harvard/iq/dataverse/dataaccess/SwiftAccessIO.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/java/edu/harvard/iq/dataverse/dataaccess/SwiftAccessIO.java b/src/main/java/edu/harvard/iq/dataverse/dataaccess/SwiftAccessIO.java index 218459789a0..690b1136975 100644 --- a/src/main/java/edu/harvard/iq/dataverse/dataaccess/SwiftAccessIO.java +++ b/src/main/java/edu/harvard/iq/dataverse/dataaccess/SwiftAccessIO.java @@ -79,7 +79,7 @@ public SwiftAccessIO(String swiftLocation) { //TODO: should this be dynamically generated based on size of file? //Also, this is in seconds - private static int TEMP_URL_EXPIRES = Integer.parseInt(System.getProperty("dataverse.files.temp_url_expire", "60")); + private static int TEMP_URL_EXPIRES = Integer.parseInt(System.getProperty("dataverse.files.swift-temporary-url-expiry-time", "60")); private static int LIST_PAGE_LIMIT = 100; From 87f4cad28554a5704632c4a3f1310695df92c45f Mon Sep 17 00:00:00 2001 From: Carlos McGregor Date: Fri, 22 Mar 2019 17:22:23 -0400 Subject: [PATCH 5/9] Update Swift installation guide --- .../source/installation/config.rst | 30 ++++++++----------- 1 file changed, 13 insertions(+), 17 deletions(-) diff --git a/doc/sphinx-guides/source/installation/config.rst b/doc/sphinx-guides/source/installation/config.rst index 3d637b0e9d4..85c819f77e9 100644 --- a/doc/sphinx-guides/source/installation/config.rst +++ b/doc/sphinx-guides/source/installation/config.rst @@ -202,19 +202,19 @@ Swift Storage Rather than storing data files on the filesystem, you can opt for an experimental setup with a `Swift Object Storage `_ backend. Each dataset that users create gets a corresponding "container" on the Swift side, and each data file is saved as a file within that container. -**In order to configure a Swift installation,** there are two steps you need to complete: +**In order to configure a Swift installation,** you need to complete these steps to properly modify the JVM options: -First, create a file named ``swift.properties`` as follows in the ``config`` directory for your installation of Glassfish (by default, this would be ``/usr/local/glassfish4/glassfish/domains/domain1/config/swift.properties``): +First, run all the following create commands with your Swift endpoint information and credentials: .. code-block:: none - swift.default.endpoint=endpoint1 - swift.auth_type.endpoint1=your-authentication-type - swift.auth_url.endpoint1=your-auth-url - swift.tenant.endpoint1=your-tenant-name - swift.username.endpoint1=your-username - swift.password.endpoint1=your-password - swift.swift_endpoint.endpoint1=your-swift-endpoint + ./asadmin $ASADMIN_OPTS create-jvm-options "\-Ddataverse.file.swift-default-endpoint=endpoint1" + ./asadmin $ASADMIN_OPTS create-jvm-options "\-Ddataverse.file.swift-auth-type.endpoint1=your-auth-type" + ./asadmin $ASADMIN_OPTS create-jvm-options "\-Ddataverse.file.swift-auth-url.endpoint1=your-auth-url" + ./asadmin $ASADMIN_OPTS create-jvm-options "\-Ddataverse.file.swift-tenant.endpoint1=your-tenant-name" + ./asadmin $ASADMIN_OPTS create-jvm-options "\-Ddataverse.file.swift-username.endpoint1=your-username" + ./asadmin $ASADMIN_OPTS create-jvm-options "\-Ddataverse.file.swift-password.endpoint1='${ALIAS=your-password}'" + ./asadmin $ASADMIN_OPTS create-jvm-options "\-Ddataverse.file.swift-endpoint.endpoint1=your-swift-endpoint" ``auth_type`` can either be ``keystone``, ``keystone_v3``, or it will assumed to be ``basic``. ``auth_url`` should be your keystone authentication URL which includes the tokens (e.g. for keystone, ``https://openstack.example.edu:35357/v2.0/tokens`` and for keystone_v3, ``https://openstack.example.edu:35357/v3/auth/tokens``). ``swift_endpoint`` is a URL that look something like ``http://rdgw.swift.example.org/swift/v1``. @@ -232,17 +232,13 @@ You also have the option to set a **custom container name separator.** It is ini By default, your Swift installation will be public-only, meaning users will be unable to put access restrictions on their data. If you are comfortable with this level of privacy, the final step in your setup is to set the :ref:`:PublicInstall` setting to `true`. -In order to **enable file access restrictions**, you must enable Swift to use temporary URLs for file access. To enable usage of temporary URLs, set a hash key both on your swift endpoint and in your swift.properties file. You can do so by adding +In order to **enable file access restrictions**, you must enable Swift to use temporary URLs for file access. To enable usage of temporary URLs, set a hash key both on your swift endpoint and in your swift.properties file. You can do so by running the create command: -.. code-block:: none - - swift.hash_key.endpoint1=your-hash-key - -to your swift.properties file. +``./asadmin $ASADMIN_OPTS create-jvm-options "\-Ddataverse.file.swift-hash-key.endpoint1=your-hash-key"`` -You also have the option to set a custom expiration length for a generated temporary URL. It is initialized to 60 seconds, but you can change it by running the create command: +You also have the option to set a custom expiration length, in seconds, for a generated temporary URL. It is initialized to 60 seconds, but you can change it by running the create command: -``./asadmin $ASADMIN_OPTS create-jvm-options "\-Ddataverse.files.temp_url_expire=3600"`` +``./asadmin $ASADMIN_OPTS create-jvm-options "\-Ddataverse.files.swift-temporary-url-expiry-time=3600"`` In this example, you would be setting the expiration length for one hour. From 1417237b7038c8d740f8f3d871c889a57d9f9d52 Mon Sep 17 00:00:00 2001 From: Carlos McGregor Date: Mon, 25 Mar 2019 17:13:38 -0400 Subject: [PATCH 6/9] Change Swift settings to dot and camel case in Swift driver and documentation Add full create-password-alias instructions in documentation Add missing swiftEndPoint --- .../source/installation/config.rst | 28 ++++++++++------- .../dataverse/dataaccess/SwiftAccessIO.java | 30 ++++++++++--------- 2 files changed, 34 insertions(+), 24 deletions(-) diff --git a/doc/sphinx-guides/source/installation/config.rst b/doc/sphinx-guides/source/installation/config.rst index 85c819f77e9..eea559562ec 100644 --- a/doc/sphinx-guides/source/installation/config.rst +++ b/doc/sphinx-guides/source/installation/config.rst @@ -208,16 +208,24 @@ First, run all the following create commands with your Swift endpoint informatio .. code-block:: none - ./asadmin $ASADMIN_OPTS create-jvm-options "\-Ddataverse.file.swift-default-endpoint=endpoint1" - ./asadmin $ASADMIN_OPTS create-jvm-options "\-Ddataverse.file.swift-auth-type.endpoint1=your-auth-type" - ./asadmin $ASADMIN_OPTS create-jvm-options "\-Ddataverse.file.swift-auth-url.endpoint1=your-auth-url" - ./asadmin $ASADMIN_OPTS create-jvm-options "\-Ddataverse.file.swift-tenant.endpoint1=your-tenant-name" - ./asadmin $ASADMIN_OPTS create-jvm-options "\-Ddataverse.file.swift-username.endpoint1=your-username" - ./asadmin $ASADMIN_OPTS create-jvm-options "\-Ddataverse.file.swift-password.endpoint1='${ALIAS=your-password}'" - ./asadmin $ASADMIN_OPTS create-jvm-options "\-Ddataverse.file.swift-endpoint.endpoint1=your-swift-endpoint" + ./asadmin $ASADMIN_OPTS create-jvm-options "\-Ddataverse.file.swift.defaultEndpoint=endpoint1" + ./asadmin $ASADMIN_OPTS create-jvm-options "\-Ddataverse.file.swift.authType.endpoint1=your-auth-type" + ./asadmin $ASADMIN_OPTS create-jvm-options "\-Ddataverse.file.swift.authUrl.endpoint1=your-auth-url" + ./asadmin $ASADMIN_OPTS create-jvm-options "\-Ddataverse.file.swift.tenant.endpoint1=your-tenant-name" + ./asadmin $ASADMIN_OPTS create-jvm-options "\-Ddataverse.file.swift.username.endpoint1=your-username" + ./asadmin $ASADMIN_OPTS create-jvm-options "\-Ddataverse.file.swift.endpoint.endpoint1=your-swift-endpoint" ``auth_type`` can either be ``keystone``, ``keystone_v3``, or it will assumed to be ``basic``. ``auth_url`` should be your keystone authentication URL which includes the tokens (e.g. for keystone, ``https://openstack.example.edu:35357/v2.0/tokens`` and for keystone_v3, ``https://openstack.example.edu:35357/v3/auth/tokens``). ``swift_endpoint`` is a URL that look something like ``http://rdgw.swift.example.org/swift/v1``. +Then create a password alias by running (without changes): + +.. code-block:: none + + ./asadmin $ASADMIN_OPTS create-jvm-options "\-Ddataverse.file.swift.password.endpoint1='${ALIAS=swiftpassword-alias}'" + ./asadmin $ASADMIN_OPTS create-password-alias swiftpassword-alias + +The second command will trigger an interactive prompt asking you to input your Swift password. + Second, update the JVM option ``dataverse.files.storage-driver-id`` by running the delete command: ``./asadmin $ASADMIN_OPTS delete-jvm-options "\-Ddataverse.files.storage-driver-id=file"`` @@ -228,17 +236,17 @@ Then run the create command: You also have the option to set a **custom container name separator.** It is initialized to ``_``, but you can change it by running the create command: -``./asadmin $ASADMIN_OPTS create-jvm-options "\-Ddataverse.files.swift-folder-path-separator=-"`` +``./asadmin $ASADMIN_OPTS create-jvm-options "\-Ddataverse.files.swift.folderPathSeparator=-"`` By default, your Swift installation will be public-only, meaning users will be unable to put access restrictions on their data. If you are comfortable with this level of privacy, the final step in your setup is to set the :ref:`:PublicInstall` setting to `true`. In order to **enable file access restrictions**, you must enable Swift to use temporary URLs for file access. To enable usage of temporary URLs, set a hash key both on your swift endpoint and in your swift.properties file. You can do so by running the create command: -``./asadmin $ASADMIN_OPTS create-jvm-options "\-Ddataverse.file.swift-hash-key.endpoint1=your-hash-key"`` +``./asadmin $ASADMIN_OPTS create-jvm-options "\-Ddataverse.file.swift.hashKey.endpoint1=your-hash-key"`` You also have the option to set a custom expiration length, in seconds, for a generated temporary URL. It is initialized to 60 seconds, but you can change it by running the create command: -``./asadmin $ASADMIN_OPTS create-jvm-options "\-Ddataverse.files.swift-temporary-url-expiry-time=3600"`` +``./asadmin $ASADMIN_OPTS create-jvm-options "\-Ddataverse.files.swift.temporaryUrlExpiryTime=3600"`` In this example, you would be setting the expiration length for one hour. diff --git a/src/main/java/edu/harvard/iq/dataverse/dataaccess/SwiftAccessIO.java b/src/main/java/edu/harvard/iq/dataverse/dataaccess/SwiftAccessIO.java index 690b1136975..73ee28e17b9 100644 --- a/src/main/java/edu/harvard/iq/dataverse/dataaccess/SwiftAccessIO.java +++ b/src/main/java/edu/harvard/iq/dataverse/dataaccess/SwiftAccessIO.java @@ -70,16 +70,16 @@ public SwiftAccessIO(String swiftLocation) { private Account account = null; private StoredObject swiftFileObject = null; private Container swiftContainer = null; - private boolean isPublicContainer = Boolean.parseBoolean(System.getProperty("dataverse.file.swift-is-public-container", "true")); - private String swiftFolderPathSeparator = System.getProperty("dataverse.files.swift-folder-path-separator", "_"); - private String swiftDefaultEndpoint = System.getProperty("dataverse.file.swift-default-endpoint"); + private boolean isPublicContainer = Boolean.parseBoolean(System.getProperty("dataverse.files.swift.isPublicContainer", "true")); + private String swiftFolderPathSeparator = System.getProperty("dataverse.files.swift.folderPathSeparator", "_"); + private String swiftDefaultEndpoint = System.getProperty("dataverse.files.swift.defaultEndpoint"); //for hash private static final String HMAC_SHA1_ALGORITHM = "HmacSHA1"; //TODO: should this be dynamically generated based on size of file? //Also, this is in seconds - private static int TEMP_URL_EXPIRES = Integer.parseInt(System.getProperty("dataverse.files.swift-temporary-url-expiry-time", "60")); + private static int TEMP_URL_EXPIRES = Integer.parseInt(System.getProperty("dataverse.files.swift.temporaryUrlExpiryTime", "60")); private static int LIST_PAGE_LIMIT = 100; @@ -531,6 +531,8 @@ private StoredObject initializeSwiftFileObject(boolean writeAccess, String auxIt // object! throw new IOException("IO driver mismatch: SwiftAccessIO called on a non-swift stored object."); } else if (this.isWriteAccess) { + swiftEndPoint = swiftDefaultEndpoint; + // Swift uses this to create pseudo-hierarchical folders String swiftPseudoFolderPathSeparator = "/"; @@ -583,7 +585,7 @@ private StoredObject initializeSwiftFileObject(boolean writeAccess, String auxIt // object! throw new IOException("IO driver mismatch: SwiftAccessIO called on a non-swift stored object."); } else if (this.isWriteAccess) { - swiftEndPoint = System.getProperty("dataverse.file.swift-default-endpoint"); + swiftEndPoint = swiftDefaultEndpoint; // Swift uses this to create pseudo-hierarchical folders String swiftPseudoFolderPathSeparator = "/"; @@ -730,12 +732,12 @@ private StoredObject openSwiftAuxFile(boolean writeAccess, String auxItemTag) th } Account authenticateWithSwift(String swiftEndPoint) throws IOException { - String swiftEndPointAuthUrl = System.getProperty("dataverse.file.swift-auth-url." + swiftEndPoint); - String swiftEndPointUsername = System.getProperty("dataverse.file.swift-username." + swiftEndPoint); - String swiftEndPointSecretKey = System.getProperty("dataverse.file.swift-password." + swiftEndPoint); - String swiftEndPointTenantName = System.getProperty("dataverse.file.swift-tenant." + swiftEndPoint); - String swiftEndPointAuthMethod = System.getProperty("dataverse.file.swift-auth-type." + swiftEndPoint); - String swiftEndPointTenantId = System.getProperty("dataverse.file.swift-tenant-id." + swiftEndPoint); + String swiftEndPointAuthUrl = System.getProperty("dataverse.files.swift.authUrl." + swiftEndPoint); + String swiftEndPointUsername = System.getProperty("dataverse.files.swift.username." + swiftEndPoint); + String swiftEndPointSecretKey = System.getProperty("dataverse.files.swift.password." + swiftEndPoint); + String swiftEndPointTenantName = System.getProperty("dataverse.files.swift.tenant." + swiftEndPoint); + String swiftEndPointAuthMethod = System.getProperty("dataverse.files.swift.authType." + swiftEndPoint); + String swiftEndPointTenantId = System.getProperty("dataverse.files.swift.tenant." + swiftEndPoint); if (swiftEndPointAuthUrl == null || swiftEndPointUsername == null || swiftEndPointSecretKey == null || "".equals(swiftEndPointAuthUrl) || "".equals(swiftEndPointUsername) || "".equals(swiftEndPointSecretKey)) { @@ -804,9 +806,9 @@ private String getSwiftFileURI(StoredObject fileObject) throws IOException { private String hmac = null; public String generateTempUrlSignature(String swiftEndPoint, String containerName, String objectName, int duration) throws IOException { if (hmac == null || isExpiryExpired(generateTempUrlExpiry(duration, System.currentTimeMillis()), duration, System.currentTimeMillis())) { - String secretKey = System.getProperty("dataverse.file.swift-hash-key." + swiftEndPoint); + String secretKey = System.getProperty("dataverse.files.swift.hashKey." + swiftEndPoint); if (secretKey == null) { - throw new IOException("Please input a hash key under dataverse.file.swift-hash-key." + swiftEndPoint); + throw new IOException("Please input a hash key under dataverse.files.swift.hashKey." + swiftEndPoint); } String path = "/v1/" + containerName + "/" + objectName; Long expires = generateTempUrlExpiry(duration, System.currentTimeMillis()); @@ -831,7 +833,7 @@ public long generateTempUrlExpiry(int duration, long currentTime) { private String temporaryUrl = null; private String generateTemporarySwiftUrl(String swiftEndPoint, String containerName, String objectName, int duration) throws IOException { - String baseUrl = System.getProperty("dataverse.file.swift-endpoint." + swiftEndPoint); + String baseUrl = System.getProperty("dataverse.files.swift.endpoint." + swiftEndPoint); String path = "/v1/" + containerName + "/" + objectName; if (temporaryUrl == null || isExpiryExpired(generateTempUrlExpiry(duration, System.currentTimeMillis()), duration, System.currentTimeMillis())) { From 166c62f0a07334b43647664a1ee16a55321fa838 Mon Sep 17 00:00:00 2001 From: Carlos McGregor Date: Mon, 25 Mar 2019 18:03:53 -0400 Subject: [PATCH 7/9] Fix tiny typo --- doc/sphinx-guides/source/installation/config.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/sphinx-guides/source/installation/config.rst b/doc/sphinx-guides/source/installation/config.rst index eea559562ec..1795bbc35bc 100644 --- a/doc/sphinx-guides/source/installation/config.rst +++ b/doc/sphinx-guides/source/installation/config.rst @@ -215,7 +215,7 @@ First, run all the following create commands with your Swift endpoint informatio ./asadmin $ASADMIN_OPTS create-jvm-options "\-Ddataverse.file.swift.username.endpoint1=your-username" ./asadmin $ASADMIN_OPTS create-jvm-options "\-Ddataverse.file.swift.endpoint.endpoint1=your-swift-endpoint" -``auth_type`` can either be ``keystone``, ``keystone_v3``, or it will assumed to be ``basic``. ``auth_url`` should be your keystone authentication URL which includes the tokens (e.g. for keystone, ``https://openstack.example.edu:35357/v2.0/tokens`` and for keystone_v3, ``https://openstack.example.edu:35357/v3/auth/tokens``). ``swift_endpoint`` is a URL that look something like ``http://rdgw.swift.example.org/swift/v1``. +``auth_type`` can either be ``keystone``, ``keystone_v3``, or it will assumed to be ``basic``. ``auth_url`` should be your keystone authentication URL which includes the tokens (e.g. for keystone, ``https://openstack.example.edu:35357/v2.0/tokens`` and for keystone_v3, ``https://openstack.example.edu:35357/v3/auth/tokens``). ``swift_endpoint`` is a URL that looks something like ``http://rdgw.swift.example.org/swift/v1``. Then create a password alias by running (without changes): From eed33a3ae548483d1caa45e47ebf8db0e6acb321 Mon Sep 17 00:00:00 2001 From: Carlos Mc Gregor Date: Mon, 25 Mar 2019 20:08:57 -0400 Subject: [PATCH 8/9] Add missing s's --- doc/sphinx-guides/source/installation/config.rst | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/doc/sphinx-guides/source/installation/config.rst b/doc/sphinx-guides/source/installation/config.rst index 1795bbc35bc..a8377454b37 100644 --- a/doc/sphinx-guides/source/installation/config.rst +++ b/doc/sphinx-guides/source/installation/config.rst @@ -208,12 +208,12 @@ First, run all the following create commands with your Swift endpoint informatio .. code-block:: none - ./asadmin $ASADMIN_OPTS create-jvm-options "\-Ddataverse.file.swift.defaultEndpoint=endpoint1" - ./asadmin $ASADMIN_OPTS create-jvm-options "\-Ddataverse.file.swift.authType.endpoint1=your-auth-type" - ./asadmin $ASADMIN_OPTS create-jvm-options "\-Ddataverse.file.swift.authUrl.endpoint1=your-auth-url" - ./asadmin $ASADMIN_OPTS create-jvm-options "\-Ddataverse.file.swift.tenant.endpoint1=your-tenant-name" - ./asadmin $ASADMIN_OPTS create-jvm-options "\-Ddataverse.file.swift.username.endpoint1=your-username" - ./asadmin $ASADMIN_OPTS create-jvm-options "\-Ddataverse.file.swift.endpoint.endpoint1=your-swift-endpoint" + ./asadmin $ASADMIN_OPTS create-jvm-options "\-Ddataverse.files.swift.defaultEndpoint=endpoint1" + ./asadmin $ASADMIN_OPTS create-jvm-options "\-Ddataverse.files.swift.authType.endpoint1=your-auth-type" + ./asadmin $ASADMIN_OPTS create-jvm-options "\-Ddataverse.files.swift.authUrl.endpoint1=your-auth-url" + ./asadmin $ASADMIN_OPTS create-jvm-options "\-Ddataverse.files.swift.tenant.endpoint1=your-tenant-name" + ./asadmin $ASADMIN_OPTS create-jvm-options "\-Ddataverse.files.swift.username.endpoint1=your-username" + ./asadmin $ASADMIN_OPTS create-jvm-options "\-Ddataverse.files.swift.endpoint.endpoint1=your-swift-endpoint" ``auth_type`` can either be ``keystone``, ``keystone_v3``, or it will assumed to be ``basic``. ``auth_url`` should be your keystone authentication URL which includes the tokens (e.g. for keystone, ``https://openstack.example.edu:35357/v2.0/tokens`` and for keystone_v3, ``https://openstack.example.edu:35357/v3/auth/tokens``). ``swift_endpoint`` is a URL that looks something like ``http://rdgw.swift.example.org/swift/v1``. @@ -221,7 +221,7 @@ Then create a password alias by running (without changes): .. code-block:: none - ./asadmin $ASADMIN_OPTS create-jvm-options "\-Ddataverse.file.swift.password.endpoint1='${ALIAS=swiftpassword-alias}'" + ./asadmin $ASADMIN_OPTS create-jvm-options "\-Ddataverse.files.swift.password.endpoint1='${ALIAS=swiftpassword-alias}'" ./asadmin $ASADMIN_OPTS create-password-alias swiftpassword-alias The second command will trigger an interactive prompt asking you to input your Swift password. @@ -242,7 +242,7 @@ By default, your Swift installation will be public-only, meaning users will be u In order to **enable file access restrictions**, you must enable Swift to use temporary URLs for file access. To enable usage of temporary URLs, set a hash key both on your swift endpoint and in your swift.properties file. You can do so by running the create command: -``./asadmin $ASADMIN_OPTS create-jvm-options "\-Ddataverse.file.swift.hashKey.endpoint1=your-hash-key"`` +``./asadmin $ASADMIN_OPTS create-jvm-options "\-Ddataverse.files.swift.hashKey.endpoint1=your-hash-key"`` You also have the option to set a custom expiration length, in seconds, for a generated temporary URL. It is initialized to 60 seconds, but you can change it by running the create command: From e0ab3526d82c1c4ec663a48147ac885daf077657 Mon Sep 17 00:00:00 2001 From: Carlos McGregor Date: Thu, 11 Apr 2019 15:33:55 -0400 Subject: [PATCH 9/9] Add release notes for 5478 --- doc/release-notes/5478-refactor-swift-properties.md | 7 +++++++ 1 file changed, 7 insertions(+) create mode 100644 doc/release-notes/5478-refactor-swift-properties.md diff --git a/doc/release-notes/5478-refactor-swift-properties.md b/doc/release-notes/5478-refactor-swift-properties.md new file mode 100644 index 00000000000..5d87173f499 --- /dev/null +++ b/doc/release-notes/5478-refactor-swift-properties.md @@ -0,0 +1,7 @@ +Now all Swift properties have been migrated to `domain.xml`, no longer needing to maintain a separate +`swift.properties` file, and offering better governability and performance. Furthermore, now the Swift +credential's password is stored using `create-password-alias`, which encrypts the password so that it does +not appear in plain text on `domain.xml`. + +In order to migrate to these new configuration settings, please visit +`doc/sphinx-guides/source/installation/config.rst#swift-storage`. \ No newline at end of file