diff --git a/doc/release-notes/5478-refactor-swift-properties.md b/doc/release-notes/5478-refactor-swift-properties.md new file mode 100644 index 00000000000..5d87173f499 --- /dev/null +++ b/doc/release-notes/5478-refactor-swift-properties.md @@ -0,0 +1,7 @@ +Now all Swift properties have been migrated to `domain.xml`, no longer needing to maintain a separate +`swift.properties` file, and offering better governability and performance. Furthermore, now the Swift +credential's password is stored using `create-password-alias`, which encrypts the password so that it does +not appear in plain text on `domain.xml`. + +In order to migrate to these new configuration settings, please visit +`doc/sphinx-guides/source/installation/config.rst#swift-storage`. \ No newline at end of file diff --git a/doc/sphinx-guides/source/installation/config.rst b/doc/sphinx-guides/source/installation/config.rst index 7476ddba974..6574142e1e8 100644 --- a/doc/sphinx-guides/source/installation/config.rst +++ b/doc/sphinx-guides/source/installation/config.rst @@ -207,21 +207,29 @@ Swift Storage Rather than storing data files on the filesystem, you can opt for an experimental setup with a `Swift Object Storage `_ backend. Each dataset that users create gets a corresponding "container" on the Swift side, and each data file is saved as a file within that container. -**In order to configure a Swift installation,** there are two steps you need to complete: +**In order to configure a Swift installation,** you need to complete these steps to properly modify the JVM options: -First, create a file named ``swift.properties`` as follows in the ``config`` directory for your installation of Glassfish (by default, this would be ``/usr/local/glassfish4/glassfish/domains/domain1/config/swift.properties``): +First, run all the following create commands with your Swift endpoint information and credentials: .. code-block:: none - swift.default.endpoint=endpoint1 - swift.auth_type.endpoint1=your-authentication-type - swift.auth_url.endpoint1=your-auth-url - swift.tenant.endpoint1=your-tenant-name - swift.username.endpoint1=your-username - swift.password.endpoint1=your-password - swift.swift_endpoint.endpoint1=your-swift-endpoint + ./asadmin $ASADMIN_OPTS create-jvm-options "\-Ddataverse.files.swift.defaultEndpoint=endpoint1" + ./asadmin $ASADMIN_OPTS create-jvm-options "\-Ddataverse.files.swift.authType.endpoint1=your-auth-type" + ./asadmin $ASADMIN_OPTS create-jvm-options "\-Ddataverse.files.swift.authUrl.endpoint1=your-auth-url" + ./asadmin $ASADMIN_OPTS create-jvm-options "\-Ddataverse.files.swift.tenant.endpoint1=your-tenant-name" + ./asadmin $ASADMIN_OPTS create-jvm-options "\-Ddataverse.files.swift.username.endpoint1=your-username" + ./asadmin $ASADMIN_OPTS create-jvm-options "\-Ddataverse.files.swift.endpoint.endpoint1=your-swift-endpoint" -``auth_type`` can either be ``keystone``, ``keystone_v3``, or it will assumed to be ``basic``. ``auth_url`` should be your keystone authentication URL which includes the tokens (e.g. for keystone, ``https://openstack.example.edu:35357/v2.0/tokens`` and for keystone_v3, ``https://openstack.example.edu:35357/v3/auth/tokens``). ``swift_endpoint`` is a URL that look something like ``http://rdgw.swift.example.org/swift/v1``. +``auth_type`` can either be ``keystone``, ``keystone_v3``, or it will assumed to be ``basic``. ``auth_url`` should be your keystone authentication URL which includes the tokens (e.g. for keystone, ``https://openstack.example.edu:35357/v2.0/tokens`` and for keystone_v3, ``https://openstack.example.edu:35357/v3/auth/tokens``). ``swift_endpoint`` is a URL that looks something like ``http://rdgw.swift.example.org/swift/v1``. + +Then create a password alias by running (without changes): + +.. code-block:: none + + ./asadmin $ASADMIN_OPTS create-jvm-options "\-Ddataverse.files.swift.password.endpoint1='${ALIAS=swiftpassword-alias}'" + ./asadmin $ASADMIN_OPTS create-password-alias swiftpassword-alias + +The second command will trigger an interactive prompt asking you to input your Swift password. Second, update the JVM option ``dataverse.files.storage-driver-id`` by running the delete command: @@ -233,21 +241,17 @@ Then run the create command: You also have the option to set a **custom container name separator.** It is initialized to ``_``, but you can change it by running the create command: -``./asadmin $ASADMIN_OPTS create-jvm-options "\-Ddataverse.files.swift-folder-path-separator=-"`` +``./asadmin $ASADMIN_OPTS create-jvm-options "\-Ddataverse.files.swift.folderPathSeparator=-"`` By default, your Swift installation will be public-only, meaning users will be unable to put access restrictions on their data. If you are comfortable with this level of privacy, the final step in your setup is to set the :ref:`:PublicInstall` setting to `true`. -In order to **enable file access restrictions**, you must enable Swift to use temporary URLs for file access. To enable usage of temporary URLs, set a hash key both on your swift endpoint and in your swift.properties file. You can do so by adding - -.. code-block:: none - - swift.hash_key.endpoint1=your-hash-key +In order to **enable file access restrictions**, you must enable Swift to use temporary URLs for file access. To enable usage of temporary URLs, set a hash key both on your swift endpoint and in your swift.properties file. You can do so by running the create command: -to your swift.properties file. +``./asadmin $ASADMIN_OPTS create-jvm-options "\-Ddataverse.files.swift.hashKey.endpoint1=your-hash-key"`` -You also have the option to set a custom expiration length for a generated temporary URL. It is initialized to 60 seconds, but you can change it by running the create command: +You also have the option to set a custom expiration length, in seconds, for a generated temporary URL. It is initialized to 60 seconds, but you can change it by running the create command: -``./asadmin $ASADMIN_OPTS create-jvm-options "\-Ddataverse.files.temp_url_expire=3600"`` +``./asadmin $ASADMIN_OPTS create-jvm-options "\-Ddataverse.files.swift.temporaryUrlExpiryTime=3600"`` In this example, you would be setting the expiration length for one hour. diff --git a/src/main/java/edu/harvard/iq/dataverse/dataaccess/SwiftAccessIO.java b/src/main/java/edu/harvard/iq/dataverse/dataaccess/SwiftAccessIO.java index f6279d1e360..73ee28e17b9 100644 --- a/src/main/java/edu/harvard/iq/dataverse/dataaccess/SwiftAccessIO.java +++ b/src/main/java/edu/harvard/iq/dataverse/dataaccess/SwiftAccessIO.java @@ -6,7 +6,6 @@ import edu.harvard.iq.dataverse.datavariable.DataVariable; import edu.harvard.iq.dataverse.util.StringUtil; import java.io.File; -import java.io.FileInputStream; import java.io.FileNotFoundException; import java.io.IOException; import java.io.InputStream; @@ -68,22 +67,23 @@ public SwiftAccessIO(String swiftLocation) { this.swiftLocation = swiftLocation; } - private Properties swiftProperties = null; private Account account = null; private StoredObject swiftFileObject = null; private Container swiftContainer = null; - //TODO: when swift containers can be private, change this -SF - boolean publicSwiftContainer = true; + private boolean isPublicContainer = Boolean.parseBoolean(System.getProperty("dataverse.files.swift.isPublicContainer", "true")); + private String swiftFolderPathSeparator = System.getProperty("dataverse.files.swift.folderPathSeparator", "_"); + private String swiftDefaultEndpoint = System.getProperty("dataverse.files.swift.defaultEndpoint"); - //for hash private static final String HMAC_SHA1_ALGORITHM = "HmacSHA1"; //TODO: should this be dynamically generated based on size of file? //Also, this is in seconds - private static int TEMP_URL_EXPIRES = System.getProperty("dataverse.files.temp_url_expire") != null ? Integer.parseInt(System.getProperty("dataverse.files.temp_url_expire")) : 60; + private static int TEMP_URL_EXPIRES = Integer.parseInt(System.getProperty("dataverse.files.swift.temporaryUrlExpiryTime", "60")); + + private static int LIST_PAGE_LIMIT = 100; - private static int LIST_PAGE_LIMIT = 100; + public static String SWIFT_IDENTIFIER_PREFIX = "swift"; @Override public void open(DataAccessOption... options) throws IOException { @@ -98,13 +98,14 @@ public void open(DataAccessOption... options) throws IOException { } if (dvObject instanceof DataFile) { + String storageIdentifier = dvObject.getStorageIdentifier(); DataFile dataFile = this.getDataFile(); if (req != null && req.getParameter("noVarHeader") != null) { this.setNoVarHeader(true); } - if (dataFile.getStorageIdentifier() == null || "".equals(dataFile.getStorageIdentifier())) { + if (storageIdentifier == null || "".equals(storageIdentifier)) { throw new IOException("Data Access: No local storage identifier defined for this datafile."); } @@ -269,7 +270,7 @@ public Channel openAuxChannel(String auxItemTag, DataAccessOption... options) th } @Override - public boolean isAuxObjectCached(String auxItemTag) throws IOException { + public boolean isAuxObjectCached(String auxItemTag) { StoredObject swiftAuxObject; try { swiftAuxObject = openSwiftAuxFile(auxItemTag); @@ -502,7 +503,7 @@ private StoredObject initializeSwiftFileObject(boolean writeAccess, String auxIt if (dvObject instanceof DataFile) { Dataset owner = this.getDataFile().getOwner(); - if (storageIdentifier.startsWith("swift://")) { + if (storageIdentifier.startsWith(SWIFT_IDENTIFIER_PREFIX + "://")) { // This is a call on an already existing swift object. String[] swiftStorageTokens = storageIdentifier.substring(8).split(":", 3); @@ -530,8 +531,7 @@ private StoredObject initializeSwiftFileObject(boolean writeAccess, String auxIt // object! throw new IOException("IO driver mismatch: SwiftAccessIO called on a non-swift stored object."); } else if (this.isWriteAccess) { - Properties p = getSwiftProperties(); - swiftEndPoint = p.getProperty("swift.default.endpoint"); + swiftEndPoint = swiftDefaultEndpoint; // Swift uses this to create pseudo-hierarchical folders String swiftPseudoFolderPathSeparator = "/"; @@ -547,18 +547,18 @@ private StoredObject initializeSwiftFileObject(boolean writeAccess, String auxIt //setSwiftContainerName(swiftFolderPath); //swiftFileName = dataFile.getDisplayName(); //Storage Identifier is now updated after the object is uploaded on Swift. - dvObject.setStorageIdentifier("swift://" + swiftEndPoint + ":" + swiftFolderPath + ":" + swiftFileName); + dvObject.setStorageIdentifier(SWIFT_IDENTIFIER_PREFIX + "://" + swiftDefaultEndpoint + ":" + swiftFolderPath + ":" + swiftFileName); } else { throw new IOException("SwiftAccessIO: unknown access mode."); } } else if (dvObject instanceof Dataset) { Dataset dataset = this.getDataset(); - if (storageIdentifier.startsWith("swift://")) { + if (storageIdentifier.startsWith(SWIFT_IDENTIFIER_PREFIX + "://")) { // This is a call on an already existing swift object. //TODO: determine how storage identifer will give us info - String[] swiftStorageTokens = storageIdentifier.substring(8).split(":", 3); + String[] swiftStorageTokens = storageIdentifier.substring(8).split(":", 3); //number of tokens should be two because there is not main file if (swiftStorageTokens.length != 2) { // bad storage identifier @@ -585,9 +585,7 @@ private StoredObject initializeSwiftFileObject(boolean writeAccess, String auxIt // object! throw new IOException("IO driver mismatch: SwiftAccessIO called on a non-swift stored object."); } else if (this.isWriteAccess) { - Properties p = getSwiftProperties(); - swiftEndPoint = p.getProperty("swift.default.endpoint"); - String swiftFolderPathSeparator = "-"; + swiftEndPoint = swiftDefaultEndpoint; // Swift uses this to create pseudo-hierarchical folders String swiftPseudoFolderPathSeparator = "/"; @@ -598,7 +596,7 @@ private StoredObject initializeSwiftFileObject(boolean writeAccess, String auxIt swiftPseudoFolderPathSeparator + dataset.getIdentifierForFileStorage(); swiftFileName = auxItemTag; - dvObject.setStorageIdentifier("swift://" + swiftEndPoint + ":" + swiftFolderPath); + dvObject.setStorageIdentifier(SWIFT_IDENTIFIER_PREFIX + "://" + swiftEndPoint + ":" + swiftFolderPath); } else { throw new IOException("SwiftAccessIO: unknown access mode."); } @@ -625,7 +623,7 @@ private StoredObject initializeSwiftFileObject(boolean writeAccess, String auxIt other swiftContainerName Object Store pseudo-folder can be created, which is not provide by the joss Java swift library as of yet. */ - if (storageIdentifier.startsWith("swift://")) { + if (storageIdentifier.startsWith(SWIFT_IDENTIFIER_PREFIX + "://")) { // An existing swift object; the container must already exist as well. this.swiftContainer = account.getContainer(swiftContainerName); } else { @@ -636,7 +634,7 @@ private StoredObject initializeSwiftFileObject(boolean writeAccess, String auxIt if (writeAccess) { //creates a private data container swiftContainer.create(); - if (publicSwiftContainer) { + if (isPublicContainer) { try { //creates a public data container this.swiftContainer.makePublic(); @@ -733,29 +731,13 @@ private StoredObject openSwiftAuxFile(boolean writeAccess, String auxItemTag) th return initializeSwiftFileObject(writeAccess, auxItemTag); } - private Properties getSwiftProperties() throws IOException { - if (swiftProperties == null) { - String domainRoot = System.getProperties().getProperty("com.sun.aas.instanceRoot"); - String swiftPropertiesFile = domainRoot + File.separator + "config" + File.separator + "swift.properties"; - swiftProperties = new Properties(); - swiftProperties.load(new FileInputStream(new File(swiftPropertiesFile))); - } - - return swiftProperties; - } - Account authenticateWithSwift(String swiftEndPoint) throws IOException { - - Properties p = getSwiftProperties(); - - // (this will throw an IOException, if the swift properties file - // is missing or corrupted) - String swiftEndPointAuthUrl = p.getProperty("swift.auth_url." + swiftEndPoint); - String swiftEndPointUsername = p.getProperty("swift.username." + swiftEndPoint); - String swiftEndPointSecretKey = p.getProperty("swift.password." + swiftEndPoint); - String swiftEndPointTenantName = p.getProperty("swift.tenant." + swiftEndPoint); - String swiftEndPointAuthMethod = p.getProperty("swift.auth_type." + swiftEndPoint); - String swiftEndPointTenantId = p.getProperty("swift.tenant_id." + swiftEndPoint); + String swiftEndPointAuthUrl = System.getProperty("dataverse.files.swift.authUrl." + swiftEndPoint); + String swiftEndPointUsername = System.getProperty("dataverse.files.swift.username." + swiftEndPoint); + String swiftEndPointSecretKey = System.getProperty("dataverse.files.swift.password." + swiftEndPoint); + String swiftEndPointTenantName = System.getProperty("dataverse.files.swift.tenant." + swiftEndPoint); + String swiftEndPointAuthMethod = System.getProperty("dataverse.files.swift.authType." + swiftEndPoint); + String swiftEndPointTenantId = System.getProperty("dataverse.files.swift.tenant." + swiftEndPoint); if (swiftEndPointAuthUrl == null || swiftEndPointUsername == null || swiftEndPointSecretKey == null || "".equals(swiftEndPointAuthUrl) || "".equals(swiftEndPointUsername) || "".equals(swiftEndPointSecretKey)) { @@ -824,10 +806,9 @@ private String getSwiftFileURI(StoredObject fileObject) throws IOException { private String hmac = null; public String generateTempUrlSignature(String swiftEndPoint, String containerName, String objectName, int duration) throws IOException { if (hmac == null || isExpiryExpired(generateTempUrlExpiry(duration, System.currentTimeMillis()), duration, System.currentTimeMillis())) { - Properties p = getSwiftProperties(); - String secretKey = p.getProperty("swift.hash_key." + swiftEndPoint); + String secretKey = System.getProperty("dataverse.files.swift.hashKey." + swiftEndPoint); if (secretKey == null) { - throw new IOException("Please input a hash key in swift.properties"); + throw new IOException("Please input a hash key under dataverse.files.swift.hashKey." + swiftEndPoint); } String path = "/v1/" + containerName + "/" + objectName; Long expires = generateTempUrlExpiry(duration, System.currentTimeMillis()); @@ -852,8 +833,7 @@ public long generateTempUrlExpiry(int duration, long currentTime) { private String temporaryUrl = null; private String generateTemporarySwiftUrl(String swiftEndPoint, String containerName, String objectName, int duration) throws IOException { - Properties p = getSwiftProperties(); - String baseUrl = p.getProperty("swift.swift_endpoint." + swiftEndPoint); + String baseUrl = System.getProperty("dataverse.files.swift.endpoint." + swiftEndPoint); String path = "/v1/" + containerName + "/" + objectName; if (temporaryUrl == null || isExpiryExpired(generateTempUrlExpiry(duration, System.currentTimeMillis()), duration, System.currentTimeMillis())) { @@ -881,10 +861,6 @@ public InputStream getAuxFileAsInputStream(String auxItemTag) throws IOException @Override public String getSwiftContainerName() { - String swiftFolderPathSeparator = System.getProperty("dataverse.files.swift-folder-path-separator"); - if (swiftFolderPathSeparator == null) { - swiftFolderPathSeparator = "_"; - } if (dvObject instanceof DataFile) { String authorityNoSlashes = this.getDataFile().getOwner().getAuthorityForFileStorage().replace("/", swiftFolderPathSeparator); return this.getDataFile().getOwner().getProtocolForFileStorage() + swiftFolderPathSeparator