diff --git a/.gitignore b/.gitignore index e638a09ae210..59902bc26542 100644 --- a/.gitignore +++ b/.gitignore @@ -104,3 +104,4 @@ scripts/.pydevproject venv node_modules .vscode +!/plugins/storage/object/huawei-obs/local-huawei-sdk/com/huawei/storage/esdk-obs-java/3.23.9/esdk-obs-java-3.23.9.jar diff --git a/client/pom.xml b/client/pom.xml index 91399097d648..6dbe49446493 100644 --- a/client/pom.xml +++ b/client/pom.xml @@ -652,6 +652,11 @@ cloud-plugin-storage-object-minio ${project.version} + + org.apache.cloudstack + cloud-plugin-storage-object-huawei-obs + ${project.version} + org.apache.cloudstack cloud-plugin-storage-object-simulator diff --git a/plugins/pom.xml b/plugins/pom.xml index cbfba8f82177..c73495b2342d 100755 --- a/plugins/pom.xml +++ b/plugins/pom.xml @@ -138,6 +138,7 @@ storage/volume/primera storage/object/minio storage/object/simulator + storage/object/huawei-obs storage-allocators/random @@ -155,7 +156,7 @@ - + org.apache.cloudstack cloud-server ${project.version} diff --git a/plugins/storage/object/huawei-obs/local-huawei-sdk/com/huawei/storage/esdk-obs-java/3.23.9/esdk-obs-java-3.23.9.jar b/plugins/storage/object/huawei-obs/local-huawei-sdk/com/huawei/storage/esdk-obs-java/3.23.9/esdk-obs-java-3.23.9.jar new file mode 100644 index 000000000000..b8c5480dc84d Binary files /dev/null and b/plugins/storage/object/huawei-obs/local-huawei-sdk/com/huawei/storage/esdk-obs-java/3.23.9/esdk-obs-java-3.23.9.jar differ diff --git a/plugins/storage/object/huawei-obs/local-huawei-sdk/com/huawei/storage/esdk-obs-java/3.23.9/esdk-obs-java-3.23.9.pom b/plugins/storage/object/huawei-obs/local-huawei-sdk/com/huawei/storage/esdk-obs-java/3.23.9/esdk-obs-java-3.23.9.pom new file mode 100644 index 000000000000..618690e01cc0 --- /dev/null +++ b/plugins/storage/object/huawei-obs/local-huawei-sdk/com/huawei/storage/esdk-obs-java/3.23.9/esdk-obs-java-3.23.9.pom @@ -0,0 +1,9 @@ + + + 4.0.0 + com.huawei.storage + esdk-obs-java + 3.23.9 + POM was created from install:install-file + diff --git a/plugins/storage/object/huawei-obs/local-huawei-sdk/com/huawei/storage/esdk-obs-java/maven-metadata-local.xml b/plugins/storage/object/huawei-obs/local-huawei-sdk/com/huawei/storage/esdk-obs-java/maven-metadata-local.xml new file mode 100644 index 000000000000..a8576018874b --- /dev/null +++ b/plugins/storage/object/huawei-obs/local-huawei-sdk/com/huawei/storage/esdk-obs-java/maven-metadata-local.xml @@ -0,0 +1,12 @@ + + + com.huawei.storage + esdk-obs-java + + 3.23.9 + + 3.23.9 + + 20240130160938 + + diff --git a/plugins/storage/object/huawei-obs/pom.xml b/plugins/storage/object/huawei-obs/pom.xml new file mode 100644 index 000000000000..a9bc51ce3724 --- /dev/null +++ b/plugins/storage/object/huawei-obs/pom.xml @@ -0,0 +1,78 @@ + + 4.0.0 + cloud-plugin-storage-object-huawei-obs + Apache CloudStack Plugin - Huawei OBS object storage provider + + org.apache.cloudstack + cloudstack-plugins + 4.19.0.0-SNAPSHOT + ../../../pom.xml + + + + + local-huawei-sdk + file:///${project.basedir}/local-huawei-sdk + + + + + + org.apache.cloudstack + cloud-engine-storage + ${project.version} + + + org.apache.cloudstack + cloud-engine-storage-object + ${project.version} + + + org.apache.cloudstack + cloud-engine-schema + ${project.version} + + + com.huawei.storage + esdk-obs-java + 3.23.9 + + + com.mikesamuel + json-sanitizer + 1.2.2 + + + com.jamesmurty.utils + java-xmlbuilder + 1.3 + + + net.iharder + base64 + + + + + com.squareup.okhttp3 + okhttp + 4.10.0 + + + com.squareup.okio + okio + 3.0.0 + + + com.huaweicloud.sdk + huaweicloud-sdk-iam + 3.1.69 + + + org.bouncycastle + bcprov-jdk18on + + + + + diff --git a/plugins/storage/object/huawei-obs/src/main/java/org/apache/cloudstack/storage/datastore/driver/HuaweiObsObjectStoreDriverImpl.java b/plugins/storage/object/huawei-obs/src/main/java/org/apache/cloudstack/storage/datastore/driver/HuaweiObsObjectStoreDriverImpl.java new file mode 100644 index 000000000000..a3fc6cd1bb43 --- /dev/null +++ b/plugins/storage/object/huawei-obs/src/main/java/org/apache/cloudstack/storage/datastore/driver/HuaweiObsObjectStoreDriverImpl.java @@ -0,0 +1,453 @@ +package org.apache.cloudstack.storage.datastore.driver; + +import com.amazonaws.services.s3.model.AccessControlList; +import com.amazonaws.services.s3.model.BucketPolicy; +import com.amazonaws.services.s3.model.CanonicalGrantee; +import com.amazonaws.services.s3.model.Grant; +import com.amazonaws.services.s3.model.Grantee; +import com.amazonaws.services.s3.model.GroupGrantee; +import com.amazonaws.services.s3.model.Owner; +import com.amazonaws.services.s3.model.Permission; +import com.cloud.agent.api.to.DataStoreTO; +import com.cloud.storage.BucketVO; +import com.cloud.storage.dao.BucketDao; +import com.cloud.user.Account; +import com.cloud.user.AccountDetailsDao; +import com.cloud.user.dao.AccountDao; +import com.cloud.utils.exception.CloudRuntimeException; +import com.huaweicloud.sdk.core.HttpListener; +import com.huaweicloud.sdk.core.auth.BasicCredentials; +import com.huaweicloud.sdk.core.http.HttpConfig; +import com.huaweicloud.sdk.iam.v3.IamClient; +import com.huaweicloud.sdk.iam.v3.model.CreateCredentialOption; +import com.huaweicloud.sdk.iam.v3.model.CreatePermanentAccessKeyRequest; +import com.huaweicloud.sdk.iam.v3.model.CreatePermanentAccessKeyRequestBody; +import com.huaweicloud.sdk.iam.v3.model.CreatePermanentAccessKeyResponse; +import com.huaweicloud.sdk.iam.v3.model.CreateUserOption; +import com.huaweicloud.sdk.iam.v3.model.CreateUserRequest; +import com.huaweicloud.sdk.iam.v3.model.CreateUserRequestBody; +import com.huaweicloud.sdk.iam.v3.model.ShowUserRequest; +import com.huaweicloud.sdk.iam.v3.model.ShowUserResult; +import com.huaweicloud.sdk.iam.v3.model.UpdateUserOption; +import com.huaweicloud.sdk.iam.v3.model.UpdateUserRequest; +import com.huaweicloud.sdk.iam.v3.model.UpdateUserRequestBody; +import com.obs.services.ObsClient; +import com.obs.services.model.BucketEncryption; +import com.obs.services.model.BucketQuota; +import com.obs.services.model.BucketStorageInfo; +import com.obs.services.model.BucketVersioningConfiguration; +import com.obs.services.model.CreateBucketRequest; +import com.obs.services.model.GrantAndPermission; +import com.obs.services.model.GranteeInterface; +import com.obs.services.model.ListBucketsRequest; +import com.obs.services.model.ObjectListing; +import com.obs.services.model.ObsBucket; +import com.obs.services.model.SSEAlgorithmEnum; +import com.obs.services.model.VersioningStatusEnum; +import java.net.URI; +import org.apache.cloudstack.engine.subsystem.api.storage.DataStore; +import org.apache.cloudstack.storage.datastore.db.ObjectStoreDao; +import org.apache.cloudstack.storage.datastore.db.ObjectStoreDetailsDao; +import org.apache.cloudstack.storage.object.BaseObjectStoreDriverImpl; +import org.apache.cloudstack.storage.object.Bucket; +import org.apache.cloudstack.storage.object.BucketObject; +import org.apache.commons.codec.binary.Base64; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.stream.Collectors; +import javax.crypto.KeyGenerator; +import javax.crypto.SecretKey; +import javax.inject.Inject; +import org.apache.cloudstack.storage.datastore.db.ObjectStoreVO; + +public class HuaweiObsObjectStoreDriverImpl extends BaseObjectStoreDriverImpl { + + @Inject + AccountDao _accountDao; + @Inject + AccountDetailsDao _accountDetailsDao; + @Inject + ObjectStoreDao _storeDao; + @Inject + BucketDao _bucketDao; + @Inject + ObjectStoreDetailsDao _storeDetailsDao; + + private static final String ACCESS_KEY = "accesskey"; + private static final String SECRET_KEY = "secretkey"; + private static final String OBS_ACCESS_KEY = "huawei-obs-accesskey"; + private static final String OBS_SECRET_KEY = "huawei-obs-secretkey"; + + @Override + public DataStoreTO getStoreTO(DataStore store) { + return null; + } + + @Override + public Bucket createBucket(Bucket bucket, boolean objectLock) { + long accountId = bucket.getAccountId(); + long storeId = bucket.getObjectStoreId(); + Account account = _accountDao.findById(accountId); + + if ((_accountDetailsDao.findDetail(accountId, OBS_ACCESS_KEY) == null) || (_accountDetailsDao.findDetail(accountId, OBS_SECRET_KEY) == null)) { + throw new CloudRuntimeException("Bucket access credentials unavailable for account: " + account.getAccountName()); + } + + try (ObsClient obsClient = getObsClient(storeId)) { + String bucketName = bucket.getName(); + + if (obsClient.headBucket(bucketName)) { + throw new CloudRuntimeException("A bucket with the name " + bucketName + " already exists"); + } + + CreateBucketRequest createBucketRequest = new CreateBucketRequest(bucketName); + createBucketRequest.setAcl(com.obs.services.model.AccessControlList.REST_CANNED_PUBLIC_READ_WRITE); + obsClient.createBucket(createBucketRequest); + + BucketVO bucketVO = _bucketDao.findById(bucket.getId()); + String accountAccessKey = _accountDetailsDao.findDetail(accountId, OBS_ACCESS_KEY).getValue(); + String accountSecretKey = _accountDetailsDao.findDetail(accountId, OBS_SECRET_KEY).getValue(); + String endpoint = _storeDao.findById(storeId).getUrl(); + String scheme = new URI(endpoint).getScheme() + "://"; + String everythingelse = endpoint.substring(scheme.length()); + bucketVO.setAccessKey(accountAccessKey); + bucketVO.setSecretKey(accountSecretKey); + bucketVO.setBucketURL(scheme + bucketName + "." + everythingelse); + _bucketDao.update(bucket.getId(), bucketVO); + return bucket; + } catch (Exception ex) { + throw new CloudRuntimeException(ex); + } + } + + @Override + public List listBuckets(long storeId) { + List bucketsList = new ArrayList<>(); + try (ObsClient obsClient = getObsClient(storeId)) { + ListBucketsRequest request = new ListBucketsRequest(); + for (ObsBucket obsBucket : obsClient.listBuckets(request)) { + Bucket bucket = new BucketObject(); + bucket.setName(obsBucket.getBucketName()); + bucketsList.add(bucket); + } + } catch (Exception ex) { + throw new CloudRuntimeException(ex); + } + return bucketsList; + } + + @Override + public boolean deleteBucket(String bucketName, long storeId) { + try (ObsClient obsClient = getObsClient(storeId)) { + + if (!obsClient.headBucket(bucketName)) { + throw new CloudRuntimeException("Bucket does not exist: " + bucketName); + } + + ObjectListing objectListing = obsClient.listObjects(bucketName); + if (objectListing == null || objectListing.getObjects().isEmpty()) { + obsClient.deleteBucket(bucketName); + } else { + throw new CloudRuntimeException("Bucket " + bucketName + " cannot be deleted because it is not empty"); + } + } catch (Exception ex) { + throw new CloudRuntimeException(ex); + } + return true; + } + + @Override + public AccessControlList getBucketAcl(String bucketName, long storeId) { + AccessControlList accessControlList = new AccessControlList(); + try (ObsClient obsClient = getObsClient(storeId)) { + com.obs.services.model.AccessControlList obsAccessControlList = obsClient.getBucketAcl(bucketName); + com.obs.services.model.Owner obsOwner = obsAccessControlList.getOwner(); + Owner owner = new Owner(obsOwner.getId(), obsOwner.getDisplayName()); + accessControlList.setOwner(owner); + for (GrantAndPermission grantAndPermission : obsAccessControlList.getGrantAndPermissions()) { + com.obs.services.model.Permission obsPermission = grantAndPermission.getPermission(); + Permission permission = castPermission(obsPermission); + GranteeInterface granteeInterface = grantAndPermission.getGrantee(); + if (granteeInterface instanceof com.obs.services.model.CanonicalGrantee) { + Grantee grantee = new CanonicalGrantee(granteeInterface.getIdentifier()); + accessControlList.grantPermission(grantee, permission); + } else if (granteeInterface instanceof com.obs.services.model.GroupGrantee) { + com.obs.services.model.GroupGrantee obsGroupGrantee = (com.obs.services.model.GroupGrantee) granteeInterface; + if (obsGroupGrantee.getGroupGranteeType() == com.obs.services.model.GroupGranteeEnum.ALL_USERS) { + accessControlList.grantPermission(GroupGrantee.AllUsers, permission); + } else if (obsGroupGrantee.getGroupGranteeType() == com.obs.services.model.GroupGranteeEnum.LOG_DELIVERY) { + accessControlList.grantPermission(GroupGrantee.LogDelivery, permission); + } else if (obsGroupGrantee.getGroupGranteeType() == com.obs.services.model.GroupGranteeEnum.AUTHENTICATED_USERS) { + accessControlList.grantPermission(GroupGrantee.AuthenticatedUsers, permission); + } + } + } + } catch (Exception ex) { + throw new CloudRuntimeException(ex); + } + return accessControlList; + } + + private Permission castPermission(com.obs.services.model.Permission obsPermission) { + if (com.obs.services.model.Permission.PERMISSION_FULL_CONTROL == obsPermission) { + return Permission.FullControl; + } else if (com.obs.services.model.Permission.PERMISSION_READ == obsPermission) { + return Permission.Read; + } else if (com.obs.services.model.Permission.PERMISSION_READ_ACP == obsPermission) { + return Permission.ReadAcp; + } else if (com.obs.services.model.Permission.PERMISSION_WRITE == obsPermission) { + return Permission.Write; + } else if (com.obs.services.model.Permission.PERMISSION_WRITE_ACP == obsPermission) { + return Permission.WriteAcp; + } + return Permission.FullControl; + } + + private com.obs.services.model.Permission castPermission(Permission permission) { + if (Permission.FullControl == permission) { + return com.obs.services.model.Permission.PERMISSION_FULL_CONTROL; + } else if (Permission.Read == permission) { + return com.obs.services.model.Permission.PERMISSION_READ; + } else if (Permission.ReadAcp == permission) { + return com.obs.services.model.Permission.PERMISSION_READ_ACP; + } else if (Permission.Write == permission) { + return com.obs.services.model.Permission.PERMISSION_WRITE; + } else if (Permission.WriteAcp == permission) { + return com.obs.services.model.Permission.PERMISSION_WRITE_ACP; + } + return com.obs.services.model.Permission.PERMISSION_FULL_CONTROL; + } + + @Override + public void setBucketAcl(String bucketName, AccessControlList accessControlList, long storeId) { + com.obs.services.model.AccessControlList obsAccessControlList = new com.obs.services.model.AccessControlList(); + Owner owner = accessControlList.getOwner(); + com.obs.services.model.Owner obsOwner = new com.obs.services.model.Owner(); + obsOwner.setId(owner.getId()); + obsOwner.setDisplayName(owner.getDisplayName()); + obsAccessControlList.setOwner(obsOwner); + for (Grant grant : accessControlList.getGrantsAsList()) { + if (grant.getGrantee() instanceof CanonicalGrantee) { + com.obs.services.model.CanonicalGrantee canonicalGrantee = new com.obs.services.model.CanonicalGrantee(grant.getGrantee().getIdentifier()); + obsAccessControlList.grantPermission(canonicalGrantee, castPermission(grant.getPermission())); + } else if (grant.getGrantee() instanceof GroupGrantee) { + GroupGrantee groupGrantee = (GroupGrantee) grant.getGrantee(); + if (GroupGrantee.AllUsers == groupGrantee) { + obsAccessControlList.grantPermission(com.obs.services.model.GroupGrantee.ALL_USERS, castPermission(grant.getPermission())); + } else if (GroupGrantee.LogDelivery == groupGrantee) { + obsAccessControlList.grantPermission(com.obs.services.model.GroupGrantee.LOG_DELIVERY, castPermission(grant.getPermission())); + } else if (GroupGrantee.AuthenticatedUsers == groupGrantee) { + obsAccessControlList.grantPermission(com.obs.services.model.GroupGrantee.AUTHENTICATED_USERS, castPermission(grant.getPermission())); + } + + } + } + try (ObsClient obsClient = getObsClient(storeId)) { + obsClient.setBucketAcl(bucketName, obsAccessControlList); + } catch (Exception ex) { + throw new CloudRuntimeException(ex); + } + } + + @Override + public void setBucketPolicy(String bucketName, String policy, long storeId) { + if (policy.equalsIgnoreCase("public") || policy.equalsIgnoreCase("private")) { + StringBuilder publicPolicyBuilder = new StringBuilder(); + publicPolicyBuilder.append("{\n"); + publicPolicyBuilder.append(" \"Statement\": [\n"); + publicPolicyBuilder.append(" {\n"); + if (policy.equalsIgnoreCase("public")) { + publicPolicyBuilder.append(" \"Effect\": \"Allow\",\n"); + } else if (policy.equalsIgnoreCase("private")) { + publicPolicyBuilder.append(" \"Effect\": \"Deny\",\n"); + } + publicPolicyBuilder.append(" \"Action\": \"*\",\n"); + publicPolicyBuilder.append(" \"Principal\": \"*\",\n"); + publicPolicyBuilder.append(" \"Resource\": [\"arn:aws:s3:::").append(bucketName).append("/*\"]\n"); + publicPolicyBuilder.append(" }\n"); + publicPolicyBuilder.append(" ]\n"); + publicPolicyBuilder.append("}\n"); + policy = publicPolicyBuilder.toString(); + } + + try (ObsClient obsClient = getObsClient(storeId)) { + obsClient.setBucketPolicy(bucketName, policy); + } catch (Exception ex) { + throw new CloudRuntimeException(ex); + } + } + + @Override + public BucketPolicy getBucketPolicy(String bucketName, long storeId) { + try (ObsClient obsClient = getObsClient(storeId)) { + String policy = obsClient.getBucketPolicy(bucketName); + BucketPolicy bucketPolicy = new BucketPolicy(); + bucketPolicy.setPolicyText(policy); + return bucketPolicy; + } catch (Exception ex) { + throw new CloudRuntimeException(ex); + } + } + + @Override + public void deleteBucketPolicy(String bucketName, long storeId) { + try (ObsClient obsClient = getObsClient(storeId)) { + obsClient.deleteBucketPolicy(bucketName); + } catch (Exception ex) { + throw new CloudRuntimeException(ex); + } + } + + @Override + public boolean setBucketEncryption(String bucketName, long storeId) { + try (ObsClient obsClient = getObsClient(storeId)) { + BucketEncryption bucketEncryption = new BucketEncryption(SSEAlgorithmEnum.KMS); + obsClient.setBucketEncryption(bucketName, bucketEncryption); + } catch (Exception ex) { + throw new CloudRuntimeException(ex); + } + return true; + } + + @Override + public boolean deleteBucketEncryption(String bucketName, long storeId) { + try (ObsClient obsClient = getObsClient(storeId)) { + obsClient.deleteBucketEncryption(bucketName); + } catch (Exception ex) { + throw new CloudRuntimeException(ex); + } + return true; + } + + @Override + public boolean setBucketVersioning(String bucketName, long storeId) { + try (ObsClient obsClient = getObsClient(storeId)) { + BucketVersioningConfiguration bucketVersioningConfiguration = new BucketVersioningConfiguration(VersioningStatusEnum.ENABLED); + obsClient.setBucketVersioning(bucketName, bucketVersioningConfiguration); + } catch (Exception ex) { + throw new CloudRuntimeException(ex); + } + return true; + } + + @Override + public boolean deleteBucketVersioning(String bucketName, long storeId) { + try (ObsClient obsClient = getObsClient(storeId)) { + BucketVersioningConfiguration bucketVersioningConfiguration = new BucketVersioningConfiguration(VersioningStatusEnum.SUSPENDED); + obsClient.setBucketVersioning(bucketName, bucketVersioningConfiguration); + } catch (Exception ex) { + throw new CloudRuntimeException(ex); + } + return true; + } + + @Override + public void setBucketQuota(String bucketName, long storeId, long size) { + try (ObsClient obsClient = getObsClient(storeId)) { + BucketQuota quota = new BucketQuota(); + quota.setBucketQuota(size); + obsClient.setBucketQuota(bucketName, quota); + } catch (Exception ex) { + throw new CloudRuntimeException(ex); + } + } + + @Override + public Map getAllBucketsUsage(long storeId) { + Map allBucketsUsage = new HashMap<>(); + try (ObsClient obsClient = getObsClient(storeId)) { + for (Bucket bucket : listBuckets(storeId)) { + String bucketName = bucket.getName(); + BucketStorageInfo storageInfo = obsClient.getBucketStorageInfo(bucketName); + allBucketsUsage.put(bucketName, storageInfo.getSize()); + } + } catch (Exception ex) { + throw new CloudRuntimeException(ex); + } + return allBucketsUsage; + } + + @Override + public boolean createUser(long accountId, long storeId) { + Account account = _accountDao.findById(accountId); + String username = account.getAccountName(); + Map storeDetails = _storeDetailsDao.getDetails(storeId); + String endpoint = _storeDao.findById(storeId).getUrl(); + String clientAccessKey = storeDetails.get(ACCESS_KEY); + String clientSecretKey = storeDetails.get(SECRET_KEY); + + try { + HttpConfig httpConfig = HttpConfig.getDefaultHttpConfig(); + httpConfig.setIgnoreSSLVerification(true); + HttpListener requestListener = HttpListener.forRequestListener(listener + -> System.out.printf("> Request %s %s\n> Headers:\n%s\n> Body: %s\n", + listener.httpMethod(), + listener.uri(), + listener.headers().entrySet().stream() + .flatMap(entry -> entry.getValue().stream().map( + value -> "\t" + entry.getKey() + ": " + value)) + .collect(Collectors.joining("\n")), + listener.body().orElse(""))); + httpConfig.addHttpListener(requestListener); + HttpListener responseListener = HttpListener.forResponseListener(listener + -> System.out.printf("< Response %s %s %s\n< Headers:\n%s\n< Body: %s\n", + listener.httpMethod(), + listener.uri(), + listener.statusCode(), + listener.headers().entrySet().stream() + .flatMap(entry -> entry.getValue().stream().map( + value -> "\t" + entry.getKey() + ": " + value)) + .collect(Collectors.joining("\n")), + listener.body().orElse(""))); + httpConfig.addHttpListener(responseListener); + + BasicCredentials basicCredentials = new BasicCredentials().withAk(clientAccessKey).withSk(clientSecretKey).withIamEndpoint(endpoint); + List endpoints = new ArrayList<>(); + endpoints.add(basicCredentials.getIamEndpoint()); + IamClient iamClient = IamClient.newBuilder().withEndpoints(endpoints).withCredential(basicCredentials).withHttpConfig(httpConfig).build(); + ShowUserRequest showUserRequest = new ShowUserRequest().withUserId(username); + ShowUserResult showUserResult = iamClient.showUser(showUserRequest).getUser(); + if (showUserResult == null || showUserResult.getPwdStatus()) { + KeyGenerator generator = KeyGenerator.getInstance("HmacSHA1"); + SecretKey key = generator.generateKey(); + String secretKey = Base64.encodeBase64URLSafeString(key.getEncoded()); + CreateUserOption createUserOption = new CreateUserOption().withName(username).withPassword(secretKey).withEnabled(Boolean.TRUE); + CreateUserRequestBody createUserRequestBody = new CreateUserRequestBody().withUser(createUserOption); + CreateUserRequest createUserRequest = new CreateUserRequest().withBody(createUserRequestBody); + iamClient.createUser(createUserRequest); + CreateCredentialOption createCredentialOption = new CreateCredentialOption().withUserId(username); + CreatePermanentAccessKeyRequestBody createPermanentAccessKeyRequestBody = new CreatePermanentAccessKeyRequestBody().withCredential(createCredentialOption); + CreatePermanentAccessKeyRequest createPermanentAccessKeyRequest = new CreatePermanentAccessKeyRequest().withBody(createPermanentAccessKeyRequestBody); + CreatePermanentAccessKeyResponse createPermanentAccessKeyResponse = iamClient.createPermanentAccessKey(createPermanentAccessKeyRequest); + String accessKey = createPermanentAccessKeyResponse.getCredential().getAccess(); + String secret = createPermanentAccessKeyResponse.getCredential().getSecret(); + String status = createPermanentAccessKeyResponse.getCredential().getStatus(); + + // Store user credentials + Map details = new HashMap<>(); + details.put(OBS_ACCESS_KEY, accessKey); + details.put(OBS_SECRET_KEY, secretKey); + _accountDetailsDao.persist(accountId, details); + } else if (!showUserResult.getEnabled()) { + UpdateUserOption updateUserOption = new UpdateUserOption().withName(clientAccessKey).withEnabled(Boolean.TRUE); + UpdateUserRequestBody updateUserRequestBody = new UpdateUserRequestBody().withUser(updateUserOption); + UpdateUserRequest updateUserRequest = new UpdateUserRequest().withBody(updateUserRequestBody); + iamClient.updateUser(updateUserRequest); + } + } catch (Exception ex) { + throw new CloudRuntimeException(ex); + } + return true; + } + + protected ObsClient getObsClient(long storeId) { + ObjectStoreVO store = _storeDao.findById(storeId); + String endpoint = store.getUrl(); + Map storeDetails = _storeDetailsDao.getDetails(storeId); + String clientAccessKey = storeDetails.get(ACCESS_KEY); + String clientSecretKey = storeDetails.get(SECRET_KEY); + return new ObsClient(clientAccessKey, clientSecretKey, endpoint); + } +} diff --git a/plugins/storage/object/huawei-obs/src/main/java/org/apache/cloudstack/storage/datastore/lifecycle/HuaweiObsObjectStoreLifeCycleImpl.java b/plugins/storage/object/huawei-obs/src/main/java/org/apache/cloudstack/storage/datastore/lifecycle/HuaweiObsObjectStoreLifeCycleImpl.java new file mode 100644 index 000000000000..bf935311ef05 --- /dev/null +++ b/plugins/storage/object/huawei-obs/src/main/java/org/apache/cloudstack/storage/datastore/lifecycle/HuaweiObsObjectStoreLifeCycleImpl.java @@ -0,0 +1,111 @@ +package org.apache.cloudstack.storage.datastore.lifecycle; + +import com.cloud.agent.api.StoragePoolInfo; +import com.cloud.hypervisor.Hypervisor.HypervisorType; +import com.cloud.utils.exception.CloudRuntimeException; +import com.obs.services.ObsClient; +import com.obs.services.model.ListBucketsRequest; +import org.apache.cloudstack.engine.subsystem.api.storage.ClusterScope; +import org.apache.cloudstack.engine.subsystem.api.storage.DataStore; +import org.apache.cloudstack.engine.subsystem.api.storage.HostScope; +import org.apache.cloudstack.engine.subsystem.api.storage.ZoneScope; +import org.apache.cloudstack.storage.datastore.db.ObjectStoreVO; +import org.apache.cloudstack.storage.object.datastore.ObjectStoreHelper; +import org.apache.cloudstack.storage.object.datastore.ObjectStoreProviderManager; +import org.apache.cloudstack.storage.object.store.lifecycle.ObjectStoreLifeCycle; +import org.apache.log4j.Logger; + +import javax.inject.Inject; +import java.util.HashMap; +import java.util.Map; + +public class HuaweiObsObjectStoreLifeCycleImpl implements ObjectStoreLifeCycle { + + private static final Logger LOG = Logger.getLogger(HuaweiObsObjectStoreLifeCycleImpl.class); + + @Inject + ObjectStoreHelper objectStoreHelper; + @Inject + ObjectStoreProviderManager objectStoreMgr; + + public HuaweiObsObjectStoreLifeCycleImpl() { + } + + @SuppressWarnings("unchecked") + @Override + public DataStore initialize(Map dsInfos) { + + String url = (String) dsInfos.get("url"); + String name = (String) dsInfos.get("name"); + String providerName = (String) dsInfos.get("providerName"); + Map details = (Map) dsInfos.get("details"); + if (details == null) { + throw new CloudRuntimeException("Huawei OBS credentials are missing"); + } + String accessKey = details.get("accesskey"); + String secretKey = details.get("secretkey"); + + Map objectStoreParameters = new HashMap(); + objectStoreParameters.put("name", name); + objectStoreParameters.put("url", url); + + objectStoreParameters.put("providerName", providerName); + objectStoreParameters.put("accesskey", accessKey); + objectStoreParameters.put("secretkey", secretKey); + + try { + //check credentials + ObsClient obsClient = new ObsClient(accessKey, secretKey, url); + // Test connection by listing buckets + ListBucketsRequest request = new ListBucketsRequest(); + request.setQueryLocation(true); + obsClient.listBucketsV2(request); + LOG.debug("Successfully connected to Huawei OBS EndPoint: " + url); + } catch (Exception ex) { + LOG.debug("Error while initializing Huawei OBS Object Store: " + ex.getMessage()); + throw new RuntimeException("Error while initializing Huawei OBS Object Store. Invalid credentials or endpoint URL"); + } + + ObjectStoreVO objectStore = objectStoreHelper.createObjectStore(objectStoreParameters, details); + return objectStoreMgr.getObjectStore(objectStore.getId()); + } + + @Override + public boolean attachCluster(DataStore store, ClusterScope scope) { + return false; + } + + @Override + public boolean attachHost(DataStore store, HostScope scope, StoragePoolInfo existingInfo) { + return false; + } + + @Override + public boolean attachZone(DataStore dataStore, ZoneScope scope, HypervisorType hypervisorType) { + return false; + } + + @Override + public boolean maintain(DataStore store) { + return false; + } + + @Override + public boolean cancelMaintain(DataStore store) { + return false; + } + + @Override + public boolean deleteDataStore(DataStore store) { + return false; + } + + /* (non-Javadoc) + * @see org.apache.cloudstack.engine.subsystem.api.storage.DataStoreLifeCycle#migrateToObjectStore(org.apache.cloudstack.engine.subsystem.api.storage.DataStore) + */ + @Override + public boolean migrateToObjectStore(DataStore store) { + return false; + } + +} diff --git a/plugins/storage/object/huawei-obs/src/main/java/org/apache/cloudstack/storage/datastore/provider/HuaweiObsObjectStoreProviderImpl.java b/plugins/storage/object/huawei-obs/src/main/java/org/apache/cloudstack/storage/datastore/provider/HuaweiObsObjectStoreProviderImpl.java new file mode 100644 index 000000000000..ac1ad5e46169 --- /dev/null +++ b/plugins/storage/object/huawei-obs/src/main/java/org/apache/cloudstack/storage/datastore/provider/HuaweiObsObjectStoreProviderImpl.java @@ -0,0 +1,64 @@ +package org.apache.cloudstack.storage.datastore.provider; + +import com.cloud.utils.component.ComponentContext; +import org.apache.cloudstack.engine.subsystem.api.storage.DataStoreDriver; +import org.apache.cloudstack.engine.subsystem.api.storage.DataStoreLifeCycle; +import org.apache.cloudstack.engine.subsystem.api.storage.HypervisorHostListener; +import org.apache.cloudstack.engine.subsystem.api.storage.ObjectStoreProvider; +import org.apache.cloudstack.storage.datastore.driver.HuaweiObsObjectStoreDriverImpl; +import org.apache.cloudstack.storage.datastore.lifecycle.HuaweiObsObjectStoreLifeCycleImpl; +import org.apache.cloudstack.storage.object.ObjectStoreDriver; +import org.apache.cloudstack.storage.object.datastore.ObjectStoreProviderManager; +import org.apache.cloudstack.storage.object.store.lifecycle.ObjectStoreLifeCycle; +import org.springframework.stereotype.Component; + +import javax.inject.Inject; +import java.util.HashSet; +import java.util.Map; +import java.util.Set; + +@Component +public class HuaweiObsObjectStoreProviderImpl implements ObjectStoreProvider { + + @Inject + ObjectStoreProviderManager storeMgr; + + private final String providerName = "Huawei OBS"; + protected ObjectStoreLifeCycle lifeCycle; + protected ObjectStoreDriver driver; + + @Override + public DataStoreLifeCycle getDataStoreLifeCycle() { + return lifeCycle; + } + + @Override + public String getName() { + return this.providerName; + } + + @Override + public boolean configure(Map params) { + lifeCycle = ComponentContext.inject(HuaweiObsObjectStoreLifeCycleImpl.class); + driver = ComponentContext.inject(HuaweiObsObjectStoreDriverImpl.class); + storeMgr.registerDriver(this.getName(), driver); + return true; + } + + @Override + public DataStoreDriver getDataStoreDriver() { + return this.driver; + } + + @Override + public HypervisorHostListener getHostListener() { + return null; + } + + @Override + public Set getTypes() { + Set types = new HashSet<>(); + types.add(DataStoreProviderType.OBJECT); + return types; + } +} diff --git a/plugins/storage/object/huawei-obs/src/main/resources/META-INF/cloudstack/storage-object-huawei-obs/module.properties b/plugins/storage/object/huawei-obs/src/main/resources/META-INF/cloudstack/storage-object-huawei-obs/module.properties new file mode 100644 index 000000000000..ca6a0f9f7af8 --- /dev/null +++ b/plugins/storage/object/huawei-obs/src/main/resources/META-INF/cloudstack/storage-object-huawei-obs/module.properties @@ -0,0 +1,2 @@ +name=storage-object-huawei-obs +parent=storage diff --git a/plugins/storage/object/huawei-obs/src/main/resources/META-INF/cloudstack/storage-object-huawei-obs/spring-storage-object-huawei-obs-context.xml b/plugins/storage/object/huawei-obs/src/main/resources/META-INF/cloudstack/storage-object-huawei-obs/spring-storage-object-huawei-obs-context.xml new file mode 100644 index 000000000000..12fdd46f4de2 --- /dev/null +++ b/plugins/storage/object/huawei-obs/src/main/resources/META-INF/cloudstack/storage-object-huawei-obs/spring-storage-object-huawei-obs-context.xml @@ -0,0 +1,12 @@ + + + + diff --git a/plugins/storage/object/huawei-obs/src/test/java/org/apache/cloudstack/storage/datastore/driver/HuaweiObsObjectStoreDriverImplTest.java b/plugins/storage/object/huawei-obs/src/test/java/org/apache/cloudstack/storage/datastore/driver/HuaweiObsObjectStoreDriverImplTest.java new file mode 100644 index 000000000000..d14fddcc2951 --- /dev/null +++ b/plugins/storage/object/huawei-obs/src/test/java/org/apache/cloudstack/storage/datastore/driver/HuaweiObsObjectStoreDriverImplTest.java @@ -0,0 +1,87 @@ +package org.apache.cloudstack.storage.datastore.driver; + +import com.cloud.storage.BucketVO; +import com.cloud.storage.dao.BucketDao; +import com.cloud.user.AccountDetailVO; +import com.cloud.user.AccountDetailsDao; +import com.cloud.user.AccountVO; +import com.cloud.user.dao.AccountDao; +import com.obs.services.ObsClient; +import com.obs.services.model.CreateBucketRequest; +import org.apache.cloudstack.storage.datastore.db.ObjectStoreDao; +import org.apache.cloudstack.storage.datastore.db.ObjectStoreDetailsDao; +import org.apache.cloudstack.storage.datastore.db.ObjectStoreVO; +import org.apache.cloudstack.storage.object.Bucket; +import org.junit.Before; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.mockito.Mock; +import org.mockito.Spy; +import org.mockito.junit.MockitoJUnitRunner; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertTrue; +import org.mockito.Mockito; + +@RunWith(MockitoJUnitRunner.class) +public class HuaweiObsObjectStoreDriverImplTest { + + @Spy + HuaweiObsObjectStoreDriverImpl huaweiObsObjectStoreDriverImpl = new HuaweiObsObjectStoreDriverImpl(); + + @Mock + ObsClient obsClient; + @Mock + ObjectStoreDao objectStoreDao; + @Mock + ObjectStoreVO objectStoreVO; + @Mock + ObjectStoreDetailsDao objectStoreDetailsDao; + @Mock + AccountDao accountDao; + @Mock + BucketDao bucketDao; + @Mock + AccountVO account; + @Mock + AccountDetailsDao accountDetailsDao; + + Bucket bucket; + String bucketName = "test-bucket"; + + @Before + public void setUp() { + huaweiObsObjectStoreDriverImpl._storeDao = objectStoreDao; + huaweiObsObjectStoreDriverImpl._storeDetailsDao = objectStoreDetailsDao; + huaweiObsObjectStoreDriverImpl._accountDao = accountDao; + huaweiObsObjectStoreDriverImpl._bucketDao = bucketDao; + huaweiObsObjectStoreDriverImpl._accountDetailsDao = accountDetailsDao; + bucket = new BucketVO(0, 0, 0, bucketName, 100, false, false, false, "public"); + } + + @Test + public void testCreateBucket() throws Exception { + Mockito.doReturn(obsClient).when(huaweiObsObjectStoreDriverImpl).getObsClient(Mockito.anyLong()); + Mockito.when(accountDao.findById(Mockito.anyLong())).thenReturn(account); + Mockito.when(accountDetailsDao.findDetail(Mockito.anyLong(), Mockito.anyString())).thenReturn(new AccountDetailVO(1L, "abc", "def")); + Mockito.when(obsClient.headBucket(bucketName)).thenReturn(false); + CreateBucketRequest createBucketRequest = new CreateBucketRequest(bucketName); + createBucketRequest.setAcl(com.obs.services.model.AccessControlList.REST_CANNED_PUBLIC_READ_WRITE); + Mockito.when(bucketDao.findById(Mockito.anyLong())).thenReturn(new BucketVO(0, 0, 0, bucketName, 100, false, false, false, "public")); + Mockito.when(objectStoreVO.getUrl()).thenReturn("http://test-bucket.localhost:9000"); + Mockito.when(objectStoreDao.findById(Mockito.any())).thenReturn(objectStoreVO); + Bucket bucketRet = huaweiObsObjectStoreDriverImpl.createBucket(bucket, false); + assertEquals(bucketRet.getName(), bucket.getName()); + Mockito.verify(obsClient, Mockito.times(1)).headBucket(Mockito.anyString()); + Mockito.verify(obsClient, Mockito.times(1)).createBucket(Mockito.any(CreateBucketRequest.class)); + } + + @Test + public void testDeleteBucket() throws Exception { + Mockito.doReturn(obsClient).when(huaweiObsObjectStoreDriverImpl).getObsClient(Mockito.anyLong()); + Mockito.when(obsClient.headBucket(bucketName)).thenReturn(true); + boolean success = huaweiObsObjectStoreDriverImpl.deleteBucket(bucketName, 1L); + assertTrue(success); + Mockito.verify(obsClient, Mockito.times(1)).headBucket(Mockito.anyString()); + Mockito.verify(obsClient, Mockito.times(1)).deleteBucket(Mockito.anyString()); + } +} diff --git a/plugins/storage/object/huawei-obs/src/test/java/org/apache/cloudstack/storage/datastore/provider/HuaweiObsObjectStoreProviderImplTest.java b/plugins/storage/object/huawei-obs/src/test/java/org/apache/cloudstack/storage/datastore/provider/HuaweiObsObjectStoreProviderImplTest.java new file mode 100644 index 000000000000..32839aaae845 --- /dev/null +++ b/plugins/storage/object/huawei-obs/src/test/java/org/apache/cloudstack/storage/datastore/provider/HuaweiObsObjectStoreProviderImplTest.java @@ -0,0 +1,34 @@ +package org.apache.cloudstack.storage.datastore.provider; + +import org.apache.cloudstack.engine.subsystem.api.storage.DataStoreProvider.DataStoreProviderType; +import org.junit.Before; +import org.junit.Test; +import org.mockito.MockitoAnnotations; + +import java.util.Set; + +import static org.junit.Assert.assertEquals; + +public class HuaweiObsObjectStoreProviderImplTest { + + private HuaweiObsObjectStoreProviderImpl huaweiObsObjectStoreProviderImpl; + + @Before + public void setUp() { + MockitoAnnotations.initMocks(this); + huaweiObsObjectStoreProviderImpl = new HuaweiObsObjectStoreProviderImpl(); + } + + @Test + public void testGetName() { + String name = huaweiObsObjectStoreProviderImpl.getName(); + assertEquals("Huawei OBS", name); + } + + @Test + public void testGetTypes() { + Set types = huaweiObsObjectStoreProviderImpl.getTypes(); + assertEquals(1, types.size()); + assertEquals("OBJECT", types.toArray()[0].toString()); + } +} diff --git a/ui/src/views/infra/AddObjectStorage.vue b/ui/src/views/infra/AddObjectStorage.vue index 4aacd6adc0f1..6db69ffcdb4a 100644 --- a/ui/src/views/infra/AddObjectStorage.vue +++ b/ui/src/views/infra/AddObjectStorage.vue @@ -82,7 +82,7 @@ export default { inject: ['parentFetchData'], data () { return { - providers: ['MinIO', 'Simulator'], + providers: ['MinIO', 'Huawei OBS', 'Simulator'], zones: [], loading: false }