From 40fb5328be711e07df287e991848b158b8e95eec Mon Sep 17 00:00:00 2001 From: swagle Date: Tue, 24 Mar 2020 14:14:19 -0700 Subject: [PATCH 01/10] HDDS-3224. Enforce volume and bucket name rule at create time. --- .../om/TestOzoneManagerRestInterface.java | 20 +++++++++++++++++++ .../apache/hadoop/ozone/om/OzoneManager.java | 10 +++++++--- 2 files changed, 27 insertions(+), 3 deletions(-) diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestOzoneManagerRestInterface.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestOzoneManagerRestInterface.java index 8168d27a5d19..a469e1dfe8ba 100644 --- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestOzoneManagerRestInterface.java +++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestOzoneManagerRestInterface.java @@ -24,10 +24,13 @@ import org.apache.hadoop.hdds.protocol.DatanodeDetails; import org.apache.hadoop.net.NetUtils; import org.apache.hadoop.ozone.MiniOzoneCluster; +import org.apache.hadoop.ozone.om.helpers.OmBucketInfo; +import org.apache.hadoop.ozone.om.helpers.OmVolumeArgs; import org.apache.hadoop.ozone.om.helpers.ServiceInfo; import org.apache.hadoop.ozone.protocol.proto .OzoneManagerProtocolProtos.ServicePort; import org.apache.hadoop.hdds.protocol.proto.HddsProtos; +import org.apache.hadoop.test.LambdaTestUtils; import org.apache.http.HttpResponse; import org.apache.http.client.HttpClient; import org.apache.http.client.methods.HttpGet; @@ -132,4 +135,21 @@ public void testGetServiceList() throws Exception { } } + @Test + public void testInvalidVolumeBucketNamesThrowException() throws Exception { + OmVolumeArgs volumeArgs = OmVolumeArgs.newBuilder() + .setVolume("v1") + .setOwnerName("owner1") + .setAdminName("admin1") + .build(); + LambdaTestUtils.intercept(IllegalArgumentException.class, + "Bucket or Volume length is illegal", + () -> cluster.getOzoneManager().createVolume(volumeArgs)); + OmBucketInfo bucketInfo = OmBucketInfo.newBuilder().setVolumeName( + "volume1").setBucketName("b1").build(); + LambdaTestUtils.intercept(IllegalArgumentException.class, + "Bucket or Volume length is illegal", + () -> cluster.getOzoneManager().createBucket(bucketInfo)); + } + } diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OzoneManager.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OzoneManager.java index 627be1f81df7..8c0d6a058e6b 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OzoneManager.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OzoneManager.java @@ -57,6 +57,7 @@ import org.apache.hadoop.hdds.protocol.proto.HddsProtos; import org.apache.hadoop.hdds.protocol.proto.SCMSecurityProtocolProtos.SCMGetCertResponseProto; import org.apache.hadoop.hdds.protocolPB.SCMSecurityProtocolClientSideTranslatorPB; +import org.apache.hadoop.hdds.scm.client.HddsClientUtils; import org.apache.hadoop.hdds.server.http.RatisDropwizardExports; import org.apache.hadoop.hdds.utils.HddsServerUtil; import org.apache.hadoop.hdds.scm.ScmInfo; @@ -1560,8 +1561,9 @@ public void cancelDelegationToken(Token token) @Override public void createVolume(OmVolumeArgs args) throws IOException { try { - metrics.incNumVolumeCreates(); checkAdmin(); + HddsClientUtils.verifyResourceName(args.getVolume()); + metrics.incNumVolumeCreates(); volumeManager.createVolume(args); AUDIT.logWriteSuccess(buildAuditMessageForSuccess(OMAction.CREATE_VOLUME, (args == null) ? null : args.toAuditMap())); @@ -1883,7 +1885,7 @@ public List listAllVolumes(String prefix, String prevKey, int } private void checkAdmin() throws OMException { - if(isAclEnabled) { + if (isAclEnabled) { if (!ozAdmins.contains(OZONE_ADMINISTRATORS_WILDCARD) && !ozAdmins.contains(ProtobufRpcEngine.Server.getRemoteUser() .getUserName())) { @@ -1905,10 +1907,12 @@ private void checkAdmin() throws OMException { @Override public void createBucket(OmBucketInfo bucketInfo) throws IOException { try { - if(isAclEnabled) { + if (isAclEnabled) { checkAcls(ResourceType.VOLUME, StoreType.OZONE, ACLType.CREATE, bucketInfo.getVolumeName(), bucketInfo.getBucketName(), null); } + HddsClientUtils.verifyResourceName(bucketInfo.getVolumeName(), + bucketInfo.getBucketName()); metrics.incNumBucketCreates(); bucketManager.createBucket(bucketInfo); AUDIT.logWriteSuccess(buildAuditMessageForSuccess(OMAction.CREATE_BUCKET, From b0ef030c2a83866d6b725a3573c582e225f489b4 Mon Sep 17 00:00:00 2001 From: swagle Date: Tue, 24 Mar 2020 21:56:03 -0700 Subject: [PATCH 02/10] Addressed review, comments, still kept original changes since code path is reachable. --- .../hadoop/ozone/client/rpc/RpcClient.java | 8 ++--- .../java/org/apache/hadoop/ozone/OmUtils.java | 29 +++++++++++++++++++ .../ozone/om/exceptions/OMException.java | 2 ++ .../src/main/proto/OzoneManagerProtocol.proto | 2 ++ .../apache/hadoop/ozone/om/OzoneManager.java | 5 ++-- .../request/bucket/OMBucketCreateRequest.java | 4 +++ .../request/volume/OMVolumeCreateRequest.java | 6 ++-- .../bucket/TestOMBucketCreateRequest.java | 5 ++++ .../volume/TestOMVolumeCreateRequest.java | 5 ++++ 9 files changed, 55 insertions(+), 11 deletions(-) diff --git a/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/rpc/RpcClient.java b/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/rpc/RpcClient.java index ab930e896c0c..21a7df1bb1e9 100644 --- a/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/rpc/RpcClient.java +++ b/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/rpc/RpcClient.java @@ -33,6 +33,7 @@ import org.apache.hadoop.hdds.scm.client.HddsClientUtils; import org.apache.hadoop.hdds.tracing.TracingUtil; import org.apache.hadoop.io.IOUtils; +import org.apache.hadoop.ozone.OmUtils; import org.apache.hadoop.ozone.OzoneConfigKeys; import org.apache.hadoop.ozone.OzoneConsts; import org.apache.hadoop.ozone.OzoneSecurityUtil; @@ -763,12 +764,7 @@ public void createS3Bucket(String userName, String s3BucketName) Preconditions.checkArgument(Strings.isNotBlank(s3BucketName), "bucket " + "name cannot be null or empty."); - try { - HddsClientUtils.verifyResourceName(s3BucketName); - } catch (IllegalArgumentException exception) { - throw new OMException("Invalid bucket name: " + s3BucketName, - OMException.ResultCodes.INVALID_BUCKET_NAME); - } + OmUtils.validateBucketName(s3BucketName); ozoneManagerClient.createS3Bucket(userName, s3BucketName); } diff --git a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/OmUtils.java b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/OmUtils.java index 3552e79f4138..bacd3bd1a4de 100644 --- a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/OmUtils.java +++ b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/OmUtils.java @@ -49,8 +49,11 @@ import org.apache.commons.compress.utils.IOUtils; import org.apache.commons.lang3.StringUtils; import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.hdds.HddsUtils; +import org.apache.hadoop.hdds.scm.client.HddsClientUtils; import org.apache.hadoop.hdds.utils.db.DBCheckpoint; import org.apache.hadoop.net.NetUtils; +import org.apache.hadoop.ozone.om.exceptions.OMException; import org.apache.hadoop.ozone.om.helpers.OmKeyInfo; import org.apache.hadoop.ozone.om.helpers.RepeatedOmKeyInfo; import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos; @@ -528,4 +531,30 @@ public static RepeatedOmKeyInfo prepareKeyForDelete(OmKeyInfo keyInfo, return repeatedOmKeyInfo; } + + /** + * Verify volume name is a valid DNS name. + */ + public static void validateVolumeName(String volumeName) throws OMException { + try { + HddsClientUtils.verifyResourceName(volumeName); + } catch (IllegalArgumentException e) { + throw new OMException("Invalid volume name: " + volumeName, + OMException.ResultCodes.INVALID_VOLUME_NAME); + } + } + + /** + * Verify bucket name is a valid DNS name. + */ + public static void validateBucketName(String bucketName) + throws OMException { + try { + HddsClientUtils.verifyResourceName(bucketName); + } catch (IllegalArgumentException e) { + throw new OMException("Invalid bucket name: " + bucketName, + OMException.ResultCodes.INVALID_BUCKET_NAME); + } + } + } diff --git a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/exceptions/OMException.java b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/exceptions/OMException.java index 279fda72d70a..cb01c811d2c2 100644 --- a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/exceptions/OMException.java +++ b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/exceptions/OMException.java @@ -227,6 +227,8 @@ public enum ResultCodes { DIRECTORY_ALREADY_EXISTS, + INVALID_VOLUME_NAME, + REPLAY // When ratis logs are replayed. } } diff --git a/hadoop-ozone/common/src/main/proto/OzoneManagerProtocol.proto b/hadoop-ozone/common/src/main/proto/OzoneManagerProtocol.proto index 5237136fc194..21a0b31ec6da 100644 --- a/hadoop-ozone/common/src/main/proto/OzoneManagerProtocol.proto +++ b/hadoop-ozone/common/src/main/proto/OzoneManagerProtocol.proto @@ -315,6 +315,8 @@ enum Status { CANNOT_CREATE_DIRECTORY_AT_ROOT = 59; DIRECTORY_ALREADY_EXISTS = 60; + INVALID_VOLUME_NAME = 61; + // When transactions are replayed REPLAY = 100; } diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OzoneManager.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OzoneManager.java index 8c0d6a058e6b..d29e26253863 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OzoneManager.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OzoneManager.java @@ -1562,7 +1562,7 @@ public void cancelDelegationToken(Token token) public void createVolume(OmVolumeArgs args) throws IOException { try { checkAdmin(); - HddsClientUtils.verifyResourceName(args.getVolume()); + OmUtils.validateVolumeName(args.getVolume()); metrics.incNumVolumeCreates(); volumeManager.createVolume(args); AUDIT.logWriteSuccess(buildAuditMessageForSuccess(OMAction.CREATE_VOLUME, @@ -1911,8 +1911,7 @@ public void createBucket(OmBucketInfo bucketInfo) throws IOException { checkAcls(ResourceType.VOLUME, StoreType.OZONE, ACLType.CREATE, bucketInfo.getVolumeName(), bucketInfo.getBucketName(), null); } - HddsClientUtils.verifyResourceName(bucketInfo.getVolumeName(), - bucketInfo.getBucketName()); + OmUtils.validateBucketName(bucketInfo.getBucketName()); metrics.incNumBucketCreates(); bucketManager.createBucket(bucketInfo); AUDIT.logWriteSuccess(buildAuditMessageForSuccess(OMAction.CREATE_BUCKET, diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/bucket/OMBucketCreateRequest.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/bucket/OMBucketCreateRequest.java index 95ead237ec51..185f260dcef1 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/bucket/OMBucketCreateRequest.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/bucket/OMBucketCreateRequest.java @@ -24,6 +24,8 @@ import java.util.stream.Collectors; import com.google.common.base.Optional; + +import org.apache.hadoop.ozone.OmUtils; import org.apache.hadoop.ozone.OzoneAcl; import org.apache.hadoop.ozone.om.helpers.OmVolumeArgs; import org.apache.hadoop.ozone.om.helpers.OzoneAclUtil; @@ -88,6 +90,8 @@ public OMRequest preExecute(OzoneManager ozoneManager) throws IOException { CreateBucketRequest createBucketRequest = getOmRequest().getCreateBucketRequest(); BucketInfo bucketInfo = createBucketRequest.getBucketInfo(); + // Verify resource name + OmUtils.validateBucketName(bucketInfo.getBucketName()); // Get KMS provider. KeyProviderCryptoExtension kmsProvider = diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/volume/OMVolumeCreateRequest.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/volume/OMVolumeCreateRequest.java index 47d839312304..6c2436e2e7ab 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/volume/OMVolumeCreateRequest.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/volume/OMVolumeCreateRequest.java @@ -24,6 +24,8 @@ import java.util.Map; import com.google.common.base.Preconditions; + +import org.apache.hadoop.ozone.OmUtils; import org.apache.hadoop.ozone.om.ratis.utils.OzoneManagerDoubleBufferHelper; import org.apache.hadoop.ozone.om.request.file.OMFileRequest; import org.slf4j.Logger; @@ -71,17 +73,17 @@ public OMRequest preExecute(OzoneManager ozoneManager) throws IOException { VolumeInfo volumeInfo = getOmRequest().getCreateVolumeRequest().getVolumeInfo(); + // Verify resource name + OmUtils.validateVolumeName(volumeInfo.getVolume()); // Set creation time VolumeInfo updatedVolumeInfo = volumeInfo.toBuilder().setCreationTime(Time.now()).build(); - return getOmRequest().toBuilder().setCreateVolumeRequest( CreateVolumeRequest.newBuilder().setVolumeInfo(updatedVolumeInfo)) .setUserInfo(getUserInfo()) .build(); - } @Override diff --git a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/bucket/TestOMBucketCreateRequest.java b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/bucket/TestOMBucketCreateRequest.java index 561bd143a3d4..ab5ad6b46580 100644 --- a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/bucket/TestOMBucketCreateRequest.java +++ b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/bucket/TestOMBucketCreateRequest.java @@ -21,6 +21,8 @@ import java.util.UUID; +import org.apache.hadoop.ozone.om.exceptions.OMException; +import org.apache.hadoop.test.LambdaTestUtils; import org.junit.Assert; import org.junit.Test; @@ -48,6 +50,9 @@ public void testPreExecute() throws Exception { String volumeName = UUID.randomUUID().toString(); String bucketName = UUID.randomUUID().toString(); doPreExecute(volumeName, bucketName); + // Verify invalid bucket name throws exception + LambdaTestUtils.intercept(OMException.class, "Invalid bucket name: b1", + () -> doPreExecute("volume1", "b1")); } diff --git a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/volume/TestOMVolumeCreateRequest.java b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/volume/TestOMVolumeCreateRequest.java index 481304e60679..05d0eba8de2b 100644 --- a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/volume/TestOMVolumeCreateRequest.java +++ b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/volume/TestOMVolumeCreateRequest.java @@ -20,9 +20,11 @@ import java.util.UUID; +import org.apache.hadoop.ozone.om.exceptions.OMException; import org.apache.hadoop.ozone.om.request.file.OMFileRequest; import org.apache.hadoop.ozone.om.response.volume.OMVolumeCreateResponse; import org.apache.hadoop.test.GenericTestUtils; +import org.apache.hadoop.test.LambdaTestUtils; import org.junit.Assert; import org.junit.Test; @@ -49,6 +51,9 @@ public void testPreExecute() throws Exception { String adminName = UUID.randomUUID().toString(); String ownerName = UUID.randomUUID().toString(); doPreExecute(volumeName, adminName, ownerName); + // Verify exception thrown on invalid volume name + LambdaTestUtils.intercept(OMException.class, "Invalid volume name: v1", + () -> doPreExecute("v1", adminName, ownerName)); } @Test From 7e0dd29e8c99495c1eae5eb1c992af321d1fa7bc Mon Sep 17 00:00:00 2001 From: swagle Date: Wed, 25 Mar 2020 16:24:34 -0700 Subject: [PATCH 03/10] Addressed review comments. --- .../om/TestOzoneManagerRestInterface.java | 20 ------------------- .../apache/hadoop/ozone/om/OzoneManager.java | 9 +++------ 2 files changed, 3 insertions(+), 26 deletions(-) diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestOzoneManagerRestInterface.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestOzoneManagerRestInterface.java index a469e1dfe8ba..8168d27a5d19 100644 --- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestOzoneManagerRestInterface.java +++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestOzoneManagerRestInterface.java @@ -24,13 +24,10 @@ import org.apache.hadoop.hdds.protocol.DatanodeDetails; import org.apache.hadoop.net.NetUtils; import org.apache.hadoop.ozone.MiniOzoneCluster; -import org.apache.hadoop.ozone.om.helpers.OmBucketInfo; -import org.apache.hadoop.ozone.om.helpers.OmVolumeArgs; import org.apache.hadoop.ozone.om.helpers.ServiceInfo; import org.apache.hadoop.ozone.protocol.proto .OzoneManagerProtocolProtos.ServicePort; import org.apache.hadoop.hdds.protocol.proto.HddsProtos; -import org.apache.hadoop.test.LambdaTestUtils; import org.apache.http.HttpResponse; import org.apache.http.client.HttpClient; import org.apache.http.client.methods.HttpGet; @@ -135,21 +132,4 @@ public void testGetServiceList() throws Exception { } } - @Test - public void testInvalidVolumeBucketNamesThrowException() throws Exception { - OmVolumeArgs volumeArgs = OmVolumeArgs.newBuilder() - .setVolume("v1") - .setOwnerName("owner1") - .setAdminName("admin1") - .build(); - LambdaTestUtils.intercept(IllegalArgumentException.class, - "Bucket or Volume length is illegal", - () -> cluster.getOzoneManager().createVolume(volumeArgs)); - OmBucketInfo bucketInfo = OmBucketInfo.newBuilder().setVolumeName( - "volume1").setBucketName("b1").build(); - LambdaTestUtils.intercept(IllegalArgumentException.class, - "Bucket or Volume length is illegal", - () -> cluster.getOzoneManager().createBucket(bucketInfo)); - } - } diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OzoneManager.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OzoneManager.java index d29e26253863..627be1f81df7 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OzoneManager.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OzoneManager.java @@ -57,7 +57,6 @@ import org.apache.hadoop.hdds.protocol.proto.HddsProtos; import org.apache.hadoop.hdds.protocol.proto.SCMSecurityProtocolProtos.SCMGetCertResponseProto; import org.apache.hadoop.hdds.protocolPB.SCMSecurityProtocolClientSideTranslatorPB; -import org.apache.hadoop.hdds.scm.client.HddsClientUtils; import org.apache.hadoop.hdds.server.http.RatisDropwizardExports; import org.apache.hadoop.hdds.utils.HddsServerUtil; import org.apache.hadoop.hdds.scm.ScmInfo; @@ -1561,9 +1560,8 @@ public void cancelDelegationToken(Token token) @Override public void createVolume(OmVolumeArgs args) throws IOException { try { - checkAdmin(); - OmUtils.validateVolumeName(args.getVolume()); metrics.incNumVolumeCreates(); + checkAdmin(); volumeManager.createVolume(args); AUDIT.logWriteSuccess(buildAuditMessageForSuccess(OMAction.CREATE_VOLUME, (args == null) ? null : args.toAuditMap())); @@ -1885,7 +1883,7 @@ public List listAllVolumes(String prefix, String prevKey, int } private void checkAdmin() throws OMException { - if (isAclEnabled) { + if(isAclEnabled) { if (!ozAdmins.contains(OZONE_ADMINISTRATORS_WILDCARD) && !ozAdmins.contains(ProtobufRpcEngine.Server.getRemoteUser() .getUserName())) { @@ -1907,11 +1905,10 @@ private void checkAdmin() throws OMException { @Override public void createBucket(OmBucketInfo bucketInfo) throws IOException { try { - if (isAclEnabled) { + if(isAclEnabled) { checkAcls(ResourceType.VOLUME, StoreType.OZONE, ACLType.CREATE, bucketInfo.getVolumeName(), bucketInfo.getBucketName(), null); } - OmUtils.validateBucketName(bucketInfo.getBucketName()); metrics.incNumBucketCreates(); bucketManager.createBucket(bucketInfo); AUDIT.logWriteSuccess(buildAuditMessageForSuccess(OMAction.CREATE_BUCKET, From 6b76587ffcc644b11f8054d9d3c250ab69248bfc Mon Sep 17 00:00:00 2001 From: swagle Date: Thu, 26 Mar 2020 15:53:44 -0700 Subject: [PATCH 04/10] Checkstyle fixes. --- .../main/java/org/apache/hadoop/ozone/client/rpc/RpcClient.java | 1 - .../common/src/main/java/org/apache/hadoop/ozone/OmUtils.java | 1 - 2 files changed, 2 deletions(-) diff --git a/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/rpc/RpcClient.java b/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/rpc/RpcClient.java index 21a7df1bb1e9..5836017969cc 100644 --- a/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/rpc/RpcClient.java +++ b/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/rpc/RpcClient.java @@ -48,7 +48,6 @@ import org.apache.hadoop.ozone.client.io.OzoneInputStream; import org.apache.hadoop.ozone.client.io.OzoneOutputStream; import org.apache.hadoop.ozone.client.protocol.ClientProtocol; -import org.apache.hadoop.ozone.om.exceptions.OMException; import org.apache.hadoop.ozone.om.ha.OMFailoverProxyProvider; import org.apache.hadoop.ozone.om.helpers.BucketEncryptionKeyInfo; import org.apache.hadoop.ozone.om.helpers.OmBucketArgs; diff --git a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/OmUtils.java b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/OmUtils.java index bacd3bd1a4de..44333c67fc4e 100644 --- a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/OmUtils.java +++ b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/OmUtils.java @@ -49,7 +49,6 @@ import org.apache.commons.compress.utils.IOUtils; import org.apache.commons.lang3.StringUtils; import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.hdds.HddsUtils; import org.apache.hadoop.hdds.scm.client.HddsClientUtils; import org.apache.hadoop.hdds.utils.db.DBCheckpoint; import org.apache.hadoop.net.NetUtils; From dbaa634b73b4175649818e3f95f99d6d0db7b404 Mon Sep 17 00:00:00 2001 From: swagle Date: Thu, 26 Mar 2020 19:04:11 -0700 Subject: [PATCH 05/10] Changes missed to S3BucketCreateRequest. --- .../om/request/s3/bucket/S3BucketCreateRequest.java | 11 +++-------- .../request/s3/bucket/TestS3BucketCreateRequest.java | 4 ++-- 2 files changed, 5 insertions(+), 10 deletions(-) diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/s3/bucket/S3BucketCreateRequest.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/s3/bucket/S3BucketCreateRequest.java index 9f916ef7d991..6d65b50f39e7 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/s3/bucket/S3BucketCreateRequest.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/s3/bucket/S3BucketCreateRequest.java @@ -26,6 +26,8 @@ import com.google.common.annotations.VisibleForTesting; import com.google.common.base.Optional; import com.google.common.base.Preconditions; + +import org.apache.hadoop.ozone.OmUtils; import org.apache.hadoop.ozone.OzoneAcl; import org.apache.hadoop.ozone.om.ratis.utils.OzoneManagerDoubleBufferHelper; import org.apache.hadoop.ozone.om.request.file.OMFileRequest; @@ -98,14 +100,7 @@ public OMRequest preExecute(OzoneManager ozoneManager) throws IOException { // TODO: Do we need to enforce the bucket rules in this code path? // https://docs.aws.amazon.com/AmazonS3/latest/dev/BucketRestrictions.html - // For now only checked the length. - int bucketLength = s3CreateBucketRequest.getS3Bucketname().length(); - if (bucketLength < S3_BUCKET_MIN_LENGTH || - bucketLength >= S3_BUCKET_MAX_LENGTH) { - throw new OMException("S3BucketName must be at least 3 and not more " + - "than 63 characters long", - OMException.ResultCodes.S3_BUCKET_INVALID_LENGTH); - } + OmUtils.validateBucketName(s3CreateBucketRequest.getS3Bucketname()); return getOmRequest().toBuilder() .setCreateS3BucketRequest(newS3CreateBucketRequest) diff --git a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/s3/bucket/TestS3BucketCreateRequest.java b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/s3/bucket/TestS3BucketCreateRequest.java index 9ad27ea588fb..30bf12159376 100644 --- a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/s3/bucket/TestS3BucketCreateRequest.java +++ b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/s3/bucket/TestS3BucketCreateRequest.java @@ -60,7 +60,7 @@ public void testPreExecuteInvalidBucketLength() throws Exception { doPreExecute(userName, s3BucketName); fail("testPreExecuteInvalidBucketLength failed"); } catch (OMException ex) { - GenericTestUtils.assertExceptionContains("S3_BUCKET_INVALID_LENGTH", ex); + GenericTestUtils.assertExceptionContains("INVALID_BUCKET_NAME", ex); } // set bucket name which is greater than 63 characters length @@ -70,7 +70,7 @@ public void testPreExecuteInvalidBucketLength() throws Exception { doPreExecute(userName, s3BucketName); fail("testPreExecuteInvalidBucketLength failed"); } catch (OMException ex) { - GenericTestUtils.assertExceptionContains("S3_BUCKET_INVALID_LENGTH", ex); + GenericTestUtils.assertExceptionContains("INVALID_BUCKET_NAME", ex); } } From b43350364e778c3d5f7e07f20dcf7c374260c596 Mon Sep 17 00:00:00 2001 From: swagle Date: Thu, 26 Mar 2020 19:08:16 -0700 Subject: [PATCH 06/10] Checkstyle fix. --- .../ozone/om/request/s3/bucket/S3BucketCreateRequest.java | 2 -- 1 file changed, 2 deletions(-) diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/s3/bucket/S3BucketCreateRequest.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/s3/bucket/S3BucketCreateRequest.java index 6d65b50f39e7..d28f1075c81b 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/s3/bucket/S3BucketCreateRequest.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/s3/bucket/S3BucketCreateRequest.java @@ -66,8 +66,6 @@ import org.apache.hadoop.hdds.utils.db.cache.CacheValue; import static org.apache.hadoop.ozone.OzoneConsts.OM_S3_VOLUME_PREFIX; -import static org.apache.hadoop.ozone.OzoneConsts.S3_BUCKET_MAX_LENGTH; -import static org.apache.hadoop.ozone.OzoneConsts.S3_BUCKET_MIN_LENGTH; import static org.apache.hadoop.ozone.om.lock.OzoneManagerLock.Resource.BUCKET_LOCK; import static org.apache.hadoop.ozone.om.lock.OzoneManagerLock.Resource.S3_BUCKET_LOCK; import static org.apache.hadoop.ozone.om.lock.OzoneManagerLock.Resource.USER_LOCK; From fd56dd8710e91f70d0256106286e6397c1de2d30 Mon Sep 17 00:00:00 2001 From: swagle Date: Tue, 31 Mar 2020 23:30:06 -0700 Subject: [PATCH 07/10] Robot test chnages to fix volume name. --- .../dist/src/main/compose/ozone-mr/common-config | 2 +- .../dist/src/main/compose/ozonesecure-mr/README.md | 12 ++++++------ .../src/main/compose/ozonesecure-mr/docker-config | 2 +- .../dist/src/main/smoketest/createbucketenv.robot | 2 +- .../dist/src/main/smoketest/createmrenv.robot | 4 ++-- hadoop-ozone/dist/src/main/smoketest/mapreduce.robot | 4 ++-- .../dist/src/main/smoketest/ozonefs/hadoopo3fs.robot | 4 ++-- 7 files changed, 15 insertions(+), 15 deletions(-) diff --git a/hadoop-ozone/dist/src/main/compose/ozone-mr/common-config b/hadoop-ozone/dist/src/main/compose/ozone-mr/common-config index a356c92e15d2..46e7527315cc 100644 --- a/hadoop-ozone/dist/src/main/compose/ozone-mr/common-config +++ b/hadoop-ozone/dist/src/main/compose/ozone-mr/common-config @@ -30,7 +30,7 @@ OZONE-SITE.XML_hdds.datanode.dir=/data/hdds HDFS-SITE.XML_dfs.datanode.address=0.0.0.0:1019 HDFS-SITE.XML_dfs.datanode.http.address=0.0.0.0:1012 -CORE-SITE.xml_fs.defaultFS=o3fs://bucket1.vol1/ +CORE-SITE.xml_fs.defaultFS=o3fs://bucket1.volume1/ MAPRED-SITE.XML_mapreduce.framework.name=yarn MAPRED-SITE.XML_yarn.app.mapreduce.am.env=HADOOP_MAPRED_HOME=$HADOOP_HOME diff --git a/hadoop-ozone/dist/src/main/compose/ozonesecure-mr/README.md b/hadoop-ozone/dist/src/main/compose/ozonesecure-mr/README.md index 14262701da51..63a09a9221a4 100644 --- a/hadoop-ozone/dist/src/main/compose/ozonesecure-mr/README.md +++ b/hadoop-ozone/dist/src/main/compose/ozonesecure-mr/README.md @@ -31,13 +31,13 @@ docker-compose exec om bash kinit -kt /etc/security/keytabs/testuser.keytab testuser/om@EXAMPLE.COM -ozone sh volume create /vol1 +ozone sh volume create /volume1 -ozone sh bucket create /vol1/bucket1 +ozone sh bucket create /volume1/bucket1 -ozone sh key put /vol1/bucket1/key1 LICENSE.txt +ozone sh key put /volume1/bucket1/key1 LICENSE.txt -ozone fs -ls o3fs://bucket1.vol1/ +ozone fs -ls o3fs://bucket1.volume1/ ``` ## Yarn Resource Manager Setup @@ -57,7 +57,7 @@ hadoop fs -mkdir /user/hadoop ### WordCount ``` -yarn jar $HADOOP_MAPRED_HOME/hadoop-mapreduce-examples-*.jar wordcount o3fs://bucket1.vol1/key1 o3fs://bucket1.vol1/key1.count +yarn jar $HADOOP_MAPRED_HOME/hadoop-mapreduce-examples-*.jar wordcount o3fs://bucket1.volume1/key1 o3fs://bucket1.volume1/key1.count hadoop fs -cat /key1.count/part-r-00000 ``` @@ -69,5 +69,5 @@ yarn jar $HADOOP_MAPRED_HOME/hadoop-mapreduce-examples-*.jar pi 10 100 ### RandomWrite ``` -yarn jar $HADOOP_MAPRED_HOME/hadoop-mapreduce-examples-*.jar randomwriter -Dtest.randomwrite.total_bytes=10000000 o3fs://bucket1.vol1/randomwrite.out +yarn jar $HADOOP_MAPRED_HOME/hadoop-mapreduce-examples-*.jar randomwriter -Dtest.randomwrite.total_bytes=10000000 o3fs://bucket1.volume1/randomwrite.out ``` diff --git a/hadoop-ozone/dist/src/main/compose/ozonesecure-mr/docker-config b/hadoop-ozone/dist/src/main/compose/ozonesecure-mr/docker-config index f0b7f5cb22da..0b08316131ad 100644 --- a/hadoop-ozone/dist/src/main/compose/ozonesecure-mr/docker-config +++ b/hadoop-ozone/dist/src/main/compose/ozonesecure-mr/docker-config @@ -64,7 +64,7 @@ HDFS-SITE.XML_rpc.metrics.quantile.enable=true HDFS-SITE.XML_rpc.metrics.percentiles.intervals=60,300 CORE-SITE.XML_fs.AbstractFileSystem.o3fs.impl=org.apache.hadoop.fs.ozone.OzFs -CORE-SITE.XML_fs.defaultFS=o3fs://bucket1.vol1/ +CORE-SITE.XML_fs.defaultFS=o3fs://bucket1.volume1/ MAPRED-SITE.XML_mapreduce.framework.name=yarn MAPRED-SITE.XML_yarn.app.mapreduce.am.env=HADOOP_MAPRED_HOME=$HADOOP_HOME diff --git a/hadoop-ozone/dist/src/main/smoketest/createbucketenv.robot b/hadoop-ozone/dist/src/main/smoketest/createbucketenv.robot index da97001725b2..c89abfc40a7a 100644 --- a/hadoop-ozone/dist/src/main/smoketest/createbucketenv.robot +++ b/hadoop-ozone/dist/src/main/smoketest/createbucketenv.robot @@ -21,7 +21,7 @@ Test Timeout 2 minute *** Variables *** -${volume} vol1 +${volume} volume1 ${bucket} bucket1 diff --git a/hadoop-ozone/dist/src/main/smoketest/createmrenv.robot b/hadoop-ozone/dist/src/main/smoketest/createmrenv.robot index 2f93e6c99e95..579fdde058b9 100644 --- a/hadoop-ozone/dist/src/main/smoketest/createmrenv.robot +++ b/hadoop-ozone/dist/src/main/smoketest/createmrenv.robot @@ -21,7 +21,7 @@ Test Timeout 2 minute *** Variables *** -${volume} vol1 +${volume} volume1 ${bucket} bucket1 @@ -40,7 +40,7 @@ Create test volume, bucket and key Run Keyword if "BUCKET_NOT_FOUND" in """${result}""" Create bucket ${result} = Execute ozone sh bucket info /${volume}/${bucket} Should not contain ${result} NOT_FOUND - Execute ozone sh key put /vol1/bucket1/key1 LICENSE.txt + Execute ozone sh key put /volume1/bucket1/key1 LICENSE.txt Create user dir for hadoop Execute ozone fs -mkdir /user diff --git a/hadoop-ozone/dist/src/main/smoketest/mapreduce.robot b/hadoop-ozone/dist/src/main/smoketest/mapreduce.robot index 789ec4f7fc15..c1e32853c224 100644 --- a/hadoop-ozone/dist/src/main/smoketest/mapreduce.robot +++ b/hadoop-ozone/dist/src/main/smoketest/mapreduce.robot @@ -21,7 +21,7 @@ Test Timeout 4 minute *** Variables *** -${volume} vol1 +${volume} volume1 ${bucket} bucket1 ${hadoop.version} 3.2.0 @@ -33,5 +33,5 @@ Execute PI calculation Execute WordCount ${random} Generate Random String 2 [NUMBERS] - ${output} = Execute yarn jar ./share/hadoop/mapreduce/hadoop-mapreduce-examples-${hadoop.version}.jar wordcount o3fs://bucket1.vol1/key1 o3fs://bucket1.vol1/key1-${random}.count + ${output} = Execute yarn jar ./share/hadoop/mapreduce/hadoop-mapreduce-examples-${hadoop.version}.jar wordcount o3fs://bucket1.volume1/key1 o3fs://bucket1.volume1/key1-${random}.count Should Contain ${output} completed successfully diff --git a/hadoop-ozone/dist/src/main/smoketest/ozonefs/hadoopo3fs.robot b/hadoop-ozone/dist/src/main/smoketest/ozonefs/hadoopo3fs.robot index 3336b39603b2..df9f3af474f5 100644 --- a/hadoop-ozone/dist/src/main/smoketest/ozonefs/hadoopo3fs.robot +++ b/hadoop-ozone/dist/src/main/smoketest/ozonefs/hadoopo3fs.robot @@ -28,6 +28,6 @@ ${PREFIX} ozone Test hadoop dfs ${random} = Generate Random String 5 [NUMBERS] - ${result} = Execute hdfs dfs -put /opt/hadoop/NOTICE.txt o3fs://bucket1.vol1/${PREFIX}-${random} - ${result} = Execute hdfs dfs -ls o3fs://bucket1.vol1/ + ${result} = Execute hdfs dfs -put /opt/hadoop/NOTICE.txt o3fs://bucket1.volume1/${PREFIX}-${random} + ${result} = Execute hdfs dfs -ls o3fs://bucket1.volume1/ Should contain ${result} ${PREFIX}-${random} From 9c8e841c63e4f89041ba2494138b74de3215708f Mon Sep 17 00:00:00 2001 From: swagle Date: Wed, 1 Apr 2020 11:45:07 -0700 Subject: [PATCH 08/10] Change to url in config. --- hadoop-ozone/dist/src/main/compose/ozone-om-ha/docker-config | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/hadoop-ozone/dist/src/main/compose/ozone-om-ha/docker-config b/hadoop-ozone/dist/src/main/compose/ozone-om-ha/docker-config index feafda931dac..d5892102d574 100644 --- a/hadoop-ozone/dist/src/main/compose/ozone-om-ha/docker-config +++ b/hadoop-ozone/dist/src/main/compose/ozone-om-ha/docker-config @@ -15,7 +15,7 @@ # limitations under the License. CORE-SITE.XML_fs.o3fs.impl=org.apache.hadoop.fs.ozone.OzoneFileSystem -CORE-SITE.XML_fs.defaultFS=o3fs://bucket.volume.omservice +CORE-SITE.XML_fs.defaultFS=o3fs://bucket1.volume1.omservice OZONE-SITE.XML_ozone.om.service.ids=omservice OZONE-SITE.XML_ozone.om.nodes.omservice=om1,om2,om3 OZONE-SITE.XML_ozone.om.address.omservice.om1=om1 From 17d8a68df484c552ebcb039221f1a9b36f80be60 Mon Sep 17 00:00:00 2001 From: swagle Date: Wed, 1 Apr 2020 11:47:01 -0700 Subject: [PATCH 09/10] Addressed review comment. --- .../java/org/apache/hadoop/ozone/freon/BaseFreonGenerator.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/freon/BaseFreonGenerator.java b/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/freon/BaseFreonGenerator.java index 6b7396353825..6ffbc0d343fd 100644 --- a/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/freon/BaseFreonGenerator.java +++ b/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/freon/BaseFreonGenerator.java @@ -237,7 +237,7 @@ public void init() { attemptCounter = new AtomicLong(0); if (prefix.length() == 0) { - prefix = RandomStringUtils.randomAlphanumeric(10); + prefix = RandomStringUtils.randomAlphanumeric(10).toLowerCase(); } else { //replace environment variables to support multi-node execution prefix = resolvePrefix(prefix); From 43a3caa0f5bd7bf4e1caa6abbb651cd99e13110f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Elek=20M=C3=A1rton?= Date: Fri, 3 Apr 2020 16:40:30 +0200 Subject: [PATCH 10/10] retrigger build with empty commit