From d131c50b19bf3f8bb667388af07475104abe44de Mon Sep 17 00:00:00 2001 From: Siyao Meng <50227127+smengcl@users.noreply.github.com> Date: Tue, 13 Jul 2021 14:14:57 -0700 Subject: [PATCH 1/4] HDDS-5279. OFS mkdir -p does not work when Volume is not pre-created Change-Id: I9e59ec7c2c895dd3c70ef20e6beba4a113ea2f24 --- .../BasicRootedOzoneClientAdapterImpl.java | 41 +++++++++---------- 1 file changed, 20 insertions(+), 21 deletions(-) diff --git a/hadoop-ozone/ozonefs-common/src/main/java/org/apache/hadoop/fs/ozone/BasicRootedOzoneClientAdapterImpl.java b/hadoop-ozone/ozonefs-common/src/main/java/org/apache/hadoop/fs/ozone/BasicRootedOzoneClientAdapterImpl.java index f25dd45f31ec..5a59e819757b 100644 --- a/hadoop-ozone/ozonefs-common/src/main/java/org/apache/hadoop/fs/ozone/BasicRootedOzoneClientAdapterImpl.java +++ b/hadoop-ozone/ozonefs-common/src/main/java/org/apache/hadoop/fs/ozone/BasicRootedOzoneClientAdapterImpl.java @@ -223,31 +223,30 @@ private OzoneBucket getBucket(String volumeStr, String bucketStr, try { bucket = proxy.getBucketDetails(volumeStr, bucketStr); } catch (OMException ex) { - // Note: always create bucket if volumeStr matches "tmp" so -put works if (createIfNotExist) { - // Note: getBucketDetails always throws BUCKET_NOT_FOUND, even if - // the volume doesn't exist. - if (ex.getResult().equals(BUCKET_NOT_FOUND)) { - OzoneVolume volume; + // getBucketDetails throws VOLUME_NOT_FOUND when the parent volume + // doesn't exist; throws BUCKET_NOT_FOUND when parent volume exists but + // the bucket doesn't exist. + if (ex.getResult().equals(VOLUME_NOT_FOUND)) { try { - volume = proxy.getVolumeDetails(volumeStr); - } catch (OMException getVolEx) { - if (getVolEx.getResult().equals(VOLUME_NOT_FOUND)) { - // Volume doesn't exist. Create it - try { - objectStore.createVolume(volumeStr); - } catch (OMException newVolEx) { - // Ignore the case where another client created the volume - if (!newVolEx.getResult().equals(VOLUME_ALREADY_EXISTS)) { - throw newVolEx; - } - } - } else { - throw getVolEx; + objectStore.createVolume(volumeStr); + } catch (OMException newVolEx) { + // Ignore the case where another client created the volume + if (!newVolEx.getResult().equals(VOLUME_ALREADY_EXISTS)) { + throw newVolEx; + } + } + OzoneVolume volume = proxy.getVolumeDetails(volumeStr); + try { + volume.createBucket(bucketStr); + } catch (OMException newBucEx) { + // Ignore the case where another client created the bucket + if (!newBucEx.getResult().equals(BUCKET_ALREADY_EXISTS)) { + throw newBucEx; } - // Try get volume again - volume = proxy.getVolumeDetails(volumeStr); } + } else if (ex.getResult().equals(BUCKET_NOT_FOUND)) { + OzoneVolume volume = proxy.getVolumeDetails(volumeStr); // Create the bucket try { volume.createBucket(bucketStr); From 50d211b94fbc8577b746f56ed1a02f2a06b70264 Mon Sep 17 00:00:00 2001 From: Siyao Meng <50227127+smengcl@users.noreply.github.com> Date: Wed, 14 Jul 2021 00:27:47 -0700 Subject: [PATCH 2/4] Address UT failure. Change-Id: Ifb321f6d1278c8c48c282876be586220cfcd288c --- .../BasicRootedOzoneClientAdapterImpl.java | 25 ++++++++----------- 1 file changed, 11 insertions(+), 14 deletions(-) diff --git a/hadoop-ozone/ozonefs-common/src/main/java/org/apache/hadoop/fs/ozone/BasicRootedOzoneClientAdapterImpl.java b/hadoop-ozone/ozonefs-common/src/main/java/org/apache/hadoop/fs/ozone/BasicRootedOzoneClientAdapterImpl.java index 5a59e819757b..4b88c51a29f9 100644 --- a/hadoop-ozone/ozonefs-common/src/main/java/org/apache/hadoop/fs/ozone/BasicRootedOzoneClientAdapterImpl.java +++ b/hadoop-ozone/ozonefs-common/src/main/java/org/apache/hadoop/fs/ozone/BasicRootedOzoneClientAdapterImpl.java @@ -224,10 +224,12 @@ private OzoneBucket getBucket(String volumeStr, String bucketStr, bucket = proxy.getBucketDetails(volumeStr, bucketStr); } catch (OMException ex) { if (createIfNotExist) { - // getBucketDetails throws VOLUME_NOT_FOUND when the parent volume - // doesn't exist; throws BUCKET_NOT_FOUND when parent volume exists but - // the bucket doesn't exist. - if (ex.getResult().equals(VOLUME_NOT_FOUND)) { + // getBucketDetails can throw VOLUME_NOT_FOUND when the parent volume + // doesn't exist and ACL is enabled; it can only throw BUCKET_NOT_FOUND + // when ACL is disabled. Both exceptions need to be handled. + switch (ex.getResult()) { + case VOLUME_NOT_FOUND: + case BUCKET_NOT_FOUND: try { objectStore.createVolume(volumeStr); } catch (OMException newVolEx) { @@ -236,16 +238,7 @@ private OzoneBucket getBucket(String volumeStr, String bucketStr, throw newVolEx; } } - OzoneVolume volume = proxy.getVolumeDetails(volumeStr); - try { - volume.createBucket(bucketStr); - } catch (OMException newBucEx) { - // Ignore the case where another client created the bucket - if (!newBucEx.getResult().equals(BUCKET_ALREADY_EXISTS)) { - throw newBucEx; - } - } - } else if (ex.getResult().equals(BUCKET_NOT_FOUND)) { + OzoneVolume volume = proxy.getVolumeDetails(volumeStr); // Create the bucket try { @@ -256,6 +249,10 @@ private OzoneBucket getBucket(String volumeStr, String bucketStr, throw newBucEx; } } + break; + default: + // Throw unhandled exception + throw ex; } // Try get bucket again bucket = proxy.getBucketDetails(volumeStr, bucketStr); From 1c17d7cee26cb53cac2d030317a617bed75fd33c Mon Sep 17 00:00:00 2001 From: Siyao Meng <50227127+smengcl@users.noreply.github.com> Date: Wed, 14 Jul 2021 11:41:27 -0700 Subject: [PATCH 3/4] Update UT, adding new enableAcl param. Change-Id: I9c288a4ffa66a66a25819282d6a09fedfae1fba9 --- .../fs/ozone/TestRootedOzoneFileSystem.java | 16 +++++++++++----- .../ozone/TestRootedOzoneFileSystemWithFSO.java | 9 +++++---- 2 files changed, 16 insertions(+), 9 deletions(-) diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/fs/ozone/TestRootedOzoneFileSystem.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/fs/ozone/TestRootedOzoneFileSystem.java index 780b64a9355c..2403719ab79d 100644 --- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/fs/ozone/TestRootedOzoneFileSystem.java +++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/fs/ozone/TestRootedOzoneFileSystem.java @@ -35,6 +35,7 @@ import org.apache.hadoop.ozone.MiniOzoneCluster; import org.apache.hadoop.ozone.OFSPath; import org.apache.hadoop.ozone.OzoneAcl; +import org.apache.hadoop.ozone.OzoneConfigKeys; import org.apache.hadoop.ozone.OzoneConsts; import org.apache.hadoop.ozone.TestDataUtil; import org.apache.hadoop.ozone.client.ObjectStore; @@ -108,16 +109,19 @@ public class TestRootedOzoneFileSystem { @Parameterized.Parameters public static Collection data() { return Arrays.asList( - new Object[]{true, true}, - new Object[]{true, false}, - new Object[]{false, true}, - new Object[]{false, false}); + new Object[]{true, true, true}, + new Object[]{true, true, false}, + new Object[]{true, false, false}, + new Object[]{false, true, false}, + new Object[]{false, false, false} + ); } public TestRootedOzoneFileSystem(boolean setDefaultFs, - boolean enableOMRatis) { + boolean enableOMRatis, boolean isAclEnabled) { enabledFileSystemPaths = setDefaultFs; omRatisEnabled = enableOMRatis; + enableAcl = isAclEnabled; } public static FileSystem getFs() { @@ -134,6 +138,7 @@ public static Path getBucketPath() { private static boolean enabledFileSystemPaths; private static boolean omRatisEnabled; private static boolean isBucketFSOptimized = false; + private static boolean enableAcl; private static OzoneConfiguration conf; private static MiniOzoneCluster cluster = null; @@ -165,6 +170,7 @@ public static void init() throws Exception { conf.setBoolean(OMConfigKeys.OZONE_OM_ENABLE_FILESYSTEM_PATHS, enabledFileSystemPaths); } + conf.setBoolean(OzoneConfigKeys.OZONE_ACL_ENABLED, enableAcl); cluster = MiniOzoneCluster.newBuilder(conf) .setNumDatanodes(3) .build(); diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/fs/ozone/TestRootedOzoneFileSystemWithFSO.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/fs/ozone/TestRootedOzoneFileSystemWithFSO.java index 686339397e01..14d667ba5ba8 100644 --- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/fs/ozone/TestRootedOzoneFileSystemWithFSO.java +++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/fs/ozone/TestRootedOzoneFileSystemWithFSO.java @@ -45,13 +45,14 @@ public class TestRootedOzoneFileSystemWithFSO @Parameterized.Parameters public static Collection data() { return Arrays.asList( - new Object[]{true, true}, - new Object[]{true, false}); + new Object[]{true, true, false}, + new Object[]{true, false, false} + ); } public TestRootedOzoneFileSystemWithFSO(boolean setDefaultFs, - boolean enableOMRatis) throws Exception { - super(setDefaultFs, enableOMRatis); + boolean enableOMRatis, boolean enableAcl) { + super(setDefaultFs, enableOMRatis, enableAcl); } @BeforeClass From db7c054afaef129f6ced249e324c4df6ad9c9096 Mon Sep 17 00:00:00 2001 From: Siyao Meng <50227127+smengcl@users.noreply.github.com> Date: Wed, 21 Jul 2021 11:01:53 -0700 Subject: [PATCH 4/4] Retrigger CI Change-Id: Id2a5bca01f7300a4c190f0f16a9cc09e00644f62