From 64b952ee3996ff22eb3e6c0e8def9c01e0ad56e1 Mon Sep 17 00:00:00 2001 From: Jyotinder Singh Date: Fri, 17 Jun 2022 17:34:13 +0530 Subject: [PATCH 1/8] HDDS-6907. OFS should create buckets with FILE_SYSTEM_OPTIMIZED layout. --- .../org/apache/hadoop/ozone/OzoneConfigKeys.java | 9 +++++++++ .../hadoop/fs/ozone/TestRootedOzoneFileSystem.java | 2 ++ .../fs/ozone/BasicRootedOzoneClientAdapterImpl.java | 12 +++++++++++- 3 files changed, 22 insertions(+), 1 deletion(-) diff --git a/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/OzoneConfigKeys.java b/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/OzoneConfigKeys.java index 1a47ad9fd8f0..205abb78a934 100644 --- a/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/OzoneConfigKeys.java +++ b/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/OzoneConfigKeys.java @@ -476,6 +476,15 @@ public final class OzoneConfigKeys { OZONE_CLIENT_BUCKET_REPLICATION_CONFIG_REFRESH_PERIOD_DEFAULT_MS = 300 * 1000; + public static final String OZONE_FS_DEFAULT_BUCKET_LAYOUT = + "ozone.fs.default.bucket.layout"; + + public static final String OZONE_FS_DEFAULT_BUCKET_LAYOUT_DEFAULT = + "FILE_SYSTEM_OPTIMIZED"; + + public static final String OZONE_FS_DEFAULT_BUCKET_LAYOUT_LEGACY = + "LEGACY"; + public static final String OZONE_AUDIT_LOG_DEBUG_CMD_LIST_OMAUDIT = "ozone.audit.log.debug.cmd.list.omaudit"; /** diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/fs/ozone/TestRootedOzoneFileSystem.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/fs/ozone/TestRootedOzoneFileSystem.java index 695d22d376ba..330181009073 100644 --- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/fs/ozone/TestRootedOzoneFileSystem.java +++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/fs/ozone/TestRootedOzoneFileSystem.java @@ -224,6 +224,8 @@ public static void initClusterAndEnv() throws IOException, bucketLayout.name()); } else { bucketLayout = BucketLayout.LEGACY; + conf.set(OzoneConfigKeys.OZONE_FS_DEFAULT_BUCKET_LAYOUT, + OzoneConfigKeys.OZONE_FS_DEFAULT_BUCKET_LAYOUT_LEGACY); conf.set(OMConfigKeys.OZONE_DEFAULT_BUCKET_LAYOUT, bucketLayout.name()); conf.setBoolean(OMConfigKeys.OZONE_OM_ENABLE_FILESYSTEM_PATHS, diff --git a/hadoop-ozone/ozonefs-common/src/main/java/org/apache/hadoop/fs/ozone/BasicRootedOzoneClientAdapterImpl.java b/hadoop-ozone/ozonefs-common/src/main/java/org/apache/hadoop/fs/ozone/BasicRootedOzoneClientAdapterImpl.java index 052c6c8ba6ca..7f0fe8e81cc1 100644 --- a/hadoop-ozone/ozonefs-common/src/main/java/org/apache/hadoop/fs/ozone/BasicRootedOzoneClientAdapterImpl.java +++ b/hadoop-ozone/ozonefs-common/src/main/java/org/apache/hadoop/fs/ozone/BasicRootedOzoneClientAdapterImpl.java @@ -62,6 +62,7 @@ import org.apache.hadoop.ozone.client.OzoneVolume; import org.apache.hadoop.ozone.client.io.OzoneOutputStream; import org.apache.hadoop.ozone.client.protocol.ClientProtocol; +import org.apache.hadoop.ozone.client.BucketArgs; import org.apache.hadoop.ozone.om.exceptions.OMException; import org.apache.hadoop.ozone.om.helpers.BucketLayout; import org.apache.hadoop.ozone.om.helpers.OmKeyInfo; @@ -103,6 +104,7 @@ public class BasicRootedOzoneClientAdapterImpl private ReplicationConfig clientConfiguredReplicationConfig; private boolean securityEnabled; private int configuredDnPort; + private BucketLayout defaultOFSBucketLayout; private OzoneConfiguration config; /** @@ -190,6 +192,12 @@ public BasicRootedOzoneClientAdapterImpl(String omHost, int omPort, this.configuredDnPort = conf.getInt( OzoneConfigKeys.DFS_CONTAINER_IPC_PORT, OzoneConfigKeys.DFS_CONTAINER_IPC_PORT_DEFAULT); + + // Fetches the bucket layout to be used by OFS. + this.defaultOFSBucketLayout = BucketLayout.fromString( + conf.get(OzoneConfigKeys.OZONE_FS_DEFAULT_BUCKET_LAYOUT, + OzoneConfigKeys.OZONE_FS_DEFAULT_BUCKET_LAYOUT_DEFAULT)); + config = conf; } finally { Thread.currentThread().setContextClassLoader(contextClassLoader); @@ -258,7 +266,9 @@ private OzoneBucket getBucket(String volumeStr, String bucketStr, // Create the bucket try { // Buckets created by OFS should be in FSO layout - volume.createBucket(bucketStr); + volume.createBucket(bucketStr, + BucketArgs.newBuilder().setBucketLayout( + this.defaultOFSBucketLayout).build()); } catch (OMException newBucEx) { // Ignore the case where another client created the bucket if (!newBucEx.getResult().equals(BUCKET_ALREADY_EXISTS)) { From f7cc9d454d3f7baf4de2ad786ca07e3926b40036 Mon Sep 17 00:00:00 2001 From: Jyotinder Singh Date: Sun, 19 Jun 2022 23:58:59 +0530 Subject: [PATCH 2/8] Address reviews and added tests. --- .../hadoop/hdds/scm/OzoneClientConfig.java | 18 ++ .../apache/hadoop/ozone/OzoneConfigKeys.java | 8 +- .../src/main/resources/ozone-default.xml | 13 + .../fs/ozone/TestOzoneFSBucketLayout.java | 224 ++++++++++++++++++ .../fs/ozone/TestRootedOzoneFileSystem.java | 4 +- .../BasicRootedOzoneClientAdapterImpl.java | 24 +- 6 files changed, 282 insertions(+), 9 deletions(-) create mode 100644 hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/fs/ozone/TestOzoneFSBucketLayout.java diff --git a/hadoop-hdds/client/src/main/java/org/apache/hadoop/hdds/scm/OzoneClientConfig.java b/hadoop-hdds/client/src/main/java/org/apache/hadoop/hdds/scm/OzoneClientConfig.java index 63dd5115962f..14a8aacbc015 100644 --- a/hadoop-hdds/client/src/main/java/org/apache/hadoop/hdds/scm/OzoneClientConfig.java +++ b/hadoop-hdds/client/src/main/java/org/apache/hadoop/hdds/scm/OzoneClientConfig.java @@ -176,6 +176,14 @@ public enum ChecksumCombineMode { private String checksumCombineMode = ChecksumCombineMode.COMPOSITE_CRC.name(); + @Config(key = "fs.default.bucket.layout", + defaultValue = "FILE_SYSTEM_OPTIMIZED", + type = ConfigType.STRING, + description = "The bucket layout used by buckets created using OFS. " + + "Valid values include FILE_SYSTEM_OPTIMIZED and LEGACY", + tags = ConfigTag.CLIENT) + private String fsDefaultBucketLayout = "FILE_SYSTEM_OPTIMIZED"; + @PostConstruct private void validate() { Preconditions.checkState(streamBufferSize > 0); @@ -307,4 +315,14 @@ public void setEcReconstructStripeReadPoolLimit(int poolLimit) { public int getEcReconstructStripeReadPoolLimit() { return ecReconstructStripeReadPoolLimit; } + + public void setFsDefaultBucketLayout(String bucketLayout) { + if (!bucketLayout.isEmpty()) { + this.fsDefaultBucketLayout = bucketLayout; + } + } + + public String getFsDefaultBucketLayout() { + return fsDefaultBucketLayout; + } } diff --git a/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/OzoneConfigKeys.java b/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/OzoneConfigKeys.java index 205abb78a934..a71d87495bec 100644 --- a/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/OzoneConfigKeys.java +++ b/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/OzoneConfigKeys.java @@ -476,13 +476,13 @@ public final class OzoneConfigKeys { OZONE_CLIENT_BUCKET_REPLICATION_CONFIG_REFRESH_PERIOD_DEFAULT_MS = 300 * 1000; - public static final String OZONE_FS_DEFAULT_BUCKET_LAYOUT = - "ozone.fs.default.bucket.layout"; + public static final String OZONE_CLIENT_FS_DEFAULT_BUCKET_LAYOUT = + "ozone.client.fs.default.bucket.layout"; - public static final String OZONE_FS_DEFAULT_BUCKET_LAYOUT_DEFAULT = + public static final String OZONE_CLIENT_FS_BUCKET_LAYOUT_DEFAULT = "FILE_SYSTEM_OPTIMIZED"; - public static final String OZONE_FS_DEFAULT_BUCKET_LAYOUT_LEGACY = + public static final String OZONE_CLIENT_FS_BUCKET_LAYOUT_LEGACY = "LEGACY"; public static final String OZONE_AUDIT_LOG_DEBUG_CMD_LIST_OMAUDIT = diff --git a/hadoop-hdds/common/src/main/resources/ozone-default.xml b/hadoop-hdds/common/src/main/resources/ozone-default.xml index 5eab4bd2b275..32e37979927f 100644 --- a/hadoop-hdds/common/src/main/resources/ozone-default.xml +++ b/hadoop-hdds/common/src/main/resources/ozone-default.xml @@ -3257,4 +3257,17 @@ If the timeout has been reached, a warning message will be logged. + + + ozone.client.fs.default.bucket.layout + FILE_SYSTEM_OPTIMIZED + OZONE, CLIENT + + Default bucket layout value used when buckets are created using OFS. + Supported values are LEGACY and FILE_SYSTEM_OPTIMIZED. + FILE_SYSTEM_OPTIMIZED: This layout allows the bucket to support atomic rename/delete operations and + also allows interoperability between S3 and FS APIs. Keys written via S3 API with a "/" delimiter + will create intermediate directories. + + diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/fs/ozone/TestOzoneFSBucketLayout.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/fs/ozone/TestOzoneFSBucketLayout.java new file mode 100644 index 000000000000..9c9bfb9c71fb --- /dev/null +++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/fs/ozone/TestOzoneFSBucketLayout.java @@ -0,0 +1,224 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

+ * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.fs.ozone; + +import org.apache.commons.io.IOUtils; +import org.apache.hadoop.hdds.conf.OzoneConfiguration; +import org.apache.hadoop.hdds.scm.OzoneClientConfig; +import org.apache.hadoop.ozone.OzoneConfigKeys; +import org.apache.hadoop.ozone.client.ObjectStore; +import org.apache.hadoop.ozone.client.OzoneBucket; +import org.apache.hadoop.ozone.om.TrashPolicyOzone; +import org.apache.hadoop.ozone.om.exceptions.OMException; +import org.apache.hadoop.ozone.om.helpers.BucketLayout; +import org.junit.Assert; +import org.junit.Before; +import org.junit.After; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.junit.runners.Parameterized; + +import java.io.IOException; +import java.util.concurrent.TimeoutException; + +import static org.apache.hadoop.ozone.OzoneConfigKeys.OZONE_FS_ITERATE_BATCH_SIZE; +import static org.apache.hadoop.ozone.OzoneConsts.OZONE_URI_DELIMITER; +import static org.apache.hadoop.ozone.om.OMConfigKeys.OZONE_OM_ADDRESS_KEY; +import static org.junit.Assert.fail; + +import org.apache.hadoop.fs.CommonConfigurationKeysPublic; +import org.apache.hadoop.fs.FileSystem; +import org.apache.hadoop.fs.Path; +import org.apache.hadoop.fs.TrashPolicy; +import org.apache.hadoop.ozone.MiniOzoneCluster; +import org.apache.hadoop.ozone.OzoneConsts; +import org.apache.hadoop.ozone.TestDataUtil; + +import java.util.Arrays; +import java.util.Collection; + +/** + * Ozone file system tests to validate default bucket layout configuration + * and behaviour. + * TODO: Refactor this and TestOzoneFileSystem to reduce duplication. + */ +@RunWith(Parameterized.class) +public class TestOzoneFSBucketLayout { + + private static BucketLayout defaultBucketLayout; + + private static MiniOzoneCluster cluster = null; + private static FileSystem fs; + private static ObjectStore objectStore; + private static BasicRootedOzoneClientAdapterImpl adapter; + + private static String volumeName; + private static Path volumePath; + + @Parameterized.Parameters + public static Collection data() { + return Arrays.asList( + null, + BucketLayout.FILE_SYSTEM_OPTIMIZED, + BucketLayout.LEGACY, + BucketLayout.OBJECT_STORE + ); + } + + public TestOzoneFSBucketLayout(BucketLayout bucketLayout) { + // Ignored. Actual init done in initParam(). + // This empty constructor is still required to avoid argument exception. + } + + @Parameterized.BeforeParam + public static void initParam(BucketLayout bucketLayout) + throws IOException, InterruptedException, TimeoutException { + // Initialize the cluster before EACH set of parameters + defaultBucketLayout = bucketLayout; + + initClusterAndEnv(); + } + + @Parameterized.AfterParam + public static void teardownParam() { + // Tear down the cluster after EACH set of parameters + if (cluster != null) { + cluster.shutdown(); + } + IOUtils.closeQuietly(fs); + } + + @Before + public void createVolumeAndBucket() throws IOException { + // create a volume and a bucket to be used by RootedOzoneFileSystem (OFS) + volumeName = + TestDataUtil.createVolumeAndBucket(cluster) + .getVolumeName(); + volumePath = new Path(OZONE_URI_DELIMITER, volumeName); + } + + @After + public void cleanup() throws IOException { + if (defaultBucketLayout != null && + defaultBucketLayout.equals(BucketLayout.OBJECT_STORE)) { + return; + } + fs.delete(volumePath, true); + } + + public static FileSystem getFs() { + return fs; + } + + + public static void initClusterAndEnv() throws IOException, + InterruptedException, TimeoutException { + OzoneConfiguration conf = new OzoneConfiguration(); + + OzoneClientConfig clientConfig = conf.getObject(OzoneClientConfig.class); + if (defaultBucketLayout != null) { + clientConfig.setFsDefaultBucketLayout(defaultBucketLayout.name()); + } else { + clientConfig.setFsDefaultBucketLayout(""); + } + + conf.setFromObject(clientConfig); + cluster = MiniOzoneCluster.newBuilder(conf) + .setNumDatanodes(3) + .build(); + cluster.waitForClusterToBeReady(); + objectStore = cluster.getClient().getObjectStore(); + String rootPath = String.format("%s://%s/", + OzoneConsts.OZONE_OFS_URI_SCHEME, conf.get(OZONE_OM_ADDRESS_KEY)); + + // Set the fs.defaultFS and start the filesystem + conf.set(CommonConfigurationKeysPublic.FS_DEFAULT_NAME_KEY, rootPath); + // Set the number of keys to be processed during batch operate. + conf.setInt(OZONE_FS_ITERATE_BATCH_SIZE, 5); + + + // In case OZONE_CLIENT_FS_DEFAULT_BUCKET_LAYOUT is set to OBS, + // FS initialization should fail. + if (defaultBucketLayout != null && + defaultBucketLayout.equals(BucketLayout.OBJECT_STORE)) { + try { + fs = FileSystem.get(conf); + fail("File System initialization should fail in case " + + BucketLayout.OBJECT_STORE + " is used as the default value for " + + OzoneConfigKeys.OZONE_CLIENT_FS_DEFAULT_BUCKET_LAYOUT); + } catch (OMException oe) { + Assert.assertTrue(oe.getMessage().contains( + "Buckets created with OBJECT_STORE layout do not support file " + + "system semantics.")); + return; + } + } + + // fs.ofs.impl would be loaded from META-INF, no need to manually set it + fs = FileSystem.get(conf); + conf.setClass("fs.trash.classname", TrashPolicyOzone.class, + TrashPolicy.class); + RootedOzoneFileSystem ofs = (RootedOzoneFileSystem) fs; + adapter = (BasicRootedOzoneClientAdapterImpl) ofs.getAdapter(); + } + + @Test + public void testClientFsDefaultBucketLayout() throws Exception { + if (defaultBucketLayout != null && + defaultBucketLayout.equals(BucketLayout.OBJECT_STORE)) { + // We do not need to test with OBS. FS initialization fails in this case. + return; + } + + Path root = new Path("/" + volumeName); + + String bucketName = getBucketName(); + Path dir1 = new Path(root, bucketName); + + adapter.createDirectory(dir1.toString()); + + OzoneBucket bucketInfo = + objectStore.getClientProxy().getBucketDetails(volumeName, bucketName); + + if (defaultBucketLayout != null) { + // Make sure the bucket layout matches the config. + Assert.assertEquals(defaultBucketLayout, bucketInfo.getBucketLayout()); + } else { + // In case we provide an empty config, the File System should use the + // default config. + Assert.assertEquals( + "FS should use default config value in case of empty config", + OzoneConfigKeys.OZONE_CLIENT_FS_BUCKET_LAYOUT_DEFAULT, + bucketInfo.getBucketLayout().toString()); + } + } + + private String getBucketName() { + String bucketSuffix; + if (defaultBucketLayout != null) { + bucketSuffix = defaultBucketLayout + .toString() + .toLowerCase() + .replaceAll("_", "-"); + } else { + bucketSuffix = "null"; + } + + return "bucket-" + bucketSuffix; + } +} diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/fs/ozone/TestRootedOzoneFileSystem.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/fs/ozone/TestRootedOzoneFileSystem.java index 330181009073..8b0b8d724855 100644 --- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/fs/ozone/TestRootedOzoneFileSystem.java +++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/fs/ozone/TestRootedOzoneFileSystem.java @@ -224,8 +224,8 @@ public static void initClusterAndEnv() throws IOException, bucketLayout.name()); } else { bucketLayout = BucketLayout.LEGACY; - conf.set(OzoneConfigKeys.OZONE_FS_DEFAULT_BUCKET_LAYOUT, - OzoneConfigKeys.OZONE_FS_DEFAULT_BUCKET_LAYOUT_LEGACY); + conf.set(OzoneConfigKeys.OZONE_CLIENT_FS_DEFAULT_BUCKET_LAYOUT, + OzoneConfigKeys.OZONE_CLIENT_FS_BUCKET_LAYOUT_LEGACY); conf.set(OMConfigKeys.OZONE_DEFAULT_BUCKET_LAYOUT, bucketLayout.name()); conf.setBoolean(OMConfigKeys.OZONE_OM_ENABLE_FILESYSTEM_PATHS, diff --git a/hadoop-ozone/ozonefs-common/src/main/java/org/apache/hadoop/fs/ozone/BasicRootedOzoneClientAdapterImpl.java b/hadoop-ozone/ozonefs-common/src/main/java/org/apache/hadoop/fs/ozone/BasicRootedOzoneClientAdapterImpl.java index 7f0fe8e81cc1..fb8aa2925276 100644 --- a/hadoop-ozone/ozonefs-common/src/main/java/org/apache/hadoop/fs/ozone/BasicRootedOzoneClientAdapterImpl.java +++ b/hadoop-ozone/ozonefs-common/src/main/java/org/apache/hadoop/fs/ozone/BasicRootedOzoneClientAdapterImpl.java @@ -194,9 +194,27 @@ public BasicRootedOzoneClientAdapterImpl(String omHost, int omPort, OzoneConfigKeys.DFS_CONTAINER_IPC_PORT_DEFAULT); // Fetches the bucket layout to be used by OFS. - this.defaultOFSBucketLayout = BucketLayout.fromString( - conf.get(OzoneConfigKeys.OZONE_FS_DEFAULT_BUCKET_LAYOUT, - OzoneConfigKeys.OZONE_FS_DEFAULT_BUCKET_LAYOUT_DEFAULT)); + try { + this.defaultOFSBucketLayout = BucketLayout.fromString( + conf.get(OzoneConfigKeys.OZONE_CLIENT_FS_DEFAULT_BUCKET_LAYOUT, + OzoneConfigKeys.OZONE_CLIENT_FS_BUCKET_LAYOUT_DEFAULT)); + } catch (IllegalArgumentException iae) { + throw new OMException("Unsupported value provided for " + + OzoneConfigKeys.OZONE_CLIENT_FS_DEFAULT_BUCKET_LAYOUT + + ". Supported values are " + BucketLayout.FILE_SYSTEM_OPTIMIZED + + " and " + BucketLayout.LEGACY + ".", + OMException.ResultCodes.INVALID_REQUEST); + } + + // Bucket Layout for buckets created with OFS cannot be OBJECT_STORE. + if (defaultOFSBucketLayout.equals(BucketLayout.OBJECT_STORE)) { + throw new OMException( + "Buckets created with OBJECT_STORE layout do not support file " + + "system semantics. Supported values for config " + + OzoneConfigKeys.OZONE_CLIENT_FS_DEFAULT_BUCKET_LAYOUT + + " include " + BucketLayout.FILE_SYSTEM_OPTIMIZED + " and " + + BucketLayout.LEGACY, OMException.ResultCodes.INVALID_REQUEST); + } config = conf; } finally { From 604a91efaa208be11ddf2629aab72160c2ea29ae Mon Sep 17 00:00:00 2001 From: Jyotinder Singh Date: Mon, 20 Jun 2022 11:03:20 +0530 Subject: [PATCH 3/8] findbugs.sh fix --- .../org/apache/hadoop/fs/ozone/TestOzoneFSBucketLayout.java | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/fs/ozone/TestOzoneFSBucketLayout.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/fs/ozone/TestOzoneFSBucketLayout.java index 9c9bfb9c71fb..2257a26071ef 100644 --- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/fs/ozone/TestOzoneFSBucketLayout.java +++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/fs/ozone/TestOzoneFSBucketLayout.java @@ -106,10 +106,11 @@ public static void teardownParam() { @Before public void createVolumeAndBucket() throws IOException { // create a volume and a bucket to be used by RootedOzoneFileSystem (OFS) - volumeName = + TestOzoneFSBucketLayout.volumeName = TestDataUtil.createVolumeAndBucket(cluster) .getVolumeName(); - volumePath = new Path(OZONE_URI_DELIMITER, volumeName); + TestOzoneFSBucketLayout.volumePath = + new Path(OZONE_URI_DELIMITER, volumeName); } @After From e2d71cf3f8b208ed40ccff0fdb8ea7841a6b5441 Mon Sep 17 00:00:00 2001 From: Jyotinder Singh Date: Mon, 20 Jun 2022 11:15:32 +0530 Subject: [PATCH 4/8] findbugs.sh fix --- .../apache/hadoop/fs/ozone/TestOzoneFSBucketLayout.java | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/fs/ozone/TestOzoneFSBucketLayout.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/fs/ozone/TestOzoneFSBucketLayout.java index 2257a26071ef..a2c162651346 100644 --- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/fs/ozone/TestOzoneFSBucketLayout.java +++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/fs/ozone/TestOzoneFSBucketLayout.java @@ -67,8 +67,8 @@ public class TestOzoneFSBucketLayout { private static ObjectStore objectStore; private static BasicRootedOzoneClientAdapterImpl adapter; - private static String volumeName; - private static Path volumePath; + private String volumeName; + private Path volumePath; @Parameterized.Parameters public static Collection data() { @@ -106,11 +106,10 @@ public static void teardownParam() { @Before public void createVolumeAndBucket() throws IOException { // create a volume and a bucket to be used by RootedOzoneFileSystem (OFS) - TestOzoneFSBucketLayout.volumeName = + volumeName = TestDataUtil.createVolumeAndBucket(cluster) .getVolumeName(); - TestOzoneFSBucketLayout.volumePath = - new Path(OZONE_URI_DELIMITER, volumeName); + volumePath = new Path(OZONE_URI_DELIMITER, volumeName); } @After From c1f160a12c9e4097f1560d31c2d5f5d319e2d402 Mon Sep 17 00:00:00 2001 From: Jyotinder Singh Date: Tue, 21 Jun 2022 13:07:45 +0530 Subject: [PATCH 5/8] address reviews: fix tests. --- .../fs/ozone/TestOzoneFSBucketLayout.java | 174 ++++++++---------- .../ozone/om/TestObjectStoreWithLegacyFS.java | 46 +++-- .../BasicRootedOzoneClientAdapterImpl.java | 53 +++--- 3 files changed, 144 insertions(+), 129 deletions(-) diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/fs/ozone/TestOzoneFSBucketLayout.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/fs/ozone/TestOzoneFSBucketLayout.java index a2c162651346..078f40e89c4b 100644 --- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/fs/ozone/TestOzoneFSBucketLayout.java +++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/fs/ozone/TestOzoneFSBucketLayout.java @@ -18,36 +18,36 @@ package org.apache.hadoop.fs.ozone; import org.apache.commons.io.IOUtils; +import org.apache.commons.lang3.StringUtils; import org.apache.hadoop.hdds.conf.OzoneConfiguration; import org.apache.hadoop.hdds.scm.OzoneClientConfig; import org.apache.hadoop.ozone.OzoneConfigKeys; import org.apache.hadoop.ozone.client.ObjectStore; import org.apache.hadoop.ozone.client.OzoneBucket; -import org.apache.hadoop.ozone.om.TrashPolicyOzone; import org.apache.hadoop.ozone.om.exceptions.OMException; import org.apache.hadoop.ozone.om.helpers.BucketLayout; -import org.junit.Assert; -import org.junit.Before; -import org.junit.After; +import org.junit.BeforeClass; +import org.junit.AfterClass; import org.junit.Test; +import org.junit.Assert; import org.junit.runner.RunWith; import org.junit.runners.Parameterized; import java.io.IOException; -import java.util.concurrent.TimeoutException; +import java.util.HashMap; +import java.util.Map; -import static org.apache.hadoop.ozone.OzoneConfigKeys.OZONE_FS_ITERATE_BATCH_SIZE; import static org.apache.hadoop.ozone.OzoneConsts.OZONE_URI_DELIMITER; import static org.apache.hadoop.ozone.om.OMConfigKeys.OZONE_OM_ADDRESS_KEY; -import static org.junit.Assert.fail; import org.apache.hadoop.fs.CommonConfigurationKeysPublic; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; -import org.apache.hadoop.fs.TrashPolicy; import org.apache.hadoop.ozone.MiniOzoneCluster; import org.apache.hadoop.ozone.OzoneConsts; import org.apache.hadoop.ozone.TestDataUtil; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; import java.util.Arrays; import java.util.Collection; @@ -60,51 +60,67 @@ @RunWith(Parameterized.class) public class TestOzoneFSBucketLayout { - private static BucketLayout defaultBucketLayout; - + private static String defaultBucketLayout; private static MiniOzoneCluster cluster = null; - private static FileSystem fs; private static ObjectStore objectStore; private static BasicRootedOzoneClientAdapterImpl adapter; - - private String volumeName; - private Path volumePath; + private static String rootPath; + private static FileSystem fs; + private static String volumeName; + private static Path volumePath; + + private static final String INVALID_CONFIG = "INVALID"; + private static final Map ERROR_MAP = new HashMap<>(); + + private static final Logger LOG = + LoggerFactory.getLogger(TestOzoneFSBucketLayout.class); + + // Initialize error map. + static { + ERROR_MAP.put(BucketLayout.OBJECT_STORE.name(), + "Buckets created with OBJECT_STORE layout do not support file " + + "system semantics."); + ERROR_MAP.put(INVALID_CONFIG, "Unsupported value provided for " + + OzoneConfigKeys.OZONE_CLIENT_FS_DEFAULT_BUCKET_LAYOUT); + } @Parameterized.Parameters - public static Collection data() { + public static Collection data() { return Arrays.asList( - null, - BucketLayout.FILE_SYSTEM_OPTIMIZED, - BucketLayout.LEGACY, - BucketLayout.OBJECT_STORE + // Empty Config + "", + // Invalid Config + INVALID_CONFIG, + // Unsupported Bucket Layout for OFS + BucketLayout.OBJECT_STORE.name(), + // Supported bucket layouts. + BucketLayout.FILE_SYSTEM_OPTIMIZED.name(), + BucketLayout.LEGACY.name() ); } - public TestOzoneFSBucketLayout(BucketLayout bucketLayout) { + public TestOzoneFSBucketLayout(String bucketLayout) { // Ignored. Actual init done in initParam(). // This empty constructor is still required to avoid argument exception. } @Parameterized.BeforeParam - public static void initParam(BucketLayout bucketLayout) - throws IOException, InterruptedException, TimeoutException { - // Initialize the cluster before EACH set of parameters + public static void initDefaultLayout(String bucketLayout) { defaultBucketLayout = bucketLayout; - - initClusterAndEnv(); + LOG.info("Default bucket layout: {}", defaultBucketLayout); } - @Parameterized.AfterParam - public static void teardownParam() { - // Tear down the cluster after EACH set of parameters - if (cluster != null) { - cluster.shutdown(); - } - IOUtils.closeQuietly(fs); - } + @BeforeClass + public static void initCluster() throws Exception { + OzoneConfiguration conf = new OzoneConfiguration(); + cluster = MiniOzoneCluster.newBuilder(conf) + .setNumDatanodes(3) + .build(); + cluster.waitForClusterToBeReady(); + objectStore = cluster.getClient().getObjectStore(); + rootPath = String.format("%s://%s/", + OzoneConsts.OZONE_OFS_URI_SCHEME, conf.get(OZONE_OM_ADDRESS_KEY)); - @Before - public void createVolumeAndBucket() throws IOException { // create a volume and a bucket to be used by RootedOzoneFileSystem (OFS) volumeName = TestDataUtil.createVolumeAndBucket(cluster) @@ -112,79 +128,48 @@ public void createVolumeAndBucket() throws IOException { volumePath = new Path(OZONE_URI_DELIMITER, volumeName); } - @After - public void cleanup() throws IOException { - if (defaultBucketLayout != null && - defaultBucketLayout.equals(BucketLayout.OBJECT_STORE)) { - return; + @AfterClass + public static void teardown() throws IOException { + // Tear down the cluster after EACH set of parameters + IOUtils.closeQuietly(fs); + if (cluster != null) { + cluster.shutdown(); } - fs.delete(volumePath, true); } - public static FileSystem getFs() { - return fs; - } - - - public static void initClusterAndEnv() throws IOException, - InterruptedException, TimeoutException { + @Test + public void testFileSystemBucketLayoutConfiguration() throws IOException { OzoneConfiguration conf = new OzoneConfiguration(); OzoneClientConfig clientConfig = conf.getObject(OzoneClientConfig.class); - if (defaultBucketLayout != null) { - clientConfig.setFsDefaultBucketLayout(defaultBucketLayout.name()); - } else { - clientConfig.setFsDefaultBucketLayout(""); - } + clientConfig.setFsDefaultBucketLayout(defaultBucketLayout); conf.setFromObject(clientConfig); - cluster = MiniOzoneCluster.newBuilder(conf) - .setNumDatanodes(3) - .build(); - cluster.waitForClusterToBeReady(); - objectStore = cluster.getClient().getObjectStore(); - String rootPath = String.format("%s://%s/", - OzoneConsts.OZONE_OFS_URI_SCHEME, conf.get(OZONE_OM_ADDRESS_KEY)); // Set the fs.defaultFS and start the filesystem conf.set(CommonConfigurationKeysPublic.FS_DEFAULT_NAME_KEY, rootPath); - // Set the number of keys to be processed during batch operate. - conf.setInt(OZONE_FS_ITERATE_BATCH_SIZE, 5); - // In case OZONE_CLIENT_FS_DEFAULT_BUCKET_LAYOUT is set to OBS, // FS initialization should fail. - if (defaultBucketLayout != null && - defaultBucketLayout.equals(BucketLayout.OBJECT_STORE)) { + if (ERROR_MAP.containsKey(defaultBucketLayout)) { try { - fs = FileSystem.get(conf); - fail("File System initialization should fail in case " + - BucketLayout.OBJECT_STORE + " is used as the default value for " + + fs = FileSystem.newInstance(conf); + Assert.fail("File System initialization should fail in case " + + " of invalid configuration of " + OzoneConfigKeys.OZONE_CLIENT_FS_DEFAULT_BUCKET_LAYOUT); } catch (OMException oe) { - Assert.assertTrue(oe.getMessage().contains( - "Buckets created with OBJECT_STORE layout do not support file " + - "system semantics.")); + Assert.assertTrue( + oe.getMessage().contains(ERROR_MAP.get(defaultBucketLayout))); return; } } - // fs.ofs.impl would be loaded from META-INF, no need to manually set it - fs = FileSystem.get(conf); - conf.setClass("fs.trash.classname", TrashPolicyOzone.class, - TrashPolicy.class); + // initialize FS and adapter. + fs = FileSystem.newInstance(conf); RootedOzoneFileSystem ofs = (RootedOzoneFileSystem) fs; adapter = (BasicRootedOzoneClientAdapterImpl) ofs.getAdapter(); - } - - @Test - public void testClientFsDefaultBucketLayout() throws Exception { - if (defaultBucketLayout != null && - defaultBucketLayout.equals(BucketLayout.OBJECT_STORE)) { - // We do not need to test with OBS. FS initialization fails in this case. - return; - } + // Create a new directory, which in turn creates a new bucket. Path root = new Path("/" + volumeName); String bucketName = getBucketName(); @@ -192,31 +177,26 @@ public void testClientFsDefaultBucketLayout() throws Exception { adapter.createDirectory(dir1.toString()); + // Make sure the bucket layout of created bucket matches the config. OzoneBucket bucketInfo = objectStore.getClientProxy().getBucketDetails(volumeName, bucketName); - - if (defaultBucketLayout != null) { - // Make sure the bucket layout matches the config. - Assert.assertEquals(defaultBucketLayout, bucketInfo.getBucketLayout()); + if (StringUtils.isNotBlank(defaultBucketLayout)) { + Assert.assertEquals(defaultBucketLayout, + bucketInfo.getBucketLayout().name()); } else { - // In case we provide an empty config, the File System should use the - // default config. - Assert.assertEquals( - "FS should use default config value in case of empty config", - OzoneConfigKeys.OZONE_CLIENT_FS_BUCKET_LAYOUT_DEFAULT, - bucketInfo.getBucketLayout().toString()); + Assert.assertEquals(OzoneConfigKeys.OZONE_CLIENT_FS_BUCKET_LAYOUT_DEFAULT, + bucketInfo.getBucketLayout().name()); } } private String getBucketName() { String bucketSuffix; - if (defaultBucketLayout != null) { + if (StringUtils.isNotBlank(defaultBucketLayout)) { bucketSuffix = defaultBucketLayout - .toString() .toLowerCase() .replaceAll("_", "-"); } else { - bucketSuffix = "null"; + bucketSuffix = "empty"; } return "bucket-" + bucketSuffix; diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestObjectStoreWithLegacyFS.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestObjectStoreWithLegacyFS.java index 546b1762ada1..152e502b70a8 100644 --- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestObjectStoreWithLegacyFS.java +++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestObjectStoreWithLegacyFS.java @@ -40,6 +40,7 @@ import org.apache.hadoop.ozone.om.helpers.OmMultipartCommitUploadPartInfo; import org.apache.hadoop.ozone.om.helpers.OmMultipartUploadCompleteInfo; +import org.apache.ozone.test.GenericTestUtils; import org.junit.AfterClass; import org.junit.Assert; import org.junit.Before; @@ -118,6 +119,7 @@ public void init() throws Exception { @Test public void testFlatKeyStructureWithOBS() throws Exception { OzoneBucket ozoneBucket = volume.getBucket(bucketName); + String keyName = "dir1/dir2/dir3/key-1"; OzoneOutputStream stream = ozoneBucket .createKey("dir1/dir2/dir3/key-1", 0); stream.close(); @@ -125,24 +127,46 @@ public void testFlatKeyStructureWithOBS() throws Exception { cluster.getOzoneManager().getMetadataManager() .getKeyTable(BucketLayout.OBJECT_STORE); - TableIterator> - iterator = keyTable.iterator(); - String seekKey = "dir"; String dbKey = cluster.getOzoneManager().getMetadataManager() .getOzoneKey(volumeName, bucketName, seekKey); - iterator.seek(dbKey); + GenericTestUtils + .waitFor(() -> assertKeyCount(keyTable, dbKey, 1, keyName), 500, + 60000); + + ozoneBucket.renameKey(keyName, "dir1/NewKey-1"); + + GenericTestUtils + .waitFor(() -> assertKeyCount(keyTable, dbKey, 1, "dir1/NewKey-1"), 500, + 60000); + } + + private boolean assertKeyCount( + Table keyTable, + String dbKey, int expectedCnt, String keyName) { + TableIterator> + itr = keyTable.iterator(); int countKeys = 0; - while (iterator.hasNext()) { - Table.KeyValue keyValue = iterator.next(); - if (!keyValue.getKey().startsWith(dbKey)) { - break; + try { + itr.seek(dbKey); + while (itr.hasNext()) { + + Table.KeyValue keyValue = itr.next(); + if (!keyValue.getKey().startsWith(dbKey)) { + break; + } + countKeys++; + Assert.assertTrue(keyValue.getKey().endsWith(keyName)); } - countKeys++; - Assert.assertTrue(keyValue.getKey().endsWith("dir1/dir2/dir3/key-1")); + } catch (IOException ex) { + LOG.info("Test failed with: " + ex.getMessage(), ex); + Assert.fail("Test failed with: " + ex.getMessage()); + } + if (countKeys != expectedCnt) { + LOG.info("Couldn't find KeyName:{} in KeyTable, retrying...", keyName); } - Assert.assertEquals(1, countKeys); + return countKeys == expectedCnt; } @Test diff --git a/hadoop-ozone/ozonefs-common/src/main/java/org/apache/hadoop/fs/ozone/BasicRootedOzoneClientAdapterImpl.java b/hadoop-ozone/ozonefs-common/src/main/java/org/apache/hadoop/fs/ozone/BasicRootedOzoneClientAdapterImpl.java index fb8aa2925276..51ba277bf709 100644 --- a/hadoop-ozone/ozonefs-common/src/main/java/org/apache/hadoop/fs/ozone/BasicRootedOzoneClientAdapterImpl.java +++ b/hadoop-ozone/ozonefs-common/src/main/java/org/apache/hadoop/fs/ozone/BasicRootedOzoneClientAdapterImpl.java @@ -194,27 +194,7 @@ public BasicRootedOzoneClientAdapterImpl(String omHost, int omPort, OzoneConfigKeys.DFS_CONTAINER_IPC_PORT_DEFAULT); // Fetches the bucket layout to be used by OFS. - try { - this.defaultOFSBucketLayout = BucketLayout.fromString( - conf.get(OzoneConfigKeys.OZONE_CLIENT_FS_DEFAULT_BUCKET_LAYOUT, - OzoneConfigKeys.OZONE_CLIENT_FS_BUCKET_LAYOUT_DEFAULT)); - } catch (IllegalArgumentException iae) { - throw new OMException("Unsupported value provided for " + - OzoneConfigKeys.OZONE_CLIENT_FS_DEFAULT_BUCKET_LAYOUT + - ". Supported values are " + BucketLayout.FILE_SYSTEM_OPTIMIZED + - " and " + BucketLayout.LEGACY + ".", - OMException.ResultCodes.INVALID_REQUEST); - } - - // Bucket Layout for buckets created with OFS cannot be OBJECT_STORE. - if (defaultOFSBucketLayout.equals(BucketLayout.OBJECT_STORE)) { - throw new OMException( - "Buckets created with OBJECT_STORE layout do not support file " + - "system semantics. Supported values for config " + - OzoneConfigKeys.OZONE_CLIENT_FS_DEFAULT_BUCKET_LAYOUT + - " include " + BucketLayout.FILE_SYSTEM_OPTIMIZED + " and " + - BucketLayout.LEGACY, OMException.ResultCodes.INVALID_REQUEST); - } + initDefaultFsBucketLayout(conf); config = conf; } finally { @@ -222,6 +202,37 @@ public BasicRootedOzoneClientAdapterImpl(String omHost, int omPort, } } + /** + * Initialize the default bucket layout to be used by OFS. + * + * @param conf OzoneConfiguration + * @throws OMException In case of unsupported value provided in the config. + */ + private void initDefaultFsBucketLayout(OzoneConfiguration conf) + throws OMException { + try { + this.defaultOFSBucketLayout = BucketLayout.fromString( + conf.get(OzoneConfigKeys.OZONE_CLIENT_FS_DEFAULT_BUCKET_LAYOUT, + OzoneConfigKeys.OZONE_CLIENT_FS_BUCKET_LAYOUT_DEFAULT)); + } catch (IllegalArgumentException iae) { + throw new OMException("Unsupported value provided for " + + OzoneConfigKeys.OZONE_CLIENT_FS_DEFAULT_BUCKET_LAYOUT + + ". Supported values are " + BucketLayout.FILE_SYSTEM_OPTIMIZED + + " and " + BucketLayout.LEGACY + ".", + OMException.ResultCodes.INVALID_REQUEST); + } + + // Bucket Layout for buckets created with OFS cannot be OBJECT_STORE. + if (defaultOFSBucketLayout.equals(BucketLayout.OBJECT_STORE)) { + throw new OMException( + "Buckets created with OBJECT_STORE layout do not support file " + + "system semantics. Supported values for config " + + OzoneConfigKeys.OZONE_CLIENT_FS_DEFAULT_BUCKET_LAYOUT + + " include " + BucketLayout.FILE_SYSTEM_OPTIMIZED + " and " + + BucketLayout.LEGACY, OMException.ResultCodes.INVALID_REQUEST); + } + } + OzoneBucket getBucket(OFSPath ofsPath, boolean createIfNotExist) throws IOException { From 2058552e23170bfde995d4cddee2a219014a7cc7 Mon Sep 17 00:00:00 2001 From: Jyotinder Singh Date: Tue, 21 Jun 2022 15:05:50 +0530 Subject: [PATCH 6/8] findbugs.sh --- .../apache/hadoop/fs/ozone/TestOzoneFSBucketLayout.java | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/fs/ozone/TestOzoneFSBucketLayout.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/fs/ozone/TestOzoneFSBucketLayout.java index 078f40e89c4b..f80777eac562 100644 --- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/fs/ozone/TestOzoneFSBucketLayout.java +++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/fs/ozone/TestOzoneFSBucketLayout.java @@ -63,9 +63,8 @@ public class TestOzoneFSBucketLayout { private static String defaultBucketLayout; private static MiniOzoneCluster cluster = null; private static ObjectStore objectStore; - private static BasicRootedOzoneClientAdapterImpl adapter; + private BasicRootedOzoneClientAdapterImpl adapter; private static String rootPath; - private static FileSystem fs; private static String volumeName; private static Path volumePath; @@ -131,7 +130,6 @@ public static void initCluster() throws Exception { @AfterClass public static void teardown() throws IOException { // Tear down the cluster after EACH set of parameters - IOUtils.closeQuietly(fs); if (cluster != null) { cluster.shutdown(); } @@ -151,6 +149,7 @@ public void testFileSystemBucketLayoutConfiguration() throws IOException { // In case OZONE_CLIENT_FS_DEFAULT_BUCKET_LAYOUT is set to OBS, // FS initialization should fail. + FileSystem fs; if (ERROR_MAP.containsKey(defaultBucketLayout)) { try { fs = FileSystem.newInstance(conf); @@ -187,6 +186,9 @@ public void testFileSystemBucketLayoutConfiguration() throws IOException { Assert.assertEquals(OzoneConfigKeys.OZONE_CLIENT_FS_BUCKET_LAYOUT_DEFAULT, bucketInfo.getBucketLayout().name()); } + + // cleanup + IOUtils.closeQuietly(fs); } private String getBucketName() { From 8bb6577f29f7eb171f12049643331c0f20f96441 Mon Sep 17 00:00:00 2001 From: Jyotinder Singh Date: Tue, 21 Jun 2022 16:36:20 +0530 Subject: [PATCH 7/8] findbugs.sh --- .../org/apache/hadoop/fs/ozone/TestOzoneFSBucketLayout.java | 3 +++ 1 file changed, 3 insertions(+) diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/fs/ozone/TestOzoneFSBucketLayout.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/fs/ozone/TestOzoneFSBucketLayout.java index f80777eac562..37e8121a9071 100644 --- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/fs/ozone/TestOzoneFSBucketLayout.java +++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/fs/ozone/TestOzoneFSBucketLayout.java @@ -156,6 +156,9 @@ public void testFileSystemBucketLayoutConfiguration() throws IOException { Assert.fail("File System initialization should fail in case " + " of invalid configuration of " + OzoneConfigKeys.OZONE_CLIENT_FS_DEFAULT_BUCKET_LAYOUT); + // Dummy read op to avoid compiler warning. This is not expected to + // be executed. + fs.mkdirs(new Path("/dummy/operation/for/findbugs/warning")); } catch (OMException oe) { Assert.assertTrue( oe.getMessage().contains(ERROR_MAP.get(defaultBucketLayout))); From 03f65c510d1bc69cc1ffd15da35a794b57357151 Mon Sep 17 00:00:00 2001 From: Jyotinder Singh Date: Tue, 21 Jun 2022 16:54:05 +0530 Subject: [PATCH 8/8] findbugs.sh --- .../apache/hadoop/fs/ozone/TestOzoneFSBucketLayout.java | 9 +++------ 1 file changed, 3 insertions(+), 6 deletions(-) diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/fs/ozone/TestOzoneFSBucketLayout.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/fs/ozone/TestOzoneFSBucketLayout.java index 37e8121a9071..3dda1d21a810 100644 --- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/fs/ozone/TestOzoneFSBucketLayout.java +++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/fs/ozone/TestOzoneFSBucketLayout.java @@ -149,16 +149,13 @@ public void testFileSystemBucketLayoutConfiguration() throws IOException { // In case OZONE_CLIENT_FS_DEFAULT_BUCKET_LAYOUT is set to OBS, // FS initialization should fail. - FileSystem fs; + if (ERROR_MAP.containsKey(defaultBucketLayout)) { try { - fs = FileSystem.newInstance(conf); + FileSystem.newInstance(conf); Assert.fail("File System initialization should fail in case " + " of invalid configuration of " + OzoneConfigKeys.OZONE_CLIENT_FS_DEFAULT_BUCKET_LAYOUT); - // Dummy read op to avoid compiler warning. This is not expected to - // be executed. - fs.mkdirs(new Path("/dummy/operation/for/findbugs/warning")); } catch (OMException oe) { Assert.assertTrue( oe.getMessage().contains(ERROR_MAP.get(defaultBucketLayout))); @@ -167,7 +164,7 @@ public void testFileSystemBucketLayoutConfiguration() throws IOException { } // initialize FS and adapter. - fs = FileSystem.newInstance(conf); + FileSystem fs = FileSystem.newInstance(conf); RootedOzoneFileSystem ofs = (RootedOzoneFileSystem) fs; adapter = (BasicRootedOzoneClientAdapterImpl) ofs.getAdapter();