From 251b16a0927b1fb76e25051a49bcf4dc2173d94f Mon Sep 17 00:00:00 2001 From: "Doroszlai, Attila" Date: Sat, 15 Mar 2025 16:42:01 +0100 Subject: [PATCH 01/10] HDDS-12608. Race condition in datanode version file creation --- .../src/main/java/org/apache/hadoop/hdds/server/YamlUtils.java | 3 ++- .../src/main/java/org/apache/hadoop/hdds/utils/IOUtils.java | 3 ++- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/server/YamlUtils.java b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/server/YamlUtils.java index 5699978e3246..f644a62843e0 100644 --- a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/server/YamlUtils.java +++ b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/server/YamlUtils.java @@ -55,7 +55,8 @@ private static Yaml getYamlForLoad() { } public static void dump(Yaml yaml, Object data, File file, Logger log) throws IOException { - try (OutputStream out = new AtomicFileOutputStream(file); + File tempFile = File.createTempFile("temp", "yaml", file.getParentFile()); + try (OutputStream out = new AtomicFileOutputStream(file, tempFile); OutputStreamWriter writer = new OutputStreamWriter(out, StandardCharsets.UTF_8)) { yaml.dump(data, writer); } catch (IOException e) { diff --git a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/utils/IOUtils.java b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/utils/IOUtils.java index ce42c9660e45..382f2f474ed3 100644 --- a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/utils/IOUtils.java +++ b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/utils/IOUtils.java @@ -106,7 +106,8 @@ public static void closeQuietly(Collection closeables) /** Write {@code properties} to the file at {@code path}, truncating any existing content. */ public static void writePropertiesToFile(File file, Properties properties) throws IOException { - try (OutputStream out = new AtomicFileOutputStream(file)) { + File tempFile = File.createTempFile("temp", "properties", file.getParentFile()); + try (OutputStream out = new AtomicFileOutputStream(file, tempFile)) { properties.store(out, null); } } From 98699e3089bb465f796aa2e90b78603a39d3d853 Mon Sep 17 00:00:00 2001 From: "Doroszlai, Attila" Date: Mon, 17 Mar 2025 07:20:17 +0100 Subject: [PATCH 02/10] Revert "HDDS-12608. Race condition in datanode version file creation" This reverts commit 251b16a0927b1fb76e25051a49bcf4dc2173d94f. --- .../src/main/java/org/apache/hadoop/hdds/server/YamlUtils.java | 3 +-- .../src/main/java/org/apache/hadoop/hdds/utils/IOUtils.java | 3 +-- 2 files changed, 2 insertions(+), 4 deletions(-) diff --git a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/server/YamlUtils.java b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/server/YamlUtils.java index f644a62843e0..5699978e3246 100644 --- a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/server/YamlUtils.java +++ b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/server/YamlUtils.java @@ -55,8 +55,7 @@ private static Yaml getYamlForLoad() { } public static void dump(Yaml yaml, Object data, File file, Logger log) throws IOException { - File tempFile = File.createTempFile("temp", "yaml", file.getParentFile()); - try (OutputStream out = new AtomicFileOutputStream(file, tempFile); + try (OutputStream out = new AtomicFileOutputStream(file); OutputStreamWriter writer = new OutputStreamWriter(out, StandardCharsets.UTF_8)) { yaml.dump(data, writer); } catch (IOException e) { diff --git a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/utils/IOUtils.java b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/utils/IOUtils.java index 382f2f474ed3..ce42c9660e45 100644 --- a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/utils/IOUtils.java +++ b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/utils/IOUtils.java @@ -106,8 +106,7 @@ public static void closeQuietly(Collection closeables) /** Write {@code properties} to the file at {@code path}, truncating any existing content. */ public static void writePropertiesToFile(File file, Properties properties) throws IOException { - File tempFile = File.createTempFile("temp", "properties", file.getParentFile()); - try (OutputStream out = new AtomicFileOutputStream(file, tempFile)) { + try (OutputStream out = new AtomicFileOutputStream(file)) { properties.store(out, null); } } From 4c9a1b66f7c600857a30388bb5cd3de6d264b547 Mon Sep 17 00:00:00 2001 From: "Doroszlai, Attila" Date: Mon, 17 Mar 2025 07:20:25 +0100 Subject: [PATCH 03/10] HDDS-12608. Race condition in datanode version file creation --- .../hadoop/ozone/HddsDatanodeService.java | 4 +- .../states/datanode/RunningDatanodeState.java | 3 +- .../states/endpoint/VersionEndpointTask.java | 68 ++----------------- .../common/utils/StorageVolumeUtil.java | 11 +-- .../container/ozoneimpl/OzoneContainer.java | 38 +++++++++++ .../hadoop/ozone/TestHddsDatanodeService.java | 3 +- .../container/common/ContainerTestUtils.java | 5 +- .../common/TestBlockDeletingService.java | 2 +- .../common/impl/TestContainerPersistence.java | 4 +- .../common/utils/TestStorageVolumeUtil.java | 12 +--- .../keyvalue/TestKeyValueBlockIterator.java | 2 +- .../keyvalue/TestKeyValueContainer.java | 11 ++- .../TestKeyValueContainerIntegrityChecks.java | 2 +- .../keyvalue/impl/TestBlockManagerImpl.java | 3 +- .../ozoneimpl/TestContainerReader.java | 14 ++-- .../ozoneimpl/TestOzoneContainer.java | 2 +- .../container/upgrade/UpgradeTestHelper.java | 3 +- .../ozone/container/common/TestEndPoint.java | 20 ++---- .../container/upgrade/TestUpgradeManager.java | 3 +- .../freon/ClosedContainerReplicator.java | 2 +- 20 files changed, 79 insertions(+), 133 deletions(-) diff --git a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/HddsDatanodeService.java b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/HddsDatanodeService.java index 5df8a235050e..7334e1885f7f 100644 --- a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/HddsDatanodeService.java +++ b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/HddsDatanodeService.java @@ -376,9 +376,7 @@ private void startRatisForTest() throws IOException { for (Map.Entry entry : volumeMap.entrySet()) { HddsVolume hddsVolume = (HddsVolume) entry.getValue(); - boolean result = StorageVolumeUtil.checkVolume(hddsVolume, clusterId, - clusterId, conf, LOG, null); - if (!result) { + if (!StorageVolumeUtil.checkVolume(hddsVolume, clusterId, LOG, null)) { volumeSet.failVolume(hddsVolume.getHddsRootDir().getPath()); } } diff --git a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/states/datanode/RunningDatanodeState.java b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/states/datanode/RunningDatanodeState.java index 9f843864aa5e..87bd7d63e411 100644 --- a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/states/datanode/RunningDatanodeState.java +++ b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/states/datanode/RunningDatanodeState.java @@ -121,8 +121,7 @@ private Callable buildEndPointTask( EndpointStateMachine endpoint) { switch (endpoint.getState()) { case GETVERSION: - return new VersionEndpointTask(endpoint, conf, - context.getParent().getContainer()); + return new VersionEndpointTask(endpoint, context.getParent().getContainer()); case REGISTER: return RegisterEndpointTask.newBuilder() .setConfig(conf) diff --git a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/states/endpoint/VersionEndpointTask.java b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/states/endpoint/VersionEndpointTask.java index 2fb7b9c69b21..639975e51172 100644 --- a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/states/endpoint/VersionEndpointTask.java +++ b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/states/endpoint/VersionEndpointTask.java @@ -17,18 +17,13 @@ package org.apache.hadoop.ozone.container.common.states.endpoint; -import com.google.common.base.Preconditions; import java.io.IOException; import java.net.BindException; +import java.util.Objects; import java.util.concurrent.Callable; -import org.apache.hadoop.hdds.conf.ConfigurationSource; import org.apache.hadoop.hdds.protocol.proto.StorageContainerDatanodeProtocolProtos.SCMVersionResponseProto; import org.apache.hadoop.ozone.OzoneConsts; -import org.apache.hadoop.ozone.container.common.DatanodeLayoutStorage; import org.apache.hadoop.ozone.container.common.statemachine.EndpointStateMachine; -import org.apache.hadoop.ozone.container.common.utils.StorageVolumeUtil; -import org.apache.hadoop.ozone.container.common.volume.MutableVolumeSet; -import org.apache.hadoop.ozone.container.common.volume.StorageVolume; import org.apache.hadoop.ozone.container.ozoneimpl.OzoneContainer; import org.apache.hadoop.ozone.protocol.VersionResponse; import org.apache.hadoop.util.DiskChecker.DiskOutOfSpaceException; @@ -43,22 +38,15 @@ public class VersionEndpointTask implements public static final Logger LOG = LoggerFactory.getLogger(VersionEndpointTask .class); private final EndpointStateMachine rpcEndPoint; - private final ConfigurationSource configuration; private final OzoneContainer ozoneContainer; public VersionEndpointTask(EndpointStateMachine rpcEndPoint, - ConfigurationSource conf, OzoneContainer container) { + OzoneContainer container) { this.rpcEndPoint = rpcEndPoint; - this.configuration = conf; this.ozoneContainer = container; } - /** - * Computes a result, or throws an exception if unable to do so. - * - * @return computed result - * @throws Exception if unable to compute a result - */ + /** Get version information from SCM, start {@link OzoneContainer}. */ @Override public EndpointStateMachine.EndPointStates call() throws Exception { rpcEndPoint.lock(); @@ -73,29 +61,11 @@ public EndpointStateMachine.EndPointStates call() throws Exception { rpcEndPoint.setVersion(response); if (!rpcEndPoint.isPassive()) { - // If end point is passive, datanode does not need to check volumes. - String scmId = response.getValue(OzoneConsts.SCM_ID); - String clusterId = response.getValue(OzoneConsts.CLUSTER_ID); - - Preconditions.checkNotNull(scmId, - "Reply from SCM: scmId cannot be null"); - Preconditions.checkNotNull(clusterId, + String clusterId = Objects.requireNonNull(response.getValue(OzoneConsts.CLUSTER_ID), "Reply from SCM: clusterId cannot be null"); - - // Check DbVolumes, format DbVolume at first register time. - checkVolumeSet(ozoneContainer.getDbVolumeSet(), scmId, clusterId); - - // Check HddsVolumes - checkVolumeSet(ozoneContainer.getVolumeSet(), scmId, clusterId); - - DatanodeLayoutStorage layoutStorage - = new DatanodeLayoutStorage(configuration); - layoutStorage.setClusterId(clusterId); - layoutStorage.persistCurrentState(); - - // Start the container services after getting the version information ozoneContainer.start(clusterId); } + EndpointStateMachine.EndPointStates nextState = rpcEndPoint.getState().getNextState(); rpcEndPoint.setState(nextState); @@ -113,32 +83,4 @@ public EndpointStateMachine.EndPointStates call() throws Exception { } return rpcEndPoint.getState(); } - - private void checkVolumeSet(MutableVolumeSet volumeSet, - String scmId, String clusterId) throws DiskOutOfSpaceException { - if (volumeSet == null) { - return; - } - - volumeSet.writeLock(); - try { - // If version file does not exist - // create version file and also set scm ID or cluster ID. - for (StorageVolume volume : volumeSet.getVolumeMap().values()) { - boolean result = StorageVolumeUtil.checkVolume(volume, - scmId, clusterId, configuration, LOG, - ozoneContainer.getDbVolumeSet()); - if (!result) { - volumeSet.failVolume(volume.getStorageDir().getPath()); - } - } - if (volumeSet.getVolumesList().isEmpty()) { - // All volumes are in inconsistent state - throw new DiskOutOfSpaceException( - "All configured Volumes are in Inconsistent State"); - } - } finally { - volumeSet.writeUnlock(); - } - } } diff --git a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/utils/StorageVolumeUtil.java b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/utils/StorageVolumeUtil.java index 5e6fe086a165..a5801d3e1244 100644 --- a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/utils/StorageVolumeUtil.java +++ b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/utils/StorageVolumeUtil.java @@ -25,7 +25,6 @@ import java.util.UUID; import java.util.stream.Collectors; import org.apache.commons.lang3.StringUtils; -import org.apache.hadoop.hdds.conf.ConfigurationSource; import org.apache.hadoop.ozone.OzoneConsts; import org.apache.hadoop.ozone.common.InconsistentStorageStateException; import org.apache.hadoop.ozone.container.common.HDDSVolumeLayoutVersion; @@ -184,16 +183,10 @@ public static String getProperty( * /hdds/} as a symlink pointing to {@code /hdds/} when SCM HA is finalized. * - * @param volume - * @param scmId - * @param clusterId - * @param conf - * @param logger - * @param dbVolumeSet * @return true - if volume is in consistent state, otherwise false. */ - public static boolean checkVolume(StorageVolume volume, String scmId, - String clusterId, ConfigurationSource conf, Logger logger, + public static boolean checkVolume(StorageVolume volume, + String clusterId, Logger logger, MutableVolumeSet dbVolumeSet) { File volumeRoot = volume.getStorageDir(); String volumeRootPath = volumeRoot.getPath(); diff --git a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/ozoneimpl/OzoneContainer.java b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/ozoneimpl/OzoneContainer.java index b3fa5133823e..112c322a89a1 100644 --- a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/ozoneimpl/OzoneContainer.java +++ b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/ozoneimpl/OzoneContainer.java @@ -65,6 +65,7 @@ import org.apache.hadoop.hdds.utils.db.Table; import org.apache.hadoop.hdds.utils.db.TableIterator; import org.apache.hadoop.ozone.HddsDatanodeService; +import org.apache.hadoop.ozone.container.common.DatanodeLayoutStorage; import org.apache.hadoop.ozone.container.common.helpers.ContainerMetrics; import org.apache.hadoop.ozone.container.common.impl.BlockDeletingService; import org.apache.hadoop.ozone.container.common.impl.ContainerSet; @@ -81,6 +82,7 @@ import org.apache.hadoop.ozone.container.common.transport.server.ratis.XceiverServerRatis; import org.apache.hadoop.ozone.container.common.utils.ContainerInspectorUtil; import org.apache.hadoop.ozone.container.common.utils.HddsVolumeUtil; +import org.apache.hadoop.ozone.container.common.utils.StorageVolumeUtil; import org.apache.hadoop.ozone.container.common.volume.HddsVolume; import org.apache.hadoop.ozone.container.common.volume.MutableVolumeSet; import org.apache.hadoop.ozone.container.common.volume.StorageVolume; @@ -484,6 +486,17 @@ public void start(String clusterId) throws IOException { return; } + // Check DbVolumes, format DbVolume at first register time. + checkVolumeSet(getDbVolumeSet(), clusterId); + + // Check HddsVolumes + checkVolumeSet(getVolumeSet(), clusterId); + + DatanodeLayoutStorage layoutStorage + = new DatanodeLayoutStorage(config); + layoutStorage.setClusterId(clusterId); + layoutStorage.persistCurrentState(); + buildContainerSet(); // Start background volume checks, which will begin after the configured @@ -657,4 +670,29 @@ public void compactDb() { } } + + private void checkVolumeSet(MutableVolumeSet volumeSet, String clusterId) + throws DiskOutOfSpaceException { + if (volumeSet == null) { + return; + } + + volumeSet.writeLock(); + try { + // If version file does not exist + // create version file and also set scm ID or cluster ID. + for (StorageVolume volume : volumeSet.getVolumeMap().values()) { + if (!StorageVolumeUtil.checkVolume(volume, clusterId, LOG, getDbVolumeSet())) { + volumeSet.failVolume(volume.getStorageDir().getPath()); + } + } + if (volumeSet.getVolumesList().isEmpty()) { + // All volumes are in inconsistent state + throw new DiskOutOfSpaceException( + "All configured Volumes are in Inconsistent State"); + } + } finally { + volumeSet.writeUnlock(); + } + } } diff --git a/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/TestHddsDatanodeService.java b/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/TestHddsDatanodeService.java index 63157450a19e..3256b313d275 100644 --- a/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/TestHddsDatanodeService.java +++ b/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/TestHddsDatanodeService.java @@ -128,8 +128,7 @@ public void testDeletedContainersClearedOnShutdown(String schemaVersion) assertInstanceOf(HddsVolume.class, volume); HddsVolume hddsVolume = (HddsVolume) volume; - StorageVolumeUtil.checkVolume(hddsVolume, clusterId, - clusterId, conf, LOG, null); + StorageVolumeUtil.checkVolume(hddsVolume, clusterId, LOG, null); // Create a container and move it under the tmp delete dir. KeyValueContainer container = ContainerTestUtils .addContainerToDeletedDir( diff --git a/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/common/ContainerTestUtils.java b/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/common/ContainerTestUtils.java index e33a4c4a3ce5..9dc2d832c980 100644 --- a/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/common/ContainerTestUtils.java +++ b/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/common/ContainerTestUtils.java @@ -191,7 +191,7 @@ public static void disableSchemaV3(OzoneConfiguration conf) { } public static void createDbInstancesForTestIfNeeded( - MutableVolumeSet hddsVolumeSet, String scmID, String clusterID, + MutableVolumeSet hddsVolumeSet, String clusterID, ConfigurationSource conf) { DatanodeConfiguration dc = conf.getObject(DatanodeConfiguration.class); if (!dc.getContainerSchemaV3Enabled()) { @@ -200,8 +200,7 @@ public static void createDbInstancesForTestIfNeeded( for (HddsVolume volume : StorageVolumeUtil.getHddsVolumesList( hddsVolumeSet.getVolumesList())) { - StorageVolumeUtil.checkVolume(volume, scmID, clusterID, conf, - null, null); + StorageVolumeUtil.checkVolume(volume, clusterID, null, null); } } diff --git a/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/common/TestBlockDeletingService.java b/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/common/TestBlockDeletingService.java index 4e8ad609f2b5..a86dcd9673a1 100644 --- a/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/common/TestBlockDeletingService.java +++ b/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/common/TestBlockDeletingService.java @@ -137,7 +137,7 @@ public void init() throws IOException { datanodeUuid = UUID.randomUUID().toString(); volumeSet = new MutableVolumeSet(datanodeUuid, scmId, conf, null, StorageVolume.VolumeType.DATA_VOLUME, null); - createDbInstancesForTestIfNeeded(volumeSet, scmId, scmId, conf); + createDbInstancesForTestIfNeeded(volumeSet, scmId, conf); } @AfterEach diff --git a/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/common/impl/TestContainerPersistence.java b/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/common/impl/TestContainerPersistence.java index 0a31b9746d57..86961af63b05 100644 --- a/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/common/impl/TestContainerPersistence.java +++ b/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/common/impl/TestContainerPersistence.java @@ -158,9 +158,7 @@ public void setupPaths() throws IOException { // Initialize volume directories. for (HddsVolume volume : StorageVolumeUtil.getHddsVolumesList( volumeSet.getVolumesList())) { - boolean success = StorageVolumeUtil.checkVolume(volume, SCM_ID, SCM_ID, - conf, null, null); - assertTrue(success); + assertTrue(StorageVolumeUtil.checkVolume(volume, SCM_ID, null, null)); } blockManager = new BlockManagerImpl(conf); chunkManager = ChunkManagerFactory.createChunkManager(conf, blockManager, diff --git a/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/common/utils/TestStorageVolumeUtil.java b/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/common/utils/TestStorageVolumeUtil.java index 92929c6641cc..7a76798e7db4 100644 --- a/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/common/utils/TestStorageVolumeUtil.java +++ b/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/common/utils/TestStorageVolumeUtil.java @@ -83,21 +83,15 @@ public void testCheckVolumeNoDupDbStoreCreated() throws IOException { .thenReturn(Collections.singletonList(dbVolume)); // check the dbVolume first for hddsVolume to use - boolean res = StorageVolumeUtil.checkVolume(dbVolume, CLUSTER_ID, - CLUSTER_ID, CONF, null, null); - assertTrue(res); + assertTrue(StorageVolumeUtil.checkVolume(dbVolume, CLUSTER_ID, null, null)); // checkVolume for the 1st time: rootFiles.length == 1 - res = StorageVolumeUtil.checkVolume(spyHddsVolume, CLUSTER_ID, - CLUSTER_ID, CONF, null, dbVolumeSet); - assertTrue(res); + assertTrue(StorageVolumeUtil.checkVolume(spyHddsVolume, CLUSTER_ID, null, dbVolumeSet)); // createDbStore called as expected verify(spyHddsVolume, times(1)).createDbStore(dbVolumeSet); // checkVolume for the 2nd time: rootFiles.length == 2 - res = StorageVolumeUtil.checkVolume(spyHddsVolume, CLUSTER_ID, - CLUSTER_ID, CONF, LOG, dbVolumeSet); - assertTrue(res); + assertTrue(StorageVolumeUtil.checkVolume(spyHddsVolume, CLUSTER_ID, LOG, dbVolumeSet)); // should only call createDbStore once, so no dup db instance verify(spyHddsVolume, times(1)).createDbStore(dbVolumeSet); diff --git a/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/keyvalue/TestKeyValueBlockIterator.java b/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/keyvalue/TestKeyValueBlockIterator.java index 577f65c2e583..08bc144d6f4b 100644 --- a/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/keyvalue/TestKeyValueBlockIterator.java +++ b/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/keyvalue/TestKeyValueBlockIterator.java @@ -109,7 +109,7 @@ public void setup() throws Exception { conf.set(OzoneConfigKeys.OZONE_METADATA_DIRS, testRoot.getAbsolutePath()); volumeSet = new MutableVolumeSet(datanodeID, clusterID, conf, null, StorageVolume.VolumeType.DATA_VOLUME, null); - createDbInstancesForTestIfNeeded(volumeSet, clusterID, clusterID, conf); + createDbInstancesForTestIfNeeded(volumeSet, clusterID, conf); containerData = new KeyValueContainerData(105L, layout, diff --git a/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/keyvalue/TestKeyValueContainer.java b/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/keyvalue/TestKeyValueContainer.java index 52478cf4b105..cdc7bdc73f67 100644 --- a/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/keyvalue/TestKeyValueContainer.java +++ b/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/keyvalue/TestKeyValueContainer.java @@ -143,8 +143,7 @@ private void init(ContainerTestVersionInfo versionInfo) throws Exception { hddsVolumes.add(new HddsVolume.Builder(folder.toString()) .conf(CONF).datanodeUuid(datanodeId .toString()).build()); - StorageVolumeUtil.checkVolume(hddsVolumes.get(0), scmId, scmId, CONF, - null, null); + StorageVolumeUtil.checkVolume(hddsVolumes.get(0), scmId, null, null); volumeSet = mock(MutableVolumeSet.class); volumeChoosingPolicy = mock(RoundRobinVolumeChoosingPolicy.class); @@ -226,7 +225,7 @@ public void testNextVolumeTriedOnWriteFailure( .toFile().getAbsolutePath(); HddsVolume newVolume = new HddsVolume.Builder(volumeDirPath) .conf(CONF).datanodeUuid(datanodeId.toString()).build(); - StorageVolumeUtil.checkVolume(newVolume, scmId, scmId, CONF, null, null); + StorageVolumeUtil.checkVolume(newVolume, scmId, null, null); hddsVolumes.add(newVolume); // Override the class, so that the first time we call it, it throws @@ -816,7 +815,7 @@ void testAutoCompactionSmallSstFile( .getAbsolutePath(); HddsVolume newVolume = new HddsVolume.Builder(volumeDirPath) .conf(CONF).datanodeUuid(datanodeId.toString()).build(); - StorageVolumeUtil.checkVolume(newVolume, scmId, scmId, CONF, null, null); + StorageVolumeUtil.checkVolume(newVolume, scmId, null, null); List volumeList = new ArrayList<>(); HddsVolume hddsVolume = hddsVolumes.get(0); volumeList.add(hddsVolume); @@ -1036,7 +1035,7 @@ private void testMixedSchemaImport(String dir, HddsVolume hddsVolume1 = new HddsVolume.Builder(dir1) .conf(conf).datanodeUuid(datanodeId.toString()).build(); conf.setBoolean(CONTAINER_SCHEMA_V3_ENABLED, schemaV3Enabled); - StorageVolumeUtil.checkVolume(hddsVolume1, scmId, scmId, conf, null, null); + StorageVolumeUtil.checkVolume(hddsVolume1, scmId, null, null); hddsVolumes.clear(); hddsVolumes.add(hddsVolume1); @@ -1080,7 +1079,7 @@ private void testMixedSchemaImport(String dir, final String dir2 = dir + (schemaV3Enabled ? "/v2" : "/v3"); HddsVolume hddsVolume2 = new HddsVolume.Builder(dir2) .conf(conf).datanodeUuid(datanodeId.toString()).build(); - StorageVolumeUtil.checkVolume(hddsVolume2, scmId, scmId, conf, null, null); + StorageVolumeUtil.checkVolume(hddsVolume2, scmId, null, null); hddsVolumes.clear(); hddsVolumes.add(hddsVolume2); diff --git a/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/keyvalue/TestKeyValueContainerIntegrityChecks.java b/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/keyvalue/TestKeyValueContainerIntegrityChecks.java index 3b4cfff93790..a2b0636e792c 100644 --- a/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/keyvalue/TestKeyValueContainerIntegrityChecks.java +++ b/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/keyvalue/TestKeyValueContainerIntegrityChecks.java @@ -91,7 +91,7 @@ private void setup() throws Exception { containerLayoutTestInfo.updateConfig(conf); volumeSet = new MutableVolumeSet(UUID.randomUUID().toString(), clusterID, conf, null, StorageVolume.VolumeType.DATA_VOLUME, null); - createDbInstancesForTestIfNeeded(volumeSet, clusterID, clusterID, conf); + createDbInstancesForTestIfNeeded(volumeSet, clusterID, conf); chunkManager = containerLayoutTestInfo.createChunkManager(true, null); } diff --git a/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/keyvalue/impl/TestBlockManagerImpl.java b/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/keyvalue/impl/TestBlockManagerImpl.java index 07414d701aad..4f89a1b01ed1 100644 --- a/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/keyvalue/impl/TestBlockManagerImpl.java +++ b/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/keyvalue/impl/TestBlockManagerImpl.java @@ -90,8 +90,7 @@ private void initilaze() throws Exception { .conf(config) .datanodeUuid(datanodeId.toString()) .build(); - StorageVolumeUtil.checkVolume(hddsVolume, scmId, scmId, config, - null, null); + StorageVolumeUtil.checkVolume(hddsVolume, scmId, null, null); volumeSet = mock(MutableVolumeSet.class); diff --git a/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/ozoneimpl/TestContainerReader.java b/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/ozoneimpl/TestContainerReader.java index e315ced9f48d..fe3e59490cea 100644 --- a/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/ozoneimpl/TestContainerReader.java +++ b/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/ozoneimpl/TestContainerReader.java @@ -109,8 +109,7 @@ private void setup(ContainerTestVersionInfo versionInfo) throws Exception { hddsVolume = new HddsVolume.Builder(volumeDir .getAbsolutePath()).conf(conf).datanodeUuid(datanodeId .toString()).clusterID(clusterId).build(); - StorageVolumeUtil.checkVolume(hddsVolume, clusterId, clusterId, conf, - null, null); + StorageVolumeUtil.checkVolume(hddsVolume, clusterId, null, null); volumeSet = mock(MutableVolumeSet.class); volumeChoosingPolicy = mock(RoundRobinVolumeChoosingPolicy.class); @@ -277,8 +276,7 @@ public void testContainerReaderWithLoadException( hddsVolume1 = new HddsVolume.Builder(volumeDir1 .getAbsolutePath()).conf(conf).datanodeUuid(datanode .toString()).clusterID(clusterId).build(); - StorageVolumeUtil.checkVolume(hddsVolume1, clusterId, clusterId, conf, - null, null); + StorageVolumeUtil.checkVolume(hddsVolume1, clusterId, null, null); volumeChoosingPolicy1 = mock(RoundRobinVolumeChoosingPolicy.class); when(volumeChoosingPolicy1.chooseVolume(anyList(), anyLong())) .thenReturn(hddsVolume1); @@ -327,8 +325,7 @@ public void testContainerReaderWithInvalidDbPath( hddsVolume1 = new HddsVolume.Builder(volumeDir1 .getAbsolutePath()).conf(conf).datanodeUuid(datanode .toString()).clusterID(clusterId).build(); - StorageVolumeUtil.checkVolume(hddsVolume1, clusterId, clusterId, conf, - null, null); + StorageVolumeUtil.checkVolume(hddsVolume1, clusterId, null, null); volumeChoosingPolicy1 = mock(RoundRobinVolumeChoosingPolicy.class); when(volumeChoosingPolicy1.chooseVolume(anyList(), anyLong())) .thenReturn(hddsVolume1); @@ -383,10 +380,9 @@ public void testMultipleContainerReader(ContainerTestVersionInfo versionInfo) new MutableVolumeSet(datanodeId.toString(), clusterId, conf, null, StorageVolume.VolumeType.DATA_VOLUME, null); for (StorageVolume v : volumeSets.getVolumesList()) { - StorageVolumeUtil.checkVolume(v, clusterId, clusterId, conf, - null, null); + StorageVolumeUtil.checkVolume(v, clusterId, null, null); } - createDbInstancesForTestIfNeeded(volumeSets, clusterId, clusterId, conf); + createDbInstancesForTestIfNeeded(volumeSets, clusterId, conf); ContainerCache cache = ContainerCache.getInstance(conf); cache.shutdownCache(); diff --git a/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/ozoneimpl/TestOzoneContainer.java b/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/ozoneimpl/TestOzoneContainer.java index 34619c19183f..ea1abd48d84e 100644 --- a/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/ozoneimpl/TestOzoneContainer.java +++ b/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/ozoneimpl/TestOzoneContainer.java @@ -97,7 +97,7 @@ private void setup() throws Exception { commitSpaceMap = new HashMap<>(); volumeSet = new MutableVolumeSet(datanodeDetails.getUuidString(), clusterId, conf, null, StorageVolume.VolumeType.DATA_VOLUME, null); - createDbInstancesForTestIfNeeded(volumeSet, clusterId, clusterId, conf); + createDbInstancesForTestIfNeeded(volumeSet, clusterId, conf); volumeChoosingPolicy = new RoundRobinVolumeChoosingPolicy(); } diff --git a/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/upgrade/UpgradeTestHelper.java b/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/upgrade/UpgradeTestHelper.java index 04bfdf30762b..769c79d88ac7 100644 --- a/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/upgrade/UpgradeTestHelper.java +++ b/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/upgrade/UpgradeTestHelper.java @@ -128,8 +128,7 @@ public static void callVersionEndpointTask( throws Exception { try (EndpointStateMachine esm = ContainerTestUtils.createEndpoint(conf, address, 1000)) { - VersionEndpointTask vet = new VersionEndpointTask(esm, conf, - container); + VersionEndpointTask vet = new VersionEndpointTask(esm, container); esm.setState(EndpointStateMachine.EndPointStates.GETVERSION); vet.call(); } diff --git a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/ozone/container/common/TestEndPoint.java b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/ozone/container/common/TestEndPoint.java index 64f903f45057..a877e9170974 100644 --- a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/ozone/container/common/TestEndPoint.java +++ b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/ozone/container/common/TestEndPoint.java @@ -157,8 +157,7 @@ public void testGetVersionTask() throws Exception { OzoneContainer ozoneContainer = new OzoneContainer(dnDetails, ozoneConf, ContainerTestUtils.getMockContext(dnDetails, ozoneConf)); rpcEndPoint.setState(EndpointStateMachine.EndPointStates.GETVERSION); - VersionEndpointTask versionTask = new VersionEndpointTask(rpcEndPoint, - ozoneConf, ozoneContainer); + VersionEndpointTask versionTask = new VersionEndpointTask(rpcEndPoint, ozoneContainer); EndpointStateMachine.EndPointStates newState = versionTask.call(); // if version call worked the endpoint should automatically move to the @@ -200,8 +199,7 @@ public void testDeletedContainersClearedOnStartup() throws Exception { rpcEndPoint.setState(EndpointStateMachine.EndPointStates.GETVERSION); // versionTask.call() cleans the tmp dir and removes container from DB - VersionEndpointTask versionTask = new VersionEndpointTask(rpcEndPoint, - ozoneConf, ozoneContainer); + VersionEndpointTask versionTask = new VersionEndpointTask(rpcEndPoint, ozoneContainer); EndpointStateMachine.EndPointStates newState = versionTask.call(); assertEquals(EndpointStateMachine.EndPointStates.REGISTER, newState); @@ -232,8 +230,7 @@ public void testCheckVersionResponse() throws Exception { OzoneContainer ozoneContainer = new OzoneContainer(dnDetails, ozoneConf, ContainerTestUtils.getMockContext(dnDetails, ozoneConf)); rpcEndPoint.setState(EndpointStateMachine.EndPointStates.GETVERSION); - VersionEndpointTask versionTask = new VersionEndpointTask(rpcEndPoint, - ozoneConf, ozoneContainer); + VersionEndpointTask versionTask = new VersionEndpointTask(rpcEndPoint, ozoneContainer); EndpointStateMachine.EndPointStates newState = versionTask.call(); // if version call worked the endpoint should automatically move to the @@ -275,8 +272,7 @@ public void testDnLayoutVersionFile() throws Exception { OzoneContainer ozoneContainer = new OzoneContainer(dnDetails, ozoneConf, ContainerTestUtils.getMockContext(dnDetails, ozoneConf)); rpcEndPoint.setState(EndpointStateMachine.EndPointStates.GETVERSION); - VersionEndpointTask versionTask = new VersionEndpointTask(rpcEndPoint, - ozoneConf, ozoneContainer); + VersionEndpointTask versionTask = new VersionEndpointTask(rpcEndPoint, ozoneContainer); versionTask.call(); // After the version call, the datanode layout file should @@ -329,8 +325,7 @@ public void testGetVersionToInvalidEndpoint() throws Exception { DatanodeDetails datanodeDetails = randomDatanodeDetails(); OzoneContainer ozoneContainer = new OzoneContainer(datanodeDetails, conf, ContainerTestUtils.getMockContext(datanodeDetails, ozoneConf)); - VersionEndpointTask versionTask = new VersionEndpointTask(rpcEndPoint, - conf, ozoneContainer); + VersionEndpointTask versionTask = new VersionEndpointTask(rpcEndPoint, ozoneContainer); EndpointStateMachine.EndPointStates newState = versionTask.call(); // This version call did NOT work, so endpoint should remain in the same @@ -356,8 +351,7 @@ public void testGetVersionAssertRpcTimeOut() throws Exception { DatanodeDetails datanodeDetails = randomDatanodeDetails(); OzoneContainer ozoneContainer = new OzoneContainer(datanodeDetails, conf, ContainerTestUtils.getMockContext(datanodeDetails, ozoneConf)); - VersionEndpointTask versionTask = new VersionEndpointTask(rpcEndPoint, - conf, ozoneContainer); + VersionEndpointTask versionTask = new VersionEndpointTask(rpcEndPoint, ozoneContainer); scmServerImpl.setRpcResponseDelay(1500); long start = Time.monotonicNow(); @@ -654,7 +648,7 @@ private OzoneContainer createVolume(OzoneConfiguration conf) MutableVolumeSet volumeSet = ozoneContainer.getVolumeSet(); ContainerTestUtils.createDbInstancesForTestIfNeeded(volumeSet, - clusterId, clusterId, conf); + clusterId, conf); // VolumeSet for this test, contains only 1 volume assertEquals(1, volumeSet.getVolumesList().size()); StorageVolume volume = volumeSet.getVolumesList().get(0); diff --git a/hadoop-hdds/tools/src/test/java/org/apache/hadoop/hdds/scm/cli/container/upgrade/TestUpgradeManager.java b/hadoop-hdds/tools/src/test/java/org/apache/hadoop/hdds/scm/cli/container/upgrade/TestUpgradeManager.java index 4d9501887678..f9165e9ad86e 100644 --- a/hadoop-hdds/tools/src/test/java/org/apache/hadoop/hdds/scm/cli/container/upgrade/TestUpgradeManager.java +++ b/hadoop-hdds/tools/src/test/java/org/apache/hadoop/hdds/scm/cli/container/upgrade/TestUpgradeManager.java @@ -116,8 +116,7 @@ public void setup() throws Exception { final List volumes = new ArrayList<>(); for (StorageVolume storageVolume : volumeSet.getVolumesList()) { HddsVolume hddsVolume = (HddsVolume) storageVolume; - StorageVolumeUtil.checkVolume(hddsVolume, SCM_ID, SCM_ID, CONF, null, - null); + StorageVolumeUtil.checkVolume(hddsVolume, SCM_ID, null, null); volumes.add(hddsVolume); } diff --git a/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/freon/ClosedContainerReplicator.java b/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/freon/ClosedContainerReplicator.java index 8d4a4e99af57..391fd244e1f1 100644 --- a/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/freon/ClosedContainerReplicator.java +++ b/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/freon/ClosedContainerReplicator.java @@ -211,7 +211,7 @@ private void initializeReplicationSupervisor( volumeSet, dbVolumeSet, false, LOG); for (StorageVolume volume : volumeSet.getVolumesList()) { - StorageVolumeUtil.checkVolume(volume, scmID, clusterID, conf, LOG, dbVolumeSet); + StorageVolumeUtil.checkVolume(volume, clusterID, LOG, dbVolumeSet); } } From 6c892f245d28bfcfefa593536dbba98abb72530f Mon Sep 17 00:00:00 2001 From: "Doroszlai, Attila" Date: Mon, 17 Mar 2025 07:25:15 +0100 Subject: [PATCH 04/10] fix checkstyle --- .../ozone/container/ozoneimpl/OzoneContainer.java | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/ozoneimpl/OzoneContainer.java b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/ozoneimpl/OzoneContainer.java index 112c322a89a1..2e6483376873 100644 --- a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/ozoneimpl/OzoneContainer.java +++ b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/ozoneimpl/OzoneContainer.java @@ -671,28 +671,28 @@ public void compactDb() { } - private void checkVolumeSet(MutableVolumeSet volumeSet, String clusterId) + private void checkVolumeSet(MutableVolumeSet set, String clusterId) throws DiskOutOfSpaceException { - if (volumeSet == null) { + if (set == null) { return; } - volumeSet.writeLock(); + set.writeLock(); try { // If version file does not exist // create version file and also set scm ID or cluster ID. - for (StorageVolume volume : volumeSet.getVolumeMap().values()) { + for (StorageVolume volume : set.getVolumeMap().values()) { if (!StorageVolumeUtil.checkVolume(volume, clusterId, LOG, getDbVolumeSet())) { - volumeSet.failVolume(volume.getStorageDir().getPath()); + set.failVolume(volume.getStorageDir().getPath()); } } - if (volumeSet.getVolumesList().isEmpty()) { + if (set.getVolumesList().isEmpty()) { // All volumes are in inconsistent state throw new DiskOutOfSpaceException( "All configured Volumes are in Inconsistent State"); } } finally { - volumeSet.writeUnlock(); + set.writeUnlock(); } } } From 9d7cf17e249a71c536dcde35516beb2741a25986 Mon Sep 17 00:00:00 2001 From: "Doroszlai, Attila" Date: Mon, 17 Mar 2025 08:09:00 +0100 Subject: [PATCH 05/10] fix findbugs --- .../org/apache/hadoop/ozone/freon/ClosedContainerReplicator.java | 1 - 1 file changed, 1 deletion(-) diff --git a/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/freon/ClosedContainerReplicator.java b/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/freon/ClosedContainerReplicator.java index 391fd244e1f1..458c3e769315 100644 --- a/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/freon/ClosedContainerReplicator.java +++ b/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/freon/ClosedContainerReplicator.java @@ -184,7 +184,6 @@ private void checkDestinationDirectory(String dirUrl) throws IOException { private void initializeReplicationSupervisor( ConfigurationSource conf, int queueSize) throws IOException { - String scmID = UUID.randomUUID().toString(); String clusterID = UUID.randomUUID().toString(); String fakeDatanodeUuid = datanode; From 1a92ae24fb18ef3105a42d9db0cf658dacdaa0d6 Mon Sep 17 00:00:00 2001 From: "Doroszlai, Attila" Date: Mon, 17 Mar 2025 09:21:20 +0100 Subject: [PATCH 06/10] Revert "fix findbugs" This reverts commit 9d7cf17e249a71c536dcde35516beb2741a25986. --- .../org/apache/hadoop/ozone/freon/ClosedContainerReplicator.java | 1 + 1 file changed, 1 insertion(+) diff --git a/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/freon/ClosedContainerReplicator.java b/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/freon/ClosedContainerReplicator.java index 458c3e769315..391fd244e1f1 100644 --- a/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/freon/ClosedContainerReplicator.java +++ b/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/freon/ClosedContainerReplicator.java @@ -184,6 +184,7 @@ private void checkDestinationDirectory(String dirUrl) throws IOException { private void initializeReplicationSupervisor( ConfigurationSource conf, int queueSize) throws IOException { + String scmID = UUID.randomUUID().toString(); String clusterID = UUID.randomUUID().toString(); String fakeDatanodeUuid = datanode; From db6dc2ec0436610d5a6ed2cccd9605dc55565b0f Mon Sep 17 00:00:00 2001 From: "Doroszlai, Attila" Date: Mon, 17 Mar 2025 09:21:20 +0100 Subject: [PATCH 07/10] Revert "fix checkstyle" This reverts commit 6c892f245d28bfcfefa593536dbba98abb72530f. --- .../ozone/container/ozoneimpl/OzoneContainer.java | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/ozoneimpl/OzoneContainer.java b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/ozoneimpl/OzoneContainer.java index 2e6483376873..112c322a89a1 100644 --- a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/ozoneimpl/OzoneContainer.java +++ b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/ozoneimpl/OzoneContainer.java @@ -671,28 +671,28 @@ public void compactDb() { } - private void checkVolumeSet(MutableVolumeSet set, String clusterId) + private void checkVolumeSet(MutableVolumeSet volumeSet, String clusterId) throws DiskOutOfSpaceException { - if (set == null) { + if (volumeSet == null) { return; } - set.writeLock(); + volumeSet.writeLock(); try { // If version file does not exist // create version file and also set scm ID or cluster ID. - for (StorageVolume volume : set.getVolumeMap().values()) { + for (StorageVolume volume : volumeSet.getVolumeMap().values()) { if (!StorageVolumeUtil.checkVolume(volume, clusterId, LOG, getDbVolumeSet())) { - set.failVolume(volume.getStorageDir().getPath()); + volumeSet.failVolume(volume.getStorageDir().getPath()); } } - if (set.getVolumesList().isEmpty()) { + if (volumeSet.getVolumesList().isEmpty()) { // All volumes are in inconsistent state throw new DiskOutOfSpaceException( "All configured Volumes are in Inconsistent State"); } } finally { - set.writeUnlock(); + volumeSet.writeUnlock(); } } } From cc8aeebe975a5227721ce9c80b9c814c28b1340a Mon Sep 17 00:00:00 2001 From: "Doroszlai, Attila" Date: Mon, 17 Mar 2025 09:21:20 +0100 Subject: [PATCH 08/10] Revert "HDDS-12608. Race condition in datanode version file creation" This reverts commit 4c9a1b66f7c600857a30388bb5cd3de6d264b547. --- .../hadoop/ozone/HddsDatanodeService.java | 4 +- .../states/datanode/RunningDatanodeState.java | 3 +- .../states/endpoint/VersionEndpointTask.java | 68 +++++++++++++++++-- .../common/utils/StorageVolumeUtil.java | 11 ++- .../container/ozoneimpl/OzoneContainer.java | 38 ----------- .../hadoop/ozone/TestHddsDatanodeService.java | 3 +- .../container/common/ContainerTestUtils.java | 5 +- .../common/TestBlockDeletingService.java | 2 +- .../common/impl/TestContainerPersistence.java | 4 +- .../common/utils/TestStorageVolumeUtil.java | 12 +++- .../keyvalue/TestKeyValueBlockIterator.java | 2 +- .../keyvalue/TestKeyValueContainer.java | 11 +-- .../TestKeyValueContainerIntegrityChecks.java | 2 +- .../keyvalue/impl/TestBlockManagerImpl.java | 3 +- .../ozoneimpl/TestContainerReader.java | 14 ++-- .../ozoneimpl/TestOzoneContainer.java | 2 +- .../container/upgrade/UpgradeTestHelper.java | 3 +- .../ozone/container/common/TestEndPoint.java | 20 ++++-- .../container/upgrade/TestUpgradeManager.java | 3 +- .../freon/ClosedContainerReplicator.java | 2 +- 20 files changed, 133 insertions(+), 79 deletions(-) diff --git a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/HddsDatanodeService.java b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/HddsDatanodeService.java index 7334e1885f7f..5df8a235050e 100644 --- a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/HddsDatanodeService.java +++ b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/HddsDatanodeService.java @@ -376,7 +376,9 @@ private void startRatisForTest() throws IOException { for (Map.Entry entry : volumeMap.entrySet()) { HddsVolume hddsVolume = (HddsVolume) entry.getValue(); - if (!StorageVolumeUtil.checkVolume(hddsVolume, clusterId, LOG, null)) { + boolean result = StorageVolumeUtil.checkVolume(hddsVolume, clusterId, + clusterId, conf, LOG, null); + if (!result) { volumeSet.failVolume(hddsVolume.getHddsRootDir().getPath()); } } diff --git a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/states/datanode/RunningDatanodeState.java b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/states/datanode/RunningDatanodeState.java index 87bd7d63e411..9f843864aa5e 100644 --- a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/states/datanode/RunningDatanodeState.java +++ b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/states/datanode/RunningDatanodeState.java @@ -121,7 +121,8 @@ private Callable buildEndPointTask( EndpointStateMachine endpoint) { switch (endpoint.getState()) { case GETVERSION: - return new VersionEndpointTask(endpoint, context.getParent().getContainer()); + return new VersionEndpointTask(endpoint, conf, + context.getParent().getContainer()); case REGISTER: return RegisterEndpointTask.newBuilder() .setConfig(conf) diff --git a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/states/endpoint/VersionEndpointTask.java b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/states/endpoint/VersionEndpointTask.java index 639975e51172..2fb7b9c69b21 100644 --- a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/states/endpoint/VersionEndpointTask.java +++ b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/states/endpoint/VersionEndpointTask.java @@ -17,13 +17,18 @@ package org.apache.hadoop.ozone.container.common.states.endpoint; +import com.google.common.base.Preconditions; import java.io.IOException; import java.net.BindException; -import java.util.Objects; import java.util.concurrent.Callable; +import org.apache.hadoop.hdds.conf.ConfigurationSource; import org.apache.hadoop.hdds.protocol.proto.StorageContainerDatanodeProtocolProtos.SCMVersionResponseProto; import org.apache.hadoop.ozone.OzoneConsts; +import org.apache.hadoop.ozone.container.common.DatanodeLayoutStorage; import org.apache.hadoop.ozone.container.common.statemachine.EndpointStateMachine; +import org.apache.hadoop.ozone.container.common.utils.StorageVolumeUtil; +import org.apache.hadoop.ozone.container.common.volume.MutableVolumeSet; +import org.apache.hadoop.ozone.container.common.volume.StorageVolume; import org.apache.hadoop.ozone.container.ozoneimpl.OzoneContainer; import org.apache.hadoop.ozone.protocol.VersionResponse; import org.apache.hadoop.util.DiskChecker.DiskOutOfSpaceException; @@ -38,15 +43,22 @@ public class VersionEndpointTask implements public static final Logger LOG = LoggerFactory.getLogger(VersionEndpointTask .class); private final EndpointStateMachine rpcEndPoint; + private final ConfigurationSource configuration; private final OzoneContainer ozoneContainer; public VersionEndpointTask(EndpointStateMachine rpcEndPoint, - OzoneContainer container) { + ConfigurationSource conf, OzoneContainer container) { this.rpcEndPoint = rpcEndPoint; + this.configuration = conf; this.ozoneContainer = container; } - /** Get version information from SCM, start {@link OzoneContainer}. */ + /** + * Computes a result, or throws an exception if unable to do so. + * + * @return computed result + * @throws Exception if unable to compute a result + */ @Override public EndpointStateMachine.EndPointStates call() throws Exception { rpcEndPoint.lock(); @@ -61,11 +73,29 @@ public EndpointStateMachine.EndPointStates call() throws Exception { rpcEndPoint.setVersion(response); if (!rpcEndPoint.isPassive()) { - String clusterId = Objects.requireNonNull(response.getValue(OzoneConsts.CLUSTER_ID), + // If end point is passive, datanode does not need to check volumes. + String scmId = response.getValue(OzoneConsts.SCM_ID); + String clusterId = response.getValue(OzoneConsts.CLUSTER_ID); + + Preconditions.checkNotNull(scmId, + "Reply from SCM: scmId cannot be null"); + Preconditions.checkNotNull(clusterId, "Reply from SCM: clusterId cannot be null"); + + // Check DbVolumes, format DbVolume at first register time. + checkVolumeSet(ozoneContainer.getDbVolumeSet(), scmId, clusterId); + + // Check HddsVolumes + checkVolumeSet(ozoneContainer.getVolumeSet(), scmId, clusterId); + + DatanodeLayoutStorage layoutStorage + = new DatanodeLayoutStorage(configuration); + layoutStorage.setClusterId(clusterId); + layoutStorage.persistCurrentState(); + + // Start the container services after getting the version information ozoneContainer.start(clusterId); } - EndpointStateMachine.EndPointStates nextState = rpcEndPoint.getState().getNextState(); rpcEndPoint.setState(nextState); @@ -83,4 +113,32 @@ public EndpointStateMachine.EndPointStates call() throws Exception { } return rpcEndPoint.getState(); } + + private void checkVolumeSet(MutableVolumeSet volumeSet, + String scmId, String clusterId) throws DiskOutOfSpaceException { + if (volumeSet == null) { + return; + } + + volumeSet.writeLock(); + try { + // If version file does not exist + // create version file and also set scm ID or cluster ID. + for (StorageVolume volume : volumeSet.getVolumeMap().values()) { + boolean result = StorageVolumeUtil.checkVolume(volume, + scmId, clusterId, configuration, LOG, + ozoneContainer.getDbVolumeSet()); + if (!result) { + volumeSet.failVolume(volume.getStorageDir().getPath()); + } + } + if (volumeSet.getVolumesList().isEmpty()) { + // All volumes are in inconsistent state + throw new DiskOutOfSpaceException( + "All configured Volumes are in Inconsistent State"); + } + } finally { + volumeSet.writeUnlock(); + } + } } diff --git a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/utils/StorageVolumeUtil.java b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/utils/StorageVolumeUtil.java index a5801d3e1244..5e6fe086a165 100644 --- a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/utils/StorageVolumeUtil.java +++ b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/utils/StorageVolumeUtil.java @@ -25,6 +25,7 @@ import java.util.UUID; import java.util.stream.Collectors; import org.apache.commons.lang3.StringUtils; +import org.apache.hadoop.hdds.conf.ConfigurationSource; import org.apache.hadoop.ozone.OzoneConsts; import org.apache.hadoop.ozone.common.InconsistentStorageStateException; import org.apache.hadoop.ozone.container.common.HDDSVolumeLayoutVersion; @@ -183,10 +184,16 @@ public static String getProperty( * /hdds/} as a symlink pointing to {@code /hdds/} when SCM HA is finalized. * + * @param volume + * @param scmId + * @param clusterId + * @param conf + * @param logger + * @param dbVolumeSet * @return true - if volume is in consistent state, otherwise false. */ - public static boolean checkVolume(StorageVolume volume, - String clusterId, Logger logger, + public static boolean checkVolume(StorageVolume volume, String scmId, + String clusterId, ConfigurationSource conf, Logger logger, MutableVolumeSet dbVolumeSet) { File volumeRoot = volume.getStorageDir(); String volumeRootPath = volumeRoot.getPath(); diff --git a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/ozoneimpl/OzoneContainer.java b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/ozoneimpl/OzoneContainer.java index 112c322a89a1..b3fa5133823e 100644 --- a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/ozoneimpl/OzoneContainer.java +++ b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/ozoneimpl/OzoneContainer.java @@ -65,7 +65,6 @@ import org.apache.hadoop.hdds.utils.db.Table; import org.apache.hadoop.hdds.utils.db.TableIterator; import org.apache.hadoop.ozone.HddsDatanodeService; -import org.apache.hadoop.ozone.container.common.DatanodeLayoutStorage; import org.apache.hadoop.ozone.container.common.helpers.ContainerMetrics; import org.apache.hadoop.ozone.container.common.impl.BlockDeletingService; import org.apache.hadoop.ozone.container.common.impl.ContainerSet; @@ -82,7 +81,6 @@ import org.apache.hadoop.ozone.container.common.transport.server.ratis.XceiverServerRatis; import org.apache.hadoop.ozone.container.common.utils.ContainerInspectorUtil; import org.apache.hadoop.ozone.container.common.utils.HddsVolumeUtil; -import org.apache.hadoop.ozone.container.common.utils.StorageVolumeUtil; import org.apache.hadoop.ozone.container.common.volume.HddsVolume; import org.apache.hadoop.ozone.container.common.volume.MutableVolumeSet; import org.apache.hadoop.ozone.container.common.volume.StorageVolume; @@ -486,17 +484,6 @@ public void start(String clusterId) throws IOException { return; } - // Check DbVolumes, format DbVolume at first register time. - checkVolumeSet(getDbVolumeSet(), clusterId); - - // Check HddsVolumes - checkVolumeSet(getVolumeSet(), clusterId); - - DatanodeLayoutStorage layoutStorage - = new DatanodeLayoutStorage(config); - layoutStorage.setClusterId(clusterId); - layoutStorage.persistCurrentState(); - buildContainerSet(); // Start background volume checks, which will begin after the configured @@ -670,29 +657,4 @@ public void compactDb() { } } - - private void checkVolumeSet(MutableVolumeSet volumeSet, String clusterId) - throws DiskOutOfSpaceException { - if (volumeSet == null) { - return; - } - - volumeSet.writeLock(); - try { - // If version file does not exist - // create version file and also set scm ID or cluster ID. - for (StorageVolume volume : volumeSet.getVolumeMap().values()) { - if (!StorageVolumeUtil.checkVolume(volume, clusterId, LOG, getDbVolumeSet())) { - volumeSet.failVolume(volume.getStorageDir().getPath()); - } - } - if (volumeSet.getVolumesList().isEmpty()) { - // All volumes are in inconsistent state - throw new DiskOutOfSpaceException( - "All configured Volumes are in Inconsistent State"); - } - } finally { - volumeSet.writeUnlock(); - } - } } diff --git a/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/TestHddsDatanodeService.java b/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/TestHddsDatanodeService.java index 3256b313d275..63157450a19e 100644 --- a/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/TestHddsDatanodeService.java +++ b/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/TestHddsDatanodeService.java @@ -128,7 +128,8 @@ public void testDeletedContainersClearedOnShutdown(String schemaVersion) assertInstanceOf(HddsVolume.class, volume); HddsVolume hddsVolume = (HddsVolume) volume; - StorageVolumeUtil.checkVolume(hddsVolume, clusterId, LOG, null); + StorageVolumeUtil.checkVolume(hddsVolume, clusterId, + clusterId, conf, LOG, null); // Create a container and move it under the tmp delete dir. KeyValueContainer container = ContainerTestUtils .addContainerToDeletedDir( diff --git a/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/common/ContainerTestUtils.java b/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/common/ContainerTestUtils.java index 9dc2d832c980..e33a4c4a3ce5 100644 --- a/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/common/ContainerTestUtils.java +++ b/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/common/ContainerTestUtils.java @@ -191,7 +191,7 @@ public static void disableSchemaV3(OzoneConfiguration conf) { } public static void createDbInstancesForTestIfNeeded( - MutableVolumeSet hddsVolumeSet, String clusterID, + MutableVolumeSet hddsVolumeSet, String scmID, String clusterID, ConfigurationSource conf) { DatanodeConfiguration dc = conf.getObject(DatanodeConfiguration.class); if (!dc.getContainerSchemaV3Enabled()) { @@ -200,7 +200,8 @@ public static void createDbInstancesForTestIfNeeded( for (HddsVolume volume : StorageVolumeUtil.getHddsVolumesList( hddsVolumeSet.getVolumesList())) { - StorageVolumeUtil.checkVolume(volume, clusterID, null, null); + StorageVolumeUtil.checkVolume(volume, scmID, clusterID, conf, + null, null); } } diff --git a/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/common/TestBlockDeletingService.java b/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/common/TestBlockDeletingService.java index a86dcd9673a1..4e8ad609f2b5 100644 --- a/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/common/TestBlockDeletingService.java +++ b/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/common/TestBlockDeletingService.java @@ -137,7 +137,7 @@ public void init() throws IOException { datanodeUuid = UUID.randomUUID().toString(); volumeSet = new MutableVolumeSet(datanodeUuid, scmId, conf, null, StorageVolume.VolumeType.DATA_VOLUME, null); - createDbInstancesForTestIfNeeded(volumeSet, scmId, conf); + createDbInstancesForTestIfNeeded(volumeSet, scmId, scmId, conf); } @AfterEach diff --git a/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/common/impl/TestContainerPersistence.java b/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/common/impl/TestContainerPersistence.java index 86961af63b05..0a31b9746d57 100644 --- a/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/common/impl/TestContainerPersistence.java +++ b/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/common/impl/TestContainerPersistence.java @@ -158,7 +158,9 @@ public void setupPaths() throws IOException { // Initialize volume directories. for (HddsVolume volume : StorageVolumeUtil.getHddsVolumesList( volumeSet.getVolumesList())) { - assertTrue(StorageVolumeUtil.checkVolume(volume, SCM_ID, null, null)); + boolean success = StorageVolumeUtil.checkVolume(volume, SCM_ID, SCM_ID, + conf, null, null); + assertTrue(success); } blockManager = new BlockManagerImpl(conf); chunkManager = ChunkManagerFactory.createChunkManager(conf, blockManager, diff --git a/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/common/utils/TestStorageVolumeUtil.java b/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/common/utils/TestStorageVolumeUtil.java index 7a76798e7db4..92929c6641cc 100644 --- a/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/common/utils/TestStorageVolumeUtil.java +++ b/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/common/utils/TestStorageVolumeUtil.java @@ -83,15 +83,21 @@ public void testCheckVolumeNoDupDbStoreCreated() throws IOException { .thenReturn(Collections.singletonList(dbVolume)); // check the dbVolume first for hddsVolume to use - assertTrue(StorageVolumeUtil.checkVolume(dbVolume, CLUSTER_ID, null, null)); + boolean res = StorageVolumeUtil.checkVolume(dbVolume, CLUSTER_ID, + CLUSTER_ID, CONF, null, null); + assertTrue(res); // checkVolume for the 1st time: rootFiles.length == 1 - assertTrue(StorageVolumeUtil.checkVolume(spyHddsVolume, CLUSTER_ID, null, dbVolumeSet)); + res = StorageVolumeUtil.checkVolume(spyHddsVolume, CLUSTER_ID, + CLUSTER_ID, CONF, null, dbVolumeSet); + assertTrue(res); // createDbStore called as expected verify(spyHddsVolume, times(1)).createDbStore(dbVolumeSet); // checkVolume for the 2nd time: rootFiles.length == 2 - assertTrue(StorageVolumeUtil.checkVolume(spyHddsVolume, CLUSTER_ID, LOG, dbVolumeSet)); + res = StorageVolumeUtil.checkVolume(spyHddsVolume, CLUSTER_ID, + CLUSTER_ID, CONF, LOG, dbVolumeSet); + assertTrue(res); // should only call createDbStore once, so no dup db instance verify(spyHddsVolume, times(1)).createDbStore(dbVolumeSet); diff --git a/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/keyvalue/TestKeyValueBlockIterator.java b/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/keyvalue/TestKeyValueBlockIterator.java index 08bc144d6f4b..577f65c2e583 100644 --- a/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/keyvalue/TestKeyValueBlockIterator.java +++ b/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/keyvalue/TestKeyValueBlockIterator.java @@ -109,7 +109,7 @@ public void setup() throws Exception { conf.set(OzoneConfigKeys.OZONE_METADATA_DIRS, testRoot.getAbsolutePath()); volumeSet = new MutableVolumeSet(datanodeID, clusterID, conf, null, StorageVolume.VolumeType.DATA_VOLUME, null); - createDbInstancesForTestIfNeeded(volumeSet, clusterID, conf); + createDbInstancesForTestIfNeeded(volumeSet, clusterID, clusterID, conf); containerData = new KeyValueContainerData(105L, layout, diff --git a/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/keyvalue/TestKeyValueContainer.java b/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/keyvalue/TestKeyValueContainer.java index cdc7bdc73f67..52478cf4b105 100644 --- a/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/keyvalue/TestKeyValueContainer.java +++ b/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/keyvalue/TestKeyValueContainer.java @@ -143,7 +143,8 @@ private void init(ContainerTestVersionInfo versionInfo) throws Exception { hddsVolumes.add(new HddsVolume.Builder(folder.toString()) .conf(CONF).datanodeUuid(datanodeId .toString()).build()); - StorageVolumeUtil.checkVolume(hddsVolumes.get(0), scmId, null, null); + StorageVolumeUtil.checkVolume(hddsVolumes.get(0), scmId, scmId, CONF, + null, null); volumeSet = mock(MutableVolumeSet.class); volumeChoosingPolicy = mock(RoundRobinVolumeChoosingPolicy.class); @@ -225,7 +226,7 @@ public void testNextVolumeTriedOnWriteFailure( .toFile().getAbsolutePath(); HddsVolume newVolume = new HddsVolume.Builder(volumeDirPath) .conf(CONF).datanodeUuid(datanodeId.toString()).build(); - StorageVolumeUtil.checkVolume(newVolume, scmId, null, null); + StorageVolumeUtil.checkVolume(newVolume, scmId, scmId, CONF, null, null); hddsVolumes.add(newVolume); // Override the class, so that the first time we call it, it throws @@ -815,7 +816,7 @@ void testAutoCompactionSmallSstFile( .getAbsolutePath(); HddsVolume newVolume = new HddsVolume.Builder(volumeDirPath) .conf(CONF).datanodeUuid(datanodeId.toString()).build(); - StorageVolumeUtil.checkVolume(newVolume, scmId, null, null); + StorageVolumeUtil.checkVolume(newVolume, scmId, scmId, CONF, null, null); List volumeList = new ArrayList<>(); HddsVolume hddsVolume = hddsVolumes.get(0); volumeList.add(hddsVolume); @@ -1035,7 +1036,7 @@ private void testMixedSchemaImport(String dir, HddsVolume hddsVolume1 = new HddsVolume.Builder(dir1) .conf(conf).datanodeUuid(datanodeId.toString()).build(); conf.setBoolean(CONTAINER_SCHEMA_V3_ENABLED, schemaV3Enabled); - StorageVolumeUtil.checkVolume(hddsVolume1, scmId, null, null); + StorageVolumeUtil.checkVolume(hddsVolume1, scmId, scmId, conf, null, null); hddsVolumes.clear(); hddsVolumes.add(hddsVolume1); @@ -1079,7 +1080,7 @@ private void testMixedSchemaImport(String dir, final String dir2 = dir + (schemaV3Enabled ? "/v2" : "/v3"); HddsVolume hddsVolume2 = new HddsVolume.Builder(dir2) .conf(conf).datanodeUuid(datanodeId.toString()).build(); - StorageVolumeUtil.checkVolume(hddsVolume2, scmId, null, null); + StorageVolumeUtil.checkVolume(hddsVolume2, scmId, scmId, conf, null, null); hddsVolumes.clear(); hddsVolumes.add(hddsVolume2); diff --git a/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/keyvalue/TestKeyValueContainerIntegrityChecks.java b/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/keyvalue/TestKeyValueContainerIntegrityChecks.java index a2b0636e792c..3b4cfff93790 100644 --- a/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/keyvalue/TestKeyValueContainerIntegrityChecks.java +++ b/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/keyvalue/TestKeyValueContainerIntegrityChecks.java @@ -91,7 +91,7 @@ private void setup() throws Exception { containerLayoutTestInfo.updateConfig(conf); volumeSet = new MutableVolumeSet(UUID.randomUUID().toString(), clusterID, conf, null, StorageVolume.VolumeType.DATA_VOLUME, null); - createDbInstancesForTestIfNeeded(volumeSet, clusterID, conf); + createDbInstancesForTestIfNeeded(volumeSet, clusterID, clusterID, conf); chunkManager = containerLayoutTestInfo.createChunkManager(true, null); } diff --git a/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/keyvalue/impl/TestBlockManagerImpl.java b/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/keyvalue/impl/TestBlockManagerImpl.java index 4f89a1b01ed1..07414d701aad 100644 --- a/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/keyvalue/impl/TestBlockManagerImpl.java +++ b/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/keyvalue/impl/TestBlockManagerImpl.java @@ -90,7 +90,8 @@ private void initilaze() throws Exception { .conf(config) .datanodeUuid(datanodeId.toString()) .build(); - StorageVolumeUtil.checkVolume(hddsVolume, scmId, null, null); + StorageVolumeUtil.checkVolume(hddsVolume, scmId, scmId, config, + null, null); volumeSet = mock(MutableVolumeSet.class); diff --git a/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/ozoneimpl/TestContainerReader.java b/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/ozoneimpl/TestContainerReader.java index fe3e59490cea..e315ced9f48d 100644 --- a/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/ozoneimpl/TestContainerReader.java +++ b/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/ozoneimpl/TestContainerReader.java @@ -109,7 +109,8 @@ private void setup(ContainerTestVersionInfo versionInfo) throws Exception { hddsVolume = new HddsVolume.Builder(volumeDir .getAbsolutePath()).conf(conf).datanodeUuid(datanodeId .toString()).clusterID(clusterId).build(); - StorageVolumeUtil.checkVolume(hddsVolume, clusterId, null, null); + StorageVolumeUtil.checkVolume(hddsVolume, clusterId, clusterId, conf, + null, null); volumeSet = mock(MutableVolumeSet.class); volumeChoosingPolicy = mock(RoundRobinVolumeChoosingPolicy.class); @@ -276,7 +277,8 @@ public void testContainerReaderWithLoadException( hddsVolume1 = new HddsVolume.Builder(volumeDir1 .getAbsolutePath()).conf(conf).datanodeUuid(datanode .toString()).clusterID(clusterId).build(); - StorageVolumeUtil.checkVolume(hddsVolume1, clusterId, null, null); + StorageVolumeUtil.checkVolume(hddsVolume1, clusterId, clusterId, conf, + null, null); volumeChoosingPolicy1 = mock(RoundRobinVolumeChoosingPolicy.class); when(volumeChoosingPolicy1.chooseVolume(anyList(), anyLong())) .thenReturn(hddsVolume1); @@ -325,7 +327,8 @@ public void testContainerReaderWithInvalidDbPath( hddsVolume1 = new HddsVolume.Builder(volumeDir1 .getAbsolutePath()).conf(conf).datanodeUuid(datanode .toString()).clusterID(clusterId).build(); - StorageVolumeUtil.checkVolume(hddsVolume1, clusterId, null, null); + StorageVolumeUtil.checkVolume(hddsVolume1, clusterId, clusterId, conf, + null, null); volumeChoosingPolicy1 = mock(RoundRobinVolumeChoosingPolicy.class); when(volumeChoosingPolicy1.chooseVolume(anyList(), anyLong())) .thenReturn(hddsVolume1); @@ -380,9 +383,10 @@ public void testMultipleContainerReader(ContainerTestVersionInfo versionInfo) new MutableVolumeSet(datanodeId.toString(), clusterId, conf, null, StorageVolume.VolumeType.DATA_VOLUME, null); for (StorageVolume v : volumeSets.getVolumesList()) { - StorageVolumeUtil.checkVolume(v, clusterId, null, null); + StorageVolumeUtil.checkVolume(v, clusterId, clusterId, conf, + null, null); } - createDbInstancesForTestIfNeeded(volumeSets, clusterId, conf); + createDbInstancesForTestIfNeeded(volumeSets, clusterId, clusterId, conf); ContainerCache cache = ContainerCache.getInstance(conf); cache.shutdownCache(); diff --git a/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/ozoneimpl/TestOzoneContainer.java b/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/ozoneimpl/TestOzoneContainer.java index ea1abd48d84e..34619c19183f 100644 --- a/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/ozoneimpl/TestOzoneContainer.java +++ b/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/ozoneimpl/TestOzoneContainer.java @@ -97,7 +97,7 @@ private void setup() throws Exception { commitSpaceMap = new HashMap<>(); volumeSet = new MutableVolumeSet(datanodeDetails.getUuidString(), clusterId, conf, null, StorageVolume.VolumeType.DATA_VOLUME, null); - createDbInstancesForTestIfNeeded(volumeSet, clusterId, conf); + createDbInstancesForTestIfNeeded(volumeSet, clusterId, clusterId, conf); volumeChoosingPolicy = new RoundRobinVolumeChoosingPolicy(); } diff --git a/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/upgrade/UpgradeTestHelper.java b/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/upgrade/UpgradeTestHelper.java index 769c79d88ac7..04bfdf30762b 100644 --- a/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/upgrade/UpgradeTestHelper.java +++ b/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/upgrade/UpgradeTestHelper.java @@ -128,7 +128,8 @@ public static void callVersionEndpointTask( throws Exception { try (EndpointStateMachine esm = ContainerTestUtils.createEndpoint(conf, address, 1000)) { - VersionEndpointTask vet = new VersionEndpointTask(esm, container); + VersionEndpointTask vet = new VersionEndpointTask(esm, conf, + container); esm.setState(EndpointStateMachine.EndPointStates.GETVERSION); vet.call(); } diff --git a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/ozone/container/common/TestEndPoint.java b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/ozone/container/common/TestEndPoint.java index a877e9170974..64f903f45057 100644 --- a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/ozone/container/common/TestEndPoint.java +++ b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/ozone/container/common/TestEndPoint.java @@ -157,7 +157,8 @@ public void testGetVersionTask() throws Exception { OzoneContainer ozoneContainer = new OzoneContainer(dnDetails, ozoneConf, ContainerTestUtils.getMockContext(dnDetails, ozoneConf)); rpcEndPoint.setState(EndpointStateMachine.EndPointStates.GETVERSION); - VersionEndpointTask versionTask = new VersionEndpointTask(rpcEndPoint, ozoneContainer); + VersionEndpointTask versionTask = new VersionEndpointTask(rpcEndPoint, + ozoneConf, ozoneContainer); EndpointStateMachine.EndPointStates newState = versionTask.call(); // if version call worked the endpoint should automatically move to the @@ -199,7 +200,8 @@ public void testDeletedContainersClearedOnStartup() throws Exception { rpcEndPoint.setState(EndpointStateMachine.EndPointStates.GETVERSION); // versionTask.call() cleans the tmp dir and removes container from DB - VersionEndpointTask versionTask = new VersionEndpointTask(rpcEndPoint, ozoneContainer); + VersionEndpointTask versionTask = new VersionEndpointTask(rpcEndPoint, + ozoneConf, ozoneContainer); EndpointStateMachine.EndPointStates newState = versionTask.call(); assertEquals(EndpointStateMachine.EndPointStates.REGISTER, newState); @@ -230,7 +232,8 @@ public void testCheckVersionResponse() throws Exception { OzoneContainer ozoneContainer = new OzoneContainer(dnDetails, ozoneConf, ContainerTestUtils.getMockContext(dnDetails, ozoneConf)); rpcEndPoint.setState(EndpointStateMachine.EndPointStates.GETVERSION); - VersionEndpointTask versionTask = new VersionEndpointTask(rpcEndPoint, ozoneContainer); + VersionEndpointTask versionTask = new VersionEndpointTask(rpcEndPoint, + ozoneConf, ozoneContainer); EndpointStateMachine.EndPointStates newState = versionTask.call(); // if version call worked the endpoint should automatically move to the @@ -272,7 +275,8 @@ public void testDnLayoutVersionFile() throws Exception { OzoneContainer ozoneContainer = new OzoneContainer(dnDetails, ozoneConf, ContainerTestUtils.getMockContext(dnDetails, ozoneConf)); rpcEndPoint.setState(EndpointStateMachine.EndPointStates.GETVERSION); - VersionEndpointTask versionTask = new VersionEndpointTask(rpcEndPoint, ozoneContainer); + VersionEndpointTask versionTask = new VersionEndpointTask(rpcEndPoint, + ozoneConf, ozoneContainer); versionTask.call(); // After the version call, the datanode layout file should @@ -325,7 +329,8 @@ public void testGetVersionToInvalidEndpoint() throws Exception { DatanodeDetails datanodeDetails = randomDatanodeDetails(); OzoneContainer ozoneContainer = new OzoneContainer(datanodeDetails, conf, ContainerTestUtils.getMockContext(datanodeDetails, ozoneConf)); - VersionEndpointTask versionTask = new VersionEndpointTask(rpcEndPoint, ozoneContainer); + VersionEndpointTask versionTask = new VersionEndpointTask(rpcEndPoint, + conf, ozoneContainer); EndpointStateMachine.EndPointStates newState = versionTask.call(); // This version call did NOT work, so endpoint should remain in the same @@ -351,7 +356,8 @@ public void testGetVersionAssertRpcTimeOut() throws Exception { DatanodeDetails datanodeDetails = randomDatanodeDetails(); OzoneContainer ozoneContainer = new OzoneContainer(datanodeDetails, conf, ContainerTestUtils.getMockContext(datanodeDetails, ozoneConf)); - VersionEndpointTask versionTask = new VersionEndpointTask(rpcEndPoint, ozoneContainer); + VersionEndpointTask versionTask = new VersionEndpointTask(rpcEndPoint, + conf, ozoneContainer); scmServerImpl.setRpcResponseDelay(1500); long start = Time.monotonicNow(); @@ -648,7 +654,7 @@ private OzoneContainer createVolume(OzoneConfiguration conf) MutableVolumeSet volumeSet = ozoneContainer.getVolumeSet(); ContainerTestUtils.createDbInstancesForTestIfNeeded(volumeSet, - clusterId, conf); + clusterId, clusterId, conf); // VolumeSet for this test, contains only 1 volume assertEquals(1, volumeSet.getVolumesList().size()); StorageVolume volume = volumeSet.getVolumesList().get(0); diff --git a/hadoop-hdds/tools/src/test/java/org/apache/hadoop/hdds/scm/cli/container/upgrade/TestUpgradeManager.java b/hadoop-hdds/tools/src/test/java/org/apache/hadoop/hdds/scm/cli/container/upgrade/TestUpgradeManager.java index f9165e9ad86e..4d9501887678 100644 --- a/hadoop-hdds/tools/src/test/java/org/apache/hadoop/hdds/scm/cli/container/upgrade/TestUpgradeManager.java +++ b/hadoop-hdds/tools/src/test/java/org/apache/hadoop/hdds/scm/cli/container/upgrade/TestUpgradeManager.java @@ -116,7 +116,8 @@ public void setup() throws Exception { final List volumes = new ArrayList<>(); for (StorageVolume storageVolume : volumeSet.getVolumesList()) { HddsVolume hddsVolume = (HddsVolume) storageVolume; - StorageVolumeUtil.checkVolume(hddsVolume, SCM_ID, null, null); + StorageVolumeUtil.checkVolume(hddsVolume, SCM_ID, SCM_ID, CONF, null, + null); volumes.add(hddsVolume); } diff --git a/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/freon/ClosedContainerReplicator.java b/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/freon/ClosedContainerReplicator.java index 391fd244e1f1..8d4a4e99af57 100644 --- a/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/freon/ClosedContainerReplicator.java +++ b/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/freon/ClosedContainerReplicator.java @@ -211,7 +211,7 @@ private void initializeReplicationSupervisor( volumeSet, dbVolumeSet, false, LOG); for (StorageVolume volume : volumeSet.getVolumesList()) { - StorageVolumeUtil.checkVolume(volume, clusterID, LOG, dbVolumeSet); + StorageVolumeUtil.checkVolume(volume, scmID, clusterID, conf, LOG, dbVolumeSet); } } From db0b13072627639bb6fae4db0997b389c4bbb92b Mon Sep 17 00:00:00 2001 From: "Doroszlai, Attila" Date: Mon, 17 Mar 2025 09:43:26 +0100 Subject: [PATCH 09/10] HDDS-12608. Race condition in datanode version file creation --- .../common/states/endpoint/VersionEndpointTask.java | 6 ------ .../hadoop/ozone/container/ozoneimpl/OzoneContainer.java | 6 ++++++ .../hadoop/ozone/container/common/TestEndPoint.java | 8 ++++++++ 3 files changed, 14 insertions(+), 6 deletions(-) diff --git a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/states/endpoint/VersionEndpointTask.java b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/states/endpoint/VersionEndpointTask.java index 2fb7b9c69b21..0f52fedf63df 100644 --- a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/states/endpoint/VersionEndpointTask.java +++ b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/states/endpoint/VersionEndpointTask.java @@ -24,7 +24,6 @@ import org.apache.hadoop.hdds.conf.ConfigurationSource; import org.apache.hadoop.hdds.protocol.proto.StorageContainerDatanodeProtocolProtos.SCMVersionResponseProto; import org.apache.hadoop.ozone.OzoneConsts; -import org.apache.hadoop.ozone.container.common.DatanodeLayoutStorage; import org.apache.hadoop.ozone.container.common.statemachine.EndpointStateMachine; import org.apache.hadoop.ozone.container.common.utils.StorageVolumeUtil; import org.apache.hadoop.ozone.container.common.volume.MutableVolumeSet; @@ -88,11 +87,6 @@ public EndpointStateMachine.EndPointStates call() throws Exception { // Check HddsVolumes checkVolumeSet(ozoneContainer.getVolumeSet(), scmId, clusterId); - DatanodeLayoutStorage layoutStorage - = new DatanodeLayoutStorage(configuration); - layoutStorage.setClusterId(clusterId); - layoutStorage.persistCurrentState(); - // Start the container services after getting the version information ozoneContainer.start(clusterId); } diff --git a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/ozoneimpl/OzoneContainer.java b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/ozoneimpl/OzoneContainer.java index b3fa5133823e..91ff1511bd5c 100644 --- a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/ozoneimpl/OzoneContainer.java +++ b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/ozoneimpl/OzoneContainer.java @@ -65,6 +65,7 @@ import org.apache.hadoop.hdds.utils.db.Table; import org.apache.hadoop.hdds.utils.db.TableIterator; import org.apache.hadoop.ozone.HddsDatanodeService; +import org.apache.hadoop.ozone.container.common.DatanodeLayoutStorage; import org.apache.hadoop.ozone.container.common.helpers.ContainerMetrics; import org.apache.hadoop.ozone.container.common.impl.BlockDeletingService; import org.apache.hadoop.ozone.container.common.impl.ContainerSet; @@ -484,6 +485,11 @@ public void start(String clusterId) throws IOException { return; } + DatanodeLayoutStorage layoutStorage + = new DatanodeLayoutStorage(config); + layoutStorage.setClusterId(clusterId); + layoutStorage.persistCurrentState(); + buildContainerSet(); // Start background volume checks, which will begin after the configured diff --git a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/ozone/container/common/TestEndPoint.java b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/ozone/container/common/TestEndPoint.java index 64f903f45057..3bbe889d1a69 100644 --- a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/ozone/container/common/TestEndPoint.java +++ b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/ozone/container/common/TestEndPoint.java @@ -63,6 +63,7 @@ import org.apache.hadoop.ipc.RPC; import org.apache.hadoop.ozone.OzoneConfigKeys; import org.apache.hadoop.ozone.OzoneConsts; +import org.apache.hadoop.ozone.common.Storage.StorageState; import org.apache.hadoop.ozone.container.common.statemachine.DatanodeStateMachine; import org.apache.hadoop.ozone.container.common.statemachine.EndpointStateMachine; import org.apache.hadoop.ozone.container.common.statemachine.StateContext; @@ -310,6 +311,13 @@ public void testDnLayoutVersionFile() throws Exception { assertEquals("different_cluster_id", layout1.getClusterID()); assertNotEquals(scmServerImpl.getClusterId(), layout1.getClusterID()); + + // another call() with OzoneContainer already started should not write the file + FileUtils.forceDelete(layout1.getVersionFile()); + rpcEndPoint.setState(EndpointStateMachine.EndPointStates.GETVERSION); + versionTask.call(); + assertEquals(StorageState.NOT_INITIALIZED, new DatanodeLayoutStorage(ozoneConf, "any").getState()); + FileUtils.forceDelete(storageDir); } } From 381e0ede4f269e120edf159ff20254cd3b79a3a9 Mon Sep 17 00:00:00 2001 From: "Doroszlai, Attila" Date: Mon, 17 Mar 2025 10:55:32 +0100 Subject: [PATCH 10/10] fix OzoneContainer integration tests --- .../ozone/container/common/ContainerTestUtils.java | 11 +++++++++++ .../ozone/container/ozoneimpl/TestOzoneContainer.java | 2 ++ .../ozoneimpl/TestOzoneContainerWithTLS.java | 1 + .../container/ozoneimpl/TestSecureOzoneContainer.java | 1 + 4 files changed, 15 insertions(+) diff --git a/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/common/ContainerTestUtils.java b/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/common/ContainerTestUtils.java index e33a4c4a3ce5..3cff94ed9bd6 100644 --- a/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/common/ContainerTestUtils.java +++ b/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/common/ContainerTestUtils.java @@ -17,6 +17,7 @@ package org.apache.hadoop.ozone.container.common; +import static org.apache.hadoop.ozone.common.Storage.StorageState.INITIALIZED; import static org.mockito.Mockito.any; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.when; @@ -46,6 +47,7 @@ import org.apache.hadoop.ipc.ProtobufRpcEngine; import org.apache.hadoop.ipc.RPC; import org.apache.hadoop.net.NetUtils; +import org.apache.hadoop.ozone.HddsDatanodeService; import org.apache.hadoop.ozone.OzoneConfigKeys; import org.apache.hadoop.ozone.container.ContainerTestHelper; import org.apache.hadoop.ozone.container.common.impl.ContainerData; @@ -350,4 +352,13 @@ public static XceiverServerRatis newXceiverServerRatis( getNoopContainerDispatcher(), getEmptyContainerController(), null, null); } + + /** Initialize {@link DatanodeLayoutStorage}. Normally this is done during {@link HddsDatanodeService} start, + * have to do the same for tests that create {@link OzoneContainer} manually. */ + public static void initializeDatanodeLayout(ConfigurationSource conf, DatanodeDetails dn) throws IOException { + DatanodeLayoutStorage layoutStorage = new DatanodeLayoutStorage(conf, dn.getUuidString()); + if (layoutStorage.getState() != INITIALIZED) { + layoutStorage.initialize(); + } + } } diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/container/ozoneimpl/TestOzoneContainer.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/container/ozoneimpl/TestOzoneContainer.java index cd3aeb9400d8..cb0dbe97199c 100644 --- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/container/ozoneimpl/TestOzoneContainer.java +++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/container/ozoneimpl/TestOzoneContainer.java @@ -82,6 +82,7 @@ public void testCreateOzoneContainer( .getOzoneContainer(datanodeDetails, conf); StorageVolumeUtil.getHddsVolumesList(container.getVolumeSet().getVolumesList()) .forEach(hddsVolume -> hddsVolume.setDbParentDir(tempDir.toFile())); + ContainerTestUtils.initializeDatanodeLayout(conf, datanodeDetails); //Set clusterId and manually start ozone container. container.start(UUID.randomUUID().toString()); @@ -112,6 +113,7 @@ void testOzoneContainerStart( DatanodeDetails datanodeDetails = randomDatanodeDetails(); container = ContainerTestUtils .getOzoneContainer(datanodeDetails, conf); + ContainerTestUtils.initializeDatanodeLayout(conf, datanodeDetails); String clusterId = UUID.randomUUID().toString(); container.start(clusterId); diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/container/ozoneimpl/TestOzoneContainerWithTLS.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/container/ozoneimpl/TestOzoneContainerWithTLS.java index 62214ca2144e..91285b551da8 100644 --- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/container/ozoneimpl/TestOzoneContainerWithTLS.java +++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/container/ozoneimpl/TestOzoneContainerWithTLS.java @@ -313,6 +313,7 @@ private OzoneContainer createAndStartOzoneContainerInstance() { MutableVolumeSet volumeSet = container.getVolumeSet(); StorageVolumeUtil.getHddsVolumesList(volumeSet.getVolumesList()) .forEach(hddsVolume -> hddsVolume.setDbParentDir(tempFolder.toFile())); + ContainerTestUtils.initializeDatanodeLayout(conf, dn); container.start(clusterID); } catch (Throwable e) { if (container != null) { diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/container/ozoneimpl/TestSecureOzoneContainer.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/container/ozoneimpl/TestSecureOzoneContainer.java index e3c3b53a22d1..de63bfd7c8ee 100644 --- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/container/ozoneimpl/TestSecureOzoneContainer.java +++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/container/ozoneimpl/TestSecureOzoneContainer.java @@ -138,6 +138,7 @@ void testCreateOzoneContainer(boolean requireToken, boolean hasToken, MutableVolumeSet volumeSet = container.getVolumeSet(); StorageVolumeUtil.getHddsVolumesList(volumeSet.getVolumesList()) .forEach(hddsVolume -> hddsVolume.setDbParentDir(tempFolder.toFile())); + ContainerTestUtils.initializeDatanodeLayout(conf, dn); //Set scmId and manually start ozone container. container.start(UUID.randomUUID().toString());