From d9c7f418a5a4126a96fb5f991a01ef895183cf23 Mon Sep 17 00:00:00 2001 From: rich7420 Date: Fri, 21 Nov 2025 13:04:00 +0800 Subject: [PATCH 01/10] Increase available space requirement during pipeline/container allocation --- .../apache/hadoop/hdds/scm/ScmConfigKeys.java | 4 ++ .../src/main/resources/ozone-default.xml | 16 ++++++++ .../scm/container/ContainerManagerImpl.java | 14 +++++-- .../hdds/scm/pipeline/ECPipelineProvider.java | 7 +++- .../scm/pipeline/RatisPipelineProvider.java | 9 ++++- .../container/TestContainerManagerImpl.java | 40 +++++++++++++++++++ 6 files changed, 83 insertions(+), 7 deletions(-) diff --git a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/ScmConfigKeys.java b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/ScmConfigKeys.java index 23400b1a06b4..83937404edf4 100644 --- a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/ScmConfigKeys.java +++ b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/ScmConfigKeys.java @@ -359,6 +359,10 @@ public final class ScmConfigKeys { "ozone.scm.container.size"; public static final String OZONE_SCM_CONTAINER_SIZE_DEFAULT = "5GB"; + public static final String OZONE_SCM_CONTAINER_SPACE_REQUIREMENT_MULTIPLIER = + "ozone.scm.container.space.requirement.multiplier"; + public static final double OZONE_SCM_CONTAINER_SPACE_REQUIREMENT_MULTIPLIER_DEFAULT = 2.0; + public static final String OZONE_SCM_CONTAINER_LOCK_STRIPE_SIZE = "ozone.scm.container.lock.stripes"; public static final int OZONE_SCM_CONTAINER_LOCK_STRIPE_SIZE_DEFAULT = 512; diff --git a/hadoop-hdds/common/src/main/resources/ozone-default.xml b/hadoop-hdds/common/src/main/resources/ozone-default.xml index 5d36eb3b8f29..c07946d7fd31 100644 --- a/hadoop-hdds/common/src/main/resources/ozone-default.xml +++ b/hadoop-hdds/common/src/main/resources/ozone-default.xml @@ -1035,6 +1035,22 @@ balances the amount of metadata. + + ozone.scm.container.space.requirement.multiplier + 2.0 + OZONE, SCM, MANAGEMENT + + Multiplier for container space requirement when checking if a datanode + has enough space for container allocation. The required space is calculated + as container size multiplied by this value. This prevents concurrent clients + from all passing the space check when there's only enough space for one + container. For example, with default container size of 5GB and multiplier + of 2.0, the system will require 10GB of available space before allocating + a new container. This ensures that if only 6GB is remaining, the check will + fail, preventing multiple clients from attempting to create containers + concurrently when there's only space for one. + + ozone.scm.container.lock.stripes 512 diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/ContainerManagerImpl.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/ContainerManagerImpl.java index d255bc9a672d..f54f595f8889 100644 --- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/ContainerManagerImpl.java +++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/ContainerManagerImpl.java @@ -82,6 +82,7 @@ public class ContainerManagerImpl implements ContainerManager { private final Random random = new Random(); private final long maxContainerSize; + private final long containerSpaceRequirement; /** * @@ -115,6 +116,11 @@ public ContainerManagerImpl( maxContainerSize = (long) conf.getStorageSize(ScmConfigKeys.OZONE_SCM_CONTAINER_SIZE, ScmConfigKeys.OZONE_SCM_CONTAINER_SIZE_DEFAULT, StorageUnit.BYTES); + double multiplier = conf.getDouble( + ScmConfigKeys.OZONE_SCM_CONTAINER_SPACE_REQUIREMENT_MULTIPLIER, + ScmConfigKeys.OZONE_SCM_CONTAINER_SPACE_REQUIREMENT_MULTIPLIER_DEFAULT); + this.containerSpaceRequirement = (long) (maxContainerSize * multiplier); + this.scmContainerManagerMetrics = SCMContainerManagerMetrics.create(); } @@ -352,23 +358,23 @@ public ContainerInfo getMatchingContainer(final long size, final String owner, synchronized (pipeline.getId()) { containerIDs = getContainersForOwner(pipeline, owner); if (containerIDs.size() < getOpenContainerCountPerPipeline(pipeline)) { - if (pipelineManager.hasEnoughSpace(pipeline, maxContainerSize)) { + if (pipelineManager.hasEnoughSpace(pipeline, containerSpaceRequirement)) { allocateContainer(pipeline, owner); containerIDs = getContainersForOwner(pipeline, owner); } else { LOG.debug("Cannot allocate a new container because pipeline {} does not have the required space {}.", - pipeline, maxContainerSize); + pipeline, containerSpaceRequirement); } } containerIDs.removeAll(excludedContainerIDs); containerInfo = containerStateManager.getMatchingContainer( size, owner, pipeline.getId(), containerIDs); if (containerInfo == null) { - if (pipelineManager.hasEnoughSpace(pipeline, maxContainerSize)) { + if (pipelineManager.hasEnoughSpace(pipeline, containerSpaceRequirement)) { containerInfo = allocateContainer(pipeline, owner); } else { LOG.debug("Cannot allocate a new container because pipeline {} does not have the required space {}.", - pipeline, maxContainerSize); + pipeline, containerSpaceRequirement); } } return containerInfo; diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/pipeline/ECPipelineProvider.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/pipeline/ECPipelineProvider.java index c1f14d8cc655..7d345f8567e8 100644 --- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/pipeline/ECPipelineProvider.java +++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/pipeline/ECPipelineProvider.java @@ -62,6 +62,7 @@ public class ECPipelineProvider extends PipelineProvider { private final ConfigurationSource conf; private final PlacementPolicy placementPolicy; private final long containerSizeBytes; + private final long containerSpaceRequirement; public ECPipelineProvider(NodeManager nodeManager, PipelineStateManager stateManager, @@ -73,6 +74,10 @@ public ECPipelineProvider(NodeManager nodeManager, this.containerSizeBytes = (long) this.conf .getStorageSize(ScmConfigKeys.OZONE_SCM_CONTAINER_SIZE, ScmConfigKeys.OZONE_SCM_CONTAINER_SIZE_DEFAULT, StorageUnit.BYTES); + double multiplier = conf.getDouble( + ScmConfigKeys.OZONE_SCM_CONTAINER_SPACE_REQUIREMENT_MULTIPLIER, + ScmConfigKeys.OZONE_SCM_CONTAINER_SPACE_REQUIREMENT_MULTIPLIER_DEFAULT); + this.containerSpaceRequirement = (long) (containerSizeBytes * multiplier); } @Override @@ -88,7 +93,7 @@ protected Pipeline create(ECReplicationConfig replicationConfig, throws IOException { List dns = placementPolicy .chooseDatanodes(excludedNodes, favoredNodes, - replicationConfig.getRequiredNodes(), 0, this.containerSizeBytes); + replicationConfig.getRequiredNodes(), 0, this.containerSpaceRequirement); return create(replicationConfig, dns); } diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/pipeline/RatisPipelineProvider.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/pipeline/RatisPipelineProvider.java index 491e4d15adf2..6cc5d931257f 100644 --- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/pipeline/RatisPipelineProvider.java +++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/pipeline/RatisPipelineProvider.java @@ -65,6 +65,7 @@ public class RatisPipelineProvider private final LeaderChoosePolicy leaderChoosePolicy; private final SCMContext scmContext; private final long containerSizeBytes; + private final long containerSpaceRequirement; private final long minRatisVolumeSizeBytes; @VisibleForTesting @@ -89,6 +90,10 @@ public RatisPipelineProvider(NodeManager nodeManager, ScmConfigKeys.OZONE_SCM_CONTAINER_SIZE, ScmConfigKeys.OZONE_SCM_CONTAINER_SIZE_DEFAULT, StorageUnit.BYTES); + double multiplier = conf.getDouble( + ScmConfigKeys.OZONE_SCM_CONTAINER_SPACE_REQUIREMENT_MULTIPLIER, + ScmConfigKeys.OZONE_SCM_CONTAINER_SPACE_REQUIREMENT_MULTIPLIER_DEFAULT); + this.containerSpaceRequirement = (long) (containerSizeBytes * multiplier); this.minRatisVolumeSizeBytes = (long) this.conf.getStorageSize( ScmConfigKeys.OZONE_DATANODE_RATIS_VOLUME_FREE_SPACE_MIN, ScmConfigKeys.OZONE_DATANODE_RATIS_VOLUME_FREE_SPACE_MIN_DEFAULT, @@ -165,7 +170,7 @@ public synchronized Pipeline create(RatisReplicationConfig replicationConfig, switch (factor) { case ONE: dns = pickNodesNotUsed(replicationConfig, minRatisVolumeSizeBytes, - containerSizeBytes, conf); + containerSpaceRequirement, conf); break; case THREE: List excludeDueToEngagement = filterPipelineEngagement(); @@ -178,7 +183,7 @@ public synchronized Pipeline create(RatisReplicationConfig replicationConfig, } dns = placementPolicy.chooseDatanodes(excludedNodes, favoredNodes, factor.getNumber(), minRatisVolumeSizeBytes, - containerSizeBytes); + containerSpaceRequirement); break; default: throw new IllegalStateException("Unknown factor: " + factor.name()); diff --git a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/TestContainerManagerImpl.java b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/TestContainerManagerImpl.java index dd5edf381930..db77c91f4024 100644 --- a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/TestContainerManagerImpl.java +++ b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/TestContainerManagerImpl.java @@ -27,8 +27,11 @@ import static org.junit.jupiter.api.Assertions.assertTrue; import static org.mockito.ArgumentMatchers.any; import static org.mockito.ArgumentMatchers.anyLong; +import static org.mockito.ArgumentMatchers.eq; +import static org.mockito.Mockito.atLeast; import static org.mockito.Mockito.doReturn; import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.reset; import static org.mockito.Mockito.spy; import static org.mockito.Mockito.times; import static org.mockito.Mockito.verify; @@ -47,6 +50,7 @@ import org.apache.hadoop.hdds.protocol.proto.HddsProtos; import org.apache.hadoop.hdds.protocol.proto.HddsProtos.LifeCycleState; import org.apache.hadoop.hdds.protocol.proto.HddsProtos.ReplicationFactor; +import org.apache.hadoop.hdds.scm.ScmConfigKeys; import org.apache.hadoop.hdds.scm.container.replication.ContainerReplicaPendingOps; import org.apache.hadoop.hdds.scm.container.states.ContainerStateMap; import org.apache.hadoop.hdds.scm.ha.SCMHAManager; @@ -179,6 +183,42 @@ public void testGetMatchingContainerReturnsContainerWhenEnoughSpaceInDatanodes() assertNotNull(container); } + @Test + public void testContainerSpaceRequirementMultiplier() throws IOException { + long sizeRequired = 256 * 1024 * 1024; + double multiplier = 2.0; + long containerSize = 5L * 1024 * 1024 * 1024; + long expectedSpaceRequirement = (long) (containerSize * multiplier); + + PipelineManager spyPipelineManager = spy(pipelineManager); + File tempDir = new File(testDir, "tempDir"); + OzoneConfiguration conf = SCMTestUtils.getConf(tempDir); + conf.setDouble(ScmConfigKeys.OZONE_SCM_CONTAINER_SPACE_REQUIREMENT_MULTIPLIER, multiplier); + + ContainerManager manager = new ContainerManagerImpl(conf, + scmhaManager, sequenceIdGen, spyPipelineManager, + SCMDBDefinition.CONTAINERS.getTable(dbStore), pendingOpsMock); + + Pipeline pipeline = spyPipelineManager.getPipelines().iterator().next(); + + doReturn(false).when(spyPipelineManager) + .hasEnoughSpace(any(Pipeline.class), anyLong()); + ContainerInfo container = manager + .getMatchingContainer(sizeRequired, "test", pipeline, Collections.emptySet()); + assertNull(container); + verify(spyPipelineManager, atLeast(1)) + .hasEnoughSpace(eq(pipeline), eq(expectedSpaceRequirement)); + + reset(spyPipelineManager); + doReturn(true).when(spyPipelineManager) + .hasEnoughSpace(any(Pipeline.class), anyLong()); + container = manager + .getMatchingContainer(sizeRequired, "test", pipeline, Collections.emptySet()); + assertNotNull(container); + verify(spyPipelineManager, atLeast(1)) + .hasEnoughSpace(eq(pipeline), eq(expectedSpaceRequirement)); + } + @Test void testUpdateContainerState() throws Exception { final ContainerInfo container = containerManager.allocateContainer( From 7ba1b2e295006b37a9de49e610e551f1b901fa20 Mon Sep 17 00:00:00 2001 From: rich7420 Date: Fri, 21 Nov 2025 15:15:07 +0800 Subject: [PATCH 02/10] fix error --- .../hdds/scm/pipeline/TestECPipelineProvider.java | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/pipeline/TestECPipelineProvider.java b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/pipeline/TestECPipelineProvider.java index f2511e624f77..5c2cf871e9d9 100644 --- a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/pipeline/TestECPipelineProvider.java +++ b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/pipeline/TestECPipelineProvider.java @@ -68,17 +68,21 @@ public class TestECPipelineProvider { private PipelineStateManager stateManager = mock(PipelineStateManager.class); private PlacementPolicy placementPolicy = mock(PlacementPolicy.class); - private long containerSizeBytes; + private long containerSpaceRequirement; @BeforeEach public void setup() throws IOException, NodeNotFoundException { OzoneConfiguration conf = new OzoneConfiguration(); provider = new ECPipelineProvider( nodeManager, stateManager, conf, placementPolicy); - this.containerSizeBytes = (long) conf.getStorageSize( + long containerSizeBytes = (long) conf.getStorageSize( ScmConfigKeys.OZONE_SCM_CONTAINER_SIZE, ScmConfigKeys.OZONE_SCM_CONTAINER_SIZE_DEFAULT, StorageUnit.BYTES); + double multiplier = conf.getDouble( + ScmConfigKeys.OZONE_SCM_CONTAINER_SPACE_REQUIREMENT_MULTIPLIER, + ScmConfigKeys.OZONE_SCM_CONTAINER_SPACE_REQUIREMENT_MULTIPLIER_DEFAULT); + this.containerSpaceRequirement = (long) (containerSizeBytes * multiplier); // Placement policy will always return EC number of random nodes. when(placementPolicy.chooseDatanodes(anyList(), anyList(), anyInt(), anyLong(), @@ -200,7 +204,7 @@ public void testExcludedAndFavoredNodesPassedToPlacementPolicy() assertEquals(ecConf.getData() + ecConf.getParity(), pipeline.getNodes().size()); verify(placementPolicy).chooseDatanodes(excludedNodes, favoredNodes, - ecConf.getRequiredNodes(), 0, containerSizeBytes); + ecConf.getRequiredNodes(), 0, containerSpaceRequirement); } private Set createContainerReplicas(int number) { From 73843b466c9e7eea6f21bf1ba47f3891c5e3f615 Mon Sep 17 00:00:00 2001 From: rich7420 Date: Sun, 23 Nov 2025 11:10:34 +0800 Subject: [PATCH 03/10] fix errors --- hadoop-ozone/dist/src/main/k8s/definitions/ozone/config.yaml | 1 + .../src/main/k8s/examples/getting-started/config-configmap.yaml | 1 + .../dist/src/main/k8s/examples/minikube/config-configmap.yaml | 1 + .../dist/src/main/k8s/examples/ozone-dev/config-configmap.yaml | 1 + .../dist/src/main/k8s/examples/ozone-ha/config-configmap.yaml | 1 + .../dist/src/main/k8s/examples/ozone/config-configmap.yaml | 1 + 6 files changed, 6 insertions(+) diff --git a/hadoop-ozone/dist/src/main/k8s/definitions/ozone/config.yaml b/hadoop-ozone/dist/src/main/k8s/definitions/ozone/config.yaml index e646b1c0af88..83421b3a0514 100644 --- a/hadoop-ozone/dist/src/main/k8s/definitions/ozone/config.yaml +++ b/hadoop-ozone/dist/src/main/k8s/definitions/ozone/config.yaml @@ -31,6 +31,7 @@ data: OZONE-SITE.XML_hdds.scm.safemode.min.datanode: "3" OZONE-SITE.XML_ozone.datanode.pipeline.limit: "1" OZONE-SITE.XML_hdds.datanode.volume.min.free.space: "1GB" + OZONE-SITE.XML_ozone.scm.container.space.requirement.multiplier: "1.0" OZONE-SITE.XML_ozone.metadata.dirs: "/data/metadata" OZONE-SITE.XML_ozone.om.address: "om-0.om" OZONE-SITE.XML_ozone.recon.address: "recon-0.recon" diff --git a/hadoop-ozone/dist/src/main/k8s/examples/getting-started/config-configmap.yaml b/hadoop-ozone/dist/src/main/k8s/examples/getting-started/config-configmap.yaml index 666be83378cc..fd01af485d6b 100644 --- a/hadoop-ozone/dist/src/main/k8s/examples/getting-started/config-configmap.yaml +++ b/hadoop-ozone/dist/src/main/k8s/examples/getting-started/config-configmap.yaml @@ -31,6 +31,7 @@ data: OZONE-SITE.XML_hdds.scm.safemode.min.datanode: "3" OZONE-SITE.XML_ozone.datanode.pipeline.limit: "1" OZONE-SITE.XML_hdds.datanode.volume.min.free.space: "1GB" + OZONE-SITE.XML_ozone.scm.container.space.requirement.multiplier: "1.0" OZONE-SITE.XML_ozone.metadata.dirs: /data/metadata OZONE-SITE.XML_ozone.om.address: om-0.om OZONE-SITE.XML_ozone.recon.address: recon-0.recon diff --git a/hadoop-ozone/dist/src/main/k8s/examples/minikube/config-configmap.yaml b/hadoop-ozone/dist/src/main/k8s/examples/minikube/config-configmap.yaml index 666be83378cc..fd01af485d6b 100644 --- a/hadoop-ozone/dist/src/main/k8s/examples/minikube/config-configmap.yaml +++ b/hadoop-ozone/dist/src/main/k8s/examples/minikube/config-configmap.yaml @@ -31,6 +31,7 @@ data: OZONE-SITE.XML_hdds.scm.safemode.min.datanode: "3" OZONE-SITE.XML_ozone.datanode.pipeline.limit: "1" OZONE-SITE.XML_hdds.datanode.volume.min.free.space: "1GB" + OZONE-SITE.XML_ozone.scm.container.space.requirement.multiplier: "1.0" OZONE-SITE.XML_ozone.metadata.dirs: /data/metadata OZONE-SITE.XML_ozone.om.address: om-0.om OZONE-SITE.XML_ozone.recon.address: recon-0.recon diff --git a/hadoop-ozone/dist/src/main/k8s/examples/ozone-dev/config-configmap.yaml b/hadoop-ozone/dist/src/main/k8s/examples/ozone-dev/config-configmap.yaml index 55c865fe224b..ee028bc2526d 100644 --- a/hadoop-ozone/dist/src/main/k8s/examples/ozone-dev/config-configmap.yaml +++ b/hadoop-ozone/dist/src/main/k8s/examples/ozone-dev/config-configmap.yaml @@ -31,6 +31,7 @@ data: OZONE-SITE.XML_hdds.scm.safemode.min.datanode: "3" OZONE-SITE.XML_ozone.datanode.pipeline.limit: "1" OZONE-SITE.XML_hdds.datanode.volume.min.free.space: "1GB" + OZONE-SITE.XML_ozone.scm.container.space.requirement.multiplier: "1.0" OZONE-SITE.XML_ozone.metadata.dirs: /data/metadata OZONE-SITE.XML_ozone.om.address: om-0.om OZONE-SITE.XML_ozone.recon.address: recon-0.recon diff --git a/hadoop-ozone/dist/src/main/k8s/examples/ozone-ha/config-configmap.yaml b/hadoop-ozone/dist/src/main/k8s/examples/ozone-ha/config-configmap.yaml index 596fddfd9c39..b0cbe626c5aa 100644 --- a/hadoop-ozone/dist/src/main/k8s/examples/ozone-ha/config-configmap.yaml +++ b/hadoop-ozone/dist/src/main/k8s/examples/ozone-ha/config-configmap.yaml @@ -31,6 +31,7 @@ data: OZONE-SITE.XML_hdds.scm.safemode.min.datanode: "3" OZONE-SITE.XML_ozone.datanode.pipeline.limit: "1" OZONE-SITE.XML_hdds.datanode.volume.min.free.space: "1GB" + OZONE-SITE.XML_ozone.scm.container.space.requirement.multiplier: "1.0" OZONE-SITE.XML_ozone.metadata.dirs: /data/metadata OZONE-SITE.XML_ozone.om.address: om-0.om OZONE-SITE.XML_ozone.recon.address: recon-0.recon diff --git a/hadoop-ozone/dist/src/main/k8s/examples/ozone/config-configmap.yaml b/hadoop-ozone/dist/src/main/k8s/examples/ozone/config-configmap.yaml index 68a5697992c0..30b9ae985d09 100644 --- a/hadoop-ozone/dist/src/main/k8s/examples/ozone/config-configmap.yaml +++ b/hadoop-ozone/dist/src/main/k8s/examples/ozone/config-configmap.yaml @@ -31,6 +31,7 @@ data: OZONE-SITE.XML_hdds.scm.safemode.min.datanode: "3" OZONE-SITE.XML_ozone.datanode.pipeline.limit: "1" OZONE-SITE.XML_hdds.datanode.volume.min.free.space: "1GB" + OZONE-SITE.XML_ozone.scm.container.space.requirement.multiplier: "1.0" OZONE-SITE.XML_ozone.metadata.dirs: /data/metadata OZONE-SITE.XML_ozone.om.address: om-0.om OZONE-SITE.XML_ozone.recon.address: recon-0.recon From 3a816c7bec81d1834ec23456ecf2d54a4078ec14 Mon Sep 17 00:00:00 2001 From: rich7420 Date: Mon, 24 Nov 2025 23:06:13 +0800 Subject: [PATCH 04/10] try fix error --- .github/workflows/check.yml | 2 +- .github/workflows/intermittent-test-check.yml | 2 +- .github/workflows/repeat-acceptance.yml | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/check.yml b/.github/workflows/check.yml index a94508b7c6cd..791b5a5ff940 100644 --- a/.github/workflows/check.yml +++ b/.github/workflows/check.yml @@ -247,7 +247,7 @@ jobs: if [[ -s "target/${{ inputs.script }}/summary.md" ]]; then cat target/${{ inputs.script }}/summary.md >> $GITHUB_STEP_SUMMARY fi - hadoop-ozone/dev-support/checks/_summary.sh target/${{ inputs.script }}/summary.txt + hadoop-ozone/dev-support/checks/_summary.sh target/${{ inputs.script }}/summary.txt || true - name: Archive build results if: ${{ !cancelled() }} diff --git a/.github/workflows/intermittent-test-check.yml b/.github/workflows/intermittent-test-check.yml index 1dffcd8a9677..6d483b88ae8c 100644 --- a/.github/workflows/intermittent-test-check.yml +++ b/.github/workflows/intermittent-test-check.yml @@ -223,7 +223,7 @@ jobs: env: DEVELOCITY_ACCESS_KEY: ${{ secrets.DEVELOCITY_ACCESS_KEY }} - name: Summary of failures - run: hadoop-ozone/dev-support/checks/_summary.sh target/unit/summary.txt + run: hadoop-ozone/dev-support/checks/_summary.sh target/unit/summary.txt || true if: ${{ !cancelled() }} - name: Archive build results uses: actions/upload-artifact@v4 diff --git a/.github/workflows/repeat-acceptance.yml b/.github/workflows/repeat-acceptance.yml index 252e93518cc6..4660156d3402 100644 --- a/.github/workflows/repeat-acceptance.yml +++ b/.github/workflows/repeat-acceptance.yml @@ -150,7 +150,7 @@ jobs: KEEP_IMAGE: false continue-on-error: true - name: Summary of failures - run: hadoop-ozone/dev-support/checks/_summary.sh target/${{ github.job }}/summary.txt + run: hadoop-ozone/dev-support/checks/_summary.sh target/${{ github.job }}/summary.txt || true if: ${{ !cancelled() }} - name: Archive build results uses: actions/upload-artifact@v4 From 6757c3a1e628885aabdc2373ce3295702678e0c8 Mon Sep 17 00:00:00 2001 From: rich7420 Date: Tue, 25 Nov 2025 09:35:18 +0800 Subject: [PATCH 05/10] fix it --- .../hadoop/ozone/client/rpc/TestFailureHandlingByClient.java | 1 + 1 file changed, 1 insertion(+) diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/TestFailureHandlingByClient.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/TestFailureHandlingByClient.java index c262b8267f1a..1978d0482d75 100644 --- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/TestFailureHandlingByClient.java +++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/TestFailureHandlingByClient.java @@ -99,6 +99,7 @@ public void init() throws Exception { conf = new OzoneConfiguration(); chunkSize = (int) OzoneConsts.MB; blockSize = 4 * chunkSize; + conf.setDouble(ScmConfigKeys.OZONE_SCM_CONTAINER_SPACE_REQUIREMENT_MULTIPLIER, 1.0); conf.setTimeDuration(OZONE_SCM_STALENODE_INTERVAL, 100, TimeUnit.SECONDS); RatisClientConfig ratisClientConfig = From 744a85a5820a974cf82759f52fb4f85e87bd33e2 Mon Sep 17 00:00:00 2001 From: rich7420 Date: Wed, 26 Nov 2025 22:28:59 +0800 Subject: [PATCH 06/10] apply suggestions --- .github/workflows/check.yml | 4 +++- .github/workflows/intermittent-test-check.yml | 5 ++++- .github/workflows/repeat-acceptance.yml | 5 ++++- .../scm/container/ContainerManagerImpl.java | 17 +++++++---------- .../hdds/scm/pipeline/ECPipelineProvider.java | 9 +++------ .../scm/pipeline/RatisPipelineProvider.java | 11 ++++------- .../scm/container/TestContainerManagerImpl.java | 7 ++----- .../scm/pipeline/TestECPipelineProvider.java | 5 +---- .../src/main/k8s/definitions/ozone/config.yaml | 1 - .../getting-started/config-configmap.yaml | 1 - .../k8s/examples/minikube/config-configmap.yaml | 1 - .../examples/ozone-dev/config-configmap.yaml | 1 - .../k8s/examples/ozone-ha/config-configmap.yaml | 1 - .../k8s/examples/ozone/config-configmap.yaml | 1 - .../client/rpc/TestFailureHandlingByClient.java | 1 - 15 files changed, 28 insertions(+), 42 deletions(-) diff --git a/.github/workflows/check.yml b/.github/workflows/check.yml index 791b5a5ff940..702605a9a1d6 100644 --- a/.github/workflows/check.yml +++ b/.github/workflows/check.yml @@ -247,7 +247,9 @@ jobs: if [[ -s "target/${{ inputs.script }}/summary.md" ]]; then cat target/${{ inputs.script }}/summary.md >> $GITHUB_STEP_SUMMARY fi - hadoop-ozone/dev-support/checks/_summary.sh target/${{ inputs.script }}/summary.txt || true + if [[ -f "hadoop-ozone/dev-support/checks/_summary.sh" ]]; then + hadoop-ozone/dev-support/checks/_summary.sh target/${{ inputs.script }}/summary.txt + fi - name: Archive build results if: ${{ !cancelled() }} diff --git a/.github/workflows/intermittent-test-check.yml b/.github/workflows/intermittent-test-check.yml index 6d483b88ae8c..91f1922fd933 100644 --- a/.github/workflows/intermittent-test-check.yml +++ b/.github/workflows/intermittent-test-check.yml @@ -223,8 +223,11 @@ jobs: env: DEVELOCITY_ACCESS_KEY: ${{ secrets.DEVELOCITY_ACCESS_KEY }} - name: Summary of failures - run: hadoop-ozone/dev-support/checks/_summary.sh target/unit/summary.txt || true if: ${{ !cancelled() }} + run: | + if [[ -f "hadoop-ozone/dev-support/checks/_summary.sh" ]]; then + hadoop-ozone/dev-support/checks/_summary.sh target/unit/summary.txt + fi - name: Archive build results uses: actions/upload-artifact@v4 if: ${{ failure() }} diff --git a/.github/workflows/repeat-acceptance.yml b/.github/workflows/repeat-acceptance.yml index 4660156d3402..d10d946df9da 100644 --- a/.github/workflows/repeat-acceptance.yml +++ b/.github/workflows/repeat-acceptance.yml @@ -150,8 +150,11 @@ jobs: KEEP_IMAGE: false continue-on-error: true - name: Summary of failures - run: hadoop-ozone/dev-support/checks/_summary.sh target/${{ github.job }}/summary.txt || true if: ${{ !cancelled() }} + run: | + if [[ -f "hadoop-ozone/dev-support/checks/_summary.sh" ]]; then + hadoop-ozone/dev-support/checks/_summary.sh target/${{ github.job }}/summary.txt + fi - name: Archive build results uses: actions/upload-artifact@v4 if: always() diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/ContainerManagerImpl.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/ContainerManagerImpl.java index f54f595f8889..95ebb2cb237a 100644 --- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/ContainerManagerImpl.java +++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/ContainerManagerImpl.java @@ -46,6 +46,7 @@ import org.apache.hadoop.hdds.scm.ha.SequenceIdGenerator; import org.apache.hadoop.hdds.scm.pipeline.Pipeline; import org.apache.hadoop.hdds.scm.pipeline.PipelineManager; +import org.apache.hadoop.hdds.utils.HddsServerUtil; import org.apache.hadoop.hdds.utils.db.Table; import org.apache.hadoop.ozone.common.statemachine.InvalidStateTransitionException; import org.apache.hadoop.util.Time; @@ -82,7 +83,6 @@ public class ContainerManagerImpl implements ContainerManager { private final Random random = new Random(); private final long maxContainerSize; - private final long containerSpaceRequirement; /** * @@ -116,11 +116,6 @@ public ContainerManagerImpl( maxContainerSize = (long) conf.getStorageSize(ScmConfigKeys.OZONE_SCM_CONTAINER_SIZE, ScmConfigKeys.OZONE_SCM_CONTAINER_SIZE_DEFAULT, StorageUnit.BYTES); - double multiplier = conf.getDouble( - ScmConfigKeys.OZONE_SCM_CONTAINER_SPACE_REQUIREMENT_MULTIPLIER, - ScmConfigKeys.OZONE_SCM_CONTAINER_SPACE_REQUIREMENT_MULTIPLIER_DEFAULT); - this.containerSpaceRequirement = (long) (maxContainerSize * multiplier); - this.scmContainerManagerMetrics = SCMContainerManagerMetrics.create(); } @@ -358,23 +353,25 @@ public ContainerInfo getMatchingContainer(final long size, final String owner, synchronized (pipeline.getId()) { containerIDs = getContainersForOwner(pipeline, owner); if (containerIDs.size() < getOpenContainerCountPerPipeline(pipeline)) { - if (pipelineManager.hasEnoughSpace(pipeline, containerSpaceRequirement)) { + long requiredSpace = HddsServerUtil.requiredReplicationSpace(maxContainerSize); + if (pipelineManager.hasEnoughSpace(pipeline, requiredSpace)) { allocateContainer(pipeline, owner); containerIDs = getContainersForOwner(pipeline, owner); } else { LOG.debug("Cannot allocate a new container because pipeline {} does not have the required space {}.", - pipeline, containerSpaceRequirement); + pipeline, requiredSpace); } } containerIDs.removeAll(excludedContainerIDs); containerInfo = containerStateManager.getMatchingContainer( size, owner, pipeline.getId(), containerIDs); if (containerInfo == null) { - if (pipelineManager.hasEnoughSpace(pipeline, containerSpaceRequirement)) { + long requiredSpace = HddsServerUtil.requiredReplicationSpace(maxContainerSize); + if (pipelineManager.hasEnoughSpace(pipeline, requiredSpace)) { containerInfo = allocateContainer(pipeline, owner); } else { LOG.debug("Cannot allocate a new container because pipeline {} does not have the required space {}.", - pipeline, containerSpaceRequirement); + pipeline, requiredSpace); } } return containerInfo; diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/pipeline/ECPipelineProvider.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/pipeline/ECPipelineProvider.java index 7d345f8567e8..1b4de8619634 100644 --- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/pipeline/ECPipelineProvider.java +++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/pipeline/ECPipelineProvider.java @@ -35,6 +35,7 @@ import org.apache.hadoop.hdds.scm.node.NodeManager; import org.apache.hadoop.hdds.scm.node.NodeStatus; import org.apache.hadoop.hdds.scm.node.states.NodeNotFoundException; +import org.apache.hadoop.hdds.utils.HddsServerUtil; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -62,7 +63,6 @@ public class ECPipelineProvider extends PipelineProvider { private final ConfigurationSource conf; private final PlacementPolicy placementPolicy; private final long containerSizeBytes; - private final long containerSpaceRequirement; public ECPipelineProvider(NodeManager nodeManager, PipelineStateManager stateManager, @@ -74,10 +74,6 @@ public ECPipelineProvider(NodeManager nodeManager, this.containerSizeBytes = (long) this.conf .getStorageSize(ScmConfigKeys.OZONE_SCM_CONTAINER_SIZE, ScmConfigKeys.OZONE_SCM_CONTAINER_SIZE_DEFAULT, StorageUnit.BYTES); - double multiplier = conf.getDouble( - ScmConfigKeys.OZONE_SCM_CONTAINER_SPACE_REQUIREMENT_MULTIPLIER, - ScmConfigKeys.OZONE_SCM_CONTAINER_SPACE_REQUIREMENT_MULTIPLIER_DEFAULT); - this.containerSpaceRequirement = (long) (containerSizeBytes * multiplier); } @Override @@ -91,9 +87,10 @@ public synchronized Pipeline create(ECReplicationConfig replicationConfig) protected Pipeline create(ECReplicationConfig replicationConfig, List excludedNodes, List favoredNodes) throws IOException { + long requiredSpace = HddsServerUtil.requiredReplicationSpace(containerSizeBytes); List dns = placementPolicy .chooseDatanodes(excludedNodes, favoredNodes, - replicationConfig.getRequiredNodes(), 0, this.containerSpaceRequirement); + replicationConfig.getRequiredNodes(), 0, requiredSpace); return create(replicationConfig, dns); } diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/pipeline/RatisPipelineProvider.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/pipeline/RatisPipelineProvider.java index 6cc5d931257f..3a2918090294 100644 --- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/pipeline/RatisPipelineProvider.java +++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/pipeline/RatisPipelineProvider.java @@ -41,6 +41,7 @@ import org.apache.hadoop.hdds.scm.pipeline.leader.choose.algorithms.LeaderChoosePolicy; import org.apache.hadoop.hdds.scm.pipeline.leader.choose.algorithms.LeaderChoosePolicyFactory; import org.apache.hadoop.hdds.server.events.EventPublisher; +import org.apache.hadoop.hdds.utils.HddsServerUtil; import org.apache.hadoop.ozone.protocol.commands.ClosePipelineCommand; import org.apache.hadoop.ozone.protocol.commands.CommandForDatanode; import org.apache.hadoop.ozone.protocol.commands.CreatePipelineCommand; @@ -65,7 +66,6 @@ public class RatisPipelineProvider private final LeaderChoosePolicy leaderChoosePolicy; private final SCMContext scmContext; private final long containerSizeBytes; - private final long containerSpaceRequirement; private final long minRatisVolumeSizeBytes; @VisibleForTesting @@ -90,10 +90,6 @@ public RatisPipelineProvider(NodeManager nodeManager, ScmConfigKeys.OZONE_SCM_CONTAINER_SIZE, ScmConfigKeys.OZONE_SCM_CONTAINER_SIZE_DEFAULT, StorageUnit.BYTES); - double multiplier = conf.getDouble( - ScmConfigKeys.OZONE_SCM_CONTAINER_SPACE_REQUIREMENT_MULTIPLIER, - ScmConfigKeys.OZONE_SCM_CONTAINER_SPACE_REQUIREMENT_MULTIPLIER_DEFAULT); - this.containerSpaceRequirement = (long) (containerSizeBytes * multiplier); this.minRatisVolumeSizeBytes = (long) this.conf.getStorageSize( ScmConfigKeys.OZONE_DATANODE_RATIS_VOLUME_FREE_SPACE_MIN, ScmConfigKeys.OZONE_DATANODE_RATIS_VOLUME_FREE_SPACE_MIN_DEFAULT, @@ -167,10 +163,11 @@ public synchronized Pipeline create(RatisReplicationConfig replicationConfig, final ReplicationFactor factor = replicationConfig.getReplicationFactor(); + long requiredSpace = HddsServerUtil.requiredReplicationSpace(containerSizeBytes); switch (factor) { case ONE: dns = pickNodesNotUsed(replicationConfig, minRatisVolumeSizeBytes, - containerSpaceRequirement, conf); + requiredSpace, conf); break; case THREE: List excludeDueToEngagement = filterPipelineEngagement(); @@ -183,7 +180,7 @@ public synchronized Pipeline create(RatisReplicationConfig replicationConfig, } dns = placementPolicy.chooseDatanodes(excludedNodes, favoredNodes, factor.getNumber(), minRatisVolumeSizeBytes, - containerSpaceRequirement); + requiredSpace); break; default: throw new IllegalStateException("Unknown factor: " + factor.name()); diff --git a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/TestContainerManagerImpl.java b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/TestContainerManagerImpl.java index db77c91f4024..f500454f5681 100644 --- a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/TestContainerManagerImpl.java +++ b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/TestContainerManagerImpl.java @@ -50,7 +50,6 @@ import org.apache.hadoop.hdds.protocol.proto.HddsProtos; import org.apache.hadoop.hdds.protocol.proto.HddsProtos.LifeCycleState; import org.apache.hadoop.hdds.protocol.proto.HddsProtos.ReplicationFactor; -import org.apache.hadoop.hdds.scm.ScmConfigKeys; import org.apache.hadoop.hdds.scm.container.replication.ContainerReplicaPendingOps; import org.apache.hadoop.hdds.scm.container.states.ContainerStateMap; import org.apache.hadoop.hdds.scm.ha.SCMHAManager; @@ -184,16 +183,14 @@ public void testGetMatchingContainerReturnsContainerWhenEnoughSpaceInDatanodes() } @Test - public void testContainerSpaceRequirementMultiplier() throws IOException { + public void testContainerSpaceRequirement() throws IOException { long sizeRequired = 256 * 1024 * 1024; - double multiplier = 2.0; long containerSize = 5L * 1024 * 1024 * 1024; - long expectedSpaceRequirement = (long) (containerSize * multiplier); + long expectedSpaceRequirement = 2 * containerSize; PipelineManager spyPipelineManager = spy(pipelineManager); File tempDir = new File(testDir, "tempDir"); OzoneConfiguration conf = SCMTestUtils.getConf(tempDir); - conf.setDouble(ScmConfigKeys.OZONE_SCM_CONTAINER_SPACE_REQUIREMENT_MULTIPLIER, multiplier); ContainerManager manager = new ContainerManagerImpl(conf, scmhaManager, sequenceIdGen, spyPipelineManager, diff --git a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/pipeline/TestECPipelineProvider.java b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/pipeline/TestECPipelineProvider.java index 5c2cf871e9d9..69e361ccb02b 100644 --- a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/pipeline/TestECPipelineProvider.java +++ b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/pipeline/TestECPipelineProvider.java @@ -79,10 +79,7 @@ public void setup() throws IOException, NodeNotFoundException { ScmConfigKeys.OZONE_SCM_CONTAINER_SIZE, ScmConfigKeys.OZONE_SCM_CONTAINER_SIZE_DEFAULT, StorageUnit.BYTES); - double multiplier = conf.getDouble( - ScmConfigKeys.OZONE_SCM_CONTAINER_SPACE_REQUIREMENT_MULTIPLIER, - ScmConfigKeys.OZONE_SCM_CONTAINER_SPACE_REQUIREMENT_MULTIPLIER_DEFAULT); - this.containerSpaceRequirement = (long) (containerSizeBytes * multiplier); + this.containerSpaceRequirement = 2 * containerSizeBytes; // Placement policy will always return EC number of random nodes. when(placementPolicy.chooseDatanodes(anyList(), anyList(), anyInt(), anyLong(), diff --git a/hadoop-ozone/dist/src/main/k8s/definitions/ozone/config.yaml b/hadoop-ozone/dist/src/main/k8s/definitions/ozone/config.yaml index 83421b3a0514..e646b1c0af88 100644 --- a/hadoop-ozone/dist/src/main/k8s/definitions/ozone/config.yaml +++ b/hadoop-ozone/dist/src/main/k8s/definitions/ozone/config.yaml @@ -31,7 +31,6 @@ data: OZONE-SITE.XML_hdds.scm.safemode.min.datanode: "3" OZONE-SITE.XML_ozone.datanode.pipeline.limit: "1" OZONE-SITE.XML_hdds.datanode.volume.min.free.space: "1GB" - OZONE-SITE.XML_ozone.scm.container.space.requirement.multiplier: "1.0" OZONE-SITE.XML_ozone.metadata.dirs: "/data/metadata" OZONE-SITE.XML_ozone.om.address: "om-0.om" OZONE-SITE.XML_ozone.recon.address: "recon-0.recon" diff --git a/hadoop-ozone/dist/src/main/k8s/examples/getting-started/config-configmap.yaml b/hadoop-ozone/dist/src/main/k8s/examples/getting-started/config-configmap.yaml index fd01af485d6b..666be83378cc 100644 --- a/hadoop-ozone/dist/src/main/k8s/examples/getting-started/config-configmap.yaml +++ b/hadoop-ozone/dist/src/main/k8s/examples/getting-started/config-configmap.yaml @@ -31,7 +31,6 @@ data: OZONE-SITE.XML_hdds.scm.safemode.min.datanode: "3" OZONE-SITE.XML_ozone.datanode.pipeline.limit: "1" OZONE-SITE.XML_hdds.datanode.volume.min.free.space: "1GB" - OZONE-SITE.XML_ozone.scm.container.space.requirement.multiplier: "1.0" OZONE-SITE.XML_ozone.metadata.dirs: /data/metadata OZONE-SITE.XML_ozone.om.address: om-0.om OZONE-SITE.XML_ozone.recon.address: recon-0.recon diff --git a/hadoop-ozone/dist/src/main/k8s/examples/minikube/config-configmap.yaml b/hadoop-ozone/dist/src/main/k8s/examples/minikube/config-configmap.yaml index fd01af485d6b..666be83378cc 100644 --- a/hadoop-ozone/dist/src/main/k8s/examples/minikube/config-configmap.yaml +++ b/hadoop-ozone/dist/src/main/k8s/examples/minikube/config-configmap.yaml @@ -31,7 +31,6 @@ data: OZONE-SITE.XML_hdds.scm.safemode.min.datanode: "3" OZONE-SITE.XML_ozone.datanode.pipeline.limit: "1" OZONE-SITE.XML_hdds.datanode.volume.min.free.space: "1GB" - OZONE-SITE.XML_ozone.scm.container.space.requirement.multiplier: "1.0" OZONE-SITE.XML_ozone.metadata.dirs: /data/metadata OZONE-SITE.XML_ozone.om.address: om-0.om OZONE-SITE.XML_ozone.recon.address: recon-0.recon diff --git a/hadoop-ozone/dist/src/main/k8s/examples/ozone-dev/config-configmap.yaml b/hadoop-ozone/dist/src/main/k8s/examples/ozone-dev/config-configmap.yaml index ee028bc2526d..55c865fe224b 100644 --- a/hadoop-ozone/dist/src/main/k8s/examples/ozone-dev/config-configmap.yaml +++ b/hadoop-ozone/dist/src/main/k8s/examples/ozone-dev/config-configmap.yaml @@ -31,7 +31,6 @@ data: OZONE-SITE.XML_hdds.scm.safemode.min.datanode: "3" OZONE-SITE.XML_ozone.datanode.pipeline.limit: "1" OZONE-SITE.XML_hdds.datanode.volume.min.free.space: "1GB" - OZONE-SITE.XML_ozone.scm.container.space.requirement.multiplier: "1.0" OZONE-SITE.XML_ozone.metadata.dirs: /data/metadata OZONE-SITE.XML_ozone.om.address: om-0.om OZONE-SITE.XML_ozone.recon.address: recon-0.recon diff --git a/hadoop-ozone/dist/src/main/k8s/examples/ozone-ha/config-configmap.yaml b/hadoop-ozone/dist/src/main/k8s/examples/ozone-ha/config-configmap.yaml index b0cbe626c5aa..596fddfd9c39 100644 --- a/hadoop-ozone/dist/src/main/k8s/examples/ozone-ha/config-configmap.yaml +++ b/hadoop-ozone/dist/src/main/k8s/examples/ozone-ha/config-configmap.yaml @@ -31,7 +31,6 @@ data: OZONE-SITE.XML_hdds.scm.safemode.min.datanode: "3" OZONE-SITE.XML_ozone.datanode.pipeline.limit: "1" OZONE-SITE.XML_hdds.datanode.volume.min.free.space: "1GB" - OZONE-SITE.XML_ozone.scm.container.space.requirement.multiplier: "1.0" OZONE-SITE.XML_ozone.metadata.dirs: /data/metadata OZONE-SITE.XML_ozone.om.address: om-0.om OZONE-SITE.XML_ozone.recon.address: recon-0.recon diff --git a/hadoop-ozone/dist/src/main/k8s/examples/ozone/config-configmap.yaml b/hadoop-ozone/dist/src/main/k8s/examples/ozone/config-configmap.yaml index 30b9ae985d09..68a5697992c0 100644 --- a/hadoop-ozone/dist/src/main/k8s/examples/ozone/config-configmap.yaml +++ b/hadoop-ozone/dist/src/main/k8s/examples/ozone/config-configmap.yaml @@ -31,7 +31,6 @@ data: OZONE-SITE.XML_hdds.scm.safemode.min.datanode: "3" OZONE-SITE.XML_ozone.datanode.pipeline.limit: "1" OZONE-SITE.XML_hdds.datanode.volume.min.free.space: "1GB" - OZONE-SITE.XML_ozone.scm.container.space.requirement.multiplier: "1.0" OZONE-SITE.XML_ozone.metadata.dirs: /data/metadata OZONE-SITE.XML_ozone.om.address: om-0.om OZONE-SITE.XML_ozone.recon.address: recon-0.recon diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/TestFailureHandlingByClient.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/TestFailureHandlingByClient.java index 1978d0482d75..c262b8267f1a 100644 --- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/TestFailureHandlingByClient.java +++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/TestFailureHandlingByClient.java @@ -99,7 +99,6 @@ public void init() throws Exception { conf = new OzoneConfiguration(); chunkSize = (int) OzoneConsts.MB; blockSize = 4 * chunkSize; - conf.setDouble(ScmConfigKeys.OZONE_SCM_CONTAINER_SPACE_REQUIREMENT_MULTIPLIER, 1.0); conf.setTimeDuration(OZONE_SCM_STALENODE_INTERVAL, 100, TimeUnit.SECONDS); RatisClientConfig ratisClientConfig = From 74db8cf5740782aa1f935fd9fddaa112b15e6425 Mon Sep 17 00:00:00 2001 From: rich7420 Date: Thu, 27 Nov 2025 09:36:50 +0800 Subject: [PATCH 07/10] fix k8s test error with set 1GB container size --- hadoop-ozone/dist/src/main/k8s/definitions/ozone/config.yaml | 1 + .../src/main/k8s/examples/getting-started/config-configmap.yaml | 1 + .../dist/src/main/k8s/examples/minikube/config-configmap.yaml | 1 + .../dist/src/main/k8s/examples/ozone-dev/config-configmap.yaml | 1 + .../dist/src/main/k8s/examples/ozone-ha/config-configmap.yaml | 1 + .../dist/src/main/k8s/examples/ozone/config-configmap.yaml | 1 + 6 files changed, 6 insertions(+) diff --git a/hadoop-ozone/dist/src/main/k8s/definitions/ozone/config.yaml b/hadoop-ozone/dist/src/main/k8s/definitions/ozone/config.yaml index e646b1c0af88..e7914e55e87f 100644 --- a/hadoop-ozone/dist/src/main/k8s/definitions/ozone/config.yaml +++ b/hadoop-ozone/dist/src/main/k8s/definitions/ozone/config.yaml @@ -31,6 +31,7 @@ data: OZONE-SITE.XML_hdds.scm.safemode.min.datanode: "3" OZONE-SITE.XML_ozone.datanode.pipeline.limit: "1" OZONE-SITE.XML_hdds.datanode.volume.min.free.space: "1GB" + OZONE-SITE.XML_ozone.scm.container.size: "1GB" OZONE-SITE.XML_ozone.metadata.dirs: "/data/metadata" OZONE-SITE.XML_ozone.om.address: "om-0.om" OZONE-SITE.XML_ozone.recon.address: "recon-0.recon" diff --git a/hadoop-ozone/dist/src/main/k8s/examples/getting-started/config-configmap.yaml b/hadoop-ozone/dist/src/main/k8s/examples/getting-started/config-configmap.yaml index 666be83378cc..cef7ba08057b 100644 --- a/hadoop-ozone/dist/src/main/k8s/examples/getting-started/config-configmap.yaml +++ b/hadoop-ozone/dist/src/main/k8s/examples/getting-started/config-configmap.yaml @@ -31,6 +31,7 @@ data: OZONE-SITE.XML_hdds.scm.safemode.min.datanode: "3" OZONE-SITE.XML_ozone.datanode.pipeline.limit: "1" OZONE-SITE.XML_hdds.datanode.volume.min.free.space: "1GB" + OZONE-SITE.XML_ozone.scm.container.size: "1GB" OZONE-SITE.XML_ozone.metadata.dirs: /data/metadata OZONE-SITE.XML_ozone.om.address: om-0.om OZONE-SITE.XML_ozone.recon.address: recon-0.recon diff --git a/hadoop-ozone/dist/src/main/k8s/examples/minikube/config-configmap.yaml b/hadoop-ozone/dist/src/main/k8s/examples/minikube/config-configmap.yaml index 666be83378cc..cef7ba08057b 100644 --- a/hadoop-ozone/dist/src/main/k8s/examples/minikube/config-configmap.yaml +++ b/hadoop-ozone/dist/src/main/k8s/examples/minikube/config-configmap.yaml @@ -31,6 +31,7 @@ data: OZONE-SITE.XML_hdds.scm.safemode.min.datanode: "3" OZONE-SITE.XML_ozone.datanode.pipeline.limit: "1" OZONE-SITE.XML_hdds.datanode.volume.min.free.space: "1GB" + OZONE-SITE.XML_ozone.scm.container.size: "1GB" OZONE-SITE.XML_ozone.metadata.dirs: /data/metadata OZONE-SITE.XML_ozone.om.address: om-0.om OZONE-SITE.XML_ozone.recon.address: recon-0.recon diff --git a/hadoop-ozone/dist/src/main/k8s/examples/ozone-dev/config-configmap.yaml b/hadoop-ozone/dist/src/main/k8s/examples/ozone-dev/config-configmap.yaml index 55c865fe224b..ccd8d2c96130 100644 --- a/hadoop-ozone/dist/src/main/k8s/examples/ozone-dev/config-configmap.yaml +++ b/hadoop-ozone/dist/src/main/k8s/examples/ozone-dev/config-configmap.yaml @@ -31,6 +31,7 @@ data: OZONE-SITE.XML_hdds.scm.safemode.min.datanode: "3" OZONE-SITE.XML_ozone.datanode.pipeline.limit: "1" OZONE-SITE.XML_hdds.datanode.volume.min.free.space: "1GB" + OZONE-SITE.XML_ozone.scm.container.size: "1GB" OZONE-SITE.XML_ozone.metadata.dirs: /data/metadata OZONE-SITE.XML_ozone.om.address: om-0.om OZONE-SITE.XML_ozone.recon.address: recon-0.recon diff --git a/hadoop-ozone/dist/src/main/k8s/examples/ozone-ha/config-configmap.yaml b/hadoop-ozone/dist/src/main/k8s/examples/ozone-ha/config-configmap.yaml index 596fddfd9c39..f433cc04187c 100644 --- a/hadoop-ozone/dist/src/main/k8s/examples/ozone-ha/config-configmap.yaml +++ b/hadoop-ozone/dist/src/main/k8s/examples/ozone-ha/config-configmap.yaml @@ -31,6 +31,7 @@ data: OZONE-SITE.XML_hdds.scm.safemode.min.datanode: "3" OZONE-SITE.XML_ozone.datanode.pipeline.limit: "1" OZONE-SITE.XML_hdds.datanode.volume.min.free.space: "1GB" + OZONE-SITE.XML_ozone.scm.container.size: "1GB" OZONE-SITE.XML_ozone.metadata.dirs: /data/metadata OZONE-SITE.XML_ozone.om.address: om-0.om OZONE-SITE.XML_ozone.recon.address: recon-0.recon diff --git a/hadoop-ozone/dist/src/main/k8s/examples/ozone/config-configmap.yaml b/hadoop-ozone/dist/src/main/k8s/examples/ozone/config-configmap.yaml index 68a5697992c0..4faf6888bde5 100644 --- a/hadoop-ozone/dist/src/main/k8s/examples/ozone/config-configmap.yaml +++ b/hadoop-ozone/dist/src/main/k8s/examples/ozone/config-configmap.yaml @@ -31,6 +31,7 @@ data: OZONE-SITE.XML_hdds.scm.safemode.min.datanode: "3" OZONE-SITE.XML_ozone.datanode.pipeline.limit: "1" OZONE-SITE.XML_hdds.datanode.volume.min.free.space: "1GB" + OZONE-SITE.XML_ozone.scm.container.size: "1GB" OZONE-SITE.XML_ozone.metadata.dirs: /data/metadata OZONE-SITE.XML_ozone.om.address: om-0.om OZONE-SITE.XML_ozone.recon.address: recon-0.recon From 42adbc71f3bf4c288be434f15eba5c8921de36f7 Mon Sep 17 00:00:00 2001 From: rich7420 Date: Sat, 6 Dec 2025 10:36:49 +0800 Subject: [PATCH 08/10] update --- .github/workflows/check.yml | 4 +--- .github/workflows/intermittent-test-check.yml | 5 +---- .github/workflows/repeat-acceptance.yml | 5 +---- .../java/org/apache/hadoop/hdds/scm/ScmConfigKeys.java | 2 +- hadoop-hdds/common/src/main/resources/ozone-default.xml | 4 ++-- .../java/org/apache/hadoop/hdds/utils/HddsServerUtil.java | 7 +++++++ .../hadoop/hdds/scm/container/ContainerManagerImpl.java | 6 ++++-- .../hadoop/hdds/scm/pipeline/ECPipelineProvider.java | 2 +- .../hadoop/hdds/scm/pipeline/RatisPipelineProvider.java | 2 +- .../hdds/scm/container/TestContainerManagerImpl.java | 3 ++- .../hadoop/hdds/scm/pipeline/TestECPipelineProvider.java | 3 ++- 11 files changed, 23 insertions(+), 20 deletions(-) diff --git a/.github/workflows/check.yml b/.github/workflows/check.yml index 702605a9a1d6..a94508b7c6cd 100644 --- a/.github/workflows/check.yml +++ b/.github/workflows/check.yml @@ -247,9 +247,7 @@ jobs: if [[ -s "target/${{ inputs.script }}/summary.md" ]]; then cat target/${{ inputs.script }}/summary.md >> $GITHUB_STEP_SUMMARY fi - if [[ -f "hadoop-ozone/dev-support/checks/_summary.sh" ]]; then - hadoop-ozone/dev-support/checks/_summary.sh target/${{ inputs.script }}/summary.txt - fi + hadoop-ozone/dev-support/checks/_summary.sh target/${{ inputs.script }}/summary.txt - name: Archive build results if: ${{ !cancelled() }} diff --git a/.github/workflows/intermittent-test-check.yml b/.github/workflows/intermittent-test-check.yml index 91f1922fd933..1dffcd8a9677 100644 --- a/.github/workflows/intermittent-test-check.yml +++ b/.github/workflows/intermittent-test-check.yml @@ -223,11 +223,8 @@ jobs: env: DEVELOCITY_ACCESS_KEY: ${{ secrets.DEVELOCITY_ACCESS_KEY }} - name: Summary of failures + run: hadoop-ozone/dev-support/checks/_summary.sh target/unit/summary.txt if: ${{ !cancelled() }} - run: | - if [[ -f "hadoop-ozone/dev-support/checks/_summary.sh" ]]; then - hadoop-ozone/dev-support/checks/_summary.sh target/unit/summary.txt - fi - name: Archive build results uses: actions/upload-artifact@v4 if: ${{ failure() }} diff --git a/.github/workflows/repeat-acceptance.yml b/.github/workflows/repeat-acceptance.yml index d10d946df9da..252e93518cc6 100644 --- a/.github/workflows/repeat-acceptance.yml +++ b/.github/workflows/repeat-acceptance.yml @@ -150,11 +150,8 @@ jobs: KEEP_IMAGE: false continue-on-error: true - name: Summary of failures + run: hadoop-ozone/dev-support/checks/_summary.sh target/${{ github.job }}/summary.txt if: ${{ !cancelled() }} - run: | - if [[ -f "hadoop-ozone/dev-support/checks/_summary.sh" ]]; then - hadoop-ozone/dev-support/checks/_summary.sh target/${{ github.job }}/summary.txt - fi - name: Archive build results uses: actions/upload-artifact@v4 if: always() diff --git a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/ScmConfigKeys.java b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/ScmConfigKeys.java index 83937404edf4..0d99f1116ec4 100644 --- a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/ScmConfigKeys.java +++ b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/ScmConfigKeys.java @@ -361,7 +361,7 @@ public final class ScmConfigKeys { public static final String OZONE_SCM_CONTAINER_SPACE_REQUIREMENT_MULTIPLIER = "ozone.scm.container.space.requirement.multiplier"; - public static final double OZONE_SCM_CONTAINER_SPACE_REQUIREMENT_MULTIPLIER_DEFAULT = 2.0; + public static final double OZONE_SCM_CONTAINER_SPACE_REQUIREMENT_MULTIPLIER_DEFAULT = 5.0; public static final String OZONE_SCM_CONTAINER_LOCK_STRIPE_SIZE = "ozone.scm.container.lock.stripes"; diff --git a/hadoop-hdds/common/src/main/resources/ozone-default.xml b/hadoop-hdds/common/src/main/resources/ozone-default.xml index c07946d7fd31..59150ff320ac 100644 --- a/hadoop-hdds/common/src/main/resources/ozone-default.xml +++ b/hadoop-hdds/common/src/main/resources/ozone-default.xml @@ -1037,7 +1037,7 @@ ozone.scm.container.space.requirement.multiplier - 2.0 + 5.0 OZONE, SCM, MANAGEMENT Multiplier for container space requirement when checking if a datanode @@ -1045,7 +1045,7 @@ as container size multiplied by this value. This prevents concurrent clients from all passing the space check when there's only enough space for one container. For example, with default container size of 5GB and multiplier - of 2.0, the system will require 10GB of available space before allocating + of 5.0, the system will require 25GB of available space before allocating a new container. This ensures that if only 6GB is remaining, the check will fail, preventing multiple clients from attempting to create containers concurrently when there's only space for one. diff --git a/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/utils/HddsServerUtil.java b/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/utils/HddsServerUtil.java index 2da151faed6a..01624aac76cd 100644 --- a/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/utils/HddsServerUtil.java +++ b/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/utils/HddsServerUtil.java @@ -432,6 +432,13 @@ public static long requiredReplicationSpace(long defaultContainerSize) { return 2 * defaultContainerSize; } + public static long requiredReplicationSpace(long defaultContainerSize, ConfigurationSource conf) { + double multiplier = conf.getDouble( + ScmConfigKeys.OZONE_SCM_CONTAINER_SPACE_REQUIREMENT_MULTIPLIER, + ScmConfigKeys.OZONE_SCM_CONTAINER_SPACE_REQUIREMENT_MULTIPLIER_DEFAULT); + return (long) (multiplier * defaultContainerSize); + } + public static Collection getDatanodeStorageDirs(ConfigurationSource conf) { Collection rawLocations = conf.getTrimmedStringCollection(HDDS_DATANODE_DIR_KEY); if (rawLocations.isEmpty()) { diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/ContainerManagerImpl.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/ContainerManagerImpl.java index 95ebb2cb237a..709b53579848 100644 --- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/ContainerManagerImpl.java +++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/ContainerManagerImpl.java @@ -72,6 +72,7 @@ public class ContainerManagerImpl implements ContainerManager { private final SCMHAManager haManager; private final SequenceIdGenerator sequenceIdGen; + private final Configuration conf; // TODO: Revisit this. // Metrics related to operations should be moved to ProtocolServer @@ -100,6 +101,7 @@ public ContainerManagerImpl( this.pipelineManager = pipelineManager; this.haManager = scmHaManager; this.sequenceIdGen = sequenceIdGen; + this.conf = conf; this.containerStateManager = ContainerStateManagerImpl.newBuilder() .setConfiguration(conf) .setPipelineManager(pipelineManager) @@ -353,7 +355,7 @@ public ContainerInfo getMatchingContainer(final long size, final String owner, synchronized (pipeline.getId()) { containerIDs = getContainersForOwner(pipeline, owner); if (containerIDs.size() < getOpenContainerCountPerPipeline(pipeline)) { - long requiredSpace = HddsServerUtil.requiredReplicationSpace(maxContainerSize); + long requiredSpace = HddsServerUtil.requiredReplicationSpace(maxContainerSize, conf); if (pipelineManager.hasEnoughSpace(pipeline, requiredSpace)) { allocateContainer(pipeline, owner); containerIDs = getContainersForOwner(pipeline, owner); @@ -366,7 +368,7 @@ public ContainerInfo getMatchingContainer(final long size, final String owner, containerInfo = containerStateManager.getMatchingContainer( size, owner, pipeline.getId(), containerIDs); if (containerInfo == null) { - long requiredSpace = HddsServerUtil.requiredReplicationSpace(maxContainerSize); + long requiredSpace = HddsServerUtil.requiredReplicationSpace(maxContainerSize, conf); if (pipelineManager.hasEnoughSpace(pipeline, requiredSpace)) { containerInfo = allocateContainer(pipeline, owner); } else { diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/pipeline/ECPipelineProvider.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/pipeline/ECPipelineProvider.java index 1b4de8619634..4da90e8a3f15 100644 --- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/pipeline/ECPipelineProvider.java +++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/pipeline/ECPipelineProvider.java @@ -87,7 +87,7 @@ public synchronized Pipeline create(ECReplicationConfig replicationConfig) protected Pipeline create(ECReplicationConfig replicationConfig, List excludedNodes, List favoredNodes) throws IOException { - long requiredSpace = HddsServerUtil.requiredReplicationSpace(containerSizeBytes); + long requiredSpace = HddsServerUtil.requiredReplicationSpace(containerSizeBytes, conf); List dns = placementPolicy .chooseDatanodes(excludedNodes, favoredNodes, replicationConfig.getRequiredNodes(), 0, requiredSpace); diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/pipeline/RatisPipelineProvider.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/pipeline/RatisPipelineProvider.java index 3a2918090294..7ec47f9d1b1b 100644 --- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/pipeline/RatisPipelineProvider.java +++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/pipeline/RatisPipelineProvider.java @@ -163,7 +163,7 @@ public synchronized Pipeline create(RatisReplicationConfig replicationConfig, final ReplicationFactor factor = replicationConfig.getReplicationFactor(); - long requiredSpace = HddsServerUtil.requiredReplicationSpace(containerSizeBytes); + long requiredSpace = HddsServerUtil.requiredReplicationSpace(containerSizeBytes, conf); switch (factor) { case ONE: dns = pickNodesNotUsed(replicationConfig, minRatisVolumeSizeBytes, diff --git a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/TestContainerManagerImpl.java b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/TestContainerManagerImpl.java index f500454f5681..4daebb72cd57 100644 --- a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/TestContainerManagerImpl.java +++ b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/TestContainerManagerImpl.java @@ -60,6 +60,7 @@ import org.apache.hadoop.hdds.scm.pipeline.MockPipelineManager; import org.apache.hadoop.hdds.scm.pipeline.Pipeline; import org.apache.hadoop.hdds.scm.pipeline.PipelineManager; +import org.apache.hadoop.hdds.utils.HddsServerUtil; import org.apache.hadoop.hdds.utils.db.DBStore; import org.apache.hadoop.hdds.utils.db.DBStoreBuilder; import org.apache.hadoop.ozone.common.statemachine.InvalidStateTransitionException; @@ -186,11 +187,11 @@ public void testGetMatchingContainerReturnsContainerWhenEnoughSpaceInDatanodes() public void testContainerSpaceRequirement() throws IOException { long sizeRequired = 256 * 1024 * 1024; long containerSize = 5L * 1024 * 1024 * 1024; - long expectedSpaceRequirement = 2 * containerSize; PipelineManager spyPipelineManager = spy(pipelineManager); File tempDir = new File(testDir, "tempDir"); OzoneConfiguration conf = SCMTestUtils.getConf(tempDir); + long expectedSpaceRequirement = HddsServerUtil.requiredReplicationSpace(containerSize, conf); ContainerManager manager = new ContainerManagerImpl(conf, scmhaManager, sequenceIdGen, spyPipelineManager, diff --git a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/pipeline/TestECPipelineProvider.java b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/pipeline/TestECPipelineProvider.java index 69e361ccb02b..b39beebdfa9c 100644 --- a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/pipeline/TestECPipelineProvider.java +++ b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/pipeline/TestECPipelineProvider.java @@ -55,6 +55,7 @@ import org.apache.hadoop.hdds.scm.node.NodeManager; import org.apache.hadoop.hdds.scm.node.NodeStatus; import org.apache.hadoop.hdds.scm.node.states.NodeNotFoundException; +import org.apache.hadoop.hdds.utils.HddsServerUtil; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; @@ -79,7 +80,7 @@ public void setup() throws IOException, NodeNotFoundException { ScmConfigKeys.OZONE_SCM_CONTAINER_SIZE, ScmConfigKeys.OZONE_SCM_CONTAINER_SIZE_DEFAULT, StorageUnit.BYTES); - this.containerSpaceRequirement = 2 * containerSizeBytes; + this.containerSpaceRequirement = HddsServerUtil.requiredReplicationSpace(containerSizeBytes, conf); // Placement policy will always return EC number of random nodes. when(placementPolicy.chooseDatanodes(anyList(), anyList(), anyInt(), anyLong(), From b808305a5a66acd5b922a0be8158d337d51926e1 Mon Sep 17 00:00:00 2001 From: rich7420 Date: Sat, 6 Dec 2025 10:44:03 +0800 Subject: [PATCH 09/10] fix errors --- .../hadoop/hdds/scm/container/ContainerManagerImpl.java | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/ContainerManagerImpl.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/ContainerManagerImpl.java index 709b53579848..b7eaaafd4d99 100644 --- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/ContainerManagerImpl.java +++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/ContainerManagerImpl.java @@ -32,6 +32,7 @@ import java.util.concurrent.locks.ReentrantLock; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.conf.StorageUnit; +import org.apache.hadoop.hdds.conf.OzoneConfiguration; import org.apache.hadoop.hdds.client.ECReplicationConfig; import org.apache.hadoop.hdds.client.ReplicationConfig; import org.apache.hadoop.hdds.protocol.proto.HddsProtos; @@ -72,7 +73,7 @@ public class ContainerManagerImpl implements ContainerManager { private final SCMHAManager haManager; private final SequenceIdGenerator sequenceIdGen; - private final Configuration conf; + private final OzoneConfiguration conf; // TODO: Revisit this. // Metrics related to operations should be moved to ProtocolServer @@ -101,7 +102,11 @@ public ContainerManagerImpl( this.pipelineManager = pipelineManager; this.haManager = scmHaManager; this.sequenceIdGen = sequenceIdGen; - this.conf = conf; + if (conf instanceof OzoneConfiguration) { + this.conf = (OzoneConfiguration) conf; + } else { + this.conf = new OzoneConfiguration(conf); + } this.containerStateManager = ContainerStateManagerImpl.newBuilder() .setConfiguration(conf) .setPipelineManager(pipelineManager) From c3118ab34196cf4d483c4c2a56e435ab32f12dd4 Mon Sep 17 00:00:00 2001 From: rich7420 Date: Sat, 6 Dec 2025 10:52:14 +0800 Subject: [PATCH 10/10] fix import errors --- .../apache/hadoop/hdds/scm/container/ContainerManagerImpl.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/ContainerManagerImpl.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/ContainerManagerImpl.java index b7eaaafd4d99..921fedc77371 100644 --- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/ContainerManagerImpl.java +++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/ContainerManagerImpl.java @@ -32,9 +32,9 @@ import java.util.concurrent.locks.ReentrantLock; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.conf.StorageUnit; -import org.apache.hadoop.hdds.conf.OzoneConfiguration; import org.apache.hadoop.hdds.client.ECReplicationConfig; import org.apache.hadoop.hdds.client.ReplicationConfig; +import org.apache.hadoop.hdds.conf.OzoneConfiguration; import org.apache.hadoop.hdds.protocol.proto.HddsProtos; import org.apache.hadoop.hdds.protocol.proto.HddsProtos.ContainerInfoProto; import org.apache.hadoop.hdds.protocol.proto.HddsProtos.LifeCycleEvent;