From 2a191c713b074e8b9571904c5dc77e4b0684d6d0 Mon Sep 17 00:00:00 2001 From: Sreeja Chintalapati Date: Wed, 5 Feb 2025 15:14:29 +0530 Subject: [PATCH 1/4] HDDS-10607. Removed ozone.block.deleting.container.limit.per.interval property as it's not used anymore --- .../common/src/main/resources/ozone-default.xml | 11 ----------- .../container/common/TestBlockDeletingService.java | 2 -- .../common/TestSchemaOneBackwardsCompatibility.java | 3 +-- .../common/TestSchemaTwoBackwardsCompatibility.java | 2 -- 4 files changed, 1 insertion(+), 17 deletions(-) diff --git a/hadoop-hdds/common/src/main/resources/ozone-default.xml b/hadoop-hdds/common/src/main/resources/ozone-default.xml index f7ff9089b926..649a5d32d03d 100644 --- a/hadoop-hdds/common/src/main/resources/ozone-default.xml +++ b/hadoop-hdds/common/src/main/resources/ozone-default.xml @@ -426,17 +426,6 @@ or be directly or indirectly in a group defined in this property. - - ozone.block.deleting.container.limit.per.interval - 10 - OZONE, PERFORMANCE, SCM - A maximum number of containers to be scanned by block deleting - service per - time interval. The block deleting service spawns a thread to handle block - deletions in a container. This property is used to throttle the number of - threads spawned for block deletions. - - ozone.block.deleting.limit.per.task 1000 diff --git a/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/common/TestBlockDeletingService.java b/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/common/TestBlockDeletingService.java index bc56141fb080..057b94437842 100644 --- a/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/common/TestBlockDeletingService.java +++ b/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/common/TestBlockDeletingService.java @@ -95,7 +95,6 @@ import static java.nio.charset.StandardCharsets.UTF_8; import static org.apache.commons.lang3.RandomStringUtils.randomAlphanumeric; -import static org.apache.hadoop.ozone.OzoneConfigKeys.OZONE_BLOCK_DELETING_CONTAINER_LIMIT_PER_INTERVAL; import static org.apache.hadoop.ozone.OzoneConfigKeys.OZONE_BLOCK_DELETING_LIMIT_PER_CONTAINER; import static org.apache.hadoop.ozone.OzoneConfigKeys.OZONE_BLOCK_DELETING_SERVICE_INTERVAL; import static org.apache.hadoop.ozone.OzoneConsts.SCHEMA_V1; @@ -770,7 +769,6 @@ public void testShutdownService(ContainerTestVersionInfo versionInfo) setLayoutAndSchemaForTest(versionInfo); conf.setTimeDuration(OZONE_BLOCK_DELETING_SERVICE_INTERVAL, 500, TimeUnit.MILLISECONDS); - conf.setInt(OZONE_BLOCK_DELETING_CONTAINER_LIMIT_PER_INTERVAL, 10); conf.setInt(OZONE_BLOCK_DELETING_LIMIT_PER_CONTAINER, 10); ContainerSet containerSet = new ContainerSet(1000); diff --git a/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/common/TestSchemaOneBackwardsCompatibility.java b/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/common/TestSchemaOneBackwardsCompatibility.java index 2235b23ce882..5ca17ba09dda 100644 --- a/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/common/TestSchemaOneBackwardsCompatibility.java +++ b/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/common/TestSchemaOneBackwardsCompatibility.java @@ -66,7 +66,7 @@ import java.util.Arrays; import java.util.stream.Collectors; -import static org.apache.hadoop.ozone.OzoneConfigKeys.OZONE_BLOCK_DELETING_CONTAINER_LIMIT_PER_INTERVAL; + import static org.assertj.core.api.Assertions.assertThat; import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertNull; @@ -555,7 +555,6 @@ public void testReadDeletedBlocks(String schemaVersion) throws Exception { private void runBlockDeletingService(KeyValueHandler keyValueHandler) throws Exception { - conf.setInt(OZONE_BLOCK_DELETING_CONTAINER_LIMIT_PER_INTERVAL, 10); conf.setInt(OzoneConfigKeys.OZONE_BLOCK_DELETING_LIMIT_PER_CONTAINER, 2); OzoneContainer container = makeMockOzoneContainer(keyValueHandler); diff --git a/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/common/TestSchemaTwoBackwardsCompatibility.java b/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/common/TestSchemaTwoBackwardsCompatibility.java index 0c4612b79fa2..98d19ea64c65 100644 --- a/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/common/TestSchemaTwoBackwardsCompatibility.java +++ b/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/common/TestSchemaTwoBackwardsCompatibility.java @@ -68,7 +68,6 @@ import static java.nio.charset.StandardCharsets.UTF_8; import static org.apache.commons.lang3.RandomStringUtils.randomAlphanumeric; -import static org.apache.hadoop.ozone.OzoneConfigKeys.OZONE_BLOCK_DELETING_CONTAINER_LIMIT_PER_INTERVAL; import static org.apache.hadoop.ozone.OzoneConsts.BLOCK_COUNT; import static org.apache.hadoop.ozone.OzoneConsts.CONTAINER_BYTES_USED; import static org.apache.hadoop.ozone.OzoneConsts.PENDING_DELETE_BLOCK_COUNT; @@ -224,7 +223,6 @@ public void testReadMetadata() throws IOException { @Test public void testDeleteViaTransation() throws IOException, TimeoutException, InterruptedException { - conf.setInt(OZONE_BLOCK_DELETING_CONTAINER_LIMIT_PER_INTERVAL, 10); conf.setInt(OzoneConfigKeys.OZONE_BLOCK_DELETING_LIMIT_PER_CONTAINER, BLOCKS_PER_CONTAINER); From 13aadda42c19633bb52fb23585f5ddc61c8f494b Mon Sep 17 00:00:00 2001 From: Sreeja Chintalapati Date: Wed, 5 Feb 2025 16:42:28 +0530 Subject: [PATCH 2/4] HDDS-10607. Removed the ozone.block.deleting.container.limit.per.interval property as it's not used anymore --- .../main/java/org/apache/hadoop/ozone/OzoneConfigKeys.java | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/OzoneConfigKeys.java b/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/OzoneConfigKeys.java index 4c0df91e1a63..de2850f38d8e 100644 --- a/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/OzoneConfigKeys.java +++ b/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/OzoneConfigKeys.java @@ -333,10 +333,7 @@ public final class OzoneConfigKeys { public static final int OZONE_BLOCK_DELETING_LIMIT_PER_CONTAINER_DEFAULT = 1000; - public static final String OZONE_BLOCK_DELETING_CONTAINER_LIMIT_PER_INTERVAL - = "ozone.block.deleting.container.limit.per.interval"; - public static final int - OZONE_BLOCK_DELETING_CONTAINER_LIMIT_PER_INTERVAL_DEFAULT = 10; + public static final String HDDS_CONTAINER_RATIS_ENABLED_KEY = ScmConfigKeys.HDDS_CONTAINER_RATIS_ENABLED_KEY; From e4b5f6ba8bdbe9b0b3cec8e6ed9bd3211fce72c1 Mon Sep 17 00:00:00 2001 From: sreejasahithi <115860222+sreejasahithi@users.noreply.github.com> Date: Wed, 5 Feb 2025 21:28:32 +0530 Subject: [PATCH 3/4] Update hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/OzoneConfigKeys.java Co-authored-by: Doroszlai, Attila <6454655+adoroszlai@users.noreply.github.com> --- .../src/main/java/org/apache/hadoop/ozone/OzoneConfigKeys.java | 2 -- 1 file changed, 2 deletions(-) diff --git a/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/OzoneConfigKeys.java b/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/OzoneConfigKeys.java index de2850f38d8e..102207dd111d 100644 --- a/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/OzoneConfigKeys.java +++ b/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/OzoneConfigKeys.java @@ -333,8 +333,6 @@ public final class OzoneConfigKeys { public static final int OZONE_BLOCK_DELETING_LIMIT_PER_CONTAINER_DEFAULT = 1000; - - public static final String HDDS_CONTAINER_RATIS_ENABLED_KEY = ScmConfigKeys.HDDS_CONTAINER_RATIS_ENABLED_KEY; public static final boolean HDDS_CONTAINER_RATIS_ENABLED_DEFAULT From 08fc90db255e03811a6312847e1c609d07f09098 Mon Sep 17 00:00:00 2001 From: sreejasahithi <115860222+sreejasahithi@users.noreply.github.com> Date: Wed, 5 Feb 2025 21:28:53 +0530 Subject: [PATCH 4/4] Update hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/common/TestSchemaOneBackwardsCompatibility.java Co-authored-by: Doroszlai, Attila <6454655+adoroszlai@users.noreply.github.com> --- .../container/common/TestSchemaOneBackwardsCompatibility.java | 1 - 1 file changed, 1 deletion(-) diff --git a/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/common/TestSchemaOneBackwardsCompatibility.java b/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/common/TestSchemaOneBackwardsCompatibility.java index 5ca17ba09dda..e05192f78916 100644 --- a/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/common/TestSchemaOneBackwardsCompatibility.java +++ b/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/common/TestSchemaOneBackwardsCompatibility.java @@ -66,7 +66,6 @@ import java.util.Arrays; import java.util.stream.Collectors; - import static org.assertj.core.api.Assertions.assertThat; import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertNull;