From bd2675549fe57b4eb0f21042ee633514e4b5b6df Mon Sep 17 00:00:00 2001 From: jianghuazhu <740087514@qq.com> Date: Sun, 29 Sep 2024 00:06:48 +0800 Subject: [PATCH 1/4] HDDS-11494. Improve the duration option of freon --- .../hadoop/ozone/freon/BaseFreonGenerator.java | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/freon/BaseFreonGenerator.java b/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/freon/BaseFreonGenerator.java index 20acad0562ab..c0599c789271 100644 --- a/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/freon/BaseFreonGenerator.java +++ b/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/freon/BaseFreonGenerator.java @@ -189,12 +189,12 @@ private void taskLoop(TaskProvider provider) { completed.set(true); break; } - } else { - //in case of an other failed test, we shouldn't execute more tasks. - if (counter >= testNo || (!failAtEnd && failureCounter.get() > 0)) { - completed.set(true); - break; - } + } + + //in case of an other failed test, we shouldn't execute more tasks. + if (counter >= testNo || (!failAtEnd && failureCounter.get() > 0)) { + completed.set(true); + break; } tryNextTask(provider, counter % testNo); From 804b0304fa0ba376998b018da202c2d28adf481d Mon Sep 17 00:00:00 2001 From: jianghuazhu <740087514@qq.com> Date: Sun, 29 Sep 2024 21:07:19 +0800 Subject: [PATCH 2/4] Optimize duration option. --- .../ozone/freon/BaseFreonGenerator.java | 29 ++++++++++++++----- .../hadoop/ozone/freon/OmBucketGenerator.java | 12 ++++++++ 2 files changed, 33 insertions(+), 8 deletions(-) diff --git a/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/freon/BaseFreonGenerator.java b/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/freon/BaseFreonGenerator.java index c0599c789271..0f5f39cf3729 100644 --- a/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/freon/BaseFreonGenerator.java +++ b/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/freon/BaseFreonGenerator.java @@ -189,12 +189,12 @@ private void taskLoop(TaskProvider provider) { completed.set(true); break; } - } - - //in case of an other failed test, we shouldn't execute more tasks. - if (counter >= testNo || (!failAtEnd && failureCounter.get() > 0)) { - completed.set(true); - break; + } else { + //in case of an other failed test, we shouldn't execute more tasks. + if (counter >= testNo || (!failAtEnd && failureCounter.get() > 0)) { + completed.set(true); + break; + } } tryNextTask(provider, counter % testNo); @@ -290,7 +290,7 @@ public void init() { //replace environment variables to support multi-node execution prefix = resolvePrefix(prefix); } - if (duration != null) { + if (duration != null && allowDuration()) { durationInSecond = TimeDurationUtil.getTimeDurationHelper( "--runtime", duration, TimeUnit.SECONDS); if (durationInSecond <= 0) { @@ -327,7 +327,7 @@ public void init() { executor = Executors.newFixedThreadPool(threadNo); long maxValue; LongSupplier supplier; - if (duration != null) { + if (duration != null && allowDuration()) { maxValue = durationInSecond; supplier = () -> Duration.between( Instant.ofEpochMilli(startTime), Instant.now()).getSeconds(); @@ -554,6 +554,19 @@ public String getPrefix() { return prefix; } + /** + * Whether to enable Duration. + * If enabled, the command will load the --duration option. + * If not enabled, the command will not load the --duration option. + */ + public boolean allowDuration() { + return true; + } + + public String getDuration() { + return duration; + } + public MetricRegistry getMetrics() { return metrics; } diff --git a/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/freon/OmBucketGenerator.java b/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/freon/OmBucketGenerator.java index a3e21d58e2f7..65c55a1ebce4 100644 --- a/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/freon/OmBucketGenerator.java +++ b/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/freon/OmBucketGenerator.java @@ -26,6 +26,8 @@ import org.apache.hadoop.ozone.om.protocol.OzoneManagerProtocol; import com.codahale.metrics.Timer; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; import picocli.CommandLine.Command; import picocli.CommandLine.Option; @@ -41,6 +43,8 @@ public class OmBucketGenerator extends BaseFreonGenerator implements Callable { + private static final Logger LOG = LoggerFactory.getLogger(OmBucketGenerator.class); + @Option(names = {"-v", "--volume"}, description = "Name of the volume which contains the test data. Will be" + " created if missing.", @@ -57,8 +61,16 @@ public class OmBucketGenerator extends BaseFreonGenerator private Timer bucketCreationTimer; + @Override + public boolean allowDuration() { + return false; + } + @Override public Void call() throws Exception { + if (getDuration() != null) { + LOG.warn("When running ombg, setting --duration has no effect."); + } init(); From 85807e3b8c50e0b81999b4fc07271a0b748341bc Mon Sep 17 00:00:00 2001 From: jianghuazhu <740087514@qq.com> Date: Mon, 30 Sep 2024 22:28:23 +0800 Subject: [PATCH 3/4] Optimize duration option. --- .../hadoop/ozone/freon/BaseFreonGenerator.java | 12 ++++++------ .../apache/hadoop/ozone/freon/OmBucketGenerator.java | 7 ------- 2 files changed, 6 insertions(+), 13 deletions(-) diff --git a/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/freon/BaseFreonGenerator.java b/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/freon/BaseFreonGenerator.java index 0f5f39cf3729..ee9fdc9865b1 100644 --- a/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/freon/BaseFreonGenerator.java +++ b/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/freon/BaseFreonGenerator.java @@ -290,7 +290,11 @@ public void init() { //replace environment variables to support multi-node execution prefix = resolvePrefix(prefix); } - if (duration != null && allowDuration()) { + if (!allowDuration()) { + LOG.warn("--duration is ignored"); + duration = null; + } + if (duration != null) { durationInSecond = TimeDurationUtil.getTimeDurationHelper( "--runtime", duration, TimeUnit.SECONDS); if (durationInSecond <= 0) { @@ -327,7 +331,7 @@ public void init() { executor = Executors.newFixedThreadPool(threadNo); long maxValue; LongSupplier supplier; - if (duration != null && allowDuration()) { + if (duration != null) { maxValue = durationInSecond; supplier = () -> Duration.between( Instant.ofEpochMilli(startTime), Instant.now()).getSeconds(); @@ -563,10 +567,6 @@ public boolean allowDuration() { return true; } - public String getDuration() { - return duration; - } - public MetricRegistry getMetrics() { return metrics; } diff --git a/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/freon/OmBucketGenerator.java b/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/freon/OmBucketGenerator.java index 65c55a1ebce4..27ebc8776330 100644 --- a/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/freon/OmBucketGenerator.java +++ b/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/freon/OmBucketGenerator.java @@ -26,8 +26,6 @@ import org.apache.hadoop.ozone.om.protocol.OzoneManagerProtocol; import com.codahale.metrics.Timer; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; import picocli.CommandLine.Command; import picocli.CommandLine.Option; @@ -43,8 +41,6 @@ public class OmBucketGenerator extends BaseFreonGenerator implements Callable { - private static final Logger LOG = LoggerFactory.getLogger(OmBucketGenerator.class); - @Option(names = {"-v", "--volume"}, description = "Name of the volume which contains the test data. Will be" + " created if missing.", @@ -68,9 +64,6 @@ public boolean allowDuration() { @Override public Void call() throws Exception { - if (getDuration() != null) { - LOG.warn("When running ombg, setting --duration has no effect."); - } init(); From 98d6dddb9cacbc9ed93ef2c67e265179954a57a4 Mon Sep 17 00:00:00 2001 From: "Doroszlai, Attila" <6454655+adoroszlai@users.noreply.github.com> Date: Tue, 1 Oct 2024 11:09:36 +0200 Subject: [PATCH 4/4] only warn if --duration was set --- .../java/org/apache/hadoop/ozone/freon/BaseFreonGenerator.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/freon/BaseFreonGenerator.java b/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/freon/BaseFreonGenerator.java index ee9fdc9865b1..3627b917f006 100644 --- a/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/freon/BaseFreonGenerator.java +++ b/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/freon/BaseFreonGenerator.java @@ -290,7 +290,7 @@ public void init() { //replace environment variables to support multi-node execution prefix = resolvePrefix(prefix); } - if (!allowDuration()) { + if (duration != null && !allowDuration()) { LOG.warn("--duration is ignored"); duration = null; }