From bb24b768967b9a83d2d72119696f793e5f2cf798 Mon Sep 17 00:00:00 2001 From: captainzmc Date: Tue, 18 Jan 2022 11:01:20 +0800 Subject: [PATCH 1/9] freon ockg support EC --- .../ozone/freon/OzoneClientKeyGenerator.java | 28 +++++++++++++------ 1 file changed, 19 insertions(+), 9 deletions(-) diff --git a/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/freon/OzoneClientKeyGenerator.java b/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/freon/OzoneClientKeyGenerator.java index cf5e2604e783..1d42060ad896 100644 --- a/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/freon/OzoneClientKeyGenerator.java +++ b/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/freon/OzoneClientKeyGenerator.java @@ -22,7 +22,7 @@ import java.util.concurrent.Callable; import org.apache.hadoop.hdds.cli.HddsVersionProvider; -import org.apache.hadoop.hdds.client.ReplicationFactor; +import org.apache.hadoop.hdds.client.ReplicationConfig; import org.apache.hadoop.hdds.client.ReplicationType; import org.apache.hadoop.hdds.conf.OzoneConfiguration; import org.apache.hadoop.ozone.client.OzoneBucket; @@ -66,23 +66,31 @@ public class OzoneClientKeyGenerator extends BaseFreonGenerator defaultValue = "4096") private int bufferSize; - @Option(names = { "-F", "--factor" }, - description = "Replication factor (ONE, THREE)", - defaultValue = "THREE" - ) - private ReplicationFactor factor = ReplicationFactor.THREE; - @Option( names = "--om-service-id", description = "OM Service ID" ) private String omServiceID = null; + @Option(names = {"--replication"}, + description = + "Replication configuration of the new key. (this is replication " + + "specific. for RATIS/STANDALONE you can use ONE or THREE, " + + "for EC you can use rs-3-2-1024k, rs-6-3-1024 or rs-10-4-1024k.)" + + " Default is specified in the cluster-wide config.") + private String replication; + + @Option(names = {"--type"}, + description = "Replication type of the new key. (use RATIS or " + + "STAND_ALONE or EC) Default is specified in the cluster-wide config.") + private ReplicationType replicationType; + private Timer timer; private OzoneBucket bucket; private ContentGenerator contentGenerator; private Map metadata; + private ReplicationConfig replicationConfig; @Override public Void call() throws Exception { @@ -93,6 +101,8 @@ public Void call() throws Exception { contentGenerator = new ContentGenerator(keySize, bufferSize); metadata = new HashMap<>(); + replicationConfig = ReplicationConfig.parse(replicationType, replication, + ozoneConfiguration); try (OzoneClient rpcClient = createOzoneClient(omServiceID, ozoneConfiguration)) { @@ -111,8 +121,8 @@ private void createKey(long counter) throws Exception { final String key = generateObjectName(counter); timer.time(() -> { - try (OutputStream stream = bucket.createKey(key, keySize, - ReplicationType.RATIS, factor, metadata)) { + try(OutputStream stream = bucket.createKey(key, keySize, + replicationConfig, metadata)) { contentGenerator.write(stream); stream.flush(); } From 58a9c54ba9be176af61cf233c93bc16a85b1d373 Mon Sep 17 00:00:00 2001 From: Kaijie Chen Date: Thu, 10 Feb 2022 16:46:41 +0800 Subject: [PATCH 2/9] ockg: enable bucket-level replication config --- .../ozone/freon/OzoneClientKeyGenerator.java | 23 ++++++++++--------- 1 file changed, 12 insertions(+), 11 deletions(-) diff --git a/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/freon/OzoneClientKeyGenerator.java b/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/freon/OzoneClientKeyGenerator.java index 1d42060ad896..51a317ee9c7f 100644 --- a/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/freon/OzoneClientKeyGenerator.java +++ b/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/freon/OzoneClientKeyGenerator.java @@ -21,6 +21,7 @@ import java.util.Map; import java.util.concurrent.Callable; +import org.apache.hadoop.fs.ozone.OzoneClientUtils; import org.apache.hadoop.hdds.cli.HddsVersionProvider; import org.apache.hadoop.hdds.client.ReplicationConfig; import org.apache.hadoop.hdds.client.ReplicationType; @@ -66,23 +67,22 @@ public class OzoneClientKeyGenerator extends BaseFreonGenerator defaultValue = "4096") private int bufferSize; - @Option( - names = "--om-service-id", + @Option(names = "--om-service-id", description = "OM Service ID" ) - private String omServiceID = null; + private String omServiceID; @Option(names = {"--replication"}, description = - "Replication configuration of the new key. (this is replication " - + "specific. for RATIS/STANDALONE you can use ONE or THREE, " + - "for EC you can use rs-3-2-1024k, rs-6-3-1024 or rs-10-4-1024k.)" - + " Default is specified in the cluster-wide config.") + "Replication configuration of the new key." + + "(ONE, THREE) for RATIS or STAND_ALONE, " + + "(rs-3-2-1024k, rs-6-3-1024k or rs-10-4-1024k) for EC." + ) private String replication; @Option(names = {"--type"}, - description = "Replication type of the new key. (use RATIS or " + - "STAND_ALONE or EC) Default is specified in the cluster-wide config.") + description = "Replication type of the new key. (RATIS, STAND_ALONE, EC)" + ) private ReplicationType replicationType; private Timer timer; @@ -101,8 +101,9 @@ public Void call() throws Exception { contentGenerator = new ContentGenerator(keySize, bufferSize); metadata = new HashMap<>(); - replicationConfig = ReplicationConfig.parse(replicationType, replication, - ozoneConfiguration); + replicationConfig = OzoneClientUtils + .validateAndGetClientReplicationConfig(replicationType, replication, + ozoneConfiguration); try (OzoneClient rpcClient = createOzoneClient(omServiceID, ozoneConfiguration)) { From a918ce3c77f97d787c6a96514d644561dc54da2b Mon Sep 17 00:00:00 2001 From: Kaijie Chen Date: Thu, 10 Feb 2022 17:05:23 +0800 Subject: [PATCH 3/9] fix style --- .../org/apache/hadoop/ozone/freon/OzoneClientKeyGenerator.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/freon/OzoneClientKeyGenerator.java b/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/freon/OzoneClientKeyGenerator.java index 51a317ee9c7f..12d446e07dde 100644 --- a/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/freon/OzoneClientKeyGenerator.java +++ b/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/freon/OzoneClientKeyGenerator.java @@ -122,7 +122,7 @@ private void createKey(long counter) throws Exception { final String key = generateObjectName(counter); timer.time(() -> { - try(OutputStream stream = bucket.createKey(key, keySize, + try (OutputStream stream = bucket.createKey(key, keySize, replicationConfig, metadata)) { contentGenerator.write(stream); stream.flush(); From 01b5181ed17041d75de86c8cccb079d45c3542e7 Mon Sep 17 00:00:00 2001 From: Kaijie Chen Date: Thu, 10 Feb 2022 17:32:56 +0800 Subject: [PATCH 4/9] ockg: restore --factor option for backward compatibility --- .../ozone/freon/OzoneClientKeyGenerator.java | 23 ++++++++++++++++--- 1 file changed, 20 insertions(+), 3 deletions(-) diff --git a/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/freon/OzoneClientKeyGenerator.java b/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/freon/OzoneClientKeyGenerator.java index 12d446e07dde..eb6e4a37a561 100644 --- a/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/freon/OzoneClientKeyGenerator.java +++ b/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/freon/OzoneClientKeyGenerator.java @@ -24,6 +24,7 @@ import org.apache.hadoop.fs.ozone.OzoneClientUtils; import org.apache.hadoop.hdds.cli.HddsVersionProvider; import org.apache.hadoop.hdds.client.ReplicationConfig; +import org.apache.hadoop.hdds.client.ReplicationFactor; import org.apache.hadoop.hdds.client.ReplicationType; import org.apache.hadoop.hdds.conf.OzoneConfiguration; import org.apache.hadoop.ozone.client.OzoneBucket; @@ -31,7 +32,9 @@ import com.codahale.metrics.Timer; import picocli.CommandLine.Command; +import picocli.CommandLine.Model.CommandSpec; import picocli.CommandLine.Option; +import picocli.CommandLine.Spec; /** * Data generator tool test om performance. @@ -45,6 +48,8 @@ public class OzoneClientKeyGenerator extends BaseFreonGenerator implements Callable { + @Spec CommandSpec spec; + @Option(names = {"-v", "--volume"}, description = "Name of the bucket which contains the test data. Will be" + " created if missing.", @@ -67,6 +72,12 @@ public class OzoneClientKeyGenerator extends BaseFreonGenerator defaultValue = "4096") private int bufferSize; + @Option(names = { "-F", "--factor" }, + description = "[Deprecated] Replication factor (ONE, THREE)", + defaultValue = "THREE" + ) + private ReplicationFactor factor = ReplicationFactor.THREE; + @Option(names = "--om-service-id", description = "OM Service ID" ) @@ -101,9 +112,15 @@ public Void call() throws Exception { contentGenerator = new ContentGenerator(keySize, bufferSize); metadata = new HashMap<>(); - replicationConfig = OzoneClientUtils - .validateAndGetClientReplicationConfig(replicationType, replication, - ozoneConfiguration); + + if (spec.commandLine().getParseResult().hasMatchedOption("--factor")) { + replicationConfig = ReplicationConfig + .fromTypeAndFactor(ReplicationType.RATIS, factor); + } else { + replicationConfig = OzoneClientUtils + .validateAndGetClientReplicationConfig(replicationType, replication, + ozoneConfiguration); + } try (OzoneClient rpcClient = createOzoneClient(omServiceID, ozoneConfiguration)) { From 8e9793da71aa49a237f8538a83389ac048f6e8e2 Mon Sep 17 00:00:00 2001 From: Kaijie Chen Date: Thu, 10 Feb 2022 17:38:14 +0800 Subject: [PATCH 5/9] fix checkstyle --- .../org/apache/hadoop/ozone/freon/OzoneClientKeyGenerator.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/freon/OzoneClientKeyGenerator.java b/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/freon/OzoneClientKeyGenerator.java index eb6e4a37a561..87af3cc92893 100644 --- a/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/freon/OzoneClientKeyGenerator.java +++ b/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/freon/OzoneClientKeyGenerator.java @@ -48,7 +48,7 @@ public class OzoneClientKeyGenerator extends BaseFreonGenerator implements Callable { - @Spec CommandSpec spec; + @Spec private CommandSpec spec; @Option(names = {"-v", "--volume"}, description = "Name of the bucket which contains the test data. Will be" From d9ed09a2bf752ba9b57d3601fc64b23d3a75a774 Mon Sep 17 00:00:00 2001 From: Kaijie Chen Date: Fri, 11 Feb 2022 14:57:24 +0800 Subject: [PATCH 6/9] ockg: catch NotImplementedException for ECKeyOutputStream#flush() --- .../org/apache/hadoop/ozone/freon/OzoneClientKeyGenerator.java | 3 +++ 1 file changed, 3 insertions(+) diff --git a/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/freon/OzoneClientKeyGenerator.java b/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/freon/OzoneClientKeyGenerator.java index 87af3cc92893..d870f02fa8b3 100644 --- a/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/freon/OzoneClientKeyGenerator.java +++ b/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/freon/OzoneClientKeyGenerator.java @@ -21,6 +21,7 @@ import java.util.Map; import java.util.concurrent.Callable; +import org.apache.commons.lang3.NotImplementedException; import org.apache.hadoop.fs.ozone.OzoneClientUtils; import org.apache.hadoop.hdds.cli.HddsVersionProvider; import org.apache.hadoop.hdds.client.ReplicationConfig; @@ -143,6 +144,8 @@ private void createKey(long counter) throws Exception { replicationConfig, metadata)) { contentGenerator.write(stream); stream.flush(); + } catch (NotImplementedException e) { + // flush() is not implemented in ECKeyOutputStream } return null; }); From 287a1776616a55089a2e7e7932ac84e02e8f8576 Mon Sep 17 00:00:00 2001 From: Kaijie Chen Date: Fri, 11 Feb 2022 15:09:25 +0800 Subject: [PATCH 7/9] Revert "ockg: catch NotImplementedException for ECKeyOutputStream#flush()" This reverts commit d9ed09a2bf752ba9b57d3601fc64b23d3a75a774. --- .../org/apache/hadoop/ozone/freon/OzoneClientKeyGenerator.java | 3 --- 1 file changed, 3 deletions(-) diff --git a/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/freon/OzoneClientKeyGenerator.java b/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/freon/OzoneClientKeyGenerator.java index d870f02fa8b3..87af3cc92893 100644 --- a/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/freon/OzoneClientKeyGenerator.java +++ b/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/freon/OzoneClientKeyGenerator.java @@ -21,7 +21,6 @@ import java.util.Map; import java.util.concurrent.Callable; -import org.apache.commons.lang3.NotImplementedException; import org.apache.hadoop.fs.ozone.OzoneClientUtils; import org.apache.hadoop.hdds.cli.HddsVersionProvider; import org.apache.hadoop.hdds.client.ReplicationConfig; @@ -144,8 +143,6 @@ private void createKey(long counter) throws Exception { replicationConfig, metadata)) { contentGenerator.write(stream); stream.flush(); - } catch (NotImplementedException e) { - // flush() is not implemented in ECKeyOutputStream } return null; }); From ac6aa139dd897e2a6dfd1fc8ce9a91d95b5c97b9 Mon Sep 17 00:00:00 2001 From: Kaijie Chen Date: Fri, 11 Feb 2022 15:35:49 +0800 Subject: [PATCH 8/9] ockg: do not call flush() for EC Keys --- .../hadoop/ozone/freon/OzoneClientKeyGenerator.java | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/freon/OzoneClientKeyGenerator.java b/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/freon/OzoneClientKeyGenerator.java index 87af3cc92893..0b7195590b89 100644 --- a/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/freon/OzoneClientKeyGenerator.java +++ b/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/freon/OzoneClientKeyGenerator.java @@ -16,7 +16,6 @@ */ package org.apache.hadoop.ozone.freon; -import java.io.OutputStream; import java.util.HashMap; import java.util.Map; import java.util.concurrent.Callable; @@ -29,6 +28,8 @@ import org.apache.hadoop.hdds.conf.OzoneConfiguration; import org.apache.hadoop.ozone.client.OzoneBucket; import org.apache.hadoop.ozone.client.OzoneClient; +import org.apache.hadoop.ozone.client.io.ECKeyOutputStream; +import org.apache.hadoop.ozone.client.io.OzoneOutputStream; import com.codahale.metrics.Timer; import picocli.CommandLine.Command; @@ -139,10 +140,13 @@ private void createKey(long counter) throws Exception { final String key = generateObjectName(counter); timer.time(() -> { - try (OutputStream stream = bucket.createKey(key, keySize, + try (OzoneOutputStream stream = bucket.createKey(key, keySize, replicationConfig, metadata)) { contentGenerator.write(stream); - stream.flush(); + if (!(stream.getOutputStream() instanceof ECKeyOutputStream)) { + // ECKeyOutputStream#flush() is not implemented yet. + stream.flush(); + } } return null; }); From abb01da56086152f5281512f731483d480c6866e Mon Sep 17 00:00:00 2001 From: Kaijie Chen Date: Fri, 11 Feb 2022 19:21:37 +0800 Subject: [PATCH 9/9] trigger CI