diff --git a/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/freon/ContentGenerator.java b/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/freon/ContentGenerator.java index 542634c4884b..724ff56430d4 100644 --- a/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/freon/ContentGenerator.java +++ b/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/freon/ContentGenerator.java @@ -22,6 +22,7 @@ import com.google.common.annotations.VisibleForTesting; import org.apache.commons.lang3.RandomStringUtils; +import org.apache.hadoop.fs.FSDataOutputStream; /** * Utility class to write random keys from a limited buffer. @@ -31,13 +32,13 @@ public class ContentGenerator { /** * Size of the destination object (key or file). */ - private long keySize; + private final long keySize; /** * Buffer for the pre-allocated content (will be reused if less than the * keySize). */ - private int bufferSize; + private final int bufferSize; /** * Number of bytes to write in one call. @@ -48,14 +49,37 @@ public class ContentGenerator { private final byte[] buffer; + /** + * Issue Hsync after every write ( Cannot be used with Hflush ). + */ + private final boolean hSync; + + /** + * Issue Hflush after every write ( Cannot be used with Hsync ). + */ + private final boolean hFlush; + ContentGenerator(long keySize, int bufferSize) { - this(keySize, bufferSize, bufferSize); + this(keySize, bufferSize, bufferSize, false, false); } ContentGenerator(long keySize, int bufferSize, int copyBufferSize) { + this(keySize, bufferSize, copyBufferSize, false, false); + } + + ContentGenerator(long keySize, int bufferSize, boolean hsync, + boolean hflush) { + this(keySize, bufferSize, bufferSize, hsync, hflush); + } + + ContentGenerator(long keySize, int bufferSize, int copyBufferSize, + boolean hSync, boolean hFlush) { this.keySize = keySize; this.bufferSize = bufferSize; this.copyBufferSize = copyBufferSize; + this.hSync = hSync; + this.hFlush = hFlush; + buffer = RandomStringUtils.randomAscii(bufferSize) .getBytes(StandardCharsets.UTF_8); } @@ -70,16 +94,28 @@ public void write(OutputStream outputStream) throws IOException { if (copyBufferSize == 1) { for (int i = 0; i < curSize; i++) { outputStream.write(buffer[i]); + flushOrSync(outputStream); } } else { for (int i = 0; i < curSize; i += copyBufferSize) { outputStream.write(buffer, i, Math.min(copyBufferSize, curSize - i)); + flushOrSync(outputStream); } } } } + private void flushOrSync(OutputStream outputStream) throws IOException { + if (outputStream instanceof FSDataOutputStream) { + if (hSync) { + ((FSDataOutputStream) outputStream).hsync(); + } else if (hFlush) { + ((FSDataOutputStream) outputStream).hflush(); + } + } + } + @VisibleForTesting byte[] getBuffer() { return buffer; diff --git a/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/freon/Freon.java b/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/freon/Freon.java index c5b9a3a882cd..31144021d3fe 100644 --- a/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/freon/Freon.java +++ b/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/freon/Freon.java @@ -63,7 +63,8 @@ GeneratorDatanode.class, ClosedContainerReplicator.class, StreamingGenerator.class, - SCMThroughputBenchmark.class}, + SCMThroughputBenchmark.class, + WriteFileThroughputBenchmark.class}, versionProvider = HddsVersionProvider.class, mixinStandardHelpOptions = true) public class Freon extends GenericCli { diff --git a/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/freon/WriteFileThroughputBenchmark.java b/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/freon/WriteFileThroughputBenchmark.java new file mode 100644 index 000000000000..ff712f2dcb0f --- /dev/null +++ b/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/freon/WriteFileThroughputBenchmark.java @@ -0,0 +1,245 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with this + * work for additional information regarding copyright ownership. The ASF + * licenses this file to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + *
+ * http://www.apache.org/licenses/LICENSE-2.0 + *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations under
+ * the License.
+ */
+package org.apache.hadoop.ozone.freon;
+
+import com.codahale.metrics.Timer;
+import org.apache.hadoop.fs.FSDataOutputStream;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.hdds.cli.HddsVersionProvider;
+import org.apache.hadoop.hdds.conf.OzoneConfiguration;
+import org.slf4j.LoggerFactory;
+import picocli.CommandLine;
+import picocli.CommandLine.Option;
+import org.slf4j.Logger;
+
+import java.io.IOException;
+import java.io.UncheckedIOException;
+import java.net.URI;
+import java.util.concurrent.Callable;
+
+/**
+ * File Generator for populating the cluster with random files.
+ */
+@CommandLine.Command(name = "wtb",
+ aliases = "write-throughput-benchmark",
+ description = "Benchmark for creating a file",
+ versionProvider = HddsVersionProvider.class,
+ mixinStandardHelpOptions = true,
+ showDefaultValues = true)
+public class WriteFileThroughputBenchmark extends BaseFreonGenerator
+ implements Callable