From cce60fb4d69424c90551956e14f584e00bbc5839 Mon Sep 17 00:00:00 2001 From: Aryan Gupta Date: Tue, 28 Jul 2020 01:15:56 +0530 Subject: [PATCH 1/7] HDDS-4034. Add Unit Test for HadoopNestedDirGenerator. --- .../freon/TestHadoopNestedDirGenerator.java | 201 ++++++++++++++++++ 1 file changed, 201 insertions(+) create mode 100644 hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/freon/TestHadoopNestedDirGenerator.java diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/freon/TestHadoopNestedDirGenerator.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/freon/TestHadoopNestedDirGenerator.java new file mode 100644 index 000000000000..52ec4375c6d5 --- /dev/null +++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/freon/TestHadoopNestedDirGenerator.java @@ -0,0 +1,201 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with this + * work for additional information regarding copyright ownership. The ASF + * licenses this file to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

+ * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ +package org.apache.hadoop.ozone.freon; + +import org.apache.commons.io.FileUtils; +import org.apache.hadoop.fs.FileStatus; +import org.apache.hadoop.fs.FileSystem; +import org.apache.hadoop.fs.Path; +import org.apache.hadoop.hdds.conf.OzoneConfiguration; +import org.apache.hadoop.ozone.MiniOzoneCluster; +import org.apache.hadoop.ozone.client.ObjectStore; +import org.apache.hadoop.ozone.client.OzoneClientFactory; +import org.apache.hadoop.ozone.client.OzoneVolume; +import org.apache.hadoop.test.GenericTestUtils; +import org.apache.ratis.server.impl.RaftServerImpl; +import org.apache.ratis.server.raftlog.RaftLog; +import org.jooq.meta.derby.sys.Sys; +import java.util.LinkedList; +import org.junit.Assert; +import org.junit.Before; +import org.junit.Test; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.slf4j.event.Level; + +import java.io.File; +import java.io.FileOutputStream; +import java.io.IOException; +import java.net.URI; + +/** + * Test for HadoopNestedDirGenerator. + */ +public class TestHadoopNestedDirGenerator { + + private String path; + private OzoneConfiguration conf = null; + private MiniOzoneCluster cluster = null; + private ObjectStore store = null; + private static final Logger LOG = + LoggerFactory.getLogger(TestHadoopNestedDirGenerator.class); + @Before + public void setup() { + path = GenericTestUtils + .getTempPath(TestOzoneClientKeyGenerator.class.getSimpleName()); + GenericTestUtils.setLogLevel(RaftLog.LOG, Level.DEBUG); + GenericTestUtils.setLogLevel(RaftServerImpl.LOG, Level.DEBUG); + File baseDir = new File(path); + baseDir.mkdirs(); + } + + /** + * Shutdown MiniDFSCluster. + */ + private void shutdown() throws IOException { + if (cluster != null) { + cluster.shutdown(); + FileUtils.deleteDirectory(new File(path)); + } + } + + /** + * Create a MiniDFSCluster for testing. + * + * @throws IOException + */ + /** + * Below we have 5 datanodes because only 5 tests are activated + * As you need more tests increase the datanodes accordingly. + */ + private void startCluster() throws Exception { + conf = new OzoneConfiguration(); + + cluster = MiniOzoneCluster.newBuilder(conf).setNumDatanodes(5).build(); + cluster.waitForClusterToBeReady(); + cluster.waitTobeOutOfSafeMode(); + + store = OzoneClientFactory.getRpcClient(conf).getObjectStore(); + } + + @Test + public void testNestedDirTreeGeneration() throws Exception { + try { + System.out.println("cloudera\n"); + startCluster(); + FileOutputStream out = FileUtils.openOutputStream(new File(path, + "conf")); + cluster.getConf().writeXml(out); + out.getFD().sync(); + out.close(); + verifyDirTree("vol1", + "bucket1", 1, 1); + verifyDirTree("vol2", + "bucket1", 1, 5); + verifyDirTree("vol3", + "bucket1", 2, 0); + verifyDirTree("vol4", + "bucket1", 3, 2); + verifyDirTree("vol5", + "bucket1", 5, 4); + } finally { + shutdown(); + } + } + + private void verifyDirTree(String volumeName, String bucketName, + int actualDepth, int span) + throws IOException { + + store.createVolume(volumeName); + OzoneVolume volume = store.getVolume(volumeName); + volume.createBucket(bucketName); + String rootPath = "o3fs://" + bucketName + "." + volumeName; + String confPath = new File(path, "conf").getAbsolutePath(); + new Freon().execute( + new String[]{"-conf", confPath, "ddsg", "-d", actualDepth + "" , + "-s", span + "", "-n", "1", "-r", rootPath }); + // verify the directory structure + FileSystem fileSystem = FileSystem.get(URI.create(rootPath), + conf); + Path rootDir = new Path(rootPath.concat("/")); + // verify root path details + FileStatus[] fileStatuses = fileSystem.listStatus(rootDir); + Path p = null; + for (FileStatus fileStatus : fileStatuses) { + // verify the num of peer directories and span directories + p = depthBFS(fileSystem, fileStatuses, span, actualDepth); + int actualSpan = spanCheck(fileSystem, span, p); + Assert.assertEquals("Mismatch span in a path", + span, actualSpan); + } + } + + /** + * Using BFS(Breadth First Search) to find the depth of nested + * directories. First we push the directory at level 1 to + * queue and follow BFS, as we encounter the child directories + * we put them in an array and increment the depth variable by 1. + */ + + private Path depthBFS(FileSystem fs, FileStatus[] fileStatuses, + int span, int actualDepth) throws IOException { + int depth = 0; + Path p = null; + if(span > 0) depth = 0; + else if(span == 0) depth = 1; + else LOG.info("Span value can never be negative"); + LinkedList queue = new LinkedList(); + FileStatus f1 = fileStatuses[0]; + queue.add(f1); + while(queue.size() != 0){ + FileStatus f = queue.poll(); + FileStatus[] temp = fs.listStatus(f.getPath()); + if(temp.length > 0){ + ++depth; + for(int i=0;i= 0) depth = 0; + else LOG.info("Span value can never be negative"); + FileStatus[] fileStatuses = fs.listStatus(p); + for (FileStatus fileStatus : fileStatuses){ + if(fileStatus.isDirectory()){ + ++sp; + } + } + return sp; + } +} From 4728c4a5dbaa0814e61702a1f46926741d71f218 Mon Sep 17 00:00:00 2001 From: Aryan Gupta Date: Tue, 28 Jul 2020 01:21:52 +0530 Subject: [PATCH 2/7] HDDS-4034. Add Unit Test for HadoopNestedDirGenerator. --- .../hadoop/ozone/freon/TestHadoopNestedDirGenerator.java | 4 ---- 1 file changed, 4 deletions(-) diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/freon/TestHadoopNestedDirGenerator.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/freon/TestHadoopNestedDirGenerator.java index 52ec4375c6d5..a8cdd7ca0ce7 100644 --- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/freon/TestHadoopNestedDirGenerator.java +++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/freon/TestHadoopNestedDirGenerator.java @@ -36,7 +36,6 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.slf4j.event.Level; - import java.io.File; import java.io.FileOutputStream; import java.io.IOException; @@ -84,11 +83,9 @@ private void shutdown() throws IOException { */ private void startCluster() throws Exception { conf = new OzoneConfiguration(); - cluster = MiniOzoneCluster.newBuilder(conf).setNumDatanodes(5).build(); cluster.waitForClusterToBeReady(); cluster.waitTobeOutOfSafeMode(); - store = OzoneClientFactory.getRpcClient(conf).getObjectStore(); } @@ -120,7 +117,6 @@ public void testNestedDirTreeGeneration() throws Exception { private void verifyDirTree(String volumeName, String bucketName, int actualDepth, int span) throws IOException { - store.createVolume(volumeName); OzoneVolume volume = store.getVolume(volumeName); volume.createBucket(bucketName); From 868344cb0277242eead71fd9b9af478bdb0c9136 Mon Sep 17 00:00:00 2001 From: Aryan Gupta Date: Tue, 28 Jul 2020 01:26:54 +0530 Subject: [PATCH 3/7] HDDS-4034. Add Unit Test for HadoopNestedDirGenerator. --- .../apache/hadoop/ozone/freon/TestHadoopNestedDirGenerator.java | 1 - 1 file changed, 1 deletion(-) diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/freon/TestHadoopNestedDirGenerator.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/freon/TestHadoopNestedDirGenerator.java index a8cdd7ca0ce7..cefb484ed62f 100644 --- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/freon/TestHadoopNestedDirGenerator.java +++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/freon/TestHadoopNestedDirGenerator.java @@ -92,7 +92,6 @@ private void startCluster() throws Exception { @Test public void testNestedDirTreeGeneration() throws Exception { try { - System.out.println("cloudera\n"); startCluster(); FileOutputStream out = FileUtils.openOutputStream(new File(path, "conf")); From efe4373d36c355e2e0cbcb9def3dab11da248de7 Mon Sep 17 00:00:00 2001 From: Aryan Gupta Date: Tue, 28 Jul 2020 01:31:32 +0530 Subject: [PATCH 4/7] HDDS-4034. Add Unit Test for HadoopNestedDirGenerator. --- .../apache/hadoop/ozone/freon/TestHadoopNestedDirGenerator.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/freon/TestHadoopNestedDirGenerator.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/freon/TestHadoopNestedDirGenerator.java index cefb484ed62f..550be4f37ee0 100644 --- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/freon/TestHadoopNestedDirGenerator.java +++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/freon/TestHadoopNestedDirGenerator.java @@ -162,7 +162,7 @@ private Path depthBFS(FileSystem fs, FileStatus[] fileStatuses, FileStatus[] temp = fs.listStatus(f.getPath()); if(temp.length > 0){ ++depth; - for(int i=0;i Date: Tue, 28 Jul 2020 15:55:01 +0530 Subject: [PATCH 5/7] HDDS-4034. CheckStyle Fixed. --- .../freon/TestHadoopNestedDirGenerator.java | 226 +++++++++--------- 1 file changed, 119 insertions(+), 107 deletions(-) diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/freon/TestHadoopNestedDirGenerator.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/freon/TestHadoopNestedDirGenerator.java index 550be4f37ee0..1d02ca8666c3 100644 --- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/freon/TestHadoopNestedDirGenerator.java +++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/freon/TestHadoopNestedDirGenerator.java @@ -28,7 +28,6 @@ import org.apache.hadoop.test.GenericTestUtils; import org.apache.ratis.server.impl.RaftServerImpl; import org.apache.ratis.server.raftlog.RaftLog; -import org.jooq.meta.derby.sys.Sys; import java.util.LinkedList; import org.junit.Assert; import org.junit.Before; @@ -44,101 +43,104 @@ /** * Test for HadoopNestedDirGenerator. */ + public class TestHadoopNestedDirGenerator { - private String path; - private OzoneConfiguration conf = null; - private MiniOzoneCluster cluster = null; - private ObjectStore store = null; - private static final Logger LOG = - LoggerFactory.getLogger(TestHadoopNestedDirGenerator.class); - @Before + private String path; + private OzoneConfiguration conf = null; + private MiniOzoneCluster cluster = null; + private ObjectStore store = null; + private static final Logger LOG = + LoggerFactory.getLogger(TestHadoopNestedDirGenerator.class); + @Before public void setup() { - path = GenericTestUtils + path = GenericTestUtils .getTempPath(TestOzoneClientKeyGenerator.class.getSimpleName()); - GenericTestUtils.setLogLevel(RaftLog.LOG, Level.DEBUG); - GenericTestUtils.setLogLevel(RaftServerImpl.LOG, Level.DEBUG); - File baseDir = new File(path); - baseDir.mkdirs(); - } + GenericTestUtils.setLogLevel(RaftLog.LOG, Level.DEBUG); + GenericTestUtils.setLogLevel(RaftServerImpl.LOG, Level.DEBUG); + File baseDir = new File(path); + baseDir.mkdirs(); + } /** * Shutdown MiniDFSCluster. */ - private void shutdown() throws IOException { - if (cluster != null) { - cluster.shutdown(); - FileUtils.deleteDirectory(new File(path)); - } + + private void shutdown() throws IOException { + if (cluster != null) { + cluster.shutdown(); + FileUtils.deleteDirectory(new File(path)); } + } /** * Create a MiniDFSCluster for testing. * * @throws IOException */ + /** * Below we have 5 datanodes because only 5 tests are activated * As you need more tests increase the datanodes accordingly. */ - private void startCluster() throws Exception { - conf = new OzoneConfiguration(); - cluster = MiniOzoneCluster.newBuilder(conf).setNumDatanodes(5).build(); - cluster.waitForClusterToBeReady(); - cluster.waitTobeOutOfSafeMode(); - store = OzoneClientFactory.getRpcClient(conf).getObjectStore(); - } - @Test + private void startCluster() throws Exception { + conf = new OzoneConfiguration(); + cluster = MiniOzoneCluster.newBuilder(conf).setNumDatanodes(5).build(); + cluster.waitForClusterToBeReady(); + cluster.waitTobeOutOfSafeMode(); + store = OzoneClientFactory.getRpcClient(conf).getObjectStore(); + } + + @Test public void testNestedDirTreeGeneration() throws Exception { - try { - startCluster(); - FileOutputStream out = FileUtils.openOutputStream(new File(path, - "conf")); - cluster.getConf().writeXml(out); - out.getFD().sync(); - out.close(); - verifyDirTree("vol1", - "bucket1", 1, 1); - verifyDirTree("vol2", - "bucket1", 1, 5); - verifyDirTree("vol3", - "bucket1", 2, 0); - verifyDirTree("vol4", - "bucket1", 3, 2); - verifyDirTree("vol5", - "bucket1", 5, 4); - } finally { - shutdown(); - } + try { + startCluster(); + FileOutputStream out = FileUtils.openOutputStream(new File(path, + "conf")); + cluster.getConf().writeXml(out); + out.getFD().sync(); + out.close(); + verifyDirTree("vol1", + "bucket1", 1, 1); + verifyDirTree("vol2", + "bucket1", 1, 5); + verifyDirTree("vol3", + "bucket1", 2, 0); + verifyDirTree("vol4", + "bucket1", 3, 2); + verifyDirTree("vol5", + "bucket1", 5, 4); + } finally { + shutdown(); } + } - private void verifyDirTree(String volumeName, String bucketName, - int actualDepth, int span) + private void verifyDirTree(String volumeName, String bucketName, + int actualDepth, int span) throws IOException { - store.createVolume(volumeName); - OzoneVolume volume = store.getVolume(volumeName); - volume.createBucket(bucketName); - String rootPath = "o3fs://" + bucketName + "." + volumeName; - String confPath = new File(path, "conf").getAbsolutePath(); - new Freon().execute( - new String[]{"-conf", confPath, "ddsg", "-d", actualDepth + "" , - "-s", span + "", "-n", "1", "-r", rootPath }); - // verify the directory structure - FileSystem fileSystem = FileSystem.get(URI.create(rootPath), - conf); - Path rootDir = new Path(rootPath.concat("/")); - // verify root path details - FileStatus[] fileStatuses = fileSystem.listStatus(rootDir); - Path p = null; - for (FileStatus fileStatus : fileStatuses) { - // verify the num of peer directories and span directories - p = depthBFS(fileSystem, fileStatuses, span, actualDepth); - int actualSpan = spanCheck(fileSystem, span, p); - Assert.assertEquals("Mismatch span in a path", - span, actualSpan); - } + store.createVolume(volumeName); + OzoneVolume volume = store.getVolume(volumeName); + volume.createBucket(bucketName); + String rootPath = "o3fs://" + bucketName + "." + volumeName; + String confPath = new File(path, "conf").getAbsolutePath(); + new Freon().execute(new String[]{"-conf", confPath, "ddsg", "-d", + actualDepth + "", "-s", span + "", "-n", "1", "-r", rootPath}); + // verify the directory structure + FileSystem fileSystem = FileSystem.get(URI.create(rootPath), + conf); + Path rootDir = new Path(rootPath.concat("/")); + // verify root path details + FileStatus[] fileStatuses = fileSystem.listStatus(rootDir); + Path p = null; + for (FileStatus fileStatus : fileStatuses) { + // verify the num of peer directories and span directories + p = depthBFS(fileSystem, fileStatuses, span, actualDepth); + int actualSpan = spanCheck(fileSystem, span, p); + Assert.assertEquals("Mismatch span in a path", + span, actualSpan); } + } /** * Using BFS(Breadth First Search) to find the depth of nested @@ -147,32 +149,39 @@ private void verifyDirTree(String volumeName, String bucketName, * we put them in an array and increment the depth variable by 1. */ - private Path depthBFS(FileSystem fs, FileStatus[] fileStatuses, - int span, int actualDepth) throws IOException { - int depth = 0; - Path p = null; - if(span > 0) depth = 0; - else if(span == 0) depth = 1; - else LOG.info("Span value can never be negative"); - LinkedList queue = new LinkedList(); - FileStatus f1 = fileStatuses[0]; - queue.add(f1); - while(queue.size() != 0){ - FileStatus f = queue.poll(); - FileStatus[] temp = fs.listStatus(f.getPath()); - if(temp.length > 0){ - ++depth; - for(int i = 0; i < temp.length; i++){ - queue.add(temp[i]); - } - } - if(span == 0) p = f.getPath(); - else p = f.getPath().getParent(); + private Path depthBFS(FileSystem fs, FileStatus[] fileStatuses, + int span, int actualDepth) throws IOException { + int depth = 0; + Path p = null; + if(span > 0){ + depth = 0; + } else if(span == 0){ + depth = 1; + } else{ + LOG.info("Span value can never be negative"); + } + LinkedList queue = new LinkedList(); + FileStatus f1 = fileStatuses[0]; + queue.add(f1); + while(queue.size() != 0){ + FileStatus f = queue.poll(); + FileStatus[] temp = fs.listStatus(f.getPath()); + if(temp.length > 0){ + ++depth; + for(int i = 0; i < temp.length; i++){ + queue.add(temp[i]); } - Assert.assertEquals("Mismatch depth in a path", - depth, actualDepth); - return p; + } + if(span == 0){ + p = f.getPath(); + } else{ + p = f.getPath().getParent(); + } } + Assert.assertEquals("Mismatch depth in a path", + depth, actualDepth); + return p; + } /** * We get the path of last parent directory or leaf parent directory @@ -180,17 +189,20 @@ private Path depthBFS(FileSystem fs, FileStatus[] fileStatuses, * and count the span directories. */ - private int spanCheck(FileSystem fs, int span, Path p) throws IOException{ - int sp = 0; - int depth = 0; - if(span >= 0) depth = 0; - else LOG.info("Span value can never be negative"); - FileStatus[] fileStatuses = fs.listStatus(p); - for (FileStatus fileStatus : fileStatuses){ - if(fileStatus.isDirectory()){ - ++sp; - } - } - return sp; + private int spanCheck(FileSystem fs, int span, Path p) throws IOException{ + int sp = 0; + int depth = 0; + if(span >= 0){ + depth = 0; + } else{ + LOG.info("Span value can never be negative"); + } + FileStatus[] fileStatuses = fs.listStatus(p); + for (FileStatus fileStatus : fileStatuses){ + if(fileStatus.isDirectory()){ + ++sp; + } } + return sp; + } } From a0430d2fae32f55631f1ad391d0b0660fdbe8fca Mon Sep 17 00:00:00 2001 From: Aryan Gupta Date: Tue, 28 Jul 2020 19:19:30 +0530 Subject: [PATCH 6/7] HDDS-4034. Trigger CI. From 5e72af64421c184d50c778949d26e2f66aa0b396 Mon Sep 17 00:00:00 2001 From: Aryan Gupta Date: Thu, 30 Jul 2020 16:02:40 +0530 Subject: [PATCH 7/7] HDDS-4034. Add Unit Test for HadoopNestedDirGenerator - Fixed Javadoc. --- .../hadoop/ozone/freon/TestHadoopNestedDirGenerator.java | 5 ----- 1 file changed, 5 deletions(-) diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/freon/TestHadoopNestedDirGenerator.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/freon/TestHadoopNestedDirGenerator.java index 1d02ca8666c3..69731ea666b9 100644 --- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/freon/TestHadoopNestedDirGenerator.java +++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/freon/TestHadoopNestedDirGenerator.java @@ -79,11 +79,6 @@ private void shutdown() throws IOException { * @throws IOException */ - /** - * Below we have 5 datanodes because only 5 tests are activated - * As you need more tests increase the datanodes accordingly. - */ - private void startCluster() throws Exception { conf = new OzoneConfiguration(); cluster = MiniOzoneCluster.newBuilder(conf).setNumDatanodes(5).build();