HDDS-7205. DiskBalancer CLI

hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/SCMClientProtocolServer.java

@@ -1160,14 +1160,14 @@ public List<DatanodeAdminError> startDiskBalancer(Optional<Double> threshold,
       Optional<Long> bandwidthInMB, Optional<Integer> parallelThread,
       Optional<List<String>> hosts) throws IOException {
     // TODO: Send message to datanodes
-    return null;
+    return new ArrayList<DatanodeAdminError>();
   }
 
   @Override
   public List<DatanodeAdminError> stopDiskBalancer(Optional<List<String>> hosts)
       throws IOException {
     // TODO: Send message to datanodes
-    return null;
+    return new ArrayList<DatanodeAdminError>();
   }
 
 
@@ -1177,7 +1177,7 @@ public List<DatanodeAdminError> updateDiskBalancerConfiguration(
       Optional<Integer> parallelThread, Optional<List<String>> hosts)
       throws IOException {
     // TODO: Send message to datanodes
-    return null;
+    return new ArrayList<DatanodeAdminError>();
   }
 
   /**

hadoop-hdds/tools/src/main/java/org/apache/hadoop/hdds/scm/cli/datanode/DatanodeCommands.java

@@ -41,7 +41,8 @@
         DecommissionSubCommand.class,
         MaintenanceSubCommand.class,
         RecommissionSubCommand.class,
-        UsageInfoSubcommand.class
+        UsageInfoSubcommand.class,
+        DiskBalancerCommands.class
     })
 @MetaInfServices(SubcommandWithParent.class)
 public class DatanodeCommands implements Callable<Void>, SubcommandWithParent {

hadoop-hdds/tools/src/main/java/org/apache/hadoop/hdds/scm/cli/datanode/DiskBalancerCommands.java

@@ -0,0 +1,99 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hdds.scm.cli.datanode;
+
+import org.apache.hadoop.hdds.cli.GenericCli;
+import org.apache.hadoop.hdds.cli.HddsVersionProvider;
+import picocli.CommandLine.Command;
+import picocli.CommandLine.Model.CommandSpec;
+import picocli.CommandLine.Spec;
+
+import java.util.concurrent.Callable;
+
+/**
+ * Subcommand to group disk balancer related operations.
+ *
+ * <p>The balancer is a tool that balances space usage on an Ozone datanode
+ * when some disks become full or when new empty disks were added to a datanode.
+ *
+ * <p>SYNOPSIS
+ * <pre>
+ * To start:
+ *      ozone admin datanode diskbalancer start
+ *      [ -t/--threshold {@literal <threshold>}]
+ *      [ -b/--bandwidthInMB {@literal <bandwidthInMB>}]
+ *      [ -p/--parallelThread {@literal <parallelThread>}]
+ *      [ -a/--alldatanodes {@literal <alldatanodes>}]
+ *      [ {@literal <hosts>}]
+ *      Examples:
+ *      ozone admin datanode diskbalancer start {@literal <hosts>}
+ *        start balancer with default values in the configuration on specified
+ *        datanodes
+ *      ozone admin datanode diskbalancer start -a
+ *        start balancer with default values in the configuration on all
+ *        datanodes in the cluster
+ *      ozone admin datanode diskbalancer start -t 5 {@literal <hosts>}
+ *        start balancer with a threshold of 5%
+ *      ozone admin datanode diskbalancer start -b 20 {@literal <hosts>}
+ *        start balancer with maximum 20MB/s diskbandwidth
+ *      ozone admin datanode diskbalancer start -p 5 {@literal <hosts>}
+ *        start balancer with 5 parallel thread on each datanode
+ * To stop:
+ *      ozone admin datanode diskbalancer stop -a
+ *        stop diskblancer on all datanodes
+ *      ozone admin datanode diskbalancer stop {@literal <hosts>};
+ *        stop diskblancer on all datanodes
+ * To update:
+ *      ozone admin datanode diskbalancer update -a
+ *        update diskblancer configuration on all datanodes
+ *      ozone admin datanode diskbalancer update {@literal <hosts>};
+ *        update diskblancer configuration on all datanodes
+ * To get report:
+ *      ozone admin datanode diskbalancer report -c 10
+ *        retrieve at most 10 datanodes that needs diskbalance most
+ * To get status:
+ *      ozone admin datanode diskbalancer status -s RUNNING {@literal <hosts>}
+ *        return the diskbalancer status on datanodes where diskbalancer are in
+ *        Running state
+ *
+ * </pre>
+ */
+
+@Command(
+    name = "diskbalancer",
+    description = "DiskBalancer specific operations",
+    mixinStandardHelpOptions = true,
+    versionProvider = HddsVersionProvider.class,
+    subcommands = {
+        DiskBalancerStartSubcommand.class,
+        DiskBalancerStopSubcommand.class,
+        DiskBalancerUpdateSubcommand.class,
+        DiskBalancerReportSubcommand.class,
+        DiskBalancerStatusSubcommand.class
+    })
+public class DiskBalancerCommands implements Callable<Void> {
+
+  @Spec
+  private CommandSpec spec;
+
+  @Override
+  public Void call() throws Exception {
+    GenericCli.missingSubcommand(spec);
+    return null;
+  }
+}

hadoop-hdds/tools/src/main/java/org/apache/hadoop/hdds/scm/cli/datanode/DiskBalancerReportSubcommand.java

@@ -0,0 +1,70 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hdds.scm.cli.datanode;
+
+import org.apache.hadoop.hdds.cli.HddsVersionProvider;
+import org.apache.hadoop.hdds.protocol.proto.HddsProtos;
+import org.apache.hadoop.hdds.scm.cli.ScmSubcommand;
+import org.apache.hadoop.hdds.scm.client.ScmClient;
+import picocli.CommandLine.Command;
+import picocli.CommandLine.Option;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.List;
+
+/**
+ * Handler to get Datanode Volume Density report.
+ */
+@Command(
+    name = "report",
+    description = "Get Datanode Volume Density Report",
+    mixinStandardHelpOptions = true,
+    versionProvider = HddsVersionProvider.class)
+public class DiskBalancerReportSubcommand extends ScmSubcommand {
+  @Option(names = {"-c", "--count"},
+      description = "Result count to return. Sort by Volume Density " +
+          "in descending order.")
+  private int count;
+
+  @Override
+  public void execute(ScmClient scmClient) throws IOException {
+    List<HddsProtos.DatanodeDiskBalancerInfoProto> resultProto =
+        scmClient.getDiskBalancerReport(count);
+    System.out.println(generateReport(resultProto));
+  }
+
+  private String generateReport(
+      List<HddsProtos.DatanodeDiskBalancerInfoProto> protos) {
+    StringBuilder formatBuilder = new StringBuilder("Report result:%n" +
+        "%-50s %s%n");
+
+    List<String> contentList = new ArrayList<>();
+    contentList.add("Datanode");
+    contentList.add("VolumeDensity");
+
+    for (HddsProtos.DatanodeDiskBalancerInfoProto proto: protos) {
+      formatBuilder.append("%-50s %s%n");
+      contentList.add(proto.getNode().getHostName());
+      contentList.add(String.valueOf(proto.getCurrentVolumeDensitySum()));
+    }
+
+    return String.format(formatBuilder.toString(),
+        contentList.toArray(new String[0]));
+  }
+}

hadoop-hdds/tools/src/main/java/org/apache/hadoop/hdds/scm/cli/datanode/DiskBalancerStartSubcommand.java

@@ -0,0 +1,100 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hdds.scm.cli.datanode;
+
+import com.google.common.annotations.VisibleForTesting;
+import org.apache.hadoop.hdds.cli.HddsVersionProvider;
+import org.apache.hadoop.hdds.scm.DatanodeAdminError;
+import org.apache.hadoop.hdds.scm.cli.ScmSubcommand;
+import org.apache.hadoop.hdds.scm.client.ScmClient;
+import picocli.CommandLine.Command;
+import picocli.CommandLine.Option;
+import picocli.CommandLine.Parameters;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Optional;
+
+/**
+ * Handler to start disk balancer.
+ */
+@Command(
+    name = "start",
+    description = "Start DiskBalancer",
+    mixinStandardHelpOptions = true,
+    versionProvider = HddsVersionProvider.class)
+public class DiskBalancerStartSubcommand extends ScmSubcommand {
+
+  @Option(names = {"-t", "--threshold"},
+      description = "Percentage deviation from average utilization of " +
+          "the disks after which a datanode will be rebalanced (for " +
+          "example, '10' for 10%%).")
+  private Optional<Double> threshold;
+
+  @Option(names = {"-b", "--bandwidthInMB"},
+      description = "Maximum bandwidth for DiskBalancer per second.")
+  private Optional<Long> bandwidthInMB;
+
+  @Option(names = {"-p", "--parallelThread"},
+      description = "Max parallelThread for DiskBalancer.")
+  private Optional<Integer> parallelThread;
+
+  @Option(names = {"-a", "--allDatanodes"},
+      description = "Start diskBalancer on all datanodes.")
+  private boolean allHosts;
+
+  @Parameters(description = "List of fully qualified host names")
+  private List<String> hosts = new ArrayList<>();
+
+  @Override
+  public void execute(ScmClient scmClient) throws IOException {
+    if (hosts.size() == 0 && !allHosts) {
+      System.out.println("Datanode not specified. Please specify " +
+          "\"--allDatanodes\" to start diskBalancer on all datanodes");
+      return;
+    }
+    if (hosts.size() != 0 && allHosts) {
+      System.out.println("Confused options. Omit \"--allDatanodes\" or " +
+          "Datanodes.");
+      return;
+    }
+    List<DatanodeAdminError> errors =
+        scmClient.startDiskBalancer(threshold, bandwidthInMB, parallelThread,
+            hosts.size() == 0 ? Optional.empty() : Optional.of(hosts));
+
+    System.out.println("Start DiskBalancer on datanode(s):\n" +
+        (allHosts ? "All datanodes" : String.join("\n", hosts)));
+
+    if (errors.size() > 0) {
+      for (DatanodeAdminError error : errors) {
+        System.err.println("Error: " + error.getHostname() + ": "
+            + error.getError());
+      }
+      // Throwing the exception will cause a non-zero exit status for the
+      // command.
+      throw new IOException(
+          "Some nodes could not start DiskBalancer.");
+    }
+  }
+
+  @VisibleForTesting
+  public void setAllHosts(boolean allHosts) {
+    this.allHosts = allHosts;
+  }
+}