From b9e0ccb6ce58dbc869c1237734b89de412e25f39 Mon Sep 17 00:00:00 2001 From: "Doroszlai, Attila" Date: Fri, 31 Jul 2020 17:02:30 +0200 Subject: [PATCH 1/9] HDDS-4056. Refactor WithScmClient to mixin --- .../scm/cli/ReplicationManagerCommands.java | 11 +-- .../ReplicationManagerStartSubcommand.java | 19 ++--- .../ReplicationManagerStatusSubcommand.java | 30 +++----- .../cli/ReplicationManagerStopSubcommand.java | 23 +++--- .../hdds/scm/cli/SafeModeCheckSubcommand.java | 40 +++++------ .../hadoop/hdds/scm/cli/SafeModeCommands.java | 16 +---- .../hdds/scm/cli/SafeModeExitSubcommand.java | 22 ++---- .../hdds/scm/cli/SafeModeWaitSubcommand.java | 13 ++-- .../apache/hadoop/hdds/scm/cli/ScmOption.java | 72 +++++++++++++++++++ .../WithScmClient.java => ScmSubcommand.java} | 24 +++++-- .../hdds/scm/cli/TopologySubcommand.java | 27 ++++--- .../scm/cli/container/CloseSubcommand.java | 20 ++---- .../scm/cli/container/ContainerCommands.java | 10 +-- .../scm/cli/container/CreateSubcommand.java | 26 +++---- .../scm/cli/container/DeleteSubcommand.java | 20 ++---- .../scm/cli/container/InfoSubcommand.java | 40 +++++------ .../scm/cli/container/ListSubcommand.java | 32 ++++----- .../scm/cli/datanode/DatanodeCommands.java | 10 +-- .../scm/cli/datanode/ListInfoSubcommand.java | 48 ++++++------- .../pipeline/ActivatePipelineSubcommand.java | 19 ++--- .../cli/pipeline/ClosePipelineSubcommand.java | 19 ++--- .../pipeline/CreatePipelineSubcommand.java | 38 +++++----- .../DeactivatePipelineSubcommand.java | 19 ++--- .../cli/pipeline/ListPipelinesSubcommand.java | 40 +++++------ .../scm/cli/pipeline/PipelineCommands.java | 11 +-- .../ozone/shell/TestOzoneDatanodeShell.java | 2 +- .../apache/hadoop/ozone/admin/OzoneAdmin.java | 40 +---------- ...stGenerateOzoneRequiredConfigurations.java | 5 +- pom.xml | 2 +- 29 files changed, 294 insertions(+), 404 deletions(-) create mode 100644 hadoop-hdds/tools/src/main/java/org/apache/hadoop/hdds/scm/cli/ScmOption.java rename hadoop-hdds/tools/src/main/java/org/apache/hadoop/hdds/scm/cli/{container/WithScmClient.java => ScmSubcommand.java} (60%) diff --git a/hadoop-hdds/tools/src/main/java/org/apache/hadoop/hdds/scm/cli/ReplicationManagerCommands.java b/hadoop-hdds/tools/src/main/java/org/apache/hadoop/hdds/scm/cli/ReplicationManagerCommands.java index fcb9ad6b2f6a..ca16a32f3cd3 100644 --- a/hadoop-hdds/tools/src/main/java/org/apache/hadoop/hdds/scm/cli/ReplicationManagerCommands.java +++ b/hadoop-hdds/tools/src/main/java/org/apache/hadoop/hdds/scm/cli/ReplicationManagerCommands.java @@ -1,4 +1,4 @@ -/** +/* * Licensed to the Apache Software Foundation (ASF) under one * or more contributor license agreements. See the NOTICE file * distributed with this work for additional information @@ -21,11 +21,9 @@ import org.apache.hadoop.hdds.cli.GenericCli; import org.apache.hadoop.hdds.cli.HddsVersionProvider; -import org.apache.hadoop.hdds.scm.cli.container.WithScmClient; import picocli.CommandLine.Command; import picocli.CommandLine.Model.CommandSpec; -import picocli.CommandLine.ParentCommand; import picocli.CommandLine.Spec; /** @@ -46,13 +44,6 @@ public class ReplicationManagerCommands implements Callable { @Spec private CommandSpec spec; - @ParentCommand - private WithScmClient parent; - - public WithScmClient getParent() { - return parent; - } - @Override public Void call() throws Exception { GenericCli.missingSubcommand(spec); diff --git a/hadoop-hdds/tools/src/main/java/org/apache/hadoop/hdds/scm/cli/ReplicationManagerStartSubcommand.java b/hadoop-hdds/tools/src/main/java/org/apache/hadoop/hdds/scm/cli/ReplicationManagerStartSubcommand.java index 1adec6b0c4b4..3b55ce5f2eeb 100644 --- a/hadoop-hdds/tools/src/main/java/org/apache/hadoop/hdds/scm/cli/ReplicationManagerStartSubcommand.java +++ b/hadoop-hdds/tools/src/main/java/org/apache/hadoop/hdds/scm/cli/ReplicationManagerStartSubcommand.java @@ -1,4 +1,4 @@ -/** +/* * Licensed to the Apache Software Foundation (ASF) under one * or more contributor license agreements. See the NOTICE file * distributed with this work for additional information @@ -22,9 +22,8 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; import picocli.CommandLine.Command; -import picocli.CommandLine.ParentCommand; -import java.util.concurrent.Callable; +import java.io.IOException; /** * This is the handler that process safe mode check command. @@ -34,20 +33,14 @@ description = "Start ReplicationManager", mixinStandardHelpOptions = true, versionProvider = HddsVersionProvider.class) -public class ReplicationManagerStartSubcommand implements Callable { +public class ReplicationManagerStartSubcommand extends ScmSubcommand { private static final Logger LOG = LoggerFactory.getLogger(ReplicationManagerStartSubcommand.class); - @ParentCommand - private ReplicationManagerCommands parent; - @Override - public Void call() throws Exception { - try (ScmClient scmClient = parent.getParent().createScmClient()) { - scmClient.startReplicationManager(); - LOG.info("Starting ReplicationManager..."); - return null; - } + public void execute(ScmClient scmClient) throws IOException { + scmClient.startReplicationManager(); + LOG.info("Starting ReplicationManager..."); } } diff --git a/hadoop-hdds/tools/src/main/java/org/apache/hadoop/hdds/scm/cli/ReplicationManagerStatusSubcommand.java b/hadoop-hdds/tools/src/main/java/org/apache/hadoop/hdds/scm/cli/ReplicationManagerStatusSubcommand.java index 2ebf28c80741..c735894b0686 100644 --- a/hadoop-hdds/tools/src/main/java/org/apache/hadoop/hdds/scm/cli/ReplicationManagerStatusSubcommand.java +++ b/hadoop-hdds/tools/src/main/java/org/apache/hadoop/hdds/scm/cli/ReplicationManagerStatusSubcommand.java @@ -1,4 +1,4 @@ -/** +/* * Licensed to the Apache Software Foundation (ASF) under one * or more contributor license agreements. See the NOTICE file * distributed with this work for additional information @@ -22,9 +22,8 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; import picocli.CommandLine.Command; -import picocli.CommandLine.ParentCommand; -import java.util.concurrent.Callable; +import java.io.IOException; /** * This is the handler that process safe mode check command. @@ -34,27 +33,20 @@ description = "Check if ReplicationManager is running or not", mixinStandardHelpOptions = true, versionProvider = HddsVersionProvider.class) -public class ReplicationManagerStatusSubcommand implements Callable { +public class ReplicationManagerStatusSubcommand extends ScmSubcommand { private static final Logger LOG = LoggerFactory.getLogger(ReplicationManagerStatusSubcommand.class); - @ParentCommand - private ReplicationManagerCommands parent; - @Override - public Void call() throws Exception { - try (ScmClient scmClient = parent.getParent().createScmClient()) { - - boolean execReturn = scmClient.getReplicationManagerStatus(); - - // Output data list - if(execReturn){ - LOG.info("ReplicationManager is Running."); - } else { - LOG.info("ReplicationManager is Not Running."); - } - return null; + public void execute(ScmClient scmClient) throws IOException { + boolean execReturn = scmClient.getReplicationManagerStatus(); + + // Output data list + if(execReturn){ + LOG.info("ReplicationManager is Running."); + } else { + LOG.info("ReplicationManager is Not Running."); } } } diff --git a/hadoop-hdds/tools/src/main/java/org/apache/hadoop/hdds/scm/cli/ReplicationManagerStopSubcommand.java b/hadoop-hdds/tools/src/main/java/org/apache/hadoop/hdds/scm/cli/ReplicationManagerStopSubcommand.java index 7cafd01b12d6..17c9034d2212 100644 --- a/hadoop-hdds/tools/src/main/java/org/apache/hadoop/hdds/scm/cli/ReplicationManagerStopSubcommand.java +++ b/hadoop-hdds/tools/src/main/java/org/apache/hadoop/hdds/scm/cli/ReplicationManagerStopSubcommand.java @@ -1,4 +1,4 @@ -/** +/* * Licensed to the Apache Software Foundation (ASF) under one * or more contributor license agreements. See the NOTICE file * distributed with this work for additional information @@ -22,9 +22,8 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; import picocli.CommandLine.Command; -import picocli.CommandLine.ParentCommand; -import java.util.concurrent.Callable; +import java.io.IOException; /** * This is the handler that process safe mode check command. @@ -34,22 +33,16 @@ description = "Stop ReplicationManager", mixinStandardHelpOptions = true, versionProvider = HddsVersionProvider.class) -public class ReplicationManagerStopSubcommand implements Callable { +public class ReplicationManagerStopSubcommand extends ScmSubcommand { private static final Logger LOG = LoggerFactory.getLogger(ReplicationManagerStopSubcommand.class); - @ParentCommand - private ReplicationManagerCommands parent; - @Override - public Void call() throws Exception { - try (ScmClient scmClient = parent.getParent().createScmClient()) { - scmClient.stopReplicationManager(); - LOG.info("Stopping ReplicationManager..."); - LOG.info("Requested SCM to stop ReplicationManager, " + - "it might take sometime for the ReplicationManager to stop."); - return null; - } + public void execute(ScmClient scmClient) throws IOException { + scmClient.stopReplicationManager(); + LOG.info("Stopping ReplicationManager..."); + LOG.info("Requested SCM to stop ReplicationManager, " + + "it might take sometime for the ReplicationManager to stop."); } } diff --git a/hadoop-hdds/tools/src/main/java/org/apache/hadoop/hdds/scm/cli/SafeModeCheckSubcommand.java b/hadoop-hdds/tools/src/main/java/org/apache/hadoop/hdds/scm/cli/SafeModeCheckSubcommand.java index b2cfea3daaaa..ba359af1c59b 100644 --- a/hadoop-hdds/tools/src/main/java/org/apache/hadoop/hdds/scm/cli/SafeModeCheckSubcommand.java +++ b/hadoop-hdds/tools/src/main/java/org/apache/hadoop/hdds/scm/cli/SafeModeCheckSubcommand.java @@ -1,4 +1,4 @@ -/** +/* * Licensed to the Apache Software Foundation (ASF) under one * or more contributor license agreements. See the NOTICE file * distributed with this work for additional information @@ -17,8 +17,8 @@ */ package org.apache.hadoop.hdds.scm.cli; +import java.io.IOException; import java.util.Map; -import java.util.concurrent.Callable; import org.apache.commons.lang3.tuple.Pair; import org.apache.hadoop.hdds.cli.HddsVersionProvider; @@ -28,7 +28,6 @@ import org.slf4j.LoggerFactory; import picocli.CommandLine; import picocli.CommandLine.Command; -import picocli.CommandLine.ParentCommand; /** * This is the handler that process safe mode check command. @@ -38,39 +37,32 @@ description = "Check if SCM is in safe mode", mixinStandardHelpOptions = true, versionProvider = HddsVersionProvider.class) -public class SafeModeCheckSubcommand implements Callable { +public class SafeModeCheckSubcommand extends ScmSubcommand { private static final Logger LOG = LoggerFactory.getLogger(SafeModeCheckSubcommand.class); - @ParentCommand - private SafeModeCommands parent; - @CommandLine.Option(names = {"--verbose"}, description = "Show detailed status of rules.") private boolean verbose; @Override - public Void call() throws Exception { - try (ScmClient scmClient = parent.getParent().createScmClient()) { - - boolean execReturn = scmClient.inSafeMode(); + public void execute(ScmClient scmClient) throws IOException { + boolean execReturn = scmClient.inSafeMode(); - // Output data list - if(execReturn){ - LOG.info("SCM is in safe mode."); - if (verbose) { - for (Map.Entry> entry : - scmClient.getSafeModeRuleStatuses().entrySet()) { - Pair value = entry.getValue(); - LOG.info("validated:{}, {}, {}", - value.getLeft(), entry.getKey(), value.getRight()); - } + // Output data list + if(execReturn){ + LOG.info("SCM is in safe mode."); + if (verbose) { + for (Map.Entry> entry : + scmClient.getSafeModeRuleStatuses().entrySet()) { + Pair value = entry.getValue(); + LOG.info("validated:{}, {}, {}", + value.getLeft(), entry.getKey(), value.getRight()); } - } else { - LOG.info("SCM is out of safe mode."); } - return null; + } else { + LOG.info("SCM is out of safe mode."); } } } diff --git a/hadoop-hdds/tools/src/main/java/org/apache/hadoop/hdds/scm/cli/SafeModeCommands.java b/hadoop-hdds/tools/src/main/java/org/apache/hadoop/hdds/scm/cli/SafeModeCommands.java index 017e1ba3c2c7..96c77c102cbe 100644 --- a/hadoop-hdds/tools/src/main/java/org/apache/hadoop/hdds/scm/cli/SafeModeCommands.java +++ b/hadoop-hdds/tools/src/main/java/org/apache/hadoop/hdds/scm/cli/SafeModeCommands.java @@ -1,4 +1,4 @@ -/** +/* * Licensed to the Apache Software Foundation (ASF) under one * or more contributor license agreements. See the NOTICE file * distributed with this work for additional information @@ -21,13 +21,9 @@ import org.apache.hadoop.hdds.cli.GenericCli; import org.apache.hadoop.hdds.cli.HddsVersionProvider; -import org.apache.hadoop.hdds.scm.cli.container.WithScmClient; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; import picocli.CommandLine.Command; import picocli.CommandLine.Model.CommandSpec; -import picocli.CommandLine.ParentCommand; import picocli.CommandLine.Spec; /** @@ -45,19 +41,9 @@ }) public class SafeModeCommands implements Callable { - private static final Logger LOG = - LoggerFactory.getLogger(SafeModeCommands.class); - @Spec private CommandSpec spec; - @ParentCommand - private WithScmClient parent; - - public WithScmClient getParent() { - return parent; - } - @Override public Void call() throws Exception { GenericCli.missingSubcommand(spec); diff --git a/hadoop-hdds/tools/src/main/java/org/apache/hadoop/hdds/scm/cli/SafeModeExitSubcommand.java b/hadoop-hdds/tools/src/main/java/org/apache/hadoop/hdds/scm/cli/SafeModeExitSubcommand.java index 9f1db45bb4e2..12490c5c2c51 100644 --- a/hadoop-hdds/tools/src/main/java/org/apache/hadoop/hdds/scm/cli/SafeModeExitSubcommand.java +++ b/hadoop-hdds/tools/src/main/java/org/apache/hadoop/hdds/scm/cli/SafeModeExitSubcommand.java @@ -1,4 +1,4 @@ -/** +/* * Licensed to the Apache Software Foundation (ASF) under one * or more contributor license agreements. See the NOTICE file * distributed with this work for additional information @@ -17,7 +17,7 @@ */ package org.apache.hadoop.hdds.scm.cli; -import java.util.concurrent.Callable; +import java.io.IOException; import org.apache.hadoop.hdds.cli.HddsVersionProvider; import org.apache.hadoop.hdds.scm.client.ScmClient; @@ -25,7 +25,6 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; import picocli.CommandLine.Command; -import picocli.CommandLine.ParentCommand; /** * This is the handler that process safe mode exit command. @@ -35,23 +34,16 @@ description = "Force SCM out of safe mode", mixinStandardHelpOptions = true, versionProvider = HddsVersionProvider.class) -public class SafeModeExitSubcommand implements Callable { +public class SafeModeExitSubcommand extends ScmSubcommand { private static final Logger LOG = LoggerFactory.getLogger(SafeModeExitSubcommand.class); - @ParentCommand - private SafeModeCommands parent; - @Override - public Void call() throws Exception { - try (ScmClient scmClient = parent.getParent().createScmClient()) { - - boolean execReturn = scmClient.forceExitSafeMode(); - if(execReturn){ - LOG.info("SCM exit safe mode successfully."); - } - return null; + public void execute(ScmClient scmClient) throws IOException { + boolean execReturn = scmClient.forceExitSafeMode(); + if(execReturn){ + LOG.info("SCM exit safe mode successfully."); } } } diff --git a/hadoop-hdds/tools/src/main/java/org/apache/hadoop/hdds/scm/cli/SafeModeWaitSubcommand.java b/hadoop-hdds/tools/src/main/java/org/apache/hadoop/hdds/scm/cli/SafeModeWaitSubcommand.java index 7668a47d24d2..e3fb5c1e718e 100644 --- a/hadoop-hdds/tools/src/main/java/org/apache/hadoop/hdds/scm/cli/SafeModeWaitSubcommand.java +++ b/hadoop-hdds/tools/src/main/java/org/apache/hadoop/hdds/scm/cli/SafeModeWaitSubcommand.java @@ -1,4 +1,4 @@ -/** +/* * Licensed to the Apache Software Foundation (ASF) under one * or more contributor license agreements. See the NOTICE file * distributed with this work for additional information @@ -27,7 +27,7 @@ import org.slf4j.LoggerFactory; import picocli.CommandLine.Command; import picocli.CommandLine.Option; -import picocli.CommandLine.ParentCommand; +import picocli.CommandLine.Mixin; /** * This is the handler that process safe mode wait command. @@ -45,21 +45,20 @@ public class SafeModeWaitSubcommand implements Callable { @Option(description = "Define timeout (in second) to wait until (exit code 1) " + "or until safemode is ended (exit code 0).", defaultValue = "30", - required = false, names = { - "-t", "--timeout"}) + names = { "-t", "--timeout"}) private long timeoutSeconds; private long startTestTime; - @ParentCommand - private SafeModeCommands parent; + @Mixin + private ScmOption scmOption; @Override public Void call() throws Exception { startTestTime = System.currentTimeMillis(); while (getRemainingTimeInSec() > 0) { - try (ScmClient scmClient = parent.getParent().createScmClient()) { + try (ScmClient scmClient = scmOption.createScmClient()) { while (getRemainingTimeInSec() > 0) { boolean isSafeModeActive = scmClient.inSafeMode(); diff --git a/hadoop-hdds/tools/src/main/java/org/apache/hadoop/hdds/scm/cli/ScmOption.java b/hadoop-hdds/tools/src/main/java/org/apache/hadoop/hdds/scm/cli/ScmOption.java new file mode 100644 index 000000000000..5b8b81436c78 --- /dev/null +++ b/hadoop-hdds/tools/src/main/java/org/apache/hadoop/hdds/scm/cli/ScmOption.java @@ -0,0 +1,72 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

+ * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hdds.scm.cli; + +import org.apache.commons.lang3.StringUtils; +import org.apache.hadoop.hdds.HddsUtils; +import org.apache.hadoop.hdds.cli.GenericParentCommand; +import org.apache.hadoop.hdds.conf.MutableConfigurationSource; +import org.apache.hadoop.hdds.conf.OzoneConfiguration; +import org.apache.hadoop.hdds.scm.ScmConfigKeys; +import org.apache.hadoop.hdds.scm.client.ScmClient; +import picocli.CommandLine; + +import java.io.IOException; + +import static org.apache.hadoop.hdds.scm.ScmConfigKeys.OZONE_SCM_CLIENT_ADDRESS_KEY; +import static picocli.CommandLine.Spec.Target.MIXEE; + +/** + * Defines command-line option for SCM address. + */ +public class ScmOption { + + @CommandLine.Spec(MIXEE) + private CommandLine.Model.CommandSpec spec; + + @CommandLine.Option(names = {"--scm"}, + description = "The destination scm (host:port)") + private String scm; + + public ScmClient createScmClient() { + try { + GenericParentCommand parent = (GenericParentCommand) + spec.root().userObject(); + OzoneConfiguration conf = parent.createOzoneConfiguration(); + checkAndSetSCMAddressArg(conf); + + return new ContainerOperationClient(conf); + } catch (IOException ex) { + throw new IllegalArgumentException("Can't create SCM client", ex); + } + } + + private void checkAndSetSCMAddressArg(MutableConfigurationSource conf) { + if (StringUtils.isNotEmpty(scm)) { + conf.set(OZONE_SCM_CLIENT_ADDRESS_KEY, scm); + } + if (!HddsUtils.getHostNameFromConfigKeys(conf, + ScmConfigKeys.OZONE_SCM_CLIENT_ADDRESS_KEY).isPresent()) { + + throw new IllegalArgumentException( + ScmConfigKeys.OZONE_SCM_CLIENT_ADDRESS_KEY + + " should be set in ozone-site.xml or with the --scm option"); + } + } + +} diff --git a/hadoop-hdds/tools/src/main/java/org/apache/hadoop/hdds/scm/cli/container/WithScmClient.java b/hadoop-hdds/tools/src/main/java/org/apache/hadoop/hdds/scm/cli/ScmSubcommand.java similarity index 60% rename from hadoop-hdds/tools/src/main/java/org/apache/hadoop/hdds/scm/cli/container/WithScmClient.java rename to hadoop-hdds/tools/src/main/java/org/apache/hadoop/hdds/scm/cli/ScmSubcommand.java index 9852d50fc8c9..6dc09c2cbecd 100644 --- a/hadoop-hdds/tools/src/main/java/org/apache/hadoop/hdds/scm/cli/container/WithScmClient.java +++ b/hadoop-hdds/tools/src/main/java/org/apache/hadoop/hdds/scm/cli/ScmSubcommand.java @@ -1,4 +1,4 @@ -/** +/* * Licensed to the Apache Software Foundation (ASF) under one * or more contributor license agreements. See the NOTICE file * distributed with this work for additional information @@ -15,15 +15,29 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.apache.hadoop.hdds.scm.cli.container; +package org.apache.hadoop.hdds.scm.cli; import org.apache.hadoop.hdds.scm.client.ScmClient; +import picocli.CommandLine; + +import java.io.IOException; +import java.util.concurrent.Callable; /** - * Command which provides a SCM client based on the current config. + * Base class for admin commands that connect via SCM client. */ -public interface WithScmClient { +public abstract class ScmSubcommand implements Callable { + + @CommandLine.Mixin + private ScmOption scmOption; - ScmClient createScmClient(); + protected abstract void execute(ScmClient client) throws IOException; + @Override + public final Void call() throws Exception { + try (ScmClient scmClient = scmOption.createScmClient()) { + execute(scmClient); + return null; + } + } } diff --git a/hadoop-hdds/tools/src/main/java/org/apache/hadoop/hdds/scm/cli/TopologySubcommand.java b/hadoop-hdds/tools/src/main/java/org/apache/hadoop/hdds/scm/cli/TopologySubcommand.java index 214da34561b5..3efb2b3ebbb8 100644 --- a/hadoop-hdds/tools/src/main/java/org/apache/hadoop/hdds/scm/cli/TopologySubcommand.java +++ b/hadoop-hdds/tools/src/main/java/org/apache/hadoop/hdds/scm/cli/TopologySubcommand.java @@ -1,4 +1,4 @@ -/** +/* * Licensed to the Apache Software Foundation (ASF) under one * or more contributor license agreements. See the NOTICE file * distributed with this work for additional information @@ -29,7 +29,6 @@ import org.apache.hadoop.hdds.cli.HddsVersionProvider; import org.apache.hadoop.hdds.protocol.DatanodeDetails; import org.apache.hadoop.hdds.protocol.proto.HddsProtos; -import org.apache.hadoop.hdds.scm.cli.container.WithScmClient; import org.apache.hadoop.hdds.scm.client.ScmClient; import static org.apache.hadoop.hdds.protocol.proto.HddsProtos.NodeState.DEAD; @@ -54,17 +53,17 @@ public class TopologySubcommand implements Callable { @Spec private CommandSpec spec; - @CommandLine.ParentCommand - private WithScmClient parent; + @CommandLine.Mixin + private ScmOption scmOption; - private static List stateArray = new ArrayList<>(); + private static final List STATES = new ArrayList<>(); static { - stateArray.add(HEALTHY); - stateArray.add(STALE); - stateArray.add(DEAD); - stateArray.add(DECOMMISSIONING); - stateArray.add(DECOMMISSIONED); + STATES.add(HEALTHY); + STATES.add(STALE); + STATES.add(DEAD); + STATES.add(DECOMMISSIONING); + STATES.add(DECOMMISSIONED); } @CommandLine.Option(names = {"-o", "--order"}, @@ -77,11 +76,11 @@ public class TopologySubcommand implements Callable { @Override public Void call() throws Exception { - try (ScmClient scmClient = parent.createScmClient()) { - for (HddsProtos.NodeState state : stateArray) { + try (ScmClient scmClient = scmOption.createScmClient()) { + for (HddsProtos.NodeState state : STATES) { List nodes = scmClient.queryNode(state, HddsProtos.QueryScope.CLUSTER, ""); - if (nodes != null && nodes.size() > 0) { + if (nodes != null && !nodes.isEmpty()) { // show node state System.out.println("State = " + state.toString()); if (order) { @@ -124,7 +123,7 @@ private String formatPortOutput(List ports) { StringBuilder sb = new StringBuilder(); for (int i = 0; i < ports.size(); i++) { HddsProtos.Port port = ports.get(i); - sb.append(port.getName() + "=" + port.getValue()); + sb.append(port.getName()).append("=").append(port.getValue()); if (i < ports.size() - 1) { sb.append(","); } diff --git a/hadoop-hdds/tools/src/main/java/org/apache/hadoop/hdds/scm/cli/container/CloseSubcommand.java b/hadoop-hdds/tools/src/main/java/org/apache/hadoop/hdds/scm/cli/container/CloseSubcommand.java index cd81d32b8a82..53cbd2f63da3 100644 --- a/hadoop-hdds/tools/src/main/java/org/apache/hadoop/hdds/scm/cli/container/CloseSubcommand.java +++ b/hadoop-hdds/tools/src/main/java/org/apache/hadoop/hdds/scm/cli/container/CloseSubcommand.java @@ -1,4 +1,4 @@ -/** +/* * Licensed to the Apache Software Foundation (ASF) under one * or more contributor license agreements. See the NOTICE file * distributed with this work for additional information @@ -17,15 +17,15 @@ */ package org.apache.hadoop.hdds.scm.cli.container; -import java.util.concurrent.Callable; +import java.io.IOException; import org.apache.hadoop.hdds.cli.HddsVersionProvider; +import org.apache.hadoop.hdds.scm.cli.ScmSubcommand; import org.apache.hadoop.hdds.scm.client.ScmClient; import static org.apache.hadoop.hdds.scm.cli.container.ContainerCommands.checkContainerExists; import picocli.CommandLine.Command; import picocli.CommandLine.Parameters; -import picocli.CommandLine.ParentCommand; /** * The handler of close container command. @@ -35,21 +35,15 @@ description = "close container", mixinStandardHelpOptions = true, versionProvider = HddsVersionProvider.class) -public class CloseSubcommand implements Callable { - - @ParentCommand - private ContainerCommands parent; +public class CloseSubcommand extends ScmSubcommand { @Parameters(description = "Id of the container to close") private long containerId; @Override - public Void call() throws Exception { - try (ScmClient scmClient = parent.getParent().createScmClient()) { - checkContainerExists(scmClient, containerId); - scmClient.closeContainer(containerId); - return null; - } + public void execute(ScmClient scmClient) throws IOException { + checkContainerExists(scmClient, containerId); + scmClient.closeContainer(containerId); } } diff --git a/hadoop-hdds/tools/src/main/java/org/apache/hadoop/hdds/scm/cli/container/ContainerCommands.java b/hadoop-hdds/tools/src/main/java/org/apache/hadoop/hdds/scm/cli/container/ContainerCommands.java index cf665b008f72..d1b9d180dec0 100644 --- a/hadoop-hdds/tools/src/main/java/org/apache/hadoop/hdds/scm/cli/container/ContainerCommands.java +++ b/hadoop-hdds/tools/src/main/java/org/apache/hadoop/hdds/scm/cli/container/ContainerCommands.java @@ -1,4 +1,4 @@ -/** +/* * Licensed to the Apache Software Foundation (ASF) under one * or more contributor license agreements. See the NOTICE file * distributed with this work for additional information @@ -27,7 +27,6 @@ import picocli.CommandLine.Command; import picocli.CommandLine.Model.CommandSpec; -import picocli.CommandLine.ParentCommand; import picocli.CommandLine.Spec; /** @@ -50,13 +49,6 @@ public class ContainerCommands implements Callable { @Spec private CommandSpec spec; - @ParentCommand - private WithScmClient parent; - - public WithScmClient getParent() { - return parent; - } - @Override public Void call() throws Exception { GenericCli.missingSubcommand(spec); diff --git a/hadoop-hdds/tools/src/main/java/org/apache/hadoop/hdds/scm/cli/container/CreateSubcommand.java b/hadoop-hdds/tools/src/main/java/org/apache/hadoop/hdds/scm/cli/container/CreateSubcommand.java index eb79e50506e2..9eedbf858958 100644 --- a/hadoop-hdds/tools/src/main/java/org/apache/hadoop/hdds/scm/cli/container/CreateSubcommand.java +++ b/hadoop-hdds/tools/src/main/java/org/apache/hadoop/hdds/scm/cli/container/CreateSubcommand.java @@ -1,4 +1,4 @@ -/** +/* * Licensed to the Apache Software Foundation (ASF) under one * or more contributor license agreements. See the NOTICE file * distributed with this work for additional information @@ -17,9 +17,10 @@ */ package org.apache.hadoop.hdds.scm.cli.container; -import java.util.concurrent.Callable; +import java.io.IOException; import org.apache.hadoop.hdds.cli.HddsVersionProvider; +import org.apache.hadoop.hdds.scm.cli.ScmSubcommand; import org.apache.hadoop.hdds.scm.client.ScmClient; import org.apache.hadoop.hdds.scm.container.common.helpers .ContainerWithPipeline; @@ -28,7 +29,6 @@ import org.slf4j.LoggerFactory; import picocli.CommandLine.Command; import picocli.CommandLine.Option; -import picocli.CommandLine.ParentCommand; /** * This is the handler that process container creation command. @@ -38,27 +38,19 @@ description = "Create container", mixinStandardHelpOptions = true, versionProvider = HddsVersionProvider.class) -public class CreateSubcommand implements Callable { +public class CreateSubcommand extends ScmSubcommand { private static final Logger LOG = LoggerFactory.getLogger(CreateSubcommand.class); - @ParentCommand - private ContainerCommands parent; - @Option(description = "Owner of the new container", defaultValue = "OZONE", - required = false, names = { - "-o", "--owner"}) - + names = { "-o", "--owner"}) private String owner; @Override - public Void call() throws Exception { - try (ScmClient scmClient = parent.getParent().createScmClient()) { - ContainerWithPipeline container = scmClient.createContainer(owner); - LOG.info("Container {} is created.", - container.getContainerInfo().getContainerID()); - return null; - } + public void execute(ScmClient scmClient) throws IOException { + ContainerWithPipeline container = scmClient.createContainer(owner); + LOG.info("Container {} is created.", + container.getContainerInfo().getContainerID()); } } diff --git a/hadoop-hdds/tools/src/main/java/org/apache/hadoop/hdds/scm/cli/container/DeleteSubcommand.java b/hadoop-hdds/tools/src/main/java/org/apache/hadoop/hdds/scm/cli/container/DeleteSubcommand.java index a438fe906131..62d1b8ab2ae3 100644 --- a/hadoop-hdds/tools/src/main/java/org/apache/hadoop/hdds/scm/cli/container/DeleteSubcommand.java +++ b/hadoop-hdds/tools/src/main/java/org/apache/hadoop/hdds/scm/cli/container/DeleteSubcommand.java @@ -1,4 +1,4 @@ -/** +/* * Licensed to the Apache Software Foundation (ASF) under one * or more contributor license agreements. See the NOTICE file * distributed with this work for additional information @@ -18,16 +18,16 @@ package org.apache.hadoop.hdds.scm.cli.container; -import java.util.concurrent.Callable; +import java.io.IOException; import org.apache.hadoop.hdds.cli.HddsVersionProvider; +import org.apache.hadoop.hdds.scm.cli.ScmSubcommand; import org.apache.hadoop.hdds.scm.client.ScmClient; import static org.apache.hadoop.hdds.scm.cli.container.ContainerCommands.checkContainerExists; import picocli.CommandLine.Command; import picocli.CommandLine.Option; import picocli.CommandLine.Parameters; -import picocli.CommandLine.ParentCommand; /** * This is the handler that process delete container command. @@ -37,7 +37,7 @@ description = "Delete container", mixinStandardHelpOptions = true, versionProvider = HddsVersionProvider.class) -public class DeleteSubcommand implements Callable { +public class DeleteSubcommand extends ScmSubcommand { @Parameters(description = "Id of the container to close") private long containerId; @@ -46,15 +46,9 @@ public class DeleteSubcommand implements Callable { "--force"}, description = "forcibly delete the container") private boolean force; - @ParentCommand - private ContainerCommands parent; - @Override - public Void call() throws Exception { - try (ScmClient scmClient = parent.getParent().createScmClient()) { - checkContainerExists(scmClient, containerId); - scmClient.deleteContainer(containerId, force); - return null; - } + public void execute(ScmClient scmClient) throws IOException { + checkContainerExists(scmClient, containerId); + scmClient.deleteContainer(containerId, force); } } diff --git a/hadoop-hdds/tools/src/main/java/org/apache/hadoop/hdds/scm/cli/container/InfoSubcommand.java b/hadoop-hdds/tools/src/main/java/org/apache/hadoop/hdds/scm/cli/container/InfoSubcommand.java index 31e2a45dfc58..5defc2456a42 100644 --- a/hadoop-hdds/tools/src/main/java/org/apache/hadoop/hdds/scm/cli/container/InfoSubcommand.java +++ b/hadoop-hdds/tools/src/main/java/org/apache/hadoop/hdds/scm/cli/container/InfoSubcommand.java @@ -1,4 +1,4 @@ -/** +/* * Licensed to the Apache Software Foundation (ASF) under one * or more contributor license agreements. See the NOTICE file * distributed with this work for additional information @@ -17,11 +17,12 @@ */ package org.apache.hadoop.hdds.scm.cli.container; -import java.util.concurrent.Callable; +import java.io.IOException; import java.util.stream.Collectors; import org.apache.hadoop.hdds.cli.HddsVersionProvider; import org.apache.hadoop.hdds.protocol.DatanodeDetails; +import org.apache.hadoop.hdds.scm.cli.ScmSubcommand; import org.apache.hadoop.hdds.scm.client.ScmClient; import org.apache.hadoop.hdds.scm.container.common.helpers .ContainerWithPipeline; @@ -31,7 +32,6 @@ import org.slf4j.LoggerFactory; import picocli.CommandLine.Command; import picocli.CommandLine.Parameters; -import picocli.CommandLine.ParentCommand; /** * This is the handler that process container info command. @@ -41,36 +41,30 @@ description = "Show information about a specific container", mixinStandardHelpOptions = true, versionProvider = HddsVersionProvider.class) -public class InfoSubcommand implements Callable { +public class InfoSubcommand extends ScmSubcommand { private static final Logger LOG = LoggerFactory.getLogger(InfoSubcommand.class); - @ParentCommand - private ContainerCommands parent; - @Parameters(description = "Decimal id of the container.") private long containerID; @Override - public Void call() throws Exception { - try (ScmClient scmClient = parent.getParent().createScmClient()) { - final ContainerWithPipeline container = scmClient. - getContainerWithPipeline(containerID); - Preconditions.checkNotNull(container, "Container cannot be null"); + public void execute(ScmClient scmClient) throws IOException { + final ContainerWithPipeline container = scmClient. + getContainerWithPipeline(containerID); + Preconditions.checkNotNull(container, "Container cannot be null"); - // Print container report info. - LOG.info("Container id: {}", containerID); - LOG.info("Pipeline id: {}", container.getPipeline().getId().getId()); - LOG.info("Container State: {}", container.getContainerInfo().getState()); + // Print container report info. + LOG.info("Container id: {}", containerID); + LOG.info("Pipeline id: {}", container.getPipeline().getId().getId()); + LOG.info("Container State: {}", container.getContainerInfo().getState()); - // Print pipeline of an existing container. - String machinesStr = container.getPipeline().getNodes().stream().map( - InfoSubcommand::buildDatanodeDetails) - .collect(Collectors.joining(",\n")); - LOG.info("Datanodes: [{}]", machinesStr); - return null; - } + // Print pipeline of an existing container. + String machinesStr = container.getPipeline().getNodes().stream().map( + InfoSubcommand::buildDatanodeDetails) + .collect(Collectors.joining(",\n")); + LOG.info("Datanodes: [{}]", machinesStr); } private static String buildDatanodeDetails(DatanodeDetails details) { diff --git a/hadoop-hdds/tools/src/main/java/org/apache/hadoop/hdds/scm/cli/container/ListSubcommand.java b/hadoop-hdds/tools/src/main/java/org/apache/hadoop/hdds/scm/cli/container/ListSubcommand.java index 3ffc118b57be..e9b0b7dc9a50 100644 --- a/hadoop-hdds/tools/src/main/java/org/apache/hadoop/hdds/scm/cli/container/ListSubcommand.java +++ b/hadoop-hdds/tools/src/main/java/org/apache/hadoop/hdds/scm/cli/container/ListSubcommand.java @@ -1,4 +1,4 @@ -/** +/* * Licensed to the Apache Software Foundation (ASF) under one * or more contributor license agreements. See the NOTICE file * distributed with this work for additional information @@ -19,9 +19,9 @@ import java.io.IOException; import java.util.List; -import java.util.concurrent.Callable; import org.apache.hadoop.hdds.cli.HddsVersionProvider; +import org.apache.hadoop.hdds.scm.cli.ScmSubcommand; import org.apache.hadoop.hdds.scm.client.ScmClient; import org.apache.hadoop.hdds.scm.container.ContainerInfo; @@ -36,7 +36,6 @@ import picocli.CommandLine.Command; import picocli.CommandLine.Help.Visibility; import picocli.CommandLine.Option; -import picocli.CommandLine.ParentCommand; /** * This is the handler that process container list command. @@ -46,22 +45,19 @@ description = "List containers", mixinStandardHelpOptions = true, versionProvider = HddsVersionProvider.class) -public class ListSubcommand implements Callable { +public class ListSubcommand extends ScmSubcommand { private static final Logger LOG = LoggerFactory.getLogger(ListSubcommand.class); - @ParentCommand - private ContainerCommands parent; - @Option(names = {"-s", "--start"}, - description = "Container id to start the iteration", required = false) - private long startId = 0; + description = "Container id to start the iteration") + private long startId; @Option(names = {"-c", "--count"}, description = "Maximum number of containers to list", defaultValue = "20", showDefaultValue = Visibility.ALWAYS) - private int count = 20; + private int count; private static final ObjectWriter WRITER; @@ -83,17 +79,13 @@ private void outputContainerInfo(ContainerInfo containerInfo) } @Override - public Void call() throws Exception { - try (ScmClient scmClient = parent.getParent().createScmClient()) { - - List containerList = - scmClient.listContainer(startId, count); + public void execute(ScmClient scmClient) throws IOException { + List containerList = + scmClient.listContainer(startId, count); - // Output data list - for (ContainerInfo container : containerList) { - outputContainerInfo(container); - } - return null; + // Output data list + for (ContainerInfo container : containerList) { + outputContainerInfo(container); } } } diff --git a/hadoop-hdds/tools/src/main/java/org/apache/hadoop/hdds/scm/cli/datanode/DatanodeCommands.java b/hadoop-hdds/tools/src/main/java/org/apache/hadoop/hdds/scm/cli/datanode/DatanodeCommands.java index b7ba59c77604..da0b87b6437e 100644 --- a/hadoop-hdds/tools/src/main/java/org/apache/hadoop/hdds/scm/cli/datanode/DatanodeCommands.java +++ b/hadoop-hdds/tools/src/main/java/org/apache/hadoop/hdds/scm/cli/datanode/DatanodeCommands.java @@ -1,4 +1,4 @@ -/** +/* * Licensed to the Apache Software Foundation (ASF) under one * or more contributor license agreements. See the NOTICE file * distributed with this work for additional information @@ -21,7 +21,6 @@ import org.apache.hadoop.hdds.cli.GenericCli; import org.apache.hadoop.hdds.cli.HddsVersionProvider; -import org.apache.hadoop.hdds.scm.cli.container.WithScmClient; import picocli.CommandLine; import picocli.CommandLine.Model.CommandSpec; @@ -43,13 +42,6 @@ public class DatanodeCommands implements Callable { @Spec private CommandSpec spec; - @CommandLine.ParentCommand - private WithScmClient parent; - - public WithScmClient getParent() { - return parent; - } - @Override public Void call() throws Exception { GenericCli.missingSubcommand(spec); diff --git a/hadoop-hdds/tools/src/main/java/org/apache/hadoop/hdds/scm/cli/datanode/ListInfoSubcommand.java b/hadoop-hdds/tools/src/main/java/org/apache/hadoop/hdds/scm/cli/datanode/ListInfoSubcommand.java index e4060b3dadaf..80c5ecaae820 100644 --- a/hadoop-hdds/tools/src/main/java/org/apache/hadoop/hdds/scm/cli/datanode/ListInfoSubcommand.java +++ b/hadoop-hdds/tools/src/main/java/org/apache/hadoop/hdds/scm/cli/datanode/ListInfoSubcommand.java @@ -1,4 +1,4 @@ -/** +/* * Licensed to the Apache Software Foundation (ASF) under one * or more contributor license agreements. See the NOTICE file * distributed with this work for additional information @@ -21,13 +21,13 @@ import org.apache.hadoop.hdds.cli.HddsVersionProvider; import org.apache.hadoop.hdds.protocol.DatanodeDetails; import org.apache.hadoop.hdds.protocol.proto.HddsProtos; +import org.apache.hadoop.hdds.scm.cli.ScmSubcommand; import org.apache.hadoop.hdds.scm.client.ScmClient; import org.apache.hadoop.hdds.scm.pipeline.Pipeline; import picocli.CommandLine; import java.io.IOException; import java.util.List; -import java.util.concurrent.Callable; import java.util.stream.Collectors; import java.util.stream.Stream; @@ -39,44 +39,36 @@ description = "List info of datanodes", mixinStandardHelpOptions = true, versionProvider = HddsVersionProvider.class) -public class ListInfoSubcommand implements Callable { - - @CommandLine.ParentCommand - private DatanodeCommands parent; +public class ListInfoSubcommand extends ScmSubcommand { @CommandLine.Option(names = {"--ip"}, description = "Show info by ip address.", - defaultValue = "", - required = false) + defaultValue = "") private String ipaddress; @CommandLine.Option(names = {"--id"}, description = "Show info by datanode UUID.", - defaultValue = "", - required = false) + defaultValue = "") private String uuid; private List pipelines; @Override - public Void call() throws Exception { - try (ScmClient scmClient = parent.getParent().createScmClient()) { - pipelines = scmClient.listPipelines(); - if (Strings.isNullOrEmpty(ipaddress) && Strings.isNullOrEmpty(uuid)) { - getAllNodes(scmClient).stream().forEach(p -> printDatanodeInfo(p)); - } else { - Stream allNodes = getAllNodes(scmClient).stream(); - if (!Strings.isNullOrEmpty(ipaddress)) { - allNodes = allNodes.filter(p -> p.getIpAddress() - .compareToIgnoreCase(ipaddress) == 0); - } - if (!Strings.isNullOrEmpty(uuid)) { - allNodes = allNodes.filter(p -> p.getUuid().toString().equals(uuid)); - } - allNodes.forEach(p -> printDatanodeInfo(p)); + public void execute(ScmClient scmClient) throws IOException { + pipelines = scmClient.listPipelines(); + if (Strings.isNullOrEmpty(ipaddress) && Strings.isNullOrEmpty(uuid)) { + getAllNodes(scmClient).forEach(this::printDatanodeInfo); + } else { + Stream allNodes = getAllNodes(scmClient).stream(); + if (!Strings.isNullOrEmpty(ipaddress)) { + allNodes = allNodes.filter(p -> p.getIpAddress() + .compareToIgnoreCase(ipaddress) == 0); + } + if (!Strings.isNullOrEmpty(uuid)) { + allNodes = allNodes.filter(p -> p.getUuid().toString().equals(uuid)); } - return null; + allNodes.forEach(this::printDatanodeInfo); } } @@ -101,7 +93,7 @@ private void printDatanodeInfo(DatanodeDetails datanode) { " or the node is not in Healthy state."); } else { relatedPipelineNum = relatedPipelines.size(); - relatedPipelines.stream().forEach( + relatedPipelines.forEach( p -> pipelineListInfo.append(p.getId().getId().toString()) .append("/").append(p.getFactor().toString()).append("/") .append(p.getType().toString()).append("/") @@ -118,4 +110,4 @@ private void printDatanodeInfo(DatanodeDetails datanode) { + "/" + datanode.getHostName() + "/" + relatedPipelineNum + " pipelines) \n" + "Related pipelines: \n" + pipelineListInfo); } -} \ No newline at end of file +} diff --git a/hadoop-hdds/tools/src/main/java/org/apache/hadoop/hdds/scm/cli/pipeline/ActivatePipelineSubcommand.java b/hadoop-hdds/tools/src/main/java/org/apache/hadoop/hdds/scm/cli/pipeline/ActivatePipelineSubcommand.java index ec4b1b789e8c..a61655dc66b0 100644 --- a/hadoop-hdds/tools/src/main/java/org/apache/hadoop/hdds/scm/cli/pipeline/ActivatePipelineSubcommand.java +++ b/hadoop-hdds/tools/src/main/java/org/apache/hadoop/hdds/scm/cli/pipeline/ActivatePipelineSubcommand.java @@ -1,4 +1,4 @@ -/** +/* * Licensed to the Apache Software Foundation (ASF) under one * or more contributor license agreements. See the NOTICE file * distributed with this work for additional information @@ -20,10 +20,11 @@ import org.apache.hadoop.hdds.cli.HddsVersionProvider; import org.apache.hadoop.hdds.protocol.proto.HddsProtos; +import org.apache.hadoop.hdds.scm.cli.ScmSubcommand; import org.apache.hadoop.hdds.scm.client.ScmClient; import picocli.CommandLine; -import java.util.concurrent.Callable; +import java.io.IOException; /** * Handler of activate pipeline command. @@ -33,20 +34,14 @@ description = "Activates the given Pipeline", mixinStandardHelpOptions = true, versionProvider = HddsVersionProvider.class) -public class ActivatePipelineSubcommand implements Callable { - - @CommandLine.ParentCommand - private PipelineCommands parent; +public class ActivatePipelineSubcommand extends ScmSubcommand { @CommandLine.Parameters(description = "ID of the pipeline to activate") private String pipelineId; @Override - public Void call() throws Exception { - try (ScmClient scmClient = parent.getParent().createScmClient()) { - scmClient.activatePipeline( - HddsProtos.PipelineID.newBuilder().setId(pipelineId).build()); - return null; - } + public void execute(ScmClient scmClient) throws IOException { + scmClient.activatePipeline( + HddsProtos.PipelineID.newBuilder().setId(pipelineId).build()); } } diff --git a/hadoop-hdds/tools/src/main/java/org/apache/hadoop/hdds/scm/cli/pipeline/ClosePipelineSubcommand.java b/hadoop-hdds/tools/src/main/java/org/apache/hadoop/hdds/scm/cli/pipeline/ClosePipelineSubcommand.java index 89a280e805c0..78b83e56db07 100644 --- a/hadoop-hdds/tools/src/main/java/org/apache/hadoop/hdds/scm/cli/pipeline/ClosePipelineSubcommand.java +++ b/hadoop-hdds/tools/src/main/java/org/apache/hadoop/hdds/scm/cli/pipeline/ClosePipelineSubcommand.java @@ -1,4 +1,4 @@ -/** +/* * Licensed to the Apache Software Foundation (ASF) under one * or more contributor license agreements. See the NOTICE file * distributed with this work for additional information @@ -20,10 +20,11 @@ import org.apache.hadoop.hdds.cli.HddsVersionProvider; import org.apache.hadoop.hdds.protocol.proto.HddsProtos; +import org.apache.hadoop.hdds.scm.cli.ScmSubcommand; import org.apache.hadoop.hdds.scm.client.ScmClient; import picocli.CommandLine; -import java.util.concurrent.Callable; +import java.io.IOException; /** * Handler of close pipeline command. @@ -33,20 +34,14 @@ description = "Close pipeline", mixinStandardHelpOptions = true, versionProvider = HddsVersionProvider.class) -public class ClosePipelineSubcommand implements Callable { - - @CommandLine.ParentCommand - private PipelineCommands parent; +public class ClosePipelineSubcommand extends ScmSubcommand { @CommandLine.Parameters(description = "ID of the pipeline to close") private String pipelineId; @Override - public Void call() throws Exception { - try (ScmClient scmClient = parent.getParent().createScmClient()) { - scmClient.closePipeline( - HddsProtos.PipelineID.newBuilder().setId(pipelineId).build()); - return null; - } + public void execute(ScmClient scmClient) throws IOException { + scmClient.closePipeline( + HddsProtos.PipelineID.newBuilder().setId(pipelineId).build()); } } diff --git a/hadoop-hdds/tools/src/main/java/org/apache/hadoop/hdds/scm/cli/pipeline/CreatePipelineSubcommand.java b/hadoop-hdds/tools/src/main/java/org/apache/hadoop/hdds/scm/cli/pipeline/CreatePipelineSubcommand.java index e0bdddb7797e..c784be88b376 100644 --- a/hadoop-hdds/tools/src/main/java/org/apache/hadoop/hdds/scm/cli/pipeline/CreatePipelineSubcommand.java +++ b/hadoop-hdds/tools/src/main/java/org/apache/hadoop/hdds/scm/cli/pipeline/CreatePipelineSubcommand.java @@ -1,4 +1,4 @@ -/** +/* * Licensed to the Apache Software Foundation (ASF) under one * or more contributor license agreements. See the NOTICE file * distributed with this work for additional information @@ -20,11 +20,12 @@ import org.apache.hadoop.hdds.cli.HddsVersionProvider; import org.apache.hadoop.hdds.protocol.proto.HddsProtos; +import org.apache.hadoop.hdds.scm.cli.ScmSubcommand; import org.apache.hadoop.hdds.scm.client.ScmClient; import org.apache.hadoop.hdds.scm.pipeline.Pipeline; import picocli.CommandLine; -import java.util.concurrent.Callable; +import java.io.IOException; /** * Handler of createPipeline command. @@ -34,44 +35,37 @@ description = "create pipeline", mixinStandardHelpOptions = true, versionProvider = HddsVersionProvider.class) -public class CreatePipelineSubcommand implements Callable { - @CommandLine.ParentCommand - private PipelineCommands parent; +public class CreatePipelineSubcommand extends ScmSubcommand { @CommandLine.Option( names = {"-t", "--replicationType"}, description = "Replication type (STAND_ALONE, RATIS)", defaultValue = "STAND_ALONE" ) - private HddsProtos.ReplicationType type - = HddsProtos.ReplicationType.STAND_ALONE; + private HddsProtos.ReplicationType type; @CommandLine.Option( names = {"-f", "--replicationFactor"}, description = "Replication factor (ONE, THREE)", defaultValue = "ONE" ) - private HddsProtos.ReplicationFactor factor - = HddsProtos.ReplicationFactor.ONE; + private HddsProtos.ReplicationFactor factor; @Override - public Void call() throws Exception { + public void execute(ScmClient scmClient) throws IOException { if (type == HddsProtos.ReplicationType.CHAINED) { throw new IllegalArgumentException(type.name() + " is not supported yet."); } - try (ScmClient scmClient = parent.getParent().createScmClient()) { - Pipeline pipeline = scmClient.createReplicationPipeline( - type, - factor, - HddsProtos.NodePool.getDefaultInstance()); + Pipeline pipeline = scmClient.createReplicationPipeline( + type, + factor, + HddsProtos.NodePool.getDefaultInstance()); - if (pipeline != null) { - System.out.println(pipeline.getId().toString() + - " is created. Factor: " + pipeline.getFactor() + - ", Type: " + pipeline.getType()); - } - return null; + if (pipeline != null) { + System.out.println(pipeline.getId().toString() + + " is created. Factor: " + pipeline.getFactor() + + ", Type: " + pipeline.getType()); } } -} \ No newline at end of file +} diff --git a/hadoop-hdds/tools/src/main/java/org/apache/hadoop/hdds/scm/cli/pipeline/DeactivatePipelineSubcommand.java b/hadoop-hdds/tools/src/main/java/org/apache/hadoop/hdds/scm/cli/pipeline/DeactivatePipelineSubcommand.java index 4f4f741a3647..70df4d91fae9 100644 --- a/hadoop-hdds/tools/src/main/java/org/apache/hadoop/hdds/scm/cli/pipeline/DeactivatePipelineSubcommand.java +++ b/hadoop-hdds/tools/src/main/java/org/apache/hadoop/hdds/scm/cli/pipeline/DeactivatePipelineSubcommand.java @@ -1,4 +1,4 @@ -/** +/* * Licensed to the Apache Software Foundation (ASF) under one * or more contributor license agreements. See the NOTICE file * distributed with this work for additional information @@ -20,10 +20,11 @@ import org.apache.hadoop.hdds.cli.HddsVersionProvider; import org.apache.hadoop.hdds.protocol.proto.HddsProtos; +import org.apache.hadoop.hdds.scm.cli.ScmSubcommand; import org.apache.hadoop.hdds.scm.client.ScmClient; import picocli.CommandLine; -import java.util.concurrent.Callable; +import java.io.IOException; /** * Handler of deactivate pipeline command. @@ -33,20 +34,14 @@ description = "Deactivates the given Pipeline", mixinStandardHelpOptions = true, versionProvider = HddsVersionProvider.class) -public class DeactivatePipelineSubcommand implements Callable { - - @CommandLine.ParentCommand - private PipelineCommands parent; +public class DeactivatePipelineSubcommand extends ScmSubcommand { @CommandLine.Parameters(description = "ID of the pipeline to deactivate") private String pipelineId; @Override - public Void call() throws Exception { - try (ScmClient scmClient = parent.getParent().createScmClient()) { - scmClient.deactivatePipeline( - HddsProtos.PipelineID.newBuilder().setId(pipelineId).build()); - return null; - } + public void execute(ScmClient scmClient) throws IOException { + scmClient.deactivatePipeline( + HddsProtos.PipelineID.newBuilder().setId(pipelineId).build()); } } diff --git a/hadoop-hdds/tools/src/main/java/org/apache/hadoop/hdds/scm/cli/pipeline/ListPipelinesSubcommand.java b/hadoop-hdds/tools/src/main/java/org/apache/hadoop/hdds/scm/cli/pipeline/ListPipelinesSubcommand.java index 729daeae56bc..58ae26e500e1 100644 --- a/hadoop-hdds/tools/src/main/java/org/apache/hadoop/hdds/scm/cli/pipeline/ListPipelinesSubcommand.java +++ b/hadoop-hdds/tools/src/main/java/org/apache/hadoop/hdds/scm/cli/pipeline/ListPipelinesSubcommand.java @@ -1,4 +1,4 @@ -/** +/* * Licensed to the Apache Software Foundation (ASF) under one * or more contributor license agreements. See the NOTICE file * distributed with this work for additional information @@ -20,11 +20,12 @@ import com.google.common.base.Strings; import org.apache.hadoop.hdds.cli.HddsVersionProvider; +import org.apache.hadoop.hdds.scm.cli.ScmSubcommand; import org.apache.hadoop.hdds.scm.client.ScmClient; import org.apache.hadoop.hdds.scm.pipeline.Pipeline; import picocli.CommandLine; -import java.util.concurrent.Callable; +import java.io.IOException; import java.util.stream.Stream; /** @@ -35,38 +36,29 @@ description = "List all active pipelines", mixinStandardHelpOptions = true, versionProvider = HddsVersionProvider.class) -public class ListPipelinesSubcommand implements Callable { - - @CommandLine.ParentCommand - private PipelineCommands parent; +public class ListPipelinesSubcommand extends ScmSubcommand { @CommandLine.Option(names = {"-ffc", "--filterByFactor"}, description = "Filter listed pipelines by Factor(ONE/one)", - defaultValue = "", - required = false) + defaultValue = "") private String factor; @CommandLine.Option(names = {"-fst", "--filterByState"}, description = "Filter listed pipelines by State(OPEN/CLOSE)", - defaultValue = "", - required = false) + defaultValue = "") private String state; - @Override - public Void call() throws Exception { - try (ScmClient scmClient = parent.getParent().createScmClient()) { - Stream stream = scmClient.listPipelines().stream(); - if (!Strings.isNullOrEmpty(factor)) { - stream = stream.filter( - p -> p.getFactor().toString().compareToIgnoreCase(factor) == 0); - } - if (!Strings.isNullOrEmpty(state)) { - stream = stream.filter(p -> p.getPipelineState().toString() - .compareToIgnoreCase(state) == 0); - } - stream.forEach(System.out::println); - return null; + public void execute(ScmClient scmClient) throws IOException { + Stream stream = scmClient.listPipelines().stream(); + if (!Strings.isNullOrEmpty(factor)) { + stream = stream.filter( + p -> p.getFactor().toString().compareToIgnoreCase(factor) == 0); + } + if (!Strings.isNullOrEmpty(state)) { + stream = stream.filter(p -> p.getPipelineState().toString() + .compareToIgnoreCase(state) == 0); } + stream.forEach(System.out::println); } } diff --git a/hadoop-hdds/tools/src/main/java/org/apache/hadoop/hdds/scm/cli/pipeline/PipelineCommands.java b/hadoop-hdds/tools/src/main/java/org/apache/hadoop/hdds/scm/cli/pipeline/PipelineCommands.java index d5c0234d01f4..6d5d3ff5f303 100644 --- a/hadoop-hdds/tools/src/main/java/org/apache/hadoop/hdds/scm/cli/pipeline/PipelineCommands.java +++ b/hadoop-hdds/tools/src/main/java/org/apache/hadoop/hdds/scm/cli/pipeline/PipelineCommands.java @@ -1,4 +1,4 @@ -/** +/* * Licensed to the Apache Software Foundation (ASF) under one * or more contributor license agreements. See the NOTICE file * distributed with this work for additional information @@ -21,11 +21,9 @@ import org.apache.hadoop.hdds.cli.GenericCli; import org.apache.hadoop.hdds.cli.HddsVersionProvider; -import org.apache.hadoop.hdds.scm.cli.container.WithScmClient; import picocli.CommandLine.Command; import picocli.CommandLine.Model.CommandSpec; -import picocli.CommandLine.ParentCommand; import picocli.CommandLine.Spec; /** @@ -48,13 +46,6 @@ public class PipelineCommands implements Callable { @Spec private CommandSpec spec; - @ParentCommand - private WithScmClient parent; - - public WithScmClient getParent() { - return parent; - } - @Override public Void call() throws Exception { GenericCli.missingSubcommand(spec); diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/shell/TestOzoneDatanodeShell.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/shell/TestOzoneDatanodeShell.java index c2ed02ee4914..d52dd33f570f 100644 --- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/shell/TestOzoneDatanodeShell.java +++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/shell/TestOzoneDatanodeShell.java @@ -130,7 +130,7 @@ public void testDatanodeCommand() { @Test public void testDatanodeInvalidParamCommand() { LOG.info("Running testDatanodeIncompleteCommand"); - String expectedError = "Unknown option: -invalidParam"; + String expectedError = "Unknown option: '-invalidParam'"; //executing 'ozone datanode -invalidParam' String[] args = new String[]{"-invalidParam"}; diff --git a/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/admin/OzoneAdmin.java b/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/admin/OzoneAdmin.java index 81f8f649b694..5e0ae76d7aa3 100644 --- a/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/admin/OzoneAdmin.java +++ b/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/admin/OzoneAdmin.java @@ -17,35 +17,24 @@ */ package org.apache.hadoop.ozone.admin; -import java.io.IOException; - -import org.apache.hadoop.hdds.HddsUtils; import org.apache.hadoop.hdds.cli.GenericCli; import org.apache.hadoop.hdds.cli.HddsVersionProvider; -import org.apache.hadoop.hdds.conf.MutableConfigurationSource; import org.apache.hadoop.hdds.conf.OzoneConfiguration; -import org.apache.hadoop.hdds.scm.ScmConfigKeys; -import org.apache.hadoop.hdds.scm.cli.ContainerOperationClient; import org.apache.hadoop.hdds.scm.cli.ReplicationManagerCommands; import org.apache.hadoop.hdds.scm.cli.SafeModeCommands; import org.apache.hadoop.hdds.scm.cli.TopologySubcommand; import org.apache.hadoop.hdds.scm.cli.container.ContainerCommands; -import org.apache.hadoop.hdds.scm.cli.container.WithScmClient; import org.apache.hadoop.hdds.scm.cli.datanode.DatanodeCommands; import org.apache.hadoop.hdds.scm.cli.pipeline.PipelineCommands; -import org.apache.hadoop.hdds.scm.client.ScmClient; import org.apache.hadoop.ozone.admin.om.OMAdmin; import org.apache.hadoop.util.NativeCodeLoader; -import org.apache.commons.lang3.StringUtils; -import static org.apache.hadoop.hdds.scm.ScmConfigKeys.OZONE_SCM_CLIENT_ADDRESS_KEY; import org.apache.log4j.ConsoleAppender; import org.apache.log4j.Level; import org.apache.log4j.LogManager; import org.apache.log4j.Logger; import org.apache.log4j.PatternLayout; import picocli.CommandLine; -import picocli.CommandLine.Option; /** * Ozone Admin Command line tool. @@ -64,13 +53,10 @@ ReplicationManagerCommands.class }, mixinStandardHelpOptions = true) -public class OzoneAdmin extends GenericCli implements WithScmClient { +public class OzoneAdmin extends GenericCli { private OzoneConfiguration ozoneConf; - @Option(names = {"--scm"}, description = "The destination scm (host:port)") - private String scm = ""; - public OzoneConfiguration getOzoneConf() { if (ozoneConf == null) { ozoneConf = createOzoneConfiguration(); @@ -93,28 +79,4 @@ public static void main(String[] argv) throws Exception { new OzoneAdmin().run(argv); } - - public ScmClient createScmClient() { - try { - OzoneConfiguration conf = createOzoneConfiguration(); - checkAndSetSCMAddressArg(conf); - - return new ContainerOperationClient(conf); - } catch (IOException ex) { - throw new IllegalArgumentException("Can't create SCM client", ex); - } - } - - private void checkAndSetSCMAddressArg(MutableConfigurationSource conf) { - if (StringUtils.isNotEmpty(scm)) { - conf.set(OZONE_SCM_CLIENT_ADDRESS_KEY, scm); - } - if (!HddsUtils.getHostNameFromConfigKeys(conf, - ScmConfigKeys.OZONE_SCM_CLIENT_ADDRESS_KEY).isPresent()) { - - throw new IllegalArgumentException( - ScmConfigKeys.OZONE_SCM_CLIENT_ADDRESS_KEY - + " should be set in ozone-site.xml or with the --scm option"); - } - } } diff --git a/hadoop-ozone/tools/src/test/java/org/apache/hadoop/ozone/genconf/TestGenerateOzoneRequiredConfigurations.java b/hadoop-ozone/tools/src/test/java/org/apache/hadoop/ozone/genconf/TestGenerateOzoneRequiredConfigurations.java index 9279d7f226d3..e520190e4c95 100644 --- a/hadoop-ozone/tools/src/test/java/org/apache/hadoop/ozone/genconf/TestGenerateOzoneRequiredConfigurations.java +++ b/hadoop-ozone/tools/src/test/java/org/apache/hadoop/ozone/genconf/TestGenerateOzoneRequiredConfigurations.java @@ -143,7 +143,8 @@ public List handleExecutionException(ExecutionException ex, cmd.parseWithHandlers(new CommandLine.RunLast(), exceptionHandler, args); }catch(Exception ex){ - Assert.assertTrue(ex.getMessage().contains(msg)); + Assert.assertTrue("Expected " + msg + ", but got: " + ex.getMessage(), + ex.getMessage().contains(msg)); } } @@ -225,7 +226,7 @@ public void genconfFailureByInvalidPath() throws Exception { public void genconfPathNotSpecified() throws Exception { File tempPath = getRandomTempDir(); String[] args = new String[]{}; - executeWithException(args, "Missing required parameter: "); + executeWithException(args, "Missing required parameter: ''"); } /** diff --git a/pom.xml b/pom.xml index 969e5c64e3d5..5f1fdf48ce4c 100644 --- a/pom.xml +++ b/pom.xml @@ -260,7 +260,7 @@ xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xs info.picocli picocli - 3.9.6 + 4.4.0 jdiff From fab61e37d0629376bbacc68b5417644511e36435 Mon Sep 17 00:00:00 2001 From: "Doroszlai, Attila" Date: Fri, 31 Jul 2020 20:06:49 +0200 Subject: [PATCH 2/9] HDDS-4056. Improve admin pipeline test --- .../main/smoketest/admincli/pipeline.robot | 39 ++++++++++++++++--- 1 file changed, 34 insertions(+), 5 deletions(-) diff --git a/hadoop-ozone/dist/src/main/smoketest/admincli/pipeline.robot b/hadoop-ozone/dist/src/main/smoketest/admincli/pipeline.robot index b514ae7b07ad..b8ffd9836e67 100644 --- a/hadoop-ozone/dist/src/main/smoketest/admincli/pipeline.robot +++ b/hadoop-ozone/dist/src/main/smoketest/admincli/pipeline.robot @@ -21,14 +21,43 @@ Resource ../commonlib.robot Test Timeout 5 minutes *** Variables *** - +${PIPELINE} *** Test Cases *** -Run list pipeline +Create pipeline + ${output} = Execute ozone admin pipeline create + Should contain ${output} is created. Factor: ONE, Type: STAND_ALONE + ${pipeline} = Execute echo "${output}" | grep 'is created' | cut -f1 -d' ' | cut -f2 -d'=' + Set Suite Variable ${PIPELINE} ${pipeline} + +List pipeline ${output} = Execute ozone admin pipeline list Should contain ${output} Type: Should contain ${output} Factor:ONE, State: -Run create pipeline - ${output} = Execute ozone admin pipeline create - Should contain ${output} is created. Factor: ONE, Type: STAND_ALONE \ No newline at end of file +Deactivate pipeline + Execute ozone admin pipeline deactivate "${PIPELINE}" + ${output} = Execute ozone admin pipeline list | grep "${PIPELINE}" + Should contain ${output} DORMANT + +Activate pipeline + Execute ozone admin pipeline activate "${PIPELINE}" + ${output} = Execute ozone admin pipeline list | grep "${PIPELINE}" + Should contain ${output} OPEN + +Close pipeline + Execute ozone admin pipeline close "${PIPELINE}" + ${output} = Execute ozone admin pipeline list | grep "${PIPELINE}" + Should contain ${output} CLOSED + +Incomplete command + ${output} = Execute And Ignore Error ozone admin pipeline + Should contain ${output} Incomplete command + Should contain ${output} close + Should contain ${output} create + Should contain ${output} deactivate + Should contain ${output} list + +List pipelines on unknown host + ${output} = Execute And Ignore Error ozone admin --verbose pipeline list --scm unknown-host + Should contain ${output} Invalid host name From 1d4d1b65c469637c2e52e26aabb5e67ba7f1f260 Mon Sep 17 00:00:00 2001 From: "Doroszlai, Attila" Date: Mon, 3 Aug 2020 06:31:04 +0200 Subject: [PATCH 3/9] HDDS-4056. TopologySubcommand extends ScmSubcommand --- .../hdds/scm/cli/TopologySubcommand.java | 37 +++++++------------ 1 file changed, 13 insertions(+), 24 deletions(-) diff --git a/hadoop-hdds/tools/src/main/java/org/apache/hadoop/hdds/scm/cli/TopologySubcommand.java b/hadoop-hdds/tools/src/main/java/org/apache/hadoop/hdds/scm/cli/TopologySubcommand.java index 3efb2b3ebbb8..eb39cbd23a27 100644 --- a/hadoop-hdds/tools/src/main/java/org/apache/hadoop/hdds/scm/cli/TopologySubcommand.java +++ b/hadoop-hdds/tools/src/main/java/org/apache/hadoop/hdds/scm/cli/TopologySubcommand.java @@ -18,13 +18,13 @@ package org.apache.hadoop.hdds.scm.cli; +import java.io.IOException; import java.util.ArrayList; import java.util.Collection; import java.util.Collections; import java.util.HashMap; import java.util.List; import java.util.TreeSet; -import java.util.concurrent.Callable; import org.apache.hadoop.hdds.cli.HddsVersionProvider; import org.apache.hadoop.hdds.protocol.DatanodeDetails; @@ -37,8 +37,6 @@ import static org.apache.hadoop.hdds.protocol.proto.HddsProtos.NodeState.HEALTHY; import static org.apache.hadoop.hdds.protocol.proto.HddsProtos.NodeState.STALE; import picocli.CommandLine; -import picocli.CommandLine.Model.CommandSpec; -import picocli.CommandLine.Spec; /** * Handler of printTopology command. @@ -48,13 +46,7 @@ description = "Print a tree of the network topology as reported by SCM", mixinStandardHelpOptions = true, versionProvider = HddsVersionProvider.class) -public class TopologySubcommand implements Callable { - - @Spec - private CommandSpec spec; - - @CommandLine.Mixin - private ScmOption scmOption; +public class TopologySubcommand extends ScmSubcommand { private static final List STATES = new ArrayList<>(); @@ -75,22 +67,19 @@ public class TopologySubcommand implements Callable { private boolean fullInfo; @Override - public Void call() throws Exception { - try (ScmClient scmClient = scmOption.createScmClient()) { - for (HddsProtos.NodeState state : STATES) { - List nodes = scmClient.queryNode(state, - HddsProtos.QueryScope.CLUSTER, ""); - if (nodes != null && !nodes.isEmpty()) { - // show node state - System.out.println("State = " + state.toString()); - if (order) { - printOrderedByLocation(nodes); - } else { - printNodesWithLocation(nodes); - } + protected void execute(ScmClient scmClient) throws IOException { + for (HddsProtos.NodeState state : STATES) { + List nodes = scmClient.queryNode(state, + HddsProtos.QueryScope.CLUSTER, ""); + if (nodes != null && !nodes.isEmpty()) { + // show node state + System.out.println("State = " + state.toString()); + if (order) { + printOrderedByLocation(nodes); + } else { + printNodesWithLocation(nodes); } } - return null; } } From 62e639ed6e003dccf55d6291335687f9b54db637 Mon Sep 17 00:00:00 2001 From: "Doroszlai, Attila" Date: Mon, 3 Aug 2020 06:32:26 +0200 Subject: [PATCH 4/9] Fix leftover javadoc --- .../hadoop/hdds/scm/cli/ReplicationManagerStartSubcommand.java | 2 +- .../hadoop/hdds/scm/cli/ReplicationManagerStatusSubcommand.java | 2 +- .../hadoop/hdds/scm/cli/ReplicationManagerStopSubcommand.java | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/hadoop-hdds/tools/src/main/java/org/apache/hadoop/hdds/scm/cli/ReplicationManagerStartSubcommand.java b/hadoop-hdds/tools/src/main/java/org/apache/hadoop/hdds/scm/cli/ReplicationManagerStartSubcommand.java index 3b55ce5f2eeb..ff82b82ec87a 100644 --- a/hadoop-hdds/tools/src/main/java/org/apache/hadoop/hdds/scm/cli/ReplicationManagerStartSubcommand.java +++ b/hadoop-hdds/tools/src/main/java/org/apache/hadoop/hdds/scm/cli/ReplicationManagerStartSubcommand.java @@ -26,7 +26,7 @@ import java.io.IOException; /** - * This is the handler that process safe mode check command. + * Handler to start replication manager. */ @Command( name = "start", diff --git a/hadoop-hdds/tools/src/main/java/org/apache/hadoop/hdds/scm/cli/ReplicationManagerStatusSubcommand.java b/hadoop-hdds/tools/src/main/java/org/apache/hadoop/hdds/scm/cli/ReplicationManagerStatusSubcommand.java index c735894b0686..c6800befd8cd 100644 --- a/hadoop-hdds/tools/src/main/java/org/apache/hadoop/hdds/scm/cli/ReplicationManagerStatusSubcommand.java +++ b/hadoop-hdds/tools/src/main/java/org/apache/hadoop/hdds/scm/cli/ReplicationManagerStatusSubcommand.java @@ -26,7 +26,7 @@ import java.io.IOException; /** - * This is the handler that process safe mode check command. + * Handler to query status of replication manager. */ @Command( name = "status", diff --git a/hadoop-hdds/tools/src/main/java/org/apache/hadoop/hdds/scm/cli/ReplicationManagerStopSubcommand.java b/hadoop-hdds/tools/src/main/java/org/apache/hadoop/hdds/scm/cli/ReplicationManagerStopSubcommand.java index 17c9034d2212..7d3063a7636c 100644 --- a/hadoop-hdds/tools/src/main/java/org/apache/hadoop/hdds/scm/cli/ReplicationManagerStopSubcommand.java +++ b/hadoop-hdds/tools/src/main/java/org/apache/hadoop/hdds/scm/cli/ReplicationManagerStopSubcommand.java @@ -26,7 +26,7 @@ import java.io.IOException; /** - * This is the handler that process safe mode check command. + * Handler to stop replication manager. */ @Command( name = "stop", From 3b598a4e9880cc3b4a8afcfb6ad075135abc9d50 Mon Sep 17 00:00:00 2001 From: "Doroszlai, Attila" Date: Tue, 11 Aug 2020 11:58:52 +0200 Subject: [PATCH 5/9] HDDS-4056. Convert OzoneAdmin subcommands to META-INF-based --- hadoop-hdds/tools/pom.xml | 8 +++++++ .../apache/hadoop/hdds/cli}/OzoneAdmin.java | 23 +++---------------- .../scm/cli/ReplicationManagerCommands.java | 12 +++++++++- .../hadoop/hdds/scm/cli/SafeModeCommands.java | 11 ++++++++- .../hdds/scm/cli/TopologySubcommand.java | 13 ++++++++++- .../scm/cli/container/ContainerCommands.java | 11 ++++++++- .../scm/cli/datanode/DatanodeCommands.java | 11 ++++++++- .../scm/cli/pipeline/PipelineCommands.java | 11 ++++++++- hadoop-ozone/dist/src/shell/ozone/ozone | 2 +- hadoop-ozone/tools/pom.xml | 2 -- .../apache/hadoop/ozone/admin/om/OMAdmin.java | 2 +- pom.xml | 6 +++++ 12 files changed, 82 insertions(+), 30 deletions(-) rename {hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/admin => hadoop-hdds/tools/src/main/java/org/apache/hadoop/hdds/cli}/OzoneAdmin.java (71%) diff --git a/hadoop-hdds/tools/pom.xml b/hadoop-hdds/tools/pom.xml index f362a0bfea61..fcc553fb4306 100644 --- a/hadoop-hdds/tools/pom.xml +++ b/hadoop-hdds/tools/pom.xml @@ -66,6 +66,14 @@ https://maven.apache.org/xsd/maven-4.0.0.xsd"> commons-cli commons-cli + + log4j + log4j + + + org.kohsuke.metainf-services + metainf-services + org.xerial sqlite-jdbc diff --git a/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/admin/OzoneAdmin.java b/hadoop-hdds/tools/src/main/java/org/apache/hadoop/hdds/cli/OzoneAdmin.java similarity index 71% rename from hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/admin/OzoneAdmin.java rename to hadoop-hdds/tools/src/main/java/org/apache/hadoop/hdds/cli/OzoneAdmin.java index 4dd22d49c765..aca8a4cf8d79 100644 --- a/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/admin/OzoneAdmin.java +++ b/hadoop-hdds/tools/src/main/java/org/apache/hadoop/hdds/cli/OzoneAdmin.java @@ -1,4 +1,4 @@ -/** +/* * Licensed to the Apache Software Foundation (ASF) under one * or more contributor license agreements. See the NOTICE file * distributed with this work for additional information @@ -15,17 +15,9 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.apache.hadoop.ozone.admin; +package org.apache.hadoop.hdds.cli; -import org.apache.hadoop.hdds.cli.GenericCli; -import org.apache.hadoop.hdds.cli.HddsVersionProvider; import org.apache.hadoop.hdds.conf.OzoneConfiguration; -import org.apache.hadoop.hdds.scm.cli.ReplicationManagerCommands; -import org.apache.hadoop.hdds.scm.cli.SafeModeCommands; -import org.apache.hadoop.hdds.scm.cli.TopologySubcommand; -import org.apache.hadoop.hdds.scm.cli.container.ContainerCommands; -import org.apache.hadoop.hdds.scm.cli.datanode.DatanodeCommands; -import org.apache.hadoop.hdds.scm.cli.pipeline.PipelineCommands; import org.apache.hadoop.util.NativeCodeLoader; import org.apache.log4j.ConsoleAppender; @@ -42,14 +34,6 @@ hidden = true, description = "Developer tools for Ozone Admin operations", versionProvider = HddsVersionProvider.class, - subcommands = { - SafeModeCommands.class, - ContainerCommands.class, - PipelineCommands.class, - DatanodeCommands.class, - TopologySubcommand.class, - ReplicationManagerCommands.class - }, mixinStandardHelpOptions = true) public class OzoneAdmin extends GenericCli { @@ -70,9 +54,8 @@ public OzoneConfiguration getOzoneConf() { * Main for the Ozone Admin shell Command handling. * * @param argv - System Args Strings[] - * @throws Exception */ - public static void main(String[] argv) throws Exception { + public static void main(String[] argv) { LogManager.resetConfiguration(); Logger.getRootLogger().setLevel(Level.INFO); Logger.getRootLogger() diff --git a/hadoop-hdds/tools/src/main/java/org/apache/hadoop/hdds/scm/cli/ReplicationManagerCommands.java b/hadoop-hdds/tools/src/main/java/org/apache/hadoop/hdds/scm/cli/ReplicationManagerCommands.java index ca16a32f3cd3..cd5aba3a82e2 100644 --- a/hadoop-hdds/tools/src/main/java/org/apache/hadoop/hdds/scm/cli/ReplicationManagerCommands.java +++ b/hadoop-hdds/tools/src/main/java/org/apache/hadoop/hdds/scm/cli/ReplicationManagerCommands.java @@ -21,7 +21,10 @@ import org.apache.hadoop.hdds.cli.GenericCli; import org.apache.hadoop.hdds.cli.HddsVersionProvider; +import org.apache.hadoop.hdds.cli.OzoneAdmin; +import org.apache.hadoop.hdds.cli.SubcommandWithParent; +import org.kohsuke.MetaInfServices; import picocli.CommandLine.Command; import picocli.CommandLine.Model.CommandSpec; import picocli.CommandLine.Spec; @@ -39,7 +42,9 @@ ReplicationManagerStopSubcommand.class, ReplicationManagerStatusSubcommand.class }) -public class ReplicationManagerCommands implements Callable { +@MetaInfServices(SubcommandWithParent.class) +public class ReplicationManagerCommands implements Callable, + SubcommandWithParent { @Spec private CommandSpec spec; @@ -49,4 +54,9 @@ public Void call() throws Exception { GenericCli.missingSubcommand(spec); return null; } + + @Override + public Class getParentType() { + return OzoneAdmin.class; + } } diff --git a/hadoop-hdds/tools/src/main/java/org/apache/hadoop/hdds/scm/cli/SafeModeCommands.java b/hadoop-hdds/tools/src/main/java/org/apache/hadoop/hdds/scm/cli/SafeModeCommands.java index 96c77c102cbe..6ba7cf295470 100644 --- a/hadoop-hdds/tools/src/main/java/org/apache/hadoop/hdds/scm/cli/SafeModeCommands.java +++ b/hadoop-hdds/tools/src/main/java/org/apache/hadoop/hdds/scm/cli/SafeModeCommands.java @@ -21,7 +21,10 @@ import org.apache.hadoop.hdds.cli.GenericCli; import org.apache.hadoop.hdds.cli.HddsVersionProvider; +import org.apache.hadoop.hdds.cli.OzoneAdmin; +import org.apache.hadoop.hdds.cli.SubcommandWithParent; +import org.kohsuke.MetaInfServices; import picocli.CommandLine.Command; import picocli.CommandLine.Model.CommandSpec; import picocli.CommandLine.Spec; @@ -39,7 +42,8 @@ SafeModeExitSubcommand.class, SafeModeWaitSubcommand.class }) -public class SafeModeCommands implements Callable { +@MetaInfServices(SubcommandWithParent.class) +public class SafeModeCommands implements Callable, SubcommandWithParent { @Spec private CommandSpec spec; @@ -49,4 +53,9 @@ public Void call() throws Exception { GenericCli.missingSubcommand(spec); return null; } + + @Override + public Class getParentType() { + return OzoneAdmin.class; + } } diff --git a/hadoop-hdds/tools/src/main/java/org/apache/hadoop/hdds/scm/cli/TopologySubcommand.java b/hadoop-hdds/tools/src/main/java/org/apache/hadoop/hdds/scm/cli/TopologySubcommand.java index eb39cbd23a27..c1aebaeec22f 100644 --- a/hadoop-hdds/tools/src/main/java/org/apache/hadoop/hdds/scm/cli/TopologySubcommand.java +++ b/hadoop-hdds/tools/src/main/java/org/apache/hadoop/hdds/scm/cli/TopologySubcommand.java @@ -27,6 +27,8 @@ import java.util.TreeSet; import org.apache.hadoop.hdds.cli.HddsVersionProvider; +import org.apache.hadoop.hdds.cli.OzoneAdmin; +import org.apache.hadoop.hdds.cli.SubcommandWithParent; import org.apache.hadoop.hdds.protocol.DatanodeDetails; import org.apache.hadoop.hdds.protocol.proto.HddsProtos; import org.apache.hadoop.hdds.scm.client.ScmClient; @@ -36,6 +38,8 @@ import static org.apache.hadoop.hdds.protocol.proto.HddsProtos.NodeState.DECOMMISSIONING; import static org.apache.hadoop.hdds.protocol.proto.HddsProtos.NodeState.HEALTHY; import static org.apache.hadoop.hdds.protocol.proto.HddsProtos.NodeState.STALE; + +import org.kohsuke.MetaInfServices; import picocli.CommandLine; /** @@ -46,7 +50,9 @@ description = "Print a tree of the network topology as reported by SCM", mixinStandardHelpOptions = true, versionProvider = HddsVersionProvider.class) -public class TopologySubcommand extends ScmSubcommand { +@MetaInfServices(SubcommandWithParent.class) +public class TopologySubcommand extends ScmSubcommand + implements SubcommandWithParent { private static final List STATES = new ArrayList<>(); @@ -66,6 +72,11 @@ public class TopologySubcommand extends ScmSubcommand { description = "Print Topology with full node infos") private boolean fullInfo; + @Override + public Class getParentType() { + return OzoneAdmin.class; + } + @Override protected void execute(ScmClient scmClient) throws IOException { for (HddsProtos.NodeState state : STATES) { diff --git a/hadoop-hdds/tools/src/main/java/org/apache/hadoop/hdds/scm/cli/container/ContainerCommands.java b/hadoop-hdds/tools/src/main/java/org/apache/hadoop/hdds/scm/cli/container/ContainerCommands.java index d1b9d180dec0..de1015d141e7 100644 --- a/hadoop-hdds/tools/src/main/java/org/apache/hadoop/hdds/scm/cli/container/ContainerCommands.java +++ b/hadoop-hdds/tools/src/main/java/org/apache/hadoop/hdds/scm/cli/container/ContainerCommands.java @@ -22,9 +22,12 @@ import org.apache.hadoop.hdds.cli.GenericCli; import org.apache.hadoop.hdds.cli.HddsVersionProvider; +import org.apache.hadoop.hdds.cli.OzoneAdmin; +import org.apache.hadoop.hdds.cli.SubcommandWithParent; import org.apache.hadoop.hdds.scm.client.ScmClient; import org.apache.hadoop.hdds.scm.container.ContainerInfo; +import org.kohsuke.MetaInfServices; import picocli.CommandLine.Command; import picocli.CommandLine.Model.CommandSpec; import picocli.CommandLine.Spec; @@ -44,7 +47,8 @@ CreateSubcommand.class, CloseSubcommand.class }) -public class ContainerCommands implements Callable { +@MetaInfServices(SubcommandWithParent.class) +public class ContainerCommands implements Callable, SubcommandWithParent { @Spec private CommandSpec spec; @@ -55,6 +59,11 @@ public Void call() throws Exception { return null; } + @Override + public Class getParentType() { + return OzoneAdmin.class; + } + public static void checkContainerExists(ScmClient scmClient, long containerId) throws IOException { ContainerInfo container = scmClient.getContainer(containerId); diff --git a/hadoop-hdds/tools/src/main/java/org/apache/hadoop/hdds/scm/cli/datanode/DatanodeCommands.java b/hadoop-hdds/tools/src/main/java/org/apache/hadoop/hdds/scm/cli/datanode/DatanodeCommands.java index da0b87b6437e..7e77c60f6e1e 100644 --- a/hadoop-hdds/tools/src/main/java/org/apache/hadoop/hdds/scm/cli/datanode/DatanodeCommands.java +++ b/hadoop-hdds/tools/src/main/java/org/apache/hadoop/hdds/scm/cli/datanode/DatanodeCommands.java @@ -21,7 +21,10 @@ import org.apache.hadoop.hdds.cli.GenericCli; import org.apache.hadoop.hdds.cli.HddsVersionProvider; +import org.apache.hadoop.hdds.cli.OzoneAdmin; +import org.apache.hadoop.hdds.cli.SubcommandWithParent; +import org.kohsuke.MetaInfServices; import picocli.CommandLine; import picocli.CommandLine.Model.CommandSpec; import picocli.CommandLine.Spec; @@ -37,7 +40,8 @@ subcommands = { ListInfoSubcommand.class }) -public class DatanodeCommands implements Callable { +@MetaInfServices(SubcommandWithParent.class) +public class DatanodeCommands implements Callable, SubcommandWithParent { @Spec private CommandSpec spec; @@ -47,4 +51,9 @@ public Void call() throws Exception { GenericCli.missingSubcommand(spec); return null; } + + @Override + public Class getParentType() { + return OzoneAdmin.class; + } } diff --git a/hadoop-hdds/tools/src/main/java/org/apache/hadoop/hdds/scm/cli/pipeline/PipelineCommands.java b/hadoop-hdds/tools/src/main/java/org/apache/hadoop/hdds/scm/cli/pipeline/PipelineCommands.java index 6d5d3ff5f303..ba7371e6214a 100644 --- a/hadoop-hdds/tools/src/main/java/org/apache/hadoop/hdds/scm/cli/pipeline/PipelineCommands.java +++ b/hadoop-hdds/tools/src/main/java/org/apache/hadoop/hdds/scm/cli/pipeline/PipelineCommands.java @@ -21,7 +21,10 @@ import org.apache.hadoop.hdds.cli.GenericCli; import org.apache.hadoop.hdds.cli.HddsVersionProvider; +import org.apache.hadoop.hdds.cli.OzoneAdmin; +import org.apache.hadoop.hdds.cli.SubcommandWithParent; +import org.kohsuke.MetaInfServices; import picocli.CommandLine.Command; import picocli.CommandLine.Model.CommandSpec; import picocli.CommandLine.Spec; @@ -41,7 +44,8 @@ CreatePipelineSubcommand.class, ClosePipelineSubcommand.class }) -public class PipelineCommands implements Callable { +@MetaInfServices(SubcommandWithParent.class) +public class PipelineCommands implements Callable, SubcommandWithParent { @Spec private CommandSpec spec; @@ -51,4 +55,9 @@ public Void call() throws Exception { GenericCli.missingSubcommand(spec); return null; } + + @Override + public Class getParentType() { + return OzoneAdmin.class; + } } diff --git a/hadoop-ozone/dist/src/shell/ozone/ozone b/hadoop-ozone/dist/src/shell/ozone/ozone index e957f7f39b43..c536484e9b56 100755 --- a/hadoop-ozone/dist/src/shell/ozone/ozone +++ b/hadoop-ozone/dist/src/shell/ozone/ozone @@ -214,7 +214,7 @@ function ozonecmd_case OZONE_RUN_ARTIFACT_NAME="hadoop-ozone-tools" ;; admin) - HADOOP_CLASSNAME=org.apache.hadoop.ozone.admin.OzoneAdmin + HADOOP_CLASSNAME=org.apache.hadoop.hdds.cli.OzoneAdmin OZONE_RUN_ARTIFACT_NAME="hadoop-ozone-tools" ;; debug) diff --git a/hadoop-ozone/tools/pom.xml b/hadoop-ozone/tools/pom.xml index 267e8ef58199..791f7b7363cb 100644 --- a/hadoop-ozone/tools/pom.xml +++ b/hadoop-ozone/tools/pom.xml @@ -112,8 +112,6 @@ https://maven.apache.org/xsd/maven-4.0.0.xsd"> org.kohsuke.metainf-services metainf-services - 1.1 - true com.github.spotbugs diff --git a/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/admin/om/OMAdmin.java b/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/admin/om/OMAdmin.java index ba5fe8154aac..f9321ab5cf2f 100644 --- a/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/admin/om/OMAdmin.java +++ b/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/admin/om/OMAdmin.java @@ -19,12 +19,12 @@ import org.apache.hadoop.hdds.cli.GenericCli; import org.apache.hadoop.hdds.cli.HddsVersionProvider; +import org.apache.hadoop.hdds.cli.OzoneAdmin; import org.apache.hadoop.hdds.cli.SubcommandWithParent; import org.apache.hadoop.hdds.conf.OzoneConfiguration; import org.apache.hadoop.ipc.ProtobufRpcEngine; import org.apache.hadoop.ipc.RPC; import org.apache.hadoop.ozone.OmUtils; -import org.apache.hadoop.ozone.admin.OzoneAdmin; import org.apache.hadoop.ozone.client.OzoneClientException; import org.apache.hadoop.ozone.client.OzoneClientFactory; import org.apache.hadoop.ozone.client.protocol.ClientProtocol; diff --git a/pom.xml b/pom.xml index 5f1fdf48ce4c..9db70430381a 100644 --- a/pom.xml +++ b/pom.xml @@ -1332,6 +1332,12 @@ xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xs hsqldb ${hsqldb.version} + + org.kohsuke.metainf-services + metainf-services + 1.1 + true + io.dropwizard.metrics metrics-core From 5abb2e0eefcbb05c98b6116af487c49dc74a44a8 Mon Sep 17 00:00:00 2001 From: "Doroszlai, Attila" Date: Tue, 11 Aug 2020 13:03:49 +0200 Subject: [PATCH 6/9] HDDS-4056. Add more smoketests --- .../src/main/smoketest/admincli/admin.robot | 32 +++++++++ .../main/smoketest/admincli/container.robot | 67 +++++++++++++++++++ .../main/smoketest/admincli/datanode.robot | 19 ++++-- .../main/smoketest/admincli/pipeline.robot | 12 ++-- .../admincli/replicationmanager.robot | 52 ++++++++++++++ .../main/smoketest/admincli/safemode.robot | 45 +++++++++++++ 6 files changed, 215 insertions(+), 12 deletions(-) create mode 100644 hadoop-ozone/dist/src/main/smoketest/admincli/admin.robot create mode 100644 hadoop-ozone/dist/src/main/smoketest/admincli/container.robot create mode 100644 hadoop-ozone/dist/src/main/smoketest/admincli/replicationmanager.robot create mode 100644 hadoop-ozone/dist/src/main/smoketest/admincli/safemode.robot diff --git a/hadoop-ozone/dist/src/main/smoketest/admincli/admin.robot b/hadoop-ozone/dist/src/main/smoketest/admincli/admin.robot new file mode 100644 index 000000000000..a28888b23f4b --- /dev/null +++ b/hadoop-ozone/dist/src/main/smoketest/admincli/admin.robot @@ -0,0 +1,32 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +*** Settings *** +Documentation Test ozone admin command +Library BuiltIn +Resource ../commonlib.robot +Test Timeout 5 minutes + +*** Test Cases *** +Incomplete command + ${output} = Execute And Ignore Error ozone admin + Should contain ${output} Incomplete command + Should contain ${output} container + Should contain ${output} datanode + Should contain ${output} om + Should contain ${output} pipeline + Should contain ${output} replicationmanager + Should contain ${output} safemode + Should contain ${output} printTopology diff --git a/hadoop-ozone/dist/src/main/smoketest/admincli/container.robot b/hadoop-ozone/dist/src/main/smoketest/admincli/container.robot new file mode 100644 index 000000000000..6ea45628419c --- /dev/null +++ b/hadoop-ozone/dist/src/main/smoketest/admincli/container.robot @@ -0,0 +1,67 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +*** Settings *** +Documentation Test ozone admin container command +Library BuiltIn +Resource ../commonlib.robot +Test Timeout 5 minutes +Suite Setup Create test data + +*** Variables *** +${CONTAINER} + +*** Keywords *** +Create test data + Execute ozone freon ockg -n1 -t1 -p container + +*** Test Cases *** +Create container + ${output} = Execute ozone admin container create + Should contain ${output} is created + ${container} = Execute echo "${output}" | grep 'is created' | cut -f2 -d' ' + Set Suite Variable ${CONTAINER} ${container} + +List containers + ${output} = Execute ozone admin container list + Should contain ${output} OPEN + +List containers with explicit host + ${output} = Execute ozone admin container list --scm scm + Should contain ${output} OPEN + +Container info + ${output} = Execute ozone admin container info "${CONTAINER}" + Should contain ${output} Container id: ${CONTAINER} + Should contain ${output} Datanodes + +Close container + Execute ozone admin container close "${CONTAINER}" + ${output} = Execute ozone admin container info "${CONTAINER}" + Should contain ${output} CLOS + +Incomplete command + ${output} = Execute And Ignore Error ozone admin container + Should contain ${output} Incomplete command + Should contain ${output} list + Should contain ${output} info + Should contain ${output} delete + Should contain ${output} create + Should contain ${output} close + +List containers on unknown host + ${output} = Execute And Ignore Error ozone admin --verbose container list --scm unknown-host + Should contain ${output} Invalid host name + diff --git a/hadoop-ozone/dist/src/main/smoketest/admincli/datanode.robot b/hadoop-ozone/dist/src/main/smoketest/admincli/datanode.robot index cb16bc8bc86a..b34f3af6255a 100644 --- a/hadoop-ozone/dist/src/main/smoketest/admincli/datanode.robot +++ b/hadoop-ozone/dist/src/main/smoketest/admincli/datanode.robot @@ -14,17 +14,22 @@ # limitations under the License. *** Settings *** -Documentation Smoketest ozone cluster startup -Library OperatingSystem +Documentation Test ozone admin datanode command Library BuiltIn Resource ../commonlib.robot Test Timeout 5 minutes -*** Variables *** - - *** Test Cases *** -Run list datanodes +List datanodes ${output} = Execute ozone admin datanode list Should contain ${output} Datanode: - Should contain ${output} Related pipelines: \ No newline at end of file + Should contain ${output} Related pipelines: + +Incomplete command + ${output} = Execute And Ignore Error ozone admin datanode + Should contain ${output} Incomplete command + Should contain ${output} list + +List datanodes on unknown host + ${output} = Execute And Ignore Error ozone admin --verbose datanode list --scm unknown-host + Should contain ${output} Invalid host name diff --git a/hadoop-ozone/dist/src/main/smoketest/admincli/pipeline.robot b/hadoop-ozone/dist/src/main/smoketest/admincli/pipeline.robot index b8ffd9836e67..3a97f8394977 100644 --- a/hadoop-ozone/dist/src/main/smoketest/admincli/pipeline.robot +++ b/hadoop-ozone/dist/src/main/smoketest/admincli/pipeline.robot @@ -14,8 +14,7 @@ # limitations under the License. *** Settings *** -Documentation Smoketest ozone cluster startup -Library OperatingSystem +Documentation Test ozone admin pipeline command Library BuiltIn Resource ../commonlib.robot Test Timeout 5 minutes @@ -30,10 +29,13 @@ Create pipeline ${pipeline} = Execute echo "${output}" | grep 'is created' | cut -f1 -d' ' | cut -f2 -d'=' Set Suite Variable ${PIPELINE} ${pipeline} -List pipeline +List pipelines ${output} = Execute ozone admin pipeline list - Should contain ${output} Type: - Should contain ${output} Factor:ONE, State: + Should contain ${output} Factor:ONE + +List pipelines with explicit host + ${output} = Execute ozone admin pipeline list --scm scm + Should contain ${output} Factor:ONE Deactivate pipeline Execute ozone admin pipeline deactivate "${PIPELINE}" diff --git a/hadoop-ozone/dist/src/main/smoketest/admincli/replicationmanager.robot b/hadoop-ozone/dist/src/main/smoketest/admincli/replicationmanager.robot new file mode 100644 index 000000000000..58e04dfd1790 --- /dev/null +++ b/hadoop-ozone/dist/src/main/smoketest/admincli/replicationmanager.robot @@ -0,0 +1,52 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +*** Settings *** +Documentation Test ozone admin replicationmanager command +Library BuiltIn +Resource ../commonlib.robot +Test Timeout 5 minutes + +*** Test Cases *** +Check replicationmanager + ${output} = Execute ozone admin replicationmanager status + Should contain ${output} ReplicationManager is Not Running + +Check replicationmanager with explicit host + ${output} = Execute ozone admin replicationmanager status --scm scm + Should contain ${output} ReplicationManager + Should contain ${output} Running + +Start replicationmanager + ${output} = Execute ozone admin replicationmanager start + Should contain ${output} Starting ReplicationManager + Wait Until Keyword Succeeds 30sec 5sec Execute ozone admin replicationmanager status | grep -q 'is Running' + +Stop replicationmanager + ${output} = Execute ozone admin replicationmanager stop + Should contain ${output} Stopping ReplicationManager + Wait Until Keyword Succeeds 30sec 5sec Execute ozone admin replicationmanager status | grep -q 'is Not Running' + +Incomplete command + ${output} = Execute And Ignore Error ozone admin replicationmanager + Should contain ${output} Incomplete command + Should contain ${output} start + Should contain ${output} stop + Should contain ${output} status + +Check replicationmanager on unknown host + ${output} = Execute And Ignore Error ozone admin --verbose replicationmanager status --scm unknown-host + Should contain ${output} Invalid host name + diff --git a/hadoop-ozone/dist/src/main/smoketest/admincli/safemode.robot b/hadoop-ozone/dist/src/main/smoketest/admincli/safemode.robot new file mode 100644 index 000000000000..114d846e0e07 --- /dev/null +++ b/hadoop-ozone/dist/src/main/smoketest/admincli/safemode.robot @@ -0,0 +1,45 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +*** Settings *** +Documentation Test ozone admin safemode command +Library BuiltIn +Resource ../commonlib.robot +Test Timeout 5 minutes + +*** Test Cases *** +Check safemode + ${output} = Execute ozone admin safemode status + Should contain ${output} SCM is out of safe mode + +Check safemode with explicit host + ${output} = Execute ozone admin safemode status --scm scm + Should contain ${output} SCM is out of safe mode + +Wait for safemode exit + ${output} = Execute ozone admin safemode wait -t 2 + Should contain ${output} SCM is out of safe mode + +Incomplete command + ${output} = Execute And Ignore Error ozone admin safemode + Should contain ${output} Incomplete command + Should contain ${output} status + Should contain ${output} exit + Should contain ${output} wait + +Check safemode on unknown host + ${output} = Execute And Ignore Error ozone admin --verbose safemode status --scm unknown-host + Should contain ${output} Invalid host name + From 7c06f168802ffea7c547a8ba71d53d90900e9e4c Mon Sep 17 00:00:00 2001 From: "Doroszlai, Attila" Date: Tue, 11 Aug 2020 13:09:47 +0200 Subject: [PATCH 7/9] HDDS-4056. Fix checkstyle --- .../apache/hadoop/hdds/cli/package-info.java | 4 ++-- .../apache/hadoop/hdds/cli/package-info.java | 22 +++++++++++++++++++ 2 files changed, 24 insertions(+), 2 deletions(-) create mode 100644 hadoop-hdds/tools/src/main/java/org/apache/hadoop/hdds/cli/package-info.java diff --git a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/cli/package-info.java b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/cli/package-info.java index 8dcc1d1a3c91..aabad6f14464 100644 --- a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/cli/package-info.java +++ b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/cli/package-info.java @@ -1,4 +1,4 @@ -/** +/* * Licensed to the Apache Software Foundation (ASF) under one * or more contributor license agreements. See the NOTICE file * distributed with this work for additional information @@ -19,4 +19,4 @@ /** * Generic helper class to make instantiate picocli based cli tools. */ -package org.apache.hadoop.hdds.cli; \ No newline at end of file +package org.apache.hadoop.hdds.cli; diff --git a/hadoop-hdds/tools/src/main/java/org/apache/hadoop/hdds/cli/package-info.java b/hadoop-hdds/tools/src/main/java/org/apache/hadoop/hdds/cli/package-info.java new file mode 100644 index 000000000000..82fbd722932e --- /dev/null +++ b/hadoop-hdds/tools/src/main/java/org/apache/hadoop/hdds/cli/package-info.java @@ -0,0 +1,22 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/** + * Command-line tools for HDDS. + */ +package org.apache.hadoop.hdds.cli; From df186da10c9e2e968169bd8488e395fd1a969382 Mon Sep 17 00:00:00 2001 From: "Doroszlai, Attila" Date: Tue, 11 Aug 2020 15:01:42 +0200 Subject: [PATCH 8/9] HDDS-4056. Kinit before freon --- hadoop-ozone/dist/src/main/smoketest/admincli/container.robot | 1 + 1 file changed, 1 insertion(+) diff --git a/hadoop-ozone/dist/src/main/smoketest/admincli/container.robot b/hadoop-ozone/dist/src/main/smoketest/admincli/container.robot index 6ea45628419c..0560880de79d 100644 --- a/hadoop-ozone/dist/src/main/smoketest/admincli/container.robot +++ b/hadoop-ozone/dist/src/main/smoketest/admincli/container.robot @@ -25,6 +25,7 @@ ${CONTAINER} *** Keywords *** Create test data + Run Keyword if '${SECURITY_ENABLED}' == 'true' Kinit test user testuser testuser.keytab Execute ozone freon ockg -n1 -t1 -p container *** Test Cases *** From 1afe288f614350a6fd88f455315a0b9a1c720e25 Mon Sep 17 00:00:00 2001 From: "Doroszlai, Attila" Date: Tue, 11 Aug 2020 16:49:38 +0200 Subject: [PATCH 9/9] HDDS-4056. ReplicationManager may or may not be running at the start of the test --- .../dist/src/main/smoketest/admincli/replicationmanager.robot | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/hadoop-ozone/dist/src/main/smoketest/admincli/replicationmanager.robot b/hadoop-ozone/dist/src/main/smoketest/admincli/replicationmanager.robot index 58e04dfd1790..cef294f1e8d7 100644 --- a/hadoop-ozone/dist/src/main/smoketest/admincli/replicationmanager.robot +++ b/hadoop-ozone/dist/src/main/smoketest/admincli/replicationmanager.robot @@ -22,7 +22,8 @@ Test Timeout 5 minutes *** Test Cases *** Check replicationmanager ${output} = Execute ozone admin replicationmanager status - Should contain ${output} ReplicationManager is Not Running + Should contain ${output} ReplicationManager + Should contain ${output} Running Check replicationmanager with explicit host ${output} = Execute ozone admin replicationmanager status --scm scm