diff --git a/hadoop-hdds/tools/src/main/java/org/apache/hadoop/hdds/scm/cli/container/InfoSubcommand.java b/hadoop-hdds/tools/src/main/java/org/apache/hadoop/hdds/scm/cli/container/InfoSubcommand.java index 9524ce94716f..8ed9f520b29d 100644 --- a/hadoop-hdds/tools/src/main/java/org/apache/hadoop/hdds/scm/cli/container/InfoSubcommand.java +++ b/hadoop-hdds/tools/src/main/java/org/apache/hadoop/hdds/scm/cli/container/InfoSubcommand.java @@ -24,6 +24,7 @@ import java.util.HashMap; import java.util.ArrayList; import java.time.Instant; +import java.util.Scanner; import java.util.stream.Collectors; import org.apache.hadoop.hdds.cli.GenericParentCommand; @@ -73,21 +74,101 @@ public class InfoSubcommand extends ScmSubcommand { description = "Format output as JSON") private boolean json; - @Parameters(description = "Decimal id of the container.") - private long containerID; + @Parameters(description = "One or more container IDs separated by spaces. " + + "To read from stdin, specify '-' and supply the container IDs " + + "separated by newlines.", + arity = "1..*", + paramLabel = "") + private String[] containerList; + + private boolean multiContainer = false; @Override public void execute(ScmClient scmClient) throws IOException { - final ContainerWithPipeline container = scmClient. - getContainerWithPipeline(containerID); - Preconditions.checkNotNull(container, "Container cannot be null"); + boolean first = true; + boolean stdin = false; + if (containerList.length > 1) { + multiContainer = true; + } else if (containerList[0].equals("-")) { + stdin = true; + // Assume multiple containers if reading from stdin + multiContainer = true; + } + + printHeader(); + if (stdin) { + Scanner scanner = new Scanner(System.in, "UTF-8"); + while (scanner.hasNextLine()) { + String id = scanner.nextLine().trim(); + printOutput(scmClient, id, first); + first = false; + } + } else { + for (String id : containerList) { + printOutput(scmClient, id, first); + first = false; + } + } + printFooter(); + } + + private void printOutput(ScmClient scmClient, String id, boolean first) + throws IOException { + long containerID; + try { + containerID = Long.parseLong(id); + } catch (NumberFormatException e) { + printError("Invalid container ID: " + id); + return; + } + printDetails(scmClient, containerID, first); + } + + private void printHeader() { + if (json && multiContainer) { + LOG.info("["); + } + } + + private void printFooter() { + if (json && multiContainer) { + LOG.info("]"); + } + } + + private void printError(String error) { + System.err.println(error); + } + + private void printBreak() { + if (json) { + LOG.info(","); + } else { + LOG.info(""); + } + } + + private void printDetails(ScmClient scmClient, long containerID, + boolean first) throws IOException { + final ContainerWithPipeline container; + try { + container = scmClient.getContainerWithPipeline(containerID); + Preconditions.checkNotNull(container, "Container cannot be null"); + } catch (IOException e) { + printError("Unable to retrieve the container details for " + containerID); + return; + } + List replicas = null; try { replicas = scmClient.getContainerReplicas(containerID); } catch (IOException e) { - LOG.error("Unable to retrieve the replica details", e); + printError("Unable to retrieve the replica details: " + e.getMessage()); } + if (!first) { + printBreak(); + } if (json) { if (container.getPipeline().size() != 0) { ContainerWithPipelineAndReplicas wrapper = @@ -125,14 +206,14 @@ public void execute(ScmClient scmClient) throws IOException { ioe) instanceof PipelineNotFoundException) { LOG.info("Write Pipeline State: CLOSED"); } else { - LOG.error("Failed to retrieve pipeline info"); + printError("Failed to retrieve pipeline info"); } } LOG.info("Container State: {}", container.getContainerInfo().getState()); // Print pipeline of an existing container. String machinesStr = container.getPipeline().getNodes().stream().map( - InfoSubcommand::buildDatanodeDetails) + InfoSubcommand::buildDatanodeDetails) .collect(Collectors.joining(",\n")); LOG.info("Datanodes: [{}]", machinesStr); diff --git a/hadoop-hdds/tools/src/test/java/org/apache/hadoop/hdds/scm/cli/container/TestInfoSubCommand.java b/hadoop-hdds/tools/src/test/java/org/apache/hadoop/hdds/scm/cli/container/TestInfoSubCommand.java index 6058546c97db..9b264312fdf4 100644 --- a/hadoop-hdds/tools/src/test/java/org/apache/hadoop/hdds/scm/cli/container/TestInfoSubCommand.java +++ b/hadoop-hdds/tools/src/test/java/org/apache/hadoop/hdds/scm/cli/container/TestInfoSubCommand.java @@ -29,7 +29,6 @@ import org.apache.hadoop.hdds.scm.pipeline.PipelineID; import org.apache.hadoop.hdds.scm.pipeline.PipelineNotFoundException; import org.apache.log4j.AppenderSkeleton; -import org.apache.log4j.Level; import org.apache.log4j.Logger; import org.apache.log4j.spi.LoggingEvent; import org.junit.jupiter.api.AfterEach; @@ -39,7 +38,13 @@ import org.mockito.Mockito; import picocli.CommandLine; +import java.io.ByteArrayInputStream; +import java.io.ByteArrayOutputStream; import java.io.IOException; +import java.io.InputStream; +import java.io.PrintStream; +import java.io.UnsupportedEncodingException; +import java.nio.charset.StandardCharsets; import java.util.ArrayList; import java.util.List; import java.util.UUID; @@ -49,6 +54,8 @@ import static org.apache.hadoop.hdds.protocol.proto.HddsProtos.LifeCycleState.CLOSED; import static org.apache.hadoop.hdds.protocol.proto.HddsProtos.ReplicationFactor.THREE; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.assertTrue; import static org.mockito.ArgumentMatchers.anyLong; import static org.mockito.ArgumentMatchers.any; import static org.mockito.Mockito.mock; @@ -64,22 +71,39 @@ public class TestInfoSubCommand { private Logger logger; private TestAppender appender; + private final ByteArrayOutputStream outContent = new ByteArrayOutputStream(); + private final ByteArrayOutputStream errContent = new ByteArrayOutputStream(); + private ByteArrayInputStream inContent; + private final PrintStream originalOut = System.out; + private final PrintStream originalErr = System.err; + private final InputStream originalIn = System.in; + + private static final String DEFAULT_ENCODING = StandardCharsets.UTF_8.name(); + @BeforeEach public void setup() throws IOException { scmClient = mock(ScmClient.class); datanodes = createDatanodeDetails(3); Mockito.when(scmClient.getContainerWithPipeline(anyLong())) - .thenReturn(getContainerWithPipeline()); + .then(i -> getContainerWithPipeline(i.getArgument(0))); + Mockito.when(scmClient.getPipeline(any())) + .thenThrow(new PipelineNotFoundException("Pipeline not found.")); appender = new TestAppender(); logger = Logger.getLogger( org.apache.hadoop.hdds.scm.cli.container.InfoSubcommand.class); logger.addAppender(appender); + + System.setOut(new PrintStream(outContent, false, DEFAULT_ENCODING)); + System.setErr(new PrintStream(errContent, false, DEFAULT_ENCODING)); } @AfterEach public void after() { logger.removeAppender(appender); + System.setOut(originalOut); + System.setErr(originalErr); + System.setIn(originalIn); } @Test @@ -92,13 +116,101 @@ public void testReplicaIndexInOutput() throws Exception { testReplicaIncludedInOutput(true); } + @Test + public void testErrorWhenNoContainerIDParam() throws Exception { + cmd = new InfoSubcommand(); + assertThrows(CommandLine.MissingParameterException.class, () -> { + CommandLine c = new CommandLine(cmd); + c.parseArgs(); + cmd.execute(scmClient); + }); + } + + @Test + public void testMultipleContainersCanBePassed() throws Exception { + Mockito.when(scmClient.getContainerReplicas(anyLong())) + .thenReturn(getReplicas(true)); + cmd = new InfoSubcommand(); + CommandLine c = new CommandLine(cmd); + c.parseArgs("1", "123", "456", "invalid", "789"); + cmd.execute(scmClient); + validateMultiOutput(); + } + + @Test + public void testContainersCanBeReadFromStdin() throws IOException { + String input = "1\n123\n456\ninvalid\n789\n"; + inContent = new ByteArrayInputStream(input.getBytes(DEFAULT_ENCODING)); + System.setIn(inContent); + cmd = new InfoSubcommand(); + CommandLine c = new CommandLine(cmd); + c.parseArgs("-"); + cmd.execute(scmClient); + + validateMultiOutput(); + } + + private void validateMultiOutput() throws UnsupportedEncodingException { + // Ensure we have a log line for each containerID + List logs = appender.getLog(); + List replica = logs.stream() + .filter(m -> m.getRenderedMessage() + .matches("(?s)^Container id: (1|123|456|789).*")) + .collect(Collectors.toList()); + Assertions.assertEquals(4, replica.size()); + + Pattern p = Pattern.compile( + "^Invalid\\scontainer\\sID:\\sinvalid.*", Pattern.MULTILINE); + Matcher m = p.matcher(errContent.toString(DEFAULT_ENCODING)); + assertTrue(m.find()); + } + + @Test + public void testContainersCanBeReadFromStdinJson() + throws IOException { + String input = "1\n123\n456\ninvalid\n789\n"; + inContent = new ByteArrayInputStream(input.getBytes(DEFAULT_ENCODING)); + System.setIn(inContent); + cmd = new InfoSubcommand(); + CommandLine c = new CommandLine(cmd); + c.parseArgs("-", "--json"); + cmd.execute(scmClient); + + validateJsonMultiOutput(); + } + + + @Test + public void testMultipleContainersCanBePassedJson() throws Exception { + Mockito.when(scmClient.getContainerReplicas(anyLong())) + .thenReturn(getReplicas(true)); + cmd = new InfoSubcommand(); + CommandLine c = new CommandLine(cmd); + c.parseArgs("1", "123", "456", "invalid", "789", "--json"); + cmd.execute(scmClient); + + validateJsonMultiOutput(); + } + + private void validateJsonMultiOutput() throws UnsupportedEncodingException { + // Ensure we have a log line for each containerID + List logs = appender.getLog(); + List replica = logs.stream() + .filter(m -> m.getRenderedMessage() + .matches("(?s)^.*\"containerInfo\".*")) + .collect(Collectors.toList()); + Assertions.assertEquals(4, replica.size()); + + Pattern p = Pattern.compile( + "^Invalid\\scontainer\\sID:\\sinvalid.*", Pattern.MULTILINE); + Matcher m = p.matcher(errContent.toString(DEFAULT_ENCODING)); + assertTrue(m.find()); + } private void testReplicaIncludedInOutput(boolean includeIndex) throws IOException { Mockito.when(scmClient.getContainerReplicas(anyLong())) .thenReturn(getReplicas(includeIndex)); - Mockito.when(scmClient.getPipeline(any())) - .thenThrow(new PipelineNotFoundException("Pipeline not found.")); cmd = new InfoSubcommand(); CommandLine c = new CommandLine(cmd); c.parseArgs("1"); @@ -139,8 +251,6 @@ private void testReplicaIncludedInOutput(boolean includeIndex) public void testReplicasNotOutputIfError() throws IOException { Mockito.when(scmClient.getContainerReplicas(anyLong())) .thenThrow(new IOException("Error getting Replicas")); - Mockito.when(scmClient.getPipeline(any())) - .thenThrow(new PipelineNotFoundException("Pipeline not found.")); cmd = new InfoSubcommand(); CommandLine c = new CommandLine(cmd); c.parseArgs("1"); @@ -153,13 +263,10 @@ public void testReplicasNotOutputIfError() throws IOException { .collect(Collectors.toList()); Assertions.assertEquals(0, replica.size()); - // Ensure we have an error logged: - List error = logs.stream() - .filter(m -> m.getLevel() == Level.ERROR) - .collect(Collectors.toList()); - Assertions.assertEquals(1, error.size()); - Assertions.assertTrue(error.get(0).getRenderedMessage() - .matches("(?s)^Unable to retrieve the replica details.*")); + Pattern p = Pattern.compile( + "^Unable to retrieve the replica details.*", Pattern.MULTILINE); + Matcher m = p.matcher(errContent.toString(DEFAULT_ENCODING)); + assertTrue(m.find()); } @Test @@ -172,12 +279,9 @@ public void testReplicasNotOutputIfErrorWithJson() throws IOException { cmd.execute(scmClient); List logs = appender.getLog(); - Assertions.assertEquals(2, logs.size()); - String error = logs.get(0).getRenderedMessage(); - String json = logs.get(1).getRenderedMessage(); + Assertions.assertEquals(1, logs.size()); + String json = logs.get(0).getRenderedMessage(); - Assertions.assertTrue(error - .matches("(?s)^Unable to retrieve the replica details.*")); Assertions.assertFalse(json.matches("(?s).*replicas.*")); } @@ -252,7 +356,7 @@ private List getReplicas(boolean includeIndex) { return replicas; } - private ContainerWithPipeline getContainerWithPipeline() { + private ContainerWithPipeline getContainerWithPipeline(long containerID) { Pipeline pipeline = new Pipeline.Builder() .setState(Pipeline.PipelineState.CLOSED) .setReplicationConfig(RatisReplicationConfig.getInstance(THREE)) @@ -261,6 +365,7 @@ private ContainerWithPipeline getContainerWithPipeline() { .build(); ContainerInfo container = new ContainerInfo.Builder() + .setContainerID(containerID) .setSequenceId(1) .setPipelineID(pipeline.getId()) .setUsedBytes(1234)