Skip to content

HDDS-9700. Allow ozone admin container info to list multiple containers #5659

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
adoroszlai merged 6 commits into from
Nov 23, 2023
Merged

HDDS-9700. Allow ozone admin container info to list multiple containers #5659

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
6 commits
Select commit Hold shift + click to select a range
36e759b
Enable multiple IDs passed with spaces on the command line
Nov 15, 2023
1912c88
Changes to ensure correctly formatted JSON
Nov 16, 2023
18aa33c
Read from stdin
Nov 17, 2023
bc61619
Read from stdin only if dash is passed
Nov 21, 2023
cc1aec8
Errors to stdout and some formatting changes
Nov 22, 2023
db531be
Fix findbugs and failing tests caused by rebase
Nov 22, 2023
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
97 changes: 89 additions & 8 deletions hadoop-hdds/tools/src/main/java/org/apache/hadoop/hdds/scm/cli/container/InfoSubcommand.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -24,6 +24,7 @@
import java.util.HashMap;
import java.util.ArrayList;
import java.time.Instant;
import java.util.Scanner;
import java.util.stream.Collectors;

import org.apache.hadoop.hdds.cli.GenericParentCommand;
Expand Down Expand Up @@ -73,21 +74,101 @@ public class InfoSubcommand extends ScmSubcommand {
description = "Format output as JSON")
private boolean json;

@Parameters(description = "Decimal id of the container.")
private long containerID;
@Parameters(description = "One or more container IDs separated by spaces. " +
"To read from stdin, specify '-' and supply the container IDs " +
"separated by newlines.",
arity = "1..*",
paramLabel = "<container ID>")
private String[] containerList;

private boolean multiContainer = false;

@Override
public void execute(ScmClient scmClient) throws IOException {
final ContainerWithPipeline container = scmClient.
getContainerWithPipeline(containerID);
Preconditions.checkNotNull(container, "Container cannot be null");
boolean first = true;
boolean stdin = false;
if (containerList.length > 1) {
multiContainer = true;
} else if (containerList[0].equals("-")) {
stdin = true;
// Assume multiple containers if reading from stdin
multiContainer = true;
}

printHeader();
if (stdin) {
Scanner scanner = new Scanner(System.in, "UTF-8");
while (scanner.hasNextLine()) {
String id = scanner.nextLine().trim();
printOutput(scmClient, id, first);
first = false;
}
} else {
for (String id : containerList) {
printOutput(scmClient, id, first);
first = false;
}
}
printFooter();
}

private void printOutput(ScmClient scmClient, String id, boolean first)
throws IOException {
long containerID;
try {
containerID = Long.parseLong(id);
} catch (NumberFormatException e) {
printError("Invalid container ID: " + id);
return;
}
printDetails(scmClient, containerID, first);
}

private void printHeader() {
if (json && multiContainer) {
LOG.info("[");
}
}

private void printFooter() {
if (json && multiContainer) {
LOG.info("]");
}
}

private void printError(String error) {
System.err.println(error);
}

private void printBreak() {
if (json) {
LOG.info(",");
} else {
LOG.info("");
}
}

private void printDetails(ScmClient scmClient, long containerID,
boolean first) throws IOException {
final ContainerWithPipeline container;
try {
container = scmClient.getContainerWithPipeline(containerID);
Preconditions.checkNotNull(container, "Container cannot be null");
} catch (IOException e) {
printError("Unable to retrieve the container details for " + containerID);
return;
}

List<ContainerReplicaInfo> replicas = null;
try {
replicas = scmClient.getContainerReplicas(containerID);
} catch (IOException e) {
LOG.error("Unable to retrieve the replica details", e);
printError("Unable to retrieve the replica details: " + e.getMessage());
}

if (!first) {
printBreak();
}
if (json) {
if (container.getPipeline().size() != 0) {
ContainerWithPipelineAndReplicas wrapper =
Expand Down Expand Up @@ -125,14 +206,14 @@ public void execute(ScmClient scmClient) throws IOException {
ioe) instanceof PipelineNotFoundException) {
LOG.info("Write Pipeline State: CLOSED");
} else {
LOG.error("Failed to retrieve pipeline info");
printError("Failed to retrieve pipeline info");
}
}
LOG.info("Container State: {}", container.getContainerInfo().getState());

// Print pipeline of an existing container.
String machinesStr = container.getPipeline().getNodes().stream().map(
InfoSubcommand::buildDatanodeDetails)
InfoSubcommand::buildDatanodeDetails)
.collect(Collectors.joining(",\n"));
LOG.info("Datanodes: [{}]", machinesStr);

Expand Down
143 changes: 124 additions & 19 deletions ...hdds/tools/src/test/java/org/apache/hadoop/hdds/scm/cli/container/TestInfoSubCommand.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -29,7 +29,6 @@
import org.apache.hadoop.hdds.scm.pipeline.PipelineID;
import org.apache.hadoop.hdds.scm.pipeline.PipelineNotFoundException;
import org.apache.log4j.AppenderSkeleton;
import org.apache.log4j.Level;
import org.apache.log4j.Logger;
import org.apache.log4j.spi.LoggingEvent;
import org.junit.jupiter.api.AfterEach;
Expand All @@ -39,7 +38,13 @@
import org.mockito.Mockito;
import picocli.CommandLine;

import java.io.ByteArrayInputStream;
import java.io.ByteArrayOutputStream;
import java.io.IOException;
import java.io.InputStream;
import java.io.PrintStream;
import java.io.UnsupportedEncodingException;
import java.nio.charset.StandardCharsets;
import java.util.ArrayList;
import java.util.List;
import java.util.UUID;
Expand All @@ -49,6 +54,8 @@

import static org.apache.hadoop.hdds.protocol.proto.HddsProtos.LifeCycleState.CLOSED;
import static org.apache.hadoop.hdds.protocol.proto.HddsProtos.ReplicationFactor.THREE;
import static org.junit.jupiter.api.Assertions.assertThrows;
import static org.junit.jupiter.api.Assertions.assertTrue;
import static org.mockito.ArgumentMatchers.anyLong;
import static org.mockito.ArgumentMatchers.any;
import static org.mockito.Mockito.mock;
Expand All @@ -64,22 +71,39 @@ public class TestInfoSubCommand {
private Logger logger;
private TestAppender appender;

private final ByteArrayOutputStream outContent = new ByteArrayOutputStream();
private final ByteArrayOutputStream errContent = new ByteArrayOutputStream();
private ByteArrayInputStream inContent;
private final PrintStream originalOut = System.out;
private final PrintStream originalErr = System.err;
private final InputStream originalIn = System.in;

private static final String DEFAULT_ENCODING = StandardCharsets.UTF_8.name();

@BeforeEach
public void setup() throws IOException {
scmClient = mock(ScmClient.class);
datanodes = createDatanodeDetails(3);
Mockito.when(scmClient.getContainerWithPipeline(anyLong()))
.thenReturn(getContainerWithPipeline());
.then(i -> getContainerWithPipeline(i.getArgument(0)));
Mockito.when(scmClient.getPipeline(any()))
.thenThrow(new PipelineNotFoundException("Pipeline not found."));

appender = new TestAppender();
logger = Logger.getLogger(
org.apache.hadoop.hdds.scm.cli.container.InfoSubcommand.class);
logger.addAppender(appender);

System.setOut(new PrintStream(outContent, false, DEFAULT_ENCODING));
System.setErr(new PrintStream(errContent, false, DEFAULT_ENCODING));
}

@AfterEach
public void after() {
logger.removeAppender(appender);
System.setOut(originalOut);
System.setErr(originalErr);
System.setIn(originalIn);
}

@Test
Expand All @@ -92,13 +116,101 @@ public void testReplicaIndexInOutput() throws Exception {
testReplicaIncludedInOutput(true);
}

@Test
public void testErrorWhenNoContainerIDParam() throws Exception {
cmd = new InfoSubcommand();
assertThrows(CommandLine.MissingParameterException.class, () -> {
CommandLine c = new CommandLine(cmd);
c.parseArgs();
cmd.execute(scmClient);
});
}

@Test
public void testMultipleContainersCanBePassed() throws Exception {
Mockito.when(scmClient.getContainerReplicas(anyLong()))
.thenReturn(getReplicas(true));
cmd = new InfoSubcommand();
CommandLine c = new CommandLine(cmd);
c.parseArgs("1", "123", "456", "invalid", "789");
cmd.execute(scmClient);
validateMultiOutput();
}

@Test
public void testContainersCanBeReadFromStdin() throws IOException {
String input = "1\n123\n456\ninvalid\n789\n";
inContent = new ByteArrayInputStream(input.getBytes(DEFAULT_ENCODING));
System.setIn(inContent);
cmd = new InfoSubcommand();
CommandLine c = new CommandLine(cmd);
c.parseArgs("-");
cmd.execute(scmClient);

validateMultiOutput();
}

private void validateMultiOutput() throws UnsupportedEncodingException {
// Ensure we have a log line for each containerID
List<LoggingEvent> logs = appender.getLog();
List<LoggingEvent> replica = logs.stream()
.filter(m -> m.getRenderedMessage()
.matches("(?s)^Container id: (1|123|456|789).*"))
.collect(Collectors.toList());
Assertions.assertEquals(4, replica.size());

Pattern p = Pattern.compile(
"^Invalid\\scontainer\\sID:\\sinvalid.*", Pattern.MULTILINE);
Matcher m = p.matcher(errContent.toString(DEFAULT_ENCODING));
assertTrue(m.find());
}

@Test
public void testContainersCanBeReadFromStdinJson()
throws IOException {
String input = "1\n123\n456\ninvalid\n789\n";
inContent = new ByteArrayInputStream(input.getBytes(DEFAULT_ENCODING));
System.setIn(inContent);
cmd = new InfoSubcommand();
CommandLine c = new CommandLine(cmd);
c.parseArgs("-", "--json");
cmd.execute(scmClient);

validateJsonMultiOutput();
}


@Test
public void testMultipleContainersCanBePassedJson() throws Exception {
Mockito.when(scmClient.getContainerReplicas(anyLong()))
.thenReturn(getReplicas(true));
cmd = new InfoSubcommand();
CommandLine c = new CommandLine(cmd);
c.parseArgs("1", "123", "456", "invalid", "789", "--json");
cmd.execute(scmClient);

validateJsonMultiOutput();
}

private void validateJsonMultiOutput() throws UnsupportedEncodingException {
// Ensure we have a log line for each containerID
List<LoggingEvent> logs = appender.getLog();
List<LoggingEvent> replica = logs.stream()
.filter(m -> m.getRenderedMessage()
.matches("(?s)^.*\"containerInfo\".*"))
.collect(Collectors.toList());
Assertions.assertEquals(4, replica.size());

Pattern p = Pattern.compile(
"^Invalid\\scontainer\\sID:\\sinvalid.*", Pattern.MULTILINE);
Matcher m = p.matcher(errContent.toString(DEFAULT_ENCODING));
assertTrue(m.find());
}

private void testReplicaIncludedInOutput(boolean includeIndex)
throws IOException {
Mockito.when(scmClient.getContainerReplicas(anyLong()))
.thenReturn(getReplicas(includeIndex));
Mockito.when(scmClient.getPipeline(any()))
.thenThrow(new PipelineNotFoundException("Pipeline not found."));
cmd = new InfoSubcommand();
CommandLine c = new CommandLine(cmd);
c.parseArgs("1");
Expand Down Expand Up @@ -139,8 +251,6 @@ private void testReplicaIncludedInOutput(boolean includeIndex)
public void testReplicasNotOutputIfError() throws IOException {
Mockito.when(scmClient.getContainerReplicas(anyLong()))
.thenThrow(new IOException("Error getting Replicas"));
Mockito.when(scmClient.getPipeline(any()))
.thenThrow(new PipelineNotFoundException("Pipeline not found."));
cmd = new InfoSubcommand();
CommandLine c = new CommandLine(cmd);
c.parseArgs("1");
Expand All @@ -153,13 +263,10 @@ public void testReplicasNotOutputIfError() throws IOException {
.collect(Collectors.toList());
Assertions.assertEquals(0, replica.size());

// Ensure we have an error logged:
List<LoggingEvent> error = logs.stream()
.filter(m -> m.getLevel() == Level.ERROR)
.collect(Collectors.toList());
Assertions.assertEquals(1, error.size());
Assertions.assertTrue(error.get(0).getRenderedMessage()
.matches("(?s)^Unable to retrieve the replica details.*"));
Pattern p = Pattern.compile(
"^Unable to retrieve the replica details.*", Pattern.MULTILINE);
Matcher m = p.matcher(errContent.toString(DEFAULT_ENCODING));
assertTrue(m.find());
}

@Test
Expand All @@ -172,12 +279,9 @@ public void testReplicasNotOutputIfErrorWithJson() throws IOException {
cmd.execute(scmClient);

List<LoggingEvent> logs = appender.getLog();
Assertions.assertEquals(2, logs.size());
String error = logs.get(0).getRenderedMessage();
String json = logs.get(1).getRenderedMessage();
Assertions.assertEquals(1, logs.size());
String json = logs.get(0).getRenderedMessage();

Assertions.assertTrue(error
.matches("(?s)^Unable to retrieve the replica details.*"));
Assertions.assertFalse(json.matches("(?s).*replicas.*"));
}

Expand Down Expand Up @@ -252,7 +356,7 @@ private List<ContainerReplicaInfo> getReplicas(boolean includeIndex) {
return replicas;
}

private ContainerWithPipeline getContainerWithPipeline() {
private ContainerWithPipeline getContainerWithPipeline(long containerID) {
Pipeline pipeline = new Pipeline.Builder()
.setState(Pipeline.PipelineState.CLOSED)
.setReplicationConfig(RatisReplicationConfig.getInstance(THREE))
Expand All @@ -261,6 +365,7 @@ private ContainerWithPipeline getContainerWithPipeline() {
.build();

ContainerInfo container = new ContainerInfo.Builder()
.setContainerID(containerID)
.setSequenceId(1)
.setPipelineID(pipeline.getId())
.setUsedBytes(1234)
Expand Down