From ec02348d3aef6e249f96bfd060afcbbc7b7804bb Mon Sep 17 00:00:00 2001 From: Ritesh H Shukla Date: Mon, 10 Mar 2025 20:21:55 -0700 Subject: [PATCH 1/8] HDDS-12553. Support ozone admin container list to print list in Json array Change-Id: Ic978e701f62e926106bc8b0febce5400219ccd78 --- .../scm/cli/container/ListSubcommand.java | 117 +++++++++++++++--- .../main/smoketest/admincli/container.robot | 27 ++++ 2 files changed, 125 insertions(+), 19 deletions(-) diff --git a/hadoop-hdds/tools/src/main/java/org/apache/hadoop/hdds/scm/cli/container/ListSubcommand.java b/hadoop-hdds/tools/src/main/java/org/apache/hadoop/hdds/scm/cli/container/ListSubcommand.java index 2997e0dd7fa5..a1091818193d 100644 --- a/hadoop-hdds/tools/src/main/java/org/apache/hadoop/hdds/scm/cli/container/ListSubcommand.java +++ b/hadoop-hdds/tools/src/main/java/org/apache/hadoop/hdds/scm/cli/container/ListSubcommand.java @@ -76,6 +76,11 @@ public class ListSubcommand extends ScmSubcommand { description = "Container replication (ONE, THREE for Ratis, " + "rs-6-3-1024k for EC)") private String replication; + + @Option(names = {"--json"}, + description = "Output the entire list in JSON array format", + defaultValue = "false") + private boolean jsonFormat; private static final ObjectWriter WRITER; @@ -90,11 +95,35 @@ public class ListSubcommand extends ScmSubcommand { } - private void outputContainerInfo(ContainerInfo containerInfo) - throws IOException { - // Print container report info. + private void outputContainerInfo(ContainerInfo containerInfo) throws IOException { + // Original behavior - just print the container JSON System.out.println(WRITER.writeValueAsString(containerInfo)); } + + private void outputContainerInfoAsJsonMember(ContainerInfo containerInfo, boolean isFirst, + boolean isLast) throws IOException { + // JSON array format with proper brackets and commas + if (isFirst) { + // Start of array + System.out.print("["); + } + + // Print the container JSON + System.out.print(WRITER.writeValueAsString(containerInfo)); + + if (!isLast) { + // Add comma between elements + System.out.print(","); + } + + if (isLast) { + // End of array + System.out.println("]"); + } else { + // Add newline for readability + System.out.println(); + } + } @Override public void execute(ScmClient scmClient) throws IOException { @@ -124,8 +153,20 @@ public void execute(ScmClient scmClient) throws IOException { count = maxCountAllowed; } containerListAndTotalCount = scmClient.listContainer(startId, count, state, type, repConfig); - for (ContainerInfo container : containerListAndTotalCount.getContainerInfoList()) { - outputContainerInfo(container); + + int totalSize = containerListAndTotalCount.getContainerInfoList().size(); + + if (jsonFormat) { + // JSON array format + for (int i = 0; i < totalSize; i++) { + ContainerInfo container = containerListAndTotalCount.getContainerInfoList().get(i); + outputContainerInfoAsJsonMember(container, i == 0, i == totalSize - 1); + } + } else { + // Original format - one JSON object per line + for (ContainerInfo container : containerListAndTotalCount.getContainerInfoList()) { + outputContainerInfo(container); + } } if (containerListAndTotalCount.getTotalCount() > count) { @@ -138,20 +179,58 @@ public void execute(ScmClient scmClient) throws IOException { int batchSize = (count > 0) ? count : maxCountAllowed; long currentStartId = startId; int fetchedCount; - - do { - // Fetch containers in batches of 'batchSize' - containerListAndTotalCount = scmClient.listContainer(currentStartId, batchSize, state, type, repConfig); - fetchedCount = containerListAndTotalCount.getContainerInfoList().size(); - - for (ContainerInfo container : containerListAndTotalCount.getContainerInfoList()) { - outputContainerInfo(container); - } - - if (fetchedCount > 0) { - currentStartId = containerListAndTotalCount.getContainerInfoList().get(fetchedCount - 1).getContainerID() + 1; - } - } while (fetchedCount > 0); + + if (jsonFormat) { + // JSON array format for all containers + boolean isFirstContainer = true; + + // Start JSON array + System.out.print("["); + + do { + // Fetch containers in batches of 'batchSize' + containerListAndTotalCount = scmClient.listContainer(currentStartId, batchSize, state, type, repConfig); + fetchedCount = containerListAndTotalCount.getContainerInfoList().size(); + + for (int i = 0; i < fetchedCount; i++) { + ContainerInfo container = containerListAndTotalCount.getContainerInfoList().get(i); + + // Only the first container overall doesn't need a preceding comma + if (!isFirstContainer) { + System.out.print(","); + System.out.println(); + } + + // Print the container JSON + System.out.print(WRITER.writeValueAsString(container)); + isFirstContainer = false; + } + + if (fetchedCount > 0) { + currentStartId = + containerListAndTotalCount.getContainerInfoList().get(fetchedCount - 1).getContainerID() + 1; + } + } while (fetchedCount > 0); + + // Close the JSON array + System.out.println("]"); + } else { + // Original format - one JSON object per line + do { + // Fetch containers in batches of 'batchSize' + containerListAndTotalCount = scmClient.listContainer(currentStartId, batchSize, state, type, repConfig); + fetchedCount = containerListAndTotalCount.getContainerInfoList().size(); + + for (ContainerInfo container : containerListAndTotalCount.getContainerInfoList()) { + outputContainerInfo(container); + } + + if (fetchedCount > 0) { + currentStartId = + containerListAndTotalCount.getContainerInfoList().get(fetchedCount - 1).getContainerID() + 1; + } + } while (fetchedCount > 0); + } } } } diff --git a/hadoop-ozone/dist/src/main/smoketest/admincli/container.robot b/hadoop-ozone/dist/src/main/smoketest/admincli/container.robot index 564fd1f5d699..322ccf68aaf0 100644 --- a/hadoop-ozone/dist/src/main/smoketest/admincli/container.robot +++ b/hadoop-ozone/dist/src/main/smoketest/admincli/container.robot @@ -96,6 +96,33 @@ List all containers from a particular container ID ${output} = Execute ozone admin container list --all --start 1 Should contain ${output} OPEN +List containers in JSON array format + ${output} = Execute ozone admin container list --json | jq -r '.' + Should Start With ${output} [ + Should Contain ${output} containerID + Should End With ${output} ] + +List all containers in JSON array format + ${output} = Execute ozone admin container list --all --json | jq -r '.' + Should Start With ${output} [ + Should Contain ${output} containerID + Should End With ${output} ] + +List containers with state in JSON array format + ${output} = Execute ozone admin container list --state=OPEN --json | jq -r '.' + Should Start With ${output} [ + Should Contain ${output} OPEN + Should Not Contain ${output} CLOSED + Should End With ${output} ] + +List containers with count in JSON array format + ${output} = Execute ozone admin container list --count 5 --json | jq -r '.' + Should Start With ${output} [ + Should Contain ${output} containerID + Should End With ${output} ] + ${count} = Execute echo '${output}' | jq -r 'length' + Should Be True ${count} <= 5 + Close container ${container} = Execute ozone admin container list --state OPEN | jq -r 'select(.replicationConfig.replicationFactor == "THREE") | .containerID' | head -1 Execute ozone admin container close "${container}" From f189a0ada4affd7a26ca5a879c7294ef9438ef51 Mon Sep 17 00:00:00 2001 From: Ritesh H Shukla Date: Tue, 11 Mar 2025 17:37:36 -0700 Subject: [PATCH 2/8] HDDS-12553. Use SequenceWriter for JSON array output in container list - Improved JSON array output using Jackson's SequenceWriter - Avoided creating an in-memory copy of all containers - Streaming containers directly to SequenceWriter as they're fetched - Improved robot tests to better handle JSON output per review feedback - Fixed test format to separate command execution and jq processing - Removed unused helper method Change-Id: I8f4d2a63d67fe1f6018daaa9e63fc29c1c076a2f --- .../scm/cli/container/ListSubcommand.java | 85 ++++++------------- .../main/smoketest/admincli/container.robot | 17 ++-- 2 files changed, 35 insertions(+), 67 deletions(-) diff --git a/hadoop-hdds/tools/src/main/java/org/apache/hadoop/hdds/scm/cli/container/ListSubcommand.java b/hadoop-hdds/tools/src/main/java/org/apache/hadoop/hdds/scm/cli/container/ListSubcommand.java index a1091818193d..888a01f21153 100644 --- a/hadoop-hdds/tools/src/main/java/org/apache/hadoop/hdds/scm/cli/container/ListSubcommand.java +++ b/hadoop-hdds/tools/src/main/java/org/apache/hadoop/hdds/scm/cli/container/ListSubcommand.java @@ -21,6 +21,7 @@ import com.fasterxml.jackson.annotation.PropertyAccessor; import com.fasterxml.jackson.databind.ObjectMapper; import com.fasterxml.jackson.databind.ObjectWriter; +import com.fasterxml.jackson.databind.SequenceWriter; import com.fasterxml.jackson.databind.SerializationFeature; import com.fasterxml.jackson.datatype.jsr310.JavaTimeModule; import com.google.common.base.Strings; @@ -99,31 +100,6 @@ private void outputContainerInfo(ContainerInfo containerInfo) throws IOException // Original behavior - just print the container JSON System.out.println(WRITER.writeValueAsString(containerInfo)); } - - private void outputContainerInfoAsJsonMember(ContainerInfo containerInfo, boolean isFirst, - boolean isLast) throws IOException { - // JSON array format with proper brackets and commas - if (isFirst) { - // Start of array - System.out.print("["); - } - - // Print the container JSON - System.out.print(WRITER.writeValueAsString(containerInfo)); - - if (!isLast) { - // Add comma between elements - System.out.print(","); - } - - if (isLast) { - // End of array - System.out.println("]"); - } else { - // Add newline for readability - System.out.println(); - } - } @Override public void execute(ScmClient scmClient) throws IOException { @@ -154,14 +130,13 @@ public void execute(ScmClient scmClient) throws IOException { } containerListAndTotalCount = scmClient.listContainer(startId, count, state, type, repConfig); - int totalSize = containerListAndTotalCount.getContainerInfoList().size(); - if (jsonFormat) { - // JSON array format - for (int i = 0; i < totalSize; i++) { - ContainerInfo container = containerListAndTotalCount.getContainerInfoList().get(i); - outputContainerInfoAsJsonMember(container, i == 0, i == totalSize - 1); + // JSON array format using SequenceWriter directly + try (SequenceWriter sequenceWriter = WRITER.writeValues(System.out)) { + sequenceWriter.init(true); // Initialize as a JSON array + sequenceWriter.writeAll(containerListAndTotalCount.getContainerInfoList()); } + System.out.println(); // Add final newline } else { // Original format - one JSON object per line for (ContainerInfo container : containerListAndTotalCount.getContainerInfoList()) { @@ -181,39 +156,27 @@ public void execute(ScmClient scmClient) throws IOException { int fetchedCount; if (jsonFormat) { - // JSON array format for all containers - boolean isFirstContainer = true; - - // Start JSON array - System.out.print("["); - - do { - // Fetch containers in batches of 'batchSize' - containerListAndTotalCount = scmClient.listContainer(currentStartId, batchSize, state, type, repConfig); - fetchedCount = containerListAndTotalCount.getContainerInfoList().size(); - - for (int i = 0; i < fetchedCount; i++) { - ContainerInfo container = containerListAndTotalCount.getContainerInfoList().get(i); + // For JSON array format, use SequenceWriter to stream containers as we fetch them + try (SequenceWriter sequenceWriter = WRITER.writeValues(System.out)) { + sequenceWriter.init(true); // Initialize as a JSON array + + do { + // Fetch containers in batches of 'batchSize' + containerListAndTotalCount = scmClient.listContainer(currentStartId, batchSize, state, type, repConfig); + fetchedCount = containerListAndTotalCount.getContainerInfoList().size(); - // Only the first container overall doesn't need a preceding comma - if (!isFirstContainer) { - System.out.print(","); - System.out.println(); + // Write containers directly to the SequenceWriter + for (ContainerInfo container : containerListAndTotalCount.getContainerInfoList()) { + sequenceWriter.write(container); } - // Print the container JSON - System.out.print(WRITER.writeValueAsString(container)); - isFirstContainer = false; - } - - if (fetchedCount > 0) { - currentStartId = - containerListAndTotalCount.getContainerInfoList().get(fetchedCount - 1).getContainerID() + 1; - } - } while (fetchedCount > 0); - - // Close the JSON array - System.out.println("]"); + if (fetchedCount > 0) { + currentStartId = + containerListAndTotalCount.getContainerInfoList().get(fetchedCount - 1).getContainerID() + 1; + } + } while (fetchedCount > 0); + } + System.out.println(); // Add final newline } else { // Original format - one JSON object per line do { diff --git a/hadoop-ozone/dist/src/main/smoketest/admincli/container.robot b/hadoop-ozone/dist/src/main/smoketest/admincli/container.robot index 322ccf68aaf0..cf9dfb540e40 100644 --- a/hadoop-ozone/dist/src/main/smoketest/admincli/container.robot +++ b/hadoop-ozone/dist/src/main/smoketest/admincli/container.robot @@ -97,26 +97,31 @@ List all containers from a particular container ID Should contain ${output} OPEN List containers in JSON array format - ${output} = Execute ozone admin container list --json | jq -r '.' + ${output} = Execute ozone admin container list --json Should Start With ${output} [ Should Contain ${output} containerID Should End With ${output} ] + ${containerIDs} = Execute echo '${output}' | jq -r '.[].containerID' + Should Not Be Empty ${containerIDs} List all containers in JSON array format - ${output} = Execute ozone admin container list --all --json | jq -r '.' + ${output} = Execute ozone admin container list --all --json Should Start With ${output} [ Should Contain ${output} containerID Should End With ${output} ] + ${containers} = Execute echo '${output}' | jq -r '.[]' + Should Not Be Empty ${containers} List containers with state in JSON array format - ${output} = Execute ozone admin container list --state=OPEN --json | jq -r '.' + ${output} = Execute ozone admin container list --state=OPEN --json Should Start With ${output} [ - Should Contain ${output} OPEN - Should Not Contain ${output} CLOSED Should End With ${output} ] + ${states} = Execute echo '${output}' | jq -r '.[].state' + Should Contain ${states} OPEN + Should Not Contain ${states} CLOSED List containers with count in JSON array format - ${output} = Execute ozone admin container list --count 5 --json | jq -r '.' + ${output} = Execute ozone admin container list --count 5 --json Should Start With ${output} [ Should Contain ${output} containerID Should End With ${output} ] From 74b2e6b454e96d3fe1c3a755bcc7d07aa10f55a5 Mon Sep 17 00:00:00 2001 From: Ritesh H Shukla Date: Thu, 13 Mar 2025 21:36:10 -0700 Subject: [PATCH 3/8] HDDS-12555. Change container list default output format to JSON array - Remove --json option and make JSON array the default output format - Reduce code duplication between partial and full listing cases - Update robot tests to handle the new JSON array format in: - admincli/container.robot - balancer/testBalancer.robot - freon/echoRPCLoad.robot Change-Id: Ie8be8e845a23136aeef2e09f73d4a2ab23028ddf --- .../scm/cli/container/ListSubcommand.java | 110 ++++++------------ .../main/smoketest/admincli/container.robot | 38 +++--- .../smoketest/balancer/testBalancer.robot | 4 +- .../main/smoketest/freon/echoRPCLoad.robot | 2 +- 4 files changed, 63 insertions(+), 91 deletions(-) diff --git a/hadoop-hdds/tools/src/main/java/org/apache/hadoop/hdds/scm/cli/container/ListSubcommand.java b/hadoop-hdds/tools/src/main/java/org/apache/hadoop/hdds/scm/cli/container/ListSubcommand.java index 888a01f21153..616856fbafb8 100644 --- a/hadoop-hdds/tools/src/main/java/org/apache/hadoop/hdds/scm/cli/container/ListSubcommand.java +++ b/hadoop-hdds/tools/src/main/java/org/apache/hadoop/hdds/scm/cli/container/ListSubcommand.java @@ -77,11 +77,6 @@ public class ListSubcommand extends ScmSubcommand { description = "Container replication (ONE, THREE for Ratis, " + "rs-6-3-1024k for EC)") private String replication; - - @Option(names = {"--json"}, - description = "Output the entire list in JSON array format", - defaultValue = "false") - private boolean jsonFormat; private static final ObjectWriter WRITER; @@ -95,12 +90,6 @@ public class ListSubcommand extends ScmSubcommand { WRITER = mapper.writerWithDefaultPrettyPrinter(); } - - private void outputContainerInfo(ContainerInfo containerInfo) throws IOException { - // Original behavior - just print the container JSON - System.out.println(WRITER.writeValueAsString(containerInfo)); - } - @Override public void execute(ScmClient scmClient) throws IOException { if (!Strings.isNullOrEmpty(replication) && type == null) { @@ -119,81 +108,56 @@ public void execute(ScmClient scmClient) throws IOException { .getInt(ScmConfigKeys.OZONE_SCM_CONTAINER_LIST_MAX_COUNT, ScmConfigKeys.OZONE_SCM_CONTAINER_LIST_MAX_COUNT_DEFAULT); - ContainerListResult containerListAndTotalCount; - - if (!all) { - if (count > maxCountAllowed) { - System.err.printf("Attempting to list the first %d records of containers." + - " However it exceeds the cluster's current limit of %d. The results will be capped at the" + - " maximum allowed count.%n", count, ScmConfigKeys.OZONE_SCM_CONTAINER_LIST_MAX_COUNT_DEFAULT); - count = maxCountAllowed; - } - containerListAndTotalCount = scmClient.listContainer(startId, count, state, type, repConfig); + // Use SequenceWriter to output JSON array format for all cases + try (SequenceWriter sequenceWriter = WRITER.writeValues(System.out)) { + sequenceWriter.init(true); // Initialize as a JSON array - if (jsonFormat) { - // JSON array format using SequenceWriter directly - try (SequenceWriter sequenceWriter = WRITER.writeValues(System.out)) { - sequenceWriter.init(true); // Initialize as a JSON array - sequenceWriter.writeAll(containerListAndTotalCount.getContainerInfoList()); + if (!all) { + // Regular listing with count limit + if (count > maxCountAllowed) { + System.err.printf("Attempting to list the first %d records of containers." + + " However it exceeds the cluster's current limit of %d. The results will be capped at the" + + " maximum allowed count.%n", count, maxCountAllowed); + count = maxCountAllowed; } - System.out.println(); // Add final newline - } else { - // Original format - one JSON object per line - for (ContainerInfo container : containerListAndTotalCount.getContainerInfoList()) { - outputContainerInfo(container); + + ContainerListResult containerListResult = + scmClient.listContainer(startId, count, state, type, repConfig); + + // Write all containers in the result to the JSON array + for (ContainerInfo container : containerListResult.getContainerInfoList()) { + sequenceWriter.write(container); } - } - - if (containerListAndTotalCount.getTotalCount() > count) { - System.err.printf("Displaying %d out of %d containers. " + - "Container list has more containers.%n", - count, containerListAndTotalCount.getTotalCount()); - } - } else { - // Batch size is either count passed through cli or maxCountAllowed - int batchSize = (count > 0) ? count : maxCountAllowed; - long currentStartId = startId; - int fetchedCount; - - if (jsonFormat) { - // For JSON array format, use SequenceWriter to stream containers as we fetch them - try (SequenceWriter sequenceWriter = WRITER.writeValues(System.out)) { - sequenceWriter.init(true); // Initialize as a JSON array - - do { - // Fetch containers in batches of 'batchSize' - containerListAndTotalCount = scmClient.listContainer(currentStartId, batchSize, state, type, repConfig); - fetchedCount = containerListAndTotalCount.getContainerInfoList().size(); - - // Write containers directly to the SequenceWriter - for (ContainerInfo container : containerListAndTotalCount.getContainerInfoList()) { - sequenceWriter.write(container); - } - - if (fetchedCount > 0) { - currentStartId = - containerListAndTotalCount.getContainerInfoList().get(fetchedCount - 1).getContainerID() + 1; - } - } while (fetchedCount > 0); + + if (containerListResult.getTotalCount() > count) { + System.err.printf("Displaying %d out of %d containers. " + + "Container list has more containers.%n", + count, containerListResult.getTotalCount()); } - System.out.println(); // Add final newline } else { - // Original format - one JSON object per line + // List all containers by fetching in batches + int batchSize = (count > 0) ? count : maxCountAllowed; + long currentStartId = startId; + int fetchedCount; + do { // Fetch containers in batches of 'batchSize' - containerListAndTotalCount = scmClient.listContainer(currentStartId, batchSize, state, type, repConfig); - fetchedCount = containerListAndTotalCount.getContainerInfoList().size(); - - for (ContainerInfo container : containerListAndTotalCount.getContainerInfoList()) { - outputContainerInfo(container); + ContainerListResult containerListResult = + scmClient.listContainer(currentStartId, batchSize, state, type, repConfig); + fetchedCount = containerListResult.getContainerInfoList().size(); + + // Write containers directly to the SequenceWriter + for (ContainerInfo container : containerListResult.getContainerInfoList()) { + sequenceWriter.write(container); } - + if (fetchedCount > 0) { currentStartId = - containerListAndTotalCount.getContainerInfoList().get(fetchedCount - 1).getContainerID() + 1; + containerListResult.getContainerInfoList().get(fetchedCount - 1).getContainerID() + 1; } } while (fetchedCount > 0); } } + System.out.println(); // Add final newline } } diff --git a/hadoop-ozone/dist/src/main/smoketest/admincli/container.robot b/hadoop-ozone/dist/src/main/smoketest/admincli/container.robot index cf9dfb540e40..394403d9df76 100644 --- a/hadoop-ozone/dist/src/main/smoketest/admincli/container.robot +++ b/hadoop-ozone/dist/src/main/smoketest/admincli/container.robot @@ -44,22 +44,32 @@ Create container List containers ${output} = Execute ozone admin container list Should contain ${output} OPEN + Should Start With ${output} [ + Should End With ${output} ] List containers with explicit host ${output} = Execute ozone admin container list --scm ${SCM} Should contain ${output} OPEN + Should Start With ${output} [ + Should End With ${output} ] List containers with container state ${output} = Execute ozone admin container list --state=CLOSED Should Not contain ${output} OPEN + Should Start With ${output} [ + Should End With ${output} ] List containers with replication factor ONE ${output} = Execute ozone admin container list -t RATIS -r ONE Should Not contain ${output} THREE + Should Start With ${output} [ + Should End With ${output} ] List containers with replication factor THREE ${output} = Execute ozone admin container list -t RATIS -r THREE Should Not contain ${output} ONE + Should Start With ${output} [ + Should End With ${output} ] Container info ${output} = Execute ozone admin container info "${CONTAINER}" @@ -87,41 +97,39 @@ Report containers as JSON List all containers ${output} = Execute ozone admin container list --all Should contain ${output} OPEN + Should Start With ${output} [ + Should End With ${output} ] List all containers according to count (batchSize) ${output} = Execute ozone admin container list --all --count 10 Should contain ${output} OPEN + Should Start With ${output} [ + Should End With ${output} ] List all containers from a particular container ID ${output} = Execute ozone admin container list --all --start 1 Should contain ${output} OPEN - -List containers in JSON array format - ${output} = Execute ozone admin container list --json Should Start With ${output} [ - Should Contain ${output} containerID Should End With ${output} ] - ${containerIDs} = Execute echo '${output}' | jq -r '.[].containerID' - Should Not Be Empty ${containerIDs} -List all containers in JSON array format - ${output} = Execute ozone admin container list --all --json +Check JSON array parsing + ${output} = Execute ozone admin container list Should Start With ${output} [ Should Contain ${output} containerID Should End With ${output} ] - ${containers} = Execute echo '${output}' | jq -r '.[]' - Should Not Be Empty ${containers} + ${containerIDs} = Execute echo '${output}' | jq -r '.[].containerID' + Should Not Be Empty ${containerIDs} -List containers with state in JSON array format - ${output} = Execute ozone admin container list --state=OPEN --json +Check state filtering with JSON array format + ${output} = Execute ozone admin container list --state=OPEN Should Start With ${output} [ Should End With ${output} ] ${states} = Execute echo '${output}' | jq -r '.[].state' Should Contain ${states} OPEN Should Not Contain ${states} CLOSED -List containers with count in JSON array format - ${output} = Execute ozone admin container list --count 5 --json +Check count limit with JSON array format + ${output} = Execute ozone admin container list --count 5 Should Start With ${output} [ Should Contain ${output} containerID Should End With ${output} ] @@ -129,7 +137,7 @@ List containers with count in JSON array format Should Be True ${count} <= 5 Close container - ${container} = Execute ozone admin container list --state OPEN | jq -r 'select(.replicationConfig.replicationFactor == "THREE") | .containerID' | head -1 + ${container} = Execute ozone admin container list --state OPEN | jq -r '.[] | select(.replicationConfig.replicationFactor == "THREE") | .containerID' | head -1 Execute ozone admin container close "${container}" ${output} = Execute ozone admin container info "${container}" Should contain ${output} CLOS diff --git a/hadoop-ozone/dist/src/main/smoketest/balancer/testBalancer.robot b/hadoop-ozone/dist/src/main/smoketest/balancer/testBalancer.robot index 641bc1462bbc..d4feebc5feef 100644 --- a/hadoop-ozone/dist/src/main/smoketest/balancer/testBalancer.robot +++ b/hadoop-ozone/dist/src/main/smoketest/balancer/testBalancer.robot @@ -134,7 +134,7 @@ Get Uuid Close All Containers FOR ${INDEX} IN RANGE 15 - ${container} = Execute ozone admin container list --state OPEN | jq -r 'select(.replicationConfig.data == 3) | .containerID' | head -1 + ${container} = Execute ozone admin container list --state OPEN | jq -r '.[] | select(.replicationConfig.data == 3) | .containerID' | head -1 EXIT FOR LOOP IF "${container}" == "${EMPTY}" ${message} = Execute And Ignore Error ozone admin container close "${container}" Run Keyword If '${message}' != '${EMPTY}' Should Contain ${message} is in closing state @@ -145,7 +145,7 @@ Close All Containers All container is closed ${output} = Execute ozone admin container list --state OPEN - Should Be Empty ${output} + Should Be Equal ${output} [] Get Datanode Ozone Used Bytes Info [arguments] ${uuid} diff --git a/hadoop-ozone/dist/src/main/smoketest/freon/echoRPCLoad.robot b/hadoop-ozone/dist/src/main/smoketest/freon/echoRPCLoad.robot index c6ea4e63468e..abcd9417b30b 100644 --- a/hadoop-ozone/dist/src/main/smoketest/freon/echoRPCLoad.robot +++ b/hadoop-ozone/dist/src/main/smoketest/freon/echoRPCLoad.robot @@ -25,7 +25,7 @@ ${n} 1 *** Test Cases *** Get Container ID ${result} = Execute ozone admin container create - ${containerID} = Execute ozone admin container list --count 1 --state=OPEN | grep -o '"containerID" *: *[^,}]*' | awk -F'[:,]' '{print $2}' | tr -d '" ' + ${containerID} = Execute ozone admin container list --count 1 --state=OPEN | jq -r '.[0].containerID' Set Suite Variable ${containerID} [Read] Ozone DataNode Echo RPC Load Generator with request payload and response payload From bbbe43c978d5fdbbc713e87803e2daa0af0c4466 Mon Sep 17 00:00:00 2001 From: Ritesh H Shukla Date: Fri, 14 Mar 2025 00:07:36 -0700 Subject: [PATCH 4/8] HDDS-12555. Change container list default output format to JSON array - fix robot test failure Change-Id: Iae0e74ffa1dd7407e36bf2592b48055768d7de85 --- .../dist/src/main/smoketest/balancer/testBalancer.robot | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/hadoop-ozone/dist/src/main/smoketest/balancer/testBalancer.robot b/hadoop-ozone/dist/src/main/smoketest/balancer/testBalancer.robot index d4feebc5feef..21490e84d646 100644 --- a/hadoop-ozone/dist/src/main/smoketest/balancer/testBalancer.robot +++ b/hadoop-ozone/dist/src/main/smoketest/balancer/testBalancer.robot @@ -145,7 +145,7 @@ Close All Containers All container is closed ${output} = Execute ozone admin container list --state OPEN - Should Be Equal ${output} [] + Should Be Equal ${output} [ ] Get Datanode Ozone Used Bytes Info [arguments] ${uuid} @@ -186,4 +186,4 @@ Verify Container Balancer for RATIS/EC containers #We need to ensure that after balancing, the amount of data recorded on each datanode falls within the following ranges: #{SIZE}*3 < used < {SIZE}*3.5 for RATIS containers, and {SIZE}*0.7 < used < {SIZE}*1.5 for EC containers. Should Be True ${datanodeOzoneUsedBytesInfoAfterContainerBalancing} < ${SIZE} * ${UPPER_LIMIT} - Should Be True ${datanodeOzoneUsedBytesInfoAfterContainerBalancing} > ${SIZE} * ${LOWER_LIMIT} \ No newline at end of file + Should Be True ${datanodeOzoneUsedBytesInfoAfterContainerBalancing} > ${SIZE} * ${LOWER_LIMIT} From f93eb1e803a398b5d4b4a10e95f20adc73575934 Mon Sep 17 00:00:00 2001 From: Ritesh H Shukla Date: Wed, 19 Mar 2025 23:36:22 -0700 Subject: [PATCH 5/8] Deal with sequence writer closing System.out and not printing the final new line Change-Id: I71faf7cbeebb22b0695c79eaac5009af65e6346d --- .../scm/cli/container/ListSubcommand.java | 154 ++++++++++++------ .../hadoop/ozone/shell/TestOzoneShellHA.java | 2 +- 2 files changed, 106 insertions(+), 50 deletions(-) diff --git a/hadoop-hdds/tools/src/main/java/org/apache/hadoop/hdds/scm/cli/container/ListSubcommand.java b/hadoop-hdds/tools/src/main/java/org/apache/hadoop/hdds/scm/cli/container/ListSubcommand.java index 616856fbafb8..d9dcd5b0956b 100644 --- a/hadoop-hdds/tools/src/main/java/org/apache/hadoop/hdds/scm/cli/container/ListSubcommand.java +++ b/hadoop-hdds/tools/src/main/java/org/apache/hadoop/hdds/scm/cli/container/ListSubcommand.java @@ -26,6 +26,9 @@ import com.fasterxml.jackson.datatype.jsr310.JavaTimeModule; import com.google.common.base.Strings; import java.io.IOException; +import java.io.OutputStream; +import java.io.PrintStream; +import java.util.List; import org.apache.hadoop.hdds.cli.HddsVersionProvider; import org.apache.hadoop.hdds.client.ReplicationConfig; import org.apache.hadoop.hdds.client.ReplicationType; @@ -41,7 +44,13 @@ import picocli.CommandLine.Option; /** - * This is the handler that process container list command. + * The ListSubcommand class represents a command to list containers in a structured way. + * It provides options to control how the list is generated, including specifying + * starting container ID, maximum number of containers to list, and other filtering criteria + * such as container state or replication type. + * + * This command connects to the SCM (Storage Container Manager) client to fetch the + * container details and outputs the result in a JSON format. */ @Command( name = "list", @@ -109,55 +118,102 @@ public void execute(ScmClient scmClient) throws IOException { ScmConfigKeys.OZONE_SCM_CONTAINER_LIST_MAX_COUNT_DEFAULT); // Use SequenceWriter to output JSON array format for all cases - try (SequenceWriter sequenceWriter = WRITER.writeValues(System.out)) { - sequenceWriter.init(true); // Initialize as a JSON array - - if (!all) { - // Regular listing with count limit - if (count > maxCountAllowed) { - System.err.printf("Attempting to list the first %d records of containers." + - " However it exceeds the cluster's current limit of %d. The results will be capped at the" + - " maximum allowed count.%n", count, maxCountAllowed); - count = maxCountAllowed; - } - - ContainerListResult containerListResult = - scmClient.listContainer(startId, count, state, type, repConfig); - - // Write all containers in the result to the JSON array - for (ContainerInfo container : containerListResult.getContainerInfoList()) { - sequenceWriter.write(container); - } - - if (containerListResult.getTotalCount() > count) { - System.err.printf("Displaying %d out of %d containers. " + - "Container list has more containers.%n", - count, containerListResult.getTotalCount()); - } - } else { - // List all containers by fetching in batches - int batchSize = (count > 0) ? count : maxCountAllowed; - long currentStartId = startId; - int fetchedCount; - - do { - // Fetch containers in batches of 'batchSize' - ContainerListResult containerListResult = - scmClient.listContainer(currentStartId, batchSize, state, type, repConfig); - fetchedCount = containerListResult.getContainerInfoList().size(); - - // Write containers directly to the SequenceWriter - for (ContainerInfo container : containerListResult.getContainerInfoList()) { - sequenceWriter.write(container); - } - - if (fetchedCount > 0) { - currentStartId = - containerListResult.getContainerInfoList().get(fetchedCount - 1).getContainerID() + 1; - } - } while (fetchedCount > 0); + SequenceWriter sequenceWriter = WRITER.writeValues(new NonClosingOutputStream(System.out)); + sequenceWriter.init(true); // Initialize as a JSON array + + if (!all) { + // Regular listing with count limit + if (count > maxCountAllowed) { + System.err.printf("Attempting to list the first %d records of containers." + + " However it exceeds the cluster's current limit of %d. The results will be capped at the" + + " maximum allowed count.%n", count, maxCountAllowed); + count = maxCountAllowed; + } + + ContainerListResult containerListResult = + scmClient.listContainer(startId, count, state, type, repConfig); + + writeContainers(sequenceWriter, containerListResult.getContainerInfoList()); + + + if (containerListResult.getTotalCount() > count) { + System.err.printf("Displaying %d out of %d containers. " + + "Container list has more containers.%n", + count, containerListResult.getTotalCount()); + } + } else { + // List all containers by fetching in batches + int batchSize = (count > 0) ? count : maxCountAllowed; + listAllContainers(scmClient, sequenceWriter, batchSize, repConfig); + } + } + + private void writeContainers(SequenceWriter writer, List containers) + throws IOException { + for (ContainerInfo container : containers) { + writer.write(container); + } + } + + private void closeStream(SequenceWriter writer) throws IOException { + writer.flush(); + writer.close(); + // Add the final newline + System.out.println(); + } + + private void listAllContainers(ScmClient scmClient, SequenceWriter writer, + int batchSize, ReplicationConfig repConfig) + throws IOException { + long currentStartId = startId; + int fetchedCount; + + do { + ContainerListResult result = + scmClient.listContainer(currentStartId, batchSize, state, type, repConfig); + fetchedCount = result.getContainerInfoList().size(); + + writeContainers(writer, result.getContainerInfoList()); + + if (fetchedCount > 0) { + currentStartId = + result.getContainerInfoList().get(fetchedCount - 1).getContainerID() + 1; } + } while (fetchedCount > 0); + } + + + private static class NonClosingOutputStream extends OutputStream { + + private final OutputStream delegate; + + public NonClosingOutputStream(OutputStream delegate) { + this.delegate = delegate; + } + + @Override + public void write(int b) throws IOException { + delegate.write(b); + } + + @Override + public void write(byte[] b) throws IOException { + delegate.write(b); + } + + @Override + public void write(byte[] b, int off, int len) throws IOException { + delegate.write(b, off, len); + } + + @Override + public void flush() throws IOException { + delegate.flush(); + } + + @Override + public void close() { + // Ignore close to keep the underlying stream open } - System.out.println(); // Add final newline } } diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/shell/TestOzoneShellHA.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/shell/TestOzoneShellHA.java index 19acdbd275fb..87ed487087b2 100644 --- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/shell/TestOzoneShellHA.java +++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/shell/TestOzoneShellHA.java @@ -961,7 +961,7 @@ public void testOzoneAdminCmdListAllContainer() execute(ozoneAdminShell, args1); //results will be capped at the maximum allowed count assertEquals(1, getNumOfContainers()); - + reset(); String[] args2 = new String[] {"container", "list", "-a", "--scm", "localhost:" + cluster.getStorageContainerManager().getClientRpcPort()}; execute(ozoneAdminShell, args2); From e34b38ee857db99a1f6f8c2499231d12be6531c1 Mon Sep 17 00:00:00 2001 From: Ritesh H Shukla Date: Thu, 20 Mar 2025 00:01:50 -0700 Subject: [PATCH 6/8] Address checkstyle and errors Change-Id: I20d5d8e27f55a3c3e97a078de7e4c1c6b1ba9264 --- .../hadoop/hdds/scm/cli/container/ListSubcommand.java | 6 +++--- .../org/apache/hadoop/ozone/shell/TestOzoneShellHA.java | 3 ++- 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/hadoop-hdds/tools/src/main/java/org/apache/hadoop/hdds/scm/cli/container/ListSubcommand.java b/hadoop-hdds/tools/src/main/java/org/apache/hadoop/hdds/scm/cli/container/ListSubcommand.java index d9dcd5b0956b..57053f76e08f 100644 --- a/hadoop-hdds/tools/src/main/java/org/apache/hadoop/hdds/scm/cli/container/ListSubcommand.java +++ b/hadoop-hdds/tools/src/main/java/org/apache/hadoop/hdds/scm/cli/container/ListSubcommand.java @@ -27,7 +27,6 @@ import com.google.common.base.Strings; import java.io.IOException; import java.io.OutputStream; -import java.io.PrintStream; import java.util.List; import org.apache.hadoop.hdds.cli.HddsVersionProvider; import org.apache.hadoop.hdds.client.ReplicationConfig; @@ -135,7 +134,7 @@ public void execute(ScmClient scmClient) throws IOException { writeContainers(sequenceWriter, containerListResult.getContainerInfoList()); - + closeStream(sequenceWriter); if (containerListResult.getTotalCount() > count) { System.err.printf("Displaying %d out of %d containers. " + "Container list has more containers.%n", @@ -145,6 +144,7 @@ public void execute(ScmClient scmClient) throws IOException { // List all containers by fetching in batches int batchSize = (count > 0) ? count : maxCountAllowed; listAllContainers(scmClient, sequenceWriter, batchSize, repConfig); + closeStream(sequenceWriter); } } @@ -187,7 +187,7 @@ private static class NonClosingOutputStream extends OutputStream { private final OutputStream delegate; - public NonClosingOutputStream(OutputStream delegate) { + NonClosingOutputStream(OutputStream delegate) { this.delegate = delegate; } diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/shell/TestOzoneShellHA.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/shell/TestOzoneShellHA.java index 87ed487087b2..d1aa84c5808a 100644 --- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/shell/TestOzoneShellHA.java +++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/shell/TestOzoneShellHA.java @@ -961,7 +961,8 @@ public void testOzoneAdminCmdListAllContainer() execute(ozoneAdminShell, args1); //results will be capped at the maximum allowed count assertEquals(1, getNumOfContainers()); - reset(); + out.reset(); + err.reset(); String[] args2 = new String[] {"container", "list", "-a", "--scm", "localhost:" + cluster.getStorageContainerManager().getClientRpcPort()}; execute(ozoneAdminShell, args2); From c7d086ba9043dd72c858511362c552c13dfb71f3 Mon Sep 17 00:00:00 2001 From: Ritesh H Shukla Date: Thu, 20 Mar 2025 14:56:53 -0700 Subject: [PATCH 7/8] Fix container count robot test Change-Id: I0b57ac48cec313f7437970abcb08d00bf748e651 --- .../dist/src/main/smoketest/admincli/container.robot | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/hadoop-ozone/dist/src/main/smoketest/admincli/container.robot b/hadoop-ozone/dist/src/main/smoketest/admincli/container.robot index 394403d9df76..e650cd88941a 100644 --- a/hadoop-ozone/dist/src/main/smoketest/admincli/container.robot +++ b/hadoop-ozone/dist/src/main/smoketest/admincli/container.robot @@ -129,12 +129,22 @@ Check state filtering with JSON array format Should Not Contain ${states} CLOSED Check count limit with JSON array format + ${output} = Execute ozone admin container create + Should contain ${output} is created + ${output} = Execute ozone admin container create + Should contain ${output} is created + ${output} = Execute ozone admin container create + Should contain ${output} is created + ${output} = Execute ozone admin container create + Should contain ${output} is created + ${output} = Execute ozone admin container create + Should contain ${output} is created ${output} = Execute ozone admin container list --count 5 Should Start With ${output} [ Should Contain ${output} containerID Should End With ${output} ] ${count} = Execute echo '${output}' | jq -r 'length' - Should Be True ${count} <= 5 + Should Be True ${count} = 5 Close container ${container} = Execute ozone admin container list --state OPEN | jq -r '.[] | select(.replicationConfig.replicationFactor == "THREE") | .containerID' | head -1 From 905852a45cbb0365ffd2e2e33154d52301d1bb23 Mon Sep 17 00:00:00 2001 From: Ritesh H Shukla Date: Thu, 20 Mar 2025 17:54:17 -0700 Subject: [PATCH 8/8] Fix the issues caused by std error output for successful CLI runs and clean up robot tests Change-Id: Ia054d91e9ba83708da1a16fbbd89549b8122b276 --- .../dist/src/main/smoketest/admincli/container.robot | 11 ++++------- 1 file changed, 4 insertions(+), 7 deletions(-) diff --git a/hadoop-ozone/dist/src/main/smoketest/admincli/container.robot b/hadoop-ozone/dist/src/main/smoketest/admincli/container.robot index e650cd88941a..1f3279c6bdca 100644 --- a/hadoop-ozone/dist/src/main/smoketest/admincli/container.robot +++ b/hadoop-ozone/dist/src/main/smoketest/admincli/container.robot @@ -107,7 +107,7 @@ List all containers according to count (batchSize) Should End With ${output} ] List all containers from a particular container ID - ${output} = Execute ozone admin container list --all --start 1 + ${output} = Execute ozone admin container list --all --start 2 Should contain ${output} OPEN Should Start With ${output} [ Should End With ${output} ] @@ -139,15 +139,12 @@ Check count limit with JSON array format Should contain ${output} is created ${output} = Execute ozone admin container create Should contain ${output} is created - ${output} = Execute ozone admin container list --count 5 - Should Start With ${output} [ - Should Contain ${output} containerID - Should End With ${output} ] + ${output} = Execute And Ignore Error ozone admin container list --count 5 2> /dev/null # This logs to error that the list is incomplete ${count} = Execute echo '${output}' | jq -r 'length' - Should Be True ${count} = 5 + Should Be True ${count} == 5 Close container - ${container} = Execute ozone admin container list --state OPEN | jq -r '.[] | select(.replicationConfig.replicationFactor == "THREE") | .containerID' | head -1 + ${container} = Execute ozone admin container list --state OPEN | jq -r '.[] | select(.replicationConfig.replicationFactor == "ONE") | .containerID' | head -1 Execute ozone admin container close "${container}" ${output} = Execute ozone admin container info "${container}" Should contain ${output} CLOS